diff --git a/.bumpversion.toml b/.bumpversion.toml index 6840e4f3138..94519e628f6 100644 --- a/.bumpversion.toml +++ b/.bumpversion.toml @@ -1,5 +1,5 @@ [tool.bumpversion] -current_version = "9.0.0-beta.16" +current_version = "11.0.0-beta.6" parse = "(?P\\d+)\\.(?P\\d+)\\.(?P\\d+)(-(?P(beta|rc))\\.(?P\\d+))?" serialize = [ "{major}.{minor}.{patch}-{prerelease}.{prerelease_num}", @@ -80,6 +80,11 @@ filename = "Cargo.toml" search = 'lance-index = {{ version = "={current_version}"' replace = 'lance-index = {{ version = "={new_version}"' +[[tool.bumpversion.files]] +filename = "Cargo.toml" +search = 'lance-index-core = {{ version = "={current_version}"' +replace = 'lance-index-core = {{ version = "={new_version}"' + [[tool.bumpversion.files]] filename = "Cargo.toml" search = 'lance-io = {{ version = "={current_version}"' diff --git a/.cargo/config.toml b/.cargo/config.toml index c455c4a978d..d8b67ef519c 100644 --- a/.cargo/config.toml +++ b/.cargo/config.toml @@ -22,7 +22,9 @@ lto = "thin" codegen-units = 16 [target.x86_64-unknown-linux-gnu] -rustflags = ["-C", "target-cpu=haswell", "-C", "target-feature=+avx2,+fma,+f16c"] +# Keep the workspace baseline below AVX so binaries can load before runtime +# feature detection selects optimized SIMD kernels. +rustflags = ["-C", "target-cpu=x86-64-v2"] [target.aarch64-apple-darwin] rustflags = ["-C", "target-cpu=apple-m1", "-C", "target-feature=+neon,+fp16,+fhm,+dotprod"] diff --git a/.coderabbit.yaml b/.coderabbit.yaml new file mode 100644 index 00000000000..9779b0ff8e4 --- /dev/null +++ b/.coderabbit.yaml @@ -0,0 +1,17 @@ +# yaml-language-server: $schema=https://coderabbit.ai/integrations/schema.v2.json +language: en-US +early_access: false +reviews: + profile: quiet + high_level_summary: false + poem: false + review_status: true + auto_review: + enabled: true + auto_incremental_review: true + ignore_title_keywords: + - WIP + - Draft + drafts: false + base_branches: + - main diff --git a/.gitattributes b/.gitattributes index f3351d043eb..a447781cffc 100644 --- a/.gitattributes +++ b/.gitattributes @@ -1,3 +1,4 @@ notebooks/** linguist-vendored Cargo.lock linguist-generated -**/Cargo.lock linguist-generated \ No newline at end of file +**/Cargo.lock linguist-generated +rust/lance-file/test_data/**/*.lance binary diff --git a/.github/ISSUE_TEMPLATE/bug_report.yml b/.github/ISSUE_TEMPLATE/bug_report.yml new file mode 100644 index 00000000000..0e7363697b0 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug_report.yml @@ -0,0 +1,79 @@ +name: Bug report +description: Report incorrect, unexpected, or crashing behavior. +labels: ["bug"] +body: + - type: markdown + attributes: + value: | + Thanks for taking the time to file a bug report! Please search + [existing issues](https://github.com/lance-format/lance/issues) first to + avoid filing a duplicate. + - type: textarea + id: description + attributes: + label: Description + description: A clear and concise description of what the bug is. + validations: + required: true + - type: textarea + id: reproduction + attributes: + label: Steps to reproduce + description: >- + A minimal, self-contained code snippet or sequence of steps that + reproduces the problem. The more we can copy-paste and run, the faster + we can fix it. + placeholder: | + 1. Write a dataset with `...` + 2. Scan with filter `...` + 3. Observe `...` + render: python + validations: + required: true + - type: textarea + id: expected + attributes: + label: Expected behavior + description: What did you expect to happen? + validations: + required: false + - type: input + id: lance-version + attributes: + label: Lance version + description: >- + Output of `python -c "import lance; print(lance.__version__)"`, or the + `lance`/`pylance` version from your `Cargo.toml` / `pyproject.toml`. + placeholder: "e.g. 0.40.0" + validations: + required: true + - type: dropdown + id: language + attributes: + label: Language binding + multiple: true + options: + - Python + - Rust + - Java + - Other / not sure + validations: + required: true + - type: input + id: environment + attributes: + label: Environment + description: OS, architecture, and storage backend (local, S3, GCS, Azure, ...). + placeholder: "e.g. Ubuntu 22.04, x86_64, S3" + validations: + required: false + - type: textarea + id: logs + attributes: + label: Logs / traceback + description: >- + Any relevant log output or stack trace. This will be automatically + formatted as code, so no need for backticks. + render: shell + validations: + required: false diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml new file mode 100644 index 00000000000..32389799971 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/config.yml @@ -0,0 +1,7 @@ +blank_issues_enabled: true +contact_links: + - name: Question / usage help + url: https://discord.gg/lance + about: >- + For questions and general discussion, please ask in the Lance Discord + rather than opening an issue. diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.github/ISSUE_TEMPLATE/feature_request.md new file mode 100644 index 00000000000..41d6278fc02 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/feature_request.md @@ -0,0 +1,11 @@ +--- +name: Feature request +about: Suggest a new capability or an improvement to an existing one. +labels: feature +--- + + diff --git a/.github/ISSUE_TEMPLATE/performance_issue.md b/.github/ISSUE_TEMPLATE/performance_issue.md new file mode 100644 index 00000000000..65fcd0be6f4 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/performance_issue.md @@ -0,0 +1,12 @@ +--- +name: Performance issue +about: Report slow operations, high memory use, or a performance regression. +labels: performance +--- + + diff --git a/.github/dependabot.yml b/.github/dependabot.yml index a7b7d11c92b..5c5f1aa6113 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -1,45 +1,86 @@ version: 2 updates: + # Rust — one grouped version-bump PR per directory each week, and security + # advisories that land together in a directory batched into a single PR + # instead of one PR per advisory. - package-ecosystem: "cargo" - directory: "/" + directories: + - "/" + - "/python" + - "/java/lance-jni" versioning-strategy: lockfile-only schedule: interval: "weekly" day: "wednesday" groups: cargo: + applies-to: version-updates + patterns: + - "*" + cargo-security: + applies-to: security-updates patterns: - "*" - - package-ecosystem: "cargo" + # Python (uv) — same weekly-grouped + grouped-security treatment. + - package-ecosystem: "uv" directory: "/python" versioning-strategy: lockfile-only schedule: interval: "weekly" day: "wednesday" groups: - cargo: + uv: + applies-to: version-updates + patterns: + - "*" + uv-security: + applies-to: security-updates patterns: - "*" + # Standalone Rust lockfiles for dev/test fixtures. Dependabot security updates + # scan these regardless of config, so we group them; open-pull-requests-limit: 0 + # disables the (unwanted) weekly version bumps for these crates. - package-ecosystem: "cargo" - directory: "/java/lance-jni" - versioning-strategy: lockfile-only + directories: + - "/memtest" + - "/test_data/fri_straddle_pre_6610/datagen" + open-pull-requests-limit: 0 schedule: interval: "weekly" day: "wednesday" groups: - cargo: + cargo-fixtures-security: + applies-to: security-updates patterns: - "*" - - package-ecosystem: "uv" - directory: "/python" - versioning-strategy: lockfile-only + # Java (Maven). Security updates fire regardless of config; group them and + # skip version bumps (limit 0) — we bump Java deps manually. + - package-ecosystem: "maven" + directory: "/java" + open-pull-requests-limit: 0 schedule: interval: "weekly" day: "wednesday" groups: - uv: + maven-security: + applies-to: security-updates + patterns: + - "*" + + # Python benchmark scripts (pip / requirements.txt). Group security updates + # and skip version bumps (limit 0). + - package-ecosystem: "pip" + directories: + - "/benchmarks/*" + open-pull-requests-limit: 0 + schedule: + interval: "weekly" + day: "wednesday" + groups: + pip-security: + applies-to: security-updates patterns: - "*" diff --git a/.github/labeler-issues.yml b/.github/labeler-issues.yml new file mode 100644 index 00000000000..625b321cd6c --- /dev/null +++ b/.github/labeler-issues.yml @@ -0,0 +1,28 @@ +version: 1 +# Never remove labels a template or a human already applied. +appendOnly: true +# Content-based labels for issues, applied by srvaroa/labeler via +# .github/workflows/issue-labeler.yml. This covers issues that bypass the +# .github/ISSUE_TEMPLATE forms entirely — those filed via `gh issue create`, +# the REST API, or an agent, which never see the web-UI template chooser. +# +# The primary signal is a leading marker in the title (e.g. "bug:", "[feature]", +# "perf -"), which many people type by hand; a modest body keyword match is a +# secondary fallback. Rules are OR'd (one label per matching entry), so an +# unmatched issue gets no label and is left for human / LLM triage rather than +# guessed at. appendOnly means matches stack harmlessly with template labels. +labels: +# --- Primary signal: a leading bug/feature/perf marker in the title --- +- label: bug + title: "(?i)^\\s*[\\[(]?\\s*bug\\b" +- label: feature + title: "(?i)^\\s*[\\[(]?\\s*(feature|feat)\\b" +- label: performance + title: "(?i)^\\s*[\\[(]?\\s*(perf|performance)\\b" +# --- Secondary fallback: keywords anywhere in the body --- +- label: bug + body: "(?i)(panic|segfault|traceback|stack ?trace|crash|corrupt|incorrect result|wrong result)" +- label: performance + body: "(?i)(regression|latency|throughput|\\bOOM\\b|out of memory|memory usage|too slow)" +- label: feature + body: "(?i)(feature request|would be (nice|great|useful)|it would be nice|support for|add .+ support)" diff --git a/.github/workflows/ci-benchmarks.yml b/.github/workflows/ci-benchmarks.yml deleted file mode 100644 index aa6e73ae74f..00000000000 --- a/.github/workflows/ci-benchmarks.yml +++ /dev/null @@ -1,78 +0,0 @@ -name: Run Regression Benchmarks - -on: - workflow_dispatch: - push: - branches: - - main - -permissions: - contents: read - -jobs: - bench_regress: - timeout-minutes: 120 - runs-on: warp-custom-gcp-storage-benchmark - env: - # Need up-to-date compilers for kernels - CC: clang-18 - CXX: clang++-18 - defaults: - run: - shell: bash - working-directory: python - steps: - - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 - with: - fetch-depth: 0 - lfs: true - - name: Authenticate with GCS - uses: google-github-actions/auth@c200f3691d83b41bf9bbd8638997a462592937ed # v2 - with: - credentials_json: "${{ secrets.GCLOUD_BENCH_STORAGE_USER_KEY }}" - - name: Install bencher - uses: bencherdev/bencher@8151077aa7b1bceaac11c4b308265417cae60e2b # v0.5.10 - - name: Set up Python - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5 - with: - python-version: 3.11 # Ray does not support 3.12 yet. - - uses: Swatinem/rust-cache@779680da715d629ac1d338a641029a2f4372abb5 # v2 - with: - workspaces: python - - name: Install dependencies - run: | - sudo apt update - sudo apt install -y protobuf-compiler libssl-dev - - name: Build - run: | - python -m venv venv - source venv/bin/activate - pip install maturin==1.13.3 uv duckdb requests pytest pytest-benchmark datasets - maturin develop --uv --locked --release --features datagen - - name: Build memtest - run: | - source venv/bin/activate - make -C ../memtest build-release - - name: Generate datasets - run: | - source venv/bin/activate - python python/ci_benchmarks/datagen/gen_all.py - - name: Run benchmarks - run: | - source venv/bin/activate - bencher run --project weston-lancedb --token ${{ secrets.LANCE_BENCHER_TOKEN }} --adapter python_pytest \ - --branch main --testbed google-genoa --err --file results.json "python -mpytest --benchmark-json \ - results.json python/ci_benchmarks" - - name: Run IO/memory benchmarks - run: | - source venv/bin/activate - LIB_PATH=$(lance-memtest) - LD_PRELOAD=$LIB_PATH pytest python/ci_benchmarks \ - -k "io_mem_" \ - --benchmark-stats-json io_mem_stats.json - - name: Upload IO/memory stats to bencher - run: | - source venv/bin/activate - bencher run --project weston-lancedb --token ${{ secrets.LANCE_BENCHER_TOKEN }} \ - --adapter json --branch main --testbed google-genoa \ - --err --file io_mem_stats.json diff --git a/.github/workflows/compat-pair.yml b/.github/workflows/compat-pair.yml index ba3332d8d7b..f4290d9ab59 100644 --- a/.github/workflows/compat-pair.yml +++ b/.github/workflows/compat-pair.yml @@ -21,7 +21,7 @@ on: required: false default: "" kinds: - description: "Comma-separated index kinds (INVERTED,BTREE,...) or 'all'." + description: "Comma-separated index kinds (INVERTED,BTREE,IVF_PQ,...) or 'all'." required: false default: "all" max_length: @@ -63,7 +63,7 @@ jobs: KINDS_IN: ${{ inputs.kinds }} run: | if [ -z "$KINDS_IN" ] || [ "$KINDS_IN" = "all" ]; then - echo "value=INVERTED,BTREE,BITMAP,LABEL_LIST,NGRAM,ZONEMAP,BLOOMFILTER" >> "$GITHUB_OUTPUT" + echo "value=INVERTED,BTREE,BITMAP,LABEL_LIST,NGRAM,ZONEMAP,BLOOMFILTER,IVF_PQ" >> "$GITHUB_OUTPUT" else echo "value=$KINDS_IN" >> "$GITHUB_OUTPUT" fi diff --git a/.github/workflows/docs-check.yml b/.github/workflows/docs-check.yml deleted file mode 100644 index 2eda3de66b4..00000000000 --- a/.github/workflows/docs-check.yml +++ /dev/null @@ -1,43 +0,0 @@ -name: Check docs - -on: - push: - branches: - - main - - release/** - pull_request: - branches: - - main - - release/** - paths: - - docs/** - - .github/workflows/docs-check.yml - -permissions: - contents: read - -env: - RUSTFLAGS: "-C debuginfo=0" - # according to: https://matklad.github.io/2021/09/04/fast-rust-builds.html - # CI builds are faster with incremental disabled. - CARGO_INCREMENTAL: "0" - -jobs: - # Single deploy job since we're just deploying - check-docs: - runs-on: ubuntu-24.04 - steps: - - name: Checkout - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 - - name: "Set up Python" - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5 - with: - python-version-file: "docs/pyproject.toml" - - name: Install uv - uses: astral-sh/setup-uv@d0cc045d04ccac9d8b7881df0226f9e82c39688e # v6 - with: - enable-cache: true - - name: Check links - working-directory: docs - run: | - uv run mkdocs-linkcheck src diff --git a/.github/workflows/docs-deploy.yml b/.github/workflows/docs-deploy.yml index 0733e5ce32c..aeaea0bbcba 100644 --- a/.github/workflows/docs-deploy.yml +++ b/.github/workflows/docs-deploy.yml @@ -62,6 +62,11 @@ jobs: with: repository: lance-format/lance-trino path: lance-trino + - name: Checkout lance-context + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 + with: + repository: lance-format/lance-context + path: lance-context - name: Configure Git Credentials run: | git config user.name github-actions[bot] @@ -83,6 +88,7 @@ jobs: LANCE_TRINO_REPO: ${{ github.workspace }}/lance-trino LANCE_DUCKDB_REPO: ${{ github.workspace }}/lance-duckdb LANCE_HUGGINGFACE_REPO: ${{ github.workspace }}/lance-huggingface + LANCE_CONTEXT_REPO: ${{ github.workspace }}/lance-context run: | docs/make-full-website.sh - name: Deploy diff --git a/.github/workflows/docs-link-check.yml b/.github/workflows/docs-link-check.yml new file mode 100644 index 00000000000..a92bf4812d7 --- /dev/null +++ b/.github/workflows/docs-link-check.yml @@ -0,0 +1,166 @@ +name: Check doc links + +# Checking external links is inherently noisy: third-party sites rate-limit +# automated clients, reject non-browser user agents, and go down temporarily. +# Blocking pull requests on that trades a lot of false failures for very little +# signal, so this runs on a schedule and reports findings in a single tracking +# issue instead of failing anyone's build. +on: + schedule: + - cron: "0 7 * * *" + workflow_dispatch: + +# The report lives in one repository-global issue, so runs must not overlap: a +# lookup racing a create produces duplicate issues, and a healthy run closing +# the issue while a failing run only rewrites its body would leave a broken +# report closed. The group is deliberately ref-independent so that a manual +# dispatch serializes against the scheduled run. +concurrency: + group: docs-link-check + cancel-in-progress: false + +permissions: {} + +env: + REPORT_TITLE: "Docs link checker report" + +jobs: + scan: + name: Scan links + runs-on: ubuntu-24.04 + # lychee-action is pinned by SHA, but its wrapper downloads the lychee + # release tarball at run time without verifying a digest, and hands the + # resulting binary a GitHub token. Release assets remain replaceable, so + # that binary is confined to a job whose token can only read public + # content; everything that writes runs in the report job below. + permissions: + contents: read + outputs: + exit_code: ${{ steps.lychee.outputs.exit_code }} + steps: + - name: Checkout + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 + with: + # workflow_dispatch can run from any ref, but the report is + # repository-global. Always measure the default branch so a manual + # run from a topic branch cannot close a report that main warrants, + # or overwrite it with branch-only findings. + ref: ${{ github.event.repository.default_branch }} + persist-credentials: false + + - name: Check links + id: lychee + uses: lycheeverse/lychee-action@e7477775783ea5526144ba13e8db5eec57747ce8 # v2.9.0 + with: + # Restricted to http(s) on purpose. Relative links resolve against + # the assembled multi-repo site (docs/make-full-website.sh pulls + # docs/src/format/catalog, format/namespace, integrations/spark and + # friends from other repositories), so they cannot be verified from + # this checkout alone and would report as broken on every run. + # + # crates.io answers automated clients with 403/404 no matter the user + # agent, so it is excluded rather than reported every day. + args: >- + --scheme https + --scheme http + --no-progress + --max-retries 3 + --timeout 20 + --exclude '^https://crates\.io/' + 'docs/src/**/*.md' + format: markdown + output: ./lychee/out.md + jobSummary: true + # The report, not a red build, is the signal for broken links. The + # report job below still fails the run if the check itself breaks. + fail: false + + - name: Upload report + if: steps.lychee.outputs.exit_code == 2 + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: link-report + path: ./lychee/out.md + retention-days: 7 + + report: + name: Update report issue + needs: scan + runs-on: ubuntu-24.04 + # Deliberately no checkout: this job needs the report artifact and the + # issues API, not the repository contents. + permissions: + issues: write + env: + EXIT_CODE: ${{ needs.scan.outputs.exit_code }} + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + steps: + - name: Classify checker result + # lychee exits 0 when every link resolves and 2 when links fail. + # Anything else (1 runtime, 3 bad config) means the check never + # produced a link verdict, which must surface as a failed run rather + # than be published as "broken documentation links". + run: | + case "$EXIT_CODE" in + 0|2) + echo "lychee exit code $EXIT_CODE" + ;; + *) + echo "::error::lychee exited with '$EXIT_CODE': the link check did not complete. Leaving the report issue untouched." + exit 1 + ;; + esac + + - name: Find existing report issue + id: report + # Matched on title alone, and through search rather than a listing: + # the issue action applies labels in a separate call after creating the + # issue, so a label filter misses a half-created report, and this + # repository has far more open issues than one listing page holds. + run: | + number=$(gh issue list --repo "$GITHUB_REPOSITORY" --state open \ + --search "in:title \"$REPORT_TITLE\" author:app/github-actions" \ + --limit 50 --json number,title \ + --jq "[.[] | select(.title == \"$REPORT_TITLE\") | .number] | first // empty") + echo "number=$number" >> "$GITHUB_OUTPUT" + + - name: Download report + if: env.EXIT_CODE == 2 + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: link-report + path: ./lychee + + - name: Compose report + if: env.EXIT_CODE == 2 + run: | + run_url="$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" + { + echo "Broken documentation links found by [\`$GITHUB_WORKFLOW\`]($run_url)." + echo + echo "This issue is rewritten by every scheduled run and closed automatically once all links resolve." + echo + echo "Entries can be false positives: some sites rate-limit or block automated clients while working fine in a browser. Confirm before editing the docs, and add persistent offenders to \`--exclude\` in \`.github/workflows/docs-link-check.yml\`." + echo + cat ./lychee/out.md + } > ./lychee/issue.md + + - name: Report broken links + if: env.EXIT_CODE == 2 + uses: peter-evans/create-issue-from-file@fca9117c27cdc29c6c4db3b86c48e4115a786710 # v6.0.0 + with: + # Empty on the first failing run, which creates the issue; afterwards + # the same issue is updated in place. + issue-number: ${{ steps.report.outputs.number }} + title: ${{ env.REPORT_TITLE }} + content-filepath: ./lychee/issue.md + labels: documentation + + - name: Close report issue once links are healthy + if: env.EXIT_CODE == 0 && steps.report.outputs.number != '' + env: + ISSUE_NUMBER: ${{ steps.report.outputs.number }} + run: | + run_url="$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" + gh issue close "$ISSUE_NUMBER" --repo "$GITHUB_REPOSITORY" \ + --comment "All documentation links resolved in [the latest run]($run_url)." diff --git a/.github/workflows/issue-labeler.yml b/.github/workflows/issue-labeler.yml new file mode 100644 index 00000000000..36f5a9cdbca --- /dev/null +++ b/.github/workflows/issue-labeler.yml @@ -0,0 +1,29 @@ +name: Issue Labeler + +# Applies bug / feature / performance labels to issues based on their title and +# body. This complements the .github/ISSUE_TEMPLATE forms, which only apply +# labels for issues opened through the web UI; this catches issues opened via +# `gh issue create`, the REST API, or an agent, which bypass templates. +# +# Issues never originate from a fork, so unlike the PR labelers this uses a +# plain `issues` trigger (no pull_request_target) with a scoped GITHUB_TOKEN. +on: + issues: + types: [opened, edited] + +concurrency: + group: ${{ github.workflow }}-${{ github.event.issue.number }} + cancel-in-progress: true + +jobs: + label: + name: Apply issue labels + permissions: + issues: write + runs-on: ubuntu-latest + steps: + - uses: srvaroa/labeler@bf262763a8a8e191f5847873aecc0f29df84f957 # v1.14.0 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + config_path: .github/labeler-issues.yml diff --git a/.github/workflows/nightly_run.yml b/.github/workflows/nightly_run.yml index 228b0b7de22..a944ce61234 100644 --- a/.github/workflows/nightly_run.yml +++ b/.github/workflows/nightly_run.yml @@ -11,7 +11,7 @@ permissions: jobs: run: runs-on: ubuntu-24.04 - if: github.repository == 'lancedb/lance' + if: github.repository == 'lance-format/lance' permissions: actions: write steps: @@ -25,7 +25,7 @@ jobs: jumbo-tests: # jumbo tests need more resources runs-on: ubuntu-24.04-8x - if: github.repository == 'lancedb/lance' + if: github.repository == 'lance-format/lance' timeout-minutes: 60 permissions: contents: read @@ -51,7 +51,7 @@ jobs: # slow and unbounded in depth, so it runs nightly with a generous timeout rather than blocking # merges; the manual escape hatch for arbitrary ref pairs lives in compat-pair.yml. compat-sequence: - if: github.repository == 'lancedb/lance' + if: github.repository == 'lance-format/lance' timeout-minutes: 360 runs-on: ubuntu-24.04 name: Index Sequence Compat diff --git a/.github/workflows/python.yml b/.github/workflows/python.yml index 601f2dd9dfc..98831ac6648 100644 --- a/.github/workflows/python.yml +++ b/.github/workflows/python.yml @@ -93,11 +93,66 @@ jobs: run: | source venv/bin/activate pytest --doctest-modules python/lance + linux-wheel: + timeout-minutes: 45 + name: Python Linux x86_64 wheel + runs-on: "ubuntu-24.04-4x" + defaults: + run: + shell: bash + steps: + - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 + with: + fetch-depth: 0 + lfs: true + - name: Set up Python + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5 + with: + python-version: "3.10" + - uses: Swatinem/rust-cache@779680da715d629ac1d338a641029a2f4372abb5 # v2 + with: + workspaces: python + prefix-key: ${{ env.CACHE_PREFIX }} + # python/Cargo.toml enables abi3-py310, so this wheel is reusable by + # every CPython version in the Linux test matrix. + - name: Build cp310 ABI3 wheel + uses: ./.github/workflows/build_linux_wheel + with: + python-minor-version: "10" + args: "--profile ci" + - name: Upload wheel + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + with: + name: linux-x86_64-wheel + path: python/target/wheels/pylance-*.whl + if-no-files-found: error + + linux-memtest: + timeout-minutes: 15 + name: Python Linux x86_64 memtest + runs-on: ubuntu-24.04 + steps: + - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 + - uses: actions-rust-lang/setup-rust-toolchain@a0b538fa0b742a6aa35d6e2c169b4bd06d225a98 # v1 + with: + cache: "false" + rustflags: "" + - name: Build memtest + working-directory: memtest + run: cargo build --release + - name: Upload memtest + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + with: + name: linux-x86_64-memtest + path: memtest/target/release/libmemtest.so + if-no-files-found: error + linux: + needs: [linux-wheel, linux-memtest] timeout-minutes: 45 strategy: matrix: - python-minor-version: ["10", "13"] + python-minor-version: ["10", "13", "14"] name: "Python Linux 3.${{ matrix.python-minor-version }} x86_64" runs-on: "ubuntu-24.04-4x" defaults: @@ -113,25 +168,22 @@ jobs: uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5 with: python-version: 3.${{ matrix.python-minor-version }} - - uses: Swatinem/rust-cache@779680da715d629ac1d338a641029a2f4372abb5 # v2 + - name: Download wheel + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4 with: - workspaces: python - prefix-key: ${{ env.CACHE_PREFIX }} - - uses: ./.github/workflows/build_linux_wheel + name: linux-x86_64-wheel + path: python/target/wheels + - name: Download memtest + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4 with: - args: "--profile ci" + name: linux-x86_64-memtest + path: memtest/python/memtest - uses: ./.github/workflows/run_tests with: memtest: true - - name: Upload wheels as artifacts - if: ${{ matrix.python-minor-version == '13' }} - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 - with: - name: linux-wheels - path: python/target/wheels/pylance-*.whl compat: - needs: linux + needs: linux-wheel timeout-minutes: 60 runs-on: ubuntu-24.04 name: Compatibility Tests @@ -147,15 +199,15 @@ jobs: - name: Set up Python uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5 with: - python-version: 3.13 - - name: Download wheels + python-version: 3.14 + - name: Download wheel uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4 with: - name: linux-wheels - path: python/wheels + name: linux-x86_64-wheel + path: python/target/wheels - name: Install dependencies run: | - pip install $(ls wheels/pylance-*.whl)[tests,ray] + pip install $(ls target/wheels/pylance-*.whl)[tests,ray] - name: Run compatibility tests run: | make compattest @@ -165,7 +217,7 @@ jobs: linux-arm: timeout-minutes: 45 runs-on: ubuntu-24.04-arm64-4x - name: Python Linux 3.13 ARM + name: Python Linux 3.14 ARM defaults: run: shell: bash @@ -178,7 +230,7 @@ jobs: - name: Set up Python uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5 with: - python-version: 3.13 + python-version: 3.14 - uses: Swatinem/rust-cache@779680da715d629ac1d338a641029a2f4372abb5 # v2 with: workspaces: python @@ -195,7 +247,7 @@ jobs: - uses: ./.github/workflows/run_tests mac: timeout-minutes: 45 - name: Python macOS 3.13 ARM + name: Python macOS 3.14 ARM runs-on: "warp-macos-14-arm64-6x" defaults: run: @@ -209,7 +261,7 @@ jobs: - name: Set up Python uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5 with: - python-version: "3.13" + python-version: "3.14" - uses: Swatinem/rust-cache@779680da715d629ac1d338a641029a2f4372abb5 # v2 with: workspaces: python @@ -242,6 +294,7 @@ jobs: args: "--profile ci" - uses: ./.github/workflows/run_tests aws-integtest: + needs: linux-wheel timeout-minutes: 45 runs-on: "ubuntu-24.04-4x" defaults: @@ -257,13 +310,11 @@ jobs: uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5 with: python-version: "3.11" # TODO: upgrade when ray supports 3.12 - - uses: Swatinem/rust-cache@779680da715d629ac1d338a641029a2f4372abb5 # v2 - with: - workspaces: python - prefix-key: ${{ env.CACHE_PREFIX }} - - uses: ./.github/workflows/build_linux_wheel + - name: Download wheel + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4 with: - args: "--profile ci" + name: linux-x86_64-wheel + path: python/target/wheels - name: Install dependencies run: | pip install ray[data] diff --git a/.github/workflows/recurring-tests.yml b/.github/workflows/recurring-tests.yml deleted file mode 100644 index dd8205a9567..00000000000 --- a/.github/workflows/recurring-tests.yml +++ /dev/null @@ -1,124 +0,0 @@ -name: Recurring Tests - -on: - schedule: - - cron: "0 0 * * 0" # Runs at 00:00 UTC every Sunday - workflow_dispatch: - -permissions: - contents: read - -jobs: - get-pylance-versions: - runs-on: ubuntu-latest - outputs: - matrix: ${{ steps.set-matrix.outputs.matrix }} - steps: - - name: Fetch latest 2 stable pylance versions - id: set-matrix - run: | - # Get all versions from PyPI - pypi_versions=$(curl -s https://pypi.org/pypi/pylance/json | jq -r '.releases | keys_unsorted | .[]') - - # Use only PyPI versions - all_versions=$(echo -e "$pypi_versions" | sort -u -V) - - # Get latest 2 stable versions - stable_versions=$(echo "$all_versions" | grep -E '^[0-9]+\.[0-9]+\.[0-9]+$' | sort -V | tail -n 2) - - # Create matrix array - matrix_versions=() - while IFS= read -r version; do - if [ -n "$version" ]; then - matrix_versions+=("$version") - fi - done <<< "$stable_versions" - - # Create JSON array manually - json_array="[" - for i in "${!matrix_versions[@]}"; do - if [ $i -gt 0 ]; then - json_array="$json_array," - fi - json_array="$json_array\"${matrix_versions[$i]}\"" - done - json_array="$json_array]" - - matrix="{\"pylance-version\": $json_array}" - echo "matrix=$matrix" >> $GITHUB_OUTPUT - - # This job is used to test the recurring tests on the latest 2 stable versions of Lance, - # and the back-compat of the recurring tests. - recurring-linux: - needs: get-pylance-versions - name: "Recurring: Linux (Pylance ${{ matrix.pylance-version }})" - runs-on: ubuntu-24.04 - timeout-minutes: 7200 - strategy: - fail-fast: false - matrix: ${{ fromJson(needs.get-pylance-versions.outputs.matrix) }} - defaults: - run: - shell: bash - working-directory: python - steps: - - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 - with: - fetch-depth: 0 - lfs: true - - name: Install protobuf - run: | - sudo apt update - sudo apt install -y protobuf-compiler - - name: Set up Python - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5 - with: - python-version: "3.13" - - name: Install dependencies - working-directory: python - shell: bash - run: | - pip install -e ".[tests]" - - uses: Swatinem/rust-cache@779680da715d629ac1d338a641029a2f4372abb5 # v2 - with: - workspaces: python - - name: Install Pylance - run: | - pip install pylance==${{ matrix.pylance-version }} - - name: Run recurring tests - id: run_recurring_tests - run: pytest -vvv -s python/tests/recurring/test_recurring.py - - name: Upgrade Pylance - run: pip install -e ".[tests]" - - name: Run recurring tests again - run: pytest -vvv -s python/tests/recurring/test_recurring.py - - # This job is used to test the recurring tests on the main branch. - recurring-linux-fresh-main: - name: "Recurring: Linux (main)" - runs-on: ubuntu-24.04 - timeout-minutes: 7200 - defaults: - run: - shell: bash - working-directory: python - steps: - - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 - with: - fetch-depth: 0 - lfs: true - - name: Install protobuf - run: | - sudo apt update - sudo apt install -y protobuf-compiler - - name: Set up Python - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5 - with: - python-version: "3.13" - - uses: Swatinem/rust-cache@779680da715d629ac1d338a641029a2f4372abb5 # v2 - with: - workspaces: python - - name: Install Lance - run: pip install -e ".[tests]" - - name: Run recurring tests - run: pytest -vvv -s python/tests/recurring/test_recurring.py diff --git a/.github/workflows/run_tests/action.yml b/.github/workflows/run_tests/action.yml index ab761532eeb..c5942a9c2ed 100644 --- a/.github/workflows/run_tests/action.yml +++ b/.github/workflows/run_tests/action.yml @@ -11,7 +11,7 @@ inputs: default: "false" memtest: required: false - description: "Run memtest" + description: "Install and preload the prebuilt memtest library" default: "false" runs: using: "composite" @@ -36,8 +36,9 @@ runs: if: inputs.memtest == 'true' shell: bash run: | - make build-release - echo "LD_PRELOAD=$(lance-memtest)" >> $GITHUB_ENV + test -f python/memtest/libmemtest.so + pip install -e . + echo "LD_PRELOAD=$(lance-memtest)" >> "$GITHUB_ENV" - name: Run python tests shell: bash working-directory: python diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index e3c17671ce3..8d2df7b0697 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -42,6 +42,25 @@ jobs: - name: Check formatting run: cargo fmt -- --check + package: + name: Check crates are publishable + runs-on: ubuntu-24.04 + timeout-minutes: 20 + steps: + - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 + - uses: actions-rust-lang/setup-rust-toolchain@a0b538fa0b742a6aa35d6e2c169b4bd06d225a98 # v1 + # Catches manifest problems that only surface at publish time (missing + # readme/license files, path dependencies without a version, excluded + # files a crate needs). `--no-verify` skips building each packaged crate, + # which the other jobs already cover. Keep the excludes in sync with + # .github/workflows/cargo-publish.yml. + - name: Package crates + run: | + cargo package --workspace --no-verify \ + --exclude lance-arrow-stats \ + --exclude lance-arrow-scalar \ + --all-features + rustdoc: runs-on: ubuntu-24.04 timeout-minutes: 30 @@ -83,9 +102,9 @@ jobs: log-level: warn command: check - linux-build: + linux-coverage-build: runs-on: "ubuntu-24.04-8x" - timeout-minutes: 60 + timeout-minutes: 75 env: # Need up-to-date compilers for kernels CC: clang @@ -96,22 +115,155 @@ jobs: - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 - name: Setup rust toolchain run: | - rustup toolchain install nightly - rustup default nightly + # Temporary mitigation for https://github.com/rust-lang/rust/issues/159261. + rustup toolchain install nightly-2026-07-13 --component llvm-tools-preview + rustup default nightly-2026-07-13 - uses: rui314/setup-mold@725a8794d15fc7563f59595bd9556495c0564878 # v1 - uses: Swatinem/rust-cache@779680da715d629ac1d338a641029a2f4372abb5 # v2 + with: + key: llvm-cov-ci - name: Install dependencies run: | sudo apt update sudo apt install -y protobuf-compiler libssl-dev - - name: Start DynamodDB and S3 + - name: Install cargo-llvm-cov + uses: taiki-e/install-action@66068bfca13dcb2ea07c3f613ca2836a37c755d5 # cargo-llvm-cov + with: + tool: cargo-llvm-cov + - name: Install cargo-nextest + uses: taiki-e/install-action@66068bfca13dcb2ea07c3f613ca2836a37c755d5 # nextest + with: + tool: nextest + - name: Build coverage tests + run: | + ALL_FEATURES=`cargo metadata --format-version=1 --no-deps | jq -r '.packages[] | .features | keys | .[]' | grep -v -e protoc | sort | uniq | paste -s -d "," -` + cargo +nightly-2026-07-13 llvm-cov nextest-archive \ + --cargo-profile ci \ + --locked \ + --workspace \ + --features ${ALL_FEATURES} \ + --archive-file rust-coverage-tests.tar.zst + - name: Upload coverage tests + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + with: + name: rust-coverage-tests + path: rust-coverage-tests.tar.zst + compression-level: 0 + retention-days: 1 + if-no-files-found: error + + linux-coverage-test: + needs: linux-coverage-build + runs-on: "ubuntu-24.04-8x" + timeout-minutes: 45 + strategy: + fail-fast: false + matrix: + shard: [1, 2, 3] + steps: + - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 + - name: Setup rust toolchain + run: | + # Temporary mitigation for https://github.com/rust-lang/rust/issues/159261. + rustup toolchain install nightly-2026-07-13 --component llvm-tools-preview + rustup default nightly-2026-07-13 + - name: Install dependencies + run: | + sudo apt update + sudo apt install -y libssl-dev + - name: Start DynamoDB and S3 run: docker compose -f docker-compose.yml up -d --wait - name: Install cargo-llvm-cov uses: taiki-e/install-action@66068bfca13dcb2ea07c3f613ca2836a37c755d5 # cargo-llvm-cov - - name: Run tests + with: + tool: cargo-llvm-cov + - name: Install cargo-nextest + uses: taiki-e/install-action@66068bfca13dcb2ea07c3f613ca2836a37c755d5 # nextest + with: + tool: nextest + - name: Download coverage tests + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4 + with: + name: rust-coverage-tests + path: coverage-artifacts + - name: Run coverage tests + run: | + cargo +nightly-2026-07-13 llvm-cov nextest \ + --no-report \ + --archive-file coverage-artifacts/rust-coverage-tests.tar.zst \ + --partition slice:${{ matrix.shard }}/3 + - name: Merge shard coverage + run: | + TOOLCHAIN_ROOT=`rustc +nightly-2026-07-13 --print sysroot` + HOST_TRIPLE=`rustc +nightly-2026-07-13 -vV | awk '/^host:/ {print $2}'` + LLVM_PROFDATA="${TOOLCHAIN_ROOT}/lib/rustlib/${HOST_TRIPLE}/bin/llvm-profdata" + find target/llvm-cov-target -maxdepth 1 -name '*.profraw' -print > coverage-artifacts/profraw-files + "${LLVM_PROFDATA}" merge \ + -sparse \ + -f coverage-artifacts/profraw-files \ + -o coverage-artifacts/coverage-shard-${{ matrix.shard }}.profdata + - name: Upload shard coverage + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + with: + name: rust-coverage-profile-${{ matrix.shard }} + path: coverage-artifacts/coverage-shard-${{ matrix.shard }}.profdata + retention-days: 1 + if-no-files-found: error + + linux-build: + needs: [linux-coverage-build, linux-coverage-test] + runs-on: ubuntu-24.04 + timeout-minutes: 20 + steps: + - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 + - name: Setup rust toolchain + run: | + # Temporary mitigation for https://github.com/rust-lang/rust/issues/159261. + rustup toolchain install nightly-2026-07-13 --component llvm-tools-preview + rustup default nightly-2026-07-13 + - name: Install cargo-llvm-cov + uses: taiki-e/install-action@66068bfca13dcb2ea07c3f613ca2836a37c755d5 # cargo-llvm-cov + with: + tool: cargo-llvm-cov + - name: Install cargo-nextest + uses: taiki-e/install-action@66068bfca13dcb2ea07c3f613ca2836a37c755d5 # nextest + with: + tool: nextest + - name: Download coverage tests + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4 + with: + name: rust-coverage-tests + path: coverage-artifacts + - name: Download shard coverage + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4 + with: + pattern: rust-coverage-profile-* + path: coverage-artifacts/profiles + merge-multiple: true + - name: Merge coverage run: | - ALL_FEATURES=`cargo metadata --format-version=1 --no-deps | jq -r '.packages[] | .features | keys | .[]' | grep -v -e protoc -e slow_tests | sort | uniq | paste -s -d "," -` - cargo +nightly llvm-cov --profile ci --locked --workspace --codecov --output-path coverage.codecov --features ${ALL_FEATURES} + mkdir -p target/llvm-cov-target + cargo +nightly-2026-07-13 nextest list \ + --archive-file coverage-artifacts/rust-coverage-tests.tar.zst \ + --extract-to target/llvm-cov-target > /dev/null + TOOLCHAIN_ROOT=`rustc +nightly-2026-07-13 --print sysroot` + HOST_TRIPLE=`rustc +nightly-2026-07-13 -vV | awk '/^host:/ {print $2}'` + LLVM_PROFDATA="${TOOLCHAIN_ROOT}/lib/rustlib/${HOST_TRIPLE}/bin/llvm-profdata" + "${LLVM_PROFDATA}" merge \ + -sparse \ + coverage-artifacts/profiles/*.profdata \ + -o target/llvm-cov-target/lance.profdata + cargo +nightly-2026-07-13 llvm-cov report \ + --nextest-archive-file coverage-artifacts/rust-coverage-tests.tar.zst \ + --codecov \ + --output-path coverage.codecov + - name: Upload coverage artifact + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + with: + name: rust-coverage-codecov + path: coverage.codecov + retention-days: 1 + if-no-files-found: error - name: Upload coverage to Codecov uses: codecov/codecov-action@b9fd7d16f6d7d1b5d2bec1a2887e65ceed900238 # v4 with: @@ -120,6 +272,7 @@ jobs: files: coverage.codecov flags: unittests fail_ci_if_error: false + linux-arm: runs-on: ubuntu-24.04-arm64-8x timeout-minutes: 75 @@ -137,41 +290,14 @@ jobs: sudo apt install -y protobuf-compiler libssl-dev pkg-config - name: Build tests run: | - ALL_FEATURES=`cargo metadata --format-version=1 --no-deps | jq -r '.packages[] | .features | keys | .[]' | grep -v -e protoc -e slow_tests | sort | uniq | paste -s -d "," -` + ALL_FEATURES=`cargo metadata --format-version=1 --no-deps | jq -r '.packages[] | .features | keys | .[]' | grep -v -e protoc | sort | uniq | paste -s -d "," -` cargo test --profile ci --locked --features ${ALL_FEATURES} --no-run - name: Start DynamodDB and S3 run: docker compose -f docker-compose.yml up -d --wait - name: Run tests run: | - ALL_FEATURES=`cargo metadata --format-version=1 --no-deps | jq -r '.packages[] | .features | keys | .[]' | grep -v -e protoc -e slow_tests | sort | uniq | paste -s -d "," -` + ALL_FEATURES=`cargo metadata --format-version=1 --no-deps | jq -r '.packages[] | .features | keys | .[]' | grep -v -e protoc | sort | uniq | paste -s -d "," -` cargo test --profile ci --locked --features ${ALL_FEATURES} - query-integration-tests: - runs-on: ubuntu-24.04-4x - timeout-minutes: 75 - env: - # We use opt-level 1 which makes some tests 5x faster to run. - RUSTFLAGS: "-C debuginfo=1 -C opt-level=1" - steps: - - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 - - name: Setup rust toolchain - run: | - rustup toolchain install stable - rustup default stable - - uses: rui314/setup-mold@725a8794d15fc7563f59595bd9556495c0564878 # v1 - - uses: Swatinem/rust-cache@779680da715d629ac1d338a641029a2f4372abb5 # v2 - with: - cache-targets: false - cache-workspace-crates: true - - name: Install dependencies - run: | - sudo apt -y -qq update - sudo apt install -y protobuf-compiler libssl-dev pkg-config - - name: Build query integration tests - run: | - cargo build --locked -p lance --no-default-features --features fp16kernels,slow_tests --tests --test integration_tests - - name: Run query integration tests - run: | - cargo test --locked -p lance --no-default-features --features fp16kernels,slow_tests --test integration_tests build-no-lock: runs-on: ubuntu-24.04-8x timeout-minutes: 30 @@ -195,7 +321,7 @@ jobs: sudo apt install -y protobuf-compiler libssl-dev - name: Build all run: | - ALL_FEATURES=`cargo metadata --format-version=1 --no-deps | jq -r '.packages[] | .features | keys | .[]' | grep -v -e protoc -e slow_tests | sort | uniq | paste -s -d "," -` + ALL_FEATURES=`cargo metadata --format-version=1 --no-deps | jq -r '.packages[] | .features | keys | .[]' | grep -v -e protoc | sort | uniq | paste -s -d "," -` cargo build --profile ci --benches --features ${ALL_FEATURES} --tests mac-build: runs-on: warp-macos-14-arm64-6x @@ -253,6 +379,96 @@ jobs: - name: Check benchmarks run: cargo check --profile ci --benches + qemu-pre-haswell: + # Verifies that lance-linalg's runtime SIMD dispatch still works + # correctly when the binary is built with the lower x86-64-v2 baseline + # (the legacy build path documented in CONTRIBUTING.md). Emulates a + # Nehalem CPU under qemu-user; catches any accidental AVX2/FMA + # instructions that leak past the runtime dispatch. + # + # The published-wheel default baseline (`target-cpu=haswell`) is set in + # `.cargo/config.toml`; this job overrides RUSTFLAGS for one job to + # exercise the legacy path without affecting any other build. + name: pre-Haswell SIGILL check (qemu Nehalem) + runs-on: ubuntu-24.04 + timeout-minutes: 60 + env: + CC: clang + CXX: clang++ + RUSTFLAGS: "-C target-cpu=x86-64-v2" + CARGO_TARGET_X86_64_UNKNOWN_LINUX_GNU_RUNNER: "qemu-x86_64 -cpu Nehalem" + steps: + - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 + - name: Setup rust toolchain + run: | + rustup toolchain install stable + rustup default stable + - uses: Swatinem/rust-cache@779680da715d629ac1d338a641029a2f4372abb5 # v2 + - name: Restore QEMU 8.2.10 + id: qemu-cache + uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + path: ${{ runner.temp }}/lance-qemu/8.2.10/qemu-x86_64 + key: qemu-user-8.2.10-x86_64-linux-user-${{ runner.os }}-${{ runner.arch }}-v1 + - name: Install dependencies + run: | + sudo apt update + sudo apt install -y protobuf-compiler libssl-dev + - name: Install QEMU build dependencies + if: steps.qemu-cache.outputs.cache-hit != 'true' + run: | + sudo apt install -y \ + ninja-build \ + pkg-config \ + libglib2.0-dev \ + python3-venv \ + curl \ + xz-utils + - name: Build QEMU 8.2.10 + if: steps.qemu-cache.outputs.cache-hit != 'true' + run: | + QEMU_VERSION=8.2.10 + QEMU_ARCHIVE="qemu-${QEMU_VERSION}.tar.xz" + QEMU_BIN_DIR="${RUNNER_TEMP}/lance-qemu/${QEMU_VERSION}" + QEMU_SOURCE_DIR="${RUNNER_TEMP}/qemu-${QEMU_VERSION}" + + # Ubuntu 24.04 ships QEMU 8.2.2, which is affected by + # https://gitlab.com/qemu-project/qemu/-/issues/2170. + curl --fail --location --silent --show-error \ + "https://download.qemu.org/${QEMU_ARCHIVE}" \ + --output "${RUNNER_TEMP}/${QEMU_ARCHIVE}" + echo "37b4a643da8ed6015eef35f5d7f06e7259d9c95359965a0a98e9667c621ab2bb ${RUNNER_TEMP}/${QEMU_ARCHIVE}" \ + | sha256sum --check - + tar --extract \ + --file "${RUNNER_TEMP}/${QEMU_ARCHIVE}" \ + --directory "${RUNNER_TEMP}" + mkdir "${QEMU_SOURCE_DIR}/build" + ( + cd "${QEMU_SOURCE_DIR}/build" + ../configure --target-list=x86_64-linux-user --disable-docs + ninja qemu-x86_64 + ) + mkdir -p "${QEMU_BIN_DIR}" + install -m 0755 \ + "${QEMU_SOURCE_DIR}/build/qemu-x86_64" \ + "${QEMU_BIN_DIR}/qemu-x86_64" + - name: Add QEMU to PATH + run: | + QEMU_BIN_DIR="${RUNNER_TEMP}/lance-qemu/8.2.10" + echo "${QEMU_BIN_DIR}" >> "${GITHUB_PATH}" + QEMU_VERSION_OUTPUT="$("${QEMU_BIN_DIR}/qemu-x86_64" --version)" + echo "${QEMU_VERSION_OUTPUT}" + if [[ "${QEMU_VERSION_OUTPUT}" != *"version 8.2.10"* ]]; then + echo "Expected QEMU 8.2.10, got: ${QEMU_VERSION_OUTPUT}" >&2 + exit 1 + fi + - name: Build lance-linalg lib tests in release mode + run: | + cargo test --release -p lance-linalg --lib --no-run + - name: Run lance-linalg lib tests under qemu Nehalem + run: | + cargo test --release -p lance-linalg --lib + msrv: # Check the minimum supported Rust version name: MSRV Check - Rust v${{ matrix.msrv }} @@ -280,5 +496,5 @@ jobs: env: RUSTUP_TOOLCHAIN: ${{ matrix.msrv }} run: | - ALL_FEATURES=`cargo metadata --format-version=1 --no-deps | jq -r '.packages[] | .features | keys | .[]' | grep -v -e protoc -e slow_tests | sort | uniq | paste -s -d "," -` + ALL_FEATURES=`cargo metadata --format-version=1 --no-deps | jq -r '.packages[] | .features | keys | .[]' | grep -v -e protoc | sort | uniq | paste -s -d "," -` cargo check --profile ci --workspace --tests --benches --features ${ALL_FEATURES} diff --git a/.gitignore b/.gitignore index dcc9c5089ff..b916613b9ce 100644 --- a/.gitignore +++ b/.gitignore @@ -105,6 +105,8 @@ test_data/venv **/*.profraw *.lance +!test_data/**/*.lance +!rust/lance-file/test_data/**/*.lance # Pytest benchmarks .benchmarks/ diff --git a/.typos.toml b/.typos.toml index 9285dd52a23..30f5fb4965f 100644 --- a/.typos.toml +++ b/.typos.toml @@ -23,6 +23,7 @@ nprob = "nprobe" extend-exclude = [ "notebooks/*.ipynb", "*_THIRD_PARTY_LICENSES.*", + "rust/lance-file/test_data/exact_versions/*.lance", "rust/lance-tokenizer/src/stop_word_filter/stopwords.rs", ] # If a line ends with # or // and has spellchecker:disable-line, ignore it diff --git a/AGENTS.md b/AGENTS.md index ee9b3b07e9d..3a154733bb3 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -4,31 +4,17 @@ Lance is a modern columnar data format optimized for ML workflows and datasets, Also see directory-specific guidelines: [rust/](rust/AGENTS.md) | [python/](python/AGENTS.md) | [java/](java/AGENTS.md) | [protos/](protos/AGENTS.md) | [docs/src/format/](docs/src/format/AGENTS.md) -## Architecture - -Rust workspace with Python and Java bindings: - -- `rust/lance/` - Main library implementing the columnar format -- `rust/lance-core/` - Core types, traits, and utilities -- `rust/lance-arrow/` - Apache Arrow integration layer -- `rust/lance-encoding/` - Data encoding and compression algorithms -- `rust/lance-file/` - File format reading/writing -- `rust/lance-index/` - Vector and scalar indexing -- `rust/lance-io/` - I/O operations and object store integration -- `rust/lance-linalg/` - Linear algebra for vector search -- `rust/lance-table/` - Table format and operations -- `rust/lance-geo/` - Geospatial data support -- `rust/lance-datagen/` - Data generation for tests and benchmarks -- `rust/lance-namespace/` / `rust/lance-namespace-impls/` - Namespace/catalog interfaces -- `rust/lance-test-macros/` / `rust/lance-testing/` - Test infrastructure -- `rust/lance-tools/` - CLI and developer tooling -- `rust/examples/` - Sample binaries and demonstrations -- `rust/compression/bitpacking/` / `rust/compression/fsst/` - Compression codecs -- `rust/lance-datafusion/` - DataFusion integration (built separately) -- `python/` - Python bindings (PyO3/maturin) -- `java/` - Java bindings (JNI) - -Key technical traits: async-first (tokio), Arrow-native, versioned writes with manifest tracking, custom ML-optimized encodings, unified object store interface (local/S3/Azure/GCS). +## File Format Stability and Compatibility + +- Treat every file format marked stable as a durable compatibility contract. All changes to a stable format must preserve both backward and forward compatibility. +- Treat every file format marked unstable as disposable. It may change freely; do not add compatibility code, migrations, fallbacks, or tests for files written by earlier unstable revisions. +- Evaluate compatibility against the latest released stable version while continuing to honor all stable format contracts. Changes that exist only on the current branch or `main` are not compatibility constraints; do not compromise a cleaner or more complete design to preserve those intermediate states. + +### Legacy Compatibility Boundaries + +- Treat formats and code paths that current writers no longer emit as frozen compatibility surfaces. Preserve their existing read behavior, but exclude them from new feature design unless legacy support is explicitly required. +- Implement new features in the current format and write paths. Do not extend legacy writers, retrofit new capabilities into legacy readers, or reuse legacy implementations as the foundation for new code. +- Avoid refactoring or otherwise modifying legacy code during feature work. If a shared boundary makes a legacy change unavoidable, isolate the change, preserve existing behavior, and add targeted regression coverage using released historical fixtures. ## Development Commands @@ -45,23 +31,12 @@ Key technical traits: async-first (tokio), Arrow-native, versioned writes with m * Use `release-with-debug` for benchmarks and profiling so optimized builds keep debug symbols without a rebuild. * Use `release-no-lto` only for local debugging, IO-bound benchmarks, or compile-time-sensitive performance investigation where LTO would not affect the measured bottleneck. -### Python / Java - -See [python/AGENTS.md](python/AGENTS.md) and [java/AGENTS.md](java/AGENTS.md). - ## Language-Specific Environment Contract - For language-specific tasks, always follow the environment and command rules in the corresponding subdirectory guide before running build, test, lint, format, or tooling commands. - Do not substitute a different environment manager or toolchain just because a command appears missing, unavailable, or slow. - If a language-specific command fails outside the documented workflow, treat that as an environment usage mistake first. Fix the environment usage, rerun with the prescribed commands, and only then conclude that a dependency or tool is unavailable. -### Integration Testing - -```bash -cd test_data && docker compose up -d -AWS_DEFAULT_REGION=us-east-1 pytest --run-integration python/tests/test_s3_ddb.py -``` - ## Coding Standards ### General @@ -70,7 +45,6 @@ AWS_DEFAULT_REGION=us-east-1 pytest --run-integration python/tests/test_s3_ddb.p - Code is for readability, not just execution. Only add meaningful comments and tests. - Comments should explain non-obvious "why" reasoning, not restate what the code does. - Remove debug prints (`println!`, `dbg!`, `print()`) before merging — use `tracing` or logging frameworks. -- Extract logic repeated in 2+ places into a shared helper; inline single-use logic at its call site. - Think carefully before adding a helper: only introduce one when it materially reduces cognitive load or eliminates substantial duplication, and do not add thin wrappers that only rename or forward existing calls. - Keep PRs focused — no drive-by refactors, reformatting, or cosmetic changes. - Be mindful of memory use: avoid collecting streams of `RecordBatch` into memory; use `RoaringBitmap` instead of `HashSet`. @@ -134,8 +108,14 @@ AWS_DEFAULT_REGION=us-east-1 pytest --run-integration python/tests/test_s3_ddb.p - Indent content under MkDocs admonition directives (`!!! note`, etc.) with 4 spaces. - Proofread comments and docs for typos before committing. +## Filing Issues + +- When opening an issue with `gh issue create` or the API, classify it and pass the matching label: `--label bug`, `--label feature`, or `--label performance`. These paths bypass the `.github/ISSUE_TEMPLATE` forms, so the label is not applied automatically. +- Prefix the title to match, e.g. `bug: ...`, `feature: ...`, or `perf: ...`. A content-based labeler (`.github/workflows/issue-labeler.yml`) uses this as a fallback signal, but an explicit `--label` is the reliable path. + ## Pull Requests +- Before creating a PR, search for similar PRs and inspect any PRs linked to the issue being addressed. If a matching PR exists, verify its current status and scope before proceeding to avoid creating duplicate work. - PR titles must follow the Conventional Commits specification because `.github/workflows/pr-title.yml` validates the PR title and body with commitlint. Use prefixes like `feat:`, `fix:`, `docs:`, `perf:`, `ci:`, `test:`, `build:`, `style:`, or `chore:`; add a scope when useful. - Before creating or updating a PR, run the lint checks for every touched language surface, even when they are expensive. For Rust changes, run `cargo fmt --all` and `cargo clippy --all --tests --benches -- -D warnings`. For Python changes, follow the environment workflow in `python/AGENTS.md` and run `uv run make lint` from `python/`. If a required lint check cannot be run, state the blocker explicitly in the PR summary. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 8f3ec285f31..d70e95d0c72 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -25,6 +25,10 @@ Currently Lance is implemented in Rust and comes with a Python wrapper. So you'l a. Install pre-commit: https://pre-commit.com/#install b. Run `pre-commit install` in the root of the repo +## x86_64 CPU compatibility + +The default workspace build targets `x86-64-v2` (SSE4.2), so binaries can run on pre-Haswell silicon that does not support AVX2. Runtime SIMD dispatch in `lance-linalg::distance` picks the appropriate tier (scalar / AVX / AVX+FMA / AVX2+FMA / AVX-512) based on the host. From Python, use `lance.simd_info()` to verify which tier was selected. + ## Sample Workflow 1. Fork the repo diff --git a/Cargo.lock b/Cargo.lock index f537f382479..3d5ca376262 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -148,9 +148,9 @@ dependencies = [ [[package]] name = "anyhow" -version = "1.0.102" +version = "1.0.104" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f202df86484c868dbad7eaa557ef785d5c66295e41b460ef922eca0723b842c" +checksum = "330a5ed07fa54e4702c9d6c4174f74427fc0ef6e214bbd677ae50a5099946470" [[package]] name = "approx" @@ -205,9 +205,9 @@ dependencies = [ [[package]] name = "arrow-arith" -version = "58.3.0" +version = "58.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a0ab212d2c1886e802f51c5212d78ebbcbb0bec980fff9dadc1eb8d45cd0b738" +checksum = "0a41203398f0eaa6f7ec8e62c0da742a21abf282c148fc157f6c35c90e29981a" dependencies = [ "arrow-array", "arrow-buffer", @@ -219,9 +219,9 @@ dependencies = [ [[package]] name = "arrow-array" -version = "58.3.0" +version = "58.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cfd33d3e92f207444098c75b42de99d329562be0cf686b307b097cc52b4e999e" +checksum = "ae33dad492b7df00a217563a7b0ef2874df68a0deea1b1a3acf628152f7f7a69" dependencies = [ "ahash", "arrow-buffer", @@ -238,9 +238,9 @@ dependencies = [ [[package]] name = "arrow-buffer" -version = "58.3.0" +version = "58.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c6cd424c2693bcdbc150d843dc9d4d137dd2de4782ce6df491ad11a3a0416c0" +checksum = "b9552f96391c005e6ab449fa941420935e7e062489b12b8b1b08879b2163f5b5" dependencies = [ "bytes", "half", @@ -250,9 +250,9 @@ dependencies = [ [[package]] name = "arrow-cast" -version = "58.3.0" +version = "58.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c5aefb56a2c02e9e2b30746241058b85f8983f0fcff2ba0c6d09006e1cded7f" +checksum = "3a8a327c9649f30d8406995f27642b68df354713cca3baaaf100f076f18d5f34" dependencies = [ "arrow-array", "arrow-buffer", @@ -287,9 +287,9 @@ dependencies = [ [[package]] name = "arrow-data" -version = "58.3.0" +version = "58.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3c88210023a2bfee1896af366309a3028fc3bcbd6515fa29a7990ee1baa08ee0" +checksum = "2b24852db04738907e06c04ea61e42fe7fda962a34513022dc0d0e754fb7976b" dependencies = [ "arrow-buffer", "arrow-schema", @@ -300,9 +300,9 @@ dependencies = [ [[package]] name = "arrow-ipc" -version = "58.3.0" +version = "58.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "238438f0834483703d88896db6fe5a7138b2230debc31b34c0336c2996e3c64f" +checksum = "29a908a11fcfb3fb2f6730f4ac15e367bc644e419155e96238f68cf3adde572b" dependencies = [ "arrow-array", "arrow-buffer", @@ -341,9 +341,9 @@ dependencies = [ [[package]] name = "arrow-ord" -version = "58.3.0" +version = "58.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1bffd8fd2579286a5d63bac898159873e5094a79009940bcb42bbfce4f19f1d0" +checksum = "63a083ec750f5c043f02946b4baf05fcdbb55f4560a3277055caca5cc99f3eb0" dependencies = [ "arrow-array", "arrow-buffer", @@ -354,9 +354,9 @@ dependencies = [ [[package]] name = "arrow-row" -version = "58.3.0" +version = "58.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bab5994731204603c73ba69267616c50f80780774c6bb0476f1f830625115e0c" +checksum = "514ba0ef0d4c5896202dae736251ce415abb43a950bed570fb7981b8716c0e4c" dependencies = [ "arrow-array", "arrow-buffer", @@ -367,9 +367,9 @@ dependencies = [ [[package]] name = "arrow-schema" -version = "58.3.0" +version = "58.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f633dbfdf39c039ada1bf9e34c694816eb71fbb7dc78f613993b7245e078a1ed" +checksum = "21ca356ad6425cecb6eb7b28e4f659f1ee7880fbb1a16127de7dd62901efee9e" dependencies = [ "bitflags 2.13.0", "serde_core", @@ -378,9 +378,9 @@ dependencies = [ [[package]] name = "arrow-select" -version = "58.3.0" +version = "58.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8cd065c54172ac787cf3f2f8d4107e0d3fdc26edba76fdf4f4cc170258942222" +checksum = "c58da39eb3d8350ad4a549e5c2bc49284dac554016c69829310350f1731b0aad" dependencies = [ "ahash", "arrow-array", @@ -460,18 +460,18 @@ checksum = "3b43422f69d8ff38f95f1b2bb76517c91589a924d1559a0e935d7c8ce0274c11" dependencies = [ "proc-macro2", "quote", - "syn 2.0.118", + "syn 2.0.119", ] [[package]] name = "async-trait" -version = "0.1.89" +version = "0.1.91" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9035ad2d096bed7955a320ee7e2230574d28fd3c3a0f186cbea1ff3c7eed5dbb" +checksum = "ae36dc4177970ef04fde5178d3e2429882def40e57a451f919c098f72baa6cec" dependencies = [ "proc-macro2", "quote", - "syn 2.0.118", + "syn 3.0.3", ] [[package]] @@ -647,7 +647,7 @@ dependencies = [ "http 0.2.12", "http 1.4.2", "http-body 1.0.1", - "lru", + "lru 0.16.4", "percent-encoding", "regex-lite", "sha2 0.10.9", @@ -1052,6 +1052,12 @@ version = "0.22.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" +[[package]] +name = "base64" +version = "0.23.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b25655df2c3cdd83c5e5b293b88acd880332b2ddadd7c30ac43144fdc0033da9" + [[package]] name = "base64-simd" version = "0.8.0" @@ -1225,7 +1231,7 @@ checksum = "89385e82b5d1821d2219e0b095efa2cc1f246cbf99080f3be46a1a85c0d392d9" dependencies = [ "proc-macro2", "quote", - "syn 2.0.118", + "syn 2.0.119", ] [[package]] @@ -1236,9 +1242,23 @@ checksum = "175812e0be2bccb6abe50bb8d566126198344f707e304f45c648fd8f2cc0365e" [[package]] name = "bytemuck" -version = "1.25.0" +version = "1.25.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c8efb64bd706a16a1bdde310ae86b351e4d21550d98d056f22f8a7f7a2183fec" +checksum = "95832e849adfb21180ccb6826a99da14e5d266ae5c2e668e1602cf234f153797" +dependencies = [ + "bytemuck_derive", +] + +[[package]] +name = "bytemuck_derive" +version = "1.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9abbd1bc6865053c427f7198e6af43bfdedc55ab791faed4fbd361d789575ff" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] [[package]] name = "byteorder" @@ -1248,9 +1268,9 @@ checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" [[package]] name = "bytes" -version = "1.12.0" +version = "1.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ae3f5d315924270530207e2a68396c3cc547f6dca3fbdca317cfb1a51edb593" +checksum = "fc652a48c352aef3ea3aed32080501cf3ef6ed5da78602a020c991775b0aff04" [[package]] name = "bytes-utils" @@ -1279,9 +1299,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.2.65" +version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e228eec9be7c17ccb640b59b36a5cd805ea2a564a4c5e162c2f659fea30d3b96" +checksum = "c89588d05638b5b4594a3348a2d6c20277e43a7f5c5202b05cc56888475a47b8" dependencies = [ "find-msvc-tools", "jobserver", @@ -1390,9 +1410,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.6.1" +version = "4.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ddb117e43bbf7dacf0a4190fef4d345b9bad68dfc649cb349e7d17d28428e51" +checksum = "d91e0c145792ef73a6ad36d27c75ac09f1832222a3c209689d90f534685ee5b7" dependencies = [ "clap_builder", "clap_derive", @@ -1400,9 +1420,9 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.6.0" +version = "4.6.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "714a53001bf66416adb0e2ef5ac857140e7dc3a0c48fb28b2f10762fc4b5069f" +checksum = "f09628afdcc538b57f3c6341e9c8e9970f18e4a481690a64974d7023bd33548b" dependencies = [ "anstream", "anstyle", @@ -1412,14 +1432,14 @@ dependencies = [ [[package]] name = "clap_derive" -version = "4.6.1" +version = "4.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f2ce8604710f6733aa641a2b3731eaa1e8b3d9973d5e3565da11800813f997a9" +checksum = "d012d2b9d65aca7f18f4d9878a045bc17899bba951561ba5ec3c2ba1eed9a061" dependencies = [ "heck", "proc-macro2", "quote", - "syn 2.0.118", + "syn 3.0.3", ] [[package]] @@ -1656,16 +1676,7 @@ dependencies = [ "crc", "digest 0.10.7", "rustversion", - "spin 0.10.0", -] - -[[package]] -name = "crc32c" -version = "0.6.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3a47af21622d091a8f0fb295b88bc886ac74efcc613efc19f5d0b21de5c89e47" -dependencies = [ - "rustc_version", + "spin 0.10.1", ] [[package]] @@ -1734,18 +1745,18 @@ dependencies = [ [[package]] name = "crossbeam-epoch" -version = "0.9.18" +version = "0.9.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e" +checksum = "2d6914041f254d6e9176c01941b21115dcfb7089e55135a35411081bd106ef3f" dependencies = [ "crossbeam-utils", ] [[package]] name = "crossbeam-queue" -version = "0.3.12" +version = "0.3.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0f58bbc28f91df819d0aa2a2c00cd19754769c2fad90579b3592b1c9ba7a3115" +checksum = "803d13fb3b09d88be9f4dbc29062c66b19bf7170867ceb746d2a8689bf6c7a26" dependencies = [ "crossbeam-utils", ] @@ -1908,7 +1919,7 @@ dependencies = [ "proc-macro2", "quote", "strsim 0.11.1", - "syn 2.0.118", + "syn 2.0.119", ] [[package]] @@ -1921,7 +1932,7 @@ dependencies = [ "proc-macro2", "quote", "strsim 0.11.1", - "syn 2.0.118", + "syn 2.0.119", ] [[package]] @@ -1943,7 +1954,7 @@ checksum = "fc34b93ccb385b40dc71c6fceac4b2ad23662c7eeb248cf10d529b7e055b6ead" dependencies = [ "darling_core 0.20.11", "quote", - "syn 2.0.118", + "syn 2.0.119", ] [[package]] @@ -1954,7 +1965,7 @@ checksum = "ac3984ec7bd6cfa798e62b4a642426a5be0e68f9401cfc2a01e3fa9ea2fcdb8d" dependencies = [ "darling_core 0.23.0", "quote", - "syn 2.0.118", + "syn 2.0.119", ] [[package]] @@ -1973,14 +1984,13 @@ dependencies = [ [[package]] name = "datafusion" -version = "53.1.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "93db0e623840612f7f2cd757f7e8a8922064192363732c88692e0870016e141b" +checksum = "754ef4e8f073922a26f5b23133b9db4829342362b09be0bc94309cf261c2f098" dependencies = [ "arrow", "arrow-schema", "async-trait", - "bytes", "chrono", "datafusion-catalog", "datafusion-catalog-listing", @@ -2007,12 +2017,11 @@ dependencies = [ "datafusion-session", "datafusion-sql", "futures", + "indexmap 2.14.0", "itertools 0.14.0", "log", "object_store", "parking_lot", - "rand 0.9.4", - "regex", "sqlparser", "tempfile", "tokio", @@ -2022,9 +2031,9 @@ dependencies = [ [[package]] name = "datafusion-catalog" -version = "53.1.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "37cefde60b26a7f4ff61e9d2ff2833322f91df2b568d7238afe67bde5bdffb66" +checksum = "06afd1e38dd27bbb1258685a1fc6524df6aff4e07b25b393a47de59635178d99" dependencies = [ "arrow", "async-trait", @@ -2047,9 +2056,9 @@ dependencies = [ [[package]] name = "datafusion-catalog-listing" -version = "53.1.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "17e112307715d6a7a331111a4c2330ff54bc237183511c319e3708a4cff431fb" +checksum = "f0668fb32c12065ec242be0e5b4bc62bd7a06a0be3ecd83791ef877e4be67e02" dependencies = [ "arrow", "async-trait", @@ -2070,32 +2079,33 @@ dependencies = [ [[package]] name = "datafusion-common" -version = "53.1.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d72a11ca44a95e1081870d3abb80c717496e8a7acb467a1d3e932bb636af5cc2" +checksum = "ca43b263cdff57042cfa8fb817fb3469f4878933380dccff25f5e793580abbf9" dependencies = [ - "ahash", "arrow", "arrow-ipc", + "arrow-schema", "chrono", + "foldhash 0.2.0", "half", - "hashbrown 0.16.1", + "hashbrown 0.17.1", "indexmap 2.14.0", "itertools 0.14.0", "libc", "log", "object_store", - "paste", "sqlparser", "tokio", + "uuid", "web-time", ] [[package]] name = "datafusion-common-runtime" -version = "53.1.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "89f4afaed29670ec4fd6053643adc749fe3f4bc9d1ce1b8c5679b22c67d12def" +checksum = "05f0ba2b864792bdca4d76c59a1de0ab6e1b61946596b9936888dbd6360035f2" dependencies = [ "futures", "log", @@ -2104,9 +2114,9 @@ dependencies = [ [[package]] name = "datafusion-datasource" -version = "53.1.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e9fb386e1691355355a96419978a0022b7947b44d4a24a6ea99f00b6b485cbb6" +checksum = "b840a8bce0bcbf5afad02946d438591e7c373f7afccaf3d874c04485772514dd" dependencies = [ "arrow", "async-trait", @@ -2126,16 +2136,17 @@ dependencies = [ "itertools 0.14.0", "log", "object_store", - "rand 0.9.4", + "parking_lot", + "rand 0.9.5", "tokio", "url", ] [[package]] name = "datafusion-datasource-arrow" -version = "53.1.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ffa6c52cfed0734c5f93754d1c0175f558175248bf686c944fb05c373e5fc096" +checksum = "a24cc0b9cf6e367f27f27406eff13abf48a11b72446aaa40b3105c0ded5c17d9" dependencies = [ "arrow", "arrow-ipc", @@ -2157,9 +2168,9 @@ dependencies = [ [[package]] name = "datafusion-datasource-csv" -version = "53.1.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "503f29e0582c1fc189578d665ff57d9300da1f80c282777d7eb67bb79fb8cdca" +checksum = "e1abe56b2a7a2d1d6de5117dd1a203181e267f28529faa5da546947621b697d7" dependencies = [ "arrow", "async-trait", @@ -2180,9 +2191,9 @@ dependencies = [ [[package]] name = "datafusion-datasource-json" -version = "53.1.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e33804749abc8d0c8cb7473228483cb8070e524c6f6086ee1b85a64debe2b3d2" +checksum = "9c3e467f0611ad7bdd5aad17c63c9bb6182d04e5282e5496d897ea2b49c024ba" dependencies = [ "arrow", "async-trait", @@ -2197,27 +2208,25 @@ dependencies = [ "datafusion-session", "futures", "object_store", - "serde_json", "tokio", "tokio-stream", ] [[package]] name = "datafusion-doc" -version = "53.1.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8de6ac0df1662b9148ad3c987978b32cbec7c772f199b1d53520c8fa764a87ee" +checksum = "d69bb69d8769e34f76839c960dbde24c1ac0c885a79b6c3c2287bdc56ec67891" [[package]] name = "datafusion-execution" -version = "53.1.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c03c7fbdaefcca4ef6ffe425a5fc2325763bfb426599bb0bf4536466efabe709" +checksum = "d8eac0a09bc8d263f52025cad9e001da4d8138d633fa288edda4d06b1772eae6" dependencies = [ "arrow", "arrow-buffer", "async-trait", - "chrono", "dashmap", "datafusion-common", "datafusion-expr", @@ -2226,18 +2235,19 @@ dependencies = [ "log", "object_store", "parking_lot", - "rand 0.9.4", + "rand 0.9.5", "tempfile", "url", ] [[package]] name = "datafusion-expr" -version = "53.1.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "574b9b6977fedbd2a611cbff12e5caf90f31640ad9dc5870f152836d94bad0dd" +checksum = "eeb14d374767ee0fc62dc79a5ba8bcf8a63c14e993c7d992d0e63adfa23d77d3" dependencies = [ "arrow", + "arrow-schema", "async-trait", "chrono", "datafusion-common", @@ -2248,29 +2258,27 @@ dependencies = [ "datafusion-physical-expr-common", "indexmap 2.14.0", "itertools 0.14.0", - "paste", "serde_json", "sqlparser", ] [[package]] name = "datafusion-expr-common" -version = "53.1.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7d7c3adf3db8bf61e92eb90cb659c8e8b734593a8f7c8e12a843c7ddba24b87e" +checksum = "b7b19a8c95522bee8cbb313d74263b85e355d2b52f42e67ef5694bf5de9e9356" dependencies = [ "arrow", "datafusion-common", "indexmap 2.14.0", "itertools 0.14.0", - "paste", ] [[package]] name = "datafusion-functions" -version = "53.1.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f28aa4e10384e782774b10e72aca4d93ef7b31aa653095d9d4536b0a3dbc51b6" +checksum = "5f64c983bbbdcb729d921a2b2ac3375598719b5cc0c30345ad664936f3176fc7" dependencies = [ "arrow", "arrow-buffer", @@ -2285,26 +2293,25 @@ dependencies = [ "datafusion-expr", "datafusion-expr-common", "datafusion-macros", + "datafusion-physical-expr-common", "hex", "itertools 0.14.0", "log", - "md-5 0.10.6", + "md-5 0.11.0", "memchr", "num-traits", - "rand 0.9.4", + "rand 0.9.5", "regex", - "sha2 0.10.9", - "unicode-segmentation", + "sha2 0.11.0", "uuid", ] [[package]] name = "datafusion-functions-aggregate" -version = "53.1.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "00aa6217e56098ba84e0a338176fe52f0a84cca398021512c6c8c5eff806d0ad" +checksum = "89bc17041e424a47ed062f43df24d84aab8b57c4c3221e5c1a5eef46d6c5718b" dependencies = [ - "ahash", "arrow", "datafusion-common", "datafusion-doc", @@ -2314,19 +2321,18 @@ dependencies = [ "datafusion-macros", "datafusion-physical-expr", "datafusion-physical-expr-common", + "foldhash 0.2.0", "half", "log", "num-traits", - "paste", ] [[package]] name = "datafusion-functions-aggregate-common" -version = "53.1.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b511250349407db7c43832ab2de63f5557b19a20dfd236b39ca2c04468b50d47" +checksum = "97dd2a9e865c6108059f5b37b77934f84b50bfb108f837bd0e5c9536e03f0545" dependencies = [ - "ahash", "arrow", "datafusion-common", "datafusion-expr-common", @@ -2335,9 +2341,9 @@ dependencies = [ [[package]] name = "datafusion-functions-nested" -version = "53.1.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ef13a858e20d50f0a9bb5e96e7ac82b4e7597f247515bccca4fdd2992df0212a" +checksum = "75f0bdfeef16d96417b9632ef855645376b242e9006a126dfd0bedfc54a93f5f" dependencies = [ "arrow", "arrow-ord", @@ -2351,34 +2357,34 @@ dependencies = [ "datafusion-functions-aggregate-common", "datafusion-macros", "datafusion-physical-expr-common", - "hashbrown 0.16.1", + "hashbrown 0.17.1", "itertools 0.14.0", "itoa", "log", - "paste", + "memchr", ] [[package]] name = "datafusion-functions-table" -version = "53.1.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "72b40d3f5bbb3905f9ccb1ce9485a9595c77b69758a7c24d3ba79e334ff51e7e" +checksum = "f4e4941673c917819616877e9993da4503e4f4739812be0bc32c5356184c6383" dependencies = [ "arrow", "async-trait", "datafusion-catalog", "datafusion-common", "datafusion-expr", + "datafusion-physical-expr", "datafusion-physical-plan", "parking_lot", - "paste", ] [[package]] name = "datafusion-functions-window" -version = "53.1.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d4e88ec9d57c9b685d02f58bfee7be62d72610430ddcedb82a08e5d9925dbfb6" +checksum = "12dd2e16c12b84b6f6b41b19f55b366dd1c46876bb35b86896c6349067379e8d" dependencies = [ "arrow", "datafusion-common", @@ -2389,14 +2395,13 @@ dependencies = [ "datafusion-physical-expr", "datafusion-physical-expr-common", "log", - "paste", ] [[package]] name = "datafusion-functions-window-common" -version = "53.1.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8307bb93519b1a91913723a1130cfafeee3f72200d870d88e91a6fc5470ede5c" +checksum = "4cdc5e4b6f8b6ef823cc1c761f85088ad4c884fe8df64df3cbcc6b2b84698441" dependencies = [ "datafusion-common", "datafusion-physical-expr-common", @@ -2404,20 +2409,20 @@ dependencies = [ [[package]] name = "datafusion-macros" -version = "53.1.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2e367e6a71051d0ebdd29b2f85d12059b38b1d1f172c6906e80016da662226bd" +checksum = "1a3614234dd93578c92428cb4f408e020874f0d2b7e6c90c928d9d28b5df2ceb" dependencies = [ "datafusion-doc", "quote", - "syn 2.0.118", + "syn 2.0.119", ] [[package]] name = "datafusion-optimizer" -version = "53.1.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e929015451a67f77d9d8b727b2bf3a40c4445fdef6cdc53281d7d97c76888ace" +checksum = "a0635620b050b81bb92764e99250868f654e2cd5ad1bece413283b3f73c83179" dependencies = [ "arrow", "chrono", @@ -2434,11 +2439,10 @@ dependencies = [ [[package]] name = "datafusion-physical-expr" -version = "53.1.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4b1e68aba7a4b350401cfdf25a3d6f989ad898a7410164afe9ca52080244cb59" +checksum = "8cabf7a86eb70b816729e33c81bf7767c936ee1226f607a114f5dac2decac8d0" dependencies = [ - "ahash", "arrow", "datafusion-common", "datafusion-expr", @@ -2446,20 +2450,19 @@ dependencies = [ "datafusion-functions-aggregate-common", "datafusion-physical-expr-common", "half", - "hashbrown 0.16.1", + "hashbrown 0.17.1", "indexmap 2.14.0", "itertools 0.14.0", "parking_lot", - "paste", "petgraph", "tokio", ] [[package]] name = "datafusion-physical-expr-adapter" -version = "53.1.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ea22315f33cf2e0adc104e8ec42e285f6ed93998d565c65e82fec6a9ee9f9db4" +checksum = "de222e04f7e6744501555a54ab0abe26bfdfebee380af79a9bdc175704246859" dependencies = [ "arrow", "datafusion-common", @@ -2472,26 +2475,26 @@ dependencies = [ [[package]] name = "datafusion-physical-expr-common" -version = "53.1.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b04b45ea8ad3ac2d78f2ea2a76053e06591c9629c7a603eda16c10649ecf4362" +checksum = "72d0d0057fc5a502d45c870cb6d47c66eb7bdd5edb1bd71ad6f3f724975ac2a8" dependencies = [ - "ahash", "arrow", "chrono", "datafusion-common", "datafusion-expr-common", - "hashbrown 0.16.1", + "hashbrown 0.17.1", "indexmap 2.14.0", "itertools 0.14.0", "parking_lot", + "pin-project", ] [[package]] name = "datafusion-physical-optimizer" -version = "53.1.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7cb13397809a425918f608dfe8653f332015a3e330004ab191b4404187238b95" +checksum = "86046eed10950c5f9aaed9acfd148e9bd2e1dfdfe4f9aef607d1447b271e4183" dependencies = [ "arrow", "datafusion-common", @@ -2507,12 +2510,13 @@ dependencies = [ [[package]] name = "datafusion-physical-plan" -version = "53.1.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5edc023675791af9d5fb4cc4c24abf5f7bd3bd4dcf9e5bd90ea1eff6976dcc79" +checksum = "9bc84da934c903407ba297971ebcc020c4c1a38aafd765d6c144c76eff3fa6a1" dependencies = [ - "ahash", "arrow", + "arrow-data", + "arrow-ipc", "arrow-ord", "arrow-schema", "async-trait", @@ -2527,7 +2531,7 @@ dependencies = [ "datafusion-physical-expr-common", "futures", "half", - "hashbrown 0.16.1", + "hashbrown 0.17.1", "indexmap 2.14.0", "itertools 0.14.0", "log", @@ -2539,9 +2543,9 @@ dependencies = [ [[package]] name = "datafusion-pruning" -version = "53.1.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ac8c76860e355616555081cab5968cec1af7a80701ff374510860bcd567e365a" +checksum = "eb63eeac6de19be40f487b65dd84e546195f783c5a9928618e0c4f2a3569b0d7" dependencies = [ "arrow", "datafusion-common", @@ -2550,15 +2554,14 @@ dependencies = [ "datafusion-physical-expr", "datafusion-physical-expr-common", "datafusion-physical-plan", - "itertools 0.14.0", "log", ] [[package]] name = "datafusion-session" -version = "53.1.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5412111aa48e2424ba926112e192f7a6b7e4ccb450145d25ce5ede9f19dc491e" +checksum = "5f961d209177f91bd014db5cbb2c33b7d28a2597b9003e77f17aeb712964315a" dependencies = [ "async-trait", "datafusion-common", @@ -2570,9 +2573,9 @@ dependencies = [ [[package]] name = "datafusion-sql" -version = "53.1.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fa0d133ddf8b9b3b872acac900157f783e7b879fe9a6bccf389abebbfac45ec1" +checksum = "b1d71cb454da682b2af7488e1fc1ddd72ee1b28f19297b8ccad73f0a21ee9a69" dependencies = [ "arrow", "bigdecimal", @@ -2588,9 +2591,9 @@ dependencies = [ [[package]] name = "datafusion-substrait" -version = "53.1.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "98494539a5468979cc42d86c7bc5f0f8cb71ee5c742694c26fc34efdd29dd2e5" +checksum = "f047a6fbf967b6b523758a48d2377bb5f9373a20e7ae4c4326d3c569f80ba3d7" dependencies = [ "async-recursion", "async-trait", @@ -2650,7 +2653,6 @@ version = "0.5.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7cd812cc2bc1d69d4764bd80df88b4317eaef9e773c75226407d9bc0876b211c" dependencies = [ - "powerfmt", "serde_core", ] @@ -2693,7 +2695,7 @@ dependencies = [ "darling 0.20.11", "proc-macro2", "quote", - "syn 2.0.118", + "syn 2.0.119", ] [[package]] @@ -2713,7 +2715,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ab63b0e2bf4d5928aff72e83a7dace85d7bba5fe12dcc3c5a572d78caffd3f3c" dependencies = [ "derive_builder_core 0.20.2", - "syn 2.0.118", + "syn 2.0.119", ] [[package]] @@ -2775,7 +2777,7 @@ checksum = "1ac70aa55017e108007fbaf5aa0f54b021c98f92ff8af59d42eda9da96e3dd4f" dependencies = [ "proc-macro2", "quote", - "syn 2.0.118", + "syn 2.0.119", ] [[package]] @@ -2860,6 +2862,12 @@ dependencies = [ "encoding_rs", ] +[[package]] +name = "endian-type" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c34f04666d835ff5d62e058c3995147c06f42fe86ff053337632bca83e42702d" + [[package]] name = "env_filter" version = "2.0.0" @@ -2900,7 +2908,7 @@ checksum = "44f23cf4b44bfce11a86ace86f8a73ffdec849c9fd00a386a53d278bd9e81fb3" dependencies = [ "proc-macro2", "quote", - "syn 2.0.118", + "syn 2.0.119", ] [[package]] @@ -2936,11 +2944,10 @@ checksum = "40404c3f5f511ec4da6fe866ddf6a717c309fdbb69fbbad7b0f3edab8f2e835f" [[package]] name = "event-listener" -version = "5.4.1" +version = "5.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e13b66accf52311f30a0db42147dadea9850cb48cd070028831ae5f5d4b856ab" +checksum = "5a23add41df1562121a9393cb065eab5146a1242410f23a644851e90cfd669d2" dependencies = [ - "concurrent-queue", "parking", "pin-project-lite", ] @@ -3068,6 +3075,12 @@ dependencies = [ "futures-core", ] +[[package]] +name = "frostem" +version = "1.20260804.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d0ae10cfccbae085dd8612669ccc116fe88bd5dfe39d202a8fa68c24c1e546f" + [[package]] name = "fs_extra" version = "1.3.0" @@ -3076,10 +3089,10 @@ checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c" [[package]] name = "fsst" -version = "9.0.0-beta.16" +version = "11.0.0-beta.6" dependencies = [ "arrow-array", - "rand 0.9.4", + "rand 0.9.5", "test-log", "tokio", ] @@ -3155,7 +3168,7 @@ checksum = "e835b70203e41293343137df5c0664546da5745f82ec9b84d40be8336958447b" dependencies = [ "proc-macro2", "quote", - "syn 2.0.118", + "syn 2.0.119", ] [[package]] @@ -3312,12 +3325,13 @@ dependencies = [ [[package]] name = "geodatafusion" -version = "0.4.0" +version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "af7cd430f1a1f59bc97053d824ad410ea6fd123c8977b3c1a75335e289233b8b" +checksum = "fecbdd00d0fff2b04635c1b1e4129c217908f0c2d17539e0a2275308afce2552" dependencies = [ "arrow-arith", "arrow-array", + "arrow-buffer", "arrow-schema", "datafusion", "geo", @@ -3369,11 +3383,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd" dependencies = [ "cfg-if 1.0.4", - "js-sys", "libc", "r-efi 5.3.0", "wasip2", - "wasm-bindgen", ] [[package]] @@ -3413,7 +3425,7 @@ checksum = "53010ccb100b96a67bc32c0175f0ed1426b31b655d562898e57325f81c023ac0" dependencies = [ "proc-macro2", "quote", - "syn 2.0.118", + "syn 2.0.119", ] [[package]] @@ -3436,17 +3448,25 @@ dependencies = [ [[package]] name = "goosefs-sdk" -version = "0.1.5" +version = "0.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ae079b88ffe7772d12cfc5c40a5a324babb357893d95b5e3a22ae857f236c5f" +checksum = "4a9bc9414e3b2cb0bd08dfe0eb315b177e86b119c7fa5e16179c92fc7b184860" dependencies = [ + "arc-swap", "async-trait", "bytes", "dashmap", + "futures", "hostname", + "io-uring", + "itoa", + "libc", + "lru 0.12.5", + "memmap2", + "moka", "prost", "prost-types", - "rand 0.9.4", + "rand 0.9.5", "reqwest 0.12.28", "serde", "thiserror 2.0.18", @@ -3456,6 +3476,7 @@ dependencies = [ "tonic-prost", "tracing", "uuid", + "xxhash-rust", ] [[package]] @@ -3483,6 +3504,7 @@ version = "2.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6ea2d84b969582b4b1864a92dc5d27cd2b77b622a8d79306834f1be5ba20d84b" dependencies = [ + "bytemuck", "cfg-if 1.0.4", "crunchy", "num-traits", @@ -3516,6 +3538,8 @@ version = "0.15.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9229cfe53dfd69f0609a49f65461bd93001ea1ef889cd5529dd176593f5338a1" dependencies = [ + "allocator-api2", + "equivalent", "foldhash 0.1.5", ] @@ -3535,6 +3559,11 @@ name = "hashbrown" version = "0.17.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" +dependencies = [ + "allocator-api2", + "equivalent", + "foldhash 0.2.0", +] [[package]] name = "heapify" @@ -3584,7 +3613,7 @@ dependencies = [ "log", "native-tls", "num_cpus", - "rand 0.9.4", + "rand 0.9.5", "reqwest 0.12.28", "serde", "serde_json", @@ -3714,9 +3743,9 @@ checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" [[package]] name = "humantime" -version = "2.3.0" +version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "135b12329e5e3ce057a9f972339ea52bc954fe1e9358ef27f95e89716fbc5424" +checksum = "15cdd26707701c53297e2fa6afb323d55fbc1d0810c3aec078ae3ef0424c3c15" [[package]] name = "hybrid-array" @@ -4198,18 +4227,18 @@ checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" [[package]] name = "jieba-macros" -version = "0.10.1" +version = "0.10.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "46adade69b634535a8f495cf87710ed893cff53e1dbc9dd750c2ab81c5defb82" +checksum = "38fc0f3831de71556de69643b80a08a5c8cd260a23c6b8dbeb7cd923c779cac5" dependencies = [ "phf_codegen", ] [[package]] name = "jieba-rs" -version = "0.10.1" +version = "0.10.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "11b53580aaa8ec8b713da271da434f8947409242c537a9ab3f7b76bdbb19e8a9" +checksum = "bb5bdea4dc241d589e179f39d2a778f31490f3370aa2f626223dbd930ebc5c9d" dependencies = [ "bytecount", "cedarwood", @@ -4244,7 +4273,7 @@ checksum = "782d32378dddf207193ac91cefb848ad41abb58195c95168e1291227a0832b47" dependencies = [ "proc-macro2", "quote", - "syn 2.0.118", + "syn 2.0.119", ] [[package]] @@ -4289,7 +4318,7 @@ dependencies = [ "quote", "rustc_version", "simd_cesu8", - "syn 2.0.118", + "syn 2.0.119", ] [[package]] @@ -4308,7 +4337,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "38c0b942f458fe50cdac086d2f946512305e5631e720728f2a61aabcd47a6264" dependencies = [ "quote", - "syn 2.0.118", + "syn 2.0.119", ] [[package]] @@ -4346,7 +4375,7 @@ dependencies = [ "nom 8.0.0", "num-traits", "ordered-float 5.3.0", - "rand 0.9.4", + "rand 0.9.5", "serde", "serde_json", "zmij", @@ -4380,7 +4409,7 @@ checksum = "e037a2e1d8d5fdbd49b16a4ea09d5d6401c1f29eca5ff29d03d3824dba16256a" [[package]] name = "lance" -version = "9.0.0-beta.16" +version = "11.0.0-beta.6" dependencies = [ "all_asserts", "approx", @@ -4457,7 +4486,7 @@ dependencies = [ "prost-build", "prost-types", "protobuf-src", - "rand 0.9.4", + "rand 0.9.5", "rayon", "reqwest 0.12.28", "roaring", @@ -4483,7 +4512,7 @@ dependencies = [ [[package]] name = "lance-arrow" -version = "9.0.0-beta.16" +version = "11.0.0-beta.6" dependencies = [ "arrow-array", "arrow-buffer", @@ -4492,13 +4521,14 @@ dependencies = [ "arrow-ord", "arrow-schema", "arrow-select", + "bytemuck", "bytes", "futures", "getrandom 0.2.17", "half", "jsonb", "num-traits", - "rand 0.9.4", + "rand 0.9.5", ] [[package]] @@ -4531,7 +4561,7 @@ dependencies = [ [[package]] name = "lance-bitpacking" -version = "9.0.0-beta.16" +version = "11.0.0-beta.6" dependencies = [ "arrayref", "bitpacking", @@ -4542,15 +4572,17 @@ dependencies = [ [[package]] name = "lance-core" -version = "9.0.0-beta.16" +version = "11.0.0-beta.6" dependencies = [ "arrow-array", "arrow-buffer", "arrow-data", "arrow-schema", "async-trait", + "blake3", "byteorder", "bytes", + "criterion", "datafusion-common", "datafusion-sql", "futures", @@ -4566,7 +4598,8 @@ dependencies = [ "pin-project", "proptest", "prost", - "rand 0.9.4", + "quick_cache", + "rand 0.9.5", "roaring", "rstest", "serde_json", @@ -4582,7 +4615,7 @@ dependencies = [ [[package]] name = "lance-datafusion" -version = "9.0.0-beta.16" +version = "11.0.0-beta.6" dependencies = [ "arrow", "arrow-array", @@ -4615,7 +4648,7 @@ dependencies = [ [[package]] name = "lance-datagen" -version = "9.0.0-beta.16" +version = "11.0.0-beta.6" dependencies = [ "arrow", "arrow-array", @@ -4627,23 +4660,23 @@ dependencies = [ "half", "hex", "lance-testing", - "rand 0.9.4", + "rand 0.9.5", "rand_distr", "rand_xoshiro", ] [[package]] name = "lance-derive" -version = "9.0.0-beta.16" +version = "11.0.0-beta.6" dependencies = [ "proc-macro2", "quote", - "syn 2.0.118", + "syn 2.0.119", ] [[package]] name = "lance-encoding" -version = "9.0.0-beta.16" +version = "11.0.0-beta.6" dependencies = [ "arrow-arith", "arrow-array", @@ -4674,11 +4707,10 @@ dependencies = [ "prost", "prost-build", "protobuf-src", - "rand 0.9.4", + "rand 0.9.5", "rand_xoshiro", "rstest", "serial_test", - "strum 0.26.3", "test-log", "tokio", "tracing", @@ -4688,7 +4720,7 @@ dependencies = [ [[package]] name = "lance-examples" -version = "9.0.0-beta.16" +version = "11.0.0-beta.6" dependencies = [ "all_asserts", "arrow", @@ -4706,7 +4738,7 @@ dependencies = [ "lance-linalg", "object_store", "parquet", - "rand 0.9.4", + "rand 0.9.5", "tempfile", "tokenizers", "tokio", @@ -4714,11 +4746,12 @@ dependencies = [ [[package]] name = "lance-file" -version = "9.0.0-beta.16" +version = "11.0.0-beta.6" dependencies = [ "arrow-arith", "arrow-array", "arrow-buffer", + "arrow-cast", "arrow-data", "arrow-schema", "arrow-select", @@ -4745,6 +4778,7 @@ dependencies = [ "prost-build", "prost-types", "protobuf-src", + "rand 0.9.5", "rstest", "test-log", "tokio", @@ -4753,7 +4787,7 @@ dependencies = [ [[package]] name = "lance-geo" -version = "9.0.0-beta.16" +version = "11.0.0-beta.6" dependencies = [ "datafusion", "geo-traits", @@ -4767,13 +4801,14 @@ dependencies = [ [[package]] name = "lance-index" -version = "9.0.0-beta.16" +version = "11.0.0-beta.6" dependencies = [ "approx", "arc-swap", "arrow", "arrow-arith", "arrow-array", + "arrow-ipc", "arrow-ord", "arrow-schema", "arrow-select", @@ -4810,6 +4845,7 @@ dependencies = [ "lance-encoding", "lance-file", "lance-geo", + "lance-index-core", "lance-io", "lance-linalg", "lance-select", @@ -4825,7 +4861,7 @@ dependencies = [ "prost-build", "prost-types", "protobuf-src", - "rand 0.9.4", + "rand 0.9.5", "rand_distr", "rangemap", "rayon", @@ -4842,19 +4878,35 @@ dependencies = [ "uuid", ] +[[package]] +name = "lance-index-core" +version = "11.0.0-beta.6" +dependencies = [ + "arrow-array", + "arrow-schema", + "arrow-select", + "async-trait", + "bytes", + "datafusion", + "datafusion-common", + "datafusion-expr", + "futures", + "lance-core", + "lance-io", + "lance-select", + "prost-types", + "roaring", + "serde", + "serde_json", +] + [[package]] name = "lance-io" -version = "9.0.0-beta.16" +version = "11.0.0-beta.6" dependencies = [ "arrow", - "arrow-arith", "arrow-array", - "arrow-buffer", - "arrow-cast", - "arrow-data", "arrow-schema", - "arrow-select", - "async-recursion", "async-trait", "aws-config", "aws-credential-types", @@ -4870,6 +4922,8 @@ dependencies = [ "lance-namespace", "lance-testing", "log", + "metrics", + "metrics-util", "mock_instant", "mockall", "moka", @@ -4879,7 +4933,7 @@ dependencies = [ "path_abs", "pin-project", "prost", - "rand 0.9.4", + "rand 0.9.5", "rstest", "serde", "tempfile", @@ -4892,7 +4946,7 @@ dependencies = [ [[package]] name = "lance-linalg" -version = "9.0.0-beta.16" +version = "11.0.0-beta.6" dependencies = [ "approx", "arrow-array", @@ -4906,13 +4960,14 @@ dependencies = [ "lance-testing", "num-traits", "proptest", - "rand 0.9.4", + "rand 0.9.5", "rayon", + "rstest", ] [[package]] name = "lance-namespace" -version = "9.0.0-beta.16" +version = "11.0.0-beta.6" dependencies = [ "arrow", "async-trait", @@ -4924,7 +4979,7 @@ dependencies = [ [[package]] name = "lance-namespace-datafusion" -version = "9.0.0-beta.16" +version = "11.0.0-beta.6" dependencies = [ "arrow-array", "arrow-schema", @@ -4940,7 +4995,7 @@ dependencies = [ [[package]] name = "lance-namespace-impls" -version = "9.0.0-beta.16" +version = "11.0.0-beta.6" dependencies = [ "arrow", "arrow-array", @@ -4968,8 +5023,8 @@ dependencies = [ "log", "object_store", "opendal", - "quick-xml 0.38.4", - "rand 0.9.4", + "quick-xml 0.40.1", + "rand 0.9.5", "reqwest 0.12.28", "ring", "roaring", @@ -4979,7 +5034,6 @@ dependencies = [ "serde_json", "sha2 0.10.9", "tempfile", - "time", "tokio", "tower", "tower-http 0.5.2", @@ -5004,7 +5058,7 @@ dependencies = [ [[package]] name = "lance-select" -version = "9.0.0-beta.16" +version = "11.0.0-beta.6" dependencies = [ "arrow-array", "arrow-buffer", @@ -5022,7 +5076,7 @@ dependencies = [ [[package]] name = "lance-table" -version = "9.0.0-beta.16" +version = "11.0.0-beta.6" dependencies = [ "arrow", "arrow-array", @@ -5032,6 +5086,7 @@ dependencies = [ "async-trait", "aws-credential-types", "aws-sdk-dynamodb", + "blake3", "byteorder", "bytes", "chrono", @@ -5052,7 +5107,7 @@ dependencies = [ "prost-build", "prost-types", "protobuf-src", - "rand 0.9.4", + "rand 0.9.5", "rangemap", "roaring", "rstest", @@ -5068,16 +5123,16 @@ dependencies = [ [[package]] name = "lance-test-macros" -version = "9.0.0-beta.16" +version = "11.0.0-beta.6" dependencies = [ "proc-macro2", "quote", - "syn 2.0.118", + "syn 2.0.119", ] [[package]] name = "lance-testing" -version = "9.0.0-beta.16" +version = "11.0.0-beta.6" dependencies = [ "arrow-array", "arrow-schema", @@ -5085,17 +5140,17 @@ dependencies = [ "lance-arrow", "num-traits", "pprof", - "rand 0.9.4", + "rand 0.9.5", ] [[package]] name = "lance-tokenizer" -version = "9.0.0-beta.16" +version = "11.0.0-beta.6" dependencies = [ + "frostem", "icu_segmenter", "jieba-rs", "lindera", - "rust-stemmers", "serde", "stop-words", "unicode-normalization", @@ -5103,7 +5158,7 @@ dependencies = [ [[package]] name = "lance-tools" -version = "9.0.0-beta.16" +version = "11.0.0-beta.6" dependencies = [ "clap", "lance-core", @@ -5120,7 +5175,7 @@ version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" dependencies = [ - "spin 0.9.8", + "spin 0.9.9", ] [[package]] @@ -5182,9 +5237,9 @@ dependencies = [ [[package]] name = "libc" -version = "0.2.186" +version = "0.2.189" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66" +checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2" [[package]] name = "libm" @@ -5229,8 +5284,8 @@ dependencies = [ "serde", "serde_json", "serde_yaml_ng", - "strum 0.28.0", - "strum_macros 0.28.0", + "strum", + "strum_macros", "unicode-blocks", "unicode-normalization", "unicode-segmentation", @@ -5259,8 +5314,8 @@ dependencies = [ "rkyv", "serde", "serde_json", - "strum 0.28.0", - "strum_macros 0.28.0", + "strum", + "strum_macros", "thiserror 2.0.18", ] @@ -5316,6 +5371,15 @@ dependencies = [ "tracing-subscriber", ] +[[package]] +name = "lru" +version = "0.12.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "234cf4f4a04dc1f57e24b96cc0cd600cf2af460d4161ac5ecdd0af8e1f3b2a38" +dependencies = [ + "hashbrown 0.15.5", +] + [[package]] name = "lru" version = "0.16.4" @@ -5451,13 +5515,43 @@ checksum = "88904434abc2901f197fe8cc55f0445e7ded921dba5911dad2e2b39b48e663c4" [[package]] name = "memmap2" -version = "0.9.10" +version = "0.9.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "714098028fe011992e1c3962653c96b2d578c4b4bce9036e15ff220319b1e0e3" +checksum = "d1219ed1b7f229ee7104d281dd01d6802fe28bb6e95d292942c4daacdeb798c0" dependencies = [ "libc", ] +[[package]] +name = "metrics" +version = "0.24.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "89550ee9f79e88fef3119de263694973a8adb26c21d75322164fb8c493039fe2" +dependencies = [ + "portable-atomic", + "rapidhash", +] + +[[package]] +name = "metrics-util" +version = "0.19.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8496cc523d1f94c1385dd8f0f0c2c480b2b8aeccb5b7e4485ad6365523ae376" +dependencies = [ + "aho-corasick", + "crossbeam-epoch", + "crossbeam-utils", + "hashbrown 0.15.5", + "indexmap 2.14.0", + "metrics", + "ordered-float 4.6.0", + "quanta", + "radix_trie", + "rand 0.9.5", + "rand_xoshiro", + "sketches-ddsketch", +] + [[package]] name = "mime" version = "0.3.17" @@ -5530,7 +5624,7 @@ dependencies = [ "cfg-if 1.0.4", "proc-macro2", "quote", - "syn 2.0.118", + "syn 2.0.119", ] [[package]] @@ -5572,7 +5666,7 @@ checksum = "e4db6d5580af57bf992f59068d4ea26fd518574ff48d7639b255a36f9de6e7e9" dependencies = [ "proc-macro2", "quote", - "syn 2.0.118", + "syn 2.0.119", ] [[package]] @@ -5604,7 +5698,7 @@ checksum = "4568f25ccbd45ab5d5603dc34318c1ec56b117531781260002151b8530a9f931" dependencies = [ "proc-macro2", "quote", - "syn 2.0.118", + "syn 2.0.119", ] [[package]] @@ -5639,6 +5733,15 @@ dependencies = [ "rawpointer", ] +[[package]] +name = "nibble_vec" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77a5d83df9f36fe23f0c3648c6bbb8b0298bb5f1939c8f2704431371f4b84d43" +dependencies = [ + "smallvec", +] + [[package]] name = "nix" version = "0.26.4" @@ -5797,7 +5900,7 @@ dependencies = [ "proc-macro-crate", "proc-macro2", "quote", - "syn 2.0.118", + "syn 2.0.119", ] [[package]] @@ -5885,9 +5988,9 @@ dependencies = [ [[package]] name = "object_store_opendal" -version = "0.57.0" +version = "0.58.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0eb12a624a41fce745838d0ef3701ff6c47797c13cd18ad3612fd2a3134fdbd8" +checksum = "88f165780495c17aa3ce86846600504198c3fffd99073521552751c2430fa6ac" dependencies = [ "async-trait", "bytes", @@ -5948,12 +6051,13 @@ checksum = "d6790f58c7ff633d8771f42965289203411a5e5c68388703c06e14f24770b41e" [[package]] name = "opendal" -version = "0.57.0" +version = "0.58.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "96c9c85ce253ff87225e7669979d877a20c98a06604ec9d6dd5f4473e08f1ae1" +checksum = "4f20562cc7447fcc915fc5c23df305a412ea80a733c9f2fd9e2d267e2815be6d" dependencies = [ "ctor 1.0.7", "opendal-core", + "opendal-http-transport-reqwest", "opendal-layer-concurrent-limit", "opendal-layer-logging", "opendal-layer-retry", @@ -5971,24 +6075,22 @@ dependencies = [ [[package]] name = "opendal-core" -version = "0.57.0" +version = "0.58.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c4f8607c90e2c963a91467f50fb49fbc7fb3d573f88cea219ca59ccd3740b309" +checksum = "ec75551ff4cf3e57da98979f6a937aaa9ddb3915bf68cc17d03df733be6646ed" dependencies = [ "anyhow", - "base64 0.22.1", + "base64 0.23.0", "bytes", "futures", "http 1.4.2", - "http-body 1.0.1", "jiff", "log", "md-5 0.11.0", "mea", "percent-encoding", - "quick-xml 0.39.4", + "quick-xml 0.41.0", "reqsign-core", - "reqwest 0.13.4", "serde", "serde_json", "tokio", @@ -5997,11 +6099,25 @@ dependencies = [ "web-time", ] +[[package]] +name = "opendal-http-transport-reqwest" +version = "0.58.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ad4d4f19c3ce01126a30611f8e544eaa217104a278c889ac17c9374fe4f9e4ef" +dependencies = [ + "bytes", + "futures", + "http 1.4.2", + "http-body 1.0.1", + "opendal-core", + "reqwest 0.13.4", +] + [[package]] name = "opendal-layer-concurrent-limit" -version = "0.57.0" +version = "0.58.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0d6f81ba6960e3fae1882f253b114b21d7e444e1534f209c7737a79f6243eb6f" +checksum = "249ac5b0aa5a7a6c3737342d10456067937f9c9a6f3f02544271f7908ab91081" dependencies = [ "futures", "http 1.4.2", @@ -6011,9 +6127,9 @@ dependencies = [ [[package]] name = "opendal-layer-logging" -version = "0.57.0" +version = "0.58.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "58ada45c6d81d1aa4c9305d0c7d4bc317c59c85866a0908a2d75a7a978aa5ee2" +checksum = "5c75411ab00f77851ff086b686c1e9ca8175ac18c15afa2cb75b9036436cb06c" dependencies = [ "log", "opendal-core", @@ -6021,9 +6137,9 @@ dependencies = [ [[package]] name = "opendal-layer-retry" -version = "0.57.0" +version = "0.58.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b2a25a718afb81fad81cb9a0580a1cb989221fa2317f888c6a37f8dad408eb7" +checksum = "80b7738bd5f233ad8da39af9b9316b9b7a4eaddd91e8e32a1e19b7030688121d" dependencies = [ "backon", "log", @@ -6032,9 +6148,9 @@ dependencies = [ [[package]] name = "opendal-layer-timeout" -version = "0.57.0" +version = "0.58.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e91f731724c213af81e9d03517859c8fc47b4578e64ad61ae4f099f10fe36e3" +checksum = "a704141924500f3803c05ed871b53305d2a2f11cb5ef20160c3ee688a1857f66" dependencies = [ "opendal-core", "tokio", @@ -6042,17 +6158,17 @@ dependencies = [ [[package]] name = "opendal-service-azblob" -version = "0.57.0" +version = "0.58.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0030644366ef5d8cbe3a4a5822bf99a4aafddc1666e9d24b44d158d9062fc76a" +checksum = "b3310fbbb48f111c6f590473c2cd15e1b7f8e384444b0d4e328f0464c864d767" dependencies = [ - "base64 0.22.1", + "base64 0.23.0", "bytes", "http 1.4.2", "log", "opendal-core", "opendal-service-azure-common", - "quick-xml 0.39.4", + "quick-xml 0.41.0", "reqsign-azure-storage", "reqsign-core", "reqsign-file-read-tokio", @@ -6063,17 +6179,18 @@ dependencies = [ [[package]] name = "opendal-service-azdls" -version = "0.57.0" +version = "0.58.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6dea4908d490143a9b0b7f7a790e139ff829b06a023f670455ed3d44f664b361" +checksum = "2e3c406729935fe214ce574d68681a1ff7e0b322548f14094912bdbfe50e5c53" dependencies = [ - "base64 0.22.1", + "base64 0.23.0", "bytes", "http 1.4.2", "log", + "mea", "opendal-core", "opendal-service-azure-common", - "quick-xml 0.39.4", + "quick-xml 0.41.0", "reqsign-azure-storage", "reqsign-core", "reqsign-file-read-tokio", @@ -6083,9 +6200,9 @@ dependencies = [ [[package]] name = "opendal-service-azure-common" -version = "0.57.0" +version = "0.58.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b489f13c42e69d69bdd72952b634356ec43a7881a20259b38b540fcecdf4051" +checksum = "7348c88edf15af435b7be930077746b569fac5e738c1bf6a363b675e7317c9df" dependencies = [ "http 1.4.2", "opendal-core", @@ -6093,15 +6210,15 @@ dependencies = [ [[package]] name = "opendal-service-cos" -version = "0.57.0" +version = "0.58.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aa8cafe9729213375c7331019b0cb756ad3e1aff7f45cd32c45eae91ebde8901" +checksum = "d533d4582105d269c8aebeee5f0e8bcf960f41b8aab6197df7012254d9f39bf0" dependencies = [ "bytes", "http 1.4.2", "log", "opendal-core", - "quick-xml 0.39.4", + "quick-xml 0.41.0", "reqsign-core", "reqsign-file-read-tokio", "reqsign-tencent-cos", @@ -6110,9 +6227,9 @@ dependencies = [ [[package]] name = "opendal-service-gcs" -version = "0.57.0" +version = "0.58.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "48de101aac565ed06af4b47903c24eafd249075553ec1fb18256751c45148d47" +checksum = "007f3fba63c21e516c956b891e96ff9892d8175662bfb781cdada9d3766a11e6" dependencies = [ "async-trait", "bytes", @@ -6120,7 +6237,7 @@ dependencies = [ "log", "opendal-core", "percent-encoding", - "quick-xml 0.39.4", + "quick-xml 0.41.0", "reqsign-core", "reqsign-file-read-tokio", "reqsign-google", @@ -6131,9 +6248,9 @@ dependencies = [ [[package]] name = "opendal-service-goosefs" -version = "0.57.0" +version = "0.58.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "69e43048bde419947ba826fbdc2f134d6c03f44ebf48bd33a03b72f9fc45fcb4" +checksum = "60871e6386f04d831e6a5bdbc032af4a91aeba49963252d0ef456a2cf36a9b78" dependencies = [ "bytes", "goosefs-sdk", @@ -6145,9 +6262,9 @@ dependencies = [ [[package]] name = "opendal-service-hf" -version = "0.57.0" +version = "0.58.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c4922661976a1d40794a2adfbdb888cc3c23097690f825a92f773af38908a848" +checksum = "b41fd41eb7ed03c5e66cefda61e8e117808ffd2908f2916737cb020a6beb02c7" dependencies = [ "bytes", "hf-xet", @@ -6155,22 +6272,21 @@ dependencies = [ "log", "opendal-core", "percent-encoding", - "reqwest 0.13.4", "serde", "serde_json", ] [[package]] name = "opendal-service-oss" -version = "0.57.0" +version = "0.58.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "328fa55e8888cbdfe00826bfea2a79042422b720e8369e9e021e46121dea5ace" +checksum = "cd528ec2d49c5ca69e674ffed7b3e0686fb9cfcfea0596870de381467fda4f1b" dependencies = [ "bytes", "http 1.4.2", "log", "opendal-core", - "quick-xml 0.39.4", + "quick-xml 0.41.0", "reqsign-aliyun-oss", "reqsign-core", "reqsign-file-read-tokio", @@ -6179,18 +6295,18 @@ dependencies = [ [[package]] name = "opendal-service-s3" -version = "0.57.0" +version = "0.58.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "313d46c9f5ae70bca26b7c3e3fbb9b639292625f28af73aa016f47e788af9deb" +checksum = "58e80cdf192d7eff05feed747894d64f81905ac4eaf132edf7ea270abdd2d663" dependencies = [ - "base64 0.22.1", + "base64 0.23.0", "bytes", - "crc32c", + "crc-fast", "http 1.4.2", "log", "md-5 0.11.0", "opendal-core", - "quick-xml 0.39.4", + "quick-xml 0.41.0", "reqsign-aws-v4", "reqsign-core", "reqsign-file-read-tokio", @@ -6200,14 +6316,14 @@ dependencies = [ [[package]] name = "opendal-service-tos" -version = "0.57.0" +version = "0.58.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6f2f7a4c32e5202eb4ac72e76c4b5e30c86ab60762811172f4111103b9d673a1" +checksum = "7841a1a09485bd08eeac34d67804321b62456c855027c580bce12a75564f4609" dependencies = [ "bytes", "http 1.4.2", "opendal-core", - "quick-xml 0.39.4", + "quick-xml 0.41.0", "reqsign-core", "reqsign-file-read-tokio", "reqsign-volcengine-tos", @@ -6237,7 +6353,7 @@ checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.118", + "syn 2.0.119", ] [[package]] @@ -6273,6 +6389,15 @@ dependencies = [ "num-traits", ] +[[package]] +name = "ordered-float" +version = "4.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7bb71e1b3fa6ca1c61f383464aaf2bb0e2f8e772a1f01d486832464de363b951" +dependencies = [ + "num-traits", +] + [[package]] name = "ordered-float" version = "5.3.0" @@ -6348,9 +6473,9 @@ dependencies = [ [[package]] name = "parquet" -version = "58.3.0" +version = "58.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5dafa7d01085b62a47dd0c1829550a0a36710ea9c4fe358a05a85477cec8a908" +checksum = "d298093b2dec60289dce0684c986d0f7679e9dd15771c2c65406e1aaf604a704" dependencies = [ "ahash", "arrow-array", @@ -6557,7 +6682,7 @@ checksum = "c96395f0a926bc13b1c17622aaddda1ecb55d49c8f1bf9777e4d877800a43f8b" dependencies = [ "proc-macro2", "quote", - "syn 2.0.118", + "syn 2.0.119", ] [[package]] @@ -6692,7 +6817,7 @@ dependencies = [ "nix", "once_cell", "smallvec", - "spin 0.10.0", + "spin 0.10.1", "symbolic-demangle", "tempfile", "thiserror 2.0.18", @@ -6750,7 +6875,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "479ca8adacdd7ce8f1fb39ce9ecccbfe93a3f1344b3d0d97f20bc0196208f62b" dependencies = [ "proc-macro2", - "syn 2.0.118", + "syn 2.0.119", ] [[package]] @@ -6764,9 +6889,9 @@ dependencies = [ [[package]] name = "proc-macro2" -version = "1.0.106" +version = "1.0.107" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8fd00f0bb2e90d81d1044c2b32617f68fcb9fa3bb7640c23e9c748e53fb30934" +checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9" dependencies = [ "unicode-ident", ] @@ -6781,7 +6906,7 @@ dependencies = [ "bit-vec", "bitflags 2.13.0", "num-traits", - "rand 0.9.4", + "rand 0.9.5", "rand_chacha 0.9.0", "rand_xorshift", "regex-syntax", @@ -6815,7 +6940,7 @@ dependencies = [ "prost", "prost-types", "regex", - "syn 2.0.118", + "syn 2.0.119", "tempfile", ] @@ -6829,7 +6954,7 @@ dependencies = [ "itertools 0.14.0", "proc-macro2", "quote", - "syn 2.0.118", + "syn 2.0.119", ] [[package]] @@ -6867,7 +6992,22 @@ checksum = "7347867d0a7e1208d93b46767be83e2b8f978c3dad35f775ac8d8847551d6fe1" dependencies = [ "proc-macro2", "quote", - "syn 2.0.118", + "syn 2.0.119", +] + +[[package]] +name = "quanta" +version = "0.12.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f3ab5a9d756f0d97bdc89019bd2e4ea098cf9cde50ee7564dde6b81ccc8f06c7" +dependencies = [ + "crossbeam-utils", + "libc", + "once_cell", + "raw-cpuid", + "wasi 0.11.1+wasi-snapshot-preview1", + "web-sys", + "winapi", ] [[package]] @@ -6887,33 +7027,45 @@ dependencies = [ [[package]] name = "quick-xml" -version = "0.38.4" +version = "0.39.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b66c2058c55a409d601666cffe35f04333cf1013010882cec174a7467cd4e21c" +checksum = "cdcc8dd4e2f670d309a5f0e83fe36dfdc05af317008fea29144da1a2ac858e5e" dependencies = [ "memchr", + "serde", ] [[package]] name = "quick-xml" -version = "0.39.4" +version = "0.40.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cdcc8dd4e2f670d309a5f0e83fe36dfdc05af317008fea29144da1a2ac858e5e" +checksum = "2474bd2e5029e7ccb6abb2ba48cf2383a333851dedf495901544281590c7da7f" dependencies = [ "memchr", - "serde", ] [[package]] name = "quick-xml" -version = "0.40.1" +version = "0.41.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2474bd2e5029e7ccb6abb2ba48cf2383a333851dedf495901544281590c7da7f" +checksum = "e660451e55124f798a69a5af3f49ccfbefbd41910eefd25caf2393e1f3473ec1" dependencies = [ "memchr", "serde", ] +[[package]] +name = "quick_cache" +version = "0.6.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9c6658afe513a3b484e3abfdaa0d03ef3c0bbf017542c178dd55f94eb3051f9" +dependencies = [ + "ahash", + "equivalent", + "hashbrown 0.16.1", + "parking_lot", +] + [[package]] name = "quinn" version = "0.11.9" @@ -6936,15 +7088,16 @@ dependencies = [ [[package]] name = "quinn-proto" -version = "0.11.14" +version = "0.11.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "434b42fec591c96ef50e21e886936e66d3cc3f737104fdb9b737c40ffb94c098" +checksum = "2f4bfc015262b9df63c8845072ce59068853ff5872180c2ce2f13038b970e560" dependencies = [ "aws-lc-rs", "bytes", - "getrandom 0.3.4", + "getrandom 0.4.3", "lru-slab", - "rand 0.9.4", + "rand 0.10.1", + "rand_pcg", "ring", "rustc-hash", "rustls", @@ -6972,9 +7125,9 @@ dependencies = [ [[package]] name = "quote" -version = "1.0.46" +version = "1.0.47" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dfbc457d0c7a0759a614551b11a6409e5951f6c7537be1f1b7682b9ae9230368" +checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001" dependencies = [ "proc-macro2", ] @@ -6997,6 +7150,16 @@ version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dc33ff2d4973d518d823d61aa239014831e521c75da58e3df4840d3f47749d09" +[[package]] +name = "radix_trie" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c069c179fcdc6a2fe24d8d18305cf085fdbd4f922c041943e203685d6a1c58fd" +dependencies = [ + "endian-type", + "nibble_vec", +] + [[package]] name = "rancor" version = "0.1.1" @@ -7019,9 +7182,9 @@ dependencies = [ [[package]] name = "rand" -version = "0.9.4" +version = "0.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "44c5af06bb1b7d3216d91932aed5265164bf384dc89cd6ba05cf59a35f5f76ea" +checksum = "b9ef1d0d795eb7d84685bca4f72f3649f064e6641543d3a8c415898726a57b41" dependencies = [ "rand_chacha 0.9.0", "rand_core 0.9.5", @@ -7089,7 +7252,16 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6a8615d50dcf34fa31f7ab52692afec947c4dd0ab803cc87cb3b0b4570ff7463" dependencies = [ "num-traits", - "rand 0.9.4", + "rand 0.9.5", +] + +[[package]] +name = "rand_pcg" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "caa0f4137e1c0a72f4c651489402276c8e8e1cf081f3b0ba156d2cbeef09e86a" +dependencies = [ + "rand_core 0.10.1", ] [[package]] @@ -7116,6 +7288,24 @@ version = "1.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "973443cf09a9c8656b574a866ab68dfa19f0867d0340648c7d2f6a71b8a8ea68" +[[package]] +name = "rapidhash" +version = "4.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32b266a82f4aa99bb5c25e28d11cc44ace63d91adbcbcee4d323e2ae3d49ef37" +dependencies = [ + "rustversion", +] + +[[package]] +name = "raw-cpuid" +version = "11.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "498cd0dc59d73224351ee52a95fee0f1a617a2eae0e7d9d720cc622c73a54186" +dependencies = [ + "bitflags 2.13.0", +] + [[package]] name = "rawpointer" version = "0.2.1" @@ -7199,7 +7389,7 @@ checksum = "b7186006dcb21920990093f30e3dea63b7d6e977bf1256be20c3563a5db070da" dependencies = [ "proc-macro2", "quote", - "syn 2.0.118", + "syn 2.0.119", ] [[package]] @@ -7264,9 +7454,9 @@ dependencies = [ [[package]] name = "reqsign-aliyun-oss" -version = "3.1.0" +version = "3.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "372266b4733756738eeb199a98188037d27a0989980e2600ae7ce1faf00a867d" +checksum = "9c0f9f69a519dd6958c4b43606bb8e1278cdc76d611fc8fed4b796eee548dc0f" dependencies = [ "anyhow", "form_urlencoded", @@ -7281,9 +7471,9 @@ dependencies = [ [[package]] name = "reqsign-aws-v4" -version = "3.0.1" +version = "3.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b75624bd8a466e37ddc0a7b6c33ac859a85347c153a916e1dd9d0b68338f74a" +checksum = "cc883bc56889f3e4a419265c87facea222a921debc5c6f15c7fd8b68ec4b36b2" dependencies = [ "anyhow", "bytes", @@ -7292,7 +7482,7 @@ dependencies = [ "http 1.4.2", "log", "percent-encoding", - "quick-xml 0.40.1", + "quick-xml 0.41.0", "reqsign-core", "rust-ini", "serde", @@ -7303,9 +7493,9 @@ dependencies = [ [[package]] name = "reqsign-azure-storage" -version = "3.0.1" +version = "3.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "62b96928e73ad984de1d99e382749d09e5dab7dd707b767974f7e40aa926b82f" +checksum = "a6ebd8524185ce9c64063e3095f83968acfa90922f00c601a4a0f3aca15b077e" dependencies = [ "anyhow", "base64 0.22.1", @@ -7324,14 +7514,13 @@ dependencies = [ [[package]] name = "reqsign-core" -version = "3.0.1" +version = "3.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a5fa5cb48808693614d1701fcd3db0b30fa292e0f18e122ae068b6d32eaeed3f" +checksum = "7e38b44697c60a823705ccef85cb04d8e0527c9d16ed7c58bf1c6395bdd24ceb" dependencies = [ "anyhow", "base64 0.22.1", "bytes", - "form_urlencoded", "futures", "hex", "hmac 0.13.0", @@ -7349,9 +7538,9 @@ dependencies = [ [[package]] name = "reqsign-file-read-tokio" -version = "3.0.1" +version = "3.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6a4b6f3a3fd29ffcc99a90aec585a65217783badfd73acddf847b63ae683bda9" +checksum = "688ff0ae421b8d4b92b53fdafaf53df2de28f428a9962edcf21702990b26f74b" dependencies = [ "anyhow", "reqsign-core", @@ -7360,9 +7549,9 @@ dependencies = [ [[package]] name = "reqsign-google" -version = "3.0.1" +version = "3.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eb215d0876a18b6bd9cdd380b589e5292aaa638ca15266de794b1122d898b6b2" +checksum = "a96da0b579b846d358090cb06b9e3c2ad1375529efbe3e0c45f96bd7bcf043ea" dependencies = [ "form_urlencoded", "http 1.4.2", @@ -7378,9 +7567,9 @@ dependencies = [ [[package]] name = "reqsign-tencent-cos" -version = "3.0.1" +version = "3.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "84110aabba799fbcd48b3abb51fbbff4749f879252e5806b6f5d0cbe0fef6abb" +checksum = "f6497dd9f6e3d1349b420521484099b284f95e8d3a65f088fccef42493a7b644" dependencies = [ "anyhow", "http 1.4.2", @@ -7393,9 +7582,9 @@ dependencies = [ [[package]] name = "reqsign-volcengine-tos" -version = "3.0.1" +version = "3.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "91d083a363b3577f519ce8425bb50f902622a28a83f7c4a26a5c990b66ec75b3" +checksum = "4335f949a3fd8b53867fd716dac97fbd545e45bcaed44343796517f2e19cd609" dependencies = [ "anyhow", "http 1.4.2", @@ -7532,9 +7721,9 @@ dependencies = [ [[package]] name = "rkyv" -version = "0.8.16" +version = "0.8.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "73389e0c99e664f919275ab5b5b0471391fe9a8de61e1dff9b1eaf56a90f16e3" +checksum = "815cc8a37159a463064825246cadb07961e25cd9885908606f6d08a98d8f8874" dependencies = [ "bytecheck", "bytes", @@ -7551,13 +7740,13 @@ dependencies = [ [[package]] name = "rkyv_derive" -version = "0.8.16" +version = "0.8.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5d2ed0b54125315fb36bd021e82d314d1c126548f871634b483f46b31d13cac6" +checksum = "c0ed1a78a1b19d184b0daa629dd9a024573173ec7d485b287cb369fb3607cc1c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.118", + "syn 2.0.119", ] [[package]] @@ -7633,7 +7822,7 @@ dependencies = [ "regex", "relative-path", "rustc_version", - "syn 2.0.118", + "syn 2.0.119", "unicode-ident", ] @@ -7647,16 +7836,6 @@ dependencies = [ "ordered-multimap", ] -[[package]] -name = "rust-stemmers" -version = "1.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e46a2036019fdb888131db7a4c847a1063a7493f971ed94ea82c67eada63ca54" -dependencies = [ - "serde", - "serde_derive", -] - [[package]] name = "rustc-demangle" version = "0.1.27" @@ -7665,9 +7844,9 @@ checksum = "b50b8869d9fc858ce7266cce0194bd74df58b9d0e3f6df3a9fc8eb470d95c09d" [[package]] name = "rustc-hash" -version = "2.1.2" +version = "2.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "94300abf3f1ae2e2b8ffb7b58043de3d399c73fa6f4b73826402a5c457614dbe" +checksum = "6b1e7f9a428571be2dc5bc0505c13fb6bf936822b894ec87abf8a08a4e51742d" [[package]] name = "rustc_version" @@ -7870,7 +8049,7 @@ dependencies = [ "proc-macro2", "quote", "serde_derive_internals", - "syn 2.0.118", + "syn 2.0.119", ] [[package]] @@ -7937,9 +8116,9 @@ checksum = "1bc711410fbe7399f390ca1c3b60ad0f53f80e95c5eb935e52268a0e2cd49acc" [[package]] name = "serde" -version = "1.0.228" +version = "1.0.229" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e" +checksum = "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba" dependencies = [ "serde_core", "serde_derive", @@ -7947,22 +8126,22 @@ dependencies = [ [[package]] name = "serde_core" -version = "1.0.228" +version = "1.0.229" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "41d385c7d4ca58e59fc732af25c3983b67ac852c1a25000afe1175de458b67ad" +checksum = "67dca2c9c51e58a4791a4b1ed58308b39c64224d349a935ab5039aa360942a48" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.228" +version = "1.0.229" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" +checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348" dependencies = [ "proc-macro2", "quote", - "syn 2.0.118", + "syn 3.0.3", ] [[package]] @@ -7973,15 +8152,16 @@ checksum = "18d26a20a969b9e3fdf2fc2d9f21eda6c40e2de84c9408bb5d3b05d499aae711" dependencies = [ "proc-macro2", "quote", - "syn 2.0.118", + "syn 2.0.119", ] [[package]] name = "serde_json" -version = "1.0.150" +version = "1.0.151" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e8014e44b4736ed0538adeecded0fce2a272f22dc9578a7eb6b2d9993c74cfb9" +checksum = "c841b55ecdae098c80dcae9cf767f6f8a0c2cdb3416bbef72181df4d0fe73f14" dependencies = [ + "indexmap 2.14.0", "itoa", "memchr", "serde", @@ -8008,7 +8188,7 @@ checksum = "175ee3e80ae9982737ca543e96133087cbd9a485eecc3bc4de9c1a37b47ea59c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.118", + "syn 2.0.119", ] [[package]] @@ -8020,7 +8200,7 @@ dependencies = [ "proc-macro2", "quote", "serde", - "syn 2.0.118", + "syn 2.0.119", ] [[package]] @@ -8064,7 +8244,7 @@ dependencies = [ "darling 0.23.0", "proc-macro2", "quote", - "syn 2.0.118", + "syn 2.0.119", ] [[package]] @@ -8115,7 +8295,7 @@ checksum = "94e153fc76e1c6a068703d6d29c508a0b15c061c4b7e43da59cc097bc342673c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.118", + "syn 2.0.119", ] [[package]] @@ -8246,6 +8426,12 @@ version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8ee5873ec9cce0195efcb7a4e9507a04cd49aec9c83d0389df45b1ef7ba2e649" +[[package]] +name = "sketches-ddsketch" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c6f73aeb92d671e0cc4dca167e59b2deb6387c375391bc99ee743f326994a2b" + [[package]] name = "slab" version = "0.4.12" @@ -8260,23 +8446,23 @@ checksum = "8ed6a63f02c8539c91a8685a86f4099661ba3da017932f6ebbea6de3f0fa7c90" [[package]] name = "snafu" -version = "0.9.1" +version = "0.9.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d1a012328be2e3f5d5f6f3218147ca02588cea4cb865e876849ab6debcf36522" +checksum = "e45cb604038abb7b926b679887b3226d8d0f23874b66623625a0454be425a4b7" dependencies = [ "snafu-derive", ] [[package]] name = "snafu-derive" -version = "0.9.1" +version = "0.9.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f103c50866b8743da9429b8a581d81a27c2d3a9c4ac7df8f8571c1dd7896eda" +checksum = "287f59010008f0d7cf5e3b03196d666c1acc46c8d3e9cf34c28a1a7157601e72" dependencies = [ "heck", "proc-macro2", "quote", - "syn 2.0.118", + "syn 2.0.119", ] [[package]] @@ -8314,15 +8500,15 @@ dependencies = [ [[package]] name = "spin" -version = "0.9.8" +version = "0.9.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67" +checksum = "3763264f6b73151db08c50ff20d7d8a0b8796e021cdea7ceedad07b80155fa0e" [[package]] name = "spin" -version = "0.10.0" +version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d5fe4ccb98d9c292d56fec89a5e07da7fc4cf0dc11e156b41793132775d3e591" +checksum = "023a211cb3138dbc438680b32560ad89f699977624c9f8dbb95a47d5b4c07dd3" dependencies = [ "lock_api", ] @@ -8351,9 +8537,9 @@ dependencies = [ [[package]] name = "sqlparser" -version = "0.61.0" +version = "0.62.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dbf5ea8d4d7c808e1af1cbabebca9a2abe603bcefc22294c5b95018d53200cb7" +checksum = "13c6d1b651dc4edf07eead2a0c6c78016ce971bc2c10da5266861b13f25e7cec" dependencies = [ "log", "sqlparser_derive", @@ -8367,7 +8553,7 @@ checksum = "a6dd45d8fc1c79299bfbb7190e42ccbbdf6a5f52e4a6ad98d92357ea965bd289" dependencies = [ "proc-macro2", "quote", - "syn 2.0.118", + "syn 2.0.119", ] [[package]] @@ -8431,35 +8617,13 @@ version = "0.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" -[[package]] -name = "strum" -version = "0.26.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8fec0f0aef304996cf250b31b5a10dee7980c85da9d759361292b8bca5a18f06" -dependencies = [ - "strum_macros 0.26.4", -] - [[package]] name = "strum" version = "0.28.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9628de9b8791db39ceda2b119bbe13134770b56c138ec1d3af810d045c04f9bd" dependencies = [ - "strum_macros 0.28.0", -] - -[[package]] -name = "strum_macros" -version = "0.26.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c6bee85a5a24955dc440386795aa378cd9cf82acd5f764469152d2270e581be" -dependencies = [ - "heck", - "proc-macro2", - "quote", - "rustversion", - "syn 2.0.118", + "strum_macros", ] [[package]] @@ -8471,16 +8635,17 @@ dependencies = [ "heck", "proc-macro2", "quote", - "syn 2.0.118", + "syn 2.0.119", ] [[package]] name = "substrait" -version = "0.62.2" +version = "0.63.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "62fc4b483a129b9772ccb9c3f7945a472112fdd9140da87f8a4e7f1d44e045d0" +checksum = "e620ff4d5c02fd6f7752931aa74b16a26af66a63022cc1ad412c77edbe0bab47" dependencies = [ "heck", + "indexmap 2.14.0", "pbjson", "pbjson-build", "pbjson-types", @@ -8494,7 +8659,7 @@ dependencies = [ "serde", "serde_json", "serde_yaml", - "syn 2.0.118", + "syn 2.0.119", "typify", "walkdir", ] @@ -8547,9 +8712,20 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.118" +version = "2.0.119" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1b9ae57f904213ebb649ce6895b8a66c66f0203b9319718f69a5612a065b1422" +checksum = "872831b642d1a07999a962a351ed35b955ea2cfc8f3862091e2a240a84f17297" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "syn" +version = "3.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53e9bae58849f64dfa4f5d5ae372c8341f7305f82a3868709269343628b659a3" dependencies = [ "proc-macro2", "quote", @@ -8573,7 +8749,7 @@ checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2" dependencies = [ "proc-macro2", "quote", - "syn 2.0.118", + "syn 2.0.119", ] [[package]] @@ -8630,7 +8806,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" dependencies = [ "fastrand", - "getrandom 0.4.3", + "getrandom 0.3.4", "once_cell", "rustix", "windows-sys 0.61.2", @@ -8661,7 +8837,7 @@ checksum = "c26ef8b00e4d382e59f6a8ddb3cd790b3a5bb29f21a358a9a69ea2f29f13f27b" dependencies = [ "proc-macro2", "quote", - "syn 2.0.118", + "syn 2.0.119", ] [[package]] @@ -8670,7 +8846,7 @@ version = "0.2.21" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "944ad38adcbb71eaa682c56bceeb079e4ca82b4b3edc2a0fde5cb297b77dac8d" dependencies = [ - "syn 2.0.118", + "syn 2.0.119", "test-log-core", ] @@ -8700,7 +8876,7 @@ checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" dependencies = [ "proc-macro2", "quote", - "syn 2.0.118", + "syn 2.0.119", ] [[package]] @@ -8711,7 +8887,7 @@ checksum = "ebc4ee7f67670e9b64d05fa4253e753e016c6c95ff35b89b7941d6b856dec1d5" dependencies = [ "proc-macro2", "quote", - "syn 2.0.118", + "syn 2.0.119", ] [[package]] @@ -8745,12 +8921,11 @@ dependencies = [ [[package]] name = "time" -version = "0.3.47" +version = "0.3.55" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "743bd48c283afc0388f9b8827b976905fb217ad9e647fae3a379a9283c4def2c" +checksum = "cdb87b95ec50ddfa440816d227a17b2ccbdda963a316a727fda0fc4334f7d134" dependencies = [ "deranged", - "itoa", "num-conv", "powerfmt", "serde_core", @@ -8760,15 +8935,15 @@ dependencies = [ [[package]] name = "time-core" -version = "0.1.8" +version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7694e1cfe791f8d31026952abf09c69ca6f6fa4e1a1229e18988f06a04a12dca" +checksum = "9e1c906769ad99c88eaa54e728060edef082f8e358ff32030cb7c7d315e81109" [[package]] name = "time-macros" -version = "0.2.27" +version = "0.2.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2e70e4c5a0e0a8a4823ad65dfe1a6930e4f4d756dcd9dd7939022b5e8c501215" +checksum = "7e689342a48d2ea927c87ea50cabf8594854bf940e9310208848d680d668ed85" dependencies = [ "num-conv", "time-core", @@ -8854,9 +9029,9 @@ dependencies = [ [[package]] name = "tokio" -version = "1.52.3" +version = "1.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8fc7f01b389ac15039e4dc9531aa973a135d7a4135281b12d7c1bc79fd57fffe" +checksum = "202caea871b69668250d242070849eb495be178ed697a3e98aebce5bc81a0bed" dependencies = [ "bytes", "libc", @@ -8877,7 +9052,7 @@ checksum = "385a6cb71ab9ab790c5fe8d67f1645e6c450a7ce006a33de03daa956cf70a496" dependencies = [ "proc-macro2", "quote", - "syn 2.0.118", + "syn 2.0.119", ] [[package]] @@ -8913,9 +9088,9 @@ dependencies = [ [[package]] name = "tokio-stream" -version = "0.1.18" +version = "0.1.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "32da49809aab5c3bc678af03902d4ccddea2a87d028d86392a4b1560c6906c70" +checksum = "a3d06f0b082ba57c26b79407372e57cf2a1e28124f78e9479fe80322cf53420b" dependencies = [ "futures-core", "pin-project-lite", @@ -8925,13 +9100,14 @@ dependencies = [ [[package]] name = "tokio-util" -version = "0.7.18" +version = "0.7.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ae9cec805b01e8fc3fd2fe289f89149a9b66dd16786abd8b19cfa7b48cb0098" +checksum = "494815d09bf52b5548659851081238f0ca39ff638363907596da739561c62c52" dependencies = [ "bytes", "futures-core", "futures-sink", + "libc", "pin-project-lite", "tokio", ] @@ -9109,7 +9285,7 @@ checksum = "7490cfa5ec963746568740651ac6781f701c9c5ea257c58e057f3ba8cf69e8da" dependencies = [ "proc-macro2", "quote", - "syn 2.0.118", + "syn 2.0.119", ] [[package]] @@ -9203,11 +9379,11 @@ checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" [[package]] name = "twox-hash" -version = "2.1.2" +version = "2.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ea3136b675547379c4bd395ca6b938e5ad3c3d20fad76e7fe85f9e0d011419c" +checksum = "8464ec13c3691491391d9fce00f6416c9a48e46972f72d7865688be2080192c9" dependencies = [ - "rand 0.9.4", + "rand 0.10.1", ] [[package]] @@ -9247,7 +9423,7 @@ dependencies = [ "semver", "serde", "serde_json", - "syn 2.0.118", + "syn 2.0.119", "thiserror 2.0.18", "unicode-ident", ] @@ -9265,7 +9441,7 @@ dependencies = [ "serde", "serde_json", "serde_tokenstream", - "syn 2.0.118", + "syn 2.0.119", "typify-impl", ] @@ -9399,9 +9575,9 @@ checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" [[package]] name = "uuid" -version = "1.23.4" +version = "1.24.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bf80a72845275afea99e7f2b434723d3bc7e38470fcd1c7ed39a599c73319a53" +checksum = "bf3923a6f5c4c6382e0b653c4117f48d631ea17f38ed86e2a828e6f7412f5239" dependencies = [ "getrandom 0.4.3", "js-sys", @@ -9536,7 +9712,7 @@ dependencies = [ "bumpalo", "proc-macro2", "quote", - "syn 2.0.118", + "syn 2.0.119", "wasm-bindgen-shared", ] @@ -9719,7 +9895,7 @@ checksum = "053e2e040ab57b9dc951b72c264860db7eb3b0200ba345b4e4c3b14f67855ddf" dependencies = [ "proc-macro2", "quote", - "syn 2.0.118", + "syn 2.0.119", ] [[package]] @@ -9730,7 +9906,7 @@ checksum = "3f316c4a2570ba26bbec722032c4099d8c8bc095efccdc15688708623367e358" dependencies = [ "proc-macro2", "quote", - "syn 2.0.118", + "syn 2.0.119", ] [[package]] @@ -10185,9 +10361,9 @@ checksum = "66fee0b777b0f5ac1c69bb06d361268faafa61cd4682ae064a171c16c433e9e4" [[package]] name = "xxhash-rust" -version = "0.8.16" +version = "0.8.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4d93c89cdc2d3a63c3ec48ffe926931bdc069eafa8e4402fe6d8f790c9d1e576" +checksum = "aee1b19627c7c60102ab80d3a9cbe18de90bfe03bfa6c3715447681f0e8c8af6" [[package]] name = "yansi" @@ -10214,7 +10390,7 @@ checksum = "de844c262c8848816172cef550288e7dc6c7b7814b4ee56b3e1553f275f1858e" dependencies = [ "proc-macro2", "quote", - "syn 2.0.118", + "syn 2.0.119", "synstructure", ] @@ -10235,7 +10411,7 @@ checksum = "1ae7f38b72ec2a254e2b87ef277cf2cd4fb97cbebf944faa6f33354da0867930" dependencies = [ "proc-macro2", "quote", - "syn 2.0.118", + "syn 2.0.119", ] [[package]] @@ -10255,7 +10431,7 @@ checksum = "11532158c46691caf0f2593ea8358fed6bbf68a0315e80aae9bd41fbade684a1" dependencies = [ "proc-macro2", "quote", - "syn 2.0.118", + "syn 2.0.119", "synstructure", ] @@ -10297,7 +10473,7 @@ checksum = "625dc425cab0dca6dc3c3319506e6593dcb08a9f387ea3b284dbd52a92c40555" dependencies = [ "proc-macro2", "quote", - "syn 2.0.118", + "syn 2.0.119", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 486aa1e56a2..c25ad8f1218 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -9,6 +9,7 @@ members = [ "rust/lance-file", "rust/lance-geo", "rust/lance-index", + "rust/lance-index-core", "rust/lance-io", "rust/lance-linalg", "rust/lance-namespace", @@ -32,7 +33,7 @@ resolver = "3" [workspace.package] -version = "9.0.0-beta.16" +version = "11.0.0-beta.6" edition = "2024" authors = ["Lance Devs "] license = "Apache-2.0" @@ -57,27 +58,28 @@ rust-version = "1.91.0" [workspace.dependencies] arc-swap = "1.7" libc = "0.2.176" -lance = { version = "=9.0.0-beta.16", path = "./rust/lance", default-features = false } -lance-arrow = { version = "=9.0.0-beta.16", path = "./rust/lance-arrow" } -lance-core = { version = "=9.0.0-beta.16", path = "./rust/lance-core" } -lance-datafusion = { version = "=9.0.0-beta.16", path = "./rust/lance-datafusion" } -lance-datagen = { version = "=9.0.0-beta.16", path = "./rust/lance-datagen" } -lance-derive = { version = "=9.0.0-beta.16", path = "./rust/lance-derive" } -lance-encoding = { version = "=9.0.0-beta.16", path = "./rust/lance-encoding" } -lance-file = { version = "=9.0.0-beta.16", path = "./rust/lance-file" } -lance-geo = { version = "=9.0.0-beta.16", path = "./rust/lance-geo" } -lance-index = { version = "=9.0.0-beta.16", path = "./rust/lance-index" } -lance-io = { version = "=9.0.0-beta.16", path = "./rust/lance-io", default-features = false } -lance-linalg = { version = "=9.0.0-beta.16", path = "./rust/lance-linalg" } -lance-namespace = { version = "=9.0.0-beta.16", path = "./rust/lance-namespace" } -lance-namespace-impls = { version = "=9.0.0-beta.16", path = "./rust/lance-namespace-impls" } +lance = { version = "=11.0.0-beta.6", path = "./rust/lance", default-features = false } +lance-arrow = { version = "=11.0.0-beta.6", path = "./rust/lance-arrow" } +lance-core = { version = "=11.0.0-beta.6", path = "./rust/lance-core" } +lance-datafusion = { version = "=11.0.0-beta.6", path = "./rust/lance-datafusion" } +lance-datagen = { version = "=11.0.0-beta.6", path = "./rust/lance-datagen" } +lance-derive = { version = "=11.0.0-beta.6", path = "./rust/lance-derive" } +lance-encoding = { version = "=11.0.0-beta.6", path = "./rust/lance-encoding" } +lance-file = { version = "=11.0.0-beta.6", path = "./rust/lance-file" } +lance-geo = { version = "=11.0.0-beta.6", path = "./rust/lance-geo" } +lance-index = { version = "=11.0.0-beta.6", path = "./rust/lance-index" } +lance-index-core = { version = "=11.0.0-beta.6", path = "./rust/lance-index-core" } +lance-io = { version = "=11.0.0-beta.6", path = "./rust/lance-io", default-features = false } +lance-linalg = { version = "=11.0.0-beta.6", path = "./rust/lance-linalg" } +lance-namespace = { version = "=11.0.0-beta.6", path = "./rust/lance-namespace" } +lance-namespace-impls = { version = "=11.0.0-beta.6", path = "./rust/lance-namespace-impls" } lance-namespace-datafusion = { version = "=7.0.0-beta.9", path = "./rust/lance-namespace-datafusion" } lance-namespace-reqwest-client = "0.8.6" -lance-select = { version = "=9.0.0-beta.16", path = "./rust/lance-select" } -lance-tokenizer = { version = "=9.0.0-beta.16", path = "./rust/lance-tokenizer" } -lance-table = { version = "=9.0.0-beta.16", path = "./rust/lance-table" } -lance-test-macros = { version = "=9.0.0-beta.16", path = "./rust/lance-test-macros" } -lance-testing = { version = "=9.0.0-beta.16", path = "./rust/lance-testing" } +lance-select = { version = "=11.0.0-beta.6", path = "./rust/lance-select" } +lance-tokenizer = { version = "=11.0.0-beta.6", path = "./rust/lance-tokenizer" } +lance-table = { version = "=11.0.0-beta.6", path = "./rust/lance-table" } +lance-test-macros = { version = "=11.0.0-beta.6", path = "./rust/lance-test-macros" } +lance-testing = { version = "=11.0.0-beta.6", path = "./rust/lance-testing" } approx = "0.5.1" # Note that this one does not include pyarrow arrow = { version = "58.0.0", optional = false, features = ["prettyprint"] } @@ -103,10 +105,15 @@ aws-sdk-s3 = { version = "1.38.0", default-features = false } half = { "version" = "2.1", default-features = false, features = [ "num-traits", "std", + "bytemuck", ] } -lance-bitpacking = { version = "=9.0.0-beta.16", path = "./rust/compression/bitpacking" } +lance-bitpacking = { version = "=11.0.0-beta.6", path = "./rust/compression/bitpacking" } bitpacking = "0.9" bitvec = "1" +blake3 = "1.8.5" +bytemuck = { version = "1", default-features = false, features = [ + "extern_crate_alloc", +] } bytes = "1.11.1" byteorder = "1.5" clap = { version = "4", features = ["derive"] } @@ -122,7 +129,7 @@ criterion = { version = "0.8.2", features = [ ] } crossbeam-queue = "0.3" crossbeam-skiplist = "0.1" -datafusion = { version = "53.0.0", default-features = false, features = [ +datafusion = { version = "54.0.0", default-features = false, features = [ "crypto_expressions", "datetime_expressions", "encoding_expressions", @@ -132,22 +139,22 @@ datafusion = { version = "53.0.0", default-features = false, features = [ "string_expressions", "unicode_expressions", ] } -datafusion-common = "53.0.0" -datafusion-functions = { version = "53.0.0", default-features = false, features = ["regex_expressions"] } -datafusion-sql = "53.0.0" -datafusion-expr = "53.0.0" -datafusion-ffi = "53.0.0" -datafusion-physical-expr = "53.0.0" -datafusion-physical-plan = "53.0.0" -datafusion-substrait = { version = "53.0.0", default-features = false } +datafusion-common = "54.0.0" +datafusion-functions = { version = "54.0.0", default-features = false, features = ["regex_expressions"] } +datafusion-sql = "54.0.0" +datafusion-expr = "54.0.0" +datafusion-ffi = "54.0.0" +datafusion-physical-expr = "54.0.0" +datafusion-physical-plan = "54.0.0" +datafusion-substrait = { version = "54.0.0", default-features = false } dirs = "6.0.0" either = "1.0" fst = { version = "0.4.7", features = ["levenshtein"] } -fsst = { version = "=9.0.0-beta.16", path = "./rust/compression/fsst" } +fsst = { version = "=11.0.0-beta.6", path = "./rust/compression/fsst" } futures = "0.3" geoarrow-array = "0.8" geoarrow-schema = "0.8" -geodatafusion = "0.4.0" +geodatafusion = "0.5.0" geo-traits = "0.3.0" geo-types = "0.7.16" http = "1.1.0" @@ -160,14 +167,16 @@ jieba-rs = { version = "0.10.0", default-features = false } jsonb = { version = "0.5.3", default-features = false, features = ["databend"] } libm = "0.2.15" log = "0.4" +metrics = { version = "0.24" } +metrics-util = { version = "0.19" } mockall = { version = "0.14.0" } mock_instant = { version = "0.6.0" } moka = { version = "0.12", features = ["future", "sync"] } ndarray = { version = "0.16.1", features = ["matrixmultiply-threading"] } num-traits = "0.2" object_store = { version = "0.13.2" } -opendal = { version = "0.57" } -object_store_opendal = { version = "0.57" } +opendal = { version = "0.58.1" } +object_store_opendal = { version = "0.58" } pin-project = "1.0" path_abs = "0.5" pprof = { version = "0.15.0", features = ["flamegraph"] } @@ -188,7 +197,6 @@ serde_json = { version = "1" } semver = "1.0" serial_test = "3" snafu = "0.9" -strum = "0.26" lindera = { version = "3.0.7" } tempfile = "3" test-log = { version = "0.2.15" } diff --git a/README.md b/README.md index 886fd70425e..08716c84ead 100644 --- a/README.md +++ b/README.md @@ -56,6 +56,17 @@ For more details, see the full [Lance format specification](https://lance.org/fo > [!TIP] > Lance is in active development and we welcome contributions. Please see our [contributing guide](https://lance.org/community/contributing/) for more information. +## File format stability + +Lance releases frequently because the SDKs, integrations, and performance work are moving quickly. This does not mean the Lance file format changes incompatibly in every release. The Lance file format is identified by the `data_storage_version` stored in each dataset, and stable storage versions are a long-term compatibility contract. + +* Once a dataset is written with a stable `data_storage_version`, future Lance releases will continue to support reading that storage version. +* SDK and API compatibility is separate from file format compatibility. SDK/API changes follow semantic versioning and are documented in the [migration guide](https://lance.org/guide/migration/). +* Older Lance releases may not understand file format versions introduced later. If you run mixed Lance versions, pin `data_storage_version` for deterministic writes. +* The `next` file format alias is unstable and should only be used for experimentation, never for production data. + +For production, write data with a stable `data_storage_version`. See the [format versioning guide](https://lance.org/format/file/versioning/) for the current compatibility matrix. + ## Quick Start **Installation** diff --git a/deny.toml b/deny.toml index 75b92e53447..aac7bd58058 100644 --- a/deny.toml +++ b/deny.toml @@ -85,6 +85,7 @@ ignore = [ { id = "RUSTSEC-2025-0119", reason = "`number_prefix` used by hf-hub in examples" }, { id = "RUSTSEC-2026-0194", reason = "`quick-xml` <0.41 pulled transitively via object_store (datafusion) and reqsign-aws-v4 (opendal); upstream must upgrade first" }, { id = "RUSTSEC-2026-0195", reason = "`quick-xml` <0.41 pulled transitively via object_store (datafusion) and reqsign-aws-v4 (opendal); upstream must upgrade first" }, + { id = "RUSTSEC-2026-0002", reason = "`lru` 0.12.x pulled transitively via goosefs-sdk; fixed in >=0.16.3, goosefs-sdk must upgrade first" }, ] # If this is true, then cargo deny will use the git executable to fetch advisory database. # If this is false, then it uses a built-in git library. diff --git a/docs/Makefile b/docs/Makefile index 4f70f375142..08f5108b770 100644 --- a/docs/Makefile +++ b/docs/Makefile @@ -6,7 +6,7 @@ help: @echo " make build - Build documentation (auto-installs deps)" @echo " make make-full-website - Assemble the full website from local repo checkouts" @echo " make clean-full-website - Remove generated website content" - @echo " Run `make make-full-website` before `make build` or `make serve` for the full multi-repo site" + @echo " Run 'make make-full-website' before 'make build' or 'make serve' for the full multi-repo site" @echo " Override any repo with env vars like LANCE_NAMESPACE_REPO=..., LANCE_SPARK_REPO=..., LANCE_RAY_REPO=..." @echo " make clean - Clean build artifacts" @echo " make check-links - Check for broken links" diff --git a/docs/README.md b/docs/README.md index 80092b157c7..8cb33f86387 100644 --- a/docs/README.md +++ b/docs/README.md @@ -1,6 +1,7 @@ # Lance Documentation -This directory contains the documentation for Lance, built with MkDocs and Material theme. +This directory contains the documentation for Lance, built with MkDocs and a +custom theme (`theme/`) implementing the Lance Docs design. ## Getting Started with uv @@ -62,5 +63,18 @@ uv sync --upgrade ## Project Structure - `src/` - Source markdown files for documentation +- `theme/` - Custom MkDocs theme implementing the Lance Docs design + (templates in `*.html`, styles/behaviour in `theme/assets/`) - `mkdocs.yml` - MkDocs configuration - `pyproject.toml` - Python project configuration (uv compatible) + +## Theme Notes + +- Light/dark mode follows `prefers-color-scheme`, is toggleable from the + header, and persists in `localStorage` (`ld-theme`). +- The GitHub star count is fetched from the public GitHub API and cached in + `localStorage` for an hour; the button degrades to a plain link offline. +- Search is a lightweight overlay (`/` or `Cmd/Ctrl+K`) over the standard + `search` plugin index — no external search dependencies. +- Mermaid diagrams render client-side; the library is loaded from a CDN only + on pages that contain a diagram. diff --git a/docs/clean-full-website.sh b/docs/clean-full-website.sh index db8013cd744..3ffa8ffe15b 100755 --- a/docs/clean-full-website.sh +++ b/docs/clean-full-website.sh @@ -15,6 +15,7 @@ rm -rf "$docs_src/integrations/duckdb" rm -rf "$docs_src/integrations/spark" rm -rf "$docs_src/integrations/ray" rm -rf "$docs_src/integrations/trino" +rm -rf "$docs_src/integrations/context" rm -f "$docs_src/community/project-specific/.pages" rm -rf "$docs_src/community/project-specific/lance" rm -f "$docs_src/community/project-specific/namespace.md" @@ -22,6 +23,7 @@ rm -f "$docs_src/community/project-specific/namespace-impls.md" rm -f "$docs_src/community/project-specific/ray.md" rm -f "$docs_src/community/project-specific/spark.md" rm -f "$docs_src/community/project-specific/trino.md" +rm -f "$docs_src/community/project-specific/context.md" cat > "$docs_src/format/.pages" <<'EOF' nav: @@ -37,9 +39,9 @@ cat > "$docs_src/integrations/.pages" <<'EOF' nav: - Overview: index.md - Apache DataFusion: datafusion.md - - PostgreSQL: https://github.com/lancedb/pglance + - PostgreSQL: https://github.com/lance-format/pglance - PyTorch: pytorch.md - - Tensorflow: tensorflow.md + - TensorFlow: tensorflow.md EOF mkdir -p "$docs_src/format/catalog/dir" diff --git a/docs/make-full-website.sh b/docs/make-full-website.sh index bb446a2f070..414b3831fb0 100755 --- a/docs/make-full-website.sh +++ b/docs/make-full-website.sh @@ -13,6 +13,7 @@ Override any repo path with the matching environment variable: LANCE_TRINO_REPO LANCE_DUCKDB_REPO LANCE_HUGGINGFACE_REPO + LANCE_CONTEXT_REPO Defaults: LANCE_NAMESPACE_REPO=$HOME/oss/lance-namespace LANCE_NAMESPACE_IMPLS_REPO=$HOME/oss/lance-namespace-impls @@ -21,6 +22,7 @@ Defaults: LANCE_TRINO_REPO=$HOME/oss/lance-trino LANCE_DUCKDB_REPO=$HOME/oss/lance-duckdb LANCE_HUGGINGFACE_REPO=$HOME/oss/lance-huggingface + LANCE_CONTEXT_REPO=$HOME/oss/lance-context EOF } @@ -60,6 +62,7 @@ ray_repo_input=${LANCE_RAY_REPO:-$HOME/oss/lance-ray} trino_repo_input=${LANCE_TRINO_REPO:-$HOME/oss/lance-trino} duckdb_repo_input=${LANCE_DUCKDB_REPO:-$HOME/oss/lance-duckdb} huggingface_repo_input=${LANCE_HUGGINGFACE_REPO:-$HOME/oss/lance-huggingface} +context_repo_input=${LANCE_CONTEXT_REPO:-$HOME/oss/lance-context} copy_docs_dir() { local source_dir="$1" @@ -134,6 +137,7 @@ ray_repo=$(resolve_repo_dir "$ray_repo_input") trino_repo=$(resolve_repo_dir "$trino_repo_input") duckdb_repo=$(resolve_repo_dir "$duckdb_repo_input") huggingface_repo=$(resolve_repo_dir "$huggingface_repo_input") +context_repo=$(resolve_repo_dir "$context_repo_input") "$script_dir/clean-full-website.sh" @@ -265,6 +269,12 @@ else warn_missing_repo "Lance Trino docs" "$trino_repo/docs/src" fi +if copy_docs_dir "$context_repo/docs/src" "$docs_src/integrations/context"; then + integration_entries+=(" - Lance Context: context") +else + warn_missing_repo "Lance Context docs" "$context_repo/docs/src" +fi + { echo "nav:" for entry in "${integration_entries[@]}"; do @@ -305,6 +315,10 @@ if copy_file_if_exists "$trino_repo/CONTRIBUTING.md" "$docs_src/community/projec project_entries+=(" - Lance Trino: trino.md") fi +if copy_file_if_exists "$context_repo/CONTRIBUTING.md" "$docs_src/community/project-specific/context.md"; then + project_entries+=(" - Lance Context: context.md") +fi + { echo "nav:" for entry in "${project_entries[@]}"; do diff --git a/docs/mkdocs.yml b/docs/mkdocs.yml index 8144a42e5f5..de75e65f5d9 100644 --- a/docs/mkdocs.yml +++ b/docs/mkdocs.yml @@ -6,37 +6,13 @@ docs_dir: src repo_name: lance-format/lance repo_url: https://github.com/lance-format/lance +# Custom theme implementing the "Lance Docs" design (see theme/). theme: - name: material - custom_dir: overrides - logo: logo/white.png - favicon: logo/logo.png - palette: - - scheme: default - primary: custom - accent: custom - toggle: - icon: material/brightness-7 - name: Switch to dark mode - - scheme: slate - primary: custom - accent: custom - toggle: - icon: material/brightness-4 - name: Switch to light mode - features: - - navigation.tabs - - navigation.sections - - navigation.instant - - navigation.indexes - - navigation.tracking - - navigation.top - - search.highlight - - search.share - - content.code.copy - - content.code.annotate - icon: - repo: fontawesome/brands/github + name: null + custom_dir: theme + locale: en + static_templates: + - 404.html markdown_extensions: - admonition @@ -51,9 +27,16 @@ markdown_extensions: line_spans: __span pygments_lang_class: true - pymdownx.inlinehilite - - pymdownx.snippets + - pymdownx.snippets: + # Allow snippets to pull in files from the repo root (e.g. shared docs + # kept alongside Rust source). Paths are relative to the docs/ dir. + base_path: + - . + - .. - pymdownx.tabbed: alternate_style: true + # Autolink bare URLs (e.g. repository tables on the community pages). + - pymdownx.magiclink - attr_list - md_in_html - tables @@ -66,18 +49,4 @@ plugins: - mkdocs_protobuf: proto_dir: ../protos -extra: - generator: false - social: - - icon: fontawesome/brands/github - link: https://github.com/lance-format/lance - - icon: fontawesome/brands/discord - link: https://discord.gg/lance - copyright: © 2025 Lance Format. All rights reserved. - -extra_css: - - assets/stylesheets/home.css -extra_javascript: - - assets/javascripts/nav-expand.js - diff --git a/docs/overrides/home.html b/docs/overrides/home.html deleted file mode 100644 index 26eb288aee6..00000000000 --- a/docs/overrides/home.html +++ /dev/null @@ -1,295 +0,0 @@ -{% extends "main.html" %} - -{% block tabs %} - {{ super() }} - - - - - - -
-
-
- -

The Open Lakehouse Format for Multimodal AI

-
- -
-
-
- - -
-
-
-

What is Lance?

-

- Lance is a modern, open source lakehouse format for multimodal AI. It contains a file format, table format, and catalog spec, - allowing you to build a complete open lakehouse on top of object storage to power your AI workflows. - Lance brings high-performance vector search, full-text search, random access, and feature - engineering capabilities to the lakehouse, while you can still get all the existing lakehouse benefits - like SQL analytics, ACID transactions, time travel, and integrations with open engines (Apache Spark, Ray, PyTorch, Trino, DuckDB, etc.) - and open catalogs (Apache Polaris, Unity Catalog, Apache Gravitino, Hive Metastore, etc.) -

-

- Learn more about Lance's technical details by reading our - research paper - published at VLDB 2025. -

- Read the Docs -
-
-
- - -
-
-
-
-

Expressive Hybrid Search

-

- Lance enables powerful hybrid search combining vector similarity, full-text search, - and SQL analytics on the same dataset. All query types are accelerated by corresponding - secondary indexes as part of the Lance specification. -

-

- Run semantic search on embeddings, BM25 search on keywords, and apply complex SQL predicates - - all using a single table with a unified interface. -

- Learn More -
-
- Hybrid Search Example -
-
-
-
- - -
-
-
-
-

Lightning-fast Random Access

-

- Lance delivers 100x faster random access compared to Parquet or Iceberg. - Unlike traditional formats, Lance maintains high performance even when - randomly accessing scattered rows across your entire dataset. -

-

- With a highly optimized file format plus efficient row-addressing and secondary indexes at table level, - you can access individual records across multiple files instantly, - making it perfect for real-time ML serving, random sampling, and interactive applications. -

- Learn More -
-
- Random Access Example -
-
-
-
- - -
-
-
-
-

Native Multimodal Data Support

-

- Store images, videos, audio, text, and embeddings alongside your traditional tabular data in a single unified format. - Lance's blob encoding efficiently handles large binary objects with lazy loading, - while optimized vector storage accelerates similarity search. -

-

- Perfect for AI/ML workloads where you need to store raw data, ML features, generated captions and embeddings - all together for multimodal retrieval and genAI workflows. -

- Learn More -
-
- Multimodal Data Example -
-
-
-
- - -
-
-
-
-

Data Evolution > Schema Evolution

-

- Schema evolution in most open table formats are metadata only and fast. - But when trying to backfill column values in existing rows, a full table rewrite is typically required. - Lance supports data evolution (efficient schema evolution with backfill), making it perfect for ML - feature engineering, embedding and media content management. -

-

- Adding a new column with data is as simple as writing new Lance files to the Lance table - - no need to rewrite your entire dataset. -

- Learn More -
-
- Data Evolution Example -
-
-
-
- - -
-
-
-
-

Rich Ecosystem Integrations

-

- As an open format, Lance integrates seamlessly with the Python data ecosystem and modern data platforms. - Work with your favorite tools including Pandas, Polars, Ray and PyTorch for data processing and machine learning. -

-

- Connect with leading query engines like Apache DataFusion, DuckDB, Apache Spark, Trino, and Apache Flink/Fluss - to run SQL analytics and distributed processing on your Lance datasets. -

- View Integrations -
-
- Lance Ecosystem Integrations -
-
-
-
- - -{% endblock %} - -{% block content %}{% endblock %} -{% block footer %} - {{ super() }} -{% endblock %} diff --git a/docs/pyproject.toml b/docs/pyproject.toml index 4112230aec5..3aa6847e4f3 100644 --- a/docs/pyproject.toml +++ b/docs/pyproject.toml @@ -6,10 +6,12 @@ readme = "README.md" requires-python = ">=3.10,<3.11" dependencies = [ "mkdocs>=1.5.0", - "mkdocs-material>=9.4.0", + "pymdown-extensions>=10.0", + "pygments>=2.16", "mkdocs-protobuf>=0.1.0", "mkdocs-linkcheck>=1.0.0", - "mkdocs-awesome-pages-plugin>=2.10.1" + "mkdocs-awesome-pages-plugin>=2.10.1", + "requests>=2.31.0" # mkdocs-linkcheck imports it but doesn't declare it ] [tool.uv] diff --git a/docs/src/assets/javascripts/nav-expand.js b/docs/src/assets/javascripts/nav-expand.js deleted file mode 100644 index 17171f5a295..00000000000 --- a/docs/src/assets/javascripts/nav-expand.js +++ /dev/null @@ -1,16 +0,0 @@ -// Auto-expand sidebar navigation to 2 levels on page load. -// Level 1 sections are already expanded by navigation.sections. -// This expands level 2 (e.g. Operations, Models become visible) -// but leaves level 3+ collapsed. -document.addEventListener("DOMContentLoaded", function () { - // In mkdocs-material with navigation.sections, the top-level items - // are rendered as non-collapsible sections. The collapsible items - // start at the next level. We want to expand one more level. - // - // Selector: inside the primary nav, find toggle checkboxes that are - // exactly 2 nesting levels deep (the second-level sections). - var toggles = document.querySelectorAll( - ".md-sidebar--primary .md-nav--primary > .md-nav__list > .md-nav__item > .md-nav > .md-nav__list > .md-nav__item > .md-nav__toggle" - ); - toggles.forEach(function (t) { t.checked = true; }); -}); diff --git a/docs/src/assets/stylesheets/home.css b/docs/src/assets/stylesheets/home.css deleted file mode 100644 index eafdd5afbf9..00000000000 --- a/docs/src/assets/stylesheets/home.css +++ /dev/null @@ -1,244 +0,0 @@ -/* Lance Homepage Styles */ - -/* Override with custom color #625EFF site-wide */ -:root > * { - --md-primary-fg-color: #625EFF; - --md-primary-fg-color--light: #8481FF; - --md-primary-fg-color--dark: #4A46CC; - --md-accent-fg-color: #625EFF; - --md-accent-fg-color--transparent: rgba(98, 94, 255, 0.1); -} - -* { - box-sizing: border-box; -} - -.container { - width: 100%; - max-width: 1140px; - margin-right: auto; - margin-left: auto; - padding-right: 15px; - padding-left: 15px; -} - -/* Hero Section - Fullscreen with Background Image */ -.mdx-container { - text-align: center; - color: #f8f8f8; - background: url("../images/lance-mj.png") no-repeat center center; - background-size: cover; - min-height: 100vh; - height: 100vh; - display: flex; - align-items: center; - justify-content: center; -} - -.intro-message { - position: relative; - padding: 40px 20px; - font-family: "Lato", -apple-system, BlinkMacSystemFont, "Segoe UI", Helvetica, Arial, sans-serif; - max-width: 1000px; - margin: 0 auto; -} - -.hero-logo { - display: inline-flex; - align-items: center; - margin-bottom: 16px; -} - -.hero-logo img { - height: 120px; - width: auto; - margin-right: 24px; - margin-top: 12px; - filter: drop-shadow(3px 3px 8px rgba(0, 0, 0, 0.9)); -} - -.intro-message h1 { - font-weight: 400; - margin: 0; - display: inline-block; - text-shadow: 3px 3px 8px rgba(0, 0, 0, 0.9), 1px 1px 3px rgba(0, 0, 0, 1); - font-size: 8em; - line-height: 1.2; - color: #ffffff; - vertical-align: middle; -} - -.intro-message h1 sup { - font-size: 2rem; - text-shadow: 2px 2px 6px rgba(0, 0, 0, 0.9); -} - -.intro-message h3 { - font-size: 1.1rem; - text-shadow: 2px 2px 6px rgba(0, 0, 0, 0.9), 1px 1px 3px rgba(0, 0, 0, 1); - font-weight: 600; - margin-bottom: 32px; - color: #ffffff; -} - -.intro-divider { - width: 400px; - max-width: 80%; - border-top: 1px solid rgba(255, 255, 255, 0.8); - border-bottom: 1px solid rgba(0, 0, 0, 0.2); - margin: 24px auto; -} - -.list-inline { - padding-left: 0; - margin-left: -5px; - list-style: none; - margin-bottom: 0; -} - -.list-inline li { - display: inline-block; - padding-right: 5px; - padding-left: 5px; -} - -.intro-message .md-button { - margin: 8px; - padding: 14px 36px; - font-size: 1.1rem; - font-weight: 600; - text-shadow: 2px 2px 4px rgba(0, 0, 0, 0.8); - transition: all 0.3s ease; -} - -.intro-message .md-button:hover { - transform: translateY(-2px); - box-shadow: 0 8px 16px rgba(0, 0, 0, 0.3); -} - -.intro-message .md-button--primary:hover { - box-shadow: 0 8px 20px rgba(98, 94, 255, 0.5); -} - -.intro-message .md-button:not(.md-button--primary):hover { - box-shadow: 0 8px 20px rgba(255, 255, 255, 0.4); -} - -/* What is Lance Section */ -.lance-intro-section { - padding: 80px 0; - background-color: rgba(128, 128, 128, 0.03); - border-bottom: 1px solid rgba(128, 128, 128, 0.1); -} - -.lance-intro-content { - max-width: 900px; - margin: 0 auto; - text-align: center; -} - -.lance-intro-content h2 { - font-size: 36px; - font-weight: 500; - margin-bottom: 32px; - color: var(--md-primary-fg-color); -} - -.lance-intro-content p { - font-size: 16px; - line-height: 1.8; - margin-bottom: 32px; - opacity: 0.9; - text-align: left; -} - -.lance-paper-link { - color: var(--md-primary-fg-color); - text-decoration: none; -} - -.lance-paper-link:hover { - color: var(--md-primary-fg-color); - text-decoration: none; -} - -.lance-intro-content a:hover { - color: #757575; - text-decoration: none; -} - -.lance-intro-content .md-button { - margin-top: 16px; - padding: 10px 28px; - font-size: 14px; - border: 2px solid currentColor; - background-color: transparent; - transition: all 0.3s ease; -} - -.lance-intro-content .md-button:hover { - color: var(--md-primary-fg-color); - background-color: transparent; -} - -/* Feature Sections */ -.lance-feature-section { - padding: 80px 0; - border-bottom: 1px solid rgba(128, 128, 128, 0.1); -} - -.lance-feature-section:last-child { - border-bottom: none; -} - -.lance-feature-content { - display: flex; - flex-wrap: wrap; - align-items: center; - gap: 60px; -} - -.lance-feature-text { - flex: 1; - min-width: 300px; -} - -.lance-feature-text h2 { - font-size: 30px; - font-weight: 500; - margin-bottom: 16px; - color: var(--md-primary-fg-color); -} - -.lance-feature-text p { - font-size: 15px; - line-height: 1.6; - opacity: 0.85; - margin-bottom: 16px; -} - -.lance-feature-text .md-button { - font-size: 0.6rem; - padding: 0; - transition: all 0.3s ease; -} - -.lance-feature-text .md-button:hover { - transform: translateX(4px); - color: var(--md-primary-fg-color); -} - -.lance-feature-demo { - flex: 1; - min-width: 400px; - display: flex; - justify-content: center; - overflow: hidden; -} - -/* Alternating layout */ -.lance-feature-section.reverse .lance-feature-content { - flex-direction: row-reverse; -} - - diff --git a/docs/src/community/index.md b/docs/src/community/index.md index 4bf1ca30842..5b9c9bdcbe4 100644 --- a/docs/src/community/index.md +++ b/docs/src/community/index.md @@ -72,7 +72,7 @@ Here is the list of current subprojects: | lance-huggingface | https://github.com/lance-format/lance-huggingface | Hugging Face integration for Lance | | lance-namespace | https://github.com/lance-format/lance-namespace | Lance namespace format specification, Rust/Python/Java Codegen SDKs | | lance-namespace-impls | https://github.com/lance-format/lance-namespace-impls | Lance Namespace Implementations - Apache Hive, Apache Polaris, Apache Gravitino, Unity Catalog, AWS Glue and more | -| lance-python-docs | https://github.com/lance-format/lance-python-docs | Lance Python SDK generated docs and integration hook with readthedocs | +| lance-python-doc | https://github.com/lance-format/lance-python-doc | Lance Python SDK generated docs and integration hook with readthedocs | | lance-ray | https://github.com/lance-format/lance-ray | Ray integration for Lance | | lance-spark | https://github.com/lance-format/lance-spark | Apache Spark connector for Lance | diff --git a/docs/src/format/CLAUDE.md b/docs/src/format/CLAUDE.md new file mode 120000 index 00000000000..47dc3e3d863 --- /dev/null +++ b/docs/src/format/CLAUDE.md @@ -0,0 +1 @@ +AGENTS.md \ No newline at end of file diff --git a/docs/src/format/file/encoding.md b/docs/src/format/file/encoding.md index 4ca053d4fa6..1cb83d581e9 100644 --- a/docs/src/format/file/encoding.md +++ b/docs/src/format/file/encoding.md @@ -327,6 +327,148 @@ The protobuf for the full zip layout describes the compression of the data buffe size of the control words and how many bits we have per value (for fixed-width data) or how many bits we have per offset (for variable-width data). +### Sparse Page Layout + +Sparse pages require Lance 2.3. They represent flat or nested Arrow structure directly as slot-domain mappings instead +of dense repetition and definition events. Writers emit this layout only in files declared as 2.3 or above. The layout is +identified only by `PageLayout`; field metadata does not identify the layout of an existing page. + +A domain is a layer-local integer coordinate space `[0, num_slots)`, and a slot is one element in that space. The +outer-most domain contains the page's top-level rows. Each layer maps its parent domain to the next layer's parent +domain, and the terminal child domain contains the leaf value slots stored in value chunks. + +Structural layers are ordered from outer-most to inner-most: + +- validity maps a nullable item or struct slot to valid or null +- list maps non-empty parent slots to variable-size child ranges +- fixed-size-list maps each parent slot to a child range of a fixed dimension + +The layer list may be empty for a flat, non-nullable leaf page. In that case the scheduling domain and +`num_visible_items` must be equal. The explicit writer currently emits its normalized all-valid layer even when a flat +page could use this shorter wire representation. + +A list slot that is valid and absent from `non_empty_positions` is an empty list. Maps use the same structural contract +as lists. The terminal child-domain size equals `SparseLayout.num_visible_items`. + +`SparseLayout.num_visible_items` is the number of leaf value slots encoded in value chunks. Null leaf slots count +because they still occupy positions in Arrow's leaf value buffer; a nullable primitive with 100 slots, including 30 +nulls, has 100 visible items. `SparseLayout.num_items` is the number of entries in the equivalent dense repetition and +definition stream. It equals `num_visible_items` plus one structural placeholder for every list slot without children. +The first layer's `num_slots` is the logical top-level row count used for projection. + +Position sets have four semantic representations: `empty`, `all`, one non-empty `range`, or an `explicit` +delta-compressed `u64` buffer. Count sets are `empty`, one positive `constant` value, or an `explicit` compressed `u64` +buffer. Every layer has a `SparseValiditySet` whose meaning is explicit: + +- `SPARSE_VALIDITY_NULL_POSITIONS`: stored positions are null and all other positions are valid +- `SPARSE_VALIDITY_VALID_POSITIONS`: stored positions are valid and all other positions are null + +The unspecified validity meaning is invalid. Both polarities are part of the wire contract and have identical Arrow +semantics after normalization. + +#### Writer Selection + +Writers may emit this layout only for Lance 2.3+ fields. A field can request it explicitly with +`lance-encoding:structural-encoding=sparse`; the same request is an input error for earlier file versions. Without an +explicit structural encoding, the Lance 2.3 writer selects sparse only when the dense mini-block repetition/definition +budget would split the page or one top-level row exceeds that budget, and only when the value path is supported by the +sparse writer. Explicit `miniblock`, `fullzip`, and `sparse` requests are not changed by this automatic policy. Lance +2.2 and earlier writers never select sparse. + +Unsupported sparse value paths, including dictionary values and variable-width packed structs, retain their dense +behavior. Writers normalize Arrow validity and list structure once. Within-budget dense pages do not build sparse +position/count plans. All-valid layers use null positions plus `empty`; all-null layers use valid positions plus +`empty`. Other layers choose the validity polarity with the lower semantic encoded cost, with ties using null +positions. Field metadata controls writer selection only: readers always use `PageLayout` to determine the layout of +an encoded page and must not use field metadata for that decision. + +Pages without a value payload keep the existing canonical `ConstantLayout`: structural-only types such as an empty +struct, and leaf pages whose visible values are all null, do not emit `SparseLayout`. An explicitly sparse page with +at least one non-null visible value does emit `SparseLayout`, even when all non-null values are equal. This boundary +avoids introducing a second structural-only representation without evidence that it improves the existing constant +encoding. + +#### Buffers and Selective Reads + +A sparse page contains the following physical buffers: + +| Buffer | Contents | +| ------ | -------- | +| 0 | Value chunk metadata, one 8-byte entry per chunk | +| 1 | Mini-block compressed value chunks without repetition or definition levels | +| 2+ | One buffer for each explicit position or count set, in structural-layer field order | + +Each value chunk metadata entry stores `(chunk_size / 8) - 1` as little-endian `u32`, followed by its visible value +count as little-endian `u32`. Chunk sizes must be positive multiples of 8 and fit this representation. The sum of +chunk sizes must equal buffer 1 exactly and the sum of chunk value counts must equal `num_visible_items`. A value chunk +contains at most 32,768 visible values. `num_buffers` describes the number of value buffers inside every chunk and +excludes the structural buffers. + +General-compressed sparse buffers use the existing length-prefixed LZ4 or Zstd representation and must not contain +another general-compression wrapper. SparseLayout does not impose additional size or descriptor-complexity limits on +otherwise representable buffers. + +Readers normalize structural metadata once, project requested top-level ranges through each layer, and read only value +chunks that intersect the resulting leaf ranges. When no leaf range remains, readers rebuild offsets and validity from +the structural plan without reading buffer 1. + +#### Caching and Point Reads + +Reader initialization loads buffer 0 and every explicit structural buffer, then validates and normalizes them into a +cached page plan. The cached state contains parsed value-chunk descriptors and prefix offsets, decoded semantic +position/count sets, validity, and the ordered structural layers. It does not contain value payload bytes from buffer +1. The plan is cached per field and page and reused by later scans, range reads, and takes. + +After that plan is cached, reading one primitive leaf value reads only the value chunk that contains it. A cold read +first loads the structural metadata and then the intersecting value chunk. Reading one top-level list or +fixed-size-list value may intersect multiple leaf chunks and reads each intersecting chunk. A selection whose projected +structure contains no leaf slots reads no value chunk. + +#### Validation + +Readers must reject malformed sparse metadata instead of inferring or repairing it. Required checks include: + +- physical buffer count, chunk-count bounds, and every checked offset/size range +- first-layer row domain, adjacent parent/child domain chaining, and terminal visible-value domain +- semantic set cardinality, explicit position ordering and bounds, and validity meaning +- exact `num_items` +- list non-empty positions being valid, count cardinality, positive counts, and child-count sum +- fixed-size-list dimension and checked child-domain multiplication +- value chunk byte/value sums, size representation and alignment, general-compression headers, descriptor buffer + count, and complete chunk consumption + +```protobuf +%%% proto.message.SparseLayout %%% +``` + +```protobuf +%%% proto.message.SparseStructuralLayer %%% +``` + +```protobuf +%%% proto.message.SparseValidityLayer %%% +``` + +```protobuf +%%% proto.message.SparseListLayer %%% +``` + +```protobuf +%%% proto.message.SparseFixedSizeListLayer %%% +``` + +```protobuf +%%% proto.message.SparseValiditySet %%% +``` + +```protobuf +%%% proto.message.SparsePositionSet %%% +``` + +```protobuf +%%% proto.message.SparseCountSet %%% +``` + ### Constant Page Layout This layout is used when all (visible) values in the page are the same scalar value. @@ -549,7 +691,7 @@ options. However, they can also be set in the field metadata in the schema. | `lance-encoding:dict-values-compression-level` | Integers (scheme dependent) | Varies by scheme | Compression level for dictionary values general compression | | `lance-encoding:general` | `off`, `on` | `off` | Whether to apply general compression. | | `lance-encoding:packed` | Any string | Not set | Whether to apply packed struct encoding (see above). | -| `lance-encoding:structural-encoding` | `miniblock`, `fullzip` | Not set | Force a particular structural encoding to be applied (only useful for testing purposes) | +| `lance-encoding:structural-encoding` | `miniblock`, `fullzip`, `sparse` | Not set | Force a structural encoding; `sparse` requires Lance 2.3. | ### Configuration Details diff --git a/docs/src/format/file/versioning.md b/docs/src/format/file/versioning.md index 2add82ef2fa..8024a46fbc8 100644 --- a/docs/src/format/file/versioning.md +++ b/docs/src/format/file/versioning.md @@ -5,10 +5,10 @@ major number is changed when the file format itself is modified while the minor strategy is modified. Newer versions will typically have better performance and compression but may not be readable by older versions of Lance. -In addition, the `next` alias points to an unstable format version and should not be used for production use cases. -Breaking changes could be made to unstable encodings and that would mean that files written with these encodings are -no longer readable by any newer versions of Lance. The `next` version should only be used for experimentation and -benchmarking upcoming features. +Any version explicitly labeled unstable, including the current 2.3 format and the `next` alias, should not be used for +production use cases. Unstable formats have no compatibility guarantee: breaking encoding changes may make files +written by one Lance build unreadable by later builds. They should only be used for experimentation and benchmarking +upcoming features. The `stable` and `next` aliases are resolved by the specific Lance release you are using. During a format rollout (for example, 2.3), prefer explicit version pinning for deterministic behavior across environments. @@ -21,7 +21,7 @@ The following values are supported: | 2.0 | 0.16.0 | Any | Rework of the Lance file format that removed row groups and introduced null support for lists, fixed size lists, and primitives | | 2.1 | 0.38.1 | Any | Enhances integer and string compression, adds support for nulls in struct fields, and improves random access performance with nested fields. | | 2.2 | None | Any | Adds support for newer nested type/encoding capabilities (including map support) and 2.2-era storage features. | -| 2.3 (unstable) | None | Any | Adds experimental encodings for upcoming features. | +| 2.3 (unstable) | None | Unspecified | Adds sparse structural pages and other experimental encodings. | | legacy | N/A | N/A | Alias for 0.1 | | stable | N/A | N/A | Alias for the default version for new datasets in the Lance release you are running. | | next | N/A | N/A | Alias for the latest unstable version in the Lance release you are running.| diff --git a/docs/src/format/index/index.md b/docs/src/format/index/index.md index 4a2e33c60a4..af8fdf595b4 100644 --- a/docs/src/format/index/index.md +++ b/docs/src/format/index/index.md @@ -177,7 +177,7 @@ or updated. These should be filtered out during query execution. -There are three situations to consider: +There are four situations to consider: 1. **A fragment has some deleted rows.** A few of the rows in the fragment have been marked as deleted, but some of the rows are still present. The row addresses from the deletion @@ -188,6 +188,17 @@ There are three situations to consider: 3. **A fragment has had the indexed column updated in place.** This cannot be detected just by examining metadata. To prevent reading invalid data, the engine should filter out any row addresses that are not in the index's current `fragment_bitmap`. +4. **A fragment has an updated value in an [overlay file](../table/data_overlay_file.md).** + This can be detected by checking if any of the fragments in the index's `fragment_bitmap` + have overlay files. For each overlay whose `committed_version` is greater than the index + segment's `dataset_version`, the overlay carries updated values not reflected in the index, + so its covered rows must be excluded from index results. Excluded rows are re-evaluated + against their current (overlaid) values on the flat path — dropping them without + re-evaluation would silently lose rows that match under the new value. Exclusion is + field-aware: only overlays covering the indexed field matter. You may exclude just the + affected rows or the whole fragment; the latter is simpler and safer but re-evaluates more + rows than necessary. See [Data Overlay Files](../table/data_overlay_file.md#index-integration) + for the exclusion set, re-evaluation, and correctness invariant. ## Compaction and remapping diff --git a/docs/src/format/index/scalar/.pages b/docs/src/format/index/scalar/.pages index ba297222b07..98e5ebda5cb 100644 --- a/docs/src/format/index/scalar/.pages +++ b/docs/src/format/index/scalar/.pages @@ -7,4 +7,5 @@ nav: - Bloom Filter: bloom_filter.md - Full Text Search: fts.md - N-gram: ngram.md + - FM-Index: fmindex.md - RTree: rtree.md diff --git a/docs/src/format/index/scalar/bloom_filter.md b/docs/src/format/index/scalar/bloom_filter.md index 5a0e08e4228..6d924f8705d 100644 --- a/docs/src/format/index/scalar/bloom_filter.md +++ b/docs/src/format/index/scalar/bloom_filter.md @@ -4,6 +4,9 @@ Bloom filters are probabilistic data structures that allow for fast membership t They are space-efficient and can test whether an element is a member of a set. It's an inexact filter - they may include false positives but never false negatives. +In addition, since finding NULLs is a common query pattern, the index also maintains a +bitmap of null rows which allows it to return exact results for IS NULL queries. + ## Index Details ```protobuf @@ -32,6 +35,13 @@ The bloom filter index stores zone-based bloom filters in a single file: |---------------------------|--------|-------------------------------------------------------------| | `bloomfilter_item` | String | Expected number of items per zone (default: "8192") | | `bloomfilter_probability` | String | False positive probability (default: "0.00057", ~1 in 1754) | +| `null_bitmap` | UInt32 | Index of null bitmap global buffer | + +### Global Buffers + +| Metadata Key | Description | +|---------------------|------------------------------------------------------------| +| `null_bitmap` | A serialized RowAddrTreeMap specifying which rows are null | ## Bloom Filter Spec @@ -122,10 +132,11 @@ Offset 60-63: Block 1, Word 7 (32-bit LE) ## Accelerated Queries -The bloom filter index provides inexact results for the following query types: +The bloom filter index provides inexact results for the following query types (nullability queries +return exact results): | Query Type | Description | Operation | Result Type | |------------|---------------------------|-------------------------------------------|-------------| | **Equals** | `column = value` | Tests if value exists in bloom filter | AtMost | | **IsIn** | `column IN (v1, v2, ...)` | Tests if any value exists in bloom filter | AtMost | -| **IsNull** | `column IS NULL` | Returns zones where has_null is true | AtMost | \ No newline at end of file +| **IsNull** | `column IS NULL` | Returns zones where has_null is true | Exact | diff --git a/docs/src/format/index/scalar/fts.md b/docs/src/format/index/scalar/fts.md index adc7f94d65e..563d7ebf994 100644 --- a/docs/src/format/index/scalar/fts.md +++ b/docs/src/format/index/scalar/fts.md @@ -34,6 +34,15 @@ An FTS index may contain multiple partitions. Each partition has its own set of | `_rowid` | UInt64 | false | Document row ID | | `_num_tokens` | UInt32 | false | Number of tokens in the document | +Partitioned `docs.lance` files may include the optional schema metadata key +`total_tokens`. Its decimal `UInt64` value is the sum of `_num_tokens` in that +file. `_num_tokens` remains the canonical per-document data. Readers use the +metadata value to construct exact corpus statistics without scanning the column; +when the key is absent, they compute the sum from `_num_tokens`. Writers produce +the key from the same document table in the same file commit. A present value +that cannot be parsed, or that differs when `_num_tokens` is subsequently +loaded, is file corruption. + ### FTS List File Schema | Column | Type | Nullable | Description | @@ -43,6 +52,8 @@ An FTS index may contain multiple partitions. Each partition has its own set of | `_length` | UInt32 | false | Number of documents containing the token | | `_compressed_position` | List> | true | Optional compressed position lists for phrase queries | +The posting-list file schema metadata includes `posting_block_size`, the number of documents encoded per compressed posting block. Older indexes that do not have this metadata use the legacy block size `128`. + ### Metadata File Schema The metadata file contains JSON-serialized configuration and partition information: @@ -67,6 +78,7 @@ The metadata file contains JSON-serialized configuration and partition informati | `min_gram` | UInt32 | 2 | Minimum n-gram length (only for ngram tokenizer) | | `max_gram` | UInt32 | 15 | Maximum n-gram length (only for ngram tokenizer) | | `prefix_only` | Boolean | false | Generate only prefix n-grams (only for ngram tokenizer) | +| `block_size` | UInt32 | 128 | Documents per compressed posting block. Must be 128 or 256. Missing values from older indexes read as 128. `256` is experimental and may introduce breaking changes. | ## Tokenizers diff --git a/docs/src/format/index/scalar/zonemap.md b/docs/src/format/index/scalar/zonemap.md index 256edc2671d..552d476cc86 100644 --- a/docs/src/format/index/scalar/zonemap.md +++ b/docs/src/format/index/scalar/zonemap.md @@ -8,6 +8,9 @@ zones that cannot contain matching values. Zone maps are "inexact" filters - they can definitively exclude zones but may include false positives that require rechecking. +In addition, since finding NULLs is a common query pattern, the index also maintains a +bitmap of null rows which allows it to return exact results for IS NULL queries. + ## Index Details ```protobuf @@ -34,17 +37,25 @@ The zone map index stores zone statistics in a single file: ### Schema Metadata -| Key | Type | Description | -|-----------------|--------|-------------------------------------------| -| `rows_per_zone` | String | Number of rows per zone (default: "8192") | +| Key | Type | Description | +|---------------------|--------|-------------------------------------------| +| `rows_per_zone` | String | Number of rows per zone (default: "8192") | +| `null_bitmap` | UInt32 | Index of null bitmap global buffer | + +### Global Buffers + +| Metadata Key | Description | +|---------------------|------------------------------------------------------------| +| `null_bitmap` | A serialized RowAddrTreeMap specifying which rows are null | ## Accelerated Queries -The zone map index provides inexact results for the following query types: +The zone map index provides inexact results for the following query types (nullability queries +return exact results): | Query Type | Description | Operation | Result Type | |------------|---------------------------|---------------------------------------------|-------------| | **Equals** | `column = value` | Includes zones where min ≤ value ≤ max | AtMost | | **Range** | `column BETWEEN a AND b` | Includes zones where ranges overlap | AtMost | | **IsIn** | `column IN (v1, v2, ...)` | Includes zones that could contain any value | AtMost | -| **IsNull** | `column IS NULL` | Includes zones where null_count > 0 | AtMost | \ No newline at end of file +| **IsNull** | `column IS NULL` | Includes zones where null_count > 0 | Exact | diff --git a/docs/src/format/index/system/mem_wal.md b/docs/src/format/index/system/mem_wal.md index 44515f0af28..2f0a5cb1fa5 100644 --- a/docs/src/format/index/system/mem_wal.md +++ b/docs/src/format/index/system/mem_wal.md @@ -1,7 +1,7 @@ # MemWAL Index The MemWAL Index is a system index that serves as the centralized structure for all MemWAL metadata. -It stores configuration (shard specs, indexes to maintain), merge progress, and shard state snapshots. +It stores configuration (shard specs, indexes to maintain), SSTable compaction progress, and shard state snapshots. A table has at most one MemWAL index. The table may be a primary-key table or an append-only table without primary-key metadata. diff --git a/docs/src/format/index/vector/index.md b/docs/src/format/index/vector/index.md index 7aaf9b55996..91d73a23a16 100644 --- a/docs/src/format/index/vector/index.md +++ b/docs/src/format/index/vector/index.md @@ -68,7 +68,7 @@ The index file stores the search structure with graph or flat organization. The Arrow schema of the Lance file varies depending on the sub-index type used. !!! note -All partitions are stored in the same file, and partitions must be written in order. + All partitions are stored in the same file, and partitions must be written in order. ##### FLAT @@ -84,12 +84,12 @@ HNSW (Hierarchical Navigable Small World) indices provide fast approximate searc | Column | Type | Nullable | Description | | ------------- | ------------- | -------- | ---------------------- | -| `__vector_id` | uint64 | false | Vector identifier | -| `__neighbors` | list | false | Neighbor node IDs | -| `_distance` | list | false | Distances to neighbors | +| `__vector_id` | uint32 | true | Vector identifier | +| `__neighbors` | list | true | Neighbor node IDs | +| `_distance` | list | true | Distances to neighbors | !!! note -HNSW consists of multiple levels, and all levels must be written in order starting from level 0. + HNSW consists of multiple levels, and all levels must be written in order starting from level 0. #### Arrow Schema Metadata @@ -111,8 +111,8 @@ References the IVF metadata stored in the Lance file global buffer. This value records the global buffer index, currently this is always "1". !!! note -Global buffer indices in Lance files are 1-based, -so you need to subtract 1 when accessing them through code. + Global buffer indices in Lance files are 1-based, + so you need to subtract 1 when accessing them through code. ##### "lance:flat" @@ -159,7 +159,7 @@ Since the auxiliary file stores the actual (quantized) vectors, the Arrow schema of the Lance file varies depending on the quantization method used. !!! note -All partitions are stored in the same file, and partitions must be written in order. + All partitions are stored in the same file, and partitions must be written in order. ##### FLAT @@ -167,17 +167,17 @@ No quantization applied - stores original vectors in their full precision: | Column | Type | Nullable | Description | | -------- | ------------------------ | -------- | ----------------------------------------------------- | -| `_rowid` | uint64 | false | Row identifier | -| `flat` | list[dimension] | false | Original vector values (list_size = vector dimension) | +| `_rowid` | uint64 | true | Row identifier | +| `flat` | list[dimension] | true | Original vector values (list_size = vector dimension) | ##### PQ Compresses vectors using product quantization for significant memory savings: -| Column | Type | Nullable | Description | -| ----------- | -------------- | -------- | ------------------------------------------- | -| `_rowid` | uint64 | false | Row identifier | -| `__pq_code` | list[m] | false | PQ codes (list_size = number of subvectors) | +| Column | Type | Nullable | Description | +| ----------- | ----------------------------------------------- | -------- | --------------------------------------------- | +| `_rowid` | uint64 | true | Row identifier | +| `__pq_code` | list[num_sub_vectors * num_bits / 8] | true | PQ codes, packed to `num_bits` per subvector | ##### SQ @@ -185,23 +185,28 @@ Compresses vectors using scalar quantization for moderate memory savings: | Column | Type | Nullable | Description | | ----------- | ---------------------- | -------- | --------------------------------------- | -| `_rowid` | uint64 | false | Row identifier | -| `__sq_code` | list[dimension] | false | SQ codes (list_size = vector dimension) | +| `_rowid` | uint64 | true | Row identifier | +| `__sq_code` | list[dimension] | true | SQ codes (list_size = vector dimension) | ##### RQ Compresses vectors using RabitQ with random rotation and binary quantization for extreme compression: -| Column | Type | Nullable | Description | -| -------------------- | ------------------------------------------------ | ------------------------ | --------------------------------------------------------------- | -| `_rowid` | uint64 | false | Row identifier | -| `_rabit_codes` | list[dimension / 8] | false | Binary quantized codes (1 bit per dimension, packed into bytes) | -| `__add_factors` | float32 | false | Additive correction factors for distance computation | -| `__scale_factors` | float32 | false | Scale correction factors for distance computation | -| `__error_factors` | float32 | false for `raw_query` | Error factors for raw-query lower-bound pruning | -| `__ex_codes` | list[ceil(dimension * (num_bits - 1) / 8)] | false for `num_bits > 1` | Extra RabitQ code bits for multi-bit RQ | -| `__add_factors_ex` | float32 | false for `num_bits > 1` | Additive correction factors for ex-code distance computation | -| `__scale_factors_ex` | float32 | false for `num_bits > 1` | Scale correction factors for ex-code distance computation | +| Column | Type | Nullable | Present when | Description | +| -------------------- | ------------------------------------------------ | -------- | --------------------------- | --------------------------------------------------------------- | +| `_rowid` | uint64 | true | always | Row identifier | +| `_rabit_codes` | list[ceil(code_dim / 8)] | true | always | Binary quantized codes (1 bit per dimension, packed into bytes) | +| `__add_factors` | float32 | true | always | Additive correction factors for distance computation | +| `__scale_factors` | float32 | true | always | Scale correction factors for distance computation | +| `__error_factors` | float32 | true | `raw_query` estimator | Error factors for raw-query lower-bound pruning | +| `__blocked_ex_codes` | list[next_multiple_of(code_dim, 64) * (num_bits - 1) / 8] | true | `num_bits > 1` | Extra RabitQ code bits for multi-bit RQ, in the blocked layout | +| `__add_factors_ex` | float32 | true | `num_bits > 1` | Additive correction factors for ex-code distance computation | +| `__scale_factors_ex` | float32 | true | `num_bits > 1` | Scale correction factors for ex-code distance computation | + +!!! note + Indexes written before the blocked ex-code layout store the same bits in + `__ex_codes`, sized `ceil(dimension * (num_bits - 1) / 8)`. Readers still + accept that column and repack it at load time; writers no longer emit it. #### Arrow Schema Metadata @@ -280,7 +285,7 @@ to rotate vectors before binary quantization: The rotation matrix has shape `[code_dim, code_dim]` where `code_dim` is the rotated vector dimension. IVF_RQ always stores the 1-bit binary sign code in `_rabit_codes`; for `num_bits > 1`, -the remaining `num_bits - 1` ex-code bits are stored in `__ex_codes` instead of widening the +the remaining `num_bits - 1` ex-code bits are stored in `__blocked_ex_codes` instead of widening the binary code path. New IVF_RQ indexes store raw-query estimator factors. `num_bits=1` indexes only store the binary-code factor columns; multi-bit indexes also store separate ex-code additive and scale factors. @@ -319,7 +324,7 @@ PQ uses 16 num_sub_vectors (m=16) with 8 num_bits per subvector, and distance ty ```python pa.schema([ pa.field("_rowid", pa.uint64()), - pa.field("__pq_code", pa.list(pa.uint8(), list_size=16)), # m subvector codes + pa.field("__pq_code", pa.list_(pa.uint8(), list_size=16)), # num_sub_vectors * num_bits / 8 = 16 * 8 / 8 ]) ``` @@ -327,7 +332,7 @@ pa.schema([ This example shows how an `IVF_RQ` index is physically laid out. Assume vectors have dimension 128, RQ uses 1 bit per dimension (`num_bits=1`), and distance type is "l2". For `num_bits > 1`, the -auxiliary schema also includes `__ex_codes`, `__add_factors_ex`, and `__scale_factors_ex`. +auxiliary schema also includes `__blocked_ex_codes`, `__add_factors_ex`, and `__scale_factors_ex`. #### Index File @@ -356,7 +361,7 @@ auxiliary schema also includes `__ex_codes`, `__add_factors_ex`, and `__scale_fa ```python pa.schema([ pa.field("_rowid", pa.uint64()), - pa.field("_rabit_codes", pa.list(pa.uint8(), list_size=16)), # dimension/8 = 128/8 = 16 bytes + pa.field("_rabit_codes", pa.list_(pa.uint8(), list_size=16)), # ceil(code_dim / 8) = ceil(128 / 8) pa.field("__add_factors", pa.float32()), pa.field("__scale_factors", pa.float32()), pa.field("__error_factors", pa.float32()), diff --git a/docs/src/format/table/.pages b/docs/src/format/table/.pages index 16c20058608..5b0cb0e95e6 100644 --- a/docs/src/format/table/.pages +++ b/docs/src/format/table/.pages @@ -6,4 +6,5 @@ nav: - Layout: layout.md - Branch & Tag: branch_tag.md - Row ID & Lineage: row_id_lineage.md + - Data Overlay Files: data_overlay_file.md - MemTable & WAL: mem_wal.md diff --git a/docs/src/format/table/data_overlay_file.md b/docs/src/format/table/data_overlay_file.md new file mode 100644 index 00000000000..9da739a29e3 --- /dev/null +++ b/docs/src/format/table/data_overlay_file.md @@ -0,0 +1,388 @@ +# Data Overlay Files + +!!! warning "Experimental" + + This feature is currently experimental and not yet supported in any library. + + + +!!! note "Overlay files require feature flag 64 (data overlay files)" + + A reader or writer that does not understand overlay files must refuse a + dataset that uses them. Silently ignoring an overlay would return stale base + values, which is a correctness bug rather than a degraded experience. + +Overlay files supply new values for a subset of `(row offset, field)` cells +within a fragment **without rewriting the fragment's base data files**. They make +updates cheap when only a small fraction of rows and/or columns change: instead +of rewriting whole columns or moving rows to a new fragment, a writer appends a +small file carrying just the changed cells. + +This is Lance's third mechanism for changing data in place, alongside +[deletion files](index.md#deletion-files) (which remove rows) and +[data evolution](index.md#data-evolution) (which adds or rewrites whole columns). +An overlay changes individual cells. + +## Concepts + +### Coverage and resolution + +Each overlay declares which cells it provides through a **coverage** bitmap (or, +for sparse overlays, one bitmap per field). The bitmaps index **physical row +offsets**. They include deleted rows and are stable even as deletion vectors change. + +To resolve a cell `(offset, field)` on read, walk the fragment's overlays from +**newest to oldest**. The first overlay that covers `(offset, field)` wins; its +value is used. If no overlay covers the cell, the value falls through to the base +data file (or is `NULL` if no base data file holds that field). + +Precedence among overlays is determined by: + +1. `committed_version` — higher wins (see [Versioning](#versioning-and-ordering)). +2. Position in `DataFragment.overlays` as a tiebreaker — a later entry is newer. + +A covered offset whose value is `NULL` overrides the cell **to** `NULL`. This is +distinct from an offset that is simply absent from the bitmap, which falls +through to the base. Coverage, not value-nullness, decides whether an overlay +applies. + +### Interaction with deletions + +Deletions take precedence over overlays. If a row offset is marked deleted in the +fragment's deletion file, any overlay value for that offset is dead and is +ignored, regardless of commit order. + +### Physical layout + +An overlay's data file stores **one value column per field**, in the order of +`data_file.fields`. It does **not** store a row-offset key column. The position of +a covered offset's value within its column is the **rank** of that offset in the +field's coverage bitmap — the number of set bits below it. Resolving a cell is a +rank lookup plus one value fetch, with no separate offset column to store or +search. + +Because different fields may cover different offset sets, the value columns of a +single sparse overlay may have **different lengths**. The Lance file format +permits columns of differing item counts within one file, so a sparse overlay is +representable as a single file. (See [Writer support](#writer-support) for the +current implementation status.) + +### Dense vs. sparse overlays + +A single overlay is one of two shapes: + +- **Dense (rectangular).** One `shared_offset_bitmap` applies to every field. Every + covered offset has a value for every field. This is the common case for a plain + `UPDATE`, where one `SET` list is applied to one set of rows. +- **Sparse.** A `FieldCoverage` carries one bitmap per field, used when different + fields cover different offset sets — for example a `MERGE` with multiple + `WHEN MATCHED` branches, where different rows update different columns. A dense + overlay would have to widen to the bounding rectangle and fill the untouched + cells with their current values (post-images), which for wide columns such as + embeddings means re-storing data that did not change. A sparse overlay stores + exactly the changed cells. + +## Protobuf + +
+DataOverlayFile protobuf message + +```protobuf +%%% proto.message.DataOverlayFile %%% +``` + +
+ +
+FieldCoverage protobuf message + +```protobuf +%%% proto.message.FieldCoverage %%% +``` + +
+ +## Versioning and ordering + +Overlays reuse the dataset version as their ordering clock rather than +introducing a separate generation counter. + +`committed_version` is the dataset version at which an overlay **became +effective** — the version of the commit that introduced it, **not** the version +it was read from. It is stamped at commit time and re-stamped if the commit is +retried, in the same way as the created-at / last-updated-at version sequences. + +This single value drives every ordering decision: + +- **Overlay vs. overlay** (read precedence): higher `committed_version` wins. +- **Overlay vs. index** (query correctness): an index records the + `dataset_version` it was built from. An index whose `dataset_version >= + committed_version` already incorporates the overlay. An overlay whose + `committed_version > index.dataset_version` is newer than the index and its + cells must be excluded from index results and re-evaluated. +- **Scheduler signal**: the gap between an overlay's `committed_version` and an + index's `dataset_version`, or between an overlay and the base, is a staleness + measure the compaction scheduler can use. + +!!! note "Why effective version, not read version" + + Suppose an overlay reads version 5 and commits at version 6, while an index + is built reading version 5 (before the overlay) and commits at version 7 with + `dataset_version = 5`. If the overlay stored its *read* version (5), the test + `5 > 5` is false, the row would not be excluded, and the index — which never + saw the overlay — would return a stale result. Storing the *effective* + version (6) makes `6 > 5` true, the cell is excluded and re-evaluated, and the + result is correct. + +## Index integration + +Building an index over a fragment that has overlays does **not** require dropping +the fragment from the index's coverage. The fragment stays indexed, and the query +path reconciles overlays at query time using an **exclusion set**. + +The exclusion set for an index on field `F` is the union of the coverage bitmaps, +restricted to field `F`, of every overlay whose `committed_version > +index.dataset_version`. The exclusion is **field-aware**: an overlay that touches +only unrelated columns does not exclude anything from the index on `F`. + +The query then proceeds as: + +1. Run the index search as usual, producing candidate rows. +2. Remove any candidate in the exclusion set. (Its indexed value may be stale.) +3. **Re-evaluate** the excluded rows against their current values — the same flat + path already used for the unindexed tail of fragments. For a scalar predicate + this re-applies the filter; for a vector query it re-scores the row's current + vector. Rows that still match are added back to the result. + +Step 3 is what makes exclusion correct rather than merely safe: removing a row +from index candidates without re-evaluating it would silently drop a row that +should match under its new value. + +Exclusion is always *sufficient* because a write changes a cell only by adding an +overlay, and that overlay's `committed_version` — the version of the commit that +adds it — necessarily exceeds the `dataset_version` of any pre-existing index. So +every cell a write changes is guaranteed to fall in that index's exclusion set. +Compaction may remove an overlay only if no index still relies on it for exclusion +(see [Compaction](#compaction)). + +## Compaction + +Overlays accumulate read cost — every overlay is a bitmap to test, a possible +file to open, and additional work to interleave values. Compaction bounds that cost in two modes: + +- **Overlay → overlay.** Merge several overlays into fewer, computing the + post-image per `(offset, field)` by walking the merged overlays newest-first. + The merged overlay takes the **maximum** `committed_version` of its inputs, so + the exclusion semantics are preserved. The merged overlays must be **contiguous + in `committed_version`** — with overlays at v10, v30, and v50 you cannot merge + just v10 and v50, because stamping the result v50 would incorrectly promote + v10's values above the intervening v30 for any cell v30 also covers. Indexes can + still be re-used, but they may now need to exclude more rows. This is cheap to + write and does not touch the base. +- **Overlay → base.** Fold overlays into a fresh base data file, computing the + post-image for every covered cell, then clear the overlays. The base is + complete, so every post-image is well defined. Overlay offsets are physical, so + they cannot survive a rewrite that reorders rows; folding therefore materializes + values rather than carrying overlays forward. + +!!! warning "Folding an indexed field must update its index" + + An overlay→base fold removes the overlay, which removes the exclusion signal + that kept an index correct. Folding an overlay that covers an indexed field + `F` is therefore equivalent to a column rewrite of `F` and must, in the same + commit, either rebuild the index to a `dataset_version` at least the folded + overlay's `committed_version`, or remove the fragment from the index's + coverage so the rows fall to the flat path. Otherwise the index would serve + stale values with no overlay to exclude them. This is the same rule that + already governs rewriting a column that an index is built on. + +When a fragment with overlays is compacted by a row-rewriting operation +(`RewriteRows`, which produces new fragments with new row addresses), the +overlays are folded into the new base as part of the rewrite, and existing +[fragment-reuse remapping](row_id_lineage.md) handles the row-address changes as +it does today. + +## Row lineage + +An overlay write updates the `last_updated_at_version` of every covered row, so +change-data-feed and time-travel queries observe the update. Because overlays are +addressed by physical offset, they do **not** require stable row IDs to be +enabled; lineage updates apply only when those features are on. + +## Worked example + +The following example illustrates how overlays function across their lifecycle, to make the rules above concrete. + +A table `users` with stable row IDs enabled and these fields: + +| field id | name | type | +|----------|-----------|-------------------------| +| 1 | id | `int32` (primary key) | +| 2 | name | `utf8` | +| 3 | age | `int32` | +| 4 | embedding | `fixed_size_list`| + +Created at version 1 as a single fragment `0` with one base data file +`data/file0.lance` holding all four columns. `physical_rows = 4`: + +| offset | id | name | age | embedding | +|--------|----|-------|-----|------------------| +| 0 | 1 | Alice | 30 | … | +| 1 | 2 | Bob | 25 | … | +| 2 | 3 | Carol | 40 | … | +| 3 | 4 | Dave | 22 | … | + +A BTree scalar index on `age` is built at version 1, covering fragment `0` +(`dataset_version = 1`). + +### Step 1 — write an overlay + +```sql +UPDATE users SET age = age + 1 WHERE id IN (2, 4); -- Bob (offset 1), Dave (offset 3) +``` + +This touches one field (`age`) for two rows, so the writer emits a dense overlay — +one shared bitmap covering both offsets — and commits it as version 2. Fragment +`0` gains: + +```text +DataOverlayFile { + data_file: { path: "data/overlay-.lance", fields: [3], column_indices: [0] } + coverage: shared_offset_bitmap = {1, 3} + committed_version: 2 +} +``` + +The overlay file stores a single `age` column with two values, `[26, 23]`, at +ranks `{1,3}.rank(1) = 0` and `{1,3}.rank(3) = 1`. `last_updated_at_version` is +set to 2 for offsets 1 and 3. + +### Step 2 — read + +`SELECT id, age FROM users` reads base ages `[30, 25, 40, 22]`. For `age` +(field 3), the overlay covers offsets 1 and 3, so `age[1]` is replaced with the +overlay value at rank `{1,3}.rank(1) = 0` → `26`, and `age[3]` with the value at +rank `{1,3}.rank(3) = 1` → `23`. Result ages: `[30, 26, 40, 23]`. + +### Step 3 — index query + +```sql +SELECT * FROM users WHERE age = 26; +``` + +The `age` index was built at `dataset_version = 1`; the overlay's +`committed_version` is 2. Since `2 > 1`, the overlay's coverage for `age`, `{1, 3}`, +is the exclusion set for this query. + +- The index (built at v1) holds Bob's *old* `age = 25`, so a lookup for `26` + returns nothing from the index. +- The whole exclusion set is re-evaluated on the flat path, not just the rows the + index returned. Offset 1's current `age` (26, via the overlay) matches, so Bob + is returned; offset 3's current `age` (23) does not match and is dropped. + +The mirror case `WHERE age = 25` shows exclusion preventing a stale hit: the index +returns offset 1 (stale `25`), but offset 1 is excluded, re-evaluated to `26`, and +correctly dropped. + +### Step 4 — a second, non-rectangular write + +```sql +MERGE INTO users USING staged ON users.id = staged.id +WHEN MATCHED AND staged.kind = 'rename' THEN UPDATE SET name = staged.name -- Carol(2), Dave(3) +WHEN MATCHED AND staged.kind = 'embed' THEN UPDATE SET embedding = staged.embedding -- Bob(1) +``` + +`name` is updated for offsets `{2, 3}` and `embedding` for offset `{1}` — different +fields over different rows. This is a sparse overlay, committed as version 3: + +```text +DataOverlayFile { + data_file: { path: "data/overlay-.lance", fields: [2, 4], column_indices: [0, 1] } + coverage: field_coverage { offset_bitmaps: [ {2,3}, {1} ] } + // name (field 2) ^ ^ embedding (field 4) + committed_version: 3 +} +``` + +The file's `name` column has **two** values (`["Caroline", "David"]`, at +ranks 0 and 1 of `{2,3}`) and its `embedding` column has **one** value (at rank 0 +of `{1}`) — columns of different lengths in one file. + +### Step 5 — read after the second write + +`SELECT name, age, embedding FROM users` resolves each field independently, +newest overlay first: + +- `name`: the v3 overlay covers `{2,3}` → `["Alice", "Bob", "Caroline", "David"]`. +- `age`: the v3 overlay does not cover `age`; the v2 overlay still applies at + offsets 1 and 3 → `[30, 26, 40, 23]`. +- `embedding`: the v3 overlay covers `{1}` → Bob's vector is the new one, others + from base. + +Overlays from different versions coexist and apply per field. + +### Step 6 — compaction (overlay → base) + +The scheduler folds both overlays into fragment `0` at version 4, computing +post-images for `age`, `name`, and `embedding`, and writing a new base data file +`data/file1.lance` with those columns. In the old file, fields 2, 3, and 4 are +marked with a tombstone (`-2`); field 1 (`id`) remains. The fragment's `overlays` list is +cleared. Row addresses are preserved (a column rewrite, not a row rewrite), so +stable row IDs and the deletion vector are untouched. + +Because the fold removed the overlay that was excluding offsets 1 and 3 from the +`age` index, the commit must drop fragment `0` from its coverage so `age` queries +fall to the flat path. + +## Guidance + +!!! note "This section is a stub." + + The following are implementation considerations, not part of the on-disk + specification. + +### When to overlay vs. rewrite a column vs. move rows + + + +*(To be expanded.)* The choice between appending an overlay, rewriting a full +column (data evolution), and moving updated rows to a new fragment depends on the +fraction of rows changed, the fraction of columns changed, column width, the +presence of indexes on the changed columns, and the accumulated overlay read +cost. Roughly: few rows changed favors overlays; most rows in a few columns +favors a column rewrite; most columns changed favors moving rows to a new +fragment. + +### Writer support + + + +*(To be expanded.)* Dense (rectangular) overlays write with the existing +equal-length file writer today. Sparse overlays stored as a **single** file +require the writer to emit columns of independent lengths, which the current v2 +writer does not yet do (it advances all columns from one global row counter). +Until that support lands, a writer can express a sparse update as multiple dense +overlays in one transaction. + +### Scheduling compaction + + + +*(To be expanded.)* The overlay→overlay and overlay→base modes have very +different costs; a cost/benefit scheduler decides when each is worthwhile, using +the version gap as a staleness signal. + +## Related specifications + +- [Table format overview](index.md) +- [Transactions: DataOverlay operation](transaction.md#dataoverlay) — write path + and conflict semantics +- [Row ID & Lineage](row_id_lineage.md) +- [Index Formats: handling overlay rows](../index/index.md#handling-deleted-and-invalidated-rows) +- [Format Versioning](versioning.md) diff --git a/docs/src/format/table/index.md b/docs/src/format/table/index.md index 94ea4b90dc9..f9da132cf3b 100644 --- a/docs/src/format/table/index.md +++ b/docs/src/format/table/index.md @@ -168,6 +168,29 @@ However, this invalidates row addresses and requires rebuilding indices, which c +## Data Overlay Files + +!!! warning "Experimental" + + This feature is currently experimental and not yet supported in any library. + + + +!!! note "Overlay files require feature flag 64 (data overlay files)" + +Overlay files supply new values for a subset of cells within +a fragment without rewriting the base data files. They make updates cheap when only +a small percentage of rows and/or columns change: a writer appends a small file +carrying just the changed cells instead of rewriting whole columns or moving rows +to a new fragment. + +For the full specification — coverage and resolution rules, dense vs. sparse layout, +versioning, index integration, compaction, and a worked example — see the +[Data Overlay Files Specification](data_overlay_file.md). + + + ## Related Specifications ### Storage Layout diff --git a/docs/src/format/table/mem_wal.md b/docs/src/format/table/mem_wal.md index 8a228721123..c8ed7f6435f 100644 --- a/docs/src/format/table/mem_wal.md +++ b/docs/src/format/table/mem_wal.md @@ -7,684 +7,695 @@ scan, point lookup, vector search and full-text search. ![MemWAL Overview](../../images/mem_wal_overview.png) -A Lance table is called a **base table** under the context of the MemWAL spec. -It may have an [unenforced primary key](index.md#unenforced-primary-key) defined in the table schema. -Primary keys are required for primary-key lookups and last-write-wins upsert semantics, -but append-only MemWAL tables may omit them. +A Lance table is called the **base table** in this document. +The base table may have an [unenforced primary key](index.md#unenforced-primary-key) in its schema. +Primary keys are required for primary-key lookups and last-write-wins upsert semantics. +Append-only MemWAL tables may omit a primary key. -On top of the base table, the MemWAL spec defines a set of shards. -Writers write to shards, and data in each shard is merged into the base table asynchronously. -An index is kept in the base table for readers to quickly discover the state of all shards at a point of time. +MemWAL adds a set of shards on top of the base table. +Writers append to shards. +Each shard keeps recent data in an in-memory MemTable, persists writes to a per-shard WAL, flushes MemTables as small Lance datasets, and later compacts those SSTables into the base table. + +The base table manifest contains one MemWAL system index entry named `__lance_mem_wal`. +This index stores MemWAL configuration and global progress metadata inline in `IndexMetadata.index_details`. +Each shard's own manifest remains authoritative for shard-local mutable state. ### MemWAL Shard -A **MemWAL Shard** is the main unit to horizontally scale out writes. +A **MemWAL shard** is the unit of horizontal write scaling. +Each shard has exactly one active writer epoch at a time. +Writers claim a shard, append WAL entries, update the in-memory MemTable, and publish SSTable generations by updating the shard manifest. -Each shard has exactly one active writer at any time. -Writers claim a shard and then write data to that shard. -Data in each shard is expected to be merged into the base table asynchronously. +For primary-key tables, all rows for the same primary key must map to the same shard. +If one primary key can appear in multiple shards, asynchronous compaction order between shards can make an older row overwrite a newer row. +Append-only tables without a primary key do not rely on last-write-wins conflict resolution and may use any deterministic shard assignment suitable for the workload. -For tables with a primary key, rows of the same primary key must be written to one and only one shard. -If two shards contain rows with the same primary key, the following scenario can cause data corruption: +### MemWAL Index -1. Shard A receives a write with primary key `pk=1` at time T1 -2. Shard B receives a write with primary key `pk=1` at time T2 (T2 > T1) -3. The row in shard B is merged into the base table first -4. The row in shard A is merged into the base table second -5. The row from Shard A (older) now overwrites the row from Shard B (newer) +The MemWAL index is a system index entry on the base table. +It has `name = "__lance_mem_wal"`, no indexed fields, and no index files. +`IndexMetadata.files` is `None`. +All MemWAL index data is stored in the `MemWalIndexDetails` protobuf message in `IndexMetadata.index_details`. -This violates the expected "last write wins" semantics. -By ensuring each primary key is assigned to exactly one shard via the sharding spec, -merge order between shards becomes irrelevant for correctness. -Append-only tables without a primary key do not rely on last-write-wins conflict resolution -and may shard by any deterministic append key or partitioning column. +The index stores: -See [MemWAL Shard Architecture](#shard-architecture) for the complete shard architecture. +- **Configuration**: `sharding_specs`, `maintained_indexes`, and `writer_config_defaults`. +- **Compaction progress**: `compacted_sstables`, the last SSTable compacted into the base table for each shard. +- **Index catchup progress**: `index_catchup`, the compacted SSTable generation covered by each base-table index. +- **Shard snapshots**: optional point-in-time snapshot fields for read optimization. -### MemWAL Index +Shard snapshots are not authoritative. +Readers that need the latest shard set list `_mem_wal/` and read each shard's latest manifest. -A **MemWAL Index** is the centralized structure for all MemWAL metadata on top of a base table. -A table has at most one MemWAL index. It stores: +## Shard Architecture -- **Configuration**: Sharding specs defining how rows map to shards, and which indexes to maintain -- **Merge progress**: Last generation merged to base table for each shard -- **Index catchup progress**: Which merged generation each base table index has been rebuilt to cover -- **Shard snapshots**: Point-in-time snapshot of shard states for read optimization +![Shard Architecture](../../images/mem_wal_shard.png) -The index is the source of truth for **configuration**, **merge progress** and **index catchup progress** -Writers and mergers read the MemWAL index to get these configurations before writing. +Within a shard, writes first enter an in-memory **MemTable** and are durably appended to the shard **write-ahead log (WAL)**. +The MemTable is periodically **flushed** to storage as a Lance dataset. +SSTables are asynchronously **compacted** into the base table. -Each [shard's manifest](#shard-manifest) is authoritative for its own state. -Readers may use **shard snapshots** as a read-only optimization to see a point-in-time view of shards without opening each shard manifest. -Readers that need the latest shard set must discover shard directories in storage and read each shard's latest manifest. +### MemTable -See [MemWAL Index Details](#memwal-index-details) for the complete structure. +A MemTable holds rows inserted into a shard before those rows are flushed to storage. +It serves two purposes: -## Shard Architecture +1. It buffers data and per-MemTable indexes before an SSTable is written. +2. It lets readers access data that has not been flushed yet when strong consistency is required. -![Shard Architecture](../../images/mem_wal_regional.png) +The storage format does not prescribe the in-memory MemTable layout. +Conceptually, a MemTable is an append log of Arrow record batches. +Later appends have larger in-memory row positions. +For primary-key tables, in-memory reads use the largest visible row position as the newest row for a key. -Within a shard, writes are stored in an **in-memory table (MemTable)**. -It is also written to the shard's **Write-Ahead Log (WAL)** for durability guarantee. -The MemTable is periodically **flushed** to storage based on memory pressure and other conditions. -**Flushed MemTables** in storage are then asynchronously **merged** into the base table. +### SSTable Generation -### MemTable +Within each shard, SSTables have monotonically increasing generation numbers starting from 1. +When a MemTable is flushed, the resulting SSTable is assigned the shard manifest's `current_generation`, and `current_generation` advances to the next SSTable generation. +A MemTable does not have a generation. -A MemTable holds rows inserted into the shard before flushing to storage. -It serves 2 purposes: +SSTable generation numbers order persisted data freshness within one shard: -1. build up data and related indexes to be flushed to storage as a flushed MemTable -2. allow a reader to potentially access data that is not flushed to storage yet +- Base table data is modeled as generation 0. +- Higher SSTable generations are newer. +- The active MemTable is newer than every published SSTable. +- Within the active MemTable, higher row positions are newer. +- Within an SSTable, flush-time deletion vectors hide older duplicate primary-key rows, so readers see at most the newest row for each primary key. -#### MemTable Format +## WAL -The complete in-memory format of a MemTable is implementation-specific and out of the scope of this spec. -The Lance core Rust SDK maintains one default implementation and is available through all its language binding SDKs, -but integrations are free to build their own MemTable format depending on the specific use cases, -as long as it follows the MemWAL storage layout, reader and writer requirements when flushing MemTable. +The WAL is the durable append log for a shard. +Every durable WAL append creates one **WAL entry**. -Conceptually, because Lance uses [Arrow as its in-memory data exchange format](https://arrow.apache.org/docs/format/index.html), -for the ease of explanation in this spec, we will treat MemTable as a list of Arrow record batches, -and each write into the MemTable is a new Arrow record batch. +### WAL Entry Positions -#### MemTable Generation +WAL entry positions are 1-based. +The first data entry is position 1. +Position 0 is reserved as the sentinel value meaning no WAL entry has been covered. -Based on conditions like memory limit and durability requirements, -a MemTable needs to be **flushed** to storage and discarded. -When that happens, new writes go to a new MemTable and the cycle repeats. -Each MemTable is assigned a monotonically increasing generation number starting from 1. -When MemTable of generation `N` is discarded, the next MemTable gets assigned generation `N+1`. +Writers append WAL entries in increasing position order. +If entry `N` is not fully written, entry `N + 1` must not exist. +Recovery replays from `replay_after_wal_entry_position + 1`. -### WAL +### WAL Entry Format -WAL serves as the durable storage of all MemTables in a shard. -It consists of data in MemTables ordered by generation. -Every time we write to the WAL, we call it a **WAL Flush**. +Each WAL entry is an Apache Arrow IPC stream file. +The Arrow schema metadata includes: -#### WAL Durability +- `writer_epoch`: decimal string containing the writer epoch that created the entry. +- `fence_sentinel`: optional marker for a data-less fence sentinel entry. -When a write is flushed to WAL, the specific write becomes durable. -Otherwise, if the MemTable is lost, data is also lost. +A normal WAL entry contains one or more record batches. +A fence sentinel entry contains no batches and is skipped during replay. +Sentinels are used so an older writer collides on the next WAL position and discovers that it has been fenced. -Multiple writes can be batched together in a single WAL flush to reduce WAL flush frequency and improve throughput. -The more writes a single WAL flush batches, the longer it takes for a write to be durable. +### WAL Storage Layout -The whole LSM tree's durability is determined by the durability of the WAL. -For example, if WAL is stored in Amazon S3, it has 99.999999999% durability. -If it is stored in local disk, the data will be lost if the local disk is damaged. +WAL entries live under `_mem_wal/{shard_id}/wal/`. +Filenames use bit-reversed 64-bit binary names with the `.arrow` suffix: -#### WAL Entry +```text +_mem_wal/{shard_id}/wal/{bit_reversed_position}.arrow +``` -Each time a WAL flush happens, it adds a new **WAL Entry** to the WAL. -In other words, a WAL consists of an ordered list of WAL entries starting from position 0. -Writer must flush WAL entries in sequential order from lower to higher position. -If WAL entry `N` is not flushed fully, WAL entry `N+1` must not exist in storage. +The bit-reversal spreads sequential positions across object-store keyspace. +For example, position 5 is encoded as: -#### WAL Replay +```text +1010000000000000000000000000000000000000000000000000000000000000.arrow +``` -**Replaying** a WAL means to read data in the WAL from a lower to a higher position. -This is commonly used to recover the latest MemTable after it is lost, -by reading from the start position of the latest MemTable generation till the highest position in the WAL, -assuming proper fencing to guard against multiple writers to the same shard. +## SSTable -See [Writer Fencing](#writer-fencing) for the full fencing mechanism. +An SSTable is the immutable result of flushing a MemTable. +It is stored as a Lance dataset under its shard directory. -#### WAL Entry Format +!!! note + Unlike a classic LSM sorted string table, a MemWAL SSTable is not sorted by key; random access is instead served by its BTree primary-key sidecar. It is called an SSTable because it is an immutable, persisted, indexed run. -Each WAL entry is a file in storage following the [Apache Arrow IPC stream format](https://arrow.apache.org/docs/format/Columnar.html#ipc-streaming-format) to store the batch of writes in the MemTable. -The writer epoch is stored in the stream's Arrow schema metadata with key `writer_epoch` for fencing validation during replay. +### SSTable Storage Layout -#### WAL Storage Layout +An SSTable with generation `i` is written to: -Each WAL entry is stored within the WAL directory of the shard located at `_mem_wal/{shard_id}/wal`. +```text +_mem_wal/{shard_id}/{random8}_gen_{i}/ +``` -WAL files use bit-reversed 64-bit binary naming to distribute files evenly across the directory keyspace. -This optimizes S3 throughput by spreading sequential writes across S3's internal partitions, minimizing throttling. -The filename is the bit-reversed binary representation of the entry ID with suffix `.arrow`. -For example, entry ID 5 (binary `000...101`) becomes `1010000000000000000000000000000000000000000000000000000000000000.arrow`. +`{random8}` is an 8-character random hex value generated for each flush attempt. +If a flush attempt fails, a retry writes a different directory instead of reusing a partially written one. +The shard manifest records the successful directory name in the SSTable's `path`. + +The SSTable directory is a standard Lance dataset written with the base table's data storage version. +Each SSTable is written as one fragment. +Additional MemWAL sidecars may be present: + +```text +{random8}_gen_{i}/ +├── _versions/ +│ └── {version}.manifest +├── _deletions/ # Present when within-SSTable dedup deletes rows +├── _indices/ # Present when maintained user indexes are built +│ └── {index_uuid}/ +├── _pk_index/ # Primary-key sidecar BTree, not a manifest index +└── bloom_filter.bin # Primary-key bloom filter +``` -### Flushed MemTable +The exact Lance dataset internals follow the [Lance table storage layout](layout.md). -A flushed MemTable is created by flushing the MemTable to storage. -In Lance MemWAL spec, a flushed MemTable must be a Lance table following the Lance table format spec. +### SSTable Row Order -!!!note -This is called Sorted String Table (SSTable) or Sorted Run in many LSM-tree literatures and implementations. -However, since our MemTable is not sorted, we just use the term flushed MemTable to avoid confusion. +SSTable rows are written in forward insert order. +Physical row offsets increase with write time. +For a duplicate primary key within one SSTable, the newest row has the largest physical offset. -#### Flushed MemTable Storage Layout +Primary-key SSTables use a deletion vector to expose last-write-wins semantics. +During flush, the writer scans rows in forward order, keeps the last occurrence of each primary key, and marks all earlier duplicate offsets deleted. +The deletion vector is attached to fragment 0 in the SSTable's Lance manifest. -The MemTable of generation `i` is flushed to `_mem_wal/{shard_id}/{random_hex}_gen_{i}/` directory, -where `{random_hex}` is a random 8-character hex value generated at flush time. -The random hex value is necessary to ensure if one MemTable flush attempt fails, -The retry can use another directory. -The content within the generation directory follows the [Lance table storage layout](layout.md). +Append-only SSTables without a primary key do not perform primary-key deduplication and retain every row. -#### Merging MemTable to Base Table +### Tombstone Rows -Generation numbers determine merge order of flushed MemTable into base table: -lower numbers represent older data and must be merged to the base table first to preserve correct upsert semantics. +Delete operations are represented as rows with the internal `_tombstone` column. +Tombstone rows follow the same forward row ordering and deletion-vector rules as ordinary rows. +If the newest row for a primary key is a tombstone, the deletion vector keeps that tombstone row and hides older rows for the key. +Read planning then filters `_tombstone = false`, so the key is absent from query results. -Within a single flushed MemTable for a primary-key table, -if there are multiple rows of the same primary key, the row that is last inserted wins. -Append-only tables without a primary key retain all inserted rows. +### SSTable Primary-Key Sidecars -### Shard Manifest +Primary-key MemTables maintain an implicit BTree for primary-key deduplication, independent of `maintained_indexes`. +When a primary-key MemTable is flushed, the SSTable writes two primary-key sidecars: -Each shard has a manifest file. This is the source of truth for the state of a shard. +- `bloom_filter.bin` stores the SSTable's primary-key bloom filter and lets point lookups skip SSTables that cannot contain the queried key. +- `_pk_index/` stores a standalone BTree over primary-key values to forward row ids. -#### Shard Manifest Contents +The `_pk_index/` sidecar is not a maintained user index, is not registered in the SSTable's Lance manifest, and has no manifest UUID. +Its identity is its immutable SSTable path. +Readers open it directly from `{sstable_path}/_pk_index`. -The manifest contains: +The `_pk_index/` directory is a Lance scalar BTree index store: -- **Fencing state**: `writer_epoch` as the latest writer fencing token, see [Writer Fencing](#writer-fencing) for more details. -- **Shard assignment**: `shard_spec_id` and `shard_field_values` record how this shard maps to its sharding spec. `shard_field_values` is a map from shard field id to the raw Arrow scalar bytes of the computed value; the matching `ShardingField.result_type` in the `ShardingSpec` determines how to interpret each entry (e.g., 4 little-endian bytes for int32, raw UTF-8 bytes for utf8). -- **WAL pointers**: `replay_after_wal_entry_position` (last entry position flushed to MemTable, 0-based), `wal_entry_position_last_seen` (last entry position seen at manifest update, 0-based) -- **Generation trackers**: `current_generation` (next generation to flush), `flushed_generations` list of generation number and directory path pairs (e.g., generation 1 at `a1b2c3d4_gen_1`) +```text +_pk_index/ +├── page_data.lance +└── page_lookup.lance +``` -Note: `wal_entry_position_last_seen` is a hint that may be stale since it's not updated on WAL write. -It is updated opportunistically by any reader that can update the shard manifest. -The manifest itself is atomically written, but recovery must try to get newer WAL files to find the actual state beyond this hint. +Readers load this directory as a BTree index using `BTreeIndexDetails` with default parameters. +The primary-key index type is the Arrow type of the primary-key column for a single-column primary key, or `Binary` for a composite primary key. -The manifest is serialized as a protobuf binary file using the `ShardManifest` message. +The `page_lookup.lance` file has the following schema: -
-ShardManifest protobuf message +| Column | Type | Nullable | Description | +|--------------|-------------------------|----------|------------------------------------------------| +| `min` | {PrimaryKeyIndexType} | true | Minimum primary-key index value in the page | +| `max` | {PrimaryKeyIndexType} | true | Maximum primary-key index value in the page | +| `null_count` | UInt32 | false | Number of null values in the page | +| `page_idx` | UInt32 | false | Page number pointing into `page_data.lance` | -```protobuf -%%% mem_wal.message.ShardManifest %%% -``` +The `page_data.lance` file has the following schema: -
+| Column | Type | Nullable | Description | +|----------|-----------------------|----------|-------------------------------------------------------------------| +| `values` | {PrimaryKeyIndexType} | true | Sorted primary-key index values | +| `ids` | UInt64 | false | Forward row ids corresponding to each primary-key index value | -#### Shard Manifest Versioning +For a single-column primary key, the indexed value stores the primary-key scalar directly. +For a composite primary key, the indexed value stores an order-preserving binary tuple encoding of all primary-key columns in primary-key column order. +Each tuple column is encoded as: -Manifests are versioned starting from 1 and immutable. -Each update creates a new manifest file at the next version number. -Updates use put-if-not-exists or file rename to ensure atomicity depending on the storage system. -If two processes compete, one wins and the other retries. +- `0x00` for null. +- `0x01` followed by the non-null value encoding otherwise. -To commit a manifest version: +Supported non-null value encodings are: -1. Compute the next version number -2. Write the manifest to `{bit_reversed_version}.binpb` using put-if-not-exists -3. In parallel best-effort write to `version_hint.json` with `{"version": }` (failure is acceptable) +- Signed integers and date values: sign-flipped 8-byte big-endian integer bytes. +- Unsigned integers: 8-byte big-endian unsigned integer bytes. +- Boolean: one byte, `0x00` for false and `0x01` for true. +- UTF-8 and binary values: raw bytes, with each `0x00` byte escaped as `0x00 0xff`, followed by a `0x00 0x00` terminator. -To read the latest manifest version: +This encoding is injective and preserves primary-key tuple ordering under lexicographic byte comparison. +Composite primary-key columns must use one of the supported encodings above. -1. Read `version_hint.json` to get the latest version hint. If not found, start from version 1 -2. Check existence for subsequent versions from the starting version -3. Continue until a version is not found -4. The latest version is the last found version +The sidecar row ids are in the same forward row-position space as the data files, deletion vector, and maintained user indexes. +The sidecar is used for cross-generation membership and block-list checks. +It is not used to choose the newest row inside the same SSTable; the deletion vector has already hidden older same-generation duplicates. -!!!note -This works because the write rate to shard manifests is significantly lower than read rates. Shard manifests are only updated when shard metadata changes (MemTable flush), not on every write. This ensures HEAD requests will eventually terminate and find the latest version. +### Maintained User Indexes -#### Shard Manifest Storage Layout +When the MemWAL index lists `maintained_indexes`, flush may build matching indexes inside the SSTable. +These index files live in the SSTable's `_indices/{index_uuid}/` directory and are recorded in the SSTable's Lance manifest. +The implicit primary-key BTree sidecar is not included in `maintained_indexes` and does not live under `_indices/`. -All shard manifest versions are stored in `_mem_wal/{shard_id}/manifest` directory. +These indexes use the same row-position space as the forward-written data files. +If the SSTable has a primary key, its deletion vector masks stale duplicate rows for indexed reads as well. -Each shard manifest version file uses bit-reversed 64-bit binary naming, the same scheme as WAL files. -For example, version 5 becomes `1010000000000000000000000000000000000000000000000000000000000000.binpb`. +### SSTable Compaction -## MemWAL Index Details +SSTables are compacted into the base table in ascending generation order within each shard. +Lower generation numbers are older and must be compacted before higher generation numbers. +SSTable compaction uses merge-insert semantics so newer rows overwrite older rows for the same primary key. -The MemWAL Index uses the [standard index storage](../index/index.md#index-storage) at `_indices/{UUID}/`. +## Shard Manifest -The index stores its data in two parts: +Each shard has a versioned manifest. +The latest shard manifest is the source of truth for shard-local state. -1. **Index details** (`index_details` in `IndexMetadata`): Contains configuration, merge progress, and snapshot metadata -2. **Shard snapshots**: Stored as a Lance file or inline, depending on shard count +### Shard Manifest Contents -### Index Details +The manifest contains: + +- **Identity**: `shard_id`, `shard_spec_id`, and `shard_field_entries`. +- **Fencing state**: `writer_epoch`. +- **WAL pointers**: `replay_after_wal_entry_position` and `wal_entry_position_last_seen`. +- **SSTable generation state**: `current_generation` and `sstables`. +- **Lifecycle state**: `status`, either `ACTIVE` or `SEALED`. + +`shard_field_entries` stores computed shard field values as raw Arrow scalar bytes keyed by `ShardingField.field_id`. +The matching `ShardingField.result_type` determines how to decode each value. +For example, `int32` values are four little-endian bytes and `utf8` values are raw UTF-8 bytes. + +`replay_after_wal_entry_position` is the most recent 1-based WAL position covered by an SSTable. +The default value 0 means no WAL entry has been covered and recovery starts at position 1. + +`wal_entry_position_last_seen` is a best-effort hint for the most recent WAL position observed at manifest update time. +It is not authoritative because it is not updated on every WAL write. +Recovery must still probe or list WAL files to find the actual tail. -The `index_details` field in `IndexMetadata` contains a `MemWalIndexDetails` protobuf message with the following key fields: +`current_generation` is the generation number to assign to the next SSTable created by flushing the MemTable. +Each entry in `sstables` records a published SSTable's `generation` and `path`. -- **Configuration fields** (`sharding_specs`, `maintained_indexes`) are the source of truth for MemWAL configuration. - Writers read these fields to determine how to partition data and which indexes to maintain. -- **Merge progress** (`merged_generations`) tracks the last generation merged to the base table for each shard. - This field is updated atomically with merge-insert data commits, enabling conflict resolution when multiple mergers operate concurrently. - Each entry contains the shard UUID and generation number. -- **Index catchup progress** (`index_catchup`) tracks which merged generation each base table index has been rebuilt to cover. - When data is merged from a flushed MemTable to the base table, the base table's indexes may be rebuilt asynchronously. - During this window, queries should use the flushed MemTable's pre-built indexes instead of scanning unindexed data in the base table. - See [Indexed Read Plan](#indexed-read-plan) for details. -- **Shard snapshot fields** (`snapshot_ts_millis`, `num_shards`, `inline_snapshots`) provide a snapshot of shard states. - The actual shard manifests remain authoritative for shard state. - When `num_shards` is 0, the `inline_snapshots` field may be `None` or an empty Lance file with 0 rows but proper schema. +`status = SEALED` marks a reversible in-flight drop-table operation. +Sealed shards refuse new writer claims. + +The manifest is serialized as the `ShardManifest` protobuf message.
-MemWalIndexDetails protobuf message +ShardManifest protobuf message ```protobuf -%%% mem_wal.message.MemWalIndexDetails %%% +%%% mem_wal.message.ShardManifest %%% ```
-### Shard Identifier +### Shard Manifest Versioning -Each shard has a unique UUID identifier within the table. -When a new shard is created, implementations may assign either a random UUID or -a deterministic UUID derived from the shard assignment when deterministic -writer fencing is required. +Manifest versions start at 1. +Each update writes a new immutable protobuf file: -### Shard Discovery +```text +_mem_wal/{shard_id}/manifest/{bit_reversed_version}.binpb +``` -The MemWAL index can store shard snapshots for read optimization, but those snapshots may lag the latest shard set. -Implementations that need to discover the current shard set should list `_mem_wal/` shard directories and read each shard's latest [shard manifest](#shard-manifest). +Writers use put-if-not-exists or atomic rename, depending on storage support. +If two processes race to write the same next version, one wins and the other reloads and retries. -Each shard manifest records the shard UUID, sharding spec ID, and computed shard field values needed to map the shard back to a sharding spec assignment. +After a successful version write, the writer best-effort updates: -### Sharding Spec +```json +{"version": } +``` -A **Sharding Spec** defines how all rows in a table are logically divided into different shards, -enabling automatic shard assignment and query-time shard pruning. +in: -Each sharding spec has: +```text +_mem_wal/{shard_id}/manifest/version_hint.json +``` -- **Spec ID**: A positive integer that uniquely identifies this spec within the MemWAL index. IDs are never reused. -- **Sharding fields**: An array of field definitions that determine how to compute shard values. +Readers use `version_hint.json` as a starting point and then probe subsequent versions until a version is missing. +The latest manifest is the last existing version. -Each shard is bound to a specific sharding spec ID, recorded in its [manifest](#shard-manifest). -Shards without a spec ID (`spec_id = 0`) are manually-created shards not governed by any spec. +## MemWAL Index Details -A sharding spec's field array consists of **sharding field** definitions. -Each sharding field has the following properties: +The MemWAL index is stored inline in the base table's `IndexMetadata`. +It is a system index with no file directory. +The `index_details` field contains a `MemWalIndexDetails` protobuf message. -| Property | Description | -| ------------- | ------------------------------------------------------------------------- | -| `field_id` | Unique string identifier for this sharding field | -| `source_ids` | Array of field IDs referencing source columns in the schema | -| `transform` | A well-known shard expression, specify this or `expression` | -| `expression` | A DataFusion SQL expression for custom logic, specify this or `transform` | -| `result_type` | The output type of the shard value | +Important fields: -#### Shard Expression +- `sharding_specs`: sharding configuration used by writers and shard pruning. +- `maintained_indexes`: names of base-table indexes to maintain in MemTables and SSTables. +- `writer_config_defaults`: string map of default writer configuration values persisted for all writers. +- `compacted_sstables`: per-shard compaction progress, updated atomically with base-table compaction commits. +- `index_catchup`: per-index coverage progress after data has been compacted into the base table. +- `snapshot_ts_millis`, `num_shards`, and `inline_snapshots`: optional shard snapshot fields for read optimization. -A **Shard Expression** is a [DataFusion SQL expression](https://datafusion.apache.org/user-guide/sql/index.html) that derives a shard value from source column(s). -Source columns are referenced as `col0`, `col1`, etc., corresponding to the order of field IDs in `source_ids`. +What an absent entry means depends on the `FLAG_MEM_WAL_INDEX_CATCHUP` feature bit: -Shard expressions must satisfy the following requirements: +- **Without the bit**, a shard absent from `index_catchup` for an index means that index is assumed fully caught up for the shard. +- **With the bit**, absence means the opposite: the index is *not* known to have caught up, so the shard's SSTables must be retained until some commit records that it has. -1. **Deterministic**: The same input value must always produce the same output value. -2. **Stateless**: The expression must not depend on external state (e.g., current time, random values, session variables). -3. **Type-promotion resistant**: The expression must produce the same result for equivalent values regardless of their numeric type (e.g., `int32(5)` and `int64(5)` must yield the same shard value). -4. **Column removal resistant**: If a source field ID is not found in the schema, the column should be interpreted as NULL. -5. **NULL-safe**: The expression should properly handle NULL inputs and have defined behavior (e.g., return NULL if input is NULL for single-column expressions). -6. **Consistent with result type**: The expression's return type must be consistent with `result_type` in non-NULL cases. +Catch-up is derived at commit time, not reported by the writer. An index whose segments together span every fragment live at the transaction's read version holds every row compaction had copied into the base table by then, so the commit records it as caught up to that version's `compacted_sstables`. That is the only proof available — nothing maps a compaction generation to the fragments its rows landed in — so covering the table as the transaction read it is how an index shows it covered those rows. Fragments appended since that read are a later catch-up gap and are not required. -#### Shard Transform +Two rules bound what a commit may record. It never credits more than its own `compacted_sstables`, and it clamps to that value, so a position can only describe generations the base table has actually taken in. Otherwise it never lowers a position an index already held, provided that index is unchanged by this commit. "Unchanged" compares each segment's UUID together with its fragment bitmap, not the UUID alone, because an operation can prune the bitmap in place while keeping the UUID; the remaining metadata does not affect which rows the index answers for. An index this commit changes keeps no position it cannot re-earn. -A **Shard Transform** is a well-known shard expression with a predefined name. -When a transform is specified, the expression is derived automatically. +Because the position is derived rather than transmitted, it cannot go stale between inspection and commit, and it survives rebase — `read_version` is fixed for a transaction's life, so what a commit can prove does not move, though a rebased attempt may record a different result because the head it commits against has changed. Any commit can earn a position, so an ordinary index build that happens to cover the table records catch-up as a side effect. A dedicated repair is still needed where no such commit occurs, or where an index does not yet span the table. -| Transform | Parameters | Shard Expression | Result Type | -| -------------- | ------------- | --------------------------------------------------------- | -------------- | -| `identity` | (none) | `col0` | same as source | -| `year` | (none) | `date_part('year', col0)` | `int32` | -| `month` | (none) | `date_part('month', col0)` | `int32` | -| `day` | (none) | `date_part('day', col0)` | `int32` | -| `hour` | (none) | `date_part('hour', col0)` | `int32` | -| `bucket` | `num_buckets` | `abs(murmur3(col0)) % N` | `int32` | -| `multi_bucket` | `num_buckets` | `abs(murmur3_multi(col0, col1, ...)) % N` | `int32` | -| `truncate` | `width` | `left(col0, W)` (string) or `col0 - (col0 % W)` (numeric) | same as source | +A read version with no fragments proves nothing, even though an index trivially covers an empty table. An empty fragment list is also what a manifest written before the `UpdateMemWalState` fragment fix looks like, where the SSTables are the last copy of those rows; crediting coverage there would retire them. The cost is that a table whose rows have all been deleted keeps its SSTables. -The `bucket` and `multi_bucket` transforms use Murmur3 hash functions: +Setting the bit is one-way. Once SSTables have stopped being served against a recorded catch-up position, reading absence as "caught up" again could drop rows that only those SSTables still hold, so the bit is never cleared as a rollback. Writers must also hold the bit: a writer that does not maintain `index_catchup` can change an index without withdrawing the position recorded for it, leaving a position that no longer describes the index it names. -- **`murmur3(col)`**: Computes the 32-bit Murmur3 hash (x86 variant, seed 0) of a single column. Returns a signed 32-bit integer. Returns NULL if input is NULL. -- **`murmur3_multi(col0, col1, ...)`**: Computes the Murmur3 hash across multiple columns. Returns a signed 32-bit integer. NULL fields are ignored during hashing; returns NULL only if all inputs are NULL. +Shard snapshots, when present, use the following Lance file schema: -The hash result is wrapped with `abs()` and modulo `N` to produce a non-negative bucket number in the range `[0, N)`. +| Column | Type | Nullable | Description | +|----------------------------|------------------------------|----------|--------------------------------------------------------| +| `shard_id` | Utf8 | false | Shard UUID string | +| `shard_spec_id` | UInt32 | false | Sharding spec that produced the shard | +| `shard_field_{field_id}` | `ShardingField.result_type` | false | Computed shard field value for the given sharding field | -### Shard Snapshot Storage +The MemWAL index data is stored inline. +Readers discover the latest shard set by listing `_mem_wal/` shard directories and reading shard manifests. -Shard snapshots are stored using one of two strategies based on the number of shards: +
+MemWalIndexDetails protobuf message -| Shard Count | Storage Strategy | Location | -| ------------------ | ------------------- | ----------------------------------------- | -| <= 100 (threshold) | Inline | `inline_snapshots` field in index details | -| > 100 | External Lance file | `_indices/{UUID}/index.lance` | +```protobuf +%%% mem_wal.message.MemWalIndexDetails %%% +``` -The threshold (100 shards) is implementation-defined and may vary. +
+ +## Sharding + +A **ShardingSpec** defines how rows map to shards. +Each spec has a positive `spec_id` and one or more `ShardingField` entries. +Each shard manifest records the `shard_spec_id` and the computed shard field values for that shard. +`spec_id = 0` means the shard was manually created and is not governed by a sharding spec. -**Inline storage**: For small shard counts, snapshots are serialized as a Lance file and stored in the `inline_snapshots` field. -This keeps the index metadata compact while avoiding an additional file read for common cases. +Each `ShardingField` contains: -**External Lance file**: For large shard counts, snapshots are stored as a Lance file at `_indices/{UUID}/index.lance`. -This file uses standard Lance format with the shard snapshot schema, enabling efficient columnar access and compression. +- `field_id`: stable identifier for the computed shard field. +- `source_ids`: field IDs of source columns in the Lance schema. +- `transform`: well-known transform name, when using built-in transform evaluation. +- `expression`: reserved custom expression text, mutually exclusive with `transform`. +- `result_type`: Arrow type name for the computed value. +- `parameters`: transform-specific string parameters. -### Shard Snapshot Arrow Schema +The supported built-in transforms are: -Shard snapshots are stored as a Lance file with one row per shard. -The snapshot schema is optimized for shard discovery. Full mutable shard state -remains in the authoritative shard manifest files. +- `unsharded`: takes no source columns, always returns `int32` value 0, and creates one shard. +- `bucket`: takes one source column and `num_buckets`, hashes the value, and returns an `int32` bucket id in `[0, num_buckets)`. +- `identity`: takes one source column and returns the raw scalar value as the shard value. -| Column | Type | Description | -| ------------------------ | ------------- | ---------------------------------------------------------------------------------------------------------- | -| `shard_id` | `utf8` | Shard UUID string | -| `shard_spec_id` | `uint32` | Sharding spec ID (0 if manual) | -| `shard_field_{field_id}` | varies | One column per sharding field defined in the sharding spec, typed to match the field's `ShardingField.result_type`. | +`bucket` computes a deterministic 32-bit hash with seed 0 and then computes: + +```text +(hash & i32::MAX) % num_buckets +``` -For example, with a sharding spec containing a field `user_bucket` of type `int32`: +`num_buckets` must be in `[1, 1024]`. +Null bucket values hash to 0 and therefore map to bucket 0. +See [Appendix 3: Bucket Hashing](#appendix-3-bucket-hashing) for the exact hash algorithm and test vectors. -| Column | Type | Description | -| -------------------------- | ------- | ---------------------------- | -| ... | ... | (base columns above) | -| `shard_field_user_bucket` | `int32` | Bucket value for this shard | +The `bucket` transform supports scalar boolean, integer, floating-point, date32, time, timestamp, utf8, and large_utf8 source types. +The `identity` transform supports scalar boolean, integer, utf8, and large_utf8 source types. -This schema records the fields needed to map each shard back to its sharding spec -assignment. Readers that need fencing epochs, WAL positions, or flushed -generation state must read the latest shard manifests directly. +The `year`, `month`, `day`, `hour`, `multi_bucket`, and `truncate` transform names are not supported MemWAL sharding transforms and must not be used in `ShardingSpec.transform`. ## Storage Layout -Here is a recap of the storage layout with all the files and concepts defined so far: +The MemWAL storage layout is: -``` +```text {table_path}/ +├── _versions/ +│ └── ... # Base table manifests, including __lance_mem_wal index metadata ├── _indices/ -│ └── {index_uuid}/ # MemWAL Index (uses standard index storage) -│ └── index.lance # Serialized shard snapshots (Lance file) -│ +│ └── ... # Ordinary base table index files; MemWAL index has no files └── _mem_wal/ - └── {shard_id}/ # Shard directory (UUID v4) + └── {shard_id}/ ├── manifest/ - │ ├── {bit_reversed_version}.binpb # Serialized shard manifest (bit-reversed naming) - │ └── version_hint.json # Version hint file + │ ├── {bit_reversed_version}.binpb + │ └── version_hint.json ├── wal/ - │ ├── {bit_reversed_entry_id}.arrow # WAL data files (bit-reversed naming) + │ ├── {bit_reversed_position}.arrow │ └── ... - └── {random_hash}_gen_{i}/ # Flushed MemTable (generation i, random prefix) + └── {random8}_gen_{generation}/ ├── _versions/ - │ └── {version}.manifest # Table manifest (V2 naming scheme) - ├── _indices/ # Indexes - │ ├── {vector_index}/ - │ └── {scalar_index}/ - └── bloom_filter.bin # Primary key bloom filter + │ └── {version}.manifest + ├── _deletions/ + ├── _indices/ + │ └── {index_uuid}/ + ├── _pk_index/ + └── bloom_filter.bin ``` -## Implementation Expectation - -This specification describes the storage layout for the LSM tree architecture. Implementations are free to use any approach to fulfill the storage layout requirements. Once data is written to the expected storage layout, the reader and writer expectations apply. +Some SSTable subdirectories are conditional. +For example, `_deletions/` is present only when the SSTable's Lance manifest references a deletion vector, `_indices/` is present only when maintained user indexes are built, and `_pk_index/` plus `bloom_filter.bin` are meaningful for primary-key tables. -The specification defines: - -- **Storage layout**: The directory structure, file formats, and naming conventions for WAL entries, flushed MemTables, shard manifests, and the MemWAL index -- **Durability guarantees**: How data is persisted through WAL entries and flushed MemTables -- **Consistency model**: How readers and writers coordinate through manifests and epoch-based fencing +## Implementation Expectation -Implementations may choose different approaches for: +This document specifies the storage layout and observable reader and writer invariants. +Implementations may choose different in-memory structures, buffering policies, background scheduling, and query execution plans. -- In-memory data structures and indexing -- Buffering strategies before WAL flush -- Background task scheduling and concurrency -- Query execution strategies +An implementation is compatible when it: -As long as the storage layout is correct and the documented invariants are maintained, implementations can optimize for their specific use cases. +1. Writes WAL entries, shard manifests, SSTables, and MemWAL index metadata using the documented layout. +2. Preserves WAL position, writer fencing, and manifest versioning invariants. +3. Exposes last-write-wins semantics for primary-key tables. +4. Preserves append-only semantics for tables without primary keys. +5. Maintains generation ordering when compacting SSTables into the base table. ## Writer Expectations -A writer operates on a single shard and is responsible for: +A writer operates on one shard and is responsible for: -1. Claiming the shard using epoch-based fencing -2. Writing data to WAL entries and flushed MemTables following the [storage layout](#storage-layout) -3. Maintaining the shard manifest to track WAL and generation progress +1. Claiming the shard with epoch-based fencing. +2. Appending WAL entries in sequential 1-based positions. +3. Maintaining in-memory MemTable state. +4. Flushing MemTables to SSTable Lance datasets. +5. Updating the shard manifest after an SSTable is durably written. ### Writer Fencing -Writers use epoch-based fencing to ensure single-writer semantics per shard. +Writers use `writer_epoch` to enforce single-writer semantics per shard. To claim a shard: -1. Load the latest shard manifest -2. Increment `writer_epoch` by one -3. Atomically write a new manifest version -4. If the write fails (another writer claimed the epoch), reload and retry with a higher epoch +1. Load the latest shard manifest. +2. Verify the shard is `ACTIVE`. +3. Increment `writer_epoch`. +4. Atomically write a new manifest version. +5. If the manifest write loses a race, reload and retry. -Before any manifest update, a writer must verify its `writer_epoch` remains valid: +Before a manifest update, a writer verifies its local epoch is still current: -- If `local_writer_epoch == stored_writer_epoch`: The writer is still active and may proceed -- If `local_writer_epoch < stored_writer_epoch`: The writer has been fenced and must abort +- If `local_writer_epoch == stored_writer_epoch`, the writer may proceed. +- If `local_writer_epoch < stored_writer_epoch`, the writer has been fenced and must abort. -For a concrete example, see [Appendix 1: Writer Fencing Example](#appendix-1-writer-fencing-example). +WAL append conflicts also detect fencing. +If an older writer collides with a newer writer's WAL entry at the same position, it reloads the manifest and observes the higher epoch. +Fence sentinel entries make this collision path explicit without storing data batches. ## Background Job Expectations -Background jobs handle merging flushed MemTables to the base table and garbage collection. +Background jobs compact SSTables into the base table and remove obsolete shard data. -### MemTable Merger +### SSTable Compactor -Flushed MemTables must be merged to the base table in **ascending generation order** within each shard. This ordering is essential for correct upsert semantics: newer generations must overwrite older ones. +SSTables must be compacted into the base table in ascending generation order within each shard. +The compaction uses Lance merge-insert semantics and updates `compacted_sstables[shard_id]` atomically with the base-table commit. -The merge uses Lance's merge-insert operation with atomic transaction semantics: +On commit conflict, a compactor reloads the conflicting base-table version: -- `merged_generations[shard_id]` is updated atomically with the data commit -- On commit conflict, check the conflicting commit's `merged_generations` to determine if the generation was already merged - -For a concrete example, see [Appendix 2: Concurrent Merger Example](#appendix-2-concurrent-merger-example). +- If the committed `compacted_sstables[shard_id]` is already greater than or equal to the generation being compacted, the compactor skips that generation. +- Otherwise, the compactor retries from the latest base-table version. ### Garbage Collector -The garbage collector removes obsolete data from shard directories. Flushed MemTables and their referenced WAL files may be deleted after: - -1. The generation has been merged to the base table (`generation <= merged_generations[shard_id]`) -2. All maintained indexes have caught up (`generation <= min(index_catchup[I].caught_up_generation)`) -3. No retained base table version references the generation for time travel +The garbage collector may remove obsolete SSTables after: -!!!warning - Deleting WAL files weakens [writer fencing](#writer-fencing) and can lead to silent acknowledgement of lost writes. +1. The SSTable has been compacted into the base table. +2. Every index a query may rely on has caught up to cover the SSTable's generation, or the SSTable is no longer needed for indexed reads. With `FLAG_MEM_WAL_INDEX_CATCHUP` set, an index absent from `index_catchup` has *not* caught up, so this condition is not met for it. +3. No retained base-table version needs the SSTable for time travel or consistency. - Fencing detects a stalled writer when its `put-if-not-exists` for the next WAL entry collides with a newer writer's entry at the same position — only that collision triggers the epoch check. If GC has already removed the WAL file at that position, the stalled writer's PUT lands on empty space and succeeds against its old `writer_epoch`. The entry is acknowledged to the client, but the new manifest's `replay_after_wal_entry_position` has already advanced past it, so the data is never replayed. +!!! warning + Deleting WAL files can weaken writer fencing. - Implementations that GC WAL files must compensate, for example by re-checking fence state after each successful WAL write, encoding the writer epoch into the WAL filename so positions are partitioned by epoch, or otherwise guaranteeing a stalled writer cannot land at a position that has been or will be GC'd. + Fencing detects a stalled writer when its put-if-not-exists for the next WAL entry collides with a newer writer's entry at the same position. + If garbage collection has removed that WAL file, the stalled writer may write into empty space with an old `writer_epoch`. + Implementations that garbage collect WAL files must compensate by re-checking fence state after WAL writes, partitioning WAL positions by epoch, or otherwise preventing stale writers from landing at positions that have been garbage collected. ## Reader Expectations ### LSM Tree Merging Read -For tables with a primary key, readers **MUST** merge results from multiple data sources -(base table, flushed MemTables, in-memory MemTables) by primary key to ensure correctness. - -When the same primary key exists in multiple sources, the reader must keep only the newest version based on: - -1. **Generation number** (`_gen`): Higher generation wins. The base table has generation 0, MemTables have positive integers starting from 1. -2. **Row address** (`_rowaddr`): Within the same generation, higher row address wins (later writes within a batch overwrite earlier ones). - -The ordering for "newest" is: highest `_gen` first, then highest `_rowaddr`. +For primary-key tables, readers merge rows from the base table, SSTables, and optionally in-memory MemTables by primary key. +The newest row wins. -This deduplication is essential because: +Freshness ordering within one shard is: -- A row updated in a MemTable also exists (with older data) in the base table -- A flushed MemTable that has been merged to the base table may not yet be garbage collected, causing the same row to appear in both -- A single write batch may contain multiple updates to the same primary key +1. The active MemTable wins over every SSTable and the base table. +2. Among SSTables, higher generation wins. +3. Any uncompacted SSTable wins over the base table. +4. Within the active MemTable, higher row position wins. +5. Within an SSTable, its deletion vector has already hidden older duplicate primary-key rows. -Without proper merging, queries would return duplicate or stale rows. +For freshness comparisons, the base table uses the sentinel generation 0. +SSTable generations are positive. +This ordering applies only to sources selected for the same read plan. +Readers must not include an SSTable that is already covered by the base table according to `compacted_sstables[shard_id]`, because otherwise the positive SSTable generation would incorrectly outrank base-table rows during deduplication. +Rows from different shards do not need primary-key deduplication if the sharding spec guarantees that each primary key maps to exactly one shard. Append-only tables without a primary key do not perform primary-key deduplication. -Readers should include the relevant base table, flushed MemTables, and in-memory MemTables -according to the requested consistency level; duplicate values are treated as distinct appended rows. +Rows from all selected sources are distinct appended rows. -### Reader Consistency - -Reader consistency depends on two factors: +### Tombstones -1. access to in-memory MemTables -2. the source of shard metadata (either through MemWAL index or shard manifests) +Readers must treat `_tombstone = true` rows as delete markers. +In SSTables, deletion vectors first resolve same-generation duplicate primary keys. +Then query planning filters tombstone rows from user-visible results. +In active in-memory MemTables, the newest visible row position for a primary key wins; if that row is a tombstone, the key is absent. -Strong consistency requires access to in-memory MemTables for all shards involved in the query and reading shard manifests directly. -Otherwise, the query is eventually consistent due to missing unflushed data or stale MemWAL Index snapshots. - -!!!note -Reading a stale MemWAL Index does not impact correctness, only freshness: +### Reader Consistency - - **Merged MemTable still in index**: If a flushed MemTable has been merged to the base table but still shows in the MemWAL index, readers query both. This results in some inefficiency for querying the same data twice, but [LSM-tree merging](#lsm-tree-merging-read) ensures correct results since both contain the same data. The inefficiency is also compensated by the fact that the data is covered by index and we rarely end up scanning both data. - - **Garbage collected MemTable still in index**: If a flushed MemTable has been garbage collected, but is still in the MemWAL index, readers would fail to open it and skip it. This is also safe because if it is garbage collected, the data must already exist in the base table. - - **Newly flushed MemTable not in index**: If a newly flushed MemTable is added after the snapshot was built, it is not queried. The result is eventually consistent but correct for the snapshot's point in time. +Reader consistency depends on: -### Query Planning +1. Whether the reader can access active in-memory MemTables. +2. Whether shard metadata comes from latest shard manifests or from an older MemWAL index snapshot. -#### MemTable Collection +Strong consistency requires active in-memory MemTable access for relevant shards and direct reads of latest shard manifests. +Otherwise, reads are eventually consistent because unflushed data or newly-created shards may be absent from the read plan. -The query planner collects datasets from multiple sources and assembles them for unified query execution. -Datasets come from: +Reading a stale MemWAL index snapshot does not corrupt last-write-wins ordering, but it can reduce freshness: -1. base table (representing already-merged data) -2. flushed MemTables (persisted but not yet merged) -3. optionally in-memory MemTables (if accessible). +- If a compacted SSTable is still listed, readers must skip it when `generation <= compacted_sstables[shard_id]`. + For primary-key tables, including it would let an older SSTable row outrank newer base-table contents because SSTable generations are positive and the base table is modeled as generation 0. + For append-only tables, including it would return the same append twice. +- If a garbage-collected SSTable is still listed, readers may skip it after failing to open it because its data must already be in the base table or be filtered out by `compacted_sstables`. +- If a new SSTable is not listed, the read is consistent with the older snapshot but may miss fresher data. -Each dataset is tagged with a generation number: 0 for the base table, and positive integers for MemTable generations. -Within a shard, the generation number determines data freshness, with higher numbers representing newer data. -For primary-key tables, rows from different shards do not need deduplication -since each primary key maps to exactly one shard. -Append-only tables without a primary key do not require cross-shard primary-key deduplication. +Readers that require latest shard membership should list `_mem_wal/` and read shard manifests instead of relying only on snapshots. -The planner also collects bloom filters from each generation for staleness detection during search queries. +### Query Planning -#### Shard Pruning +A query planner collects sources from: -Before executing queries, if sharding spec is available, -the planner evaluates filter predicates against sharding specs to determine which shards may contain matching data. -This pruning step reduces the number of shards to scan. +1. The base table. +2. SSTables that are not yet safely replaceable by base-table indexed reads. +3. Active in-memory MemTables, when available and required by the requested consistency level. -For each filter predicate: +Each source is tagged with its shard and freshness tier. +SSTable sources are also tagged with their generation. +For primary-key reads, the planner applies LSM deduplication across selected sources. +For append-only reads, the planner concatenates selected sources without primary-key deduplication. -1. Extract predicates on columns used in sharding specs -2. Evaluate which shard values can satisfy the predicate -3. Prune shards whose values cannot match +Bloom filters and `_pk_index/` sidecars help prune SSTables during point lookups and cross-generation deduplication. -For example, with a sharding spec using `bucket(user_id, 10)` and a filter `user_id = 123`: +### Shard Pruning -1. Compute `bucket(123, 10) = 3` -2. Only scan shards with bucket value 3 -3. Skip all other shards +When sharding specs are available, the planner evaluates query predicates against shard fields and skips shards whose computed shard values cannot match. -Shard pruning applies to both scan queries and prefilters in search queries. +For example, with `bucket(user_id, 10)` and predicate `user_id = 123`: -#### Indexed Read Plan +1. Compute the bucket id for `123`. +2. Scan only shards whose manifest has the same computed bucket value. +3. Skip all other bucket shards. -When data is merged from a flushed MemTable to the base table, the base table's indexes are rebuilt asynchronously by the base table index builders. -During this window, the merged data exists in the base table but is not yet covered by the base table's indexes. +### Indexed Read Plan -Without special handling, indexed queries would fall back to expensive full scans for the unindexed part of the base table. -To maintain indexed read performance, the query planner should use `index_catchup` progress to determine the optimal data source for each query. +When data is compacted from an SSTable into the base table, base-table indexes may lag behind the data commit. +`index_catchup` records which compacted generation each base-table index covers. -The key insight is that flushed MemTables serve as a bridge between the base table's index catchup and the current merged state. -For a query that requires a specific index for acceleration, when `index_gen < merged_gen`, -the generations in the gap `(index_gen, merged_gen]` have data already merged in the base table but are not covered by the base table's index. -Since flushed MemTables contain pre-built indexes (created during [MemTable flush](#flushed-memtable)), queries can use these indexes instead of scanning unindexed data in the base table. -This ensures all reads remain indexed regardless of how far behind the async index builder is. +If an indexed query needs index `I` and `I` has only caught up to generation `G` while `compacted_sstables[shard_id]` is higher, the planner should read the gap from SSTable indexes instead of scanning unindexed base-table rows. +Once index `I` catches up, the planner can use the base-table index for those compacted rows. ## Appendices ### Appendix 1: Writer Fencing Example -This example demonstrates how epoch-based fencing prevents data corruption when two writers compete for the same shard. - -#### Initial State +Initial shard manifest: +```text +version: 1 +writer_epoch: 5 +replay_after_wal_entry_position: 10 +wal_entry_position_last_seen: 12 +status: ACTIVE ``` -Shard manifest (version 1): - writer_epoch: 5 - replay_after_wal_entry_position: 10 - wal_entry_position_last_seen: 12 -``` - -#### Scenario - -| Step | Writer A | Writer B | Manifest State | -| ---- | --------------------------------------------- | ----------------------------------------- | ------------------ | -| 1 | Loads manifest, sees epoch=5 | | epoch=5, version=1 | -| 2 | Increments to epoch=6, writes manifest v2 | | epoch=6, version=2 | -| 3 | Starts writing WAL entries 13, 14, 15 | | | -| 4 | | Loads manifest v2, sees epoch=6 | epoch=6, version=2 | -| 5 | | Increments to epoch=7, writes manifest v3 | epoch=7, version=3 | -| 6 | | Starts writing WAL entries 16, 17 | | -| 7 | Tries to flush MemTable, loads manifest | | | -| 8 | Sees epoch=7, but local epoch=6 | | | -| 9 | **Writer A is fenced!** Aborts all operations | | | -| 10 | | Continues writing normally | epoch=7, version=3 | - -#### What Happens to Writer A's WAL Entries? - -Writer A wrote WAL entries 13, 14, 15 with `writer_epoch=6` in their schema metadata. - -When Writer B performs crash recovery or MemTable flush: - -1. Reads WAL entries sequentially starting from `replay_after_wal_entry_position + 1` (entry 11, since positions are 0-based) -2. For each entry, checks existence using HEAD request on the bit-reversed filename -3. Continues until an entry is not found (e.g., entry 18 doesn't exist) -4. Finds entries 13, 14, 15, 16, 17 -5. Reads each file's `writer_epoch` from schema metadata -6. Entries 13, 14, 15 have `writer_epoch=6` which is <= current epoch (7) -> **valid, will be replayed** -7. Entries 16, 17 have `writer_epoch=7` -> **valid, will be replayed** -#### Key Points +Writer A loads version 1, claims epoch 6, and writes manifest version 2. +It appends WAL entries 13, 14, and 15 with `writer_epoch = 6`. -1. **No data loss**: Writer A's entries are not discarded. They were written with a valid epoch at the time and will be included in recovery. +Writer B then loads version 2, claims epoch 7, and writes manifest version 3. +It appends WAL entries 16 and 17 with `writer_epoch = 7`. -2. **Consistency preserved**: Writer A is prevented from making further writes that could conflict with Writer B. +When Writer A later tries to flush or update the shard manifest, it reloads the manifest and sees stored epoch 7 while its local epoch is 6. +Writer A is fenced and must abort. -3. **Orphaned files are safe**: WAL files from fenced writers remain on storage and are replayed by the new writer. They are only garbage collected after being included in a flushed MemTable that has been merged. +Recovery starts from `replay_after_wal_entry_position + 1`, which is entry 11. +Entries 13, 14, 15, 16, and 17 are valid replay inputs because they were written by epochs that were valid at write time and are not greater than the current shard epoch. -4. **Epoch validation timing**: Writers check their epoch before manifest updates (MemTable flush), not on every WAL write. This keeps the hot path fast while ensuring consistency at commit boundaries. +### Appendix 2: Concurrent Compactor Example -### Appendix 2: Concurrent Merger Example +Initial state: -This example demonstrates how MemWAL Index and conflict resolution handle concurrent mergers safely. +```text +MemWAL index: + compacted_sstables: {shard: 5} -#### Initial State - -``` -MemWAL Index: - merged_generations: {shard: 5} - -Shard manifest (version 1): +Shard manifest: current_generation: 8 - flushed_generations: [(6, "abc123_gen_6"), (7, "def456_gen_7")] + sstables: + - generation: 6, path: "abc12345_gen_6" + - generation: 7, path: "def67890_gen_7" ``` -#### Scenario 1: Racing on the Same Generation - -Two mergers both try to merge generation 6 concurrently. - -| Step | Merger A | Merger B | MemWAL Index | -| ---- | ------------------------- | ------------------------------ | ---------------- | -| 1 | Reads index: merged_gen=5 | | merged_gen=5 | -| 2 | Reads shard manifest | | | -| 3 | Starts merging gen 6 | | | -| 4 | | Reads index: merged_gen=5 | merged_gen=5 | -| 5 | | Reads shard manifest | | -| 6 | | Starts merging gen 6 | | -| 7 | Commits (merged_gen=6) | | **merged_gen=6** | -| 8 | | Tries to commit | | -| 9 | | **Conflict**: reads new index | | -| 10 | | Sees merged_gen=6 >= 6, aborts | | -| 11 | | Reloads, continues to gen 7 | | +Two compactors both try to compact the SSTable at generation 6. +Compactor A commits first and updates `compacted_sstables[shard]` to 6 in the same base-table commit as the data. +Compactor B then hits a commit conflict, reloads the latest MemWAL index, sees `compacted_sstables[shard] >= 6`, skips generation 6, and continues with generation 7. -Merger B's conflict resolution detected that generation 6 was already merged by checking the MemWAL Index in the conflicting commit. +The MemWAL index is the authoritative compaction-progress record because it is committed atomically with the base-table data changes. -#### Scenario 2: Crash After Table Commit +### Appendix 3: Bucket Hashing -Merger A crashes after committing to the table. +The bucket transform hash uses 32-bit wrapping arithmetic with these mixing functions. +Right shifts in `fmix` are logical shifts of the `u32` bit pattern. -| Step | Merger A | Merger B | MemWAL Index | -| ---- | ------------------------- | -------------------------------- | ---------------- | -| 1 | Reads index: merged_gen=5 | | merged_gen=5 | -| 2 | Merges gen 6, commits | | **merged_gen=6** | -| 3 | **CRASH** | | merged_gen=6 | -| 4 | | Reads index: merged_gen=6 | merged_gen=6 | -| 5 | | Reads shard manifest | | -| 6 | | **Skips gen 6** (already merged) | | -| 7 | | Merges gen 7, commits | **merged_gen=7** | - -The MemWAL Index is the single source of truth. Merger B correctly used it to determine that generation 6 was already merged. - -#### Key Points +```text +mix_k1(k) = rotl32(k * 0xcc9e2d51, 15) * 0x1b873593 +mix_h1(h, k) = rotl32(h ^ k, 13) * 5 + 0xe6546b64 +fmix(h, len) = + h = h ^ len + h = (h ^ (h >> 16)) * 0x85ebca6b + h = (h ^ (h >> 13)) * 0xc2b2ae35 + h ^ (h >> 16) +``` -1. **Single source of truth**: `merged_generations` is the authoritative source for merge progress, updated atomically with data. +Signed and unsigned casts use two's-complement wrapping. +Values are normalized and hashed as follows: + +- `bool`: `false` as `0`, `true` as `1`, then `hash_i32`. +- `int8`, `int16`, `int32`, `uint8`, `uint16`, `uint32`, `date32`, `time32`: cast to `i32`, then `hash_i32`. +- `int64`, `uint64`, `timestamp`, `time64`: cast to `i64`, then `hash_i64`. +- `float32`: `-0.0` and `+0.0` normalize to bits `0`; all NaNs normalize to `0x7fc00000`; other values use IEEE 754 bits cast to `i32`, then `hash_i32`. +- `float64`: `-0.0` and `+0.0` normalize to bits `0`; all NaNs normalize to `0x7ff8000000000000`; other values use IEEE 754 bits cast to `i64`, then `hash_i64`. +- `utf8` and `large_utf8`: hash the UTF-8 bytes with `hash_bytes`. + +The helper hashes are: + +```text +hash_i32(v) = fmix(mix_h1(0, mix_k1(v)), 4) + +hash_i64(v) = + low = low 32 bits of v as i32 + high = high 32 bits of v as i32 + fmix(mix_h1(mix_h1(0, mix_k1(low)), mix_k1(high)), 8) + +hash_bytes(bytes) = + h = 0 + for each complete 4-byte little-endian chunk: + h = mix_h1(h, mix_k1(chunk_as_i32)) + for each remaining byte: + h = mix_h1(h, mix_k1(sign_extend_i8(byte))) + fmix(h, byte_length) +``` -2. **Conflict resolution uses MemWAL Index**: When a commit conflicts, the merger checks the conflicting commit's MemWAL Index. +Test vectors for `num_buckets = 8`: -3. **No progress regression**: Because MemWAL Index is updated atomically with data, concurrent mergers cannot regress the merge progress. +- `int32` or `date32`: `1 -> 2`, `2 -> 7`, `null -> 0`, `3 -> 1`. +- `utf8`: `"a" -> 1`, `"b" -> 5`, `null -> 0`. +- `bool`: `true -> 2`. +- `float32`: `1.25 -> 0`. +- `float64`: `1.25 -> 0`. diff --git a/docs/src/format/table/transaction.md b/docs/src/format/table/transaction.md index 78dd5301fb8..ef3384c6254 100644 --- a/docs/src/format/table/transaction.md +++ b/docs/src/format/table/transaction.md @@ -466,6 +466,64 @@ The following operations are retryable conflicts with DataReplacement: A concurrent Delete or Update that only adds a deletion vector to a target fragment (without removing it) is compatible: the positional column file stays aligned and the rebase preserves the deletion vector. +### DataOverlay + +Attaches [overlay files](data_overlay_file.md) to fragments, supplying new values +for a subset of `(row offset, field)` cells without rewriting the fragments' base +data files. The overlays are appended to each fragment's existing `overlays` list, +so overlays written by concurrent commits are preserved. Each overlay's +`committed_version` is stamped to the new dataset version at commit time (and +re-stamped on retry), like the created-at / last-updated-at version sequences. + +
+DataOverlay protobuf message + +```protobuf +%%% proto.message.DataOverlay %%% + +%%% proto.message.DataOverlayGroup %%% +``` + +
+ +#### DataOverlay Compatibility + +A DataOverlay operation only changes cells within existing fragments and preserves +physical row addresses, so — like DataReplacement — it is intentionally permissive. +Because overlays stack and the higher `committed_version` wins each covered cell, +independent backfills never conflict, and a concurrent Delete simply makes the +overlay value for a deleted offset inert. Here are the operations that conflict +with DataOverlay: + +- Overwrite +- Restore +- UpdateMemWalState + +The following operations are retryable conflicts with DataOverlay: + +- Rewrite (only if overlapping fragments) — row-rewriting compaction or an + overlay→base fold changes physical row addresses or consumes the overlays, so + the overlay's offsets are no longer valid; the writer must re-read the new + fragment, recompute, and retry. +- Merge (always). +- A row-moving Update that touches an overlaid fragment — a delete-and-reinsert + update (any update that is not a `REWRITE_COLUMNS` column rewrite) relocates the + updated rows into new fragments, so the overlay's physical offsets no longer + address them; the writer must re-read and retry. + +DataOverlay is compatible with another DataOverlay (any fields), Append, Delete, a +`REWRITE_COLUMNS` column rewrite, and DataReplacement, because all of these +preserve physical row addresses: overlay offsets stay valid, the overlay is newer +and wins its covered cells, and the version gate excludes those cells from any +rebuilt index. + +When a DataReplacement or a `REWRITE_COLUMNS` update writes new base values for a +field, it supersedes any older overlay on that field: the writer tombstones the +overlay's entry for the rewritten field — replacing the field id with the obsolete +sentinel, as with obsolete base columns — so the fresh base values are not silently +shadowed. Overlay entries for other fields are preserved, and an overlay left with +no live fields is dropped. + ### UpdateMemWalState Updates the state of MemWal indices (write-ahead log based indices). diff --git a/docs/src/format/table/versioning.md b/docs/src/format/table/versioning.md index 745dd1ccd87..e0b7d480702 100644 --- a/docs/src/format/table/versioning.md +++ b/docs/src/format/table/versioning.md @@ -28,7 +28,10 @@ they should return an "unsupported" error on any read or write operation. | 4 | `FLAG_USE_V2_FORMAT_DEPRECATED` | No | No | Files are written with the new v2 format. This flag is deprecated and no longer used. | | 8 | `FLAG_TABLE_CONFIG` | No | Yes | Table config is present in the manifest. | | 16 | `FLAG_BASE_PATHS` | Yes | Yes | Dataset uses multiple base paths (for shallow clones or multi-base datasets). | +| 32 | `FLAG_DISABLE_TRANSACTION_FILE` | No | Yes | Transactions are recorded in the manifest rather than in a separate transaction file. | +| 64 | `FLAG_UNSTABLE_DATA_OVERLAY_FILES` | Yes | Yes | Fragments may carry data overlay files. Unstable: release builds reject it unless explicitly opted in. | +| 128 | `FLAG_MEM_WAL_INDEX_CATCHUP` | Yes | Yes | `index_catchup` is maintained on this table, so an index absent from it is *not* caught up. See [MemWAL](mem_wal.md). | -Flags with bit values 32 and above are unknown and will cause implementations to reject the dataset with an "unsupported" error. +Flags with bit values 256 and above are unknown and will cause implementations to reject the dataset with an "unsupported" error. diff --git a/docs/src/guide/.pages b/docs/src/guide/.pages index 46ddd475799..7a0fd817b1c 100644 --- a/docs/src/guide/.pages +++ b/docs/src/guide/.pages @@ -6,6 +6,7 @@ nav: - JSON Support: json.md - Tags and Branches: tags_and_branches.md - Object Store Configuration: object_store.md + - Observability: observability.md - Distributed Write: distributed_write.md - Distributed Indexing: distributed_indexing.md - Migration Guide: migration.md diff --git a/docs/src/guide/arrays.md b/docs/src/guide/arrays.md index f824dd641c1..5765ca7675b 100644 --- a/docs/src/guide/arrays.md +++ b/docs/src/guide/arrays.md @@ -119,11 +119,11 @@ calling `lance.arrow.ImageTensorArray.to_encoded`. A `lance.arrow.EncodedImageArray.to_tensor` method is provided to decode encoded images and return them as `lance.arrow.FixedShapeImageTensorArray`, from -which they can be converted to numpy arrays or TensorFlow tensors. +which they can be converted to numpy arrays. For decoding images, it will first attempt to use a decoder provided via the optional function parameter. If decoder is not provided it will attempt to use -[Pillow](https://pillow.readthedocs.io/en/stable/) and [tensorflow](https://www.tensorflow.org/api_docs/python/tf/io/encode_png) in that -order. If neither library or custom decoder is available an exception will be raised. +[Pillow](https://pillow.readthedocs.io/en/stable/). If neither Pillow nor a custom +decoder is available an exception will be raised. ```python from lance.arrow import ImageURIArray @@ -132,13 +132,16 @@ uris = [os.path.join(os.path.dirname(__file__), "images/1.png")] encoded_images = ImageURIArray.from_uris(uris).read_uris() print(encoded_images.to_tensor()) -def tensorflow_decoder(images): - import tensorflow as tf +def pillow_decoder(images): + import io import numpy as np + from PIL import Image - return np.stack(tf.io.decode_png(img.as_py(), channels=3) for img in images.storage) + return np.stack( + np.asarray(Image.open(io.BytesIO(img.as_py()))) for img in images.storage + ) -print(encoded_images.to_tensor(tensorflow_decoder)) +print(encoded_images.to_tensor(pillow_decoder)) # # [[42, 42, 42, 255]] # @@ -164,8 +167,8 @@ created by calling `lance.arrow.ImageArray.from_array` and passing in a It can be encoded into to `lance.arrow.EncodedImageArray` by calling `lance.arrow.FixedShapeImageTensorArray.to_encoded` and passing custom encoder If encoder is not provided it will attempt to use -[tensorflow](https://www.tensorflow.org/api_docs/python/tf/io/encode_png) and [Pillow](https://pillow.readthedocs.io/en/stable/) in that order. Default encoders will -encode to PNG. If neither library is available it will raise an exception. +[Pillow](https://pillow.readthedocs.io/en/stable/). The default encoder will encode +to PNG. If neither Pillow nor a custom encoder is available it will raise an exception. ```python from lance.arrow import ImageURIArray @@ -176,4 +179,4 @@ tensor_images.to_encoded() # # [... # b'\x89PNG\r\n\x1a...' -``` \ No newline at end of file +``` diff --git a/docs/src/guide/blob.md b/docs/src/guide/blob.md index 00bd5d086e3..8e902a95a9c 100644 --- a/docs/src/guide/blob.md +++ b/docs/src/guide/blob.md @@ -115,6 +115,9 @@ Note: metadata for the same column are rejected. - `blob_pack_file_size_threshold` is a write option for rolling packed `.blob` sidecar files. It does not control inline-vs-packed placement. +- Blob v2 fields can be nested inside structs and variable-length lists. Blob-aware + scans preserve the surrounding nested layout; use `blob_handling="all_binary"` + to materialize nested blob payloads as bytes. ### Example: packed external blobs (single container file) @@ -174,8 +177,9 @@ Choose the read API based on the payload shape you want: | API | Returns | Use When | |---|---|---| -| `read_blobs` | `List[Tuple[int, bytes]]` | You need complete blob payloads in memory, such as training loaders or batch preprocessing. | -| `take_blobs` | `List[BlobFile]` | You need file-like objects for streaming, seeking, or partial reads. | +| `read_blobs` | `List[Tuple[int, Optional[bytes]]]` | You need complete blob payloads in memory, such as training loaders or batch preprocessing. | +| `read_blob_ranges` | `List[Tuple[int, int, Optional[bytes]]]` | You need selected byte ranges from multiple rows without materializing complete blobs. | +| `take_blobs` | `List[Optional[BlobFile]]` | You need file-like objects for streaming, seeking, or partial reads. | | `scanner(..., blob_handling="all_binary")` | Arrow binary columns | You want blob columns in a scan result or `pyarrow.Table`. | Do not wrap `take_blobs` in your own thread pool just to call `read()` or @@ -183,7 +187,8 @@ Do not wrap `take_blobs` in your own thread pool just to call `read()` or batched blob reads through Lance's scheduler. Exactly one selector must be provided to `read_blobs` or `take_blobs`: `ids`, -`indices`, or `addresses`. +`indices`, or `addresses`. `read_blob_ranges` accepts the same selector kinds +through its required `selector` argument. | Selector | Typical Use | Stability | |---|---|---| @@ -223,6 +228,48 @@ row_addrs = ds.to_table(columns=[], with_row_address=True).column("_rowaddr").to rows = ds.read_blobs("blob", addresses=row_addrs[:2]) ``` +Blob selection APIs preserve logical result cardinality. `read_blobs()` and +`take_blobs()` return one element per selected row, and `read_blob_ranges()` +returns one element per request. A null blob is returned as `None`; a valid +empty blob remains a non-null empty payload or zero-length `BlobFile`. + +### Read row-specific byte ranges + +Use `read_blob_ranges` to read multiple blob-local ranges with one planned API +call. Each request is a `(row, offset, length)` tuple, and `selector` determines +whether every `row` is interpreted as a row ID, row address, or dataset index. + +```python +import lance + +ds = lance.dataset("./blobs_v22.lance") +results = ds.read_blob_ranges( + "blob", + requests=[ + (7, 0, 1024), + (7, 4096, 1024), + (12, 0, 0), + ], + selector="indices", +) + +for request_index, row_address, data in results: + if data is None: + # The selected blob is null. + continue + print(request_index, row_address, len(data)) +``` + +Each result contains the zero-based `request_index`, the resolved physical row +address, and the requested bytes. `request_index` identifies the original +request when the same row appears more than once. + +A request on a null blob returns `None`, including when its range is empty. An +empty range on a non-null blob returns `b""` without payload I/O. For every +request, `offset + length` must fit in an unsigned 64-bit integer. A range on a +non-null blob must not extend beyond its logical size; blob-local bounds are not +evaluated for null blobs because they have no logical payload length. + ### Read blob columns as Arrow binary ```python @@ -241,8 +288,10 @@ import lance ds = lance.dataset("./blobs_v22.lance") blobs = ds.take_blobs("blob", indices=[0, 1]) -with blobs[0] as f: - header = f.read(1024) +blob = blobs[0] +if blob is not None: + with blob as f: + header = f.read(1024) ``` ### Example: decode video frames lazily @@ -253,6 +302,8 @@ import lance ds = lance.dataset("./videos_v22.lance") blob = ds.take_blobs("video", indices=[0])[0] +if blob is None: + raise ValueError("video blob is null") start_ms, end_ms = 500, 1000 @@ -347,7 +398,7 @@ lance.write_dataset( Not every binary column needs to be a blob column. Plain Arrow `binary`/`large_binary` stores bytes *inline*, interleaved with your other columns, which is simplest and fastest for really small blobs (e.g., thumbnail images). Using a blob column to store the binary payload makes sense when either of these holds: -- **You need partial or streaming reads.** Inline binary is always read in full; there is no way to fetch a byte range without materializing the entire value. Blob columns expose `take_blobs` → `BlobFile` handles that seek and range-read, so you pay only for the bytes you touch. +- **You need partial or streaming reads.** Inline binary is always read in full; there is no way to fetch a byte range without materializing the entire value. Blob columns expose `read_blob_ranges` for planned row-specific range reads and `take_blobs` → `BlobFile` handles for caller-driven seeks, so you pay only for the bytes you touch. - **Your values are large (roughly 1 MB or more on average).** Operations that rewrite entire rows, such as compaction or some updates, must copy the large inline payloads forward into the new version — even when those bytes never changed. The bigger the payload, the more bytes you rewrite per logical change (write amplification). A blob column keeps large payloads in separate `.blob` files that are referenced rather than re-copied, so these operations don't rewrite the heavy bytes. !!! tip diff --git a/docs/src/guide/data_types.md b/docs/src/guide/data_types.md index 06f26da1db3..c3f222d36ba 100644 --- a/docs/src/guide/data_types.md +++ b/docs/src/guide/data_types.md @@ -408,7 +408,7 @@ This maps to Lance's `FixedSizeList(Float32, 384)` type, which is optimized for: 3. **Align dimensions for SIMD**: Vector dimensions divisible by 8 enable optimal SIMD acceleration. Common dimensions: 128, 256, 384, 512, 768, 1024, 1536. -4. **Create indexes for large datasets**: For datasets with more than ~10,000 vectors, create an ANN index for fast search: +4. **Create indices for large datasets**: For datasets with more than ~10,000 vectors, create an ANN index for fast search: ```python # IVF_PQ is recommended for most use cases diff --git a/docs/src/guide/distributed_write.md b/docs/src/guide/distributed_write.md index 4fbc43a1058..ef476fdadb1 100644 --- a/docs/src/guide/distributed_write.md +++ b/docs/src/guide/distributed_write.md @@ -263,3 +263,114 @@ Output: 6 7 Gracie 88 7 8 Henry 82 ``` + +### Handling stable row id + +On a dataset created with `enable_stable_row_ids=True`, each row keeps the same +`_rowid` for its lifetime, even when an update rewrites it into a different +fragment. Lance cannot infer which new row replaces which old one, so when you +assemble the transaction yourself, carrying those ids across is your job: read +the rows you are rewriting with `with_row_id=True` and attach their ids to the +new fragment with `lance.fragment.RowIdSequence`. + +Rows you leave without an id are treated as newly inserted. That is not an error, +so a fragment written without `row_id_meta` commits successfully while silently +giving every rewritten row a fresh identity, breaking `_rowid` for anything +downstream that relies on it. + +You do **not** need to supply `created_at_version_meta` or +`last_updated_at_version_meta`. Leave them as `None`. Lance derives both while +building the manifest: `last_updated_at_version_meta` becomes the version being +committed, and `created_at_version_meta` is copied from whichever existing row +carries the same stable row id, so a rewritten row keeps the version it first +appeared in. + +#### Mixing updated and new rows + +A single fragment may hold both rewritten rows and brand new ones. Order it so +that **the rewritten rows come first and the new rows last**, then pass only the +row ids of the rewritten rows. The row ids bind to the leading rows in fragment +order, and the commit mints ids for the remaining rows. + +Do not mint ids for the new rows yourself. Row ids are handed out from a counter +in the manifest, and a commit that loses a race is retried against the version +that won, which may have consumed the very ids you picked. Only the commit knows +which values are free, so it assigns them after conflict resolution has settled. +Supplying more row ids than the fragment has rows is rejected. + +```python +import lance +import pyarrow as pa +import pyarrow.compute as pc +from lance.fragment import RowIdSequence, write_fragments + +schema = pa.schema([("id", pa.int64()), ("score", pa.int64())]) +dataset_uri = "./stable_row_ids.lance" +dataset = lance.write_dataset( + pa.table({"id": [1, 2, 3, 4], "score": [85, 90, 75, 80]}, schema=schema), + dataset_uri, + enable_stable_row_ids=True, +) + +# On a worker: read the rows to rewrite, keeping their stable row ids. +rows = dataset.to_table(columns=["id", "score"], with_row_id=True) +rewritten = rows.filter(pc.field("id").isin([2, 3])) + +# Rewritten rows first, then the row that did not exist before. +new_data = pa.table( + { + "id": rewritten["id"].to_pylist() + [5], + "score": [95, 70, 60], + }, + schema=schema, +) +fragments = write_fragments(new_data, dataset_uri, schema=schema) +assert len(fragments) == 1 + +# Only the rewritten rows have ids. The trailing row gets one at commit time. +fragments[0].row_id_meta = RowIdSequence(rewritten["_rowid"]).to_inline_metadata() + +# On the committing worker: tombstone the old copies of the rewritten rows. +updated_fragment = dataset.get_fragments()[0].delete("id in (2, 3)") + +op = lance.LanceOperation.Update( + updated_fragments=[updated_fragment], + new_fragments=fragments, +) +dataset = lance.LanceDataset.commit(dataset_uri, op, read_version=dataset.version) + +print(dataset.to_table(with_row_id=True).to_pandas()) +``` + +Output: +``` + id score _rowid +0 1 85 0 +1 4 80 3 +2 2 95 1 +3 3 70 2 +4 5 60 4 +``` + +Row ids 1 and 2 followed their rows into the new fragment, and the inserted row +received the next unused id. Reading the lineage columns shows that the rewritten +rows kept their original creation version while the inserted row is stamped with +the version that added it: + +```python +print( + dataset.to_table( + columns=["id", "_row_created_at_version", "_row_last_updated_at_version"] + ).to_pandas() +) +``` + +Output: +``` + id _row_created_at_version _row_last_updated_at_version +0 1 1 1 +1 4 1 1 +2 2 1 2 +3 3 1 2 +4 5 2 2 +``` diff --git a/docs/src/guide/migration.md b/docs/src/guide/migration.md index 5efd7b26b7f..56d7737941f 100644 --- a/docs/src/guide/migration.md +++ b/docs/src/guide/migration.md @@ -8,24 +8,19 @@ migrate. ## 9.0.0 -* Newly created FTS / inverted indexes now default to format v2 instead of v1. - The `LANCE_FTS_FORMAT_VERSION` environment variable no longer controls the - format used for newly created indexes. Users who need a specific index layout - should pass the index creation parameter `format_version` explicitly. - -* This affects users who create FTS / inverted indexes and need those indexes to - be readable by older Lance versions, or who depend on the v1 index layout. In - those cases, pass `format_version=1` when creating the index. Otherwise, newly - created indexes will use v2 by default, and older Lance readers may not be able - to read them. - - ```python - dataset.create_scalar_index("text", "INVERTED", format_version=1) - ``` - -* Existing v1 FTS indexes remain queryable. Operations that maintain an existing - v1 index, including append, incremental indexing, optimize, and mem-wal - maintained-index flush, should continue preserving the v1 format. +* Unless overridden, newly created FTS indexes use format v2. The code analyzer + and `block_size=256` require format v3, so readers must support v3 before an + index using either option is created. `document_granularity="list_element"` + also requires v3 reader capability, independently of the posting format. + +* To keep new indexes readable by nodes that support at most format v1 or v2, + set `format_version` in the index creation parameters, or set + `LANCE_FTS_FORMAT_VERSION` for a rollout-wide override. Formats v1 and v2 + require the text analyzer and `block_size=128`. + +* Operations that maintain an existing FTS index, including append, incremental + indexing, optimize, and mem-wal maintained-index flush, preserve its format + version. ## 7.2.0 diff --git a/docs/src/guide/object_store.md b/docs/src/guide/object_store.md index 9ce4800b800..70445837228 100644 --- a/docs/src/guide/object_store.md +++ b/docs/src/guide/object_store.md @@ -110,6 +110,38 @@ The following keys can be used as both environment variables or keys in the | `aws_sse_kms_key_id` | The KMS key ID to use for server-side encryption. If set, `aws_server_side_encryption` must be `"aws:kms"` or `"aws:kms:dsse"`. | | `aws_sse_bucket_key_enabled` | Whether to use bucket keys for server-side encryption. | +### Credential provider selection + +By default, Lance uses the standard AWS credential provider chain (environment +variables, shared config file, web identity tokens, ECS, EC2 instance metadata). + +The `aws_provider_scheme` storage option pins a dataset to a specific credential +provider, which is useful when two datasets in the same process need different +AWS auth (for example, one bucket using IRSA and another using ECS container +credentials). + +| Value | Behavior | +|-------|----------| +| `token` | Use static access-key credentials. Returns an error if `aws_access_key_id` and `aws_secret_access_key` are not set. | +| `ecs` | Use the ECS/Pod Identity container credential endpoint. Reads `AWS_CONTAINER_CREDENTIALS_FULL_URI` or `AWS_CONTAINER_CREDENTIALS_RELATIVE_URI` from the environment. | +| `irsa` | Use IRSA (IAM Roles for Service Accounts) web identity token credentials. Reads `AWS_WEB_IDENTITY_TOKEN_FILE` and `AWS_ROLE_ARN` from the environment. | + +```python +import lance + +# Bucket A — use IRSA (web identity token from the environment) +ds_a = lance.dataset( + "s3://bucket-a/path", + storage_options={"aws_provider_scheme": "irsa"}, +) + +# Bucket B — use ECS container credentials +ds_b = lance.dataset( + "s3://bucket-b/path", + storage_options={"aws_provider_scheme": "ecs"}, +) +``` + ### S3-compatible stores Lance can also connect to S3-compatible stores, such as MinIO. To do so, you must @@ -326,6 +358,15 @@ parameter; explicit `storage_options` override environment variables: | `cos_secret_key` | Secret key used for COS authentication. Optional if credentials are provided by environment. | | `cos_enable_versioning` | Whether to enable object versioning on the bucket. Optional. | +!!! warning + + Tencent COS does not reliably enforce put-if-not-exists on buckets that have + ever had versioning enabled, even if versioning is now suspended. To prevent + silent manifest overwrites, Lance requires a custom distributed commit lock + for COS writes. Pass the same `commit_lock` implementation to every Python + writer, or provide a custom `CommitHandler` in Rust. Reads do not require a + commit lock. + !!! note The OpenDAL `CosConfig` currently exposes a limited set of options. Additional @@ -340,6 +381,31 @@ is `goosefs://host:port/path`, where `host:port` is the GooseFS Master address (default port: `9200`, may be omitted, e.g. `goosefs://10.0.0.1/path`) and `/path` is the filesystem path within GooseFS. +Manifest commits on `goosefs://` use `ConditionalPutCommitHandler` +(`PutMode::Create` / if-not-exists), backed by GooseFS master's atomic +no-replace rename so concurrent writers cannot clobber each other's +versioned manifests. + +!!! warning "Mixed-version writers are NOT safe" + + The `if-not-exists` guarantee only holds when **every** writer for a + dataset routes through this new handler. A writer running an older + Lance release still selects `UnsafeCommitHandler` for `goosefs://` + and writes the version path unconditionally, which can overwrite a + manifest that an upgraded writer has already won. Safe concurrent + commits therefore require: + + - all writers for the dataset run a Lance release that includes this + routing change, **or** + - writers share an external coordination boundary (e.g. a single-writer + queue, table-level lock, or a gateway that serializes commits) that + prevents the old code path from racing the new one. + + When upgrading in place, quiesce all writers (drain jobs, scale + clients to zero, or route traffic through a writer coordinator) before + rolling out the new Lance version, then bring writers back on the new + version together. + !!! note "About the dataset path" `/path` is just an arbitrary directory inside GooseFS — Lance does **not** diff --git a/docs/src/guide/observability.md b/docs/src/guide/observability.md new file mode 100644 index 00000000000..9cec3154e82 --- /dev/null +++ b/docs/src/guide/observability.md @@ -0,0 +1,100 @@ +# Observability + +Lance can publish operational metrics to your monitoring stack. The table below +is the authoritative catalogue of the metrics Lance emits, shared verbatim with +the Rust [`lance::metrics`](https://github.com/lance-format/lance/blob/main/rust/lance/src/metrics.md) +module documentation. + +--8<-- "rust/lance/src/metrics.md" + +## Collecting metrics + +Lance emits through the [`metrics`](https://docs.rs/metrics) crate facade, so it +is not tied to a specific backend — you install a recorder/exporter and route +the metrics wherever you like. Metrics are available from the Rust, Python, and +Java APIs. + +### Rust + +Enable the `metrics` feature on the `lance` crate: + +```toml +lance = { version = "...", features = ["metrics"] } +``` + +Then install any `metrics`-compatible recorder once at startup, before opening +datasets. For example, with +[`metrics-exporter-prometheus`](https://docs.rs/metrics-exporter-prometheus): + +```rust +metrics_exporter_prometheus::PrometheusBuilder::new() + .install() + .expect("install Prometheus recorder"); +``` + +Any recorder works — Prometheus, StatsD, an OpenTelemetry bridge, and so on. +When no recorder is installed, emission is a cheap no-op. + +### Python + +Unlike Rust, the Python bindings do not let you plug in an arbitrary recorder: +bridging one across the FFI boundary into the Rust `metrics` facade would be +complicated and inefficient. Instead `pylance` standardizes on OpenTelemetry, +which has good Python support, as its recorder. + +The `pylance` wheels are built with the `metrics` feature enabled. Install the +OpenTelemetry extra and call `instrument_lance_metrics`, which registers Lance's +metrics as observable instruments on your OpenTelemetry `MeterProvider`: + +```bash +pip install "pylance[otel]" +``` + +```python +from lance.otel import instrument_lance_metrics + +# Uses the global MeterProvider; pass meter_provider=... to target a specific one. +instrument_lance_metrics() +``` + +### Java + +The Java SDK includes an OpenTelemetry bridge in `org.lance.otel`. Register it +before opening datasets or performing Lance IO so the process-global Rust +recorder sees every emitted metric: + +```java +import org.lance.otel.LanceMetrics; + +LanceMetrics.instrument(); +``` + +The OpenTelemetry API is a dependency of the Java SDK. The application must +still configure an OpenTelemetry SDK, metric reader, and exporter for collection +and delivery. + +The no-argument method uses OpenTelemetry's global `MeterProvider`. To register +with an explicitly configured provider, pass it directly: + +```java +SdkMeterProvider provider = SdkMeterProvider.builder() + .registerMetricReader(metricReader) + .build(); +LanceMetrics.instrument(provider); +``` + +Repeated calls with the same provider are idempotent. Passing a different +provider unregisters the existing callback before registering the new one. Call +`LanceMetrics.close()` to stop exporting while retaining the process-global Rust +metric state. + +From there the metrics flow through whatever OpenTelemetry pipeline you have +configured (OTLP, Prometheus, console, …). Because OpenTelemetry has no +asynchronous histogram instrument, histograms are exported Prometheus-style as +three observable counters: `_bucket`, `_count`, and `_sum`. +Each `_bucket` sample carries an `le` ("less than or equal") attribute +giving that bucket's inclusive upper bound in the metric's unit; the bucket +count is cumulative, covering every observation at or below `le`. For example, a +`lance_object_store_request_duration_seconds_bucket` sample with `le="0.5"` +counts all requests that completed in 0.5 seconds or less, while `le="+Inf"` is +the total count. diff --git a/docs/src/guide/performance.md b/docs/src/guide/performance.md index 14181eb69af..5b8255577be 100644 --- a/docs/src/guide/performance.md +++ b/docs/src/guide/performance.md @@ -141,7 +141,7 @@ Keys are often a composite of multiple fields and all keys are scoped to the dat | Deletion Files | Dataset URI, fragment_id, version, id, file_type | The deletion vector for a frag | | Row Id Mask | Dataset URI, version | The row id sequence for the dataset | | Row Id Index | Dataset URI, version | The row id index for the dataset | -| Row Id Sequence | Dataset URI, fragment_id | The row id sequence for a fragment | +| Row Id Sequence | Dataset URI, fragment_id, row_id_meta | The row id sequence for a fragment | | Index Metadata | Dataset URI, version | The index metadata for the dataset | | Index Details¹ | Dataset URI, index uuid | The index details for an index | | File Global Meta | Dataset URI, file path | The global metadata for a file | @@ -189,6 +189,41 @@ working with 1024-dimensional vector embeddings (e.g. 32-bit floats) then 8192 r spread that across 16 CPU threads then you would need 512MB of compute memory per scan. You might find working with 1024 rows per batch is more appropriate. +#### Tuning remote scans + +An ordered dataset scan still overlaps I/O from multiple fragments. `scan_in_order=True` controls the order in +which batches are returned; it does not make fragment reads sequential. This is why a dataset scan can issue +more concurrent requests than scanning one fragment directly. The following controls tune different parts of +the scan: + +* `fragment_readahead` limits how many fragments may have reads scheduled concurrently. Set it to `1` to match + the fragment-level I/O pattern, then increase it if the storage connection has spare bandwidth. +* `LANCE_IO_THREADS` limits concurrent storage requests for the process. Cloud stores default to 64, which is + intended for high-bandwidth, in-region access and can be too aggressive across regions or over the public + internet. +* `io_buffer_size` limits buffered I/O bytes and applies backpressure when decoding falls behind. +* `batch_readahead` limits concurrent batch decoding. It does not control the size of storage range requests. + +For a bandwidth-constrained remote connection, start with conservative settings and tune upward: + +```shell +LANCE_IO_THREADS=8 python scan.py +``` + +```python +scanner = dataset.scanner( + fragment_readahead=1, + batch_readahead=2, + io_buffer_size=64 * 1024 * 1024, +) +for batch in scanner.to_batches(): + process(batch) +``` + +Lance reads encoded pages from storage, so reducing `batch_size` changes the returned and decoded batch sizes +but may not reduce the initial range request. The first batch can require loading one encoded page for each +selected column. + In summary, scans could use up to `(2 * io_buffer_size) + (batch_size * num_compute_threads)` bytes of memory. Keep in mind that `io_buffer_size` is a soft limit (e.g. we cannot read less than one page at a time right now) and so it is not necessarily a bug if you see memory usage exceed this limit by a small margin. @@ -219,6 +254,37 @@ use cases. For example, S3 can typically get up to 5000 req/s and with these settings we should get there in about 10 seconds. +## Fragment Sizing + +A Lance table is a collection of fragments tracked by a manifest. How you size those fragments +trades off two classes of work: + +- **Manifest-level operations** scale with the *number* of fragments. Every dataset mutation + (appends, metadata updates, schema changes, compactions, etc.) rewrites the manifest, so a + larger fragment list makes every write slower. Reads pay a similar cost up front: opening a + dataset, listing fragments, planning a scan, and resolving transaction conflicts at the + dataset level all walk the manifest. +- **Fragment-level operations** scale with the *size* of a fragment. These include scans + against a matching fragment, compaction, updates, deletes, and `merge_insert`. Conflict + detection for these operations is also done at the fragment level. + +Fewer, larger fragments make manifest-level operations cheap but make each fragment-level +operation heavier and increase the chance of conflicts when many writers target the same +fragment. More, smaller fragments do the reverse. + +Practical guidance: + +- The default of 1M rows per fragment works well up to ~1B rows. Past that, bumping toward + ~100M rows per fragment is reasonable, though fragment-count limits are rarely the bottleneck + in practice. +- Tens of thousands of fragments per table is generally fine. +- Keep individual fragments well under object-store object-size limits (S3 caps at 5 TB, and + stores tend to misbehave well before that). 10 GB–100 GB per fragment is a reasonable upper + range; 1 TB is a hard ceiling. +- If you run many concurrent updates, deletes, or `merge_insert` operations, err toward more + fragments — conflict detection is per-fragment, so too few fragments leads to excess + retries. + ## Conflict Handling Lance supports concurrent operations on the same table using optimistic concurrency control. When two diff --git a/docs/src/guide/read_and_write.md b/docs/src/guide/read_and_write.md index ec6cde5173a..c7feb69144b 100644 --- a/docs/src/guide/read_and_write.md +++ b/docs/src/guide/read_and_write.md @@ -456,3 +456,146 @@ affected files are no longer part of any ANN index if they were before. Because of this, it's recommended to rewrite files before re-building indices. + +### Cleanup old versions + +Lance is an immutable format — every write creates a new version. The new version +only writes the data that changed, so an insert writes the new rows and an update +rewrites the affected columns for the affected rows. Even a delete creates a +small deletion file. However, old versions still reference the previous data +files, so those files are kept on disk until explicitly removed. Over time this +means storage grows with each operation — inserts, updates, and deletes alike. + +Keeping old versions has important benefits: readers that opened an older version +can continue reading it without interference from concurrent writers, providing +snapshot isolation. Old versions also enable time travel queries, letting you +read the dataset as it existed at any prior point in time. + +`cleanup_old_versions` deletes old version metadata and any data files that are +no longer referenced by any version, reclaiming the accumulated storage. + +!!! warning + + Once old versions are cleaned up, time travel queries to those versions are + no longer possible. Choose your retention window (`older_than`) accordingly — + any version removed by cleanup cannot be recovered. + +```python +import lance + +dataset = lance.dataset("./my_dataset.lance") +dataset.cleanup_old_versions() +``` + +By default, versions older than 7 days are removed. You can override this with +the `older_than` parameter (a `timedelta`): + +```python +from datetime import timedelta + +dataset.cleanup_old_versions(older_than=timedelta(days=1)) +``` + +!!! note + + Tagged versions are exempt from cleanup. See [Tags and Branches](tags_and_branches.md) + for details. + +By default, Lance only removes files that it can **verify** are no longer needed. +A file is verified when Lance can see that it was referenced by an older version +and is no longer referenced by any newer version. However, some orphaned files +cannot be verified this way — for example, files left behind by aborted or failed +commits that were never recorded in any version. These files are +indistinguishable from files being written by an in-progress operation. + +Cleanup will never delete the current (active) version. This means passing +`older_than=timedelta(0)` is safe and will delete all versions except the current +one. + +The `delete_unverified` flag enables a more aggressive strategy that will also +delete these unverified files: + +```python +dataset.cleanup_old_versions( + older_than=timedelta(hours=2), + delete_unverified=True, +) +``` + +!!! danger + + Only use `delete_unverified=True` when you are confident that no other + concurrent operation has been in-progress for longer than the `older_than` + duration. Lance uses the file's age to decide whether an unverified file is + safe to remove, so any operation that is still running past the `older_than` + window risks having its files deleted out from under it. + + In particular, combining `delete_unverified=True` with `older_than=timedelta(0)` + is **extremely dangerous** — if any other operation is in-progress at all, + its data files may be deleted, leading to dataset corruption. + +### Automatic cleanup + +Instead of calling `cleanup_old_versions` manually, you can configure Lance to +clean up old versions automatically during writes. When auto cleanup is enabled, +Lance will run cleanup every *N* commits (the **interval**), removing versions +older than a specified duration. + +Auto cleanup can be enabled when creating a new dataset: + +```python +import lance +import pyarrow as pa +from lance.dataset import AutoCleanupConfig + +table = pa.table({"id": range(100)}) +ds = lance.write_dataset( + table, + "./my_dataset.lance", + auto_cleanup_options=AutoCleanupConfig( + interval=20, # run cleanup every 20 commits + older_than_seconds=3600, # remove versions older than 1 hour + ), +) +``` + +Or enabled on an existing dataset: + +```python +ds = lance.dataset("./my_dataset.lance") +ds.optimize.enable_auto_cleanup( + AutoCleanupConfig( + interval=20, + older_than_seconds=3600, + ) +) +``` + +And disabled again: + +```python +ds.optimize.disable_auto_cleanup() +``` + +Auto cleanup parameters can also be set directly via dataset config keys: + +```python +ds.update_config({ + "lance.auto_cleanup.interval": "20", + "lance.auto_cleanup.older_than": "3600s", +}) +``` + +!!! warning + + Auto cleanup runs as part of the commit path. If your writer does not have + delete permissions, or you are doing high-frequency writes where the extra + latency matters, pass `skip_auto_cleanup=True` to `write_dataset` to skip it + on a per-write basis. + +### Other cleanup strategies + +It is common to run cleanup as a periodic background task on a dedicated server +(for example, via a cron job or scheduled workflow). This keeps cleanup off the +write path entirely, avoiding any impact to write latency, but requires setting +up and maintaining additional infrastructure. diff --git a/docs/src/guide/tags_and_branches.md b/docs/src/guide/tags_and_branches.md index 8af2302bfb0..4b371ea6dae 100644 --- a/docs/src/guide/tags_and_branches.md +++ b/docs/src/guide/tags_and_branches.md @@ -87,7 +87,7 @@ import pyarrow as pa # Open dataset ds = lance.dataset("/tmp/test.lance") -# Create branch from latest version (default: current branch's latest) +# Create branch from the currently checked-out version experiment_branch = ds.create_branch("experiment") experimental_data = pa.Table.from_pydict({"a": [11], "b": [12]}) lance.write_dataset(experimental_data, experiment_branch, mode="append") diff --git a/docs/src/guide/tokenizer.md b/docs/src/guide/tokenizer.md index eedac585fe9..0d3605df30b 100644 --- a/docs/src/guide/tokenizer.md +++ b/docs/src/guide/tokenizer.md @@ -12,6 +12,39 @@ ${system data directory}/lance/language_models It also supports configuring user dictionaries, which makes it convenient for users to expand their own dictionaries without retraining the language models. +## Inspect Query Tokenization + +Use `lance.tokenize` to inspect the tokens that a full-text query will produce +without creating a dataset or index: + +```python +import lance + +tokens = lance.tokenize("the Cats and Dogs") +[(token.text, token.position) for token in tokens] +# [("cat", 0), ("dog", 2)] +``` + +Positions start at the first retained query token and preserve gaps left by stop +word removal and other filters. This is the same representation used for phrase +matching. The function accepts the tokenizer-related options supported by +`LanceDataset.create_scalar_index`, including custom stop words, n-grams, and the +code analyzer: + +```python +tokens = lance.tokenize( + "getUserName::value42", + analyzer="code", + split_identifiers=True, + index_operators=True, +) +``` + +Options set to `None` use the selected analyzer profile's default. For example, +the code analyzer disables stemming and stop-word removal unless explicitly +overridden. The exception is `max_token_length`: omitting it keeps the default +length limit of 40, while `max_token_length=None` disables the limit. + ## ICU Tokenizer ICU uses Unicode word boundary rules and bundled dictionary data for complex scripts. It is useful for mixed-language text and does not require downloading a language model. @@ -54,7 +87,7 @@ Create a file named config.json in the root directory of the current model. ``` - The "main" field is optional. If not filled, the default is "dict.txt". -- "users" is the path of the user dictionary. For the format of the user dictionary, please refer to https://github.com/messense/jieba-rs/blob/main/src/data/dict.txt. +- "users" is the path of the user dictionary. For the format of the user dictionary, please refer to https://github.com/messense/jieba-rs/blob/main/jieba/src/data/dict.txt. ## Language Models of Lindera diff --git a/docs/src/images/mem_wal_overview.png b/docs/src/images/mem_wal_overview.png index 008c84d0724..1c9f64bc647 100644 Binary files a/docs/src/images/mem_wal_overview.png and b/docs/src/images/mem_wal_overview.png differ diff --git a/docs/src/images/mem_wal_regional.png b/docs/src/images/mem_wal_regional.png deleted file mode 100644 index 5681fa27b8b..00000000000 Binary files a/docs/src/images/mem_wal_regional.png and /dev/null differ diff --git a/docs/src/images/mem_wal_shard.png b/docs/src/images/mem_wal_shard.png new file mode 100644 index 00000000000..ad8bed9e4b9 Binary files /dev/null and b/docs/src/images/mem_wal_shard.png differ diff --git a/docs/src/integrations/.pages b/docs/src/integrations/.pages index 62feffae067..ab9c2c0dcee 100644 --- a/docs/src/integrations/.pages +++ b/docs/src/integrations/.pages @@ -1,9 +1,6 @@ nav: - Overview: index.md - Apache DataFusion: datafusion.md - - PostgreSQL: https://github.com/lancedb/pglance + - PostgreSQL: https://github.com/lance-format/pglance - PyTorch: pytorch.md - - Tensorflow: tensorflow.md - - Apache Spark: spark - - Ray: ray - - Trino: trino + - TensorFlow: tensorflow.md diff --git a/docs/src/integrations/index.md b/docs/src/integrations/index.md index 0304f8f5277..eb3b25051cc 100644 --- a/docs/src/integrations/index.md +++ b/docs/src/integrations/index.md @@ -27,7 +27,7 @@ GitHub organization. | Integration | Description | Source | |---|---|---| | [PyTorch](pytorch.md) | Use `lance.torch.data.LanceDataset` as a `torch.utils.data.IterableDataset` for training and inference. | Built-in | -| [TensorFlow](tensorflow.md) | Use `lance.tf.data.from_lance` to stream Lance data into `tf.data.Dataset` pipelines. | Built-in | +| [TensorFlow](tensorflow.md) | Use `lance_tensorflow.from_lance` to stream Lance data into `tf.data.Dataset` pipelines. | [lance-format/lance-tensorflow](https://github.com/lance-format/lance-tensorflow) | | [Ray](ray) | Distributed read/write of Lance datasets with Ray Data. | [lance-format/lance-ray](https://github.com/lance-format/lance-ray) | | [Hugging Face](huggingface) | Convert and load Hugging Face datasets to and from Lance in a single call. | [lance-format/lance-huggingface](https://github.com/lance-format/lance-huggingface) | diff --git a/docs/src/integrations/tensorflow.md b/docs/src/integrations/tensorflow.md index 1c5d6b87157..03a5c5c5eca 100644 --- a/docs/src/integrations/tensorflow.md +++ b/docs/src/integrations/tensorflow.md @@ -1,92 +1,71 @@ -# Tensorflow Integration +--- +title: TensorFlow +description: Stream Lance datasets into TensorFlow tf.data pipelines with lance-tensorflow. +--- -Lance can be used as a regular [tf.data.Dataset](https://www.tensorflow.org/api_docs/python/tf/data/Dataset) -in [Tensorflow](https://www.tensorflow.org/). +# TensorFlow Integration -!!! warning +The TensorFlow integration is maintained in the +[lance-format/lance-tensorflow](https://github.com/lance-format/lance-tensorflow) +project. - This feature is experimental and the APIs may change in the future. +The main Lance Python package no longer includes `lance.tf`. Install +`lance-tensorflow` and import `lance_tensorflow` instead. + +```bash +pip install lance-tensorflow +``` ## Reading from Lance -Using `lance.tf.data.from_lance`, you can create an `tf.data.Dataset` easily. +Use `lance_tensorflow.from_lance` to create a `tf.data.Dataset` from a Lance +dataset. ```python -import tensorflow as tf -import lance +from lance_tensorflow import from_lance -# Create tf dataset -ds = lance.tf.data.from_lance("s3://my-bucket/my-dataset") - -# Chain tf dataset with other tf primitives +ds = from_lance( + "s3://my-bucket/my-dataset", + columns=["image", "label"], + filter="split = 'train'", + batch_size=256, +) -for batch in ds.shuffling(32).map(lambda x: tf.io.decode_png(x["image"])): - print(batch) +for batch in ds: + print(batch["label"]) ``` -Backed by the Lance [columnar format](../format/index.md), using `lance.tf.data.from_lance` supports -efficient column selection, filtering, and more. +## Dataset Convenience Methods + +If you want `tf.data.Dataset.from_lance`, register the convenience methods +explicitly after importing `lance_tensorflow`. ```python -ds = lance.tf.data.from_lance( - "s3://my-bucket/my-dataset", - columns=["image", "label"], - filter="split = 'train' AND collected_time > timestamp '2020-01-01'", - batch_size=256) -``` +import tensorflow as tf +import lance_tensorflow -By default, Lance will infer the Tensor spec from the projected columns. You can also specify `tf.TensorSpec` manually. +lance_tensorflow.register_tensorflow_dataset() -```python -batch_size = 256 -ds = lance.tf.data.from_lance( - "s3://my-bucket/my-dataset", - columns=["image", "labels"], - batch_size=batch_size, - output_signature={ - "image": tf.TensorSpec(shape=(), dtype=tf.string), - "labels": tf.RaggedTensorSpec( - dtype=tf.int32, shape=(batch_size, None), ragged_rank=1), - }, +ds = tf.data.Dataset.from_lance("s3://my-bucket/my-dataset") ``` -## Distributed Training and Shuffling +## Migration -Since [a Lance Dataset is a set of Fragments](../format/index.md), we can distribute and shuffle Fragments to different -workers. +Replace old imports: ```python -import tensorflow as tf -from lance.tf.data import from_lance, lance_fragments +import lance.tf.data -world_size = 32 -rank = 10 -seed = 123 # -epoch = 100 +ds = lance.tf.data.from_lance(uri) +``` -dataset_uri = "s3://my-bucket/my-dataset" +with: -# Shuffle fragments distributedly. -fragments = - lance_fragments("s3://my-bucket/my-dataset") - .shuffling(32, seed=seed) - .repeat(epoch) - .enumerate() - .filter(lambda i, _: i % world_size == rank) - .map(lambda _, fid: fid) +```python +from lance_tensorflow import from_lance -ds = from_lance( - uri, - columns=["image", "label"], - fragments=fragments, - batch_size=32 - ) -for batch in ds: - print(batch) +ds = from_lance(uri) ``` -!!! warning - - For multiprocessing you should probably not use fork as lance is - multi-threaded internally and fork and multi-thread do not work well. - Refer to [this discussion](https://discuss.python.org/t/concerns-regarding-deprecation-of-fork-with-alive-threads/33555). \ No newline at end of file +See the [lance-tensorflow README](https://github.com/lance-format/lance-tensorflow) +for the current installation and compatibility details. diff --git a/docs/src/quickstart/full-text-search.md b/docs/src/quickstart/full-text-search.md index f990b2bd589..de6bec7dd2c 100644 --- a/docs/src/quickstart/full-text-search.md +++ b/docs/src/quickstart/full-text-search.md @@ -98,6 +98,7 @@ ds.create_scalar_index( remove_stop_words=True, # Remove stop words (language-dependent) custom_stop_words=None, # Optional additional stop words (only used if remove_stop_words=True) ascii_folding=True, # Fold accents to ASCII when possible (e.g., "é" -> "e") + block_size=128, # Posting block size: 128 or 256; 256 is experimental ) ``` @@ -241,6 +242,9 @@ query_result = ds.to_table(full_text_query=(q1 & q2)) To combine `OR` queries via operators, use the pattern `q1 | q2`. +Every query combined with `AND` becomes a scoring `MUST` clause: all clauses must match, +and every matching clause contributes to the final `_score`. + #### Exclude terms: `NOT` Queries that exclude specific keywords are explicitly written using `BooleanQuery`/`Occur` diff --git a/docs/src/quickstart/index.md b/docs/src/quickstart/index.md index 34367c7177f..f10c5749014 100644 --- a/docs/src/quickstart/index.md +++ b/docs/src/quickstart/index.md @@ -105,4 +105,4 @@ dataset.to_table().to_pandas() Now that you've mastered the basics of creating Lance datasets, here's what you can explore next: - **[Versioning Your Datasets with Lance](versioning.md)** - Learn how to track changes over time with native versioning -- **[Vector Indexing and Vector Search With Lance](vector-search.md)** - Build high-performance vector search capabilities with ANN indexes +- **[Vector Indexing and Vector Search With Lance](vector-search.md)** - Build high-performance vector search capabilities with ANN indices diff --git a/docs/src/quickstart/vector-search.md b/docs/src/quickstart/vector-search.md index 6b1f6a5e516..c21bc6d9f37 100644 --- a/docs/src/quickstart/vector-search.md +++ b/docs/src/quickstart/vector-search.md @@ -1,13 +1,13 @@ --- title: Vector Search -description: High-performance vector search with ANN indexes, including IVF_PQ, IVF_HNSW_PQ, and IVF_HNSW_SQ +description: High-performance vector search with ANN indices, including IVF_PQ, IVF_HNSW_PQ, and IVF_HNSW_SQ --- # Vector Indexing and Vector Search With Lance -Lance provides high-performance vector search capabilities with ANN (Approximate Nearest Neighbor) indexes. +Lance provides high-performance vector search capabilities with ANN (Approximate Nearest Neighbor) indices. -By the end of this tutorial, you'll be able to build and use ANN indexes to dramatically speed up vector search operations while maintaining high accuracy. You'll also learn how to tune search parameters for optimal performance and combine vector search with metadata queries in a single operation. +By the end of this tutorial, you'll be able to build and use ANN indices to dramatically speed up vector search operations while maintaining high accuracy. You'll also learn how to tune search parameters for optimal performance and combine vector search with metadata queries in a single operation. ## Install the Python SDK diff --git a/docs/theme/404.html b/docs/theme/404.html new file mode 100644 index 00000000000..51897bd0805 --- /dev/null +++ b/docs/theme/404.html @@ -0,0 +1,13 @@ +{% extends "base.html" %} + +{% block container %} +
+
404 · Page not found
+

This page doesn't exist.

+

The page you're looking for may have moved.

+
+ Back to home + Getting started +
+
+{% endblock %} diff --git a/docs/theme/assets/site.css b/docs/theme/assets/site.css new file mode 100644 index 00000000000..a9d5286244c --- /dev/null +++ b/docs/theme/assets/site.css @@ -0,0 +1,792 @@ +/* Lance docs site chrome — faithful port of the "Lance Docs" design prototype. + Layout/nav classes (ld-*) come from the theme templates; article content + styles target python-markdown + pymdownx output. */ + +[hidden] { display: none !important; } + +.ld-shell { + min-height: 100vh; + display: flex; + flex-direction: column; + background: var(--surface-page); + font-family: var(--font-body); + color: var(--text-body); +} + +/* ---------- header ---------- */ +.ld-header { + position: sticky; + top: 0; + z-index: 60; + display: flex; + align-items: center; + gap: 20px; + height: 60px; + padding: 0 clamp(16px, 3vw, 32px); + min-width: 0; + background: var(--surface-header); + backdrop-filter: blur(12px); + border-bottom: 1px solid var(--line-1); +} +.ld-brand { + display: inline-flex; + align-items: center; + text-decoration: none; + user-select: none; + flex-shrink: 0; +} +.ld-brand img { display: block; height: 24px; width: auto; } +.ld-topnav { + display: flex; + gap: 2px; + margin-right: auto; + overflow-x: auto; + scrollbar-width: none; + min-width: 0; + flex: 1; +} +.ld-topnav::-webkit-scrollbar { display: none; } +.ld-toptab { + font-family: var(--font-body); + font-size: 14px; + font-weight: 500; + color: var(--text-muted); + text-decoration: none; + padding: 7px 12px; + cursor: pointer; + transition: color var(--dur-fast) var(--ease-out); + position: relative; + white-space: nowrap; +} +.ld-toptab:hover { color: var(--text-body); } +.ld-toptab.active { color: var(--fg-1); } +.ld-toptab.active::after { + content: ""; + position: absolute; + left: 12px; + right: 12px; + bottom: -10px; + height: 2px; + background: var(--beam-400); +} +.ld-header__actions { display: flex; gap: 10px; align-items: center; flex-shrink: 0; } +/* All header controls share one 34px height regardless of content (icon/text). */ +.ld-header__actions .ld-btn { height: 34px; padding-top: 0; padding-bottom: 0; } +.ld-header__actions .ld-btn--icon { width: 34px; padding: 0; justify-content: center; } + +/* ---------- buttons ---------- */ +.ld-btn { + display: inline-flex; + align-items: center; + gap: 8px; + text-decoration: none; + white-space: nowrap; + flex-shrink: 0; +} +.ld-btn--primary { + font-family: var(--font-body); + font-size: 13.5px; + font-weight: 500; + color: #ffffff; + background: var(--beam-400); + padding: 8px 16px; +} +.ld-btn--primary:hover { background: var(--beam-600); color: #ffffff; } +.ld-btn--ghost { + font-family: var(--font-mono); + font-size: 12.5px; + color: var(--text-secondary); + border: 1px solid var(--line-1); + padding: 7px 14px; +} +.ld-btn--ghost:hover { border-color: var(--beam-400); color: var(--fg-1); } +.ld-btn--icon { padding: 7px 9px; background: none; cursor: pointer; } +.ld-btn__count { + border-left: 1px solid var(--line-2); + padding-left: 8px; + color: var(--fg-1); + font-weight: 600; +} +:root[data-theme="light"] .ld-icon-sun { display: none; } +:root[data-theme="dark"] .ld-icon-moon { display: none; } +.ld-btn--outline { + font-size: 14px; + font-weight: 500; + color: var(--fg-1); + border: 1px solid var(--line-1); + padding: 11px 24px; +} +.ld-btn--outline:hover { border-color: var(--beam-400); color: var(--fg-1); } +.ld-btn--text { + font-size: 14px; + font-weight: 500; + color: var(--text-secondary); + padding: 11px 12px; +} +.ld-btn--text:hover { color: var(--fg-1); } +.ld-btn--lg.ld-btn--primary { font-size: 14px; padding: 12px 24px; } + +/* ---------- home: hero ---------- */ +.ld-hero { max-width: var(--container-max); margin: 0 auto; padding: 96px 32px 72px; width: 100%; } +.ld-kicker { + font-family: var(--font-mono); + font-size: 12.5px; + letter-spacing: var(--tracking-caps); + text-transform: uppercase; + color: var(--beam-600); + margin-bottom: 24px; +} +/* Wrap only between segments (at the · separators), never inside one. */ +.ld-kicker span, .ld-kicker a { white-space: nowrap; } +.ld-kicker a { color: inherit; text-decoration: underline; text-underline-offset: 3px; } +.ld-kicker a:hover { color: var(--beam-700); } +.ld-hero h1 { + font-family: var(--font-display); + font-size: 72px; + font-weight: 600; + letter-spacing: var(--tracking-display); + line-height: 0.97; + color: var(--fg-1); + margin: 0 0 28px; + max-width: 900px; + text-wrap: balance; +} +.ld-hero__lead { + font-size: 17px; + line-height: 1.6; + color: var(--text-secondary); + max-width: 640px; + margin: 0 0 36px; + text-wrap: pretty; +} +.ld-hero__cta { display: flex; gap: 12px; flex-wrap: wrap; } + +/* ---------- home: stats band ---------- */ +.ld-band { max-width: var(--container-max); margin: 0 auto; padding: 0 32px; width: 100%; } +.ld-stats { + display: grid; + grid-template-columns: 1fr 1fr 1fr; + border-top: 1px solid var(--line-1); + border-bottom: 1px solid var(--line-1); +} +.ld-stat { padding: 28px 32px; } +.ld-stat:first-child { padding-left: 0; } +.ld-stat:last-child { padding-right: 0; } +.ld-stat + .ld-stat { border-left: 1px solid var(--line-2); } +.ld-stat--link { display: block; text-decoration: none; } +.ld-stat--link .ld-stat__label { transition: color var(--dur-fast) var(--ease-out); } +.ld-stat--link:hover .ld-stat__label { color: var(--beam-600); } +.ld-stat__value { + font-family: var(--font-display); + font-size: 40px; + font-weight: 600; + letter-spacing: -0.03em; + color: var(--fg-1); +} +.ld-stat__value--beam { color: var(--beam-400); } +.ld-stat__label { + font-family: var(--font-mono); + font-size: 12px; + letter-spacing: 0.1em; + text-transform: uppercase; + color: var(--text-muted); + margin-top: 6px; +} + +/* ---------- home: what is lance ---------- */ +.ld-what { max-width: var(--container-max); margin: 0 auto; padding: 72px 32px 40px; width: 100%; } +.ld-what h2 { + font-family: var(--font-display); + font-size: 34px; + font-weight: 600; + letter-spacing: -0.03em; + color: var(--fg-1); + margin: 0 0 16px; +} +.ld-what__body { + font-size: 16px; + line-height: 1.65; + color: var(--text-secondary); + max-width: 760px; + margin: 0 0 12px; + text-wrap: pretty; +} +.ld-what__more { font-size: 15px; line-height: 1.6; color: var(--text-secondary); margin: 0; } +.ld-inline-link { + color: var(--beam-600); + text-decoration: underline; + text-decoration-thickness: 2px; + text-underline-offset: 3px; +} + +/* ---------- home: features ---------- */ +.ld-features { max-width: var(--container-max); margin: 0 auto; padding: 24px 32px 96px; width: 100%; } +.ld-feature { + display: grid; + grid-template-columns: 90px minmax(0, 1fr) minmax(0, 460px); + gap: 40px; + align-items: center; + border-top: 1px solid var(--line-1); + padding: 40px 0; +} +.ld-feature__num { + font-family: var(--font-mono); + font-size: 15px; + color: var(--beam-600); + align-self: start; + padding-top: 6px; +} +.ld-feature__body h3 { + font-family: var(--font-display); + font-size: 24px; + font-weight: 600; + letter-spacing: -0.02em; + color: var(--fg-1); + margin: 0 0 12px; +} +.ld-feature__body p { + font-size: 15px; + line-height: 1.65; + color: var(--text-secondary); + margin: 0 0 16px; + text-wrap: pretty; +} +.ld-more { + font-family: var(--font-mono); + font-size: 13px; + color: var(--beam-600); + text-decoration: none; +} +.ld-more:hover { text-decoration: underline; } +/* Logo grids stay on white in both themes — the artwork assumes a light background. */ +.ld-feature__img { border: 1px solid var(--line-1); padding: 16px; background: #ffffff; } +.ld-feature__img img { display: block; width: 100%; height: auto; } + +/* homepage code windows (always ink-dark, like docs code blocks) */ +.ld-feature__code { border: 1px solid var(--line-2); background: var(--ink-900); min-width: 0; border-radius: var(--radius-box); overflow: hidden; } +.ld-win { display: flex; gap: 6px; padding: 12px 16px 0; } +.ld-win i { width: 10px; height: 10px; border-radius: 999px; background: var(--ink-600); } +.ld-feature__code pre { + margin: 0; + padding: 14px 20px 18px; + overflow-x: auto; + font-family: var(--font-mono); + font-size: 12.5px; + line-height: 1.7; + color: #EDEBF5; +} +.ld-feature__code code { font-family: inherit; } +.tok-kw { color: #B8A8FF; } +.tok-str { color: #7CC0FF; } +.tok-com { color: #6F6A85; font-style: italic; } +.tok-num { color: #F2CE6B; } +.tok-arg { color: #FF95A8; } +.tok-fn { color: #8FE8CE; } + +/* ---------- docs layout ---------- */ +.ld-docs { + flex: 1; + display: grid; + grid-template-columns: 240px minmax(0, 1fr) 190px; + max-width: 1280px; + width: 100%; + margin: 0 auto; + gap: 44px; + padding: 0 32px; +} +.ld-sidenav { + border-right: 1px solid var(--line-2); + padding: 36px 24px 48px 0; + position: sticky; + top: 60px; + align-self: start; + height: calc(100vh - 60px); + overflow-y: auto; +} +/* Mobile-only disclosure for the sidenav; hidden on desktop where the + sidenav is a sticky column. */ +.ld-sidenav-toggle { + display: none; + align-items: center; + justify-content: space-between; + gap: 8px; + width: 100%; + font-family: var(--font-mono); + font-size: 11px; + font-weight: 600; + letter-spacing: var(--tracking-caps); + text-transform: uppercase; + color: var(--text-secondary); + background: none; + border: 1px solid var(--line-2); + padding: 9px 12px; + margin: 16px 0 0; + cursor: pointer; +} +.ld-sidenav-toggle:hover { color: var(--fg-1); border-color: var(--beam-400); } +.ld-sidenav-toggle svg { transition: transform var(--dur-fast) var(--ease-out); } +.ld-sidenav-toggle.open svg { transform: rotate(180deg); } +.ld-sidenav__group { margin-bottom: 6px; } +.ld-sidenav__label { + font-family: var(--font-mono); + font-size: 10.5px; + letter-spacing: var(--tracking-caps); + text-transform: uppercase; + color: var(--text-muted); + margin: 22px 0 8px; +} +.ld-sidenav a { + display: block; + font-size: 13.5px; + color: var(--text-secondary); + text-decoration: none; + padding: 5px 12px; + cursor: pointer; + border-left: 1px solid var(--line-2); + line-height: 1.45; +} +.ld-sidenav a:hover { color: var(--text-body); background: var(--surface-overlay); } +.ld-sidenav a.active { + color: var(--beam-600); + border-left: 2px solid var(--beam-400); + padding-left: 11px; + font-weight: 500; +} +.ld-sidenav a .ext { color: var(--text-muted); font-size: 11px; margin-left: 4px; } + +/* ---------- right-hand page TOC ---------- */ +.ld-toc { + position: sticky; + top: 60px; + align-self: start; + max-height: calc(100vh - 60px); + overflow-y: auto; + padding: 44px 0 48px; +} +.ld-toc__label { + font-family: var(--font-mono); + font-size: 10.5px; + letter-spacing: var(--tracking-caps); + text-transform: uppercase; + color: var(--text-muted); + margin: 0 0 10px; +} +.ld-toc a { + display: block; + font-size: 12.5px; + line-height: 1.45; + color: var(--text-muted); + text-decoration: none; + border-left: 1px solid var(--line-2); + padding: 4px 0 4px 12px; +} +.ld-toc a.ld-toc__h3 { padding-left: 24px; } +.ld-toc a:hover { color: var(--text-body); } +.ld-toc a.active { + color: var(--beam-600); + border-left: 2px solid var(--beam-400); + padding-left: 11px; + font-weight: 500; +} +.ld-toc a.ld-toc__h3.active { padding-left: 23px; } + +.ld-crumbs { + font-family: var(--font-mono); + font-size: 11.5px; + color: var(--text-muted); + display: flex; + gap: 8px; + flex-wrap: wrap; + margin-bottom: 24px; +} +.ld-crumbs b { color: var(--beam-600); font-weight: 500; } + +.ld-pagenav { + display: flex; + justify-content: space-between; + gap: 16px; + border-top: 1px solid var(--line-1); + margin-top: 56px; + padding-top: 20px; +} +.ld-pagenav a { text-decoration: none; color: var(--fg-1); } +.ld-pagenav a:hover { color: var(--beam-600); } +.ld-pagenav a.next { text-align: right; } +.ld-pagenav__k { + font-family: var(--font-mono); + font-size: 10.5px; + letter-spacing: var(--tracking-caps); + text-transform: uppercase; + color: var(--text-muted); + margin-bottom: 4px; +} +.ld-pagenav__t { font-size: 14.5px; font-weight: 500; } + +/* ---------- article prose (python-markdown output) ---------- */ +.ld-article-col { max-width: 760px; min-width: 0; padding: 40px 0 96px; } +.ld-article { font-family: var(--font-body); color: var(--text-secondary); } +/* Keep anchor targets clear of the sticky header. */ +.ld-article :is(h1, h2, h3, h4, h5, h6) { scroll-margin-top: 76px; } +.ld-article h1 { font-family: var(--font-display); font-size: 42px; font-weight: 600; letter-spacing: -0.03em; line-height: 1.05; color: var(--fg-1); margin: 0 0 20px; text-wrap: balance; } +.ld-article h2 { font-family: var(--font-display); font-size: 26px; font-weight: 600; letter-spacing: -0.02em; color: var(--fg-1); margin: 48px 0 14px; padding-top: 28px; border-top: 1px solid var(--line-2); } +.ld-article h3 { font-family: var(--font-display); font-size: 19px; font-weight: 600; letter-spacing: -0.01em; color: var(--fg-1); margin: 32px 0 10px; } +.ld-article h4, .ld-article h5, .ld-article h6 { font-family: var(--font-mono); font-size: 12.5px; font-weight: 600; letter-spacing: .14em; text-transform: uppercase; color: var(--fg-1); margin: 28px 0 8px; } +.ld-article p { font-size: 15.5px; line-height: 1.65; margin: 0 0 16px; text-wrap: pretty; } +.ld-article a { color: var(--beam-600); text-decoration: underline; text-decoration-thickness: 2px; text-underline-offset: 3px; text-decoration-color: rgba(98, 94, 255, .35); } +.ld-article a:hover { text-decoration-color: var(--beam-600); } +.ld-article code { font-family: var(--font-mono); font-size: 13px; background: var(--surface-inline-code); border-radius: var(--radius-chip); padding: 1px 5px; color: var(--fg-1); } +.ld-article ul, .ld-article ol { margin: 0 0 16px; padding-left: 22px; font-size: 15.5px; line-height: 1.65; } +.ld-article li { margin-bottom: 6px; } +.ld-article li p { margin-bottom: 8px; } +.ld-article hr { border: 0; border-top: 1px solid var(--line-1); margin: 40px 0; } +.ld-article blockquote { margin: 0 0 16px; padding: 4px 0 4px 20px; border-left: 2px solid var(--beam-400); } +.ld-article blockquote p:last-child { margin-bottom: 0; } + +/* heading permalinks (toc: permalink) */ +.ld-article .headerlink { + margin-left: 8px; + color: var(--beam-400); + text-decoration: none; + opacity: 0; + transition: opacity var(--dur-fast) var(--ease-out); +} +.ld-article :is(h1, h2, h3, h4, h5, h6):hover .headerlink { opacity: 1; } + +/* figures: lone images sit on a white plate in both themes */ +.ld-article img { max-width: 100%; height: auto; } +.ld-article p > img:only-child { + display: block; + margin: 24px auto; + border: 1px solid var(--line-1); + padding: 16px; + background: #ffffff; +} + +/* tables (JS wraps them in .ld-tablewrap for overflow) */ +.ld-tablewrap { overflow-x: auto; margin: 0 0 20px; border: 1px solid var(--line-1); } +.ld-article table { border-collapse: collapse; width: 100%; font-size: 14px; } +.ld-article th { font-family: var(--font-mono); font-size: 11px; font-weight: 600; letter-spacing: .12em; text-transform: uppercase; text-align: left; color: var(--fg-1); background: var(--surface-overlay); padding: 10px 14px; border-bottom: 1px solid var(--line-1); } +.ld-article td { padding: 10px 14px; border-bottom: 1px solid var(--line-2); vertical-align: top; line-height: 1.55; color: var(--text-secondary); } +.ld-article tr:last-child td { border-bottom: 0; } + +/* ---------- code blocks (pymdownx.highlight output, JS adds the bar) ---------- */ +.ld-code { margin: 0 0 20px; border: 1px solid var(--line-2); border-radius: var(--radius-box); overflow: hidden; } +.ld-code__bar { display: flex; justify-content: space-between; align-items: center; padding: 7px 14px; border-bottom: 1px solid var(--line-2); background: var(--surface-card); } +.ld-code__bar span { display: inline-flex; align-items: center; gap: 7px; font-family: var(--font-mono); font-size: 11px; letter-spacing: .12em; text-transform: uppercase; color: var(--text-muted); } +.ld-code__bar span svg { width: 13px; height: 13px; display: block; flex-shrink: 0; } +.ld-copy { font-family: var(--font-mono); font-size: 11px; letter-spacing: .08em; text-transform: uppercase; background: none; border: none; color: var(--beam-600); cursor: pointer; padding: 2px 0; } +.ld-copy:hover { color: var(--beam-700); } +.ld-code .highlight { margin: 0; } +.ld-code pre { margin: 0; background: var(--surface-code); padding: 16px 18px; overflow-x: auto; } +.ld-code pre code { background: none; border: none; padding: 0; font-size: 13px; line-height: 1.6; color: var(--code-fg); } +/* Ink code surfaces (dark-theme docs blocks + the always-dark homepage windows) + need an explicit light selection color; light-theme docs blocks use the default. */ +:root[data-theme="dark"] .ld-code pre ::selection, :root[data-theme="dark"] .ld-code pre::selection, +.ld-feature__code pre ::selection, .ld-feature__code pre::selection { + background: rgba(98, 94, 255, 0.55); + color: #ffffff; +} + +/* Pygments tokens — colors resolve per theme via the --code-* palette */ +.highlight .k, .highlight .kn, .highlight .kd, .highlight .kt, .highlight .kr, .highlight .kp, .highlight .ow { color: var(--code-kw); } +.highlight .kc { color: var(--code-kw); } +.highlight .s, .highlight .s1, .highlight .s2, .highlight .sb, .highlight .sd, .highlight .sa, .highlight .se, .highlight .si, .highlight .sx, .highlight .sr, .highlight .ss, .highlight .sh { color: var(--code-str); } +.highlight .c, .highlight .c1, .highlight .cm, .highlight .ch, .highlight .cs, .highlight .cp, .highlight .cpf { color: var(--code-com); font-style: italic; } +.highlight .m, .highlight .mi, .highlight .mf, .highlight .mh, .highlight .mo, .highlight .mb, .highlight .il { color: var(--code-num); } +.highlight .nf, .highlight .fm, .highlight .nd, .highlight .ne { color: var(--code-fn); } +.highlight .nc, .highlight .nn { color: var(--code-name); font-weight: 500; } +.highlight .nb, .highlight .bp { color: var(--code-kw); } +.highlight .nt { color: var(--code-tag); } +.highlight .na { color: var(--code-tag); } +.highlight .o { color: var(--code-punct); } +.highlight .p { color: var(--code-punct); } +.highlight .gp { color: var(--code-prompt); } +.highlight .go { color: var(--code-punct); } +.highlight .gh, .highlight .gu { color: var(--code-name); font-weight: 600; } +.highlight .hll { background: var(--code-hll); display: block; } + +/* mermaid diagrams: white plate (rendered by JS) */ +.ld-article pre.mermaid, .ld-article div.mermaid { + border: 1px solid var(--line-1); + padding: 16px; + background: #ffffff; + margin: 0 0 20px; + text-align: center; + overflow-x: auto; +} + +/* ---------- admonitions ---------- */ +.ld-article .admonition { + border: 1px solid var(--line-2); + border-radius: var(--radius-box); + overflow: hidden; + margin: 0 0 20px; + background: var(--surface-card); + padding: 0; + font-size: 14.5px; +} +.ld-article .admonition > .admonition-title { + display: flex; + align-items: center; + gap: 8px; + font-family: var(--font-mono); + font-size: 11px; + font-weight: 600; + letter-spacing: .14em; + text-transform: uppercase; + color: var(--beam-600); + padding: 9px 16px; + margin: 0; + border-bottom: 1px solid var(--line-2); + background: none; +} +/* Material-style type icon, drawn in the title color via a CSS mask */ +.ld-article .admonition > .admonition-title::before { + content: ""; + width: 14px; + height: 14px; + flex-shrink: 0; + background-color: currentColor; + -webkit-mask: var(--ld-adm-icon) no-repeat center / contain; + mask: var(--ld-adm-icon) no-repeat center / contain; +} +/* Icon shapes are Material Design Icons paths (Apache 2.0), matching mkdocs-material's defaults. */ +.ld-article .admonition { --ld-adm-icon: url('data:image/svg+xml;charset=utf-8,%3Csvg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"%3E%3Cpath d="M20.71,7.04C21.1,6.65 21.1,6 20.71,5.63L18.37,3.29C18,2.9 17.35,2.9 16.96,3.29L15.12,5.12L18.87,8.87M3,17.25V21H6.75L17.81,9.93L14.06,6.18L3,17.25Z"/%3E%3C/svg%3E'); } +.ld-article :is(.admonition.abstract, .admonition.summary, .admonition.tldr) { --ld-adm-icon: url('data:image/svg+xml;charset=utf-8,%3Csvg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"%3E%3Cpath d="M17,9H7V7H17M17,13H7V11H17M14,17H7V15H14M12,3A1,1 0 0,1 13,4A1,1 0 0,1 12,5A1,1 0 0,1 11,4A1,1 0 0,1 12,3M19,3H14.82C14.4,1.84 13.3,1 12,1C10.7,1 9.6,1.84 9.18,3H5A2,2 0 0,0 3,5V19A2,2 0 0,0 5,21H19A2,2 0 0,0 21,19V5A2,2 0 0,0 19,3Z"/%3E%3C/svg%3E'); } +.ld-article :is(.admonition.info, .admonition.todo) { --ld-adm-icon: url('data:image/svg+xml;charset=utf-8,%3Csvg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"%3E%3Cpath d="M13,9H11V7H13M13,17H11V11H13M12,2A10,10 0 0,0 2,12A10,10 0 0,0 12,22A10,10 0 0,0 22,12A10,10 0 0,0 12,2Z"/%3E%3C/svg%3E'); } +.ld-article :is(.admonition.tip, .admonition.hint, .admonition.important) { --ld-adm-icon: url('data:image/svg+xml;charset=utf-8,%3Csvg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"%3E%3Cpath d="M17.66,11.2C17.43,10.9 17.15,10.64 16.89,10.38C16.22,9.78 15.46,9.35 14.82,8.72C13.33,7.26 13,4.85 13.95,3C13,3.23 12.17,3.75 11.46,4.32C8.87,6.4 7.85,10.07 9.07,13.22C9.11,13.32 9.15,13.42 9.15,13.55C9.15,13.77 9,13.97 8.8,14.05C8.57,14.15 8.33,14.09 8.14,13.93C8.08,13.88 8.04,13.83 8,13.76C6.87,12.33 6.69,10.28 7.45,8.64C5.78,10 4.87,12.3 5,14.47C5.06,14.97 5.12,15.47 5.29,15.97C5.43,16.57 5.7,17.17 6,17.7C7.08,19.43 8.95,20.67 10.96,20.92C13.1,21.19 15.39,20.8 17.03,19.32C18.86,17.66 19.5,15 18.56,12.72L18.43,12.46C18.22,12 17.66,11.2 17.66,11.2M14.5,17.5C14.22,17.74 13.76,18 13.4,18.1C12.28,18.5 11.16,17.94 10.5,17.28C11.69,17 12.4,16.12 12.61,15.23C12.78,14.43 12.46,13.77 12.33,13C12.21,12.26 12.23,11.63 12.5,10.94C12.69,11.32 12.89,11.7 13.13,12C13.9,13 15.11,13.44 15.37,14.8C15.41,14.94 15.43,15.08 15.43,15.23C15.46,16.05 15.1,16.95 14.5,17.5H14.5Z"/%3E%3C/svg%3E'); } +.ld-article :is(.admonition.success, .admonition.check, .admonition.done) { --ld-adm-icon: url('data:image/svg+xml;charset=utf-8,%3Csvg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"%3E%3Cpath d="M9,20.42L2.79,14.21L5.62,11.38L9,14.77L18.88,4.88L21.71,7.71L9,20.42Z"/%3E%3C/svg%3E'); } +.ld-article :is(.admonition.question, .admonition.help, .admonition.faq) { --ld-adm-icon: url('data:image/svg+xml;charset=utf-8,%3Csvg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"%3E%3Cpath d="M15.07,11.25L14.17,12.17C13.45,12.89 13,13.5 13,15H11V14.5C11,13.39 11.45,12.39 12.17,11.67L13.41,10.41C13.78,10.05 14,9.55 14,9C14,7.89 13.1,7 12,7A2,2 0 0,0 10,9H8A4,4 0 0,1 12,5A4,4 0 0,1 16,9C16,9.88 15.64,10.67 15.07,11.25M13,19H11V17H13M12,2A10,10 0 0,0 2,12A10,10 0 0,0 12,22A10,10 0 0,0 22,12C22,6.47 17.5,2 12,2Z"/%3E%3C/svg%3E'); } +.ld-article :is(.admonition.warning, .admonition.caution, .admonition.attention) { --ld-adm-icon: url('data:image/svg+xml;charset=utf-8,%3Csvg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"%3E%3Cpath d="M13,14H11V9H13M13,18H11V16H13M1,21H23L12,2L1,21Z"/%3E%3C/svg%3E'); } +.ld-article :is(.admonition.failure, .admonition.fail, .admonition.missing) { --ld-adm-icon: url('data:image/svg+xml;charset=utf-8,%3Csvg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"%3E%3Cpath d="M20 6.91L17.09 4L12 9.09L6.91 4L4 6.91L9.09 12L4 17.09L6.91 20L12 14.91L17.09 20L20 17.09L14.91 12L20 6.91Z"/%3E%3C/svg%3E'); } +.ld-article :is(.admonition.danger, .admonition.error) { --ld-adm-icon: url('data:image/svg+xml;charset=utf-8,%3Csvg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"%3E%3Cpath d="M11,15H6L13,1V9H18L11,23V15Z"/%3E%3C/svg%3E'); } +.ld-article .admonition.bug { --ld-adm-icon: url('data:image/svg+xml;charset=utf-8,%3Csvg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"%3E%3Cpath d="M14,12H10V10H14M14,16H10V14H14M20,8H17.19C16.74,7.22 16.12,6.55 15.37,6.04L17,4.41L15.59,3L13.42,5.17C12.96,5.06 12.5,5 12,5C11.5,5 11.04,5.06 10.59,5.17L8.41,3L7,4.41L8.62,6.04C7.88,6.55 7.26,7.22 6.81,8H4V10H6.09C6.04,10.33 6,10.66 6,11V12H4V14H6V15C6,15.34 6.04,15.67 6.09,16H4V18H6.81C7.85,19.79 9.78,21 12,21C14.22,21 16.15,19.79 17.19,18H20V16H17.91C17.96,15.67 18,15.34 18,15V14H20V12H18V11C18,10.66 17.96,10.33 17.91,10H20V8Z"/%3E%3C/svg%3E'); } +.ld-article .admonition.example { --ld-adm-icon: url('data:image/svg+xml;charset=utf-8,%3Csvg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"%3E%3Cpath d="M7,13V11H21V13H7M7,19V17H21V19H7M7,7V5H21V7H7M3,8V5H2V4H4V8H3M2,17V16H5V20H2V19H4V18.5H3V17.5H4V17H2M4.25,10A0.75,0.75 0 0,1 5,10.75C5,10.95 4.92,11.14 4.79,11.27L3.12,13H5V14H2V13.08L4,11H2V10H4.25Z"/%3E%3C/svg%3E'); } +.ld-article :is(.admonition.quote, .admonition.cite) { --ld-adm-icon: url('data:image/svg+xml;charset=utf-8,%3Csvg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"%3E%3Cpath d="M14,17H17L19,13V7H13V13H16M6,17H9L11,13V7H5V13H8L6,17Z"/%3E%3C/svg%3E'); } +.ld-article .admonition > :not(.admonition-title) { margin: 14px 16px; font-size: 14.5px; } +.ld-article .admonition > .highlight, .ld-article .admonition > .ld-code { margin: 14px 16px; } +.ld-article .admonition.warning > .admonition-title, +.ld-article .admonition.caution > .admonition-title, +.ld-article .admonition.attention > .admonition-title { color: var(--warn-500); } +.ld-article .admonition.danger > .admonition-title, +.ld-article .admonition.error > .admonition-title, +.ld-article .admonition.bug > .admonition-title, +.ld-article .admonition.failure > .admonition-title { color: var(--danger-500); } +.ld-article .admonition.success > .admonition-title, +.ld-article .admonition.check > .admonition-title { color: var(--ok-500); } + +/* ---------- details / summary (pymdownx.details) ---------- */ +.ld-article details { + border: 1px solid var(--line-2); + border-radius: var(--radius-box); + overflow: hidden; + margin: 0 0 20px; + background: var(--surface-card); +} +.ld-article details > summary { + font-family: var(--font-mono); + font-size: 12px; + font-weight: 600; + letter-spacing: .1em; + text-transform: uppercase; + color: var(--beam-600); + padding: 10px 16px; + cursor: pointer; + user-select: none; + list-style: none; +} +.ld-article details > summary::-webkit-details-marker { display: none; } +.ld-article details > summary:hover { background: var(--surface-overlay); } +.ld-article details[open] > summary { border-bottom: 1px solid var(--line-2); } +.ld-article details > :not(summary) { margin: 14px 16px; } + +/* ---------- content tabs (pymdownx.tabbed, alternate style) ---------- */ +.ld-article .tabbed-set { margin: 0 0 20px; border: 1px solid var(--line-2); border-radius: var(--radius-box); overflow: hidden; position: relative; } +.ld-article .tabbed-set > input { position: absolute; opacity: 0; pointer-events: none; } +.ld-article .tabbed-labels { + display: flex; + border-bottom: 1px solid var(--line-2); + background: var(--surface-card); + overflow-x: auto; + scrollbar-width: none; +} +.ld-article .tabbed-labels > label { + font-family: var(--font-mono); + font-size: 12px; + letter-spacing: .06em; + padding: 9px 18px; + border-right: 1px solid var(--line-2); + color: var(--text-muted); + cursor: pointer; + white-space: nowrap; +} +.ld-article .tabbed-labels > label:hover { color: var(--fg-1); } +.ld-article .tabbed-content { display: block; } +.ld-article .tabbed-block { display: none; padding: 16px 16px 0; } +.ld-article .tabbed-block > :last-child { margin-bottom: 16px; } +.ld-article .tabbed-block > .ld-code:last-child { margin-bottom: 16px; } +/* nth-input → nth-label/nth-block mapping (supports up to 8 tabs) */ +.ld-article .tabbed-set > input:nth-child(1):checked ~ .tabbed-labels > label:nth-child(1), +.ld-article .tabbed-set > input:nth-child(2):checked ~ .tabbed-labels > label:nth-child(2), +.ld-article .tabbed-set > input:nth-child(3):checked ~ .tabbed-labels > label:nth-child(3), +.ld-article .tabbed-set > input:nth-child(4):checked ~ .tabbed-labels > label:nth-child(4), +.ld-article .tabbed-set > input:nth-child(5):checked ~ .tabbed-labels > label:nth-child(5), +.ld-article .tabbed-set > input:nth-child(6):checked ~ .tabbed-labels > label:nth-child(6), +.ld-article .tabbed-set > input:nth-child(7):checked ~ .tabbed-labels > label:nth-child(7), +.ld-article .tabbed-set > input:nth-child(8):checked ~ .tabbed-labels > label:nth-child(8) { + color: var(--fg-1); + box-shadow: inset 0 -2px 0 var(--beam-400); +} +.ld-article .tabbed-set > input:nth-child(1):checked ~ .tabbed-content > .tabbed-block:nth-child(1), +.ld-article .tabbed-set > input:nth-child(2):checked ~ .tabbed-content > .tabbed-block:nth-child(2), +.ld-article .tabbed-set > input:nth-child(3):checked ~ .tabbed-content > .tabbed-block:nth-child(3), +.ld-article .tabbed-set > input:nth-child(4):checked ~ .tabbed-content > .tabbed-block:nth-child(4), +.ld-article .tabbed-set > input:nth-child(5):checked ~ .tabbed-content > .tabbed-block:nth-child(5), +.ld-article .tabbed-set > input:nth-child(6):checked ~ .tabbed-content > .tabbed-block:nth-child(6), +.ld-article .tabbed-set > input:nth-child(7):checked ~ .tabbed-content > .tabbed-block:nth-child(7), +.ld-article .tabbed-set > input:nth-child(8):checked ~ .tabbed-content > .tabbed-block:nth-child(8) { + display: block; +} + +/* ---------- search overlay ---------- */ +.ld-search { position: fixed; inset: 0; z-index: 100; } +.ld-search__scrim { position: absolute; inset: 0; background: rgba(14, 12, 20, 0.5); } +.ld-search__panel { + position: relative; + max-width: 640px; + margin: 96px auto 0; + background: var(--surface-page); + border: 1px solid var(--line-1); +} +.ld-search__bar { display: flex; align-items: center; border-bottom: 1px solid var(--line-1); } +.ld-search__input { + flex: 1; + font-family: var(--font-body); + font-size: 15px; + color: var(--fg-1); + background: none; + border: none; + outline: none; + padding: 14px 16px; +} +.ld-search__input::placeholder { color: var(--text-muted); } +.ld-search__close { + font-family: var(--font-mono); + font-size: 10.5px; + letter-spacing: .1em; + text-transform: uppercase; + color: var(--text-muted); + background: none; + border: 1px solid var(--line-2); + margin-right: 12px; + padding: 3px 8px; + cursor: pointer; +} +.ld-search__close:hover { color: var(--fg-1); border-color: var(--beam-400); } +.ld-search__results { max-height: 55vh; overflow-y: auto; } +.ld-search__hit { display: block; padding: 12px 16px; text-decoration: none; border-top: 1px solid var(--line-2); } +.ld-search__hit:first-child { border-top: 0; } +.ld-search__hit:hover, .ld-search__hit.active { background: var(--surface-overlay); } +.ld-search__hit-title { font-size: 14px; font-weight: 500; color: var(--fg-1); } +.ld-search__hit-title mark, .ld-search__hit-text mark { background: none; color: var(--beam-600); } +.ld-search__hit-crumb { font-family: var(--font-mono); font-size: 10.5px; letter-spacing: .08em; text-transform: uppercase; color: var(--text-muted); margin-bottom: 2px; } +.ld-search__hit-text { font-size: 12.5px; color: var(--text-muted); line-height: 1.5; margin-top: 2px; } +.ld-search__empty { font-family: var(--font-mono); font-size: 12px; letter-spacing: .08em; text-transform: uppercase; color: var(--text-muted); padding: 20px 16px; } + +/* ---------- 404 ---------- */ +.ld-notfound { max-width: var(--container-max); margin: 0 auto; padding: 120px 32px 160px; width: 100%; flex: 1; } +.ld-notfound h1 { + font-family: var(--font-display); + font-size: 56px; + font-weight: 600; + letter-spacing: var(--tracking-display); + color: var(--fg-1); + margin: 0 0 16px; +} +.ld-notfound p { color: var(--text-secondary); margin: 0 0 32px; } + +/* ---------- footer ---------- */ +.ld-footer { + border-top: 1px solid var(--line-1); + padding: 40px 32px; + display: flex; + align-items: flex-start; + justify-content: space-between; + gap: 32px; + flex-wrap: wrap; +} +.ld-footer__brand img { display: block; height: 24px; width: auto; } +.ld-footer__brand p { + font-size: 12.5px; + color: var(--text-muted); + max-width: 300px; + line-height: 1.6; + margin: 12px 0 0; +} +.ld-footer__cols { display: flex; gap: 64px; flex-wrap: wrap; } +.ld-footer__col { display: flex; flex-direction: column; gap: 8px; } +.ld-footer__col h5 { + font-family: var(--font-mono); + font-size: 10.5px; + letter-spacing: var(--tracking-caps); + text-transform: uppercase; + color: var(--text-muted); + margin: 0 0 4px; +} +.ld-footer__col a { font-size: 13.5px; color: var(--text-secondary); text-decoration: none; } +.ld-footer__col a:hover { color: var(--fg-1); } + +/* ---------- responsive ---------- */ +@media (max-width: 1180px) { + .ld-docs { grid-template-columns: 240px minmax(0, 1fr); } + .ld-toc { display: none; } +} +@media (max-width: 960px) { + /* Header folds into two rows: brand + icon actions, then a scrollable + section-tab row. Text-heavy controls collapse to icons. */ + .ld-header { flex-wrap: wrap; height: auto; gap: 0 10px; padding: 10px 16px 0; } + .ld-header__actions { margin-left: auto; } + .ld-header__actions .ld-btn--primary, + .ld-header__actions .ld-btn__label, + .ld-header__actions .ld-btn__count { display: none; } + .ld-topnav { order: 1; flex-basis: 100%; margin: 4px 0 0; padding-bottom: 10px; } + .ld-toptab { padding: 7px 10px; } + .ld-toptab:first-child { margin-left: -10px; } + .ld-toptab.active::after { left: 10px; right: 10px; } + + .ld-article :is(h1, h2, h3, h4, h5, h6) { scroll-margin-top: 104px; } + .ld-article h1 { font-size: 32px; } + + .ld-hero h1 { font-size: clamp(38px, 9vw, 56px); } + .ld-stats { grid-template-columns: 1fr; } + .ld-stat { padding: 20px 0; } + .ld-stat + .ld-stat { border-left: 0; border-top: 1px solid var(--line-2); } + .ld-feature { grid-template-columns: 1fr; gap: 16px; } + .ld-feature__num { padding-top: 0; } + + /* Docs: sidenav collapses behind a disclosure button; article leads. */ + .ld-docs { display: block; padding: 0 20px; } + .ld-sidenav-toggle { display: flex; } + .ld-sidenav { + display: none; + position: static; + height: auto; + border-right: 0; + border-bottom: 1px solid var(--line-2); + padding: 0 0 20px; + } + .ld-sidenav.open { display: block; } + .ld-sidenav__group:first-child .ld-sidenav__label { margin-top: 16px; } + .ld-article-col { padding-top: 24px; } + + .ld-hero, .ld-band, .ld-what, .ld-features { padding-left: 20px; padding-right: 20px; } + .ld-hero { padding-top: 56px; padding-bottom: 48px; } + .ld-notfound { padding: 72px 20px 96px; } + .ld-footer { padding: 32px 20px; } + .ld-footer__cols { gap: 28px 40px; } +} +@media (max-width: 700px) { + /* Search becomes an edge-to-edge sheet under the top of the screen. */ + .ld-search__panel { margin: 0; max-width: none; border-left: 0; border-right: 0; border-top: 0; } + .ld-search__results { max-height: calc(100dvh - 110px); } +} diff --git a/docs/theme/assets/site.js b/docs/theme/assets/site.js new file mode 100644 index 00000000000..34378c820b2 --- /dev/null +++ b/docs/theme/assets/site.js @@ -0,0 +1,263 @@ +/* Lance docs theme behaviours: theme toggle, GitHub stars, code block chrome, + TOC scroll-spy, search overlay, mermaid rendering. */ +(function () { + "use strict"; + + var BASE = (window.LD_BASE || ".").replace(/\/$/, ""); + + /* ---------- theme toggle ---------- */ + var themeBtn = document.getElementById("theme-toggle"); + if (themeBtn) { + themeBtn.addEventListener("click", function () { + var next = document.documentElement.dataset.theme === "dark" ? "light" : "dark"; + document.documentElement.dataset.theme = next; + try { localStorage.setItem("ld-theme", next); } catch (e) { /* private mode */ } + }); + } + + /* ---------- GitHub stars ---------- */ + (function loadStars() { + var el = document.getElementById("gh-stars"); + if (!el) return; + var TTL = 3600e3; + function show(count) { + if (!(count > 0)) return; + var label = count >= 1000 ? (count / 1000).toFixed(1).replace(/\.0$/, "") + "k" : String(count); + el.textContent = "★ " + label; + el.hidden = false; + } + try { + var cached = JSON.parse(localStorage.getItem("ld-gh-stars") || "null"); + if (cached && Date.now() - cached.t < TTL) { show(cached.v); return; } + } catch (e) { /* fall through to fetch */ } + fetch("https://api.github.com/repos/lance-format/lance") + .then(function (r) { if (!r.ok) throw new Error(r.status); return r.json(); }) + .then(function (d) { + show(d.stargazers_count); + try { localStorage.setItem("ld-gh-stars", JSON.stringify({ v: d.stargazers_count, t: Date.now() })); } catch (e) { /* ignore */ } + }) + .catch(function () { /* rate-limited or offline: button still works without the count */ }); + })(); + + /* ---------- mobile sidenav toggle ---------- */ + var sidenavToggle = document.querySelector(".ld-sidenav-toggle"); + var sidenav = document.querySelector(".ld-sidenav"); + if (sidenavToggle && sidenav) { + sidenavToggle.addEventListener("click", function () { + var open = sidenav.classList.toggle("open"); + sidenavToggle.classList.toggle("open", open); + sidenavToggle.setAttribute("aria-expanded", open ? "true" : "false"); + }); + } + + /* ---------- article enhancements ---------- */ + var article = document.querySelector(".ld-article"); + + if (article) { + // Language logos for code-bar labels (Simple Icons + Devicon paths, drawn in currentColor). + var LANG_ICONS = { + python: ["0 0 24 24", "M14.25.18l.9.2.73.26.59.3.45.32.34.34.25.34.16.33.1.3.04.26.02.2-.01.13V8.5l-.05.63-.13.55-.21.46-.26.38-.3.31-.33.25-.35.19-.35.14-.33.1-.3.07-.26.04-.21.02H8.77l-.69.05-.59.14-.5.22-.41.27-.33.32-.27.35-.2.36-.15.37-.1.35-.07.32-.04.27-.02.21v3.06H3.17l-.21-.03-.28-.07-.32-.12-.35-.18-.36-.26-.36-.36-.35-.46-.32-.59-.28-.73-.21-.88-.14-1.05-.05-1.23.06-1.22.16-1.04.24-.87.32-.71.36-.57.4-.44.42-.33.42-.24.4-.16.36-.1.32-.05.24-.01h.16l.06.01h8.16v-.83H6.18l-.01-2.75-.02-.37.05-.34.11-.31.17-.28.25-.26.31-.23.38-.2.44-.18.51-.15.58-.12.64-.1.71-.06.77-.04.84-.02 1.27.05zm-6.3 1.98l-.23.33-.08.41.08.41.23.34.33.22.41.09.41-.09.33-.22.23-.34.08-.41-.08-.41-.23-.33-.33-.22-.41-.09-.41.09zm13.09 3.95l.28.06.32.12.35.18.36.27.36.35.35.47.32.59.28.73.21.88.14 1.04.05 1.23-.06 1.23-.16 1.04-.24.86-.32.71-.36.57-.4.45-.42.33-.42.24-.4.16-.36.09-.32.05-.24.02-.16-.01h-8.22v.82h5.84l.01 2.76.02.36-.05.34-.11.31-.17.29-.25.25-.31.24-.38.2-.44.17-.51.15-.58.13-.64.09-.71.07-.77.04-.84.01-1.27-.04-1.07-.14-.9-.2-.73-.25-.59-.3-.45-.33-.34-.34-.25-.34-.16-.33-.1-.3-.04-.25-.02-.2.01-.13v-5.34l.05-.64.13-.54.21-.46.26-.38.3-.32.33-.24.35-.2.35-.14.33-.1.3-.06.26-.04.21-.02.13-.01h5.84l.69-.05.59-.14.5-.21.41-.28.33-.32.27-.35.2-.36.15-.36.1-.35.07-.32.04-.28.02-.21V6.07h2.09l.14.01zm-6.47 14.25l-.23.33-.08.41.08.41.23.33.33.23.41.08.41-.08.33-.23.23-.33.08-.41-.08-.41-.23-.33-.33-.23-.41-.08-.41.08z"], + rust: ["0 0 24 24", "M23.8346 11.7033l-1.0073-.6236a13.7268 13.7268 0 00-.0283-.2936l.8656-.8069a.3483.3483 0 00-.1154-.578l-1.1066-.414a8.4958 8.4958 0 00-.087-.2856l.6904-.9587a.3462.3462 0 00-.2257-.5446l-1.1663-.1894a9.3574 9.3574 0 00-.1407-.2622l.49-1.0761a.3437.3437 0 00-.0274-.3361.3486.3486 0 00-.3006-.154l-1.1845.0416a6.7444 6.7444 0 00-.1873-.2268l.2723-1.153a.3472.3472 0 00-.417-.4172l-1.1532.2724a14.0183 14.0183 0 00-.2278-.1873l.0415-1.1845a.3442.3442 0 00-.49-.328l-1.076.491c-.0872-.0476-.1742-.0952-.2623-.1407l-.1903-1.1673A.3483.3483 0 0016.256.955l-.9597.6905a8.4867 8.4867 0 00-.2855-.086l-.414-1.1066a.3483.3483 0 00-.5781-.1154l-.8069.8666a9.2936 9.2936 0 00-.2936-.0284L12.2946.1683a.3462.3462 0 00-.5892 0l-.6236 1.0073a13.7383 13.7383 0 00-.2936.0284L9.9803.3374a.3462.3462 0 00-.578.1154l-.4141 1.1065c-.0962.0274-.1903.0567-.2855.086L7.744.955a.3483.3483 0 00-.5447.2258L7.009 2.348a9.3574 9.3574 0 00-.2622.1407l-1.0762-.491a.3462.3462 0 00-.49.328l.0416 1.1845a7.9826 7.9826 0 00-.2278.1873L3.8413 3.425a.3472.3472 0 00-.4171.4171l.2713 1.1531c-.0628.075-.1255.1509-.1863.2268l-1.1845-.0415a.3462.3462 0 00-.328.49l.491 1.0761a9.167 9.167 0 00-.1407.2622l-1.1662.1894a.3483.3483 0 00-.2258.5446l.6904.9587a13.303 13.303 0 00-.087.2855l-1.1065.414a.3483.3483 0 00-.1155.5781l.8656.807a9.2936 9.2936 0 00-.0283.2935l-1.0073.6236a.3442.3442 0 000 .5892l1.0073.6236c.008.0982.0182.1964.0283.2936l-.8656.8079a.3462.3462 0 00.1155.578l1.1065.4141c.0273.0962.0567.1914.087.2855l-.6904.9587a.3452.3452 0 00.2268.5447l1.1662.1893c.0456.088.0922.1751.1408.2622l-.491 1.0762a.3462.3462 0 00.328.49l1.1834-.0415c.0618.0769.1235.1528.1873.2277l-.2713 1.1541a.3462.3462 0 00.4171.4161l1.153-.2713c.075.0638.151.1255.2279.1863l-.0415 1.1845a.3442.3442 0 00.49.327l1.0761-.49c.087.0486.1741.0951.2622.1407l.1903 1.1662a.3483.3483 0 00.5447.2268l.9587-.6904a9.299 9.299 0 00.2855.087l.414 1.1066a.3452.3452 0 00.5781.1154l.8079-.8656c.0972.0111.1954.0203.2936.0294l.6236 1.0073a.3472.3472 0 00.5892 0l.6236-1.0073c.0982-.0091.1964-.0183.2936-.0294l.8069.8656a.3483.3483 0 00.578-.1154l.4141-1.1066a8.4626 8.4626 0 00.2855-.087l.9587.6904a.3452.3452 0 00.5447-.2268l.1903-1.1662c.088-.0456.1751-.0931.2622-.1407l1.0762.49a.3472.3472 0 00.49-.327l-.0415-1.1845a6.7267 6.7267 0 00.2267-.1863l1.1531.2713a.3472.3472 0 00.4171-.416l-.2713-1.1542c.0628-.0749.1255-.1508.1863-.2278l1.1845.0415a.3442.3442 0 00.328-.49l-.49-1.076c.0475-.0872.0951-.1742.1407-.2623l1.1662-.1893a.3483.3483 0 00.2258-.5447l-.6904-.9587.087-.2855 1.1066-.414a.3462.3462 0 00.1154-.5781l-.8656-.8079c.0101-.0972.0202-.1954.0283-.2936l1.0073-.6236a.3442.3442 0 000-.5892zm-6.7413 8.3551a.7138.7138 0 01.2986-1.396.714.714 0 11-.2997 1.396zm-.3422-2.3142a.649.649 0 00-.7715.5l-.3573 1.6685c-1.1035.501-2.3285.7795-3.6193.7795a8.7368 8.7368 0 01-3.6951-.814l-.3574-1.6684a.648.648 0 00-.7714-.499l-1.473.3158a8.7216 8.7216 0 01-.7613-.898h7.1676c.081 0 .1356-.0141.1356-.088v-2.536c0-.074-.0536-.0881-.1356-.0881h-2.0966v-1.6077h2.2677c.2065 0 1.1065.0587 1.394 1.2088.0901.3533.2875 1.5044.4232 1.8729.1346.413.6833 1.2381 1.2685 1.2381h3.5716a.7492.7492 0 00.1296-.0131 8.7874 8.7874 0 01-.8119.9526zM6.8369 20.024a.714.714 0 11-.2997-1.396.714.714 0 01.2997 1.396zM4.1177 8.9972a.7137.7137 0 11-1.304.5791.7137.7137 0 011.304-.579zm-.8352 1.9813l1.5347-.6824a.65.65 0 00.33-.8585l-.3158-.7147h1.2432v5.6025H3.5669a8.7753 8.7753 0 01-.2834-3.348zm6.7343-.5437V8.7836h2.9601c.153 0 1.0792.1772 1.0792.8697 0 .575-.7107.7815-1.2948.7815zm10.7574 1.4862c0 .2187-.008.4363-.0243.651h-.9c-.09 0-.1265.0586-.1265.1477v.413c0 .973-.5487 1.1846-1.0296 1.2382-.4576.0517-.9648-.1913-1.0275-.4717-.2704-1.5186-.7198-1.8436-1.4305-2.4034.8817-.5599 1.799-1.386 1.799-2.4915 0-1.1936-.819-1.9458-1.3769-2.3153-.7825-.5163-1.6491-.6195-1.883-.6195H5.4682a8.7651 8.7651 0 014.907-2.7699l1.0974 1.151a.648.648 0 00.9182.0213l1.227-1.1743a8.7753 8.7753 0 016.0044 4.2762l-.8403 1.8982a.652.652 0 00.33.8585l1.6178.7188c.0283.2875.0425.577.0425.8717zm-9.3006-9.5993a.7128.7128 0 11.984 1.0316.7137.7137 0 01-.984-1.0316zm8.3389 6.71a.7107.7107 0 01.9395-.3625.7137.7137 0 11-.9405.3635z"], + java: ["0 0 128 128", "M47.617 98.12c-19.192 5.362 11.677 16.439 36.115 5.969-4.003-1.556-6.874-3.351-6.874-3.351-10.897 2.06-15.952 2.222-25.844 1.092-8.164-.935-3.397-3.71-3.397-3.71zm33.189-10.46c-14.444 2.779-22.787 2.69-33.354 1.6-8.171-.845-2.822-4.805-2.822-4.805-21.137 7.016 11.767 14.977 41.309 6.336-3.14-1.106-5.133-3.131-5.133-3.131zm11.319-60.575c.001 0-42.731 10.669-22.323 34.187 6.024 6.935-1.58 13.17-1.58 13.17s15.289-7.891 8.269-17.777c-6.559-9.215-11.587-13.793 15.634-29.58zm9.998 81.144s3.529 2.91-3.888 5.159c-14.102 4.272-58.706 5.56-71.095.171-4.45-1.938 3.899-4.625 6.526-5.192 2.739-.593 4.303-.485 4.303-.485-4.952-3.487-32.013 6.85-13.742 9.815 49.821 8.076 90.817-3.637 77.896-9.468zM85 77.896c2.395-1.634 5.703-3.053 5.703-3.053s-9.424 1.685-18.813 2.474c-11.494.964-23.823 1.154-30.012.326-14.652-1.959 8.033-7.348 8.033-7.348s-8.812-.596-19.644 4.644C17.455 81.134 61.958 83.958 85 77.896zm5.609 15.145c-.108.29-.468.616-.468.616 31.273-8.221 19.775-28.979 4.822-23.725-1.312.464-2 1.543-2 1.543s.829-.334 2.678-.72c7.559-1.575 18.389 10.119-5.032 22.286zM64.181 70.069c-4.614-10.429-20.26-19.553.007-35.559C89.459 14.563 76.492 1.587 76.492 1.587c5.23 20.608-18.451 26.833-26.999 39.667-5.821 8.745 2.857 18.142 14.688 28.815zm27.274 51.748c-19.187 3.612-42.854 3.191-56.887.874 0 0 2.874 2.38 17.646 3.331 22.476 1.437 57-.8 57.816-11.436.001 0-1.57 4.032-18.575 7.231z"], + bash: ["0 0 24 24", "M21.038,4.9l-7.577-4.498C13.009,0.134,12.505,0,12,0c-0.505,0-1.009,0.134-1.462,0.403L2.961,4.9 C2.057,5.437,1.5,6.429,1.5,7.503v8.995c0,1.073,0.557,2.066,1.462,2.603l7.577,4.497C10.991,23.866,11.495,24,12,24 c0.505,0,1.009-0.134,1.461-0.402l7.577-4.497c0.904-0.537,1.462-1.529,1.462-2.603V7.503C22.5,6.429,21.943,5.437,21.038,4.9z M15.17,18.946l0.013,0.646c0.001,0.078-0.05,0.167-0.111,0.198l-0.383,0.22c-0.061,0.031-0.111-0.007-0.112-0.085L14.57,19.29 c-0.328,0.136-0.66,0.169-0.872,0.084c-0.04-0.016-0.057-0.075-0.041-0.142l0.139-0.584c0.011-0.046,0.036-0.092,0.069-0.121 c0.012-0.011,0.024-0.02,0.036-0.026c0.022-0.011,0.043-0.014,0.062-0.006c0.229,0.077,0.521,0.041,0.802-0.101 c0.357-0.181,0.596-0.545,0.592-0.907c-0.003-0.328-0.181-0.465-0.613-0.468c-0.55,0.001-1.064-0.107-1.072-0.917 c-0.007-0.667,0.34-1.361,0.889-1.8l-0.007-0.652c-0.001-0.08,0.048-0.168,0.111-0.2l0.37-0.236 c0.061-0.031,0.111,0.007,0.112,0.087l0.006,0.653c0.273-0.109,0.511-0.138,0.726-0.088c0.047,0.012,0.067,0.076,0.048,0.151 l-0.144,0.578c-0.011,0.044-0.036,0.088-0.065,0.116c-0.012,0.012-0.025,0.021-0.038,0.028c-0.019,0.01-0.038,0.013-0.057,0.009 c-0.098-0.022-0.332-0.073-0.699,0.113c-0.385,0.195-0.52,0.53-0.517,0.778c0.003,0.297,0.155,0.387,0.681,0.396 c0.7,0.012,1.003,0.318,1.01,1.023C16.105,17.747,15.736,18.491,15.17,18.946z M19.143,17.859c0,0.06-0.008,0.116-0.058,0.145 l-1.916,1.164c-0.05,0.029-0.09,0.004-0.09-0.056v-0.494c0-0.06,0.037-0.093,0.087-0.122l1.887-1.129 c0.05-0.029,0.09-0.004,0.09,0.056V17.859z M20.459,6.797l-7.168,4.427c-0.894,0.523-1.553,1.109-1.553,2.187v8.833 c0,0.645,0.26,1.063,0.66,1.184c-0.131,0.023-0.264,0.039-0.398,0.039c-0.42,0-0.833-0.114-1.197-0.33L3.226,18.64 c-0.741-0.44-1.201-1.261-1.201-2.142V7.503c0-0.881,0.46-1.702,1.201-2.142l7.577-4.498c0.363-0.216,0.777-0.33,1.197-0.33 c0.419,0,0.833,0.114,1.197,0.33l7.577,4.498c0.624,0.371,1.046,1.013,1.164,1.732C21.686,6.557,21.12,6.411,20.459,6.797z"], + }; + var LANG_ALIASES = { py: "python", python3: "python", rs: "rust", sh: "bash", shell: "bash", console: "bash", zsh: "bash" }; + + // Code blocks: wrap .highlight in a window with a language bar + copy button. + article.querySelectorAll("div.highlight").forEach(function (hl) { + if (hl.closest(".ld-code")) return; + var code = hl.querySelector("code"); + var m = (hl.className + " " + (code ? code.className : "")).match(/language-([\w+-]+)/); + var lang = m ? m[1] : "text"; + var wrap = document.createElement("div"); + wrap.className = "ld-code"; + var bar = document.createElement("div"); + bar.className = "ld-code__bar"; + bar.innerHTML = ""; + var span = bar.querySelector("span"); + var icon = LANG_ICONS[LANG_ALIASES[lang] || lang]; + if (icon) span.innerHTML = ''; + span.appendChild(document.createTextNode(lang)); + hl.parentNode.insertBefore(wrap, hl); + wrap.appendChild(bar); + wrap.appendChild(hl); + }); + + document.addEventListener("click", function (e) { + var copy = e.target.closest(".ld-copy"); + if (!copy) return; + var pre = copy.closest(".ld-code").querySelector("pre"); + if (navigator.clipboard && pre) navigator.clipboard.writeText(pre.textContent); + copy.textContent = "Copied"; + setTimeout(function () { copy.textContent = "Copy"; }, 1400); + }); + + // Tables: wrap for horizontal overflow. + article.querySelectorAll("table").forEach(function (t) { + if (t.closest(".ld-tablewrap")) return; + var wrap = document.createElement("div"); + wrap.className = "ld-tablewrap"; + t.parentNode.insertBefore(wrap, t); + wrap.appendChild(t); + }); + + // Mermaid: render fenced diagrams on demand. + var mermaidNodes = article.querySelectorAll("pre.mermaid, div.mermaid"); + if (mermaidNodes.length) { + import("https://cdn.jsdelivr.net/npm/mermaid@11/dist/mermaid.esm.min.mjs").then(function (mod) { + var mermaid = mod.default; + mermaidNodes.forEach(function (node) { + if (node.tagName === "PRE") { + var div = document.createElement("div"); + div.className = "mermaid"; + div.textContent = node.textContent; + node.replaceWith(div); + } + }); + mermaid.initialize({ startOnLoad: false, securityLevel: "loose", theme: "neutral" }); + mermaid.run({ querySelector: ".ld-article div.mermaid" }); + }).catch(function () { /* offline: leave the diagram source visible */ }); + } + } + + /* ---------- TOC scroll-spy ---------- */ + var tocLinks = Array.prototype.slice.call(document.querySelectorAll(".ld-toc a")); + if (tocLinks.length && article) { + var targets = tocLinks.map(function (a) { + var id = decodeURIComponent((a.getAttribute("href") || "").replace(/^#/, "")); + return document.getElementById(id); + }); + var update = function () { + var active = 0; + for (var i = 0; i < targets.length; i++) { + if (targets[i] && targets[i].getBoundingClientRect().top <= 90) active = i; + } + tocLinks.forEach(function (a, i) { a.classList.toggle("active", i === active); }); + }; + window.addEventListener("scroll", update, { passive: true }); + update(); + } + + /* ---------- search overlay ---------- */ + var overlay = document.getElementById("search-overlay"); + var input = document.getElementById("search-input"); + var results = document.getElementById("search-results"); + var indexPromise = null; + + function loadIndex() { + if (!indexPromise) { + indexPromise = fetch(BASE + "/search/search_index.json") + .then(function (r) { if (!r.ok) throw new Error(r.status); return r.json(); }) + .then(function (d) { + return d.docs.map(function (doc) { + return { + location: doc.location, + title: doc.title || "", + text: (doc.text || "").replace(/\s+/g, " "), + }; + }); + }); + } + return indexPromise; + } + + function esc(s) { + return s.replace(/&/g, "&").replace(//g, ">"); + } + + function highlight(text, terms) { + var out = esc(text); + terms.forEach(function (t) { + out = out.replace(new RegExp("(" + t.replace(/[.*+?^${}()|[\]\\]/g, "\\$&") + ")", "ig"), "$1"); + }); + return out; + } + + function search(docs, query) { + var terms = query.toLowerCase().split(/\s+/).filter(Boolean); + if (!terms.length) return []; + var scored = []; + docs.forEach(function (doc) { + var title = doc.title.toLowerCase(); + var text = doc.text.toLowerCase(); + var score = 0; + for (var i = 0; i < terms.length; i++) { + var t = terms[i]; + var inTitle = title.indexOf(t) !== -1; + var inText = text.indexOf(t) !== -1; + if (!inTitle && !inText) { score = 0; break; } + score += (inTitle ? 10 : 0) + (inText ? 1 : 0); + } + // Prefer page-level entries slightly over deep anchors. + if (score > 0) scored.push({ doc: doc, score: score + (doc.location.indexOf("#") === -1 ? 2 : 0) }); + }); + scored.sort(function (a, b) { return b.score - a.score; }); + return scored.slice(0, 12).map(function (s) { return s.doc; }); + } + + function snippet(text, terms) { + var lower = text.toLowerCase(); + var pos = -1; + for (var i = 0; i < terms.length; i++) { + pos = lower.indexOf(terms[i]); + if (pos !== -1) break; + } + if (pos === -1) pos = 0; + var start = Math.max(0, pos - 60); + var s = (start > 0 ? "…" : "") + text.slice(start, start + 160) + (start + 160 < text.length ? "…" : ""); + return s; + } + + function render(docs, query) { + var terms = query.toLowerCase().split(/\s+/).filter(Boolean); + if (!docs.length) { + results.innerHTML = "
No results
"; + return; + } + results.innerHTML = docs.map(function (doc) { + var crumb = doc.location.split("#")[0].replace(/\/$/, "").replace(/\//g, " / ") || "home"; + return "" + + "
" + esc(crumb) + "
" + + "
" + highlight(doc.title, terms) + "
" + + "
" + highlight(snippet(doc.text, terms), terms) + "
" + + ""; + }).join(""); + } + + function openSearch() { + overlay.hidden = false; + input.value = ""; + results.innerHTML = ""; + input.focus(); + loadIndex(); + } + function closeSearch() { overlay.hidden = true; } + + if (overlay && input && results) { + var openBtn = document.getElementById("search-open"); + if (openBtn) openBtn.addEventListener("click", openSearch); + overlay.addEventListener("click", function (e) { + if (e.target.closest("[data-search-close]")) closeSearch(); + }); + document.addEventListener("keydown", function (e) { + if (e.key === "Escape" && !overlay.hidden) { closeSearch(); return; } + var typing = /^(INPUT|TEXTAREA|SELECT)$/.test((document.activeElement || {}).tagName || ""); + if ((e.key === "/" && !typing) || ((e.metaKey || e.ctrlKey) && e.key.toLowerCase() === "k")) { + e.preventDefault(); + if (overlay.hidden) openSearch(); else closeSearch(); + } + }); + var pending = 0; + input.addEventListener("input", function () { + var q = input.value.trim(); + var seq = ++pending; + if (q.length < 2) { results.innerHTML = ""; return; } + loadIndex().then(function (docs) { + if (seq !== pending) return; + render(search(docs, q), q); + }).catch(function () { + results.innerHTML = "
Search index unavailable
"; + }); + }); + } +})(); diff --git a/docs/theme/assets/tokens.css b/docs/theme/assets/tokens.css new file mode 100644 index 00000000000..29c08f2711e --- /dev/null +++ b/docs/theme/assets/tokens.css @@ -0,0 +1,142 @@ +/* Lance Design System — tokens (Swiss editorial: white, ink, brand purple #625EFF). + Merged from the design project's ds/tokens set; dark theme mirrors the same + structure with paper rules on ink. */ +@import url("https://fonts.googleapis.com/css2?family=Space+Grotesk:wght@400;500;600;700&family=IBM+Plex+Sans:wght@400;500;600&family=Roboto+Mono:wght@400;500;600&display=swap"); + +:root { + color-scheme: light; + + /* ---- Typography ---- */ + --font-display: "Space Grotesk", "Segoe UI", sans-serif; + --font-body: "IBM Plex Sans", "Segoe UI", sans-serif; + --font-mono: "Roboto Mono", "SF Mono", monospace; + --text-base: 16px; + --leading-body: 1.6; + --tracking-display: -0.045em; + --tracking-caps: 0.14em; + + /* ---- Ink scale (dark values — code surfaces & tooltips only) ---- */ + --ink-950: #0E0C14; + --ink-900: #14111D; + --ink-850: #1A1626; + --ink-800: #221D30; + --ink-700: #2C2640; + --ink-600: #3A3352; + + /* ---- Foreground scale (on white) ---- */ + --fg-1: #0E0C14; + --fg-2: #4A4459; + --fg-3: #8A8499; + --fg-inverse: #ffffff; + + /* ---- Brand purple ramp (official #625EFF) ---- */ + --beam-300: #B3B1FF; + --beam-400: #625EFF; + --beam-500: #625EFF; + --beam-600: #4B47E0; + --beam-700: #3936B4; + --beam-glow: rgba(98, 94, 255, 0.25); + --beam-dim: rgba(98, 94, 255, 0.09); + + /* ---- Semantic status ---- */ + --ok-500: #0E8F63; + --warn-500: #B26205; + --danger-500: #D92D20; + + /* ---- Lines: ink rules carry the structure ---- */ + --line-1: #14111D; + --line-2: rgba(20, 17, 29, 0.16); + + /* ---- Surfaces ---- */ + --surface-page: #FFFFFF; + --surface-card: #FFFFFF; + --surface-overlay: #F5F4F8; + --surface-header: rgba(255, 255, 255, 0.88); + --surface-code: #F5F4F8; /* light code surface */ + --surface-inline-code: rgba(20, 17, 29, 0.05); + + --text-body: var(--fg-1); + --text-secondary: var(--fg-2); + --text-muted: var(--fg-3); + + /* ---- Syntax highlighting (Pygments), light theme ---- */ + --code-fg: #2A2635; + --code-kw: #7C3AED; /* keywords, builtins */ + --code-str: #0B62C4; /* strings */ + --code-com: #6E6A80; /* comments */ + --code-num: #9A6700; /* numbers */ + --code-fn: #0F766E; /* functions, decorators */ + --code-name: #2A2635; /* class/module names, headings */ + --code-tag: #BE123C; /* HTML tags, attributes */ + --code-punct: #57534E; /* operators, punctuation, output */ + --code-prompt: #8A8499; /* REPL prompts */ + --code-hll: rgba(98, 94, 255, 0.12); /* highlighted line */ + + /* ---- Layout ---- */ + --container-max: 1200px; + --radius-box: 6px; /* content boxes: code, admonitions, tabs */ + --radius-chip: 4px; /* inline code chips */ + + /* ---- Motion — fast, precise, no bounce ---- */ + --ease-out: cubic-bezier(0.2, 0.8, 0.2, 1); + --dur-fast: 120ms; +} + +/* ---- Dark theme: same structure, paper rules on ink ---- */ +:root[data-theme="dark"] { + color-scheme: dark; + + --fg-1: #F2F0FA; + --fg-2: #B7B1C6; + --fg-3: #837D96; + --fg-inverse: #0E0C14; + + /* text accents need a lighter purple to stay readable on ink */ + --beam-600: #8E8BFF; + --beam-700: #A9A6FF; + + --ok-500: #34C08E; + --warn-500: #E8A13C; + --danger-500: #FF6B5E; + + --line-1: rgba(237, 235, 245, 0.7); + --line-2: rgba(237, 235, 245, 0.15); + + --surface-page: #0E0C14; + --surface-card: #14111D; + --surface-overlay: #1A1626; + --surface-header: rgba(14, 12, 20, 0.85); + --surface-code: #14111D; + --surface-inline-code: rgba(237, 235, 245, 0.08); + + /* ---- Syntax highlighting (Pygments), dark theme (on ink) ---- */ + --code-fg: #EDEBF5; + --code-kw: #B8A8FF; + --code-str: #7CC0FF; + --code-com: #6F6A85; + --code-num: #F2CE6B; + --code-fn: #8FE8CE; + --code-name: #EDEBF5; + --code-tag: #FF95A8; + --code-punct: #B7B1C6; + --code-prompt: #6F6A85; + --code-hll: rgba(98, 94, 255, 0.22); +} + +/* ---- Minimal base ---- */ +* { box-sizing: border-box; } + +body { + margin: 0; + background: var(--surface-page); + color: var(--text-body); + font-family: var(--font-body); + font-size: var(--text-base); + line-height: var(--leading-body); + -webkit-font-smoothing: antialiased; +} + +/* No `color` here: forcing ink text breaks selection inside dark code blocks. */ +::selection { background: rgba(98, 94, 255, 0.3); } + +code, kbd, pre { font-family: var(--font-mono); } diff --git a/docs/theme/base.html b/docs/theme/base.html new file mode 100644 index 00000000000..d90b2ae02c4 --- /dev/null +++ b/docs/theme/base.html @@ -0,0 +1,118 @@ +{#- Lance Docs custom theme — shared skeleton: head, header, footer, scripts. -#} +{%- macro first_url(item) -%} + {%- if item.is_link or item.is_page -%} + {{- item.url -}} + {%- elif item.children -%} + {{- first_url(item.children[0]) -}} + {%- endif -%} +{%- endmacro -%} + + + + + +{% if page and page.title and not page.is_homepage %} +{{ page.title }} — {{ config.site_name }} +{% else %} +{{ config.site_name }} — The open lakehouse format for multimodal AI +{% endif %} +{% if page and page.meta and page.meta.description %} + +{% elif config.site_description %} + +{% endif %} +{% if page and page.canonical_url %}{% endif %} + + + + + + +
+
+ + {{ config.site_name }} + + +
+ + + + GitHub + + + + + + + Get started +
+
+ + {% block container %}{% endblock %} + + +
+ + + + + + + + diff --git a/docs/theme/home.html b/docs/theme/home.html new file mode 100644 index 00000000000..1b361168d46 --- /dev/null +++ b/docs/theme/home.html @@ -0,0 +1,147 @@ +{% extends "base.html" %} + +{% block container %} +
+
+
Lance format · Open source · Apache-2.0 · VLDB '25 paper ↗
+

The open lakehouse format for multimodal AI.

+

A file format, table format, and catalog spec for building a complete lakehouse on object storage — powering vector and full-text search, feature engineering, and model training with the fast random access and scans that AI workloads need.

+
+ Get started + Read the spec + Train an LLM + Join Discord → +
+
+ +
+
+
+
100×
+
Faster random access than Parquet
+
+
+
1 line
+
To convert Parquet to Lance
+
+ +
VLDB '25
+
Peer-reviewed research paper →
+ +
+
+ +
+

What is Lance?

+

Lance is a modern, open source lakehouse format for multimodal AI. It brings high-performance vector and full-text search, feature engineering, and model training to the lakehouse, powered by fast random access and scans — while keeping SQL analytics, ACID transactions, time travel, and integrations with open engines (Apache Spark, Ray, PyTorch, Trino, DuckDB) and open catalogs (Apache Polaris, Unity Catalog, Apache Gravitino, Hive Metastore).

+

Learn more in the research paper published at VLDB 2025.

+
+ +
+
+
01
+
+

Expressive hybrid search

+

Combine vector similarity, full-text search (BM25), and SQL analytics on the same dataset. All query types are accelerated by secondary indexes that are part of the Lance specification.

+ Learn more → +
+
+ + 
import lance
+
+ds = lance.dataset("s3://my-bucket/docs")
+
+# Full text search
+ds.to_table(full_text_query="machine learning")
+
+# Hybrid search
+ds.to_table(
+    nearest={
+        "column": "embedding", "q": query_vec, "k": 10
+    },
+    filter="year > 2020",
+)
+
+
+ +
+
02
+
+

Lightning-fast random access

+

100x faster random access than Parquet or Iceberg. An optimized file format plus row addressing and secondary indexes let you fetch individual records across files instantly — for ML serving, sampling, and interactive apps.

+ Learn more → +
+
+ + 
import lance
+
+ds = lance.dataset("s3://my-bucket/embeddings.lance")
+
+# Access the 2nd & 51st rows
+ds.take([2, 51], columns=["id", "vec_gemma3"])
+
+# Take 1000 random samples
+ds.sample(1000, columns=["id", "vec_llama"])
+
+
+ +
+
03
+
+

Native multimodal data

+

Store images, videos, audio, text, and embeddings alongside tabular data in one format. Blob encoding handles large binary objects with lazy loading; optimized vector storage accelerates similarity search.

+ Learn more → +
+
+ + 
import lance
+import av
+
+ds = lance.dataset("s3://my-bucket/videos.lance")
+
+# Get blobs from the 2nd and 51st rows
+blobs = ds.take_blobs("video", ids=[2, 51])
+
+for blob in blobs:
+    with av.open(blob) as container:
+        stream = container.streams.video[0]
+        container.seek(start_time=500, stream=stream)
+
+
+ +
+
04
+
+

Data evolution > schema evolution

+

Backfilling column values normally forces a full table rewrite. Lance supports efficient schema evolution with backfill — adding a column with data is just writing new Lance files to the table.

+ Learn more → +
+
+ + 
import lance
+
+dataset = lance.dataset("my_data.lance")
+
+@lance.batch_udf()
+def add_embeddings(batch):
+    vectors = model.encode(batch["text"])
+    return {"embedding": vectors}
+
+dataset.add_columns(add_embeddings)
+
+
+ +
+
05
+
+

Rich ecosystem integrations

+

Works with Pandas, Polars, Ray, and PyTorch for processing and ML. Connects to Apache DataFusion, DuckDB, Apache Spark, Trino, and Apache Flink for SQL analytics and distributed processing.

+ View integrations → +
+
+ Lance ecosystem integrations +
+
+
+
+{% endblock %} diff --git a/docs/theme/main.html b/docs/theme/main.html new file mode 100644 index 00000000000..f9a4a7e6bd9 --- /dev/null +++ b/docs/theme/main.html @@ -0,0 +1,92 @@ +{% extends "base.html" %} + +{#- Render one nav section as sidenav groups: its direct pages/links under the + section's own label, then each child section as a further group. -#} +{%- macro sidenav_section(sec, label) -%} + {%- set direct = sec.children | selectattr("is_section", "ne", true) | list -%} + {%- if direct -%} +
+
{{ label }}
+ {%- for child in direct %} + {%- if child.is_link %} + {{ child.title }} ↗ + {%- else %} + {{ child.title }} + {%- endif %} + {%- endfor %} +
+ {%- endif -%} + {%- for child in sec.children if child.is_section -%} + {{ sidenav_section(child, child.title) }} + {%- endfor -%} +{%- endmacro -%} + +{% block container %} + {%- set active_section = namespace(item=none) -%} + {%- for item in nav %}{% if item.active %}{% set active_section.item = item %}{% endif %}{% endfor %} + +
+ + + +
+
+ Docs + {%- for crumb in page.ancestors | reverse %} + /{{ crumb.title }} + {%- endfor %} + /{{ page.title }} +
+ +
+ {{ page.content }} +
+ +
+ {%- if page.previous_page %} + +
← Previous
+
{{ page.previous_page.title }}
+ + {%- else %}{% endif %} + {%- if page.next_page %} + + {%- endif %} +
+
+ + +
+{% endblock %} diff --git a/java/lance-jni/Cargo.lock b/java/lance-jni/Cargo.lock index e392fd82c76..be2695f28ef 100644 --- a/java/lance-jni/Cargo.lock +++ b/java/lance-jni/Cargo.lock @@ -109,9 +109,9 @@ dependencies = [ [[package]] name = "anyhow" -version = "1.0.102" +version = "1.0.104" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f202df86484c868dbad7eaa557ef785d5c66295e41b460ef922eca0723b842c" +checksum = "330a5ed07fa54e4702c9d6c4174f74427fc0ef6e214bbd677ae50a5099946470" [[package]] name = "approx" @@ -145,9 +145,9 @@ checksum = "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50" [[package]] name = "arrow" -version = "58.3.0" +version = "58.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "378530e55cd479eda3c14eb345310799717e6f76d0c332041e8487022166b471" +checksum = "6cfdd0833e32a9874d2b55089333ad310c0be208aafa277385ce2461dec90be3" dependencies = [ "arrow-arith", "arrow-array", @@ -166,9 +166,9 @@ dependencies = [ [[package]] name = "arrow-arith" -version = "58.3.0" +version = "58.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a0ab212d2c1886e802f51c5212d78ebbcbb0bec980fff9dadc1eb8d45cd0b738" +checksum = "0a41203398f0eaa6f7ec8e62c0da742a21abf282c148fc157f6c35c90e29981a" dependencies = [ "arrow-array", "arrow-buffer", @@ -180,9 +180,9 @@ dependencies = [ [[package]] name = "arrow-array" -version = "58.3.0" +version = "58.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cfd33d3e92f207444098c75b42de99d329562be0cf686b307b097cc52b4e999e" +checksum = "ae33dad492b7df00a217563a7b0ef2874df68a0deea1b1a3acf628152f7f7a69" dependencies = [ "ahash", "arrow-buffer", @@ -199,9 +199,9 @@ dependencies = [ [[package]] name = "arrow-buffer" -version = "58.3.0" +version = "58.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c6cd424c2693bcdbc150d843dc9d4d137dd2de4782ce6df491ad11a3a0416c0" +checksum = "b9552f96391c005e6ab449fa941420935e7e062489b12b8b1b08879b2163f5b5" dependencies = [ "bytes", "half", @@ -211,9 +211,9 @@ dependencies = [ [[package]] name = "arrow-cast" -version = "58.3.0" +version = "58.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c5aefb56a2c02e9e2b30746241058b85f8983f0fcff2ba0c6d09006e1cded7f" +checksum = "3a8a327c9649f30d8406995f27642b68df354713cca3baaaf100f076f18d5f34" dependencies = [ "arrow-array", "arrow-buffer", @@ -222,7 +222,7 @@ dependencies = [ "arrow-schema", "arrow-select", "atoi", - "base64", + "base64 0.22.1", "chrono", "comfy-table", "half", @@ -233,9 +233,9 @@ dependencies = [ [[package]] name = "arrow-csv" -version = "58.3.0" +version = "58.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e94e8cf7e517657a52b91ea1263acf38c4ca62a84655d72458a3359b12ab97de" +checksum = "af0dd6d90d1955e9f9a014c1e563ee8aeffc21909085d25623e1da44d96eca26" dependencies = [ "arrow-array", "arrow-cast", @@ -248,9 +248,9 @@ dependencies = [ [[package]] name = "arrow-data" -version = "58.3.0" +version = "58.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3c88210023a2bfee1896af366309a3028fc3bcbd6515fa29a7990ee1baa08ee0" +checksum = "2b24852db04738907e06c04ea61e42fe7fda962a34513022dc0d0e754fb7976b" dependencies = [ "arrow-buffer", "arrow-schema", @@ -261,9 +261,9 @@ dependencies = [ [[package]] name = "arrow-ipc" -version = "58.3.0" +version = "58.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "238438f0834483703d88896db6fe5a7138b2230debc31b34c0336c2996e3c64f" +checksum = "29a908a11fcfb3fb2f6730f4ac15e367bc644e419155e96238f68cf3adde572b" dependencies = [ "arrow-array", "arrow-buffer", @@ -277,9 +277,9 @@ dependencies = [ [[package]] name = "arrow-json" -version = "58.3.0" +version = "58.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "205ca2119e6d679d5c133c6f30e68f027738d95ed948cf77677ea69c7800036b" +checksum = "b8a96aed3931c076adee39ec2a40d8219fc7f09e79bcdaca1df16272993e1e14" dependencies = [ "arrow-array", "arrow-buffer", @@ -302,9 +302,9 @@ dependencies = [ [[package]] name = "arrow-ord" -version = "58.3.0" +version = "58.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1bffd8fd2579286a5d63bac898159873e5094a79009940bcb42bbfce4f19f1d0" +checksum = "63a083ec750f5c043f02946b4baf05fcdbb55f4560a3277055caca5cc99f3eb0" dependencies = [ "arrow-array", "arrow-buffer", @@ -315,9 +315,9 @@ dependencies = [ [[package]] name = "arrow-row" -version = "58.3.0" +version = "58.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bab5994731204603c73ba69267616c50f80780774c6bb0476f1f830625115e0c" +checksum = "514ba0ef0d4c5896202dae736251ce415abb43a950bed570fb7981b8716c0e4c" dependencies = [ "arrow-array", "arrow-buffer", @@ -328,9 +328,9 @@ dependencies = [ [[package]] name = "arrow-schema" -version = "58.3.0" +version = "58.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f633dbfdf39c039ada1bf9e34c694816eb71fbb7dc78f613993b7245e078a1ed" +checksum = "21ca356ad6425cecb6eb7b28e4f659f1ee7880fbb1a16127de7dd62901efee9e" dependencies = [ "bitflags", "serde_core", @@ -339,9 +339,9 @@ dependencies = [ [[package]] name = "arrow-select" -version = "58.3.0" +version = "58.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8cd065c54172ac787cf3f2f8d4107e0d3fdc26edba76fdf4f4cc170258942222" +checksum = "c58da39eb3d8350ad4a549e5c2bc49284dac554016c69829310350f1731b0aad" dependencies = [ "ahash", "arrow-array", @@ -353,9 +353,9 @@ dependencies = [ [[package]] name = "arrow-string" -version = "58.3.0" +version = "58.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "29dd7cda3ab9692f43a2e4acc444d760cc17b12bb6d8232ddf64e9bab7c06b42" +checksum = "b6789b388467525e3271326b6b4915666ecfdf5142aef09779445c954b67543c" dependencies = [ "arrow-array", "arrow-buffer", @@ -411,18 +411,18 @@ checksum = "3b43422f69d8ff38f95f1b2bb76517c91589a924d1559a0e935d7c8ce0274c11" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.118", ] [[package]] name = "async-trait" -version = "0.1.89" +version = "0.1.91" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9035ad2d096bed7955a320ee7e2230574d28fd3c3a0f186cbea1ff3c7eed5dbb" +checksum = "ae36dc4177970ef04fde5178d3e2429882def40e57a451f919c098f72baa6cec" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 3.0.2", ] [[package]] @@ -883,6 +883,12 @@ version = "0.22.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" +[[package]] +name = "base64" +version = "0.23.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b25655df2c3cdd83c5e5b293b88acd880332b2ddadd7c30ac43144fdc0033da9" + [[package]] name = "base64-simd" version = "0.8.0" @@ -1008,9 +1014,23 @@ checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649" [[package]] name = "bytemuck" -version = "1.25.0" +version = "1.25.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c8efb64bd706a16a1bdde310ae86b351e4d21550d98d056f22f8a7f7a2183fec" +checksum = "95832e849adfb21180ccb6826a99da14e5d266ae5c2e668e1602cf234f153797" +dependencies = [ + "bytemuck_derive", +] + +[[package]] +name = "bytemuck_derive" +version = "1.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9abbd1bc6865053c427f7198e6af43bfdedc55ab791faed4fbd361d789575ff" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.118", +] [[package]] name = "byteorder" @@ -1020,9 +1040,9 @@ checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" [[package]] name = "bytes" -version = "1.12.0" +version = "1.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ae3f5d315924270530207e2a68396c3cc547f6dca3fbdca317cfb1a51edb593" +checksum = "fc652a48c352aef3ea3aed32080501cf3ef6ed5da78602a020c991775b0aff04" [[package]] name = "bytes-utils" @@ -1045,9 +1065,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.2.65" +version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e228eec9be7c17ccb640b59b36a5cd805ea2a564a4c5e162c2f659fea30d3b96" +checksum = "c89588d05638b5b4594a3348a2d6c20277e43a7f5c5202b05cc56888475a47b8" dependencies = [ "find-msvc-tools", "jobserver", @@ -1126,9 +1146,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.6.1" +version = "4.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ddb117e43bbf7dacf0a4190fef4d345b9bad68dfc649cb349e7d17d28428e51" +checksum = "d91e0c145792ef73a6ad36d27c75ac09f1832222a3c209689d90f534685ee5b7" dependencies = [ "clap_builder", "clap_derive", @@ -1136,9 +1156,9 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.6.0" +version = "4.6.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "714a53001bf66416adb0e2ef5ac857140e7dc3a0c48fb28b2f10762fc4b5069f" +checksum = "f09628afdcc538b57f3c6341e9c8e9970f18e4a481690a64974d7023bd33548b" dependencies = [ "anstream", "anstyle", @@ -1148,14 +1168,14 @@ dependencies = [ [[package]] name = "clap_derive" -version = "4.6.1" +version = "4.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f2ce8604710f6733aa641a2b3731eaa1e8b3d9973d5e3565da11800813f997a9" +checksum = "d012d2b9d65aca7f18f4d9878a045bc17899bba951561ba5ec3c2ba1eed9a061" dependencies = [ "heck", "proc-macro2", "quote", - "syn", + "syn 3.0.2", ] [[package]] @@ -1347,12 +1367,13 @@ dependencies = [ ] [[package]] -name = "crc32c" -version = "0.6.8" +name = "crc-fast" +version = "1.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3a47af21622d091a8f0fb295b88bc886ac74efcc613efc19f5d0b21de5c89e47" +checksum = "e75b2483e97a5a7da73ac68a05b629f9c53cff58d8ed1c77866079e18b00dba5" dependencies = [ - "rustc_version", + "digest 0.10.7", + "spin 0.10.1", ] [[package]] @@ -1385,18 +1406,18 @@ dependencies = [ [[package]] name = "crossbeam-epoch" -version = "0.9.18" +version = "0.9.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e" +checksum = "2d6914041f254d6e9176c01941b21115dcfb7089e55135a35411081bd106ef3f" dependencies = [ "crossbeam-utils", ] [[package]] name = "crossbeam-queue" -version = "0.3.12" +version = "0.3.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0f58bbc28f91df819d0aa2a2c00cd19754769c2fad90579b3592b1c9ba7a3115" +checksum = "803d13fb3b09d88be9f4dbc29062c66b19bf7170867ceb746d2a8689bf6c7a26" dependencies = [ "crossbeam-utils", ] @@ -1518,7 +1539,7 @@ dependencies = [ "proc-macro2", "quote", "strsim", - "syn", + "syn 2.0.118", ] [[package]] @@ -1529,7 +1550,7 @@ checksum = "ac3984ec7bd6cfa798e62b4a642426a5be0e68f9401cfc2a01e3fa9ea2fcdb8d" dependencies = [ "darling_core", "quote", - "syn", + "syn 2.0.118", ] [[package]] @@ -1548,14 +1569,13 @@ dependencies = [ [[package]] name = "datafusion" -version = "53.1.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "93db0e623840612f7f2cd757f7e8a8922064192363732c88692e0870016e141b" +checksum = "754ef4e8f073922a26f5b23133b9db4829342362b09be0bc94309cf261c2f098" dependencies = [ "arrow", "arrow-schema", "async-trait", - "bytes", "chrono", "datafusion-catalog", "datafusion-catalog-listing", @@ -1582,12 +1602,11 @@ dependencies = [ "datafusion-session", "datafusion-sql", "futures", + "indexmap 2.14.0", "itertools 0.14.0", "log", "object_store", "parking_lot", - "rand 0.9.4", - "regex", "sqlparser", "tempfile", "tokio", @@ -1597,9 +1616,9 @@ dependencies = [ [[package]] name = "datafusion-catalog" -version = "53.1.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "37cefde60b26a7f4ff61e9d2ff2833322f91df2b568d7238afe67bde5bdffb66" +checksum = "06afd1e38dd27bbb1258685a1fc6524df6aff4e07b25b393a47de59635178d99" dependencies = [ "arrow", "async-trait", @@ -1622,9 +1641,9 @@ dependencies = [ [[package]] name = "datafusion-catalog-listing" -version = "53.1.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "17e112307715d6a7a331111a4c2330ff54bc237183511c319e3708a4cff431fb" +checksum = "f0668fb32c12065ec242be0e5b4bc62bd7a06a0be3ecd83791ef877e4be67e02" dependencies = [ "arrow", "async-trait", @@ -1645,32 +1664,33 @@ dependencies = [ [[package]] name = "datafusion-common" -version = "53.1.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d72a11ca44a95e1081870d3abb80c717496e8a7acb467a1d3e932bb636af5cc2" +checksum = "ca43b263cdff57042cfa8fb817fb3469f4878933380dccff25f5e793580abbf9" dependencies = [ - "ahash", "arrow", "arrow-ipc", + "arrow-schema", "chrono", + "foldhash 0.2.0", "half", - "hashbrown 0.16.1", + "hashbrown 0.17.1", "indexmap 2.14.0", "itertools 0.14.0", "libc", "log", "object_store", - "paste", "sqlparser", "tokio", + "uuid", "web-time", ] [[package]] name = "datafusion-common-runtime" -version = "53.1.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "89f4afaed29670ec4fd6053643adc749fe3f4bc9d1ce1b8c5679b22c67d12def" +checksum = "05f0ba2b864792bdca4d76c59a1de0ab6e1b61946596b9936888dbd6360035f2" dependencies = [ "futures", "log", @@ -1679,9 +1699,9 @@ dependencies = [ [[package]] name = "datafusion-datasource" -version = "53.1.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e9fb386e1691355355a96419978a0022b7947b44d4a24a6ea99f00b6b485cbb6" +checksum = "b840a8bce0bcbf5afad02946d438591e7c373f7afccaf3d874c04485772514dd" dependencies = [ "arrow", "async-trait", @@ -1701,16 +1721,17 @@ dependencies = [ "itertools 0.14.0", "log", "object_store", - "rand 0.9.4", + "parking_lot", + "rand 0.9.5", "tokio", "url", ] [[package]] name = "datafusion-datasource-arrow" -version = "53.1.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ffa6c52cfed0734c5f93754d1c0175f558175248bf686c944fb05c373e5fc096" +checksum = "a24cc0b9cf6e367f27f27406eff13abf48a11b72446aaa40b3105c0ded5c17d9" dependencies = [ "arrow", "arrow-ipc", @@ -1732,9 +1753,9 @@ dependencies = [ [[package]] name = "datafusion-datasource-csv" -version = "53.1.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "503f29e0582c1fc189578d665ff57d9300da1f80c282777d7eb67bb79fb8cdca" +checksum = "e1abe56b2a7a2d1d6de5117dd1a203181e267f28529faa5da546947621b697d7" dependencies = [ "arrow", "async-trait", @@ -1755,9 +1776,9 @@ dependencies = [ [[package]] name = "datafusion-datasource-json" -version = "53.1.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e33804749abc8d0c8cb7473228483cb8070e524c6f6086ee1b85a64debe2b3d2" +checksum = "9c3e467f0611ad7bdd5aad17c63c9bb6182d04e5282e5496d897ea2b49c024ba" dependencies = [ "arrow", "async-trait", @@ -1772,27 +1793,25 @@ dependencies = [ "datafusion-session", "futures", "object_store", - "serde_json", "tokio", "tokio-stream", ] [[package]] name = "datafusion-doc" -version = "53.1.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8de6ac0df1662b9148ad3c987978b32cbec7c772f199b1d53520c8fa764a87ee" +checksum = "d69bb69d8769e34f76839c960dbde24c1ac0c885a79b6c3c2287bdc56ec67891" [[package]] name = "datafusion-execution" -version = "53.1.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c03c7fbdaefcca4ef6ffe425a5fc2325763bfb426599bb0bf4536466efabe709" +checksum = "d8eac0a09bc8d263f52025cad9e001da4d8138d633fa288edda4d06b1772eae6" dependencies = [ "arrow", "arrow-buffer", "async-trait", - "chrono", "dashmap", "datafusion-common", "datafusion-expr", @@ -1801,18 +1820,19 @@ dependencies = [ "log", "object_store", "parking_lot", - "rand 0.9.4", + "rand 0.9.5", "tempfile", "url", ] [[package]] name = "datafusion-expr" -version = "53.1.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "574b9b6977fedbd2a611cbff12e5caf90f31640ad9dc5870f152836d94bad0dd" +checksum = "eeb14d374767ee0fc62dc79a5ba8bcf8a63c14e993c7d992d0e63adfa23d77d3" dependencies = [ "arrow", + "arrow-schema", "async-trait", "chrono", "datafusion-common", @@ -1823,33 +1843,31 @@ dependencies = [ "datafusion-physical-expr-common", "indexmap 2.14.0", "itertools 0.14.0", - "paste", "serde_json", "sqlparser", ] [[package]] name = "datafusion-expr-common" -version = "53.1.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7d7c3adf3db8bf61e92eb90cb659c8e8b734593a8f7c8e12a843c7ddba24b87e" +checksum = "b7b19a8c95522bee8cbb313d74263b85e355d2b52f42e67ef5694bf5de9e9356" dependencies = [ "arrow", "datafusion-common", "indexmap 2.14.0", "itertools 0.14.0", - "paste", ] [[package]] name = "datafusion-functions" -version = "53.1.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f28aa4e10384e782774b10e72aca4d93ef7b31aa653095d9d4536b0a3dbc51b6" +checksum = "5f64c983bbbdcb729d921a2b2ac3375598719b5cc0c30345ad664936f3176fc7" dependencies = [ "arrow", "arrow-buffer", - "base64", + "base64 0.22.1", "blake2", "blake3", "chrono", @@ -1860,26 +1878,25 @@ dependencies = [ "datafusion-expr", "datafusion-expr-common", "datafusion-macros", + "datafusion-physical-expr-common", "hex", "itertools 0.14.0", "log", - "md-5 0.10.6", + "md-5 0.11.0", "memchr", "num-traits", - "rand 0.9.4", + "rand 0.9.5", "regex", - "sha2 0.10.9", - "unicode-segmentation", + "sha2 0.11.0", "uuid", ] [[package]] name = "datafusion-functions-aggregate" -version = "53.1.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "00aa6217e56098ba84e0a338176fe52f0a84cca398021512c6c8c5eff806d0ad" +checksum = "89bc17041e424a47ed062f43df24d84aab8b57c4c3221e5c1a5eef46d6c5718b" dependencies = [ - "ahash", "arrow", "datafusion-common", "datafusion-doc", @@ -1889,19 +1906,18 @@ dependencies = [ "datafusion-macros", "datafusion-physical-expr", "datafusion-physical-expr-common", + "foldhash 0.2.0", "half", "log", "num-traits", - "paste", ] [[package]] name = "datafusion-functions-aggregate-common" -version = "53.1.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b511250349407db7c43832ab2de63f5557b19a20dfd236b39ca2c04468b50d47" +checksum = "97dd2a9e865c6108059f5b37b77934f84b50bfb108f837bd0e5c9536e03f0545" dependencies = [ - "ahash", "arrow", "datafusion-common", "datafusion-expr-common", @@ -1910,9 +1926,9 @@ dependencies = [ [[package]] name = "datafusion-functions-nested" -version = "53.1.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ef13a858e20d50f0a9bb5e96e7ac82b4e7597f247515bccca4fdd2992df0212a" +checksum = "75f0bdfeef16d96417b9632ef855645376b242e9006a126dfd0bedfc54a93f5f" dependencies = [ "arrow", "arrow-ord", @@ -1926,34 +1942,34 @@ dependencies = [ "datafusion-functions-aggregate-common", "datafusion-macros", "datafusion-physical-expr-common", - "hashbrown 0.16.1", + "hashbrown 0.17.1", "itertools 0.14.0", "itoa", "log", - "paste", + "memchr", ] [[package]] name = "datafusion-functions-table" -version = "53.1.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "72b40d3f5bbb3905f9ccb1ce9485a9595c77b69758a7c24d3ba79e334ff51e7e" +checksum = "f4e4941673c917819616877e9993da4503e4f4739812be0bc32c5356184c6383" dependencies = [ "arrow", "async-trait", "datafusion-catalog", "datafusion-common", "datafusion-expr", + "datafusion-physical-expr", "datafusion-physical-plan", "parking_lot", - "paste", ] [[package]] name = "datafusion-functions-window" -version = "53.1.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d4e88ec9d57c9b685d02f58bfee7be62d72610430ddcedb82a08e5d9925dbfb6" +checksum = "12dd2e16c12b84b6f6b41b19f55b366dd1c46876bb35b86896c6349067379e8d" dependencies = [ "arrow", "datafusion-common", @@ -1964,14 +1980,13 @@ dependencies = [ "datafusion-physical-expr", "datafusion-physical-expr-common", "log", - "paste", ] [[package]] name = "datafusion-functions-window-common" -version = "53.1.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8307bb93519b1a91913723a1130cfafeee3f72200d870d88e91a6fc5470ede5c" +checksum = "4cdc5e4b6f8b6ef823cc1c761f85088ad4c884fe8df64df3cbcc6b2b84698441" dependencies = [ "datafusion-common", "datafusion-physical-expr-common", @@ -1979,20 +1994,20 @@ dependencies = [ [[package]] name = "datafusion-macros" -version = "53.1.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2e367e6a71051d0ebdd29b2f85d12059b38b1d1f172c6906e80016da662226bd" +checksum = "1a3614234dd93578c92428cb4f408e020874f0d2b7e6c90c928d9d28b5df2ceb" dependencies = [ "datafusion-doc", "quote", - "syn", + "syn 2.0.118", ] [[package]] name = "datafusion-optimizer" -version = "53.1.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e929015451a67f77d9d8b727b2bf3a40c4445fdef6cdc53281d7d97c76888ace" +checksum = "a0635620b050b81bb92764e99250868f654e2cd5ad1bece413283b3f73c83179" dependencies = [ "arrow", "chrono", @@ -2009,11 +2024,10 @@ dependencies = [ [[package]] name = "datafusion-physical-expr" -version = "53.1.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4b1e68aba7a4b350401cfdf25a3d6f989ad898a7410164afe9ca52080244cb59" +checksum = "8cabf7a86eb70b816729e33c81bf7767c936ee1226f607a114f5dac2decac8d0" dependencies = [ - "ahash", "arrow", "datafusion-common", "datafusion-expr", @@ -2021,20 +2035,19 @@ dependencies = [ "datafusion-functions-aggregate-common", "datafusion-physical-expr-common", "half", - "hashbrown 0.16.1", + "hashbrown 0.17.1", "indexmap 2.14.0", "itertools 0.14.0", "parking_lot", - "paste", "petgraph", "tokio", ] [[package]] name = "datafusion-physical-expr-adapter" -version = "53.1.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ea22315f33cf2e0adc104e8ec42e285f6ed93998d565c65e82fec6a9ee9f9db4" +checksum = "de222e04f7e6744501555a54ab0abe26bfdfebee380af79a9bdc175704246859" dependencies = [ "arrow", "datafusion-common", @@ -2047,26 +2060,26 @@ dependencies = [ [[package]] name = "datafusion-physical-expr-common" -version = "53.1.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b04b45ea8ad3ac2d78f2ea2a76053e06591c9629c7a603eda16c10649ecf4362" +checksum = "72d0d0057fc5a502d45c870cb6d47c66eb7bdd5edb1bd71ad6f3f724975ac2a8" dependencies = [ - "ahash", "arrow", "chrono", "datafusion-common", "datafusion-expr-common", - "hashbrown 0.16.1", + "hashbrown 0.17.1", "indexmap 2.14.0", "itertools 0.14.0", "parking_lot", + "pin-project", ] [[package]] name = "datafusion-physical-optimizer" -version = "53.1.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7cb13397809a425918f608dfe8653f332015a3e330004ab191b4404187238b95" +checksum = "86046eed10950c5f9aaed9acfd148e9bd2e1dfdfe4f9aef607d1447b271e4183" dependencies = [ "arrow", "datafusion-common", @@ -2082,12 +2095,13 @@ dependencies = [ [[package]] name = "datafusion-physical-plan" -version = "53.1.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5edc023675791af9d5fb4cc4c24abf5f7bd3bd4dcf9e5bd90ea1eff6976dcc79" +checksum = "9bc84da934c903407ba297971ebcc020c4c1a38aafd765d6c144c76eff3fa6a1" dependencies = [ - "ahash", "arrow", + "arrow-data", + "arrow-ipc", "arrow-ord", "arrow-schema", "async-trait", @@ -2102,7 +2116,7 @@ dependencies = [ "datafusion-physical-expr-common", "futures", "half", - "hashbrown 0.16.1", + "hashbrown 0.17.1", "indexmap 2.14.0", "itertools 0.14.0", "log", @@ -2114,9 +2128,9 @@ dependencies = [ [[package]] name = "datafusion-pruning" -version = "53.1.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ac8c76860e355616555081cab5968cec1af7a80701ff374510860bcd567e365a" +checksum = "eb63eeac6de19be40f487b65dd84e546195f783c5a9928618e0c4f2a3569b0d7" dependencies = [ "arrow", "datafusion-common", @@ -2125,15 +2139,14 @@ dependencies = [ "datafusion-physical-expr", "datafusion-physical-expr-common", "datafusion-physical-plan", - "itertools 0.14.0", "log", ] [[package]] name = "datafusion-session" -version = "53.1.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5412111aa48e2424ba926112e192f7a6b7e4ccb450145d25ce5ede9f19dc491e" +checksum = "5f961d209177f91bd014db5cbb2c33b7d28a2597b9003e77f17aeb712964315a" dependencies = [ "async-trait", "datafusion-common", @@ -2145,9 +2158,9 @@ dependencies = [ [[package]] name = "datafusion-sql" -version = "53.1.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fa0d133ddf8b9b3b872acac900157f783e7b879fe9a6bccf389abebbfac45ec1" +checksum = "b1d71cb454da682b2af7488e1fc1ddd72ee1b28f19297b8ccad73f0a21ee9a69" dependencies = [ "arrow", "bigdecimal", @@ -2163,9 +2176,9 @@ dependencies = [ [[package]] name = "datafusion-substrait" -version = "53.1.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "98494539a5468979cc42d86c7bc5f0f8cb71ee5c742694c26fc34efdd29dd2e5" +checksum = "f047a6fbf967b6b523758a48d2377bb5f9373a20e7ae4c4326d3c569f80ba3d7" dependencies = [ "async-recursion", "async-trait", @@ -2255,7 +2268,7 @@ checksum = "1ac70aa55017e108007fbaf5aa0f54b021c98f92ff8af59d42eda9da96e3dd4f" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.118", ] [[package]] @@ -2366,11 +2379,10 @@ checksum = "40404c3f5f511ec4da6fe866ddf6a717c309fdbb69fbbad7b0f3edab8f2e835f" [[package]] name = "event-listener" -version = "5.4.1" +version = "5.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e13b66accf52311f30a0db42147dadea9850cb48cd070028831ae5f5d4b856ab" +checksum = "5a23add41df1562121a9393cb065eab5146a1242410f23a644851e90cfd669d2" dependencies = [ - "concurrent-queue", "parking", "pin-project-lite", ] @@ -2462,6 +2474,12 @@ dependencies = [ "percent-encoding", ] +[[package]] +name = "frostem" +version = "1.20260804.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d0ae10cfccbae085dd8612669ccc116fe88bd5dfe39d202a8fa68c24c1e546f" + [[package]] name = "fs_extra" version = "1.3.0" @@ -2470,10 +2488,10 @@ checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c" [[package]] name = "fsst" -version = "9.0.0-beta.16" +version = "11.0.0-beta.6" dependencies = [ "arrow-array", - "rand 0.9.4", + "rand 0.9.5", ] [[package]] @@ -2547,7 +2565,7 @@ checksum = "e835b70203e41293343137df5c0664546da5745f82ec9b84d40be8336958447b" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.118", ] [[package]] @@ -2698,12 +2716,13 @@ dependencies = [ [[package]] name = "geodatafusion" -version = "0.4.0" +version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "af7cd430f1a1f59bc97053d824ad410ea6fd123c8977b3c1a75335e289233b8b" +checksum = "fecbdd00d0fff2b04635c1b1e4129c217908f0c2d17539e0a2275308afce2552" dependencies = [ "arrow-arith", "arrow-array", + "arrow-buffer", "arrow-schema", "datafusion", "geo", @@ -2755,11 +2774,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd" dependencies = [ "cfg-if 1.0.4", - "js-sys", "libc", "r-efi 5.3.0", "wasip2", - "wasm-bindgen", ] [[package]] @@ -2793,7 +2810,7 @@ checksum = "53010ccb100b96a67bc32c0175f0ed1426b31b655d562898e57325f81c023ac0" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.118", ] [[package]] @@ -2816,17 +2833,25 @@ dependencies = [ [[package]] name = "goosefs-sdk" -version = "0.1.5" +version = "0.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ae079b88ffe7772d12cfc5c40a5a324babb357893d95b5e3a22ae857f236c5f" +checksum = "4a9bc9414e3b2cb0bd08dfe0eb315b177e86b119c7fa5e16179c92fc7b184860" dependencies = [ + "arc-swap", "async-trait", "bytes", "dashmap", + "futures", "hostname", + "io-uring", + "itoa", + "libc", + "lru", + "memmap2", + "moka", "prost", "prost-types", - "rand 0.9.4", + "rand 0.9.5", "reqwest 0.12.28", "serde", "thiserror 2.0.18", @@ -2836,6 +2861,7 @@ dependencies = [ "tonic-prost", "tracing", "uuid", + "xxhash-rust", ] [[package]] @@ -2863,6 +2889,7 @@ version = "2.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6ea2d84b969582b4b1864a92dc5d27cd2b77b622a8d79306834f1be5ba20d84b" dependencies = [ + "bytemuck", "cfg-if 1.0.4", "crunchy", "num-traits", @@ -2896,6 +2923,8 @@ version = "0.15.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9229cfe53dfd69f0609a49f65461bd93001ea1ef889cd5529dd176593f5338a1" dependencies = [ + "allocator-api2", + "equivalent", "foldhash 0.1.5", ] @@ -2915,6 +2944,11 @@ name = "hashbrown" version = "0.17.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" +dependencies = [ + "allocator-api2", + "equivalent", + "foldhash 0.2.0", +] [[package]] name = "heapify" @@ -3141,7 +3175,7 @@ version = "0.1.20" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "96547c2556ec9d12fb1578c4eaf448b04993e7fb79cbaad930a656880a6bdfa0" dependencies = [ - "base64", + "base64 0.22.1", "bytes", "futures-channel", "futures-util", @@ -3495,7 +3529,7 @@ checksum = "782d32378dddf207193ac91cefb848ad41abb58195c95168e1291227a0832b47" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.118", ] [[package]] @@ -3556,7 +3590,7 @@ dependencies = [ "quote", "rustc_version", "simd_cesu8", - "syn", + "syn 2.0.118", ] [[package]] @@ -3584,7 +3618,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "38c0b942f458fe50cdac086d2f946512305e5631e720728f2a61aabcd47a6264" dependencies = [ "quote", - "syn", + "syn 2.0.118", ] [[package]] @@ -3622,7 +3656,7 @@ dependencies = [ "nom", "num-traits", "ordered-float", - "rand 0.9.4", + "rand 0.9.5", "serde", "serde_json", "zmij", @@ -3647,7 +3681,7 @@ checksum = "e037a2e1d8d5fdbd49b16a4ea09d5d6401c1f29eca5ff29d03d3824dba16256a" [[package]] name = "lance" -version = "9.0.0-beta.16" +version = "11.0.0-beta.6" dependencies = [ "arc-swap", "arrow", @@ -3702,7 +3736,7 @@ dependencies = [ "prost", "prost-build", "prost-types", - "rand 0.9.4", + "rand 0.9.5", "rayon", "roaring", "rustc-hash", @@ -3720,7 +3754,7 @@ dependencies = [ [[package]] name = "lance-arrow" -version = "9.0.0-beta.16" +version = "11.0.0-beta.6" dependencies = [ "arrow-array", "arrow-buffer", @@ -3729,13 +3763,14 @@ dependencies = [ "arrow-ord", "arrow-schema", "arrow-select", + "bytemuck", "bytes", "futures", "getrandom 0.2.17", "half", "jsonb", "num-traits", - "rand 0.9.4", + "rand 0.9.5", ] [[package]] @@ -3762,7 +3797,7 @@ dependencies = [ [[package]] name = "lance-bitpacking" -version = "9.0.0-beta.16" +version = "11.0.0-beta.6" dependencies = [ "arrayref", "crunchy", @@ -3772,13 +3807,14 @@ dependencies = [ [[package]] name = "lance-core" -version = "9.0.0-beta.16" +version = "11.0.0-beta.6" dependencies = [ "arrow-array", "arrow-buffer", "arrow-data", "arrow-schema", "async-trait", + "blake3", "byteorder", "bytes", "datafusion-common", @@ -3795,7 +3831,8 @@ dependencies = [ "object_store", "pin-project", "prost", - "rand 0.9.4", + "quick_cache", + "rand 0.9.5", "roaring", "serde_json", "snafu", @@ -3810,7 +3847,7 @@ dependencies = [ [[package]] name = "lance-datafusion" -version = "9.0.0-beta.16" +version = "11.0.0-beta.6" dependencies = [ "arrow", "arrow-array", @@ -3842,7 +3879,7 @@ dependencies = [ [[package]] name = "lance-datagen" -version = "9.0.0-beta.16" +version = "11.0.0-beta.6" dependencies = [ "arrow", "arrow-array", @@ -3852,23 +3889,23 @@ dependencies = [ "futures", "half", "hex", - "rand 0.9.4", + "rand 0.9.5", "rand_distr", "rand_xoshiro", ] [[package]] name = "lance-derive" -version = "9.0.0-beta.16" +version = "11.0.0-beta.6" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.118", ] [[package]] name = "lance-encoding" -version = "9.0.0-beta.16" +version = "11.0.0-beta.6" dependencies = [ "arrow-arith", "arrow-array", @@ -3893,8 +3930,7 @@ dependencies = [ "num-traits", "prost", "prost-build", - "rand 0.9.4", - "strum", + "rand 0.9.5", "tokio", "tracing", "xxhash-rust", @@ -3903,11 +3939,12 @@ dependencies = [ [[package]] name = "lance-file" -version = "9.0.0-beta.16" +version = "11.0.0-beta.6" dependencies = [ "arrow-arith", "arrow-array", "arrow-buffer", + "arrow-cast", "arrow-data", "arrow-schema", "arrow-select", @@ -3933,7 +3970,7 @@ dependencies = [ [[package]] name = "lance-geo" -version = "9.0.0-beta.16" +version = "11.0.0-beta.6" dependencies = [ "datafusion", "geo-traits", @@ -3947,12 +3984,13 @@ dependencies = [ [[package]] name = "lance-index" -version = "9.0.0-beta.16" +version = "11.0.0-beta.6" dependencies = [ "arc-swap", "arrow", "arrow-arith", "arrow-array", + "arrow-ipc", "arrow-ord", "arrow-schema", "arrow-select", @@ -3985,6 +4023,7 @@ dependencies = [ "lance-encoding", "lance-file", "lance-geo", + "lance-index-core", "lance-io", "lance-linalg", "lance-select", @@ -3998,7 +4037,7 @@ dependencies = [ "prost", "prost-build", "prost-types", - "rand 0.9.4", + "rand 0.9.5", "rand_distr", "rangemap", "rayon", @@ -4013,19 +4052,35 @@ dependencies = [ "uuid", ] +[[package]] +name = "lance-index-core" +version = "11.0.0-beta.6" +dependencies = [ + "arrow-array", + "arrow-schema", + "arrow-select", + "async-trait", + "bytes", + "datafusion", + "datafusion-common", + "datafusion-expr", + "futures", + "lance-core", + "lance-io", + "lance-select", + "prost-types", + "roaring", + "serde", + "serde_json", +] + [[package]] name = "lance-io" -version = "9.0.0-beta.16" +version = "11.0.0-beta.6" dependencies = [ "arrow", - "arrow-arith", "arrow-array", - "arrow-buffer", - "arrow-cast", - "arrow-data", "arrow-schema", - "arrow-select", - "async-recursion", "async-trait", "aws-config", "aws-credential-types", @@ -4039,6 +4094,7 @@ dependencies = [ "lance-core", "lance-namespace", "log", + "metrics", "moka", "object_store", "object_store_opendal", @@ -4046,7 +4102,7 @@ dependencies = [ "path_abs", "pin-project", "prost", - "rand 0.9.4", + "rand 0.9.5", "serde", "tempfile", "tokio", @@ -4056,7 +4112,7 @@ dependencies = [ [[package]] name = "lance-jni" -version = "9.0.0-beta.16" +version = "11.0.0-beta.6" dependencies = [ "arrow", "arrow-array", @@ -4080,6 +4136,8 @@ dependencies = [ "lance-namespace-impls", "lance-table", "log", + "metrics", + "metrics-util", "object_store", "prost", "prost-types", @@ -4092,7 +4150,7 @@ dependencies = [ [[package]] name = "lance-linalg" -version = "9.0.0-beta.16" +version = "11.0.0-beta.6" dependencies = [ "arrow-array", "arrow-buffer", @@ -4102,13 +4160,13 @@ dependencies = [ "lance-arrow", "lance-core", "num-traits", - "rand 0.9.4", + "rand 0.9.5", "rayon", ] [[package]] name = "lance-namespace" -version = "9.0.0-beta.16" +version = "11.0.0-beta.6" dependencies = [ "arrow", "async-trait", @@ -4120,7 +4178,7 @@ dependencies = [ [[package]] name = "lance-namespace-impls" -version = "9.0.0-beta.16" +version = "11.0.0-beta.6" dependencies = [ "arrow", "arrow-ipc", @@ -4140,12 +4198,11 @@ dependencies = [ "lance-table", "log", "object_store", - "rand 0.9.4", + "rand 0.9.5", "reqwest 0.12.28", "roaring", "serde", "serde_json", - "time", "tokio", "tower", "tower-http 0.5.2", @@ -4169,7 +4226,7 @@ dependencies = [ [[package]] name = "lance-select" -version = "9.0.0-beta.16" +version = "11.0.0-beta.6" dependencies = [ "arrow-array", "arrow-buffer", @@ -4184,7 +4241,7 @@ dependencies = [ [[package]] name = "lance-table" -version = "9.0.0-beta.16" +version = "11.0.0-beta.6" dependencies = [ "arrow", "arrow-array", @@ -4192,6 +4249,7 @@ dependencies = [ "arrow-ipc", "arrow-schema", "async-trait", + "blake3", "byteorder", "bytes", "chrono", @@ -4206,7 +4264,7 @@ dependencies = [ "prost", "prost-build", "prost-types", - "rand 0.9.4", + "rand 0.9.5", "rangemap", "roaring", "semver", @@ -4221,10 +4279,10 @@ dependencies = [ [[package]] name = "lance-tokenizer" -version = "9.0.0-beta.16" +version = "11.0.0-beta.6" dependencies = [ + "frostem", "icu_segmenter", - "rust-stemmers", "serde", "stop-words", "unicode-normalization", @@ -4236,7 +4294,7 @@ version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" dependencies = [ - "spin", + "spin 0.9.9", ] [[package]] @@ -4298,9 +4356,9 @@ dependencies = [ [[package]] name = "libc" -version = "0.2.186" +version = "0.2.189" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66" +checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2" [[package]] name = "libm" @@ -4378,6 +4436,15 @@ dependencies = [ "tracing-subscriber", ] +[[package]] +name = "lru" +version = "0.12.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "234cf4f4a04dc1f57e24b96cc0cd600cf2af460d4161ac5ecdd0af8e1f3b2a38" +dependencies = [ + "hashbrown 0.15.5", +] + [[package]] name = "lru-slab" version = "0.1.2" @@ -4475,6 +4542,40 @@ version = "2.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "88904434abc2901f197fe8cc55f0445e7ded921dba5911dad2e2b39b48e663c4" +[[package]] +name = "memmap2" +version = "0.9.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d1219ed1b7f229ee7104d281dd01d6802fe28bb6e95d292942c4daacdeb798c0" +dependencies = [ + "libc", +] + +[[package]] +name = "metrics" +version = "0.24.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "89550ee9f79e88fef3119de263694973a8adb26c21d75322164fb8c493039fe2" +dependencies = [ + "portable-atomic", + "rapidhash", +] + +[[package]] +name = "metrics-util" +version = "0.19.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8496cc523d1f94c1385dd8f0f0c2c480b2b8aeccb5b7e4485ad6365523ae376" +dependencies = [ + "crossbeam-epoch", + "crossbeam-utils", + "hashbrown 0.15.5", + "metrics", + "rand 0.9.5", + "rand_xoshiro", + "sketches-ddsketch", +] + [[package]] name = "mime" version = "0.3.17" @@ -4686,7 +4787,7 @@ dependencies = [ "proc-macro-crate", "proc-macro2", "quote", - "syn", + "syn 2.0.118", ] [[package]] @@ -4724,7 +4825,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "622acbc9100d3c10e2ee15804b0caa40e55c933d5aa53814cd520805b7958a49" dependencies = [ "async-trait", - "base64", + "base64 0.22.1", "bytes", "chrono", "form_urlencoded", @@ -4759,9 +4860,9 @@ dependencies = [ [[package]] name = "object_store_opendal" -version = "0.57.0" +version = "0.58.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0eb12a624a41fce745838d0ef3701ff6c47797c13cd18ad3612fd2a3134fdbd8" +checksum = "88f165780495c17aa3ce86846600504198c3fffd99073521552751c2430fa6ac" dependencies = [ "async-trait", "bytes", @@ -4794,12 +4895,13 @@ checksum = "269bca4c2591a28585d6bf10d9ed0332b7d76900a1b02bec41bdc3a2cdcda107" [[package]] name = "opendal" -version = "0.57.0" +version = "0.58.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "96c9c85ce253ff87225e7669979d877a20c98a06604ec9d6dd5f4473e08f1ae1" +checksum = "4f20562cc7447fcc915fc5c23df305a412ea80a733c9f2fd9e2d267e2815be6d" dependencies = [ "ctor 1.0.7", "opendal-core", + "opendal-http-transport-reqwest", "opendal-layer-concurrent-limit", "opendal-layer-logging", "opendal-layer-retry", @@ -4817,24 +4919,22 @@ dependencies = [ [[package]] name = "opendal-core" -version = "0.57.0" +version = "0.58.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c4f8607c90e2c963a91467f50fb49fbc7fb3d573f88cea219ca59ccd3740b309" +checksum = "ec75551ff4cf3e57da98979f6a937aaa9ddb3915bf68cc17d03df733be6646ed" dependencies = [ "anyhow", - "base64", + "base64 0.23.0", "bytes", "futures", "http 1.4.2", - "http-body 1.0.1", "jiff", "log", "md-5 0.11.0", "mea", "percent-encoding", - "quick-xml 0.39.4", + "quick-xml 0.41.0", "reqsign-core", - "reqwest 0.13.4", "serde", "serde_json", "tokio", @@ -4843,11 +4943,25 @@ dependencies = [ "web-time", ] +[[package]] +name = "opendal-http-transport-reqwest" +version = "0.58.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ad4d4f19c3ce01126a30611f8e544eaa217104a278c889ac17c9374fe4f9e4ef" +dependencies = [ + "bytes", + "futures", + "http 1.4.2", + "http-body 1.0.1", + "opendal-core", + "reqwest 0.13.4", +] + [[package]] name = "opendal-layer-concurrent-limit" -version = "0.57.0" +version = "0.58.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0d6f81ba6960e3fae1882f253b114b21d7e444e1534f209c7737a79f6243eb6f" +checksum = "249ac5b0aa5a7a6c3737342d10456067937f9c9a6f3f02544271f7908ab91081" dependencies = [ "futures", "http 1.4.2", @@ -4857,9 +4971,9 @@ dependencies = [ [[package]] name = "opendal-layer-logging" -version = "0.57.0" +version = "0.58.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "58ada45c6d81d1aa4c9305d0c7d4bc317c59c85866a0908a2d75a7a978aa5ee2" +checksum = "5c75411ab00f77851ff086b686c1e9ca8175ac18c15afa2cb75b9036436cb06c" dependencies = [ "log", "opendal-core", @@ -4867,9 +4981,9 @@ dependencies = [ [[package]] name = "opendal-layer-retry" -version = "0.57.0" +version = "0.58.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b2a25a718afb81fad81cb9a0580a1cb989221fa2317f888c6a37f8dad408eb7" +checksum = "80b7738bd5f233ad8da39af9b9316b9b7a4eaddd91e8e32a1e19b7030688121d" dependencies = [ "backon", "log", @@ -4878,9 +4992,9 @@ dependencies = [ [[package]] name = "opendal-layer-timeout" -version = "0.57.0" +version = "0.58.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e91f731724c213af81e9d03517859c8fc47b4578e64ad61ae4f099f10fe36e3" +checksum = "a704141924500f3803c05ed871b53305d2a2f11cb5ef20160c3ee688a1857f66" dependencies = [ "opendal-core", "tokio", @@ -4888,17 +5002,17 @@ dependencies = [ [[package]] name = "opendal-service-azblob" -version = "0.57.0" +version = "0.58.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0030644366ef5d8cbe3a4a5822bf99a4aafddc1666e9d24b44d158d9062fc76a" +checksum = "b3310fbbb48f111c6f590473c2cd15e1b7f8e384444b0d4e328f0464c864d767" dependencies = [ - "base64", + "base64 0.23.0", "bytes", "http 1.4.2", "log", "opendal-core", "opendal-service-azure-common", - "quick-xml 0.39.4", + "quick-xml 0.41.0", "reqsign-azure-storage", "reqsign-core", "reqsign-file-read-tokio", @@ -4909,17 +5023,18 @@ dependencies = [ [[package]] name = "opendal-service-azdls" -version = "0.57.0" +version = "0.58.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6dea4908d490143a9b0b7f7a790e139ff829b06a023f670455ed3d44f664b361" +checksum = "2e3c406729935fe214ce574d68681a1ff7e0b322548f14094912bdbfe50e5c53" dependencies = [ - "base64", + "base64 0.23.0", "bytes", "http 1.4.2", "log", + "mea", "opendal-core", "opendal-service-azure-common", - "quick-xml 0.39.4", + "quick-xml 0.41.0", "reqsign-azure-storage", "reqsign-core", "reqsign-file-read-tokio", @@ -4929,9 +5044,9 @@ dependencies = [ [[package]] name = "opendal-service-azure-common" -version = "0.57.0" +version = "0.58.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b489f13c42e69d69bdd72952b634356ec43a7881a20259b38b540fcecdf4051" +checksum = "7348c88edf15af435b7be930077746b569fac5e738c1bf6a363b675e7317c9df" dependencies = [ "http 1.4.2", "opendal-core", @@ -4939,15 +5054,15 @@ dependencies = [ [[package]] name = "opendal-service-cos" -version = "0.57.0" +version = "0.58.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aa8cafe9729213375c7331019b0cb756ad3e1aff7f45cd32c45eae91ebde8901" +checksum = "d533d4582105d269c8aebeee5f0e8bcf960f41b8aab6197df7012254d9f39bf0" dependencies = [ "bytes", "http 1.4.2", "log", "opendal-core", - "quick-xml 0.39.4", + "quick-xml 0.41.0", "reqsign-core", "reqsign-file-read-tokio", "reqsign-tencent-cos", @@ -4956,9 +5071,9 @@ dependencies = [ [[package]] name = "opendal-service-gcs" -version = "0.57.0" +version = "0.58.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "48de101aac565ed06af4b47903c24eafd249075553ec1fb18256751c45148d47" +checksum = "007f3fba63c21e516c956b891e96ff9892d8175662bfb781cdada9d3766a11e6" dependencies = [ "async-trait", "bytes", @@ -4966,7 +5081,7 @@ dependencies = [ "log", "opendal-core", "percent-encoding", - "quick-xml 0.39.4", + "quick-xml 0.41.0", "reqsign-core", "reqsign-file-read-tokio", "reqsign-google", @@ -4977,9 +5092,9 @@ dependencies = [ [[package]] name = "opendal-service-goosefs" -version = "0.57.0" +version = "0.58.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "69e43048bde419947ba826fbdc2f134d6c03f44ebf48bd33a03b72f9fc45fcb4" +checksum = "60871e6386f04d831e6a5bdbc032af4a91aeba49963252d0ef456a2cf36a9b78" dependencies = [ "bytes", "goosefs-sdk", @@ -4991,9 +5106,9 @@ dependencies = [ [[package]] name = "opendal-service-hf" -version = "0.57.0" +version = "0.58.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c4922661976a1d40794a2adfbdb888cc3c23097690f825a92f773af38908a848" +checksum = "b41fd41eb7ed03c5e66cefda61e8e117808ffd2908f2916737cb020a6beb02c7" dependencies = [ "bytes", "hf-xet", @@ -5001,22 +5116,21 @@ dependencies = [ "log", "opendal-core", "percent-encoding", - "reqwest 0.13.4", "serde", "serde_json", ] [[package]] name = "opendal-service-oss" -version = "0.57.0" +version = "0.58.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "328fa55e8888cbdfe00826bfea2a79042422b720e8369e9e021e46121dea5ace" +checksum = "cd528ec2d49c5ca69e674ffed7b3e0686fb9cfcfea0596870de381467fda4f1b" dependencies = [ "bytes", "http 1.4.2", "log", "opendal-core", - "quick-xml 0.39.4", + "quick-xml 0.41.0", "reqsign-aliyun-oss", "reqsign-core", "reqsign-file-read-tokio", @@ -5025,18 +5139,18 @@ dependencies = [ [[package]] name = "opendal-service-s3" -version = "0.57.0" +version = "0.58.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "313d46c9f5ae70bca26b7c3e3fbb9b639292625f28af73aa016f47e788af9deb" +checksum = "58e80cdf192d7eff05feed747894d64f81905ac4eaf132edf7ea270abdd2d663" dependencies = [ - "base64", + "base64 0.23.0", "bytes", - "crc32c", + "crc-fast", "http 1.4.2", "log", "md-5 0.11.0", "opendal-core", - "quick-xml 0.39.4", + "quick-xml 0.41.0", "reqsign-aws-v4", "reqsign-core", "reqsign-file-read-tokio", @@ -5046,14 +5160,14 @@ dependencies = [ [[package]] name = "opendal-service-tos" -version = "0.57.0" +version = "0.58.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6f2f7a4c32e5202eb4ac72e76c4b5e30c86ab60762811172f4111103b9d673a1" +checksum = "7841a1a09485bd08eeac34d67804321b62456c855027c580bce12a75564f4609" dependencies = [ "bytes", "http 1.4.2", "opendal-core", - "quick-xml 0.39.4", + "quick-xml 0.41.0", "reqsign-core", "reqsign-file-read-tokio", "reqsign-volcengine-tos", @@ -5160,7 +5274,7 @@ version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "898bac3fa00d0ba57a4e8289837e965baa2dee8c3749f3b11d45a64b4223d9c3" dependencies = [ - "base64", + "base64 0.22.1", "serde", ] @@ -5207,7 +5321,7 @@ version = "3.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1d30c53c26bc5b31a98cd02d20f25a7c8567146caf63ed593a9d87b2775291be" dependencies = [ - "base64", + "base64 0.22.1", "serde_core", ] @@ -5279,7 +5393,7 @@ checksum = "c96395f0a926bc13b1c17622aaddda1ecb55d49c8f1bf9777e4d877800a43f8b" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.118", ] [[package]] @@ -5386,7 +5500,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "479ca8adacdd7ce8f1fb39ce9ecccbfe93a3f1344b3d0d97f20bc0196208f62b" dependencies = [ "proc-macro2", - "syn", + "syn 2.0.118", ] [[package]] @@ -5424,7 +5538,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "03da047801ff44bb6a4d407d4860c05fd70bb81714e6b2f3812603d5b145b042" dependencies = [ "heck", - "itertools 0.14.0", + "itertools 0.11.0", "log", "multimap", "petgraph", @@ -5432,7 +5546,7 @@ dependencies = [ "prost", "prost-types", "regex", - "syn", + "syn 2.0.118", "tempfile", ] @@ -5443,10 +5557,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b570b25f7617e43d59005d0990ccb79e950a423952cea19671b7a876da390adf" dependencies = [ "anyhow", - "itertools 0.14.0", + "itertools 0.11.0", "proc-macro2", "quote", - "syn", + "syn 2.0.118", ] [[package]] @@ -5470,14 +5584,26 @@ dependencies = [ [[package]] name = "quick-xml" -version = "0.40.1" +version = "0.41.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2474bd2e5029e7ccb6abb2ba48cf2383a333851dedf495901544281590c7da7f" +checksum = "e660451e55124f798a69a5af3f49ccfbefbd41910eefd25caf2393e1f3473ec1" dependencies = [ "memchr", "serde", ] +[[package]] +name = "quick_cache" +version = "0.6.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9c6658afe513a3b484e3abfdaa0d03ef3c0bbf017542c178dd55f94eb3051f9" +dependencies = [ + "ahash", + "equivalent", + "hashbrown 0.16.1", + "parking_lot", +] + [[package]] name = "quinn" version = "0.11.9" @@ -5500,15 +5626,16 @@ dependencies = [ [[package]] name = "quinn-proto" -version = "0.11.14" +version = "0.11.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "434b42fec591c96ef50e21e886936e66d3cc3f737104fdb9b737c40ffb94c098" +checksum = "2f4bfc015262b9df63c8845072ce59068853ff5872180c2ce2f13038b970e560" dependencies = [ "aws-lc-rs", "bytes", - "getrandom 0.3.4", + "getrandom 0.4.3", "lru-slab", - "rand 0.9.4", + "rand 0.10.1", + "rand_pcg", "ring", "rustc-hash", "rustls", @@ -5573,9 +5700,9 @@ dependencies = [ [[package]] name = "rand" -version = "0.9.4" +version = "0.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "44c5af06bb1b7d3216d91932aed5265164bf384dc89cd6ba05cf59a35f5f76ea" +checksum = "b9ef1d0d795eb7d84685bca4f72f3649f064e6641543d3a8c415898726a57b41" dependencies = [ "rand_chacha 0.9.0", "rand_core 0.9.5", @@ -5643,7 +5770,16 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6a8615d50dcf34fa31f7ab52692afec947c4dd0ab803cc87cb3b0b4570ff7463" dependencies = [ "num-traits", - "rand 0.9.4", + "rand 0.9.5", +] + +[[package]] +name = "rand_pcg" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "caa0f4137e1c0a72f4c651489402276c8e8e1cf081f3b0ba156d2cbeef09e86a" +dependencies = [ + "rand_core 0.10.1", ] [[package]] @@ -5661,6 +5797,15 @@ version = "1.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "973443cf09a9c8656b574a866ab68dfa19f0867d0340648c7d2f6a71b8a8ea68" +[[package]] +name = "rapidhash" +version = "4.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32b266a82f4aa99bb5c25e28d11cc44ace63d91adbcbcee4d323e2ae3d49ef37" +dependencies = [ + "rustversion", +] + [[package]] name = "rawpointer" version = "0.2.1" @@ -5733,7 +5878,7 @@ checksum = "b7186006dcb21920990093f30e3dea63b7d6e977bf1256be20c3563a5db070da" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.118", ] [[package]] @@ -5783,9 +5928,9 @@ dependencies = [ [[package]] name = "reqsign-aliyun-oss" -version = "3.1.0" +version = "3.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "372266b4733756738eeb199a98188037d27a0989980e2600ae7ce1faf00a867d" +checksum = "9c0f9f69a519dd6958c4b43606bb8e1278cdc76d611fc8fed4b796eee548dc0f" dependencies = [ "anyhow", "form_urlencoded", @@ -5800,9 +5945,9 @@ dependencies = [ [[package]] name = "reqsign-aws-v4" -version = "3.0.1" +version = "3.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b75624bd8a466e37ddc0a7b6c33ac859a85347c153a916e1dd9d0b68338f74a" +checksum = "cc883bc56889f3e4a419265c87facea222a921debc5c6f15c7fd8b68ec4b36b2" dependencies = [ "anyhow", "bytes", @@ -5811,7 +5956,7 @@ dependencies = [ "http 1.4.2", "log", "percent-encoding", - "quick-xml 0.40.1", + "quick-xml 0.41.0", "reqsign-core", "rust-ini", "serde", @@ -5822,12 +5967,12 @@ dependencies = [ [[package]] name = "reqsign-azure-storage" -version = "3.0.1" +version = "3.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "62b96928e73ad984de1d99e382749d09e5dab7dd707b767974f7e40aa926b82f" +checksum = "a6ebd8524185ce9c64063e3095f83968acfa90922f00c601a4a0f3aca15b077e" dependencies = [ "anyhow", - "base64", + "base64 0.22.1", "bytes", "form_urlencoded", "http 1.4.2", @@ -5843,14 +5988,13 @@ dependencies = [ [[package]] name = "reqsign-core" -version = "3.0.1" +version = "3.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a5fa5cb48808693614d1701fcd3db0b30fa292e0f18e122ae068b6d32eaeed3f" +checksum = "7e38b44697c60a823705ccef85cb04d8e0527c9d16ed7c58bf1c6395bdd24ceb" dependencies = [ "anyhow", - "base64", + "base64 0.22.1", "bytes", - "form_urlencoded", "futures", "hex", "hmac 0.13.0", @@ -5868,9 +6012,9 @@ dependencies = [ [[package]] name = "reqsign-file-read-tokio" -version = "3.0.1" +version = "3.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6a4b6f3a3fd29ffcc99a90aec585a65217783badfd73acddf847b63ae683bda9" +checksum = "688ff0ae421b8d4b92b53fdafaf53df2de28f428a9962edcf21702990b26f74b" dependencies = [ "anyhow", "reqsign-core", @@ -5879,9 +6023,9 @@ dependencies = [ [[package]] name = "reqsign-google" -version = "3.0.1" +version = "3.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eb215d0876a18b6bd9cdd380b589e5292aaa638ca15266de794b1122d898b6b2" +checksum = "a96da0b579b846d358090cb06b9e3c2ad1375529efbe3e0c45f96bd7bcf043ea" dependencies = [ "form_urlencoded", "http 1.4.2", @@ -5897,9 +6041,9 @@ dependencies = [ [[package]] name = "reqsign-tencent-cos" -version = "3.0.1" +version = "3.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "84110aabba799fbcd48b3abb51fbbff4749f879252e5806b6f5d0cbe0fef6abb" +checksum = "f6497dd9f6e3d1349b420521484099b284f95e8d3a65f088fccef42493a7b644" dependencies = [ "anyhow", "http 1.4.2", @@ -5912,9 +6056,9 @@ dependencies = [ [[package]] name = "reqsign-volcengine-tos" -version = "3.0.1" +version = "3.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "91d083a363b3577f519ce8425bb50f902622a28a83f7c4a26a5c990b66ec75b3" +checksum = "4335f949a3fd8b53867fd716dac97fbd545e45bcaed44343796517f2e19cd609" dependencies = [ "anyhow", "http 1.4.2", @@ -5929,7 +6073,7 @@ version = "0.12.28" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "eddd3ca559203180a307f12d114c268abf583f59b03cb906fd0b3ff8646c1147" dependencies = [ - "base64", + "base64 0.22.1", "bytes", "encoding_rs", "futures-core", @@ -5975,7 +6119,7 @@ version = "0.13.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "219c5811de6525e5416c7d5d53bb656d3afdbc6c5af816e0802bcfa42dbdc1c3" dependencies = [ - "base64", + "base64 0.22.1", "bytes", "futures-core", "futures-util", @@ -6095,21 +6239,11 @@ dependencies = [ "ordered-multimap", ] -[[package]] -name = "rust-stemmers" -version = "1.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e46a2036019fdb888131db7a4c847a1063a7493f971ed94ea82c67eada63ca54" -dependencies = [ - "serde", - "serde_derive", -] - [[package]] name = "rustc-hash" -version = "2.1.2" +version = "2.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "94300abf3f1ae2e2b8ffb7b58043de3d399c73fa6f4b73826402a5c457614dbe" +checksum = "6b1e7f9a428571be2dc5bc0505c13fb6bf936822b894ec87abf8a08a4e51742d" [[package]] name = "rustc_version" @@ -6299,7 +6433,7 @@ dependencies = [ "proc-macro2", "quote", "serde_derive_internals", - "syn", + "syn 2.0.118", ] [[package]] @@ -6366,9 +6500,9 @@ checksum = "1bc711410fbe7399f390ca1c3b60ad0f53f80e95c5eb935e52268a0e2cd49acc" [[package]] name = "serde" -version = "1.0.228" +version = "1.0.229" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e" +checksum = "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba" dependencies = [ "serde_core", "serde_derive", @@ -6376,22 +6510,22 @@ dependencies = [ [[package]] name = "serde_core" -version = "1.0.228" +version = "1.0.229" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "41d385c7d4ca58e59fc732af25c3983b67ac852c1a25000afe1175de458b67ad" +checksum = "67dca2c9c51e58a4791a4b1ed58308b39c64224d349a935ab5039aa360942a48" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.228" +version = "1.0.229" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" +checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 3.0.2", ] [[package]] @@ -6402,15 +6536,16 @@ checksum = "18d26a20a969b9e3fdf2fc2d9f21eda6c40e2de84c9408bb5d3b05d499aae711" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.118", ] [[package]] name = "serde_json" -version = "1.0.150" +version = "1.0.151" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e8014e44b4736ed0538adeecded0fce2a272f22dc9578a7eb6b2d9993c74cfb9" +checksum = "c841b55ecdae098c80dcae9cf767f6f8a0c2cdb3416bbef72181df4d0fe73f14" dependencies = [ + "indexmap 2.14.0", "itoa", "memchr", "serde", @@ -6437,7 +6572,7 @@ checksum = "175ee3e80ae9982737ca543e96133087cbd9a485eecc3bc4de9c1a37b47ea59c" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.118", ] [[package]] @@ -6449,7 +6584,7 @@ dependencies = [ "proc-macro2", "quote", "serde", - "syn", + "syn 2.0.118", ] [[package]] @@ -6470,7 +6605,7 @@ version = "3.21.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "76a5c54c7310e7b8b9577c286d7e399ddd876c3e12b3ed917a8aabc4b96e9e8c" dependencies = [ - "base64", + "base64 0.22.1", "bs58", "chrono", "hex", @@ -6493,7 +6628,7 @@ dependencies = [ "darling", "proc-macro2", "quote", - "syn", + "syn 2.0.118", ] [[package]] @@ -6626,6 +6761,12 @@ version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8ee5873ec9cce0195efcb7a4e9507a04cd49aec9c83d0389df45b1ef7ba2e649" +[[package]] +name = "sketches-ddsketch" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c6f73aeb92d671e0cc4dca167e59b2deb6387c375391bc99ee743f326994a2b" + [[package]] name = "slab" version = "0.4.12" @@ -6640,23 +6781,23 @@ checksum = "8ed6a63f02c8539c91a8685a86f4099661ba3da017932f6ebbea6de3f0fa7c90" [[package]] name = "snafu" -version = "0.9.1" +version = "0.9.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d1a012328be2e3f5d5f6f3218147ca02588cea4cb865e876849ab6debcf36522" +checksum = "e45cb604038abb7b926b679887b3226d8d0f23874b66623625a0454be425a4b7" dependencies = [ "snafu-derive", ] [[package]] name = "snafu-derive" -version = "0.9.1" +version = "0.9.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f103c50866b8743da9429b8a581d81a27c2d3a9c4ac7df8f8571c1dd7896eda" +checksum = "287f59010008f0d7cf5e3b03196d666c1acc46c8d3e9cf34c28a1a7157601e72" dependencies = [ "heck", "proc-macro2", "quote", - "syn", + "syn 2.0.118", ] [[package]] @@ -6683,9 +6824,15 @@ dependencies = [ [[package]] name = "spin" -version = "0.9.8" +version = "0.9.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3763264f6b73151db08c50ff20d7d8a0b8796e021cdea7ceedad07b80155fa0e" + +[[package]] +name = "spin" +version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67" +checksum = "023a211cb3138dbc438680b32560ad89f699977624c9f8dbb95a47d5b4c07dd3" [[package]] name = "spki" @@ -6699,9 +6846,9 @@ dependencies = [ [[package]] name = "sqlparser" -version = "0.61.0" +version = "0.62.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dbf5ea8d4d7c808e1af1cbabebca9a2abe603bcefc22294c5b95018d53200cb7" +checksum = "13c6d1b651dc4edf07eead2a0c6c78016ce971bc2c10da5266861b13f25e7cec" dependencies = [ "log", "sqlparser_derive", @@ -6715,7 +6862,7 @@ checksum = "a6dd45d8fc1c79299bfbb7190e42ccbbdf6a5f52e4a6ad98d92357ea965bd289" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.118", ] [[package]] @@ -6767,35 +6914,14 @@ version = "0.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" -[[package]] -name = "strum" -version = "0.26.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8fec0f0aef304996cf250b31b5a10dee7980c85da9d759361292b8bca5a18f06" -dependencies = [ - "strum_macros", -] - -[[package]] -name = "strum_macros" -version = "0.26.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c6bee85a5a24955dc440386795aa378cd9cf82acd5f764469152d2270e581be" -dependencies = [ - "heck", - "proc-macro2", - "quote", - "rustversion", - "syn", -] - [[package]] name = "substrait" -version = "0.62.2" +version = "0.63.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "62fc4b483a129b9772ccb9c3f7945a472112fdd9140da87f8a4e7f1d44e045d0" +checksum = "e620ff4d5c02fd6f7752931aa74b16a26af66a63022cc1ad412c77edbe0bab47" dependencies = [ "heck", + "indexmap 2.14.0", "pbjson", "pbjson-build", "pbjson-types", @@ -6809,7 +6935,7 @@ dependencies = [ "serde", "serde_json", "serde_yaml", - "syn", + "syn 2.0.118", "typify", "walkdir", ] @@ -6837,6 +6963,17 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "syn" +version = "3.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a207d6d6a2b7fc470b80443726053f18a2481b7e1eee970597051596567987a3" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + [[package]] name = "sync_wrapper" version = "1.0.2" @@ -6854,7 +6991,7 @@ checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.118", ] [[package]] @@ -6943,7 +7080,7 @@ checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.118", ] [[package]] @@ -6954,7 +7091,7 @@ checksum = "ebc4ee7f67670e9b64d05fa4253e753e016c6c95ff35b89b7941d6b856dec1d5" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.118", ] [[package]] @@ -7043,9 +7180,9 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" -version = "1.52.3" +version = "1.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8fc7f01b389ac15039e4dc9531aa973a135d7a4135281b12d7c1bc79fd57fffe" +checksum = "202caea871b69668250d242070849eb495be178ed697a3e98aebce5bc81a0bed" dependencies = [ "bytes", "libc", @@ -7066,7 +7203,7 @@ checksum = "385a6cb71ab9ab790c5fe8d67f1645e6c450a7ce006a33de03daa956cf70a496" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.118", ] [[package]] @@ -7092,9 +7229,9 @@ dependencies = [ [[package]] name = "tokio-stream" -version = "0.1.18" +version = "0.1.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "32da49809aab5c3bc678af03902d4ccddea2a87d028d86392a4b1560c6906c70" +checksum = "a3d06f0b082ba57c26b79407372e57cf2a1e28124f78e9479fe80322cf53420b" dependencies = [ "futures-core", "pin-project-lite", @@ -7104,13 +7241,14 @@ dependencies = [ [[package]] name = "tokio-util" -version = "0.7.18" +version = "0.7.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ae9cec805b01e8fc3fd2fe289f89149a9b66dd16786abd8b19cfa7b48cb0098" +checksum = "494815d09bf52b5548659851081238f0ca39ff638363907596da739561c62c52" dependencies = [ "bytes", "futures-core", "futures-sink", + "libc", "pin-project-lite", "tokio", ] @@ -7152,7 +7290,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ac2a5518c70fa84342385732db33fb3f44bc4cc748936eb5833d2df34d6445ef" dependencies = [ "async-trait", - "base64", + "base64 0.22.1", "bytes", "h2", "http 1.4.2", @@ -7288,7 +7426,7 @@ checksum = "7490cfa5ec963746568740651ac6781f701c9c5ea257c58e057f3ba8cf69e8da" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.118", ] [[package]] @@ -7351,11 +7489,11 @@ checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" [[package]] name = "twox-hash" -version = "2.1.2" +version = "2.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ea3136b675547379c4bd395ca6b938e5ad3c3d20fad76e7fe85f9e0d011419c" +checksum = "8464ec13c3691491391d9fce00f6416c9a48e46972f72d7865688be2080192c9" dependencies = [ - "rand 0.9.4", + "rand 0.10.1", ] [[package]] @@ -7395,7 +7533,7 @@ dependencies = [ "semver", "serde", "serde_json", - "syn", + "syn 2.0.118", "thiserror 2.0.18", "unicode-ident", ] @@ -7413,7 +7551,7 @@ dependencies = [ "serde", "serde_json", "serde_tokenstream", - "syn", + "syn 2.0.118", "typify-impl", ] @@ -7500,9 +7638,9 @@ checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" [[package]] name = "uuid" -version = "1.23.4" +version = "1.24.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bf80a72845275afea99e7f2b434723d3bc7e38470fcd1c7ed39a599c73319a53" +checksum = "bf3923a6f5c4c6382e0b653c4117f48d631ea17f38ed86e2a828e6f7412f5239" dependencies = [ "getrandom 0.4.3", "js-sys", @@ -7622,7 +7760,7 @@ dependencies = [ "bumpalo", "proc-macro2", "quote", - "syn", + "syn 2.0.118", "wasm-bindgen-shared", ] @@ -7796,7 +7934,7 @@ checksum = "053e2e040ab57b9dc951b72c264860db7eb3b0200ba345b4e4c3b14f67855ddf" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.118", ] [[package]] @@ -7807,7 +7945,7 @@ checksum = "3f316c4a2570ba26bbec722032c4099d8c8bc095efccdc15688708623367e358" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.118", ] [[package]] @@ -8149,7 +8287,7 @@ checksum = "3e1e496dcbe6a09017acdfaf48e1a646735e7ff5b2a49e2c7e081cca77a59bc8" dependencies = [ "anyhow", "async-trait", - "base64", + "base64 0.22.1", "bytes", "clap", "crc32fast", @@ -8186,7 +8324,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cb838aa8eb67d730af301584cf003caad407487606058292a6750711b603fbee" dependencies = [ "async-trait", - "base64", + "base64 0.22.1", "blake3", "bytemuck", "bytes", @@ -8296,9 +8434,9 @@ checksum = "66fee0b777b0f5ac1c69bb06d361268faafa61cd4682ae064a171c16c433e9e4" [[package]] name = "xxhash-rust" -version = "0.8.16" +version = "0.8.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4d93c89cdc2d3a63c3ec48ffe926931bdc069eafa8e4402fe6d8f790c9d1e576" +checksum = "aee1b19627c7c60102ab80d3a9cbe18de90bfe03bfa6c3715447681f0e8c8af6" [[package]] name = "yoke" @@ -8319,7 +8457,7 @@ checksum = "de844c262c8848816172cef550288e7dc6c7b7814b4ee56b3e1553f275f1858e" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.118", "synstructure", ] @@ -8340,7 +8478,7 @@ checksum = "1ae7f38b72ec2a254e2b87ef277cf2cd4fb97cbebf944faa6f33354da0867930" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.118", ] [[package]] @@ -8360,7 +8498,7 @@ checksum = "11532158c46691caf0f2593ea8358fed6bbf68a0315e80aae9bd41fbade684a1" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.118", "synstructure", ] @@ -8402,7 +8540,7 @@ checksum = "625dc425cab0dca6dc3c3319506e6593dcb08a9f387ea3b284dbd52a92c40555" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.118", ] [[package]] diff --git a/java/lance-jni/Cargo.toml b/java/lance-jni/Cargo.toml index d8f839b273a..4247a0b4f66 100644 --- a/java/lance-jni/Cargo.toml +++ b/java/lance-jni/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "lance-jni" -version = "9.0.0-beta.16" +version = "11.0.0-beta.6" edition = "2024" authors = ["Lance Devs "] rust-version = "1.91" @@ -20,14 +20,15 @@ crate-type = ["cdylib"] [features] default = [] +backtrace = ["lance/backtrace", "lance-core/backtrace"] [dependencies] -lance = { path = "../../rust/lance", features = ["substrait"] } +lance = { path = "../../rust/lance", features = ["substrait", "metrics"] } lance-datafusion = { path = "../../rust/lance-datafusion" } lance-encoding = { path = "../../rust/lance-encoding" } lance-linalg = { path = "../../rust/lance-linalg" } lance-index = { path = "../../rust/lance-index" } -lance-io = { path = "../../rust/lance-io" } +lance-io = { path = "../../rust/lance-io", features = ["metrics"] } lance-namespace = { path = "../../rust/lance-namespace" } lance-namespace-impls = { path = "../../rust/lance-namespace-impls", features = ["rest", "rest-adapter", "dir-goosefs"] } lance-core = { path = "../../rust/lance-core" } @@ -36,8 +37,8 @@ lance-table = { path = "../../rust/lance-table" } arrow = { version = "58.0.0", features = ["ffi"] } arrow-array = "58.0.0" arrow-schema = "58.0.0" -datafusion = { version = "53.0.0", default-features = false } -datafusion-common = "53.0.0" +datafusion = { version = "54.0.0", default-features = false } +datafusion-common = "54.0.0" object_store = { version = "0.13.2" } tokio = { version = "1.23", features = [ "rt-multi-thread", @@ -51,6 +52,8 @@ serde = { version = "1.0", features = ["derive"] } serde_json = { version = "1" } bytes = "1.11" log = "0.4" +metrics = "0.24" +metrics-util = { version = "0.19", default-features = false, features = ["registry"] } env_logger = "0.11.7" uuid = { version = "1.17.0", features = ["v4"] } prost = "0.14.1" @@ -71,5 +74,8 @@ debug-assertions = false strip = "debuginfo" incremental = false +[profile.release] +strip = true + [lints.clippy] disallowed_macros = "deny" diff --git a/java/lance-jni/src/async_scanner.rs b/java/lance-jni/src/async_scanner.rs index 6da10479266..4f65deda0d4 100644 --- a/java/lance-jni/src/async_scanner.rs +++ b/java/lance-jni/src/async_scanner.rs @@ -3,12 +3,12 @@ use std::sync::Arc; -use crate::RT; use crate::blocking_dataset::{BlockingDataset, NATIVE_DATASET}; use crate::blocking_scanner::{ScannerOptions, build_scanner_with_options}; use crate::dispatcher::{DISPATCHER, DispatcherMessage}; use crate::error::Result; use crate::task_tracker::{TASK_TRACKER, TaskInfo}; +use crate::{RT, block_on}; use arrow::ffi::FFI_ArrowSchema; use jni::JNIEnv; use jni::objects::JObject; @@ -82,7 +82,7 @@ impl AsyncScanner { // Will be aborted when real handle is registered }); - RT.block_on(async { + block_on(async { TASK_TRACKER .register( task_id, @@ -163,7 +163,7 @@ impl AsyncScanner { }); // Step 3: Update registration with real handle - RT.block_on(async { + block_on(async { TASK_TRACKER.update_handle(task_id, handle).await; }); } @@ -181,6 +181,8 @@ pub extern "system" fn Java_org_lance_ipc_AsyncScanner_createAsyncScanner<'local substrait_filter_obj: JObject<'local>, filter_obj: JObject<'local>, batch_size_obj: JObject<'local>, + batch_size_bytes_obj: JObject<'local>, + io_buffer_size_obj: JObject<'local>, limit_obj: JObject<'local>, offset_obj: JObject<'local>, query_obj: JObject<'local>, @@ -189,6 +191,9 @@ pub extern "system" fn Java_org_lance_ipc_AsyncScanner_createAsyncScanner<'local with_row_id: jboolean, with_row_address: jboolean, batch_readahead: jint, + fragment_readahead_obj: JObject<'local>, + scan_in_order: jboolean, + late_materialization_obj: JObject<'local>, column_orderings: JObject<'local>, use_scalar_index: jboolean, fast_search: jboolean, @@ -207,6 +212,8 @@ pub extern "system" fn Java_org_lance_ipc_AsyncScanner_createAsyncScanner<'local substrait_filter_obj, filter_obj, batch_size_obj, + batch_size_bytes_obj, + io_buffer_size_obj, limit_obj, offset_obj, query_obj, @@ -215,6 +222,9 @@ pub extern "system" fn Java_org_lance_ipc_AsyncScanner_createAsyncScanner<'local with_row_id, with_row_address, batch_readahead, + fragment_readahead_obj, + scan_in_order, + late_materialization_obj, column_orderings, use_scalar_index, fast_search, @@ -235,6 +245,8 @@ fn inner_create_async_scanner<'local>( substrait_filter_obj: JObject<'local>, filter_obj: JObject<'local>, batch_size_obj: JObject<'local>, + batch_size_bytes_obj: JObject<'local>, + io_buffer_size_obj: JObject<'local>, limit_obj: JObject<'local>, offset_obj: JObject<'local>, query_obj: JObject<'local>, @@ -243,6 +255,9 @@ fn inner_create_async_scanner<'local>( with_row_id: jboolean, with_row_address: jboolean, batch_readahead: jint, + fragment_readahead_obj: JObject<'local>, + scan_in_order: jboolean, + late_materialization_obj: JObject<'local>, column_orderings: JObject<'local>, use_scalar_index: jboolean, fast_search: jboolean, @@ -262,6 +277,8 @@ fn inner_create_async_scanner<'local>( substrait_filter_obj, filter_obj, batch_size_obj, + batch_size_bytes_obj, + io_buffer_size_obj, limit_obj, offset_obj, query_obj, @@ -270,6 +287,9 @@ fn inner_create_async_scanner<'local>( with_row_id, with_row_address, batch_readahead, + fragment_readahead_obj, + scan_in_order, + late_materialization_obj, column_orderings, use_scalar_index, fast_search, @@ -323,7 +343,7 @@ pub extern "system" fn Java_org_lance_ipc_AsyncScanner_nativeCancelTask( _j_scanner: JObject, task_id: jlong, ) { - RT.block_on(async { + block_on(async { TASK_TRACKER.cancel(task_id as u64).await; }); } @@ -361,7 +381,7 @@ fn inner_import_async_ffi_schema( let scanner_guard = unsafe { env.get_rust_field::<_, _, AsyncScanner>(j_scanner, NATIVE_ASYNC_SCANNER)? }; - let schema = RT.block_on(scanner_guard.inner.schema())?; + let schema = block_on(scanner_guard.inner.schema())?; let ffi_schema = FFI_ArrowSchema::try_from(&*schema)?; unsafe { std::ptr::write_unaligned(schema_addr as *mut FFI_ArrowSchema, ffi_schema) } Ok(()) diff --git a/java/lance-jni/src/blocking_blob.rs b/java/lance-jni/src/blocking_blob.rs index 002fa817cf6..77531053571 100755 --- a/java/lance-jni/src/blocking_blob.rs +++ b/java/lance-jni/src/blocking_blob.rs @@ -4,7 +4,7 @@ use crate::blocking_dataset::{BlockingDataset, NATIVE_DATASET}; use crate::error::Result; use crate::traits::{FromJString, IntoJava}; -use crate::{JNIEnvExt, RT}; +use crate::{JNIEnvExt, block_on}; use jni::JNIEnv; use jni::objects::{JByteArray, JObject, JString, JValueGen}; use jni::sys::{jbyteArray, jint, jlong}; @@ -59,23 +59,26 @@ fn inner_take_blobs<'local>( let blobs = { let dataset = unsafe { env.get_rust_field::<_, _, BlockingDataset>(jdataset, NATIVE_DATASET) }?; - RT.block_on(Arc::new(dataset.inner.clone()).take_blobs(&row_ids_u64, col_name))? + block_on(Arc::new(dataset.inner.clone()).take_blobs(&row_ids_u64, col_name))? }; let j_blobs = blobs .into_iter() - .map(BlockingBlobFile::from) - .collect::>(); + .map(|blob| blob.map(BlockingBlobFile::from)) + .collect::>(); transform_vec(env, j_blobs) } fn transform_vec<'local>( env: &mut JNIEnv<'local>, - vec: Vec, + vec: Vec>, ) -> Result> { let array_list_class = env.find_class("java/util/ArrayList")?; let array_list = env.new_object(array_list_class, "()V", &[])?; for blob_file in vec { - let blob_file_obj = blob_file.into_java(env)?; + let blob_file_obj = match blob_file { + Some(blob_file) => blob_file.into_java(env)?, + None => JObject::null(), + }; env.call_method( &array_list, "add", @@ -111,14 +114,12 @@ fn inner_take_blobs_by_indices<'local>( let blobs = { let dataset = unsafe { env.get_rust_field::<_, _, BlockingDataset>(jdataset, NATIVE_DATASET) }?; - RT.block_on( - Arc::new(dataset.inner.clone()).take_blobs_by_indices(&row_indices_u64, col_name), - )? + block_on(Arc::new(dataset.inner.clone()).take_blobs_by_indices(&row_indices_u64, col_name))? }; let j_blobs = blobs .into_iter() - .map(BlockingBlobFile::from) - .collect::>(); + .map(|blob| blob.map(BlockingBlobFile::from)) + .collect::>(); transform_vec(env, j_blobs) } @@ -137,7 +138,7 @@ pub extern "system" fn Java_org_lance_BlobFile_nativeRead( fn inner_blob_read<'local>(env: &mut JNIEnv<'local>, jblob: JObject) -> Result> { let bytes = { let blob = unsafe { env.get_rust_field::<_, _, BlockingBlobFile>(jblob, NATIVE_BLOB) }?; - RT.block_on(blob.inner.read())? + block_on(blob.inner.read())? }; let arr = env.new_byte_array(bytes.len() as jint)?; let u8_slice: &[u8] = bytes.as_ref(); @@ -167,7 +168,7 @@ fn inner_blob_read_up_to<'local>( ) -> Result> { let bytes = { let blob = unsafe { env.get_rust_field::<_, _, BlockingBlobFile>(jblob, NATIVE_BLOB) }?; - RT.block_on(blob.inner.read_up_to(len as usize))? + block_on(blob.inner.read_up_to(len as usize))? }; let arr = env.new_byte_array(bytes.len() as jint)?; let u8_slice: &[u8] = bytes.as_ref(); @@ -202,7 +203,7 @@ fn inner_blob_read_range<'local>( .ok_or_else(|| lance_core::Error::invalid_input("offset + len overflowed".to_string()))?; let bytes = { let blob = unsafe { env.get_rust_field::<_, _, BlockingBlobFile>(jblob, NATIVE_BLOB) }?; - RT.block_on(blob.inner.read_range(offset as u64..end))? + block_on(blob.inner.read_range(offset as u64..end))? }; let arr = env.new_byte_array(bytes.len() as jint)?; let u8_slice: &[u8] = bytes.as_ref(); @@ -223,7 +224,7 @@ pub extern "system" fn Java_org_lance_BlobFile_nativeSeek( fn inner_blob_seek(env: &mut JNIEnv, jblob: JObject, new_cursor: jlong) -> Result<()> { let blob = unsafe { env.get_rust_field::<_, _, BlockingBlobFile>(jblob, NATIVE_BLOB) }?; - RT.block_on(blob.inner.seek(new_cursor as u64))?; + block_on(blob.inner.seek(new_cursor as u64))?; Ok(()) } @@ -237,7 +238,7 @@ pub extern "system" fn Java_org_lance_BlobFile_nativeTell( fn inner_blob_tell(env: &mut JNIEnv, jblob: JObject) -> Result { let blob = unsafe { env.get_rust_field::<_, _, BlockingBlobFile>(jblob, NATIVE_BLOB) }?; - Ok(RT.block_on(blob.inner.tell())?) + Ok(block_on(blob.inner.tell())?) } #[unsafe(no_mangle)] @@ -260,6 +261,6 @@ pub extern "system" fn Java_org_lance_BlobFile_nativeClose(mut env: JNIEnv, jblo fn inner_blob_close(env: &mut JNIEnv, jblob: JObject) -> Result<()> { let blob = unsafe { env.take_rust_field::<_, _, BlockingBlobFile>(jblob, NATIVE_BLOB)? }; - RT.block_on(blob.inner.close())?; + block_on(blob.inner.close())?; Ok(()) } diff --git a/java/lance-jni/src/blocking_dataset.rs b/java/lance-jni/src/blocking_dataset.rs index caf837b371a..8bf5994e7a3 100644 --- a/java/lance-jni/src/blocking_dataset.rs +++ b/java/lance-jni/src/blocking_dataset.rs @@ -3,6 +3,7 @@ use crate::error::{Error, Result}; use crate::ffi::JNIEnvExt; +use crate::index_progress::JavaIndexBuildProgress; use crate::namespace::{ BlockingDirectoryNamespace, BlockingRestNamespace, create_java_lance_namespace, }; @@ -13,7 +14,7 @@ use crate::utils::{ extract_write_params, get_scalar_index_params, get_vector_index_params, to_java_map, to_rust_map, }; -use crate::{RT, traits::IntoJava}; +use crate::{block_on, traits::IntoJava}; use arrow::array::RecordBatchReader; use arrow::datatypes::Schema; use arrow::ffi::FFI_ArrowSchema; @@ -21,7 +22,6 @@ use arrow::ffi_stream::ArrowArrayStreamReader; use arrow::ffi_stream::FFI_ArrowArrayStream; use arrow::ipc::writer::StreamWriter; use arrow::record_batch::RecordBatchIterator; -use arrow_schema::DataType; use arrow_schema::Schema as ArrowSchema; use chrono::{DateTime, Utc}; use jni::objects::{JMap, JString, JValue}; @@ -41,7 +41,7 @@ use lance::dataset::{ ColumnAlteration, CommitBuilder, Dataset, NewColumnTransform, ProjectionRequest, ReadParams, Version, WriteParams, }; -use lance::index::{DatasetIndexExt, IndexSegment}; +use lance::index::{DatasetIndexExt, IndexSegment, IntoIndexSegment}; use lance::io::commit::namespace_manifest::LanceNamespaceExternalManifestStore; use lance::io::{ObjectStore, ObjectStoreParams}; use lance::session::Session as LanceSession; @@ -61,7 +61,6 @@ use lance_table::io::commit::external_manifest::ExternalManifestCommitHandler; use std::collections::HashMap; use std::future::IntoFuture; use std::iter::empty; -use std::str::FromStr; use std::sync::Arc; use std::time::{Duration, UNIX_EPOCH}; use uuid::Uuid; @@ -103,13 +102,13 @@ impl BlockingDataset { /// If a storage options provider was configured and credentials are expiring, /// this will refresh them. pub fn latest_storage_options(&self) -> Result>> { - RT.block_on(async { self.inner.latest_storage_options().await }) + block_on(async { self.inner.latest_storage_options().await }) .map(|opt| opt.map(|opts| opts.0)) .map_err(|e| Error::io_error(e.to_string())) } pub fn drop(uri: &str, storage_options: HashMap) -> Result<()> { - RT.block_on(async move { + block_on(async move { let registry = Arc::new(ObjectStoreRegistry::default()); let object_store_params = ObjectStoreParams { storage_options_accessor: Some(Arc::new( @@ -132,7 +131,7 @@ impl BlockingDataset { uri: &str, params: Option, ) -> Result { - let inner = RT.block_on(Dataset::write(reader, uri, params))?; + let inner = block_on(Dataset::write(reader, uri, params))?; Ok(Self { inner }) } @@ -212,7 +211,7 @@ impl BlockingDataset { builder = builder.with_commit_handler(commit_handler); } - let inner = RT.block_on(builder.load())?; + let inner = block_on(builder.load())?; Ok(Self { inner }) } @@ -229,7 +228,7 @@ impl BlockingDataset { lance::io::StorageOptionsAccessor::with_static_options(storage_options), )) }; - let inner = RT.block_on(Dataset::commit( + let inner = block_on(Dataset::commit( uri, operation, read_version, @@ -245,51 +244,55 @@ impl BlockingDataset { } pub fn latest_version(&self) -> Result { - let version = RT.block_on(self.inner.latest_version_id())?; + let version = block_on(self.inner.latest_version_id())?; Ok(version) } pub fn list_versions(&self) -> Result> { - let versions = RT.block_on(self.inner.versions())?; + let versions = block_on(self.inner.versions())?; Ok(versions) } + pub fn count_versions(&self) -> Result { + Ok(block_on(self.inner.count_versions())?) + } + pub fn version(&self) -> Result { Ok(self.inner.version()) } pub fn checkout_version(&mut self, version: u64) -> Result { - let inner = RT.block_on(self.inner.checkout_version(version))?; + let inner = block_on(self.inner.checkout_version(version))?; Ok(Self { inner }) } pub fn checkout_tag(&mut self, tag: &str) -> Result { - let inner = RT.block_on(self.inner.checkout_version(tag))?; + let inner = block_on(self.inner.checkout_version(tag))?; Ok(Self { inner }) } pub fn checkout_latest(&mut self) -> Result<()> { - RT.block_on(self.inner.checkout_latest())?; + block_on(self.inner.checkout_latest())?; Ok(()) } pub fn restore(&mut self) -> Result<()> { - RT.block_on(self.inner.restore())?; + block_on(self.inner.restore())?; Ok(()) } pub fn list_tags(&self) -> Result> { - let tags = RT.block_on(self.inner.tags().list())?; + let tags = block_on(self.inner.tags().list())?; Ok(tags) } pub fn list_branches(&self) -> Result> { - let branches = RT.block_on(self.inner.branches().list())?; + let branches = block_on(self.inner.branches().list())?; Ok(branches) } pub fn delete_branch(&mut self, branch: &str) -> Result<()> { - RT.block_on(self.inner.branches().delete(branch, true))?; + block_on(self.inner.branches().delete(branch, true))?; Ok(()) } @@ -304,22 +307,22 @@ impl BlockingDataset { } else { Ref::Version(branch, version) }; - let inner = RT.block_on(self.inner.checkout_version(reference))?; + let inner = block_on(self.inner.checkout_version(reference))?; Ok(Self { inner }) } pub fn create_tag(&mut self, tag: &str, reference: Ref) -> Result<()> { - RT.block_on(self.inner.tags().create(tag, reference))?; + block_on(self.inner.tags().create(tag, reference))?; Ok(()) } pub fn delete_tag(&mut self, tag: &str) -> Result<()> { - RT.block_on(self.inner.tags().delete(tag))?; + block_on(self.inner.tags().delete(tag))?; Ok(()) } pub fn update_tag(&mut self, tag: &str, reference: Ref) -> Result<()> { - RT.block_on(self.inner.tags().update(tag, reference))?; + block_on(self.inner.tags().update(tag, reference))?; Ok(()) } @@ -328,7 +331,7 @@ impl BlockingDataset { tag: &str, metadata: HashMap, ) -> Result<()> { - RT.block_on(self.inner.tags().replace_metadata(tag, metadata))?; + block_on(self.inner.tags().replace_metadata(tag, metadata))?; Ok(()) } @@ -337,27 +340,27 @@ impl BlockingDataset { branch: &str, metadata: HashMap, ) -> Result<()> { - RT.block_on(self.inner.branches().replace_metadata(branch, metadata))?; + block_on(self.inner.branches().replace_metadata(branch, metadata))?; Ok(()) } pub fn get_version(&self, tag: &str) -> Result { - let version = RT.block_on(self.inner.tags().get_version(tag))?; + let version = block_on(self.inner.tags().get_version(tag))?; Ok(version) } pub fn count_rows(&self, filter: Option) -> Result { - let rows = RT.block_on(self.inner.count_rows(filter))?; + let rows = block_on(self.inner.count_rows(filter))?; Ok(rows) } pub fn calculate_data_stats(&self) -> Result { - let stats = RT.block_on(Arc::new(self.clone().inner).calculate_data_stats())?; + let stats = block_on(Arc::new(self.clone().inner).calculate_data_stats())?; Ok(stats) } pub fn list_indexes(&self) -> Result>> { - let indexes = RT.block_on(self.inner.load_indices())?; + let indexes = block_on(self.inner.load_indices())?; Ok(indexes) } @@ -395,12 +398,12 @@ impl BlockingDataset { if let Some(handler) = commit_handler { builder = builder.with_commit_handler(handler); } - let new_dataset = RT.block_on(builder.execute(transaction))?; + let new_dataset = block_on(builder.execute(transaction))?; Ok(BlockingDataset { inner: new_dataset }) } pub fn read_transaction(&self) -> Result> { - let transaction = RT.block_on(self.inner.read_transaction())?; + let transaction = block_on(self.inner.read_transaction())?; Ok(transaction) } @@ -409,12 +412,12 @@ impl BlockingDataset { } pub fn compact(&mut self, options: RustCompactionOptions) -> Result<()> { - RT.block_on(compact_files(&mut self.inner, options, None))?; + block_on(compact_files(&mut self.inner, options, None))?; Ok(()) } pub fn cleanup_with_policy(&mut self, policy: CleanupPolicy) -> Result { - Ok(RT.block_on(self.inner.cleanup_with_policy(policy))?) + Ok(block_on(self.inner.cleanup_with_policy(policy))?) } pub fn explain_cleanup_with_policy( @@ -426,7 +429,7 @@ impl BlockingDataset { if let Some(limit) = max_candidate_files { op = op.with_max_candidate_files(limit); } - Ok(RT.block_on(op.explain())?) + Ok(block_on(op.explain())?) } pub fn close(&self) {} @@ -552,7 +555,7 @@ pub extern "system" fn Java_org_lance_Dataset_nativeMigrateManifestPathsV2( fn inner_native_migrate_manifest_paths_v2(env: &mut JNIEnv, java_dataset: JObject) -> Result<()> { let mut dataset_guard = unsafe { env.get_rust_field::<_, _, BlockingDataset>(java_dataset, NATIVE_DATASET) }?; - RT.block_on(dataset_guard.inner.migrate_manifest_paths_v2())?; + block_on(dataset_guard.inner.migrate_manifest_paths_v2())?; Ok(()) } @@ -1062,9 +1065,9 @@ fn inner_create_index<'local>( } if skip_commit { - RT.block_on(index_builder.execute_uncommitted())? + block_on(index_builder.execute_uncommitted())? } else { - RT.block_on(index_builder.into_future())? + block_on(index_builder.into_future())? } }; @@ -1084,7 +1087,7 @@ fn inner_drop_index(env: &mut JNIEnv, java_dataset: JObject, name: JString) -> R let name = name.extract(env)?; let mut dataset_guard = unsafe { env.get_rust_field::<_, _, BlockingDataset>(java_dataset, NATIVE_DATASET) }?; - RT.block_on(dataset_guard.inner.drop_index(&name))?; + block_on(dataset_guard.inner.drop_index(&name))?; Ok(()) } @@ -1115,6 +1118,92 @@ fn inner_merge_index_metadata( index_type_code_jobj: jint, batch_readhead_jobj: JObject, // Optional ) -> Result<()> { + let (index_uuid, index_type, batch_readhead) = parse_merge_index_metadata_args( + env, + index_uuid, + index_type_code_jobj, + batch_readhead_jobj, + )?; + + // Clone the inner Dataset out of the `get_rust_field` guard and drop the + // guard before the long-lived merge. Otherwise nested JNI callbacks that + // touch the same Dataset would deadlock on the native field mutex. + let inner_dataset = unsafe { + let dataset_guard = + env.get_rust_field::<_, _, BlockingDataset>(java_dataset, NATIVE_DATASET)?; + dataset_guard.inner.clone() + }; + + block_on(async { + inner_dataset + .merge_index_metadata(&index_uuid, index_type, batch_readhead, noop_progress()) + .await + })?; + Ok(()) +} + +#[unsafe(no_mangle)] +pub extern "system" fn Java_org_lance_Dataset_innerMergeIndexMetadataWithProgress<'local>( + mut env: JNIEnv<'local>, + java_dataset: JObject, + index_uuid: JString, + index_type_code_jobj: jint, + batch_readhead_jobj: JObject, + progress_jobj: JObject, +) { + ok_or_throw_without_return!( + env, + inner_merge_index_metadata_with_progress( + &mut env, + java_dataset, + index_uuid, + index_type_code_jobj, + batch_readhead_jobj, + progress_jobj, + ) + ); +} + +fn inner_merge_index_metadata_with_progress( + env: &mut JNIEnv, + java_dataset: JObject, + index_uuid: JString, + index_type_code_jobj: jint, + batch_readhead_jobj: JObject, + progress_jobj: JObject, +) -> Result<()> { + let (index_uuid, index_type, batch_readhead) = parse_merge_index_metadata_args( + env, + index_uuid, + index_type_code_jobj, + batch_readhead_jobj, + )?; + let progress = Arc::new(JavaIndexBuildProgress::new(env, &progress_jobj)?); + + // Clone the inner Dataset out of the `get_rust_field` guard and drop the + // guard before the long-lived merge. Progress callbacks are allowed to + // re-enter Dataset JNI methods; holding the guard across those callbacks + // would deadlock on the native field mutex (see update.rs). + let inner_dataset = unsafe { + let dataset_guard = + env.get_rust_field::<_, _, BlockingDataset>(java_dataset, NATIVE_DATASET)?; + dataset_guard.inner.clone() + }; + + block_on(async { + inner_dataset + .merge_index_metadata(&index_uuid, index_type, batch_readhead, progress) + .await + })?; + Ok(()) +} + +fn parse_merge_index_metadata_args( + env: &mut JNIEnv, + index_uuid: JString, + index_type_code_jobj: jint, + batch_readhead_jobj: JObject, +) -> Result<(Uuid, IndexType, Option)> { let index_uuid_str = index_uuid.extract(env)?; let index_uuid = Uuid::parse_str(&index_uuid_str) .map_err(|e| Error::input_error(format!("Invalid UUID string for index_uuid: {e}")))?; @@ -1122,17 +1211,7 @@ fn inner_merge_index_metadata( let batch_readhead = env .get_int_opt(&batch_readhead_jobj)? .map(|val| val as usize); - - let dataset_guard = - unsafe { env.get_rust_field::<_, _, BlockingDataset>(java_dataset, NATIVE_DATASET) }?; - - RT.block_on(async { - dataset_guard - .inner - .merge_index_metadata(&index_uuid, index_type, batch_readhead, noop_progress()) - .await - })?; - Ok(()) + Ok((index_uuid, index_type, batch_readhead)) } #[unsafe(no_mangle)] @@ -1156,7 +1235,7 @@ fn inner_merge_existing_index_segments<'local>( let merged_segment = { let dataset_guard = unsafe { env.get_rust_field::<_, _, BlockingDataset>(java_dataset, NATIVE_DATASET) }?; - RT.block_on(dataset_guard.inner.merge_existing_index_segments(segments))? + block_on(dataset_guard.inner.merge_existing_index_segments(segments))? }; (&merged_segment).into_java(env) } @@ -1200,37 +1279,19 @@ fn inner_commit_existing_index_segments<'local>( let committed = { let mut dataset_guard = unsafe { env.get_rust_field::<_, _, BlockingDataset>(java_dataset, NATIVE_DATASET) }?; - RT.block_on(dataset_guard.inner.commit_existing_index_segments( + block_on(dataset_guard.inner.commit_existing_index_segments( &index_name, &column, segments, ))?; - RT.block_on(dataset_guard.inner.load_indices_by_name(&index_name))? + block_on(dataset_guard.inner.load_indices_by_name(&index_name))? }; export_vec(env, &committed) } fn index_metadata_to_segment(metadata: &IndexMetadata) -> Result { - let fragment_bitmap = metadata.fragment_bitmap.clone().ok_or_else(|| { - Error::input_error(format!( - "Segment '{}' is missing fragment coverage metadata", - metadata.uuid - )) - })?; - let index_details = metadata.index_details.clone().ok_or_else(|| { - Error::input_error(format!( - "Segment '{}' is missing index details metadata", - metadata.uuid - )) - })?; - - Ok(IndexSegment::new( - metadata.uuid, - fragment_bitmap, - index_details, - metadata.index_version, - )) + Ok(metadata.clone().into_index_segment()?) } #[unsafe(no_mangle)] @@ -1277,7 +1338,7 @@ fn inner_optimize_indices( let mut dataset_guard = unsafe { env.get_rust_field::<_, _, BlockingDataset>(java_dataset, NATIVE_DATASET) }?; - RT.block_on(dataset_guard.inner.optimize_indices(&options))?; + block_on(dataset_guard.inner.optimize_indices(&options))?; Ok(()) } @@ -1466,6 +1527,47 @@ fn inner_get_fragments<'local>( export_vec(env, &fragments) } +#[unsafe(no_mangle)] +pub extern "system" fn Java_org_lance_Dataset_nativeGetFragmentStatistics<'a>( + mut env: JNIEnv<'a>, + jdataset: JObject, +) -> JObject<'a> { + ok_or_throw!(env, inner_get_fragment_statistics(&mut env, jdataset)) +} + +/// Returns per-fragment statistics flattened as [id0, rowCount0, dataFileNum0, id1, ...]. +/// +/// Row count semantics match Java `FragmentMetadata.getNumRows()`: +/// physical rows minus deleted rows, with absent values treated as 0. +/// Data file count is the number of data files in the fragment. +fn inner_get_fragment_statistics<'local>( + env: &mut JNIEnv<'local>, + jdataset: JObject, +) -> Result> { + let stats: Vec = { + let dataset = + unsafe { env.get_rust_field::<_, _, BlockingDataset>(jdataset, NATIVE_DATASET) }?; + let fragments = dataset.inner.get_fragments(); + let mut stats = Vec::with_capacity(fragments.len() * 3); + for f in fragments.iter() { + let meta = f.metadata(); + let physical_rows = meta.physical_rows.unwrap_or(0) as i64; + let deleted_rows = meta + .deletion_file + .as_ref() + .and_then(|d| d.num_deleted_rows) + .unwrap_or(0) as i64; + stats.push(f.id() as i64); + stats.push(physical_rows - deleted_rows); + stats.push(meta.files.len() as i64); + } + stats + }; + let jarray = env.new_long_array(stats.len() as i32)?; + env.set_long_array_region(&jarray, 0, &stats)?; + Ok(jarray.into()) +} + #[unsafe(no_mangle)] pub extern "system" fn Java_org_lance_Dataset_getFragmentNative<'a>( mut env: JNIEnv<'a>, @@ -1571,6 +1673,20 @@ pub extern "system" fn Java_org_lance_Dataset_nativeListVersions<'local>( ok_or_throw!(env, inner_list_versions(&mut env, java_dataset)) } +#[unsafe(no_mangle)] +pub extern "system" fn Java_org_lance_Dataset_nativeGetVersionCount( + mut env: JNIEnv, + java_dataset: JObject, +) -> jlong { + ok_or_throw_with_return!(env, inner_get_version_count(&mut env, java_dataset), -1) as jlong +} + +fn inner_get_version_count(env: &mut JNIEnv, java_dataset: JObject) -> Result { + let dataset_guard = + unsafe { env.get_rust_field::<_, _, BlockingDataset>(java_dataset, NATIVE_DATASET) }?; + dataset_guard.count_versions() +} + fn inner_list_versions<'local>( env: &mut JNIEnv<'local>, java_dataset: JObject, @@ -1793,7 +1909,7 @@ fn inner_shallow_clone<'local>( let new_ds = { let mut dataset_guard = unsafe { env.get_rust_field::<_, _, BlockingDataset>(java_dataset, NATIVE_DATASET) }?; - RT.block_on(dataset_guard.inner.shallow_clone( + block_on(dataset_guard.inner.shallow_clone( target_path_str.as_str(), reference, storage_opts, @@ -1976,12 +2092,13 @@ fn inner_get_lance_file_format_version<'local>( let version_string = { let dataset_guard = unsafe { env.get_rust_field::<_, _, BlockingDataset>(java_dataset, NATIVE_DATASET) }?; - let version = dataset_guard + dataset_guard .inner .manifest() .data_storage_format - .lance_file_version()?; - version.to_string() + .lance_file_format() + .to_manifest_string() + .to_string() }; Ok(env @@ -2023,7 +2140,7 @@ fn inner_take( let projection = ProjectionRequest::from_columns(columns, dataset.schema()); - match RT.block_on(dataset.take(indices_slice, projection)) { + match block_on(dataset.take(indices_slice, projection)) { Ok(res) => res, Err(e) => { return Err(e.into()); @@ -2075,7 +2192,7 @@ fn inner_take_rows( let projection = ProjectionRequest::from_columns(columns, dataset.schema()); - match RT.block_on(dataset.take_rows(&row_ids_u64, projection)) { + match block_on(dataset.take_rows(&row_ids_u64, projection)) { Ok(res) => res, Err(e) => { return Err(e.into()); @@ -2133,7 +2250,7 @@ fn inner_sample( .project_preserve_system_columns(&columns) .map_err(|e| Error::runtime_error(e.to_string()))?; - match RT.block_on(dataset.sample(n as usize, &projection, fragment_ids_u32.as_deref())) { + match block_on(dataset.sample(n as usize, &projection, fragment_ids_u32.as_deref())) { Ok(res) => res, Err(e) => { return Err(e.into()); @@ -2165,7 +2282,7 @@ fn inner_delete(env: &mut JNIEnv, java_dataset: JObject, predicate: JString) -> let predicate_str = predicate.extract(env)?; let mut dataset_guard = unsafe { env.get_rust_field::<_, _, BlockingDataset>(java_dataset, NATIVE_DATASET) }?; - RT.block_on(dataset_guard.inner.delete(&predicate_str))?; + block_on(dataset_guard.inner.delete(&predicate_str))?; Ok(()) } @@ -2180,7 +2297,7 @@ pub extern "system" fn Java_org_lance_Dataset_nativeTruncateTable( fn inner_truncate_table(env: &mut JNIEnv, java_dataset: JObject) -> Result<()> { let mut dataset_guard = unsafe { env.get_rust_field::<_, _, BlockingDataset>(java_dataset, NATIVE_DATASET) }?; - RT.block_on(dataset_guard.inner.truncate_table())?; + block_on(dataset_guard.inner.truncate_table())?; Ok(()) } @@ -2205,7 +2322,7 @@ fn inner_drop_columns( let columns_slice: Vec<&str> = columns.iter().map(AsRef::as_ref).collect(); let mut dataset_guard = unsafe { env.get_rust_field::<_, _, BlockingDataset>(java_dataset, NATIVE_DATASET) }?; - RT.block_on(dataset_guard.inner.drop_columns(&columns_slice))?; + block_on(dataset_guard.inner.drop_columns(&columns_slice))?; Ok(()) } @@ -2214,17 +2331,23 @@ pub extern "system" fn Java_org_lance_Dataset_nativeAlterColumns( mut env: JNIEnv, java_dataset: JObject, column_alterations_obj: JObject, // List + cast_schema_addr: jlong, ) { ok_or_throw_without_return!( env, - inner_alter_columns(&mut env, java_dataset, column_alterations_obj) + inner_alter_columns( + &mut env, + java_dataset, + column_alterations_obj, + cast_schema_addr + ) ) } fn create_column_alteration( env: &mut JNIEnv, column_alteration_jobj: JObject, // ColumnAlteration -) -> Result { +) -> Result<(ColumnAlteration, bool)> { let path_obj = env .get_field(&column_alteration_jobj, "path", "Ljava/lang/String;")? .l()?; @@ -2263,54 +2386,61 @@ fn create_column_alteration( None }; + // The cast target type (if any) is not read here: it is transferred separately through the + // Arrow C Data Interface (see inner_alter_columns), because ArrowType#toString() does not + // round-trip through DataType::from_str for parameterized types. This flag records whether a + // cast was requested so the caller can attach the imported type in order. let data_type_obj = env .get_field(&column_alteration_jobj, "dataType", "Ljava/util/Optional;")? .l()?; - let data_type = if env + let wants_cast = env .call_method(&data_type_obj, "isPresent", "()Z", &[])? - .z()? - { - let j_data_type: JObject = env - .call_method(data_type_obj, "get", "()Ljava/lang/Object;", &[])? - .l()?; - let jstring: JString = env - .call_method(j_data_type, "toString", "()Ljava/lang/String;", &[])? - .l()? - .into(); - let data_type_str: String = env.get_string(&jstring)?.into(); // Intermediate variable - DataType::from_str(&data_type_str) - .map_err(|e| Error::input_error(e.to_string())) - .ok() - } else { - None - }; + .z()?; - Ok(ColumnAlteration { + let alteration = ColumnAlteration { path, rename, nullable, - data_type, - }) + data_type: None, + }; + Ok((alteration, wants_cast)) } fn inner_alter_columns( env: &mut JNIEnv, java_dataset: JObject, column_alterations_obj: JObject, // List + cast_schema_addr: jlong, ) -> Result<()> { let list = env.get_list(&column_alterations_obj)?; let mut iter = list.iter(env)?; let mut column_alterations = Vec::new(); + let mut cast_flags = Vec::new(); while let Some(elem) = iter.next(env)? { - let alteration = create_column_alteration(env, elem)?; + let (alteration, wants_cast) = create_column_alteration(env, elem)?; column_alterations.push(alteration); + cast_flags.push(wants_cast); + } + + // Cast target types arrive as one Arrow schema field per requested cast, in the same order + // as the alterations that requested one. + let cast_schema = unsafe { FFI_ArrowSchema::from_raw(cast_schema_addr as *mut _) }; + let cast_schema = ArrowSchema::try_from(&cast_schema) + .map_err(|_| Error::input_error("ArrowSchema conversion error".to_string()))?; + let mut cast_types = cast_schema.fields.iter().map(|f| f.data_type().clone()); + for (alteration, wants_cast) in column_alterations.iter_mut().zip(cast_flags) { + if wants_cast { + alteration.data_type = Some(cast_types.next().ok_or_else(|| { + Error::input_error("Missing cast type for column alteration".to_string()) + })?); + } } let mut dataset_guard = unsafe { env.get_rust_field::<_, _, BlockingDataset>(java_dataset, NATIVE_DATASET) }?; - RT.block_on(dataset_guard.inner.alter_columns(&column_alterations))?; + block_on(dataset_guard.inner.alter_columns(&column_alterations))?; Ok(()) } @@ -2368,7 +2498,7 @@ fn inner_add_columns_by_sql_expressions( let mut dataset_guard = unsafe { env.get_rust_field::<_, _, BlockingDataset>(java_dataset, NATIVE_DATASET) }?; - RT.block_on( + block_on( dataset_guard .inner .add_columns(rust_transform, None, batch_size), @@ -2413,7 +2543,7 @@ fn inner_add_columns_by_reader( let mut dataset_guard = unsafe { env.get_rust_field::<_, _, BlockingDataset>(java_dataset, NATIVE_DATASET) }?; - RT.block_on(dataset_guard.inner.add_columns(transform, None, batch_size))?; + block_on(dataset_guard.inner.add_columns(transform, None, batch_size))?; Ok(()) } @@ -2444,7 +2574,7 @@ fn inner_add_columns_by_schema( let mut dataset_guard = unsafe { env.get_rust_field::<_, _, BlockingDataset>(java_dataset, NATIVE_DATASET) }?; - RT.block_on(dataset_guard.inner.add_columns(transform, None, None))?; + block_on(dataset_guard.inner.add_columns(transform, None, None))?; Ok(()) } @@ -2744,7 +2874,7 @@ fn inner_create_branch<'local>( let new_blocking_dataset = { let mut dataset_guard = unsafe { env.get_rust_field::<_, _, BlockingDataset>(java_dataset, NATIVE_DATASET) }?; - let inner = RT.block_on(dataset_guard.inner.create_branch( + let inner = block_on(dataset_guard.inner.create_branch( branch_name.as_str(), reference, storage_opts, @@ -3367,7 +3497,7 @@ fn inner_get_index_statistics<'local>( let stats_json = { let dataset_guard = unsafe { env.get_rust_field::<_, _, BlockingDataset>(java_dataset, NATIVE_DATASET) }?; - RT.block_on(dataset_guard.inner.index_statistics(&index_name))? + block_on(dataset_guard.inner.index_statistics(&index_name))? }; let jstats = env.new_string(stats_json)?; Ok(jstats) @@ -3402,6 +3532,7 @@ fn inner_describe_indices<'local>( for_column: for_column.as_deref(), has_name: has_name.as_deref(), must_support_fts, + fts_document_granularity: None, must_support_exact_equality, }) })?; @@ -3409,7 +3540,7 @@ fn inner_describe_indices<'local>( let descriptions = { let dataset_guard = unsafe { env.get_rust_field::<_, _, BlockingDataset>(java_dataset, NATIVE_DATASET) }?; - RT.block_on(dataset_guard.inner.describe_indices(index_criteria))? + block_on(dataset_guard.inner.describe_indices(index_criteria))? }; export_vec(env, &descriptions) @@ -3467,7 +3598,7 @@ fn inner_count_indexed_rows( // This ensures we only count rows in the specified fragments let inner = dataset_guard.inner.clone(); - RT.block_on(async { + block_on(async { let mut scanner = inner.scan(); // Apply filter @@ -3572,13 +3703,14 @@ fn inner_get_zonemap_stats<'local>( })?; // Do all async work in a single block_on call to avoid nested runtime issues - RT.block_on(async { + block_on(async { // Find the zonemap index for this column using describe_indices let descriptions = dataset .describe_indices(Some(lance_index::IndexCriteria { for_column: Some(&column_name), has_name: None, must_support_fts: false, + fts_document_granularity: None, must_support_exact_equality: false, })) .await diff --git a/java/lance-jni/src/blocking_scanner.rs b/java/lance-jni/src/blocking_scanner.rs index 335cb2a4fa3..6a97d23e029 100644 --- a/java/lance-jni/src/blocking_scanner.rs +++ b/java/lance-jni/src/blocking_scanner.rs @@ -15,19 +15,22 @@ use jni::sys::{JNI_TRUE, jboolean, jint}; use jni::{JNIEnv, sys::jlong}; use lance::dataset::scanner::{ AggregateExpr, ColumnOrdering, DatasetRecordBatchStream, ExecutionStatsCallback, - ExecutionSummaryCounts, Scanner, + ExecutionSummaryCounts, MaterializationStyle, Scanner, }; use lance_index::scalar::FullTextSearchQuery; -use lance_index::scalar::inverted::query::{ - BooleanQuery as FtsBooleanQuery, BoostQuery as FtsBoostQuery, FtsQuery, - MatchQuery as FtsMatchQuery, MultiMatchQuery as FtsMultiMatchQuery, Occur as FtsOccur, - PhraseQuery as FtsPhraseQuery, +use lance_index::scalar::inverted::{ + DocumentGranularity, + query::{ + BooleanQuery as FtsBooleanQuery, BoostQuery as FtsBoostQuery, FtsQuery, + MatchQuery as FtsMatchQuery, MultiMatchQuery as FtsMultiMatchQuery, Occur as FtsOccur, + PhraseQuery as FtsPhraseQuery, + }, }; use lance_io::ffi::to_ffi_arrow_array_stream; use lance_linalg::distance::DistanceType; use crate::{ - RT, + RT, block_on, blocking_dataset::{BlockingDataset, NATIVE_DATASET}, traits::IntoJava, utils::parse_approx_mode, @@ -66,17 +69,17 @@ impl BlockingScanner { pub fn open_stream(&self) -> Result { self.reset_stats(); - let res = RT.block_on(self.inner.try_into_stream())?; + let res = block_on(self.inner.try_into_stream())?; Ok(res) } pub fn schema(&self) -> Result { - let res = RT.block_on(self.inner.schema())?; + let res = block_on(self.inner.schema())?; Ok(res) } pub fn count_rows(&self) -> Result { - let res = RT.block_on(self.inner.count_rows())?; + let res = block_on(self.inner.count_rows())?; Ok(res) } @@ -114,6 +117,7 @@ pub(crate) fn build_full_text_search_query<'a>( let max_expansions = env.get_int_as_usize_from_method(&java_obj, "getMaxExpansions")?; let operator = env.get_fts_operator_from_method(&java_obj)?; let prefix_length = env.get_u32_from_method(&java_obj, "getPrefixLength")?; + let document_granularity = get_document_granularity(env, &java_obj)?; let mut query = FtsMatchQuery::new(query_text); query = query.with_column(Some(column)); @@ -123,6 +127,9 @@ pub(crate) fn build_full_text_search_query<'a>( .with_max_expansions(max_expansions) .with_operator(operator) .with_prefix_length(prefix_length); + if let Some(document_granularity) = document_granularity { + query = query.with_document_granularity(document_granularity); + } Ok(FtsQuery::Match(query)) } @@ -130,10 +137,14 @@ pub(crate) fn build_full_text_search_query<'a>( let query_text = env.get_string_from_method(&java_obj, "getQueryText")?; let column = env.get_string_from_method(&java_obj, "getColumn")?; let slop = env.get_u32_from_method(&java_obj, "getSlop")?; + let document_granularity = get_document_granularity(env, &java_obj)?; let mut query = FtsPhraseQuery::new(query_text); query = query.with_column(Some(column)); query = query.with_slop(slop); + if let Some(document_granularity) = document_granularity { + query = query.with_document_granularity(document_granularity); + } Ok(FtsQuery::Phrase(query)) } @@ -229,6 +240,20 @@ pub(crate) fn build_full_text_search_query<'a>( } } +fn get_document_granularity( + env: &mut JNIEnv<'_>, + java_obj: &JObject, +) -> Result> { + env.get_optional_from_method( + java_obj, + "getDocumentGranularity", + |env, granularity_obj| { + let value = env.get_string_from_method(&granularity_obj, "toRustString")?; + DocumentGranularity::try_from(value.as_str()).map_err(Error::from) + }, + ) +} + /// Scanner options passed from JNI - shared between blocking and async scanners pub(crate) struct ScannerOptions<'a> { pub fragment_ids_obj: JObject<'a>, @@ -236,6 +261,8 @@ pub(crate) struct ScannerOptions<'a> { pub substrait_filter_obj: JObject<'a>, pub filter_obj: JObject<'a>, pub batch_size_obj: JObject<'a>, + pub batch_size_bytes_obj: JObject<'a>, + pub io_buffer_size_obj: JObject<'a>, pub limit_obj: JObject<'a>, pub offset_obj: JObject<'a>, pub query_obj: JObject<'a>, @@ -244,6 +271,9 @@ pub(crate) struct ScannerOptions<'a> { pub with_row_id: jboolean, pub with_row_address: jboolean, pub batch_readahead: jint, + pub fragment_readahead_obj: JObject<'a>, + pub scan_in_order: jboolean, + pub late_materialization_obj: JObject<'a>, pub column_orderings: JObject<'a>, pub use_scalar_index: jboolean, pub fast_search: jboolean, @@ -283,7 +313,7 @@ pub(crate) fn build_scanner_with_options<'a>( let substrait_opt = env.get_bytes_opt(&options.substrait_filter_obj)?; if let Some(substrait) = substrait_opt { - RT.block_on(async { scanner.filter_substrait(substrait) })?; + block_on(async { scanner.filter_substrait(substrait) })?; } let filter_opt = env.get_string_opt(&options.filter_obj)?; @@ -296,6 +326,26 @@ pub(crate) fn build_scanner_with_options<'a>( scanner.batch_size(batch_size as usize); } + let batch_size_bytes_opt = env.get_long_opt(&options.batch_size_bytes_obj)?; + if let Some(batch_size_bytes) = batch_size_bytes_opt { + let batch_size_bytes = u64::try_from(batch_size_bytes).map_err(|_| { + Error::input_error(format!( + "batchSizeBytes must be non-negative, got {batch_size_bytes}" + )) + })?; + scanner.batch_size_bytes(batch_size_bytes); + } + + let io_buffer_size_opt = env.get_long_opt(&options.io_buffer_size_obj)?; + if let Some(io_buffer_size) = io_buffer_size_opt { + let io_buffer_size = u64::try_from(io_buffer_size).map_err(|_| { + Error::input_error(format!( + "ioBufferSize must be non-negative, got {io_buffer_size}" + )) + })?; + scanner.io_buffer_size(io_buffer_size); + } + let limit_opt = env.get_long_opt(&options.limit_obj)?; let offset_opt = env.get_long_opt(&options.offset_obj)?; scanner @@ -377,6 +427,47 @@ pub(crate) fn build_scanner_with_options<'a>( scanner.batch_readahead(options.batch_readahead as usize); + let fragment_readahead_opt = env.get_int_opt(&options.fragment_readahead_obj)?; + if let Some(fragment_readahead) = fragment_readahead_opt { + let fragment_readahead = usize::try_from(fragment_readahead).map_err(|_| { + Error::input_error(format!( + "fragmentReadahead must be greater than 0, got {fragment_readahead}" + )) + })?; + if fragment_readahead == 0 { + return Err(Error::input_error( + "fragmentReadahead must be greater than 0, got 0".to_string(), + )); + } + scanner.fragment_readahead(fragment_readahead); + } + + scanner.scan_in_order(options.scan_in_order == JNI_TRUE); + + env.get_optional(&options.late_materialization_obj, |env, java_obj| { + let style_name = env.get_string_from_method(&java_obj, "toRustString")?; + let style = match style_name.as_str() { + "heuristic" => MaterializationStyle::Heuristic, + "all_late" => MaterializationStyle::AllLate, + "all_early" => MaterializationStyle::AllEarly, + "all_early_except" => { + let columns: Vec = + import_vec_from_method(env, &java_obj, "getColumns", |env, elem| { + let jstr = JString::from(elem); + Ok(env.get_string(&jstr)?.into()) + })?; + MaterializationStyle::all_early_except(&columns, dataset.schema())? + } + other => { + return Err(Error::input_error(format!( + "Unsupported materialization style: {other}" + ))); + } + }; + scanner.materialization_style(style); + Ok(()) + })?; + env.get_optional(&options.column_orderings, |env, java_obj| { let list = env.get_list(&java_obj)?; let mut iter = list.iter(env)?; @@ -427,6 +518,8 @@ pub extern "system" fn Java_org_lance_ipc_LanceScanner_createScanner<'local>( substrait_filter_obj: JObject<'local>, // Optional filter_obj: JObject<'local>, // Optional batch_size_obj: JObject<'local>, // Optional + batch_size_bytes_obj: JObject<'local>, // Optional + io_buffer_size_obj: JObject<'local>, // Optional limit_obj: JObject<'local>, // Optional offset_obj: JObject<'local>, // Optional query_obj: JObject<'local>, // Optional @@ -435,6 +528,9 @@ pub extern "system" fn Java_org_lance_ipc_LanceScanner_createScanner<'local>( with_row_id: jboolean, // boolean with_row_address: jboolean, // boolean batch_readahead: jint, // int + fragment_readahead_obj: JObject<'local>, // Optional + scan_in_order: jboolean, // boolean + late_materialization_obj: JObject<'local>, // Optional column_orderings: JObject<'local>, // Optional> use_scalar_index: jboolean, // boolean fast_search: jboolean, // boolean @@ -454,6 +550,8 @@ pub extern "system" fn Java_org_lance_ipc_LanceScanner_createScanner<'local>( substrait_filter_obj, filter_obj, batch_size_obj, + batch_size_bytes_obj, + io_buffer_size_obj, limit_obj, offset_obj, query_obj, @@ -462,6 +560,9 @@ pub extern "system" fn Java_org_lance_ipc_LanceScanner_createScanner<'local>( with_row_id, with_row_address, batch_readahead, + fragment_readahead_obj, + scan_in_order, + late_materialization_obj, column_orderings, use_scalar_index, fast_search, @@ -483,6 +584,8 @@ fn inner_create_scanner<'local>( substrait_filter_obj: JObject<'local>, filter_obj: JObject<'local>, batch_size_obj: JObject<'local>, + batch_size_bytes_obj: JObject<'local>, + io_buffer_size_obj: JObject<'local>, limit_obj: JObject<'local>, offset_obj: JObject<'local>, query_obj: JObject<'local>, @@ -491,6 +594,9 @@ fn inner_create_scanner<'local>( with_row_id: jboolean, with_row_address: jboolean, batch_readahead: jint, + fragment_readahead_obj: JObject<'local>, + scan_in_order: jboolean, + late_materialization_obj: JObject<'local>, column_orderings: JObject<'local>, use_scalar_index: jboolean, fast_search: jboolean, @@ -511,6 +617,8 @@ fn inner_create_scanner<'local>( substrait_filter_obj, filter_obj, batch_size_obj, + batch_size_bytes_obj, + io_buffer_size_obj, limit_obj, offset_obj, query_obj, @@ -519,6 +627,9 @@ fn inner_create_scanner<'local>( with_row_id, with_row_address, batch_readahead, + fragment_readahead_obj, + scan_in_order, + late_materialization_obj, column_orderings, use_scalar_index, fast_search, @@ -637,7 +748,7 @@ fn inner_count_rows(env: &mut JNIEnv, j_scanner: JObject) -> Result { } const SCAN_STATS_CLASS: &str = "org/lance/ipc/ScanStats"; -const SCAN_STATS_CONSTRUCTOR_SIG: &str = "(JJJJJJLjava/util/Map;Ljava/util/Map;)V"; +const SCAN_STATS_CONSTRUCTOR_SIG: &str = "(JJJJJJJJLjava/util/Map;Ljava/util/Map;)V"; fn export_usize_map<'a>(env: &mut JNIEnv<'a>, map: &HashMap) -> Result> { let hash_map = env.new_object("java/util/HashMap", "()V", &[])?; @@ -669,6 +780,8 @@ impl IntoJava for &ExecutionSummaryCounts { JValueGen::Long(self.indices_loaded as i64), JValueGen::Long(self.parts_loaded as i64), JValueGen::Long(self.index_comparisons as i64), + JValueGen::Long(self.index_cache_hits() as i64), + JValueGen::Long(self.index_cache_misses() as i64), JValueGen::Object(&all_counts), JValueGen::Object(&all_times), ], diff --git a/java/lance-jni/src/delta.rs b/java/lance-jni/src/delta.rs index d5a6b0f3a27..511d9be3bfc 100755 --- a/java/lance-jni/src/delta.rs +++ b/java/lance-jni/src/delta.rs @@ -1,11 +1,11 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright The Lance Authors -use crate::RT; use crate::blocking_dataset::{BlockingDataset, NATIVE_DATASET}; use crate::error::Result; use crate::ffi::JNIEnvExt; use crate::transaction::convert_to_java_transaction; +use crate::{RT, block_on}; use arrow::ffi_stream::FFI_ArrowArrayStream; use jni::JNIEnv; use jni::objects::{JObject, JValue}; @@ -123,7 +123,7 @@ fn inner_list_transactions<'local>( let txs: Vec = { let delta_guard = unsafe { env.get_rust_field::<_, _, BlockingDatasetDelta>(&j_delta, NATIVE_DELTA) }?; - RT.block_on(delta_guard.inner.list_transactions())? + block_on(delta_guard.inner.list_transactions())? }; let array_list = env.new_object("java/util/ArrayList", "()V", &[])?; @@ -156,7 +156,7 @@ fn inner_get_inserted_rows<'local>( let delta_guard = unsafe { env.get_rust_field::<_, _, BlockingDatasetDelta>(&j_delta, NATIVE_DELTA) }?; - let stream: DatasetRecordBatchStream = RT.block_on(delta_guard.inner.get_inserted_rows())?; + let stream: DatasetRecordBatchStream = block_on(delta_guard.inner.get_inserted_rows())?; let ffi_stream = to_ffi_arrow_array_stream(stream, RT.handle().clone())?; unsafe { std::ptr::write_unaligned(stream_addr as *mut FFI_ArrowArrayStream, ffi_stream) } @@ -180,7 +180,7 @@ fn inner_get_updated_rows<'local>( let delta_guard = unsafe { env.get_rust_field::<_, _, BlockingDatasetDelta>(&j_delta, NATIVE_DELTA) }?; - let stream: DatasetRecordBatchStream = RT.block_on(delta_guard.inner.get_updated_rows())?; + let stream: DatasetRecordBatchStream = block_on(delta_guard.inner.get_updated_rows())?; let ffi_stream = to_ffi_arrow_array_stream(stream, RT.handle().clone())?; unsafe { std::ptr::write_unaligned(stream_addr as *mut FFI_ArrowArrayStream, ffi_stream) } diff --git a/java/lance-jni/src/error.rs b/java/lance-jni/src/error.rs index cdb922a3cef..0affdca8f98 100644 --- a/java/lance-jni/src/error.rs +++ b/java/lance-jni/src/error.rs @@ -181,29 +181,36 @@ impl std::fmt::Display for Error { impl From for Error { fn from(err: LanceError) -> Self { + let backtrace_suffix = err + .backtrace() + .map(|bt| format!("\n\nRust backtrace:\n{}", bt)) + .unwrap_or_default(); + let message = format!("{}{}", err, backtrace_suffix); + match &err { LanceError::DatasetNotFound { .. } | LanceError::DatasetAlreadyExists { .. } | LanceError::CommitConflict { .. } - | LanceError::InvalidInput { .. } => Self::input_error(err.to_string()), - LanceError::IO { .. } => Self::io_error(err.to_string()), - LanceError::Timeout { .. } => Self::timeout_error(err.to_string()), - LanceError::NotSupported { .. } => Self::unsupported_error(err.to_string()), - LanceError::NotFound { .. } => Self::io_error(err.to_string()), + | LanceError::InvalidInput { .. } => Self::input_error(message), + LanceError::IO { .. } => Self::io_error(message), + LanceError::Timeout { .. } => Self::timeout_error(message), + LanceError::NotSupported { .. } => Self::unsupported_error(message), + LanceError::NotFound { .. } => Self::io_error(message), LanceError::Namespace { source, .. } => { // Try to downcast to NamespaceError and get the error code if let Some(ns_err) = source.downcast_ref::() { - Self::namespace_error(ns_err.code().as_u32(), ns_err.to_string()) + let ns_message = format!("{}{}", ns_err, backtrace_suffix); + Self::namespace_error(ns_err.code().as_u32(), ns_message) } else { log::warn!( "Failed to downcast NamespaceError source, falling back to runtime error. \ This may indicate a version mismatch. Source type: {:?}", source ); - Self::runtime_error(err.to_string()) + Self::runtime_error(message) } } - _ => Self::runtime_error(err.to_string()), + _ => Self::runtime_error(message), } } } @@ -241,3 +248,104 @@ impl From for Error { Self::input_error(err.to_string()) } } + +#[cfg(test)] +mod tests { + use super::*; + + // Helper: extract the java_class from an Error via Display output + fn java_class(err: &Error) -> &JavaExceptionClass { + &err.java_class + } + + #[test] + fn test_invalid_input_maps_to_illegal_argument() { + let lance_err = LanceError::invalid_input("bad input"); + let jni_err: Error = lance_err.into(); + assert_eq!( + *java_class(&jni_err), + JavaExceptionClass::IllegalArgumentException + ); + assert!(jni_err.message.contains("bad input")); + } + + #[test] + fn test_dataset_not_found_maps_to_illegal_argument() { + let lance_err = LanceError::dataset_not_found("my_dataset", "not found".to_string().into()); + let jni_err: Error = lance_err.into(); + assert_eq!( + *java_class(&jni_err), + JavaExceptionClass::IllegalArgumentException + ); + assert!(jni_err.message.contains("my_dataset")); + } + + #[test] + fn test_dataset_already_exists_maps_to_illegal_argument() { + let lance_err = LanceError::dataset_already_exists("my_dataset"); + let jni_err: Error = lance_err.into(); + assert_eq!( + *java_class(&jni_err), + JavaExceptionClass::IllegalArgumentException + ); + assert!(jni_err.message.contains("my_dataset")); + } + + #[test] + fn test_commit_conflict_maps_to_illegal_argument() { + let lance_err = LanceError::commit_conflict_source(42, "conflict".to_string().into()); + let jni_err: Error = lance_err.into(); + assert_eq!( + *java_class(&jni_err), + JavaExceptionClass::IllegalArgumentException + ); + } + + #[test] + fn test_io_maps_to_ioexception() { + let lance_err = LanceError::io("disk failure"); + let jni_err: Error = lance_err.into(); + assert_eq!(*java_class(&jni_err), JavaExceptionClass::IOException); + assert!(jni_err.message.contains("disk failure")); + } + + #[test] + fn test_not_supported_maps_to_unsupported() { + let lance_err = LanceError::not_supported("nope"); + let jni_err: Error = lance_err.into(); + assert_eq!( + *java_class(&jni_err), + JavaExceptionClass::UnsupportedOperationException + ); + assert!(jni_err.message.contains("nope")); + } + + #[test] + fn test_not_found_maps_to_ioexception() { + let lance_err = LanceError::not_found("missing_uri"); + let jni_err: Error = lance_err.into(); + assert_eq!(*java_class(&jni_err), JavaExceptionClass::IOException); + assert!(jni_err.message.contains("missing_uri")); + } + + #[test] + fn test_fallthrough_maps_to_runtime() { + let lance_err = LanceError::internal("internal oops"); + let jni_err: Error = lance_err.into(); + assert_eq!(*java_class(&jni_err), JavaExceptionClass::RuntimeException); + assert!(jni_err.message.contains("internal oops")); + } + + #[test] + fn test_no_backtrace_suffix_when_backtrace_is_none() { + // Without the backtrace feature enabled in lance-core default tests, + // backtrace() returns None, so no suffix should be appended. + let lance_err = LanceError::io("clean message"); + let jni_err: Error = lance_err.into(); + assert!( + !jni_err.message.contains("Rust backtrace:"), + "Expected no backtrace suffix, got: {}", + jni_err.message + ); + } +} diff --git a/java/lance-jni/src/file_reader.rs b/java/lance-jni/src/file_reader.rs index 3df9766d066..35d2ecd8f77 100644 --- a/java/lance-jni/src/file_reader.rs +++ b/java/lance-jni/src/file_reader.rs @@ -7,7 +7,7 @@ use std::sync::{Arc, Mutex}; use crate::utils::to_rust_map; use crate::{ - JNIEnvExt, RT, + JNIEnvExt, RT, block_on, error::{Error, Result}, traits::IntoJava, }; @@ -23,8 +23,8 @@ use lance::io::ObjectStore; use lance_core::cache::LanceCache; use lance_core::datatypes::{BlobHandling, OnMissing, Projection, Schema}; use lance_encoding::decoder::{DecoderPlugins, FilterExpression}; -use lance_encoding::version::LanceFileVersion; use lance_file::reader::{FileReader, FileReaderOptions, ReaderProjection}; +use lance_file::versions as file_versions; use lance_io::object_store::{ObjectStoreParams, ObjectStoreRegistry}; use lance_io::{ ReadBatchParams, @@ -53,16 +53,15 @@ impl BlockingFileReader { filter_expression: FilterExpression, ) -> Result> { let reader = self.inner.clone(); - Ok(RT - .block_on(RT.spawn_blocking(move || { - reader.read_stream_projected_blocking( - read_batch_params, - batch_size, - reader_projection, - filter_expression, - ) - })) - .unwrap()?) + Ok(block_on(RT.spawn_blocking(move || { + reader.read_stream_projected_blocking( + read_batch_params, + batch_size, + reader_projection, + filter_expression, + ) + })) + .unwrap()?) } pub fn schema(&self) -> Result { @@ -112,7 +111,7 @@ fn inner_open<'local>( let file_uri_str: String = env.get_string(&file_uri)?.into(); let jmap = JMap::from_env(env, &storage_options_obj)?; let storage_options = to_rust_map(env, &jmap)?; - let reader = RT.block_on(async move { + let reader = block_on(async move { let object_params = ObjectStoreParams { storage_options_accessor: Some(Arc::new( lance::io::StorageOptionsAccessor::with_static_options(storage_options), @@ -262,18 +261,17 @@ pub extern "system" fn Java_org_lance_file_LanceFileReader_readAllNative( let transformed_schema = projection.to_bare_schema(); - let field_id_to_column_index = base_schema - .fields_pre_order() - .filter(|field| { - file_version < LanceFileVersion::V2_1 - || field.is_leaf() - || field.is_packed_struct() + let (field_ids, column_indices) = + file_versions::data_file_columns(file_version, &base_schema); + let field_id_to_column_index = field_ids + .into_iter() + .zip(column_indices) + .filter_map(|(field_id, column_index)| { + (column_index >= 0).then_some((field_id as u32, column_index as u32)) }) - .enumerate() - .map(|(idx, field)| (field.id as u32, idx as u32)) .collect::>(); - Some(ReaderProjection::from_field_ids( + Some(file_versions::reader_projection_from_field_ids( file_version, &transformed_schema, &field_id_to_column_index, diff --git a/java/lance-jni/src/file_writer.rs b/java/lance-jni/src/file_writer.rs index 40b48bd686b..c8cffe8b642 100644 --- a/java/lance-jni/src/file_writer.rs +++ b/java/lance-jni/src/file_writer.rs @@ -5,7 +5,7 @@ use std::sync::{Arc, Mutex}; use crate::utils::to_rust_map; use crate::{ - JNIEnvExt, RT, + JNIEnvExt, block_on, error::{Error, Result}, traits::IntoJava, }; @@ -23,6 +23,7 @@ use jni::{ use lance::io::ObjectStore; use lance_file::{ version::LanceFileVersion, + versions as file_versions, writer::{FileWriter, FileWriterOptions}, }; use lance_io::object_store::{ObjectStoreParams, ObjectStoreRegistry}; @@ -92,7 +93,7 @@ fn inner_open<'local>( let data_storage_version_opt = env.get_string_opt(&data_storage_version)?; let storage_options = to_rust_map(env, &jmap)?; - let writer = RT.block_on(async move { + let writer = block_on(async move { let object_params = ObjectStoreParams { storage_options_accessor: Some(Arc::new( lance::io::StorageOptionsAccessor::with_static_options(storage_options), @@ -108,15 +109,12 @@ fn inner_open<'local>( let obj_store = Arc::new(obj_store); let obj_writer = obj_store.create(&path).await?; - Result::Ok(FileWriter::new_lazy( - obj_writer, - FileWriterOptions { - format_version: data_storage_version_opt - .map(|v| v.parse::()) - .transpose()?, - ..Default::default() - }, - )) + let version = data_storage_version_opt + .map(|value| value.parse::()) + .transpose()? + .unwrap_or_default() + .resolve(); + file_versions::create_lazy_writer(version, obj_writer, FileWriterOptions::default()) })?; let writer = BlockingFileWriter::create(writer); @@ -141,7 +139,7 @@ pub extern "system" fn Java_org_lance_file_LanceFileWriter_closeNative<'local>( } }; if let Some(writer) = writer { - match RT.block_on(writer.inner.lock().unwrap().finish()) { + match block_on(writer.inner.lock().unwrap().finish()) { Ok(_) => {} Err(e) => { Error::from(e).throw(&mut env); @@ -213,6 +211,6 @@ fn inner_write_batch( let writer = unsafe { env.get_rust_field::<_, _, BlockingFileWriter>(writer, NATIVE_WRITER) }?; let mut writer = writer.inner.lock().unwrap(); - RT.block_on(writer.write_batch(&record_batch))?; + block_on(writer.write_batch(&record_batch))?; Ok(()) } diff --git a/java/lance-jni/src/fragment.rs b/java/lance-jni/src/fragment.rs index d6603925947..69cbf04a013 100644 --- a/java/lance-jni/src/fragment.rs +++ b/java/lance-jni/src/fragment.rs @@ -5,7 +5,7 @@ use arrow::array::{RecordBatch, RecordBatchIterator, StructArray}; use arrow::ffi::{FFI_ArrowArray, FFI_ArrowSchema, from_ffi_and_data_type}; use arrow::ffi_stream::{ArrowArrayStreamReader, FFI_ArrowArrayStream}; use arrow_schema::{DataType, Schema as ArrowSchema}; -use jni::objects::{JIntArray, JValue, JValueGen}; +use jni::objects::{JByteArray, JIntArray, JValue, JValueGen}; use jni::{ JNIEnv, objects::{JClass, JLongArray, JObject, JString}, @@ -19,6 +19,8 @@ use lance_io::utils::CachedFileSize; use lance_table::rowids::{RowIdSequence, write_row_ids}; use std::iter::once; +use roaring::RoaringBitmap; + use lance::dataset::fragment::write::FragmentCreateBuilder; use lance::io::ObjectStoreParams; use lance_datafusion::utils::StreamingWriteSource; @@ -29,10 +31,11 @@ use std::sync::Arc; use crate::blocking_dataset::extract_namespace_info; use crate::error::{Error, Result}; use crate::ffi::JNIEnvExt; +use crate::session::session_from_handle; use crate::traits::{FromJObjectWithEnv, IntoJava, JLance, export_vec, import_vec}; use crate::utils::extract_storage_options; use crate::{ - RT, + block_on, blocking_dataset::{BlockingDataset, NATIVE_DATASET}, traits::FromJString, utils::extract_write_params, @@ -48,8 +51,8 @@ pub(crate) struct FragmentMergeResult { pub(crate) struct FragmentUpdateResult { updated_fragment: Fragment, fields_modified: Vec, - /// Physical row offsets that received column updates (from `_rowaddr` low bits). - updated_row_offsets: Vec, + /// Matched row offsets serialized as portable RoaringBitmap bytes. + updated_row_offset_bytes: Vec, } ////////////////// @@ -80,7 +83,7 @@ fn inner_count_rows_native( "Fragment not found: {fragment_id}" ))); }; - let res = RT.block_on(fragment.count_rows(None))?; + let res = block_on(fragment.count_rows(None))?; Ok(res) } @@ -109,6 +112,7 @@ pub extern "system" fn Java_org_lance_Fragment_createWithFfiArray<'local>( allow_external_blob_outside_bases: JObject, // Optional blob_pack_file_size_threshold: JObject, // Optional schema_addr: jlong, + session_handle: jlong, // Session handle, 0 means no session ) -> JObject<'local> { ok_or_throw_with_return!( env, @@ -132,6 +136,7 @@ pub extern "system" fn Java_org_lance_Fragment_createWithFfiArray<'local>( allow_external_blob_outside_bases, blob_pack_file_size_threshold, schema_addr, + session_handle, ), JObject::default() ) @@ -158,6 +163,7 @@ fn inner_create_with_ffi_array<'local>( allow_external_blob_outside_bases: JObject, // Optional blob_pack_file_size_threshold: JObject, // Optional schema_addr: jlong, + session_handle: jlong, // Session handle, 0 means no session ) -> Result> { let c_array_ptr = arrow_array_addr as *mut FFI_ArrowArray; let c_schema_ptr = arrow_schema_addr as *mut FFI_ArrowSchema; @@ -190,6 +196,7 @@ fn inner_create_with_ffi_array<'local>( allow_external_blob_outside_bases, blob_pack_file_size_threshold, schema_addr, + session_handle, reader, ) } @@ -215,6 +222,7 @@ pub extern "system" fn Java_org_lance_Fragment_createWithFfiStream<'a>( allow_external_blob_outside_bases: JObject, // Optional blob_pack_file_size_threshold: JObject, // Optional schema_addr: jlong, + session_handle: jlong, // Session handle, 0 means no session ) -> JObject<'a> { ok_or_throw_with_return!( env, @@ -237,6 +245,7 @@ pub extern "system" fn Java_org_lance_Fragment_createWithFfiStream<'a>( allow_external_blob_outside_bases, blob_pack_file_size_threshold, schema_addr, + session_handle, ), JObject::null() ) @@ -262,6 +271,7 @@ fn inner_create_with_ffi_stream<'local>( allow_external_blob_outside_bases: JObject, // Optional blob_pack_file_size_threshold: JObject, // Optional schema_addr: jlong, + session_handle: jlong, // Session handle, 0 means no session ) -> Result> { let stream_ptr = arrow_array_stream_addr as *mut FFI_ArrowArrayStream; let reader = unsafe { ArrowArrayStreamReader::from_raw(stream_ptr) }?; @@ -284,6 +294,7 @@ fn inner_create_with_ffi_stream<'local>( allow_external_blob_outside_bases, blob_pack_file_size_threshold, schema_addr, + session_handle, reader, ) } @@ -307,6 +318,7 @@ fn create_fragment<'a>( allow_external_blob_outside_bases: JObject, // Optional blob_pack_file_size_threshold: JObject, // Optional schema_addr: jlong, + session_handle: jlong, // Session handle, 0 means no session source: impl StreamingWriteSource, ) -> Result> { let path_str = dataset_uri.extract(env)?; @@ -328,6 +340,8 @@ fn create_fragment<'a>( &blob_pack_file_size_threshold, )?; + write_params.session = session_from_handle(session_handle); + // Set up storage options provider if namespace is provided let namespace_info = extract_namespace_info(env, &namespace_obj, &table_id_obj)?; if let Some((namespace, table_id)) = namespace_info { @@ -366,7 +380,7 @@ fn create_fragment<'a>( builder = builder.schema(&schema); } - let fragments = RT.block_on(builder.write_fragments(source))?; + let fragments = block_on(builder.write_fragments(source))?; export_vec(env, &fragments) } @@ -408,7 +422,7 @@ fn inner_delete_rows<'local>( .map(|x| x as u32) .collect(); - let res = RT.block_on(async move { fragment.extend_deletions(indexes).await }); + let res = block_on(async move { fragment.extend_deletions(indexes).await }); let obj = match res { Ok(Some(f)) => f.metadata().into_java(env)?, @@ -480,7 +494,7 @@ fn inner_merge_column<'local>( let right_on_str: String = right_on.extract(env)?; let (new_frag, new_schema) = - RT.block_on(fragment.merge_columns(reader, &left_on_str, &right_on_str, max_field_id))?; + block_on(fragment.merge_columns(reader, &left_on_str, &right_on_str, max_field_id))?; let result = FragmentMergeResult { fragment: new_frag, schema: new_schema, @@ -537,17 +551,112 @@ fn inner_update_column<'local>( let reader = unsafe { ArrowArrayStreamReader::from_raw(stream_ptr) }?; let left_on_str: String = left_on.extract(env)?; let right_on_str: String = right_on.extract(env)?; - let r = - RT.block_on(fragment.update_columns_with_offsets(reader, &left_on_str, &right_on_str))?; - let updated_row_offsets: Vec = r.matched_offsets.iter().map(|o| o as i64).collect(); + let r = block_on(fragment.update_columns_with_offsets(reader, &left_on_str, &right_on_str))?; + let updated_row_offset_bytes = serialize_matched_offsets(&r.matched_offsets)?; let result = FragmentUpdateResult { updated_fragment: r.fragment, fields_modified: r.fields_modified, - updated_row_offsets, + updated_row_offset_bytes, }; result.into_java(env) } +fn serialize_matched_offsets(bitmap: &RoaringBitmap) -> Result> { + let mut buf = Vec::new(); + bitmap.serialize_into(&mut buf).map_err(|e| { + Error::runtime_error(format!( + "failed to serialize matched row offsets RoaringBitmap: {e}" + )) + })?; + Ok(buf) +} + +fn deserialize_row_offset_bytes(bytes: &[u8]) -> Result { + if bytes.is_empty() { + return Ok(RoaringBitmap::new()); + } + RoaringBitmap::deserialize_from(bytes).map_err(|e| { + Error::input_error(format!( + "invalid updatedRowOffsetBytes RoaringBitmap bytes: {e}" + )) + }) +} + +fn expand_row_offset_bytes_to_i64(bitmap: &RoaringBitmap) -> Vec { + bitmap.iter().map(|o| o as i64).collect() +} + +#[unsafe(no_mangle)] +pub extern "system" fn Java_org_lance_fragment_FragmentUpdateResult_expandRowOffsetsFromBytes< + 'local, +>( + mut env: JNIEnv<'local>, + _cls: JClass, + jbytes: JByteArray, +) -> JLongArray<'local> { + ok_or_throw_with_return!( + env, + inner_expand_updated_row_offset_bytes(&mut env, jbytes), + unsafe { JLongArray::from_raw(std::ptr::null_mut()) } + ) +} + +fn inner_expand_updated_row_offset_bytes<'local>( + env: &mut JNIEnv<'local>, + jbytes: JByteArray, +) -> Result> { + let buf = env.convert_byte_array(&jbytes)?; + let bitmap = deserialize_row_offset_bytes(&buf)?; + let offsets = expand_row_offset_bytes_to_i64(&bitmap); + let arr = env.new_long_array(offsets.len() as i32)?; + if !offsets.is_empty() { + env.set_long_array_region(&arr, 0, &offsets)?; + } + Ok(arr) +} + +#[unsafe(no_mangle)] +pub extern "system" fn Java_org_lance_fragment_FragmentUpdateResult_encodeRowOffsetsToBytes< + 'local, +>( + mut env: JNIEnv<'local>, + _cls: JClass, + joffsets: JLongArray, +) -> JByteArray<'local> { + ok_or_throw_with_return!( + env, + inner_encode_updated_row_offset_bytes(&mut env, joffsets), + unsafe { JByteArray::from_raw(std::ptr::null_mut()) } + ) +} + +fn inner_encode_updated_row_offset_bytes<'local>( + env: &mut JNIEnv<'local>, + joffsets: JLongArray, +) -> Result> { + let len = env.get_array_length(&joffsets)?; + let mut buf: Vec = vec![0; len as usize]; + if len > 0 { + env.get_long_array_region(&joffsets, 0, buf.as_mut_slice())?; + } + let mut bitmap = RoaringBitmap::new(); + for offset in buf { + if offset < 0 { + return Err(Error::input_error(format!( + "updatedRowOffsets must be non-negative, got {offset}" + ))); + } + if offset > u32::MAX as i64 { + return Err(Error::input_error(format!( + "updatedRowOffsets value {offset} exceeds u32::MAX" + ))); + } + bitmap.insert(offset as u32); + } + let bytes = serialize_matched_offsets(&bitmap)?; + Ok(env.byte_array_from_slice(&bytes)?) +} + #[unsafe(no_mangle)] pub extern "system" fn Java_org_lance_fragment_RowIdMeta_nativeEncodeRowIds( mut env: JNIEnv, @@ -569,7 +678,7 @@ fn inner_encode_row_ids(env: &mut JNIEnv, row_ids: &JLongArray) -> Result = buf.into_iter().map(|x| x as u64).collect(); let seq = RowIdSequence::from(ids.as_slice()); - let meta = RowIdMeta::Inline(write_row_ids(&seq)); + let meta = RowIdMeta::Inline(write_row_ids(&seq).into()); let json = serde_json::to_string(&meta)?; Ok(json) } @@ -591,7 +700,7 @@ const FRAGMENT_MERGE_RESULT_CLASS: &str = "org/lance/fragment/FragmentMergeResul const FRAGMENT_MERGE_RESULT_CONSTRUCTOR_SIG: &str = "(Lorg/lance/FragmentMetadata;Lorg/lance/schema/LanceSchema;)V"; const FRAGMENT_UPDATE_RESULT_CLASS: &str = "org/lance/fragment/FragmentUpdateResult"; -const FRAGMENT_UPDATE_RESULT_CONSTRUCTOR_SIG: &str = "(Lorg/lance/FragmentMetadata;[J[J)V"; +const FRAGMENT_UPDATE_RESULT_CONSTRUCTOR_SIG: &str = "(Lorg/lance/FragmentMetadata;[J[B)V"; impl IntoJava for &FragmentMergeResult { fn into_java<'a>(self, env: &mut JNIEnv<'a>) -> Result> { @@ -612,14 +721,15 @@ impl IntoJava for &FragmentUpdateResult { fn into_java<'a>(self, env: &mut JNIEnv<'a>) -> Result> { let java_updated_fragment = self.updated_fragment.into_java(env)?; let java_fields_modified = JLance(self.fields_modified.clone()).into_java(env)?; - let java_updated_row_offsets = JLance(self.updated_row_offsets.clone()).into_java(env)?; + let java_updated_row_offset_bytes = + env.byte_array_from_slice(&self.updated_row_offset_bytes)?; Ok(env.new_object( FRAGMENT_UPDATE_RESULT_CLASS, FRAGMENT_UPDATE_RESULT_CONSTRUCTOR_SIG, &[ JValueGen::Object(&java_updated_fragment), JValueGen::Object(&java_fields_modified), - JValueGen::Object(&java_updated_row_offsets), + JValueGen::Object(&java_updated_row_offset_bytes), ], )?) } @@ -828,6 +938,9 @@ impl FromJObjectWithEnv for JObject<'_> { row_id_meta, created_at_version_meta, last_updated_at_version_meta, + // Overlays are not exposed to Java yet, and the reverse conversion + // does not export them, so this round-trip is overlay-free. + overlays: vec![], }) } } diff --git a/java/lance-jni/src/index.rs b/java/lance-jni/src/index.rs index 6cb64a05a81..865abf27bb3 100644 --- a/java/lance-jni/src/index.rs +++ b/java/lance-jni/src/index.rs @@ -35,10 +35,15 @@ impl IntoJava for &Arc { let metadata_list = export_vec(env, self.metadata())?; let details_json = self.details()?; let details = env.new_string(details_json)?; + let total_size_bytes = if let Some(size) = self.total_size_bytes() { + env.new_object("java/lang/Long", "(J)V", &[JValue::Long(size as i64)])? + } else { + JObject::null() + }; let j_index_desc = env.new_object( "org/lance/index/IndexDescription", - "(Ljava/lang/String;Ljava/util/List;Ljava/lang/String;Ljava/lang/String;JLjava/util/List;Ljava/lang/String;)V", + "(Ljava/lang/String;Ljava/util/List;Ljava/lang/String;Ljava/lang/String;JLjava/util/List;Ljava/lang/String;Ljava/lang/Long;)V", &[ JValue::Object(&name), JValue::Object(&field_ids_list), @@ -47,6 +52,7 @@ impl IntoJava for &Arc { JValue::Long(rows_indexed), JValue::Object(&metadata_list), JValue::Object(&details), + JValue::Object(&total_size_bytes), ], )?; Ok(j_index_desc) @@ -125,13 +131,19 @@ impl IntoJava for &IndexMetadata { JObject::null() }; + let size_bytes = if let Some(size) = self.total_size_bytes() { + env.new_object("java/lang/Long", "(J)V", &[JValue::Long(size as i64)])? + } else { + JObject::null() + }; + // Determine index type from index_details type_url let index_type = determine_index_type(env, &self.index_details)?; // Create Index object Ok(env.new_object( "org/lance/index/Index", - "(Ljava/util/UUID;Ljava/util/List;Ljava/lang/String;JLjava/util/List;[BILjava/time/Instant;Ljava/lang/Integer;Lorg/lance/index/IndexType;)V", + "(Ljava/util/UUID;Ljava/util/List;Ljava/lang/String;JLjava/util/List;[BILjava/time/Instant;Ljava/lang/Integer;Ljava/lang/Long;Lorg/lance/index/IndexType;)V", &[ JValue::Object(&uuid), JValue::Object(&fields), @@ -142,6 +154,7 @@ impl IntoJava for &IndexMetadata { JValue::Int(self.index_version), JValue::Object(&created_at), JValue::Object(&base_id), + JValue::Object(&size_bytes), JValue::Object(&index_type), ], )?) diff --git a/java/lance-jni/src/index_progress.rs b/java/lance-jni/src/index_progress.rs new file mode 100644 index 00000000000..a9275668db1 --- /dev/null +++ b/java/lance-jni/src/index_progress.rs @@ -0,0 +1,214 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright The Lance Authors + +use std::sync::Arc; + +use async_trait::async_trait; +use jni::objects::{GlobalRef, JObject, JString, JValue}; +use jni::{JNIEnv, JavaVM}; +use lance_index::progress::IndexBuildProgress; + +use crate::error::{Error, Result}; + +/// Bridges Rust index progress events to a Java callback. +pub(crate) struct JavaIndexBuildProgress { + callback: GlobalRef, + jvm: Arc, +} + +impl std::fmt::Debug for JavaIndexBuildProgress { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str("JavaIndexBuildProgress") + } +} + +impl JavaIndexBuildProgress { + pub(crate) fn new(env: &mut JNIEnv, callback: &JObject) -> Result { + if callback.is_null() { + return Err(Error::input_error( + "IndexBuildProgress callback cannot be null".to_string(), + )); + } + Ok(Self { + callback: env.new_global_ref(callback)?, + jvm: Arc::new(env.get_java_vm()?), + }) + } + + fn call_stage_start( + &self, + stage: &str, + total: Option, + unit: &str, + ) -> lance_core::Result<()> { + let total = total + .map(|value| { + i64::try_from(value).map_err(|_| { + lance_core::Error::invalid_input(format!( + "IndexBuildProgress total exceeds Java long for stage '{stage}': {value}" + )) + }) + }) + .transpose()?; + let mut env = self.jvm.attach_current_thread().map_err(|error| { + lance_core::Error::internal(format!( + "IndexBuildProgress.stageStart failed to attach JVM thread for stage '{stage}': {error}" + )) + })?; + + env.with_local_frame(16, |env| { + let stage_obj = env + .new_string(stage) + .map_err(|error| callback_error(env, "stageStart", stage, error))?; + let unit_obj = env + .new_string(unit) + .map_err(|error| callback_error(env, "stageStart", stage, error))?; + let total_obj = match total { + Some(value) => env + .new_object("java/lang/Long", "(J)V", &[JValue::Long(value)]) + .map_err(|error| callback_error(env, "stageStart", stage, error))?, + None => JObject::null(), + }; + let total_optional = env + .call_static_method( + "java/util/Optional", + "ofNullable", + "(Ljava/lang/Object;)Ljava/util/Optional;", + &[JValue::Object(&total_obj)], + ) + .and_then(|value| value.l()) + .map_err(|error| callback_error(env, "stageStart", stage, error))?; + + env.call_method( + &self.callback, + "stageStart", + "(Ljava/lang/String;Ljava/util/Optional;Ljava/lang/String;)V", + &[ + JValue::Object(&stage_obj), + JValue::Object(&total_optional), + JValue::Object(&unit_obj), + ], + ) + .map_err(|error| callback_error(env, "stageStart", stage, error))?; + Ok::<(), Error>(()) + }) + .map_err(|error| lance_core::Error::internal(error.to_string())) + } + + fn call_stage_progress(&self, stage: &str, completed: u64) -> lance_core::Result<()> { + let completed = i64::try_from(completed).map_err(|_| { + lance_core::Error::invalid_input(format!( + "IndexBuildProgress completed value exceeds Java long for stage '{stage}': {completed}" + )) + })?; + let mut env = self.jvm.attach_current_thread().map_err(|error| { + lance_core::Error::internal(format!( + "IndexBuildProgress.stageProgress failed to attach JVM thread for stage '{stage}': {error}" + )) + })?; + + env.with_local_frame(16, |env| { + let stage_obj = env + .new_string(stage) + .map_err(|error| callback_error(env, "stageProgress", stage, error))?; + + env.call_method( + &self.callback, + "stageProgress", + "(Ljava/lang/String;J)V", + &[JValue::Object(&stage_obj), JValue::Long(completed)], + ) + .map_err(|error| callback_error(env, "stageProgress", stage, error))?; + Ok::<(), Error>(()) + }) + .map_err(|error| lance_core::Error::internal(error.to_string())) + } + + fn call_stage_complete(&self, stage: &str) -> lance_core::Result<()> { + let mut env = self.jvm.attach_current_thread().map_err(|error| { + lance_core::Error::internal(format!( + "IndexBuildProgress.stageComplete failed to attach JVM thread for stage '{stage}': {error}" + )) + })?; + + env.with_local_frame(16, |env| { + let stage_obj = env + .new_string(stage) + .map_err(|error| callback_error(env, "stageComplete", stage, error))?; + + env.call_method( + &self.callback, + "stageComplete", + "(Ljava/lang/String;)V", + &[JValue::Object(&stage_obj)], + ) + .map_err(|error| callback_error(env, "stageComplete", stage, error))?; + Ok::<(), Error>(()) + }) + .map_err(|error| lance_core::Error::internal(error.to_string())) + } +} + +fn callback_error(env: &mut JNIEnv, method: &str, stage: &str, error: jni::errors::Error) -> Error { + let java_exception = take_pending_java_exception(env); + let detail = java_exception.unwrap_or_else(|| error.to_string()); + Error::runtime_error(format!( + "IndexBuildProgress.{method} callback failed for stage '{stage}': {detail}" + )) +} + +fn take_pending_java_exception(env: &mut JNIEnv) -> Option { + if !env.exception_check().unwrap_or(false) { + return None; + } + + let throwable = env.exception_occurred().ok(); + let _ = env.exception_clear(); + + let description = throwable.and_then(|throwable| { + if throwable.is_null() { + return None; + } + let description = env + .call_method(&throwable, "toString", "()Ljava/lang/String;", &[]) + .and_then(|value| value.l()) + .ok()?; + if description.is_null() { + return None; + } + let description = JString::from(description); + env.get_string(&description).ok().map(|value| value.into()) + }); + + if env.exception_check().unwrap_or(false) { + let _ = env.exception_clear(); + } + description +} + +#[async_trait] +impl IndexBuildProgress for JavaIndexBuildProgress { + async fn stage_start( + &self, + stage: &str, + total: Option, + unit: &str, + ) -> lance_core::Result<()> { + self.call_stage_start(stage, total, unit) + } + + async fn stage_progress(&self, stage: &str, completed: u64) -> lance_core::Result<()> { + self.call_stage_progress(stage, completed) + } + + async fn stage_complete(&self, stage: &str) -> lance_core::Result<()> { + if let Err(error) = self.call_stage_complete(stage) { + log::warn!( + "Ignoring IndexBuildProgress.stageComplete callback failure for stage '{}': {}", + stage, + error + ); + } + Ok(()) + } +} diff --git a/java/lance-jni/src/lib.rs b/java/lance-jni/src/lib.rs index 37eeff66693..3e342a3eb9a 100644 --- a/java/lance-jni/src/lib.rs +++ b/java/lance-jni/src/lib.rs @@ -51,10 +51,12 @@ mod file_reader; mod file_writer; mod fragment; mod index; +mod index_progress; mod mem_wal; mod merge_insert; mod namespace; mod optimize; +mod otel; mod schema; mod session; mod sql; @@ -85,6 +87,22 @@ pub static RT: LazyLock = LazyLock::new(|| { .expect("Failed to create tokio runtime") }); +/// Drive a future on the shared JNI runtime, including nested calls. +/// +/// Progress callbacks (and similar JNI re-entry) may invoke Dataset methods while +/// already inside `RT.block_on`. Calling `Runtime::block_on` again panics with +/// "Cannot start a runtime from within a runtime". When a Tokio handle is already +/// available, use `block_in_place` + `Handle::block_on` instead. +/// +/// JNI entry points should use this helper instead of calling `RT.block_on` +/// directly so they remain safe when invoked from a callback. +pub fn block_on(future: F) -> F::Output { + match tokio::runtime::Handle::try_current() { + Ok(handle) => tokio::task::block_in_place(|| handle.block_on(future)), + Err(_) => RT.block_on(future), + } +} + fn set_timestamp_precision(builder: &mut env_logger::Builder) { if let Ok(timestamp_precision) = env::var("LANCE_LOG_TS_PRECISION") { match timestamp_precision.as_str() { diff --git a/java/lance-jni/src/mem_wal.rs b/java/lance-jni/src/mem_wal.rs index 37fe377ed17..649ffb2cc04 100644 --- a/java/lance-jni/src/mem_wal.rs +++ b/java/lance-jni/src/mem_wal.rs @@ -26,7 +26,7 @@ use jni::objects::{JClass, JMap, JObject, JString, JValueGen}; use jni::sys::{jdouble, jint, jlong}; use lance::dataset::Dataset as LanceDataset; use lance::dataset::mem_wal::scanner::{ - FlushedGeneration, LsmDataSourceCollector, LsmPointLookupPlanner, LsmVectorSearchPlanner, + LsmDataSourceCollector, LsmPointLookupPlanner, LsmVectorSearchPlanner, SsTable, parse_filter_expr as parse_lsm_filter_expr, write_pk_sidecar, }; use lance::dataset::mem_wal::write::{MemTableStats, WriteStatsSnapshot}; @@ -41,7 +41,6 @@ use lance_io::ffi::to_ffi_arrow_array_stream; use lance_linalg::distance::DistanceType; use uuid::Uuid; -use crate::RT; use crate::blocking_dataset::{BlockingDataset, NATIVE_DATASET}; use crate::error::{Error, Result}; use crate::ffi::JNIEnvExt; @@ -49,6 +48,7 @@ use crate::traits::{ FromJString, IntoJava, export_vec, import_vec, import_vec_from_method, import_vec_to_rust, }; use crate::utils::to_rust_map; +use crate::{RT, block_on}; const NATIVE_SHARD_WRITER: &str = "nativeShardWriterHandle"; const NATIVE_LSM_SCANNER: &str = "nativeLsmScannerHandle"; @@ -126,7 +126,7 @@ fn inner_create_shard_writer<'local>( Arc::new(guard.inner.clone()) }; - let writer = RT.block_on(dataset.mem_wal_writer(uuid, writer_config))?; + let writer = block_on(dataset.mem_wal_writer(uuid, writer_config))?; let blocking = BlockingShardWriter { writer, shard_id: uuid, @@ -177,14 +177,37 @@ fn inner_put(env: &mut JNIEnv, this: JObject, stream_addr: jlong) -> Result<()> let guard = unsafe { env.get_rust_field::<_, _, BlockingShardWriter>(&this, NATIVE_SHARD_WRITER) }?; - RT.block_on(guard.writer.put(batches))?; + block_on(guard.writer.put(batches))?; Ok(()) } -/// Test-support: write a primary-key dedup sidecar (`_pk_index/`) for a -/// flushed-generation dataset already staged at `gen_path`, mirroring what -/// production flush emits. Lets Java tests stage a *faithful* flushed -/// generation (dataset + sidecar); production always writes the sidecar during +#[unsafe(no_mangle)] +pub extern "system" fn Java_org_lance_memwal_ShardWriter_nativeDelete( + mut env: JNIEnv, + this: JObject, + stream_addr: jlong, +) { + ok_or_throw_without_return!(env, inner_delete(&mut env, this, stream_addr)); +} + +fn inner_delete(env: &mut JNIEnv, this: JObject, stream_addr: jlong) -> Result<()> { + let stream_ptr = stream_addr as *mut FFI_ArrowArrayStream; + let reader = unsafe { ArrowArrayStreamReader::from_raw(stream_ptr) }?; + let batches: Vec = reader.collect::>()?; + if batches.is_empty() { + return Ok(()); + } + + let guard = + unsafe { env.get_rust_field::<_, _, BlockingShardWriter>(&this, NATIVE_SHARD_WRITER) }?; + block_on(guard.writer.delete(batches))?; + Ok(()) +} + +/// Test-support: write a primary-key dedup sidecar (`_pk_index/`) for an +/// SSTable dataset already staged at `gen_path`, mirroring what +/// production flush emits. Lets Java tests stage a *faithful* SSTable +/// (dataset + sidecar); production always writes the sidecar during /// flush, so a dataset-without-sidecar is not a state the system produces. /// Mirrors the Python `_write_pk_sidecar` binding. #[unsafe(no_mangle)] @@ -213,7 +236,7 @@ fn inner_write_pk_sidecar( let reader = unsafe { ArrowArrayStreamReader::from_raw(stream_ptr) }?; let batches: Vec = reader.collect::>()?; let pk_refs: Vec<&str> = pk_columns.iter().map(String::as_str).collect(); - RT.block_on(write_pk_sidecar(&gen_path, &batches, &pk_refs))?; + block_on(write_pk_sidecar(&gen_path, &batches, &pk_refs))?; Ok(()) } @@ -252,7 +275,7 @@ fn inner_memtable_stats<'local>( let stats = { let guard = unsafe { env.get_rust_field::<_, _, BlockingShardWriter>(&this, NATIVE_SHARD_WRITER) }?; - RT.block_on(guard.writer.memtable_stats())? + block_on(guard.writer.memtable_stats())? }; memtable_stats_to_java(env, &stats) } @@ -283,9 +306,8 @@ fn inner_writer_lsm_scanner<'local>( // Capture the active memtable *and* any frozen-awaiting-flush memtables // so a concurrent flush rollover cannot hide acknowledged writes from // this read-your-writes scanner. - let in_memory_memtables = RT.block_on(guard.writer.in_memory_memtable_refs())?; - let writer_snapshot = RT - .block_on(guard.writer.manifest())? + let in_memory_memtables = block_on(guard.writer.in_memory_memtable_refs())?; + let writer_snapshot = block_on(guard.writer.manifest())? .map(shard_snapshot_from_manifest) .unwrap_or_else(|| ShardSnapshot::new(guard.shard_id)); ( @@ -316,7 +338,7 @@ pub extern "system" fn Java_org_lance_memwal_ShardWriter_releaseNativeShardWrite fn inner_release_shard_writer(env: &mut JNIEnv, this: JObject) -> Result<()> { let blocking: BlockingShardWriter = unsafe { env.take_rust_field(&this, NATIVE_SHARD_WRITER) }?; - RT.block_on(blocking.writer.close())?; + block_on(blocking.writer.close())?; Ok(()) } @@ -475,7 +497,7 @@ fn inner_scanner_open_stream(env: &mut JNIEnv, this: JObject, stream_addr: jlong let scanner = guard.inner.as_ref().ok_or_else(|| { Error::runtime_error("LsmScanner is no longer usable because an earlier builder call (e.g. filter) failed; create a new scanner".to_string()) })?; - RT.block_on(scanner.try_into_stream())? + block_on(scanner.try_into_stream())? }; let ffi_stream = to_ffi_arrow_array_stream(DatasetRecordBatchStream::new(stream), RT.handle().clone())?; @@ -498,7 +520,7 @@ fn inner_scanner_count_rows(env: &mut JNIEnv, this: JObject) -> Result { .inner .as_ref() .ok_or_else(|| Error::runtime_error("LsmScanner is no longer usable because an earlier builder call (e.g. filter) failed; create a new scanner".to_string()))?; - Ok(RT.block_on(scanner.count_rows())?) + Ok(block_on(scanner.count_rows())?) } #[unsafe(no_mangle)] @@ -625,12 +647,11 @@ fn inner_plan_open_stream(env: &mut JNIEnv, this: JObject, stream_addr: jlong) - guard.plan.clone() }; let schema = plan.schema(); - let batches = RT - .block_on(async move { - let ctx = SessionContext::new(); - collect(plan, ctx.task_ctx()).await - }) - .map_err(|e| Error::io_error(format!("Plan execution failed: {}", e)))?; + let batches = block_on(async move { + let ctx = SessionContext::new(); + collect(plan, ctx.task_ctx()).await + }) + .map_err(|e| Error::io_error(format!("Plan execution failed: {}", e)))?; let reader: Box = Box::new(RecordBatchIterator::new( batches.into_iter().map(Ok), @@ -790,7 +811,7 @@ fn inner_plan_lookup<'local>( env.get_rust_field::<_, _, BlockingLsmPointLookupPlanner>(&this, NATIVE_LOOKUP_PLANNER) }?; let pk_values = scalar_values_from_pk_value(pk_value.as_ref(), &guard.pk_columns)?; - let plan = RT.block_on(guard.planner.plan_lookup(&pk_values, columns.as_deref()))?; + let plan = block_on(guard.planner.plan_lookup(&pk_values, columns.as_deref()))?; (plan, guard.dataset_schema.clone()) }; attach_execution_plan(env, plan, dataset_schema) @@ -961,7 +982,7 @@ fn inner_plan_search<'local>( Arc::new(float32_array.clone()), None, )?; - let plan = RT.block_on(guard.planner.plan_search( + let plan = block_on(guard.planner.plan_search( &fsl, k as usize, nprobes as usize, @@ -1049,7 +1070,7 @@ fn inner_initialize_mem_wal(env: &mut JNIEnv, jdataset: JObject, params: JObject if let Some(config) = writer_config { builder = builder.writer_config_defaults(config); } - RT.block_on(builder.execute())?; + block_on(builder.execute())?; Ok(()) } @@ -1068,7 +1089,7 @@ fn inner_mem_wal_index_details<'local>( let details = { let guard = unsafe { env.get_rust_field::<_, _, BlockingDataset>(&jdataset, NATIVE_DATASET) }?; - RT.block_on(guard.inner.mem_wal_index_details())? + block_on(guard.inner.mem_wal_index_details())? }; match details { Some(details) => index_details_to_java(env, &details), @@ -1096,13 +1117,13 @@ fn read_shard_snapshots(env: &mut JNIEnv, list_obj: &JObject) -> Result ShardSnapshot { shard_id: manifest.shard_id, spec_id: manifest.shard_spec_id, current_generation: manifest.current_generation, - flushed_generations: manifest - .flushed_generations + sstables: manifest + .sstables .into_iter() - .map(|generation| FlushedGeneration { - generation: generation.generation, - path: generation.path, + .map(|sstable| SsTable { + generation: sstable.generation, + path: sstable.path, }) .collect(), } @@ -1242,9 +1263,6 @@ fn build_writer_config(env: &mut JNIEnv, config: &JObject) -> Result Result( fn memtable_stats_to_java<'a>(env: &mut JNIEnv<'a>, stats: &MemTableStats) -> Result> { let max_buffered = box_u64_opt(env, stats.max_buffered_batch_position)?; - let max_flushed = box_u64_opt(env, stats.max_flushed_batch_position)?; let pending_start = box_u64_opt(env, stats.pending_wal_start_batch_position)?; let pending_end = box_u64_opt(env, stats.pending_wal_end_batch_position)?; Ok(env.new_object( "org/lance/memwal/MemTableStats", - "(JJJJLjava/lang/Long;Ljava/lang/Long;Ljava/lang/Long;Ljava/lang/Long;JJJ)V", + "(JJJJLjava/lang/Long;JJLjava/lang/Long;Ljava/lang/Long;JJJ)V", &[ JValueGen::Long(stats.row_count as i64), JValueGen::Long(stats.batch_count as i64), JValueGen::Long(stats.estimated_size as i64), JValueGen::Long(stats.generation as i64), JValueGen::Object(&max_buffered), - JValueGen::Object(&max_flushed), + JValueGen::Long(stats.durable_batch_count as i64), + JValueGen::Long(stats.global_offset as i64), JValueGen::Object(&pending_start), JValueGen::Object(&pending_end), JValueGen::Long(stats.pending_wal_batch_count as i64), diff --git a/java/lance-jni/src/merge_insert.rs b/java/lance-jni/src/merge_insert.rs index df4d63bd2f6..2be6a914a57 100644 --- a/java/lance-jni/src/merge_insert.rs +++ b/java/lance-jni/src/merge_insert.rs @@ -5,17 +5,18 @@ use crate::blocking_dataset::{BlockingDataset, NATIVE_DATASET}; use crate::error::Result; use crate::traits::import_vec_to_rust; use crate::traits::{FromJString, IntoJava}; -use crate::{Error, JNIEnvExt, RT}; +use crate::{Error, JNIEnvExt, block_on}; use arrow::ffi_stream::{ArrowArrayStreamReader, FFI_ArrowArrayStream}; use jni::JNIEnv; use jni::objects::{JObject, JString, JValueGen}; use jni::sys::jlong; use lance::dataset::scanner::ExprFilter; use lance::dataset::{ - MergeInsertBuilder, MergeStats, WhenMatched, WhenNotMatched, WhenNotMatchedBySource, + MergeInsertBuilder, MergeStats, SourceDedupeBehavior, WhenMatched, WhenNotMatched, + WhenNotMatchedBySource, }; use lance_core::datatypes::Schema; -use lance_index::mem_wal::MergedGeneration; +use lance_index::mem_wal::CompactedSsTable; use std::sync::Arc; use std::time::Duration; use uuid::Uuid; @@ -52,7 +53,8 @@ fn inner_merge_insert<'local>( let retry_timeout_ms = extract_retry_timeout_ms(env, &jparam)?; let skip_auto_cleanup = extract_skip_auto_cleanup(env, &jparam)?; let use_index = extract_use_index(env, &jparam)?; - let marked_generations = extract_marked_generations(env, &jparam)?; + let compacted_sstables = extract_compacted_sstables(env, &jparam)?; + let source_dedupe_behavior = extract_source_dedupe_behavior(env, &jparam)?; let (new_ds, merge_stats) = unsafe { let dataset = env.get_rust_field::<_, _, BlockingDataset>(jdataset, NATIVE_DATASET)?; @@ -71,13 +73,14 @@ fn inner_merge_insert<'local>( .retry_timeout(Duration::from_millis(retry_timeout_ms as u64)) .skip_auto_cleanup(skip_auto_cleanup) .use_index(use_index) - .mark_generations_as_merged(marked_generations) + .mark_sstables_as_compacted(compacted_sstables) + .source_dedupe_behavior(source_dedupe_behavior) .try_build()?; let stream_ptr = batch_address as *mut FFI_ArrowArrayStream; let source_stream = ArrowArrayStreamReader::from_raw(stream_ptr)?; - RT.block_on(async move { merge_insert_job.execute_reader(source_stream).await })? + block_on(async move { merge_insert_job.execute_reader(source_stream).await })? }; MergeResult( @@ -241,23 +244,46 @@ fn extract_use_index<'local>(env: &mut JNIEnv<'local>, jparam: &JObject) -> Resu Ok(use_index) } -fn extract_marked_generations<'local>( +fn extract_source_dedupe_behavior<'local>( env: &mut JNIEnv<'local>, jparam: &JObject, -) -> Result> { +) -> Result { + let behavior: JString = env + .call_method( + jparam, + "sourceDedupeBehaviorValue", + "()Ljava/lang/String;", + &[], + )? + .l()? + .into(); + let behavior = behavior.extract(env)?; + match behavior.as_str() { + "Fail" => Ok(SourceDedupeBehavior::Fail), + "FirstSeen" => Ok(SourceDedupeBehavior::FirstSeen), + _ => Err(Error::input_error(format!( + "Illegal source_dedupe_behavior: {behavior}", + ))), + } +} + +fn extract_compacted_sstables<'local>( + env: &mut JNIEnv<'local>, + jparam: &JObject, +) -> Result> { let list = env - .call_method(jparam, "markedGenerations", "()Ljava/util/List;", &[])? + .call_method(jparam, "getCompactedSstables", "()Ljava/util/List;", &[])? .l()?; import_vec_to_rust(env, &list, |env, obj| { let shard_id: JString = env - .call_method(&obj, "shardId", "()Ljava/lang/String;", &[])? + .call_method(&obj, "getShardId", "()Ljava/lang/String;", &[])? .l()? .into(); let shard_id = shard_id.extract(env)?; - let generation = env.call_method(&obj, "generation", "()J", &[])?.j()? as u64; + let generation = env.call_method(&obj, "getGeneration", "()J", &[])?.j()? as u64; let uuid = Uuid::parse_str(&shard_id) .map_err(|e| Error::input_error(format!("Invalid shard_id UUID: {}", e)))?; - Ok(MergedGeneration::new(uuid, generation)) + Ok(CompactedSsTable::new(uuid, generation)) }) } diff --git a/java/lance-jni/src/namespace.rs b/java/lance-jni/src/namespace.rs index f0da7ff79ae..9978fcfa7a2 100644 --- a/java/lance-jni/src/namespace.rs +++ b/java/lance-jni/src/namespace.rs @@ -17,7 +17,7 @@ use lance_namespace_impls::{ }; use serde::{Deserialize, Serialize}; -use crate::RT; +use crate::block_on; use crate::error::{Error, Result}; use crate::utils::to_rust_map; @@ -1581,8 +1581,7 @@ fn create_directory_namespace_internal( builder = builder.context_provider(Arc::new(java_provider)); } - let namespace = RT - .block_on(builder.build()) + let namespace = block_on(builder.build()) .map_err(|e| Error::runtime_error(format!("Failed to build DirectoryNamespace: {}", e)))?; let blocking_namespace = BlockingDirectoryNamespace { @@ -1631,7 +1630,7 @@ pub extern "system" fn Java_org_lance_namespace_DirectoryNamespace_listNamespace ok_or_throw_with_return!( env, call_namespace_method(&mut env, handle, request_json, |namespace_client, req| { - RT.block_on(namespace_client.inner.list_namespaces(req)) + block_on(namespace_client.inner.list_namespaces(req)) }), std::ptr::null_mut() ) @@ -1648,7 +1647,7 @@ pub extern "system" fn Java_org_lance_namespace_DirectoryNamespace_describeNames ok_or_throw_with_return!( env, call_namespace_method(&mut env, handle, request_json, |namespace_client, req| { - RT.block_on(namespace_client.inner.describe_namespace(req)) + block_on(namespace_client.inner.describe_namespace(req)) }), std::ptr::null_mut() ) @@ -1665,7 +1664,7 @@ pub extern "system" fn Java_org_lance_namespace_DirectoryNamespace_createNamespa ok_or_throw_with_return!( env, call_namespace_method(&mut env, handle, request_json, |namespace_client, req| { - RT.block_on(namespace_client.inner.create_namespace(req)) + block_on(namespace_client.inner.create_namespace(req)) }), std::ptr::null_mut() ) @@ -1682,7 +1681,7 @@ pub extern "system" fn Java_org_lance_namespace_DirectoryNamespace_dropNamespace ok_or_throw_with_return!( env, call_namespace_method(&mut env, handle, request_json, |namespace_client, req| { - RT.block_on(namespace_client.inner.drop_namespace(req)) + block_on(namespace_client.inner.drop_namespace(req)) }), std::ptr::null_mut() ) @@ -1699,7 +1698,7 @@ pub extern "system" fn Java_org_lance_namespace_DirectoryNamespace_namespaceExis ok_or_throw_without_return!( env, call_namespace_void_method(&mut env, handle, request_json, |namespace_client, req| { - RT.block_on(namespace_client.inner.namespace_exists(req)) + block_on(namespace_client.inner.namespace_exists(req)) }) ) } @@ -1714,7 +1713,7 @@ pub extern "system" fn Java_org_lance_namespace_DirectoryNamespace_listTablesNat ok_or_throw_with_return!( env, call_namespace_method(&mut env, handle, request_json, |namespace_client, req| { - RT.block_on(namespace_client.inner.list_tables(req)) + block_on(namespace_client.inner.list_tables(req)) }), std::ptr::null_mut() ) @@ -1731,7 +1730,7 @@ pub extern "system" fn Java_org_lance_namespace_DirectoryNamespace_describeTable ok_or_throw_with_return!( env, call_namespace_method(&mut env, handle, request_json, |namespace_client, req| { - RT.block_on(namespace_client.inner.describe_table(req)) + block_on(namespace_client.inner.describe_table(req)) }), std::ptr::null_mut() ) @@ -1748,7 +1747,7 @@ pub extern "system" fn Java_org_lance_namespace_DirectoryNamespace_registerTable ok_or_throw_with_return!( env, call_namespace_method(&mut env, handle, request_json, |namespace_client, req| { - RT.block_on(namespace_client.inner.register_table(req)) + block_on(namespace_client.inner.register_table(req)) }), std::ptr::null_mut() ) @@ -1765,7 +1764,7 @@ pub extern "system" fn Java_org_lance_namespace_DirectoryNamespace_tableExistsNa ok_or_throw_without_return!( env, call_namespace_void_method(&mut env, handle, request_json, |namespace_client, req| { - RT.block_on(namespace_client.inner.table_exists(req)) + block_on(namespace_client.inner.table_exists(req)) }) ) } @@ -1780,7 +1779,7 @@ pub extern "system" fn Java_org_lance_namespace_DirectoryNamespace_dropTableNati ok_or_throw_with_return!( env, call_namespace_method(&mut env, handle, request_json, |namespace_client, req| { - RT.block_on(namespace_client.inner.drop_table(req)) + block_on(namespace_client.inner.drop_table(req)) }), std::ptr::null_mut() ) @@ -1797,7 +1796,7 @@ pub extern "system" fn Java_org_lance_namespace_DirectoryNamespace_deregisterTab ok_or_throw_with_return!( env, call_namespace_method(&mut env, handle, request_json, |namespace_client, req| { - RT.block_on(namespace_client.inner.deregister_table(req)) + block_on(namespace_client.inner.deregister_table(req)) }), std::ptr::null_mut() ) @@ -1834,7 +1833,7 @@ pub extern "system" fn Java_org_lance_namespace_DirectoryNamespace_createTableNa request_json, request_data, |namespace_client, req, data| { - RT.block_on(namespace_client.inner.create_table(req, data)) + block_on(namespace_client.inner.create_table(req, data)) } ), std::ptr::null_mut() @@ -1852,7 +1851,7 @@ pub extern "system" fn Java_org_lance_namespace_DirectoryNamespace_declareTableN ok_or_throw_with_return!( env, call_namespace_method(&mut env, handle, request_json, |namespace_client, req| { - RT.block_on(namespace_client.inner.declare_table(req)) + block_on(namespace_client.inner.declare_table(req)) }), std::ptr::null_mut() ) @@ -1869,7 +1868,7 @@ pub extern "system" fn Java_org_lance_namespace_DirectoryNamespace_renameTableNa ok_or_throw_with_return!( env, call_namespace_method(&mut env, handle, request_json, |namespace_client, req| { - RT.block_on(namespace_client.inner.rename_table(req)) + block_on(namespace_client.inner.rename_table(req)) }), std::ptr::null_mut() ) @@ -1892,7 +1891,7 @@ pub extern "system" fn Java_org_lance_namespace_DirectoryNamespace_insertIntoTab request_json, request_data, |namespace_client, req, data| { - RT.block_on(namespace_client.inner.insert_into_table(req, data)) + block_on(namespace_client.inner.insert_into_table(req, data)) } ), std::ptr::null_mut() @@ -1916,7 +1915,7 @@ pub extern "system" fn Java_org_lance_namespace_DirectoryNamespace_mergeInsertIn request_json, request_data, |namespace_client, req, data| { - RT.block_on(namespace_client.inner.merge_insert_into_table(req, data)) + block_on(namespace_client.inner.merge_insert_into_table(req, data)) } ), std::ptr::null_mut() @@ -1934,7 +1933,7 @@ pub extern "system" fn Java_org_lance_namespace_DirectoryNamespace_updateTableNa ok_or_throw_with_return!( env, call_namespace_method(&mut env, handle, request_json, |namespace_client, req| { - RT.block_on(namespace_client.inner.update_table(req)) + block_on(namespace_client.inner.update_table(req)) }), std::ptr::null_mut() ) @@ -1951,7 +1950,7 @@ pub extern "system" fn Java_org_lance_namespace_DirectoryNamespace_deleteFromTab ok_or_throw_with_return!( env, call_namespace_method(&mut env, handle, request_json, |namespace_client, req| { - RT.block_on(namespace_client.inner.delete_from_table(req)) + block_on(namespace_client.inner.delete_from_table(req)) }), std::ptr::null_mut() ) @@ -1983,7 +1982,7 @@ pub extern "system" fn Java_org_lance_namespace_DirectoryNamespace_createTableIn ok_or_throw_with_return!( env, call_namespace_method(&mut env, handle, request_json, |namespace_client, req| { - RT.block_on(namespace_client.inner.create_table_index(req)) + block_on(namespace_client.inner.create_table_index(req)) }), std::ptr::null_mut() ) @@ -2000,7 +1999,7 @@ pub extern "system" fn Java_org_lance_namespace_DirectoryNamespace_listTableIndi ok_or_throw_with_return!( env, call_namespace_method(&mut env, handle, request_json, |namespace_client, req| { - RT.block_on(namespace_client.inner.list_table_indices(req)) + block_on(namespace_client.inner.list_table_indices(req)) }), std::ptr::null_mut() ) @@ -2017,7 +2016,7 @@ pub extern "system" fn Java_org_lance_namespace_DirectoryNamespace_describeTable ok_or_throw_with_return!( env, call_namespace_method(&mut env, handle, request_json, |namespace_client, req| { - RT.block_on(namespace_client.inner.describe_table_index_stats(req)) + block_on(namespace_client.inner.describe_table_index_stats(req)) }), std::ptr::null_mut() ) @@ -2034,7 +2033,7 @@ pub extern "system" fn Java_org_lance_namespace_DirectoryNamespace_describeTrans ok_or_throw_with_return!( env, call_namespace_method(&mut env, handle, request_json, |namespace_client, req| { - RT.block_on(namespace_client.inner.describe_transaction(req)) + block_on(namespace_client.inner.describe_transaction(req)) }), std::ptr::null_mut() ) @@ -2051,7 +2050,7 @@ pub extern "system" fn Java_org_lance_namespace_DirectoryNamespace_alterTransact ok_or_throw_with_return!( env, call_namespace_method(&mut env, handle, request_json, |namespace_client, req| { - RT.block_on(namespace_client.inner.alter_transaction(req)) + block_on(namespace_client.inner.alter_transaction(req)) }), std::ptr::null_mut() ) @@ -2068,7 +2067,7 @@ pub extern "system" fn Java_org_lance_namespace_DirectoryNamespace_listTableVers ok_or_throw_with_return!( env, call_namespace_method(&mut env, handle, request_json, |namespace_client, req| { - RT.block_on(namespace_client.inner.list_table_versions(req)) + block_on(namespace_client.inner.list_table_versions(req)) }), std::ptr::null_mut() ) @@ -2085,7 +2084,7 @@ pub extern "system" fn Java_org_lance_namespace_DirectoryNamespace_createTableVe ok_or_throw_with_return!( env, call_namespace_method(&mut env, handle, request_json, |namespace_client, req| { - RT.block_on(namespace_client.inner.create_table_version(req)) + block_on(namespace_client.inner.create_table_version(req)) }), std::ptr::null_mut() ) @@ -2102,7 +2101,7 @@ pub extern "system" fn Java_org_lance_namespace_DirectoryNamespace_describeTable ok_or_throw_with_return!( env, call_namespace_method(&mut env, handle, request_json, |namespace_client, req| { - RT.block_on(namespace_client.inner.describe_table_version(req)) + block_on(namespace_client.inner.describe_table_version(req)) }), std::ptr::null_mut() ) @@ -2119,7 +2118,7 @@ pub extern "system" fn Java_org_lance_namespace_DirectoryNamespace_batchDeleteTa ok_or_throw_with_return!( env, call_namespace_method(&mut env, handle, request_json, |namespace_client, req| { - RT.block_on(namespace_client.inner.batch_delete_table_versions(req)) + block_on(namespace_client.inner.batch_delete_table_versions(req)) }), std::ptr::null_mut() ) @@ -2136,7 +2135,7 @@ pub extern "system" fn Java_org_lance_namespace_DirectoryNamespace_createTableSc ok_or_throw_with_return!( env, call_namespace_method(&mut env, handle, request_json, |namespace_client, req| { - RT.block_on(namespace_client.inner.create_table_scalar_index(req)) + block_on(namespace_client.inner.create_table_scalar_index(req)) }), std::ptr::null_mut() ) @@ -2153,7 +2152,7 @@ pub extern "system" fn Java_org_lance_namespace_DirectoryNamespace_dropTableInde ok_or_throw_with_return!( env, call_namespace_method(&mut env, handle, request_json, |namespace_client, req| { - RT.block_on(namespace_client.inner.drop_table_index(req)) + block_on(namespace_client.inner.drop_table_index(req)) }), std::ptr::null_mut() ) @@ -2170,7 +2169,7 @@ pub extern "system" fn Java_org_lance_namespace_DirectoryNamespace_listAllTables ok_or_throw_with_return!( env, call_namespace_method(&mut env, handle, request_json, |namespace_client, req| { - RT.block_on(namespace_client.inner.list_all_tables(req)) + block_on(namespace_client.inner.list_all_tables(req)) }), std::ptr::null_mut() ) @@ -2187,7 +2186,7 @@ pub extern "system" fn Java_org_lance_namespace_DirectoryNamespace_restoreTableN ok_or_throw_with_return!( env, call_namespace_method(&mut env, handle, request_json, |namespace_client, req| { - RT.block_on(namespace_client.inner.restore_table(req)) + block_on(namespace_client.inner.restore_table(req)) }), std::ptr::null_mut() ) @@ -2204,7 +2203,7 @@ pub extern "system" fn Java_org_lance_namespace_DirectoryNamespace_updateTableSc ok_or_throw_with_return!( env, call_namespace_method(&mut env, handle, request_json, |namespace_client, req| { - RT.block_on(namespace_client.inner.update_table_schema_metadata(req)) + block_on(namespace_client.inner.update_table_schema_metadata(req)) }), std::ptr::null_mut() ) @@ -2221,7 +2220,7 @@ pub extern "system" fn Java_org_lance_namespace_DirectoryNamespace_getTableStats ok_or_throw_with_return!( env, call_namespace_method(&mut env, handle, request_json, |namespace_client, req| { - RT.block_on(namespace_client.inner.get_table_stats(req)) + block_on(namespace_client.inner.get_table_stats(req)) }), std::ptr::null_mut() ) @@ -2238,7 +2237,7 @@ pub extern "system" fn Java_org_lance_namespace_DirectoryNamespace_explainTableQ ok_or_throw_with_return!( env, call_namespace_string_method(&mut env, handle, request_json, |namespace_client, req| { - RT.block_on(namespace_client.inner.explain_table_query_plan(req)) + block_on(namespace_client.inner.explain_table_query_plan(req)) }), std::ptr::null_mut() ) @@ -2255,7 +2254,7 @@ pub extern "system" fn Java_org_lance_namespace_DirectoryNamespace_analyzeTableQ ok_or_throw_with_return!( env, call_namespace_string_method(&mut env, handle, request_json, |namespace_client, req| { - RT.block_on(namespace_client.inner.analyze_table_query_plan(req)) + block_on(namespace_client.inner.analyze_table_query_plan(req)) }), std::ptr::null_mut() ) @@ -2272,7 +2271,7 @@ pub extern "system" fn Java_org_lance_namespace_DirectoryNamespace_alterTableAdd ok_or_throw_with_return!( env, call_namespace_method(&mut env, handle, request_json, |namespace_client, req| { - RT.block_on(namespace_client.inner.alter_table_add_columns(req)) + block_on(namespace_client.inner.alter_table_add_columns(req)) }), std::ptr::null_mut() ) @@ -2289,7 +2288,7 @@ pub extern "system" fn Java_org_lance_namespace_DirectoryNamespace_alterTableAlt ok_or_throw_with_return!( env, call_namespace_method(&mut env, handle, request_json, |namespace_client, req| { - RT.block_on(namespace_client.inner.alter_table_alter_columns(req)) + block_on(namespace_client.inner.alter_table_alter_columns(req)) }), std::ptr::null_mut() ) @@ -2306,7 +2305,7 @@ pub extern "system" fn Java_org_lance_namespace_DirectoryNamespace_alterTableDro ok_or_throw_with_return!( env, call_namespace_method(&mut env, handle, request_json, |namespace_client, req| { - RT.block_on(namespace_client.inner.alter_table_drop_columns(req)) + block_on(namespace_client.inner.alter_table_drop_columns(req)) }), std::ptr::null_mut() ) @@ -2323,7 +2322,7 @@ pub extern "system" fn Java_org_lance_namespace_DirectoryNamespace_alterTableBac ok_or_throw_with_return!( env, call_namespace_method(&mut env, handle, request_json, |namespace_client, req| { - RT.block_on(namespace_client.inner.alter_table_backfill_columns(req)) + block_on(namespace_client.inner.alter_table_backfill_columns(req)) }), std::ptr::null_mut() ) @@ -2340,7 +2339,7 @@ pub extern "system" fn Java_org_lance_namespace_DirectoryNamespace_refreshMateri ok_or_throw_with_return!( env, call_namespace_method(&mut env, handle, request_json, |namespace_client, req| { - RT.block_on(namespace_client.inner.refresh_materialized_view(req)) + block_on(namespace_client.inner.refresh_materialized_view(req)) }), std::ptr::null_mut() ) @@ -2357,7 +2356,7 @@ pub extern "system" fn Java_org_lance_namespace_DirectoryNamespace_listTableTags ok_or_throw_with_return!( env, call_namespace_method(&mut env, handle, request_json, |namespace_client, req| { - RT.block_on(namespace_client.inner.list_table_tags(req)) + block_on(namespace_client.inner.list_table_tags(req)) }), std::ptr::null_mut() ) @@ -2374,7 +2373,7 @@ pub extern "system" fn Java_org_lance_namespace_DirectoryNamespace_getTableTagVe ok_or_throw_with_return!( env, call_namespace_method(&mut env, handle, request_json, |namespace_client, req| { - RT.block_on(namespace_client.inner.get_table_tag_version(req)) + block_on(namespace_client.inner.get_table_tag_version(req)) }), std::ptr::null_mut() ) @@ -2391,7 +2390,7 @@ pub extern "system" fn Java_org_lance_namespace_DirectoryNamespace_createTableTa ok_or_throw_with_return!( env, call_namespace_method(&mut env, handle, request_json, |namespace_client, req| { - RT.block_on(namespace_client.inner.create_table_tag(req)) + block_on(namespace_client.inner.create_table_tag(req)) }), std::ptr::null_mut() ) @@ -2408,7 +2407,7 @@ pub extern "system" fn Java_org_lance_namespace_DirectoryNamespace_deleteTableTa ok_or_throw_with_return!( env, call_namespace_method(&mut env, handle, request_json, |namespace_client, req| { - RT.block_on(namespace_client.inner.delete_table_tag(req)) + block_on(namespace_client.inner.delete_table_tag(req)) }), std::ptr::null_mut() ) @@ -2425,7 +2424,7 @@ pub extern "system" fn Java_org_lance_namespace_DirectoryNamespace_updateTableTa ok_or_throw_with_return!( env, call_namespace_method(&mut env, handle, request_json, |namespace_client, req| { - RT.block_on(namespace_client.inner.update_table_tag(req)) + block_on(namespace_client.inner.update_table_tag(req)) }), std::ptr::null_mut() ) @@ -2442,7 +2441,7 @@ pub extern "system" fn Java_org_lance_namespace_DirectoryNamespace_createMateria ok_or_throw_with_return!( env, call_namespace_method(&mut env, handle, request_json, |namespace_client, req| { - RT.block_on(namespace_client.inner.create_materialized_view(req)) + block_on(namespace_client.inner.create_materialized_view(req)) }), std::ptr::null_mut() ) @@ -2581,7 +2580,7 @@ pub extern "system" fn Java_org_lance_namespace_RestNamespace_listNamespacesNati ok_or_throw_with_return!( env, call_rest_namespace_method(&mut env, handle, request_json, |namespace_client, req| { - RT.block_on(namespace_client.inner.list_namespaces(req)) + block_on(namespace_client.inner.list_namespaces(req)) }), std::ptr::null_mut() ) @@ -2598,7 +2597,7 @@ pub extern "system" fn Java_org_lance_namespace_RestNamespace_describeNamespaceN ok_or_throw_with_return!( env, call_rest_namespace_method(&mut env, handle, request_json, |namespace_client, req| { - RT.block_on(namespace_client.inner.describe_namespace(req)) + block_on(namespace_client.inner.describe_namespace(req)) }), std::ptr::null_mut() ) @@ -2615,7 +2614,7 @@ pub extern "system" fn Java_org_lance_namespace_RestNamespace_createNamespaceNat ok_or_throw_with_return!( env, call_rest_namespace_method(&mut env, handle, request_json, |namespace_client, req| { - RT.block_on(namespace_client.inner.create_namespace(req)) + block_on(namespace_client.inner.create_namespace(req)) }), std::ptr::null_mut() ) @@ -2632,7 +2631,7 @@ pub extern "system" fn Java_org_lance_namespace_RestNamespace_dropNamespaceNativ ok_or_throw_with_return!( env, call_rest_namespace_method(&mut env, handle, request_json, |namespace_client, req| { - RT.block_on(namespace_client.inner.drop_namespace(req)) + block_on(namespace_client.inner.drop_namespace(req)) }), std::ptr::null_mut() ) @@ -2649,7 +2648,7 @@ pub extern "system" fn Java_org_lance_namespace_RestNamespace_namespaceExistsNat ok_or_throw_without_return!( env, call_rest_namespace_void_method(&mut env, handle, request_json, |namespace_client, req| { - RT.block_on(namespace_client.inner.namespace_exists(req)) + block_on(namespace_client.inner.namespace_exists(req)) }) ) } @@ -2664,7 +2663,7 @@ pub extern "system" fn Java_org_lance_namespace_RestNamespace_listTablesNative( ok_or_throw_with_return!( env, call_rest_namespace_method(&mut env, handle, request_json, |namespace_client, req| { - RT.block_on(namespace_client.inner.list_tables(req)) + block_on(namespace_client.inner.list_tables(req)) }), std::ptr::null_mut() ) @@ -2681,7 +2680,7 @@ pub extern "system" fn Java_org_lance_namespace_RestNamespace_describeTableNativ ok_or_throw_with_return!( env, call_rest_namespace_method(&mut env, handle, request_json, |namespace_client, req| { - RT.block_on(namespace_client.inner.describe_table(req)) + block_on(namespace_client.inner.describe_table(req)) }), std::ptr::null_mut() ) @@ -2698,7 +2697,7 @@ pub extern "system" fn Java_org_lance_namespace_RestNamespace_registerTableNativ ok_or_throw_with_return!( env, call_rest_namespace_method(&mut env, handle, request_json, |namespace_client, req| { - RT.block_on(namespace_client.inner.register_table(req)) + block_on(namespace_client.inner.register_table(req)) }), std::ptr::null_mut() ) @@ -2715,7 +2714,7 @@ pub extern "system" fn Java_org_lance_namespace_RestNamespace_tableExistsNative( ok_or_throw_without_return!( env, call_rest_namespace_void_method(&mut env, handle, request_json, |namespace_client, req| { - RT.block_on(namespace_client.inner.table_exists(req)) + block_on(namespace_client.inner.table_exists(req)) }) ) } @@ -2730,7 +2729,7 @@ pub extern "system" fn Java_org_lance_namespace_RestNamespace_dropTableNative( ok_or_throw_with_return!( env, call_rest_namespace_method(&mut env, handle, request_json, |namespace_client, req| { - RT.block_on(namespace_client.inner.drop_table(req)) + block_on(namespace_client.inner.drop_table(req)) }), std::ptr::null_mut() ) @@ -2747,7 +2746,7 @@ pub extern "system" fn Java_org_lance_namespace_RestNamespace_deregisterTableNat ok_or_throw_with_return!( env, call_rest_namespace_method(&mut env, handle, request_json, |namespace_client, req| { - RT.block_on(namespace_client.inner.deregister_table(req)) + block_on(namespace_client.inner.deregister_table(req)) }), std::ptr::null_mut() ) @@ -2784,7 +2783,7 @@ pub extern "system" fn Java_org_lance_namespace_RestNamespace_createTableNative( request_json, request_data, |namespace_client, req, data| { - RT.block_on(namespace_client.inner.create_table(req, data)) + block_on(namespace_client.inner.create_table(req, data)) } ), std::ptr::null_mut() @@ -2802,7 +2801,7 @@ pub extern "system" fn Java_org_lance_namespace_RestNamespace_declareTableNative ok_or_throw_with_return!( env, call_rest_namespace_method(&mut env, handle, request_json, |namespace_client, req| { - RT.block_on(namespace_client.inner.declare_table(req)) + block_on(namespace_client.inner.declare_table(req)) }), std::ptr::null_mut() ) @@ -2819,7 +2818,7 @@ pub extern "system" fn Java_org_lance_namespace_RestNamespace_renameTableNative( ok_or_throw_with_return!( env, call_rest_namespace_method(&mut env, handle, request_json, |namespace_client, req| { - RT.block_on(namespace_client.inner.rename_table(req)) + block_on(namespace_client.inner.rename_table(req)) }), std::ptr::null_mut() ) @@ -2842,7 +2841,7 @@ pub extern "system" fn Java_org_lance_namespace_RestNamespace_insertIntoTableNat request_json, request_data, |namespace_client, req, data| { - RT.block_on(namespace_client.inner.insert_into_table(req, data)) + block_on(namespace_client.inner.insert_into_table(req, data)) } ), std::ptr::null_mut() @@ -2866,7 +2865,7 @@ pub extern "system" fn Java_org_lance_namespace_RestNamespace_mergeInsertIntoTab request_json, request_data, |namespace_client, req, data| { - RT.block_on(namespace_client.inner.merge_insert_into_table(req, data)) + block_on(namespace_client.inner.merge_insert_into_table(req, data)) } ), std::ptr::null_mut() @@ -2884,7 +2883,7 @@ pub extern "system" fn Java_org_lance_namespace_RestNamespace_updateTableNative( ok_or_throw_with_return!( env, call_rest_namespace_method(&mut env, handle, request_json, |namespace_client, req| { - RT.block_on(namespace_client.inner.update_table(req)) + block_on(namespace_client.inner.update_table(req)) }), std::ptr::null_mut() ) @@ -2901,7 +2900,7 @@ pub extern "system" fn Java_org_lance_namespace_RestNamespace_deleteFromTableNat ok_or_throw_with_return!( env, call_rest_namespace_method(&mut env, handle, request_json, |namespace_client, req| { - RT.block_on(namespace_client.inner.delete_from_table(req)) + block_on(namespace_client.inner.delete_from_table(req)) }), std::ptr::null_mut() ) @@ -2933,7 +2932,7 @@ pub extern "system" fn Java_org_lance_namespace_RestNamespace_createTableIndexNa ok_or_throw_with_return!( env, call_rest_namespace_method(&mut env, handle, request_json, |namespace_client, req| { - RT.block_on(namespace_client.inner.create_table_index(req)) + block_on(namespace_client.inner.create_table_index(req)) }), std::ptr::null_mut() ) @@ -2950,7 +2949,7 @@ pub extern "system" fn Java_org_lance_namespace_RestNamespace_listTableIndicesNa ok_or_throw_with_return!( env, call_rest_namespace_method(&mut env, handle, request_json, |namespace_client, req| { - RT.block_on(namespace_client.inner.list_table_indices(req)) + block_on(namespace_client.inner.list_table_indices(req)) }), std::ptr::null_mut() ) @@ -2967,7 +2966,7 @@ pub extern "system" fn Java_org_lance_namespace_RestNamespace_describeTableIndex ok_or_throw_with_return!( env, call_rest_namespace_method(&mut env, handle, request_json, |namespace_client, req| { - RT.block_on(namespace_client.inner.describe_table_index_stats(req)) + block_on(namespace_client.inner.describe_table_index_stats(req)) }), std::ptr::null_mut() ) @@ -2984,7 +2983,7 @@ pub extern "system" fn Java_org_lance_namespace_RestNamespace_describeTransactio ok_or_throw_with_return!( env, call_rest_namespace_method(&mut env, handle, request_json, |namespace_client, req| { - RT.block_on(namespace_client.inner.describe_transaction(req)) + block_on(namespace_client.inner.describe_transaction(req)) }), std::ptr::null_mut() ) @@ -3001,7 +3000,7 @@ pub extern "system" fn Java_org_lance_namespace_RestNamespace_alterTransactionNa ok_or_throw_with_return!( env, call_rest_namespace_method(&mut env, handle, request_json, |namespace_client, req| { - RT.block_on(namespace_client.inner.alter_transaction(req)) + block_on(namespace_client.inner.alter_transaction(req)) }), std::ptr::null_mut() ) @@ -3018,7 +3017,7 @@ pub extern "system" fn Java_org_lance_namespace_RestNamespace_listTableVersionsN ok_or_throw_with_return!( env, call_rest_namespace_method(&mut env, handle, request_json, |namespace_client, req| { - RT.block_on(namespace_client.inner.list_table_versions(req)) + block_on(namespace_client.inner.list_table_versions(req)) }), std::ptr::null_mut() ) @@ -3035,7 +3034,7 @@ pub extern "system" fn Java_org_lance_namespace_RestNamespace_createTableVersion ok_or_throw_with_return!( env, call_rest_namespace_method(&mut env, handle, request_json, |namespace_client, req| { - RT.block_on(namespace_client.inner.create_table_version(req)) + block_on(namespace_client.inner.create_table_version(req)) }), std::ptr::null_mut() ) @@ -3052,7 +3051,7 @@ pub extern "system" fn Java_org_lance_namespace_RestNamespace_describeTableVersi ok_or_throw_with_return!( env, call_rest_namespace_method(&mut env, handle, request_json, |namespace_client, req| { - RT.block_on(namespace_client.inner.describe_table_version(req)) + block_on(namespace_client.inner.describe_table_version(req)) }), std::ptr::null_mut() ) @@ -3069,7 +3068,7 @@ pub extern "system" fn Java_org_lance_namespace_RestNamespace_batchDeleteTableVe ok_or_throw_with_return!( env, call_rest_namespace_method(&mut env, handle, request_json, |namespace_client, req| { - RT.block_on(namespace_client.inner.batch_delete_table_versions(req)) + block_on(namespace_client.inner.batch_delete_table_versions(req)) }), std::ptr::null_mut() ) @@ -3086,7 +3085,7 @@ pub extern "system" fn Java_org_lance_namespace_RestNamespace_createTableScalarI ok_or_throw_with_return!( env, call_rest_namespace_method(&mut env, handle, request_json, |namespace_client, req| { - RT.block_on(namespace_client.inner.create_table_scalar_index(req)) + block_on(namespace_client.inner.create_table_scalar_index(req)) }), std::ptr::null_mut() ) @@ -3103,7 +3102,7 @@ pub extern "system" fn Java_org_lance_namespace_RestNamespace_dropTableIndexNati ok_or_throw_with_return!( env, call_rest_namespace_method(&mut env, handle, request_json, |namespace_client, req| { - RT.block_on(namespace_client.inner.drop_table_index(req)) + block_on(namespace_client.inner.drop_table_index(req)) }), std::ptr::null_mut() ) @@ -3120,7 +3119,7 @@ pub extern "system" fn Java_org_lance_namespace_RestNamespace_listAllTablesNativ ok_or_throw_with_return!( env, call_rest_namespace_method(&mut env, handle, request_json, |namespace_client, req| { - RT.block_on(namespace_client.inner.list_all_tables(req)) + block_on(namespace_client.inner.list_all_tables(req)) }), std::ptr::null_mut() ) @@ -3137,7 +3136,7 @@ pub extern "system" fn Java_org_lance_namespace_RestNamespace_restoreTableNative ok_or_throw_with_return!( env, call_rest_namespace_method(&mut env, handle, request_json, |namespace_client, req| { - RT.block_on(namespace_client.inner.restore_table(req)) + block_on(namespace_client.inner.restore_table(req)) }), std::ptr::null_mut() ) @@ -3154,7 +3153,7 @@ pub extern "system" fn Java_org_lance_namespace_RestNamespace_updateTableSchemaM ok_or_throw_with_return!( env, call_rest_namespace_method(&mut env, handle, request_json, |namespace_client, req| { - RT.block_on(namespace_client.inner.update_table_schema_metadata(req)) + block_on(namespace_client.inner.update_table_schema_metadata(req)) }), std::ptr::null_mut() ) @@ -3171,7 +3170,7 @@ pub extern "system" fn Java_org_lance_namespace_RestNamespace_getTableStatsNativ ok_or_throw_with_return!( env, call_rest_namespace_method(&mut env, handle, request_json, |namespace_client, req| { - RT.block_on(namespace_client.inner.get_table_stats(req)) + block_on(namespace_client.inner.get_table_stats(req)) }), std::ptr::null_mut() ) @@ -3192,7 +3191,7 @@ pub extern "system" fn Java_org_lance_namespace_RestNamespace_explainTableQueryP handle, request_json, |namespace_client, req| { - RT.block_on(namespace_client.inner.explain_table_query_plan(req)) + block_on(namespace_client.inner.explain_table_query_plan(req)) } ), std::ptr::null_mut() @@ -3214,7 +3213,7 @@ pub extern "system" fn Java_org_lance_namespace_RestNamespace_analyzeTableQueryP handle, request_json, |namespace_client, req| { - RT.block_on(namespace_client.inner.analyze_table_query_plan(req)) + block_on(namespace_client.inner.analyze_table_query_plan(req)) } ), std::ptr::null_mut() @@ -3232,7 +3231,7 @@ pub extern "system" fn Java_org_lance_namespace_RestNamespace_alterTableAddColum ok_or_throw_with_return!( env, call_rest_namespace_method(&mut env, handle, request_json, |namespace_client, req| { - RT.block_on(namespace_client.inner.alter_table_add_columns(req)) + block_on(namespace_client.inner.alter_table_add_columns(req)) }), std::ptr::null_mut() ) @@ -3249,7 +3248,7 @@ pub extern "system" fn Java_org_lance_namespace_RestNamespace_alterTableAlterCol ok_or_throw_with_return!( env, call_rest_namespace_method(&mut env, handle, request_json, |namespace_client, req| { - RT.block_on(namespace_client.inner.alter_table_alter_columns(req)) + block_on(namespace_client.inner.alter_table_alter_columns(req)) }), std::ptr::null_mut() ) @@ -3266,7 +3265,7 @@ pub extern "system" fn Java_org_lance_namespace_RestNamespace_alterTableDropColu ok_or_throw_with_return!( env, call_rest_namespace_method(&mut env, handle, request_json, |namespace_client, req| { - RT.block_on(namespace_client.inner.alter_table_drop_columns(req)) + block_on(namespace_client.inner.alter_table_drop_columns(req)) }), std::ptr::null_mut() ) @@ -3283,7 +3282,7 @@ pub extern "system" fn Java_org_lance_namespace_RestNamespace_alterTableBackfill ok_or_throw_with_return!( env, call_rest_namespace_method(&mut env, handle, request_json, |namespace_client, req| { - RT.block_on(namespace_client.inner.alter_table_backfill_columns(req)) + block_on(namespace_client.inner.alter_table_backfill_columns(req)) }), std::ptr::null_mut() ) @@ -3300,7 +3299,7 @@ pub extern "system" fn Java_org_lance_namespace_RestNamespace_refreshMaterialize ok_or_throw_with_return!( env, call_rest_namespace_method(&mut env, handle, request_json, |namespace_client, req| { - RT.block_on(namespace_client.inner.refresh_materialized_view(req)) + block_on(namespace_client.inner.refresh_materialized_view(req)) }), std::ptr::null_mut() ) @@ -3317,7 +3316,7 @@ pub extern "system" fn Java_org_lance_namespace_RestNamespace_listTableTagsNativ ok_or_throw_with_return!( env, call_rest_namespace_method(&mut env, handle, request_json, |namespace_client, req| { - RT.block_on(namespace_client.inner.list_table_tags(req)) + block_on(namespace_client.inner.list_table_tags(req)) }), std::ptr::null_mut() ) @@ -3334,7 +3333,7 @@ pub extern "system" fn Java_org_lance_namespace_RestNamespace_getTableTagVersion ok_or_throw_with_return!( env, call_rest_namespace_method(&mut env, handle, request_json, |namespace_client, req| { - RT.block_on(namespace_client.inner.get_table_tag_version(req)) + block_on(namespace_client.inner.get_table_tag_version(req)) }), std::ptr::null_mut() ) @@ -3351,7 +3350,7 @@ pub extern "system" fn Java_org_lance_namespace_RestNamespace_createTableTagNati ok_or_throw_with_return!( env, call_rest_namespace_method(&mut env, handle, request_json, |namespace_client, req| { - RT.block_on(namespace_client.inner.create_table_tag(req)) + block_on(namespace_client.inner.create_table_tag(req)) }), std::ptr::null_mut() ) @@ -3368,7 +3367,7 @@ pub extern "system" fn Java_org_lance_namespace_RestNamespace_deleteTableTagNati ok_or_throw_with_return!( env, call_rest_namespace_method(&mut env, handle, request_json, |namespace_client, req| { - RT.block_on(namespace_client.inner.delete_table_tag(req)) + block_on(namespace_client.inner.delete_table_tag(req)) }), std::ptr::null_mut() ) @@ -3385,7 +3384,7 @@ pub extern "system" fn Java_org_lance_namespace_RestNamespace_updateTableTagNati ok_or_throw_with_return!( env, call_rest_namespace_method(&mut env, handle, request_json, |namespace_client, req| { - RT.block_on(namespace_client.inner.update_table_tag(req)) + block_on(namespace_client.inner.update_table_tag(req)) }), std::ptr::null_mut() ) @@ -3402,7 +3401,7 @@ pub extern "system" fn Java_org_lance_namespace_RestNamespace_createMaterialized ok_or_throw_with_return!( env, call_rest_namespace_method(&mut env, handle, request_json, |namespace_client, req| { - RT.block_on(namespace_client.inner.create_materialized_view(req)) + block_on(namespace_client.inner.create_materialized_view(req)) }), std::ptr::null_mut() ) @@ -3522,9 +3521,7 @@ fn call_namespace_count_method( let request: CountTableRowsRequest = serde_json::from_str(&request_str) .map_err(|e| Error::input_error(format!("Failed to parse request JSON: {}", e)))?; - let count = RT - .block_on(namespace.inner.count_table_rows(request)) - .map_err(Error::from)?; + let count = block_on(namespace.inner.count_table_rows(request)).map_err(Error::from)?; Ok(count) } @@ -3569,9 +3566,7 @@ fn call_namespace_query_method<'local>( let request: QueryTableRequest = serde_json::from_str(&request_str) .map_err(|e| Error::input_error(format!("Failed to parse request JSON: {}", e)))?; - let result_bytes = RT - .block_on(namespace.inner.query_table(request)) - .map_err(Error::from)?; + let result_bytes = block_on(namespace.inner.query_table(request)).map_err(Error::from)?; let byte_array = env.byte_array_from_slice(&result_bytes)?; Ok(byte_array) @@ -3655,9 +3650,7 @@ fn call_rest_namespace_count_method( let request: CountTableRowsRequest = serde_json::from_str(&request_str) .map_err(|e| Error::input_error(format!("Failed to parse request JSON: {}", e)))?; - let count = RT - .block_on(namespace.inner.count_table_rows(request)) - .map_err(Error::from)?; + let count = block_on(namespace.inner.count_table_rows(request)).map_err(Error::from)?; Ok(count) } @@ -3702,9 +3695,7 @@ fn call_rest_namespace_query_method<'local>( let request: QueryTableRequest = serde_json::from_str(&request_str) .map_err(|e| Error::input_error(format!("Failed to parse request JSON: {}", e)))?; - let result_bytes = RT - .block_on(namespace.inner.query_table(request)) - .map_err(Error::from)?; + let result_bytes = block_on(namespace.inner.query_table(request)).map_err(Error::from)?; let byte_array = env.byte_array_from_slice(&result_bytes)?; Ok(byte_array) @@ -3756,8 +3747,7 @@ fn create_rest_adapter_internal( builder = builder.property(k, v); } - let backend = RT - .block_on(builder.connect()) + let backend = block_on(builder.connect()) .map_err(|e| Error::runtime_error(format!("Failed to build backend namespace: {}", e)))?; // Build config with defaults, overriding if values provided @@ -3799,7 +3789,7 @@ pub extern "system" fn Java_org_lance_namespace_RestAdapter_start( fn start_internal(handle: jlong) -> Result<()> { let adapter = unsafe { &mut *(handle as *mut BlockingRestAdapter) }; let rest_adapter = RestAdapter::new(adapter.backend.clone(), adapter.config.clone()); - let server_handle = RT.block_on(rest_adapter.start())?; + let server_handle = block_on(rest_adapter.start())?; adapter.server_handle = Some(server_handle); Ok(()) } diff --git a/java/lance-jni/src/optimize.rs b/java/lance-jni/src/optimize.rs index 0ce92baeec8..11281b69b0b 100644 --- a/java/lance-jni/src/optimize.rs +++ b/java/lance-jni/src/optimize.rs @@ -17,7 +17,7 @@ use lance::dataset::{ }; use crate::{ - RT, + block_on, blocking_dataset::{BlockingDataset, NATIVE_DATASET}, traits::{ FromJObjectWithEnv, IntoJava, export_vec, import_vec_from_method, import_vec_to_rust, @@ -108,7 +108,7 @@ fn inner_plan_compaction<'local>( let plan = { let dataset = unsafe { env.get_rust_field::<_, _, BlockingDataset>(java_dataset, NATIVE_DATASET) }?; - RT.block_on(plan_compaction(&dataset.inner, &compaction_options))? + block_on(plan_compaction(&dataset.inner, &compaction_options))? }; plan.into_java(env) } @@ -197,7 +197,7 @@ fn inner_commit_compaction<'local>( let committed_metrics = { let mut dataset = unsafe { env.get_rust_field::<_, _, BlockingDataset>(java_dataset, NATIVE_DATASET) }?; - RT.block_on(commit_compaction( + block_on(commit_compaction( &mut dataset.inner, completed_tasks, remap_options, @@ -296,7 +296,7 @@ fn inner_execute_task<'local>( let rewrite_result = { let dataset = unsafe { env.get_rust_field::<_, _, BlockingDataset>(java_dataset, NATIVE_DATASET) }?; - RT.block_on(compaction_task.execute(&dataset.inner))? + block_on(compaction_task.execute(&dataset.inner))? }; rewrite_result.into_java(env) } @@ -312,6 +312,7 @@ const REWRITE_RESULT_CLASS: &str = "org/lance/compaction/RewriteResult"; const REWRITE_RESULT_CONSTRUCTOR_SIG: &str = "(Lorg/lance/compaction/CompactionMetrics;Ljava/util/List;Ljava/util/List;J[B)V"; const COMPACTION_OPTIONS_CLASS: &str = "org/lance/compaction/CompactionOptions"; +const COMPACTION_MODE_CLASS: &str = "org/lance/compaction/CompactionMode"; const COMPACTION_OPTIONS_CONSTRUCTOR_SIG: &str = "(Ljava/util/Optional;Ljava/util/Optional;Ljava/util/Optional;Ljava/util/Optional;Ljava/util/Optional;Ljava/util/Optional;Ljava/util/Optional;Ljava/util/Optional;Ljava/util/Optional;Ljava/util/Optional;Ljava/util/Optional;)V"; impl IntoJava for &TaskData { @@ -361,13 +362,20 @@ impl IntoJava for &CompactionOptions { let batch_size_opt = to_java_optional(env, batch_size)?; let defer_index_remap = to_java_boolean_obj(env, Some(self.defer_index_remap))?; let defer_index_remap_opt = to_java_optional(env, defer_index_remap)?; - let compaction_mode_str = self.compaction_mode.as_ref().map(|mode| match mode { - CompactionMode::Reencode => "reencode", - CompactionMode::TryBinaryCopy => "try_binary_copy", - CompactionMode::ForceBinaryCopy => "force_binary_copy", - }); - let compaction_mode_obj = match compaction_mode_str { - Some(s) => env.new_string(s)?.into(), + let compaction_mode_obj = match self.compaction_mode { + Some(mode) => { + let name = match mode { + CompactionMode::Reencode => "REENCODE", + CompactionMode::TryBinaryCopy => "TRY_BINARY_COPY", + CompactionMode::ForceBinaryCopy => "FORCE_BINARY_COPY", + }; + env.get_static_field( + COMPACTION_MODE_CLASS, + name, + format!("L{};", COMPACTION_MODE_CLASS), + )? + .l()? + } None => JObject::null(), }; let compaction_mode_opt = to_java_optional(env, compaction_mode_obj)?; diff --git a/java/lance-jni/src/otel.rs b/java/lance-jni/src/otel.rs new file mode 100644 index 00000000000..54f46c7955a --- /dev/null +++ b/java/lance-jni/src/otel.rs @@ -0,0 +1,541 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright The Lance Authors + +//! Bridge from the [`metrics`] crate facade to Java OpenTelemetry. +//! +//! The Java layer owns the actual OpenTelemetry instruments. This module +//! installs the process-global Rust [`metrics::Recorder`], keeps cumulative +//! metric state, and exposes catalog/snapshot calls over JNI. + +use std::collections::HashMap; +use std::sync::atomic::Ordering; +use std::sync::{Arc, LazyLock, Mutex, OnceLock, RwLock}; + +use jni::JNIEnv; +use jni::objects::{JClass, JObject, JValue}; +use jni::sys::{jboolean, jobject}; +use metrics::atomics::AtomicU64; +use metrics::{Counter, Gauge, Histogram, Key, KeyName, Metadata, Recorder, SharedString, Unit}; +use metrics_util::registry::{Registry, Storage}; + +use crate::error::Result; + +const DEFAULT_BOUNDS: &[f64] = &[ + 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1.0, 2.5, 5.0, 10.0, 30.0, 60.0, 120.0, 300.0, +]; + +#[derive(Clone, Copy)] +enum MetricKind { + Counter, + Gauge, + Histogram, +} + +impl MetricKind { + fn as_str(self) -> &'static str { + match self { + Self::Counter => "counter", + Self::Gauge => "gauge", + Self::Histogram => "histogram", + } + } +} + +struct MetricDescription { + kind: MetricKind, + unit: Option, + description: String, +} + +static CATALOG: LazyLock>> = + LazyLock::new(|| Mutex::new(HashMap::new())); + +static HISTOGRAM_BOUNDS: LazyLock>>> = + LazyLock::new(|| RwLock::new(HashMap::new())); + +static REGISTRY: OnceLock>> = OnceLock::new(); + +fn bounds_for(name: &str) -> Arc<[f64]> { + HISTOGRAM_BOUNDS + .read() + .unwrap() + .get(name) + .cloned() + .unwrap_or_else(|| Arc::from(DEFAULT_BOUNDS)) +} + +struct BucketedHistogram { + bounds: Arc<[f64]>, + counts: Box<[AtomicU64]>, + count: AtomicU64, + sum_bits: AtomicU64, +} + +impl BucketedHistogram { + fn new(bounds: Arc<[f64]>) -> Self { + let counts = (0..=bounds.len()) + .map(|_| AtomicU64::new(0)) + .collect::>(); + Self { + bounds, + counts, + count: AtomicU64::new(0), + sum_bits: AtomicU64::new(0), + } + } + + fn add_to_sum(&self, value: f64) { + let mut current = self.sum_bits.load(Ordering::Relaxed); + loop { + let updated = (f64::from_bits(current) + value).to_bits(); + match self.sum_bits.compare_exchange_weak( + current, + updated, + Ordering::Relaxed, + Ordering::Relaxed, + ) { + Ok(_) => break, + Err(actual) => current = actual, + } + } + } + + fn snapshot(&self) -> MetricValue { + let mut cumulative = 0u64; + let mut buckets = Vec::with_capacity(self.bounds.len() + 1); + for (i, bound) in self.bounds.iter().enumerate() { + cumulative += self.counts[i].load(Ordering::Relaxed); + buckets.push((bound.to_string(), cumulative)); + } + cumulative += self.counts[self.bounds.len()].load(Ordering::Relaxed); + buckets.push(("+Inf".to_string(), cumulative)); + MetricValue::Histogram { + buckets, + count: self.count.load(Ordering::Relaxed), + sum: f64::from_bits(self.sum_bits.load(Ordering::Relaxed)), + } + } +} + +impl metrics::HistogramFn for BucketedHistogram { + fn record(&self, value: f64) { + let idx = self.bounds.partition_point(|&bound| bound < value); + self.counts[idx].fetch_add(1, Ordering::Relaxed); + self.count.fetch_add(1, Ordering::Relaxed); + self.add_to_sum(value); + } +} + +struct LanceStorage; + +impl Storage for LanceStorage { + type Counter = Arc; + type Gauge = Arc; + type Histogram = Arc; + + fn counter(&self, _key: &Key) -> Self::Counter { + Arc::new(AtomicU64::new(0)) + } + + fn gauge(&self, _key: &Key) -> Self::Gauge { + Arc::new(AtomicU64::new(0)) + } + + fn histogram(&self, key: &Key) -> Self::Histogram { + Arc::new(BucketedHistogram::new(bounds_for(key.name()))) + } +} + +struct LanceRecorder { + registry: Arc>, +} + +impl LanceRecorder { + fn describe( + &self, + key: KeyName, + kind: MetricKind, + unit: Option, + description: SharedString, + ) { + CATALOG.lock().unwrap().insert( + key.as_str().to_string(), + MetricDescription { + kind, + unit: unit.map(|u| u.as_canonical_label().to_string()), + description: description.into_owned(), + }, + ); + } +} + +impl Recorder for LanceRecorder { + fn describe_counter(&self, key: KeyName, unit: Option, description: SharedString) { + self.describe(key, MetricKind::Counter, unit, description); + } + + fn describe_gauge(&self, key: KeyName, unit: Option, description: SharedString) { + self.describe(key, MetricKind::Gauge, unit, description); + } + + fn describe_histogram(&self, key: KeyName, unit: Option, description: SharedString) { + self.describe(key, MetricKind::Histogram, unit, description); + } + + fn register_counter(&self, key: &Key, _metadata: &Metadata<'_>) -> Counter { + self.registry + .get_or_create_counter(key, |counter| Counter::from_arc(counter.clone())) + } + + fn register_gauge(&self, key: &Key, _metadata: &Metadata<'_>) -> Gauge { + self.registry + .get_or_create_gauge(key, |gauge| Gauge::from_arc(gauge.clone())) + } + + fn register_histogram(&self, key: &Key, _metadata: &Metadata<'_>) -> Histogram { + self.registry + .get_or_create_histogram(key, |histogram| Histogram::from_arc(histogram.clone())) + } +} + +fn register_bounds() { + let mut bounds = HISTOGRAM_BOUNDS.write().unwrap(); + for (name, values) in lance_io::object_store::metrics::histogram_bounds() { + bounds.insert((*name).to_string(), Arc::from(*values)); + } +} + +fn describe_all() { + lance_io::object_store::metrics::describe_metrics(); +} + +enum MetricValue { + Scalar(f64), + Histogram { + buckets: Vec<(String, u64)>, + count: u64, + sum: f64, + }, +} + +struct MetricPoint { + name: String, + kind: &'static str, + attributes: HashMap, + value: MetricValue, +} + +fn labels(key: &Key) -> HashMap { + key.labels() + .map(|label| (label.key().to_string(), label.value().to_string())) + .collect() +} + +fn collect_points(registry: &Registry) -> Vec { + let mut points = Vec::new(); + for (key, handle) in registry.get_counter_handles() { + points.push(MetricPoint { + name: key.name().to_string(), + kind: "counter", + attributes: labels(&key), + value: MetricValue::Scalar(handle.load(Ordering::Relaxed) as f64), + }); + } + for (key, handle) in registry.get_gauge_handles() { + points.push(MetricPoint { + name: key.name().to_string(), + kind: "gauge", + attributes: labels(&key), + value: MetricValue::Scalar(f64::from_bits(handle.load(Ordering::Relaxed))), + }); + } + for (key, handle) in registry.get_histogram_handles() { + points.push(MetricPoint { + name: key.name().to_string(), + kind: "histogram", + attributes: labels(&key), + value: handle.snapshot(), + }); + } + points +} + +fn register_lance_metrics_recorder() -> bool { + if REGISTRY.get().is_some() { + return true; + } + let registry = Arc::new(Registry::new(LanceStorage)); + let recorder = LanceRecorder { + registry: registry.clone(), + }; + register_bounds(); + match metrics::set_global_recorder(recorder) { + Ok(()) => { + let _ = REGISTRY.set(registry); + describe_all(); + true + } + Err(_) => false, + } +} + +fn object_list<'local>(env: &mut JNIEnv<'local>) -> Result> { + Ok(env.new_object("java/util/ArrayList", "()V", &[])?) +} + +fn add_to_list(env: &mut JNIEnv, list: &JObject, item: &JObject) -> Result<()> { + env.call_method( + list, + "add", + "(Ljava/lang/Object;)Z", + &[JValue::Object(item)], + )?; + Ok(()) +} + +fn string_object<'local>(env: &mut JNIEnv<'local>, value: Option<&str>) -> Result> { + match value { + Some(value) => Ok(env.new_string(value)?.into()), + None => Ok(JObject::null()), + } +} + +fn attributes_to_java<'local>( + env: &mut JNIEnv<'local>, + attributes: &HashMap, +) -> Result> { + let java_map = env.new_object("java/util/HashMap", "()V", &[])?; + for (key, value) in attributes { + let java_key = env.new_string(key)?; + let java_value = env.new_string(value)?; + env.call_method( + &java_map, + "put", + "(Ljava/lang/Object;Ljava/lang/Object;)Ljava/lang/Object;", + &[JValue::Object(&java_key), JValue::Object(&java_value)], + )?; + env.delete_local_ref(java_key)?; + env.delete_local_ref(java_value)?; + } + Ok(java_map) +} + +fn buckets_to_java<'local>( + env: &mut JNIEnv<'local>, + buckets: Option<&[(String, u64)]>, +) -> Result> { + let Some(buckets) = buckets else { + return Ok(JObject::null()); + }; + let list = object_list(env)?; + for (le, cumulative_count) in buckets { + let le = env.new_string(le)?; + let bucket = env.new_object( + "org/lance/otel/MetricBucket", + "(Ljava/lang/String;J)V", + &[JValue::Object(&le), JValue::Long(*cumulative_count as i64)], + )?; + add_to_list(env, &list, &bucket)?; + env.delete_local_ref(le)?; + env.delete_local_ref(bucket)?; + } + Ok(list) +} + +fn boxed_double<'local>(env: &mut JNIEnv<'local>, value: Option) -> Result> { + match value { + Some(value) => Ok(env.new_object("java/lang/Double", "(D)V", &[JValue::Double(value)])?), + None => Ok(JObject::null()), + } +} + +fn boxed_long<'local>(env: &mut JNIEnv<'local>, value: Option) -> Result> { + match value { + Some(value) => { + Ok(env.new_object("java/lang/Long", "(J)V", &[JValue::Long(value as i64)])?) + } + None => Ok(JObject::null()), + } +} + +fn metric_description_to_java<'local>( + env: &mut JNIEnv<'local>, + name: &str, + desc: &MetricDescription, +) -> Result> { + let name = env.new_string(name)?; + let kind = env.new_string(desc.kind.as_str())?; + let unit = string_object(env, desc.unit.as_deref())?; + let description = env.new_string(&desc.description)?; + Ok(env.new_object( + "org/lance/otel/MetricDescription", + "(Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;)V", + &[ + JValue::Object(&name), + JValue::Object(&kind), + JValue::Object(&unit), + JValue::Object(&description), + ], + )?) +} + +fn metric_point_to_java<'local>( + env: &mut JNIEnv<'local>, + point: MetricPoint, +) -> Result> { + let (value, buckets, count, sum) = match point.value { + MetricValue::Scalar(value) => (Some(value), None, None, None), + MetricValue::Histogram { + buckets, + count, + sum, + } => (None, Some(buckets), Some(count), Some(sum)), + }; + let name = env.new_string(&point.name)?; + let kind = env.new_string(point.kind)?; + let attributes = attributes_to_java(env, &point.attributes)?; + let value = boxed_double(env, value)?; + let buckets = buckets_to_java(env, buckets.as_deref())?; + let count = boxed_long(env, count)?; + let sum = boxed_double(env, sum)?; + Ok(env.new_object( + "org/lance/otel/MetricPoint", + "(Ljava/lang/String;Ljava/lang/String;Ljava/util/Map;Ljava/lang/Double;Ljava/util/List;Ljava/lang/Long;Ljava/lang/Double;)V", + &[ + JValue::Object(&name), + JValue::Object(&kind), + JValue::Object(&attributes), + JValue::Object(&value), + JValue::Object(&buckets), + JValue::Object(&count), + JValue::Object(&sum), + ], + )?) +} + +fn lance_metrics_catalog_native<'local>(env: &mut JNIEnv<'local>) -> Result> { + let catalog = CATALOG.lock().unwrap(); + let list = object_list(env)?; + for (name, desc) in catalog.iter() { + env.with_local_frame(16, |env| { + let item = metric_description_to_java(env, name, desc)?; + add_to_list(env, &list, &item) + })?; + } + Ok(list) +} + +fn snapshot_lance_metrics_native<'local>(env: &mut JNIEnv<'local>) -> Result> { + let list = object_list(env)?; + if let Some(registry) = REGISTRY.get() { + for point in collect_points(registry) { + env.with_local_frame(64, |env| { + let item = metric_point_to_java(env, point)?; + add_to_list(env, &list, &item) + })?; + } + } + Ok(list) +} + +#[unsafe(no_mangle)] +pub extern "system" fn Java_org_lance_otel_LanceMetrics_registerLanceMetricsRecorderNative( + _env: JNIEnv, + _class: JClass, +) -> jboolean { + register_lance_metrics_recorder() as jboolean +} + +#[unsafe(no_mangle)] +pub extern "system" fn Java_org_lance_otel_LanceMetrics_lanceMetricsCatalogNative( + mut env: JNIEnv, + _class: JClass, +) -> jobject { + ok_or_throw_with_return!( + env, + lance_metrics_catalog_native(&mut env), + std::ptr::null_mut() + ) + .into_raw() +} + +#[unsafe(no_mangle)] +pub extern "system" fn Java_org_lance_otel_LanceMetrics_snapshotLanceMetricsNative( + mut env: JNIEnv, + _class: JClass, +) -> jobject { + ok_or_throw_with_return!( + env, + snapshot_lance_metrics_native(&mut env), + std::ptr::null_mut() + ) + .into_raw() +} + +#[cfg(test)] +mod tests { + use super::*; + use metrics::HistogramFn; + + fn bucket_count(buckets: &[(String, u64)], le: &str) -> u64 { + buckets + .iter() + .find(|(bound, _)| bound == le) + .map(|(_, count)| *count) + .unwrap_or_else(|| panic!("no bucket with le={le}")) + } + + #[test] + fn test_bucketed_histogram_records_cumulative_buckets() { + let histogram = BucketedHistogram::new(Arc::from([0.1f64, 1.0, 10.0].as_slice())); + histogram.record(0.05); + histogram.record(0.1); + histogram.record(0.5); + histogram.record(1.0); + histogram.record(5.0); + histogram.record(50.0); + + let MetricValue::Histogram { + buckets, + count, + sum, + } = histogram.snapshot() + else { + panic!("expected histogram"); + }; + + assert_eq!(bucket_count(&buckets, "0.1"), 2); + assert_eq!(bucket_count(&buckets, "1"), 4); + assert_eq!(bucket_count(&buckets, "10"), 5); + assert_eq!(bucket_count(&buckets, "+Inf"), 6); + assert_eq!(bucket_count(&buckets, "+Inf"), count); + assert!((sum - 56.65).abs() < 1e-9); + } + + #[test] + fn test_registry_aggregates_counters_and_gauges() { + let registry = Arc::new(Registry::new(LanceStorage)); + let recorder = LanceRecorder { + registry: registry.clone(), + }; + metrics::with_local_recorder(&recorder, || { + metrics::counter!("test_counter", "operation" => "get").increment(2); + metrics::counter!("test_counter", "operation" => "get").increment(3); + metrics::gauge!("test_gauge", "operation" => "get").increment(3.5); + metrics::gauge!("test_gauge", "operation" => "get").decrement(1.25); + }); + + let points = collect_points(®istry); + let counter = points + .iter() + .find(|point| point.name == "test_counter") + .expect("counter recorded"); + let gauge = points + .iter() + .find(|point| point.name == "test_gauge") + .expect("gauge recorded"); + + assert!(matches!(counter.value, MetricValue::Scalar(value) if value == 5.0)); + assert!(matches!(gauge.value, MetricValue::Scalar(value) if (value - 2.25).abs() < 1e-9)); + } +} diff --git a/java/lance-jni/src/session.rs b/java/lance-jni/src/session.rs index 17b59c12ecb..6afae56f43a 100644 --- a/java/lance-jni/src/session.rs +++ b/java/lance-jni/src/session.rs @@ -4,13 +4,15 @@ use std::sync::Arc; use jni::JNIEnv; -use jni::objects::JObject; +use jni::objects::{JMap, JObject, JString, JValue}; use jni::sys::jlong; -use lance::dataset::{DEFAULT_INDEX_CACHE_SIZE, DEFAULT_METADATA_CACHE_SIZE}; -use lance::session::Session as LanceSession; +use lance::session::{CacheSpec, Session as LanceSession}; +use lance_core::cache::{BackendConfig, build_from_config, build_from_uri}; use lance_io::object_store::ObjectStoreRegistry; +use crate::block_on; use crate::error::{Error, Result}; +use crate::utils::to_rust_map; /// Creates a new Session and returns a handle to it. /// @@ -22,33 +24,64 @@ pub extern "system" fn Java_org_lance_Session_createNative( _obj: JObject, index_cache_size_bytes: jlong, metadata_cache_size_bytes: jlong, + index_cache_backend_uri: JString, + index_cache_backend_kind: JString, + index_cache_backend_options: JObject, + metadata_cache_backend_uri: JString, + metadata_cache_backend_kind: JString, + metadata_cache_backend_options: JObject, ) -> jlong { ok_or_throw_with_return!( env, - create_session(index_cache_size_bytes, metadata_cache_size_bytes), + create_session( + &mut env, + index_cache_size_bytes, + metadata_cache_size_bytes, + index_cache_backend_uri, + index_cache_backend_kind, + index_cache_backend_options, + metadata_cache_backend_uri, + metadata_cache_backend_kind, + metadata_cache_backend_options, + ), 0 ) } +#[allow(clippy::too_many_arguments)] fn create_session( + env: &mut JNIEnv, index_cache_size_bytes: jlong, metadata_cache_size_bytes: jlong, + index_cache_backend_uri: JString, + index_cache_backend_kind: JString, + index_cache_backend_options: JObject, + metadata_cache_backend_uri: JString, + metadata_cache_backend_kind: JString, + metadata_cache_backend_options: JObject, ) -> Result { - let index_cache_size = if index_cache_size_bytes >= 0 { - index_cache_size_bytes as usize - } else { - DEFAULT_INDEX_CACHE_SIZE - }; - - let metadata_cache_size = if metadata_cache_size_bytes >= 0 { - metadata_cache_size_bytes as usize - } else { - DEFAULT_METADATA_CACHE_SIZE - }; + let index_cache = resolve_cache_spec( + env, + "indexCacheBackend", + "indexCacheSizeBytes", + index_cache_size_bytes, + index_cache_backend_uri, + index_cache_backend_kind, + index_cache_backend_options, + )?; + let metadata_cache = resolve_cache_spec( + env, + "metadataCacheBackend", + "metadataCacheSizeBytes", + metadata_cache_size_bytes, + metadata_cache_backend_uri, + metadata_cache_backend_kind, + metadata_cache_backend_options, + )?; - let session = LanceSession::new( - index_cache_size, - metadata_cache_size, + let session = LanceSession::with_cache_backends( + index_cache, + metadata_cache, Arc::new(ObjectStoreRegistry::default()), ); @@ -58,6 +91,63 @@ fn create_session( Ok(handle) } +#[allow(clippy::too_many_arguments)] +fn resolve_cache_spec( + env: &mut JNIEnv, + backend_field: &str, + size_field: &str, + size_bytes: jlong, + backend_uri: JString, + backend_kind: JString, + backend_options: JObject, +) -> Result { + let has_uri = !backend_uri.is_null(); + let has_kind = !backend_kind.is_null(); + if has_uri && has_kind { + return Err(Error::input_error(format!( + "{} must use either a URI or a structured config, not both", + backend_field + ))); + } + if size_bytes >= 0 && (has_uri || has_kind) { + return Err(Error::input_error(format!( + "{} and {} are mutually exclusive; set one or the other", + size_field, backend_field + ))); + } + + if has_uri { + let uri: String = env.get_string(&backend_uri)?.into(); + return build_from_uri(&uri) + .map(CacheSpec::Backend) + .map_err(Error::from); + } + + if has_kind { + let kind: String = env.get_string(&backend_kind)?.into(); + let mut config = BackendConfig::new(&kind)?; + if !backend_options.is_null() { + let options = JMap::from_env(env, &backend_options)?; + config.options = to_rust_map(env, &options)?; + } + return build_from_config(&config) + .map(CacheSpec::Backend) + .map_err(Error::from); + } + + if size_bytes >= 0 { + let size = usize::try_from(size_bytes).map_err(|_| { + Error::input_error(format!( + "{} value {} does not fit in usize", + size_field, size_bytes + )) + })?; + Ok(CacheSpec::Size(size)) + } else { + Ok(CacheSpec::Default) + } +} + /// Returns the current size of the session in bytes. #[unsafe(no_mangle)] pub extern "system" fn Java_org_lance_Session_sizeBytesNative( @@ -78,6 +168,41 @@ fn size_bytes_native(env: &mut JNIEnv, obj: JObject) -> Result { Ok(session_arc.size_bytes() as jlong) } +/// Returns statistics for the session's metadata cache as an org.lance.CacheStats object. +#[unsafe(no_mangle)] +pub extern "system" fn Java_org_lance_Session_metadataCacheStatsNative<'local>( + mut env: JNIEnv<'local>, + obj: JObject, +) -> JObject<'local> { + ok_or_throw!(env, metadata_cache_stats_native(&mut env, obj)) +} + +fn metadata_cache_stats_native<'local>( + env: &mut JNIEnv<'local>, + obj: JObject, +) -> Result> { + let handle = get_session_handle(env, &obj)?; + if handle == 0 { + return Err(Error::input_error("Session is closed".to_string())); + } + + // Safety: We trust that the handle is valid and was created by createNative + let session_arc = unsafe { &*(handle as *const Arc) }; + let stats = block_on(session_arc.metadata_cache_stats()); + + let stats_obj = env.new_object( + "org/lance/CacheStats", + "(JJJJ)V", + &[ + JValue::Long(stats.hits as jlong), + JValue::Long(stats.misses as jlong), + JValue::Long(stats.num_entries as jlong), + JValue::Long(stats.size_bytes as jlong), + ], + )?; + Ok(stats_obj) +} + /// Releases the native session handle. #[unsafe(no_mangle)] pub extern "system" fn Java_org_lance_Session_releaseNative( diff --git a/java/lance-jni/src/sql.rs b/java/lance-jni/src/sql.rs index f378577eedf..2111d02c9fd 100644 --- a/java/lance-jni/src/sql.rs +++ b/java/lance-jni/src/sql.rs @@ -4,7 +4,7 @@ use crate::blocking_dataset::{BlockingDataset, NATIVE_DATASET}; use crate::error::Result; use crate::traits::FromJString; -use crate::{Error, JNIEnvExt, RT}; +use crate::{Error, JNIEnvExt, RT, block_on}; use arrow::ffi_stream::FFI_ArrowArrayStream; use jni::JNIEnv; use jni::objects::{JClass, JObject, JString}; @@ -57,7 +57,7 @@ fn inner_into_batch_records( with_row_addr, )?; - let stream = RT.block_on(async move { + let stream = block_on(async move { let query = builder.build().await?; query.into_stream().await })?; diff --git a/java/lance-jni/src/transaction.rs b/java/lance-jni/src/transaction.rs index 4f899f56ff2..5a4a295b0f2 100644 --- a/java/lance-jni/src/transaction.rs +++ b/java/lance-jni/src/transaction.rs @@ -3,7 +3,7 @@ use crate::Error; use crate::JNIEnvExt; -use crate::RT; +use crate::block_on; use crate::blocking_dataset::{BlockingDataset, NATIVE_DATASET, extract_namespace_info}; use crate::error::Result; use crate::traits::{ @@ -19,14 +19,14 @@ use jni::sys::{jboolean, jint, jlong}; use lance::dataset::CommitBuilder; use lance::dataset::transaction::{ DataReplacementGroup, Operation, RewriteGroup, RewrittenIndex, Transaction, TransactionBuilder, - UpdateMap, UpdateMapEntry, UpdateMode, + UpdateMap, UpdateMapEntry, UpdateMode, UpdatedFragmentOffsets, }; use lance::io::ObjectStoreParams; use lance::io::commit::namespace_manifest::LanceNamespaceExternalManifestStore; use lance::table::format::{Fragment, IndexMetadata}; use lance_core::datatypes::Field; use lance_core::datatypes::Schema as LanceSchema; -use lance_file::version::LanceFileVersion; +use lance_file::version::{LanceFileVersion, V2_FORMAT_2_0, V2_FORMAT_2_1, V2_FORMAT_2_2}; use lance_io::object_store::{LanceNamespaceStorageOptionsProvider, StorageOptionsProvider}; use lance_table::io::commit::CommitHandler; use lance_table::io::commit::external_manifest::ExternalManifestCommitHandler; @@ -411,6 +411,7 @@ fn convert_to_java_operation_inner<'local>( Operation::CreateIndex { new_indices, removed_indices, + .. } => { let java_new_indices = export_vec(env, &new_indices)?; let java_removed_indices = export_vec(env, &removed_indices)?; @@ -429,11 +430,11 @@ fn convert_to_java_operation_inner<'local>( updated_fragments, new_fragments, fields_modified, - merged_generations: _, + compacted_sstables: _, fields_for_preserving_frag_bitmap, update_mode, inserted_rows_filter: _, - updated_fragment_offsets: _, + updated_fragment_offsets, } => { let removed_ids: Vec> = removed_fragment_ids .iter() @@ -457,9 +458,48 @@ fn convert_to_java_operation_inner<'local>( &[JValue::Object(&update_mode)], )? .l()?; + // Serialize updated_fragment_offsets to Java Map. + // Values are portable RoaringBitmap bytes so the JNI boundary stays O(bitmap size) + // rather than O(n rows). Empty HashMap when None so the Java constructor always + // receives a non-null map. + let java_offsets_map = { + let java_map = env.new_object("java/util/HashMap", "()V", &[])?; + if let Some(UpdatedFragmentOffsets(ref map)) = updated_fragment_offsets { + for (frag_id, bitmap) in map { + let mut buf: Vec = Vec::new(); + bitmap.serialize_into(&mut buf).map_err(|e| { + Error::runtime_error(format!( + "failed to serialize updatedFragmentOffsets for fragment \ + {frag_id}: {e}" + )) + })?; + // JNI byte arrays are signed i8; reinterpret without copying. + let buf_i8: &[i8] = unsafe { + std::slice::from_raw_parts(buf.as_ptr() as *const i8, buf.len()) + }; + env.with_local_frame(4, |env| { + let java_key = env.new_object( + "java/lang/Long", + "(J)V", + &[JValue::Long(*frag_id as i64)], + )?; + let java_arr = env.new_byte_array(buf_i8.len() as i32)?; + env.set_byte_array_region(&java_arr, 0, buf_i8)?; + env.call_method( + &java_map, + "put", + "(Ljava/lang/Object;Ljava/lang/Object;)Ljava/lang/Object;", + &[JValue::Object(&java_key), JValue::Object(&*java_arr)], + )?; + Ok::(JObject::null()) + })?; + } + } + java_map + }; Ok(env.new_object( "org/lance/operation/Update", - "(Ljava/util/List;Ljava/util/List;Ljava/util/List;[J[JLjava/util/Optional;)V", + "(Ljava/util/List;Ljava/util/List;Ljava/util/List;[J[JLjava/util/Optional;Ljava/util/Map;)V", &[ JValue::Object(&removed_fragment_ids_obj), JValue::Object(&updated_fragments_obj), @@ -467,16 +507,23 @@ fn convert_to_java_operation_inner<'local>( JValueGen::Object(&fields_modified), JValueGen::Object(&fields_for_preserving_frag_bitmap), JValue::Object(&update_mode_optional), + JValue::Object(&java_offsets_map), ], )?) } - Operation::Project { schema } => { + Operation::Project { + schema, + preserves_nullability, + } => { let java_schema = convert_to_java_schema(env, schema)?; Ok(env.new_object( "org/lance/operation/Project", - "(Lorg/apache/arrow/vector/types/pojo/Schema;)V", - &[JValue::Object(&java_schema)], + "(Lorg/apache/arrow/vector/types/pojo/Schema;Z)V", + &[ + JValue::Object(&java_schema), + JValue::Bool(preserves_nullability as u8), + ], )?) } Operation::Rewrite { @@ -552,16 +599,18 @@ fn convert_to_java_operation_inner<'local>( Operation::Merge { fragments: rust_fragments, schema, + preserves_nullability, } => { let java_fragments = export_vec(env, &rust_fragments)?; let java_schema = convert_to_java_schema(env, schema)?; Ok(env.new_object( "org/lance/operation/Merge", - "(Ljava/util/List;Lorg/apache/arrow/vector/types/pojo/Schema;)V", + "(Ljava/util/List;Lorg/apache/arrow/vector/types/pojo/Schema;Z)V", &[ JValue::Object(&java_fragments), JValue::Object(&java_schema), + JValue::Bool(preserves_nullability as u8), ], )?) } @@ -594,19 +643,38 @@ pub(crate) fn convert_to_java_schema<'local>( .l()?) } +/// Parse a `CommitBuilder.storageFormat` string into a [`LanceFileVersion`]. +/// +/// The canonical spellings ("2.1", "stable", ...) are the ones every other Lance +/// binding accepts and the ones [`LanceFileVersion`]'s `Display` emits. +/// +/// The `v`-prefixed spellings are a Java-only accident: this function originally +/// hand-rolled its match by walking the `LanceFileVersion` variant identifiers +/// (`V2_1` -> `"v2_1"`) instead of delegating to `FromStr`, so it accepted those +/// identifiers and rejected the canonical "2.1". They were documented on +/// `CommitBuilder.storageFormat` and shipped from 3.0.0, so they are translated +/// here for compatibility. The set is deliberately frozen to what shipped — +/// newer versions are reachable only by their canonical name. fn parse_storage_format(name: &str) -> Result { - match name.to_lowercase().as_str() { - "legacy" => Ok(LanceFileVersion::Legacy), - "v2_0" | "v2.0" => Ok(LanceFileVersion::V2_0), - "stable" => Ok(LanceFileVersion::Stable), - "v2_1" | "v2.1" => Ok(LanceFileVersion::V2_1), - "next" => Ok(LanceFileVersion::Next), - "v2_2" | "v2.2" => Ok(LanceFileVersion::V2_2), - _ => Err(Error::input_error(format!( - "Unknown storage format: {}", - name - ))), + let requested = name.to_lowercase(); + let canonical = match requested.as_str() { + "v2_0" | "v2.0" => V2_FORMAT_2_0, + "v2_1" | "v2.1" => V2_FORMAT_2_1, + "v2_2" | "v2.2" => V2_FORMAT_2_2, + _ => requested.as_str(), + }; + + if canonical != requested { + log::warn!( + "Storage format \"{}\" is deprecated and will be removed in a future release; use \"{}\" instead", + name, + canonical + ); } + + canonical + .parse::() + .map_err(|_| Error::input_error(format!("Unknown storage format: {}", name))) } /// Translate the Java `commitTimeoutNanos` sentinel into an @@ -1025,6 +1093,8 @@ fn convert_to_rust_operation( let op_name = env.get_string_from_method(java_operation, "name")?; let op = match op_name.as_str() { "Project" => Operation::Project { + preserves_nullability: env + .get_boolean_from_method(java_operation, "preservesNullability")?, schema: convert_schema_from_operation( env, java_operation, @@ -1238,16 +1308,69 @@ fn convert_to_rust_operation( update_mode.extract_object(env) })?; + let updated_fragment_offsets = { + let offsets_obj = env + .call_method( + java_operation, + "updatedFragmentOffsets", + "()Ljava/util/Map;", + &[], + )? + .l()?; + if offsets_obj.is_null() { + None + } else { + let jmap = JMap::from_env(env, &offsets_obj)?; + let mut iter = jmap.iter(env)?; + let mut offsets: HashMap = HashMap::new(); + // Per-iteration local frame: iterator key/value JNI refs are released each + // loop so large multi-fragment maps cannot exhaust the local reference table. + loop { + let entry = env.with_local_frame( + 8, + |env| -> Result> { + let Some((key, value)) = iter.next(env)? else { + return Ok(None); + }; + let frag_id = + env.call_method(&key, "longValue", "()J", &[])?.j()? as u64; + let buf: Vec = + env.convert_byte_array(JByteArray::from(value))?; + let bitmap = RoaringBitmap::deserialize_from(buf.as_slice()) + .map_err(|e| { + Error::input_error(format!( + "invalid updatedFragmentOffsets RoaringBitmap bytes \ + for fragment {frag_id}: {e}" + )) + })?; + Ok(Some((frag_id, bitmap))) + }, + )?; + match entry { + None => break, + Some((frag_id, bitmap)) => { + offsets.insert(frag_id, bitmap); + } + } + } + if offsets.is_empty() { + None + } else { + Some(UpdatedFragmentOffsets(offsets)) + } + } + }; + Operation::Update { removed_fragment_ids, updated_fragments, new_fragments, fields_modified, - merged_generations: vec![], + compacted_sstables: vec![], fields_for_preserving_frag_bitmap, update_mode, inserted_rows_filter: None, - updated_fragment_offsets: None, + updated_fragment_offsets, } } "DataReplacement" => { @@ -1264,6 +1387,8 @@ fn convert_to_rust_operation( })?; Operation::Merge { fragments, + preserves_nullability: env + .get_boolean_from_method(java_operation, "preservesNullability")?, schema: convert_schema_from_operation( env, java_operation, @@ -1574,7 +1699,7 @@ fn inner_commit_to_uri<'local>( builder = builder.with_commit_handler(commit_handler); } - let dataset = RT.block_on(builder.execute(transaction))?; + let dataset = block_on(builder.execute(transaction))?; let blocking_ds = BlockingDataset { inner: dataset }; blocking_ds.into_java(env) } @@ -1836,4 +1961,80 @@ mod tests { HashMap::from([("new_schema_k".to_string(), "new_schema_v".to_string())]) ); } + + #[test] + fn test_parse_storage_format_canonical_forms() { + let cases = [ + ("2.0", LanceFileVersion::V2_0), + ("2.1", LanceFileVersion::V2_1), + ("2.2", LanceFileVersion::V2_2), + ("2.3", LanceFileVersion::V2_3), + ("0.1", LanceFileVersion::Legacy), + ("legacy", LanceFileVersion::Legacy), + ("stable", LanceFileVersion::Stable), + ("next", LanceFileVersion::Next), + ]; + for (input, expected) in cases { + assert_eq!( + parse_storage_format(input).unwrap(), + expected, + "parse_storage_format({:?}) failed", + input + ); + } + } + + /// The `v`-prefixed spellings shipped in the `CommitBuilder.storageFormat` + /// Javadoc and must keep working for existing Java callers. + #[test] + fn test_parse_storage_format_deprecated_aliases() { + let cases = [ + ("v2_0", LanceFileVersion::V2_0), + ("v2.0", LanceFileVersion::V2_0), + ("v2_1", LanceFileVersion::V2_1), + ("v2.1", LanceFileVersion::V2_1), + ("v2_2", LanceFileVersion::V2_2), + ("v2.2", LanceFileVersion::V2_2), + ]; + for (input, expected) in cases { + assert_eq!( + parse_storage_format(input).unwrap(), + expected, + "parse_storage_format({:?}) failed", + input + ); + } + } + + /// The alias set is frozen to what shipped, so versions added after the + /// aliases were deprecated are reachable only by their canonical name. + #[test] + fn test_parse_storage_format_does_not_extend_aliases_to_new_versions() { + assert!(parse_storage_format("v2_3").is_err()); + assert!(parse_storage_format("v2.3").is_err()); + assert_eq!(parse_storage_format("2.3").unwrap(), LanceFileVersion::V2_3); + } + + #[test] + fn test_parse_storage_format_case_insensitive() { + assert_eq!( + parse_storage_format("LEGACY").unwrap(), + LanceFileVersion::Legacy + ); + assert_eq!( + parse_storage_format("Stable").unwrap(), + LanceFileVersion::Stable + ); + assert_eq!( + parse_storage_format("V2_1").unwrap(), + LanceFileVersion::V2_1 + ); + } + + #[test] + fn test_parse_storage_format_rejects_invalid() { + assert!(parse_storage_format("v3.0").is_err()); + assert!(parse_storage_format("").is_err()); + assert!(parse_storage_format("foo").is_err()); + } } diff --git a/java/lance-jni/src/update.rs b/java/lance-jni/src/update.rs index 3544ce9c9f2..3a6f9360062 100644 --- a/java/lance-jni/src/update.rs +++ b/java/lance-jni/src/update.rs @@ -5,7 +5,7 @@ use crate::blocking_dataset::{BlockingDataset, NATIVE_DATASET}; use crate::error::Result; use crate::traits::IntoJava; use crate::utils::to_rust_map; -use crate::{JNIEnvExt, RT}; +use crate::{JNIEnvExt, block_on}; use jni::JNIEnv; use jni::objects::{JMap, JObject, JValueGen}; use lance::dataset::UpdateBuilder; @@ -53,7 +53,7 @@ fn inner_update<'local>( } let job = builder.build()?; - let update_result = RT.block_on(job.execute())?; + let update_result = block_on(job.execute())?; // Avoid panicking if Lance core retains a clone of the Arc; fall back to a // deep clone so the JNI boundary stays panic-free. diff --git a/java/lance-jni/src/utils.rs b/java/lance-jni/src/utils.rs index 94372ef27cc..0c2b2280415 100644 --- a/java/lance-jni/src/utils.rs +++ b/java/lance-jni/src/utils.rs @@ -12,7 +12,7 @@ use lance::dataset::optimize::{CompactionMode, CompactionOptions}; use lance::dataset::{WriteMode, WriteParams}; use lance::index::vector::{IndexFileVersion, StageParams, VectorIndexParams}; use lance::io::ObjectStoreParams; -use lance_encoding::version::LanceFileVersion; +use lance_file::version::LanceFileVersion; use lance_index::IndexParams; use lance_index::vector::bq::RQBuildParams; use lance_index::vector::hnsw::builder::HnswBuildParams; diff --git a/java/lance-jni/src/vector_trainer.rs b/java/lance-jni/src/vector_trainer.rs index 798b88baaff..d988aebd468 100755 --- a/java/lance-jni/src/vector_trainer.rs +++ b/java/lance-jni/src/vector_trainer.rs @@ -3,7 +3,7 @@ use std::sync::Arc; -use crate::RT; +use crate::block_on; use crate::blocking_dataset::{BlockingDataset, NATIVE_DATASET}; use crate::error::{Error, Result}; use crate::ffi::JNIEnvExt; @@ -122,7 +122,7 @@ fn inner_train_ivf_centroids<'local>( let dim = get_vector_dim(dataset.schema(), &column)?; - let ivf_model = RT.block_on(lance::index::vector::ivf::build_ivf_model( + let ivf_model = block_on(lance::index::vector::ivf::build_ivf_model( dataset, &column, dim, @@ -185,7 +185,7 @@ fn inner_train_pq_codebook<'local>( let dim = get_vector_dim(dataset.schema(), &column)?; - let pq = RT.block_on(lance::index::vector::pq::build_pq_model( + let pq = block_on(lance::index::vector::pq::build_pq_model( dataset, &column, dim, diff --git a/java/pom.xml b/java/pom.xml index 89b567c5f3e..2857bd915ae 100644 --- a/java/pom.xml +++ b/java/pom.xml @@ -7,7 +7,7 @@ org.lance lance-core Lance Core - 9.0.0-beta.16 + 11.0.0-beta.6 jar Lance Format Java API @@ -29,6 +29,7 @@ UTF-8 18.3.0 + 1.64.0 0.28.1 false 2.43.0 @@ -39,6 +40,7 @@ 3.7.5 package false + false org.lance.shaded @@ -121,6 +123,11 @@ jackson-databind 2.15.2 + + io.opentelemetry + opentelemetry-api + ${opentelemetry.version} + software.amazon.awssdk @@ -134,6 +141,18 @@ 2.20.26 test + + io.opentelemetry + opentelemetry-sdk + ${opentelemetry.version} + test + + + io.opentelemetry + opentelemetry-sdk-testing + ${opentelemetry.version} + test + @@ -396,6 +415,9 @@ lance-jni ${rust.release.build} + + ${rust.features} + ${project.build.directory}/classes/nativelib true @@ -409,6 +431,9 @@ lance-jni ${rust.release.build} + + ${rust.features} + -v diff --git a/java/src/main/java/org/lance/CacheBackendConfig.java b/java/src/main/java/org/lance/CacheBackendConfig.java new file mode 100644 index 00000000000..b43a49e9404 --- /dev/null +++ b/java/src/main/java/org/lance/CacheBackendConfig.java @@ -0,0 +1,113 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.lance; + +import org.apache.arrow.util.Preconditions; + +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; + +/** + * Structured configuration for a cache backend registered with the native Lance runtime. + * + *

The {@code kind} selects a registered backend constructor, while {@code options} contains + * backend-specific string settings. For example: + * + * 

{@code
+ * CacheBackendConfig config = CacheBackendConfig.builder("moka")
+ *     .option("capacity", "1048576")
+ *     .build();
+ * }
+ * + *

Third-party native backend crates register their constructors with Lance at application + * startup. This class selects and configures one of those registered constructors; it does not + * register a Java implementation as a native cache backend. + */ +public final class CacheBackendConfig { + private final String kind; + private final Map options; + + private CacheBackendConfig(Builder builder) { + this.kind = builder.kind; + this.options = Collections.unmodifiableMap(new HashMap<>(builder.options)); + } + + /** + * Creates a builder for a registered backend kind. + * + * @param kind registered backend identifier, such as {@code moka} + * @return a new builder + */ + public static Builder builder(String kind) { + return new Builder(kind); + } + + /** Returns the registered backend identifier. */ + public String getKind() { + return kind; + } + + /** Returns an immutable map of backend-specific options. */ + public Map getOptions() { + return options; + } + + /** Builder for {@link CacheBackendConfig}. */ + public static final class Builder { + private final String kind; + private final Map options = new HashMap<>(); + + private Builder(String kind) { + Preconditions.checkNotNull(kind, "kind must not be null"); + Preconditions.checkArgument(!kind.isEmpty(), "kind must not be empty"); + this.kind = kind; + } + + /** + * Adds a backend-specific option. + * + * @param key option name + * @param value option value + * @return this builder + */ + public Builder option(String key, String value) { + Preconditions.checkNotNull(key, "cache backend option key must not be null"); + Preconditions.checkNotNull(value, "cache backend option value must not be null"); + Preconditions.checkArgument(!key.isEmpty(), "cache backend option key must not be empty"); + options.put(key, value); + return this; + } + + /** + * Replaces the current backend options. + * + * @param options backend-specific options + * @return this builder + */ + public Builder options(Map options) { + Preconditions.checkNotNull(options, "options must not be null"); + this.options.clear(); + for (Map.Entry option : options.entrySet()) { + option(option.getKey(), option.getValue()); + } + return this; + } + + /** Builds the immutable backend configuration. */ + public CacheBackendConfig build() { + return new CacheBackendConfig(this); + } + } +} diff --git a/java/src/main/java/org/lance/CacheStats.java b/java/src/main/java/org/lance/CacheStats.java new file mode 100644 index 00000000000..9df3388dabb --- /dev/null +++ b/java/src/main/java/org/lance/CacheStats.java @@ -0,0 +1,105 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.lance; + +/** + * Statistics for a session cache. + * + *

A snapshot of cache activity, useful for monitoring hit/miss rates over time. + */ +public class CacheStats { + private final long hits; + private final long misses; + private final long numEntries; + private final long sizeBytes; + + /** + * Constructs cache statistics. Instances are created by the native layer. + * + * @param hits number of cache lookups that found an item + * @param misses number of cache lookups that did not find an item + * @param numEntries number of entries currently in the cache + * @param sizeBytes total size in bytes of all entries in the cache + */ + public CacheStats(long hits, long misses, long numEntries, long sizeBytes) { + if (hits < 0 || misses < 0 || numEntries < 0 || sizeBytes < 0) { + throw new IllegalArgumentException( + String.format( + "Cache statistics must be non-negative: " + + "hits=%d, misses=%d, numEntries=%d, sizeBytes=%d", + hits, misses, numEntries, sizeBytes)); + } + this.hits = hits; + this.misses = misses; + this.numEntries = numEntries; + this.sizeBytes = sizeBytes; + } + + /** + * Returns the number of cache lookups that found an item in the cache. + * + * @return the number of cache hits + */ + public long getHits() { + return hits; + } + + /** + * Returns the number of cache lookups that did not find an item in the cache. + * + * @return the number of cache misses + */ + public long getMisses() { + return misses; + } + + /** + * Returns the number of entries currently in the cache. + * + * @return the number of cache entries + */ + public long getNumEntries() { + return numEntries; + } + + /** + * Returns the total size in bytes of all entries in the cache. + * + * @return the cache size in bytes + */ + public long getSizeBytes() { + return sizeBytes; + } + + /** + * Returns the ratio of hits to total lookups, or 0 if there have been no lookups. + * + * @return the cache hit ratio in the range [0, 1] + */ + public double getHitRatio() { + // Sum in double to avoid long overflow for very large counters + double total = (double) hits + (double) misses; + if (total == 0) { + return 0.0; + } + return hits / total; + } + + @Override + public String toString() { + return String.format( + "CacheStats(hits=%d, misses=%d, numEntries=%d, sizeBytes=%d)", + hits, misses, numEntries, sizeBytes); + } +} diff --git a/java/src/main/java/org/lance/CommitBuilder.java b/java/src/main/java/org/lance/CommitBuilder.java index 62861b7ef73..fdb95828942 100644 --- a/java/src/main/java/org/lance/CommitBuilder.java +++ b/java/src/main/java/org/lance/CommitBuilder.java @@ -200,8 +200,14 @@ public CommitBuilder useStableRowIds(boolean useStableRowIds) { * Set the storage format to use for the dataset. * *

This is only needed when creating a new empty table. If any data files are passed, the - * storage format will be inferred from the data files. Valid values: "legacy", "v2_0", "stable", - * "v2_1", "next", "v2_2". + * storage format will be inferred from the data files. Valid values are the numeric versions + * ("0.1", "2.0", "2.1", "2.2", "2.3") and the release selectors ("legacy", "stable", "next"), + * matching {@link WriteParams.Builder#withDataStorageVersion(String)}. Parsing is + * case-insensitive. + * + *

The {@code v}-prefixed spellings ("v2_0", "v2.0", "v2_1", "v2.1", "v2_2", "v2.2") are + * deprecated. They were accepted only by this method, never by the rest of Lance, and will be + * removed in a future release — use the numeric version instead ("v2_1" becomes "2.1"). * * @param storageFormat the storage format name * @return this builder instance diff --git a/java/src/main/java/org/lance/Dataset.java b/java/src/main/java/org/lance/Dataset.java index 8f79d7cbaea..72a9a5fa0c7 100644 --- a/java/src/main/java/org/lance/Dataset.java +++ b/java/src/main/java/org/lance/Dataset.java @@ -19,6 +19,7 @@ import org.lance.compaction.CompactionOptions; import org.lance.delta.DatasetDelta; import org.lance.index.Index; +import org.lance.index.IndexBuildProgress; import org.lance.index.IndexCriteria; import org.lance.index.IndexDescription; import org.lance.index.IndexOptions; @@ -54,6 +55,7 @@ import org.apache.arrow.vector.ipc.ArrowReader; import org.apache.arrow.vector.ipc.ArrowStreamReader; import org.apache.arrow.vector.types.pojo.Field; +import org.apache.arrow.vector.types.pojo.FieldType; import org.apache.arrow.vector.types.pojo.Schema; import java.io.ByteArrayInputStream; @@ -62,6 +64,7 @@ import java.nio.ByteBuffer; import java.nio.channels.Channels; import java.nio.channels.ReadableByteChannel; +import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -732,11 +735,31 @@ public void dropColumns(List columns) { public void alterColumns(List columnAlterations) { try (LockManager.WriteLock writeLock = lockManager.acquireWriteLock()) { Preconditions.checkArgument(nativeDatasetHandle != 0, "Dataset is closed"); - nativeAlterColumns(columnAlterations); + // Cast target types are carried across the FFI boundary through the Arrow C Data + // Interface rather than ArrowType#toString(), which does not round-trip reliably on + // the native side (parameterized types such as Int(64, true) fail to parse and the + // cast would otherwise be silently dropped). One field is exported per alteration that + // requests a type change, in the same order as {@code columnAlterations}. + List castFields = new ArrayList<>(); + int castIndex = 0; + for (ColumnAlteration alteration : columnAlterations) { + if (alteration.getDataType().isPresent()) { + castFields.add(new Field("f" + castIndex++, castFieldType(alteration), null)); + } + } + try (ArrowSchema castSchema = ArrowSchema.allocateNew(allocator)) { + Data.exportSchema(allocator, new Schema(castFields), null, castSchema); + nativeAlterColumns(columnAlterations, castSchema.memoryAddress()); + } } } - private native void nativeAlterColumns(List columnAlterations); + private static FieldType castFieldType(ColumnAlteration alteration) { + boolean nullable = alteration.getNullable().orElse(true); + return new FieldType(nullable, alteration.getDataType().get(), null); + } + + private native void nativeAlterColumns(List columnAlterations, long castAddr); /** * Create a new Dataset Scanner. @@ -955,6 +978,23 @@ public List listVersions() { private native List nativeListVersions(); + /** + * Get the number of versions in the current version history. + * + *

Unlike {@link #listVersions()}, this method does not read or deserialize every manifest. + * Detached versions are not included. + * + * @return the number of versions + */ + public long getVersionCount() { + try (LockManager.ReadLock readLock = lockManager.acquireReadLock()) { + Preconditions.checkArgument(nativeDatasetHandle != 0, "Dataset is closed"); + return nativeGetVersionCount(); + } + } + + private native long nativeGetVersionCount(); + /** * @return the latest version of the dataset. */ @@ -1153,6 +1193,29 @@ public void mergeIndexMetadata( private native void innerMergeIndexMetadata( String indexUUID, int indexType, Optional batchReadHead); + /** + * Merge distributed index metadata while reporting stage-level progress. + * + * @param indexUUID shared UUID used by the distributed index parts + * @param indexType type of index metadata to merge + * @param batchReadHead optional limit for metadata read concurrency + * @param progress thread-safe progress callback + */ + public void mergeIndexMetadata( + String indexUUID, + IndexType indexType, + Optional batchReadHead, + IndexBuildProgress progress) { + Preconditions.checkNotNull(progress, "progress cannot be null"); + innerMergeIndexMetadataWithProgress(indexUUID, indexType.getValue(), batchReadHead, progress); + } + + private native void innerMergeIndexMetadataWithProgress( + String indexUUID, + int indexType, + Optional batchReadHead, + IndexBuildProgress progress); + /** Merge one caller-defined group of existing uncommitted vector index segments. */ public Index mergeExistingIndexSegments(List segments) { Preconditions.checkNotNull(segments, "segments cannot be null"); @@ -1303,6 +1366,36 @@ public List getFragments() { private native List getFragmentsNative(); + /** + * Get per-fragment statistics for all fragments in this dataset version. + * + *

Unlike {@link #getFragments()}, this is a metadata-only bulk operation: no per-fragment Java + * objects are materialized, making it suitable for planning over datasets with a very large + * number of fragments. Row counts match {@link FragmentMetadata#getNumRows()} (physical rows + * minus deleted rows). + * + * @return per-fragment statistics as parallel arrays, in manifest order + */ + public FragmentStatistics getFragmentStatistics() { + try (LockManager.ReadLock readLock = lockManager.acquireReadLock()) { + Preconditions.checkArgument(nativeDatasetHandle != 0, "Dataset is closed"); + // Flattened as [id0, rowCount0, dataFileNum0, id1, ...] to keep the JNI surface primitive + long[] flat = nativeGetFragmentStatistics(); + int count = flat.length / 3; + int[] ids = new int[count]; + long[] rowCounts = new long[count]; + int[] dataFileNums = new int[count]; + for (int i = 0; i < count; i++) { + ids[i] = (int) flat[3 * i]; + rowCounts[i] = flat[3 * i + 1]; + dataFileNums[i] = (int) flat[3 * i + 2]; + } + return new FragmentStatistics(ids, rowCounts, dataFileNums); + } + } + + private native long[] nativeGetFragmentStatistics(); + /** * Gets the arrow schema of the dataset. * @@ -1621,11 +1714,27 @@ public void close() { private native List nativeTakeBlobsByIndices(List rowIndices, String column); /** - * Open blob files for given row ids on a blob column. Names and semantics align with Rust/Python. + * Open {@link BlobFile} handles for given row IDs on a blob column. Names and semantics align + * with Rust/Python. + * + *

Pass logical row IDs read from {@code _rowid}, not physical row addresses from {@code + * _rowaddr}. + * + * 

{@code
+   * long rowId = 42L; // Example value from the _rowid column.
+   * List blobs = dataset.takeBlobs(List.of(rowId), "images");
+   * for (BlobFile blob : blobs) {
+   *   if (blob != null) {
+   *     try (BlobFile file = blob) {
+   *       byte[] data = file.read();
+   *     }
+   *   }
+   * }
+   * }
* - * @param rowIds stable row ids (row addresses) + * @param rowIds logical row IDs from the {@code _rowid} column * @param column blob column name - * @return list of BlobFile objects + * @return one {@link BlobFile} per row ID; null blob values are represented by null elements */ public List takeBlobs(List rowIds, String column) { try (LockManager.ReadLock readLock = lockManager.acquireReadLock()) { @@ -1639,11 +1748,22 @@ public List takeBlobs(List rowIds, String column) { } /** - * Open blob files for given row indices on a blob column. + * Open {@link BlobFile} handles for given row indices on a blob column. + * + * 
{@code
+   * List blobs = dataset.takeBlobsByIndices(List.of(0L), "images");
+   * for (BlobFile blob : blobs) {
+   *   if (blob != null) {
+   *     try (BlobFile file = blob) {
+   *       byte[] data = file.read();
+   *     }
+   *   }
+   * }
+   * }
* * @param rowIndices row offsets within dataset * @param column blob column name - * @return list of BlobFile objects + * @return one {@link BlobFile} per row index; null blob values are represented by null elements */ public List takeBlobsByIndices(List rowIndices, String column) { try (LockManager.ReadLock readLock = lockManager.acquireReadLock()) { diff --git a/java/src/main/java/org/lance/DocumentGranularity.java b/java/src/main/java/org/lance/DocumentGranularity.java new file mode 100644 index 00000000000..c540cc1ff9c --- /dev/null +++ b/java/src/main/java/org/lance/DocumentGranularity.java @@ -0,0 +1,34 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.lance; + +/** The unit treated as one full-text-search document. */ +public enum DocumentGranularity { + /** All text selected from one dataset row belongs to one document. */ + ROW("row"), + + /** Each element of the deepest list on the field path is one document. */ + LIST_ELEMENT("list_element"); + + private final String rustString; + + DocumentGranularity(String rustString) { + this.rustString = rustString; + } + + /** Return the stable value understood by the Rust API and serialized index parameters. */ + public String toRustString() { + return rustString; + } +} diff --git a/java/src/main/java/org/lance/Fragment.java b/java/src/main/java/org/lance/Fragment.java index 3b12e158617..0040fdbf869 100644 --- a/java/src/main/java/org/lance/Fragment.java +++ b/java/src/main/java/org/lance/Fragment.java @@ -261,7 +261,7 @@ static List create( WriteParams params, LanceNamespace namespaceClient, List tableId) { - return create(datasetUri, allocator, root, params, namespaceClient, tableId, null); + return create(datasetUri, allocator, root, params, namespaceClient, tableId, null, null); } /** Create a fragment from the given arrow array and schema. */ @@ -272,11 +272,13 @@ static List create( WriteParams params, LanceNamespace namespaceClient, List tableId, - LanceSchema schema) { + LanceSchema schema, + Session session) { Preconditions.checkNotNull(datasetUri); Preconditions.checkNotNull(allocator); Preconditions.checkNotNull(root); Preconditions.checkNotNull(params); + long sessionHandle = getSessionHandle(session); try (ArrowSchema arrowSchema = ArrowSchema.allocateNew(allocator); ArrowArray arrowArray = ArrowArray.allocateNew(allocator)) { Data.exportVectorSchemaRoot(allocator, root, null, arrowArray, arrowSchema); @@ -301,7 +303,8 @@ static List create( tableId, params.getAllowExternalBlobOutsideBases(), params.getBlobPackFileSizeThreshold(), - lanceSchema.memoryAddress()); + lanceSchema.memoryAddress(), + sessionHandle); } } return createWithFfiArray( @@ -322,7 +325,8 @@ static List create( tableId, params.getAllowExternalBlobOutsideBases(), params.getBlobPackFileSizeThreshold(), - 0L); + 0L, + sessionHandle); } } @@ -333,7 +337,7 @@ static List create( WriteParams params, LanceNamespace namespaceClient, List tableId) { - return create(datasetUri, null, stream, params, namespaceClient, tableId, null); + return create(datasetUri, null, stream, params, namespaceClient, tableId, null, null); } /** Create a fragment from the given arrow stream. */ @@ -344,10 +348,12 @@ static List create( WriteParams params, LanceNamespace namespaceClient, List tableId, - LanceSchema schema) { + LanceSchema schema, + Session session) { Preconditions.checkNotNull(datasetUri); Preconditions.checkNotNull(stream); Preconditions.checkNotNull(params); + long sessionHandle = getSessionHandle(session); if (schema != null) { Preconditions.checkNotNull(allocator, "allocator is required with schema"); try (ArrowSchema lanceSchema = ArrowSchema.allocateNew(allocator)) { @@ -369,7 +375,8 @@ static List create( tableId, params.getAllowExternalBlobOutsideBases(), params.getBlobPackFileSizeThreshold(), - lanceSchema.memoryAddress()); + lanceSchema.memoryAddress(), + sessionHandle); } } return createWithFfiStream( @@ -389,7 +396,16 @@ static List create( tableId, params.getAllowExternalBlobOutsideBases(), params.getBlobPackFileSizeThreshold(), - 0L); + 0L, + sessionHandle); + } + + /** + * Resolves the native handle of an optional session. A closed session has a zero handle and is + * treated as absent, matching how Dataset handles closed sessions. + */ + private static long getSessionHandle(Session session) { + return session == null ? 0L : session.getNativeHandle(); } /** Create a fragment from the given arrow array and schema. */ @@ -411,7 +427,8 @@ private static native List createWithFfiArray( List tableId, Optional allowExternalBlobOutsideBases, Optional blobPackFileSizeThreshold, - long schemaMemoryAddress); + long schemaMemoryAddress, + long sessionHandle); /** Create a fragment from the given arrow stream. */ private static native List createWithFfiStream( @@ -431,5 +448,6 @@ private static native List createWithFfiStream( List tableId, Optional allowExternalBlobOutsideBases, Optional blobPackFileSizeThreshold, - long schemaMemoryAddress); + long schemaMemoryAddress, + long sessionHandle); } diff --git a/java/src/main/java/org/lance/FragmentStatistics.java b/java/src/main/java/org/lance/FragmentStatistics.java new file mode 100644 index 00000000000..8e4cd53bc54 --- /dev/null +++ b/java/src/main/java/org/lance/FragmentStatistics.java @@ -0,0 +1,55 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.lance; + +/** + * Per-fragment statistics of a dataset version as parallel primitive arrays: {@code ids[i]}, {@code + * rowCounts[i]} and {@code dataFileNums[i]} describe the same fragment. Returned by {@link + * Dataset#getFragmentStatistics()} as a lightweight alternative to materializing full {@link + * Fragment} objects. + */ +public final class FragmentStatistics { + private final int[] ids; + private final long[] rowCounts; + private final int[] dataFileNums; + + FragmentStatistics(int[] ids, long[] rowCounts, int[] dataFileNums) { + this.ids = ids; + this.rowCounts = rowCounts; + this.dataFileNums = dataFileNums; + } + + /** Fragment IDs in manifest order. */ + public int[] getIds() { + return ids; + } + + /** + * Row count per fragment, aligned with {@link #getIds()}. Matches {@link + * FragmentMetadata#getNumRows()}: physical rows minus deleted rows. + */ + public long[] getRowCounts() { + return rowCounts; + } + + /** Number of data files per fragment, aligned with {@link #getIds()}. */ + public int[] getDataFileNums() { + return dataFileNums; + } + + /** Number of fragments described. */ + public int size() { + return ids.length; + } +} diff --git a/java/src/main/java/org/lance/LanceException.java b/java/src/main/java/org/lance/LanceException.java new file mode 100644 index 00000000000..0b5c180e319 --- /dev/null +++ b/java/src/main/java/org/lance/LanceException.java @@ -0,0 +1,48 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.lance; + +/** + * Thrown when a Lance operation fails and its API cannot expose a checked exception. + * + *

For example, scanner I/O failures can be handled separately from programming errors: + * + * 

{@code
+ * try (ArrowReader reader = scanner.scanBatches()) {
+ *   // Consume batches.
+ * } catch (LanceException e) {
+ *   // Handle the failed Lance operation.
+ * }
+ * }
+ */ +public class LanceException extends RuntimeException { + /** + * Creates an exception with a message describing the failed operation. + * + * @param message description of the failure + */ + public LanceException(String message) { + super(message); + } + + /** + * Creates an exception with a message and its underlying cause. + * + * @param message description of the failure + * @param cause underlying cause of the failure + */ + public LanceException(String message, Throwable cause) { + super(message, cause); + } +} diff --git a/java/src/main/java/org/lance/Session.java b/java/src/main/java/org/lance/Session.java index 0fe4a59736c..95a92cc8b57 100644 --- a/java/src/main/java/org/lance/Session.java +++ b/java/src/main/java/org/lance/Session.java @@ -16,6 +16,8 @@ import org.apache.arrow.util.Preconditions; import java.io.Closeable; +import java.util.Collections; +import java.util.Map; /** * A user session that holds runtime state for Lance datasets. @@ -35,6 +37,14 @@ * .metadataCacheSizeBytes(512L * 1024 * 1024) // 512 MiB * .build(); * + * // Select registered cache backends using URIs or structured configuration + * Session backendSession = Session.builder() + * .indexCacheBackend("moka://?capacity=1073741824") + * .metadataCacheBackend(CacheBackendConfig.builder("moka") + * .option("capacity", "268435456") + * .build()) + * .build(); + * * // Open multiple datasets with shared session * Dataset ds1 = Dataset.open() * .uri("s3://bucket/table1.lance") @@ -110,8 +120,12 @@ public static Session create(long indexCacheSizeBytes, long metadataCacheSizeByt /** Builder for creating Session instances with custom configuration. */ public static class Builder { - private long indexCacheSizeBytes = DEFAULT_INDEX_CACHE_SIZE_BYTES; - private long metadataCacheSizeBytes = DEFAULT_METADATA_CACHE_SIZE_BYTES; + private Long indexCacheSizeBytes; + private Long metadataCacheSizeBytes; + private String indexCacheBackendUri; + private CacheBackendConfig indexCacheBackendConfig; + private String metadataCacheBackendUri; + private CacheBackendConfig metadataCacheBackendConfig; private Builder() {} @@ -127,6 +141,37 @@ public Builder indexCacheSizeBytes(long indexCacheSizeBytes) { return this; } + /** + * Selects a registered index cache backend using a backend URI. + * + *

For example, {@code moka://?capacity=1048576}. This option is mutually exclusive with + * {@link #indexCacheSizeBytes(long)}. + * + * @param backendUri backend URI whose scheme identifies the registered backend + * @return this builder instance + */ + public Builder indexCacheBackend(String backendUri) { + Preconditions.checkNotNull(backendUri, "backendUri must not be null"); + this.indexCacheBackendUri = backendUri; + this.indexCacheBackendConfig = null; + return this; + } + + /** + * Selects a registered index cache backend using structured configuration. + * + *

This option is mutually exclusive with {@link #indexCacheSizeBytes(long)}. + * + * @param backendConfig backend kind and backend-specific options + * @return this builder instance + */ + public Builder indexCacheBackend(CacheBackendConfig backendConfig) { + Preconditions.checkNotNull(backendConfig, "backendConfig must not be null"); + this.indexCacheBackendConfig = backendConfig; + this.indexCacheBackendUri = null; + return this; + } + /** * Sets the size of the metadata cache in bytes. * @@ -140,15 +185,76 @@ public Builder metadataCacheSizeBytes(long metadataCacheSizeBytes) { return this; } + /** + * Selects a registered metadata cache backend using a backend URI. + * + *

For example, {@code moka://?capacity=1048576}. This option is mutually exclusive with + * {@link #metadataCacheSizeBytes(long)}. + * + * @param backendUri backend URI whose scheme identifies the registered backend + * @return this builder instance + */ + public Builder metadataCacheBackend(String backendUri) { + Preconditions.checkNotNull(backendUri, "backendUri must not be null"); + this.metadataCacheBackendUri = backendUri; + this.metadataCacheBackendConfig = null; + return this; + } + + /** + * Selects a registered metadata cache backend using structured configuration. + * + *

This option is mutually exclusive with {@link #metadataCacheSizeBytes(long)}. + * + * @param backendConfig backend kind and backend-specific options + * @return this builder instance + */ + public Builder metadataCacheBackend(CacheBackendConfig backendConfig) { + Preconditions.checkNotNull(backendConfig, "backendConfig must not be null"); + this.metadataCacheBackendConfig = backendConfig; + this.metadataCacheBackendUri = null; + return this; + } + /** * Builds the Session with the configured settings. * * @return a new Session instance */ public Session build() { - long handle = createNative(indexCacheSizeBytes, metadataCacheSizeBytes); + validateCacheConfiguration(); + long handle = + createNative( + indexCacheSizeBytes == null ? -1 : indexCacheSizeBytes, + metadataCacheSizeBytes == null ? -1 : metadataCacheSizeBytes, + indexCacheBackendUri, + backendKind(indexCacheBackendConfig), + backendOptions(indexCacheBackendConfig), + metadataCacheBackendUri, + backendKind(metadataCacheBackendConfig), + backendOptions(metadataCacheBackendConfig)); return new Session(handle); } + + private void validateCacheConfiguration() { + Preconditions.checkArgument( + indexCacheSizeBytes == null + || (indexCacheBackendUri == null && indexCacheBackendConfig == null), + "indexCacheSizeBytes and indexCacheBackend are mutually exclusive; set one or the other"); + Preconditions.checkArgument( + metadataCacheSizeBytes == null + || (metadataCacheBackendUri == null && metadataCacheBackendConfig == null), + "metadataCacheSizeBytes and metadataCacheBackend are mutually exclusive; " + + "set one or the other"); + } + + private static String backendKind(CacheBackendConfig config) { + return config == null ? null : config.getKind(); + } + + private static Map backendOptions(CacheBackendConfig config) { + return config == null ? Collections.emptyMap() : config.getOptions(); + } } /** @@ -176,6 +282,19 @@ public long sizeBytes() { return sizeBytesNative(); } + /** + * Returns statistics for the metadata cache of this session. + * + *

The returned statistics are a snapshot; call this method periodically to observe how hits + * and misses evolve over time. + * + * @return the metadata cache statistics + */ + public CacheStats metadataCacheStats() { + Preconditions.checkArgument(nativeSessionHandle != 0, "Session is closed"); + return metadataCacheStatsNative(); + } + /** * Returns whether the other session is the same as this one. * @@ -238,10 +357,20 @@ public String toString() { return String.format("Session(sizeBytes=%d)", sizeBytes()); } - private static native long createNative(long indexCacheSizeBytes, long metadataCacheSizeBytes); + private static native long createNative( + long indexCacheSizeBytes, + long metadataCacheSizeBytes, + String indexCacheBackendUri, + String indexCacheBackendKind, + Map indexCacheBackendOptions, + String metadataCacheBackendUri, + String metadataCacheBackendKind, + Map metadataCacheBackendOptions); private native long sizeBytesNative(); + private native CacheStats metadataCacheStatsNative(); + private static native void releaseNative(long handle); private static native boolean isSameAsNative(long handle1, long handle2); diff --git a/java/src/main/java/org/lance/WriteDatasetBuilder.java b/java/src/main/java/org/lance/WriteDatasetBuilder.java index 96a894fc944..9d406406291 100644 --- a/java/src/main/java/org/lance/WriteDatasetBuilder.java +++ b/java/src/main/java/org/lance/WriteDatasetBuilder.java @@ -70,6 +70,7 @@ public class WriteDatasetBuilder { private WriteParams.WriteMode mode = WriteParams.WriteMode.CREATE; private Schema schema; private Map storageOptions = new HashMap<>(); + private Map properties = new HashMap<>(); private Map> baseStoreParams = new HashMap<>(); private boolean ignoreNamespaceStorageOptions = false; private Optional maxRowsPerFile = Optional.empty(); @@ -207,6 +208,27 @@ public WriteDatasetBuilder storageOptions(Map storageOptions) { return this; } + /** + * Sets the table properties to forward to the namespace on table creation. + * + *

These are Lance-namespace properties: catalog-level key-value metadata stored by the + * namespace outside the Lance table (available even if the table manifest does not exist), as + * distinct from the manifest-stored {@code config} (read/write behavior) and {@code metadata} + * (business metadata), and from non-persisted {@code storageOptions}. They are attached to the + * underlying declareTable request via its {@code properties} field. + * + *

Only used when a namespace client is configured via namespaceClient()+tableId() and the + * write creates the table (CREATE mode). Ignored for direct-URI writes and for APPEND/OVERWRITE + * modes. + * + * @param properties Table properties to forward on declareTable + * @return this builder instance + */ + public WriteDatasetBuilder properties(Map properties) { + this.properties = new HashMap<>(properties); + return this; + } + /** * Sets runtime-only object store parameters for registered base paths. * @@ -410,6 +432,9 @@ private Dataset executeWithNamespaceClient() { if (mode == WriteParams.WriteMode.CREATE) { DeclareTableRequest declareRequest = new DeclareTableRequest(); declareRequest.setId(tableId); + if (properties != null && !properties.isEmpty()) { + declareRequest.setProperties(properties); + } DeclareTableResponse declareResponse = namespaceClient.declareTable(declareRequest); tableUri = declareResponse.getLocation(); diff --git a/java/src/main/java/org/lance/WriteFragmentBuilder.java b/java/src/main/java/org/lance/WriteFragmentBuilder.java index 2dbef873849..6e4a6b007df 100644 --- a/java/src/main/java/org/lance/WriteFragmentBuilder.java +++ b/java/src/main/java/org/lance/WriteFragmentBuilder.java @@ -51,6 +51,7 @@ public class WriteFragmentBuilder { private WriteParams.Builder writeParamsBuilder; private LanceNamespace namespaceClient; private List tableId; + private Session session; WriteFragmentBuilder() {} @@ -185,6 +186,19 @@ public WriteFragmentBuilder tableId(List tableId) { return this; } + /** + * Set a session to reuse across operations. + * + *

The session holds shared caches (metadata and index) and the object store registry. + * + * @param session the session to share + * @return this builder + */ + public WriteFragmentBuilder session(Session session) { + this.session = session; + return this; + } + /** * Set the maximum number of rows per file. * @@ -302,7 +316,8 @@ public List execute() { finalWriteParams, namespaceClient, tableId, - schema); + schema, + session); } else { return Fragment.create( datasetUri, @@ -311,7 +326,8 @@ public List execute() { finalWriteParams, namespaceClient, tableId, - schema); + schema, + session); } } diff --git a/java/src/main/java/org/lance/fragment/FragmentUpdateResult.java b/java/src/main/java/org/lance/fragment/FragmentUpdateResult.java index cadb22b0e6a..34dbbe3908c 100644 --- a/java/src/main/java/org/lance/fragment/FragmentUpdateResult.java +++ b/java/src/main/java/org/lance/fragment/FragmentUpdateResult.java @@ -14,6 +14,7 @@ package org.lance.fragment; import org.lance.FragmentMetadata; +import org.lance.JniLoader; import com.google.common.base.MoreObjects; import org.apache.arrow.c.ArrowArrayStream; @@ -23,22 +24,51 @@ * Fragment.updateColumns()}. */ public class FragmentUpdateResult { + static { + JniLoader.ensureLoaded(); + } + private final FragmentMetadata updatedFragment; private final long[] fieldsModified; - /** Local physical row offsets within the fragment that received updates (see RowAddress). */ - private final long[] updatedRowOffsets; + /** + * Matched physical row offsets within the fragment, serialized as portable RoaringBitmap bytes + * (little-endian, same format as {@link org.lance.operation.Update#updatedFragmentOffsets()}). + */ + private final byte[] updatedRowOffsetBytes; + + /** Primary public API for constructing a result with portable RoaringBitmap offset bytes. */ + public static FragmentUpdateResult create( + FragmentMetadata updatedFragment, long[] updatedFieldIds, byte[] updatedRowOffsetBytes) { + return new FragmentUpdateResult(updatedFragment, updatedFieldIds, updatedRowOffsetBytes); + } /** Two-argument form for callers that do not track per-row offsets; offsets default to empty. */ public FragmentUpdateResult(FragmentMetadata updatedFragment, long[] updatedFieldIds) { - this(updatedFragment, updatedFieldIds, new long[0]); + this(updatedFragment, updatedFieldIds, new byte[0]); } - public FragmentUpdateResult( - FragmentMetadata updatedFragment, long[] updatedFieldIds, long[] updatedRowOffsets) { + private FragmentUpdateResult( + FragmentMetadata updatedFragment, long[] updatedFieldIds, byte[] updatedRowOffsetBytes) { this.updatedFragment = updatedFragment; this.fieldsModified = updatedFieldIds; - this.updatedRowOffsets = updatedRowOffsets; + this.updatedRowOffsetBytes = + updatedRowOffsetBytes != null ? updatedRowOffsetBytes : new byte[0]; + } + + /** + * @deprecated Use {@link #create(FragmentMetadata, long[], byte[])} instead. This constructor + * encodes the expanded {@code long[]} offsets into portable RoaringBitmap bytes via JNI and + * is retained for backward compatibility with callers compiled against the prior long[]-based + * API. + */ + @Deprecated + public FragmentUpdateResult( + FragmentMetadata updatedFragment, long[] updatedFieldIds, long[] updatedRowOffsets) { + this( + updatedFragment, + updatedFieldIds, + encodeRowOffsetsToBytes(updatedRowOffsets != null ? updatedRowOffsets : new long[0])); } public FragmentMetadata getUpdatedFragment() { @@ -49,17 +79,34 @@ public long[] getFieldsModified() { return fieldsModified; } - /** Physical row offsets (0-based within the fragment) whose columns were rewritten. */ + /** + * Physical row offsets (0-based within the fragment) whose columns were rewritten, as portable + * RoaringBitmap bytes. + */ + public byte[] getUpdatedRowOffsetBytes() { + return updatedRowOffsetBytes; + } + + /** + * Physical row offsets (0-based within the fragment) whose columns were rewritten. + * + * @deprecated Use {@link #getUpdatedRowOffsetBytes()} instead. + */ + @Deprecated public long[] getUpdatedRowOffsets() { - return updatedRowOffsets; + return expandRowOffsetsFromBytes(updatedRowOffsetBytes); } + private static native byte[] encodeRowOffsetsToBytes(long[] rowOffsets); + + private static native long[] expandRowOffsetsFromBytes(byte[] rowOffsetBytes); + @Override public String toString() { return MoreObjects.toStringHelper(this) .add("fragmentMetadata", updatedFragment) .add("updatedFieldIds", fieldsModified) - .add("updatedRowOffsets", updatedRowOffsets) + .add("updatedRowOffsetBytesLength", updatedRowOffsetBytes.length) .toString(); } } diff --git a/java/src/main/java/org/lance/index/Index.java b/java/src/main/java/org/lance/index/Index.java index 955835496ed..e696ca844b7 100644 --- a/java/src/main/java/org/lance/index/Index.java +++ b/java/src/main/java/org/lance/index/Index.java @@ -36,6 +36,7 @@ public class Index { private final int indexVersion; private final Instant createdAt; private final Integer baseId; + private final Long sizeBytes; private final IndexType indexType; private Index( @@ -48,6 +49,7 @@ private Index( int indexVersion, Instant createdAt, Integer baseId, + Long sizeBytes, IndexType indexType) { this.uuid = uuid; this.fields = fields; @@ -58,6 +60,7 @@ private Index( this.indexVersion = indexVersion; this.createdAt = createdAt; this.baseId = baseId; + this.sizeBytes = sizeBytes; this.indexType = indexType; } @@ -104,6 +107,17 @@ public Optional baseId() { return Optional.ofNullable(baseId); } + /** + * Get the total size of all files in this physical index segment. + * + *

The size is unavailable for indices created before index file sizes were tracked. + * + * @return the segment size in bytes, or empty if unavailable + */ + public Optional getSizeBytes() { + return Optional.ofNullable(sizeBytes); + } + /** * Get the index version. * @@ -145,6 +159,7 @@ public boolean equals(Object o) { && Arrays.equals(indexDetails, index.indexDetails) && Objects.equals(createdAt, index.createdAt) && Objects.equals(baseId, index.baseId) + && Objects.equals(sizeBytes, index.sizeBytes) && indexType == index.indexType; } @@ -159,6 +174,7 @@ public int hashCode() { indexVersion, createdAt, baseId, + sizeBytes, fragments, indexType); result = 31 * result + Arrays.hashCode(indexDetails); @@ -176,6 +192,7 @@ public String toString() { .add("indexType", indexType) .add("createdAt", createdAt) .add("baseId", baseId) + .add("sizeBytes", sizeBytes) .toString(); } @@ -199,6 +216,7 @@ public static class Builder { private int indexVersion; private Instant createdAt; private Integer baseId; + private Long sizeBytes; private IndexType indexType; private Builder() {} @@ -248,6 +266,11 @@ public Builder baseId(Integer baseId) { return this; } + public Builder sizeBytes(Long sizeBytes) { + this.sizeBytes = sizeBytes; + return this; + } + public Builder indexType(IndexType indexType) { this.indexType = indexType; return this; @@ -264,6 +287,7 @@ public Index build() { indexVersion, createdAt, baseId, + sizeBytes, indexType); } } diff --git a/java/src/main/java/org/lance/index/IndexBuildProgress.java b/java/src/main/java/org/lance/index/IndexBuildProgress.java new file mode 100644 index 00000000000..20cd0dd6d8d --- /dev/null +++ b/java/src/main/java/org/lance/index/IndexBuildProgress.java @@ -0,0 +1,53 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.lance.index; + +import java.util.Optional; + +/** Receives stage-level progress while Lance builds or finalizes an index. */ +public interface IndexBuildProgress { + + /** + * Reports that a stage has started. + * + *

Implementations must be thread-safe. Lance may invoke callbacks concurrently from native + * runtime threads. Callbacks may re-enter the same {@code Dataset} through JNI methods. An + * exception thrown by this method terminates the index operation. + * + * @param stage stable, index-type-specific stage name + * @param total number of work units, or empty when the total is unknown + * @param unit description of the work unit, such as {@code files} or {@code partitions} + */ + void stageStart(String stage, Optional total, String unit); + + /** + * Reports completed work within a stage. + * + *

An exception thrown by this method terminates the index operation. + * + * @param stage stage name previously reported to {@link #stageStart} + * @param completed number of completed work units + */ + void stageProgress(String stage, long completed); + + /** + * Reports that a stage has completed. + * + *

The stage work has already succeeded when this method is called. Lance therefore logs and + * ignores exceptions thrown by this callback instead of failing the completed operation. + * + * @param stage completed stage name + */ + void stageComplete(String stage); +} diff --git a/java/src/main/java/org/lance/index/IndexDescription.java b/java/src/main/java/org/lance/index/IndexDescription.java index 1b5e5a3a8f8..b1e2689a00c 100755 --- a/java/src/main/java/org/lance/index/IndexDescription.java +++ b/java/src/main/java/org/lance/index/IndexDescription.java @@ -15,6 +15,7 @@ import java.util.List; import java.util.Objects; +import java.util.Optional; /** * High-level description of an index, aggregating metadata across all segments. @@ -31,6 +32,7 @@ public final class IndexDescription { private final long rowsIndexed; private final List metadata; private final String detailsJson; + private final Long totalSizeBytes; public IndexDescription( String name, @@ -40,6 +42,18 @@ public IndexDescription( long rowsIndexed, List metadata, String detailsJson) { + this(name, fieldIds, typeUrl, indexType, rowsIndexed, metadata, detailsJson, null); + } + + public IndexDescription( + String name, + List fieldIds, + String typeUrl, + String indexType, + long rowsIndexed, + List metadata, + String detailsJson, + Long totalSizeBytes) { this.name = Objects.requireNonNull(name, "name must not be null"); this.fieldIds = Objects.requireNonNull(fieldIds, "fieldIds must not be null"); this.typeUrl = Objects.requireNonNull(typeUrl, "typeUrl must not be null"); @@ -47,6 +61,7 @@ public IndexDescription( this.rowsIndexed = rowsIndexed; this.metadata = Objects.requireNonNull(metadata, "metadata must not be null"); this.detailsJson = detailsJson; + this.totalSizeBytes = totalSizeBytes; } /** The logical name of the index. */ @@ -100,4 +115,15 @@ public List getSegments() { public String getDetailsJson() { return detailsJson; } + + /** + * Total size of all files across all physical index segments. + * + *

The size is unavailable if any segment predates index file size tracking. + * + * @return the logical index size in bytes, or empty if unavailable + */ + public Optional getTotalSizeBytes() { + return Optional.ofNullable(totalSizeBytes); + } } diff --git a/java/src/main/java/org/lance/index/scalar/InvertedIndexParams.java b/java/src/main/java/org/lance/index/scalar/InvertedIndexParams.java index 9b29d7a0795..6928a78c4d9 100755 --- a/java/src/main/java/org/lance/index/scalar/InvertedIndexParams.java +++ b/java/src/main/java/org/lance/index/scalar/InvertedIndexParams.java @@ -13,6 +13,7 @@ */ package org.lance.index.scalar; +import org.lance.DocumentGranularity; import org.lance.util.JsonUtils; import com.google.common.base.Preconditions; @@ -53,8 +54,10 @@ public static final class Builder { private Integer minNgramLength; private Integer maxNgramLength; private Boolean prefixOnly; + private Integer blockSize = 128; private Boolean skipMerge; private Integer formatVersion; + private DocumentGranularity documentGranularity = DocumentGranularity.ROW; /** * Configure the base tokenizer. @@ -66,6 +69,7 @@ public static final class Builder { *

  • {@code "whitespace"}: splits tokens on whitespace *
  • {@code "raw"}: no tokenization *
  • {@code "ngram"}: N-Gram tokenizer + *
  • {@code "code"}: code-aware tokenizer *
  • {@code "icu"}: ICU dictionary-based Unicode word segmentation *
  • {@code "icu/split"}: ICU segmentation with simple-style delimiter splitting *
  • {@code "lindera/*"}: Lindera tokenizer @@ -226,6 +230,27 @@ public Builder prefixOnly(boolean prefixOnly) { return this; } + /** + * Configure the number of documents in each compressed posting block. + * + *

    Supported values are {@code 128} and {@code 256}. New indexes default to {@code 128} when + * this is not set. + * + *

    {@code blockSize = 256} requires FTS format v3. Format v3 also supports the default {@code + * blockSize = 128}. + * + * @param blockSize posting block size + * @return this builder + * @throws IllegalArgumentException if {@code blockSize} is unsupported + */ + public Builder blockSize(int blockSize) { + if (blockSize != 128 && blockSize != 256) { + throw new IllegalArgumentException("blockSize must be one of 128 or 256"); + } + this.blockSize = blockSize; + return this; + } + /** * Configure whether to skip the partition merge stage after indexing. If true, skip the * partition merge stage after indexing. This can be useful for distributed indexing where merge @@ -242,22 +267,47 @@ public Builder skipMerge(boolean skipMerge) { /** * Configure the on-disk FTS format version to write when creating a new index. * - *

    If unset, Lance chooses the current default format. + *

    If unset, Lance uses {@code LANCE_FTS_FORMAT_VERSION} when present and otherwise selects + * v3 for the code analyzer or {@code blockSize = 256}, and v2 for other indexes. Format v3 + * supports both posting block sizes. Formats v1 and v2 support only {@code blockSize = 128} and + * cannot be used with the code analyzer. * - * @param formatVersion FTS format version, must be 1 or 2 + * @param formatVersion FTS format version, must be 1, 2, or 3 * @return this builder * @throws IllegalArgumentException */ public Builder formatVersion(int formatVersion) { - if (formatVersion != 1 && formatVersion != 2) { - throw new IllegalArgumentException("formatVersion must be 1 or 2"); + if (formatVersion != 1 && formatVersion != 2 && formatVersion != 3) { + throw new IllegalArgumentException("formatVersion must be 1, 2, or 3"); } this.formatVersion = formatVersion; return this; } + /** + * Configure the unit treated as one FTS document. + * + *

    {@link DocumentGranularity#LIST_ELEMENT} uses each element of the deepest list on the + * indexed field path as one document. The default is {@link DocumentGranularity#ROW}. + * + * @param documentGranularity document boundary semantics + * @return this builder + */ + public Builder documentGranularity(DocumentGranularity documentGranularity) { + this.documentGranularity = + Objects.requireNonNull(documentGranularity, "documentGranularity must not be null"); + return this; + } + /** Build a {@link ScalarIndexParams} instance for an inverted index. */ public ScalarIndexParams build() { + if (formatVersion != null) { + Preconditions.checkArgument( + formatVersion == 3 || blockSize == 128, "formatVersion 1 and 2 require blockSize 128"); + Preconditions.checkArgument( + !"code".equals(baseTokenizer) || formatVersion == 3, + "baseTokenizer 'code' requires formatVersion 3"); + } Map params = new HashMap<>(); if (baseTokenizer != null) { params.put("base_tokenizer", baseTokenizer); @@ -300,12 +350,16 @@ public ScalarIndexParams build() { if (prefixOnly != null) { params.put("prefix_only", prefixOnly); } + if (blockSize != null) { + params.put("block_size", blockSize); + } if (skipMerge != null) { params.put("skip_merge", skipMerge); } if (formatVersion != null) { params.put("format_version", formatVersion); } + params.put("document_granularity", documentGranularity.toRustString()); String json = JsonUtils.toJson(params); return ScalarIndexParams.create(INDEX_TYPE, json); diff --git a/java/src/main/java/org/lance/ipc/AsyncScanner.java b/java/src/main/java/org/lance/ipc/AsyncScanner.java index 6e515e3546c..b60d91c25f3 100644 --- a/java/src/main/java/org/lance/ipc/AsyncScanner.java +++ b/java/src/main/java/org/lance/ipc/AsyncScanner.java @@ -14,6 +14,7 @@ package org.lance.ipc; import org.lance.Dataset; +import org.lance.LanceException; import org.lance.LockManager; import org.apache.arrow.c.ArrowArrayStream; @@ -69,6 +70,8 @@ public static AsyncScanner create( options.getSubstraitFilter(), options.getFilter(), options.getBatchSize(), + options.getBatchSizeBytes(), + options.getIoBufferSize(), options.getLimit(), options.getOffset(), options.getNearest(), @@ -77,6 +80,9 @@ public static AsyncScanner create( options.isWithRowId(), options.isWithRowAddress(), options.getBatchReadahead(), + options.getFragmentReadahead(), + options.isScanInOrder(), + options.getLateMaterialization(), options.getColumnOrderings(), options.isUseScalarIndex(), options.isFastSearch(), @@ -95,6 +101,8 @@ static native AsyncScanner createAsyncScanner( Optional substraitFilter, Optional filter, Optional batchSize, + Optional batchSizeBytes, + Optional ioBufferSize, Optional limit, Optional offset, Optional query, @@ -103,6 +111,9 @@ static native AsyncScanner createAsyncScanner( boolean withRowId, boolean withRowAddress, int batchReadahead, + Optional fragmentReadahead, + boolean scanInOrder, + Optional lateMaterialization, Optional> columnOrderings, boolean useScalarIndex, boolean fastSearch, @@ -137,18 +148,18 @@ public CompletableFuture scanBatchesAsync() { pendingTasks.remove(taskId); if (error != null) { - throw new RuntimeException("Scan failed", error); + throw new LanceException("Scan failed", error); } if (streamPtr < 0) { - throw new RuntimeException("Native scan error"); + throw new LanceException("Native scan returned an invalid stream pointer"); } try { ArrowArrayStream stream = ArrowArrayStream.wrap(streamPtr); return Data.importArrayStream(allocator, stream); } catch (Exception e) { - throw new RuntimeException(e); + throw new LanceException("Failed to import scan stream", e); } }); } @@ -166,7 +177,7 @@ private void completeTask(long taskId, long resultPtr) { private void failTask(long taskId, String errorMessage) { CompletableFuture future = pendingTasks.get(taskId); if (future != null) { - future.completeExceptionally(new RuntimeException(errorMessage)); + future.completeExceptionally(new LanceException(errorMessage)); } } diff --git a/java/src/main/java/org/lance/ipc/FullTextQuery.java b/java/src/main/java/org/lance/ipc/FullTextQuery.java index 0163def2f76..73badab9a49 100755 --- a/java/src/main/java/org/lance/ipc/FullTextQuery.java +++ b/java/src/main/java/org/lance/ipc/FullTextQuery.java @@ -13,6 +13,8 @@ */ package org.lance.ipc; +import org.lance.DocumentGranularity; + import com.google.common.base.MoreObjects; import org.apache.arrow.util.Preconditions; @@ -21,7 +23,13 @@ import java.util.Objects; import java.util.Optional; -/** Base type for full text search queries used by Lance scanner. */ +/** + * Base type for full text search queries used by Lance scanner. + * + *

    Match and phrase overloads without a {@link DocumentGranularity} infer the unique indexed + * granularity for the field. They use row documents when no index exists and fail as ambiguous when + * row and list-element indexes coexist. + */ public abstract class FullTextQuery { public enum Type { MATCH, @@ -37,8 +45,11 @@ public enum Operator { } public enum Occur { + /** The clause may match and contributes its score when it does. */ SHOULD, + /** The clause must match and contributes its score. */ MUST, + /** The clause must not match and never contributes to the score. */ MUST_NOT } @@ -63,7 +74,14 @@ public FullTextQuery getQuery() { public abstract Type getType(); public static FullTextQuery match(String queryText, String column) { - return match(queryText, column, 1.0f, Optional.empty(), 50, Operator.OR, 0); + return new MatchQuery( + queryText, column, 1.0f, Optional.empty(), 50, Operator.OR, 0, Optional.empty()); + } + + public static FullTextQuery match( + String queryText, String column, DocumentGranularity documentGranularity) { + return match( + queryText, column, 1.0f, Optional.empty(), 50, Operator.OR, 0, documentGranularity); } public static FullTextQuery match( @@ -75,15 +93,58 @@ public static FullTextQuery match( Operator operator, int prefixLength) { return new MatchQuery( - queryText, column, boost, fuzziness, maxExpansions, operator, prefixLength); + queryText, + column, + boost, + fuzziness, + maxExpansions, + operator, + prefixLength, + Optional.empty()); + } + + public static FullTextQuery match( + String queryText, + String column, + float boost, + Optional fuzziness, + int maxExpansions, + Operator operator, + int prefixLength, + DocumentGranularity documentGranularity) { + return new MatchQuery( + queryText, + column, + boost, + fuzziness, + maxExpansions, + operator, + prefixLength, + Optional.of( + Objects.requireNonNull(documentGranularity, "documentGranularity must not be null"))); } public static FullTextQuery phrase(String queryText, String column) { - return phrase(queryText, column, 0); + return new PhraseQuery(queryText, column, 0, Optional.empty()); + } + + public static FullTextQuery phrase( + String queryText, String column, DocumentGranularity documentGranularity) { + return phrase(queryText, column, 0, documentGranularity); } public static FullTextQuery phrase(String queryText, String column, int slop) { - return new PhraseQuery(queryText, column, slop); + return new PhraseQuery(queryText, column, slop, Optional.empty()); + } + + public static FullTextQuery phrase( + String queryText, String column, int slop, DocumentGranularity documentGranularity) { + return new PhraseQuery( + queryText, + column, + slop, + Optional.of( + Objects.requireNonNull(documentGranularity, "documentGranularity must not be null"))); } public static FullTextQuery multiMatch(String queryText, List columns) { @@ -117,6 +178,7 @@ public static final class MatchQuery extends FullTextQuery { private final int maxExpansions; private final Operator operator; private final int prefixLength; + private final Optional documentGranularity; MatchQuery( String queryText, @@ -125,7 +187,8 @@ public static final class MatchQuery extends FullTextQuery { Optional fuzziness, int maxExpansions, Operator operator, - int prefixLength) { + int prefixLength, + Optional documentGranularity) { Preconditions.checkArgument( queryText != null && !queryText.isEmpty(), "queryText must not be null or empty"); Preconditions.checkArgument( @@ -140,6 +203,7 @@ public static final class MatchQuery extends FullTextQuery { this.maxExpansions = maxExpansions; this.operator = operator == null ? Operator.OR : operator; this.prefixLength = prefixLength; + this.documentGranularity = Objects.requireNonNull(documentGranularity); } @Override @@ -175,6 +239,11 @@ public int getPrefixLength() { return prefixLength; } + /** Returns the explicit granularity, or empty when query planning should infer it. */ + public Optional getDocumentGranularity() { + return documentGranularity; + } + @Override public boolean equals(Object o) { if (this == o) return true; @@ -184,6 +253,7 @@ public boolean equals(Object o) { && maxExpansions == other.maxExpansions && prefixLength == other.prefixLength && operator == other.operator + && Objects.equals(documentGranularity, other.documentGranularity) && Objects.equals(queryText, other.queryText) && Objects.equals(column, other.column) && Objects.equals(fuzziness, other.fuzziness); @@ -192,7 +262,14 @@ public boolean equals(Object o) { @Override public int hashCode() { return Objects.hash( - queryText, column, boost, fuzziness, maxExpansions, operator, prefixLength); + queryText, + column, + boost, + fuzziness, + maxExpansions, + operator, + prefixLength, + documentGranularity); } @Override @@ -206,6 +283,7 @@ public String toString() { .add("maxExpansions", maxExpansions) .add("operator", operator) .add("prefixLength", prefixLength) + .add("documentGranularity", documentGranularity) .toString(); } } @@ -215,8 +293,13 @@ public static final class PhraseQuery extends FullTextQuery { private final String queryText; private final String column; private final int slop; + private final Optional documentGranularity; - PhraseQuery(String queryText, String column, int slop) { + PhraseQuery( + String queryText, + String column, + int slop, + Optional documentGranularity) { Preconditions.checkArgument( queryText != null && !queryText.isEmpty(), "queryText must not be null or empty"); Preconditions.checkArgument( @@ -226,6 +309,7 @@ public static final class PhraseQuery extends FullTextQuery { this.queryText = queryText; this.column = column; this.slop = slop; + this.documentGranularity = Objects.requireNonNull(documentGranularity); } @Override @@ -245,19 +329,25 @@ public int getSlop() { return slop; } + /** Returns the explicit granularity, or empty when query planning should infer it. */ + public Optional getDocumentGranularity() { + return documentGranularity; + } + @Override public boolean equals(Object o) { if (this == o) return true; if (!(o instanceof PhraseQuery)) return false; PhraseQuery other = (PhraseQuery) o; return slop == other.slop + && Objects.equals(documentGranularity, other.documentGranularity) && Objects.equals(queryText, other.queryText) && Objects.equals(column, other.column); } @Override public int hashCode() { - return Objects.hash(queryText, column, slop); + return Objects.hash(queryText, column, slop, documentGranularity); } @Override @@ -267,6 +357,7 @@ public String toString() { .add("queryText", queryText) .add("column", column) .add("slop", slop) + .add("documentGranularity", documentGranularity) .toString(); } } diff --git a/java/src/main/java/org/lance/ipc/LanceScanner.java b/java/src/main/java/org/lance/ipc/LanceScanner.java index 3a413e0ccfd..4cf50f6bdcf 100644 --- a/java/src/main/java/org/lance/ipc/LanceScanner.java +++ b/java/src/main/java/org/lance/ipc/LanceScanner.java @@ -14,6 +14,7 @@ package org.lance.ipc; import org.lance.Dataset; +import org.lance.LanceException; import org.lance.LockManager; import org.apache.arrow.c.ArrowArrayStream; @@ -65,6 +66,8 @@ public static LanceScanner create( options.getSubstraitFilter(), options.getFilter(), options.getBatchSize(), + options.getBatchSizeBytes(), + options.getIoBufferSize(), options.getLimit(), options.getOffset(), options.getNearest(), @@ -73,6 +76,9 @@ public static LanceScanner create( options.isWithRowId(), options.isWithRowAddress(), options.getBatchReadahead(), + options.getFragmentReadahead(), + options.isScanInOrder(), + options.getLateMaterialization(), options.getColumnOrderings(), options.isUseScalarIndex(), options.isFastSearch(), @@ -94,6 +100,8 @@ static native LanceScanner createScanner( Optional substraitFilter, Optional filter, Optional batchSize, + Optional batchSizeBytes, + Optional ioBufferSize, Optional limit, Optional offset, Optional query, @@ -102,6 +110,9 @@ static native LanceScanner createScanner( boolean withRowId, boolean withRowAddress, int batchReadahead, + Optional fragmentReadahead, + boolean scanInOrder, + Optional lateMaterialization, Optional> columnOrderings, boolean useScalarIndex, boolean fastSearch, @@ -140,8 +151,7 @@ public ArrowReader scanBatches() { openStream(s.memoryAddress()); return Data.importArrayStream(allocator, s); } catch (IOException e) { - // TODO: handle IO exception? - throw new RuntimeException(e); + throw new LanceException("Failed to open scan stream", e); } } } diff --git a/java/src/main/java/org/lance/ipc/MaterializationStyle.java b/java/src/main/java/org/lance/ipc/MaterializationStyle.java new file mode 100644 index 00000000000..e0ab095e06c --- /dev/null +++ b/java/src/main/java/org/lance/ipc/MaterializationStyle.java @@ -0,0 +1,117 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.lance.ipc; + +import org.apache.arrow.util.Preconditions; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +/** + * Controls whether projected columns are fetched before or after filtering. + * + *

    For example, a selective filter can fetch a large payload column only for matching rows: + * + * 

    {@code
+ * ScanOptions options = new ScanOptions.Builder()
+ *     .filter("id = 42")
+ *     .lateMaterialization(MaterializationStyle.allEarlyExcept(
+ *         Collections.singletonList("payload")))
+ *     .build();
+ * }
    + * + *

    This policy only affects projected columns that are not used by a filter in a regular scan. + * Vector search and full-text search always use late materialization. + */ +public final class MaterializationStyle { + /** Available materialization policies. */ + public enum Mode { + /** Let Lance choose based on the storage system and column size. */ + HEURISTIC("heuristic"), + /** Fetch all eligible projected columns after filtering. */ + ALL_LATE("all_late"), + /** Fetch all projected columns before filtering. */ + ALL_EARLY("all_early"), + /** Fetch only the specified projected columns after filtering. */ + ALL_EARLY_EXCEPT("all_early_except"); + + private final String rustValue; + + Mode(String rustValue) { + this.rustValue = rustValue; + } + + /** Returns the explicit value expected by the Rust binding. */ + public String toRustString() { + return rustValue; + } + } + + private final Mode mode; + private final List columns; + + private MaterializationStyle(Mode mode, List columns) { + this.mode = mode; + this.columns = Collections.unmodifiableList(new ArrayList<>(columns)); + } + + /** Use Lance's storage-aware heuristic. This is the default when no style is specified. */ + public static MaterializationStyle heuristic() { + return new MaterializationStyle(Mode.HEURISTIC, Collections.emptyList()); + } + + /** Fetch all eligible projected columns after filtering. */ + public static MaterializationStyle allLate() { + return new MaterializationStyle(Mode.ALL_LATE, Collections.emptyList()); + } + + /** Fetch all projected columns before filtering. */ + public static MaterializationStyle allEarly() { + return new MaterializationStyle(Mode.ALL_EARLY, Collections.emptyList()); + } + + /** + * Fetch only the specified projected columns after filtering; all other columns are fetched + * before filtering. An empty list is equivalent to {@link #allEarly()}. + */ + public static MaterializationStyle allEarlyExcept(List lateColumns) { + Preconditions.checkNotNull(lateColumns, "lateColumns must not be null"); + for (String column : lateColumns) { + Preconditions.checkArgument( + column != null && !column.isEmpty(), "lateColumns must not contain null or empty names"); + } + return new MaterializationStyle(Mode.ALL_EARLY_EXCEPT, lateColumns); + } + + /** Returns the selected materialization mode. */ + public Mode getMode() { + return mode; + } + + /** Returns the explicit value expected by the Rust binding. */ + public String toRustString() { + return mode.toRustString(); + } + + /** Columns to fetch after filtering when using {@link Mode#ALL_EARLY_EXCEPT}. */ + public List getColumns() { + return columns; + } + + @Override + public String toString() { + return "MaterializationStyle{" + "mode=" + mode + ", columns=" + columns + '}'; + } +} diff --git a/java/src/main/java/org/lance/ipc/ScanOptions.java b/java/src/main/java/org/lance/ipc/ScanOptions.java index a9aad590c2b..1287652d37b 100644 --- a/java/src/main/java/org/lance/ipc/ScanOptions.java +++ b/java/src/main/java/org/lance/ipc/ScanOptions.java @@ -24,6 +24,8 @@ public class ScanOptions { private final Optional> fragmentIds; private final Optional batchSize; + private final Optional batchSizeBytes; + private final Optional ioBufferSize; private final Optional> columns; private final Optional filter; private final Optional substraitFilter; @@ -35,6 +37,9 @@ public class ScanOptions { private final boolean withRowId; private final boolean withRowAddress; private final int batchReadahead; + private final Optional fragmentReadahead; + private final boolean scanInOrder; + private final Optional lateMaterialization; private final Optional> columnOrderings; private final boolean useScalarIndex; private final Optional substraitAggregate; @@ -131,11 +136,86 @@ public ScanOptions( boolean includeDeletedRows, boolean strictBatchSize, boolean disableScoringAutoprojection) { + this( + fragmentIds, + batchSize, + Optional.empty(), + Optional.empty(), + columns, + filter, + substraitFilter, + limit, + offset, + nearest, + fullTextQuery, + prefilter, + withRowId, + withRowAddress, + batchReadahead, + Optional.empty(), + true, + Optional.empty(), + columnOrderings, + useScalarIndex, + substraitAggregate, + collectStats, + fastSearch, + includeDeletedRows, + strictBatchSize, + disableScoringAutoprojection); + } + + private ScanOptions( + Optional> fragmentIds, + Optional batchSize, + Optional batchSizeBytes, + Optional ioBufferSize, + Optional> columns, + Optional filter, + Optional substraitFilter, + Optional limit, + Optional offset, + Optional nearest, + Optional fullTextQuery, + boolean prefilter, + boolean withRowId, + boolean withRowAddress, + int batchReadahead, + Optional fragmentReadahead, + boolean scanInOrder, + Optional lateMaterialization, + Optional> columnOrderings, + boolean useScalarIndex, + Optional substraitAggregate, + boolean collectStats, + boolean fastSearch, + boolean includeDeletedRows, + boolean strictBatchSize, + boolean disableScoringAutoprojection) { Preconditions.checkArgument( !(filter.isPresent() && substraitFilter.isPresent()), "cannot set both substrait filter and string filter"); + Preconditions.checkArgument( + batchReadahead > 0, "batchReadahead must be greater than 0, got %s", batchReadahead); + batchSizeBytes.ifPresent( + value -> + Preconditions.checkArgument( + value > 0, "batchSizeBytes must be greater than 0, got %s", value)); + ioBufferSize.ifPresent( + value -> + Preconditions.checkArgument( + value > 0, "ioBufferSize must be greater than 0, got %s", value)); + fragmentReadahead.ifPresent( + value -> + Preconditions.checkArgument( + value > 0, "fragmentReadahead must be greater than 0, got %s", value)); + Preconditions.checkArgument( + !(strictBatchSize && batchSizeBytes.isPresent()), + "strictBatchSize=true cannot be combined with batchSizeBytes"); this.fragmentIds = fragmentIds; this.batchSize = batchSize; + this.batchSizeBytes = batchSizeBytes; + this.ioBufferSize = ioBufferSize; this.columns = columns; this.filter = filter; this.substraitFilter = substraitFilter; @@ -147,6 +227,9 @@ public ScanOptions( this.withRowId = withRowId; this.withRowAddress = withRowAddress; this.batchReadahead = batchReadahead; + this.fragmentReadahead = fragmentReadahead; + this.scanInOrder = scanInOrder; + this.lateMaterialization = lateMaterialization; this.columnOrderings = columnOrderings; this.useScalarIndex = useScalarIndex; this.substraitAggregate = substraitAggregate; @@ -175,6 +258,24 @@ public Optional getBatchSize() { return batchSize; } + /** + * Get the target batch size in bytes. + * + * @return Optional containing the target batch size in bytes if specified, otherwise empty. + */ + public Optional getBatchSizeBytes() { + return batchSizeBytes; + } + + /** + * Get the I/O buffer size in bytes. + * + * @return Optional containing the I/O buffer size if specified, otherwise empty. + */ + public Optional getIoBufferSize() { + return ioBufferSize; + } + /** * Get the columns. * @@ -274,6 +375,33 @@ public int getBatchReadahead() { return batchReadahead; } + /** + * Get the number of fragments to read ahead concurrently. + * + * @return Optional containing the fragment readahead if specified, otherwise empty. + */ + public Optional getFragmentReadahead() { + return fragmentReadahead; + } + + /** + * Get whether batches should be returned in fragment and batch order. + * + * @return true to preserve scan order, false to return batches as soon as they are ready. + */ + public boolean isScanInOrder() { + return scanInOrder; + } + + /** + * Get the materialization policy for projected columns. + * + * @return Optional containing the materialization policy if specified, otherwise empty. + */ + public Optional getLateMaterialization() { + return lateMaterialization; + } + public Optional> getColumnOrderings() { return columnOrderings; } @@ -341,6 +469,8 @@ public String toString() { return MoreObjects.toStringHelper(this) .add("fragmentIds", fragmentIds.orElse(null)) .add("batchSize", batchSize.orElse(null)) + .add("batchSizeBytes", batchSizeBytes.orElse(null)) + .add("ioBufferSize", ioBufferSize.orElse(null)) .add("columns", columns.orElse(null)) .add("filter", filter.orElse(null)) .add( @@ -354,6 +484,9 @@ public String toString() { .add("withRowId", withRowId) .add("WithRowAddress", withRowAddress) .add("batchReadahead", batchReadahead) + .add("fragmentReadahead", fragmentReadahead.orElse(null)) + .add("scanInOrder", scanInOrder) + .add("lateMaterialization", lateMaterialization.orElse(null)) .add("columnOrdering", columnOrderings) .add("useScalarIndex", useScalarIndex) .add("fastSearch", fastSearch) @@ -371,6 +504,8 @@ public String toString() { public static class Builder { private Optional> fragmentIds = Optional.empty(); private Optional batchSize = Optional.empty(); + private Optional batchSizeBytes = Optional.empty(); + private Optional ioBufferSize = Optional.empty(); private Optional> columns = Optional.empty(); private Optional filter = Optional.empty(); private Optional substraitFilter = Optional.empty(); @@ -382,6 +517,9 @@ public static class Builder { private boolean withRowId = false; private boolean withRowAddress = false; private int batchReadahead = 16; + private Optional fragmentReadahead = Optional.empty(); + private boolean scanInOrder = true; + private Optional lateMaterialization = Optional.empty(); private Optional> columnOrderings = Optional.empty(); private boolean useScalarIndex = true; private boolean fastSearch = false; @@ -401,6 +539,8 @@ public Builder() {} public Builder(ScanOptions options) { this.fragmentIds = options.getFragmentIds(); this.batchSize = options.getBatchSize(); + this.batchSizeBytes = options.getBatchSizeBytes(); + this.ioBufferSize = options.getIoBufferSize(); this.columns = options.getColumns(); this.filter = options.getFilter(); this.substraitFilter = options.getSubstraitFilter(); @@ -412,6 +552,9 @@ public Builder(ScanOptions options) { this.withRowId = options.isWithRowId(); this.withRowAddress = options.isWithRowAddress(); this.batchReadahead = options.getBatchReadahead(); + this.fragmentReadahead = options.getFragmentReadahead(); + this.scanInOrder = options.isScanInOrder(); + this.lateMaterialization = options.getLateMaterialization(); this.columnOrderings = options.getColumnOrderings(); this.useScalarIndex = options.isUseScalarIndex(); this.fastSearch = options.isFastSearch(); @@ -444,6 +587,34 @@ public Builder batchSize(long batchSize) { return this; } + /** + * Set the approximate target size of each returned Arrow record batch in bytes. + * + *

    If {@link #batchSize(long)} is also set, the limit reached first determines the batch + * size. This option cannot be combined with {@link #strictBatchSize(boolean)}. + * + * @param batchSizeBytes target batch size in bytes; must be greater than zero. + * @return Builder instance for method chaining. + */ + public Builder batchSizeBytes(long batchSizeBytes) { + this.batchSizeBytes = Optional.of(batchSizeBytes); + return this; + } + + /** + * Set the amount of memory reserved for buffering storage I/O. + * + *

    This controls scanner backpressure but is not a hard upper bound on total scanner memory. + * It currently applies only to v2 files. + * + * @param ioBufferSize I/O buffer size in bytes; must be greater than zero. + * @return Builder instance for method chaining. + */ + public Builder ioBufferSize(long ioBufferSize) { + this.ioBufferSize = Optional.of(ioBufferSize); + return this; + } + /** * Set the columns. * @@ -565,6 +736,46 @@ public Builder batchReadahead(int batchReadahead) { return this; } + /** + * Set the number of fragments to read ahead concurrently. + * + * @param fragmentReadahead number of fragments to read ahead; must be greater than zero. + * @return Builder instance for method chaining. + */ + public Builder fragmentReadahead(int fragmentReadahead) { + this.fragmentReadahead = Optional.of(fragmentReadahead); + return this; + } + + /** + * Set whether batches are returned in fragment and batch order. + * + *

    Disabling ordering can increase scan concurrency at the cost of additional memory. This + * option is ignored for v2 files, which always scan in order. It is also ignored when a column + * ordering or nearest-neighbor query is configured; the query determines the result order. + * + * @param scanInOrder true to preserve scan order, false to return ready batches immediately. + * @return Builder instance for method chaining. + */ + public Builder scanInOrder(boolean scanInOrder) { + this.scanInOrder = scanInOrder; + return this; + } + + /** + * Set when projected columns that are not used by the filter should be materialized. + * + *

    This option only affects regular scans. Vector search and full-text search always use late + * materialization. + * + * @param lateMaterialization materialization policy. + * @return Builder instance for method chaining. + */ + public Builder lateMaterialization(MaterializationStyle lateMaterialization) { + this.lateMaterialization = Optional.of(lateMaterialization); + return this; + } + public Builder setColumnOrderings(List columnOrderings) { this.columnOrderings = Optional.of(columnOrderings); return this; @@ -667,6 +878,8 @@ public ScanOptions build() { return new ScanOptions( fragmentIds, batchSize, + batchSizeBytes, + ioBufferSize, columns, filter, substraitFilter, @@ -678,6 +891,9 @@ public ScanOptions build() { withRowId, withRowAddress, batchReadahead, + fragmentReadahead, + scanInOrder, + lateMaterialization, columnOrderings, useScalarIndex, substraitAggregate, diff --git a/java/src/main/java/org/lance/ipc/ScanStats.java b/java/src/main/java/org/lance/ipc/ScanStats.java index 24926d00911..e2160d0f371 100755 --- a/java/src/main/java/org/lance/ipc/ScanStats.java +++ b/java/src/main/java/org/lance/ipc/ScanStats.java @@ -30,6 +30,13 @@ public final class ScanStats { private final long indicesLoaded; private final long partsLoaded; private final long indexComparisons; + + /** Number of index cache page lookups served from memory in this scan. */ + private final long indexCacheHits; + + /** Number of index cache page lookups that had to load from storage in this scan. */ + private final long indexCacheMisses; + private final Map allCounts; private final Map allTimes; @@ -40,6 +47,8 @@ public ScanStats( long indicesLoaded, long partsLoaded, long indexComparisons, + long indexCacheHits, + long indexCacheMisses, Map allCounts, Map allTimes) { this.iops = iops; @@ -48,10 +57,42 @@ public ScanStats( this.indicesLoaded = indicesLoaded; this.partsLoaded = partsLoaded; this.indexComparisons = indexComparisons; + this.indexCacheHits = indexCacheHits; + this.indexCacheMisses = indexCacheMisses; this.allCounts = freezeMap(allCounts); this.allTimes = freezeMap(allTimes); } + /** + * Backwards-compatible constructor kept for existing callers that predate the addition of + * per-query index cache statistics. New code should use the 10-argument constructor that also + * accepts {@code indexCacheHits} and {@code indexCacheMisses}. + * + * @deprecated Use {@link #ScanStats(long, long, long, long, long, long, long, long, Map, Map)}. + */ + @Deprecated + public ScanStats( + long iops, + long requests, + long bytesRead, + long indicesLoaded, + long partsLoaded, + long indexComparisons, + Map allCounts, + Map allTimes) { + this( + iops, + requests, + bytesRead, + indicesLoaded, + partsLoaded, + indexComparisons, + 0L, + 0L, + allCounts, + allTimes); + } + private static Map freezeMap(Map map) { if (map == null || map.isEmpty()) { return Collections.emptyMap(); @@ -83,6 +124,41 @@ public long getIndexComparisons() { return indexComparisons; } + /** + * Number of index cache page lookups where the loader was not executed in this scan. + * + *

    Counts both true cache hits on already-populated entries and coalesced concurrent loads (a + * follower attached to another caller's in-flight load). + * + *

    Instrumented boundaries in this release: BTree, IVF v2 (write-cache scan path), inverted + * posting list (grouped and per-token) and its per-token metadata, inverted phrase positions, + * bitmap (Equals / Range / IsIn), ngram, rtree. + * + *

    Caveats: + * + *

      + *
    • IVF v2 streaming scans and legacy v1 IVF partitions bypass the cache by design and are + * therefore reported as a miss on every call. + *
    • A cold posting-list lookup on the grouped inverted layout can record up to two misses + * (group + per-token metadata) for a single term. + *
    + * + *

    Uninstrumented paths (HNSW graph pages, quantizer codebooks) do not contribute to either + * counter. See the sibling {@link #getIndexCacheMisses()} for the paired counter. + */ + public long getIndexCacheHits() { + return indexCacheHits; + } + + /** + * Number of index cache page lookups where the loader ran in this scan (the page was not resident + * and had to be materialised, typically from storage). See {@link #getIndexCacheHits()} for the + * paired counter and the list of instrumented boundaries. + */ + public long getIndexCacheMisses() { + return indexCacheMisses; + } + public Map getAllCounts() { return allCounts; } @@ -106,6 +182,8 @@ public boolean equals(Object o) { && indicesLoaded == that.indicesLoaded && partsLoaded == that.partsLoaded && indexComparisons == that.indexComparisons + && indexCacheHits == that.indexCacheHits + && indexCacheMisses == that.indexCacheMisses && Objects.equals(allCounts, that.allCounts) && Objects.equals(allTimes, that.allTimes); } @@ -119,6 +197,8 @@ public int hashCode() { indicesLoaded, partsLoaded, indexComparisons, + indexCacheHits, + indexCacheMisses, allCounts, allTimes); } @@ -138,6 +218,10 @@ public String toString() { + partsLoaded + ", indexComparisons=" + indexComparisons + + ", indexCacheHits=" + + indexCacheHits + + ", indexCacheMisses=" + + indexCacheMisses + ", allCounts=" + allCounts + ", allTimes=" diff --git a/java/src/main/java/org/lance/memwal/MergedGeneration.java b/java/src/main/java/org/lance/memwal/CompactedSsTable.java similarity index 77% rename from java/src/main/java/org/lance/memwal/MergedGeneration.java rename to java/src/main/java/org/lance/memwal/CompactedSsTable.java index 481e79ba3c0..cb703b42641 100644 --- a/java/src/main/java/org/lance/memwal/MergedGeneration.java +++ b/java/src/main/java/org/lance/memwal/CompactedSsTable.java @@ -17,32 +17,32 @@ import com.google.common.base.Preconditions; /** - * Identifies a flushed MemWAL generation that has been merged into the base table. + * Points to an SSTable compacted into the base table. * - *

    Pass a list of these to {@link org.lance.merge.MergeInsertParams#markGenerationsAsMerged} so - * Lance knows which generations are now part of the base table. + *

    Pass a list of these to {@link org.lance.merge.MergeInsertParams#markSstablesAsCompacted} so + * Lance can record compaction progress. */ -public class MergedGeneration { +public class CompactedSsTable { private final String shardId; private final long generation; /** * @param shardId UUID string for the write shard - * @param generation generation number from {@link ShardSnapshot#flushedGenerations()} + * @param generation generation number from {@link ShardSnapshot#sstables()} */ - public MergedGeneration(String shardId, long generation) { + public CompactedSsTable(String shardId, long generation) { Preconditions.checkNotNull(shardId, "shardId must not be null"); this.shardId = shardId; this.generation = generation; } /** UUID string for the write shard. */ - public String shardId() { + public String getShardId() { return shardId; } - /** The merged generation number. */ - public long generation() { + /** The compacted SSTable's generation number. */ + public long getGeneration() { return generation; } diff --git a/java/src/main/java/org/lance/memwal/LsmPointLookupPlanner.java b/java/src/main/java/org/lance/memwal/LsmPointLookupPlanner.java index 0488f9e4530..d58e4e81d57 100644 --- a/java/src/main/java/org/lance/memwal/LsmPointLookupPlanner.java +++ b/java/src/main/java/org/lance/memwal/LsmPointLookupPlanner.java @@ -44,7 +44,7 @@ public class LsmPointLookupPlanner implements AutoCloseable { /** * @param dataset the base dataset - * @param shardSnapshots shard snapshots specifying the flushed generations to include + * @param shardSnapshots shard snapshots specifying the SSTables to include */ public LsmPointLookupPlanner(Dataset dataset, List shardSnapshots) { this(dataset, shardSnapshots, null); @@ -52,7 +52,7 @@ public LsmPointLookupPlanner(Dataset dataset, List shardSnapshots /** * @param dataset the base dataset - * @param shardSnapshots shard snapshots specifying the flushed generations to include + * @param shardSnapshots shard snapshots specifying the SSTables to include * @param pkColumns primary key column names; inferred from schema metadata when {@code null} */ public LsmPointLookupPlanner( diff --git a/java/src/main/java/org/lance/memwal/LsmScanner.java b/java/src/main/java/org/lance/memwal/LsmScanner.java index 29bfb097375..509e70f7b0a 100644 --- a/java/src/main/java/org/lance/memwal/LsmScanner.java +++ b/java/src/main/java/org/lance/memwal/LsmScanner.java @@ -31,8 +31,8 @@ * LSM-aware scanner covering all MemWAL data levels. * *

    Results are deduplicated by primary key, always returning the newest version of each row - * across the base table, flushed MemTables, and (when created from a {@link ShardWriter}) the - * active MemTable. + * across the base table, SSTables, and (when created from a {@link ShardWriter}) the active + * MemTable. * *

    The builder methods ({@link #project}, {@link #filter}, {@link #limit}, {@link * #withRowAddress}, {@link #withMemtableGen}) mutate this scanner and return it for chaining. @@ -55,7 +55,7 @@ private LsmScanner() {} * read-your-writes consistency. * * @param dataset the base dataset to scan - * @param shardSnapshots shard snapshots specifying the flushed generations to include + * @param shardSnapshots shard snapshots specifying the SSTables to include * @return an LSM scanner */ public static LsmScanner fromSnapshots(Dataset dataset, List shardSnapshots) { diff --git a/java/src/main/java/org/lance/memwal/LsmVectorSearchPlanner.java b/java/src/main/java/org/lance/memwal/LsmVectorSearchPlanner.java index 75c0d86c475..b17bb8e37a0 100644 --- a/java/src/main/java/org/lance/memwal/LsmVectorSearchPlanner.java +++ b/java/src/main/java/org/lance/memwal/LsmVectorSearchPlanner.java @@ -44,7 +44,7 @@ public class LsmVectorSearchPlanner implements AutoCloseable { /** * @param dataset the base dataset - * @param shardSnapshots shard snapshots specifying the flushed generations to include + * @param shardSnapshots shard snapshots specifying the SSTables to include * @param vectorColumn name of the {@code FixedSizeList} vector column */ public LsmVectorSearchPlanner( @@ -54,7 +54,7 @@ public LsmVectorSearchPlanner( /** * @param dataset the base dataset - * @param shardSnapshots shard snapshots specifying the flushed generations to include + * @param shardSnapshots shard snapshots specifying the SSTables to include * @param vectorColumn name of the {@code FixedSizeList} vector column * @param pkColumns primary key column names; inferred from schema metadata when {@code null} * @param distanceType distance metric, one of {@code "l2"}, {@code "cosine"}, {@code "dot"}, @@ -71,7 +71,7 @@ public LsmVectorSearchPlanner( /** * @param dataset the base dataset - * @param shardSnapshots shard snapshots specifying the flushed generations to include + * @param shardSnapshots shard snapshots specifying the SSTables to include * @param vectorColumn name of the {@code FixedSizeList} vector column * @param pkColumns primary key column names; inferred from schema metadata when {@code null} * @param distanceType distance metric, one of {@code "l2"}, {@code "cosine"}, {@code "dot"}, diff --git a/java/src/main/java/org/lance/memwal/MemTableStats.java b/java/src/main/java/org/lance/memwal/MemTableStats.java index a993d771b42..b1fbeede299 100644 --- a/java/src/main/java/org/lance/memwal/MemTableStats.java +++ b/java/src/main/java/org/lance/memwal/MemTableStats.java @@ -24,7 +24,8 @@ public class MemTableStats { private final long estimatedSizeBytes; private final long generation; private final Optional maxBufferedBatchPosition; - private final Optional maxFlushedBatchPosition; + private final long durableBatchCount; + private final long globalOffset; private final Optional pendingWalStartBatchPosition; private final Optional pendingWalEndBatchPosition; private final long pendingWalBatchCount; @@ -37,7 +38,8 @@ public MemTableStats( long estimatedSizeBytes, long generation, Long maxBufferedBatchPosition, - Long maxFlushedBatchPosition, + long durableBatchCount, + long globalOffset, Long pendingWalStartBatchPosition, Long pendingWalEndBatchPosition, long pendingWalBatchCount, @@ -48,7 +50,8 @@ public MemTableStats( this.estimatedSizeBytes = estimatedSizeBytes; this.generation = generation; this.maxBufferedBatchPosition = Optional.ofNullable(maxBufferedBatchPosition); - this.maxFlushedBatchPosition = Optional.ofNullable(maxFlushedBatchPosition); + this.durableBatchCount = durableBatchCount; + this.globalOffset = globalOffset; this.pendingWalStartBatchPosition = Optional.ofNullable(pendingWalStartBatchPosition); this.pendingWalEndBatchPosition = Optional.ofNullable(pendingWalEndBatchPosition); this.pendingWalBatchCount = pendingWalBatchCount; @@ -81,9 +84,17 @@ public Optional maxBufferedBatchPosition() { return maxBufferedBatchPosition; } - /** Highest WAL batch position flushed from the MemTable, if any. */ - public Optional maxFlushedBatchPosition() { - return maxFlushedBatchPosition; + /** + * Writer-global count of WAL-durable batches (exclusive; 0 means none). Compare against {@code + * globalOffset() + batchCount()} to see what this MemTable still owes the WAL. + */ + public long durableBatchCount() { + return durableBatchCount; + } + + /** Writer-global coordinate of this MemTable's batch 0. */ + public long globalOffset() { + return globalOffset; } /** First WAL batch position pending flush, if any. */ @@ -119,7 +130,8 @@ public String toString() { .add("estimatedSizeBytes", estimatedSizeBytes) .add("generation", generation) .add("maxBufferedBatchPosition", maxBufferedBatchPosition.orElse(null)) - .add("maxFlushedBatchPosition", maxFlushedBatchPosition.orElse(null)) + .add("durableBatchCount", durableBatchCount) + .add("globalOffset", globalOffset) .add("pendingWalStartBatchPosition", pendingWalStartBatchPosition.orElse(null)) .add("pendingWalEndBatchPosition", pendingWalEndBatchPosition.orElse(null)) .add("pendingWalBatchCount", pendingWalBatchCount) diff --git a/java/src/main/java/org/lance/memwal/ShardSnapshot.java b/java/src/main/java/org/lance/memwal/ShardSnapshot.java index 493b2fce496..42e20bb5177 100644 --- a/java/src/main/java/org/lance/memwal/ShardSnapshot.java +++ b/java/src/main/java/org/lance/memwal/ShardSnapshot.java @@ -24,13 +24,13 @@ * Snapshot of a MemWAL shard's state, used when constructing scanners and planners. * *

    The builder methods ({@link #withSpecId}, {@link #withCurrentGeneration}, {@link - * #withFlushedGeneration}) mutate this instance and return it for chaining. + * #withSsTable}) mutate this instance and return it for chaining. */ public class ShardSnapshot { private final String shardId; private int specId = 0; private long currentGeneration = 0; - private final List flushedGenerations = new ArrayList<>(); + private final List sstables = new ArrayList<>(); /** * @param shardId UUID string for the write shard @@ -52,10 +52,10 @@ public ShardSnapshot withCurrentGeneration(long currentGeneration) { return this; } - /** Add a flushed generation with its storage path. */ - public ShardSnapshot withFlushedGeneration(long generation, String path) { + /** Add an SSTable with its storage path. */ + public ShardSnapshot withSsTable(long generation, String path) { Preconditions.checkNotNull(path, "path must not be null"); - this.flushedGenerations.add(new FlushedGeneration(generation, path)); + this.sstables.add(new SsTable(generation, path)); return this; } @@ -74,9 +74,9 @@ public long currentGeneration() { return currentGeneration; } - /** The flushed generations included in this snapshot. */ - public List flushedGenerations() { - return Collections.unmodifiableList(flushedGenerations); + /** The SSTables included in this snapshot. */ + public List sstables() { + return Collections.unmodifiableList(sstables); } @Override @@ -85,7 +85,7 @@ public String toString() { .add("shardId", shardId) .add("specId", specId) .add("currentGeneration", currentGeneration) - .add("flushedGenerations", flushedGenerations) + .add("sstables", sstables) .toString(); } } diff --git a/java/src/main/java/org/lance/memwal/ShardWriter.java b/java/src/main/java/org/lance/memwal/ShardWriter.java index a5a3000bdba..da4c621e934 100644 --- a/java/src/main/java/org/lance/memwal/ShardWriter.java +++ b/java/src/main/java/org/lance/memwal/ShardWriter.java @@ -36,6 +36,7 @@ * 

    {@code
  * try (ShardWriter writer = dataset.memWalWriter(shardId)) {
  *   writer.put(reader);
+ *   writer.delete(keys);
  * }
  * }
    * @@ -99,6 +100,34 @@ public void put(ArrowReader reader) { private native void nativePut(long streamAddress); + /** + * Delete rows from the MemWAL by primary key. + * + *

    Each batch in {@code reader} must carry this shard's primary key column(s); other columns + * are ignored. Lance builds a tombstone row per key — the primary key plus {@code _tombstone = + * true} and null in every other column — and appends it like an ordinary write. The tombstone is + * the newest value for its key: it wins newest-per-PK resolution (suppressing the older real row) + * and is then dropped from query results. + * + *

    Only supported in memtable mode. Because a tombstone nulls every non-PK column, those + * columns must be nullable in the base schema; deleting against a schema with a non-nullable + * non-PK column errors. Deleting on a shard with no primary key columns also errors. + * + * @param reader the keys to delete; consumed fully by this call + */ + public void delete(ArrowReader reader) { + Preconditions.checkNotNull(reader, "reader must not be null"); + try (LockManager.ReadLock readLock = lockManager.acquireReadLock()) { + Preconditions.checkArgument(nativeShardWriterHandle != 0, "ShardWriter is closed"); + try (ArrowArrayStream stream = ArrowArrayStream.allocateNew(allocator)) { + Data.exportArrayStream(allocator, reader, stream); + nativeDelete(stream.memoryAddress()); + } + } + } + + private native void nativeDelete(long streamAddress); + /** Return a snapshot of cumulative write statistics. */ public WriteStats stats() { try (LockManager.ReadLock readLock = lockManager.acquireReadLock()) { @@ -122,9 +151,8 @@ public MemTableStats memtableStats() { /** * Create an LSM scanner that includes this writer's active MemTable. * - *

    The scanner covers the base table, the given flushed generations, and the current active - * MemTable, providing read-your-writes consistency. This writer's own shard is included - * automatically. + *

    The scanner covers the base table, the given SSTables, and the current active MemTable, + * providing read-your-writes consistency. This writer's own shard is included automatically. * * @param shardSnapshots snapshots of other shards to include * @return an LSM scanner diff --git a/java/src/main/java/org/lance/memwal/ShardWriterConfig.java b/java/src/main/java/org/lance/memwal/ShardWriterConfig.java index 40310907d3e..d6bf2aeb383 100644 --- a/java/src/main/java/org/lance/memwal/ShardWriterConfig.java +++ b/java/src/main/java/org/lance/memwal/ShardWriterConfig.java @@ -28,7 +28,6 @@ */ public class ShardWriterConfig { private Optional durableWrite = Optional.empty(); - private Optional syncIndexedWrite = Optional.empty(); private Optional maxWalBufferSize = Optional.empty(); private Optional maxWalFlushIntervalMs = Optional.empty(); private Optional maxMemtableSize = Optional.empty(); @@ -36,8 +35,6 @@ public class ShardWriterConfig { private Optional maxMemtableBatches = Optional.empty(); private Optional maxUnflushedMemtableBytes = Optional.empty(); private Optional manifestScanBatchSize = Optional.empty(); - private Optional asyncIndexBufferRows = Optional.empty(); - private Optional asyncIndexIntervalMs = Optional.empty(); private Optional backpressureLogIntervalMs = Optional.empty(); private Optional statsLogIntervalMs = Optional.empty(); private List hnswParams = Collections.emptyList(); @@ -48,12 +45,6 @@ public ShardWriterConfig withDurableWrite(boolean durableWrite) { return this; } - /** Whether indexed writes are applied synchronously. */ - public ShardWriterConfig withSyncIndexedWrite(boolean syncIndexedWrite) { - this.syncIndexedWrite = Optional.of(syncIndexedWrite); - return this; - } - /** Maximum size of the in-memory WAL buffer, in bytes. */ public ShardWriterConfig withMaxWalBufferSize(long maxWalBufferSize) { Preconditions.checkArgument( @@ -118,26 +109,6 @@ public ShardWriterConfig withManifestScanBatchSize(long manifestScanBatchSize) { return this; } - /** Number of rows buffered before an asynchronous index update is triggered. */ - public ShardWriterConfig withAsyncIndexBufferRows(long asyncIndexBufferRows) { - Preconditions.checkArgument( - asyncIndexBufferRows >= 0, - "asyncIndexBufferRows must not be negative, got %s", - asyncIndexBufferRows); - this.asyncIndexBufferRows = Optional.of(asyncIndexBufferRows); - return this; - } - - /** Interval between asynchronous index updates, in milliseconds. */ - public ShardWriterConfig withAsyncIndexIntervalMs(long asyncIndexIntervalMs) { - Preconditions.checkArgument( - asyncIndexIntervalMs >= 0, - "asyncIndexIntervalMs must not be negative, got %s", - asyncIndexIntervalMs); - this.asyncIndexIntervalMs = Optional.of(asyncIndexIntervalMs); - return this; - } - /** Interval between backpressure log messages, in milliseconds. */ public ShardWriterConfig withBackpressureLogIntervalMs(long backpressureLogIntervalMs) { Preconditions.checkArgument( @@ -172,10 +143,6 @@ public Optional durableWrite() { return durableWrite; } - public Optional syncIndexedWrite() { - return syncIndexedWrite; - } - public Optional maxWalBufferSize() { return maxWalBufferSize; } @@ -204,14 +171,6 @@ public Optional manifestScanBatchSize() { return manifestScanBatchSize; } - public Optional asyncIndexBufferRows() { - return asyncIndexBufferRows; - } - - public Optional asyncIndexIntervalMs() { - return asyncIndexIntervalMs; - } - public Optional backpressureLogIntervalMs() { return backpressureLogIntervalMs; } diff --git a/java/src/main/java/org/lance/memwal/FlushedGeneration.java b/java/src/main/java/org/lance/memwal/SsTable.java similarity index 78% rename from java/src/main/java/org/lance/memwal/FlushedGeneration.java rename to java/src/main/java/org/lance/memwal/SsTable.java index 66288161438..55cb2e63f77 100644 --- a/java/src/main/java/org/lance/memwal/FlushedGeneration.java +++ b/java/src/main/java/org/lance/memwal/SsTable.java @@ -15,22 +15,22 @@ import com.google.common.base.MoreObjects; -/** A flushed MemWAL generation and the storage path of its Lance files. */ -public class FlushedGeneration { +/** An SSTable and the storage path of its Lance files. */ +public class SsTable { private final long generation; private final String path; - public FlushedGeneration(long generation, String path) { + public SsTable(long generation, String path) { this.generation = generation; this.path = path; } - /** The generation number of this flushed MemTable. */ + /** The generation number of this SSTable. */ public long generation() { return generation; } - /** The storage path of the flushed Lance files. */ + /** The storage path of the SSTable Lance files. */ public String path() { return path; } diff --git a/java/src/main/java/org/lance/merge/MergeInsertParams.java b/java/src/main/java/org/lance/merge/MergeInsertParams.java index 2ae27b67cba..fedff8cbf26 100644 --- a/java/src/main/java/org/lance/merge/MergeInsertParams.java +++ b/java/src/main/java/org/lance/merge/MergeInsertParams.java @@ -13,7 +13,7 @@ */ package org.lance.merge; -import org.lance.memwal.MergedGeneration; +import org.lance.memwal.CompactedSsTable; import com.google.common.base.MoreObjects; import com.google.common.base.Preconditions; @@ -39,7 +39,8 @@ public class MergeInsertParams { private long retryTimeoutMs = 30 * 1000; private boolean skipAutoCleanup = false; private boolean useIndex = true; - private List markedGenerations = Collections.emptyList(); + private List compactedSstables = Collections.emptyList(); + private SourceDedupeBehavior sourceDedupeBehavior = SourceDedupeBehavior.Fail; public MergeInsertParams(List on) { this.on = on; @@ -245,17 +246,48 @@ public MergeInsertParams withUseIndex(boolean useIndex) { } /** - * Mark MemWAL generations as merged into the base table. + * Control how duplicate source rows are handled. * - *

    Use this when the merge insert incorporates data from MemWAL flushed generations. It updates - * the MemWAL generation tracking to prevent the same generations from being merged again. + *

    Default is {@link SourceDedupeBehavior#Fail}, which errors only when multiple source rows + * match the same target row (an ambiguous update or delete). It does not reject + * duplicate join keys among unmatched rows — those are all inserted. Use {@link + * SourceDedupeBehavior#FirstSeen} to keep the first row for each join key and skip subsequent + * duplicates, including unmatched rows that would otherwise insert the same non-null key more + * than once. * - * @param generations the flushed generations being merged + *

    Rows whose join keys contain NULL are never duplicates, because merge insert uses SQL + * equality where NULL does not equal NULL. + * + *

    If the source contains duplicates and {@code FirstSeen} behavior doesn't match your needs, + * sort the source data before passing it to the merge insert operation. + * + * @param sourceDedupeBehavior The behavior to apply when duplicate source rows are found + * @return This MergeInsertParams instance + */ + public MergeInsertParams withSourceDedupeBehavior(SourceDedupeBehavior sourceDedupeBehavior) { + Preconditions.checkNotNull(sourceDedupeBehavior); + this.sourceDedupeBehavior = sourceDedupeBehavior; + return this; + } + + /** + * Mark MemWAL SSTables as compacted into the base table. + * + *

    Use this when merge insert compacts MemWAL SSTables. It updates MemWAL compaction progress + * to prevent the same SSTables from being compacted again, in the same commit as the data. + * + *

    For multi-pass compaction, call this only on the final successful data-changing pass. + * Intermediate passes must not carry compaction progress. Lance cannot tell whether a caller has + * another pass planned, so it cannot enforce this: if a delete pass carried the marker and the + * process then died before the matching upsert, the recorded progress would claim rows were + * copied in that never were. + * + * @param sstables the SSTables being compacted * @return This MergeInsertParams instance */ - public MergeInsertParams markGenerationsAsMerged(List generations) { - Preconditions.checkNotNull(generations, "generations must not be null"); - this.markedGenerations = generations; + public MergeInsertParams markSstablesAsCompacted(List sstables) { + Preconditions.checkNotNull(sstables, "sstables must not be null"); + this.compactedSstables = sstables; return this; } @@ -263,8 +295,8 @@ public List on() { return on; } - public List markedGenerations() { - return markedGenerations; + public List getCompactedSstables() { + return compactedSstables; } public WhenMatched whenMatched() { @@ -319,6 +351,14 @@ public boolean useIndex() { return useIndex; } + public SourceDedupeBehavior sourceDedupeBehavior() { + return sourceDedupeBehavior; + } + + public String sourceDedupeBehaviorValue() { + return sourceDedupeBehavior.name(); + } + @Override public String toString() { return MoreObjects.toStringHelper(this) @@ -337,6 +377,7 @@ public String toString() { .add("retryTimeoutMs", retryTimeoutMs) .add("skipAutoCleanup", skipAutoCleanup) .add("useIndex", useIndex) + .add("sourceDedupeBehavior", sourceDedupeBehavior) .toString(); } @@ -396,4 +437,25 @@ public enum WhenNotMatchedBySource { */ DeleteIf, } + + /** + * Describes how to handle duplicate source rows. + * + *

    If the source contains duplicates and {@code FirstSeen} behavior doesn't match your needs, + * sort the source data before passing it to the merge insert operation. Rows whose join keys + * contain NULL are not duplicates because merge insert uses SQL equality, where NULL does not + * equal NULL. + */ + public enum SourceDedupeBehavior { + /** Fail if multiple source rows match the same target row (default). */ + Fail, + + /** + * Keep the first row for each join key and skip subsequent rows. + * + *

    This applies both to rows that match a target row and to unmatched rows that would + * otherwise insert the same non-null join key more than once. + */ + FirstSeen, + } } diff --git a/java/src/main/java/org/lance/operation/Merge.java b/java/src/main/java/org/lance/operation/Merge.java index bd83657384b..07ecb880d90 100644 --- a/java/src/main/java/org/lance/operation/Merge.java +++ b/java/src/main/java/org/lance/operation/Merge.java @@ -27,16 +27,26 @@ */ public class Merge extends SchemaOperation { private final List fragments; + // True when this merge makes no nullability-affecting schema change: it + // introduces no field that data staged against an earlier schema could not + // safely omit. Without the assertion the merge conservatively conflicts with + // concurrent appends, whose fragments would omit new columns and read as null. + private final boolean preservesNullability; - protected Merge(List fragments, Schema schema) { + protected Merge(List fragments, Schema schema, boolean preservesNullability) { super(schema); this.fragments = fragments; + this.preservesNullability = preservesNullability; } public List fragments() { return fragments; } + public boolean preservesNullability() { + return preservesNullability; + } + @Override public String name() { return "Merge"; @@ -47,6 +57,7 @@ public String toString() { return MoreObjects.toStringHelper(this) .add("fragments", fragments) .add("schema", schema()) + .add("preservesNullability", preservesNullability) .toString(); } @@ -56,12 +67,13 @@ public boolean equals(Object o) { if (o == null || getClass() != o.getClass()) return false; if (!super.equals(o)) return false; Merge that = (Merge) o; - return Objects.equals(fragments, that.fragments); + return Objects.equals(fragments, that.fragments) + && preservesNullability == that.preservesNullability; } @Override public int hashCode() { - return Objects.hash(super.hashCode(), fragments); + return Objects.hash(super.hashCode(), fragments, preservesNullability); } public static Builder builder() { @@ -71,6 +83,8 @@ public static Builder builder() { public static class Builder { private List fragments; private Schema schema; + // No assertion by default, which conservatively conflicts. + private boolean preservesNullability = false; private Builder() {} @@ -84,8 +98,13 @@ public Builder schema(Schema schema) { return this; } + public Builder preservesNullability(boolean preservesNullability) { + this.preservesNullability = preservesNullability; + return this; + } + public Merge build() { - return new Merge(fragments, schema); + return new Merge(fragments, schema, preservesNullability); } } } diff --git a/java/src/main/java/org/lance/operation/Project.java b/java/src/main/java/org/lance/operation/Project.java index a4c718c4c4b..8c79431a7d1 100644 --- a/java/src/main/java/org/lance/operation/Project.java +++ b/java/src/main/java/org/lance/operation/Project.java @@ -16,14 +16,26 @@ import com.google.common.base.MoreObjects; import org.apache.arrow.vector.types.pojo.Schema; +import java.util.Objects; + /** * Project to a new schema. This Operation only changes the schema, not the data. Note: 1. For * removing columns. The data will be removed after compaction. 2. Project will modify column * positions, not ids(a.k.a. field id) */ public class Project extends SchemaOperation { - private Project(Schema schema) { + // True when this projection makes no nullability-affecting schema change, + // as a rename or a drop does not. Without the assertion the projection + // conservatively conflicts with concurrent value writes. + private final boolean preservesNullability; + + private Project(Schema schema, boolean preservesNullability) { super(schema); + this.preservesNullability = preservesNullability; + } + + public boolean preservesNullability() { + return preservesNullability; } @Override @@ -33,7 +45,23 @@ public String name() { @Override public String toString() { - return MoreObjects.toStringHelper(this).add("schema", schema()).toString(); + return MoreObjects.toStringHelper(this) + .add("schema", schema()) + .add("preservesNullability", preservesNullability) + .toString(); + } + + @Override + public boolean equals(Object o) { + if (!super.equals(o)) { + return false; + } + return preservesNullability == ((Project) o).preservesNullability; + } + + @Override + public int hashCode() { + return Objects.hash(schema(), preservesNullability); } public static Builder builder() { @@ -42,6 +70,7 @@ public static Builder builder() { public static class Builder { private Schema schema; + private boolean preservesNullability; public Builder() {} @@ -50,8 +79,13 @@ public Builder schema(Schema schema) { return this; } + public Builder preservesNullability(boolean preservesNullability) { + this.preservesNullability = preservesNullability; + return this; + } + public Project build() { - return new Project(schema); + return new Project(schema, preservesNullability); } } } diff --git a/java/src/main/java/org/lance/operation/Update.java b/java/src/main/java/org/lance/operation/Update.java index f886942b4b9..721bbd84b47 100644 --- a/java/src/main/java/org/lance/operation/Update.java +++ b/java/src/main/java/org/lance/operation/Update.java @@ -20,6 +20,7 @@ import java.util.Arrays; import java.util.Collections; import java.util.List; +import java.util.Map; import java.util.Objects; import java.util.Optional; @@ -31,19 +32,30 @@ public class Update implements Operation { private final long[] fieldsForPreservingFragBitmap; private final Optional updateMode; + /** + * Per-fragment matched row offsets serialized as portable RoaringBitmap bytes (little-endian, + * spec-compliant). Keys are fragment ids; values are the serialized bitmap for the local physical + * row offsets (0-based) within the fragment whose columns were rewritten. Empty map means the + * caller did not supply offsets and the partial last_updated refresh in build_manifest will not + * activate. + */ + private final Map updatedFragmentOffsets; + private Update( List removedFragmentIds, List updatedFragments, List newFragments, long[] fieldsModified, long[] fieldsForPreservingFragBitmap, - Optional updateMode) { + Optional updateMode, + Map updatedFragmentOffsets) { this.removedFragmentIds = removedFragmentIds; this.updatedFragments = updatedFragments; this.newFragments = newFragments; this.fieldsModified = fieldsModified; this.fieldsForPreservingFragBitmap = fieldsForPreservingFragBitmap; this.updateMode = updateMode; + this.updatedFragmentOffsets = updatedFragmentOffsets; } public static Builder builder() { @@ -74,6 +86,10 @@ public Optional updateMode() { return updateMode; } + public Map updatedFragmentOffsets() { + return updatedFragmentOffsets; + } + @Override public String name() { return "Update"; @@ -87,6 +103,7 @@ public String toString() { .add("fieldsModified", fieldsModified) .add("fieldsForPreservingFragBitmap", fieldsForPreservingFragBitmap) .add("updateMode", updateMode) + .add("updatedFragmentOffsets", updatedFragmentOffsets) .toString(); } @@ -100,7 +117,32 @@ public boolean equals(Object o) { && Objects.equals(newFragments, that.newFragments) && Arrays.equals(fieldsModified, that.fieldsModified) && Arrays.equals(fieldsForPreservingFragBitmap, that.fieldsForPreservingFragBitmap) - && Objects.equals(updateMode, that.updateMode); + && Objects.equals(updateMode, that.updateMode) + && offsetMapsEqual(updatedFragmentOffsets, that.updatedFragmentOffsets); + } + + /** Deep-equality for {@code Map}: keys by value, arrays by content. */ + private static boolean offsetMapsEqual(Map a, Map b) { + if (a == b) return true; + if (a.size() != b.size()) return false; + for (Map.Entry entry : a.entrySet()) { + if (!Arrays.equals(entry.getValue(), b.get(entry.getKey()))) return false; + } + return true; + } + + @Override + public int hashCode() { + int h = Objects.hash(removedFragmentIds, updatedFragments, newFragments, updateMode); + h = 31 * h + Arrays.hashCode(fieldsModified); + h = 31 * h + Arrays.hashCode(fieldsForPreservingFragBitmap); + // Sum entry hashes (XOR key ^ array-content hash) so result is insertion-order-independent. + int mapHash = 0; + for (Map.Entry entry : updatedFragmentOffsets.entrySet()) { + mapHash += Long.hashCode(entry.getKey()) ^ Arrays.hashCode(entry.getValue()); + } + h = 31 * h + mapHash; + return h; } public enum UpdateMode { @@ -115,6 +157,7 @@ public static class Builder { private long[] fieldsModified = new long[0]; private long[] fieldsForPreservingFragBitmap = new long[0]; private Optional updateMode = Optional.empty(); + private Map updatedFragmentOffsets = Collections.emptyMap(); private Builder() {} @@ -148,6 +191,20 @@ public Builder updateMode(Optional updateMode) { return this; } + /** + * Set the per-fragment matched row offsets for a RewriteColumns commit. + * + *

    Keys are fragment ids; values are portable RoaringBitmap bytes (little-endian, + * spec-compliant serialization) encoding the local physical row offsets (0-based) within the + * fragment that matched the update_columns hash join. When non-empty and update mode is + * RewriteColumns with stable row IDs enabled, build_manifest will call the partial last_updated + * refresh for those offsets only. + */ + public Builder updatedFragmentOffsets(Map updatedFragmentOffsets) { + this.updatedFragmentOffsets = updatedFragmentOffsets; + return this; + } + public Update build() { return new Update( removedFragmentIds, @@ -155,7 +212,8 @@ public Update build() { newFragments, fieldsModified, fieldsForPreservingFragBitmap, - updateMode); + updateMode, + updatedFragmentOffsets); } } } diff --git a/java/src/main/java/org/lance/otel/LanceMetrics.java b/java/src/main/java/org/lance/otel/LanceMetrics.java new file mode 100644 index 00000000000..07b8595ff62 --- /dev/null +++ b/java/src/main/java/org/lance/otel/LanceMetrics.java @@ -0,0 +1,283 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.lance.otel; + +import org.lance.JniLoader; + +import io.opentelemetry.api.GlobalOpenTelemetry; +import io.opentelemetry.api.common.Attributes; +import io.opentelemetry.api.common.AttributesBuilder; +import io.opentelemetry.api.metrics.BatchCallback; +import io.opentelemetry.api.metrics.Meter; +import io.opentelemetry.api.metrics.MeterProvider; +import io.opentelemetry.api.metrics.ObservableDoubleMeasurement; +import io.opentelemetry.api.metrics.ObservableMeasurement; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.logging.Logger; + +/** Bridges Lance's internal Rust metrics into OpenTelemetry observable instruments. */ +public final class LanceMetrics { + private static final Logger LOGGER = Logger.getLogger(LanceMetrics.class.getName()); + private static final String METER_NAME = "lance"; + private static final List INSTRUMENTS = new ArrayList<>(); + private static boolean instrumented; + private static MeterProvider instrumentedProvider; + + private LanceMetrics() {} + + /** + * Register Lance metrics on the global OpenTelemetry {@link MeterProvider}. + * + * @return true if the recorder is installed and instruments are registered, false if a different + * Rust metrics recorder is already installed in this process + */ + public static synchronized boolean instrument() { + return instrument(GlobalOpenTelemetry.get().getMeterProvider()); + } + + /** + * Register Lance metrics as OpenTelemetry observable instruments. + * + *

    This installs a process-global Rust metrics recorder. If a different Rust metrics recorder + * is already installed, this returns false and does not create instruments. Repeated successful + * calls with the same provider are safe and return true without creating duplicate instruments. + * Calls with a different provider close the existing instruments and register new callbacks on + * the supplied provider. + * + * @param meterProvider the OpenTelemetry meter provider that owns the instruments + * @return true if the recorder is installed and instruments are registered, false otherwise + */ + public static synchronized boolean instrument(MeterProvider meterProvider) { + Objects.requireNonNull(meterProvider, "meterProvider"); + JniLoader.ensureLoaded(); + + if (!registerLanceMetricsRecorderNative()) { + LOGGER.warning( + "Could not install the Lance metrics recorder: another Rust metrics recorder is already" + + " installed in this process. Lance metrics will not be exported via" + + " OpenTelemetry."); + return false; + } + + if (instrumented) { + if (instrumentedProvider == meterProvider) { + return true; + } + closeInstruments(); + } + + Meter meter = meterProvider.meterBuilder(METER_NAME).build(); + Map registeredMetrics = new HashMap<>(); + List measurements = new ArrayList<>(); + + for (MetricDescription desc : supportedMetrics(catalog())) { + String unit = desc.getUnit() == null ? "" : desc.getUnit(); + String description = desc.getDescription(); + switch (desc.getKind()) { + case "counter": + ObservableDoubleMeasurement counter = + meter + .counterBuilder(desc.getName()) + .ofDoubles() + .setUnit(unit) + .setDescription(description) + .buildObserver(); + registeredMetrics.put(desc.getName(), RegisteredMetric.scalar(counter)); + measurements.add(counter); + break; + case "gauge": + ObservableDoubleMeasurement gauge = + meter + .gaugeBuilder(desc.getName()) + .setUnit(unit) + .setDescription(description) + .buildObserver(); + registeredMetrics.put(desc.getName(), RegisteredMetric.scalar(gauge)); + measurements.add(gauge); + break; + case "histogram": + ObservableDoubleMeasurement buckets = + meter + .counterBuilder(desc.getName() + "_bucket") + .ofDoubles() + .setDescription(description + " (cumulative buckets)") + .buildObserver(); + ObservableDoubleMeasurement count = + meter + .counterBuilder(desc.getName() + "_count") + .ofDoubles() + .setDescription(description + " (count)") + .buildObserver(); + ObservableDoubleMeasurement sum = + meter + .counterBuilder(desc.getName() + "_sum") + .ofDoubles() + .setUnit(unit) + .setDescription(description + " (sum)") + .buildObserver(); + registeredMetrics.put(desc.getName(), RegisteredMetric.histogram(buckets, count, sum)); + measurements.add(buckets); + measurements.add(count); + measurements.add(sum); + break; + default: + LOGGER.warning( + "Skipping Lance metric " + desc.getName() + " with unknown kind: " + desc.getKind()); + continue; + } + } + + if (!measurements.isEmpty()) { + ObservableMeasurement first = measurements.get(0); + ObservableMeasurement[] rest = + measurements.subList(1, measurements.size()).toArray(new ObservableMeasurement[0]); + BatchCallback callback = + meter.batchCallback(() -> recordSnapshot(registeredMetrics), first, rest); + INSTRUMENTS.add(callback); + } + + instrumented = true; + instrumentedProvider = meterProvider; + return true; + } + + /** Close registered OpenTelemetry callbacks. Rust metric state remains process-global. */ + public static synchronized void close() { + closeInstruments(); + } + + /** Return the catalog of Lance metrics described by the native recorder. */ + public static List catalog() { + JniLoader.ensureLoaded(); + return Collections.unmodifiableList(lanceMetricsCatalogNative()); + } + + /** Return a point-in-time snapshot of all recorded Lance metric series. */ + public static List snapshot() { + JniLoader.ensureLoaded(); + return Collections.unmodifiableList(snapshotLanceMetricsNative()); + } + + static List supportedMetrics(List catalog) { + List supported = new ArrayList<>(); + for (MetricDescription desc : catalog) { + switch (desc.getKind()) { + case "counter": + case "gauge": + case "histogram": + supported.add(desc); + break; + default: + LOGGER.warning( + "Skipping Lance metric " + desc.getName() + " with unknown kind: " + desc.getKind()); + } + } + return supported; + } + + private static void recordSnapshot(Map registeredMetrics) { + for (MetricPoint point : snapshot()) { + RegisteredMetric metric = registeredMetrics.get(point.getName()); + if (metric == null) { + continue; + } + metric.record(point); + } + } + + private static void closeInstruments() { + for (AutoCloseable instrument : INSTRUMENTS) { + try { + instrument.close(); + } catch (Exception e) { + LOGGER.warning("Failed to close Lance OpenTelemetry instrument: " + e.getMessage()); + } + } + INSTRUMENTS.clear(); + instrumented = false; + instrumentedProvider = null; + } + + private static AttributesBuilder attributesBuilder(Map values) { + AttributesBuilder builder = Attributes.builder(); + for (Map.Entry entry : values.entrySet()) { + builder.put(entry.getKey(), entry.getValue()); + } + return builder; + } + + private static Attributes attributes(Map values) { + return attributesBuilder(values).build(); + } + + private static final class RegisteredMetric { + private final ObservableDoubleMeasurement scalar; + private final ObservableDoubleMeasurement buckets; + private final ObservableDoubleMeasurement count; + private final ObservableDoubleMeasurement sum; + + private RegisteredMetric( + ObservableDoubleMeasurement scalar, + ObservableDoubleMeasurement buckets, + ObservableDoubleMeasurement count, + ObservableDoubleMeasurement sum) { + this.scalar = scalar; + this.buckets = buckets; + this.count = count; + this.sum = sum; + } + + private static RegisteredMetric scalar(ObservableDoubleMeasurement measurement) { + return new RegisteredMetric(measurement, null, null, null); + } + + private static RegisteredMetric histogram( + ObservableDoubleMeasurement buckets, + ObservableDoubleMeasurement count, + ObservableDoubleMeasurement sum) { + return new RegisteredMetric(null, buckets, count, sum); + } + + private void record(MetricPoint point) { + if (scalar != null && point.getValue() != null) { + scalar.record(point.getValue(), attributes(point.getAttributes())); + } + if (buckets != null && point.getBuckets() != null) { + for (MetricBucket bucket : point.getBuckets()) { + AttributesBuilder attributes = attributesBuilder(point.getAttributes()); + attributes.put("le", bucket.getLe()); + buckets.record(bucket.getCumulativeCount(), attributes.build()); + } + } + if (count != null && point.getCount() != null) { + count.record(point.getCount().doubleValue(), attributes(point.getAttributes())); + } + if (sum != null && point.getSum() != null) { + sum.record(point.getSum(), attributes(point.getAttributes())); + } + } + } + + private static native boolean registerLanceMetricsRecorderNative(); + + private static native List lanceMetricsCatalogNative(); + + private static native List snapshotLanceMetricsNative(); +} diff --git a/java/src/main/java/org/lance/otel/MetricBucket.java b/java/src/main/java/org/lance/otel/MetricBucket.java new file mode 100644 index 00000000000..4efa1058a30 --- /dev/null +++ b/java/src/main/java/org/lance/otel/MetricBucket.java @@ -0,0 +1,33 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.lance.otel; + +/** A cumulative histogram bucket exported with Prometheus-style {@code le} semantics. */ +public final class MetricBucket { + private final String le; + private final long cumulativeCount; + + public MetricBucket(String le, long cumulativeCount) { + this.le = le; + this.cumulativeCount = cumulativeCount; + } + + public String getLe() { + return le; + } + + public long getCumulativeCount() { + return cumulativeCount; + } +} diff --git a/java/src/main/java/org/lance/otel/MetricDescription.java b/java/src/main/java/org/lance/otel/MetricDescription.java new file mode 100644 index 00000000000..a350e2be963 --- /dev/null +++ b/java/src/main/java/org/lance/otel/MetricDescription.java @@ -0,0 +1,45 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.lance.otel; + +/** Metadata for one Lance metric available through the OpenTelemetry bridge. */ +public final class MetricDescription { + private final String name; + private final String kind; + private final String unit; + private final String description; + + public MetricDescription(String name, String kind, String unit, String description) { + this.name = name; + this.kind = kind; + this.unit = unit; + this.description = description; + } + + public String getName() { + return name; + } + + public String getKind() { + return kind; + } + + public String getUnit() { + return unit; + } + + public String getDescription() { + return description; + } +} diff --git a/java/src/main/java/org/lance/otel/MetricPoint.java b/java/src/main/java/org/lance/otel/MetricPoint.java new file mode 100644 index 00000000000..f9a44b95f20 --- /dev/null +++ b/java/src/main/java/org/lance/otel/MetricPoint.java @@ -0,0 +1,75 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.lance.otel; + +import java.util.Collections; +import java.util.List; +import java.util.Map; + +/** A cumulative Lance metric data point captured by the native metrics recorder. */ +public final class MetricPoint { + private final String name; + private final String kind; + private final Map attributes; + private final Double value; + private final List buckets; + private final Long count; + private final Double sum; + + public MetricPoint( + String name, + String kind, + Map attributes, + Double value, + List buckets, + Long count, + Double sum) { + this.name = name; + this.kind = kind; + this.attributes = + attributes == null ? Collections.emptyMap() : Collections.unmodifiableMap(attributes); + this.value = value; + this.buckets = buckets == null ? null : Collections.unmodifiableList(buckets); + this.count = count; + this.sum = sum; + } + + public String getName() { + return name; + } + + public String getKind() { + return kind; + } + + public Map getAttributes() { + return attributes; + } + + public Double getValue() { + return value; + } + + public List getBuckets() { + return buckets; + } + + public Long getCount() { + return count; + } + + public Double getSum() { + return sum; + } +} diff --git a/java/src/test/java/org/lance/AsyncScannerTest.java b/java/src/test/java/org/lance/AsyncScannerTest.java index fc786ff57c2..5b75a22cee2 100644 --- a/java/src/test/java/org/lance/AsyncScannerTest.java +++ b/java/src/test/java/org/lance/AsyncScannerTest.java @@ -18,6 +18,7 @@ import org.lance.index.IndexType; import org.lance.index.scalar.ScalarIndexParams; import org.lance.ipc.AsyncScanner; +import org.lance.ipc.MaterializationStyle; import org.lance.ipc.ScanOptions; import org.apache.arrow.memory.BufferAllocator; @@ -32,6 +33,9 @@ import java.io.BufferedReader; import java.io.InputStreamReader; +import java.lang.reflect.Constructor; +import java.lang.reflect.Field; +import java.lang.reflect.Method; import java.net.URL; import java.net.URLClassLoader; import java.nio.file.Path; @@ -40,6 +44,8 @@ import java.util.List; import java.util.Optional; import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CompletionException; +import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.TimeUnit; @@ -47,6 +53,7 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; /** @@ -68,6 +75,29 @@ static void tearDown() { } } + @Test + @SuppressWarnings("unchecked") + void testNativeScanFailureUsesLanceException() throws Exception { + Constructor constructor = AsyncScanner.class.getDeclaredConstructor(); + constructor.setAccessible(true); + AsyncScanner scanner = constructor.newInstance(); + + Field pendingTasksField = AsyncScanner.class.getDeclaredField("pendingTasks"); + pendingTasksField.setAccessible(true); + ConcurrentHashMap> pendingTasks = + (ConcurrentHashMap>) pendingTasksField.get(scanner); + CompletableFuture pendingTask = new CompletableFuture<>(); + pendingTasks.put(1L, pendingTask); + + Method failTask = AsyncScanner.class.getDeclaredMethod("failTask", long.class, String.class); + failTask.setAccessible(true); + failTask.invoke(scanner, 1L, "scan I/O failed"); + + CompletionException failure = assertThrows(CompletionException.class, pendingTask::join); + assertEquals(LanceException.class, failure.getCause().getClass()); + assertEquals("scan I/O failed", failure.getCause().getMessage()); + } + /** * Example 1: Basic async scan with CompletableFuture. * @@ -142,6 +172,35 @@ void testAsyncScanWithFilter(@TempDir Path tempDir) throws Exception { } } + @Test + void testAsyncScanWithPerformanceOptions(@TempDir Path tempDir) throws Exception { + String datasetPath = tempDir.resolve("async_scanner_performance_options").toString(); + try (BufferAllocator allocator = new RootAllocator()) { + TestUtils.SimpleTestDataset testDataset = + new TestUtils.SimpleTestDataset(allocator, datasetPath); + testDataset.createEmptyDataset().close(); + + try (Dataset dataset = testDataset.write(1, 40)) { + ScanOptions options = + new ScanOptions.Builder() + .filter("id < 20") + .batchSize(10) + .batchSizeBytes(1024) + .ioBufferSize(1024 * 1024) + .batchReadahead(2) + .fragmentReadahead(2) + .scanInOrder(false) + .lateMaterialization(MaterializationStyle.allEarly()) + .build(); + + try (AsyncScanner scanner = AsyncScanner.create(dataset, options, allocator); + ArrowReader reader = scanner.scanBatchesAsync().get(10, TimeUnit.SECONDS)) { + assertEquals(20, countRows(reader)); + } + } + } + } + @Test void testFastSearchSkipsUnindexedFragments(@TempDir Path tempDir) throws Exception { String datasetPath = tempDir.resolve("async_scanner_fast_search_scalar_index").toString(); diff --git a/java/src/test/java/org/lance/CommitBuilderStorageFormatTest.java b/java/src/test/java/org/lance/CommitBuilderStorageFormatTest.java new file mode 100644 index 00000000000..b500369cbb9 --- /dev/null +++ b/java/src/test/java/org/lance/CommitBuilderStorageFormatTest.java @@ -0,0 +1,151 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.lance; + +import org.lance.operation.Append; +import org.lance.operation.Delete; +import org.lance.operation.OperationTestBase; + +import org.apache.arrow.memory.RootAllocator; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import java.nio.file.Path; +import java.util.Collections; +import java.util.List; +import java.util.stream.Collectors; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +public class CommitBuilderStorageFormatTest extends OperationTestBase { + + /** + * Append to a freshly created (2.1) dataset with the given storage format and return the + * committed dataset's format version. + */ + private String commitWithStorageFormat(String datasetPath, String storageFormat) + throws Exception { + try (RootAllocator allocator = new RootAllocator(Long.MAX_VALUE)) { + TestUtils.SimpleTestDataset testDataset = + new TestUtils.SimpleTestDataset(allocator, datasetPath); + dataset = testDataset.createEmptyDataset(); + FragmentMetadata fragment = testDataset.createNewFragment(10); + try (Transaction txn = + new Transaction.Builder() + .readVersion(dataset.version()) + .operation(Append.builder().fragments(Collections.singletonList(fragment)).build()) + .build()) { + try (Dataset committed = + new CommitBuilder(dataset).storageFormat(storageFormat).execute(txn)) { + assertEquals(2, committed.version()); + return committed.getLanceFileFormatVersion(); + } + } + } + } + + /** + * The numeric versions are what {@link Dataset#getLanceFileFormatVersion()} returns and what + * {@link WriteParams.Builder#withDataStorageVersion(String)} accepts, so they must work here too + * — a caller that encodes fragments as "2.1" has to be able to commit them as "2.1". + */ + @Test + void testCanonicalVersionAccepted(@TempDir Path tempDir) throws Exception { + assertEquals( + LanceConstants.FILE_FORMAT_VERSION_2_1, + commitWithStorageFormat( + tempDir.resolve("canonical").toString(), LanceConstants.FILE_FORMAT_VERSION_2_1)); + } + + /** The "v"-prefixed spelling shipped in this method's Javadoc and stays accepted. */ + @Test + void testDeprecatedAliasAccepted(@TempDir Path tempDir) throws Exception { + assertEquals( + LanceConstants.FILE_FORMAT_VERSION_2_1, + commitWithStorageFormat(tempDir.resolve("alias").toString(), "v2_1")); + } + + /** + * A delete adds no data files, so nothing about it depends on the storage format — but {@link + * CommitBuilder#storageFormat(String)} is still validated against the existing dataset for any + * operation other than overwrite. A caller that forwards a configured format on every commit hits + * this on row-level operations against a table written in a different version, so the failure is + * a mismatch error rather than anything to do with parsing. + */ + @Test + void testMismatchedFormatRejectedOnRowLevelOperation(@TempDir Path tempDir) throws Exception { + String datasetPath = tempDir.resolve("mismatch").toString(); + try (RootAllocator allocator = new RootAllocator(Long.MAX_VALUE)) { + TestUtils.SimpleTestDataset testDataset = + new TestUtils.SimpleTestDataset(allocator, datasetPath); + dataset = testDataset.createEmptyDataset(); + FragmentMetadata fragment = testDataset.createNewFragment(10); + try (Transaction appendTxn = + new Transaction.Builder() + .readVersion(dataset.version()) + .operation(Append.builder().fragments(Collections.singletonList(fragment)).build()) + .build()) { + dataset = new CommitBuilder(dataset).execute(appendTxn); + } + assertEquals(LanceConstants.FILE_FORMAT_VERSION_2_1, dataset.getLanceFileFormatVersion()); + + List fragmentIds = + dataset.getFragments().stream() + .map(f -> Long.valueOf(f.getId())) + .collect(Collectors.toList()); + + // "2.2" parses fine, so a failure here is the mismatch guard and not the parser. + try (Transaction deleteTxn = deleteAll(fragmentIds)) { + IllegalArgumentException error = + assertThrows( + IllegalArgumentException.class, + () -> + new CommitBuilder(dataset) + .storageFormat(LanceConstants.FILE_FORMAT_VERSION_2_2) + .execute(deleteTxn)); + assertTrue(error.getMessage().contains("Storage format mismatch"), error.getMessage()); + } + + // The same delete succeeds when the format agrees with the dataset. + try (Transaction deleteTxn = deleteAll(fragmentIds)) { + try (Dataset deleted = + new CommitBuilder(dataset) + .storageFormat(LanceConstants.FILE_FORMAT_VERSION_2_1) + .execute(deleteTxn)) { + assertEquals(0, deleted.countRows()); + } + } + } + } + + private Transaction deleteAll(List fragmentIds) { + return new Transaction.Builder() + .readVersion(dataset.version()) + .operation(Delete.builder().deletedFragmentIds(fragmentIds).predicate("1=1").build()) + .build(); + } + + @Test + void testUnknownFormatRejected(@TempDir Path tempDir) { + assertThrows( + IllegalArgumentException.class, + () -> commitWithStorageFormat(tempDir.resolve("bogus").toString(), "bogus")); + // The alias set is frozen at what shipped, so it does not extend to newer versions. + assertThrows( + IllegalArgumentException.class, + () -> commitWithStorageFormat(tempDir.resolve("v23").toString(), "v2_3")); + } +} diff --git a/java/src/test/java/org/lance/CompactionTest.java b/java/src/test/java/org/lance/CompactionTest.java index 9ac96804a61..f357c37754f 100644 --- a/java/src/test/java/org/lance/CompactionTest.java +++ b/java/src/test/java/org/lance/CompactionTest.java @@ -15,6 +15,7 @@ import org.lance.compaction.Compaction; import org.lance.compaction.CompactionMetrics; +import org.lance.compaction.CompactionMode; import org.lance.compaction.CompactionOptions; import org.lance.compaction.CompactionPlan; import org.lance.compaction.CompactionTask; @@ -23,6 +24,8 @@ import org.apache.arrow.memory.RootAllocator; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.EnumSource; import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; @@ -31,6 +34,7 @@ import java.io.ObjectOutputStream; import java.nio.file.Path; import java.util.Collections; +import java.util.Optional; import static org.junit.jupiter.api.Assertions.assertEquals; @@ -130,6 +134,40 @@ public void testDeletionCompaction(@TempDir Path tempDir) throws Exception { } } + @ParameterizedTest + @EnumSource(CompactionMode.class) + public void testCompactionModeRoundTrip(CompactionMode mode, @TempDir Path tempDir) + throws Exception { + String datasetPath = tempDir.resolve("test_dataset_for_compaction").toString(); + try (RootAllocator allocator = new RootAllocator(Long.MAX_VALUE)) { + TestUtils.SimpleTestDataset testDataset = + new TestUtils.SimpleTestDataset(allocator, datasetPath); + testDataset.createEmptyDataset().close(); + + testDataset.write(1, 10).close(); + try (Dataset dataset = testDataset.write(2, 10)) { + CompactionOptions compactionOptions = + CompactionOptions.builder() + .withTargetRowsPerFragment(100) + .withNumThreads(1) + .withCompactionMode(mode) + .build(); + CompactionPlan compactionPlan = Compaction.planCompaction(dataset, compactionOptions); + + // The plan's options are rebuilt by the native layer; the mode must come + // back as a CompactionMode enum, not a raw String. + assertEquals( + Optional.of(mode.getValue()), + compactionPlan.getCompactionOptions().getCompactionMode()); + + CompactionTask task = serializeAndDeserialize(compactionPlan.getCompactionTasks().get(0)); + RewriteResult result = task.execute(dataset); + assertEquals(2, result.getMetrics().getFragmentsRemoved()); + assertEquals(1, result.getMetrics().getFragmentsAdded()); + } + } + } + private static T serializeAndDeserialize(T object) throws IOException, ClassNotFoundException { ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); diff --git a/java/src/test/java/org/lance/DatasetTest.java b/java/src/test/java/org/lance/DatasetTest.java index 45466a0367c..80681f38bed 100644 --- a/java/src/test/java/org/lance/DatasetTest.java +++ b/java/src/test/java/org/lance/DatasetTest.java @@ -80,6 +80,7 @@ import static org.junit.jupiter.api.Assertions.assertInstanceOf; import static org.junit.jupiter.api.Assertions.assertNotEquals; import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.junit.jupiter.api.Assertions.fail; @@ -240,6 +241,9 @@ void testDatasetVersion(@TempDir Path tempDir) { List versions = dataset.listVersions(); assertEquals(3, versions.size()); + assertEquals(3, dataset.getVersionCount()); + assertEquals(3, dataset2.getVersionCount()); + assertEquals(3, dataset3.getVersionCount()); assertEquals(1, versions.get(0).getId()); assertEquals(2, versions.get(1).getId()); assertEquals(3, versions.get(2).getId()); @@ -659,6 +663,40 @@ void testAlterColumns(@TempDir Path tempDir) { } } + @Test + void testAlterColumnsCastType(@TempDir Path tempDir) { + String testMethodName = new Object() {}.getClass().getEnclosingMethod().getName(); + String datasetPath = tempDir.resolve(testMethodName).toString(); + try (RootAllocator allocator = new RootAllocator(Long.MAX_VALUE)) { + TestUtils.SimpleTestDataset testDataset = + new TestUtils.SimpleTestDataset(allocator, datasetPath); + dataset = testDataset.createEmptyDataset(); + + // Widen "id" from Int32 to Int64. The cast target type is a parameterized ArrowType, which + // must survive the trip to the native side; regression test for a dropped cast that left the + // stored type unchanged. + ColumnAlteration widenId = + new ColumnAlteration.Builder("id").castTo(new ArrowType.Int(64, true)).build(); + dataset.alterColumns(Collections.singletonList(widenId)); + + assertEquals(new ArrowType.Int(64, true), dataset.getSchema().findField("id").getType()); + + // A cast combined with rename must apply both. + ColumnAlteration renameAndWiden = + new ColumnAlteration.Builder("id") + .rename("id_long") + .castTo(new ArrowType.Int(64, true)) + .build(); + dataset.alterColumns(Collections.singletonList(renameAndWiden)); + + List fieldNames = + dataset.getSchema().getFields().stream().map(Field::getName).collect(Collectors.toList()); + assertFalse(fieldNames.contains("id")); + assertTrue(fieldNames.contains("id_long")); + assertEquals(new ArrowType.Int(64, true), dataset.getSchema().findField("id_long").getType()); + } + } + @Test void testAddColumnBySqlExpressions(@TempDir Path tempDir) { String testMethodName = new Object() {}.getClass().getEnclosingMethod().getName(); @@ -2026,6 +2064,16 @@ void testReadZeroLengthBlob(@TempDir Path tempDir) throws Exception { } } + @Test + void testTakeNullBlobPreservesSelection(@TempDir Path tempDir) throws Exception { + String base = tempDir.resolve("testTakeNullBlobPreservesSelection").toString(); + try (Dataset ds = TestUtils.createBlobDataset(base, 128, 8)) { + List blobs = ds.takeBlobsByIndices(Collections.singletonList(15L), "blobs"); + assertEquals(1, blobs.size()); + assertNull(blobs.get(0)); + } + } + @Test void testReadLargeBlobAndRanges(@TempDir Path tempDir) throws Exception { String base = tempDir.resolve("testReadLargeBlobAndRanges").toString(); @@ -2142,6 +2190,13 @@ public void testDescribeIndicesByName(@TempDir Path tempDir) throws Exception { assertEquals(1, desc.getSegments().size(), "Expected exactly one physical segment"); assertEquals("index1", desc.getSegments().get(0).name()); + assertTrue( + desc.getSegments().get(0).getSizeBytes().orElse(0L) > 0, + "segment size should be positive"); + assertEquals( + desc.getSegments().get(0).getSizeBytes(), + desc.getTotalSizeBytes(), + "single-segment size should equal the logical index size"); descriptions = dataset.describeIndices(); assertEquals(2, descriptions.size(), "Expected exactly one matching index"); @@ -2154,6 +2209,8 @@ public void testDescribeIndicesByName(@TempDir Path tempDir) throws Exception { indexDesc.getSegments(), "segments alias should match metadata"); assertNotNull(indexDesc.getDetailsJson(), "Details JSON should not be null"); + assertTrue( + indexDesc.getTotalSizeBytes().orElse(0L) > 0, "total index size should be positive"); } } } diff --git a/java/src/test/java/org/lance/FileReaderWriterTest.java b/java/src/test/java/org/lance/FileReaderWriterTest.java index a849a87c576..85c7430f087 100644 --- a/java/src/test/java/org/lance/FileReaderWriterTest.java +++ b/java/src/test/java/org/lance/FileReaderWriterTest.java @@ -256,7 +256,9 @@ void testInvalidPath() { LanceFileReader.open("/tmp/does_not_exist.lance", allocator); fail("Expected LanceException to be thrown"); } catch (IOException e) { - assertTrue(e.getMessage().contains("Object at location /tmp/does_not_exist.lance not found")); + String message = e.getMessage(); + assertTrue(message.contains("/tmp/does_not_exist.lance")); + assertTrue(message.toLowerCase().contains("not found")); } try { LanceFileReader.open("", allocator); diff --git a/java/src/test/java/org/lance/FragmentTest.java b/java/src/test/java/org/lance/FragmentTest.java index 29a21b5258a..3f30fa49863 100644 --- a/java/src/test/java/org/lance/FragmentTest.java +++ b/java/src/test/java/org/lance/FragmentTest.java @@ -150,6 +150,64 @@ void testWriteFragmentWithSchemaOverride(@TempDir Path tempDir) throws Exception } } + @Test + void testWriteFragmentWithSession(@TempDir Path tempDir) { + String datasetPath = tempDir.resolve("fragment_with_session").toString(); + try (RootAllocator allocator = new RootAllocator(Long.MAX_VALUE); + Session session = Session.builder().build()) { + TestUtils.SimpleTestDataset testDataset = + new TestUtils.SimpleTestDataset(allocator, datasetPath); + testDataset.createEmptyDataset().close(); + + long sizeBefore = session.sizeBytes(); + try (VectorSchemaRoot root = VectorSchemaRoot.create(testDataset.getSchema(), allocator)) { + root.allocateNew(); + VarCharVector nameVector = (VarCharVector) root.getVector("name"); + IntVector idVector = (IntVector) root.getVector("id"); + nameVector.setSafe(0, "Person 1".getBytes(StandardCharsets.UTF_8)); + idVector.setSafe(0, 1); + root.setRowCount(1); + + // First append in APPEND mode without an explicit schema: the + // manifest load for schema inference populates the shared session's + // metadata cache. + List firstFragments = + appendWithSession(datasetPath, allocator, root, session); + assertEquals(1, firstFragments.size()); + assertEquals(1, firstFragments.get(0).getPhysicalRows()); + assertTrue(session.sizeBytes() > sizeBefore); + long hitsAfterFirst = session.metadataCacheStats().getHits(); + + // Second append through the same session: schema inference reads the + // manifest cached by the first write, so cache hits must increase. + List secondFragments = + appendWithSession(datasetPath, allocator, root, session); + assertEquals(1, secondFragments.size()); + assertEquals(1, secondFragments.get(0).getPhysicalRows()); + assertTrue(session.metadataCacheStats().getHits() > hitsAfterFirst); + + // A closed session has a zero native handle and degrades to "no + // session", matching Dataset's behavior for closed sessions. + Session closedSession = Session.builder().build(); + closedSession.close(); + List fragmentsWithClosedSession = + appendWithSession(datasetPath, allocator, root, closedSession); + assertEquals(1, fragmentsWithClosedSession.size()); + } + } + } + + private static List appendWithSession( + String datasetPath, RootAllocator allocator, VectorSchemaRoot root, Session session) { + return Fragment.write() + .datasetUri(datasetPath) + .allocator(allocator) + .data(root) + .mode(WriteParams.WriteMode.APPEND) + .session(session) + .execute(); + } + @Test void commitWithoutVersion(@TempDir Path tempDir) { String datasetPath = tempDir.resolve("commit_without_version").toString(); @@ -413,4 +471,47 @@ void testMergeColumns(@TempDir Path tempDir) throws Exception { } } } + + @Test + void testFragmentStatistics(@TempDir Path tempDir) { + String datasetPath = tempDir.resolve("fragment_statistics").toString(); + try (RootAllocator allocator = new RootAllocator(Long.MAX_VALUE)) { + TestUtils.SimpleTestDataset testDataset = + new TestUtils.SimpleTestDataset(allocator, datasetPath); + testDataset.createEmptyDataset().close(); + + // Two fragments with different row counts + FragmentMetadata frag1 = testDataset.createNewFragment(21); + FragmentMetadata frag2 = testDataset.createNewFragment(9); + FragmentOperation.Append appendOp = new FragmentOperation.Append(Arrays.asList(frag1, frag2)); + try (Dataset dataset = Dataset.commit(allocator, datasetPath, appendOp, Optional.of(1L))) { + List fragments = dataset.getFragments(); + FragmentStatistics stats = dataset.getFragmentStatistics(); + assertEquals(fragments.size(), stats.size()); + + // Parity with getFragments across all three arrays + assertArrayEquals(fragments.stream().mapToInt(Fragment::getId).toArray(), stats.getIds()); + assertArrayEquals( + fragments.stream().mapToLong(f -> f.metadata().getNumRows()).toArray(), + stats.getRowCounts()); + assertArrayEquals( + fragments.stream().mapToInt(f -> f.metadata().getFiles().size()).toArray(), + stats.getDataFileNums()); + + assertEquals(30, Arrays.stream(stats.getRowCounts()).sum()); + } + } + } + + @Test + void testFragmentStatisticsOnEmptyDataset(@TempDir Path tempDir) { + String datasetPath = tempDir.resolve("fragment_statistics_empty").toString(); + try (RootAllocator allocator = new RootAllocator(Long.MAX_VALUE)) { + TestUtils.SimpleTestDataset testDataset = + new TestUtils.SimpleTestDataset(allocator, datasetPath); + try (Dataset dataset = testDataset.createEmptyDataset()) { + assertEquals(0, dataset.getFragmentStatistics().size()); + } + } + } } diff --git a/java/src/test/java/org/lance/MergeInsertTest.java b/java/src/test/java/org/lance/MergeInsertTest.java index b738ef8852d..72b2e948282 100644 --- a/java/src/test/java/org/lance/MergeInsertTest.java +++ b/java/src/test/java/org/lance/MergeInsertTest.java @@ -25,6 +25,7 @@ import org.apache.arrow.vector.ipc.ArrowReader; import org.apache.arrow.vector.ipc.ArrowStreamReader; import org.apache.arrow.vector.ipc.ArrowStreamWriter; +import org.apache.arrow.vector.types.pojo.Field; import org.apache.arrow.vector.types.pojo.Schema; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.Assertions; @@ -257,6 +258,79 @@ private VectorSchemaRoot buildSource(Schema schema, RootAllocator allocator) { return root; } + /** + * Build a source whose join key {@code id=0} appears twice ("First 0", then "Second 0"), so the + * source-dedupe behavior is exercised. Remaining ids (3, 4) are unique matches. + */ + private VectorSchemaRoot buildDuplicateKeySource(Schema schema, RootAllocator allocator) { + List sourceIds = Arrays.asList(0, 0, 3, 4); + List sourceNames = Arrays.asList("First 0", "Second 0", "Source 3", "Source 4"); + + VectorSchemaRoot root = VectorSchemaRoot.create(schema, allocator); + root.allocateNew(); + + IntVector idVector = (IntVector) root.getVector("id"); + VarCharVector nameVector = (VarCharVector) root.getVector("name"); + + for (int i = 0; i < sourceIds.size(); i++) { + idVector.setSafe(i, sourceIds.get(i)); + nameVector.setSafe(i, sourceNames.get(i).getBytes(StandardCharsets.UTF_8)); + } + + root.setRowCount(sourceIds.size()); + + return root; + } + + /** + * Build a key-only source (only the join key column) whose key {@code id=0} appears twice. A pure + * delete over such a source routes through the delete-only plan, exercising source dedupe there. + */ + private VectorSchemaRoot buildDuplicateKeyOnlySource(Schema schema, RootAllocator allocator) { + Field idField = schema.findField("id"); + Schema keyOnlySchema = new Schema(Collections.singletonList(idField)); + + List sourceIds = Arrays.asList(0, 0); + + VectorSchemaRoot root = VectorSchemaRoot.create(keyOnlySchema, allocator); + root.allocateNew(); + + IntVector idVector = (IntVector) root.getVector("id"); + for (int i = 0; i < sourceIds.size(); i++) { + idVector.setSafe(i, sourceIds.get(i)); + } + + root.setRowCount(sourceIds.size()); + + return root; + } + + /** + * Build a full-schema source from parallel {@code (id, name)} lists. A {@code null} id is written + * as a NULL join key, which merge insert never treats as a duplicate (SQL {@code NULL != NULL}). + */ + private VectorSchemaRoot buildSourceFrom( + Schema schema, RootAllocator allocator, List ids, List names) { + VectorSchemaRoot root = VectorSchemaRoot.create(schema, allocator); + root.allocateNew(); + + IntVector idVector = (IntVector) root.getVector("id"); + VarCharVector nameVector = (VarCharVector) root.getVector("name"); + + for (int i = 0; i < ids.size(); i++) { + if (ids.get(i) == null) { + idVector.setNull(i); + } else { + idVector.setSafe(i, ids.get(i)); + } + nameVector.setSafe(i, names.get(i).getBytes(StandardCharsets.UTF_8)); + } + + root.setRowCount(ids.size()); + + return root; + } + private ArrowArrayStream convertToStream(VectorSchemaRoot root, RootAllocator allocator) throws Exception { ByteArrayOutputStream out = new ByteArrayOutputStream(); @@ -275,6 +349,269 @@ private ArrowArrayStream convertToStream(VectorSchemaRoot root, RootAllocator al return stream; } + @Test + public void testSourceDedupeFirstSeenKeepsFirst() throws Exception { + // Source has two rows for id=0 ("First 0" then "Second 0"). FirstSeen keeps + // the first encountered row and skips the duplicate. + + try (VectorSchemaRoot source = buildDuplicateKeySource(testDataset.getSchema(), allocator)) { + try (ArrowArrayStream sourceStream = convertToStream(source, allocator)) { + MergeInsertResult result = + dataset.mergeInsert( + new MergeInsertParams(Collections.singletonList("id")) + .withMatchedUpdateAll() + .withNotMatched(MergeInsertParams.WhenNotMatched.InsertAll) + .withSourceDedupeBehavior(MergeInsertParams.SourceDedupeBehavior.FirstSeen), + sourceStream); + + Assertions.assertEquals( + "{0=First 0, 1=Person 1, 2=Person 2, 3=Source 3, 4=Source 4}", + readAll(result.dataset()).toString(), + "FirstSeen should keep the first duplicate source row (id=0) and update unique matches"); + } + } + } + + @Test + public void testSourceDedupeFailWithDuplicates() throws Exception { + // Default behavior (Fail) must error when the source contains duplicate join keys. + + try (VectorSchemaRoot source = buildDuplicateKeySource(testDataset.getSchema(), allocator)) { + try (ArrowArrayStream sourceStream = convertToStream(source, allocator)) { + String originalDataset = readAll(dataset).toString(); + + Exception ex = + Assertions.assertThrows( + Exception.class, + () -> + dataset.mergeInsert( + new MergeInsertParams(Collections.singletonList("id")) + .withMatchedUpdateAll() + .withNotMatched(MergeInsertParams.WhenNotMatched.InsertAll) + .withSourceDedupeBehavior(MergeInsertParams.SourceDedupeBehavior.Fail), + sourceStream)); + + Assertions.assertNotNull(ex.getMessage(), "exception should carry a message"); + Assertions.assertTrue( + ex.getMessage().contains("Ambiguous merge inserts are prohibited"), + "Fail should report the ambiguous-merge cause, got: " + ex.getMessage()); + + Assertions.assertEquals( + originalDataset, + readAll(dataset).toString(), + "Dataset should remain unchanged after a failed mergeInsert"); + } + } + } + + @Test + public void testSourceDedupeDeleteFullSchemaFirstSeen() throws Exception { + // Full-schema delete: source id=0 is duplicated ("First 0", then "Second 0"). + // FirstSeen deletes the matched target row once and skips the duplicate; ids + // 3 and 4 are unique matches that are also deleted. + + try (VectorSchemaRoot source = buildDuplicateKeySource(testDataset.getSchema(), allocator)) { + try (ArrowArrayStream sourceStream = convertToStream(source, allocator)) { + MergeInsertResult result = + dataset.mergeInsert( + new MergeInsertParams(Collections.singletonList("id")) + .withMatchedDelete() + .withNotMatched(MergeInsertParams.WhenNotMatched.InsertAll) + .withSourceDedupeBehavior(MergeInsertParams.SourceDedupeBehavior.FirstSeen), + sourceStream); + + Assertions.assertEquals( + "{1=Person 1, 2=Person 2}", + readAll(result.dataset()).toString(), + "FirstSeen should delete each matched target row once, skipping the duplicate id=0"); + } + } + } + + @Test + public void testSourceDedupeDeleteFullSchemaFailWithDuplicates() throws Exception { + // Full-schema delete with Fail must reject the ambiguous duplicate source key + // and leave the target unchanged. + + try (VectorSchemaRoot source = buildDuplicateKeySource(testDataset.getSchema(), allocator)) { + try (ArrowArrayStream sourceStream = convertToStream(source, allocator)) { + String originalDataset = readAll(dataset).toString(); + + Exception ex = + Assertions.assertThrows( + Exception.class, + () -> + dataset.mergeInsert( + new MergeInsertParams(Collections.singletonList("id")) + .withMatchedDelete() + .withNotMatched(MergeInsertParams.WhenNotMatched.InsertAll) + .withSourceDedupeBehavior(MergeInsertParams.SourceDedupeBehavior.Fail), + sourceStream)); + + Assertions.assertNotNull(ex.getMessage(), "exception should carry a message"); + Assertions.assertTrue( + ex.getMessage().contains("Ambiguous merge inserts are prohibited"), + "Fail should report the ambiguous-merge cause, got: " + ex.getMessage()); + + Assertions.assertEquals( + originalDataset, + readAll(dataset).toString(), + "Dataset should remain unchanged after a failed delete mergeInsert"); + } + } + } + + @Test + public void testSourceDedupeDeleteOnlyKeyOnlyFirstSeen() throws Exception { + // Key-only delete-only plan: the source carries just the join key and id=0 is + // duplicated. FirstSeen deletes the matched target row once and skips the + // duplicate; no other rows are touched. + + try (VectorSchemaRoot source = + buildDuplicateKeyOnlySource(testDataset.getSchema(), allocator)) { + try (ArrowArrayStream sourceStream = convertToStream(source, allocator)) { + MergeInsertResult result = + dataset.mergeInsert( + new MergeInsertParams(Collections.singletonList("id")) + .withMatchedDelete() + .withNotMatched(MergeInsertParams.WhenNotMatched.DoNothing) + .withSourceDedupeBehavior(MergeInsertParams.SourceDedupeBehavior.FirstSeen), + sourceStream); + + Assertions.assertEquals( + "{1=Person 1, 2=Person 2, 3=Person 3, 4=Person 4}", + readAll(result.dataset()).toString(), + "FirstSeen should delete the matched id=0 once and skip the duplicate"); + } + } + } + + @Test + public void testSourceDedupeDeleteOnlyKeyOnlyFailWithDuplicates() throws Exception { + // Key-only delete-only plan with Fail must reject the ambiguous duplicate + // source key and leave the target unchanged. + + try (VectorSchemaRoot source = + buildDuplicateKeyOnlySource(testDataset.getSchema(), allocator)) { + try (ArrowArrayStream sourceStream = convertToStream(source, allocator)) { + String originalDataset = readAll(dataset).toString(); + + Exception ex = + Assertions.assertThrows( + Exception.class, + () -> + dataset.mergeInsert( + new MergeInsertParams(Collections.singletonList("id")) + .withMatchedDelete() + .withNotMatched(MergeInsertParams.WhenNotMatched.DoNothing) + .withSourceDedupeBehavior(MergeInsertParams.SourceDedupeBehavior.Fail), + sourceStream)); + + Assertions.assertNotNull(ex.getMessage(), "exception should carry a message"); + Assertions.assertTrue( + ex.getMessage().contains("Ambiguous merge inserts are prohibited"), + "Fail should report the ambiguous-merge cause, got: " + ex.getMessage()); + + Assertions.assertEquals( + originalDataset, + readAll(dataset).toString(), + "Dataset should remain unchanged after a failed delete-only mergeInsert"); + } + } + } + + @Test + public void testSourceDedupeFailAllowsUnmatchedDuplicateKeys() throws Exception { + // Boundary: Fail only rejects multiple source rows matching the SAME target row. + // Two unmatched source rows share id=10 (no target match), so Fail must NOT error + // and both rows are inserted. + + try (VectorSchemaRoot source = + buildSourceFrom( + testDataset.getSchema(), + allocator, + Arrays.asList(10, 10), + Arrays.asList("First 10", "Second 10"))) { + try (ArrowArrayStream sourceStream = convertToStream(source, allocator)) { + MergeInsertResult result = + dataset.mergeInsert( + new MergeInsertParams(Collections.singletonList("id")) + .withMatchedUpdateAll() + .withNotMatched(MergeInsertParams.WhenNotMatched.InsertAll) + .withSourceDedupeBehavior(MergeInsertParams.SourceDedupeBehavior.Fail), + sourceStream); + + Assertions.assertEquals( + 2, + result.stats().numInsertedRows(), + "Fail should insert both unmatched duplicate-key rows, not reject them"); + Assertions.assertEquals( + 7, result.dataset().countRows(), "Base 5 rows plus 2 unmatched inserts for id=10"); + } + } + } + + @Test + public void testSourceDedupeFirstSeenDedupesUnmatchedKeys() throws Exception { + // Boundary: FirstSeen deduplicates unmatched rows that would otherwise insert the + // same non-null join key more than once. Two unmatched id=10 rows collapse to one. + + try (VectorSchemaRoot source = + buildSourceFrom( + testDataset.getSchema(), + allocator, + Arrays.asList(10, 10), + Arrays.asList("First 10", "Second 10"))) { + try (ArrowArrayStream sourceStream = convertToStream(source, allocator)) { + MergeInsertResult result = + dataset.mergeInsert( + new MergeInsertParams(Collections.singletonList("id")) + .withMatchedUpdateAll() + .withNotMatched(MergeInsertParams.WhenNotMatched.InsertAll) + .withSourceDedupeBehavior(MergeInsertParams.SourceDedupeBehavior.FirstSeen), + sourceStream); + + Assertions.assertEquals( + 1, + result.stats().numInsertedRows(), + "FirstSeen should insert only the first row for the duplicate key id=10"); + Assertions.assertEquals( + 6, result.dataset().countRows(), "Base 5 rows plus 1 deduplicated insert for id=10"); + } + } + } + + @Test + public void testSourceDedupeNullKeysNeverDuplicate() throws Exception { + // Boundary: rows whose join key is NULL are never duplicates, because merge insert + // uses SQL equality where NULL does not equal NULL. Even under FirstSeen, two NULL-key + // rows are both inserted. + + try (VectorSchemaRoot source = + buildSourceFrom( + testDataset.getSchema(), + allocator, + Arrays.asList((Integer) null, (Integer) null), + Arrays.asList("Null A", "Null B"))) { + try (ArrowArrayStream sourceStream = convertToStream(source, allocator)) { + MergeInsertResult result = + dataset.mergeInsert( + new MergeInsertParams(Collections.singletonList("id")) + .withMatchedUpdateAll() + .withNotMatched(MergeInsertParams.WhenNotMatched.InsertAll) + .withSourceDedupeBehavior(MergeInsertParams.SourceDedupeBehavior.FirstSeen), + sourceStream); + + Assertions.assertEquals( + 2, + result.stats().numInsertedRows(), + "NULL join keys are never duplicates, so both rows insert even under FirstSeen"); + Assertions.assertEquals( + 7, result.dataset().countRows(), "Base 5 rows plus 2 NULL-key inserts"); + } + } + } + @Test public void testMergeInsertWithoutIndex() throws Exception { // Verify that merge insert with useIndex=false still completes and diff --git a/java/src/test/java/org/lance/ScannerTest.java b/java/src/test/java/org/lance/ScannerTest.java index 00434034b64..d56ce868654 100644 --- a/java/src/test/java/org/lance/ScannerTest.java +++ b/java/src/test/java/org/lance/ScannerTest.java @@ -19,6 +19,7 @@ import org.lance.index.scalar.ScalarIndexParams; import org.lance.ipc.ColumnOrdering; import org.lance.ipc.LanceScanner; +import org.lance.ipc.MaterializationStyle; import org.lance.ipc.ScanOptions; import org.lance.ipc.ScanStats; @@ -37,6 +38,8 @@ import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.MethodSource; import java.io.IOException; import java.nio.file.Path; @@ -46,9 +49,11 @@ import java.util.List; import java.util.Optional; import java.util.stream.Collectors; +import java.util.stream.Stream; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; public class ScannerTest { @@ -65,6 +70,90 @@ static void tearDown() { } } + @Test + void testScannerPerformanceOptions() { + MaterializationStyle materialization = + MaterializationStyle.allEarlyExcept(Collections.singletonList("name")); + ScanOptions options = + new ScanOptions.Builder() + .batchSize(1024) + .batchSizeBytes(64 * 1024) + .ioBufferSize(8 * 1024 * 1024) + .batchReadahead(4) + .fragmentReadahead(2) + .scanInOrder(false) + .lateMaterialization(materialization) + .build(); + + assertEquals(1024L, options.getBatchSize().orElseThrow()); + assertEquals(64 * 1024L, options.getBatchSizeBytes().orElseThrow()); + assertEquals(8 * 1024 * 1024L, options.getIoBufferSize().orElseThrow()); + assertEquals(4, options.getBatchReadahead()); + assertEquals(2, options.getFragmentReadahead().orElseThrow()); + assertFalse(options.isScanInOrder()); + assertEquals(materialization, options.getLateMaterialization().orElseThrow()); + assertEquals("heuristic", MaterializationStyle.heuristic().toRustString()); + assertEquals("all_late", MaterializationStyle.allLate().toRustString()); + assertEquals("all_early", MaterializationStyle.allEarly().toRustString()); + assertEquals("all_early_except", materialization.toRustString()); + assertEquals(Collections.singletonList("name"), materialization.getColumns()); + + ScanOptions defaults = new ScanOptions.Builder().build(); + assertTrue(defaults.getBatchSizeBytes().isEmpty()); + assertTrue(defaults.getIoBufferSize().isEmpty()); + assertTrue(defaults.getFragmentReadahead().isEmpty()); + assertTrue(defaults.isScanInOrder()); + assertTrue(defaults.getLateMaterialization().isEmpty()); + + assertThrows( + IllegalArgumentException.class, () -> new ScanOptions.Builder().batchSizeBytes(0).build()); + assertThrows( + IllegalArgumentException.class, () -> new ScanOptions.Builder().ioBufferSize(0).build()); + assertThrows( + IllegalArgumentException.class, + () -> new ScanOptions.Builder().fragmentReadahead(0).build()); + assertThrows( + IllegalArgumentException.class, + () -> new ScanOptions.Builder().batchSizeBytes(1024).strictBatchSize(true).build()); + assertTrue(MaterializationStyle.allEarlyExcept(Collections.emptyList()).getColumns().isEmpty()); + } + + static Stream materializationStyles() { + return Stream.of( + MaterializationStyle.heuristic(), + MaterializationStyle.allLate(), + MaterializationStyle.allEarly(), + MaterializationStyle.allEarlyExcept(Collections.emptyList())); + } + + @ParameterizedTest + @MethodSource("materializationStyles") + void testMaterializationStylesAcrossJni( + MaterializationStyle materializationStyle, @TempDir Path tempDir) throws Exception { + String datasetPath = + tempDir.resolve("materialization_" + materializationStyle.getMode().name()).toString(); + try (BufferAllocator allocator = new RootAllocator()) { + TestUtils.SimpleTestDataset testDataset = + new TestUtils.SimpleTestDataset(allocator, datasetPath); + testDataset.createEmptyDataset().close(); + + try (Dataset dataset = testDataset.write(1, 40); + LanceScanner scanner = + dataset.newScan( + new ScanOptions.Builder() + .filter("id < 20") + .lateMaterialization(materializationStyle) + .build()); + ArrowReader reader = scanner.scanBatches()) { + int rowCount = 0; + while (reader.loadNextBatch()) { + rowCount += reader.getVectorSchemaRoot().getRowCount(); + } + assertEquals(20, rowCount); + } + } + } + @Test void testDatasetScanner(@TempDir Path tempDir) throws IOException { String datasetPath = tempDir.resolve("dataset_scanner").toString(); @@ -212,6 +301,10 @@ void testDatasetScannerStats(@TempDir Path tempDir) throws Exception { assertTrue(statsOpt.isPresent()); ScanStats stats = statsOpt.get(); assertTrue(stats.getBytesRead() > 0 || !stats.getAllCounts().isEmpty()); + // Even without an index on this dataset, the two new counters must + // still marshal through JNI and default to zero rather than throwing. + assertTrue(stats.getIndexCacheHits() >= 0); + assertTrue(stats.getIndexCacheMisses() >= 0); } } } @@ -422,31 +515,99 @@ void testDatasetScannerBatchReadahead(@TempDir Path tempDir) throws Exception { TestUtils.SimpleTestDataset testDataset = new TestUtils.SimpleTestDataset(allocator, datasetPath); testDataset.createEmptyDataset().close(); - int totalRows = 1000; - int batchSize = 100; - int batchReadahead = 5; - try (Dataset dataset = testDataset.write(1, totalRows)) { + + int totalRows = 2000; + int maxRowsPerFile = 100; // ~20 fragments + List fragments = testDataset.createNewFragment(totalRows, maxRowsPerFile); + assertTrue(fragments.size() > 1, "expected multiple fragments, got " + fragments.size()); + + FragmentOperation.Append append = new FragmentOperation.Append(fragments); + try (Dataset dataset = Dataset.commit(allocator, datasetPath, append, Optional.of(1L))) { + int batchReadahead = 2; // far below the default (num compute CPUs) try (LanceScanner scanner = dataset.newScan( - new ScanOptions.Builder() - .batchSize(batchSize) - .batchReadahead(batchReadahead) - .build())) { - // This test is more about ensuring that the batchReadahead parameter is accepted - // and doesn't cause errors. The actual effect of batchReadahead might not be - // directly observable in this test. + new ScanOptions.Builder().batchSize(50).batchReadahead(batchReadahead).build())) { try (ArrowReader reader = scanner.scanBatches()) { int rowCount = 0; + long idSum = 0; while (reader.loadNextBatch()) { - rowCount += reader.getVectorSchemaRoot().getRowCount(); + VectorSchemaRoot root = reader.getVectorSchemaRoot(); + IntVector ids = (IntVector) root.getVector("id"); + for (int i = 0; i < root.getRowCount(); i++) { + idSum += ids.get(i); + } + rowCount += root.getRowCount(); } assertEquals(totalRows, rowCount); + // ids are the contiguous range [0, totalRows) + assertEquals((long) totalRows * (totalRows - 1) / 2, idSum); } } } } } + @Test + void testDatasetScannerPerformanceOptions(@TempDir Path tempDir) throws Exception { + String datasetPath = tempDir.resolve("dataset_scanner_performance_options").toString(); + try (BufferAllocator allocator = new RootAllocator()) { + TestUtils.SimpleTestDataset testDataset = + new TestUtils.SimpleTestDataset(allocator, datasetPath); + testDataset.createEmptyDataset().close(); + + int totalRows = 2000; + int matchingRows = 1000; + WriteParams writeParams = + new WriteParams.Builder() + .withMaxRowsPerFile(100) + .withDataStorageVersion(LanceConstants.FILE_FORMAT_VERSION_STABLE) + .build(); + List fragments = testDataset.createNewFragment(totalRows, writeParams); + FragmentOperation.Append append = new FragmentOperation.Append(fragments); + try (Dataset dataset = Dataset.commit(allocator, datasetPath, append, Optional.of(1L))) { + try (LanceScanner scanner = + dataset.newScan( + new ScanOptions.Builder() + .filter("id < " + matchingRows) + .batchSize(200) + .batchSizeBytes(256) + .ioBufferSize(1024 * 1024) + .batchReadahead(2) + .fragmentReadahead(2) + .scanInOrder(false) + .lateMaterialization(MaterializationStyle.allEarly()) + .build()); + ArrowReader reader = scanner.scanBatches()) { + int rowCount = 0; + long idSum = 0; + while (reader.loadNextBatch()) { + VectorSchemaRoot root = reader.getVectorSchemaRoot(); + assertTrue( + root.getRowCount() < 200, + "batchSizeBytes should split batches before the row limit"); + IntVector ids = (IntVector) root.getVector("id"); + for (int i = 0; i < root.getRowCount(); i++) { + idSum += ids.get(i); + } + rowCount += root.getRowCount(); + } + assertEquals(matchingRows, rowCount); + assertEquals((long) matchingRows * (matchingRows - 1) / 2, idSum); + } + + try (LanceScanner scanner = + dataset.newScan( + new ScanOptions.Builder() + .filter("id < " + matchingRows) + .lateMaterialization( + MaterializationStyle.allEarlyExcept(Collections.singletonList("name"))) + .build())) { + assertEquals(matchingRows, scanner.countRows()); + } + } + } + } + @Test void testDatasetScannerSortBy(@TempDir Path tempDir) throws Exception { String datasetPath = tempDir.resolve("testDatasetScannerSortBy").toString(); diff --git a/java/src/test/java/org/lance/SessionTest.java b/java/src/test/java/org/lance/SessionTest.java index b2ed3baa343..5c2f9fb53b8 100644 --- a/java/src/test/java/org/lance/SessionTest.java +++ b/java/src/test/java/org/lance/SessionTest.java @@ -19,6 +19,8 @@ import org.junit.jupiter.api.io.TempDir; import java.nio.file.Path; +import java.util.Collections; +import java.util.Map; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; @@ -68,6 +70,135 @@ void testCreateSessionWithPartialCustomCacheSizes() { } } + @Test + void testCreateSessionWithCacheBackendUris() { + try (Session session = + Session.builder() + .indexCacheBackend("moka://?capacity=1048576") + .metadataCacheBackend("moka://?capacity=524288") + .build()) { + assertNotNull(session); + assertFalse(session.isClosed()); + assertEquals(0, session.metadataCacheStats().getNumEntries()); + } + } + + @Test + void testCreateSessionWithStructuredCacheBackendConfigs() { + CacheBackendConfig indexBackend = + CacheBackendConfig.builder("moka").option("capacity", "1048576").build(); + CacheBackendConfig metadataBackend = + CacheBackendConfig.builder("moka") + .options(Collections.singletonMap("capacity", "524288")) + .build(); + + assertEquals("moka", indexBackend.getKind()); + assertEquals(Collections.singletonMap("capacity", "1048576"), indexBackend.getOptions()); + assertThrows( + UnsupportedOperationException.class, () -> indexBackend.getOptions().put("capacity", "1")); + + try (Session session = + Session.builder() + .indexCacheBackend(indexBackend) + .metadataCacheBackend(metadataBackend) + .build()) { + assertNotNull(session); + assertFalse(session.isClosed()); + } + } + + @Test + void testCacheBackendReplacesPreviousDescriptorForSameTier() { + CacheBackendConfig structuredBackend = + CacheBackendConfig.builder("moka").option("capacity", "1048576").build(); + + try (Session session = + Session.builder() + .indexCacheBackend("missing://") + .indexCacheBackend(structuredBackend) + .build()) { + assertNotNull(session); + } + + try (Session session = + Session.builder() + .metadataCacheBackend(structuredBackend) + .metadataCacheBackend("moka://?capacity=1048576") + .build()) { + assertNotNull(session); + } + } + + @Test + void testCacheBackendRejectsSizeAndBackend() { + IllegalArgumentException indexError = + assertThrows( + IllegalArgumentException.class, + () -> + Session.builder() + .indexCacheSizeBytes(1024) + .indexCacheBackend("moka://?capacity=1048576") + .build()); + assertTrue( + indexError + .getMessage() + .contains("indexCacheSizeBytes and indexCacheBackend are mutually exclusive")); + + IllegalArgumentException metadataError = + assertThrows( + IllegalArgumentException.class, + () -> + Session.builder() + .metadataCacheBackend( + CacheBackendConfig.builder("moka").option("capacity", "1048576").build()) + .metadataCacheSizeBytes(1024) + .build()); + assertTrue( + metadataError + .getMessage() + .contains("metadataCacheSizeBytes and metadataCacheBackend are mutually exclusive")); + } + + @Test + void testCacheBackendRejectsUnknownKind() { + IllegalArgumentException error = + assertThrows( + IllegalArgumentException.class, + () -> Session.builder().indexCacheBackend("missing://").build()); + assertTrue(error.getMessage().contains("unknown cache backend kind")); + } + + @Test + void testCacheBackendRejectsInvalidMokaConfig() { + IllegalArgumentException missingCapacity = + assertThrows( + IllegalArgumentException.class, + () -> Session.builder().indexCacheBackend("moka://").build()); + assertTrue(missingCapacity.getMessage().contains("capacity is required")); + + IllegalArgumentException unknownOption = + assertThrows( + IllegalArgumentException.class, + () -> + Session.builder() + .metadataCacheBackend( + CacheBackendConfig.builder("moka").option("unknown", "value").build()) + .build()); + assertTrue(unknownOption.getMessage().contains("unknown option")); + } + + @Test + void testCacheBackendConfigValidatesInputs() { + assertThrows(NullPointerException.class, () -> CacheBackendConfig.builder(null)); + assertThrows(IllegalArgumentException.class, () -> CacheBackendConfig.builder("")); + assertThrows( + NullPointerException.class, + () -> CacheBackendConfig.builder("moka").options((Map) null)); + assertThrows( + NullPointerException.class, + () -> CacheBackendConfig.builder("moka").option("capacity", null)); + } + @Test void testSessionClose() { Session session = Session.builder().build(); @@ -259,6 +390,49 @@ void testSessionToString() { assertEquals("Session(closed)", closedSession.toString()); } + @Test + void testMetadataCacheStats(@TempDir Path tempDir) { + String datasetPath = tempDir.resolve("dataset_metadata_cache_stats").toString(); + + try (BufferAllocator allocator = new RootAllocator(); + Session session = Session.builder().build()) { + CacheStats initialStats = session.metadataCacheStats(); + assertEquals(0, initialStats.getHits()); + assertEquals(0, initialStats.getMisses()); + assertEquals(0, initialStats.getNumEntries()); + assertEquals(0, initialStats.getSizeBytes()); + assertEquals(0.0, initialStats.getHitRatio()); + + TestUtils.SimpleTestDataset testDataset = + new TestUtils.SimpleTestDataset(allocator, datasetPath); + testDataset.createEmptyDataset().close(); + + // First open populates the metadata cache, producing misses + try (Dataset ds = + Dataset.open().allocator(allocator).uri(datasetPath).session(session).build()) { + ds.countRows(); + } + CacheStats statsAfterFirstOpen = session.metadataCacheStats(); + assertTrue(statsAfterFirstOpen.getMisses() > 0); + assertTrue(statsAfterFirstOpen.getNumEntries() > 0); + assertTrue(statsAfterFirstOpen.getSizeBytes() > 0); + + // Reopening the same dataset should hit the shared metadata cache + try (Dataset ds = + Dataset.open().allocator(allocator).uri(datasetPath).session(session).build()) { + ds.countRows(); + } + CacheStats statsAfterSecondOpen = session.metadataCacheStats(); + assertTrue(statsAfterSecondOpen.getHits() > statsAfterFirstOpen.getHits()); + assertTrue(statsAfterSecondOpen.getHitRatio() > 0.0); + + // Stats are not accessible once the session is closed (close is idempotent, + // so the implicit close from try-with-resources remains safe) + session.close(); + assertThrows(IllegalArgumentException.class, session::metadataCacheStats); + } + } + @Test void testInvalidCacheSizes() { assertThrows( diff --git a/java/src/test/java/org/lance/TestUtils.java b/java/src/test/java/org/lance/TestUtils.java index c9033361103..1989a5243bc 100644 --- a/java/src/test/java/org/lance/TestUtils.java +++ b/java/src/test/java/org/lance/TestUtils.java @@ -111,6 +111,11 @@ public FragmentMetadata createNewFragment(int rowCount) { } public List createNewFragment(int rowCount, int maxRowsPerFile) { + return createNewFragment( + rowCount, new WriteParams.Builder().withMaxRowsPerFile(maxRowsPerFile).build()); + } + + public List createNewFragment(int rowCount, WriteParams writeParams) { List fragmentMetas; try (VectorSchemaRoot root = VectorSchemaRoot.create(getSchema(), allocator)) { root.allocateNew(); @@ -124,12 +129,7 @@ public List createNewFragment(int rowCount, int maxRowsPerFile } root.setRowCount(rowCount); - fragmentMetas = - Fragment.create( - datasetPath, - allocator, - root, - new WriteParams.Builder().withMaxRowsPerFile(maxRowsPerFile).build()); + fragmentMetas = Fragment.create(datasetPath, allocator, root, writeParams); } return fragmentMetas; } @@ -694,7 +694,8 @@ public Dataset createEmptyDataset() { /** * Create a single fragment with given row count and return its metadata. The fragment contains * deterministic blob payloads: - Every 16th row starting at 0 has zero-length blob - Every 16th - * row starting at 1 has a ~1 MiB payload - Others have small variable blobs (128..383 bytes) + * row starting at 1 has a ~1 MiB payload - Every 16th row starting at 15 is null - Others have + * small variable blobs (128..383 bytes) */ public FragmentMetadata createBlobFragment(int rowCount, int maxRowsPerFile) { Preconditions.checkArgument(rowCount >= 0, "rowCount must be non-negative"); @@ -716,6 +717,8 @@ public FragmentMetadata createBlobFragment(int rowCount, int maxRowsPerFile) { byte[] big = new byte[1024 * 1024]; Arrays.fill(big, (byte) 0xAB); blobsVec.setSafe(i, big); + } else if (i % 16 == 15) { + blobsVec.setNull(i); } else { // small variable blob int sz = 128 + (i % 256); diff --git a/java/src/test/java/org/lance/fragment/FragmentUpdateResultTest.java b/java/src/test/java/org/lance/fragment/FragmentUpdateResultTest.java new file mode 100644 index 00000000000..1161f18b415 --- /dev/null +++ b/java/src/test/java/org/lance/fragment/FragmentUpdateResultTest.java @@ -0,0 +1,106 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.lance.fragment; + +import org.lance.CommitBuilder; +import org.lance.Dataset; +import org.lance.Fragment; +import org.lance.FragmentMetadata; +import org.lance.TestUtils; +import org.lance.Transaction; +import org.lance.operation.Append; + +import org.apache.arrow.memory.RootAllocator; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import java.nio.file.Path; +import java.util.Collections; + +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +public class FragmentUpdateResultTest { + + /** Portable RoaringBitmap bytes for offsets {1, 3, 5} (see UpdateTest round-trip fixture). */ + private static final byte[] PORTABLE_ROARING_BYTES_135 = + new byte[] { + (byte) 0x3A, + (byte) 0x30, + (byte) 0x00, + (byte) 0x00, + (byte) 0x01, + (byte) 0x00, + (byte) 0x00, + (byte) 0x00, + (byte) 0x00, + (byte) 0x00, + (byte) 0x02, + (byte) 0x00, + (byte) 0x10, + (byte) 0x00, + (byte) 0x00, + (byte) 0x00, + (byte) 0x01, + (byte) 0x00, + (byte) 0x03, + (byte) 0x00, + (byte) 0x05, + (byte) 0x00 + }; + + @Test + void testGetUpdatedRowOffsetBytesRoundTripViaDeprecatedGetter() { + FragmentUpdateResult result = + FragmentUpdateResult.create(null, new long[0], PORTABLE_ROARING_BYTES_135); + assertArrayEquals(PORTABLE_ROARING_BYTES_135, result.getUpdatedRowOffsetBytes()); + assertArrayEquals(new long[] {1, 3, 5}, result.getUpdatedRowOffsets()); + } + + @Test + void testDeprecatedLongArrayConstructorEncodesToBytes() { + FragmentUpdateResult result = new FragmentUpdateResult(null, new long[0], new long[] {1, 3, 5}); + assertArrayEquals(new long[] {1, 3, 5}, result.getUpdatedRowOffsets()); + + // Stored bytes from the deprecated long[] constructor decode to the same offsets. + FragmentUpdateResult fromEncodedBytes = + FragmentUpdateResult.create(null, new long[0], result.getUpdatedRowOffsetBytes()); + assertArrayEquals(new long[] {1, 3, 5}, fromEncodedBytes.getUpdatedRowOffsets()); + } + + @Test + void testUpdateColumnsReturnsMatchedRowOffsetBytes(@TempDir Path tempDir) throws Exception { + String datasetPath = tempDir.resolve("testUpdateColumnsRowOffsetBytes").toString(); + try (RootAllocator allocator = new RootAllocator(Long.MAX_VALUE)) { + TestUtils.UpdateColumnTestDataset testDataset = + new TestUtils.UpdateColumnTestDataset(allocator, datasetPath); + try (Dataset dataset = testDataset.createEmptyDataset()) { + FragmentMetadata fragmentMeta = testDataset.createNewFragment(6); + try (Transaction appendTxn = + new Transaction.Builder() + .readVersion(dataset.version()) + .operation( + Append.builder().fragments(Collections.singletonList(fragmentMeta)).build()) + .build()) { + try (Dataset appended = new CommitBuilder(dataset).execute(appendTxn)) { + Fragment fragment = appended.getFragments().get(0); + FragmentUpdateResult updateResult = testDataset.updateColumn(fragment, 4); + assertTrue(updateResult.getUpdatedRowOffsetBytes().length > 0); + assertArrayEquals(new long[] {0, 1, 2, 3}, updateResult.getUpdatedRowOffsets()); + } + } + } + } + } +} diff --git a/java/src/test/java/org/lance/index/ScalarIndexTest.java b/java/src/test/java/org/lance/index/ScalarIndexTest.java index cb090e7c955..6e3e309d681 100644 --- a/java/src/test/java/org/lance/index/ScalarIndexTest.java +++ b/java/src/test/java/org/lance/index/ScalarIndexTest.java @@ -40,6 +40,7 @@ import org.apache.arrow.vector.types.pojo.Schema; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.Timeout; import org.junit.jupiter.api.io.TempDir; import java.io.ByteArrayInputStream; @@ -52,6 +53,8 @@ import java.util.List; import java.util.Optional; import java.util.UUID; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; @@ -60,6 +63,113 @@ public class ScalarIndexTest { + private static final class RecordingIndexBuildProgress implements IndexBuildProgress { + private final List events = Collections.synchronizedList(new ArrayList<>()); + + @Override + public void stageStart(String stage, Optional total, String unit) { + events.add( + "start:" + stage + ":" + total.map(String::valueOf).orElse("unknown") + ":" + unit); + } + + @Override + public void stageProgress(String stage, long completed) { + events.add("progress:" + stage + ":" + completed); + } + + @Override + public void stageComplete(String stage) { + events.add("complete:" + stage); + } + + private List snapshot() { + synchronized (events) { + return new ArrayList<>(events); + } + } + } + + private static final class FailingProgressIndexBuildProgress implements IndexBuildProgress { + private final RecordingIndexBuildProgress recorder = new RecordingIndexBuildProgress(); + + @Override + public void stageStart(String stage, Optional total, String unit) { + recorder.stageStart(stage, total, unit); + } + + @Override + public void stageProgress(String stage, long completed) { + recorder.stageProgress(stage, completed); + throw new IllegalStateException("progress callback failure"); + } + + @Override + public void stageComplete(String stage) { + recorder.stageComplete(stage); + } + } + + private static final class FailingCompleteIndexBuildProgress implements IndexBuildProgress { + private final RecordingIndexBuildProgress recorder = new RecordingIndexBuildProgress(); + + @Override + public void stageStart(String stage, Optional total, String unit) { + recorder.stageStart(stage, total, unit); + } + + @Override + public void stageProgress(String stage, long completed) { + recorder.stageProgress(stage, completed); + } + + @Override + public void stageComplete(String stage) { + recorder.stageComplete(stage); + throw new IllegalStateException("complete callback failure"); + } + } + + /** + * Progress callback that re-enters the same Dataset via JNI. Without releasing the native field + * lock before merge starts, these calls would deadlock. + */ + private static final class ReentrantDatasetIndexBuildProgress implements IndexBuildProgress { + private final Dataset dataset; + private final RecordingIndexBuildProgress recorder = new RecordingIndexBuildProgress(); + private final AtomicInteger reentries = new AtomicInteger(); + + private ReentrantDatasetIndexBuildProgress(Dataset dataset) { + this.dataset = dataset; + } + + @Override + public void stageStart(String stage, Optional total, String unit) { + recorder.stageStart(stage, total, unit); + touchDataset(); + } + + @Override + public void stageProgress(String stage, long completed) { + recorder.stageProgress(stage, completed); + touchDataset(); + } + + @Override + public void stageComplete(String stage) { + recorder.stageComplete(stage); + touchDataset(); + } + + private void touchDataset() { + assertNotNull(dataset.uri()); + assertTrue(dataset.version() > 0); + assertTrue(dataset.countRows() > 0); + assertFalse(dataset.getFragments().isEmpty()); + assertFalse(dataset.memWalIndexDetails().isPresent()); + reentries.incrementAndGet(); + } + } + @Test public void testCreateBTreeIndex(@TempDir Path tempDir) throws Exception { String datasetPath = tempDir.resolve("btree_test").toString(); @@ -276,6 +386,175 @@ public void testBtreeMergeIndexMetadataSoftBreak(@TempDir Path tempDir) throws E } } + @Test + public void testMergeInvertedIndexMetadataReportsProgress(@TempDir Path tempDir) + throws Exception { + String datasetPath = tempDir.resolve("inverted_merge_progress").toString(); + try (RootAllocator allocator = new RootAllocator(Long.MAX_VALUE)) { + TestUtils.SimpleTestDataset testDataset = + new TestUtils.SimpleTestDataset(allocator, datasetPath); + testDataset.createEmptyDataset().close(); + testDataset.write(1, 10).close(); + try (Dataset dataset = testDataset.write(2, 10)) { + String indexUuid = createDistributedInvertedIndex(dataset); + RecordingIndexBuildProgress progress = new RecordingIndexBuildProgress(); + + dataset.mergeIndexMetadata(indexUuid, IndexType.INVERTED, Optional.empty(), progress); + + List events = progress.snapshot(); + assertEventsInOrder( + events, + "start:read_partition_metadata:", + "complete:read_partition_metadata", + "start:remap_partition_files:", + "complete:remap_partition_files", + "start:write_merged_metadata:", + "complete:write_merged_metadata"); + assertTrue( + events.contains("progress:read_partition_metadata:2"), + "Expected metadata progress to reach both fragments, got: " + events); + assertTrue( + events.stream().anyMatch(event -> event.startsWith("progress:remap_partition_files:")), + "Expected remap progress, got: " + events); + assertTrue( + events.contains("progress:write_merged_metadata:1"), + "Expected merged metadata write progress, got: " + events); + } + } + } + + @Test + public void testMergeInvertedIndexMetadataPropagatesProgressFailure(@TempDir Path tempDir) + throws Exception { + String datasetPath = tempDir.resolve("inverted_merge_progress_failure").toString(); + try (RootAllocator allocator = new RootAllocator(Long.MAX_VALUE)) { + TestUtils.SimpleTestDataset testDataset = + new TestUtils.SimpleTestDataset(allocator, datasetPath); + testDataset.createEmptyDataset().close(); + testDataset.write(1, 10).close(); + try (Dataset dataset = testDataset.write(2, 10)) { + String indexUuid = createDistributedInvertedIndex(dataset); + + RuntimeException failure = + Assertions.assertThrows( + RuntimeException.class, + () -> + dataset.mergeIndexMetadata( + indexUuid, + IndexType.INVERTED, + Optional.empty(), + new FailingProgressIndexBuildProgress())); + + assertFalse( + failure instanceof IllegalArgumentException, + "Progress callback failures should not be reported as invalid input: " + failure); + assertTrue( + causeChainContains(failure, "stageProgress") + && causeChainContains(failure, "read_partition_metadata") + && causeChainContains(failure, "java.lang.IllegalStateException") + && causeChainContains(failure, "progress callback failure"), + "Expected callback context and original Java exception details, got: " + failure); + } + } + } + + @Test + public void testMergeInvertedIndexMetadataIgnoresCompleteFailure(@TempDir Path tempDir) + throws Exception { + String datasetPath = tempDir.resolve("inverted_merge_complete_failure").toString(); + try (RootAllocator allocator = new RootAllocator(Long.MAX_VALUE)) { + TestUtils.SimpleTestDataset testDataset = + new TestUtils.SimpleTestDataset(allocator, datasetPath); + testDataset.createEmptyDataset().close(); + testDataset.write(1, 10).close(); + try (Dataset dataset = testDataset.write(2, 10)) { + String indexUuid = createDistributedInvertedIndex(dataset); + FailingCompleteIndexBuildProgress progress = new FailingCompleteIndexBuildProgress(); + + dataset.mergeIndexMetadata(indexUuid, IndexType.INVERTED, Optional.empty(), progress); + + assertTrue( + progress.recorder.snapshot().contains("complete:write_merged_metadata"), + "Expected merge to continue after stageComplete callback failures"); + } + } + } + + @Test + @Timeout(value = 60, unit = TimeUnit.SECONDS) + public void testMergeInvertedIndexMetadataAllowsReentrantDatasetAccess(@TempDir Path tempDir) + throws Exception { + String datasetPath = tempDir.resolve("inverted_merge_reentrant_dataset").toString(); + try (RootAllocator allocator = new RootAllocator(Long.MAX_VALUE)) { + TestUtils.SimpleTestDataset testDataset = + new TestUtils.SimpleTestDataset(allocator, datasetPath); + testDataset.createEmptyDataset().close(); + testDataset.write(1, 10).close(); + try (Dataset dataset = testDataset.write(2, 10)) { + String indexUuid = createDistributedInvertedIndex(dataset); + ReentrantDatasetIndexBuildProgress progress = + new ReentrantDatasetIndexBuildProgress(dataset); + + dataset.mergeIndexMetadata(indexUuid, IndexType.INVERTED, Optional.empty(), progress); + + assertTrue( + progress.reentries.get() > 0, + "Expected progress callbacks to re-enter Dataset JNI methods"); + assertTrue( + progress.recorder.snapshot().contains("complete:write_merged_metadata"), + "Expected merge to finish after re-entrant Dataset access, got: " + + progress.recorder.snapshot()); + } + } + } + + private static String createDistributedInvertedIndex(Dataset dataset) { + ScalarIndexParams scalarParams = + ScalarIndexParams.create( + "inverted", + "{\"base_tokenizer\":\"simple\",\"language\":\"English\"," + + "\"max_token_length\":40,\"lower_case\":true,\"stem\":false," + + "\"remove_stop_words\":false}"); + IndexParams indexParams = IndexParams.builder().setScalarIndexParams(scalarParams).build(); + String indexUuid = UUID.randomUUID().toString(); + for (Fragment fragment : dataset.getFragments()) { + dataset.createIndex( + IndexOptions.builder(Collections.singletonList("name"), IndexType.INVERTED, indexParams) + .replace(true) + .withIndexName("inverted_progress_idx") + .withIndexUUID(indexUuid) + .withFragmentIds(Collections.singletonList(fragment.getId())) + .build()); + } + return indexUuid; + } + + private static void assertEventsInOrder(List events, String... prefixes) { + int previous = -1; + for (String prefix : prefixes) { + int current = -1; + for (int i = previous + 1; i < events.size(); i++) { + if (events.get(i).startsWith(prefix)) { + current = i; + break; + } + } + assertTrue( + current >= 0, + "Missing event '" + prefix + "' after position " + previous + ": " + events); + previous = current; + } + } + + private static boolean causeChainContains(Throwable failure, String expected) { + for (Throwable current = failure; current != null; current = current.getCause()) { + if (current.getMessage() != null && current.getMessage().contains(expected)) { + return true; + } + } + return false; + } + @Test public void testCreateZonemapIndex(@TempDir Path tempDir) throws Exception { String datasetPath = tempDir.resolve("zonemap_test").toString(); diff --git a/java/src/test/java/org/lance/index/scalar/InvertedIndexParamsTest.java b/java/src/test/java/org/lance/index/scalar/InvertedIndexParamsTest.java index e5024a95c2a..9fb8de4375b 100644 --- a/java/src/test/java/org/lance/index/scalar/InvertedIndexParamsTest.java +++ b/java/src/test/java/org/lance/index/scalar/InvertedIndexParamsTest.java @@ -13,19 +13,93 @@ */ package org.lance.index.scalar; +import org.lance.DocumentGranularity; +import org.lance.util.JsonUtils; + import org.junit.jupiter.api.Test; +import java.util.Map; + import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; -public class InvertedIndexParamsTest { +class InvertedIndexParamsTest { @Test - public void testIcuSplitTokenizerVariant() { + void testIcuSplitTokenizerVariant() { ScalarIndexParams params = InvertedIndexParams.builder().baseTokenizer("icu/split").build(); assertEquals("inverted", params.getIndexType()); String jsonParams = params.getJsonParams().orElseThrow(AssertionError::new); assertTrue(jsonParams.contains("\"base_tokenizer\":\"icu/split\"")); } + + @Test + void defaultBlockSizeIsSerialized() { + ScalarIndexParams params = InvertedIndexParams.builder().build(); + + Map json = JsonUtils.fromJson(params.getJsonParams().orElseThrow()); + assertEquals(128, ((Number) json.get("block_size")).intValue()); + assertEquals("row", json.get("document_granularity")); + } + + @Test + void documentGranularityIsSerialized() { + ScalarIndexParams params = + InvertedIndexParams.builder().documentGranularity(DocumentGranularity.LIST_ELEMENT).build(); + + Map json = JsonUtils.fromJson(params.getJsonParams().orElseThrow()); + assertEquals("list_element", json.get("document_granularity")); + } + + @Test + void blockSizeIsSerialized() { + ScalarIndexParams params = InvertedIndexParams.builder().blockSize(128).build(); + + assertEquals("inverted", params.getIndexType()); + Map json = JsonUtils.fromJson(params.getJsonParams().orElseThrow()); + assertEquals(128, ((Number) json.get("block_size")).intValue()); + } + + @Test + void invalidBlockSizeIsRejected() { + assertThrows( + IllegalArgumentException.class, () -> InvertedIndexParams.builder().blockSize(129)); + assertThrows( + IllegalArgumentException.class, () -> InvertedIndexParams.builder().blockSize(512)); + } + + @Test + void formatVersionThreeSupportsBothBlockSizes() { + for (int blockSize : new int[] {128, 256}) { + ScalarIndexParams params = + InvertedIndexParams.builder().blockSize(blockSize).formatVersion(3).build(); + Map json = JsonUtils.fromJson(params.getJsonParams().orElseThrow()); + assertEquals(blockSize, ((Number) json.get("block_size")).intValue()); + assertEquals(3, ((Number) json.get("format_version")).intValue()); + } + + assertThrows( + IllegalArgumentException.class, + () -> InvertedIndexParams.builder().blockSize(256).formatVersion(2).build()); + } + + @Test + void formatVersionFourIsRejected() { + assertThrows( + IllegalArgumentException.class, () -> InvertedIndexParams.builder().formatVersion(4)); + } + + @Test + void codeAnalyzerRequiresFormatVersionThreeWhenExplicit() { + ScalarIndexParams params = + InvertedIndexParams.builder().baseTokenizer("code").formatVersion(3).build(); + Map json = JsonUtils.fromJson(params.getJsonParams().orElseThrow()); + assertEquals(3, ((Number) json.get("format_version")).intValue()); + + assertThrows( + IllegalArgumentException.class, + () -> InvertedIndexParams.builder().baseTokenizer("code").formatVersion(2).build()); + } } diff --git a/java/src/test/java/org/lance/ipc/FullTextQueryTest.java b/java/src/test/java/org/lance/ipc/FullTextQueryTest.java index 3dcf4276277..b84a19db4c8 100755 --- a/java/src/test/java/org/lance/ipc/FullTextQueryTest.java +++ b/java/src/test/java/org/lance/ipc/FullTextQueryTest.java @@ -13,6 +13,8 @@ */ package org.lance.ipc; +import org.lance.DocumentGranularity; + import org.junit.jupiter.api.Test; import java.util.Arrays; @@ -39,6 +41,7 @@ void testMatchQueryDefaults() { assertEquals(50, q.getMaxExpansions()); assertEquals(FullTextQuery.Operator.OR, q.getOperator()); assertEquals(0, q.getPrefixLength()); + assertEquals(Optional.empty(), q.getDocumentGranularity()); } @Test @@ -46,7 +49,14 @@ void testMatchQueryCustomParameters() { FullTextQuery.MatchQuery q = (FullTextQuery.MatchQuery) FullTextQuery.match( - "hello", "title", 2.0f, Optional.of(1), 10, FullTextQuery.Operator.AND, 3); + "hello", + "title", + 2.0f, + Optional.of(1), + 10, + FullTextQuery.Operator.AND, + 3, + DocumentGranularity.LIST_ELEMENT); assertEquals(FullTextQuery.Type.MATCH, q.getType()); assertEquals("hello", q.getQueryText()); @@ -56,6 +66,7 @@ void testMatchQueryCustomParameters() { assertEquals(10, q.getMaxExpansions()); assertEquals(FullTextQuery.Operator.AND, q.getOperator()); assertEquals(3, q.getPrefixLength()); + assertEquals(Optional.of(DocumentGranularity.LIST_ELEMENT), q.getDocumentGranularity()); } @Test @@ -67,17 +78,20 @@ void testPhraseQueryDefaults() { assertEquals("exact match", q.getQueryText()); assertEquals("content", q.getColumn()); assertEquals(0, q.getSlop()); + assertEquals(Optional.empty(), q.getDocumentGranularity()); } @Test void testPhraseQueryCustomSlop() { FullTextQuery.PhraseQuery q = - (FullTextQuery.PhraseQuery) FullTextQuery.phrase("ordered terms", "content", 2); + (FullTextQuery.PhraseQuery) + FullTextQuery.phrase("ordered terms", "content", 2, DocumentGranularity.LIST_ELEMENT); assertEquals(FullTextQuery.Type.MATCH_PHRASE, q.getType()); assertEquals("ordered terms", q.getQueryText()); assertEquals("content", q.getColumn()); assertEquals(2, q.getSlop()); + assertEquals(Optional.of(DocumentGranularity.LIST_ELEMENT), q.getDocumentGranularity()); } @Test diff --git a/java/src/test/java/org/lance/ipc/LanceScannerFullTextSearchTest.java b/java/src/test/java/org/lance/ipc/LanceScannerFullTextSearchTest.java index 1c46b399195..9098084ac9f 100755 --- a/java/src/test/java/org/lance/ipc/LanceScannerFullTextSearchTest.java +++ b/java/src/test/java/org/lance/ipc/LanceScannerFullTextSearchTest.java @@ -14,6 +14,7 @@ package org.lance.ipc; import org.lance.Dataset; +import org.lance.DocumentGranularity; import org.lance.WriteParams; import org.lance.index.IndexOptions; import org.lance.index.IndexParams; @@ -41,12 +42,31 @@ import java.util.Collections; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; class LanceScannerFullTextSearchTest { @Test void testMatchQuery() throws Exception { - runFtsQuery("memory://fts_java_match", FullTextQuery.match("hello", "doc"), 2L); + runFtsQuery( + "memory://fts_java_match", + FullTextQuery.match("hello", "doc", DocumentGranularity.ROW), + 2L); + } + + @Test + void testExplicitListElementGranularityReachesRustRouting() { + RuntimeException error = + assertThrows( + RuntimeException.class, + () -> + runFtsQuery( + "memory://fts_java_list_element_validation", + FullTextQuery.match("hello", "doc", DocumentGranularity.LIST_ELEMENT), + 0L)); + assertTrue(error.getMessage().contains("requested ListElement"), error.getMessage()); + assertTrue(error.getMessage().contains("'doc_idx' (Row)"), error.getMessage()); } @Test diff --git a/java/src/test/java/org/lance/memwal/MemWalTest.java b/java/src/test/java/org/lance/memwal/MemWalTest.java index de0124bb1b3..e9bd053f298 100644 --- a/java/src/test/java/org/lance/memwal/MemWalTest.java +++ b/java/src/test/java/org/lance/memwal/MemWalTest.java @@ -97,6 +97,22 @@ private static VectorSchemaRoot lookupRoot(BufferAllocator allocator, long[] ids return root; } + /** Build a single-batch root carrying only the {@code id} primary key, for deletes. */ + private static VectorSchemaRoot keysRoot(BufferAllocator allocator, long[] ids) { + VectorSchemaRoot root = + VectorSchemaRoot.create( + new Schema( + Collections.singletonList(Field.nullable("id", new ArrowType.Int(64, true)))), + allocator); + BigIntVector idVector = (BigIntVector) root.getVector("id"); + idVector.allocateNew(ids.length); + for (int i = 0; i < ids.length; i++) { + idVector.set(i, ids[i]); + } + root.setRowCount(ids.length); + return root; + } + /** Build a single-batch append-only root without primary-key metadata. */ private static VectorSchemaRoot appendOnlyRoot( BufferAllocator allocator, long[] ids, String prefix) { @@ -144,12 +160,12 @@ private static Dataset writeAppendOnlyDataset( } /** - * Stage a faithful flushed generation at {@code genPath}: the Lance dataset plus its - * primary-key dedup sidecar ({@code _pk_index/}), mirroring what production flush emits. The LSM - * scanner's cross-generation block-list opens the sidecar, so a dataset alone (no sidecar) is not - * a state production produces. Mirrors the Python {@code _write_flushed_gen} test helper. + * Stage a faithful SSTable at {@code genPath}: the Lance dataset plus its primary-key + * dedup sidecar ({@code _pk_index/}), mirroring what production flush emits. The LSM scanner's + * cross-generation block-list opens the sidecar, so a dataset alone (no sidecar) is not a state + * production produces. Mirrors the Python {@code _write_sstable} test helper. */ - private static void writeFlushedGen( + private static void writeSsTable( BufferAllocator allocator, String genPath, long[] ids, String prefix) throws Exception { writeLookupDataset(allocator, genPath, ids, prefix).close(); try (VectorSchemaRoot root = lookupRoot(allocator, ids, prefix); @@ -161,8 +177,8 @@ private static void writeFlushedGen( } /** - * Test-support native: write the primary-key dedup sidecar for a flushed-generation dataset - * already staged at {@code genPath}. See {@link #writeFlushedGen}. + * Test-support native: write the primary-key dedup sidecar for an SSTable dataset already staged + * at {@code genPath}. See {@link #writeSsTable}. */ private static native void nativeWritePkSidecar( String genPath, long streamAddress, List pkColumns); @@ -382,6 +398,51 @@ void testShardWriterPutAndLsmScanner(@TempDir Path tempDir) throws Exception { } } + @Test + void testShardWriterDeleteMasksBaseRow(@TempDir Path tempDir) throws Exception { + String path = tempDir.resolve("base").toString(); + String shardId = UUID.randomUUID().toString(); + try (BufferAllocator allocator = new RootAllocator(); + Dataset dataset = writeLookupDataset(allocator, path, new long[] {1, 2, 3}, "base")) { + dataset.initializeMemWal(new InitializeMemWalParams()); + + ShardWriterConfig config = + new ShardWriterConfig() + .withDurableWrite(true) + .withMaxWalBufferSize(1) + .withMaxWalFlushIntervalMs(10); + + try (ShardWriter writer = dataset.memWalWriter(shardId, config)) { + try (VectorSchemaRoot root = lookupRoot(allocator, new long[] {4}, "writer"); + ArrowReader reader = toReader(allocator, root)) { + writer.put(reader); + } + try (VectorSchemaRoot keys = keysRoot(allocator, new long[] {2}); + ArrowReader reader = toReader(allocator, keys)) { + writer.delete(reader); + } + + Map byId = Collections.emptyMap(); + long deadline = System.currentTimeMillis() + 10_000; + while (System.currentTimeMillis() < deadline) { + try (LsmScanner scanner = writer.lsmScanner(); + ArrowReader reader = scanner.scanBatches()) { + byId = readByName(reader); + } + if (!byId.containsKey(2L) && "writer_4".equals(byId.get(4L))) { + break; + } + Thread.sleep(50); + } + + assertEquals("base_1", byId.get(1L)); + assertFalse(byId.containsKey(2L), "deleted base row should be masked by the tombstone"); + assertEquals("base_3", byId.get(3L)); + assertEquals("writer_4", byId.get(4L)); + } + } + } + @Test void testLsmScannerFromSnapshots(@TempDir Path tempDir) throws Exception { String basePath = tempDir.resolve("base").toString(); @@ -390,12 +451,12 @@ void testLsmScannerFromSnapshots(@TempDir Path tempDir) throws Exception { Dataset dataset = writeLookupDataset(allocator, basePath, new long[] {1, 2, 3}, "base")) { dataset.initializeMemWal(new InitializeMemWalParams()); - // Flushed generation overwrites id=2. + // SSTable overwrites id=2. String genPath = basePath + "/_mem_wal/" + shardId + "/gen_1"; - writeFlushedGen(allocator, genPath, new long[] {2}, "gen1"); + writeSsTable(allocator, genPath, new long[] {2}, "gen1"); ShardSnapshot snapshot = - new ShardSnapshot(shardId).withFlushedGeneration(1, "gen_1").withCurrentGeneration(2); + new ShardSnapshot(shardId).withSsTable(1, "gen_1").withCurrentGeneration(2); try (LsmScanner scanner = LsmScanner.fromSnapshots(dataset, Collections.singletonList(snapshot)); @@ -403,7 +464,7 @@ void testLsmScannerFromSnapshots(@TempDir Path tempDir) throws Exception { Map byId = readByName(reader); assertEquals(3, byId.size(), "Expected 3 deduplicated rows"); assertEquals("base_1", byId.get(1L)); - assertEquals("gen1_2", byId.get(2L), "Flushed generation must win over base"); + assertEquals("gen1_2", byId.get(2L), "SSTable must win over base"); assertEquals("base_3", byId.get(3L)); } @@ -426,14 +487,14 @@ void testPointLookup(@TempDir Path tempDir) throws Exception { dataset.initializeMemWal(new InitializeMemWalParams()); String genPath = basePath + "/_mem_wal/" + shardId + "/gen_1"; - writeFlushedGen(allocator, genPath, new long[] {2}, "gen1"); + writeSsTable(allocator, genPath, new long[] {2}, "gen1"); ShardSnapshot snapshot = - new ShardSnapshot(shardId).withFlushedGeneration(1, "gen_1").withCurrentGeneration(2); + new ShardSnapshot(shardId).withSsTable(1, "gen_1").withCurrentGeneration(2); try (LsmPointLookupPlanner planner = new LsmPointLookupPlanner(dataset, Collections.singletonList(snapshot))) { - // id=2 must resolve to the flushed-generation value. + // id=2 must resolve to the SSTable value. assertEquals("gen1_2", lookup(planner, allocator, 2L)); // id=1 only exists in the base table. assertEquals("base_1", lookup(planner, allocator, 1L)); @@ -459,7 +520,7 @@ private static String lookup(LsmPointLookupPlanner planner, BufferAllocator allo } @Test - void testMergeInsertMarkGenerationsAsMerged(@TempDir Path tempDir) throws Exception { + void testMergeInsertMarkSstablesAsCompacted(@TempDir Path tempDir) throws Exception { String path = tempDir.resolve("base").toString(); String shardId = UUID.randomUUID().toString(); try (BufferAllocator allocator = new RootAllocator()) { @@ -471,8 +532,8 @@ void testMergeInsertMarkGenerationsAsMerged(@TempDir Path tempDir) throws Except new MergeInsertParams(Collections.singletonList("id")) .withMatchedUpdateAll() .withNotMatched(MergeInsertParams.WhenNotMatched.InsertAll) - .markGenerationsAsMerged( - Collections.singletonList(new MergedGeneration(shardId, 1))); + .markSstablesAsCompacted( + Collections.singletonList(new CompactedSsTable(shardId, 1))); try (VectorSchemaRoot root = lookupRoot(allocator, new long[] {2, 4}, "merged"); ArrowReader reader = toReader(allocator, root); diff --git a/java/src/test/java/org/lance/operation/MergeTest.java b/java/src/test/java/org/lance/operation/MergeTest.java index 841ad41d5ea..1e037285217 100644 --- a/java/src/test/java/org/lance/operation/MergeTest.java +++ b/java/src/test/java/org/lance/operation/MergeTest.java @@ -104,10 +104,14 @@ void testMergeNewColumn(@TempDir Path tempDir) throws Exception { Merge.builder() .fragments(Collections.singletonList(evolvedFragment)) .schema(evolvedSchema) + .preservesNullability(true) .build()) .build()) { try (Dataset evolvedDataset = new CommitBuilder(initialDataset).execute(mergeTxn)) { Assertions.assertEquals(3, evolvedDataset.version()); + // The explicit non-default assertion must survive the JNI round trip. + Transaction readBack = evolvedDataset.readTransaction().orElseThrow(); + Assertions.assertTrue(((Merge) readBack.operation()).preservesNullability()); Assertions.assertEquals(rowCount, evolvedDataset.countRows()); Assertions.assertEquals(evolvedSchema, evolvedDataset.getSchema()); Assertions.assertEquals(3, evolvedDataset.getSchema().getFields().size()); @@ -136,6 +140,27 @@ void testMergeNewColumn(@TempDir Path tempDir) throws Exception { } } + @Test + void testPreservesNullabilityEquality() { + Schema schema = + new Schema( + Collections.singletonList(Field.nullable("id", new ArrowType.Int(32, true))), null); + // No assertion by default, and the assertion is part of the operation's identity. + Assertions.assertFalse( + Merge.builder() + .fragments(Collections.emptyList()) + .schema(schema) + .build() + .preservesNullability()); + Assertions.assertNotEquals( + Merge.builder().fragments(Collections.emptyList()).schema(schema).build(), + Merge.builder() + .fragments(Collections.emptyList()) + .schema(schema) + .preservesNullability(true) + .build()); + } + @Test void testMergeNewColumnWithNonContiguousFieldId(@TempDir Path tempDir) throws Exception { String datasetPath = tempDir.resolve("testMergeNewColumnWithNonContiguousFieldId").toString(); diff --git a/java/src/test/java/org/lance/operation/ProjectTest.java b/java/src/test/java/org/lance/operation/ProjectTest.java index fa0c92cc15f..bd3dd7d2960 100644 --- a/java/src/test/java/org/lance/operation/ProjectTest.java +++ b/java/src/test/java/org/lance/operation/ProjectTest.java @@ -30,6 +30,8 @@ import java.util.List; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; public class ProjectTest extends OperationTestBase { @@ -69,4 +71,36 @@ void testProjection(@TempDir Path tempDir) { } } } + + @Test + void testPreservesNullabilityEqualityAndRoundTrip(@TempDir Path tempDir) { + String datasetPath = tempDir.resolve("testAssertsNonNull").toString(); + try (RootAllocator allocator = new RootAllocator(Long.MAX_VALUE)) { + TestUtils.SimpleTestDataset testDataset = + new TestUtils.SimpleTestDataset(allocator, datasetPath); + dataset = testDataset.createEmptyDataset(); + Schema schema = testDataset.getSchema(); + + // The assertion is part of the operation's identity. + assertNotEquals( + Project.builder().schema(schema).preservesNullability(true).build(), + Project.builder().schema(schema).preservesNullability(false).build()); + assertEquals( + Project.builder().schema(schema).preservesNullability(true).build(), + Project.builder().schema(schema).preservesNullability(true).build()); + + // The explicit non-default assertion must survive the JNI round trip. + try (Transaction txn = + new Transaction.Builder() + .readVersion(dataset.version()) + .operation(Project.builder().schema(schema).preservesNullability(true).build()) + .build()) { + try (Dataset committed = new CommitBuilder(dataset).execute(txn)) { + Transaction readBack = committed.readTransaction().orElseThrow(); + Project project = (Project) readBack.operation(); + assertTrue(project.preservesNullability()); + } + } + } + } } diff --git a/java/src/test/java/org/lance/operation/UpdateTest.java b/java/src/test/java/org/lance/operation/UpdateTest.java index bb39a5f4d12..0e69707c972 100644 --- a/java/src/test/java/org/lance/operation/UpdateTest.java +++ b/java/src/test/java/org/lance/operation/UpdateTest.java @@ -36,10 +36,14 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; +import java.util.HashMap; import java.util.List; +import java.util.Map; import java.util.Optional; +import static org.junit.jupiter.api.Assertions.assertArrayEquals; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertInstanceOf; import static org.junit.jupiter.api.Assertions.assertThrows; public class UpdateTest extends OperationTestBase { @@ -104,6 +108,88 @@ void testUpdate(@TempDir Path tempDir) throws Exception { } } + @Test + void testUpdatedFragmentOffsetsRoundTrip(@TempDir Path tempDir) throws Exception { + String datasetPath = tempDir.resolve("testUpdatedFragmentOffsetsRoundTrip").toString(); + try (RootAllocator allocator = new RootAllocator(Long.MAX_VALUE)) { + TestUtils.SimpleTestDataset testDataset = + new TestUtils.SimpleTestDataset(allocator, datasetPath); + dataset = testDataset.createEmptyDataset(); + + // Append an initial fragment so we have a real fragment id. + FragmentMetadata fragmentMeta = testDataset.createNewFragment(10); + try (Transaction appendTxn = + new Transaction.Builder() + .readVersion(dataset.version()) + .operation( + Append.builder().fragments(Collections.singletonList(fragmentMeta)).build()) + .build()) { + new CommitBuilder(dataset).execute(appendTxn).close(); + } + + dataset = Dataset.open(datasetPath, allocator); + Fragment existingFragment = dataset.getFragments().get(0); + long fragmentId = existingFragment.getId(); + // Use the committed fragment's own metadata as the updatedFragment so that + // the updatedFragmentOffsets key is valid (must match a fragment in updatedFragments). + FragmentMetadata existingFragmentMeta = existingFragment.metadata(); + + // Build Update with non-empty updatedFragmentOffsets. Values are portable RoaringBitmap + // bytes encoding {1, 3, 5}: cookie(4) + containerCount(4) + key(2) + card-1(2) + + // offset(4) + elems(6). Offset = 16 (start of container data from beginning of stream). + Map offsets = new HashMap<>(); + offsets.put( + fragmentId, + new byte[] { + (byte) 0x3A, + (byte) 0x30, + (byte) 0x00, + (byte) 0x00, // cookie = 12346 + (byte) 0x01, + (byte) 0x00, + (byte) 0x00, + (byte) 0x00, // 1 container + (byte) 0x00, + (byte) 0x00, // container key 0 + (byte) 0x02, + (byte) 0x00, // cardinality - 1 = 2 + (byte) 0x10, + (byte) 0x00, + (byte) 0x00, + (byte) 0x00, // offset = 16 + (byte) 0x01, + (byte) 0x00, // element 1 + (byte) 0x03, + (byte) 0x00, // element 3 + (byte) 0x05, + (byte) 0x00 // element 5 + }); + + try (Transaction updateTxn = + new Transaction.Builder() + .readVersion(dataset.version()) + .operation( + Update.builder() + .updatedFragments(Collections.singletonList(existingFragmentMeta)) + .updateMode(Optional.of(UpdateMode.RewriteColumns)) + .updatedFragmentOffsets(offsets) + .build()) + .build()) { + try (Dataset committed = new CommitBuilder(dataset).execute(updateTxn)) { + // Read the committed transaction back (exercises the IntoJava JNI path). + try (Transaction readTx = committed.readTransaction().orElseThrow()) { + assertInstanceOf(Update.class, readTx.operation()); + Update readOp = (Update) readTx.operation(); + + Map readOffsets = readOp.updatedFragmentOffsets(); + assertEquals(1, readOffsets.size()); + assertArrayEquals(offsets.get(fragmentId), readOffsets.get(fragmentId)); + } + } + } + } + } + @Test void testUpdateColumns(@TempDir Path tempDir) throws Exception { String datasetPath = tempDir.resolve("testUpdateColumns").toString(); diff --git a/java/src/test/java/org/lance/otel/LanceMetricsTest.java b/java/src/test/java/org/lance/otel/LanceMetricsTest.java new file mode 100644 index 00000000000..b165bec3d36 --- /dev/null +++ b/java/src/test/java/org/lance/otel/LanceMetricsTest.java @@ -0,0 +1,172 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.lance.otel; + +import org.lance.Dataset; +import org.lance.TestUtils; + +import io.opentelemetry.api.common.AttributeKey; +import io.opentelemetry.api.common.Attributes; +import io.opentelemetry.sdk.metrics.SdkMeterProvider; +import io.opentelemetry.sdk.metrics.data.DoublePointData; +import io.opentelemetry.sdk.metrics.data.MetricData; +import io.opentelemetry.sdk.testing.exporter.InMemoryMetricReader; +import org.apache.arrow.memory.RootAllocator; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import java.nio.file.Path; +import java.util.Arrays; +import java.util.Collection; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.function.Function; +import java.util.stream.Collectors; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +public class LanceMetricsTest { + private static final String REQUESTS = "lance_object_store_requests_total"; + private static final String DURATION = "lance_object_store_request_duration_seconds"; + private static final String RETRYABLE = "lance_object_store_retryable_responses_total"; + private static final String IN_FLIGHT = "lance_object_store_in_flight_requests"; + + @AfterEach + void closeLanceMetrics() { + LanceMetrics.close(); + } + + @Test + void testInstrumentLanceMetricsExportsObjectStoreMetrics(@TempDir Path tempDir) { + InMemoryMetricReader reader = InMemoryMetricReader.create(); + SdkMeterProvider provider = SdkMeterProvider.builder().registerMetricReader(reader).build(); + + try { + assertTrue(LanceMetrics.instrument(provider)); + + Map catalog = + LanceMetrics.catalog().stream() + .collect(Collectors.toMap(MetricDescription::getName, Function.identity())); + assertTrue(catalog.containsKey(REQUESTS)); + assertEquals("histogram", catalog.get(DURATION).getKind()); + assertEquals("counter", catalog.get(RETRYABLE).getKind()); + assertEquals("gauge", catalog.get(IN_FLIGHT).getKind()); + + generateObjectStoreMetrics(tempDir.resolve("otel_metrics.lance")); + + MetricPoint requests = + LanceMetrics.snapshot().stream() + .filter(point -> REQUESTS.equals(point.getName())) + .findFirst() + .orElseThrow(() -> new AssertionError("expected object store request metric")); + assertNotNull(requests.getValue()); + assertTrue(requests.getValue() > 0); + assertTrue(requests.getAttributes().containsKey("operation")); + assertTrue(requests.getAttributes().containsKey("base")); + + Collection metrics = reader.collectAllMetrics(); + Map metricsByName = + metrics.stream().collect(Collectors.toMap(MetricData::getName, Function.identity())); + Set names = metricsByName.keySet(); + assertTrue(names.contains(REQUESTS)); + assertTrue(names.contains(DURATION + "_bucket")); + assertTrue(names.contains(DURATION + "_count")); + assertTrue(names.contains(DURATION + "_sum")); + + Collection bucketPoints = + metricsByName.get(DURATION + "_bucket").getDoubleSumData().getPoints(); + Collection countPoints = + metricsByName.get(DURATION + "_count").getDoubleSumData().getPoints(); + Collection sumPoints = + metricsByName.get(DURATION + "_sum").getDoubleSumData().getPoints(); + + assertFalse(bucketPoints.isEmpty()); + assertTrue( + bucketPoints.stream() + .allMatch(point -> point.getAttributes().get(AttributeKey.stringKey("le")) != null)); + + Map countsByAttributes = + countPoints.stream() + .collect(Collectors.toMap(DoublePointData::getAttributes, DoublePointData::getValue)); + Map infiniteBucketsByAttributes = new HashMap<>(); + for (DoublePointData bucketPoint : bucketPoints) { + if ("+Inf".equals(bucketPoint.getAttributes().get(AttributeKey.stringKey("le")))) { + Attributes attributes = + bucketPoint.getAttributes().toBuilder().remove(AttributeKey.stringKey("le")).build(); + infiniteBucketsByAttributes.put(attributes, bucketPoint.getValue()); + } + } + assertEquals(countsByAttributes, infiniteBucketsByAttributes); + assertTrue(sumPoints.stream().anyMatch(point -> point.getValue() > 0)); + } finally { + provider.close(); + } + } + + @Test + void testInstrumentReRegistersWithNewProvider(@TempDir Path tempDir) { + InMemoryMetricReader firstReader = InMemoryMetricReader.create(); + InMemoryMetricReader secondReader = InMemoryMetricReader.create(); + SdkMeterProvider firstProvider = + SdkMeterProvider.builder().registerMetricReader(firstReader).build(); + SdkMeterProvider secondProvider = + SdkMeterProvider.builder().registerMetricReader(secondReader).build(); + + try { + assertTrue(LanceMetrics.instrument(firstProvider)); + generateObjectStoreMetrics(tempDir.resolve("first_provider.lance")); + assertTrue(metricNames(firstReader).contains(REQUESTS)); + + assertTrue(LanceMetrics.instrument(secondProvider)); + assertTrue(metricNames(secondReader).contains(REQUESTS)); + assertFalse(metricNames(firstReader).contains(REQUESTS)); + } finally { + firstProvider.close(); + secondProvider.close(); + } + } + + @Test + void testSupportedMetricsSkipsUnknownKinds() { + MetricDescription counter = new MetricDescription("counter", "counter", null, "counter"); + MetricDescription unknown = new MetricDescription("unknown", "summary", null, "unknown"); + + List supported = + LanceMetrics.supportedMetrics(Arrays.asList(counter, unknown)); + + assertEquals(1, supported.size()); + assertEquals("counter", supported.get(0).getName()); + } + + private static void generateObjectStoreMetrics(Path datasetPath) { + try (RootAllocator allocator = new RootAllocator(Long.MAX_VALUE)) { + TestUtils.SimpleTestDataset testDataset = + new TestUtils.SimpleTestDataset(allocator, datasetPath.toString()); + try (Dataset dataset = testDataset.createEmptyDataset()) { + assertEquals(0, dataset.countRows()); + } + } + } + + private static Set metricNames(InMemoryMetricReader reader) { + Collection metrics = reader.collectAllMetrics(); + return metrics.stream().map(MetricData::getName).collect(Collectors.toSet()); + } +} diff --git a/protos/AGENTS.md b/protos/AGENTS.md index 23aef9fc196..2dba0e23dcb 100644 --- a/protos/AGENTS.md +++ b/protos/AGENTS.md @@ -4,7 +4,8 @@ Also see [root AGENTS.md](../AGENTS.md) for cross-language standards. ## Compatibility -- All changes must be backwards compatible. Never re-use or change field numbers of existing fields. +- Protobuf schemas that are part of a stable file format or any other stable persisted contract must remain backwards compatible. Never reuse or change their existing field numbers. +- Protobuf schemas used exclusively by an unstable file format follow the root file-format stability contract: do not preserve compatibility with prior unstable revisions. Before making a breaking protobuf change, verify that the schema is not shared with a stable format or another persisted contract. ## Schema Design diff --git a/protos/encodings_v2_1.proto b/protos/encodings_v2_1.proto index 46fd012fb58..51427332063 100644 --- a/protos/encodings_v2_1.proto +++ b/protos/encodings_v2_1.proto @@ -147,6 +147,128 @@ message FullZipLayout { repeated RepDefLayer layers = 8; } +// A layout used for sparse flat or nested pages where Arrow structure is represented directly +// in layer-local slot domains instead of as dense repetition / definition events. +// +// Structural layers are ordered from outer-most to inner-most. Values remain mini-block +// compressed and are split into independently readable chunks. +message SparseLayout { + // Description of the compression of values. + CompressiveEncoding value_compression = 1; + // Number of value buffers in each mini-block chunk. This does not include structural buffers. + uint64 num_buffers = 2; + // Number of entries in the equivalent dense repetition / definition stream. This equals + // num_visible_items plus one structural placeholder for every list slot without children. + // Null leaf slots count as visible items because they still occupy positions in Arrow's + // leaf value buffer. For example, a nullable primitive with 100 slots, 30 of them null, + // has num_items = num_visible_items = 100. + uint64 num_items = 3; + // Number of leaf value slots encoded in the value chunks, including null leaf slots. + uint64 num_visible_items = 4; + // If true, chunk-local value buffer sizes use u32. Otherwise they use u16. + bool has_large_chunk = 5; + // Structural layers ordered from outer-most to inner-most. This may be empty for a flat, + // non-nullable leaf page whose scheduling domain equals num_visible_items. + repeated SparseStructuralLayer structural_layers = 6; +} + +// A domain is a layer-local integer coordinate space [0, num_slots). A slot is one +// element in that space. The outer-most domain is the page's top-level rows; each +// layer's child domain is the next layer's parent domain, and the terminal child +// domain contains num_visible_items leaf value slots. +message SparseStructuralLayer { + // Exactly one layer kind is required. + oneof layer { + SparseValidityLayer validity = 1; + SparseListLayer list = 2; + SparseFixedSizeListLayer fixed_size_list = 3; + } +} + +message SparseValidityLayer { + // Number of nullable item or struct slots in this layer's parent and child domain. + uint64 num_slots = 1; + // Validity for the slots in this layer. + SparseValiditySet validity = 2; +} + +message SparseListLayer { + // Number of list, large-list, or map slots in this layer's parent domain. + uint64 num_slots = 1; + // Number of slots in this layer's child domain. + uint64 num_child_slots = 2; + // Non-empty parent slots. Valid parent slots absent from this set are empty lists. + SparsePositionSet non_empty_positions = 3; + // Positive child counts corresponding one-for-one with non_empty_positions. + SparseCountSet counts = 4; + // Validity for the parent slots in this layer. + SparseValiditySet validity = 5; +} + +message SparseFixedSizeListLayer { + // Number of fixed-size-list slots in this layer's parent domain. + uint64 num_slots = 1; + // Number of children per parent slot. The child domain has num_slots * dimension slots. + uint64 dimension = 2; + // Validity for the parent slots in this layer. + SparseValiditySet validity = 3; +} + +message SparseValiditySet { + enum Meaning { + SPARSE_VALIDITY_UNSPECIFIED = 0; + // Stored positions are null; all other positions are valid. + SPARSE_VALIDITY_NULL_POSITIONS = 1; + // Stored positions are valid; all other positions are null. + SPARSE_VALIDITY_VALID_POSITIONS = 2; + } + + Meaning meaning = 1; + SparsePositionSet positions = 2; +} + +message SparsePositionEmpty {} + +message SparsePositionAll {} + +message SparsePositionRange { + uint64 start = 1; + uint64 length = 2; +} + +message SparsePositionSet { + oneof positions { + // Delta-compressed u64 positions. Cardinality is num_positions. + CompressiveEncoding explicit = 1; + // One contiguous, non-empty range. + SparsePositionRange range = 2; + // Every position in the domain. + SparsePositionAll all = 3; + // No positions in the domain. + SparsePositionEmpty empty = 4; + } + // Semantic cardinality of this set. + uint64 num_positions = 5; +} + +message SparseCountEmpty {} + +message SparseCountConstant { + // Child count shared by every non-empty list slot. + uint64 value = 1; +} + +message SparseCountSet { + oneof counts { + // Compressed u64 child counts. Cardinality comes from the containing position set. + CompressiveEncoding explicit = 1; + // One positive child count shared by every non-empty list slot. + SparseCountConstant constant = 2; + // No counts; valid only when there are no non-empty list slots. + SparseCountEmpty empty = 3; + } +} + // A layout used for pages where all (visible) values are the same scalar value. // // This generalizes the prior AllNullLayout semantics for file_version >= 2.2. @@ -206,6 +328,8 @@ message PageLayout { // A layout where large binary data is encoded externally // and only the descriptions are put in the page BlobLayout blob_layout = 4; + // A sparse structural layout. This variant requires file version 2.3 or later. + SparseLayout sparse_layout = 5; } } diff --git a/protos/file2.proto b/protos/file2.proto index da0b1d5e96c..650a1568da6 100644 --- a/protos/file2.proto +++ b/protos/file2.proto @@ -207,4 +207,4 @@ message ColumnMetadata { // // This file format is extremely minimal. It is a building block for // creating more useful readers and writers and not terribly useful by itself. -// Other protobuf files will describe how this can be extended. \ No newline at end of file +// Other protobuf files will describe how this can be extended. diff --git a/protos/index_old.proto b/protos/index_old.proto index 601aa2681da..236d1f110c0 100644 --- a/protos/index_old.proto +++ b/protos/index_old.proto @@ -24,8 +24,55 @@ message BTreeIndexDetails {} message BitmapIndexDetails {} message LabelListIndexDetails {} message NGramIndexDetails {} -message ZoneMapIndexDetails {} +message ZoneMapIndexDetails { + // Number of rows per zone. Optional for backwards compatibility: absent on + // datasets written before this field was added. When absent, no seed writer + // is created for the index. + optional uint64 rows_per_zone = 1; + // Whether seed-based incremental updates are enabled for this index. + // On-disk semantics: absent means seeds are disabled (old datasets written + // before this field was added). Present false means explicitly disabled. + // Present true means seeds are enabled: the index will embed per-fragment + // seed buffers in data files and harvest them during incremental updates + // to skip full column scans. + // Creation-time default: index creation code sets this to true for + // variable-length types (strings, binary) and fixed-width types wider than + // 8 bytes, and to false for narrow fixed-width types (e.g. Int64, Float64). + optional bool use_seeds = 2; + // Whether this index tracks exact null row addresses in a separate bitmap. + // Absent or false means legacy format: null positions are not tracked and + // IS NULL searches fall back to approximate zone-level statistics. Present + // true means IS NULL is exact and IS NOT NULL can be answered without a + // full scan. + optional bool has_null_bitmap = 3; +} message InvertedIndexDetails { + enum DocumentGranularity { + ROW = 0; + LIST_ELEMENT = 1; + } + + message CodeTokenizerConfig { + // Split one lexical identifier into subwords, e.g. getUserName -> + // get/user/name. + bool split_identifiers = 1; + // Split identifier subwords across letter/number boundaries, e.g. + // HTML2JSON -> html/2/json. An absent value uses the code tokenizer default; + // a present value records the explicit index-time choice. + optional bool split_on_numerics = 2; + // Keep the complete lexical identifier in addition to subwords, e.g. + // user_name plus user/name. An absent value uses the code tokenizer default; + // a present value records the explicit index-time choice. + optional bool preserve_original = 3; + // Index operator tokens such as "::", "->", and "!=". Operators are not + // indexed by default because they are often high-frequency noise. + bool index_operators = 4; + } + + // Lexical tokenizer used after document-level text extraction. This is an + // implementation component such as "simple", "icu", "ngram", or "code". + // Input-time analyzer profiles are expanded into this field and the concrete + // options below before these details are persisted. // Marking this field as optional as old versions of the index store blank details and we // need to make sure we have a proper optional field to detect this. optional string base_tokenizer = 1; @@ -39,4 +86,19 @@ message InvertedIndexDetails { uint32 min_ngram_length = 9; uint32 max_ngram_length = 10; bool prefix_only = 11; + // Number of documents per compressed posting block. An absent value means + // the index predates this field and must use the legacy block size of 128. + // A present value records the block size used by the index; 256 is valid + // with format versions 3 and 4. + optional uint32 block_size = 12; + // Options for base_tokenizer = "code". Presence records the code tokenizer + // configuration used to build the index; absence means there is no + // code-specific configuration to apply. + CodeTokenizerConfig code_config = 13; + // The logical FTS document boundary. The protobuf default preserves the + // legacy row-document behavior when this field is absent. + DocumentGranularity document_granularity = 14; + // The posting-list payload format. This is separate from index_version, + // which identifies the overall inverted-index layout. + optional uint32 posting_format_version = 15; } diff --git a/protos/table.proto b/protos/table.proto index 8d0cb249fda..0f8a17d0ff6 100644 --- a/protos/table.proto +++ b/protos/table.proto @@ -115,6 +115,16 @@ message Manifest { // * 1 << 3: table config is present // * 1 << 4: dataset uses multiple base paths // * 1 << 5: transaction file writes are disabled + // * 1 << 6: data overlay files are present (see DataOverlayFile). Readers that do + // not understand overlays must refuse the dataset, since ignoring an overlay + // would silently return stale base values. + // * 1 << 7: index_catchup is maintained on this table, so an index absent from + // it is *not* caught up rather than fully caught up (see MemWalIndexDetails). + // Readers that do not understand it must refuse the dataset, since reading + // absence as caught up could answer from an index missing rows that only the + // MemWAL SSTables still hold. Writers must refuse it too: one that does not + // maintain index_catchup can change an index without withdrawing the + // position recorded for it. Setting it is one-way. uint64 reader_feature_flags = 9; // Feature flags for writers. @@ -313,6 +323,15 @@ message DataFragment { repeated DataFile files = 2; + // Optional overlay files for this fragment, which supply new values for a + // subset of cells without rewriting the base data files. This MUST be empty + // if the data overlay files feature flag (64) is not set in the manifest. + // + // Order is significant: a later entry is newer than an earlier one. When two + // overlays cover the same (offset, field) and share a `committed_version`, the + // later entry wins. See DataOverlayFile for the full resolution rules. + repeated DataOverlayFile overlays = 11; + // File that indicates which rows, if any, should be considered deleted. DeletionFile deletion_file = 3; @@ -435,6 +454,66 @@ message DataFile { optional uint32 base_id = 7; } // DataFile +// An overlay file supplies new values for a subset of (row offset, field) cells +// within a fragment, without rewriting the fragment's base data files. It is +// used for efficient updates when only a small fraction of rows and/or columns +// change. +// +// On read, a cell is resolved by consulting the fragment's overlays from newest +// to oldest: the first overlay that covers that (offset, field) wins; if none +// cover it, the value falls through to the base data file. Because deletions +// take precedence over overlays, an overlay value for an offset that is also +// marked deleted is dead and is ignored. +// +// The overlay's data file does NOT store a row-offset key column. Within a value +// column, the position of a covered offset's value is the rank (0-based count of +// set bits below it) of that offset within the field's coverage bitmap. Because +// fields may cover different offset sets, the value columns of a single overlay +// data file may have different lengths (which the Lance file format permits). +message DataOverlayFile { + // The data file storing the overlay's new cell values, one value column per + // field in `data_file.fields`. No row-offset key column is stored. + DataFile data_file = 1; + + // Which (offset, field) cells this overlay provides values for. + oneof coverage { + // A single 32-bit Roaring bitmap of physical row offsets that applies to + // every field in `data_file.fields` (a "dense" / rectangular overlay). + // Every covered offset has a value for every field. This is the common case + // for a plain UPDATE, where one SET list is applied to one set of rows. + bytes shared_offset_bitmap = 2; + // Per-field coverage for a "sparse" overlay, used when different fields cover + // different offset sets (e.g. a MERGE with multiple WHEN MATCHED branches). + FieldCoverage field_coverage = 4; + } + + // The dataset version at which this overlay became effective: the version of + // the commit that introduced it, NOT the version it was read from. It is + // stamped at commit time and re-stamped if the commit is retried, in the same + // way as the created-at / last-updated-at version sequences. + // + // This drives two orderings: + // * Versus index builds: an index whose `dataset_version` >= this value + // already incorporates this overlay. Otherwise the overlay's covered cells + // are excluded from index results for the affected fields and re-evaluated + // against their current values (see the Data Overlay Files specification). + // * Versus other overlays: when two overlays cover the same (offset, field), + // the one with the higher `committed_version` wins. Overlays that share a + // `committed_version` are ordered by their position in + // `DataFragment.overlays`, where a later entry is newer and wins. + uint64 committed_version = 3; +} + +// Per-field coverage for a sparse overlay. +message FieldCoverage { + // One entry per field in the overlay's `data_file.fields`, in the same order. + // Each is a 32-bit Roaring bitmap of the physical row offsets covered for that + // field. An offset present in a field's bitmap but mapped to a NULL value + // means the cell is overridden to NULL (distinct from an offset that is absent, + // which falls through to the base data file). + repeated bytes offset_bitmaps = 1; +} + // Deletion File // // The path of the deletion file is constructed as: @@ -570,14 +649,14 @@ message ShardManifest { // files to find actual state. uint64 wal_entry_position_last_seen = 4; - // Next generation ID to create (incremented after each MemTable flush). + // Generation to assign to the next SSTable (incremented after each MemTable flush). uint64 current_generation = 6; - // Field 7 removed: merged_generation moved to MemWalIndexDetails.merged_generations - // which is the authoritative source for merge progress. + // Field 7 removed: compaction progress lives in + // MemWalIndexDetails.compacted_sstables. - // List of flushed MemTable generations and their directory paths. - repeated FlushedGeneration flushed_generations = 8; + // List of SSTables created by flushing MemTables and their directory paths. + repeated SsTable sstables = 8; // Lifecycle status. Default ACTIVE; SEALED marks an in-flight drop // (drop-table 2PC). A SEALED manifest refuses claims at claim_epoch. @@ -594,40 +673,44 @@ message ShardFieldEntry { bytes value = 2; } -// A flushed MemTable generation and its storage location. -message FlushedGeneration { - // Generation number. +// An SSTable: the immutable result of flushing a MemTable, stored as a Lance dataset. +message SsTable { + // Generation number identifying this SSTable. uint64 generation = 1; // Directory name relative to the shard directory. string path = 2; } -// A shard's merged generation, used in MemWalIndexDetails. -message MergedGeneration { +// A pointer to the latest SSTable compacted for a shard. +message CompactedSsTable { // Shard identifier (UUID v4). UUID shard_id = 1; - // Last generation merged to base table for this shard. + // Generation of the latest SSTable compacted into the base table for this shard. uint64 generation = 2; } -// Tracks which merged generation a base table index has been rebuilt to cover. -// Used to determine whether to read from flushed MemTable indexes or base table. +// Tracks which compacted SSTable generation a base table index has been rebuilt to cover. +// Used to determine whether to read from SSTable indexes or base table. message IndexCatchupProgress { // Name of the base table index (must match an entry in maintained_indexes). string index_name = 1; // Per-shard progress: the generation up to which this index covers. - // If a shard is not present, the index is assumed to be fully caught up - // (i.e., caught_up_generation >= merged_generation for that shard). - repeated MergedGeneration caught_up_generations = 2; + // + // An absent shard means "fully caught up" ONLY on a legacy table. On a table + // with the MemWAL index-catchup feature bit set it means *unknown*: this + // index has recorded no catch-up for that shard, so its SSTables must be retained + // and a repair scheduled. The manifest feature bit, not this message, selects + // which reading applies. + repeated CompactedSsTable caught_up_generations = 2; } // Index details for MemWAL Index, stored in IndexMetadata.index_details. // This is the centralized structure for all MemWAL metadata: // - Configuration (sharding specs, indexes to maintain) -// - Merge progress (merged generations per shard) +// - SSTable compaction progress // - Shard state snapshots // // Writers read this index to get configuration before writing. @@ -669,24 +752,29 @@ message MemWalIndexDetails { // SQ params) from the base table index to ensure distance comparability. repeated string maintained_indexes = 8; - // Last generation merged to base table for each shard. + // Latest SSTable compacted into the base table for each shard. // This is updated atomically with merge-insert data commits, enabling - // conflict resolution when multiple mergers operate concurrently. + // conflict resolution when multiple compactors operate concurrently. // // Note: This is separate from shard snapshots because: - // 1. merged_generations is updated by mergers (atomic with data commit) + // 1. compacted_sstables is updated by compactors (atomic with data commit) // 2. shard snapshots are updated by background index builder - repeated MergedGeneration merged_generations = 9; + repeated CompactedSsTable compacted_sstables = 9; // Per-index catchup progress tracking. - // When data is merged to the base table, base table indexes are rebuilt + // When data is compacted into the base table, base table indexes are rebuilt // asynchronously. This field tracks which generation each index covers. // - // For indexed queries, if an index's caught_up_generation < merged_generation, - // readers should use flushed MemTable indexes for the gap instead of + // For indexed queries, if an index's caught_up_generation < compacted_generation, + // readers should use SSTable indexes for the gap instead of // scanning unindexed data in the base table. // - // If an index is not present in this list, it is assumed to be fully caught up. + // An index absent from this list is "fully caught up" ONLY on a legacy table. + // With the MemWAL index-catchup feature bit set, absence means the index has + // recorded no catch-up, so the SSTables it would need stay live until a repair + // records it. Only the dedicated WAL index-repair path may add entries here; + // ordinary index operations have their entry removed automatically when they + // change an index, since they do not report what the new index covers. repeated IndexCatchupProgress index_catchup = 10; // Default ShardWriter configuration values for this MemWAL index. diff --git a/protos/transaction.proto b/protos/transaction.proto index e72e95025a4..bbde96fed5d 100644 --- a/protos/transaction.proto +++ b/protos/transaction.proto @@ -144,12 +144,24 @@ message Transaction { repeated lance.file.Field schema = 2; // Schema metadata. map schema_metadata = 3; + // Set when this merge makes no nullability-affecting schema change: it + // introduces no field that data staged against an earlier schema could + // not safely omit. Without the assertion (including transactions written + // before this field existed) the merge conservatively conflicts with + // concurrent value-writes, which can only cause a retry. + bool preserves_nullability = 4; } // An operation that projects a subset of columns, altering the schema. message Project { // The new schema repeated lance.file.Field schema = 1; + // Set when this projection makes no nullability-affecting schema change, + // as a rename or a drop does not. Without the assertion (including + // transactions written before this field existed) the projection + // conservatively conflicts with concurrent value-writes, which can only + // cause a retry. A nullability tightening must not set this. + bool preserves_nullability = 2; } // An operation that restores a dataset to a previous version. @@ -243,8 +255,8 @@ message Transaction { repeated DataFragment new_fragments = 3; // The ids of the fields that have been modified. repeated uint32 fields_modified = 4; - /// List of MemWAL shard generations to mark as merged after this transaction - repeated MergedGeneration merged_generations = 5; + /// SSTables to mark as compacted after this transaction. + repeated CompactedSsTable compacted_sstables = 5; /// The fields that used to judge whether to preserve the new frag's id into /// the frag bitmap of the specified indices. repeated uint32 fields_for_preserving_frag_bitmap = 6; @@ -254,7 +266,12 @@ message Transaction { // Only tracks keys from INSERT operations during merge insert, not updates. optional KeyExistenceFilter inserted_rows = 8; // Per-fragment physical row offsets that matched an update_columns hash join (RewriteColumns). + // Deprecated: use updated_fragment_offset_bitmaps (field 10) instead. map updated_fragment_offsets = 9; + // Per-fragment matched offsets as portable RoaringBitmap bytes (replaces field 9). + // Writers emit field 10 only. Readers prefer field 10; fall back to field 9 for + // manifests written before this change. + map updated_fragment_offset_bitmaps = 10; } // The mode of update operation @@ -315,12 +332,42 @@ message Transaction { repeated DataReplacementGroup replacements = 1; } - // Update the merged generations in MemWAL index. + // Overlay files to append to a single fragment, in order (the last entry is + // newest). The overlays are appended to the fragment's existing `overlays` + // list; they do not replace it, so overlays written by concurrent commits are + // preserved. + message DataOverlayGroup { + uint64 fragment_id = 1; + // Each DataOverlayFile.committed_version is left 0 by the writer and stamped + // to the new dataset version at commit time (re-stamped on retry), in the + // same way as the created-at / last-updated-at version sequences. The fields + // touched are read from each overlay's `data_file.fields`. + repeated DataOverlayFile overlays = 2; + } + + // Attach overlay files to fragments, supplying new values for a subset of + // (row offset, field) cells without rewriting the fragments' base data files. + // See the DataOverlayFile message in table.proto for resolution, coverage, and + // versioning rules, and the Data Overlay Files and Transactions specifications + // for the (intentionally permissive) conflict semantics. + message DataOverlay { + repeated DataOverlayGroup groups = 1; + } + + // Update SSTable compaction progress in the MemWAL index. // This operation is used during merge-insert to atomically record which - // generations have been merged to the base table. + // SSTables have been compacted into the base table. message UpdateMemWalState { - // Shards and generations being marked as merged. - repeated MergedGeneration merged_generations = 1; + // SSTables being marked as compacted. + repeated CompactedSsTable compacted_sstables = 1; + + // Presence with true requests the one-way migration to required catch-up. + // False is invalid; absence is an ordinary progress update. + // + // The migration is one-way because returning to legacy semantics — where a + // missing coverage entry reads as "fully caught up" — is unsafe once any + // SSTable has been retired against a recorded catch-up position. + optional bool require_index_catchup = 2; } // An operation that updates base paths in the dataset. @@ -346,6 +393,7 @@ message Transaction { UpdateMemWalState update_mem_wal_state = 112; Clone clone = 113; UpdateBases update_bases = 114; + DataOverlay data_overlay = 115; } // Fields 200/202 (`blob_append` / `blob_overwrite`) previously represented blob dataset ops. diff --git a/python/.cargo/config.toml b/python/.cargo/config.toml index f9f9bc0544a..96837e35cc8 100644 --- a/python/.cargo/config.toml +++ b/python/.cargo/config.toml @@ -22,7 +22,9 @@ rustflags = [ ] [target.x86_64-unknown-linux-gnu] -rustflags = ["-C", "target-cpu=haswell", "-C", "target-feature=+avx2,+fma,+f16c"] +# Published extensions must load before runtime SIMD detection selects an +# optimized kernel, including on x86_64 hosts without AVX or AVX2. +rustflags = ["-C", "target-cpu=x86-64-v2"] [target.aarch64-apple-darwin] rustflags = ["-C", "target-cpu=apple-m1", "-C", "target-feature=+neon,+fp16"] diff --git a/python/AGENTS.md b/python/AGENTS.md index 2025f4ef9df..40301a4ddc7 100644 --- a/python/AGENTS.md +++ b/python/AGENTS.md @@ -7,12 +7,10 @@ Also see [root AGENTS.md](../AGENTS.md) for cross-language standards. * Environment: use `uv` for all local Python environment setup in this repository. * First step in every new worktree or fresh checkout: run `make install` from `python/` before any Python command. This runs `uv sync` (dev and test dependencies are included by default via `[tool.uv] default-groups`) and sets up pre-commit hooks. Add `--group benchmarks`, `--extra torch`, or `--extra geo` only when needed. * `uv sync` builds the local `pylance` Rust extension as part of environment setup. This can take a long time. Start it early, let it finish, and do not interrupt it or switch to a different setup path just because the build is slow. -* After the initial `make install`, use `uv run make test` to run tests. * Only run `uv sync` again when dependencies change (e.g., after pulling new commits that update `pyproject.toml` or `uv.lock`). * Command execution: always use `uv run ...` for Python-related repository commands. Do not rely on a globally activated environment. * Never invoke bare `python`, `pytest`, `pip`, `maturin`, `make test`, `make doctest`, `make lint`, or `make format` for repository work. * If a Python command fails outside `uv run`, that does not count as a dependency or test failure. Fix the environment usage first and rerun correctly. -* Build time expectations: `make install` and `make build` build the local `pylance` Rust extension as part of the environment workflow. This can be slow, especially on the first run or after Rust dependency changes; treat that as expected and do not switch to a different environment manager or shortcut around the build just because it takes time. * Build: `make build` (required after Rust changes) * Test: `uv run make test` * Run single test: `uv run pytest python/tests/.py::` @@ -39,11 +37,4 @@ Also see [root AGENTS.md](../AGENTS.md) for cross-language standards. ## Testing -- Use `@pytest.mark.parametrize` for tests that differ only in inputs — extract shared setup into helpers. - Add tests to existing `test_{module}.py` files rather than creating new test files for the same module. -- Replace `print()` in tests with `assert` statements. - -## Common Failure Mode - -- A missing module or missing command error from bare `python`, `pytest`, `pip`, `maturin`, or `make` is usually an environment usage mistake, not a repository issue. -- Before reporting a Python dependency as unavailable, verify that `uv sync` has been run in the current worktree and that the failing command was executed with `uv run ...`. diff --git a/python/Cargo.lock b/python/Cargo.lock index 440538da1ee..d9950ae10aa 100644 --- a/python/Cargo.lock +++ b/python/Cargo.lock @@ -2,54 +2,6 @@ # It is not intended for manual editing. version = 4 -[[package]] -name = "abi_stable" -version = "0.11.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "69d6512d3eb05ffe5004c59c206de7f99c34951504056ce23fc953842f12c445" -dependencies = [ - "abi_stable_derive", - "abi_stable_shared", - "const_panic", - "core_extensions", - "crossbeam-channel", - "generational-arena", - "libloading", - "lock_api", - "parking_lot", - "paste", - "repr_offset", - "rustc_version", - "serde", - "serde_derive", - "serde_json", -] - -[[package]] -name = "abi_stable_derive" -version = "0.11.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d7178468b407a4ee10e881bc7a328a65e739f0863615cca4429d43916b05e898" -dependencies = [ - "abi_stable_shared", - "as_derive_utils", - "core_extensions", - "proc-macro2", - "quote", - "rustc_version", - "syn 1.0.109", - "typed-arena", -] - -[[package]] -name = "abi_stable_shared" -version = "0.11.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b2b5df7688c123e63f4d4d649cba63f2967ba7f7861b1664fca3f77d3dad2b63" -dependencies = [ - "core_extensions", -] - [[package]] name = "adler2" version = "2.0.1" @@ -172,9 +124,9 @@ dependencies = [ [[package]] name = "anyhow" -version = "1.0.102" +version = "1.0.104" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f202df86484c868dbad7eaa557ef785d5c66295e41b460ef922eca0723b842c" +checksum = "330a5ed07fa54e4702c9d6c4174f74427fc0ef6e214bbd677ae50a5099946470" [[package]] name = "approx" @@ -208,9 +160,9 @@ checksum = "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50" [[package]] name = "arrow" -version = "58.3.0" +version = "58.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "378530e55cd479eda3c14eb345310799717e6f76d0c332041e8487022166b471" +checksum = "6cfdd0833e32a9874d2b55089333ad310c0be208aafa277385ce2461dec90be3" dependencies = [ "arrow-arith", "arrow-array", @@ -230,9 +182,9 @@ dependencies = [ [[package]] name = "arrow-arith" -version = "58.3.0" +version = "58.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a0ab212d2c1886e802f51c5212d78ebbcbb0bec980fff9dadc1eb8d45cd0b738" +checksum = "0a41203398f0eaa6f7ec8e62c0da742a21abf282c148fc157f6c35c90e29981a" dependencies = [ "arrow-array", "arrow-buffer", @@ -244,9 +196,9 @@ dependencies = [ [[package]] name = "arrow-array" -version = "58.3.0" +version = "58.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cfd33d3e92f207444098c75b42de99d329562be0cf686b307b097cc52b4e999e" +checksum = "ae33dad492b7df00a217563a7b0ef2874df68a0deea1b1a3acf628152f7f7a69" dependencies = [ "ahash", "arrow-buffer", @@ -263,9 +215,9 @@ dependencies = [ [[package]] name = "arrow-buffer" -version = "58.3.0" +version = "58.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c6cd424c2693bcdbc150d843dc9d4d137dd2de4782ce6df491ad11a3a0416c0" +checksum = "b9552f96391c005e6ab449fa941420935e7e062489b12b8b1b08879b2163f5b5" dependencies = [ "bytes", "half", @@ -275,9 +227,9 @@ dependencies = [ [[package]] name = "arrow-cast" -version = "58.3.0" +version = "58.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c5aefb56a2c02e9e2b30746241058b85f8983f0fcff2ba0c6d09006e1cded7f" +checksum = "3a8a327c9649f30d8406995f27642b68df354713cca3baaaf100f076f18d5f34" dependencies = [ "arrow-array", "arrow-buffer", @@ -286,7 +238,7 @@ dependencies = [ "arrow-schema", "arrow-select", "atoi", - "base64", + "base64 0.22.1", "chrono", "comfy-table", "half", @@ -297,9 +249,9 @@ dependencies = [ [[package]] name = "arrow-csv" -version = "58.3.0" +version = "58.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e94e8cf7e517657a52b91ea1263acf38c4ca62a84655d72458a3359b12ab97de" +checksum = "af0dd6d90d1955e9f9a014c1e563ee8aeffc21909085d25623e1da44d96eca26" dependencies = [ "arrow-array", "arrow-cast", @@ -312,9 +264,9 @@ dependencies = [ [[package]] name = "arrow-data" -version = "58.3.0" +version = "58.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3c88210023a2bfee1896af366309a3028fc3bcbd6515fa29a7990ee1baa08ee0" +checksum = "2b24852db04738907e06c04ea61e42fe7fda962a34513022dc0d0e754fb7976b" dependencies = [ "arrow-buffer", "arrow-schema", @@ -325,9 +277,9 @@ dependencies = [ [[package]] name = "arrow-ipc" -version = "58.3.0" +version = "58.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "238438f0834483703d88896db6fe5a7138b2230debc31b34c0336c2996e3c64f" +checksum = "29a908a11fcfb3fb2f6730f4ac15e367bc644e419155e96238f68cf3adde572b" dependencies = [ "arrow-array", "arrow-buffer", @@ -341,9 +293,9 @@ dependencies = [ [[package]] name = "arrow-json" -version = "58.3.0" +version = "58.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "205ca2119e6d679d5c133c6f30e68f027738d95ed948cf77677ea69c7800036b" +checksum = "b8a96aed3931c076adee39ec2a40d8219fc7f09e79bcdaca1df16272993e1e14" dependencies = [ "arrow-array", "arrow-buffer", @@ -366,9 +318,9 @@ dependencies = [ [[package]] name = "arrow-ord" -version = "58.3.0" +version = "58.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1bffd8fd2579286a5d63bac898159873e5094a79009940bcb42bbfce4f19f1d0" +checksum = "63a083ec750f5c043f02946b4baf05fcdbb55f4560a3277055caca5cc99f3eb0" dependencies = [ "arrow-array", "arrow-buffer", @@ -379,9 +331,9 @@ dependencies = [ [[package]] name = "arrow-pyarrow" -version = "58.3.0" +version = "58.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d29abdf672a81c1aeb57fd2661457f9918964d49aed0e9f18932535f2a9e49ce" +checksum = "3ffb9be5a873590f825aef50df20e0f8dff5fd42a77058e28bbe7bd44bb53dec" dependencies = [ "arrow-array", "arrow-data", @@ -391,9 +343,9 @@ dependencies = [ [[package]] name = "arrow-row" -version = "58.3.0" +version = "58.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bab5994731204603c73ba69267616c50f80780774c6bb0476f1f830625115e0c" +checksum = "514ba0ef0d4c5896202dae736251ce415abb43a950bed570fb7981b8716c0e4c" dependencies = [ "arrow-array", "arrow-buffer", @@ -404,9 +356,9 @@ dependencies = [ [[package]] name = "arrow-schema" -version = "58.3.0" +version = "58.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f633dbfdf39c039ada1bf9e34c694816eb71fbb7dc78f613993b7245e078a1ed" +checksum = "21ca356ad6425cecb6eb7b28e4f659f1ee7880fbb1a16127de7dd62901efee9e" dependencies = [ "bitflags 2.13.0", "serde_core", @@ -415,9 +367,9 @@ dependencies = [ [[package]] name = "arrow-select" -version = "58.3.0" +version = "58.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8cd065c54172ac787cf3f2f8d4107e0d3fdc26edba76fdf4f4cc170258942222" +checksum = "c58da39eb3d8350ad4a549e5c2bc49284dac554016c69829310350f1731b0aad" dependencies = [ "ahash", "arrow-array", @@ -429,9 +381,9 @@ dependencies = [ [[package]] name = "arrow-string" -version = "58.3.0" +version = "58.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "29dd7cda3ab9692f43a2e4acc444d760cc17b12bb6d8232ddf64e9bab7c06b42" +checksum = "b6789b388467525e3271326b6b4915666ecfdf5142aef09779445c954b67543c" dependencies = [ "arrow-array", "arrow-buffer", @@ -444,18 +396,6 @@ dependencies = [ "regex-syntax", ] -[[package]] -name = "as_derive_utils" -version = "0.11.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ff3c96645900a44cf11941c111bd08a6573b0e2f9f69bc9264b179d8fae753c4" -dependencies = [ - "core_extensions", - "proc-macro2", - "quote", - "syn 1.0.109", -] - [[package]] name = "async-channel" version = "2.5.0" @@ -485,9 +425,6 @@ name = "async-ffi" version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f4de21c0feef7e5a556e51af767c953f0501f7f300ba785cc99c47bdc8081a50" -dependencies = [ - "abi_stable", -] [[package]] name = "async-lock" @@ -513,13 +450,13 @@ dependencies = [ [[package]] name = "async-trait" -version = "0.1.89" +version = "0.1.91" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9035ad2d096bed7955a320ee7e2230574d28fd3c3a0f186cbea1ff3c7eed5dbb" +checksum = "ae36dc4177970ef04fde5178d3e2429882def40e57a451f919c098f72baa6cec" dependencies = [ "proc-macro2", "quote", - "syn 2.0.118", + "syn 3.0.2", ] [[package]] @@ -1007,6 +944,12 @@ version = "0.22.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" +[[package]] +name = "base64" +version = "0.23.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b25655df2c3cdd83c5e5b293b88acd880332b2ddadd7c30ac43144fdc0033da9" + [[package]] name = "base64-simd" version = "0.8.0" @@ -1188,9 +1131,23 @@ checksum = "175812e0be2bccb6abe50bb8d566126198344f707e304f45c648fd8f2cc0365e" [[package]] name = "bytemuck" -version = "1.25.0" +version = "1.25.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c8efb64bd706a16a1bdde310ae86b351e4d21550d98d056f22f8a7f7a2183fec" +checksum = "95832e849adfb21180ccb6826a99da14e5d266ae5c2e668e1602cf234f153797" +dependencies = [ + "bytemuck_derive", +] + +[[package]] +name = "bytemuck_derive" +version = "1.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9abbd1bc6865053c427f7198e6af43bfdedc55ab791faed4fbd361d789575ff" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.118", +] [[package]] name = "byteorder" @@ -1200,9 +1157,9 @@ checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" [[package]] name = "bytes" -version = "1.12.0" +version = "1.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ae3f5d315924270530207e2a68396c3cc547f6dca3fbdca317cfb1a51edb593" +checksum = "fc652a48c352aef3ea3aed32080501cf3ef6ed5da78602a020c991775b0aff04" [[package]] name = "bytes-utils" @@ -1225,9 +1182,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.2.65" +version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e228eec9be7c17ccb640b59b36a5cd805ea2a564a4c5e162c2f659fea30d3b96" +checksum = "c89588d05638b5b4594a3348a2d6c20277e43a7f5c5202b05cc56888475a47b8" dependencies = [ "find-msvc-tools", "jobserver", @@ -1309,9 +1266,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.6.1" +version = "4.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ddb117e43bbf7dacf0a4190fef4d345b9bad68dfc649cb349e7d17d28428e51" +checksum = "d91e0c145792ef73a6ad36d27c75ac09f1832222a3c209689d90f534685ee5b7" dependencies = [ "clap_builder", "clap_derive", @@ -1319,9 +1276,9 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.6.0" +version = "4.6.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "714a53001bf66416adb0e2ef5ac857140e7dc3a0c48fb28b2f10762fc4b5069f" +checksum = "f09628afdcc538b57f3c6341e9c8e9970f18e4a481690a64974d7023bd33548b" dependencies = [ "anstream", "anstyle", @@ -1331,14 +1288,14 @@ dependencies = [ [[package]] name = "clap_derive" -version = "4.6.1" +version = "4.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f2ce8604710f6733aa641a2b3731eaa1e8b3d9973d5e3565da11800813f997a9" +checksum = "d012d2b9d65aca7f18f4d9878a045bc17899bba951561ba5ec3c2ba1eed9a061" dependencies = [ "heck", "proc-macro2", "quote", - "syn 2.0.118", + "syn 3.0.2", ] [[package]] @@ -1502,21 +1459,6 @@ version = "0.8.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" -[[package]] -name = "core_extensions" -version = "1.5.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "42bb5e5d0269fd4f739ea6cedaf29c16d81c27a7ce7582008e90eb50dcd57003" -dependencies = [ - "core_extensions_proc_macros", -] - -[[package]] -name = "core_extensions_proc_macros" -version = "1.5.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "533d38ecd2709b7608fb8e18e4504deb99e9a72879e6aa66373a76d8dc4259ea" - [[package]] name = "countio" version = "0.3.0" @@ -1545,12 +1487,13 @@ dependencies = [ ] [[package]] -name = "crc32c" -version = "0.6.8" +name = "crc-fast" +version = "1.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3a47af21622d091a8f0fb295b88bc886ac74efcc613efc19f5d0b21de5c89e47" +checksum = "e75b2483e97a5a7da73ac68a05b629f9c53cff58d8ed1c77866079e18b00dba5" dependencies = [ - "rustc_version", + "digest 0.10.7", + "spin 0.10.1", ] [[package]] @@ -1583,18 +1526,18 @@ dependencies = [ [[package]] name = "crossbeam-epoch" -version = "0.9.18" +version = "0.9.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e" +checksum = "2d6914041f254d6e9176c01941b21115dcfb7089e55135a35411081bd106ef3f" dependencies = [ "crossbeam-utils", ] [[package]] name = "crossbeam-queue" -version = "0.3.12" +version = "0.3.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0f58bbc28f91df819d0aa2a2c00cd19754769c2fad90579b3592b1c9ba7a3115" +checksum = "803d13fb3b09d88be9f4dbc29062c66b19bf7170867ceb746d2a8689bf6c7a26" dependencies = [ "crossbeam-utils", ] @@ -1787,14 +1730,13 @@ dependencies = [ [[package]] name = "datafusion" -version = "53.1.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "93db0e623840612f7f2cd757f7e8a8922064192363732c88692e0870016e141b" +checksum = "754ef4e8f073922a26f5b23133b9db4829342362b09be0bc94309cf261c2f098" dependencies = [ "arrow", "arrow-schema", "async-trait", - "bytes", "chrono", "datafusion-catalog", "datafusion-catalog-listing", @@ -1821,12 +1763,11 @@ dependencies = [ "datafusion-session", "datafusion-sql", "futures", + "indexmap 2.14.0", "itertools 0.14.0", "log", "object_store", "parking_lot", - "rand 0.9.4", - "regex", "sqlparser", "tempfile", "tokio", @@ -1836,9 +1777,9 @@ dependencies = [ [[package]] name = "datafusion-catalog" -version = "53.1.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "37cefde60b26a7f4ff61e9d2ff2833322f91df2b568d7238afe67bde5bdffb66" +checksum = "06afd1e38dd27bbb1258685a1fc6524df6aff4e07b25b393a47de59635178d99" dependencies = [ "arrow", "async-trait", @@ -1861,9 +1802,9 @@ dependencies = [ [[package]] name = "datafusion-catalog-listing" -version = "53.1.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "17e112307715d6a7a331111a4c2330ff54bc237183511c319e3708a4cff431fb" +checksum = "f0668fb32c12065ec242be0e5b4bc62bd7a06a0be3ecd83791ef877e4be67e02" dependencies = [ "arrow", "async-trait", @@ -1884,33 +1825,34 @@ dependencies = [ [[package]] name = "datafusion-common" -version = "53.1.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d72a11ca44a95e1081870d3abb80c717496e8a7acb467a1d3e932bb636af5cc2" +checksum = "ca43b263cdff57042cfa8fb817fb3469f4878933380dccff25f5e793580abbf9" dependencies = [ - "ahash", "arrow", "arrow-ipc", + "arrow-schema", "chrono", + "foldhash 0.2.0", "half", - "hashbrown 0.16.1", + "hashbrown 0.17.1", "indexmap 2.14.0", "itertools 0.14.0", "libc", "log", "object_store", "parquet", - "paste", "sqlparser", "tokio", + "uuid", "web-time", ] [[package]] name = "datafusion-common-runtime" -version = "53.1.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "89f4afaed29670ec4fd6053643adc749fe3f4bc9d1ce1b8c5679b22c67d12def" +checksum = "05f0ba2b864792bdca4d76c59a1de0ab6e1b61946596b9936888dbd6360035f2" dependencies = [ "futures", "log", @@ -1919,9 +1861,9 @@ dependencies = [ [[package]] name = "datafusion-datasource" -version = "53.1.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e9fb386e1691355355a96419978a0022b7947b44d4a24a6ea99f00b6b485cbb6" +checksum = "b840a8bce0bcbf5afad02946d438591e7c373f7afccaf3d874c04485772514dd" dependencies = [ "arrow", "async-trait", @@ -1941,16 +1883,17 @@ dependencies = [ "itertools 0.14.0", "log", "object_store", - "rand 0.9.4", + "parking_lot", + "rand 0.9.5", "tokio", "url", ] [[package]] name = "datafusion-datasource-arrow" -version = "53.1.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ffa6c52cfed0734c5f93754d1c0175f558175248bf686c944fb05c373e5fc096" +checksum = "a24cc0b9cf6e367f27f27406eff13abf48a11b72446aaa40b3105c0ded5c17d9" dependencies = [ "arrow", "arrow-ipc", @@ -1972,9 +1915,9 @@ dependencies = [ [[package]] name = "datafusion-datasource-csv" -version = "53.1.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "503f29e0582c1fc189578d665ff57d9300da1f80c282777d7eb67bb79fb8cdca" +checksum = "e1abe56b2a7a2d1d6de5117dd1a203181e267f28529faa5da546947621b697d7" dependencies = [ "arrow", "async-trait", @@ -1995,9 +1938,9 @@ dependencies = [ [[package]] name = "datafusion-datasource-json" -version = "53.1.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e33804749abc8d0c8cb7473228483cb8070e524c6f6086ee1b85a64debe2b3d2" +checksum = "9c3e467f0611ad7bdd5aad17c63c9bb6182d04e5282e5496d897ea2b49c024ba" dependencies = [ "arrow", "async-trait", @@ -2012,16 +1955,15 @@ dependencies = [ "datafusion-session", "futures", "object_store", - "serde_json", "tokio", "tokio-stream", ] [[package]] name = "datafusion-datasource-parquet" -version = "53.1.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "32a8e0365e0e08e8ff94d912f0ababcf9065a1a304018ba90b1fc83c855b4997" +checksum = "4cc35b92cd560082155e80d9c826929c852d3c51543f4affd3a51c464a0aab3a" dependencies = [ "arrow", "async-trait", @@ -2031,6 +1973,7 @@ dependencies = [ "datafusion-datasource", "datafusion-execution", "datafusion-expr", + "datafusion-functions", "datafusion-functions-aggregate-common", "datafusion-physical-expr", "datafusion-physical-expr-adapter", @@ -2049,20 +1992,19 @@ dependencies = [ [[package]] name = "datafusion-doc" -version = "53.1.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8de6ac0df1662b9148ad3c987978b32cbec7c772f199b1d53520c8fa764a87ee" +checksum = "d69bb69d8769e34f76839c960dbde24c1ac0c885a79b6c3c2287bdc56ec67891" [[package]] name = "datafusion-execution" -version = "53.1.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c03c7fbdaefcca4ef6ffe425a5fc2325763bfb426599bb0bf4536466efabe709" +checksum = "d8eac0a09bc8d263f52025cad9e001da4d8138d633fa288edda4d06b1772eae6" dependencies = [ "arrow", "arrow-buffer", "async-trait", - "chrono", "dashmap", "datafusion-common", "datafusion-expr", @@ -2071,18 +2013,19 @@ dependencies = [ "log", "object_store", "parking_lot", - "rand 0.9.4", + "rand 0.9.5", "tempfile", "url", ] [[package]] name = "datafusion-expr" -version = "53.1.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "574b9b6977fedbd2a611cbff12e5caf90f31640ad9dc5870f152836d94bad0dd" +checksum = "eeb14d374767ee0fc62dc79a5ba8bcf8a63c14e993c7d992d0e63adfa23d77d3" dependencies = [ "arrow", + "arrow-schema", "async-trait", "chrono", "datafusion-common", @@ -2093,35 +2036,33 @@ dependencies = [ "datafusion-physical-expr-common", "indexmap 2.14.0", "itertools 0.14.0", - "paste", "serde_json", "sqlparser", ] [[package]] name = "datafusion-expr-common" -version = "53.1.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7d7c3adf3db8bf61e92eb90cb659c8e8b734593a8f7c8e12a843c7ddba24b87e" +checksum = "b7b19a8c95522bee8cbb313d74263b85e355d2b52f42e67ef5694bf5de9e9356" dependencies = [ "arrow", "datafusion-common", "indexmap 2.14.0", "itertools 0.14.0", - "paste", ] [[package]] name = "datafusion-ffi" -version = "53.1.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b95173344d04ba62755c949bf44f8d1a6e4414cf6392a635db96c07e711b9a3c" +checksum = "d9a4a09f24f1387408810acca070207a9cf6c7fc2368122982a8e7671f643d1e" dependencies = [ - "abi_stable", "arrow", "arrow-schema", "async-ffi", "async-trait", + "chrono", "datafusion-catalog", "datafusion-common", "datafusion-datasource", @@ -2130,26 +2071,29 @@ dependencies = [ "datafusion-functions-aggregate-common", "datafusion-physical-expr", "datafusion-physical-expr-common", + "datafusion-physical-optimizer", "datafusion-physical-plan", "datafusion-proto", "datafusion-proto-common", "datafusion-session", "futures", + "libloading", "log", "prost", "semver", + "stabby", "tokio", ] [[package]] name = "datafusion-functions" -version = "53.1.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f28aa4e10384e782774b10e72aca4d93ef7b31aa653095d9d4536b0a3dbc51b6" +checksum = "5f64c983bbbdcb729d921a2b2ac3375598719b5cc0c30345ad664936f3176fc7" dependencies = [ "arrow", "arrow-buffer", - "base64", + "base64 0.22.1", "blake2", "blake3", "chrono", @@ -2160,26 +2104,25 @@ dependencies = [ "datafusion-expr", "datafusion-expr-common", "datafusion-macros", + "datafusion-physical-expr-common", "hex", "itertools 0.14.0", "log", - "md-5 0.10.6", + "md-5 0.11.0", "memchr", "num-traits", - "rand 0.9.4", + "rand 0.9.5", "regex", - "sha2 0.10.9", - "unicode-segmentation", + "sha2 0.11.0", "uuid", ] [[package]] name = "datafusion-functions-aggregate" -version = "53.1.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "00aa6217e56098ba84e0a338176fe52f0a84cca398021512c6c8c5eff806d0ad" +checksum = "89bc17041e424a47ed062f43df24d84aab8b57c4c3221e5c1a5eef46d6c5718b" dependencies = [ - "ahash", "arrow", "datafusion-common", "datafusion-doc", @@ -2189,19 +2132,18 @@ dependencies = [ "datafusion-macros", "datafusion-physical-expr", "datafusion-physical-expr-common", + "foldhash 0.2.0", "half", "log", "num-traits", - "paste", ] [[package]] name = "datafusion-functions-aggregate-common" -version = "53.1.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b511250349407db7c43832ab2de63f5557b19a20dfd236b39ca2c04468b50d47" +checksum = "97dd2a9e865c6108059f5b37b77934f84b50bfb108f837bd0e5c9536e03f0545" dependencies = [ - "ahash", "arrow", "datafusion-common", "datafusion-expr-common", @@ -2210,9 +2152,9 @@ dependencies = [ [[package]] name = "datafusion-functions-nested" -version = "53.1.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ef13a858e20d50f0a9bb5e96e7ac82b4e7597f247515bccca4fdd2992df0212a" +checksum = "75f0bdfeef16d96417b9632ef855645376b242e9006a126dfd0bedfc54a93f5f" dependencies = [ "arrow", "arrow-ord", @@ -2226,34 +2168,34 @@ dependencies = [ "datafusion-functions-aggregate-common", "datafusion-macros", "datafusion-physical-expr-common", - "hashbrown 0.16.1", + "hashbrown 0.17.1", "itertools 0.14.0", "itoa", "log", - "paste", + "memchr", ] [[package]] name = "datafusion-functions-table" -version = "53.1.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "72b40d3f5bbb3905f9ccb1ce9485a9595c77b69758a7c24d3ba79e334ff51e7e" +checksum = "f4e4941673c917819616877e9993da4503e4f4739812be0bc32c5356184c6383" dependencies = [ "arrow", "async-trait", "datafusion-catalog", "datafusion-common", "datafusion-expr", + "datafusion-physical-expr", "datafusion-physical-plan", "parking_lot", - "paste", ] [[package]] name = "datafusion-functions-window" -version = "53.1.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d4e88ec9d57c9b685d02f58bfee7be62d72610430ddcedb82a08e5d9925dbfb6" +checksum = "12dd2e16c12b84b6f6b41b19f55b366dd1c46876bb35b86896c6349067379e8d" dependencies = [ "arrow", "datafusion-common", @@ -2264,14 +2206,13 @@ dependencies = [ "datafusion-physical-expr", "datafusion-physical-expr-common", "log", - "paste", ] [[package]] name = "datafusion-functions-window-common" -version = "53.1.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8307bb93519b1a91913723a1130cfafeee3f72200d870d88e91a6fc5470ede5c" +checksum = "4cdc5e4b6f8b6ef823cc1c761f85088ad4c884fe8df64df3cbcc6b2b84698441" dependencies = [ "datafusion-common", "datafusion-physical-expr-common", @@ -2279,9 +2220,9 @@ dependencies = [ [[package]] name = "datafusion-macros" -version = "53.1.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2e367e6a71051d0ebdd29b2f85d12059b38b1d1f172c6906e80016da662226bd" +checksum = "1a3614234dd93578c92428cb4f408e020874f0d2b7e6c90c928d9d28b5df2ceb" dependencies = [ "datafusion-doc", "quote", @@ -2290,9 +2231,9 @@ dependencies = [ [[package]] name = "datafusion-optimizer" -version = "53.1.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e929015451a67f77d9d8b727b2bf3a40c4445fdef6cdc53281d7d97c76888ace" +checksum = "a0635620b050b81bb92764e99250868f654e2cd5ad1bece413283b3f73c83179" dependencies = [ "arrow", "chrono", @@ -2309,11 +2250,10 @@ dependencies = [ [[package]] name = "datafusion-physical-expr" -version = "53.1.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4b1e68aba7a4b350401cfdf25a3d6f989ad898a7410164afe9ca52080244cb59" +checksum = "8cabf7a86eb70b816729e33c81bf7767c936ee1226f607a114f5dac2decac8d0" dependencies = [ - "ahash", "arrow", "datafusion-common", "datafusion-expr", @@ -2321,20 +2261,19 @@ dependencies = [ "datafusion-functions-aggregate-common", "datafusion-physical-expr-common", "half", - "hashbrown 0.16.1", + "hashbrown 0.17.1", "indexmap 2.14.0", "itertools 0.14.0", "parking_lot", - "paste", "petgraph", "tokio", ] [[package]] name = "datafusion-physical-expr-adapter" -version = "53.1.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ea22315f33cf2e0adc104e8ec42e285f6ed93998d565c65e82fec6a9ee9f9db4" +checksum = "de222e04f7e6744501555a54ab0abe26bfdfebee380af79a9bdc175704246859" dependencies = [ "arrow", "datafusion-common", @@ -2347,26 +2286,26 @@ dependencies = [ [[package]] name = "datafusion-physical-expr-common" -version = "53.1.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b04b45ea8ad3ac2d78f2ea2a76053e06591c9629c7a603eda16c10649ecf4362" +checksum = "72d0d0057fc5a502d45c870cb6d47c66eb7bdd5edb1bd71ad6f3f724975ac2a8" dependencies = [ - "ahash", "arrow", "chrono", "datafusion-common", "datafusion-expr-common", - "hashbrown 0.16.1", + "hashbrown 0.17.1", "indexmap 2.14.0", "itertools 0.14.0", "parking_lot", + "pin-project", ] [[package]] name = "datafusion-physical-optimizer" -version = "53.1.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7cb13397809a425918f608dfe8653f332015a3e330004ab191b4404187238b95" +checksum = "86046eed10950c5f9aaed9acfd148e9bd2e1dfdfe4f9aef607d1447b271e4183" dependencies = [ "arrow", "datafusion-common", @@ -2382,12 +2321,13 @@ dependencies = [ [[package]] name = "datafusion-physical-plan" -version = "53.1.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5edc023675791af9d5fb4cc4c24abf5f7bd3bd4dcf9e5bd90ea1eff6976dcc79" +checksum = "9bc84da934c903407ba297971ebcc020c4c1a38aafd765d6c144c76eff3fa6a1" dependencies = [ - "ahash", "arrow", + "arrow-data", + "arrow-ipc", "arrow-ord", "arrow-schema", "async-trait", @@ -2402,7 +2342,7 @@ dependencies = [ "datafusion-physical-expr-common", "futures", "half", - "hashbrown 0.16.1", + "hashbrown 0.17.1", "indexmap 2.14.0", "itertools 0.14.0", "log", @@ -2414,9 +2354,9 @@ dependencies = [ [[package]] name = "datafusion-proto" -version = "53.1.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6a387aaef949dc16bb6abc81bd1af850ec7449183aef011214f9724957495738" +checksum = "67791dcfacd142a9d95f8f73c2a161094cc936d231d80c235f5017dacf24e84e" dependencies = [ "arrow", "chrono", @@ -2437,14 +2377,13 @@ dependencies = [ "datafusion-proto-common", "object_store", "prost", - "rand 0.9.4", ] [[package]] name = "datafusion-proto-common" -version = "53.1.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "16e614c7c53a9c304c6a850b821010bb492e57300311835f1180613f9d2c63d9" +checksum = "b8cd9e80d637891645d074db0f6c650b591117367247deb313fdfb78dff559cb" dependencies = [ "arrow", "datafusion-common", @@ -2453,9 +2392,9 @@ dependencies = [ [[package]] name = "datafusion-pruning" -version = "53.1.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ac8c76860e355616555081cab5968cec1af7a80701ff374510860bcd567e365a" +checksum = "eb63eeac6de19be40f487b65dd84e546195f783c5a9928618e0c4f2a3569b0d7" dependencies = [ "arrow", "datafusion-common", @@ -2464,15 +2403,14 @@ dependencies = [ "datafusion-physical-expr", "datafusion-physical-expr-common", "datafusion-physical-plan", - "itertools 0.14.0", "log", ] [[package]] name = "datafusion-session" -version = "53.1.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5412111aa48e2424ba926112e192f7a6b7e4ccb450145d25ce5ede9f19dc491e" +checksum = "5f961d209177f91bd014db5cbb2c33b7d28a2597b9003e77f17aeb712964315a" dependencies = [ "async-trait", "datafusion-common", @@ -2484,9 +2422,9 @@ dependencies = [ [[package]] name = "datafusion-sql" -version = "53.1.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fa0d133ddf8b9b3b872acac900157f783e7b879fe9a6bccf389abebbfac45ec1" +checksum = "b1d71cb454da682b2af7488e1fc1ddd72ee1b28f19297b8ccad73f0a21ee9a69" dependencies = [ "arrow", "bigdecimal", @@ -2502,9 +2440,9 @@ dependencies = [ [[package]] name = "datafusion-substrait" -version = "53.1.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "98494539a5468979cc42d86c7bc5f0f8cb71ee5c742694c26fc34efdd29dd2e5" +checksum = "f047a6fbf967b6b523758a48d2377bb5f9373a20e7ae4c4326d3c569f80ba3d7" dependencies = [ "async-recursion", "async-trait", @@ -2698,6 +2636,12 @@ dependencies = [ "encoding_rs", ] +[[package]] +name = "endian-type" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c34f04666d835ff5d62e058c3995147c06f42fe86ff053337632bca83e42702d" + [[package]] name = "env_filter" version = "2.0.0" @@ -2745,11 +2689,10 @@ checksum = "40404c3f5f511ec4da6fe866ddf6a717c309fdbb69fbbad7b0f3edab8f2e835f" [[package]] name = "event-listener" -version = "5.4.1" +version = "5.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e13b66accf52311f30a0db42147dadea9850cb48cd070028831ae5f5d4b856ab" +checksum = "5a23add41df1562121a9393cb065eab5146a1242410f23a644851e90cfd669d2" dependencies = [ - "concurrent-queue", "parking", "pin-project-lite", ] @@ -2842,6 +2785,12 @@ dependencies = [ "percent-encoding", ] +[[package]] +name = "frostem" +version = "1.20260804.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d0ae10cfccbae085dd8612669ccc116fe88bd5dfe39d202a8fa68c24c1e546f" + [[package]] name = "fs_extra" version = "1.3.0" @@ -2850,10 +2799,10 @@ checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c" [[package]] name = "fsst" -version = "9.0.0-beta.16" +version = "11.0.0-beta.6" dependencies = [ "arrow-array", - "rand 0.9.4", + "rand 0.9.5", ] [[package]] @@ -2968,15 +2917,6 @@ dependencies = [ "cfg-if 0.1.10", ] -[[package]] -name = "generational-arena" -version = "0.2.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "877e94aff08e743b651baaea359664321055749b398adff8740a7399af7796e7" -dependencies = [ - "cfg-if 1.0.4", -] - [[package]] name = "generator" version = "0.8.9" @@ -3087,12 +3027,13 @@ dependencies = [ [[package]] name = "geodatafusion" -version = "0.4.0" +version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "af7cd430f1a1f59bc97053d824ad410ea6fd123c8977b3c1a75335e289233b8b" +checksum = "fecbdd00d0fff2b04635c1b1e4129c217908f0c2d17539e0a2275308afce2552" dependencies = [ "arrow-arith", "arrow-array", + "arrow-buffer", "arrow-schema", "datafusion", "geo", @@ -3205,17 +3146,25 @@ dependencies = [ [[package]] name = "goosefs-sdk" -version = "0.1.5" +version = "0.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ae079b88ffe7772d12cfc5c40a5a324babb357893d95b5e3a22ae857f236c5f" +checksum = "4a9bc9414e3b2cb0bd08dfe0eb315b177e86b119c7fa5e16179c92fc7b184860" dependencies = [ + "arc-swap", "async-trait", "bytes", "dashmap", + "futures", "hostname", + "io-uring", + "itoa", + "libc", + "lru", + "memmap2", + "moka", "prost", "prost-types", - "rand 0.9.4", + "rand 0.9.5", "reqwest 0.12.28", "serde", "thiserror 2.0.18", @@ -3225,6 +3174,7 @@ dependencies = [ "tonic-prost", "tracing", "uuid", + "xxhash-rust", ] [[package]] @@ -3252,6 +3202,7 @@ version = "2.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6ea2d84b969582b4b1864a92dc5d27cd2b77b622a8d79306834f1be5ba20d84b" dependencies = [ + "bytemuck", "cfg-if 1.0.4", "crunchy", "num-traits", @@ -3285,6 +3236,8 @@ version = "0.15.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9229cfe53dfd69f0609a49f65461bd93001ea1ef889cd5529dd176593f5338a1" dependencies = [ + "allocator-api2", + "equivalent", "foldhash 0.1.5", ] @@ -3304,6 +3257,11 @@ name = "hashbrown" version = "0.17.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" +dependencies = [ + "allocator-api2", + "equivalent", + "foldhash 0.2.0", +] [[package]] name = "heapify" @@ -3459,9 +3417,9 @@ checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" [[package]] name = "humantime" -version = "2.3.0" +version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "135b12329e5e3ce057a9f972339ea52bc954fe1e9358ef27f95e89716fbc5424" +checksum = "15cdd26707701c53297e2fa6afb323d55fbc1d0810c3aec078ae3ef0424c3c15" [[package]] name = "hybrid-array" @@ -3530,7 +3488,7 @@ version = "0.1.20" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "96547c2556ec9d12fb1578c4eaf448b04993e7fb79cbaad930a656880a6bdfa0" dependencies = [ - "base64", + "base64 0.22.1", "bytes", "futures-channel", "futures-util", @@ -3867,18 +3825,18 @@ checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" [[package]] name = "jieba-macros" -version = "0.10.1" +version = "0.10.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "46adade69b634535a8f495cf87710ed893cff53e1dbc9dd750c2ab81c5defb82" +checksum = "38fc0f3831de71556de69643b80a08a5c8cd260a23c6b8dbeb7cd923c779cac5" dependencies = [ "phf_codegen", ] [[package]] name = "jieba-rs" -version = "0.10.1" +version = "0.10.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "11b53580aaa8ec8b713da271da434f8947409242c537a9ab3f7b76bdbb19e8a9" +checksum = "bb5bdea4dc241d589e179f39d2a778f31490f3370aa2f626223dbd930ebc5c9d" dependencies = [ "bytecount", "cedarwood", @@ -4015,7 +3973,7 @@ dependencies = [ "nom", "num-traits", "ordered-float 5.3.0", - "rand 0.9.4", + "rand 0.9.5", "serde", "serde_json", "zmij", @@ -4049,7 +4007,7 @@ checksum = "e037a2e1d8d5fdbd49b16a4ea09d5d6401c1f29eca5ff29d03d3824dba16256a" [[package]] name = "lance" -version = "9.0.0-beta.16" +version = "11.0.0-beta.6" dependencies = [ "arc-swap", "arrow", @@ -4105,7 +4063,7 @@ dependencies = [ "prost", "prost-build", "prost-types", - "rand 0.9.4", + "rand 0.9.5", "rayon", "roaring", "rustc-hash", @@ -4123,7 +4081,7 @@ dependencies = [ [[package]] name = "lance-arrow" -version = "9.0.0-beta.16" +version = "11.0.0-beta.6" dependencies = [ "arrow-array", "arrow-buffer", @@ -4132,13 +4090,14 @@ dependencies = [ "arrow-ord", "arrow-schema", "arrow-select", + "bytemuck", "bytes", "futures", "getrandom 0.2.17", "half", "jsonb", "num-traits", - "rand 0.9.4", + "rand 0.9.5", ] [[package]] @@ -4165,7 +4124,7 @@ dependencies = [ [[package]] name = "lance-bitpacking" -version = "9.0.0-beta.16" +version = "11.0.0-beta.6" dependencies = [ "arrayref", "crunchy", @@ -4175,13 +4134,14 @@ dependencies = [ [[package]] name = "lance-core" -version = "9.0.0-beta.16" +version = "11.0.0-beta.6" dependencies = [ "arrow-array", "arrow-buffer", "arrow-data", "arrow-schema", "async-trait", + "blake3", "byteorder", "bytes", "datafusion-common", @@ -4198,7 +4158,8 @@ dependencies = [ "object_store", "pin-project", "prost", - "rand 0.9.4", + "quick_cache", + "rand 0.9.5", "roaring", "serde_json", "snafu", @@ -4213,7 +4174,7 @@ dependencies = [ [[package]] name = "lance-datafusion" -version = "9.0.0-beta.16" +version = "11.0.0-beta.6" dependencies = [ "arrow", "arrow-array", @@ -4245,7 +4206,7 @@ dependencies = [ [[package]] name = "lance-datagen" -version = "9.0.0-beta.16" +version = "11.0.0-beta.6" dependencies = [ "arrow", "arrow-array", @@ -4255,14 +4216,14 @@ dependencies = [ "futures", "half", "hex", - "rand 0.9.4", + "rand 0.9.5", "rand_distr", "rand_xoshiro", ] [[package]] name = "lance-derive" -version = "9.0.0-beta.16" +version = "11.0.0-beta.6" dependencies = [ "proc-macro2", "quote", @@ -4271,7 +4232,7 @@ dependencies = [ [[package]] name = "lance-encoding" -version = "9.0.0-beta.16" +version = "11.0.0-beta.6" dependencies = [ "arrow-arith", "arrow-array", @@ -4296,8 +4257,7 @@ dependencies = [ "num-traits", "prost", "prost-build", - "rand 0.9.4", - "strum 0.26.3", + "rand 0.9.5", "tokio", "tracing", "xxhash-rust", @@ -4306,11 +4266,12 @@ dependencies = [ [[package]] name = "lance-file" -version = "9.0.0-beta.16" +version = "11.0.0-beta.6" dependencies = [ "arrow-arith", "arrow-array", "arrow-buffer", + "arrow-cast", "arrow-data", "arrow-schema", "arrow-select", @@ -4336,7 +4297,7 @@ dependencies = [ [[package]] name = "lance-geo" -version = "9.0.0-beta.16" +version = "11.0.0-beta.6" dependencies = [ "datafusion", "geo-traits", @@ -4350,12 +4311,13 @@ dependencies = [ [[package]] name = "lance-index" -version = "9.0.0-beta.16" +version = "11.0.0-beta.6" dependencies = [ "arc-swap", "arrow", "arrow-arith", "arrow-array", + "arrow-ipc", "arrow-ord", "arrow-schema", "arrow-select", @@ -4389,6 +4351,7 @@ dependencies = [ "lance-encoding", "lance-file", "lance-geo", + "lance-index-core", "lance-io", "lance-linalg", "lance-select", @@ -4402,7 +4365,7 @@ dependencies = [ "prost", "prost-build", "prost-types", - "rand 0.9.4", + "rand 0.9.5", "rand_distr", "rangemap", "rayon", @@ -4417,19 +4380,35 @@ dependencies = [ "uuid", ] +[[package]] +name = "lance-index-core" +version = "11.0.0-beta.6" +dependencies = [ + "arrow-array", + "arrow-schema", + "arrow-select", + "async-trait", + "bytes", + "datafusion", + "datafusion-common", + "datafusion-expr", + "futures", + "lance-core", + "lance-io", + "lance-select", + "prost-types", + "roaring", + "serde", + "serde_json", +] + [[package]] name = "lance-io" -version = "9.0.0-beta.16" +version = "11.0.0-beta.6" dependencies = [ "arrow", - "arrow-arith", "arrow-array", - "arrow-buffer", - "arrow-cast", - "arrow-data", "arrow-schema", - "arrow-select", - "async-recursion", "async-trait", "aws-config", "aws-credential-types", @@ -4443,6 +4422,7 @@ dependencies = [ "lance-core", "lance-namespace", "log", + "metrics", "moka", "object_store", "object_store_opendal", @@ -4450,7 +4430,7 @@ dependencies = [ "path_abs", "pin-project", "prost", - "rand 0.9.4", + "rand 0.9.5", "serde", "tempfile", "tokio", @@ -4460,7 +4440,7 @@ dependencies = [ [[package]] name = "lance-linalg" -version = "9.0.0-beta.16" +version = "11.0.0-beta.6" dependencies = [ "arrow-array", "arrow-buffer", @@ -4470,13 +4450,13 @@ dependencies = [ "lance-arrow", "lance-core", "num-traits", - "rand 0.9.4", + "rand 0.9.5", "rayon", ] [[package]] name = "lance-namespace" -version = "9.0.0-beta.16" +version = "11.0.0-beta.6" dependencies = [ "arrow", "async-trait", @@ -4488,7 +4468,7 @@ dependencies = [ [[package]] name = "lance-namespace-impls" -version = "9.0.0-beta.16" +version = "11.0.0-beta.6" dependencies = [ "arrow", "arrow-ipc", @@ -4508,12 +4488,11 @@ dependencies = [ "lance-table", "log", "object_store", - "rand 0.9.4", + "rand 0.9.5", "reqwest 0.12.28", "roaring", "serde", "serde_json", - "time", "tokio", "tower", "tower-http 0.5.2", @@ -4537,7 +4516,7 @@ dependencies = [ [[package]] name = "lance-select" -version = "9.0.0-beta.16" +version = "11.0.0-beta.6" dependencies = [ "arrow-array", "arrow-buffer", @@ -4552,7 +4531,7 @@ dependencies = [ [[package]] name = "lance-table" -version = "9.0.0-beta.16" +version = "11.0.0-beta.6" dependencies = [ "arrow", "arrow-array", @@ -4562,6 +4541,7 @@ dependencies = [ "async-trait", "aws-credential-types", "aws-sdk-dynamodb", + "blake3", "byteorder", "bytes", "chrono", @@ -4576,7 +4556,7 @@ dependencies = [ "prost", "prost-build", "prost-types", - "rand 0.9.4", + "rand 0.9.5", "rangemap", "roaring", "semver", @@ -4591,12 +4571,12 @@ dependencies = [ [[package]] name = "lance-tokenizer" -version = "9.0.0-beta.16" +version = "11.0.0-beta.6" dependencies = [ + "frostem", "icu_segmenter", "jieba-rs", "lindera", - "rust-stemmers", "serde", "stop-words", "unicode-normalization", @@ -4608,7 +4588,7 @@ version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" dependencies = [ - "spin", + "spin 0.9.9", ] [[package]] @@ -4670,18 +4650,18 @@ dependencies = [ [[package]] name = "libc" -version = "0.2.186" +version = "0.2.189" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66" +checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2" [[package]] name = "libloading" -version = "0.7.4" +version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b67380fd3b2fbe7527a606e18729d21c6f3951633d0500574c4dc22d2d638b9f" +checksum = "754ca22de805bb5744484a5b151a9e1a8e837d5dc232c2d7d8c2e3492edc8b60" dependencies = [ "cfg-if 1.0.4", - "winapi", + "windows-link", ] [[package]] @@ -4727,8 +4707,8 @@ dependencies = [ "serde", "serde_json", "serde_yaml_ng", - "strum 0.28.0", - "strum_macros 0.28.0", + "strum", + "strum_macros", "unicode-blocks", "unicode-normalization", "unicode-segmentation", @@ -4757,8 +4737,8 @@ dependencies = [ "rkyv", "serde", "serde_json", - "strum 0.28.0", - "strum_macros 0.28.0", + "strum", + "strum_macros", "thiserror 2.0.18", ] @@ -4814,6 +4794,15 @@ dependencies = [ "tracing-subscriber", ] +[[package]] +name = "lru" +version = "0.12.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "234cf4f4a04dc1f57e24b96cc0cd600cf2af460d4161ac5ecdd0af8e1f3b2a38" +dependencies = [ + "hashbrown 0.15.5", +] + [[package]] name = "lru-slab" version = "0.1.2" @@ -4913,13 +4902,43 @@ checksum = "88904434abc2901f197fe8cc55f0445e7ded921dba5911dad2e2b39b48e663c4" [[package]] name = "memmap2" -version = "0.9.10" +version = "0.9.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "714098028fe011992e1c3962653c96b2d578c4b4bce9036e15ff220319b1e0e3" +checksum = "d1219ed1b7f229ee7104d281dd01d6802fe28bb6e95d292942c4daacdeb798c0" dependencies = [ "libc", ] +[[package]] +name = "metrics" +version = "0.24.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "89550ee9f79e88fef3119de263694973a8adb26c21d75322164fb8c493039fe2" +dependencies = [ + "portable-atomic", + "rapidhash", +] + +[[package]] +name = "metrics-util" +version = "0.19.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8496cc523d1f94c1385dd8f0f0c2c480b2b8aeccb5b7e4485ad6365523ae376" +dependencies = [ + "aho-corasick", + "crossbeam-epoch", + "crossbeam-utils", + "hashbrown 0.15.5", + "indexmap 2.14.0", + "metrics", + "ordered-float 4.6.0", + "quanta", + "radix_trie", + "rand 0.9.5", + "rand_xoshiro", + "sketches-ddsketch", +] + [[package]] name = "mime" version = "0.3.17" @@ -5024,6 +5043,15 @@ dependencies = [ "rawpointer", ] +[[package]] +name = "nibble_vec" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77a5d83df9f36fe23f0c3648c6bbb8b0298bb5f1939c8f2704431371f4b84d43" +dependencies = [ + "smallvec", +] + [[package]] name = "nom" version = "8.0.0" @@ -5189,7 +5217,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "622acbc9100d3c10e2ee15804b0caa40e55c933d5aa53814cd520805b7958a49" dependencies = [ "async-trait", - "base64", + "base64 0.22.1", "bytes", "chrono", "form_urlencoded", @@ -5224,9 +5252,9 @@ dependencies = [ [[package]] name = "object_store_opendal" -version = "0.57.0" +version = "0.58.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0eb12a624a41fce745838d0ef3701ff6c47797c13cd18ad3612fd2a3134fdbd8" +checksum = "88f165780495c17aa3ce86846600504198c3fffd99073521552751c2430fa6ac" dependencies = [ "async-trait", "bytes", @@ -5259,12 +5287,13 @@ checksum = "269bca4c2591a28585d6bf10d9ed0332b7d76900a1b02bec41bdc3a2cdcda107" [[package]] name = "opendal" -version = "0.57.0" +version = "0.58.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "96c9c85ce253ff87225e7669979d877a20c98a06604ec9d6dd5f4473e08f1ae1" +checksum = "4f20562cc7447fcc915fc5c23df305a412ea80a733c9f2fd9e2d267e2815be6d" dependencies = [ "ctor 1.0.7", "opendal-core", + "opendal-http-transport-reqwest", "opendal-layer-concurrent-limit", "opendal-layer-logging", "opendal-layer-retry", @@ -5282,24 +5311,22 @@ dependencies = [ [[package]] name = "opendal-core" -version = "0.57.0" +version = "0.58.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c4f8607c90e2c963a91467f50fb49fbc7fb3d573f88cea219ca59ccd3740b309" +checksum = "ec75551ff4cf3e57da98979f6a937aaa9ddb3915bf68cc17d03df733be6646ed" dependencies = [ "anyhow", - "base64", + "base64 0.23.0", "bytes", "futures", "http 1.4.2", - "http-body 1.0.1", "jiff", "log", "md-5 0.11.0", "mea", "percent-encoding", - "quick-xml 0.39.4", + "quick-xml 0.41.0", "reqsign-core", - "reqwest 0.13.4", "serde", "serde_json", "tokio", @@ -5308,11 +5335,25 @@ dependencies = [ "web-time", ] +[[package]] +name = "opendal-http-transport-reqwest" +version = "0.58.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ad4d4f19c3ce01126a30611f8e544eaa217104a278c889ac17c9374fe4f9e4ef" +dependencies = [ + "bytes", + "futures", + "http 1.4.2", + "http-body 1.0.1", + "opendal-core", + "reqwest 0.13.4", +] + [[package]] name = "opendal-layer-concurrent-limit" -version = "0.57.0" +version = "0.58.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0d6f81ba6960e3fae1882f253b114b21d7e444e1534f209c7737a79f6243eb6f" +checksum = "249ac5b0aa5a7a6c3737342d10456067937f9c9a6f3f02544271f7908ab91081" dependencies = [ "futures", "http 1.4.2", @@ -5322,9 +5363,9 @@ dependencies = [ [[package]] name = "opendal-layer-logging" -version = "0.57.0" +version = "0.58.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "58ada45c6d81d1aa4c9305d0c7d4bc317c59c85866a0908a2d75a7a978aa5ee2" +checksum = "5c75411ab00f77851ff086b686c1e9ca8175ac18c15afa2cb75b9036436cb06c" dependencies = [ "log", "opendal-core", @@ -5332,9 +5373,9 @@ dependencies = [ [[package]] name = "opendal-layer-retry" -version = "0.57.0" +version = "0.58.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b2a25a718afb81fad81cb9a0580a1cb989221fa2317f888c6a37f8dad408eb7" +checksum = "80b7738bd5f233ad8da39af9b9316b9b7a4eaddd91e8e32a1e19b7030688121d" dependencies = [ "backon", "log", @@ -5343,9 +5384,9 @@ dependencies = [ [[package]] name = "opendal-layer-timeout" -version = "0.57.0" +version = "0.58.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e91f731724c213af81e9d03517859c8fc47b4578e64ad61ae4f099f10fe36e3" +checksum = "a704141924500f3803c05ed871b53305d2a2f11cb5ef20160c3ee688a1857f66" dependencies = [ "opendal-core", "tokio", @@ -5353,17 +5394,17 @@ dependencies = [ [[package]] name = "opendal-service-azblob" -version = "0.57.0" +version = "0.58.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0030644366ef5d8cbe3a4a5822bf99a4aafddc1666e9d24b44d158d9062fc76a" +checksum = "b3310fbbb48f111c6f590473c2cd15e1b7f8e384444b0d4e328f0464c864d767" dependencies = [ - "base64", + "base64 0.23.0", "bytes", "http 1.4.2", "log", "opendal-core", "opendal-service-azure-common", - "quick-xml 0.39.4", + "quick-xml 0.41.0", "reqsign-azure-storage", "reqsign-core", "reqsign-file-read-tokio", @@ -5374,17 +5415,18 @@ dependencies = [ [[package]] name = "opendal-service-azdls" -version = "0.57.0" +version = "0.58.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6dea4908d490143a9b0b7f7a790e139ff829b06a023f670455ed3d44f664b361" +checksum = "2e3c406729935fe214ce574d68681a1ff7e0b322548f14094912bdbfe50e5c53" dependencies = [ - "base64", + "base64 0.23.0", "bytes", "http 1.4.2", "log", + "mea", "opendal-core", "opendal-service-azure-common", - "quick-xml 0.39.4", + "quick-xml 0.41.0", "reqsign-azure-storage", "reqsign-core", "reqsign-file-read-tokio", @@ -5394,9 +5436,9 @@ dependencies = [ [[package]] name = "opendal-service-azure-common" -version = "0.57.0" +version = "0.58.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b489f13c42e69d69bdd72952b634356ec43a7881a20259b38b540fcecdf4051" +checksum = "7348c88edf15af435b7be930077746b569fac5e738c1bf6a363b675e7317c9df" dependencies = [ "http 1.4.2", "opendal-core", @@ -5404,15 +5446,15 @@ dependencies = [ [[package]] name = "opendal-service-cos" -version = "0.57.0" +version = "0.58.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aa8cafe9729213375c7331019b0cb756ad3e1aff7f45cd32c45eae91ebde8901" +checksum = "d533d4582105d269c8aebeee5f0e8bcf960f41b8aab6197df7012254d9f39bf0" dependencies = [ "bytes", "http 1.4.2", "log", "opendal-core", - "quick-xml 0.39.4", + "quick-xml 0.41.0", "reqsign-core", "reqsign-file-read-tokio", "reqsign-tencent-cos", @@ -5421,9 +5463,9 @@ dependencies = [ [[package]] name = "opendal-service-gcs" -version = "0.57.0" +version = "0.58.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "48de101aac565ed06af4b47903c24eafd249075553ec1fb18256751c45148d47" +checksum = "007f3fba63c21e516c956b891e96ff9892d8175662bfb781cdada9d3766a11e6" dependencies = [ "async-trait", "bytes", @@ -5431,7 +5473,7 @@ dependencies = [ "log", "opendal-core", "percent-encoding", - "quick-xml 0.39.4", + "quick-xml 0.41.0", "reqsign-core", "reqsign-file-read-tokio", "reqsign-google", @@ -5442,9 +5484,9 @@ dependencies = [ [[package]] name = "opendal-service-goosefs" -version = "0.57.0" +version = "0.58.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "69e43048bde419947ba826fbdc2f134d6c03f44ebf48bd33a03b72f9fc45fcb4" +checksum = "60871e6386f04d831e6a5bdbc032af4a91aeba49963252d0ef456a2cf36a9b78" dependencies = [ "bytes", "goosefs-sdk", @@ -5456,9 +5498,9 @@ dependencies = [ [[package]] name = "opendal-service-hf" -version = "0.57.0" +version = "0.58.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c4922661976a1d40794a2adfbdb888cc3c23097690f825a92f773af38908a848" +checksum = "b41fd41eb7ed03c5e66cefda61e8e117808ffd2908f2916737cb020a6beb02c7" dependencies = [ "bytes", "hf-xet", @@ -5466,22 +5508,21 @@ dependencies = [ "log", "opendal-core", "percent-encoding", - "reqwest 0.13.4", "serde", "serde_json", ] [[package]] name = "opendal-service-oss" -version = "0.57.0" +version = "0.58.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "328fa55e8888cbdfe00826bfea2a79042422b720e8369e9e021e46121dea5ace" +checksum = "cd528ec2d49c5ca69e674ffed7b3e0686fb9cfcfea0596870de381467fda4f1b" dependencies = [ "bytes", "http 1.4.2", "log", "opendal-core", - "quick-xml 0.39.4", + "quick-xml 0.41.0", "reqsign-aliyun-oss", "reqsign-core", "reqsign-file-read-tokio", @@ -5490,18 +5531,18 @@ dependencies = [ [[package]] name = "opendal-service-s3" -version = "0.57.0" +version = "0.58.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "313d46c9f5ae70bca26b7c3e3fbb9b639292625f28af73aa016f47e788af9deb" +checksum = "58e80cdf192d7eff05feed747894d64f81905ac4eaf132edf7ea270abdd2d663" dependencies = [ - "base64", + "base64 0.23.0", "bytes", - "crc32c", + "crc-fast", "http 1.4.2", "log", "md-5 0.11.0", "opendal-core", - "quick-xml 0.39.4", + "quick-xml 0.41.0", "reqsign-aws-v4", "reqsign-core", "reqsign-file-read-tokio", @@ -5511,14 +5552,14 @@ dependencies = [ [[package]] name = "opendal-service-tos" -version = "0.57.0" +version = "0.58.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6f2f7a4c32e5202eb4ac72e76c4b5e30c86ab60762811172f4111103b9d673a1" +checksum = "7841a1a09485bd08eeac34d67804321b62456c855027c580bce12a75564f4609" dependencies = [ "bytes", "http 1.4.2", "opendal-core", - "quick-xml 0.39.4", + "quick-xml 0.41.0", "reqsign-core", "reqsign-file-read-tokio", "reqsign-volcengine-tos", @@ -5547,6 +5588,15 @@ dependencies = [ "num-traits", ] +[[package]] +name = "ordered-float" +version = "4.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7bb71e1b3fa6ca1c61f383464aaf2bb0e2f8e772a1f01d486832464de363b951" +dependencies = [ + "num-traits", +] + [[package]] name = "ordered-float" version = "5.3.0" @@ -5612,9 +5662,9 @@ dependencies = [ [[package]] name = "parquet" -version = "58.3.0" +version = "58.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5dafa7d01085b62a47dd0c1829550a0a36710ea9c4fe358a05a85477cec8a908" +checksum = "d298093b2dec60289dce0684c986d0f7679e9dd15771c2c65406e1aaf604a704" dependencies = [ "ahash", "arrow-array", @@ -5623,7 +5673,7 @@ dependencies = [ "arrow-ipc", "arrow-schema", "arrow-select", - "base64", + "base64 0.22.1", "brotli", "bytes", "chrono", @@ -5670,7 +5720,7 @@ version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "898bac3fa00d0ba57a4e8289837e965baa2dee8c3749f3b11d45a64b4223d9c3" dependencies = [ - "base64", + "base64 0.22.1", "serde", ] @@ -5717,7 +5767,7 @@ version = "3.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1d30c53c26bc5b31a98cd02d20f25a7c8567146caf63ed593a9d87b2775291be" dependencies = [ - "base64", + "base64 0.22.1", "serde_core", ] @@ -5973,7 +6023,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "03da047801ff44bb6a4d407d4860c05fd70bb81714e6b2f3812603d5b145b042" dependencies = [ "heck", - "itertools 0.14.0", + "itertools 0.11.0", "log", "multimap", "petgraph", @@ -5992,7 +6042,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b570b25f7617e43d59005d0990ccb79e950a423952cea19671b7a876da390adf" dependencies = [ "anyhow", - "itertools 0.14.0", + "itertools 0.11.0", "proc-macro2", "quote", "syn 2.0.118", @@ -6029,7 +6079,7 @@ dependencies = [ [[package]] name = "pylance" -version = "9.0.0-beta.16" +version = "11.0.0-beta.6" dependencies = [ "alloc-stdlib", "arrow", @@ -6061,6 +6111,8 @@ dependencies = [ "lance-table", "libc", "log", + "metrics", + "metrics-util", "object_store", "prost", "prost-types", @@ -6146,6 +6198,21 @@ dependencies = [ "serde", ] +[[package]] +name = "quanta" +version = "0.12.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f3ab5a9d756f0d97bdc89019bd2e4ea098cf9cde50ee7564dde6b81ccc8f06c7" +dependencies = [ + "crossbeam-utils", + "libc", + "once_cell", + "raw-cpuid", + "wasi 0.11.1+wasi-snapshot-preview1", + "web-sys", + "winapi", +] + [[package]] name = "quick-xml" version = "0.39.4" @@ -6158,14 +6225,26 @@ dependencies = [ [[package]] name = "quick-xml" -version = "0.40.1" +version = "0.41.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2474bd2e5029e7ccb6abb2ba48cf2383a333851dedf495901544281590c7da7f" +checksum = "e660451e55124f798a69a5af3f49ccfbefbd41910eefd25caf2393e1f3473ec1" dependencies = [ "memchr", "serde", ] +[[package]] +name = "quick_cache" +version = "0.6.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9c6658afe513a3b484e3abfdaa0d03ef3c0bbf017542c178dd55f94eb3051f9" +dependencies = [ + "ahash", + "equivalent", + "hashbrown 0.16.1", + "parking_lot", +] + [[package]] name = "quinn" version = "0.11.9" @@ -6196,7 +6275,7 @@ dependencies = [ "bytes", "getrandom 0.3.4", "lru-slab", - "rand 0.9.4", + "rand 0.9.5", "ring", "rustc-hash", "rustls", @@ -6249,6 +6328,16 @@ version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dc33ff2d4973d518d823d61aa239014831e521c75da58e3df4840d3f47749d09" +[[package]] +name = "radix_trie" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c069c179fcdc6a2fe24d8d18305cf085fdbd4f922c041943e203685d6a1c58fd" +dependencies = [ + "endian-type", + "nibble_vec", +] + [[package]] name = "rancor" version = "0.1.1" @@ -6270,9 +6359,9 @@ dependencies = [ [[package]] name = "rand" -version = "0.9.4" +version = "0.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "44c5af06bb1b7d3216d91932aed5265164bf384dc89cd6ba05cf59a35f5f76ea" +checksum = "b9ef1d0d795eb7d84685bca4f72f3649f064e6641543d3a8c415898726a57b41" dependencies = [ "rand_chacha 0.9.0", "rand_core 0.9.5", @@ -6340,7 +6429,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6a8615d50dcf34fa31f7ab52692afec947c4dd0ab803cc87cb3b0b4570ff7463" dependencies = [ "num-traits", - "rand 0.9.4", + "rand 0.9.5", ] [[package]] @@ -6358,6 +6447,24 @@ version = "1.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "973443cf09a9c8656b574a866ab68dfa19f0867d0340648c7d2f6a71b8a8ea68" +[[package]] +name = "rapidhash" +version = "4.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32b266a82f4aa99bb5c25e28d11cc44ace63d91adbcbcee4d323e2ae3d49ef37" +dependencies = [ + "rustversion", +] + +[[package]] +name = "raw-cpuid" +version = "11.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "498cd0dc59d73224351ee52a95fee0f1a617a2eae0e7d9d720cc622c73a54186" +dependencies = [ + "bitflags 2.13.0", +] + [[package]] name = "rawpointer" version = "0.2.1" @@ -6487,20 +6594,11 @@ dependencies = [ "bytecheck", ] -[[package]] -name = "repr_offset" -version = "0.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fb1070755bd29dffc19d0971cab794e607839ba2ef4b69a9e6fbc8733c1b72ea" -dependencies = [ - "tstr", -] - [[package]] name = "reqsign-aliyun-oss" -version = "3.1.0" +version = "3.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "372266b4733756738eeb199a98188037d27a0989980e2600ae7ce1faf00a867d" +checksum = "9c0f9f69a519dd6958c4b43606bb8e1278cdc76d611fc8fed4b796eee548dc0f" dependencies = [ "anyhow", "form_urlencoded", @@ -6515,9 +6613,9 @@ dependencies = [ [[package]] name = "reqsign-aws-v4" -version = "3.0.1" +version = "3.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b75624bd8a466e37ddc0a7b6c33ac859a85347c153a916e1dd9d0b68338f74a" +checksum = "cc883bc56889f3e4a419265c87facea222a921debc5c6f15c7fd8b68ec4b36b2" dependencies = [ "anyhow", "bytes", @@ -6526,7 +6624,7 @@ dependencies = [ "http 1.4.2", "log", "percent-encoding", - "quick-xml 0.40.1", + "quick-xml 0.41.0", "reqsign-core", "rust-ini", "serde", @@ -6537,12 +6635,12 @@ dependencies = [ [[package]] name = "reqsign-azure-storage" -version = "3.0.1" +version = "3.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "62b96928e73ad984de1d99e382749d09e5dab7dd707b767974f7e40aa926b82f" +checksum = "a6ebd8524185ce9c64063e3095f83968acfa90922f00c601a4a0f3aca15b077e" dependencies = [ "anyhow", - "base64", + "base64 0.22.1", "bytes", "form_urlencoded", "http 1.4.2", @@ -6558,14 +6656,13 @@ dependencies = [ [[package]] name = "reqsign-core" -version = "3.0.1" +version = "3.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a5fa5cb48808693614d1701fcd3db0b30fa292e0f18e122ae068b6d32eaeed3f" +checksum = "7e38b44697c60a823705ccef85cb04d8e0527c9d16ed7c58bf1c6395bdd24ceb" dependencies = [ "anyhow", - "base64", + "base64 0.22.1", "bytes", - "form_urlencoded", "futures", "hex", "hmac 0.13.0", @@ -6583,9 +6680,9 @@ dependencies = [ [[package]] name = "reqsign-file-read-tokio" -version = "3.0.1" +version = "3.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6a4b6f3a3fd29ffcc99a90aec585a65217783badfd73acddf847b63ae683bda9" +checksum = "688ff0ae421b8d4b92b53fdafaf53df2de28f428a9962edcf21702990b26f74b" dependencies = [ "anyhow", "reqsign-core", @@ -6594,9 +6691,9 @@ dependencies = [ [[package]] name = "reqsign-google" -version = "3.0.1" +version = "3.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eb215d0876a18b6bd9cdd380b589e5292aaa638ca15266de794b1122d898b6b2" +checksum = "a96da0b579b846d358090cb06b9e3c2ad1375529efbe3e0c45f96bd7bcf043ea" dependencies = [ "form_urlencoded", "http 1.4.2", @@ -6612,9 +6709,9 @@ dependencies = [ [[package]] name = "reqsign-tencent-cos" -version = "3.0.1" +version = "3.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "84110aabba799fbcd48b3abb51fbbff4749f879252e5806b6f5d0cbe0fef6abb" +checksum = "f6497dd9f6e3d1349b420521484099b284f95e8d3a65f088fccef42493a7b644" dependencies = [ "anyhow", "http 1.4.2", @@ -6627,9 +6724,9 @@ dependencies = [ [[package]] name = "reqsign-volcengine-tos" -version = "3.0.1" +version = "3.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "91d083a363b3577f519ce8425bb50f902622a28a83f7c4a26a5c990b66ec75b3" +checksum = "4335f949a3fd8b53867fd716dac97fbd545e45bcaed44343796517f2e19cd609" dependencies = [ "anyhow", "http 1.4.2", @@ -6644,7 +6741,7 @@ version = "0.12.28" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "eddd3ca559203180a307f12d114c268abf583f59b03cb906fd0b3ff8646c1147" dependencies = [ - "base64", + "base64 0.22.1", "bytes", "encoding_rs", "futures-core", @@ -6690,7 +6787,7 @@ version = "0.13.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "219c5811de6525e5416c7d5d53bb656d3afdbc6c5af816e0802bcfa42dbdc1c3" dependencies = [ - "base64", + "base64 0.22.1", "bytes", "futures-core", "futures-util", @@ -6754,9 +6851,9 @@ dependencies = [ [[package]] name = "rkyv" -version = "0.8.16" +version = "0.8.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "73389e0c99e664f919275ab5b5b0471391fe9a8de61e1dff9b1eaf56a90f16e3" +checksum = "815cc8a37159a463064825246cadb07961e25cd9885908606f6d08a98d8f8874" dependencies = [ "bytecheck", "bytes", @@ -6773,9 +6870,9 @@ dependencies = [ [[package]] name = "rkyv_derive" -version = "0.8.16" +version = "0.8.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5d2ed0b54125315fb36bd021e82d314d1c126548f871634b483f46b31d13cac6" +checksum = "c0ed1a78a1b19d184b0daa629dd9a024573173ec7d485b287cb369fb3607cc1c" dependencies = [ "proc-macro2", "quote", @@ -6840,21 +6937,11 @@ dependencies = [ "ordered-multimap", ] -[[package]] -name = "rust-stemmers" -version = "1.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e46a2036019fdb888131db7a4c847a1063a7493f971ed94ea82c67eada63ca54" -dependencies = [ - "serde", - "serde_derive", -] - [[package]] name = "rustc-hash" -version = "2.1.2" +version = "2.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "94300abf3f1ae2e2b8ffb7b58043de3d399c73fa6f4b73826402a5c457614dbe" +checksum = "6b1e7f9a428571be2dc5bc0505c13fb6bf936822b894ec87abf8a08a4e51742d" [[package]] name = "rustc_version" @@ -7111,9 +7198,9 @@ checksum = "1bc711410fbe7399f390ca1c3b60ad0f53f80e95c5eb935e52268a0e2cd49acc" [[package]] name = "serde" -version = "1.0.228" +version = "1.0.229" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e" +checksum = "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba" dependencies = [ "serde_core", "serde_derive", @@ -7121,22 +7208,22 @@ dependencies = [ [[package]] name = "serde_core" -version = "1.0.228" +version = "1.0.229" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "41d385c7d4ca58e59fc732af25c3983b67ac852c1a25000afe1175de458b67ad" +checksum = "67dca2c9c51e58a4791a4b1ed58308b39c64224d349a935ab5039aa360942a48" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.228" +version = "1.0.229" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" +checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348" dependencies = [ "proc-macro2", "quote", - "syn 2.0.118", + "syn 3.0.2", ] [[package]] @@ -7152,10 +7239,11 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.150" +version = "1.0.151" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e8014e44b4736ed0538adeecded0fce2a272f22dc9578a7eb6b2d9993c74cfb9" +checksum = "c841b55ecdae098c80dcae9cf767f6f8a0c2cdb3416bbef72181df4d0fe73f14" dependencies = [ + "indexmap 2.14.0", "itoa", "memchr", "serde", @@ -7215,7 +7303,7 @@ version = "3.21.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "76a5c54c7310e7b8b9577c286d7e399ddd876c3e12b3ed917a8aabc4b96e9e8c" dependencies = [ - "base64", + "base64 0.22.1", "bs58", "chrono", "hex", @@ -7310,6 +7398,12 @@ dependencies = [ "cc", ] +[[package]] +name = "sha2-const-stable" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5f179d4e11094a893b82fff208f74d448a7512f99f5a0acbd5c679b705f83ed9" + [[package]] name = "sharded-slab" version = "0.1.7" @@ -7384,6 +7478,12 @@ version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8ee5873ec9cce0195efcb7a4e9507a04cd49aec9c83d0389df45b1ef7ba2e649" +[[package]] +name = "sketches-ddsketch" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c6f73aeb92d671e0cc4dca167e59b2deb6387c375391bc99ee743f326994a2b" + [[package]] name = "slab" version = "0.4.12" @@ -7398,18 +7498,18 @@ checksum = "8ed6a63f02c8539c91a8685a86f4099661ba3da017932f6ebbea6de3f0fa7c90" [[package]] name = "snafu" -version = "0.9.1" +version = "0.9.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d1a012328be2e3f5d5f6f3218147ca02588cea4cb865e876849ab6debcf36522" +checksum = "e45cb604038abb7b926b679887b3226d8d0f23874b66623625a0454be425a4b7" dependencies = [ "snafu-derive", ] [[package]] name = "snafu-derive" -version = "0.9.1" +version = "0.9.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f103c50866b8743da9429b8a581d81a27c2d3a9c4ac7df8f8571c1dd7896eda" +checksum = "287f59010008f0d7cf5e3b03196d666c1acc46c8d3e9cf34c28a1a7157601e72" dependencies = [ "heck", "proc-macro2", @@ -7447,9 +7547,15 @@ dependencies = [ [[package]] name = "spin" -version = "0.9.8" +version = "0.9.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67" +checksum = "3763264f6b73151db08c50ff20d7d8a0b8796e021cdea7ceedad07b80155fa0e" + +[[package]] +name = "spin" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "023a211cb3138dbc438680b32560ad89f699977624c9f8dbb95a47d5b4c07dd3" [[package]] name = "spki" @@ -7463,9 +7569,9 @@ dependencies = [ [[package]] name = "sqlparser" -version = "0.61.0" +version = "0.62.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dbf5ea8d4d7c808e1af1cbabebca9a2abe603bcefc22294c5b95018d53200cb7" +checksum = "13c6d1b651dc4edf07eead2a0c6c78016ce971bc2c10da5266861b13f25e7cec" dependencies = [ "log", "sqlparser_derive", @@ -7482,6 +7588,40 @@ dependencies = [ "syn 2.0.118", ] +[[package]] +name = "stabby" +version = "72.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7b834ec7ced12095fea1e4b07dcb7e8cf2b59b18afa3eac52494d835965a5ec" +dependencies = [ + "rustversion", + "stabby-abi", +] + +[[package]] +name = "stabby-abi" +version = "72.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff1a4f477858a5bdf927c9fab7f579899de9b13e39f8b3b3b300c89fbab632f4" +dependencies = [ + "rustc_version", + "rustversion", + "sha2-const-stable", + "stabby-macros", +] + +[[package]] +name = "stabby-macros" +version = "72.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b31c4b2434980b67ad83f300a58088ba14d59454dcd79ba3d87419bbd924d31e" +dependencies = [ + "proc-macro-crate", + "proc-macro2", + "quote", + "syn 2.0.118", +] + [[package]] name = "stable_deref_trait" version = "1.2.1" @@ -7531,35 +7671,13 @@ version = "0.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" -[[package]] -name = "strum" -version = "0.26.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8fec0f0aef304996cf250b31b5a10dee7980c85da9d759361292b8bca5a18f06" -dependencies = [ - "strum_macros 0.26.4", -] - [[package]] name = "strum" version = "0.28.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9628de9b8791db39ceda2b119bbe13134770b56c138ec1d3af810d045c04f9bd" dependencies = [ - "strum_macros 0.28.0", -] - -[[package]] -name = "strum_macros" -version = "0.26.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c6bee85a5a24955dc440386795aa378cd9cf82acd5f764469152d2270e581be" -dependencies = [ - "heck", - "proc-macro2", - "quote", - "rustversion", - "syn 2.0.118", + "strum_macros", ] [[package]] @@ -7576,11 +7694,12 @@ dependencies = [ [[package]] name = "substrait" -version = "0.62.2" +version = "0.63.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "62fc4b483a129b9772ccb9c3f7945a472112fdd9140da87f8a4e7f1d44e045d0" +checksum = "e620ff4d5c02fd6f7752931aa74b16a26af66a63022cc1ad412c77edbe0bab47" dependencies = [ "heck", + "indexmap 2.14.0", "pbjson", "pbjson-build", "pbjson-types", @@ -7613,9 +7732,9 @@ checksum = "a7973cce6668464ea31f176d85b13c7ab3bba2cb3b77a2ed26abd7801688010a" [[package]] name = "syn" -version = "1.0.109" +version = "2.0.118" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237" +checksum = "1b9ae57f904213ebb649ce6895b8a66c66f0203b9319718f69a5612a065b1422" dependencies = [ "proc-macro2", "quote", @@ -7624,9 +7743,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.118" +version = "3.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1b9ae57f904213ebb649ce6895b8a66c66f0203b9319718f69a5612a065b1422" +checksum = "a207d6d6a2b7fc470b80443726053f18a2481b7e1eee970597051596567987a3" dependencies = [ "proc-macro2", "quote", @@ -7713,7 +7832,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" dependencies = [ "fastrand", - "getrandom 0.4.3", + "getrandom 0.3.4", "once_cell", "rustix", "windows-sys 0.61.2", @@ -7856,9 +7975,9 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" -version = "1.52.3" +version = "1.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8fc7f01b389ac15039e4dc9531aa973a135d7a4135281b12d7c1bc79fd57fffe" +checksum = "202caea871b69668250d242070849eb495be178ed697a3e98aebce5bc81a0bed" dependencies = [ "bytes", "libc", @@ -7905,9 +8024,9 @@ dependencies = [ [[package]] name = "tokio-stream" -version = "0.1.18" +version = "0.1.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "32da49809aab5c3bc678af03902d4ccddea2a87d028d86392a4b1560c6906c70" +checksum = "a3d06f0b082ba57c26b79407372e57cf2a1e28124f78e9479fe80322cf53420b" dependencies = [ "futures-core", "pin-project-lite", @@ -7917,13 +8036,14 @@ dependencies = [ [[package]] name = "tokio-util" -version = "0.7.18" +version = "0.7.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ae9cec805b01e8fc3fd2fe289f89149a9b66dd16786abd8b19cfa7b48cb0098" +checksum = "494815d09bf52b5548659851081238f0ca39ff638363907596da739561c62c52" dependencies = [ "bytes", "futures-core", "futures-sink", + "libc", "pin-project-lite", "tokio", ] @@ -7965,7 +8085,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ac2a5518c70fa84342385732db33fb3f44bc4cc748936eb5833d2df34d6445ef" dependencies = [ "async-trait", - "base64", + "base64 0.22.1", "bytes", "h2", "http 1.4.2", @@ -8173,36 +8293,15 @@ version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" -[[package]] -name = "tstr" -version = "0.2.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f8e0294f14baae476d0dd0a2d780b2e24d66e349a9de876f5126777a37bdba7" -dependencies = [ - "tstr_proc_macros", -] - -[[package]] -name = "tstr_proc_macros" -version = "0.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e78122066b0cb818b8afd08f7ed22f7fdbc3e90815035726f0840d0d26c0747a" - [[package]] name = "twox-hash" -version = "2.1.2" +version = "2.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ea3136b675547379c4bd395ca6b938e5ad3c3d20fad76e7fe85f9e0d011419c" +checksum = "8464ec13c3691491391d9fce00f6416c9a48e46972f72d7865688be2080192c9" dependencies = [ - "rand 0.9.4", + "rand 0.10.1", ] -[[package]] -name = "typed-arena" -version = "2.0.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6af6ae20167a9ece4bcb41af5b80f8a1f1df981f6391189ce00fd257af04126a" - [[package]] name = "typenum" version = "1.20.1" @@ -8351,9 +8450,9 @@ checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" [[package]] name = "uuid" -version = "1.23.4" +version = "1.24.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bf80a72845275afea99e7f2b434723d3bc7e38470fcd1c7ed39a599c73319a53" +checksum = "bf3923a6f5c4c6382e0b653c4117f48d631ea17f38ed86e2a828e6f7412f5239" dependencies = [ "getrandom 0.4.3", "js-sys", @@ -8934,7 +9033,7 @@ checksum = "3e1e496dcbe6a09017acdfaf48e1a646735e7ff5b2a49e2c7e081cca77a59bc8" dependencies = [ "anyhow", "async-trait", - "base64", + "base64 0.22.1", "bytes", "clap", "crc32fast", @@ -8971,7 +9070,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cb838aa8eb67d730af301584cf003caad407487606058292a6750711b603fbee" dependencies = [ "async-trait", - "base64", + "base64 0.22.1", "blake3", "bytemuck", "bytes", @@ -9081,9 +9180,9 @@ checksum = "66fee0b777b0f5ac1c69bb06d361268faafa61cd4682ae064a171c16c433e9e4" [[package]] name = "xxhash-rust" -version = "0.8.16" +version = "0.8.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4d93c89cdc2d3a63c3ec48ffe926931bdc069eafa8e4402fe6d8f790c9d1e576" +checksum = "aee1b19627c7c60102ab80d3a9cbe18de90bfe03bfa6c3715447681f0e8c8af6" [[package]] name = "yoke" diff --git a/python/Cargo.toml b/python/Cargo.toml index f0c84c01a16..457ec602423 100644 --- a/python/Cargo.toml +++ b/python/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "pylance" -version = "9.0.0-beta.16" +version = "11.0.0-beta.6" edition = "2024" authors = ["Lance Devs "] license = "Apache-2.0" @@ -25,9 +25,9 @@ arrow-cast = "58.0.0" arrow-data = "58.0.0" arrow-schema = "58.0.0" object_store = "0.13.2" -datafusion = { version = "53.0.0", default-features = false } -datafusion-ffi = "53.0.0" -datafusion-common = "53.0.0" +datafusion = { version = "54.0.0", default-features = false } +datafusion-ffi = "54.0.0" +datafusion-common = "54.0.0" # Keep the Python FFI build on the working Brotli allocator resolution until # datafusion-ffi no longer enables datafusion-proto/default. # See https://github.com/lance-format/lance/issues/7271. @@ -44,6 +44,7 @@ lance = { path = "../rust/lance", features = [ "goosefs", "dynamodb", "substrait", + "metrics", ] } lance-arrow = { path = "../rust/lance-arrow" } lance-core = { path = "../rust/lance-core" } @@ -54,7 +55,7 @@ lance-index = { path = "../rust/lance-index", features = [ "tokenizer-lindera", "tokenizer-jieba", ] } -lance-io = { path = "../rust/lance-io" } +lance-io = { path = "../rust/lance-io", features = ["metrics"] } lance-linalg = { path = "../rust/lance-linalg" } lance-namespace = { path = "../rust/lance-namespace" } lance-namespace-impls = { path = "../rust/lance-namespace-impls", features = ["rest", "rest-adapter", "dir-goosefs"] } @@ -62,6 +63,8 @@ lance-table = { path = "../rust/lance-table" } lance-datafusion = { path = "../rust/lance-datafusion" } libc = "0.2.176" log = "0.4" +metrics = "0.24" +metrics-util = "0.19" prost = "0.14.1" prost-types = "0.14.1" pyo3 = { version = "0.28", features = [ @@ -80,7 +83,7 @@ serde_yaml = "0.9.34" tracing-chrome = "0.7.1" tracing-subscriber = "0.3.17" tracing = { version = "0.1" } -bytes = "1.4" +bytes = "1.11.1" [features] default = [] diff --git a/python/Makefile b/python/Makefile index d5077019f35..73a2945cfb9 100644 --- a/python/Makefile +++ b/python/Makefile @@ -1,7 +1,7 @@ .DEFAULT_GOAL := help .PHONY: help install build test integtest doctest compattest format format-python lint lint-python lint-rust clean PYTHON ?= -PYTEST_ARGS ?= -vvv -s -m "not recurring" +PYTEST_ARGS ?= -vvv -s KEEP_COMPOSE ?= 0 COMPOSE_FILE ?= ../docker-compose.yml UV_SYNC = uv sync @@ -32,7 +32,7 @@ install: ## Sync dependencies and set up local development tools build: ## Build the local Rust extension with maturin $(UV_RUN) maturin develop --uv -test: ## Run Python tests except recurring tests +test: ## Run Python tests pytest $(PYTEST_ARGS) python/tests integtest: ## Start LocalStack and run integration tests diff --git a/python/pyproject.toml b/python/pyproject.toml index dd11344324c..6c8defcab9e 100644 --- a/python/pyproject.toml +++ b/python/pyproject.toml @@ -49,21 +49,20 @@ build-backend = "maturin" [project.optional-dependencies] tests = [ "boto3", - "datasets", - "duckdb", + "datasets==4.4.0", + "duckdb>=1.5.0,<1.6.0", "ml_dtypes", "pillow", "pandas", "polars[pyarrow,pandas]", "psutil", "pytest", - # Only test tensorflow on linux for now. We will deprecate tensorflow soon. - "tensorflow; sys_platform == 'linux'", "tqdm", - "datafusion>=53,<54", + "datafusion>=54,<55", ] dev = ["ruff==0.11.2", "pyright"] benchmarks = ["pytest-benchmark"] +otel = ["opentelemetry-api", "opentelemetry-sdk"] torch = ["torch>=2.0"] geo = [ "geoarrow-rust-core", @@ -73,17 +72,17 @@ geo = [ [dependency-groups] tests = [ "boto3==1.40.43", - "datasets==4.1.1", - "duckdb==1.4.0", + "datasets==4.4.0", + "duckdb>=1.5.0,<1.6.0", "ml_dtypes==0.5.3", "pillow==11.3.0", "pandas==2.3.3", "polars[pyarrow,pandas]==1.34.0", "psutil==7.1.0", "pytest==8.4.2", - "tensorflow==2.20.0; sys_platform == 'linux'", "tqdm==4.67.1", - "datafusion==53.0.0", + "datafusion==54.0.0", + "opentelemetry-sdk==1.30.0", ] dev = [ "maturin==1.13.3", @@ -104,7 +103,7 @@ lint.select = ["F", "E", "W", "I", "G", "TCH", "PERF", "B019"] "*.pyi" = ["E301", "E302"] [tool.pyright] -pythonVersion = "3.13" +pythonVersion = "3.14" # TODO: expand this list as we fix more files. include = [ "python/lance/util.py", @@ -132,14 +131,10 @@ markers = [ "gpu: tests which rely on pytorch and some kind of gpu", "slow", "torch: tests which rely on pytorch being installed", - "recurring: marks tests as recurring tests", ] filterwarnings = [ 'error::FutureWarning', 'error::DeprecationWarning', - # TensorFlow import can emit NumPy deprecation FutureWarnings in some environments. - # We keep FutureWarnings as errors generally, but ignore this known-noisy import-time warning. - 'ignore:.*`np\\.object` will be defined as the corresponding NumPy scalar\\..*:FutureWarning', # Boto3 'ignore:.*datetime\.datetime\.utcnow\(\) is deprecated.*:DeprecationWarning', # Hugging Face Hub calls this deprecated hf-xet API internally. @@ -153,9 +148,8 @@ filterwarnings = [ 'ignore:.*the load_module\(\) method is deprecated.*:DeprecationWarning', # Pytorch uses deprecated jit.script_method internally (torch/utils/mkldnn.py) 'ignore:.*torch\.jit\.script_method.*is deprecated.*:DeprecationWarning', + # Pytorch uses the same API internally on Python 3.14+ with a different warning message. + 'ignore:.*torch\.jit\.script_method.*is not supported in Python 3\.14\+.*:DeprecationWarning', # huggingface_hub still calls the deprecated hf_xet.download_files() during Xet downloads 'ignore:.*hf_xet\.download_files\(\) is deprecated.*:DeprecationWarning', - # TensorFlow/Keras import can emit NumPy deprecation FutureWarnings in some environments. - # Keep FutureWarnings as errors generally, but ignore this known-noisy import-time warning. - 'ignore:.*np\.object.*:FutureWarning', ] diff --git a/python/python/benchmarks/test_blob.py b/python/python/benchmarks/test_blob.py new file mode 100644 index 00000000000..c2465f36fad --- /dev/null +++ b/python/python/benchmarks/test_blob.py @@ -0,0 +1,73 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright The Lance Authors + +from itertools import count +from time import process_time + +import pyarrow as pa +import pytest +from lance.file import LanceFileSession + +# Many small blobs isolate per-row Python overhead; fewer large blobs show how +# the bulk path behaves once payload copying accounts for more of the CPU time. +WORKLOADS = [ + pytest.param(50_000, 256, id="50000x256b"), + pytest.param(2_000, 64 * 1024, id="2000x64kib"), +] + + +def _packed_blob_benchmark(benchmark, tmpdir_factory, row_count, payload_size, mode): + payload = b"x" * payload_size + payloads = pa.repeat(payload, row_count) + files = LanceFileSession(str(tmpdir_factory.mktemp("packed_blob_writer"))) + file_number = count() + + if mode == "scalar_preconverted": + python_payloads = payloads.to_pylist() + + def write(): + writer = files.open_packed_blob_writer( + f"scalar-{next(file_number)}.lance", 1 + ) + for value in python_payloads: + writer.write_blob(value) + return writer.finish() + + elif mode == "scalar_from_arrow": + + def write(): + writer = files.open_packed_blob_writer( + f"scalar-arrow-{next(file_number)}.lance", 1 + ) + for value in payloads.to_pylist(): + writer.write_blob(value) + return writer.finish() + + elif mode == "bulk": + + def write(): + writer = files.open_packed_blob_writer(f"bulk-{next(file_number)}.lance", 1) + writer.write_blobs(payloads) + return writer.finish_array("blob") + + else: + raise ValueError(f"Unknown benchmark mode: {mode}") + + result = benchmark.pedantic(write, iterations=1, rounds=5) + assert len(result) == row_count + + +@pytest.mark.benchmark(group="packed_blob_writer_cpu", timer=process_time) +@pytest.mark.parametrize("row_count,payload_size", WORKLOADS) +@pytest.mark.parametrize( + "mode", + ["scalar_preconverted", "scalar_from_arrow", "bulk"], +) +def test_packed_blob_writer(benchmark, tmpdir_factory, row_count, payload_size, mode): + _packed_blob_benchmark( + benchmark, + tmpdir_factory, + row_count, + payload_size, + mode, + ) diff --git a/python/python/benchmarks/zone_map_seeds.py b/python/python/benchmarks/zone_map_seeds.py new file mode 100644 index 00000000000..023e14cfcf7 --- /dev/null +++ b/python/python/benchmarks/zone_map_seeds.py @@ -0,0 +1,351 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright The Lance Authors +""" +Benchmark: Zone Map Index Seeds vs. No Seeds + +Measures the time to update a zone map index when new data is appended to an +already-indexed dataset, with and without use_seeds. + +Seeds allow Lance to embed per-zone min/max statistics in data files at write +time. When optimize_indices() runs, it harvests those pre-computed stats +instead of re-scanning the full column, which eliminates the read I/O for +wide data types. + +WHY the workflow matters +------------------------ +Seeds only engage when merging new unindexed fragments into an existing +non-empty index segment. A first optimize_indices() call on a freshly +created empty-index always does a full rebuild (no old fragments to merge +against), so seeds would never be used. + +Correct workflow: + 1. Write an initial batch of data and build the zone map index on it. + This creates a non-empty indexed segment; all subsequent writes will + embed seed buffers in their data files (when use_seeds=True). + 2. Append the bulk of the data in multiple batches. + 3. optimize_indices() — harvests seeds (phase 2 data) and merges them into + the segment from phase 1. Without seeds, phase 3 must re-read all of + the phase-2 data from disk. + +Phases timed: + initial_write write seed_fraction of total rows + create/build index + ingest append remaining rows (seeds collected here when enabled) + update_index optimize_indices() — seed harvest happens here + total sum of the three phases above + +Three scenarios: + int64 10 M rows ~80 MB on disk + FixedSizeBinary(4096) 10 M rows ~38 GB on disk + LargeBinary(20 KiB) 1 M rows ~19 GB on disk + +Usage: + uv run python python/benchmarks/zone_map_seeds.py + uv run python python/benchmarks/zone_map_seeds.py --tmpdir /fast/nvme/bench + uv run python python/benchmarks/zone_map_seeds.py --scale 0.01 +""" + +import argparse +import os +import shutil +import tempfile +import time +from pathlib import Path + +import lance +import numpy as np +import pyarrow as pa +from lance.indices import IndexConfig + +KiB = 1024 + +# Fraction of total rows written in the initial indexed batch. +SEED_FRACTION = 0.1 + + +# --------------------------------------------------------------------------- +# Data generators (yield pa.Table one batch at a time) +# --------------------------------------------------------------------------- + + +def gen_int_batches(num_rows: int, batch_size: int): + rng = np.random.default_rng() + for start in range(0, num_rows, batch_size): + n = min(batch_size, num_rows - start) + yield pa.table({"value": pa.array(rng.integers(0, 2**31, n, dtype=np.int64))}) + + +def gen_fsb_batches(num_rows: int, blob_bytes: int, batch_size: int): + for start in range(0, num_rows, batch_size): + n = min(batch_size, num_rows - start) + raw = os.urandom(n * blob_bytes) + arr = pa.FixedSizeBinaryArray.from_buffers( + pa.binary(blob_bytes), n, [None, pa.py_buffer(raw)] + ) + yield pa.table({"value": arr}) + + +def gen_large_binary_batches(num_rows: int, blob_bytes: int, batch_size: int): + for start in range(0, num_rows, batch_size): + n = min(batch_size, num_rows - start) + yield pa.table( + { + "value": pa.array( + [os.urandom(blob_bytes) for _ in range(n)], + type=pa.large_binary(), + ) + } + ) + + +# --------------------------------------------------------------------------- +# Core benchmark runner +# --------------------------------------------------------------------------- + + +def run_scenario( + dataset_path: Path, + schema: pa.Schema, + seed_rows: int, + seed_batch_size: int, + gen_seed_batches, # () -> Iterator[pa.Table] for the initial write + gen_bulk_batches, # () -> Iterator[pa.Table] for the appended data + use_seeds: bool, +) -> dict: + """ + Run one full scenario and return per-phase timings. + + Seeds are only engaged by optimize_indices() when there is at least one + previously indexed fragment. This scenario ensures that by writing + seed_rows of data and building the index before any bulk ingest. + """ + if dataset_path.exists(): + shutil.rmtree(dataset_path) + + index_config = IndexConfig( + index_type="ZONEMAP", parameters={"use_seeds": use_seeds} + ) + timings = {} + + # Phase 1: initial write + index build + t0 = time.perf_counter() + ds = None + for batch in gen_seed_batches(): + if ds is None: + ds = lance.write_dataset(batch, dataset_path, schema=schema) + else: + ds = lance.write_dataset(batch, dataset_path, mode="append") + ds.create_scalar_index("value", index_type=index_config, replace=True) + timings["initial_write"] = time.perf_counter() - t0 + + # Phase 2: bulk ingest — seeds are collected per-batch when enabled + t0 = time.perf_counter() + for batch in gen_bulk_batches(): + ds = lance.write_dataset(batch, dataset_path, mode="append") + timings["ingest"] = time.perf_counter() - t0 + + # Phase 3: index update — seed harvest happens here + t0 = time.perf_counter() + ds.optimize.optimize_indices() + timings["update_index"] = time.perf_counter() - t0 + + timings["total"] = sum(timings.values()) + shutil.rmtree(dataset_path) + return timings + + +# --------------------------------------------------------------------------- +# Output formatting +# --------------------------------------------------------------------------- + + +def print_comparison(title: str, no_seeds: dict, with_seeds: dict) -> None: + phases = ["initial_write", "ingest", "update_index", "total"] + col_w = 13 + + print(f"\n{'=' * 70}") + print(f" {title}") + print(f"{'=' * 70}") + print( + f" {'Phase':<20} {'No Seeds':>{col_w}} " + + f"{'With Seeds':>{col_w}} {'Speedup':>{col_w}}" + ) + print(f" {'-' * (20 + col_w * 3 + 6)}") + for phase in phases: + t_no = no_seeds[phase] + t_yes = with_seeds[phase] + speedup = t_no / t_yes if t_yes > 0 else float("inf") + marker = "* " if phase == "total" else " " + print( + f"{marker}{phase:<20} {t_no:>{col_w}.2f}s" + f" {t_yes:>{col_w}.2f}s" + f" {speedup:>{col_w - 1}.2f}x" + ) + + +# --------------------------------------------------------------------------- +# Per-type benchmark wrappers +# --------------------------------------------------------------------------- + + +def bench_integers(base: Path, num_rows: int) -> None: + seed_rows = max(1, int(num_rows * SEED_FRACTION)) + bulk_rows = num_rows - seed_rows + # ~400 KB per seed batch; ~4 MB per bulk batch → ~20 bulk fragments + seed_bs = max(1, seed_rows // 5) + bulk_bs = max(1, bulk_rows // 20) + print( + f"\nInteger (int64): {num_rows:,} rows " + f"(initial={seed_rows:,}, bulk={bulk_rows:,}) ..." + ) + schema = pa.schema([pa.field("value", pa.int64())]) + + no_seeds = run_scenario( + base / "int_no_seeds", + schema, + seed_rows, + seed_bs, + lambda: gen_int_batches(seed_rows, seed_bs), + lambda: gen_int_batches(bulk_rows, bulk_bs), + use_seeds=False, + ) + with_seeds = run_scenario( + base / "int_with_seeds", + schema, + seed_rows, + seed_bs, + lambda: gen_int_batches(seed_rows, seed_bs), + lambda: gen_int_batches(bulk_rows, bulk_bs), + use_seeds=True, + ) + print_comparison(f"int64 — {num_rows:,} rows", no_seeds, with_seeds) + + +def bench_vectors(base: Path, num_rows: int) -> None: + blob_bytes = 4 * KiB + seed_rows = max(1, int(num_rows * SEED_FRACTION)) + bulk_rows = num_rows - seed_rows + total_gb = num_rows * blob_bytes / (1024**3) + # ~200 MB per batch to stay memory-friendly + seed_bs = max(1, (200 * KiB * KiB) // blob_bytes) + bulk_bs = seed_bs + print( + f"\nVector (FixedSizeBinary {blob_bytes // KiB} KiB): {num_rows:,} rows" + f" (~{total_gb:.1f} GB) (initial={seed_rows:,}, bulk={bulk_rows:,}) ..." + ) + schema = pa.schema([pa.field("value", pa.binary(blob_bytes))]) + + no_seeds = run_scenario( + base / "vec_no_seeds", + schema, + seed_rows, + seed_bs, + lambda: gen_fsb_batches(seed_rows, blob_bytes, seed_bs), + lambda: gen_fsb_batches(bulk_rows, blob_bytes, bulk_bs), + use_seeds=False, + ) + with_seeds = run_scenario( + base / "vec_with_seeds", + schema, + seed_rows, + seed_bs, + lambda: gen_fsb_batches(seed_rows, blob_bytes, seed_bs), + lambda: gen_fsb_batches(bulk_rows, blob_bytes, bulk_bs), + use_seeds=True, + ) + print_comparison( + f"FixedSizeBinary({blob_bytes // KiB} KiB) — {num_rows:,} rows", + no_seeds, + with_seeds, + ) + + +def bench_large_binary(base: Path, num_rows: int) -> None: + blob_bytes = 20 * KiB + seed_rows = max(1, int(num_rows * SEED_FRACTION)) + bulk_rows = num_rows - seed_rows + total_gb = num_rows * blob_bytes / (1024**3) + # ~200 MB per batch + seed_bs = max(1, (200 * KiB * KiB) // blob_bytes) + bulk_bs = seed_bs + print( + f"\nLargeBinary ({blob_bytes // KiB} KiB blobs): {num_rows:,} rows" + f" (~{total_gb:.1f} GB) (initial={seed_rows:,}, bulk={bulk_rows:,}) ..." + ) + schema = pa.schema([pa.field("value", pa.large_binary())]) + + no_seeds = run_scenario( + base / "bin_no_seeds", + schema, + seed_rows, + seed_bs, + lambda: gen_large_binary_batches(seed_rows, blob_bytes, seed_bs), + lambda: gen_large_binary_batches(bulk_rows, blob_bytes, bulk_bs), + use_seeds=False, + ) + with_seeds = run_scenario( + base / "bin_with_seeds", + schema, + seed_rows, + seed_bs, + lambda: gen_large_binary_batches(seed_rows, blob_bytes, seed_bs), + lambda: gen_large_binary_batches(bulk_rows, blob_bytes, bulk_bs), + use_seeds=True, + ) + print_comparison( + f"LargeBinary({blob_bytes // KiB} KiB) — {num_rows:,} rows", + no_seeds, + with_seeds, + ) + + +# --------------------------------------------------------------------------- +# Entry point +# --------------------------------------------------------------------------- + + +def main() -> None: + parser = argparse.ArgumentParser( + description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter + ) + parser.add_argument( + "--tmpdir", + type=Path, + default=None, + help="Directory for benchmark datasets (default: system temp dir)", + ) + parser.add_argument( + "--scale", + type=float, + default=1.0, + help="Row-count scale factor (default 1.0; use 0.01 for a quick smoke test)", + ) + args = parser.parse_args() + + int_rows = max(2, int(10_000_000 * args.scale)) + vec_rows = max(2, int(10_000_000 * args.scale)) + bin_rows = max(2, int(1_000_000 * args.scale)) + + print( + f"Scale: {args.scale} → " + f"{int_rows:,} int / {vec_rows:,} vector / {bin_rows:,} binary rows" + ) + + def run(tmp_dir: Path) -> None: + bench_integers(tmp_dir, int_rows) + bench_vectors(tmp_dir, vec_rows) + bench_large_binary(tmp_dir, bin_rows) + + if args.tmpdir is not None: + args.tmpdir.mkdir(parents=True, exist_ok=True) + with tempfile.TemporaryDirectory( + prefix="lance_zonemap_bench_", dir=args.tmpdir + ) as tmp: + run(Path(tmp)) + else: + with tempfile.TemporaryDirectory(prefix="lance_zonemap_bench_") as tmp: + run(Path(tmp)) + + +if __name__ == "__main__": + main() diff --git a/python/python/ci_benchmarks/benchmark.py b/python/python/ci_benchmarks/benchmark.py index 7d80596e305..8d17bef5ae5 100644 --- a/python/python/ci_benchmarks/benchmark.py +++ b/python/python/ci_benchmarks/benchmark.py @@ -20,7 +20,7 @@ def workload(dataset): import json from dataclasses import dataclass -from typing import Any, Callable, List +from typing import Any, Callable, List, Optional import pytest @@ -90,6 +90,7 @@ def __call__( func: Callable, dataset: Any, warmup: bool = True, + setup: Optional[Callable[[], Any]] = None, ) -> Any: """ Run a benchmark function with IO and memory tracking. @@ -102,28 +103,47 @@ def __call__( The dataset to pass to the function. warmup : bool, default True Whether to run a warmup iteration before measuring. + setup : Callable, optional + Called before the warmup run and again before the measured run. + Required for workloads that mutate the dataset, so each run starts + from the same state. Setup runs before IO stats are reset, so its + own IO is never attributed to the benchmark. If it returns a value, + that value is passed to ``func`` instead of ``dataset`` -- use this + when the reset produces a new dataset handle. Returns ------- Any The return value of the benchmark function. """ + + def prepare() -> Any: + if setup is None: + return dataset + replacement = setup() + return dataset if replacement is None else replacement + # Warmup run (not measured) if warmup: - func(dataset) + func(prepare()) + + target = prepare() - # Reset IO stats before the measured run + # Reset IO stats before the measured run. Stats are tracked on the + # ObjectStore, which is shared by every handle from the same session, + # so resetting/reading through `dataset` also captures work that `func` + # performs through a handle returned by `setup`. dataset.io_stats_incremental() # Run with memory tracking if available if MEMTEST_AVAILABLE: memtest.reset_stats() - result = func(dataset) + result = func(target) mem_stats = memtest.get_stats() self._stats.peak_bytes = mem_stats["peak_bytes"] self._stats.total_allocations = mem_stats["total_allocations"] else: - result = func(dataset) + result = func(target) # Capture IO stats io_stats = dataset.io_stats_incremental() @@ -159,7 +179,11 @@ def workload(dataset): if marker is None: # Not an io_memory_benchmark test, return a simple passthrough class PassthroughBenchmark: - def __call__(self, func, dataset, warmup=True): + def __call__(self, func, dataset, warmup=True, setup=None): + if setup is not None: + replacement = setup() + if replacement is not None: + dataset = replacement return func(dataset) yield PassthroughBenchmark() diff --git a/python/python/ci_benchmarks/benchmarks/test_merge_insert.py b/python/python/ci_benchmarks/benchmarks/test_merge_insert.py new file mode 100644 index 00000000000..04a5ec02c35 --- /dev/null +++ b/python/python/ci_benchmarks/benchmarks/test_merge_insert.py @@ -0,0 +1,780 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright The Lance Authors + +"""Benchmarks for `merge_insert`. + +merge_insert has two execution paths and the routing between them is +structural, not a flag. `use_index` is the one knob that selects between them, +so every benchmark here is parametrized on it: + + ``v1_indexed`` — ``use_index(True)`` with an indexed key. Takes the legacy + indexed-scan path (``create_indexed_scan_joined_stream``). + ``v2_hash`` — ``use_index(False)``. Disables the index gate in + ``can_use_create_plan``, so the DataFusion path + (``LanceRead + HashJoin``) runs instead. + +For a partial-schema source the same knob also selects the write sink: v1 +patches columns in place (``UpdateMode::RewriteColumns``) while v2 rewrites +whole rows (``RewriteRows``). That makes ``write_bytes`` the interesting +metric for the ``test_update_*`` benchmarks. + +Targets are mutated, so each measured run is preceded by an untimed restore to +the ``merge_insert_base`` tag written by ``datagen/merge_insert.py``. +""" + +from __future__ import annotations + +from dataclasses import dataclass +from datetime import timedelta +from typing import TYPE_CHECKING, Callable, Iterable, Optional, Sequence + +import lance +import numpy as np +import pyarrow as pa +import pytest +from ci_benchmarks.datagen.merge_insert import ( + BASE_TAG, + DELETED_ROW_STRIDE, + FRAGS_NUM_ROWS, + FRAGS_SCHEMA, + NARROW_NUM_ROWS, + NARROW_SCHEMA, + UNINDEXED_TAIL_ROWS, + WIDE_NUM_ROWS, + WIDE_ROWS_PER_FRAGMENT, + WIDE_SCALAR_COLUMNS, + WIDE_SCHEMA, + WIDE_VECTOR_DIM, + narrow_batch, +) +from ci_benchmarks.datasets import get_dataset_uri + +if TYPE_CHECKING: + from lance.dataset import ExecuteResult + +PLANS = ["v1_indexed", "v2_hash"] + +# Brackets the cold-random break-even, which the design analysis puts at +# roughly target_rows / 4096 -- about 2.4K rows for the 10M-row narrow target. +SOURCE_SIZES = [1_000, 10_000, 100_000] + +NARROW_KEYS = ["id_int", "id_uuid7", "id_uuid4"] + +# DataFusion accounts each 8,192-row slice of the 1.32 GiB full-schema source +# for its shared backing buffers. Budget the resulting ~159 GiB of reservations +# without changing the source layout measured by this benchmark. +_MEM_POOL_SIZE = 160 * 1024**3 + + +@pytest.fixture(scope="module", autouse=True) +def _merge_insert_mem_pool() -> Iterable[None]: + """Use a bounded pool large enough for every merge_insert benchmark.""" + with pytest.MonkeyPatch.context() as monkeypatch: + monkeypatch.setenv("LANCE_MEM_POOL_SIZE", str(_MEM_POOL_SIZE)) + yield + + +# --------------------------------------------------------------------------- +# Target management +# --------------------------------------------------------------------------- + + +@dataclass +class Target: + """A merge_insert target that can be rewound to a pristine state. + + ``reset`` returns a handle sitting on a fresh version whose contents match + the ``merge_insert_base`` tag. It is called from an untimed benchmark + setup hook, so the restore never lands in the measurement. + """ + + uri: str + dataset: lance.LanceDataset + base_version: int + + def reset(self, cold: bool = False) -> lance.LanceDataset: + if cold: + # A new session means an empty index cache. `checkout_version` + # deliberately shares the cache, so a cold run cannot reuse + # `self.dataset`. + handle = lance.dataset(self.uri) + else: + handle = self.dataset + reverted = handle.checkout_version(self.base_version) + reverted.restore() + return reverted + + +def _open_target(name: str) -> Iterable[Target]: + uri = get_dataset_uri(name) + dataset = lance.dataset(uri) + tags = dataset.tags.list() + if BASE_TAG not in tags: + pytest.skip( + f"Dataset {name} has no {BASE_TAG} tag; " + "run python/ci_benchmarks/datagen/gen_all.py" + ) + base_version = tags[BASE_TAG]["version"] + + yield Target(uri=uri, dataset=dataset, base_version=base_version) + + # Leave the dataset pristine and drop the versions and data files the + # benchmarks produced. Cleanup never deletes a tagged version, so the tag + # has to move onto the restored version first or the old base would be + # retained forever. + final = dataset.checkout_version(base_version) + final.restore() + final.tags.update(BASE_TAG, final.version) + final.cleanup_old_versions(older_than=timedelta(0), delete_unverified=True) + + +@pytest.fixture(scope="module") +def narrow() -> Iterable[Target]: + yield from _open_target("merge_insert_narrow") + + +@pytest.fixture(scope="module") +def wide() -> Iterable[Target]: + yield from _open_target("merge_insert_wide") + + +@pytest.fixture(scope="module") +def frags() -> Iterable[Target]: + yield from _open_target("merge_insert_frags") + + +@pytest.fixture(scope="module") +def deleted() -> Iterable[Target]: + yield from _open_target("merge_insert_deleted") + + +@pytest.fixture(scope="module") +def unindexed_tail() -> Iterable[Target]: + yield from _open_target("merge_insert_unindexed_tail") + + +# --------------------------------------------------------------------------- +# Source construction +# --------------------------------------------------------------------------- +# +# Sources are built from a contiguous run of row indices. Which key column the +# merge joins on then decides whether those keys are clustered or scattered in +# the index's key order: `id_int` and `id_uuid7` are monotonic in the row index, +# `id_uuid4` scrambles it. This keeps one source builder for all key shapes. + + +def narrow_source(row_indices: np.ndarray) -> pa.Table: + """A full-schema narrow source. ``value`` is offset so updates are real.""" + return pa.Table.from_batches([narrow_batch(row_indices, value_offset=1)]) + + +def existing_rows(num_rows: int, offset: int = 0) -> np.ndarray: + return np.arange(offset, offset + num_rows, dtype=np.int64) + + +def new_rows(num_rows: int) -> np.ndarray: + """Row indices past the end of the target, so every key is a new key.""" + return np.arange(NARROW_NUM_ROWS, NARROW_NUM_ROWS + num_rows, dtype=np.int64) + + +def wide_row_indices(fraction: float) -> np.ndarray: + """Row indices covering ``fraction`` of every fragment's rows. + + Spread evenly within each fragment rather than contiguously: that is the + harder case for the in-place column updater, which has to interleave + updated and untouched values. + """ + per_fragment = max(1, round(fraction * WIDE_ROWS_PER_FRAGMENT)) + num_fragments = WIDE_NUM_ROWS // WIDE_ROWS_PER_FRAGMENT + return np.concatenate( + [ + fragment * WIDE_ROWS_PER_FRAGMENT + + np.linspace(0, WIDE_ROWS_PER_FRAGMENT - 1, per_fragment, dtype=np.int64) + for fragment in range(num_fragments) + ] + ) + + +def wide_source(row_indices: np.ndarray, columns: Sequence[str]) -> pa.Table: + """A wide source carrying ``id_int`` plus ``columns``.""" + arrays = [pa.array(row_indices)] + names = ["id_int"] + for column in columns: + names.append(column) + if column == "vec": + values = np.linspace( + 1.0, + 2.0, + num=len(row_indices) * WIDE_VECTOR_DIM, + dtype=np.float32, + ) + arrays.append( + pa.FixedSizeListArray.from_arrays(pa.array(values), WIDE_VECTOR_DIM) + ) + elif WIDE_SCHEMA.field(column).type == pa.string(): + arrays.append( + pa.array([f"updated_{column}_{v}" for v in row_indices], pa.string()) + ) + else: + arrays.append(pa.array(row_indices + 1)) + return pa.table(arrays, schema=pa.schema([WIDE_SCHEMA.field(n) for n in names])) + + +# --------------------------------------------------------------------------- +# Runner +# --------------------------------------------------------------------------- + + +def run( + benchmark, + target: Target, + job: Callable[[lance.LanceDataset], ExecuteResult], + *, + rounds: int = 3, + cold: bool = False, + warmup: bool = True, + expected_rows: Optional[int] = None, +) -> None: + """Benchmark ``job``, restoring the target before every round. + + ``cold`` opens a fresh session for each round so the index cache starts + empty. ``warmup`` runs one unmeasured round first; turn it off for + expensive shapes where a second pass is not worth the wall clock. + + ``expected_rows`` guards against a benchmark that silently stops doing + work: a semantics regression should fail the run, not post a fast time. + Row count is checked instead of the returned stats because it is + independent of which execution path ran. + """ + state: dict = {} + + def setup() -> None: + state["dataset"] = target.reset(cold=cold) + + def bench() -> None: + dataset = state["dataset"] + state["stats"] = job(dataset) + if expected_rows is not None: + state["row_count"] = dataset.count_rows() + + benchmark.pedantic( + bench, + setup=setup, + rounds=rounds, + iterations=1, + # `setup` runs before each warmup round too, so a warmup never leaves + # the target dirty for the measured rounds. + warmup_rounds=1 if warmup and not cold else 0, + ) + + if expected_rows is not None: + assert state["row_count"] == expected_rows, ( + f"expected {expected_rows} rows after merge_insert, " + f"got {state['row_count']} (stats: {state['stats']})" + ) + + +def upsert( + key: str, source: pa.Table, *, use_index: bool +) -> Callable[[lance.LanceDataset], ExecuteResult]: + def job(dataset: lance.LanceDataset) -> ExecuteResult: + return ( + dataset.merge_insert(key) + .when_matched_update_all() + .when_not_matched_insert_all() + .use_index(use_index) + .execute(source) + ) + + return job + + +def uses_index(plan: str) -> bool: + return plan == "v1_indexed" + + +# --------------------------------------------------------------------------- +# A. Cost model core -- merge_insert_narrow, 10M rows +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize("plan", PLANS) +def test_upsert_point(benchmark, narrow: Target, plan: str) -> None: + """Single-row upsert latency -- the extreme where a probe should win.""" + source = narrow_source(existing_rows(1)) + run( + benchmark, + narrow, + upsert("id_int", source, use_index=uses_index(plan)), + rounds=5, + expected_rows=NARROW_NUM_ROWS, + ) + + +@pytest.mark.parametrize("plan", PLANS) +@pytest.mark.parametrize("key", NARROW_KEYS) +@pytest.mark.parametrize("num_rows", SOURCE_SIZES) +def test_upsert_ratio_sweep( + benchmark, narrow: Target, num_rows: int, key: str, plan: str +) -> None: + """Source/target ratio sweep against a warm index.""" + source = narrow_source(existing_rows(num_rows)) + run( + benchmark, + narrow, + upsert(key, source, use_index=uses_index(plan)), + expected_rows=NARROW_NUM_ROWS, + ) + + +@pytest.mark.parametrize("plan", PLANS) +@pytest.mark.parametrize("key", ["id_int", "id_uuid4"]) +@pytest.mark.parametrize("num_rows", [1_000, 100_000]) +def test_upsert_cold_ratio_sweep( + benchmark, narrow: Target, num_rows: int, key: str, plan: str +) -> None: + """Same sweep with a cold index cache, where page reads are not free.""" + source = narrow_source(existing_rows(num_rows)) + run( + benchmark, + narrow, + upsert(key, source, use_index=uses_index(plan)), + cold=True, + expected_rows=NARROW_NUM_ROWS, + ) + + +@pytest.mark.parametrize("plan", PLANS) +@pytest.mark.parametrize("num_rows", [10_000, 100_000]) +def test_upsert_all_new_keys( + benchmark, narrow: Target, num_rows: int, plan: str +) -> None: + """Time-ordered ingest: every key is new, so no index page should be read.""" + source = narrow_source(new_rows(num_rows)) + run( + benchmark, + narrow, + upsert("id_uuid7", source, use_index=uses_index(plan)), + expected_rows=NARROW_NUM_ROWS + num_rows, + ) + + +@pytest.mark.parametrize("num_rows", [1_000, 100_000]) +def test_upsert_unindexed_baseline(benchmark, narrow: Target, num_rows: int) -> None: + """Joining on a column with no index at all -- the no-index reference.""" + source = narrow_source(existing_rows(num_rows)) + run( + benchmark, + narrow, + upsert("id_no_index", source, use_index=True), + expected_rows=NARROW_NUM_ROWS, + ) + + +# --------------------------------------------------------------------------- +# B. Write path -- merge_insert_wide, 1M rows +# --------------------------------------------------------------------------- + +# Fractions are of each fragment's rows, which is what decides whether the +# in-place updater interleaves or rewrites a whole column file. +ROW_FRACTIONS = [0.001, 0.01, 0.1, 1.0] +FRACTION_IDS = ["0.1pct", "1pct", "10pct", "100pct"] + +# Isolates the two pressures a partial-column update can exert: field count +# (scalar columns, cheap per field) and byte volume (the vector column). +PROJECTIONS = { + "one_scalar": WIDE_SCALAR_COLUMNS[:1], + "ten_scalars": WIDE_SCALAR_COLUMNS[:10], + "vector": ["vec"], + "vector_and_ten_scalars": ["vec"] + WIDE_SCALAR_COLUMNS[:10], +} + + +def _wide_rounds(fraction: float) -> int: + # A full-fragment update rewrites every column file it touches, which for + # the vector column is ~1 GB per round. + return 1 if fraction == 1.0 else 3 + + +def update_subset( + source: pa.Table, *, use_index: bool +) -> Callable[[lance.LanceDataset], ExecuteResult]: + """Partial-schema update. No insert clause: matched rows only.""" + + def job(dataset: lance.LanceDataset) -> ExecuteResult: + return ( + dataset.merge_insert("id_int") + .when_matched_update_all() + .use_index(use_index) + .execute(source) + ) + + return job + + +@pytest.mark.parametrize("plan", PLANS) +@pytest.mark.parametrize("fraction", ROW_FRACTIONS, ids=FRACTION_IDS) +def test_update_subset_row_fraction( + benchmark, wide: Target, fraction: float, plan: str +) -> None: + """Row-fraction sweep at a fixed, minimal projection. + + Locates the crossover between patching a column in place and rewriting the + whole column file, without byte volume confounding it. + """ + source = wide_source(wide_row_indices(fraction), PROJECTIONS["one_scalar"]) + run( + benchmark, + wide, + update_subset(source, use_index=uses_index(plan)), + rounds=_wide_rounds(fraction), + warmup=fraction != 1.0, + expected_rows=WIDE_NUM_ROWS, + ) + + +@pytest.mark.parametrize("plan", PLANS) +@pytest.mark.parametrize("fraction", [0.01, 1.0], ids=["1pct", "100pct"]) +@pytest.mark.parametrize("projection", list(PROJECTIONS), ids=list(PROJECTIONS)) +def test_update_subset_projection( + benchmark, wide: Target, projection: str, fraction: float, plan: str +) -> None: + """Projection sweep: field count vs byte volume, plus the cross term.""" + source = wide_source(wide_row_indices(fraction), PROJECTIONS[projection]) + run( + benchmark, + wide, + update_subset(source, use_index=uses_index(plan)), + rounds=_wide_rounds(fraction), + warmup=fraction != 1.0, + expected_rows=WIDE_NUM_ROWS, + ) + + +@pytest.mark.parametrize("plan", PLANS) +@pytest.mark.parametrize("fraction", [0.01, 1.0], ids=["1pct", "100pct"]) +def test_upsert_wide_full_schema( + benchmark, wide: Target, fraction: float, plan: str +) -> None: + """Full-schema baseline for the partial-column benchmarks above.""" + row_indices = wide_row_indices(fraction) + source = wide_source( + row_indices, [f.name for f in WIDE_SCHEMA if f.name != "id_int"] + ) + run( + benchmark, + wide, + upsert("id_int", source, use_index=uses_index(plan)), + rounds=_wide_rounds(fraction), + warmup=fraction != 1.0, + expected_rows=WIDE_NUM_ROWS, + ) + + +# --------------------------------------------------------------------------- +# C. Clause shapes -- merge_insert_narrow, 10K-row source +# --------------------------------------------------------------------------- + +CLAUSE_SOURCE_ROWS = 10_000 + + +@pytest.mark.parametrize("plan", PLANS) +def test_insert_if_not_exists(benchmark, narrow: Target, plan: str) -> None: + """Dedup ingest: half the keys already exist, matched rows are untouched. + + The probe only needs to know whether a key exists, so no target payload has + to be read. + """ + half = CLAUSE_SOURCE_ROWS // 2 + row_indices = np.concatenate([existing_rows(half), new_rows(half)]) + source = narrow_source(row_indices) + + def job(dataset: lance.LanceDataset) -> ExecuteResult: + return ( + dataset.merge_insert("id_int") + .when_not_matched_insert_all() + .use_index(uses_index(plan)) + .execute(source) + ) + + run(benchmark, narrow, job, expected_rows=NARROW_NUM_ROWS + half) + + +@pytest.mark.parametrize("plan", PLANS) +def test_update_only(benchmark, narrow: Target, plan: str) -> None: + """No insert clause: unmatched source rows are dropped.""" + half = CLAUSE_SOURCE_ROWS // 2 + row_indices = np.concatenate([existing_rows(half), new_rows(half)]) + source = narrow_source(row_indices) + + def job(dataset: lance.LanceDataset) -> ExecuteResult: + return ( + dataset.merge_insert("id_int") + .when_matched_update_all() + .use_index(uses_index(plan)) + .execute(source) + ) + + run(benchmark, narrow, job, expected_rows=NARROW_NUM_ROWS) + + +def test_delete_by_source(benchmark, narrow: Target) -> None: + """Deleting rows absent from the source requires a full target scan. + + The index gate in `can_use_create_plan` rejects this shape outright, so + there is no v1 variant to compare against. + """ + source = narrow_source(existing_rows(CLAUSE_SOURCE_ROWS)) + + def job(dataset: lance.LanceDataset) -> ExecuteResult: + return ( + dataset.merge_insert("id_int") + .when_matched_update_all() + .when_not_matched_insert_all() + .when_not_matched_by_source_delete() + .execute(source) + ) + + run(benchmark, narrow, job, expected_rows=CLAUSE_SOURCE_ROWS) + + +@pytest.mark.parametrize("plan", PLANS) +def test_conditional_update(benchmark, narrow: Target, plan: str) -> None: + """A condition on target columns forces the target payload to be read.""" + source = narrow_source(existing_rows(CLAUSE_SOURCE_ROWS)) + + def job(dataset: lance.LanceDataset) -> ExecuteResult: + return ( + dataset.merge_insert("id_int") + .when_matched_update_all(condition="source.value > target.value") + .when_not_matched_insert_all() + .use_index(uses_index(plan)) + .execute(source) + ) + + run(benchmark, narrow, job, expected_rows=NARROW_NUM_ROWS) + + +@pytest.mark.parametrize("plan", PLANS) +def test_composite_key_fully_indexed(benchmark, narrow: Target, plan: str) -> None: + """Both key columns indexed: v1 probes each index and AND-folds.""" + source = narrow_source(existing_rows(CLAUSE_SOURCE_ROWS)) + + def job(dataset: lance.LanceDataset) -> ExecuteResult: + return ( + dataset.merge_insert(["composite_a", "composite_b"]) + .when_matched_update_all() + .when_not_matched_insert_all() + .use_index(uses_index(plan)) + .execute(source) + ) + + run(benchmark, narrow, job, expected_rows=NARROW_NUM_ROWS) + + +def test_composite_key_partially_indexed(benchmark, narrow: Target) -> None: + """One key column unindexed, which forces the hash join regardless of flag. + + A partial index probe would under-match, so `can_use_create_plan` keeps + this shape off the indexed path. + """ + source = narrow_source(existing_rows(CLAUSE_SOURCE_ROWS)) + + def job(dataset: lance.LanceDataset) -> ExecuteResult: + # `composite_a` is indexed, `id_no_index` is not. Both are equal to the + # row index in source and target, so every source row matches. + return ( + dataset.merge_insert(["composite_a", "id_no_index"]) + .when_matched_update_all() + .when_not_matched_insert_all() + .execute(source) + ) + + run(benchmark, narrow, job, expected_rows=NARROW_NUM_ROWS) + + +# --------------------------------------------------------------------------- +# D. Target shape +# --------------------------------------------------------------------------- + +TARGET_SHAPE_ROWS = 10_000 + + +@pytest.mark.parametrize("plan", PLANS) +def test_upsert_unindexed_tail(benchmark, unindexed_tail: Target, plan: str) -> None: + """10% of the target was appended after indexing, so a scan is unioned in.""" + source = narrow_source(existing_rows(TARGET_SHAPE_ROWS)) + total_rows = NARROW_NUM_ROWS + UNINDEXED_TAIL_ROWS + run( + benchmark, + unindexed_tail, + upsert("id_int", source, use_index=uses_index(plan)), + expected_rows=total_rows, + ) + + +@pytest.mark.parametrize("plan", PLANS) +def test_upsert_with_deletion_files(benchmark, deleted: Target, plan: str) -> None: + """Every fragment carries a deletion file, which the probe has to mask.""" + source = narrow_source(existing_rows(TARGET_SHAPE_ROWS)) + # Row 0 and every DELETED_ROW_STRIDE-th row after it were deleted at + # generation time, so those source rows insert rather than update. + reinserted = TARGET_SHAPE_ROWS // DELETED_ROW_STRIDE + total_rows = NARROW_NUM_ROWS - NARROW_NUM_ROWS // DELETED_ROW_STRIDE + reinserted + run( + benchmark, + deleted, + upsert("id_int", source, use_index=uses_index(plan)), + expected_rows=total_rows, + ) + + +@pytest.mark.parametrize("plan", PLANS) +def test_upsert_many_small_fragments(benchmark, frags: Target, plan: str) -> None: + """10K fragments of 1K rows, to expose per-fragment overhead.""" + row_indices = existing_rows(TARGET_SHAPE_ROWS) + source = pa.table( + [pa.array(row_indices), pa.array(row_indices + 1)], schema=FRAGS_SCHEMA + ) + run( + benchmark, + frags, + upsert("id_int", source, use_index=uses_index(plan)), + expected_rows=FRAGS_NUM_ROWS, + ) + + +# --------------------------------------------------------------------------- +# E. Memory and regime edges +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize("plan", PLANS) +def test_upsert_streaming_source(benchmark, narrow: Target, plan: str) -> None: + """A one-shot reader source, which v1 has to buffer in full to fork it. + + Peak memory is the number of interest here. + """ + num_rows = 1_000_000 + batch_size = 100_000 + + def make_reader() -> pa.RecordBatchReader: + def batches(): + for start in range(0, num_rows, batch_size): + yield narrow_batch( + existing_rows(batch_size, offset=start), value_offset=1 + ) + + return pa.RecordBatchReader.from_batches(NARROW_SCHEMA, batches()) + + def job(dataset: lance.LanceDataset) -> ExecuteResult: + return ( + dataset.merge_insert("id_int") + .when_matched_update_all() + .when_not_matched_insert_all() + .use_index(uses_index(plan)) + .execute(make_reader()) + ) + + run( + benchmark, + narrow, + job, + rounds=1, + warmup=False, + expected_rows=NARROW_NUM_ROWS, + ) + + +def test_upsert_source_equals_target(benchmark, narrow: Target) -> None: + """Source the same size as the target -- the probe must not be chosen here. + + v2 only: the v1 indexed path cannot run this shape at all. Its source-side + hash join asks for more than the whole memory pool and the operation fails + with "Resources exhausted". So there is no v1 baseline to compare against, + and this benchmark exists to keep the v2 path honest at this size. + """ + source = narrow_source(existing_rows(NARROW_NUM_ROWS)) + run( + benchmark, + narrow, + upsert("id_int", source, use_index=False), + rounds=1, + warmup=False, + expected_rows=NARROW_NUM_ROWS, + ) + + +# --------------------------------------------------------------------------- +# IO / memory variants +# --------------------------------------------------------------------------- +# +# A subset of the above, re-run under the io_memory_benchmark fixture. Write +# amplification (`write_bytes`) is the headline for the partial-column cases and +# peak memory is the headline for the streaming source. + + +@pytest.mark.io_memory_benchmark() +@pytest.mark.parametrize("plan", PLANS) +@pytest.mark.parametrize("key", NARROW_KEYS) +@pytest.mark.parametrize("num_rows", [1_000, 100_000]) +def test_io_mem_upsert_ratio( + io_mem_benchmark, narrow: Target, num_rows: int, key: str, plan: str +) -> None: + source = narrow_source(existing_rows(num_rows)) + job = upsert(key, source, use_index=uses_index(plan)) + io_mem_benchmark(job, narrow.dataset, setup=lambda: narrow.reset()) + + +@pytest.mark.io_memory_benchmark() +@pytest.mark.parametrize("plan", PLANS) +@pytest.mark.parametrize("fraction", ROW_FRACTIONS, ids=FRACTION_IDS) +def test_io_mem_update_subset_row_fraction( + io_mem_benchmark, wide: Target, fraction: float, plan: str +) -> None: + source = wide_source(wide_row_indices(fraction), PROJECTIONS["one_scalar"]) + job = update_subset(source, use_index=uses_index(plan)) + io_mem_benchmark( + job, + wide.dataset, + warmup=fraction != 1.0, + setup=lambda: wide.reset(), + ) + + +@pytest.mark.io_memory_benchmark() +@pytest.mark.parametrize("plan", PLANS) +@pytest.mark.parametrize("projection", list(PROJECTIONS), ids=list(PROJECTIONS)) +def test_io_mem_update_subset_projection( + io_mem_benchmark, wide: Target, projection: str, plan: str +) -> None: + source = wide_source(wide_row_indices(0.01), PROJECTIONS[projection]) + job = update_subset(source, use_index=uses_index(plan)) + io_mem_benchmark(job, wide.dataset, setup=lambda: wide.reset()) + + +@pytest.mark.io_memory_benchmark() +@pytest.mark.parametrize("plan", PLANS) +def test_io_mem_upsert_streaming_source( + io_mem_benchmark, narrow: Target, plan: str +) -> None: + num_rows = 1_000_000 + batch_size = 100_000 + + def job(dataset: lance.LanceDataset) -> ExecuteResult: + def batches(): + for start in range(0, num_rows, batch_size): + yield narrow_batch( + existing_rows(batch_size, offset=start), value_offset=1 + ) + + reader = pa.RecordBatchReader.from_batches(NARROW_SCHEMA, batches()) + return ( + dataset.merge_insert("id_int") + .when_matched_update_all() + .when_not_matched_insert_all() + .use_index(uses_index(plan)) + .execute(reader) + ) + + io_mem_benchmark(job, narrow.dataset, warmup=False, setup=lambda: narrow.reset()) diff --git a/python/python/ci_benchmarks/benchmarks/test_overlay_manifest.py b/python/python/ci_benchmarks/benchmarks/test_overlay_manifest.py new file mode 100644 index 00000000000..4a9d8c75e83 --- /dev/null +++ b/python/python/ci_benchmarks/benchmarks/test_overlay_manifest.py @@ -0,0 +1,50 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright The Lance Authors + +"""Impact of overlay files on manifest size. + +Each committed overlay adds a ``DataOverlayFile`` entry (a value-file pointer +plus a serialized coverage bitmap) to every overlaid fragment's metadata in the +manifest. This measures how the manifest grows with the number of overlays and +the coverage size, since manifests are read on every dataset open. +""" + +import pytest +from ci_benchmarks.overlays import ( + commit_overlay_layers, + make_base_dataset, + manifest_size, +) + +NUM_ROWS = 1_000_000 +# Single fragment so every overlay lands on the same fragment's metadata. +ROWS_PER_FILE = NUM_ROWS + +# Strided 10% coverage is the worst case for bitmap size, so manifest growth is +# measured at its upper bound rather than swept across coverage shapes. +COVERAGE_FRACTION = 0.1 +COVERAGE_PATTERN = "stride" + + +@pytest.mark.parametrize("num_overlays", [0, 4, 64]) +def test_overlay_manifest_size(tmp_path, record_property, num_overlays): + base = str(tmp_path / "ds") + ds = make_base_dataset(base, NUM_ROWS, ROWS_PER_FILE, "int32") + base_bytes = manifest_size(ds) + + if num_overlays: + ds = commit_overlay_layers( + ds, num_overlays, COVERAGE_FRACTION, COVERAGE_PATTERN, "int32" + ) + + total = manifest_size(ds) + growth = total - base_bytes + # Guard the fixture: committed overlays must enlarge the manifest, else the + # benchmark would report growth=0 for overlays that were never recorded. + if num_overlays: + assert growth > 0, "overlays did not grow the manifest" + per_overlay = growth / num_overlays if num_overlays else 0 + + record_property("manifest_bytes", total) + record_property("manifest_growth_bytes", growth) + record_property("bytes_per_overlay", per_overlay) diff --git a/python/python/ci_benchmarks/benchmarks/test_overlay_read.py b/python/python/ci_benchmarks/benchmarks/test_overlay_read.py new file mode 100644 index 00000000000..a5774e5ea85 --- /dev/null +++ b/python/python/ci_benchmarks/benchmarks/test_overlay_read.py @@ -0,0 +1,139 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright The Lance Authors + +"""Impact of overlay files on take and scan workloads. + +On read, every overlay covering a requested cell must be consulted and its value +merged over the base. This measures how take and full-scan cost scale with: +- the number of overlay layers stacked on a fragment (compaction payoff), +- fragmentation (contiguous run vs. strided -> pages touched), and +- value width (a 4-byte int32 vs. a wide fixed-size-list embedding). + +Coverage is fixed at 1% and only the default storage version is measured — these +exist to catch regressions, not to explore the parameter space. + +Wall time is measured warm via pytest-benchmark; read IO (bytes + IOPS) is +measured once cold, after dropping the page cache, via ``io_stats_incremental``. +""" + +import random + +import lance +import pytest +from ci_benchmarks.overlays import commit_overlay_layers, make_base_dataset +from ci_benchmarks.utils import wipe_os_cache + +NUM_ROWS = 1_000_000 +ROWS_PER_FILE = NUM_ROWS # single fragment: isolates overlay-layer scaling +TAKE_ROWS = 100 + +# Wide value column: a 3072-d float32 embedding is 12 KiB/row, ~750x an int32 +# cell. Fewer rows keep the base file to ~1.2 GiB while each cell still dominates +# read cost, so the merge/interleave a scan pays per overlay layer moves real +# payload rather than 4-byte integers. +WIDE_EMBEDDING_DIM = 3072 +NUM_ROWS_WIDE = 100_000 + +# Coverage size is held constant so the sweeps isolate layer count and coverage +# shape; 1% models a small targeted update, the case overlays are built for. +COVERAGE_FRACTION = 0.01 + + +def _take_indices(num_rows: int) -> list[int]: + rng = random.Random(0) + return sorted(rng.sample(range(num_rows), TAKE_ROWS)) + + +def _covered_value(ds: lance.LanceDataset): + """``val`` at offset 0, which every coverage pattern includes. + + Used to guard the fixture: after committing overlays a covered cell must + read back a new value, otherwise a read-invisible overlay would let the + benchmark silently time plain base reads. + """ + return ds.take([0], columns=["val"]).column("val").to_pylist()[0] + + +def _measure_cold_io(ds: lance.LanceDataset, base: str, work): + """Drop the page cache, run ``work`` once, return its read IO stats.""" + wipe_os_cache(base) + ds.io_stats_incremental() # reset + work() + stats = ds.io_stats_incremental() + return stats.read_bytes, stats.read_iops + + +def _run_read(benchmark, record_property, base, ds, workload, num_rows): + if workload == "take": + indices = _take_indices(num_rows) + + def work(): + ds.take(indices, columns=["val"]) + else: + + def work(): + ds.to_table(columns=["val"]) + + read_bytes, read_iops = _measure_cold_io(ds, base, work) + record_property("cold_read_bytes", read_bytes) + record_property("cold_read_iops", read_iops) + + benchmark(work) + + +@pytest.mark.parametrize("workload", ["take", "scan"]) +@pytest.mark.parametrize("num_overlays", [0, 4]) +@pytest.mark.parametrize("pattern", ["contiguous", "stride"]) +def test_overlay_read_scaling( + benchmark, + tmp_path, + record_property, + workload, + num_overlays, + pattern, +): + base = str(tmp_path / "ds") + ds = make_base_dataset(base, NUM_ROWS, ROWS_PER_FILE, "int32") + if num_overlays: + base_val = _covered_value(ds) + ds = commit_overlay_layers( + ds, num_overlays, COVERAGE_FRACTION, pattern, "int32" + ) + assert _covered_value(ds) != base_val, "overlay not visible on read" + _run_read(benchmark, record_property, base, ds, workload, NUM_ROWS) + + +# Mirror test_overlay_read_scaling but on a wide 3072-d embedding column, so the +# take/scan-vs-layers story can be read for a fat value column rather than a +# 4-byte one. +@pytest.mark.parametrize("workload", ["take", "scan"]) +@pytest.mark.parametrize("num_overlays", [0, 4]) +@pytest.mark.parametrize("pattern", ["contiguous", "stride"]) +def test_overlay_read_wide( + benchmark, + tmp_path, + record_property, + workload, + num_overlays, + pattern, +): + base = str(tmp_path / "ds") + ds = make_base_dataset( + base, + NUM_ROWS_WIDE, + NUM_ROWS_WIDE, + "embedding", + embedding_dim=WIDE_EMBEDDING_DIM, + ) + if num_overlays: + base_val = _covered_value(ds) + ds = commit_overlay_layers( + ds, + num_overlays, + COVERAGE_FRACTION, + pattern, + "embedding", + embedding_dim=WIDE_EMBEDDING_DIM, + ) + assert _covered_value(ds) != base_val, "overlay not visible on read" + _run_read(benchmark, record_property, base, ds, workload, NUM_ROWS_WIDE) diff --git a/python/python/ci_benchmarks/datagen/gen_all.py b/python/python/ci_benchmarks/datagen/gen_all.py index d5120d20ff7..79725708bb8 100644 --- a/python/python/ci_benchmarks/datagen/gen_all.py +++ b/python/python/ci_benchmarks/datagen/gen_all.py @@ -8,6 +8,7 @@ from ci_benchmarks.datagen.basic import gen_basic from ci_benchmarks.datagen.count_rows import gen_count_rows from ci_benchmarks.datagen.lineitems import gen_tcph +from ci_benchmarks.datagen.merge_insert import gen_merge_insert from ci_benchmarks.datagen.wikipedia import gen_wikipedia @@ -44,6 +45,9 @@ def setup_logging(): LOGGER.info("Generating count_rows benchmark dataset...") gen_count_rows() + LOGGER.info("Generating merge_insert benchmark datasets...") + gen_merge_insert() + LOGGER.info("=" * 80) LOGGER.info("All datasets generated successfully!") LOGGER.info("=" * 80) diff --git a/python/python/ci_benchmarks/datagen/merge_insert.py b/python/python/ci_benchmarks/datagen/merge_insert.py new file mode 100644 index 00000000000..d69a7f4c85f --- /dev/null +++ b/python/python/ci_benchmarks/datagen/merge_insert.py @@ -0,0 +1,423 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright The Lance Authors + +"""Generate the merge_insert benchmark datasets. + +merge_insert benchmarks mutate their target, so each dataset carries a +``merge_insert_base`` tag pointing at a pristine version. Benchmarks restore +to that tag before every measured run (see ``benchmarks/test_merge_insert.py``), +which makes the suite tolerant of a crashed run leaving stale versions behind. + +Datasets +-------- +``merge_insert_narrow`` + 10M rows, 10 fragments. The key columns differ along two independent + axes so benchmarks can separate them: + + * ``id_int`` vs ``id_uuid7`` -- key *type* (int64 vs string), both + clustered. + * ``id_uuid7`` vs ``id_uuid4`` -- key *distribution* (clustered vs + random), both string. + + ``id_no_index`` holds the same values as ``id_int`` with no index, for the + "user never built an index" baseline. ``composite_a``/``composite_b`` are + both indexed, for composite-key probes. + +``merge_insert_wide`` + 1M rows, 10 fragments. 20 narrow scalar columns (~200 MB) plus one + 256-dim float32 vector column (~1 GB). These exert different pressures on + a partial-column update: the scalar columns stress field count and the + number of distinct buffers to schedule, the vector column stresses raw + byte volume. + +``merge_insert_frags`` + 10M rows in 10K fragments of 1K rows, for per-fragment overhead. + +``merge_insert_deleted`` + ``merge_insert_narrow`` layout with a deletion file on every fragment. + +``merge_insert_unindexed_tail`` + ``merge_insert_narrow`` layout where 10% of the rows were appended after + the index was built, so the probe must union an unindexed scan. +""" + +from __future__ import annotations + +import os +from contextlib import contextmanager + +import lance +import numpy as np +import pyarrow as pa +from lance.log import LOGGER + +from ci_benchmarks.datasets import get_dataset_uri + +# Tag marking the pristine version that benchmarks restore to. +BASE_TAG = "merge_insert_base" + +# BTREE training sorts the full column under DataFusion's FairSpillPool. The +# default 150 MiB pool is tight for 10M 32-byte string keys (uuid columns): +# ExternalSorterMerge has peaked near the pool edge and aborted datagen with +# ResourcesExhausted. Index build is one-shot setup, not a measured path, so +# temporarily enlarge the pool only around create_scalar_index. +_INDEX_BUILD_MEM_POOL_SIZE = str(1024 * 1024 * 1024) + +NARROW_NUM_ROWS = 10_000_000 +NARROW_ROWS_PER_FRAGMENT = 1_000_000 + +WIDE_NUM_ROWS = 1_000_000 +WIDE_ROWS_PER_FRAGMENT = 100_000 +WIDE_NUM_SCALAR_COLUMNS = 20 +WIDE_VECTOR_DIM = 256 + +FRAGS_NUM_ROWS = 10_000_000 +FRAGS_ROWS_PER_FRAGMENT = 1_000 + +# Rows appended to `merge_insert_unindexed_tail` after the index is built. +UNINDEXED_TAIL_ROWS = NARROW_NUM_ROWS // 10 + +# `merge_insert_deleted` deletes every Nth row, which puts a deletion file on +# every fragment. +DELETED_ROW_STRIDE = 1000 + +_BATCH_SIZE = 1_000_000 + +NARROW_INDEXED_COLUMNS = [ + "id_int", + "id_uuid7", + "id_uuid4", + "composite_a", + "composite_b", +] + +NARROW_SCHEMA = pa.schema( + [ + ("id_int", pa.int64()), + ("id_uuid7", pa.string()), + ("id_uuid4", pa.string()), + ("id_no_index", pa.int64()), + ("composite_a", pa.int64()), + ("composite_b", pa.int64()), + ("value", pa.int64()), + ] +) + +FRAGS_SCHEMA = pa.schema([("id_int", pa.int64()), ("value", pa.int64())]) + +WIDE_SCALAR_COLUMNS = [f"scalar_{i}" for i in range(WIDE_NUM_SCALAR_COLUMNS)] + +# Half int64, half string, so a partial-column update touches a mix of +# fixed-width and variable-width buffers. +WIDE_SCHEMA = pa.schema( + [("id_int", pa.int64())] + + [ + (name, pa.int64() if i % 2 == 0 else pa.string()) + for i, name in enumerate(WIDE_SCALAR_COLUMNS) + ] + + [("vec", pa.list_(pa.float32(), WIDE_VECTOR_DIM))] +) + + +_GOLDEN = np.uint64(0x9E3779B97F4A7C15) +_MIX1 = np.uint64(0xBF58476D1CE4E5B9) +_MIX2 = np.uint64(0x94D049BB133111EB) +_HEX_DIGITS = np.frombuffer(b"0123456789abcdef", dtype="S1") +_NIBBLE_SHIFTS = np.arange(60, -4, -4, dtype=np.uint64) +# Two uint64 halves rendered as hex. +_KEY_WIDTH = 32 + + +def _scramble(values: np.ndarray) -> np.ndarray: + """splitmix64 finalizer. Wrapping uint64 arithmetic, vectorized.""" + x = values.astype(np.uint64) * _GOLDEN + x = (x ^ (x >> np.uint64(30))) * _MIX1 + x = (x ^ (x >> np.uint64(27))) * _MIX2 + return x ^ (x >> np.uint64(31)) + + +def _hex_keys(high: np.ndarray, low: np.ndarray) -> pa.Array: + """Format two uint64 columns as 32-character lowercase hex strings. + + Vectorized because the narrow dataset needs 10M of these and the + benchmarks rebuild source keys from row indices on every run. + """ + # One nibble at a time into a preallocated byte matrix. Broadcasting all 16 + # shifts at once would materialize an (n, 16) uint64 array per half, which is + # 2.5 GB of scratch for a 10M-row source. + num_rows = len(high) + half = len(_NIBBLE_SHIFTS) + digits = np.empty((num_rows, _KEY_WIDTH), dtype="S1") + for position, shift in enumerate(_NIBBLE_SHIFTS): + digits[:, position] = _HEX_DIGITS[(high >> shift) & np.uint64(0xF)] + digits[:, position + half] = _HEX_DIGITS[(low >> shift) & np.uint64(0xF)] + + # Built from buffers rather than `pa.array`, which splits a numpy byte array + # into chunks above ~1M elements. RecordBatch needs a contiguous Array. + offsets = np.arange(0, _KEY_WIDTH * (num_rows + 1), _KEY_WIDTH, dtype=np.int32) + return pa.StringArray.from_buffers( + num_rows, pa.py_buffer(offsets), pa.py_buffer(digits) + ) + + +def uuid7_keys(row_indices: np.ndarray) -> pa.Array: + """Sortable, UUIDv7-shaped keys: monotonic prefix, scrambled suffix. + + Real UUIDv7 keys are time-ordered, so a stream of them lands in a narrow, + advancing slice of the index. Reproducing that ordering is what matters + for the benchmark; the exact bit layout is not. + + Deterministic in the row index so benchmarks can reconstruct a key without + reading the dataset. + """ + row_indices = row_indices.astype(np.uint64) + return _hex_keys(row_indices, _scramble(row_indices)) + + +def uuid4_keys(row_indices: np.ndarray) -> pa.Array: + """Same width as `uuid7_keys`, but scattered across the index key space. + + Deterministic for the same reason, but the leading bytes are scrambled, so + a contiguous run of row indices maps to keys spread over the whole index + rather than a narrow slice. + """ + row_indices = row_indices.astype(np.uint64) + return _hex_keys(_scramble(row_indices), row_indices) + + +def narrow_batch(row_indices: np.ndarray, value_offset: int = 0) -> pa.RecordBatch: + """Build a full-schema narrow batch from row indices. + + Shared with the benchmarks, which reconstruct source rows from row indices. + ``value_offset`` shifts the payload column so an update writes a value that + differs from what the target already holds. + """ + return pa.record_batch( + [ + pa.array(row_indices), + uuid7_keys(row_indices), + uuid4_keys(row_indices), + pa.array(row_indices), + pa.array(row_indices), + # `composite_b` is a deterministic function of `composite_a` so a + # source row can target an existing composite key without the + # benchmark tracking extra state. + pa.array(row_indices % 1024), + pa.array(row_indices + value_offset), + ], + schema=NARROW_SCHEMA, + ) + + +def _narrow_data(num_rows: int, offset: int = 0): + LOGGER.info("Generating %d narrow rows starting at %d", num_rows, offset) + for start in range(offset, offset + num_rows, _BATCH_SIZE): + count = min(_BATCH_SIZE, offset + num_rows - start) + yield narrow_batch(np.arange(start, start + count, dtype=np.int64)) + + +def _frags_data(num_rows: int): + LOGGER.info("Generating %d small-fragment rows", num_rows) + for start in range(0, num_rows, _BATCH_SIZE): + ids = np.arange( + start, start + min(_BATCH_SIZE, num_rows - start), dtype=np.int64 + ) + yield pa.record_batch([pa.array(ids), pa.array(ids)], schema=FRAGS_SCHEMA) + + +def _wide_batch(offset: int, num_rows: int) -> pa.RecordBatch: + ids = np.arange(offset, offset + num_rows, dtype=np.int64) + columns: list[pa.Array] = [pa.array(ids)] + for i in range(WIDE_NUM_SCALAR_COLUMNS): + if i % 2 == 0: + columns.append(pa.array(ids + i)) + else: + columns.append(pa.array([f"s{i}_{v}" for v in ids], type=pa.string())) + # Deterministic float payload; the values are irrelevant, the byte volume + # is the point. + vectors = np.linspace( + 0.0, 1.0, num=num_rows * WIDE_VECTOR_DIM, dtype=np.float32 + ).reshape(num_rows, WIDE_VECTOR_DIM) + columns.append( + pa.FixedSizeListArray.from_arrays( + pa.array(vectors.reshape(-1)), WIDE_VECTOR_DIM + ) + ) + return pa.record_batch(columns, schema=WIDE_SCHEMA) + + +def _wide_data(num_rows: int): + LOGGER.info("Generating %d wide rows", num_rows) + # The vector column makes full batches large, so use a smaller batch here. + batch_size = WIDE_ROWS_PER_FRAGMENT + for start in range(0, num_rows, batch_size): + yield _wide_batch(start, min(batch_size, num_rows - start)) + + +def _tag_base(ds: lance.LanceDataset) -> None: + """Point BASE_TAG at the current version, creating the tag if needed.""" + if BASE_TAG in ds.tags.list(): + ds.tags.update(BASE_TAG, ds.version) + else: + ds.tags.create(BASE_TAG, ds.version) + + +def _already_generated(uri: str, expected_rows: int) -> bool: + """True when a usable dataset with a pristine base tag already exists. + + A previous benchmark run may have left extra versions behind, so the row + count is checked at the tagged version rather than at the latest one. + + Incomplete generations (dataset written, tag never created) and missing + tags both return False so the caller can overwrite and retag. + """ + try: + ds = lance.dataset(uri) + except ValueError: + return False + tags = ds.tags.list() + if BASE_TAG not in tags: + return False + base_version = tags[BASE_TAG]["version"] + return ds.checkout_version(base_version).count_rows() == expected_rows + + +@contextmanager +def _enlarged_mem_pool_for_index_build(): + """Raise LANCE_MEM_POOL_SIZE only for the duration of index training.""" + previous = os.environ.get("LANCE_MEM_POOL_SIZE") + os.environ["LANCE_MEM_POOL_SIZE"] = _INDEX_BUILD_MEM_POOL_SIZE + try: + yield + finally: + if previous is None: + os.environ.pop("LANCE_MEM_POOL_SIZE", None) + else: + os.environ["LANCE_MEM_POOL_SIZE"] = previous + + +def _gen( + name: str, + data, + schema: pa.Schema, + expected_rows: int, + rows_per_fragment: int, + indexed_columns: list[str], +) -> lance.LanceDataset: + dataset_uri = get_dataset_uri(name) + if _already_generated(dataset_uri, expected_rows): + LOGGER.info("Dataset %s already exists, skipping", name) + return lance.dataset(dataset_uri) + + LOGGER.info("Creating dataset %s", name) + ds = lance.write_dataset( + data, + dataset_uri, + schema=schema, + mode="overwrite", + max_rows_per_file=rows_per_fragment, + max_rows_per_group=min(rows_per_fragment, 100_000), + ) + with _enlarged_mem_pool_for_index_build(): + for column in indexed_columns: + LOGGER.info("Building BTREE index on %s.%s", name, column) + ds.create_scalar_index(column, "BTREE") + _tag_base(ds) + return ds + + +def gen_merge_insert_narrow() -> lance.LanceDataset: + return _gen( + "merge_insert_narrow", + _narrow_data(NARROW_NUM_ROWS), + NARROW_SCHEMA, + NARROW_NUM_ROWS, + NARROW_ROWS_PER_FRAGMENT, + NARROW_INDEXED_COLUMNS, + ) + + +def gen_merge_insert_wide() -> lance.LanceDataset: + return _gen( + "merge_insert_wide", + _wide_data(WIDE_NUM_ROWS), + WIDE_SCHEMA, + WIDE_NUM_ROWS, + WIDE_ROWS_PER_FRAGMENT, + ["id_int"], + ) + + +def gen_merge_insert_frags() -> lance.LanceDataset: + return _gen( + "merge_insert_frags", + _frags_data(FRAGS_NUM_ROWS), + FRAGS_SCHEMA, + FRAGS_NUM_ROWS, + FRAGS_ROWS_PER_FRAGMENT, + ["id_int"], + ) + + +def gen_merge_insert_deleted() -> lance.LanceDataset: + """Narrow layout with a deletion file on every fragment. + + One row in every `DELETED_ROW_STRIDE` is deleted, which lands in every + fragment. + """ + name = "merge_insert_deleted" + dataset_uri = get_dataset_uri(name) + expected_rows = NARROW_NUM_ROWS - NARROW_NUM_ROWS // DELETED_ROW_STRIDE + if _already_generated(dataset_uri, expected_rows): + LOGGER.info("Dataset %s already exists, skipping", name) + return lance.dataset(dataset_uri) + + ds = _gen( + name, + _narrow_data(NARROW_NUM_ROWS), + NARROW_SCHEMA, + NARROW_NUM_ROWS, + NARROW_ROWS_PER_FRAGMENT, + NARROW_INDEXED_COLUMNS, + ) + LOGGER.info("Deleting rows from %s to create deletion files", name) + ds.delete(f"id_int % {DELETED_ROW_STRIDE} == 0") + _tag_base(ds) + return ds + + +def gen_merge_insert_unindexed_tail() -> lance.LanceDataset: + """Narrow layout where the last 10% of rows are not covered by the index.""" + name = "merge_insert_unindexed_tail" + dataset_uri = get_dataset_uri(name) + expected_rows = NARROW_NUM_ROWS + UNINDEXED_TAIL_ROWS + if _already_generated(dataset_uri, expected_rows): + LOGGER.info("Dataset %s already exists, skipping", name) + return lance.dataset(dataset_uri) + + _gen( + name, + _narrow_data(NARROW_NUM_ROWS), + NARROW_SCHEMA, + NARROW_NUM_ROWS, + NARROW_ROWS_PER_FRAGMENT, + NARROW_INDEXED_COLUMNS, + ) + LOGGER.info("Appending %d unindexed rows to %s", UNINDEXED_TAIL_ROWS, name) + ds = lance.write_dataset( + _narrow_data(UNINDEXED_TAIL_ROWS, offset=NARROW_NUM_ROWS), + dataset_uri, + schema=NARROW_SCHEMA, + mode="append", + max_rows_per_file=NARROW_ROWS_PER_FRAGMENT, + ) + _tag_base(ds) + return ds + + +def gen_merge_insert() -> None: + gen_merge_insert_narrow() + gen_merge_insert_wide() + gen_merge_insert_frags() + gen_merge_insert_deleted() + gen_merge_insert_unindexed_tail() diff --git a/python/python/ci_benchmarks/overlays.py b/python/python/ci_benchmarks/overlays.py new file mode 100644 index 00000000000..9d9d182de12 --- /dev/null +++ b/python/python/ci_benchmarks/overlays.py @@ -0,0 +1,168 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright The Lance Authors + +"""Shared helpers for the data-overlay benchmark suite. + +Data overlay files supply replacement values for a subset of (row offset, field) +cells in a fragment, merged on read, without rewriting the base data file. These +helpers build synthetic base datasets, commit overlay layers through the public +``lance.LanceOperation.DataOverlay`` operation, and measure their cost. +""" + +import os + +# Data overlay support is gated off in release builds unless this is set (it is +# always on in debug builds). Benchmarks usually run against a release build, so +# enable it here, before lance is imported, or reading overlay datasets fails. +os.environ.setdefault("LANCE_ENABLE_UNSTABLE_DATA_OVERLAY_FILES", "1") + +from typing import List # noqa: E402 +from urllib.parse import urlparse # noqa: E402 + +import lance # noqa: E402 +import numpy as np # noqa: E402 +import pyarrow as pa # noqa: E402 +from lance._datagen import rand_batches # noqa: E402 +from lance.file import LanceFileWriter # noqa: E402 + +# Default width for the ``embedding`` dtype. Individual benchmarks override it +# via ``embedding_dim`` to model narrow vs. wide (e.g. 3072-d) value columns. +EMBEDDING_DIM = 128 + + +def _value_type(dtype: str, embedding_dim: int = EMBEDDING_DIM) -> pa.DataType: + if dtype == "int32": + return pa.int32() + if dtype == "embedding": + return pa.list_(pa.float32(), embedding_dim) + raise ValueError(f"unknown overlay benchmark dtype {dtype!r}") + + +def _gen_values( + dtype: str, + n: int, + rng: np.random.Generator, + embedding_dim: int = EMBEDDING_DIM, +) -> pa.Array: + if dtype == "int32": + return pa.array(rng.integers(0, 1 << 30, size=n, dtype=np.int32)) + if dtype == "embedding": + flat = rng.random(n * embedding_dim, dtype=np.float32) + return pa.FixedSizeListArray.from_arrays(pa.array(flat), embedding_dim) + raise ValueError(f"unknown overlay benchmark dtype {dtype!r}") + + +def make_base_dataset( + base_path: str, + num_rows: int, + rows_per_file: int, + dtype: str, + embedding_dim: int = EMBEDDING_DIM, +) -> lance.LanceDataset: + """Create a base dataset with an ``id`` column and a ``val`` column. + + ``val`` is the column overlays target. ``rows_per_file`` controls the number + of fragments (``num_rows // rows_per_file``), which must divide ``num_rows``. + ``embedding_dim`` sets the width of the ``val`` column when ``dtype`` is + ``embedding`` (e.g. 3072 for a wide embedding). + """ + if num_rows % rows_per_file: + raise ValueError( + f"num_rows ({num_rows}) must be a multiple of rows_per_file " + f"({rows_per_file})" + ) + schema = pa.schema({"id": pa.int64(), "val": _value_type(dtype, embedding_dim)}) + # One fragment per batch; lance-datagen fills both columns with random data. + reader = rand_batches( + schema, num_batches=num_rows // rows_per_file, rows_per_batch=rows_per_file + ) + return lance.write_dataset(reader, base_path, max_rows_per_file=rows_per_file) + + +def coverage_offsets(num_rows: int, fraction: float, pattern: str) -> List[int]: + """Offsets within a fragment covered by an overlay. + + ``contiguous`` packs the covered cells into a single leading run (few pages + touched); ``stride`` spreads them evenly across the fragment (many pages + touched). Both cover ``round(num_rows * fraction)`` cells. + """ + count = max(1, int(round(num_rows * fraction))) + if pattern == "contiguous": + return list(range(count)) + if pattern == "stride": + step = max(1, num_rows // count) + return list(range(0, num_rows, step))[:count] + raise ValueError(f"unknown coverage pattern {pattern!r}") + + +def _val_field_id(ds: lance.LanceDataset) -> int: + base_df = ds.get_fragments()[0].metadata.files[0] + names = [f.name for f in ds.schema] + return base_df.fields[names.index("val")] + + +def commit_overlay_layers( + ds: lance.LanceDataset, + num_layers: int, + fraction: float, + pattern: str, + dtype: str, + *, + seed: int = 0, + embedding_dim: int = EMBEDDING_DIM, +) -> lance.LanceDataset: + """Commit ``num_layers`` overlays on ``val``, each covering the same offsets + in every fragment so that all layers must be consulted on read (the case + that motivates compaction). Returns the updated dataset. + """ + base_df = ds.get_fragments()[0].metadata.files[0] + field_id = _val_field_id(ds) + data_dir = os.path.join(_local_path(ds), "data") + for layer in range(num_layers): + rng = np.random.default_rng(seed + layer + 1) + groups = [] + for frag in ds.get_fragments(): + offsets = coverage_offsets(frag.count_rows(), fraction, pattern) + values = _gen_values(dtype, len(offsets), rng, embedding_dim) + batch = pa.record_batch([values], names=["val"]) + name = f"overlay_l{layer}_f{frag.fragment_id}.lance" + path = os.path.join(data_dir, name) + with LanceFileWriter(path) as writer: + writer.write_batch(batch) + df = lance.fragment.DataFile( + path=name, + fields=[field_id], + column_indices=[0], + file_major_version=base_df.file_major_version, + file_minor_version=base_df.file_minor_version, + file_size_bytes=os.path.getsize(path), + ) + groups.append( + lance.LanceOperation.DataOverlayGroup( + fragment_id=frag.fragment_id, + overlays=[ + lance.LanceOperation.DataOverlayFile( + data_file=df, offsets=offsets + ) + ], + ) + ) + op = lance.LanceOperation.DataOverlay(groups=groups) + ds = lance.LanceDataset.commit(ds, op, read_version=ds.version) + return ds + + +# --- Measurement helpers ---------------------------------------------------- + + +def _local_path(ds: lance.LanceDataset) -> str: + parsed = urlparse(ds.uri) + return parsed.path if parsed.scheme == "file" else ds.uri + + +def manifest_size(ds: lance.LanceDataset) -> int: + """Size in bytes of the manifest for the dataset's current version.""" + # Manifests are named `{u64::MAX - version}.manifest` so that a plain + # lexicographic directory listing yields newest-version-first. + name = f"{(1 << 64) - 1 - ds.version}.manifest" + return os.path.getsize(os.path.join(_local_path(ds), "_versions", name)) diff --git a/python/python/lance/__init__.py b/python/python/lance/__init__.py index 61d94b5550a..981140858c7 100644 --- a/python/python/lance/__init__.py +++ b/python/python/lance/__init__.py @@ -44,16 +44,19 @@ CleanupStats, DatasetBasePath, FFILanceTableProvider, + FtsToken, ScanStatistics, bytes_read_counter, iops_counter, + simd_info, + tokenize, ) from .mem_wal import ( + CompactedSsTable, ExecutionPlan, LsmPointLookupPlanner, LsmScanner, LsmVectorSearchPlanner, - MergedGeneration, ShardingField, ShardingSpec, ShardSnapshot, @@ -97,6 +100,7 @@ "DataStatistics", "FieldStatistics", "FragmentMetadata", + "FtsToken", "Index", "IndexFile", "LanceDataset", @@ -115,6 +119,8 @@ "json_to_schema", "schema_to_json", "set_logger", + "simd_info", + "tokenize", "write_dataset", "FFILanceTableProvider", "IndexProgress", @@ -122,7 +128,7 @@ "LsmPointLookupPlanner", "LsmScanner", "LsmVectorSearchPlanner", - "MergedGeneration", + "CompactedSsTable", "ShardSnapshot", "ShardWriter", "ShardingField", diff --git a/python/python/lance/_datagen.py b/python/python/lance/_datagen.py index b156066eca6..45826503c5b 100644 --- a/python/python/lance/_datagen.py +++ b/python/python/lance/_datagen.py @@ -21,10 +21,13 @@ def rand_batches( *, num_batches: Optional[int] = None, batch_size_bytes: Optional[int] = None, + rows_per_batch: Optional[int] = None, ): if not datagen.is_datagen_supported(): raise NotImplementedError( "This version of lance was not built with the datagen feature" ) - batch_iter = datagen.rand_batches(schema, num_batches, batch_size_bytes) + batch_iter = datagen.rand_batches( + schema, num_batches, batch_size_bytes, rows_per_batch + ) return pa.RecordBatchReader.from_batches(schema, batch_iter) diff --git a/python/python/lance/arrow.py b/python/python/lance/arrow.py index da022cdc8f6..5e9f671ab80 100644 --- a/python/python/lance/arrow.py +++ b/python/python/lance/arrow.py @@ -310,16 +310,7 @@ def pillow_metadata_decoder(images): img = Image.open(io.BytesIO(images[0].as_py())) return img - def tensorflow_metadata_decoder(images): - import tensorflow as tf - - img = tf.io.decode_image(images[0].as_py()) - return img - - decoders = ( - ("tensorflow", tensorflow_metadata_decoder), - ("PIL", pillow_metadata_decoder), - ) + decoders = (("PIL", pillow_metadata_decoder),) decoder = None for libname, metadata_decoder in decoders: @@ -351,7 +342,7 @@ def to_tensor( decoder : Callable[pa.binary()], optional A function that takes a binary array and returns a numpy.ndarray or pa.fixed_shape_tensor. If not provided, will attempt to use - tensorflow and then pillow decoder in that order. + pillow. Returns ------- @@ -385,20 +376,7 @@ def pillow_decoder(images) -> "np.ndarray": ] ) - def tensorflow_decoder(images) -> "np.ndarray": - import tensorflow as tf - - decoded_to_tensor = tuple( - tf.io.decode_image(img) for img in images.to_pylist() - ) - return tf.stack( # pyright: ignore[reportOptionalCall] - decoded_to_tensor, axis=0 - ).numpy() - - decoders = [ - ("tensorflow", tensorflow_decoder), - ("PIL", pillow_decoder), - ] + decoders = [("PIL", pillow_decoder)] for libname, decoder_function in decoders: try: __import__(libname) @@ -408,8 +386,8 @@ def tensorflow_decoder(images) -> "np.ndarray": pass else: raise ValueError( - "No image decoder available. Please either install one of " - "tensorflow, pillow, or pass a decoder argument." + "No image decoder available. Please install pillow or pass a " + "decoder argument." ) image_array = decoder(self.storage) @@ -499,19 +477,8 @@ def pillow_encoder(x): encoded_images.append(buf.getvalue()) return pa.array(encoded_images, type=storage_type) - def tensorflow_encoder(x): - import tensorflow as tf - - encoded_images = ( - tf.io.encode_png(y).numpy() for y in tf.convert_to_tensor(x) - ) - return pa.array(encoded_images, type=storage_type) - if not encoder: - encoders = ( - ("PIL", pillow_encoder), - ("tensorflow", tensorflow_encoder), - ) + encoders = (("PIL", pillow_encoder),) for libname, encoder_function in encoders: try: __import__(libname) @@ -521,8 +488,8 @@ def tensorflow_encoder(x): pass else: raise ValueError( - "No image encoder available. Please either install one of " - "tensorflow, pillow, or pass an encoder argument." + "No image encoder available. Please install pillow or pass an " + "encoder argument." ) return EncodedImageArray.from_storage( diff --git a/python/python/lance/blob.py b/python/python/lance/blob.py index 6d1af6797dc..f4a92633807 100644 --- a/python/python/lance/blob.py +++ b/python/python/lance/blob.py @@ -383,6 +383,28 @@ def read_range(self, offset: int, length: int) -> bytes: """Read a blob-local byte range without changing the current cursor.""" return self.inner.read_range(offset, length) + def read_ranges(self, ranges: list[tuple[int, int]]) -> list[bytes]: + """ + Read multiple blob-local byte ranges without changing the current cursor. + + Each range is an ``(offset, length)`` pair, matching + :py:meth:`read_range`. The underlying physical reads may be reordered, + coalesced, or split for efficiency. For every range, offset plus length + must fit in an unsigned 64-bit integer and must not extend beyond the + blob size. + + Parameters + ---------- + ranges : List[Tuple[int, int]] + The ``(offset, length)`` byte ranges to read. + + Returns + ------- + data : List[bytes] + One payload per requested range, in input order. + """ + return self.inner.read_ranges(ranges) + def readinto(self, b: bytearray) -> int: return self.inner.read_into(b) diff --git a/python/python/lance/dataset.py b/python/python/lance/dataset.py index 6be0a78d2e8..0273063c3b3 100644 --- a/python/python/lance/dataset.py +++ b/python/python/lance/dataset.py @@ -9,6 +9,7 @@ import operator import os import random +import re import time import uuid import warnings @@ -44,6 +45,9 @@ from .dependencies import ( _check_for_numpy, _check_for_torch, + _is_pydantic_base_model_class, + _validate_pydantic_list, + model_to_dict, torch, ) from .dependencies import numpy as np @@ -70,12 +74,16 @@ ) from .lance import __version__ as __version__ from .lance import _Session as Session -from .query import FullTextQuery -from .types import _coerce_reader +from .query import DocumentGranularity, FullTextQuery +from .types import _coerce_reader, _is_materialized from .udf import BatchUDF, normalize_transform from .udf import BatchUDFCheckpoint as BatchUDFCheckpoint from .udf import batch_udf as batch_udf -from .util import _target_partition_size_to_num_partitions, td_to_micros +from .util import ( + _normalize_index_segment_ids, + _target_partition_size_to_num_partitions, + td_to_micros, +) if TYPE_CHECKING: from pyarrow._compute import Expression @@ -235,14 +243,6 @@ def _is_null_blob_description(description: Any) -> bool: return False if description.keys() == {"position", "size"}: return description["position"] == 1 and description["size"] == 0 - if description.keys() == {"kind", "position", "size", "blob_id", "blob_uri"}: - return ( - description["kind"] == 0 - and description["position"] == 0 - and description["size"] == 0 - and description["blob_id"] == 0 - and description["blob_uri"] == "" - ) return False @@ -393,6 +393,12 @@ def execute(self, data_obj: ReaderLike, *, schema: Optional[pa.Schema] = None): """ reader = _coerce_reader(data_obj, schema) + # Materialized sources are wrapped in an in-memory table so retries never + # spill and the source's statistics can drive the join; everything else is + # treated as a one-shot stream. + if _is_materialized(data_obj): + return super(MergeInsertBuilder, self).execute_batches(reader) + return super(MergeInsertBuilder, self).execute(reader) def execute_uncommitted( @@ -417,6 +423,9 @@ def execute_uncommitted( """ reader = _coerce_reader(data_obj, schema) + if _is_materialized(data_obj): + return super(MergeInsertBuilder, self).execute_uncommitted_batches(reader) + return super(MergeInsertBuilder, self).execute_uncommitted(reader) # These next three overrides exist only to document the methods @@ -735,23 +744,30 @@ def analyze_plan( reader = _coerce_reader(data_obj, schema) return super(MergeInsertBuilder, self).analyze_plan(reader) - def mark_generations_as_merged( - self, generations: "List[mem_wal.MergedGeneration]" + def mark_sstables_as_compacted( + self, sstables: "List[mem_wal.CompactedSsTable]" ) -> "MergeInsertBuilder": - """Mark MemWAL generations as merged into the base table. + """Mark MemWAL SSTables as compacted into the base table. + + Call this before executing merge_insert when it compacts MemWAL SSTables. + The progress is recorded in the same commit as the data. - Call this before executing the merge_insert when the source data - includes rows from MemWAL flushed generations. + For multi-pass compaction, call this only on the final successful + data-changing pass. Intermediate passes must not carry compaction + progress. Lance cannot tell whether a caller has another pass planned, + so it cannot enforce this: if a delete pass carried the marker and the + process then died before the matching upsert, the recorded progress + would claim rows were copied in that never were. Parameters ---------- - generations : list of MergedGeneration - Generations to mark as merged. + sstables : list of CompactedSsTable + SSTables to mark as compacted. """ - from .mem_wal import _to_raw_merged_generations + from .mem_wal import _to_raw_compacted_sstables - raw_gens = _to_raw_merged_generations(generations) - super(MergeInsertBuilder, self).mark_generations_as_merged(raw_gens) + raw_sstables = _to_raw_compacted_sstables(sstables) + super(MergeInsertBuilder, self).mark_sstables_as_compacted(raw_sstables) return self @@ -842,6 +858,55 @@ def __deserialize__( base_store_params=base_store_params, ) + @classmethod + def from_pydantic_model( + cls, + model_class, + data, + uri: Optional[Union[str, Path]] = None, + mode: str = "create", + **kwargs, + ) -> "LanceDataset": + """Create a LanceDataset from a Pydantic model class and a list of instances. + + The table name is inferred from the model class name converted to snake_case. + The schema is inferred from the model class's field annotations, not from + the data, so optional fields are typed correctly even if every value in a + given batch happens to be None. + + Parameters + ---------- + model_class : type + A Pydantic BaseModel subclass. + data : list + A list of Pydantic model instances. + uri : str or Path, optional + The URI to write the dataset to. If not provided, the model class name + converted to snake_case is used as the path. + mode : str, optional + The write mode. One of "create", "overwrite", or "append". + **kwargs + Additional arguments passed to write_dataset(). + """ + if not _is_pydantic_base_model_class(model_class): + raise TypeError( + f"`model_class` must be a Pydantic BaseModel subclass, " + f"got {model_class!r}" + ) + _validate_pydantic_list(data, model_class) + if not data: + raise ValueError( + "`data` must be a non-empty list of Pydantic model instances." + ) + if uri is None: + uri = re.sub(r"(? List[BlobFile]: + ) -> List[Optional[BlobFile]]: """ Select blobs by row IDs. @@ -2189,7 +2263,9 @@ def take_blobs( Returns ------- - blob_files : List[BlobFile] + blob_files : List[Optional[BlobFile]] + One element per selected row. Null blob values return ``None``; + valid empty blobs return a ``BlobFile`` with size zero. """ selection_kind, selection_values = _resolve_blob_selection( ids, addresses, indices @@ -2205,7 +2281,10 @@ def take_blobs( lance_blob_files = self._ds.take_blobs_by_indices( selection_values, blob_column ) - return [BlobFile(lance_blob_file) for lance_blob_file in lance_blob_files] + return [ + BlobFile(lance_blob_file) if lance_blob_file is not None else None + for lance_blob_file in lance_blob_files + ] def read_blobs( self, @@ -2216,7 +2295,7 @@ def read_blobs( *, io_buffer_size: Optional[int] = None, preserve_order: Optional[bool] = None, - ) -> List[Tuple[int, bytes]]: + ) -> List[Tuple[int, Optional[bytes]]]: """ Read blobs directly into memory using Lance's planned blob reader. @@ -2244,8 +2323,9 @@ def read_blobs( Returns ------- - blobs : List[Tuple[int, bytes]] - A list of ``(row_address, blob_bytes)`` pairs. + blobs : List[Tuple[int, Optional[bytes]]] + One ``(row_address, blob_bytes)`` pair per selected row. Null blob + values return ``None``; valid empty blobs return ``b""``. """ selection_kind, selection_values = _resolve_blob_selection( ids, addresses, indices @@ -2263,6 +2343,74 @@ def read_blobs( ) return self._ds.read_blobs_by_indices(selection_values, blob_column, **kwargs) + def read_blob_ranges( + self, + blob_column: str, + requests: Sequence[Tuple[int, int, int]], + *, + selector: Literal["ids", "addresses", "indices"], + io_buffer_size: Optional[int] = None, + preserve_order: Optional[bool] = None, + ) -> List[Tuple[int, int, Optional[bytes]]]: + """ + Read row-specific blob-local byte ranges with one planned API call. + + Each request is a ``(row, offset, length)`` tuple. ``selector`` defines + whether every ``row`` value is interpreted as a stable row ID, physical + row address, or dataset index. Repeat a row value to read multiple ranges + from the same blob. Planning, range validation, physical-source grouping, + request coalescing, and bounded I/O scheduling all happen in Rust; this + method does not use a Python thread pool. + + Every request produces one result. Requests on null blobs return ``None``, + including when the requested range is empty. Empty ranges on non-null + blobs return empty bytes without issuing payload I/O. For every request, + offset plus length must fit in an unsigned 64-bit integer. Ranges on + non-null blobs must not extend beyond the logical blob size. Blob-local + bounds are not evaluated for null values because they have no logical + payload length. By default results preserve the input request order. + + Parameters + ---------- + blob_column : str + The name of the blob column to read. + requests : Sequence[Tuple[int, int, int]] + Complete ``(row, offset, length)`` requests. + selector : {"ids", "addresses", "indices"} + Interpretation of every request's ``row`` value. + io_buffer_size : int, optional + Override the scheduler I/O buffer size used while materializing ranges. + preserve_order : bool, optional + If True, returned results follow request order. If False, + ``request_index`` still identifies every result. + + Examples + -------- + Read two disjoint ranges from the same row index: + + .. code-block:: python + + dataset.read_blob_ranges( + "images", + requests=[(7, 0, 1024), (7, 4096, 1024)], + selector="indices", + ) + + Returns + ------- + results : List[Tuple[int, int, Optional[bytes]]] + One ``(request_index, row_address, data)`` tuple per request. + The Python list retains all returned payload bytes in memory; peak + result memory is therefore proportional to their total logical size. + """ + return self._ds.read_blob_ranges( + list(requests), + blob_column, + selector, + io_buffer_size=io_buffer_size, + preserve_order=preserve_order, + ) + def head(self, num_rows, **kwargs): """ Load the first N rows of the dataset. @@ -3006,7 +3154,7 @@ def cleanup_old_versions( ``retain_versions`` are not specified, this will default to two weeks. retain_versions: int, optional - Retain the last N versions of the dataset. + Retain the last N versions of the dataset. Must be positive. delete_unverified: bool, default False Files leftover from a failed transaction may appear to be part of an @@ -3065,7 +3213,7 @@ def explain_cleanup_old_versions( ``retain_versions`` are not specified, this will default to two weeks. retain_versions: int, optional - Retain the last N versions of the dataset. + Retain the last N versions of the dataset. Must be positive. delete_unverified: bool, default False Include unverified files that cleanup would remove when this is set. @@ -3120,8 +3268,6 @@ def _prepare_scalar_index_request( column = column[0] lance_field = self._ds.lance_schema.field_case_insensitive(column) - if lance_field is None: - raise KeyError(f"{column} not found in schema") if isinstance(index_type, str): index_type = index_type.upper() @@ -3144,6 +3290,12 @@ def _prepare_scalar_index_request( ) ) + if lance_field is None: + if index_type in ["INVERTED", "FTS"]: + # Rust resolves public FTS paths through intervening list layers. + return column, index_type, index_type + raise KeyError(f"{column} not found in schema") + field = lance_field.to_arrow() field_type = field.type @@ -3151,7 +3303,7 @@ def _prepare_scalar_index_request( if hasattr(field_type, "storage_type"): field_type = field_type.storage_type - if index_type in ["BTREE", "BITMAP", "ZONEMAP"]: + if index_type in ["BTREE", "BITMAP"]: if ( not pa.types.is_integer(field_type) and not pa.types.is_floating(field_type) @@ -3162,12 +3314,16 @@ def _prepare_scalar_index_request( and not pa.types.is_fixed_size_binary(field_type) ): raise TypeError( - f"BTREE/BITMAP/ZONEMAP index column {column} must be int", + f"BTREE/BITMAP index column {column} must be int", ", float, bool, str, large_str, fixed-size-binary, or temporal", ) elif index_type == "LABEL_LIST": - if not pa.types.is_list(field_type): - raise TypeError(f"LABEL_LIST index column {column} must be a list") + if not ( + pa.types.is_list(field_type) or pa.types.is_large_list(field_type) + ): + raise TypeError( + f"LABEL_LIST index column {column} must be a list or large list" + ) elif index_type == "NGRAM": if not pa.types.is_string(field_type) and not pa.types.is_large_string( field_type @@ -3197,7 +3353,12 @@ def _prepare_scalar_index_request( return column, index_type, index_type elif isinstance(index_type, IndexConfig): logical_index_type = index_type.index_type.upper() - config = json.dumps(index_type.parameters) + if lance_field is None and logical_index_type not in ["INVERTED", "FTS"]: + raise KeyError(f"{column} not found in schema") + parameters = dict(index_type.parameters) + if logical_index_type in ["INVERTED", "FTS"]: + parameters["document_granularity"] = kwargs["document_granularity"] + config = json.dumps(parameters) kwargs["config"] = indices.IndexConfig(index_type.index_type, config) return column, "scalar", logical_index_type else: @@ -3221,7 +3382,11 @@ def _is_segment_native_scalar_index_type( "BITMAP", "INVERTED", "FTS", + "NGRAM", + "RTREE", "ZONEMAP", + "BLOOMFILTER", + "LABEL_LIST", } @classmethod @@ -3232,7 +3397,11 @@ def _requires_uncommitted_scalar_index( return cls._normalized_index_type(index_type) in { "BTREE", "BITMAP", + "NGRAM", + "RTREE", "ZONEMAP", + "BLOOMFILTER", + "LABEL_LIST", } def create_scalar_index( @@ -3258,6 +3427,7 @@ def create_scalar_index( index_uuid: Optional[str] = None, progress_callback: Optional[Callable[[IndexProgress], None]] = None, format_version: Optional[Union[int, str]] = None, + document_granularity: DocumentGranularity = DocumentGranularity.ROW, **kwargs, ): """Create a scalar index on a column. @@ -3366,8 +3536,15 @@ def create_scalar_index( format_version: int or str, optional This is for the ``INVERTED`` / ``FTS`` index. Explicit on-disk FTS format version to write when creating a new index. Accepts ``1``, - ``2``, ``"v1"``, or ``"v2"``. If unset, Lance chooses the current - default format. + ``2``, ``3``, ``"v1"``, ``"v2"``, or ``"v3"``. + If unset, Lance uses ``LANCE_FTS_FORMAT_VERSION`` when present and + otherwise writes v2 for text analysis with ``block_size=128`` and + v3 for code analysis or ``block_size=256``. + + document_granularity: DocumentGranularity, default ROW + This is for the ``INVERTED`` / ``FTS`` index. ``ROW`` treats all + selected text in one dataset row as one document. ``LIST_ELEMENT`` + treats each element of the deepest list on ``column`` as one document. with_position: bool, default False This is for the ``INVERTED`` index. If True, the index will store the @@ -3375,6 +3552,12 @@ def create_scalar_index( query. This will significantly increase the index size. It won't impact the performance of non-phrase queries even if it is set to True. + block_size: int, default 128 + This is for the ``INVERTED`` index. Number of documents per compressed + posting block. Must be one of ``128`` or ``256``. + ``block_size=256`` is experimental and may introduce breaking changes. + Use ``128`` when stable compatibility with the legacy posting layout is + required. memory_limit: int, optional This is for the ``INVERTED`` index. Total build-time memory limit in MiB. If set, Lance divides this budget evenly across the workers. If unset, @@ -3456,6 +3639,11 @@ def create_scalar_index( ``MaterializeIndex`` operator. """ + if not isinstance(document_granularity, DocumentGranularity): + raise TypeError( + "document_granularity must be a lance.query.DocumentGranularity" + ) + kwargs["document_granularity"] = document_granularity.value column, index_type, logical_index_type = self._prepare_scalar_index_request( column, index_type, kwargs ) @@ -3477,7 +3665,6 @@ def create_scalar_index( kwargs["progress_callback"] = progress_callback if format_version is not None: kwargs["format_version"] = format_version - self._ds.create_index([column], index_type, name, replace, train, None, kwargs) def _create_index_impl( @@ -4180,10 +4367,10 @@ def create_index_uncommitted( Create one segment without publishing it and return its metadata. This is the public distributed-build API for vector, BTREE scalar, - canonical bitmap scalar, INVERTED scalar, and ZONEMAP scalar index - construction. Unlike - :meth:`create_index`, this method does not publish the index into the - dataset manifest. Instead, it writes one segment under + canonical bitmap scalar, INVERTED scalar, NGRAM scalar, RTREE scalar, + ZONEMAP scalar, BLOOMFILTER scalar, and LABEL_LIST scalar index construction. + Unlike :meth:`create_index`, this method does not publish the index into + the dataset manifest. Instead, it writes one segment under ``_indices//`` and returns the resulting :class:`Index` metadata. @@ -4197,8 +4384,11 @@ def create_index_uncommitted( 4. commit the final segment list with :meth:`commit_existing_index_segments` - BTREE, BITMAP, INVERTED, and ZONEMAP segments may - be merged with :meth:`merge_existing_index_segments` before commit. + BTREE, BITMAP, INVERTED, NGRAM, RTREE, ZONEMAP, BLOOMFILTER, and + LABEL_LIST segments may be merged with + :meth:`merge_existing_index_segments` before commit. NGRAM segments + built before a deferred compaction must be merged before commit so + their postings can be rebuilt against current row addresses. Parameters are the same as :meth:`create_index`, with one additional requirement: @@ -4294,13 +4484,22 @@ def drop_index(self, name: str): """ return self._ds.drop_index(name) - def prewarm_index(self, name: str, *, with_position: bool = False): + def prewarm_index( + self, + name: str, + *, + with_position: bool = False, + index_segments: Optional[Iterable[Union[str, uuid.UUID]]] = None, + ): """ Prewarm an index - This will load the entire index into memory. This can help avoid cold start - issues with index queries. If the index does not fit in the index cache, then - this will result in wasted I/O. + By default, this will load the entire index into memory. This can help + avoid cold start issues with index queries. If the index does not fit in + the index cache, then this will result in wasted I/O. + + Use ``session().index_cache_size_bytes()`` before and after prewarm to + inspect how much the index cache grew. Parameters ---------- @@ -4310,8 +4509,16 @@ def prewarm_index(self, name: str, *, with_position: bool = False): This is only supported for ``INVERTED`` indices. If True, positions are also loaded into the cache during prewarm so phrase queries do not need a separate lazy positions read. - """ - return self._ds.prewarm_index(name, with_position=with_position) + index_segments: iterable of str or uuid.UUID, default None + If specified, prewarm only these physical index segment UUIDs from the + named logical index. Use :meth:`describe_indices` to inspect logical + indices and obtain segment UUIDs from ``IndexDescription.segments``. + """ + return self._ds.prewarm_index( + name, + with_position=with_position, + index_segments=_normalize_index_segment_ids(index_segments), + ) def merge_index_metadata( self, @@ -5057,7 +5264,6 @@ def initialize_mem_wal( identity_column: Optional[str] = None, unsharded: bool = False, durable_write: Optional[bool] = None, - sync_indexed_write: Optional[bool] = None, max_wal_buffer_size: Optional[int] = None, max_wal_flush_interval_ms: Optional[int] = None, max_memtable_size: Optional[int] = None, @@ -5065,8 +5271,6 @@ def initialize_mem_wal( max_memtable_batches: Optional[int] = None, max_unflushed_memtable_bytes: Optional[int] = None, manifest_scan_batch_size: Optional[int] = None, - async_index_buffer_rows: Optional[int] = None, - async_index_interval_ms: Optional[int] = None, backpressure_log_interval_ms: Optional[int] = None, stats_log_interval_ms: Optional[int] = None, hnsw_params: Optional[Dict[str, Dict[str, int]]] = None, @@ -5127,7 +5331,6 @@ def initialize_mem_wal( identity_column=identity_column, unsharded=unsharded, durable_write=durable_write, - sync_indexed_write=sync_indexed_write, max_wal_buffer_size=max_wal_buffer_size, max_wal_flush_interval_ms=max_wal_flush_interval_ms, max_memtable_size=max_memtable_size, @@ -5135,8 +5338,6 @@ def initialize_mem_wal( max_memtable_batches=max_memtable_batches, max_unflushed_memtable_bytes=max_unflushed_memtable_bytes, manifest_scan_batch_size=manifest_scan_batch_size, - async_index_buffer_rows=async_index_buffer_rows, - async_index_interval_ms=async_index_interval_ms, backpressure_log_interval_ms=backpressure_log_interval_ms, stats_log_interval_ms=stats_log_interval_ms, hnsw_params=hnsw_params, @@ -5159,7 +5360,6 @@ def mem_wal_writer( shard_id: str, *, durable_write: Optional[bool] = None, - sync_indexed_write: Optional[bool] = None, max_wal_buffer_size: Optional[int] = None, max_wal_flush_interval_ms: Optional[int] = None, max_memtable_size: Optional[int] = None, @@ -5167,8 +5367,6 @@ def mem_wal_writer( max_memtable_batches: Optional[int] = None, max_unflushed_memtable_bytes: Optional[int] = None, manifest_scan_batch_size: Optional[int] = None, - async_index_buffer_rows: Optional[int] = None, - async_index_interval_ms: Optional[int] = None, backpressure_log_interval_ms: Optional[int] = None, stats_log_interval_ms: Optional[int] = None, hnsw_params: Optional[Dict[str, Dict[str, int]]] = None, @@ -5186,8 +5384,6 @@ def mem_wal_writer( ``str(uuid.uuid4())``). durable_write : bool, optional Whether to fsync WAL writes (default: ``True``). - sync_indexed_write : bool, optional - Whether index updates are synchronous (default: ``True``). max_wal_buffer_size : int, optional Maximum WAL buffer size in bytes (default: 10 MB). max_wal_flush_interval_ms : int, optional @@ -5202,10 +5398,6 @@ def mem_wal_writer( Maximum unflushed bytes before backpressure (default: 1 GB). manifest_scan_batch_size : int, optional Batch size for manifest scans (default: 2). - async_index_buffer_rows : int, optional - Buffer rows for async index updates (default: 10 000). - async_index_interval_ms : int, optional - Interval for async index updates in milliseconds (default: 1000). backpressure_log_interval_ms : int, optional Interval for backpressure log messages in milliseconds (default: 30 000). @@ -5253,7 +5445,6 @@ def mem_wal_writer( name: val for name, val in [ ("durable_write", durable_write), - ("sync_indexed_write", sync_indexed_write), ("max_wal_buffer_size", max_wal_buffer_size), ("max_wal_flush_interval_ms", max_wal_flush_interval_ms), ("max_memtable_size", max_memtable_size), @@ -5261,8 +5452,6 @@ def mem_wal_writer( ("max_memtable_batches", max_memtable_batches), ("max_unflushed_memtable_bytes", max_unflushed_memtable_bytes), ("manifest_scan_batch_size", manifest_scan_batch_size), - ("async_index_buffer_rows", async_index_buffer_rows), - ("async_index_interval_ms", async_index_interval_ms), ("backpressure_log_interval_ms", backpressure_log_interval_ms), ("stats_log_interval_ms", stats_log_interval_ms), ("hnsw_params", hnsw_params), @@ -5414,6 +5603,21 @@ def with_row_addr(self, with_row_addr: bool = True) -> "SqlQueryBuilder": self._builder = self._builder.with_row_addr(with_row_addr) return self + def blob_handling( + self, + blob_handling: Literal["all_binary", "blobs_descriptions", "all_descriptions"], + ) -> "SqlQueryBuilder": + """ + Control how blob columns are returned by this SQL query. + + - ``"all_binary"`` materializes blob columns as binary values. + - ``"blobs_descriptions"`` returns blob descriptors (the default). + - ``"all_descriptions"`` returns descriptions for all binary-like + columns. + """ + self._builder = self._builder.blob_handling(blob_handling) + return self + def build(self) -> SqlQuery: """ Build the query. @@ -5615,7 +5819,13 @@ class Overwrite(BaseOperation): new_schema: pyarrow.Schema The schema of the new dataset. fragments: list[FragmentMetadata] - The fragments that make up the new dataset. + The newly written fragments that make up the new dataset. They are + assigned fresh ids when the operation is committed, continuing from + the highest id the dataset has ever used, so any id they carry is + ignored. Since we reassign fragment ids, a fragment with a deletion + file is rejected: use :class:`LanceOperation.Delete` to commit + deletions, or :class:`LanceOperation.Merge` to change the schema of + existing fragments. initial_bases: list[DatasetBasePath], optional Base paths to register when creating a new dataset (CREATE mode only). **Only valid in CREATE mode**. Will raise an error if used with @@ -5786,8 +5996,15 @@ class Update(BaseOperation): The ids of the fragments that have been removed entirely. updated_fragments: list[FragmentMetadata] The fragments that have been updated with new deletion vectors. + These are used as given, so pass back the metadata read from the + dataset rather than a freshly constructed object, or the fragment + loses its row id and version metadata. new_fragments: list[FragmentMetadata] - The fragments that contain the new rows. + The fragments that contain the new rows. On a dataset that uses + stable row ids, set ``row_id_meta`` on these to carry the ids of + rewritten rows over; see :class:`lance.fragment.RowIdSequence`. The + created-at and last-updated-at version metadata are derived during + the commit and should be left as None. fields_modified: list[int] If any fields are modified in updated_fragments, then they must be listed here so those fragments can be removed from indices covering @@ -5795,6 +6012,11 @@ class Update(BaseOperation): fields_for_preserving_frag_bitmap: list[int] The fields that used to judge whether to preserve the new frag's id into the frag bitmap of the specified indices. + updated_fragment_offsets: dict[int, bytes], optional + Physical row offsets that matched the update, keyed by fragment id, + each serialized in the portable RoaringBitmap format. Set on + ``rewrite_columns`` updates over stable row ids so the commit + refreshes row-level version metadata for the matched rows only. """ removed_fragment_ids: List[int] = dataclasses.field(default_factory=list) @@ -5807,6 +6029,7 @@ class Update(BaseOperation): default_factory=list ) update_mode: str = "" + updated_fragment_offsets: Optional[Dict[int, bytes]] = None def __post_init__(self): LanceOperation._validate_fragments(self.updated_fragments) @@ -5825,6 +6048,14 @@ class Merge(BaseOperation): schema: LanceSchema or pyarrow.Schema The schema of the new dataset. Passing a LanceSchema is preferred, and passing a pyarrow.Schema is deprecated. + preserves_nullability: bool + True when this merge makes no nullability-affecting schema change: + it introduces no field that data staged against an earlier schema + could not safely omit. Without the assertion (the default) the + merge conservatively conflicts with concurrent appends, whose + fragments would omit new columns and read as null; that can only + cause a retry. Pass True when every column this merge introduces + is nullable to let concurrent appends commit without conflict. Warning ------- @@ -5870,6 +6101,7 @@ class Merge(BaseOperation): fragments: Iterable[FragmentMetadata] schema: LanceSchema | pa.Schema + preserves_nullability: bool = False def __post_init__(self): if isinstance(self.schema, pa.Schema): @@ -5962,6 +6194,76 @@ class DataReplacement(BaseOperation): replacements: List[LanceOperation.DataReplacementGroup] + @dataclass + class DataOverlayFile: + """ + An overlay file supplying new values for a subset of + ``(physical offset, field)`` cells of a fragment, resolved on read and + layered over the base data without rewriting the base files. + + The overlay is dense or sparse depending on the shape of ``offsets``: + pass a flat ``List[int]`` for a dense overlay (one offset list shared by + every field in ``data_file``) or a ``List[List[int]]`` for a sparse + overlay (one offset list per field, in the order of the file's fields). + Offsets are **physical** row offsets (positions in the base files, + counting deleted rows), like deletion vectors. + + Attributes + ---------- + data_file : DataFile + The Lance data file storing the overlay's new cell values — one + value column per covered field. The value at each covered offset is + stored at the rank (0-based count of covered offsets below it) of + that offset in the field's coverage. + offsets : Union[List[int], List[List[int]]] + The covered physical row offsets. A flat list is dense coverage + (shared by every field); a list of per-field lists is sparse + coverage (in field order). Each list must be strictly ascending + with no duplicates, since the Nth offset maps to the Nth value row + in ``data_file``; a non-ascending list raises ``ValueError``. + committed_version : Optional[int] + The dataset version at which this overlay became effective. Leave as + ``None`` when creating an overlay to commit — the commit stamps it. + It is populated when reading an existing fragment's overlays so they + round-trip through :class:`FragmentMetadata`. + """ + + data_file: DataFile + offsets: Union[List[int], List[List[int]]] + committed_version: Optional[int] = None + + @dataclass + class DataOverlayGroup: + """ + Overlay files to append to a single fragment. + + Attributes + ---------- + fragment_id : int + The id of the fragment the overlays apply to. + overlays : List[LanceOperation.DataOverlayFile] + The overlay files to append, ordered oldest-first (a later entry is + newer and wins where coverage overlaps). + """ + + fragment_id: int + overlays: List[LanceOperation.DataOverlayFile] + + @dataclass + class DataOverlay(BaseOperation): + """ + Operation that appends data overlay files to fragments. + + Overlays are appended to each fragment's existing overlays (overlays + written by concurrent commits are preserved) and resolved on read + over the base data without rewriting it. + + If multiple groups target the same data then the values in the + latest group take precedence. + """ + + groups: List[LanceOperation.DataOverlayGroup] + @dataclass class Project(BaseOperation): """ @@ -5972,6 +6274,11 @@ class Project(BaseOperation): ---------- schema: LanceSchema The lance schema of the new dataset. + preserves_nullability: bool + True when this projection makes no nullability-affecting schema + change, as a rename or a drop does not. Without the assertion + (the default) the projection conservatively conflicts with + concurrent writes, which can only cause a retry. Examples -------- @@ -6000,6 +6307,7 @@ class Project(BaseOperation): """ schema: LanceSchema + preserves_nullability: bool = False @dataclass class UpdateMap: @@ -6138,9 +6446,8 @@ def apply_defaults(self, default_opts: Dict[str, Any]) -> ScannerBuilder: def batch_size(self, batch_size: int) -> ScannerBuilder: """Set the maximum number of rows per batch. - Note: this can be overridden by ``batch_size_bytes`` or by a - dataset-level ``batch_size_bytes`` configured via - ``FileReaderOptions``. + If a byte limit is also configured, both limits apply and the one + reached first determines the batch size. """ self._batch_size = batch_size return self @@ -6149,7 +6456,8 @@ def batch_size_bytes(self, batch_size_bytes: int) -> ScannerBuilder: """Set the target batch size in bytes. When set, the scanner will produce batches whose total size in bytes - is approximately this value, overriding the row-based ``batch_size``. + is approximately this value. If ``batch_size`` is also set, both + limits apply and the one reached first determines the batch size. This can also be configured at the dataset level via ``FileReaderOptions``. A scanner-level setting takes precedence @@ -6182,10 +6490,12 @@ def io_buffer_size(self, io_buffer_size: int) -> ScannerBuilder: def batch_readahead(self, nbatches: Optional[int] = None) -> ScannerBuilder: """ - This parameter is ignored when reading v2 files + Set the maximum number of batches to decode concurrently. + + This parameter must be greater than zero. """ - if nbatches is not None and int(nbatches) < 0: - raise ValueError("batch_readahead must be non-negative") + if nbatches is not None and int(nbatches) <= 0: + raise ValueError("batch_readahead must be greater than 0") self._batch_readahead = nbatches return self @@ -6394,19 +6704,7 @@ def with_fragments( def with_index_segments( self, index_segments: Optional[Iterable[Union[str, uuid.UUID]]] ) -> ScannerBuilder: - if index_segments is not None: - segment_ids = [] - for segment_id in index_segments: - if isinstance(segment_id, (str, uuid.UUID)): - segment_ids.append(str(segment_id)) - else: - raise TypeError( - "index_segments must be an iterable of str or uuid.UUID. " - f"Got {type(segment_id)} instead." - ) - index_segments = segment_ids - - self._index_segments = index_segments + self._index_segments = _normalize_index_segment_ids(index_segments) return self def nearest( @@ -6538,6 +6836,9 @@ def strict_batch_size(self, strict_batch_size: bool = False) -> ScannerBuilder: If this is true then small batches will need to be merged together which will require a data copy and incur a (typically very small) performance penalty. + + This cannot be combined with ``batch_size_bytes`` because merging + batches to the strict row count can exceed the byte limit. """ self._strict_batch_size = strict_batch_size return self @@ -6795,6 +7096,9 @@ def explain_plan(self, verbose=False) -> str: def analyze_plan(self, count_rows: bool = False) -> str: """Execute the plan for this scanner and display with runtime metrics. + Full-text-search nodes include the execution-time ``tokenized_query`` + text and positions. + Parameters ---------- count_rows : bool, default False @@ -6829,6 +7133,7 @@ def compact_files( Literal["reencode", "try_binary_copy", "force_binary_copy"] ] = None, binary_copy_read_batch_bytes: Optional[int] = None, + max_source_fragments: Optional[int] = None, ) -> CompactionMetrics: """Compacts small files in the dataset, reducing total number of files. @@ -6859,7 +7164,8 @@ def compact_files( ``lance.compaction.defer_index_remap``, ``lance.compaction.batch_size``, ``lance.compaction.compaction_mode``, - ``lance.compaction.binary_copy_read_batch_bytes``. + ``lance.compaction.binary_copy_read_batch_bytes``, + ``lance.compaction.max_source_fragments``. Parameters ---------- @@ -6912,6 +7218,12 @@ def compact_files( The batch size in bytes for reading during binary copy operations. Controls how much data is read at once when performing binary copy. Defaults to 16MB. + max_source_fragments: int, optional + Maximum number of source fragments to compact in a single run. + Compaction tasks are included until adding the next task would + exceed this limit, allowing compaction to proceed incrementally. + Fragments are processed oldest first. If not specified, uses the + manifest config value, or applies no limit. Returns ------- @@ -6935,6 +7247,7 @@ def compact_files( batch_size=batch_size, compaction_mode=compaction_mode, binary_copy_read_batch_bytes=binary_copy_read_batch_bytes, + max_source_fragments=max_source_fragments, ).items() if v is not None } @@ -7018,7 +7331,7 @@ def list(self) -> dict[str, Tag]: """ return self._ds.tags() - def get_version(self, tag: str) -> Optional[int]: + def get_version(self, tag: str) -> int: """ Get the version of a specific tag by name. @@ -7029,8 +7342,13 @@ def get_version(self, tag: str) -> Optional[int]: Returns ------- - int or None - The version number of the tag if it exists, otherwise None. + int + The version number of the tag. + + Raises + ------ + ValueError + If the tag does not exist. Use :meth:`list` to check for presence. """ return self._ds.get_version(tag) @@ -7564,8 +7882,7 @@ def _coerce_query_vector(query: QueryVectorLike) -> tuple[pa.Array, int]: if isinstance(query.type, pa.FixedSizeListType): query = query.values elif isinstance(query, (list, tuple)) or ( - _check_for_numpy(query), - isinstance(query, np.ndarray), + _check_for_numpy(query) and isinstance(query, np.ndarray) ): query = np.array(query).astype("float64") # workaround for GH-608 query = pa.FloatingPointArray.from_pandas(query, type=pa.float32()) diff --git a/python/python/lance/dependencies.py b/python/python/lance/dependencies.py index def9a052504..e1435e6fca1 100644 --- a/python/python/lance/dependencies.py +++ b/python/python/lance/dependencies.py @@ -27,7 +27,7 @@ _CAGRA_AVAILABLE = True _RAFT_COMMON_AVAILABLE = True _HUGGING_FACE_AVAILABLE = True -_TENSORFLOW_AVAILABLE = True +_PYDANTIC_AVAILABLE = True class _LazyModule(ModuleType): @@ -49,7 +49,6 @@ class _LazyModule(ModuleType): "pandas": "pd.", "polars": "pl.", "torch": "torch.", - "tensorflow": "tf.", } def __init__( @@ -163,7 +162,6 @@ def _lazy_import(module_name: str) -> tuple[ModuleType, bool]: import numpy import pandas import polars - import tensorflow # type: ignore[reportMissingImports] import torch # type: ignore[reportMissingImports] else: # heavy/optional third party libs @@ -172,7 +170,7 @@ def _lazy_import(module_name: str) -> tuple[ModuleType, bool]: polars, _POLARS_AVAILABLE = _lazy_import("polars") torch, _TORCH_AVAILABLE = _lazy_import("torch") datasets, _HUGGING_FACE_AVAILABLE = _lazy_import("datasets") - tensorflow, _TENSORFLOW_AVAILABLE = _lazy_import("tensorflow") + _, _PYDANTIC_AVAILABLE = _lazy_import("pydantic") @lru_cache(maxsize=None) @@ -215,33 +213,77 @@ def _check_for_hugging_face(obj: Any, *, check_type: bool = True) -> bool: ) -def _check_for_tensorflow(obj: Any, *, check_type: bool = True) -> bool: - return _TENSORFLOW_AVAILABLE and _might_be( - cast("Hashable", type(obj) if check_type else obj), "tensorflow" +def _check_for_pydantic(obj: Any, *, check_type: bool = True) -> bool: + return _PYDANTIC_AVAILABLE and _might_be( + cast("Hashable", type(obj) if check_type else obj), "pydantic" ) +def _is_pydantic_base_model(obj: Any) -> bool: + if not _PYDANTIC_AVAILABLE: + return False + from pydantic import BaseModel + + return isinstance(obj, BaseModel) + + +def _is_pydantic_base_model_class(obj: Any) -> bool: + if not _PYDANTIC_AVAILABLE: + return False + from pydantic import BaseModel + + return isinstance(obj, type) and issubclass(obj, BaseModel) + + +def model_to_dict(obj: Any) -> dict[str, Any]: + return obj.model_dump() if hasattr(obj, "model_dump") else obj.dict() + + +def _validate_pydantic_list(data: Any, model_class: type) -> None: + """Validate that `data` is a list of exact `model_class` instances. + + Rejects non-list iterables (e.g. generators), which would otherwise be + drained by this validation loop and leave nothing for the caller to + serialize, and rejects subclass instances, whose extra/overridden fields + would not match the schema derived from `model_class`. + """ + if not isinstance(data, list): + raise TypeError( + f"Pydantic model data must be provided as a list, got {type(data)!r}" + ) + for i, item in enumerate(data): + if type(item) is not model_class: + raise TypeError( + f"data[{i}] must be an instance of {model_class!r} exactly " + f"(subclasses are not accepted), got {type(item)!r} " + f"(data has {len(data)} items)" + ) + + __all__ = [ # lazy-load third party libs "datasets", "numpy", "pandas", "polars", - "tensorflow", "torch", # lazy utilities "_check_for_hugging_face", "_check_for_numpy", "_check_for_pandas", "_check_for_polars", - "_check_for_tensorflow", + "_check_for_pydantic", "_check_for_torch", + "_is_pydantic_base_model", + "_is_pydantic_base_model_class", "_LazyModule", + "model_to_dict", + "_validate_pydantic_list", # exported flags/guards "_NUMPY_AVAILABLE", "_PANDAS_AVAILABLE", "_POLARS_AVAILABLE", + "_PYDANTIC_AVAILABLE", "_TORCH_AVAILABLE", "_HUGGING_FACE_AVAILABLE", - "_TENSORFLOW_AVAILABLE", ] diff --git a/python/python/lance/file.py b/python/python/lance/file.py index 5926241977e..89210099ccf 100644 --- a/python/python/lance/file.py +++ b/python/python/lance/file.py @@ -482,6 +482,12 @@ class LanceFileWriter: This class is used to write Lance data files, a low level structure optimized for storing multi-modal tabular data. If you are working with Lance datasets then you should use the LanceDataset class instead. + + Attributes + ---------- + size_bytes: Optional[int] + The final size of the file in bytes. This is None until `close` is + called. """ def __init__( @@ -548,6 +554,7 @@ def __init__( **kwargs, ) self.closed = False + self.size_bytes: Optional[int] = None def write_batch(self, batch: Union[pa.RecordBatch, pa.Table]) -> None: """ @@ -569,11 +576,17 @@ def close(self) -> Optional[int]: Write the file metadata and close the file Returns the number of rows written to the file + + After this returns, ``size_bytes`` holds the final size of the file. This + is reported by the writer itself, so it is available for object stores + without issuing a separate metadata request. """ if self.closed: return self.closed = True - return self._writer.finish() + summary = self._writer.finish() + self.size_bytes = summary.size_bytes + return summary.num_rows def add_schema_metadata(self, key: str, value: str) -> None: """ diff --git a/python/python/lance/fragment.py b/python/python/lance/fragment.py index adf220e59f6..d42ae6b3310 100644 --- a/python/python/lance/fragment.py +++ b/python/python/lance/fragment.py @@ -35,7 +35,11 @@ from .lance import ( RowIdMeta as RowIdMeta, ) +from .lance import ( + RowIdSequence as RowIdSequence, +) from .lance import _Fragment, _write_fragments, _write_fragments_transaction +from .lance import _Session as Session from .progress import FragmentWriteProgress, NoopFragmentWriteProgress from .types import _coerce_reader from .udf import BatchUDF, normalize_transform @@ -45,6 +49,7 @@ ColumnOrdering, DatasetBasePath, LanceDataset, + LanceOperation, LanceScanner, ReaderLike, Transaction, @@ -73,11 +78,26 @@ class FragmentMetadata: deletion_file : Optional[DeletionFile] The deletion file, if any. row_id_meta : Optional[RowIdMeta] - The row id metadata, if any. + The stable row ids of this fragment's rows, if any. When committing a + transaction by hand on a dataset that uses stable row ids, set this to + carry the ids of rewritten rows over to their new fragment; build it with + :class:`RowIdSequence`. Rows left without an id are treated as newly + inserted and are assigned ids during the commit. created_at_version_meta : Optional[RowDatasetVersionMeta] - The row created at version metadata, if any. + The dataset version each row was created in. Derived during the commit + from ``row_id_meta`` -- a rewritten row keeps the version it first + appeared in -- so leave this as None when building a transaction. Any + value set here is ignored for newly written fragments. last_updated_at_version_meta : Optional[RowDatasetVersionMeta] - The row last updated at version metadata, if any. + The dataset version each row was last modified in. Derived during the + commit, like ``created_at_version_meta``; leave this as None. It cannot + be computed ahead of time because a commit that loses a race is retried + against a later version than the one it was built for. + overlays : List[LanceOperation.DataOverlayFile] + The data overlay files layered over this fragment's base data, if any. + Overlays are created via :class:`LanceOperation.DataOverlay`; they are + carried here so they survive operations that round-trip fragment + metadata (e.g. a manual ``Delete``, ``Update``, or ``Merge`` commit). """ id: int @@ -87,6 +107,7 @@ class FragmentMetadata: row_id_meta: Optional[RowIdMeta] = None created_at_version_meta: Optional[RowDatasetVersionMeta] = None last_updated_at_version_meta: Optional[RowDatasetVersionMeta] = None + overlays: List["LanceOperation.DataOverlayFile"] = field(default_factory=list) @property def num_deletions(self) -> int: @@ -110,12 +131,25 @@ def data_files(self) -> List[DataFile]: def to_json(self) -> dict: """Get this as a simple JSON-serializable dictionary.""" - files = [asdict(f) for f in self.files] - for f in files: - f["path"] = f.pop("_path") + + def _data_file_to_json(f: DataFile) -> dict: + d = asdict(f) + d["path"] = d.pop("_path") + return d + + files = [_data_file_to_json(f) for f in self.files] + overlays = [ + dict( + data_file=_data_file_to_json(o.data_file), + offsets=o.offsets, + committed_version=o.committed_version, + ) + for o in self.overlays + ] return dict( id=self.id, files=files, + overlays=overlays, physical_rows=self.physical_rows, deletion_file=( self.deletion_file.asdict() if self.deletion_file is not None else None @@ -159,6 +193,20 @@ def from_json(json_data: str) -> FragmentMetadata: json.dumps(last_updated_at_version_meta) ) + overlays = [] + overlays_json = json_data.get("overlays") + if overlays_json: + from .dataset import LanceOperation + + overlays = [ + LanceOperation.DataOverlayFile( + data_file=DataFile(**o["data_file"]), + offsets=o["offsets"], + committed_version=o.get("committed_version"), + ) + for o in overlays_json + ] + return FragmentMetadata( id=json_data["id"], files=[DataFile(**f) for f in json_data["files"]], @@ -167,6 +215,7 @@ def from_json(json_data: str) -> FragmentMetadata: row_id_meta=row_id_meta, created_at_version_meta=created_at_version_meta, last_updated_at_version_meta=last_updated_at_version_meta, + overlays=overlays, ) @@ -304,10 +353,7 @@ def __repr__(self): return self._fragment.__repr__() def __reduce__(self): - from .dataset import LanceDataset - - ds = LanceDataset(self._ds.uri, self._ds.version) - return LanceFragment, (ds, self.fragment_id) + return LanceFragment, (self._ds, self.fragment_id) @staticmethod def create_from_file( @@ -349,6 +395,7 @@ def create( storage_options: Optional[Dict[str, str]] = None, namespace_client: Optional["LanceNamespace"] = None, table_id: Optional[List[str]] = None, + session: Optional[Session] = None, ) -> FragmentMetadata: """Create a :class:`FragmentMetadata` from the given data. @@ -397,6 +444,9 @@ def create( table_id : optional, List[str] The table identifier when using a namespace (e.g., ["my_table"]). Must be provided together with `namespace_client`. + session : optional, Session + A session to reuse across operations. The session holds shared + caches (metadata and index) and the object store registry. See Also -------- @@ -450,6 +500,7 @@ def create( storage_options=storage_options, namespace_client=namespace_client, table_id=table_id, + session=session, ) @property @@ -480,6 +531,16 @@ def physical_rows(self) -> int: """ return self._fragment.physical_rows + def validate(self) -> None: + """ + Validate the fragment. + + This checks the integrity of the fragment and will raise an exception if + the fragment is corrupted. Unlike :meth:`lance.LanceDataset.validate`, + which checks every fragment, this validates only this fragment. + """ + self._fragment.validate() + @property def physical_schema(self) -> pa.Schema: # override the pyarrow super class method otherwise causes segfault @@ -1036,6 +1097,7 @@ def write_fragments( allow_external_blob_outside_bases: bool = False, namespace_client: Optional[LanceNamespace] = None, table_id: Optional[List[str]] = None, + session: Optional[Session] = None, ) -> Transaction: ... @overload @@ -1062,6 +1124,7 @@ def write_fragments( allow_external_blob_outside_bases: bool = False, namespace_client: Optional[LanceNamespace] = None, table_id: Optional[List[str]] = None, + session: Optional[Session] = None, ) -> List[FragmentMetadata]: ... @@ -1088,6 +1151,7 @@ def write_fragments( allow_external_blob_outside_bases: bool = False, namespace_client: Optional[LanceNamespace] = None, table_id: Optional[List[str]] = None, + session: Optional[Session] = None, ) -> List[FragmentMetadata] | Transaction: """ Write data into one or more fragments. @@ -1196,6 +1260,9 @@ def write_fragments( table_id : optional, List[str] The table identifier when using a namespace (e.g., ["my_table"]). Must be provided together with `namespace_client`. + session : optional, Session + A session to reuse across operations. The session holds shared caches + (metadata and index) and the object store registry. Returns ------- @@ -1232,6 +1299,12 @@ def write_fragments( base_store_params = dataset_uri._base_store_params if storage_options is None: storage_options = dataset_uri._storage_options + if session is not None and not session.is_same_as(dataset_uri.session()): + raise ValueError( + "The provided session is not the destination dataset's own " + "session. Please pass the dataset's session or omit the " + "'session' parameter." + ) dataset_uri = dataset_uri._ds elif not isinstance(dataset_uri, str): raise TypeError(f"Unknown dataset_uri type {type(dataset_uri)}") @@ -1267,6 +1340,7 @@ def write_fragments( base_store_params=base_store_params, external_blob_mode=external_blob_mode, allow_external_blob_outside_bases=allow_external_blob_outside_bases, + session=session, ) diff --git a/python/python/lance/indices/builder.py b/python/python/lance/indices/builder.py index 6059166d6ba..e235f348979 100644 --- a/python/python/lance/indices/builder.py +++ b/python/python/lance/indices/builder.py @@ -164,6 +164,7 @@ def train_pq( *, sample_rate: int = 256, max_iters: int = 50, + num_bits: int = 8, fragment_ids: Optional[list[int]] = None, ) -> PqModel: """ @@ -195,6 +196,8 @@ def train_pq( This parameter is used in the same way as in the IVF model. max_iters: int This parameter is used in the same way as in the IVF model. + num_bits: int + The number of bits used to encode each PQ centroid. fragment_ids: list[int], optional If provided, train using only the specified fragments from the dataset. """ @@ -202,7 +205,7 @@ def train_pq( num_rows = self._count_rows(fragment_ids) num_subvectors = self._normalize_pq_params(num_subvectors, self.dimension) - self._verify_pq_sample_rate(num_rows, sample_rate) + self._verify_pq_sample_rate(num_rows, sample_rate, num_bits) distance_type = ivf_model.distance_type pq_codebook = indices.train_pq_model( self.dataset._ds, @@ -214,8 +217,9 @@ def train_pq( max_iters, ivf_model.centroids, fragment_ids, + num_bits, ) - return PqModel(num_subvectors, pq_codebook) + return PqModel(num_subvectors, pq_codebook, num_bits=num_bits) def prepare_global_ivf_pq( self, @@ -226,6 +230,7 @@ def prepare_global_ivf_pq( accelerator: Optional[Union[str, "torch.Device"]] = None, sample_rate: int = 256, max_iters: int = 50, + num_bits: int = 8, fragment_ids: Optional[list[int]] = None, ) -> dict: """ @@ -267,6 +272,7 @@ def prepare_global_ivf_pq( num_subvectors, sample_rate=sample_rate, max_iters=max_iters, + num_bits=num_bits, fragment_ids=fragment_ids, ) @@ -381,6 +387,7 @@ def transform_vectors( dest_uri, fragments, partition_ds_uri, + pq.num_bits, ) def shuffle_transformed_vectors( @@ -471,6 +478,7 @@ def load_shuffled_vectors( num_subvectors, distance_type, index_name, + pq.num_bits, ) else: raise ValueError("filenames must be a list of strings") @@ -526,13 +534,17 @@ def _verify_base_sample_rate(self, sample_rate: int): f"The sample_rate must be an int greater than 1, got {sample_rate}" ) - def _verify_pq_sample_rate(self, num_rows: int, sample_rate: int): + def _verify_pq_sample_rate( + self, num_rows: int, sample_rate: int, num_bits: int = 8 + ): self._verify_base_sample_rate(sample_rate) - if 256 * sample_rate > num_rows: + required_rows = (2**num_bits) * sample_rate + if required_rows > num_rows: raise ValueError( "There are not enough rows in the dataset to create PQ" - f" codebook with a sample rate of {sample_rate}. {sample_rate * 256}" - f" rows needed and there are {num_rows}" + f" codebook with a sample rate of {sample_rate} and num_bits" + f" of {num_bits}. {required_rows} rows needed and there are" + f" {num_rows}" ) def _verify_ivf_sample_rate( diff --git a/python/python/lance/indices/pq.py b/python/python/lance/indices/pq.py index b3aeb50bcbe..e3d334ccf48 100644 --- a/python/python/lance/indices/pq.py +++ b/python/python/lance/indices/pq.py @@ -14,9 +14,13 @@ class PqModel: Can be saved / loaded to checkpoint progress. """ - def __init__(self, num_subvectors: int, codebook: pa.FixedSizeListArray): + def __init__( + self, num_subvectors: int, codebook: pa.FixedSizeListArray, *, num_bits: int = 8 + ): self.num_subvectors = num_subvectors """The number of subvectors to divide source vectors into""" + self.num_bits = num_bits + """The number of bits used to encode each PQ centroid""" self.codebook = codebook """The centroids of the PQ clusters""" @@ -42,7 +46,10 @@ def save(self, uri: str, *, storage_options: Optional[Dict[str, str]] = None): uri, pa.schema( [pa.field("codebook", self.codebook.type)], - metadata={b"num_subvectors": str(self.num_subvectors).encode()}, + metadata={ + b"num_subvectors": str(self.num_subvectors).encode(), + b"num_bits": str(self.num_bits).encode(), + }, ), storage_options=storage_options, ) as writer: @@ -65,9 +72,10 @@ def load(cls, uri: str, *, storage_options: Optional[Dict[str, str]] = None): """ reader = LanceFileReader(uri, storage_options=storage_options) num_rows = reader.metadata().num_rows - metadata = reader.metadata().schema.metadata + metadata = reader.metadata().schema.metadata or {} num_subvectors = int(metadata[b"num_subvectors"].decode()) + num_bits = int(metadata.get(b"num_bits", b"8").decode()) codebook = ( reader.read_all(batch_size=num_rows).to_table().column("codebook").chunk(0) ) - return cls(num_subvectors, codebook) + return cls(num_subvectors, codebook, num_bits=num_bits) diff --git a/python/python/lance/lance/__init__.pyi b/python/python/lance/lance/__init__.pyi index f050c4c8422..1ddadae051e 100644 --- a/python/python/lance/lance/__init__.pyi +++ b/python/python/lance/lance/__init__.pyi @@ -63,6 +63,9 @@ from .fragment import ( from .fragment import ( RowIdMeta as RowIdMeta, ) +from .fragment import ( + RowIdSequence as RowIdSequence, +) from .indices import IndexDescription as IndexDescription from .indices import IndexSegment as IndexSegment from .lance import PySearchFilter @@ -87,6 +90,56 @@ from .trace import capture_trace_events as capture_trace_events from .trace import shutdown_tracing as shutdown_tracing from .trace import trace_to_chrome as trace_to_chrome +class MetricPoint: + name: str + kind: str + attributes: Dict[str, str] + value: Optional[float] + buckets: Optional[List[Tuple[str, int]]] + count: Optional[int] + sum: Optional[float] + +class MetricDescription: + name: str + kind: str + unit: Optional[str] + description: str + +def register_lance_metrics_recorder() -> bool: ... +def lance_metrics_catalog() -> List[MetricDescription]: ... +def snapshot_lance_metrics() -> List[MetricPoint]: ... + +class FtsToken: + text: str + position: int + def __repr__(self) -> str: ... + +def tokenize( + query: str, + *, + analyzer: Optional[Literal["text", "code"]] = None, + base_tokenizer: Optional[str] = None, + language: Optional[str] = None, + max_token_length: Optional[int] = 40, + lower_case: Optional[bool] = None, + stem: Optional[bool] = None, + remove_stop_words: Optional[bool] = None, + custom_stop_words: Optional[List[str]] = None, + ascii_folding: Optional[bool] = None, + min_ngram_length: Optional[int] = None, + max_ngram_length: Optional[int] = None, + prefix_only: Optional[bool] = None, + split_identifiers: Optional[bool] = None, + split_on_numerics: Optional[bool] = None, + preserve_original: Optional[bool] = None, + index_operators: Optional[bool] = None, +) -> List[FtsToken]: + """Tokenize an FTS query without an index. + + ``max_token_length`` defaults to 40; pass ``None`` to disable the limit. + """ + ... + class CleanupStats: bytes_removed: int old_versions: int @@ -115,6 +168,12 @@ class CleanupExplanation: referenced_branches: List[CleanupReferencedBranch] warnings: List[str] +class LanceFileWriteSummary: + num_rows: int + size_bytes: int + + def __repr__(self) -> str: ... + class LanceFileWriter: def __init__( self, @@ -129,7 +188,7 @@ class LanceFileWriter: max_page_bytes: Optional[int], ): ... def write_batch(self, batch: pa.RecordBatch) -> None: ... - def finish(self) -> int: ... + def finish(self) -> LanceFileWriteSummary: ... def add_schema_metadata(self, key: str, value: str) -> None: ... def add_global_buffer(self, data: bytes) -> int: ... @@ -141,8 +200,15 @@ class PackedBlobWriter: def blob_id(self) -> int: ... @property def path(self) -> str: ... + @property + def field(self) -> pa.Field: ... def write_blob(self, data: bytes) -> None: ... + def write_blobs( + self, + payloads: Union[pa.BinaryArray, pa.LargeBinaryArray, pa.ChunkedArray], + ) -> None: ... def finish(self) -> List[BlobDescriptor]: ... + def finish_array(self, field_name: str) -> pa.StructArray: ... class DedicatedBlobWriter: @property @@ -252,7 +318,25 @@ class LanceColumnStatistics: size_bytes: int class _Session: + def __init__( + self, + index_cache_size_bytes: Optional[int] = None, + metadata_cache_size_bytes: Optional[int] = None, + index_cache_backend: Optional[str | Dict[str, Any]] = None, + metadata_cache_backend: Optional[str | Dict[str, Any]] = None, + ) -> None: + """Create a Lance session. + + Cache backends may be backend URI strings such as + ``"moka://?capacity=1048576"`` or dictionaries such as + ``{"kind": "moka", "options": {"capacity": "1048576"}}``. + ``index_cache_backend`` is mutually exclusive with + ``index_cache_size_bytes``. ``metadata_cache_backend`` is mutually + exclusive with ``metadata_cache_size_bytes``. + """ + ... def size_bytes(self) -> int: ... + def index_cache_size_bytes(self) -> int: ... class LanceBlobFile: def close(self): ... @@ -261,6 +345,8 @@ class LanceBlobFile: def tell(self) -> int: ... def size(self) -> int: ... def readall(self) -> bytes: ... + def read_range(self, offset: int, length: int) -> bytes: ... + def read_ranges(self, ranges: List[Tuple[int, int]]) -> List[bytes]: ... def read_into(self, b: bytearray) -> int: ... class _Dataset: @@ -344,38 +430,46 @@ class _Dataset: self, row_ids: List[int], blob_column: str, - ) -> List[LanceBlobFile]: ... + ) -> List[Optional[LanceBlobFile]]: ... def take_blobs_by_addresses( self, row_addresses: List[int], blob_column: str, - ) -> List[LanceBlobFile]: ... + ) -> List[Optional[LanceBlobFile]]: ... def take_blobs_by_indices( self, row_indices: List[int], blob_column: str, - ) -> List[LanceBlobFile]: ... + ) -> List[Optional[LanceBlobFile]]: ... def read_blobs( self, row_ids: List[int], blob_column: str, io_buffer_size: Optional[int] = None, preserve_order: Optional[bool] = None, - ) -> List[Tuple[int, bytes]]: ... + ) -> List[Tuple[int, Optional[bytes]]]: ... def read_blobs_by_addresses( self, row_addresses: List[int], blob_column: str, io_buffer_size: Optional[int] = None, preserve_order: Optional[bool] = None, - ) -> List[Tuple[int, bytes]]: ... + ) -> List[Tuple[int, Optional[bytes]]]: ... def read_blobs_by_indices( self, row_indices: List[int], blob_column: str, io_buffer_size: Optional[int] = None, preserve_order: Optional[bool] = None, - ) -> List[Tuple[int, bytes]]: ... + ) -> List[Tuple[int, Optional[bytes]]]: ... + def read_blob_ranges( + self, + requests: List[Tuple[int, int, int]], + blob_column: str, + selector: Literal["ids", "addresses", "indices"], + io_buffer_size: Optional[int] = None, + preserve_order: Optional[bool] = None, + ) -> List[Tuple[int, int, Optional[bytes]]]: ... def take_scan( self, row_slices: Iterable[Tuple[int, int]], @@ -470,7 +564,13 @@ class _Dataset: kwargs: Optional[Dict[str, Any]] = None, ): ... def drop_index(self, name: str): ... - def prewarm_index(self, name: str, *, with_position: bool = False): ... + def prewarm_index( + self, + name: str, + *, + with_position: bool = False, + index_segments: Optional[List[str]] = None, + ): ... def merge_index_metadata( self, index_uuid: str, @@ -542,8 +642,13 @@ class _Dataset: index_name: str, partition_id: int, hamming_threshold: int, + index_segments: Optional[List[str]] = None, ) -> pa.RecordBatchReader: ... - def get_ivf_partition_info(self, index_name: str) -> List[dict]: ... + def get_ivf_partition_info( + self, + index_name: str, + index_segments: Optional[List[str]] = None, + ) -> List[dict]: ... def hamming_clustering_for_sample( self, column: str, @@ -568,6 +673,13 @@ class _MergeInsertBuilder: def target_bases(self, bases: list[str]) -> Self: ... def target_all_bases(self, include_primary: bool = True) -> Self: ... def execute(self, new_data: pa.RecordBatchReader) -> ExecuteResult: ... + def execute_batches(self, new_data: pa.RecordBatchReader) -> ExecuteResult: ... + def execute_uncommitted( + self, new_data: pa.RecordBatchReader + ) -> tuple[Transaction, ExecuteResult]: ... + def execute_uncommitted_batches( + self, new_data: pa.RecordBatchReader + ) -> tuple[Transaction, ExecuteResult]: ... class _Scanner: @property @@ -633,6 +745,7 @@ class _Fragment: def physical_rows(self) -> int: ... @property def num_deletions(self) -> int: ... + def validate(self) -> None: ... def iops_counter() -> int: ... def bytes_read_counter() -> int: ... @@ -659,6 +772,7 @@ def _write_fragments( base_store_params: Optional[Dict[str, Dict[str, str]]] = None, external_blob_mode: Literal["reference", "ingest"] = "reference", allow_external_blob_outside_bases: bool = False, + session: Optional[_Session] = None, ): ... def _write_fragments_transaction( dataset_uri: str | Path | _Dataset, @@ -679,6 +793,7 @@ def _write_fragments_transaction( base_store_params: Optional[Dict[str, Dict[str, str]]] = None, external_blob_mode: Literal["reference", "ingest"] = "reference", allow_external_blob_outside_bases: bool = False, + session: Optional[_Session] = None, ) -> Transaction: ... def _json_to_schema(schema_json: str) -> pa.Schema: ... def _schema_to_json(schema: pa.Schema) -> str: ... @@ -690,7 +805,7 @@ def _evaluate_sharding_spec( schema: LanceSchema, ) -> pa.RecordBatch: ... -class _MergedGeneration: +class _CompactedSsTable: shard_id: str generation: int def __init__(self, shard_id: str, generation: int) -> None: ... @@ -700,11 +815,12 @@ class _ShardSnapshot: def __init__(self, shard_id: str) -> None: ... def with_spec_id(self, spec_id: int) -> Self: ... def with_current_generation(self, generation: int) -> Self: ... - def with_flushed_generation(self, generation: int, path: str) -> Self: ... + def with_sstable(self, generation: int, path: str) -> Self: ... class _ShardWriter: shard_id: str def put(self, data: Any) -> None: ... + def delete(self, keys: Any) -> None: ... def close(self) -> None: ... def stats(self) -> Dict[str, Any]: ... def memtable_stats(self) -> Dict[str, Any]: ... @@ -838,6 +954,31 @@ class ScanStatistics: indices_loaded: int parts_loaded: int index_comparisons: int + index_cache_hits: int + """Number of index cache page lookups where the loader was not executed + in this scan. Counts both true cache hits on already-populated entries + and coalesced concurrent loads (a follower attached to another caller's + in-flight load). + + Instrumented boundaries in this release: BTree, IVF v2 (write-cache scan + path), inverted posting list (grouped and per-token) and its per-token + metadata, inverted phrase positions, bitmap (Equals / Range / IsIn), + ngram, rtree. + + Caveats: + + * IVF v2 streaming scans and legacy v1 IVF partitions bypass the cache + by design and are therefore reported as a miss on every call. + * A cold posting-list lookup on the grouped inverted layout can record + up to two misses (group + per-token metadata) for a single term. + + Uninstrumented paths (HNSW graph pages, quantizer codebooks) do not + contribute to either counter.""" + index_cache_misses: int + """Number of index cache page lookups where the loader ran (the page was + not resident and had to be materialised, typically from storage). See + the sibling ``index_cache_hits`` for the paired counter and the list of + instrumented boundaries.""" all_counts: Dict[ str, int ] # Additional metrics for debugging purposes. Subject to change. diff --git a/python/python/lance/lance/fragment.pyi b/python/python/lance/lance/fragment.pyi index 6e80f847620..75bdee1de6b 100644 --- a/python/python/lance/lance/fragment.pyi +++ b/python/python/lance/lance/fragment.pyi @@ -1,7 +1,9 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright The Lance Authors -from typing import Literal, Optional +from typing import Iterable, Iterator, Literal, Optional, Union + +import pyarrow as pa class DeletionFile: """ @@ -120,3 +122,86 @@ class RowIdMeta: ... def __reduce__(self) -> tuple: ... + +class RowIdSequence: + """ + The stable row ids of the rows in a single fragment, in fragment order. + + Use this to attach pre-existing row ids to a fragment when assembling a + transaction manually, so that rewritten rows keep their identity:: + + sequence = RowIdSequence([7, 12]) + fragment = FragmentMetadata(..., row_id_meta=sequence.to_inline_metadata()) + + The sequence may be shorter than the fragment's ``physical_rows``. The ids + bind to the leading rows and the commit mints ids for the remaining ones, + which is how a fragment holding both rewritten and newly inserted rows is + expressed: write the rewritten rows first and supply only their ids. Passing + more ids than the fragment has rows is rejected. + + Warning + ------- + Only duplicates within this sequence are rejected. Row ids must also be + unique across the dataset, and Lance does not re-check that when + committing, so the caller owns it. Supply only row ids that already exist + and are being relocated by the same transaction, which must also remove + every earlier occurrence of them. Do not mint ids for new rows yourself -- + they come from a counter in the manifest that a concurrent commit can + advance, so only the commit knows which values are free. Do not supply + unused row ids either: a sequence covering all of a fragment's rows leaves + the dataset's row id allocator untouched, so a later append will hand the + same id out again. + + Parameters + ---------- + row_ids : range | pa.Array | pa.ChunkedArray | Iterable[int] + The row ids, in the order the corresponding rows appear in the + fragment. A ``range`` with a step of one is stored compactly without + materializing its values. + """ + + def __init__( + self, row_ids: Union[range, pa.Array, pa.ChunkedArray, Iterable[int]] + ) -> None: ... + @staticmethod + def from_inline_metadata(metadata: RowIdMeta) -> RowIdSequence: + """ + Read back a sequence stored inline in fragment row id metadata. + + Parameters + ---------- + metadata : RowIdMeta + Row id metadata holding an inline sequence. Metadata pointing at an + external file is not supported. + + Returns + ------- + RowIdSequence + """ + ... + + def to_inline_metadata(self) -> RowIdMeta: + """ + Encode the sequence as row id metadata to store inline in the manifest. + + Returns + ------- + RowIdMeta + Suitable for the ``row_id_meta`` argument of + :class:`lance.fragment.FragmentMetadata`. + """ + ... + + def to_pyarrow(self) -> pa.UInt64Array: + """ + Get the row ids as a ``uint64`` array, in sequence order. + + Returns + ------- + pa.UInt64Array + """ + ... + + def __len__(self) -> int: ... + def __iter__(self) -> Iterator[int]: ... + def __reduce__(self) -> tuple: ... diff --git a/python/python/lance/lance/indices/__init__.pyi b/python/python/lance/lance/indices/__init__.pyi index 0f5db7037df..181ec93b2d1 100644 --- a/python/python/lance/lance/indices/__init__.pyi +++ b/python/python/lance/lance/indices/__init__.pyi @@ -17,6 +17,8 @@ from typing import Optional import pyarrow as pa +from .. import _Fragment + class IndexConfig: index_type: str config: str @@ -48,6 +50,7 @@ def train_pq_model( max_iters: int, ivf_model: pa.Array, fragment_ids: Optional[list[int]] = None, + num_bits: int = 8, ) -> pa.Array: ... def transform_vectors( dataset, @@ -58,6 +61,9 @@ def transform_vectors( ivf_centroids: pa.Array, pq_codebook: pa.Array, dst_uri: str, + fragments: list[_Fragment], + partitions_ds_uri: Optional[str] = None, + num_bits: int = 8, ): ... def build_rq_model( dimension: int, diff --git a/python/python/lance/mem_wal.py b/python/python/lance/mem_wal.py index f87e811f830..325d97607e3 100644 --- a/python/python/lance/mem_wal.py +++ b/python/python/lance/mem_wal.py @@ -9,8 +9,8 @@ 1. **WAL** – append-only durable log (raw writes) 2. **Active MemTable** – in-memory write buffer -3. **Flushed MemTable** – Lance files written to object store -4. **Base table** – canonical Lance dataset files (after merge_insert) +3. **SSTable** – Lance files written to object store +4. **Base table** – canonical Lance dataset files (after SSTable compaction) """ from __future__ import annotations @@ -22,12 +22,12 @@ import pyarrow as pa from .lance import ( + _CompactedSsTable, _evaluate_sharding_spec, _ExecutionPlan, _LsmPointLookupPlanner, _LsmScanner, _LsmVectorSearchPlanner, - _MergedGeneration, _ShardSnapshot, _ShardWriter, ) @@ -40,7 +40,7 @@ "ShardingField", "ShardingSpec", "evaluate_sharding_spec", - "MergedGeneration", + "CompactedSsTable", "ShardSnapshot", "ShardWriter", "LsmScanner", @@ -124,19 +124,19 @@ def _sharding_spec_to_dict(spec: Union[ShardingSpec, Mapping[str, object]]) -> d @dataclass -class MergedGeneration: - """Identifies a flushed MemWAL generation that has been merged. +class CompactedSsTable: + """Points to an SSTable compacted into the base table. - Pass a list of these to mark_generations_as_merged - so Lance knows which generations are now in the base table. + Pass a list of these to mark_sstables_as_compacted + so Lance can record compaction progress. Parameters ---------- shard_id : str UUID string for the write shard. generation : int - Generation number (from - :attr:`ShardSnapshot.flushed_generations`). + Generation number of the compacted SSTable (as passed to + :meth:`ShardSnapshot.with_sstable`). """ shard_id: str @@ -170,9 +170,9 @@ def with_current_generation(self, generation: int) -> "ShardSnapshot": self._raw = self._raw.with_current_generation(generation) return self - def with_flushed_generation(self, generation: int, path: str) -> "ShardSnapshot": - """Add a flushed generation with its storage path.""" - self._raw = self._raw.with_flushed_generation(generation, path) + def with_sstable(self, generation: int, path: str) -> "ShardSnapshot": + """Add an SSTable with its storage path.""" + self._raw = self._raw.with_sstable(generation, path) return self def __repr__(self) -> str: @@ -187,6 +187,7 @@ class ShardWriter: with dataset.mem_wal_writer(shard_id) as writer: writer.put(batch) + writer.delete(pa.table({"id": [1]})) Parameters ---------- @@ -224,6 +225,36 @@ def put(self, data, *, schema: Optional[pa.Schema] = None) -> None: reader = _coerce_reader(data, schema) self._raw.put(reader) + def delete(self, keys, *, schema: Optional[pa.Schema] = None) -> None: + """Delete rows by primary key from the MemWAL. + + Parameters + ---------- + keys : ReaderLike + Any Arrow-compatible data containing this shard's primary key + column(s). Non-primary-key columns, if present, are ignored by + the Rust core delete path. + schema : pa.Schema, optional + Schema hint, needed when *keys* is a generator. + + Raises + ------ + IOError + If delete validation fails, WAL flush fails, or the writer has + already been closed. Delete validation is centralized in Rust and + includes checks for primary-key metadata and tombstone-compatible + nullable non-key columns. + + Examples + -------- + :: + + with dataset.mem_wal_writer(shard_id) as writer: + writer.delete(pa.table({"id": [42]})) + """ + reader = _coerce_reader(keys, schema) + self._raw.delete(reader) + def close(self) -> None: """Flush and close the writer. @@ -261,7 +292,7 @@ def lsm_scanner( ) -> "LsmScanner": """Create an LSM scanner that includes the active MemTable. - This scanner covers the base table, the given flushed generations, + This scanner covers the base table, the given SSTables, and the current active MemTable — providing strong read-your-writes consistency. @@ -290,10 +321,10 @@ class LsmScanner: """LSM-aware scanner covering all data levels. Deduplicates by primary key, always returning the newest version of - each row across base table, flushed MemTables, and the active MemTable. + each row across base table, SSTables, and the active MemTable. Obtain an instance from `ShardWriter.lsm_scanner` (includes - active MemTable) or `LsmScanner.from_snapshots` (flushed only). + active MemTable) or `LsmScanner.from_snapshots` (SSTables only). The builder methods (`project`, `filter`, `limit`) return ``self`` for chaining. @@ -323,7 +354,7 @@ def from_snapshots( dataset : LanceDataset The base dataset to scan. shard_snapshots : list of ShardSnapshot - Shard snapshots specifying flushed generations to include. + Shard snapshots specifying SSTables to include. """ raw = _LsmScanner.from_snapshots(dataset._ds, [s._raw for s in shard_snapshots]) return LsmScanner(raw) @@ -425,7 +456,7 @@ class LsmPointLookupPlanner: dataset : LanceDataset The base dataset. shard_snapshots : list of ShardSnapshot - Shard snapshots specifying flushed generations to include. + Shard snapshots specifying SSTables to include. pk_columns : list of str, optional Primary key column names. Inferred from schema metadata if omitted. @@ -484,7 +515,7 @@ class LsmVectorSearchPlanner: dataset : LanceDataset The base dataset. shard_snapshots : list of ShardSnapshot - Shard snapshots specifying flushed generations to include. + Shard snapshots specifying SSTables to include. vector_column : str Name of the ``FixedSizeList`` vector column. pk_columns : list of str, optional @@ -596,8 +627,8 @@ def _unwrap_shard_id(shard_id: str) -> str: return shard_id -def _to_raw_merged_generations( - generations: Iterable[MergedGeneration], +def _to_raw_compacted_sstables( + sstables: Iterable[CompactedSsTable], ) -> list: - """Convert Python MergedGeneration list to PyO3 _MergedGeneration list.""" - return [_MergedGeneration(g.shard_id, g.generation) for g in generations] + """Convert Python CompactedSsTable list to PyO3 _CompactedSsTable list.""" + return [_CompactedSsTable(s.shard_id, s.generation) for s in sstables] diff --git a/python/python/lance/namespace.py b/python/python/lance/namespace.py index fec3a1cfb1e..6df5b9aa4bc 100644 --- a/python/python/lance/namespace.py +++ b/python/python/lance/namespace.py @@ -342,12 +342,13 @@ class DirectoryNamespace(LanceNamespace): >>> >>> # With AWS credential vending (requires credential-vendor-aws feature) >>> # Use **dict to pass property names with dots - >>> ns = lance.namespace.DirectoryNamespace(**{ + >>> aws_properties = { ... "root": "s3://my-bucket/data", ... "credential_vendor.enabled": "true", ... "credential_vendor.aws_role_arn": "arn:aws:iam::123456789012:role/MyRole", ... "credential_vendor.aws_duration_millis": "3600000", - ... }) + ... } + >>> # ns = lance.namespace.DirectoryNamespace(**aws_properties) With dynamic context provider: diff --git a/python/python/lance/otel.py b/python/python/lance/otel.py new file mode 100644 index 00000000000..0c9ee1790fd --- /dev/null +++ b/python/python/lance/otel.py @@ -0,0 +1,166 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright The Lance Authors + +"""Bridge Lance's internal metrics into OpenTelemetry. + +Lance core publishes metrics (currently object store request counts, bytes, +latency, errors, and throttles) through the Rust ``metrics`` facade. This module +installs a process-global recorder that aggregates them and registers +OpenTelemetry observable instruments that report the aggregated values into the +user's ``MeterProvider``. + +The bridge is generic: every metric Lance describes is surfaced automatically, +with no per-metric Python code. Histograms have no asynchronous OpenTelemetry +instrument, so each is exported Prometheus-style as cumulative ``le`` buckets +plus ``_count`` and ``_sum`` observable counters. +""" + +from __future__ import annotations + +import warnings +from typing import TYPE_CHECKING, Optional + +from .lance import ( + lance_metrics_catalog, + register_lance_metrics_recorder, + snapshot_lance_metrics, +) + +if TYPE_CHECKING: + from opentelemetry.metrics import MeterProvider + +_INSTRUMENTED = False + + +def instrument_lance_metrics(meter_provider: Optional["MeterProvider"] = None) -> bool: + """Register Lance metrics as OpenTelemetry observable instruments. + + Installs a process-global metrics recorder and creates one observable + instrument per Lance metric on the given (or global) ``MeterProvider``. The + user's configured ``MetricReader`` then collects them on its own schedule. + + Counters and gauges map directly to observable counters/gauges. Each + histogram is exported as cumulative ``le`` bucket counts (``_bucket``, + with an ``le`` attribute) plus ``_count`` and ``_sum``. + + Parameters + ---------- + meter_provider : opentelemetry.metrics.MeterProvider, optional + The provider to register instruments on. Defaults to the global provider + from ``opentelemetry.metrics.get_meter_provider()``. + + Returns + ------- + bool + ``True`` if the recorder is installed and instruments are registered. + ``False`` if a different ``metrics`` recorder is already installed in + this process (``metrics`` permits only one global recorder), in which + case a warning is emitted and no instruments are created. + + Notes + ----- + Requires the OpenTelemetry SDK (``pip install pylance[otel]``). Calling this + more than once is safe; instruments are created only on the first successful + call. + """ + global _INSTRUMENTED + + try: + from opentelemetry.metrics import Observation, get_meter_provider + except ImportError as exc: + raise ImportError( + "instrument_lance_metrics requires the OpenTelemetry API/SDK. " + "Install it with `pip install pylance[otel]` or " + "`pip install opentelemetry-sdk`." + ) from exc + + if not register_lance_metrics_recorder(): + warnings.warn( + "Could not install the Lance metrics recorder: another `metrics` " + "recorder is already installed in this process. Lance metrics will " + "not be exported via OpenTelemetry.", + stacklevel=2, + ) + return False + + if _INSTRUMENTED: + return True + + provider = meter_provider or get_meter_provider() + meter = provider.get_meter("lance") + + def scalar_callback(metric_name: str): + def callback(_options): + return [ + Observation(point.value, point.attributes) + for point in snapshot_lance_metrics() + if point.name == metric_name and point.value is not None + ] + + return callback + + def bucket_callback(metric_name: str): + def callback(_options): + observations = [] + for point in snapshot_lance_metrics(): + if point.name != metric_name or point.buckets is None: + continue + for le, cumulative in point.buckets: + attributes = dict(point.attributes) + attributes["le"] = le + observations.append(Observation(cumulative, attributes)) + return observations + + return callback + + def field_callback(metric_name: str, field: str): + def callback(_options): + observations = [] + for point in snapshot_lance_metrics(): + if point.name != metric_name: + continue + value = getattr(point, field) + if value is not None: + observations.append(Observation(value, point.attributes)) + return observations + + return callback + + for desc in lance_metrics_catalog(): + unit = desc.unit or "" + if desc.kind == "counter": + meter.create_observable_counter( + desc.name, + callbacks=[scalar_callback(desc.name)], + unit=unit, + description=desc.description, + ) + elif desc.kind == "gauge": + meter.create_observable_gauge( + desc.name, + callbacks=[scalar_callback(desc.name)], + unit=unit, + description=desc.description, + ) + elif desc.kind == "histogram": + # `_bucket` and `_count` observe cumulative counts, so they are + # unitless; only `_sum` carries the histogram's unit (e.g. seconds). + meter.create_observable_counter( + f"{desc.name}_bucket", + callbacks=[bucket_callback(desc.name)], + description=f"{desc.description} (cumulative buckets)", + ) + meter.create_observable_counter( + f"{desc.name}_count", + callbacks=[field_callback(desc.name, "count")], + description=f"{desc.description} (count)", + ) + meter.create_observable_counter( + f"{desc.name}_sum", + callbacks=[field_callback(desc.name, "sum")], + unit=unit, + description=f"{desc.description} (sum)", + ) + + _INSTRUMENTED = True + return True diff --git a/python/python/lance/pydantic.py b/python/python/lance/pydantic.py new file mode 100644 index 00000000000..99226fcf664 --- /dev/null +++ b/python/python/lance/pydantic.py @@ -0,0 +1,434 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright The Lance Authors +# +# Pydantic-to-Arrow conversion, ported from +# https://github.com/lancedb/lancedb/blob/f8dc2f78ee3219084d457a647e230b23e2f391b0/python/python/lancedb/pydantic.py +# +# Embedding-function-specific code (LanceModel, parse_embedding_functions, +# EmbeddingFunctionRegistry) is intentionally excluded -- lance has no concept +# of embedding functions, that stays in lancedb. Version detection uses +# hasattr checks against the field/model objects instead of branching on +# pydantic's version number, since pydantic is an optional dependency here. + +"""Pydantic (v1 / v2) to Arrow schema conversion""" + +from __future__ import annotations + +import inspect +import sys +import types +from abc import ABC, abstractmethod +from datetime import date, datetime +from enum import Enum +from typing import Any, Callable, Dict, Generator, List, Type, Union, _GenericAlias + +import pyarrow as pa +import pydantic + +from .dependencies import numpy as np + +_PYDANTIC_V2 = hasattr(pydantic, "GetCoreSchemaHandler") + +try: + from pydantic_core import CoreSchema, core_schema +except ImportError: + if _PYDANTIC_V2: + raise + + +class FixedSizeListMixin(ABC): + @staticmethod + @abstractmethod + def dim() -> int: + raise NotImplementedError + + @staticmethod + @abstractmethod + def value_arrow_type() -> pa.DataType: + raise NotImplementedError + + +def Vector( + dim: int, value_type: pa.DataType = pa.float32(), nullable: bool = True +) -> Type[FixedSizeListMixin]: + """Pydantic type for a fixed-size-list embedding vector column. + + Examples + -------- + >>> import pydantic + >>> from lance.pydantic import Vector + ... + >>> class MyModel(pydantic.BaseModel): + ... id: int + ... embedding: Vector(768) + """ + + class FixedSizeList(list, FixedSizeListMixin): + def __repr__(self): + return f"FixedSizeList(dim={dim})" + + @staticmethod + def nullable() -> bool: + return nullable + + @staticmethod + def dim() -> int: + return dim + + @staticmethod + def value_arrow_type() -> pa.DataType: + return value_type + + @classmethod + def __get_pydantic_core_schema__( + cls, _source_type: Any, _handler: pydantic.GetCoreSchemaHandler + ) -> CoreSchema: + return core_schema.no_info_after_validator_function( + cls, + core_schema.list_schema( + min_length=dim, + max_length=dim, + items_schema=core_schema.float_schema(), + ), + ) + + @classmethod + def __get_validators__(cls) -> Generator[Callable, None, None]: + yield cls.validate + + # For pydantic v1 + @classmethod + def validate(cls, v): + if not isinstance(v, (list, range, np.ndarray)) or len(v) != dim: + raise TypeError("A list of numbers or numpy.ndarray is needed") + return cls(v) + + if not _PYDANTIC_V2: + + @classmethod + def __modify_schema__(cls, field_schema: Dict[str, Any]): + field_schema["items"] = {"type": "number"} + field_schema["maxItems"] = dim + field_schema["minItems"] = dim + + return FixedSizeList + + +def MultiVector( + dim: int, value_type: pa.DataType = pa.float32(), nullable: bool = True +) -> Type[FixedSizeListMixin]: + """Pydantic type for a list of fixed-size-list embedding vectors. + + Useful for models that produce multiple embeddings per input (e.g. + ColPali-style multi-vector embeddings). + + Examples + -------- + >>> import pydantic + >>> from lance.pydantic import MultiVector + ... + >>> class MyModel(pydantic.BaseModel): + ... id: int + ... embeddings: MultiVector(128) + """ + + class MultiVectorList(list, FixedSizeListMixin): + def __repr__(self): + return f"MultiVector(dim={dim})" + + @staticmethod + def nullable() -> bool: + return nullable + + @staticmethod + def dim() -> int: + return dim + + @staticmethod + def value_arrow_type() -> pa.DataType: + return value_type + + @staticmethod + def is_multi_vector() -> bool: + return True + + @classmethod + def __get_pydantic_core_schema__( + cls, _source_type: Any, _handler: pydantic.GetCoreSchemaHandler + ) -> CoreSchema: + return core_schema.no_info_after_validator_function( + cls, + core_schema.list_schema( + items_schema=core_schema.list_schema( + min_length=dim, + max_length=dim, + items_schema=core_schema.float_schema(), + ), + ), + ) + + @classmethod + def __get_validators__(cls) -> Generator[Callable, None, None]: + yield cls.validate + + # For pydantic v1 + @classmethod + def validate(cls, v): + if not isinstance(v, (list, range)): + raise TypeError("A list of vectors is needed") + for vec in v: + if not isinstance(vec, (list, range, np.ndarray)) or len(vec) != dim: + raise TypeError(f"Each vector must be a list of {dim} numbers") + return cls(v) + + if not _PYDANTIC_V2: + + @classmethod + def __modify_schema__(cls, field_schema: Dict[str, Any]): + field_schema["items"] = { + "type": "array", + "items": {"type": "number"}, + "minItems": dim, + "maxItems": dim, + } + + return MultiVectorList + + +def _field_annotation(field: Any) -> Any: + """Get the type annotation off a pydantic v1 ModelField or v2 FieldInfo.""" + if hasattr(field, "annotation"): + # Pydantic v2 FieldInfo + return field.annotation + # Pydantic v1 ModelField -- Optional-ness is tracked via `allow_none`, + # not folded into `outer_type_`. + return field.outer_type_ + + +def get_extras(field: Any, key: str) -> Any: + """Get extra metadata (from `json_schema_extra`) off a pydantic field.""" + if hasattr(field, "json_schema_extra"): + # Pydantic v2 FieldInfo + return (field.json_schema_extra or {}).get(key) + # Pydantic v1 ModelField + return (field.field_info.extra or {}).get("json_schema_extra", {}).get(key) + + +def _py_type_to_arrow_type(py_type: Type[Any], field: Any) -> pa.DataType: + """Convert a field with native Python type to Arrow data type. + + Raises + ------ + TypeError + If the type is not supported. + """ + if py_type is int: + return pa.int64() + elif py_type is float: + return pa.float64() + elif py_type is str: + return pa.utf8() + elif py_type is bool: + return pa.bool_() + elif py_type is bytes: + return pa.binary() + elif py_type is date: + return pa.date32() + elif py_type is datetime: + tz = get_extras(field, "tz") + return pa.timestamp("us", tz=tz) + elif getattr(py_type, "__origin__", None) in (list, tuple): + # A bare, unparameterised ``typing.List`` / ``typing.Tuple`` matches + # this branch (its ``__origin__`` is ``list`` / ``tuple``) but has no + # ``__args__``, so we cannot infer the element type. Raise a clear + # ``TypeError`` instead of crashing with an opaque ``AttributeError``. + args = getattr(py_type, "__args__", None) + if not args: + raise TypeError( + "Converting Pydantic type to Arrow Type: unsupported type " + f"{py_type}. Specify the element type, e.g. List[int] instead " + "of a bare List." + ) + child = args[0] + return _pydantic_list_child_to_arrow(child, field) + raise TypeError( + f"Converting Pydantic type to Arrow Type: unsupported type {py_type}." + ) + + +def _pydantic_model_to_fields(model: Type[pydantic.BaseModel]) -> List[pa.Field]: + if hasattr(model, "model_fields"): + # Pydantic v2 + return [ + _pydantic_to_field(name, field) + for name, field in model.model_fields.items() + ] + # Pydantic v1 + return [_pydantic_to_field(name, field) for name, field in model.__fields__.items()] + + +def _pydantic_type_to_arrow_type(tp: Any, field: Any) -> pa.DataType: + def _safe_issubclass(candidate: Any, base: type) -> bool: + try: + return issubclass(candidate, base) + except TypeError: + return False + + if inspect.isclass(tp): + if _safe_issubclass(tp, pydantic.BaseModel): + # Struct + fields = _pydantic_model_to_fields(tp) + return pa.struct(fields) + if _safe_issubclass(tp, FixedSizeListMixin): + if getattr(tp, "is_multi_vector", lambda: False)(): + return pa.list_(pa.list_(tp.value_arrow_type(), tp.dim())) + # For regular Vector + return pa.list_(tp.value_arrow_type(), tp.dim()) + if _safe_issubclass(tp, Enum): + # Map Enum to the Arrow type of its value. + # For string-valued enums, use dictionary encoding for efficiency. + # For integer enums, use the native type. + # Fall back to utf8 for mixed-type or empty enums. + value_types = {type(m.value) for m in tp} + if len(value_types) == 1: + value_type = value_types.pop() + if value_type is str: + # Use dictionary encoding for string enums + return pa.dictionary(pa.int32(), pa.utf8()) + return _py_type_to_arrow_type(value_type, field) + return pa.utf8() + return _py_type_to_arrow_type(tp, field) + + +def _pydantic_list_child_to_arrow(child: Any, field: Any) -> pa.DataType: + unwrapped = _unwrap_optional_annotation(child) + if unwrapped is not None: + return pa.list_( + pa.field("item", _pydantic_type_to_arrow_type(unwrapped, field), True) + ) + return pa.list_(_pydantic_type_to_arrow_type(child, field)) + + +def _unwrap_optional_annotation(annotation: Any) -> Any | None: + if isinstance(annotation, (_GenericAlias, types.GenericAlias)): + origin = annotation.__origin__ + args = annotation.__args__ + if origin == Union: + non_none = [arg for arg in args if arg is not type(None)] + if len(non_none) == 1 and len(non_none) != len(args): + return non_none[0] + elif sys.version_info >= (3, 10) and isinstance(annotation, types.UnionType): + args = annotation.__args__ + non_none = [arg for arg in args if arg is not type(None)] + if len(non_none) == 1 and len(non_none) != len(args): + return non_none[0] + return None + + +def _pydantic_to_arrow_type(field: Any) -> pa.DataType: + """Convert a pydantic field (v1 ModelField or v2 FieldInfo) to Arrow DataType""" + annotation = _field_annotation(field) + unwrapped = _unwrap_optional_annotation(annotation) + if unwrapped is not None: + return _pydantic_type_to_arrow_type(unwrapped, field) + if isinstance(annotation, (_GenericAlias, types.GenericAlias)): + origin = annotation.__origin__ + args = annotation.__args__ + + if origin is list: + child = args[0] + return _pydantic_list_child_to_arrow(child, field) + return _pydantic_type_to_arrow_type(annotation, field) + + +def is_nullable(field: Any) -> bool: + """Check if a pydantic field (v1 ModelField or v2 FieldInfo) is nullable. + + Only a true ``Optional``/``Union[..., None]`` annotation (or a nullable + ``Vector``/``MultiVector``) makes a field nullable -- a field with a + plain default value but a non-Optional type is not. + """ + if not hasattr(field, "annotation"): + # Pydantic v1 ModelField: Optional-ness is tracked via `allow_none` + # directly, since `outer_type_` already has Optional stripped. A + # nullable Vector/MultiVector still overrides this, same as v2. + v1_type = _field_annotation(field) + if inspect.isclass(v1_type): + try: + if issubclass(v1_type, FixedSizeListMixin): + return v1_type.nullable() + except TypeError: + pass + return bool(field.allow_none) + + annotation = field.annotation + if _unwrap_optional_annotation(annotation) is not None: + return True + if isinstance(annotation, (_GenericAlias, types.GenericAlias)): + origin = annotation.__origin__ + args = annotation.__args__ + if origin == Union: + if any(typ is type(None) for typ in args): + return True + elif sys.version_info >= (3, 10) and isinstance(annotation, types.UnionType): + args = annotation.__args__ + for typ in args: + if typ is type(None): + return True + elif inspect.isclass(annotation): + try: + if issubclass(annotation, FixedSizeListMixin): + return annotation.nullable() + except TypeError: + return False + return False + + +def _pydantic_to_field(name: str, field: Any) -> pa.Field: + """Convert a pydantic field (v1 ModelField or v2 FieldInfo) to a PyArrow Field.""" + dt = _pydantic_to_arrow_type(field) + return pa.field(name, dt, is_nullable(field)) + + +def pydantic_to_schema(model: Type[pydantic.BaseModel]) -> pa.Schema: + """Convert a [Pydantic Model][pydantic.BaseModel] to a + [PyArrow Schema][pyarrow.Schema]. + + Supports nested ``BaseModel`` fields (-> struct), ``Enum`` fields + (string-valued enums are dictionary-encoded), timezone-aware + ``datetime`` fields (via ``Field(json_schema_extra={"tz": ...})``), and + the ``Vector``/``MultiVector`` fixed-size-list types, in addition to + plain scalar/``Optional``/``List`` types. + + Parameters + ---------- + model : Type[pydantic.BaseModel] + The Pydantic BaseModel to convert to Arrow Schema. + + Returns + ------- + pyarrow.Schema + The Arrow Schema + + Examples + -------- + + >>> from typing import List, Optional + >>> import pydantic + >>> from lance.pydantic import pydantic_to_schema, Vector + >>> class FooModel(pydantic.BaseModel): + ... id: int + ... s: str + ... vec: Vector(1536) # fixed_size_list[1536] + ... li: List[int] + ... + >>> schema = pydantic_to_schema(FooModel) + >>> assert schema == pa.schema([ + ... pa.field("id", pa.int64(), False), + ... pa.field("s", pa.utf8(), False), + ... pa.field("vec", pa.list_(pa.float32(), 1536)), + ... pa.field("li", pa.list_(pa.int64()), False), + ... ]) + """ + fields = _pydantic_model_to_fields(model) + return pa.schema(fields) diff --git a/python/python/lance/query.py b/python/python/lance/query.py index 85bbf121e6e..a462f9e3f64 100644 --- a/python/python/lance/query.py +++ b/python/python/lance/query.py @@ -22,6 +22,13 @@ class FullTextOperator(Enum): OR = "OR" +class DocumentGranularity(str, Enum): + """The unit treated as one full-text-search document.""" + + ROW = "row" + LIST_ELEMENT = "list_element" + + class Occur(Enum): SHOULD = "SHOULD" MUST = "MUST" @@ -98,6 +105,7 @@ def __init__( max_expansions: int = 50, operator: FullTextOperator = FullTextOperator.OR, prefix_length: int = 0, + document_granularity: Optional[DocumentGranularity] = None, ): """ Match query for full-text search. @@ -129,6 +137,10 @@ def __init__( prefix_length : int, default 0 The number of beginning characters being unchanged for fuzzy matching. This is useful to achieve prefix matching. + document_granularity : DocumentGranularity, optional + Explicitly select row or deepest-list-element documents. If omitted, + the indexed granularity is inferred. When both granularities are indexed + for the field, this must be specified. With no index, ``ROW`` is used. """ self._inner = PyFullTextQuery.match_query( query, @@ -138,6 +150,9 @@ def __init__( max_expansions=max_expansions, operator=operator.value, prefix_length=prefix_length, + document_granularity=( + document_granularity.value if document_granularity is not None else None + ), ) def query_type(self) -> FullTextQueryType: @@ -145,7 +160,14 @@ def query_type(self) -> FullTextQueryType: class PhraseQuery(FullTextQuery): - def __init__(self, query: str, column: str, *, slop: int = 0): + def __init__( + self, + query: str, + column: str, + *, + slop: int = 0, + document_granularity: Optional[DocumentGranularity] = None, + ): """ Phrase query for full-text search. @@ -155,8 +177,21 @@ def __init__(self, query: str, column: str, *, slop: int = 0): The query string to match against. column : str The name of the column to match against. + slop : int, default 0 + The maximum number of intervening positions permitted in the phrase. + document_granularity : DocumentGranularity, optional + Explicitly select row or deepest-list-element documents. If omitted, + the indexed granularity is inferred. When both granularities are indexed + for the field, this must be specified. With no index, ``ROW`` is used. """ - self._inner = PyFullTextQuery.phrase_query(query, column, slop) + self._inner = PyFullTextQuery.phrase_query( + query, + column, + slop, + document_granularity=( + document_granularity.value if document_granularity is not None else None + ), + ) def query_type(self) -> FullTextQueryType: return FullTextQueryType.MATCH_PHRASE @@ -235,7 +270,9 @@ def __init__(self, queries: list[tuple[Occur, FullTextQuery]]): Parameters ---------- queries : list[tuple(Occur, FullTextQuery)] - The list of queries with their occurrence requirements. + The list of queries with their occurrence requirements. Every MUST + clause must match and contributes its score; matching SHOULD scores + are also added, while MUST_NOT clauses only exclude documents. """ self._inner = PyFullTextQuery.boolean_query( [(occur.value, query.inner) for occur, query in queries] diff --git a/python/python/lance/sampler.py b/python/python/lance/sampler.py index b7e7230dfc6..4c17825ab04 100644 --- a/python/python/lance/sampler.py +++ b/python/python/lance/sampler.py @@ -193,7 +193,7 @@ def maybe_sample( This is employed to minimize the number of random reads necessary for sampling. A sufficiently large value can provide an effective random sample without the need for excessive random reads. - filter : str, optional + filt : str, optional The filter to apply to the dataset, by default None. If a filter is provided, then we will first load all row ids in memory and then batch through the ids in random order until enough matches have been found. diff --git a/python/python/lance/schema.py b/python/python/lance/schema.py index aacdcb73e8e..e97e97e3322 100644 --- a/python/python/lance/schema.py +++ b/python/python/lance/schema.py @@ -12,17 +12,19 @@ def schema_to_json(schema: pa.Schema) -> Dict[str, Any]: """ - Converts a pyarrow schema to a JSON string. + Converts a pyarrow schema to a JSON-compatible dict. Parameters ---------- + schema: pa.Schema + The PyArrow schema to convert. """ return json.loads(_schema_to_json(schema)) def json_to_schema(schema_json: Dict[str, Any]) -> pa.Schema: """ - Converts a JSON string to a PyArrow schema. + Converts a JSON-compatible dict to a PyArrow schema. Parameters ---------- diff --git a/python/python/lance/tf/__init__.py b/python/python/lance/tf/__init__.py deleted file mode 100644 index 1aa41beb99c..00000000000 --- a/python/python/lance/tf/__init__.py +++ /dev/null @@ -1,10 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright The Lance Authors - -import importlib.util - -if importlib.util.find_spec("tensorflow") is None: - raise ImportError( - "Tensorflow is not installed. Please install tensorflow" - + " to use lance.tf module.", - ) diff --git a/python/python/lance/tf/data.py b/python/python/lance/tf/data.py deleted file mode 100644 index 68280f9211f..00000000000 --- a/python/python/lance/tf/data.py +++ /dev/null @@ -1,410 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright The Lance Authors - - -"""Tensorflow Dataset (`tf.data `_) -implementation for Lance. - -.. warning:: - - Experimental feature. API stability is not guaranteed. -""" - -from __future__ import annotations - -from functools import partial -from typing import TYPE_CHECKING, Dict, Iterable, List, Optional, Tuple, Union - -import pyarrow as pa - -import lance -from lance import LanceDataset -from lance.arrow import EncodedImageType, FixedShapeImageTensorType, ImageURIType -from lance.dependencies import _check_for_numpy -from lance.dependencies import numpy as np -from lance.dependencies import tensorflow as tf -from lance.fragment import FragmentMetadata, LanceFragment -from lance.log import LOGGER - -if TYPE_CHECKING: - from pathlib import Path - - from lance import LanceNamespace - - -def arrow_data_type_to_tf(dt: pa.DataType) -> tf.DType: - """Convert Pyarrow DataType to Tensorflow.""" - if pa.types.is_boolean(dt): - return tf.bool - elif pa.types.is_int8(dt): - return tf.int8 - elif pa.types.is_int16(dt): - return tf.int16 - elif pa.types.is_int32(dt): - return tf.int32 - elif pa.types.is_int64(dt): - return tf.int64 - elif pa.types.is_uint8(dt): - return tf.uint8 - elif pa.types.is_uint16(dt): - return tf.uint16 - elif pa.types.is_uint32(dt): - return tf.uint32 - elif pa.types.is_uint64(dt): - return tf.uint64 - elif pa.types.is_float16(dt): - return tf.float16 - elif pa.types.is_float32(dt): - return tf.float32 - elif pa.types.is_float64(dt): - return tf.float64 - elif ( - pa.types.is_string(dt) - or pa.types.is_large_string(dt) - or pa.types.is_binary(dt) - or pa.types.is_large_binary(dt) - ): - return tf.string - - raise TypeError(f"Arrow/Tf conversion: Unsupported arrow data type: {dt}") - - -def data_type_to_tensor_spec(dt: pa.DataType) -> tf.TensorSpec: - """Convert PyArrow DataType to Tensorflow TensorSpec.""" - if ( - pa.types.is_boolean(dt) - or pa.types.is_integer(dt) - or pa.types.is_floating(dt) - or pa.types.is_string(dt) - or pa.types.is_binary(dt) - ): - return tf.TensorSpec(shape=(None,), dtype=arrow_data_type_to_tf(dt)) - elif isinstance(dt, pa.FixedShapeTensorType): - return tf.TensorSpec( - shape=(None, *dt.shape), dtype=arrow_data_type_to_tf(dt.value_type) - ) - elif pa.types.is_fixed_size_list(dt): - return tf.TensorSpec( - shape=(None, dt.list_size), dtype=arrow_data_type_to_tf(dt.value_type) - ) - elif pa.types.is_list(dt) or pa.types.is_large_list(dt): - return tf.TensorSpec( - shape=( - None, - None, - ), - dtype=arrow_data_type_to_tf(dt.value_type), - ) - elif pa.types.is_struct(dt): - return {field.name: data_type_to_tensor_spec(field.type) for field in dt} - elif isinstance(dt, (EncodedImageType, ImageURIType)): - return tf.TensorSpec(shape=(None,), dtype=tf.string) - elif isinstance(dt, FixedShapeImageTensorType): - return tf.TensorSpec( - shape=(None, *dt.shape), dtype=arrow_data_type_to_tf(dt.arrow_type) - ) - - raise TypeError("Unsupported data type: ", dt) - - -def schema_to_spec(schema: pa.Schema) -> tf.TypeSpec: - """Convert PyArrow Schema to Tensorflow output signature.""" - signature = {} - for name in schema.names: - field = schema.field(name) - signature[name] = data_type_to_tensor_spec(field.type) - return signature - - -def column_to_tensor(array: pa.Array, tensor_spec: tf.TensorSpec) -> tf.Tensor: - """Convert a PyArrow array into a TensorFlow tensor.""" - if isinstance(tensor_spec, tf.RaggedTensorSpec): - return tf.ragged.constant(array.to_pylist(), dtype=tensor_spec.dtype) - elif isinstance(array.type, pa.FixedShapeTensorType): - return tf.constant(array.to_numpy_ndarray(), dtype=tensor_spec.dtype) - elif isinstance(array.type, FixedShapeImageTensorType): - return tf.constant(array.to_numpy(), dtype=tensor_spec.dtype) - elif isinstance(array.type, pa.StructType): - return { - field.name: column_to_tensor(array.field(i), tensor_spec[field.name]) - for (i, field) in enumerate(array.type) - } - else: - return tf.constant(array.to_pylist(), dtype=tensor_spec.dtype) - - -def from_lance( - dataset: Optional[Union[str, Path, LanceDataset]] = None, - *, - columns: Optional[Union[List[str], Dict[str, str]]] = None, - batch_size: int = 256, - filter: Optional[str] = None, - fragments: Union[Iterable[int], Iterable[LanceFragment], tf.data.Dataset] = None, - output_signature: Optional[Dict[str, tf.TypeSpec]] = None, - namespace_client: Optional["LanceNamespace"] = None, - table_id: Optional[List[str]] = None, - ignore_namespace_table_storage_options: bool = False, -) -> tf.data.Dataset: - """Create a ``tf.data.Dataset`` from a Lance dataset. - - Parameters - ---------- - dataset : Union[str, Path, LanceDataset], optional - Lance dataset or dataset URI/path. Either ``dataset`` or both - ``namespace_client`` and ``table_id`` must be provided. - columns : Optional[List[str]], optional - List of columns to include in the output dataset. - If not set, all columns will be read. - batch_size : int, optional - Batch size, by default 256 - filter : Optional[str], optional - SQL filter expression, by default None. - fragments : Union[List[LanceFragment], tf.data.Dataset], optional - If provided, only the fragments are read. It can be used to feed - for distributed training. - output_signature : Optional[tf.TypeSpec], optional - Override output signature of the returned tensors. If not provided, - the output signature is inferred from the projection Schema. - namespace_client : Optional[LanceNamespace], optional - Namespace client to resolve the table location when ``table_id`` is - provided. - table_id : Optional[List[str]], optional - Table identifier used together with ``namespace_client`` to locate - the table. - ignore_namespace_table_storage_options : bool, default False - When using ``namespace_client``/``table_id``, ignore storage options - returned by the namespace. - - Examples - -------- - - .. code-block:: python - - import tensorflow as tf - import lance.tf.data - - ds = lance.tf.data.from_lance( - "s3://bucket/path", - columns=["image", "id"], - filter="catalog = 'train' AND split = 'train'", - batch_size=100) - - for batch in ds.repeat(10).shuffle(128).map(io_decode): - print(batch["image"].shape) - - ``from_lance`` can take an iterator or ``tf.data.Dataset`` of - Fragments. So that it can be used to feed for distributed training. - - .. code-block:: python - - import tensorflow as tf - import lance.tf.data - - seed = 200 # seed to shuffle the fragments in distributed machines. - fragments = lance.tf.data.lance_fragments("s3://bucket/path") - repeat(10).shuffle(4, seed=seed) - ds = lance.tf.data.from_lance( - "s3://bucket/path", - columns=["image", "id"], - filter="catalog = 'train' AND split = 'train'", - fragments=fragments, - batch_size=100) - for batch in ds.shuffle(128).map(io_decode): - print(batch["image"].shape) - - """ - if isinstance(dataset, LanceDataset): - if namespace_client is not None or table_id is not None: - raise ValueError( - "Cannot specify 'namespace_client' or 'table_id' when passing " - "a LanceDataset instance" - ) - else: - dataset = lance.dataset( - dataset, - namespace_client=namespace_client, - table_id=table_id, - ignore_namespace_table_storage_options=ignore_namespace_table_storage_options, - ) - - if isinstance(fragments, tf.data.Dataset): - fragments = list(fragments.as_numpy_iterator()) - elif _check_for_numpy(fragments) and isinstance(fragments, np.ndarray): - fragments = list(fragments) - - if fragments is not None: - - def gen_fragments(fragments): - for f in fragments: - if isinstance(f, int) or ( - _check_for_numpy(f) and isinstance(f, np.integer) - ): - yield LanceFragment(dataset, int(f)) - elif isinstance(f, FragmentMetadata): - yield LanceFragment(dataset, f.id) - elif isinstance(f, LanceFragment): - yield f - else: - raise TypeError(f"Invalid type passed to `fragments`: {type(f)}") - - # A Generator of Fragments - fragments = gen_fragments(fragments) - - scanner = dataset.scanner( - filter=filter, columns=columns, batch_size=batch_size, fragments=fragments - ) - - if output_signature is None: - schema = scanner.projected_schema - output_signature = schema_to_spec(schema) - LOGGER.debug("Output signature: %s", output_signature) - - def generator(): - for batch in scanner.to_batches(): - yield { - name: column_to_tensor(batch[name], output_signature[name]) - for name in batch.schema.names - } - - return tf.data.Dataset.from_generator(generator, output_signature=output_signature) - - -def lance_fragments(dataset: Union[str, Path, LanceDataset]) -> tf.data.Dataset: - """Create a ``tf.data.Dataset`` of Lance Fragments in the dataset. - - Parameters - ---------- - dataset : Union[str, Path, LanceDataset] - A Lance Dataset or dataset URI/path. - """ - if not isinstance(dataset, LanceDataset): - dataset = lance.dataset(dataset) - return tf.data.Dataset.from_tensor_slices( - [f.fragment_id for f in dataset.get_fragments()] - ) - - -def _ith_batch(i: int, batch_size: int, total_size: int) -> Tuple[int, int]: - """ - Get the start and end index of the ith batch. - - This takes into account the total_size, the total number of rows in the dataset. - """ - start = i * batch_size - end = tf.math.minimum(start + batch_size, total_size) - return (start, end) - - -def from_lance_batches( - dataset: Union[str, Path, LanceDataset], - *, - shuffle: bool = False, - seed: Optional[int] = None, - batch_size: int = 1024, - skip: int = 0, -) -> tf.data.Dataset: - """ - Create a ``tf.data.Dataset`` of batch indices for a Lance dataset. - - Parameters - ---------- - dataset : Union[str, Path, LanceDataset] - A Lance Dataset or dataset URI/path. - shuffle : bool, optional - Shuffle the batches, by default False - seed : Optional[int], optional - Random seed for shuffling, by default None - batch_size : int, optional - Batch size, by default 1024 - skip : int, optional - Number of batches to skip. - - Returns - ------- - tf.data.Dataset - A tensorflow dataset of batch slice ranges. These can be passed to - :func:`lance_take_batches` to create a Tensorflow dataset of batches. - """ - if not isinstance(dataset, LanceDataset): - dataset = lance.dataset(dataset) - num_rows = dataset.count_rows() - num_batches = (num_rows + batch_size - 1) // batch_size - indices = tf.data.Dataset.range(num_batches, dtype=tf.int64) - if shuffle: - indices = indices.shuffle(num_batches, seed=seed) - if skip > 0: - indices = indices.skip(skip) - return indices.map(partial(_ith_batch, batch_size=batch_size, total_size=num_rows)) - - -def lance_take_batches( - dataset: Union[str, Path, LanceDataset], - batch_ranges: Iterable[Tuple[int, int]], - *, - columns: Optional[List[str]] = None, - output_signature: Optional[Dict[str, tf.TypeSpec]] = None, - batch_readahead: int = 10, -) -> tf.data.Dataset: - """ - Create a ``tf.data.Dataset`` of batches from a Lance dataset. - - Parameters - ---------- - dataset : Union[str, Path, LanceDataset] - A Lance Dataset or dataset URI/path. - batch_ranges : Iterable[Tuple[int, int]] - Iterable of batch indices. - columns : Optional[List[str]], optional - List of columns to include in the output dataset. - If not set, all columns will be read. - output_signature : Optional[tf.TypeSpec], optional - Override output signature of the returned tensors. If not provided, - the output signature is inferred from the projection Schema. - batch_readahead : int, default 10 - The number of batches to read ahead in parallel. - - Examples - -------- - You can compose this with ``from_lance_batches`` to create a randomized Tensorflow - dataset. With ``from_lance_batches``, you can deterministically randomized the - batches by setting ``seed``. - - .. code-block:: python - - batch_iter = from_lance_batches(dataset, batch_size=100, shuffle=True, seed=200) - batch_iter = batch_iter.as_numpy_iterator() - lance_ds = lance_take_batches(dataset, batch_iter) - lance_ds = lance_ds.unbatch().shuffle(500, seed=42).batch(100) - """ - if not isinstance(dataset, LanceDataset): - dataset = lance.dataset(dataset) - - if output_signature is None: - schema = dataset.scanner(columns=columns).projected_schema - output_signature = schema_to_spec(schema) - LOGGER.debug("Output signature: %s", output_signature) - - def gen_ranges(): - for start, end in batch_ranges: - yield (start, end) - - def gen_batches(): - batches = dataset._ds.take_scan( - gen_ranges(), - columns=columns, - batch_readahead=batch_readahead, - ) - for batch in batches: - yield { - name: column_to_tensor(batch[name], output_signature[name]) - for name in batch.schema.names - } - - return tf.data.Dataset.from_generator( - gen_batches, output_signature=output_signature - ) - - -# Register `from_lance` to ``tf.data.Dataset``. -tf.data.Dataset.from_lance = from_lance -tf.data.Dataset.from_lance_batches = from_lance_batches diff --git a/python/python/lance/torch/bench_utils.py b/python/python/lance/torch/bench_utils.py index b0b19fc22a9..41d8f8278f3 100644 --- a/python/python/lance/torch/bench_utils.py +++ b/python/python/lance/torch/bench_utils.py @@ -128,7 +128,7 @@ def recall(expected: np.ndarray, actual: np.ndarray) -> np.ndarray: ---------- expected: ndarray The ground truth - results: ndarray + actual: ndarray The ANN results """ assert expected.shape == actual.shape diff --git a/python/python/lance/torch/kmeans.py b/python/python/lance/torch/kmeans.py index 44fca9ae60a..84874c47f46 100644 --- a/python/python/lance/torch/kmeans.py +++ b/python/python/lance/torch/kmeans.py @@ -177,7 +177,7 @@ def _updated_centroids( for idx in zero_counts.nonzero(as_tuple=False): # split the largest cluster and remove empty cluster max_idx = torch.argmax(counts).item() - # add 1% gassuian noise to the largest centroid + # add 1% gaussian noise to the largest centroid # do this twice so we effectively split the largest cluster into 2 # rand_like returns on [0, 1) so we need to shift it to [-0.5, 0.5) noise = (torch.rand_like(centroids[idx]) - 0.5) * 0.01 + 1 diff --git a/python/python/lance/types.py b/python/python/lance/types.py index 41cc191e4d6..3b5cc5b0683 100644 --- a/python/python/lance/types.py +++ b/python/python/lance/types.py @@ -9,7 +9,13 @@ from pyarrow import RecordBatch from . import dataset -from .dependencies import _check_for_hugging_face, _check_for_pandas +from .dependencies import ( + _check_for_hugging_face, + _check_for_pandas, + _is_pydantic_base_model, + _validate_pydantic_list, + model_to_dict, +) from .dependencies import pandas as pd if TYPE_CHECKING: @@ -52,6 +58,34 @@ def _casting_recordbatch_iter( yield batch +def _is_materialized(data_obj: ReaderLike) -> bool: + """Whether ``data_obj`` is fully materialized in memory. + + Materialized sources (tables, in-memory frames) can be wrapped in an + in-memory table for replay without spilling and to expose exact statistics. + Streaming or re-readable sources (readers, scanners, datasets, generators) + are not considered materialized. + """ + if _check_for_pandas(data_obj) and isinstance(data_obj, pd.DataFrame): + return True + if isinstance(data_obj, (pa.Table, pa.RecordBatch)): + return True + if ( + type(data_obj).__module__.startswith("polars") + and data_obj.__class__.__name__ == "DataFrame" + ): + return True + if isinstance(data_obj, dict): + return True + if ( + isinstance(data_obj, list) + and len(data_obj) > 0 + and isinstance(data_obj[0], dict) + ): + return True + return False + + def _coerce_reader( data_obj: ReaderLike, schema: Optional[pa.Schema] = None ) -> pa.RecordBatchReader: @@ -116,6 +150,20 @@ def batch_iter(): # List of dictionaries batch = pa.RecordBatch.from_pylist(data_obj, schema=schema) return pa.RecordBatchReader.from_batches(batch.schema, [batch]) + elif ( + isinstance(data_obj, list) + and len(data_obj) > 0 + and _is_pydantic_base_model(data_obj[0]) + ): + model_class = type(data_obj[0]) + _validate_pydantic_list(data_obj, model_class) + if schema is None: + from .pydantic import pydantic_to_schema + + schema = pydantic_to_schema(model_class) + dicts = [model_to_dict(item) for item in data_obj] + batch = pa.RecordBatch.from_pylist(dicts, schema=schema) + return pa.RecordBatchReader.from_batches(batch.schema, [batch]) # for other iterables, assume they are of type Iterable[RecordBatch] elif isinstance(data_obj, Iterable): if schema is not None: diff --git a/python/python/lance/util.py b/python/python/lance/util.py index 2161c4e0d45..180e2441b43 100644 --- a/python/python/lance/util.py +++ b/python/python/lance/util.py @@ -3,8 +3,18 @@ from __future__ import annotations +import uuid from datetime import datetime, timedelta -from typing import TYPE_CHECKING, Iterator, Literal, Optional, Union, cast +from typing import ( + TYPE_CHECKING, + Iterable, + Iterator, + List, + Literal, + Optional, + Union, + cast, +) import pyarrow as pa @@ -51,6 +61,29 @@ def td_to_micros(td: timedelta) -> int: return round(td / timedelta(microseconds=1)) +def _normalize_index_segment_ids( + index_segments: Optional[Iterable[Union[str, uuid.UUID]]], +) -> Optional[List[str]]: + """Normalize a physical index segment selection to a list of UUID strings.""" + if index_segments is None: + return None + if isinstance(index_segments, (str, uuid.UUID)): + raise TypeError( + "index_segments must be an iterable of str or uuid.UUID, " + f"not a single {type(index_segments)}." + ) + segment_ids = [] + for segment_id in index_segments: + if isinstance(segment_id, (str, uuid.UUID)): + segment_ids.append(str(segment_id)) + else: + raise TypeError( + "index_segments must be an iterable of str or uuid.UUID. " + f"Got {type(segment_id)} instead." + ) + return segment_ids + + class KMeans: """KMean model for clustering. diff --git a/python/python/lance/vector.py b/python/python/lance/vector.py index 5ce5e8b61e5..37e331313dc 100644 --- a/python/python/lance/vector.py +++ b/python/python/lance/vector.py @@ -19,9 +19,10 @@ ) from .dependencies import numpy as np from .log import LOGGER -from .util import MetricType, _normalize_metric_type +from .util import MetricType, _normalize_index_segment_ids, _normalize_metric_type if TYPE_CHECKING: + import uuid from pathlib import Path from . import LanceDataset @@ -288,12 +289,12 @@ def compute_pq_codes( Dataset to compute pq codes for. kmeans_list: List[lance.torch.kmeans.KMeans] KMeans models to use to compute pq (one per subspace) - batch_size: int, default 10240 + batch_size: int, default 40960 The batch size used to read the dataset. dst_dataset_uri: Union[str, Path], optional The path to store the partitions. If not specified a random directory is used instead - allow_tf32: bool, default True + allow_cuda_tf32: bool, default True Whether to allow tf32 for matmul on CUDA. Returns @@ -417,12 +418,12 @@ def compute_partitions( Column name of the vector column. kmeans: lance.torch.kmeans.KMeans KMeans model to use to compute partitions. - batch_size: int, default 10240 + batch_size: int, default 40960 The batch size used to read the dataset. dst_dataset_uri: Union[str, Path], optional The path to store the partitions. If not specified a random directory is used instead - allow_tf32: bool, default True + allow_cuda_tf32: bool, default True Whether to allow tf32 for matmul on CUDA. Returns @@ -761,13 +762,17 @@ def hamming_clustering_for_ivf_partition( index_name: str, partition_id: int, hamming_threshold: int, + *, + index_segments: Optional[Iterable[Union[str, uuid.UUID]]] = None, ) -> pa.RecordBatchReader: """ Perform hamming clustering on a partition of an IVF_FLAT index. - Loads a partition from an IVF_FLAT index on a hash column, computes - pairwise hamming distances between all hashes in the partition, - filters by threshold, and clusters the results using union-find. + Loads a partition from every segment of an IVF_FLAT index on a hash + column, computes pairwise hamming distances between all hashes in the + combined partition, filters by threshold, and clusters the results using + union-find. All segments of the logical index must share the same global + IVF centroids; an error is raised if they do not. Parameters ---------- @@ -779,6 +784,11 @@ def hamming_clustering_for_ivf_partition( The partition ID within the IVF_FLAT index hamming_threshold : int Maximum hamming distance to consider as similar + index_segments : iterable of str or uuid.UUID, optional + If specified, only these physical index segment UUIDs of the named + logical index contribute rows. Use + :meth:`LanceDataset.describe_indices` to obtain segment UUIDs from + ``IndexDescription.segments``. Defaults to all segments. Returns ------- @@ -789,30 +799,43 @@ def hamming_clustering_for_ivf_partition( - 'duplicates': list - List of duplicate row IDs in each cluster """ return dataset._ds.hamming_clustering_for_ivf_partition( - index_name, partition_id, hamming_threshold + index_name, + partition_id, + hamming_threshold, + _normalize_index_segment_ids(index_segments), ) def get_ivf_partition_info( dataset: "LanceDataset", index_name: str, + *, + index_segments: Optional[Iterable[Union[str, uuid.UUID]]] = None, ) -> List[dict]: """ Get partition information for an IVF_FLAT index. + Partition sizes are aggregated across all segments of the logical index + unless a subset is selected via ``index_segments``. + Parameters ---------- dataset : LanceDataset The Lance dataset containing the hash column with an IVF_FLAT index. index_name : str Name of the IVF_FLAT index + index_segments : iterable of str or uuid.UUID, optional + If specified, only these physical index segment UUIDs of the named + logical index contribute to the sizes. Defaults to all segments. Returns ------- list[dict] List of partition info dicts with 'partition_id' and 'size' """ - return dataset._ds.get_ivf_partition_info(index_name) + return dataset._ds.get_ivf_partition_info( + index_name, _normalize_index_segment_ids(index_segments) + ) def hamming_clustering_for_sample( @@ -833,7 +856,8 @@ def hamming_clustering_for_sample( dataset : LanceDataset The Lance dataset containing the hash column. column : str - Name of the hash column (must be FixedSizeList) + Name of the hash column (must be FixedSizeList where N is a + positive multiple of 8 bytes) sample_size : int, optional Number of rows to sample. If None, uses all rows. hamming_threshold : int, default 10 @@ -875,7 +899,8 @@ def hamming_clustering_for_range( dataset : LanceDataset The Lance dataset containing the hash column. column : str - Name of the hash column (must be FixedSizeList) + Name of the hash column (must be FixedSizeList where N is a + positive multiple of 8 bytes) fragment_id : int The fragment ID to read from start_row : int diff --git a/python/python/tests/compat/compat_sequence.py b/python/python/tests/compat/compat_sequence.py index 48df46500ca..40330c6a1e0 100644 --- a/python/python/tests/compat/compat_sequence.py +++ b/python/python/tests/compat/compat_sequence.py @@ -11,8 +11,8 @@ without hand-coding the triggering sequence. The scenario is parameterized by index *kind* so every scalar index type gets the same -aged-lifecycle, cross-version treatment. The oracle runs the same predicate twice -- -normally and with use_scalar_index=False (lance ignores the index) -- and requires +aged-lifecycle, cross-version treatment. The scalar oracle runs the same predicate twice +-- normally and with use_scalar_index=False (lance ignores the index) -- and requires the results to match. If the two query plans are identical the index wasn't used, so the comparison is skipped rather than failed (uninformative, not a regression). FTS has no "ignore the index" mode to diff against, so its oracle reconstructs ground truth from a @@ -22,9 +22,15 @@ pin this with the create-index `format_version` parameter; old Lance versions still use `LANCE_FTS_FORMAT_VERSION`. -The op vocabulary and bounds are deliberately small so the search is runnable; this is -exhaustive over the maintenance-lifecycle grammar up to the configured lengths, not over -every op permutation. +IVF_PQ uses a separate bounded grammar because exhaustively applying the scalar grammar +to a trained vector index would be prohibitively expensive. It covers every ordered pair +of vector/scalar maintenance operations plus a few deeper lifecycle sequences. Its +oracle compares scalar-prefiltered ANN results with an exact, index-free KNN scan and +requires at least 0.5 recall, as well as checking the scalar filter exactly. + +The op vocabulary and bounds are deliberately small so the search is runnable. Scalar +and FTS cases are exhaustive over their maintenance grammar up to the configured length; +vector cases cover every ordered pair plus the curated deeper lifecycles above. """ import itertools @@ -33,9 +39,26 @@ from pathlib import Path ROWS_PER_WRITE = 200 +VECTOR_ROWS_PER_WRITE = 512 +VECTOR_DIM = 8 +VECTOR_K = 10 +VECTOR_KIND = "IVF_PQ" +VECTOR_INDEX_NAME = "vector_idx" +VECTOR_SCALAR_INDEX_NAME = "scalar_idx" SETUP_TAIL_OPS = ["D", "C", "W"] EXERCISE_OPS = ["W", "D", "C", "Oa", "Om", "Od"] +VECTOR_OPS = ("W", "D", "C", "Os", "Ov", "Om") + +# All ordered operation pairs are searched below. These longer cases preserve the +# state combinations that motivated the old recurring test without growing as 6**N. +VECTOR_CRITICAL_SEQUENCES = ( + ("W", "Ov", "W", "Ov", "W"), # two vector deltas, then unindexed rows + ("W", "Os", "W", "Ov", "Om"), + ("D", "C", "W", "Ov", "Om"), + ("W", "D", "Os", "C", "Ov"), + ("W", "Ov", "D", "C", "Om"), +) OP_NAMES = { "W": "write rows", @@ -45,6 +68,8 @@ "Oa": "optimize (append)", "Om": "optimize (merge)", "Od": "optimize", + "Os": "optimize scalar index (append)", + "Ov": "optimize vector index (append)", } @@ -58,7 +83,7 @@ def describe(kind, from_ref, to_ref, setup_ops, exercise_ops, fts_version=None): # Index kinds covered by the maintenance-sequence search. SCALAR_KINDS = ["BTREE", "BITMAP", "LABEL_LIST", "NGRAM", "ZONEMAP", "BLOOMFILTER"] -ALL_KINDS = ["INVERTED", *SCALAR_KINDS] +ALL_KINDS = ["INVERTED", *SCALAR_KINDS, VECTOR_KIND] class IndexScenario: @@ -83,6 +108,19 @@ def _batch(self, a, b): import pyarrow as pa idx = list(range(a, b)) + if self.kind == VECTOR_KIND: + # Deterministic pseudo-random vectors keep every appended batch in the + # training distribution without depending on numpy or process RNG state. + flat = [] + for i in idx: + state = ((i + 1) * 2654435761) & 0xFFFFFFFF + for _ in range(VECTOR_DIM): + state = (state * 1664525 + 1013904223) & 0xFFFFFFFF + flat.append(state / 4294967296.0) + vector = pa.FixedSizeListArray.from_arrays( + pa.array(flat, type=pa.float32()), VECTOR_DIM + ) + return pa.table({"idx": idx, "vector": vector}) if self.kind == "INVERTED": # Each row's text mixes tokens of different frequency: a unique term, a # mid-frequency bucket (~1/7 of rows), and one shared by every row. Sampling @@ -115,7 +153,8 @@ def _oracle_pred(self): def _op_W(self): import lance - a, b = self.next_idx, self.next_idx + ROWS_PER_WRITE + num_rows = VECTOR_ROWS_PER_WRITE if self.kind == VECTOR_KIND else ROWS_PER_WRITE + a, b = self.next_idx, self.next_idx + num_rows self.next_idx = b tbl = self._batch(a, b) if not os.path.exists(self.path): @@ -124,6 +163,18 @@ def _op_W(self): self._open().insert(tbl) def _op_I(self): + if self.kind == VECTOR_KIND: + self._open().create_scalar_index( + "idx", "BTREE", name=VECTOR_SCALAR_INDEX_NAME + ) + self._open().create_index( + "vector", + index_type="IVF_PQ", + name=VECTOR_INDEX_NAME, + num_partitions=2, + num_sub_vectors=2, + ) + return kwargs = {"with_position": True} if self.kind == "INVERTED" else {} if self.kind == "INVERTED" and self.fts_version is not None: kwargs["format_version"] = int(self.fts_version) @@ -145,11 +196,27 @@ def _op_Oa(self): self._open().optimize.optimize_indices(num_indices_to_merge=0) def _op_Om(self): - self._open().optimize.optimize_indices(num_indices_to_merge=10) + kwargs = {"num_indices_to_merge": 10} + if self.kind == VECTOR_KIND: + kwargs = { + "num_indices_to_merge": 1, + "index_names": [VECTOR_INDEX_NAME], + } + self._open().optimize.optimize_indices(**kwargs) def _op_Od(self): self._open().optimize.optimize_indices() + def _op_Os(self): + self._open().optimize.optimize_indices( + num_indices_to_merge=0, index_names=[VECTOR_SCALAR_INDEX_NAME] + ) + + def _op_Ov(self): + self._open().optimize.optimize_indices( + num_indices_to_merge=0, index_names=[VECTOR_INDEX_NAME] + ) + def _run(self, ops): for op in ops: getattr(self, f"_op_{op}")() @@ -164,6 +231,9 @@ def setup(self): def exercise_and_check(self): self._run(self.exercise_ops) ds = self._open() + if self.kind == VECTOR_KIND: + self._check_vector_prefilter(ds) + return if self.kind == "INVERTED": # Differential oracle: rebuild the token -> rows map from a full (unindexed) # scan, then require an FTS search for a spread of sampled terms to return @@ -208,6 +278,136 @@ def exercise_and_check(self): f"{self.kind}: index gave {got} rows, full scan {expected}, for '{pred}'" ) + def _check_vector_prefilter(self, ds): + """Check both BTREE filtering and IVF_PQ recall against index-free scans.""" + # Both ranges avoid the deterministic delete window. The first is always in + # the original index while the newest range may be an unindexed append, so + # the same query covers the indexed + unindexed prefilter path. + filter_rows = VECTOR_ROWS_PER_WRITE // 8 + lo = self.next_idx - filter_rows + pred = f"idx < {filter_rows} OR (idx >= {lo} AND idx < {self.next_idx})" + + filtered_scan = ds.to_table( + columns=["idx", "vector"], filter=pred, use_scalar_index=False + ) + filtered_rows = sorted( + zip( + filtered_scan.column("idx").to_pylist(), + filtered_scan.column("vector").to_pylist(), + ) + ) + filtered_ids = [idx for idx, _ in filtered_rows] + filtered_id_set = set(filtered_ids) + assert len(filtered_ids) == len(filtered_id_set), ( + f"BTREE prefilter returned duplicate row ids for '{pred}'" + ) + assert len(filtered_ids) >= VECTOR_K, ( + f"not enough live rows ({len(filtered_ids)}) for vector oracle '{pred}'" + ) + + scalar_plan = ds.scanner(filter=pred).explain_plan(True) + scan_plan = ds.scanner(filter=pred, use_scalar_index=False).explain_plan(True) + scalar_markers = ("ScalarIndexQuery", "MaterializeIndex") + assert any(marker in scalar_plan for marker in scalar_markers), ( + f"BTREE index was not used for '{pred}':\n{scalar_plan}" + ) + assert not any(marker in scan_plan for marker in scalar_markers), ( + f"BTREE index disabling was ignored for '{pred}':\n{scan_plan}" + ) + scalar_ids = ds.to_table(columns=["idx"], filter=pred).column("idx").to_pylist() + assert len(scalar_ids) == len(set(scalar_ids)), ( + f"BTREE returned duplicate row ids for '{pred}'" + ) + assert set(scalar_ids) == filtered_id_set, ( + f"BTREE returned {len(scalar_ids)} rows, full scan returned " + f"{len(filtered_ids)}, for '{pred}'" + ) + + query_positions = (0, len(filtered_ids) // 2, len(filtered_ids) - 1) + vectors = [vector for _, vector in filtered_rows] + for query_position in query_positions: + query = vectors[query_position] + indexed_nearest = { + "column": "vector", + "q": query, + "k": VECTOR_K, + "nprobes": 2, + "refine_factor": 10, + } + exact_nearest = { + "column": "vector", + "q": query, + "k": VECTOR_K, + "use_index": False, + } + + ann_plan = ds.scanner( + nearest=indexed_nearest, filter=pred, prefilter=True + ).explain_plan(True) + exact_plan = ds.scanner( + nearest=exact_nearest, + filter=pred, + prefilter=True, + use_scalar_index=False, + ).explain_plan(True) + ann_markers = ("ANNSubIndex", "ANNIvfPartition") + assert any(marker in ann_plan for marker in ann_markers), ( + f"IVF_PQ index was not used by vector search:\n{ann_plan}" + ) + assert not any(marker in exact_plan for marker in ann_markers), ( + f"IVF_PQ index disabling was ignored:\n{exact_plan}" + ) + assert any(marker in ann_plan for marker in scalar_markers), ( + f"BTREE index was not used by vector prefilter:\n{ann_plan}" + ) + assert not any(marker in exact_plan for marker in scalar_markers), ( + f"BTREE index disabling was ignored by exact prefilter:\n{exact_plan}" + ) + + got = ( + ds.to_table( + columns=["idx", "_distance"], + nearest=indexed_nearest, + filter=pred, + prefilter=True, + ) + .column("idx") + .to_pylist() + ) + expected = ( + ds.to_table( + columns=["idx", "_distance"], + nearest=exact_nearest, + filter=pred, + prefilter=True, + use_scalar_index=False, + ) + .column("idx") + .to_pylist() + ) + + assert len(got) == len(set(got)), "IVF_PQ search returned duplicate row ids" + assert len(expected) == len(set(expected)), ( + "exact vector search returned duplicate row ids" + ) + assert set(got) <= filtered_id_set, ( + f"IVF_PQ prefilter returned ids outside '{pred}': " + f"{sorted(set(got) - filtered_id_set)[:5]}" + ) + assert set(expected) <= filtered_id_set, ( + f"exact prefilter returned ids outside '{pred}': " + f"{sorted(set(expected) - filtered_id_set)[:5]}" + ) + assert len(got) == len(expected) == VECTOR_K, ( + f"IVF_PQ returned {len(got)} rows, exact search returned " + f"{len(expected)}" + ) + recall = len(set(got) & set(expected)) / VECTOR_K + assert recall >= 0.5, ( + f"IVF_PQ prefilter recall@{VECTOR_K}={recall:.3f}; " + f"expected at least 0.5 (got={got}, exact={expected})" + ) + def generate(max_length): """Yield every (setup_ops, exercise_ops) whose combined length is 1..max_length, @@ -223,6 +423,31 @@ def generate(max_length): yield list(s), list(e) +def generate_vector(max_length): + """Yield a bounded IVF_PQ + BTREE maintenance search space. + + Every ordered pair of operations is covered on both sides of the version split. + A small set of deeper cases captures multi-delta, unindexed-row, delete, compact, + and merge interactions without expanding the full operation grammar to 6**N. + """ + seen = set() + for total in range(1, min(max_length, 2) + 1): + for sequence in itertools.product(VECTOR_OPS, repeat=total): + for setup_len in range(total): + case = (sequence[:setup_len], sequence[setup_len:]) + if case not in seen: + seen.add(case) + yield list(case[0]), list(case[1]) + for sequence in VECTOR_CRITICAL_SEQUENCES: + if len(sequence) > max_length: + continue + for setup_len in range(len(sequence)): + case = (sequence[:setup_len], sequence[setup_len:]) + if case not in seen: + seen.add(case) + yield list(case[0]), list(case[1]) + + def search( venv_factory, from_ref, @@ -254,7 +479,10 @@ def search( # than rebuilding the index). Cached per shard, keyed by the setup ops. snapshots = {} # tuple(setup) -> (snapshot_path, next_idx), or None if setup failed try: - for i, (setup_tail, exercise) in enumerate(generate(max_length)): + cases = ( + generate_vector(max_length) if kind == VECTOR_KIND else generate(max_length) + ) + for i, (setup_tail, exercise) in enumerate(cases): if i % num_shards != shard: continue key = tuple(setup_tail) diff --git a/python/python/tests/compat/test_index_sequence.py b/python/python/tests/compat/test_index_sequence.py index 4d0db694064..0ab1fd65869 100644 --- a/python/python/tests/compat/test_index_sequence.py +++ b/python/python/tests/compat/test_index_sequence.py @@ -11,16 +11,51 @@ Refs and max length are environment-driven so the suite can run between two refs (versions, commits, or branches): COMPAT_FROM_REF / COMPAT_TO_REF / COMPAT_MAX_LENGTH / -COMPAT_KINDS (comma-separated subset of kinds) / COMPAT_SHARDS (split each kind's search -into this many cases so pytest-xdist (`-n auto`) parallelizes them across cores). +COMPAT_VECTOR_MAX_LENGTH / COMPAT_KINDS (comma-separated subset of kinds) / +COMPAT_SHARDS (split each scalar/FTS kind's search into this many cases so pytest-xdist +(`-n auto`) parallelizes them across cores) / COMPAT_VECTOR_SHARDS (the bounded IVF_PQ +search uses fewer shards to avoid repeatedly training the same small index). """ import os +from itertools import product import pytest from .compat_decorator import pylance_stable_versions -from .compat_sequence import ALL_KINDS, search +from .compat_sequence import ( + ALL_KINDS, + VECTOR_KIND, + VECTOR_OPS, + generate_vector, + search, +) + + +def test_vector_sequence_generation_is_bounded_and_covers_high_risk_orders(): + cases = list(generate_vector(max_length=5)) + combined = [tuple(setup + exercise) for setup, exercise in cases] + + assert cases + assert len(cases) < 128 + assert len(cases) == len(set((tuple(s), tuple(e)) for s, e in cases)) + assert all(exercise for _, exercise in cases) + assert all(1 <= len(sequence) <= 5 for sequence in combined) + + split_cases = {(tuple(setup), tuple(exercise)) for setup, exercise in cases} + for pair in product(VECTOR_OPS, repeat=2): + assert ((), pair) in split_cases + assert (pair[:1], pair[1:]) in split_cases + + two_delta_then_unindexed = ("W", "Ov", "W", "Ov", "W") + assert { + (tuple(setup), tuple(exercise)) + for setup, exercise in cases + if tuple(setup + exercise) == two_delta_then_unindexed + } == { + (two_delta_then_unindexed[:split], two_delta_then_unindexed[split:]) + for split in range(len(two_delta_then_unindexed)) + } def _default_refs(): @@ -35,10 +70,15 @@ def _default_refs(): FROM_REF = os.environ.get("COMPAT_FROM_REF") or _default_from TO_REF = os.environ.get("COMPAT_TO_REF") or _default_to MAX_LENGTH = int(os.environ.get("COMPAT_MAX_LENGTH", "4")) +VECTOR_MAX_LENGTH = int(os.environ.get("COMPAT_VECTOR_MAX_LENGTH", str(MAX_LENGTH))) KINDS = os.environ.get("COMPAT_KINDS", ",".join(ALL_KINDS)).split(",") # Many small shards (default 4x cores) so xdist's dynamic scheduler keeps every worker # busy and an oversubscribed `-n` has work to overlap. NUM_SHARDS = int(os.environ.get("COMPAT_SHARDS", str((os.cpu_count() or 1) * 4))) +# Training IVF_PQ once per generic shard would multiply total work without expanding +# coverage. Four cases still parallelize the bounded vector search while retaining +# snapshot reuse within each case. +VECTOR_NUM_SHARDS = int(os.environ.get("COMPAT_VECTOR_SHARDS", str(min(NUM_SHARDS, 4)))) def _cases(): @@ -53,25 +93,42 @@ def _cases(): return cases -CASES = _cases() -CASE_IDS = [k if v is None else f"{k}-fmtv{v}" for k, v in CASES] +def _search_cases(): + cases = [] + for kind, fts_version in _cases(): + num_shards = VECTOR_NUM_SHARDS if kind == VECTOR_KIND else NUM_SHARDS + kind_id = kind if fts_version is None else f"{kind}-fmtv{fts_version}" + cases.extend( + pytest.param( + kind, + fts_version, + shard, + num_shards, + id=f"{kind_id}-shard{shard}", + ) + for shard in range(num_shards) + ) + return cases + + +SEARCH_CASES = _search_cases() @pytest.mark.compat -@pytest.mark.parametrize("kind,fts_version", CASES, ids=CASE_IDS) -@pytest.mark.parametrize("shard", range(NUM_SHARDS)) +@pytest.mark.parametrize("kind,fts_version,shard,num_shards", SEARCH_CASES) def test_index_maintenance_sequence_search( - venv_factory, tmp_path, kind, fts_version, shard + venv_factory, tmp_path, kind, fts_version, shard, num_shards ): + max_length = VECTOR_MAX_LENGTH if kind == VECTOR_KIND else MAX_LENGTH failures = search( venv_factory, FROM_REF, TO_REF, tmp_path, kind, - max_length=MAX_LENGTH, + max_length=max_length, shard=shard, - num_shards=NUM_SHARDS, + num_shards=num_shards, fts_version=fts_version, ) # First line is the failure itself so it shows in pytest's bottom summary; the rest diff --git a/python/python/tests/compat/test_scalar_indices.py b/python/python/tests/compat/test_scalar_indices.py index c3bf301eee0..850c1212532 100644 --- a/python/python/tests/compat/test_scalar_indices.py +++ b/python/python/tests/compat/test_scalar_indices.py @@ -9,6 +9,7 @@ and written by other versions. """ +import os import shutil from pathlib import Path @@ -193,12 +194,17 @@ def __init__(self, path: Path): self.path = path def create(self): - """Create dataset with ZONEMAP and BLOOMFILTER indices.""" + """Create dataset with ZONEMAP and BLOOMFILTER indices. + + The zonemap column contains nulls at rows 0 and 500 so that IS NULL + queries can be verified across version boundaries. + """ shutil.rmtree(self.path, ignore_errors=True) + zonemap_values = [None if i in (0, 500) else i for i in range(1000)] data = pa.table( { "idx": pa.array(range(1000)), - "zonemap": pa.array(range(1000)), + "zonemap": pa.array(zonemap_values, type=pa.int64()), "bloomfilter": pa.array(range(1000)), } ) @@ -215,11 +221,22 @@ def check_read(self): """Verify ZONEMAP and BLOOMFILTER indices can be queried.""" ds = lance.dataset(self.path) - # Test ZONEMAP + # Test ZONEMAP equality table = ds.to_table(filter="zonemap == 7") assert table.num_rows == 1 assert table.column("idx").to_pylist() == [7] + # Test ZONEMAP IS NULL — two nulls were inserted at rows 0 and 500. + # Older versions without a null bitmap fall back to a zone scan, which + # is still correct; newer versions may return an exact result. + table = ds.to_table(filter="zonemap IS NULL") + if 1000 in table.column("idx").to_pylist(): + # After write, there are 3 NULLs + assert table.num_rows == 3 + else: + # Before write, there are 2 NULLs + assert table.num_rows == 2 + # Test BLOOMFILTER table = ds.to_table(filter="bloomfilter == 7") assert table.num_rows == 1 @@ -231,7 +248,7 @@ def check_write(self): data = pa.table( { "idx": pa.array([1000]), - "zonemap": pa.array([1000]), + "zonemap": pa.array([None], type=pa.int64()), "bloomfilter": pa.array([1000]), } ) @@ -239,6 +256,16 @@ def check_write(self): ds.optimize.optimize_indices() ds.optimize.compact_files() + # IS NULL must still return results after the index is updated and + # files are compacted. The newly inserted null must be found + # regardless of which version handles the seed-based index update. + table = ds.to_table(filter="zonemap IS NULL") + assert table.num_rows >= 1 + + def skip_downgrade(self, version: str) -> bool: + # In 0.X the zonemap index did not properly handle NULL in filters + return version.startswith("0.") + @compat_test(min_version="0.36.0") class JsonIndex(UpgradeDowngradeTest): @@ -320,9 +347,12 @@ def create(self): max_rows_per_file=100, data_storage_version=safe_data_storage_version(self.compat_version), ) - dataset.create_scalar_index( - "text", "INVERTED", with_position=True, format_version=1 - ) + kwargs = {"with_position": True} + # Downgrade reads use older wheels, so current-created FTS indexes must + # stay on the legacy posting block layout. + if os.environ.get("LANCE_COMPAT_FTS_LEGACY_BLOCK_SIZE") == "1": + kwargs["block_size"] = 128 + dataset.create_scalar_index("text", "INVERTED", format_version=1, **kwargs) def check_read(self): """Verify FTS index can be queried.""" @@ -351,6 +381,16 @@ def check_write(self): def skip_downgrade(self, version: str) -> bool: return version.startswith("0.") + def current_env(self, method_name: str) -> dict[str, str]: + if method_name == "create": + return { + "LANCE_COMPAT_FTS_LEGACY_BLOCK_SIZE": "1", + "LANCE_FTS_FORMAT_VERSION": "1", + } + if method_name == "check_write": + return {"LANCE_FTS_FORMAT_VERSION": "2"} + return {} + def compat_env(self, version: str, method_name: str) -> dict[str, str]: if method_name in {"create", "check_write"}: return {"LANCE_FTS_FORMAT_VERSION": "1"} diff --git a/python/python/tests/compat/test_venv_manager.py b/python/python/tests/compat/test_venv_manager.py index ebe3dfc766b..57f8b94910a 100644 --- a/python/python/tests/compat/test_venv_manager.py +++ b/python/python/tests/compat/test_venv_manager.py @@ -1,9 +1,12 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright The Lance Authors +import subprocess + import pytest -from .venv_manager import _lance_namespace_dependency +from . import venv_manager +from .venv_manager import _lance_namespace_dependency, _pip_install @pytest.mark.parametrize( @@ -19,3 +22,57 @@ ) def test_lance_namespace_dependency(version: str, expected: str): assert _lance_namespace_dependency(version) == expected + + +@pytest.fixture +def no_sleep(monkeypatch): + sleeps: list[float] = [] + monkeypatch.setattr(venv_manager.time, "sleep", sleeps.append) + return sleeps + + +def test_pip_install_failure_includes_pip_output(monkeypatch, no_sleep): + calls: list[list[str]] = [] + + def failing_run(cmd, **kwargs): + calls.append(cmd) + raise subprocess.CalledProcessError( + 2, cmd, output="resolving deps", stderr="No matching distribution found" + ) + + monkeypatch.setattr(venv_manager.subprocess, "run", failing_run) + with pytest.raises(RuntimeError) as exc_info: + _pip_install("python", ["pylance==0.38.0"]) + # The error must surface pip's captured output, which CalledProcessError hides. + message = str(exc_info.value) + assert "No matching distribution found" in message + assert "resolving deps" in message + assert "pylance==0.38.0" in message + assert len(calls) == venv_manager._PIP_INSTALL_ATTEMPTS + assert len(no_sleep) == venv_manager._PIP_INSTALL_ATTEMPTS - 1 + + +def test_pip_install_retries_transient_failure(monkeypatch, no_sleep): + calls: list[list[str]] = [] + + def flaky_run(cmd, **kwargs): + calls.append(cmd) + if len(calls) == 1: + raise subprocess.CalledProcessError( + 2, cmd, output="", stderr="ReadTimeoutError: pypi.fury.io" + ) + + monkeypatch.setattr(venv_manager.subprocess, "run", flaky_run) + _pip_install("python", ["pylance==0.38.0"]) + assert len(calls) == 2 + assert len(no_sleep) == 1 + + +def test_pip_install_succeeds_first_try_without_sleeping(monkeypatch, no_sleep): + calls: list[list[str]] = [] + monkeypatch.setattr( + venv_manager.subprocess, "run", lambda cmd, **kwargs: calls.append(cmd) + ) + _pip_install("python", ["pytest"]) + assert calls == [["python", "-m", "pip", "install", "--quiet", "pytest"]] + assert no_sleep == [] diff --git a/python/python/tests/compat/venv_manager.py b/python/python/tests/compat/venv_manager.py index c4b23486cd3..04e00d7af00 100644 --- a/python/python/tests/compat/venv_manager.py +++ b/python/python/tests/compat/venv_manager.py @@ -17,9 +17,11 @@ import struct import subprocess import sys +import time from pathlib import Path -from typing import Any, Optional +from typing import Any, Optional, Union +import pytest from packaging.version import InvalidVersion, Version try: @@ -42,6 +44,35 @@ def _venv_lock(lock_path: Path): fcntl.flock(handle, fcntl.LOCK_UN) +_PIP_INSTALL_ATTEMPTS = 3 +_PIP_RETRY_BACKOFF_SECONDS = 2.0 + + +def _pip_install(python: Union[str, Path], args: list[str]) -> None: + """Run ``pip install`` with output captured, retrying transient failures. + + pip installs hit the network (PyPI / fury.io) and a single flaky download + should not fail a whole test run, so retry a bounded number of times with a + short backoff. On final failure raise an error that includes pip's + stdout/stderr, which a bare CalledProcessError from ``capture_output=True`` + would hide from the test log. + """ + cmd = [str(python), "-m", "pip", "install", "--quiet", *args] + for attempt in range(1, _PIP_INSTALL_ATTEMPTS + 1): + try: + subprocess.run(cmd, check=True, capture_output=True, text=True) + return + except subprocess.CalledProcessError as exc: # noqa: PERF203 + if attempt == _PIP_INSTALL_ATTEMPTS: + raise RuntimeError( + f"pip install failed after {attempt} attempts " + f"(exit status {exc.returncode}): {cmd}\n" + f"stdout:\n{exc.stdout}\n" + f"stderr:\n{exc.stderr}" + ) from exc + time.sleep(_PIP_RETRY_BACKOFF_SECONDS * attempt) + + NAMESPACE_0_6_DEPENDENCY = "lance-namespace<0.7" NAMESPACE_0_7_DEPENDENCY = "lance-namespace>=0.7.2,<0.8" NAMESPACE_0_8_DEPENDENCY = "lance-namespace>=0.8.0,<0.9" @@ -96,6 +127,36 @@ def _safe(ref: str) -> str: return re.sub(r"[^A-Za-z0-9._-]", "_", ref) +# Explicit skip whitelist for known (lance_version, python_version) incompatibilities. +# Each entry is (max_lance_version_inclusive, min_python_version_inclusive). +# A test is skipped when the tested lance version <= max_lance AND the current +# Python >= min_python. Add entries here only after verifying the incompatibility +# is a runtime/ABI issue rather than a format regression — anything NOT listed +# will surface as a test failure so new problems are visible. +_COMPAT_SKIP: list[tuple[str, tuple[int, int]]] = [ + # lance 0.22.x abi3 wheel was built with old PyO3 (<0.23) that crashes on + # Python 3.14+ due to removed internal CPython APIs. + ("0.22.0", (3, 14)), +] + + +def _skip_reason(lance_version: str) -> Optional[str]: + """Return a skip reason if this lance/Python combo is whitelisted, else None.""" + try: + ver = Version(lance_version) + except InvalidVersion: + return None + py = sys.version_info[:2] + for max_lance, min_python in _COMPAT_SKIP: + if ver <= Version(max_lance) and py >= min_python: + py_str = f"{py[0]}.{py[1]}" + return ( + f"Lance {lance_version} + Python {py_str}: whitelisted skip " + f"(see _COMPAT_SKIP in venv_manager.py)" + ) + return None + + class VenvExecutor: """Manages a virtual environment with a specific Lance version.""" @@ -130,15 +191,29 @@ def python_path(self) -> Path: def _marker_path(self) -> Path: return self.venv_path / ".compat_ref" + @staticmethod + def _python_version_tag() -> str: + return f"{sys.version_info.major}.{sys.version_info.minor}" + def _validate_venv(self) -> bool: - """A cached venv is reusable if it exists and its recorded ref matches. A marker - file is used (not `pip show`) so source-built commit refs also validate.""" + """A cached venv is reusable if it exists, its recorded ref matches, and it was + built with the same Python major.minor as the current interpreter. + + The marker file format is two lines: `\\n`. + Old single-line markers (no Python version) are treated as stale so the venv + is rebuilt — this handles cached venvs from a different Python installation.""" if not self.python_path.exists(): return False try: - return self._marker_path.read_text().strip() == self.version + lines = self._marker_path.read_text().strip().splitlines() except OSError: return False + if not lines or lines[0] != self.version: + return False + # Require a Python version line; single-line markers are stale. + if len(lines) < 2 or lines[1] != self._python_version_tag(): + return False + return True def create(self): """Create the virtual environment and install the specified Lance version.""" @@ -170,24 +245,18 @@ def create(self): self._install_release_wheel() else: self._build_from_source() - self._marker_path.write_text(self.version) + self._marker_path.write_text( + f"{self.version}\n{self._python_version_tag()}" + ) self._created = True def _install_wheel(self, wheel: str): - subprocess.run( - [str(self.python_path), "-m", "pip", "install", "--quiet", wheel, "pytest"], - check=True, - capture_output=True, - ) + _pip_install(self.python_path, [wheel, "pytest"]) def _install_release_wheel(self): - subprocess.run( + _pip_install( + self.python_path, [ - str(self.python_path), - "-m", - "pip", - "install", - "--quiet", "--pre", "--extra-index-url", "https://pypi.fury.io/lance-format/", @@ -201,8 +270,6 @@ def _install_release_wheel(self): _lance_namespace_dependency(self.version), "pytest", ], - check=True, - capture_output=True, ) def _build_from_source(self): @@ -225,11 +292,7 @@ def _build_from_source(self): check=True, capture_output=True, ) - subprocess.run( - [py, "-m", "pip", "install", "--quiet", "maturin", "pytest", "pyarrow"], - check=True, - capture_output=True, - ) + _pip_install(py, ["maturin", "pytest", "pyarrow"]) wheels = src / "target" / "compat-wheels" subprocess.run( [ @@ -249,11 +312,7 @@ def _build_from_source(self): capture_output=True, ) wheel = next(wheels.glob("pylance-*.whl")) - subprocess.run( - [py, "-m", "pip", "install", "--quiet", str(wheel)], - check=True, - capture_output=True, - ) + _pip_install(py, [str(wheel)]) def _ensure_subprocess(self): """Ensure the persistent subprocess is running.""" @@ -264,10 +323,13 @@ def _ensure_subprocess(self): # Start persistent subprocess runner_script = Path(__file__).parent / "venv_runner.py" - # Set PYTHONPATH to include the tests directory + # Set PYTHONPATH so the subprocess can import compat test modules. + # pytest adds the `tests/` directory (the first ancestor without __init__.py) + # to sys.path, so test modules are imported as `compat.`. env = os.environ.copy() tests_dir = Path(__file__).parent.parent env["PYTHONPATH"] = str(tests_dir) + env.setdefault("RUST_BACKTRACE", "full") # Capture stderr to a file so a Rust panic (which crashes the runner) can be # surfaced in the error instead of an opaque "broken pipe". @@ -357,6 +419,10 @@ def execute_method( if not self._created: raise RuntimeError("Virtual environment not created. Call create() first.") + reason = _skip_reason(self.version) + if reason: + pytest.skip(reason) + # Ensure subprocess is running self._ensure_subprocess() try: @@ -379,14 +445,16 @@ def execute_method( except (BrokenPipeError, EOFError, struct.error) as e: # Subprocess died (usually a Rust panic); flush it, then surface that. + returncode = "unknown" if self._subprocess is not None: try: self._subprocess.wait(timeout=2) + returncode = str(self._subprocess.returncode) except Exception: pass panic = self._last_panic() detail = panic or f"subprocess communication failed: {e}" - raise RuntimeError(f"Lance {self.version}: {detail}") + raise RuntimeError(f"Lance {self.version} (exit={returncode}): {detail}") def cleanup(self): """Remove the virtual environment directory and terminate subprocess.""" @@ -404,8 +472,6 @@ def cleanup(self): # Remove venv directory if self.venv_path.exists(): - import shutil - shutil.rmtree(self.venv_path) self._created = False diff --git a/python/python/tests/recurring/test_recurring.py b/python/python/tests/recurring/test_recurring.py deleted file mode 100644 index ab39269528a..00000000000 --- a/python/python/tests/recurring/test_recurring.py +++ /dev/null @@ -1,231 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright The Lance Authors - -# Recurring tests that runs all operations on a large dataset, -# these operations are ran in random order repeated 10 times - -import abc -import itertools -from datetime import timedelta -from typing import Optional - -import lance -import numpy as np -import pyarrow as pa -import pytest - -# For testing, use smaller numbers to make tests run faster -# In production, you might want to use: NUM_ROWS = 1_000_000 -NUM_ROWS = 1_000_000 -BATCH_SIZE = 1_000 -DIM = 32 - -schema = pa.schema( - [ - pa.field("id", pa.int64()), - pa.field("vector", pa.list_(pa.float32(), DIM)), - pa.field("text", pa.string()), - ] -) -words = ["hello", "world", "this", "is", "a", "test", "sentence"] - - -def random_text(num_words: int) -> str: - return " ".join(np.random.choice(words, num_words)) - - -def random_batch(start_id: int, batch_size: int) -> pa.Table: - return pa.Table.from_arrays( - [ - pa.array(np.arange(start_id, start_id + batch_size)), - pa.array(np.random.rand(batch_size, DIM).tolist()), - pa.array( - [random_text(np.random.randint(1, 10)) for _ in range(batch_size)] - ), - ], - schema=schema, - ) - - -def create_or_load_dataset(dataset_name: str, kwargs: dict): - uri = f"tests/recurring/{dataset_name}" - - # Try to open existing dataset first - try: - ds = lance.dataset(uri) - if ds.count_rows() > 0: - return ds - except Exception: - pass - - # Create new dataset with initial data - initial_batch = random_batch(0, BATCH_SIZE) - ds = lance.write_dataset(initial_batch, uri, schema=schema, mode="overwrite") - - # Add remaining data - for i in range(BATCH_SIZE, NUM_ROWS, BATCH_SIZE): - batch = random_batch(i, BATCH_SIZE) - ds.insert(batch) - - # Create indices - ds.create_scalar_index("id", index_type="BTREE", replace=True) - ds.create_index( - "vector", - index_type="IVF_PQ", - metric="cosine", - num_partitions=128, - num_sub_vectors=DIM // 8, - replace=True, - ) - - # Note: FTS index creation is async, but we'll handle this differently for pytest - # For now, we'll skip the async part and create it synchronously if possible - try: - ds.create_scalar_index( - "text", - index_type="INVERTED", - with_position=kwargs.get("with_position", False), - replace=True, - ) - except Exception as e: - print(f"Warning: Could not create FTS index: {e}") - - return ds - - -class Operation(abc.ABC): - @abc.abstractmethod - def read_only(self) -> bool: ... - - @abc.abstractmethod - def run(self, ds: lance.LanceDataset): ... - - -class ReadOnlyOperation(Operation): - def read_only(self) -> bool: - return True - - -class WriteOperation(Operation): - def read_only(self) -> bool: - return False - - -class Append(WriteOperation): - def run(self, ds: lance.LanceDataset): - batch = random_batch(ds.count_rows(), BATCH_SIZE) - ds.insert(batch) - - -class Delete(WriteOperation): - def __init__(self, delete_num_rows: int = 100): - self.delete_num_rows = delete_num_rows - - def run(self, ds: lance.LanceDataset): - num_rows = ds.count_rows() - to_delete = np.random.randint(0, num_rows, self.delete_num_rows) - to_delete = ", ".join([str(v) for v in to_delete]) - ds.delete(f"id IN ({to_delete})") - - -class Optimize(WriteOperation): - def __init__(self, num_indices_to_merge: int, column: str): - self.num_indices_to_merge = num_indices_to_merge - self.column = column - - def run(self, ds: lance.LanceDataset): - ds.optimize.optimize_indices( - num_indices_to_merge=self.num_indices_to_merge, - index_names=[f"{self.column}_idx"], - ) - - -class Compact(WriteOperation): - def run(self, ds: lance.LanceDataset): - ds.optimize.compact_files() - - -class VectorSearch(ReadOnlyOperation): - def __init__(self, filter: Optional[str] = None): - self.filter = filter - - def run(self, ds: lance.LanceDataset): - stats = ds.stats.index_stats("vector_idx") - if stats is None: - print("No vector index found") - return - query_vector = np.random.rand(DIM).tolist() - query = ds.scanner( - nearest={ - "q": query_vector, - "k": 10, - "column": "vector", - }, - filter=self.filter, - ) - query.analyze_plan() - - -class FullTextSearch(ReadOnlyOperation): - def __init__(self, has_position: bool, filter: Optional[str] = None): - self.has_position = has_position - self.filter = filter - - def run(self, ds: lance.LanceDataset): - stats = ds.stats.index_stats("text_idx") - if stats is None: - print("No text index found") - return - query_text = random_text(np.random.randint(1, 10)) - self.do_query(ds, query_text) - - if self.has_position: - query_text = f'"{query_text}"' - self.do_query(ds, query_text) - - def do_query(self, ds: lance.LanceDataset, query_text: str): - query: lance.LanceScanner = ds.scanner( - full_text_query=query_text, - filter=self.filter, - limit=10, - ) - query.analyze_plan() - - -@pytest.mark.recurring -@pytest.mark.parametrize("with_position", [True]) -def test_all_permutations(with_position): - """Test all operations on dataset without FTS position tracking""" - dataset_name = f"test_table_with_position_{with_position}" - ds = create_or_load_dataset(dataset_name, {"with_position": with_position}) - - write_operations = [ - Append(), - Delete(delete_num_rows=1000), - Optimize(num_indices_to_merge=0, column="id"), - Optimize(num_indices_to_merge=0, column="vector"), # delta index - Optimize(num_indices_to_merge=1, column="vector"), # merge index - Optimize(num_indices_to_merge=0, column="text"), - Compact(), - ] - - read_only_operations = [ - # Read only operations - VectorSearch(), - VectorSearch(filter="id >= 1000 and id < 8000"), - FullTextSearch(has_position=False), - FullTextSearch(has_position=False, filter="id >= 1000 and id < 8000"), - ] - - for permutation in itertools.permutations(range(len(write_operations))): - for idx in permutation: - write_operation = write_operations[idx] - print(f"Running {write_operation.__class__.__name__}") - write_operation.run(ds) - ds.cleanup_old_versions(older_than=timedelta(seconds=0)) - - # write operation changed the status of the table, - # then we need to run all read only operations after it - for read_only_operation in read_only_operations: - print(f"Running {read_only_operation.__class__.__name__}") - read_only_operation.run(ds) diff --git a/python/python/tests/test_arrow.py b/python/python/tests/test_arrow.py index 92ab52021ff..a0be09846e1 100644 --- a/python/python/tests/test_arrow.py +++ b/python/python/tests/test_arrow.py @@ -273,8 +273,6 @@ def test_image_uri_arrays(tmp_path: Path, png_uris): def test_image_tensor_arrays(tmp_path: Path, png_uris): - tf = pytest.importorskip("tensorflow") - n = 10 encoded_image_array = ImageURIArray.from_uris(png_uris).read_uris() @@ -297,22 +295,22 @@ def test_image_tensor_arrays(tmp_path: Path, png_uris): assert tensor_image_array.storage.type == pa.list_(pa.uint8(), 4) assert tensor_image_array[2].as_py() == [42, 42, 42, 255] - test_tensor = tf.constant( - np.array([42, 42, 42, 255] * n, dtype=np.uint8).reshape((n, 1, 1, 4)) - ) + test_tensor = np.array([42, 42, 42, 255] * n, dtype=np.uint8).reshape((n, 1, 1, 4)) assert test_tensor.shape == (n, 1, 1, 4) - assert tf.math.reduce_all( - tf.convert_to_tensor(tensor_image_array.to_numpy()) == test_tensor - ) + assert np.array_equal(tensor_image_array.to_numpy(), test_tensor) assert tensor_image_array.to_encoded().to_tensor() == tensor_image_array def png_encoder(images): - import tensorflow as tf + import io - encoded_images = ( - tf.io.encode_png(x).numpy() for x in tf.convert_to_tensor(images) - ) + from PIL import Image # pyright: ignore[reportMissingImports] + + encoded_images = [] + for image in images: + with io.BytesIO() as buf: + Image.fromarray(image).save(buf, format="PNG") + encoded_images.append(buf.getvalue()) return pa.array(encoded_images, type=pa.binary()) assert tensor_image_array.to_encoded(png_encoder).to_tensor() == tensor_image_array @@ -324,20 +322,18 @@ def png_encoder(images): uris = [str(Path(x)) for x in uris] encoded_image_array = ImageArray.from_array(uris).read_uris() - with pytest.raises( - tf.errors.InvalidArgumentError, match="Shapes of all inputs must match" - ): + with pytest.raises(ValueError, match="all input arrays must have the same shape"): encoded_image_array.to_tensor() pattern = r"(object at) 0x[\w\d]+(:?>)" repl = r"\1 0x..\2" - assert re.sub(pattern, repl, encoded_image_array.__repr__()) == ( - "\n" - "[, ..]\n" + repr_ = re.sub(pattern, repl, encoded_image_array.__repr__()) + assert repr_.startswith("\n[ 0 + assert writer.size_bytes == os.path.getsize(path) + + +def test_size_bytes_with_session(tmp_path): + session = LanceFileSession(tmp_path) + with session.open_writer("foo.lance") as writer: + writer.write_batch(pa.table({"a": [1, 2, 3]})) + assert writer.size_bytes == os.path.getsize(tmp_path / "foo.lance") def test_version(tmp_path): diff --git a/python/python/tests/test_filter.py b/python/python/tests/test_filter.py index 9416c191e36..16bd3c0061d 100644 --- a/python/python/tests/test_filter.py +++ b/python/python/tests/test_filter.py @@ -371,8 +371,8 @@ def test_filter_on_column_beside_root_extension_type(tmp_path): @pytest.mark.skip( - reason="enable this in recurring test https://github.com/lance-format/lance/pull/4190" - " as it requires release mode" + reason="requires a release build; see " + "https://github.com/lance-format/lance/pull/4190" ) def test_filter_depth_limit(): column_name = "a_very_long_column_name" diff --git a/python/python/tests/test_fragment.py b/python/python/tests/test_fragment.py index b05888df31f..09721c21f4a 100644 --- a/python/python/tests/test_fragment.py +++ b/python/python/tests/test_fragment.py @@ -22,7 +22,7 @@ ) from lance.debug import format_fragment from lance.file import LanceFileWriter -from lance.fragment import write_fragments +from lance.fragment import RowIdMeta, RowIdSequence, write_fragments from lance.progress import FileSystemFragmentWriteProgress @@ -279,7 +279,7 @@ def test_fragment_meta(): "file_size_bytes=100), DataFile(path='1.lance', fields=[1], column_indices=[], " "file_major_version=0, file_minor_version=0, file_size_bytes=None)], " "physical_rows=100, deletion_file=None, row_id_meta=None, " - "created_at_version_meta=None, last_updated_at_version_meta=None)" + "created_at_version_meta=None, last_updated_at_version_meta=None, overlays=[])" ) @@ -617,6 +617,139 @@ def test_fragment_update_columns_with_custom_join_key(tmp_path): assert result["name"][2] == "Chase" # id=3 should have name Chase +def test_fragment_update_columns_with_blob_v2(tmp_path): + data = pa.table( + { + "id": pa.array([1, 2, 3, 4]), + "payload": lance.blob_array([b"one", b"two", b"", None]), + } + ) + dataset_uri = tmp_path / "test_dataset_update_columns_blob_v2" + dataset = lance.write_dataset( + data, + dataset_uri, + data_storage_version="2.2", + ) + + fragment = dataset.get_fragment(0) + updated_fragment, fields_modified = fragment.update_columns( + pa.table( + { + "id": pa.array([2]), + "payload": lance.blob_array([b"NEW"]), + } + ), + left_on="id", + ) + + operation = LanceOperation.Update( + updated_fragments=[updated_fragment], + fields_modified=fields_modified, + ) + updated_dataset = LanceDataset.commit( + dataset_uri, + operation, + read_version=dataset.version, + ) + + result = updated_dataset.to_table(blob_handling="all_binary") + assert result["id"].to_pylist() == [1, 2, 3, 4] + assert result["payload"].to_pylist() == [b"one", b"NEW", b"", None] + + +def test_fragment_update_columns_with_nested_blob_v2(tmp_path): + def info_array(names, payloads): + fields = [pa.field("name", pa.string()), lance.blob_field("blob")] + return pa.StructArray.from_arrays( + [pa.array(names), lance.blob_array(payloads)], fields=fields + ) + + dataset_uri = tmp_path / "test_dataset_update_columns_nested_blob_v2" + dataset = lance.write_dataset( + pa.table( + { + "id": pa.array([1, 2]), + "info": info_array(["a", "b"], [b"one", b"two"]), + } + ), + dataset_uri, + data_storage_version="2.2", + ) + + updated_fragment, fields_modified = dataset.get_fragment(0).update_columns( + pa.table( + { + "id": pa.array([2]), + "info": info_array(["B"], [b"NEW"]), + } + ), + left_on="id", + ) + updated_dataset = LanceDataset.commit( + dataset_uri, + LanceOperation.Update( + updated_fragments=[updated_fragment], + fields_modified=fields_modified, + ), + read_version=dataset.version, + ) + + info = updated_dataset.to_table(blob_handling="all_binary")["info"].combine_chunks() + assert info.field("name").to_pylist() == ["a", "B"] + assert info.field("blob").to_pylist() == [b"one", b"NEW"] + + +def test_fragment_update_columns_preserves_external_blob_v2(tmp_path): + dataset_uri = tmp_path / "test_dataset_update_columns_external_blob_v2" + external = tmp_path / "existing-payload.bin" + external.write_bytes(b"outside") + dataset = lance.write_dataset( + pa.table( + { + "id": pa.array([1, 2]), + "payload": lance.blob_array([external.as_uri(), b"two"]), + } + ), + dataset_uri, + data_storage_version="2.2", + allow_external_blob_outside_bases=True, + ) + + updated_fragment, fields_modified = dataset.get_fragment(0).update_columns( + pa.table( + { + "id": pa.array([2]), + "payload": lance.blob_array([b"NEW"]), + } + ), + left_on="id", + ) + updated_dataset = LanceDataset.commit( + dataset_uri, + LanceOperation.Update( + updated_fragments=[updated_fragment], + fields_modified=fields_modified, + ), + read_version=dataset.version, + ) + + result = updated_dataset.to_table(blob_handling="all_binary") + assert result["payload"].to_pylist() == [b"outside", b"NEW"] + + new_external = tmp_path / "new-payload.bin" + new_external.write_bytes(b"new outside") + with pytest.raises(ValueError, match="outside registered external bases"): + updated_dataset.get_fragment(0).update_columns( + pa.table( + { + "id": pa.array([2]), + "payload": lance.blob_array([new_external.as_uri()]), + } + ), + left_on="id", + ) + + def test_fragment_update_columns_with_nulls(tmp_path): """Test fragment update columns with null values.""" # Create initial dataset @@ -865,3 +998,322 @@ def test_fragment_take_with_json_column(tmp_path): assert metas[0] == '{"val":1}' assert metas[1] == '{"val":4}' assert metas[2] == '{"val":7}' + + +def test_fragment_create_with_json_column(tmp_path): + """Test that LanceFragment.create works with Arrow JSON extension type. + + Previously the single-fragment create path skipped the Arrow JSON (Utf8) -> + Lance JSON (JSONB LargeBinary) conversion that write_dataset/write_fragments + perform, so the raw UTF-8 string bytes were written into a column whose schema + declared JSONB. Reads then miss-decoded the bytes and returned garbage. + """ + json_type = pa.json_() + data = pa.table( + { + "uid": pa.array(["a", "b", "c", "d"], type=pa.utf8()), + "payload": pa.array( + ['{"x":1}', '{"x":2}', '{"y":3}', '{"y":4}'], + type=json_type, + ), + } + ) + + frag = LanceFragment.create(tmp_path, data) + operation = LanceOperation.Overwrite(data.schema, [frag]) + dataset = LanceDataset.commit(tmp_path, operation) + + result = dataset.to_table() + assert result.column("uid").to_pylist() == ["a", "b", "c", "d"] + payloads = result.column("payload").to_pylist() + assert [json.loads(p) for p in payloads] == [ + {"x": 1}, + {"x": 2}, + {"y": 3}, + {"y": 4}, + ] + + +def test_fragment_update_columns_with_json_column(tmp_path): + """Test that fragment update_columns works with Arrow JSON extension type. + + Previously this would fail with a type mismatch error because the + HashJoiner didn't convert Arrow JSON (Utf8) to Lance JSON (LargeBinary). + """ + # Create initial dataset with a JSON extension type column + json_type = pa.json_() + data = pa.table( + { + "id": pa.array([1, 2, 3, 4, 5], type=pa.int64()), + "name": pa.array(["a", "b", "c", "d", "e"], type=pa.utf8()), + "meta": pa.array( + ['{"x":1}', '{"x":2}', '{"x":3}', '{"x":4}', '{"x":5}'], + type=json_type, + ), + } + ) + dataset_uri = tmp_path / "test_update_cols_json" + dataset = lance.write_dataset(data, dataset_uri) + + # Prepare update data: update the JSON column for some rows + update_data = pa.table( + { + "_rowid": pa.array([1, 3], type=pa.uint64()), + "meta": pa.array( + ['{"updated":true,"id":2}', '{"updated":true,"id":4}'], + type=json_type, + ), + } + ) + + # This should NOT raise a type mismatch error + fragment = dataset.get_fragment(0) + updated_fragment, fields_modified = fragment.update_columns(update_data) + + assert len(fields_modified) > 0 + + # Commit and verify + op = LanceOperation.Update( + updated_fragments=[updated_fragment], + fields_modified=fields_modified, + ) + updated_dataset = lance.LanceDataset.commit( + str(dataset_uri), op, read_version=dataset.version + ) + + result = updated_dataset.to_table() + ids = result.column("id").to_pylist() + metas = result.column("meta").to_pylist() + + for i, (id_val, meta_val) in enumerate(zip(ids, metas)): + meta = json.loads(meta_val) if isinstance(meta_val, str) else meta_val + if id_val == 2 or id_val == 4: + assert "updated" in meta_val or meta.get("updated") is True, ( + f"id={id_val} should be updated, got {meta_val}" + ) + else: + assert "x" in meta_val or "x" in str(meta), ( + f"id={id_val} should have original value, got {meta_val}" + ) + + +def test_row_id_sequence_from_range(): + # A step-of-one range is the compact case and must not materialize its ids. + sequence = RowIdSequence(range(10)) + + assert len(sequence) == 10 + assert sequence.to_pyarrow() == pa.array(range(10), type=pa.uint64()) + assert sequence.to_pyarrow().type == pa.uint64() + assert list(sequence) == list(range(10)) + + +@pytest.mark.parametrize( + "row_ids", + [ + pytest.param(pa.array([1, 2, 3]), id="pyarrow_array"), + pytest.param(pa.array([1, 2, 3], type=pa.uint64()), id="pyarrow_uint64_array"), + pytest.param(pa.array([1, 2, 3], type=pa.int8()), id="pyarrow_int8_array"), + pytest.param(pa.array([1, 2, 3], type=pa.uint16()), id="pyarrow_uint16_array"), + # A slice carries an offset into a larger buffer; only the slice counts. + pytest.param(pa.array([9, 1, 2, 3, 9]).slice(1, 3), id="pyarrow_sliced_array"), + pytest.param(pa.chunked_array([[1], [2, 3]]), id="pyarrow_chunked_array"), + pytest.param((x for x in [1, 2, 3]), id="generator"), + pytest.param([1, 2, 3], id="list"), + pytest.param(range(1, 4), id="range"), + pytest.param(range(3, 0, -1), id="descending_range"), + ], +) +def test_row_id_sequence_accepts_input_types(row_ids): + sequence = RowIdSequence(row_ids) + + assert sorted(sequence) == [1, 2, 3] + + +@pytest.mark.parametrize( + "row_ids", + [ + pytest.param([], id="empty_list"), + pytest.param(range(0), id="empty_range"), + pytest.param(pa.array([], type=pa.uint64()), id="empty_array"), + ], +) +def test_row_id_sequence_empty(row_ids): + sequence = RowIdSequence(row_ids) + + assert len(sequence) == 0 + assert list(sequence) == [] + assert sequence.to_pyarrow() == pa.array([], type=pa.uint64()) + + +@pytest.mark.parametrize( + "row_ids", + [ + pytest.param(list(range(4100)), id="contiguous"), + pytest.param(list(range(0, 8200, 2)), id="gapped"), + pytest.param(list(range(4100))[::-1], id="unsorted"), + ], +) +def test_row_id_sequence_iterates_large_sequences(row_ids): + # Each shape picks a different segment encoding, so all of them have to + # round-trip through iteration. + sequence = RowIdSequence(row_ids) + + assert list(sequence) == row_ids + # Each call must hand back a fresh iterator rather than a spent one. + assert list(sequence) == row_ids + + +def test_row_id_sequence_from_range_above_isize(): + # Range bounds are read as isize; beyond that the values are read one at a + # time instead, which must still cover the whole uint64 row id domain. + start = 2**63 + sequence = RowIdSequence(range(start, start + 3)) + + assert list(sequence) == [start, start + 1, start + 2] + + +def test_row_id_sequence_unsorted_round_trips(): + sequence = RowIdSequence([12, 11, 10]) + + assert list(sequence) == [12, 11, 10] + assert sequence.to_pyarrow() == pa.array([12, 11, 10], type=pa.uint64()) + + +@pytest.mark.parametrize( + "row_ids", + [ + pytest.param([1, 1, 2], id="adjacent"), + pytest.param([1, 2, 3, 1], id="separated"), + pytest.param(pa.array([5, 3, 5]), id="unsorted_array"), + ], +) +def test_row_id_sequence_rejects_duplicates(row_ids): + with pytest.raises(ValueError, match="Row ids must be unique"): + RowIdSequence(row_ids) + + +@pytest.mark.parametrize( + ("row_ids", "message"), + [ + pytest.param(pa.array([1, None, 3]), "must not be null", id="null_in_array"), + pytest.param(pa.array([1.5, 2.5]), "array of integers", id="float_array"), + pytest.param(pa.array([-1, 2]), "uint64", id="negative_in_array"), + pytest.param(range(-5, 5), "non-negative", id="negative_range"), + pytest.param(5, "iterable of integers", id="not_iterable"), + ], +) +def test_row_id_sequence_rejects_invalid_input(row_ids, message): + with pytest.raises((ValueError, TypeError), match=message): + RowIdSequence(row_ids) + + +def test_row_id_sequence_metadata_round_trip(): + sequence = RowIdSequence([7, 12, 3]) + + metadata = sequence.to_inline_metadata() + assert isinstance(metadata, RowIdMeta) + assert RowIdSequence.from_inline_metadata(metadata) == sequence + + +def test_row_id_sequence_equality_and_repr(): + assert RowIdSequence(range(3)) == RowIdSequence([0, 1, 2]) + assert RowIdSequence(range(3)) != RowIdSequence([0, 1]) + # Comparing against an unrelated type is False rather than an error. + assert RowIdSequence(range(3)) != "not a sequence" + + assert repr(RowIdSequence([1, 2])) == "RowIdSequence([1, 2])" + assert repr(RowIdSequence(range(12))) == ( + "RowIdSequence([0, 1, 2, 3, 4, 5, 6, 7, 8, 9, ...], len=12)" + ) + + +def test_row_id_sequence_pickle(): + sequence = RowIdSequence([7, 12, 3]) + + assert pickle.loads(pickle.dumps(sequence)) == sequence + + +def _row_ids_by_id(dataset: LanceDataset) -> dict: + table = dataset.to_table(columns=["id"], with_row_id=True) + return dict(zip(table["id"].to_pylist(), table["_rowid"].to_pylist())) + + +def test_row_id_sequence_preserves_ids_in_manual_update(tmp_path: Path): + # An update assembled externally (delete the old row, append a replacement) + # keeps the row's identity only if the new fragment carries its row id. + dataset = write_dataset( + pa.table({"id": [1, 2, 3, 4], "v": [10, 20, 30, 40]}), + tmp_path, + max_rows_per_file=2, + enable_stable_row_ids=True, + ) + row_ids_before = _row_ids_by_id(dataset) + + updated_fragment = dataset.get_fragments()[0].delete("id = 2") + (new_fragment,) = write_fragments(pa.table({"id": [2], "v": [99]}), tmp_path) + new_fragment.row_id_meta = RowIdSequence([row_ids_before[2]]).to_inline_metadata() + + dataset = LanceDataset.commit( + tmp_path, + LanceOperation.Update( + removed_fragment_ids=[], + updated_fragments=[updated_fragment], + new_fragments=[new_fragment], + fields_modified=[], + ), + read_version=dataset.version, + ) + + assert _row_ids_by_id(dataset) == row_ids_before + assert dataset.to_table().sort_by("id").to_pydict() == { + "id": [1, 2, 3, 4], + "v": [10, 99, 30, 40], + } + + +def test_row_id_sequence_reads_back_fragment_metadata(tmp_path: Path): + dataset = write_dataset( + pa.table({"a": range(10)}), + tmp_path, + max_rows_per_file=5, + enable_stable_row_ids=True, + ) + + sequences = [ + RowIdSequence.from_inline_metadata(fragment.metadata.row_id_meta) + for fragment in dataset.get_fragments() + ] + + assert [list(sequence) for sequence in sequences] == [ + [0, 1, 2, 3, 4], + [5, 6, 7, 8, 9], + ] + + +def test_fragment_validate(tmp_path: Path): + dataset = write_dataset( + pa.table({"a": range(100), "b": range(100)}), + tmp_path, + max_rows_per_file=50, + ) + # A valid fragment validates without raising. + for fragment in dataset.get_fragments(): + assert fragment.validate() is None + + +def test_fragment_validate_across_data_files(tmp_path: Path): + # add_columns writes a second data file per fragment; validate must still + # pass (field ids increasing and unique across a fragment's data files). + dataset = write_dataset(pa.table({"a": range(100)}), tmp_path, max_rows_per_file=50) + dataset.add_columns({"b": "a + 1"}) + for fragment in dataset.get_fragments(): + assert len(fragment.data_files()) > 1 + fragment.validate() + + +def test_fragment_validate_after_delete(tmp_path: Path): + dataset = write_dataset(pa.table({"a": range(100)}), tmp_path, max_rows_per_file=50) + dataset.delete("a < 10") + # A fragment carrying a deletion vector still validates. + for fragment in dataset.get_fragments(): + fragment.validate() diff --git a/python/python/tests/test_geo.py b/python/python/tests/test_geo.py index c011c2de3de..f7d9d7b3e1a 100644 --- a/python/python/tests/test_geo.py +++ b/python/python/tests/test_geo.py @@ -19,6 +19,19 @@ ) +def _query_point_ids(dataset: lance.LanceDataset, wkt: str) -> list[int]: + sql = f""" + SELECT id, point + FROM dataset + WHERE St_Intersects(point, ST_GeomFromText('{wkt}')) + """ + return [ + value + for batch in dataset.sql(sql).build().to_batch_records() + for value in batch.column("id").to_pylist() + ] + + def test_geo_types(tmp_path: Path): uri = str(tmp_path / "test_geo_types.lance") # Points @@ -153,3 +166,276 @@ def query(ds: lance.LanceDataset, has_index=False): table_with_index = query(ds, has_index=True) assert table_with_index == table_without_index + + +def test_rtree_segment_merge_and_commit(tmp_path: Path): + num_points = 120 + points_2d = points( + [ + np.arange(num_points, dtype=np.float64), + np.arange(num_points, dtype=np.float64), + ] + ) + schema = pa.schema( + [ + pa.field("id", pa.int64()), + pa.field(point("xy")).with_name("point"), + ] + ) + table = pa.Table.from_arrays( + [np.arange(num_points, dtype=np.int64), points_2d], schema=schema + ) + ds = lance.write_dataset( + table, + str(tmp_path / "segmented_rtree.lance"), + max_rows_per_file=40, + ) + fragments = ds.get_fragments() + assert len(fragments) == 3 + segments = [ + ds.create_index_uncommitted( + column="point", + index_type="RTREE", + name="point_rtree", + fragment_ids=[fragment.fragment_id], + ) + for fragment in fragments + ] + + merged = ds.merge_existing_index_segments(segments) + assert set(merged.fragment_ids) == {fragment.fragment_id for fragment in fragments} + ds = ds.commit_existing_index_segments("point_rtree", "point", [merged]) + + sql = """ + SELECT id, point + FROM dataset + WHERE St_Intersects(point, ST_GeomFromText('LINESTRING (10 10, 110 110)')) + """ + indexed = pa.Table.from_batches(ds.sql(sql).build().to_batch_records()) + assert indexed["id"].to_pylist() == list(range(10, 111)) + explain = ( + pa.Table.from_batches( + ds.sql("EXPLAIN ANALYZE " + sql).build().to_batch_records() + ) + .to_pandas() + .to_string() + ) + assert "ScalarIndexQuery" in explain + + +def test_staged_rtree_after_rewrite_columns(tmp_path: Path): + uri = str(tmp_path / "stale_rtree.lance") + point_type = point("xy") + schema = pa.schema( + [ + pa.field("id", pa.int64()), + pa.field(point_type).with_name("point"), + ] + ) + dataset = lance.write_dataset( + pa.Table.from_arrays( + [ + pa.array([0, 1], type=pa.int64()), + points([np.array([0.0, 1.0]), np.array([0.0, 1.0])]), + ], + schema=schema, + ), + uri, + ) + segment = dataset.create_index_uncommitted( + column="point", + index_type="RTREE", + name="point_rtree", + fragment_ids=[0], + ) + + update_schema = pa.schema( + [ + pa.field("_rowid", pa.uint64()), + pa.field(point_type).with_name("point"), + ] + ) + update = pa.Table.from_arrays( + [ + pa.array([0], type=pa.uint64()), + points([np.array([10.0]), np.array([10.0])]), + ], + schema=update_schema, + ) + fragment, fields = dataset.get_fragment(0).update_columns(update) + updated = lance.LanceDataset.commit( + uri, + lance.LanceOperation.Update( + updated_fragments=[fragment], + fields_modified=fields, + ), + read_version=dataset.version, + ) + + assert _query_point_ids(updated, "POINT (10 10)") == [0] + committed = dataset.commit_existing_index_segments( + "point_rtree", + "point", + [segment], + ) + assert committed.describe_indices()[0].segments[0].fragment_ids == set() + assert _query_point_ids(committed, "POINT (10 10)") == [0] + + +def test_rtree_rejects_distributed_uuid_reuse(tmp_path: Path): + uri = str(tmp_path / "uuid_reuse.lance") + num_points = 120 + point_type = point("xy") + schema = pa.schema( + [ + pa.field("id", pa.int64()), + pa.field(point_type).with_name("point"), + ] + ) + dataset = lance.write_dataset( + pa.Table.from_arrays( + [ + pa.array(range(num_points), type=pa.int64()), + points( + [ + np.arange(num_points, dtype=np.float64), + np.arange(num_points, dtype=np.float64), + ] + ), + ], + schema=schema, + ), + uri, + max_rows_per_file=40, + ) + dataset.create_scalar_index("point", "RTREE") + index_uuid = dataset.describe_indices()[0].segments[0].uuid + + with pytest.raises( + ValueError, + match="index_uuid is no longer accepted for RTree distributed index builds", + ): + dataset.create_index_uncommitted( + column="point", + index_type="RTREE", + name="point_rtree_reuse", + fragment_ids=[0], + index_uuid=index_uuid, + ) + + assert _query_point_ids( + lance.dataset(uri), + "LINESTRING (100 100, 110 110)", + ) == list(range(100, 111)) + + +def test_rtree_merge_all_deleted_stable_row_ids(tmp_path: Path): + uri = str(tmp_path / "all_deleted.lance") + point_type = point("xy") + schema = pa.schema( + [ + pa.field("id", pa.int64()), + pa.field(point_type).with_name("point"), + ] + ) + dataset = lance.write_dataset( + pa.Table.from_arrays( + [ + pa.array([0, 1], type=pa.int64()), + points([np.array([0.0, 1.0]), np.array([0.0, 1.0])]), + ], + schema=schema, + ), + uri, + enable_stable_row_ids=True, + ) + segment = dataset.create_index_uncommitted( + column="point", + index_type="RTREE", + name="point_rtree", + fragment_ids=[0], + ) + + dataset.delete("true") + merged = dataset.merge_existing_index_segments([segment]) + assert merged.fragment_ids == set() + committed = dataset.commit_existing_index_segments( + "point_rtree", + "point", + [merged], + ) + assert _query_point_ids(committed, "POINT (0 0)") == [] + + +def test_rtree_merge_preserves_newer_fragment_coverage(tmp_path: Path): + uri = str(tmp_path / "mixed_versions.lance") + point_type = point("xy") + schema = pa.schema( + [ + pa.field("id", pa.int64()), + pa.field(point_type).with_name("point"), + ] + ) + + def batch(start: int, stop: int) -> pa.Table: + values = np.arange(start, stop, dtype=np.float64) + return pa.Table.from_arrays( + [ + pa.array(range(start, stop), type=pa.int64()), + points([values, values]), + ], + schema=schema, + ) + + dataset = lance.write_dataset(batch(0, 40), uri) + first = dataset.create_index_uncommitted( + column="point", + index_type="RTREE", + name="point_rtree", + fragment_ids=[0], + ) + dataset = lance.write_dataset(batch(40, 80), uri, mode="append") + second = dataset.create_index_uncommitted( + column="point", + index_type="RTREE", + name="point_rtree", + fragment_ids=[1], + ) + + merged = dataset.merge_existing_index_segments([first, second]) + assert merged.fragment_ids == {0, 1} + assert merged.dataset_version == dataset.version + + update_schema = pa.schema( + [ + pa.field("_rowid", pa.uint64()), + pa.field(point_type).with_name("point"), + ] + ) + update = pa.Table.from_arrays( + [ + pa.array([1 << 32], type=pa.uint64()), + points([np.array([100.0]), np.array([100.0])]), + ], + schema=update_schema, + ) + fragment, fields = dataset.get_fragment(1).update_columns(update) + dataset = lance.LanceDataset.commit( + uri, + lance.LanceOperation.Update( + updated_fragments=[fragment], + fields_modified=fields, + ), + read_version=dataset.version, + ) + + committed = dataset.commit_existing_index_segments( + "point_rtree", + "point", + [merged], + ) + assert committed.describe_indices()[0].segments[0].fragment_ids == {0} + assert _query_point_ids( + committed, + "POINT (100 100)", + ) == [40] diff --git a/python/python/tests/test_indices.py b/python/python/tests/test_indices.py index 02cf64541d6..ae51e6ddb0d 100644 --- a/python/python/tests/test_indices.py +++ b/python/python/tests/test_indices.py @@ -8,7 +8,7 @@ import numpy as np import pyarrow as pa import pytest -from lance.file import LanceFileReader +from lance.file import LanceFileReader, LanceFileWriter from lance.indices import IndicesBuilder, IvfModel, PqModel NUM_ROWS_PER_FRAGMENT = 10000 @@ -209,6 +209,29 @@ def test_gen_pq(tmpdir, rand_dataset, rand_ivf): assert pq.dimension == reloaded.dimension assert pq.codebook == reloaded.codebook + pq_4bit = IndicesBuilder(rand_dataset, "vectors").train_pq( + rand_ivf, + sample_rate=2, + num_bits=4, + ) + assert pq_4bit.num_bits == 4 + assert len(pq_4bit.codebook) == 16 + + pq_4bit.save(str(tmpdir / "pq_4bit")) + reloaded = PqModel.load(str(tmpdir / "pq_4bit")) + assert reloaded.num_bits == 4 + + legacy_pq_uri = str(tmpdir / "legacy_pq") + with LanceFileWriter( + legacy_pq_uri, + pa.schema( + [pa.field("codebook", pq.codebook.type)], + metadata={b"num_subvectors": str(pq.num_subvectors).encode()}, + ), + ) as writer: + writer.write_batch(pa.table([pq.codebook], names=["codebook"])) + assert PqModel.load(legacy_pq_uri).num_bits == 8 + def test_ivf_centroids_fragment_ids(tmpdir): rows_per_fragment = 32 @@ -300,7 +323,7 @@ def test_indices_builder_multivector_distributed_dimensions(tmpdir, monkeypatch) captured_dimensions = {} - def train_pq_model(*args): + def train_pq_model(*args, **kwargs): captured_dimensions["train_pq"] = args[2] return codebook diff --git a/python/python/tests/test_lance.py b/python/python/tests/test_lance.py index 0162e370665..4ba9281823d 100644 --- a/python/python/tests/test_lance.py +++ b/python/python/tests/test_lance.py @@ -248,6 +248,74 @@ def test_io_counters(tmp_path): assert lance.bytes_read_counter() > starting_bytes +def test_simd_info(): + info = lance.simd_info() + assert info["tier"] in ( + "none", + "sse", + "avx", + "avx_fma", + "avx2", + "avx512", + "avx512_fp16", + "neon", + "lsx", + "lasx", + ) + assert isinstance(info["target_arch"], str) and info["target_arch"] + if info["target_arch"] == "x86_64": + # The x86_64 ABI mandates SSE2. + assert "sse2" in info["host_features"] + else: + assert info["host_features"] == [] + + +@pytest.mark.parametrize( + ("query", "options", "expected"), + [ + ("the cats and dogs", {}, [("cat", 0), ("dog", 2)]), + ( + "the skip alpha", + {"stem": False, "custom_stop_words": ["skip"]}, + [("the", 0), ("alpha", 2)], + ), + ( + "getUserName", + {"analyzer": "code", "split_identifiers": True}, + [ + ("getusername", 0), + ("get", 0), + ("user", 1), + ("name", 2), + ], + ), + ( + "a::b", + {"analyzer": "code", "index_operators": True}, + [("a", 0), ("::", 1), ("b", 2)], + ), + ], +) +def test_tokenize_fts_query(query, options, expected): + tokens = lance.tokenize(query, **options) + assert [(token.text, token.position) for token in tokens] == expected + + +def test_tokenize_fts_query_validates_analyzer_options(): + with pytest.raises(ValueError, match="code analyzer flags require analyzer='code'"): + lance.tokenize("getUserName", split_identifiers=True) + + with pytest.raises(ValueError, match="unknown base tokenizer"): + lance.tokenize("hello", base_tokenizer="unknown") + + +def test_tokenize_fts_query_can_disable_max_token_length(): + long_token = "x" * 41 + assert lance.tokenize(long_token, stem=False) == [] + unlimited_tokens = lance.tokenize(long_token, max_token_length=None, stem=False) + assert [token.text for token in unlimited_tokens] == [long_token] + + @pytest.mark.parametrize( "row_param, column_name", [("with_row_id", "_rowid"), ("with_row_address", "_rowaddr")], diff --git a/python/python/tests/test_mem_wal.py b/python/python/tests/test_mem_wal.py index 95596a7123e..4a688c4168b 100644 --- a/python/python/tests/test_mem_wal.py +++ b/python/python/tests/test_mem_wal.py @@ -9,6 +9,7 @@ import pyarrow as pa import pytest from lance.mem_wal import ( + CompactedSsTable, LsmPointLookupPlanner, LsmScanner, LsmVectorSearchPlanner, @@ -56,15 +57,15 @@ def _append_only_table(ids, prefix: str) -> pa.Table: ) -def _write_flushed_gen(base_path: str, shard_id: str, gen_folder: str, data: pa.Table): - """Write a flushed-generation Lance dataset at the expected sub-path. +def _write_sstable(base_path: str, shard_id: str, gen_folder: str, data: pa.Table): + """Write an SSTable Lance dataset at the expected sub-path. - The collector resolves flushed generation paths as: + The collector resolves SSTable paths as: {base_dataset_path}/_mem_wal/{shard_id}/{gen_folder} Production flush also writes a primary-key dedup sidecar (`_pk_index/`) that the LSM scanner opens to dedup across generations; stage it here too so the - flushed generation faithfully matches what flush produces. + SSTable faithfully matches what flush produces. """ from lance.lance import _write_pk_sidecar @@ -73,17 +74,36 @@ def _write_flushed_gen(base_path: str, shard_id: str, gen_folder: str, data: pa. _write_pk_sidecar(gen_path, data, ["id"]) +def test_mark_sstables_as_compacted(tmp_path): + ds_path = str(tmp_path / "base") + shard_id = str(uuid.uuid4()) + dataset = lance.write_dataset( + _lookup_table([1, 2, 3], "base"), ds_path, schema=_LOOKUP_SCHEMA + ) + dataset.initialize_mem_wal() + + ( + dataset.merge_insert("id") + .when_matched_update_all() + .when_not_matched_insert_all() + .mark_sstables_as_compacted([CompactedSsTable(shard_id, 1)]) + .execute(_lookup_table([2, 4], "compacted")) + ) + + assert dataset.count_rows() == 4 + + def test_point_lookup_with_memtables(tmp_path): """ - Lookup against a base table that has one flushed generation containing an - update. The flushed version must win over the base table version. + Lookup against a base table that has one SSTable containing an + update. The SSTable version must win over the base table version. Setup ----- base : ids [1, 2, 3] names ["base_1", "base_2", "base_3"] gen_1 : ids [2] names ["gen1_2"] ← update to id=2 - ShardSnapshot: flushed_generation(gen=1, path="gen_1"), current_generation=2 + ShardSnapshot: sstable(gen=1, path="gen_1"), current_generation=2 """ ds_path = str(tmp_path / "base") shard_id = str(uuid.uuid4()) @@ -94,28 +114,25 @@ def test_point_lookup_with_memtables(tmp_path): ) base_ds.initialize_mem_wal() - # --- Flushed generation: overwrites id=2 --- - _write_flushed_gen(ds_path, shard_id, "gen_1", _lookup_table([2], "gen1")) + # --- SSTable: overwrites id=2 --- + _write_sstable(ds_path, shard_id, "gen_1", _lookup_table([2], "gen1")) - # --- ShardSnapshot describing the flushed state --- - snap = ( - ShardSnapshot(shard_id) - .with_flushed_generation(1, "gen_1") - .with_current_generation(2) - ) + # --- ShardSnapshot describing the SSTable state --- + snap = ShardSnapshot(shard_id).with_sstable(1, "gen_1").with_current_generation(2) planner = LsmPointLookupPlanner(base_ds, [snap]) assert not hasattr(planner, "lookup") - # id=2 must return the flushed version + # id=2 must return the SSTable version plan = planner.plan_lookup(pa.array([2], type=pa.int64())) assert plan.schema.names == ["id", "name"] assert plan.dataset_schema.names == ["id", "name"] - assert "Take" in plan.explain() or "Scan" in plan.explain() + explained = plan.explain() + assert "Take" in explained or "Scan" in explained or "LanceRead" in explained result = plan.to_table() assert len(result) == 1, "Expected exactly one row for id=2" assert result.column("name")[0].as_py() == "gen1_2", ( - "Flushed generation must win over base table" + "SSTable must win over base table" ) # id=1 is only in the base table @@ -146,13 +163,9 @@ def test_lsm_scanner_with_memtables(tmp_path): ) base_ds.initialize_mem_wal() - _write_flushed_gen(ds_path, shard_id, "gen_1", _lookup_table([2], "gen1")) + _write_sstable(ds_path, shard_id, "gen_1", _lookup_table([2], "gen1")) - snap = ( - ShardSnapshot(shard_id) - .with_flushed_generation(1, "gen_1") - .with_current_generation(2) - ) + snap = ShardSnapshot(shard_id).with_sstable(1, "gen_1").with_current_generation(2) scanner = LsmScanner.from_snapshots(base_ds, [snap]) table = scanner.to_table() @@ -161,7 +174,7 @@ def test_lsm_scanner_with_memtables(tmp_path): name_by_id = {row["id"]: row["name"] for row in table.to_pylist()} assert name_by_id[1] == "base_1" - assert name_by_id[2] == "gen1_2", "Flushed gen must overwrite base for id=2" + assert name_by_id[2] == "gen1_2", "SSTable gen must overwrite base for id=2" assert name_by_id[3] == "base_3" offset_table = ( @@ -170,7 +183,7 @@ def test_lsm_scanner_with_memtables(tmp_path): assert len(offset_table) == 2, "Offset-only LSM scan should not require a limit" -def test_shard_writer_lsm_scanner_includes_own_flushed_generations(tmp_path): +def test_shard_writer_lsm_scanner_includes_own_sstables(tmp_path): ds_path = str(tmp_path / "base") shard_id = str(uuid.uuid4()) ds = lance.write_dataset(_lookup_table([0], "base"), ds_path, schema=_LOOKUP_SCHEMA) @@ -192,10 +205,37 @@ def test_shard_writer_lsm_scanner_includes_own_flushed_generations(tmp_path): if name_by_id.get(1) == "writer_1" and name_by_id.get(2) == "writer_2": break if time.time() >= deadline: - assert False, "writer.lsm_scanner() did not include flushed writer rows" + assert False, "writer.lsm_scanner() did not include SSTable writer rows" time.sleep(0.05) +def test_shard_writer_delete_binding_masks_base_row(tmp_path): + ds_path = str(tmp_path / "base") + shard_id = str(uuid.uuid4()) + ds = lance.write_dataset( + _lookup_table([1, 2, 3], "base"), ds_path, schema=_LOOKUP_SCHEMA + ) + ds.initialize_mem_wal() + + delete_keys = pa.table({"id": pa.array([2], type=pa.int64())}) + + with ds.mem_wal_writer( + shard_id, + durable_write=True, + max_wal_buffer_size=1, + max_wal_flush_interval_ms=10, + ) as writer: + writer.put(_lookup_table([4], "writer")) + writer.delete(delete_keys) + table = writer.lsm_scanner().to_table() + + rows = {row["id"]: row["name"] for row in table.to_pylist()} + assert rows[1] == "base_1" + assert 2 not in rows, "deleted base row should be masked by the tombstone" + assert rows[3] == "base_3" + assert rows[4] == "writer_4" + + _VDIM = 4 # matches Rust test fixture dimension @@ -305,7 +345,7 @@ def test_shard_writer_e2e_correctness(tmp_path): End-to-end correctness test for ShardWriter covering: - Multi-round writes that trigger WAL and MemTable flushes - File-system layout verification (_mem_wal//wal/ and manifest/) - - Flushed generation data readable via LsmScanner + - SSTable data readable via LsmScanner - New writer created after close can write and scan correctly Mirrors Rust test: shard_writer_tests::test_shard_writer_e2e_correctness @@ -327,7 +367,6 @@ def test_shard_writer_e2e_correctness(tmp_path): writer = ds.mem_wal_writer( shard_id, durable_write=True, - sync_indexed_write=True, max_wal_buffer_size=10 * 1024, # 10 KB max_wal_flush_interval_ms=50, max_memtable_size=80, # flush after ~80 rows @@ -377,9 +416,7 @@ def test_shard_writer_e2e_correctness(tmp_path): # === New writer: write and read back via active MemTable scanner === ds2 = lance.dataset(ds_path) shard_id2 = str(uuid.uuid4()) - with ds2.mem_wal_writer( - shard_id2, durable_write=False, sync_indexed_write=True - ) as writer2: + with ds2.mem_wal_writer(shard_id2, durable_write=False) as writer2: verify_batch = _e2e_batch(schema, start_id=10000, num_rows=10) writer2.put(pa.Table.from_batches([verify_batch])) result = writer2.lsm_scanner().to_table() @@ -494,7 +531,6 @@ def test_initialize_mem_wal_writer_config_defaults(tmp_path): # Duration knobs are recorded in milliseconds with a `_ms` suffix. assert defaults["max_wal_flush_interval_ms"] == "250" # Every ShardWriterConfig tunable is recorded once any default is set. - assert "sync_indexed_write" in defaults assert "enable_memtable" in defaults diff --git a/python/python/tests/test_optimize.py b/python/python/tests/test_optimize.py index e35093dc370..48557c63cd2 100644 --- a/python/python/tests/test_optimize.py +++ b/python/python/tests/test_optimize.py @@ -1,5 +1,6 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright The Lance Authors +import json import pickle import random import re @@ -37,6 +38,25 @@ def test_dataset_optimize(tmp_path: Path): assert dataset.version == 3 +def test_compact_files_max_source_fragments(tmp_path: Path): + rows_per_fragment = 256 * 1024 + dataset = lance.write_dataset( + pa.table({"a": pa.nulls(10 * rows_per_fragment)}), + tmp_path / "dataset", + max_rows_per_file=rows_per_fragment, + ) + assert len(dataset.get_fragments()) == 10 + + metrics = dataset.optimize.compact_files( + max_source_fragments=4, + num_threads=1, + ) + + assert metrics.fragments_removed == 4 + assert metrics.fragments_added == 1 + assert len(dataset.get_fragments()) == 7 + + def test_blob_compaction(tmp_path: Path): base_dir = tmp_path / "blob_dataset" blob_field = pa.field( @@ -69,6 +89,121 @@ def test_blob_compaction(tmp_path: Path): assert contents == blobs +def test_blob_compaction_with_nested_json_sibling(tmp_path: Path): + dataset_uri = tmp_path / "nested_blob_json" + info_fields = [lance.blob_field("blob"), pa.field("meta", pa.json_())] + schema = pa.schema( + [ + pa.field("id", pa.int64()), + pa.field("info", pa.struct(info_fields)), + ] + ) + for index, row_id in enumerate([1, 2]): + info = pa.StructArray.from_arrays( + [ + lance.blob_array([f"blob-{row_id}".encode()]), + pa.array([json.dumps({"row": row_id})], type=pa.json_()), + ], + fields=info_fields, + ) + lance.write_dataset( + pa.Table.from_arrays([pa.array([row_id]), info], schema=schema), + dataset_uri, + mode="create" if index == 0 else "append", + data_storage_version="2.2", + ) + + dataset = lance.dataset(dataset_uri) + dataset.optimize.compact_files(num_threads=1) + + assert len(dataset.get_fragments()) == 1 + assert [data for _, data in dataset.read_blobs("info.blob", indices=[0, 1])] == [ + b"blob-1", + b"blob-2", + ] + assert [ + json.loads(value) + for value in dataset.to_table(columns=["info.meta"])["info.meta"].to_pylist() + ] == [{"row": 1}, {"row": 2}] + + +@pytest.mark.parametrize("storage_version", ["2.0", "2.1", "2.2"]) +def test_blob_compaction_preserves_null_empty_and_read_parity( + tmp_path: Path, storage_version: str +): + base_dir = tmp_path / f"blob_dataset_{storage_version}" + if storage_version == "2.2": + blob_field = lance.blob_field("blob") + else: + blob_field = pa.field( + "blob", + pa.large_binary(), + metadata={"lance-encoding:blob": "true"}, + ) + schema = pa.schema([pa.field("id", pa.int64()), blob_field]) + + def make_blob_array(values): + if storage_version == "2.2": + return lance.blob_array(values) + return pa.array(values, type=pa.large_binary()) + + for index, (ids, values) in enumerate( + [ + ([1, 2], [b"P1", None]), + ([3, 4, 5], [b"P3", None, b""]), + ] + ): + lance.write_dataset( + pa.Table.from_arrays( + [ + pa.array(ids, type=pa.int64()), + make_blob_array(values), + ], + schema=schema, + ), + base_dir, + mode="create" if index == 0 else "append", + data_storage_version=storage_version, + ) + + dataset = lance.dataset(base_dir) + dataset.delete("id = 2") + dataset.optimize.compact_files(num_threads=1) + + expected = {1: b"P1", 3: b"P3", 4: None, 5: b""} + binary_table = dataset.to_table(columns=["id", "blob"], blob_handling="all_binary") + scan_values = dict( + zip( + binary_table.column("id").to_pylist(), + binary_table.column("blob").to_pylist(), + ) + ) + + ids = dataset.to_table(columns=["id"]).column("id").to_pylist() + blob_files = dataset.take_blobs("blob", indices=range(len(ids))) + take_values = { + row_id: None if blob_file is None else blob_file.readall() + for row_id, blob_file in zip(ids, blob_files) + } + + assert take_values == expected + assert scan_values == take_values + + descriptor_table = dataset.to_table(columns=["id", "blob"]) + descriptions = dict( + zip( + descriptor_table.column("id").to_pylist(), + descriptor_table.column("blob").to_pylist(), + ) + ) + if storage_version == "2.0": + assert descriptions[4] == {"position": 1, "size": 0} + else: + assert descriptions[4] is None + assert descriptions[5] is not None + assert descriptions[5]["size"] == 0 + + def test_optimize_max_bytes(tmp_path: Path): base_dir = tmp_path / "dataset" arr = pa.array(range(4 * 1024 * 1024)) @@ -492,6 +627,20 @@ def test_optimize_indices_second_call_is_noop(tmp_path: Path): must not write any new files to the dataset directory.""" base_dir = tmp_path / "dataset" + def assert_optimize_is_noop(dataset: lance.LanceDataset): + version_before = dataset.version + files_before = { + path.relative_to(base_dir) for path in base_dir.rglob("*") if path.is_file() + } + + dataset.optimize.optimize_indices() + + files_after = { + path.relative_to(base_dir) for path in base_dir.rglob("*") if path.is_file() + } + assert dataset.version == version_before + assert files_after == files_before + n = 1024 rng = np.random.default_rng(0) vectors = rng.standard_normal((n, 8)).astype(np.float32) @@ -546,16 +695,20 @@ def test_optimize_indices_second_call_is_noop(tmp_path: Path): # First optimize: should pull the new fragment into each index. dataset.optimize.optimize_indices() + assert_optimize_is_noop(dataset) - files_before = {p.relative_to(base_dir) for p in base_dir.rglob("*") if p.is_file()} + # Consolidate the vector delta so every fragment has the same physical index + # coverage; compaction intentionally avoids mixing fragments that do not. + dataset.optimize.optimize_indices(num_indices_to_merge=10) - # Second optimize: nothing has changed, so this must be a no-op on disk. + # Compaction invalidates the old fragment coverage, so one optimize is needed + # to rebuild each index. Further calls must return to the same steady state. + compaction = dataset.optimize.compact_files( + target_rows_per_fragment=2 * (n + extra_rows), num_threads=1 + ) + assert compaction.fragments_removed > 0 dataset.optimize.optimize_indices() - - files_after = {p.relative_to(base_dir) for p in base_dir.rglob("*") if p.is_file()} - - new_files = files_after - files_before - assert not new_files, f"second optimize_indices created new files: {new_files}" + assert_optimize_is_noop(dataset) def test_compaction_generates_rewrite_transaction(tmp_path: Path): @@ -591,6 +744,10 @@ def test_remap_row_addrs(tmp_path: Path): before = ds.scanner(columns=["id"], with_row_address=True).to_table() old = dict(zip(before["id"].to_pylist(), before["_rowaddr"].to_pylist())) + # A deferred-remap compaction records a fragment-reuse index only when it + # rewrites data an index covers, so index a column first. + ds.create_scalar_index("id", "BTREE") + ds.optimize.compact_files( target_rows_per_fragment=1_000, defer_index_remap=True, num_threads=1 ) diff --git a/python/python/tests/test_otel.py b/python/python/tests/test_otel.py new file mode 100644 index 00000000000..a96e0125f56 --- /dev/null +++ b/python/python/tests/test_otel.py @@ -0,0 +1,93 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright The Lance Authors + +import lance +import pyarrow as pa +import pytest + +# The metrics recorder is process-global and installed once, so the whole +# bridge is exercised in a single test to avoid cross-test global-state coupling. + + +def _metrics_by_name(reader): + data = reader.get_metrics_data() + result = {} + for resource_metrics in data.resource_metrics: + for scope_metrics in resource_metrics.scope_metrics: + for metric in scope_metrics.metrics: + result[metric.name] = metric + return result + + +def test_instrument_lance_metrics_exports_object_store_metrics(tmp_path): + pytest.importorskip("opentelemetry.sdk.metrics") + from lance.otel import instrument_lance_metrics + from opentelemetry.sdk.metrics import MeterProvider + from opentelemetry.sdk.metrics.export import InMemoryMetricReader + + reader = InMemoryMetricReader() + provider = MeterProvider(metric_readers=[reader]) + assert instrument_lance_metrics(provider) + + # The catalog is populated once the recorder is installed. + from lance.lance import lance_metrics_catalog + + catalog = {desc.name: desc for desc in lance_metrics_catalog()} + assert "lance_object_store_requests_total" in catalog + assert catalog["lance_object_store_request_duration_seconds"].kind == "histogram" + # Gauges and the retryable counter are described too, so they surface in the + # export even when a plain write doesn't happen to emit them. + assert catalog["lance_object_store_retryable_responses_total"].kind == "counter" + assert catalog["lance_object_store_in_flight_requests"].kind == "gauge" + + # Generate object store activity on the local filesystem (scheme "file"). + table = pa.table({"id": pa.array(range(256))}) + dataset = lance.write_dataset(table, str(tmp_path / "ds.lance")) + assert dataset.to_table().num_rows == 256 + + metrics = _metrics_by_name(reader) + + requests = metrics["lance_object_store_requests_total"] + points = list(requests.data.data_points) + assert points, "expected at least one request data point" + # The `base` label carries the store scheme ("file") by default. + assert all("base" in p.attributes and "operation" in p.attributes for p in points) + assert sum(p.value for p in points) > 0 + + # Histograms are decomposed into bucket / count / sum observable counters. + bucket = metrics["lance_object_store_request_duration_seconds_bucket"] + bucket_points = list(bucket.data.data_points) + assert bucket_points + assert all("le" in p.attributes for p in bucket_points) + # The implicit +Inf bucket must be present and is the cumulative maximum. + assert any(p.attributes["le"] == "+Inf" for p in bucket_points) + + count = metrics["lance_object_store_request_duration_seconds_count"] + assert sum(p.value for p in count.data.data_points) > 0 + + # The `_sum` instrument must also be wired and report positive latency. + duration_sum = metrics["lance_object_store_request_duration_seconds_sum"] + assert sum(p.value for p in duration_sum.data.data_points) > 0 + + +def test_snapshot_empty_before_install_is_safe(): + # snapshot is callable regardless of installation state and never raises. + from lance.lance import snapshot_lance_metrics + + assert isinstance(snapshot_lance_metrics(), list) + + +def test_instrument_warns_when_recorder_unavailable(monkeypatch): + # A foreign `metrics` recorder already installed -> register returns False; + # instrument_lance_metrics must warn and return False without instrumenting. + pytest.importorskip("opentelemetry.sdk.metrics") + import lance.otel as otel + from opentelemetry.sdk.metrics import MeterProvider + from opentelemetry.sdk.metrics.export import InMemoryMetricReader + + monkeypatch.setattr(otel, "register_lance_metrics_recorder", lambda: False) + + reader = InMemoryMetricReader() + provider = MeterProvider(metric_readers=[reader]) + with pytest.warns(UserWarning, match="recorder"): + assert otel.instrument_lance_metrics(provider) is False diff --git a/python/python/tests/test_pydantic.py b/python/python/tests/test_pydantic.py new file mode 100644 index 00000000000..3db60885f6b --- /dev/null +++ b/python/python/tests/test_pydantic.py @@ -0,0 +1,177 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright The Lance Authors + +from enum import Enum +from typing import List, Optional + +import pyarrow as pa +import pytest + +BaseModel = pytest.importorskip("pydantic").BaseModel +Field = pytest.importorskip("pydantic").Field + +from lance.pydantic import ( # noqa: E402 + MultiVector, + Vector, + is_nullable, + pydantic_to_schema, +) + + +def test_scalar_and_optional_and_list_unchanged(): + class Simple(BaseModel): + name: str + score: float + tag: Optional[str] = None + values: List[int] + + schema = pydantic_to_schema(Simple) + assert schema == pa.schema( + [ + pa.field("name", pa.utf8(), False), + pa.field("score", pa.float64(), False), + pa.field("tag", pa.utf8(), True), + pa.field("values", pa.list_(pa.int64()), False), + ] + ) + + +def test_nested_model_becomes_struct(): + class Address(BaseModel): + city: str + zip_code: str + + class Person(BaseModel): + name: str + address: Address + + schema = pydantic_to_schema(Person) + address_type = pa.struct( + [ + pa.field("city", pa.utf8(), False), + pa.field("zip_code", pa.utf8(), False), + ] + ) + assert schema == pa.schema( + [ + pa.field("name", pa.utf8(), False), + pa.field("address", address_type, False), + ] + ) + + +def test_string_enum_is_dictionary_encoded(): + class Color(str, Enum): + RED = "red" + GREEN = "green" + + class Item(BaseModel): + color: Color + + schema = pydantic_to_schema(Item) + assert schema.field("color").type == pa.dictionary(pa.int32(), pa.utf8()) + + +def test_int_enum_uses_native_type(): + class Priority(int, Enum): + LOW = 0 + HIGH = 1 + + class Task(BaseModel): + priority: Priority + + schema = pydantic_to_schema(Task) + assert schema.field("priority").type == pa.int64() + + +def test_vector_field(): + class Doc(BaseModel): + id: int + embedding: Vector(8) + + schema = pydantic_to_schema(Doc) + assert schema.field("embedding").type == pa.list_(pa.float32(), 8) + assert schema.field("embedding").nullable is True + + +def test_vector_field_not_nullable(): + class Doc(BaseModel): + id: int + embedding: Vector(8, nullable=False) + + schema = pydantic_to_schema(Doc) + assert schema.field("embedding").nullable is False + + +def test_multi_vector_field(): + class Doc(BaseModel): + id: int + embeddings: MultiVector(4) + + schema = pydantic_to_schema(Doc) + assert schema.field("embeddings").type == pa.list_(pa.list_(pa.float32(), 4)) + + +def test_tz_aware_datetime_field(): + from datetime import datetime + + class Event(BaseModel): + occurred_at: datetime = Field(json_schema_extra={"tz": "UTC"}) + + schema = pydantic_to_schema(Event) + assert schema.field("occurred_at").type == pa.timestamp("us", tz="UTC") + + +def test_naive_datetime_field(): + from datetime import datetime + + class Event(BaseModel): + occurred_at: datetime + + schema = pydantic_to_schema(Event) + assert schema.field("occurred_at").type == pa.timestamp("us", tz=None) + + +def test_defaulted_non_optional_field_is_not_nullable(): + """A plain default value (without Optional) should not make a field + nullable -- this matches lancedb's behavior, tightened from lance's + previous default-implies-nullable inference.""" + + class Counter(BaseModel): + count: int = 0 + + schema = pydantic_to_schema(Counter) + assert schema.field("count").nullable is False + + field_info = Counter.model_fields["count"] + assert is_nullable(field_info) is False + + +def test_optional_field_is_nullable(): + class Counter(BaseModel): + count: Optional[int] = None + + schema = pydantic_to_schema(Counter) + assert schema.field("count").nullable is True + + +def test_v1_style_field_nullable_vector_overrides_allow_none(): + """Simulates a Pydantic v1 ModelField (no `.annotation` attribute; type + is read off `.outer_type_`, nullability off `.allow_none`) to verify a + nullable Vector/MultiVector annotation overrides `allow_none=False` + under v1, matching the v2 behavior tested in test_vector_field above.""" + + class FakeV1Field: + def __init__(self, outer_type_, allow_none): + self.outer_type_ = outer_type_ + self.allow_none = allow_none + + nullable_vector = Vector(8, nullable=True) + assert is_nullable(FakeV1Field(nullable_vector, allow_none=False)) is True + + non_nullable_vector = Vector(8, nullable=False) + assert is_nullable(FakeV1Field(non_nullable_vector, allow_none=False)) is False + + # Plain (non-Vector) types still fall back to `allow_none`. + assert is_nullable(FakeV1Field(str, allow_none=True)) is True + assert is_nullable(FakeV1Field(str, allow_none=False)) is False diff --git a/python/python/tests/test_s3_ddb.py b/python/python/tests/test_s3_ddb.py index dc9744115e2..5289dd176fb 100644 --- a/python/python/tests/test_s3_ddb.py +++ b/python/python/tests/test_s3_ddb.py @@ -315,6 +315,29 @@ def test_file_writer_reader(s3_bucket: str): == global_buffer_text ) + # The writer reports the size of the object it just wrote, so callers do not + # need a second client to stat it. + s3 = get_boto3_client("s3", endpoint_url=CONFIG["aws_endpoint"]) + head = s3.head_object(Bucket=s3_bucket, Key="foo.lance") + assert writer.size_bytes == head["ContentLength"] + + +@pytest.mark.integration +def test_file_writer_size_bytes_multipart(s3_bucket: str): + # Large enough to exceed the single-PUT threshold, so the write goes through + # the multipart path where the reported size is filled in on completion. + storage_options = copy.deepcopy(CONFIG) + del storage_options["dynamodb_endpoint"] + table = pa.table({"a": pa.array(range(4 * 1024 * 1024), type=pa.int64())}) + file_path = f"s3://{s3_bucket}/multipart.lance" + with LanceFileWriter(str(file_path), storage_options=storage_options) as writer: + writer.write_batch(table) + + s3 = get_boto3_client("s3", endpoint_url=CONFIG["aws_endpoint"]) + head = s3.head_object(Bucket=s3_bucket, Key="multipart.lance") + assert head["ContentLength"] > 5 * 1024 * 1024 + assert writer.size_bytes == head["ContentLength"] + @pytest.mark.integration def test_file_session_upload_download(s3_bucket: str, tmp_path): diff --git a/python/python/tests/test_scalar_index.py b/python/python/tests/test_scalar_index.py index ae92abdd427..1007f8b3231 100644 --- a/python/python/tests/test_scalar_index.py +++ b/python/python/tests/test_scalar_index.py @@ -7,6 +7,8 @@ import re import shutil import string +import subprocess +import sys import uuid import zipfile from datetime import date, datetime, timedelta @@ -21,6 +23,8 @@ from lance.query import ( BooleanQuery, BoostQuery, + DocumentGranularity, + FullTextOperator, MatchQuery, MultiMatchQuery, Occur, @@ -624,13 +628,13 @@ def make_fts_search(ds): plan = make_vec_search(ds).explain_plan() assert "ScalarIndexQuery" in plan assert "KNNVectorDistance" not in plan - assert "LanceRead" not in plan + assert "num_fragments" not in plan # no scan; the take prints as LanceRead assert make_vec_search(ds).to_table().num_rows == 6 plan = make_fts_search(ds).explain_plan() assert "ScalarIndexQuery" in plan assert "KNNVectorDistance" not in plan - assert "LanceRead" not in plan + assert "num_fragments" not in plan # no scan; the take prints as LanceRead assert make_fts_search(ds).to_table().num_rows == 6 # Add new data (including 6 more results) @@ -806,6 +810,341 @@ def test_full_text_search(dataset, with_position, base_tokenizer): ) +@pytest.mark.parametrize("block_size", [128, 256]) +def test_code_analyzer_does_not_split_identifiers_by_default(tmp_path, block_size): + table = pa.table({"code": ["GetUserName", "GetUserEmail", "user"]}) + ds = lance.write_dataset(table, tmp_path) + ds.create_scalar_index( + "code", + index_type="INVERTED", + analyzer="code", + block_size=block_size, + ) + assert ds.describe_indices()[0].segments[0].index_version == 3 + + results = ds.to_table( + columns=["code"], + full_text_query=MatchQuery("user", "code"), + ) + assert results["code"].to_pylist() == ["user"] + + stats = ds.stats.index_stats("code_idx")["indices"][0] + params = stats["params"] + assert "analyzer" not in params + assert params["base_tokenizer"] == "code" + assert params["split_identifiers"] is False + + +def test_code_analyzer_full_text_search_with_identifier_splitting(tmp_path): + table = pa.table( + { + "code": [ + "getUserName", + "set_user_name", + "user-name", + "username", + "other", + ] + } + ) + ds = lance.write_dataset(table, tmp_path) + ds.create_scalar_index( + "code", + index_type="INVERTED", + analyzer="code", + split_identifiers=True, + ) + + results = ds.to_table( + columns=["code"], + full_text_query=MatchQuery("user", "code"), + ) + assert set(results["code"].to_pylist()) == { + "getUserName", + "set_user_name", + "user-name", + } + + stats = ds.stats.index_stats("code_idx")["indices"][0] + params = stats["params"] + assert "analyzer" not in params + assert params["base_tokenizer"] == "code" + assert params["split_identifiers"] is True + assert params["split_on_numerics"] is True + assert params["preserve_original"] is True + assert params["stem"] is False + assert params["remove_stop_words"] is False + + +def test_code_analyzer_operator_search_matches_rust_turbofish(tmp_path): + table = pa.table( + { + "path": ["turbofish.rs", "comparison.rs"], + "code": ["value.parse::()", "value.parse()"], + } + ) + ds = lance.write_dataset(table, tmp_path) + ds.create_scalar_index( + "code", + index_type="INVERTED", + analyzer="code", + index_operators=True, + ) + + results = ds.to_table( + columns=["path"], + full_text_query=MatchQuery("::", "code", operator=FullTextOperator.OR), + ) + assert results["path"].to_pylist() == ["turbofish.rs"] + + +def test_code_analyzer_exact_identifier_survives_grouped_top_k(tmp_path): + table = pa.table( + { + "path": ["split_0.rs", "split_1.rs", "split_2.rs", "exact.rs"], + "code": ["get user name", "get user name", "get user name", "getUserName"], + } + ) + ds = lance.write_dataset(table, tmp_path) + ds.create_scalar_index( + "code", + index_type="INVERTED", + analyzer="code", + split_identifiers=True, + ) + + results = ds.scanner( + columns=["path", "_score"], + full_text_query=MatchQuery( + "getUserName", "code", operator=FullTextOperator.AND + ), + limit=1, + ).to_table() + assert results["path"].to_pylist() == ["exact.rs"] + + +def test_code_analyzer_flags_require_code_analyzer(tmp_path): + table = pa.table({"text": ["getUserName"]}) + ds = lance.write_dataset(table, tmp_path) + + with pytest.raises(ValueError, match="code analyzer flags require analyzer='code'"): + ds.create_scalar_index( + "text", + index_type="INVERTED", + split_identifiers=True, + ) + + +def test_code_analyzer_requires_fts_v3(tmp_path): + table = pa.table({"code": ["getUserName"]}) + ds = lance.write_dataset(table, tmp_path) + + with pytest.raises(ValueError, match="requires FTS format_version=3"): + ds.create_scalar_index( + "code", + index_type="INVERTED", + analyzer="code", + format_version=2, + ) + + +def test_code_analyzer_complex_code_constructs(tmp_path): + table = pa.table( + { + "path": [ + "edge/trait.rs", + "edge/impl.rs", + "edge/fn_pointer.rs", + "edge/unit_result.rs", + "edge/hrtb.rs", + "edge/associated.rs", + "edge/operators.rs", + ], + "code": [ + """ +pub trait EdgeAsyncRepository<'a, T: Send + Sync> +where + T: TryFrom<&'a str, Error = EdgeParseError>, +{ + type Output<'b>: Iterator> + where + Self: 'b; + + async fn fetch_by_key( + &'a self, + key: [u8; N], + ) -> Result, EdgeRepoError>; +} +""", + """ +impl<'a, T, S> EdgeAsyncRepository<'a, T> for EdgeStore +where + T: TryFrom<&'a str, Error = EdgeParseError> + Clone + Send + Sync, + S: EdgeBackend + ?Sized, +{ + type Output<'b> = std::vec::IntoIter> where Self: 'b; + + async fn fetch_by_key( + &'a self, + key: [u8; N], + ) -> Result, EdgeRepoError> { + self.backend.fetch::(key).await + } +} +""", + """ +pub fn build_edge_handler( + factory: F, +) -> impl Fn() -> Result, EdgeError> +where + F: FnOnce() -> Result + Send + 'static, + T: Default + Send + Sync + 'static, +{ + move || factory().map(EdgeHandler::new) +} +""", + """ +pub fn edge_unit_result_callback() -> Result<()> { + Ok(()) +} +""", + """ +pub fn edge_higher_ranked<'a, T>( + visitor: impl for<'b> Fn(&'b T) -> Result<&'b str, EdgeVisitError>, + value: &'a T, +) -> Result<&'a str, EdgeVisitError> { + visitor(value) +} +""", + """ +pub fn edge_collect_stream(items: I) -> Result, E::Error> +where + I: IntoIterator, + E: EdgeExtract, +{ + items.into_iter().map(E::extract).collect() +} +""", + """ +pub fn edge_operator_arrow() -> Result { + let variant = EdgeModule::EdgeVariant; + if variant != EdgeModule::Default && EdgeMask::enabled() { + return Ok(EdgeArrow::new(variant)); + } + Err(EdgeError::empty()) +} +""", + ], + } + ) + table = table.append_column("code_ops", table["code"]) + ds = lance.write_dataset(table, tmp_path) + ds.create_scalar_index("code", index_type="INVERTED", analyzer="code") + ds.create_scalar_index( + "code_ops", + index_type="INVERTED", + analyzer="code", + index_operators=True, + ) + + ds.insert( + pa.table( + { + "path": ["edge/flat_unindexed.rs"], + "code": [ + """ +pub async fn edge_flat_generic_return() -> Result +where + T: TryFrom + Send, + E: Into, +{ + T::try_from(String::new()).map_err(Into::into) +} +""" + ], + "code_ops": [ + """ +pub async fn edge_flat_operator() -> Result { + EdgeFlat::try_new() -> Result +} +""" + ], + } + ) + ) + ds = lance.dataset(tmp_path) + + def assert_search(column, query, expected_path, operator=FullTextOperator.AND): + result = ds.scanner( + columns=["path", "_score"], + full_text_query=MatchQuery(query, column, operator=operator), + limit=50, + ).to_table() + assert expected_path in result["path"].to_pylist() + + assert_search( + "code", + "EdgeAsyncRepository fetch_by_key TryFrom EdgeRepoError", + "edge/trait.rs", + ) + assert_search( + "code", + "EdgeStore fetch_by_key const usize where Result", + "edge/impl.rs", + ) + assert_search( + "code", + "build_edge_handler FnOnce Result EdgeHandler", + "edge/fn_pointer.rs", + ) + assert_search( + "code", + "edge_unit_result_callback fn () -> Result", + "edge/unit_result.rs", + ) + assert_search( + "code", + "edge_higher_ranked for Fn EdgeVisitError Result", + "edge/hrtb.rs", + ) + assert_search( + "code", + "edge_collect_stream IntoIterator Item Error Result", + "edge/associated.rs", + ) + assert_search( + "code", + "edge_flat_generic_return TryFrom EdgeFlatError Result", + "edge/flat_unindexed.rs", + ) + assert_search( + "code_ops", + "edge_operator_arrow -> Result", + "edge/operators.rs", + ) + assert_search( + "code_ops", + "EdgeModule :: EdgeVariant !=", + "edge/operators.rs", + ) + assert_search( + "code_ops", + "edge_flat_operator -> Result EdgeFlatError", + "edge/flat_unindexed.rs", + ) + + default_operator_results = ds.scanner( + columns=["path", "_score"], + full_text_query=MatchQuery("->", "code", operator=FullTextOperator.OR), + ).to_table() + operator_results = ds.scanner( + columns=["path", "_score"], + full_text_query=MatchQuery("->", "code_ops", operator=FullTextOperator.OR), + ).to_table() + assert default_operator_results.num_rows == 0 + assert operator_results.num_rows > 0 + + def test_unindexed_full_text_search_on_empty_index(tmp_path): # Create fts index on empty table. schema = pa.schema({"text": pa.string()}) @@ -949,6 +1288,39 @@ def test_create_scalar_index_fts_alias(dataset): assert any(idx.index_type == "Inverted" for idx in dataset.describe_indices()) +def test_create_scalar_index_fts_block_size(dataset): + dataset.create_scalar_index( + "doc", index_type="INVERTED", with_position=False, block_size=256 + ) + indices = dataset.describe_indices() + doc_index = next(index for index in indices if index.name == "doc_idx") + assert doc_index.segments[0].index_version == 3 + + row = dataset.take(indices=[0], columns=["doc"]) + query = row.column(0)[0].as_py().split(" ")[0] + results = dataset.scanner(columns=["doc"], full_text_query=query).to_table() + assert results.num_rows > 0 + + with pytest.raises(ValueError, match="block_size"): + dataset.create_scalar_index( + "doc", index_type="INVERTED", name="doc_invalid_129", block_size=129 + ) + + with pytest.raises(ValueError, match="block_size"): + dataset.create_scalar_index( + "doc", index_type="INVERTED", name="doc_invalid_512", block_size=512 + ) + + with pytest.raises(ValueError, match="block_size=256"): + dataset.create_scalar_index( + "doc", + index_type="INVERTED", + name="doc_invalid_v2_256", + block_size=256, + format_version=2, + ) + + def test_multi_index_create(tmp_path): dataset = lance.write_dataset( pa.table({"ints": range(1024)}), tmp_path, max_rows_per_file=100 @@ -1318,6 +1690,136 @@ def test_fts_on_list(tmp_path): assert results.num_rows == 6 +@pytest.mark.parametrize( + "list_type", + [ + pa.list_(pa.string()), + pa.list_(pa.large_string()), + pa.large_list(pa.string()), + pa.large_list(pa.large_string()), + ], +) +def test_fts_on_list_elements(tmp_path, list_type): + data = pa.table( + { + "id": pa.array([0, 1, 2]), + "tags": pa.array( + [ + ["alpha beta", "gamma alpha", None, "", "delta"], + ["beta", "gamma"], + None, + ], + type=list_type, + ), + } + ) + ds = lance.write_dataset(data, tmp_path) + + def hits(table): + return sorted(zip(table["id"].to_pylist(), table["_doc_index"].to_pylist())) + + list_element = DocumentGranularity.LIST_ELEMENT + query = MatchQuery("alpha", "tags", document_granularity=list_element) + flat = ds.to_table(full_text_query=query) + assert hits(flat) == [(0, [0]), (0, [1])] + assert pa.types.is_list(flat.schema.field("_doc_index").type) + assert flat.schema.field("_doc_index").type.value_type == pa.uint32() + assert hits( + ds.to_table( + full_text_query=MatchQuery( + "delta", "tags", document_granularity=list_element + ) + ) + ) == [(0, [4])] + + ds.create_scalar_index( + "tags", + "INVERTED", + with_position=True, + document_granularity=list_element, + ) + ds.create_scalar_index( + "tags", + IndexConfig(index_type="inverted", parameters={"with_position": True}), + document_granularity=list_element, + ) + inferred = ds.to_table(full_text_query=MatchQuery("alpha", "tags")) + assert hits(inferred) == [(0, [0]), (0, [1])] + with pytest.raises(ValueError, match=r"requested Row.*ListElement"): + ds.to_table( + full_text_query=MatchQuery( + "alpha", "tags", document_granularity=DocumentGranularity.ROW + ) + ) + + ds.create_scalar_index("tags", "INVERTED", with_position=True) + index_names = {index.name for index in ds.describe_indices()} + assert {"tags_idx", "tags_list_element_idx"}.issubset(index_names) + row_auto = ds.to_table(full_text_query="alpha") + assert row_auto["id"].to_pylist() == [0] + assert "_doc_index" not in row_auto.column_names + with pytest.raises(ValueError, match=r"ambiguous.*document_granularity"): + ds.to_table(full_text_query=MatchQuery("alpha", "tags")) + indexed = ds.to_table(full_text_query=query) + assert hits(indexed) == [(0, [0]), (0, [1])] + assert pa.types.is_list(indexed.schema.field("_doc_index").type) + assert indexed.schema.field("_doc_index").type.value_type == pa.uint32() + filtered = ds.to_table(full_text_query=query, filter="id = 0", prefilter=True) + assert hits(filtered) == [(0, [0]), (0, [1])] + assert hits( + ds.to_table( + full_text_query=MatchQuery( + "delta", "tags", document_granularity=list_element + ) + ) + ) == [(0, [4])] + + phrase = ds.to_table( + full_text_query=PhraseQuery( + "beta gamma", "tags", document_granularity=list_element + ) + ) + assert phrase.num_rows == 0 + assert hits( + ds.to_table( + full_text_query=PhraseQuery( + "alpha beta", "tags", document_granularity=list_element + ) + ) + ) == [(0, [0])] + row_phrase = ds.to_table( + full_text_query=PhraseQuery( + "beta gamma", + "tags", + document_granularity=DocumentGranularity.ROW, + ) + ) + assert sorted(row_phrase["id"].to_pylist()) == [0, 1] + assert "_doc_index" not in row_phrase.column_names + + ds.insert( + pa.table( + { + "id": pa.array([3]), + "tags": pa.array([["alpha", "alpha again"]], type=list_type), + } + ) + ) + assert hits(ds.to_table(full_text_query=query)) == [ + (0, [0]), + (0, [1]), + (3, [0]), + (3, [1]), + ] + + with pytest.raises(RuntimeError, match=r"tags\[\*\]"): + ds.create_scalar_index( + "tags[*]", + "INVERTED", + document_granularity=list_element, + ) + + def test_fts_fuzzy_query(tmp_path): data = pa.table( { @@ -2165,6 +2667,87 @@ def scan_stats_callback(stats: lance.ScanStatistics): assert small_bytes_read < large_bytes_read +@pytest.mark.parametrize("index_type", ["ZONEMAP", "BLOOMFILTER"]) +def test_address_domain_index_with_stable_row_ids(tmp_path: Path, index_type): + """Regression test for issue #7434. + + Address-domain scalar indices (zonemap, bloom filter) report matches as + physical row addresses. On a stable-row-id dataset a row's stable id differs + from its physical address in every fragment except fragment 0, so the index + result must be translated back to the row-id domain before it prefilters the + scan. Without that translation the index silently drops matching rows in + fragments other than fragment 0 (often returning an empty result). + """ + + # A single value "a" is placed in non-adjacent fragments (0, 2, 4) so its + # matching rows span multiple fragments and diverge from fragment 0. + def block(v, n): + return [v] * n + + vals = ( + block("a", 5_000) + + block("b", 5_000) + + block("a", 5_000) + + block("c", 5_000) + + block("a", 5_000) + ) + tbl = pa.table({"category": pa.array(vals, pa.string()), "id": range(len(vals))}) + ds = lance.write_dataset( + tbl, tmp_path, max_rows_per_file=5_000, enable_stable_row_ids=True + ) + assert ds.has_stable_row_ids + assert len(ds.get_fragments()) == 5 + + true_count = ds.scanner( + filter="category = 'a'", use_scalar_index=False + ).count_rows() + assert true_count == 15_000 + + ds.create_scalar_index("category", index_type=index_type, replace=True) + + # The index must be consulted and must return the same rows as a full scan. + assert "ScalarIndexQuery" in ds.scanner(filter="category = 'a'").explain_plan() + indexed = ds.to_table(filter="category = 'a'", columns=["id"]) + assert indexed.num_rows == true_count + assert sorted(indexed["id"].to_pylist()) == sorted( + ds.to_table(filter="category = 'a'", columns=["id"], use_scalar_index=False)[ + "id" + ].to_pylist() + ) + + +def test_zonemap_with_stable_row_ids_after_compaction(tmp_path: Path): + """Zonemap results stay correct after a compaction relocates rows under + stable row ids (physical address != stable id for the surviving rows).""" + ds = lance.write_dataset( + pa.table({"x": range(0, 5_000)}), + tmp_path, + max_rows_per_file=5_000, + enable_stable_row_ids=True, + ) + ds = lance.write_dataset( + pa.table({"x": range(5_000, 10_000)}), + tmp_path, + mode="append", + max_rows_per_file=5_000, + enable_stable_row_ids=True, + ) + # Delete part of the first fragment, then compact so surviving rows keep + # their small stable ids but move to a freshly numbered fragment. + ds.delete("x >= 1000 AND x < 2000") + ds.optimize.compact_files(target_rows_per_fragment=100_000) + ds = lance.dataset(tmp_path) + assert len(ds.get_fragments()) == 1 + + ds.create_scalar_index("x", index_type="ZONEMAP") + + filter_expr = "x >= 6000 AND x <= 6500" + assert "ScalarIndexQuery" in ds.scanner(filter=filter_expr).explain_plan() + expected = ds.to_table(filter=filter_expr, use_scalar_index=False)["x"].to_pylist() + actual = ds.to_table(filter=filter_expr)["x"].to_pylist() + assert sorted(actual) == sorted(expected) == list(range(6000, 6501)) + + def test_zonemap_deletion_handling(tmp_path: Path): """Test zonemap deletion handling""" data = pa.table( @@ -2194,6 +2777,33 @@ def test_zonemap_deletion_handling(tmp_path: Path): assert ids == [0, 2, 4, 6, 8] +@pytest.mark.parametrize("index_type", ["ZONEMAP", "BLOOMFILTER"]) +def test_address_domain_index_not_query_with_stable_row_ids(tmp_path: Path, index_type): + """Regression test: != queries return correct results on stable-row-id datasets. + + Address-domain indices (zonemap, bloom filter) search returns physical row + addresses. Translation to row IDs happens at the Query leaf before the NOT + node is evaluated, so the NOT operates on a correctly translated AllowList. + Without the address-to-row-id translation the AllowList contains wrong IDs, + and the subsequent NOT excludes the wrong rows. + """ + vals = list(range(5_000)) + list(range(5_000, 10_000)) + tbl = pa.table({"x": vals}) + ds = lance.write_dataset( + tbl, tmp_path, max_rows_per_file=5_000, enable_stable_row_ids=True + ) + assert ds.has_stable_row_ids + + ds.create_scalar_index("x", index_type=index_type, replace=True) + + # Without address translation the NOT excludes the wrong rows, producing an + # incorrect result set rather than crashing. + expected = ds.to_table(filter="x != 42", use_scalar_index=False)["x"].to_pylist() + actual = ds.to_table(filter="x != 42")["x"].to_pylist() + assert sorted(actual) == sorted(expected) + assert len(actual) == 9_999 + + def test_zonemap_index_remapping(tmp_path: Path): """Test zonemap index remapping after compaction and optimization""" # Create a dataset with 5 fragments by writing data in chunks @@ -2250,6 +2860,54 @@ def test_zonemap_index_remapping(tmp_path: Path): assert result.num_rows == 501 # 1000..1500 inclusive +def test_zonemap_fsl_column(tmp_path: Path): + """Zone map can be created on a FixedSizeList column and accelerates IS NULL.""" + dim = 8 + n = 1000 + rng = np.random.default_rng(42) + vectors = rng.standard_normal((n, dim)).astype(np.float32) + vec_type = pa.list_(pa.float32(), dim) + # Every 10th row is null + vec_list = [None if i % 10 == 0 else v.tolist() for i, v in enumerate(vectors)] + tbl = pa.table({"vec": pa.array(vec_list, type=vec_type), "id": pa.array(range(n))}) + ds = lance.write_dataset(tbl, tmp_path) + ds.create_scalar_index("vec", index_type="ZONEMAP") + + scanner = ds.scanner(filter="vec IS NULL", prefilter=True) + plan = scanner.explain_plan() + assert "ScalarIndexQuery" in plan + result = scanner.to_table() + assert result.num_rows == 100 # every 10th row is null + + +def test_vector_and_zonemap_on_fsl_column(tmp_path: Path): + """Vector index and zone map can coexist on the same FSL column.""" + dim = 16 + n = 2000 + rng = np.random.default_rng(0) + vectors = rng.standard_normal((n, dim)).astype(np.float32) + vec_type = pa.list_(pa.float32(), dim) + # Every 20th row is null + vec_list = [None if i % 20 == 0 else v.tolist() for i, v in enumerate(vectors)] + tbl = pa.table({"vec": pa.array(vec_list, type=vec_type), "id": pa.array(range(n))}) + ds = lance.write_dataset(tbl, tmp_path) + + ds.create_index("vec", index_type="IVF_PQ", num_partitions=4, num_sub_vectors=2) + ds.create_scalar_index("vec", index_type="ZONEMAP") + + # Vector search still works + query = vectors[5] + result = ds.scanner(nearest={"column": "vec", "q": query, "k": 10}).to_table() + assert result.num_rows == 10 + + # IS NULL is zone-map-accelerated + scanner = ds.scanner(filter="vec IS NULL", prefilter=True) + plan = scanner.explain_plan() + assert "ScalarIndexQuery" in plan + null_result = scanner.to_table() + assert null_result.num_rows == 100 # every 20th row is null + + def test_bloomfilter_index(tmp_path: Path): """Test create bloomfilter index""" tbl = pa.Table.from_arrays([pa.array([i for i in range(10000)])], names=["values"]) @@ -2332,6 +2990,36 @@ def test_json_index(): ) +def test_json_index_non_exact_floats(): + # A JSON-path btree index is trained on the value extracted from the JSON + # column, not on the raw column itself, so the index build must sort by + # the extracted value rather than assuming the raw column's order matches + # it. Without that, range/equality queries silently miss rows whenever + # the extracted floats are not exactly representable in float64 (#7485). + vals = ['{"latitude": 10.5}', '{"latitude": 40.1}', '{"latitude": -3.2}'] + tbl = pa.table({"data": pa.array(vals, pa.json_())}) + ds = lance.write_dataset(tbl, "memory://test") + ds.create_scalar_index( + "data", + IndexConfig( + index_type="json", + parameters={"target_index_type": "btree", "path": "latitude"}, + ), + ) + + for filter in [ + "json_get_float(data, 'latitude') > 0", + "json_get_float(data, 'latitude') >= 10.5", + "json_get_float(data, 'latitude') = 40.1", + "json_get_float(data, 'latitude') = 10.5", + "json_get_float(data, 'latitude') < 100", + ]: + assert "ScalarIndexQuery" in ds.scanner(filter=filter).explain_plan() + assert ds.to_table(filter=filter) == ds.to_table( + filter=filter, use_scalar_index=False + ), filter + + def test_null_handling(): tbl = pa.table( { @@ -2793,6 +3481,15 @@ def scan_stats_callback(stats: lance.ScanStatistics): cache_entries_after_query = ds._ds.index_cache_entry_count() assert cache_entries_after_query == cache_entries_after_prewarm + segment_uuid = ds.describe_indices()[0].segments[0].uuid + ds = lance.dataset(phrase_path) + ds.prewarm_index("fts_idx", with_position=True, index_segments=[segment_uuid]) + cache_entries_after_prewarm = ds._ds.index_cache_entry_count() + results = ds.to_table(full_text_query=PhraseQuery("word word", "fts")) + assert results.num_rows == test_table_size + cache_entries_after_query = ds._ds.index_cache_entry_count() + assert cache_entries_after_query == cache_entries_after_prewarm + with pytest.raises( TypeError, match="takes 2 positional arguments", @@ -2828,6 +3525,166 @@ def scan_stats_callback(stats: lance.ScanStatistics): assert scan_stats.parts_loaded == 0 +def test_btree_index_cache_hit_miss_stats(tmp_path: Path): + """Cold scan reports index cache misses; warm scan reports hits. + + ScanStatistics.index_cache_{hits,misses} are populated at page-level cache + boundaries. On a freshly-loaded dataset the BTree page fetch must be a + miss; a second scan against the same in-memory Dataset re-uses the cached + page and therefore reports a hit with zero misses. + """ + scan_stats = None + + def scan_stats_callback(stats: lance.ScanStatistics): + nonlocal scan_stats + scan_stats = stats + + test_table = pa.table({"val": list(range(1000))}) + ds = lance.write_dataset(test_table, tmp_path) + ds.create_scalar_index("val", index_type="BTREE") + + # Reopen so the session cache starts cold. A single-key point lookup on + # a small dataset resolves to exactly one BTree page, so cold/warm counts + # are deterministic 1/0 and 0/1. + ds = lance.dataset(tmp_path) + ds.scanner(filter="val = 42", scan_stats_callback=scan_stats_callback).to_table() + assert scan_stats is not None + assert scan_stats.index_cache_misses == 1 + assert scan_stats.index_cache_hits == 0 + + # Same Dataset, warm cache — no new page loads, only hits. + ds.scanner(filter="val = 42", scan_stats_callback=scan_stats_callback).to_table() + assert scan_stats.index_cache_hits == 1 + assert scan_stats.index_cache_misses == 0 + + +def test_bitmap_index_cache_hit_miss_stats(tmp_path: Path): + """Bitmap Range/IN queries report cold misses and warm hits; a value + that is not in the index never reaches the loader and must not count. + + Guards against the ``BitmapIndex::search`` regressions where the + ``Range`` / ``IsIn`` branches used to drop the ``MetricsCollector`` (so + every lookup was silently ``0/0``), and where an equality on a value + absent from ``index_map`` recorded a spurious miss before short-circuiting + to the empty result. + """ + scan_stats = None + + def scan_stats_callback(stats: lance.ScanStatistics): + nonlocal scan_stats + scan_stats = stats + + test_table = pa.table({"color": ["red", "green", "blue", "yellow"] * 25}) + ds = lance.write_dataset(test_table, tmp_path) + ds.create_scalar_index("color", index_type="BITMAP") + + # Reopen so the session cache starts cold. + ds = lance.dataset(tmp_path) + ds.scanner( + filter="color IN ('red', 'blue')", scan_stats_callback=scan_stats_callback + ).to_table() + assert scan_stats is not None + assert scan_stats.index_cache_misses == 2 + assert scan_stats.index_cache_hits == 0 + + ds.scanner( + filter="color IN ('red', 'blue')", scan_stats_callback=scan_stats_callback + ).to_table() + assert scan_stats.index_cache_hits == 2 + assert scan_stats.index_cache_misses == 0 + + # A value that is not in the index short-circuits before the loader and + # must not touch either counter. + ds.scanner( + filter="color = 'purple'", scan_stats_callback=scan_stats_callback + ).to_table() + assert scan_stats.index_cache_hits == 0 + assert scan_stats.index_cache_misses == 0 + + +def test_phrase_query_cache_hit_miss_stats(tmp_path: Path): + """Phrase-query fallback populates ``PositionKey``; that boundary must + show up in per-query cache statistics. + + Guards against ``read_positions`` silently using the non-metric + ``get_or_insert_with_key`` API — before this fix, a warm phrase query + would report zero hits for the phrase-position cache slot even though + the loader was skipped. + + The cold path can already record a few hits (``bm25_stats_for_terms`` + populates ``PostingMetadataKey``, which is then re-read on the + posting-list path as a cross-boundary hit), so the cold assertion is + ``misses > hits`` rather than a strict zero. + """ + scan_stats = None + + def scan_stats_callback(stats: lance.ScanStatistics): + nonlocal scan_stats + scan_stats = stats + + test_table = pa.table( + {"text": ["quick brown fox jumps over lazy dog" for _ in range(50)]} + ) + ds = lance.write_dataset(test_table, tmp_path) + ds.create_scalar_index("text", index_type="INVERTED", with_position=True) + + ds = lance.dataset(tmp_path) + ds.scanner( + scan_stats_callback=scan_stats_callback, + full_text_query='"quick brown"', + ).to_table() + assert scan_stats is not None + assert scan_stats.index_cache_misses > 0 + assert scan_stats.index_cache_misses > scan_stats.index_cache_hits + + ds.scanner( + scan_stats_callback=scan_stats_callback, + full_text_query='"quick brown"', + ).to_table() + assert scan_stats.index_cache_hits > 0 + assert scan_stats.index_cache_misses == 0 + + +def test_fts_index_cache_hit_miss_stats(tmp_path: Path): + """Cold FTS scan reports misses; warm FTS scan reports hits. + + Guards the wrapper-forwarding fix in ``FtsIndexMetrics``: previously the + two new cache-hit/miss trait methods had default no-op implementations + that swallowed FTS-side events, so cache activity was reported as ``0/0`` + even for hot inverted-index scans. + + The cold path can still record a few hits when the same cache key is + read across boundaries in one query (e.g. ``bm25_stats_for_terms`` + populates ``PostingMetadataKey`` before ``posting_list`` re-reads it), + so the cold assertion is ``misses > hits`` rather than a strict zero. + """ + scan_stats = None + + def scan_stats_callback(stats: lance.ScanStatistics): + nonlocal scan_stats + scan_stats = stats + + test_table = pa.table({"fts": ["word" for _ in range(100)]}) + ds = lance.write_dataset(test_table, tmp_path) + ds.create_scalar_index("fts", index_type="INVERTED") + + # Reopen so the session cache starts cold. + ds = lance.dataset(tmp_path) + ds.scanner( + scan_stats_callback=scan_stats_callback, full_text_query="word" + ).to_table() + assert scan_stats is not None + assert scan_stats.index_cache_misses > 0 + assert scan_stats.index_cache_misses > scan_stats.index_cache_hits + + # Same Dataset, warm cache — posting-list / metadata reads must now hit. + ds.scanner( + scan_stats_callback=scan_stats_callback, full_text_query="word" + ).to_table() + assert scan_stats.index_cache_hits > 0 + assert scan_stats.index_cache_misses == 0 + + def test_fts_backward_v0_27_0(tmp_path: Path): path = ( Path(__file__).parent.parent.parent.parent @@ -2853,6 +3710,14 @@ def test_fts_backward_v0_27_0(tmp_path: Path): "frodo was a happy puppy", } + # Requiring both disjoint terms advances "happy" past its final document while + # "tail" remains live. Legacy WAND must terminate without reading the exhausted + # posting. + results = ds.to_table( + full_text_query=MatchQuery("happy tail", "text", operator=FullTextOperator.AND) + ) + assert results.num_rows == 0 + data = pa.table( { "text": [ @@ -4151,6 +5016,126 @@ def test_bitmap_uncommitted_segments_can_be_committed_from_python(tmp_path): ) +def test_ngram_segment_merge_and_commit_from_python(tmp_path): + ds = lance.write_dataset( + pa.table( + { + "text": [ + "alpha needle", + None, + "beta needle", + "gamma stack", + "delta needle", + "", + ] + } + ), + tmp_path, + max_rows_per_file=2, + ) + index_name = "text_ngram_segments" + fragment_ids = [fragment.fragment_id for fragment in ds.get_fragments()] + staged_segments = [ + ds.create_index_uncommitted( + column="text", + index_type="NGRAM", + name=index_name, + fragment_ids=[fragment_id], + ) + for fragment_id in fragment_ids + ] + source_version = staged_segments[0].dataset_version + + for segment, fragment_id in zip(staged_segments, fragment_ids): + assert segment.fragment_ids == {fragment_id} + assert any(file.path == "ngram_postings.lance" for file in segment.files) + + merged_segment = ds.merge_existing_index_segments(staged_segments) + assert merged_segment.dataset_version == source_version + assert merged_segment.fragment_ids == set(fragment_ids) + assert any(file.path == "ngram_postings.lance" for file in merged_segment.files) + + ds.insert(pa.table({"text": ["new stack"]})) + assert ds.version > source_version + ds = ds.commit_existing_index_segments(index_name, "text", [merged_segment]) + descriptions = {index.name: index for index in ds.describe_indices()} + assert descriptions[index_name].index_type == "NGram" + assert len(descriptions[index_name].segments) == 1 + assert ( + descriptions[index_name].segments[0].dataset_version_at_last_update + == source_version + ) + assert ds.count_rows("contains(text, 'needle')") == 3 + assert ds.count_rows("text IS NULL") == 1 + + +@pytest.mark.parametrize( + "label_type", + [pa.list_(pa.string()), pa.large_list(pa.string())], + ids=["list", "large_list"], +) +def test_label_list_segment_index(tmp_path, label_type): + rows_per_fragment = 8 + ds = lance.write_dataset( + pa.table( + { + "id": pa.array(range(rows_per_fragment * 4), type=pa.int32()), + "labels": pa.array( + [ + ["distributed"] if row_id % 2 == 0 else ["other"] + for row_id in range(rows_per_fragment * 4) + ], + type=label_type, + ), + } + ), + tmp_path, + max_rows_per_file=rows_per_fragment, + ) + + fragment_ids = [fragment.fragment_id for fragment in ds.get_fragments()] + assert len(fragment_ids) == 4 + + with pytest.raises(ValueError, match="create_index_uncommitted"): + ds.create_scalar_index( + column="labels", + index_type="LABEL_LIST", + fragment_ids=[fragment_ids[0]], + ) + + index_name = "labels_segment_idx" + segments = [ + ds.create_index_uncommitted( + column="labels", + index_type="LABEL_LIST", + name=index_name, + fragment_ids=[fragment_id], + ) + for fragment_id in fragment_ids + ] + + merged_segment = ds.merge_existing_index_segments(segments) + ds = ds.commit_existing_index_segments(index_name, "labels", [merged_segment]) + + filter_expr = "array_has_any(labels, ['distributed'])" + without_index = ds.scanner( + filter=filter_expr, + columns=["id", "labels"], + use_scalar_index=False, + ).to_table() + with_index = ds.scanner( + filter=filter_expr, + columns=["id", "labels"], + use_scalar_index=True, + ).to_table() + + assert with_index.equals(without_index) + assert ( + "ScalarIndexQuery" + in ds.scanner(filter=filter_expr, use_scalar_index=True).explain_plan() + ) + + def test_zonemap_fragment_ids_parameter_validation(tmp_path): ds = generate_multi_fragment_dataset( tmp_path, num_fragments=2, rows_per_fragment=100 @@ -4224,6 +5209,54 @@ def test_zonemap_segment_merge_and_commit_from_python(tmp_path): ) +def test_bloomfilter_segment_merge_and_commit_from_python(tmp_path): + ds = generate_multi_fragment_dataset( + tmp_path, num_fragments=3, rows_per_fragment=100 + ) + + index_name = "id_bloomfilter_segments" + fragment_ids = [fragment.fragment_id for fragment in ds.get_fragments()] + staged_segments = [ + ds.create_index_uncommitted( + column="id", + index_type="BLOOMFILTER", + name=index_name, + fragment_ids=[fragment_id], + ) + for fragment_id in fragment_ids + ] + + for segment, fragment_id in zip(staged_segments, fragment_ids): + assert segment.fragment_ids == {fragment_id} + assert any(file.path == "bloomfilter.lance" for file in segment.files) + + merged_segment = ds.merge_existing_index_segments(staged_segments) + assert merged_segment.fragment_ids == set(fragment_ids) + assert any(file.path == "bloomfilter.lance" for file in merged_segment.files) + + ds = ds.commit_existing_index_segments(index_name, "id", [merged_segment]) + descriptions = {index.name: index for index in ds.describe_indices()} + assert descriptions[index_name].index_type == "BloomFilter" + assert len(descriptions[index_name].segments) == 1 + + filter_expr = "id = 117" + without_index = ds.scanner( + filter=filter_expr, + columns=["id", "text"], + use_scalar_index=False, + ).to_table() + with_index = ds.scanner( + filter=filter_expr, + columns=["id", "text"], + use_scalar_index=True, + ).to_table() + assert with_index.to_pydict() == without_index.to_pydict() + assert ( + "ScalarIndexQuery" + in ds.scanner(filter=filter_expr, use_scalar_index=True).explain_plan() + ) + + def test_merge_index_metadata_btree_soft_break(tmp_path): ds = generate_multi_fragment_dataset( tmp_path, num_fragments=2, rows_per_fragment=100 @@ -4719,6 +5752,77 @@ def test_nested_field_fts_index(tmp_path): assert results.num_rows == 50 +def test_multiple_nested_field_fts_indices_e2e(tmp_path): + """Test FTS queries against multiple indexed nested string fields.""" + + def make_table(ids, text_values, summary_values): + return pa.table( + { + "id": ids, + "data": pa.StructArray.from_arrays( + [ + pa.array(text_values, type=pa.string()), + pa.array(summary_values, type=pa.string()), + ], + names=["text", "summary"], + ), + } + ) + + def result_ids(query): + return sorted(ds.to_table(full_text_query=query)["id"].to_pylist()) + + ds = lance.write_dataset( + make_table( + [0, 1, 2, 3], + [ + "lance nested alpha", + "plain text", + None, + "phrase target here", + ], + [ + "metadata only", + "database nested beta", + "lance beta", + "other", + ], + ), + tmp_path, + ) + + ds.create_scalar_index("data.text", index_type="INVERTED", with_position=True) + ds.create_scalar_index("data.summary", index_type="INVERTED", with_position=False) + + indexed_fields = { + tuple(index.field_names) + for index in ds.describe_indices() + if index.index_type == "Inverted" + } + assert indexed_fields == {("data.text",), ("data.summary",)} + + assert result_ids(MatchQuery("alpha", "data.text")) == [0] + assert result_ids(MatchQuery("beta", "data.summary")) == [1, 2] + assert result_ids("lance") == [0, 2] + assert result_ids(MultiMatchQuery("nested", ["data.text", "data.summary"])) == [ + 0, + 1, + ] + assert result_ids(PhraseQuery("phrase target", "data.text")) == [3] + + ds = lance.write_dataset( + make_table( + [4, 5], + ["fresh lance append", "plain append"], + ["other", "fresh beta append"], + ), + tmp_path, + mode="append", + ) + + assert result_ids("fresh") == [4, 5] + + def test_nested_field_bitmap_index(tmp_path): """Test BITMAP index creation and querying on nested fields""" # Create dataset with nested categorical field @@ -4853,7 +5957,7 @@ def test_json_inverted_match_query(tmp_path): @pytest.mark.parametrize( ("format_version", "expected_format_version"), - [(1, 1), (2, 2), ("v1", 1), ("v2", 2)], + [(1, 1), (2, 2), (3, 3), ("v1", 1), ("v2", 2), ("v3", 3)], ) def test_describe_indices(tmp_path, format_version, expected_format_version): data = pa.table( @@ -4903,7 +6007,6 @@ def test_describe_indices(tmp_path, format_version, expected_format_version): assert details["lower_case"] assert details["stem"] assert details["remove_stop_words"] - assert details["custom_stop_words"] is None assert details["ascii_folding"] assert details["min_ngram_length"] == 3 assert details["max_ngram_length"] == 3 @@ -4984,15 +6087,92 @@ def test_describe_indices(tmp_path, format_version, expected_format_version): assert index.num_rows_indexed == 50 -def test_create_inverted_index_defaults_to_v2_and_ignores_env(tmp_path, monkeypatch): - monkeypatch.setenv("LANCE_FTS_FORMAT_VERSION", "1") - data = pa.table({"text": ["document about lance database"]}) - ds = lance.write_dataset(data, tmp_path) +def _run_fts_format_creation_probe( + tmp_path, env_value, creation_options=None, expected_format_version=None +): + script = """ +import json +import sys - ds.create_scalar_index("text", index_type="INVERTED") +import lance +import pyarrow as pa - indices = ds.describe_indices() - assert indices[0].segments[0].index_version == 2 +dataset = lance.write_dataset( + pa.table({"text": ["document about lance database"]}), sys.argv[1] +) +dataset.create_scalar_index( + "text", index_type="INVERTED", **json.loads(sys.argv[2]) +) +expected_format_version = json.loads(sys.argv[3]) +if expected_format_version is not None: + actual_format_version = dataset.describe_indices()[0].segments[0].index_version + assert actual_format_version == expected_format_version +""" + env = os.environ.copy() + if env_value is None: + env.pop("LANCE_FTS_FORMAT_VERSION", None) + else: + env["LANCE_FTS_FORMAT_VERSION"] = env_value + return subprocess.run( + [ + sys.executable, + "-c", + script, + str(tmp_path), + json.dumps(creation_options or {}), + json.dumps(expected_format_version), + ], + capture_output=True, + env=env, + text=True, + ) + + +@pytest.mark.parametrize( + ("env_value", "creation_options", "expected_format_version"), + [ + ("1", {}, 1), + ("2", {}, 2), + ("3", {}, 3), + ("3", {"block_size": 256}, 3), + ], +) +def test_create_inverted_index_uses_env_format_version( + tmp_path, env_value, creation_options, expected_format_version +): + result = _run_fts_format_creation_probe( + tmp_path, + env_value, + creation_options, + expected_format_version, + ) + + assert result.returncode == 0, result.stderr + + +def test_create_inverted_index_explicit_format_version_overrides_env(tmp_path): + result = _run_fts_format_creation_probe( + tmp_path, + "invalid", + {"format_version": 1}, + 1, + ) + + assert result.returncode == 0, result.stderr + + +def test_create_text_inverted_index_defaults_to_v2_without_env(tmp_path): + result = _run_fts_format_creation_probe(tmp_path, None, expected_format_version=2) + + assert result.returncode == 0, result.stderr + + +def test_create_inverted_index_rejects_invalid_env_format_version(tmp_path): + result = _run_fts_format_creation_probe(tmp_path, "invalid") + + assert result.returncode != 0 + assert "LANCE_FTS_FORMAT_VERSION" in result.stderr + assert "invalid" in result.stderr def test_create_inverted_index_rejects_invalid_format_version(tmp_path): @@ -5000,7 +6180,10 @@ def test_create_inverted_index_rejects_invalid_format_version(tmp_path): ds = lance.write_dataset(data, tmp_path) with pytest.raises(ValueError, match="unsupported FTS format version"): - ds.create_scalar_index("text", index_type="INVERTED", format_version="v3") + ds.create_scalar_index("text", index_type="INVERTED", format_version="v5") + + with pytest.raises(ValueError, match="unsupported FTS format version"): + ds.create_scalar_index("text", index_type="INVERTED", format_version="v4") def test_vector_filter_fts_search(tmp_path): diff --git a/python/python/tests/test_schema.py b/python/python/tests/test_schema.py index fcff283ebe2..c384466082f 100644 --- a/python/python/tests/test_schema.py +++ b/python/python/tests/test_schema.py @@ -6,6 +6,7 @@ import lance import pyarrow as pa +import pytest from lance.schema import LanceSchema @@ -60,3 +61,15 @@ def test_lance_schema(tmp_path: Path): s_fields = fields[1].children() assert s_fields[0].name() == "new_name" assert s_fields[0].id() == 2 + + +def test_lance_schema_from_protos_rejects_missing_parent(): + # name (field 2): child; id (field 3): 7; parent_id (field 4): 42; + # logical_type (field 5): int32. + field_proto = b"\x12\x05child\x18\x07\x20\x2a\x2a\x05int32" + + with pytest.raises( + ValueError, + match="Field 'child' \\(id=7\\) references parent id 42", + ): + LanceSchema._from_protos("{}", field_proto) diff --git a/python/python/tests/test_schema_evolution.py b/python/python/tests/test_schema_evolution.py index 7df6962789e..abd89e0cded 100644 --- a/python/python/tests/test_schema_evolution.py +++ b/python/python/tests/test_schema_evolution.py @@ -15,6 +15,7 @@ import pytest from lance import LanceDataset from lance.file import LanceFileReader, LanceFileWriter +from lance.fragment import write_fragments def test_drop_columns(tmp_path: Path): @@ -573,3 +574,56 @@ def test_add_cols_all_null_with_sql(tmp_path: Path): "b": pa.int32(), } ) + + +def test_merge_nullability_assertion(tmp_path: Path): + tbl = pa.table({"value": pa.array([1, 2], pa.int32())}) + lance.write_dataset(tbl, tmp_path) + written_at = lance.dataset(tmp_path).version + + # Stage an append against the original schema, then add a non-nullable + # column. The merge claims non-null, and reading it back must preserve + # the claim, or recommitting it would silently drop the barrier. + fragments = write_fragments( + pa.table({"value": pa.array([7], pa.int32())}), tmp_path, mode="append" + ) + lance.dataset(tmp_path).add_columns({"one": "1"}) + txn = lance.dataset(tmp_path).read_transaction(2) + assert txn is not None + assert txn.operation.preserves_nullability is False + + # The stale append omits the required column, so its rows would read as + # null under the merged schema; the claim refuses it. + op = lance.LanceOperation.Append(fragments) + with pytest.raises(Exception, match="preempted"): + lance.LanceDataset.commit(tmp_path, op, read_version=written_at) + + # A nullable add preserves nullability and skips the barrier. + lance.dataset(tmp_path).add_columns({"copied": "value"}) + txn = lance.dataset(tmp_path).read_transaction(3) + assert txn is not None + assert txn.operation.preserves_nullability is True + + +def test_project_nullability_assertion_round_trips(tmp_path: Path): + tbl = pa.table({"value": pa.array([1, 2], pa.int32())}) + lance.write_dataset(tbl, tmp_path) + lance.dataset(tmp_path).alter_columns({"path": "value", "nullable": False}) + + # Reading the tightening back must preserve the claim, or recommitting it + # would silently drop the concurrency barrier. + txn = lance.dataset(tmp_path).read_transaction(2) + assert txn is not None + assert txn.operation.preserves_nullability is False + + # A Python-built non-assertion must reach the barrier: race an append. + written_at = lance.dataset(tmp_path).version + appended = lance.write_dataset( + pa.table({"value": pa.array([7], pa.int32())}), tmp_path, mode="append" + ) + assert appended.version > written_at + relax = lance.LanceOperation.Project( + schema=txn.operation.schema, preserves_nullability=False + ) + with pytest.raises(Exception, match="preempted"): + lance.LanceDataset.commit(tmp_path, relax, read_version=written_at) diff --git a/python/python/tests/test_session.py b/python/python/tests/test_session.py index 18ffb6df7fa..05b8a637628 100644 --- a/python/python/tests/test_session.py +++ b/python/python/tests/test_session.py @@ -5,6 +5,7 @@ import lance import pyarrow as pa +import pytest def test_cache_size_bytes( @@ -37,3 +38,86 @@ def test_share_session(tmp_path: Path): assert ds1.session().size_bytes() == ds2.session().size_bytes() assert ds1.to_table() == ds2.to_table() + + +def test_fragment_write_with_session(tmp_path: Path): + from lance.fragment import LanceFragment, write_fragments + + data = pa.table({"a": range(10), "b": [str(i) for i in range(10)]}) + ds = lance.write_dataset(data, tmp_path) + # Drop a column so the surviving field id is non-trivial (!= 0). Appends + # that infer the schema must pick up this field id from the dataset. + ds.drop_columns(["a"]) + field_id = ds.lance_schema.field_case_insensitive("b").id() + assert field_id != 0 + + session = ds.session() + size_before = session.size_bytes() + + append_data = pa.table({"b": ["x", "y"]}) + fragments = write_fragments( + append_data, str(tmp_path), mode="append", session=session + ) + assert len(fragments) == 1 + assert fragments[0].files[0].fields == [field_id] + + fragment = LanceFragment.create( + str(tmp_path), append_data, mode="append", session=session + ) + assert fragment.files[0].fields == [field_id] + + # The manifest loads for schema inference went through the shared session. + assert session.size_bytes() > size_before + + # A LanceDataset destination always uses its own session; a different + # explicit session is rejected. + with pytest.raises(ValueError, match="not the destination dataset's own session"): + write_fragments(append_data, ds, mode="append", session=lance.Session()) + + +def test_cache_backend_uri_config(): + session = lance.Session(index_cache_backend="moka://?capacity=1048576") + + assert session.index_cache_size_bytes() == 0 + + +def test_cache_backend_dict_config(): + session = lance.Session( + index_cache_backend={ + "kind": "MOKA", + "options": {"capacity": "1048576"}, + }, + ) + + assert session.index_cache_size_bytes() == 0 + + +def test_cache_backend_rejects_size_and_backend(): + with pytest.raises( + ValueError, + match="index_cache_size_bytes and index_cache_backend are mutually exclusive", + ): + lance.Session( + index_cache_size_bytes=1024, + index_cache_backend="moka://?capacity=1048576", + ) + + +def test_cache_backend_rejects_unknown_dict_key(): + with pytest.raises(ValueError, match="unknown dict key"): + lance.Session( + index_cache_backend={ + "kind": "moka", + "capacity": "1048576", + }, + ) + + +def test_cache_backend_rejects_moka_without_capacity(): + with pytest.raises(ValueError, match="capacity is required"): + lance.Session(index_cache_backend="moka://") + + +def test_cache_backend_rejects_moka_empty_capacity(): + with pytest.raises(ValueError, match="capacity must not be empty"): + lance.Session(index_cache_backend="moka://?capacity=") diff --git a/python/python/tests/test_tf.py b/python/python/tests/test_tf.py deleted file mode 100644 index 3652df0e938..00000000000 --- a/python/python/tests/test_tf.py +++ /dev/null @@ -1,351 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright The Lance Authors - -import os -import warnings - -import lance -import numpy as np -import pandas as pd -import pyarrow as pa -import pytest -from lance.arrow import ImageArray -from lance.fragment import LanceFragment - -pytest.skip("Skip tensorflow tests", allow_module_level=True) - -try: - with warnings.catch_warnings(): - warnings.filterwarnings("ignore", category=DeprecationWarning) - import tensorflow as tf # noqa: F401 -except ImportError: - pytest.skip( - "Tensorflow is not installed. Please install tensorflow to " - + "test lance.tf module.", - allow_module_level=True, - ) - -from lance.tf.data import ( # noqa: E402 - from_lance, - from_lance_batches, - lance_fragments, - lance_take_batches, -) - - -@pytest.fixture -def tf_dataset(tmp_path): - df = pd.DataFrame( - { - "a": range(10000), - "s": [f"val-{i}" for i in range(10000)], - "vec": [[i * 0.2] * 128 for i in range(10000)], - } - ) - - schema = pa.schema( - [ - pa.field("a", pa.int64()), - pa.field("s", pa.string()), - pa.field("vec", pa.list_(pa.float32(), 128)), - ] - ) - tbl = pa.Table.from_pandas(df, schema=schema) - uri = tmp_path / "dataset.lance" - lance.write_dataset( - tbl, - uri, - schema=tbl.schema, - max_rows_per_group=100, - max_rows_per_file=1000, - ) - return uri - - -def test_fragment_dataset(tf_dataset): - ds = from_lance(tf_dataset, batch_size=100) - for idx, batch in enumerate(ds): - assert batch["a"].numpy()[0] == idx * 100 - assert batch["s"].numpy()[0] == f"val-{idx * 100}".encode("utf-8") - assert batch["a"].shape == (100,) - assert batch["vec"].shape == ( - 100, - 128, - ) # Fixed size list - - -def test_projection(tf_dataset): - ds = from_lance(tf_dataset, batch_size=100, columns=["a"]) - - for idx, batch in enumerate(ds): - assert list(batch.keys()) == ["a"] - assert batch["a"].numpy()[0] == idx * 100 - assert batch["a"].shape == (100,) - - -def test_filter(tf_dataset): - ds = from_lance(tf_dataset, batch_size=100, filter="a >= 5000") - - for idx, batch in enumerate(ds): - assert batch["a"].numpy()[0] == idx * 100 + 5000 - assert batch["a"].shape == (100,) - - -def test_namespace_table_id(monkeypatch): - calls = {} - - class DummyScanner: - def __init__(self): - self._batch = pa.record_batch([pa.array([1, 2])], names=["a"]) - self.projected_schema = self._batch.schema - - def to_batches(self): - yield self._batch - - class DummyDataset: - def scanner(self, **kwargs): - return DummyScanner() - - def fake_dataset(uri=None, **kwargs): - calls["uri"] = uri - calls["kwargs"] = kwargs - return DummyDataset() - - monkeypatch.setattr(lance, "dataset", fake_dataset) - - ns = object() - ds = from_lance( - None, - namespace_client=ns, - table_id=["tbl"], - ignore_namespace_table_storage_options=True, - ) - - assert calls["kwargs"]["namespace_client"] is ns - assert calls["kwargs"]["table_id"] == ["tbl"] - assert calls["kwargs"]["ignore_namespace_table_storage_options"] is True - - batches = list(ds) - assert [b["a"].numpy().tolist() for b in batches] == [[1, 2]] - - -def test_scan_use_tf_data(tf_dataset): - ds = tf.data.Dataset.from_lance(tf_dataset) - for idx, batch in enumerate(ds): - assert batch["a"].numpy()[0] == idx * 100 - assert batch["s"].numpy()[0] == f"val-{idx * 100}".encode("utf-8") - assert batch["a"].shape == (100,) - assert batch["vec"].shape == ( - 100, - 128, - ) # Fixed size list - - -def test_pass_fragments(tf_dataset): - # Can pass fragments directly to from_lance - dataset = lance.dataset(tf_dataset) - ds = from_lance(tf_dataset, fragments=dataset.get_fragments(), batch_size=100) - ds_default = from_lance(tf_dataset, batch_size=100) - for batch, batch_default in zip(ds, ds_default): - assert batch["a"].numpy()[0] == batch_default["a"].numpy()[0] - assert batch["a"].numpy().shape == (100,) - assert batch["vec"].shape == ( - 100, - 128, - ) - - # Can pass ids directly to from_lance - ds = from_lance(tf_dataset, fragments=[0, 1, 2], batch_size=100) - for idx, batch in enumerate(ds): - assert batch["a"].numpy()[0] == idx * 100 - assert batch["a"].numpy().shape == (100,) - - -def test_shuffle(tf_dataset): - fragments = lance_fragments(tf_dataset).shuffle(4, seed=20).take(3) - - ds = from_lance(tf_dataset, fragments=fragments, batch_size=100) - raw_ds = lance.dataset(tf_dataset) - scanner = raw_ds.scanner( - fragments=[LanceFragment(raw_ds, fid) for fid in [0, 3, 1]], batch_size=100 - ) - - for batch, raw_batch in zip(ds, scanner.to_batches()): - assert batch["a"].numpy()[0] == raw_batch.to_pydict()["a"][0] - assert batch["a"].numpy().shape == (100,) - assert batch["vec"].shape == ( - 100, - 128, - ) # Fixed size list - - -def test_dataset_batches(tf_dataset): - tf_dataset = lance.dataset(tf_dataset) - batch_size = 300 - batches = list( - from_lance_batches(tf_dataset, batch_size=batch_size).as_numpy_iterator() - ) - assert tf_dataset.count_rows() // batch_size + 1 == len(batches) - assert all(end - start == batch_size for start, end in batches[:-2]) - assert batches[-1][1] - batches[-1][0] == tf_dataset.count_rows() % batch_size - - skip = 5 - batches_skipped = list( - from_lance_batches( - tf_dataset, batch_size=batch_size, skip=skip - ).as_numpy_iterator() - ) - assert batches_skipped == batches[skip:] - - batches_shuffled = list( - from_lance_batches( - tf_dataset, batch_size=batch_size, shuffle=True, seed=42 - ).as_numpy_iterator() - ) - # make sure it does a shuffle - assert batches_shuffled != batches - batches_shuffled2 = list( - from_lance_batches( - tf_dataset, batch_size=batch_size, shuffle=True, seed=42 - ).as_numpy_iterator() - ) - # make sure the shuffle can be deterministic - assert batches_shuffled == batches_shuffled2 - - -def test_take_dataset(tf_dataset): - tf_dataset = lance.dataset(tf_dataset) - batch_ds = from_lance_batches( - tf_dataset, batch_size=100, shuffle=True, seed=42 - ).as_numpy_iterator() - lance_ds = lance_take_batches(tf_dataset, batch_ds) - lance_ds = lance_ds.unbatch().shuffle(400, seed=42).batch(100) - - for batch in lance_ds: - assert batch["a"].numpy().shape == (100,) - - batches = [(0, 200), (100, 200)] - lance_ds = lance_take_batches(tf_dataset, batches, columns=["a"]) - for (start, end), batch in zip(batches, lance_ds): - assert batch["a"].numpy().tolist() == np.arange(start, end).tolist() - assert batch.keys() == {"a"} - - -def test_var_length_list(tmp_path): - """Treat var length list as RaggedTensor.""" - df = pd.DataFrame( - { - "a": range(200), - "l": [[i] * (i % 5 + 1) for i in range(200)], - } - ) - - schema = pa.schema( - [ - pa.field("a", pa.int64()), - pa.field("l", pa.list_(pa.int32())), - ] - ) - tbl = pa.Table.from_pandas(df, schema=schema) - - uri = tmp_path / "dataset.lance" - lance.write_dataset( - tbl, - uri, - schema=tbl.schema, - ) - - output_signature = { - "a": tf.TensorSpec(shape=(None,), dtype=tf.int64), - "l": tf.RaggedTensorSpec(dtype=tf.dtypes.int32, shape=(8, None), ragged_rank=1), - } - - ds = tf.data.Dataset.from_lance( - uri, - batch_size=8, - output_signature=output_signature, - ) - for idx, batch in enumerate(ds): - assert batch["a"].numpy()[0] == idx * 8 - assert batch["l"].shape == (8, None) - assert isinstance(batch["l"], tf.RaggedTensor) - - -def test_nested_struct(tmp_path): - table = pa.table( - { - "x": pa.array( - [ - { - "a": 1, - "json": {"b": "hello", "x": b"abc"}, - }, - { - "a": 24, - "json": {"b": "world", "x": b"def"}, - }, - ] - ) - } - ) - uri = tmp_path / "dataset.lance" - dataset = lance.write_dataset(table, uri) - - ds = tf.data.Dataset.from_lance( - dataset, - batch_size=8, - ) - - for batch in ds: - tf.debugging.assert_equal(batch["x"]["a"], tf.constant([1, 24], dtype=tf.int64)) - tf.debugging.assert_equal( - batch["x"]["json"]["b"], tf.constant(["hello", "world"]) - ) - tf.debugging.assert_equal( - batch["x"]["json"]["x"], tf.constant([b"abc", b"def"]) - ) - - -def test_tensor(tmp_path): - arr = np.array([[[1, 2, 3], [4, 5, 6]], [[1, 2, 3], [4, 5, 6]]], dtype=np.float32) - table = pa.table({"x": pa.FixedShapeTensorArray.from_numpy_ndarray(arr)}) - - uri = tmp_path / "dataset.lance" - dataset = lance.write_dataset(table, uri) - ds = tf.data.Dataset.from_lance(dataset) - - for batch in ds: - assert batch["x"].shape == (2, 2, 3) - assert batch["x"].dtype == tf.float32 - assert batch["x"].numpy().tolist() == arr.tolist() - - -def test_image_types(tmp_path): - path = [os.path.join(os.path.dirname(__file__), "images/1.png")] - uris = ImageArray.from_array(path * 3) - encoded_images = uris.read_uris() - tensors = encoded_images.to_tensor() - table = pa.table( - { - "uris": uris, - "encoded_images": encoded_images, - "tensor_images": tensors, - } - ) - - uri = tmp_path / "dataset.lance" - dataset = lance.write_dataset(table, uri) - ds = tf.data.Dataset.from_lance(dataset) - - for batch in ds: - assert batch["uris"].shape == (3,) - assert batch["uris"].dtype == tf.string - assert batch["uris"].numpy().astype("str").tolist() == uris.tolist() - - assert batch["encoded_images"].shape == (3,) - assert batch["encoded_images"].dtype == tf.string - assert batch["encoded_images"].numpy().tolist() == encoded_images.tolist() - - assert batch["tensor_images"].shape == (3, 1, 1, 4) - assert batch["tensor_images"].dtype == tf.uint8 - assert batch["tensor_images"].numpy().tolist() == tensors.to_numpy().tolist() diff --git a/python/python/tests/test_vector.py b/python/python/tests/test_vector.py index 4ea4e7d425e..12caec5f0d5 100644 --- a/python/python/tests/test_vector.py +++ b/python/python/tests/test_vector.py @@ -5,7 +5,12 @@ import numpy as np import pyarrow as pa import pytest -from lance.vector import hamming_clustering_for_sample, vec_to_table +from lance.vector import ( + get_ivf_partition_info, + hamming_clustering_for_ivf_partition, + hamming_clustering_for_sample, + vec_to_table, +) def test_dict(): @@ -150,21 +155,26 @@ def test_binary_vectors_invalid_metric(tmp_path): def _hash_table(hashes): - """Build a table with a ``hash`` column of FixedSizeList. + """Build a table with a ``hash`` column of FixedSizeList. - ``hashes`` is a list of 8-byte sequences, one per row. + ``hashes`` is a list of byte sequences, one per row. The byte width must + be a positive multiple of 8. """ + byte_width = len(hashes[0]) + assert byte_width > 0 and byte_width % 8 == 0 + assert all(len(row) == byte_width for row in hashes) flat = [byte for row in hashes for byte in row] values = pa.FixedSizeListArray.from_arrays( - pa.array(flat, type=pa.uint8()), list_size=8 + pa.array(flat, type=pa.uint8()), list_size=byte_width ) return pa.Table.from_arrays([values], names=["hash"]) -def test_hamming_clustering_for_sample(tmp_path): - hash_a = [0, 0, 0, 0, 0, 0, 0, 0] - hash_b = [255, 0, 0, 0, 0, 0, 0, 0] # 8 bits from hash_a - hash_c = [1, 2, 3, 4, 5, 6, 7, 8] # far from both +@pytest.mark.parametrize("byte_width", [8, 16]) +def test_hamming_clustering_for_sample(tmp_path, byte_width): + hash_a = [0] * byte_width + hash_b = [0] * (byte_width - 8) + [255] + [0] * 7 # 8 bits from hash_a + hash_c = list(range(1, byte_width + 1)) # far from both # Rows 0,1,2 share hash_a; rows 3,4 share hash_b; row 5 is unique. table = _hash_table([hash_a, hash_a, hash_a, hash_b, hash_b, hash_c]) dataset = lance.write_dataset(table, tmp_path / "hashes") @@ -182,3 +192,89 @@ def test_hamming_clustering_for_sample(tmp_path): } # Singleton row 5 is not emitted as a cluster. assert clusters == {0: [1, 2], 3: [4]} + + +@pytest.mark.parametrize("byte_width", [8, 16]) +def test_hamming_clustering_multi_segment(tmp_path, byte_width): + mask = (1 << 64) - 1 + + def hash_bytes(value): + if byte_width == 8: + lanes = [(value * 0x9E3779B97F4A7C15) & mask] + else: + # Adjacent logical values share the first 64-bit lane and differ in + # later lanes, so threshold-0 clustering must compare every lane. + lanes = [ + ((value // 2) * 0x9E3779B97F4A7C15) & mask, + ((value * 0xD6E8FEB86659FD93) ^ 0xA5A5A5A5A5A5A5A5) & mask, + ] + return [ + byte for lane_value in lanes for byte in lane_value.to_bytes(8, "little") + ] + + # 25 distinct hash values, two copies each; the same table is written to + # fragment 0 and appended as fragment 1. + values = [i // 2 for i in range(50)] + table = _hash_table([hash_bytes(value) for value in values]) + dataset = lance.write_dataset(table, tmp_path / "hashes") + dataset.create_index( + "hash", index_type="IVF_FLAT", num_partitions=4, metric="hamming" + ) + dataset = lance.write_dataset(table, tmp_path / "hashes", mode="append") + # Optimizing with merge disabled creates a delta segment for fragment 1. + dataset.optimize.optimize_indices(num_indices_to_merge=0) + + index = dataset.describe_indices()[0] + assert len(index.segments) == 2 + + infos = get_ivf_partition_info(dataset, index.name) + assert sum(info["size"] for info in infos) == 100 + + # All four copies of each value cluster together across both fragments. + frag1_start = 1 << 32 + clusters = [] + for info in infos: + result = hamming_clustering_for_ivf_partition( + dataset, index.name, info["partition_id"], 0 + ).read_all() + clusters.extend( + zip( + result["representative"].to_pylist(), + result["duplicates"].to_pylist(), + ) + ) + assert len(clusters) == 25 + for representative, duplicates in clusters: + assert representative < frag1_start + assert len(duplicates) == 3 + assert any(dup >= frag1_start for dup in duplicates) + + # Selecting the fragment-0 segment reproduces the single-segment scope. + first_segment = next( + segment for segment in index.segments if segment.fragment_ids == {0} + ) + infos = get_ivf_partition_info( + dataset, index.name, index_segments=[first_segment.uuid] + ) + assert sum(info["size"] for info in infos) == 50 + num_selected_clusters = 0 + for info in infos: + result = hamming_clustering_for_ivf_partition( + dataset, + index.name, + info["partition_id"], + 0, + index_segments=[first_segment.uuid], + ).read_all() + for duplicates in result["duplicates"].to_pylist(): + num_selected_clusters += 1 + assert duplicates == [dup for dup in duplicates if dup < frag1_start] + assert len(duplicates) == 1 + assert num_selected_clusters == 25 + + with pytest.raises(ValueError, match="invalid index segment uuid"): + get_ivf_partition_info(dataset, index.name, index_segments=["not-a-uuid"]) + with pytest.raises(TypeError, match="str or uuid.UUID"): + get_ivf_partition_info(dataset, index.name, index_segments=[123]) + with pytest.raises(TypeError, match="not a single"): + get_ivf_partition_info(dataset, index.name, index_segments=first_segment.uuid) diff --git a/python/python/tests/test_vector_index.py b/python/python/tests/test_vector_index.py index fa2c4047cd0..21ae2aaac8e 100644 --- a/python/python/tests/test_vector_index.py +++ b/python/python/tests/test_vector_index.py @@ -2053,6 +2053,34 @@ def test_read_partition(indexed_dataset): VectorIndexReader(indexed_dataset, "id_idx") +def test_read_partition_nested_vector_quoted_field(tmp_path): + num_rows = 1024 + dimensions = 8 + rng = np.random.default_rng(42) + values = rng.integers(0, 256, size=num_rows * dimensions, dtype=np.uint8) + vectors = pa.FixedSizeListArray.from_arrays(pa.array(values), dimensions) + nested = pa.StructArray.from_arrays([vectors], names=["embedding.v1"]) + dataset = lance.write_dataset(pa.table({"data": nested}), tmp_path) + # Match nested uint8 pHash indexes without introducing PQ training setup. + dataset = dataset.create_index( + "data.`embedding.v1`", + index_type="IVF_FLAT", + name="vector_idx", + metric="hamming", + num_partitions=4, + ) + + reader = VectorIndexReader(dataset, "vector_idx") + for with_vector in (False, True): + partitions = [ + reader.read_partition(partition_id, with_vector=with_vector) + for partition_id in range(reader.num_partitions()) + ] + + assert all("_rowid" in partition.column_names for partition in partitions) + assert sum(partition.num_rows for partition in partitions) == num_rows + + def test_vector_index_with_prefilter_and_scalar_index(indexed_dataset): uri = indexed_dataset.uri new_table = create_table() @@ -2272,6 +2300,14 @@ def test_nested_field_vector_index(tmp_path): assert len(indices) == 1 assert indices[0].field_names == ["data.embedding"] + reader = VectorIndexReader(dataset, indices[0].name) + for with_vector in (False, True): + partition_rows = sum( + reader.read_partition(partition_id, with_vector=with_vector).num_rows + for partition_id in range(reader.num_partitions()) + ) + assert partition_rows == num_rows + # Test querying with the index query_vec = vectors[0] result = dataset.to_table( @@ -2468,7 +2504,7 @@ def test_vector_index_distance_range(tmp_path): assert np.all(index_distances >= distance_range[0]) and np.all( index_distances < distance_range[1] ) - assert np.allclose(brute_distances, index_distances, rtol=0.0, atol=0.0) + assert np.allclose(brute_distances, index_distances, rtol=1e-5, atol=0.0) # ============================================================================= diff --git a/python/python/tests/torch_tests/test_torch_kmeans.py b/python/python/tests/torch_tests/test_torch_kmeans.py index edfb0a0329a..96bbf76c51f 100644 --- a/python/python/tests/torch_tests/test_torch_kmeans.py +++ b/python/python/tests/torch_tests/test_torch_kmeans.py @@ -10,15 +10,16 @@ torch = pytest.importorskip("torch") -from lance.torch import preferred_device # noqa: E402 from lance.torch.kmeans import KMeans # noqa: E402 from lance.vector import train_ivf_centroids_on_accelerator # noqa: E402 -@pytest.mark.skip(reason="flaky") def test_kmeans(): arr = np.array(range(128)).reshape(-1, 8).astype(np.float32) - kmeans = KMeans(4, device="cpu") + # These duplicate centroids reproduce the empty clusters that made this test + # depend on random initialization before empty-cluster recovery was fixed. + centroids = torch.from_numpy(arr[[5, 5, 13, 13]]) + kmeans = KMeans(4, centroids=centroids, device="cpu") kmeans.fit(arr) cluster_ids = kmeans.transform(arr) @@ -27,7 +28,6 @@ def test_kmeans(): assert len(cnts) == 4 # all cluster has data -@pytest.mark.skip(reason="TODO: async dataset hangs on github CI") def test_torch_kmeans_accept_torch_device(tmp_path: Path): values = pa.array(np.array(range(128)).astype(np.float32)) arr = pa.FixedSizeListArray.from_arrays(values, 8) @@ -39,7 +39,7 @@ def test_torch_kmeans_accept_torch_device(tmp_path: Path): "vector", 2, metric_type="L2", - accelerator=preferred_device(), + accelerator=torch.device("cpu"), ) diff --git a/python/src/blob.rs b/python/src/blob.rs index 82e8a01ae8a..1f03cd9abe5 100644 --- a/python/src/blob.rs +++ b/python/src/blob.rs @@ -2,18 +2,166 @@ // SPDX-FileCopyrightText: Copyright The Lance Authors use crate::{error::PythonErrorExt, rt}; -use arrow::pyarrow::ToPyArrow; +use arrow::{ + array::{Array, ArrayRef, GenericBinaryArray, OffsetSizeTrait, cast::AsArray, make_array}, + pyarrow::{FromPyArrow, ToPyArrow}, +}; +use arrow_data::ArrayData; +use arrow_schema::{DataType, Field}; use bytes::Bytes; use lance::{ BlobDescriptor, BlobDescriptorArrayBuilder, BlobRange, DedicatedBlobWriter, PackedBlobWriter, }; use pyo3::{ - Bound, PyResult, - exceptions::PyValueError, + Bound, PyErr, PyResult, + exceptions::{PyRuntimeError, PyValueError}, pyclass, pymethods, - types::{PyAny, PyAnyMethods, PyDict, PyList, PyListMethods, PyModule}, + types::{PyAny, PyAnyMethods, PyDict, PyList, PyListMethods, PyModule, PyTypeMethods}, +}; +use std::{ + borrow::Cow, + sync::{Arc, Mutex}, }; -use std::sync::Arc; + +fn with_writer( + inner: &Mutex>, + writer_name: &str, + operation: impl FnOnce(&W) -> R, +) -> PyResult { + let guard = inner.lock().map_err(|_| poisoned_writer(writer_name))?; + let writer = guard.as_ref().ok_or_else(|| finished_writer(writer_name))?; + Ok(operation(writer)) +} + +fn writer_mut<'a, W>(inner: &'a mut Mutex>, writer_name: &str) -> PyResult<&'a mut W> { + inner + .get_mut() + .map_err(|_| poisoned_writer(writer_name))? + .as_mut() + .ok_or_else(|| finished_writer(writer_name)) +} + +fn take_writer(inner: &mut Mutex>, writer_name: &str) -> PyResult { + inner + .get_mut() + .map_err(|_| poisoned_writer(writer_name))? + .take() + .ok_or_else(|| finished_writer(writer_name)) +} + +fn finished_writer(writer_name: &str) -> PyErr { + PyValueError::new_err(format!("{writer_name} is already finished")) +} + +fn poisoned_writer(writer_name: &str) -> PyErr { + PyRuntimeError::new_err(format!("{writer_name} lock is poisoned")) +} + +/// Reconstruct the PyArrow equivalent of [`BlobDescriptorArrayBuilder::field`]. +/// +/// Arrow's array bridge does not carry the enclosing extension field, so this +/// rebuilds the canonical six nullable blob-v2 children and +/// `ARROW:extension:name = lance.blob.v2` metadata. +fn descriptor_field_to_pyarrow<'py>( + field: &Field, + py: pyo3::Python<'py>, +) -> PyResult> { + let pyarrow = PyModule::import(py, "pyarrow")?; + let child_fields = PyList::empty(py); + for (name, type_fn) in [ + ("kind", "uint8"), + ("data", "large_binary"), + ("uri", "utf8"), + ("blob_id", "uint32"), + ("blob_size", "uint64"), + ("position", "uint64"), + ] { + let data_type = pyarrow.getattr(type_fn)?.call0()?; + let child = pyarrow.call_method1("field", (name, data_type, true))?; + child_fields.append(child)?; + } + let data_type = pyarrow.call_method1("struct", (child_fields,))?; + let metadata = PyDict::new(py); + metadata.set_item("ARROW:extension:name", "lance.blob.v2")?; + let kwargs = PyDict::new(py); + kwargs.set_item("nullable", field.is_nullable())?; + kwargs.set_item("metadata", metadata)?; + pyarrow.call_method("field", (field.name().as_str(), data_type), Some(&kwargs)) +} + +/// Normalize inputs accepted by [`PyPackedBlobWriter::write_blobs`] into Arrow arrays. +/// +/// BinaryArray, LargeBinaryArray, and ChunkedArray values of either binary type +/// are accepted. Chunk boundaries, nulls, and empty values remain in the arrays; +/// each row is later passed to the core writer as an optional byte slice. +fn extract_blob_payloads(payloads: &Bound<'_, PyAny>) -> PyResult> { + match ArrayData::from_pyarrow_bound(payloads) { + Ok(data) => Ok(vec![validated_blob_payload(data, None)?]), + Err(_) => { + let pyarrow = PyModule::import(payloads.py(), "pyarrow")?; + let chunked_array_type = pyarrow.getattr("ChunkedArray")?; + if !payloads.is_instance(&chunked_array_type)? { + return Err(PyValueError::new_err(format!( + "payloads must be a pyarrow BinaryArray, LargeBinaryArray, or ChunkedArray, got {}", + payloads.get_type().name()? + ))); + } + + let chunked_data_type = DataType::from_pyarrow_bound(&payloads.getattr("type")?)?; + if !matches!(chunked_data_type, DataType::Binary | DataType::LargeBinary) { + return Err(PyValueError::new_err(format!( + "Packed blob payloads must have Arrow type Binary or LargeBinary, got {chunked_data_type}" + ))); + } + + let chunks = payloads.getattr("chunks")?; + let mut arrays = Vec::with_capacity(chunks.len()?); + for (chunk_index, chunk) in chunks.try_iter()?.enumerate() { + let data = ArrayData::from_pyarrow_bound(&chunk?)?; + arrays.push(validated_blob_payload(data, Some(chunk_index))?); + } + Ok(arrays) + } + } +} + +fn validated_blob_payload(data: ArrayData, chunk_index: Option) -> PyResult { + let context = chunk_index + .map(|index| format!("Packed blob payload chunk {index}")) + .unwrap_or_else(|| "Packed blob payload array".to_string()); + if !matches!(data.data_type(), DataType::Binary | DataType::LargeBinary) { + return Err(PyValueError::new_err(format!( + "{context} must have Arrow type Binary or LargeBinary, got {}", + data.data_type() + ))); + } + if data.is_empty() { + // PyArrow may export an empty slice without the values preceding its + // nonzero first offset. Normalize it because an empty array never + // observes those buffers, and Arrow validation would reject the slice. + return Ok(make_array(ArrayData::new_empty(data.data_type()))); + } + data.validate_full().map_err(|error| { + PyValueError::new_err(format!("{context} contains invalid Arrow data: {error}")) + })?; + Ok(make_array(data)) +} + +/// Stream one Arrow binary array into the core writer as zero-copy row slices. +/// +/// Null rows become `None` so the core writer records null descriptors, keeping +/// its output row-aligned with the input. +async fn write_binary_payloads( + writer: &mut PackedBlobWriter, + payloads: &GenericBinaryArray, +) -> PyResult<()> { + writer + .write_packed_blobs( + (0..payloads.len()).map(|row| payloads.is_valid(row).then(|| payloads.value(row))), + ) + .await + .infer_error() +} #[pyclass(name = "BlobDescriptor", skip_from_py_object)] #[derive(Clone)] @@ -61,31 +209,7 @@ impl PyBlobDescriptorArrayBuilder { #[getter] pub fn field<'py>(&self, py: pyo3::Python<'py>) -> PyResult> { - let pyarrow = PyModule::import(py, "pyarrow")?; - let child_fields = PyList::empty(py); - for (name, type_fn) in [ - ("kind", "uint8"), - ("data", "large_binary"), - ("uri", "utf8"), - ("blob_id", "uint32"), - ("blob_size", "uint64"), - ("position", "uint64"), - ] { - let data_type = pyarrow.getattr(type_fn)?.call0()?; - let child = pyarrow.call_method1("field", (name, data_type, true))?; - child_fields.append(child)?; - } - let data_type = pyarrow.call_method1("struct", (child_fields,))?; - let metadata = PyDict::new(py); - metadata.set_item("ARROW:extension:name", "lance.blob.v2")?; - let kwargs = PyDict::new(py); - kwargs.set_item("nullable", self.field.is_nullable())?; - kwargs.set_item("metadata", metadata)?; - pyarrow.call_method( - "field", - (self.field.name().as_str(), data_type), - Some(&kwargs), - ) + descriptor_field_to_pyarrow(&self.field, py) } pub fn extend_packed( @@ -150,9 +274,10 @@ impl PyBlobDescriptorArrayBuilder { } } -#[pyclass(name = "PackedBlobWriter", skip_from_py_object, unsendable)] +#[pyclass(name = "PackedBlobWriter", skip_from_py_object)] pub struct PyPackedBlobWriter { - inner: Option, + field: Option, + inner: Mutex>, } impl PyPackedBlobWriter { @@ -165,19 +290,22 @@ impl PyPackedBlobWriter { PackedBlobWriter::try_new(object_store.as_ref().clone(), data_file_path, blob_id) .await .infer_error()?; - Ok(Self { inner: Some(inner) }) + Ok(Self { + field: None, + inner: Mutex::new(Some(inner)), + }) } - fn inner(&self) -> PyResult<&PackedBlobWriter> { - self.inner - .as_ref() - .ok_or_else(|| PyValueError::new_err("PackedBlobWriter is already finished")) + fn with_inner(&self, operation: impl FnOnce(&PackedBlobWriter) -> R) -> PyResult { + with_writer(&self.inner, "PackedBlobWriter", operation) } fn inner_mut(&mut self) -> PyResult<&mut PackedBlobWriter> { - self.inner - .as_mut() - .ok_or_else(|| PyValueError::new_err("PackedBlobWriter is already finished")) + writer_mut(&mut self.inner, "PackedBlobWriter") + } + + fn take_inner(&mut self) -> PyResult { + take_writer(&mut self.inner, "PackedBlobWriter") } } @@ -185,32 +313,142 @@ impl PyPackedBlobWriter { impl PyPackedBlobWriter { #[getter] pub fn blob_id(&self) -> PyResult { - Ok(self.inner()?.blob_id()) + self.with_inner(PackedBlobWriter::blob_id) } #[getter] pub fn path(&self) -> PyResult { - Ok(self.inner()?.path().to_string()) + self.with_inner(|writer| writer.path().to_string()) + } + + /// The descriptor field associated with the array returned by + /// :meth:`finish_array`. + /// + /// The field uses the name passed to ``finish_array`` and carries the + /// ``lance.blob.v2`` extension metadata. It is available only after + /// ``finish_array`` succeeds; accessing it earlier raises ``ValueError``. + #[getter] + pub fn field<'py>(&self, py: pyo3::Python<'py>) -> PyResult> { + let field = self.field.as_ref().ok_or_else(|| { + PyValueError::new_err("PackedBlobWriter field is available after finish_array") + })?; + descriptor_field_to_pyarrow(field, py) } - pub fn write_blob(&mut self, data: Vec) -> PyResult<()> { - rt().block_on(None, self.inner_mut()?.write_blob(data))? + /// Append one packed blob. + /// + /// Python ``bytes`` are borrowed without copying. Other compatible byte + /// sequences use owned storage for the duration of the write. + pub fn write_blob(&mut self, data: Cow<'_, [u8]>) -> PyResult<()> { + rt().block_on(None, self.inner_mut()?.write_blob(data.as_ref()))? .infer_error() } + /// Append a batch of packed blob payloads. + /// + /// Parameters + /// ---------- + /// payloads : pyarrow.BinaryArray, pyarrow.LargeBinaryArray, or pyarrow.ChunkedArray + /// A binary Arrow array. Every chunk of a chunked array must be binary. + /// Each input row produces one descriptor row, in order, across chunks + /// and repeated calls. Null rows produce null descriptors; empty but + /// non-null byte strings produce valid zero-length blobs. + /// + /// Examples + /// -------- + /// >>> import pyarrow as pa + /// >>> payloads = pa.array([b"first", None, b""], type=pa.large_binary()) + /// >>> writer.write_blobs(payloads) + /// >>> descriptors = writer.finish_array("blob") + /// >>> len(descriptors) + /// 3 + pub fn write_blobs(&mut self, payloads: &Bound<'_, PyAny>) -> PyResult<()> { + let payloads = extract_blob_payloads(payloads)?; + let result = { + let writer = self.inner_mut()?; + rt().block_on(None, async { + for payloads in payloads { + match payloads.data_type() { + DataType::Binary => { + write_binary_payloads(writer, payloads.as_binary::()).await? + } + DataType::LargeBinary => { + write_binary_payloads(writer, payloads.as_binary::()).await? + } + data_type => { + return Err(PyValueError::new_err(format!( + "Packed blob payloads must have Arrow type Binary or LargeBinary, got {data_type}" + ))); + } + } + } + Ok(()) + }) + }; + match result { + Ok(result) => result, + Err(error) => { + // KeyboardInterrupt drops the async batch future. Remove the core + // writer as well so RAII cleanup runs and a completed prefix cannot + // be reused as a new batch. + self.take_inner()?; + Err(error) + } + } + } + pub fn finish(&mut self) -> PyResult> { - let inner = self - .inner - .take() - .ok_or_else(|| PyValueError::new_err("PackedBlobWriter is already finished"))?; + let inner = self.take_inner()?; let values = rt().block_on(None, inner.finish())?.infer_error()?; Ok(values.into_iter().map(Into::into).collect()) } + + /// Finish the upload and return its blob descriptors as a PyArrow array. + /// + /// The returned ``pyarrow.StructArray`` has one row per payload previously + /// passed to :meth:`write_blob` or :meth:`write_blobs`. The writer is consumed + /// by this call. After it succeeds, :attr:`field` returns the matching + /// extension field with ``field_name`` as its name. + /// + /// Parameters + /// ---------- + /// field_name : str + /// Name for the descriptor field exposed by :attr:`field`. + /// + /// Returns + /// ------- + /// pyarrow.StructArray + /// Row-aligned blob descriptors, including null rows from bulk input. + /// + /// Examples + /// -------- + /// >>> import pyarrow as pa + /// >>> writer.write_blobs(pa.array([b"value", None])) + /// >>> descriptors = writer.finish_array("payload") + /// >>> descriptors.is_null().to_pylist() + /// [False, True] + /// >>> writer.field.name + /// 'payload' + pub fn finish_array<'py>( + &mut self, + py: pyo3::Python<'py>, + field_name: String, + ) -> PyResult> { + let inner = self.take_inner()?; + let values = rt().block_on(None, inner.finish())?.infer_error()?; + let mut builder = BlobDescriptorArrayBuilder::new(field_name); + builder.extend(values).infer_error()?; + let column = builder.finish().infer_error()?; + let (field, array) = column.into_parts(); + let array = array.to_data().to_pyarrow(py)?; + self.field = Some(field); + Ok(array) + } } -#[pyclass(name = "DedicatedBlobWriter", skip_from_py_object, unsendable)] +#[pyclass(name = "DedicatedBlobWriter", skip_from_py_object)] pub struct PyDedicatedBlobWriter { - inner: Option, + inner: Mutex>, } impl PyDedicatedBlobWriter { @@ -223,19 +461,21 @@ impl PyDedicatedBlobWriter { DedicatedBlobWriter::try_new(object_store.as_ref().clone(), data_file_path, blob_id) .await .infer_error()?; - Ok(Self { inner: Some(inner) }) + Ok(Self { + inner: Mutex::new(Some(inner)), + }) } - fn inner(&self) -> PyResult<&DedicatedBlobWriter> { - self.inner - .as_ref() - .ok_or_else(|| PyValueError::new_err("DedicatedBlobWriter is already finished")) + fn with_inner(&self, operation: impl FnOnce(&DedicatedBlobWriter) -> R) -> PyResult { + with_writer(&self.inner, "DedicatedBlobWriter", operation) } fn inner_mut(&mut self) -> PyResult<&mut DedicatedBlobWriter> { - self.inner - .as_mut() - .ok_or_else(|| PyValueError::new_err("DedicatedBlobWriter is already finished")) + writer_mut(&mut self.inner, "DedicatedBlobWriter") + } + + fn take_inner(&mut self) -> PyResult { + take_writer(&mut self.inner, "DedicatedBlobWriter") } } @@ -243,12 +483,12 @@ impl PyDedicatedBlobWriter { impl PyDedicatedBlobWriter { #[getter] pub fn blob_id(&self) -> PyResult { - Ok(self.inner()?.blob_id()) + self.with_inner(DedicatedBlobWriter::blob_id) } #[getter] pub fn path(&self) -> PyResult { - Ok(self.inner()?.path().to_string()) + self.with_inner(|writer| writer.path().to_string()) } pub fn write(&mut self, data: Vec) -> PyResult<()> { @@ -257,10 +497,7 @@ impl PyDedicatedBlobWriter { } pub fn finish(&mut self) -> PyResult { - let inner = self - .inner - .take() - .ok_or_else(|| PyValueError::new_err("DedicatedBlobWriter is already finished"))?; + let inner = self.take_inner()?; let value = rt().block_on(None, inner.finish())?.infer_error()?; Ok(value.into()) } diff --git a/python/src/datagen.rs b/python/src/datagen.rs index 8b046c37f24..194ba3d6a73 100644 --- a/python/src/datagen.rs +++ b/python/src/datagen.rs @@ -1,7 +1,7 @@ use arrow::pyarrow::PyArrowType; -use arrow_array::RecordBatch; +use arrow_array::{RecordBatch, RecordBatchReader}; use arrow_schema::Schema; -use lance_datagen::{BatchCount, ByteCount}; +use lance_datagen::{BatchCount, ByteCount, RowCount}; use pyo3::{ Bound, PyResult, Python, pyfunction, types::{PyModule, PyModuleMethods}, @@ -16,22 +16,44 @@ pub fn is_datagen_supported() -> bool { true } +/// Generate `batch_count` batches of random data for `schema`. +/// +/// Batch size is set either by `rows_in_batch` (exact rows per batch) or +/// `bytes_in_batch` (approximate bytes per batch); the two are mutually +/// exclusive. When neither is given the byte-based default is used. #[pyfunction] -#[pyo3(signature=(schema, batch_count=None, bytes_in_batch=None))] +#[pyo3(signature=(schema, batch_count=None, bytes_in_batch=None, rows_in_batch=None))] pub fn rand_batches( schema: PyArrowType, batch_count: Option, bytes_in_batch: Option, + rows_in_batch: Option, ) -> PyResult>> { - lance_datagen::rand(&schema.0) - .into_reader_bytes( - ByteCount::from(bytes_in_batch.unwrap_or(DEFAULT_BATCH_SIZE_BYTES)), - BatchCount::from(batch_count.unwrap_or(DEFAULT_BATCH_COUNT)), - lance_datagen::RoundingBehavior::RoundUp, - ) - .map_err(|e| { - pyo3::exceptions::PyValueError::new_err(format!("Failed to generate batches: {}", e)) - })? + if rows_in_batch.is_some() && bytes_in_batch.is_some() { + return Err(pyo3::exceptions::PyValueError::new_err( + "rows_in_batch and bytes_in_batch are mutually exclusive", + )); + } + let builder = lance_datagen::rand(&schema.0); + let batch_count = BatchCount::from(batch_count.unwrap_or(DEFAULT_BATCH_COUNT)); + let reader: Box = match rows_in_batch { + Some(rows) => Box::new(builder.into_reader_rows(RowCount::from(rows), batch_count)), + None => Box::new( + builder + .into_reader_bytes( + ByteCount::from(bytes_in_batch.unwrap_or(DEFAULT_BATCH_SIZE_BYTES)), + batch_count, + lance_datagen::RoundingBehavior::RoundUp, + ) + .map_err(|e| { + pyo3::exceptions::PyValueError::new_err(format!( + "Failed to generate batches: {}", + e + )) + })?, + ), + }; + reader .map(|item| { item.map(PyArrowType::from).map_err(|e| { pyo3::exceptions::PyValueError::new_err(format!("Failed to generate batch: {}", e)) diff --git a/python/src/dataset.rs b/python/src/dataset.rs index 350428c89aa..f4bff47d13f 100644 --- a/python/src/dataset.rs +++ b/python/src/dataset.rs @@ -61,7 +61,8 @@ use lance::dataset::{ }; use lance::index::vector::utils::get_vector_type; use lance::index::{ - DatasetIndexExt, DatasetIndexInternalExt, IndexSegment, vector::VectorIndexParams, + DatasetIndexExt, DatasetIndexInternalExt, IndexSegment, IntoIndexSegment, + vector::VectorIndexParams, }; use lance::{dataset::builder::DatasetBuilder, index::vector::IndexFileVersion}; use lance_arrow::as_fixed_size_list_array; @@ -78,7 +79,7 @@ use lance_index::{ FtsPrewarmOptions, IndexParams, IndexType, PrewarmOptions, optimize::OptimizeOptions, progress::{IndexBuildProgress, NoopIndexBuildProgress}, - scalar::inverted::InvertedListFormatVersion, + scalar::inverted::{DocumentGranularity, InvertedListFormatVersion}, scalar::{FullTextSearchQuery, InvertedIndexParams, ScalarIndexParams}, vector::{ ApproxMode, DEFAULT_QUERY_PARALLELISM, Query as VectorQuery, @@ -97,6 +98,7 @@ use lance_table::io::commit::external_manifest::ExternalManifestCommitHandler; use crate::error::PythonErrorExt; use crate::file::object_store_from_uri_or_path; use crate::fragment::FileFragment; +use crate::fts::FtsTokenizerOptions; use crate::indices::{PyIndexConfig, PyIndexDescription, PyIndexSegment}; use crate::namespace::extract_namespace_arc; use crate::rt; @@ -125,13 +127,44 @@ const DEFAULT_NPROBES: usize = 1; const LANCE_COMMIT_MESSAGE_KEY: &str = "__lance_commit_message"; const INDEX_PROGRESS_QUEUE_SIZE: usize = 1024; -fn read_blobs_to_python( - py: Python<'_>, - blobs: Vec, -) -> Vec<(u64, Py)> { +type PyBlobBytes = Option>; +type PyReadBlob = (u64, PyBlobBytes); +type PyReadBlobRange = (usize, u64, PyBlobBytes); + +fn read_blobs_to_python(py: Python<'_>, blobs: Vec) -> Vec { blobs .into_iter() - .map(|blob| (blob.row_address, PyBytes::new(py, &blob.data).unbind())) + .map(|blob| { + ( + blob.row_address, + blob.data.map(|data| PyBytes::new(py, &data).unbind()), + ) + }) + .collect() +} + +fn read_blob_ranges_to_python( + py: Python<'_>, + ranges: Vec, +) -> Vec { + ranges + .into_iter() + .map(|range| { + ( + range.request_index, + range.row_address, + range.data.map(|data| PyBytes::new(py, &data).unbind()), + ) + }) + .collect() +} + +fn blob_range_requests_from_tuples( + requests: Vec<(u64, u64, u64)>, +) -> Vec { + requests + .into_iter() + .map(|(row, offset, length)| lance::dataset::BlobRangeRequest::new(row, offset, length)) .collect() } @@ -149,6 +182,20 @@ fn configure_read_blobs_builder( builder } +fn configure_read_blob_ranges_builder( + mut builder: lance::dataset::ReadBlobRangesBuilder, + io_buffer_size: Option, + preserve_order: Option, +) -> lance::dataset::ReadBlobRangesBuilder { + if let Some(bytes) = io_buffer_size { + builder = builder.with_io_buffer_size_bytes(bytes); + } + if let Some(preserve) = preserve_order { + builder = builder.preserve_order(preserve); + } + builder +} + fn stats_log_interval_from_millis(ms: u64) -> Option { if ms == 0 { None @@ -164,7 +211,6 @@ fn stats_log_interval_from_millis(ms: u64) -> Option { #[allow(clippy::too_many_arguments)] fn writer_config_from_kwargs( durable_write: Option, - sync_indexed_write: Option, max_wal_buffer_size: Option, max_wal_flush_interval_ms: Option, max_memtable_size: Option, @@ -172,8 +218,6 @@ fn writer_config_from_kwargs( max_memtable_batches: Option, max_unflushed_memtable_bytes: Option, manifest_scan_batch_size: Option, - async_index_buffer_rows: Option, - async_index_interval_ms: Option, backpressure_log_interval_ms: Option, stats_log_interval_ms: Option, hnsw_params: Option>>, @@ -186,10 +230,6 @@ fn writer_config_from_kwargs( config = config.with_durable_write(v); any = true; } - if let Some(v) = sync_indexed_write { - config = config.with_sync_indexed_write(v); - any = true; - } if let Some(v) = max_wal_buffer_size { config = config.with_max_wal_buffer_size(v); any = true; @@ -218,14 +258,6 @@ fn writer_config_from_kwargs( config = config.with_manifest_scan_batch_size(v); any = true; } - if let Some(v) = async_index_buffer_rows { - config = config.with_async_index_buffer_rows(v); - any = true; - } - if let Some(v) = async_index_interval_ms { - config = config.with_async_index_interval(Duration::from_millis(v)); - any = true; - } if let Some(v) = backpressure_log_interval_ms { config = config.with_backpressure_log_interval(Duration::from_millis(v)); any = true; @@ -456,6 +488,60 @@ impl MergeInsertBuilder { Ok((PyLance(transaction), stats)) } + /// Execute the merge insert from fully-materialized data. + /// + /// The data is read into memory and wrapped in an in-memory table, so retries + /// never spill to disk and the source's statistics drive the join. Callers + /// should only route in-memory inputs (e.g. a `pa.Table`) here. + pub fn execute_batches(&mut self, new_data: &Bound) -> PyResult> { + let py = new_data.py(); + let reader = convert_reader(new_data)?; + let batches = reader + .collect::, _>>() + .map_err(|err| PyValueError::new_err(err.to_string()))?; + + let job = self + .builder + .try_build() + .map_err(|err| PyValueError::new_err(err.to_string()))?; + + let (new_dataset, stats) = rt() + .spawn(Some(py), job.execute_batches(batches))? + .map_err(|err: lance::Error| PyIOError::new_err(err.to_string()))?; + + let dataset = self.dataset.bind(py); + dataset.borrow_mut().ds = new_dataset; + + Ok(Self::build_stats(&stats, py)?.into()) + } + + /// [`Self::execute_batches`] without committing; returns the transaction. + pub fn execute_uncommitted_batches<'a>( + &mut self, + new_data: &Bound<'a, PyAny>, + ) -> PyResult<(PyLance, Bound<'a, PyDict>)> { + let py = new_data.py(); + let reader = convert_reader(new_data)?; + let batches = reader + .collect::, _>>() + .map_err(|err| PyValueError::new_err(err.to_string()))?; + + let job = self + .builder + .try_build() + .map_err(|err| PyValueError::new_err(err.to_string()))?; + + let UncommittedMergeInsert { + transaction, stats, .. + } = rt() + .spawn(Some(py), job.execute_uncommitted_batches(batches))? + .map_err(|err: lance::Error| PyIOError::new_err(err.to_string()))?; + + let stats = Self::build_stats(&stats, py)?; + + Ok((PyLance(transaction), stats)) + } + #[pyo3(signature=(schema = None, verbose = false))] pub fn explain_plan( &mut self, @@ -487,46 +573,27 @@ impl MergeInsertBuilder { .map_err(|err| PyIOError::new_err(err.to_string())) } - /// Mark MemWAL generations as merged into the base table. + /// Mark MemWAL SSTables as compacted into the base table. /// - /// Call this when executing a merge_insert that incorporates MemWAL - /// flushed generation data. This updates the MemWAL generation tracking - /// to prevent duplicate merges. - pub fn mark_generations_as_merged<'a>( + /// Call this when executing a merge_insert that compacts MemWAL SSTables. + /// This updates MemWAL compaction progress to prevent duplicate compactions. + pub fn mark_sstables_as_compacted<'a>( mut slf: PyRefMut<'a, Self>, - generations: Vec>, + sstables: Vec>, ) -> PyResult> { - use lance_index::mem_wal::MergedGeneration; + use lance_index::mem_wal::CompactedSsTable; - let gens: Vec = generations + let compacted_sstables: Vec = sstables .iter() - .map(|g| g.borrow().to_lance()) + .map(|sstable| sstable.borrow().to_lance()) .collect::>()?; - slf.builder.mark_generations_as_merged(gens); + slf.builder.mark_sstables_as_compacted(compacted_sstables); Ok(slf) } } fn index_metadata_to_segment(metadata: IndexMetadata) -> PyResult { - let fragment_bitmap = metadata.fragment_bitmap.ok_or_else(|| { - PyValueError::new_err(format!( - "Index metadata {} is missing fragment coverage", - metadata.uuid - )) - })?; - let index_details = metadata.index_details.ok_or_else(|| { - PyValueError::new_err(format!( - "Index metadata {} is missing index details", - metadata.uuid - )) - })?; - - Ok(IndexSegment::new( - metadata.uuid, - fragment_bitmap.iter(), - index_details, - metadata.index_version, - )) + metadata.into_index_segment().infer_error() } fn extract_index_segments(segments: &Bound<'_, PyAny>) -> PyResult> { @@ -549,6 +616,23 @@ fn extract_index_segments(segments: &Bound<'_, PyAny>) -> PyResult>) -> PyResult>> { + index_segments + .map(|segments| { + segments + .into_iter() + .map(|segment| { + Uuid::parse_str(&segment).map_err(|err| { + PyValueError::new_err(format!( + "invalid index segment uuid '{segment}': {err}" + )) + }) + }) + .collect::>>() + }) + .transpose() +} + impl MergeInsertBuilder { fn build_stats<'a>(stats: &MergeStats, py: Python<'a>) -> PyResult> { let dict = PyDict::new(py); @@ -1019,7 +1103,13 @@ impl Dataset { #[getter(data_storage_version)] fn data_storage_version(&self) -> PyResult { - Ok(self.ds.manifest().data_storage_format.version.clone()) + Ok(self + .ds + .manifest() + .data_storage_format + .version + .to_manifest_string() + .to_string()) } #[getter(has_stable_row_ids)] @@ -1194,6 +1284,13 @@ impl Dataset { let is_phrase = query.len() >= 2 && query.starts_with('"') && query.ends_with('"'); let is_multi_match = columns.as_ref().map(|cols| cols.len() > 1).unwrap_or(false); + let document_granularity = match full_text_query.get_item("document_granularity")? { + Some(value) if !value.is_none() => Some( + DocumentGranularity::try_from(value.extract::()?.as_str()) + .map_err(|err| PyValueError::new_err(err.to_string()))?, + ), + _ => None, + }; if is_phrase { // Remove the surrounding quotes for phrase queries @@ -1201,8 +1298,20 @@ impl Dataset { } let query: FtsQuery = match (is_phrase, is_multi_match) { - (false, _) => MatchQuery::new(query).into(), - (true, false) => PhraseQuery::new(query).into(), + (false, _) => { + let mut query = MatchQuery::new(query); + if let Some(document_granularity) = document_granularity { + query = query.with_document_granularity(document_granularity); + } + query.into() + } + (true, false) => { + let mut query = PhraseQuery::new(query); + if let Some(document_granularity) = document_granularity { + query = query.with_document_granularity(document_granularity); + } + query.into() + } (true, true) => { return Err(PyValueError::new_err( "Phrase queries cannot be used with multiple columns.", @@ -1523,18 +1632,21 @@ impl Dataset { self_: PyRef<'_, Self>, row_ids: Vec, blob_column: &str, - ) -> PyResult> { + ) -> PyResult>> { let blobs = rt() .block_on(Some(self_.py()), self_.ds.take_blobs(&row_ids, blob_column))? .infer_error()?; - Ok(blobs.into_iter().map(LanceBlobFile::from).collect()) + Ok(blobs + .into_iter() + .map(|blob| blob.map(LanceBlobFile::from)) + .collect()) } fn take_blobs_by_addresses( self_: PyRef<'_, Self>, row_addresses: Vec, blob_column: &str, - ) -> PyResult> { + ) -> PyResult>> { let blobs = rt() .block_on( Some(self_.py()), @@ -1543,21 +1655,27 @@ impl Dataset { .take_blobs_by_addresses(&row_addresses, blob_column), )? .infer_error()?; - Ok(blobs.into_iter().map(LanceBlobFile::from).collect()) + Ok(blobs + .into_iter() + .map(|blob| blob.map(LanceBlobFile::from)) + .collect()) } fn take_blobs_by_indices( self_: PyRef<'_, Self>, row_indices: Vec, blob_column: &str, - ) -> PyResult> { + ) -> PyResult>> { let blobs = rt() .block_on( Some(self_.py()), self_.ds.take_blobs_by_indices(&row_indices, blob_column), )? .infer_error()?; - Ok(blobs.into_iter().map(LanceBlobFile::from).collect()) + Ok(blobs + .into_iter() + .map(|blob| blob.map(LanceBlobFile::from)) + .collect()) } #[pyo3(signature=( @@ -1572,7 +1690,7 @@ impl Dataset { blob_column: &str, io_buffer_size: Option, preserve_order: Option, - ) -> PyResult)>> { + ) -> PyResult> { let builder = configure_read_blobs_builder( self_ .ds @@ -1600,7 +1718,7 @@ impl Dataset { blob_column: &str, io_buffer_size: Option, preserve_order: Option, - ) -> PyResult)>> { + ) -> PyResult> { let builder = configure_read_blobs_builder( self_ .ds @@ -1628,7 +1746,7 @@ impl Dataset { blob_column: &str, io_buffer_size: Option, preserve_order: Option, - ) -> PyResult)>> { + ) -> PyResult> { let builder = configure_read_blobs_builder( self_ .ds @@ -1644,6 +1762,40 @@ impl Dataset { Ok(read_blobs_to_python(self_.py(), blobs)) } + #[pyo3(signature=( + requests, + blob_column, + selector, + io_buffer_size=None, + preserve_order=None + ))] + fn read_blob_ranges( + self_: PyRef<'_, Self>, + requests: Vec<(u64, u64, u64)>, + blob_column: &str, + selector: &str, + io_buffer_size: Option, + preserve_order: Option, + ) -> PyResult> { + let requests = blob_range_requests_from_tuples(requests); + let builder = self_.ds.read_blob_ranges(blob_column).infer_error()?; + let builder = match selector { + "ids" => builder.with_row_ids(requests), + "addresses" => builder.with_row_addresses(requests), + "indices" => builder.with_row_indices(requests), + selector => { + return Err(PyValueError::new_err(format!( + "selector must be one of 'ids', 'addresses', or 'indices', got {selector:?}" + ))); + } + }; + let builder = configure_read_blob_ranges_builder(builder, io_buffer_size, preserve_order); + let ranges = rt() + .block_on(Some(self_.py()), builder.execute())? + .infer_error()?; + Ok(read_blob_ranges_to_python(self_.py(), ranges)) + } + #[pyo3(signature = (row_slices, columns = None, batch_readahead = 10))] fn take_scan( &self, @@ -2404,48 +2556,58 @@ impl Dataset { "INVERTED" | "FTS" => { let mut params = InvertedIndexParams::default(); if let Some(kwargs) = kwargs { - if let Some(with_position) = kwargs.get_item("with_position")? { - params = params.with_position(with_position.extract()?); - } - if let Some(base_tokenizer) = kwargs.get_item("base_tokenizer")? { - params = params.base_tokenizer(base_tokenizer.extract()?); - } - if let Some(language) = kwargs.get_item("language")? { - let language: PyBackedStr = - language.cast::()?.clone().try_into()?; - params = params.language(&language).map_err(|e| { - PyValueError::new_err(format!( - "can't set tokenizer language to {}: {:?}", - language, e - )) - })?; - } - if let Some(max_token_length) = kwargs.get_item("max_token_length")? { - params = params.max_token_length(max_token_length.extract()?); - } - if let Some(lower_case) = kwargs.get_item("lower_case")? { - params = params.lower_case(lower_case.extract()?); - } - if let Some(stem) = kwargs.get_item("stem")? { - params = params.stem(stem.extract()?); - } - if let Some(remove_stop_words) = kwargs.get_item("remove_stop_words")? { - params = params.remove_stop_words(remove_stop_words.extract()?); - } - if let Some(stop_words_file) = kwargs.get_item("custom_stop_words")? { - params = params.custom_stop_words(stop_words_file.extract()?); - } - if let Some(ascii_folding) = kwargs.get_item("ascii_folding")? { - params = params.ascii_folding(ascii_folding.extract()?); + let allowed_kwargs = [ + "analyzer", + "document_granularity", + "with_position", + "base_tokenizer", + "language", + "max_token_length", + "lower_case", + "stem", + "remove_stop_words", + "custom_stop_words", + "ascii_folding", + "min_ngram_length", + "max_ngram_length", + "prefix_only", + "block_size", + "split_identifiers", + "split_on_numerics", + "preserve_original", + "index_operators", + "memory_limit", + "num_workers", + "format_version", + "fragment_ids", + "index_uuid", + "progress_callback", + ]; + for (key, _) in kwargs.iter() { + let key: String = key.extract()?; + if !allowed_kwargs.contains(&key.as_str()) { + return Err(PyValueError::new_err(format!( + "unknown FTS index parameter '{}'", + key + ))); + } } - if let Some(min_ngram_length) = kwargs.get_item("min_ngram_length")? { - params = params.ngram_min_length(min_ngram_length.extract()?); + + params = FtsTokenizerOptions::from_kwargs(kwargs)?.apply(params)?; + if let Some(document_granularity) = kwargs.get_item("document_granularity")? { + let document_granularity: String = document_granularity.extract()?; + params = params.document_granularity( + DocumentGranularity::try_from(document_granularity.as_str()) + .map_err(|err| PyValueError::new_err(err.to_string()))?, + ); } - if let Some(max_ngram_length) = kwargs.get_item("max_ngram_length")? { - params = params.ngram_max_length(max_ngram_length.extract()?); + if let Some(with_position) = kwargs.get_item("with_position")? { + params = params.with_position(with_position.extract()?); } - if let Some(prefix_only) = kwargs.get_item("prefix_only")? { - params = params.ngram_prefix_only(prefix_only.extract()?); + if let Some(block_size) = kwargs.get_item("block_size")? { + params = params + .block_size(block_size.extract()?) + .map_err(|e| PyValueError::new_err(e.to_string()))?; } if let Some(memory_limit) = kwargs.get_item("memory_limit")? { params = params.memory_limit_mb(memory_limit.extract()?); @@ -2462,7 +2624,7 @@ impl Dataset { value.to_string() } else { return Err(PyValueError::new_err( - "format_version must be 1, 2, 'v1', or 'v2'", + "format_version must be 1, 2, 3, 'v1', 'v2', or 'v3'", )); }; let format_version = value @@ -2600,18 +2762,31 @@ impl Dataset { Ok(()) } - #[pyo3(signature = (name, *, with_position = false))] - fn prewarm_index(&self, name: &str, with_position: bool) -> PyResult<()> { + #[pyo3(signature = (name, *, with_position = false, index_segments = None))] + fn prewarm_index( + &self, + name: &str, + with_position: bool, + index_segments: Option>, + ) -> PyResult<()> { + let index_segments = parse_index_segment_ids(index_segments)?; + rt().block_on(None, async { if with_position { - self.ds - .prewarm_index_with_options( - name, - &PrewarmOptions::Fts(FtsPrewarmOptions::new().with_position(true)), - ) - .await + let options = PrewarmOptions::Fts(FtsPrewarmOptions::new().with_position(true)); + if let Some(index_segments) = index_segments.as_deref() { + self.ds + .prewarm_index_segments_with_options(name, index_segments, &options) + .await + } else { + self.ds.prewarm_index_with_options(name, &options).await + } } else { - self.ds.prewarm_index(name).await + if let Some(index_segments) = index_segments.as_deref() { + self.ds.prewarm_index_segments(name, index_segments).await + } else { + self.ds.prewarm_index(name).await + } } })? .infer_error() @@ -3379,7 +3554,6 @@ impl Dataset { identity_column=None, unsharded=false, durable_write=None, - sync_indexed_write=None, max_wal_buffer_size=None, max_wal_flush_interval_ms=None, max_memtable_size=None, @@ -3387,8 +3561,6 @@ impl Dataset { max_memtable_batches=None, max_unflushed_memtable_bytes=None, manifest_scan_batch_size=None, - async_index_buffer_rows=None, - async_index_interval_ms=None, backpressure_log_interval_ms=None, stats_log_interval_ms=None, hnsw_params=None, @@ -3402,7 +3574,6 @@ impl Dataset { identity_column: Option, unsharded: bool, durable_write: Option, - sync_indexed_write: Option, max_wal_buffer_size: Option, max_wal_flush_interval_ms: Option, max_memtable_size: Option, @@ -3410,8 +3581,6 @@ impl Dataset { max_memtable_batches: Option, max_unflushed_memtable_bytes: Option, manifest_scan_batch_size: Option, - async_index_buffer_rows: Option, - async_index_interval_ms: Option, backpressure_log_interval_ms: Option, stats_log_interval_ms: Option, hnsw_params: Option>>, @@ -3439,7 +3608,6 @@ impl Dataset { let writer_config = writer_config_from_kwargs( durable_write, - sync_indexed_write, max_wal_buffer_size, max_wal_flush_interval_ms, max_memtable_size, @@ -3447,8 +3615,6 @@ impl Dataset { max_memtable_batches, max_unflushed_memtable_bytes, manifest_scan_batch_size, - async_index_buffer_rows, - async_index_interval_ms, backpressure_log_interval_ms, stats_log_interval_ms, hnsw_params, @@ -3530,7 +3696,6 @@ impl Dataset { shard_id, *, durable_write=None, - sync_indexed_write=None, max_wal_buffer_size=None, max_wal_flush_interval_ms=None, max_memtable_size=None, @@ -3538,8 +3703,6 @@ impl Dataset { max_memtable_batches=None, max_unflushed_memtable_bytes=None, manifest_scan_batch_size=None, - async_index_buffer_rows=None, - async_index_interval_ms=None, backpressure_log_interval_ms=None, stats_log_interval_ms=None, hnsw_params=None, @@ -3549,7 +3712,6 @@ impl Dataset { py: Python<'_>, shard_id: String, durable_write: Option, - sync_indexed_write: Option, max_wal_buffer_size: Option, max_wal_flush_interval_ms: Option, max_memtable_size: Option, @@ -3557,8 +3719,6 @@ impl Dataset { max_memtable_batches: Option, max_unflushed_memtable_bytes: Option, manifest_scan_batch_size: Option, - async_index_buffer_rows: Option, - async_index_interval_ms: Option, backpressure_log_interval_ms: Option, stats_log_interval_ms: Option, hnsw_params: Option>>, @@ -3570,7 +3730,6 @@ impl Dataset { let config = writer_config_from_kwargs( durable_write, - sync_indexed_write, max_wal_buffer_size, max_wal_flush_interval_ms, max_memtable_size, @@ -3578,8 +3737,6 @@ impl Dataset { max_memtable_batches, max_unflushed_memtable_bytes, manifest_scan_batch_size, - async_index_buffer_rows, - async_index_interval_ms, backpressure_log_interval_ms, stats_log_interval_ms, hnsw_params, @@ -3603,9 +3760,10 @@ impl Dataset { /// Perform pairwise hamming distance clustering on a partition of an IVF_FLAT index. /// - /// This function loads a specific partition from an IVF_FLAT index on a hash column, - /// computes pairwise hamming distances between all hashes in the partition, - /// filters by threshold, and clusters the results using union-find. + /// This function loads a specific partition from every segment of an IVF_FLAT + /// index on a hash column, computes pairwise hamming distances between all + /// hashes in the combined partition, filters by threshold, and clusters the + /// results using union-find. /// /// Parameters /// ---------- @@ -3615,6 +3773,9 @@ impl Dataset { /// The partition ID within the IVF_FLAT index /// hamming_threshold : int /// Maximum hamming distance to consider as similar + /// index_segments : list of str, optional + /// If specified, only these physical index segment UUIDs of the named + /// logical index contribute rows. Defaults to all segments. /// /// Returns /// ------- @@ -3622,27 +3783,45 @@ impl Dataset { /// A reader yielding batches with columns: /// - 'representative': uint64 - The representative row ID for each cluster /// - 'duplicates': list - List of duplicate row IDs in each cluster - #[pyo3(signature = (index_name, partition_id, hamming_threshold))] + #[pyo3(signature = (index_name, partition_id, hamming_threshold, index_segments=None))] fn hamming_clustering_for_ivf_partition( &self, py: Python<'_>, index_name: &str, partition_id: usize, hamming_threshold: u32, + index_segments: Option>, ) -> PyResult>> { - use lance::index::vector::hamming::hamming_clustering_for_ivf_partition; + use lance::index::vector::hamming::{ + hamming_clustering_for_ivf_partition, hamming_clustering_for_ivf_partition_segments, + }; + let segment_ids = parse_index_segment_ids(index_segments)?; let ds = self.ds.as_ref(); let reader = rt() - .block_on( - Some(py), - hamming_clustering_for_ivf_partition( - ds, - index_name, - partition_id, - hamming_threshold, - ), - )? + .block_on(Some(py), async { + match segment_ids.as_deref() { + Some(segment_ids) => { + hamming_clustering_for_ivf_partition_segments( + ds, + index_name, + segment_ids, + partition_id, + hamming_threshold, + ) + .await + } + None => { + hamming_clustering_for_ivf_partition( + ds, + index_name, + partition_id, + hamming_threshold, + ) + .await + } + } + })? .map_err(|err| PyValueError::new_err(err.to_string()))?; Ok(PyArrowType(reader)) @@ -3650,26 +3829,43 @@ impl Dataset { /// Get partition information for an IVF_FLAT index. /// + /// Partition sizes are aggregated across all segments of the logical index + /// unless a subset is selected via ``index_segments``. + /// /// Parameters /// ---------- /// index_name : str /// Name of the IVF_FLAT index + /// index_segments : list of str, optional + /// If specified, only these physical index segment UUIDs of the named + /// logical index contribute to the sizes. Defaults to all segments. /// /// Returns /// ------- /// List[dict] /// List of partition info dicts with 'partition_id' and 'size' - #[pyo3(signature = (index_name))] + #[pyo3(signature = (index_name, index_segments=None))] fn get_ivf_partition_info( &self, py: Python<'_>, index_name: &str, + index_segments: Option>, ) -> PyResult>> { - use lance::index::vector::hamming::get_ivf_partition_info; + use lance::index::vector::hamming::{ + get_ivf_partition_info, get_ivf_partition_info_segments, + }; + let segment_ids = parse_index_segment_ids(index_segments)?; let ds = self.ds.as_ref(); let result = rt() - .block_on(Some(py), get_ivf_partition_info(ds, index_name))? + .block_on(Some(py), async { + match segment_ids.as_deref() { + Some(segment_ids) => { + get_ivf_partition_info_segments(ds, index_name, segment_ids).await + } + None => get_ivf_partition_info(ds, index_name).await, + } + })? .map_err(|err| PyValueError::new_err(err.to_string()))?; let partitions: PyResult> = result @@ -3694,7 +3890,8 @@ impl Dataset { /// Parameters /// ---------- /// column : str - /// Name of the hash column (must be FixedSizeList) + /// Name of the hash column (must be FixedSizeList where N is + /// a positive multiple of 8 bytes) /// sample_size : int, optional /// Number of rows to sample (if None or >= total rows, uses all rows) /// hamming_threshold : int @@ -3737,7 +3934,8 @@ impl Dataset { /// Parameters /// ---------- /// column : str - /// Name of the hash column (must be FixedSizeList) + /// Name of the hash column (must be FixedSizeList where N is + /// a positive multiple of 8 bytes) /// fragment_id : int /// The fragment ID to read from /// start_row : int @@ -3878,6 +4076,23 @@ impl SqlQueryBuilder { } } + #[pyo3(signature = (blob_handling))] + fn blob_handling(&self, blob_handling: &str) -> PyResult { + let blob_handling = match blob_handling { + "all_binary" => BlobHandling::AllBinary, + "blobs_descriptions" => BlobHandling::BlobsDescriptions, + "all_descriptions" => BlobHandling::AllDescriptions, + other => { + return Err(PyValueError::new_err(format!( + "Invalid blob_handling: {other}. Expected one of: all_binary, blobs_descriptions, all_descriptions" + ))); + } + }; + Ok(Self { + builder: self.builder.clone().blob_handling(blob_handling), + }) + } + /// Build the SQL query. fn build(&self) -> PyResult { Ok(SqlQuery { @@ -4444,6 +4659,9 @@ pub fn get_write_params( if let Some(progress) = get_dict_opt::>(options, "progress")? { p.progress = Arc::new(PyWriteProgress::new(progress.into_py_any(options.py())?)); } + if let Some(session) = get_dict_opt::(options, "session")? { + p.session = Some(session.inner.clone()); + } let storage_options = get_dict_opt::>(options, "storage_options")?; @@ -5005,7 +5223,8 @@ pub struct PyFullTextQuery { #[pymethods] impl PyFullTextQuery { #[staticmethod] - #[pyo3(signature = (query, column, boost=1.0, fuzziness=Some(0), max_expansions=50, operator="OR", prefix_length=0))] + #[pyo3(signature = (query, column, boost=1.0, fuzziness=Some(0), max_expansions=50, operator="OR", prefix_length=0, document_granularity=None))] + #[allow(clippy::too_many_arguments)] fn match_query( query: String, column: String, @@ -5014,30 +5233,48 @@ impl PyFullTextQuery { max_expansions: usize, operator: &str, prefix_length: u32, + document_granularity: Option<&str>, ) -> PyResult { + let mut query = MatchQuery::new(query) + .with_column(Some(column)) + .with_boost(boost) + .with_fuzziness(fuzziness) + .with_max_expansions(max_expansions) + .with_operator( + Operator::try_from(operator) + .map_err(|e| PyValueError::new_err(format!("Invalid operator: {}", e)))?, + ) + .with_prefix_length(prefix_length); + if let Some(document_granularity) = document_granularity { + query = query.with_document_granularity( + DocumentGranularity::try_from(document_granularity) + .map_err(|err| PyValueError::new_err(err.to_string()))?, + ); + } Ok(Self { - inner: MatchQuery::new(query) - .with_column(Some(column)) - .with_boost(boost) - .with_fuzziness(fuzziness) - .with_max_expansions(max_expansions) - .with_operator( - Operator::try_from(operator) - .map_err(|e| PyValueError::new_err(format!("Invalid operator: {}", e)))?, - ) - .with_prefix_length(prefix_length) - .into(), + inner: query.into(), }) } #[staticmethod] - #[pyo3(signature = (query, column, slop))] - fn phrase_query(query: String, column: String, slop: u32) -> PyResult { + #[pyo3(signature = (query, column, slop, document_granularity=None))] + fn phrase_query( + query: String, + column: String, + slop: u32, + document_granularity: Option<&str>, + ) -> PyResult { + let mut query = PhraseQuery::new(query) + .with_column(Some(column)) + .with_slop(slop); + if let Some(document_granularity) = document_granularity { + query = query.with_document_granularity( + DocumentGranularity::try_from(document_granularity) + .map_err(|err| PyValueError::new_err(err.to_string()))?, + ); + } Ok(Self { - inner: PhraseQuery::new(query) - .with_column(Some(column)) - .with_slop(slop) - .into(), + inner: query.into(), }) } diff --git a/python/src/dataset/blob.rs b/python/src/dataset/blob.rs index 1f6272075f2..7952ce76fb1 100644 --- a/python/src/dataset/blob.rs +++ b/python/src/dataset/blob.rs @@ -79,6 +79,32 @@ impl LanceBlobFile { Ok(PyBytes::new(py, &data)) } + /// Read multiple blob-local `(offset, length)` ranges without changing the current cursor. + pub fn read_ranges<'py>( + &self, + py: Python<'py>, + ranges: Vec<(u64, u64)>, + ) -> PyResult>> { + let ranges = ranges + .into_iter() + .enumerate() + .map(|(i, (offset, length))| { + let end = offset.checked_add(length).ok_or_else(|| { + PyValueError::new_err(format!( + "Blob range request {i} offset + length overflowed u64: \ + offset={offset}, length={length}" + )) + })?; + Ok(offset..end) + }) + .collect::>>()?; + let inner = self.inner.clone(); + let data = rt() + .block_on(Some(py), inner.read_ranges(&ranges))? + .infer_error()?; + Ok(data.iter().map(|bytes| PyBytes::new(py, bytes)).collect()) + } + pub fn read_into(&self, dst: Bound<'_, PyByteArray>) -> PyResult { let inner = self.inner.clone(); diff --git a/python/src/dataset/commit.rs b/python/src/dataset/commit.rs index 41a0ef2f69a..bb904f89bf7 100644 --- a/python/src/dataset/commit.rs +++ b/python/src/dataset/commit.rs @@ -23,8 +23,7 @@ use pyo3::{exceptions::PyIOError, prelude::*}; static PY_CONFLICT_ERROR: LazyLock>> = LazyLock::new(|| { Python::attach(|py| { - py.import("lance") - .and_then(|lance| lance.getattr("commit")) + py.import("lance.commit") .and_then(|commit| commit.getattr("CommitConflictError")) .map(|err| err.unbind()) }) diff --git a/python/src/executor.rs b/python/src/executor.rs index e2e573a8193..e2c1ca4b383 100644 --- a/python/src/executor.rs +++ b/python/src/executor.rs @@ -19,6 +19,10 @@ use pyo3::{PyResult, Python, exceptions::PyRuntimeError}; pub const SIGNAL_CHECK_INTERVAL: std::time::Duration = std::time::Duration::from_millis(100); +fn is_python314_or_later(py: Option>) -> bool { + py.is_some_and(|py| py.version_info() >= (3, 14)) +} + /// A wrapper around tokio runtime. /// /// This is used to spawn tasks in the background and wait synchronously for them @@ -195,6 +199,8 @@ impl BackgroundExecutor { F::Output: Send, P: FnMut() -> PyResult<()>, { + let should_propagate_on_completion = is_python314_or_later(py); + let mut future = std::pin::pin!(future); loop { @@ -236,11 +242,22 @@ impl BackgroundExecutor { }; if let Some(output) = maybe_output { - if let Err(err) = pump() { - log::warn!( - "Ignoring progress callback error after operation completed successfully: {}", - err - ); + // When the index build finishes so fast that no pump cycles + // occurred during execution, pending events sit in the channel + // buffer and get drained after completion. Python ≥ 3.14 changed + // GIL/async scheduling timing such that callback invocations may + // only be visible in this post-completion drain, so we propagate + // errors for 3.14+. For ≤ 3.13 we keep the old tolerant behavior + // to avoid spurious failures when scheduling shifts slightly. + if should_propagate_on_completion { + pump()?; + } else { + if let Err(err) = pump() { + log::warn!( + "Ignoring progress callback error after operation completed successfully: {}", + err + ); + } } return Ok(output); } diff --git a/python/src/file.rs b/python/src/file.rs index 2a3dd09e17f..8b3511496b0 100644 --- a/python/src/file.rs +++ b/python/src/file.rs @@ -26,10 +26,9 @@ use lance_core::utils::path::LancePathExt; use lance_encoding::decoder::{DecoderPlugins, FilterExpression}; use lance_file::reader::{ BufferDescriptor, CachedFileMetadata, FileReader, FileReaderOptions, FileStatistics, - ReaderProjection, }; use lance_file::writer::{FileWriter, FileWriterOptions}; -use lance_file::{LanceEncodingsIo, version::LanceFileVersion}; +use lance_file::{LanceEncodingsIo, version::LanceFileVersion, versions as file_versions}; use lance_io::object_store::{LanceNamespaceStorageOptionsProvider, ObjectStoreParams}; use lance_io::{ ReadBatchParams, @@ -230,6 +229,26 @@ impl LanceFileMetadata { } } +/// Summary of a completed Lance file write +#[pyclass(get_all, skip_from_py_object)] +#[derive(Clone, Debug, Serialize)] +pub struct LanceFileWriteSummary { + /// The number of rows written to the file + pub num_rows: u64, + /// The final size of the file, in bytes + pub size_bytes: u64, +} + +#[pymethods] +impl LanceFileWriteSummary { + fn __repr__(&self) -> String { + format!( + "FileWriteSummary(num_rows={}, size_bytes={})", + self.num_rows, self.size_bytes + ) + } +} + #[pyclass] pub struct LanceFileWriter { inner: Arc>>, @@ -275,21 +294,23 @@ impl LanceFileWriter { max_page_bytes: Option, ) -> PyResult { let object_writer = object_store.create(&path).await.infer_error()?; + let version = version + .map(|value| value.parse::()) + .transpose() + .infer_error()? + .unwrap_or_default() + .resolve(); let options = FileWriterOptions { data_cache_bytes, keep_original_array, max_page_bytes, - format_version: version - .map(|v| v.parse::()) - .transpose() - .infer_error()?, - ..Default::default() }; let inner = if let Some(schema) = schema { let lance_schema = lance_core::datatypes::Schema::try_from(&schema.0).infer_error()?; - FileWriter::try_new(object_writer, lance_schema, options).infer_error() + file_versions::create_writer(version, object_writer, lance_schema, options) + .infer_error() } else { - Ok(FileWriter::new_lazy(object_writer, options)) + file_versions::create_lazy_writer(version, object_writer, options).infer_error() }?; Ok(Self { inner: Arc::new(Mutex::new(Box::new(inner))), @@ -347,11 +368,18 @@ impl LanceFileWriter { .infer_error() } - pub fn finish(&self) -> PyResult { - rt().block_on(None, async { - self.inner.lock().await.finish().await.map(|s| s.num_rows) - })? - .infer_error() + /// Finish the file and return the row count and the final file size + /// + /// The size is reported by the object writer once the file has been closed + /// and so it is accurate for object stores as well as local filesystems. + pub fn finish(&self) -> PyResult { + let summary = rt() + .block_on(None, async { self.inner.lock().await.finish().await })? + .infer_error()?; + Ok(LanceFileWriteSummary { + num_rows: summary.num_rows, + size_bytes: summary.size_bytes, + }) } pub fn add_global_buffer(&self, bytes: Vec) -> PyResult { @@ -817,7 +845,7 @@ impl LanceFileReader { let mut base_projection = None; if let Some(columns) = columns { base_projection = Some( - ReaderProjection::from_column_names( + file_versions::reader_projection_from_column_names( file_metadata.version(), &file_metadata.file_schema, &columns.iter().map(|s| s.as_str()).collect::>(), diff --git a/python/src/fragment.rs b/python/src/fragment.rs index dbe5c426903..6c52673432c 100644 --- a/python/src/fragment.rs +++ b/python/src/fragment.rs @@ -26,6 +26,7 @@ use lance::dataset::transaction::{Operation, Transaction}; use lance::dataset::{InsertBuilder, NewColumnTransform, WriteParams}; use lance_core::datatypes::BlobHandling; use lance_io::utils::CachedFileSize; +use lance_table::format::overlay::DataOverlayFile; use lance_table::format::{ DataFile, DeletionFile, DeletionFileType, Fragment, RowDatasetVersionMeta, RowIdMeta, }; @@ -452,6 +453,11 @@ impl FileFragment { rt().block_on(None, self.fragment.physical_rows())? .map_err(|err| PyIOError::new_err(err.to_string())) } + + fn validate(&self) -> PyResult<()> { + rt().block_on(None, self.fragment.validate())? + .map_err(|err| PyIOError::new_err(err.to_string())) + } } impl From for LanceFragment { @@ -825,6 +831,10 @@ impl FromPyObject<'_, '_> for PyLance { row_id_meta, last_updated_at_version_meta, created_at_version_meta, + // Round-tripped so overlays survive operations that pass existing + // fragments back (a manual Delete/Update/Merge commit). Sorting + // newest-last is deferred to the manifest reload after commit. + overlays: extract_vec::(&ob.getattr("overlays")?)?, })) } } @@ -857,6 +867,7 @@ impl<'py> IntoPyObject<'py> for PyLance<&Fragment> { .created_at_version_meta .as_ref() .map(|r| PyRowDatasetVersionMeta(r.clone())); + let overlays = export_vec(py, &self.0.overlays)?; cls.call1(( self.0.id, @@ -866,6 +877,7 @@ impl<'py> IntoPyObject<'py> for PyLance<&Fragment> { row_id_meta, created_at_version_meta, last_updated_at_version_meta, + overlays, )) } } diff --git a/python/src/fts.rs b/python/src/fts.rs new file mode 100644 index 00000000000..c24cefd6300 --- /dev/null +++ b/python/src/fts.rs @@ -0,0 +1,258 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright The Lance Authors + +use lance_index::scalar::InvertedIndexParams; +use lance_index::scalar::inverted::query::collect_query_tokens; +use pyo3::exceptions::PyValueError; +use pyo3::prelude::*; +use pyo3::types::PyDict; + +fn extract_kwarg<'py, T>(kwargs: &Bound<'py, PyDict>, key: &str) -> PyResult> +where + T: FromPyObjectOwned<'py>, +{ + kwargs + .get_item(key)? + .map(|value| value.extract::().map_err(Into::into)) + .transpose() +} + +pub(crate) struct FtsTokenizerOptions { + analyzer: Option, + base_tokenizer: Option, + language: Option, + // The outer option tracks omission; the inner None explicitly disables the limit. + max_token_length: Option>, + lower_case: Option, + stem: Option, + remove_stop_words: Option, + // Preserve omitted versus explicit None to match create-index keyword semantics. + custom_stop_words: Option>>, + ascii_folding: Option, + min_ngram_length: Option, + max_ngram_length: Option, + prefix_only: Option, + split_identifiers: Option, + split_on_numerics: Option, + preserve_original: Option, + index_operators: Option, +} + +impl FtsTokenizerOptions { + pub(crate) fn from_kwargs(kwargs: &Bound<'_, PyDict>) -> PyResult { + Ok(Self { + analyzer: extract_kwarg(kwargs, "analyzer")?, + base_tokenizer: extract_kwarg(kwargs, "base_tokenizer")?, + language: extract_kwarg(kwargs, "language")?, + max_token_length: extract_kwarg(kwargs, "max_token_length")?, + lower_case: extract_kwarg(kwargs, "lower_case")?, + stem: extract_kwarg(kwargs, "stem")?, + remove_stop_words: extract_kwarg(kwargs, "remove_stop_words")?, + custom_stop_words: extract_kwarg(kwargs, "custom_stop_words")?, + ascii_folding: extract_kwarg(kwargs, "ascii_folding")?, + min_ngram_length: extract_kwarg(kwargs, "min_ngram_length")?, + max_ngram_length: extract_kwarg(kwargs, "max_ngram_length")?, + prefix_only: extract_kwarg(kwargs, "prefix_only")?, + split_identifiers: extract_kwarg(kwargs, "split_identifiers")?, + split_on_numerics: extract_kwarg(kwargs, "split_on_numerics")?, + preserve_original: extract_kwarg(kwargs, "preserve_original")?, + index_operators: extract_kwarg(kwargs, "index_operators")?, + }) + } + + pub(crate) fn apply(self, mut params: InvertedIndexParams) -> PyResult { + match (self.analyzer.as_deref(), self.base_tokenizer.as_deref()) { + (Some("text"), Some("code")) => { + return Err(PyValueError::new_err( + "base_tokenizer='code' requires analyzer='code'", + )); + } + (Some("code"), Some(base_tokenizer)) if base_tokenizer != "code" => { + return Err(PyValueError::new_err(format!( + "analyzer='code' requires base_tokenizer='code', got '{base_tokenizer}'" + ))); + } + _ => {} + } + + let uses_code_analyzer = match self.analyzer.as_deref() { + Some("code") => true, + Some("text") | None => self.base_tokenizer.as_deref() == Some("code"), + Some(_) => true, + }; + if !uses_code_analyzer + && [ + self.split_identifiers, + self.split_on_numerics, + self.preserve_original, + self.index_operators, + ] + .into_iter() + .flatten() + .any(|value| value) + { + return Err(PyValueError::new_err( + "code analyzer flags require analyzer='code'", + )); + } + + if let Some(analyzer) = self.analyzer { + params = params + .analyzer(&analyzer) + .map_err(|err| PyValueError::new_err(err.to_string()))?; + } + if let Some(base_tokenizer) = self.base_tokenizer { + params = params.base_tokenizer(base_tokenizer); + } + if let Some(language) = self.language { + params = params.language(&language).map_err(|err| { + PyValueError::new_err(format!("can't set tokenizer language to {language}: {err}")) + })?; + } + if let Some(max_token_length) = self.max_token_length { + params = params.max_token_length(max_token_length); + } + if let Some(lower_case) = self.lower_case { + params = params.lower_case(lower_case); + } + if let Some(stem) = self.stem { + params = params.stem(stem); + } + if let Some(remove_stop_words) = self.remove_stop_words { + params = params.remove_stop_words(remove_stop_words); + } + if let Some(custom_stop_words) = self.custom_stop_words { + params = params.custom_stop_words(custom_stop_words); + } + if let Some(ascii_folding) = self.ascii_folding { + params = params.ascii_folding(ascii_folding); + } + if let Some(min_ngram_length) = self.min_ngram_length { + params = params.ngram_min_length(min_ngram_length); + } + if let Some(max_ngram_length) = self.max_ngram_length { + params = params.ngram_max_length(max_ngram_length); + } + if let Some(prefix_only) = self.prefix_only { + params = params.ngram_prefix_only(prefix_only); + } + if let Some(split_identifiers) = self.split_identifiers { + params = params.split_identifiers(split_identifiers); + } + if let Some(split_on_numerics) = self.split_on_numerics { + params = params.split_on_numerics(split_on_numerics); + } + if let Some(preserve_original) = self.preserve_original { + params = params.preserve_original(preserve_original); + } + if let Some(index_operators) = self.index_operators { + params = params.index_operators(index_operators); + } + Ok(params) + } +} + +/// A token produced by the full-text search query tokenizer. +#[pyclass(get_all, skip_from_py_object)] +#[derive(Clone, Debug)] +pub struct FtsToken { + /// The token text after all configured filters have been applied. + pub text: String, + /// The position used by full-text query matching. + pub position: u32, +} + +#[pymethods] +impl FtsToken { + fn __repr__(&self) -> String { + format!("FtsToken(text={:?}, position={})", self.text, self.position) + } +} + +/// Tokenize a full-text search query without creating a dataset or index. +/// +/// The tokenizer options are the same as the tokenizer-related options accepted by +/// ``LanceDataset.create_scalar_index(..., index_type="INVERTED")``. ``None`` uses +/// the selected analyzer profile's default, except ``max_token_length=None``, which +/// disables the length limit; omitting it keeps the default limit of 40. Returned +/// positions are normalized to the first retained token while preserving gaps left +/// by token filters. +/// +/// Examples +/// -------- +/// >>> import lance +/// >>> tokens = lance.tokenize("the Cats and Dogs") +/// >>> [(token.text, token.position) for token in tokens] +/// [('cat', 0), ('dog', 2)] +#[pyfunction] +#[pyo3(signature = ( + query, + *, + analyzer = None, + base_tokenizer = None, + language = None, + max_token_length = Some(40), + lower_case = None, + stem = None, + remove_stop_words = None, + custom_stop_words = None, + ascii_folding = None, + min_ngram_length = None, + max_ngram_length = None, + prefix_only = None, + split_identifiers = None, + split_on_numerics = None, + preserve_original = None, + index_operators = None, +))] +#[allow(clippy::too_many_arguments)] +pub fn tokenize( + query: &str, + analyzer: Option, + base_tokenizer: Option, + language: Option, + max_token_length: Option, + lower_case: Option, + stem: Option, + remove_stop_words: Option, + custom_stop_words: Option>, + ascii_folding: Option, + min_ngram_length: Option, + max_ngram_length: Option, + prefix_only: Option, + split_identifiers: Option, + split_on_numerics: Option, + preserve_original: Option, + index_operators: Option, +) -> PyResult> { + let params = FtsTokenizerOptions { + analyzer, + base_tokenizer, + language, + max_token_length: Some(max_token_length), + lower_case, + stem, + remove_stop_words, + custom_stop_words: custom_stop_words.map(Some), + ascii_folding, + min_ngram_length, + max_ngram_length, + prefix_only, + split_identifiers, + split_on_numerics, + preserve_original, + index_operators, + } + .apply(InvertedIndexParams::default())?; + + let mut tokenizer = params + .build() + .map_err(|err| PyValueError::new_err(format!("Failed to build tokenizer: {err}")))?; + let tokens = collect_query_tokens(query, &mut tokenizer); + Ok((0..tokens.len()) + .map(|index| FtsToken { + text: tokens.get_token(index).to_string(), + position: tokens.position(index), + }) + .collect()) +} diff --git a/python/src/indices.rs b/python/src/indices.rs index 7ce7a297924..28f5affa9b9 100644 --- a/python/src/indices.rs +++ b/python/src/indices.rs @@ -232,6 +232,7 @@ async fn do_train_pq_model( distance_type: &str, sample_rate: u32, max_iters: u32, + num_bits: u32, ivf_model: IvfModel, fragment_ids: Option>, ) -> PyResult { @@ -239,7 +240,7 @@ async fn do_train_pq_model( let distance_type = DistanceType::try_from(distance_type).unwrap(); let params = PQBuildParams { num_sub_vectors: num_subvectors as usize, - num_bits: 8, + num_bits: num_bits as usize, max_iters: max_iters as usize, sample_rate: sample_rate as usize, ..Default::default() @@ -260,7 +261,7 @@ async fn do_train_pq_model( #[pyfunction] #[allow(clippy::too_many_arguments)] -#[pyo3(signature=(dataset, column, dimension, num_subvectors, distance_type, sample_rate, max_iters, ivf_centroids, fragment_ids=None))] +#[pyo3(signature=(dataset, column, dimension, num_subvectors, distance_type, sample_rate, max_iters, ivf_centroids, fragment_ids=None, num_bits=8))] fn train_pq_model<'py>( py: Python<'py>, dataset: &Dataset, @@ -272,6 +273,7 @@ fn train_pq_model<'py>( max_iters: u32, ivf_centroids: PyArrowType, fragment_ids: Option>, + num_bits: u32, ) -> PyResult> { let ivf_centroids = ivf_centroids.0; let ivf_centroids = FixedSizeListArray::from(ivf_centroids); @@ -291,6 +293,7 @@ fn train_pq_model<'py>( distance_type, sample_rate, max_iters, + num_bits, ivf_model, fragment_ids, ), @@ -398,7 +401,7 @@ async fn do_transform_vectors( #[pyfunction] #[allow(clippy::too_many_arguments)] -#[pyo3(signature=(dataset, column, dimension, num_subvectors, distance_type, ivf_centroids, pq_codebook, dst_uri, fragments, partitions_ds_uri=None))] +#[pyo3(signature=(dataset, column, dimension, num_subvectors, distance_type, ivf_centroids, pq_codebook, dst_uri, fragments, partitions_ds_uri=None, num_bits=8))] pub fn transform_vectors( py: Python<'_>, dataset: &Dataset, @@ -411,6 +414,7 @@ pub fn transform_vectors( dst_uri: &str, fragments: Vec, partitions_ds_uri: Option<&str>, + num_bits: u32, ) -> PyResult<()> { let ivf_centroids = ivf_centroids.0; let ivf_centroids = FixedSizeListArray::from(ivf_centroids); @@ -419,7 +423,7 @@ pub fn transform_vectors( let distance_type = DistanceType::try_from(distance_type).unwrap(); let pq = ProductQuantizer::new( num_subvectors as usize, - /*num_bits=*/ 8, + num_bits, dimension, codebook, distance_type, @@ -539,21 +543,7 @@ async fn do_load_shuffled_vectors( base_id: None, files: Some(files), }; - let segment = IndexSegment::new( - metadata.uuid, - metadata - .fragment_bitmap - .as_ref() - .expect("vector metadata should include fragment coverage") - .iter(), - metadata - .index_details - .as_ref() - .expect("vector metadata should include index details") - .clone(), - metadata.index_version, - ); - ds.commit_existing_index_segments(index_name, column, vec![segment]) + ds.commit_existing_index_segments(index_name, column, vec![metadata]) .await .infer_error()?; @@ -561,7 +551,7 @@ async fn do_load_shuffled_vectors( } #[pyfunction] -#[pyo3(signature=(filenames, dir_path, dataset, column, ivf_centroids, pq_codebook, pq_dimension, num_subvectors, distance_type, index_name=None))] +#[pyo3(signature=(filenames, dir_path, dataset, column, ivf_centroids, pq_codebook, pq_dimension, num_subvectors, distance_type, index_name=None, num_bits=8))] #[allow(clippy::too_many_arguments)] pub fn load_shuffled_vectors( filenames: Vec, @@ -574,6 +564,7 @@ pub fn load_shuffled_vectors( num_subvectors: u32, distance_type: &str, index_name: Option<&str>, + num_bits: u32, ) -> PyResult<()> { let mut default_idx_name = column.to_string(); default_idx_name.push_str("_idx"); @@ -595,7 +586,7 @@ pub fn load_shuffled_vectors( let distance_type = DistanceType::try_from(distance_type).unwrap(); let pq_model = ProductQuantizer::new( num_subvectors as usize, - /*num_bits=*/ 8, + num_bits, pq_dimension, codebook, distance_type, @@ -701,7 +692,7 @@ impl PyIndexDescription { .map(|field| { dataset .schema() - .field_path(*field as i32) + .field_path_minimal(*field as i32) .unwrap_or_else(|_| "".to_string()) }) .collect(); diff --git a/python/src/lib.rs b/python/src/lib.rs index 466d4ea90f2..9fa5a05e2bf 100644 --- a/python/src/lib.rs +++ b/python/src/lib.rs @@ -49,7 +49,7 @@ use dataset::{DatasetBasePath, MergeInsertBuilder, PyFullTextQuery, PySearchFilt use env_logger::{Builder, Env}; use file::{ LanceBufferDescriptor, LanceColumnMetadata, LanceFileMetadata, LanceFileReader, - LanceFileStatistics, LanceFileWriter, LancePageMetadata, stable_version, + LanceFileStatistics, LanceFileWriteSummary, LanceFileWriter, LancePageMetadata, stable_version, }; use log::Level; use pyo3::exceptions::PyIOError; @@ -71,10 +71,13 @@ pub(crate) mod error; pub(crate) mod executor; pub(crate) mod file; pub(crate) mod fragment; +pub(crate) mod fts; pub(crate) mod indices; pub(crate) mod mem_wal; pub(crate) mod namespace; +pub(crate) mod otel; pub(crate) mod reader; +pub(crate) mod rowids; pub(crate) mod scanner; pub(crate) mod schema; pub(crate) mod session; @@ -95,6 +98,7 @@ pub use dataset::write_dataset; use fragment::{FileFragment, PyDeletionFile, PyRowDatasetVersionMeta, PyRowIdMeta}; pub use indices::register_indices; pub use reader::LanceReader; +use rowids::{PyRowIdSequence, PyRowIdSequenceIterator}; pub use scanner::Scanner; use crate::blob::{ @@ -258,6 +262,8 @@ fn lance(py: Python, m: &Bound<'_, PyModule>) -> PyResult<()> { m.add_class::()?; m.add_class::()?; m.add_class::()?; + m.add_class::()?; + m.add_class::()?; m.add_class::()?; m.add_class::()?; m.add_class::()?; @@ -267,6 +273,7 @@ fn lance(py: Python, m: &Bound<'_, PyModule>) -> PyResult<()> { m.add_class::()?; m.add_class::()?; m.add_class::()?; + m.add_class::()?; m.add_class::()?; m.add_class::()?; m.add_class::()?; @@ -290,6 +297,7 @@ fn lance(py: Python, m: &Bound<'_, PyModule>) -> PyResult<()> { m.add_class::()?; m.add_class::()?; m.add_class::()?; + m.add_class::()?; m.add_class::()?; m.add_class::()?; m.add_class::()?; @@ -298,7 +306,7 @@ fn lance(py: Python, m: &Bound<'_, PyModule>) -> PyResult<()> { m.add_class::()?; m.add_class::()?; // MemWAL classes - m.add_class::()?; + m.add_class::()?; m.add_class::()?; m.add_class::()?; m.add_class::()?; @@ -318,10 +326,18 @@ fn lance(py: Python, m: &Bound<'_, PyModule>) -> PyResult<()> { m.add_wrapped(wrap_pyfunction!(trace_to_chrome))?; m.add_wrapped(wrap_pyfunction!(capture_trace_events))?; m.add_wrapped(wrap_pyfunction!(shutdown_tracing))?; + m.add_wrapped(wrap_pyfunction!(fts::tokenize))?; + // OpenTelemetry metrics bridge + m.add_class::()?; + m.add_class::()?; + m.add_wrapped(wrap_pyfunction!(otel::register_lance_metrics_recorder))?; + m.add_wrapped(wrap_pyfunction!(otel::lance_metrics_catalog))?; + m.add_wrapped(wrap_pyfunction!(otel::snapshot_lance_metrics))?; m.add_wrapped(wrap_pyfunction!(manifest_needs_migration))?; m.add_wrapped(wrap_pyfunction!(language_model_home))?; m.add_wrapped(wrap_pyfunction!(bytes_read_counter))?; m.add_wrapped(wrap_pyfunction!(iops_counter))?; + m.add_wrapped(wrap_pyfunction!(simd_info))?; m.add_wrapped(wrap_pyfunction!(stable_version))?; // Debug functions m.add_wrapped(wrap_pyfunction!(debug::format_schema))?; @@ -340,6 +356,37 @@ fn iops_counter() -> PyResult { Ok(::lance::io::iops_counter()) } +/// Returns a dict describing which SIMD tier the lance runtime dispatches to +/// on this host, plus the raw CPU feature flags it detected. +/// +/// Mirrors `pyarrow.runtime_info()`: a cheap, transparent way to verify that +/// the host is hitting the expected SIMD tier (e.g., `"avx512_fp16"`, +/// `"avx2"`) when debugging vector-search performance. +/// +/// Returns: +/// { +/// "tier": str, # e.g. "avx2", "avx_fma", "neon", "none" +/// "target_arch": str, # e.g. "x86_64", "aarch64", "loongarch64" +/// "host_features": list[str], # raw CPU feature flags (x86_64 only) +/// } +/// +/// Examples: +/// >>> import lance +/// >>> info = lance.simd_info() +/// >>> sorted(info) +/// ['host_features', 'target_arch', 'tier'] +/// >>> isinstance(info["tier"], str) +/// True +#[pyfunction] +pub fn simd_info(py: Python<'_>) -> PyResult> { + let info = lance_core::utils::cpu::simd_info(); + let dict = pyo3::types::PyDict::new(py); + dict.set_item("tier", info.tier.to_string())?; + dict.set_item("target_arch", info.target_arch)?; + dict.set_item("host_features", info.host_features)?; + Ok(dict.into()) +} + #[pyfunction(name = "bytes_read_counter")] fn bytes_read_counter() -> PyResult { Ok(::lance::io::bytes_read_counter()) diff --git a/python/src/mem_wal.rs b/python/src/mem_wal.rs index 812c5f42f22..7dd75659e72 100644 --- a/python/src/mem_wal.rs +++ b/python/src/mem_wal.rs @@ -18,13 +18,13 @@ use datafusion::prelude::SessionContext; use futures::TryStreamExt; use lance::dataset::Dataset as LanceDataset; use lance::dataset::mem_wal::scanner::{ - FlushedGeneration, LsmDataSourceCollector, LsmPointLookupPlanner, LsmVectorSearchPlanner, + LsmDataSourceCollector, LsmPointLookupPlanner, LsmVectorSearchPlanner, SsTable, parse_filter_expr as parse_lsm_filter_expr, }; use lance::dataset::mem_wal::write::{MemTableStats, WriteStatsSnapshot}; use lance::dataset::mem_wal::{LsmScanner, ShardSnapshot, ShardWriter, evaluate_sharding_spec}; use lance_index::mem_wal::{ - MergedGeneration as LanceMergedGeneration, ShardingField, ShardingSpec, + CompactedSsTable as LanceCompactedSsTable, ShardingField, ShardingSpec, }; use lance_linalg::distance::DistanceType; use pyo3::exceptions::{PyIOError, PyRuntimeError, PyValueError}; @@ -52,10 +52,10 @@ pub fn py_evaluate_sharding_spec<'py>( result.to_pyarrow(py) } -/// Write a primary-key dedup sidecar (`_pk_index/`) for a flushed-generation +/// Write a primary-key dedup sidecar (`_pk_index/`) for an SSTable /// dataset already written at `gen_path`, mirroring what production flush emits. /// -/// Test-support only: lets Python tests stage a *faithful* flushed generation +/// Test-support only: lets Python tests stage a *faithful* SSTable /// (dataset + sidecar). Production always writes the sidecar during flush, so a /// dataset-without-sidecar is not a state the system otherwise produces. #[pyfunction(name = "_write_pk_sidecar", signature = (gen_path, data, pk_columns))] @@ -115,16 +115,17 @@ fn optional_string(value: Bound<'_, PyAny>) -> PyResult> { } } -/// Represents a single generation of a MemWAL shard that has been merged -/// into the base table. Used with `MergeInsertBuilder.mark_generations_as_merged()`. -#[pyclass(name = "_MergedGeneration", module = "_lib")] -pub struct PyMergedGeneration { +/// Points to an SSTable compacted into the base table. +/// +/// Used with `MergeInsertBuilder.mark_sstables_as_compacted()`. +#[pyclass(name = "_CompactedSsTable", module = "_lib")] +pub struct PyCompactedSsTable { pub shard_id: String, pub generation: u64, } #[pymethods] -impl PyMergedGeneration { +impl PyCompactedSsTable { #[new] pub fn new(shard_id: String, generation: u64) -> Self { Self { @@ -145,23 +146,23 @@ impl PyMergedGeneration { pub fn __repr__(&self) -> String { format!( - "_MergedGeneration(shard_id='{}', generation={})", + "_CompactedSsTable(shard_id='{}', generation={})", self.shard_id, self.generation ) } } -impl PyMergedGeneration { - pub fn to_lance(&self) -> PyResult { +impl PyCompactedSsTable { + pub fn to_lance(&self) -> PyResult { let uuid = Uuid::parse_str(&self.shard_id) .map_err(|e| PyValueError::new_err(format!("Invalid shard_id UUID: {}", e)))?; - Ok(LanceMergedGeneration::new(uuid, self.generation)) + Ok(LanceCompactedSsTable::new(uuid, self.generation)) } } /// Snapshot of a MemWAL shard's state at a point in time. /// -/// Used to specify which flushed generations to include when creating an +/// Used to specify which SSTables to include when creating an /// `_LsmScanner`. Supports a builder pattern for adding generations. #[pyclass(name = "_ShardSnapshot", module = "_lib", skip_from_py_object)] #[derive(Clone)] @@ -195,13 +196,13 @@ impl PyShardSnapshot { slf } - /// Add a flushed generation by its generation number and storage path. - pub fn with_flushed_generation( + /// Add an SSTable by its generation number and storage path. + pub fn with_sstable( mut slf: PyRefMut<'_, Self>, generation: u64, path: String, ) -> PyRefMut<'_, Self> { - slf.inner = slf.inner.clone().with_flushed_generation(generation, path); + slf.inner = slf.inner.clone().with_sstable(generation, path); slf } @@ -212,10 +213,10 @@ impl PyShardSnapshot { pub fn __repr__(&self) -> String { format!( - "_ShardSnapshot(shard_id='{}', current_gen={}, flushed_gens={})", + "_ShardSnapshot(shard_id='{}', current_gen={}, sstables={})", self.inner.shard_id, self.inner.current_generation, - self.inner.flushed_generations.len() + self.inner.sstables.len() ) } } @@ -238,6 +239,14 @@ struct ClosedShardWriterState { memtable_stats: MemTableStats, } +fn collect_record_batches(data: &Bound<'_, PyAny>) -> PyResult> { + let reader = ArrowArrayStreamReader::from_pyarrow_bound(data) + .map_err(|e| PyValueError::new_err(format!("Cannot read data as Arrow: {}", e)))?; + reader + .collect::>() + .map_err(|e| PyIOError::new_err(format!("Failed to read batches: {}", e))) +} + #[pymethods] impl PyShardWriter { /// Write data batches to the MemWAL. @@ -245,11 +254,7 @@ impl PyShardWriter { /// Accepts any PyArrow-compatible data source (RecordBatch, Table, /// or an Arrow stream reader). pub fn put(&self, py: Python<'_>, data: &Bound<'_, PyAny>) -> PyResult<()> { - let reader = ArrowArrayStreamReader::from_pyarrow_bound(data) - .map_err(|e| PyValueError::new_err(format!("Cannot read data as Arrow: {}", e)))?; - let batches: Vec = reader - .collect::>() - .map_err(|e| PyIOError::new_err(format!("Failed to read batches: {}", e)))?; + let batches = collect_record_batches(data)?; if batches.is_empty() { return Ok(()); @@ -268,6 +273,31 @@ impl PyShardWriter { .map_err(|e: lance::Error| PyIOError::new_err(e.to_string())) } + /// Delete rows from the MemWAL by primary key. + /// + /// Accepts any PyArrow-compatible data source carrying the shard's primary + /// key column(s). Rust core validates that primary keys exist and builds the + /// tombstone rows. + pub fn delete(&self, py: Python<'_>, keys: &Bound<'_, PyAny>) -> PyResult<()> { + let batches = collect_record_batches(keys)?; + + if batches.is_empty() { + return Ok(()); + } + + let inner = self.inner.clone(); + rt().block_on(Some(py), async move { + let guard = inner.lock().await; + match guard.as_ref() { + Some(writer) => writer.delete(batches).await.map(|_| ()), + None => Err(lance_core::Error::invalid_input( + "ShardWriter is already closed", + )), + } + })? + .map_err(|e: lance::Error| PyIOError::new_err(e.to_string())) + } + /// Flush pending data and close the writer. /// /// After close(), calling put() will raise an error. @@ -355,7 +385,7 @@ impl PyShardWriter { /// Create an LSM scanner that includes the active MemTable for strong consistency. /// - /// The scanner covers: base table + given flushed generations + current active MemTable. + /// The scanner covers: base table + given SSTables + current active MemTable. #[pyo3(signature = (shard_snapshots=vec![]))] pub fn lsm_scanner( &self, @@ -508,7 +538,7 @@ impl PyExecutionPlan { } } -/// LSM-aware scanner covering base table, flushed MemTables, and active MemTable. +/// LSM-aware scanner covering base table, SSTables, and active MemTable. /// /// Provides deduplication by primary key, always returning the newest version /// of each row across all LSM levels. @@ -920,10 +950,8 @@ fn memtable_stats_to_pydict(py: Python<'_>, stats: &MemTableStats) -> PyResult

    MemTableStats { + // Close awaits every frozen memtable's flush, so nothing is owed afterwards + // regardless of whether the active memtable had buffered batches. + let stats_before_close = MemTableStats { + frozen_count: 0, + frozen_bytes: 0, + ..stats_before_close + }; + if stats_before_close.batch_count == 0 { return stats_before_close; } + // After a successful close every buffered batch is flushed and WAL-durable, + // so the synthesized empty memtable starts at the writer's global end and the + // durable cursor has caught up to it. + let global_end = stats_before_close.global_offset + stats_before_close.batch_count; MemTableStats { row_count: 0, batch_count: 0, estimated_size: 0, generation: stats_before_close.generation.saturating_add(1), max_buffered_batch_position: None, - max_flushed_batch_position: None, + durable_batch_count: global_end, + global_offset: global_end, pending_wal_start_batch_position: None, pending_wal_end_batch_position: None, pending_wal_batch_count: 0, pending_wal_row_count: 0, pending_wal_estimated_bytes: 0, + frozen_count: 0, + frozen_bytes: 0, } } diff --git a/python/src/otel.rs b/python/src/otel.rs new file mode 100644 index 00000000000..71d191d7db9 --- /dev/null +++ b/python/src/otel.rs @@ -0,0 +1,640 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright The Lance Authors + +//! Bridge from the [`metrics`] crate facade to Python OpenTelemetry. +//! +//! Lance core publishes metrics through the global [`metrics`] facade without +//! choosing a backend. This module installs a process-global [`Recorder`] that +//! aggregates those metrics into lock-free cumulative storage, and exposes that +//! state to Python so the bindings can feed it into the user's OpenTelemetry +//! `MeterProvider`. +//! +//! While it targets OpenTelemetry on the Python side, the recorder is agnostic +//! to the metric *source*: it records any metric emitted through the facade, +//! keyed by name and labels. Object store metrics are the first producer, but +//! nothing here is specific to them. New metrics flow through automatically; +//! they only need to be described (see [`describe_all`]) so the Python layer can +//! discover their name, kind, and unit up front. +//! +//! ## Why pull, not push +//! +//! OpenTelemetry collects on its own schedule and invokes observable-instrument +//! callbacks at collection time. Cumulative counters map directly onto OTel's +//! `ObservableCounter` semantics. So the bridge aggregates in Rust and lets the +//! Python collection thread pull a [`snapshot`](snapshot_lance_metrics). The +//! snapshot is lock-free, but it still walks every registered series and +//! allocates owned copies of their names and labels, so it runs with the GIL +//! released to avoid stalling other Python threads during collection. +//! +//! ## Histograms +//! +//! OpenTelemetry has no asynchronous histogram instrument, so histograms cannot +//! be pulled as-is. Instead each histogram is aggregated into fixed buckets +//! (Prometheus style) and exposed as cumulative `le` bucket counts plus a count +//! and sum, which the Python layer surfaces as observable counters. + +use std::collections::HashMap; +use std::sync::atomic::{AtomicU64, Ordering}; +use std::sync::{Arc, LazyLock, Mutex, OnceLock, RwLock}; + +use metrics::{Counter, Gauge, Histogram, Key, KeyName, Metadata, Recorder, SharedString, Unit}; +use metrics_util::registry::{Registry, Storage}; +use pyo3::prelude::*; + +/// Bucket boundaries used when a histogram has no registered bounds. Covers a +/// broad latency range so unknown histograms still produce useful buckets. +const DEFAULT_BOUNDS: &[f64] = &[ + 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1.0, 2.5, 5.0, 10.0, 30.0, 60.0, 120.0, 300.0, +]; + +/// The kind of a metric, mirroring the three `metrics` instrument types. +#[derive(Clone, Copy)] +enum MetricKind { + Counter, + Gauge, + Histogram, +} + +impl MetricKind { + fn as_str(self) -> &'static str { + match self { + Self::Counter => "counter", + Self::Gauge => "gauge", + Self::Histogram => "histogram", + } + } +} + +/// Description of a metric, populated by the recorder's `describe_*` methods. +struct MetricDescription { + kind: MetricKind, + unit: Option, + description: String, +} + +/// Catalog of described metrics, keyed by metric name. The Python layer reads +/// this to create one OpenTelemetry instrument per metric up front. +static CATALOG: LazyLock>> = + LazyLock::new(|| Mutex::new(HashMap::new())); + +/// Per-metric histogram bucket boundaries, keyed by metric name. Producers +/// register their recommended bounds before any metric is recorded. +static HISTOGRAM_BOUNDS: LazyLock>>> = + LazyLock::new(|| RwLock::new(HashMap::new())); + +/// The installed recorder's registry, available once installation succeeds. +static REGISTRY: OnceLock>> = OnceLock::new(); + +fn bounds_for(name: &str) -> Arc<[f64]> { + HISTOGRAM_BOUNDS + .read() + .unwrap() + .get(name) + .cloned() + .unwrap_or_else(|| Arc::from(DEFAULT_BOUNDS)) +} + +/// A histogram that buckets samples at record time into fixed boundaries, +/// keeping a cumulative count and sum. Bucketing eagerly keeps memory bounded +/// (unlike retaining raw samples) and produces Prometheus-style `le` buckets. +struct BucketedHistogram { + /// Sorted, finite upper bounds. A sample `v` falls in the first bucket whose + /// bound is `>= v`; samples above all bounds fall in the implicit `+Inf` + /// bucket stored as the final entry of `counts`. + bounds: Arc<[f64]>, + /// Per-bucket (non-cumulative) counts; length is `bounds.len() + 1`. + counts: Box<[AtomicU64]>, + count: AtomicU64, + /// Running sum of recorded values, stored as `f64` bits (there is no atomic + /// f64, so the bit pattern is held in a `u64`; see [`Self::add_to_sum`]). + sum_bits: AtomicU64, +} + +// All atomics here use `Ordering::Relaxed`: each metric counter is independent, +// so no happens-before relationship is needed between them, and a snapshot +// reader tolerates slightly stale values. This matches `metrics_util`'s +// `AtomicStorage`. + +impl BucketedHistogram { + fn new(bounds: Arc<[f64]>) -> Self { + let counts = (0..bounds.len() + 1) + .map(|_| AtomicU64::new(0)) + .collect::>() + .into_boxed_slice(); + Self { + bounds, + counts, + count: AtomicU64::new(0), + sum_bits: AtomicU64::new(0), + } + } + + fn add_to_sum(&self, value: f64) { + // No atomic offers an f64 add, so read the current bit pattern, add in + // float space, and CAS it back, retrying if another thread won the race. + let mut current = self.sum_bits.load(Ordering::Relaxed); + loop { + let updated = (f64::from_bits(current) + value).to_bits(); + match self.sum_bits.compare_exchange_weak( + current, + updated, + Ordering::Relaxed, + Ordering::Relaxed, + ) { + Ok(_) => break, + Err(actual) => current = actual, + } + } + } + + /// Cumulative `le` buckets, total count, and sum at this instant. + fn snapshot(&self) -> MetricValue { + let mut cumulative = 0u64; + let mut buckets = Vec::with_capacity(self.bounds.len() + 1); + for (i, bound) in self.bounds.iter().enumerate() { + cumulative += self.counts[i].load(Ordering::Relaxed); + buckets.push((format!("{}", bound), cumulative)); + } + cumulative += self.counts[self.bounds.len()].load(Ordering::Relaxed); + buckets.push(("+Inf".to_string(), cumulative)); + MetricValue::Histogram { + buckets, + count: self.count.load(Ordering::Relaxed), + sum: f64::from_bits(self.sum_bits.load(Ordering::Relaxed)), + } + } +} + +impl metrics::HistogramFn for BucketedHistogram { + fn record(&self, value: f64) { + let idx = self.bounds.partition_point(|&bound| bound < value); + self.counts[idx].fetch_add(1, Ordering::Relaxed); + self.count.fetch_add(1, Ordering::Relaxed); + self.add_to_sum(value); + } +} + +/// Storage backing the registry. Counters and gauges are plain atomics (as in +/// `metrics_util`'s `AtomicStorage`); histograms use [`BucketedHistogram`]. +struct LanceStorage; + +impl Storage for LanceStorage { + type Counter = Arc; + type Gauge = Arc; + type Histogram = Arc; + + fn counter(&self, _key: &Key) -> Self::Counter { + Arc::new(AtomicU64::new(0)) + } + + fn gauge(&self, _key: &Key) -> Self::Gauge { + // The `metrics` facade writes the f64 bit pattern into this `u64` (the + // snapshot decodes it with `f64::from_bits`), matching `AtomicStorage`. + // `0` decodes to `0.0`, the correct initial value. + Arc::new(AtomicU64::new(0)) + } + + fn histogram(&self, key: &Key) -> Self::Histogram { + Arc::new(BucketedHistogram::new(bounds_for(key.name()))) + } +} + +struct LanceRecorder { + registry: Arc>, +} + +impl LanceRecorder { + fn describe( + &self, + key: KeyName, + kind: MetricKind, + unit: Option, + description: SharedString, + ) { + CATALOG.lock().unwrap().insert( + key.as_str().to_string(), + MetricDescription { + kind, + unit: unit.map(|u| u.as_canonical_label().to_string()), + description: description.into_owned(), + }, + ); + } +} + +impl Recorder for LanceRecorder { + fn describe_counter(&self, key: KeyName, unit: Option, description: SharedString) { + self.describe(key, MetricKind::Counter, unit, description); + } + + fn describe_gauge(&self, key: KeyName, unit: Option, description: SharedString) { + self.describe(key, MetricKind::Gauge, unit, description); + } + + fn describe_histogram(&self, key: KeyName, unit: Option, description: SharedString) { + self.describe(key, MetricKind::Histogram, unit, description); + } + + fn register_counter(&self, key: &Key, _metadata: &Metadata<'_>) -> Counter { + self.registry + .get_or_create_counter(key, |c| Counter::from_arc(c.clone())) + } + + fn register_gauge(&self, key: &Key, _metadata: &Metadata<'_>) -> Gauge { + self.registry + .get_or_create_gauge(key, |g| Gauge::from_arc(g.clone())) + } + + fn register_histogram(&self, key: &Key, _metadata: &Metadata<'_>) -> Histogram { + self.registry + .get_or_create_histogram(key, |h| Histogram::from_arc(h.clone())) + } +} + +/// Register the recommended histogram bounds for every metric-emitting +/// subsystem. New subsystems add their `histogram_bounds()` here. +fn register_bounds() { + let mut bounds = HISTOGRAM_BOUNDS.write().unwrap(); + for (name, values) in lance_io::object_store::metrics::histogram_bounds() { + bounds.insert((*name).to_string(), Arc::from(*values)); + } +} + +/// Describe every metric-emitting subsystem so the catalog is populated. Must +/// run after the recorder is installed. New subsystems add their +/// `describe_metrics()` here. +fn describe_all() { + lance_io::object_store::metrics::describe_metrics(); +} + +enum MetricValue { + Scalar(f64), + Histogram { + buckets: Vec<(String, u64)>, + count: u64, + sum: f64, + }, +} + +struct MetricPoint { + name: String, + kind: &'static str, + attributes: HashMap, + value: MetricValue, +} + +fn labels(key: &Key) -> HashMap { + key.labels() + .map(|label| (label.key().to_string(), label.value().to_string())) + .collect() +} + +fn collect_points(registry: &Registry) -> Vec { + let mut points = Vec::new(); + for (key, handle) in registry.get_counter_handles() { + points.push(MetricPoint { + name: key.name().to_string(), + kind: "counter", + attributes: labels(&key), + // OpenTelemetry observations are float; counts stay well within the + // f64-exact integer range (2^53), so this cast is lossless in practice. + value: MetricValue::Scalar(handle.load(Ordering::Relaxed) as f64), + }); + } + for (key, handle) in registry.get_gauge_handles() { + points.push(MetricPoint { + name: key.name().to_string(), + kind: "gauge", + attributes: labels(&key), + value: MetricValue::Scalar(f64::from_bits(handle.load(Ordering::Relaxed))), + }); + } + for (key, handle) in registry.get_histogram_handles() { + points.push(MetricPoint { + name: key.name().to_string(), + kind: "histogram", + attributes: labels(&key), + value: handle.snapshot(), + }); + } + points +} + +/// One metric data point exposed to Python. For counters and gauges only +/// `value` is set; for histograms `buckets` (cumulative `le` counts), `count`, +/// and `sum` are set. +#[pyclass(name = "MetricPoint", get_all)] +pub struct PyMetricPoint { + name: String, + kind: String, + attributes: HashMap, + value: Option, + buckets: Option>, + count: Option, + sum: Option, +} + +impl From for PyMetricPoint { + fn from(point: MetricPoint) -> Self { + let (value, buckets, count, sum) = match point.value { + MetricValue::Scalar(v) => (Some(v), None, None, None), + MetricValue::Histogram { + buckets, + count, + sum, + } => (None, Some(buckets), Some(count), Some(sum)), + }; + Self { + name: point.name, + kind: point.kind.to_string(), + attributes: point.attributes, + value, + buckets, + count, + sum, + } + } +} + +/// A described metric, used by the Python layer to create instruments up front. +#[pyclass(name = "MetricDescription", get_all)] +pub struct PyMetricDescription { + name: String, + kind: String, + unit: Option, + description: String, +} + +/// Install the Lance metrics recorder as the process-global `metrics` recorder. +/// +/// Returns `True` if the recorder is installed (now or previously). Returns +/// `False` if a *different* recorder is already installed — `metrics` allows +/// only one global recorder per process, so Lance cannot coexist with another. +#[pyfunction] +pub fn register_lance_metrics_recorder() -> bool { + if REGISTRY.get().is_some() { + return true; + } + let registry = Arc::new(Registry::new(LanceStorage)); + let recorder = LanceRecorder { + registry: registry.clone(), + }; + // Register histogram bounds *before* installing the recorder. Once the + // recorder is global, another thread can emit a metric and create the + // histogram handle concurrently; if the bounds aren't registered yet that + // handle would be built with the fallback bounds and keep them for the + // life of the process. `register_bounds()` doesn't need the recorder. + register_bounds(); + match metrics::set_global_recorder(recorder) { + Ok(()) => { + let _ = REGISTRY.set(registry); + describe_all(); + true + } + Err(_) => false, + } +} + +/// The catalog of described Lance metrics. Empty until the recorder is installed. +#[pyfunction] +pub fn lance_metrics_catalog() -> Vec { + CATALOG + .lock() + .unwrap() + .iter() + .map(|(name, desc)| PyMetricDescription { + name: name.clone(), + kind: desc.kind.as_str().to_string(), + unit: desc.unit.clone(), + description: desc.description.clone(), + }) + .collect() +} + +/// A point-in-time snapshot of every recorded metric. Empty until the recorder +/// is installed. The read is lock-free but walks every series and allocates, so +/// it runs with the GIL released. +#[pyfunction] +pub fn snapshot_lance_metrics(py: Python<'_>) -> Vec { + let Some(registry) = REGISTRY.get() else { + return Vec::new(); + }; + let points = py.detach(|| collect_points(registry)); + points.into_iter().map(PyMetricPoint::from).collect() +} + +#[cfg(test)] +mod tests { + use super::*; + use metrics::HistogramFn; + + fn bucket_count(buckets: &[(String, u64)], le: &str) -> u64 { + buckets + .iter() + .find(|(b, _)| b == le) + .map(|(_, c)| *c) + .unwrap_or_else(|| panic!("no bucket with le={le}")) + } + + #[test] + fn bucketed_histogram_records_cumulative_buckets() { + let hist = BucketedHistogram::new(Arc::from([0.1f64, 1.0, 10.0].as_slice())); + hist.record(0.05); // le=0.1 + hist.record(0.5); // le=1 + hist.record(0.5); // le=1 + hist.record(50.0); // +Inf + + let MetricValue::Histogram { + buckets, + count, + sum, + } = hist.snapshot() + else { + panic!("expected histogram"); + }; + + // Buckets are cumulative (Prometheus `le` semantics). + assert_eq!(bucket_count(&buckets, "0.1"), 1); + assert_eq!(bucket_count(&buckets, "1"), 3); + assert_eq!(bucket_count(&buckets, "10"), 3); + assert_eq!(bucket_count(&buckets, "+Inf"), 4); + assert_eq!(count, 4); + assert!((sum - 51.05).abs() < 1e-9); + } + + #[test] + fn bucketed_histogram_boundary_is_inclusive() { + let hist = BucketedHistogram::new(Arc::from([1.0f64].as_slice())); + hist.record(1.0); // exactly the bound -> le=1, not +Inf + let MetricValue::Histogram { buckets, .. } = hist.snapshot() else { + panic!("expected histogram"); + }; + assert_eq!(bucket_count(&buckets, "1"), 1); + assert_eq!(bucket_count(&buckets, "+Inf"), 1); + } + + #[test] + fn bucketed_histogram_boundary_is_inclusive_mid_range() { + // A value equal to a middle bound lands in that bucket, not the next. + let hist = BucketedHistogram::new(Arc::from([0.1f64, 1.0, 10.0].as_slice())); + hist.record(1.0); + let MetricValue::Histogram { buckets, .. } = hist.snapshot() else { + panic!("expected histogram"); + }; + assert_eq!(bucket_count(&buckets, "0.1"), 0); + assert_eq!(bucket_count(&buckets, "1"), 1); + assert_eq!(bucket_count(&buckets, "10"), 1); // cumulative, so still 1 + assert_eq!(bucket_count(&buckets, "+Inf"), 1); + } + + #[test] + fn recorder_aggregates_counters_with_labels() { + let registry = Arc::new(Registry::new(LanceStorage)); + let recorder = LanceRecorder { + registry: registry.clone(), + }; + metrics::with_local_recorder(&recorder, || { + metrics::counter!("test_requests_total", "operation" => "get", "scheme" => "s3") + .increment(2); + metrics::counter!("test_requests_total", "operation" => "get", "scheme" => "s3") + .increment(3); + // A distinct label set must produce a separate point, not merge. + metrics::counter!("test_requests_total", "operation" => "put", "scheme" => "gs") + .increment(7); + }); + + let scalar = |attrs: &[(&str, &str)]| { + let points = collect_points(®istry); + let point = points + .into_iter() + .find(|p| { + p.name == "test_requests_total" + && attrs + .iter() + .all(|(k, v)| p.attributes.get(*k).map(String::as_str) == Some(*v)) + }) + .expect("counter recorded for label set"); + assert_eq!(point.kind, "counter"); + match point.value { + MetricValue::Scalar(v) => v, + _ => panic!("expected scalar"), + } + }; + + // Same labels aggregate; distinct labels stay separate. + assert!((scalar(&[("operation", "get"), ("scheme", "s3")]) - 5.0).abs() < 1e-9); + assert!((scalar(&[("operation", "put"), ("scheme", "gs")]) - 7.0).abs() < 1e-9); + } + + #[test] + fn recorder_records_gauges() { + let registry = Arc::new(Registry::new(LanceStorage)); + let recorder = LanceRecorder { + registry: registry.clone(), + }; + // Gauges store the f64 bit pattern in a u64; the snapshot must decode it. + metrics::with_local_recorder(&recorder, || { + metrics::gauge!("test_gauge", "scheme" => "s3").set(3.5); + }); + + let points = collect_points(®istry); + let point = points + .iter() + .find(|p| p.name == "test_gauge") + .expect("gauge recorded"); + assert_eq!(point.kind, "gauge"); + assert!(matches!(point.value, MetricValue::Scalar(v) if (v - 3.5).abs() < 1e-9)); + } + + #[test] + fn recorder_falls_back_to_default_bounds() { + // A histogram with no registered bounds uses DEFAULT_BOUNDS. + let name = "test_unregistered_histogram"; + assert!(!HISTOGRAM_BOUNDS.read().unwrap().contains_key(name)); + + let registry = Arc::new(Registry::new(LanceStorage)); + let recorder = LanceRecorder { + registry: registry.clone(), + }; + metrics::with_local_recorder(&recorder, || { + metrics::histogram!(name).record(0.02); + }); + + let points = collect_points(®istry); + let point = points.iter().find(|p| p.name == name).expect("recorded"); + let MetricValue::Histogram { buckets, count, .. } = &point.value else { + panic!("expected histogram"); + }; + assert_eq!(*count, 1); + // DEFAULT_BOUNDS yields one bucket per bound plus the implicit `+Inf`. + assert_eq!(buckets.len(), DEFAULT_BOUNDS.len() + 1); + // 0.02 falls in the le=0.025 bucket (the third DEFAULT_BOUNDS entry). + assert_eq!(bucket_count(buckets, "0.025"), 1); + assert_eq!(bucket_count(buckets, "0.01"), 0); + assert_eq!(bucket_count(buckets, "+Inf"), 1); + } + + #[test] + fn recorder_uses_registered_histogram_bounds() { + let name = "test_recorder_bounds_seconds"; + HISTOGRAM_BOUNDS + .write() + .unwrap() + .insert(name.to_string(), Arc::from([0.1f64, 1.0].as_slice())); + + let registry = Arc::new(Registry::new(LanceStorage)); + let recorder = LanceRecorder { + registry: registry.clone(), + }; + metrics::with_local_recorder(&recorder, || { + metrics::histogram!(name).record(0.05); + metrics::histogram!(name).record(5.0); + }); + + let points = collect_points(®istry); + let point = points.iter().find(|p| p.name == name).expect("recorded"); + let MetricValue::Histogram { buckets, count, .. } = &point.value else { + panic!("expected histogram"); + }; + assert_eq!(*count, 2); + assert_eq!(bucket_count(buckets, "0.1"), 1); + assert_eq!(bucket_count(buckets, "+Inf"), 2); + } + + #[test] + fn describe_populates_catalog() { + let name = "test_describe_catalog_total"; + let registry = Arc::new(Registry::new(LanceStorage)); + let recorder = LanceRecorder { registry }; + metrics::with_local_recorder(&recorder, || { + metrics::describe_counter!(name, Unit::Count, "a test counter"); + }); + + let catalog = CATALOG.lock().unwrap(); + let desc = catalog.get(name).expect("described"); + assert!(matches!(desc.kind, MetricKind::Counter)); + assert_eq!(desc.description, "a test counter"); + } + + #[test] + fn describe_all_covers_object_store_metrics() { + use lance_io::object_store::metrics as os; + + let registry = Arc::new(Registry::new(LanceStorage)); + let recorder = LanceRecorder { registry }; + metrics::with_local_recorder(&recorder, describe_all); + + let catalog = CATALOG.lock().unwrap(); + let kind = |name: &str| catalog.get(name).expect("described").kind; + // Every emitted object store metric must be described so the OTel + // bridge can create an instrument for it, including the gauge and the + // retryable counter that a plain request path might never emit. + assert!(matches!(kind(os::METRIC_REQUESTS), MetricKind::Counter)); + assert!(matches!(kind(os::METRIC_BYTES), MetricKind::Counter)); + assert!(matches!(kind(os::METRIC_ERRORS), MetricKind::Counter)); + assert!(matches!(kind(os::METRIC_THROTTLE), MetricKind::Counter)); + assert!(matches!(kind(os::METRIC_RETRYABLE), MetricKind::Counter)); + assert!(matches!(kind(os::METRIC_IN_FLIGHT), MetricKind::Gauge)); + assert!(matches!(kind(os::METRIC_DURATION), MetricKind::Histogram)); + } +} diff --git a/python/src/rowids.rs b/python/src/rowids.rs new file mode 100644 index 00000000000..0ad83c90579 --- /dev/null +++ b/python/src/rowids.rs @@ -0,0 +1,223 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright The Lance Authors + +use std::ops::Range; + +use arrow::array::{Array, UInt64Array, make_array}; +use arrow::compute::CastOptions; +use arrow::compute::kernels::cast::cast_with_options; +use arrow::datatypes::DataType; +use arrow::pyarrow::{FromPyArrow, ToPyArrow}; +use arrow_array::cast::AsArray; +use arrow_array::types::UInt64Type; +use arrow_data::ArrayData; +use lance_table::format::RowIdMeta; +use lance_table::rowids::{RowIdSequence, read_row_ids, write_row_ids}; +use pyo3::basic::CompareOp; +use pyo3::exceptions::{PyNotImplementedError, PyTypeError, PyValueError}; +use pyo3::prelude::*; +use pyo3::types::{PyRange, PyRangeMethods, PyTuple}; +use pyo3::{IntoPyObjectExt, intern}; + +use crate::error::PythonErrorExt; +use crate::fragment::PyRowIdMeta; + +/// The number of row ids shown in `RowIdSequence.__repr__` before eliding. +const REPR_PREVIEW_LEN: usize = 10; + +/// A sequence of stable row ids belonging to a single fragment. +#[pyclass(name = "RowIdSequence", module = "lance.fragment")] +pub struct PyRowIdSequence(pub RowIdSequence); + +#[pymethods] +impl PyRowIdSequence { + #[new] + fn new(row_ids: &Bound<'_, PyAny>) -> PyResult { + let sequence = match contiguous_range(row_ids)? { + // A `Range` segment is the most compact encoding, and taking it + // directly avoids materializing the ids of a large range. + Some(range) if range.is_empty() => RowIdSequence::new(), + Some(range) => RowIdSequence::from(range), + None => RowIdSequence::try_from_iter(extract_row_ids(row_ids)?).infer_error()?, + }; + Ok(Self(sequence)) + } + + /// Read back the sequence stored inline in fragment row id metadata. + #[staticmethod] + fn from_inline_metadata(metadata: PyRef<'_, PyRowIdMeta>) -> PyResult { + match &metadata.0 { + RowIdMeta::Inline(data) => read_row_ids(data).infer_error().map(Self), + RowIdMeta::External(_) => Err(PyNotImplementedError::new_err( + "Row ids stored in an external file cannot be read into a RowIdSequence", + )), + } + } + + /// Encode the sequence as row id metadata stored inline in the manifest. + fn to_inline_metadata(&self) -> PyRowIdMeta { + PyRowIdMeta(RowIdMeta::Inline(write_row_ids(&self.0).into())) + } + + fn to_pyarrow<'py>(&self, py: Python<'py>) -> PyResult> { + let array = UInt64Array::from(self.0.iter().collect::>()); + array.into_data().to_pyarrow(py) + } + + fn __len__(&self) -> usize { + self.0.len() as usize + } + + fn __iter__(&self, py: Python<'_>) -> PyResult> { + let row_ids: Vec = self.0.iter().collect(); + Py::new(py, PyRowIdSequenceIterator(row_ids.into_iter())) + } + + fn __repr__(&self) -> String { + let len = self.0.len(); + let preview = self + .0 + .iter() + .take(REPR_PREVIEW_LEN) + .map(|row_id| row_id.to_string()) + .collect::>() + .join(", "); + if len > REPR_PREVIEW_LEN as u64 { + format!("RowIdSequence([{}, ...], len={})", preview, len) + } else { + format!("RowIdSequence([{}])", preview) + } + } + + fn __richcmp__( + &self, + other: &Bound<'_, PyAny>, + op: CompareOp, + py: Python<'_>, + ) -> PyResult> { + let Ok(other) = other.cast::() else { + return Ok(py.NotImplemented()); + }; + let equal = self.0 == other.borrow().0; + match op { + CompareOp::Eq => equal.into_py_any(py), + CompareOp::Ne => (!equal).into_py_any(py), + _ => Ok(py.NotImplemented()), + } + } + + fn __reduce__(&self, py: Python<'_>) -> PyResult<(Py, Py)> { + let from_inline_metadata = PyModule::import(py, "lance.fragment")? + .getattr("RowIdSequence")? + .getattr("from_inline_metadata")? + .extract()?; + let metadata = Py::new(py, self.to_inline_metadata())?; + let state = PyTuple::new(py, [metadata])?.extract()?; + Ok((from_inline_metadata, state)) + } +} + +/// The row ids are materialized up front because `RowIdSequence::iter` borrows +/// the sequence and a `#[pyclass]` cannot hold a borrowing iterator. +/// +/// Refilling a bounded buffer from `RowIdSequence::slice` looks like the way to +/// avoid that, but it is quadratic: slicing takes an absolute offset, and for +/// the gapped `RangeWithHoles` and `RangeWithBitmap` encodings the slice skips +/// its prefix one element at a time. Making iteration both lazy and linear +/// needs a forward cursor over the private segments, which belongs in +/// `lance-table` rather than here. Bulk callers should use `to_pyarrow`. +#[pyclass(name = "RowIdSequenceIterator", module = "lance.fragment")] +pub struct PyRowIdSequenceIterator(std::vec::IntoIter); + +#[pymethods] +impl PyRowIdSequenceIterator { + fn __iter__(slf: PyRef<'_, Self>) -> PyRef<'_, Self> { + slf + } + + fn __next__(mut slf: PyRefMut<'_, Self>) -> Option { + slf.0.next() + } +} + +/// Recognize a `range` with a step of one, whose row ids need no materialization. +/// +/// Returns `None` for anything else, including strided and descending ranges, +/// which are handled by the general iterable path. +fn contiguous_range(ob: &Bound<'_, PyAny>) -> PyResult>> { + let Ok(range) = ob.cast::() else { + return Ok(None); + }; + // Bounds are read as `isize`. A range reaching past that is well beyond any + // dataset's row count, so it falls through to the element-wise path, which + // reads each value as a u64, rather than failing. + let (Ok(start), Ok(stop), Ok(step)) = (range.start(), range.stop(), range.step()) else { + return Ok(None); + }; + if step != 1 { + return Ok(None); + } + if start < 0 { + return Err(PyValueError::new_err(format!( + "Row ids must be non-negative, but the range starts at {}", + start + ))); + } + if stop <= start { + return Ok(Some(0..0)); + } + Ok(Some(start as u64..stop as u64)) +} + +fn extract_row_ids(ob: &Bound<'_, PyAny>) -> PyResult> { + let py = ob.py(); + if ob.hasattr(intern!(py, "__arrow_c_array__"))? { + return row_ids_from_arrow(ob); + } + // A `pyarrow.ChunkedArray`, such as a `_rowid` column taken from a table. + let chunks = intern!(py, "chunks"); + if ob.hasattr(chunks)? { + let mut row_ids = Vec::new(); + for chunk in ob.getattr(chunks)?.try_iter()? { + row_ids.extend(row_ids_from_arrow(&chunk?)?); + } + return Ok(row_ids); + } + + let iter = ob.try_iter().map_err(|_| { + PyTypeError::new_err(format!( + "Row ids must be an iterable of integers or an Arrow array, but got {}", + ob.get_type().name().map_or_else( + |_| "an object of unknown type".to_string(), + |name| name.to_string() + ) + )) + })?; + iter.map(|row_id| row_id?.extract::()).collect() +} + +fn row_ids_from_arrow(array: &Bound<'_, PyAny>) -> PyResult> { + let array = make_array(ArrayData::from_pyarrow_bound(array)?); + if !array.data_type().is_integer() { + return Err(PyTypeError::new_err(format!( + "Row ids must be an array of integers, but got an array of type {}", + array.data_type() + ))); + } + if array.null_count() > 0 { + return Err(PyValueError::new_err(format!( + "Row ids must not be null, but the array has {} null values", + array.null_count() + ))); + } + + // `safe: false` so that negative values raise instead of wrapping around + // into the top of the row id space. + let cast_options = CastOptions { + safe: false, + ..Default::default() + }; + let array = cast_with_options(&array, &DataType::UInt64, &cast_options) + .map_err(|err| PyValueError::new_err(format!("Row ids must fit in a uint64: {}", err)))?; + Ok(array.as_primitive::().values().to_vec()) +} diff --git a/python/src/scanner.rs b/python/src/scanner.rs index bbf1b3f35a3..8702537a340 100644 --- a/python/src/scanner.rs +++ b/python/src/scanner.rs @@ -66,6 +66,10 @@ pub struct ScanStatistics { pub parts_loaded: usize, /// Number of index comparisons performed pub index_comparisons: usize, + /// Number of index cache page lookups that were served from memory + pub index_cache_hits: usize, + /// Number of index cache page lookups that had to load from storage + pub index_cache_misses: usize, /// Additional metrics for more detailed statistics. These are subject to change in the future /// and should only be used for debugging purposes. pub all_counts: HashMap, @@ -80,6 +84,8 @@ impl ScanStatistics { indices_loaded: stats.indices_loaded, parts_loaded: stats.parts_loaded, index_comparisons: stats.index_comparisons, + index_cache_hits: stats.index_cache_hits(), + index_cache_misses: stats.index_cache_misses(), all_counts: stats.all_counts.clone(), } } @@ -89,13 +95,15 @@ impl ScanStatistics { impl ScanStatistics { fn __repr__(&self) -> String { format!( - "ScanStatistics(iops={}, requests={}, bytes_read={}, indices_loaded={}, parts_loaded={}, index_comparisons={}, all_counts={:?})", + "ScanStatistics(iops={}, requests={}, bytes_read={}, indices_loaded={}, parts_loaded={}, index_comparisons={}, index_cache_hits={}, index_cache_misses={}, all_counts={:?})", self.iops, self.requests, self.bytes_read, self.indices_loaded, self.parts_loaded, self.index_comparisons, + self.index_cache_hits, + self.index_cache_misses, self.all_counts ) } diff --git a/python/src/schema.rs b/python/src/schema.rs index db4f7369710..8cdc2115cd1 100644 --- a/python/src/schema.rs +++ b/python/src/schema.rs @@ -179,7 +179,8 @@ impl LanceSchema { fields: Fields(fields), metadata, }; - let schema = Schema::from(fields_with_meta); + let schema = Schema::try_from(fields_with_meta) + .map_err(|err| PyValueError::new_err(format!("Failed to reconstruct schema: {err}")))?; Ok(Self(schema)) } diff --git a/python/src/session.rs b/python/src/session.rs index c91329ec1ee..c15d17230d6 100644 --- a/python/src/session.rs +++ b/python/src/session.rs @@ -1,18 +1,40 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright The Lance Authors +use std::collections::HashMap; use std::sync::Arc; -use pyo3::{pyclass, pymethods}; +use pyo3::exceptions::PyValueError; +use pyo3::types::{PyAnyMethods, PyDict, PyDictMethods, PyString}; +use pyo3::{Bound, PyAny, PyResult, pyclass, pymethods}; -use lance::dataset::{DEFAULT_INDEX_CACHE_SIZE, DEFAULT_METADATA_CACHE_SIZE}; -use lance::session::Session as LanceSession; +use lance::session::{CacheSpec, Session as LanceSession}; +use lance_core::cache::{BackendConfig, build_from_config, build_from_uri}; use crate::rt; /// The Session holds stateful information for a dataset. /// /// The session contains caches for opened indices and file metadata. +/// +/// Parameters +/// ---------- +/// index_cache_size_bytes : int, optional +/// Capacity of the default index cache in bytes. +/// metadata_cache_size_bytes : int, optional +/// Capacity of the default metadata cache in bytes. +/// index_cache_backend : str or dict, optional +/// Custom index cache backend. Strings are backend URIs such as +/// ``"moka://?capacity=1048576"``. Dicts must contain ``"kind"`` and may +/// contain ``"options"``, for example +/// ``{"kind": "moka", "options": {"capacity": "1048576"}}``. +/// metadata_cache_backend : str or dict, optional +/// Custom metadata cache backend with the same format as +/// ``index_cache_backend``. +/// +/// ``index_cache_backend`` is mutually exclusive with +/// ``index_cache_size_bytes``. ``metadata_cache_backend`` is mutually +/// exclusive with ``metadata_cache_size_bytes``. #[pyclass(name = "_Session", module = "_lib", from_py_object)] #[derive(Clone)] pub struct Session { @@ -25,22 +47,152 @@ impl Session { } } +/// Turn a Python-supplied backend descriptor into an `Arc`, +/// or return `Ok(None)` when the caller did not pass one. +/// +/// Accepts: +/// * `str` — treated as a URI (`moka://?capacity=...`) and passed to +/// [`build_from_uri`]. +/// * `dict` — must have string keys `kind` (required) and `options` +/// (optional `dict[str, str]`) matching [`BackendConfig`]; passed to +/// [`build_from_config`]. +/// +/// Any other Python type is rejected with a clear `TypeError`-style +/// `PyValueError`. +/// +/// If `size_field_set` is `true` and `backend` is `Some`, both a size and a +/// backend were provided for the same cache. Rather than silently letting +/// one override the other (Proposal §7), this is rejected up-front so the +/// operator gets an actionable error. +fn resolve_cache_spec( + backend_field: &str, + backend: Option<&Bound<'_, PyAny>>, + size_field: &str, + size: Option, +) -> PyResult { + if backend.is_some() && size.is_some() { + return Err(PyValueError::new_err(format!( + "{} and {} are mutually exclusive; set one or the other", + size_field, backend_field, + ))); + } + + let Some(value) = backend else { + return Ok(size.map(CacheSpec::Size).unwrap_or(CacheSpec::Default)); + }; + + if value.cast::().is_ok() { + let uri: String = value.extract()?; + return build_from_uri(&uri) + .map(CacheSpec::Backend) + .map_err(|e| PyValueError::new_err(format!("{}: {}", backend_field, e))); + } + + if let Ok(dict) = value.cast::() { + let cfg = backend_config_from_dict(backend_field, dict)?; + return build_from_config(&cfg) + .map(CacheSpec::Backend) + .map_err(|e| PyValueError::new_err(format!("{}: {}", backend_field, e))); + } + + let type_name: String = value.get_type().getattr("__name__")?.extract()?; + Err(PyValueError::new_err(format!( + "{}: expected str (URI) or dict with 'kind'/'options' keys, got {}", + backend_field, type_name, + ))) +} + +fn backend_config_from_dict(field: &str, dict: &Bound<'_, PyDict>) -> PyResult { + for (key, _) in dict.iter() { + if key.cast::().is_err() { + return Err(PyValueError::new_err(format!( + "{}: dict keys must be strings", + field + ))); + } + let key: String = key.extract()?; + if key != "kind" && key != "options" { + return Err(PyValueError::new_err(format!( + "{}: unknown dict key {:?}; expected 'kind' or 'options'", + field, key + ))); + } + } + + let kind_obj = dict.get_item("kind")?.ok_or_else(|| { + PyValueError::new_err(format!("{}: dict must contain a 'kind' key", field)) + })?; + if kind_obj.cast::().is_err() { + return Err(PyValueError::new_err(format!( + "{}: 'kind' must be a string", + field + ))); + } + let kind: String = kind_obj.extract()?; + + let mut options: HashMap = HashMap::new(); + if let Some(options_obj) = dict.get_item("options")? { + let options_dict = options_obj.cast::().map_err(|_| { + PyValueError::new_err(format!("{}: 'options' must be a dict[str, str]", field)) + })?; + for (k, v) in options_dict.iter() { + if k.cast::().is_err() { + return Err(PyValueError::new_err(format!( + "{}: 'options' keys must be strings", + field + ))); + } + if v.cast::().is_err() { + return Err(PyValueError::new_err(format!( + "{}: 'options' values must be strings", + field + ))); + } + let key: String = k.extract()?; + let value: String = v.extract()?; + options.insert(key, value); + } + } + + let mut config = BackendConfig::new(&kind) + .map_err(|e| PyValueError::new_err(format!("{}: {}", field, e)))?; + config.options = options; + Ok(config) +} + #[pymethods] impl Session { #[new] - #[pyo3(signature=(index_cache_size_bytes=None, metadata_cache_size_bytes=None))] + #[pyo3(signature=( + index_cache_size_bytes=None, + metadata_cache_size_bytes=None, + index_cache_backend=None, + metadata_cache_backend=None, + ))] fn create( index_cache_size_bytes: Option, metadata_cache_size_bytes: Option, - ) -> Self { - let session = LanceSession::new( - index_cache_size_bytes.unwrap_or(DEFAULT_INDEX_CACHE_SIZE), - metadata_cache_size_bytes.unwrap_or(DEFAULT_METADATA_CACHE_SIZE), - Default::default(), - ); - Self { + index_cache_backend: Option>, + metadata_cache_backend: Option>, + ) -> PyResult { + let index_cache = resolve_cache_spec( + "index_cache_backend", + index_cache_backend.as_ref(), + "index_cache_size_bytes", + index_cache_size_bytes, + )?; + let metadata_cache = resolve_cache_spec( + "metadata_cache_backend", + metadata_cache_backend.as_ref(), + "metadata_cache_size_bytes", + metadata_cache_size_bytes, + )?; + + let session = + LanceSession::with_cache_backends(index_cache, metadata_cache, Default::default()); + Ok(Self { inner: Arc::new(session), - } + }) } fn __repr__(&self) -> String { @@ -63,6 +215,13 @@ impl Session { self.inner.size_bytes() } + /// Return the current size of the index cache in bytes. + pub fn index_cache_size_bytes(&self) -> PyResult { + rt().block_on(None, async move { + self.inner.index_cache_stats().await.size_bytes as u64 + }) + } + /// Return whether the other session is the same as this one. pub fn is_same_as(&self, other: &Self) -> bool { Arc::ptr_eq(&self.inner, &other.inner) diff --git a/python/src/transaction.rs b/python/src/transaction.rs index 1b659395099..bf70cf3dd6d 100644 --- a/python/src/transaction.rs +++ b/python/src/transaction.rs @@ -7,10 +7,11 @@ use crate::utils::{PyLance, class_name, export_vec, extract_vec}; use arrow::pyarrow::PyArrowType; use arrow_schema::Schema as ArrowSchema; use lance::dataset::transaction::{ - DataReplacementGroup, Operation, RewriteGroup, RewrittenIndex, Transaction, UpdateMap, - UpdateMapEntry, UpdateMode, + DataOverlayGroup, DataReplacementGroup, Operation, RewriteGroup, RewrittenIndex, Transaction, + UpdateMap, UpdateMapEntry, UpdateMode, UpdatedFragmentOffsets, }; use lance::datatypes::Schema; +use lance_table::format::overlay::{DataOverlayFile, OverlayCoverage}; use lance_table::format::{BasePath, DataFile, Fragment, IndexFile, IndexMetadata}; use pyo3::exceptions::PyValueError; use pyo3::types::PySet; @@ -206,6 +207,128 @@ impl<'py> IntoPyObject<'py> for PyLance<&DataReplacementGroup> { } } +// The Nth offset in an overlay list positionally maps to the Nth value row in +// `data_file`, but `RoaringBitmap` stores offsets in ascending order and drops +// duplicates. A caller-supplied list that isn't strictly ascending would be +// silently reordered, breaking that mapping, so reject it here instead. This can +// go away once we expose RoaringBitmap directly to Python (issue #7695). +fn bitmap_from_sorted_offsets(offsets: Vec) -> PyResult { + if offsets.windows(2).any(|w| w[0] >= w[1]) { + return Err(PyValueError::new_err( + "DataOverlayFile.offsets must be strictly ascending with no duplicates; \ + each offset positionally maps to a value row in data_file", + )); + } + Ok(RoaringBitmap::from_sorted_iter(offsets).expect("offsets verified strictly ascending")) +} + +impl FromPyObject<'_, '_> for PyLance { + type Error = PyErr; + fn extract(ob: Borrowed<'_, '_, PyAny>) -> PyResult { + let data_file = ob.getattr("data_file")?.extract::>()?.0; + let offsets = ob.getattr("offsets")?; + + // A flat list of offsets is a dense overlay (one coverage shared by every + // field); a list of per-field lists is a sparse overlay. Differentiate by + // shape, trying the dense form first. + let coverage = if let Ok(shared) = offsets.extract::>() { + OverlayCoverage::dense(bitmap_from_sorted_offsets(shared)?) + } else if let Ok(per_field) = offsets.extract::>>() { + OverlayCoverage::sparse( + per_field + .into_iter() + .map(bitmap_from_sorted_offsets) + .collect::>>()?, + ) + } else { + return Err(PyValueError::new_err( + "DataOverlayFile.offsets must be a list of ints (dense coverage shared by \ + every field) or a list of per-field int lists (sparse coverage)", + )); + }; + + // Present (and preserved) when round-tripping an existing fragment's + // overlays; None/0 when creating an overlay to commit, since the + // DataOverlay commit stamps the effective version. + let committed_version = ob + .getattr("committed_version")? + .extract::>()? + .unwrap_or(0); + + Ok(Self(DataOverlayFile { + data_file, + coverage, + committed_version, + })) + } +} + +impl<'py> IntoPyObject<'py> for PyLance<&DataOverlayFile> { + type Target = PyAny; + type Output = Bound<'py, Self::Target>; + type Error = PyErr; + + fn into_pyobject(self, py: Python<'py>) -> Result { + let namespace = py + .import(intern!(py, "lance")) + .and_then(|module| module.getattr(intern!(py, "LanceOperation"))) + .expect("Failed to import LanceOperation namespace"); + + let data_file = PyLance(&self.0.data_file).into_pyobject(py)?; + let cls = namespace + .getattr("DataOverlayFile") + .expect("Failed to get DataOverlayFile class"); + + let committed_version = self.0.committed_version; + + // Mirror the read side: a dense overlay becomes a flat list of offsets, a + // sparse overlay a list of per-field lists. + match &self.0.coverage { + OverlayCoverage::Shared(bitmap) => { + let offsets: Vec = bitmap.iter().collect(); + cls.call1((data_file, offsets, committed_version)) + } + OverlayCoverage::PerField(bitmaps) => { + let offsets: Vec> = bitmaps.iter().map(|b| b.iter().collect()).collect(); + cls.call1((data_file, offsets, committed_version)) + } + } + } +} + +impl FromPyObject<'_, '_> for PyLance { + type Error = PyErr; + fn extract(ob: Borrowed<'_, '_, PyAny>) -> PyResult { + let fragment_id = ob.getattr("fragment_id")?.extract::()?; + let overlays = extract_vec(&ob.getattr("overlays")?)?; + Ok(Self(DataOverlayGroup { + fragment_id, + overlays, + })) + } +} + +impl<'py> IntoPyObject<'py> for PyLance<&DataOverlayGroup> { + type Target = PyAny; + type Output = Bound<'py, Self::Target>; + type Error = PyErr; + + fn into_pyobject(self, py: Python<'py>) -> Result { + let namespace = py + .import(intern!(py, "lance")) + .and_then(|module| module.getattr(intern!(py, "LanceOperation"))) + .expect("Failed to import LanceOperation namespace"); + + let fragment_id = self.0.fragment_id; + let overlays = export_vec(py, self.0.overlays.as_slice())?; + + let cls = namespace + .getattr("DataOverlayGroup") + .expect("Failed to get DataOverlayGroup class"); + cls.call1((fragment_id, overlays)) + } +} + #[derive(Debug, Clone)] pub struct PyUpdateMode(pub UpdateMode); @@ -290,16 +413,41 @@ impl FromPyObject<'_, '_> for PyLance { .ok() .map(|py_mode| py_mode.0); + // Absent on objects predating the field. + let updated_fragment_offsets = ob + .getattr("updated_fragment_offsets") + .ok() + .map(|v| v.extract::>>>()) + .transpose()? + .flatten() + .map(|offsets| { + offsets + .into_iter() + .map(|(frag_id, bytes)| { + RoaringBitmap::deserialize_from(&bytes[..]) + .map(|bitmap| (frag_id, bitmap)) + .map_err(|e| { + PyValueError::new_err(format!( + "updated_fragment_offsets[{frag_id}]: invalid \ + portable RoaringBitmap bytes: {e}" + )) + }) + }) + .collect::>>() + }) + .transpose()? + .map(UpdatedFragmentOffsets); + let op = Operation::Update { removed_fragment_ids, updated_fragments, new_fragments, fields_modified, - merged_generations: vec![], + compacted_sstables: vec![], fields_for_preserving_frag_bitmap, update_mode, inserted_rows_filter: None, - updated_fragment_offsets: None, + updated_fragment_offsets, }; Ok(Self(op)) } @@ -311,7 +459,18 @@ impl FromPyObject<'_, '_> for PyLance { .extract::>>()?; let fragments = fragments.into_iter().map(|f| f.0).collect(); - let op = Operation::Merge { schema, fragments }; + // Absent on objects predating the field: no assertion, which + // conservatively conflicts. + let preserves_nullability = ob + .getattr("preserves_nullability") + .and_then(|v| v.extract()) + .unwrap_or(false); + + let op = Operation::Merge { + schema, + fragments, + preserves_nullability, + }; Ok(Self(op)) } "Restore" => { @@ -350,10 +509,26 @@ impl FromPyObject<'_, '_> for PyLance { Ok(Self(op)) } + "DataOverlay" => { + let groups = extract_vec(&ob.getattr("groups")?)?; + + let op = Operation::DataOverlay { groups }; + + Ok(Self(op)) + } "Project" => { let schema = extract_schema(&ob.getattr("schema")?)?; - - let op = Operation::Project { schema }; + // Absent on objects predating the field: no assertion, which + // conservatively conflicts. + let preserves_nullability = ob + .getattr("preserves_nullability") + .and_then(|v| v.extract()) + .unwrap_or(false); + + let op = Operation::Project { + schema, + preserves_nullability, + }; Ok(Self(op)) } "UpdateConfig" => { @@ -451,6 +626,7 @@ impl<'py> IntoPyObject<'py> for PyLance<&Operation> { fields_modified, fields_for_preserving_frag_bitmap, update_mode, + updated_fragment_offsets, .. } => { let removed_fragment_ids = removed_fragment_ids.into_pyobject(py)?; @@ -468,6 +644,21 @@ impl<'py> IntoPyObject<'py> for PyLance<&Operation> { }, None => "rewrite_rows", }; + let updated_fragment_offsets = + updated_fragment_offsets + .as_ref() + .map(|UpdatedFragmentOffsets(offsets)| { + offsets + .iter() + .map(|(frag_id, bitmap)| { + let mut buf = Vec::with_capacity(bitmap.serialized_size()); + bitmap + .serialize_into(&mut buf) + .expect("RoaringBitmap serialization cannot fail"); + (*frag_id, buf) + }) + .collect::>>() + }); let cls = namespace .getattr("Update") .expect("Failed to get Update class"); @@ -478,6 +669,7 @@ impl<'py> IntoPyObject<'py> for PyLance<&Operation> { fields_modified, fields_for_preserving_frag_bitmap, update_mode, + updated_fragment_offsets, )) } Operation::DataReplacement { replacements } => { @@ -487,6 +679,13 @@ impl<'py> IntoPyObject<'py> for PyLance<&Operation> { .expect("Failed to get DataReplacement class"); cls.call1((replacements,)) } + Operation::DataOverlay { groups } => { + let groups = export_vec(py, groups.as_slice())?; + let cls = namespace + .getattr("DataOverlay") + .expect("Failed to get DataOverlay class"); + cls.call1((groups,)) + } Operation::Delete { updated_fragments, deleted_fragment_ids, @@ -499,13 +698,17 @@ impl<'py> IntoPyObject<'py> for PyLance<&Operation> { .expect("Failed to get Delete class"); cls.call1((updated_fragments, deleted_fragment_ids, predicate)) } - Operation::Merge { fragments, schema } => { + Operation::Merge { + fragments, + schema, + preserves_nullability, + } => { let fragments_py = export_vec(py, fragments.as_slice())?; let schema_py = LanceSchema(schema.clone()); let cls = namespace .getattr("Merge") .expect("Failed to get Merge class"); - cls.call1((fragments_py, schema_py)) + cls.call1((fragments_py, schema_py, *preserves_nullability)) } Operation::Restore { version } => { let cls = namespace @@ -528,6 +731,7 @@ impl<'py> IntoPyObject<'py> for PyLance<&Operation> { Operation::CreateIndex { new_indices, removed_indices, + .. } => { let new_indices_py = export_vec(py, new_indices.as_slice())?; let removed_indices_py = export_vec(py, removed_indices.as_slice())?; @@ -537,12 +741,15 @@ impl<'py> IntoPyObject<'py> for PyLance<&Operation> { .expect("Failed to get CreateIndex class"); cls.call1((new_indices_py, removed_indices_py)) } - Operation::Project { schema } => { + Operation::Project { + schema, + preserves_nullability, + } => { let schema_py = LanceSchema(schema.clone()); let cls = namespace .getattr("Project") .expect("Failed to get Project class"); - cls.call1((schema_py,)) + cls.call1((schema_py, *preserves_nullability)) } Operation::ReserveFragments { num_fragments } => { if let Ok(cls) = namespace.getattr("ReserveFragments") { diff --git a/python/src/utils.rs b/python/src/utils.rs index 4f7d6d7dde2..5b3801e79aa 100644 --- a/python/src/utils.rs +++ b/python/src/utils.rs @@ -25,8 +25,7 @@ use arrow_schema::DataType; use lance::Result; use lance::datatypes::Schema; use lance_arrow::FixedSizeListArrayExt; -use lance_file::previous::writer::FileWriter as PreviousFileWriter; -use lance_index::scalar::IndexWriter; +use lance_file::versions::v1::writer::FileWriter as V1FileWriter; use lance_index::vector::hnsw::{HNSW, builder::HnswBuildParams}; use lance_index::vector::kmeans::{ KMeans as LanceKMeans, KMeansAlgoFloat, KMeansParams, compute_partitions, @@ -243,7 +242,7 @@ impl Hnsw { let mut writer = rt() .block_on( Some(py), - PreviousFileWriter::::try_new( + V1FileWriter::::try_new( &object_store, &path, Schema::try_from(HNSW::schema().as_ref()) @@ -255,7 +254,7 @@ impl Hnsw { rt().block_on(Some(py), async { let batch = self.hnsw.to_batch()?; let metadata = batch.schema_ref().metadata().clone(); - writer.write_record_batch(batch).await?; + writer.write(&[batch]).await?; writer.finish_with_metadata(&metadata).await?; Result::Ok(()) })? diff --git a/python/uv.lock b/python/uv.lock index a8a7febeea4..fe581b05222 100644 --- a/python/uv.lock +++ b/python/uv.lock @@ -9,27 +9,18 @@ resolution-markers = [ "python_full_version < '3.11'", ] -[[package]] -name = "absl-py" -version = "2.3.1" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/10/2a/c93173ffa1b39c1d0395b7e842bbdc62e556ca9d8d3b5572926f3e4ca752/absl_py-2.3.1.tar.gz", hash = "sha256:a97820526f7fbfd2ec1bce83f3f25e3a14840dac0d8e02a0b71cd75db3f77fc9", size = 116588, upload-time = "2025-07-03T09:31:44.05Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/8f/aa/ba0014cc4659328dc818a28827be78e6d97312ab0cb98105a770924dc11e/absl_py-2.3.1-py3-none-any.whl", hash = "sha256:eeecf07f0c2a93ace0772c92e596ace6d3d3996c042b2128459aaae2a76de11d", size = 135811, upload-time = "2025-07-03T09:31:42.253Z" }, -] - [[package]] name = "aiohappyeyeballs" -version = "2.6.1" +version = "2.7.1" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/26/30/f84a107a9c4331c14b2b586036f40965c128aa4fee4dda5d3d51cb14ad54/aiohappyeyeballs-2.6.1.tar.gz", hash = "sha256:c3f9d0113123803ccadfdf3f0faa505bc78e6a72d1cc4806cbd719826e943558", size = 22760, upload-time = "2025-03-12T01:42:48.764Z" } +sdist = { url = "https://files.pythonhosted.org/packages/ce/f4/eec0465c2f67b2664688d0240b3212d5196fd89e741df67ddb81f8d35658/aiohappyeyeballs-2.7.1.tar.gz", hash = "sha256:065665c041c42a5938ed220bdcd7230f22527fbec085e1853d2402c8a3615d9d", size = 24757, upload-time = "2026-07-01T17:11:55.501Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/0f/15/5bf3b99495fb160b63f95972b81750f18f7f4e02ad051373b669d17d44f2/aiohappyeyeballs-2.6.1-py3-none-any.whl", hash = "sha256:f349ba8f4b75cb25c99c5c2d84e997e485204d2902a9597802b0371f09331fb8", size = 15265, upload-time = "2025-03-12T01:42:47.083Z" }, + { url = "https://files.pythonhosted.org/packages/71/43/1947f06babed6b3f1d7f38b0c767f52df66bfb2bc10b468c4a7de9eceff2/aiohappyeyeballs-2.7.1-py3-none-any.whl", hash = "sha256:9243213661e29250eb41368e5daa826fc017156c3b8a11440826b2e3ed376472", size = 15038, upload-time = "2026-07-01T17:11:54.055Z" }, ] [[package]] name = "aiohttp" -version = "3.14.0" +version = "3.14.1" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "aiohappyeyeballs" }, @@ -42,126 +33,126 @@ dependencies = [ { name = "typing-extensions", marker = "python_full_version < '3.13'" }, { name = "yarl" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/ee/ab/93ce242f899b68c51b0578c027aafa791ab3614cb9345fa5d37b5f5c8e3e/aiohttp-3.14.0.tar.gz", hash = "sha256:2882de819734c715fd1b9c11c97e09fa020d14438203d1d354d8ed1702791c9b", size = 7940674, upload-time = "2026-06-01T19:41:02.763Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/ef/f0/f81190ba488cd106c2fc6d92680e56bb223bbbbf1e6908c2617011290112/aiohttp-3.14.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:692e409052e7436029bbb32977cd7c5bf806ac5fa4085b973996785ffadad33c", size = 760606, upload-time = "2026-06-01T19:36:39.054Z" }, - { url = "https://files.pythonhosted.org/packages/f6/54/444d37eebf0f15db661ca44ec7caf93962f3c5ca92eb4c9a5d888b70aaa2/aiohttp-3.14.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:40af7ebe53c7990e110dc4ad03566b12c3ac996254298a3d39046dd69cfcb2c2", size = 514677, upload-time = "2026-06-01T19:36:42.408Z" }, - { url = "https://files.pythonhosted.org/packages/d0/d1/da280e23321c132c0a3fa7c8cc2830621d79174edc64c829443346489a36/aiohttp-3.14.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:02cb2ffbb7da32f82e21ad9952669c45bd88a80e0878264c2f59fe1c6fb2badd", size = 510155, upload-time = "2026-06-01T19:36:44.072Z" }, - { url = "https://files.pythonhosted.org/packages/09/b8/2e36d54d0991ec5bba451444004591ee0af58cb1662a3a81c562878b9c1f/aiohttp-3.14.0-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:2e2514cb7195f6d7c219339635bea71ae47d1569b051300d32df9dcfabcdb869", size = 1699947, upload-time = "2026-06-01T19:36:45.762Z" }, - { url = "https://files.pythonhosted.org/packages/57/95/a31d8ea1a0b9ecc084f5a7dd0b431ce64ef585918bb7bdc82afe11843877/aiohttp-3.14.0-cp310-cp310-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:30e8b7eeb42d02c120ca90d6c6e076a221a16b70a6dac9ae44c7ab5104cc7fe4", size = 1664364, upload-time = "2026-06-01T19:36:47.653Z" }, - { url = "https://files.pythonhosted.org/packages/01/f6/5de3ddffc87a9e8d09b3be38fbd6dd1a736b2ad477a7e787dcb85f57f338/aiohttp-3.14.0-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:63e38be0d75a654deaa06be32fb4cab883a4222940be1d05861b6717679cbadb", size = 1761186, upload-time = "2026-06-01T19:36:49.355Z" }, - { url = "https://files.pythonhosted.org/packages/33/8c/03c5438ec35d7e3a4f33fe895d6c3ec7540a7cec46065f21851211e1ee4d/aiohttp-3.14.0-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:1210d4c87cc00128160c7384ab41877a701295b97cffa6362f908a49b6e8a7ca", size = 1849727, upload-time = "2026-06-01T19:36:51.478Z" }, - { url = "https://files.pythonhosted.org/packages/22/32/5a05303b0874458920b73f48b8779cc3a93d503f121b38dcc0456dbd698c/aiohttp-3.14.0-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:1a78a77366ed158a0a54b076990e575d7b7cdb728cbfd02711eadab150f2269f", size = 1708197, upload-time = "2026-06-01T19:36:53.241Z" }, - { url = "https://files.pythonhosted.org/packages/7d/62/478f169488d61414c0a05e7fe423b59ae3d9dcc933d1f0e4acc2c5d5bc3e/aiohttp-3.14.0-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:f4d2038c64f36df96cfd3fa0937910e231eafbf897e70a06c155a817bb632fa6", size = 1578147, upload-time = "2026-06-01T19:36:55.154Z" }, - { url = "https://files.pythonhosted.org/packages/1d/af/b20af85765658972d3337834bd5eebba91b962794f2b4fc3e0ee8c85c0e1/aiohttp-3.14.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:4714c70067a08b604d0bf3bc4dfdf82e52944afab41d0428d460862763d2f79b", size = 1665836, upload-time = "2026-06-01T19:36:56.94Z" }, - { url = "https://files.pythonhosted.org/packages/8d/a3/771879cfd59948f4544b172189048905feff802f20f1c6c5411e998a3e06/aiohttp-3.14.0-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:f79bfd2847513a7ac801bbafd1de02348a37926ac439eeb4bfe96fcff4eada15", size = 1680335, upload-time = "2026-06-01T19:36:58.642Z" }, - { url = "https://files.pythonhosted.org/packages/f4/16/582e36ad1d32133cd40659f3bc98e71c22179665a1cfbbb4713bce339c06/aiohttp-3.14.0-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:25e9f1d2465a210d60edb64d7b204a147e85d4c194eecef3d1604fb5ace678ce", size = 1731180, upload-time = "2026-06-01T19:37:00.583Z" }, - { url = "https://files.pythonhosted.org/packages/11/bc/80708fe3f64a07a2c306a42fc7b009118a952709761d215f6d1b4c57195b/aiohttp-3.14.0-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:b5314743ebe926c2fda35d0a298c565c885505f6635c2a30936363404cf274a7", size = 1565805, upload-time = "2026-06-01T19:37:02.446Z" }, - { url = "https://files.pythonhosted.org/packages/57/8f/8d25897f8273a32fe4ad40a8885eec4f397377ed46e8e383078169f60316/aiohttp-3.14.0-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:28eee8de1d69711c53116df8202f1c2aa0e3f80ef912a88fc18d159d53e7110b", size = 1742496, upload-time = "2026-06-01T19:37:04.222Z" }, - { url = "https://files.pythonhosted.org/packages/9f/7d/c341d32ab2dec56c8478740695743dc6c21b383cace9376a3eab16311a07/aiohttp-3.14.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:89ed35666c95d3efe1955056afcde09e62a57a34e2a4398b17f9f6c1564f0b25", size = 1691240, upload-time = "2026-06-01T19:37:06.277Z" }, - { url = "https://files.pythonhosted.org/packages/37/0f/a81207dd7a2d4a4f645b3a3f8b5a1da1159dc63117ffb137b698fd6df50f/aiohttp-3.14.0-cp310-cp310-win32.whl", hash = "sha256:5e4646e9a6af29af354204011bf5769cb0276ec5b64653e42f90b3e13845169f", size = 454686, upload-time = "2026-06-01T19:37:07.96Z" }, - { url = "https://files.pythonhosted.org/packages/7f/ae/842357f2afb9c915715c6f5775239d987f5d0f845abf7675fa794e0a9d40/aiohttp-3.14.0-cp310-cp310-win_amd64.whl", hash = "sha256:22a8d06f204e0518a586d770032db3c7043c9ba3693081b3e3ad425e1458d594", size = 478677, upload-time = "2026-06-01T19:37:09.652Z" }, - { url = "https://files.pythonhosted.org/packages/6b/d1/330fb22c9535ec177b52396905131c6e39447244b6ca876262939af668ef/aiohttp-3.14.0-cp310-cp310-win_arm64.whl", hash = "sha256:4acfc34bd4d3c58754fc9f22ff1b5e92aabce68f3d4bf7b71a0b732d9bceb78a", size = 450364, upload-time = "2026-06-01T19:37:11.279Z" }, - { url = "https://files.pythonhosted.org/packages/67/47/7727bfe8db93f8835a001bd4359d8480cc68d1259b8bce334668f8be97bd/aiohttp-3.14.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:54bf3522d6f7351e55f89a62d5c2bf138ad557b031670266c5df604ae88e0b5a", size = 759147, upload-time = "2026-06-01T19:37:12.918Z" }, - { url = "https://files.pythonhosted.org/packages/eb/f2/cd3fedff6fade73d71df9ec908c210cec518ef90fd00289250684b90aecf/aiohttp-3.14.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:0746d9fb0ac4fdef643a84494efe3f06d50335dd8c7a530228b86448aae0a803", size = 513705, upload-time = "2026-06-01T19:37:14.633Z" }, - { url = "https://files.pythonhosted.org/packages/5a/fe/49746b6b610144a06323bebd8e1211a390310d8c69b98dd6d52df341bc3e/aiohttp-3.14.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:9f3a96b6d39a4872222beee72e1df41d2ff886ae96152cf3e757ef8c5673ef0e", size = 509627, upload-time = "2026-06-01T19:37:16.385Z" }, - { url = "https://files.pythonhosted.org/packages/4c/3f/28f2f6cf3d5c0e7b01b27140d0e7873fd11fb341169ad3ce78ad04aba628/aiohttp-3.14.0-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d336820adbb914debbc90a1d8c1bfc4bea55996aecf64866a989d35d1f9fd903", size = 1769293, upload-time = "2026-06-01T19:37:18.067Z" }, - { url = "https://files.pythonhosted.org/packages/97/6f/2e5f1b525d5474b12b3c60abf733a755845f3bceff21542081ada515f837/aiohttp-3.14.0-cp311-cp311-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:71b2604c9bfc1b115547d63a094d5244b3f02799833513a99a68aaa7b167c4cb", size = 1732363, upload-time = "2026-06-01T19:37:20.138Z" }, - { url = "https://files.pythonhosted.org/packages/a8/ce/596120faa85ca7b19cd061e3f2f3be23aa8f11a0aedf9191db9e0da1bd76/aiohttp-3.14.0-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:610d68800435903e303ca0542b9d3e4eb72a12ff33a6d471a070c1d81eebd3c2", size = 1840375, upload-time = "2026-06-01T19:37:22.104Z" }, - { url = "https://files.pythonhosted.org/packages/72/3c/a7ffe05a757a4a7867643da69357ec41f506879fbd1b231d2ed90af246b2/aiohttp-3.14.0-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:514db9a79337068981ee2137310283a07b4b885c584991097a91a4da419bcb81", size = 1921484, upload-time = "2026-06-01T19:37:24.068Z" }, - { url = "https://files.pythonhosted.org/packages/93/fa/2c861170bbd4a491de93a69e081db1d971092569e0d593a98ef62c384dc1/aiohttp-3.14.0-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c452d17eeb95d563fc8b936f3050301dbd1d268126c4632d8b70ede9696202ee", size = 1774153, upload-time = "2026-06-01T19:37:26.256Z" }, - { url = "https://files.pythonhosted.org/packages/9d/da/1d2f5a165f47ec9b1f69d37b8b977fdc4d501aa72ffb7930db27bb9e49ea/aiohttp-3.14.0-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:ed94a81506e3d1bdbad5108f497a58f2a2354aedb4ca314d5326f07d1fd1ac2d", size = 1632569, upload-time = "2026-06-01T19:37:28.192Z" }, - { url = "https://files.pythonhosted.org/packages/46/1d/7a6e295c4257252f70f69e90864fdad74b6a1293054fb3f9e65a15de6d63/aiohttp-3.14.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:1394dce36e0f0d260ac0b555a654de19cb989f3c1b8bdd24f505314dfea18a00", size = 1740325, upload-time = "2026-06-01T19:37:30.08Z" }, - { url = "https://files.pythonhosted.org/packages/f1/7e/e1899b1ca3ec62f1eab2a5cbde14039b97493f7f53eb88d9b668562ffa8d/aiohttp-3.14.0-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:d1467d1e7b48a73ca7237e0ee4335f3d02b923dbc27b82fd254bc301c97d4026", size = 1748691, upload-time = "2026-06-01T19:37:32.211Z" }, - { url = "https://files.pythonhosted.org/packages/ec/54/4e6b61c1fe7d3433f82bcc6bd7e4d7c683a742a10c9b12a025fd3695c047/aiohttp-3.14.0-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:6a5f3532125233c261cf61f32df4059cfcf482eb793c7d3db8452e3142028b86", size = 1814477, upload-time = "2026-06-01T19:37:34.173Z" }, - { url = "https://files.pythonhosted.org/packages/9c/38/86fd51be2e08d8e45c83d879d255f10391903cd9fe2a16512f7591a15873/aiohttp-3.14.0-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:3ea81eb518a2ecb319d8ec6d1424a37c773f6634bd87d6985eb606b2faac419f", size = 1623393, upload-time = "2026-06-01T19:37:36.281Z" }, - { url = "https://files.pythonhosted.org/packages/78/49/466e947a42a88ee23c486d036e7e5d1b097f1bafd8084ad9c9a0a92f0f43/aiohttp-3.14.0-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:32e735c3182de7b64f6941a4ede48b38c7f47d9437bd615dd30b5bda8fa1bc93", size = 1824097, upload-time = "2026-06-01T19:37:38.421Z" }, - { url = "https://files.pythonhosted.org/packages/f3/89/35f3410bc284682338a1be6b6ea0c5abfa05f063942cfaa9256608440434/aiohttp-3.14.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:c21ca9a1c63d4509158f478aeb9d02914dcc52adc68d1bc9dee2452284ee5996", size = 1764790, upload-time = "2026-06-01T19:37:40.755Z" }, - { url = "https://files.pythonhosted.org/packages/42/80/2d4291bd5724d3d17e5951aff5a3e02281483fb47295f0788276ee66cd73/aiohttp-3.14.0-cp311-cp311-win32.whl", hash = "sha256:19ca5fc84130675ba11c6ca5c7da5cb65f7bf8a32cdd2b616bf49cd334688aae", size = 454176, upload-time = "2026-06-01T19:37:42.837Z" }, - { url = "https://files.pythonhosted.org/packages/59/ed/41d0ad4f6ececffc32bdf1f7b494e5498f7ca5c849ea2e3cc9bbd1668251/aiohttp-3.14.0-cp311-cp311-win_amd64.whl", hash = "sha256:d488e6e9d3bb8ba5ae7066d5be885ae9670eba021b8c6ccb9a3a568e6b19d6e5", size = 479334, upload-time = "2026-06-01T19:37:44.776Z" }, - { url = "https://files.pythonhosted.org/packages/d1/86/c0b5e305c770053f8c3d069bb52b8196917ba91949d1962d52eb307fb0d2/aiohttp-3.14.0-cp311-cp311-win_arm64.whl", hash = "sha256:8b93618102caf12801638a01a2b478a55410ddd71bd41cfaf6f707953a49ac43", size = 450262, upload-time = "2026-06-01T19:37:46.461Z" }, - { url = "https://files.pythonhosted.org/packages/89/97/2b6889bfb6b6847520d50d95eb8c4307a45e28aaca39faf4a9454b3d1b2f/aiohttp-3.14.0-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:b29518c9c2ec7e373e68259206a137c7f4f5439c58baaec4b5ab3ab799850a4e", size = 750194, upload-time = "2026-06-01T19:37:48.164Z" }, - { url = "https://files.pythonhosted.org/packages/21/e2/62634b7fff918ed98c3c6b2f0e70d520f7f28846cb412d451b04354c6459/aiohttp-3.14.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:dbec68ce61b64cb73cab4d33df9433427b1713c8bcccb181dce695c1b6f8e87c", size = 506966, upload-time = "2026-06-01T19:37:50.014Z" }, - { url = "https://files.pythonhosted.org/packages/dd/fb/5ce075150828c797a5106f1c2fb26034e709d4289b9d2bf8b07f1e59fac6/aiohttp-3.14.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:3cdf534aa455593e589302990c5097aa5c92c06c4262a20da22934f9186a5fff", size = 507527, upload-time = "2026-06-01T19:37:51.96Z" }, - { url = "https://files.pythonhosted.org/packages/01/d5/405a0ae4e6b081754a3609c1c97c63a950e000a2def16046f1e736933a0e/aiohttp-3.14.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:cb6c657104393b5fbff01a5f59b2023db74058a8077d94475d6c25d03882a108", size = 1762420, upload-time = "2026-06-01T19:37:53.839Z" }, - { url = "https://files.pythonhosted.org/packages/ae/1d/e05a7c896b15a6bc6fb8fc5319eb437861c2c49c34559ef928add6590315/aiohttp-3.14.0-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:46fbbec4e4fab7428d4396a3823f9320e4560aa3113b89eeebce712c27c9ed5a", size = 1733672, upload-time = "2026-06-01T19:37:55.791Z" }, - { url = "https://files.pythonhosted.org/packages/cc/22/a72f7c459e195fa41bf4f7abd1f925b91fe91f8097e51c654229ba144a33/aiohttp-3.14.0-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:2c2c7e05dd5335b298085abf45ddf98673934c3ee1c083d0b9ea13d4186ad500", size = 1805064, upload-time = "2026-06-01T19:37:57.931Z" }, - { url = "https://files.pythonhosted.org/packages/80/50/e85bdaba0be59ca4838005ebfef4048fcdd5f35a02b07057a9a123394440/aiohttp-3.14.0-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:3c7139100fbaae76515b73051d8f0aa3a3ff02e415eec8a8eee8e2223d9ba955", size = 1902125, upload-time = "2026-06-01T19:38:00.225Z" }, - { url = "https://files.pythonhosted.org/packages/19/d8/51de5c6b971c27bb1ef620293b8d1ca611ec78736b34b3f6ccf68e4c8785/aiohttp-3.14.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:78d6f9286a629ce52728430afe18f8ed2b6c39a1fddb3802d7244b9983910ad2", size = 1783112, upload-time = "2026-06-01T19:38:02.641Z" }, - { url = "https://files.pythonhosted.org/packages/73/ae/b4402bfde77e43dfb1b6ccff83c7b7ab63ed06b50c4754f0c5423fb374fe/aiohttp-3.14.0-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:cc3c3e12cdaeb92d7dcf13db00e9f6b1956b910e47256e696df1cfa946d02159", size = 1586356, upload-time = "2026-06-01T19:38:04.637Z" }, - { url = "https://files.pythonhosted.org/packages/bc/05/750a3265ca4dc54a460bd0cb1121a8f2ce9171fce4a135fb47ea7fd594d2/aiohttp-3.14.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:4d6a998191f5ebe3b8c28463ff72bc030250008b3193c402464efadd08b5ca02", size = 1723119, upload-time = "2026-06-01T19:38:06.713Z" }, - { url = "https://files.pythonhosted.org/packages/37/01/8c0812c50b3b1b1c37b323bf170d6be8847a8f234060485b7d1e71953f60/aiohttp-3.14.0-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:0fc2b75ae8d169d853be2862d960be8550da6c5c65711d5476407eb3fdb006bd", size = 1757216, upload-time = "2026-06-01T19:38:08.736Z" }, - { url = "https://files.pythonhosted.org/packages/47/2a/50fb98028a26887cbe48dcc1df92a90825615bc73b5584301304090cded8/aiohttp-3.14.0-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:16eee56bcc72d04600bc56c1759982c2385ec0b41d3fd3521f836bf64a0957ef", size = 1770500, upload-time = "2026-06-01T19:38:11.111Z" }, - { url = "https://files.pythonhosted.org/packages/bd/32/0ffd598a2fa2b9a423daf242e700cfdabda35d6e602394ad9ae58972c1c7/aiohttp-3.14.0-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:5a2e7ca615c3ddc15b82687e05a624e5f5cba3f1d6c20cb81172d70ea498451e", size = 1576224, upload-time = "2026-06-01T19:38:13.391Z" }, - { url = "https://files.pythonhosted.org/packages/0b/f9/b9fc381dd9b66afb33f2634c40e229d106467be0afcabe79648631ab6712/aiohttp-3.14.0-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:f0b7b8bbbec3ce9467ee0ebe334622fd90624f593edd3136c567811453fc4fae", size = 1794252, upload-time = "2026-06-01T19:38:15.498Z" }, - { url = "https://files.pythonhosted.org/packages/a8/fb/05d9214c975f23225a8cd5c439325e338c7c377b315480ef3871db51f54e/aiohttp-3.14.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:5ba10966d4f03dd96a14365be4b8e37c327c76f11c3ca867116966cdd9f98066", size = 1760193, upload-time = "2026-06-01T19:38:17.624Z" }, - { url = "https://files.pythonhosted.org/packages/d9/4b/02992fc4fb9e1b6673ee3f888a8e587a6447afda1f6f4aca776c148c2876/aiohttp-3.14.0-cp312-cp312-win32.whl", hash = "sha256:101df7779c80c0636014a6b2c6642acd3efb5b355d48347c9d7dfb720aee9430", size = 448650, upload-time = "2026-06-01T19:38:19.545Z" }, - { url = "https://files.pythonhosted.org/packages/39/e9/246532214c3abda518477cbaaf16d420295ad8effa5233844cbb38f299ab/aiohttp-3.14.0-cp312-cp312-win_amd64.whl", hash = "sha256:b0a5747586d4467efd1f932710b269131c9717a872dce082cd92a00c1c13123a", size = 476145, upload-time = "2026-06-01T19:38:21.505Z" }, - { url = "https://files.pythonhosted.org/packages/2b/c3/63f8c20090048915711598b0adf475b149216d736157961de06480a45b15/aiohttp-3.14.0-cp312-cp312-win_arm64.whl", hash = "sha256:5f1c5be60add78fabb4aacd13c5a348ae79d2fcbfc7fa78da8f1eb192273b370", size = 444250, upload-time = "2026-06-01T19:38:24.027Z" }, - { url = "https://files.pythonhosted.org/packages/21/61/d11f7d9a3144bffe825247d6367cd93053666da50b94707c9129c78868d5/aiohttp-3.14.0-cp313-cp313-android_21_arm64_v8a.whl", hash = "sha256:25400d710641a8040bf022a8a99f579e581ffa1c5bd42c33255d7d6f3957c127", size = 502399, upload-time = "2026-06-01T19:38:25.955Z" }, - { url = "https://files.pythonhosted.org/packages/4f/9b/a7e317625d36356844f8bb022cabd305b541f968856cc3c2e0b58e53ee6e/aiohttp-3.14.0-cp313-cp313-android_21_x86_64.whl", hash = "sha256:c5492b9929826e07cc3fcb9739ae87aab05dff6b5e67a9b73fd1700c6d008981", size = 510068, upload-time = "2026-06-01T19:38:27.828Z" }, - { url = "https://files.pythonhosted.org/packages/11/41/cc2d2cfbfbdc3126ba258f3cd27d1ac8a33492ae3c35a4583ee21f0ba7f1/aiohttp-3.14.0-cp313-cp313-ios_13_0_arm64_iphoneos.whl", hash = "sha256:3366751d68d237c621264233a32f3078bbc21b7904ab90a77e03d21390c742c6", size = 481670, upload-time = "2026-06-01T19:38:29.836Z" }, - { url = "https://files.pythonhosted.org/packages/3c/07/381f4023c3b08cb616e520f566d8c58957abad54e56441d41fe67cfb0195/aiohttp-3.14.0-cp313-cp313-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:57ea07d28695a7a40304d42251892a8df765e5588c10ee32afeddcd5df33c0a2", size = 487591, upload-time = "2026-06-01T19:38:31.704Z" }, - { url = "https://files.pythonhosted.org/packages/fb/4d/4506fdb7a022bdf70011a3bbb4ca00c5c570026ef6a3c5bd7bc70c39089c/aiohttp-3.14.0-cp313-cp313-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:076cb014191ae2e65d949e1ad01f1dcfe33e32789b5172510f3e79c79fc04d50", size = 496503, upload-time = "2026-06-01T19:38:33.6Z" }, - { url = "https://files.pythonhosted.org/packages/ef/7d/c814111e04894a45d9e2defc94443879a6f118d9633d5fedfe6e2e8af5f0/aiohttp-3.14.0-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:2f3fc37054564dee64a855b5b092d87ec35dcddfaabf7dacb1c8a2b1f83dc0a9", size = 745870, upload-time = "2026-06-01T19:38:36.013Z" }, - { url = "https://files.pythonhosted.org/packages/c6/ee/80eee0efddfe187e7cd05027086b7ce1c0e492e82a4eda58f5c5543a44a0/aiohttp-3.14.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:8fcaef74d2ab0f607d7ff85a0d15e21bb5a258c4a58df1908396eb50d7f4ed3c", size = 505588, upload-time = "2026-06-01T19:38:38.282Z" }, - { url = "https://files.pythonhosted.org/packages/d6/f8/0f28f04eef75d52fc9c715dde7ce9c0abb810fd20cfeb0fea7afd2ab1e98/aiohttp-3.14.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:e4c01b0bfc6209590960e68eac083cd22d5d87c21f974dd6208cafa5d3542bc8", size = 504492, upload-time = "2026-06-01T19:38:40.611Z" }, - { url = "https://files.pythonhosted.org/packages/ff/db/44c755232085545065c94378dfce38641b1aee647f4939fcd32f5b32e719/aiohttp-3.14.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f12eb7896e81caf403a2b18c9406426f1207361e7239c057ab29c076d4257e83", size = 1752111, upload-time = "2026-06-01T19:38:42.682Z" }, - { url = "https://files.pythonhosted.org/packages/5e/6a/42e030a46743841414402a3b00cd3d78419055e86c66fb5822c14b5abfc6/aiohttp-3.14.0-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:6c79a044cacf360ec46738d863d2f41c9300d2a06ef4a7402ea0df306a350e61", size = 1729674, upload-time = "2026-06-01T19:38:44.79Z" }, - { url = "https://files.pythonhosted.org/packages/34/26/3199beb415202e3108e7b83ecebe10914d806d33fb9860c3e4aa60a19be3/aiohttp-3.14.0-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:85e0675f47be4eff0636bf88c02140ea89168ae0df3ff1f3f464e9de9610d277", size = 1798808, upload-time = "2026-06-01T19:38:47.01Z" }, - { url = "https://files.pythonhosted.org/packages/bd/94/b9b6fcf0ee17c21d0d19fb8c22bf83ad18f82e702a9c3bd901a868f5e446/aiohttp-3.14.0-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:7b33e751cab03fdc960095b1e326cb5a03f5ee577d6ded59f3d1c100f8668882", size = 1891921, upload-time = "2026-06-01T19:38:49.233Z" }, - { url = "https://files.pythonhosted.org/packages/c5/a3/3800dbd095cb2bb165a7ea5d94d790914677e27f45638c7d80e3f34c8945/aiohttp-3.14.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:26d9224c6dd7f5c749aba4f61315a894601448b28d94d12f4dea0903e26d2096", size = 1777241, upload-time = "2026-06-01T19:38:52.04Z" }, - { url = "https://files.pythonhosted.org/packages/21/2a/45be91ad1b860508557448d4cc2e165a2ee68dd865657b73bf66cc5a00fb/aiohttp-3.14.0-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:6281aecdf2732940f4fe06bd6adec5ae4d59b78b080b8e3a6b81467301010988", size = 1579554, upload-time = "2026-06-01T19:38:54.508Z" }, - { url = "https://files.pythonhosted.org/packages/b4/3d/dc94df99ed1511fdf28314f722643ed334112643cab00223577085e788c4/aiohttp-3.14.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:23e8314e7aed8576fbe33314d218bd81447a3adbc91dc36f1163bf583cd3084c", size = 1714864, upload-time = "2026-06-01T19:38:56.788Z" }, - { url = "https://files.pythonhosted.org/packages/ae/e4/1f1c8acbb3acd5c8f795473b92c9c3d44eb60a5692c6104256c8a1c83a0c/aiohttp-3.14.0-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:3b54fbff46127aeafdd764cecd0d99fa2f24a0e37ea5c18a7c3a4ac450df1db3", size = 1749803, upload-time = "2026-06-01T19:38:59.367Z" }, - { url = "https://files.pythonhosted.org/packages/0b/c8/c45ea6e7ed84cebba939b9c334498a045ba19d79c61b0110df5f21580de3/aiohttp-3.14.0-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:b27d89af91a555f58e08e4902dbcbc48862fd40095720ca705990476bd93b7ac", size = 1765023, upload-time = "2026-06-01T19:39:01.651Z" }, - { url = "https://files.pythonhosted.org/packages/a8/a1/a932941784432962fe390e1066823aaef64b4e5ac9fa595df57b5fe472a9/aiohttp-3.14.0-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:25d2326a4967bf705a9f9913a13005e93b6020ad8a9f6bd6bd78850d5171332e", size = 1571671, upload-time = "2026-06-01T19:39:04.044Z" }, - { url = "https://files.pythonhosted.org/packages/b0/01/e1280feac522597a4d46eb67a0cdfa053cfae263033030b761ab146f29fb/aiohttp-3.14.0-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:a1d209375c503472b3c0a340cdf3c55fcd82e84b46dda7caeaced59faba373ec", size = 1789904, upload-time = "2026-06-01T19:39:06.294Z" }, - { url = "https://files.pythonhosted.org/packages/fa/10/ab28818262f4d26bdb47ed5f1fc7999b69e2fc6e0370b02d0f49011f45ea/aiohttp-3.14.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:666c7c5036df57b693026398b69b41874a1931ac5b3485fd910e57bfac253869", size = 1754516, upload-time = "2026-06-01T19:39:08.788Z" }, - { url = "https://files.pythonhosted.org/packages/af/cc/c122eabd7a1b7e0c9bbdd6be60e4715905b858399145d9df872bb94f1427/aiohttp-3.14.0-cp313-cp313-win32.whl", hash = "sha256:23f094a1ef64823fd35854ddf5c7a80a078162f37f9d2f7c6142b51a6affa456", size = 448656, upload-time = "2026-06-01T19:39:11.171Z" }, - { url = "https://files.pythonhosted.org/packages/41/a5/bab07d79848a00eedd8ed979ccb302aaea3ac6eb9fa16bd0ed87135869b4/aiohttp-3.14.0-cp313-cp313-win_amd64.whl", hash = "sha256:e03abdaa17d553f17e1d1d06bb266b3970106c78051d06795723e748d8e49d11", size = 475803, upload-time = "2026-06-01T19:39:13.439Z" }, - { url = "https://files.pythonhosted.org/packages/d1/a0/f03ade8566c153666a3871afccbedf6d99911da006325e1fc6cf72a2de99/aiohttp-3.14.0-cp313-cp313-win_arm64.whl", hash = "sha256:acdb400538cf4769543548bb5d1eb23d39bed4f96554a6078cb728c7cb2c268b", size = 443889, upload-time = "2026-06-01T19:39:15.945Z" }, - { url = "https://files.pythonhosted.org/packages/28/03/5f36ab196a88ba5e9648ae5643e6531e67a3a8c0e96f9c6510ff41540fec/aiohttp-3.14.0-cp314-cp314-android_24_arm64_v8a.whl", hash = "sha256:363ef9e91014e7891679bfb2ac0a7c6ea93435dbbfd10ecf41b9f06fcf506c5f", size = 503330, upload-time = "2026-06-01T19:39:18.195Z" }, - { url = "https://files.pythonhosted.org/packages/2c/ce/8b49ec2f30f68e02f314f4832186cd45e583360a5a386058be36855d23b6/aiohttp-3.14.0-cp314-cp314-android_24_x86_64.whl", hash = "sha256:884a4edbdad77be9d0ef36142c8b504351b170df0bf62b51e784fadabf311c42", size = 509822, upload-time = "2026-06-01T19:39:20.396Z" }, - { url = "https://files.pythonhosted.org/packages/1a/fe/6edbf5d39bf29322b6816365b17ed8ede4dace164a3aea1abcd30110eb78/aiohttp-3.14.0-cp314-cp314-ios_13_0_arm64_iphoneos.whl", hash = "sha256:70ea956f6cc4a37620966b56c2e205d88ca3e6d85ec063277e414b1035cddad3", size = 483329, upload-time = "2026-06-01T19:39:22.607Z" }, - { url = "https://files.pythonhosted.org/packages/1b/5a/fae531bdbc6456fb6241f46b7b81e4d8a0dd3fc09118a0055dc7141ac1ec/aiohttp-3.14.0-cp314-cp314-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:ea3b9806c89f61da22fddf1f12dd524fb368e5e28f1261fbdafe5c3cd8ce893b", size = 489502, upload-time = "2026-06-01T19:39:24.881Z" }, - { url = "https://files.pythonhosted.org/packages/36/f4/48a7b0414db7fed77a03d5dde34508c026afd83510ab6bca08c313855776/aiohttp-3.14.0-cp314-cp314-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:a071be341c2bd9b0188e62d173509f024e0a35b1c342c53c50f8daaeda8c3bd8", size = 497357, upload-time = "2026-06-01T19:39:27.197Z" }, - { url = "https://files.pythonhosted.org/packages/75/75/e85a13a370acc007fca5feb1fd1b88ac2d8426e6dadd625479b7cadd55a3/aiohttp-3.14.0-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:198cfe61bf253b19da1fb3e0fa122249dc4f14c12709493fed8054aa0411cc76", size = 750898, upload-time = "2026-06-01T19:39:29.563Z" }, - { url = "https://files.pythonhosted.org/packages/9e/e4/3d637f800c724eff0e2bed64df72557444482366fd0a35b0cec0e6968f6c/aiohttp-3.14.0-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:9dc203d6ce6b9106d54e2a93f41dfdfebfbca2d99962ba503bfd3e5921a6549e", size = 506986, upload-time = "2026-06-01T19:39:31.872Z" }, - { url = "https://files.pythonhosted.org/packages/1d/df/35161f3598bf7501d2b2a805b41ab4f45a2e34150c421bcb4ef8c0d281a7/aiohttp-3.14.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:9e19d17ab02bf16832a2c8c0d55a486792c5b1645665652ee9531aebcc30cb72", size = 508033, upload-time = "2026-06-01T19:39:34.137Z" }, - { url = "https://files.pythonhosted.org/packages/e5/39/b36e5d3d31e850fb4691dd3e941684ac490a2559249f6fa634b6b0fdf020/aiohttp-3.14.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d925fba0c14d5b498a8028b0107beebdfd16c5d48d702ff54f879cb017aaaca3", size = 1746213, upload-time = "2026-06-01T19:39:36.654Z" }, - { url = "https://files.pythonhosted.org/packages/b1/28/24e1409e605a9aa5d84abe0e2acb365354b70ae56d40948101cabe3341ab/aiohttp-3.14.0-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:d33e61021222ce7f9792bcac870d6f58d8adfceda33ab857b01264f4560f2c5f", size = 1705862, upload-time = "2026-06-01T19:39:38.968Z" }, - { url = "https://files.pythonhosted.org/packages/8c/d0/e5eb3ff1daeaf644c7e36a957517672494122628e067c38b263fa04eda77/aiohttp-3.14.0-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:44eca38755d0105bb32f47d085f5dd449846a449e1245fc105889e3279dcf8e3", size = 1798909, upload-time = "2026-06-01T19:39:41.334Z" }, - { url = "https://files.pythonhosted.org/packages/d3/ba/8943f906f0570342886ababb9a722a44e360f786a028c5e0b0e29e3f735b/aiohttp-3.14.0-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:f13087e06f68fea4941c21a0c541c00553aa16e4f8fd7bbe2b198df761e964d6", size = 1868892, upload-time = "2026-06-01T19:39:43.807Z" }, - { url = "https://files.pythonhosted.org/packages/3a/05/27df32c844b2156e1675a8d8ec22d963e3c8ba469ed7ceb1863320c7b521/aiohttp-3.14.0-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ff82be7f1ef73634cb77890a770743239bc3d487b848669be1c599889336dc0a", size = 1751659, upload-time = "2026-06-01T19:39:46.398Z" }, - { url = "https://files.pythonhosted.org/packages/7f/62/da182e5910ab912b2e88aa919b61a16046a37a95714a5795b02eb57b2d18/aiohttp-3.14.0-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:a150c0875ac8fd87f1c398650841308a30d65facf7416b12dbdb9cfdcbe5a48c", size = 1578775, upload-time = "2026-06-01T19:39:48.902Z" }, - { url = "https://files.pythonhosted.org/packages/66/e3/53c67097e8a5ce98625e91e3fa7f43c9c6940de680345d03b3509a72a078/aiohttp-3.14.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:edc01ea4e1ec5a1649a28866262bf24195889ff7b27bdd947029a6086741de9b", size = 1710090, upload-time = "2026-06-01T19:39:51.392Z" }, - { url = "https://files.pythonhosted.org/packages/dd/55/0e2732ca598c7a4dfe8a775662376d0ca2977cb1030e48386d4da5d9a456/aiohttp-3.14.0-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:540632bf882ff8fc88f2e1697be0761578e89e0d79fb4a8a6d65dc5da7e729d4", size = 1715016, upload-time = "2026-06-01T19:39:53.807Z" }, - { url = "https://files.pythonhosted.org/packages/5a/96/f0b73730798c9ca525afc30b39f1f81bbe24e245d9654c54d3b39d63212d/aiohttp-3.14.0-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:860a86bc2c80237f5dff52edcf427e10a8d8352271fd84845429a3e60199e02c", size = 1763810, upload-time = "2026-06-01T19:39:56.31Z" }, - { url = "https://files.pythonhosted.org/packages/71/cc/11acb6c4518f448323405a7312b6f255d0f974a34373ad1db7633c4aadc8/aiohttp-3.14.0-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:5cbd50e6a50d6b99283a826b18cbdebf65b0797689a7535cb0e9dd37be0f63c3", size = 1573064, upload-time = "2026-06-01T19:39:58.718Z" }, - { url = "https://files.pythonhosted.org/packages/de/2d/28c31dde0a7dc98c0ee7d0da2ddcec3f7688c4fc131e5989e278d0c03c0a/aiohttp-3.14.0-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:20144819e99db593e22bbd2f3f2691a5e149f879142d6b8670254708853ff4fb", size = 1775765, upload-time = "2026-06-01T19:40:01.195Z" }, - { url = "https://files.pythonhosted.org/packages/b8/69/155c4ef3aec96417d47024800472b33b16c5d8a665371dcd044c2afdf25d/aiohttp-3.14.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:26b6d79aa54cb4ed50cc7d41ed14e99e0f1fc8e7c2d42f2e05b37aea897b2b52", size = 1733716, upload-time = "2026-06-01T19:40:03.631Z" }, - { url = "https://files.pythonhosted.org/packages/5f/44/6126116fd8a316b712bb615660b855c78466bb67ba1bb1742427eafcf7ac/aiohttp-3.14.0-cp314-cp314-win32.whl", hash = "sha256:106ed074a856f3e21d186b8579e2c8afb6da598e267cdaab01059e13db2fc44d", size = 453684, upload-time = "2026-06-01T19:40:06.277Z" }, - { url = "https://files.pythonhosted.org/packages/a2/d7/eff4c58a88c5cac5e38b55f44fb8a6d3929c3cbd77356e383e094d3220bd/aiohttp-3.14.0-cp314-cp314-win_amd64.whl", hash = "sha256:4f770846edae8f00ecc57af825bce811f787f87a7dcf0e90d191790efe5b31f7", size = 481758, upload-time = "2026-06-01T19:40:08.653Z" }, - { url = "https://files.pythonhosted.org/packages/d7/ed/17b5bd9fbcb46e688f02e572f517754a9a75831e7b54702f027761dc4fa5/aiohttp-3.14.0-cp314-cp314-win_arm64.whl", hash = "sha256:acf1581c4f21ed4b80a2dded504d87b055a071a84d5737ea966435f768275ac6", size = 450557, upload-time = "2026-06-01T19:40:11.03Z" }, - { url = "https://files.pythonhosted.org/packages/12/34/6180103ce9aabc8ebff3f7bb55a1228ffe60f61042823031d9692cb7b101/aiohttp-3.14.0-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:6aa1a40f9cbb3da9f80714c5966b8946c21e6a2530d809b9498b33161e3c8733", size = 787878, upload-time = "2026-06-01T19:40:13.401Z" }, - { url = "https://files.pythonhosted.org/packages/92/e9/08954a40e8b7baa3d8beadd2b074b186e9b1e9c8ddabc288678a6265de50/aiohttp-3.14.0-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:b62af5a8cc96a194eaa01a9ed7b34a3ffa58d3d8daaa1a0d7a749353ad12d228", size = 524400, upload-time = "2026-06-01T19:40:15.972Z" }, - { url = "https://files.pythonhosted.org/packages/08/6a/b5965a634ac4d5ba99a463314cf4ab214ca073fcdc38a15e0294273701fc/aiohttp-3.14.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:6eb63b1417efaf7d1002a6ad034a40d44376afcc16508a57f8e74b49ad26a095", size = 527904, upload-time = "2026-06-01T19:40:18.28Z" }, - { url = "https://files.pythonhosted.org/packages/06/b4/932bcdd850c354d9bcca30f360e475d7852e30413fbbd44b182782ed5432/aiohttp-3.14.0-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c20b9ad156a79eb97be5cf9e069eec01d2f0dc8472ffbd75299a8b2d4c2cbbde", size = 1912162, upload-time = "2026-06-01T19:40:20.825Z" }, - { url = "https://files.pythonhosted.org/packages/c6/85/ce79bab0310d2e3fd2d7bc7e44412abeff7c8338f8a21dd0f2f1714989e5/aiohttp-3.14.0-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:40ae7b0642c25632c7eabc4a04754012691864d2a1b93becf7cddb76027b838a", size = 1778813, upload-time = "2026-06-01T19:40:23.726Z" }, - { url = "https://files.pythonhosted.org/packages/05/54/ba62ac2d1bc87e010aad23751e383b8794e45d931df67677313a2da78823/aiohttp-3.14.0-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:95f5217e76a046b9f228a101717ef8d42b1eb3d9d196d15202db5bf41df88936", size = 1899969, upload-time = "2026-06-01T19:40:26.406Z" }, - { url = "https://files.pythonhosted.org/packages/dc/82/7cc7907725d83a19f31551334061e1ab8e108b1d7ac52632a2a844a4acb5/aiohttp-3.14.0-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:1a4a9f17e85b80878c176695c1998c790e83731d8271881e5d356488652a1f9e", size = 1991771, upload-time = "2026-06-01T19:40:29.061Z" }, - { url = "https://files.pythonhosted.org/packages/d0/1c/a57de71a4508c93a830b77c28af3d08cd97f606dedfc6b94275347744508/aiohttp-3.14.0-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:145262119b07d7f95abc1839add35ba2bfc84551d4b4660ca11542c0b215455b", size = 1868606, upload-time = "2026-06-01T19:40:31.843Z" }, - { url = "https://files.pythonhosted.org/packages/9c/ae/3839726cd49150a53ed340cc24ce5ba09d4c2117020ef9d45542bec5eb2f/aiohttp-3.14.0-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:49a33ded29b0b2fa7a367a02cf0fb89af602bb87542a16177ec8ce1c9c51d12a", size = 1665437, upload-time = "2026-06-01T19:40:35.01Z" }, - { url = "https://files.pythonhosted.org/packages/35/1e/c237923232c7da7f0392ea25d89fc5e60c0e93f685f4ebca8e7bcdd5271c/aiohttp-3.14.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:2cc736a9c9fc2bc4dd71fd404815741b6573df27c3f985948ec4076989ac57de", size = 1834090, upload-time = "2026-06-01T19:40:37.733Z" }, - { url = "https://files.pythonhosted.org/packages/98/02/a5a7a2524f92d3911761b405a7c067c751891942144adc13e2ad79611e39/aiohttp-3.14.0-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:b4141a3e5342ee3053a9cab54d25b64ed28289c1041e4c54b3d99839314d90ce", size = 1816907, upload-time = "2026-06-01T19:40:40.46Z" }, - { url = "https://files.pythonhosted.org/packages/fa/76/a8b9f0d09234d516af9f2d7dd715557f33b5da3b0b56ead41d1170e86e3c/aiohttp-3.14.0-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:e30871b2d58996cb81aac52d2b1d15ac05257131ef0f90f18c2115a380fbfe7c", size = 1840382, upload-time = "2026-06-01T19:40:43.48Z" }, - { url = "https://files.pythonhosted.org/packages/c9/8e/140e715a0a4bbc211979ea30ec8396ad2ed5bf90ab87d8058fc4668b1923/aiohttp-3.14.0-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:667b881d083ccae3900ea5a241e17e5007ca78844c53ed389bb63d48f729d9c7", size = 1659497, upload-time = "2026-06-01T19:40:46.265Z" }, - { url = "https://files.pythonhosted.org/packages/10/c7/7ba5de8af9650b9767b063c675427b8685f43fa7ce563673a7bc3af60f08/aiohttp-3.14.0-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:b584dfe615d151e9b8f0a8ecb3aee6147f2927ec5b95ba25fe621f5377510928", size = 1870829, upload-time = "2026-06-01T19:40:49.583Z" }, - { url = "https://files.pythonhosted.org/packages/cc/bc/2aaab2f85cadb26ea59c091fa2b8e370d625154b5c14b478f1b489d07551/aiohttp-3.14.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:6199707cc40e0e9cd39c36fbc97bec416c704e1d0ddce03412bb3b3e6a90ccd0", size = 1832281, upload-time = "2026-06-01T19:40:52.303Z" }, - { url = "https://files.pythonhosted.org/packages/39/98/31b9ad9fbc01f0075ee7221002df5fd2d10b647f451ca5f30edc802d9dd6/aiohttp-3.14.0-cp314-cp314t-win32.whl", hash = "sha256:a8d93334d4961c9d566b1f046c81dee475b7c21eb730728d38237bfa70d1c8e6", size = 490597, upload-time = "2026-06-01T19:40:54.937Z" }, - { url = "https://files.pythonhosted.org/packages/59/1f/299b21441c8de42ff70fddc7cfe65e92f810abcf740739a09b56f7835364/aiohttp-3.14.0-cp314-cp314t-win_amd64.whl", hash = "sha256:2d2ffe9b614f50f069068b3b52e73414e4107fc10b7efc939a76acff9251fdd2", size = 525789, upload-time = "2026-06-01T19:40:57.306Z" }, - { url = "https://files.pythonhosted.org/packages/70/11/7f83fcba9ee05d4c54d61b3f8104da0d43a59adac44dd28effc0c9a10422/aiohttp-3.14.0-cp314-cp314t-win_arm64.whl", hash = "sha256:7a3fc4358e65826c515350f199c210de747cf669998211b1ee6c2e46de364b24", size = 467399, upload-time = "2026-06-01T19:40:59.993Z" }, +sdist = { url = "https://files.pythonhosted.org/packages/82/78/8ea7308cac6934de8c74a14f3d5f65d1c89287426688be79538d0e5c013d/aiohttp-3.14.1.tar.gz", hash = "sha256:307f2cff90a764d329e77040603fa032db89c5c24fdad50c4c15334cba744035", size = 7955794, upload-time = "2026-06-07T21:09:35.529Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/6d/67/58ded4b3f2e10f94972d8928050c85330e249a31dd45a0e5f3c0e9c3fa05/aiohttp-3.14.1-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:8f6bb621e5863cfe8fe5ff5468002d200ec31f30f1280b259dc505b02595099e", size = 766140, upload-time = "2026-06-07T21:05:37.471Z" }, + { url = "https://files.pythonhosted.org/packages/18/68/4ae5b4e08943f316594bb68da89957d3baf5760588fa09509594bd777e4b/aiohttp-3.14.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:4f7215cb3933784f79ed20e5f050e15984f390424339b22375d5a53c933a0491", size = 519430, upload-time = "2026-06-07T21:05:40.751Z" }, + { url = "https://files.pythonhosted.org/packages/cb/c1/316c8f3549dbe5245f92bfd523ec6f32dd4d98cafe21df3f6a19b1184c75/aiohttp-3.14.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:d9d4e294455b23a68c9b8f042d0e8e377a265bcb15332753695f6e5b6819e0ce", size = 514406, upload-time = "2026-06-07T21:05:42.111Z" }, + { url = "https://files.pythonhosted.org/packages/5a/ee/fb0ac28684e8d753b83c8a4eebc19a5846912aa0a4daaabb6a9936363840/aiohttp-3.14.1-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b238af795833d5731d049d82bc84b768ae6f8f97f0495963b3ed9935c5901cc3", size = 1703649, upload-time = "2026-06-07T21:05:43.427Z" }, + { url = "https://files.pythonhosted.org/packages/3b/57/aa2beab673331f111885db8a7b69dfe3ab0e53e446a0ace18ca694b4dc58/aiohttp-3.14.1-cp310-cp310-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:e4e5e0ae56914ecdbf446493addefc0159053dd53962cef37d7839f37f73d505", size = 1675126, upload-time = "2026-06-07T21:05:44.897Z" }, + { url = "https://files.pythonhosted.org/packages/47/ea/dad128abe365e79be03b16ed464198ac73e0d257e8260c6f7d6f31cbef26/aiohttp-3.14.1-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:092e4ce3619a7c6dee52a6bdabda973d9b34b66781f840ce93c7e0cec30cf521", size = 1771558, upload-time = "2026-06-07T21:05:46.405Z" }, + { url = "https://files.pythonhosted.org/packages/63/f3/b5b4e10327cb85d34d24232c6b71b64602f190b3ccb238a043ac6b187dac/aiohttp-3.14.1-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:bb33777ea21e8b7ecde0e6fc84f598be0a1192eab1a63bc746d75aa75d38e7bd", size = 1856631, upload-time = "2026-06-07T21:05:47.844Z" }, + { url = "https://files.pythonhosted.org/packages/2b/9d/93294c3045775c708ac8310eb3d3622a11d2951345ad590d532d62a1faa4/aiohttp-3.14.1-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:23119f8fd4f5d16902ed459b63b100bcd269628075162bddac56cc7b5273b3fb", size = 1714139, upload-time = "2026-06-07T21:05:49.982Z" }, + { url = "https://files.pythonhosted.org/packages/29/c4/93067c85a0373492ce8e577435203c5947c454af074ac48ed4f3a1b9dd4a/aiohttp-3.14.1-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:57fc6745a4b7d0f5a9eb4f40a69718be6c0bc1b8368cc9fe89e90118719f4f42", size = 1588321, upload-time = "2026-06-07T21:05:51.431Z" }, + { url = "https://files.pythonhosted.org/packages/c4/39/9ff91aaf02af8b7b8222a987466da539f154c3e01732c22b5f5a20a8ee66/aiohttp-3.14.1-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:6fd35beba67c4183b09375c5fff9accb47524191a244a99f95fd4472f5402c2b", size = 1670375, upload-time = "2026-06-07T21:05:53.109Z" }, + { url = "https://files.pythonhosted.org/packages/aa/e4/77452a3676b8d99ac1375f77691d6bf65ea6e9f4b201b82ef77c916dc767/aiohttp-3.14.1-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:672b9d65f42eb877f5c3f234a4547e4e1a226ca8c2eed879bb34670a0ce51192", size = 1690933, upload-time = "2026-06-07T21:05:54.902Z" }, + { url = "https://files.pythonhosted.org/packages/7d/84/b0059a7c7fc05ea23f3bc1596ba91c12f79588b9450564a24cac37536d0a/aiohttp-3.14.1-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:24ba13339fed9251d9b1a1bec8c7ab84c0d1675d79d33501e11f94f8b9a84e05", size = 1740798, upload-time = "2026-06-07T21:05:56.458Z" }, + { url = "https://files.pythonhosted.org/packages/8f/3a/e2a513ecbfc362591caa51a7f7e011b3bfc8938b388ae44cd95560d36999/aiohttp-3.14.1-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:94da27378da0610e341c4d30de29a191672683cc82b8f9556e8f7c7212a020fe", size = 1576412, upload-time = "2026-06-07T21:05:57.953Z" }, + { url = "https://files.pythonhosted.org/packages/a1/10/08f1654f538f93d36dcac66310a06eefce4641cdafca83f9f0a5317be254/aiohttp-3.14.1-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:52cdac9432d8b4a719f35094a818d95adcae0f0b4fe9b9b921909e0c87de9e7d", size = 1750199, upload-time = "2026-06-07T21:05:59.488Z" }, + { url = "https://files.pythonhosted.org/packages/99/e4/d91b70c57d8b8e9611e4a2e52238ca3698d3dc1c2efe25b7a9bf594ac584/aiohttp-3.14.1-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:672ac254412a24d0d0cf00a9e6c238877e4be5e5fa2d188832c1244f45f31966", size = 1699356, upload-time = "2026-06-07T21:06:01.131Z" }, + { url = "https://files.pythonhosted.org/packages/3d/f1/15340176f35ff61b95dbe34020bcf43f9e624a2d7bbac934715ff97d2033/aiohttp-3.14.1-cp310-cp310-win32.whl", hash = "sha256:2fe3607e71acc6ebb0ec8e492a247bf7a291226192dc0084236dfc12478916f6", size = 458939, upload-time = "2026-06-07T21:06:02.86Z" }, + { url = "https://files.pythonhosted.org/packages/c3/c2/a2f1ec5b37f903109e43ae2862268cfe4a67a60c1b2cf43169fcdff5995f/aiohttp-3.14.1-cp310-cp310-win_amd64.whl", hash = "sha256:30099eda75a53c32efb0920e9c33c195314d2cc1c680fbfd30894932ac5f27df", size = 482583, upload-time = "2026-06-07T21:06:04.666Z" }, + { url = "https://files.pythonhosted.org/packages/d0/7a/7b56f6732ef79530afaa72aa335d41b67c8d79b946995f0b11ad72985435/aiohttp-3.14.1-cp310-cp310-win_arm64.whl", hash = "sha256:5a837f49d901f9e368651b676912bff1104ed8c1a83b280bcd7b29adccef5c9c", size = 453470, upload-time = "2026-06-07T21:06:06.322Z" }, + { url = "https://files.pythonhosted.org/packages/26/dd/bf526e6f0a1120dd6f2df2e97bacfe4d358f13d17a0ff5847301a1375a51/aiohttp-3.14.1-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:aa00140699487bd435fde4342d85c94cb256b7cd3a5b9c3396c67f19922afda2", size = 765225, upload-time = "2026-06-07T21:06:07.957Z" }, + { url = "https://files.pythonhosted.org/packages/8f/e1/a2872aa55495a70f61310d411541c6ee23812d9a884e000c716e1bc3edbf/aiohttp-3.14.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:1c1af67559445498b502030c35c59db59966f47041ca9de5b4e707f86bd10b5f", size = 518743, upload-time = "2026-06-07T21:06:09.749Z" }, + { url = "https://files.pythonhosted.org/packages/5b/e7/c60c7b209e509cc787de3cea0550a518538cfc08003e1c1e14c1c63fff71/aiohttp-3.14.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:d44ec478e713ee7f29b439f7eb8dc2b9d4079e11ae114d2c2ac3d5daf30516c8", size = 514139, upload-time = "2026-06-07T21:06:11.26Z" }, + { url = "https://files.pythonhosted.org/packages/5b/8d/614ace2f579702c9840ab1e1447fd8509e35b0b904f7196418fa2f57b25d/aiohttp-3.14.1-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d3b1a184a9a8f548a6b73f1e26b96b052193e4b3175ed7342aaf1151a1f00a04", size = 1784088, upload-time = "2026-06-07T21:06:12.887Z" }, + { url = "https://files.pythonhosted.org/packages/49/e0/726e90f99542bf292f81a96a12cc4847deb86f3ccf62c6f4014a201f4d33/aiohttp-3.14.1-cp311-cp311-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:5f2504bc0322437c9a1ff6d3333ca56c7477b727c995f036b976ae17b98372c8", size = 1737835, upload-time = "2026-06-07T21:06:14.564Z" }, + { url = "https://files.pythonhosted.org/packages/0b/4b/d176d5c4db9d33dacf0543102ea59503bc1d528af4cfd0b719949ca49389/aiohttp-3.14.1-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:73f05ea02013e02512c3bf42714f1208c57168c779cc6fe23516e4543089d0a6", size = 1842801, upload-time = "2026-06-07T21:06:16.228Z" }, + { url = "https://files.pythonhosted.org/packages/dc/d6/5a99b563690ea0cbed912ae94a2ce33993a5709a651a3a4fe761e7dd973a/aiohttp-3.14.1-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:797457503c2d426bee06eef808d07b31ede30b65e054444e7de64cad0061b7af", size = 1929992, upload-time = "2026-06-07T21:06:17.947Z" }, + { url = "https://files.pythonhosted.org/packages/76/7f/a987b14a3859094b3cea3f4825219c3e5536242564af6e3f9c2f6c994eb2/aiohttp-3.14.1-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b821a1f7dedf7e37450654e620038ac3b2e81e8fa6ea269337e97101978ec730", size = 1786989, upload-time = "2026-06-07T21:06:19.677Z" }, + { url = "https://files.pythonhosted.org/packages/f1/1a/420e5c85a3e73349372ed22ce0b6af86bfa6ce16a4b20a64a2e94608c781/aiohttp-3.14.1-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:4cd96b5ba05d67ed0cf00b5b405c8cd99586d8e3481e8ee0a831057591af7621", size = 1640129, upload-time = "2026-06-07T21:06:22.558Z" }, + { url = "https://files.pythonhosted.org/packages/a7/80/18a592ed3be0a402cc03670bd72ee1f8563ddbe1d8d5542dbf868f274136/aiohttp-3.14.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:1d459b98a932296c6f0e94f87511a0b1b90a8a02c30a50e60a297619cd5a58ee", size = 1756576, upload-time = "2026-06-07T21:06:24.8Z" }, + { url = "https://files.pythonhosted.org/packages/ec/0b/8b3d5713373858ff71a617daf6e3b0e81ad63e79d09a3cf2f6b6b983939c/aiohttp-3.14.1-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:764457a7be60825fb770a644852ff717bcbb5042f189f2bd16df61a81b3f6573", size = 1754668, upload-time = "2026-06-07T21:06:26.528Z" }, + { url = "https://files.pythonhosted.org/packages/9f/49/fd564575cf225821d7ba5a117cb8bc27213d8a7e1811162afb43ae077039/aiohttp-3.14.1-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:f7a16ef45b081454ef844502d87a848876c490c4cb5c650c230f6ec79ed2c1e7", size = 1817019, upload-time = "2026-06-07T21:06:28.297Z" }, + { url = "https://files.pythonhosted.org/packages/ed/1b/e850c9ae6fc91356552ae668bb6c51e93fa29c8aef13398a10b56678557f/aiohttp-3.14.1-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:2fbc3ed048b3475b9f0cbcb9978e9d2d3511acd91ead203af26ed9f0056004cf", size = 1631638, upload-time = "2026-06-07T21:06:30.242Z" }, + { url = "https://files.pythonhosted.org/packages/eb/94/3c337ba72451a89806ace6f75bddc92bafc5b8d53d90115a512858024b63/aiohttp-3.14.1-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:bedb0cd073cc2dc035e30aeb99444389d3cd2113afe4ef9fcd23d439f5bade85", size = 1835660, upload-time = "2026-06-07T21:06:31.943Z" }, + { url = "https://files.pythonhosted.org/packages/2b/9c/9c18cf367a0498212d9ba7daf990b504a5e8ae064cda4b504e2647c89c03/aiohttp-3.14.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:b6feea921016eb3d4e04d65fc4e9ca402d1a3801f562aef94989f54694917af3", size = 1775698, upload-time = "2026-06-07T21:06:33.72Z" }, + { url = "https://files.pythonhosted.org/packages/b5/63/a251a9d2a6cb45065b2ddc0bde2b3dd10108740a9a42f632c66405a761a2/aiohttp-3.14.1-cp311-cp311-win32.whl", hash = "sha256:313701e488100074ce99850404ee36e741abf6330179fec908a1944ecf570126", size = 458386, upload-time = "2026-06-07T21:06:35.279Z" }, + { url = "https://files.pythonhosted.org/packages/17/ca/69274c51dcd6e8947d77b2806cf47a4a15f2c846e2cbeb1882547d3da283/aiohttp-3.14.1-cp311-cp311-win_amd64.whl", hash = "sha256:03ab4530fdcb3a543a122ba4b65ac9919da9fe9f78a03d328a6e38ff962f7aa5", size = 483406, upload-time = "2026-06-07T21:06:36.824Z" }, + { url = "https://files.pythonhosted.org/packages/2c/8a/c25904f77690c3688ec140f87591ef11a0cfe36bf3d5c0f1f38056fb62b3/aiohttp-3.14.1-cp311-cp311-win_arm64.whl", hash = "sha256:486f7d16ed54c39c2cbd7ca71fd8ba2b8bb7860df65bd7b6ed640bab96a38a8b", size = 452987, upload-time = "2026-06-07T21:06:38.371Z" }, + { url = "https://files.pythonhosted.org/packages/1d/21/151624b51cd92553d95424daf4bf19f19ce9be9002d19253e7e7ce67197b/aiohttp-3.14.1-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:d35143e27778b4bb0fb189562d7f275bff79c62ab8e98459717c0ea617ff2480", size = 757402, upload-time = "2026-06-07T21:06:40.311Z" }, + { url = "https://files.pythonhosted.org/packages/c2/82/280619e0bd7bf2454987e19282616e84762255dd9c8468f62382e8c191f1/aiohttp-3.14.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:bcfb80a2cc36fba2534e5e5b5264dc7ae6fcd9bf15256da3e53d2f499e6fa29d", size = 512310, upload-time = "2026-06-07T21:06:42.207Z" }, + { url = "https://files.pythonhosted.org/packages/55/b2/2aac325583aaa1353045f96dffa586d8a34e8322e14a7ba49cffeb103ab4/aiohttp-3.14.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:27fd7c91e51729b4f7e1577865fa6d34c9adccbc39aabe9000285b48af9f0ec2", size = 512448, upload-time = "2026-06-07T21:06:43.813Z" }, + { url = "https://files.pythonhosted.org/packages/8a/72/a60607cb849faa8af8a356c9329ea2eb6f395d49e82cc82ccba1fd8deb8f/aiohttp-3.14.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:64c567bf9eaf664280116a8688f63016e6b32db2505908e2bdaca1b6438142f2", size = 1766854, upload-time = "2026-06-07T21:06:45.391Z" }, + { url = "https://files.pythonhosted.org/packages/b5/d3/d9fe1c9ec7557ab4d0d82bebaa728c6418f0b93295ec2f4ab015f7710cc7/aiohttp-3.14.1-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:f5e6ff2bdbb8f4cd3fbe41f99e25bbcd58e3bf9f13d3dd31a11e7917251cc77a", size = 1740884, upload-time = "2026-06-07T21:06:47.413Z" }, + { url = "https://files.pythonhosted.org/packages/c1/dc/f2cecfaf9337ba3e63f181500814ff502aa3d00d9c7ec93a9d23d10a27b2/aiohttp-3.14.1-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:2f73e01dc37122325caf079982621262f96d74823c179038a82fddfc50359264", size = 1810034, upload-time = "2026-06-07T21:06:50.165Z" }, + { url = "https://files.pythonhosted.org/packages/66/d7/2ff65c5e65c0d7476daf7e15c032e0805e36811185b9623e3238ad6c763e/aiohttp-3.14.1-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:bb2c0c80d431c0d03f2c7dbf125150fedd4f0de17366a7ca33f7ccb822391842", size = 1904054, upload-time = "2026-06-07T21:06:52.035Z" }, + { url = "https://files.pythonhosted.org/packages/20/9c/d445818389df371f56d141d881153ba23183c4735a03f7356ffb43f7757d/aiohttp-3.14.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3e6fc1a85fa7194a1a7d19f44e8609180f4a8eb5fa4c7ed8b4355f080fad235c", size = 1790278, upload-time = "2026-06-07T21:06:54.049Z" }, + { url = "https://files.pythonhosted.org/packages/4d/aa/bf04cb4d865fc6101c2229a294ad744973b72e513fdc5a6b791e6983d72a/aiohttp-3.14.1-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:686b6c0d3911ec387b444ddf5dc62fb7f7c0a7d5186a7861626496a5ab4aff95", size = 1591795, upload-time = "2026-06-07T21:06:55.911Z" }, + { url = "https://files.pythonhosted.org/packages/dc/b4/4dac0038960427ba832f6609dfb4ea5437d7fd80c72001b9e48f834f428b/aiohttp-3.14.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:c6fa4dc7ad6f8109c70bb1499e589f76b0b792baf39f9b017eb92c8a81d0a199", size = 1728397, upload-time = "2026-06-07T21:06:57.777Z" }, + { url = "https://files.pythonhosted.org/packages/2b/f9/7cd4e8ad7aa3b75f17d56bb5498dd604a93d4e6eece822ba0568c413fff0/aiohttp-3.14.1-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:87a5eea1b2a5e21e1ebdbb33ad4165359189327e63fc4e4894693e7f821ac817", size = 1766504, upload-time = "2026-06-07T21:07:00.009Z" }, + { url = "https://files.pythonhosted.org/packages/f9/df/fc01d9fcad0f73fed3f3d361f1f94f975947b50dff82919f6dc2bf4316cc/aiohttp-3.14.1-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:1c1421eb01d4fd608d88cc8290211d177a58532b55ad94076fb349c5bf467f0a", size = 1777806, upload-time = "2026-06-07T21:07:02.064Z" }, + { url = "https://files.pythonhosted.org/packages/41/09/47e2d090bddcc8fb4ccb4c314aadc32d7c5d9bb55f50f6ad1c92fc15d501/aiohttp-3.14.1-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:34b257ec41345c1e8f2df68fa908a7952f5de932723871eb633ecbbff396c9a4", size = 1580707, upload-time = "2026-06-07T21:07:03.942Z" }, + { url = "https://files.pythonhosted.org/packages/3d/36/f1a4ce904ae0b6930cfe9afc96d0896f7ec1a620c400405d63783bb95a9c/aiohttp-3.14.1-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:de538791a80e5d862addbc183f70f0158ac9b9bb872bb147f1fd2a683691e087", size = 1798121, upload-time = "2026-06-07T21:07:05.987Z" }, + { url = "https://files.pythonhosted.org/packages/70/0a/e0075ce9ca0279ee1d4f0c0b85f54fea02ebc83c3007651a72bece658fec/aiohttp-3.14.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:6f71173be42d3241d428f760122febb748de0623f44308a6f120d0dd9ec572e3", size = 1767580, upload-time = "2026-06-07T21:07:07.873Z" }, + { url = "https://files.pythonhosted.org/packages/3e/61/a0c0a8f327a9c52095cdd8e312391b00d3ed64ab6c72bb5c33d8ec251cf7/aiohttp-3.14.1-cp312-cp312-win32.whl", hash = "sha256:ec8dc383ee57ea3e883477dcca3f11b65d58199f1080acaf4cd6ad9a99698be4", size = 452771, upload-time = "2026-06-07T21:07:09.669Z" }, + { url = "https://files.pythonhosted.org/packages/df/d9/ea367c75f16ac9c6cdc8febb25e8318fa21a2b1bc8d6514d4b2d890bface/aiohttp-3.14.1-cp312-cp312-win_amd64.whl", hash = "sha256:2aa92c87868cd13674989f9ee83e5f9f7ea4237589b728048e1f0c8f6caa3271", size = 479873, upload-time = "2026-06-07T21:07:11.538Z" }, + { url = "https://files.pythonhosted.org/packages/03/64/8d96784a7851156db8a4c6c3f6f91042fdf39fb15a4cc38c8b3c14833c45/aiohttp-3.14.1-cp312-cp312-win_arm64.whl", hash = "sha256:2c840c90759922cb5e6dda94596e079a30fb5a5ba548e7e0dc00574703940847", size = 448073, upload-time = "2026-06-07T21:07:13.637Z" }, + { url = "https://files.pythonhosted.org/packages/bc/97/bd137012dd97e1649162b099135a80e1fd59aaa807b2430fc448d1029aff/aiohttp-3.14.1-cp313-cp313-android_21_arm64_v8a.whl", hash = "sha256:b3a03285a7f9c7b016324574a6d92a1c895da6b978cb8f1deee3ac72bc6da178", size = 506882, upload-time = "2026-06-07T21:07:15.501Z" }, + { url = "https://files.pythonhosted.org/packages/ef/79/e5cc690e9d922a66887ceeaca53a8ffd5a7b0be3816142b7abc433742d89/aiohttp-3.14.1-cp313-cp313-android_21_x86_64.whl", hash = "sha256:2a73f487ab8ef5abbb24b7aa9b73e98eaba9e9e031804ff2416f02eca315ccaf", size = 515270, upload-time = "2026-06-07T21:07:17.53Z" }, + { url = "https://files.pythonhosted.org/packages/fe/22/a73ccbf9dbd6e26dda0b24d5fd5db7da92ee3383a79f47677ffb834c5c5b/aiohttp-3.14.1-cp313-cp313-ios_13_0_arm64_iphoneos.whl", hash = "sha256:915fbb7b41b115192259f8c9ae58f3ddc444d2b5579917270211858e606a4afd", size = 485841, upload-time = "2026-06-07T21:07:19.555Z" }, + { url = "https://files.pythonhosted.org/packages/3b/b9/57ed8eaf596321c2ad747bd480fb1700dbd7177c60dfc9e4c187f629662e/aiohttp-3.14.1-cp313-cp313-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:7fb4bdf95b0561a79f259f9d28fbc109728c5ee7f27aff6391f0ca703a329abe", size = 492088, upload-time = "2026-06-07T21:07:21.581Z" }, + { url = "https://files.pythonhosted.org/packages/78/c0/5ebe5270a7c140d7c6f79dcb018640225f14d406c149e4eec04a7d82fe71/aiohttp-3.14.1-cp313-cp313-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:1b9748363260121d2927704f5d4fc498150669ca3ae93625986ee89c8f80dcd4", size = 501564, upload-time = "2026-06-07T21:07:23.388Z" }, + { url = "https://files.pythonhosted.org/packages/75/7f/8cdaa24fc7983865e0915153b96a9ac5bcdd3548d64c5a27d17cecccad2d/aiohttp-3.14.1-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:86a6dab78b0e43e2897a3bbe15745aa60dc5423ca437b7b0b164c069bf91b876", size = 751998, upload-time = "2026-06-07T21:07:25.046Z" }, + { url = "https://files.pythonhosted.org/packages/b2/f4/c4227aacfacc5cb0cc2d119b65301d177912a6842cd64e120c47af76064f/aiohttp-3.14.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:4dfd6e47d3c44c2279907607f73a4240b88c69eb8b90da7e2441a8045dfd21da", size = 510918, upload-time = "2026-06-07T21:07:27.28Z" }, + { url = "https://files.pythonhosted.org/packages/ab/01/a2d5f96cd4e74424864d30bc0a7e44d0a12dacdcfa91b5b2d1bd3dca6bf3/aiohttp-3.14.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:317acd9f8602858dc7d59679812c376c7f0b97bcbbf16e0d6237f54141d8a8a6", size = 508657, upload-time = "2026-06-07T21:07:29.252Z" }, + { url = "https://files.pythonhosted.org/packages/e8/ed/3c0fb5c500fdd8e7ebc10d1889c04384fffa1a9163eac1356088ca9da1b1/aiohttp-3.14.1-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:bd869c427324e5cb15195793de951295710db28be7d818247f3097b4ab5d4b96", size = 1757907, upload-time = "2026-06-07T21:07:31.03Z" }, + { url = "https://files.pythonhosted.org/packages/0b/ab/d4c924d9bd5be3050c226612413ce68cb54c70d2c31b661bfc8d9a5b6a70/aiohttp-3.14.1-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:93b032b5ec3255473c143627d21a69ac74ae12f7f33974cb587c564d11b1066f", size = 1737565, upload-time = "2026-06-07T21:07:33.031Z" }, + { url = "https://files.pythonhosted.org/packages/19/2a/37326821ff779084020cdc33224d20b19f42f4183a500ff92022a739eda7/aiohttp-3.14.1-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:f234b4deb12f3ad59127e037bc57c40c21e45b45282df7d3a55a0f409f595296", size = 1799018, upload-time = "2026-06-07T21:07:35.003Z" }, + { url = "https://files.pythonhosted.org/packages/b3/4f/6e947ba73e4ce09070761c05ed3a8ceb7c21f5e46798671d8b2aac0e4626/aiohttp-3.14.1-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:9af6779bfb46abf124068327abcdf9ce95c9ef8287a3e8da76ccf2d0f16c28fa", size = 1894416, upload-time = "2026-06-07T21:07:36.956Z" }, + { url = "https://files.pythonhosted.org/packages/9d/6e/dbf1d0625dc711fb2851f4f3c3055c39ed58bae92082d8c627dbe6013736/aiohttp-3.14.1-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:faccab372e66bc76d5731525e7f1143c922271725b9d38c9f97edcc66266b451", size = 1783881, upload-time = "2026-06-07T21:07:39.063Z" }, + { url = "https://files.pythonhosted.org/packages/44/c2/5e25098a67268ed369483ae7d1a58bd0a13d03aab860d2a0e4a6eb25b046/aiohttp-3.14.1-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:f380468b09d2a81633ee863b0ec5648d364bd17bb8ecfb8c2f387f7ac1faf42c", size = 1587572, upload-time = "2026-06-07T21:07:41.058Z" }, + { url = "https://files.pythonhosted.org/packages/2a/bd/cf9cee17e140f942a3de73e658a543aa8fbf35a5fc67a9d2538d52d77f0b/aiohttp-3.14.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:97e704dcd26271f5bda3fa07c3ce0fb76d6d3f8659f4baa1a24442cc9ba177ca", size = 1722137, upload-time = "2026-06-07T21:07:43.014Z" }, + { url = "https://files.pythonhosted.org/packages/89/6d/5684f8c59045c96f81a18cefbc1fbbd79d25b88f1c622f2a5c5c08fcb632/aiohttp-3.14.1-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:269b76ac5394092b95bc4a098f4fc6c191c083c3bd12775d1e30e663132f6a09", size = 1755953, upload-time = "2026-06-07T21:07:45.933Z" }, + { url = "https://files.pythonhosted.org/packages/a8/40/35caf3170f8359760740a7d9aa0fff2e344bef98e1d1186f5a0f6dec17e6/aiohttp-3.14.1-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:5c0b3e614340c889d575451696374c9d17affd54cd607ca0babed8f8c37b9397", size = 1766479, upload-time = "2026-06-07T21:07:48.047Z" }, + { url = "https://files.pythonhosted.org/packages/6d/a1/b0c61e7a137f0d81de49a82023a6df73c3c16d6fefb0f8e4a93d21639002/aiohttp-3.14.1-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:5663ee9257cfa1add7253a7da3035a02f31b6600ec48261585e1800a81533080", size = 1580077, upload-time = "2026-06-07T21:07:50.069Z" }, + { url = "https://files.pythonhosted.org/packages/0b/41/194ea4623693009fcefebef7aef63c141754f153e9cd0d39d3b9e36c175c/aiohttp-3.14.1-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:603a2c834142172ffddc054067f5ec0ca65d57a0aa98a71bc81952573208e345", size = 1791688, upload-time = "2026-06-07T21:07:52.106Z" }, + { url = "https://files.pythonhosted.org/packages/ba/45/4de841f005cfe1fd63e2a2fe011262c515e2a62aa6994b15947e7d717ac9/aiohttp-3.14.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:cb21957bb8aca671c1765e32f58164cf0c50e6bf41c0bbbd16da20732ecaf588", size = 1761094, upload-time = "2026-06-07T21:07:54.113Z" }, + { url = "https://files.pythonhosted.org/packages/e4/ae/dbce10533d3896d544d5053939ed75b7dc31a1b0973d959b1b5ae21028d6/aiohttp-3.14.1-cp313-cp313-win32.whl", hash = "sha256:e509a55f681e6158c20f70f102f9cf61fb20fbc382272bc6d94b7343f2582780", size = 452662, upload-time = "2026-06-07T21:07:56.06Z" }, + { url = "https://files.pythonhosted.org/packages/7b/d9/0bf1a19362c32f06229da5e7ddfcec91f93474d6307f7a2d3135e9c674dc/aiohttp-3.14.1-cp313-cp313-win_amd64.whl", hash = "sha256:1ac8531b638959718e18c2207fbfe297819875da46a740b29dfa29beba64355a", size = 479748, upload-time = "2026-06-07T21:07:58.319Z" }, + { url = "https://files.pythonhosted.org/packages/22/0a/62e7232dc9484fbec112ceb32efb6a624cc7994ec6e2b019286f17c4e8f2/aiohttp-3.14.1-cp313-cp313-win_arm64.whl", hash = "sha256:250d14af67f6b6a1a4a811049b1afa69d61d617fca6bf33149b3ab1a6dbcf7b8", size = 447723, upload-time = "2026-06-07T21:08:00.154Z" }, + { url = "https://files.pythonhosted.org/packages/c4/a1/5fafa04e1ca91ddb47608699d60649c1c6db3cf41c99e78fc4056f9513db/aiohttp-3.14.1-cp314-cp314-android_24_arm64_v8a.whl", hash = "sha256:7c106c26852ca1c2047c6b80384f17100b4e439af276f21ef3d4e2f450ae7e15", size = 508531, upload-time = "2026-06-07T21:08:02.093Z" }, + { url = "https://files.pythonhosted.org/packages/fa/2e/bfa02f699d87ffc86d5959270b28f1cb410add3ccaced8ed2e0b8a5238fc/aiohttp-3.14.1-cp314-cp314-android_24_x86_64.whl", hash = "sha256:20205f7f5ade7aaec9f4b500549bbc071b046453aed72f9c06dcab87896a83e8", size = 514718, upload-time = "2026-06-07T21:08:04.476Z" }, + { url = "https://files.pythonhosted.org/packages/85/a5/9594ad6289eebbc97d167c44213d557807f90e59115caad24de21ad2c3b1/aiohttp-3.14.1-cp314-cp314-ios_13_0_arm64_iphoneos.whl", hash = "sha256:62a759436b29e677181a9e76bab8b8f689a29cb9c535f45f7c48c9c830d3f8c3", size = 487918, upload-time = "2026-06-07T21:08:06.377Z" }, + { url = "https://files.pythonhosted.org/packages/b4/61/16a32c36c3c49edec122a3dc811f2057df2f94d3b14aa107c8017d981618/aiohttp-3.14.1-cp314-cp314-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:2964cbf553df4d7a57348da44d961d871895fc1ee4e8c322b2a95612c7b17fba", size = 494014, upload-time = "2026-06-07T21:08:08.263Z" }, + { url = "https://files.pythonhosted.org/packages/9b/89/3ebcf96ed99c05bec9c434aaac6963fd3cbab4a786ae739908a144d9ce44/aiohttp-3.14.1-cp314-cp314-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:237651caadc3a59badd39319c54642b5299e9cc98a3a194310e55d5bb9f5e397", size = 502398, upload-time = "2026-06-07T21:08:10.244Z" }, + { url = "https://files.pythonhosted.org/packages/fd/3d/b74870a0c2d40c355928cd5b96c7a11fa821b8a40fc41365e64479b151fb/aiohttp-3.14.1-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:896e12dfdbbab9d8f7e16d2b28c6769a60126fa92095d1ebf9473d02593a2448", size = 758018, upload-time = "2026-06-07T21:08:12.447Z" }, + { url = "https://files.pythonhosted.org/packages/d3/66/f42f5c984d99e49c6cff5f26f590750f2e2f7ef1fcfb99966ab5be1b632e/aiohttp-3.14.1-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:d03f281ed22579314ba00821ce20115a7c0ac430660b4cc05704a3f818b3e004", size = 512462, upload-time = "2026-06-07T21:08:14.624Z" }, + { url = "https://files.pythonhosted.org/packages/e9/a7/248e1aebe0c7810b0271e021a0f2a5eb6e78a051885b3c9df49f42a5802d/aiohttp-3.14.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:07eabb979d236335fed927e137a928c9adfb7df3b9ec7aa31726f133a62be983", size = 512824, upload-time = "2026-06-07T21:08:16.572Z" }, + { url = "https://files.pythonhosted.org/packages/26/97/2aa0e5ba0727dc3bd5aaebb7ccbc510f7dfb7fb961ec87497cd496635ab1/aiohttp-3.14.1-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:4fe1f1087cbadb280b5e1bb054a4f00d1423c74d6626c5e48400d871d34ecefe", size = 1749898, upload-time = "2026-06-07T21:08:18.635Z" }, + { url = "https://files.pythonhosted.org/packages/00/8d/e97f6c96c891d457c8479d92a514ba194d0412f981d72c70341ee18488ed/aiohttp-3.14.1-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:367a9314fdc79dab0fac96e216cb41dd73c85bdca85306ce8999118ba7e0f333", size = 1710114, upload-time = "2026-06-07T21:08:20.892Z" }, + { url = "https://files.pythonhosted.org/packages/6f/e6/aa8d7e863048c8fceb5cd6ce74017311cec3ead07847387e12265fb4444e/aiohttp-3.14.1-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:a24f677ebe83749039e7bdf862ff0bbb16818ae4193d4ef96505e269375bcce0", size = 1802541, upload-time = "2026-06-07T21:08:23.044Z" }, + { url = "https://files.pythonhosted.org/packages/83/a8/72193137de57fda4ebfae4563182d082c8856e3b6e9871d0b46f028fb369/aiohttp-3.14.1-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:c83afe0ba876be7e943d2e0ba645809ad441575d2840c895c21ee5de93b9377a", size = 1875776, upload-time = "2026-06-07T21:08:25.288Z" }, + { url = "https://files.pythonhosted.org/packages/a0/18/938441025db6769a3464596b2410af3afde0b21eb2f204c6f766f68af4bd/aiohttp-3.14.1-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:634e385930fb6d2d479cf3aa66515955863b77a5e3c2b5894ca259a25b308602", size = 1760329, upload-time = "2026-06-07T21:08:27.363Z" }, + { url = "https://files.pythonhosted.org/packages/60/29/bf2496b4065e76e09fe48015aaffe5ce161d8f089b06ac6982070f653076/aiohttp-3.14.1-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:eeea07c4397bbc57719c4eed8f9c284874d4f175f9b6d57f7a1546b976d455ca", size = 1587293, upload-time = "2026-06-07T21:08:29.805Z" }, + { url = "https://files.pythonhosted.org/packages/49/a2/2136674d52123b1354bd05dd5753c318db47dc0c927cc70b27bab3755456/aiohttp-3.14.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:335c0cc3e3545ce98dcb9cfcb836f40c3411f43fa03dab757597d80c89af8a35", size = 1714756, upload-time = "2026-06-07T21:08:32.094Z" }, + { url = "https://files.pythonhosted.org/packages/a7/b9/e5fd2e6f915503081c0f9b1e8540947037929c70c191da2e4d54b31a21a1/aiohttp-3.14.1-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:ae6be797afdef264e8a84864a85b196ca06045586481b3df8a967322fd2fa844", size = 1721052, upload-time = "2026-06-07T21:08:34.167Z" }, + { url = "https://files.pythonhosted.org/packages/63/5a/2833e324a2263e104e31e2e91bc5bbee81bc499afd32203faee048a883f0/aiohttp-3.14.1-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:8560b4d712474335d08907db7973f71912d3a9a8f1dee992ec06b5d2fe359496", size = 1766888, upload-time = "2026-06-07T21:08:36.95Z" }, + { url = "https://files.pythonhosted.org/packages/57/fa/dea6511870913162f3b2e8c42a7614eb203a4540b8c2da43e0bfb0548f3c/aiohttp-3.14.1-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:2b7edd08e0a5deb1e8564a2fcd8f4561014a3f05252334671bbf55ddd47db0e5", size = 1581679, upload-time = "2026-06-07T21:08:39.292Z" }, + { url = "https://files.pythonhosted.org/packages/14/bd/3cf0d55e71784b33534e9710a67d382d900598b4787fbce6cc7317f8c42a/aiohttp-3.14.1-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:b6ff7fcee63287ae57b5df3e4f5957ce032122802509246dec1a5bcc55904c95", size = 1782021, upload-time = "2026-06-07T21:08:41.407Z" }, + { url = "https://files.pythonhosted.org/packages/c1/af/14bb5843eccbe234f4dfb78ab73e549d99727247e62ae5d62cbd22eaf5b0/aiohttp-3.14.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:6ffbb2f4ec1ceaff7e07d43922954da26b223d188bf30658e561b98e23089444", size = 1742574, upload-time = "2026-06-07T21:08:43.795Z" }, + { url = "https://files.pythonhosted.org/packages/f2/1e/fbeb7af9210a67ac0f9c9bec0f8f4568497924e33137a3d5b48e1cf85f3f/aiohttp-3.14.1-cp314-cp314-win32.whl", hash = "sha256:a9875b46d910cff3ea2f5962f9d266b465459fe634e22556ab9bd6fc1192eea0", size = 457773, upload-time = "2026-06-07T21:08:46.168Z" }, + { url = "https://files.pythonhosted.org/packages/f0/2b/13e8d741a9ec5db7d900c060554cf8352ab85e44e2a4469ebb9d377bda17/aiohttp-3.14.1-cp314-cp314-win_amd64.whl", hash = "sha256:af8b4b81a960eeaf1234971ac3cd0ba5901f3cd42eae42a46b4d089a8b492719", size = 485001, upload-time = "2026-06-07T21:08:48.401Z" }, + { url = "https://files.pythonhosted.org/packages/df/30/491acfa2c4d6c3ff59c49a14fc1b50be3241e25bbb0c84c09e2da4d11395/aiohttp-3.14.1-cp314-cp314-win_arm64.whl", hash = "sha256:cf4491381b1b57425c315a56a439251b1bdac07b2275f19a8c44bc57744532ec", size = 453809, upload-time = "2026-06-07T21:08:50.7Z" }, + { url = "https://files.pythonhosted.org/packages/34/e3/19dbe1a1f4cc6230eb9e314de7fe68053b0992f9302b27d12141a0b5db53/aiohttp-3.14.1-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:819c054312f1af92947e6a55883d1b66feefab11531a7fc45e0fb9b63880b5c2", size = 793320, upload-time = "2026-06-07T21:08:52.775Z" }, + { url = "https://files.pythonhosted.org/packages/7f/20/1b7182219ba1b108430d6e4dc53d25ae02dcfcf5a045b33af4e8c5167527/aiohttp-3.14.1-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:10ee9c1753a8f706345b22496c79fbddb5be0599e0823f3738b1534058e25340", size = 529077, upload-time = "2026-06-07T21:08:55Z" }, + { url = "https://files.pythonhosted.org/packages/b9/c8/14ce60ec31a2e5f5274bb17d383a6f7a3aabca31ac04eee05585bbadab16/aiohttp-3.14.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:1601cc37baf5750ccacae618ec2daf020769581695550e3b654a911f859c563d", size = 532476, upload-time = "2026-06-07T21:08:57.176Z" }, + { url = "https://files.pythonhosted.org/packages/7e/02/9ac85e081e53da2e061b02fa7758fe0a12d17b8ce2d1f5e6c7cb76730328/aiohttp-3.14.1-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:4d6e0ac9da31c9c04c84e1c0182ad8d6df35965a85cae29cd71d089621b3ae94", size = 1922347, upload-time = "2026-06-07T21:08:59.563Z" }, + { url = "https://files.pythonhosted.org/packages/c0/3e/d3ba07a0ab38b5389e10bec4362d21e10a4f667cba2d79ba30837b3a5059/aiohttp-3.14.1-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:9e8f2d660c350b3d0e259c7a7e3d9b7fc8b41210cbcc3d4a7076ff0a5e5c2fdc", size = 1786465, upload-time = "2026-06-07T21:09:01.909Z" }, + { url = "https://files.pythonhosted.org/packages/0b/cb/e2ee978a00cfb2df829704a69528b18154eba5939f45bc1efa8f33aee4c5/aiohttp-3.14.1-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:4691802dda97be727f79d86818acaad7eb8e9252626a1d6b519fedbb92d5e251", size = 1909423, upload-time = "2026-06-07T21:09:04.357Z" }, + { url = "https://files.pythonhosted.org/packages/73/5d/1430334858b1022b58ae50399a918f0bd6fe8fa7fa183598d657ff61e040/aiohttp-3.14.1-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:c389c482a7e9b9dc3ee2701ac46c4125297a3818875b9c305ddb603c04828fd1", size = 2001906, upload-time = "2026-06-07T21:09:06.722Z" }, + { url = "https://files.pythonhosted.org/packages/66/4e/560c7472d3d198a23aa5c8b19a5115bf6a9b77b7d3e4bb363da320430ad2/aiohttp-3.14.1-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:fc0cacab7ba4e56f0f81c82a98c09bed2f39c940107b03a34b168bdf7597edd3", size = 1877095, upload-time = "2026-06-07T21:09:09.011Z" }, + { url = "https://files.pythonhosted.org/packages/0d/f1/4745806578d447db4a784a8591e2dae3afdfc2bcb96f8f81271b13df6543/aiohttp-3.14.1-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:979ed4717f59b8bb12e3963378fa285d93d367e15bcd66c721311826d3c44a6c", size = 1676222, upload-time = "2026-06-07T21:09:11.461Z" }, + { url = "https://files.pythonhosted.org/packages/6a/c9/48255813cca749a229ef0ab476004ec623728ad79a9c0840616f6c076325/aiohttp-3.14.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:38e1e7daaea81df51c952e18483f323d878499a1e2bfe564790e0f9701d6f203", size = 1842922, upload-time = "2026-06-07T21:09:14.118Z" }, + { url = "https://files.pythonhosted.org/packages/3d/c0/bbd054e2bee909f529523a5af3891052606af5143c09f5f183ec3b234676/aiohttp-3.14.1-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:4132e72c608fe9fecb8f409113567605915b83e9bdd3ea56538d2f9cd35002f1", size = 1825035, upload-time = "2026-06-07T21:09:16.447Z" }, + { url = "https://files.pythonhosted.org/packages/a8/ae/90395d4376deceb74e09ec26b6adf7d2015a6f8802d6d84446af860fef04/aiohttp-3.14.1-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:eefd9cc9b6d4a2db5f00a26bc3e4f9acf71926a6ec557cd56c9c6f27c290b665", size = 1849512, upload-time = "2026-06-07T21:09:18.742Z" }, + { url = "https://files.pythonhosted.org/packages/93/bd/fb25f3049957553d4ce0ba6ae480aa2f592a6985497fca590837d16c1be0/aiohttp-3.14.1-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:b165790117eea512d7f3fb22f1f6dad3d55a7189571993eb015591c1401276d1", size = 1668571, upload-time = "2026-06-07T21:09:21.458Z" }, + { url = "https://files.pythonhosted.org/packages/3f/22/7f73303d64dd567ff3addca90b556690ed1233a47b8f55d242fb90af3681/aiohttp-3.14.1-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:ed09c7eb1c391271c2ed0314a51903e72a3acb653d5ccfc264cdf3ef11f8269d", size = 1881159, upload-time = "2026-06-07T21:09:23.813Z" }, + { url = "https://files.pythonhosted.org/packages/44/be/0474c5a8b5640e1e4aa1923430a91f4151be82e511373fe764189b89aef5/aiohttp-3.14.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:99abd37084b82f5830c635fddd0b4993b9742a66eb746dacf433c8590e8f9e3c", size = 1841409, upload-time = "2026-06-07T21:09:26.207Z" }, + { url = "https://files.pythonhosted.org/packages/7b/3c/bb4a7cba26956cb3da4553cc2056cf67be5b5ff6e6d8fa4fbdff73bfb7ae/aiohttp-3.14.1-cp314-cp314t-win32.whl", hash = "sha256:47ddf841cdecc810749921d25606dee45857d12d2ad5ddb7b5bd7eab12e4b365", size = 494166, upload-time = "2026-06-07T21:09:28.505Z" }, + { url = "https://files.pythonhosted.org/packages/8a/84/ec80c2c1f66a952555a9f86df6b33af65108a6febfa0471b69013a12f807/aiohttp-3.14.1-cp314-cp314t-win_amd64.whl", hash = "sha256:5e78b522b7a6e27e0b25d19b247b75039ac4c94f99823e3c9e53ae1603a9f7e9", size = 530255, upload-time = "2026-06-07T21:09:30.843Z" }, + { url = "https://files.pythonhosted.org/packages/2a/71/6e22be134a4061ada85a92951b842f2657f17d926b727f3f94c56ae963d6/aiohttp-3.14.1-cp314-cp314t-win_arm64.whl", hash = "sha256:90d53f1609c29ccc2193945ef732428382a28f78d0456ae4d3daf0d48b74f0f6", size = 469640, upload-time = "2026-06-07T21:09:33.028Z" }, ] [[package]] @@ -187,78 +178,94 @@ wheels = [ ] [[package]] -name = "arro3-core" -version = "0.6.5" +name = "anyio" +version = "4.14.2" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "typing-extensions", marker = "python_full_version < '3.12'" }, + { name = "exceptiongroup", marker = "python_full_version < '3.11'" }, + { name = "idna" }, + { name = "typing-extensions", marker = "python_full_version < '3.13'" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/2a/01/f06342d2eb822153f63d188153e41fbeabb29b48247f7a11ce76c538f7d1/arro3_core-0.6.5.tar.gz", hash = "sha256:768078887cd7ac82de4736f94bbd91f6d660f10779848bd5b019f511badd9d75", size = 107522, upload-time = "2025-10-13T23:12:38.872Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/7a/8a/24b35cf01a68621f5f07e3191ca96f70a145022ca367347266901eb504a7/arro3_core-0.6.5-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:da193dc2fb8c2005d0b3887b09d1a90d42cec1f59f17a8a1a5791f0de90946ae", size = 2678116, upload-time = "2025-10-13T23:09:04.198Z" }, - { url = "https://files.pythonhosted.org/packages/5a/7a/4398bb0582fb22d575f256f2b9ac7be735c765222cc61fb214d606bdb77c/arro3_core-0.6.5-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:ed1a760ec39fe19c65e98f45515582408002d0212df5db227a5959ffeb07ad4a", size = 2383214, upload-time = "2025-10-13T23:09:06.841Z" }, - { url = "https://files.pythonhosted.org/packages/82/3f/a321501c5da4bf3ff7438c3e5eb6e63bcecb5630c0f4a89a017cbfa8e4a0/arro3_core-0.6.5-cp310-cp310-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:6584a3d28007740afcef1e301332876e2b785bd8edd59a458a6bc9b051bce052", size = 2883536, upload-time = "2025-10-13T23:09:08.877Z" }, - { url = "https://files.pythonhosted.org/packages/0d/50/1d1e55b9a8c4cf2fdeb954947aa135010554a3333b709e8cad3d5d084be2/arro3_core-0.6.5-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:8e0af4789618f02bead4a0cd4d0a54abd9c8aa4fcedf9872b4891d2e3e984161", size = 2908828, upload-time = "2025-10-13T23:09:10.958Z" }, - { url = "https://files.pythonhosted.org/packages/12/75/b4b1de1ccb17890bada9a3f4131cf3137f145d5d10490db51de6b8799926/arro3_core-0.6.5-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:c73f212e549e9b6d11cfe3f14bbf3fba9d0891426afb5916688d16d0df724085", size = 3145458, upload-time = "2025-10-13T23:09:13.275Z" }, - { url = "https://files.pythonhosted.org/packages/08/4f/f42ce1840490fd0863bfbc56f28eaaec3bcb4eb322079af9c070111657e5/arro3_core-0.6.5-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:89f88f62e4e276a9e84f250722d2e5ffc078af9a3f67ac691f572a0e05dd6095", size = 2775793, upload-time = "2025-10-13T23:09:15.342Z" }, - { url = "https://files.pythonhosted.org/packages/2b/aa/9637efc8d8733c34bedef44e5b2c170dea14d15ab56b3566d8d7963c2616/arro3_core-0.6.5-cp310-cp310-manylinux_2_24_aarch64.whl", hash = "sha256:b2635e4c227f25ff8784dc8efb38cb7c1674646cfdc68ded53f2426289885f0e", size = 2516697, upload-time = "2025-10-13T23:09:17.584Z" }, - { url = "https://files.pythonhosted.org/packages/60/84/1fcfadf956bc25eb5251b1ea7a7099f05198a55764635d2fc9ceafdbdbd1/arro3_core-0.6.5-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:a5f3e936686bcd8542fafc94c68fdb23ec42d1d51a4777967ae815c90aff7296", size = 3023625, upload-time = "2025-10-13T23:09:21.556Z" }, - { url = "https://files.pythonhosted.org/packages/58/d0/52d0cb3c0dfa8e94ba2118b7e91a70da76d6ede9de4e70374f831f38cfdf/arro3_core-0.6.5-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:705c32fec03dadc08f807d69ce557882005d43eb20ec62699f7036340f0d580f", size = 2701346, upload-time = "2025-10-13T23:09:25.031Z" }, - { url = "https://files.pythonhosted.org/packages/69/bf/42a6f6501805c31cb65d8a6e3379eeec4fa6c26dc07c9ce894f363ccad1c/arro3_core-0.6.5-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:56d8166235a4c54e4f7ba082ec76890c820fa8c1b6c995ec59cead62a9698e59", size = 3153207, upload-time = "2025-10-13T23:09:28.254Z" }, - { url = "https://files.pythonhosted.org/packages/4f/e5/41fdee468b33759b42958347c2d70b0461bf8f70ba1762a94cdf2e9b0142/arro3_core-0.6.5-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:1ba43ba9081c00767083195222b6be74913de668296f55599658c4b0bb7cd327", size = 3105033, upload-time = "2025-10-13T23:09:31.545Z" }, - { url = "https://files.pythonhosted.org/packages/03/e0/b6d733b4540c05bac546162e045b547031f4d88c67b7c864929d9bce29ad/arro3_core-0.6.5-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:4f5df13c6742e3f0b494cfe9025dccdc8426a74cc9e3e5a1239311e07a4b24e0", size = 2954793, upload-time = "2025-10-13T23:09:34.988Z" }, - { url = "https://files.pythonhosted.org/packages/c0/34/8353ba79c8d0498eaacc077d58b384ef785e0b69c9cbff7c2580136b8fe3/arro3_core-0.6.5-cp310-cp310-win_amd64.whl", hash = "sha256:34676b728178236df63c9ea10b21432392d4b5bb51e2030e77c68eed4dede2ad", size = 2837495, upload-time = "2025-10-13T23:09:38.539Z" }, - { url = "https://files.pythonhosted.org/packages/78/85/20e46d3ed59d2f93be4a4d1abea4f6bef3e96acd59bf5a50726f84303c51/arro3_core-0.6.5-cp311-abi3-macosx_10_12_x86_64.whl", hash = "sha256:9d5999506daec1ab31096b3deb1e3573041d6ecadb4ca99c96f7ab26720c592c", size = 2685615, upload-time = "2025-10-13T23:09:41.793Z" }, - { url = "https://files.pythonhosted.org/packages/d0/9c/427d578f7d2bf3149515a8b75217e7189e7b1d74e5c5609e1a7e7f0f8d3c/arro3_core-0.6.5-cp311-abi3-macosx_11_0_arm64.whl", hash = "sha256:bd3e251184c2dd6ade81c5613256b6d85ab3ddbd5af838b1de657e0ddec017f8", size = 2391944, upload-time = "2025-10-13T23:09:45.266Z" }, - { url = "https://files.pythonhosted.org/packages/90/24/7e4af478eb889bfa401e1c1b8868048ca692e6205affbf81cf3666347852/arro3_core-0.6.5-cp311-abi3-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:7cadb29349960d3821b0515d9df80f2725cea155ad966c699f6084de32e313cb", size = 2888376, upload-time = "2025-10-13T23:09:48.737Z" }, - { url = "https://files.pythonhosted.org/packages/70/3b/01006a96bc980275aa4d2eb759c5f10afb7c85fcdce3c36ddb18635ad23b/arro3_core-0.6.5-cp311-abi3-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a922e560ed2ccee3293d51b39e013b51cc233895d25ddafcacfb83c540a19e6f", size = 2916568, upload-time = "2025-10-13T23:09:51.95Z" }, - { url = "https://files.pythonhosted.org/packages/a2/2f/4e04c7f5687de6fb6f88aa7590b16bcf507ba17ddbd268525f27b70b7a68/arro3_core-0.6.5-cp311-abi3-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:68fe6672bf51f039b12046a209cba0a9405e10ae44e5a0d557f091b356a62051", size = 3144223, upload-time = "2025-10-13T23:09:55.387Z" }, - { url = "https://files.pythonhosted.org/packages/31/4a/72dc383d1a0d14f1d453e334e3461e229762edb1bf3f75b3ab977e9386ed/arro3_core-0.6.5-cp311-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5c3ee95603e375401a58ff763ce2c8aa858e0c4f757c1fb719f48fb070f540b2", size = 2781862, upload-time = "2025-10-13T23:09:59.035Z" }, - { url = "https://files.pythonhosted.org/packages/14/dc/0df7684b683114eaf8e57989b4230edb359cbfb6e98b8770d69128b27572/arro3_core-0.6.5-cp311-abi3-manylinux_2_24_aarch64.whl", hash = "sha256:fbaf6b65213630007b798b565e0701c2092a330deeba16bd3d896d401f7e9f28", size = 2522442, upload-time = "2025-10-13T23:10:02.134Z" }, - { url = "https://files.pythonhosted.org/packages/c9/04/75f8627cd7fe4d103eca51760d50269cfbc0bf6beaf83a3cdefb4ebd37c7/arro3_core-0.6.5-cp311-abi3-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:20679f874558bb2113e96325522625ec64a72687000b7a9578031a4d082c6ef5", size = 3033454, upload-time = "2025-10-13T23:10:05.192Z" }, - { url = "https://files.pythonhosted.org/packages/ea/19/f2d54985da65bf6d3da76218bee56383285035541c8d0cadb53095845b3e/arro3_core-0.6.5-cp311-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:d82d6ec32d5c7c73057fb9c528390289fd5bc94b8d8f28fca9c56fc8e41c412c", size = 2705984, upload-time = "2025-10-13T23:10:08.518Z" }, - { url = "https://files.pythonhosted.org/packages/6c/53/b1d7742d6db7b4aa44d3785956955d651b3ac36db321625fd15466be1aca/arro3_core-0.6.5-cp311-abi3-musllinux_1_2_armv7l.whl", hash = "sha256:4cba4db0a4203a3ccf131c3fb7804d77f0740d6165ec9efa3aa3acbca87c43a3", size = 3157472, upload-time = "2025-10-13T23:10:11.976Z" }, - { url = "https://files.pythonhosted.org/packages/05/31/68711327dbdd480aed54158fc1c46ab245e860ab0286e0916ce788f9889e/arro3_core-0.6.5-cp311-abi3-musllinux_1_2_i686.whl", hash = "sha256:e358affc4a0fe5c1b5dccf4f92c43a836aaa4c4eab0906c83b00b60275de3b6d", size = 3117099, upload-time = "2025-10-13T23:10:15.374Z" }, - { url = "https://files.pythonhosted.org/packages/31/e3/15ffca0797d9500b23759ae4477cf052fde8dd47a3890f4e4e1d04639016/arro3_core-0.6.5-cp311-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:324e43f07b7681846d00a8995b78bdc4b4a719047aa0d34426b462b8f208ee98", size = 2963677, upload-time = "2025-10-13T23:10:18.828Z" }, - { url = "https://files.pythonhosted.org/packages/bc/02/69e60dbe3bbe2bfc8b6dfa4f4bfcb8d1dd240a137bf2a5f7bcc84703f05c/arro3_core-0.6.5-cp311-abi3-win_amd64.whl", hash = "sha256:285f802c8a42fe29ecb84584d1700bc4c4f974552b75f805e1f4362d28b97080", size = 2850445, upload-time = "2025-10-13T23:10:22.345Z" }, - { url = "https://files.pythonhosted.org/packages/b1/29/2e5b091f6b5cffb6489dbe7ed353841568dde8ac4d1232c77321da1d0925/arro3_core-0.6.5-cp313-cp313t-macosx_10_12_x86_64.whl", hash = "sha256:8c20e69c3b3411fd6ed56091f388e699072651e880e682be5bd14f3a392ed3e8", size = 2671985, upload-time = "2025-10-13T23:10:25.515Z" }, - { url = "https://files.pythonhosted.org/packages/30/74/764ac4b58fef3fdfc655416c42349206156db5c687fa24a0674acaeaadbb/arro3_core-0.6.5-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:92211f1d03221ff74d0b535a576b39601083d8e98e9d47228314573f9d4f9ae2", size = 2382931, upload-time = "2025-10-13T23:10:29.893Z" }, - { url = "https://files.pythonhosted.org/packages/6a/07/bd8c92e218240ae8a30150a5d7a2dab359b452ab54a8bb7b90effe806e3d/arro3_core-0.6.5-cp313-cp313t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:280d933b75f2649779d76e32a07f91d2352a952f2c97ddf7b320e267f440cd42", size = 2879900, upload-time = "2025-10-13T23:10:33.238Z" }, - { url = "https://files.pythonhosted.org/packages/0f/d4/253725019fe2ae5f5fde87928118ffa568cc59f07b2d6a0e90620938c537/arro3_core-0.6.5-cp313-cp313t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:bfc3f6b93b924f43fb7985b06202343c30b43da6bd5055ba8b84eda431e494d4", size = 2904149, upload-time = "2025-10-13T23:10:36.547Z" }, - { url = "https://files.pythonhosted.org/packages/f0/b0/7a3dea641ac8de041c1a34859a2f2a82d3cdf3c3360872101c1d198a1e24/arro3_core-0.6.5-cp313-cp313t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a5963635eb698ebc7da689e641f68b3998864bab894cf0ca84bd058b8c60d97f", size = 3143477, upload-time = "2025-10-13T23:10:40.232Z" }, - { url = "https://files.pythonhosted.org/packages/a7/05/1a50575be33fe9240898a1b5a8574658a905b5675865285585e070dcf7e2/arro3_core-0.6.5-cp313-cp313t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ac291b3e74b57e56e03373d57530540cbbbfd92e4219fe2778ea531006673fe9", size = 2776522, upload-time = "2025-10-13T23:10:43.413Z" }, - { url = "https://files.pythonhosted.org/packages/2e/bd/e7b03207e7906e94e327cd4190fdb2d26ae52bc4ee1edeb057fed760796b/arro3_core-0.6.5-cp313-cp313t-manylinux_2_24_aarch64.whl", hash = "sha256:5d3f4cc58a654037d61f61ba230419da2c8f88a0ac82b9d41fe307f7cf9fda97", size = 2515426, upload-time = "2025-10-13T23:10:46.926Z" }, - { url = "https://files.pythonhosted.org/packages/f9/ed/82d1febd5c104eccdfb82434e3619125c328c36da143e19dfa3c86de4a81/arro3_core-0.6.5-cp313-cp313t-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:93cddac90238d64451f5e66c630ded89d0b5fd6d2c099bf3a5151dde2c1ddf1d", size = 3024759, upload-time = "2025-10-13T23:10:50.281Z" }, - { url = "https://files.pythonhosted.org/packages/da/cd/00e06907e42e404c21eb08282dee94ac7a1961facfa9a96d116829031721/arro3_core-0.6.5-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:1fa7ac10db5846c33f4e8b66a6eaa705d84998e38575a835acac9a6a6649933d", size = 2700191, upload-time = "2025-10-13T23:10:53.776Z" }, - { url = "https://files.pythonhosted.org/packages/a3/11/a4bb9a900f456a6905d481bd2289f7a2371dcde024de56779621fd6a92c3/arro3_core-0.6.5-cp313-cp313t-musllinux_1_2_armv7l.whl", hash = "sha256:ca69f698a065cdbf845d59d412bc204e8f8af12f93737d82e6a18f3cff812349", size = 3149963, upload-time = "2025-10-13T23:10:57.163Z" }, - { url = "https://files.pythonhosted.org/packages/28/8a/79c76ad88b16f2fac25684f7313593738f353355eb1af2307e43efd7b1ca/arro3_core-0.6.5-cp313-cp313t-musllinux_1_2_i686.whl", hash = "sha256:de74a2512e2e2366d4b064c498c38672bf6ddea38acec8b1999b4e66182dd001", size = 3104663, upload-time = "2025-10-13T23:11:00.582Z" }, - { url = "https://files.pythonhosted.org/packages/20/66/9152feaa87f851a37c1a2bd74fb89d7e82e4c76447ee590bf8e6fff5e9d8/arro3_core-0.6.5-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:806ca8e20507675b2de68b3d009f76e898cc3c3e441c834ea5220866f68aac50", size = 2956440, upload-time = "2025-10-13T23:11:03.769Z" }, - { url = "https://files.pythonhosted.org/packages/ad/66/f4179ef64d5c18fe76ec93cfbff42c0f401438ef771c6766b880044d7e13/arro3_core-0.6.5-cp313-cp313t-win_amd64.whl", hash = "sha256:8f6f0cc78877ade7ad6e678a4671b191406547e7b407bc9637436869c017ed47", size = 2845345, upload-time = "2025-10-13T23:11:07.447Z" }, - { url = "https://files.pythonhosted.org/packages/10/ca/b2139dbb25f9fefb9b1cdce8a73785615de6763af6a16bf6ff96a3b630f2/arro3_core-0.6.5-pp310-pypy310_pp73-macosx_10_12_x86_64.whl", hash = "sha256:26d5b50139f1a96727fa1760b4d70393acf5ee0fba45346ad2d4f69824d3bdc2", size = 2676788, upload-time = "2025-10-13T23:11:56.965Z" }, - { url = "https://files.pythonhosted.org/packages/34/a1/c68dde2944f493c8ccfcb91bf6da6d27a27c3674316dd09c9560f9e6ab1a/arro3_core-0.6.5-pp310-pypy310_pp73-macosx_11_0_arm64.whl", hash = "sha256:b65b3d8d7f65f2f3c36002dc467380d7a31ea771132986dddc6341c5a9dc726f", size = 2382809, upload-time = "2025-10-13T23:12:00.175Z" }, - { url = "https://files.pythonhosted.org/packages/c6/fc/2fb81d42a3cecd632deace97dc23ac74083d60d158106440c783bae4ff01/arro3_core-0.6.5-pp310-pypy310_pp73-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:6c3442a79a757ed3fbd7793de180019ae3201f04237537c2e2e3f1e3dd99b31c", size = 2882818, upload-time = "2025-10-13T23:12:03.721Z" }, - { url = "https://files.pythonhosted.org/packages/58/7f/16f741e1d49ba5c5a893ce6f8eb0283d64bc68d6cc9e07ac62f96eaadfae/arro3_core-0.6.5-pp310-pypy310_pp73-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:def7b0065a684d6f903a658d2567da47e2fcecde716e0b34eff4d899c6468c8d", size = 2907503, upload-time = "2025-10-13T23:12:07.066Z" }, - { url = "https://files.pythonhosted.org/packages/eb/45/2eb7972e0bbec0ee0ab22b0f166ec1ea74b53bd76c93a18ced434713e495/arro3_core-0.6.5-pp310-pypy310_pp73-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:cbfe2f2d4d0d393833cd6a4bd9c15266a02307a3028f159155a1c536469c3ae7", size = 3143706, upload-time = "2025-10-13T23:12:10.492Z" }, - { url = "https://files.pythonhosted.org/packages/2d/af/b78e28842faa675e4e6c4d82e861accf21ac08bbab80a65fa80c578f80a1/arro3_core-0.6.5-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a191a3e4f72c34f7ace7724a94f2d90b06c804a6cbece4ae0f18d36325479cf3", size = 2775462, upload-time = "2025-10-13T23:12:14.026Z" }, - { url = "https://files.pythonhosted.org/packages/45/df/950e57e4915e0457acadaaca13c4423d5e2652e403135eb7606d5e6e5443/arro3_core-0.6.5-pp310-pypy310_pp73-manylinux_2_24_aarch64.whl", hash = "sha256:e3f6ab4c6ea96c451eff72aa6c5b9835a0ea8a9847cfe3995c88cce0c7701fb5", size = 2516212, upload-time = "2025-10-13T23:12:17.548Z" }, - { url = "https://files.pythonhosted.org/packages/07/73/821640d0827a829ed2565c2d4812080ab7fb86f0d271b462f9b37e6d946e/arro3_core-0.6.5-pp310-pypy310_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:27df5239835330299636a02977f2cb34d5c460cc03b2ae1d6ab6a03d28051b08", size = 3023342, upload-time = "2025-10-13T23:12:21.308Z" }, - { url = "https://files.pythonhosted.org/packages/fd/30/51302d2f4d1b627dd11e2be979f2c48550b782d8d58d0378316342e284a8/arro3_core-0.6.5-pp310-pypy310_pp73-musllinux_1_2_aarch64.whl", hash = "sha256:71dce89c0e91be4cfb42591f03809235bbc374c396e08acdf93c4d85b09e40f5", size = 2700740, upload-time = "2025-10-13T23:12:24.968Z" }, - { url = "https://files.pythonhosted.org/packages/1d/e8/0c8a345a013bb64abea60b4864bacc01e43b8699b8874794baec9c8a7e76/arro3_core-0.6.5-pp310-pypy310_pp73-musllinux_1_2_armv7l.whl", hash = "sha256:d380c28f85568ed99c1686fb9d64b5a811d76d569f367cbec8ef7e58f6e2fdf9", size = 3152749, upload-time = "2025-10-13T23:12:28.393Z" }, - { url = "https://files.pythonhosted.org/packages/6a/42/003b30c4da394366d5967a5b993f7471a74182c983d8f757891b3dd5d594/arro3_core-0.6.5-pp310-pypy310_pp73-musllinux_1_2_i686.whl", hash = "sha256:8e359c0c4fe9992f5a863a4a31502ea58eb2f92988fc2e501850540b3eff0328", size = 3104676, upload-time = "2025-10-13T23:12:31.711Z" }, - { url = "https://files.pythonhosted.org/packages/0b/fd/4f8dac58ea17e05978bf35cb9a3e485b1ff3cdd6e2cc29deb08f54080de4/arro3_core-0.6.5-pp310-pypy310_pp73-musllinux_1_2_x86_64.whl", hash = "sha256:9a58acbc61480b533aa84d735db04b1e68fc7f6807ab694d606c03b5e694d83d", size = 2954405, upload-time = "2025-10-13T23:12:35.328Z" }, -] - -[[package]] -name = "astunparse" -version = "1.6.3" +sdist = { url = "https://files.pythonhosted.org/packages/61/cc/a381afa6efea9f496eff839d4a6a1aed3bfafc7b3ab4b0d1b243a12573dd/anyio-4.14.2.tar.gz", hash = "sha256:cfa139f3ed1a23ee8f88a145ddb5ac7605b8bbfd8592baacd7ce3d8bb4313c7f", size = 260176, upload-time = "2026-07-12T20:29:07.082Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/da/35/f2287558c17e29fafc8ef3daf819bb9834061cfa43bff8014f7df7f63bdc/anyio-4.14.2-py3-none-any.whl", hash = "sha256:9f505dda5ac9f0c8309b5e8bd445a8c2bf7246f3ce950121e45ea15bc41d1494", size = 125813, upload-time = "2026-07-12T20:29:05.763Z" }, +] + +[[package]] +name = "arro3-core" +version = "0.8.1" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "six" }, - { name = "wheel" }, + { name = "typing-extensions", marker = "python_full_version < '3.12'" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/f3/af/4182184d3c338792894f34a62672919db7ca008c89abee9b564dd34d8029/astunparse-1.6.3.tar.gz", hash = "sha256:5ad93a8456f0d084c3456d059fd9a92cce667963232cbf763eac3bc5b7940872", size = 18290, upload-time = "2019-12-22T18:12:13.129Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/2b/03/13dde6512ad7b4557eb792fbcf0c653af6076b81e5941d36ec61f7ce6028/astunparse-1.6.3-py2.py3-none-any.whl", hash = "sha256:c2652417f2c8b5bb325c885ae329bdf3f86424075c4fd1a128674bc6fba4b8e8", size = 12732, upload-time = "2019-12-22T18:12:11.297Z" }, +sdist = { url = "https://files.pythonhosted.org/packages/c0/c8/fc5bacb6fc264dc61e46d4832f690015b7f6c693ff5dea8a1e53b63cb772/arro3_core-0.8.1.tar.gz", hash = "sha256:1df54a8e2c14a877f291d90de65f00bafe9cc6d5958417ff749f178f059dcd39", size = 93684, upload-time = "2026-06-11T18:01:37.508Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ea/18/81ff17882bb43bcef22bfc67d5b9522ee52f39b0bc6516eda718f99fb1aa/arro3_core-0.8.1-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:88b15a937ebe7e64f63cbbd1134d18daa3cb3f769e936a1f0284136375707ebb", size = 3077128, upload-time = "2026-06-11T17:59:25.637Z" }, + { url = "https://files.pythonhosted.org/packages/a2/c1/f9b9675f12f0cc2da922f4e1d029600f76b834879e11eed22b1a2b5d9521/arro3_core-0.8.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:0501662cb3c7c6a05dcd6b998db55f9a97743aa571da42203733366c42166a7a", size = 2798583, upload-time = "2026-06-11T17:59:27.193Z" }, + { url = "https://files.pythonhosted.org/packages/39/80/62e19c6935d5633452a76354881be62f7a25f1bc7d8ad2e74faf3c68790a/arro3_core-0.8.1-cp310-cp310-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:5674a052c68721963f462b10ce92596a7e85cfacd3fbdc3839633182844f6f3c", size = 3270157, upload-time = "2026-06-11T17:59:28.785Z" }, + { url = "https://files.pythonhosted.org/packages/9a/7d/0eadbcc314d61cad40679878bab76ae224a476059097af271051e4f36024/arro3_core-0.8.1-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ebc5e8917ccc638f276292532e7a59dad3d89ba8a26e874e596a6bc4ffb0ca4b", size = 3404256, upload-time = "2026-06-11T17:59:30.276Z" }, + { url = "https://files.pythonhosted.org/packages/76/6f/6b2ed8c8c0f4bff723e9485ac64daf3d34e424f3d266517e002644b9171d/arro3_core-0.8.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:3f30c53a51934884416b752e95ef50b1958d6b55de6a386e919c202ceec7d4ff", size = 3467310, upload-time = "2026-06-11T17:59:31.889Z" }, + { url = "https://files.pythonhosted.org/packages/34/48/0fe2f381f29e16e639e691392e1bbc628d275f950f3a8dec3bab7d04742f/arro3_core-0.8.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:31b966b0f9b2a6fb3f285339ed5ec4eac69591bbb92a9fdb2cf80ad995b759af", size = 3190458, upload-time = "2026-06-11T17:59:33.385Z" }, + { url = "https://files.pythonhosted.org/packages/73/63/fd6af88fe75a1316f7742ac11937fb9ab94ffbc582e9e722b807cf2957fa/arro3_core-0.8.1-cp310-cp310-manylinux_2_24_aarch64.whl", hash = "sha256:f27615f05e55e7c277b4386fd18356d7b052e0c92be66cc13cf823286d94b141", size = 2950698, upload-time = "2026-06-11T17:59:34.97Z" }, + { url = "https://files.pythonhosted.org/packages/40/a6/5c3f9c4a8f0f8c510e6b0fea41329f704d6be569cc52f7479c179a84d196/arro3_core-0.8.1-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:f6c9e50b3bc594260cbcc94e4b9fff590c5fd7f35a6c5729ecc6a6ff0a117981", size = 3398996, upload-time = "2026-06-11T17:59:36.369Z" }, + { url = "https://files.pythonhosted.org/packages/aa/0c/b52e17b284e9b0b3cb0c402071e1057bf422309407e7f9c3bc80818ee581/arro3_core-0.8.1-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:97584eb695f6d9a2cb5e66aa4866fdd17316c313d1724257660bcb0f37a59fb1", size = 3128930, upload-time = "2026-06-11T17:59:37.914Z" }, + { url = "https://files.pythonhosted.org/packages/58/99/6e6b15a8e9b6ea8995f2b0ccd01a91fd3e72af74ef45b856b7675bb58b04/arro3_core-0.8.1-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:35dfbc1c19afb63371ae23bd653202b57fd99b57f17ed1561849c4b5994b63d9", size = 3547956, upload-time = "2026-06-11T17:59:39.327Z" }, + { url = "https://files.pythonhosted.org/packages/dc/3e/82d1df9edcd26632c5b6271ebbc6c825307abb76dc9c8de04094510b3dc6/arro3_core-0.8.1-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:2dd781426d776fad17dba1c991df422d071d22e59cfd0db80b104648b6c96040", size = 3505514, upload-time = "2026-06-11T17:59:41.219Z" }, + { url = "https://files.pythonhosted.org/packages/c1/c8/ffc2d069ba82444fbeb0a41b24a4c51ddfe99935b11c8572ce3c1911c19f/arro3_core-0.8.1-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:540f85f0b6a31820347d069c217b002488fe60f8e509d134bd1e0bc2eeffbfff", size = 3405220, upload-time = "2026-06-11T17:59:43.275Z" }, + { url = "https://files.pythonhosted.org/packages/df/12/d7de53951ed8b89bc3f390b416e25e92dd765bc2faa13eedcd467248bc24/arro3_core-0.8.1-cp310-cp310-win_amd64.whl", hash = "sha256:507c4a58c993af697eb2e7b324c1f2c4ced22a72c85e9c2adbc23d28c6a6c20b", size = 3357259, upload-time = "2026-06-11T17:59:44.961Z" }, + { url = "https://files.pythonhosted.org/packages/ca/75/29517738623cccba1d60d0aa65b896bdd046ab4a82c5cd5a9fca115f6d7a/arro3_core-0.8.1-cp311-abi3-macosx_10_12_x86_64.whl", hash = "sha256:64d3ea60da4279e9c6a9b2e60abf7f4fefb6643544ecb2dfde899f72f1f91bad", size = 3085640, upload-time = "2026-06-11T17:59:46.597Z" }, + { url = "https://files.pythonhosted.org/packages/c0/b5/bff1842ed8dbb2134b004f78c52d977b52a6df1d9389b1284aad02bf4d93/arro3_core-0.8.1-cp311-abi3-macosx_11_0_arm64.whl", hash = "sha256:463db0b7f698abc19f4274d6df697560bc9c19463cfdbd01094bb0fa6ddd1206", size = 2800705, upload-time = "2026-06-11T17:59:48.242Z" }, + { url = "https://files.pythonhosted.org/packages/32/20/ef60404d5008f84bca50510f65d7acfe26812e61638ee19a416bf3f11530/arro3_core-0.8.1-cp311-abi3-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:881afcb9c83334ac26498b429265223c676e028cd5d88fb0c909578f8e0330de", size = 3272103, upload-time = "2026-06-11T17:59:49.641Z" }, + { url = "https://files.pythonhosted.org/packages/f5/8e/6687efc8e0414cfde6281b1f4ea3b5b44aeb318584e39113d645d1f913a2/arro3_core-0.8.1-cp311-abi3-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:d1dc03e1e5965528bfc7e26a6ee857e3b6fc5f201ff530a03dc731b9f3fb14fd", size = 3405750, upload-time = "2026-06-11T17:59:51.037Z" }, + { url = "https://files.pythonhosted.org/packages/2e/38/2567f26cd041387a2c1f381f1757b5246184de22dd8bd9dbbcb91a1b3586/arro3_core-0.8.1-cp311-abi3-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:410cdca92392be39a5580059ebf9a6f89a0410af6dc10a8baa76e9d511fa6624", size = 3467290, upload-time = "2026-06-11T17:59:52.672Z" }, + { url = "https://files.pythonhosted.org/packages/85/38/2c5af3a3e8c806188f1b3a651cbaa6c22d1f094c41e82900d40219c2b337/arro3_core-0.8.1-cp311-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f4082c6bbff7f164619d99dffcbc2313ed69024653a9e58d9f94cc65d9226f6c", size = 3195582, upload-time = "2026-06-11T17:59:54.536Z" }, + { url = "https://files.pythonhosted.org/packages/e4/8d/d7d8686901a273743c53aae98f898f00caaedea807e28854200f2a7365e4/arro3_core-0.8.1-cp311-abi3-manylinux_2_24_aarch64.whl", hash = "sha256:40da2ee0701f217cd482e61228023ee9d8993984daccaf6ded089035b5fdc132", size = 2950909, upload-time = "2026-06-11T17:59:56.05Z" }, + { url = "https://files.pythonhosted.org/packages/5b/8d/f44198a859b13f519b331906d08079c7c281f8267d718082a3ca858c93ee/arro3_core-0.8.1-cp311-abi3-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:d79b12a83403b1a100731587d33f09a818a4ff93472f84fcdf3a38d3934dfc5e", size = 3403497, upload-time = "2026-06-11T17:59:57.651Z" }, + { url = "https://files.pythonhosted.org/packages/88/6d/431d2ef9942a30599db4a86cf13b7f1de92d340717438074a25659d382f4/arro3_core-0.8.1-cp311-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:3ff2da3c888ec504291deafd6965f2364f14f1fb63977f202f2f7680f10909f6", size = 3130901, upload-time = "2026-06-11T17:59:59.262Z" }, + { url = "https://files.pythonhosted.org/packages/ce/9e/458e8596c675fb88075137ab21d4c4a2f9e585101a97d77e910839862168/arro3_core-0.8.1-cp311-abi3-musllinux_1_2_armv7l.whl", hash = "sha256:0bbcfb60ddf9b571387b4c0aaf78171d2ee7e0d335c08514bc247685f01bc086", size = 3550071, upload-time = "2026-06-11T18:00:00.808Z" }, + { url = "https://files.pythonhosted.org/packages/93/cd/8fa1f4a49e707ea364022620ca5033ba69b6396215e0bae1fb9fa9efcfe0/arro3_core-0.8.1-cp311-abi3-musllinux_1_2_i686.whl", hash = "sha256:44f6256cae47d369fe9076e9fa61f5c2899447b76ca5bda641668ad07cf26bf6", size = 3508957, upload-time = "2026-06-11T18:00:02.341Z" }, + { url = "https://files.pythonhosted.org/packages/7f/02/0126ff3b2ff48187315a9782280c0f6412c066559e22bf206ceec2791299/arro3_core-0.8.1-cp311-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:b2517ea3fb7cb15a41e2d0e3496d91afe4703eca86a88072f917bef044c2b8ee", size = 3408923, upload-time = "2026-06-11T18:00:03.97Z" }, + { url = "https://files.pythonhosted.org/packages/d8/fd/a19cb50480a769b1cc3f347efa1a808e13b853bb3b1d94b289677115fe92/arro3_core-0.8.1-cp311-abi3-win_amd64.whl", hash = "sha256:6a96df94a4538ab9acf01873eecf35161ad0c54ab79134247e69a42cd66515dc", size = 3368045, upload-time = "2026-06-11T18:00:05.578Z" }, + { url = "https://files.pythonhosted.org/packages/5c/86/1ac6eed229482b1ef6ac6f3211c6760012aaaa026b0a385a18e42ec0ff74/arro3_core-0.8.1-cp313-cp313-pyemscripten_2025_0_wasm32.whl", hash = "sha256:71165a75a7d22c16226085ed2255afe84741369cd35eb01cc153c4687e85b49f", size = 1724964, upload-time = "2026-06-11T18:00:07.653Z" }, + { url = "https://files.pythonhosted.org/packages/4a/32/c2e4a65b3b4f7e00552d280e9c2eaaca59c19466b0f95e4e08d6a020ab2d/arro3_core-0.8.1-cp313-cp313t-macosx_10_12_x86_64.whl", hash = "sha256:dc81e079d182dfc2ba6bace815608dbab0158809b9268c4eb59816685f7871fb", size = 3071070, upload-time = "2026-06-11T18:00:09.12Z" }, + { url = "https://files.pythonhosted.org/packages/f0/32/56016de27757bcc0d66999ae2fec624eaa27bd097ee69ac7a5a0d52d407b/arro3_core-0.8.1-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:7695e9fca1e2c0064571bea65d9b7909a957ed9be4c81718d19356e578818649", size = 2794497, upload-time = "2026-06-11T18:00:10.872Z" }, + { url = "https://files.pythonhosted.org/packages/d2/6e/a7aac0e87d6d63672d4ebb7e8466b276fde2a6413579008fadd1e8e919c5/arro3_core-0.8.1-cp313-cp313t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:ad0f507dfafb1e8a8c15e8a902bccdd9a9c2cf2102fc39ad35c22e23ca76a65e", size = 3272090, upload-time = "2026-06-11T18:00:12.561Z" }, + { url = "https://files.pythonhosted.org/packages/1b/fd/8b96e57eea8ebf5fa4e2556b06c84a29ea25ded924f56d300807ff377a94/arro3_core-0.8.1-cp313-cp313t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:3fc91a59cb27c7660134b9ed0c067d53c0f5bb1722f16f2cf7d4c446ce1086c4", size = 3399753, upload-time = "2026-06-11T18:00:13.992Z" }, + { url = "https://files.pythonhosted.org/packages/80/8b/76f1385ecd0448fcc60902a4474f3f00bd2197fe6d7fa4434bb6a0b3dee9/arro3_core-0.8.1-cp313-cp313t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:71bc86880256cca5f22802ca444a92b3e9edbb5c9e5b74df1cbfcfe2ca788097", size = 3468912, upload-time = "2026-06-11T18:00:15.502Z" }, + { url = "https://files.pythonhosted.org/packages/0f/15/e1c3aefee23d926be9f75f125d6d94f189f28826aa01713498f1de60b080/arro3_core-0.8.1-cp313-cp313t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:94528515f59146b5b4421e468bc269f682754ff082434cccf0ecbfb0d74d0b38", size = 3193338, upload-time = "2026-06-11T18:00:17.064Z" }, + { url = "https://files.pythonhosted.org/packages/de/3d/bfa3963bd09941f69d810a387bbea61deb8a091101c1765216266ec61394/arro3_core-0.8.1-cp313-cp313t-manylinux_2_24_aarch64.whl", hash = "sha256:e348fa5349e6655df4b1ea281ab71e0794c749ba842da54248eb942201f93fb8", size = 2950166, upload-time = "2026-06-11T18:00:18.607Z" }, + { url = "https://files.pythonhosted.org/packages/21/07/67bdc66582294d5914b8f7462907b1e44377ccb77a0f7e7b7760c9f81fd1/arro3_core-0.8.1-cp313-cp313t-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:d97170d56690655699977f4535120f170c5047813341c2b0264abc3af00625dc", size = 3401097, upload-time = "2026-06-11T18:00:20.135Z" }, + { url = "https://files.pythonhosted.org/packages/6d/a3/ab0d46e8fe2337f0ce1789949c5078e63d532fd628af4f254702e1359989/arro3_core-0.8.1-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:6f71fdd46e1a86bc1db0733361130e51447d2d3c7d5fca13937381e2ba085dca", size = 3128454, upload-time = "2026-06-11T18:00:21.598Z" }, + { url = "https://files.pythonhosted.org/packages/c2/6e/e90cea2555aaa429c16173cbc121b41fb6a656e70ae9f21edbe3f9e73eb0/arro3_core-0.8.1-cp313-cp313t-musllinux_1_2_armv7l.whl", hash = "sha256:3e2c2e4c6b36a0f493f07222eadf7c628155f4a306c650a14c4d6d343870043c", size = 3550294, upload-time = "2026-06-11T18:00:24.344Z" }, + { url = "https://files.pythonhosted.org/packages/15/38/f3db72e411fbaea83844d19b1965abbdf3dd387083f99a715d3b3752b461/arro3_core-0.8.1-cp313-cp313t-musllinux_1_2_i686.whl", hash = "sha256:90d31db38bc7cbbe6061d23ae11b3e830b55ce60ba6f0550c70915bab4cafe41", size = 3507806, upload-time = "2026-06-11T18:00:26.143Z" }, + { url = "https://files.pythonhosted.org/packages/47/73/5d011dad78ac58aaa61e9cc65d349b879afec6781c5aa78f80c839fe359f/arro3_core-0.8.1-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:9d28d87625d6c743fee1d2e20252531db6cff6c5218b89055a9f7492af99c14c", size = 3408910, upload-time = "2026-06-11T18:00:27.843Z" }, + { url = "https://files.pythonhosted.org/packages/6a/43/be86ea01c2d53d32376602cb2efa9a9b7a25d98e14ad4b5bf0d7da8b0564/arro3_core-0.8.1-cp313-cp313t-win_amd64.whl", hash = "sha256:b394218ad71ce266088e0e65bfad0542556adf0eb93197b66afaa2009358662f", size = 3356174, upload-time = "2026-06-11T18:00:29.395Z" }, + { url = "https://files.pythonhosted.org/packages/7a/0f/2f7b5b458fd38ecf9277fd59919e4c90047795d1abdfd47f0081c48c28fa/arro3_core-0.8.1-cp314-cp314-pyemscripten_2026_0_wasm32.whl", hash = "sha256:a7c227982c0fd762271d550a6242e54ff7d2aa7aff73c918e127042cf9601ec0", size = 1710676, upload-time = "2026-06-11T18:00:30.947Z" }, + { url = "https://files.pythonhosted.org/packages/4d/9e/b64c9325173b7d87030955007af55348b866a1c02545382261d8966d8bb1/arro3_core-0.8.1-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:5861781e6363db0707e1a4ba5166025a1113d921ccff870a0a097f27ab2dece1", size = 3072327, upload-time = "2026-06-11T18:00:32.313Z" }, + { url = "https://files.pythonhosted.org/packages/3c/bf/2c58a549b2409439fcac4ce2abe31ef060d05114a1e5412ce3088e2a2cde/arro3_core-0.8.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:8c1d9ff8b2848bf48bbcad962271b2adeda922457cf23591464c12af9b0ddbf6", size = 2794872, upload-time = "2026-06-11T18:00:33.92Z" }, + { url = "https://files.pythonhosted.org/packages/1f/21/dee8d1c9309820783fe30ad29149508638743cdc61d2851f0132b370ac49/arro3_core-0.8.1-cp314-cp314t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:246057dc9d283a283cb05b228f710e1ada3b4a8c483e519b0ec8b0d8499204db", size = 3272432, upload-time = "2026-06-11T18:00:35.563Z" }, + { url = "https://files.pythonhosted.org/packages/b6/b2/dbc6a3d5ece2cb11c3041821c8322df500b8c8030534d9f791b66b29c090/arro3_core-0.8.1-cp314-cp314t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:5d55178ad56c637f278286370532e58256eff48f71d3c36a4e5f8652eb7300b3", size = 3400822, upload-time = "2026-06-11T18:00:37.129Z" }, + { url = "https://files.pythonhosted.org/packages/a4/44/0ca0e96eebc7f96af4ad8b78c545a9477e8db271aa090043bb8f797e7f58/arro3_core-0.8.1-cp314-cp314t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:7a917195d7553a188bbb9fa815d430ff70ee53357fcc6fdb395d0e4436a66243", size = 3469220, upload-time = "2026-06-11T18:00:39.375Z" }, + { url = "https://files.pythonhosted.org/packages/8e/70/3e9c63e9e4499d373304ceb1315afc8dd326395dd0692ccdebcfec703dab/arro3_core-0.8.1-cp314-cp314t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2435dc4057d604650a34d195e6977ef40b26af0b9dea8f1b842a924a270de2fc", size = 3193813, upload-time = "2026-06-11T18:00:41.281Z" }, + { url = "https://files.pythonhosted.org/packages/a2/a8/48842a836d7fc2bdac16ecaf4e60c9fd98f46d1e9c80f45b85b071ccaffb/arro3_core-0.8.1-cp314-cp314t-manylinux_2_24_aarch64.whl", hash = "sha256:c8b90ad6fbdd3507c20bdbc2b23c88929a3f7c49a8a43697f316e084b5664289", size = 2950818, upload-time = "2026-06-11T18:00:42.987Z" }, + { url = "https://files.pythonhosted.org/packages/bb/46/635ca59c20b972f0575b9fd1b7debebd94bcba1d6fd68271bfff65d3ae91/arro3_core-0.8.1-cp314-cp314t-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:8918dd0fcf5c3650332678ddbfb7c13ce1f0534f9669ab6839c38069885318cf", size = 3401202, upload-time = "2026-06-11T18:00:44.654Z" }, + { url = "https://files.pythonhosted.org/packages/4a/03/020d032cf4937cca0652f434613d327f8f11e10b4a3fbd4ccf48846c3158/arro3_core-0.8.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:6cc8c7c9b81cad2b4eaaeb29da25854bddb5bb373a278f1f1046b277fa7f2ee6", size = 3129352, upload-time = "2026-06-11T18:00:46.322Z" }, + { url = "https://files.pythonhosted.org/packages/b2/55/ce1c840af64e8c5d4f8b684a4c00c5f6d260659c3d2f732c6074eb5597fe/arro3_core-0.8.1-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:2aa0bda5251e93c67318aa8315e563b02fe39d5577b72654c335d69fce802d03", size = 3550475, upload-time = "2026-06-11T18:00:48.021Z" }, + { url = "https://files.pythonhosted.org/packages/19/79/fa8f1a53cf10fe33029619d14f06525e08ea35c1bcdce42157b230098e1b/arro3_core-0.8.1-cp314-cp314t-musllinux_1_2_i686.whl", hash = "sha256:cd462e7ef7735c341c27f754756b244d90a9485818c5ea090fa46009bd8ae252", size = 3507877, upload-time = "2026-06-11T18:00:49.798Z" }, + { url = "https://files.pythonhosted.org/packages/a3/e5/bf8349b2e5613e6c0caf2f4f0758d61dc273649c2bdcfc96978ab9bc1686/arro3_core-0.8.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:480c33d62d66adf92fd731e77728f81b7bd3fcde4e2eb313f5616b5097b1875f", size = 3409360, upload-time = "2026-06-11T18:00:51.393Z" }, + { url = "https://files.pythonhosted.org/packages/a4/0d/e13efbbc448bc817f8343bf12606d22f83011682c821ccd32dda304816be/arro3_core-0.8.1-cp314-cp314t-win_amd64.whl", hash = "sha256:51bea8dc0bc0230b6af8d1b258cf2a7e4f69770ed99062592ce1b43b17732727", size = 3355557, upload-time = "2026-06-11T18:00:52.922Z" }, + { url = "https://files.pythonhosted.org/packages/de/bc/ab3081cc65cd38a5ceda6dd112ed9d70a481e2810c8f92d37441edab6a8a/arro3_core-0.8.1-pp311-pypy311_pp73-macosx_10_12_x86_64.whl", hash = "sha256:d70c68665525744154dc1f65b5038fe97600a5204e8a875b3fe7675455363024", size = 3076025, upload-time = "2026-06-11T18:01:16.781Z" }, + { url = "https://files.pythonhosted.org/packages/0b/5b/b278236c7425f36fe1d25087964ca143d2a11f99b69b19f3c050cf3a0f4b/arro3_core-0.8.1-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:b89c867f65852de7fa0586a4883195a2cca1ee3a7bca8c95c309a02ce6be15b5", size = 2797246, upload-time = "2026-06-11T18:01:18.648Z" }, + { url = "https://files.pythonhosted.org/packages/1c/a5/a3a96bef25a7d0d7fd49ea3762d901a17d908fda774c868abe9aa065e0f2/arro3_core-0.8.1-pp311-pypy311_pp73-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:66d685f7c53137798e05322a17dabc09445a818c9c8959d21e9fb3e084371f51", size = 3269959, upload-time = "2026-06-11T18:01:20.317Z" }, + { url = "https://files.pythonhosted.org/packages/f7/2f/4da148dea5890ae24eade70d3f9dfc93d191f501bf425d906c21dee89510/arro3_core-0.8.1-pp311-pypy311_pp73-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:5a7563856e609dd24be03efde72edd5b1c16dd1fb3d67fd84970df6d9ca36b96", size = 3400877, upload-time = "2026-06-11T18:01:22.112Z" }, + { url = "https://files.pythonhosted.org/packages/7b/9a/2ed015d4a03c131b9e3da322ee8681c5921320a90218865a711d7fe39768/arro3_core-0.8.1-pp311-pypy311_pp73-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b7379e14f8f49e99d53e364c26533293d2136920a3cf818138718a9a9bc114d0", size = 3466119, upload-time = "2026-06-11T18:01:23.909Z" }, + { url = "https://files.pythonhosted.org/packages/25/a8/154478c9b3093602eaec8a5d71b657205177f0ac9fd9da2eef9ddfb42fe0/arro3_core-0.8.1-pp311-pypy311_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:af88a17346e9e7897b6d1bc165f686b87ea928b4c7e89666156b2281f0236cee", size = 3190211, upload-time = "2026-06-11T18:01:25.786Z" }, + { url = "https://files.pythonhosted.org/packages/5b/65/60c6323bbaffd2399f03336a90d7c02450a5b9120348808eb0260651de56/arro3_core-0.8.1-pp311-pypy311_pp73-manylinux_2_24_aarch64.whl", hash = "sha256:1e9e04cbbba877f31073146798b328537b63675f65d0b7e2154754d5203260ed", size = 2950345, upload-time = "2026-06-11T18:01:27.423Z" }, + { url = "https://files.pythonhosted.org/packages/21/81/3a0c786ec86a8c3aee946e4588b188f08a3ccbba71c4e0db144a0151b56c/arro3_core-0.8.1-pp311-pypy311_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:35c46c49442853f8a1b09d37ceabe4ce31e2a276d4a73c407ba3a25fcf1b9221", size = 3398153, upload-time = "2026-06-11T18:01:29.036Z" }, + { url = "https://files.pythonhosted.org/packages/f6/52/e230d0af9881056e527f91e58c518b85000e9ae49f8706c3d0e027cbadd4/arro3_core-0.8.1-pp311-pypy311_pp73-musllinux_1_2_aarch64.whl", hash = "sha256:4e4af152b4ccf7694d634573db906e1a9b28f5beda84bac5adeb60f7e98262a8", size = 3128123, upload-time = "2026-06-11T18:01:30.733Z" }, + { url = "https://files.pythonhosted.org/packages/62/79/9bc36279f1ef2754fde18e1c3c282b2cabe6093b7ca8bd8c0c6c735685ed/arro3_core-0.8.1-pp311-pypy311_pp73-musllinux_1_2_armv7l.whl", hash = "sha256:7bf8cb1053cb51529437544eedbc6643819912eec15c0491d7304d6bad0fb151", size = 3547345, upload-time = "2026-06-11T18:01:32.603Z" }, + { url = "https://files.pythonhosted.org/packages/f9/33/cee8670131009d3a23e140dd0a6cd454111dac54a9a2e83567a6f972ad46/arro3_core-0.8.1-pp311-pypy311_pp73-musllinux_1_2_i686.whl", hash = "sha256:212dfde17bd26193754aaee6024f7e99624163e0d44966745e8752af79c4c454", size = 3504672, upload-time = "2026-06-11T18:01:34.207Z" }, + { url = "https://files.pythonhosted.org/packages/3f/5f/daa7a8af9ad53eebbdb818078d5034525663f298ec9bfdb1933cfd0cc694/arro3_core-0.8.1-pp311-pypy311_pp73-musllinux_1_2_x86_64.whl", hash = "sha256:81426cb122d6bfccacd2e0ae5df8b847dc3c3e07b5f336547cb18f873d186f54", size = 3404906, upload-time = "2026-06-11T18:01:36.097Z" }, ] [[package]] @@ -272,11 +279,11 @@ wheels = [ [[package]] name = "attrs" -version = "25.3.0" +version = "26.1.0" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/5a/b0/1367933a8532ee6ff8d63537de4f1177af4bff9f3e829baf7331f595bb24/attrs-25.3.0.tar.gz", hash = "sha256:75d7cefc7fb576747b2c81b4442d4d4a1ce0900973527c011d1030fd3bf4af1b", size = 812032, upload-time = "2025-03-13T11:10:22.779Z" } +sdist = { url = "https://files.pythonhosted.org/packages/9a/8e/82a0fe20a541c03148528be8cac2408564a6c9a0cc7e9171802bc1d26985/attrs-26.1.0.tar.gz", hash = "sha256:d03ceb89cb322a8fd706d4fb91940737b6642aa36998fe130a9bc96c985eff32", size = 952055, upload-time = "2026-03-19T14:22:25.026Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/77/06/bb80f5f86020c4551da315d78b3ab75e8228f89f0162f2c3a819e407941a/attrs-25.3.0-py3-none-any.whl", hash = "sha256:427318ce031701fea540783410126f03899a97ffc6f61596ad581ac2e40e3bc3", size = 63815, upload-time = "2025-03-13T11:10:21.14Z" }, + { url = "https://files.pythonhosted.org/packages/64/b4/17d4b0b2a2dc85a6df63d1157e028ed19f90d4cd97c36717afef2bc2f395/attrs-26.1.0-py3-none-any.whl", hash = "sha256:c647aa4a12dfbad9333ca4e71fe62ddc36f4e63b2d260a37a8b83d2f043ac309", size = 67548, upload-time = "2026-03-19T14:22:23.645Z" }, ] [[package]] @@ -295,89 +302,133 @@ wheels = [ [[package]] name = "botocore" -version = "1.40.43" +version = "1.40.76" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "jmespath" }, { name = "python-dateutil" }, { name = "urllib3" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/49/d0/3888673417202262ddd7e6361cab8e01ee2705e39643af8445e2eb276eab/botocore-1.40.43.tar.gz", hash = "sha256:d87412dc1ea785df156f412627d3417c9f9eb45601fd0846d8fe96fe3c78b630", size = 14389164, upload-time = "2025-10-01T19:38:16.06Z" } +sdist = { url = "https://files.pythonhosted.org/packages/07/eb/50e2d280589a3c20c3b649bb66262d2b53a25c03262e4cc492048ac7540a/botocore-1.40.76.tar.gz", hash = "sha256:2b16024d68b29b973005adfb5039adfe9099ebe772d40a90ca89f2e165c495dc", size = 14494001, upload-time = "2025-11-18T20:22:59.131Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/79/46/2eb4802e15e38befbea6cab7dafa1ab796722ab6f0833991c2a05e9f8ef0/botocore-1.40.43-py3-none-any.whl", hash = "sha256:1639f38999fc0cf42c92c5c83c5fbe189a4857a86f55b842be868e3283c6d3bb", size = 14057986, upload-time = "2025-10-01T19:38:13.714Z" }, + { url = "https://files.pythonhosted.org/packages/7f/6c/522e05388aa6fc66cf8ea46c6b29809a1a6f527ea864998b01ffb368ca36/botocore-1.40.76-py3-none-any.whl", hash = "sha256:fe425d386e48ac64c81cbb4a7181688d813df2e2b4c78b95ebe833c9e868c6f4", size = 14161738, upload-time = "2025-11-18T20:22:55.332Z" }, ] [[package]] name = "certifi" -version = "2025.8.3" +version = "2026.6.17" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/dc/67/960ebe6bf230a96cda2e0abcf73af550ec4f090005363542f0765df162e0/certifi-2025.8.3.tar.gz", hash = "sha256:e564105f78ded564e3ae7c923924435e1daa7463faeab5bb932bc53ffae63407", size = 162386, upload-time = "2025-08-03T03:07:47.08Z" } +sdist = { url = "https://files.pythonhosted.org/packages/c9/c7/424b75da314c1045981bd9777432fad05a9e0c69daa4ed7e308bbaffe405/certifi-2026.6.17.tar.gz", hash = "sha256:024c88eeec92ca068db80f02b8b07c9cef7b9fe261d1d535abfd5abd6f6af432", size = 134594, upload-time = "2026-06-17T10:31:07.894Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/e5/48/1549795ba7742c948d2ad169c1c8cdbae65bc450d6cd753d124b17c8cd32/certifi-2025.8.3-py3-none-any.whl", hash = "sha256:f6c12493cfb1b06ba2ff328595af9350c65d6644968e5d3a2ffd78699af217a5", size = 161216, upload-time = "2025-08-03T03:07:45.777Z" }, + { url = "https://files.pythonhosted.org/packages/ef/2f/c5464532e965badff2f4c4c1a3a83f5697f0d7c407ed0cda44aaa99bb451/certifi-2026.6.17-py3-none-any.whl", hash = "sha256:2227dcbaafe0d2f59279d1762ddddc37783ed4354594f194ffc31d20f41fc3db", size = 133289, upload-time = "2026-06-17T10:31:06.348Z" }, ] [[package]] name = "charset-normalizer" -version = "3.4.3" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/83/2d/5fd176ceb9b2fc619e63405525573493ca23441330fcdaee6bef9460e924/charset_normalizer-3.4.3.tar.gz", hash = "sha256:6fce4b8500244f6fcb71465d4a4930d132ba9ab8e71a7859e6a5d59851068d14", size = 122371, upload-time = "2025-08-09T07:57:28.46Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/d6/98/f3b8013223728a99b908c9344da3aa04ee6e3fa235f19409033eda92fb78/charset_normalizer-3.4.3-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:fb7f67a1bfa6e40b438170ebdc8158b78dc465a5a67b6dde178a46987b244a72", size = 207695, upload-time = "2025-08-09T07:55:36.452Z" }, - { url = "https://files.pythonhosted.org/packages/21/40/5188be1e3118c82dcb7c2a5ba101b783822cfb413a0268ed3be0468532de/charset_normalizer-3.4.3-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:cc9370a2da1ac13f0153780040f465839e6cccb4a1e44810124b4e22483c93fe", size = 147153, upload-time = "2025-08-09T07:55:38.467Z" }, - { url = "https://files.pythonhosted.org/packages/37/60/5d0d74bc1e1380f0b72c327948d9c2aca14b46a9efd87604e724260f384c/charset_normalizer-3.4.3-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:07a0eae9e2787b586e129fdcbe1af6997f8d0e5abaa0bc98c0e20e124d67e601", size = 160428, upload-time = "2025-08-09T07:55:40.072Z" }, - { url = "https://files.pythonhosted.org/packages/85/9a/d891f63722d9158688de58d050c59dc3da560ea7f04f4c53e769de5140f5/charset_normalizer-3.4.3-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:74d77e25adda8581ffc1c720f1c81ca082921329452eba58b16233ab1842141c", size = 157627, upload-time = "2025-08-09T07:55:41.706Z" }, - { url = "https://files.pythonhosted.org/packages/65/1a/7425c952944a6521a9cfa7e675343f83fd82085b8af2b1373a2409c683dc/charset_normalizer-3.4.3-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d0e909868420b7049dafd3a31d45125b31143eec59235311fc4c57ea26a4acd2", size = 152388, upload-time = "2025-08-09T07:55:43.262Z" }, - { url = "https://files.pythonhosted.org/packages/f0/c9/a2c9c2a355a8594ce2446085e2ec97fd44d323c684ff32042e2a6b718e1d/charset_normalizer-3.4.3-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:c6f162aabe9a91a309510d74eeb6507fab5fff92337a15acbe77753d88d9dcf0", size = 150077, upload-time = "2025-08-09T07:55:44.903Z" }, - { url = "https://files.pythonhosted.org/packages/3b/38/20a1f44e4851aa1c9105d6e7110c9d020e093dfa5836d712a5f074a12bf7/charset_normalizer-3.4.3-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:4ca4c094de7771a98d7fbd67d9e5dbf1eb73efa4f744a730437d8a3a5cf994f0", size = 161631, upload-time = "2025-08-09T07:55:46.346Z" }, - { url = "https://files.pythonhosted.org/packages/a4/fa/384d2c0f57edad03d7bec3ebefb462090d8905b4ff5a2d2525f3bb711fac/charset_normalizer-3.4.3-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:02425242e96bcf29a49711b0ca9f37e451da7c70562bc10e8ed992a5a7a25cc0", size = 159210, upload-time = "2025-08-09T07:55:47.539Z" }, - { url = "https://files.pythonhosted.org/packages/33/9e/eca49d35867ca2db336b6ca27617deed4653b97ebf45dfc21311ce473c37/charset_normalizer-3.4.3-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:78deba4d8f9590fe4dae384aeff04082510a709957e968753ff3c48399f6f92a", size = 153739, upload-time = "2025-08-09T07:55:48.744Z" }, - { url = "https://files.pythonhosted.org/packages/2a/91/26c3036e62dfe8de8061182d33be5025e2424002125c9500faff74a6735e/charset_normalizer-3.4.3-cp310-cp310-win32.whl", hash = "sha256:d79c198e27580c8e958906f803e63cddb77653731be08851c7df0b1a14a8fc0f", size = 99825, upload-time = "2025-08-09T07:55:50.305Z" }, - { url = "https://files.pythonhosted.org/packages/e2/c6/f05db471f81af1fa01839d44ae2a8bfeec8d2a8b4590f16c4e7393afd323/charset_normalizer-3.4.3-cp310-cp310-win_amd64.whl", hash = "sha256:c6e490913a46fa054e03699c70019ab869e990270597018cef1d8562132c2669", size = 107452, upload-time = "2025-08-09T07:55:51.461Z" }, - { url = "https://files.pythonhosted.org/packages/7f/b5/991245018615474a60965a7c9cd2b4efbaabd16d582a5547c47ee1c7730b/charset_normalizer-3.4.3-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:b256ee2e749283ef3ddcff51a675ff43798d92d746d1a6e4631bf8c707d22d0b", size = 204483, upload-time = "2025-08-09T07:55:53.12Z" }, - { url = "https://files.pythonhosted.org/packages/c7/2a/ae245c41c06299ec18262825c1569c5d3298fc920e4ddf56ab011b417efd/charset_normalizer-3.4.3-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:13faeacfe61784e2559e690fc53fa4c5ae97c6fcedb8eb6fb8d0a15b475d2c64", size = 145520, upload-time = "2025-08-09T07:55:54.712Z" }, - { url = "https://files.pythonhosted.org/packages/3a/a4/b3b6c76e7a635748c4421d2b92c7b8f90a432f98bda5082049af37ffc8e3/charset_normalizer-3.4.3-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:00237675befef519d9af72169d8604a067d92755e84fe76492fef5441db05b91", size = 158876, upload-time = "2025-08-09T07:55:56.024Z" }, - { url = "https://files.pythonhosted.org/packages/e2/e6/63bb0e10f90a8243c5def74b5b105b3bbbfb3e7bb753915fe333fb0c11ea/charset_normalizer-3.4.3-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:585f3b2a80fbd26b048a0be90c5aae8f06605d3c92615911c3a2b03a8a3b796f", size = 156083, upload-time = "2025-08-09T07:55:57.582Z" }, - { url = "https://files.pythonhosted.org/packages/87/df/b7737ff046c974b183ea9aa111b74185ac8c3a326c6262d413bd5a1b8c69/charset_normalizer-3.4.3-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0e78314bdc32fa80696f72fa16dc61168fda4d6a0c014e0380f9d02f0e5d8a07", size = 150295, upload-time = "2025-08-09T07:55:59.147Z" }, - { url = "https://files.pythonhosted.org/packages/61/f1/190d9977e0084d3f1dc169acd060d479bbbc71b90bf3e7bf7b9927dec3eb/charset_normalizer-3.4.3-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:96b2b3d1a83ad55310de8c7b4a2d04d9277d5591f40761274856635acc5fcb30", size = 148379, upload-time = "2025-08-09T07:56:00.364Z" }, - { url = "https://files.pythonhosted.org/packages/4c/92/27dbe365d34c68cfe0ca76f1edd70e8705d82b378cb54ebbaeabc2e3029d/charset_normalizer-3.4.3-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:939578d9d8fd4299220161fdd76e86c6a251987476f5243e8864a7844476ba14", size = 160018, upload-time = "2025-08-09T07:56:01.678Z" }, - { url = "https://files.pythonhosted.org/packages/99/04/baae2a1ea1893a01635d475b9261c889a18fd48393634b6270827869fa34/charset_normalizer-3.4.3-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:fd10de089bcdcd1be95a2f73dbe6254798ec1bda9f450d5828c96f93e2536b9c", size = 157430, upload-time = "2025-08-09T07:56:02.87Z" }, - { url = "https://files.pythonhosted.org/packages/2f/36/77da9c6a328c54d17b960c89eccacfab8271fdaaa228305330915b88afa9/charset_normalizer-3.4.3-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:1e8ac75d72fa3775e0b7cb7e4629cec13b7514d928d15ef8ea06bca03ef01cae", size = 151600, upload-time = "2025-08-09T07:56:04.089Z" }, - { url = "https://files.pythonhosted.org/packages/64/d4/9eb4ff2c167edbbf08cdd28e19078bf195762e9bd63371689cab5ecd3d0d/charset_normalizer-3.4.3-cp311-cp311-win32.whl", hash = "sha256:6cf8fd4c04756b6b60146d98cd8a77d0cdae0e1ca20329da2ac85eed779b6849", size = 99616, upload-time = "2025-08-09T07:56:05.658Z" }, - { url = "https://files.pythonhosted.org/packages/f4/9c/996a4a028222e7761a96634d1820de8a744ff4327a00ada9c8942033089b/charset_normalizer-3.4.3-cp311-cp311-win_amd64.whl", hash = "sha256:31a9a6f775f9bcd865d88ee350f0ffb0e25936a7f930ca98995c05abf1faf21c", size = 107108, upload-time = "2025-08-09T07:56:07.176Z" }, - { url = "https://files.pythonhosted.org/packages/e9/5e/14c94999e418d9b87682734589404a25854d5f5d0408df68bc15b6ff54bb/charset_normalizer-3.4.3-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:e28e334d3ff134e88989d90ba04b47d84382a828c061d0d1027b1b12a62b39b1", size = 205655, upload-time = "2025-08-09T07:56:08.475Z" }, - { url = "https://files.pythonhosted.org/packages/7d/a8/c6ec5d389672521f644505a257f50544c074cf5fc292d5390331cd6fc9c3/charset_normalizer-3.4.3-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0cacf8f7297b0c4fcb74227692ca46b4a5852f8f4f24b3c766dd94a1075c4884", size = 146223, upload-time = "2025-08-09T07:56:09.708Z" }, - { url = "https://files.pythonhosted.org/packages/fc/eb/a2ffb08547f4e1e5415fb69eb7db25932c52a52bed371429648db4d84fb1/charset_normalizer-3.4.3-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:c6fd51128a41297f5409deab284fecbe5305ebd7e5a1f959bee1c054622b7018", size = 159366, upload-time = "2025-08-09T07:56:11.326Z" }, - { url = "https://files.pythonhosted.org/packages/82/10/0fd19f20c624b278dddaf83b8464dcddc2456cb4b02bb902a6da126b87a1/charset_normalizer-3.4.3-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:3cfb2aad70f2c6debfbcb717f23b7eb55febc0bb23dcffc0f076009da10c6392", size = 157104, upload-time = "2025-08-09T07:56:13.014Z" }, - { url = "https://files.pythonhosted.org/packages/16/ab/0233c3231af734f5dfcf0844aa9582d5a1466c985bbed6cedab85af9bfe3/charset_normalizer-3.4.3-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:1606f4a55c0fd363d754049cdf400175ee96c992b1f8018b993941f221221c5f", size = 151830, upload-time = "2025-08-09T07:56:14.428Z" }, - { url = "https://files.pythonhosted.org/packages/ae/02/e29e22b4e02839a0e4a06557b1999d0a47db3567e82989b5bb21f3fbbd9f/charset_normalizer-3.4.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:027b776c26d38b7f15b26a5da1044f376455fb3766df8fc38563b4efbc515154", size = 148854, upload-time = "2025-08-09T07:56:16.051Z" }, - { url = "https://files.pythonhosted.org/packages/05/6b/e2539a0a4be302b481e8cafb5af8792da8093b486885a1ae4d15d452bcec/charset_normalizer-3.4.3-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:42e5088973e56e31e4fa58eb6bd709e42fc03799c11c42929592889a2e54c491", size = 160670, upload-time = "2025-08-09T07:56:17.314Z" }, - { url = "https://files.pythonhosted.org/packages/31/e7/883ee5676a2ef217a40ce0bffcc3d0dfbf9e64cbcfbdf822c52981c3304b/charset_normalizer-3.4.3-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:cc34f233c9e71701040d772aa7490318673aa7164a0efe3172b2981218c26d93", size = 158501, upload-time = "2025-08-09T07:56:18.641Z" }, - { url = "https://files.pythonhosted.org/packages/c1/35/6525b21aa0db614cf8b5792d232021dca3df7f90a1944db934efa5d20bb1/charset_normalizer-3.4.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:320e8e66157cc4e247d9ddca8e21f427efc7a04bbd0ac8a9faf56583fa543f9f", size = 153173, upload-time = "2025-08-09T07:56:20.289Z" }, - { url = "https://files.pythonhosted.org/packages/50/ee/f4704bad8201de513fdc8aac1cabc87e38c5818c93857140e06e772b5892/charset_normalizer-3.4.3-cp312-cp312-win32.whl", hash = "sha256:fb6fecfd65564f208cbf0fba07f107fb661bcd1a7c389edbced3f7a493f70e37", size = 99822, upload-time = "2025-08-09T07:56:21.551Z" }, - { url = "https://files.pythonhosted.org/packages/39/f5/3b3836ca6064d0992c58c7561c6b6eee1b3892e9665d650c803bd5614522/charset_normalizer-3.4.3-cp312-cp312-win_amd64.whl", hash = "sha256:86df271bf921c2ee3818f0522e9a5b8092ca2ad8b065ece5d7d9d0e9f4849bcc", size = 107543, upload-time = "2025-08-09T07:56:23.115Z" }, - { url = "https://files.pythonhosted.org/packages/65/ca/2135ac97709b400c7654b4b764daf5c5567c2da45a30cdd20f9eefe2d658/charset_normalizer-3.4.3-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:14c2a87c65b351109f6abfc424cab3927b3bdece6f706e4d12faaf3d52ee5efe", size = 205326, upload-time = "2025-08-09T07:56:24.721Z" }, - { url = "https://files.pythonhosted.org/packages/71/11/98a04c3c97dd34e49c7d247083af03645ca3730809a5509443f3c37f7c99/charset_normalizer-3.4.3-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:41d1fc408ff5fdfb910200ec0e74abc40387bccb3252f3f27c0676731df2b2c8", size = 146008, upload-time = "2025-08-09T07:56:26.004Z" }, - { url = "https://files.pythonhosted.org/packages/60/f5/4659a4cb3c4ec146bec80c32d8bb16033752574c20b1252ee842a95d1a1e/charset_normalizer-3.4.3-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:1bb60174149316da1c35fa5233681f7c0f9f514509b8e399ab70fea5f17e45c9", size = 159196, upload-time = "2025-08-09T07:56:27.25Z" }, - { url = "https://files.pythonhosted.org/packages/86/9e/f552f7a00611f168b9a5865a1414179b2c6de8235a4fa40189f6f79a1753/charset_normalizer-3.4.3-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:30d006f98569de3459c2fc1f2acde170b7b2bd265dc1943e87e1a4efe1b67c31", size = 156819, upload-time = "2025-08-09T07:56:28.515Z" }, - { url = "https://files.pythonhosted.org/packages/7e/95/42aa2156235cbc8fa61208aded06ef46111c4d3f0de233107b3f38631803/charset_normalizer-3.4.3-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:416175faf02e4b0810f1f38bcb54682878a4af94059a1cd63b8747244420801f", size = 151350, upload-time = "2025-08-09T07:56:29.716Z" }, - { url = "https://files.pythonhosted.org/packages/c2/a9/3865b02c56f300a6f94fc631ef54f0a8a29da74fb45a773dfd3dcd380af7/charset_normalizer-3.4.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:6aab0f181c486f973bc7262a97f5aca3ee7e1437011ef0c2ec04b5a11d16c927", size = 148644, upload-time = "2025-08-09T07:56:30.984Z" }, - { url = "https://files.pythonhosted.org/packages/77/d9/cbcf1a2a5c7d7856f11e7ac2d782aec12bdfea60d104e60e0aa1c97849dc/charset_normalizer-3.4.3-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:fdabf8315679312cfa71302f9bd509ded4f2f263fb5b765cf1433b39106c3cc9", size = 160468, upload-time = "2025-08-09T07:56:32.252Z" }, - { url = "https://files.pythonhosted.org/packages/f6/42/6f45efee8697b89fda4d50580f292b8f7f9306cb2971d4b53f8914e4d890/charset_normalizer-3.4.3-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:bd28b817ea8c70215401f657edef3a8aa83c29d447fb0b622c35403780ba11d5", size = 158187, upload-time = "2025-08-09T07:56:33.481Z" }, - { url = "https://files.pythonhosted.org/packages/70/99/f1c3bdcfaa9c45b3ce96f70b14f070411366fa19549c1d4832c935d8e2c3/charset_normalizer-3.4.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:18343b2d246dc6761a249ba1fb13f9ee9a2bcd95decc767319506056ea4ad4dc", size = 152699, upload-time = "2025-08-09T07:56:34.739Z" }, - { url = "https://files.pythonhosted.org/packages/a3/ad/b0081f2f99a4b194bcbb1934ef3b12aa4d9702ced80a37026b7607c72e58/charset_normalizer-3.4.3-cp313-cp313-win32.whl", hash = "sha256:6fb70de56f1859a3f71261cbe41005f56a7842cc348d3aeb26237560bfa5e0ce", size = 99580, upload-time = "2025-08-09T07:56:35.981Z" }, - { url = "https://files.pythonhosted.org/packages/9a/8f/ae790790c7b64f925e5c953b924aaa42a243fb778fed9e41f147b2a5715a/charset_normalizer-3.4.3-cp313-cp313-win_amd64.whl", hash = "sha256:cf1ebb7d78e1ad8ec2a8c4732c7be2e736f6e5123a4146c5b89c9d1f585f8cef", size = 107366, upload-time = "2025-08-09T07:56:37.339Z" }, - { url = "https://files.pythonhosted.org/packages/8e/91/b5a06ad970ddc7a0e513112d40113e834638f4ca1120eb727a249fb2715e/charset_normalizer-3.4.3-cp314-cp314-macosx_10_13_universal2.whl", hash = "sha256:3cd35b7e8aedeb9e34c41385fda4f73ba609e561faedfae0a9e75e44ac558a15", size = 204342, upload-time = "2025-08-09T07:56:38.687Z" }, - { url = "https://files.pythonhosted.org/packages/ce/ec/1edc30a377f0a02689342f214455c3f6c2fbedd896a1d2f856c002fc3062/charset_normalizer-3.4.3-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b89bc04de1d83006373429975f8ef9e7932534b8cc9ca582e4db7d20d91816db", size = 145995, upload-time = "2025-08-09T07:56:40.048Z" }, - { url = "https://files.pythonhosted.org/packages/17/e5/5e67ab85e6d22b04641acb5399c8684f4d37caf7558a53859f0283a650e9/charset_normalizer-3.4.3-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:2001a39612b241dae17b4687898843f254f8748b796a2e16f1051a17078d991d", size = 158640, upload-time = "2025-08-09T07:56:41.311Z" }, - { url = "https://files.pythonhosted.org/packages/f1/e5/38421987f6c697ee3722981289d554957c4be652f963d71c5e46a262e135/charset_normalizer-3.4.3-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:8dcfc373f888e4fb39a7bc57e93e3b845e7f462dacc008d9749568b1c4ece096", size = 156636, upload-time = "2025-08-09T07:56:43.195Z" }, - { url = "https://files.pythonhosted.org/packages/a0/e4/5a075de8daa3ec0745a9a3b54467e0c2967daaaf2cec04c845f73493e9a1/charset_normalizer-3.4.3-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:18b97b8404387b96cdbd30ad660f6407799126d26a39ca65729162fd810a99aa", size = 150939, upload-time = "2025-08-09T07:56:44.819Z" }, - { url = "https://files.pythonhosted.org/packages/02/f7/3611b32318b30974131db62b4043f335861d4d9b49adc6d57c1149cc49d4/charset_normalizer-3.4.3-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:ccf600859c183d70eb47e05a44cd80a4ce77394d1ac0f79dbd2dd90a69a3a049", size = 148580, upload-time = "2025-08-09T07:56:46.684Z" }, - { url = "https://files.pythonhosted.org/packages/7e/61/19b36f4bd67f2793ab6a99b979b4e4f3d8fc754cbdffb805335df4337126/charset_normalizer-3.4.3-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:53cd68b185d98dde4ad8990e56a58dea83a4162161b1ea9272e5c9182ce415e0", size = 159870, upload-time = "2025-08-09T07:56:47.941Z" }, - { url = "https://files.pythonhosted.org/packages/06/57/84722eefdd338c04cf3030ada66889298eaedf3e7a30a624201e0cbe424a/charset_normalizer-3.4.3-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:30a96e1e1f865f78b030d65241c1ee850cdf422d869e9028e2fc1d5e4db73b92", size = 157797, upload-time = "2025-08-09T07:56:49.756Z" }, - { url = "https://files.pythonhosted.org/packages/72/2a/aff5dd112b2f14bcc3462c312dce5445806bfc8ab3a7328555da95330e4b/charset_normalizer-3.4.3-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:d716a916938e03231e86e43782ca7878fb602a125a91e7acb8b5112e2e96ac16", size = 152224, upload-time = "2025-08-09T07:56:51.369Z" }, - { url = "https://files.pythonhosted.org/packages/b7/8c/9839225320046ed279c6e839d51f028342eb77c91c89b8ef2549f951f3ec/charset_normalizer-3.4.3-cp314-cp314-win32.whl", hash = "sha256:c6dbd0ccdda3a2ba7c2ecd9d77b37f3b5831687d8dc1b6ca5f56a4880cc7b7ce", size = 100086, upload-time = "2025-08-09T07:56:52.722Z" }, - { url = "https://files.pythonhosted.org/packages/ee/7a/36fbcf646e41f710ce0a563c1c9a343c6edf9be80786edeb15b6f62e17db/charset_normalizer-3.4.3-cp314-cp314-win_amd64.whl", hash = "sha256:73dc19b562516fc9bcf6e5d6e596df0b4eb98d87e4f79f3ae71840e6ed21361c", size = 107400, upload-time = "2025-08-09T07:56:55.172Z" }, - { url = "https://files.pythonhosted.org/packages/8a/1f/f041989e93b001bc4e44bb1669ccdcf54d3f00e628229a85b08d330615c5/charset_normalizer-3.4.3-py3-none-any.whl", hash = "sha256:ce571ab16d890d23b5c278547ba694193a45011ff86a9162a71307ed9f86759a", size = 53175, upload-time = "2025-08-09T07:57:26.864Z" }, +version = "3.4.9" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/bd/2a/23f34ec9d04624958e137efdc394888716353190e75f25dd22c7a2c7a8aa/charset_normalizer-3.4.9.tar.gz", hash = "sha256:673611bbd43f0810bec0b0f028ddeaaa501190339cac411f347ac76917c3ae7b", size = 152439, upload-time = "2026-07-07T14:34:58.454Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ad/81/8e983840c6e5b93b33c2ba81aa3d52c2e42f0e9a690ce7607a2e61da4a5c/charset_normalizer-3.4.9-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:cd6280cf040f233bd7d3407b743b4b4c74f70e8e1c4199cb112a62c941c0772a", size = 322240, upload-time = "2026-07-07T14:32:36.236Z" }, + { url = "https://files.pythonhosted.org/packages/de/d1/b4319dc3229d8272fba305e206fc0a148e2de8d4087917ce62ae6382f359/charset_normalizer-3.4.9-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:aa99adc8f081b475a12843953db36831eaf83ec33eb46a90629ca6a5de45a616", size = 216475, upload-time = "2026-07-07T14:32:38.142Z" }, + { url = "https://files.pythonhosted.org/packages/80/33/6c99c1b3e6b8bf730e1bc809b9a2608f224145069114c479a2e9e1494346/charset_normalizer-3.4.9-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:c1225416b463483160e4af85d5fc3a9690ccb53fd4b1865a6437825f5ede3209", size = 238670, upload-time = "2026-07-07T14:32:39.658Z" }, + { url = "https://files.pythonhosted.org/packages/7f/f4/ffbb83546e1f198ecc70ecd372b65cf2b50f9068b380abd67640f17a8e18/charset_normalizer-3.4.9-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:16d10d789dd9bcca1173c95af82c58433122564b7bc39385124be735a35cbe99", size = 233476, upload-time = "2026-07-07T14:32:41.155Z" }, + { url = "https://files.pythonhosted.org/packages/e8/5f/b98b8da398637b551e427e7be922bdec19177dc54d6811dcdaa503f23aac/charset_normalizer-3.4.9-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9bb41182d93ea91f60b4bc8fbf4c820c69ef8a12ab2d917f3f1834f1acad07e8", size = 223817, upload-time = "2026-07-07T14:32:42.592Z" }, + { url = "https://files.pythonhosted.org/packages/36/31/a276bb2e66243072a3fd06fdcab9cbb61a305b02143d70d2bda21d888fa8/charset_normalizer-3.4.9-cp310-cp310-manylinux_2_31_armv7l.whl", hash = "sha256:bcf74c1df76758a395bf0af608c04c82257523f55c9868b334f06270d0f2112b", size = 207974, upload-time = "2026-07-07T14:32:44.258Z" }, + { url = "https://files.pythonhosted.org/packages/5e/be/7ee4453d7e88dfbc4104ccd34900b9f2c7c17dac22881865fe0e82424a25/charset_normalizer-3.4.9-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:b5314963fce9b0b12743891de876e724997864ee22aa496f903f426c7e2fa5b2", size = 221655, upload-time = "2026-07-07T14:32:45.64Z" }, + { url = "https://files.pythonhosted.org/packages/1d/85/181c652953eb5276d198f375b1dd641047392050098100a3a02d6534f657/charset_normalizer-3.4.9-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:e9701d0049d92c16703a42771b98d560b95248949f23f8cf7b4eddd201814fb9", size = 219229, upload-time = "2026-07-07T14:32:47.376Z" }, + { url = "https://files.pythonhosted.org/packages/0c/e7/aaf6da33fc9f4691cda8f7efbc9f69179d3d39ec8a4799baf273ee1d8db0/charset_normalizer-3.4.9-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:65a7ff3f705e57d392f7261b6d0550fe137c3019477431f1c355e0db0a7d3e15", size = 209704, upload-time = "2026-07-07T14:32:48.855Z" }, + { url = "https://files.pythonhosted.org/packages/63/01/f2fb3bd3a73be48b173ee0c6aa8d2497af97d5663a8c4c4b491de4c62f7a/charset_normalizer-3.4.9-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:79580094b00d1789d1f93ea55bc43cb2f611910c72235b7657f3482ddcc1b22d", size = 226243, upload-time = "2026-07-07T14:32:50.239Z" }, + { url = "https://files.pythonhosted.org/packages/c4/02/c57a22739fe05246b0b5783b3bfb6afaac4eebb46f3ececdfb2f048f780e/charset_normalizer-3.4.9-cp310-cp310-win32.whl", hash = "sha256:432786d3561e69aeeae6c7e8648964ce0ad05736120135601f87ac26b9c83381", size = 150935, upload-time = "2026-07-07T14:32:51.676Z" }, + { url = "https://files.pythonhosted.org/packages/37/8d/ca39a7559a4797505530d084fd3a49a2c959efbbbff146302fb7be4e3b35/charset_normalizer-3.4.9-cp310-cp310-win_amd64.whl", hash = "sha256:8c041122946b7ba21bb32c45b1aa57b1be35527690aeb3c5c234521085632eee", size = 162314, upload-time = "2026-07-07T14:32:53.193Z" }, + { url = "https://files.pythonhosted.org/packages/01/da/a44bd7a13d426e69e4894557106cd58669097bfad4a8681123b618fbfc5d/charset_normalizer-3.4.9-cp310-cp310-win_arm64.whl", hash = "sha256:375b83ed0aecfce76c16d198fbc21f3b11b337d68662bea0a995046682a11419", size = 153075, upload-time = "2026-07-07T14:32:54.554Z" }, + { url = "https://files.pythonhosted.org/packages/0b/e3/85ec501f206fb049259288c1f3506e53876937fb00edb47009348e66756b/charset_normalizer-3.4.9-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:0e94703ec9684807f20cfb5eed95c70f67f2a8f21ad620146d7b5a13677b93e5", size = 317075, upload-time = "2026-07-07T14:32:56.021Z" }, + { url = "https://files.pythonhosted.org/packages/c3/69/2a5385192e67175f7d8bd5ce4f57c24bc956439adeae5c13a99aa28a53d1/charset_normalizer-3.4.9-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:2a441ea71902098ffe78c5abe6c494f44160b4af614ed16c3d9a3b1d17fd8ee2", size = 213837, upload-time = "2026-07-07T14:32:57.78Z" }, + { url = "https://files.pythonhosted.org/packages/b3/46/03ddc7da576d814fe0a36dd1f0fd3258e95404b4b2e3c026b7923d7e133f/charset_normalizer-3.4.9-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:304b13570067b2547562e308af560b3963857b1fa90bd6afd978130130fe2d6a", size = 235503, upload-time = "2026-07-07T14:32:59.205Z" }, + { url = "https://files.pythonhosted.org/packages/4e/6e/de0229a7ef40f6f9d28a837eebf4ec47bdca5dab4e900c84f22919af636a/charset_normalizer-3.4.9-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:4773092f8019072343a7447203308b176e10199920eb02d6195e81bbb3274c29", size = 229944, upload-time = "2026-07-07T14:33:00.803Z" }, + { url = "https://files.pythonhosted.org/packages/a5/34/49b9060e8418b14fb5cba9cf6bfb383111e2538a03a1fb18e66a95aeb3d5/charset_normalizer-3.4.9-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:04ce310cb89c15df659582aee80a0603788732a5e017d5bd5c81158106ce249c", size = 221276, upload-time = "2026-07-07T14:33:02.199Z" }, + { url = "https://files.pythonhosted.org/packages/44/95/80282cce0fae9c3061203d723ee87da996aed79679e65d8935050ee7ca1f/charset_normalizer-3.4.9-cp311-cp311-manylinux_2_31_armv7l.whl", hash = "sha256:c0323c9daef75ef2e5083624b4585018a0c9d5e3b40f607eed81a311270b934b", size = 205260, upload-time = "2026-07-07T14:33:03.698Z" }, + { url = "https://files.pythonhosted.org/packages/0c/74/2f62c8821b969ea3bd67cc2e6976834f48ca5d12664d2559ebcd9bcfbed7/charset_normalizer-3.4.9-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:871ff67ea1aad4dfd91736464934d56b32dac49f9fbe16cddba36198a7b3a0db", size = 217786, upload-time = "2026-07-07T14:33:05.12Z" }, + { url = "https://files.pythonhosted.org/packages/d9/8d/feabb82cb49fcad14515b1d7d1ca4787b0da7fc723a212bf89bc9e0fac52/charset_normalizer-3.4.9-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:67830fc78e67501f47bb950471b2dcb9b35b140084429318e862895a8e89c993", size = 216798, upload-time = "2026-07-07T14:33:06.629Z" }, + { url = "https://files.pythonhosted.org/packages/a5/ff/c946d63bc3786d5b84d960b0f7ab7e25b828486a946b5aa997625bcaf6a6/charset_normalizer-3.4.9-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:3d92613ec25e43b05f042302531ec0f00b8445190e43325880cbd6ab7c2581da", size = 206429, upload-time = "2026-07-07T14:33:08.006Z" }, + { url = "https://files.pythonhosted.org/packages/af/ba/5e5007c370702f85d2ef75791fac7943ed41e080364a673b20142e430e3e/charset_normalizer-3.4.9-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:280081916dc341820640489a66e4696049401ef1cf6dd672f672e70ad915aca3", size = 223066, upload-time = "2026-07-07T14:33:09.783Z" }, + { url = "https://files.pythonhosted.org/packages/83/d5/9096aa3cf532dfad237861544eb47a0f20d5adbf1039760fed8eaae935d9/charset_normalizer-3.4.9-cp311-cp311-win32.whl", hash = "sha256:ac351b3b8014eead140e77e9717e2992c6bbe30b63bc3422422eb84865412e3d", size = 150456, upload-time = "2026-07-07T14:33:11.217Z" }, + { url = "https://files.pythonhosted.org/packages/ed/a1/e29995109e455dc8eff8d0fac6ae509be39561318a7cfeac5d33ad029213/charset_normalizer-3.4.9-cp311-cp311-win_amd64.whl", hash = "sha256:6366a16e1a25018694d6a5d784d09b046edc9eac40ea2b54065c3052672516a1", size = 161410, upload-time = "2026-07-07T14:33:12.743Z" }, + { url = "https://files.pythonhosted.org/packages/4f/8d/1569f4d0032d6ba2a4fe4591c35bf87868c600c41a71eb5c2e1ffa8464c2/charset_normalizer-3.4.9-cp311-cp311-win_arm64.whl", hash = "sha256:1d22856ffbe153a602df38e4a5464f0b748a54002e0d69ac6d2ad0a197cc99ec", size = 152649, upload-time = "2026-07-07T14:33:14.173Z" }, + { url = "https://files.pythonhosted.org/packages/70/4a/ecbd131485c07fcdfad54e28946d513e3da22ef3b4bd854dcafae54ec739/charset_normalizer-3.4.9-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:45b0cc4e3556cd875e09102988d1ab8356c998b596c9fced84547c8138b487a0", size = 319300, upload-time = "2026-07-07T14:33:15.666Z" }, + { url = "https://files.pythonhosted.org/packages/ec/96/5d9364e3342d69f3a045e1777bc47c85c383e6e9466d561b33fdb419d1f9/charset_normalizer-3.4.9-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9b2aff1c7b3884512b9512c3eaadd9bab39fb45042ffaaa1dd08ff2b9f8109d9", size = 215802, upload-time = "2026-07-07T14:33:17.031Z" }, + { url = "https://files.pythonhosted.org/packages/4b/4c/5361f9aa7f2cb58d94f2ab831b3d493f69efb1d239654b4744e3c09527cb/charset_normalizer-3.4.9-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:9104ed0bd76a429d46f9ec0dbc9b08ad1d2dcdf2b00a5a0daa1c145329b35b44", size = 237171, upload-time = "2026-07-07T14:33:18.576Z" }, + { url = "https://files.pythonhosted.org/packages/50/78/ce342ca4ff30b2eb49fe6d9578df85974f90c67d294113e94efdd9664cbd/charset_normalizer-3.4.9-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:7b86a2b16095d250c6f58b3d9b2eee6f4147754344f3dab0922f7c9bf7d226c9", size = 233075, upload-time = "2026-07-07T14:33:20.084Z" }, + { url = "https://files.pythonhosted.org/packages/01/c4/4fa4c8b3097a11f3c5f09a35b72ed6855fb1d332469504962ab7bafcc702/charset_normalizer-3.4.9-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:5e226f6218febc71f6c1fc2fafb91c226f75bdc1d8fb12d66823716e891608fd", size = 224256, upload-time = "2026-07-07T14:33:21.747Z" }, + { url = "https://files.pythonhosted.org/packages/87/3a/ad914516df7e358a81aae018caa5e0470ba827fa6d763b1d2e87d920a5f6/charset_normalizer-3.4.9-cp312-cp312-manylinux_2_31_armv7l.whl", hash = "sha256:90c44bc373b7687f6948b693cceaea1348ae0975d7474746559494468e3c1d84", size = 208784, upload-time = "2026-07-07T14:33:23.313Z" }, + { url = "https://files.pythonhosted.org/packages/d7/74/3c12f9755717dfe5c5c87da63f35d765fa0c00382ec26bf23f7fae34f2ba/charset_normalizer-3.4.9-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:9cdef90ae47919cae358d8ab15797a800ed41da7aba5d72419fb510729e2ed4b", size = 219928, upload-time = "2026-07-07T14:33:24.814Z" }, + { url = "https://files.pythonhosted.org/packages/33/9a/895095b83e7907abd6d3d99aad3a38ad0d9686cc186cb0c94c24320fe63e/charset_normalizer-3.4.9-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:60f44ade2cf573dad7a277e6f8ca9a51a21dda572b13bd7d8539bb3cd5dbedde", size = 218489, upload-time = "2026-07-07T14:33:26.42Z" }, + { url = "https://files.pythonhosted.org/packages/a1/34/ef5c05f412f42520d7709b7d3784d19640839eb7366ded1755511585429f/charset_normalizer-3.4.9-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:a1786910334ed46ab1dd73222f2cd1e05c2c3bb39f6dddb4f8b36fc382058a39", size = 210267, upload-time = "2026-07-07T14:33:27.952Z" }, + { url = "https://files.pythonhosted.org/packages/83/dc/9b29fa4412b318bf3bfea985c35d67eb55e04b59a7c3f2237168b0e0be6f/charset_normalizer-3.4.9-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:03d07803992c6c7bbc976327f34b18b6160327fc81cb82c9d504720ac0be3b62", size = 226030, upload-time = "2026-07-07T14:33:29.397Z" }, + { url = "https://files.pythonhosted.org/packages/0e/42/6dbc00b8cd16011691203e33570fa42ed5746599a2e878112d16eab403a3/charset_normalizer-3.4.9-cp312-cp312-win32.whl", hash = "sha256:78841cccf1af7b40f6f716338d50c0902dbe88d9f800b3c973b7a9a0a693a642", size = 151185, upload-time = "2026-07-07T14:33:30.781Z" }, + { url = "https://files.pythonhosted.org/packages/80/cc/f920afd1a23c58ccd53c1d36085a71893a4737ff5e66e0371efab6809850/charset_normalizer-3.4.9-cp312-cp312-win_amd64.whl", hash = "sha256:4b3dac63058cc36820b0dd072f89898604e2d39686fe05321729d00d8ac185a0", size = 162557, upload-time = "2026-07-07T14:33:32.176Z" }, + { url = "https://files.pythonhosted.org/packages/f0/e6/0386d43a261ff4e4b30c5857af7df877254b46bec7b9d1b74b6bf969a90b/charset_normalizer-3.4.9-cp312-cp312-win_arm64.whl", hash = "sha256:78fa18e436a1a0e58dbd7e02fc4473f3f32cceb12df9dfca542d075961c307d2", size = 152665, upload-time = "2026-07-07T14:33:33.711Z" }, + { url = "https://files.pythonhosted.org/packages/b2/06/97ec2aeae780b31d742b6352218b43841a6871e2564578ca522dce4a45c3/charset_normalizer-3.4.9-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:440eede837960000d74978f0eba527be106b5b9aee0daf779d395276ed0b0614", size = 317688, upload-time = "2026-07-07T14:33:35.408Z" }, + { url = "https://files.pythonhosted.org/packages/d0/39/8ff066c672434225f8d25f8b739f992af250944392173dcc88362681c9bf/charset_normalizer-3.4.9-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:21e764fd1e70b6a3e205a0e46f3051701f98a8cb3fad66eeb80e48bb502f8698", size = 214982, upload-time = "2026-07-07T14:33:36.996Z" }, + { url = "https://files.pythonhosted.org/packages/92/8f/3a47a3667c83c2df9483d91644c6c107de3bf8874aa1793da9d3012eb986/charset_normalizer-3.4.9-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:e4fd89cc178bced6ad29cb3e6dd4aa63fa5017c3524dbd0b25998fb64a87cc8b", size = 236460, upload-time = "2026-07-07T14:33:38.536Z" }, + { url = "https://files.pythonhosted.org/packages/f1/60/b22cdbee7e4013dab8b0d7647fc6181120fbbbc8f7025c226d15bd5a47fc/charset_normalizer-3.4.9-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:bd47ba7fc3ca94896759ea0109775132d3e7ab921fbf54038e1bab2e46c313c9", size = 232003, upload-time = "2026-07-07T14:33:40.059Z" }, + { url = "https://files.pythonhosted.org/packages/ea/f8/72eb13dcabe7257035cea8aefd922caad2f110d252bf9f67c4c2ca763aee/charset_normalizer-3.4.9-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:84fd18bcc17526fc2b3c1af7d2b9217d32c9c04448c16ec693b9b4f1985c3d33", size = 223149, upload-time = "2026-07-07T14:33:41.631Z" }, + { url = "https://files.pythonhosted.org/packages/b0/3e/faee8f9de92b14ee1198e9163252bb15efee7301b31256a3b6d9ebfdd0dd/charset_normalizer-3.4.9-cp313-cp313-manylinux_2_31_armv7l.whl", hash = "sha256:5b10cd92fc5c498b35a8635df6d5a100207f88b63a4dc1de7ef9a548e1e2cd63", size = 207901, upload-time = "2026-07-07T14:33:43.209Z" }, + { url = "https://files.pythonhosted.org/packages/3a/25/45f30093ae27dd7b92a793b61882a38685f993700113ca36e0c9c14965e1/charset_normalizer-3.4.9-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:a4fbdde9dd4a9ce5fd52c2b3a347bb50cc89483ef783f1cb00d408c13f7a96c0", size = 219176, upload-time = "2026-07-07T14:33:44.725Z" }, + { url = "https://files.pythonhosted.org/packages/48/18/c8f397329c35e32f6a837e488986f4ae03bd2abebc453b48714991630c2f/charset_normalizer-3.4.9-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:416c229f77e5ea25b3dfd4b582f8d73d7e43c22320302b9ab128a2d3a0b38efe", size = 217356, upload-time = "2026-07-07T14:33:46.192Z" }, + { url = "https://files.pythonhosted.org/packages/86/7e/5ce0bba863470fd1902d5e5843968951bddf38abe4742fc97116ef4598b3/charset_normalizer-3.4.9-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:75286256590a6320cf106a0d28970d3560aad9ee09aa7b34fb40524792436d35", size = 209614, upload-time = "2026-07-07T14:33:47.705Z" }, + { url = "https://files.pythonhosted.org/packages/6c/ef/2473d3c4d869155be4af1191111d59c4d5c4e0173026f7e85b176e23bf65/charset_normalizer-3.4.9-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:69b157c5d3292bcd443faca052f3096f637f1e074b98212a933c074ae23dc3b8", size = 224991, upload-time = "2026-07-07T14:33:49.238Z" }, + { url = "https://files.pythonhosted.org/packages/d0/a3/53ddae3db108a088156aa8ddfafd411ebbc1340f48c5573f697b27f69a39/charset_normalizer-3.4.9-cp313-cp313-win32.whl", hash = "sha256:51307f5c71007673a2bf8232ad973483d281e74cb99c8c5a990af1eefa6277d9", size = 150622, upload-time = "2026-07-07T14:33:50.711Z" }, + { url = "https://files.pythonhosted.org/packages/e8/ef/6953a77c7cf2c2ff9998e6f575ab3e380119f100223381565a4f94c1f836/charset_normalizer-3.4.9-cp313-cp313-win_amd64.whl", hash = "sha256:fe2c7201c642b7c308f1675355ad7ff7b66acfe3541625efe5a3ad38f29d6115", size = 161947, upload-time = "2026-07-07T14:33:52.197Z" }, + { url = "https://files.pythonhosted.org/packages/6e/fb/d560d1d1555debbfe7849d9cac6145c1b537709d79576bf22557ed803b82/charset_normalizer-3.4.9-cp313-cp313-win_arm64.whl", hash = "sha256:611057cc5d5c0afc743ba8be6bd828c17e0aaa8643f9d0a9b9bb7dea80eb8012", size = 152594, upload-time = "2026-07-07T14:33:53.486Z" }, + { url = "https://files.pythonhosted.org/packages/7e/8d/496817fa0944239ecae662dd57ea765cfeaec6a735f9f025d4b7b72e7143/charset_normalizer-3.4.9-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:0327fcd59a935777d83410750c50600ee9571af2846f71ce40f25b13da1ef380", size = 317253, upload-time = "2026-07-07T14:33:54.994Z" }, + { url = "https://files.pythonhosted.org/packages/2b/f9/ef4a69ea338ad3c0deceea0f5f7d2380ae8b52132b06d652cb0d2cd86706/charset_normalizer-3.4.9-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:8a79d9f4d8001473a30c163556b3c3bfebec837495a412dde78b51672f6134f9", size = 215898, upload-time = "2026-07-07T14:33:56.334Z" }, + { url = "https://files.pythonhosted.org/packages/8c/e7/5ddfd76fc061eb52de219658a4aa431cbacadf0a0219c8854f00da50d289/charset_normalizer-3.4.9-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:33bdcc2a32c0a0e861f60841a512c8acc658c87c2ac59d89e3a46dacf7d866e4", size = 236718, upload-time = "2026-07-07T14:33:57.9Z" }, + { url = "https://files.pythonhosted.org/packages/49/ba/768fa3f36048d81c477a0ce61f813bc1454d80917ccfe550abd9f44f5e24/charset_normalizer-3.4.9-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:f840ed6d8ecba8255df8c42b87fadeda98ddfc6eeec05e2dc66e26d46dd6f58a", size = 232519, upload-time = "2026-07-07T14:33:59.811Z" }, + { url = "https://files.pythonhosted.org/packages/f4/c4/b3e049d2aa3766180c78507110543d9d50894cc97f57de543f1be521dcdc/charset_normalizer-3.4.9-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c25fe15c70c59eb7c5ce8c06a1f3fa1da0ecc5ea1e7a5922c40fd2fa9b0d5046", size = 223143, upload-time = "2026-07-07T14:34:01.517Z" }, + { url = "https://files.pythonhosted.org/packages/19/79/55c32d06d76ae4feafe053f061f3e3ab70bcf19f4007797ce8c3efda7830/charset_normalizer-3.4.9-cp314-cp314-manylinux_2_31_armv7l.whl", hash = "sha256:f7fb7d750cfa0a070d2c24e831fd3481019a60dd317ea2b39acbcebc08b6ed81", size = 206742, upload-time = "2026-07-07T14:34:03.04Z" }, + { url = "https://files.pythonhosted.org/packages/10/e0/47c079dd82d217c807479cd59ffd30af56307ea31c108b75758970459ad3/charset_normalizer-3.4.9-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:4d1c96a7a18b9690a4d46df09e3e3382406ae3213727cd1019ebade1c4a81917", size = 219191, upload-time = "2026-07-07T14:34:04.657Z" }, + { url = "https://files.pythonhosted.org/packages/42/ab/b9bc2e77d6b44a7e46ef62ec5cac1c9a6ba7b9135a5d560f002696ec9995/charset_normalizer-3.4.9-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:a4cfde78a9f2880208d16a93b795726a3017d5977e08d1e162a7a31322479c41", size = 218328, upload-time = "2026-07-07T14:34:06.115Z" }, + { url = "https://files.pythonhosted.org/packages/f1/78/c9c71d599f5aa2d42bcdd35cbbd46d7f535351a57e40ff7d8e5a7e219401/charset_normalizer-3.4.9-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:d4d6fcde76f94f5cb9e43e9e9a61f16dacefd228cbbf6f1a09bd9b219a92f1a1", size = 207406, upload-time = "2026-07-07T14:34:07.554Z" }, + { url = "https://files.pythonhosted.org/packages/f6/39/c914445c321a845097ce4f6ac7de9a18228a77b766272125a1ce00d851eb/charset_normalizer-3.4.9-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:898f0e9068ca27d37f8e83a5b962821df851532e6c4a7d615c1c033f9da6eedf", size = 225157, upload-time = "2026-07-07T14:34:09.061Z" }, + { url = "https://files.pythonhosted.org/packages/9b/f2/c0d4b8508565a36bc5c624e88ed297f5b0b1095011034d7f5b83a69908b5/charset_normalizer-3.4.9-cp314-cp314-win32.whl", hash = "sha256:c1c948747b03be832dceed96ca815cef7360de9aa19d37c730f8e3f6101aca48", size = 151095, upload-time = "2026-07-07T14:34:10.901Z" }, + { url = "https://files.pythonhosted.org/packages/49/fd/a1d26144398c67486422a72bf5812cda22cb4ccfcd95a290fb41ceb4b8e2/charset_normalizer-3.4.9-cp314-cp314-win_amd64.whl", hash = "sha256:16b65ea0f2465b6fb52aa22de5eca612aa964ddfec00a912e26f4656cbef890b", size = 162796, upload-time = "2026-07-07T14:34:12.47Z" }, + { url = "https://files.pythonhosted.org/packages/20/95/d75e82f8ce9fd323ebf059c16c9aadefb22a1ecde13b7840b35835e4886c/charset_normalizer-3.4.9-cp314-cp314-win_arm64.whl", hash = "sha256:40a126142a56b2dfc0aacbad1de8310cbf60da7656db0e6b16eebd48e3e93519", size = 153334, upload-time = "2026-07-07T14:34:14.044Z" }, + { url = "https://files.pythonhosted.org/packages/00/5e/17398df3a139985ba9d11ed072531986f408c8fca952835ef1ab1820c02b/charset_normalizer-3.4.9-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:609b3ba8fcc0fb5ab7af00719d0fb6ad0cb518e48e7712d12fd68f1327951198", size = 338848, upload-time = "2026-07-07T14:34:15.688Z" }, + { url = "https://files.pythonhosted.org/packages/cd/91/7253a32e86b7e1d1239b1b36ba6dd0f021a21107ab33054b53119cc083b9/charset_normalizer-3.4.9-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:51447e9aa2684679af07ca5021c3db526e0284347ebf4ffcec1154c3350cfe32", size = 223022, upload-time = "2026-07-07T14:34:17.248Z" }, + { url = "https://files.pythonhosted.org/packages/cb/32/2e64bd2be10e89c61e57ebe6a93fd98ae88eb7ebe414b5121f22c96c69eb/charset_normalizer-3.4.9-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:cc1b0fff8ead343dae06305f954eb8468ba0ec1a97881f42489d198e4ce3c632", size = 241590, upload-time = "2026-07-07T14:34:18.813Z" }, + { url = "https://files.pythonhosted.org/packages/3d/ef/d96ec496cfea0c21db43b0ad03891308b02388d054cc902cf0e5a1ad6a88/charset_normalizer-3.4.9-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:fa36ec09ef71d158186bc79e359ff5fdd6e7996fe8ab638f00d6b93139ba4fcf", size = 239584, upload-time = "2026-07-07T14:34:20.52Z" }, + { url = "https://files.pythonhosted.org/packages/d4/ce/9af95f7876194bd7a14e3dfe4a4de2e0bff02666a3910d72beafd06cc297/charset_normalizer-3.4.9-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:df115d4d83168fdf2cae48ef1ff6d1cb4c466364e30861b37121de0f3bf1b990", size = 230224, upload-time = "2026-07-07T14:34:22.189Z" }, + { url = "https://files.pythonhosted.org/packages/52/94/af74dde74a3996bd959c350709bfe50e297823d70a8c1cbd54b838880863/charset_normalizer-3.4.9-cp314-cp314t-manylinux_2_31_armv7l.whl", hash = "sha256:f86c6358749bd4fda175388691e3ba8c46e24c5347d0afd20f9b7edfc9faf07d", size = 212667, upload-time = "2026-07-07T14:34:23.857Z" }, + { url = "https://files.pythonhosted.org/packages/ee/f0/f1c4fe746c395922961b5916ed1d7d6e7d4c84851d19ed43cc89980ec953/charset_normalizer-3.4.9-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:32286a2c8d167e897177b673176c1e3e00d4057caf5d2b64eef9a3666b03018e", size = 227179, upload-time = "2026-07-07T14:34:25.586Z" }, + { url = "https://files.pythonhosted.org/packages/e4/56/6c745619ac397e8871e2bcd3cea1eec86b877488f33888b3aef5c3ed506e/charset_normalizer-3.4.9-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:83aed2c10721ddd90f68140685391b50811a880af20654c59af6b6c66c40513c", size = 225372, upload-time = "2026-07-07T14:34:27.212Z" }, + { url = "https://files.pythonhosted.org/packages/78/ad/98aae8630ac71f16711968e38a5acfecce41b778bf2f0312851020f565a8/charset_normalizer-3.4.9-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:cd6c3d4b783c556fa00bf540854e42f135e2f256abd29669fcd0da0f2dec79c2", size = 215222, upload-time = "2026-07-07T14:34:28.774Z" }, + { url = "https://files.pythonhosted.org/packages/f7/40/9593d54209765207a7f11073c06494c1721e4ca4a0a426c597679bf7f91e/charset_normalizer-3.4.9-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:ee2f2a527e3c1a6e6411eb4209642e138b544a2d72fe5d0d76daf77b24063534", size = 231958, upload-time = "2026-07-07T14:34:30.345Z" }, + { url = "https://files.pythonhosted.org/packages/b1/27/693ee5e8a18191eb38647360c51cd505013e2bd3b366aa43fd5344c21e3c/charset_normalizer-3.4.9-cp314-cp314t-win32.whl", hash = "sha256:0d861473f743244d349b50f850d10eb87aeb22bbdcc8e64f79273c94af5a8226", size = 155580, upload-time = "2026-07-07T14:34:31.884Z" }, + { url = "https://files.pythonhosted.org/packages/80/3f/bd97d3d9c613013d07cb7733d299385b41df37f0471310f5a73dc359f0b8/charset_normalizer-3.4.9-cp314-cp314t-win_amd64.whl", hash = "sha256:9b8e0f3107e2200b76f6054de99016eac3ee6762713587b36baaa7e4bd2ae177", size = 167620, upload-time = "2026-07-07T14:34:33.438Z" }, + { url = "https://files.pythonhosted.org/packages/3d/c6/eee9dca4439b1061f76373f06ea855678cc4a64c1c3c90b50e479edbb8eb/charset_normalizer-3.4.9-cp314-cp314t-win_arm64.whl", hash = "sha256:19ac87f93086ce37b86e098888555c4b4bc48102279bae3350098c0ed664b501", size = 158037, upload-time = "2026-07-07T14:34:35.018Z" }, + { url = "https://files.pythonhosted.org/packages/98/2b/f97f1c193fb855c345d678f5077d6926034db0722df74c8f057020e05a25/charset_normalizer-3.4.9-py3-none-any.whl", hash = "sha256:68e5f26a1ad57ded6d1cfb85331d1c1a195314756471d97758c48498bb4dcdf5", size = 64538, upload-time = "2026-07-07T14:34:56.993Z" }, +] + +[[package]] +name = "click" +version = "8.4.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "colorama", marker = "sys_platform == 'win32'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/76/d4/81420972a676e8ffea40450d8c8c92943e7218a78fe9b64359836cc9876b/click-8.4.2.tar.gz", hash = "sha256:9a6cea6e60b17ebe0a44c5cc636d94f09bd66142c1cd7d8b4cd731c4917a15f6", size = 338000, upload-time = "2026-06-24T17:45:15.148Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/fb/e2/79c688af8b210d232694e31e59da9f6ec747bae31c3f5946e4e9b98860d5/click-8.4.2-py3-none-any.whl", hash = "sha256:e6f9f66136c816745b9d65817da91d61d957fb16e02e4dcd0552553c5a197b76", size = 119243, upload-time = "2026-06-24T17:45:13.73Z" }, +] + +[[package]] +name = "cloudpickle" +version = "3.1.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/27/fb/576f067976d320f5f0114a8d9fa1215425441bb35627b1993e5afd8111e5/cloudpickle-3.1.2.tar.gz", hash = "sha256:7fda9eb655c9c230dab534f1983763de5835249750e85fbcef43aaa30a9a2414", size = 22330, upload-time = "2025-11-03T09:25:26.604Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/88/39/799be3f2f0f38cc727ee3b4f1445fe6d5e4133064ec2e4115069418a5bb6/cloudpickle-3.1.2-py3-none-any.whl", hash = "sha256:9acb47f6afd73f60dc1df93bb801b472f05ff42fa6c84167d25cb206be1fbf4a", size = 22228, upload-time = "2025-11-03T09:25:25.534Z" }, ] [[package]] @@ -413,81 +464,98 @@ wheels = [ [[package]] name = "cuda-pathfinder" -version = "1.5.5" +version = "1.5.6" source = { registry = "https://pypi.org/simple" } wheels = [ - { url = "https://files.pythonhosted.org/packages/11/c8/26f2e4aae92f11522a96043892ba39a90eac610d5242523aa863212bc1c7/cuda_pathfinder-1.5.5-py3-none-any.whl", hash = "sha256:0228c023f95d1480f143ef5c8922d27a2ab052087a942e81dc289c9eb8f91689", size = 51671, upload-time = "2026-05-27T01:21:25.413Z" }, + { url = "https://files.pythonhosted.org/packages/d2/53/8fc9b0cdc5b7f62746e6a01b85b6461e5ae27f871010a5fcf8fa6950766d/cuda_pathfinder-1.5.6-py3-none-any.whl", hash = "sha256:7e4c07c117b78ba1fb35dac4c444d21f3677b1b1ff56175c53a8e3025c5b43c0", size = 52972, upload-time = "2026-06-30T00:58:04.34Z" }, ] [[package]] name = "cuda-toolkit" -version = "13.0.2" +version = "13.0.3.0" source = { registry = "https://pypi.org/simple" } wheels = [ - { url = "https://files.pythonhosted.org/packages/57/b2/453099f5f3b698d7d0eab38916aac44c7f76229f451709e2eb9db6615dcd/cuda_toolkit-13.0.2-py2.py3-none-any.whl", hash = "sha256:b198824cf2f54003f50d64ada3a0f184b42ca0846c1c94192fa269ecd97a66eb", size = 2364, upload-time = "2025-12-19T23:24:07.328Z" }, + { url = "https://files.pythonhosted.org/packages/d1/c7/a79086a62c98befcdb8349656c6f114e2db3b8b2422f6e25c97a7f2a9a3c/cuda_toolkit-13.0.3.0-py2.py3-none-any.whl", hash = "sha256:d693caaa261214ddd7dbb60d68e71cbed884e68c2be7509778f3051da0b91c3f", size = 2512, upload-time = "2026-04-14T00:50:08.173Z" }, ] [package.optional-dependencies] +cublas = [ + { name = "nvidia-cublas", marker = "(platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') or (platform_machine == 'AMD64' and sys_platform == 'win32')" }, + { name = "nvidia-cuda-nvrtc", marker = "(platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') or (platform_machine == 'AMD64' and sys_platform == 'win32')" }, +] cudart = [ - { name = "nvidia-cuda-runtime", marker = "sys_platform == 'linux' or sys_platform == 'win32'" }, + { name = "nvidia-cuda-runtime", marker = "(platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') or (platform_machine == 'AMD64' and sys_platform == 'win32')" }, ] cufft = [ - { name = "nvidia-cufft", marker = "sys_platform == 'linux' or sys_platform == 'win32'" }, + { name = "nvidia-cufft", marker = "(platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') or (platform_machine == 'AMD64' and sys_platform == 'win32')" }, + { name = "nvidia-nvjitlink", marker = "(platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') or (platform_machine == 'AMD64' and sys_platform == 'win32')" }, ] cufile = [ - { name = "nvidia-cufile", marker = "sys_platform == 'linux'" }, + { name = "nvidia-cufile", marker = "(platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, ] cupti = [ - { name = "nvidia-cuda-cupti", marker = "sys_platform == 'linux' or sys_platform == 'win32'" }, + { name = "nvidia-cuda-cupti", marker = "(platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') or (platform_machine == 'AMD64' and sys_platform == 'win32')" }, ] curand = [ - { name = "nvidia-curand", marker = "sys_platform == 'linux' or sys_platform == 'win32'" }, + { name = "nvidia-curand", marker = "(platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') or (platform_machine == 'AMD64' and sys_platform == 'win32')" }, ] cusolver = [ - { name = "nvidia-cusolver", marker = "sys_platform == 'linux' or sys_platform == 'win32'" }, + { name = "nvidia-cublas", marker = "(platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') or (platform_machine == 'AMD64' and sys_platform == 'win32')" }, + { name = "nvidia-cusolver", marker = "(platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') or (platform_machine == 'AMD64' and sys_platform == 'win32')" }, + { name = "nvidia-cusparse", marker = "(platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') or (platform_machine == 'AMD64' and sys_platform == 'win32')" }, + { name = "nvidia-nvjitlink", marker = "(platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') or (platform_machine == 'AMD64' and sys_platform == 'win32')" }, ] cusparse = [ - { name = "nvidia-cusparse", marker = "sys_platform == 'linux' or sys_platform == 'win32'" }, + { name = "nvidia-cusparse", marker = "(platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') or (platform_machine == 'AMD64' and sys_platform == 'win32')" }, + { name = "nvidia-nvjitlink", marker = "(platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') or (platform_machine == 'AMD64' and sys_platform == 'win32')" }, ] nvjitlink = [ - { name = "nvidia-nvjitlink", marker = "sys_platform == 'linux' or sys_platform == 'win32'" }, + { name = "nvidia-nvjitlink", marker = "(platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') or (platform_machine == 'AMD64' and sys_platform == 'win32')" }, ] nvrtc = [ - { name = "nvidia-cuda-nvrtc", marker = "sys_platform == 'linux' or sys_platform == 'win32'" }, + { name = "nvidia-cuda-nvrtc", marker = "(platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') or (platform_machine == 'AMD64' and sys_platform == 'win32')" }, ] nvtx = [ - { name = "nvidia-nvtx", marker = "sys_platform == 'linux' or sys_platform == 'win32'" }, + { name = "nvidia-nvtx", marker = "(platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') or (platform_machine == 'AMD64' and sys_platform == 'win32')" }, ] [[package]] name = "datafusion" -version = "53.0.0" +version = "54.0.0" source = { registry = "https://pypi.org/simple" } dependencies = [ + { name = "cloudpickle" }, { name = "pyarrow" }, { name = "typing-extensions", marker = "python_full_version < '3.13'" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/58/2b/0f96f12b70839c93930c4e17d767fc32b6c77d548c78784128049e944701/datafusion-53.0.0.tar.gz", hash = "sha256:ba9a5ec06b5453fbd8710d6aeeb515a8bcac4b6c140e254409bb53a5f322ef22", size = 224267, upload-time = "2026-04-13T00:45:02.686Z" } +sdist = { url = "https://files.pythonhosted.org/packages/60/90/886f7e9cf827f07ebd60bd293e54e0a028a50dd49bbaef0ee42aae1981ea/datafusion-54.0.0.tar.gz", hash = "sha256:cfe7e8dfc026efc05824f49b53ad6a72caf5c2d6820759b6212a09e245a427ed", size = 276448, upload-time = "2026-06-29T11:19:34.816Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/af/4c/60e052813d81f1ffe3123ead013dbdd2cf961daa576cb9056cbb80228e6b/datafusion-53.0.0-cp310-abi3-macosx_10_12_x86_64.whl", hash = "sha256:a0bd1a98d736571321416dc4ed361a9d1225da1ec9f6c5fad818d75f547697a7", size = 35774913, upload-time = "2026-04-13T00:44:46.235Z" }, - { url = "https://files.pythonhosted.org/packages/6e/59/beabe5301df3338d8206446cd624079e43bdad46e20377a6336017fb6ccf/datafusion-53.0.0-cp310-abi3-macosx_11_0_arm64.whl", hash = "sha256:ce186a8d2405afd67e11e2fb75715019f16b00d070b8d0da89d8aa61cc74c8b5", size = 32667118, upload-time = "2026-04-13T00:44:50.269Z" }, - { url = "https://files.pythonhosted.org/packages/ae/94/636ab61ade98395daea6e733e225e9c7beef111c7c5b575ac851513e203c/datafusion-53.0.0-cp310-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:288a00a7ef03e2807a4667683f7560efd80d60ed1d41696ac15ca9ded14c8251", size = 35585824, upload-time = "2026-04-13T00:44:53.683Z" }, - { url = "https://files.pythonhosted.org/packages/34/80/b9f4889209af02f8d14bccb0e6f0519c329b072bc4d2595025a1303f144c/datafusion-53.0.0-cp310-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:8fef0004f0161fcfc556c025a7201f9cc3169aa3adb97a86419ebb34182d9efb", size = 38083690, upload-time = "2026-04-13T00:44:57.188Z" }, - { url = "https://files.pythonhosted.org/packages/4b/1a/ea4831fc6aeefedbcf186c9f6a273d507b1787c03cbb905bded7e1149a6a/datafusion-53.0.0-cp310-abi3-win_amd64.whl", hash = "sha256:4c8410f5f659b926677be6c7d443bbc05d825c078c970b7d8cf977ebcf948314", size = 38120687, upload-time = "2026-04-13T00:45:00.633Z" }, + { url = "https://files.pythonhosted.org/packages/46/58/4c5b981e3d9ade32a906c15a4941eef50c9b862781cdc14bf4dff48d026a/datafusion-54.0.0-cp310-abi3-macosx_10_12_x86_64.whl", hash = "sha256:946f55e48b8d523d7b4ac106bdf588b4493c2c66f81877d6952aafeaf7c3ec73", size = 39810553, upload-time = "2026-06-29T11:19:02.1Z" }, + { url = "https://files.pythonhosted.org/packages/66/e5/5e4dbd42ce9a2affb3be90d9ab17cebde1a6f28b0d9fb4b83d612d5c8e42/datafusion-54.0.0-cp310-abi3-macosx_11_0_arm64.whl", hash = "sha256:2a3bf43185c7e43e25242e5fb17b6a11b86bf976434c0bc493fdedbd9a080969", size = 37145255, upload-time = "2026-06-29T11:19:05.491Z" }, + { url = "https://files.pythonhosted.org/packages/c6/5e/dbb9e6e3e5006d34f295d7ac73f1302c8f2df140666402a06e6c55028edb/datafusion-54.0.0-cp310-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:9432bf162381e9282cbc74915b8b773895de18be836f7e3f6d0de4d981f24630", size = 38853856, upload-time = "2026-06-29T11:19:08.732Z" }, + { url = "https://files.pythonhosted.org/packages/a8/81/e69008e3479f4d0134875bc4ae39503bedcd55ca2597e71392c963c651b4/datafusion-54.0.0-cp310-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:3bcd4d213fa74710e75e6e182cc468c2bdbc5ffc74a08c8155d414fbbfa1b3f6", size = 41050149, upload-time = "2026-06-29T11:19:12.108Z" }, + { url = "https://files.pythonhosted.org/packages/61/d4/8ba6e3fe3291c9ccc94b5ca3ec3c1fbcbfbe5ece5ffb965e4550844e2c56/datafusion-54.0.0-cp310-abi3-win_amd64.whl", hash = "sha256:b934e097e1bdca7d5768a81ac1bc4a1812cb459269f8b1a5d892a5d930f18376", size = 43444869, upload-time = "2026-06-29T11:19:15.963Z" }, + { url = "https://files.pythonhosted.org/packages/9d/41/5608323226f21a0fa180823c531dbc0ed270e9b694f299b7647505cb6a06/datafusion-54.0.0-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:c4e79048da82ad89b768bd0be7df39254cd2a0afe2b719d1f129e8a7229af683", size = 39796248, upload-time = "2026-06-29T11:19:19.208Z" }, + { url = "https://files.pythonhosted.org/packages/18/81/392ee323104ab14ca689384723b69e137064a828233c165574f97a74c0e9/datafusion-54.0.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:fe57038003b18e28b90752c1e32b44af74ec4f552a1904aee725e1129a00c447", size = 37153577, upload-time = "2026-06-29T11:19:22.397Z" }, + { url = "https://files.pythonhosted.org/packages/40/c4/ebd5ef5349ecbea7f5f9da76c213581c13e7bbe1b5735c9925b279eeb4eb/datafusion-54.0.0-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:574f642832a106456cfc4f32aa82484c504fc32f4be2b510202bcb579de8e6d1", size = 38849839, upload-time = "2026-06-29T11:19:25.783Z" }, + { url = "https://files.pythonhosted.org/packages/5a/b9/2383d30d317bb913cab97dbf2e6e1d5f37f594860d5c5bc176e025cf7d4a/datafusion-54.0.0-cp314-cp314t-manylinux_2_28_x86_64.whl", hash = "sha256:796fd5683927443c5bc61999d00b9007ef9b5ce107725ea8d241df718860985d", size = 41074623, upload-time = "2026-06-29T11:19:29.119Z" }, + { url = "https://files.pythonhosted.org/packages/35/5c/553fd1107dede0a56727fda7216a7198d41394f2d19697f4fb104cc695ea/datafusion-54.0.0-cp314-cp314t-win_amd64.whl", hash = "sha256:64973c63874ec31670dd97b32b18af7b07fad679cb20d58ed154038e3a5c204e", size = 43438801, upload-time = "2026-06-29T11:19:32.799Z" }, ] [[package]] name = "datasets" -version = "4.1.1" +version = "4.4.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "dill" }, { name = "filelock" }, { name = "fsspec", extra = ["http"] }, + { name = "httpx" }, { name = "huggingface-hub" }, { name = "multiprocess" }, { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, - { name = "numpy", version = "2.3.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" }, + { name = "numpy", version = "2.4.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.11.*'" }, + { name = "numpy", version = "2.5.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.12'" }, { name = "packaging" }, { name = "pandas" }, { name = "pyarrow" }, @@ -496,9 +564,21 @@ dependencies = [ { name = "tqdm" }, { name = "xxhash" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/91/a4/73f8e6ef52c535e1d20d5b2ca83bfe6de399d8b8b8a61ccc8d63d60735aa/datasets-4.1.1.tar.gz", hash = "sha256:7d8d5ba8b12861d2c44bfff9c83484ebfafff1ff553371e5901a8d3aab5450e2", size = 579324, upload-time = "2025-09-18T13:14:27.108Z" } +sdist = { url = "https://files.pythonhosted.org/packages/57/13/f05a80bbbac5f62e492e5e463ec59a4479647ef9c376b1fdfaa4d3ed01cc/datasets-4.4.0.tar.gz", hash = "sha256:0430d39b9f13b53c37afb80c23c7e5d8c6ceccc014c14a14d15fa2b4e8688d2a", size = 585143, upload-time = "2025-11-04T10:36:22.73Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/1e/31/d552336985f747b19f0a852d98ca7a2ef4727ba956b38041cfbda08dde0a/datasets-4.4.0-py3-none-any.whl", hash = "sha256:b7e6d1d48c2e1d3a95d6b378e8fc3d7ab29f24f14ddf505a8d417dd09c692f19", size = 511463, upload-time = "2025-11-04T10:36:20.062Z" }, +] + +[[package]] +name = "deprecated" +version = "1.3.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "wrapt" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/49/85/12f0a49a7c4ffb70572b6c2ef13c90c88fd190debda93b23f026b25f9634/deprecated-1.3.1.tar.gz", hash = "sha256:b1b50e0ff0c1fddaa5708a2c6b0a6588bb09b892825ab2b214ac9ea9d92a5223", size = 2932523, upload-time = "2025-10-30T08:19:02.757Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/f4/c8/09012ac195a0aab58755800d2efdc0e7d5905053509f12cb5d136c911cda/datasets-4.1.1-py3-none-any.whl", hash = "sha256:62e4f6899a36be9ec74a7e759a6951253cc85b3fcfa0a759b0efa8353b149dac", size = 503623, upload-time = "2025-09-18T13:14:25.111Z" }, + { url = "https://files.pythonhosted.org/packages/84/d0/205d54408c08b13550c733c4b85429e7ead111c7f0014309637425520a9a/deprecated-1.3.1-py2.py3-none-any.whl", hash = "sha256:597bfef186b6f60181535a29fbe44865ce137a5079f295b479886c82729d5f3f", size = 11298, upload-time = "2025-10-30T08:19:00.758Z" }, ] [[package]] @@ -512,167 +592,195 @@ wheels = [ [[package]] name = "duckdb" -version = "1.4.0" +version = "1.5.5" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/82/93/adc0d183642fc9a602ca9b97cb16754c84b8c1d92e5b99aec412e0c419a8/duckdb-1.4.0.tar.gz", hash = "sha256:bd5edee8bd5a73b5822f2b390668597b5fcdc2d3292c244d8d933bb87ad6ac4c", size = 18453175, upload-time = "2025-09-16T10:22:41.509Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/0f/4a/b2e17dbe2953481b084f355f162ed319a67ef760e28794c6870058583aec/duckdb-1.4.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:e24e981a6c87e299201694b9bb24fff0beb04ccad399fca6f13072a59814488f", size = 31293005, upload-time = "2025-09-16T10:21:28.296Z" }, - { url = "https://files.pythonhosted.org/packages/a9/89/e34ed03cce7e35b83c1f056126aa4e8e8097eb93e7324463020f85d5cbfa/duckdb-1.4.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:db500ef2c8cb7dc1ca078740ecf1dceaa20d3f5dc5bce269be45d5cff4170c0f", size = 17288207, upload-time = "2025-09-16T10:21:31.129Z" }, - { url = "https://files.pythonhosted.org/packages/f8/17/7ff24799ee98c4dbb177c3ec6c93e38e9513828785c31757c727b47ad71e/duckdb-1.4.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:a65739b8a7106634e6e77d0e110fc5e057b88edc9df6cb1683d499a1e5aa3177", size = 14817523, upload-time = "2025-09-16T10:21:33.397Z" }, - { url = "https://files.pythonhosted.org/packages/fc/ab/7a482a76ff75212b5cf4f2172a802f2a59b4ab096416e5821aa62a305bc4/duckdb-1.4.0-cp310-cp310-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1d59f7be24862adb803a1ddfc9c3b8cb09e6005bca0c9c6f7c631a1da1c3aa0c", size = 18410654, upload-time = "2025-09-16T10:21:35.864Z" }, - { url = "https://files.pythonhosted.org/packages/1e/f6/a235233b973652b31448b6d600604620d02fc552b90ab94ca7f645fd5ac0/duckdb-1.4.0-cp310-cp310-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7d052a87e9edf4eb3bab0b7a6ac995676018c6083b8049421628dfa3b983a2d4", size = 20399121, upload-time = "2025-09-16T10:21:38.524Z" }, - { url = "https://files.pythonhosted.org/packages/b1/cf/63fedb74d00d7c4e19ffc73a1d8d98ee8d3d6498cf2865509c104aa8e799/duckdb-1.4.0-cp310-cp310-win_amd64.whl", hash = "sha256:0329b81e587f745b2fc6f3a488ea3188b0f029c3b5feef43792a25eaac84ac01", size = 12283288, upload-time = "2025-09-16T10:21:40.732Z" }, - { url = "https://files.pythonhosted.org/packages/60/e9/b29cc5bceac52e049b20d613551a2171a092df07f26d4315f3f9651c80d4/duckdb-1.4.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:6505fed1ccae8df9f574e744c48fa32ee2feaeebe5346c2daf4d4d10a8dac5aa", size = 31290878, upload-time = "2025-09-16T10:21:43.256Z" }, - { url = "https://files.pythonhosted.org/packages/1f/68/d88a15dba48bf6a4b33f1be5097ef45c83f7b9e97c854cc638a85bb07d70/duckdb-1.4.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:36974a04b29c74ac2143457e95420a7422016d050e28573060b89a90b9cf2b57", size = 17288823, upload-time = "2025-09-16T10:21:45.716Z" }, - { url = "https://files.pythonhosted.org/packages/8c/7e/e3d2101dc6bbd60f2b3c1d748351ff541fc8c48790ac1218c0199cb930f6/duckdb-1.4.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:90484b896e5059f145d1facfabea38e22c54a2dcc2bd62dd6c290423f0aee258", size = 14819684, upload-time = "2025-09-16T10:21:48.117Z" }, - { url = "https://files.pythonhosted.org/packages/c4/bb/4ec8e4d03cb5b77d75b9ee0057c2c714cffaa9bda1e55ffec833458af0a3/duckdb-1.4.0-cp311-cp311-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a969d624b385853b31a43b0a23089683297da2f14846243921c6dbec8382d659", size = 18410075, upload-time = "2025-09-16T10:21:50.517Z" }, - { url = "https://files.pythonhosted.org/packages/ec/21/e896616d892d50dc1e0c142428e9359b483d4dd6e339231d822e57834ad3/duckdb-1.4.0-cp311-cp311-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:5935644f96a75e9f6f3c3eeb3da14cdcaf7bad14d1199c08439103decb29466a", size = 20402984, upload-time = "2025-09-16T10:21:52.808Z" }, - { url = "https://files.pythonhosted.org/packages/c4/c0/b5eb9497e4a9167d23fbad745969eaa36e28d346648e17565471892d1b33/duckdb-1.4.0-cp311-cp311-win_amd64.whl", hash = "sha256:300aa0e963af97969c38440877fffd576fc1f49c1f5914789a9d01f2fe7def91", size = 12282971, upload-time = "2025-09-16T10:21:55.314Z" }, - { url = "https://files.pythonhosted.org/packages/e8/6d/0c774d6af1aed82dbe855d266cb000a1c09ea31ed7d6c3a79e2167a38e7a/duckdb-1.4.0-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:18b3a048fca6cc7bafe08b10e1b0ab1509d7a0381ffb2c70359e7dc56d8a705d", size = 31307425, upload-time = "2025-09-16T10:21:57.83Z" }, - { url = "https://files.pythonhosted.org/packages/d3/c0/1fd7b7b2c0c53d8d748d2f28ea9096df5ee9dc39fa736cca68acabe69656/duckdb-1.4.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:2c1271cb85aeacccfd0b1284e816280a7450df1dd4dd85ccb2848563cfdf90e9", size = 17295727, upload-time = "2025-09-16T10:22:02.242Z" }, - { url = "https://files.pythonhosted.org/packages/98/d3/4d4c4bd667b7ada5f6c207c2f127591ebb8468333f207f8f10ff0532578e/duckdb-1.4.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:55064dd2e25711eeaa6a72c25405bdd7994c81a3221657e94309a2faf65d25a6", size = 14826879, upload-time = "2025-09-16T10:22:05.162Z" }, - { url = "https://files.pythonhosted.org/packages/b0/48/e0c1b97d76fb7567c53db5739931323238fad54a642707008104f501db37/duckdb-1.4.0-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0536d7c81bc506532daccf373ddbc8c6add46aeb70ef3cd5ee70ad5c2b3165ea", size = 18417856, upload-time = "2025-09-16T10:22:07.919Z" }, - { url = "https://files.pythonhosted.org/packages/12/78/297b838f3b9511589badc8f472f70b31cf3bbf9eb99fa0a4d6e911d3114a/duckdb-1.4.0-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:784554e3ddfcfc5c5c7b1aa1f9925fedb7938f6628729adba48f7ea37554598f", size = 20427154, upload-time = "2025-09-16T10:22:10.216Z" }, - { url = "https://files.pythonhosted.org/packages/ea/57/500d251b886494f6c52d56eeab8a1860572ee62aed05d7d50c71ba2320f3/duckdb-1.4.0-cp312-cp312-win_amd64.whl", hash = "sha256:c5d2aa4d6981f525ada95e6db41bb929403632bb5ff24bd6d6dd551662b1b613", size = 12290108, upload-time = "2025-09-16T10:22:12.668Z" }, - { url = "https://files.pythonhosted.org/packages/2f/64/ee22b2b8572746e1523143b9f28d606575782e0204de5020656a1d15dd14/duckdb-1.4.0-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:1d94d010a09b1a62d9021a2a71cf266188750f3c9b1912ccd6afe104a6ce8010", size = 31307662, upload-time = "2025-09-16T10:22:14.9Z" }, - { url = "https://files.pythonhosted.org/packages/76/2e/4241cd00046ca6b781bd1d9002e8223af061e85d1cc21830aa63e7a7db7c/duckdb-1.4.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:c61756fa8b3374627e5fa964b8e0d5b58e364dce59b87dba7fb7bc6ede196b26", size = 17295617, upload-time = "2025-09-16T10:22:17.239Z" }, - { url = "https://files.pythonhosted.org/packages/f7/98/5ab136bc7b12ac18580350a220db7c00606be9eac2d89de259cce733f64c/duckdb-1.4.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:e70d7d9881ea2c0836695de70ea68c970e18a2856ba3d6502e276c85bd414ae7", size = 14826727, upload-time = "2025-09-16T10:22:19.415Z" }, - { url = "https://files.pythonhosted.org/packages/23/32/57866cf8881288b3dfb9212720221fb890daaa534dbdc6fe3fff3979ecd1/duckdb-1.4.0-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:2de258a93435c977a0ec3a74ec8f60c2f215ddc73d427ee49adc4119558facd3", size = 18421289, upload-time = "2025-09-16T10:22:21.564Z" }, - { url = "https://files.pythonhosted.org/packages/a0/83/7438fb43be451a7d4a04650aaaf662b2ff2d95895bbffe3e0e28cbe030c9/duckdb-1.4.0-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a6d3659641d517dd9ed1ab66f110cdbdaa6900106f116effaf2dbedd83c38de3", size = 20426547, upload-time = "2025-09-16T10:22:23.759Z" }, - { url = "https://files.pythonhosted.org/packages/21/b2/98fb89ae81611855f35984e96f648d871f3967bb3f524b51d1372d052f0c/duckdb-1.4.0-cp313-cp313-win_amd64.whl", hash = "sha256:07fcc612ea5f0fe6032b92bcc93693034eb00e7a23eb9146576911d5326af4f7", size = 12290467, upload-time = "2025-09-16T10:22:25.923Z" }, +sdist = { url = "https://files.pythonhosted.org/packages/7d/19/e57151753576373c6696a12022648546cca6038e8833fda2908ee2342d9b/duckdb-1.5.5.tar.gz", hash = "sha256:72f33ee57ca7595b23957671a2cc7f7fe2be0ecc2d68f63abedcfcaa3a5c1238", size = 18066741, upload-time = "2026-07-22T10:55:17.819Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/52/d4/298acf9331a80b3ce6ac64dd940e7e13f4058fb69d18914445f02e3c7bfe/duckdb-1.5.5-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:3b805507f88171b428b21c966c30e9a3d54e30b24528918a44ed0032542bc26f", size = 32702934, upload-time = "2026-07-22T10:53:19.069Z" }, + { url = "https://files.pythonhosted.org/packages/d5/90/c489fb63d64b2e7ee109ce8460bdede003a0f256e5b41a03a2a1c4764058/duckdb-1.5.5-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:b08e19cc856220d8a26fa62abc2264b349aff67255e9373c6a3f607addd56dc6", size = 17343604, upload-time = "2026-07-22T10:53:22.767Z" }, + { url = "https://files.pythonhosted.org/packages/0b/27/effa80a15b1f0c61c235622f797868485359e8c9ad6a8e358e7a0c479151/duckdb-1.5.5-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:a17e6a922e42a5c06ed2353fe78c5dff2610f6632d603836f9606ad0bf754079", size = 15488179, upload-time = "2026-07-22T10:53:25.945Z" }, + { url = "https://files.pythonhosted.org/packages/5d/07/21212345c8d24ba62dceaa20be3b21f5c46f1510b1b42ce93bb058afe0c4/duckdb-1.5.5-cp310-cp310-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1bdc38922c365c37720149f90d90b1e9823eb82dad6830855b5f87537fa6fc0c", size = 19367323, upload-time = "2026-07-22T10:53:30.23Z" }, + { url = "https://files.pythonhosted.org/packages/3f/d0/10371ae875fb4b5ef61bb892743b4b2e90c512b371fdf29317deb744857d/duckdb-1.5.5-cp310-cp310-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e238060db5ca59879882a6e9b015e2c65d5c64ddf281ba1d7a9a2033764152cf", size = 21476568, upload-time = "2026-07-22T10:53:33.486Z" }, + { url = "https://files.pythonhosted.org/packages/b2/35/09568ce617dd7bc0757b3d7b6a981660b9e4f0b7594de8ed776755eae740/duckdb-1.5.5-cp310-cp310-win_amd64.whl", hash = "sha256:4acc72798ba1885a9c17d1242903d2cd502f13b1271c7677f7cab25d8578eceb", size = 13156129, upload-time = "2026-07-22T10:53:37.55Z" }, + { url = "https://files.pythonhosted.org/packages/9c/c2/b62ec24d57bb8df4e24b0b58f7f8facb32f5fdb9f1895aed9e9fcdded168/duckdb-1.5.5-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:1b543841b0ae18a9c982345cfa3987e9c065d3a4b0f067daa473d92d1e65f528", size = 32708371, upload-time = "2026-07-22T10:53:41.642Z" }, + { url = "https://files.pythonhosted.org/packages/8a/ce/769171ba45f0b73632dc3bc3108d891e81dd6c6bbfba630a34a75b4dcc0f/duckdb-1.5.5-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:1a925d06c2a4c3b64553d6cc1aced5028d376d4479bed689a7d47e9b1dccd80a", size = 17343979, upload-time = "2026-07-22T10:53:44.951Z" }, + { url = "https://files.pythonhosted.org/packages/46/59/a8e3384ee916e00d5dcf985194c1511d61978540778a1e96fa47f9fb3e0d/duckdb-1.5.5-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:0c42757cb34722144bd4dfb94b6f336339e7b2468f6813fa7fa9a319ba07bab4", size = 15493704, upload-time = "2026-07-22T10:53:47.912Z" }, + { url = "https://files.pythonhosted.org/packages/6f/1d/9840179c2607b90523a2884a129c4d4e6dbdc1178ba62a976c1043beba88/duckdb-1.5.5-cp311-cp311-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:2e72f9e1a4f90a5c8483ad4d540e495bf0834ba61c360b52499a573d7ed62a3f", size = 19366574, upload-time = "2026-07-22T10:53:51.876Z" }, + { url = "https://files.pythonhosted.org/packages/b5/55/f9641a4eebcc2f4df631287d6c3b9ed2eea3b92644f93acbad825e3972b6/duckdb-1.5.5-cp311-cp311-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b9b6f86ed85d4ef5e0211eaebf75d057bd8bb520bba438a95dd0f4e42234bbfe", size = 21477952, upload-time = "2026-07-22T10:53:55.575Z" }, + { url = "https://files.pythonhosted.org/packages/3c/3a/07c3556e37a5c97b95917b029c8fdde4a25fbd76a660bacdac195cf20dcb/duckdb-1.5.5-cp311-cp311-win_amd64.whl", hash = "sha256:9f4287f97ccf0c1f3d471e7115be2b067cbf99627e2d34bffd462dd64703cddc", size = 13156986, upload-time = "2026-07-22T10:53:58.823Z" }, + { url = "https://files.pythonhosted.org/packages/4f/ff/07b48eef2078ca033847e9caa46cc7633b714c5f91ad1ce091c8ca89d792/duckdb-1.5.5-cp311-cp311-win_arm64.whl", hash = "sha256:179633a3fc6296c75d57c69c1e239fa9e5cdcb670fd1dbff88a02663f932905c", size = 14001317, upload-time = "2026-07-22T10:54:01.724Z" }, + { url = "https://files.pythonhosted.org/packages/d6/40/2e05d324400fdaa5656c9f48d6298da421cb034d85e509fa0e6e325cf04b/duckdb-1.5.5-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:d4dd65f8941a604b947e0b9b4b4f7165988e29a23ec0b69b4038520956d9933e", size = 32753858, upload-time = "2026-07-22T10:54:05.514Z" }, + { url = "https://files.pythonhosted.org/packages/79/15/5ceb58ffb5bb8a62b3fd7abb39c41467cdf94850ece02e6d88664dfc75ce/duckdb-1.5.5-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:33db46679b071f108d57139493dee2d37e1f5efcf5c5c039c2969eed11a6c8a7", size = 17368293, upload-time = "2026-07-22T10:54:09.139Z" }, + { url = "https://files.pythonhosted.org/packages/bf/5c/bf02da0b354fe83cca4f95a4fbf762181af466f7d551ab2a093f7698882a/duckdb-1.5.5-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:f0b88535a5d86fdd63dba6ea02ab68c003dfb9e4892b11256ef24c4da208baae", size = 15509131, upload-time = "2026-07-22T10:54:12.228Z" }, + { url = "https://files.pythonhosted.org/packages/ea/a9/5f1f09da421d8e930e0b063d11c1b3f90363f40ede74438cd188afdd13a2/duckdb-1.5.5-cp312-cp312-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f316eae2323d9a851883fdf2dee91c1f9efe251ab33e14a2272f82a913422ed6", size = 19391959, upload-time = "2026-07-22T10:54:15.551Z" }, + { url = "https://files.pythonhosted.org/packages/4f/98/6549769f158126fa64fd6c1ac2eb59a18282146c939867a3eb31b7c1db07/duckdb-1.5.5-cp312-cp312-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7a6d2d11859d82a936ebdcb30ce3d8a1cbb3e990bff05c12abb9b54c44fa7bd1", size = 21510909, upload-time = "2026-07-22T10:54:19.681Z" }, + { url = "https://files.pythonhosted.org/packages/af/b7/5753b41d3124838f868f9f523362812d9fc45409e9e4dd70dcbb0a25826e/duckdb-1.5.5-cp312-cp312-win_amd64.whl", hash = "sha256:ddfbdb096c11d51ee22492397d342c90a82e62c5d09961477895934d0a25372f", size = 13168544, upload-time = "2026-07-22T10:54:22.789Z" }, + { url = "https://files.pythonhosted.org/packages/5c/28/44b679c7d46245f8398feae7edac959d1b83d4eb143e25b3fce0630b78bd/duckdb-1.5.5-cp312-cp312-win_arm64.whl", hash = "sha256:2725d2b9ace3a4e75d72fc5a239f6a44b502c580edadb8fb2676db772c5f9282", size = 13988684, upload-time = "2026-07-22T10:54:26.003Z" }, + { url = "https://files.pythonhosted.org/packages/47/37/4a38116e7700720fd152c666292214fd3abdf916496991296d8d1f66efbf/duckdb-1.5.5-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:cd98829b67788609017e65c761bd42a5dd0f9129441bed8bda4d6881ccf819f0", size = 32754294, upload-time = "2026-07-22T10:54:29.822Z" }, + { url = "https://files.pythonhosted.org/packages/66/42/7d392f1ba1eee0eaf4ab4c8c7a604bfe3536cd63f979cf5c98798664f807/duckdb-1.5.5-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:feead93c56679b79592d437c62975d39cb67adedffa7592c763baf8160ac7366", size = 17368211, upload-time = "2026-07-22T10:54:33.359Z" }, + { url = "https://files.pythonhosted.org/packages/9f/a5/0a6f4fa60562faa615e55e15bd1953a2f2b17a8edd8105e5cda215e43457/duckdb-1.5.5-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:49c963d9469373d7aba8d750d9ea565ab823e94166efed953f184dd9b169b98c", size = 15509136, upload-time = "2026-07-22T10:54:36.369Z" }, + { url = "https://files.pythonhosted.org/packages/e4/cb/023c89f51978545b9fab318581bba0c457a58e7530d2d933e54ae7d8647c/duckdb-1.5.5-cp313-cp313-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a736217825461732b5442d05a220f3da2e23a0dae114efbf08c9bf171b53098a", size = 19392147, upload-time = "2026-07-22T10:54:39.551Z" }, + { url = "https://files.pythonhosted.org/packages/3e/c5/41bef391fb8b23dbc133c9f2ba016e7a7a8124513d2cc1b430f1897d87e4/duckdb-1.5.5-cp313-cp313-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:078e6a60dd8eedde5832f45422ca5c4a6b8c837aeabd8a56ca0b7d933f588053", size = 21511060, upload-time = "2026-07-22T10:54:42.788Z" }, + { url = "https://files.pythonhosted.org/packages/07/9f/c44dfc1f924ac29b3252dc1b91393c01d009dbfe9f8ed33f10b986151bd1/duckdb-1.5.5-cp313-cp313-win_amd64.whl", hash = "sha256:6826504277dba513c0c5d71d828456c94d729c9d2482f94b2e289f90a9167e28", size = 13168028, upload-time = "2026-07-22T10:54:46.127Z" }, + { url = "https://files.pythonhosted.org/packages/ca/88/591384b2cd59abddd6f5dc175e60374f9abae6064429f0c4402854c10f44/duckdb-1.5.5-cp313-cp313-win_arm64.whl", hash = "sha256:baa9c5702002fabb559ded2a39008f9f421fcbc7237d388b8213eff1e08858de", size = 13989955, upload-time = "2026-07-22T10:54:49.262Z" }, + { url = "https://files.pythonhosted.org/packages/3e/56/12c65bfa2d2605b81981b264788891bcf11ec72227889554cead5d8d13b9/duckdb-1.5.5-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:8e6413dd40facb7b8ab21bd844450cd8f549b29e138635be9cf090ef4d2049e2", size = 32761946, upload-time = "2026-07-22T10:54:53.412Z" }, + { url = "https://files.pythonhosted.org/packages/b9/46/682ce155f17e0d2822d4f13ee3db9ca4b5b7c2da61b841b2629035e1f4bc/duckdb-1.5.5-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:64078acfd16541132ac6e191eb81b2845554444a0305cc1aa581ba107e514aa8", size = 17375069, upload-time = "2026-07-22T10:54:57.269Z" }, + { url = "https://files.pythonhosted.org/packages/39/ce/a24bcbd3289c8f305a430759c5fc12242740b4af3e17f7593f3a34e333d2/duckdb-1.5.5-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:8c11775cc99a447618d5f1840126db17f2652f3eae05529df4f81f40e2df7151", size = 15519791, upload-time = "2026-07-22T10:55:00.681Z" }, + { url = "https://files.pythonhosted.org/packages/d9/76/3a01afbc615c1d418c0de58a6b68ac5ce2a8563232c0464bfbc2ce552398/duckdb-1.5.5-cp314-cp314-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:77bbc1e6ba12e1e06f9020117bdf848627ecfdf36f907550e62e008e6109dece", size = 19398251, upload-time = "2026-07-22T10:55:04.168Z" }, + { url = "https://files.pythonhosted.org/packages/a1/43/3a5e81d1728f4d234c79bfe385808ee7c04834f7c37a4b5c257459c25614/duckdb-1.5.5-cp314-cp314-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:fbf0f2d48b43c6c304d00463b463c27ead6c4b01c3c1816b750f728decf71afe", size = 21513851, upload-time = "2026-07-22T10:55:07.864Z" }, + { url = "https://files.pythonhosted.org/packages/91/41/fc7c829172c60ca22485251eab285f4f1a0d87b486a024c726f21471d86e/duckdb-1.5.5-cp314-cp314-win_amd64.whl", hash = "sha256:9dc826c4b50e64f6c4e4d07a3a9cb075ef70ba3899dc43ec5493dc3d7b04b353", size = 13691858, upload-time = "2026-07-22T10:55:11.181Z" }, + { url = "https://files.pythonhosted.org/packages/e1/2c/95d9216b79e9273689d7ebce125a54503ed0c9bd7da931f0265888e99779/duckdb-1.5.5-cp314-cp314-win_arm64.whl", hash = "sha256:63e48d4b74b15aeacd688976432a7225163df8c226eddeb8536bba2d4d4ff433", size = 14470180, upload-time = "2026-07-22T10:55:14.445Z" }, ] [[package]] name = "exceptiongroup" -version = "1.3.0" +version = "1.3.1" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "typing-extensions", marker = "python_full_version < '3.11'" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/0b/9f/a65090624ecf468cdca03533906e7c69ed7588582240cfe7cc9e770b50eb/exceptiongroup-1.3.0.tar.gz", hash = "sha256:b241f5885f560bc56a59ee63ca4c6a8bfa46ae4ad651af316d4e81817bb9fd88", size = 29749, upload-time = "2025-05-10T17:42:51.123Z" } +sdist = { url = "https://files.pythonhosted.org/packages/50/79/66800aadf48771f6b62f7eb014e352e5d06856655206165d775e675a02c9/exceptiongroup-1.3.1.tar.gz", hash = "sha256:8b412432c6055b0b7d14c310000ae93352ed6754f70fa8f7c34141f91c4e3219", size = 30371, upload-time = "2025-11-21T23:01:54.787Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/36/f4/c6e662dade71f56cd2f3735141b265c3c79293c109549c1e6933b0651ffc/exceptiongroup-1.3.0-py3-none-any.whl", hash = "sha256:4d111e6e0c13d0644cad6ddaa7ed0261a0b36971f6d23e7ec9b4b9097da78a10", size = 16674, upload-time = "2025-05-10T17:42:49.33Z" }, + { url = "https://files.pythonhosted.org/packages/8a/0e/97c33bf5009bdbac74fd2beace167cab3f978feb69cc36f1ef79360d6c4e/exceptiongroup-1.3.1-py3-none-any.whl", hash = "sha256:a7a39a3bd276781e98394987d3a5701d0c4edffb633bb7a5144577f82c773598", size = 16740, upload-time = "2025-11-21T23:01:53.443Z" }, ] [[package]] name = "filelock" -version = "3.19.1" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/40/bb/0ab3e58d22305b6f5440629d20683af28959bf793d98d11950e305c1c326/filelock-3.19.1.tar.gz", hash = "sha256:66eda1888b0171c998b35be2bcc0f6d75c388a7ce20c3f3f37aa8e96c2dddf58", size = 17687, upload-time = "2025-08-14T16:56:03.016Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/42/14/42b2651a2f46b022ccd948bca9f2d5af0fd8929c4eec235b8d6d844fbe67/filelock-3.19.1-py3-none-any.whl", hash = "sha256:d38e30481def20772f5baf097c122c3babc4fcdb7e14e57049eb9d88c6dc017d", size = 15988, upload-time = "2025-08-14T16:56:01.633Z" }, -] - -[[package]] -name = "flatbuffers" -version = "25.9.23" +version = "3.29.7" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/9d/1f/3ee70b0a55137442038f2a33469cc5fddd7e0ad2abf83d7497c18a2b6923/flatbuffers-25.9.23.tar.gz", hash = "sha256:676f9fa62750bb50cf531b42a0a2a118ad8f7f797a511eda12881c016f093b12", size = 22067, upload-time = "2025-09-24T05:25:30.106Z" } +sdist = { url = "https://files.pythonhosted.org/packages/35/94/00f2059e4835eace3ae8fde680b932c496f8ec7bdc99168dfa53fb2e6b79/filelock-3.29.7.tar.gz", hash = "sha256:5b481979797ae69e72f0b389d89a80bdd585c260c5b3f1fb9c0a5ba9bb3f195d", size = 71521, upload-time = "2026-07-08T05:46:58.716Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/ee/1b/00a78aa2e8fbd63f9af08c9c19e6deb3d5d66b4dda677a0f61654680ee89/flatbuffers-25.9.23-py2.py3-none-any.whl", hash = "sha256:255538574d6cb6d0a79a17ec8bc0d30985913b87513a01cce8bcdb6b4c44d0e2", size = 30869, upload-time = "2025-09-24T05:25:28.912Z" }, + { url = "https://files.pythonhosted.org/packages/60/02/be4a57b60c7149b55b9e3b3c13f609cd8eb5307c751f22bd8fb8d262e75b/filelock-3.29.7-py3-none-any.whl", hash = "sha256:987db6f789a3a2a59f55081801b2b3697cb97e2a736b5f1a9e99b559285fbc51", size = 46036, upload-time = "2026-07-08T05:46:57.53Z" }, ] [[package]] name = "frozenlist" -version = "1.7.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/79/b1/b64018016eeb087db503b038296fd782586432b9c077fc5c7839e9cb6ef6/frozenlist-1.7.0.tar.gz", hash = "sha256:2e310d81923c2437ea8670467121cc3e9b0f76d3043cc1d2331d56c7fb7a3a8f", size = 45078, upload-time = "2025-06-09T23:02:35.538Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/af/36/0da0a49409f6b47cc2d060dc8c9040b897b5902a8a4e37d9bc1deb11f680/frozenlist-1.7.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:cc4df77d638aa2ed703b878dd093725b72a824c3c546c076e8fdf276f78ee84a", size = 81304, upload-time = "2025-06-09T22:59:46.226Z" }, - { url = "https://files.pythonhosted.org/packages/77/f0/77c11d13d39513b298e267b22eb6cb559c103d56f155aa9a49097221f0b6/frozenlist-1.7.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:716a9973a2cc963160394f701964fe25012600f3d311f60c790400b00e568b61", size = 47735, upload-time = "2025-06-09T22:59:48.133Z" }, - { url = "https://files.pythonhosted.org/packages/37/12/9d07fa18971a44150593de56b2f2947c46604819976784bcf6ea0d5db43b/frozenlist-1.7.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:a0fd1bad056a3600047fb9462cff4c5322cebc59ebf5d0a3725e0ee78955001d", size = 46775, upload-time = "2025-06-09T22:59:49.564Z" }, - { url = "https://files.pythonhosted.org/packages/70/34/f73539227e06288fcd1f8a76853e755b2b48bca6747e99e283111c18bcd4/frozenlist-1.7.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3789ebc19cb811163e70fe2bd354cea097254ce6e707ae42e56f45e31e96cb8e", size = 224644, upload-time = "2025-06-09T22:59:51.35Z" }, - { url = "https://files.pythonhosted.org/packages/fb/68/c1d9c2f4a6e438e14613bad0f2973567586610cc22dcb1e1241da71de9d3/frozenlist-1.7.0-cp310-cp310-manylinux_2_17_armv7l.manylinux2014_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:af369aa35ee34f132fcfad5be45fbfcde0e3a5f6a1ec0712857f286b7d20cca9", size = 222125, upload-time = "2025-06-09T22:59:52.884Z" }, - { url = "https://files.pythonhosted.org/packages/b9/d0/98e8f9a515228d708344d7c6986752be3e3192d1795f748c24bcf154ad99/frozenlist-1.7.0-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ac64b6478722eeb7a3313d494f8342ef3478dff539d17002f849101b212ef97c", size = 233455, upload-time = "2025-06-09T22:59:54.74Z" }, - { url = "https://files.pythonhosted.org/packages/79/df/8a11bcec5600557f40338407d3e5bea80376ed1c01a6c0910fcfdc4b8993/frozenlist-1.7.0-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:f89f65d85774f1797239693cef07ad4c97fdd0639544bad9ac4b869782eb1981", size = 227339, upload-time = "2025-06-09T22:59:56.187Z" }, - { url = "https://files.pythonhosted.org/packages/50/82/41cb97d9c9a5ff94438c63cc343eb7980dac4187eb625a51bdfdb7707314/frozenlist-1.7.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1073557c941395fdfcfac13eb2456cb8aad89f9de27bae29fabca8e563b12615", size = 212969, upload-time = "2025-06-09T22:59:57.604Z" }, - { url = "https://files.pythonhosted.org/packages/13/47/f9179ee5ee4f55629e4f28c660b3fdf2775c8bfde8f9c53f2de2d93f52a9/frozenlist-1.7.0-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1ed8d2fa095aae4bdc7fdd80351009a48d286635edffee66bf865e37a9125c50", size = 222862, upload-time = "2025-06-09T22:59:59.498Z" }, - { url = "https://files.pythonhosted.org/packages/1a/52/df81e41ec6b953902c8b7e3a83bee48b195cb0e5ec2eabae5d8330c78038/frozenlist-1.7.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:24c34bea555fe42d9f928ba0a740c553088500377448febecaa82cc3e88aa1fa", size = 222492, upload-time = "2025-06-09T23:00:01.026Z" }, - { url = "https://files.pythonhosted.org/packages/84/17/30d6ea87fa95a9408245a948604b82c1a4b8b3e153cea596421a2aef2754/frozenlist-1.7.0-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:69cac419ac6a6baad202c85aaf467b65ac860ac2e7f2ac1686dc40dbb52f6577", size = 238250, upload-time = "2025-06-09T23:00:03.401Z" }, - { url = "https://files.pythonhosted.org/packages/8f/00/ecbeb51669e3c3df76cf2ddd66ae3e48345ec213a55e3887d216eb4fbab3/frozenlist-1.7.0-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:960d67d0611f4c87da7e2ae2eacf7ea81a5be967861e0c63cf205215afbfac59", size = 218720, upload-time = "2025-06-09T23:00:05.282Z" }, - { url = "https://files.pythonhosted.org/packages/1a/c0/c224ce0e0eb31cc57f67742071bb470ba8246623c1823a7530be0e76164c/frozenlist-1.7.0-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:41be2964bd4b15bf575e5daee5a5ce7ed3115320fb3c2b71fca05582ffa4dc9e", size = 232585, upload-time = "2025-06-09T23:00:07.962Z" }, - { url = "https://files.pythonhosted.org/packages/55/3c/34cb694abf532f31f365106deebdeac9e45c19304d83cf7d51ebbb4ca4d1/frozenlist-1.7.0-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:46d84d49e00c9429238a7ce02dc0be8f6d7cd0cd405abd1bebdc991bf27c15bd", size = 234248, upload-time = "2025-06-09T23:00:09.428Z" }, - { url = "https://files.pythonhosted.org/packages/98/c0/2052d8b6cecda2e70bd81299e3512fa332abb6dcd2969b9c80dfcdddbf75/frozenlist-1.7.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:15900082e886edb37480335d9d518cec978afc69ccbc30bd18610b7c1b22a718", size = 221621, upload-time = "2025-06-09T23:00:11.32Z" }, - { url = "https://files.pythonhosted.org/packages/c5/bf/7dcebae315436903b1d98ffb791a09d674c88480c158aa171958a3ac07f0/frozenlist-1.7.0-cp310-cp310-win32.whl", hash = "sha256:400ddd24ab4e55014bba442d917203c73b2846391dd42ca5e38ff52bb18c3c5e", size = 39578, upload-time = "2025-06-09T23:00:13.526Z" }, - { url = "https://files.pythonhosted.org/packages/8f/5f/f69818f017fa9a3d24d1ae39763e29b7f60a59e46d5f91b9c6b21622f4cd/frozenlist-1.7.0-cp310-cp310-win_amd64.whl", hash = "sha256:6eb93efb8101ef39d32d50bce242c84bcbddb4f7e9febfa7b524532a239b4464", size = 43830, upload-time = "2025-06-09T23:00:14.98Z" }, - { url = "https://files.pythonhosted.org/packages/34/7e/803dde33760128acd393a27eb002f2020ddb8d99d30a44bfbaab31c5f08a/frozenlist-1.7.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:aa51e147a66b2d74de1e6e2cf5921890de6b0f4820b257465101d7f37b49fb5a", size = 82251, upload-time = "2025-06-09T23:00:16.279Z" }, - { url = "https://files.pythonhosted.org/packages/75/a9/9c2c5760b6ba45eae11334db454c189d43d34a4c0b489feb2175e5e64277/frozenlist-1.7.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:9b35db7ce1cd71d36ba24f80f0c9e7cff73a28d7a74e91fe83e23d27c7828750", size = 48183, upload-time = "2025-06-09T23:00:17.698Z" }, - { url = "https://files.pythonhosted.org/packages/47/be/4038e2d869f8a2da165f35a6befb9158c259819be22eeaf9c9a8f6a87771/frozenlist-1.7.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:34a69a85e34ff37791e94542065c8416c1afbf820b68f720452f636d5fb990cd", size = 47107, upload-time = "2025-06-09T23:00:18.952Z" }, - { url = "https://files.pythonhosted.org/packages/79/26/85314b8a83187c76a37183ceed886381a5f992975786f883472fcb6dc5f2/frozenlist-1.7.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4a646531fa8d82c87fe4bb2e596f23173caec9185bfbca5d583b4ccfb95183e2", size = 237333, upload-time = "2025-06-09T23:00:20.275Z" }, - { url = "https://files.pythonhosted.org/packages/1f/fd/e5b64f7d2c92a41639ffb2ad44a6a82f347787abc0c7df5f49057cf11770/frozenlist-1.7.0-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:79b2ffbba483f4ed36a0f236ccb85fbb16e670c9238313709638167670ba235f", size = 231724, upload-time = "2025-06-09T23:00:21.705Z" }, - { url = "https://files.pythonhosted.org/packages/20/fb/03395c0a43a5976af4bf7534759d214405fbbb4c114683f434dfdd3128ef/frozenlist-1.7.0-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a26f205c9ca5829cbf82bb2a84b5c36f7184c4316617d7ef1b271a56720d6b30", size = 245842, upload-time = "2025-06-09T23:00:23.148Z" }, - { url = "https://files.pythonhosted.org/packages/d0/15/c01c8e1dffdac5d9803507d824f27aed2ba76b6ed0026fab4d9866e82f1f/frozenlist-1.7.0-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:bcacfad3185a623fa11ea0e0634aac7b691aa925d50a440f39b458e41c561d98", size = 239767, upload-time = "2025-06-09T23:00:25.103Z" }, - { url = "https://files.pythonhosted.org/packages/14/99/3f4c6fe882c1f5514b6848aa0a69b20cb5e5d8e8f51a339d48c0e9305ed0/frozenlist-1.7.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:72c1b0fe8fe451b34f12dce46445ddf14bd2a5bcad7e324987194dc8e3a74c86", size = 224130, upload-time = "2025-06-09T23:00:27.061Z" }, - { url = "https://files.pythonhosted.org/packages/4d/83/220a374bd7b2aeba9d0725130665afe11de347d95c3620b9b82cc2fcab97/frozenlist-1.7.0-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:61d1a5baeaac6c0798ff6edfaeaa00e0e412d49946c53fae8d4b8e8b3566c4ae", size = 235301, upload-time = "2025-06-09T23:00:29.02Z" }, - { url = "https://files.pythonhosted.org/packages/03/3c/3e3390d75334a063181625343e8daab61b77e1b8214802cc4e8a1bb678fc/frozenlist-1.7.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:7edf5c043c062462f09b6820de9854bf28cc6cc5b6714b383149745e287181a8", size = 234606, upload-time = "2025-06-09T23:00:30.514Z" }, - { url = "https://files.pythonhosted.org/packages/23/1e/58232c19608b7a549d72d9903005e2d82488f12554a32de2d5fb59b9b1ba/frozenlist-1.7.0-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:d50ac7627b3a1bd2dcef6f9da89a772694ec04d9a61b66cf87f7d9446b4a0c31", size = 248372, upload-time = "2025-06-09T23:00:31.966Z" }, - { url = "https://files.pythonhosted.org/packages/c0/a4/e4a567e01702a88a74ce8a324691e62a629bf47d4f8607f24bf1c7216e7f/frozenlist-1.7.0-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:ce48b2fece5aeb45265bb7a58259f45027db0abff478e3077e12b05b17fb9da7", size = 229860, upload-time = "2025-06-09T23:00:33.375Z" }, - { url = "https://files.pythonhosted.org/packages/73/a6/63b3374f7d22268b41a9db73d68a8233afa30ed164c46107b33c4d18ecdd/frozenlist-1.7.0-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:fe2365ae915a1fafd982c146754e1de6ab3478def8a59c86e1f7242d794f97d5", size = 245893, upload-time = "2025-06-09T23:00:35.002Z" }, - { url = "https://files.pythonhosted.org/packages/6d/eb/d18b3f6e64799a79673c4ba0b45e4cfbe49c240edfd03a68be20002eaeaa/frozenlist-1.7.0-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:45a6f2fdbd10e074e8814eb98b05292f27bad7d1883afbe009d96abdcf3bc898", size = 246323, upload-time = "2025-06-09T23:00:36.468Z" }, - { url = "https://files.pythonhosted.org/packages/5a/f5/720f3812e3d06cd89a1d5db9ff6450088b8f5c449dae8ffb2971a44da506/frozenlist-1.7.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:21884e23cffabb157a9dd7e353779077bf5b8f9a58e9b262c6caad2ef5f80a56", size = 233149, upload-time = "2025-06-09T23:00:37.963Z" }, - { url = "https://files.pythonhosted.org/packages/69/68/03efbf545e217d5db8446acfd4c447c15b7c8cf4dbd4a58403111df9322d/frozenlist-1.7.0-cp311-cp311-win32.whl", hash = "sha256:284d233a8953d7b24f9159b8a3496fc1ddc00f4db99c324bd5fb5f22d8698ea7", size = 39565, upload-time = "2025-06-09T23:00:39.753Z" }, - { url = "https://files.pythonhosted.org/packages/58/17/fe61124c5c333ae87f09bb67186d65038834a47d974fc10a5fadb4cc5ae1/frozenlist-1.7.0-cp311-cp311-win_amd64.whl", hash = "sha256:387cbfdcde2f2353f19c2f66bbb52406d06ed77519ac7ee21be0232147c2592d", size = 44019, upload-time = "2025-06-09T23:00:40.988Z" }, - { url = "https://files.pythonhosted.org/packages/ef/a2/c8131383f1e66adad5f6ecfcce383d584ca94055a34d683bbb24ac5f2f1c/frozenlist-1.7.0-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:3dbf9952c4bb0e90e98aec1bd992b3318685005702656bc6f67c1a32b76787f2", size = 81424, upload-time = "2025-06-09T23:00:42.24Z" }, - { url = "https://files.pythonhosted.org/packages/4c/9d/02754159955088cb52567337d1113f945b9e444c4960771ea90eb73de8db/frozenlist-1.7.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:1f5906d3359300b8a9bb194239491122e6cf1444c2efb88865426f170c262cdb", size = 47952, upload-time = "2025-06-09T23:00:43.481Z" }, - { url = "https://files.pythonhosted.org/packages/01/7a/0046ef1bd6699b40acd2067ed6d6670b4db2f425c56980fa21c982c2a9db/frozenlist-1.7.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:3dabd5a8f84573c8d10d8859a50ea2dec01eea372031929871368c09fa103478", size = 46688, upload-time = "2025-06-09T23:00:44.793Z" }, - { url = "https://files.pythonhosted.org/packages/d6/a2/a910bafe29c86997363fb4c02069df4ff0b5bc39d33c5198b4e9dd42d8f8/frozenlist-1.7.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:aa57daa5917f1738064f302bf2626281a1cb01920c32f711fbc7bc36111058a8", size = 243084, upload-time = "2025-06-09T23:00:46.125Z" }, - { url = "https://files.pythonhosted.org/packages/64/3e/5036af9d5031374c64c387469bfcc3af537fc0f5b1187d83a1cf6fab1639/frozenlist-1.7.0-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:c193dda2b6d49f4c4398962810fa7d7c78f032bf45572b3e04dd5249dff27e08", size = 233524, upload-time = "2025-06-09T23:00:47.73Z" }, - { url = "https://files.pythonhosted.org/packages/06/39/6a17b7c107a2887e781a48ecf20ad20f1c39d94b2a548c83615b5b879f28/frozenlist-1.7.0-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:bfe2b675cf0aaa6d61bf8fbffd3c274b3c9b7b1623beb3809df8a81399a4a9c4", size = 248493, upload-time = "2025-06-09T23:00:49.742Z" }, - { url = "https://files.pythonhosted.org/packages/be/00/711d1337c7327d88c44d91dd0f556a1c47fb99afc060ae0ef66b4d24793d/frozenlist-1.7.0-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:8fc5d5cda37f62b262405cf9652cf0856839c4be8ee41be0afe8858f17f4c94b", size = 244116, upload-time = "2025-06-09T23:00:51.352Z" }, - { url = "https://files.pythonhosted.org/packages/24/fe/74e6ec0639c115df13d5850e75722750adabdc7de24e37e05a40527ca539/frozenlist-1.7.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b0d5ce521d1dd7d620198829b87ea002956e4319002ef0bc8d3e6d045cb4646e", size = 224557, upload-time = "2025-06-09T23:00:52.855Z" }, - { url = "https://files.pythonhosted.org/packages/8d/db/48421f62a6f77c553575201e89048e97198046b793f4a089c79a6e3268bd/frozenlist-1.7.0-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:488d0a7d6a0008ca0db273c542098a0fa9e7dfaa7e57f70acef43f32b3f69dca", size = 241820, upload-time = "2025-06-09T23:00:54.43Z" }, - { url = "https://files.pythonhosted.org/packages/1d/fa/cb4a76bea23047c8462976ea7b7a2bf53997a0ca171302deae9d6dd12096/frozenlist-1.7.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:15a7eaba63983d22c54d255b854e8108e7e5f3e89f647fc854bd77a237e767df", size = 236542, upload-time = "2025-06-09T23:00:56.409Z" }, - { url = "https://files.pythonhosted.org/packages/5d/32/476a4b5cfaa0ec94d3f808f193301debff2ea42288a099afe60757ef6282/frozenlist-1.7.0-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:1eaa7e9c6d15df825bf255649e05bd8a74b04a4d2baa1ae46d9c2d00b2ca2cb5", size = 249350, upload-time = "2025-06-09T23:00:58.468Z" }, - { url = "https://files.pythonhosted.org/packages/8d/ba/9a28042f84a6bf8ea5dbc81cfff8eaef18d78b2a1ad9d51c7bc5b029ad16/frozenlist-1.7.0-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:e4389e06714cfa9d47ab87f784a7c5be91d3934cd6e9a7b85beef808297cc025", size = 225093, upload-time = "2025-06-09T23:01:00.015Z" }, - { url = "https://files.pythonhosted.org/packages/bc/29/3a32959e68f9cf000b04e79ba574527c17e8842e38c91d68214a37455786/frozenlist-1.7.0-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:73bd45e1488c40b63fe5a7df892baf9e2a4d4bb6409a2b3b78ac1c6236178e01", size = 245482, upload-time = "2025-06-09T23:01:01.474Z" }, - { url = "https://files.pythonhosted.org/packages/80/e8/edf2f9e00da553f07f5fa165325cfc302dead715cab6ac8336a5f3d0adc2/frozenlist-1.7.0-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:99886d98e1643269760e5fe0df31e5ae7050788dd288947f7f007209b8c33f08", size = 249590, upload-time = "2025-06-09T23:01:02.961Z" }, - { url = "https://files.pythonhosted.org/packages/1c/80/9a0eb48b944050f94cc51ee1c413eb14a39543cc4f760ed12657a5a3c45a/frozenlist-1.7.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:290a172aae5a4c278c6da8a96222e6337744cd9c77313efe33d5670b9f65fc43", size = 237785, upload-time = "2025-06-09T23:01:05.095Z" }, - { url = "https://files.pythonhosted.org/packages/f3/74/87601e0fb0369b7a2baf404ea921769c53b7ae00dee7dcfe5162c8c6dbf0/frozenlist-1.7.0-cp312-cp312-win32.whl", hash = "sha256:426c7bc70e07cfebc178bc4c2bf2d861d720c4fff172181eeb4a4c41d4ca2ad3", size = 39487, upload-time = "2025-06-09T23:01:06.54Z" }, - { url = "https://files.pythonhosted.org/packages/0b/15/c026e9a9fc17585a9d461f65d8593d281fedf55fbf7eb53f16c6df2392f9/frozenlist-1.7.0-cp312-cp312-win_amd64.whl", hash = "sha256:563b72efe5da92e02eb68c59cb37205457c977aa7a449ed1b37e6939e5c47c6a", size = 43874, upload-time = "2025-06-09T23:01:07.752Z" }, - { url = "https://files.pythonhosted.org/packages/24/90/6b2cebdabdbd50367273c20ff6b57a3dfa89bd0762de02c3a1eb42cb6462/frozenlist-1.7.0-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:ee80eeda5e2a4e660651370ebffd1286542b67e268aa1ac8d6dbe973120ef7ee", size = 79791, upload-time = "2025-06-09T23:01:09.368Z" }, - { url = "https://files.pythonhosted.org/packages/83/2e/5b70b6a3325363293fe5fc3ae74cdcbc3e996c2a11dde2fd9f1fb0776d19/frozenlist-1.7.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:d1a81c85417b914139e3a9b995d4a1c84559afc839a93cf2cb7f15e6e5f6ed2d", size = 47165, upload-time = "2025-06-09T23:01:10.653Z" }, - { url = "https://files.pythonhosted.org/packages/f4/25/a0895c99270ca6966110f4ad98e87e5662eab416a17e7fd53c364bf8b954/frozenlist-1.7.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:cbb65198a9132ebc334f237d7b0df163e4de83fb4f2bdfe46c1e654bdb0c5d43", size = 45881, upload-time = "2025-06-09T23:01:12.296Z" }, - { url = "https://files.pythonhosted.org/packages/19/7c/71bb0bbe0832793c601fff68cd0cf6143753d0c667f9aec93d3c323f4b55/frozenlist-1.7.0-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:dab46c723eeb2c255a64f9dc05b8dd601fde66d6b19cdb82b2e09cc6ff8d8b5d", size = 232409, upload-time = "2025-06-09T23:01:13.641Z" }, - { url = "https://files.pythonhosted.org/packages/c0/45/ed2798718910fe6eb3ba574082aaceff4528e6323f9a8570be0f7028d8e9/frozenlist-1.7.0-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:6aeac207a759d0dedd2e40745575ae32ab30926ff4fa49b1635def65806fddee", size = 225132, upload-time = "2025-06-09T23:01:15.264Z" }, - { url = "https://files.pythonhosted.org/packages/ba/e2/8417ae0f8eacb1d071d4950f32f229aa6bf68ab69aab797b72a07ea68d4f/frozenlist-1.7.0-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:bd8c4e58ad14b4fa7802b8be49d47993182fdd4023393899632c88fd8cd994eb", size = 237638, upload-time = "2025-06-09T23:01:16.752Z" }, - { url = "https://files.pythonhosted.org/packages/f8/b7/2ace5450ce85f2af05a871b8c8719b341294775a0a6c5585d5e6170f2ce7/frozenlist-1.7.0-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:04fb24d104f425da3540ed83cbfc31388a586a7696142004c577fa61c6298c3f", size = 233539, upload-time = "2025-06-09T23:01:18.202Z" }, - { url = "https://files.pythonhosted.org/packages/46/b9/6989292c5539553dba63f3c83dc4598186ab2888f67c0dc1d917e6887db6/frozenlist-1.7.0-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6a5c505156368e4ea6b53b5ac23c92d7edc864537ff911d2fb24c140bb175e60", size = 215646, upload-time = "2025-06-09T23:01:19.649Z" }, - { url = "https://files.pythonhosted.org/packages/72/31/bc8c5c99c7818293458fe745dab4fd5730ff49697ccc82b554eb69f16a24/frozenlist-1.7.0-cp313-cp313-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8bd7eb96a675f18aa5c553eb7ddc24a43c8c18f22e1f9925528128c052cdbe00", size = 232233, upload-time = "2025-06-09T23:01:21.175Z" }, - { url = "https://files.pythonhosted.org/packages/59/52/460db4d7ba0811b9ccb85af996019f5d70831f2f5f255f7cc61f86199795/frozenlist-1.7.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:05579bf020096fe05a764f1f84cd104a12f78eaab68842d036772dc6d4870b4b", size = 227996, upload-time = "2025-06-09T23:01:23.098Z" }, - { url = "https://files.pythonhosted.org/packages/ba/c9/f4b39e904c03927b7ecf891804fd3b4df3db29b9e487c6418e37988d6e9d/frozenlist-1.7.0-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:376b6222d114e97eeec13d46c486facd41d4f43bab626b7c3f6a8b4e81a5192c", size = 242280, upload-time = "2025-06-09T23:01:24.808Z" }, - { url = "https://files.pythonhosted.org/packages/b8/33/3f8d6ced42f162d743e3517781566b8481322be321b486d9d262adf70bfb/frozenlist-1.7.0-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:0aa7e176ebe115379b5b1c95b4096fb1c17cce0847402e227e712c27bdb5a949", size = 217717, upload-time = "2025-06-09T23:01:26.28Z" }, - { url = "https://files.pythonhosted.org/packages/3e/e8/ad683e75da6ccef50d0ab0c2b2324b32f84fc88ceee778ed79b8e2d2fe2e/frozenlist-1.7.0-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:3fbba20e662b9c2130dc771e332a99eff5da078b2b2648153a40669a6d0e36ca", size = 236644, upload-time = "2025-06-09T23:01:27.887Z" }, - { url = "https://files.pythonhosted.org/packages/b2/14/8d19ccdd3799310722195a72ac94ddc677541fb4bef4091d8e7775752360/frozenlist-1.7.0-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:f3f4410a0a601d349dd406b5713fec59b4cee7e71678d5b17edda7f4655a940b", size = 238879, upload-time = "2025-06-09T23:01:29.524Z" }, - { url = "https://files.pythonhosted.org/packages/ce/13/c12bf657494c2fd1079a48b2db49fa4196325909249a52d8f09bc9123fd7/frozenlist-1.7.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:e2cdfaaec6a2f9327bf43c933c0319a7c429058e8537c508964a133dffee412e", size = 232502, upload-time = "2025-06-09T23:01:31.287Z" }, - { url = "https://files.pythonhosted.org/packages/d7/8b/e7f9dfde869825489382bc0d512c15e96d3964180c9499efcec72e85db7e/frozenlist-1.7.0-cp313-cp313-win32.whl", hash = "sha256:5fc4df05a6591c7768459caba1b342d9ec23fa16195e744939ba5914596ae3e1", size = 39169, upload-time = "2025-06-09T23:01:35.503Z" }, - { url = "https://files.pythonhosted.org/packages/35/89/a487a98d94205d85745080a37860ff5744b9820a2c9acbcdd9440bfddf98/frozenlist-1.7.0-cp313-cp313-win_amd64.whl", hash = "sha256:52109052b9791a3e6b5d1b65f4b909703984b770694d3eb64fad124c835d7cba", size = 43219, upload-time = "2025-06-09T23:01:36.784Z" }, - { url = "https://files.pythonhosted.org/packages/56/d5/5c4cf2319a49eddd9dd7145e66c4866bdc6f3dbc67ca3d59685149c11e0d/frozenlist-1.7.0-cp313-cp313t-macosx_10_13_universal2.whl", hash = "sha256:a6f86e4193bb0e235ef6ce3dde5cbabed887e0b11f516ce8a0f4d3b33078ec2d", size = 84345, upload-time = "2025-06-09T23:01:38.295Z" }, - { url = "https://files.pythonhosted.org/packages/a4/7d/ec2c1e1dc16b85bc9d526009961953df9cec8481b6886debb36ec9107799/frozenlist-1.7.0-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:82d664628865abeb32d90ae497fb93df398a69bb3434463d172b80fc25b0dd7d", size = 48880, upload-time = "2025-06-09T23:01:39.887Z" }, - { url = "https://files.pythonhosted.org/packages/69/86/f9596807b03de126e11e7d42ac91e3d0b19a6599c714a1989a4e85eeefc4/frozenlist-1.7.0-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:912a7e8375a1c9a68325a902f3953191b7b292aa3c3fb0d71a216221deca460b", size = 48498, upload-time = "2025-06-09T23:01:41.318Z" }, - { url = "https://files.pythonhosted.org/packages/5e/cb/df6de220f5036001005f2d726b789b2c0b65f2363b104bbc16f5be8084f8/frozenlist-1.7.0-cp313-cp313t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9537c2777167488d539bc5de2ad262efc44388230e5118868e172dd4a552b146", size = 292296, upload-time = "2025-06-09T23:01:42.685Z" }, - { url = "https://files.pythonhosted.org/packages/83/1f/de84c642f17c8f851a2905cee2dae401e5e0daca9b5ef121e120e19aa825/frozenlist-1.7.0-cp313-cp313t-manylinux_2_17_armv7l.manylinux2014_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:f34560fb1b4c3e30ba35fa9a13894ba39e5acfc5f60f57d8accde65f46cc5e74", size = 273103, upload-time = "2025-06-09T23:01:44.166Z" }, - { url = "https://files.pythonhosted.org/packages/88/3c/c840bfa474ba3fa13c772b93070893c6e9d5c0350885760376cbe3b6c1b3/frozenlist-1.7.0-cp313-cp313t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:acd03d224b0175f5a850edc104ac19040d35419eddad04e7cf2d5986d98427f1", size = 292869, upload-time = "2025-06-09T23:01:45.681Z" }, - { url = "https://files.pythonhosted.org/packages/a6/1c/3efa6e7d5a39a1d5ef0abeb51c48fb657765794a46cf124e5aca2c7a592c/frozenlist-1.7.0-cp313-cp313t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:f2038310bc582f3d6a09b3816ab01737d60bf7b1ec70f5356b09e84fb7408ab1", size = 291467, upload-time = "2025-06-09T23:01:47.234Z" }, - { url = "https://files.pythonhosted.org/packages/4f/00/d5c5e09d4922c395e2f2f6b79b9a20dab4b67daaf78ab92e7729341f61f6/frozenlist-1.7.0-cp313-cp313t-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b8c05e4c8e5f36e5e088caa1bf78a687528f83c043706640a92cb76cd6999384", size = 266028, upload-time = "2025-06-09T23:01:48.819Z" }, - { url = "https://files.pythonhosted.org/packages/4e/27/72765be905619dfde25a7f33813ac0341eb6b076abede17a2e3fbfade0cb/frozenlist-1.7.0-cp313-cp313t-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:765bb588c86e47d0b68f23c1bee323d4b703218037765dcf3f25c838c6fecceb", size = 284294, upload-time = "2025-06-09T23:01:50.394Z" }, - { url = "https://files.pythonhosted.org/packages/88/67/c94103a23001b17808eb7dd1200c156bb69fb68e63fcf0693dde4cd6228c/frozenlist-1.7.0-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:32dc2e08c67d86d0969714dd484fd60ff08ff81d1a1e40a77dd34a387e6ebc0c", size = 281898, upload-time = "2025-06-09T23:01:52.234Z" }, - { url = "https://files.pythonhosted.org/packages/42/34/a3e2c00c00f9e2a9db5653bca3fec306349e71aff14ae45ecc6d0951dd24/frozenlist-1.7.0-cp313-cp313t-musllinux_1_2_armv7l.whl", hash = "sha256:c0303e597eb5a5321b4de9c68e9845ac8f290d2ab3f3e2c864437d3c5a30cd65", size = 290465, upload-time = "2025-06-09T23:01:53.788Z" }, - { url = "https://files.pythonhosted.org/packages/bb/73/f89b7fbce8b0b0c095d82b008afd0590f71ccb3dee6eee41791cf8cd25fd/frozenlist-1.7.0-cp313-cp313t-musllinux_1_2_i686.whl", hash = "sha256:a47f2abb4e29b3a8d0b530f7c3598badc6b134562b1a5caee867f7c62fee51e3", size = 266385, upload-time = "2025-06-09T23:01:55.769Z" }, - { url = "https://files.pythonhosted.org/packages/cd/45/e365fdb554159462ca12df54bc59bfa7a9a273ecc21e99e72e597564d1ae/frozenlist-1.7.0-cp313-cp313t-musllinux_1_2_ppc64le.whl", hash = "sha256:3d688126c242a6fabbd92e02633414d40f50bb6002fa4cf995a1d18051525657", size = 288771, upload-time = "2025-06-09T23:01:57.4Z" }, - { url = "https://files.pythonhosted.org/packages/00/11/47b6117002a0e904f004d70ec5194fe9144f117c33c851e3d51c765962d0/frozenlist-1.7.0-cp313-cp313t-musllinux_1_2_s390x.whl", hash = "sha256:4e7e9652b3d367c7bd449a727dc79d5043f48b88d0cbfd4f9f1060cf2b414104", size = 288206, upload-time = "2025-06-09T23:01:58.936Z" }, - { url = "https://files.pythonhosted.org/packages/40/37/5f9f3c3fd7f7746082ec67bcdc204db72dad081f4f83a503d33220a92973/frozenlist-1.7.0-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:1a85e345b4c43db8b842cab1feb41be5cc0b10a1830e6295b69d7310f99becaf", size = 282620, upload-time = "2025-06-09T23:02:00.493Z" }, - { url = "https://files.pythonhosted.org/packages/0b/31/8fbc5af2d183bff20f21aa743b4088eac4445d2bb1cdece449ae80e4e2d1/frozenlist-1.7.0-cp313-cp313t-win32.whl", hash = "sha256:3a14027124ddb70dfcee5148979998066897e79f89f64b13328595c4bdf77c81", size = 43059, upload-time = "2025-06-09T23:02:02.072Z" }, - { url = "https://files.pythonhosted.org/packages/bb/ed/41956f52105b8dbc26e457c5705340c67c8cc2b79f394b79bffc09d0e938/frozenlist-1.7.0-cp313-cp313t-win_amd64.whl", hash = "sha256:3bf8010d71d4507775f658e9823210b7427be36625b387221642725b515dcf3e", size = 47516, upload-time = "2025-06-09T23:02:03.779Z" }, - { url = "https://files.pythonhosted.org/packages/ee/45/b82e3c16be2182bff01179db177fe144d58b5dc787a7d4492c6ed8b9317f/frozenlist-1.7.0-py3-none-any.whl", hash = "sha256:9a5af342e34f7e97caf8c995864c7a396418ae2859cc6fdf1b1073020d516a7e", size = 13106, upload-time = "2025-06-09T23:02:34.204Z" }, +version = "1.8.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/2d/f5/c831fac6cc817d26fd54c7eaccd04ef7e0288806943f7cc5bbf69f3ac1f0/frozenlist-1.8.0.tar.gz", hash = "sha256:3ede829ed8d842f6cd48fc7081d7a41001a56f1f38603f9d49bf3020d59a31ad", size = 45875, upload-time = "2025-10-06T05:38:17.865Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/83/4a/557715d5047da48d54e659203b9335be7bfaafda2c3f627b7c47e0b3aaf3/frozenlist-1.8.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:b37f6d31b3dcea7deb5e9696e529a6aa4a898adc33db82da12e4c60a7c4d2011", size = 86230, upload-time = "2025-10-06T05:35:23.699Z" }, + { url = "https://files.pythonhosted.org/packages/a2/fb/c85f9fed3ea8fe8740e5b46a59cc141c23b842eca617da8876cfce5f760e/frozenlist-1.8.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:ef2b7b394f208233e471abc541cc6991f907ffd47dc72584acee3147899d6565", size = 49621, upload-time = "2025-10-06T05:35:25.341Z" }, + { url = "https://files.pythonhosted.org/packages/63/70/26ca3f06aace16f2352796b08704338d74b6d1a24ca38f2771afbb7ed915/frozenlist-1.8.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:a88f062f072d1589b7b46e951698950e7da00442fc1cacbe17e19e025dc327ad", size = 49889, upload-time = "2025-10-06T05:35:26.797Z" }, + { url = "https://files.pythonhosted.org/packages/5d/ed/c7895fd2fde7f3ee70d248175f9b6cdf792fb741ab92dc59cd9ef3bd241b/frozenlist-1.8.0-cp310-cp310-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:f57fb59d9f385710aa7060e89410aeb5058b99e62f4d16b08b91986b9a2140c2", size = 219464, upload-time = "2025-10-06T05:35:28.254Z" }, + { url = "https://files.pythonhosted.org/packages/6b/83/4d587dccbfca74cb8b810472392ad62bfa100bf8108c7223eb4c4fa2f7b3/frozenlist-1.8.0-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:799345ab092bee59f01a915620b5d014698547afd011e691a208637312db9186", size = 221649, upload-time = "2025-10-06T05:35:29.454Z" }, + { url = "https://files.pythonhosted.org/packages/6a/c6/fd3b9cd046ec5fff9dab66831083bc2077006a874a2d3d9247dea93ddf7e/frozenlist-1.8.0-cp310-cp310-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:c23c3ff005322a6e16f71bf8692fcf4d5a304aaafe1e262c98c6d4adc7be863e", size = 219188, upload-time = "2025-10-06T05:35:30.951Z" }, + { url = "https://files.pythonhosted.org/packages/ce/80/6693f55eb2e085fc8afb28cf611448fb5b90e98e068fa1d1b8d8e66e5c7d/frozenlist-1.8.0-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:8a76ea0f0b9dfa06f254ee06053d93a600865b3274358ca48a352ce4f0798450", size = 231748, upload-time = "2025-10-06T05:35:32.101Z" }, + { url = "https://files.pythonhosted.org/packages/97/d6/e9459f7c5183854abd989ba384fe0cc1a0fb795a83c033f0571ec5933ca4/frozenlist-1.8.0-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:c7366fe1418a6133d5aa824ee53d406550110984de7637d65a178010f759c6ef", size = 236351, upload-time = "2025-10-06T05:35:33.834Z" }, + { url = "https://files.pythonhosted.org/packages/97/92/24e97474b65c0262e9ecd076e826bfd1d3074adcc165a256e42e7b8a7249/frozenlist-1.8.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:13d23a45c4cebade99340c4165bd90eeb4a56c6d8a9d8aa49568cac19a6d0dc4", size = 218767, upload-time = "2025-10-06T05:35:35.205Z" }, + { url = "https://files.pythonhosted.org/packages/ee/bf/dc394a097508f15abff383c5108cb8ad880d1f64a725ed3b90d5c2fbf0bb/frozenlist-1.8.0-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:e4a3408834f65da56c83528fb52ce7911484f0d1eaf7b761fc66001db1646eff", size = 235887, upload-time = "2025-10-06T05:35:36.354Z" }, + { url = "https://files.pythonhosted.org/packages/40/90/25b201b9c015dbc999a5baf475a257010471a1fa8c200c843fd4abbee725/frozenlist-1.8.0-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:42145cd2748ca39f32801dad54aeea10039da6f86e303659db90db1c4b614c8c", size = 228785, upload-time = "2025-10-06T05:35:37.949Z" }, + { url = "https://files.pythonhosted.org/packages/84/f4/b5bc148df03082f05d2dd30c089e269acdbe251ac9a9cf4e727b2dbb8a3d/frozenlist-1.8.0-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:e2de870d16a7a53901e41b64ffdf26f2fbb8917b3e6ebf398098d72c5b20bd7f", size = 230312, upload-time = "2025-10-06T05:35:39.178Z" }, + { url = "https://files.pythonhosted.org/packages/db/4b/87e95b5d15097c302430e647136b7d7ab2398a702390cf4c8601975709e7/frozenlist-1.8.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:20e63c9493d33ee48536600d1a5c95eefc870cd71e7ab037763d1fbb89cc51e7", size = 217650, upload-time = "2025-10-06T05:35:40.377Z" }, + { url = "https://files.pythonhosted.org/packages/e5/70/78a0315d1fea97120591a83e0acd644da638c872f142fd72a6cebee825f3/frozenlist-1.8.0-cp310-cp310-win32.whl", hash = "sha256:adbeebaebae3526afc3c96fad434367cafbfd1b25d72369a9e5858453b1bb71a", size = 39659, upload-time = "2025-10-06T05:35:41.863Z" }, + { url = "https://files.pythonhosted.org/packages/66/aa/3f04523fb189a00e147e60c5b2205126118f216b0aa908035c45336e27e4/frozenlist-1.8.0-cp310-cp310-win_amd64.whl", hash = "sha256:667c3777ca571e5dbeb76f331562ff98b957431df140b54c85fd4d52eea8d8f6", size = 43837, upload-time = "2025-10-06T05:35:43.205Z" }, + { url = "https://files.pythonhosted.org/packages/39/75/1135feecdd7c336938bd55b4dc3b0dfc46d85b9be12ef2628574b28de776/frozenlist-1.8.0-cp310-cp310-win_arm64.whl", hash = "sha256:80f85f0a7cc86e7a54c46d99c9e1318ff01f4687c172ede30fd52d19d1da1c8e", size = 39989, upload-time = "2025-10-06T05:35:44.596Z" }, + { url = "https://files.pythonhosted.org/packages/bc/03/077f869d540370db12165c0aa51640a873fb661d8b315d1d4d67b284d7ac/frozenlist-1.8.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:09474e9831bc2b2199fad6da3c14c7b0fbdd377cce9d3d77131be28906cb7d84", size = 86912, upload-time = "2025-10-06T05:35:45.98Z" }, + { url = "https://files.pythonhosted.org/packages/df/b5/7610b6bd13e4ae77b96ba85abea1c8cb249683217ef09ac9e0ae93f25a91/frozenlist-1.8.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:17c883ab0ab67200b5f964d2b9ed6b00971917d5d8a92df149dc2c9779208ee9", size = 50046, upload-time = "2025-10-06T05:35:47.009Z" }, + { url = "https://files.pythonhosted.org/packages/6e/ef/0e8f1fe32f8a53dd26bdd1f9347efe0778b0fddf62789ea683f4cc7d787d/frozenlist-1.8.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:fa47e444b8ba08fffd1c18e8cdb9a75db1b6a27f17507522834ad13ed5922b93", size = 50119, upload-time = "2025-10-06T05:35:48.38Z" }, + { url = "https://files.pythonhosted.org/packages/11/b1/71a477adc7c36e5fb628245dfbdea2166feae310757dea848d02bd0689fd/frozenlist-1.8.0-cp311-cp311-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:2552f44204b744fba866e573be4c1f9048d6a324dfe14475103fd51613eb1d1f", size = 231067, upload-time = "2025-10-06T05:35:49.97Z" }, + { url = "https://files.pythonhosted.org/packages/45/7e/afe40eca3a2dc19b9904c0f5d7edfe82b5304cb831391edec0ac04af94c2/frozenlist-1.8.0-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:957e7c38f250991e48a9a73e6423db1bb9dd14e722a10f6b8bb8e16a0f55f695", size = 233160, upload-time = "2025-10-06T05:35:51.729Z" }, + { url = "https://files.pythonhosted.org/packages/a6/aa/7416eac95603ce428679d273255ffc7c998d4132cfae200103f164b108aa/frozenlist-1.8.0-cp311-cp311-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:8585e3bb2cdea02fc88ffa245069c36555557ad3609e83be0ec71f54fd4abb52", size = 228544, upload-time = "2025-10-06T05:35:53.246Z" }, + { url = "https://files.pythonhosted.org/packages/8b/3d/2a2d1f683d55ac7e3875e4263d28410063e738384d3adc294f5ff3d7105e/frozenlist-1.8.0-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:edee74874ce20a373d62dc28b0b18b93f645633c2943fd90ee9d898550770581", size = 243797, upload-time = "2025-10-06T05:35:54.497Z" }, + { url = "https://files.pythonhosted.org/packages/78/1e/2d5565b589e580c296d3bb54da08d206e797d941a83a6fdea42af23be79c/frozenlist-1.8.0-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:c9a63152fe95756b85f31186bddf42e4c02c6321207fd6601a1c89ebac4fe567", size = 247923, upload-time = "2025-10-06T05:35:55.861Z" }, + { url = "https://files.pythonhosted.org/packages/aa/c3/65872fcf1d326a7f101ad4d86285c403c87be7d832b7470b77f6d2ed5ddc/frozenlist-1.8.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:b6db2185db9be0a04fecf2f241c70b63b1a242e2805be291855078f2b404dd6b", size = 230886, upload-time = "2025-10-06T05:35:57.399Z" }, + { url = "https://files.pythonhosted.org/packages/a0/76/ac9ced601d62f6956f03cc794f9e04c81719509f85255abf96e2510f4265/frozenlist-1.8.0-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:f4be2e3d8bc8aabd566f8d5b8ba7ecc09249d74ba3c9ed52e54dc23a293f0b92", size = 245731, upload-time = "2025-10-06T05:35:58.563Z" }, + { url = "https://files.pythonhosted.org/packages/b9/49/ecccb5f2598daf0b4a1415497eba4c33c1e8ce07495eb07d2860c731b8d5/frozenlist-1.8.0-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:c8d1634419f39ea6f5c427ea2f90ca85126b54b50837f31497f3bf38266e853d", size = 241544, upload-time = "2025-10-06T05:35:59.719Z" }, + { url = "https://files.pythonhosted.org/packages/53/4b/ddf24113323c0bbcc54cb38c8b8916f1da7165e07b8e24a717b4a12cbf10/frozenlist-1.8.0-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:1a7fa382a4a223773ed64242dbe1c9c326ec09457e6b8428efb4118c685c3dfd", size = 241806, upload-time = "2025-10-06T05:36:00.959Z" }, + { url = "https://files.pythonhosted.org/packages/a7/fb/9b9a084d73c67175484ba2789a59f8eebebd0827d186a8102005ce41e1ba/frozenlist-1.8.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:11847b53d722050808926e785df837353bd4d75f1d494377e59b23594d834967", size = 229382, upload-time = "2025-10-06T05:36:02.22Z" }, + { url = "https://files.pythonhosted.org/packages/95/a3/c8fb25aac55bf5e12dae5c5aa6a98f85d436c1dc658f21c3ac73f9fa95e5/frozenlist-1.8.0-cp311-cp311-win32.whl", hash = "sha256:27c6e8077956cf73eadd514be8fb04d77fc946a7fe9f7fe167648b0b9085cc25", size = 39647, upload-time = "2025-10-06T05:36:03.409Z" }, + { url = "https://files.pythonhosted.org/packages/0a/f5/603d0d6a02cfd4c8f2a095a54672b3cf967ad688a60fb9faf04fc4887f65/frozenlist-1.8.0-cp311-cp311-win_amd64.whl", hash = "sha256:ac913f8403b36a2c8610bbfd25b8013488533e71e62b4b4adce9c86c8cea905b", size = 44064, upload-time = "2025-10-06T05:36:04.368Z" }, + { url = "https://files.pythonhosted.org/packages/5d/16/c2c9ab44e181f043a86f9a8f84d5124b62dbcb3a02c0977ec72b9ac1d3e0/frozenlist-1.8.0-cp311-cp311-win_arm64.whl", hash = "sha256:d4d3214a0f8394edfa3e303136d0575eece0745ff2b47bd2cb2e66dd92d4351a", size = 39937, upload-time = "2025-10-06T05:36:05.669Z" }, + { url = "https://files.pythonhosted.org/packages/69/29/948b9aa87e75820a38650af445d2ef2b6b8a6fab1a23b6bb9e4ef0be2d59/frozenlist-1.8.0-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:78f7b9e5d6f2fdb88cdde9440dc147259b62b9d3b019924def9f6478be254ac1", size = 87782, upload-time = "2025-10-06T05:36:06.649Z" }, + { url = "https://files.pythonhosted.org/packages/64/80/4f6e318ee2a7c0750ed724fa33a4bdf1eacdc5a39a7a24e818a773cd91af/frozenlist-1.8.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:229bf37d2e4acdaf808fd3f06e854a4a7a3661e871b10dc1f8f1896a3b05f18b", size = 50594, upload-time = "2025-10-06T05:36:07.69Z" }, + { url = "https://files.pythonhosted.org/packages/2b/94/5c8a2b50a496b11dd519f4a24cb5496cf125681dd99e94c604ccdea9419a/frozenlist-1.8.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:f833670942247a14eafbb675458b4e61c82e002a148f49e68257b79296e865c4", size = 50448, upload-time = "2025-10-06T05:36:08.78Z" }, + { url = "https://files.pythonhosted.org/packages/6a/bd/d91c5e39f490a49df14320f4e8c80161cfcce09f1e2cde1edd16a551abb3/frozenlist-1.8.0-cp312-cp312-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:494a5952b1c597ba44e0e78113a7266e656b9794eec897b19ead706bd7074383", size = 242411, upload-time = "2025-10-06T05:36:09.801Z" }, + { url = "https://files.pythonhosted.org/packages/8f/83/f61505a05109ef3293dfb1ff594d13d64a2324ac3482be2cedc2be818256/frozenlist-1.8.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:96f423a119f4777a4a056b66ce11527366a8bb92f54e541ade21f2374433f6d4", size = 243014, upload-time = "2025-10-06T05:36:11.394Z" }, + { url = "https://files.pythonhosted.org/packages/d8/cb/cb6c7b0f7d4023ddda30cf56b8b17494eb3a79e3fda666bf735f63118b35/frozenlist-1.8.0-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:3462dd9475af2025c31cc61be6652dfa25cbfb56cbbf52f4ccfe029f38decaf8", size = 234909, upload-time = "2025-10-06T05:36:12.598Z" }, + { url = "https://files.pythonhosted.org/packages/31/c5/cd7a1f3b8b34af009fb17d4123c5a778b44ae2804e3ad6b86204255f9ec5/frozenlist-1.8.0-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:c4c800524c9cd9bac5166cd6f55285957fcfc907db323e193f2afcd4d9abd69b", size = 250049, upload-time = "2025-10-06T05:36:14.065Z" }, + { url = "https://files.pythonhosted.org/packages/c0/01/2f95d3b416c584a1e7f0e1d6d31998c4a795f7544069ee2e0962a4b60740/frozenlist-1.8.0-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:d6a5df73acd3399d893dafc71663ad22534b5aa4f94e8a2fabfe856c3c1b6a52", size = 256485, upload-time = "2025-10-06T05:36:15.39Z" }, + { url = "https://files.pythonhosted.org/packages/ce/03/024bf7720b3abaebcff6d0793d73c154237b85bdf67b7ed55e5e9596dc9a/frozenlist-1.8.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:405e8fe955c2280ce66428b3ca55e12b3c4e9c336fb2103a4937e891c69a4a29", size = 237619, upload-time = "2025-10-06T05:36:16.558Z" }, + { url = "https://files.pythonhosted.org/packages/69/fa/f8abdfe7d76b731f5d8bd217827cf6764d4f1d9763407e42717b4bed50a0/frozenlist-1.8.0-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:908bd3f6439f2fef9e85031b59fd4f1297af54415fb60e4254a95f75b3cab3f3", size = 250320, upload-time = "2025-10-06T05:36:17.821Z" }, + { url = "https://files.pythonhosted.org/packages/f5/3c/b051329f718b463b22613e269ad72138cc256c540f78a6de89452803a47d/frozenlist-1.8.0-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:294e487f9ec720bd8ffcebc99d575f7eff3568a08a253d1ee1a0378754b74143", size = 246820, upload-time = "2025-10-06T05:36:19.046Z" }, + { url = "https://files.pythonhosted.org/packages/0f/ae/58282e8f98e444b3f4dd42448ff36fa38bef29e40d40f330b22e7108f565/frozenlist-1.8.0-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:74c51543498289c0c43656701be6b077f4b265868fa7f8a8859c197006efb608", size = 250518, upload-time = "2025-10-06T05:36:20.763Z" }, + { url = "https://files.pythonhosted.org/packages/8f/96/007e5944694d66123183845a106547a15944fbbb7154788cbf7272789536/frozenlist-1.8.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:776f352e8329135506a1d6bf16ac3f87bc25b28e765949282dcc627af36123aa", size = 239096, upload-time = "2025-10-06T05:36:22.129Z" }, + { url = "https://files.pythonhosted.org/packages/66/bb/852b9d6db2fa40be96f29c0d1205c306288f0684df8fd26ca1951d461a56/frozenlist-1.8.0-cp312-cp312-win32.whl", hash = "sha256:433403ae80709741ce34038da08511d4a77062aa924baf411ef73d1146e74faf", size = 39985, upload-time = "2025-10-06T05:36:23.661Z" }, + { url = "https://files.pythonhosted.org/packages/b8/af/38e51a553dd66eb064cdf193841f16f077585d4d28394c2fa6235cb41765/frozenlist-1.8.0-cp312-cp312-win_amd64.whl", hash = "sha256:34187385b08f866104f0c0617404c8eb08165ab1272e884abc89c112e9c00746", size = 44591, upload-time = "2025-10-06T05:36:24.958Z" }, + { url = "https://files.pythonhosted.org/packages/a7/06/1dc65480ab147339fecc70797e9c2f69d9cea9cf38934ce08df070fdb9cb/frozenlist-1.8.0-cp312-cp312-win_arm64.whl", hash = "sha256:fe3c58d2f5db5fbd18c2987cba06d51b0529f52bc3a6cdc33d3f4eab725104bd", size = 40102, upload-time = "2025-10-06T05:36:26.333Z" }, + { url = "https://files.pythonhosted.org/packages/2d/40/0832c31a37d60f60ed79e9dfb5a92e1e2af4f40a16a29abcc7992af9edff/frozenlist-1.8.0-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:8d92f1a84bb12d9e56f818b3a746f3efba93c1b63c8387a73dde655e1e42282a", size = 85717, upload-time = "2025-10-06T05:36:27.341Z" }, + { url = "https://files.pythonhosted.org/packages/30/ba/b0b3de23f40bc55a7057bd38434e25c34fa48e17f20ee273bbde5e0650f3/frozenlist-1.8.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:96153e77a591c8adc2ee805756c61f59fef4cf4073a9275ee86fe8cba41241f7", size = 49651, upload-time = "2025-10-06T05:36:28.855Z" }, + { url = "https://files.pythonhosted.org/packages/0c/ab/6e5080ee374f875296c4243c381bbdef97a9ac39c6e3ce1d5f7d42cb78d6/frozenlist-1.8.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:f21f00a91358803399890ab167098c131ec2ddd5f8f5fd5fe9c9f2c6fcd91e40", size = 49417, upload-time = "2025-10-06T05:36:29.877Z" }, + { url = "https://files.pythonhosted.org/packages/d5/4e/e4691508f9477ce67da2015d8c00acd751e6287739123113a9fca6f1604e/frozenlist-1.8.0-cp313-cp313-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:fb30f9626572a76dfe4293c7194a09fb1fe93ba94c7d4f720dfae3b646b45027", size = 234391, upload-time = "2025-10-06T05:36:31.301Z" }, + { url = "https://files.pythonhosted.org/packages/40/76/c202df58e3acdf12969a7895fd6f3bc016c642e6726aa63bd3025e0fc71c/frozenlist-1.8.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:eaa352d7047a31d87dafcacbabe89df0aa506abb5b1b85a2fb91bc3faa02d822", size = 233048, upload-time = "2025-10-06T05:36:32.531Z" }, + { url = "https://files.pythonhosted.org/packages/f9/c0/8746afb90f17b73ca5979c7a3958116e105ff796e718575175319b5bb4ce/frozenlist-1.8.0-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:03ae967b4e297f58f8c774c7eabcce57fe3c2434817d4385c50661845a058121", size = 226549, upload-time = "2025-10-06T05:36:33.706Z" }, + { url = "https://files.pythonhosted.org/packages/7e/eb/4c7eefc718ff72f9b6c4893291abaae5fbc0c82226a32dcd8ef4f7a5dbef/frozenlist-1.8.0-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:f6292f1de555ffcc675941d65fffffb0a5bcd992905015f85d0592201793e0e5", size = 239833, upload-time = "2025-10-06T05:36:34.947Z" }, + { url = "https://files.pythonhosted.org/packages/c2/4e/e5c02187cf704224f8b21bee886f3d713ca379535f16893233b9d672ea71/frozenlist-1.8.0-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:29548f9b5b5e3460ce7378144c3010363d8035cea44bc0bf02d57f5a685e084e", size = 245363, upload-time = "2025-10-06T05:36:36.534Z" }, + { url = "https://files.pythonhosted.org/packages/1f/96/cb85ec608464472e82ad37a17f844889c36100eed57bea094518bf270692/frozenlist-1.8.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:ec3cc8c5d4084591b4237c0a272cc4f50a5b03396a47d9caaf76f5d7b38a4f11", size = 229314, upload-time = "2025-10-06T05:36:38.582Z" }, + { url = "https://files.pythonhosted.org/packages/5d/6f/4ae69c550e4cee66b57887daeebe006fe985917c01d0fff9caab9883f6d0/frozenlist-1.8.0-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:517279f58009d0b1f2e7c1b130b377a349405da3f7621ed6bfae50b10adf20c1", size = 243365, upload-time = "2025-10-06T05:36:40.152Z" }, + { url = "https://files.pythonhosted.org/packages/7a/58/afd56de246cf11780a40a2c28dc7cbabbf06337cc8ddb1c780a2d97e88d8/frozenlist-1.8.0-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:db1e72ede2d0d7ccb213f218df6a078a9c09a7de257c2fe8fcef16d5925230b1", size = 237763, upload-time = "2025-10-06T05:36:41.355Z" }, + { url = "https://files.pythonhosted.org/packages/cb/36/cdfaf6ed42e2644740d4a10452d8e97fa1c062e2a8006e4b09f1b5fd7d63/frozenlist-1.8.0-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:b4dec9482a65c54a5044486847b8a66bf10c9cb4926d42927ec4e8fd5db7fed8", size = 240110, upload-time = "2025-10-06T05:36:42.716Z" }, + { url = "https://files.pythonhosted.org/packages/03/a8/9ea226fbefad669f11b52e864c55f0bd57d3c8d7eb07e9f2e9a0b39502e1/frozenlist-1.8.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:21900c48ae04d13d416f0e1e0c4d81f7931f73a9dfa0b7a8746fb2fe7dd970ed", size = 233717, upload-time = "2025-10-06T05:36:44.251Z" }, + { url = "https://files.pythonhosted.org/packages/1e/0b/1b5531611e83ba7d13ccc9988967ea1b51186af64c42b7a7af465dcc9568/frozenlist-1.8.0-cp313-cp313-win32.whl", hash = "sha256:8b7b94a067d1c504ee0b16def57ad5738701e4ba10cec90529f13fa03c833496", size = 39628, upload-time = "2025-10-06T05:36:45.423Z" }, + { url = "https://files.pythonhosted.org/packages/d8/cf/174c91dbc9cc49bc7b7aab74d8b734e974d1faa8f191c74af9b7e80848e6/frozenlist-1.8.0-cp313-cp313-win_amd64.whl", hash = "sha256:878be833caa6a3821caf85eb39c5ba92d28e85df26d57afb06b35b2efd937231", size = 43882, upload-time = "2025-10-06T05:36:46.796Z" }, + { url = "https://files.pythonhosted.org/packages/c1/17/502cd212cbfa96eb1388614fe39a3fc9ab87dbbe042b66f97acb57474834/frozenlist-1.8.0-cp313-cp313-win_arm64.whl", hash = "sha256:44389d135b3ff43ba8cc89ff7f51f5a0bb6b63d829c8300f79a2fe4fe61bcc62", size = 39676, upload-time = "2025-10-06T05:36:47.8Z" }, + { url = "https://files.pythonhosted.org/packages/d2/5c/3bbfaa920dfab09e76946a5d2833a7cbdf7b9b4a91c714666ac4855b88b4/frozenlist-1.8.0-cp313-cp313t-macosx_10_13_universal2.whl", hash = "sha256:e25ac20a2ef37e91c1b39938b591457666a0fa835c7783c3a8f33ea42870db94", size = 89235, upload-time = "2025-10-06T05:36:48.78Z" }, + { url = "https://files.pythonhosted.org/packages/d2/d6/f03961ef72166cec1687e84e8925838442b615bd0b8854b54923ce5b7b8a/frozenlist-1.8.0-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:07cdca25a91a4386d2e76ad992916a85038a9b97561bf7a3fd12d5d9ce31870c", size = 50742, upload-time = "2025-10-06T05:36:49.837Z" }, + { url = "https://files.pythonhosted.org/packages/1e/bb/a6d12b7ba4c3337667d0e421f7181c82dda448ce4e7ad7ecd249a16fa806/frozenlist-1.8.0-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:4e0c11f2cc6717e0a741f84a527c52616140741cd812a50422f83dc31749fb52", size = 51725, upload-time = "2025-10-06T05:36:50.851Z" }, + { url = "https://files.pythonhosted.org/packages/bc/71/d1fed0ffe2c2ccd70b43714c6cab0f4188f09f8a67a7914a6b46ee30f274/frozenlist-1.8.0-cp313-cp313t-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:b3210649ee28062ea6099cfda39e147fa1bc039583c8ee4481cb7811e2448c51", size = 284533, upload-time = "2025-10-06T05:36:51.898Z" }, + { url = "https://files.pythonhosted.org/packages/c9/1f/fb1685a7b009d89f9bf78a42d94461bc06581f6e718c39344754a5d9bada/frozenlist-1.8.0-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:581ef5194c48035a7de2aefc72ac6539823bb71508189e5de01d60c9dcd5fa65", size = 292506, upload-time = "2025-10-06T05:36:53.101Z" }, + { url = "https://files.pythonhosted.org/packages/e6/3b/b991fe1612703f7e0d05c0cf734c1b77aaf7c7d321df4572e8d36e7048c8/frozenlist-1.8.0-cp313-cp313t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:3ef2d026f16a2b1866e1d86fc4e1291e1ed8a387b2c333809419a2f8b3a77b82", size = 274161, upload-time = "2025-10-06T05:36:54.309Z" }, + { url = "https://files.pythonhosted.org/packages/ca/ec/c5c618767bcdf66e88945ec0157d7f6c4a1322f1473392319b7a2501ded7/frozenlist-1.8.0-cp313-cp313t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:5500ef82073f599ac84d888e3a8c1f77ac831183244bfd7f11eaa0289fb30714", size = 294676, upload-time = "2025-10-06T05:36:55.566Z" }, + { url = "https://files.pythonhosted.org/packages/7c/ce/3934758637d8f8a88d11f0585d6495ef54b2044ed6ec84492a91fa3b27aa/frozenlist-1.8.0-cp313-cp313t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:50066c3997d0091c411a66e710f4e11752251e6d2d73d70d8d5d4c76442a199d", size = 300638, upload-time = "2025-10-06T05:36:56.758Z" }, + { url = "https://files.pythonhosted.org/packages/fc/4f/a7e4d0d467298f42de4b41cbc7ddaf19d3cfeabaf9ff97c20c6c7ee409f9/frozenlist-1.8.0-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:5c1c8e78426e59b3f8005e9b19f6ff46e5845895adbde20ece9218319eca6506", size = 283067, upload-time = "2025-10-06T05:36:57.965Z" }, + { url = "https://files.pythonhosted.org/packages/dc/48/c7b163063d55a83772b268e6d1affb960771b0e203b632cfe09522d67ea5/frozenlist-1.8.0-cp313-cp313t-musllinux_1_2_armv7l.whl", hash = "sha256:eefdba20de0d938cec6a89bd4d70f346a03108a19b9df4248d3cf0d88f1b0f51", size = 292101, upload-time = "2025-10-06T05:36:59.237Z" }, + { url = "https://files.pythonhosted.org/packages/9f/d0/2366d3c4ecdc2fd391e0afa6e11500bfba0ea772764d631bbf82f0136c9d/frozenlist-1.8.0-cp313-cp313t-musllinux_1_2_ppc64le.whl", hash = "sha256:cf253e0e1c3ceb4aaff6df637ce033ff6535fb8c70a764a8f46aafd3d6ab798e", size = 289901, upload-time = "2025-10-06T05:37:00.811Z" }, + { url = "https://files.pythonhosted.org/packages/b8/94/daff920e82c1b70e3618a2ac39fbc01ae3e2ff6124e80739ce5d71c9b920/frozenlist-1.8.0-cp313-cp313t-musllinux_1_2_s390x.whl", hash = "sha256:032efa2674356903cd0261c4317a561a6850f3ac864a63fc1583147fb05a79b0", size = 289395, upload-time = "2025-10-06T05:37:02.115Z" }, + { url = "https://files.pythonhosted.org/packages/e3/20/bba307ab4235a09fdcd3cc5508dbabd17c4634a1af4b96e0f69bfe551ebd/frozenlist-1.8.0-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:6da155091429aeba16851ecb10a9104a108bcd32f6c1642867eadaee401c1c41", size = 283659, upload-time = "2025-10-06T05:37:03.711Z" }, + { url = "https://files.pythonhosted.org/packages/fd/00/04ca1c3a7a124b6de4f8a9a17cc2fcad138b4608e7a3fc5877804b8715d7/frozenlist-1.8.0-cp313-cp313t-win32.whl", hash = "sha256:0f96534f8bfebc1a394209427d0f8a63d343c9779cda6fc25e8e121b5fd8555b", size = 43492, upload-time = "2025-10-06T05:37:04.915Z" }, + { url = "https://files.pythonhosted.org/packages/59/5e/c69f733a86a94ab10f68e496dc6b7e8bc078ebb415281d5698313e3af3a1/frozenlist-1.8.0-cp313-cp313t-win_amd64.whl", hash = "sha256:5d63a068f978fc69421fb0e6eb91a9603187527c86b7cd3f534a5b77a592b888", size = 48034, upload-time = "2025-10-06T05:37:06.343Z" }, + { url = "https://files.pythonhosted.org/packages/16/6c/be9d79775d8abe79b05fa6d23da99ad6e7763a1d080fbae7290b286093fd/frozenlist-1.8.0-cp313-cp313t-win_arm64.whl", hash = "sha256:bf0a7e10b077bf5fb9380ad3ae8ce20ef919a6ad93b4552896419ac7e1d8e042", size = 41749, upload-time = "2025-10-06T05:37:07.431Z" }, + { url = "https://files.pythonhosted.org/packages/f1/c8/85da824b7e7b9b6e7f7705b2ecaf9591ba6f79c1177f324c2735e41d36a2/frozenlist-1.8.0-cp314-cp314-macosx_10_13_universal2.whl", hash = "sha256:cee686f1f4cadeb2136007ddedd0aaf928ab95216e7691c63e50a8ec066336d0", size = 86127, upload-time = "2025-10-06T05:37:08.438Z" }, + { url = "https://files.pythonhosted.org/packages/8e/e8/a1185e236ec66c20afd72399522f142c3724c785789255202d27ae992818/frozenlist-1.8.0-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:119fb2a1bd47307e899c2fac7f28e85b9a543864df47aa7ec9d3c1b4545f096f", size = 49698, upload-time = "2025-10-06T05:37:09.48Z" }, + { url = "https://files.pythonhosted.org/packages/a1/93/72b1736d68f03fda5fdf0f2180fb6caaae3894f1b854d006ac61ecc727ee/frozenlist-1.8.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:4970ece02dbc8c3a92fcc5228e36a3e933a01a999f7094ff7c23fbd2beeaa67c", size = 49749, upload-time = "2025-10-06T05:37:10.569Z" }, + { url = "https://files.pythonhosted.org/packages/a7/b2/fabede9fafd976b991e9f1b9c8c873ed86f202889b864756f240ce6dd855/frozenlist-1.8.0-cp314-cp314-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:cba69cb73723c3f329622e34bdbf5ce1f80c21c290ff04256cff1cd3c2036ed2", size = 231298, upload-time = "2025-10-06T05:37:11.993Z" }, + { url = "https://files.pythonhosted.org/packages/3a/3b/d9b1e0b0eed36e70477ffb8360c49c85c8ca8ef9700a4e6711f39a6e8b45/frozenlist-1.8.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:778a11b15673f6f1df23d9586f83c4846c471a8af693a22e066508b77d201ec8", size = 232015, upload-time = "2025-10-06T05:37:13.194Z" }, + { url = "https://files.pythonhosted.org/packages/dc/94/be719d2766c1138148564a3960fc2c06eb688da592bdc25adcf856101be7/frozenlist-1.8.0-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:0325024fe97f94c41c08872db482cf8ac4800d80e79222c6b0b7b162d5b13686", size = 225038, upload-time = "2025-10-06T05:37:14.577Z" }, + { url = "https://files.pythonhosted.org/packages/e4/09/6712b6c5465f083f52f50cf74167b92d4ea2f50e46a9eea0523d658454ae/frozenlist-1.8.0-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:97260ff46b207a82a7567b581ab4190bd4dfa09f4db8a8b49d1a958f6aa4940e", size = 240130, upload-time = "2025-10-06T05:37:15.781Z" }, + { url = "https://files.pythonhosted.org/packages/f8/d4/cd065cdcf21550b54f3ce6a22e143ac9e4836ca42a0de1022da8498eac89/frozenlist-1.8.0-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:54b2077180eb7f83dd52c40b2750d0a9f175e06a42e3213ce047219de902717a", size = 242845, upload-time = "2025-10-06T05:37:17.037Z" }, + { url = "https://files.pythonhosted.org/packages/62/c3/f57a5c8c70cd1ead3d5d5f776f89d33110b1addae0ab010ad774d9a44fb9/frozenlist-1.8.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:2f05983daecab868a31e1da44462873306d3cbfd76d1f0b5b69c473d21dbb128", size = 229131, upload-time = "2025-10-06T05:37:18.221Z" }, + { url = "https://files.pythonhosted.org/packages/6c/52/232476fe9cb64f0742f3fde2b7d26c1dac18b6d62071c74d4ded55e0ef94/frozenlist-1.8.0-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:33f48f51a446114bc5d251fb2954ab0164d5be02ad3382abcbfe07e2531d650f", size = 240542, upload-time = "2025-10-06T05:37:19.771Z" }, + { url = "https://files.pythonhosted.org/packages/5f/85/07bf3f5d0fb5414aee5f47d33c6f5c77bfe49aac680bfece33d4fdf6a246/frozenlist-1.8.0-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:154e55ec0655291b5dd1b8731c637ecdb50975a2ae70c606d100750a540082f7", size = 237308, upload-time = "2025-10-06T05:37:20.969Z" }, + { url = "https://files.pythonhosted.org/packages/11/99/ae3a33d5befd41ac0ca2cc7fd3aa707c9c324de2e89db0e0f45db9a64c26/frozenlist-1.8.0-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:4314debad13beb564b708b4a496020e5306c7333fa9a3ab90374169a20ffab30", size = 238210, upload-time = "2025-10-06T05:37:22.252Z" }, + { url = "https://files.pythonhosted.org/packages/b2/60/b1d2da22f4970e7a155f0adde9b1435712ece01b3cd45ba63702aea33938/frozenlist-1.8.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:073f8bf8becba60aa931eb3bc420b217bb7d5b8f4750e6f8b3be7f3da85d38b7", size = 231972, upload-time = "2025-10-06T05:37:23.5Z" }, + { url = "https://files.pythonhosted.org/packages/3f/ab/945b2f32de889993b9c9133216c068b7fcf257d8595a0ac420ac8677cab0/frozenlist-1.8.0-cp314-cp314-win32.whl", hash = "sha256:bac9c42ba2ac65ddc115d930c78d24ab8d4f465fd3fc473cdedfccadb9429806", size = 40536, upload-time = "2025-10-06T05:37:25.581Z" }, + { url = "https://files.pythonhosted.org/packages/59/ad/9caa9b9c836d9ad6f067157a531ac48b7d36499f5036d4141ce78c230b1b/frozenlist-1.8.0-cp314-cp314-win_amd64.whl", hash = "sha256:3e0761f4d1a44f1d1a47996511752cf3dcec5bbdd9cc2b4fe595caf97754b7a0", size = 44330, upload-time = "2025-10-06T05:37:26.928Z" }, + { url = "https://files.pythonhosted.org/packages/82/13/e6950121764f2676f43534c555249f57030150260aee9dcf7d64efda11dd/frozenlist-1.8.0-cp314-cp314-win_arm64.whl", hash = "sha256:d1eaff1d00c7751b7c6662e9c5ba6eb2c17a2306ba5e2a37f24ddf3cc953402b", size = 40627, upload-time = "2025-10-06T05:37:28.075Z" }, + { url = "https://files.pythonhosted.org/packages/c0/c7/43200656ecc4e02d3f8bc248df68256cd9572b3f0017f0a0c4e93440ae23/frozenlist-1.8.0-cp314-cp314t-macosx_10_13_universal2.whl", hash = "sha256:d3bb933317c52d7ea5004a1c442eef86f426886fba134ef8cf4226ea6ee1821d", size = 89238, upload-time = "2025-10-06T05:37:29.373Z" }, + { url = "https://files.pythonhosted.org/packages/d1/29/55c5f0689b9c0fb765055629f472c0de484dcaf0acee2f7707266ae3583c/frozenlist-1.8.0-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:8009897cdef112072f93a0efdce29cd819e717fd2f649ee3016efd3cd885a7ed", size = 50738, upload-time = "2025-10-06T05:37:30.792Z" }, + { url = "https://files.pythonhosted.org/packages/ba/7d/b7282a445956506fa11da8c2db7d276adcbf2b17d8bb8407a47685263f90/frozenlist-1.8.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:2c5dcbbc55383e5883246d11fd179782a9d07a986c40f49abe89ddf865913930", size = 51739, upload-time = "2025-10-06T05:37:32.127Z" }, + { url = "https://files.pythonhosted.org/packages/62/1c/3d8622e60d0b767a5510d1d3cf21065b9db874696a51ea6d7a43180a259c/frozenlist-1.8.0-cp314-cp314t-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:39ecbc32f1390387d2aa4f5a995e465e9e2f79ba3adcac92d68e3e0afae6657c", size = 284186, upload-time = "2025-10-06T05:37:33.21Z" }, + { url = "https://files.pythonhosted.org/packages/2d/14/aa36d5f85a89679a85a1d44cd7a6657e0b1c75f61e7cad987b203d2daca8/frozenlist-1.8.0-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:92db2bf818d5cc8d9c1f1fc56b897662e24ea5adb36ad1f1d82875bd64e03c24", size = 292196, upload-time = "2025-10-06T05:37:36.107Z" }, + { url = "https://files.pythonhosted.org/packages/05/23/6bde59eb55abd407d34f77d39a5126fb7b4f109a3f611d3929f14b700c66/frozenlist-1.8.0-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:2dc43a022e555de94c3b68a4ef0b11c4f747d12c024a520c7101709a2144fb37", size = 273830, upload-time = "2025-10-06T05:37:37.663Z" }, + { url = "https://files.pythonhosted.org/packages/d2/3f/22cff331bfad7a8afa616289000ba793347fcd7bc275f3b28ecea2a27909/frozenlist-1.8.0-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:cb89a7f2de3602cfed448095bab3f178399646ab7c61454315089787df07733a", size = 294289, upload-time = "2025-10-06T05:37:39.261Z" }, + { url = "https://files.pythonhosted.org/packages/a4/89/5b057c799de4838b6c69aa82b79705f2027615e01be996d2486a69ca99c4/frozenlist-1.8.0-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:33139dc858c580ea50e7e60a1b0ea003efa1fd42e6ec7fdbad78fff65fad2fd2", size = 300318, upload-time = "2025-10-06T05:37:43.213Z" }, + { url = "https://files.pythonhosted.org/packages/30/de/2c22ab3eb2a8af6d69dc799e48455813bab3690c760de58e1bf43b36da3e/frozenlist-1.8.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:168c0969a329b416119507ba30b9ea13688fafffac1b7822802537569a1cb0ef", size = 282814, upload-time = "2025-10-06T05:37:45.337Z" }, + { url = "https://files.pythonhosted.org/packages/59/f7/970141a6a8dbd7f556d94977858cfb36fa9b66e0892c6dd780d2219d8cd8/frozenlist-1.8.0-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:28bd570e8e189d7f7b001966435f9dac6718324b5be2990ac496cf1ea9ddb7fe", size = 291762, upload-time = "2025-10-06T05:37:46.657Z" }, + { url = "https://files.pythonhosted.org/packages/c1/15/ca1adae83a719f82df9116d66f5bb28bb95557b3951903d39135620ef157/frozenlist-1.8.0-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:b2a095d45c5d46e5e79ba1e5b9cb787f541a8dee0433836cea4b96a2c439dcd8", size = 289470, upload-time = "2025-10-06T05:37:47.946Z" }, + { url = "https://files.pythonhosted.org/packages/ac/83/dca6dc53bf657d371fbc88ddeb21b79891e747189c5de990b9dfff2ccba1/frozenlist-1.8.0-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:eab8145831a0d56ec9c4139b6c3e594c7a83c2c8be25d5bcf2d86136a532287a", size = 289042, upload-time = "2025-10-06T05:37:49.499Z" }, + { url = "https://files.pythonhosted.org/packages/96/52/abddd34ca99be142f354398700536c5bd315880ed0a213812bc491cff5e4/frozenlist-1.8.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:974b28cf63cc99dfb2188d8d222bc6843656188164848c4f679e63dae4b0708e", size = 283148, upload-time = "2025-10-06T05:37:50.745Z" }, + { url = "https://files.pythonhosted.org/packages/af/d3/76bd4ed4317e7119c2b7f57c3f6934aba26d277acc6309f873341640e21f/frozenlist-1.8.0-cp314-cp314t-win32.whl", hash = "sha256:342c97bf697ac5480c0a7ec73cd700ecfa5a8a40ac923bd035484616efecc2df", size = 44676, upload-time = "2025-10-06T05:37:52.222Z" }, + { url = "https://files.pythonhosted.org/packages/89/76/c615883b7b521ead2944bb3480398cbb07e12b7b4e4d073d3752eb721558/frozenlist-1.8.0-cp314-cp314t-win_amd64.whl", hash = "sha256:06be8f67f39c8b1dc671f5d83aaefd3358ae5cdcf8314552c57e7ed3e6475bdd", size = 49451, upload-time = "2025-10-06T05:37:53.425Z" }, + { url = "https://files.pythonhosted.org/packages/e0/a3/5982da14e113d07b325230f95060e2169f5311b1017ea8af2a29b374c289/frozenlist-1.8.0-cp314-cp314t-win_arm64.whl", hash = "sha256:102e6314ca4da683dca92e3b1355490fed5f313b768500084fbe6371fddfdb79", size = 42507, upload-time = "2025-10-06T05:37:54.513Z" }, + { url = "https://files.pythonhosted.org/packages/9a/9a/e35b4a917281c0b8419d4207f4334c8e8c5dbf4f3f5f9ada73958d937dcc/frozenlist-1.8.0-py3-none-any.whl", hash = "sha256:0c18a16eab41e82c295618a77502e17b195883241c563b00f0aa5106fc4eaa0d", size = 13409, upload-time = "2025-10-06T05:38:16.721Z" }, ] [[package]] name = "fsspec" -version = "2025.9.0" +version = "2025.10.0" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/de/e0/bab50af11c2d75c9c4a2a26a5254573c0bd97cea152254401510950486fa/fsspec-2025.9.0.tar.gz", hash = "sha256:19fd429483d25d28b65ec68f9f4adc16c17ea2c7c7bf54ec61360d478fb19c19", size = 304847, upload-time = "2025-09-02T19:10:49.215Z" } +sdist = { url = "https://files.pythonhosted.org/packages/24/7f/2747c0d332b9acfa75dc84447a066fdf812b5a6b8d30472b74d309bfe8cb/fsspec-2025.10.0.tar.gz", hash = "sha256:b6789427626f068f9a83ca4e8a3cc050850b6c0f71f99ddb4f542b8266a26a59", size = 309285, upload-time = "2025-10-30T14:58:44.036Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/47/71/70db47e4f6ce3e5c37a607355f80da8860a33226be640226ac52cb05ef2e/fsspec-2025.9.0-py3-none-any.whl", hash = "sha256:530dc2a2af60a414a832059574df4a6e10cce927f6f4a78209390fe38955cfb7", size = 199289, upload-time = "2025-09-02T19:10:47.708Z" }, + { url = "https://files.pythonhosted.org/packages/eb/02/a6b21098b1d5d6249b7c5ab69dde30108a71e4e819d4a9778f1de1d5b70d/fsspec-2025.10.0-py3-none-any.whl", hash = "sha256:7c7712353ae7d875407f97715f0e1ffcc21e33d5b24556cb1e090ae9409ec61d", size = 200966, upload-time = "2025-10-30T14:58:42.53Z" }, ] [package.optional-dependencies] @@ -680,15 +788,6 @@ http = [ { name = "aiohttp" }, ] -[[package]] -name = "gast" -version = "0.6.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/3c/14/c566f5ca00c115db7725263408ff952b8ae6d6a4e792ef9c84e77d9af7a1/gast-0.6.0.tar.gz", hash = "sha256:88fc5300d32c7ac6ca7b515310862f71e6fdf2c029bbec7c66c0f5dd47b6b1fb", size = 27708, upload-time = "2024-06-27T20:31:49.527Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/a3/61/8001b38461d751cd1a0c3a6ae84346796a5758123f3ed97a1b121dfbf4f3/gast-0.6.0-py3-none-any.whl", hash = "sha256:52b182313f7330389f72b069ba00f174cfe2a06411099547288839c6cbafbd54", size = 21173, upload-time = "2024-07-09T13:15:15.615Z" }, -] - [[package]] name = "geoarrow-rust-core" version = "0.6.3" @@ -806,133 +905,122 @@ wheels = [ ] [[package]] -name = "google-pasta" -version = "0.2.0" +name = "h11" +version = "0.16.0" source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "six" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/35/4a/0bd53b36ff0323d10d5f24ebd67af2de10a1117f5cf4d7add90df92756f1/google-pasta-0.2.0.tar.gz", hash = "sha256:c9f2c8dfc8f96d0d5808299920721be30c9eec37f2389f28904f454565c8a16e", size = 40430, upload-time = "2020-03-13T18:57:50.34Z" } +sdist = { url = "https://files.pythonhosted.org/packages/01/ee/02a2c011bdab74c6fb3c75474d40b3052059d95df7e73351460c8588d963/h11-0.16.0.tar.gz", hash = "sha256:4e35b956cf45792e4caa5885e69fba00bdbc6ffafbfa020300e549b208ee5ff1", size = 101250, upload-time = "2025-04-24T03:35:25.427Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/a3/de/c648ef6835192e6e2cc03f40b19eeda4382c49b5bafb43d88b931c4c74ac/google_pasta-0.2.0-py3-none-any.whl", hash = "sha256:b32482794a366b5366a32c92a9a9201b107821889935a02b3e51f6b432ea84ed", size = 57471, upload-time = "2020-03-13T18:57:48.872Z" }, + { url = "https://files.pythonhosted.org/packages/04/4b/29cac41a4d98d144bf5f6d33995617b185d14b22401f75ca86f384e87ff1/h11-0.16.0-py3-none-any.whl", hash = "sha256:63cf8bbe7522de3bf65932fda1d9c2772064ffb3dae62d55932da54b31cb6c86", size = 37515, upload-time = "2025-04-24T03:35:24.344Z" }, ] [[package]] -name = "grpcio" -version = "1.75.1" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "typing-extensions" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/9d/f7/8963848164c7604efb3a3e6ee457fdb3a469653e19002bd24742473254f8/grpcio-1.75.1.tar.gz", hash = "sha256:3e81d89ece99b9ace23a6916880baca613c03a799925afb2857887efa8b1b3d2", size = 12731327, upload-time = "2025-09-26T09:03:36.887Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/51/57/89fd829fb00a6d0bee3fbcb2c8a7aa0252d908949b6ab58bfae99d39d77e/grpcio-1.75.1-cp310-cp310-linux_armv7l.whl", hash = "sha256:1712b5890b22547dd29f3215c5788d8fc759ce6dd0b85a6ba6e2731f2d04c088", size = 5705534, upload-time = "2025-09-26T09:00:52.225Z" }, - { url = "https://files.pythonhosted.org/packages/9a/3d/affe2fb897804c98d56361138e73786af8f4dd876b9d9851cfe6342b53c8/grpcio-1.75.1-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:683cfc70be0c1383449097cba637317e4737a357cfc185d887fd984206380403", size = 6289953, upload-time = "2025-09-26T09:01:03.699Z" }, - { url = "https://files.pythonhosted.org/packages/87/aa/0f40b7f47a0ff10d7e482bc3af22dac767c7ff27205915f08962d5ca87a2/grpcio-1.75.1-cp310-cp310-manylinux2014_i686.manylinux_2_17_i686.whl", hash = "sha256:491444c081a54dcd5e6ada57314321ae526377f498d4aa09d975c3241c5b9e1c", size = 6949785, upload-time = "2025-09-26T09:01:07.504Z" }, - { url = "https://files.pythonhosted.org/packages/a5/45/b04407e44050781821c84f26df71b3f7bc469923f92f9f8bc27f1406dbcc/grpcio-1.75.1-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:ce08d4e112d0d38487c2b631ec8723deac9bc404e9c7b1011426af50a79999e4", size = 6465708, upload-time = "2025-09-26T09:01:11.028Z" }, - { url = "https://files.pythonhosted.org/packages/09/3e/4ae3ec0a4d20dcaafbb6e597defcde06399ccdc5b342f607323f3b47f0a3/grpcio-1.75.1-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:5a2acda37fc926ccc4547977ac3e56b1df48fe200de968e8c8421f6e3093df6c", size = 7100912, upload-time = "2025-09-26T09:01:14.393Z" }, - { url = "https://files.pythonhosted.org/packages/34/3f/a9085dab5c313bb0cb853f222d095e2477b9b8490a03634cdd8d19daa5c3/grpcio-1.75.1-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:745c5fe6bf05df6a04bf2d11552c7d867a2690759e7ab6b05c318a772739bd75", size = 8042497, upload-time = "2025-09-26T09:01:17.759Z" }, - { url = "https://files.pythonhosted.org/packages/c3/87/ea54eba931ab9ed3f999ba95f5d8d01a20221b664725bab2fe93e3dee848/grpcio-1.75.1-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:259526a7159d39e2db40d566fe3e8f8e034d0fb2db5bf9c00e09aace655a4c2b", size = 7493284, upload-time = "2025-09-26T09:01:20.896Z" }, - { url = "https://files.pythonhosted.org/packages/0c/3c/35ca9747473a306bfad0cee04504953f7098527cd112a4ab55c55af9e7bd/grpcio-1.75.1-cp311-cp311-linux_armv7l.whl", hash = "sha256:573855ca2e58e35032aff30bfbd1ee103fbcf4472e4b28d4010757700918e326", size = 5709761, upload-time = "2025-09-26T09:01:28.528Z" }, - { url = "https://files.pythonhosted.org/packages/81/40/bc07aee2911f0d426fa53fe636216100c31a8ea65a400894f280274cb023/grpcio-1.75.1-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:b1ea1bbe77ecbc1be00af2769f4ae4a88ce93be57a4f3eebd91087898ed749f9", size = 6296084, upload-time = "2025-09-26T09:01:34.596Z" }, - { url = "https://files.pythonhosted.org/packages/b8/d1/10c067f6c67396cbf46448b80f27583b5e8c4b46cdfbe18a2a02c2c2f290/grpcio-1.75.1-cp311-cp311-manylinux2014_i686.manylinux_2_17_i686.whl", hash = "sha256:e5b425aee54cc5e3e3c58f00731e8a33f5567965d478d516d35ef99fd648ab68", size = 6950403, upload-time = "2025-09-26T09:01:36.736Z" }, - { url = "https://files.pythonhosted.org/packages/3f/42/5f628abe360b84dfe8dd8f32be6b0606dc31dc04d3358eef27db791ea4d5/grpcio-1.75.1-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:0049a7bf547dafaeeb1db17079ce79596c298bfe308fc084d023c8907a845b9a", size = 6470166, upload-time = "2025-09-26T09:01:39.474Z" }, - { url = "https://files.pythonhosted.org/packages/c3/93/a24035080251324019882ee2265cfde642d6476c0cf8eb207fc693fcebdc/grpcio-1.75.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:5b8ea230c7f77c0a1a3208a04a1eda164633fb0767b4cefd65a01079b65e5b1f", size = 7107828, upload-time = "2025-09-26T09:01:41.782Z" }, - { url = "https://files.pythonhosted.org/packages/e4/f8/d18b984c1c9ba0318e3628dbbeb6af77a5007f02abc378c845070f2d3edd/grpcio-1.75.1-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:36990d629c3c9fb41e546414e5af52d0a7af37ce7113d9682c46d7e2919e4cca", size = 8045421, upload-time = "2025-09-26T09:01:45.835Z" }, - { url = "https://files.pythonhosted.org/packages/7e/b6/4bf9aacff45deca5eac5562547ed212556b831064da77971a4e632917da3/grpcio-1.75.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:b10ad908118d38c2453ade7ff790e5bce36580c3742919007a2a78e3a1e521ca", size = 7503290, upload-time = "2025-09-26T09:01:49.28Z" }, - { url = "https://files.pythonhosted.org/packages/3a/81/42be79e73a50aaa20af66731c2defeb0e8c9008d9935a64dd8ea8e8c44eb/grpcio-1.75.1-cp312-cp312-linux_armv7l.whl", hash = "sha256:7b888b33cd14085d86176b1628ad2fcbff94cfbbe7809465097aa0132e58b018", size = 5668314, upload-time = "2025-09-26T09:01:55.424Z" }, - { url = "https://files.pythonhosted.org/packages/14/85/21c71d674f03345ab183c634ecd889d3330177e27baea8d5d247a89b6442/grpcio-1.75.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:bb658f703468d7fbb5dcc4037c65391b7dc34f808ac46ed9136c24fc5eeb041d", size = 6246335, upload-time = "2025-09-26T09:02:00.76Z" }, - { url = "https://files.pythonhosted.org/packages/fd/db/3beb661bc56a385ae4fa6b0e70f6b91ac99d47afb726fe76aaff87ebb116/grpcio-1.75.1-cp312-cp312-manylinux2014_i686.manylinux_2_17_i686.whl", hash = "sha256:4b7177a1cdb3c51b02b0c0a256b0a72fdab719600a693e0e9037949efffb200b", size = 6916309, upload-time = "2025-09-26T09:02:02.894Z" }, - { url = "https://files.pythonhosted.org/packages/1e/9c/eda9fe57f2b84343d44c1b66cf3831c973ba29b078b16a27d4587a1fdd47/grpcio-1.75.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:7d4fa6ccc3ec2e68a04f7b883d354d7fea22a34c44ce535a2f0c0049cf626ddf", size = 6435419, upload-time = "2025-09-26T09:02:05.055Z" }, - { url = "https://files.pythonhosted.org/packages/c3/b8/090c98983e0a9d602e3f919a6e2d4e470a8b489452905f9a0fa472cac059/grpcio-1.75.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:3d86880ecaeb5b2f0a8afa63824de93adb8ebe4e49d0e51442532f4e08add7d6", size = 7064893, upload-time = "2025-09-26T09:02:07.275Z" }, - { url = "https://files.pythonhosted.org/packages/ec/c0/6d53d4dbbd00f8bd81571f5478d8a95528b716e0eddb4217cc7cb45aae5f/grpcio-1.75.1-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:a8041d2f9e8a742aeae96f4b047ee44e73619f4f9d24565e84d5446c623673b6", size = 8011922, upload-time = "2025-09-26T09:02:09.527Z" }, - { url = "https://files.pythonhosted.org/packages/f2/7c/48455b2d0c5949678d6982c3e31ea4d89df4e16131b03f7d5c590811cbe9/grpcio-1.75.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:3652516048bf4c314ce12be37423c79829f46efffb390ad64149a10c6071e8de", size = 7466181, upload-time = "2025-09-26T09:02:12.279Z" }, - { url = "https://files.pythonhosted.org/packages/46/74/bac4ab9f7722164afdf263ae31ba97b8174c667153510322a5eba4194c32/grpcio-1.75.1-cp313-cp313-linux_armv7l.whl", hash = "sha256:3bed22e750d91d53d9e31e0af35a7b0b51367e974e14a4ff229db5b207647884", size = 5672779, upload-time = "2025-09-26T09:02:19.11Z" }, - { url = "https://files.pythonhosted.org/packages/cf/e4/d1954dce2972e32384db6a30273275e8c8ea5a44b80347f9055589333b3f/grpcio-1.75.1-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:5bf4001d3293e3414d0cf99ff9b1139106e57c3a66dfff0c5f60b2a6286ec133", size = 6248838, upload-time = "2025-09-26T09:02:26.426Z" }, - { url = "https://files.pythonhosted.org/packages/06/43/073363bf63826ba8077c335d797a8d026f129dc0912b69c42feaf8f0cd26/grpcio-1.75.1-cp313-cp313-manylinux2014_i686.manylinux_2_17_i686.whl", hash = "sha256:9f82ff474103e26351dacfe8d50214e7c9322960d8d07ba7fa1d05ff981c8b2d", size = 6922663, upload-time = "2025-09-26T09:02:28.724Z" }, - { url = "https://files.pythonhosted.org/packages/c2/6f/076ac0df6c359117676cacfa8a377e2abcecec6a6599a15a672d331f6680/grpcio-1.75.1-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:0ee119f4f88d9f75414217823d21d75bfe0e6ed40135b0cbbfc6376bc9f7757d", size = 6436149, upload-time = "2025-09-26T09:02:30.971Z" }, - { url = "https://files.pythonhosted.org/packages/6b/27/1d08824f1d573fcb1fa35ede40d6020e68a04391709939e1c6f4193b445f/grpcio-1.75.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:664eecc3abe6d916fa6cf8dd6b778e62fb264a70f3430a3180995bf2da935446", size = 7067989, upload-time = "2025-09-26T09:02:33.233Z" }, - { url = "https://files.pythonhosted.org/packages/c6/98/98594cf97b8713feb06a8cb04eeef60b4757e3e2fb91aa0d9161da769843/grpcio-1.75.1-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:c32193fa08b2fbebf08fe08e84f8a0aad32d87c3ad42999c65e9449871b1c66e", size = 8010717, upload-time = "2025-09-26T09:02:36.011Z" }, - { url = "https://files.pythonhosted.org/packages/8c/7e/bb80b1bba03c12158f9254762cdf5cced4a9bc2e8ed51ed335915a5a06ef/grpcio-1.75.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:5cebe13088b9254f6e615bcf1da9131d46cfa4e88039454aca9cb65f639bd3bc", size = 7463822, upload-time = "2025-09-26T09:02:38.26Z" }, - { url = "https://files.pythonhosted.org/packages/f2/1b/9a0a5cecd24302b9fdbcd55d15ed6267e5f3d5b898ff9ac8cbe17ee76129/grpcio-1.75.1-cp314-cp314-linux_armv7l.whl", hash = "sha256:c05da79068dd96723793bffc8d0e64c45f316248417515f28d22204d9dae51c7", size = 5673319, upload-time = "2025-09-26T09:02:44.742Z" }, - { url = "https://files.pythonhosted.org/packages/09/7a/26da709e42c4565c3d7bf999a9569da96243ce34a8271a968dee810a7cf1/grpcio-1.75.1-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:4484f4b7287bdaa7a5b3980f3c7224c3c622669405d20f69549f5fb956ad0421", size = 6254706, upload-time = "2025-09-26T09:02:50.4Z" }, - { url = "https://files.pythonhosted.org/packages/f1/08/dcb26a319d3725f199c97e671d904d84ee5680de57d74c566a991cfab632/grpcio-1.75.1-cp314-cp314-manylinux2014_i686.manylinux_2_17_i686.whl", hash = "sha256:2720c239c1180eee69f7883c1d4c83fc1a495a2535b5fa322887c70bf02b16e8", size = 6922501, upload-time = "2025-09-26T09:02:52.711Z" }, - { url = "https://files.pythonhosted.org/packages/78/66/044d412c98408a5e23cb348845979a2d17a2e2b6c3c34c1ec91b920f49d0/grpcio-1.75.1-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:07a554fa31c668cf0e7a188678ceeca3cb8fead29bbe455352e712ec33ca701c", size = 6437492, upload-time = "2025-09-26T09:02:55.542Z" }, - { url = "https://files.pythonhosted.org/packages/4e/9d/5e3e362815152aa1afd8b26ea613effa005962f9da0eec6e0e4527e7a7d1/grpcio-1.75.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:3e71a2105210366bfc398eef7f57a664df99194f3520edb88b9c3a7e46ee0d64", size = 7081061, upload-time = "2025-09-26T09:02:58.261Z" }, - { url = "https://files.pythonhosted.org/packages/1e/1a/46615682a19e100f46e31ddba9ebc297c5a5ab9ddb47b35443ffadb8776c/grpcio-1.75.1-cp314-cp314-musllinux_1_2_i686.whl", hash = "sha256:8679aa8a5b67976776d3c6b0521e99d1c34db8a312a12bcfd78a7085cb9b604e", size = 8010849, upload-time = "2025-09-26T09:03:00.548Z" }, - { url = "https://files.pythonhosted.org/packages/67/8e/3204b94ac30b0f675ab1c06540ab5578660dc8b690db71854d3116f20d00/grpcio-1.75.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:aad1c774f4ebf0696a7f148a56d39a3432550612597331792528895258966dc0", size = 7464478, upload-time = "2025-09-26T09:03:03.096Z" }, -] - -[[package]] -name = "h5py" -version = "3.14.0" +name = "hf-xet" +version = "1.5.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/4b/2d/57fd21d84d93efb4bd0b962383790e19dd1bc053501b4264c97903b4e83e/hf_xet-1.5.1.tar.gz", hash = "sha256:51ef4500dab3764b41135ee1381a4b62ce56fc54d4c92b719b59e597d6df5bf6", size = 876636, upload-time = "2026-06-08T23:02:53.897Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/64/ee/dd9ba7beae1005e54131b7d45263cc74c8a066d47d354e6d58ae9445a388/hf_xet-1.5.1-cp313-cp313t-macosx_10_12_x86_64.whl", hash = "sha256:dbf48c0d02cf0b2e568944330c60d9120c272dabe013bd892d48e25bc6797577", size = 4069485, upload-time = "2026-06-08T23:02:13.193Z" }, + { url = "https://files.pythonhosted.org/packages/b6/bc/9cae6cfeb4e03070874e73e5c97c66eb90369d3206b6a2b1ef5f96520888/hf_xet-1.5.1-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:e78e4e5192ad2b674c2e1160b651cb9134db974f8ae1835bdfbfb0166b894a43", size = 3838493, upload-time = "2026-06-08T23:02:15.282Z" }, + { url = "https://files.pythonhosted.org/packages/ba/b4/d5c01e0eb6d9f2ca2dacd84d0d1b71e6cfbb2ef3208c968528e010e9b3d7/hf_xet-1.5.1-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:6f7a04a8ad962422e225bc49fbbac99dc1806764b1f3e54dbd154bffa7593947", size = 4505658, upload-time = "2026-06-08T23:02:17.196Z" }, + { url = "https://files.pythonhosted.org/packages/76/c5/29a7598c0c6383c523dc22186d577f4e04267a626cd95ae60f67c00bfe66/hf_xet-1.5.1-cp313-cp313t-manylinux_2_28_aarch64.whl", hash = "sha256:d48199c2bf4f8df0adc55d31d1368b6ec0e4d4f45bc86b08038089c23db0bed8", size = 4292822, upload-time = "2026-06-08T23:02:18.608Z" }, + { url = "https://files.pythonhosted.org/packages/04/9a/dceaf6ca69390126b86ea825fb354b93d01163199070b7bd849225de9468/hf_xet-1.5.1-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:97f212a88d14bbf573619a74b7fecb238de77d08fc702e54dec6f78276ca3283", size = 4491255, upload-time = "2026-06-08T23:02:20.124Z" }, + { url = "https://files.pythonhosted.org/packages/48/a7/e5a7afaacf6c1791fdbeeac42951fb81c3d2bc482992b115dedcc86d963e/hf_xet-1.5.1-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:f61e3665892a6c8c5e765395838b8ddf36185da835253d4bc4509a81e49fb342", size = 4711062, upload-time = "2026-06-08T23:02:21.863Z" }, + { url = "https://files.pythonhosted.org/packages/53/49/2802f8433c9742ce281bddc1e65c02c32268ca3098d66828b05e12e45ee2/hf_xet-1.5.1-cp313-cp313t-win_amd64.whl", hash = "sha256:f4ad3ebd4c32dd2b27099d69dc7b2df821e30767e46fb6ee6a0713778243b8ff", size = 4017205, upload-time = "2026-06-08T23:02:23.495Z" }, + { url = "https://files.pythonhosted.org/packages/9e/5a/50c71195b9fb883659f596e7252faf4c18c58e753a9013bdbf9bac5d2250/hf_xet-1.5.1-cp313-cp313t-win_arm64.whl", hash = "sha256:8298485c1e36e7e67cbd01eeb1376619b7af43d4f1ec245caae306f890a8a32d", size = 3845426, upload-time = "2026-06-08T23:02:25.124Z" }, + { url = "https://files.pythonhosted.org/packages/05/24/5e0c28f80371c17d49fed004597d9d132cb75c1f6f53db2cb95f459d2312/hf_xet-1.5.1-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:3474760d10e3bb6f92ff3f024fcb00c0b3e4001e9b035c7483e49a5dd17aa70f", size = 4069676, upload-time = "2026-06-08T23:02:26.759Z" }, + { url = "https://files.pythonhosted.org/packages/d2/17/261ba565b6a4d960fb478f61fdf919c0be5824645aaf1c319eca660c1611/hf_xet-1.5.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:6762d89b9e3267dfd502b29b2a327b4525f33b17e7b509a78d94e2151a30ce30", size = 3838509, upload-time = "2026-06-08T23:02:28.573Z" }, + { url = "https://files.pythonhosted.org/packages/4e/44/7ffdc2e184b0d41fc0f683ba3936ef669ab63cf242cf36ef50e57d683668/hf_xet-1.5.1-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:bf67e6ed10260cef62e852789dc91ebb03f382d5bdc4b1dbeb64763ea275e7d6", size = 4505881, upload-time = "2026-06-08T23:02:30.257Z" }, + { url = "https://files.pythonhosted.org/packages/63/b6/788060d5aa4d5e671f1a31bf69624c314eb2d8babab3aa562f9e5d53444e/hf_xet-1.5.1-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:c6b6cd08ca095058780b50b8ce4d6cbf6787bcf27841705d58a9d32246e3e47a", size = 4292995, upload-time = "2026-06-08T23:02:31.993Z" }, + { url = "https://files.pythonhosted.org/packages/22/93/c5540cbd6b55529b7dc42f6734e88cebee21aefbea34128b66229df56c57/hf_xet-1.5.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:e1af0de8ca6f190d4294a28b88023db64a1e2d1d719cab044baf75bec569e7a9", size = 4491570, upload-time = "2026-06-08T23:02:33.86Z" }, + { url = "https://files.pythonhosted.org/packages/03/f3/9d8ceab30f44f36c1679b1b8683054c71a0dadc787dbf07421891742d3ca/hf_xet-1.5.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:4f561cbbb92f80960772059864b7fb07eae879adde1b2e781ec6f86f6ac26c59", size = 4711565, upload-time = "2026-06-08T23:02:35.454Z" }, + { url = "https://files.pythonhosted.org/packages/cd/54/27ed9a5e2cc583b4df82f75a03a4df8dbf55f5a9fa1f47f1fadfb20dbeac/hf_xet-1.5.1-cp314-cp314t-win_amd64.whl", hash = "sha256:e7dbb40617410f432182d918e37c12303fe6700fd6aa6c5964e30a535a4461d6", size = 4017343, upload-time = "2026-06-08T23:02:37.14Z" }, + { url = "https://files.pythonhosted.org/packages/ae/12/ecb2fc8d45e767580e3a37faa97cb895608b614965567efb4f18cff67e27/hf_xet-1.5.1-cp314-cp314t-win_arm64.whl", hash = "sha256:6071d5ccb4d8d2cbd5fea5cc798da4f0ba3f44e25369591c4e89a4987050e61d", size = 3845716, upload-time = "2026-06-08T23:02:39.073Z" }, + { url = "https://files.pythonhosted.org/packages/7a/d8/5e54cf37434759d1f4f2ba9b66077ff9d4c4e1f37b6bd7975da5c40d94ab/hf_xet-1.5.1-cp37-abi3-macosx_10_12_x86_64.whl", hash = "sha256:6abd35c3221eff63836618ddfb954dcf84798603f71d8e33e3ed7b04acfdbe6e", size = 4077794, upload-time = "2026-06-08T23:02:40.656Z" }, + { url = "https://files.pythonhosted.org/packages/35/94/4b2ecfbad8f8b04701a23aefb62f540b9137d058b7e1dbef16a32676f0e9/hf_xet-1.5.1-cp37-abi3-macosx_11_0_arm64.whl", hash = "sha256:94e761bbd266bf4c03cee73753916062665ce8365aa40ed321f45afcb934b41e", size = 3845354, upload-time = "2026-06-08T23:02:42.702Z" }, + { url = "https://files.pythonhosted.org/packages/de/cc/f99f4bc7295023d7bd9ebbfd51f75cc530ca262c1227666268b8208f4b77/hf_xet-1.5.1-cp37-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:892e3a3a3aecc12aded8b93cf4f9cd059282c7de0732f7d55026f3abdf474350", size = 4514864, upload-time = "2026-06-08T23:02:44.497Z" }, + { url = "https://files.pythonhosted.org/packages/cd/6e/21f7e5a2381278bd3b7b7a5a4d90038518bb6308a0c1daf5d9f8268bb178/hf_xet-1.5.1-cp37-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:a93df2039190502835b1db8cd7e178b0b7b889fe9ab51299d5ced26e0dd879a4", size = 4303784, upload-time = "2026-06-08T23:02:46.203Z" }, + { url = "https://files.pythonhosted.org/packages/35/0e/f992bb6927ac1cb30ef74e62268f551f338bc32b2191f7c96a44c6f7283e/hf_xet-1.5.1-cp37-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:0c97106032ef70467b4f6bc2d0ccc266d7613ee076afc56516c502f87ce1c4a6", size = 4500703, upload-time = "2026-06-08T23:02:47.628Z" }, + { url = "https://files.pythonhosted.org/packages/fb/d1/90a498d05447980b977b1669246eeeeae4cfb0ea3e7a286eaba627f91bf9/hf_xet-1.5.1-cp37-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:6208adb15d192b90e4c2ad2a27ed864359b2cb0f2494eb6d7c7f3699ac02e2bf", size = 4719498, upload-time = "2026-06-08T23:02:49.268Z" }, + { url = "https://files.pythonhosted.org/packages/6d/b6/20f99cfe97cc663a711f7b33cc21d4793e51968e9a26125b4afcd77315ba/hf_xet-1.5.1-cp37-abi3-win_amd64.whl", hash = "sha256:f7b3002f95d1c13e24bcb4537baa8f0eb3838957067c91bb4959bc004a6435f5", size = 4026419, upload-time = "2026-06-08T23:02:50.829Z" }, + { url = "https://files.pythonhosted.org/packages/f9/fa/77453694888f03e5a8c8852d1514a0894d8e81c622d39edbaf308ea0dcf4/hf_xet-1.5.1-cp37-abi3-win_arm64.whl", hash = "sha256:93d090b57b211133f6c0dab0205ef5cb6d89162979ba75a74845045cc3063b8e", size = 3855178, upload-time = "2026-06-08T23:02:52.452Z" }, +] + +[[package]] +name = "httpcore" +version = "1.0.9" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, - { name = "numpy", version = "2.3.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" }, + { name = "certifi" }, + { name = "h11" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/5d/57/dfb3c5c3f1bf5f5ef2e59a22dec4ff1f3d7408b55bfcefcfb0ea69ef21c6/h5py-3.14.0.tar.gz", hash = "sha256:2372116b2e0d5d3e5e705b7f663f7c8d96fa79a4052d250484ef91d24d6a08f4", size = 424323, upload-time = "2025-06-06T14:06:15.01Z" } +sdist = { url = "https://files.pythonhosted.org/packages/06/94/82699a10bca87a5556c9c59b5963f2d039dbd239f25bc2a63907a05a14cb/httpcore-1.0.9.tar.gz", hash = "sha256:6e34463af53fd2ab5d807f399a9b45ea31c3dfa2276f15a2c3f00afff6e176e8", size = 85484, upload-time = "2025-04-24T22:06:22.219Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/fa/cd/3dd38cdb7cc9266dc4d85f27f0261680cb62f553f1523167ad7454e32b11/h5py-3.14.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:016e89d3be4c44f8d5e115fab60548e518ecd9efe9fa5c5324505a90773e6f03", size = 4324677, upload-time = "2025-06-06T14:04:23.438Z" }, - { url = "https://files.pythonhosted.org/packages/b1/45/e1a754dc7cd465ba35e438e28557119221ac89b20aaebef48282654e3dc7/h5py-3.14.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1223b902ef0b5d90bcc8a4778218d6d6cd0f5561861611eda59fa6c52b922f4d", size = 4557272, upload-time = "2025-06-06T14:04:28.863Z" }, - { url = "https://files.pythonhosted.org/packages/08/0c/5e6aaf221557314bc15ba0e0da92e40b24af97ab162076c8ae009320a42b/h5py-3.14.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8c497600c0496548810047257e36360ff551df8b59156d3a4181072eed47d8ad", size = 4298002, upload-time = "2025-06-06T14:04:47.106Z" }, - { url = "https://files.pythonhosted.org/packages/21/d4/d461649cafd5137088fb7f8e78fdc6621bb0c4ff2c090a389f68e8edc136/h5py-3.14.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:723a40ee6505bd354bfd26385f2dae7bbfa87655f4e61bab175a49d72ebfc06b", size = 4516618, upload-time = "2025-06-06T14:04:52.467Z" }, - { url = "https://files.pythonhosted.org/packages/3f/19/c8bfe8543bfdd7ccfafd46d8cfd96fce53d6c33e9c7921f375530ee1d39a/h5py-3.14.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:554ef0ced3571366d4d383427c00c966c360e178b5fb5ee5bb31a435c424db0c", size = 4708455, upload-time = "2025-06-06T14:05:11.528Z" }, - { url = "https://files.pythonhosted.org/packages/86/f9/f00de11c82c88bfc1ef22633557bfba9e271e0cb3189ad704183fc4a2644/h5py-3.14.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0cbd41f4e3761f150aa5b662df991868ca533872c95467216f2bec5fcad84882", size = 4929422, upload-time = "2025-06-06T14:05:18.399Z" }, - { url = "https://files.pythonhosted.org/packages/0d/ce/3a21d87896bc7e3e9255e0ad5583ae31ae9e6b4b00e0bcb2a67e2b6acdbc/h5py-3.14.0-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e8cbaf6910fa3983c46172666b0b8da7b7bd90d764399ca983236f2400436eeb", size = 4700675, upload-time = "2025-06-06T14:05:37.38Z" }, - { url = "https://files.pythonhosted.org/packages/e7/ec/86f59025306dcc6deee5fda54d980d077075b8d9889aac80f158bd585f1b/h5py-3.14.0-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d90e6445ab7c146d7f7981b11895d70bc1dd91278a4f9f9028bc0c95e4a53f13", size = 4921632, upload-time = "2025-06-06T14:05:43.464Z" }, + { url = "https://files.pythonhosted.org/packages/7e/f5/f66802a942d491edb555dd61e3a9961140fd64c90bce1eafd741609d334d/httpcore-1.0.9-py3-none-any.whl", hash = "sha256:2d400746a40668fc9dec9810239072b40b4484b640a8c38fd654a024c7a1bf55", size = 78784, upload-time = "2025-04-24T22:06:20.566Z" }, ] [[package]] -name = "hf-xet" -version = "1.1.10" +name = "httpx" +version = "0.28.1" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/74/31/feeddfce1748c4a233ec1aa5b7396161c07ae1aa9b7bdbc9a72c3c7dd768/hf_xet-1.1.10.tar.gz", hash = "sha256:408aef343800a2102374a883f283ff29068055c111f003ff840733d3b715bb97", size = 487910, upload-time = "2025-09-12T20:10:27.12Z" } +dependencies = [ + { name = "anyio" }, + { name = "certifi" }, + { name = "httpcore" }, + { name = "idna" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/b1/df/48c586a5fe32a0f01324ee087459e112ebb7224f646c0b5023f5e79e9956/httpx-0.28.1.tar.gz", hash = "sha256:75e98c5f16b0f35b567856f597f06ff2270a374470a5c2392242528e3e3e42fc", size = 141406, upload-time = "2024-12-06T15:37:23.222Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/f7/a2/343e6d05de96908366bdc0081f2d8607d61200be2ac802769c4284cc65bd/hf_xet-1.1.10-cp37-abi3-macosx_10_12_x86_64.whl", hash = "sha256:686083aca1a6669bc85c21c0563551cbcdaa5cf7876a91f3d074a030b577231d", size = 2761466, upload-time = "2025-09-12T20:10:22.836Z" }, - { url = "https://files.pythonhosted.org/packages/31/f9/6215f948ac8f17566ee27af6430ea72045e0418ce757260248b483f4183b/hf_xet-1.1.10-cp37-abi3-macosx_11_0_arm64.whl", hash = "sha256:71081925383b66b24eedff3013f8e6bbd41215c3338be4b94ba75fd75b21513b", size = 2623807, upload-time = "2025-09-12T20:10:21.118Z" }, - { url = "https://files.pythonhosted.org/packages/15/07/86397573efefff941e100367bbda0b21496ffcdb34db7ab51912994c32a2/hf_xet-1.1.10-cp37-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6b6bceb6361c80c1cc42b5a7b4e3efd90e64630bcf11224dcac50ef30a47e435", size = 3186960, upload-time = "2025-09-12T20:10:19.336Z" }, - { url = "https://files.pythonhosted.org/packages/01/a7/0b2e242b918cc30e1f91980f3c4b026ff2eedaf1e2ad96933bca164b2869/hf_xet-1.1.10-cp37-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:eae7c1fc8a664e54753ffc235e11427ca61f4b0477d757cc4eb9ae374b69f09c", size = 3087167, upload-time = "2025-09-12T20:10:17.255Z" }, - { url = "https://files.pythonhosted.org/packages/4a/25/3e32ab61cc7145b11eee9d745988e2f0f4fafda81b25980eebf97d8cff15/hf_xet-1.1.10-cp37-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:0a0005fd08f002180f7a12d4e13b22be277725bc23ed0529f8add5c7a6309c06", size = 3248612, upload-time = "2025-09-12T20:10:24.093Z" }, - { url = "https://files.pythonhosted.org/packages/2c/3d/ab7109e607ed321afaa690f557a9ada6d6d164ec852fd6bf9979665dc3d6/hf_xet-1.1.10-cp37-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:f900481cf6e362a6c549c61ff77468bd59d6dd082f3170a36acfef2eb6a6793f", size = 3353360, upload-time = "2025-09-12T20:10:25.563Z" }, - { url = "https://files.pythonhosted.org/packages/ee/0e/471f0a21db36e71a2f1752767ad77e92d8cde24e974e03d662931b1305ec/hf_xet-1.1.10-cp37-abi3-win_amd64.whl", hash = "sha256:5f54b19cc347c13235ae7ee98b330c26dd65ef1df47e5316ffb1e87713ca7045", size = 2804691, upload-time = "2025-09-12T20:10:28.433Z" }, + { url = "https://files.pythonhosted.org/packages/2a/39/e50c7c3a983047577ee07d2a9e53faf5a69493943ec3f6a384bdc792deb2/httpx-0.28.1-py3-none-any.whl", hash = "sha256:d909fcccc110f8c7faf814ca82a9a4d816bc5a6dbfea25d6591d6985b8ba59ad", size = 73517, upload-time = "2024-12-06T15:37:21.509Z" }, ] [[package]] name = "huggingface-hub" -version = "0.35.3" +version = "1.23.0" source = { registry = "https://pypi.org/simple" } dependencies = [ + { name = "click" }, { name = "filelock" }, { name = "fsspec" }, - { name = "hf-xet", marker = "platform_machine == 'aarch64' or platform_machine == 'amd64' or platform_machine == 'arm64' or platform_machine == 'x86_64'" }, + { name = "hf-xet", marker = "platform_machine == 'AMD64' or platform_machine == 'aarch64' or platform_machine == 'amd64' or platform_machine == 'arm64' or platform_machine == 'x86_64'" }, + { name = "httpx" }, { name = "packaging" }, { name = "pyyaml" }, - { name = "requests" }, { name = "tqdm" }, { name = "typing-extensions" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/10/7e/a0a97de7c73671863ca6b3f61fa12518caf35db37825e43d63a70956738c/huggingface_hub-0.35.3.tar.gz", hash = "sha256:350932eaa5cc6a4747efae85126ee220e4ef1b54e29d31c3b45c5612ddf0b32a", size = 461798, upload-time = "2025-09-29T14:29:58.625Z" } +sdist = { url = "https://files.pythonhosted.org/packages/1a/8f/999e4dda11c6187c78f090eac00895a47e11a0049308f07579bcb7aa3aa2/huggingface_hub-1.23.0.tar.gz", hash = "sha256:c04997fb8bbdace1e57b7703d30ed7678af51f70d00d241819ff411b92ae9a88", size = 919163, upload-time = "2026-07-09T14:49:32.315Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/31/a0/651f93d154cb72323358bf2bbae3e642bdb5d2f1bfc874d096f7cb159fa0/huggingface_hub-0.35.3-py3-none-any.whl", hash = "sha256:0e3a01829c19d86d03793e4577816fe3bdfc1602ac62c7fb220d593d351224ba", size = 564262, upload-time = "2025-09-29T14:29:55.813Z" }, + { url = "https://files.pythonhosted.org/packages/f1/ce/13b2ba57838b8db1e6bd033c1b21ce0b9f6153b87d4e4939f77074e41eb0/huggingface_hub-1.23.0-py3-none-any.whl", hash = "sha256:b1d604788f5adc7f0eb246e03e0ec19011ca06e38400218c347dccc3dffa64a2", size = 770336, upload-time = "2026-07-09T14:49:30.597Z" }, ] [[package]] name = "idna" -version = "3.15" +version = "3.18" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/82/77/7b3966d0b9d1d31a36ddf1746926a11dface89a83409bf1483f0237aa758/idna-3.15.tar.gz", hash = "sha256:ca962446ea538f7092a95e057da437618e886f4d349216d2b1e294abfdb65fdc", size = 199245, upload-time = "2026-05-12T22:45:57.011Z" } +sdist = { url = "https://files.pythonhosted.org/packages/cd/63/9496c57188a2ee585e0f1db071d75089a11e98aa86eb99d9d7618fc1edce/idna-3.18.tar.gz", hash = "sha256:ffb385a7e039654cef1ab9ef32c6fafe283c0c0467bba1d9029738ce4a14a848", size = 196711, upload-time = "2026-06-02T14:34:07.794Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/d2/23/408243171aa9aaba178d3e2559159c24c1171a641aa83b67bdd3394ead8e/idna-3.15-py3-none-any.whl", hash = "sha256:048adeaf8c2d788c40fee287673ccaa74c24ffd8dcf09ffa555a2fbb59f10ac8", size = 72340, upload-time = "2026-05-12T22:45:55.733Z" }, + { url = "https://files.pythonhosted.org/packages/1e/5e/d4e9f1a599fb8e573b7b87160658329fbf28d19eac2718f51fc3def3aa5a/idna-3.18-py3-none-any.whl", hash = "sha256:7f952cbe720b688055e3f87de14f5c3e5fdaa8bc3928985c4077ca689de849a2", size = 65455, upload-time = "2026-06-02T14:34:06.319Z" }, +] + +[[package]] +name = "importlib-metadata" +version = "8.5.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "zipp" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/cd/12/33e59336dca5be0c398a7482335911a33aa0e20776128f038019f1a95f1b/importlib_metadata-8.5.0.tar.gz", hash = "sha256:71522656f0abace1d072b9e5481a48f07c138e00f079c38c8f883823f9c26bd7", size = 55304, upload-time = "2024-09-11T14:56:08.937Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a0/d9/a1e041c5e7caa9a05c925f4bdbdfb7f006d1f74996af53467bc394c97be7/importlib_metadata-8.5.0-py3-none-any.whl", hash = "sha256:45e54197d28b7a7f1559e60b95e7c567032b602131fbd588f1497f47880aa68b", size = 26514, upload-time = "2024-09-11T14:56:07.019Z" }, ] [[package]] name = "iniconfig" -version = "2.1.0" +version = "2.3.0" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/f2/97/ebf4da567aa6827c909642694d71c9fcf53e5b504f2d96afea02718862f3/iniconfig-2.1.0.tar.gz", hash = "sha256:3abbd2e30b36733fee78f9c7f7308f2d0050e88f0087fd25c2645f63c773e1c7", size = 4793, upload-time = "2025-03-19T20:09:59.721Z" } +sdist = { url = "https://files.pythonhosted.org/packages/72/34/14ca021ce8e5dfedc35312d08ba8bf51fdd999c576889fc2c24cb97f4f10/iniconfig-2.3.0.tar.gz", hash = "sha256:c76315c77db068650d49c5b56314774a7804df16fee4402c1f19d6d15d8c4730", size = 20503, upload-time = "2025-10-18T21:55:43.219Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/2c/e1/e6716421ea10d38022b952c159d5161ca1193197fb744506875fbb87ea7b/iniconfig-2.1.0-py3-none-any.whl", hash = "sha256:9deba5723312380e77435581c6bf4935c94cbfab9b1ed33ef8d238ea168eb760", size = 6050, upload-time = "2025-03-19T20:10:01.071Z" }, + { url = "https://files.pythonhosted.org/packages/cb/b1/3846dd7f199d53cb17f49cba7e651e9ce294d8497c8c150530ed11865bb8/iniconfig-2.3.0-py3-none-any.whl", hash = "sha256:f631c04d2c48c52b84d0d0549c99ff3859c98df65b3101406327ecc7d53fbf12", size = 7484, upload-time = "2025-10-18T21:55:41.639Z" }, ] [[package]] @@ -949,31 +1037,11 @@ wheels = [ [[package]] name = "jmespath" -version = "1.0.1" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/00/2a/e867e8531cf3e36b41201936b7fa7ba7b5702dbef42922193f05c8976cd6/jmespath-1.0.1.tar.gz", hash = "sha256:90261b206d6defd58fdd5e85f478bf633a2901798906be2ad389150c5c60edbe", size = 25843, upload-time = "2022-06-17T18:00:12.224Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/31/b4/b9b800c45527aadd64d5b442f9b932b00648617eb5d63d2c7a6587b7cafc/jmespath-1.0.1-py3-none-any.whl", hash = "sha256:02e2e4cc71b5bcab88332eebf907519190dd9e6e82107fa7f83b1003a6252980", size = 20256, upload-time = "2022-06-17T18:00:10.251Z" }, -] - -[[package]] -name = "keras" -version = "3.11.3" +version = "1.1.0" source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "absl-py" }, - { name = "h5py" }, - { name = "ml-dtypes" }, - { name = "namex" }, - { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, - { name = "numpy", version = "2.3.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" }, - { name = "optree" }, - { name = "packaging" }, - { name = "rich" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/6a/89/646425fe9a46f9053430e1271f817c36041c6f33469950a3caafc3d2591e/keras-3.11.3.tar.gz", hash = "sha256:efda616835c31b7d916d72303ef9adec1257320bc9fd4b2b0138840fc65fb5b7", size = 1065906, upload-time = "2025-08-21T22:08:57.643Z" } +sdist = { url = "https://files.pythonhosted.org/packages/d3/59/322338183ecda247fb5d1763a6cbe46eff7222eaeebafd9fa65d4bf5cb11/jmespath-1.1.0.tar.gz", hash = "sha256:472c87d80f36026ae83c6ddd0f1d05d4e510134ed462851fd5f754c8c3cbb88d", size = 27377, upload-time = "2026-01-22T16:35:26.279Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/94/5b/4c778cc921ce4b864b238f63f8e3ff6e954ab19b80c9fa680593ad8093d4/keras-3.11.3-py3-none-any.whl", hash = "sha256:f484f050e05ee400455b05ec8c36ed35edc34de94256b6073f56cfe68f65491f", size = 1408438, upload-time = "2025-08-21T22:08:55.858Z" }, + { url = "https://files.pythonhosted.org/packages/14/2f/967ba146e6d58cf6a652da73885f52fc68001525b4197effc174321d70b4/jmespath-1.1.0-py3-none-any.whl", hash = "sha256:a5663118de4908c91729bea0acadca56526eb2698e83de10cd116ae0f4e97c64", size = 20419, upload-time = "2026-01-22T16:35:24.919Z" }, ] [[package]] @@ -1003,39 +1071,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/c5/90/1e27de15cd1b16785a1c7312beb0a59e75c8344a815f600f58173a565bd1/lance_namespace_urllib3_client-0.8.6-py3-none-any.whl", hash = "sha256:9d78249c3fb15aa3d15d668f78f04a275af3d08d800a7027492f37996ac4968b", size = 369950, upload-time = "2026-06-12T17:36:40.438Z" }, ] -[[package]] -name = "libclang" -version = "18.1.1" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/6e/5c/ca35e19a4f142adffa27e3d652196b7362fa612243e2b916845d801454fc/libclang-18.1.1.tar.gz", hash = "sha256:a1214966d08d73d971287fc3ead8dfaf82eb07fb197680d8b3859dbbbbf78250", size = 39612, upload-time = "2024-03-17T16:04:37.434Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/1d/fc/716c1e62e512ef1c160e7984a73a5fc7df45166f2ff3f254e71c58076f7c/libclang-18.1.1-py2.py3-none-manylinux2010_x86_64.whl", hash = "sha256:c533091d8a3bbf7460a00cb6c1a71da93bffe148f172c7d03b1c31fbf8aa2a0b", size = 24515943, upload-time = "2024-03-17T16:03:45.942Z" }, - { url = "https://files.pythonhosted.org/packages/3c/3d/f0ac1150280d8d20d059608cf2d5ff61b7c3b7f7bcf9c0f425ab92df769a/libclang-18.1.1-py2.py3-none-manylinux2014_aarch64.whl", hash = "sha256:54dda940a4a0491a9d1532bf071ea3ef26e6dbaf03b5000ed94dd7174e8f9592", size = 23784972, upload-time = "2024-03-17T16:12:47.677Z" }, - { url = "https://files.pythonhosted.org/packages/fe/2f/d920822c2b1ce9326a4c78c0c2b4aa3fde610c7ee9f631b600acb5376c26/libclang-18.1.1-py2.py3-none-manylinux2014_armv7l.whl", hash = "sha256:cf4a99b05376513717ab5d82a0db832c56ccea4fd61a69dbb7bccf2dfb207dbe", size = 20259606, upload-time = "2024-03-17T16:17:42.437Z" }, - { url = "https://files.pythonhosted.org/packages/2d/c2/de1db8c6d413597076a4259cea409b83459b2db997c003578affdd32bf66/libclang-18.1.1-py2.py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:69f8eb8f65c279e765ffd28aaa7e9e364c776c17618af8bff22a8df58677ff4f", size = 24921494, upload-time = "2024-03-17T16:14:20.132Z" }, -] - -[[package]] -name = "markdown" -version = "3.9" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/8d/37/02347f6d6d8279247a5837082ebc26fc0d5aaeaf75aa013fcbb433c777ab/markdown-3.9.tar.gz", hash = "sha256:d2900fe1782bd33bdbbd56859defef70c2e78fc46668f8eb9df3128138f2cb6a", size = 364585, upload-time = "2025-09-04T20:25:22.885Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/70/ae/44c4a6a4cbb496d93c6257954260fe3a6e91b7bed2240e5dad2a717f5111/markdown-3.9-py3-none-any.whl", hash = "sha256:9f4d91ed810864ea88a6f32c07ba8bee1346c0cc1f6b1f9f6c822f2a9667d280", size = 107441, upload-time = "2025-09-04T20:25:21.784Z" }, -] - -[[package]] -name = "markdown-it-py" -version = "4.0.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "mdurl" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/5b/f5/4ec618ed16cc4f8fb3b701563655a69816155e79e24a17b651541804721d/markdown_it_py-4.0.0.tar.gz", hash = "sha256:cb0a2b4aa34f932c007117b194e945bd74e0ec24133ceb5bac59009cda1cb9f3", size = 73070, upload-time = "2025-08-11T12:57:52.854Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/94/54/e7d793b573f298e1c9013b8c4dade17d481164aa517d1d7148619c2cedbf/markdown_it_py-4.0.0-py3-none-any.whl", hash = "sha256:87327c59b172c5011896038353a81343b6754500a08cd7a4973bb48c6d578147", size = 87321, upload-time = "2025-08-11T12:57:51.923Z" }, -] - [[package]] name = "markupsafe" version = "3.0.3" @@ -1145,22 +1180,14 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/6c/93/e32e79333f0902ba292b996f504f5f06be59587f7d02ab8d5ed1e3066445/maturin-1.13.3-py3-none-win_arm64.whl", hash = "sha256:2389fe92d017cea9d94e521fa0175314a4c52f79a1057b901fbc9f8686ef7d0b", size = 9706562, upload-time = "2026-05-11T07:43:31.743Z" }, ] -[[package]] -name = "mdurl" -version = "0.1.2" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/d6/54/cfe61301667036ec958cb99bd3efefba235e65cdeb9c84d24a8293ba1d90/mdurl-0.1.2.tar.gz", hash = "sha256:bb413d29f5eea38f31dd4754dd7377d4465116fb207585f97bf925588687c1ba", size = 8729, upload-time = "2022-08-14T12:40:10.846Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/b3/38/89ba8ad64ae25be8de66a6d463314cf1eb366222074cfda9ee839c56a4b4/mdurl-0.1.2-py3-none-any.whl", hash = "sha256:84008a41e51615a49fc9966191ff91509e3c40b939176e643fd50a5c2196b8f8", size = 9979, upload-time = "2022-08-14T12:40:09.779Z" }, -] - [[package]] name = "ml-dtypes" version = "0.5.3" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, - { name = "numpy", version = "2.3.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" }, + { name = "numpy", version = "2.4.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.11.*'" }, + { name = "numpy", version = "2.5.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.12'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/78/a7/aad060393123cfb383956dca68402aff3db1e1caffd5764887ed5153f41b/ml_dtypes-0.5.3.tar.gz", hash = "sha256:95ce33057ba4d05df50b1f3cfefab22e351868a843b3b15a46c65836283670c9", size = 692316, upload-time = "2025-07-29T18:39:19.454Z" } wheels = [ @@ -1207,131 +1234,163 @@ wheels = [ [[package]] name = "multidict" -version = "6.6.4" +version = "6.7.1" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "typing-extensions", marker = "python_full_version < '3.11'" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/69/7f/0652e6ed47ab288e3756ea9c0df8b14950781184d4bd7883f4d87dd41245/multidict-6.6.4.tar.gz", hash = "sha256:d2d4e4787672911b48350df02ed3fa3fffdc2f2e8ca06dd6afdf34189b76a9dd", size = 101843, upload-time = "2025-08-11T12:08:48.217Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/eb/6b/86f353088c1358e76fd30b0146947fddecee812703b604ee901e85cd2a80/multidict-6.6.4-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:b8aa6f0bd8125ddd04a6593437bad6a7e70f300ff4180a531654aa2ab3f6d58f", size = 77054, upload-time = "2025-08-11T12:06:02.99Z" }, - { url = "https://files.pythonhosted.org/packages/19/5d/c01dc3d3788bb877bd7f5753ea6eb23c1beeca8044902a8f5bfb54430f63/multidict-6.6.4-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:b9e5853bbd7264baca42ffc53391b490d65fe62849bf2c690fa3f6273dbcd0cb", size = 44914, upload-time = "2025-08-11T12:06:05.264Z" }, - { url = "https://files.pythonhosted.org/packages/46/44/964dae19ea42f7d3e166474d8205f14bb811020e28bc423d46123ddda763/multidict-6.6.4-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:0af5f9dee472371e36d6ae38bde009bd8ce65ac7335f55dcc240379d7bed1495", size = 44601, upload-time = "2025-08-11T12:06:06.627Z" }, - { url = "https://files.pythonhosted.org/packages/31/20/0616348a1dfb36cb2ab33fc9521de1f27235a397bf3f59338e583afadd17/multidict-6.6.4-cp310-cp310-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:d24f351e4d759f5054b641c81e8291e5d122af0fca5c72454ff77f7cbe492de8", size = 224821, upload-time = "2025-08-11T12:06:08.06Z" }, - { url = "https://files.pythonhosted.org/packages/14/26/5d8923c69c110ff51861af05bd27ca6783011b96725d59ccae6d9daeb627/multidict-6.6.4-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:db6a3810eec08280a172a6cd541ff4a5f6a97b161d93ec94e6c4018917deb6b7", size = 242608, upload-time = "2025-08-11T12:06:09.697Z" }, - { url = "https://files.pythonhosted.org/packages/5c/cc/e2ad3ba9459aa34fa65cf1f82a5c4a820a2ce615aacfb5143b8817f76504/multidict-6.6.4-cp310-cp310-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:a1b20a9d56b2d81e2ff52ecc0670d583eaabaa55f402e8d16dd062373dbbe796", size = 222324, upload-time = "2025-08-11T12:06:10.905Z" }, - { url = "https://files.pythonhosted.org/packages/19/db/4ed0f65701afbc2cb0c140d2d02928bb0fe38dd044af76e58ad7c54fd21f/multidict-6.6.4-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:8c9854df0eaa610a23494c32a6f44a3a550fb398b6b51a56e8c6b9b3689578db", size = 253234, upload-time = "2025-08-11T12:06:12.658Z" }, - { url = "https://files.pythonhosted.org/packages/94/c1/5160c9813269e39ae14b73debb907bfaaa1beee1762da8c4fb95df4764ed/multidict-6.6.4-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:4bb7627fd7a968f41905a4d6343b0d63244a0623f006e9ed989fa2b78f4438a0", size = 251613, upload-time = "2025-08-11T12:06:13.97Z" }, - { url = "https://files.pythonhosted.org/packages/05/a9/48d1bd111fc2f8fb98b2ed7f9a115c55a9355358432a19f53c0b74d8425d/multidict-6.6.4-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:caebafea30ed049c57c673d0b36238b1748683be2593965614d7b0e99125c877", size = 241649, upload-time = "2025-08-11T12:06:15.204Z" }, - { url = "https://files.pythonhosted.org/packages/85/2a/f7d743df0019408768af8a70d2037546a2be7b81fbb65f040d76caafd4c5/multidict-6.6.4-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:ad887a8250eb47d3ab083d2f98db7f48098d13d42eb7a3b67d8a5c795f224ace", size = 239238, upload-time = "2025-08-11T12:06:16.467Z" }, - { url = "https://files.pythonhosted.org/packages/cb/b8/4f4bb13323c2d647323f7919201493cf48ebe7ded971717bfb0f1a79b6bf/multidict-6.6.4-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:ed8358ae7d94ffb7c397cecb62cbac9578a83ecefc1eba27b9090ee910e2efb6", size = 233517, upload-time = "2025-08-11T12:06:18.107Z" }, - { url = "https://files.pythonhosted.org/packages/33/29/4293c26029ebfbba4f574febd2ed01b6f619cfa0d2e344217d53eef34192/multidict-6.6.4-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:ecab51ad2462197a4c000b6d5701fc8585b80eecb90583635d7e327b7b6923eb", size = 243122, upload-time = "2025-08-11T12:06:19.361Z" }, - { url = "https://files.pythonhosted.org/packages/20/60/a1c53628168aa22447bfde3a8730096ac28086704a0d8c590f3b63388d0c/multidict-6.6.4-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:c5c97aa666cf70e667dfa5af945424ba1329af5dd988a437efeb3a09430389fb", size = 248992, upload-time = "2025-08-11T12:06:20.661Z" }, - { url = "https://files.pythonhosted.org/packages/a3/3b/55443a0c372f33cae5d9ec37a6a973802884fa0ab3586659b197cf8cc5e9/multidict-6.6.4-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:9a950b7cf54099c1209f455ac5970b1ea81410f2af60ed9eb3c3f14f0bfcf987", size = 243708, upload-time = "2025-08-11T12:06:21.891Z" }, - { url = "https://files.pythonhosted.org/packages/7c/60/a18c6900086769312560b2626b18e8cca22d9e85b1186ba77f4755b11266/multidict-6.6.4-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:163c7ea522ea9365a8a57832dea7618e6cbdc3cd75f8c627663587459a4e328f", size = 237498, upload-time = "2025-08-11T12:06:23.206Z" }, - { url = "https://files.pythonhosted.org/packages/11/3d/8bdd8bcaff2951ce2affccca107a404925a2beafedd5aef0b5e4a71120a6/multidict-6.6.4-cp310-cp310-win32.whl", hash = "sha256:17d2cbbfa6ff20821396b25890f155f40c986f9cfbce5667759696d83504954f", size = 41415, upload-time = "2025-08-11T12:06:24.77Z" }, - { url = "https://files.pythonhosted.org/packages/c0/53/cab1ad80356a4cd1b685a254b680167059b433b573e53872fab245e9fc95/multidict-6.6.4-cp310-cp310-win_amd64.whl", hash = "sha256:ce9a40fbe52e57e7edf20113a4eaddfacac0561a0879734e636aa6d4bb5e3fb0", size = 46046, upload-time = "2025-08-11T12:06:25.893Z" }, - { url = "https://files.pythonhosted.org/packages/cf/9a/874212b6f5c1c2d870d0a7adc5bb4cfe9b0624fa15cdf5cf757c0f5087ae/multidict-6.6.4-cp310-cp310-win_arm64.whl", hash = "sha256:01d0959807a451fe9fdd4da3e139cb5b77f7328baf2140feeaf233e1d777b729", size = 43147, upload-time = "2025-08-11T12:06:27.534Z" }, - { url = "https://files.pythonhosted.org/packages/6b/7f/90a7f01e2d005d6653c689039977f6856718c75c5579445effb7e60923d1/multidict-6.6.4-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:c7a0e9b561e6460484318a7612e725df1145d46b0ef57c6b9866441bf6e27e0c", size = 76472, upload-time = "2025-08-11T12:06:29.006Z" }, - { url = "https://files.pythonhosted.org/packages/54/a3/bed07bc9e2bb302ce752f1dabc69e884cd6a676da44fb0e501b246031fdd/multidict-6.6.4-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:6bf2f10f70acc7a2446965ffbc726e5fc0b272c97a90b485857e5c70022213eb", size = 44634, upload-time = "2025-08-11T12:06:30.374Z" }, - { url = "https://files.pythonhosted.org/packages/a7/4b/ceeb4f8f33cf81277da464307afeaf164fb0297947642585884f5cad4f28/multidict-6.6.4-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:66247d72ed62d5dd29752ffc1d3b88f135c6a8de8b5f63b7c14e973ef5bda19e", size = 44282, upload-time = "2025-08-11T12:06:31.958Z" }, - { url = "https://files.pythonhosted.org/packages/03/35/436a5da8702b06866189b69f655ffdb8f70796252a8772a77815f1812679/multidict-6.6.4-cp311-cp311-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:105245cc6b76f51e408451a844a54e6823bbd5a490ebfe5bdfc79798511ceded", size = 229696, upload-time = "2025-08-11T12:06:33.087Z" }, - { url = "https://files.pythonhosted.org/packages/b6/0e/915160be8fecf1fca35f790c08fb74ca684d752fcba62c11daaf3d92c216/multidict-6.6.4-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:cbbc54e58b34c3bae389ef00046be0961f30fef7cb0dd9c7756aee376a4f7683", size = 246665, upload-time = "2025-08-11T12:06:34.448Z" }, - { url = "https://files.pythonhosted.org/packages/08/ee/2f464330acd83f77dcc346f0b1a0eaae10230291450887f96b204b8ac4d3/multidict-6.6.4-cp311-cp311-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:56c6b3652f945c9bc3ac6c8178cd93132b8d82dd581fcbc3a00676c51302bc1a", size = 225485, upload-time = "2025-08-11T12:06:35.672Z" }, - { url = "https://files.pythonhosted.org/packages/71/cc/9a117f828b4d7fbaec6adeed2204f211e9caf0a012692a1ee32169f846ae/multidict-6.6.4-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:b95494daf857602eccf4c18ca33337dd2be705bccdb6dddbfc9d513e6addb9d9", size = 257318, upload-time = "2025-08-11T12:06:36.98Z" }, - { url = "https://files.pythonhosted.org/packages/25/77/62752d3dbd70e27fdd68e86626c1ae6bccfebe2bb1f84ae226363e112f5a/multidict-6.6.4-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:e5b1413361cef15340ab9dc61523e653d25723e82d488ef7d60a12878227ed50", size = 254689, upload-time = "2025-08-11T12:06:38.233Z" }, - { url = "https://files.pythonhosted.org/packages/00/6e/fac58b1072a6fc59af5e7acb245e8754d3e1f97f4f808a6559951f72a0d4/multidict-6.6.4-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e167bf899c3d724f9662ef00b4f7fef87a19c22b2fead198a6f68b263618df52", size = 246709, upload-time = "2025-08-11T12:06:39.517Z" }, - { url = "https://files.pythonhosted.org/packages/01/ef/4698d6842ef5e797c6db7744b0081e36fb5de3d00002cc4c58071097fac3/multidict-6.6.4-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:aaea28ba20a9026dfa77f4b80369e51cb767c61e33a2d4043399c67bd95fb7c6", size = 243185, upload-time = "2025-08-11T12:06:40.796Z" }, - { url = "https://files.pythonhosted.org/packages/aa/c9/d82e95ae1d6e4ef396934e9b0e942dfc428775f9554acf04393cce66b157/multidict-6.6.4-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:8c91cdb30809a96d9ecf442ec9bc45e8cfaa0f7f8bdf534e082c2443a196727e", size = 237838, upload-time = "2025-08-11T12:06:42.595Z" }, - { url = "https://files.pythonhosted.org/packages/57/cf/f94af5c36baaa75d44fab9f02e2a6bcfa0cd90acb44d4976a80960759dbc/multidict-6.6.4-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:1a0ccbfe93ca114c5d65a2471d52d8829e56d467c97b0e341cf5ee45410033b3", size = 246368, upload-time = "2025-08-11T12:06:44.304Z" }, - { url = "https://files.pythonhosted.org/packages/4a/fe/29f23460c3d995f6a4b678cb2e9730e7277231b981f0b234702f0177818a/multidict-6.6.4-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:55624b3f321d84c403cb7d8e6e982f41ae233d85f85db54ba6286f7295dc8a9c", size = 253339, upload-time = "2025-08-11T12:06:45.597Z" }, - { url = "https://files.pythonhosted.org/packages/29/b6/fd59449204426187b82bf8a75f629310f68c6adc9559dc922d5abe34797b/multidict-6.6.4-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:4a1fb393a2c9d202cb766c76208bd7945bc194eba8ac920ce98c6e458f0b524b", size = 246933, upload-time = "2025-08-11T12:06:46.841Z" }, - { url = "https://files.pythonhosted.org/packages/19/52/d5d6b344f176a5ac3606f7a61fb44dc746e04550e1a13834dff722b8d7d6/multidict-6.6.4-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:43868297a5759a845fa3a483fb4392973a95fb1de891605a3728130c52b8f40f", size = 242225, upload-time = "2025-08-11T12:06:48.588Z" }, - { url = "https://files.pythonhosted.org/packages/ec/d3/5b2281ed89ff4d5318d82478a2a2450fcdfc3300da48ff15c1778280ad26/multidict-6.6.4-cp311-cp311-win32.whl", hash = "sha256:ed3b94c5e362a8a84d69642dbeac615452e8af9b8eb825b7bc9f31a53a1051e2", size = 41306, upload-time = "2025-08-11T12:06:49.95Z" }, - { url = "https://files.pythonhosted.org/packages/74/7d/36b045c23a1ab98507aefd44fd8b264ee1dd5e5010543c6fccf82141ccef/multidict-6.6.4-cp311-cp311-win_amd64.whl", hash = "sha256:d8c112f7a90d8ca5d20213aa41eac690bb50a76da153e3afb3886418e61cb22e", size = 46029, upload-time = "2025-08-11T12:06:51.082Z" }, - { url = "https://files.pythonhosted.org/packages/0f/5e/553d67d24432c5cd52b49047f2d248821843743ee6d29a704594f656d182/multidict-6.6.4-cp311-cp311-win_arm64.whl", hash = "sha256:3bb0eae408fa1996d87247ca0d6a57b7fc1dcf83e8a5c47ab82c558c250d4adf", size = 43017, upload-time = "2025-08-11T12:06:52.243Z" }, - { url = "https://files.pythonhosted.org/packages/05/f6/512ffd8fd8b37fb2680e5ac35d788f1d71bbaf37789d21a820bdc441e565/multidict-6.6.4-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:0ffb87be160942d56d7b87b0fdf098e81ed565add09eaa1294268c7f3caac4c8", size = 76516, upload-time = "2025-08-11T12:06:53.393Z" }, - { url = "https://files.pythonhosted.org/packages/99/58/45c3e75deb8855c36bd66cc1658007589662ba584dbf423d01df478dd1c5/multidict-6.6.4-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:d191de6cbab2aff5de6c5723101705fd044b3e4c7cfd587a1929b5028b9714b3", size = 45394, upload-time = "2025-08-11T12:06:54.555Z" }, - { url = "https://files.pythonhosted.org/packages/fd/ca/e8c4472a93a26e4507c0b8e1f0762c0d8a32de1328ef72fd704ef9cc5447/multidict-6.6.4-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:38a0956dd92d918ad5feff3db8fcb4a5eb7dba114da917e1a88475619781b57b", size = 43591, upload-time = "2025-08-11T12:06:55.672Z" }, - { url = "https://files.pythonhosted.org/packages/05/51/edf414f4df058574a7265034d04c935aa84a89e79ce90fcf4df211f47b16/multidict-6.6.4-cp312-cp312-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:6865f6d3b7900ae020b495d599fcf3765653bc927951c1abb959017f81ae8287", size = 237215, upload-time = "2025-08-11T12:06:57.213Z" }, - { url = "https://files.pythonhosted.org/packages/c8/45/8b3d6dbad8cf3252553cc41abea09ad527b33ce47a5e199072620b296902/multidict-6.6.4-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0a2088c126b6f72db6c9212ad827d0ba088c01d951cee25e758c450da732c138", size = 258299, upload-time = "2025-08-11T12:06:58.946Z" }, - { url = "https://files.pythonhosted.org/packages/3c/e8/8ca2e9a9f5a435fc6db40438a55730a4bf4956b554e487fa1b9ae920f825/multidict-6.6.4-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:0f37bed7319b848097085d7d48116f545985db988e2256b2e6f00563a3416ee6", size = 242357, upload-time = "2025-08-11T12:07:00.301Z" }, - { url = "https://files.pythonhosted.org/packages/0f/84/80c77c99df05a75c28490b2af8f7cba2a12621186e0a8b0865d8e745c104/multidict-6.6.4-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:01368e3c94032ba6ca0b78e7ccb099643466cf24f8dc8eefcfdc0571d56e58f9", size = 268369, upload-time = "2025-08-11T12:07:01.638Z" }, - { url = "https://files.pythonhosted.org/packages/0d/e9/920bfa46c27b05fb3e1ad85121fd49f441492dca2449c5bcfe42e4565d8a/multidict-6.6.4-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:8fe323540c255db0bffee79ad7f048c909f2ab0edb87a597e1c17da6a54e493c", size = 269341, upload-time = "2025-08-11T12:07:02.943Z" }, - { url = "https://files.pythonhosted.org/packages/af/65/753a2d8b05daf496f4a9c367fe844e90a1b2cac78e2be2c844200d10cc4c/multidict-6.6.4-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b8eb3025f17b0a4c3cd08cda49acf312a19ad6e8a4edd9dbd591e6506d999402", size = 256100, upload-time = "2025-08-11T12:07:04.564Z" }, - { url = "https://files.pythonhosted.org/packages/09/54/655be13ae324212bf0bc15d665a4e34844f34c206f78801be42f7a0a8aaa/multidict-6.6.4-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:bbc14f0365534d35a06970d6a83478b249752e922d662dc24d489af1aa0d1be7", size = 253584, upload-time = "2025-08-11T12:07:05.914Z" }, - { url = "https://files.pythonhosted.org/packages/5c/74/ab2039ecc05264b5cec73eb018ce417af3ebb384ae9c0e9ed42cb33f8151/multidict-6.6.4-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:75aa52fba2d96bf972e85451b99d8e19cc37ce26fd016f6d4aa60da9ab2b005f", size = 251018, upload-time = "2025-08-11T12:07:08.301Z" }, - { url = "https://files.pythonhosted.org/packages/af/0a/ccbb244ac848e56c6427f2392741c06302bbfba49c0042f1eb3c5b606497/multidict-6.6.4-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:4fefd4a815e362d4f011919d97d7b4a1e566f1dde83dc4ad8cfb5b41de1df68d", size = 251477, upload-time = "2025-08-11T12:07:10.248Z" }, - { url = "https://files.pythonhosted.org/packages/0e/b0/0ed49bba775b135937f52fe13922bc64a7eaf0a3ead84a36e8e4e446e096/multidict-6.6.4-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:db9801fe021f59a5b375ab778973127ca0ac52429a26e2fd86aa9508f4d26eb7", size = 263575, upload-time = "2025-08-11T12:07:11.928Z" }, - { url = "https://files.pythonhosted.org/packages/3e/d9/7fb85a85e14de2e44dfb6a24f03c41e2af8697a6df83daddb0e9b7569f73/multidict-6.6.4-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:a650629970fa21ac1fb06ba25dabfc5b8a2054fcbf6ae97c758aa956b8dba802", size = 259649, upload-time = "2025-08-11T12:07:13.244Z" }, - { url = "https://files.pythonhosted.org/packages/03/9e/b3a459bcf9b6e74fa461a5222a10ff9b544cb1cd52fd482fb1b75ecda2a2/multidict-6.6.4-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:452ff5da78d4720d7516a3a2abd804957532dd69296cb77319c193e3ffb87e24", size = 251505, upload-time = "2025-08-11T12:07:14.57Z" }, - { url = "https://files.pythonhosted.org/packages/86/a2/8022f78f041dfe6d71e364001a5cf987c30edfc83c8a5fb7a3f0974cff39/multidict-6.6.4-cp312-cp312-win32.whl", hash = "sha256:8c2fcb12136530ed19572bbba61b407f655e3953ba669b96a35036a11a485793", size = 41888, upload-time = "2025-08-11T12:07:15.904Z" }, - { url = "https://files.pythonhosted.org/packages/c7/eb/d88b1780d43a56db2cba24289fa744a9d216c1a8546a0dc3956563fd53ea/multidict-6.6.4-cp312-cp312-win_amd64.whl", hash = "sha256:047d9425860a8c9544fed1b9584f0c8bcd31bcde9568b047c5e567a1025ecd6e", size = 46072, upload-time = "2025-08-11T12:07:17.045Z" }, - { url = "https://files.pythonhosted.org/packages/9f/16/b929320bf5750e2d9d4931835a4c638a19d2494a5b519caaaa7492ebe105/multidict-6.6.4-cp312-cp312-win_arm64.whl", hash = "sha256:14754eb72feaa1e8ae528468f24250dd997b8e2188c3d2f593f9eba259e4b364", size = 43222, upload-time = "2025-08-11T12:07:18.328Z" }, - { url = "https://files.pythonhosted.org/packages/3a/5d/e1db626f64f60008320aab00fbe4f23fc3300d75892a3381275b3d284580/multidict-6.6.4-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:f46a6e8597f9bd71b31cc708195d42b634c8527fecbcf93febf1052cacc1f16e", size = 75848, upload-time = "2025-08-11T12:07:19.912Z" }, - { url = "https://files.pythonhosted.org/packages/4c/aa/8b6f548d839b6c13887253af4e29c939af22a18591bfb5d0ee6f1931dae8/multidict-6.6.4-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:22e38b2bc176c5eb9c0a0e379f9d188ae4cd8b28c0f53b52bce7ab0a9e534657", size = 45060, upload-time = "2025-08-11T12:07:21.163Z" }, - { url = "https://files.pythonhosted.org/packages/eb/c6/f5e97e5d99a729bc2aa58eb3ebfa9f1e56a9b517cc38c60537c81834a73f/multidict-6.6.4-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:5df8afd26f162da59e218ac0eefaa01b01b2e6cd606cffa46608f699539246da", size = 43269, upload-time = "2025-08-11T12:07:22.392Z" }, - { url = "https://files.pythonhosted.org/packages/dc/31/d54eb0c62516776f36fe67f84a732f97e0b0e12f98d5685bebcc6d396910/multidict-6.6.4-cp313-cp313-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:49517449b58d043023720aa58e62b2f74ce9b28f740a0b5d33971149553d72aa", size = 237158, upload-time = "2025-08-11T12:07:23.636Z" }, - { url = "https://files.pythonhosted.org/packages/c4/1c/8a10c1c25b23156e63b12165a929d8eb49a6ed769fdbefb06e6f07c1e50d/multidict-6.6.4-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ae9408439537c5afdca05edd128a63f56a62680f4b3c234301055d7a2000220f", size = 257076, upload-time = "2025-08-11T12:07:25.049Z" }, - { url = "https://files.pythonhosted.org/packages/ad/86/90e20b5771d6805a119e483fd3d1e8393e745a11511aebca41f0da38c3e2/multidict-6.6.4-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:87a32d20759dc52a9e850fe1061b6e41ab28e2998d44168a8a341b99ded1dba0", size = 240694, upload-time = "2025-08-11T12:07:26.458Z" }, - { url = "https://files.pythonhosted.org/packages/e7/49/484d3e6b535bc0555b52a0a26ba86e4d8d03fd5587d4936dc59ba7583221/multidict-6.6.4-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:52e3c8d43cdfff587ceedce9deb25e6ae77daba560b626e97a56ddcad3756879", size = 266350, upload-time = "2025-08-11T12:07:27.94Z" }, - { url = "https://files.pythonhosted.org/packages/bf/b4/aa4c5c379b11895083d50021e229e90c408d7d875471cb3abf721e4670d6/multidict-6.6.4-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:ad8850921d3a8d8ff6fbef790e773cecfc260bbfa0566998980d3fa8f520bc4a", size = 267250, upload-time = "2025-08-11T12:07:29.303Z" }, - { url = "https://files.pythonhosted.org/packages/80/e5/5e22c5bf96a64bdd43518b1834c6d95a4922cc2066b7d8e467dae9b6cee6/multidict-6.6.4-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:497a2954adc25c08daff36f795077f63ad33e13f19bfff7736e72c785391534f", size = 254900, upload-time = "2025-08-11T12:07:30.764Z" }, - { url = "https://files.pythonhosted.org/packages/17/38/58b27fed927c07035abc02befacab42491e7388ca105e087e6e0215ead64/multidict-6.6.4-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:024ce601f92d780ca1617ad4be5ac15b501cc2414970ffa2bb2bbc2bd5a68fa5", size = 252355, upload-time = "2025-08-11T12:07:32.205Z" }, - { url = "https://files.pythonhosted.org/packages/d0/a1/dad75d23a90c29c02b5d6f3d7c10ab36c3197613be5d07ec49c7791e186c/multidict-6.6.4-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:a693fc5ed9bdd1c9e898013e0da4dcc640de7963a371c0bd458e50e046bf6438", size = 250061, upload-time = "2025-08-11T12:07:33.623Z" }, - { url = "https://files.pythonhosted.org/packages/b8/1a/ac2216b61c7f116edab6dc3378cca6c70dc019c9a457ff0d754067c58b20/multidict-6.6.4-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:190766dac95aab54cae5b152a56520fd99298f32a1266d66d27fdd1b5ac00f4e", size = 249675, upload-time = "2025-08-11T12:07:34.958Z" }, - { url = "https://files.pythonhosted.org/packages/d4/79/1916af833b800d13883e452e8e0977c065c4ee3ab7a26941fbfdebc11895/multidict-6.6.4-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:34d8f2a5ffdceab9dcd97c7a016deb2308531d5f0fced2bb0c9e1df45b3363d7", size = 261247, upload-time = "2025-08-11T12:07:36.588Z" }, - { url = "https://files.pythonhosted.org/packages/c5/65/d1f84fe08ac44a5fc7391cbc20a7cedc433ea616b266284413fd86062f8c/multidict-6.6.4-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:59e8d40ab1f5a8597abcef00d04845155a5693b5da00d2c93dbe88f2050f2812", size = 257960, upload-time = "2025-08-11T12:07:39.735Z" }, - { url = "https://files.pythonhosted.org/packages/13/b5/29ec78057d377b195ac2c5248c773703a6b602e132a763e20ec0457e7440/multidict-6.6.4-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:467fe64138cfac771f0e949b938c2e1ada2b5af22f39692aa9258715e9ea613a", size = 250078, upload-time = "2025-08-11T12:07:41.525Z" }, - { url = "https://files.pythonhosted.org/packages/c4/0e/7e79d38f70a872cae32e29b0d77024bef7834b0afb406ddae6558d9e2414/multidict-6.6.4-cp313-cp313-win32.whl", hash = "sha256:14616a30fe6d0a48d0a48d1a633ab3b8bec4cf293aac65f32ed116f620adfd69", size = 41708, upload-time = "2025-08-11T12:07:43.405Z" }, - { url = "https://files.pythonhosted.org/packages/9d/34/746696dffff742e97cd6a23da953e55d0ea51fa601fa2ff387b3edcfaa2c/multidict-6.6.4-cp313-cp313-win_amd64.whl", hash = "sha256:40cd05eaeb39e2bc8939451f033e57feaa2ac99e07dbca8afe2be450a4a3b6cf", size = 45912, upload-time = "2025-08-11T12:07:45.082Z" }, - { url = "https://files.pythonhosted.org/packages/c7/87/3bac136181e271e29170d8d71929cdeddeb77f3e8b6a0c08da3a8e9da114/multidict-6.6.4-cp313-cp313-win_arm64.whl", hash = "sha256:f6eb37d511bfae9e13e82cb4d1af36b91150466f24d9b2b8a9785816deb16605", size = 43076, upload-time = "2025-08-11T12:07:46.746Z" }, - { url = "https://files.pythonhosted.org/packages/64/94/0a8e63e36c049b571c9ae41ee301ada29c3fee9643d9c2548d7d558a1d99/multidict-6.6.4-cp313-cp313t-macosx_10_13_universal2.whl", hash = "sha256:6c84378acd4f37d1b507dfa0d459b449e2321b3ba5f2338f9b085cf7a7ba95eb", size = 82812, upload-time = "2025-08-11T12:07:48.402Z" }, - { url = "https://files.pythonhosted.org/packages/25/1a/be8e369dfcd260d2070a67e65dd3990dd635cbd735b98da31e00ea84cd4e/multidict-6.6.4-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:0e0558693063c75f3d952abf645c78f3c5dfdd825a41d8c4d8156fc0b0da6e7e", size = 48313, upload-time = "2025-08-11T12:07:49.679Z" }, - { url = "https://files.pythonhosted.org/packages/26/5a/dd4ade298674b2f9a7b06a32c94ffbc0497354df8285f27317c66433ce3b/multidict-6.6.4-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:3f8e2384cb83ebd23fd07e9eada8ba64afc4c759cd94817433ab8c81ee4b403f", size = 46777, upload-time = "2025-08-11T12:07:51.318Z" }, - { url = "https://files.pythonhosted.org/packages/89/db/98aa28bc7e071bfba611ac2ae803c24e96dd3a452b4118c587d3d872c64c/multidict-6.6.4-cp313-cp313t-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:f996b87b420995a9174b2a7c1a8daf7db4750be6848b03eb5e639674f7963773", size = 229321, upload-time = "2025-08-11T12:07:52.965Z" }, - { url = "https://files.pythonhosted.org/packages/c7/bc/01ddda2a73dd9d167bd85d0e8ef4293836a8f82b786c63fb1a429bc3e678/multidict-6.6.4-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:cc356250cffd6e78416cf5b40dc6a74f1edf3be8e834cf8862d9ed5265cf9b0e", size = 249954, upload-time = "2025-08-11T12:07:54.423Z" }, - { url = "https://files.pythonhosted.org/packages/06/78/6b7c0f020f9aa0acf66d0ab4eb9f08375bac9a50ff5e3edb1c4ccd59eafc/multidict-6.6.4-cp313-cp313t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:dadf95aa862714ea468a49ad1e09fe00fcc9ec67d122f6596a8d40caf6cec7d0", size = 228612, upload-time = "2025-08-11T12:07:55.914Z" }, - { url = "https://files.pythonhosted.org/packages/00/44/3faa416f89b2d5d76e9d447296a81521e1c832ad6e40b92f990697b43192/multidict-6.6.4-cp313-cp313t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:7dd57515bebffd8ebd714d101d4c434063322e4fe24042e90ced41f18b6d3395", size = 257528, upload-time = "2025-08-11T12:07:57.371Z" }, - { url = "https://files.pythonhosted.org/packages/05/5f/77c03b89af0fcb16f018f668207768191fb9dcfb5e3361a5e706a11db2c9/multidict-6.6.4-cp313-cp313t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:967af5f238ebc2eb1da4e77af5492219fbd9b4b812347da39a7b5f5c72c0fa45", size = 256329, upload-time = "2025-08-11T12:07:58.844Z" }, - { url = "https://files.pythonhosted.org/packages/cf/e9/ed750a2a9afb4f8dc6f13dc5b67b514832101b95714f1211cd42e0aafc26/multidict-6.6.4-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:2a4c6875c37aae9794308ec43e3530e4aa0d36579ce38d89979bbf89582002bb", size = 247928, upload-time = "2025-08-11T12:08:01.037Z" }, - { url = "https://files.pythonhosted.org/packages/1f/b5/e0571bc13cda277db7e6e8a532791d4403dacc9850006cb66d2556e649c0/multidict-6.6.4-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:7f683a551e92bdb7fac545b9c6f9fa2aebdeefa61d607510b3533286fcab67f5", size = 245228, upload-time = "2025-08-11T12:08:02.96Z" }, - { url = "https://files.pythonhosted.org/packages/f3/a3/69a84b0eccb9824491f06368f5b86e72e4af54c3067c37c39099b6687109/multidict-6.6.4-cp313-cp313t-musllinux_1_2_armv7l.whl", hash = "sha256:3ba5aaf600edaf2a868a391779f7a85d93bed147854925f34edd24cc70a3e141", size = 235869, upload-time = "2025-08-11T12:08:04.746Z" }, - { url = "https://files.pythonhosted.org/packages/a9/9d/28802e8f9121a6a0804fa009debf4e753d0a59969ea9f70be5f5fdfcb18f/multidict-6.6.4-cp313-cp313t-musllinux_1_2_i686.whl", hash = "sha256:580b643b7fd2c295d83cad90d78419081f53fd532d1f1eb67ceb7060f61cff0d", size = 243446, upload-time = "2025-08-11T12:08:06.332Z" }, - { url = "https://files.pythonhosted.org/packages/38/ea/6c98add069b4878c1d66428a5f5149ddb6d32b1f9836a826ac764b9940be/multidict-6.6.4-cp313-cp313t-musllinux_1_2_ppc64le.whl", hash = "sha256:37b7187197da6af3ee0b044dbc9625afd0c885f2800815b228a0e70f9a7f473d", size = 252299, upload-time = "2025-08-11T12:08:07.931Z" }, - { url = "https://files.pythonhosted.org/packages/3a/09/8fe02d204473e14c0af3affd50af9078839dfca1742f025cca765435d6b4/multidict-6.6.4-cp313-cp313t-musllinux_1_2_s390x.whl", hash = "sha256:e1b93790ed0bc26feb72e2f08299691ceb6da5e9e14a0d13cc74f1869af327a0", size = 246926, upload-time = "2025-08-11T12:08:09.467Z" }, - { url = "https://files.pythonhosted.org/packages/37/3d/7b1e10d774a6df5175ecd3c92bff069e77bed9ec2a927fdd4ff5fe182f67/multidict-6.6.4-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:a506a77ddee1efcca81ecbeae27ade3e09cdf21a8ae854d766c2bb4f14053f92", size = 243383, upload-time = "2025-08-11T12:08:10.981Z" }, - { url = "https://files.pythonhosted.org/packages/50/b0/a6fae46071b645ae98786ab738447de1ef53742eaad949f27e960864bb49/multidict-6.6.4-cp313-cp313t-win32.whl", hash = "sha256:f93b2b2279883d1d0a9e1bd01f312d6fc315c5e4c1f09e112e4736e2f650bc4e", size = 47775, upload-time = "2025-08-11T12:08:12.439Z" }, - { url = "https://files.pythonhosted.org/packages/b2/0a/2436550b1520091af0600dff547913cb2d66fbac27a8c33bc1b1bccd8d98/multidict-6.6.4-cp313-cp313t-win_amd64.whl", hash = "sha256:6d46a180acdf6e87cc41dc15d8f5c2986e1e8739dc25dbb7dac826731ef381a4", size = 53100, upload-time = "2025-08-11T12:08:13.823Z" }, - { url = "https://files.pythonhosted.org/packages/97/ea/43ac51faff934086db9c072a94d327d71b7d8b40cd5dcb47311330929ef0/multidict-6.6.4-cp313-cp313t-win_arm64.whl", hash = "sha256:756989334015e3335d087a27331659820d53ba432befdef6a718398b0a8493ad", size = 45501, upload-time = "2025-08-11T12:08:15.173Z" }, - { url = "https://files.pythonhosted.org/packages/fd/69/b547032297c7e63ba2af494edba695d781af8a0c6e89e4d06cf848b21d80/multidict-6.6.4-py3-none-any.whl", hash = "sha256:27d8f8e125c07cb954e54d75d04905a9bba8a439c1d84aca94949d4d03d8601c", size = 12313, upload-time = "2025-08-11T12:08:46.891Z" }, +sdist = { url = "https://files.pythonhosted.org/packages/1a/c2/c2d94cbe6ac1753f3fc980da97b3d930efe1da3af3c9f5125354436c073d/multidict-6.7.1.tar.gz", hash = "sha256:ec6652a1bee61c53a3e5776b6049172c53b6aaba34f18c9ad04f82712bac623d", size = 102010, upload-time = "2026-01-26T02:46:45.979Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/84/0b/19348d4c98980c4851d2f943f8ebafdece2ae7ef737adcfa5994ce8e5f10/multidict-6.7.1-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:c93c3db7ea657dd4637d57e74ab73de31bccefe144d3d4ce370052035bc85fb5", size = 77176, upload-time = "2026-01-26T02:42:59.784Z" }, + { url = "https://files.pythonhosted.org/packages/ef/04/9de3f8077852e3d438215c81e9b691244532d2e05b4270e89ce67b7d103c/multidict-6.7.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:974e72a2474600827abaeda71af0c53d9ebbc3c2eb7da37b37d7829ae31232d8", size = 44996, upload-time = "2026-01-26T02:43:01.674Z" }, + { url = "https://files.pythonhosted.org/packages/31/5c/08c7f7fe311f32e83f7621cd3f99d805f45519cd06fafb247628b861da7d/multidict-6.7.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:cdea2e7b2456cfb6694fb113066fd0ec7ea4d67e3a35e1f4cbeea0b448bf5872", size = 44631, upload-time = "2026-01-26T02:43:03.169Z" }, + { url = "https://files.pythonhosted.org/packages/b7/7f/0e3b1390ae772f27501199996b94b52ceeb64fe6f9120a32c6c3f6b781be/multidict-6.7.1-cp310-cp310-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:17207077e29342fdc2c9a82e4b306f1127bf1ea91f8b71e02d4798a70bb99991", size = 242561, upload-time = "2026-01-26T02:43:04.733Z" }, + { url = "https://files.pythonhosted.org/packages/dd/f4/8719f4f167586af317b69dd3e90f913416c91ca610cac79a45c53f590312/multidict-6.7.1-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d4f49cb5661344764e4c7c7973e92a47a59b8fc19b6523649ec9dc4960e58a03", size = 242223, upload-time = "2026-01-26T02:43:06.695Z" }, + { url = "https://files.pythonhosted.org/packages/47/ab/7c36164cce64a6ad19c6d9a85377b7178ecf3b89f8fd589c73381a5eedfd/multidict-6.7.1-cp310-cp310-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:a9fc4caa29e2e6ae408d1c450ac8bf19892c5fca83ee634ecd88a53332c59981", size = 222322, upload-time = "2026-01-26T02:43:08.472Z" }, + { url = "https://files.pythonhosted.org/packages/f5/79/a25add6fb38035b5337bc5734f296d9afc99163403bbcf56d4170f97eb62/multidict-6.7.1-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:c5f0c21549ab432b57dcc82130f388d84ad8179824cc3f223d5e7cfbfd4143f6", size = 254005, upload-time = "2026-01-26T02:43:10.127Z" }, + { url = "https://files.pythonhosted.org/packages/4a/7b/64a87cf98e12f756fc8bd444b001232ffff2be37288f018ad0d3f0aae931/multidict-6.7.1-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:7dfb78d966b2c906ae1d28ccf6e6712a3cd04407ee5088cd276fe8cb42186190", size = 251173, upload-time = "2026-01-26T02:43:11.731Z" }, + { url = "https://files.pythonhosted.org/packages/4b/ac/b605473de2bb404e742f2cc3583d12aedb2352a70e49ae8fce455b50c5aa/multidict-6.7.1-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9b0d9b91d1aa44db9c1f1ecd0d9d2ae610b2f4f856448664e01a3b35899f3f92", size = 243273, upload-time = "2026-01-26T02:43:13.063Z" }, + { url = "https://files.pythonhosted.org/packages/03/65/11492d6a0e259783720f3bc1d9ea55579a76f1407e31ed44045c99542004/multidict-6.7.1-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:dd96c01a9dcd4889dcfcf9eb5544ca0c77603f239e3ffab0524ec17aea9a93ee", size = 238956, upload-time = "2026-01-26T02:43:14.843Z" }, + { url = "https://files.pythonhosted.org/packages/5f/a7/7ee591302af64e7c196fb63fe856c788993c1372df765102bd0448e7e165/multidict-6.7.1-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:067343c68cd6612d375710f895337b3a98a033c94f14b9a99eff902f205424e2", size = 233477, upload-time = "2026-01-26T02:43:16.025Z" }, + { url = "https://files.pythonhosted.org/packages/9c/99/c109962d58756c35fd9992fed7f2355303846ea2ff054bb5f5e9d6b888de/multidict-6.7.1-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:5884a04f4ff56c6120f6ccf703bdeb8b5079d808ba604d4d53aec0d55dc33568", size = 243615, upload-time = "2026-01-26T02:43:17.84Z" }, + { url = "https://files.pythonhosted.org/packages/d5/5f/1973e7c771c86e93dcfe1c9cc55a5481b610f6614acfc28c0d326fe6bfad/multidict-6.7.1-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:8affcf1c98b82bc901702eb73b6947a1bfa170823c153fe8a47b5f5f02e48e40", size = 249930, upload-time = "2026-01-26T02:43:19.06Z" }, + { url = "https://files.pythonhosted.org/packages/5d/a5/f170fc2268c3243853580203378cd522446b2df632061e0a5409817854c7/multidict-6.7.1-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:0d17522c37d03e85c8098ec8431636309b2682cf12e58f4dbc76121fb50e4962", size = 243807, upload-time = "2026-01-26T02:43:20.286Z" }, + { url = "https://files.pythonhosted.org/packages/de/01/73856fab6d125e5bc652c3986b90e8699a95e84b48d72f39ade6c0e74a8c/multidict-6.7.1-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:24c0cf81544ca5e17cfcb6e482e7a82cd475925242b308b890c9452a074d4505", size = 239103, upload-time = "2026-01-26T02:43:21.508Z" }, + { url = "https://files.pythonhosted.org/packages/e7/46/f1220bd9944d8aa40d8ccff100eeeee19b505b857b6f603d6078cb5315b0/multidict-6.7.1-cp310-cp310-win32.whl", hash = "sha256:d82dd730a95e6643802f4454b8fdecdf08667881a9c5670db85bc5a56693f122", size = 41416, upload-time = "2026-01-26T02:43:22.703Z" }, + { url = "https://files.pythonhosted.org/packages/68/00/9b38e272a770303692fc406c36e1a4c740f401522d5787691eb38a8925a8/multidict-6.7.1-cp310-cp310-win_amd64.whl", hash = "sha256:cf37cbe5ced48d417ba045aca1b21bafca67489452debcde94778a576666a1df", size = 46022, upload-time = "2026-01-26T02:43:23.77Z" }, + { url = "https://files.pythonhosted.org/packages/64/65/d8d42490c02ee07b6bbe00f7190d70bb4738b3cce7629aaf9f213ef730dd/multidict-6.7.1-cp310-cp310-win_arm64.whl", hash = "sha256:59bc83d3f66b41dac1e7460aac1d196edc70c9ba3094965c467715a70ecb46db", size = 43238, upload-time = "2026-01-26T02:43:24.882Z" }, + { url = "https://files.pythonhosted.org/packages/ce/f1/a90635c4f88fb913fbf4ce660b83b7445b7a02615bda034b2f8eb38fd597/multidict-6.7.1-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:7ff981b266af91d7b4b3793ca3382e53229088d193a85dfad6f5f4c27fc73e5d", size = 76626, upload-time = "2026-01-26T02:43:26.485Z" }, + { url = "https://files.pythonhosted.org/packages/a6/9b/267e64eaf6fc637a15b35f5de31a566634a2740f97d8d094a69d34f524a4/multidict-6.7.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:844c5bca0b5444adb44a623fb0a1310c2f4cd41f402126bb269cd44c9b3f3e1e", size = 44706, upload-time = "2026-01-26T02:43:27.607Z" }, + { url = "https://files.pythonhosted.org/packages/dd/a4/d45caf2b97b035c57267791ecfaafbd59c68212004b3842830954bb4b02e/multidict-6.7.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:f2a0a924d4c2e9afcd7ec64f9de35fcd96915149b2216e1cb2c10a56df483855", size = 44356, upload-time = "2026-01-26T02:43:28.661Z" }, + { url = "https://files.pythonhosted.org/packages/fd/d2/0a36c8473f0cbaeadd5db6c8b72d15bbceeec275807772bfcd059bef487d/multidict-6.7.1-cp311-cp311-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:8be1802715a8e892c784c0197c2ace276ea52702a0ede98b6310c8f255a5afb3", size = 244355, upload-time = "2026-01-26T02:43:31.165Z" }, + { url = "https://files.pythonhosted.org/packages/5d/16/8c65be997fd7dd311b7d39c7b6e71a0cb449bad093761481eccbbe4b42a2/multidict-6.7.1-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:2e2d2ed645ea29f31c4c7ea1552fcfd7cb7ba656e1eafd4134a6620c9f5fdd9e", size = 246433, upload-time = "2026-01-26T02:43:32.581Z" }, + { url = "https://files.pythonhosted.org/packages/01/fb/4dbd7e848d2799c6a026ec88ad39cf2b8416aa167fcc903baa55ecaa045c/multidict-6.7.1-cp311-cp311-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:95922cee9a778659e91db6497596435777bd25ed116701a4c034f8e46544955a", size = 225376, upload-time = "2026-01-26T02:43:34.417Z" }, + { url = "https://files.pythonhosted.org/packages/b6/8a/4a3a6341eac3830f6053062f8fbc9a9e54407c80755b3f05bc427295c2d0/multidict-6.7.1-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:6b83cabdc375ffaaa15edd97eb7c0c672ad788e2687004990074d7d6c9b140c8", size = 257365, upload-time = "2026-01-26T02:43:35.741Z" }, + { url = "https://files.pythonhosted.org/packages/f7/a2/dd575a69c1aa206e12d27d0770cdf9b92434b48a9ef0cd0d1afdecaa93c4/multidict-6.7.1-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:38fb49540705369bab8484db0689d86c0a33a0a9f2c1b197f506b71b4b6c19b0", size = 254747, upload-time = "2026-01-26T02:43:36.976Z" }, + { url = "https://files.pythonhosted.org/packages/5a/56/21b27c560c13822ed93133f08aa6372c53a8e067f11fbed37b4adcdac922/multidict-6.7.1-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:439cbebd499f92e9aa6793016a8acaa161dfa749ae86d20960189f5398a19144", size = 246293, upload-time = "2026-01-26T02:43:38.258Z" }, + { url = "https://files.pythonhosted.org/packages/5a/a4/23466059dc3854763423d0ad6c0f3683a379d97673b1b89ec33826e46728/multidict-6.7.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:6d3bc717b6fe763b8be3f2bee2701d3c8eb1b2a8ae9f60910f1b2860c82b6c49", size = 242962, upload-time = "2026-01-26T02:43:40.034Z" }, + { url = "https://files.pythonhosted.org/packages/1f/67/51dd754a3524d685958001e8fa20a0f5f90a6a856e0a9dcabff69be3dbb7/multidict-6.7.1-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:619e5a1ac57986dbfec9f0b301d865dddf763696435e2962f6d9cf2fdff2bb71", size = 237360, upload-time = "2026-01-26T02:43:41.752Z" }, + { url = "https://files.pythonhosted.org/packages/64/3f/036dfc8c174934d4b55d86ff4f978e558b0e585cef70cfc1ad01adc6bf18/multidict-6.7.1-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:0b38ebffd9be37c1170d33bc0f36f4f262e0a09bc1aac1c34c7aa51a7293f0b3", size = 245940, upload-time = "2026-01-26T02:43:43.042Z" }, + { url = "https://files.pythonhosted.org/packages/3d/20/6214d3c105928ebc353a1c644a6ef1408bc5794fcb4f170bb524a3c16311/multidict-6.7.1-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:10ae39c9cfe6adedcdb764f5e8411d4a92b055e35573a2eaa88d3323289ef93c", size = 253502, upload-time = "2026-01-26T02:43:44.371Z" }, + { url = "https://files.pythonhosted.org/packages/b1/e2/c653bc4ae1be70a0f836b82172d643fcf1dade042ba2676ab08ec08bff0f/multidict-6.7.1-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:25167cc263257660290fba06b9318d2026e3c910be240a146e1f66dd114af2b0", size = 247065, upload-time = "2026-01-26T02:43:45.745Z" }, + { url = "https://files.pythonhosted.org/packages/c8/11/a854b4154cd3bd8b1fd375e8a8ca9d73be37610c361543d56f764109509b/multidict-6.7.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:128441d052254f42989ef98b7b6a6ecb1e6f708aa962c7984235316db59f50fa", size = 241870, upload-time = "2026-01-26T02:43:47.054Z" }, + { url = "https://files.pythonhosted.org/packages/13/bf/9676c0392309b5fdae322333d22a829715b570edb9baa8016a517b55b558/multidict-6.7.1-cp311-cp311-win32.whl", hash = "sha256:d62b7f64ffde3b99d06b707a280db04fb3855b55f5a06df387236051d0668f4a", size = 41302, upload-time = "2026-01-26T02:43:48.753Z" }, + { url = "https://files.pythonhosted.org/packages/c9/68/f16a3a8ba6f7b6dc92a1f19669c0810bd2c43fc5a02da13b1cbf8e253845/multidict-6.7.1-cp311-cp311-win_amd64.whl", hash = "sha256:bdbf9f3b332abd0cdb306e7c2113818ab1e922dc84b8f8fd06ec89ed2a19ab8b", size = 45981, upload-time = "2026-01-26T02:43:49.921Z" }, + { url = "https://files.pythonhosted.org/packages/ac/ad/9dd5305253fa00cd3c7555dbef69d5bf4133debc53b87ab8d6a44d411665/multidict-6.7.1-cp311-cp311-win_arm64.whl", hash = "sha256:b8c990b037d2fff2f4e33d3f21b9b531c5745b33a49a7d6dbe7a177266af44f6", size = 43159, upload-time = "2026-01-26T02:43:51.635Z" }, + { url = "https://files.pythonhosted.org/packages/8d/9c/f20e0e2cf80e4b2e4b1c365bf5fe104ee633c751a724246262db8f1a0b13/multidict-6.7.1-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:a90f75c956e32891a4eda3639ce6dd86e87105271f43d43442a3aedf3cddf172", size = 76893, upload-time = "2026-01-26T02:43:52.754Z" }, + { url = "https://files.pythonhosted.org/packages/fe/cf/18ef143a81610136d3da8193da9d80bfe1cb548a1e2d1c775f26b23d024a/multidict-6.7.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:3fccb473e87eaa1382689053e4a4618e7ba7b9b9b8d6adf2027ee474597128cd", size = 45456, upload-time = "2026-01-26T02:43:53.893Z" }, + { url = "https://files.pythonhosted.org/packages/a9/65/1caac9d4cd32e8433908683446eebc953e82d22b03d10d41a5f0fefe991b/multidict-6.7.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:b0fa96985700739c4c7853a43c0b3e169360d6855780021bfc6d0f1ce7c123e7", size = 43872, upload-time = "2026-01-26T02:43:55.041Z" }, + { url = "https://files.pythonhosted.org/packages/cf/3b/d6bd75dc4f3ff7c73766e04e705b00ed6dbbaccf670d9e05a12b006f5a21/multidict-6.7.1-cp312-cp312-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:cb2a55f408c3043e42b40cc8eecd575afa27b7e0b956dfb190de0f8499a57a53", size = 251018, upload-time = "2026-01-26T02:43:56.198Z" }, + { url = "https://files.pythonhosted.org/packages/fd/80/c959c5933adedb9ac15152e4067c702a808ea183a8b64cf8f31af8ad3155/multidict-6.7.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:eb0ce7b2a32d09892b3dd6cc44877a0d02a33241fafca5f25c8b6b62374f8b75", size = 258883, upload-time = "2026-01-26T02:43:57.499Z" }, + { url = "https://files.pythonhosted.org/packages/86/85/7ed40adafea3d4f1c8b916e3b5cc3a8e07dfcdcb9cd72800f4ed3ca1b387/multidict-6.7.1-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:c3a32d23520ee37bf327d1e1a656fec76a2edd5c038bf43eddfa0572ec49c60b", size = 242413, upload-time = "2026-01-26T02:43:58.755Z" }, + { url = "https://files.pythonhosted.org/packages/d2/57/b8565ff533e48595503c785f8361ff9a4fde4d67de25c207cd0ba3befd03/multidict-6.7.1-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:9c90fed18bffc0189ba814749fdcc102b536e83a9f738a9003e569acd540a733", size = 268404, upload-time = "2026-01-26T02:44:00.216Z" }, + { url = "https://files.pythonhosted.org/packages/e0/50/9810c5c29350f7258180dfdcb2e52783a0632862eb334c4896ac717cebcb/multidict-6.7.1-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:da62917e6076f512daccfbbde27f46fed1c98fee202f0559adec8ee0de67f71a", size = 269456, upload-time = "2026-01-26T02:44:02.202Z" }, + { url = "https://files.pythonhosted.org/packages/f3/8d/5e5be3ced1d12966fefb5c4ea3b2a5b480afcea36406559442c6e31d4a48/multidict-6.7.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:bfde23ef6ed9db7eaee6c37dcec08524cb43903c60b285b172b6c094711b3961", size = 256322, upload-time = "2026-01-26T02:44:03.56Z" }, + { url = "https://files.pythonhosted.org/packages/31/6e/d8a26d81ac166a5592782d208dd90dfdc0a7a218adaa52b45a672b46c122/multidict-6.7.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:3758692429e4e32f1ba0df23219cd0b4fc0a52f476726fff9337d1a57676a582", size = 253955, upload-time = "2026-01-26T02:44:04.845Z" }, + { url = "https://files.pythonhosted.org/packages/59/4c/7c672c8aad41534ba619bcd4ade7a0dc87ed6b8b5c06149b85d3dd03f0cd/multidict-6.7.1-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:398c1478926eca669f2fd6a5856b6de9c0acf23a2cb59a14c0ba5844fa38077e", size = 251254, upload-time = "2026-01-26T02:44:06.133Z" }, + { url = "https://files.pythonhosted.org/packages/7b/bd/84c24de512cbafbdbc39439f74e967f19570ce7924e3007174a29c348916/multidict-6.7.1-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:c102791b1c4f3ab36ce4101154549105a53dc828f016356b3e3bcae2e3a039d3", size = 252059, upload-time = "2026-01-26T02:44:07.518Z" }, + { url = "https://files.pythonhosted.org/packages/fa/ba/f5449385510825b73d01c2d4087bf6d2fccc20a2d42ac34df93191d3dd03/multidict-6.7.1-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:a088b62bd733e2ad12c50dad01b7d0166c30287c166e137433d3b410add807a6", size = 263588, upload-time = "2026-01-26T02:44:09.382Z" }, + { url = "https://files.pythonhosted.org/packages/d7/11/afc7c677f68f75c84a69fe37184f0f82fce13ce4b92f49f3db280b7e92b3/multidict-6.7.1-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:3d51ff4785d58d3f6c91bdbffcb5e1f7ddfda557727043aa20d20ec4f65e324a", size = 259642, upload-time = "2026-01-26T02:44:10.73Z" }, + { url = "https://files.pythonhosted.org/packages/2b/17/ebb9644da78c4ab36403739e0e6e0e30ebb135b9caf3440825001a0bddcb/multidict-6.7.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:fc5907494fccf3e7d3f94f95c91d6336b092b5fc83811720fae5e2765890dfba", size = 251377, upload-time = "2026-01-26T02:44:12.042Z" }, + { url = "https://files.pythonhosted.org/packages/ca/a4/840f5b97339e27846c46307f2530a2805d9d537d8b8bd416af031cad7fa0/multidict-6.7.1-cp312-cp312-win32.whl", hash = "sha256:28ca5ce2fd9716631133d0e9a9b9a745ad7f60bac2bccafb56aa380fc0b6c511", size = 41887, upload-time = "2026-01-26T02:44:14.245Z" }, + { url = "https://files.pythonhosted.org/packages/80/31/0b2517913687895f5904325c2069d6a3b78f66cc641a86a2baf75a05dcbb/multidict-6.7.1-cp312-cp312-win_amd64.whl", hash = "sha256:fcee94dfbd638784645b066074b338bc9cc155d4b4bffa4adce1615c5a426c19", size = 46053, upload-time = "2026-01-26T02:44:15.371Z" }, + { url = "https://files.pythonhosted.org/packages/0c/5b/aba28e4ee4006ae4c7df8d327d31025d760ffa992ea23812a601d226e682/multidict-6.7.1-cp312-cp312-win_arm64.whl", hash = "sha256:ba0a9fb644d0c1a2194cf7ffb043bd852cea63a57f66fbd33959f7dae18517bf", size = 43307, upload-time = "2026-01-26T02:44:16.852Z" }, + { url = "https://files.pythonhosted.org/packages/f2/22/929c141d6c0dba87d3e1d38fbdf1ba8baba86b7776469f2bc2d3227a1e67/multidict-6.7.1-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:2b41f5fed0ed563624f1c17630cb9941cf2309d4df00e494b551b5f3e3d67a23", size = 76174, upload-time = "2026-01-26T02:44:18.509Z" }, + { url = "https://files.pythonhosted.org/packages/c7/75/bc704ae15fee974f8fccd871305e254754167dce5f9e42d88a2def741a1d/multidict-6.7.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:84e61e3af5463c19b67ced91f6c634effb89ef8bfc5ca0267f954451ed4bb6a2", size = 45116, upload-time = "2026-01-26T02:44:19.745Z" }, + { url = "https://files.pythonhosted.org/packages/79/76/55cd7186f498ed080a18440c9013011eb548f77ae1b297206d030eb1180a/multidict-6.7.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:935434b9853c7c112eee7ac891bc4cb86455aa631269ae35442cb316790c1445", size = 43524, upload-time = "2026-01-26T02:44:21.571Z" }, + { url = "https://files.pythonhosted.org/packages/e9/3c/414842ef8d5a1628d68edee29ba0e5bcf235dbfb3ccd3ea303a7fe8c72ff/multidict-6.7.1-cp313-cp313-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:432feb25a1cb67fe82a9680b4d65fb542e4635cb3166cd9c01560651ad60f177", size = 249368, upload-time = "2026-01-26T02:44:22.803Z" }, + { url = "https://files.pythonhosted.org/packages/f6/32/befed7f74c458b4a525e60519fe8d87eef72bb1e99924fa2b0f9d97a221e/multidict-6.7.1-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e82d14e3c948952a1a85503817e038cba5905a3352de76b9a465075d072fba23", size = 256952, upload-time = "2026-01-26T02:44:24.306Z" }, + { url = "https://files.pythonhosted.org/packages/03/d6/c878a44ba877f366630c860fdf74bfb203c33778f12b6ac274936853c451/multidict-6.7.1-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:4cfb48c6ea66c83bcaaf7e4dfa7ec1b6bbcf751b7db85a328902796dfde4c060", size = 240317, upload-time = "2026-01-26T02:44:25.772Z" }, + { url = "https://files.pythonhosted.org/packages/68/49/57421b4d7ad2e9e60e25922b08ceb37e077b90444bde6ead629095327a6f/multidict-6.7.1-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:1d540e51b7e8e170174555edecddbd5538105443754539193e3e1061864d444d", size = 267132, upload-time = "2026-01-26T02:44:27.648Z" }, + { url = "https://files.pythonhosted.org/packages/b7/fe/ec0edd52ddbcea2a2e89e174f0206444a61440b40f39704e64dc807a70bd/multidict-6.7.1-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:273d23f4b40f3dce4d6c8a821c741a86dec62cded82e1175ba3d99be128147ed", size = 268140, upload-time = "2026-01-26T02:44:29.588Z" }, + { url = "https://files.pythonhosted.org/packages/b0/73/6e1b01cbeb458807aa0831742232dbdd1fa92bfa33f52a3f176b4ff3dc11/multidict-6.7.1-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9d624335fd4fa1c08a53f8b4be7676ebde19cd092b3895c421045ca87895b429", size = 254277, upload-time = "2026-01-26T02:44:30.902Z" }, + { url = "https://files.pythonhosted.org/packages/6a/b2/5fb8c124d7561a4974c342bc8c778b471ebbeb3cc17df696f034a7e9afe7/multidict-6.7.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:12fad252f8b267cc75b66e8fc51b3079604e8d43a75428ffe193cd9e2195dfd6", size = 252291, upload-time = "2026-01-26T02:44:32.31Z" }, + { url = "https://files.pythonhosted.org/packages/5a/96/51d4e4e06bcce92577fcd488e22600bd38e4fd59c20cb49434d054903bd2/multidict-6.7.1-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:03ede2a6ffbe8ef936b92cb4529f27f42be7f56afcdab5ab739cd5f27fb1cbf9", size = 250156, upload-time = "2026-01-26T02:44:33.734Z" }, + { url = "https://files.pythonhosted.org/packages/db/6b/420e173eec5fba721a50e2a9f89eda89d9c98fded1124f8d5c675f7a0c0f/multidict-6.7.1-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:90efbcf47dbe33dcf643a1e400d67d59abeac5db07dc3f27d6bdeae497a2198c", size = 249742, upload-time = "2026-01-26T02:44:35.222Z" }, + { url = "https://files.pythonhosted.org/packages/44/a3/ec5b5bd98f306bc2aa297b8c6f11a46714a56b1e6ef5ebda50a4f5d7c5fb/multidict-6.7.1-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:5c4b9bfc148f5a91be9244d6264c53035c8a0dcd2f51f1c3c6e30e30ebaa1c84", size = 262221, upload-time = "2026-01-26T02:44:36.604Z" }, + { url = "https://files.pythonhosted.org/packages/cd/f7/e8c0d0da0cd1e28d10e624604e1a36bcc3353aaebdfdc3a43c72bc683a12/multidict-6.7.1-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:401c5a650f3add2472d1d288c26deebc540f99e2fb83e9525007a74cd2116f1d", size = 258664, upload-time = "2026-01-26T02:44:38.008Z" }, + { url = "https://files.pythonhosted.org/packages/52/da/151a44e8016dd33feed44f730bd856a66257c1ee7aed4f44b649fb7edeb3/multidict-6.7.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:97891f3b1b3ffbded884e2916cacf3c6fc87b66bb0dde46f7357404750559f33", size = 249490, upload-time = "2026-01-26T02:44:39.386Z" }, + { url = "https://files.pythonhosted.org/packages/87/af/a3b86bf9630b732897f6fc3f4c4714b90aa4361983ccbdcd6c0339b21b0c/multidict-6.7.1-cp313-cp313-win32.whl", hash = "sha256:e1c5988359516095535c4301af38d8a8838534158f649c05dd1050222321bcb3", size = 41695, upload-time = "2026-01-26T02:44:41.318Z" }, + { url = "https://files.pythonhosted.org/packages/b2/35/e994121b0e90e46134673422dd564623f93304614f5d11886b1b3e06f503/multidict-6.7.1-cp313-cp313-win_amd64.whl", hash = "sha256:960c83bf01a95b12b08fd54324a4eb1d5b52c88932b5cba5d6e712bb3ed12eb5", size = 45884, upload-time = "2026-01-26T02:44:42.488Z" }, + { url = "https://files.pythonhosted.org/packages/ca/61/42d3e5dbf661242a69c97ea363f2d7b46c567da8eadef8890022be6e2ab0/multidict-6.7.1-cp313-cp313-win_arm64.whl", hash = "sha256:563fe25c678aaba333d5399408f5ec3c383ca5b663e7f774dd179a520b8144df", size = 43122, upload-time = "2026-01-26T02:44:43.664Z" }, + { url = "https://files.pythonhosted.org/packages/6d/b3/e6b21c6c4f314bb956016b0b3ef2162590a529b84cb831c257519e7fde44/multidict-6.7.1-cp313-cp313t-macosx_10_13_universal2.whl", hash = "sha256:c76c4bec1538375dad9d452d246ca5368ad6e1c9039dadcf007ae59c70619ea1", size = 83175, upload-time = "2026-01-26T02:44:44.894Z" }, + { url = "https://files.pythonhosted.org/packages/fb/76/23ecd2abfe0957b234f6c960f4ade497f55f2c16aeb684d4ecdbf1c95791/multidict-6.7.1-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:57b46b24b5d5ebcc978da4ec23a819a9402b4228b8a90d9c656422b4bdd8a963", size = 48460, upload-time = "2026-01-26T02:44:46.106Z" }, + { url = "https://files.pythonhosted.org/packages/c4/57/a0ed92b23f3a042c36bc4227b72b97eca803f5f1801c1ab77c8a212d455e/multidict-6.7.1-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:e954b24433c768ce78ab7929e84ccf3422e46deb45a4dc9f93438f8217fa2d34", size = 46930, upload-time = "2026-01-26T02:44:47.278Z" }, + { url = "https://files.pythonhosted.org/packages/b5/66/02ec7ace29162e447f6382c495dc95826bf931d3818799bbef11e8f7df1a/multidict-6.7.1-cp313-cp313t-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:3bd231490fa7217cc832528e1cd8752a96f0125ddd2b5749390f7c3ec8721b65", size = 242582, upload-time = "2026-01-26T02:44:48.604Z" }, + { url = "https://files.pythonhosted.org/packages/58/18/64f5a795e7677670e872673aca234162514696274597b3708b2c0d276cce/multidict-6.7.1-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:253282d70d67885a15c8a7716f3a73edf2d635793ceda8173b9ecc21f2fb8292", size = 250031, upload-time = "2026-01-26T02:44:50.544Z" }, + { url = "https://files.pythonhosted.org/packages/c8/ed/e192291dbbe51a8290c5686f482084d31bcd9d09af24f63358c3d42fd284/multidict-6.7.1-cp313-cp313t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:0b4c48648d7649c9335cf1927a8b87fa692de3dcb15faa676c6a6f1f1aabda43", size = 228596, upload-time = "2026-01-26T02:44:51.951Z" }, + { url = "https://files.pythonhosted.org/packages/1e/7e/3562a15a60cf747397e7f2180b0a11dc0c38d9175a650e75fa1b4d325e15/multidict-6.7.1-cp313-cp313t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:98bc624954ec4d2c7cb074b8eefc2b5d0ce7d482e410df446414355d158fe4ca", size = 257492, upload-time = "2026-01-26T02:44:53.902Z" }, + { url = "https://files.pythonhosted.org/packages/24/02/7d0f9eae92b5249bb50ac1595b295f10e263dd0078ebb55115c31e0eaccd/multidict-6.7.1-cp313-cp313t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:1b99af4d9eec0b49927b4402bcbb58dea89d3e0db8806a4086117019939ad3dd", size = 255899, upload-time = "2026-01-26T02:44:55.316Z" }, + { url = "https://files.pythonhosted.org/packages/00/e3/9b60ed9e23e64c73a5cde95269ef1330678e9c6e34dd4eb6b431b85b5a10/multidict-6.7.1-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:6aac4f16b472d5b7dc6f66a0d49dd57b0e0902090be16594dc9ebfd3d17c47e7", size = 247970, upload-time = "2026-01-26T02:44:56.783Z" }, + { url = "https://files.pythonhosted.org/packages/3e/06/538e58a63ed5cfb0bd4517e346b91da32fde409d839720f664e9a4ae4f9d/multidict-6.7.1-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:21f830fe223215dffd51f538e78c172ed7c7f60c9b96a2bf05c4848ad49921c3", size = 245060, upload-time = "2026-01-26T02:44:58.195Z" }, + { url = "https://files.pythonhosted.org/packages/b2/2f/d743a3045a97c895d401e9bd29aaa09b94f5cbdf1bd561609e5a6c431c70/multidict-6.7.1-cp313-cp313t-musllinux_1_2_armv7l.whl", hash = "sha256:f5dd81c45b05518b9aa4da4aa74e1c93d715efa234fd3e8a179df611cc85e5f4", size = 235888, upload-time = "2026-01-26T02:44:59.57Z" }, + { url = "https://files.pythonhosted.org/packages/38/83/5a325cac191ab28b63c52f14f1131f3b0a55ba3b9aa65a6d0bf2a9b921a0/multidict-6.7.1-cp313-cp313t-musllinux_1_2_i686.whl", hash = "sha256:eb304767bca2bb92fb9c5bd33cedc95baee5bb5f6c88e63706533a1c06ad08c8", size = 243554, upload-time = "2026-01-26T02:45:01.054Z" }, + { url = "https://files.pythonhosted.org/packages/20/1f/9d2327086bd15da2725ef6aae624208e2ef828ed99892b17f60c344e57ed/multidict-6.7.1-cp313-cp313t-musllinux_1_2_ppc64le.whl", hash = "sha256:c9035dde0f916702850ef66460bc4239d89d08df4d02023a5926e7446724212c", size = 252341, upload-time = "2026-01-26T02:45:02.484Z" }, + { url = "https://files.pythonhosted.org/packages/e8/2c/2a1aa0280cf579d0f6eed8ee5211c4f1730bd7e06c636ba2ee6aafda302e/multidict-6.7.1-cp313-cp313t-musllinux_1_2_s390x.whl", hash = "sha256:af959b9beeb66c822380f222f0e0a1889331597e81f1ded7f374f3ecb0fd6c52", size = 246391, upload-time = "2026-01-26T02:45:03.862Z" }, + { url = "https://files.pythonhosted.org/packages/e5/03/7ca022ffc36c5a3f6e03b179a5ceb829be9da5783e6fe395f347c0794680/multidict-6.7.1-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:41f2952231456154ee479651491e94118229844dd7226541788be783be2b5108", size = 243422, upload-time = "2026-01-26T02:45:05.296Z" }, + { url = "https://files.pythonhosted.org/packages/dc/1d/b31650eab6c5778aceed46ba735bd97f7c7d2f54b319fa916c0f96e7805b/multidict-6.7.1-cp313-cp313t-win32.whl", hash = "sha256:df9f19c28adcb40b6aae30bbaa1478c389efd50c28d541d76760199fc1037c32", size = 47770, upload-time = "2026-01-26T02:45:06.754Z" }, + { url = "https://files.pythonhosted.org/packages/ac/5b/2d2d1d522e51285bd61b1e20df8f47ae1a9d80839db0b24ea783b3832832/multidict-6.7.1-cp313-cp313t-win_amd64.whl", hash = "sha256:d54ecf9f301853f2c5e802da559604b3e95bb7a3b01a9c295c6ee591b9882de8", size = 53109, upload-time = "2026-01-26T02:45:08.044Z" }, + { url = "https://files.pythonhosted.org/packages/3d/a3/cc409ba012c83ca024a308516703cf339bdc4b696195644a7215a5164a24/multidict-6.7.1-cp313-cp313t-win_arm64.whl", hash = "sha256:5a37ca18e360377cfda1d62f5f382ff41f2b8c4ccb329ed974cc2e1643440118", size = 45573, upload-time = "2026-01-26T02:45:09.349Z" }, + { url = "https://files.pythonhosted.org/packages/91/cc/db74228a8be41884a567e88a62fd589a913708fcf180d029898c17a9a371/multidict-6.7.1-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:8f333ec9c5eb1b7105e3b84b53141e66ca05a19a605368c55450b6ba208cb9ee", size = 75190, upload-time = "2026-01-26T02:45:10.651Z" }, + { url = "https://files.pythonhosted.org/packages/d5/22/492f2246bb5b534abd44804292e81eeaf835388901f0c574bac4eeec73c5/multidict-6.7.1-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:a407f13c188f804c759fc6a9f88286a565c242a76b27626594c133b82883b5c2", size = 44486, upload-time = "2026-01-26T02:45:11.938Z" }, + { url = "https://files.pythonhosted.org/packages/f1/4f/733c48f270565d78b4544f2baddc2fb2a245e5a8640254b12c36ac7ac68e/multidict-6.7.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:0e161ddf326db5577c3a4cc2d8648f81456e8a20d40415541587a71620d7a7d1", size = 43219, upload-time = "2026-01-26T02:45:14.346Z" }, + { url = "https://files.pythonhosted.org/packages/24/bb/2c0c2287963f4259c85e8bcbba9182ced8d7fca65c780c38e99e61629d11/multidict-6.7.1-cp314-cp314-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:1e3a8bb24342a8201d178c3b4984c26ba81a577c80d4d525727427460a50c22d", size = 245132, upload-time = "2026-01-26T02:45:15.712Z" }, + { url = "https://files.pythonhosted.org/packages/a7/f9/44d4b3064c65079d2467888794dea218d1601898ac50222ab8a9a8094460/multidict-6.7.1-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:97231140a50f5d447d3164f994b86a0bed7cd016e2682f8650d6a9158e14fd31", size = 252420, upload-time = "2026-01-26T02:45:17.293Z" }, + { url = "https://files.pythonhosted.org/packages/8b/13/78f7275e73fa17b24c9a51b0bd9d73ba64bb32d0ed51b02a746eb876abe7/multidict-6.7.1-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:6b10359683bd8806a200fd2909e7c8ca3a7b24ec1d8132e483d58e791d881048", size = 233510, upload-time = "2026-01-26T02:45:19.356Z" }, + { url = "https://files.pythonhosted.org/packages/4b/25/8167187f62ae3cbd52da7893f58cb036b47ea3fb67138787c76800158982/multidict-6.7.1-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:283ddac99f7ac25a4acadbf004cb5ae34480bbeb063520f70ce397b281859362", size = 264094, upload-time = "2026-01-26T02:45:20.834Z" }, + { url = "https://files.pythonhosted.org/packages/a1/e7/69a3a83b7b030cf283fb06ce074a05a02322359783424d7edf0f15fe5022/multidict-6.7.1-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:538cec1e18c067d0e6103aa9a74f9e832904c957adc260e61cd9d8cf0c3b3d37", size = 260786, upload-time = "2026-01-26T02:45:22.818Z" }, + { url = "https://files.pythonhosted.org/packages/fe/3b/8ec5074bcfc450fe84273713b4b0a0dd47c0249358f5d82eb8104ffe2520/multidict-6.7.1-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7eee46ccb30ff48a1e35bb818cc90846c6be2b68240e42a78599166722cea709", size = 248483, upload-time = "2026-01-26T02:45:24.368Z" }, + { url = "https://files.pythonhosted.org/packages/48/5a/d5a99e3acbca0e29c5d9cba8f92ceb15dce78bab963b308ae692981e3a5d/multidict-6.7.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:fa263a02f4f2dd2d11a7b1bb4362aa7cb1049f84a9235d31adf63f30143469a0", size = 248403, upload-time = "2026-01-26T02:45:25.982Z" }, + { url = "https://files.pythonhosted.org/packages/35/48/e58cd31f6c7d5102f2a4bf89f96b9cf7e00b6c6f3d04ecc44417c00a5a3c/multidict-6.7.1-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:2e1425e2f99ec5bd36c15a01b690a1a2456209c5deed58f95469ffb46039ccbb", size = 240315, upload-time = "2026-01-26T02:45:27.487Z" }, + { url = "https://files.pythonhosted.org/packages/94/33/1cd210229559cb90b6786c30676bb0c58249ff42f942765f88793b41fdce/multidict-6.7.1-cp314-cp314-musllinux_1_2_i686.whl", hash = "sha256:497394b3239fc6f0e13a78a3e1b61296e72bf1c5f94b4c4eb80b265c37a131cd", size = 245528, upload-time = "2026-01-26T02:45:28.991Z" }, + { url = "https://files.pythonhosted.org/packages/64/f2/6e1107d226278c876c783056b7db43d800bb64c6131cec9c8dfb6903698e/multidict-6.7.1-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:233b398c29d3f1b9676b4b6f75c518a06fcb2ea0b925119fb2c1bc35c05e1601", size = 258784, upload-time = "2026-01-26T02:45:30.503Z" }, + { url = "https://files.pythonhosted.org/packages/4d/c1/11f664f14d525e4a1b5327a82d4de61a1db604ab34c6603bb3c2cc63ad34/multidict-6.7.1-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:93b1818e4a6e0930454f0f2af7dfce69307ca03cdcfb3739bf4d91241967b6c1", size = 251980, upload-time = "2026-01-26T02:45:32.603Z" }, + { url = "https://files.pythonhosted.org/packages/e1/9f/75a9ac888121d0c5bbd4ecf4eead45668b1766f6baabfb3b7f66a410e231/multidict-6.7.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:f33dc2a3abe9249ea5d8360f969ec7f4142e7ac45ee7014d8f8d5acddf178b7b", size = 243602, upload-time = "2026-01-26T02:45:34.043Z" }, + { url = "https://files.pythonhosted.org/packages/9a/e7/50bf7b004cc8525d80dbbbedfdc7aed3e4c323810890be4413e589074032/multidict-6.7.1-cp314-cp314-win32.whl", hash = "sha256:3ab8b9d8b75aef9df299595d5388b14530839f6422333357af1339443cff777d", size = 40930, upload-time = "2026-01-26T02:45:36.278Z" }, + { url = "https://files.pythonhosted.org/packages/e0/bf/52f25716bbe93745595800f36fb17b73711f14da59ed0bb2eba141bc9f0f/multidict-6.7.1-cp314-cp314-win_amd64.whl", hash = "sha256:5e01429a929600e7dab7b166062d9bb54a5eed752384c7384c968c2afab8f50f", size = 45074, upload-time = "2026-01-26T02:45:37.546Z" }, + { url = "https://files.pythonhosted.org/packages/97/ab/22803b03285fa3a525f48217963da3a65ae40f6a1b6f6cf2768879e208f9/multidict-6.7.1-cp314-cp314-win_arm64.whl", hash = "sha256:4885cb0e817aef5d00a2e8451d4665c1808378dc27c2705f1bf4ef8505c0d2e5", size = 42471, upload-time = "2026-01-26T02:45:38.889Z" }, + { url = "https://files.pythonhosted.org/packages/e0/6d/f9293baa6146ba9507e360ea0292b6422b016907c393e2f63fc40ab7b7b5/multidict-6.7.1-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:0458c978acd8e6ea53c81eefaddbbee9c6c5e591f41b3f5e8e194780fe026581", size = 82401, upload-time = "2026-01-26T02:45:40.254Z" }, + { url = "https://files.pythonhosted.org/packages/7a/68/53b5494738d83558d87c3c71a486504d8373421c3e0dbb6d0db48ad42ee0/multidict-6.7.1-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:c0abd12629b0af3cf590982c0b413b1e7395cd4ec026f30986818ab95bfaa94a", size = 48143, upload-time = "2026-01-26T02:45:41.635Z" }, + { url = "https://files.pythonhosted.org/packages/37/e8/5284c53310dcdc99ce5d66563f6e5773531a9b9fe9ec7a615e9bc306b05f/multidict-6.7.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:14525a5f61d7d0c94b368a42cff4c9a4e7ba2d52e2672a7b23d84dc86fb02b0c", size = 46507, upload-time = "2026-01-26T02:45:42.99Z" }, + { url = "https://files.pythonhosted.org/packages/e4/fc/6800d0e5b3875568b4083ecf5f310dcf91d86d52573160834fb4bfcf5e4f/multidict-6.7.1-cp314-cp314t-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:17307b22c217b4cf05033dabefe68255a534d637c6c9b0cc8382718f87be4262", size = 239358, upload-time = "2026-01-26T02:45:44.376Z" }, + { url = "https://files.pythonhosted.org/packages/41/75/4ad0973179361cdf3a113905e6e088173198349131be2b390f9fa4da5fc6/multidict-6.7.1-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:7a7e590ff876a3eaf1c02a4dfe0724b6e69a9e9de6d8f556816f29c496046e59", size = 246884, upload-time = "2026-01-26T02:45:47.167Z" }, + { url = "https://files.pythonhosted.org/packages/c3/9c/095bb28b5da139bd41fb9a5d5caff412584f377914bd8787c2aa98717130/multidict-6.7.1-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:5fa6a95dfee63893d80a34758cd0e0c118a30b8dcb46372bf75106c591b77889", size = 225878, upload-time = "2026-01-26T02:45:48.698Z" }, + { url = "https://files.pythonhosted.org/packages/07/d0/c0a72000243756e8f5a277b6b514fa005f2c73d481b7d9e47cd4568aa2e4/multidict-6.7.1-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:a0543217a6a017692aa6ae5cc39adb75e587af0f3a82288b1492eb73dd6cc2a4", size = 253542, upload-time = "2026-01-26T02:45:50.164Z" }, + { url = "https://files.pythonhosted.org/packages/c0/6b/f69da15289e384ecf2a68837ec8b5ad8c33e973aa18b266f50fe55f24b8c/multidict-6.7.1-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:f99fe611c312b3c1c0ace793f92464d8cd263cc3b26b5721950d977b006b6c4d", size = 252403, upload-time = "2026-01-26T02:45:51.779Z" }, + { url = "https://files.pythonhosted.org/packages/a2/76/b9669547afa5a1a25cd93eaca91c0da1c095b06b6d2d8ec25b713588d3a1/multidict-6.7.1-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9004d8386d133b7e6135679424c91b0b854d2d164af6ea3f289f8f2761064609", size = 244889, upload-time = "2026-01-26T02:45:53.27Z" }, + { url = "https://files.pythonhosted.org/packages/7e/a9/a50d2669e506dad33cfc45b5d574a205587b7b8a5f426f2fbb2e90882588/multidict-6.7.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:e628ef0e6859ffd8273c69412a2465c4be4a9517d07261b33334b5ec6f3c7489", size = 241982, upload-time = "2026-01-26T02:45:54.919Z" }, + { url = "https://files.pythonhosted.org/packages/c5/bb/1609558ad8b456b4827d3c5a5b775c93b87878fd3117ed3db3423dfbce1b/multidict-6.7.1-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:841189848ba629c3552035a6a7f5bf3b02eb304e9fea7492ca220a8eda6b0e5c", size = 232415, upload-time = "2026-01-26T02:45:56.981Z" }, + { url = "https://files.pythonhosted.org/packages/d8/59/6f61039d2aa9261871e03ab9dc058a550d240f25859b05b67fd70f80d4b3/multidict-6.7.1-cp314-cp314t-musllinux_1_2_i686.whl", hash = "sha256:ce1bbd7d780bb5a0da032e095c951f7014d6b0a205f8318308140f1a6aba159e", size = 240337, upload-time = "2026-01-26T02:45:58.698Z" }, + { url = "https://files.pythonhosted.org/packages/a1/29/fdc6a43c203890dc2ae9249971ecd0c41deaedfe00d25cb6564b2edd99eb/multidict-6.7.1-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:b26684587228afed0d50cf804cc71062cc9c1cdf55051c4c6345d372947b268c", size = 248788, upload-time = "2026-01-26T02:46:00.862Z" }, + { url = "https://files.pythonhosted.org/packages/a9/14/a153a06101323e4cf086ecee3faadba52ff71633d471f9685c42e3736163/multidict-6.7.1-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:9f9af11306994335398293f9958071019e3ab95e9a707dc1383a35613f6abcb9", size = 242842, upload-time = "2026-01-26T02:46:02.824Z" }, + { url = "https://files.pythonhosted.org/packages/41/5f/604ae839e64a4a6efc80db94465348d3b328ee955e37acb24badbcd24d83/multidict-6.7.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:b4938326284c4f1224178a560987b6cf8b4d38458b113d9b8c1db1a836e640a2", size = 240237, upload-time = "2026-01-26T02:46:05.898Z" }, + { url = "https://files.pythonhosted.org/packages/5f/60/c3a5187bf66f6fb546ff4ab8fb5a077cbdd832d7b1908d4365c7f74a1917/multidict-6.7.1-cp314-cp314t-win32.whl", hash = "sha256:98655c737850c064a65e006a3df7c997cd3b220be4ec8fe26215760b9697d4d7", size = 48008, upload-time = "2026-01-26T02:46:07.468Z" }, + { url = "https://files.pythonhosted.org/packages/0c/f7/addf1087b860ac60e6f382240f64fb99f8bfb532bb06f7c542b83c29ca61/multidict-6.7.1-cp314-cp314t-win_amd64.whl", hash = "sha256:497bde6223c212ba11d462853cfa4f0ae6ef97465033e7dc9940cdb3ab5b48e5", size = 53542, upload-time = "2026-01-26T02:46:08.809Z" }, + { url = "https://files.pythonhosted.org/packages/4c/81/4629d0aa32302ef7b2ec65c75a728cc5ff4fa410c50096174c1632e70b3e/multidict-6.7.1-cp314-cp314t-win_arm64.whl", hash = "sha256:2bbd113e0d4af5db41d5ebfe9ccaff89de2120578164f86a5d17d5a576d1e5b2", size = 44719, upload-time = "2026-01-26T02:46:11.146Z" }, + { url = "https://files.pythonhosted.org/packages/81/08/7036c080d7117f28a4af526d794aab6a84463126db031b007717c1a6676e/multidict-6.7.1-py3-none-any.whl", hash = "sha256:55d97cc6dae627efa6a6e548885712d4864b81110ac76fa4e534c03819fa4a56", size = 12319, upload-time = "2026-01-26T02:46:44.004Z" }, ] [[package]] name = "multiprocess" -version = "0.70.16" +version = "0.70.18" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "dill" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/b5/ae/04f39c5d0d0def03247c2893d6f2b83c136bf3320a2154d7b8858f2ba72d/multiprocess-0.70.16.tar.gz", hash = "sha256:161af703d4652a0e1410be6abccecde4a7ddffd19341be0a7011b94aeb171ac1", size = 1772603, upload-time = "2024-01-28T18:52:34.85Z" } +sdist = { url = "https://files.pythonhosted.org/packages/72/fd/2ae3826f5be24c6ed87266bc4e59c46ea5b059a103f3d7e7eb76a52aeecb/multiprocess-0.70.18.tar.gz", hash = "sha256:f9597128e6b3e67b23956da07cf3d2e5cba79e2f4e0fba8d7903636663ec6d0d", size = 1798503, upload-time = "2025-04-17T03:11:27.742Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/ef/76/6e712a2623d146d314f17598df5de7224c85c0060ef63fd95cc15a25b3fa/multiprocess-0.70.16-pp310-pypy310_pp73-macosx_10_13_x86_64.whl", hash = "sha256:476887be10e2f59ff183c006af746cb6f1fd0eadcfd4ef49e605cbe2659920ee", size = 134980, upload-time = "2024-01-28T18:52:15.731Z" }, - { url = "https://files.pythonhosted.org/packages/0f/ab/1e6e8009e380e22254ff539ebe117861e5bdb3bff1fc977920972237c6c7/multiprocess-0.70.16-pp310-pypy310_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:d951bed82c8f73929ac82c61f01a7b5ce8f3e5ef40f5b52553b4f547ce2b08ec", size = 134982, upload-time = "2024-01-28T18:52:17.783Z" }, - { url = "https://files.pythonhosted.org/packages/bc/f7/7ec7fddc92e50714ea3745631f79bd9c96424cb2702632521028e57d3a36/multiprocess-0.70.16-py310-none-any.whl", hash = "sha256:c4a9944c67bd49f823687463660a2d6daae94c289adff97e0f9d696ba6371d02", size = 134824, upload-time = "2024-01-28T18:52:26.062Z" }, - { url = "https://files.pythonhosted.org/packages/50/15/b56e50e8debaf439f44befec5b2af11db85f6e0f344c3113ae0be0593a91/multiprocess-0.70.16-py311-none-any.whl", hash = "sha256:af4cabb0dac72abfb1e794fa7855c325fd2b55a10a44628a3c1ad3311c04127a", size = 143519, upload-time = "2024-01-28T18:52:28.115Z" }, - { url = "https://files.pythonhosted.org/packages/0a/7d/a988f258104dcd2ccf1ed40fdc97e26c4ac351eeaf81d76e266c52d84e2f/multiprocess-0.70.16-py312-none-any.whl", hash = "sha256:fc0544c531920dde3b00c29863377f87e1632601092ea2daca74e4beb40faa2e", size = 146741, upload-time = "2024-01-28T18:52:29.395Z" }, - { url = "https://files.pythonhosted.org/packages/ea/89/38df130f2c799090c978b366cfdf5b96d08de5b29a4a293df7f7429fa50b/multiprocess-0.70.16-py38-none-any.whl", hash = "sha256:a71d82033454891091a226dfc319d0cfa8019a4e888ef9ca910372a446de4435", size = 132628, upload-time = "2024-01-28T18:52:30.853Z" }, - { url = "https://files.pythonhosted.org/packages/da/d9/f7f9379981e39b8c2511c9e0326d212accacb82f12fbfdc1aa2ce2a7b2b6/multiprocess-0.70.16-py39-none-any.whl", hash = "sha256:a0bafd3ae1b732eac64be2e72038231c1ba97724b60b09400d68f229fcc2fbf3", size = 133351, upload-time = "2024-01-28T18:52:31.981Z" }, -] - -[[package]] -name = "namex" -version = "0.1.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/0c/c0/ee95b28f029c73f8d49d8f52edaed02a1d4a9acb8b69355737fdb1faa191/namex-0.1.0.tar.gz", hash = "sha256:117f03ccd302cc48e3f5c58a296838f6b89c83455ab8683a1e85f2a430aa4306", size = 6649, upload-time = "2025-05-26T23:17:38.918Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/b2/bc/465daf1de06409cdd4532082806770ee0d8d7df434da79c76564d0f69741/namex-0.1.0-py3-none-any.whl", hash = "sha256:e2012a474502f1e2251267062aae3114611f07df4224b6e06334c57b0f2ce87c", size = 5905, upload-time = "2025-05-26T23:17:37.695Z" }, + { url = "https://files.pythonhosted.org/packages/c8/f8/7f9a8f08bf98cea1dfaa181e05cc8bbcb59cecf044b5a9ac3cce39f9c449/multiprocess-0.70.18-pp310-pypy310_pp73-macosx_10_15_x86_64.whl", hash = "sha256:25d4012dcaaf66b9e8e955f58482b42910c2ee526d532844d8bcf661bbc604df", size = 135083, upload-time = "2025-04-17T03:11:04.223Z" }, + { url = "https://files.pythonhosted.org/packages/e5/03/b7b10dbfc17b2b3ce07d4d30b3ba8367d0ed32d6d46cd166e298f161dd46/multiprocess-0.70.18-pp310-pypy310_pp73-macosx_11_0_arm64.whl", hash = "sha256:06b19433de0d02afe5869aec8931dd5c01d99074664f806c73896b0d9e527213", size = 135128, upload-time = "2025-04-17T03:11:06.045Z" }, + { url = "https://files.pythonhosted.org/packages/c1/a3/5f8d3b9690ea5580bee5868ab7d7e2cfca74b7e826b28192b40aa3881cdc/multiprocess-0.70.18-pp310-pypy310_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:6fa1366f994373aaf2d4738b0f56e707caeaa05486e97a7f71ee0853823180c2", size = 135132, upload-time = "2025-04-17T03:11:07.533Z" }, + { url = "https://files.pythonhosted.org/packages/55/4d/9af0d1279c84618bcd35bf5fd7e371657358c7b0a523e54a9cffb87461f8/multiprocess-0.70.18-pp311-pypy311_pp73-macosx_10_15_x86_64.whl", hash = "sha256:8b8940ae30139e04b076da6c5b83e9398585ebdf0f2ad3250673fef5b2ff06d6", size = 144695, upload-time = "2025-04-17T03:11:09.161Z" }, + { url = "https://files.pythonhosted.org/packages/17/bf/87323e79dd0562474fad3373c21c66bc6c3c9963b68eb2a209deb4c8575e/multiprocess-0.70.18-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:0929ba95831adb938edbd5fb801ac45e705ecad9d100b3e653946b7716cb6bd3", size = 144742, upload-time = "2025-04-17T03:11:10.072Z" }, + { url = "https://files.pythonhosted.org/packages/dd/74/cb8c831e58dc6d5cf450b17c7db87f14294a1df52eb391da948b5e0a0b94/multiprocess-0.70.18-pp311-pypy311_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:4d77f8e4bfe6c6e2e661925bbf9aed4d5ade9a1c6502d5dfc10129b9d1141797", size = 144745, upload-time = "2025-04-17T03:11:11.453Z" }, + { url = "https://files.pythonhosted.org/packages/ba/d8/0cba6cf51a1a31f20471fbc823a716170c73012ddc4fb85d706630ed6e8f/multiprocess-0.70.18-py310-none-any.whl", hash = "sha256:60c194974c31784019c1f459d984e8f33ee48f10fcf42c309ba97b30d9bd53ea", size = 134948, upload-time = "2025-04-17T03:11:20.223Z" }, + { url = "https://files.pythonhosted.org/packages/4b/88/9039f2fed1012ef584751d4ceff9ab4a51e5ae264898f0b7cbf44340a859/multiprocess-0.70.18-py311-none-any.whl", hash = "sha256:5aa6eef98e691281b3ad923be2832bf1c55dd2c859acd73e5ec53a66aae06a1d", size = 144462, upload-time = "2025-04-17T03:11:21.657Z" }, + { url = "https://files.pythonhosted.org/packages/bf/b6/5f922792be93b82ec6b5f270bbb1ef031fd0622847070bbcf9da816502cc/multiprocess-0.70.18-py312-none-any.whl", hash = "sha256:9b78f8e5024b573730bfb654783a13800c2c0f2dfc0c25e70b40d184d64adaa2", size = 150287, upload-time = "2025-04-17T03:11:22.69Z" }, + { url = "https://files.pythonhosted.org/packages/ee/25/7d7e78e750bc1aecfaf0efbf826c69a791d2eeaf29cf20cba93ff4cced78/multiprocess-0.70.18-py313-none-any.whl", hash = "sha256:871743755f43ef57d7910a38433cfe41319e72be1bbd90b79c7a5ac523eb9334", size = 151917, upload-time = "2025-04-17T03:11:24.044Z" }, + { url = "https://files.pythonhosted.org/packages/3b/c3/ca84c19bd14cdfc21c388fdcebf08b86a7a470ebc9f5c3c084fc2dbc50f7/multiprocess-0.70.18-py38-none-any.whl", hash = "sha256:dbf705e52a154fe5e90fb17b38f02556169557c2dd8bb084f2e06c2784d8279b", size = 132636, upload-time = "2025-04-17T03:11:24.936Z" }, + { url = "https://files.pythonhosted.org/packages/6c/28/dd72947e59a6a8c856448a5e74da6201cb5502ddff644fbc790e4bd40b9a/multiprocess-0.70.18-py39-none-any.whl", hash = "sha256:e78ca805a72b1b810c690b6b4cc32579eba34f403094bbbae962b7b5bf9dfcb8", size = 133478, upload-time = "2025-04-17T03:11:26.253Z" }, ] [[package]] @@ -1348,7 +1407,7 @@ wheels = [ [[package]] name = "networkx" -version = "3.5" +version = "3.6.1" source = { registry = "https://pypi.org/simple" } resolution-markers = [ "python_full_version >= '3.14'", @@ -1356,18 +1415,18 @@ resolution-markers = [ "python_full_version == '3.12.*'", "python_full_version == '3.11.*'", ] -sdist = { url = "https://files.pythonhosted.org/packages/6c/4f/ccdb8ad3a38e583f214547fd2f7ff1fc160c43a75af88e6aec213404b96a/networkx-3.5.tar.gz", hash = "sha256:d4c6f9cf81f52d69230866796b82afbccdec3db7ae4fbd1b65ea750feed50037", size = 2471065, upload-time = "2025-05-29T11:35:07.804Z" } +sdist = { url = "https://files.pythonhosted.org/packages/6a/51/63fe664f3908c97be9d2e4f1158eb633317598cfa6e1fc14af5383f17512/networkx-3.6.1.tar.gz", hash = "sha256:26b7c357accc0c8cde558ad486283728b65b6a95d85ee1cd66bafab4c8168509", size = 2517025, upload-time = "2025-12-08T17:02:39.908Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/eb/8d/776adee7bbf76365fdd7f2552710282c79a4ead5d2a46408c9043a2b70ba/networkx-3.5-py3-none-any.whl", hash = "sha256:0030d386a9a06dee3565298b4a734b68589749a544acbb6c412dc9e2489ec6ec", size = 2034406, upload-time = "2025-05-29T11:35:04.961Z" }, + { url = "https://files.pythonhosted.org/packages/9e/c9/b2622292ea83fbb4ec318f5b9ab867d0a28ab43c5717bb85b0a5f6b3b0a4/networkx-3.6.1-py3-none-any.whl", hash = "sha256:d47fbf302e7d9cbbb9e2555a0d267983d2aa476bac30e90dfbe5669bd57f3762", size = 2068504, upload-time = "2025-12-08T17:02:38.159Z" }, ] [[package]] name = "nodeenv" -version = "1.9.1" +version = "1.10.0" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/43/16/fc88b08840de0e0a72a2f9d8c6bae36be573e475a6326ae854bcc549fc45/nodeenv-1.9.1.tar.gz", hash = "sha256:6ec12890a2dab7946721edbfbcd91f3319c6ccc9aec47be7c7e6b7011ee6645f", size = 47437, upload-time = "2024-06-04T18:44:11.171Z" } +sdist = { url = "https://files.pythonhosted.org/packages/24/bf/d1bda4f6168e0b2e9e5958945e01910052158313224ada5ce1fb2e1113b8/nodeenv-1.10.0.tar.gz", hash = "sha256:996c191ad80897d076bdfba80a41994c2b47c68e224c542b48feba42ba00f8bb", size = 55611, upload-time = "2025-12-20T14:08:54.006Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/d2/1d/1b658dbd2b9fa9c4c9f32accbfc0205d532c8c6194dc0f2a4c0428e7128a/nodeenv-1.9.1-py2.py3-none-any.whl", hash = "sha256:ba11c9782d29c27c70ffbdda2d7415098754709be8a7056d79a737cd901155c9", size = 22314, upload-time = "2024-06-04T18:44:08.352Z" }, + { url = "https://files.pythonhosted.org/packages/88/b2/d0896bdcdc8d28a7fc5717c305f1a861c26e18c05047949fb371034d98bd/nodeenv-1.10.0-py2.py3-none-any.whl", hash = "sha256:5bb13e3eed2923615535339b3c620e76779af4cb4c6a90deccc9e36b274d3827", size = 23438, upload-time = "2025-12-20T14:08:52.782Z" }, ] [[package]] @@ -1437,89 +1496,140 @@ wheels = [ [[package]] name = "numpy" -version = "2.3.3" +version = "2.4.6" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "python_full_version == '3.11.*'", +] +sdist = { url = "https://files.pythonhosted.org/packages/d0/ad/fed0499ce6a338d2a03ebae59cd15093910c8875328855781952abf6c2fe/numpy-2.4.6.tar.gz", hash = "sha256:f3a3570c4a2a16746ac2c31a7c7c7b0c186b95ce902e33db6f28094ed7387dda", size = 20735807, upload-time = "2026-05-18T23:37:14.07Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b3/49/ec46835a70be8fa6446c495126ac84fdb28cb2558e1620ffb87a10c8b64c/numpy-2.4.6-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:0280e0356c0829a18d9de1cb7eee50ec22ca639878d7240307ca0943d73cd2c4", size = 16969194, upload-time = "2026-05-18T23:33:13.503Z" }, + { url = "https://files.pythonhosted.org/packages/0e/0d/f5957185c0ee2f3e12f78715aa9e3b353fd83633316c8532b38faa37e3f6/numpy-2.4.6-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:110f8b71aacb688ec69062bb7f6938a0f8acb01b7c1c4beb453c65b6d234584d", size = 14964111, upload-time = "2026-05-18T23:33:17.795Z" }, + { url = "https://files.pythonhosted.org/packages/ad/40/40a40ee0ddf7ceb782c49af278894b686e586d65d8c1889c8b5da01a3d7d/numpy-2.4.6-cp311-cp311-macosx_14_0_arm64.whl", hash = "sha256:4cfe66903cc32a9921a6733d96b19bb6abf310397581bbad89c228f5abaf0ee8", size = 5469159, upload-time = "2026-05-18T23:33:20.654Z" }, + { url = "https://files.pythonhosted.org/packages/63/13/f9a8046535cb21deae82f8d03de9617e08882d274fad2539630761888228/numpy-2.4.6-cp311-cp311-macosx_14_0_x86_64.whl", hash = "sha256:8155154c7c691289fe18f510b5d4657c68c67989f293f0535a91360392ff6538", size = 6798936, upload-time = "2026-05-18T23:33:22.987Z" }, + { url = "https://files.pythonhosted.org/packages/33/a8/6fa8c1a345a8c85dbb21932c447bee07c30a2c2a3f31e369c0a84b300147/numpy-2.4.6-cp311-cp311-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0ab0a9c4ffb1a6d95ef519fe4247dba8eb6b18ad93999f76b7f657039acabd47", size = 15966692, upload-time = "2026-05-18T23:33:26.62Z" }, + { url = "https://files.pythonhosted.org/packages/02/03/74fe2a4cb3817d94d86402f2506554130a2f01414e299b5a843e5a8a957f/numpy-2.4.6-cp311-cp311-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:89cd468399cfd2504718f0ba50e410dca55a170b61a02ad92bb18c8a65186e93", size = 16918164, upload-time = "2026-05-18T23:33:29.955Z" }, + { url = "https://files.pythonhosted.org/packages/c5/80/3615be3313f7e7696609bc194b9f0101da809df79e859bdb84e0cd043f46/numpy-2.4.6-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:c2d37ab77531417474168eb79d6d80b14f821a966818505d03013d0833edb7a8", size = 17322877, upload-time = "2026-05-18T23:33:34.724Z" }, + { url = "https://files.pythonhosted.org/packages/ca/ac/a691e0fe2675e370d0e08ff905adc49a1c8830e8cae03efe4477e92cd55d/numpy-2.4.6-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:f407cb6b8e9d6d8c626bc73c945db1706035af8fd632295547bf1c9e46d092d6", size = 18651487, upload-time = "2026-05-18T23:33:38.217Z" }, + { url = "https://files.pythonhosted.org/packages/15/a7/9bc1cd626d7bf6869bfedf27b91b6ab5dd607758bf8e959d6fa80c6a59cb/numpy-2.4.6-cp311-cp311-win32.whl", hash = "sha256:ddea102b48f9e339f3948bf22040944184627a30fdf7f858667673b9c5f033c8", size = 6233945, upload-time = "2026-05-18T23:33:41.331Z" }, + { url = "https://files.pythonhosted.org/packages/c5/31/7fc6239c12bce7e931463251cca4426c465e1876ba3cc785402ef4dd8f4e/numpy-2.4.6-cp311-cp311-win_amd64.whl", hash = "sha256:1e254a00cdf42b1e4d5b3d68d33af63268d41340d8885df2ab6470f2e1500147", size = 12608406, upload-time = "2026-05-18T23:33:44.131Z" }, + { url = "https://files.pythonhosted.org/packages/27/83/140f85a466595a16382996a1bf06b2b54bcd597488921b0c9daaeeda72af/numpy-2.4.6-cp311-cp311-win_arm64.whl", hash = "sha256:ed9749eef4cbd126da3dc1d6bcb3a57f5eb7ac6a6484146bdbf743f552dfc577", size = 10479528, upload-time = "2026-05-18T23:33:50.725Z" }, + { url = "https://files.pythonhosted.org/packages/95/2a/3d7b5ac8aac24feaf9ad7ed58f45b0bbc06d37e4338ae84c9f2298b570f9/numpy-2.4.6-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:001fbb8e08d942dd57599e781f2472269ee7f2755fae407b4f67b2f0b17da3f1", size = 16689119, upload-time = "2026-05-18T23:33:54.065Z" }, + { url = "https://files.pythonhosted.org/packages/ea/12/92c4c131527599e8288d6918e888d88726f84d805d784b771f32408aeaef/numpy-2.4.6-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:ebfb099f8dcf083deef3ac1ca4c1503f387cf76296fcb3816b66f5ecb5f54fdb", size = 14699246, upload-time = "2026-05-18T23:33:57.621Z" }, + { url = "https://files.pythonhosted.org/packages/ad/fe/c0a6b7b2ca128a8fb228575147073b660656734b8ebe4d76c8fd748dcc79/numpy-2.4.6-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:3213d622a0283a39a93d188f3cf72b26862df52fbb4ca3697f51705016523d41", size = 5204410, upload-time = "2026-05-18T23:34:00.302Z" }, + { url = "https://files.pythonhosted.org/packages/f3/d4/9770d14ba719432bb90a421bfd443872ed0f70f7264b64bec12ea363d5fd/numpy-2.4.6-cp312-cp312-macosx_14_0_x86_64.whl", hash = "sha256:357cc07a6d7b0b182ff02249616a03742827ebb1277546b5c7cd7f7620a45698", size = 6551240, upload-time = "2026-05-18T23:34:02.852Z" }, + { url = "https://files.pythonhosted.org/packages/c9/c6/50a46a6205feba2343f1d6d17438107c5dc491ed1c736e6ea68689fd906b/numpy-2.4.6-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5f9fb9157b4ce2971008323afe46053787b526ef624fea915b261468a8421a0f", size = 15671012, upload-time = "2026-05-18T23:34:05.485Z" }, + { url = "https://files.pythonhosted.org/packages/99/60/14115e6364fa676c5397c2ad3004e527e9aa487abf5d0706ec81bbd08529/numpy-2.4.6-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:90f9849678c75fe7afa2d348ac842c168b0a4d3d61919687216dfc547976d853", size = 16645538, upload-time = "2026-05-18T23:34:09.265Z" }, + { url = "https://files.pythonhosted.org/packages/ae/c5/693cbe59e57db94d2231fa519ca3978dc9e19da5a8f088588f5c6e947ff2/numpy-2.4.6-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:c1a2af6c6ef86344a6b0db6b97834208bf598db514f2b155042439b62605601a", size = 17020706, upload-time = "2026-05-18T23:34:13.053Z" }, + { url = "https://files.pythonhosted.org/packages/ef/fc/85b7c4eff9b4966ade25c2273cf7e7012e92366c032058653934b37de044/numpy-2.4.6-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:e5805d5a22fd19c8ccff10a9561f9df94436b0545619ea579db2d3c35294bce2", size = 18368541, upload-time = "2026-05-18T23:34:17.024Z" }, + { url = "https://files.pythonhosted.org/packages/f6/81/e1b27545deedce7f4a0b348618c6b62d74e36a4dc9ccd42f3eb2f85eee32/numpy-2.4.6-cp312-cp312-win32.whl", hash = "sha256:e3eeb0aabd6bd5ce64faae67e9935203a6991b4bc2a485a767fbafb2c5125f45", size = 5962825, upload-time = "2026-05-18T23:34:20.3Z" }, + { url = "https://files.pythonhosted.org/packages/ab/ca/feab00bd44aa5fe1ad2c18f08b4d3bb92e26484b0b1d1443897809ed528c/numpy-2.4.6-cp312-cp312-win_amd64.whl", hash = "sha256:d8e8286dd7cea7895157318d1b91cdacac64c479f3cbc8dce548331728484751", size = 12321687, upload-time = "2026-05-18T23:34:23.095Z" }, + { url = "https://files.pythonhosted.org/packages/63/cf/5a6d34850a39d1093558564f77ee8e8e0bee5061151b8f05a55711001ec7/numpy-2.4.6-cp312-cp312-win_arm64.whl", hash = "sha256:4081eb135ac24158bd51cdfbef16f1c64df7063b1143f24731387137c092bec8", size = 10221482, upload-time = "2026-05-18T23:34:25.876Z" }, + { url = "https://files.pythonhosted.org/packages/fb/82/bdab26d7438c6791ca31b7c024ca37c1eab8b726ba236129005cd4a06e45/numpy-2.4.6-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:511dbaf848decaaaf4b4ca48032619fb3138710c4bf7da7617765edad1ef96b0", size = 16684648, upload-time = "2026-05-18T23:34:29.41Z" }, + { url = "https://files.pythonhosted.org/packages/1b/30/a80189bcc7f5e4258b3fbc3968d909d1756f54d023299ecc39ad6fdb9ef8/numpy-2.4.6-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:bf162abab1c1a736333192707cef898e735a5ca00f38f27eeedf44b39d9e85eb", size = 14693902, upload-time = "2026-05-18T23:34:33.013Z" }, + { url = "https://files.pythonhosted.org/packages/97/12/70b5d0d7c15e1ebb8a6a84a8caa1d19e181d84fb58bb6d70aca29099dec1/numpy-2.4.6-cp313-cp313-macosx_14_0_arm64.whl", hash = "sha256:043191bfa8eab18c776647b62723ac9dddece59743b13f49b2016094129c2b3f", size = 5198992, upload-time = "2026-05-18T23:34:36.132Z" }, + { url = "https://files.pythonhosted.org/packages/ba/8c/ebd2a8f8a83541f8d38cc5667e8c2b69cecfd30da6e45693e8158857d44b/numpy-2.4.6-cp313-cp313-macosx_14_0_x86_64.whl", hash = "sha256:6180d8b35af935aed8ece3a85e0a43f87393ae0ac87c8d2c8bd2c993f7270ef3", size = 6546944, upload-time = "2026-05-18T23:34:38.484Z" }, + { url = "https://files.pythonhosted.org/packages/bb/c5/7b863a97a91671a0338f4253bd3b5a3d3852f0692dae91711c9f4a10e787/numpy-2.4.6-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:72fbe16c6fac95aedf5937fa873445cec2110be35d8a4e9433d7501fd98dae6b", size = 15669392, upload-time = "2026-05-18T23:34:41.257Z" }, + { url = "https://files.pythonhosted.org/packages/a5/9d/3584b9984ca4c047aea75214ce1a4c4c73d849bd71b604264b7f5653f8a8/numpy-2.4.6-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a7830bab239b79cda9c08c2da014761cafb48da6150e1da17ac06283f43b6089", size = 16633220, upload-time = "2026-05-18T23:34:45.075Z" }, + { url = "https://files.pythonhosted.org/packages/05/ae/7c67fba23bd98caec7c99261f3a16072ade14813486b0282cb29846de832/numpy-2.4.6-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:ef4aea96ce4d3b074422cb4f2f64e216bf9e213004bb58ecfdf50ea02ea8eb9a", size = 17020800, upload-time = "2026-05-18T23:34:49.065Z" }, + { url = "https://files.pythonhosted.org/packages/d9/5d/3b6725cb31d983c5e66916f5d36f6d7e5521129e4c4404d64f918292a5b6/numpy-2.4.6-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:dfa20cc6ca228e6b155b11da03825975ce66aea520985dbbddf0f2a5a495c605", size = 18357600, upload-time = "2026-05-18T23:34:52.709Z" }, + { url = "https://files.pythonhosted.org/packages/f7/da/2ccc6c2fe8898dee01d90c75c5f5f914a23daf99e3e0f59516a08760c8b5/numpy-2.4.6-cp313-cp313-win32.whl", hash = "sha256:56b39e5e0622a09a25bf5baf62f4bcf0cb8a41ae6e2819cf49bbc5a74c083f91", size = 5961134, upload-time = "2026-05-18T23:34:55.618Z" }, + { url = "https://files.pythonhosted.org/packages/b5/cd/9cc4dc876fb065d5c220aae4d5e14826b2715331bb7618ce1fb07a679d99/numpy-2.4.6-cp313-cp313-win_amd64.whl", hash = "sha256:c4fc99836233ea196540b17ab0983aff60ed07941751930f5f4d05bc3b3b7359", size = 12318598, upload-time = "2026-05-18T23:34:58.928Z" }, + { url = "https://files.pythonhosted.org/packages/39/1e/c0bcba1f8694116485fe28fd1be698c278fcda4141c5b0e53a2aed8b12a8/numpy-2.4.6-cp313-cp313-win_arm64.whl", hash = "sha256:a7c711e21628b52034bb5ab8d1bce291f752fcc5e92accc615778acee1ff4778", size = 10222272, upload-time = "2026-05-18T23:35:02.167Z" }, + { url = "https://files.pythonhosted.org/packages/63/6d/cc5619247c8f4204e507f5883528372e4ac4bb189e579fb859a12e480b1f/numpy-2.4.6-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:112b06a867b235ef466ed3508ddf0238050df9c727cafb5301ac385b899189a1", size = 14821197, upload-time = "2026-05-18T23:35:05.468Z" }, + { url = "https://files.pythonhosted.org/packages/00/58/f1c39161c87d9e9bed660f1ed4bafc0e403d5ec9650b6dd77aead07d489b/numpy-2.4.6-cp313-cp313t-macosx_14_0_arm64.whl", hash = "sha256:eaf7fa2de5c0be8ae6ff8e9bea2ccd725e980541244521d8d4b5f3354a27babe", size = 5326287, upload-time = "2026-05-18T23:35:08.693Z" }, + { url = "https://files.pythonhosted.org/packages/af/57/3917ab0fd97f271a8694513581b8a36c655f111c446852c302f04ccdb6fc/numpy-2.4.6-cp313-cp313t-macosx_14_0_x86_64.whl", hash = "sha256:7265a2f3d436e54ef9f2b52b5c937e6be778781bd97a590319d7348f1c1ca997", size = 6646763, upload-time = "2026-05-18T23:35:11.459Z" }, + { url = "https://files.pythonhosted.org/packages/eb/0f/037e64c494b67581ae18193d770adef354c41f3f2c8ebf865602d949bf8f/numpy-2.4.6-cp313-cp313t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f74a575920ab21fe304421a3fc28793d82e299cae9eccb37084e9fc7f3617c20", size = 15728070, upload-time = "2026-05-18T23:35:14.79Z" }, + { url = "https://files.pythonhosted.org/packages/21/a6/5d2bae9c9542eb4df16dc9c46dc79c186e9bad53805dfa5399a6023c6db0/numpy-2.4.6-cp313-cp313t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ede83e07a75dd06bc501566c1eca2afc0d61677c1472ac9ad93fdee6e638a48d", size = 16681752, upload-time = "2026-05-18T23:35:18.836Z" }, + { url = "https://files.pythonhosted.org/packages/92/14/23d1dfb410ae362cd59ce53e936b1513d545eb40db3949ced632e19a459e/numpy-2.4.6-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:68bb27509ac1b9a3443094260f6326150663b06abe40b73a2f81160623da5b67", size = 17086024, upload-time = "2026-05-18T23:35:22.52Z" }, + { url = "https://files.pythonhosted.org/packages/4b/6e/23595a2c642cdf3bc567877064bdd7f91c8b0038a4453cf2daf7248eafe9/numpy-2.4.6-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:a0df0043bdb289bde1f62da130d20df23d58b45429f752bc7a8fc5325a225ecd", size = 18403398, upload-time = "2026-05-18T23:35:26.398Z" }, + { url = "https://files.pythonhosted.org/packages/8a/90/0ac3bc947217e66dec77e7cbc6a1979d1af70b6461b82f620d3bccd5e4c8/numpy-2.4.6-cp313-cp313t-win32.whl", hash = "sha256:29a287e0cf63ff528da061de6b9f64a4618da591ca1046aafc54062e40ca7eab", size = 6084971, upload-time = "2026-05-18T23:35:29.387Z" }, + { url = "https://files.pythonhosted.org/packages/77/71/5673e351671a1d2bd6063b91b44f70c0affea7d1516fa7a6572941ba4aa1/numpy-2.4.6-cp313-cp313t-win_amd64.whl", hash = "sha256:25c692919ac5a01f170a3bfcd62d745b24fd095c353d50812637d6fcab442e75", size = 12458532, upload-time = "2026-05-18T23:35:32.175Z" }, + { url = "https://files.pythonhosted.org/packages/3f/88/19d3503c5046e688f049274b27a3ef3d771152fa80d3ba3d01a3dff61abe/numpy-2.4.6-cp313-cp313t-win_arm64.whl", hash = "sha256:1e978ec1e8bd0e0e4de6bb75de9d30cbb74db6b6a2bb727618613703ca0167dd", size = 10291881, upload-time = "2026-05-18T23:35:35.465Z" }, + { url = "https://files.pythonhosted.org/packages/f8/91/3ab2044d05fd16d343c5ac2e69b127f1b2854040dd20b193257c78028bd3/numpy-2.4.6-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:06ca2f61ec4385a07a6977c55ba998a4466c123642b4a32694d3128fce18c079", size = 16683458, upload-time = "2026-05-18T23:35:38.353Z" }, + { url = "https://files.pythonhosted.org/packages/8e/62/764ce66fa4147ae6d73071a3abf804ffe606f174618697c571acdf26a7c9/numpy-2.4.6-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:38efbc8de75c7a0fc1ac190162d892787f3f47b57cc291231aafee36b80982b7", size = 14704559, upload-time = "2026-05-18T23:35:42.14Z" }, + { url = "https://files.pythonhosted.org/packages/60/61/23f27c172f022e04025b7dc2367f4d63c1a398120607ec896228649a6f48/numpy-2.4.6-cp314-cp314-macosx_14_0_arm64.whl", hash = "sha256:d581b735e177fdcdce6fed8e7e8880a3fb6ee4e3653a3ac6af01c6f4c03effc5", size = 5209716, upload-time = "2026-05-18T23:35:45.377Z" }, + { url = "https://files.pythonhosted.org/packages/03/71/21cf70dc6ea3e3acb95fc53a265b2fc248b981f0194ceb5b475271b8809d/numpy-2.4.6-cp314-cp314-macosx_14_0_x86_64.whl", hash = "sha256:0a041d3d761dc3c35cc56ce0351506a02bcbc25f7b169f652435141a17db9096", size = 6543947, upload-time = "2026-05-18T23:35:47.926Z" }, + { url = "https://files.pythonhosted.org/packages/d5/91/64288395ee1799bd2e0b04a305dce9666da90c961e1f3fe982a05ee1c036/numpy-2.4.6-cp314-cp314-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:40fdc1ae7125e518ea98e53e69a4ebc27e1fd50510c47b7ea130cf21e5e1d42b", size = 15685197, upload-time = "2026-05-18T23:35:50.863Z" }, + { url = "https://files.pythonhosted.org/packages/f3/eb/ebffaa97dc55502df69584a8f0dcf07f69a3e0b3e2323670a2722db9aa39/numpy-2.4.6-cp314-cp314-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a2c306dea656c12c68f51f4cea133cbe78ca7435eb28c735eac1d3ebe73be6e8", size = 16638245, upload-time = "2026-05-18T23:35:54.752Z" }, + { url = "https://files.pythonhosted.org/packages/b8/0b/54f9da33128d7e350fab89c7455902eeae70349ee52bddb448dc4a576f45/numpy-2.4.6-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:33111801a01c12a8a1e3721f0a9232f8cfc8ae2c6b7098167e6f623c6073f402", size = 17036587, upload-time = "2026-05-18T23:35:58.355Z" }, + { url = "https://files.pythonhosted.org/packages/b6/f0/fdebc1052db1cc37c64beb22072d67cd6d1c71adca1299f53dec2b5e20d3/numpy-2.4.6-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:ae506e6902902557576a26ff33eda8695e7ecb3cb36c3b573a0765dee114ebdb", size = 18363226, upload-time = "2026-05-18T23:36:02.845Z" }, + { url = "https://files.pythonhosted.org/packages/aa/b4/298628d98c72b57e57f7165ae6a481a1deaf6f3c28262a6e4c739c275930/numpy-2.4.6-cp314-cp314-win32.whl", hash = "sha256:aaf159caa35993cb1f56fb9b8e4610d35758e7ca005412eb1daa856a78c9c4b1", size = 6010196, upload-time = "2026-05-18T23:36:05.92Z" }, + { url = "https://files.pythonhosted.org/packages/df/ac/46de6dda46478f7942f839e094970be2d4a861e005c4b3bf07c92e291a09/numpy-2.4.6-cp314-cp314-win_amd64.whl", hash = "sha256:b507f5c4c1d508876d1819b6bf9a49d365b96320b5d4993426b33a23ca4b8261", size = 12450334, upload-time = "2026-05-18T23:36:09.107Z" }, + { url = "https://files.pythonhosted.org/packages/78/92/b8b798ac784102c0da830d2257d59358e3d3d90d1e2b3f2575dad976c5cf/numpy-2.4.6-cp314-cp314-win_arm64.whl", hash = "sha256:6f41ae150c4e32db4f3310cdaf64b1593a03dbabe29eec77fc9b50fe64061df6", size = 10495678, upload-time = "2026-05-18T23:36:12.766Z" }, + { url = "https://files.pythonhosted.org/packages/30/34/ec28d1aa8115971537c01469ab2011ee96827930f0a124de1000cc2a7ed7/numpy-2.4.6-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:ece3d2cfe132e7d51f44a832b303895e6f2d499c5e74dfbdb06ee246147a304a", size = 14823672, upload-time = "2026-05-18T23:36:16.473Z" }, + { url = "https://files.pythonhosted.org/packages/16/bd/f6d1fede4e54e8042a7ff97bb495510f3c220f94bcd9e8b228e87c92cc0d/numpy-2.4.6-cp314-cp314t-macosx_14_0_arm64.whl", hash = "sha256:e3e5193ef5a3dc73bceee50f7fdc2c90dbb76c42df8d8fae3d1067a583df579e", size = 5328731, upload-time = "2026-05-18T23:36:19.767Z" }, + { url = "https://files.pythonhosted.org/packages/f4/f0/e105b9e2fd728a9910103884decd6951d9dd73896b914a98d9a231de02ee/numpy-2.4.6-cp314-cp314t-macosx_14_0_x86_64.whl", hash = "sha256:17f9ade344e7d9b464a084d69bcf18fc691cb1db67c62ed80820bf4926d78f0e", size = 6649805, upload-time = "2026-05-18T23:36:22.266Z" }, + { url = "https://files.pythonhosted.org/packages/82/dd/1206a7ca6ab15e3f02069707ca96222e202af681bb73756da7527f3cb837/numpy-2.4.6-cp314-cp314t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9cd5ffd25db4e7ba6a375693b3fc0fc1791ec636c17db3720da19bde7180ec43", size = 15730496, upload-time = "2026-05-18T23:36:25.713Z" }, + { url = "https://files.pythonhosted.org/packages/51/e7/38d3ea825dcab85a591734decb2f6c67caa7c8367d374df1a1c3842f9b07/numpy-2.4.6-cp314-cp314t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7d92c3819208a60205a12a245c91ad70cb0a85336659b19b834205573ac8456e", size = 16679616, upload-time = "2026-05-18T23:36:29.652Z" }, + { url = "https://files.pythonhosted.org/packages/93/b7/caabfdf53edf663e0b4eb74d7d405d83baef09eb5e83bcd32d601d72b93e/numpy-2.4.6-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:e85b752a1e912b70eaad4fafbd4d1238007ab221de2009b9a2f5ae7461239895", size = 17085145, upload-time = "2026-05-18T23:36:33.449Z" }, + { url = "https://files.pythonhosted.org/packages/f9/45/68d7c33a6bcf3e5aa3bdbd57a367e6f615286dfd6482f97e8ffeb734306e/numpy-2.4.6-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:29cb7f67d10b479ff07c17d33e39f78c07f71c40ef30d63c153d340e96cd3fb4", size = 18403813, upload-time = "2026-05-18T23:36:37.369Z" }, + { url = "https://files.pythonhosted.org/packages/9c/50/0753655aa844c99cd9e018aacf76f130f1bd81d881bb74bc0aef5d73a8ba/numpy-2.4.6-cp314-cp314t-win32.whl", hash = "sha256:260a5d70215b61ab4fadf5c7baacd64821842975eea312125ed3c39a6391b063", size = 6156982, upload-time = "2026-05-18T23:36:40.817Z" }, + { url = "https://files.pythonhosted.org/packages/b2/d4/7c67becf668f973cb490cec3e98dfd799d866f9c989a54d355672cfa0db6/numpy-2.4.6-cp314-cp314t-win_amd64.whl", hash = "sha256:81a1cca95ed5bb92aa8b10dd2cdc9a0d3853a50fad926c28b5d7e8ea54389627", size = 12638908, upload-time = "2026-05-18T23:36:43.996Z" }, + { url = "https://files.pythonhosted.org/packages/43/bb/e1c71a4295b1b1d1393d50dbb4f2a36283c6859d9d3892e84f00ec5a91d5/numpy-2.4.6-cp314-cp314t-win_arm64.whl", hash = "sha256:0c9136e14ed34a9e343a31c533d78a9813a69a3148332bce5e9821cb2f996e66", size = 10565867, upload-time = "2026-05-18T23:36:47.114Z" }, + { url = "https://files.pythonhosted.org/packages/de/12/b422cc84439adc0d00de605bf4a308890ae5c26f2c71fbd73e5d08fbb0dd/numpy-2.4.6-pp311-pypy311_pp73-macosx_10_15_x86_64.whl", hash = "sha256:55cced7c52e981362f708ad635198e97a752dfba412cc03c23bbf3bd8d5cd662", size = 16847511, upload-time = "2026-05-18T23:36:50.673Z" }, + { url = "https://files.pythonhosted.org/packages/44/53/f481bef68011740f8849418d82db07230e825013f31f4eef5ba5b805316a/numpy-2.4.6-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:d6da64deb6b8ed903e7560180a92f2d804ee1ba5eeb849ac2748b8c1aba1f6d7", size = 14889064, upload-time = "2026-05-18T23:36:53.879Z" }, + { url = "https://files.pythonhosted.org/packages/7f/57/42ed575c10ced8af951d426bc4e1f8aff16fd851db33f067036215a7f860/numpy-2.4.6-pp311-pypy311_pp73-macosx_14_0_arm64.whl", hash = "sha256:68a5124b13fa6cc2086764a20005d30bc0548146f7f5322f02fce212ca14317f", size = 5394157, upload-time = "2026-05-18T23:36:57.194Z" }, + { url = "https://files.pythonhosted.org/packages/6a/ef/f66cc724fcc36c1e364c67f51ae9146090b8b584f27d58b97fdae3edd737/numpy-2.4.6-pp311-pypy311_pp73-macosx_14_0_x86_64.whl", hash = "sha256:948424b06129ce883307e8cff868c31396d8dc7630a59c61d70d98dbe70f222c", size = 6708728, upload-time = "2026-05-18T23:36:59.575Z" }, + { url = "https://files.pythonhosted.org/packages/1a/9c/c531f2293b91265d8b48e9b329f54fdd7ffae73cb4134ea10cca4237e9cc/numpy-2.4.6-pp311-pypy311_pp73-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5dbbdb29840ca3d91ee0fece42fc29278886d908280bfec0a5846c6f901a3eb0", size = 15798374, upload-time = "2026-05-18T23:37:02.674Z" }, + { url = "https://files.pythonhosted.org/packages/1a/b0/413077f6b1153ed3cba361401c6783bbad6114804a000cc22eb71c13e190/numpy-2.4.6-pp311-pypy311_pp73-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8ad03c0965fb3c692200e74d458ca28c1dbb4ce96f9a479a8aa041ad5fabca02", size = 16747286, upload-time = "2026-05-18T23:37:06.327Z" }, + { url = "https://files.pythonhosted.org/packages/15/ce/e5ec180bc41812edcd8daeb8639d205622c0e8c02259d8ab25a0201b3c2a/numpy-2.4.6-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:2803abfebfc990042cd494d8ce2d5f82e9d847af6d35ec486923aa19dbad5e73", size = 12504263, upload-time = "2026-05-18T23:37:09.715Z" }, +] + +[[package]] +name = "numpy" +version = "2.5.1" source = { registry = "https://pypi.org/simple" } resolution-markers = [ "python_full_version >= '3.14'", "python_full_version == '3.13.*'", "python_full_version == '3.12.*'", - "python_full_version == '3.11.*'", ] -sdist = { url = "https://files.pythonhosted.org/packages/d0/19/95b3d357407220ed24c139018d2518fab0a61a948e68286a25f1a4d049ff/numpy-2.3.3.tar.gz", hash = "sha256:ddc7c39727ba62b80dfdbedf400d1c10ddfa8eefbd7ec8dcb118be8b56d31029", size = 20576648, upload-time = "2025-09-09T16:54:12.543Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/7a/45/e80d203ef6b267aa29b22714fb558930b27960a0c5ce3c19c999232bb3eb/numpy-2.3.3-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:0ffc4f5caba7dfcbe944ed674b7eef683c7e94874046454bb79ed7ee0236f59d", size = 21259253, upload-time = "2025-09-09T15:56:02.094Z" }, - { url = "https://files.pythonhosted.org/packages/52/18/cf2c648fccf339e59302e00e5f2bc87725a3ce1992f30f3f78c9044d7c43/numpy-2.3.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:e7e946c7170858a0295f79a60214424caac2ffdb0063d4d79cb681f9aa0aa569", size = 14450980, upload-time = "2025-09-09T15:56:05.926Z" }, - { url = "https://files.pythonhosted.org/packages/93/fb/9af1082bec870188c42a1c239839915b74a5099c392389ff04215dcee812/numpy-2.3.3-cp311-cp311-macosx_14_0_arm64.whl", hash = "sha256:cd4260f64bc794c3390a63bf0728220dd1a68170c169088a1e0dfa2fde1be12f", size = 5379709, upload-time = "2025-09-09T15:56:07.95Z" }, - { url = "https://files.pythonhosted.org/packages/75/0f/bfd7abca52bcbf9a4a65abc83fe18ef01ccdeb37bfb28bbd6ad613447c79/numpy-2.3.3-cp311-cp311-macosx_14_0_x86_64.whl", hash = "sha256:f0ddb4b96a87b6728df9362135e764eac3cfa674499943ebc44ce96c478ab125", size = 6913923, upload-time = "2025-09-09T15:56:09.443Z" }, - { url = "https://files.pythonhosted.org/packages/79/55/d69adad255e87ab7afda1caf93ca997859092afeb697703e2f010f7c2e55/numpy-2.3.3-cp311-cp311-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:afd07d377f478344ec6ca2b8d4ca08ae8bd44706763d1efb56397de606393f48", size = 14589591, upload-time = "2025-09-09T15:56:11.234Z" }, - { url = "https://files.pythonhosted.org/packages/10/a2/010b0e27ddeacab7839957d7a8f00e91206e0c2c47abbb5f35a2630e5387/numpy-2.3.3-cp311-cp311-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:bc92a5dedcc53857249ca51ef29f5e5f2f8c513e22cfb90faeb20343b8c6f7a6", size = 16938714, upload-time = "2025-09-09T15:56:14.637Z" }, - { url = "https://files.pythonhosted.org/packages/1c/6b/12ce8ede632c7126eb2762b9e15e18e204b81725b81f35176eac14dc5b82/numpy-2.3.3-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:7af05ed4dc19f308e1d9fc759f36f21921eb7bbfc82843eeec6b2a2863a0aefa", size = 16370592, upload-time = "2025-09-09T15:56:17.285Z" }, - { url = "https://files.pythonhosted.org/packages/b4/35/aba8568b2593067bb6a8fe4c52babb23b4c3b9c80e1b49dff03a09925e4a/numpy-2.3.3-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:433bf137e338677cebdd5beac0199ac84712ad9d630b74eceeb759eaa45ddf30", size = 18884474, upload-time = "2025-09-09T15:56:20.943Z" }, - { url = "https://files.pythonhosted.org/packages/45/fa/7f43ba10c77575e8be7b0138d107e4f44ca4a1ef322cd16980ea3e8b8222/numpy-2.3.3-cp311-cp311-win32.whl", hash = "sha256:eb63d443d7b4ffd1e873f8155260d7f58e7e4b095961b01c91062935c2491e57", size = 6599794, upload-time = "2025-09-09T15:56:23.258Z" }, - { url = "https://files.pythonhosted.org/packages/0a/a2/a4f78cb2241fe5664a22a10332f2be886dcdea8784c9f6a01c272da9b426/numpy-2.3.3-cp311-cp311-win_amd64.whl", hash = "sha256:ec9d249840f6a565f58d8f913bccac2444235025bbb13e9a4681783572ee3caa", size = 13088104, upload-time = "2025-09-09T15:56:25.476Z" }, - { url = "https://files.pythonhosted.org/packages/79/64/e424e975adbd38282ebcd4891661965b78783de893b381cbc4832fb9beb2/numpy-2.3.3-cp311-cp311-win_arm64.whl", hash = "sha256:74c2a948d02f88c11a3c075d9733f1ae67d97c6bdb97f2bb542f980458b257e7", size = 10460772, upload-time = "2025-09-09T15:56:27.679Z" }, - { url = "https://files.pythonhosted.org/packages/51/5d/bb7fc075b762c96329147799e1bcc9176ab07ca6375ea976c475482ad5b3/numpy-2.3.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:cfdd09f9c84a1a934cde1eec2267f0a43a7cd44b2cca4ff95b7c0d14d144b0bf", size = 20957014, upload-time = "2025-09-09T15:56:29.966Z" }, - { url = "https://files.pythonhosted.org/packages/6b/0e/c6211bb92af26517acd52125a237a92afe9c3124c6a68d3b9f81b62a0568/numpy-2.3.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:cb32e3cf0f762aee47ad1ddc6672988f7f27045b0783c887190545baba73aa25", size = 14185220, upload-time = "2025-09-09T15:56:32.175Z" }, - { url = "https://files.pythonhosted.org/packages/22/f2/07bb754eb2ede9073f4054f7c0286b0d9d2e23982e090a80d478b26d35ca/numpy-2.3.3-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:396b254daeb0a57b1fe0ecb5e3cff6fa79a380fa97c8f7781a6d08cd429418fe", size = 5113918, upload-time = "2025-09-09T15:56:34.175Z" }, - { url = "https://files.pythonhosted.org/packages/81/0a/afa51697e9fb74642f231ea36aca80fa17c8fb89f7a82abd5174023c3960/numpy-2.3.3-cp312-cp312-macosx_14_0_x86_64.whl", hash = "sha256:067e3d7159a5d8f8a0b46ee11148fc35ca9b21f61e3c49fbd0a027450e65a33b", size = 6647922, upload-time = "2025-09-09T15:56:36.149Z" }, - { url = "https://files.pythonhosted.org/packages/5d/f5/122d9cdb3f51c520d150fef6e87df9279e33d19a9611a87c0d2cf78a89f4/numpy-2.3.3-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1c02d0629d25d426585fb2e45a66154081b9fa677bc92a881ff1d216bc9919a8", size = 14281991, upload-time = "2025-09-09T15:56:40.548Z" }, - { url = "https://files.pythonhosted.org/packages/51/64/7de3c91e821a2debf77c92962ea3fe6ac2bc45d0778c1cbe15d4fce2fd94/numpy-2.3.3-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d9192da52b9745f7f0766531dcfa978b7763916f158bb63bdb8a1eca0068ab20", size = 16641643, upload-time = "2025-09-09T15:56:43.343Z" }, - { url = "https://files.pythonhosted.org/packages/30/e4/961a5fa681502cd0d68907818b69f67542695b74e3ceaa513918103b7e80/numpy-2.3.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:cd7de500a5b66319db419dc3c345244404a164beae0d0937283b907d8152e6ea", size = 16056787, upload-time = "2025-09-09T15:56:46.141Z" }, - { url = "https://files.pythonhosted.org/packages/99/26/92c912b966e47fbbdf2ad556cb17e3a3088e2e1292b9833be1dfa5361a1a/numpy-2.3.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:93d4962d8f82af58f0b2eb85daaf1b3ca23fe0a85d0be8f1f2b7bb46034e56d7", size = 18579598, upload-time = "2025-09-09T15:56:49.844Z" }, - { url = "https://files.pythonhosted.org/packages/17/b6/fc8f82cb3520768718834f310c37d96380d9dc61bfdaf05fe5c0b7653e01/numpy-2.3.3-cp312-cp312-win32.whl", hash = "sha256:5534ed6b92f9b7dca6c0a19d6df12d41c68b991cef051d108f6dbff3babc4ebf", size = 6320800, upload-time = "2025-09-09T15:56:52.499Z" }, - { url = "https://files.pythonhosted.org/packages/32/ee/de999f2625b80d043d6d2d628c07d0d5555a677a3cf78fdf868d409b8766/numpy-2.3.3-cp312-cp312-win_amd64.whl", hash = "sha256:497d7cad08e7092dba36e3d296fe4c97708c93daf26643a1ae4b03f6294d30eb", size = 12786615, upload-time = "2025-09-09T15:56:54.422Z" }, - { url = "https://files.pythonhosted.org/packages/49/6e/b479032f8a43559c383acb20816644f5f91c88f633d9271ee84f3b3a996c/numpy-2.3.3-cp312-cp312-win_arm64.whl", hash = "sha256:ca0309a18d4dfea6fc6262a66d06c26cfe4640c3926ceec90e57791a82b6eee5", size = 10195936, upload-time = "2025-09-09T15:56:56.541Z" }, - { url = "https://files.pythonhosted.org/packages/7d/b9/984c2b1ee61a8b803bf63582b4ac4242cf76e2dbd663efeafcb620cc0ccb/numpy-2.3.3-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:f5415fb78995644253370985342cd03572ef8620b934da27d77377a2285955bf", size = 20949588, upload-time = "2025-09-09T15:56:59.087Z" }, - { url = "https://files.pythonhosted.org/packages/a6/e4/07970e3bed0b1384d22af1e9912527ecbeb47d3b26e9b6a3bced068b3bea/numpy-2.3.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:d00de139a3324e26ed5b95870ce63be7ec7352171bc69a4cf1f157a48e3eb6b7", size = 14177802, upload-time = "2025-09-09T15:57:01.73Z" }, - { url = "https://files.pythonhosted.org/packages/35/c7/477a83887f9de61f1203bad89cf208b7c19cc9fef0cebef65d5a1a0619f2/numpy-2.3.3-cp313-cp313-macosx_14_0_arm64.whl", hash = "sha256:9dc13c6a5829610cc07422bc74d3ac083bd8323f14e2827d992f9e52e22cd6a6", size = 5106537, upload-time = "2025-09-09T15:57:03.765Z" }, - { url = "https://files.pythonhosted.org/packages/52/47/93b953bd5866a6f6986344d045a207d3f1cfbad99db29f534ea9cee5108c/numpy-2.3.3-cp313-cp313-macosx_14_0_x86_64.whl", hash = "sha256:d79715d95f1894771eb4e60fb23f065663b2298f7d22945d66877aadf33d00c7", size = 6640743, upload-time = "2025-09-09T15:57:07.921Z" }, - { url = "https://files.pythonhosted.org/packages/23/83/377f84aaeb800b64c0ef4de58b08769e782edcefa4fea712910b6f0afd3c/numpy-2.3.3-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:952cfd0748514ea7c3afc729a0fc639e61655ce4c55ab9acfab14bda4f402b4c", size = 14278881, upload-time = "2025-09-09T15:57:11.349Z" }, - { url = "https://files.pythonhosted.org/packages/9a/a5/bf3db6e66c4b160d6ea10b534c381a1955dfab34cb1017ea93aa33c70ed3/numpy-2.3.3-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:5b83648633d46f77039c29078751f80da65aa64d5622a3cd62aaef9d835b6c93", size = 16636301, upload-time = "2025-09-09T15:57:14.245Z" }, - { url = "https://files.pythonhosted.org/packages/a2/59/1287924242eb4fa3f9b3a2c30400f2e17eb2707020d1c5e3086fe7330717/numpy-2.3.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:b001bae8cea1c7dfdb2ae2b017ed0a6f2102d7a70059df1e338e307a4c78a8ae", size = 16053645, upload-time = "2025-09-09T15:57:16.534Z" }, - { url = "https://files.pythonhosted.org/packages/e6/93/b3d47ed882027c35e94ac2320c37e452a549f582a5e801f2d34b56973c97/numpy-2.3.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:8e9aced64054739037d42fb84c54dd38b81ee238816c948c8f3ed134665dcd86", size = 18578179, upload-time = "2025-09-09T15:57:18.883Z" }, - { url = "https://files.pythonhosted.org/packages/20/d9/487a2bccbf7cc9d4bfc5f0f197761a5ef27ba870f1e3bbb9afc4bbe3fcc2/numpy-2.3.3-cp313-cp313-win32.whl", hash = "sha256:9591e1221db3f37751e6442850429b3aabf7026d3b05542d102944ca7f00c8a8", size = 6312250, upload-time = "2025-09-09T15:57:21.296Z" }, - { url = "https://files.pythonhosted.org/packages/1b/b5/263ebbbbcede85028f30047eab3d58028d7ebe389d6493fc95ae66c636ab/numpy-2.3.3-cp313-cp313-win_amd64.whl", hash = "sha256:f0dadeb302887f07431910f67a14d57209ed91130be0adea2f9793f1a4f817cf", size = 12783269, upload-time = "2025-09-09T15:57:23.034Z" }, - { url = "https://files.pythonhosted.org/packages/fa/75/67b8ca554bbeaaeb3fac2e8bce46967a5a06544c9108ec0cf5cece559b6c/numpy-2.3.3-cp313-cp313-win_arm64.whl", hash = "sha256:3c7cf302ac6e0b76a64c4aecf1a09e51abd9b01fc7feee80f6c43e3ab1b1dbc5", size = 10195314, upload-time = "2025-09-09T15:57:25.045Z" }, - { url = "https://files.pythonhosted.org/packages/11/d0/0d1ddec56b162042ddfafeeb293bac672de9b0cfd688383590090963720a/numpy-2.3.3-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:eda59e44957d272846bb407aad19f89dc6f58fecf3504bd144f4c5cf81a7eacc", size = 21048025, upload-time = "2025-09-09T15:57:27.257Z" }, - { url = "https://files.pythonhosted.org/packages/36/9e/1996ca6b6d00415b6acbdd3c42f7f03ea256e2c3f158f80bd7436a8a19f3/numpy-2.3.3-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:823d04112bc85ef5c4fda73ba24e6096c8f869931405a80aa8b0e604510a26bc", size = 14301053, upload-time = "2025-09-09T15:57:30.077Z" }, - { url = "https://files.pythonhosted.org/packages/05/24/43da09aa764c68694b76e84b3d3f0c44cb7c18cdc1ba80e48b0ac1d2cd39/numpy-2.3.3-cp313-cp313t-macosx_14_0_arm64.whl", hash = "sha256:40051003e03db4041aa325da2a0971ba41cf65714e65d296397cc0e32de6018b", size = 5229444, upload-time = "2025-09-09T15:57:32.733Z" }, - { url = "https://files.pythonhosted.org/packages/bc/14/50ffb0f22f7218ef8af28dd089f79f68289a7a05a208db9a2c5dcbe123c1/numpy-2.3.3-cp313-cp313t-macosx_14_0_x86_64.whl", hash = "sha256:6ee9086235dd6ab7ae75aba5662f582a81ced49f0f1c6de4260a78d8f2d91a19", size = 6738039, upload-time = "2025-09-09T15:57:34.328Z" }, - { url = "https://files.pythonhosted.org/packages/55/52/af46ac0795e09657d45a7f4db961917314377edecf66db0e39fa7ab5c3d3/numpy-2.3.3-cp313-cp313t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:94fcaa68757c3e2e668ddadeaa86ab05499a70725811e582b6a9858dd472fb30", size = 14352314, upload-time = "2025-09-09T15:57:36.255Z" }, - { url = "https://files.pythonhosted.org/packages/a7/b1/dc226b4c90eb9f07a3fff95c2f0db3268e2e54e5cce97c4ac91518aee71b/numpy-2.3.3-cp313-cp313t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:da1a74b90e7483d6ce5244053399a614b1d6b7bc30a60d2f570e5071f8959d3e", size = 16701722, upload-time = "2025-09-09T15:57:38.622Z" }, - { url = "https://files.pythonhosted.org/packages/9d/9d/9d8d358f2eb5eced14dba99f110d83b5cd9a4460895230f3b396ad19a323/numpy-2.3.3-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:2990adf06d1ecee3b3dcbb4977dfab6e9f09807598d647f04d385d29e7a3c3d3", size = 16132755, upload-time = "2025-09-09T15:57:41.16Z" }, - { url = "https://files.pythonhosted.org/packages/b6/27/b3922660c45513f9377b3fb42240bec63f203c71416093476ec9aa0719dc/numpy-2.3.3-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:ed635ff692483b8e3f0fcaa8e7eb8a75ee71aa6d975388224f70821421800cea", size = 18651560, upload-time = "2025-09-09T15:57:43.459Z" }, - { url = "https://files.pythonhosted.org/packages/5b/8e/3ab61a730bdbbc201bb245a71102aa609f0008b9ed15255500a99cd7f780/numpy-2.3.3-cp313-cp313t-win32.whl", hash = "sha256:a333b4ed33d8dc2b373cc955ca57babc00cd6f9009991d9edc5ddbc1bac36bcd", size = 6442776, upload-time = "2025-09-09T15:57:45.793Z" }, - { url = "https://files.pythonhosted.org/packages/1c/3a/e22b766b11f6030dc2decdeff5c2fb1610768055603f9f3be88b6d192fb2/numpy-2.3.3-cp313-cp313t-win_amd64.whl", hash = "sha256:4384a169c4d8f97195980815d6fcad04933a7e1ab3b530921c3fef7a1c63426d", size = 12927281, upload-time = "2025-09-09T15:57:47.492Z" }, - { url = "https://files.pythonhosted.org/packages/7b/42/c2e2bc48c5e9b2a83423f99733950fbefd86f165b468a3d85d52b30bf782/numpy-2.3.3-cp313-cp313t-win_arm64.whl", hash = "sha256:75370986cc0bc66f4ce5110ad35aae6d182cc4ce6433c40ad151f53690130bf1", size = 10265275, upload-time = "2025-09-09T15:57:49.647Z" }, - { url = "https://files.pythonhosted.org/packages/6b/01/342ad585ad82419b99bcf7cebe99e61da6bedb89e213c5fd71acc467faee/numpy-2.3.3-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:cd052f1fa6a78dee696b58a914b7229ecfa41f0a6d96dc663c1220a55e137593", size = 20951527, upload-time = "2025-09-09T15:57:52.006Z" }, - { url = "https://files.pythonhosted.org/packages/ef/d8/204e0d73fc1b7a9ee80ab1fe1983dd33a4d64a4e30a05364b0208e9a241a/numpy-2.3.3-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:414a97499480067d305fcac9716c29cf4d0d76db6ebf0bf3cbce666677f12652", size = 14186159, upload-time = "2025-09-09T15:57:54.407Z" }, - { url = "https://files.pythonhosted.org/packages/22/af/f11c916d08f3a18fb8ba81ab72b5b74a6e42ead4c2846d270eb19845bf74/numpy-2.3.3-cp314-cp314-macosx_14_0_arm64.whl", hash = "sha256:50a5fe69f135f88a2be9b6ca0481a68a136f6febe1916e4920e12f1a34e708a7", size = 5114624, upload-time = "2025-09-09T15:57:56.5Z" }, - { url = "https://files.pythonhosted.org/packages/fb/11/0ed919c8381ac9d2ffacd63fd1f0c34d27e99cab650f0eb6f110e6ae4858/numpy-2.3.3-cp314-cp314-macosx_14_0_x86_64.whl", hash = "sha256:b912f2ed2b67a129e6a601e9d93d4fa37bef67e54cac442a2f588a54afe5c67a", size = 6642627, upload-time = "2025-09-09T15:57:58.206Z" }, - { url = "https://files.pythonhosted.org/packages/ee/83/deb5f77cb0f7ba6cb52b91ed388b47f8f3c2e9930d4665c600408d9b90b9/numpy-2.3.3-cp314-cp314-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9e318ee0596d76d4cb3d78535dc005fa60e5ea348cd131a51e99d0bdbe0b54fe", size = 14296926, upload-time = "2025-09-09T15:58:00.035Z" }, - { url = "https://files.pythonhosted.org/packages/77/cc/70e59dcb84f2b005d4f306310ff0a892518cc0c8000a33d0e6faf7ca8d80/numpy-2.3.3-cp314-cp314-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ce020080e4a52426202bdb6f7691c65bb55e49f261f31a8f506c9f6bc7450421", size = 16638958, upload-time = "2025-09-09T15:58:02.738Z" }, - { url = "https://files.pythonhosted.org/packages/b6/5a/b2ab6c18b4257e099587d5b7f903317bd7115333ad8d4ec4874278eafa61/numpy-2.3.3-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:e6687dc183aa55dae4a705b35f9c0f8cb178bcaa2f029b241ac5356221d5c021", size = 16071920, upload-time = "2025-09-09T15:58:05.029Z" }, - { url = "https://files.pythonhosted.org/packages/b8/f1/8b3fdc44324a259298520dd82147ff648979bed085feeacc1250ef1656c0/numpy-2.3.3-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:d8f3b1080782469fdc1718c4ed1d22549b5fb12af0d57d35e992158a772a37cf", size = 18577076, upload-time = "2025-09-09T15:58:07.745Z" }, - { url = "https://files.pythonhosted.org/packages/f0/a1/b87a284fb15a42e9274e7fcea0dad259d12ddbf07c1595b26883151ca3b4/numpy-2.3.3-cp314-cp314-win32.whl", hash = "sha256:cb248499b0bc3be66ebd6578b83e5acacf1d6cb2a77f2248ce0e40fbec5a76d0", size = 6366952, upload-time = "2025-09-09T15:58:10.096Z" }, - { url = "https://files.pythonhosted.org/packages/70/5f/1816f4d08f3b8f66576d8433a66f8fa35a5acfb3bbd0bf6c31183b003f3d/numpy-2.3.3-cp314-cp314-win_amd64.whl", hash = "sha256:691808c2b26b0f002a032c73255d0bd89751425f379f7bcd22d140db593a96e8", size = 12919322, upload-time = "2025-09-09T15:58:12.138Z" }, - { url = "https://files.pythonhosted.org/packages/8c/de/072420342e46a8ea41c324a555fa90fcc11637583fb8df722936aed1736d/numpy-2.3.3-cp314-cp314-win_arm64.whl", hash = "sha256:9ad12e976ca7b10f1774b03615a2a4bab8addce37ecc77394d8e986927dc0dfe", size = 10478630, upload-time = "2025-09-09T15:58:14.64Z" }, - { url = "https://files.pythonhosted.org/packages/d5/df/ee2f1c0a9de7347f14da5dd3cd3c3b034d1b8607ccb6883d7dd5c035d631/numpy-2.3.3-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:9cc48e09feb11e1db00b320e9d30a4151f7369afb96bd0e48d942d09da3a0d00", size = 21047987, upload-time = "2025-09-09T15:58:16.889Z" }, - { url = "https://files.pythonhosted.org/packages/d6/92/9453bdc5a4e9e69cf4358463f25e8260e2ffc126d52e10038b9077815989/numpy-2.3.3-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:901bf6123879b7f251d3631967fd574690734236075082078e0571977c6a8e6a", size = 14301076, upload-time = "2025-09-09T15:58:20.343Z" }, - { url = "https://files.pythonhosted.org/packages/13/77/1447b9eb500f028bb44253105bd67534af60499588a5149a94f18f2ca917/numpy-2.3.3-cp314-cp314t-macosx_14_0_arm64.whl", hash = "sha256:7f025652034199c301049296b59fa7d52c7e625017cae4c75d8662e377bf487d", size = 5229491, upload-time = "2025-09-09T15:58:22.481Z" }, - { url = "https://files.pythonhosted.org/packages/3d/f9/d72221b6ca205f9736cb4b2ce3b002f6e45cd67cd6a6d1c8af11a2f0b649/numpy-2.3.3-cp314-cp314t-macosx_14_0_x86_64.whl", hash = "sha256:533ca5f6d325c80b6007d4d7fb1984c303553534191024ec6a524a4c92a5935a", size = 6737913, upload-time = "2025-09-09T15:58:24.569Z" }, - { url = "https://files.pythonhosted.org/packages/3c/5f/d12834711962ad9c46af72f79bb31e73e416ee49d17f4c797f72c96b6ca5/numpy-2.3.3-cp314-cp314t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0edd58682a399824633b66885d699d7de982800053acf20be1eaa46d92009c54", size = 14352811, upload-time = "2025-09-09T15:58:26.416Z" }, - { url = "https://files.pythonhosted.org/packages/a1/0d/fdbec6629d97fd1bebed56cd742884e4eead593611bbe1abc3eb40d304b2/numpy-2.3.3-cp314-cp314t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:367ad5d8fbec5d9296d18478804a530f1191e24ab4d75ab408346ae88045d25e", size = 16702689, upload-time = "2025-09-09T15:58:28.831Z" }, - { url = "https://files.pythonhosted.org/packages/9b/09/0a35196dc5575adde1eb97ddfbc3e1687a814f905377621d18ca9bc2b7dd/numpy-2.3.3-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:8f6ac61a217437946a1fa48d24c47c91a0c4f725237871117dea264982128097", size = 16133855, upload-time = "2025-09-09T15:58:31.349Z" }, - { url = "https://files.pythonhosted.org/packages/7a/ca/c9de3ea397d576f1b6753eaa906d4cdef1bf97589a6d9825a349b4729cc2/numpy-2.3.3-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:179a42101b845a816d464b6fe9a845dfaf308fdfc7925387195570789bb2c970", size = 18652520, upload-time = "2025-09-09T15:58:33.762Z" }, - { url = "https://files.pythonhosted.org/packages/fd/c2/e5ed830e08cd0196351db55db82f65bc0ab05da6ef2b72a836dcf1936d2f/numpy-2.3.3-cp314-cp314t-win32.whl", hash = "sha256:1250c5d3d2562ec4174bce2e3a1523041595f9b651065e4a4473f5f48a6bc8a5", size = 6515371, upload-time = "2025-09-09T15:58:36.04Z" }, - { url = "https://files.pythonhosted.org/packages/47/c7/b0f6b5b67f6788a0725f744496badbb604d226bf233ba716683ebb47b570/numpy-2.3.3-cp314-cp314t-win_amd64.whl", hash = "sha256:b37a0b2e5935409daebe82c1e42274d30d9dd355852529eab91dab8dcca7419f", size = 13112576, upload-time = "2025-09-09T15:58:37.927Z" }, - { url = "https://files.pythonhosted.org/packages/06/b9/33bba5ff6fb679aa0b1f8a07e853f002a6b04b9394db3069a1270a7784ca/numpy-2.3.3-cp314-cp314t-win_arm64.whl", hash = "sha256:78c9f6560dc7e6b3990e32df7ea1a50bbd0e2a111e05209963f5ddcab7073b0b", size = 10545953, upload-time = "2025-09-09T15:58:40.576Z" }, - { url = "https://files.pythonhosted.org/packages/b8/f2/7e0a37cfced2644c9563c529f29fa28acbd0960dde32ece683aafa6f4949/numpy-2.3.3-pp311-pypy311_pp73-macosx_10_15_x86_64.whl", hash = "sha256:1e02c7159791cd481e1e6d5ddd766b62a4d5acf8df4d4d1afe35ee9c5c33a41e", size = 21131019, upload-time = "2025-09-09T15:58:42.838Z" }, - { url = "https://files.pythonhosted.org/packages/1a/7e/3291f505297ed63831135a6cc0f474da0c868a1f31b0dd9a9f03a7a0d2ed/numpy-2.3.3-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:dca2d0fc80b3893ae72197b39f69d55a3cd8b17ea1b50aa4c62de82419936150", size = 14376288, upload-time = "2025-09-09T15:58:45.425Z" }, - { url = "https://files.pythonhosted.org/packages/bf/4b/ae02e985bdeee73d7b5abdefeb98aef1207e96d4c0621ee0cf228ddfac3c/numpy-2.3.3-pp311-pypy311_pp73-macosx_14_0_arm64.whl", hash = "sha256:99683cbe0658f8271b333a1b1b4bb3173750ad59c0c61f5bbdc5b318918fffe3", size = 5305425, upload-time = "2025-09-09T15:58:48.6Z" }, - { url = "https://files.pythonhosted.org/packages/8b/eb/9df215d6d7250db32007941500dc51c48190be25f2401d5b2b564e467247/numpy-2.3.3-pp311-pypy311_pp73-macosx_14_0_x86_64.whl", hash = "sha256:d9d537a39cc9de668e5cd0e25affb17aec17b577c6b3ae8a3d866b479fbe88d0", size = 6819053, upload-time = "2025-09-09T15:58:50.401Z" }, - { url = "https://files.pythonhosted.org/packages/57/62/208293d7d6b2a8998a4a1f23ac758648c3c32182d4ce4346062018362e29/numpy-2.3.3-pp311-pypy311_pp73-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:8596ba2f8af5f93b01d97563832686d20206d303024777f6dfc2e7c7c3f1850e", size = 14420354, upload-time = "2025-09-09T15:58:52.704Z" }, - { url = "https://files.pythonhosted.org/packages/ed/0c/8e86e0ff7072e14a71b4c6af63175e40d1e7e933ce9b9e9f765a95b4e0c3/numpy-2.3.3-pp311-pypy311_pp73-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e1ec5615b05369925bd1125f27df33f3b6c8bc10d788d5999ecd8769a1fa04db", size = 16760413, upload-time = "2025-09-09T15:58:55.027Z" }, - { url = "https://files.pythonhosted.org/packages/af/11/0cc63f9f321ccf63886ac203336777140011fb669e739da36d8db3c53b98/numpy-2.3.3-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:2e267c7da5bf7309670523896df97f93f6e469fb931161f483cd6882b3b1a5dc", size = 12971844, upload-time = "2025-09-09T15:58:57.359Z" }, +sdist = { url = "https://files.pythonhosted.org/packages/22/fd/89965aa4ac08c74998539fcbf24fa3540f3e15237fbeb6bcf9c908f4aade/numpy-2.5.1.tar.gz", hash = "sha256:a48a113e6afea91f5608793bafa7ef2ad481fefbda87ec5069f483de61cb9fa3", size = 20755553, upload-time = "2026-07-04T17:08:00.933Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/62/7b/14687aa674250e5e546f616f486b0d56d3631cd5b2415739141ce40bdcea/numpy-2.5.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:2c889b56fe48b1018f764b0eec8df59ab654e9148aa91faa12596043500de277", size = 16801574, upload-time = "2026-07-04T17:06:12.423Z" }, + { url = "https://files.pythonhosted.org/packages/e1/19/cc5bb2a3f2913d27d6dbb2c78d25921fabaedc6741d4a5a615a11f3c5bf3/numpy-2.5.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:ab451b59c5643c570974c43aef780703ef1d3b4965d2be07afd530615a9358d1", size = 11772250, upload-time = "2026-07-04T17:06:15.726Z" }, + { url = "https://files.pythonhosted.org/packages/42/77/fdf34a71dd30f54979b18603bee915e0aaf825b07afe79acd60b04b691e2/numpy-2.5.1-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:78798bd5b9ad744056af8efa90e3b9ddaa53272a0848a483084a1cc0a13b2dc0", size = 5331516, upload-time = "2026-07-04T17:06:17.913Z" }, + { url = "https://files.pythonhosted.org/packages/ce/e2/eb7efa015b4cce41e2517bf182a7fce0d7d5b9d9ed76a29bfa0f4fe4505c/numpy-2.5.1-cp312-cp312-macosx_14_0_x86_64.whl", hash = "sha256:2ae0ca40bcb22d6ba59c1dfd5446f49940b0f2d821fde133f10dda11f816b84e", size = 6664863, upload-time = "2026-07-04T17:06:20.02Z" }, + { url = "https://files.pythonhosted.org/packages/a9/4b/a2b32dd94ee9ffbeecb28152240042a3949db33b1c834d44090b80e1b3b8/numpy-2.5.1-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:61ac47e772e6b8ea489e1d2f441a34c5c3ac17327e7ce294cbdf535795ad4e75", size = 15167977, upload-time = "2026-07-04T17:06:21.621Z" }, + { url = "https://files.pythonhosted.org/packages/b8/a9/6e73d68500f80773f65f0654ea932019d6694329a0eb0ed0533de38df376/numpy-2.5.1-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:59fda5e192b570217ec2580c96f00e9a7e12ef6866a900eb089b62c1a32545ca", size = 16672469, upload-time = "2026-07-04T17:06:24.064Z" }, + { url = "https://files.pythonhosted.org/packages/24/7d/ad3e59015135f5261c95fd4cafeff159c955febd83a99a1d9250c4233815/numpy-2.5.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:f7119ebff1a9829e9f431a4f9d28e703023bb6b9fe7c8f724467dbfc27c94ab3", size = 16527531, upload-time = "2026-07-04T17:06:26.69Z" }, + { url = "https://files.pythonhosted.org/packages/83/d0/a39b2fbcde9cb17a1dac678f254b33a6336298af9df338824c685425d5e8/numpy-2.5.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:e824c2acf8862052246be5a44c15da1777940c60d010dd2aab897824d9c430f9", size = 18431940, upload-time = "2026-07-04T17:06:29.521Z" }, + { url = "https://files.pythonhosted.org/packages/04/12/cff070947791c1ed425ff76413189adbdc2fbe215eba7ce7fa454a03c7f8/numpy-2.5.1-cp312-cp312-win32.whl", hash = "sha256:08d60c810432eb83360958dea0999ac4cfb94531ea8efcbf0b7f277c2068aeb2", size = 6066764, upload-time = "2026-07-04T17:06:32.571Z" }, + { url = "https://files.pythonhosted.org/packages/65/66/53f31807a48a750f9d748da273bc3fcedd12b27ff1f3e373bfec55ef2dc0/numpy-2.5.1-cp312-cp312-win_amd64.whl", hash = "sha256:f7d60026c0bdb1380e83bfa7a0419c4577ee4b9a08880afcb6dadeb74c649fa2", size = 12430966, upload-time = "2026-07-04T17:06:34.926Z" }, + { url = "https://files.pythonhosted.org/packages/2b/2a/d1a88066b1c14186f5d3c0d18c94f17b064511982bab0578d49ee9d43c29/numpy-2.5.1-cp312-cp312-win_arm64.whl", hash = "sha256:17a25e09640602e10bc8de0e6fa2b3fd68eedd84ba6d7842dc8f32f9ab87bd0b", size = 10350488, upload-time = "2026-07-04T17:06:37.785Z" }, + { url = "https://files.pythonhosted.org/packages/eb/07/ec2a3f0c91761581d4b7104a740791800025983f9a4dc4e73f91a99aeac4/numpy-2.5.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:0bfebd8695f9863592fe744be833a258120b14a9f39da255e8aa8fade2c0ddd1", size = 16796419, upload-time = "2026-07-04T17:06:40.37Z" }, + { url = "https://files.pythonhosted.org/packages/ab/ab/ddb499fc4f8780354395face5b65c7fd107bcd6e1d667a5f07d046956f6f/numpy-2.5.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:30b44a6b53a7ae63c54c089a8726e5563ed302716c5b7ccc85afade40b0e7ff6", size = 11765832, upload-time = "2026-07-04T17:06:42.768Z" }, + { url = "https://files.pythonhosted.org/packages/88/b3/3c28c558a09fc72100c646dac6d2fce8e834c471b0edca01a29996706117/numpy-2.5.1-cp313-cp313-macosx_14_0_arm64.whl", hash = "sha256:6165343f81b56ef8f514f396989e529b61d9dc709b99421b07e9f3e698e2287d", size = 5325143, upload-time = "2026-07-04T17:06:45.466Z" }, + { url = "https://files.pythonhosted.org/packages/5e/0e/ce19b985bb15c596f4f05954e76cccc77c845083b3b8f938a6c68e523128/numpy-2.5.1-cp313-cp313-macosx_14_0_x86_64.whl", hash = "sha256:4939237038ada79308dda3204ac6462df056b5672b2e25db1149cf873668b3e1", size = 6659749, upload-time = "2026-07-04T17:06:47.288Z" }, + { url = "https://files.pythonhosted.org/packages/2e/20/1ee6614d64332a1bba6411f38e68cb79eec1b2459e20a623777c5c5492a2/numpy-2.5.1-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1c6759f538fb912fc46de0a6b1758ccf7b57bc7c7ebebc23974fdac3de8db0cd", size = 15164716, upload-time = "2026-07-04T17:06:49.494Z" }, + { url = "https://files.pythonhosted.org/packages/ed/a7/2bcd3fdbb87804755c35b729bf8709d62025c5f4cfd7d5b2415997097515/numpy-2.5.1-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9726558e8db4a5bf7929a70ae50f63abda4daf0efe810e3bfbab95976f75fc1a", size = 16661440, upload-time = "2026-07-04T17:06:52.061Z" }, + { url = "https://files.pythonhosted.org/packages/fc/d7/a41e3310c886fe457d36e670bbf24fae411aca8a7b6ad92a32afd924077c/numpy-2.5.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:3935f3b419b244a02732676fa5317a9193cc596a4c0646db07e5b421229ac9f7", size = 16526305, upload-time = "2026-07-04T17:06:54.605Z" }, + { url = "https://files.pythonhosted.org/packages/53/75/4333a9a707c1edd3a4e1a0c58eca52c0f31e55089fa80db02b5565b24df7/numpy-2.5.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:dc932a65ded7ce9013d120845a2514dcccb1a67bfc8deb8d37633762951904a6", size = 18423008, upload-time = "2026-07-04T17:06:57.54Z" }, + { url = "https://files.pythonhosted.org/packages/ee/90/e314a32b1c11a2ffe818ddad3a57b50b4b6e1b6c487192eb50cdef0415d0/numpy-2.5.1-cp313-cp313-win32.whl", hash = "sha256:4b4ff1608417eb7a59da7b967bbb798cacfe071d2caf526a24281cd562072ed9", size = 6063885, upload-time = "2026-07-04T17:07:00.14Z" }, + { url = "https://files.pythonhosted.org/packages/10/70/800b3fca480af32df9e8ea9f3d4a0c8feb4b32d7f195d174eabbda4829ad/numpy-2.5.1-cp313-cp313-win_amd64.whl", hash = "sha256:6c3fe51bc6a16453d452997053454f309e8e0ed7b42d6b361ce4ac8c32913d74", size = 12425674, upload-time = "2026-07-04T17:07:02.387Z" }, + { url = "https://files.pythonhosted.org/packages/8b/0b/196350c122f50f6ca56846f2d71efd5e0d24b7b2e07355e019b2e2c7a11e/numpy-2.5.1-cp313-cp313-win_arm64.whl", hash = "sha256:f7feb014281029e628ba2d5a007407443b06e418b6fe451d1e2adcbc8eba0107", size = 10350256, upload-time = "2026-07-04T17:07:04.878Z" }, + { url = "https://files.pythonhosted.org/packages/db/f4/731b6085a83faf6ca843394cbd5e217280c214399f7e8b21b9f552af0ae2/numpy-2.5.1-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:7c786fe9a5bbe360022e584c5a34cf6b54265c71bd7ec8ac3d8fec38968071f8", size = 16795063, upload-time = "2026-07-04T17:07:07.374Z" }, + { url = "https://files.pythonhosted.org/packages/bf/64/0e215f2048dd11a55bb989ed41b3585ef57452404e638d703a211a3e4157/numpy-2.5.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:32985c896d897419ef8da6917872d80b78ad0ea26d85b23245c7366ffde76d75", size = 11776652, upload-time = "2026-07-04T17:07:09.907Z" }, + { url = "https://files.pythonhosted.org/packages/b5/59/2b844c7a6e9deff69b404a66221e1542937734f65d5e6e39411876053862/numpy-2.5.1-cp314-cp314-macosx_14_0_arm64.whl", hash = "sha256:efd736408cc97c79b9e6917338dfc8f06013b2274f992e96b1d9a81a71e2a2c2", size = 5335944, upload-time = "2026-07-04T17:07:12.227Z" }, + { url = "https://files.pythonhosted.org/packages/86/51/9bf7cb2cabcebc9e017e4ec7e6322b378317a542c08b4cb68479c1efc716/numpy-2.5.1-cp314-cp314-macosx_14_0_x86_64.whl", hash = "sha256:ab84dc6b074fa881cae55bea94cc4f68e285181ba7f32497bf7dee6b1496165b", size = 6656266, upload-time = "2026-07-04T17:07:14.368Z" }, + { url = "https://files.pythonhosted.org/packages/83/3e/fb7615b211b82a32f44d5180a6d421b61f84d4fadd578b48ba4ac34e189f/numpy-2.5.1-cp314-cp314-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:caf3e317d33d60c37986b452613f4ab51246d0691350c03d0cb4a898627f4a95", size = 15179720, upload-time = "2026-07-04T17:07:16.272Z" }, + { url = "https://files.pythonhosted.org/packages/41/5f/0f992cb24560673496c5d68de61913b57166ce530ffda07c1f280e0cc464/numpy-2.5.1-cp314-cp314-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:54ad769f17bc2d833b620851989f62054fb9ab93c969d9e1dc3c8e3d56beea21", size = 16664835, upload-time = "2026-07-04T17:07:19.021Z" }, + { url = "https://files.pythonhosted.org/packages/a2/2f/97d6475ee91afe2587797d09446f9d3e475ad4cb681662d824809327b75a/numpy-2.5.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:c12afb53450fa976d4c681c50a7423729a4c51c0465ed9f32b8a9cabbc472373", size = 16539135, upload-time = "2026-07-04T17:07:22.015Z" }, + { url = "https://files.pythonhosted.org/packages/c4/5b/4db81e4ba0be7e2776b1de68c82aa862c7f8ec27e1b4927d4ae075e20678/numpy-2.5.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:e8c11c405efc5ff6816d5983c96cdfa215bab3428961243af3ff59b228490438", size = 18426684, upload-time = "2026-07-04T17:07:24.941Z" }, + { url = "https://files.pythonhosted.org/packages/1f/64/c0ba2d90724d450279a7df8f32057241070250a26a7e2b5337d77347f481/numpy-2.5.1-cp314-cp314-win32.whl", hash = "sha256:f2479a47f8d5932d1718168a681ad6e536a9df484c83cfcf9de365e164537ace", size = 6116103, upload-time = "2026-07-04T17:07:27.622Z" }, + { url = "https://files.pythonhosted.org/packages/c1/1a/837f9ed7405adcd7a40538792eb169eddd8fa5630c16a1ef49dae71a30f4/numpy-2.5.1-cp314-cp314-win_amd64.whl", hash = "sha256:24d0eb82c0541d3415a33425db64ae439dffccd7b4dbcb30e7c35120205c506a", size = 12562177, upload-time = "2026-07-04T17:07:29.887Z" }, + { url = "https://files.pythonhosted.org/packages/22/ed/49707938b6dd0a78a9178dd93227dc89e4c11af47f5c798d70366e8d0483/numpy-2.5.1-cp314-cp314-win_arm64.whl", hash = "sha256:5a4c988b38d261deeeaad9954e3deb091ad905c94e8bb6708654ef1d97f286b0", size = 10627739, upload-time = "2026-07-04T17:07:32.568Z" }, + { url = "https://files.pythonhosted.org/packages/a6/c7/bb4b882cfe7f299cbc8b66e42e7dd78cf9d14e40f9469fc5e3db7e15b3bd/numpy-2.5.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:a33276be12fa045805f477f22482088b66bb758ffbe89a9d21457de863a32e22", size = 11894709, upload-time = "2026-07-04T17:07:34.941Z" }, + { url = "https://files.pythonhosted.org/packages/40/3f/5af7f4a7f6224aef48017aa82bb6174c7a659d724be0c75017b7e64a55b4/numpy-2.5.1-cp314-cp314t-macosx_14_0_arm64.whl", hash = "sha256:f089d7b00756190aacf1f5d34bdf38c3c430ac82b4f868f8cede73380460fce7", size = 5453810, upload-time = "2026-07-04T17:07:37.495Z" }, + { url = "https://files.pythonhosted.org/packages/20/c9/3474309bc94d634d3f9c3eddf03250ecb8c22cd948ef16fef69a77cc5d7b/numpy-2.5.1-cp314-cp314t-macosx_14_0_x86_64.whl", hash = "sha256:09e9bfd8d2cf479c7d174804fb3811c53a8e9f20a37444008606b57d6b7a826d", size = 6761189, upload-time = "2026-07-04T17:07:39.563Z" }, + { url = "https://files.pythonhosted.org/packages/90/8a/558ae39fdd55d7e7f7fef9a84a6e964ac6b23edbd2a07e52bb084500507d/numpy-2.5.1-cp314-cp314t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e68d8dd1e7eba712948f2053a29ec86917bc70ba1358df869d9f06649ef9cf09", size = 15225039, upload-time = "2026-07-04T17:07:41.682Z" }, + { url = "https://files.pythonhosted.org/packages/63/27/ca7392b2d030277bdf0273e7d23255b3ee57d57a7c170a6f4fb3981e1e5d/numpy-2.5.1-cp314-cp314t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:99d5095fa265a0c4152e7bb12759e14381ef5496152f1ce58f44bdf55c44beb4", size = 16701306, upload-time = "2026-07-04T17:07:44.611Z" }, + { url = "https://files.pythonhosted.org/packages/02/42/03d53ae7996c44d4374a8262e9dc41671fd56cbb98f7d47ef85cf5da4c6b/numpy-2.5.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:ab87a91b3cc3382b8956095bd8f95e00cf679bb81554339be1a2ba404a1473c1", size = 16589955, upload-time = "2026-07-04T17:07:47.694Z" }, + { url = "https://files.pythonhosted.org/packages/7b/15/6c1784ae469640e65db111e9a34b3d0f14d91e8a38b9ce34810ced370dbb/numpy-2.5.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:224ca51130ef7da85bea2191625181cb4f337f9cb64b471f10c1a12aa8b60077", size = 18464252, upload-time = "2026-07-04T17:07:50.684Z" }, + { url = "https://files.pythonhosted.org/packages/94/a8/f98e50356cf167df656c526c2dfeec2d7dde182f2a3da4b458a5938e2776/numpy-2.5.1-cp314-cp314t-win32.whl", hash = "sha256:6eab239876581b2b3c5a242281b6007bbdbcd1c7085d7709bb57c5929b11e6bf", size = 6263298, upload-time = "2026-07-04T17:07:53.445Z" }, + { url = "https://files.pythonhosted.org/packages/72/ac/96ae880cdecad0b3275d9359fcec72667b49a4863c9f12942e43679dda02/numpy-2.5.1-cp314-cp314t-win_amd64.whl", hash = "sha256:83ce9c80d5b521b0d77ddcbe5447c218d247929b6cc056ca5351342accfff0af", size = 12748623, upload-time = "2026-07-04T17:07:55.384Z" }, + { url = "https://files.pythonhosted.org/packages/a1/5a/4d2b1601df3602dba7a14f3348ba9bfe94a18adb428e693df6154c293831/numpy-2.5.1-cp314-cp314t-win_arm64.whl", hash = "sha256:5a6db61f9aaa57e369905c67d852045d3c4f7126405b29d09b19dec118e9c9cb", size = 10697674, upload-time = "2026-07-04T17:07:58.506Z" }, ] [[package]] @@ -1649,11 +1759,11 @@ wheels = [ [[package]] name = "nvidia-nvjitlink" -version = "13.0.88" +version = "13.3.33" source = { registry = "https://pypi.org/simple" } wheels = [ - { url = "https://files.pythonhosted.org/packages/56/7a/123e033aaff487c77107195fa5a2b8686795ca537935a24efae476c41f05/nvidia_nvjitlink-13.0.88-py3-none-manylinux2010_x86_64.manylinux_2_12_x86_64.whl", hash = "sha256:13a74f429e23b921c1109976abefacc69835f2f433ebd323d3946e11d804e47b", size = 40713933, upload-time = "2025-09-04T08:35:43.553Z" }, - { url = "https://files.pythonhosted.org/packages/ab/2c/93c5250e64df4f894f1cbb397c6fd71f79813f9fd79d7cd61de3f97b3c2d/nvidia_nvjitlink-13.0.88-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:e931536ccc7d467a98ba1d8b89ff7fa7f1fa3b13f2b0069118cd7f47bff07d0c", size = 38768748, upload-time = "2025-09-04T08:35:20.008Z" }, + { url = "https://files.pythonhosted.org/packages/f0/ee/580ca6f29dcab0221db8706badca1bbbb084f1975c4d4e83329c3a7e31f0/nvidia_nvjitlink-13.3.33-py3-none-manylinux2010_x86_64.manylinux_2_12_x86_64.whl", hash = "sha256:26a6de7fb4c8fdaa7703d3dad720d6d427ddfea5c48a528fd97c11733ad830e5", size = 40742423, upload-time = "2026-05-26T16:54:51.613Z" }, + { url = "https://files.pythonhosted.org/packages/69/30/45414e35ff2eee7db3da037e5707037ccf9d2b5218ffbdb055ea4d5aa98a/nvidia_nvjitlink-13.3.33-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:ce48b37dfeb3cb1eae4cf85adacb47d7a6539ea2272870c9a3628ce275c2037e", size = 39168635, upload-time = "2026-05-26T16:54:13.906Z" }, ] [[package]] @@ -1675,73 +1785,52 @@ wheels = [ ] [[package]] -name = "opt-einsum" -version = "3.4.0" +name = "opentelemetry-api" +version = "1.30.0" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/8c/b9/2ac072041e899a52f20cf9510850ff58295003aa75525e58343591b0cbfb/opt_einsum-3.4.0.tar.gz", hash = "sha256:96ca72f1b886d148241348783498194c577fa30a8faac108586b14f1ba4473ac", size = 63004, upload-time = "2024-09-26T14:33:24.483Z" } +dependencies = [ + { name = "deprecated" }, + { name = "importlib-metadata" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/2b/6d/bbbf879826b7f3c89a45252010b5796fb1f1a0d45d9dc4709db0ef9a06c8/opentelemetry_api-1.30.0.tar.gz", hash = "sha256:375893400c1435bf623f7dfb3bcd44825fe6b56c34d0667c542ea8257b1a1240", size = 63703, upload-time = "2025-02-04T18:17:13.789Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/23/cd/066e86230ae37ed0be70aae89aabf03ca8d9f39c8aea0dec8029455b5540/opt_einsum-3.4.0-py3-none-any.whl", hash = "sha256:69bb92469f86a1565195ece4ac0323943e83477171b91d24c35afe028a90d7cd", size = 71932, upload-time = "2024-09-26T14:33:23.039Z" }, + { url = "https://files.pythonhosted.org/packages/36/0a/eea862fae6413d8181b23acf8e13489c90a45f17986ee9cf4eab8a0b9ad9/opentelemetry_api-1.30.0-py3-none-any.whl", hash = "sha256:d5f5284890d73fdf47f843dda3210edf37a38d66f44f2b5aedc1e89ed455dc09", size = 64955, upload-time = "2025-02-04T18:16:46.167Z" }, ] [[package]] -name = "optree" -version = "0.17.0" +name = "opentelemetry-sdk" +version = "1.30.0" source = { registry = "https://pypi.org/simple" } dependencies = [ + { name = "opentelemetry-api" }, + { name = "opentelemetry-semantic-conventions" }, { name = "typing-extensions" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/56/c7/0853e0c59b135dff770615d2713b547b6b3b5cde7c10995b4a5825244612/optree-0.17.0.tar.gz", hash = "sha256:5335a5ec44479920620d72324c66563bd705ab2a698605dd4b6ee67dbcad7ecd", size = 163111, upload-time = "2025-07-25T11:26:11.586Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/91/f9/6ca076fd4c6f16be031afdc711a2676c1ff15bd1717ee2e699179b1a29bc/optree-0.17.0-cp310-cp310-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:98990201f352dba253af1a995c1453818db5f08de4cae7355d85aa6023676a52", size = 350398, upload-time = "2025-07-25T11:24:26.672Z" }, - { url = "https://files.pythonhosted.org/packages/95/4c/81344cbdcf8ea8525a21c9d65892d7529010ee2146c53423b2e9a84441ba/optree-0.17.0-cp310-cp310-manylinux_2_26_i686.manylinux_2_28_i686.whl", hash = "sha256:e1a40adf6bb78a6a4b4f480879de2cb6b57d46d680a4d9834aa824f41e69c0d9", size = 404834, upload-time = "2025-07-25T11:24:28.988Z" }, - { url = "https://files.pythonhosted.org/packages/e5/c4/ac1880372a89f5c21514a7965dfa23b1afb2ad683fb9804d366727de9ecf/optree-0.17.0-cp310-cp310-manylinux_2_26_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:78a113436a0a440f900b2799584f3cc2b2eea1b245d81c3583af42ac003e333c", size = 402116, upload-time = "2025-07-25T11:24:30.396Z" }, - { url = "https://files.pythonhosted.org/packages/ff/72/ad6be4d6a03805cf3921b492494cb3371ca28060d5ad19d5a36e10c4d67d/optree-0.17.0-cp310-cp310-manylinux_2_26_s390x.manylinux_2_28_s390x.whl", hash = "sha256:0e45c16018f4283f028cf839b707b7ac734e8056a31b7198a1577161fcbe146d", size = 398491, upload-time = "2025-07-25T11:24:31.725Z" }, - { url = "https://files.pythonhosted.org/packages/d9/c1/6827fb504351f9a3935699b0eb31c8a6af59d775ee78289a25e0ba54f732/optree-0.17.0-cp310-cp310-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b698613d821d80cc216a2444ebc3145c8bf671b55a2223058a6574c1483a65f6", size = 387957, upload-time = "2025-07-25T11:24:32.759Z" }, - { url = "https://files.pythonhosted.org/packages/73/5c/13a2a864b0c0b39c3c193be534a195a3ab2463c7d0443d4a76e749e3ff83/optree-0.17.0-cp311-cp311-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:3080c564c9760711aa72d1b4d700ce1417f99ad087136f415c4eb8221169e2a3", size = 362797, upload-time = "2025-07-25T11:24:39.509Z" }, - { url = "https://files.pythonhosted.org/packages/da/f5/ff7dcb5a0108ee89c2be09aed2ebd26a7e1333d8122031aa9d9322b24ee6/optree-0.17.0-cp311-cp311-manylinux_2_26_i686.manylinux_2_28_i686.whl", hash = "sha256:834a8fb358b608240b3a38706a09b43974675624485fad64c8ee641dae2eb57d", size = 419450, upload-time = "2025-07-25T11:24:40.555Z" }, - { url = "https://files.pythonhosted.org/packages/1b/e6/48a97aefd18770b55e5ed456d8183891f325cdb6d90592e5f072ed6951f8/optree-0.17.0-cp311-cp311-manylinux_2_26_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:1a2bd263e6b5621d000d0f94de1f245414fd5dbce365a24b7b89b1ed0ef56cf9", size = 417557, upload-time = "2025-07-25T11:24:42.396Z" }, - { url = "https://files.pythonhosted.org/packages/c4/b1/4e280edab8a86be47ec1f9bd9ed4b685d2e15f0950ae62b613b26d12a1da/optree-0.17.0-cp311-cp311-manylinux_2_26_s390x.manylinux_2_28_s390x.whl", hash = "sha256:9b37daca4ad89339b1f5320cc61ac600dcf976adbb060769d36d5542d6ebfedf", size = 414174, upload-time = "2025-07-25T11:24:43.51Z" }, - { url = "https://files.pythonhosted.org/packages/db/3b/49a9a1986215dd342525974deeb17c260a83fee8fad147276fd710ac8718/optree-0.17.0-cp311-cp311-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a146a6917f3e28cfdc268ff1770aa696c346482dd3da681c3ff92153d94450ea", size = 402000, upload-time = "2025-07-25T11:24:44.819Z" }, - { url = "https://files.pythonhosted.org/packages/41/90/e12dea2cb5d8a5e17bbe3011ed4e972b89c027272a816db4897589751cad/optree-0.17.0-cp312-cp312-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e13ae51a63d69db445f269a3a4fd1d6edb064a705188d007ea47c9f034788fc5", size = 365869, upload-time = "2025-07-25T11:24:51.807Z" }, - { url = "https://files.pythonhosted.org/packages/76/ee/21af214663960a479863cd6c03d7a0abc8123ea22a6ea34689c2eed88ccd/optree-0.17.0-cp312-cp312-manylinux_2_26_i686.manylinux_2_28_i686.whl", hash = "sha256:5958f58423cc7870cb011c8c8f92687397380886e8c9d33adac752147e7bbc3f", size = 424465, upload-time = "2025-07-25T11:24:53.124Z" }, - { url = "https://files.pythonhosted.org/packages/54/a3/64b184a79373753f4f46a5cd301ea581f71d6dc1a5c103bd2394f0925d40/optree-0.17.0-cp312-cp312-manylinux_2_26_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:970ae4e47727b4c5526fc583b87d29190e576f6a2b6c19e8671589b73d256250", size = 420686, upload-time = "2025-07-25T11:24:54.212Z" }, - { url = "https://files.pythonhosted.org/packages/6c/6d/b6051b0b1ef9a49df96a66e9e62fc02620d2115d1ba659888c94e67fcfc9/optree-0.17.0-cp312-cp312-manylinux_2_26_s390x.manylinux_2_28_s390x.whl", hash = "sha256:54177fd3e6e05c08b66329e26d7d44b85f24125f25c6b74c921499a1b31b8f70", size = 421225, upload-time = "2025-07-25T11:24:55.213Z" }, - { url = "https://files.pythonhosted.org/packages/f6/f1/940bc959aaef9eede8bb1b1127833b0929c6ffa9268ec0f6cb19877e2027/optree-0.17.0-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e1959cfbc38c228c8195354967cda64887b96219924b7b3759e5ee355582c1ec", size = 408819, upload-time = "2025-07-25T11:24:56.315Z" }, - { url = "https://files.pythonhosted.org/packages/21/04/9706d11b880186e9e9d66d7c21ce249b2ce0212645137cc13fdd18247c26/optree-0.17.0-cp313-cp313-ios_13_0_arm64_iphoneos.whl", hash = "sha256:b5995a3efce4b00a14049268a81ab0379656a41ddf3c3761e3b88937fca44d48", size = 348177, upload-time = "2025-07-25T11:25:00.999Z" }, - { url = "https://files.pythonhosted.org/packages/ae/4b/0415c18816818ac871c9f3d5c7c5f4ceb83baff03ed511c9c94591ace4bc/optree-0.17.0-cp313-cp313-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:d06e8143d16fe6c0708f3cc2807b5b65f815d60ee2b52f3d79e4022c95563482", size = 354389, upload-time = "2025-07-25T11:25:02.337Z" }, - { url = "https://files.pythonhosted.org/packages/dd/12/24d4a417fd325ec06cfbce52716ac4f816ef696653b868960ac2ccb28436/optree-0.17.0-cp313-cp313-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:dfeea4aa0fd354d27922aba63ff9d86e4e126c6bf89cfb02849e68515519f1a5", size = 368513, upload-time = "2025-07-25T11:25:05.548Z" }, - { url = "https://files.pythonhosted.org/packages/30/e2/34e392209933e2c582c67594a7a6b4851bca4015c83b51c7508384b616b4/optree-0.17.0-cp313-cp313-manylinux_2_26_i686.manylinux_2_28_i686.whl", hash = "sha256:6b2ff8999a9b84d00f23a032b6b3f13678894432a335d024e0670b9880f238ca", size = 430378, upload-time = "2025-07-25T11:25:06.918Z" }, - { url = "https://files.pythonhosted.org/packages/5f/16/0a0d6139022e9a53ecb1212fb6fbc5b60eff824371071ef5f5fa481d8167/optree-0.17.0-cp313-cp313-manylinux_2_26_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:ea8bef525432b38a84e7448348da1a2dc308375bce79c77675cc50a501305851", size = 423294, upload-time = "2025-07-25T11:25:08.043Z" }, - { url = "https://files.pythonhosted.org/packages/ef/60/2e083dabb6aff6d939d8aab16ba3dbe6eee9429597a13f3fca57b33cdcde/optree-0.17.0-cp313-cp313-manylinux_2_26_s390x.manylinux_2_28_s390x.whl", hash = "sha256:f95b81aa67538d38316b184a6ff39a3725ee5c8555fba21dcb692f8d7c39302e", size = 424633, upload-time = "2025-07-25T11:25:09.141Z" }, - { url = "https://files.pythonhosted.org/packages/af/fd/0e4229b5fa3fd9d3c779a606c0f358ffbdfee717f49b3477facd04de2cec/optree-0.17.0-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e808a1125169ae90de623456ef2423eb84a8578a74f03fe48b06b8561c2cc31d", size = 414866, upload-time = "2025-07-25T11:25:10.214Z" }, - { url = "https://files.pythonhosted.org/packages/39/df/b8882f5519c85af146de3a79a08066a56fe634b23052c593fcedc70bfcd7/optree-0.17.0-cp313-cp313t-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:8e45a13b35873712e095fe0f7fd6e9c4f98f3bd5af6f5dc33c17b80357bc97fc", size = 386945, upload-time = "2025-07-25T11:25:17.728Z" }, - { url = "https://files.pythonhosted.org/packages/ca/d7/91f4efb509bda601a1591465c4a5bd55320e4bafe06b294bf80754127b0e/optree-0.17.0-cp313-cp313t-manylinux_2_26_i686.manylinux_2_28_i686.whl", hash = "sha256:bfaf04d833dc53e5cfccff3b564e934a49086158472e31d84df31fce6d4f7b1c", size = 444177, upload-time = "2025-07-25T11:25:18.749Z" }, - { url = "https://files.pythonhosted.org/packages/84/17/a4833006e925c6ed5c45ceb02e65c9e9a260e70da6523858fcf628481847/optree-0.17.0-cp313-cp313t-manylinux_2_26_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:b4c1d030ac1c881803f5c8e23d241159ae403fd00cdf57625328f282fc671ebd", size = 439198, upload-time = "2025-07-25T11:25:19.865Z" }, - { url = "https://files.pythonhosted.org/packages/ef/d1/c08fc60f6dfcb1b86ca1fdc0add08a98412a1596cd45830acbdc309f2cdb/optree-0.17.0-cp313-cp313t-manylinux_2_26_s390x.manylinux_2_28_s390x.whl", hash = "sha256:bd7738709970acab5d963896192b63b2718be93bb6c0bcea91895ea157fa2b13", size = 439391, upload-time = "2025-07-25T11:25:20.942Z" }, - { url = "https://files.pythonhosted.org/packages/05/8f/461e10201003e6ad6bff3c594a29a7e044454aba68c5f795f4c8386ce47c/optree-0.17.0-cp313-cp313t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:1644bc24b6e93cafccfdeee44157c3d4ae9bb0af3e861300602d716699865b1a", size = 426555, upload-time = "2025-07-25T11:25:21.968Z" }, - { url = "https://files.pythonhosted.org/packages/3c/21/6480d23b52b2e23b976fe254b9fbdc4b514e90a349b1ee73565b185c69f1/optree-0.17.0-cp314-cp314-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:dd21e0a89806cc3b86aaa578a73897d56085038fe432043534a23b2e559d7691", size = 369929, upload-time = "2025-07-25T11:25:28.897Z" }, - { url = "https://files.pythonhosted.org/packages/b3/29/69bb26473ff862a1792f5568c977e7a2580e08afe0fdcd7a7b3e1e4d6933/optree-0.17.0-cp314-cp314-manylinux_2_26_i686.manylinux_2_28_i686.whl", hash = "sha256:9211c61285b8b3e42fd0e803cebd6e2b0987d8b2edffe45b42923debca09a9df", size = 430381, upload-time = "2025-07-25T11:25:29.984Z" }, - { url = "https://files.pythonhosted.org/packages/c8/8b/2c0a38c0d0c2396d698b97216cd6814d6754d11997b6ac66c57d87d71bae/optree-0.17.0-cp314-cp314-manylinux_2_26_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:87938255749a45979c4e331627cb33d81aa08b0a09d024368b3e25ff67f0e9f2", size = 424461, upload-time = "2025-07-25T11:25:31.116Z" }, - { url = "https://files.pythonhosted.org/packages/a7/77/08fda3f97621190d50762225ee8bad87463a8b3a55fba451a999971ff130/optree-0.17.0-cp314-cp314-manylinux_2_26_s390x.manylinux_2_28_s390x.whl", hash = "sha256:3432858145fd1955a3be12207507466ac40a6911f428bf5d2d6c7f67486530a2", size = 427234, upload-time = "2025-07-25T11:25:32.289Z" }, - { url = "https://files.pythonhosted.org/packages/ea/b5/b4f19952c36d6448c85a6ef6be5f916dd13548de2b684ab123f04b450850/optree-0.17.0-cp314-cp314-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:5afe3e9e2f6da0a0a5c0892f32f675eb88965036b061aa555b74e6c412a05e17", size = 413863, upload-time = "2025-07-25T11:25:33.379Z" }, - { url = "https://files.pythonhosted.org/packages/88/42/6003f13e66cfbe7f0011bf8509da2479aba93068cdb9d79bf46010255089/optree-0.17.0-cp314-cp314t-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5739c03a3362be42cb7649e82457c90aa818aa3e82af9681d3100c3346f4a90f", size = 386975, upload-time = "2025-07-25T11:25:40.376Z" }, - { url = "https://files.pythonhosted.org/packages/d0/53/621642abd76eda5a941b47adc98be81f0052683160be776499d11b4af83d/optree-0.17.0-cp314-cp314t-manylinux_2_26_i686.manylinux_2_28_i686.whl", hash = "sha256:ee07b59a08bd45aedd5252241a98841f1a5082a7b9b73df2dae6a433aa2a91d8", size = 444173, upload-time = "2025-07-25T11:25:41.474Z" }, - { url = "https://files.pythonhosted.org/packages/5b/d3/8819a2d5105a240d6793d11a61d597db91756ce84da5cee08808c6b8f61f/optree-0.17.0-cp314-cp314t-manylinux_2_26_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:875c017890a4b5d566af5593cab67fe3c4845544942af57e6bb9dea17e060297", size = 439080, upload-time = "2025-07-25T11:25:42.605Z" }, - { url = "https://files.pythonhosted.org/packages/c6/ef/9dbd34dfd1ad89feb239ca9925897a14ac94f190379a3bd991afdfd94186/optree-0.17.0-cp314-cp314t-manylinux_2_26_s390x.manylinux_2_28_s390x.whl", hash = "sha256:ffa5686191139f763e13445a169765c83517164bc28e60dbedb19bed2b2655f1", size = 439422, upload-time = "2025-07-25T11:25:43.672Z" }, - { url = "https://files.pythonhosted.org/packages/86/ca/a7a7549af2951925a692df508902ed2a6a94a51bc846806d2281b1029ef9/optree-0.17.0-cp314-cp314t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:575cf48cc2190acb565bd2b26b6f9b15c4e3b60183e86031215badc9d5441345", size = 426579, upload-time = "2025-07-25T11:25:44.765Z" }, - { url = "https://files.pythonhosted.org/packages/ed/d7/3036d15c028c447b1bd65dcf8f66cfd775bfa4e52daa74b82fb1d3c88faf/optree-0.17.0-pp310-pypy310_pp73-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:adde1427e0982cfc5f56939c26b4ebbd833091a176734c79fb95c78bdf833dff", size = 350952, upload-time = "2025-07-25T11:26:02.692Z" }, - { url = "https://files.pythonhosted.org/packages/71/45/e710024ef77324e745de48efd64f6270d8c209f14107a48ffef4049ac57a/optree-0.17.0-pp310-pypy310_pp73-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a80b7e5de5dd09b9c8b62d501e29a3850b047565c336c9d004b07ee1c01f4ae1", size = 389568, upload-time = "2025-07-25T11:26:04.094Z" }, - { url = "https://files.pythonhosted.org/packages/69/c4/94a187ed3ca71194b9da6a276790e1703c7544c8f695ac915214ae8ce934/optree-0.17.0-pp311-pypy311_pp73-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f87f6f39015fc82d7adeee19900d246b89911319726e93cb2dbd4d1a809899bd", size = 363728, upload-time = "2025-07-25T11:26:07.959Z" }, - { url = "https://files.pythonhosted.org/packages/cd/99/23b7a484da8dfb814107b20ef2c93ef27c04f36aeb83bd976964a5b69e06/optree-0.17.0-pp311-pypy311_pp73-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:58b0a83a967d2ef0f343db7182f0ad074eb1166bcaea909ae33909462013f151", size = 404649, upload-time = "2025-07-25T11:26:09.463Z" }, +sdist = { url = "https://files.pythonhosted.org/packages/93/ee/d710062e8a862433d1be0b85920d0c653abe318878fef2d14dfe2c62ff7b/opentelemetry_sdk-1.30.0.tar.gz", hash = "sha256:c9287a9e4a7614b9946e933a67168450b9ab35f08797eb9bc77d998fa480fa18", size = 158633, upload-time = "2025-02-04T18:17:28.908Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/97/28/64d781d6adc6bda2260067ce2902bd030cf45aec657e02e28c5b4480b976/opentelemetry_sdk-1.30.0-py3-none-any.whl", hash = "sha256:14fe7afc090caad881addb6926cec967129bd9260c4d33ae6a217359f6b61091", size = 118717, upload-time = "2025-02-04T18:17:09.353Z" }, +] + +[[package]] +name = "opentelemetry-semantic-conventions" +version = "0.51b0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "deprecated" }, + { name = "opentelemetry-api" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/1e/c0/0f9ef4605fea7f2b83d55dd0b0d7aebe8feead247cd6facd232b30907b4f/opentelemetry_semantic_conventions-0.51b0.tar.gz", hash = "sha256:3fabf47f35d1fd9aebcdca7e6802d86bd5ebc3bc3408b7e3248dde6e87a18c47", size = 107191, upload-time = "2025-02-04T18:17:29.903Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/2e/75/d7bdbb6fd8630b4cafb883482b75c4fc276b6426619539d266e32ac53266/opentelemetry_semantic_conventions-0.51b0-py3-none-any.whl", hash = "sha256:fdc777359418e8d06c86012c3dc92c88a6453ba662e941593adb062e48c2eeae", size = 177416, upload-time = "2025-02-04T18:17:11.305Z" }, ] [[package]] name = "packaging" -version = "25.0" +version = "26.2" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/a1/d4/1fc4078c65507b51b96ca8f8c3ba19e6a61c8253c72794544580a7b6c24d/packaging-25.0.tar.gz", hash = "sha256:d443872c98d677bf60f6a1f2f8c1cb748e8fe762d2bf9d3148b5599295b0fc4f", size = 165727, upload-time = "2025-04-19T11:48:59.673Z" } +sdist = { url = "https://files.pythonhosted.org/packages/d7/f1/e7a6dd94a8d4a5626c03e4e99c87f241ba9e350cd9e6d75123f992427270/packaging-26.2.tar.gz", hash = "sha256:ff452ff5a3e828ce110190feff1178bb1f2ea2281fa2075aadb987c2fb221661", size = 228134, upload-time = "2026-04-24T20:15:23.917Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/20/12/38679034af332785aac8774540895e234f4d07f7545804097de4b666afd8/packaging-25.0-py3-none-any.whl", hash = "sha256:29572ef2b1f17581046b3a2227d5c611fb25ec70ca1ba8554b24b0e69331a484", size = 66469, upload-time = "2025-04-19T11:48:57.875Z" }, + { url = "https://files.pythonhosted.org/packages/df/b2/87e62e8c3e2f4b32e5fe99e0b86d576da1312593b39f47d8ceef365e95ed/packaging-26.2-py3-none-any.whl", hash = "sha256:5fc45236b9446107ff2415ce77c807cee2862cb6fac22b8a73826d0693b0980e", size = 100195, upload-time = "2026-04-24T20:15:22.081Z" }, ] [[package]] @@ -1750,7 +1839,8 @@ version = "2.3.3" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, - { name = "numpy", version = "2.3.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" }, + { name = "numpy", version = "2.4.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.11.*'" }, + { name = "numpy", version = "2.5.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.12'" }, { name = "python-dateutil" }, { name = "pytz" }, { name = "tzdata" }, @@ -1954,102 +2044,130 @@ wheels = [ [[package]] name = "propcache" -version = "0.3.2" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/a6/16/43264e4a779dd8588c21a70f0709665ee8f611211bdd2c87d952cfa7c776/propcache-0.3.2.tar.gz", hash = "sha256:20d7d62e4e7ef05f221e0db2856b979540686342e7dd9973b815599c7057e168", size = 44139, upload-time = "2025-06-09T22:56:06.081Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/ab/14/510deed325e262afeb8b360043c5d7c960da7d3ecd6d6f9496c9c56dc7f4/propcache-0.3.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:22d9962a358aedbb7a2e36187ff273adeaab9743373a272976d2e348d08c7770", size = 73178, upload-time = "2025-06-09T22:53:40.126Z" }, - { url = "https://files.pythonhosted.org/packages/cd/4e/ad52a7925ff01c1325653a730c7ec3175a23f948f08626a534133427dcff/propcache-0.3.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:0d0fda578d1dc3f77b6b5a5dce3b9ad69a8250a891760a548df850a5e8da87f3", size = 43133, upload-time = "2025-06-09T22:53:41.965Z" }, - { url = "https://files.pythonhosted.org/packages/63/7c/e9399ba5da7780871db4eac178e9c2e204c23dd3e7d32df202092a1ed400/propcache-0.3.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:3def3da3ac3ce41562d85db655d18ebac740cb3fa4367f11a52b3da9d03a5cc3", size = 43039, upload-time = "2025-06-09T22:53:43.268Z" }, - { url = "https://files.pythonhosted.org/packages/22/e1/58da211eb8fdc6fc854002387d38f415a6ca5f5c67c1315b204a5d3e9d7a/propcache-0.3.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9bec58347a5a6cebf239daba9bda37dffec5b8d2ce004d9fe4edef3d2815137e", size = 201903, upload-time = "2025-06-09T22:53:44.872Z" }, - { url = "https://files.pythonhosted.org/packages/c4/0a/550ea0f52aac455cb90111c8bab995208443e46d925e51e2f6ebdf869525/propcache-0.3.2-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:55ffda449a507e9fbd4aca1a7d9aa6753b07d6166140e5a18d2ac9bc49eac220", size = 213362, upload-time = "2025-06-09T22:53:46.707Z" }, - { url = "https://files.pythonhosted.org/packages/5a/af/9893b7d878deda9bb69fcf54600b247fba7317761b7db11fede6e0f28bd0/propcache-0.3.2-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:64a67fb39229a8a8491dd42f864e5e263155e729c2e7ff723d6e25f596b1e8cb", size = 210525, upload-time = "2025-06-09T22:53:48.547Z" }, - { url = "https://files.pythonhosted.org/packages/7c/bb/38fd08b278ca85cde36d848091ad2b45954bc5f15cce494bb300b9285831/propcache-0.3.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9da1cf97b92b51253d5b68cf5a2b9e0dafca095e36b7f2da335e27dc6172a614", size = 198283, upload-time = "2025-06-09T22:53:50.067Z" }, - { url = "https://files.pythonhosted.org/packages/78/8c/9fe55bd01d362bafb413dfe508c48753111a1e269737fa143ba85693592c/propcache-0.3.2-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:5f559e127134b07425134b4065be45b166183fdcb433cb6c24c8e4149056ad50", size = 191872, upload-time = "2025-06-09T22:53:51.438Z" }, - { url = "https://files.pythonhosted.org/packages/54/14/4701c33852937a22584e08abb531d654c8bcf7948a8f87ad0a4822394147/propcache-0.3.2-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:aff2e4e06435d61f11a428360a932138d0ec288b0a31dd9bd78d200bd4a2b339", size = 199452, upload-time = "2025-06-09T22:53:53.229Z" }, - { url = "https://files.pythonhosted.org/packages/16/44/447f2253d859602095356007657ee535e0093215ea0b3d1d6a41d16e5201/propcache-0.3.2-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:4927842833830942a5d0a56e6f4839bc484785b8e1ce8d287359794818633ba0", size = 191567, upload-time = "2025-06-09T22:53:54.541Z" }, - { url = "https://files.pythonhosted.org/packages/f2/b3/e4756258749bb2d3b46defcff606a2f47410bab82be5824a67e84015b267/propcache-0.3.2-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:6107ddd08b02654a30fb8ad7a132021759d750a82578b94cd55ee2772b6ebea2", size = 193015, upload-time = "2025-06-09T22:53:56.44Z" }, - { url = "https://files.pythonhosted.org/packages/1e/df/e6d3c7574233164b6330b9fd697beeac402afd367280e6dc377bb99b43d9/propcache-0.3.2-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:70bd8b9cd6b519e12859c99f3fc9a93f375ebd22a50296c3a295028bea73b9e7", size = 204660, upload-time = "2025-06-09T22:53:57.839Z" }, - { url = "https://files.pythonhosted.org/packages/b2/53/e4d31dd5170b4a0e2e6b730f2385a96410633b4833dc25fe5dffd1f73294/propcache-0.3.2-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:2183111651d710d3097338dd1893fcf09c9f54e27ff1a8795495a16a469cc90b", size = 206105, upload-time = "2025-06-09T22:53:59.638Z" }, - { url = "https://files.pythonhosted.org/packages/7f/fe/74d54cf9fbe2a20ff786e5f7afcfde446588f0cf15fb2daacfbc267b866c/propcache-0.3.2-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:fb075ad271405dcad8e2a7ffc9a750a3bf70e533bd86e89f0603e607b93aa64c", size = 196980, upload-time = "2025-06-09T22:54:01.071Z" }, - { url = "https://files.pythonhosted.org/packages/22/ec/c469c9d59dada8a7679625e0440b544fe72e99311a4679c279562051f6fc/propcache-0.3.2-cp310-cp310-win32.whl", hash = "sha256:404d70768080d3d3bdb41d0771037da19d8340d50b08e104ca0e7f9ce55fce70", size = 37679, upload-time = "2025-06-09T22:54:03.003Z" }, - { url = "https://files.pythonhosted.org/packages/38/35/07a471371ac89d418f8d0b699c75ea6dca2041fbda360823de21f6a9ce0a/propcache-0.3.2-cp310-cp310-win_amd64.whl", hash = "sha256:7435d766f978b4ede777002e6b3b6641dd229cd1da8d3d3106a45770365f9ad9", size = 41459, upload-time = "2025-06-09T22:54:04.134Z" }, - { url = "https://files.pythonhosted.org/packages/80/8d/e8b436717ab9c2cfc23b116d2c297305aa4cd8339172a456d61ebf5669b8/propcache-0.3.2-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:0b8d2f607bd8f80ddc04088bc2a037fdd17884a6fcadc47a96e334d72f3717be", size = 74207, upload-time = "2025-06-09T22:54:05.399Z" }, - { url = "https://files.pythonhosted.org/packages/d6/29/1e34000e9766d112171764b9fa3226fa0153ab565d0c242c70e9945318a7/propcache-0.3.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:06766d8f34733416e2e34f46fea488ad5d60726bb9481d3cddf89a6fa2d9603f", size = 43648, upload-time = "2025-06-09T22:54:08.023Z" }, - { url = "https://files.pythonhosted.org/packages/46/92/1ad5af0df781e76988897da39b5f086c2bf0f028b7f9bd1f409bb05b6874/propcache-0.3.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:a2dc1f4a1df4fecf4e6f68013575ff4af84ef6f478fe5344317a65d38a8e6dc9", size = 43496, upload-time = "2025-06-09T22:54:09.228Z" }, - { url = "https://files.pythonhosted.org/packages/b3/ce/e96392460f9fb68461fabab3e095cb00c8ddf901205be4eae5ce246e5b7e/propcache-0.3.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:be29c4f4810c5789cf10ddf6af80b041c724e629fa51e308a7a0fb19ed1ef7bf", size = 217288, upload-time = "2025-06-09T22:54:10.466Z" }, - { url = "https://files.pythonhosted.org/packages/c5/2a/866726ea345299f7ceefc861a5e782b045545ae6940851930a6adaf1fca6/propcache-0.3.2-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:59d61f6970ecbd8ff2e9360304d5c8876a6abd4530cb752c06586849ac8a9dc9", size = 227456, upload-time = "2025-06-09T22:54:11.828Z" }, - { url = "https://files.pythonhosted.org/packages/de/03/07d992ccb6d930398689187e1b3c718339a1c06b8b145a8d9650e4726166/propcache-0.3.2-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:62180e0b8dbb6b004baec00a7983e4cc52f5ada9cd11f48c3528d8cfa7b96a66", size = 225429, upload-time = "2025-06-09T22:54:13.823Z" }, - { url = "https://files.pythonhosted.org/packages/5d/e6/116ba39448753b1330f48ab8ba927dcd6cf0baea8a0ccbc512dfb49ba670/propcache-0.3.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c144ca294a204c470f18cf4c9d78887810d04a3e2fbb30eea903575a779159df", size = 213472, upload-time = "2025-06-09T22:54:15.232Z" }, - { url = "https://files.pythonhosted.org/packages/a6/85/f01f5d97e54e428885a5497ccf7f54404cbb4f906688a1690cd51bf597dc/propcache-0.3.2-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c5c2a784234c28854878d68978265617aa6dc0780e53d44b4d67f3651a17a9a2", size = 204480, upload-time = "2025-06-09T22:54:17.104Z" }, - { url = "https://files.pythonhosted.org/packages/e3/79/7bf5ab9033b8b8194cc3f7cf1aaa0e9c3256320726f64a3e1f113a812dce/propcache-0.3.2-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:5745bc7acdafa978ca1642891b82c19238eadc78ba2aaa293c6863b304e552d7", size = 214530, upload-time = "2025-06-09T22:54:18.512Z" }, - { url = "https://files.pythonhosted.org/packages/31/0b/bd3e0c00509b609317df4a18e6b05a450ef2d9a963e1d8bc9c9415d86f30/propcache-0.3.2-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:c0075bf773d66fa8c9d41f66cc132ecc75e5bb9dd7cce3cfd14adc5ca184cb95", size = 205230, upload-time = "2025-06-09T22:54:19.947Z" }, - { url = "https://files.pythonhosted.org/packages/7a/23/fae0ff9b54b0de4e819bbe559508da132d5683c32d84d0dc2ccce3563ed4/propcache-0.3.2-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:5f57aa0847730daceff0497f417c9de353c575d8da3579162cc74ac294c5369e", size = 206754, upload-time = "2025-06-09T22:54:21.716Z" }, - { url = "https://files.pythonhosted.org/packages/b7/7f/ad6a3c22630aaa5f618b4dc3c3598974a72abb4c18e45a50b3cdd091eb2f/propcache-0.3.2-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:eef914c014bf72d18efb55619447e0aecd5fb7c2e3fa7441e2e5d6099bddff7e", size = 218430, upload-time = "2025-06-09T22:54:23.17Z" }, - { url = "https://files.pythonhosted.org/packages/5b/2c/ba4f1c0e8a4b4c75910742f0d333759d441f65a1c7f34683b4a74c0ee015/propcache-0.3.2-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:2a4092e8549031e82facf3decdbc0883755d5bbcc62d3aea9d9e185549936dcf", size = 223884, upload-time = "2025-06-09T22:54:25.539Z" }, - { url = "https://files.pythonhosted.org/packages/88/e4/ebe30fc399e98572019eee82ad0caf512401661985cbd3da5e3140ffa1b0/propcache-0.3.2-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:85871b050f174bc0bfb437efbdb68aaf860611953ed12418e4361bc9c392749e", size = 211480, upload-time = "2025-06-09T22:54:26.892Z" }, - { url = "https://files.pythonhosted.org/packages/96/0a/7d5260b914e01d1d0906f7f38af101f8d8ed0dc47426219eeaf05e8ea7c2/propcache-0.3.2-cp311-cp311-win32.whl", hash = "sha256:36c8d9b673ec57900c3554264e630d45980fd302458e4ac801802a7fd2ef7897", size = 37757, upload-time = "2025-06-09T22:54:28.241Z" }, - { url = "https://files.pythonhosted.org/packages/e1/2d/89fe4489a884bc0da0c3278c552bd4ffe06a1ace559db5ef02ef24ab446b/propcache-0.3.2-cp311-cp311-win_amd64.whl", hash = "sha256:e53af8cb6a781b02d2ea079b5b853ba9430fcbe18a8e3ce647d5982a3ff69f39", size = 41500, upload-time = "2025-06-09T22:54:29.4Z" }, - { url = "https://files.pythonhosted.org/packages/a8/42/9ca01b0a6f48e81615dca4765a8f1dd2c057e0540f6116a27dc5ee01dfb6/propcache-0.3.2-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:8de106b6c84506b31c27168582cd3cb3000a6412c16df14a8628e5871ff83c10", size = 73674, upload-time = "2025-06-09T22:54:30.551Z" }, - { url = "https://files.pythonhosted.org/packages/af/6e/21293133beb550f9c901bbece755d582bfaf2176bee4774000bd4dd41884/propcache-0.3.2-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:28710b0d3975117239c76600ea351934ac7b5ff56e60953474342608dbbb6154", size = 43570, upload-time = "2025-06-09T22:54:32.296Z" }, - { url = "https://files.pythonhosted.org/packages/0c/c8/0393a0a3a2b8760eb3bde3c147f62b20044f0ddac81e9d6ed7318ec0d852/propcache-0.3.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:ce26862344bdf836650ed2487c3d724b00fbfec4233a1013f597b78c1cb73615", size = 43094, upload-time = "2025-06-09T22:54:33.929Z" }, - { url = "https://files.pythonhosted.org/packages/37/2c/489afe311a690399d04a3e03b069225670c1d489eb7b044a566511c1c498/propcache-0.3.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bca54bd347a253af2cf4544bbec232ab982f4868de0dd684246b67a51bc6b1db", size = 226958, upload-time = "2025-06-09T22:54:35.186Z" }, - { url = "https://files.pythonhosted.org/packages/9d/ca/63b520d2f3d418c968bf596839ae26cf7f87bead026b6192d4da6a08c467/propcache-0.3.2-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:55780d5e9a2ddc59711d727226bb1ba83a22dd32f64ee15594b9392b1f544eb1", size = 234894, upload-time = "2025-06-09T22:54:36.708Z" }, - { url = "https://files.pythonhosted.org/packages/11/60/1d0ed6fff455a028d678df30cc28dcee7af77fa2b0e6962ce1df95c9a2a9/propcache-0.3.2-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:035e631be25d6975ed87ab23153db6a73426a48db688070d925aa27e996fe93c", size = 233672, upload-time = "2025-06-09T22:54:38.062Z" }, - { url = "https://files.pythonhosted.org/packages/37/7c/54fd5301ef38505ab235d98827207176a5c9b2aa61939b10a460ca53e123/propcache-0.3.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ee6f22b6eaa39297c751d0e80c0d3a454f112f5c6481214fcf4c092074cecd67", size = 224395, upload-time = "2025-06-09T22:54:39.634Z" }, - { url = "https://files.pythonhosted.org/packages/ee/1a/89a40e0846f5de05fdc6779883bf46ba980e6df4d2ff8fb02643de126592/propcache-0.3.2-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7ca3aee1aa955438c4dba34fc20a9f390e4c79967257d830f137bd5a8a32ed3b", size = 212510, upload-time = "2025-06-09T22:54:41.565Z" }, - { url = "https://files.pythonhosted.org/packages/5e/33/ca98368586c9566a6b8d5ef66e30484f8da84c0aac3f2d9aec6d31a11bd5/propcache-0.3.2-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:7a4f30862869fa2b68380d677cc1c5fcf1e0f2b9ea0cf665812895c75d0ca3b8", size = 222949, upload-time = "2025-06-09T22:54:43.038Z" }, - { url = "https://files.pythonhosted.org/packages/ba/11/ace870d0aafe443b33b2f0b7efdb872b7c3abd505bfb4890716ad7865e9d/propcache-0.3.2-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:b77ec3c257d7816d9f3700013639db7491a434644c906a2578a11daf13176251", size = 217258, upload-time = "2025-06-09T22:54:44.376Z" }, - { url = "https://files.pythonhosted.org/packages/5b/d2/86fd6f7adffcfc74b42c10a6b7db721d1d9ca1055c45d39a1a8f2a740a21/propcache-0.3.2-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:cab90ac9d3f14b2d5050928483d3d3b8fb6b4018893fc75710e6aa361ecb2474", size = 213036, upload-time = "2025-06-09T22:54:46.243Z" }, - { url = "https://files.pythonhosted.org/packages/07/94/2d7d1e328f45ff34a0a284cf5a2847013701e24c2a53117e7c280a4316b3/propcache-0.3.2-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:0b504d29f3c47cf6b9e936c1852246c83d450e8e063d50562115a6be6d3a2535", size = 227684, upload-time = "2025-06-09T22:54:47.63Z" }, - { url = "https://files.pythonhosted.org/packages/b7/05/37ae63a0087677e90b1d14710e532ff104d44bc1efa3b3970fff99b891dc/propcache-0.3.2-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:ce2ac2675a6aa41ddb2a0c9cbff53780a617ac3d43e620f8fd77ba1c84dcfc06", size = 234562, upload-time = "2025-06-09T22:54:48.982Z" }, - { url = "https://files.pythonhosted.org/packages/a4/7c/3f539fcae630408d0bd8bf3208b9a647ccad10976eda62402a80adf8fc34/propcache-0.3.2-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:62b4239611205294cc433845b914131b2a1f03500ff3c1ed093ed216b82621e1", size = 222142, upload-time = "2025-06-09T22:54:50.424Z" }, - { url = "https://files.pythonhosted.org/packages/7c/d2/34b9eac8c35f79f8a962546b3e97e9d4b990c420ee66ac8255d5d9611648/propcache-0.3.2-cp312-cp312-win32.whl", hash = "sha256:df4a81b9b53449ebc90cc4deefb052c1dd934ba85012aa912c7ea7b7e38b60c1", size = 37711, upload-time = "2025-06-09T22:54:52.072Z" }, - { url = "https://files.pythonhosted.org/packages/19/61/d582be5d226cf79071681d1b46b848d6cb03d7b70af7063e33a2787eaa03/propcache-0.3.2-cp312-cp312-win_amd64.whl", hash = "sha256:7046e79b989d7fe457bb755844019e10f693752d169076138abf17f31380800c", size = 41479, upload-time = "2025-06-09T22:54:53.234Z" }, - { url = "https://files.pythonhosted.org/packages/dc/d1/8c747fafa558c603c4ca19d8e20b288aa0c7cda74e9402f50f31eb65267e/propcache-0.3.2-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:ca592ed634a73ca002967458187109265e980422116c0a107cf93d81f95af945", size = 71286, upload-time = "2025-06-09T22:54:54.369Z" }, - { url = "https://files.pythonhosted.org/packages/61/99/d606cb7986b60d89c36de8a85d58764323b3a5ff07770a99d8e993b3fa73/propcache-0.3.2-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:9ecb0aad4020e275652ba3975740f241bd12a61f1a784df044cf7477a02bc252", size = 42425, upload-time = "2025-06-09T22:54:55.642Z" }, - { url = "https://files.pythonhosted.org/packages/8c/96/ef98f91bbb42b79e9bb82bdd348b255eb9d65f14dbbe3b1594644c4073f7/propcache-0.3.2-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:7f08f1cc28bd2eade7a8a3d2954ccc673bb02062e3e7da09bc75d843386b342f", size = 41846, upload-time = "2025-06-09T22:54:57.246Z" }, - { url = "https://files.pythonhosted.org/packages/5b/ad/3f0f9a705fb630d175146cd7b1d2bf5555c9beaed54e94132b21aac098a6/propcache-0.3.2-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d1a342c834734edb4be5ecb1e9fb48cb64b1e2320fccbd8c54bf8da8f2a84c33", size = 208871, upload-time = "2025-06-09T22:54:58.975Z" }, - { url = "https://files.pythonhosted.org/packages/3a/38/2085cda93d2c8b6ec3e92af2c89489a36a5886b712a34ab25de9fbca7992/propcache-0.3.2-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:8a544caaae1ac73f1fecfae70ded3e93728831affebd017d53449e3ac052ac1e", size = 215720, upload-time = "2025-06-09T22:55:00.471Z" }, - { url = "https://files.pythonhosted.org/packages/61/c1/d72ea2dc83ac7f2c8e182786ab0fc2c7bd123a1ff9b7975bee671866fe5f/propcache-0.3.2-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:310d11aa44635298397db47a3ebce7db99a4cc4b9bbdfcf6c98a60c8d5261cf1", size = 215203, upload-time = "2025-06-09T22:55:01.834Z" }, - { url = "https://files.pythonhosted.org/packages/af/81/b324c44ae60c56ef12007105f1460d5c304b0626ab0cc6b07c8f2a9aa0b8/propcache-0.3.2-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4c1396592321ac83157ac03a2023aa6cc4a3cc3cfdecb71090054c09e5a7cce3", size = 206365, upload-time = "2025-06-09T22:55:03.199Z" }, - { url = "https://files.pythonhosted.org/packages/09/73/88549128bb89e66d2aff242488f62869014ae092db63ccea53c1cc75a81d/propcache-0.3.2-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:8cabf5b5902272565e78197edb682017d21cf3b550ba0460ee473753f28d23c1", size = 196016, upload-time = "2025-06-09T22:55:04.518Z" }, - { url = "https://files.pythonhosted.org/packages/b9/3f/3bdd14e737d145114a5eb83cb172903afba7242f67c5877f9909a20d948d/propcache-0.3.2-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:0a2f2235ac46a7aa25bdeb03a9e7060f6ecbd213b1f9101c43b3090ffb971ef6", size = 205596, upload-time = "2025-06-09T22:55:05.942Z" }, - { url = "https://files.pythonhosted.org/packages/0f/ca/2f4aa819c357d3107c3763d7ef42c03980f9ed5c48c82e01e25945d437c1/propcache-0.3.2-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:92b69e12e34869a6970fd2f3da91669899994b47c98f5d430b781c26f1d9f387", size = 200977, upload-time = "2025-06-09T22:55:07.792Z" }, - { url = "https://files.pythonhosted.org/packages/cd/4a/e65276c7477533c59085251ae88505caf6831c0e85ff8b2e31ebcbb949b1/propcache-0.3.2-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:54e02207c79968ebbdffc169591009f4474dde3b4679e16634d34c9363ff56b4", size = 197220, upload-time = "2025-06-09T22:55:09.173Z" }, - { url = "https://files.pythonhosted.org/packages/7c/54/fc7152e517cf5578278b242396ce4d4b36795423988ef39bb8cd5bf274c8/propcache-0.3.2-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:4adfb44cb588001f68c5466579d3f1157ca07f7504fc91ec87862e2b8e556b88", size = 210642, upload-time = "2025-06-09T22:55:10.62Z" }, - { url = "https://files.pythonhosted.org/packages/b9/80/abeb4a896d2767bf5f1ea7b92eb7be6a5330645bd7fb844049c0e4045d9d/propcache-0.3.2-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:fd3e6019dc1261cd0291ee8919dd91fbab7b169bb76aeef6c716833a3f65d206", size = 212789, upload-time = "2025-06-09T22:55:12.029Z" }, - { url = "https://files.pythonhosted.org/packages/b3/db/ea12a49aa7b2b6d68a5da8293dcf50068d48d088100ac016ad92a6a780e6/propcache-0.3.2-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:4c181cad81158d71c41a2bce88edce078458e2dd5ffee7eddd6b05da85079f43", size = 205880, upload-time = "2025-06-09T22:55:13.45Z" }, - { url = "https://files.pythonhosted.org/packages/d1/e5/9076a0bbbfb65d1198007059c65639dfd56266cf8e477a9707e4b1999ff4/propcache-0.3.2-cp313-cp313-win32.whl", hash = "sha256:8a08154613f2249519e549de2330cf8e2071c2887309a7b07fb56098f5170a02", size = 37220, upload-time = "2025-06-09T22:55:15.284Z" }, - { url = "https://files.pythonhosted.org/packages/d3/f5/b369e026b09a26cd77aa88d8fffd69141d2ae00a2abaaf5380d2603f4b7f/propcache-0.3.2-cp313-cp313-win_amd64.whl", hash = "sha256:e41671f1594fc4ab0a6dec1351864713cb3a279910ae8b58f884a88a0a632c05", size = 40678, upload-time = "2025-06-09T22:55:16.445Z" }, - { url = "https://files.pythonhosted.org/packages/a4/3a/6ece377b55544941a08d03581c7bc400a3c8cd3c2865900a68d5de79e21f/propcache-0.3.2-cp313-cp313t-macosx_10_13_universal2.whl", hash = "sha256:9a3cf035bbaf035f109987d9d55dc90e4b0e36e04bbbb95af3055ef17194057b", size = 76560, upload-time = "2025-06-09T22:55:17.598Z" }, - { url = "https://files.pythonhosted.org/packages/0c/da/64a2bb16418740fa634b0e9c3d29edff1db07f56d3546ca2d86ddf0305e1/propcache-0.3.2-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:156c03d07dc1323d8dacaa221fbe028c5c70d16709cdd63502778e6c3ccca1b0", size = 44676, upload-time = "2025-06-09T22:55:18.922Z" }, - { url = "https://files.pythonhosted.org/packages/36/7b/f025e06ea51cb72c52fb87e9b395cced02786610b60a3ed51da8af017170/propcache-0.3.2-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:74413c0ba02ba86f55cf60d18daab219f7e531620c15f1e23d95563f505efe7e", size = 44701, upload-time = "2025-06-09T22:55:20.106Z" }, - { url = "https://files.pythonhosted.org/packages/a4/00/faa1b1b7c3b74fc277f8642f32a4c72ba1d7b2de36d7cdfb676db7f4303e/propcache-0.3.2-cp313-cp313t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f066b437bb3fa39c58ff97ab2ca351db465157d68ed0440abecb21715eb24b28", size = 276934, upload-time = "2025-06-09T22:55:21.5Z" }, - { url = "https://files.pythonhosted.org/packages/74/ab/935beb6f1756e0476a4d5938ff44bf0d13a055fed880caf93859b4f1baf4/propcache-0.3.2-cp313-cp313t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:f1304b085c83067914721e7e9d9917d41ad87696bf70f0bc7dee450e9c71ad0a", size = 278316, upload-time = "2025-06-09T22:55:22.918Z" }, - { url = "https://files.pythonhosted.org/packages/f8/9d/994a5c1ce4389610838d1caec74bdf0e98b306c70314d46dbe4fcf21a3e2/propcache-0.3.2-cp313-cp313t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:ab50cef01b372763a13333b4e54021bdcb291fc9a8e2ccb9c2df98be51bcde6c", size = 282619, upload-time = "2025-06-09T22:55:24.651Z" }, - { url = "https://files.pythonhosted.org/packages/2b/00/a10afce3d1ed0287cef2e09506d3be9822513f2c1e96457ee369adb9a6cd/propcache-0.3.2-cp313-cp313t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fad3b2a085ec259ad2c2842666b2a0a49dea8463579c606426128925af1ed725", size = 265896, upload-time = "2025-06-09T22:55:26.049Z" }, - { url = "https://files.pythonhosted.org/packages/2e/a8/2aa6716ffa566ca57c749edb909ad27884680887d68517e4be41b02299f3/propcache-0.3.2-cp313-cp313t-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:261fa020c1c14deafd54c76b014956e2f86991af198c51139faf41c4d5e83892", size = 252111, upload-time = "2025-06-09T22:55:27.381Z" }, - { url = "https://files.pythonhosted.org/packages/36/4f/345ca9183b85ac29c8694b0941f7484bf419c7f0fea2d1e386b4f7893eed/propcache-0.3.2-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:46d7f8aa79c927e5f987ee3a80205c987717d3659f035c85cf0c3680526bdb44", size = 268334, upload-time = "2025-06-09T22:55:28.747Z" }, - { url = "https://files.pythonhosted.org/packages/3e/ca/fcd54f78b59e3f97b3b9715501e3147f5340167733d27db423aa321e7148/propcache-0.3.2-cp313-cp313t-musllinux_1_2_armv7l.whl", hash = "sha256:6d8f3f0eebf73e3c0ff0e7853f68be638b4043c65a70517bb575eff54edd8dbe", size = 255026, upload-time = "2025-06-09T22:55:30.184Z" }, - { url = "https://files.pythonhosted.org/packages/8b/95/8e6a6bbbd78ac89c30c225210a5c687790e532ba4088afb8c0445b77ef37/propcache-0.3.2-cp313-cp313t-musllinux_1_2_i686.whl", hash = "sha256:03c89c1b14a5452cf15403e291c0ccd7751d5b9736ecb2c5bab977ad6c5bcd81", size = 250724, upload-time = "2025-06-09T22:55:31.646Z" }, - { url = "https://files.pythonhosted.org/packages/ee/b0/0dd03616142baba28e8b2d14ce5df6631b4673850a3d4f9c0f9dd714a404/propcache-0.3.2-cp313-cp313t-musllinux_1_2_ppc64le.whl", hash = "sha256:0cc17efde71e12bbaad086d679ce575268d70bc123a5a71ea7ad76f70ba30bba", size = 268868, upload-time = "2025-06-09T22:55:33.209Z" }, - { url = "https://files.pythonhosted.org/packages/c5/98/2c12407a7e4fbacd94ddd32f3b1e3d5231e77c30ef7162b12a60e2dd5ce3/propcache-0.3.2-cp313-cp313t-musllinux_1_2_s390x.whl", hash = "sha256:acdf05d00696bc0447e278bb53cb04ca72354e562cf88ea6f9107df8e7fd9770", size = 271322, upload-time = "2025-06-09T22:55:35.065Z" }, - { url = "https://files.pythonhosted.org/packages/35/91/9cb56efbb428b006bb85db28591e40b7736847b8331d43fe335acf95f6c8/propcache-0.3.2-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:4445542398bd0b5d32df908031cb1b30d43ac848e20470a878b770ec2dcc6330", size = 265778, upload-time = "2025-06-09T22:55:36.45Z" }, - { url = "https://files.pythonhosted.org/packages/9a/4c/b0fe775a2bdd01e176b14b574be679d84fc83958335790f7c9a686c1f468/propcache-0.3.2-cp313-cp313t-win32.whl", hash = "sha256:f86e5d7cd03afb3a1db8e9f9f6eff15794e79e791350ac48a8c924e6f439f394", size = 41175, upload-time = "2025-06-09T22:55:38.436Z" }, - { url = "https://files.pythonhosted.org/packages/a4/ff/47f08595e3d9b5e149c150f88d9714574f1a7cbd89fe2817158a952674bf/propcache-0.3.2-cp313-cp313t-win_amd64.whl", hash = "sha256:9704bedf6e7cbe3c65eca4379a9b53ee6a83749f047808cbb5044d40d7d72198", size = 44857, upload-time = "2025-06-09T22:55:39.687Z" }, - { url = "https://files.pythonhosted.org/packages/cc/35/cc0aaecf278bb4575b8555f2b137de5ab821595ddae9da9d3cd1da4072c7/propcache-0.3.2-py3-none-any.whl", hash = "sha256:98f1ec44fb675f5052cccc8e609c46ed23a35a1cfd18545ad4e29002d858a43f", size = 12663, upload-time = "2025-06-09T22:56:04.484Z" }, -] - -[[package]] -name = "protobuf" -version = "6.32.1" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/fa/a4/cc17347aa2897568beece2e674674359f911d6fe21b0b8d6268cd42727ac/protobuf-6.32.1.tar.gz", hash = "sha256:ee2469e4a021474ab9baafea6cd070e5bf27c7d29433504ddea1a4ee5850f68d", size = 440635, upload-time = "2025-09-11T21:38:42.935Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/3f/be/8dd0a927c559b37d7a6c8ab79034fd167dcc1f851595f2e641ad62be8643/protobuf-6.32.1-cp39-abi3-manylinux2014_aarch64.whl", hash = "sha256:2f5b80a49e1eb7b86d85fcd23fe92df154b9730a725c3b38c4e43b9d77018bf4", size = 322874, upload-time = "2025-09-11T21:38:35.509Z" }, - { url = "https://files.pythonhosted.org/packages/5c/f6/88d77011b605ef979aace37b7703e4eefad066f7e84d935e5a696515c2dd/protobuf-6.32.1-cp39-abi3-manylinux2014_x86_64.whl", hash = "sha256:b1864818300c297265c83a4982fd3169f97122c299f56a56e2445c3698d34710", size = 322013, upload-time = "2025-09-11T21:38:37.017Z" }, - { url = "https://files.pythonhosted.org/packages/97/b7/15cc7d93443d6c6a84626ae3258a91f4c6ac8c0edd5df35ea7658f71b79c/protobuf-6.32.1-py3-none-any.whl", hash = "sha256:2601b779fc7d32a866c6b4404f9d42a3f67c5b9f3f15b4db3cccabe06b95c346", size = 169289, upload-time = "2025-09-11T21:38:41.234Z" }, +version = "0.5.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/ec/44/c87281c333769159c50594f22610f77398a47ccbfbbf23074e744e86f87c/propcache-0.5.2.tar.gz", hash = "sha256:01c4fc7480cd0598bb4b57022df55b9ca296da7fc5a8760bd8451a7e63a7d427", size = 50208, upload-time = "2026-05-08T21:02:12.199Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/5b/56/030b7b4719d53085722893e0009dffb9236aa10bca1b12121bdc5626ef16/propcache-0.5.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:d5a81be28596d6559f6131ef33e10200de6e17643b3c74ce03f9eb103be6ae8b", size = 93417, upload-time = "2026-05-08T20:59:15.597Z" }, + { url = "https://files.pythonhosted.org/packages/1a/55/1140a8e067b8ec093a18a4ae7bb0045d9db65da38a08618ddc5e2f1994aa/propcache-0.5.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:29cbaac5ea0212663e6845e04b5e188d5a6ae6dd919810ac835bf1d3b42c3f4c", size = 53847, upload-time = "2026-05-08T20:59:17.096Z" }, + { url = "https://files.pythonhosted.org/packages/20/42/0e7443c90310498561addf346e7d57fe3c6ba1914e1ba938b5464c7bbfd2/propcache-0.5.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:6bf3be92233808fcd338eba0fb4d0b59ec5772af4f4ecfcec450d1bfc0f8b5eb", size = 53512, upload-time = "2026-05-08T20:59:18.64Z" }, + { url = "https://files.pythonhosted.org/packages/b7/db/cf51a71bab2009517d1a7f0ee07657e3bd446c4d69f67e6966cf17bcf956/propcache-0.5.2-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:2f8ea531c794b9d6274acd4e8d2c2ebcac590a4361d27482edd3010b79f1325e", size = 58068, upload-time = "2026-05-08T20:59:20.683Z" }, + { url = "https://files.pythonhosted.org/packages/b7/43/39b6bdee9699fa1e1641c519feeb64a67e2a9f93bb465c70776b37a7333f/propcache-0.5.2-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:decfca4c79dd53ebab484b00cc4b6717d8c369f86e74aa4ca395a64ac651495e", size = 61020, upload-time = "2026-05-08T20:59:22.112Z" }, + { url = "https://files.pythonhosted.org/packages/26/0b/843726fbb0a29a8c5684fdb25971823638399f31e52e9d1f06a02dc9aa6b/propcache-0.5.2-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:4621064bbf28fa77ff64dd5d94367c04684c67d3a5bf1dff25f0cd0d98a38f3b", size = 62732, upload-time = "2026-05-08T20:59:23.805Z" }, + { url = "https://files.pythonhosted.org/packages/39/6e/899fed76dc1942b8a64193a4f059d7f1a2c7ef65085e8a9366ed8ec0d199/propcache-0.5.2-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b96db7141a592cbc968daf1feea83a118e6ab378af4abbc72b248c895414c22d", size = 60140, upload-time = "2026-05-08T20:59:25.389Z" }, + { url = "https://files.pythonhosted.org/packages/ab/09/3da4be9b5b879219ad234aa535b3dd4a080ed1ad48d3a73ca07a9e798f22/propcache-0.5.2-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:1ca071adabaab6e9219924bbe00af821f1ee7de113a9eca1cdc292de3d120f4d", size = 60400, upload-time = "2026-05-08T20:59:27.238Z" }, + { url = "https://files.pythonhosted.org/packages/60/2f/09b72b874a9aa0044faf52a69807a6ed618e267ceaa9ec4a63195fa5b504/propcache-0.5.2-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:e4294d04a94dcab1b3bccd8b66d962dcad411a1d19414b2a41d1445f1de32ad0", size = 58155, upload-time = "2026-05-08T20:59:28.48Z" }, + { url = "https://files.pythonhosted.org/packages/8a/37/97489848c54c95578045473954f10956d619ce6a09e7ac137b71cdcb698b/propcache-0.5.2-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:a0e399a2eccb91ed18721f86aa85757727400b6865c89e88934781deb9c8498b", size = 57037, upload-time = "2026-05-08T20:59:30.146Z" }, + { url = "https://files.pythonhosted.org/packages/22/db/6c695285ccfc49012743ee9c98212b8c5dd0aed7b63cfd816d4a0f7a1601/propcache-0.5.2-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:823581fd5cb08b12a48bfa11fe962a7916766b6170c17b028fbdf762b85eb9bf", size = 61103, upload-time = "2026-05-08T20:59:31.626Z" }, + { url = "https://files.pythonhosted.org/packages/98/a9/1e500401ca593b0bdb6bf75a70bc2d723835fd53360edff6af70692c7546/propcache-0.5.2-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:949c91d1a990cf3b2e8188dfcfb25005e0b834a06c63fa4ef9f360878ce21ecf", size = 60394, upload-time = "2026-05-08T20:59:32.829Z" }, + { url = "https://files.pythonhosted.org/packages/1f/87/f638b6e375eae0f30a1a2325d8b34fd85fdc785bb9960cf805f3bf1ec69a/propcache-0.5.2-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:cc1177027eda740fdb152706bd215a3f124e3eea15afc39f2cb9fe351b50619e", size = 63084, upload-time = "2026-05-08T20:59:35.964Z" }, + { url = "https://files.pythonhosted.org/packages/f6/18/884573f5d97b6d9eba68de759a82c901b7e39d7904d30f7b8d58d42d2a12/propcache-0.5.2-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:b05d643f944a8c3c4bd86d65ffd87bf3264b617f87791940302bc474d2ff5274", size = 60999, upload-time = "2026-05-08T20:59:38.481Z" }, + { url = "https://files.pythonhosted.org/packages/8f/1a/c3915eb059ceec9e758a56e4cfd955292bc0f201be2176a46b76d94b303a/propcache-0.5.2-cp310-cp310-win32.whl", hash = "sha256:8114f28879e0904748e831c3a7774261bd9e75f49be089f389a76f959dcd13fe", size = 39036, upload-time = "2026-05-08T20:59:40.323Z" }, + { url = "https://files.pythonhosted.org/packages/5b/02/1dfd5607501a602d19c1c449d2d193b7d1c611f9246b4059026a1189a80e/propcache-0.5.2-cp310-cp310-win_amd64.whl", hash = "sha256:5fcb98e7598b1ee0addab320d90f65b530297a867dbfe9de52ea838077e16e3d", size = 42190, upload-time = "2026-05-08T20:59:42.232Z" }, + { url = "https://files.pythonhosted.org/packages/57/93/f71588ad08b3e6f4b555b5ef215808a3c02b042d0151ad82fa6f15be677a/propcache-0.5.2-cp310-cp310-win_arm64.whl", hash = "sha256:04dc2390d9edbbaef7461f33322555976ffddf0b650a038649d026358714e6c5", size = 38545, upload-time = "2026-05-08T20:59:44.087Z" }, + { url = "https://files.pythonhosted.org/packages/e7/f1/8a8cc1c2c7e7934ab77e0163414f736fadbc0f5e8dd9673b952355ac175b/propcache-0.5.2-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:74b70780220e2dd89175ca24b81b68b67c83db499ae611e7f2313cb329801c78", size = 90744, upload-time = "2026-05-08T20:59:45.799Z" }, + { url = "https://files.pythonhosted.org/packages/c2/f4/651b1225e976bd1a2ba5cfba0c29d096581c2636b437e3a9a7ab6276270a/propcache-0.5.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:a4840ab0ae0216d952f4b53dc6d0b992bfc2bedbfe360bdd9b548bc184c08959", size = 52033, upload-time = "2026-05-08T20:59:47.408Z" }, + { url = "https://files.pythonhosted.org/packages/15/a8/8ede85d6aa1f79fc7dc2f8fd2c8d65920b8272c3892903c8a1affde48cfb/propcache-0.5.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:c6844ba6364fb12f403928a82cfd295ab103a2b315c77c747b2dbe4a41894ea7", size = 52754, upload-time = "2026-05-08T20:59:49.202Z" }, + { url = "https://files.pythonhosted.org/packages/7d/fe/b3551b41bbc2f5b5bb088fc6920567cd43101253e68fbaa261339eb96fe1/propcache-0.5.2-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:2293949b855ce597f2826452d17c2d545fb5622379c4ea6fdf525e9b8e8a2511", size = 57573, upload-time = "2026-05-08T20:59:50.778Z" }, + { url = "https://files.pythonhosted.org/packages/83/27/ab851ebd1b7172e3e161f5f8d39e315d54a91bea246f01f4d872d3376aef/propcache-0.5.2-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:0fd59b5af35f74da48d905dcbad55449ba13be91823cb05a9bd590bbf5b61660", size = 60645, upload-time = "2026-05-08T20:59:52.227Z" }, + { url = "https://files.pythonhosted.org/packages/95/7d/466b3d18022e9897cbda9c735c493c5bd747d7a4c6f5ea1480b4cec434b6/propcache-0.5.2-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:29f9309a2e42b0d273be006fdb4be2d6c39a47f6f57d8fb1cf9f81481df81b66", size = 61563, upload-time = "2026-05-08T20:59:53.866Z" }, + { url = "https://files.pythonhosted.org/packages/27/1b/16ab7f2cf2041da2f60d156ba64c2484eadf9168075b4ff43c3ef60045af/propcache-0.5.2-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:5aaa2b923c1944ac8febd6609cb373540a5563e7cbcb0fd770f75dace2eb817b", size = 58888, upload-time = "2026-05-08T20:59:55.457Z" }, + { url = "https://files.pythonhosted.org/packages/0a/67/bb777ffd907633563bf35fd859c4ce97b0512c32f4633cf5d1eb7c33512b/propcache-0.5.2-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:66ea454f095ddf5b6b14f56c064c0941c4788be11e18d2464cf643bf7203ff67", size = 59253, upload-time = "2026-05-08T20:59:57.075Z" }, + { url = "https://files.pythonhosted.org/packages/b9/42/64f8d90b73fd9cdc1499b48057ff6d9cd2a98a25734c9bb62ecf07e87061/propcache-0.5.2-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:95f1e3f4760d404b13c9976c0229b2b49a3c8e2c62a9ce92efdd2b11ada75e3f", size = 57558, upload-time = "2026-05-08T20:59:58.602Z" }, + { url = "https://files.pythonhosted.org/packages/eb/02/dba5bc03c9041f2092ea55a449caf5dfe68352c6654511b29ba0654ddb69/propcache-0.5.2-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:85341b12b9d55bad0bded24cac341bb34289469e03a11f3f583ea1cc1db0326c", size = 55007, upload-time = "2026-05-08T20:59:59.837Z" }, + { url = "https://files.pythonhosted.org/packages/14/c0/43f649c7aa2a77a3b100d84e9dea3a483120ecb608bfe36ce49eaff517fe/propcache-0.5.2-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:26a4dca084132874e639895c3135dfad5eb20bae209f62d1aeb31b03e601c3c0", size = 60355, upload-time = "2026-05-08T21:00:01.144Z" }, + { url = "https://files.pythonhosted.org/packages/83/c0/435dafd27f1cb4a495381dae60e25883ccfe4020bb72818e8184c1678092/propcache-0.5.2-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:3b199b9b2b3d6a7edf3183ba8a9a137a22b97f7df525feb5ae1eccf026d2a9c6", size = 59057, upload-time = "2026-05-08T21:00:02.401Z" }, + { url = "https://files.pythonhosted.org/packages/53/ae/6e292df9135d659944e96cb3389258e4a663e5b2b5f6c217ef0ddc8d2f73/propcache-0.5.2-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:e59bc9e66329185b93dab73f210f1a37f81cb40f321501db8017c9aea15dba27", size = 61938, upload-time = "2026-05-08T21:00:03.638Z" }, + { url = "https://files.pythonhosted.org/packages/0b/42/314ebc50d8159055411fd6b0bda322ff510e4b1f7d2e4927940ad0f6af20/propcache-0.5.2-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:552ffadf6ad409844bc5919c42a0a83d88314cedddaea0e41e80a8b8fffe881f", size = 59731, upload-time = "2026-05-08T21:00:04.881Z" }, + { url = "https://files.pythonhosted.org/packages/b8/9b/2da6dee38871c3c8772fabc2758325a5c9077d6d18c597737dc04dd884cd/propcache-0.5.2-cp311-cp311-win32.whl", hash = "sha256:cd416c1de191973c52ff1a12a57446bfc7642797b282d7caf2162d7d1b8aa9a0", size = 38966, upload-time = "2026-05-08T21:00:06.511Z" }, + { url = "https://files.pythonhosted.org/packages/42/4e/f17363fb58c0afe05b067361cb6d86ed2d29de6506779a27547c4d183075/propcache-0.5.2-cp311-cp311-win_amd64.whl", hash = "sha256:44e488ef40dbb452700b2b1f8188934121f6648f52c295055662d2191959ff82", size = 42135, upload-time = "2026-05-08T21:00:08.088Z" }, + { url = "https://files.pythonhosted.org/packages/c6/eb/6af6685077d22e8b33358d3c548e3282706a0b3cd85044ffba4e5dd08e3b/propcache-0.5.2-cp311-cp311-win_arm64.whl", hash = "sha256:54adaa85a22078d1e306304a40984dc5be99d599bf3dc0a24dc98f7daeab89ab", size = 38381, upload-time = "2026-05-08T21:00:09.692Z" }, + { url = "https://files.pythonhosted.org/packages/4a/cb/e27bc2b2737a0bb49962b275efa051e8f1c35a936df7d5139b6b658b7dc9/propcache-0.5.2-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:806719138ecd720339a12410fb9614ac9b2b2d3a5fdf8235d56981c36f4039ba", size = 95887, upload-time = "2026-05-08T21:00:11.277Z" }, + { url = "https://files.pythonhosted.org/packages/e6/13/b8ae04c59392f8d11c6cd9fb4011d1dc7c86b81225c770280300e259ffe1/propcache-0.5.2-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:db2b80ea58eab4f86b2beec3cc8b39e8ff9276ac20e96b7cce43c8ae84cd6b5a", size = 54654, upload-time = "2026-05-08T21:00:12.604Z" }, + { url = "https://files.pythonhosted.org/packages/2c/7d/49777a3e20b55863d4794384a38acd460c04157b0a00f8602b0d508b8431/propcache-0.5.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:e5cbfac9f61484f7e9f3597775500cd3ebe8274e9b050c38f9525c77c97520bf", size = 55190, upload-time = "2026-05-08T21:00:13.935Z" }, + { url = "https://files.pythonhosted.org/packages/44/c7/085d0cd63062e84044e3f05797749c3f8e3938ff3aeb0eb2f69d43fafc91/propcache-0.5.2-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5dbc581d2814337da56222fab8dc5f161cd798a434e49bac27930aaef798e144", size = 59995, upload-time = "2026-05-08T21:00:15.526Z" }, + { url = "https://files.pythonhosted.org/packages/9c/42/32cf8e3009e92b2645cf1e944f701e8ea4e924dffde1ee26db860bcbf7e4/propcache-0.5.2-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:857187f381f88c8e2fa2fe56ab94879d011b883d5a2ee5a1b60a8cd2a06846d9", size = 63422, upload-time = "2026-05-08T21:00:16.824Z" }, + { url = "https://files.pythonhosted.org/packages/9e/1b/f112433f99fc979431b87a39ef169e3f8df070d99a72792c56d6937ac48b/propcache-0.5.2-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:178b4a2cdaac1818e2bf1c5a99b94383fa73ea5382e032a48dec07dc5668dc42", size = 64342, upload-time = "2026-05-08T21:00:18.362Z" }, + { url = "https://files.pythonhosted.org/packages/14/15/5574111ae50dd6e879456888c0eadd4c5a869959775854e18e18a6b345f3/propcache-0.5.2-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:6f328175a2cde1f0ff2c4ed8ce968b9dcfb55f3a7153f39e2957ed994da13476", size = 61639, upload-time = "2026-05-08T21:00:19.692Z" }, + { url = "https://files.pythonhosted.org/packages/cc/da/4d775080b1490c0ae604acda868bd71aabe3a89ed16f2aa4339eb8a283e7/propcache-0.5.2-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:5671d09a36b06d0fd4a3da0fccbcae360e9b1570924171a15e9e0997f0249fba", size = 61588, upload-time = "2026-05-08T21:00:21.155Z" }, + { url = "https://files.pythonhosted.org/packages/04/ac/f076982cbe2195ee9cf32de5a1e46951d9fb399fc207f390562dd0fd8fb2/propcache-0.5.2-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:80168e2ebe4d3ec6599d10ad8f520304ae1cad9b6c5a95372aef1b66b7bfb53a", size = 60029, upload-time = "2026-05-08T21:00:22.713Z" }, + { url = "https://files.pythonhosted.org/packages/70/60/189be62e0dd898dce3b331e1b8c7a543cd3a405ac0c81fe8ee8a9d5d77e1/propcache-0.5.2-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:45f11346f884bc47444f6e6647131055844134c3175b629f84952e2b5cd62b64", size = 56774, upload-time = "2026-05-08T21:00:24.001Z" }, + { url = "https://files.pythonhosted.org/packages/ea/9e/93377b9c7939c1ffae98f878dee955efadfd638078bc86dbc21f9d52f651/propcache-0.5.2-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:8e778ebd44ef4f66ed60a0416b06b489687db264a9c0b3620362f26489492913", size = 63532, upload-time = "2026-05-08T21:00:25.545Z" }, + { url = "https://files.pythonhosted.org/packages/14/f9/590ef6cfb9b8028d516d287812ece32bb0bc5f11fbb9c8bf6b2e6313fec8/propcache-0.5.2-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:c0cb9ed24c8964e172768d455a38254c2dd8a552905729ce006cad3d3dda59b1", size = 61592, upload-time = "2026-05-08T21:00:27.186Z" }, + { url = "https://files.pythonhosted.org/packages/b4/5e/70958b3034c297a630bba2f17ca7abc2d5f39a803ad7e370ab79d1ecd022/propcache-0.5.2-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:1d1ad32d9d4355e2be65574fd0bfd3677e7066b009cd5b9b2dee8aa6a6393b33", size = 64788, upload-time = "2026-05-08T21:00:28.8Z" }, + { url = "https://files.pythonhosted.org/packages/12/fd/77fe5936d8c3086ca9048f7f415f122ed82e53884a9ec193646b42deef06/propcache-0.5.2-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:c80f4ba3e8f00189165999a742ee526ebeccedf6c3f7beb0c7df821e9772435a", size = 62514, upload-time = "2026-05-08T21:00:30.098Z" }, + { url = "https://files.pythonhosted.org/packages/cf/74/66bd798b5b3be70aa1b391f5cc9d6a0a5532d7fd3b19ec0b213e72e6ad9d/propcache-0.5.2-cp312-cp312-win32.whl", hash = "sha256:8c7972d8f193740d9175f0998ab38717e6cd322d5935c5b0fef8c0d323fd9031", size = 39018, upload-time = "2026-05-08T21:00:31.622Z" }, + { url = "https://files.pythonhosted.org/packages/61/7c/5c0d34aa3024694d6dcb9271cdbdd08c4e47c1c0ad95ec7e7bc74cdea145/propcache-0.5.2-cp312-cp312-win_amd64.whl", hash = "sha256:d9ee8826a7d47863a08ac44e1a5f611a462eefc3a194b492da242128bec75b42", size = 42322, upload-time = "2026-05-08T21:00:32.918Z" }, + { url = "https://files.pythonhosted.org/packages/4d/91/875812f1a3feb20ceba818ef39fbe4d92f1081e04ac815c822496d0d038b/propcache-0.5.2-cp312-cp312-win_arm64.whl", hash = "sha256:2800a4a8ead6b28cccd1ec54b59346f0def7922ee1c7598e8499c733cfbb7c84", size = 38172, upload-time = "2026-05-08T21:00:35.124Z" }, + { url = "https://files.pythonhosted.org/packages/c5/09/f049e45385503fe67db75a6b6186a7b9f0c3930366dc960522c312a825b1/propcache-0.5.2-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:099aaf4b4d1a02265b92a977edf00b5c4f63b3b17ac6de39b0d637c9cac0188a", size = 94457, upload-time = "2026-05-08T21:00:36.355Z" }, + { url = "https://files.pythonhosted.org/packages/6b/65/83d1d05655baf63113731bd5a1008435e14f8d1e5a06cbe4ec5b23ad7a31/propcache-0.5.2-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:68ce1c44c7a813a7f71ea04315a8c7b330b63db99d059a797a4651bb6f69f117", size = 53835, upload-time = "2026-05-08T21:00:38.072Z" }, + { url = "https://files.pythonhosted.org/packages/a9/12/a6ba6482bb5ea3260c000c9b20881c95fa11c6b30173715668259f844ed7/propcache-0.5.2-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:fc299c129490f55f254cd90be0deca4764e36e9a7c08b4aa588479a3bbed3098", size = 54545, upload-time = "2026-05-08T21:00:39.319Z" }, + { url = "https://files.pythonhosted.org/packages/a9/19/7fa086f5764c59ec8a8e157cd93aa8497acc00aba9dcdec56bfffb32602d/propcache-0.5.2-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a6ae2198be502c10f09b2516e7b5d019816924bc3183a43ce792a7bd6625e6f4", size = 59886, upload-time = "2026-05-08T21:00:40.621Z" }, + { url = "https://files.pythonhosted.org/packages/a1/e4/5d7663dc8235956c8f5281698a3af1d351d8820341ddd890f59d9a9127f2/propcache-0.5.2-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:6041d31504dc1779d700e1edcfb08eea334b357620b06681a4eabb57a74e574e", size = 63261, upload-time = "2026-05-08T21:00:41.775Z" }, + { url = "https://files.pythonhosted.org/packages/4a/4a/15a03adee24d6350da4292caeac44c34c033d2afe5e87eb370f38854560f/propcache-0.5.2-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:f7eabc04151c78a9f4d5bbb5f1faf571e4defeb4b585e0fe95b60ff2dbe4d3d7", size = 64184, upload-time = "2026-05-08T21:00:43.018Z" }, + { url = "https://files.pythonhosted.org/packages/8b/c6/979176efdaa3d239e36d503d5af63a0a773b36662ed8f52e5b6a6d9fd40e/propcache-0.5.2-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:4db0ba63d693afd40d249bd93f842b5f144f8fcbb83de05660373bcf30517b1d", size = 61534, upload-time = "2026-05-08T21:00:44.507Z" }, + { url = "https://files.pythonhosted.org/packages/c8/22/63e8cd1bae4c2d2be6493b6b7d10566ddafad88137cfbc99964a1119853c/propcache-0.5.2-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:1dbcf7675229b35d31abb6547d8ebc8c27a830ac3f9a794edff6254873ec7c0a", size = 61500, upload-time = "2026-05-08T21:00:45.796Z" }, + { url = "https://files.pythonhosted.org/packages/60/5a/28e5d9acbac1cc9ccb67045e8c1b943aa8d79fdf39c93bd73cacd68008ea/propcache-0.5.2-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:d310c013aad2c72f1c3f2f8dd3279d460a858c551f97aeb8c63e4693cca7b4d2", size = 59994, upload-time = "2026-05-08T21:00:47.093Z" }, + { url = "https://files.pythonhosted.org/packages/f3/40/db650677f554a95b9c01a7c9d93d629e93a15562f5deb4573c9ee136fed2/propcache-0.5.2-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:06187263ddad280d05b4d8a8b3bb7d164cbebd469236544a42e6d9b28ac6a4fa", size = 56884, upload-time = "2026-05-08T21:00:48.376Z" }, + { url = "https://files.pythonhosted.org/packages/80/45/70b39b89516ff8b96bf732fa6fded8cef20f293cb1508690101c3c07ec51/propcache-0.5.2-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:3115559b8effafd63b142ea5ed53d63a16ea6469cbc63dce4ee194b42db5d853", size = 63464, upload-time = "2026-05-08T21:00:49.954Z" }, + { url = "https://files.pythonhosted.org/packages/f9/e2/fa59d3a89eac5534293124af4f1d0d0ada091ce4a0ab4610ce03fd2bdd8d/propcache-0.5.2-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:c60462af8e6dc30c35407c7237ea908d777b22862bbee27bc4699c0d8bcdc45a", size = 61588, upload-time = "2026-05-08T21:00:51.281Z" }, + { url = "https://files.pythonhosted.org/packages/0b/97/efb547a55c4bc7381cfb202d6a2239ac621045277bc1ea5dfd3a7f0516c0/propcache-0.5.2-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:40314bca9ac559716fe374094fc81c11dcc34b64fd6c585360f5775690505704", size = 64667, upload-time = "2026-05-08T21:00:52.602Z" }, + { url = "https://files.pythonhosted.org/packages/92/56/f5c7d9b4b7595d5127da38974d791b2153f3d1eae6c674af3583ace92ad3/propcache-0.5.2-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:cfa21e036ce1e1db2be04ba3b85d2df1bb1702fa01932d984c5464c665228ff4", size = 62463, upload-time = "2026-05-08T21:00:54.303Z" }, + { url = "https://files.pythonhosted.org/packages/bd/3b/484a3a65fc9f9f60c41dcd17b428bace5389544e2c680994534a20755066/propcache-0.5.2-cp313-cp313-win32.whl", hash = "sha256:f156a3529f38063b6dbaf356e15602a7f95f8055b1295a438433a6386f10463d", size = 38621, upload-time = "2026-05-08T21:00:55.808Z" }, + { url = "https://files.pythonhosted.org/packages/1c/fd/3f0f10dba4dabad3bf53102be007abf55481067952bde0fdddff439e7c61/propcache-0.5.2-cp313-cp313-win_amd64.whl", hash = "sha256:dfed59d0a5aeb01e242e66ff0300bc4a265a7c05f612d30016f0b60b1017d757", size = 41649, upload-time = "2026-05-08T21:00:57.061Z" }, + { url = "https://files.pythonhosted.org/packages/90/ec/6ce619cc32bb500a482f811f9cd509368b4e58e638d13f2c68f370d6b475/propcache-0.5.2-cp313-cp313-win_arm64.whl", hash = "sha256:ba338430e87ceb9c8f0cf754de38a9860560261e56c00376debd628698a7364f", size = 37636, upload-time = "2026-05-08T21:00:58.646Z" }, + { url = "https://files.pythonhosted.org/packages/1b/82/c1d268bbbf2ef981c5bf0fbbe746db617c66e3bcefe431a1aa8943fbe23a/propcache-0.5.2-cp313-cp313t-macosx_10_13_universal2.whl", hash = "sha256:a592f5f3da71c8691c788c13cb6734b6d17663d2e1cb8caddf0673d01ef8847d", size = 98872, upload-time = "2026-05-08T21:00:59.889Z" }, + { url = "https://files.pythonhosted.org/packages/f4/d4/52c871e73e864e6b34c0e2d58ac1ec5ccd149497ddc7ad2137ae98323a35/propcache-0.5.2-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:6a997d0489e9668a384fcfd5061b857aa5361de73191cac204d04b889cfbbafa", size = 56257, upload-time = "2026-05-08T21:01:01.195Z" }, + { url = "https://files.pythonhosted.org/packages/67/f0/9b90ca2a210b3d09bcfcd96ecd0f55545c091535abce2a45de2775cfd357/propcache-0.5.2-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:10734b5484ea113152ee25a91dccedf81631791805d2c9ccb054958e51842c94", size = 56696, upload-time = "2026-05-08T21:01:02.941Z" }, + { url = "https://files.pythonhosted.org/packages/9d/0e/6e9d4ba07c8e56e21ddec1e75f12148142b21ca83a51871babce095334f4/propcache-0.5.2-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:cafca7e56c12bb02ae16d283742bef25a61122e9dab2b5b3f2ccbe589ce32164", size = 62378, upload-time = "2026-05-08T21:01:04.475Z" }, + { url = "https://files.pythonhosted.org/packages/65/19/c10badaa463dde8a27ce884f8ee2ec37e6035b7c9f5ff0c8f74f06f08dac/propcache-0.5.2-cp313-cp313t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:f064f8d2b59177878b7615df1735cd8fe3462ed6be8c7b217d17a276489c2b7f", size = 65283, upload-time = "2026-05-08T21:01:05.959Z" }, + { url = "https://files.pythonhosted.org/packages/b0/b6/93bea99ca80e19cef6512a8580e5b7857bbe09422d9daa7fd4ef5723306c/propcache-0.5.2-cp313-cp313t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:f78abfa8dfc32376fd1aacf597b2f2fbbe0ea751419aee718af5d4f82537ef8c", size = 66616, upload-time = "2026-05-08T21:01:07.228Z" }, + { url = "https://files.pythonhosted.org/packages/83/e4/5c7462e50625f051f37fb38b8224f7639f667184bbd34424ec83819bb1b7/propcache-0.5.2-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f7467da8a9822bf1a55336f877340c5bcbd3c482afc43a99771169f74a26dedc", size = 63773, upload-time = "2026-05-08T21:01:08.514Z" }, + { url = "https://files.pythonhosted.org/packages/ca/b6/99238894047b13c823be25027e736626cd414a52a5e30d2c3347c2733529/propcache-0.5.2-cp313-cp313t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:a6ddc6ac9e25de626c1f129c1b467d7ecd33ce2237d3fd0c4e429feef0a7ee1f", size = 63664, upload-time = "2026-05-08T21:01:09.874Z" }, + { url = "https://files.pythonhosted.org/packages/85/1e/a3a1a63116a2b8edb415a8bb9a6f0c34bd03830b1e18e8ce2904e1dc1cf4/propcache-0.5.2-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:2f22cbbac9e26a8e864c0985ff1268d5d939d53d9d9411a9824279097e03a2cb", size = 62643, upload-time = "2026-05-08T21:01:11.132Z" }, + { url = "https://files.pythonhosted.org/packages/e4/03/893cf147de2fc6543c5eaa07ad833170e7e2a2385725bbebe8c0503723bb/propcache-0.5.2-cp313-cp313t-musllinux_1_2_armv7l.whl", hash = "sha256:fc76378c62a0f04d0cd82fbb1a2cd2d7e28fcb40d5873f28a6c44e388aaa2751", size = 59595, upload-time = "2026-05-08T21:01:12.387Z" }, + { url = "https://files.pythonhosted.org/packages/86/3b/04c1a2e12c57766568ba75ba72b3bf2042818d4c1425fab6fc07155c7cff/propcache-0.5.2-cp313-cp313t-musllinux_1_2_ppc64le.whl", hash = "sha256:acd2c8edba48e31e58a363b8cf4e5c7db3b04b3f9e371f601df30d9b0d244836", size = 65711, upload-time = "2026-05-08T21:01:13.676Z" }, + { url = "https://files.pythonhosted.org/packages/1c/34/80f8d0099f8d6bacc4de1624c85672681c8cd1149ca2da0e38fd120b817f/propcache-0.5.2-cp313-cp313t-musllinux_1_2_riscv64.whl", hash = "sha256:452b5065457eb9991ec5eb38ff41d6cd4c991c9ac7c531c4d5849ae473a9a13f", size = 64247, upload-time = "2026-05-08T21:01:14.936Z" }, + { url = "https://files.pythonhosted.org/packages/f3/1a/8b08f3a5f1037e9e370c55883ceeeee0f6dd0416fb2d2d67b8bfc91f2a79/propcache-0.5.2-cp313-cp313t-musllinux_1_2_s390x.whl", hash = "sha256:3430bb2bfe1331885c427745a751e774ee679fd4344f80b97bf879815fe8fa55", size = 67102, upload-time = "2026-05-08T21:01:16.281Z" }, + { url = "https://files.pythonhosted.org/packages/34/68/8bdb7bb7756d76e005490649d10e4a8369e610c74d619f71e1aedf889e9c/propcache-0.5.2-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:cef6cea3922890dd6c9654971001fa797b526c16ab5e1e46c05fd6f877be7568", size = 64964, upload-time = "2026-05-08T21:01:17.57Z" }, + { url = "https://files.pythonhosted.org/packages/0a/aa/50fb0b5d3968b61a510926ff8b8465f1d6e976b3ab74496d7a4b9fc42515/propcache-0.5.2-cp313-cp313t-win32.whl", hash = "sha256:72d61e16dd78228b58c5d47be830ff3da7e5f139abdf0aef9d86cde1c5cf2191", size = 42546, upload-time = "2026-05-08T21:01:18.946Z" }, + { url = "https://files.pythonhosted.org/packages/ae/4c/0ddbae64321bd4a95bcbfc19307238016b5b1fee645c84626c8d539e5b74/propcache-0.5.2-cp313-cp313t-win_amd64.whl", hash = "sha256:0958834041a0166d343b8d2cedcd8bcbaeb4fdbe0cf08320c5379f143c3be6e7", size = 46330, upload-time = "2026-05-08T21:01:20.162Z" }, + { url = "https://files.pythonhosted.org/packages/00/d9/9cddc8efb78d8af264c5ec9f6d10b62f57c515feda8d321595f56010fb23/propcache-0.5.2-cp313-cp313t-win_arm64.whl", hash = "sha256:6de8bd93ddde9b992cf2b2e0d796d501a19026b5b9fd87356d7d0779531a8d96", size = 40521, upload-time = "2026-05-08T21:01:21.399Z" }, + { url = "https://files.pythonhosted.org/packages/e2/ea/23ee535d90ce8bcc465a3028eb3cc0ce3bd1005f4bb27710b30587de798d/propcache-0.5.2-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:46088abff4cba581dea21ae0467a480526cb25aa5f3c269e909f800328bc3999", size = 94662, upload-time = "2026-05-08T21:01:22.683Z" }, + { url = "https://files.pythonhosted.org/packages/b5/06/c5a52f419b5d8972f8d46a7577476090d8e3263ff589ce40b5ca4968d5be/propcache-0.5.2-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:fc88b26f08d634f7bc819a7852e5214f5802641ab8d9fd5326892292eee1993e", size = 53928, upload-time = "2026-05-08T21:01:23.986Z" }, + { url = "https://files.pythonhosted.org/packages/63/b1/4260d67d6bd85e58a66b72d54ce15d5de789b6f3870cc6bedf8ff9667401/propcache-0.5.2-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:97797ebb098e670a2f92dd66f32897e30d7615b14e7f59711de23e30a9072539", size = 54650, upload-time = "2026-05-08T21:01:25.305Z" }, + { url = "https://files.pythonhosted.org/packages/70/06/2f46c318e3307cd7a6a7481def374ce838c0fe20084b39dd54b0879d0e99/propcache-0.5.2-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ba57fffe4ac99c5d30076161b5866336d97600769bad35cc68f7774b15298a4e", size = 59912, upload-time = "2026-05-08T21:01:26.545Z" }, + { url = "https://files.pythonhosted.org/packages/4c/29/fe1aebec2ce57ab985a9c382bded1124431f85078113aa222c5d278430d4/propcache-0.5.2-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:583c19759d9eec1e5b69e2fbef36a7d9c326041be9746cb822d335c8cedc2979", size = 63300, upload-time = "2026-05-08T21:01:27.937Z" }, + { url = "https://files.pythonhosted.org/packages/b4/18/2334b26768b6c82be8c69e83671b767d5ef426aa09b0cba6c2ea47816774/propcache-0.5.2-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:d0326e2e5e1f3163fa306c834e48e8d490e5fae607a097a40c0648109b47ba80", size = 64208, upload-time = "2026-05-08T21:01:29.484Z" }, + { url = "https://files.pythonhosted.org/packages/2b/76/7f1bfd6afff4c5e38e36a3c6d68eb5f4b7311ea80baf693db78d95b603c4/propcache-0.5.2-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e00820e192c8dbebcafb383ebbf99030895f09905e7a0eb2e0340a0bcc2bc825", size = 61633, upload-time = "2026-05-08T21:01:31.068Z" }, + { url = "https://files.pythonhosted.org/packages/c4/46/b3ff8aba2b4953a3e50de2cf72f1b5748b8eca93b15f3dc2c84339084c09/propcache-0.5.2-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:c66afea89b1e43725731d2004732a046fe6fe955d51f952c3e95a7314a284a39", size = 61724, upload-time = "2026-05-08T21:01:32.374Z" }, + { url = "https://files.pythonhosted.org/packages/c5/01/814cfcafbcff954f94c01cf30e097ddc88a076b5440fbcf4570753437d40/propcache-0.5.2-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:d4dc37dec6c6cdad0b57881a5658fd14fbf53e333b1a86cf86559f190e1d9ec4", size = 60069, upload-time = "2026-05-08T21:01:33.67Z" }, + { url = "https://files.pythonhosted.org/packages/da/68/5c6f7622d510cc666a300687e06fd060c1a43361c0c9b20d284f06d8096a/propcache-0.5.2-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:5570dbcc97571c15f68068e529c92715a12f8d54030e272d264b377e22bd17a5", size = 57099, upload-time = "2026-05-08T21:01:34.915Z" }, + { url = "https://files.pythonhosted.org/packages/55/27/9cb0b4c679124085327957d42521c99dba04c88c90c3e55a6f0b633ebccc/propcache-0.5.2-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:f814362777a9f841adddb200ecdf8f5cb1e5a3c4b7a86378edbd6ccb26edd702", size = 63391, upload-time = "2026-05-08T21:01:36.231Z" }, + { url = "https://files.pythonhosted.org/packages/f0/9d/7258aaa5bdf60fc6f27591eef6fe52768cb0beda7140be477c8b12c9794a/propcache-0.5.2-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:196913dea116aeb5a2ba95af4ddcb7ea85559ae07d8eee8751688310d09168c3", size = 61626, upload-time = "2026-05-08T21:01:37.545Z" }, + { url = "https://files.pythonhosted.org/packages/8e/0d/41c602003e8a9b16fe1e7eadf62c7bfba9d5474370b24200bf48b315f45f/propcache-0.5.2-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:6e7b8719005dd1175be4ab1cd25e9b98659a5e0347331506ec6760d2773a7fb5", size = 64781, upload-time = "2026-05-08T21:01:38.83Z" }, + { url = "https://files.pythonhosted.org/packages/8b/f3/38e66b1856e9bd079deea015bc4a55f7767c0e4db2f7dcf69e7e680ba4ce/propcache-0.5.2-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:51f96d685ab16e88cab128cd37a52c5da540809c8b879fa047731bfcb4ad35a4", size = 62570, upload-time = "2026-05-08T21:01:40.415Z" }, + { url = "https://files.pythonhosted.org/packages/95/ca/bbfe9b910ce57dde8bb4876b4520fc02a4e89497c10de26be936758a3aaa/propcache-0.5.2-cp314-cp314-win32.whl", hash = "sha256:cc6fc3cc62e8501d3ed62894425040d2728ecddb1ed072737a5c70bd537aa9f0", size = 39436, upload-time = "2026-05-08T21:01:41.654Z" }, + { url = "https://files.pythonhosted.org/packages/61/d2/45c9defbaa1ea297035d9d4cce9e8f80daafbf19319c6007f157c6256ea9/propcache-0.5.2-cp314-cp314-win_amd64.whl", hash = "sha256:81e3a30b0bb60caa22033dd0f8a3618d1d67356212514f62c57db75cb0ef410c", size = 42373, upload-time = "2026-05-08T21:01:43.041Z" }, + { url = "https://files.pythonhosted.org/packages/44/68/9ea5103f41d5217d7d6ec24db90018e23aebec070c3f9a6e54d12b841fd8/propcache-0.5.2-cp314-cp314-win_arm64.whl", hash = "sha256:0d2c9bf8528f135dbb805ce027567e09164f7efa51a2be07458a2c0420f292d0", size = 38554, upload-time = "2026-05-08T21:01:44.336Z" }, + { url = "https://files.pythonhosted.org/packages/8a/81/fadf555f42d3b762eea8a53950b0489fdc0aa9da5f8ed9e10ce0a4e01b48/propcache-0.5.2-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:4bc8ff1feffc6a61c7002ffe84634c41b822e104990ae009f44a0834430070bb", size = 99395, upload-time = "2026-05-08T21:01:45.883Z" }, + { url = "https://files.pythonhosted.org/packages/f5/c9/c61e134a686949cf7971af3a390148b1156f7be81c73bc0cd12c873e2d48/propcache-0.5.2-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:79aa3ff0a9b566633b642fa9caf7e21ed1c13d6feca718187873f199e1514078", size = 56653, upload-time = "2026-05-08T21:01:47.307Z" }, + { url = "https://files.pythonhosted.org/packages/cb/73/daf935ea7048ddd7ec8eec5345b4a40b619d2d178b3c0a0900796bc3c794/propcache-0.5.2-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:1b31822f4474c4036bae62de9402710051d431a606d6a0f907fec79935a071aa", size = 56914, upload-time = "2026-05-08T21:01:48.573Z" }, + { url = "https://files.pythonhosted.org/packages/79/9f/aba959b435ea18617edd7cf0a7ad0b9c574b8fc7e3d2cd55fb59cb255d33/propcache-0.5.2-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:13fef48778b5a2a756523fdb781326b028ca75e32858b04f2cdd19f394564917", size = 62567, upload-time = "2026-05-08T21:01:49.903Z" }, + { url = "https://files.pythonhosted.org/packages/6c/a1/859942de9a791ff42f6141736f5b37749b8f53e65edfa49638c67dd67e6a/propcache-0.5.2-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:8b73ab70f1a3351fbc71f663b3e645af6dd0329100c353081cf69c37433fc6fe", size = 65542, upload-time = "2026-05-08T21:01:51.204Z" }, + { url = "https://files.pythonhosted.org/packages/b5/61/315bc0fd6c0fc7f80a528b8afd209e5fc4a875ea79571b91b8f50f442907/propcache-0.5.2-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:5538d2c13d93e4698af7e092b57bc7298fd35d1d58e656ae18f23ee0d0378e03", size = 66845, upload-time = "2026-05-08T21:01:52.539Z" }, + { url = "https://files.pythonhosted.org/packages/47/f7/9f8122e3132e8e354ac41975ef8f1099be7d5a16bc7ae562734e993665c0/propcache-0.5.2-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:cd645f03898405cabe694fb8bc35241e3a9c332ec85627584fe3de201452b335", size = 63985, upload-time = "2026-05-08T21:01:53.847Z" }, + { url = "https://files.pythonhosted.org/packages/c8/54/c317819ec157cbf6f35df9df9657a6f82daf34d5faf15948b2f639c2192e/propcache-0.5.2-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:a473b3440261e0c60706e732b2ed2f517857344fc21bf48fdfe211e2d98eb285", size = 63999, upload-time = "2026-05-08T21:01:55.179Z" }, + { url = "https://files.pythonhosted.org/packages/5a/56/387e3f7dfce0a9233df41fb888aa1c30222cb4bbbf09537c02dd9bd85fe2/propcache-0.5.2-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:7afa37062e6650640e932e4cc9297d81f9f42d9944029cc386b8247dea4da837", size = 62779, upload-time = "2026-05-08T21:01:57.489Z" }, + { url = "https://files.pythonhosted.org/packages/a1/9c/596784cb5824ed61ee960d3f8655a3f0993e107c6e98ab6c818b7fb92ccb/propcache-0.5.2-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:8a90efd5777e996e42d568db9ac740b944d691e565cbfd31b2f7832f9184b2b8", size = 59796, upload-time = "2026-05-08T21:01:58.736Z" }, + { url = "https://files.pythonhosted.org/packages/c2/3d/1a6cfa1726a48542c1e8784a0761421476a5b68e09b7f36bf95eb954aaba/propcache-0.5.2-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:f19bb891234d72535764d703bfed1153cc34f4214d5bd7150aee1eec9e8f4366", size = 66023, upload-time = "2026-05-08T21:02:00.228Z" }, + { url = "https://files.pythonhosted.org/packages/e4/0e/05fd6990369477076e4e280bcb970de760fddf0161a46e988bc95f7940ec/propcache-0.5.2-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:32775082acd2d807ee3db715c7770d38767b817870acfa08c29e057f3c4d5b56", size = 64448, upload-time = "2026-05-08T21:02:01.888Z" }, + { url = "https://files.pythonhosted.org/packages/cd/86/5f8da315a4309c62c10c0b2516b17492d5d3bbe1bb862b96604db67e2a37/propcache-0.5.2-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:9282fb1a3bccd038da9f768b927b24a0c753e466c086b7c4f3c6982851eefb2d", size = 67329, upload-time = "2026-05-08T21:02:03.484Z" }, + { url = "https://files.pythonhosted.org/packages/da/d3/3368efe79ab21f0cdf86ef49895811c9cc933131d4cde1f28a624e22e712/propcache-0.5.2-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:cc49723e2f60d6b32a0f0b08a3fd6d13203c07f1cd9566cfce0f12a917c967a2", size = 65172, upload-time = "2026-05-08T21:02:04.745Z" }, + { url = "https://files.pythonhosted.org/packages/d5/07/127e8b0bacfb325396196f9d976a22453049b89b9b2b08477cc3145faa44/propcache-0.5.2-cp314-cp314t-win32.whl", hash = "sha256:2d7aa89ebca5acc98cba9d1472d976e394782f587bad6661003602a619fd1821", size = 43813, upload-time = "2026-05-08T21:02:06.025Z" }, + { url = "https://files.pythonhosted.org/packages/88/fb/46dad6c0ae49ed230ab1b16c890c2b6314e2403e6c412976f4a72d64a527/propcache-0.5.2-cp314-cp314t-win_amd64.whl", hash = "sha256:d447bb0b3054be5818458fbb171208b1d9ff11eba14e18ca18b90cbb45767370", size = 47764, upload-time = "2026-05-08T21:02:07.353Z" }, + { url = "https://files.pythonhosted.org/packages/e7/c4/a47d0a63aa309d10d59ede6e9d4cff03a344a79d1f0f4cd0cd74997b53e0/propcache-0.5.2-cp314-cp314t-win_arm64.whl", hash = "sha256:fe67a3d11cd9b4efabfa45c3d00ffba2b26811442a73a581a94b67c2b5faccf6", size = 41140, upload-time = "2026-05-08T21:02:09.065Z" }, + { url = "https://files.pythonhosted.org/packages/3a/ed/1cdcab6ba3d6ab7feca11fc14f0eeea80755bb53ef4e892079f31b10a25f/propcache-0.5.2-py3-none-any.whl", hash = "sha256:be1ddfcbb376e3de5d2e2db1d58d6d67463e6b4f9f040c000de8e300295465fe", size = 14036, upload-time = "2026-05-08T21:02:10.673Z" }, ] [[package]] @@ -2079,64 +2197,57 @@ wheels = [ [[package]] name = "pyarrow" -version = "24.0.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/91/13/13e1069b351bdc3881266e11147ffccf687505dbb0ea74036237f5d454a5/pyarrow-24.0.0.tar.gz", hash = "sha256:85fe721a14dd823aca09127acbb06c3ca723efbd436c004f16bca601b04dcc83", size = 1180261, upload-time = "2026-04-21T10:51:25.837Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/a5/bf/a34fee1d624152124fa8355c42f34195ad5fe5233ce5bb87946432047d52/pyarrow-24.0.0-cp310-cp310-macosx_12_0_arm64.whl", hash = "sha256:7c2b98645d576a0b9616892ead22b64a83a5f043c5e2ca15ebcefcb5b70c80cb", size = 35076681, upload-time = "2026-04-21T08:51:46.845Z" }, - { url = "https://files.pythonhosted.org/packages/1d/41/64180033d7027afce12dc96d0fe1f504c6fa112190582b458acea2399530/pyarrow-24.0.0-cp310-cp310-macosx_12_0_x86_64.whl", hash = "sha256:644a246325b8c69c595ad1dd4b463eba4b0cdb731370e4a86137d433208d6147", size = 36684260, upload-time = "2026-04-21T08:51:53.642Z" }, - { url = "https://files.pythonhosted.org/packages/57/02/9b9320e673dd8a99411fac78690f3df92f6dd6f59754c750110bca66d64e/pyarrow-24.0.0-cp310-cp310-manylinux_2_28_aarch64.whl", hash = "sha256:3a577bd840ca83f646f0a625dbc571dba7044c43c2d1503afc378b570954345c", size = 45698566, upload-time = "2026-04-21T10:46:02.133Z" }, - { url = "https://files.pythonhosted.org/packages/67/33/f75e91b9a64c3f33c787e263c93b871ad91b8a4a68c1d5cebddd9840e835/pyarrow-24.0.0-cp310-cp310-manylinux_2_28_x86_64.whl", hash = "sha256:e3268e43984d0b1a185c89b4cfff282a7ead12fc93f56cfd7088bdbcbe727041", size = 48835562, upload-time = "2026-04-21T10:46:10.278Z" }, - { url = "https://files.pythonhosted.org/packages/a5/63/097510448e47e4091faa41c43ba92f97cecaab8f4535b56a3d149578f634/pyarrow-24.0.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:2392d954fcb920f42d230284b677605e4e2fbb11f2821e823e642abd67fbb491", size = 49394997, upload-time = "2026-04-21T10:46:18.08Z" }, - { url = "https://files.pythonhosted.org/packages/60/6b/c047d6222ab279024a062742d1807e2fbaf27bba88a98637299ff47b9236/pyarrow-24.0.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:bec9373df11544592b0ba7ec2af0e35059e5f0e7647c6183a854dedd193298f1", size = 51911424, upload-time = "2026-04-21T10:46:25.347Z" }, - { url = "https://files.pythonhosted.org/packages/3a/ba/464cc70761c2a525d97ebd84e21c31ebd47f3ef4bdcee117009f51c46f24/pyarrow-24.0.0-cp310-cp310-win_amd64.whl", hash = "sha256:c42ab9439498270139cc63e18847a02afe5c8b3ed9c931266533cfe378bd3591", size = 27251730, upload-time = "2026-04-21T10:46:30.913Z" }, - { url = "https://files.pythonhosted.org/packages/62/c9/a47ab7ece0d86cbe6678418a0fbd1ac4bb493b9184a3891dfa0e7f287ae0/pyarrow-24.0.0-cp311-cp311-macosx_12_0_arm64.whl", hash = "sha256:b0e131f880cda8d04e076cee175a46fc0e8bc8b65c99c6c09dff6669335fde74", size = 35068898, upload-time = "2026-04-21T10:46:36.599Z" }, - { url = "https://files.pythonhosted.org/packages/d1/bc/8db86617a9a58008acf8913d6fed68ea2a46acb6de928db28d724c891a68/pyarrow-24.0.0-cp311-cp311-macosx_12_0_x86_64.whl", hash = "sha256:1b2fe7f9a5566401a0ef2571f197eb92358925c1f0c8dba305d6e43ea0871bb3", size = 36679915, upload-time = "2026-04-21T10:46:42.602Z" }, - { url = "https://files.pythonhosted.org/packages/eb/8e/fb178720400ef69db251eb4a9c3ccf4af269bc1feb5055529b8fc87170d1/pyarrow-24.0.0-cp311-cp311-manylinux_2_28_aarch64.whl", hash = "sha256:0b3537c00fb8d384f15ac1e79b6eb6db04a16514c8c1d22e59a9b95c8ba42868", size = 45697931, upload-time = "2026-04-21T10:46:48.403Z" }, - { url = "https://files.pythonhosted.org/packages/f3/27/99c42abe8e21b44f4917f62631f3aa31404882a2c41d8a4cd5c110e13d52/pyarrow-24.0.0-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:14e31a3c9e35f1ab6356c6378f6f72830e6d2d5f1791df3774a7b097d18a6a1e", size = 48837449, upload-time = "2026-04-21T10:46:55.329Z" }, - { url = "https://files.pythonhosted.org/packages/36/b6/333749e2666e9032891125bf9c691146e92901bece62030ac1430e2e7c88/pyarrow-24.0.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:b7d9a514e73bc42711e6a35aaccf3587c520024fe0a25d830a1a8a27c15f4f57", size = 49395949, upload-time = "2026-04-21T10:47:01.869Z" }, - { url = "https://files.pythonhosted.org/packages/17/25/c5201706a2dd374e8ba6ee3fd7a8c89fb7ffc16eed5217a91fd2bd7f7626/pyarrow-24.0.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:b196eb3f931862af3fa84c2a253514d859c08e0d8fe020e07be12e75a5a9780c", size = 51912986, upload-time = "2026-04-21T10:47:09.872Z" }, - { url = "https://files.pythonhosted.org/packages/f8/d2/4d1bbba65320b21a49678d6fbdc6ff7c649251359fdcfc03568c4136231d/pyarrow-24.0.0-cp311-cp311-win_amd64.whl", hash = "sha256:35405aecb474e683fb36af650618fd5340ee5471fc65a21b36076a18bbc6c981", size = 27255371, upload-time = "2026-04-21T10:47:15.943Z" }, - { url = "https://files.pythonhosted.org/packages/b4/a9/9686d9f07837f91f775e8932659192e02c74f9d8920524b480b85212cc68/pyarrow-24.0.0-cp312-cp312-macosx_12_0_arm64.whl", hash = "sha256:6233c9ed9ab9d1db47de57d9753256d9dcffbf42db341576099f0fd9f6bf4810", size = 34981559, upload-time = "2026-04-21T10:47:22.17Z" }, - { url = "https://files.pythonhosted.org/packages/80/b6/0ddf0e9b6ead3474ab087ae598c76b031fc45532bf6a63f3a553440fb258/pyarrow-24.0.0-cp312-cp312-macosx_12_0_x86_64.whl", hash = "sha256:f7616236ec1bc2b15bfdec22a71ab38851c86f8f05ff64f379e1278cf20c634a", size = 36663654, upload-time = "2026-04-21T10:47:28.315Z" }, - { url = "https://files.pythonhosted.org/packages/7c/3b/926382efe8ce27ba729071d3566ade6dfb86bdf112f366000196b2f5780a/pyarrow-24.0.0-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:1617043b99bd33e5318ae18eb2919af09c71322ef1ca46566cdafc6e6712fb66", size = 45679394, upload-time = "2026-04-21T10:47:34.821Z" }, - { url = "https://files.pythonhosted.org/packages/b3/7a/829f7d9dfd37c207206081d6dad474d81dde29952401f07f2ba507814818/pyarrow-24.0.0-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:6165461f55ef6314f026de6638d661188e3455d3ec49834556a0ebbdbace18bb", size = 48863122, upload-time = "2026-04-21T10:47:42.056Z" }, - { url = "https://files.pythonhosted.org/packages/5f/e8/f88ce625fe8babaae64e8db2d417c7653adb3019b08aae85c5ed787dc816/pyarrow-24.0.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:3b13dedfe76a0ad2d1d859b0811b53827a4e9d93a0bcb05cf59333ab4980cc7e", size = 49376032, upload-time = "2026-04-21T10:47:48.967Z" }, - { url = "https://files.pythonhosted.org/packages/36/7a/82c363caa145fff88fb475da50d3bf52bb024f61917be5424c3392eaf878/pyarrow-24.0.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:25ea65d868eb04015cd18e6df2fbe98f07e5bda2abefabcb88fce39a947716f6", size = 51929490, upload-time = "2026-04-21T10:47:55.981Z" }, - { url = "https://files.pythonhosted.org/packages/66/1c/e3e72c8014ad2743ca64a701652c733cc5cbcee15c0463a32a8c55518d9e/pyarrow-24.0.0-cp312-cp312-win_amd64.whl", hash = "sha256:295f0a7f2e242dabd513737cf076007dc5b2d59237e3eca37b05c0c6446f3826", size = 27355660, upload-time = "2026-04-21T10:48:01.718Z" }, - { url = "https://files.pythonhosted.org/packages/6f/d3/a1abf004482026ddc17f4503db227787fa3cfe41ec5091ff20e4fea55e57/pyarrow-24.0.0-cp313-cp313-macosx_12_0_arm64.whl", hash = "sha256:02b001b3ed4723caa44f6cd1af2d5c86aa2cf9971dacc2ffa55b21237713dfba", size = 34976759, upload-time = "2026-04-21T10:48:07.258Z" }, - { url = "https://files.pythonhosted.org/packages/4f/4a/34f0a36d28a2dd32225301b79daad44e243dc1a2bb77d43b60749be255c4/pyarrow-24.0.0-cp313-cp313-macosx_12_0_x86_64.whl", hash = "sha256:04920d6a71aabd08a0417709efce97d45ea8e6fb733d9ca9ecffb13c67839f68", size = 36658471, upload-time = "2026-04-21T10:48:13.347Z" }, - { url = "https://files.pythonhosted.org/packages/1f/78/543b94712ae8bb1a6023bcc1acf1a740fbff8286747c289cd9468fced2a5/pyarrow-24.0.0-cp313-cp313-manylinux_2_28_aarch64.whl", hash = "sha256:a964266397740257f16f7bb2e4f08a0c81454004beab8ff59dd531b73610e9f2", size = 45675981, upload-time = "2026-04-21T10:48:20.201Z" }, - { url = "https://files.pythonhosted.org/packages/84/9f/8fb7c222b100d314137fa40ec050de56cd8c6d957d1cfff685ce72f15b17/pyarrow-24.0.0-cp313-cp313-manylinux_2_28_x86_64.whl", hash = "sha256:6f066b179d68c413374294bc1735f68475457c933258df594443bb9d88ddc2a0", size = 48859172, upload-time = "2026-04-21T10:48:27.541Z" }, - { url = "https://files.pythonhosted.org/packages/a7/d3/1ea72538e6c8b3b475ed78d1049a2c518e655761ea50fe1171fc855fcab7/pyarrow-24.0.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:1183baeb14c5f587b1ec52831e665718ce632caab84b7cd6b85fd44f96114495", size = 49385733, upload-time = "2026-04-21T10:48:34.7Z" }, - { url = "https://files.pythonhosted.org/packages/c3/be/c3d8b06a1ba35f2260f8e1f771abbee7d5e345c0937aab90675706b1690a/pyarrow-24.0.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:806f24b4085453c197a5078218d1ee08783ebbba271badd153d1ae22a3ee804f", size = 51934335, upload-time = "2026-04-21T10:48:42.099Z" }, - { url = "https://files.pythonhosted.org/packages/9c/62/89e07a1e7329d2cde3e3c6994ba0839a24977a2beda8be6005ea3d860b99/pyarrow-24.0.0-cp313-cp313-win_amd64.whl", hash = "sha256:e4505fc6583f7b05ab854934896bcac8253b04ac1171a77dfb73efef92076d91", size = 27271748, upload-time = "2026-04-21T10:49:42.532Z" }, - { url = "https://files.pythonhosted.org/packages/17/1a/cff3a59f80b5b1658549d46611b67163f65e0664431c076ad728bf9d5af4/pyarrow-24.0.0-cp313-cp313t-macosx_12_0_arm64.whl", hash = "sha256:1a4e45017efbf115032e4475ee876d525e0e36c742214fbe405332480ecd6275", size = 35238554, upload-time = "2026-04-21T10:48:48.526Z" }, - { url = "https://files.pythonhosted.org/packages/a8/99/cce0f42a327bfef2c420fb6078a3eb834826e5d6697bf3009fe11d2ad051/pyarrow-24.0.0-cp313-cp313t-macosx_12_0_x86_64.whl", hash = "sha256:7986f1fa71cee060ad00758bcc79d3a93bab8559bf978fab9e53472a2e25a17b", size = 36782301, upload-time = "2026-04-21T10:48:55.181Z" }, - { url = "https://files.pythonhosted.org/packages/2a/66/8e560d5ff6793ca29aca213c53eec0dd482dd46cb93b2819e5aab52e4252/pyarrow-24.0.0-cp313-cp313t-manylinux_2_28_aarch64.whl", hash = "sha256:d3e0b61e8efb24ed38898e5cdc5fffa9124be480008d401a1f8071500494ae42", size = 45721929, upload-time = "2026-04-21T10:49:03.676Z" }, - { url = "https://files.pythonhosted.org/packages/27/0c/a26e25505d030716e078d9f16eb74973cbf0b33b672884e9f9da1c83b871/pyarrow-24.0.0-cp313-cp313t-manylinux_2_28_x86_64.whl", hash = "sha256:55a3bc1e3df3b5567b7d27ef551b2283f0c68a5e86f1cd56abc569da4f31335b", size = 48825365, upload-time = "2026-04-21T10:49:11.714Z" }, - { url = "https://files.pythonhosted.org/packages/5f/eb/771f9ecb0c65e73fe9dccdd1717901b9594f08c4515d000c7c62df573811/pyarrow-24.0.0-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:641f795b361874ac9da5294f8f443dfdbee355cf2bd9e3b8d97aaac2306b9b37", size = 49451819, upload-time = "2026-04-21T10:49:21.474Z" }, - { url = "https://files.pythonhosted.org/packages/48/da/61ae89a88732f5a785646f3ec6125dbb640fa98a540eb2b9889caa561403/pyarrow-24.0.0-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:8adc8e6ce5fccf5dc707046ae4914fd537def529709cc0d285d37a7f9cd442ca", size = 51909252, upload-time = "2026-04-21T10:49:31.164Z" }, - { url = "https://files.pythonhosted.org/packages/cb/1a/8dd5cafab7b66573fa91c03d06d213356ad4edd71813aa75e08ce2b3a844/pyarrow-24.0.0-cp313-cp313t-win_amd64.whl", hash = "sha256:9b18371ad2f44044b81a8d23bc2d8a9b6a6226dca775e8e16cfee640473d6c5d", size = 27388127, upload-time = "2026-04-21T10:49:37.334Z" }, - { url = "https://files.pythonhosted.org/packages/ad/80/d022a34ff05d2cbedd8ccf841fc1f532ecfa9eb5ed1711b56d0e0ea71fc9/pyarrow-24.0.0-cp314-cp314-macosx_12_0_arm64.whl", hash = "sha256:1cc9057f0319e26333b357e17f3c2c022f1a83739b48a88b25bfd5fa2dc18838", size = 35007997, upload-time = "2026-04-21T10:49:48.796Z" }, - { url = "https://files.pythonhosted.org/packages/1a/ff/f01485fda6f4e5d441afb8dd5e7681e4db18826c1e271852f5d3957d6a80/pyarrow-24.0.0-cp314-cp314-macosx_12_0_x86_64.whl", hash = "sha256:e6f1278ee4785b6db21229374a1c9e54ec7c549de5d1efc9630b6207de7e170b", size = 36678720, upload-time = "2026-04-21T10:49:55.858Z" }, - { url = "https://files.pythonhosted.org/packages/9e/c2/2d2d5fea814237923f71b36495211f20b43a1576f9a4d6da7e751a64ec6f/pyarrow-24.0.0-cp314-cp314-manylinux_2_28_aarch64.whl", hash = "sha256:adbbedc55506cbdabb830890444fb856bfb0060c46c6f8026c6c2f2cf86ae795", size = 45741852, upload-time = "2026-04-21T10:50:04.624Z" }, - { url = "https://files.pythonhosted.org/packages/8e/3a/28ba9c1c1ebdbb5f1b94dfebb46f207e52e6a554b7fe4132540fde29a3a0/pyarrow-24.0.0-cp314-cp314-manylinux_2_28_x86_64.whl", hash = "sha256:ae8a1145af31d903fa9bb166824d7abe9b4681a000b0159c9fb99c11bc11ad26", size = 48889852, upload-time = "2026-04-21T10:50:12.293Z" }, - { url = "https://files.pythonhosted.org/packages/df/51/4a389acfd31dca009f8fb82d7f510bb4130f2b3a8e18cf00194d0687d8ac/pyarrow-24.0.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:d7027eba1df3b2069e2e8d80f644fa0918b68c46432af3d088ddd390d063ecde", size = 49445207, upload-time = "2026-04-21T10:50:20.677Z" }, - { url = "https://files.pythonhosted.org/packages/19/4b/0bab2b23d2ae901b1b9a03c0efd4b2d070256f8ce3fc43f6e58c167b2081/pyarrow-24.0.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:e56a1ffe9bf7b727432b89104cc0849c21582949dd7bdcb34f17b2001a351a76", size = 51954117, upload-time = "2026-04-21T10:50:29.14Z" }, - { url = "https://files.pythonhosted.org/packages/29/88/f4e9145da0417b3d2c12035a8492b35ff4a3dbc653e614fcfb51d9dedb38/pyarrow-24.0.0-cp314-cp314-win_amd64.whl", hash = "sha256:38be1808cdd068605b787e6ca9119b27eb275a0234e50212c3492331680c3b1e", size = 28001155, upload-time = "2026-04-21T10:51:22.337Z" }, - { url = "https://files.pythonhosted.org/packages/79/4f/46a49a63f43526da895b1a45bbb51d5baf8e4d77159f8528fc3e5490007f/pyarrow-24.0.0-cp314-cp314t-macosx_12_0_arm64.whl", hash = "sha256:418e48ce50a45a6a6c73c454677203a9c75c966cb1e92ca3370959185f197a05", size = 35250387, upload-time = "2026-04-21T10:50:35.552Z" }, - { url = "https://files.pythonhosted.org/packages/a0/da/d5e0cd5ef00796922404806d5f00325cdadc3441ce2c13fe7115f2df9a64/pyarrow-24.0.0-cp314-cp314t-macosx_12_0_x86_64.whl", hash = "sha256:2f16197705a230a78270cdd4ea8a1d57e86b2fdcbc34a1f6aebc72e65c986f9a", size = 36797102, upload-time = "2026-04-21T10:50:42.417Z" }, - { url = "https://files.pythonhosted.org/packages/34/c7/5904145b0a593a05236c882933d439b5720f0a145381179063722fbfc123/pyarrow-24.0.0-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:fb24ac194bfc5e86839d7dcd52092ee31e5fe6733fe11f5e3b06ef0812b20072", size = 45745118, upload-time = "2026-04-21T10:50:49.324Z" }, - { url = "https://files.pythonhosted.org/packages/13/d3/cca42fe166d1c6e4d5b80e530b7949104d10e17508a90ae202dac205ce2a/pyarrow-24.0.0-cp314-cp314t-manylinux_2_28_x86_64.whl", hash = "sha256:9700ebd9a51f5895ce75ff4ac4b3c47a7d4b42bc618be8e713e5d56bacf5f931", size = 48844765, upload-time = "2026-04-21T10:50:55.579Z" }, - { url = "https://files.pythonhosted.org/packages/b0/49/942c3b79878ba928324d1e17c274ed84581db8c0a749b24bcf4cbdf15bd3/pyarrow-24.0.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:d8ddd2768da81d3ee08cfea9b597f4abb4e8e1dc8ae7e204b608d23a0d3ab699", size = 49471890, upload-time = "2026-04-21T10:51:02.439Z" }, - { url = "https://files.pythonhosted.org/packages/76/97/ff71431000a75d84135a1ace5ca4ba11726a231a8007bbb320a4c54075d5/pyarrow-24.0.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:61a3d7eaa97a14768b542f3d284dc6400dd2470d9f080708b13cd46b6ae18136", size = 51932250, upload-time = "2026-04-21T10:51:10.576Z" }, - { url = "https://files.pythonhosted.org/packages/51/be/6f79d55816d5c22557cf27533543d5d70dfe692adfbee4b99f2760674f38/pyarrow-24.0.0-cp314-cp314t-win_amd64.whl", hash = "sha256:c91d00057f23b8d353039520dc3a6c09d8608164c692e9f59a175a42b2ae0c19", size = 28131282, upload-time = "2026-04-21T10:51:16.815Z" }, +version = "25.0.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/27/f3/95428098d1fa7d04432fb750eed06b41304c2f6a5d3319985e64db2d9d41/pyarrow-25.0.0.tar.gz", hash = "sha256:d2d697008b5ec06d75952ef260c2e9a8a0f6ccfce24266c04c9c8ade927cb3b4", size = 1199181, upload-time = "2026-07-10T08:29:50.116Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/4d/2a/eaa70e6d6ed430c2e90c0599e2831a41a50251879e44788ccdbc73115af1/pyarrow-25.0.0-cp310-cp310-macosx_12_0_arm64.whl", hash = "sha256:ce0ca222802087b9a8cb031a6468442cb6b67c290a45a601cac64753d34954d3", size = 35945551, upload-time = "2026-07-10T08:25:23.153Z" }, + { url = "https://files.pythonhosted.org/packages/df/e0/917086af6b246143012cdc8a7c886b018b53204f3d69fc5f9be5857a8b80/pyarrow-25.0.0-cp310-cp310-macosx_12_0_x86_64.whl", hash = "sha256:7d6da02ffc7a3a9bda3b7ded4cc2a27ff73969ab37153f3afd46bbbc1ba4f0f7", size = 37636698, upload-time = "2026-07-10T08:25:28.031Z" }, + { url = "https://files.pythonhosted.org/packages/68/6a/c87829f92503f84993721791c942f3d9aa81044de51a8cfb1da5810e5345/pyarrow-25.0.0-cp310-cp310-manylinux_2_28_aarch64.whl", hash = "sha256:dbf9fa5d4bde73b1cc16377dcaaa010f971e6fa7f5083f5d44f34b50bc1d74af", size = 46858364, upload-time = "2026-07-10T08:25:34.527Z" }, + { url = "https://files.pythonhosted.org/packages/cc/ba/2030d454c2747e26cce23e4a0338067ee0830a155b7894da04caa96783a5/pyarrow-25.0.0-cp310-cp310-manylinux_2_28_x86_64.whl", hash = "sha256:b72d943ff4e10fec8d48aedb23322d8f6ea8bc2d698b81db37e73730f69e4862", size = 50056398, upload-time = "2026-07-10T08:25:40.785Z" }, + { url = "https://files.pythonhosted.org/packages/78/ce/ba7a5ce7bf0cfc372ec48203a34ece42f73aa2f3231706f61c55e105ecd0/pyarrow-25.0.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:5fb2d837960f1df7f679ff9f1a55065e306347d379e0768cebf14781254d6194", size = 49958146, upload-time = "2026-07-10T08:25:46.98Z" }, + { url = "https://files.pythonhosted.org/packages/75/eb/c34a29fb7a70dca2f903c7d85a928928ef55af20cd56e99de6b4c0d897bc/pyarrow-25.0.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:add690feafa0953c443cdba9e9e87f5eaa198f1ea2e43a3b146ea83f202262d0", size = 53096264, upload-time = "2026-07-10T08:25:53.925Z" }, + { url = "https://files.pythonhosted.org/packages/36/f9/35b1f83a0727d84951588e4034aca2feb76dfb45b0725918c0037b0a48f7/pyarrow-25.0.0-cp310-cp310-win_amd64.whl", hash = "sha256:d293e9959b29a24c82d936d04ab2b7fd8b8d334030de2e56a99aba94f008ad7a", size = 27840572, upload-time = "2026-07-10T08:25:58.966Z" }, + { url = "https://files.pythonhosted.org/packages/a7/98/ae2b5acf9876dbeffa6f320776242c52caab062df55c8ac5501ed2679e74/pyarrow-25.0.0-cp311-cp311-macosx_12_0_arm64.whl", hash = "sha256:2e3b6544e26e393fe2cd530f523e36c1c8d3c345bbbb60cca3fd866be8322517", size = 35939080, upload-time = "2026-07-10T08:26:04.53Z" }, + { url = "https://files.pythonhosted.org/packages/80/09/3de2a968edbd496c86cb8b932cdbee2d4b08c4a28e9884a15e5c705a646b/pyarrow-25.0.0-cp311-cp311-macosx_12_0_x86_64.whl", hash = "sha256:b724d127783b4c19f088fcdfc844cbc318809246a30307bcabd5ed02045e890e", size = 37633420, upload-time = "2026-07-10T08:26:10.354Z" }, + { url = "https://files.pythonhosted.org/packages/19/86/8399243a4ce080426ec37db18d5e29148b7ec960a8a8c7f9059a7bf6ef0a/pyarrow-25.0.0-cp311-cp311-manylinux_2_28_aarch64.whl", hash = "sha256:244f98a595f70fa4fd35faa7508c4ae67e14a173397a4b3b49d2b3c360fb0062", size = 46861050, upload-time = "2026-07-10T08:26:16.397Z" }, + { url = "https://files.pythonhosted.org/packages/7b/79/72d704b02bc5fc6d06954d76a0208c1e79cad3ab370f6d6a91ffe5078870/pyarrow-25.0.0-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:0222f0071d13313962a88d21bf28b80d355ac39d81bfa6ff3fe00eeaf748e4be", size = 50056458, upload-time = "2026-07-10T08:26:23.271Z" }, + { url = "https://files.pythonhosted.org/packages/06/5d/3c31a60b6403d63cad2e0f829096f5fc5763a129ead4207a5d4690b96448/pyarrow-25.0.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:b58726f118c079f9d4ed7e904975d4f15fd69d0741ba511a4e2dcaa4ef16354f", size = 49957793, upload-time = "2026-07-10T08:26:30.232Z" }, + { url = "https://files.pythonhosted.org/packages/34/f7/8f8a019061f9863a831915329264372a87ed25eaf9109ce56eb0e84012c5/pyarrow-25.0.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:38a2c887cb3883e241b70201688db34133b6dfadd04f03c8f9213df53770c18e", size = 53100544, upload-time = "2026-07-10T08:26:36.414Z" }, + { url = "https://files.pythonhosted.org/packages/f1/e2/738071e95c5ddad7b3dfc12f569ffa992db89d7d7b4a95258fd184191249/pyarrow-25.0.0-cp311-cp311-win_amd64.whl", hash = "sha256:161649d60a7a46c613a19fd795763ea8a88c36ba997dd99d9bc66e6794ee36e8", size = 27848311, upload-time = "2026-07-10T08:26:41.429Z" }, + { url = "https://files.pythonhosted.org/packages/73/44/fdd3a4377807b7dcabe2d4b5aa99dbbc98e2e5df3f1ca4e7f0aec492d987/pyarrow-25.0.0-cp312-cp312-macosx_12_0_arm64.whl", hash = "sha256:149730a3d1f0fb59d663a0b8aa210adfd9c17c27cd94a0d143e60daea8320d4e", size = 35850884, upload-time = "2026-07-10T08:26:47.357Z" }, + { url = "https://files.pythonhosted.org/packages/bf/71/9f053177a7709b8c90abb00a2375b916286f9f0d6cfb21a5cadd4ef811e8/pyarrow-25.0.0-cp312-cp312-macosx_12_0_x86_64.whl", hash = "sha256:0721332c30fdd453fdd1fc203b2ac1f4c9db5aea28fa38d41f2574c4b068b9ec", size = 37616197, upload-time = "2026-07-10T08:26:53.564Z" }, + { url = "https://files.pythonhosted.org/packages/95/1a/22bfb6597dcdc861fa83c39c06e1457cb56f698940eff42fbb25de30e8e5/pyarrow-25.0.0-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:fa1482b3da10cac2d4db6e26b81da543e237616af2ef6d466018b31ca586496f", size = 46841966, upload-time = "2026-07-10T08:27:07.685Z" }, + { url = "https://files.pythonhosted.org/packages/55/0e/cd705c042bc4fe7022478db577fcab4abdcfabb9bc37ab7a75556b3fcb2b/pyarrow-25.0.0-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:5d1dbf24e151042f2fa3c129563f65d66674128868496fb008c4272b16bdf778", size = 50088993, upload-time = "2026-07-10T08:27:14.268Z" }, + { url = "https://files.pythonhosted.org/packages/98/ee/d822e1ee31fe31ec5d057210e0605c950b975dcd8d9a332976cc859a9df8/pyarrow-25.0.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:20887a762dd61dcc530f93a140840ab1f6aa7836b33270e42d627ab3cf11e537", size = 49941005, upload-time = "2026-07-10T08:27:21.274Z" }, + { url = "https://files.pythonhosted.org/packages/33/1b/207a90cc64619a095eb75a263ae069735f2810056d43c667befd573ec083/pyarrow-25.0.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:58d1ab556b0cea1c93fdb799b24ad58adb2f2a2788dbce782a94f64ae1a5cc9b", size = 53112355, upload-time = "2026-07-10T08:27:27.911Z" }, + { url = "https://files.pythonhosted.org/packages/7e/fe/81d1e5f8beed15c01e98649d5c6e2167b67fd395884a2488f18bf1cf0dba/pyarrow-25.0.0-cp312-cp312-win_amd64.whl", hash = "sha256:3f356afe61186395c861d5cd63dc21ff7d5fa335012a4668d979257df7fea0f5", size = 27945954, upload-time = "2026-07-10T08:27:32.903Z" }, + { url = "https://files.pythonhosted.org/packages/6c/c8/098ce17d778fd9d29e40bb8c5f19a40cc90c3f0b46c9057b0d7993f42f54/pyarrow-25.0.0-cp313-cp313-macosx_12_0_arm64.whl", hash = "sha256:8831a3ba52fa7cdb78d368d968b1dcd06171e6dff5461e16d90de91d371e47bc", size = 35844549, upload-time = "2026-07-10T08:27:37.956Z" }, + { url = "https://files.pythonhosted.org/packages/bc/66/24c28877219abf6263d909b1592c97ff82c59f13a59acbed11fc87c0654f/pyarrow-25.0.0-cp313-cp313-macosx_12_0_x86_64.whl", hash = "sha256:5f4bacb60f91dd2fca6c52f1b9a0012cd090e0294f1f781dc1881a247a352f8e", size = 37610397, upload-time = "2026-07-10T08:27:43.803Z" }, + { url = "https://files.pythonhosted.org/packages/53/55/6d1d5f5aff317ec5de9421594679ed51ed828fe7e2ce209327f819d801e4/pyarrow-25.0.0-cp313-cp313-manylinux_2_28_aarch64.whl", hash = "sha256:59516c822d5fd8e544aaa0dfe72f36fed5d4c24ea8390aab1bcd31d7e959c6be", size = 46841701, upload-time = "2026-07-10T08:27:49.741Z" }, + { url = "https://files.pythonhosted.org/packages/b5/5d/f790fb6965ab54c9da0dda7856abc75fd0d7648d865f8d603c111d203a64/pyarrow-25.0.0-cp313-cp313-manylinux_2_28_x86_64.whl", hash = "sha256:6f9dbd83e91c239a1f5ee7ce13f108b5f6c0efbe40a4375260d8f08b43ad05e9", size = 50090118, upload-time = "2026-07-10T08:27:56.051Z" }, + { url = "https://files.pythonhosted.org/packages/0c/8c/faf025357ebf31bc96777f234277aa31e2aeca6dd4ecaa391f29085473c2/pyarrow-25.0.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:18dcc8cc50b5e72eae6fcbfc6c8776c21a007176b27a3cdec5c2f5bcf126708d", size = 49945559, upload-time = "2026-07-10T08:28:01.927Z" }, + { url = "https://files.pythonhosted.org/packages/07/a1/bd051871708ea99a5e0fc711926c26c6f2c6d0130c7aaac8093e34998af6/pyarrow-25.0.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:4ec1895a87aa834c3b99b7a1e758747eb8bb57f922b32c0e0fa04afb8d6998b1", size = 53114238, upload-time = "2026-07-10T08:28:08.594Z" }, + { url = "https://files.pythonhosted.org/packages/7c/31/737f0c3cffcd6af647849477d1dd68045deac2e3963c3f9f211bedc48540/pyarrow-25.0.0-cp313-cp313-win_amd64.whl", hash = "sha256:77c8d1ae46a44b4006e8db1cc977bbcc6ce4873c92f74137d68e45503b97fb18", size = 27861162, upload-time = "2026-07-10T08:28:12.975Z" }, + { url = "https://files.pythonhosted.org/packages/55/c7/581ccbcdb3d897eb2893328d68db3d52eca373bf2a7e964d0a6276b8e85b/pyarrow-25.0.0-cp314-cp314-macosx_12_0_arm64.whl", hash = "sha256:72132b9a8a0a1840197794d4dea26080069b6b0981c116bc078762dc9691b21b", size = 35878945, upload-time = "2026-07-10T08:28:18.222Z" }, + { url = "https://files.pythonhosted.org/packages/64/d1/ccb01db7329ea0411ef4fbd9b62a04d3268b36777d4e758d5e39b91ddeab/pyarrow-25.0.0-cp314-cp314-macosx_12_0_x86_64.whl", hash = "sha256:e009ef945e498dca2f050ea10d2e9764cb44017254826fc4574fdb8d2530173b", size = 37630854, upload-time = "2026-07-10T08:28:23.452Z" }, + { url = "https://files.pythonhosted.org/packages/af/9f/2d81ba89d1e4198d0cb25fe7529de936830fdaec0db926bb52a1ef7080d4/pyarrow-25.0.0-cp314-cp314-manylinux_2_28_aarch64.whl", hash = "sha256:f57a39dbcb416345401c2e77a4373669b45fd111a1768e6cf267a7a0607ff0ec", size = 46905617, upload-time = "2026-07-10T08:28:29.376Z" }, + { url = "https://files.pythonhosted.org/packages/6a/29/0ed312ec800fb536f93783215126cee4b8977dcfeccba6f0f44df0cc87d7/pyarrow-25.0.0-cp314-cp314-manylinux_2_28_x86_64.whl", hash = "sha256:447df764beb07c544f0178a5f6b70ef44b9ecf382b3cdfad4c2d7867353c3887", size = 50119765, upload-time = "2026-07-10T08:28:35.826Z" }, + { url = "https://files.pythonhosted.org/packages/ca/88/cab5063ba0c4d46a9f6b4b7eb1c9029dc0302d65cd5ab3510c949a386568/pyarrow-25.0.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:ac5dfeee59f9ceb4d45ba76e83b026c38c24334135bb329d8274baa49cec3c62", size = 50027563, upload-time = "2026-07-10T08:28:43.848Z" }, + { url = "https://files.pythonhosted.org/packages/7b/fb/4d24f1b7fe2e042dc4ef315ef75e4e702d8e46fe10c37e63caff00502b03/pyarrow-25.0.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:f0f100dacf2c0f400601664a79d1a907ced4740514bb2b00917341038e2ce76f", size = 53162437, upload-time = "2026-07-10T08:28:52.819Z" }, + { url = "https://files.pythonhosted.org/packages/fa/65/da20806de93ca6ee91e72cb6a9b08b3ac890b46efc8d94a7326c651c4c81/pyarrow-25.0.0-cp314-cp314-win_amd64.whl", hash = "sha256:2e093efbecb5317372f819228fa4b4e6157eee48d3f0a7b0303705ebf81a7104", size = 28613262, upload-time = "2026-07-10T08:29:47.544Z" }, + { url = "https://files.pythonhosted.org/packages/86/9f/c632afb1d3ef4a7814cee236718235f3a47eac46e97eb87df40f550b6b48/pyarrow-25.0.0-cp314-cp314t-macosx_12_0_arm64.whl", hash = "sha256:26be35b80780d2d21f4bae3d568b1666337c3a89722cc1794c956a77017cb24e", size = 36120702, upload-time = "2026-07-10T08:28:59.577Z" }, + { url = "https://files.pythonhosted.org/packages/36/0a/093d53a0e72ad06e45d6443e00651bbc2d21af4211295086cbf4d873d3b9/pyarrow-25.0.0-cp314-cp314t-macosx_12_0_x86_64.whl", hash = "sha256:6f4812bfbf11ca7d8faf59eb8fff8bf4dd25ce3a38b62baa010cc17a0926d1b2", size = 37750674, upload-time = "2026-07-10T08:29:06.916Z" }, + { url = "https://files.pythonhosted.org/packages/8a/18/b37fc31a69cff4bdfb8842683def5612f551b93fff6f44375e4a4a6a5535/pyarrow-25.0.0-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:b8af8ceedf0c9c160fd2b63440f2d205b9404db85866c1217bfea601de7cfb50", size = 46912304, upload-time = "2026-07-10T08:29:14.656Z" }, + { url = "https://files.pythonhosted.org/packages/32/35/5cae19ba72493e5598022468b56f6a5571f399f485bf412f157356476caa/pyarrow-25.0.0-cp314-cp314t-manylinux_2_28_x86_64.whl", hash = "sha256:c70a5fd9a82bd1a702fd482bdc62d38dcb672fb2b449b1d7c0d7d1f4be7b7bfe", size = 50073652, upload-time = "2026-07-10T08:29:22.467Z" }, + { url = "https://files.pythonhosted.org/packages/2e/a5/ddd508424bdfd5e6945765e9e2ffc687e2f6115972badc8ecf423076c407/pyarrow-25.0.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:0490a7f8b38ffe11cc26526b50c65d111cb54ddac3717cec781806793f1244dc", size = 50058654, upload-time = "2026-07-10T08:29:29.689Z" }, + { url = "https://files.pythonhosted.org/packages/5f/a4/324d0db203ff5eebe8694ec2d6ec5a23f9aaa5d02e5b8c692914c518c33c/pyarrow-25.0.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:e83916bbcf380866b4e14255850b33323ff678dc9758411d0409cdd2523880b0", size = 53140153, upload-time = "2026-07-10T08:29:36.041Z" }, + { url = "https://files.pythonhosted.org/packages/bd/8d/d236e9c82fe315f9128885c8be3ec719f41965a1eb6b6f4b42470904cd41/pyarrow-25.0.0-cp314-cp314t-win_amd64.whl", hash = "sha256:13240f0d3dc5932ccd0bfa90cd76d835680b9d94a7661c635df4b703d40ce849", size = 28743657, upload-time = "2026-07-10T08:29:42.742Z" }, ] [[package]] name = "pydantic" -version = "2.12.4" +version = "2.13.4" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "annotated-types" }, @@ -2144,136 +2255,134 @@ dependencies = [ { name = "typing-extensions" }, { name = "typing-inspection" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/96/ad/a17bc283d7d81837c061c49e3eaa27a45991759a1b7eae1031921c6bd924/pydantic-2.12.4.tar.gz", hash = "sha256:0f8cb9555000a4b5b617f66bfd2566264c4984b27589d3b845685983e8ea85ac", size = 821038, upload-time = "2025-11-05T10:50:08.59Z" } +sdist = { url = "https://files.pythonhosted.org/packages/18/a5/b60d21ac674192f8ab0ba4e9fd860690f9b4a6e51ca5df118733b487d8d6/pydantic-2.13.4.tar.gz", hash = "sha256:c40756b57adaa8b1efeeced5c196f3f3b7c435f90e84ea7f443901bec8099ef6", size = 844775, upload-time = "2026-05-06T13:43:05.343Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/82/2f/e68750da9b04856e2a7ec56fc6f034a5a79775e9b9a81882252789873798/pydantic-2.12.4-py3-none-any.whl", hash = "sha256:92d3d202a745d46f9be6df459ac5a064fdaa3c1c4cd8adcfa332ccf3c05f871e", size = 463400, upload-time = "2025-11-05T10:50:06.732Z" }, + { url = "https://files.pythonhosted.org/packages/fd/7b/122376b1fd3c62c1ed9dc80c931ace4844b3c55407b6fb2d199377c9736f/pydantic-2.13.4-py3-none-any.whl", hash = "sha256:45a282cde31d808236fd7ea9d919b128653c8b38b393d1c4ab335c62924d9aba", size = 472262, upload-time = "2026-05-06T13:43:02.641Z" }, ] [[package]] name = "pydantic-core" -version = "2.41.5" +version = "2.46.4" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "typing-extensions" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/71/70/23b021c950c2addd24ec408e9ab05d59b035b39d97cdc1130e1bce647bb6/pydantic_core-2.41.5.tar.gz", hash = "sha256:08daa51ea16ad373ffd5e7606252cc32f07bc72b28284b6bc9c6df804816476e", size = 460952, upload-time = "2025-11-04T13:43:49.098Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/c6/90/32c9941e728d564b411d574d8ee0cf09b12ec978cb22b294995bae5549a5/pydantic_core-2.41.5-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:77b63866ca88d804225eaa4af3e664c5faf3568cea95360d21f4725ab6e07146", size = 2107298, upload-time = "2025-11-04T13:39:04.116Z" }, - { url = "https://files.pythonhosted.org/packages/fb/a8/61c96a77fe28993d9a6fb0f4127e05430a267b235a124545d79fea46dd65/pydantic_core-2.41.5-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:dfa8a0c812ac681395907e71e1274819dec685fec28273a28905df579ef137e2", size = 1901475, upload-time = "2025-11-04T13:39:06.055Z" }, - { url = "https://files.pythonhosted.org/packages/5d/b6/338abf60225acc18cdc08b4faef592d0310923d19a87fba1faf05af5346e/pydantic_core-2.41.5-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5921a4d3ca3aee735d9fd163808f5e8dd6c6972101e4adbda9a4667908849b97", size = 1918815, upload-time = "2025-11-04T13:39:10.41Z" }, - { url = "https://files.pythonhosted.org/packages/d1/1c/2ed0433e682983d8e8cba9c8d8ef274d4791ec6a6f24c58935b90e780e0a/pydantic_core-2.41.5-cp310-cp310-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:e25c479382d26a2a41b7ebea1043564a937db462816ea07afa8a44c0866d52f9", size = 2065567, upload-time = "2025-11-04T13:39:12.244Z" }, - { url = "https://files.pythonhosted.org/packages/b3/24/cf84974ee7d6eae06b9e63289b7b8f6549d416b5c199ca2d7ce13bbcf619/pydantic_core-2.41.5-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:f547144f2966e1e16ae626d8ce72b4cfa0caedc7fa28052001c94fb2fcaa1c52", size = 2230442, upload-time = "2025-11-04T13:39:13.962Z" }, - { url = "https://files.pythonhosted.org/packages/fd/21/4e287865504b3edc0136c89c9c09431be326168b1eb7841911cbc877a995/pydantic_core-2.41.5-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:6f52298fbd394f9ed112d56f3d11aabd0d5bd27beb3084cc3d8ad069483b8941", size = 2350956, upload-time = "2025-11-04T13:39:15.889Z" }, - { url = "https://files.pythonhosted.org/packages/a8/76/7727ef2ffa4b62fcab916686a68a0426b9b790139720e1934e8ba797e238/pydantic_core-2.41.5-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:100baa204bb412b74fe285fb0f3a385256dad1d1879f0a5cb1499ed2e83d132a", size = 2068253, upload-time = "2025-11-04T13:39:17.403Z" }, - { url = "https://files.pythonhosted.org/packages/d5/8c/a4abfc79604bcb4c748e18975c44f94f756f08fb04218d5cb87eb0d3a63e/pydantic_core-2.41.5-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:05a2c8852530ad2812cb7914dc61a1125dc4e06252ee98e5638a12da6cc6fb6c", size = 2177050, upload-time = "2025-11-04T13:39:19.351Z" }, - { url = "https://files.pythonhosted.org/packages/67/b1/de2e9a9a79b480f9cb0b6e8b6ba4c50b18d4e89852426364c66aa82bb7b3/pydantic_core-2.41.5-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:29452c56df2ed968d18d7e21f4ab0ac55e71dc59524872f6fc57dcf4a3249ed2", size = 2147178, upload-time = "2025-11-04T13:39:21Z" }, - { url = "https://files.pythonhosted.org/packages/16/c1/dfb33f837a47b20417500efaa0378adc6635b3c79e8369ff7a03c494b4ac/pydantic_core-2.41.5-cp310-cp310-musllinux_1_1_armv7l.whl", hash = "sha256:d5160812ea7a8a2ffbe233d8da666880cad0cbaf5d4de74ae15c313213d62556", size = 2341833, upload-time = "2025-11-04T13:39:22.606Z" }, - { url = "https://files.pythonhosted.org/packages/47/36/00f398642a0f4b815a9a558c4f1dca1b4020a7d49562807d7bc9ff279a6c/pydantic_core-2.41.5-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:df3959765b553b9440adfd3c795617c352154e497a4eaf3752555cfb5da8fc49", size = 2321156, upload-time = "2025-11-04T13:39:25.843Z" }, - { url = "https://files.pythonhosted.org/packages/7e/70/cad3acd89fde2010807354d978725ae111ddf6d0ea46d1ea1775b5c1bd0c/pydantic_core-2.41.5-cp310-cp310-win32.whl", hash = "sha256:1f8d33a7f4d5a7889e60dc39856d76d09333d8a6ed0f5f1190635cbec70ec4ba", size = 1989378, upload-time = "2025-11-04T13:39:27.92Z" }, - { url = "https://files.pythonhosted.org/packages/76/92/d338652464c6c367e5608e4488201702cd1cbb0f33f7b6a85a60fe5f3720/pydantic_core-2.41.5-cp310-cp310-win_amd64.whl", hash = "sha256:62de39db01b8d593e45871af2af9e497295db8d73b085f6bfd0b18c83c70a8f9", size = 2013622, upload-time = "2025-11-04T13:39:29.848Z" }, - { url = "https://files.pythonhosted.org/packages/e8/72/74a989dd9f2084b3d9530b0915fdda64ac48831c30dbf7c72a41a5232db8/pydantic_core-2.41.5-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:a3a52f6156e73e7ccb0f8cced536adccb7042be67cb45f9562e12b319c119da6", size = 2105873, upload-time = "2025-11-04T13:39:31.373Z" }, - { url = "https://files.pythonhosted.org/packages/12/44/37e403fd9455708b3b942949e1d7febc02167662bf1a7da5b78ee1ea2842/pydantic_core-2.41.5-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:7f3bf998340c6d4b0c9a2f02d6a400e51f123b59565d74dc60d252ce888c260b", size = 1899826, upload-time = "2025-11-04T13:39:32.897Z" }, - { url = "https://files.pythonhosted.org/packages/33/7f/1d5cab3ccf44c1935a359d51a8a2a9e1a654b744b5e7f80d41b88d501eec/pydantic_core-2.41.5-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:378bec5c66998815d224c9ca994f1e14c0c21cb95d2f52b6021cc0b2a58f2a5a", size = 1917869, upload-time = "2025-11-04T13:39:34.469Z" }, - { url = "https://files.pythonhosted.org/packages/6e/6a/30d94a9674a7fe4f4744052ed6c5e083424510be1e93da5bc47569d11810/pydantic_core-2.41.5-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:e7b576130c69225432866fe2f4a469a85a54ade141d96fd396dffcf607b558f8", size = 2063890, upload-time = "2025-11-04T13:39:36.053Z" }, - { url = "https://files.pythonhosted.org/packages/50/be/76e5d46203fcb2750e542f32e6c371ffa9b8ad17364cf94bb0818dbfb50c/pydantic_core-2.41.5-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:6cb58b9c66f7e4179a2d5e0f849c48eff5c1fca560994d6eb6543abf955a149e", size = 2229740, upload-time = "2025-11-04T13:39:37.753Z" }, - { url = "https://files.pythonhosted.org/packages/d3/ee/fed784df0144793489f87db310a6bbf8118d7b630ed07aa180d6067e653a/pydantic_core-2.41.5-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:88942d3a3dff3afc8288c21e565e476fc278902ae4d6d134f1eeda118cc830b1", size = 2350021, upload-time = "2025-11-04T13:39:40.94Z" }, - { url = "https://files.pythonhosted.org/packages/c8/be/8fed28dd0a180dca19e72c233cbf58efa36df055e5b9d90d64fd1740b828/pydantic_core-2.41.5-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f31d95a179f8d64d90f6831d71fa93290893a33148d890ba15de25642c5d075b", size = 2066378, upload-time = "2025-11-04T13:39:42.523Z" }, - { url = "https://files.pythonhosted.org/packages/b0/3b/698cf8ae1d536a010e05121b4958b1257f0b5522085e335360e53a6b1c8b/pydantic_core-2.41.5-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:c1df3d34aced70add6f867a8cf413e299177e0c22660cc767218373d0779487b", size = 2175761, upload-time = "2025-11-04T13:39:44.553Z" }, - { url = "https://files.pythonhosted.org/packages/b8/ba/15d537423939553116dea94ce02f9c31be0fa9d0b806d427e0308ec17145/pydantic_core-2.41.5-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:4009935984bd36bd2c774e13f9a09563ce8de4abaa7226f5108262fa3e637284", size = 2146303, upload-time = "2025-11-04T13:39:46.238Z" }, - { url = "https://files.pythonhosted.org/packages/58/7f/0de669bf37d206723795f9c90c82966726a2ab06c336deba4735b55af431/pydantic_core-2.41.5-cp311-cp311-musllinux_1_1_armv7l.whl", hash = "sha256:34a64bc3441dc1213096a20fe27e8e128bd3ff89921706e83c0b1ac971276594", size = 2340355, upload-time = "2025-11-04T13:39:48.002Z" }, - { url = "https://files.pythonhosted.org/packages/e5/de/e7482c435b83d7e3c3ee5ee4451f6e8973cff0eb6007d2872ce6383f6398/pydantic_core-2.41.5-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:c9e19dd6e28fdcaa5a1de679aec4141f691023916427ef9bae8584f9c2fb3b0e", size = 2319875, upload-time = "2025-11-04T13:39:49.705Z" }, - { url = "https://files.pythonhosted.org/packages/fe/e6/8c9e81bb6dd7560e33b9053351c29f30c8194b72f2d6932888581f503482/pydantic_core-2.41.5-cp311-cp311-win32.whl", hash = "sha256:2c010c6ded393148374c0f6f0bf89d206bf3217f201faa0635dcd56bd1520f6b", size = 1987549, upload-time = "2025-11-04T13:39:51.842Z" }, - { url = "https://files.pythonhosted.org/packages/11/66/f14d1d978ea94d1bc21fc98fcf570f9542fe55bfcc40269d4e1a21c19bf7/pydantic_core-2.41.5-cp311-cp311-win_amd64.whl", hash = "sha256:76ee27c6e9c7f16f47db7a94157112a2f3a00e958bc626e2f4ee8bec5c328fbe", size = 2011305, upload-time = "2025-11-04T13:39:53.485Z" }, - { url = "https://files.pythonhosted.org/packages/56/d8/0e271434e8efd03186c5386671328154ee349ff0354d83c74f5caaf096ed/pydantic_core-2.41.5-cp311-cp311-win_arm64.whl", hash = "sha256:4bc36bbc0b7584de96561184ad7f012478987882ebf9f9c389b23f432ea3d90f", size = 1972902, upload-time = "2025-11-04T13:39:56.488Z" }, - { url = "https://files.pythonhosted.org/packages/5f/5d/5f6c63eebb5afee93bcaae4ce9a898f3373ca23df3ccaef086d0233a35a7/pydantic_core-2.41.5-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:f41a7489d32336dbf2199c8c0a215390a751c5b014c2c1c5366e817202e9cdf7", size = 2110990, upload-time = "2025-11-04T13:39:58.079Z" }, - { url = "https://files.pythonhosted.org/packages/aa/32/9c2e8ccb57c01111e0fd091f236c7b371c1bccea0fa85247ac55b1e2b6b6/pydantic_core-2.41.5-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:070259a8818988b9a84a449a2a7337c7f430a22acc0859c6b110aa7212a6d9c0", size = 1896003, upload-time = "2025-11-04T13:39:59.956Z" }, - { url = "https://files.pythonhosted.org/packages/68/b8/a01b53cb0e59139fbc9e4fda3e9724ede8de279097179be4ff31f1abb65a/pydantic_core-2.41.5-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e96cea19e34778f8d59fe40775a7a574d95816eb150850a85a7a4c8f4b94ac69", size = 1919200, upload-time = "2025-11-04T13:40:02.241Z" }, - { url = "https://files.pythonhosted.org/packages/38/de/8c36b5198a29bdaade07b5985e80a233a5ac27137846f3bc2d3b40a47360/pydantic_core-2.41.5-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:ed2e99c456e3fadd05c991f8f437ef902e00eedf34320ba2b0842bd1c3ca3a75", size = 2052578, upload-time = "2025-11-04T13:40:04.401Z" }, - { url = "https://files.pythonhosted.org/packages/00/b5/0e8e4b5b081eac6cb3dbb7e60a65907549a1ce035a724368c330112adfdd/pydantic_core-2.41.5-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:65840751b72fbfd82c3c640cff9284545342a4f1eb1586ad0636955b261b0b05", size = 2208504, upload-time = "2025-11-04T13:40:06.072Z" }, - { url = "https://files.pythonhosted.org/packages/77/56/87a61aad59c7c5b9dc8caad5a41a5545cba3810c3e828708b3d7404f6cef/pydantic_core-2.41.5-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:e536c98a7626a98feb2d3eaf75944ef6f3dbee447e1f841eae16f2f0a72d8ddc", size = 2335816, upload-time = "2025-11-04T13:40:07.835Z" }, - { url = "https://files.pythonhosted.org/packages/0d/76/941cc9f73529988688a665a5c0ecff1112b3d95ab48f81db5f7606f522d3/pydantic_core-2.41.5-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:eceb81a8d74f9267ef4081e246ffd6d129da5d87e37a77c9bde550cb04870c1c", size = 2075366, upload-time = "2025-11-04T13:40:09.804Z" }, - { url = "https://files.pythonhosted.org/packages/d3/43/ebef01f69baa07a482844faaa0a591bad1ef129253ffd0cdaa9d8a7f72d3/pydantic_core-2.41.5-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:d38548150c39b74aeeb0ce8ee1d8e82696f4a4e16ddc6de7b1d8823f7de4b9b5", size = 2171698, upload-time = "2025-11-04T13:40:12.004Z" }, - { url = "https://files.pythonhosted.org/packages/b1/87/41f3202e4193e3bacfc2c065fab7706ebe81af46a83d3e27605029c1f5a6/pydantic_core-2.41.5-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:c23e27686783f60290e36827f9c626e63154b82b116d7fe9adba1fda36da706c", size = 2132603, upload-time = "2025-11-04T13:40:13.868Z" }, - { url = "https://files.pythonhosted.org/packages/49/7d/4c00df99cb12070b6bccdef4a195255e6020a550d572768d92cc54dba91a/pydantic_core-2.41.5-cp312-cp312-musllinux_1_1_armv7l.whl", hash = "sha256:482c982f814460eabe1d3bb0adfdc583387bd4691ef00b90575ca0d2b6fe2294", size = 2329591, upload-time = "2025-11-04T13:40:15.672Z" }, - { url = "https://files.pythonhosted.org/packages/cc/6a/ebf4b1d65d458f3cda6a7335d141305dfa19bdc61140a884d165a8a1bbc7/pydantic_core-2.41.5-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:bfea2a5f0b4d8d43adf9d7b8bf019fb46fdd10a2e5cde477fbcb9d1fa08c68e1", size = 2319068, upload-time = "2025-11-04T13:40:17.532Z" }, - { url = "https://files.pythonhosted.org/packages/49/3b/774f2b5cd4192d5ab75870ce4381fd89cf218af999515baf07e7206753f0/pydantic_core-2.41.5-cp312-cp312-win32.whl", hash = "sha256:b74557b16e390ec12dca509bce9264c3bbd128f8a2c376eaa68003d7f327276d", size = 1985908, upload-time = "2025-11-04T13:40:19.309Z" }, - { url = "https://files.pythonhosted.org/packages/86/45/00173a033c801cacf67c190fef088789394feaf88a98a7035b0e40d53dc9/pydantic_core-2.41.5-cp312-cp312-win_amd64.whl", hash = "sha256:1962293292865bca8e54702b08a4f26da73adc83dd1fcf26fbc875b35d81c815", size = 2020145, upload-time = "2025-11-04T13:40:21.548Z" }, - { url = "https://files.pythonhosted.org/packages/f9/22/91fbc821fa6d261b376a3f73809f907cec5ca6025642c463d3488aad22fb/pydantic_core-2.41.5-cp312-cp312-win_arm64.whl", hash = "sha256:1746d4a3d9a794cacae06a5eaaccb4b8643a131d45fbc9af23e353dc0a5ba5c3", size = 1976179, upload-time = "2025-11-04T13:40:23.393Z" }, - { url = "https://files.pythonhosted.org/packages/87/06/8806241ff1f70d9939f9af039c6c35f2360cf16e93c2ca76f184e76b1564/pydantic_core-2.41.5-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:941103c9be18ac8daf7b7adca8228f8ed6bb7a1849020f643b3a14d15b1924d9", size = 2120403, upload-time = "2025-11-04T13:40:25.248Z" }, - { url = "https://files.pythonhosted.org/packages/94/02/abfa0e0bda67faa65fef1c84971c7e45928e108fe24333c81f3bfe35d5f5/pydantic_core-2.41.5-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:112e305c3314f40c93998e567879e887a3160bb8689ef3d2c04b6cc62c33ac34", size = 1896206, upload-time = "2025-11-04T13:40:27.099Z" }, - { url = "https://files.pythonhosted.org/packages/15/df/a4c740c0943e93e6500f9eb23f4ca7ec9bf71b19e608ae5b579678c8d02f/pydantic_core-2.41.5-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0cbaad15cb0c90aa221d43c00e77bb33c93e8d36e0bf74760cd00e732d10a6a0", size = 1919307, upload-time = "2025-11-04T13:40:29.806Z" }, - { url = "https://files.pythonhosted.org/packages/9a/e3/6324802931ae1d123528988e0e86587c2072ac2e5394b4bc2bc34b61ff6e/pydantic_core-2.41.5-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:03ca43e12fab6023fc79d28ca6b39b05f794ad08ec2feccc59a339b02f2b3d33", size = 2063258, upload-time = "2025-11-04T13:40:33.544Z" }, - { url = "https://files.pythonhosted.org/packages/c9/d4/2230d7151d4957dd79c3044ea26346c148c98fbf0ee6ebd41056f2d62ab5/pydantic_core-2.41.5-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:dc799088c08fa04e43144b164feb0c13f9a0bc40503f8df3e9fde58a3c0c101e", size = 2214917, upload-time = "2025-11-04T13:40:35.479Z" }, - { url = "https://files.pythonhosted.org/packages/e6/9f/eaac5df17a3672fef0081b6c1bb0b82b33ee89aa5cec0d7b05f52fd4a1fa/pydantic_core-2.41.5-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:97aeba56665b4c3235a0e52b2c2f5ae9cd071b8a8310ad27bddb3f7fb30e9aa2", size = 2332186, upload-time = "2025-11-04T13:40:37.436Z" }, - { url = "https://files.pythonhosted.org/packages/cf/4e/35a80cae583a37cf15604b44240e45c05e04e86f9cfd766623149297e971/pydantic_core-2.41.5-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:406bf18d345822d6c21366031003612b9c77b3e29ffdb0f612367352aab7d586", size = 2073164, upload-time = "2025-11-04T13:40:40.289Z" }, - { url = "https://files.pythonhosted.org/packages/bf/e3/f6e262673c6140dd3305d144d032f7bd5f7497d3871c1428521f19f9efa2/pydantic_core-2.41.5-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:b93590ae81f7010dbe380cdeab6f515902ebcbefe0b9327cc4804d74e93ae69d", size = 2179146, upload-time = "2025-11-04T13:40:42.809Z" }, - { url = "https://files.pythonhosted.org/packages/75/c7/20bd7fc05f0c6ea2056a4565c6f36f8968c0924f19b7d97bbfea55780e73/pydantic_core-2.41.5-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:01a3d0ab748ee531f4ea6c3e48ad9dac84ddba4b0d82291f87248f2f9de8d740", size = 2137788, upload-time = "2025-11-04T13:40:44.752Z" }, - { url = "https://files.pythonhosted.org/packages/3a/8d/34318ef985c45196e004bc46c6eab2eda437e744c124ef0dbe1ff2c9d06b/pydantic_core-2.41.5-cp313-cp313-musllinux_1_1_armv7l.whl", hash = "sha256:6561e94ba9dacc9c61bce40e2d6bdc3bfaa0259d3ff36ace3b1e6901936d2e3e", size = 2340133, upload-time = "2025-11-04T13:40:46.66Z" }, - { url = "https://files.pythonhosted.org/packages/9c/59/013626bf8c78a5a5d9350d12e7697d3d4de951a75565496abd40ccd46bee/pydantic_core-2.41.5-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:915c3d10f81bec3a74fbd4faebe8391013ba61e5a1a8d48c4455b923bdda7858", size = 2324852, upload-time = "2025-11-04T13:40:48.575Z" }, - { url = "https://files.pythonhosted.org/packages/1a/d9/c248c103856f807ef70c18a4f986693a46a8ffe1602e5d361485da502d20/pydantic_core-2.41.5-cp313-cp313-win32.whl", hash = "sha256:650ae77860b45cfa6e2cdafc42618ceafab3a2d9a3811fcfbd3bbf8ac3c40d36", size = 1994679, upload-time = "2025-11-04T13:40:50.619Z" }, - { url = "https://files.pythonhosted.org/packages/9e/8b/341991b158ddab181cff136acd2552c9f35bd30380422a639c0671e99a91/pydantic_core-2.41.5-cp313-cp313-win_amd64.whl", hash = "sha256:79ec52ec461e99e13791ec6508c722742ad745571f234ea6255bed38c6480f11", size = 2019766, upload-time = "2025-11-04T13:40:52.631Z" }, - { url = "https://files.pythonhosted.org/packages/73/7d/f2f9db34af103bea3e09735bb40b021788a5e834c81eedb541991badf8f5/pydantic_core-2.41.5-cp313-cp313-win_arm64.whl", hash = "sha256:3f84d5c1b4ab906093bdc1ff10484838aca54ef08de4afa9de0f5f14d69639cd", size = 1981005, upload-time = "2025-11-04T13:40:54.734Z" }, - { url = "https://files.pythonhosted.org/packages/ea/28/46b7c5c9635ae96ea0fbb779e271a38129df2550f763937659ee6c5dbc65/pydantic_core-2.41.5-cp314-cp314-macosx_10_12_x86_64.whl", hash = "sha256:3f37a19d7ebcdd20b96485056ba9e8b304e27d9904d233d7b1015db320e51f0a", size = 2119622, upload-time = "2025-11-04T13:40:56.68Z" }, - { url = "https://files.pythonhosted.org/packages/74/1a/145646e5687e8d9a1e8d09acb278c8535ebe9e972e1f162ed338a622f193/pydantic_core-2.41.5-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:1d1d9764366c73f996edd17abb6d9d7649a7eb690006ab6adbda117717099b14", size = 1891725, upload-time = "2025-11-04T13:40:58.807Z" }, - { url = "https://files.pythonhosted.org/packages/23/04/e89c29e267b8060b40dca97bfc64a19b2a3cf99018167ea1677d96368273/pydantic_core-2.41.5-cp314-cp314-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:25e1c2af0fce638d5f1988b686f3b3ea8cd7de5f244ca147c777769e798a9cd1", size = 1915040, upload-time = "2025-11-04T13:41:00.853Z" }, - { url = "https://files.pythonhosted.org/packages/84/a3/15a82ac7bd97992a82257f777b3583d3e84bdb06ba6858f745daa2ec8a85/pydantic_core-2.41.5-cp314-cp314-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:506d766a8727beef16b7adaeb8ee6217c64fc813646b424d0804d67c16eddb66", size = 2063691, upload-time = "2025-11-04T13:41:03.504Z" }, - { url = "https://files.pythonhosted.org/packages/74/9b/0046701313c6ef08c0c1cf0e028c67c770a4e1275ca73131563c5f2a310a/pydantic_core-2.41.5-cp314-cp314-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:4819fa52133c9aa3c387b3328f25c1facc356491e6135b459f1de698ff64d869", size = 2213897, upload-time = "2025-11-04T13:41:05.804Z" }, - { url = "https://files.pythonhosted.org/packages/8a/cd/6bac76ecd1b27e75a95ca3a9a559c643b3afcd2dd62086d4b7a32a18b169/pydantic_core-2.41.5-cp314-cp314-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:2b761d210c9ea91feda40d25b4efe82a1707da2ef62901466a42492c028553a2", size = 2333302, upload-time = "2025-11-04T13:41:07.809Z" }, - { url = "https://files.pythonhosted.org/packages/4c/d2/ef2074dc020dd6e109611a8be4449b98cd25e1b9b8a303c2f0fca2f2bcf7/pydantic_core-2.41.5-cp314-cp314-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:22f0fb8c1c583a3b6f24df2470833b40207e907b90c928cc8d3594b76f874375", size = 2064877, upload-time = "2025-11-04T13:41:09.827Z" }, - { url = "https://files.pythonhosted.org/packages/18/66/e9db17a9a763d72f03de903883c057b2592c09509ccfe468187f2a2eef29/pydantic_core-2.41.5-cp314-cp314-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:2782c870e99878c634505236d81e5443092fba820f0373997ff75f90f68cd553", size = 2180680, upload-time = "2025-11-04T13:41:12.379Z" }, - { url = "https://files.pythonhosted.org/packages/d3/9e/3ce66cebb929f3ced22be85d4c2399b8e85b622db77dad36b73c5387f8f8/pydantic_core-2.41.5-cp314-cp314-musllinux_1_1_aarch64.whl", hash = "sha256:0177272f88ab8312479336e1d777f6b124537d47f2123f89cb37e0accea97f90", size = 2138960, upload-time = "2025-11-04T13:41:14.627Z" }, - { url = "https://files.pythonhosted.org/packages/a6/62/205a998f4327d2079326b01abee48e502ea739d174f0a89295c481a2272e/pydantic_core-2.41.5-cp314-cp314-musllinux_1_1_armv7l.whl", hash = "sha256:63510af5e38f8955b8ee5687740d6ebf7c2a0886d15a6d65c32814613681bc07", size = 2339102, upload-time = "2025-11-04T13:41:16.868Z" }, - { url = "https://files.pythonhosted.org/packages/3c/0d/f05e79471e889d74d3d88f5bd20d0ed189ad94c2423d81ff8d0000aab4ff/pydantic_core-2.41.5-cp314-cp314-musllinux_1_1_x86_64.whl", hash = "sha256:e56ba91f47764cc14f1daacd723e3e82d1a89d783f0f5afe9c364b8bb491ccdb", size = 2326039, upload-time = "2025-11-04T13:41:18.934Z" }, - { url = "https://files.pythonhosted.org/packages/ec/e1/e08a6208bb100da7e0c4b288eed624a703f4d129bde2da475721a80cab32/pydantic_core-2.41.5-cp314-cp314-win32.whl", hash = "sha256:aec5cf2fd867b4ff45b9959f8b20ea3993fc93e63c7363fe6851424c8a7e7c23", size = 1995126, upload-time = "2025-11-04T13:41:21.418Z" }, - { url = "https://files.pythonhosted.org/packages/48/5d/56ba7b24e9557f99c9237e29f5c09913c81eeb2f3217e40e922353668092/pydantic_core-2.41.5-cp314-cp314-win_amd64.whl", hash = "sha256:8e7c86f27c585ef37c35e56a96363ab8de4e549a95512445b85c96d3e2f7c1bf", size = 2015489, upload-time = "2025-11-04T13:41:24.076Z" }, - { url = "https://files.pythonhosted.org/packages/4e/bb/f7a190991ec9e3e0ba22e4993d8755bbc4a32925c0b5b42775c03e8148f9/pydantic_core-2.41.5-cp314-cp314-win_arm64.whl", hash = "sha256:e672ba74fbc2dc8eea59fb6d4aed6845e6905fc2a8afe93175d94a83ba2a01a0", size = 1977288, upload-time = "2025-11-04T13:41:26.33Z" }, - { url = "https://files.pythonhosted.org/packages/92/ed/77542d0c51538e32e15afe7899d79efce4b81eee631d99850edc2f5e9349/pydantic_core-2.41.5-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:8566def80554c3faa0e65ac30ab0932b9e3a5cd7f8323764303d468e5c37595a", size = 2120255, upload-time = "2025-11-04T13:41:28.569Z" }, - { url = "https://files.pythonhosted.org/packages/bb/3d/6913dde84d5be21e284439676168b28d8bbba5600d838b9dca99de0fad71/pydantic_core-2.41.5-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:b80aa5095cd3109962a298ce14110ae16b8c1aece8b72f9dafe81cf597ad80b3", size = 1863760, upload-time = "2025-11-04T13:41:31.055Z" }, - { url = "https://files.pythonhosted.org/packages/5a/f0/e5e6b99d4191da102f2b0eb9687aaa7f5bea5d9964071a84effc3e40f997/pydantic_core-2.41.5-cp314-cp314t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3006c3dd9ba34b0c094c544c6006cc79e87d8612999f1a5d43b769b89181f23c", size = 1878092, upload-time = "2025-11-04T13:41:33.21Z" }, - { url = "https://files.pythonhosted.org/packages/71/48/36fb760642d568925953bcc8116455513d6e34c4beaa37544118c36aba6d/pydantic_core-2.41.5-cp314-cp314t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:72f6c8b11857a856bcfa48c86f5368439f74453563f951e473514579d44aa612", size = 2053385, upload-time = "2025-11-04T13:41:35.508Z" }, - { url = "https://files.pythonhosted.org/packages/20/25/92dc684dd8eb75a234bc1c764b4210cf2646479d54b47bf46061657292a8/pydantic_core-2.41.5-cp314-cp314t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:5cb1b2f9742240e4bb26b652a5aeb840aa4b417c7748b6f8387927bc6e45e40d", size = 2218832, upload-time = "2025-11-04T13:41:37.732Z" }, - { url = "https://files.pythonhosted.org/packages/e2/09/f53e0b05023d3e30357d82eb35835d0f6340ca344720a4599cd663dca599/pydantic_core-2.41.5-cp314-cp314t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:bd3d54f38609ff308209bd43acea66061494157703364ae40c951f83ba99a1a9", size = 2327585, upload-time = "2025-11-04T13:41:40Z" }, - { url = "https://files.pythonhosted.org/packages/aa/4e/2ae1aa85d6af35a39b236b1b1641de73f5a6ac4d5a7509f77b814885760c/pydantic_core-2.41.5-cp314-cp314t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2ff4321e56e879ee8d2a879501c8e469414d948f4aba74a2d4593184eb326660", size = 2041078, upload-time = "2025-11-04T13:41:42.323Z" }, - { url = "https://files.pythonhosted.org/packages/cd/13/2e215f17f0ef326fc72afe94776edb77525142c693767fc347ed6288728d/pydantic_core-2.41.5-cp314-cp314t-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:d0d2568a8c11bf8225044aa94409e21da0cb09dcdafe9ecd10250b2baad531a9", size = 2173914, upload-time = "2025-11-04T13:41:45.221Z" }, - { url = "https://files.pythonhosted.org/packages/02/7a/f999a6dcbcd0e5660bc348a3991c8915ce6599f4f2c6ac22f01d7a10816c/pydantic_core-2.41.5-cp314-cp314t-musllinux_1_1_aarch64.whl", hash = "sha256:a39455728aabd58ceabb03c90e12f71fd30fa69615760a075b9fec596456ccc3", size = 2129560, upload-time = "2025-11-04T13:41:47.474Z" }, - { url = "https://files.pythonhosted.org/packages/3a/b1/6c990ac65e3b4c079a4fb9f5b05f5b013afa0f4ed6780a3dd236d2cbdc64/pydantic_core-2.41.5-cp314-cp314t-musllinux_1_1_armv7l.whl", hash = "sha256:239edca560d05757817c13dc17c50766136d21f7cd0fac50295499ae24f90fdf", size = 2329244, upload-time = "2025-11-04T13:41:49.992Z" }, - { url = "https://files.pythonhosted.org/packages/d9/02/3c562f3a51afd4d88fff8dffb1771b30cfdfd79befd9883ee094f5b6c0d8/pydantic_core-2.41.5-cp314-cp314t-musllinux_1_1_x86_64.whl", hash = "sha256:2a5e06546e19f24c6a96a129142a75cee553cc018ffee48a460059b1185f4470", size = 2331955, upload-time = "2025-11-04T13:41:54.079Z" }, - { url = "https://files.pythonhosted.org/packages/5c/96/5fb7d8c3c17bc8c62fdb031c47d77a1af698f1d7a406b0f79aaa1338f9ad/pydantic_core-2.41.5-cp314-cp314t-win32.whl", hash = "sha256:b4ececa40ac28afa90871c2cc2b9ffd2ff0bf749380fbdf57d165fd23da353aa", size = 1988906, upload-time = "2025-11-04T13:41:56.606Z" }, - { url = "https://files.pythonhosted.org/packages/22/ed/182129d83032702912c2e2d8bbe33c036f342cc735737064668585dac28f/pydantic_core-2.41.5-cp314-cp314t-win_amd64.whl", hash = "sha256:80aa89cad80b32a912a65332f64a4450ed00966111b6615ca6816153d3585a8c", size = 1981607, upload-time = "2025-11-04T13:41:58.889Z" }, - { url = "https://files.pythonhosted.org/packages/9f/ed/068e41660b832bb0b1aa5b58011dea2a3fe0ba7861ff38c4d4904c1c1a99/pydantic_core-2.41.5-cp314-cp314t-win_arm64.whl", hash = "sha256:35b44f37a3199f771c3eaa53051bc8a70cd7b54f333531c59e29fd4db5d15008", size = 1974769, upload-time = "2025-11-04T13:42:01.186Z" }, - { url = "https://files.pythonhosted.org/packages/11/72/90fda5ee3b97e51c494938a4a44c3a35a9c96c19bba12372fb9c634d6f57/pydantic_core-2.41.5-graalpy311-graalpy242_311_native-macosx_10_12_x86_64.whl", hash = "sha256:b96d5f26b05d03cc60f11a7761a5ded1741da411e7fe0909e27a5e6a0cb7b034", size = 2115441, upload-time = "2025-11-04T13:42:39.557Z" }, - { url = "https://files.pythonhosted.org/packages/1f/53/8942f884fa33f50794f119012dc6a1a02ac43a56407adaac20463df8e98f/pydantic_core-2.41.5-graalpy311-graalpy242_311_native-macosx_11_0_arm64.whl", hash = "sha256:634e8609e89ceecea15e2d61bc9ac3718caaaa71963717bf3c8f38bfde64242c", size = 1930291, upload-time = "2025-11-04T13:42:42.169Z" }, - { url = "https://files.pythonhosted.org/packages/79/c8/ecb9ed9cd942bce09fc888ee960b52654fbdbede4ba6c2d6e0d3b1d8b49c/pydantic_core-2.41.5-graalpy311-graalpy242_311_native-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:93e8740d7503eb008aa2df04d3b9735f845d43ae845e6dcd2be0b55a2da43cd2", size = 1948632, upload-time = "2025-11-04T13:42:44.564Z" }, - { url = "https://files.pythonhosted.org/packages/2e/1b/687711069de7efa6af934e74f601e2a4307365e8fdc404703afc453eab26/pydantic_core-2.41.5-graalpy311-graalpy242_311_native-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f15489ba13d61f670dcc96772e733aad1a6f9c429cc27574c6cdaed82d0146ad", size = 2138905, upload-time = "2025-11-04T13:42:47.156Z" }, - { url = "https://files.pythonhosted.org/packages/09/32/59b0c7e63e277fa7911c2fc70ccfb45ce4b98991e7ef37110663437005af/pydantic_core-2.41.5-graalpy312-graalpy250_312_native-macosx_10_12_x86_64.whl", hash = "sha256:7da7087d756b19037bc2c06edc6c170eeef3c3bafcb8f532ff17d64dc427adfd", size = 2110495, upload-time = "2025-11-04T13:42:49.689Z" }, - { url = "https://files.pythonhosted.org/packages/aa/81/05e400037eaf55ad400bcd318c05bb345b57e708887f07ddb2d20e3f0e98/pydantic_core-2.41.5-graalpy312-graalpy250_312_native-macosx_11_0_arm64.whl", hash = "sha256:aabf5777b5c8ca26f7824cb4a120a740c9588ed58df9b2d196ce92fba42ff8dc", size = 1915388, upload-time = "2025-11-04T13:42:52.215Z" }, - { url = "https://files.pythonhosted.org/packages/6e/0d/e3549b2399f71d56476b77dbf3cf8937cec5cd70536bdc0e374a421d0599/pydantic_core-2.41.5-graalpy312-graalpy250_312_native-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c007fe8a43d43b3969e8469004e9845944f1a80e6acd47c150856bb87f230c56", size = 1942879, upload-time = "2025-11-04T13:42:56.483Z" }, - { url = "https://files.pythonhosted.org/packages/f7/07/34573da085946b6a313d7c42f82f16e8920bfd730665de2d11c0c37a74b5/pydantic_core-2.41.5-graalpy312-graalpy250_312_native-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:76d0819de158cd855d1cbb8fcafdf6f5cf1eb8e470abe056d5d161106e38062b", size = 2139017, upload-time = "2025-11-04T13:42:59.471Z" }, - { url = "https://files.pythonhosted.org/packages/e6/b0/1a2aa41e3b5a4ba11420aba2d091b2d17959c8d1519ece3627c371951e73/pydantic_core-2.41.5-pp310-pypy310_pp73-macosx_10_12_x86_64.whl", hash = "sha256:b5819cd790dbf0c5eb9f82c73c16b39a65dd6dd4d1439dcdea7816ec9adddab8", size = 2103351, upload-time = "2025-11-04T13:43:02.058Z" }, - { url = "https://files.pythonhosted.org/packages/a4/ee/31b1f0020baaf6d091c87900ae05c6aeae101fa4e188e1613c80e4f1ea31/pydantic_core-2.41.5-pp310-pypy310_pp73-macosx_11_0_arm64.whl", hash = "sha256:5a4e67afbc95fa5c34cf27d9089bca7fcab4e51e57278d710320a70b956d1b9a", size = 1925363, upload-time = "2025-11-04T13:43:05.159Z" }, - { url = "https://files.pythonhosted.org/packages/e1/89/ab8e86208467e467a80deaca4e434adac37b10a9d134cd2f99b28a01e483/pydantic_core-2.41.5-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ece5c59f0ce7d001e017643d8d24da587ea1f74f6993467d85ae8a5ef9d4f42b", size = 2135615, upload-time = "2025-11-04T13:43:08.116Z" }, - { url = "https://files.pythonhosted.org/packages/99/0a/99a53d06dd0348b2008f2f30884b34719c323f16c3be4e6cc1203b74a91d/pydantic_core-2.41.5-pp310-pypy310_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:16f80f7abe3351f8ea6858914ddc8c77e02578544a0ebc15b4c2e1a0e813b0b2", size = 2175369, upload-time = "2025-11-04T13:43:12.49Z" }, - { url = "https://files.pythonhosted.org/packages/6d/94/30ca3b73c6d485b9bb0bc66e611cff4a7138ff9736b7e66bcf0852151636/pydantic_core-2.41.5-pp310-pypy310_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:33cb885e759a705b426baada1fe68cbb0a2e68e34c5d0d0289a364cf01709093", size = 2144218, upload-time = "2025-11-04T13:43:15.431Z" }, - { url = "https://files.pythonhosted.org/packages/87/57/31b4f8e12680b739a91f472b5671294236b82586889ef764b5fbc6669238/pydantic_core-2.41.5-pp310-pypy310_pp73-musllinux_1_1_armv7l.whl", hash = "sha256:c8d8b4eb992936023be7dee581270af5c6e0697a8559895f527f5b7105ecd36a", size = 2329951, upload-time = "2025-11-04T13:43:18.062Z" }, - { url = "https://files.pythonhosted.org/packages/7d/73/3c2c8edef77b8f7310e6fb012dbc4b8551386ed575b9eb6fb2506e28a7eb/pydantic_core-2.41.5-pp310-pypy310_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:242a206cd0318f95cd21bdacff3fcc3aab23e79bba5cac3db5a841c9ef9c6963", size = 2318428, upload-time = "2025-11-04T13:43:20.679Z" }, - { url = "https://files.pythonhosted.org/packages/2f/02/8559b1f26ee0d502c74f9cca5c0d2fd97e967e083e006bbbb4e97f3a043a/pydantic_core-2.41.5-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:d3a978c4f57a597908b7e697229d996d77a6d3c94901e9edee593adada95ce1a", size = 2147009, upload-time = "2025-11-04T13:43:23.286Z" }, - { url = "https://files.pythonhosted.org/packages/5f/9b/1b3f0e9f9305839d7e84912f9e8bfbd191ed1b1ef48083609f0dabde978c/pydantic_core-2.41.5-pp311-pypy311_pp73-macosx_10_12_x86_64.whl", hash = "sha256:b2379fa7ed44ddecb5bfe4e48577d752db9fc10be00a6b7446e9663ba143de26", size = 2101980, upload-time = "2025-11-04T13:43:25.97Z" }, - { url = "https://files.pythonhosted.org/packages/a4/ed/d71fefcb4263df0da6a85b5d8a7508360f2f2e9b3bf5814be9c8bccdccc1/pydantic_core-2.41.5-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:266fb4cbf5e3cbd0b53669a6d1b039c45e3ce651fd5442eff4d07c2cc8d66808", size = 1923865, upload-time = "2025-11-04T13:43:28.763Z" }, - { url = "https://files.pythonhosted.org/packages/ce/3a/626b38db460d675f873e4444b4bb030453bbe7b4ba55df821d026a0493c4/pydantic_core-2.41.5-pp311-pypy311_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:58133647260ea01e4d0500089a8c4f07bd7aa6ce109682b1426394988d8aaacc", size = 2134256, upload-time = "2025-11-04T13:43:31.71Z" }, - { url = "https://files.pythonhosted.org/packages/83/d9/8412d7f06f616bbc053d30cb4e5f76786af3221462ad5eee1f202021eb4e/pydantic_core-2.41.5-pp311-pypy311_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:287dad91cfb551c363dc62899a80e9e14da1f0e2b6ebde82c806612ca2a13ef1", size = 2174762, upload-time = "2025-11-04T13:43:34.744Z" }, - { url = "https://files.pythonhosted.org/packages/55/4c/162d906b8e3ba3a99354e20faa1b49a85206c47de97a639510a0e673f5da/pydantic_core-2.41.5-pp311-pypy311_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:03b77d184b9eb40240ae9fd676ca364ce1085f203e1b1256f8ab9984dca80a84", size = 2143141, upload-time = "2025-11-04T13:43:37.701Z" }, - { url = "https://files.pythonhosted.org/packages/1f/f2/f11dd73284122713f5f89fc940f370d035fa8e1e078d446b3313955157fe/pydantic_core-2.41.5-pp311-pypy311_pp73-musllinux_1_1_armv7l.whl", hash = "sha256:a668ce24de96165bb239160b3d854943128f4334822900534f2fe947930e5770", size = 2330317, upload-time = "2025-11-04T13:43:40.406Z" }, - { url = "https://files.pythonhosted.org/packages/88/9d/b06ca6acfe4abb296110fb1273a4d848a0bfb2ff65f3ee92127b3244e16b/pydantic_core-2.41.5-pp311-pypy311_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:f14f8f046c14563f8eb3f45f499cc658ab8d10072961e07225e507adb700e93f", size = 2316992, upload-time = "2025-11-04T13:43:43.602Z" }, - { url = "https://files.pythonhosted.org/packages/36/c7/cfc8e811f061c841d7990b0201912c3556bfeb99cdcb7ed24adc8d6f8704/pydantic_core-2.41.5-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:56121965f7a4dc965bff783d70b907ddf3d57f6eba29b6d2e5dabfaf07799c51", size = 2145302, upload-time = "2025-11-04T13:43:46.64Z" }, +sdist = { url = "https://files.pythonhosted.org/packages/9d/56/921726b776ace8d8f5db44c4ef961006580d91dc52b803c489fafd1aa249/pydantic_core-2.46.4.tar.gz", hash = "sha256:62f875393d7f270851f20523dd2e29f082bcc82292d66db2b64ea71f64b6e1c1", size = 471464, upload-time = "2026-05-06T13:37:06.98Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e7/08/f1ba952f1c8ae5581c70fa9c6da89f247b83e3dd8c09c035d5d7931fc23d/pydantic_core-2.46.4-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:a396dcc17e5a0b164dbe026896245a4fa9ff402edca1dff0be3d53a517f74de4", size = 2113146, upload-time = "2026-05-06T13:37:36.537Z" }, + { url = "https://files.pythonhosted.org/packages/56/c6/65f646c7ff09bd257f660434adb45c4dfcbbcebcc030562fecf6f5bf887d/pydantic_core-2.46.4-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:da4b951fe36dc7c3a1ccb4e3cd1747c3542b8c9ceede8fc86cae054e764485f5", size = 1949769, upload-time = "2026-05-06T13:37:46.365Z" }, + { url = "https://files.pythonhosted.org/packages/64/ba/bfb1d928fd5b49e1258935ff104ae356e9fd89384a55bf9f847e9193ad40/pydantic_core-2.46.4-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bb63e0198ca18aad131c089b9204c23079c3afa95487e561f4c522d519e55aba", size = 1974958, upload-time = "2026-05-06T13:37:28.611Z" }, + { url = "https://files.pythonhosted.org/packages/4e/74/76223bfb117b64af743c9b6670d1364516f5c0604f96b48f3272f6af6cc6/pydantic_core-2.46.4-cp310-cp310-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:f47286a97f0bc9b8859519809077b91b2cefe4ae47fcbf5e466a009c1c5d742b", size = 2042118, upload-time = "2026-05-06T13:36:55.216Z" }, + { url = "https://files.pythonhosted.org/packages/cb/7b/848732968bc8f48f3187542f08358b9d842db564147b256669426ebb1652/pydantic_core-2.46.4-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:905a0ed8ea6f2d61c1738835f99b699348d7857379083e5fc497fa0c967a407c", size = 2222876, upload-time = "2026-05-06T13:38:25.455Z" }, + { url = "https://files.pythonhosted.org/packages/b5/2f/e90b63ee2e14bd8d3db8f705a6d75d64e6ee1b7c2c8833747ce706e1e0ce/pydantic_core-2.46.4-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:ea793e075b70290d89d8142074262885d3f7da19634845135751bd6344f73b50", size = 2286703, upload-time = "2026-05-06T13:37:53.304Z" }, + { url = "https://files.pythonhosted.org/packages/ba/1e/acc4d70f88a0a277e4a1fa77ebb985ceabaf900430f875bf9338e11c9420/pydantic_core-2.46.4-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:395aebd9183f9d112f569aeb5b2214d1a10a33bec8456447f7fbdfa51d38d4cd", size = 2092042, upload-time = "2026-05-06T13:38:46.981Z" }, + { url = "https://files.pythonhosted.org/packages/a9/da/0a422b57bf8504102bf3c4ccea9c41bab5a5cee6a54650acf8faf67f5a24/pydantic_core-2.46.4-cp310-cp310-manylinux_2_31_riscv64.whl", hash = "sha256:b078afbc25f3a1436c7a1d2cd3e322497ee99615ba97c563566fdf46aff1ee01", size = 2117231, upload-time = "2026-05-06T13:39:23.146Z" }, + { url = "https://files.pythonhosted.org/packages/bd/2a/2ac13c3af305843e23c5078c53d135656b3f05a2fd78cb7bbbb12e97b473/pydantic_core-2.46.4-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:f747929cf940cddb5b3668a390056ddd5ba2e5010615ea2dcf4f9c4f3ab8791d", size = 2168388, upload-time = "2026-05-06T13:40:08.06Z" }, + { url = "https://files.pythonhosted.org/packages/72/04/2beacf7e1607e93eefe4aed1b4709f079b905fb77530179d4f7c71745f22/pydantic_core-2.46.4-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:daa27d92c36f24388fe3ad306b174781c747627f134452e4f128ea00ce1fe8c4", size = 2184769, upload-time = "2026-05-06T13:38:13.901Z" }, + { url = "https://files.pythonhosted.org/packages/9e/29/d2b9fd9f539133548eaf622c06a4ce176cb46ac59f32d0359c4abc0de047/pydantic_core-2.46.4-cp310-cp310-musllinux_1_1_armv7l.whl", hash = "sha256:19e51f073cd3df251856a8a4189fbdf1de4012c3ebacfb1884f94f1eb406079f", size = 2319312, upload-time = "2026-05-06T13:39:08.24Z" }, + { url = "https://files.pythonhosted.org/packages/7c/af/0f7a5b85fec6075bea96e3ef9187de38fccced0de92c1e7feda8d5cc7bb9/pydantic_core-2.46.4-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:c1747f85cee84c26985853c6f3d9bd3e75da5212912443fa111c113b9c246f39", size = 2361817, upload-time = "2026-05-06T13:38:43.2Z" }, + { url = "https://files.pythonhosted.org/packages/25/a4/73363fec545fd3ec025490bdda2743c56d0dd5b6266b1a53bbe9e4265375/pydantic_core-2.46.4-cp310-cp310-win32.whl", hash = "sha256:2f84c03c8607173d16b5a854ec68a2f9079ae03237a54fb506d13af47e1d018d", size = 1987085, upload-time = "2026-05-06T13:39:25.497Z" }, + { url = "https://files.pythonhosted.org/packages/01/aa/62f082da2c91fac1c234bc9ee0066257ce83f0604abd72e4c9d5991f2d84/pydantic_core-2.46.4-cp310-cp310-win_amd64.whl", hash = "sha256:8358a950c8909158e3df31538a7e4edc2d7265a7c54b47f0864d9e5bae9dcebf", size = 2074311, upload-time = "2026-05-06T13:39:59.922Z" }, + { url = "https://files.pythonhosted.org/packages/5c/fa/6d7708d2cfc1a832acb6aeb0cd16e801902df8a0f583bb3b4b527fde022e/pydantic_core-2.46.4-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:0e96592440881c74a213e5ad528e2b24d3d4f940de2766bed9010ab1d9e51594", size = 2111872, upload-time = "2026-05-06T13:40:27.596Z" }, + { url = "https://files.pythonhosted.org/packages/ae/6f/aa064a3e74b5745afbdf250594f38e7ead05e2d651bcb35994b9417a0d4d/pydantic_core-2.46.4-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:e0d65b8c354be7fb5f720c3caa8bc940bc2d20ce749c8e06135f07f8ed95dd7c", size = 1948255, upload-time = "2026-05-06T13:39:12.574Z" }, + { url = "https://files.pythonhosted.org/packages/43/3a/41114a9f7569b84b4d84e7a018c57c56347dac30c0d4a872946ec4e36c46/pydantic_core-2.46.4-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7bfb192b3f4b9e8a89b6277b6ce787564f62cfd272055f6e685726b111dc7826", size = 1972827, upload-time = "2026-05-06T13:38:19.841Z" }, + { url = "https://files.pythonhosted.org/packages/ef/25/1ab42e8048fe551934d9884e8d64daa7e990ad386f310a15981aeb6a5b08/pydantic_core-2.46.4-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:9037063db01f09b09e237c282b6792bd4da634b5402c4e7f0c61effed7701a04", size = 2041051, upload-time = "2026-05-06T13:38:10.447Z" }, + { url = "https://files.pythonhosted.org/packages/94/c2/1a934597ddf08da410385b3b7aae91956a5a76c635effef456074fad7e88/pydantic_core-2.46.4-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:fc010ab034c8c7452522748bf937df58020d256ccae0874463d1f4d01758af8e", size = 2221314, upload-time = "2026-05-06T13:40:13.089Z" }, + { url = "https://files.pythonhosted.org/packages/02/6d/9e8ad178c9c4df27ad3c8f25d1fe2a7ab0d2ba0559fad4aee5d3d1f16771/pydantic_core-2.46.4-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:8c5dac79fa1614d1e06ca695109c6105923bd9c7d1d6c918d4e637b7e6b32fd3", size = 2285146, upload-time = "2026-05-06T13:38:59.224Z" }, + { url = "https://files.pythonhosted.org/packages/80/50/540cd3aeefc041beb111125c4bff779831a2111fc6b15a9138cda277d32c/pydantic_core-2.46.4-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f9fa868638bf362d3d138ea55829cefb3d5f4b0d7f142234382a15e2485dbec4", size = 2089685, upload-time = "2026-05-06T13:38:17.762Z" }, + { url = "https://files.pythonhosted.org/packages/6b/a4/b440ad35f05f6a38f89fa0f149accb3f0e02be94ca5e15f3c449a61b4bc9/pydantic_core-2.46.4-cp311-cp311-manylinux_2_31_riscv64.whl", hash = "sha256:17299feefe090f2caa5b8e37222bb5f663e4935a8bfa6931d4102e5df1a9f398", size = 2115420, upload-time = "2026-05-06T13:37:58.195Z" }, + { url = "https://files.pythonhosted.org/packages/99/61/de4f55db8dfd57bfdfa9a12ec90fe1b57c4f41062f7ca86f08586b3e0ac0/pydantic_core-2.46.4-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:4c63ebc82684aa89d9a3bcbd13d515b3be44250dc68dd3bd81526c1cb31286c3", size = 2165122, upload-time = "2026-05-06T13:37:01.167Z" }, + { url = "https://files.pythonhosted.org/packages/f7/52/7c529d7bdb2d1068bd52f51fe32572c8301f9a4febf1948f10639f1436f5/pydantic_core-2.46.4-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:aaa2a54443eff1950ba5ddc6b6ccda0d9c84a364276a62f969bdf2a390650848", size = 2182573, upload-time = "2026-05-06T13:38:45.04Z" }, + { url = "https://files.pythonhosted.org/packages/37/b3/7c40325848ba78247f2812dcf9c7274e38cd801820ca6dd9fe63bcfb0eb4/pydantic_core-2.46.4-cp311-cp311-musllinux_1_1_armv7l.whl", hash = "sha256:18e5ceec2ab67e6d5f1a9085e5a24c9c4e2ac4545730bfe668680bca05e555f3", size = 2317139, upload-time = "2026-05-06T13:37:15.539Z" }, + { url = "https://files.pythonhosted.org/packages/d9/37/f913f81a657c865b75da6c0dbed79876073c2a43b5bd9edbe8da785e4d49/pydantic_core-2.46.4-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:a0f62d0a58f4e7da165457e995725421e0064f2255d8eccebc49f41bbc23b109", size = 2360433, upload-time = "2026-05-06T13:37:30.099Z" }, + { url = "https://files.pythonhosted.org/packages/c4/67/6acaa1be2567f9256b056d8477158cac7240813956ce86e49deae8e173b4/pydantic_core-2.46.4-cp311-cp311-win32.whl", hash = "sha256:041bde0a48fd37cf71cab1c9d56d3e8625a3793fef1f7dd232b3ff37e978ecda", size = 1985513, upload-time = "2026-05-06T13:38:15.669Z" }, + { url = "https://files.pythonhosted.org/packages/aa/e6/c505f83dfeda9a2e5c995cfd872949e4d05e12f7feb3dca72f633daefa94/pydantic_core-2.46.4-cp311-cp311-win_amd64.whl", hash = "sha256:6f2eeda33a839975441c86a4119e1383c50b47faf0cbb5176985565c6bb02c33", size = 2071114, upload-time = "2026-05-06T13:40:35.416Z" }, + { url = "https://files.pythonhosted.org/packages/0f/da/7a263a96d965d9d0df5e8de8a475f33495451117035b09acb110288c381f/pydantic_core-2.46.4-cp311-cp311-win_arm64.whl", hash = "sha256:14f4c5d6db102bd796a627bbb3a17b4cf4574b9ae861d8b7c9a9661c6dd3362d", size = 2044298, upload-time = "2026-05-06T13:38:29.754Z" }, + { url = "https://files.pythonhosted.org/packages/ce/8c/af022f0af448d7747c5154288d46b5f2bc5f17366eaa0e23e9aa04d59f3b/pydantic_core-2.46.4-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:3245406455a5d98187ec35530fd772b1d799b26667980872c8d4614991e2c4a2", size = 2106158, upload-time = "2026-05-06T13:38:57.215Z" }, + { url = "https://files.pythonhosted.org/packages/19/95/6195171e385007300f0f5574592e467c568becce2d937a0b6804f218bc49/pydantic_core-2.46.4-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:962ccbab7b642487b1d8b7df90ef677e03134cf1fd8880bf698649b22a69371f", size = 1951724, upload-time = "2026-05-06T13:37:02.697Z" }, + { url = "https://files.pythonhosted.org/packages/8e/bc/f47d1ff9cbb1620e1b5b697eef06010035735f07820180e74178226b27b3/pydantic_core-2.46.4-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8233f2947cf85404441fd7e0085f53b10c93e0ee78611099b5c7237e36aacbf7", size = 1975742, upload-time = "2026-05-06T13:37:09.448Z" }, + { url = "https://files.pythonhosted.org/packages/5b/11/9b9a5b0306345664a2da6410877af6e8082481b5884b3ddd78d47c6013ce/pydantic_core-2.46.4-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:3a233125ac121aa3ffba9a2b59edfc4a985a76092dc8279586ab4b71390875e7", size = 2052418, upload-time = "2026-05-06T13:37:38.234Z" }, + { url = "https://files.pythonhosted.org/packages/f1/b7/a65fec226f5d78fc39f4a13c4cc0c768c22b113438f60c14adc9d2865038/pydantic_core-2.46.4-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:5b712b53160b79a5850310b912a5ef8e57e56947c8ad690c227f5c9d7e561712", size = 2232274, upload-time = "2026-05-06T13:38:27.753Z" }, + { url = "https://files.pythonhosted.org/packages/68/f0/92039db98b907ef49269a8271f67db9cb78ae2fc68062ef7e4e77adb5f61/pydantic_core-2.46.4-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:9401557acd873c3a7f3eb9383edef8ac4968f9510e340f4808d427e75667e7b4", size = 2309940, upload-time = "2026-05-06T13:38:05.353Z" }, + { url = "https://files.pythonhosted.org/packages/5f/97/2aab507d3d00ca626e8e57c1eac6a79e4e5fbcc63eb99733ff55d1717f65/pydantic_core-2.46.4-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:926c9541b14b12b1681dca8a0b75feb510b06c6341b70a8e500c2fdcff837cce", size = 2094516, upload-time = "2026-05-06T13:39:10.577Z" }, + { url = "https://files.pythonhosted.org/packages/22/37/a8aca44d40d737dde2bc05b3c6c07dff0de07ce6f82e9f3167aeaf4d5dea/pydantic_core-2.46.4-cp312-cp312-manylinux_2_31_riscv64.whl", hash = "sha256:56cb4851bcaf3d117eddcef4fe66afd750a50274b0da8e22be256d10e5611987", size = 2136854, upload-time = "2026-05-06T13:40:22.59Z" }, + { url = "https://files.pythonhosted.org/packages/24/99/fcef1b79238c06a8cbec70819ac722ba76e02bc8ada9b0fd66eba40da01b/pydantic_core-2.46.4-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:c68fcd102d71ea85c5b2dfac3f4f8476eff42a9e078fd5faefff6d145063536b", size = 2180306, upload-time = "2026-05-06T13:40:10.666Z" }, + { url = "https://files.pythonhosted.org/packages/ae/6c/fc44000918855b42779d007ae63b0532794739027b2f417321cddbc44f6a/pydantic_core-2.46.4-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:b2f69dec1725e79a012d920df1707de5caf7ed5e08f3be4435e25803efc47458", size = 2190044, upload-time = "2026-05-06T13:40:43.231Z" }, + { url = "https://files.pythonhosted.org/packages/6b/65/d9cadc9f1920d7a127ad2edba16c1db7916e59719285cd6c94600b0080ba/pydantic_core-2.46.4-cp312-cp312-musllinux_1_1_armv7l.whl", hash = "sha256:8d0820e8192167f80d88d64038e609c31452eeca865b4e1d9950a27a4609b00b", size = 2329133, upload-time = "2026-05-06T13:39:57.365Z" }, + { url = "https://files.pythonhosted.org/packages/d0/cf/c873d91679f3a30bcf5e7ac280ce5573483e72295307685120d0d5ad3416/pydantic_core-2.46.4-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:fbdb89b3e1c94a30cc5edfce477c6e6a5dc4d8f84665b455c27582f211a1c72c", size = 2374464, upload-time = "2026-05-06T13:38:06.976Z" }, + { url = "https://files.pythonhosted.org/packages/47/bd/6f2fc8188f31bf10590f1e98e7b306336161fac930a8c514cd7bd828c7dc/pydantic_core-2.46.4-cp312-cp312-win32.whl", hash = "sha256:9aa768456404a8bf48a4406685ac2bec8e72b62c69313734fa3b73cf33b3a894", size = 1974823, upload-time = "2026-05-06T13:40:47.985Z" }, + { url = "https://files.pythonhosted.org/packages/40/8c/985c1d41ea1107c2534abd9870e4ed5c8e7669b5c308297835c001e7a1c4/pydantic_core-2.46.4-cp312-cp312-win_amd64.whl", hash = "sha256:e9c26f834c65f5752f3f06cb08cb86a913ceb7274d0db6e267808a708b46bc89", size = 2072919, upload-time = "2026-05-06T13:39:21.153Z" }, + { url = "https://files.pythonhosted.org/packages/c4/ba/f463d006e0c47373ca7ec5e1a261c59dc01ef4d62b2657af925fb0deee3a/pydantic_core-2.46.4-cp312-cp312-win_arm64.whl", hash = "sha256:4fc73cb559bdb54b1134a706a2802a4cddd27a0633f5abb7e53056268751ac6a", size = 2027604, upload-time = "2026-05-06T13:39:03.753Z" }, + { url = "https://files.pythonhosted.org/packages/51/a2/5d30b469c5267a17b39dec53208222f76a8d351dfac4af661888c5aee77d/pydantic_core-2.46.4-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:5d5902252db0d3cedf8d4a1bc68f70eeb430f7e4c7104c8c476753519b423008", size = 2106306, upload-time = "2026-05-06T13:37:48.029Z" }, + { url = "https://files.pythonhosted.org/packages/c1/81/4fa520eaffa8bd7d1525e644cd6d39e7d60b1592bc5b516693c7340b50f1/pydantic_core-2.46.4-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:c94f0688e7b8d0a67abf40e57a7eaaecd17cc9586706a31b76c031f63df052b4", size = 1951906, upload-time = "2026-05-06T13:37:17.012Z" }, + { url = "https://files.pythonhosted.org/packages/03/d5/fd02da45b659668b05923b17ba3a0100a0a3d5541e3bd8fcc4ecb711309e/pydantic_core-2.46.4-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f027324c56cd5406ca49c124b0db10e56c69064fec039acc571c29020cc87c76", size = 1976802, upload-time = "2026-05-06T13:37:35.113Z" }, + { url = "https://files.pythonhosted.org/packages/21/f2/95727e1368be3d3ed485eaab7adbd7dda408f33f7a36e8b48e0144002b91/pydantic_core-2.46.4-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:e739fee756ba1010f8bcccb534252e85a35fe45ae92c295a06059ce58b74ccd3", size = 2052446, upload-time = "2026-05-06T13:37:12.313Z" }, + { url = "https://files.pythonhosted.org/packages/9c/86/5d99feea3f77c7234b8718075b23db11532773c1a0dbd9b9490215dc2eeb/pydantic_core-2.46.4-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:9d56801be94b86a9da183e5f3766e6310752b99ff647e38b09a9500d88e46e76", size = 2232757, upload-time = "2026-05-06T13:39:01.149Z" }, + { url = "https://files.pythonhosted.org/packages/d2/3a/508ac615935ef7588cf6d9e9b91309fdc2da751af865e02a9098de88258c/pydantic_core-2.46.4-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:2412e734dcb48da14d4e4006b82b46b74f2518b8a26ee7e58c6844a6cd6d03c4", size = 2309275, upload-time = "2026-05-06T13:37:41.406Z" }, + { url = "https://files.pythonhosted.org/packages/07/f8/41db9de19d7987d6b04715a02b3b40aea467000275d9d758ffaa31af7d50/pydantic_core-2.46.4-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9551187363ffc0de2a00b2e47c25aeaeb1020b69b668762966df15fc5659dd5a", size = 2094467, upload-time = "2026-05-06T13:39:18.847Z" }, + { url = "https://files.pythonhosted.org/packages/2c/e2/f35033184cb11d0052daf4416e8e10a502ea2ac006fc4f459aee872727d1/pydantic_core-2.46.4-cp313-cp313-manylinux_2_31_riscv64.whl", hash = "sha256:0186750b482eefa11d7f435892b09c5c606193ef3375bcf94aa00ae6bfb66262", size = 2134417, upload-time = "2026-05-06T13:40:17.944Z" }, + { url = "https://files.pythonhosted.org/packages/7e/7b/6ceeb1cc90e193862f444ebe373d8fdf613f0a82572dde03fb10734c6c71/pydantic_core-2.46.4-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:5855698a4856556d86e8e6cd8434bc3ac0314ee8e12089ae0e143f64c6256e4e", size = 2179782, upload-time = "2026-05-06T13:40:32.618Z" }, + { url = "https://files.pythonhosted.org/packages/5a/f2/c8d7773ede6af08036423a00ae0ceffce266c3c52a096c435d68c896083f/pydantic_core-2.46.4-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:cbaf13819775b7f769bf4a1f066cb6df7a28d4480081a589828ef190226881cd", size = 2188782, upload-time = "2026-05-06T13:36:51.018Z" }, + { url = "https://files.pythonhosted.org/packages/59/31/0c864784e31f09f05cdd87606f08923b9c9e7f6e51dd27f20f62f975ce9f/pydantic_core-2.46.4-cp313-cp313-musllinux_1_1_armv7l.whl", hash = "sha256:633147d34cf4550417f12e2b1a0383973bdf5cdfde212cb09e9a581cf10820be", size = 2328334, upload-time = "2026-05-06T13:40:37.764Z" }, + { url = "https://files.pythonhosted.org/packages/c2/eb/4f6c8a41efa30baa755590f4141abf3a8c370fab610915733e74134a7270/pydantic_core-2.46.4-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:82cf5301172168103724d49a1444d3378cb20cdee30b116a1bd6031236298a5d", size = 2372986, upload-time = "2026-05-06T13:39:34.152Z" }, + { url = "https://files.pythonhosted.org/packages/5b/24/b375a480d53113860c299764bfe9f349a3dc9108b3adc0d7f0d786492ebf/pydantic_core-2.46.4-cp313-cp313-win32.whl", hash = "sha256:9fa8ae11da9e2b3126c6426f147e0fba88d96d65921799bb30c6abd1cb2c97fb", size = 1973693, upload-time = "2026-05-06T13:37:55.072Z" }, + { url = "https://files.pythonhosted.org/packages/7e/e8/cff247591966f2d22ec8c003cd7587e27b7ba7b81ab2fb888e3ab75dc285/pydantic_core-2.46.4-cp313-cp313-win_amd64.whl", hash = "sha256:6b3ace8194b0e5204818c92802dcdca7fc6d88aabbb799d7c795540d9cd6d292", size = 2071819, upload-time = "2026-05-06T13:38:49.139Z" }, + { url = "https://files.pythonhosted.org/packages/c6/1a/f4aee670d5670e9e148e0c82c7db98d780be566c6e6a97ee8035528ca0b3/pydantic_core-2.46.4-cp313-cp313-win_arm64.whl", hash = "sha256:184c081504d17f1c1066e430e117142b2c77d9448a97f7b65c6ac9fd9aee238d", size = 2027411, upload-time = "2026-05-06T13:40:45.796Z" }, + { url = "https://files.pythonhosted.org/packages/8d/74/228a26ddad29c6672b805d9fd78e8d251cd04004fa7eed0e622096cd0250/pydantic_core-2.46.4-cp314-cp314-macosx_10_12_x86_64.whl", hash = "sha256:428e04521a40150c85216fc8b85e8d39fece235a9cf5e383761238c7fa9b96fb", size = 2102079, upload-time = "2026-05-06T13:38:41.019Z" }, + { url = "https://files.pythonhosted.org/packages/ad/1f/8970b150a4b4365623ae00fc88603491f763c627311ae8031e3111356d6e/pydantic_core-2.46.4-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:23ace664830ee0bfe014a0c7bc248b1f7f25ed7ad103852c317624a1083af462", size = 1952179, upload-time = "2026-05-06T13:36:59.812Z" }, + { url = "https://files.pythonhosted.org/packages/95/30/5211a831ae054928054b2f79731661087a2bc5c01e825c672b3a4a8f1b3e/pydantic_core-2.46.4-cp314-cp314-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ce5c1d2a8b27468f433ca974829c44060b8097eedc39933e3c206a90ee49c4a9", size = 1978926, upload-time = "2026-05-06T13:37:39.933Z" }, + { url = "https://files.pythonhosted.org/packages/57/e9/689668733b1eb67adeef047db3c2e8788fcf65a7fd9c9e2b46b7744fe245/pydantic_core-2.46.4-cp314-cp314-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:7283d57845ecf5a163403eb0702dfc220cc4fbdd18919cb5ccea4f95ee1cdab4", size = 2046785, upload-time = "2026-05-06T13:38:01.995Z" }, + { url = "https://files.pythonhosted.org/packages/60/d9/6715260422ff50a2109878fd24d948a6c3446bb2664f34ee78cd972b3acd/pydantic_core-2.46.4-cp314-cp314-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:8daafc69c93ee8a0204506a3b6b30f586ef54028f52aeeeb5c4cfc5184fd5914", size = 2228733, upload-time = "2026-05-06T13:40:50.371Z" }, + { url = "https://files.pythonhosted.org/packages/18/ae/fdb2f64316afca925640f8e70bb1a564b0ec2721c1389e25b8eb4bf9a299/pydantic_core-2.46.4-cp314-cp314-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:cd2213145bcc2ba85884d0ac63d222fece9209678f77b9b4d76f054c561adb28", size = 2307534, upload-time = "2026-05-06T13:37:21.531Z" }, + { url = "https://files.pythonhosted.org/packages/89/1d/8eff589b45bb8190a9d12c49cfad0f176a5cbd1534908a6b5125e2886239/pydantic_core-2.46.4-cp314-cp314-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7a5f930472650a82629163023e630d160863fce524c616f4e5186e5de9d9a49b", size = 2099732, upload-time = "2026-05-06T13:39:31.942Z" }, + { url = "https://files.pythonhosted.org/packages/06/d5/ee5a3366637fee41dee51a1fc91562dcf12ddbc68fda34e6b253da2324bb/pydantic_core-2.46.4-cp314-cp314-manylinux_2_31_riscv64.whl", hash = "sha256:c1b3f518abeca3aa13c712fd202306e145abf59a18b094a6bafb2d2bbf59192c", size = 2129627, upload-time = "2026-05-06T13:37:25.033Z" }, + { url = "https://files.pythonhosted.org/packages/94/33/2414be571d2c6a6c4d08be21f9292b6d3fdb08949a97b6dfe985017821db/pydantic_core-2.46.4-cp314-cp314-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:1a7dd0b3ee80d90150e3495a3a13ac34dbcbfd4f012996a6a1d8900e91b5c0fb", size = 2179141, upload-time = "2026-05-06T13:37:14.046Z" }, + { url = "https://files.pythonhosted.org/packages/7b/79/7daa95be995be0eecc4cf75064cb33f9bbbfe3fe0158caf2f0d4a996a5c7/pydantic_core-2.46.4-cp314-cp314-musllinux_1_1_aarch64.whl", hash = "sha256:3fb702cd90b0446a3a1c5e470bfa0dd23c0233b676a9099ddcc964fa6ca13898", size = 2184325, upload-time = "2026-05-06T13:36:53.615Z" }, + { url = "https://files.pythonhosted.org/packages/9f/cb/d0a382f5c0de8a222dc61c65348e0ce831b1f68e0a018450d31c2cace3a5/pydantic_core-2.46.4-cp314-cp314-musllinux_1_1_armv7l.whl", hash = "sha256:b8458003118a712e66286df6a707db01c52c0f52f7db8e4a38f0da1d3b94fc4e", size = 2323990, upload-time = "2026-05-06T13:40:29.971Z" }, + { url = "https://files.pythonhosted.org/packages/05/db/d9ba624cc4a5aced1598e88c04fdbd8310c8a69b9d38b9a3d39ce3a61ed7/pydantic_core-2.46.4-cp314-cp314-musllinux_1_1_x86_64.whl", hash = "sha256:372429a130e469c9cd698925ce5fc50940b7a1336b0d82038e63d5bbc4edc519", size = 2369978, upload-time = "2026-05-06T13:37:23.027Z" }, + { url = "https://files.pythonhosted.org/packages/f2/20/d15df15ba918c423461905802bfd2981c3af0bfa0e40d05e13edbfa48bc3/pydantic_core-2.46.4-cp314-cp314-win32.whl", hash = "sha256:85bb3611ff1802f3ee7fdd7dbff26b56f343fb432d57a4728fdd49b6ef35e2f4", size = 1966354, upload-time = "2026-05-06T13:38:03.499Z" }, + { url = "https://files.pythonhosted.org/packages/fc/b6/6b8de4c0a7d7ab3004c439c80c5c1e0a3e8d78bbae19379b01960383d9e5/pydantic_core-2.46.4-cp314-cp314-win_amd64.whl", hash = "sha256:811ff8e9c313ab425368bcbb36e5c4ebd7108c2bbf4e4089cfbb0b01eff63fac", size = 2072238, upload-time = "2026-05-06T13:39:40.807Z" }, + { url = "https://files.pythonhosted.org/packages/32/36/51eb763beec1f4cf59b1db243a7dcc39cbb41230f050a09b9d69faaf0a48/pydantic_core-2.46.4-cp314-cp314-win_arm64.whl", hash = "sha256:bfec22eab3c8cc2ceec0248aec886624116dc079afa027ecc8ad4a7e62010f8a", size = 2018251, upload-time = "2026-05-06T13:37:26.72Z" }, + { url = "https://files.pythonhosted.org/packages/e8/91/855af51d625b23aa987116a19e231d2aaef9c4a415273ddc189b79a45fee/pydantic_core-2.46.4-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:af8244b2bef6aaad6d92cda81372de7f8c8d36c9f0c3ea36e827c60e7d9467a0", size = 2099593, upload-time = "2026-05-06T13:39:47.682Z" }, + { url = "https://files.pythonhosted.org/packages/fb/1b/8784a54c65edb5f49f0a14d6977cf1b209bba85a4c77445b255c2de58ab3/pydantic_core-2.46.4-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:5a4330cdbc57162e4b3aa303f588ba752257694c9c9be3e7ebb11b4aca659b5d", size = 1935226, upload-time = "2026-05-06T13:40:40.428Z" }, + { url = "https://files.pythonhosted.org/packages/e8/e7/1955d28d1afc56dd4b3ad7cc0cf39df1b9852964cf16e5d13912756d6d6b/pydantic_core-2.46.4-cp314-cp314t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:29c61fc04a3d840155ff08e475a04809278972fe6aef51e2720554e96367e34b", size = 1974605, upload-time = "2026-05-06T13:37:32.029Z" }, + { url = "https://files.pythonhosted.org/packages/93/e2/3fedbf0ba7a22850e6e9fd78117f1c0f10f950182344d8a6c535d468fdd8/pydantic_core-2.46.4-cp314-cp314t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:c50f2528cf200c5eed56faf3f4e22fcd5f38c157a8b78576e6ba3168ec35f000", size = 2030777, upload-time = "2026-05-06T13:38:55.239Z" }, + { url = "https://files.pythonhosted.org/packages/f8/61/46be275fcaaba0b4f5b9669dd852267ce1ff616592dccf7a7845588df091/pydantic_core-2.46.4-cp314-cp314t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:0cbe8b01f948de4286c74cdd6c667aceb38f5c1e26f0693b3983d9d74887c65e", size = 2236641, upload-time = "2026-05-06T13:37:08.096Z" }, + { url = "https://files.pythonhosted.org/packages/60/db/12e93e46a8bac9988be3c016860f83293daea8c716c029c9ace279036f2f/pydantic_core-2.46.4-cp314-cp314t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:617d7e2ca7dcb8c5cf6bcb8c59b8832c94b36196bbf1cbd1bfb56ed341905edd", size = 2286404, upload-time = "2026-05-06T13:40:20.221Z" }, + { url = "https://files.pythonhosted.org/packages/e2/4a/4d8b19008f38d31c53b8219cfedc2e3d5de5fe99d90076b7e767de29274f/pydantic_core-2.46.4-cp314-cp314t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7027560ee92211647d0d34e3f7cd6f50da56399d26a9c8ad0da286d3869a53f3", size = 2109219, upload-time = "2026-05-06T13:38:12.153Z" }, + { url = "https://files.pythonhosted.org/packages/88/70/3cbc40978fefb7bb09c6708d40d4ad1a5d70fd7213c3d17f971de868ec1f/pydantic_core-2.46.4-cp314-cp314t-manylinux_2_31_riscv64.whl", hash = "sha256:f99626688942fb746e545232e7726926f3be91b5975f8b55327665fafda991c7", size = 2110594, upload-time = "2026-05-06T13:40:02.971Z" }, + { url = "https://files.pythonhosted.org/packages/9d/20/b8d36736216e29491125531685b2f9e61aa5b4b2599893f8268551da3338/pydantic_core-2.46.4-cp314-cp314t-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:fc3e9034a63de20e15e8ade85358bc6efc614008cab72898b4b4952bea0509ff", size = 2159542, upload-time = "2026-05-06T13:39:27.506Z" }, + { url = "https://files.pythonhosted.org/packages/1d/a2/367df868eb584dacf6bf82a389272406d7178e301c4ac82545ab98bc2dd9/pydantic_core-2.46.4-cp314-cp314t-musllinux_1_1_aarch64.whl", hash = "sha256:97e7cf2be5c77b7d1a9713a05605d49460d02c6078d38d8bef3cbe323c548424", size = 2168146, upload-time = "2026-05-06T13:38:31.93Z" }, + { url = "https://files.pythonhosted.org/packages/c1/b8/4460f77f7e201893f649a29ab355dddd3beee8a97bcb1a320db414f9a06e/pydantic_core-2.46.4-cp314-cp314t-musllinux_1_1_armv7l.whl", hash = "sha256:3bf92c5d0e00fefaab325a4d27828fe6b6e2a21848686b5b60d2d9eeb09d76c6", size = 2306309, upload-time = "2026-05-06T13:37:44.717Z" }, + { url = "https://files.pythonhosted.org/packages/64/c4/be2639293acd87dc8ddbcec41a73cee9b2ebf996fe6d892a1a74e88ad3f7/pydantic_core-2.46.4-cp314-cp314t-musllinux_1_1_x86_64.whl", hash = "sha256:3ecbc122d18468d06ca279dc26a8c2e2d5acb10943bb35e36ae92096dc3b5565", size = 2369736, upload-time = "2026-05-06T13:37:05.645Z" }, + { url = "https://files.pythonhosted.org/packages/30/a6/9f9f380dbb301f67023bf8f707aaa75daadf84f7152d95c410fd7e81d994/pydantic_core-2.46.4-cp314-cp314t-win32.whl", hash = "sha256:e846ae7835bf0703ae43f534ab79a867146dadd59dc9ca5c8b53d5c8f7c9ef02", size = 1955575, upload-time = "2026-05-06T13:38:51.116Z" }, + { url = "https://files.pythonhosted.org/packages/40/1f/f1eb9eb350e795d1af8586289746f5c5677d16043040d63710e22abc43c9/pydantic_core-2.46.4-cp314-cp314t-win_amd64.whl", hash = "sha256:2108ba5c1c1eca18030634489dc544844144ee36357f2f9f780b93e7ddbb44b5", size = 2051624, upload-time = "2026-05-06T13:38:21.672Z" }, + { url = "https://files.pythonhosted.org/packages/f6/d2/42dd53d0a85c27606f316d3aa5d2869c4e8470a5ed6dec30e4a1abe19192/pydantic_core-2.46.4-cp314-cp314t-win_arm64.whl", hash = "sha256:4fcbe087dbc2068af7eda3aa87634eba216dbda64d1ae73c8684b621d33f6596", size = 2017325, upload-time = "2026-05-06T13:40:52.723Z" }, + { url = "https://files.pythonhosted.org/packages/ee/a4/73995fd4ebbb46ba0ee51e6fa049b8f02c40daebb762208feda8a6b7894d/pydantic_core-2.46.4-graalpy311-graalpy242_311_native-macosx_10_12_x86_64.whl", hash = "sha256:14d4edf427bdcf950a8a02d7cb44a08614388dd6e1bdcbf4f67504fa7887da9c", size = 2111589, upload-time = "2026-05-06T13:37:10.817Z" }, + { url = "https://files.pythonhosted.org/packages/fb/7f/f37d3a5e8bfcc2e403f5c57a730f2d815693fb42119e8ea48b3789335af1/pydantic_core-2.46.4-graalpy311-graalpy242_311_native-macosx_11_0_arm64.whl", hash = "sha256:0ce40cd7b21210e99342afafbd4d0f76d784eb5b1d60f3bdc566be4983c6c73b", size = 1944552, upload-time = "2026-05-06T13:36:56.717Z" }, + { url = "https://files.pythonhosted.org/packages/15/3c/d7eb777b3ff43e8433a4efb39a17aa8fd98a4ee8561a24a67ef5db07b2d6/pydantic_core-2.46.4-graalpy311-graalpy242_311_native-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:90884113d8b48f760e9587002789ddd741e76ab9f89518cd1e43b1f1a52ec44b", size = 1982984, upload-time = "2026-05-06T13:39:06.207Z" }, + { url = "https://files.pythonhosted.org/packages/63/87/70b9f40170a81afd55ca26c9b2acb25c20d64bcfbf888fafecb3ba077d4c/pydantic_core-2.46.4-graalpy311-graalpy242_311_native-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:66ce7632c22d837c95301830e111ad0128a32b8207533b60896a96c4915192ea", size = 2138417, upload-time = "2026-05-06T13:39:45.476Z" }, + { url = "https://files.pythonhosted.org/packages/9d/1d/8987ad40f65ae1432753072f214fb5c74fe47ffbd0698bb9cbbb585664f8/pydantic_core-2.46.4-graalpy312-graalpy250_312_native-macosx_10_12_x86_64.whl", hash = "sha256:1d8ba486450b14f3b1d63bc521d410ec7565e52f887b9fb671791886436a42f7", size = 2095527, upload-time = "2026-05-06T13:39:52.283Z" }, + { url = "https://files.pythonhosted.org/packages/64/d3/84c282a7eee1d3ac4c0377546ef5a1ea436ce26840d9ac3b7ed54a377507/pydantic_core-2.46.4-graalpy312-graalpy250_312_native-macosx_11_0_arm64.whl", hash = "sha256:3009f12e4e90b7f88b4f9adb1b0c4a3d58fe7820f3238c190047209d148026df", size = 1936024, upload-time = "2026-05-06T13:40:15.671Z" }, + { url = "https://files.pythonhosted.org/packages/d7/ca/eac61596cdeb4d7e174d3dc0bd8a6238f14f75f97a24e7b7db4c7e7340a0/pydantic_core-2.46.4-graalpy312-graalpy250_312_native-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ad785e92e6dc634c21555edc8bd6b64957ab844541bcb96a1366c202951ae526", size = 1990696, upload-time = "2026-05-06T13:38:34.717Z" }, + { url = "https://files.pythonhosted.org/packages/fa/c3/7c8b240552251faf6b3a957db200fcfbbcec36763c050428b601e0c9b83b/pydantic_core-2.46.4-graalpy312-graalpy250_312_native-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:00c603d540afdd6b80eb39f078f33ebd46211f02f33e34a32d9f053bba711de0", size = 2147590, upload-time = "2026-05-06T13:39:29.883Z" }, + { url = "https://files.pythonhosted.org/packages/11/cb/428de0385b6c8d44b716feba566abfacfbd23ee3c4439faa789a1456242f/pydantic_core-2.46.4-pp311-pypy311_pp73-macosx_10_12_x86_64.whl", hash = "sha256:0c563b08bca408dc7f65f700633d8442fffb2421fc47b8101377e9fd65051ff0", size = 2112782, upload-time = "2026-05-06T13:37:04.016Z" }, + { url = "https://files.pythonhosted.org/packages/0b/b5/6a17bdadd0fc1f170adfd05a20d37c832f52b117b4d9131da1f41bb097ce/pydantic_core-2.46.4-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:db06ffe51636ffe9ca531fe9023dd64bdd794be8754cb5df57c5498ae5b518a7", size = 1952146, upload-time = "2026-05-06T13:39:43.092Z" }, + { url = "https://files.pythonhosted.org/packages/2a/dc/03734d80e362cd43ef65428e9de77c730ce7f2f11c60d2b1e1b39f0fbf99/pydantic_core-2.46.4-pp311-pypy311_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:133878133d271ade3d41d1bfb2a45ec38dbdbda40bc065921c6b04e4630127e2", size = 2134492, upload-time = "2026-05-06T13:36:58.124Z" }, + { url = "https://files.pythonhosted.org/packages/de/df/5e5ffc085ed07cc22d298134d3d911c63e91f6a0eb91fe646750a3209910/pydantic_core-2.46.4-pp311-pypy311_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:9bc519fbf2b7578398853d815009ae5e4d4603d12f4e3f91da8c06852d3da3e9", size = 2156604, upload-time = "2026-05-06T13:37:49.88Z" }, + { url = "https://files.pythonhosted.org/packages/81/44/6e112a4253e56f5705467cbab7ab5e91ee7398ba3d56d358635958893d3e/pydantic_core-2.46.4-pp311-pypy311_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:c7a7bd4e39e8e4c12c39cd480356842b6a8a06e41b23a55a5e3e191718838ddf", size = 2183828, upload-time = "2026-05-06T13:37:43.053Z" }, + { url = "https://files.pythonhosted.org/packages/ac/ad/5565071e937d8e752842ac241463944c9eb14c87e2d269f2658a5bd05e98/pydantic_core-2.46.4-pp311-pypy311_pp73-musllinux_1_1_armv7l.whl", hash = "sha256:d396ec2b979760aaf3218e76c24e65bd0aca24983298653b3a9d7a45f9e47b30", size = 2310000, upload-time = "2026-05-06T13:37:56.694Z" }, + { url = "https://files.pythonhosted.org/packages/4f/c3/66883a5cec183e7fba4d024b4cbbe61851a63750ef606b0afecc46d1f2bf/pydantic_core-2.46.4-pp311-pypy311_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:86e1a4418c6cd97d60c95c71164158eaf7324fae7b0923264016baa993eba6fc", size = 2361286, upload-time = "2026-05-06T13:40:05.667Z" }, + { url = "https://files.pythonhosted.org/packages/4b/2d/69abac8f838090bbecd5df894befb2c2619e7996a98ddb949db9f3b93225/pydantic_core-2.46.4-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:d51026d73fcfd93610abc7b27789c26b313920fcfb20e27462d74a7f8b06e983", size = 2193071, upload-time = "2026-05-06T13:38:08.682Z" }, ] [[package]] name = "pygments" -version = "2.19.2" +version = "2.20.0" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/b0/77/a5b8c569bf593b0140bde72ea885a803b82086995367bf2037de0159d924/pygments-2.19.2.tar.gz", hash = "sha256:636cb2477cec7f8952536970bc533bc43743542f70392ae026374600add5b887", size = 4968631, upload-time = "2025-06-21T13:39:12.283Z" } +sdist = { url = "https://files.pythonhosted.org/packages/c3/b2/bc9c9196916376152d655522fdcebac55e66de6603a76a02bca1b6414f6c/pygments-2.20.0.tar.gz", hash = "sha256:6757cd03768053ff99f3039c1a36d6c0aa0b263438fcab17520b30a303a82b5f", size = 4955991, upload-time = "2026-03-29T13:29:33.898Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/c7/21/705964c7812476f378728bdf590ca4b771ec72385c533964653c68e86bdc/pygments-2.19.2-py3-none-any.whl", hash = "sha256:86540386c03d588bb81d44bc3928634ff26449851e99741617ecb9037ee5ec0b", size = 1225217, upload-time = "2025-06-21T13:39:07.939Z" }, + { url = "https://files.pythonhosted.org/packages/f4/7e/a72dd26f3b0f4f2bf1dd8923c85f7ceb43172af56d63c7383eb62b332364/pygments-2.20.0-py3-none-any.whl", hash = "sha256:81a9e26dd42fd28a23a2d169d86d7ac03b46e2f8b59ed4698fb4785f946d0176", size = 1231151, upload-time = "2026-03-29T13:29:30.038Z" }, ] [[package]] @@ -2282,7 +2391,8 @@ source = { editable = "." } dependencies = [ { name = "lance-namespace" }, { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, - { name = "numpy", version = "2.3.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" }, + { name = "numpy", version = "2.4.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.11.*'" }, + { name = "numpy", version = "2.5.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.12'" }, { name = "pyarrow" }, ] @@ -2298,6 +2408,10 @@ geo = [ { name = "geoarrow-rust-core" }, { name = "geoarrow-rust-io" }, ] +otel = [ + { name = "opentelemetry-api" }, + { name = "opentelemetry-sdk" }, +] tests = [ { name = "boto3" }, { name = "datafusion" }, @@ -2309,7 +2423,6 @@ tests = [ { name = "polars", extra = ["pandas", "pyarrow"] }, { name = "psutil" }, { name = "pytest" }, - { name = "tensorflow", marker = "sys_platform == 'linux'" }, { name = "tqdm" }, ] torch = [ @@ -2331,26 +2444,28 @@ tests = [ { name = "datasets" }, { name = "duckdb" }, { name = "ml-dtypes" }, + { name = "opentelemetry-sdk" }, { name = "pandas" }, { name = "pillow" }, { name = "polars", extra = ["pandas", "pyarrow"] }, { name = "psutil" }, { name = "pytest" }, - { name = "tensorflow", marker = "sys_platform == 'linux'" }, { name = "tqdm" }, ] [package.metadata] requires-dist = [ { name = "boto3", marker = "extra == 'tests'" }, - { name = "datafusion", marker = "extra == 'tests'", specifier = ">=53,<54" }, - { name = "datasets", marker = "extra == 'tests'" }, - { name = "duckdb", marker = "extra == 'tests'" }, + { name = "datafusion", marker = "extra == 'tests'", specifier = ">=54,<55" }, + { name = "datasets", marker = "extra == 'tests'", specifier = "==4.4.0" }, + { name = "duckdb", marker = "extra == 'tests'", specifier = ">=1.5.0,<1.6.0" }, { name = "geoarrow-rust-core", marker = "extra == 'geo'" }, { name = "geoarrow-rust-io", marker = "extra == 'geo'" }, { name = "lance-namespace", specifier = ">=0.8.5,<0.9" }, { name = "ml-dtypes", marker = "extra == 'tests'" }, { name = "numpy", specifier = ">=1.22" }, + { name = "opentelemetry-api", marker = "extra == 'otel'" }, + { name = "opentelemetry-sdk", marker = "extra == 'otel'" }, { name = "pandas", marker = "extra == 'tests'" }, { name = "pillow", marker = "extra == 'tests'" }, { name = "polars", extras = ["pyarrow", "pandas"], marker = "extra == 'tests'" }, @@ -2360,11 +2475,10 @@ requires-dist = [ { name = "pytest", marker = "extra == 'tests'" }, { name = "pytest-benchmark", marker = "extra == 'benchmarks'" }, { name = "ruff", marker = "extra == 'dev'", specifier = "==0.11.2" }, - { name = "tensorflow", marker = "sys_platform == 'linux' and extra == 'tests'" }, { name = "torch", marker = "extra == 'torch'", specifier = ">=2.0" }, { name = "tqdm", marker = "extra == 'tests'" }, ] -provides-extras = ["benchmarks", "dev", "geo", "tests", "torch"] +provides-extras = ["benchmarks", "dev", "geo", "otel", "tests", "torch"] [package.metadata.requires-dev] benchmarks = [{ name = "pytest-benchmark", specifier = "==5.1.0" }] @@ -2375,16 +2489,16 @@ dev = [ ] tests = [ { name = "boto3", specifier = "==1.40.43" }, - { name = "datafusion", specifier = "==53.0.0" }, - { name = "datasets", specifier = "==4.1.1" }, - { name = "duckdb", specifier = "==1.4.0" }, + { name = "datafusion", specifier = "==54.0.0" }, + { name = "datasets", specifier = "==4.4.0" }, + { name = "duckdb", specifier = ">=1.5.0,<1.6.0" }, { name = "ml-dtypes", specifier = "==0.5.3" }, + { name = "opentelemetry-sdk", specifier = "==1.30.0" }, { name = "pandas", specifier = "==2.3.3" }, { name = "pillow", specifier = "==11.3.0" }, { name = "polars", extras = ["pyarrow", "pandas"], specifier = "==1.34.0" }, { name = "psutil", specifier = "==7.1.0" }, { name = "pytest", specifier = "==8.4.2" }, - { name = "tensorflow", marker = "sys_platform == 'linux'", specifier = "==2.20.0" }, { name = "tqdm", specifier = "==4.67.1" }, ] @@ -2563,11 +2677,11 @@ wheels = [ [[package]] name = "pytz" -version = "2025.2" +version = "2026.2" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/f8/bf/abbd3cdfb8fbc7fb3d4d38d320f2441b1e7cbe29be4f23797b4a2b5d8aac/pytz-2025.2.tar.gz", hash = "sha256:360b9e3dbb49a209c21ad61809c7fb453643e048b38924c765813546746e81c3", size = 320884, upload-time = "2025-03-25T02:25:00.538Z" } +sdist = { url = "https://files.pythonhosted.org/packages/ff/46/dd499ec9038423421951e4fad73051febaa13d2df82b4064f87af8b8c0c3/pytz-2026.2.tar.gz", hash = "sha256:0e60b47b29f21574376f218fe21abc009894a2321ea16c6754f3cad6eb7cdd6a", size = 320861, upload-time = "2026-05-04T01:35:29.667Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/81/c4/34e93fe5f5429d7570ec1fa436f1986fb1f00c3e0f43a589fe2bbcd22c3f/pytz-2025.2-py2.py3-none-any.whl", hash = "sha256:5ddf76296dd8c44c26eb8f4b6f35488f3ccbf6fbbd7adee0b7262d43f0ec2f00", size = 509225, upload-time = "2025-03-25T02:24:58.468Z" }, + { url = "https://files.pythonhosted.org/packages/ec/dd/96da98f892250475bdf2328112d7468abdd4acc7b902b6af23f4ed958ea0/pytz-2026.2-py2.py3-none-any.whl", hash = "sha256:04156e608bee23d3792fd45c94ae47fae1036688e75032eea2e3bf0323d1f126", size = 510141, upload-time = "2026-05-04T01:35:27.408Z" }, ] [[package]] @@ -2636,7 +2750,7 @@ wheels = [ [[package]] name = "requests" -version = "2.33.0" +version = "2.34.2" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "certifi" }, @@ -2644,22 +2758,9 @@ dependencies = [ { name = "idna" }, { name = "urllib3" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/34/64/8860370b167a9721e8956ae116825caff829224fbca0ca6e7bf8ddef8430/requests-2.33.0.tar.gz", hash = "sha256:c7ebc5e8b0f21837386ad0e1c8fe8b829fa5f544d8df3b2253bff14ef29d7652", size = 134232, upload-time = "2026-03-25T15:10:41.586Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/56/5d/c814546c2333ceea4ba42262d8c4d55763003e767fa169adc693bd524478/requests-2.33.0-py3-none-any.whl", hash = "sha256:3324635456fa185245e24865e810cecec7b4caf933d7eb133dcde67d48cee69b", size = 65017, upload-time = "2026-03-25T15:10:40.382Z" }, -] - -[[package]] -name = "rich" -version = "14.1.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "markdown-it-py" }, - { name = "pygments" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/fe/75/af448d8e52bf1d8fa6a9d089ca6c07ff4453d86c65c145d0a300bb073b9b/rich-14.1.0.tar.gz", hash = "sha256:e497a48b844b0320d45007cdebfeaeed8db2a4f4bcf49f15e455cfc4af11eaa8", size = 224441, upload-time = "2025-07-25T07:32:58.125Z" } +sdist = { url = "https://files.pythonhosted.org/packages/ac/c3/e2a2b89f2d3e2179abd6d00ebd70bff6273f37fb3e0cc209f48b39d00cbf/requests-2.34.2.tar.gz", hash = "sha256:f288924cae4e29463698d6d60bc6a4da69c89185ad1e0bcc4104f584e960b9ed", size = 142856, upload-time = "2026-05-14T19:25:27.735Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/e3/30/3c4d035596d3cf444529e0b2953ad0466f6049528a879d27534700580395/rich-14.1.0-py3-none-any.whl", hash = "sha256:536f5f1785986d6dbdea3c75205c473f970777b4a0d6c6dd1b696aa05a3fa04f", size = 243368, upload-time = "2025-07-25T07:32:56.73Z" }, + { url = "https://files.pythonhosted.org/packages/a0/f4/c67b0b3f1b9245e8d266f0f112c500d50e5b4e83cb6f3b71b6528104182a/requests-2.34.2-py3-none-any.whl", hash = "sha256:2a0d60c172f83ac6ab31e4554906c0f3b3588d37b5cb939b1c061f4907e278e0", size = 73075, upload-time = "2026-05-14T19:25:26.443Z" }, ] [[package]] @@ -2701,11 +2802,11 @@ wheels = [ [[package]] name = "setuptools" -version = "80.9.0" +version = "83.0.0" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/18/5d/3bf57dcd21979b887f014ea83c24ae194cfcd12b9e0fda66b957c69d1fca/setuptools-80.9.0.tar.gz", hash = "sha256:f36b47402ecde768dbfafc46e8e4207b4360c654f1f3bb84475f0a28628fb19c", size = 1319958, upload-time = "2025-05-27T00:56:51.443Z" } +sdist = { url = "https://files.pythonhosted.org/packages/34/26/f5d29e25ffdb535afef2d35cdb55b325298f96debd670da4c325e08d70f4/setuptools-83.0.0.tar.gz", hash = "sha256:025bccbbf0fa05b6192bc64ae1e7b16e001fd6d6d4d5de03c97b1c1ade523bef", size = 1154254, upload-time = "2026-07-04T15:31:22.699Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/a3/dc/17031897dae0efacfea57dfd3a82fdd2a2aeb58e0ff71b77b87e44edc772/setuptools-80.9.0-py3-none-any.whl", hash = "sha256:062d34222ad13e0cc312a4c02d73f059e86a4acbfbdea8f8f76b28c99f306922", size = 1201486, upload-time = "2025-05-27T00:56:49.664Z" }, + { url = "https://files.pythonhosted.org/packages/5d/40/e1e72872c6354b306daef1703549e8e83b4d43cfea356311bf722a043752/setuptools-83.0.0-py3-none-any.whl", hash = "sha256:29b23c360f22f414dc7336bb39178cc7bcbf6021ed2733cde173f09dba19abb3", size = 1008090, upload-time = "2026-07-04T15:31:20.885Z" }, ] [[package]] @@ -2729,170 +2830,106 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/a2/09/77d55d46fd61b4a135c444fc97158ef34a095e5681d0a6c10b75bf356191/sympy-1.14.0-py3-none-any.whl", hash = "sha256:e091cc3e99d2141a0ba2847328f5479b05d94a6635cb96148ccb3f34671bd8f5", size = 6299353, upload-time = "2025-04-27T18:04:59.103Z" }, ] -[[package]] -name = "tensorboard" -version = "2.20.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "absl-py" }, - { name = "grpcio" }, - { name = "markdown" }, - { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, - { name = "numpy", version = "2.3.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" }, - { name = "packaging" }, - { name = "pillow" }, - { name = "protobuf" }, - { name = "setuptools" }, - { name = "tensorboard-data-server" }, - { name = "werkzeug" }, -] -wheels = [ - { url = "https://files.pythonhosted.org/packages/9c/d9/a5db55f88f258ac669a92858b70a714bbbd5acd993820b41ec4a96a4d77f/tensorboard-2.20.0-py3-none-any.whl", hash = "sha256:9dc9f978cb84c0723acf9a345d96c184f0293d18f166bb8d59ee098e6cfaaba6", size = 5525680, upload-time = "2025-07-17T19:20:49.638Z" }, -] - -[[package]] -name = "tensorboard-data-server" -version = "0.7.2" -source = { registry = "https://pypi.org/simple" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/7a/13/e503968fefabd4c6b2650af21e110aa8466fe21432cd7c43a84577a89438/tensorboard_data_server-0.7.2-py3-none-any.whl", hash = "sha256:7e0610d205889588983836ec05dc098e80f97b7e7bbff7e994ebb78f578d0ddb", size = 2356, upload-time = "2023-10-23T21:23:32.16Z" }, - { url = "https://files.pythonhosted.org/packages/73/c6/825dab04195756cf8ff2e12698f22513b3db2f64925bdd41671bfb33aaa5/tensorboard_data_server-0.7.2-py3-none-manylinux_2_31_x86_64.whl", hash = "sha256:ef687163c24185ae9754ed5650eb5bc4d84ff257aabdc33f0cc6f74d8ba54530", size = 6590363, upload-time = "2023-10-23T21:23:35.583Z" }, -] - -[[package]] -name = "tensorflow" -version = "2.20.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "absl-py" }, - { name = "astunparse" }, - { name = "flatbuffers" }, - { name = "gast" }, - { name = "google-pasta" }, - { name = "grpcio" }, - { name = "h5py" }, - { name = "keras" }, - { name = "libclang" }, - { name = "ml-dtypes" }, - { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, - { name = "numpy", version = "2.3.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" }, - { name = "opt-einsum" }, - { name = "packaging" }, - { name = "protobuf" }, - { name = "requests" }, - { name = "setuptools" }, - { name = "six" }, - { name = "tensorboard" }, - { name = "termcolor" }, - { name = "typing-extensions" }, - { name = "wrapt" }, -] -wheels = [ - { url = "https://files.pythonhosted.org/packages/ff/07/ea91ac67a9fd36d3372099f5a3e69860ded544f877f5f2117802388f4212/tensorflow-2.20.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:02a0293d94f5c8b7125b66abf622cc4854a33ae9d618a0d41309f95e091bbaea", size = 259307122, upload-time = "2025-08-13T16:50:47.909Z" }, - { url = "https://files.pythonhosted.org/packages/e5/9e/0d57922cf46b9e91de636cd5b5e0d7a424ebe98f3245380a713f1f6c2a0b/tensorflow-2.20.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7abd7f3a010e0d354dc804182372779a722d474c4d8a3db8f4a3f5baef2a591e", size = 620425510, upload-time = "2025-08-13T16:51:02.608Z" }, - { url = "https://files.pythonhosted.org/packages/f1/b7/a3d455db88ab5b35ce53ab885ec0dd9f28d905a86a2250423048bc8cafa0/tensorflow-2.20.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3e9568c8efcb05c0266be223e3269c62ebf7ad3498f156438311735f6fa5ced5", size = 259465882, upload-time = "2025-08-13T16:51:39.546Z" }, - { url = "https://files.pythonhosted.org/packages/ff/0c/7df285ee8a88139fab0b237003634d90690759fae9c18f55ddb7c04656ec/tensorflow-2.20.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:481499fd0f824583de8945be61d5e827898cdaa4f5ea1bc2cc28ca2ccff8229e", size = 620570129, upload-time = "2025-08-13T16:51:55.104Z" }, - { url = "https://files.pythonhosted.org/packages/ec/b4/f028a5de27d0fda10ba6145bc76e40c37ff6d2d1e95b601adb5ae17d635e/tensorflow-2.20.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2bfbfb3dd0e22bffc45fe1e922390d27753e99261fab8a882e802cf98a0e078f", size = 259533109, upload-time = "2025-08-13T16:52:31.513Z" }, - { url = "https://files.pythonhosted.org/packages/9c/d1/6aa15085d672056d5f08b5f28b1c7ce01c4e12149a23b0c98e3c79d04441/tensorflow-2.20.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:25265b0bc527e0d54b1e9cc60c44a24f44a809fe27666b905f0466471f9c52ec", size = 620682547, upload-time = "2025-08-13T16:52:46.396Z" }, - { url = "https://files.pythonhosted.org/packages/ea/4c/c1aa90c5cc92e9f7f9c78421e121ef25bae7d378f8d1d4cbad46c6308836/tensorflow-2.20.0-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:47c88e05a07f1ead4977b4894b3ecd4d8075c40191065afc4fd9355c9db3d926", size = 259663776, upload-time = "2025-08-13T16:53:24.507Z" }, - { url = "https://files.pythonhosted.org/packages/43/fb/8be8547c128613d82a2b006004026d86ed0bd672e913029a98153af4ffab/tensorflow-2.20.0-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5fa3729b0126f75a99882b89fb7d536515721eda8014a63e259e780ba0a37372", size = 620815537, upload-time = "2025-08-13T16:53:42.577Z" }, -] - -[[package]] -name = "termcolor" -version = "3.1.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/ca/6c/3d75c196ac07ac8749600b60b03f4f6094d54e132c4d94ebac6ee0e0add0/termcolor-3.1.0.tar.gz", hash = "sha256:6a6dd7fbee581909eeec6a756cff1d7f7c376063b14e4a298dc4980309e55970", size = 14324, upload-time = "2025-04-30T11:37:53.791Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/4f/bd/de8d508070629b6d84a30d01d57e4a65c69aa7f5abe7560b8fad3b50ea59/termcolor-3.1.0-py3-none-any.whl", hash = "sha256:591dd26b5c2ce03b9e43f391264626557873ce1d379019786f99b0c2bee140aa", size = 7684, upload-time = "2025-04-30T11:37:52.382Z" }, -] - [[package]] name = "tomli" -version = "2.2.1" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/18/87/302344fed471e44a87289cf4967697d07e532f2421fdaf868a303cbae4ff/tomli-2.2.1.tar.gz", hash = "sha256:cd45e1dc79c835ce60f7404ec8119f2eb06d38b1deba146f07ced3bbc44505ff", size = 17175, upload-time = "2024-11-27T22:38:36.873Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/43/ca/75707e6efa2b37c77dadb324ae7d9571cb424e61ea73fad7c56c2d14527f/tomli-2.2.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:678e4fa69e4575eb77d103de3df8a895e1591b48e740211bd1067378c69e8249", size = 131077, upload-time = "2024-11-27T22:37:54.956Z" }, - { url = "https://files.pythonhosted.org/packages/c7/16/51ae563a8615d472fdbffc43a3f3d46588c264ac4f024f63f01283becfbb/tomli-2.2.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:023aa114dd824ade0100497eb2318602af309e5a55595f76b626d6d9f3b7b0a6", size = 123429, upload-time = "2024-11-27T22:37:56.698Z" }, - { url = "https://files.pythonhosted.org/packages/f1/dd/4f6cd1e7b160041db83c694abc78e100473c15d54620083dbd5aae7b990e/tomli-2.2.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ece47d672db52ac607a3d9599a9d48dcb2f2f735c6c2d1f34130085bb12b112a", size = 226067, upload-time = "2024-11-27T22:37:57.63Z" }, - { url = "https://files.pythonhosted.org/packages/a9/6b/c54ede5dc70d648cc6361eaf429304b02f2871a345bbdd51e993d6cdf550/tomli-2.2.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6972ca9c9cc9f0acaa56a8ca1ff51e7af152a9f87fb64623e31d5c83700080ee", size = 236030, upload-time = "2024-11-27T22:37:59.344Z" }, - { url = "https://files.pythonhosted.org/packages/1f/47/999514fa49cfaf7a92c805a86c3c43f4215621855d151b61c602abb38091/tomli-2.2.1-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c954d2250168d28797dd4e3ac5cf812a406cd5a92674ee4c8f123c889786aa8e", size = 240898, upload-time = "2024-11-27T22:38:00.429Z" }, - { url = "https://files.pythonhosted.org/packages/73/41/0a01279a7ae09ee1573b423318e7934674ce06eb33f50936655071d81a24/tomli-2.2.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:8dd28b3e155b80f4d54beb40a441d366adcfe740969820caf156c019fb5c7ec4", size = 229894, upload-time = "2024-11-27T22:38:02.094Z" }, - { url = "https://files.pythonhosted.org/packages/55/18/5d8bc5b0a0362311ce4d18830a5d28943667599a60d20118074ea1b01bb7/tomli-2.2.1-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:e59e304978767a54663af13c07b3d1af22ddee3bb2fb0618ca1593e4f593a106", size = 245319, upload-time = "2024-11-27T22:38:03.206Z" }, - { url = "https://files.pythonhosted.org/packages/92/a3/7ade0576d17f3cdf5ff44d61390d4b3febb8a9fc2b480c75c47ea048c646/tomli-2.2.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:33580bccab0338d00994d7f16f4c4ec25b776af3ffaac1ed74e0b3fc95e885a8", size = 238273, upload-time = "2024-11-27T22:38:04.217Z" }, - { url = "https://files.pythonhosted.org/packages/72/6f/fa64ef058ac1446a1e51110c375339b3ec6be245af9d14c87c4a6412dd32/tomli-2.2.1-cp311-cp311-win32.whl", hash = "sha256:465af0e0875402f1d226519c9904f37254b3045fc5084697cefb9bdde1ff99ff", size = 98310, upload-time = "2024-11-27T22:38:05.908Z" }, - { url = "https://files.pythonhosted.org/packages/6a/1c/4a2dcde4a51b81be3530565e92eda625d94dafb46dbeb15069df4caffc34/tomli-2.2.1-cp311-cp311-win_amd64.whl", hash = "sha256:2d0f2fdd22b02c6d81637a3c95f8cd77f995846af7414c5c4b8d0545afa1bc4b", size = 108309, upload-time = "2024-11-27T22:38:06.812Z" }, - { url = "https://files.pythonhosted.org/packages/52/e1/f8af4c2fcde17500422858155aeb0d7e93477a0d59a98e56cbfe75070fd0/tomli-2.2.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:4a8f6e44de52d5e6c657c9fe83b562f5f4256d8ebbfe4ff922c495620a7f6cea", size = 132762, upload-time = "2024-11-27T22:38:07.731Z" }, - { url = "https://files.pythonhosted.org/packages/03/b8/152c68bb84fc00396b83e7bbddd5ec0bd3dd409db4195e2a9b3e398ad2e3/tomli-2.2.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:8d57ca8095a641b8237d5b079147646153d22552f1c637fd3ba7f4b0b29167a8", size = 123453, upload-time = "2024-11-27T22:38:09.384Z" }, - { url = "https://files.pythonhosted.org/packages/c8/d6/fc9267af9166f79ac528ff7e8c55c8181ded34eb4b0e93daa767b8841573/tomli-2.2.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4e340144ad7ae1533cb897d406382b4b6fede8890a03738ff1683af800d54192", size = 233486, upload-time = "2024-11-27T22:38:10.329Z" }, - { url = "https://files.pythonhosted.org/packages/5c/51/51c3f2884d7bab89af25f678447ea7d297b53b5a3b5730a7cb2ef6069f07/tomli-2.2.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:db2b95f9de79181805df90bedc5a5ab4c165e6ec3fe99f970d0e302f384ad222", size = 242349, upload-time = "2024-11-27T22:38:11.443Z" }, - { url = "https://files.pythonhosted.org/packages/ab/df/bfa89627d13a5cc22402e441e8a931ef2108403db390ff3345c05253935e/tomli-2.2.1-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:40741994320b232529c802f8bc86da4e1aa9f413db394617b9a256ae0f9a7f77", size = 252159, upload-time = "2024-11-27T22:38:13.099Z" }, - { url = "https://files.pythonhosted.org/packages/9e/6e/fa2b916dced65763a5168c6ccb91066f7639bdc88b48adda990db10c8c0b/tomli-2.2.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:400e720fe168c0f8521520190686ef8ef033fb19fc493da09779e592861b78c6", size = 237243, upload-time = "2024-11-27T22:38:14.766Z" }, - { url = "https://files.pythonhosted.org/packages/b4/04/885d3b1f650e1153cbb93a6a9782c58a972b94ea4483ae4ac5cedd5e4a09/tomli-2.2.1-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:02abe224de6ae62c19f090f68da4e27b10af2b93213d36cf44e6e1c5abd19fdd", size = 259645, upload-time = "2024-11-27T22:38:15.843Z" }, - { url = "https://files.pythonhosted.org/packages/9c/de/6b432d66e986e501586da298e28ebeefd3edc2c780f3ad73d22566034239/tomli-2.2.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:b82ebccc8c8a36f2094e969560a1b836758481f3dc360ce9a3277c65f374285e", size = 244584, upload-time = "2024-11-27T22:38:17.645Z" }, - { url = "https://files.pythonhosted.org/packages/1c/9a/47c0449b98e6e7d1be6cbac02f93dd79003234ddc4aaab6ba07a9a7482e2/tomli-2.2.1-cp312-cp312-win32.whl", hash = "sha256:889f80ef92701b9dbb224e49ec87c645ce5df3fa2cc548664eb8a25e03127a98", size = 98875, upload-time = "2024-11-27T22:38:19.159Z" }, - { url = "https://files.pythonhosted.org/packages/ef/60/9b9638f081c6f1261e2688bd487625cd1e660d0a85bd469e91d8db969734/tomli-2.2.1-cp312-cp312-win_amd64.whl", hash = "sha256:7fc04e92e1d624a4a63c76474610238576942d6b8950a2d7f908a340494e67e4", size = 109418, upload-time = "2024-11-27T22:38:20.064Z" }, - { url = "https://files.pythonhosted.org/packages/04/90/2ee5f2e0362cb8a0b6499dc44f4d7d48f8fff06d28ba46e6f1eaa61a1388/tomli-2.2.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:f4039b9cbc3048b2416cc57ab3bda989a6fcf9b36cf8937f01a6e731b64f80d7", size = 132708, upload-time = "2024-11-27T22:38:21.659Z" }, - { url = "https://files.pythonhosted.org/packages/c0/ec/46b4108816de6b385141f082ba99e315501ccd0a2ea23db4a100dd3990ea/tomli-2.2.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:286f0ca2ffeeb5b9bd4fcc8d6c330534323ec51b2f52da063b11c502da16f30c", size = 123582, upload-time = "2024-11-27T22:38:22.693Z" }, - { url = "https://files.pythonhosted.org/packages/a0/bd/b470466d0137b37b68d24556c38a0cc819e8febe392d5b199dcd7f578365/tomli-2.2.1-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a92ef1a44547e894e2a17d24e7557a5e85a9e1d0048b0b5e7541f76c5032cb13", size = 232543, upload-time = "2024-11-27T22:38:24.367Z" }, - { url = "https://files.pythonhosted.org/packages/d9/e5/82e80ff3b751373f7cead2815bcbe2d51c895b3c990686741a8e56ec42ab/tomli-2.2.1-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9316dc65bed1684c9a98ee68759ceaed29d229e985297003e494aa825ebb0281", size = 241691, upload-time = "2024-11-27T22:38:26.081Z" }, - { url = "https://files.pythonhosted.org/packages/05/7e/2a110bc2713557d6a1bfb06af23dd01e7dde52b6ee7dadc589868f9abfac/tomli-2.2.1-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e85e99945e688e32d5a35c1ff38ed0b3f41f43fad8df0bdf79f72b2ba7bc5272", size = 251170, upload-time = "2024-11-27T22:38:27.921Z" }, - { url = "https://files.pythonhosted.org/packages/64/7b/22d713946efe00e0adbcdfd6d1aa119ae03fd0b60ebed51ebb3fa9f5a2e5/tomli-2.2.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:ac065718db92ca818f8d6141b5f66369833d4a80a9d74435a268c52bdfa73140", size = 236530, upload-time = "2024-11-27T22:38:29.591Z" }, - { url = "https://files.pythonhosted.org/packages/38/31/3a76f67da4b0cf37b742ca76beaf819dca0ebef26d78fc794a576e08accf/tomli-2.2.1-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:d920f33822747519673ee656a4b6ac33e382eca9d331c87770faa3eef562aeb2", size = 258666, upload-time = "2024-11-27T22:38:30.639Z" }, - { url = "https://files.pythonhosted.org/packages/07/10/5af1293da642aded87e8a988753945d0cf7e00a9452d3911dd3bb354c9e2/tomli-2.2.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:a198f10c4d1b1375d7687bc25294306e551bf1abfa4eace6650070a5c1ae2744", size = 243954, upload-time = "2024-11-27T22:38:31.702Z" }, - { url = "https://files.pythonhosted.org/packages/5b/b9/1ed31d167be802da0fc95020d04cd27b7d7065cc6fbefdd2f9186f60d7bd/tomli-2.2.1-cp313-cp313-win32.whl", hash = "sha256:d3f5614314d758649ab2ab3a62d4f2004c825922f9e370b29416484086b264ec", size = 98724, upload-time = "2024-11-27T22:38:32.837Z" }, - { url = "https://files.pythonhosted.org/packages/c7/32/b0963458706accd9afcfeb867c0f9175a741bf7b19cd424230714d722198/tomli-2.2.1-cp313-cp313-win_amd64.whl", hash = "sha256:a38aa0308e754b0e3c67e344754dff64999ff9b513e691d0e786265c93583c69", size = 109383, upload-time = "2024-11-27T22:38:34.455Z" }, - { url = "https://files.pythonhosted.org/packages/6e/c2/61d3e0f47e2b74ef40a68b9e6ad5984f6241a942f7cd3bbfbdbd03861ea9/tomli-2.2.1-py3-none-any.whl", hash = "sha256:cb55c73c5f4408779d0cf3eef9f762b9c9f147a77de7b258bef0a5628adc85cc", size = 14257, upload-time = "2024-11-27T22:38:35.385Z" }, +version = "2.4.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/22/de/48c59722572767841493b26183a0d1cc411d54fd759c5607c4590b6563a6/tomli-2.4.1.tar.gz", hash = "sha256:7c7e1a961a0b2f2472c1ac5b69affa0ae1132c39adcb67aba98568702b9cc23f", size = 17543, upload-time = "2026-03-25T20:22:03.828Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f4/11/db3d5885d8528263d8adc260bb2d28ebf1270b96e98f0e0268d32b8d9900/tomli-2.4.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:f8f0fc26ec2cc2b965b7a3b87cd19c5c6b8c5e5f436b984e85f486d652285c30", size = 154704, upload-time = "2026-03-25T20:21:10.473Z" }, + { url = "https://files.pythonhosted.org/packages/6d/f7/675db52c7e46064a9aa928885a9b20f4124ecb9bc2e1ce74c9106648d202/tomli-2.4.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:4ab97e64ccda8756376892c53a72bd1f964e519c77236368527f758fbc36a53a", size = 149454, upload-time = "2026-03-25T20:21:12.036Z" }, + { url = "https://files.pythonhosted.org/packages/61/71/81c50943cf953efa35bce7646caab3cf457a7d8c030b27cfb40d7235f9ee/tomli-2.4.1-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:96481a5786729fd470164b47cdb3e0e58062a496f455ee41b4403be77cb5a076", size = 237561, upload-time = "2026-03-25T20:21:13.098Z" }, + { url = "https://files.pythonhosted.org/packages/48/c1/f41d9cb618acccca7df82aaf682f9b49013c9397212cb9f53219e3abac37/tomli-2.4.1-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:5a881ab208c0baf688221f8cecc5401bd291d67e38a1ac884d6736cbcd8247e9", size = 243824, upload-time = "2026-03-25T20:21:14.569Z" }, + { url = "https://files.pythonhosted.org/packages/22/e4/5a816ecdd1f8ca51fb756ef684b90f2780afc52fc67f987e3c61d800a46d/tomli-2.4.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:47149d5bd38761ac8be13a84864bf0b7b70bc051806bc3669ab1cbc56216b23c", size = 242227, upload-time = "2026-03-25T20:21:15.712Z" }, + { url = "https://files.pythonhosted.org/packages/6b/49/2b2a0ef529aa6eec245d25f0c703e020a73955ad7edf73e7f54ddc608aa5/tomli-2.4.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:ec9bfaf3ad2df51ace80688143a6a4ebc09a248f6ff781a9945e51937008fcbc", size = 247859, upload-time = "2026-03-25T20:21:17.001Z" }, + { url = "https://files.pythonhosted.org/packages/83/bd/6c1a630eaca337e1e78c5903104f831bda934c426f9231429396ce3c3467/tomli-2.4.1-cp311-cp311-win32.whl", hash = "sha256:ff2983983d34813c1aeb0fa89091e76c3a22889ee83ab27c5eeb45100560c049", size = 97204, upload-time = "2026-03-25T20:21:18.079Z" }, + { url = "https://files.pythonhosted.org/packages/42/59/71461df1a885647e10b6bb7802d0b8e66480c61f3f43079e0dcd315b3954/tomli-2.4.1-cp311-cp311-win_amd64.whl", hash = "sha256:5ee18d9ebdb417e384b58fe414e8d6af9f4e7a0ae761519fb50f721de398dd4e", size = 108084, upload-time = "2026-03-25T20:21:18.978Z" }, + { url = "https://files.pythonhosted.org/packages/b8/83/dceca96142499c069475b790e7913b1044c1a4337e700751f48ed723f883/tomli-2.4.1-cp311-cp311-win_arm64.whl", hash = "sha256:c2541745709bad0264b7d4705ad453b76ccd191e64aa6f0fc66b69a293a45ece", size = 95285, upload-time = "2026-03-25T20:21:20.309Z" }, + { url = "https://files.pythonhosted.org/packages/c1/ba/42f134a3fe2b370f555f44b1d72feebb94debcab01676bf918d0cb70e9aa/tomli-2.4.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:c742f741d58a28940ce01d58f0ab2ea3ced8b12402f162f4d534dfe18ba1cd6a", size = 155924, upload-time = "2026-03-25T20:21:21.626Z" }, + { url = "https://files.pythonhosted.org/packages/dc/c7/62d7a17c26487ade21c5422b646110f2162f1fcc95980ef7f63e73c68f14/tomli-2.4.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:7f86fd587c4ed9dd76f318225e7d9b29cfc5a9d43de44e5754db8d1128487085", size = 150018, upload-time = "2026-03-25T20:21:23.002Z" }, + { url = "https://files.pythonhosted.org/packages/5c/05/79d13d7c15f13bdef410bdd49a6485b1c37d28968314eabee452c22a7fda/tomli-2.4.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ff18e6a727ee0ab0388507b89d1bc6a22b138d1e2fa56d1ad494586d61d2eae9", size = 244948, upload-time = "2026-03-25T20:21:24.04Z" }, + { url = "https://files.pythonhosted.org/packages/10/90/d62ce007a1c80d0b2c93e02cab211224756240884751b94ca72df8a875ca/tomli-2.4.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:136443dbd7e1dee43c68ac2694fde36b2849865fa258d39bf822c10e8068eac5", size = 253341, upload-time = "2026-03-25T20:21:25.177Z" }, + { url = "https://files.pythonhosted.org/packages/1a/7e/caf6496d60152ad4ed09282c1885cca4eea150bfd007da84aea07bcc0a3e/tomli-2.4.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:5e262d41726bc187e69af7825504c933b6794dc3fbd5945e41a79bb14c31f585", size = 248159, upload-time = "2026-03-25T20:21:26.364Z" }, + { url = "https://files.pythonhosted.org/packages/99/e7/c6f69c3120de34bbd882c6fba7975f3d7a746e9218e56ab46a1bc4b42552/tomli-2.4.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:5cb41aa38891e073ee49d55fbc7839cfdb2bc0e600add13874d048c94aadddd1", size = 253290, upload-time = "2026-03-25T20:21:27.46Z" }, + { url = "https://files.pythonhosted.org/packages/d6/2f/4a3c322f22c5c66c4b836ec58211641a4067364f5dcdd7b974b4c5da300c/tomli-2.4.1-cp312-cp312-win32.whl", hash = "sha256:da25dc3563bff5965356133435b757a795a17b17d01dbc0f42fb32447ddfd917", size = 98141, upload-time = "2026-03-25T20:21:28.492Z" }, + { url = "https://files.pythonhosted.org/packages/24/22/4daacd05391b92c55759d55eaee21e1dfaea86ce5c571f10083360adf534/tomli-2.4.1-cp312-cp312-win_amd64.whl", hash = "sha256:52c8ef851d9a240f11a88c003eacb03c31fc1c9c4ec64a99a0f922b93874fda9", size = 108847, upload-time = "2026-03-25T20:21:29.386Z" }, + { url = "https://files.pythonhosted.org/packages/68/fd/70e768887666ddd9e9f5d85129e84910f2db2796f9096aa02b721a53098d/tomli-2.4.1-cp312-cp312-win_arm64.whl", hash = "sha256:f758f1b9299d059cc3f6546ae2af89670cb1c4d48ea29c3cacc4fe7de3058257", size = 95088, upload-time = "2026-03-25T20:21:30.677Z" }, + { url = "https://files.pythonhosted.org/packages/07/06/b823a7e818c756d9a7123ba2cda7d07bc2dd32835648d1a7b7b7a05d848d/tomli-2.4.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:36d2bd2ad5fb9eaddba5226aa02c8ec3fa4f192631e347b3ed28186d43be6b54", size = 155866, upload-time = "2026-03-25T20:21:31.65Z" }, + { url = "https://files.pythonhosted.org/packages/14/6f/12645cf7f08e1a20c7eb8c297c6f11d31c1b50f316a7e7e1e1de6e2e7b7e/tomli-2.4.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:eb0dc4e38e6a1fd579e5d50369aa2e10acfc9cace504579b2faabb478e76941a", size = 149887, upload-time = "2026-03-25T20:21:33.028Z" }, + { url = "https://files.pythonhosted.org/packages/5c/e0/90637574e5e7212c09099c67ad349b04ec4d6020324539297b634a0192b0/tomli-2.4.1-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c7f2c7f2b9ca6bdeef8f0fa897f8e05085923eb091721675170254cbc5b02897", size = 243704, upload-time = "2026-03-25T20:21:34.51Z" }, + { url = "https://files.pythonhosted.org/packages/10/8f/d3ddb16c5a4befdf31a23307f72828686ab2096f068eaf56631e136c1fdd/tomli-2.4.1-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f3c6818a1a86dd6dca7ddcaaf76947d5ba31aecc28cb1b67009a5877c9a64f3f", size = 251628, upload-time = "2026-03-25T20:21:36.012Z" }, + { url = "https://files.pythonhosted.org/packages/e3/f1/dbeeb9116715abee2485bf0a12d07a8f31af94d71608c171c45f64c0469d/tomli-2.4.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:d312ef37c91508b0ab2cee7da26ec0b3ed2f03ce12bd87a588d771ae15dcf82d", size = 247180, upload-time = "2026-03-25T20:21:37.136Z" }, + { url = "https://files.pythonhosted.org/packages/d3/74/16336ffd19ed4da28a70959f92f506233bd7cfc2332b20bdb01591e8b1d1/tomli-2.4.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:51529d40e3ca50046d7606fa99ce3956a617f9b36380da3b7f0dd3dd28e68cb5", size = 251674, upload-time = "2026-03-25T20:21:38.298Z" }, + { url = "https://files.pythonhosted.org/packages/16/f9/229fa3434c590ddf6c0aa9af64d3af4b752540686cace29e6281e3458469/tomli-2.4.1-cp313-cp313-win32.whl", hash = "sha256:2190f2e9dd7508d2a90ded5ed369255980a1bcdd58e52f7fe24b8162bf9fedbd", size = 97976, upload-time = "2026-03-25T20:21:39.316Z" }, + { url = "https://files.pythonhosted.org/packages/6a/1e/71dfd96bcc1c775420cb8befe7a9d35f2e5b1309798f009dca17b7708c1e/tomli-2.4.1-cp313-cp313-win_amd64.whl", hash = "sha256:8d65a2fbf9d2f8352685bc1364177ee3923d6baf5e7f43ea4959d7d8bc326a36", size = 108755, upload-time = "2026-03-25T20:21:40.248Z" }, + { url = "https://files.pythonhosted.org/packages/83/7a/d34f422a021d62420b78f5c538e5b102f62bea616d1d75a13f0a88acb04a/tomli-2.4.1-cp313-cp313-win_arm64.whl", hash = "sha256:4b605484e43cdc43f0954ddae319fb75f04cc10dd80d830540060ee7cd0243cd", size = 95265, upload-time = "2026-03-25T20:21:41.219Z" }, + { url = "https://files.pythonhosted.org/packages/3c/fb/9a5c8d27dbab540869f7c1f8eb0abb3244189ce780ba9cd73f3770662072/tomli-2.4.1-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:fd0409a3653af6c147209d267a0e4243f0ae46b011aa978b1080359fddc9b6cf", size = 155726, upload-time = "2026-03-25T20:21:42.23Z" }, + { url = "https://files.pythonhosted.org/packages/62/05/d2f816630cc771ad836af54f5001f47a6f611d2d39535364f148b6a92d6b/tomli-2.4.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:a120733b01c45e9a0c34aeef92bf0cf1d56cfe81ed9d47d562f9ed591a9828ac", size = 149859, upload-time = "2026-03-25T20:21:43.386Z" }, + { url = "https://files.pythonhosted.org/packages/ce/48/66341bdb858ad9bd0ceab5a86f90eddab127cf8b046418009f2125630ecb/tomli-2.4.1-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:559db847dc486944896521f68d8190be1c9e719fced785720d2216fe7022b662", size = 244713, upload-time = "2026-03-25T20:21:44.474Z" }, + { url = "https://files.pythonhosted.org/packages/df/6d/c5fad00d82b3c7a3ab6189bd4b10e60466f22cfe8a08a9394185c8a8111c/tomli-2.4.1-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:01f520d4f53ef97964a240a035ec2a869fe1a37dde002b57ebc4417a27ccd853", size = 252084, upload-time = "2026-03-25T20:21:45.62Z" }, + { url = "https://files.pythonhosted.org/packages/00/71/3a69e86f3eafe8c7a59d008d245888051005bd657760e96d5fbfb0b740c2/tomli-2.4.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:7f94b27a62cfad8496c8d2513e1a222dd446f095fca8987fceef261225538a15", size = 247973, upload-time = "2026-03-25T20:21:46.937Z" }, + { url = "https://files.pythonhosted.org/packages/67/50/361e986652847fec4bd5e4a0208752fbe64689c603c7ae5ea7cb16b1c0ca/tomli-2.4.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:ede3e6487c5ef5d28634ba3f31f989030ad6af71edfb0055cbbd14189ff240ba", size = 256223, upload-time = "2026-03-25T20:21:48.467Z" }, + { url = "https://files.pythonhosted.org/packages/8c/9a/b4173689a9203472e5467217e0154b00e260621caa227b6fa01feab16998/tomli-2.4.1-cp314-cp314-win32.whl", hash = "sha256:3d48a93ee1c9b79c04bb38772ee1b64dcf18ff43085896ea460ca8dec96f35f6", size = 98973, upload-time = "2026-03-25T20:21:49.526Z" }, + { url = "https://files.pythonhosted.org/packages/14/58/640ac93bf230cd27d002462c9af0d837779f8773bc03dee06b5835208214/tomli-2.4.1-cp314-cp314-win_amd64.whl", hash = "sha256:88dceee75c2c63af144e456745e10101eb67361050196b0b6af5d717254dddf7", size = 109082, upload-time = "2026-03-25T20:21:50.506Z" }, + { url = "https://files.pythonhosted.org/packages/d5/2f/702d5e05b227401c1068f0d386d79a589bb12bf64c3d2c72ce0631e3bc49/tomli-2.4.1-cp314-cp314-win_arm64.whl", hash = "sha256:b8c198f8c1805dc42708689ed6864951fd2494f924149d3e4bce7710f8eb5232", size = 96490, upload-time = "2026-03-25T20:21:51.474Z" }, + { url = "https://files.pythonhosted.org/packages/45/4b/b877b05c8ba62927d9865dd980e34a755de541eb65fffba52b4cc495d4d2/tomli-2.4.1-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:d4d8fe59808a54658fcc0160ecfb1b30f9089906c50b23bcb4c69eddc19ec2b4", size = 164263, upload-time = "2026-03-25T20:21:52.543Z" }, + { url = "https://files.pythonhosted.org/packages/24/79/6ab420d37a270b89f7195dec5448f79400d9e9c1826df982f3f8e97b24fd/tomli-2.4.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:7008df2e7655c495dd12d2a4ad038ff878d4ca4b81fccaf82b714e07eae4402c", size = 160736, upload-time = "2026-03-25T20:21:53.674Z" }, + { url = "https://files.pythonhosted.org/packages/02/e0/3630057d8eb170310785723ed5adcdfb7d50cb7e6455f85ba8a3deed642b/tomli-2.4.1-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1d8591993e228b0c930c4bb0db464bdad97b3289fb981255d6c9a41aedc84b2d", size = 270717, upload-time = "2026-03-25T20:21:55.129Z" }, + { url = "https://files.pythonhosted.org/packages/7a/b4/1613716072e544d1a7891f548d8f9ec6ce2faf42ca65acae01d76ea06bb0/tomli-2.4.1-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:734e20b57ba95624ecf1841e72b53f6e186355e216e5412de414e3c51e5e3c41", size = 278461, upload-time = "2026-03-25T20:21:56.228Z" }, + { url = "https://files.pythonhosted.org/packages/05/38/30f541baf6a3f6df77b3df16b01ba319221389e2da59427e221ef417ac0c/tomli-2.4.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:8a650c2dbafa08d42e51ba0b62740dae4ecb9338eefa093aa5c78ceb546fcd5c", size = 274855, upload-time = "2026-03-25T20:21:57.653Z" }, + { url = "https://files.pythonhosted.org/packages/77/a3/ec9dd4fd2c38e98de34223b995a3b34813e6bdadf86c75314c928350ed14/tomli-2.4.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:504aa796fe0569bb43171066009ead363de03675276d2d121ac1a4572397870f", size = 283144, upload-time = "2026-03-25T20:21:59.089Z" }, + { url = "https://files.pythonhosted.org/packages/ef/be/605a6261cac79fba2ec0c9827e986e00323a1945700969b8ee0b30d85453/tomli-2.4.1-cp314-cp314t-win32.whl", hash = "sha256:b1d22e6e9387bf4739fbe23bfa80e93f6b0373a7f1b96c6227c32bef95a4d7a8", size = 108683, upload-time = "2026-03-25T20:22:00.214Z" }, + { url = "https://files.pythonhosted.org/packages/12/64/da524626d3b9cc40c168a13da8335fe1c51be12c0a63685cc6db7308daae/tomli-2.4.1-cp314-cp314t-win_amd64.whl", hash = "sha256:2c1c351919aca02858f740c6d33adea0c5deea37f9ecca1cc1ef9e884a619d26", size = 121196, upload-time = "2026-03-25T20:22:01.169Z" }, + { url = "https://files.pythonhosted.org/packages/5a/cd/e80b62269fc78fc36c9af5a6b89c835baa8af28ff5ad28c7028d60860320/tomli-2.4.1-cp314-cp314t-win_arm64.whl", hash = "sha256:eab21f45c7f66c13f2a9e0e1535309cee140182a9cdae1e041d02e47291e8396", size = 100393, upload-time = "2026-03-25T20:22:02.137Z" }, + { url = "https://files.pythonhosted.org/packages/7b/61/cceae43728b7de99d9b847560c262873a1f6c98202171fd5ed62640b494b/tomli-2.4.1-py3-none-any.whl", hash = "sha256:0d85819802132122da43cb86656f8d1f8c6587d54ae7dcaf30e90533028b49fe", size = 14583, upload-time = "2026-03-25T20:22:03.012Z" }, ] [[package]] name = "torch" -version = "2.12.1" +version = "2.13.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "cuda-bindings", marker = "sys_platform == 'linux'" }, - { name = "cuda-toolkit", extra = ["cudart", "cufft", "cufile", "cupti", "curand", "cusolver", "cusparse", "nvjitlink", "nvrtc", "nvtx"], marker = "sys_platform == 'linux'" }, + { name = "cuda-bindings", marker = "python_full_version < '3.15' and sys_platform == 'linux'" }, + { name = "cuda-toolkit", extra = ["cublas", "cudart", "cufft", "cufile", "cupti", "curand", "cusolver", "cusparse", "nvjitlink", "nvrtc", "nvtx"], marker = "sys_platform == 'linux'" }, { name = "filelock" }, { name = "fsspec" }, { name = "jinja2" }, { name = "networkx", version = "3.4.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, - { name = "networkx", version = "3.5", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" }, - { name = "nvidia-cublas", marker = "sys_platform == 'linux'" }, + { name = "networkx", version = "3.6.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" }, { name = "nvidia-cudnn-cu13", marker = "sys_platform == 'linux'" }, { name = "nvidia-cusparselt-cu13", marker = "sys_platform == 'linux'" }, { name = "nvidia-nccl-cu13", marker = "sys_platform == 'linux'" }, { name = "nvidia-nvshmem-cu13", marker = "sys_platform == 'linux'" }, { name = "setuptools" }, { name = "sympy" }, - { name = "triton", marker = "sys_platform == 'linux'" }, + { name = "triton", marker = "python_full_version < '3.15' and sys_platform == 'linux'" }, { name = "typing-extensions" }, ] wheels = [ - { url = "https://files.pythonhosted.org/packages/db/ed/ff0c4f8cef63977a646dc80e40c05cae873f4097b12dc87e1cd7e1cecf42/torch-2.12.1-cp310-cp310-macosx_14_0_arm64.whl", hash = "sha256:ec56e82be6a8b0c036771a77f7d32ad3c299770571af9815b3dafe61434389d5", size = 87967927, upload-time = "2026-06-17T21:08:43.16Z" }, - { url = "https://files.pythonhosted.org/packages/85/1b/c8ecf60c9dba535f9ea341c359c600c0bd877a7ca14b3296f13316321847/torch-2.12.1-cp310-cp310-manylinux_2_28_aarch64.whl", hash = "sha256:42cd7339bf266f14944710e8274be63e7e012bb937834a8d85a8327a9860eba6", size = 426366829, upload-time = "2026-06-17T21:07:18.574Z" }, - { url = "https://files.pythonhosted.org/packages/ab/d6/73d4a3f27e00526e98086f3a64ab609af1345cca62367749fbc3c8e4b83c/torch-2.12.1-cp310-cp310-manylinux_2_28_x86_64.whl", hash = "sha256:a7817f0f89a796d9de239d06f69faf5d7e19a6a5db6710a5ead777c912f9f50a", size = 532144834, upload-time = "2026-06-17T21:08:00.633Z" }, - { url = "https://files.pythonhosted.org/packages/e3/51/4010c8fa6f9d1f42c054a321970ca95ec58e4e4494f5b53a34c3f3c9e310/torch-2.12.1-cp310-cp310-win_amd64.whl", hash = "sha256:2af3d9cc866e0a15ae7635ff0a9c61d6624a353ad657f5bcd8d86c26cdc64693", size = 122949863, upload-time = "2026-06-17T21:08:39.016Z" }, - { url = "https://files.pythonhosted.org/packages/59/38/7028d3be540f1dcdf41660a2b01d0c51d2cb73915fe370d84e4d277a6d47/torch-2.12.1-cp311-cp311-macosx_14_0_arm64.whl", hash = "sha256:ef81f503912effea2ce3d9b12a2e3a6ed488943e91271c90c7a829f60baf6aa2", size = 87975425, upload-time = "2026-06-17T21:08:34.094Z" }, - { url = "https://files.pythonhosted.org/packages/5a/e3/750b3e3548635ceac03ba255daa26dbc7ed66ca3484dc4b4d955ab7f4501/torch-2.12.1-cp311-cp311-manylinux_2_28_aarch64.whl", hash = "sha256:107df6888624bdea41508f9aeb6149d9333c737a5530ceecb56c904e811369ae", size = 426379894, upload-time = "2026-06-17T21:06:55.077Z" }, - { url = "https://files.pythonhosted.org/packages/dc/ca/ed24783da629ff3e640ba3f70a7639e9045d3d88b93ee6bc47b8a28a1f2c/torch-2.12.1-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:6e29e7e74d05bda7d955c75e99459f878ebd970ef851b4057edbd3b34a5eb4a3", size = 532169264, upload-time = "2026-06-17T21:08:17.65Z" }, - { url = "https://files.pythonhosted.org/packages/46/61/c63f0158446f3a98ea672b004d761b848911eba567ea4a624c7db5aadc04/torch-2.12.1-cp311-cp311-win_amd64.whl", hash = "sha256:a513506cfda3c1c78dabeb6574c1597538c0254b3d39af174dde35d8177f4ce3", size = 122953086, upload-time = "2026-06-17T21:08:27.69Z" }, - { url = "https://files.pythonhosted.org/packages/f0/54/efb7ebca77970012b0cc21687a55d70eb2ba514b2c2b8e18d9fb1222f3be/torch-2.12.1-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:d2dd0f2c5f7ccbddaf34cade0deaf476808368f902b9cdb7f36a2ab42301bc0e", size = 87991951, upload-time = "2026-06-17T21:07:49.309Z" }, - { url = "https://files.pythonhosted.org/packages/1e/00/4210d76ca7424981f04033ebe7e48816ab83287a62538747a58825db770c/torch-2.12.1-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:2de4e19b88a481482c6c75291f2d6a52eda3ce51f311b29aa9b68499c830c07c", size = 426382721, upload-time = "2026-06-17T21:06:41.842Z" }, - { url = "https://files.pythonhosted.org/packages/76/1f/bc9f5a5aa569307076365f25afcebacb22e9c754b1bcfbaaa146627c7fda/torch-2.12.1-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:649e4ced014ba646f76f8cb9c9726735a6323eb321b7919f942790a923f90921", size = 532261322, upload-time = "2026-06-17T21:06:06.673Z" }, - { url = "https://files.pythonhosted.org/packages/9e/49/c549461daa008159d006a76a991fbc2f26fa8bac27a4030c858463dcb20f/torch-2.12.1-cp312-cp312-win_amd64.whl", hash = "sha256:e86550597877fb272ddc52db2f85b82cb601ea7bd932576a0340152cae2200b3", size = 122988095, upload-time = "2026-06-17T21:07:44.9Z" }, - { url = "https://files.pythonhosted.org/packages/ff/4a/0300261818e1560d72cc160ac826005507e8b7ca0a35788b591436d05b4a/torch-2.12.1-cp313-cp313-macosx_14_0_arm64.whl", hash = "sha256:c75e93173c700bccd6bfcc4a9d19ce242ab6dacd1f1781483027a16239b9e650", size = 87992358, upload-time = "2026-06-17T21:07:40.299Z" }, - { url = "https://files.pythonhosted.org/packages/30/a7/874a5ca05e8f159211dca7921060f7057acc1adb26431e119fd150623efc/torch-2.12.1-cp313-cp313-manylinux_2_28_aarch64.whl", hash = "sha256:fcb61ccd20784b62bdd78ec84238a5cfb383b4994902e03bac95505ab360884c", size = 426386134, upload-time = "2026-06-17T21:07:31.481Z" }, - { url = "https://files.pythonhosted.org/packages/e1/75/20bb8fe9c1ad6538cce8cd0391b51927ae5af0b17ed1eab44b8824465dc1/torch-2.12.1-cp313-cp313-manylinux_2_28_x86_64.whl", hash = "sha256:f4afc8083dff08719edbea346644476e3cec0cf40ebe256be0ee5d5b7c7e8c0d", size = 532268019, upload-time = "2026-06-17T21:05:37.925Z" }, - { url = "https://files.pythonhosted.org/packages/d1/fa/824ddb662af55b2eabc0dbb7b57c7c0b1bcd93693754a2b8509ec4d16490/torch-2.12.1-cp313-cp313-win_amd64.whl", hash = "sha256:f92609e3b3ce72f25e2eb780d043ced2480c1a86c47c852604fc7a9108648386", size = 122987777, upload-time = "2026-06-17T21:07:09.49Z" }, - { url = "https://files.pythonhosted.org/packages/63/b7/1b49fe7086ea36839cc80abc43174c43d0ab6f676c0891c871c162f44fe3/torch-2.12.1-cp314-cp314-macosx_14_0_arm64.whl", hash = "sha256:e9b6f7d2dd66ea87a3ae620069d31335d594c06effb1a383bdd21cfe61e44ece", size = 88010025, upload-time = "2026-06-17T21:07:03.934Z" }, - { url = "https://files.pythonhosted.org/packages/d7/06/5b44063a6545036dcc680d2d303b137d9176cfb2cc1e1863e3ef94abeb52/torch-2.12.1-cp314-cp314-manylinux_2_28_aarch64.whl", hash = "sha256:7973ccd3d2cd35c74449213f7bded199bec6c6247e705cbeda7407af79703d91", size = 426392891, upload-time = "2026-06-17T21:05:52.261Z" }, - { url = "https://files.pythonhosted.org/packages/f8/dd/c9ce9a4b0eb3c5bb92d9ea56766e2c22559f0b45171149188494edcce80f/torch-2.12.1-cp314-cp314-manylinux_2_28_x86_64.whl", hash = "sha256:c64ac4aac16be5e296dcd912305605804b203333c690bf98c55bc09494ee92ad", size = 532272494, upload-time = "2026-06-17T21:06:22.72Z" }, - { url = "https://files.pythonhosted.org/packages/21/7c/f3a601fc1b1f663ff269bfe553654e638651939aa6563e8daa7167c33098/torch-2.12.1-cp314-cp314-win_amd64.whl", hash = "sha256:f6dc4caf7eb4adb38a2d9f536b51db56310fdd1254e69a2d96767e1367c892b3", size = 122987254, upload-time = "2026-06-17T21:06:33.199Z" }, - { url = "https://files.pythonhosted.org/packages/e6/8c/b8087556cf81ddd808dbeb34afb8396d7ae7a1694ab489f08b1a0004e7d0/torch-2.12.1-cp314-cp314t-macosx_14_0_arm64.whl", hash = "sha256:2afbb2bdaa8a95040e733f05492ddf133c3967c9b7ce0abd218d704b6cab437d", size = 88303173, upload-time = "2026-06-17T21:05:06.603Z" }, - { url = "https://files.pythonhosted.org/packages/4a/07/fe09d1699fbed2afa10ebc692ff2b99d113f2605b6748cea633989e2789a/torch-2.12.1-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:97eba061fcb042fed191400b15568990073d67eaacaa6ee9b7ca01dd8b790fe9", size = 426404009, upload-time = "2026-06-17T21:04:57.557Z" }, - { url = "https://files.pythonhosted.org/packages/2e/f7/0ce4f6c1962c60ded7270e0a9eb560fb615c92b89d332cf9e3dff36d5ecc/torch-2.12.1-cp314-cp314t-manylinux_2_28_x86_64.whl", hash = "sha256:3867b861391701012adb2df93360efb88494dca245a185e3bb7624495cfe3f33", size = 532184292, upload-time = "2026-06-17T21:05:17.526Z" }, - { url = "https://files.pythonhosted.org/packages/70/db/e384c12aba30320ca92aaaf557456cbcb26f04b4df307728bb8f019f5000/torch-2.12.1-cp314-cp314t-win_amd64.whl", hash = "sha256:dd15595f8fc764cffde8c6361a3beb6ef69a028c851b1b3e70e077f615980d4e", size = 123231142, upload-time = "2026-06-17T21:05:27.061Z" }, + { url = "https://files.pythonhosted.org/packages/7f/e7/19894fdb51c7dbaf94f5a79bb0871da0992e8e4241e579cb006da46d2e58/torch-2.13.0-cp310-cp310-macosx_14_0_arm64.whl", hash = "sha256:94f0de129916f77b8dc2c7a8eff644cfeddfe59e39c9f55e9f6e17543410281d", size = 111178962, upload-time = "2026-07-08T16:05:49.855Z" }, + { url = "https://files.pythonhosted.org/packages/d1/5c/b1d5de470c54e339b30a92d96683a71bcebd78f5f2a7fc714cd6dc6bbd68/torch-2.13.0-cp310-cp310-manylinux_2_28_aarch64.whl", hash = "sha256:0ab4b69f3ee03a62a002cfbf77b1ca5e88aceb4ea64cb4388bb28f638ddbb045", size = 427198333, upload-time = "2026-07-08T16:05:36.847Z" }, + { url = "https://files.pythonhosted.org/packages/50/c0/68a84105e1fcb8970144b388ff3d3e5dc15a3be28c1e247841f7d7247e41/torch-2.13.0-cp310-cp310-manylinux_2_28_x86_64.whl", hash = "sha256:c78b7b4d04461855a764cf01bae9a462bb88bc93defcfa11235cbc8fdf3e12c4", size = 526555154, upload-time = "2026-07-08T16:05:06.507Z" }, + { url = "https://files.pythonhosted.org/packages/2e/c9/0bb9d097b03cbaf96bb75b15e867347b8e41bfcdfe0539452d17d9e63993/torch-2.13.0-cp310-cp310-win_amd64.whl", hash = "sha256:2bd30b6b730d987fa386ce3898933762c5cb8cc82eb0535211d787cc3ce2dfeb", size = 122015602, upload-time = "2026-07-08T16:05:45.25Z" }, + { url = "https://files.pythonhosted.org/packages/5b/fe/cba54dc58523434919b66f13a667e36e436deddd77ca519e96553617d4ec/torch-2.13.0-cp311-cp311-macosx_14_0_arm64.whl", hash = "sha256:e76f9bcecc52b8ff711239a2f7547d5353df95878ab232f0773c1d95928b92f8", size = 111187938, upload-time = "2026-07-08T16:05:17.065Z" }, + { url = "https://files.pythonhosted.org/packages/c2/59/1e3160e18e12aa3038390efab3ce02b36a9d4d6a527ecdd8520dca2e68d8/torch-2.13.0-cp311-cp311-manylinux_2_28_aarch64.whl", hash = "sha256:092790c696a760c729fd5722835f50b9d81fd7c8f141571f3f3cf4081a8f664c", size = 427199369, upload-time = "2026-07-08T16:04:51.054Z" }, + { url = "https://files.pythonhosted.org/packages/01/79/1f2d34ad7034ee1c7ffc1cf8bf0f8213af2a81df6ecdb3997ecec107c09d/torch-2.13.0-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:60fcdcb2f3876e21146cb4524ef06397d727ca9ad5f020818547e25075fe3cb7", size = 526574961, upload-time = "2026-07-08T16:04:07.075Z" }, + { url = "https://files.pythonhosted.org/packages/6c/fd/0f2ce40f58aefbdb3392f9acce3c8171940943ae2d661f70558bfa73befb/torch-2.13.0-cp311-cp311-win_amd64.whl", hash = "sha256:a0d8b11f16a48d60e2015d8213aa0390744cbebb98e58b62b3514dddc656e330", size = 122015870, upload-time = "2026-07-08T16:05:27.59Z" }, + { url = "https://files.pythonhosted.org/packages/c4/3a/ed0f4d4d1dcde03bced7aac9a28e800abcdc0cbd06b6775044c9fbd877b7/torch-2.13.0-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:2fe228aba290d14b9f31b049be550dbd469c3fd3013d7a19705b30454da97027", size = 111213045, upload-time = "2026-07-08T16:05:22.997Z" }, + { url = "https://files.pythonhosted.org/packages/df/a9/f6a2a4d763ff1df02e9a64c477029db614295bc9367f4131223791ccc243/torch-2.13.0-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:572df8be8ffb4599c88cbd6a0726f1f854f4da65d2e3c09f0e2c2283333cd6d4", size = 427210998, upload-time = "2026-07-08T16:04:37.708Z" }, + { url = "https://files.pythonhosted.org/packages/f3/82/fea946351658e6534db52d2cc12bc53087cbf87f9440c5f180f367c1950b/torch-2.13.0-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:796633c4cdf0fe2cdced72d8f88f22e73dbcfce83132763162f6d4bff13b820b", size = 526605292, upload-time = "2026-07-08T16:04:22.81Z" }, + { url = "https://files.pythonhosted.org/packages/21/d6/e8f3c6f7e01f626f77259de9860d2a78bc84c40539e28e79b7e98b0bb659/torch-2.13.0-cp312-cp312-win_amd64.whl", hash = "sha256:024c6cc0c1b085f2f91f20a3dc27b0471d021c31ce84b81be3afdc39f791fd9d", size = 122057313, upload-time = "2026-07-08T16:03:53.43Z" }, + { url = "https://files.pythonhosted.org/packages/0d/fa/c1c10b7aff4a9a3e8956d4f0a5f468fa6db7abc3208805719076772b4833/torch-2.13.0-cp313-cp313-macosx_14_0_arm64.whl", hash = "sha256:33449899ce5496c1b84b4853179d94fd102028ae1407314d9fb956bb79e70d09", size = 111213743, upload-time = "2026-07-08T16:03:28.579Z" }, + { url = "https://files.pythonhosted.org/packages/11/18/9ecb37b56293a0be8d80f810bf672a72fe7e02f8b475d5ef1b9bf8a0d748/torch-2.13.0-cp313-cp313-manylinux_2_28_aarch64.whl", hash = "sha256:1e09d6a722504957c694faceca843acde562786df1144ebcc5a74075ec7f6005", size = 427213008, upload-time = "2026-07-08T16:03:44.106Z" }, + { url = "https://files.pythonhosted.org/packages/d4/5a/7c50ba1b7b713d71d34669c6d13dab0a11531a3eceb0307a5162dbfec0f7/torch-2.13.0-cp313-cp313-manylinux_2_28_x86_64.whl", hash = "sha256:a3a9a21312872af8a26950b2c15680335a386a1f56ed03e780653d78b9607e9e", size = 526602329, upload-time = "2026-07-08T16:03:12.649Z" }, + { url = "https://files.pythonhosted.org/packages/91/3d/e7adcc6aaf36961cd18f56cf8ad0f3058c3a5c84ccf391762176c94581b8/torch-2.13.0-cp313-cp313-win_amd64.whl", hash = "sha256:49b58f1e2c52440abb6f17c28f0335fe6c6d01ad1a7f55b0183b81e4b34d64e6", size = 122057920, upload-time = "2026-07-08T16:03:01.808Z" }, + { url = "https://files.pythonhosted.org/packages/36/76/6dcc7f0c07052102dd36f83cbc5800842a909c8c3fbf1a7f8a5844954de9/torch-2.13.0-cp314-cp314-macosx_14_0_arm64.whl", hash = "sha256:d849b390e07d8d333ce8ecaf91b273c656c598379a19c9acf1318a883f6b391c", size = 111227066, upload-time = "2026-07-08T16:03:33.6Z" }, + { url = "https://files.pythonhosted.org/packages/e9/09/2c10e8cd0e00fa5d23c052df6ce467eaa7182399f5e0f824f1e4ff42ccae/torch-2.13.0-cp314-cp314-manylinux_2_28_aarch64.whl", hash = "sha256:a3893dc2da0a972a8ca5d698c85a9f967559ac5f8ee1797b77408aa8734d073c", size = 427226309, upload-time = "2026-07-08T16:02:53.127Z" }, + { url = "https://files.pythonhosted.org/packages/76/c6/22c2102bbef14ca6a6cb4c20e42f088e49c5f812be4e160ae57502e325f9/torch-2.13.0-cp314-cp314-manylinux_2_28_x86_64.whl", hash = "sha256:49f1ea385c754e54919408a9bb3b5a72b0b755bbe2c916c1d6f70afbec4908a2", size = 526614507, upload-time = "2026-07-08T16:02:16.441Z" }, + { url = "https://files.pythonhosted.org/packages/2b/0c/7d1deb6bce5bc3e6042caf39100ac768eba3b9a098e1dddd16f75bd6489b/torch-2.13.0-cp314-cp314-win_amd64.whl", hash = "sha256:4f8573e3ce9ebcd53fe922f01077a6085ccdfbe5f12fd215883a9d87d7a744fd", size = 122051871, upload-time = "2026-07-08T16:03:23.521Z" }, + { url = "https://files.pythonhosted.org/packages/f4/ce/aa8b7f9949d32e0f2f624f342bc3b48112c1b8a130288465938bc83bcbf9/torch-2.13.0-cp314-cp314t-macosx_14_0_arm64.whl", hash = "sha256:c28def70706c2f9ecc752574766e8ae4da9b810ab6676b611166761a78a9f1e1", size = 111537025, upload-time = "2026-07-08T16:02:44.28Z" }, + { url = "https://files.pythonhosted.org/packages/69/d1/491e3a0389430946145888b0203f2b6a759ce2a61481b96a85c2da4f2ced/torch-2.13.0-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:31061ff56ed8fbf26c749806905aeb749ebeb819810fd5d52508aa5afd90dddc", size = 427219769, upload-time = "2026-07-08T16:02:31.18Z" }, + { url = "https://files.pythonhosted.org/packages/9a/1d/38006e045bf0a1fc28ef01e757c554e59e59a8770c284bc4f47b14e60441/torch-2.13.0-cp314-cp314t-manylinux_2_28_x86_64.whl", hash = "sha256:cc26eead4cf51d0b544e31e364dcf000846549c273bd148936fe9d24d29acb92", size = 526571320, upload-time = "2026-07-08T16:01:59.348Z" }, + { url = "https://files.pythonhosted.org/packages/56/94/655c91992a882bd5071aa0b5d22a07dbb130d801e872be97c0b627a7c693/torch-2.13.0-cp314-cp314t-win_amd64.whl", hash = "sha256:a7de8a313090dc5c7d7ba4bfe5c3be222528f9a4dba1acc83bddb1157360c4b8", size = 122306773, upload-time = "2026-07-08T16:02:39.832Z" }, ] [[package]] @@ -2928,11 +2965,11 @@ wheels = [ [[package]] name = "typing-extensions" -version = "4.15.0" +version = "4.16.0" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/72/94/1a15dd82efb362ac84269196e94cf00f187f7ed21c242792a923cdb1c61f/typing_extensions-4.15.0.tar.gz", hash = "sha256:0cea48d173cc12fa28ecabc3b837ea3cf6f38c6d1136f85cbaaf598984861466", size = 109391, upload-time = "2025-08-25T13:49:26.313Z" } +sdist = { url = "https://files.pythonhosted.org/packages/f6/cc/6253133b5bb138fc3306cebfbda2c520f545d36b5be2c7255cc528bb45d6/typing_extensions-4.16.0.tar.gz", hash = "sha256:dc983d19a509c94dba722ee6abd33940f7c05a89e243c47e907eb4db6f1a43e5", size = 113555, upload-time = "2026-07-02T08:40:05.92Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/18/67/36e9267722cc04a6b9f15c7f3441c2363321a3ea07da7ae0c0707beb2a9c/typing_extensions-4.15.0-py3-none-any.whl", hash = "sha256:f0fa19c6845758ab08074a0cfa8b7aecb71c999ca73d62883bc25cc018c4e548", size = 44614, upload-time = "2025-08-25T13:49:24.86Z" }, + { url = "https://files.pythonhosted.org/packages/49/d3/b8441a820a491ddfc024b0b0cf0393375b75ea13866d9c66727e54c2fc80/typing_extensions-4.16.0-py3-none-any.whl", hash = "sha256:481caa481374e813c1b176ada14e97f1f67a4539ce9cfeb3f350d78d6370c2e8", size = 45571, upload-time = "2026-07-02T08:40:04.659Z" }, ] [[package]] @@ -2949,289 +2986,386 @@ wheels = [ [[package]] name = "tzdata" -version = "2025.2" +version = "2026.3" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/95/32/1a225d6164441be760d75c2c42e2780dc0873fe382da3e98a2e1e48361e5/tzdata-2025.2.tar.gz", hash = "sha256:b60a638fcc0daffadf82fe0f57e53d06bdec2f36c4df66280ae79bce6bd6f2b9", size = 196380, upload-time = "2025-03-23T13:54:43.652Z" } +sdist = { url = "https://files.pythonhosted.org/packages/92/ff/5a28bdfd8c3ebec42564ac7d0e54ca3db65044a9314a97f9564fa7a1e926/tzdata-2026.3.tar.gz", hash = "sha256:4a1518b8993086a7982523e071643f3c0e5f213e75b21318e78bcabfff9d1415", size = 198674, upload-time = "2026-07-10T08:50:37.887Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/5c/23/c7abc0ca0a1526a0774eca151daeb8de62ec457e77262b66b359c3c7679e/tzdata-2025.2-py2.py3-none-any.whl", hash = "sha256:1a403fada01ff9221ca8044d701868fa132215d84beb92242d9acd2147f667a8", size = 347839, upload-time = "2025-03-23T13:54:41.845Z" }, + { url = "https://files.pythonhosted.org/packages/e5/6d/b53b99a9f2766d095985947a5782f1702cabb129a34f7a802d7197af832f/tzdata-2026.3-py2.py3-none-any.whl", hash = "sha256:dc096730c87af6cab1b171c9d532be840741ff5d459015e7f6947bd7d7e54931", size = 348168, upload-time = "2026-07-10T08:50:36.46Z" }, ] [[package]] name = "urllib3" -version = "2.5.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/15/22/9ee70a2574a4f4599c47dd506532914ce044817c7752a79b6a51286319bc/urllib3-2.5.0.tar.gz", hash = "sha256:3fc47733c7e419d4bc3f6b3dc2b4f890bb743906a30d56ba4a5bfa4bbff92760", size = 393185, upload-time = "2025-06-18T14:07:41.644Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/a7/c2/fe1e52489ae3122415c51f387e221dd0773709bad6c6cdaa599e8a2c5185/urllib3-2.5.0-py3-none-any.whl", hash = "sha256:e6b01673c0fa6a13e374b50871808eb3bf7046c4b125b216f6bf1cc604cff0dc", size = 129795, upload-time = "2025-06-18T14:07:40.39Z" }, -] - -[[package]] -name = "werkzeug" -version = "3.1.3" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "markupsafe" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/9f/69/83029f1f6300c5fb2471d621ab06f6ec6b3324685a2ce0f9777fd4a8b71e/werkzeug-3.1.3.tar.gz", hash = "sha256:60723ce945c19328679790e3282cc758aa4a6040e4bb330f53d30fa546d44746", size = 806925, upload-time = "2024-11-08T15:52:18.093Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/52/24/ab44c871b0f07f491e5d2ad12c9bd7358e527510618cb1b803a88e986db1/werkzeug-3.1.3-py3-none-any.whl", hash = "sha256:54b78bf3716d19a65be4fceccc0d1d7b89e608834989dfae50ea87564639213e", size = 224498, upload-time = "2024-11-08T15:52:16.132Z" }, -] - -[[package]] -name = "wheel" -version = "0.45.1" +version = "2.7.0" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/8a/98/2d9906746cdc6a6ef809ae6338005b3f21bb568bea3165cfc6a243fdc25c/wheel-0.45.1.tar.gz", hash = "sha256:661e1abd9198507b1409a20c02106d9670b2576e916d58f520316666abca6729", size = 107545, upload-time = "2024-11-23T00:18:23.513Z" } +sdist = { url = "https://files.pythonhosted.org/packages/53/0c/06f8b233b8fd13b9e5ee11424ef85419ba0d8ba0b3138bf360be2ff56953/urllib3-2.7.0.tar.gz", hash = "sha256:231e0ec3b63ceb14667c67be60f2f2c40a518cb38b03af60abc813da26505f4c", size = 433602, upload-time = "2026-05-07T16:13:18.596Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/0b/2c/87f3254fd8ffd29e4c02732eee68a83a1d3c346ae39bc6822dcbcb697f2b/wheel-0.45.1-py3-none-any.whl", hash = "sha256:708e7481cc80179af0e556bbf0cc00b8444c7321e2700b8d8580231d13017248", size = 72494, upload-time = "2024-11-23T00:18:21.207Z" }, + { url = "https://files.pythonhosted.org/packages/7f/3e/5db95bcf282c52709639744ca2a8b149baccf648e39c8cc87553df9eae0c/urllib3-2.7.0-py3-none-any.whl", hash = "sha256:9fb4c81ebbb1ce9531cce37674bbc6f1360472bc18ca9a553ede278ef7276897", size = 131087, upload-time = "2026-05-07T16:13:17.151Z" }, ] [[package]] name = "wrapt" -version = "1.17.3" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/95/8f/aeb76c5b46e273670962298c23e7ddde79916cb74db802131d49a85e4b7d/wrapt-1.17.3.tar.gz", hash = "sha256:f66eb08feaa410fe4eebd17f2a2c8e2e46d3476e9f8c783daa8e09e0faa666d0", size = 55547, upload-time = "2025-08-12T05:53:21.714Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/de/17/9f8f86755c191d6779d7ddead1a53c7a8aa18bccb7cea8e7e72dfa6a8a09/wrapt-1.17.3-cp310-cp310-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:f9b2601381be482f70e5d1051a5965c25fb3625455a2bf520b5a077b22afb775", size = 81975, upload-time = "2025-08-12T05:52:30.109Z" }, - { url = "https://files.pythonhosted.org/packages/f2/15/dd576273491f9f43dd09fce517f6c2ce6eb4fe21681726068db0d0467096/wrapt-1.17.3-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:343e44b2a8e60e06a7e0d29c1671a0d9951f59174f3709962b5143f60a2a98bd", size = 83149, upload-time = "2025-08-12T05:52:09.316Z" }, - { url = "https://files.pythonhosted.org/packages/0c/c4/5eb4ce0d4814521fee7aa806264bf7a114e748ad05110441cd5b8a5c744b/wrapt-1.17.3-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:33486899acd2d7d3066156b03465b949da3fd41a5da6e394ec49d271baefcf05", size = 82209, upload-time = "2025-08-12T05:52:10.331Z" }, - { url = "https://files.pythonhosted.org/packages/31/4b/819e9e0eb5c8dc86f60dfc42aa4e2c0d6c3db8732bce93cc752e604bb5f5/wrapt-1.17.3-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:e6f40a8aa5a92f150bdb3e1c44b7e98fb7113955b2e5394122fa5532fec4b418", size = 81551, upload-time = "2025-08-12T05:52:31.137Z" }, - { url = "https://files.pythonhosted.org/packages/5d/8f/a32a99fc03e4b37e31b57cb9cefc65050ea08147a8ce12f288616b05ef54/wrapt-1.17.3-cp311-cp311-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:b32888aad8b6e68f83a8fdccbf3165f5469702a7544472bdf41f582970ed3311", size = 82376, upload-time = "2025-08-12T05:52:32.134Z" }, - { url = "https://files.pythonhosted.org/packages/31/57/4930cb8d9d70d59c27ee1332a318c20291749b4fba31f113c2f8ac49a72e/wrapt-1.17.3-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:8cccf4f81371f257440c88faed6b74f1053eef90807b77e31ca057b2db74edb1", size = 83604, upload-time = "2025-08-12T05:52:11.663Z" }, - { url = "https://files.pythonhosted.org/packages/a8/f3/1afd48de81d63dd66e01b263a6fbb86e1b5053b419b9b33d13e1f6d0f7d0/wrapt-1.17.3-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:d8a210b158a34164de8bb68b0e7780041a903d7b00c87e906fb69928bf7890d5", size = 82782, upload-time = "2025-08-12T05:52:12.626Z" }, - { url = "https://files.pythonhosted.org/packages/1e/d7/4ad5327612173b144998232f98a85bb24b60c352afb73bc48e3e0d2bdc4e/wrapt-1.17.3-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:79573c24a46ce11aab457b472efd8d125e5a51da2d1d24387666cd85f54c05b2", size = 82076, upload-time = "2025-08-12T05:52:33.168Z" }, - { url = "https://files.pythonhosted.org/packages/9f/81/5d931d78d0eb732b95dc3ddaeeb71c8bb572fb01356e9133916cd729ecdd/wrapt-1.17.3-cp312-cp312-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:042ec3bb8f319c147b1301f2393bc19dba6e176b7da446853406d041c36c7828", size = 88036, upload-time = "2025-08-12T05:52:34.784Z" }, - { url = "https://files.pythonhosted.org/packages/ca/38/2e1785df03b3d72d34fc6252d91d9d12dc27a5c89caef3335a1bbb8908ca/wrapt-1.17.3-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:3af60380ba0b7b5aeb329bc4e402acd25bd877e98b3727b0135cb5c2efdaefe9", size = 88156, upload-time = "2025-08-12T05:52:13.599Z" }, - { url = "https://files.pythonhosted.org/packages/b3/8b/48cdb60fe0603e34e05cffda0b2a4adab81fd43718e11111a4b0100fd7c1/wrapt-1.17.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:0b02e424deef65c9f7326d8c19220a2c9040c51dc165cddb732f16198c168396", size = 87102, upload-time = "2025-08-12T05:52:14.56Z" }, - { url = "https://files.pythonhosted.org/packages/3c/51/d81abca783b58f40a154f1b2c56db1d2d9e0d04fa2d4224e357529f57a57/wrapt-1.17.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:74afa28374a3c3a11b3b5e5fca0ae03bef8450d6aa3ab3a1e2c30e3a75d023dc", size = 87732, upload-time = "2025-08-12T05:52:36.165Z" }, - { url = "https://files.pythonhosted.org/packages/0c/37/6faf15cfa41bf1f3dba80cd3f5ccc6622dfccb660ab26ed79f0178c7497f/wrapt-1.17.3-cp313-cp313-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:6fd1ad24dc235e4ab88cda009e19bf347aabb975e44fd5c2fb22a3f6e4141277", size = 88072, upload-time = "2025-08-12T05:52:37.53Z" }, - { url = "https://files.pythonhosted.org/packages/78/f2/efe19ada4a38e4e15b6dff39c3e3f3f73f5decf901f66e6f72fe79623a06/wrapt-1.17.3-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0ed61b7c2d49cee3c027372df5809a59d60cf1b6c2f81ee980a091f3afed6a2d", size = 88214, upload-time = "2025-08-12T05:52:15.886Z" }, - { url = "https://files.pythonhosted.org/packages/40/90/ca86701e9de1622b16e09689fc24b76f69b06bb0150990f6f4e8b0eeb576/wrapt-1.17.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:423ed5420ad5f5529db9ce89eac09c8a2f97da18eb1c870237e84c5a5c2d60aa", size = 87105, upload-time = "2025-08-12T05:52:17.914Z" }, - { url = "https://files.pythonhosted.org/packages/fd/e0/d10bd257c9a3e15cbf5523025252cc14d77468e8ed644aafb2d6f54cb95d/wrapt-1.17.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:e01375f275f010fcbf7f643b4279896d04e571889b8a5b3f848423d91bf07050", size = 87766, upload-time = "2025-08-12T05:52:39.243Z" }, - { url = "https://files.pythonhosted.org/packages/c3/f7/c983d2762bcce2326c317c26a6a1e7016f7eb039c27cdf5c4e30f4160f31/wrapt-1.17.3-cp314-cp314-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:281262213373b6d5e4bb4353bc36d1ba4084e6d6b5d242863721ef2bf2c2930b", size = 87163, upload-time = "2025-08-12T05:52:40.965Z" }, - { url = "https://files.pythonhosted.org/packages/e4/0f/f673f75d489c7f22d17fe0193e84b41540d962f75fce579cf6873167c29b/wrapt-1.17.3-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:dc4a8d2b25efb6681ecacad42fca8859f88092d8732b170de6a5dddd80a1c8fa", size = 87963, upload-time = "2025-08-12T05:52:20.326Z" }, - { url = "https://files.pythonhosted.org/packages/df/61/515ad6caca68995da2fac7a6af97faab8f78ebe3bf4f761e1b77efbc47b5/wrapt-1.17.3-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:373342dd05b1d07d752cecbec0c41817231f29f3a89aa8b8843f7b95992ed0c7", size = 86945, upload-time = "2025-08-12T05:52:21.581Z" }, - { url = "https://files.pythonhosted.org/packages/d3/bd/4e70162ce398462a467bc09e768bee112f1412e563620adc353de9055d33/wrapt-1.17.3-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:d40770d7c0fd5cbed9d84b2c3f2e156431a12c9a37dc6284060fb4bec0b7ffd4", size = 86857, upload-time = "2025-08-12T05:52:43.043Z" }, - { url = "https://files.pythonhosted.org/packages/64/0e/f4472f2fdde2d4617975144311f8800ef73677a159be7fe61fa50997d6c0/wrapt-1.17.3-cp314-cp314t-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:5d4478d72eb61c36e5b446e375bbc49ed002430d17cdec3cecb36993398e1a9e", size = 108571, upload-time = "2025-08-12T05:52:44.521Z" }, - { url = "https://files.pythonhosted.org/packages/cc/01/9b85a99996b0a97c8a17484684f206cbb6ba73c1ce6890ac668bcf3838fb/wrapt-1.17.3-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:223db574bb38637e8230eb14b185565023ab624474df94d2af18f1cdb625216f", size = 113094, upload-time = "2025-08-12T05:52:22.618Z" }, - { url = "https://files.pythonhosted.org/packages/25/02/78926c1efddcc7b3aa0bc3d6b33a822f7d898059f7cd9ace8c8318e559ef/wrapt-1.17.3-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:e405adefb53a435f01efa7ccdec012c016b5a1d3f35459990afc39b6be4d5056", size = 110659, upload-time = "2025-08-12T05:52:24.057Z" }, - { url = "https://files.pythonhosted.org/packages/dc/ee/c414501ad518ac3e6fe184753632fe5e5ecacdcf0effc23f31c1e4f7bfcf/wrapt-1.17.3-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:88547535b787a6c9ce4086917b6e1d291aa8ed914fdd3a838b3539dc95c12804", size = 106946, upload-time = "2025-08-12T05:52:45.976Z" }, - { url = "https://files.pythonhosted.org/packages/1f/f6/a933bd70f98e9cf3e08167fc5cd7aaaca49147e48411c0bd5ae701bb2194/wrapt-1.17.3-py3-none-any.whl", hash = "sha256:7171ae35d2c33d326ac19dd8facb1e82e5fd04ef8c6c0e394d7af55a55051c22", size = 23591, upload-time = "2025-08-12T05:53:20.674Z" }, +version = "2.2.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/fe/a4/282c8e64300a59fc834518a54bf0afabb4ff9218b5fa76958b450459a844/wrapt-2.2.2.tar.gz", hash = "sha256:0788e321027c999bf221b667bd4a54aaefd1a36283749a860ac3eb77daed0302", size = 129068, upload-time = "2026-06-20T23:49:44.49Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e1/8b/59781d0fe7b0adfbea37f600857de4be68921e454aeecf1a11bda35cdccc/wrapt-2.2.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:055e6fcfaa28e58c6a8c247d48b92be9d56f818b7068aa4f22b15b3343a09931", size = 80556, upload-time = "2026-06-20T23:47:28.473Z" }, + { url = "https://files.pythonhosted.org/packages/94/dc/66c61aca927230c9cf97a3cb005c803971a1076ff9f7d61085d035c20085/wrapt-2.2.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:8374eb6b1a58809211e84ff835a182bb17ab2807a5bfef23204c8cff38178a00", size = 81648, upload-time = "2026-06-20T23:47:30.504Z" }, + { url = "https://files.pythonhosted.org/packages/23/1b/545eee1c18f3af4cf140bb5822b6ef81ebe569df0a63ac109973103a30a5/wrapt-2.2.2-cp310-cp310-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:656593bb3f5529f03d27af4136c4d7b11990e470bcbc6fefa5ef218695bece55", size = 152956, upload-time = "2026-06-20T23:47:31.867Z" }, + { url = "https://files.pythonhosted.org/packages/44/a7/6f42a3d03e44dc612a5dcff324e7366075a7857f0be2d49a8cb8a68279b8/wrapt-2.2.2-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:dfb00cb7bb22099e2f64b7340fb96113639aa7260c0972af3797ace2297b936c", size = 154771, upload-time = "2026-06-20T23:47:33.352Z" }, + { url = "https://files.pythonhosted.org/packages/bf/55/4d76175aaa97523c38f1d28f79d18ab41a1b116814158a818bc0eba00571/wrapt-2.2.2-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:e7f10ee0bd53673bfd52b67cbce83336fe6cad90d2377b03baf66491d2bbfb91", size = 149460, upload-time = "2026-06-20T23:47:34.712Z" }, + { url = "https://files.pythonhosted.org/packages/84/9b/12e23264d8f4735e8483262f95c5a6b03c3665fd2a84bdf99a45b6a2f4ec/wrapt-2.2.2-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:4402f57c5f0d0579599858ffbdd9bf4e3f0972f51096f2bd6cc7dab6b76ee49e", size = 153648, upload-time = "2026-06-20T23:47:36.092Z" }, + { url = "https://files.pythonhosted.org/packages/d6/a3/bcd5ec37289dcd85ecd4d15395a6a6063d60bc45ff94a9d77814e1e54d64/wrapt-2.2.2-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:3a4eb7964ff4643d333c84f880bcf554652b2a1050aebc54ae696327f61acfaf", size = 148502, upload-time = "2026-06-20T23:47:37.623Z" }, + { url = "https://files.pythonhosted.org/packages/f2/be/716d708f607fa70f8a6eb47dff8ee945d5278dfc89ffeeff33039d052e63/wrapt-2.2.2-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:e542b7c5af91e2123a8aabf19894319d5ec4268d2a9ffd2f239386133fc47746", size = 152238, upload-time = "2026-06-20T23:47:39.118Z" }, + { url = "https://files.pythonhosted.org/packages/b5/c0/1a48e7e54501274f5d906f18372221b13183b0afbb5b8bb4c7ca0392c0b4/wrapt-2.2.2-cp310-cp310-win32.whl", hash = "sha256:6e7e45b43d3c774d244fe7264378f5a3f0f383bc55a54a9866434e524540110f", size = 77278, upload-time = "2026-06-20T23:47:40.476Z" }, + { url = "https://files.pythonhosted.org/packages/b0/82/9cd69a1af288fbdedf01a10e3c8a0b6890b08c7f3f96d36a213699dbcd94/wrapt-2.2.2-cp310-cp310-win_amd64.whl", hash = "sha256:955f1d6e72a352e478de8d8b503abe301c5e139a141b62eb0923bd694995025f", size = 80131, upload-time = "2026-06-20T23:47:41.785Z" }, + { url = "https://files.pythonhosted.org/packages/7f/73/8db7e27daef37ae70a53ea62bef7fe80cc51a8b5e9e9181a8be6eb9a999c/wrapt-2.2.2-cp310-cp310-win_arm64.whl", hash = "sha256:b89d8d73c82db2bb7e6090b3afd7973f980d24e905cc34394eab60b884b3bf67", size = 79615, upload-time = "2026-06-20T23:47:43.109Z" }, + { url = "https://files.pythonhosted.org/packages/27/15/0c2d55168707465abfc41f33c0b23d792a5fa9b65c26983606940900a120/wrapt-2.2.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:f1a2ff355ece6a111ca7a20dc86df6659c9205d3fcee674ca34f2a2854fd4e73", size = 80782, upload-time = "2026-06-20T23:47:44.367Z" }, + { url = "https://files.pythonhosted.org/packages/7d/b5/5c0b093eb48f8a062ef6267d3cb36e9bb1b88440181f6545a383c60efdf8/wrapt-2.2.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:55b9a899e6fff5444f229d30aa6e9ac92d2216d9d60f33c771b5d76a760d5f8e", size = 81678, upload-time = "2026-06-20T23:47:45.857Z" }, + { url = "https://files.pythonhosted.org/packages/34/f3/de70937472dd3e8a4e6811192f9c6075efdffd4a2cd9b4596bf160f89668/wrapt-2.2.2-cp311-cp311-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:a2d78c363f97d8bd718ee40432c66395685e9e98528ccaa423c3355d1715a26d", size = 159671, upload-time = "2026-06-20T23:47:47.345Z" }, + { url = "https://files.pythonhosted.org/packages/a5/ec/40aed2330e7f02ecf74386ffcfef9ccb7108c6a430f15b6a252b663b1bed/wrapt-2.2.2-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d619e1eed9bd4f6ed9f24cd61971aa086fa86505289628d464bcf8a2c2e3f328", size = 160785, upload-time = "2026-06-20T23:47:48.759Z" }, + { url = "https://files.pythonhosted.org/packages/45/04/aa5309beed5344b00220ae6b3b24055852192656194c27947bee1736306a/wrapt-2.2.2-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:518b0c5e323511ec56a38894802ddd5e1222626484e68efe63f201854ad788e5", size = 153699, upload-time = "2026-06-20T23:47:50.177Z" }, + { url = "https://files.pythonhosted.org/packages/01/df/2def7e99d1fe87eea413f95f671924cdddcb08823b1ffd212748dfa6d062/wrapt-2.2.2-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:4bccea5cdecffa9dd70e343741f0e41e0a16619313d04b72f78bb525162ebcd0", size = 159695, upload-time = "2026-06-20T23:47:51.602Z" }, + { url = "https://files.pythonhosted.org/packages/c7/f6/a906d01a2ce12157bad2404957b3e2140da354b8a70b2fa48bbf282871c0/wrapt-2.2.2-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:209112cafd963710a05d199aae431d79a28bc76eb8e6d1bbbb8ad24340722cae", size = 152813, upload-time = "2026-06-20T23:47:53.03Z" }, + { url = "https://files.pythonhosted.org/packages/02/49/bc0086292d239575b4c08f4cf8a4079fa58abbad58ec23abf84833a283ed/wrapt-2.2.2-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:e5a5290e4bf2f332fc29ce72ffb9a2fff678aaac047e2e9f5f7165cd7792e099", size = 158809, upload-time = "2026-06-20T23:47:54.391Z" }, + { url = "https://files.pythonhosted.org/packages/55/83/8fbd034de1f3e907edaa18786d5dd8f6932874edee0826c7cecb5cab03a1/wrapt-2.2.2-cp311-cp311-win32.whl", hash = "sha256:5499236ad1dc116012e2a5dd943f3f31af12fce452128e2bbcbd55a7d3d4d14c", size = 77414, upload-time = "2026-06-20T23:47:55.882Z" }, + { url = "https://files.pythonhosted.org/packages/7e/9c/23695baa331c6de4e874c3d78b8e0bed92e1d2a274e665b29858f6841672/wrapt-2.2.2-cp311-cp311-win_amd64.whl", hash = "sha256:8636809939152be6ae20a6cef0fed9fe60f411b47847d0426a826884b469e971", size = 80368, upload-time = "2026-06-20T23:47:57.237Z" }, + { url = "https://files.pythonhosted.org/packages/08/49/40cefc342bf89b234a4490d741290fce781774b831aefb39c25471da96c9/wrapt-2.2.2-cp311-cp311-win_arm64.whl", hash = "sha256:5d0a142f7af07caeb5e5da87493162a7b8efa19ba919e550a746f7446e13fb30", size = 79489, upload-time = "2026-06-20T23:47:58.56Z" }, + { url = "https://files.pythonhosted.org/packages/2a/85/180b40628b23772692a0c76e8030114e1c0ae068470ed531919f0a5f2a4a/wrapt-2.2.2-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:8417fd3c674d3c8023d080292d29301531a12daf8bd938dd419710dd2f464f2b", size = 81484, upload-time = "2026-06-20T23:47:59.924Z" }, + { url = "https://files.pythonhosted.org/packages/94/f2/21c90f2a16689702e2aaff45795b11018dff2c9b1242bac10d225483f676/wrapt-2.2.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:0e7070c7472582e31af3dfc2622b2381a0df7435110a9388ed8db5ffbce67efb", size = 82151, upload-time = "2026-06-20T23:48:01.303Z" }, + { url = "https://files.pythonhosted.org/packages/5f/b3/7e6e9fcf4fe7e1b69a49fe6cc5a44e8224bab6283c5233c97e132f14908e/wrapt-2.2.2-cp312-cp312-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:2e096c9d39a59b35b63c9aacfbbbec2088ff51ff1fc31051acc60a07f42f273a", size = 169828, upload-time = "2026-06-20T23:48:02.719Z" }, + { url = "https://files.pythonhosted.org/packages/0b/43/894f132d857ed5a9904d937baf368badcbe5ea9e436e2f1930fe21c9f1f0/wrapt-2.2.2-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6d1a6050405bf334be33bf66296f113563622972a34900ae6fa60fd283a1a900", size = 171544, upload-time = "2026-06-20T23:48:04.266Z" }, + { url = "https://files.pythonhosted.org/packages/29/de/3c833e03725b477e9ea34028224dd21a48781830101e4e036f77e8b6b102/wrapt-2.2.2-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:10adb01371408c6de504a6658b9886480f1a4919a83752748a387a504a21df79", size = 160663, upload-time = "2026-06-20T23:48:05.708Z" }, + { url = "https://files.pythonhosted.org/packages/33/be/27edce350b24e3054d9d047f65f16d4c4d4c1f3f31c4278a1f8a95c723c8/wrapt-2.2.2-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:3442eee2a5798f9b451f1b2cd7518ce8b7e28a2a364696c414460a0e295c012a", size = 169387, upload-time = "2026-06-20T23:48:07.243Z" }, + { url = "https://files.pythonhosted.org/packages/e2/c4/9fd9679af8bf38e146652c7f47b6b352c3e5795b4ad1c0b7f94e15ac2aa7/wrapt-2.2.2-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:6c99012a22f735a85eed7c4b86a3e99c30fdd57d9e115b2b45f796264b58d0bf", size = 158849, upload-time = "2026-06-20T23:48:08.91Z" }, + { url = "https://files.pythonhosted.org/packages/bc/c2/aa6c0c2206803068c6859dabe01f8c84c43744da93d4c67b8946d21655ee/wrapt-2.2.2-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:3b686cfc008776a3952d6213cb296ed7f45d782a8453936406faa89eac0835ab", size = 168147, upload-time = "2026-06-20T23:48:10.374Z" }, + { url = "https://files.pythonhosted.org/packages/42/63/3eb25da41049d20ae18fcab2dd8b056e02387c4bfa626cbdfb7c3b872e4f/wrapt-2.2.2-cp312-cp312-win32.whl", hash = "sha256:ef2cce266b5b0b07e19fa82e59673b81142b7a3607c8ed1254113d048ed668da", size = 77734, upload-time = "2026-06-20T23:48:11.769Z" }, + { url = "https://files.pythonhosted.org/packages/da/09/0390e008a305360948fa9ce69507d041ac12cb2ee5d28e34467e2ee79391/wrapt-2.2.2-cp312-cp312-win_amd64.whl", hash = "sha256:abf8c20a2d72ee69e16328b3c91342c446e723bfe48bfcc4dded3b9722ac027f", size = 80585, upload-time = "2026-06-20T23:48:13.117Z" }, + { url = "https://files.pythonhosted.org/packages/d3/b3/84c445c66969f2d3457276b183a48c91097d59bbef9af6c075366b0f8c36/wrapt-2.2.2-cp312-cp312-win_arm64.whl", hash = "sha256:c6c64c5d02578bc4c4bca4f0aef1504de933c1d5b4ac2710b9131111459506c8", size = 79553, upload-time = "2026-06-20T23:48:14.5Z" }, + { url = "https://files.pythonhosted.org/packages/43/fc/f32f4b22c6511173c11d9e541ab4e7d8467a0f1b3455acaf784115d31ff8/wrapt-2.2.2-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:9e8b648270c613720a202d9a45ebabc33261b22c3a839b115ac5bce8c0bb0d69", size = 81296, upload-time = "2026-06-20T23:48:15.881Z" }, + { url = "https://files.pythonhosted.org/packages/72/06/4d117d5d77a9344776c0248b24dae3d3dd2f58e5f765fa08cf887072e719/wrapt-2.2.2-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:e6fb7e94e8fe3e4c3067bb1653a91cce7c5e83acc119fdd41501b1bf74654617", size = 81841, upload-time = "2026-06-20T23:48:17.262Z" }, + { url = "https://files.pythonhosted.org/packages/15/ff/63ad96f98eb58a742b1a20d80f21da88924405910149950b912368150468/wrapt-2.2.2-cp313-cp313-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:fb18fc51e813df0d9c98049e3bf2298a5495a648602040e21fa3c7329371159e", size = 167882, upload-time = "2026-06-20T23:48:18.764Z" }, + { url = "https://files.pythonhosted.org/packages/20/1f/8bb62d8933df7acf3247194e6e9fc68edf9d2fa203252c89c94b319dd472/wrapt-2.2.2-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:94b00b00f806eb3ef2abe9049ed45994a81ee9284884d96e6b8314927c6cea3d", size = 167411, upload-time = "2026-06-20T23:48:20.315Z" }, + { url = "https://files.pythonhosted.org/packages/17/09/8789dcb09ee1de715727db7521aabbb68ffa68dfade3a49468440cfced49/wrapt-2.2.2-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:62415fd095bc590b842b6d092f2b5d9ccbaeb7e0b28535c03dcea2718b48636b", size = 158607, upload-time = "2026-06-20T23:48:21.728Z" }, + { url = "https://files.pythonhosted.org/packages/9c/20/66e02562d53ee67d841f175e38e3c993c2d78a3e104c576cad61c028b43c/wrapt-2.2.2-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:a41e758d80dc0ab8c210f641ac892009d356cf1f955d97db544c8dd317b4d14c", size = 166367, upload-time = "2026-06-20T23:48:23.177Z" }, + { url = "https://files.pythonhosted.org/packages/bd/a3/832ac4e41222fb263b3042d42c2f08d305db7d0f0c9b1d3a271a9eede8f6/wrapt-2.2.2-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:b84cd4058001c9727b0e9980b7a9e66325b5ca748b1b578e822cade1bc6b304f", size = 157176, upload-time = "2026-06-20T23:48:24.711Z" }, + { url = "https://files.pythonhosted.org/packages/b7/01/1bd5e4d2df9c0178989ac8da9186543465388588ee2ef153e2591accebef/wrapt-2.2.2-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:26fc73a1b15e0946d2942b9a4426d162b51676338327dc067ccd8d2d76385f94", size = 167025, upload-time = "2026-06-20T23:48:26.118Z" }, + { url = "https://files.pythonhosted.org/packages/1c/69/583ed25291ab53e1ec117135fb1c33425e2f46d2bc8f29c17f7a94cf4274/wrapt-2.2.2-cp313-cp313-win32.whl", hash = "sha256:3c4095803491f6ef72128914c28ec05bbad9758433bb35f6715a3e9c8e46fb2d", size = 77605, upload-time = "2026-06-20T23:48:27.643Z" }, + { url = "https://files.pythonhosted.org/packages/29/68/e69fc6d06e1523c68e0d00f95c9aed1158ce9908ee41603f7f2eae3d5db6/wrapt-2.2.2-cp313-cp313-win_amd64.whl", hash = "sha256:2cb07f414fab25dbe6b5c7398e1491423a5c81a6209533639969a6c928d474a4", size = 80508, upload-time = "2026-06-20T23:48:29.013Z" }, + { url = "https://files.pythonhosted.org/packages/55/21/fe7a393d9e5dc0923bed8f5d857e9dcff210f1fa0888c02cc8f3ffaa55aa/wrapt-2.2.2-cp313-cp313-win_arm64.whl", hash = "sha256:1fc7691f070220215cccb2a20836b9adbaecb8ff22ad47abe63de5f110994fac", size = 79565, upload-time = "2026-06-20T23:48:30.429Z" }, + { url = "https://files.pythonhosted.org/packages/b6/e5/c120d13bf5091164f68c3c1657e84f16f57e71d978421b626393ac5bd7eb/wrapt-2.2.2-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:ec8f83949028366531383603139403cac7a826e4011955813cdd640017845ce5", size = 83264, upload-time = "2026-06-20T23:48:31.807Z" }, + { url = "https://files.pythonhosted.org/packages/d3/b0/d4a1eb97e0e286625bdf21bc7f702637f9607787ffbbdb5ec14d50c79dbf/wrapt-2.2.2-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:4b481fb0c40d9fd90a5809911208da700987d373a20a4709dc9e3944af7a6bec", size = 83791, upload-time = "2026-06-20T23:48:33.482Z" }, + { url = "https://files.pythonhosted.org/packages/18/1e/f060df47755e87b57684cee7bfc1362b204df55fac96ffebc0631b697b79/wrapt-2.2.2-cp313-cp313t-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:0065a3b657cec06813b4241d2462ccec287f6863103d7445b725fb3a889736f9", size = 203399, upload-time = "2026-06-20T23:48:34.97Z" }, + { url = "https://files.pythonhosted.org/packages/c4/de/2316a757a1abb6453700b79d83e532146dcef2611348282d4d8889792161/wrapt-2.2.2-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:30f7424af5c5c345b7f26490e097f74a2ef45b3d08b664dc33571aee3bd3b56c", size = 210461, upload-time = "2026-06-20T23:48:36.569Z" }, + { url = "https://files.pythonhosted.org/packages/ed/29/d1160785ae18ca2495a6d82a21154103d74f656c9fd457fb35f6b11b965a/wrapt-2.2.2-cp313-cp313t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:07fdcb012821859168641acf68afad61ef9783cf37100af85f152550e9677194", size = 195313, upload-time = "2026-06-20T23:48:38.175Z" }, + { url = "https://files.pythonhosted.org/packages/f5/2d/7caa9598ae61a9cf0989cc501739cbeeb7d650ab3193cca1407b9af0c6ab/wrapt-2.2.2-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:f90038ab58fafb584801ca62d72384d7d5225d93c76f7b773c22fae545bd8066", size = 206116, upload-time = "2026-06-20T23:48:39.804Z" }, + { url = "https://files.pythonhosted.org/packages/ac/02/281ea1088b8650d865f311b35cf86fd21df89128e2909714f1161e01c9d0/wrapt-2.2.2-cp313-cp313t-musllinux_1_2_riscv64.whl", hash = "sha256:c5d7825491bfa2d08b97e9557768987952c7b9ae687d06c3320b40a37ccb7f20", size = 192668, upload-time = "2026-06-20T23:48:41.346Z" }, + { url = "https://files.pythonhosted.org/packages/be/7d/976e2d5b4b5c5babda40974edd54d0a5585cb60132ed86b46f4b80239b16/wrapt-2.2.2-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:0ad520e6daa9bbf136f14de735474dbec7dcc0891f718e1d274ce8dc92e645af", size = 198891, upload-time = "2026-06-20T23:48:43.056Z" }, + { url = "https://files.pythonhosted.org/packages/59/b7/e47651797c097f75a37e2ce86dcf04048ff576f3a674f7c558df7b5e9622/wrapt-2.2.2-cp313-cp313t-win32.whl", hash = "sha256:25904acb9475f46c24fe0423dbc8fda8cc5fbc282ab3dc6e72e919748c53f4e9", size = 78537, upload-time = "2026-06-20T23:48:44.509Z" }, + { url = "https://files.pythonhosted.org/packages/d1/6f/9fa5d59fb06d890defb5a8f727ce6a14d2932c8760153f96956628559fee/wrapt-2.2.2-cp313-cp313t-win_amd64.whl", hash = "sha256:305d4c247d61c4115794a169141823c62f719525ddb90b23aa332741c77d2c28", size = 82005, upload-time = "2026-06-20T23:48:46.391Z" }, + { url = "https://files.pythonhosted.org/packages/15/80/4c7bd9873d1f9f7d138d93556b500469dbe24f42710b877519c2b9eb380d/wrapt-2.2.2-cp313-cp313t-win_arm64.whl", hash = "sha256:c20279cd1a29800815d7b2d6338b60a6c6e78263f9d6e62e0eda251ba9cae2d0", size = 80762, upload-time = "2026-06-20T23:48:47.964Z" }, + { url = "https://files.pythonhosted.org/packages/24/05/7fd9c3f83b2c74cbfc572a0b88aa37431e04bd8aed70d2c0efd3464206de/wrapt-2.2.2-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:0e64826f920c42d9d9f87e8cc09ffae66c51ede12d59061a5a426deb9aa71745", size = 81341, upload-time = "2026-06-20T23:48:49.39Z" }, + { url = "https://files.pythonhosted.org/packages/4b/68/1bfa43100dd90d4ef74a05897b86275cf57e1313ca14aae2545bc9f872c9/wrapt-2.2.2-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:dcaa5e1451bd8751d7bd1568dfa3321c78092a52a7ecb5d1a0f18a5791e1fd00", size = 81921, upload-time = "2026-06-20T23:48:50.986Z" }, + { url = "https://files.pythonhosted.org/packages/74/eb/df7b7f0b631dbbc750f39be27d8b55f65777d8ac86da80e12be41a644c4b/wrapt-2.2.2-cp314-cp314-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:0abfd648dac9ac9c5b3aa9b523d27f1789046640b58dcd5652a720ddb325e1fc", size = 167713, upload-time = "2026-06-20T23:48:52.598Z" }, + { url = "https://files.pythonhosted.org/packages/4d/9a/d1bd36f6d088c8e652a9383cabbd49af30b8c576302a7eccddbab6963e3f/wrapt-2.2.2-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f4bfd8d1eb438153eff8b8cfe87f032ba65731e1ce06138b5090f745a33f6f95", size = 166779, upload-time = "2026-06-20T23:48:54.33Z" }, + { url = "https://files.pythonhosted.org/packages/4c/ae/24ffacd4187fac2740a1972093929e836dea092d42c87d728cd98fee11a6/wrapt-2.2.2-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:c427c9d06d859848a69f0d928fe28b5c33a941b2265d10a0e1f15cd244f1ee33", size = 158407, upload-time = "2026-06-20T23:48:55.944Z" }, + { url = "https://files.pythonhosted.org/packages/a3/ed/974427668249a356051e8d67d47fa54ef6c777f0fcf3bae9d292c047d4b6/wrapt-2.2.2-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:4250b43d1a129d947e083c4dc6baf333c9bb34edd26f912d5b0457841fc858ab", size = 166594, upload-time = "2026-06-20T23:48:57.617Z" }, + { url = "https://files.pythonhosted.org/packages/fb/5f/e1d7c6e4523f78db2fbd7826babd0348da1d5e0834c4f918b9ab5757dfae/wrapt-2.2.2-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:173e5bb5ca350a6e0abab60b7ec7cdd7992a814cb14b4de670a28f067f105663", size = 157068, upload-time = "2026-06-20T23:48:59.171Z" }, + { url = "https://files.pythonhosted.org/packages/1e/c1/7ebd1027f00700c0b0233b20aceef2b4784294ed64971424c4a78e069e34/wrapt-2.2.2-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:aa14b01804bce36c6d63d7b6a4f55df390f29f8648cc13a1f40b166f4d54680d", size = 166470, upload-time = "2026-06-20T23:49:00.737Z" }, + { url = "https://files.pythonhosted.org/packages/99/eb/974e471a6a978b8180186b8a9dc5ae3361ce269a967190b709b8ce17abfb/wrapt-2.2.2-cp314-cp314-win32.whl", hash = "sha256:58f9f8d637c9a6e245c6ef5b109b67ec187d2faed23d1405656b51d96e0a5b56", size = 78062, upload-time = "2026-06-20T23:49:02.327Z" }, + { url = "https://files.pythonhosted.org/packages/49/ec/e1281156cdc7a66693838ad7a0865ad641c74abd337a957d668b575aaffb/wrapt-2.2.2-cp314-cp314-win_amd64.whl", hash = "sha256:385cb1866f20479e83299af585375bfa0a4b0c6c9907a981483ea782ea8ae406", size = 80832, upload-time = "2026-06-20T23:49:03.837Z" }, + { url = "https://files.pythonhosted.org/packages/45/7d/1b6b5ddd94005a2dac97a4490c9838f3154977850d633abcb65b30089437/wrapt-2.2.2-cp314-cp314-win_arm64.whl", hash = "sha256:8ffbeaea6771a6eba6e6eeb09767864995726bc8240bb54baf88a9bb1db34d5c", size = 80029, upload-time = "2026-06-20T23:49:05.237Z" }, + { url = "https://files.pythonhosted.org/packages/b0/33/9ebcf8aafe91c601127cbd93708c16aa8f688f34a10bf004046803ecdc4f/wrapt-2.2.2-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:09f811d43f6f33ec7515f0be76b159569f4057ab54d3e079c3204dddb90afa2a", size = 83357, upload-time = "2026-06-20T23:49:06.632Z" }, + { url = "https://files.pythonhosted.org/packages/39/38/ec45b635153327b52e52732a0ea980e5f00b7efba65f9e018828f1e69daa/wrapt-2.2.2-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:a795d3c06e5fbf9ea2f13196180b77aeab1b4685917256ee0d014cc163d90063", size = 83794, upload-time = "2026-06-20T23:49:08.098Z" }, + { url = "https://files.pythonhosted.org/packages/4e/ea/1a89e6d3b7a83c3affe5c09cde77792c947e63e4bc85ad84cd5bb9abb0d8/wrapt-2.2.2-cp314-cp314t-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:45c2f2768e790c9f8db90f239ef23a2af8e7570f25a35619ef902df4a738447f", size = 203362, upload-time = "2026-06-20T23:49:09.811Z" }, + { url = "https://files.pythonhosted.org/packages/19/d8/3b58763d9863b5a73771c0d97110f9595d248db454009e07e1535ee905a4/wrapt-2.2.2-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:bbf00ee0cb55ec24e2b0995a71942b85b21a066db8f3f46e1dbfdb9433ffba81", size = 210449, upload-time = "2026-06-20T23:49:11.521Z" }, + { url = "https://files.pythonhosted.org/packages/2d/6f/17fd9e053103d8be148d20d5d7505facc72d5fe1f9127973904ceaed79cf/wrapt-2.2.2-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:2252f77663651b89255895f58cc6ac08fcb206d4371813e5af61bb62d4f7689c", size = 195349, upload-time = "2026-06-20T23:49:13.346Z" }, + { url = "https://files.pythonhosted.org/packages/ef/04/d0d1ccaaa12cb7dccf28a23f0279a608ba498f71e81d949d5ed54bcfd5c1/wrapt-2.2.2-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:2cd7181ab1c31192ff5219269830744b5a62020b3a6d433588c4f1c95b8f8bff", size = 206099, upload-time = "2026-06-20T23:49:15.051Z" }, + { url = "https://files.pythonhosted.org/packages/44/b3/e8aa07b619890a2aa6cde1931b1887abb08820721b564a5f80b7ca3f3aa0/wrapt-2.2.2-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:6fe35fd51b74867d8b80174c277bd6bbf6a73e443f908129dc531c4b688a20d5", size = 192728, upload-time = "2026-06-20T23:49:16.854Z" }, + { url = "https://files.pythonhosted.org/packages/b7/f0/1819fb50f0d3c9bd758d8a83b56f1b470dee8b5b8eac8702b7c137cea9d4/wrapt-2.2.2-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:11d95fc2fbad3163596c39d440e6f21ca9fccece74b56e30a37ac2fca786a07c", size = 198842, upload-time = "2026-06-20T23:49:18.504Z" }, + { url = "https://files.pythonhosted.org/packages/67/7c/e88313f16a99930b899ef970d91c281544a470749a359decad994483bbda/wrapt-2.2.2-cp314-cp314t-win32.whl", hash = "sha256:d8a15813215f33fa83667bfc978b300e35669ea8bb424e970a1426bcb7bc6cca", size = 79059, upload-time = "2026-06-20T23:49:20.107Z" }, + { url = "https://files.pythonhosted.org/packages/a0/4f/ac12fda57a55068a094ec42851fb0a40e8489d8941863d517452de62e507/wrapt-2.2.2-cp314-cp314t-win_amd64.whl", hash = "sha256:d09db0f7e8357060d3c38fc22a018aba683a796bf184360fd1a58f6fc180dc77", size = 82462, upload-time = "2026-06-20T23:49:21.631Z" }, + { url = "https://files.pythonhosted.org/packages/48/a7/df732dac86d9b2027c56bd163dbc883e037b16c3469614752e148d219c61/wrapt-2.2.2-cp314-cp314t-win_arm64.whl", hash = "sha256:f32fe639c39561ccc187bcae17e9271be0eb45f1c2952510d2f29b33ab577347", size = 81182, upload-time = "2026-06-20T23:49:23.199Z" }, + { url = "https://files.pythonhosted.org/packages/6e/d2/6317eb6d4554855bbf12d61857774af34747bf88a42c19bf306de67e2fa3/wrapt-2.2.2-py3-none-any.whl", hash = "sha256:5bad217350f19ce99ca5b5e71d406765ea86fe541628426772b657375ee1c048", size = 61460, upload-time = "2026-06-20T23:49:42.966Z" }, ] [[package]] name = "xxhash" -version = "3.6.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/02/84/30869e01909fb37a6cc7e18688ee8bf1e42d57e7e0777636bd47524c43c7/xxhash-3.6.0.tar.gz", hash = "sha256:f0162a78b13a0d7617b2845b90c763339d1f1d82bb04a4b07f4ab535cc5e05d6", size = 85160, upload-time = "2025-10-02T14:37:08.097Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/34/ee/f9f1d656ad168681bb0f6b092372c1e533c4416b8069b1896a175c46e484/xxhash-3.6.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:87ff03d7e35c61435976554477a7f4cd1704c3596a89a8300d5ce7fc83874a71", size = 32845, upload-time = "2025-10-02T14:33:51.573Z" }, - { url = "https://files.pythonhosted.org/packages/a3/b1/93508d9460b292c74a09b83d16750c52a0ead89c51eea9951cb97a60d959/xxhash-3.6.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:f572dfd3d0e2eb1a57511831cf6341242f5a9f8298a45862d085f5b93394a27d", size = 30807, upload-time = "2025-10-02T14:33:52.964Z" }, - { url = "https://files.pythonhosted.org/packages/07/55/28c93a3662f2d200c70704efe74aab9640e824f8ce330d8d3943bf7c9b3c/xxhash-3.6.0-cp310-cp310-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:89952ea539566b9fed2bbd94e589672794b4286f342254fad28b149f9615fef8", size = 193786, upload-time = "2025-10-02T14:33:54.272Z" }, - { url = "https://files.pythonhosted.org/packages/c1/96/fec0be9bb4b8f5d9c57d76380a366f31a1781fb802f76fc7cda6c84893c7/xxhash-3.6.0-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:48e6f2ffb07a50b52465a1032c3cf1f4a5683f944acaca8a134a2f23674c2058", size = 212830, upload-time = "2025-10-02T14:33:55.706Z" }, - { url = "https://files.pythonhosted.org/packages/c4/a0/c706845ba77b9611f81fd2e93fad9859346b026e8445e76f8c6fd057cc6d/xxhash-3.6.0-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:b5b848ad6c16d308c3ac7ad4ba6bede80ed5df2ba8ed382f8932df63158dd4b2", size = 211606, upload-time = "2025-10-02T14:33:57.133Z" }, - { url = "https://files.pythonhosted.org/packages/67/1e/164126a2999e5045f04a69257eea946c0dc3e86541b400d4385d646b53d7/xxhash-3.6.0-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a034590a727b44dd8ac5914236a7b8504144447a9682586c3327e935f33ec8cc", size = 444872, upload-time = "2025-10-02T14:33:58.446Z" }, - { url = "https://files.pythonhosted.org/packages/2d/4b/55ab404c56cd70a2cf5ecfe484838865d0fea5627365c6c8ca156bd09c8f/xxhash-3.6.0-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8a8f1972e75ebdd161d7896743122834fe87378160c20e97f8b09166213bf8cc", size = 193217, upload-time = "2025-10-02T14:33:59.724Z" }, - { url = "https://files.pythonhosted.org/packages/45/e6/52abf06bac316db33aa269091ae7311bd53cfc6f4b120ae77bac1b348091/xxhash-3.6.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:ee34327b187f002a596d7b167ebc59a1b729e963ce645964bbc050d2f1b73d07", size = 210139, upload-time = "2025-10-02T14:34:02.041Z" }, - { url = "https://files.pythonhosted.org/packages/34/37/db94d490b8691236d356bc249c08819cbcef9273a1a30acf1254ff9ce157/xxhash-3.6.0-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:339f518c3c7a850dd033ab416ea25a692759dc7478a71131fe8869010d2b75e4", size = 197669, upload-time = "2025-10-02T14:34:03.664Z" }, - { url = "https://files.pythonhosted.org/packages/b7/36/c4f219ef4a17a4f7a64ed3569bc2b5a9c8311abdb22249ac96093625b1a4/xxhash-3.6.0-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:bf48889c9630542d4709192578aebbd836177c9f7a4a2778a7d6340107c65f06", size = 210018, upload-time = "2025-10-02T14:34:05.325Z" }, - { url = "https://files.pythonhosted.org/packages/fd/06/bfac889a374fc2fc439a69223d1750eed2e18a7db8514737ab630534fa08/xxhash-3.6.0-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:5576b002a56207f640636056b4160a378fe36a58db73ae5c27a7ec8db35f71d4", size = 413058, upload-time = "2025-10-02T14:34:06.925Z" }, - { url = "https://files.pythonhosted.org/packages/c9/d1/555d8447e0dd32ad0930a249a522bb2e289f0d08b6b16204cfa42c1f5a0c/xxhash-3.6.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:af1f3278bd02814d6dedc5dec397993b549d6f16c19379721e5a1d31e132c49b", size = 190628, upload-time = "2025-10-02T14:34:08.669Z" }, - { url = "https://files.pythonhosted.org/packages/d1/15/8751330b5186cedc4ed4b597989882ea05e0408b53fa47bcb46a6125bfc6/xxhash-3.6.0-cp310-cp310-win32.whl", hash = "sha256:aed058764db109dc9052720da65fafe84873b05eb8b07e5e653597951af57c3b", size = 30577, upload-time = "2025-10-02T14:34:10.234Z" }, - { url = "https://files.pythonhosted.org/packages/bb/cc/53f87e8b5871a6eb2ff7e89c48c66093bda2be52315a8161ddc54ea550c4/xxhash-3.6.0-cp310-cp310-win_amd64.whl", hash = "sha256:e82da5670f2d0d98950317f82a0e4a0197150ff19a6df2ba40399c2a3b9ae5fb", size = 31487, upload-time = "2025-10-02T14:34:11.618Z" }, - { url = "https://files.pythonhosted.org/packages/9f/00/60f9ea3bb697667a14314d7269956f58bf56bb73864f8f8d52a3c2535e9a/xxhash-3.6.0-cp310-cp310-win_arm64.whl", hash = "sha256:4a082ffff8c6ac07707fb6b671caf7c6e020c75226c561830b73d862060f281d", size = 27863, upload-time = "2025-10-02T14:34:12.619Z" }, - { url = "https://files.pythonhosted.org/packages/17/d4/cc2f0400e9154df4b9964249da78ebd72f318e35ccc425e9f403c392f22a/xxhash-3.6.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:b47bbd8cf2d72797f3c2772eaaac0ded3d3af26481a26d7d7d41dc2d3c46b04a", size = 32844, upload-time = "2025-10-02T14:34:14.037Z" }, - { url = "https://files.pythonhosted.org/packages/5e/ec/1cc11cd13e26ea8bc3cb4af4eaadd8d46d5014aebb67be3f71fb0b68802a/xxhash-3.6.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:2b6821e94346f96db75abaa6e255706fb06ebd530899ed76d32cd99f20dc52fa", size = 30809, upload-time = "2025-10-02T14:34:15.484Z" }, - { url = "https://files.pythonhosted.org/packages/04/5f/19fe357ea348d98ca22f456f75a30ac0916b51c753e1f8b2e0e6fb884cce/xxhash-3.6.0-cp311-cp311-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:d0a9751f71a1a65ce3584e9cae4467651c7e70c9d31017fa57574583a4540248", size = 194665, upload-time = "2025-10-02T14:34:16.541Z" }, - { url = "https://files.pythonhosted.org/packages/90/3b/d1f1a8f5442a5fd8beedae110c5af7604dc37349a8e16519c13c19a9a2de/xxhash-3.6.0-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:8b29ee68625ab37b04c0b40c3fafdf24d2f75ccd778333cfb698f65f6c463f62", size = 213550, upload-time = "2025-10-02T14:34:17.878Z" }, - { url = "https://files.pythonhosted.org/packages/c4/ef/3a9b05eb527457d5db13a135a2ae1a26c80fecd624d20f3e8dcc4cb170f3/xxhash-3.6.0-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:6812c25fe0d6c36a46ccb002f40f27ac903bf18af9f6dd8f9669cb4d176ab18f", size = 212384, upload-time = "2025-10-02T14:34:19.182Z" }, - { url = "https://files.pythonhosted.org/packages/0f/18/ccc194ee698c6c623acbf0f8c2969811a8a4b6185af5e824cd27b9e4fd3e/xxhash-3.6.0-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:4ccbff013972390b51a18ef1255ef5ac125c92dc9143b2d1909f59abc765540e", size = 445749, upload-time = "2025-10-02T14:34:20.659Z" }, - { url = "https://files.pythonhosted.org/packages/a5/86/cf2c0321dc3940a7aa73076f4fd677a0fb3e405cb297ead7d864fd90847e/xxhash-3.6.0-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:297b7fbf86c82c550e12e8fb71968b3f033d27b874276ba3624ea868c11165a8", size = 193880, upload-time = "2025-10-02T14:34:22.431Z" }, - { url = "https://files.pythonhosted.org/packages/82/fb/96213c8560e6f948a1ecc9a7613f8032b19ee45f747f4fca4eb31bb6d6ed/xxhash-3.6.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:dea26ae1eb293db089798d3973a5fc928a18fdd97cc8801226fae705b02b14b0", size = 210912, upload-time = "2025-10-02T14:34:23.937Z" }, - { url = "https://files.pythonhosted.org/packages/40/aa/4395e669b0606a096d6788f40dbdf2b819d6773aa290c19e6e83cbfc312f/xxhash-3.6.0-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:7a0b169aafb98f4284f73635a8e93f0735f9cbde17bd5ec332480484241aaa77", size = 198654, upload-time = "2025-10-02T14:34:25.644Z" }, - { url = "https://files.pythonhosted.org/packages/67/74/b044fcd6b3d89e9b1b665924d85d3f400636c23590226feb1eb09e1176ce/xxhash-3.6.0-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:08d45aef063a4531b785cd72de4887766d01dc8f362a515693df349fdb825e0c", size = 210867, upload-time = "2025-10-02T14:34:27.203Z" }, - { url = "https://files.pythonhosted.org/packages/bc/fd/3ce73bf753b08cb19daee1eb14aa0d7fe331f8da9c02dd95316ddfe5275e/xxhash-3.6.0-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:929142361a48ee07f09121fe9e96a84950e8d4df3bb298ca5d88061969f34d7b", size = 414012, upload-time = "2025-10-02T14:34:28.409Z" }, - { url = "https://files.pythonhosted.org/packages/ba/b3/5a4241309217c5c876f156b10778f3ab3af7ba7e3259e6d5f5c7d0129eb2/xxhash-3.6.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:51312c768403d8540487dbbfb557454cfc55589bbde6424456951f7fcd4facb3", size = 191409, upload-time = "2025-10-02T14:34:29.696Z" }, - { url = "https://files.pythonhosted.org/packages/c0/01/99bfbc15fb9abb9a72b088c1d95219fc4782b7d01fc835bd5744d66dd0b8/xxhash-3.6.0-cp311-cp311-win32.whl", hash = "sha256:d1927a69feddc24c987b337ce81ac15c4720955b667fe9b588e02254b80446fd", size = 30574, upload-time = "2025-10-02T14:34:31.028Z" }, - { url = "https://files.pythonhosted.org/packages/65/79/9d24d7f53819fe301b231044ea362ce64e86c74f6e8c8e51320de248b3e5/xxhash-3.6.0-cp311-cp311-win_amd64.whl", hash = "sha256:26734cdc2d4ffe449b41d186bbeac416f704a482ed835d375a5c0cb02bc63fef", size = 31481, upload-time = "2025-10-02T14:34:32.062Z" }, - { url = "https://files.pythonhosted.org/packages/30/4e/15cd0e3e8772071344eab2961ce83f6e485111fed8beb491a3f1ce100270/xxhash-3.6.0-cp311-cp311-win_arm64.whl", hash = "sha256:d72f67ef8bf36e05f5b6c65e8524f265bd61071471cd4cf1d36743ebeeeb06b7", size = 27861, upload-time = "2025-10-02T14:34:33.555Z" }, - { url = "https://files.pythonhosted.org/packages/9a/07/d9412f3d7d462347e4511181dea65e47e0d0e16e26fbee2ea86a2aefb657/xxhash-3.6.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:01362c4331775398e7bb34e3ab403bc9ee9f7c497bc7dee6272114055277dd3c", size = 32744, upload-time = "2025-10-02T14:34:34.622Z" }, - { url = "https://files.pythonhosted.org/packages/79/35/0429ee11d035fc33abe32dca1b2b69e8c18d236547b9a9b72c1929189b9a/xxhash-3.6.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:b7b2df81a23f8cb99656378e72501b2cb41b1827c0f5a86f87d6b06b69f9f204", size = 30816, upload-time = "2025-10-02T14:34:36.043Z" }, - { url = "https://files.pythonhosted.org/packages/b7/f2/57eb99aa0f7d98624c0932c5b9a170e1806406cdbcdb510546634a1359e0/xxhash-3.6.0-cp312-cp312-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:dc94790144e66b14f67b10ac8ed75b39ca47536bf8800eb7c24b50271ea0c490", size = 194035, upload-time = "2025-10-02T14:34:37.354Z" }, - { url = "https://files.pythonhosted.org/packages/4c/ed/6224ba353690d73af7a3f1c7cdb1fc1b002e38f783cb991ae338e1eb3d79/xxhash-3.6.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:93f107c673bccf0d592cdba077dedaf52fe7f42dcd7676eba1f6d6f0c3efffd2", size = 212914, upload-time = "2025-10-02T14:34:38.6Z" }, - { url = "https://files.pythonhosted.org/packages/38/86/fb6b6130d8dd6b8942cc17ab4d90e223653a89aa32ad2776f8af7064ed13/xxhash-3.6.0-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:2aa5ee3444c25b69813663c9f8067dcfaa2e126dc55e8dddf40f4d1c25d7effa", size = 212163, upload-time = "2025-10-02T14:34:39.872Z" }, - { url = "https://files.pythonhosted.org/packages/ee/dc/e84875682b0593e884ad73b2d40767b5790d417bde603cceb6878901d647/xxhash-3.6.0-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:f7f99123f0e1194fa59cc69ad46dbae2e07becec5df50a0509a808f90a0f03f0", size = 445411, upload-time = "2025-10-02T14:34:41.569Z" }, - { url = "https://files.pythonhosted.org/packages/11/4f/426f91b96701ec2f37bb2b8cec664eff4f658a11f3fa9d94f0a887ea6d2b/xxhash-3.6.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:49e03e6fe2cac4a1bc64952dd250cf0dbc5ef4ebb7b8d96bce82e2de163c82a2", size = 193883, upload-time = "2025-10-02T14:34:43.249Z" }, - { url = "https://files.pythonhosted.org/packages/53/5a/ddbb83eee8e28b778eacfc5a85c969673e4023cdeedcfcef61f36731610b/xxhash-3.6.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:bd17fede52a17a4f9a7bc4472a5867cb0b160deeb431795c0e4abe158bc784e9", size = 210392, upload-time = "2025-10-02T14:34:45.042Z" }, - { url = "https://files.pythonhosted.org/packages/1e/c2/ff69efd07c8c074ccdf0a4f36fcdd3d27363665bcdf4ba399abebe643465/xxhash-3.6.0-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:6fb5f5476bef678f69db04f2bd1efbed3030d2aba305b0fc1773645f187d6a4e", size = 197898, upload-time = "2025-10-02T14:34:46.302Z" }, - { url = "https://files.pythonhosted.org/packages/58/ca/faa05ac19b3b622c7c9317ac3e23954187516298a091eb02c976d0d3dd45/xxhash-3.6.0-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:843b52f6d88071f87eba1631b684fcb4b2068cd2180a0224122fe4ef011a9374", size = 210655, upload-time = "2025-10-02T14:34:47.571Z" }, - { url = "https://files.pythonhosted.org/packages/d4/7a/06aa7482345480cc0cb597f5c875b11a82c3953f534394f620b0be2f700c/xxhash-3.6.0-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:7d14a6cfaf03b1b6f5f9790f76880601ccc7896aff7ab9cd8978a939c1eb7e0d", size = 414001, upload-time = "2025-10-02T14:34:49.273Z" }, - { url = "https://files.pythonhosted.org/packages/23/07/63ffb386cd47029aa2916b3d2f454e6cc5b9f5c5ada3790377d5430084e7/xxhash-3.6.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:418daf3db71e1413cfe211c2f9a528456936645c17f46b5204705581a45390ae", size = 191431, upload-time = "2025-10-02T14:34:50.798Z" }, - { url = "https://files.pythonhosted.org/packages/0f/93/14fde614cadb4ddf5e7cebf8918b7e8fac5ae7861c1875964f17e678205c/xxhash-3.6.0-cp312-cp312-win32.whl", hash = "sha256:50fc255f39428a27299c20e280d6193d8b63b8ef8028995323bf834a026b4fbb", size = 30617, upload-time = "2025-10-02T14:34:51.954Z" }, - { url = "https://files.pythonhosted.org/packages/13/5d/0d125536cbe7565a83d06e43783389ecae0c0f2ed037b48ede185de477c0/xxhash-3.6.0-cp312-cp312-win_amd64.whl", hash = "sha256:c0f2ab8c715630565ab8991b536ecded9416d615538be8ecddce43ccf26cbc7c", size = 31534, upload-time = "2025-10-02T14:34:53.276Z" }, - { url = "https://files.pythonhosted.org/packages/54/85/6ec269b0952ec7e36ba019125982cf11d91256a778c7c3f98a4c5043d283/xxhash-3.6.0-cp312-cp312-win_arm64.whl", hash = "sha256:eae5c13f3bc455a3bbb68bdc513912dc7356de7e2280363ea235f71f54064829", size = 27876, upload-time = "2025-10-02T14:34:54.371Z" }, - { url = "https://files.pythonhosted.org/packages/33/76/35d05267ac82f53ae9b0e554da7c5e281ee61f3cad44c743f0fcd354f211/xxhash-3.6.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:599e64ba7f67472481ceb6ee80fa3bd828fd61ba59fb11475572cc5ee52b89ec", size = 32738, upload-time = "2025-10-02T14:34:55.839Z" }, - { url = "https://files.pythonhosted.org/packages/31/a8/3fbce1cd96534a95e35d5120637bf29b0d7f5d8fa2f6374e31b4156dd419/xxhash-3.6.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:7d8b8aaa30fca4f16f0c84a5c8d7ddee0e25250ec2796c973775373257dde8f1", size = 30821, upload-time = "2025-10-02T14:34:57.219Z" }, - { url = "https://files.pythonhosted.org/packages/0c/ea/d387530ca7ecfa183cb358027f1833297c6ac6098223fd14f9782cd0015c/xxhash-3.6.0-cp313-cp313-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:d597acf8506d6e7101a4a44a5e428977a51c0fadbbfd3c39650cca9253f6e5a6", size = 194127, upload-time = "2025-10-02T14:34:59.21Z" }, - { url = "https://files.pythonhosted.org/packages/ba/0c/71435dcb99874b09a43b8d7c54071e600a7481e42b3e3ce1eb5226a5711a/xxhash-3.6.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:858dc935963a33bc33490128edc1c12b0c14d9c7ebaa4e387a7869ecc4f3e263", size = 212975, upload-time = "2025-10-02T14:35:00.816Z" }, - { url = "https://files.pythonhosted.org/packages/84/7a/c2b3d071e4bb4a90b7057228a99b10d51744878f4a8a6dd643c8bd897620/xxhash-3.6.0-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:ba284920194615cb8edf73bf52236ce2e1664ccd4a38fdb543506413529cc546", size = 212241, upload-time = "2025-10-02T14:35:02.207Z" }, - { url = "https://files.pythonhosted.org/packages/81/5f/640b6eac0128e215f177df99eadcd0f1b7c42c274ab6a394a05059694c5a/xxhash-3.6.0-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:4b54219177f6c6674d5378bd862c6aedf64725f70dd29c472eaae154df1a2e89", size = 445471, upload-time = "2025-10-02T14:35:03.61Z" }, - { url = "https://files.pythonhosted.org/packages/5e/1e/3c3d3ef071b051cc3abbe3721ffb8365033a172613c04af2da89d5548a87/xxhash-3.6.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:42c36dd7dbad2f5238950c377fcbf6811b1cdb1c444fab447960030cea60504d", size = 193936, upload-time = "2025-10-02T14:35:05.013Z" }, - { url = "https://files.pythonhosted.org/packages/2c/bd/4a5f68381939219abfe1c22a9e3a5854a4f6f6f3c4983a87d255f21f2e5d/xxhash-3.6.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:f22927652cba98c44639ffdc7aaf35828dccf679b10b31c4ad72a5b530a18eb7", size = 210440, upload-time = "2025-10-02T14:35:06.239Z" }, - { url = "https://files.pythonhosted.org/packages/eb/37/b80fe3d5cfb9faff01a02121a0f4d565eb7237e9e5fc66e73017e74dcd36/xxhash-3.6.0-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:b45fad44d9c5c119e9c6fbf2e1c656a46dc68e280275007bbfd3d572b21426db", size = 197990, upload-time = "2025-10-02T14:35:07.735Z" }, - { url = "https://files.pythonhosted.org/packages/d7/fd/2c0a00c97b9e18f72e1f240ad4e8f8a90fd9d408289ba9c7c495ed7dc05c/xxhash-3.6.0-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:6f2580ffab1a8b68ef2b901cde7e55fa8da5e4be0977c68f78fc80f3c143de42", size = 210689, upload-time = "2025-10-02T14:35:09.438Z" }, - { url = "https://files.pythonhosted.org/packages/93/86/5dd8076a926b9a95db3206aba20d89a7fc14dd5aac16e5c4de4b56033140/xxhash-3.6.0-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:40c391dd3cd041ebc3ffe6f2c862f402e306eb571422e0aa918d8070ba31da11", size = 414068, upload-time = "2025-10-02T14:35:11.162Z" }, - { url = "https://files.pythonhosted.org/packages/af/3c/0bb129170ee8f3650f08e993baee550a09593462a5cddd8e44d0011102b1/xxhash-3.6.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:f205badabde7aafd1a31e8ca2a3e5a763107a71c397c4481d6a804eb5063d8bd", size = 191495, upload-time = "2025-10-02T14:35:12.971Z" }, - { url = "https://files.pythonhosted.org/packages/e9/3a/6797e0114c21d1725e2577508e24006fd7ff1d8c0c502d3b52e45c1771d8/xxhash-3.6.0-cp313-cp313-win32.whl", hash = "sha256:2577b276e060b73b73a53042ea5bd5203d3e6347ce0d09f98500f418a9fcf799", size = 30620, upload-time = "2025-10-02T14:35:14.129Z" }, - { url = "https://files.pythonhosted.org/packages/86/15/9bc32671e9a38b413a76d24722a2bf8784a132c043063a8f5152d390b0f9/xxhash-3.6.0-cp313-cp313-win_amd64.whl", hash = "sha256:757320d45d2fbcce8f30c42a6b2f47862967aea7bf458b9625b4bbe7ee390392", size = 31542, upload-time = "2025-10-02T14:35:15.21Z" }, - { url = "https://files.pythonhosted.org/packages/39/c5/cc01e4f6188656e56112d6a8e0dfe298a16934b8c47a247236549a3f7695/xxhash-3.6.0-cp313-cp313-win_arm64.whl", hash = "sha256:457b8f85dec5825eed7b69c11ae86834a018b8e3df5e77783c999663da2f96d6", size = 27880, upload-time = "2025-10-02T14:35:16.315Z" }, - { url = "https://files.pythonhosted.org/packages/f3/30/25e5321c8732759e930c555176d37e24ab84365482d257c3b16362235212/xxhash-3.6.0-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:a42e633d75cdad6d625434e3468126c73f13f7584545a9cf34e883aa1710e702", size = 32956, upload-time = "2025-10-02T14:35:17.413Z" }, - { url = "https://files.pythonhosted.org/packages/9f/3c/0573299560d7d9f8ab1838f1efc021a280b5ae5ae2e849034ef3dee18810/xxhash-3.6.0-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:568a6d743219e717b07b4e03b0a828ce593833e498c3b64752e0f5df6bfe84db", size = 31072, upload-time = "2025-10-02T14:35:18.844Z" }, - { url = "https://files.pythonhosted.org/packages/7a/1c/52d83a06e417cd9d4137722693424885cc9878249beb3a7c829e74bf7ce9/xxhash-3.6.0-cp313-cp313t-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:bec91b562d8012dae276af8025a55811b875baace6af510412a5e58e3121bc54", size = 196409, upload-time = "2025-10-02T14:35:20.31Z" }, - { url = "https://files.pythonhosted.org/packages/e3/8e/c6d158d12a79bbd0b878f8355432075fc82759e356ab5a111463422a239b/xxhash-3.6.0-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:78e7f2f4c521c30ad5e786fdd6bae89d47a32672a80195467b5de0480aa97b1f", size = 215736, upload-time = "2025-10-02T14:35:21.616Z" }, - { url = "https://files.pythonhosted.org/packages/bc/68/c4c80614716345d55071a396cf03d06e34b5f4917a467faf43083c995155/xxhash-3.6.0-cp313-cp313t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:3ed0df1b11a79856df5ffcab572cbd6b9627034c1c748c5566fa79df9048a7c5", size = 214833, upload-time = "2025-10-02T14:35:23.32Z" }, - { url = "https://files.pythonhosted.org/packages/7e/e9/ae27c8ffec8b953efa84c7c4a6c6802c263d587b9fc0d6e7cea64e08c3af/xxhash-3.6.0-cp313-cp313t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:0e4edbfc7d420925b0dd5e792478ed393d6e75ff8fc219a6546fb446b6a417b1", size = 448348, upload-time = "2025-10-02T14:35:25.111Z" }, - { url = "https://files.pythonhosted.org/packages/d7/6b/33e21afb1b5b3f46b74b6bd1913639066af218d704cc0941404ca717fc57/xxhash-3.6.0-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:fba27a198363a7ef87f8c0f6b171ec36b674fe9053742c58dd7e3201c1ab30ee", size = 196070, upload-time = "2025-10-02T14:35:26.586Z" }, - { url = "https://files.pythonhosted.org/packages/96/b6/fcabd337bc5fa624e7203aa0fa7d0c49eed22f72e93229431752bddc83d9/xxhash-3.6.0-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:794fe9145fe60191c6532fa95063765529770edcdd67b3d537793e8004cabbfd", size = 212907, upload-time = "2025-10-02T14:35:28.087Z" }, - { url = "https://files.pythonhosted.org/packages/4b/d3/9ee6160e644d660fcf176c5825e61411c7f62648728f69c79ba237250143/xxhash-3.6.0-cp313-cp313t-musllinux_1_2_i686.whl", hash = "sha256:6105ef7e62b5ac73a837778efc331a591d8442f8ef5c7e102376506cb4ae2729", size = 200839, upload-time = "2025-10-02T14:35:29.857Z" }, - { url = "https://files.pythonhosted.org/packages/0d/98/e8de5baa5109394baf5118f5e72ab21a86387c4f89b0e77ef3e2f6b0327b/xxhash-3.6.0-cp313-cp313t-musllinux_1_2_ppc64le.whl", hash = "sha256:f01375c0e55395b814a679b3eea205db7919ac2af213f4a6682e01220e5fe292", size = 213304, upload-time = "2025-10-02T14:35:31.222Z" }, - { url = "https://files.pythonhosted.org/packages/7b/1d/71056535dec5c3177eeb53e38e3d367dd1d16e024e63b1cee208d572a033/xxhash-3.6.0-cp313-cp313t-musllinux_1_2_s390x.whl", hash = "sha256:d706dca2d24d834a4661619dcacf51a75c16d65985718d6a7d73c1eeeb903ddf", size = 416930, upload-time = "2025-10-02T14:35:32.517Z" }, - { url = "https://files.pythonhosted.org/packages/dc/6c/5cbde9de2cd967c322e651c65c543700b19e7ae3e0aae8ece3469bf9683d/xxhash-3.6.0-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:5f059d9faeacd49c0215d66f4056e1326c80503f51a1532ca336a385edadd033", size = 193787, upload-time = "2025-10-02T14:35:33.827Z" }, - { url = "https://files.pythonhosted.org/packages/19/fa/0172e350361d61febcea941b0cc541d6e6c8d65d153e85f850a7b256ff8a/xxhash-3.6.0-cp313-cp313t-win32.whl", hash = "sha256:1244460adc3a9be84731d72b8e80625788e5815b68da3da8b83f78115a40a7ec", size = 30916, upload-time = "2025-10-02T14:35:35.107Z" }, - { url = "https://files.pythonhosted.org/packages/ad/e6/e8cf858a2b19d6d45820f072eff1bea413910592ff17157cabc5f1227a16/xxhash-3.6.0-cp313-cp313t-win_amd64.whl", hash = "sha256:b1e420ef35c503869c4064f4a2f2b08ad6431ab7b229a05cce39d74268bca6b8", size = 31799, upload-time = "2025-10-02T14:35:36.165Z" }, - { url = "https://files.pythonhosted.org/packages/56/15/064b197e855bfb7b343210e82490ae672f8bc7cdf3ddb02e92f64304ee8a/xxhash-3.6.0-cp313-cp313t-win_arm64.whl", hash = "sha256:ec44b73a4220623235f67a996c862049f375df3b1052d9899f40a6382c32d746", size = 28044, upload-time = "2025-10-02T14:35:37.195Z" }, - { url = "https://files.pythonhosted.org/packages/7e/5e/0138bc4484ea9b897864d59fce9be9086030825bc778b76cb5a33a906d37/xxhash-3.6.0-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:a40a3d35b204b7cc7643cbcf8c9976d818cb47befcfac8bbefec8038ac363f3e", size = 32754, upload-time = "2025-10-02T14:35:38.245Z" }, - { url = "https://files.pythonhosted.org/packages/18/d7/5dac2eb2ec75fd771957a13e5dda560efb2176d5203f39502a5fc571f899/xxhash-3.6.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:a54844be970d3fc22630b32d515e79a90d0a3ddb2644d8d7402e3c4c8da61405", size = 30846, upload-time = "2025-10-02T14:35:39.6Z" }, - { url = "https://files.pythonhosted.org/packages/fe/71/8bc5be2bb00deb5682e92e8da955ebe5fa982da13a69da5a40a4c8db12fb/xxhash-3.6.0-cp314-cp314-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:016e9190af8f0a4e3741343777710e3d5717427f175adfdc3e72508f59e2a7f3", size = 194343, upload-time = "2025-10-02T14:35:40.69Z" }, - { url = "https://files.pythonhosted.org/packages/e7/3b/52badfb2aecec2c377ddf1ae75f55db3ba2d321c5e164f14461c90837ef3/xxhash-3.6.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:4f6f72232f849eb9d0141e2ebe2677ece15adfd0fa599bc058aad83c714bb2c6", size = 213074, upload-time = "2025-10-02T14:35:42.29Z" }, - { url = "https://files.pythonhosted.org/packages/a2/2b/ae46b4e9b92e537fa30d03dbc19cdae57ed407e9c26d163895e968e3de85/xxhash-3.6.0-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:63275a8aba7865e44b1813d2177e0f5ea7eadad3dd063a21f7cf9afdc7054063", size = 212388, upload-time = "2025-10-02T14:35:43.929Z" }, - { url = "https://files.pythonhosted.org/packages/f5/80/49f88d3afc724b4ac7fbd664c8452d6db51b49915be48c6982659e0e7942/xxhash-3.6.0-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:3cd01fa2aa00d8b017c97eb46b9a794fbdca53fc14f845f5a328c71254b0abb7", size = 445614, upload-time = "2025-10-02T14:35:45.216Z" }, - { url = "https://files.pythonhosted.org/packages/ed/ba/603ce3961e339413543d8cd44f21f2c80e2a7c5cfe692a7b1f2cccf58f3c/xxhash-3.6.0-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0226aa89035b62b6a86d3c68df4d7c1f47a342b8683da2b60cedcddb46c4d95b", size = 194024, upload-time = "2025-10-02T14:35:46.959Z" }, - { url = "https://files.pythonhosted.org/packages/78/d1/8e225ff7113bf81545cfdcd79eef124a7b7064a0bba53605ff39590b95c2/xxhash-3.6.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:c6e193e9f56e4ca4923c61238cdaced324f0feac782544eb4c6d55ad5cc99ddd", size = 210541, upload-time = "2025-10-02T14:35:48.301Z" }, - { url = "https://files.pythonhosted.org/packages/6f/58/0f89d149f0bad89def1a8dd38feb50ccdeb643d9797ec84707091d4cb494/xxhash-3.6.0-cp314-cp314-musllinux_1_2_i686.whl", hash = "sha256:9176dcaddf4ca963d4deb93866d739a343c01c969231dbe21680e13a5d1a5bf0", size = 198305, upload-time = "2025-10-02T14:35:49.584Z" }, - { url = "https://files.pythonhosted.org/packages/11/38/5eab81580703c4df93feb5f32ff8fa7fe1e2c51c1f183ee4e48d4bb9d3d7/xxhash-3.6.0-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:c1ce4009c97a752e682b897aa99aef84191077a9433eb237774689f14f8ec152", size = 210848, upload-time = "2025-10-02T14:35:50.877Z" }, - { url = "https://files.pythonhosted.org/packages/5e/6b/953dc4b05c3ce678abca756416e4c130d2382f877a9c30a20d08ee6a77c0/xxhash-3.6.0-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:8cb2f4f679b01513b7adbb9b1b2f0f9cdc31b70007eaf9d59d0878809f385b11", size = 414142, upload-time = "2025-10-02T14:35:52.15Z" }, - { url = "https://files.pythonhosted.org/packages/08/a9/238ec0d4e81a10eb5026d4a6972677cbc898ba6c8b9dbaec12ae001b1b35/xxhash-3.6.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:653a91d7c2ab54a92c19ccf43508b6a555440b9be1bc8be553376778be7f20b5", size = 191547, upload-time = "2025-10-02T14:35:53.547Z" }, - { url = "https://files.pythonhosted.org/packages/f1/ee/3cf8589e06c2164ac77c3bf0aa127012801128f1feebf2a079272da5737c/xxhash-3.6.0-cp314-cp314-win32.whl", hash = "sha256:a756fe893389483ee8c394d06b5ab765d96e68fbbfe6fde7aa17e11f5720559f", size = 31214, upload-time = "2025-10-02T14:35:54.746Z" }, - { url = "https://files.pythonhosted.org/packages/02/5d/a19552fbc6ad4cb54ff953c3908bbc095f4a921bc569433d791f755186f1/xxhash-3.6.0-cp314-cp314-win_amd64.whl", hash = "sha256:39be8e4e142550ef69629c9cd71b88c90e9a5db703fecbcf265546d9536ca4ad", size = 32290, upload-time = "2025-10-02T14:35:55.791Z" }, - { url = "https://files.pythonhosted.org/packages/b1/11/dafa0643bc30442c887b55baf8e73353a344ee89c1901b5a5c54a6c17d39/xxhash-3.6.0-cp314-cp314-win_arm64.whl", hash = "sha256:25915e6000338999236f1eb68a02a32c3275ac338628a7eaa5a269c401995679", size = 28795, upload-time = "2025-10-02T14:35:57.162Z" }, - { url = "https://files.pythonhosted.org/packages/2c/db/0e99732ed7f64182aef4a6fb145e1a295558deec2a746265dcdec12d191e/xxhash-3.6.0-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:c5294f596a9017ca5a3e3f8884c00b91ab2ad2933cf288f4923c3fd4346cf3d4", size = 32955, upload-time = "2025-10-02T14:35:58.267Z" }, - { url = "https://files.pythonhosted.org/packages/55/f4/2a7c3c68e564a099becfa44bb3d398810cc0ff6749b0d3cb8ccb93f23c14/xxhash-3.6.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:1cf9dcc4ab9cff01dfbba78544297a3a01dafd60f3bde4e2bfd016cf7e4ddc67", size = 31072, upload-time = "2025-10-02T14:35:59.382Z" }, - { url = "https://files.pythonhosted.org/packages/c6/d9/72a29cddc7250e8a5819dad5d466facb5dc4c802ce120645630149127e73/xxhash-3.6.0-cp314-cp314t-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:01262da8798422d0685f7cef03b2bd3f4f46511b02830861df548d7def4402ad", size = 196579, upload-time = "2025-10-02T14:36:00.838Z" }, - { url = "https://files.pythonhosted.org/packages/63/93/b21590e1e381040e2ca305a884d89e1c345b347404f7780f07f2cdd47ef4/xxhash-3.6.0-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:51a73fb7cb3a3ead9f7a8b583ffd9b8038e277cdb8cb87cf890e88b3456afa0b", size = 215854, upload-time = "2025-10-02T14:36:02.207Z" }, - { url = "https://files.pythonhosted.org/packages/ce/b8/edab8a7d4fa14e924b29be877d54155dcbd8b80be85ea00d2be3413a9ed4/xxhash-3.6.0-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:b9c6df83594f7df8f7f708ce5ebeacfc69f72c9fbaaababf6cf4758eaada0c9b", size = 214965, upload-time = "2025-10-02T14:36:03.507Z" }, - { url = "https://files.pythonhosted.org/packages/27/67/dfa980ac7f0d509d54ea0d5a486d2bb4b80c3f1bb22b66e6a05d3efaf6c0/xxhash-3.6.0-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:627f0af069b0ea56f312fd5189001c24578868643203bca1abbc2c52d3a6f3ca", size = 448484, upload-time = "2025-10-02T14:36:04.828Z" }, - { url = "https://files.pythonhosted.org/packages/8c/63/8ffc2cc97e811c0ca5d00ab36604b3ea6f4254f20b7bc658ca825ce6c954/xxhash-3.6.0-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:aa912c62f842dfd013c5f21a642c9c10cd9f4c4e943e0af83618b4a404d9091a", size = 196162, upload-time = "2025-10-02T14:36:06.182Z" }, - { url = "https://files.pythonhosted.org/packages/4b/77/07f0e7a3edd11a6097e990f6e5b815b6592459cb16dae990d967693e6ea9/xxhash-3.6.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:b465afd7909db30168ab62afe40b2fcf79eedc0b89a6c0ab3123515dc0df8b99", size = 213007, upload-time = "2025-10-02T14:36:07.733Z" }, - { url = "https://files.pythonhosted.org/packages/ae/d8/bc5fa0d152837117eb0bef6f83f956c509332ce133c91c63ce07ee7c4873/xxhash-3.6.0-cp314-cp314t-musllinux_1_2_i686.whl", hash = "sha256:a881851cf38b0a70e7c4d3ce81fc7afd86fbc2a024f4cfb2a97cf49ce04b75d3", size = 200956, upload-time = "2025-10-02T14:36:09.106Z" }, - { url = "https://files.pythonhosted.org/packages/26/a5/d749334130de9411783873e9b98ecc46688dad5db64ca6e04b02acc8b473/xxhash-3.6.0-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:9b3222c686a919a0f3253cfc12bb118b8b103506612253b5baeaac10d8027cf6", size = 213401, upload-time = "2025-10-02T14:36:10.585Z" }, - { url = "https://files.pythonhosted.org/packages/89/72/abed959c956a4bfc72b58c0384bb7940663c678127538634d896b1195c10/xxhash-3.6.0-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:c5aa639bc113e9286137cec8fadc20e9cd732b2cc385c0b7fa673b84fc1f2a93", size = 417083, upload-time = "2025-10-02T14:36:12.276Z" }, - { url = "https://files.pythonhosted.org/packages/0c/b3/62fd2b586283b7d7d665fb98e266decadf31f058f1cf6c478741f68af0cb/xxhash-3.6.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:5c1343d49ac102799905e115aee590183c3921d475356cb24b4de29a4bc56518", size = 193913, upload-time = "2025-10-02T14:36:14.025Z" }, - { url = "https://files.pythonhosted.org/packages/9a/9a/c19c42c5b3f5a4aad748a6d5b4f23df3bed7ee5445accc65a0fb3ff03953/xxhash-3.6.0-cp314-cp314t-win32.whl", hash = "sha256:5851f033c3030dd95c086b4a36a2683c2ff4a799b23af60977188b057e467119", size = 31586, upload-time = "2025-10-02T14:36:15.603Z" }, - { url = "https://files.pythonhosted.org/packages/03/d6/4cc450345be9924fd5dc8c590ceda1db5b43a0a889587b0ae81a95511360/xxhash-3.6.0-cp314-cp314t-win_amd64.whl", hash = "sha256:0444e7967dac37569052d2409b00a8860c2135cff05502df4da80267d384849f", size = 32526, upload-time = "2025-10-02T14:36:16.708Z" }, - { url = "https://files.pythonhosted.org/packages/0f/c9/7243eb3f9eaabd1a88a5a5acadf06df2d83b100c62684b7425c6a11bcaa8/xxhash-3.6.0-cp314-cp314t-win_arm64.whl", hash = "sha256:bb79b1e63f6fd84ec778a4b1916dfe0a7c3fdb986c06addd5db3a0d413819d95", size = 28898, upload-time = "2025-10-02T14:36:17.843Z" }, - { url = "https://files.pythonhosted.org/packages/93/1e/8aec23647a34a249f62e2398c42955acd9b4c6ed5cf08cbea94dc46f78d2/xxhash-3.6.0-pp311-pypy311_pp73-macosx_10_15_x86_64.whl", hash = "sha256:0f7b7e2ec26c1666ad5fc9dbfa426a6a3367ceaf79db5dd76264659d509d73b0", size = 30662, upload-time = "2025-10-02T14:37:01.743Z" }, - { url = "https://files.pythonhosted.org/packages/b8/0b/b14510b38ba91caf43006209db846a696ceea6a847a0c9ba0a5b1adc53d6/xxhash-3.6.0-pp311-pypy311_pp73-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:5dc1e14d14fa0f5789ec29a7062004b5933964bb9b02aae6622b8f530dc40296", size = 41056, upload-time = "2025-10-02T14:37:02.879Z" }, - { url = "https://files.pythonhosted.org/packages/50/55/15a7b8a56590e66ccd374bbfa3f9ffc45b810886c8c3b614e3f90bd2367c/xxhash-3.6.0-pp311-pypy311_pp73-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:881b47fc47e051b37d94d13e7455131054b56749b91b508b0907eb07900d1c13", size = 36251, upload-time = "2025-10-02T14:37:04.44Z" }, - { url = "https://files.pythonhosted.org/packages/62/b2/5ac99a041a29e58e95f907876b04f7067a0242cb85b5f39e726153981503/xxhash-3.6.0-pp311-pypy311_pp73-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c6dc31591899f5e5666f04cc2e529e69b4072827085c1ef15294d91a004bc1bd", size = 32481, upload-time = "2025-10-02T14:37:05.869Z" }, - { url = "https://files.pythonhosted.org/packages/7b/d9/8d95e906764a386a3d3b596f3c68bb63687dfca806373509f51ce8eea81f/xxhash-3.6.0-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:15e0dac10eb9309508bfc41f7f9deaa7755c69e35af835db9cb10751adebc35d", size = 31565, upload-time = "2025-10-02T14:37:06.966Z" }, +version = "3.8.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/8e/63/71aa56b151a1b28770037a61bd4e461c2619cfc8866a4fcaf1548605e325/xxhash-3.8.1.tar.gz", hash = "sha256:b0de4bf3aa66363552d52c6a89003c479911f12098cd48a53d44a0f7a25f7c46", size = 86223, upload-time = "2026-07-06T10:49:58.937Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/55/97/1a8cebf0a6650417f08a18231590e2515aacd5ce39c3ad8b9e013ebd437d/xxhash-3.8.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:27a9e475157f7315826118e3f3127909a0fe25f1b43d3d3be9c584f9d265f937", size = 34695, upload-time = "2026-07-06T10:43:40.248Z" }, + { url = "https://files.pythonhosted.org/packages/2f/cf/745b9bc0dd9c341bc074b5fc700db7bbef0f3b69ab21446492296ab37e50/xxhash-3.8.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:9b2ce44bf8f4a1d01f418b3110ff8dff32fd3f3e836c0e06333c3725f243fa6c", size = 32376, upload-time = "2026-07-06T10:43:41.97Z" }, + { url = "https://files.pythonhosted.org/packages/65/a4/8512a901b1d6ad4a9838d1b40385907a879d7e005a5afbec5d39526b69f6/xxhash-3.8.1-cp310-cp310-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:942bc86e9be6fdd6e1175048f5fe8f8fdaaf2309dd1323ef1e155a69cd346780", size = 217470, upload-time = "2026-07-06T10:43:43.572Z" }, + { url = "https://files.pythonhosted.org/packages/a0/ad/0ffd8094ea29579bb2dc42fa74d08570e9ea3d95db561e6b1105e69b9ca6/xxhash-3.8.1-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0204701e6d01f64254e0e5ff4255812b1febe027ddd7dda63372e27f98b5e91f", size = 237799, upload-time = "2026-07-06T10:43:45.248Z" }, + { url = "https://files.pythonhosted.org/packages/b3/90/783c6b3f9336bd07449fe672be32cef6833633936bbfda8d3b23ee18d202/xxhash-3.8.1-cp310-cp310-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:7dc4bdf008f77c88d544849c48c1a40faf25a5eff6cc466de2e8edc37c191fce", size = 262587, upload-time = "2026-07-06T10:43:46.733Z" }, + { url = "https://files.pythonhosted.org/packages/c4/77/ba0316a7c3e661b86830a47ae4987798616ce1b15af8d2a6358e2d89ef60/xxhash-3.8.1-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:5c566b123dce7e4867ca518434cdfb9f84e5023771235b2e3107a26c9a41cbd8", size = 238484, upload-time = "2026-07-06T10:43:48.453Z" }, + { url = "https://files.pythonhosted.org/packages/09/79/33001037c1cba90f4ced38b257161c13452024c0db44208f883e2e47f3fc/xxhash-3.8.1-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:9f23083e1bd9d901f844af7a126727c486e7eada9a1a6791c8f7e73f94fac656", size = 469909, upload-time = "2026-07-06T10:43:50.188Z" }, + { url = "https://files.pythonhosted.org/packages/45/90/237eded9dd6ae638083294e5a9f77b317aaebd480a330806b39c192a0de1/xxhash-3.8.1-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:64af54dd1c3a45a27c04942f9a1a4683322bdd127f4745cca4e02549c1d2d2bb", size = 217166, upload-time = "2026-07-06T10:43:51.816Z" }, + { url = "https://files.pythonhosted.org/packages/0b/6a/8cb439dc9920e1468e1c2d69ef77cbeb4be3b1ae9f4b5344c07a2b59af18/xxhash-3.8.1-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:8ea8a141eeced4f6262ab6dd71c681ac546a558c30bb586abe087d814b5f85ea", size = 307593, upload-time = "2026-07-06T10:43:53.436Z" }, + { url = "https://files.pythonhosted.org/packages/ec/c6/c0607d373c8affea92101a3926c4fc8b026bcf8983e05fd58f3a0380ebf8/xxhash-3.8.1-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:a98b2f95cab589e0f5e92c48431afb4d56238b8bf6668edcc66166180e9b509b", size = 234702, upload-time = "2026-07-06T10:43:55.042Z" }, + { url = "https://files.pythonhosted.org/packages/5b/cb/f4cfd456624c1f017858168b7ba9443dad810da8aac779a612658450e827/xxhash-3.8.1-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:1b86ae798a976ccbc1d02af6ccb98f5b4d24756b1f65e995f11d10fe071f486f", size = 265749, upload-time = "2026-07-06T10:43:56.749Z" }, + { url = "https://files.pythonhosted.org/packages/33/f3/9006669c04b01206e21b2177425c649461ba188930a052c2f1728d6ec6a8/xxhash-3.8.1-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:81f4ed9ca9644bc95cd976bfe10f7a4cafab8ffdc3aed52877d4600e445be7ef", size = 221992, upload-time = "2026-07-06T10:43:58.12Z" }, + { url = "https://files.pythonhosted.org/packages/2b/0b/7e6f3eaa05df5e0b6c94aa452b0672801f7031e602081f07fd441aaaaed5/xxhash-3.8.1-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:cb3fe820c27593f170770d6c8d791936cf6275d9269405fbb7b30a55363c10c8", size = 236899, upload-time = "2026-07-06T10:43:59.562Z" }, + { url = "https://files.pythonhosted.org/packages/da/cc/bbaee4987f3aab1d7b33bb430bb49e940646160af448b9167431c931126d/xxhash-3.8.1-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:7345007c12780985de4fd740148776d1eee18c0d41407c6fa1e48c5450304fe5", size = 297934, upload-time = "2026-07-06T10:44:01.132Z" }, + { url = "https://files.pythonhosted.org/packages/a7/97/6bee358660eb8b4f73c00b00b00bc616ebde00e1ab4b67c63486ce360648/xxhash-3.8.1-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:12eaeaa9ab8b9e6033a1fa5f6b338aaf55ff4df4bee11b59fd6ee03b19186ee4", size = 439315, upload-time = "2026-07-06T10:44:02.878Z" }, + { url = "https://files.pythonhosted.org/packages/c6/50/7e35275f39256bedace0c3cd5be3c72d4ac9d5aecf5e5fdc3530337cd263/xxhash-3.8.1-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:e2a845687219ba3214126f14a8a5861f97c9e065a7d0b8252adb6df13eea86fb", size = 214038, upload-time = "2026-07-06T10:44:04.504Z" }, + { url = "https://files.pythonhosted.org/packages/59/2d/69d02d096ee50bdf3ef0d208d874f52c71b1aa6906066bce3c52fedb8bc6/xxhash-3.8.1-cp310-cp310-win32.whl", hash = "sha256:656256c9f9303e47f07d5cb8ae4468285370adfafd7ba48aea33a458e7697626", size = 31939, upload-time = "2026-07-06T10:44:06.213Z" }, + { url = "https://files.pythonhosted.org/packages/c9/1d/e06fca9844919ca91c6587d530cfa1e745830ec73ad38f44f04b25d1bfb7/xxhash-3.8.1-cp310-cp310-win_amd64.whl", hash = "sha256:27cfc2f1ed76f956f36dfe0c56e5f5a3e94cd91eb78b893f63e2ef2ae404fcdf", size = 32729, upload-time = "2026-07-06T10:44:07.621Z" }, + { url = "https://files.pythonhosted.org/packages/8c/c2/800648d99039927b5a86d8ae02cd86a556a5ee1678d388216f6b44c8966c/xxhash-3.8.1-cp310-cp310-win_arm64.whl", hash = "sha256:c85949d02c85adf6d786eb94858e124989a632a4e65739835b2fc5761827fac3", size = 29215, upload-time = "2026-07-06T10:44:08.916Z" }, + { url = "https://files.pythonhosted.org/packages/8a/5a/05eaa129555f85476a3e16ff869e95f81a78bbe4647eef9d0229f515a317/xxhash-3.8.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:602efcad4a42c184e81d43a2b7e6e4f524d619878f2b6ee2ba469011f47c8147", size = 34699, upload-time = "2026-07-06T10:44:10.14Z" }, + { url = "https://files.pythonhosted.org/packages/80/59/0df1133958b2228929355e022aab1e958c7b2c43e27bf7f59bc9edfa8a54/xxhash-3.8.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:131324f719957b988861714de7d6ddf57b47abec3b0cc691302ffeaba0e05e10", size = 32373, upload-time = "2026-07-06T10:44:11.353Z" }, + { url = "https://files.pythonhosted.org/packages/3e/bf/1cfda5b5e6bf26617812b4a31662ef2220d2ad04e0a55b8ff9eb36e56a5c/xxhash-3.8.1-cp311-cp311-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:db77278a6eddadbf44ce5aae2fee5ebb4d061f026b1ce2130d058cd4d7a7b670", size = 220284, upload-time = "2026-07-06T10:44:12.683Z" }, + { url = "https://files.pythonhosted.org/packages/70/93/45dc0ad7913b69e5b08bd039236cf628380e4c9cc76a8a4c6625a328e058/xxhash-3.8.1-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1c332dd48b8cb050da2bb2a3c96d72b1664168650a250ef9718e423df7989e05", size = 240980, upload-time = "2026-07-06T10:44:14.297Z" }, + { url = "https://files.pythonhosted.org/packages/e9/02/f28ba7d17f2c1410ee397982c817ab1bd5b2701070c2d2c373539aad000a/xxhash-3.8.1-cp311-cp311-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:a5cd96f6dcdf4fa657b2d95668d71d58455248f98712ecffaa9c528edf40ccae", size = 264526, upload-time = "2026-07-06T10:44:16.017Z" }, + { url = "https://files.pythonhosted.org/packages/5c/d0/f10651cec2c7981b20d693deae6bdfc438427d92be2db4ccabb6181f0021/xxhash-3.8.1-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:c959f88160b13b4e730b0d75b459b7929fc0d2225c284c9683ac95d6feeeac6a", size = 241369, upload-time = "2026-07-06T10:44:17.698Z" }, + { url = "https://files.pythonhosted.org/packages/ff/40/136e0cbaf5db51e191423b1c98643593189f02b6cd90837bf64b19113d70/xxhash-3.8.1-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:027dee4355f3fcc41481650d846cf6cfc895c85a1ab7acd063063821a0df5b4c", size = 473186, upload-time = "2026-07-06T10:44:19.354Z" }, + { url = "https://files.pythonhosted.org/packages/4b/3f/6aa808a96bdc43dba9a740dec56c744526ee3c0019e32c75e810fa90ae4d/xxhash-3.8.1-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ad52a0e4bcc0ba956a953a169d1feec2734a64981d689e4fc8f490f7bf91af60", size = 220092, upload-time = "2026-07-06T10:44:20.956Z" }, + { url = "https://files.pythonhosted.org/packages/47/28/a8675e78a9ced96dab853416162268e10e05b452e95db7888cf69f58ac5f/xxhash-3.8.1-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:5d3dfb1f0ff146da7952867a9414f0c7a29762f8825a84879592612fd6139342", size = 309846, upload-time = "2026-07-06T10:44:22.543Z" }, + { url = "https://files.pythonhosted.org/packages/89/0f/7fe4d4ef4e69f0033e012396ee2a115886bca7b10b7e45ce398626436bfc/xxhash-3.8.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:4482380b462ca9e59994d072a877ecadd1cf51102daeeab2db696f96ab763723", size = 237659, upload-time = "2026-07-06T10:44:24.135Z" }, + { url = "https://files.pythonhosted.org/packages/38/8f/83e9e31d4ed57fe963b99cb5b13a23e3e0f0dad1885aa0ebd2a7819dd423/xxhash-3.8.1-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:950ac754d16daea42038f38e7465eb84cda4d08d7343c1c915771b29470f065a", size = 268737, upload-time = "2026-07-06T10:44:25.875Z" }, + { url = "https://files.pythonhosted.org/packages/57/79/7e7de46dbe5d1f49afc96a0bc42e6b8df24eae3d6bad6007b99e42f48430/xxhash-3.8.1-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:0418ec8b2331b9d4d575fc9284427e8e69449d7172e99e1a86fcdd1f51a0a937", size = 224955, upload-time = "2026-07-06T10:44:27.777Z" }, + { url = "https://files.pythonhosted.org/packages/ec/34/b8540839e958d5ef5c6101af6f16032109e7099698ae8edbc8dcefe4d8f4/xxhash-3.8.1-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:32a94ad2763e0263d9102037d349002c3d3c401e42770542c3eeb4801f311661", size = 239653, upload-time = "2026-07-06T10:44:29.422Z" }, + { url = "https://files.pythonhosted.org/packages/ce/87/a735d05f7f859354acadabe470ff40e2c46672275f96dcf096a761904def/xxhash-3.8.1-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:89b11a5cdd441aa463f6d34ca0241602bc09b001a76994b6059828494108c673", size = 300213, upload-time = "2026-07-06T10:44:31.401Z" }, + { url = "https://files.pythonhosted.org/packages/98/31/3e1cb020237b68117fc212dc5f9753b87f865b4dfee7c1ce62d0836955b5/xxhash-3.8.1-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:09a204dd4bb0823daf938cdd0dc8057d5f1e14fe3cbde929424255f23f9de872", size = 442508, upload-time = "2026-07-06T10:44:33.023Z" }, + { url = "https://files.pythonhosted.org/packages/23/bf/f80090622141cc734b039ce1d15ce3ff6dced375e9680249bf5b9b8c6bf9/xxhash-3.8.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:e710ad822c493fb80a4fbc1e3d0a807b1422cb90adbe64378f98291b7fa48fef", size = 216853, upload-time = "2026-07-06T10:44:34.983Z" }, + { url = "https://files.pythonhosted.org/packages/a6/a3/60157acecc307b238d3651c2483168e224b48b23a36ae6d6903588341d80/xxhash-3.8.1-cp311-cp311-win32.whl", hash = "sha256:5013be3bea7612852c62a7437f3302c1cfb91ca7e703b194459db0b2b2e0d792", size = 31936, upload-time = "2026-07-06T10:44:36.542Z" }, + { url = "https://files.pythonhosted.org/packages/59/5c/ef70c418d878d187b8da56d4cdc06aea6cf5e456b301e96e51e1d2cc8625/xxhash-3.8.1-cp311-cp311-win_amd64.whl", hash = "sha256:f377012b86c0a23a1df0cf5a1b05aa7187649e472f71c7892e5f2c2815bbe74f", size = 32724, upload-time = "2026-07-06T10:44:38.177Z" }, + { url = "https://files.pythonhosted.org/packages/2c/25/f008db952cec6b2a26445b456eeed2ebebd65e08e848ebe09ed6ac0634e6/xxhash-3.8.1-cp311-cp311-win_arm64.whl", hash = "sha256:836f11d4474d3228e9909d97216faa4f7505df41cfaf3927eb29809de785a78d", size = 29212, upload-time = "2026-07-06T10:44:39.577Z" }, + { url = "https://files.pythonhosted.org/packages/42/91/f65c34a7aa7b4e7cf4854f8e6ef3f7ee32ceac41d4f008da0780db0612f6/xxhash-3.8.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:e6e49370822c1f4d8d90e678b06dbcb08b51a026a7c4b55479e7d467f2e813bc", size = 34680, upload-time = "2026-07-06T10:44:40.932Z" }, + { url = "https://files.pythonhosted.org/packages/57/04/b10a245a4c09a9cfa88f8e9ae755029413ad1ac17047f9a61906e5ae0799/xxhash-3.8.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:220d68130f83f7cc86d6edfdeab176adc73d7200bf3a8ec10c629e8cf605c215", size = 32397, upload-time = "2026-07-06T10:44:42.196Z" }, + { url = "https://files.pythonhosted.org/packages/3a/75/45ab795b5945b6388583bd75202106af505537935566c15a1577797a0e08/xxhash-3.8.1-cp312-cp312-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:4d365ee1892c1fa803536f8c6ce21d24b29c9718ec75eb856095c07830f8c478", size = 220549, upload-time = "2026-07-06T10:44:43.603Z" }, + { url = "https://files.pythonhosted.org/packages/13/44/5ba2bd0a14ddf4193fc7d8ec29625f659f22c06d60b28f04bf46305d8330/xxhash-3.8.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:852bfe059720632e2f16a6a4745e41d20937b2bf2a42a401e2412046bb6971cc", size = 241186, upload-time = "2026-07-06T10:44:45.534Z" }, + { url = "https://files.pythonhosted.org/packages/23/32/c4147def4d1e4538b906f82731e0ba23424377fc50a7cddd03cd284c8f63/xxhash-3.8.1-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:2f8c25a7061d952de589bd0ea0eaadee32378ff83dd6a677b267f9cd86f401f8", size = 264852, upload-time = "2026-07-06T10:44:47.199Z" }, + { url = "https://files.pythonhosted.org/packages/6c/bd/71ed14f4f0318bb7fd7b2ec51999413487fa8da8d41208e84d50d1ef0f98/xxhash-3.8.1-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:868a8dcaff1a84ba78038e1cef14fc88ccf84d9b4d12ea604696e0693296aa56", size = 242663, upload-time = "2026-07-06T10:44:48.846Z" }, + { url = "https://files.pythonhosted.org/packages/91/09/70af22c565a8473b3f2ae73f88e7721af281bc4a575236dbd1970c9f76f6/xxhash-3.8.1-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:6536d8677d2fff7e64cd0b98b976df9de7aee0e69590044c2af5f51b76b7a170", size = 473510, upload-time = "2026-07-06T10:44:50.695Z" }, + { url = "https://files.pythonhosted.org/packages/18/96/34db781c8f0cf99c544ca1f2bc2e5bf55426e1eb4ca6de8ea5da56a9f352/xxhash-3.8.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:82c0cedd280eab2e8291270e6c04894dbc096f8159a39dcf1807429f026ca3cc", size = 220469, upload-time = "2026-07-06T10:44:52.422Z" }, + { url = "https://files.pythonhosted.org/packages/93/5f/9a184f615fa5a4dce30c01534f62946ce5a11ce40f73785cbd356ccabaa9/xxhash-3.8.1-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:daa86e4b68221d38e669bb236ba112d0335353829fb627c82e5909e4bbe8694c", size = 310290, upload-time = "2026-07-06T10:44:54.142Z" }, + { url = "https://files.pythonhosted.org/packages/a9/dc/9b9a9789011ee153723a5eb9e7dd7fcbae2ba9b3fe7a729249ca7c252056/xxhash-3.8.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:2bc7113e6f2b6b3922dd61796ca9f36af09da3773898e7003038dc992fc83b8d", size = 238173, upload-time = "2026-07-06T10:44:55.693Z" }, + { url = "https://files.pythonhosted.org/packages/ec/4d/71c6005ada9dcb608a4e1902e8475ecadb5f3fbfa04e1e244d276a2d0c43/xxhash-3.8.1-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:5eed32dad81d6ba8e62dc7b9ffa0500199385d7810a8dd9d4eafaceb8c6e20bb", size = 269026, upload-time = "2026-07-06T10:44:57.424Z" }, + { url = "https://files.pythonhosted.org/packages/2f/87/d6c036ba25dfbd9c8633be5aa86fc9474bbb9e2c68212a841d090abe7344/xxhash-3.8.1-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:83697b0ea1f10e7f5d8b26a4906fa851393c61546c63839643a2b7fe2d868061", size = 224970, upload-time = "2026-07-06T10:44:59.085Z" }, + { url = "https://files.pythonhosted.org/packages/48/62/4c1f035a41c5752aa05e195b6c904c07b94fe9061a16de61e72a6e6b135f/xxhash-3.8.1-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:36fc69160465ae75c6ec4ac9f781bb2aa16ae7ff869e73c26fee85fbb11b9887", size = 240820, upload-time = "2026-07-06T10:45:00.746Z" }, + { url = "https://files.pythonhosted.org/packages/da/14/d39d565069b87e86d21a2af2a31d04db79249d25aa8d5b62959056a89857/xxhash-3.8.1-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:445e0f5a31f2f3546ae0895d4811e159518cdc9d824c11419898d40cfadb677e", size = 300619, upload-time = "2026-07-06T10:45:02.716Z" }, + { url = "https://files.pythonhosted.org/packages/13/22/75467acc887edc8cf71c97ab1708feb3df7a88bda589b9f399765c6387d2/xxhash-3.8.1-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:dfe0580fbfd5e4af87d0cc52d2044f155d55ebd8c8a93568758a2ea7d8e15975", size = 443267, upload-time = "2026-07-06T10:45:04.653Z" }, + { url = "https://files.pythonhosted.org/packages/a4/b6/1da3baa5fa6ef705e3425fddd382be7dfc4dfba2686df90a20f16e9c7b1b/xxhash-3.8.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:095e1323fa108be1292c54c86da3ef3c7a7dc015b105a52133973bc07a6ad11a", size = 217338, upload-time = "2026-07-06T10:45:06.304Z" }, + { url = "https://files.pythonhosted.org/packages/78/dd/b5295a9f97484e7a1c2b283a742ca45e3104991c55a1ef670dde161829ba/xxhash-3.8.1-cp312-cp312-win32.whl", hash = "sha256:bf28f55e427e0483acb1f666bd0d869b6d5e5a716680c216ad7befe3d4cfba2e", size = 31970, upload-time = "2026-07-06T10:45:07.823Z" }, + { url = "https://files.pythonhosted.org/packages/ec/31/3fa0b807d7e21515cd975e7fe5c039d52ac3e9401a96d6ad68dae6305215/xxhash-3.8.1-cp312-cp312-win_amd64.whl", hash = "sha256:2256e80e4960ee282f63428adb349cb7f8bd8efe4db770d88eb815f4b9860724", size = 32741, upload-time = "2026-07-06T10:45:09.42Z" }, + { url = "https://files.pythonhosted.org/packages/b8/05/86feada74e239600e6875aa507afb40482a89b92700aa74a92da83bdcb77/xxhash-3.8.1-cp312-cp312-win_arm64.whl", hash = "sha256:9df56e6df96a60590935e22373041cccc91fd55858763dcffb55bf63b3a2b396", size = 29234, upload-time = "2026-07-06T10:45:10.809Z" }, + { url = "https://files.pythonhosted.org/packages/6b/8c/446bb782cd0d27007a917b5569a08dd73219c3e8d6e459014db104b27bdb/xxhash-3.8.1-cp313-cp313-android_21_arm64_v8a.whl", hash = "sha256:3c682fcd96eb4bf64be32a4d95f96107e1588005831bd8a741b324fdda01b913", size = 38562, upload-time = "2026-07-06T10:45:12.425Z" }, + { url = "https://files.pythonhosted.org/packages/d7/ec/c0c45627eaa6be7a5d6117423adf8f7a15b17ee74b4b17072cca5959a225/xxhash-3.8.1-cp313-cp313-android_21_x86_64.whl", hash = "sha256:036a024d8b9c01f70782e09ed98d532e76fd23f950ae7154bd950fe94e90ebec", size = 36656, upload-time = "2026-07-06T10:45:13.932Z" }, + { url = "https://files.pythonhosted.org/packages/f6/94/8324c04cc7597154caaeba6c094e01fbd2e7601d01e7a13eea9f5420e77b/xxhash-3.8.1-cp313-cp313-ios_13_0_arm64_iphoneos.whl", hash = "sha256:d6a5c0bce213b23b0166fe0d35bcbbe23ce4b968f257cc7eb6fd57cb8e1e6297", size = 31169, upload-time = "2026-07-06T10:45:15.687Z" }, + { url = "https://files.pythonhosted.org/packages/40/a4/beb6bb26e1184e126dbe7a5682330214ef54dcfbf882078aa9f4b5428d42/xxhash-3.8.1-cp313-cp313-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:5177aa44eddaa97c6ef0cc00c6d540edb64d51781d2f8fb941612ec61a92c9ed", size = 32177, upload-time = "2026-07-06T10:45:17.035Z" }, + { url = "https://files.pythonhosted.org/packages/56/0f/fc4c92a5a528f839b34b6419b2e53c8597f2a629d5a1f5d721f65bfa1fd6/xxhash-3.8.1-cp313-cp313-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:7801b7223db017b9c0c9ccf37e44524edb35a1544a1c032add22c061c6af0276", size = 34642, upload-time = "2026-07-06T10:45:18.39Z" }, + { url = "https://files.pythonhosted.org/packages/d4/58/edbfb141d4000767ac6a9694f8ac0763e2c2e983e65c9e31620ba56e2667/xxhash-3.8.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:9e80238259655bf69d7bcd08226a970d7f42605f3157786bfa76dd13472d7fa0", size = 34684, upload-time = "2026-07-06T10:45:20.033Z" }, + { url = "https://files.pythonhosted.org/packages/07/3f/5072f1f0f5714186f0ac2a0b5a4929ce30d4b845e94886b6c01b6ebda0be/xxhash-3.8.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:bcab50a389cc04d87f90092af78a6adba2ab3deca63175a3344ca83514045315", size = 32401, upload-time = "2026-07-06T10:45:21.414Z" }, + { url = "https://files.pythonhosted.org/packages/49/c7/802ea2f9c2ed59219934d6d65c470d502b1788043eae277a52af8658bda6/xxhash-3.8.1-cp313-cp313-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:a2489d3a776fa380cb8e71f54c7fda268a9baf3de9b1395093fd280f95735907", size = 220617, upload-time = "2026-07-06T10:45:23.234Z" }, + { url = "https://files.pythonhosted.org/packages/99/a8/e10488efd31fcb13fcd6acbc6e788f10c6f8e3a0cc4ae3eb89dc19c55a12/xxhash-3.8.1-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:32ab1e5432690276e71192be7401b55f96db2d0eedea5d44eb1f164505669cc0", size = 241295, upload-time = "2026-07-06T10:45:25.364Z" }, + { url = "https://files.pythonhosted.org/packages/18/cc/14180b17d44892a631f8ae7323c30bfbb1328efc8209e528a480293528ac/xxhash-3.8.1-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:b30e01a0b97a4bc3f519a4d7a82da3dc53251fb0de5eeea8660dcd4ff094c0c2", size = 264688, upload-time = "2026-07-06T10:45:27.09Z" }, + { url = "https://files.pythonhosted.org/packages/a9/72/a14019d0c5f6c41ee407a503036ae32787c91325ca218a96a9b5627be651/xxhash-3.8.1-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:1f44275ddb0978b67a58a951501903f04d49335a91f7681c9ce122ecb8ccb329", size = 242740, upload-time = "2026-07-06T10:45:28.753Z" }, + { url = "https://files.pythonhosted.org/packages/68/08/92550e556c6fcfcb96c6a336945eb53a431ed43120ed749636debb16c5cf/xxhash-3.8.1-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:e3b87cbd974512c0c5fc7b469c36b2cdc9ee6d76e4ec78bccb2c7184611c49b0", size = 473599, upload-time = "2026-07-06T10:45:30.524Z" }, + { url = "https://files.pythonhosted.org/packages/29/83/e361d3c1acd1b21e1d489616de6fa4aaf843365d8179f612e3743eac20a9/xxhash-3.8.1-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:98ee81b4b7f3023c9cb04a78cc67610baffcb5812d92f2096cb5a5efc6f19437", size = 220559, upload-time = "2026-07-06T10:45:32.979Z" }, + { url = "https://files.pythonhosted.org/packages/05/01/006a4243c2c2a6831827f9999f6d1c23feeef100eb023c1f886022a00bf3/xxhash-3.8.1-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:2666f059a1588a99267e33605365ed89cea92f424b3522806a9f4bd8ad2e3d62", size = 310383, upload-time = "2026-07-06T10:45:35.875Z" }, + { url = "https://files.pythonhosted.org/packages/d8/20/af388e8bf9f9a0f89eeef7d2a1935d176ee1c20bc6adeda05035879379cf/xxhash-3.8.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:b0093cf7eeb91b84776e8742113afa4bdf47533d36cf719179aaaf1f56f6f8bf", size = 238228, upload-time = "2026-07-06T10:45:38.02Z" }, + { url = "https://files.pythonhosted.org/packages/63/6b/4666579a87eebd1744663c404297355fa0658617b015cedfa58810ee7036/xxhash-3.8.1-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:3a800912a2e5e975d4128969d645c4a2a80aa886ccd6c9b1c6f44529e327e8cf", size = 269137, upload-time = "2026-07-06T10:45:39.954Z" }, + { url = "https://files.pythonhosted.org/packages/de/d3/e963a8a46f900a137d91b02144d8ea07a8f812971b138204a3b2f8b8e55c/xxhash-3.8.1-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:0fe37f72a207223d22a4eddc3149d4298993385aa9daef25c039246ca5a309f3", size = 225068, upload-time = "2026-07-06T10:45:41.718Z" }, + { url = "https://files.pythonhosted.org/packages/aa/80/9d181dbcde4b0fe48375f48833a5832d4b8cd2b349b15110c92ee472d874/xxhash-3.8.1-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:5db43f249b4be9f99ef4b967863f37094fb40e67effafb78ba4f0356b6396104", size = 240874, upload-time = "2026-07-06T10:45:43.414Z" }, + { url = "https://files.pythonhosted.org/packages/39/15/ce3ab5a1cd27ead25a5196e55a7284220f6ad6e316da494ffd900b2b600f/xxhash-3.8.1-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:c4ed42965c2cd9081f011be22f69d0e65d3b6165fe7734072fd0c232840bbd4e", size = 300702, upload-time = "2026-07-06T10:45:45.135Z" }, + { url = "https://files.pythonhosted.org/packages/96/c0/2281a8ab5f2a62dbf57a23c58a01ccc1d98abf40f71193c8a81f59e759b5/xxhash-3.8.1-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:3557bec8fcb11738a8920eeb68974bc76b75262f6947998d3147954ce0a4b893", size = 443351, upload-time = "2026-07-06T10:45:47.188Z" }, + { url = "https://files.pythonhosted.org/packages/81/2e/071a58c1a53a52d4f7a3aa0987be0c396dffd40da8204805fe1b130a81f4/xxhash-3.8.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:00de40f3b42240db23a82a5c682b55d7263d84a26a953240c1aee463409660e3", size = 217396, upload-time = "2026-07-06T10:45:48.925Z" }, + { url = "https://files.pythonhosted.org/packages/68/44/36ab58134badd9d3433fc7b53c4ca8d113d8e807782885628640f8297a4d/xxhash-3.8.1-cp313-cp313-win32.whl", hash = "sha256:b5196cc2574cfec572a5f3fb7cfa5ade27305ae3d06516a082132441aff4c83a", size = 31974, upload-time = "2026-07-06T10:45:50.591Z" }, + { url = "https://files.pythonhosted.org/packages/96/2a/2a0b84798448e766f7b89ceed073cb0cb5a43fc9ebbacbdea74a38de18e3/xxhash-3.8.1-cp313-cp313-win_amd64.whl", hash = "sha256:538f5f865df6cd8c32dd63158a0e5b4f5dd08d732a7da8b7228a5a0776c8ce55", size = 32739, upload-time = "2026-07-06T10:45:52.221Z" }, + { url = "https://files.pythonhosted.org/packages/d4/60/bb51dbf7c363ff88a7cbd50b7959718219577ef44d7cf255929ffc4a2194/xxhash-3.8.1-cp313-cp313-win_arm64.whl", hash = "sha256:a6617f30641ba0d8baa1635fbefb1dffc5165ec36d26921bd5cee13497cd937a", size = 29239, upload-time = "2026-07-06T10:45:53.714Z" }, + { url = "https://files.pythonhosted.org/packages/56/d3/827ca123c2ee5443a6aaed3c5dd199237dc2f010e2bebd7ec09ef36f3a5f/xxhash-3.8.1-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:bfcd82852c62a60e314670a9602de354c4460f8adad916e2e42a20860c7870bc", size = 34964, upload-time = "2026-07-06T10:45:55.535Z" }, + { url = "https://files.pythonhosted.org/packages/05/67/67ae2a3ccdeb8b8ef025d35aee9edd1d26c3abe5051d47da9286232afbf8/xxhash-3.8.1-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:08ea2081f5e88615fec8622a9f87fbe21b8ea58d88cfc02163ca11026ee62a92", size = 32697, upload-time = "2026-07-06T10:45:57.288Z" }, + { url = "https://files.pythonhosted.org/packages/38/5a/3d3994346e1f45493679cb5c1ffc2bf454e410e9d1e8a662d253becee91e/xxhash-3.8.1-cp313-cp313t-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:2e32855b6f9e5b18f449e59d45e3d5778bdeb660632ef2693cca267a11246c75", size = 225954, upload-time = "2026-07-06T10:45:58.897Z" }, + { url = "https://files.pythonhosted.org/packages/3f/2c/53169270309b7cd8e05504e07fe123bac053b89d00ac63617faacf0a2ec0/xxhash-3.8.1-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a6e088bd7870775624256a0d84c2a6714afd223b2eeb56b0ca58398e52a32fda", size = 249776, upload-time = "2026-07-06T10:46:00.977Z" }, + { url = "https://files.pythonhosted.org/packages/70/e0/5c551d8d592f944506f7c5185e210255c15e672a3c6008c156a1bd9b775e/xxhash-3.8.1-cp313-cp313t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:72eb5ae575cc7ae2b23f6f8064a8b10f638c7149819ae9cc6d20ebd4d37a1629", size = 274776, upload-time = "2026-07-06T10:46:02.869Z" }, + { url = "https://files.pythonhosted.org/packages/a0/2a/d3a762270cee2d7bcd0e25e28c623e5f3f5c0dc637b66e3e47dd5b0bb3f0/xxhash-3.8.1-cp313-cp313t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:d0b48cdf690a64cedf7258c3dc9506cc41fc86edd7739c40e3098952265dc068", size = 252056, upload-time = "2026-07-06T10:46:04.688Z" }, + { url = "https://files.pythonhosted.org/packages/c1/8f/b78e4373b2cb6d1c42af60ea2d7e9146ad0710b239ac7f706d5d31d5bb98/xxhash-3.8.1-cp313-cp313t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:fb9e256a357dfcede7818c6d34e70db2d6b664394803d1de4b6984d2de76c0f1", size = 482108, upload-time = "2026-07-06T10:46:06.498Z" }, + { url = "https://files.pythonhosted.org/packages/e6/0d/642d923336ea61a15f8ce64fc7e078729e6e06c3a026e517fa79b2c23b7a/xxhash-3.8.1-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:51f71a6e2ad071e70c937e41fcb6c19f82c3f9f49831eba850ed4a106ffbb647", size = 226739, upload-time = "2026-07-06T10:46:08.598Z" }, + { url = "https://files.pythonhosted.org/packages/a6/0a/a37d6da6427d45a8d23e3ee3a0ca9c9d4a90364849c6637fe2963a755f9b/xxhash-3.8.1-cp313-cp313t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:e4a6443968c4e8dc69967e12776776a5952c119cc1bd94168ad1c5ad667c2be1", size = 319658, upload-time = "2026-07-06T10:46:10.504Z" }, + { url = "https://files.pythonhosted.org/packages/4a/51/ebbd40da8a3f1bc53b4b7a9a87f8e28bd95c5f21bc14b8a57860cf367d1b/xxhash-3.8.1-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:714503083a1f2065c9ad15340dd49ac8a8e948a505a705ffa1750cb951519113", size = 246059, upload-time = "2026-07-06T10:46:12.634Z" }, + { url = "https://files.pythonhosted.org/packages/24/4c/d9014030147e1f0bb26e7da47aa240dd9ec61c763c573e558111d869f8e1/xxhash-3.8.1-cp313-cp313t-musllinux_1_2_armv7l.whl", hash = "sha256:77f74e45a1e5574bbbf80181c8027b3a4c65c2248fffbd557bd596fff13102f9", size = 275535, upload-time = "2026-07-06T10:46:14.614Z" }, + { url = "https://files.pythonhosted.org/packages/84/86/caee2db41fadcd5a25aa4323213f9afec5a8586d4e419241e3d659362bd7/xxhash-3.8.1-cp313-cp313t-musllinux_1_2_i686.whl", hash = "sha256:4e0e1b0fb0259c1b75d1251ac0bb4d7ab675d36f7a6bf4ba6aa630dae94f9ffa", size = 231292, upload-time = "2026-07-06T10:46:16.452Z" }, + { url = "https://files.pythonhosted.org/packages/0b/60/f52f08bcdc904c4514ea5c25caa19e9f3214144434a6ff96dc82dc1cbddd/xxhash-3.8.1-cp313-cp313t-musllinux_1_2_ppc64le.whl", hash = "sha256:10e4393ec33633c2f05ad01869e546ad080b1a18f2650503731f153774608b31", size = 250490, upload-time = "2026-07-06T10:46:18.318Z" }, + { url = "https://files.pythonhosted.org/packages/24/a0/94dc7ae310838f250669c6ad7168e6d6fca17d49dac1053f06dc232c4a56/xxhash-3.8.1-cp313-cp313t-musllinux_1_2_riscv64.whl", hash = "sha256:b3ba794c3d885803db6c3116686923f1ec13bc86e621e169a375282b63ea1cc6", size = 309861, upload-time = "2026-07-06T10:46:20.503Z" }, + { url = "https://files.pythonhosted.org/packages/8b/f9/adeead7d0eb28cdfc2832544ea639ffbc6749ccde47a8e228d667459182e/xxhash-3.8.1-cp313-cp313t-musllinux_1_2_s390x.whl", hash = "sha256:57189a69c0891e4818853feaa521c972d22c880a001453addea015f48e3c3398", size = 448739, upload-time = "2026-07-06T10:46:22.79Z" }, + { url = "https://files.pythonhosted.org/packages/04/a4/22ec0e07db57d901c9298ae98aa3cf2be45bafded6f07c13131e85b89032/xxhash-3.8.1-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:d59e71153fe9ff85648d00e18649b07e9b22c797291abb7e27274fa06df8b838", size = 223657, upload-time = "2026-07-06T10:46:24.831Z" }, + { url = "https://files.pythonhosted.org/packages/94/32/8a9531f37b59e5a013003db7cb7414baf4ce7e0e1268e0d5947cd3d6a2df/xxhash-3.8.1-cp313-cp313t-win32.whl", hash = "sha256:5b96f0024e9840f449bd91b2d005c921a4b666055a0d1b6492463799f32aae22", size = 32377, upload-time = "2026-07-06T10:46:26.86Z" }, + { url = "https://files.pythonhosted.org/packages/e7/ab/2ca45fd7f671de5f81fc297ef1c95080b40c86ec6be0cc6034b8f7707ac8/xxhash-3.8.1-cp313-cp313t-win_amd64.whl", hash = "sha256:37d5a56c36dcc0b9a87b814cd992598d33863ff683749de6c86081f278d5e629", size = 33274, upload-time = "2026-07-06T10:46:28.39Z" }, + { url = "https://files.pythonhosted.org/packages/5a/54/20d7163463ddb6438b73a427d1655a77a502cf9b9b0c3ada3599629d9c0a/xxhash-3.8.1-cp313-cp313t-win_arm64.whl", hash = "sha256:6696c8752aded28ff3b16f33ef28ce28fb5d209b80c206746f943199fcf5fd65", size = 29375, upload-time = "2026-07-06T10:46:29.962Z" }, + { url = "https://files.pythonhosted.org/packages/c2/8b/df2ba04f22a6cd6b39f96a6577329a8471a55c90ef8d8e2f7c102363613f/xxhash-3.8.1-cp314-cp314-android_24_arm64_v8a.whl", hash = "sha256:9db455cb649dcfe4504d6d68a6d83a7315a99a3ca59871dc3ff840671f99adba", size = 38430, upload-time = "2026-07-06T10:46:31.496Z" }, + { url = "https://files.pythonhosted.org/packages/b2/4f/6a059e8ad3ca8deedc91dfe335b211204900895152212c03ebbe721de68b/xxhash-3.8.1-cp314-cp314-android_24_x86_64.whl", hash = "sha256:affb37f152e55b5e4494bb9d0107f7bb08515c6704fbed82d9f61214d74adc17", size = 36558, upload-time = "2026-07-06T10:46:33.078Z" }, + { url = "https://files.pythonhosted.org/packages/cb/95/40be178205acce092ae418feb20ac737b32a02c7b864926ed0717354c9f8/xxhash-3.8.1-cp314-cp314-ios_13_0_arm64_iphoneos.whl", hash = "sha256:460261045936975193bfd20549a0de1cd52a33b405cbb972f0d80940c42266cd", size = 31181, upload-time = "2026-07-06T10:46:34.793Z" }, + { url = "https://files.pythonhosted.org/packages/3f/89/2da4dbf051bafa156c0e3f12012db2b0ac3b84ff37ca1f021f6bfffcdfbb/xxhash-3.8.1-cp314-cp314-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:38c887aedb696ef8bca19983206d270848558cfae4a91afa6a2fb05dde58ffc5", size = 32192, upload-time = "2026-07-06T10:46:36.393Z" }, + { url = "https://files.pythonhosted.org/packages/7c/4e/e000bbae3566bc8e0be771a8a0f294aa99075e3f0bc4ef43922ebffdebc8/xxhash-3.8.1-cp314-cp314-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:594131ce1aad18db3689781f806db1b065cdaa04f4df36b4c038d2013aefd0bf", size = 34691, upload-time = "2026-07-06T10:46:38.1Z" }, + { url = "https://files.pythonhosted.org/packages/b4/4a/ea954aacc7d1c8711880ac2b55da94429a9b4296b151c4fc0966549ca1ee/xxhash-3.8.1-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:78c794b643d214f1522e7a288bcf5a2de120d26cd170516749a4009dc92722c9", size = 34807, upload-time = "2026-07-06T10:46:39.647Z" }, + { url = "https://files.pythonhosted.org/packages/ca/29/df598e738ff37558ac627264deb2e560902d9bf7f46d3bd5175c9eee593e/xxhash-3.8.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:af0c9fedc4a2c24e8664953882fe8185f3790b8338c9c700f76f5ad660817711", size = 32410, upload-time = "2026-07-06T10:46:41.359Z" }, + { url = "https://files.pythonhosted.org/packages/59/9c/81ab40e7d33ada0b3df5d1bc884894d15dbf4f805cd645b685e4606bb8e0/xxhash-3.8.1-cp314-cp314-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:115772daeb71b2f3b9381177017f53e6cf3f3439c840737fdabd21aba6e54920", size = 220564, upload-time = "2026-07-06T10:46:43.463Z" }, + { url = "https://files.pythonhosted.org/packages/fd/6f/62ae6f5c8606320a0e2a41c2dc8c6d91cc5d63d0f84dd9582e9543779dd8/xxhash-3.8.1-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:000435984a0469b0f822fe76f35bddea0f96a4d6521b3339a60a6428cdee1edc", size = 241462, upload-time = "2026-07-06T10:46:45.509Z" }, + { url = "https://files.pythonhosted.org/packages/15/a1/9c3a0ec6cb524396f551eddd102a76690a795494eb9784fc67542b0daa37/xxhash-3.8.1-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:2f1c68394818e0595569c2ff3cbc1e6d5a36a434e796f5c526b987b80c8a8c62", size = 264491, upload-time = "2026-07-06T10:46:47.655Z" }, + { url = "https://files.pythonhosted.org/packages/64/f2/700a4674e4308eb59d2fdb973977e82eae231bea5044753fee5c9eec0e0c/xxhash-3.8.1-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:46b39976d008e2a845758650f0ff7136bca004f40da0c8798bd37ac37860154f", size = 242905, upload-time = "2026-07-06T10:46:49.857Z" }, + { url = "https://files.pythonhosted.org/packages/f3/8a/72d9874375c8d4cbc64a8cd1d659d5695a8765c3db82efa82dc5bd9f14d0/xxhash-3.8.1-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:d5006c65ec507a333479e76e00e2c368781f16c24ededa764763956b32a0e93e", size = 473873, upload-time = "2026-07-06T10:46:51.953Z" }, + { url = "https://files.pythonhosted.org/packages/03/f0/6db07590ed7e0a77f186ef0bcea8d52553bf1ba57833e09467a2411f0f2d/xxhash-3.8.1-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c31a2649bcf1fe97cf11c79848d761df33ac46b3896942d31b640557b486ff6b", size = 220765, upload-time = "2026-07-06T10:46:55.41Z" }, + { url = "https://files.pythonhosted.org/packages/8f/10/00d12d8b8beabbf49a8bbc626fb9f40445145a8887eb41a6acfb69149ac4/xxhash-3.8.1-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:8f759eed402448c2bdbb492e4fba1f20668ffe29688605ea61f0f67f9e4e386d", size = 310478, upload-time = "2026-07-06T10:46:57.729Z" }, + { url = "https://files.pythonhosted.org/packages/1f/f9/12a82394eefb0f185d15a7f7b9f627c61c475a72dd83718436a5b84b42ac/xxhash-3.8.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:7b5f97ecfede10d5b2870383620e2d25c8561e217c7bf9081073802b54248d2b", size = 238393, upload-time = "2026-07-06T10:46:59.87Z" }, + { url = "https://files.pythonhosted.org/packages/20/f3/53f963e320b9ce678337aa7273f39ce692ded8b99e3d22a866ec722159ab/xxhash-3.8.1-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:1da930bbcac3e8fbe2191850e2abb57977a99348c12c4b385e1058ac1b0a9ecc", size = 268704, upload-time = "2026-07-06T10:47:01.806Z" }, + { url = "https://files.pythonhosted.org/packages/0a/50/5b5badbd87c82d9f9b5f58ac74a3f29ef08f6fc387b324b8fd482450b862/xxhash-3.8.1-cp314-cp314-musllinux_1_2_i686.whl", hash = "sha256:747476436f6891b9773374ce8d48edcc8b12cb5b61b67c6fb6289633747d088f", size = 225015, upload-time = "2026-07-06T10:47:03.784Z" }, + { url = "https://files.pythonhosted.org/packages/30/93/3ca68265afe7b4e69435e08a7b6a1d9d0f2a071e889da1f8041ed00fe878/xxhash-3.8.1-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:4ef09bbc2519a93cd0f95f2ceb5f7b85919dffea643278e02362bf40e3c4bed1", size = 240951, upload-time = "2026-07-06T10:47:05.816Z" }, + { url = "https://files.pythonhosted.org/packages/dd/a6/27e19670c40f46b5e76e11f2f4713d21054804568425d870670e757172ad/xxhash-3.8.1-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:a5eed9d41995a83f3332b4e3396abb7f433cac584222bd7e305b606d8353861e", size = 300751, upload-time = "2026-07-06T10:47:07.95Z" }, + { url = "https://files.pythonhosted.org/packages/bc/fb/b33e27689959fe7ed2ae0b830af41560d65213943983afa9db3a8d481bce/xxhash-3.8.1-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:53f3ed9118397074ff63a79b66b7fec1c84c782eecde35c5bc94e420a971c231", size = 443480, upload-time = "2026-07-06T10:47:10Z" }, + { url = "https://files.pythonhosted.org/packages/26/60/0e0d973be5fe280753ef02fbc89349492ad6e903bf1dcb870b668f94b662/xxhash-3.8.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:d247b34bf433c92b41689318fd25d246313cab2275a6a47e2efac178b80d6efe", size = 217657, upload-time = "2026-07-06T10:47:12.196Z" }, + { url = "https://files.pythonhosted.org/packages/ad/68/c9e3ecef4a9a417d464cb5bd200aa12f73192dee677901b9e08e0ad0d1bb/xxhash-3.8.1-cp314-cp314-win32.whl", hash = "sha256:d58ce8b6cfa9c4d2f230557f69caf7c06369e318015d0b19485095bc2c5963ab", size = 32690, upload-time = "2026-07-06T10:47:14.204Z" }, + { url = "https://files.pythonhosted.org/packages/d7/99/e9e44588c0b62837bbec5ba7927816de0afa03406b1a0b6c7a7e1d1a30a0/xxhash-3.8.1-cp314-cp314-win_amd64.whl", hash = "sha256:6cee733fe4ccb1737e0997135283c82341e5cfa9cf214b165f9087fb663aaf4f", size = 33460, upload-time = "2026-07-06T10:47:16.021Z" }, + { url = "https://files.pythonhosted.org/packages/45/2b/64f36d86380b3657ad9031967ab814f3ef31307174650853f69c18932ebc/xxhash-3.8.1-cp314-cp314-win_arm64.whl", hash = "sha256:58346024d47e84f7d8b3e7f5d6faa1d58acbbe49a8771497872059f58c1d8ea5", size = 30092, upload-time = "2026-07-06T10:47:17.81Z" }, + { url = "https://files.pythonhosted.org/packages/92/cb/18b64bff88c58a0ca209dc533e63cf02d7ae5aa6b1b9a9fd14e81b5dbd60/xxhash-3.8.1-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:01cab782f8a0a05ecad2c63d7ef10f7ab475f660e0d6419d069418c14d88de7c", size = 35024, upload-time = "2026-07-06T10:47:19.821Z" }, + { url = "https://files.pythonhosted.org/packages/af/1d/72d8a70520e5dcddb472ea0486d299da3240745a10658290cd7b5690ede2/xxhash-3.8.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:717b12fdc51819833704e85e6926d76981ffa3f780ef92e33ebb8b26d46bb230", size = 32697, upload-time = "2026-07-06T10:47:21.649Z" }, + { url = "https://files.pythonhosted.org/packages/9c/b8/e041f555903c56db3d0a731b3d72a6575d75e0ed868b1bd2e5176111ca44/xxhash-3.8.1-cp314-cp314t-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:ec55d80e9b8a519d742669e0b49e8ce9e6747be42bf3c138158b6543a9c8e489", size = 226044, upload-time = "2026-07-06T10:47:23.612Z" }, + { url = "https://files.pythonhosted.org/packages/3a/7e/5cdcf06bf6ec4b5d2ac073feb23432ec1d603fd438864cbd2c09c7cb45e1/xxhash-3.8.1-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:98d8ac1129b4dd39098cffed94d1284aceb61c3aa396757ccc736ac392e4cee5", size = 249899, upload-time = "2026-07-06T10:47:25.812Z" }, + { url = "https://files.pythonhosted.org/packages/c0/c0/eb7e059cb5e1dba11fd30d2fdf882f56e5a417a3eaa43669d43623767f45/xxhash-3.8.1-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:3bc0fa90830df1e1277f33cc6e55de9990b83c0319fd8c7412866cfde38b025e", size = 274892, upload-time = "2026-07-06T10:47:27.931Z" }, + { url = "https://files.pythonhosted.org/packages/66/74/a600aaf7cd39957fd1510adeedb1749c1e7eb82bd632a1153d9c664c3135/xxhash-3.8.1-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:c73b6f652f0745425aa6378319c331293b5341756262e9408ed3d45f183375e6", size = 252243, upload-time = "2026-07-06T10:47:30.288Z" }, + { url = "https://files.pythonhosted.org/packages/ad/04/78d88fa75a6763e5d09bf1b947a392a27988903381b219006f92f3c68fc8/xxhash-3.8.1-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:f6114692261eff4266386cdec0f7d87eee24e317ab397c218b7ae6a76b4c6339", size = 482191, upload-time = "2026-07-06T10:47:32.45Z" }, + { url = "https://files.pythonhosted.org/packages/7f/06/07a8aea1108d682de8791ce608cdf367d75ff4e7e57cd3c154bdc6f47b23/xxhash-3.8.1-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:4df57c0b161ec1b3ed0526a67b0db0914b557e86ee8aae51887aec941b261542", size = 226877, upload-time = "2026-07-06T10:47:34.705Z" }, + { url = "https://files.pythonhosted.org/packages/ed/b5/86bade5618a524d2c06c4041aa2fe8e5749ce16e88afba60d67c1684a21f/xxhash-3.8.1-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:9043877a917be88ccf230aa5667c1bd059bce80f4c2727e4defa1b29b7f48b08", size = 319794, upload-time = "2026-07-06T10:47:37.08Z" }, + { url = "https://files.pythonhosted.org/packages/23/69/9b1a2b89b1621bb740fbcb7beb512f60f99480c1bdc680c0c90e1f56ff75/xxhash-3.8.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:559e3cabe522231909f9de98ef06929edbd53782046bd21aae0c72db6f2a0775", size = 246202, upload-time = "2026-07-06T10:47:39.676Z" }, + { url = "https://files.pythonhosted.org/packages/08/ea/662ed6cb49f1d34078b6a3a3e0f3d29ff93fd7b5a03c0bc9ecfd9b2159c3/xxhash-3.8.1-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:264710bd335016f303763ce1275c6486df30bb57c2245c91b224c983d7ac39b8", size = 275628, upload-time = "2026-07-06T10:47:41.99Z" }, + { url = "https://files.pythonhosted.org/packages/13/f5/49fc9e4c6728a5a3bd8fe639199d2fa67609b3a84f938aff6e8568dd3e4f/xxhash-3.8.1-cp314-cp314t-musllinux_1_2_i686.whl", hash = "sha256:e14800b9b10bb39d7a60ad4a310e403164d7b8988a27ae933d4e40618a44088e", size = 231390, upload-time = "2026-07-06T10:47:44.233Z" }, + { url = "https://files.pythonhosted.org/packages/64/9d/3acaf8f599c0e0b30e910a3a11ba32929da53c86dc73c7c55fe6a010b4e9/xxhash-3.8.1-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:ea6a3e734b0fd41b82784a400be946821900daebe610c050a5e0760838a34f99", size = 250600, upload-time = "2026-07-06T10:47:47.611Z" }, + { url = "https://files.pythonhosted.org/packages/23/64/8acab4c5ec60dbe664b5b9858fd44c2413b07e535b09556a0a5022e78aa6/xxhash-3.8.1-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:cf399fac542a1c7a4734a435b93df2c55e858c7d31abf6c1bdf46f9ae67fbfd0", size = 310032, upload-time = "2026-07-06T10:47:49.88Z" }, + { url = "https://files.pythonhosted.org/packages/56/47/a0288d7329b1fe63e2734a32d19d444a96ae2b4810f545bc61e561224917/xxhash-3.8.1-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:44c89d915a75c11d2547eaee9098fcd80398987c4bff2974a0497a925bf92c07", size = 448882, upload-time = "2026-07-06T10:47:52.631Z" }, + { url = "https://files.pythonhosted.org/packages/01/e7/3071dfd3beb5c38204ce1cf56bf7749fce08de900fa92714b81d1d8ca1f2/xxhash-3.8.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:358650d5bda9c635da699c53adf4e8134af492ecc79c960f917eebf088bb6799", size = 223728, upload-time = "2026-07-06T10:47:55.093Z" }, + { url = "https://files.pythonhosted.org/packages/12/11/b99949f0ba2b07e9f9ffe83b9c86faa685f9080725dc21a916a607313be5/xxhash-3.8.1-cp314-cp314t-win32.whl", hash = "sha256:c240939e963653054fc7e4a17c382829cda4aa88a7daf0af841715dbded1b497", size = 33150, upload-time = "2026-07-06T10:47:57.274Z" }, + { url = "https://files.pythonhosted.org/packages/54/1c/09703eb341f8416e74e58d6c6732d4b5c46de59c942363203cb237cc95b0/xxhash-3.8.1-cp314-cp314t-win_amd64.whl", hash = "sha256:7258ee276e8772599bc19e14b36f6260306e21b637190cd7cb489a2449d48684", size = 34005, upload-time = "2026-07-06T10:47:59.434Z" }, + { url = "https://files.pythonhosted.org/packages/d6/f9/6ed7251bb6a8af10ac73b1821c60583d2826e5b2064e45a979c935287c98/xxhash-3.8.1-cp314-cp314t-win_arm64.whl", hash = "sha256:8f454166c2ffed45636c8d501741e649851ba2f346c4eb73a64c07ac00428f20", size = 30239, upload-time = "2026-07-06T10:48:01.874Z" }, + { url = "https://files.pythonhosted.org/packages/99/e4/4d8040435aeac814fc69ba63621565fbeb19229a138e2568324a26b2a45c/xxhash-3.8.1-pp311-pypy311_pp73-macosx_10_15_x86_64.whl", hash = "sha256:39c9d5b61508b0bb68f29e54546de0ed2a74943c6a18585535a7e37356f1dd12", size = 32687, upload-time = "2026-07-06T10:49:42.803Z" }, + { url = "https://files.pythonhosted.org/packages/da/6a/975f1f2318c760e5bcec109ed379713ae645d8d856c2a3b9ec5d26857087/xxhash-3.8.1-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:83b9130b80b216d56fdf9e87131946b353c9627930c061955a101ea82b09fed9", size = 29879, upload-time = "2026-07-06T10:49:45.172Z" }, + { url = "https://files.pythonhosted.org/packages/08/0b/40a2a55ff52cf635bfdc5eae67a772bec85b4f44c6c737f73f6f528d51d1/xxhash-3.8.1-pp311-pypy311_pp73-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:8304be0982130954b7fd3aad18e2c6f8ee40254bc3d2e635991c16d77c91e2bd", size = 43246, upload-time = "2026-07-06T10:49:47.905Z" }, + { url = "https://files.pythonhosted.org/packages/9c/6d/56ed2b6b200f26fb474f3fd387d95d0601efcd5bb33430c90c68924bdd77/xxhash-3.8.1-pp311-pypy311_pp73-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:4b512261801b1e5fde7b6ebf2fef7977339c620cbbca88a0040ad9ad134f4d02", size = 38202, upload-time = "2026-07-06T10:49:50.59Z" }, + { url = "https://files.pythonhosted.org/packages/0d/a3/56864d895d1161a9f17502088e9c1fb7c06bde2c2efdde620d22bb7a9c43/xxhash-3.8.1-pp311-pypy311_pp73-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:49aa8692507835dcc1e8ad8021f20c74c2dc13d83b5112e87877faa2a0035b20", size = 34448, upload-time = "2026-07-06T10:49:53.242Z" }, + { url = "https://files.pythonhosted.org/packages/6b/57/5c6e0908a47f61dca96d01c8ee6fce01ed1050611eb779083ba8758fed81/xxhash-3.8.1-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:345b07b78e2bf583d71682aa34ae5b5fab575f7a1cb31e10263ebbc6f89f8c42", size = 32869, upload-time = "2026-07-06T10:49:55.972Z" }, ] [[package]] name = "yarl" -version = "1.20.1" +version = "1.24.2" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "idna" }, { name = "multidict" }, { name = "propcache" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/3c/fb/efaa23fa4e45537b827620f04cf8f3cd658b76642205162e072703a5b963/yarl-1.20.1.tar.gz", hash = "sha256:d017a4997ee50c91fd5466cef416231bb82177b93b029906cefc542ce14c35ac", size = 186428, upload-time = "2025-06-10T00:46:09.923Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/cb/65/7fed0d774abf47487c64be14e9223749468922817b5e8792b8a64792a1bb/yarl-1.20.1-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:6032e6da6abd41e4acda34d75a816012717000fa6839f37124a47fcefc49bec4", size = 132910, upload-time = "2025-06-10T00:42:31.108Z" }, - { url = "https://files.pythonhosted.org/packages/8a/7b/988f55a52da99df9e56dc733b8e4e5a6ae2090081dc2754fc8fd34e60aa0/yarl-1.20.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:2c7b34d804b8cf9b214f05015c4fee2ebe7ed05cf581e7192c06555c71f4446a", size = 90644, upload-time = "2025-06-10T00:42:33.851Z" }, - { url = "https://files.pythonhosted.org/packages/f7/de/30d98f03e95d30c7e3cc093759982d038c8833ec2451001d45ef4854edc1/yarl-1.20.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:0c869f2651cc77465f6cd01d938d91a11d9ea5d798738c1dc077f3de0b5e5fed", size = 89322, upload-time = "2025-06-10T00:42:35.688Z" }, - { url = "https://files.pythonhosted.org/packages/e0/7a/f2f314f5ebfe9200724b0b748de2186b927acb334cf964fd312eb86fc286/yarl-1.20.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:62915e6688eb4d180d93840cda4110995ad50c459bf931b8b3775b37c264af1e", size = 323786, upload-time = "2025-06-10T00:42:37.817Z" }, - { url = "https://files.pythonhosted.org/packages/15/3f/718d26f189db96d993d14b984ce91de52e76309d0fd1d4296f34039856aa/yarl-1.20.1-cp310-cp310-manylinux_2_17_armv7l.manylinux2014_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:41ebd28167bc6af8abb97fec1a399f412eec5fd61a3ccbe2305a18b84fb4ca73", size = 319627, upload-time = "2025-06-10T00:42:39.937Z" }, - { url = "https://files.pythonhosted.org/packages/a5/76/8fcfbf5fa2369157b9898962a4a7d96764b287b085b5b3d9ffae69cdefd1/yarl-1.20.1-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:21242b4288a6d56f04ea193adde174b7e347ac46ce6bc84989ff7c1b1ecea84e", size = 339149, upload-time = "2025-06-10T00:42:42.627Z" }, - { url = "https://files.pythonhosted.org/packages/3c/95/d7fc301cc4661785967acc04f54a4a42d5124905e27db27bb578aac49b5c/yarl-1.20.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:bea21cdae6c7eb02ba02a475f37463abfe0a01f5d7200121b03e605d6a0439f8", size = 333327, upload-time = "2025-06-10T00:42:44.842Z" }, - { url = "https://files.pythonhosted.org/packages/65/94/e21269718349582eee81efc5c1c08ee71c816bfc1585b77d0ec3f58089eb/yarl-1.20.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1f8a891e4a22a89f5dde7862994485e19db246b70bb288d3ce73a34422e55b23", size = 326054, upload-time = "2025-06-10T00:42:47.149Z" }, - { url = "https://files.pythonhosted.org/packages/32/ae/8616d1f07853704523519f6131d21f092e567c5af93de7e3e94b38d7f065/yarl-1.20.1-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:dd803820d44c8853a109a34e3660e5a61beae12970da479cf44aa2954019bf70", size = 315035, upload-time = "2025-06-10T00:42:48.852Z" }, - { url = "https://files.pythonhosted.org/packages/48/aa/0ace06280861ef055855333707db5e49c6e3a08840a7ce62682259d0a6c0/yarl-1.20.1-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:b982fa7f74c80d5c0c7b5b38f908971e513380a10fecea528091405f519b9ebb", size = 338962, upload-time = "2025-06-10T00:42:51.024Z" }, - { url = "https://files.pythonhosted.org/packages/20/52/1e9d0e6916f45a8fb50e6844f01cb34692455f1acd548606cbda8134cd1e/yarl-1.20.1-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:33f29ecfe0330c570d997bcf1afd304377f2e48f61447f37e846a6058a4d33b2", size = 335399, upload-time = "2025-06-10T00:42:53.007Z" }, - { url = "https://files.pythonhosted.org/packages/f2/65/60452df742952c630e82f394cd409de10610481d9043aa14c61bf846b7b1/yarl-1.20.1-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:835ab2cfc74d5eb4a6a528c57f05688099da41cf4957cf08cad38647e4a83b30", size = 338649, upload-time = "2025-06-10T00:42:54.964Z" }, - { url = "https://files.pythonhosted.org/packages/7b/f5/6cd4ff38dcde57a70f23719a838665ee17079640c77087404c3d34da6727/yarl-1.20.1-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:46b5e0ccf1943a9a6e766b2c2b8c732c55b34e28be57d8daa2b3c1d1d4009309", size = 358563, upload-time = "2025-06-10T00:42:57.28Z" }, - { url = "https://files.pythonhosted.org/packages/d1/90/c42eefd79d0d8222cb3227bdd51b640c0c1d0aa33fe4cc86c36eccba77d3/yarl-1.20.1-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:df47c55f7d74127d1b11251fe6397d84afdde0d53b90bedb46a23c0e534f9d24", size = 357609, upload-time = "2025-06-10T00:42:59.055Z" }, - { url = "https://files.pythonhosted.org/packages/03/c8/cea6b232cb4617514232e0f8a718153a95b5d82b5290711b201545825532/yarl-1.20.1-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:76d12524d05841276b0e22573f28d5fbcb67589836772ae9244d90dd7d66aa13", size = 350224, upload-time = "2025-06-10T00:43:01.248Z" }, - { url = "https://files.pythonhosted.org/packages/ce/a3/eaa0ab9712f1f3d01faf43cf6f1f7210ce4ea4a7e9b28b489a2261ca8db9/yarl-1.20.1-cp310-cp310-win32.whl", hash = "sha256:6c4fbf6b02d70e512d7ade4b1f998f237137f1417ab07ec06358ea04f69134f8", size = 81753, upload-time = "2025-06-10T00:43:03.486Z" }, - { url = "https://files.pythonhosted.org/packages/8f/34/e4abde70a9256465fe31c88ed02c3f8502b7b5dead693a4f350a06413f28/yarl-1.20.1-cp310-cp310-win_amd64.whl", hash = "sha256:aef6c4d69554d44b7f9d923245f8ad9a707d971e6209d51279196d8e8fe1ae16", size = 86817, upload-time = "2025-06-10T00:43:05.231Z" }, - { url = "https://files.pythonhosted.org/packages/b1/18/893b50efc2350e47a874c5c2d67e55a0ea5df91186b2a6f5ac52eff887cd/yarl-1.20.1-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:47ee6188fea634bdfaeb2cc420f5b3b17332e6225ce88149a17c413c77ff269e", size = 133833, upload-time = "2025-06-10T00:43:07.393Z" }, - { url = "https://files.pythonhosted.org/packages/89/ed/b8773448030e6fc47fa797f099ab9eab151a43a25717f9ac043844ad5ea3/yarl-1.20.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:d0f6500f69e8402d513e5eedb77a4e1818691e8f45e6b687147963514d84b44b", size = 91070, upload-time = "2025-06-10T00:43:09.538Z" }, - { url = "https://files.pythonhosted.org/packages/e3/e3/409bd17b1e42619bf69f60e4f031ce1ccb29bd7380117a55529e76933464/yarl-1.20.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:7a8900a42fcdaad568de58887c7b2f602962356908eedb7628eaf6021a6e435b", size = 89818, upload-time = "2025-06-10T00:43:11.575Z" }, - { url = "https://files.pythonhosted.org/packages/f8/77/64d8431a4d77c856eb2d82aa3de2ad6741365245a29b3a9543cd598ed8c5/yarl-1.20.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bad6d131fda8ef508b36be3ece16d0902e80b88ea7200f030a0f6c11d9e508d4", size = 347003, upload-time = "2025-06-10T00:43:14.088Z" }, - { url = "https://files.pythonhosted.org/packages/8d/d2/0c7e4def093dcef0bd9fa22d4d24b023788b0a33b8d0088b51aa51e21e99/yarl-1.20.1-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:df018d92fe22aaebb679a7f89fe0c0f368ec497e3dda6cb81a567610f04501f1", size = 336537, upload-time = "2025-06-10T00:43:16.431Z" }, - { url = "https://files.pythonhosted.org/packages/f0/f3/fc514f4b2cf02cb59d10cbfe228691d25929ce8f72a38db07d3febc3f706/yarl-1.20.1-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:8f969afbb0a9b63c18d0feecf0db09d164b7a44a053e78a7d05f5df163e43833", size = 362358, upload-time = "2025-06-10T00:43:18.704Z" }, - { url = "https://files.pythonhosted.org/packages/ea/6d/a313ac8d8391381ff9006ac05f1d4331cee3b1efaa833a53d12253733255/yarl-1.20.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:812303eb4aa98e302886ccda58d6b099e3576b1b9276161469c25803a8db277d", size = 357362, upload-time = "2025-06-10T00:43:20.888Z" }, - { url = "https://files.pythonhosted.org/packages/00/70/8f78a95d6935a70263d46caa3dd18e1f223cf2f2ff2037baa01a22bc5b22/yarl-1.20.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:98c4a7d166635147924aa0bf9bfe8d8abad6fffa6102de9c99ea04a1376f91e8", size = 348979, upload-time = "2025-06-10T00:43:23.169Z" }, - { url = "https://files.pythonhosted.org/packages/cb/05/42773027968968f4f15143553970ee36ead27038d627f457cc44bbbeecf3/yarl-1.20.1-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:12e768f966538e81e6e7550f9086a6236b16e26cd964cf4df35349970f3551cf", size = 337274, upload-time = "2025-06-10T00:43:27.111Z" }, - { url = "https://files.pythonhosted.org/packages/05/be/665634aa196954156741ea591d2f946f1b78ceee8bb8f28488bf28c0dd62/yarl-1.20.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:fe41919b9d899661c5c28a8b4b0acf704510b88f27f0934ac7a7bebdd8938d5e", size = 363294, upload-time = "2025-06-10T00:43:28.96Z" }, - { url = "https://files.pythonhosted.org/packages/eb/90/73448401d36fa4e210ece5579895731f190d5119c4b66b43b52182e88cd5/yarl-1.20.1-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:8601bc010d1d7780592f3fc1bdc6c72e2b6466ea34569778422943e1a1f3c389", size = 358169, upload-time = "2025-06-10T00:43:30.701Z" }, - { url = "https://files.pythonhosted.org/packages/c3/b0/fce922d46dc1eb43c811f1889f7daa6001b27a4005587e94878570300881/yarl-1.20.1-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:daadbdc1f2a9033a2399c42646fbd46da7992e868a5fe9513860122d7fe7a73f", size = 362776, upload-time = "2025-06-10T00:43:32.51Z" }, - { url = "https://files.pythonhosted.org/packages/f1/0d/b172628fce039dae8977fd22caeff3eeebffd52e86060413f5673767c427/yarl-1.20.1-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:03aa1e041727cb438ca762628109ef1333498b122e4c76dd858d186a37cec845", size = 381341, upload-time = "2025-06-10T00:43:34.543Z" }, - { url = "https://files.pythonhosted.org/packages/6b/9b/5b886d7671f4580209e855974fe1cecec409aa4a89ea58b8f0560dc529b1/yarl-1.20.1-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:642980ef5e0fa1de5fa96d905c7e00cb2c47cb468bfcac5a18c58e27dbf8d8d1", size = 379988, upload-time = "2025-06-10T00:43:36.489Z" }, - { url = "https://files.pythonhosted.org/packages/73/be/75ef5fd0fcd8f083a5d13f78fd3f009528132a1f2a1d7c925c39fa20aa79/yarl-1.20.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:86971e2795584fe8c002356d3b97ef6c61862720eeff03db2a7c86b678d85b3e", size = 371113, upload-time = "2025-06-10T00:43:38.592Z" }, - { url = "https://files.pythonhosted.org/packages/50/4f/62faab3b479dfdcb741fe9e3f0323e2a7d5cd1ab2edc73221d57ad4834b2/yarl-1.20.1-cp311-cp311-win32.whl", hash = "sha256:597f40615b8d25812f14562699e287f0dcc035d25eb74da72cae043bb884d773", size = 81485, upload-time = "2025-06-10T00:43:41.038Z" }, - { url = "https://files.pythonhosted.org/packages/f0/09/d9c7942f8f05c32ec72cd5c8e041c8b29b5807328b68b4801ff2511d4d5e/yarl-1.20.1-cp311-cp311-win_amd64.whl", hash = "sha256:26ef53a9e726e61e9cd1cda6b478f17e350fb5800b4bd1cd9fe81c4d91cfeb2e", size = 86686, upload-time = "2025-06-10T00:43:42.692Z" }, - { url = "https://files.pythonhosted.org/packages/5f/9a/cb7fad7d73c69f296eda6815e4a2c7ed53fc70c2f136479a91c8e5fbdb6d/yarl-1.20.1-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:bdcc4cd244e58593a4379fe60fdee5ac0331f8eb70320a24d591a3be197b94a9", size = 133667, upload-time = "2025-06-10T00:43:44.369Z" }, - { url = "https://files.pythonhosted.org/packages/67/38/688577a1cb1e656e3971fb66a3492501c5a5df56d99722e57c98249e5b8a/yarl-1.20.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:b29a2c385a5f5b9c7d9347e5812b6f7ab267193c62d282a540b4fc528c8a9d2a", size = 91025, upload-time = "2025-06-10T00:43:46.295Z" }, - { url = "https://files.pythonhosted.org/packages/50/ec/72991ae51febeb11a42813fc259f0d4c8e0507f2b74b5514618d8b640365/yarl-1.20.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:1112ae8154186dfe2de4732197f59c05a83dc814849a5ced892b708033f40dc2", size = 89709, upload-time = "2025-06-10T00:43:48.22Z" }, - { url = "https://files.pythonhosted.org/packages/99/da/4d798025490e89426e9f976702e5f9482005c548c579bdae792a4c37769e/yarl-1.20.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:90bbd29c4fe234233f7fa2b9b121fb63c321830e5d05b45153a2ca68f7d310ee", size = 352287, upload-time = "2025-06-10T00:43:49.924Z" }, - { url = "https://files.pythonhosted.org/packages/1a/26/54a15c6a567aac1c61b18aa0f4b8aa2e285a52d547d1be8bf48abe2b3991/yarl-1.20.1-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:680e19c7ce3710ac4cd964e90dad99bf9b5029372ba0c7cbfcd55e54d90ea819", size = 345429, upload-time = "2025-06-10T00:43:51.7Z" }, - { url = "https://files.pythonhosted.org/packages/d6/95/9dcf2386cb875b234353b93ec43e40219e14900e046bf6ac118f94b1e353/yarl-1.20.1-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:4a979218c1fdb4246a05efc2cc23859d47c89af463a90b99b7c56094daf25a16", size = 365429, upload-time = "2025-06-10T00:43:53.494Z" }, - { url = "https://files.pythonhosted.org/packages/91/b2/33a8750f6a4bc224242a635f5f2cff6d6ad5ba651f6edcccf721992c21a0/yarl-1.20.1-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:255b468adf57b4a7b65d8aad5b5138dce6a0752c139965711bdcb81bc370e1b6", size = 363862, upload-time = "2025-06-10T00:43:55.766Z" }, - { url = "https://files.pythonhosted.org/packages/98/28/3ab7acc5b51f4434b181b0cee8f1f4b77a65919700a355fb3617f9488874/yarl-1.20.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a97d67108e79cfe22e2b430d80d7571ae57d19f17cda8bb967057ca8a7bf5bfd", size = 355616, upload-time = "2025-06-10T00:43:58.056Z" }, - { url = "https://files.pythonhosted.org/packages/36/a3/f666894aa947a371724ec7cd2e5daa78ee8a777b21509b4252dd7bd15e29/yarl-1.20.1-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:8570d998db4ddbfb9a590b185a0a33dbf8aafb831d07a5257b4ec9948df9cb0a", size = 339954, upload-time = "2025-06-10T00:43:59.773Z" }, - { url = "https://files.pythonhosted.org/packages/f1/81/5f466427e09773c04219d3450d7a1256138a010b6c9f0af2d48565e9ad13/yarl-1.20.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:97c75596019baae7c71ccf1d8cc4738bc08134060d0adfcbe5642f778d1dca38", size = 365575, upload-time = "2025-06-10T00:44:02.051Z" }, - { url = "https://files.pythonhosted.org/packages/2e/e3/e4b0ad8403e97e6c9972dd587388940a032f030ebec196ab81a3b8e94d31/yarl-1.20.1-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:1c48912653e63aef91ff988c5432832692ac5a1d8f0fb8a33091520b5bbe19ef", size = 365061, upload-time = "2025-06-10T00:44:04.196Z" }, - { url = "https://files.pythonhosted.org/packages/ac/99/b8a142e79eb86c926f9f06452eb13ecb1bb5713bd01dc0038faf5452e544/yarl-1.20.1-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:4c3ae28f3ae1563c50f3d37f064ddb1511ecc1d5584e88c6b7c63cf7702a6d5f", size = 364142, upload-time = "2025-06-10T00:44:06.527Z" }, - { url = "https://files.pythonhosted.org/packages/34/f2/08ed34a4a506d82a1a3e5bab99ccd930a040f9b6449e9fd050320e45845c/yarl-1.20.1-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:c5e9642f27036283550f5f57dc6156c51084b458570b9d0d96100c8bebb186a8", size = 381894, upload-time = "2025-06-10T00:44:08.379Z" }, - { url = "https://files.pythonhosted.org/packages/92/f8/9a3fbf0968eac704f681726eff595dce9b49c8a25cd92bf83df209668285/yarl-1.20.1-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:2c26b0c49220d5799f7b22c6838409ee9bc58ee5c95361a4d7831f03cc225b5a", size = 383378, upload-time = "2025-06-10T00:44:10.51Z" }, - { url = "https://files.pythonhosted.org/packages/af/85/9363f77bdfa1e4d690957cd39d192c4cacd1c58965df0470a4905253b54f/yarl-1.20.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:564ab3d517e3d01c408c67f2e5247aad4019dcf1969982aba3974b4093279004", size = 374069, upload-time = "2025-06-10T00:44:12.834Z" }, - { url = "https://files.pythonhosted.org/packages/35/99/9918c8739ba271dcd935400cff8b32e3cd319eaf02fcd023d5dcd487a7c8/yarl-1.20.1-cp312-cp312-win32.whl", hash = "sha256:daea0d313868da1cf2fac6b2d3a25c6e3a9e879483244be38c8e6a41f1d876a5", size = 81249, upload-time = "2025-06-10T00:44:14.731Z" }, - { url = "https://files.pythonhosted.org/packages/eb/83/5d9092950565481b413b31a23e75dd3418ff0a277d6e0abf3729d4d1ce25/yarl-1.20.1-cp312-cp312-win_amd64.whl", hash = "sha256:48ea7d7f9be0487339828a4de0360d7ce0efc06524a48e1810f945c45b813698", size = 86710, upload-time = "2025-06-10T00:44:16.716Z" }, - { url = "https://files.pythonhosted.org/packages/8a/e1/2411b6d7f769a07687acee88a062af5833cf1966b7266f3d8dfb3d3dc7d3/yarl-1.20.1-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:0b5ff0fbb7c9f1b1b5ab53330acbfc5247893069e7716840c8e7d5bb7355038a", size = 131811, upload-time = "2025-06-10T00:44:18.933Z" }, - { url = "https://files.pythonhosted.org/packages/b2/27/584394e1cb76fb771371770eccad35de400e7b434ce3142c2dd27392c968/yarl-1.20.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:14f326acd845c2b2e2eb38fb1346c94f7f3b01a4f5c788f8144f9b630bfff9a3", size = 90078, upload-time = "2025-06-10T00:44:20.635Z" }, - { url = "https://files.pythonhosted.org/packages/bf/9a/3246ae92d4049099f52d9b0fe3486e3b500e29b7ea872d0f152966fc209d/yarl-1.20.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:f60e4ad5db23f0b96e49c018596707c3ae89f5d0bd97f0ad3684bcbad899f1e7", size = 88748, upload-time = "2025-06-10T00:44:22.34Z" }, - { url = "https://files.pythonhosted.org/packages/a3/25/35afe384e31115a1a801fbcf84012d7a066d89035befae7c5d4284df1e03/yarl-1.20.1-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:49bdd1b8e00ce57e68ba51916e4bb04461746e794e7c4d4bbc42ba2f18297691", size = 349595, upload-time = "2025-06-10T00:44:24.314Z" }, - { url = "https://files.pythonhosted.org/packages/28/2d/8aca6cb2cabc8f12efcb82749b9cefecbccfc7b0384e56cd71058ccee433/yarl-1.20.1-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:66252d780b45189975abfed839616e8fd2dbacbdc262105ad7742c6ae58f3e31", size = 342616, upload-time = "2025-06-10T00:44:26.167Z" }, - { url = "https://files.pythonhosted.org/packages/0b/e9/1312633d16b31acf0098d30440ca855e3492d66623dafb8e25b03d00c3da/yarl-1.20.1-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:59174e7332f5d153d8f7452a102b103e2e74035ad085f404df2e40e663a22b28", size = 361324, upload-time = "2025-06-10T00:44:27.915Z" }, - { url = "https://files.pythonhosted.org/packages/bc/a0/688cc99463f12f7669eec7c8acc71ef56a1521b99eab7cd3abb75af887b0/yarl-1.20.1-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:e3968ec7d92a0c0f9ac34d5ecfd03869ec0cab0697c91a45db3fbbd95fe1b653", size = 359676, upload-time = "2025-06-10T00:44:30.041Z" }, - { url = "https://files.pythonhosted.org/packages/af/44/46407d7f7a56e9a85a4c207724c9f2c545c060380718eea9088f222ba697/yarl-1.20.1-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d1a4fbb50e14396ba3d375f68bfe02215d8e7bc3ec49da8341fe3157f59d2ff5", size = 352614, upload-time = "2025-06-10T00:44:32.171Z" }, - { url = "https://files.pythonhosted.org/packages/b1/91/31163295e82b8d5485d31d9cf7754d973d41915cadce070491778d9c9825/yarl-1.20.1-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:11a62c839c3a8eac2410e951301309426f368388ff2f33799052787035793b02", size = 336766, upload-time = "2025-06-10T00:44:34.494Z" }, - { url = "https://files.pythonhosted.org/packages/b4/8e/c41a5bc482121f51c083c4c2bcd16b9e01e1cf8729e380273a952513a21f/yarl-1.20.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:041eaa14f73ff5a8986b4388ac6bb43a77f2ea09bf1913df7a35d4646db69e53", size = 364615, upload-time = "2025-06-10T00:44:36.856Z" }, - { url = "https://files.pythonhosted.org/packages/e3/5b/61a3b054238d33d70ea06ebba7e58597891b71c699e247df35cc984ab393/yarl-1.20.1-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:377fae2fef158e8fd9d60b4c8751387b8d1fb121d3d0b8e9b0be07d1b41e83dc", size = 360982, upload-time = "2025-06-10T00:44:39.141Z" }, - { url = "https://files.pythonhosted.org/packages/df/a3/6a72fb83f8d478cb201d14927bc8040af901811a88e0ff2da7842dd0ed19/yarl-1.20.1-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:1c92f4390e407513f619d49319023664643d3339bd5e5a56a3bebe01bc67ec04", size = 369792, upload-time = "2025-06-10T00:44:40.934Z" }, - { url = "https://files.pythonhosted.org/packages/7c/af/4cc3c36dfc7c077f8dedb561eb21f69e1e9f2456b91b593882b0b18c19dc/yarl-1.20.1-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:d25ddcf954df1754ab0f86bb696af765c5bfaba39b74095f27eececa049ef9a4", size = 382049, upload-time = "2025-06-10T00:44:42.854Z" }, - { url = "https://files.pythonhosted.org/packages/19/3a/e54e2c4752160115183a66dc9ee75a153f81f3ab2ba4bf79c3c53b33de34/yarl-1.20.1-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:909313577e9619dcff8c31a0ea2aa0a2a828341d92673015456b3ae492e7317b", size = 384774, upload-time = "2025-06-10T00:44:45.275Z" }, - { url = "https://files.pythonhosted.org/packages/9c/20/200ae86dabfca89060ec6447649f219b4cbd94531e425e50d57e5f5ac330/yarl-1.20.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:793fd0580cb9664548c6b83c63b43c477212c0260891ddf86809e1c06c8b08f1", size = 374252, upload-time = "2025-06-10T00:44:47.31Z" }, - { url = "https://files.pythonhosted.org/packages/83/75/11ee332f2f516b3d094e89448da73d557687f7d137d5a0f48c40ff211487/yarl-1.20.1-cp313-cp313-win32.whl", hash = "sha256:468f6e40285de5a5b3c44981ca3a319a4b208ccc07d526b20b12aeedcfa654b7", size = 81198, upload-time = "2025-06-10T00:44:49.164Z" }, - { url = "https://files.pythonhosted.org/packages/ba/ba/39b1ecbf51620b40ab402b0fc817f0ff750f6d92712b44689c2c215be89d/yarl-1.20.1-cp313-cp313-win_amd64.whl", hash = "sha256:495b4ef2fea40596bfc0affe3837411d6aa3371abcf31aac0ccc4bdd64d4ef5c", size = 86346, upload-time = "2025-06-10T00:44:51.182Z" }, - { url = "https://files.pythonhosted.org/packages/43/c7/669c52519dca4c95153c8ad96dd123c79f354a376346b198f438e56ffeb4/yarl-1.20.1-cp313-cp313t-macosx_10_13_universal2.whl", hash = "sha256:f60233b98423aab21d249a30eb27c389c14929f47be8430efa7dbd91493a729d", size = 138826, upload-time = "2025-06-10T00:44:52.883Z" }, - { url = "https://files.pythonhosted.org/packages/6a/42/fc0053719b44f6ad04a75d7f05e0e9674d45ef62f2d9ad2c1163e5c05827/yarl-1.20.1-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:6f3eff4cc3f03d650d8755c6eefc844edde99d641d0dcf4da3ab27141a5f8ddf", size = 93217, upload-time = "2025-06-10T00:44:54.658Z" }, - { url = "https://files.pythonhosted.org/packages/4f/7f/fa59c4c27e2a076bba0d959386e26eba77eb52ea4a0aac48e3515c186b4c/yarl-1.20.1-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:69ff8439d8ba832d6bed88af2c2b3445977eba9a4588b787b32945871c2444e3", size = 92700, upload-time = "2025-06-10T00:44:56.784Z" }, - { url = "https://files.pythonhosted.org/packages/2f/d4/062b2f48e7c93481e88eff97a6312dca15ea200e959f23e96d8ab898c5b8/yarl-1.20.1-cp313-cp313t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3cf34efa60eb81dd2645a2e13e00bb98b76c35ab5061a3989c7a70f78c85006d", size = 347644, upload-time = "2025-06-10T00:44:59.071Z" }, - { url = "https://files.pythonhosted.org/packages/89/47/78b7f40d13c8f62b499cc702fdf69e090455518ae544c00a3bf4afc9fc77/yarl-1.20.1-cp313-cp313t-manylinux_2_17_armv7l.manylinux2014_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:8e0fe9364ad0fddab2688ce72cb7a8e61ea42eff3c7caeeb83874a5d479c896c", size = 323452, upload-time = "2025-06-10T00:45:01.605Z" }, - { url = "https://files.pythonhosted.org/packages/eb/2b/490d3b2dc66f52987d4ee0d3090a147ea67732ce6b4d61e362c1846d0d32/yarl-1.20.1-cp313-cp313t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:8f64fbf81878ba914562c672024089e3401974a39767747691c65080a67b18c1", size = 346378, upload-time = "2025-06-10T00:45:03.946Z" }, - { url = "https://files.pythonhosted.org/packages/66/ad/775da9c8a94ce925d1537f939a4f17d782efef1f973039d821cbe4bcc211/yarl-1.20.1-cp313-cp313t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:f6342d643bf9a1de97e512e45e4b9560a043347e779a173250824f8b254bd5ce", size = 353261, upload-time = "2025-06-10T00:45:05.992Z" }, - { url = "https://files.pythonhosted.org/packages/4b/23/0ed0922b47a4f5c6eb9065d5ff1e459747226ddce5c6a4c111e728c9f701/yarl-1.20.1-cp313-cp313t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:56dac5f452ed25eef0f6e3c6a066c6ab68971d96a9fb441791cad0efba6140d3", size = 335987, upload-time = "2025-06-10T00:45:08.227Z" }, - { url = "https://files.pythonhosted.org/packages/3e/49/bc728a7fe7d0e9336e2b78f0958a2d6b288ba89f25a1762407a222bf53c3/yarl-1.20.1-cp313-cp313t-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c7d7f497126d65e2cad8dc5f97d34c27b19199b6414a40cb36b52f41b79014be", size = 329361, upload-time = "2025-06-10T00:45:10.11Z" }, - { url = "https://files.pythonhosted.org/packages/93/8f/b811b9d1f617c83c907e7082a76e2b92b655400e61730cd61a1f67178393/yarl-1.20.1-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:67e708dfb8e78d8a19169818eeb5c7a80717562de9051bf2413aca8e3696bf16", size = 346460, upload-time = "2025-06-10T00:45:12.055Z" }, - { url = "https://files.pythonhosted.org/packages/70/fd/af94f04f275f95da2c3b8b5e1d49e3e79f1ed8b6ceb0f1664cbd902773ff/yarl-1.20.1-cp313-cp313t-musllinux_1_2_armv7l.whl", hash = "sha256:595c07bc79af2494365cc96ddeb772f76272364ef7c80fb892ef9d0649586513", size = 334486, upload-time = "2025-06-10T00:45:13.995Z" }, - { url = "https://files.pythonhosted.org/packages/84/65/04c62e82704e7dd0a9b3f61dbaa8447f8507655fd16c51da0637b39b2910/yarl-1.20.1-cp313-cp313t-musllinux_1_2_i686.whl", hash = "sha256:7bdd2f80f4a7df852ab9ab49484a4dee8030023aa536df41f2d922fd57bf023f", size = 342219, upload-time = "2025-06-10T00:45:16.479Z" }, - { url = "https://files.pythonhosted.org/packages/91/95/459ca62eb958381b342d94ab9a4b6aec1ddec1f7057c487e926f03c06d30/yarl-1.20.1-cp313-cp313t-musllinux_1_2_ppc64le.whl", hash = "sha256:c03bfebc4ae8d862f853a9757199677ab74ec25424d0ebd68a0027e9c639a390", size = 350693, upload-time = "2025-06-10T00:45:18.399Z" }, - { url = "https://files.pythonhosted.org/packages/a6/00/d393e82dd955ad20617abc546a8f1aee40534d599ff555ea053d0ec9bf03/yarl-1.20.1-cp313-cp313t-musllinux_1_2_s390x.whl", hash = "sha256:344d1103e9c1523f32a5ed704d576172d2cabed3122ea90b1d4e11fe17c66458", size = 355803, upload-time = "2025-06-10T00:45:20.677Z" }, - { url = "https://files.pythonhosted.org/packages/9e/ed/c5fb04869b99b717985e244fd93029c7a8e8febdfcffa06093e32d7d44e7/yarl-1.20.1-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:88cab98aa4e13e1ade8c141daeedd300a4603b7132819c484841bb7af3edce9e", size = 341709, upload-time = "2025-06-10T00:45:23.221Z" }, - { url = "https://files.pythonhosted.org/packages/24/fd/725b8e73ac2a50e78a4534ac43c6addf5c1c2d65380dd48a9169cc6739a9/yarl-1.20.1-cp313-cp313t-win32.whl", hash = "sha256:b121ff6a7cbd4abc28985b6028235491941b9fe8fe226e6fdc539c977ea1739d", size = 86591, upload-time = "2025-06-10T00:45:25.793Z" }, - { url = "https://files.pythonhosted.org/packages/94/c3/b2e9f38bc3e11191981d57ea08cab2166e74ea770024a646617c9cddd9f6/yarl-1.20.1-cp313-cp313t-win_amd64.whl", hash = "sha256:541d050a355bbbc27e55d906bc91cb6fe42f96c01413dd0f4ed5a5240513874f", size = 93003, upload-time = "2025-06-10T00:45:27.752Z" }, - { url = "https://files.pythonhosted.org/packages/b4/2d/2345fce04cfd4bee161bf1e7d9cdc702e3e16109021035dbb24db654a622/yarl-1.20.1-py3-none-any.whl", hash = "sha256:83b8eb083fe4683c6115795d9fc1cfaf2cbbefb19b3a1cb68f6527460f483a77", size = 46542, upload-time = "2025-06-10T00:46:07.521Z" }, +sdist = { url = "https://files.pythonhosted.org/packages/79/12/1e8f37460ea0f7eb59c221fdaf0ed75e7ac43e97f8093b9c6f411df50a78/yarl-1.24.2.tar.gz", hash = "sha256:9ac374123c6fd7abf64d1fec93962b0bd4ee2c19751755a762a72dd96c0378f8", size = 210798, upload-time = "2026-05-19T21:31:05.599Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/3f/df/f1c7a3de0831cd83194f1a85c5bb431b13f81e6b45079314c86d1c4ef3f2/yarl-1.24.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:5249a113065c2b7a958bc699759e359cd61cfc81e3069662208f48f191b7ed12", size = 129057, upload-time = "2026-05-19T21:27:47.564Z" }, + { url = "https://files.pythonhosted.org/packages/48/41/7daafb32dd7562bf45b1ce56562e7e1a9146f6479b6456873eb8a3413c40/yarl-1.24.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:7f4425fa244fbf530b006d0c5f79ce920114cfff5b4f5f6056e669f8e160fdc0", size = 91545, upload-time = "2026-05-19T21:27:50.089Z" }, + { url = "https://files.pythonhosted.org/packages/a8/8f/7b3ec212f1ea0683f55f978e3246bc313c38818664edfc97a9f349a4901e/yarl-1.24.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:15c0b5e49d3c44e2a0b93e6a49476c5edad0a7686b92c395765a7ea775572a75", size = 91380, upload-time = "2026-05-19T21:27:51.953Z" }, + { url = "https://files.pythonhosted.org/packages/8a/1b/8bafab7db23b0567ae9db749099b329d91e3b82bc6028b2050ba583e116c/yarl-1.24.2-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:246d32a53a947c8f0189f5d699cbd4c7036de45d9359e13ba238d1239678c727", size = 105957, upload-time = "2026-05-19T21:27:53.98Z" }, + { url = "https://files.pythonhosted.org/packages/7f/77/21030c2f8d21d21559719beafc772ada2014be933418ed1eaed9cc800e42/yarl-1.24.2-cp310-cp310-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:64480fb3e4d4ed9ed71c48a91a477384fc342a50ca30071d2f8a88d51d9c9413", size = 97242, upload-time = "2026-05-19T21:27:55.981Z" }, + { url = "https://files.pythonhosted.org/packages/50/d8/f9ea63d1b6aa910a866e089d871fff6cbd49caab29b86b35221a62dfa0d5/yarl-1.24.2-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:349de4701dc3760b6e876628423a8f147ef4f5599d10aba1e10702075d424ed9", size = 114719, upload-time = "2026-05-19T21:27:58.037Z" }, + { url = "https://files.pythonhosted.org/packages/e9/a3/04e0ee98ac58a249ea7ed75223f5f901ba81a834f0b4921b58e5cec11757/yarl-1.24.2-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:d162677af8d5d3d6ebab8394b021f4d041ac107a4b705873148a77a49dc9e1b2", size = 112140, upload-time = "2026-05-19T21:27:59.618Z" }, + { url = "https://files.pythonhosted.org/packages/02/ad/0b9cc9f38a7324a7eb1d80f834eaa5283d17e9271bbda3186e598dddaeac/yarl-1.24.2-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f5f5c6ec23a9043f2d139cc072f53dd23168d202a334b9b2fda8de4c3e890d90", size = 106721, upload-time = "2026-05-19T21:28:02.586Z" }, + { url = "https://files.pythonhosted.org/packages/65/e7/a52478ebfc66ec989e085c6ae038b9f1bfa4190baa193b133b669c709e2f/yarl-1.24.2-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:60de6742447fbbf697f16f070b8a443f1b5fe6ca3826fbef9fe70ecd5328e643", size = 106478, upload-time = "2026-05-19T21:28:04.523Z" }, + { url = "https://files.pythonhosted.org/packages/04/d8/5508530fea8472542de00013ae280765fc938ee196fc4030c43a498afb36/yarl-1.24.2-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:acf93187c3710e422368eb768aee98db551ec7c85adc250207a95c16548ab7ac", size = 105423, upload-time = "2026-05-19T21:28:06.515Z" }, + { url = "https://files.pythonhosted.org/packages/84/f1/ece28505e9628e8b756e11bb4f28864a17cc33b6b44db4d2aaf0622bf630/yarl-1.24.2-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:f4b0352fd41fd34b6651934606268816afd6914d09626f9bcbbf018edb0afb3f", size = 99878, upload-time = "2026-05-19T21:28:08.637Z" }, + { url = "https://files.pythonhosted.org/packages/3f/52/fb5d34529b46dd84013afcfb30b8d2bc2832ed03d412736f577d604fa393/yarl-1.24.2-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:6b208bb939099b4b297438da4e9b25357f0b1c791888669b963e45b203ea9f36", size = 114025, upload-time = "2026-05-19T21:28:10.64Z" }, + { url = "https://files.pythonhosted.org/packages/43/f0/ff9d31aaab024f7a251c0ed308a98ae29bf9f7dc344e78f28b1322431ca2/yarl-1.24.2-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:4b85b8825e631295ff4bc8943f7471d54c533a9360bbe15ebb38e018b555bb8a", size = 105613, upload-time = "2026-05-19T21:28:12.784Z" }, + { url = "https://files.pythonhosted.org/packages/31/7d/3296fb3f3ecd52bf9ae6c16b0895c1cda7e9170a2083861552b683f70264/yarl-1.24.2-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:e26acf20c26cb4fefc631fdb75aca2a6b8fa8b7b5d7f204fb6a8f1e63c706f53", size = 111665, upload-time = "2026-05-19T21:28:14.393Z" }, + { url = "https://files.pythonhosted.org/packages/1a/74/77aa6ddaca4fbf42e45e675a465c43956dd40702281049975a2aa04eae59/yarl-1.24.2-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:819ca24f8eafcfb683c1bd5f44f2f488cea1274eb8944731ffd2e1f10f619342", size = 106914, upload-time = "2026-05-19T21:28:15.893Z" }, + { url = "https://files.pythonhosted.org/packages/d8/02/7611f22cd1d4ed7373eb7f9ee21fde1046edba2e7c0e514880d760352f48/yarl-1.24.2-cp310-cp310-win_amd64.whl", hash = "sha256:5cb0f995a901c36be096ccbf4c673591c2faabbe96279598ffaec8c030f85bf4", size = 92658, upload-time = "2026-05-19T21:28:17.471Z" }, + { url = "https://files.pythonhosted.org/packages/91/00/671d0add79938127292839ae44506ce2f7fe8909c72d5a931864f128fd0b/yarl-1.24.2-cp310-cp310-win_arm64.whl", hash = "sha256:f408eace7e22a68b467a0562e0d27d322f91fe3eaaa6f466b962c6cfaea9fa39", size = 87887, upload-time = "2026-05-19T21:28:19.021Z" }, + { url = "https://files.pythonhosted.org/packages/c5/c5/1ce244152ff2839645e7cae92f90e7bafcb2c52bea7ff586ac714f14f5df/yarl-1.24.2-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:36348bebb147b83818b9d7e673ea4debc75970afc6ffdc7e3975ad05ce5a58c1", size = 128971, upload-time = "2026-05-19T21:28:20.543Z" }, + { url = "https://files.pythonhosted.org/packages/87/5a/00f36967203ed89cb3acd2c8ed526cc3fed9418eb70ce128160a911c8499/yarl-1.24.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:1a97e42c8a2233f2f279ecadd9e4a037bcb5d813b78435e8eedd4db5a9e9708c", size = 91507, upload-time = "2026-05-19T21:28:22.556Z" }, + { url = "https://files.pythonhosted.org/packages/31/d0/1fb0c1cd27288f39f6974da4318c32768d72c9890984541fdf1e2e32a51d/yarl-1.24.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:8d027d56f1035e339d1001ac33eceab5b2ec8e42e449787bb75e289fb9a5cd1d", size = 91343, upload-time = "2026-05-19T21:28:24.092Z" }, + { url = "https://files.pythonhosted.org/packages/03/ce/d4a646508bed2f8dec6435b40166fe9308dd191262033d3f307b2bbcaecd/yarl-1.24.2-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0a6377060e7927187a42b7eb202090cbe2b34933a4eeaf90e3bd9e33432e5cae", size = 105704, upload-time = "2026-05-19T21:28:25.872Z" }, + { url = "https://files.pythonhosted.org/packages/4b/07/b3278e82d8bc41485bcf6d856cd0433262593de615b1d3dc43bd3f5bead4/yarl-1.24.2-cp311-cp311-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:17076578bce0049a5ce57d14ad1bded391b68a3b213e9b81b0097b090244999a", size = 97281, upload-time = "2026-05-19T21:28:27.352Z" }, + { url = "https://files.pythonhosted.org/packages/17/5b/4cee6e7c92e487bebe7afc797da0aa54a248ab4e776a68fe369ec29665a5/yarl-1.24.2-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:50713f1d4d6be6375bb178bb43d140ee1acb8abe589cd723320b7925a275be1e", size = 114020, upload-time = "2026-05-19T21:28:29.458Z" }, + { url = "https://files.pythonhosted.org/packages/5c/82/111076571545a7d4f9cca3fbd5c6f40615af58642be09f12328f48022468/yarl-1.24.2-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:34263e2fa8fb5bb63a0d97706cda38edbad62fddb58c7f12d6acbc092812aa50", size = 111450, upload-time = "2026-05-19T21:28:31.262Z" }, + { url = "https://files.pythonhosted.org/packages/b6/ec/08f671f69a444d704aeecebf92af659b67b97a869942411d0a578b08c334/yarl-1.24.2-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:49016d82f032b1bd1e10b01078a7d29ae71bf468eeae0ea22df8bab691e60003", size = 106384, upload-time = "2026-05-19T21:28:32.856Z" }, + { url = "https://files.pythonhosted.org/packages/e5/86/ce41e7a7a199340b2330d52b60f25c4074b6636dd0e60b1a80d31a9db042/yarl-1.24.2-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:3f6d2c216318f8f32038ca3f72501ba08536f0fd18a36e858836b121b2deed9f", size = 106153, upload-time = "2026-05-19T21:28:35.222Z" }, + { url = "https://files.pythonhosted.org/packages/c4/5d/31be8a729531ab3e55ac3e7e5c800be8c89ea98947f418b2f6ea259fb6ee/yarl-1.24.2-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:08d3a33218e0c64393e7610284e770409a9c31c429b078bcb24096ed0a783b8f", size = 105322, upload-time = "2026-05-19T21:28:36.642Z" }, + { url = "https://files.pythonhosted.org/packages/47/9b/b57afb22b386ae87ac9940f09878b98d8c333f89113e6fc96fcf4ca9eb64/yarl-1.24.2-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:5d699376c4ca3cba49bbfae3a05b5b70ded572937171ce1e0b8d87118e2ba294", size = 99057, upload-time = "2026-05-19T21:28:38.386Z" }, + { url = "https://files.pythonhosted.org/packages/a3/4f/06348c27c8389256c313e8a57d796808fc0264c915dd5e7cfd3c0e314dc7/yarl-1.24.2-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:a1cab588b4fa14bea2e55ebea27478adfb05372f47573738e1acc4a36c0b05d2", size = 113502, upload-time = "2026-05-19T21:28:40.091Z" }, + { url = "https://files.pythonhosted.org/packages/5f/1c/284f307b298e4a17b7943b07d9d7ecc4151537f8d137ba51f3bb6c31ca20/yarl-1.24.2-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:ec87ccc31bd21db7ad009d8572c127c1000f268517618a4cc09adba3c2a7f21c", size = 105253, upload-time = "2026-05-19T21:28:41.987Z" }, + { url = "https://files.pythonhosted.org/packages/c8/bf/0de123bec8619e45c80cbded9085f61b5b4a9eddb8abe6d25d28ee1ec866/yarl-1.24.2-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:d1dd47a22843b212baa8d74f37796815d43bd046b42a0f41e9da433386c3136b", size = 111345, upload-time = "2026-05-19T21:28:43.93Z" }, + { url = "https://files.pythonhosted.org/packages/90/af/0248eb065e51129d2a9b2436cd1b5c772c19a6b04e5b6a186955671e3319/yarl-1.24.2-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:7b54b9c67c2b06bd7b9a77253d242124b9c95d2c02def5a1144001ee547dd9d5", size = 106558, upload-time = "2026-05-19T21:28:45.806Z" }, + { url = "https://files.pythonhosted.org/packages/21/3c/f960d7a65ef97d8ba9b424fb5128796a4bc710fc6df2ddbbd7dfdc3bbd20/yarl-1.24.2-cp311-cp311-win_amd64.whl", hash = "sha256:f8fdbcff8b2c7c9284e60c196f693588598ddcee31e11c18e14949ce44519d45", size = 92808, upload-time = "2026-05-19T21:28:48.465Z" }, + { url = "https://files.pythonhosted.org/packages/03/1a/49fb03750e4de4d2284cd5b885a383133c34eef45bd59631b2bb8b7e81e8/yarl-1.24.2-cp311-cp311-win_arm64.whl", hash = "sha256:b32c37a7a337e90822c45797bf3d79d60875cfcccd3ecc80e9f453d87026c122", size = 87610, upload-time = "2026-05-19T21:28:50.07Z" }, + { url = "https://files.pythonhosted.org/packages/f0/da/866bcb01076ba49d2b42b309867bed3826421f1c479655eb7a607b44f20b/yarl-1.24.2-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:b975866c184564c827e0877380f0dae57dcca7e52782128381b72feff6dfceb8", size = 129957, upload-time = "2026-05-19T21:28:51.695Z" }, + { url = "https://files.pythonhosted.org/packages/bf/1d/fcefb70922ea2268a8971d8e5874d9a8218644200fb8465f1dcad55e6851/yarl-1.24.2-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:3b075301a2836a0e297b1b658cb6d6135df535d62efefdd60366bd589c2c82f2", size = 92164, upload-time = "2026-05-19T21:28:53.242Z" }, + { url = "https://files.pythonhosted.org/packages/29/b6/170e2b8d4e3bc30e6bfdcca53556537f5bf595e938632dfcb059311f3ff6/yarl-1.24.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:8ae44649b00947634ab0dab2a374a638f52923a6e67083f2c156cd5cbd1a881d", size = 91688, upload-time = "2026-05-19T21:28:54.865Z" }, + { url = "https://files.pythonhosted.org/packages/fe/a5/c9f655d5553ea0b99fdac9d6a99ad3f9b3e73b8e5758bb46f58c9831f74c/yarl-1.24.2-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:507cc19f0b45454e2d6dcd62ff7d062b9f77a2812404e62dbdaec05b50faa035", size = 102902, upload-time = "2026-05-19T21:28:56.963Z" }, + { url = "https://files.pythonhosted.org/packages/5d/bc/6b9664d815d79af4ee553337f9d606c56bbf269186ada9172de45f1b5f60/yarl-1.24.2-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:c4c17bad5a530912d2111825d3f05e89bab2dd376aaa8cbc77e449e6db63e576", size = 97931, upload-time = "2026-05-19T21:28:58.56Z" }, + { url = "https://files.pythonhosted.org/packages/98/ec/32ba48acae30fecd60928f5791188b80a9d6ee3840507ffda29fecd37b71/yarl-1.24.2-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:f5f0cbb112838a4a293985b6ed73948a547dadcc1ba6d2089938e7abdedceef8", size = 111030, upload-time = "2026-05-19T21:29:00.148Z" }, + { url = "https://files.pythonhosted.org/packages/82/5a/6f4cd081e5f4934d2ae3a8ef4abe3afacc010d26f0035ee91b35cd7d7c37/yarl-1.24.2-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:5ec8356b8a6afcf81fc7aeeef13b1ff7a49dec00f313394bbb9e83830d32ccd7", size = 110392, upload-time = "2026-05-19T21:29:02.155Z" }, + { url = "https://files.pythonhosted.org/packages/7a/da/323a01c349bd5fb01bb6652e314d9bb218cee630a736bdb810ad50e4013f/yarl-1.24.2-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7e7ebcdef69dec6c6451e616f32b622a6d4a2e92b445c992f7c8e5274a6bbc4c", size = 105612, upload-time = "2026-05-19T21:29:04.247Z" }, + { url = "https://files.pythonhosted.org/packages/7c/80/264ab684f181e1a876389374519ff05d10248725535ae2ac4e8ac4e563d6/yarl-1.24.2-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:47a55d6cf6db2f401017a9e96e5288844e5051911fb4e0c8311a3980f5e59a7d", size = 104487, upload-time = "2026-05-19T21:29:06.491Z" }, + { url = "https://files.pythonhosted.org/packages/41/07/efabe5df87e96d7ad5959760b888344be48cd6884db127b407c6b5503adc/yarl-1.24.2-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:3065657c80a2321225e804048597ad55658a7e76b32d6f5ee4074d04c50401db", size = 102333, upload-time = "2026-05-19T21:29:08.267Z" }, + { url = "https://files.pythonhosted.org/packages/44/0c/bcf7c42603e1009295f586d8890f2ba032c8b53310e815adf0a202c73d9f/yarl-1.24.2-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:cb84b80d88e19ede158619b80813968713d8d008b0e2497a576e6a0557d50712", size = 99025, upload-time = "2026-05-19T21:29:10.682Z" }, + { url = "https://files.pythonhosted.org/packages/4f/82/84482ab1a57a0f21a08afe6a7004c61d741f8f2ecc3b05c321577c612164/yarl-1.24.2-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:990de4f680b1c217e77ff0d6aa0029f9eb79889c11fb3e9a3942c7eba29c1996", size = 110507, upload-time = "2026-05-19T21:29:12.954Z" }, + { url = "https://files.pythonhosted.org/packages/c4/8d/a546ba1dfe1b0f290e05fef145cd07614c0f15df1a707195e512d1e39d1d/yarl-1.24.2-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:abb8ec0323b80161e3802da3150ef660b41d0e9be2048b76a363d93eee992c2b", size = 103719, upload-time = "2026-05-19T21:29:14.893Z" }, + { url = "https://files.pythonhosted.org/packages/1a/b6/267f2a09213138473adfce6b8a6e17791d7fee70bd4d9003218e4dec58b0/yarl-1.24.2-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:e7977781f83638a4c73e0f88425563d70173e0dfd90ac006a45c65036293ee3c", size = 110438, upload-time = "2026-05-19T21:29:16.485Z" }, + { url = "https://files.pythonhosted.org/packages/48/2d/1c8d89c7c5f9cad9fb2902445d94e2ab1d7aa35de029afbb8ae95c42d00f/yarl-1.24.2-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:e30dd55825dc554ec5b66a94953b8eda8745926514c5089dfcacecb9c99b5bd1", size = 105719, upload-time = "2026-05-19T21:29:18.367Z" }, + { url = "https://files.pythonhosted.org/packages/a7/25/722e3b93bd687009afb2d59a35e13d30ddd8f80571445bb0c4e4ce26ec66/yarl-1.24.2-cp312-cp312-win_amd64.whl", hash = "sha256:7dafe10c12ddd4d120d528c4b5599c953bd7b12845347d507b95451195bb6cad", size = 92901, upload-time = "2026-05-19T21:29:20.014Z" }, + { url = "https://files.pythonhosted.org/packages/39/47/4486ccfb674c04854a1ef8aa77868b6a6f765feaf69633409d7ca4f02cb8/yarl-1.24.2-cp312-cp312-win_arm64.whl", hash = "sha256:044a09d8401fcf8681977faef6d286b8ade1e2d2e9dceda175d1cfa5ca496f30", size = 87229, upload-time = "2026-05-19T21:29:22.1Z" }, + { url = "https://files.pythonhosted.org/packages/82/62/fcf0ce677f17e5c471c06311dd25964be38a4c586993632910d2e75278bc/yarl-1.24.2-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:491ac9141decf49ee8030199e1ee251cdff0e131f25678817ff6aa5f837a3536", size = 128978, upload-time = "2026-05-19T21:29:23.83Z" }, + { url = "https://files.pythonhosted.org/packages/d3/58/8e63299bb71ed61a834121d9d3fe6c9fcf2a6a5d09754ff4f20f2d20baf5/yarl-1.24.2-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:e89418f65eda18f99030386305bd44d7d504e328a7945db1ead514fbe03a0607", size = 91733, upload-time = "2026-05-19T21:29:25.375Z" }, + { url = "https://files.pythonhosted.org/packages/c1/24/16748d5dab6daec8b0ed81ccec639a1cded0f18dcc62a4f696b4fe366c37/yarl-1.24.2-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:cdfcce633b4a4bb8281913c57fcafd4b5933fbc19111a5e3930bbd299d6102f1", size = 91113, upload-time = "2026-05-19T21:29:26.928Z" }, + { url = "https://files.pythonhosted.org/packages/1b/66/b63fff7b71211e866624b21432d5943cbb633eb0c2872d9ee3070648f22c/yarl-1.24.2-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:863297ddede92ee49024e9a9b11ecb59f310ca85b60d8537f56bed9bbb5b1986", size = 103899, upload-time = "2026-05-19T21:29:28.842Z" }, + { url = "https://files.pythonhosted.org/packages/9d/ac/ba1974b8533909636f7733fe86cf677e3619527c3c2fa913e0ea89c48757/yarl-1.24.2-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:374423f70754a2c96942ede36a29d37dc6b0cb8f92f8d009ddf3ed78d3da5488", size = 97862, upload-time = "2026-05-19T21:29:31.086Z" }, + { url = "https://files.pythonhosted.org/packages/1b/a5/123ac993b5c2ba6f554a140305620cb8f150fa543711bbc49be3ec0a65a4/yarl-1.24.2-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:33a29b5d00ccbf3219bb3e351d7875739c19481e030779f48cc46a7a71681a9b", size = 111060, upload-time = "2026-05-19T21:29:32.657Z" }, + { url = "https://files.pythonhosted.org/packages/23/37/c472d3af3509688392134a88a825276770a187f1daa4de3f6dc0a327a751/yarl-1.24.2-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a9532c57211730c515341af11fef6e9b61d157487272a096d0c04da445642592", size = 110613, upload-time = "2026-05-19T21:29:34.379Z" }, + { url = "https://files.pythonhosted.org/packages/df/88/09c28dad91e662ccfaa1b78f1c57badde74fc9d0b23e74aef644750ecd73/yarl-1.24.2-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:91e72cf093fd833483a97ee648e0c053c7c629f51ff4a0e7edd84f806b0c5617", size = 107012, upload-time = "2026-05-19T21:29:36.216Z" }, + { url = "https://files.pythonhosted.org/packages/07/ab/9d4f69d571a94f4d112fa7e2e007200f5a54d319f58c82ac7b7baa61f5c6/yarl-1.24.2-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:b3177bc0a768ef3bacceb4f272632990b7bea352f1b2f1eee9d6d6ff16516f92", size = 105887, upload-time = "2026-05-19T21:29:38.746Z" }, + { url = "https://files.pythonhosted.org/packages/8e/9a/000b2b66c0d772a499fc531d21dab92dfeb73b640a12eed6ba89f49bb2d0/yarl-1.24.2-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:e196952aacaf3b232e265ff02980b64d483dc0972bd49bcb061171ff22ac203a", size = 103620, upload-time = "2026-05-19T21:29:40.368Z" }, + { url = "https://files.pythonhosted.org/packages/41/7c/7c1050f73450fbdaa3f0c72017059f00ce5e13366692f3dba25275a1083d/yarl-1.24.2-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:204e7a61ce99919c0de1bf904ab5d7aa188a129ea8f690a8f76cfb6e2844dc44", size = 100599, upload-time = "2026-05-19T21:29:42.66Z" }, + { url = "https://files.pythonhosted.org/packages/ec/b1/29e5756b3926705f5f6089bd5b9f50a56eaac550da6e260bf713ead44d04/yarl-1.24.2-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:4b156914620f0b9d78dc1adb3751141daee561cfec796088abb89ed49d220f1a", size = 110604, upload-time = "2026-05-19T21:29:44.632Z" }, + { url = "https://files.pythonhosted.org/packages/a3/4b/8415bc96e9b150cde942fbac9a8182985e58f40ce5c54c34ed015407d3ee/yarl-1.24.2-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:8372a2b976cf70654b2be6619ab6068acabb35f724c0fda7b277fbf53d66a5cf", size = 105161, upload-time = "2026-05-19T21:29:46.755Z" }, + { url = "https://files.pythonhosted.org/packages/8b/d4/cde059abfa229553b7298a2eadde2752e723d50aeedaef86ce59da2718ee/yarl-1.24.2-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:f9a1e9b622ca284143aab5d885848686dcd85453bb1ca9abcdb7503e64dc0056", size = 110619, upload-time = "2026-05-19T21:29:48.972Z" }, + { url = "https://files.pythonhosted.org/packages/e7/2c/d6a6c9a61549f7b6c7e6dc6937d195bcf069582b47b7200dcd0e7b256acf/yarl-1.24.2-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:810e19b685c8c3c5862f6a38160a1f4e4c0916c9390024ec347b6157a45a0992", size = 107362, upload-time = "2026-05-19T21:29:51Z" }, + { url = "https://files.pythonhosted.org/packages/92/dd/3ae5fe417e9d1c353a548553326eb9935e76b6b727161563b424cc296df3/yarl-1.24.2-cp313-cp313-win_amd64.whl", hash = "sha256:7d37fb7c38f2b6edab0f845c4f85148d4c44204f52bc127021bd2bc9fdbf1656", size = 92667, upload-time = "2026-05-19T21:29:52.743Z" }, + { url = "https://files.pythonhosted.org/packages/10/cc/a7beb239f78f27fca1b053c8e8595e4179c02e62249b4687ec218c370c50/yarl-1.24.2-cp313-cp313-win_arm64.whl", hash = "sha256:1e831894be7c2954240e49791fa4b50c05a0dc881de2552cfe3ffd8631c7f461", size = 87069, upload-time = "2026-05-19T21:29:54.442Z" }, + { url = "https://files.pythonhosted.org/packages/40/0e/e08087695fc12789263821c5dc0f8dc52b5b17efd0887cacf419f8a43ba3/yarl-1.24.2-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:f9312b3c02d9b3d23840f67952913c9c8721d7f1b7db305289faefa878f364c2", size = 129670, upload-time = "2026-05-19T21:29:56.631Z" }, + { url = "https://files.pythonhosted.org/packages/3a/98/ab4b5ed1b1b5cd973c8a3eb994c3a6aefb6ce6d399e21bb5f0316c33815c/yarl-1.24.2-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:a4f4d6cd615823bfc7fb7e9b5987c3f41666371d870d51058f77e2680fbe9630", size = 91916, upload-time = "2026-05-19T21:29:58.645Z" }, + { url = "https://files.pythonhosted.org/packages/ba/b1/5297bb6a7df4782f7605bffc43b31f5044070935fbbcaa6c705a07e6ac65/yarl-1.24.2-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:0c3063e5c0a8e8e62fae6c2596fa01da1561e4cd1da6fec5789f5cf99a8aefd8", size = 91625, upload-time = "2026-05-19T21:30:00.412Z" }, + { url = "https://files.pythonhosted.org/packages/02/a7/45baabfff76829264e623b185cff0c340d7e11bf3e1cd9ea37e7d17934bd/yarl-1.24.2-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:fecd17873a096036c1c87ab3486f1aef7f269ada7f23f7f856f93b1cc7744f14", size = 104574, upload-time = "2026-05-19T21:30:02.544Z" }, + { url = "https://files.pythonhosted.org/packages/f3/40/3a5ab144d3d650ca37d4f4b57e56169be8af3ca34c448793e064b30baaed/yarl-1.24.2-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:a46d1ab4ba4d32e6dc80daf8a28ce0bd83d08df52fbc32f3e288663427734535", size = 97534, upload-time = "2026-05-19T21:30:04.319Z" }, + { url = "https://files.pythonhosted.org/packages/9c/b5/5658fef3681fb5776b4513b052bec750009f47b3a592251c705d75375798/yarl-1.24.2-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:73e68edf6dfd5f73f9ca127d84e2a6f9213c65bdffb736bda19524c0564fcd14", size = 111481, upload-time = "2026-05-19T21:30:05.988Z" }, + { url = "https://files.pythonhosted.org/packages/4c/06/fdcd7dde037f00866dce123ed4ba23dba94beb56fc4cf561668d27be37f2/yarl-1.24.2-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a296ca617f2d25fbceafb962b88750d627e5984e75732c712154d058ae8d79a3", size = 111529, upload-time = "2026-05-19T21:30:07.738Z" }, + { url = "https://files.pythonhosted.org/packages/c2/53/d81269aaafccea0d33396c03035de997b743f11e648e6e27a0df99c72980/yarl-1.24.2-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e51b2cf5ec89a8b8470177641ed62a3ba22d74e1e898e06ad53aa77972487208", size = 107338, upload-time = "2026-05-19T21:30:09.713Z" }, + { url = "https://files.pythonhosted.org/packages/ae/04/23049463f729bd899df203a7960505a75333edd499cda8aa1d5a82b64df5/yarl-1.24.2-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:310fc687f7b2044ec54e372c8cbe923bb88f5c37bded0d3079e5791c2fc3cf50", size = 106147, upload-time = "2026-05-19T21:30:11.365Z" }, + { url = "https://files.pythonhosted.org/packages/14/18/04a4b5830b43ed5e4c5015b40e9f6241ad91487d71611061b4e111d6ac80/yarl-1.24.2-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:297a2fe352ecf858b30a98f87948746ec16f001d279f84aebdbd3bd965e2f1bd", size = 104272, upload-time = "2026-05-19T21:30:12.978Z" }, + { url = "https://files.pythonhosted.org/packages/5a/f7/8cffdf319aee7a7c1dbd07b61d91c3e3fda460c7a93b5f93e445f3806c4c/yarl-1.24.2-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:2a263e76b97bc42bdcd7c5f4953dec1f7cd62a1112fa7f869e57255229390d67", size = 99962, upload-time = "2026-05-19T21:30:15.001Z" }, + { url = "https://files.pythonhosted.org/packages/d7/39/b3cce3b7dbef64ac700ad4cea156a207d01bede0f507587616c364b5468e/yarl-1.24.2-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:822519b64cf0b474f1a0aaef1dc621438ea46bb77c94df97a5b4d213a7d8a8b1", size = 111063, upload-time = "2026-05-19T21:30:16.683Z" }, + { url = "https://files.pythonhosted.org/packages/a1/ea/100818505e7ebf165c7242ff17fdf7d9fee79e27234aeca871c1082920d7/yarl-1.24.2-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:b6067060d9dc594899ba83e6db6c48c68d1e494a6dab158156ed86977ca7bcb1", size = 105438, upload-time = "2026-05-19T21:30:18.769Z" }, + { url = "https://files.pythonhosted.org/packages/8f/d2/e075a0b32aa6625087de9e653087df0759fed5de4a435fef594181102a77/yarl-1.24.2-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:0063adad533e57171b79db3943b229d40dfafeeee579767f96541f106bac5f1b", size = 111458, upload-time = "2026-05-19T21:30:21.024Z" }, + { url = "https://files.pythonhosted.org/packages/e6/5c/ceea7ba98b65c8eb8d947fdc52f9bedfcd43c6a57c9e3c90c17be8f324a3/yarl-1.24.2-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:ee8e3fb34513e8dc082b586ef4910c98335d43a6fab688cd44d4851bacfce3e8", size = 107589, upload-time = "2026-05-19T21:30:23.412Z" }, + { url = "https://files.pythonhosted.org/packages/fa/d9/5582d57e2b2db9b85eb6663a22efdd78e08805f3f5389566e9fcad254d1b/yarl-1.24.2-cp314-cp314-win_amd64.whl", hash = "sha256:afb00d7fd8e0f285ca29a44cc50df2d622ff2f7a6d933fa641577b5f9d5f3db0", size = 94424, upload-time = "2026-05-19T21:30:25.425Z" }, + { url = "https://files.pythonhosted.org/packages/92/10/7dc07a0e22806a9280f42a57361395506e800c64e22737cd7b0886feab42/yarl-1.24.2-cp314-cp314-win_arm64.whl", hash = "sha256:68cf6eacd6028ef1142bc4b48376b81566385ca6f9e7dde3b0fa91be08ffcb57", size = 88690, upload-time = "2026-05-19T21:30:27.623Z" }, + { url = "https://files.pythonhosted.org/packages/9e/13/d5b8e2c8667db955bcb3de233f18798fefe7edf1d7429c2c9d4f9c401114/yarl-1.24.2-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:221ce1dd921ac4f603957f17d7c18c5cc0797fbb52f156941f92e04605d1d67b", size = 136248, upload-time = "2026-05-19T21:30:29.297Z" }, + { url = "https://files.pythonhosted.org/packages/de/46/a4a97c05c9c9b8fd266bb2a0df12992c7fbd02391eb9640583411b6dab32/yarl-1.24.2-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:5f3224db28173a00d7afacdee07045cc4673dfab2b15492c7ae10deddbece761", size = 95084, upload-time = "2026-05-19T21:30:31.031Z" }, + { url = "https://files.pythonhosted.org/packages/95/b2/845cf2074a015e6fe0d0808cf1a2d9e868386c4220d657ebd8302b199043/yarl-1.24.2-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:c557165320d6244ebe3a02431b2a201a20080e02f41f0cfa0ccc47a183765da8", size = 95272, upload-time = "2026-05-19T21:30:33.062Z" }, + { url = "https://files.pythonhosted.org/packages/fe/16/e69d4aa244aef45235ddfebc0e04036a6829842bc5a6a795aedc6c998d23/yarl-1.24.2-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:904065e6e85b1fa54d0d87438bd58c14c0bad97aad654ad1077fd9d87e8478ed", size = 101497, upload-time = "2026-05-19T21:30:34.842Z" }, + { url = "https://files.pythonhosted.org/packages/15/94/c07107715d621076863ee88b3ddf183fa5e9d4aba5769623c9979828410a/yarl-1.24.2-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:8cec2a38d70edc10e0e856ceda886af5327a017ccbde8e1de1bd44d300357543", size = 94002, upload-time = "2026-05-19T21:30:37.724Z" }, + { url = "https://files.pythonhosted.org/packages/a9/35/fc1bbdd895b5e4010b8fdd037f7ed3aa289d3863e08231b30231ca9a0815/yarl-1.24.2-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:e7484b9361ed222ee1ca5b4337aa4cbdcc4618ce5aff57d9ef1582fd95893fc0", size = 106524, upload-time = "2026-05-19T21:30:40.196Z" }, + { url = "https://files.pythonhosted.org/packages/1f/f2/32b66d0a4ba47c296cf86d03e2c67bff58399fe6d6d84d5205c04c66cc6d/yarl-1.24.2-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:84f9670b89f34db07f81e53aee83e0b938a3412329d51c8f922488be7fcc4024", size = 106165, upload-time = "2026-05-19T21:30:41.888Z" }, + { url = "https://files.pythonhosted.org/packages/95/47/37cb5ff50c5e825d4d38e81bb04d1b7e96bf960f7ab89f9850b162f3f114/yarl-1.24.2-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:abb2759733d63a28b4956500a5dd57140f26486c92b2caedfb964ab7d9b79dbf", size = 103010, upload-time = "2026-05-19T21:30:43.985Z" }, + { url = "https://files.pythonhosted.org/packages/6f/d2/4597912315096f7bb359e46e13bf8b60994fcbb2db29b804c0902ef4eff5/yarl-1.24.2-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:081c2bf54efe03774d0311172bc04fedf9ca01e644d4cd8c805688e527209bdc", size = 101128, upload-time = "2026-05-19T21:30:46.291Z" }, + { url = "https://files.pythonhosted.org/packages/b9/d5/c8e86e120521e646013d02a8e3b8884392e28494be8f392366e50d208efc/yarl-1.24.2-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:86746bef442aa479107fe28132e1277237f9c24c2f00b0b0cf22b3ee0904f2bb", size = 101382, upload-time = "2026-05-19T21:30:48.085Z" }, + { url = "https://files.pythonhosted.org/packages/fa/98/70b229236118f89dbeb739b76f10225bbf53b5497725502594c9a01d699a/yarl-1.24.2-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:2d07d21d0bc4b17558e8de0b02fbfdf1e347d3bb3699edd00bb92e7c57925420", size = 95964, upload-time = "2026-05-19T21:30:49.785Z" }, + { url = "https://files.pythonhosted.org/packages/87/f8/56c386981e3c8648d279fdef2397ffec577e8320fd5649745e34d54faeb7/yarl-1.24.2-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:4fb1ac3fc5fecd8ae7453ea237e4d22b49befa70266dfe1629924245c21a0c7f", size = 106204, upload-time = "2026-05-19T21:30:51.862Z" }, + { url = "https://files.pythonhosted.org/packages/1a/1e/765afe97811ca35933e2a7de70ac57b1997ea2e4ee895719ee7a231fb7e5/yarl-1.24.2-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:4da31a5512ed1729ca8d8aacde3f7faeb8843cde3165d6bcf7f88f74f17bb8aa", size = 101510, upload-time = "2026-05-19T21:30:53.62Z" }, + { url = "https://files.pythonhosted.org/packages/ee/78/393913f4b9039e1edd09ae8a9bbb9d539be909a8abf6d8a2084585bed4b7/yarl-1.24.2-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:533ded4dceb5f1f3da7906244f4e82cf46cfd40d84c69a1faf5ac506aa65ecbe", size = 105584, upload-time = "2026-05-19T21:30:55.962Z" }, + { url = "https://files.pythonhosted.org/packages/78/87/deb17b7049bbe74ea11a713b86f8f27800cc1c8648b0b797243ebb4830ba/yarl-1.24.2-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:7b3a85525f6e7eeabcfdd372862b21ee1915db1b498a04e8bf0e389b607ff0bd", size = 103410, upload-time = "2026-05-19T21:30:57.962Z" }, + { url = "https://files.pythonhosted.org/packages/8f/be/f9f7594e23b5b93affff0318e4593c1920331bcaefda326cabcad94296a1/yarl-1.24.2-cp314-cp314t-win_amd64.whl", hash = "sha256:a7624b1ca46ca5d7b864ef0d2f8efe3091454085ee1855b4e992314529972215", size = 102980, upload-time = "2026-05-19T21:30:59.735Z" }, + { url = "https://files.pythonhosted.org/packages/65/a4/ba80dccd3593ff1f01051a818694d07b58cb8232677ee9a22a5a1f93a9fc/yarl-1.24.2-cp314-cp314t-win_arm64.whl", hash = "sha256:e434a45ce2e7a947f951fc5a8944c8cc080b7e59f9c50ae80fd39107cf88126d", size = 91219, upload-time = "2026-05-19T21:31:01.934Z" }, + { url = "https://files.pythonhosted.org/packages/fd/4d/4b880086bd0d3e034d25647be1d830afc3e3f610e98c4ab3490af6b1b6d5/yarl-1.24.2-py3-none-any.whl", hash = "sha256:2783d9226db8797636cd6896e4de81feed252d1db72265686c9558d97a4d94b9", size = 53576, upload-time = "2026-05-19T21:31:03.909Z" }, +] + +[[package]] +name = "zipp" +version = "4.1.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/b9/d8/eab98a517c14134c0b2eb4e2387bc5f457334293ec5d2dd3857ec2966802/zipp-4.1.0.tar.gz", hash = "sha256:4cb57381f544315db7688e976e922a2b18cdb513d21cc194eb42232ba2a3e602", size = 26214, upload-time = "2026-05-18T20:08:57.967Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/3a/13/547360d81e6d88d58492968ffda9f9542854f11310ee556fef14260cc886/zipp-4.1.0-py3-none-any.whl", hash = "sha256:25ad4e16390cd314347dd8f1de67a2ac538ae658ed4ab9db16029c07c188e97f", size = 10238, upload-time = "2026-05-18T20:08:57.045Z" }, ] diff --git a/rust-toolchain.toml b/rust-toolchain.toml index 5699bd4d536..c4058187023 100644 --- a/rust-toolchain.toml +++ b/rust-toolchain.toml @@ -1,5 +1,5 @@ # We keep this pinned to keep clippy and rustfmt in sync between local and CI. # Feel free to upgrade to bring in new lints. [toolchain] -channel = "1.94.0" +channel = "1.97.0" components = ["rustfmt", "clippy", "rust-analyzer"] diff --git a/rust/AGENTS.md b/rust/AGENTS.md index 6b2729c6692..70a803c6c76 100644 --- a/rust/AGENTS.md +++ b/rust/AGENTS.md @@ -17,6 +17,10 @@ Also see [root AGENTS.md](../AGENTS.md) for cross-language standards. - Delete obsolete internal (`pub(crate)` / private) methods in the same PR that introduces their replacements. For public API methods, follow the deprecation path in root AGENTS.md instead. - Choose log levels by audience: `debug!` for routine/high-frequency ops, `info!` for infrequent operator-visible state changes, `warn!` for unexpected conditions. +## Concurrency + +- The closure passed to `spawn_cpu()` must only consume CPU and return — it must **never** wait on anything: **no channels** (blocking send/recv), **no I/O**, **no locks**, and no `block_on`/`.blocking_*`. The CPU pool can collapse to a single worker in resource-constrained environments (`<= 3` CPUs), so a parked closure can deadlock the whole pool with a silent 0% hang. Keep the waiting in surrounding async code and hand only the pure-CPU work to `spawn_cpu()`. Only dispatch substantial work (rule of thumb: ~100µs+ of CPU); below that the pool overhead outweighs the benefit and the work is better left inline. See the doc comment on `spawn_cpu` for the rationale. + ## API Design - Use `with_`-prefixed builder methods for optional config (e.g., `MyStruct::new(required).with_option(v)`) — don't create separate constructor variants. diff --git a/rust/CLAUDE.md b/rust/CLAUDE.md new file mode 120000 index 00000000000..47dc3e3d863 --- /dev/null +++ b/rust/CLAUDE.md @@ -0,0 +1 @@ +AGENTS.md \ No newline at end of file diff --git a/rust/compression/bitpacking/src/bitpacker_internal/bitpacker8x.rs b/rust/compression/bitpacking/src/bitpacker_internal/bitpacker8x.rs index 188a1f4ce2a..b17edacabb2 100644 --- a/rust/compression/bitpacking/src/bitpacker_internal/bitpacker8x.rs +++ b/rust/compression/bitpacking/src/bitpacker_internal/bitpacker8x.rs @@ -420,12 +420,11 @@ enum InstructionSet { Scalar, } -/// Internal 8-wide bitpacker implementation. +/// 8-wide bitpacker implementation. /// -/// One block contains 256 integers. This stays private to avoid exposing a new -/// block-size choice through the public Lance bitpacking API. +/// One block contains 256 integers. #[derive(Clone, Copy)] -pub(crate) struct BitPacker8x(InstructionSet); +pub struct BitPacker8x(InstructionSet); impl BitPacker8x { #[cfg(target_arch = "x86_64")] diff --git a/rust/compression/bitpacking/src/bitpacker_internal/mod.rs b/rust/compression/bitpacking/src/bitpacker_internal/mod.rs index c287a29da0b..80803e50ec8 100644 --- a/rust/compression/bitpacking/src/bitpacker_internal/mod.rs +++ b/rust/compression/bitpacking/src/bitpacker_internal/mod.rs @@ -20,6 +20,7 @@ mod bitpacker4x; mod bitpacker8x; pub use bitpacker4x::BitPacker4x; +pub use bitpacker8x::BitPacker8x; pub(crate) trait Available { fn available() -> bool; diff --git a/rust/compression/bitpacking/src/lib.rs b/rust/compression/bitpacking/src/lib.rs index f0e25e37e8c..c6aa6d75ad0 100644 --- a/rust/compression/bitpacking/src/lib.rs +++ b/rust/compression/bitpacking/src/lib.rs @@ -18,7 +18,7 @@ use core::mem::size_of; mod bitpacker_internal; -pub use bitpacker_internal::{BitPacker, BitPacker4x}; +pub use bitpacker_internal::{BitPacker, BitPacker4x, BitPacker8x}; pub const FL_ORDER: [usize; 8] = [0, 4, 2, 6, 1, 5, 3, 7]; diff --git a/rust/compression/fsst/Cargo.toml b/rust/compression/fsst/Cargo.toml index da5d8f01d04..7056896e3b9 100644 --- a/rust/compression/fsst/Cargo.toml +++ b/rust/compression/fsst/Cargo.toml @@ -16,7 +16,6 @@ arrow-array.workspace = true rand.workspace = true [dev-dependencies] -arrow-array.workspace = true test-log.workspace = true tokio.workspace = true diff --git a/rust/compression/fsst/examples/benchmark.rs b/rust/compression/fsst/examples/benchmark.rs index c442243e112..f71abeefedd 100644 --- a/rust/compression/fsst/examples/benchmark.rs +++ b/rust/compression/fsst/examples/benchmark.rs @@ -56,7 +56,10 @@ fn benchmark(file_path: &str) { let mut decompression_out_bufs = vec![]; let mut decompression_out_offsets_bufs = vec![]; for _ in 0..TEST_NUM { - let this_decom_out_buf = vec![0u8; BUFFER_SIZE * 3]; + // `decompress` requires the output buffer to be at least 8x the compressed input (a 1-byte + // code can expand to an 8-byte symbol). The compressed buffer is at most `BUFFER_SIZE`, so + // `BUFFER_SIZE * 8` is a safe upper bound. + let this_decom_out_buf = vec![0u8; BUFFER_SIZE * 8]; let this_decom_out_offsets_buf = vec![0i32; BUFFER_SIZE * 3]; decompression_out_bufs.push(this_decom_out_buf); decompression_out_offsets_bufs.push(this_decom_out_offsets_buf); diff --git a/rust/compression/fsst/src/fsst.rs b/rust/compression/fsst/src/fsst.rs index 0a2bf1d03d7..d00a6ed806b 100644 --- a/rust/compression/fsst/src/fsst.rs +++ b/rust/compression/fsst/src/fsst.rs @@ -57,6 +57,12 @@ use std::ptr; #[inline] fn fsst_unaligned_load_unchecked(v: *const u8) -> u64 { + // SAFETY: the caller must guarantee that `v` points to at least 8 readable bytes. All callers + // uphold this: `compress_bulk` loads from a 520-byte stack buffer at an offset < 511 (leaving + // >= 8 bytes), `build_symbol_table` guards the load with `word.len() > 7 && curr < word.len() - 7`, + // `find_longest_symbol_from_char_slice` copies into a stack `[u8; 8]` before loading, and + // `FsstDecoder::init` reads symbols from a `symbol_table` buffer already validated to be + // exactly `FSST_SYMBOL_TABLE_SIZE` bytes. unsafe { ptr::read_unaligned(v as *const u64) } } @@ -812,12 +818,31 @@ fn decompress_bulk( ) -> io::Result<()> { let symbols = decoder.symbols; let lens = decoder.lens; + // SAFETY invariant shared by every `unsafe` block in this closure: + // - `out` is sized to at least 8x `compressed_strs` (checked in `FsstDecoder::init`, which the + // sole public entry point always runs before reaching this function). Each code advances + // `out_curr` by `lens[code]`, which is 1..=8 for a well-formed symbol table, and each + // consumed input byte yields at most 8 output bytes, so `out_curr + 8 <= out.len()` at every + // 8-byte write, including the final one. This is why we can `write_unaligned` a full 8-byte + // word per code and advance by only the length. + // NOTE: `lens` is loaded verbatim from the (untrusted) symbol table and is NOT re-validated + // to be <= 8 on decode, and offsets (below) are likewise trusted. A corrupted table or + // offset buffer can violate these bounds; callers must supply structures produced by + // `compress` (or otherwise trusted). Hardening the decoder against corrupt input is a + // separate concern, not addressed here. + // - The only unchecked read is `read_unaligned::`, gated by `in_curr + 4 <= in_end`; the + // scalar paths use bounds-checked indexing. `in_end` is a caller-provided offset into + // `compressed_strs`; the read is sound only if `in_end <= compressed_strs.len()`, which is a + // trusted precondition (holds for encoder-produced offsets; not validated here). let mut decompress = |mut in_curr: usize, in_end: usize, out_curr: &mut usize| { // Do SIMD operation here by 4 bytes while in_curr + 4 <= in_end { let next_block; let mut code; let mut len; + // SAFETY: the loop guard proves `in_curr + 4 <= in_end`. Per the closure-level + // invariant, `in_end <= compressed_strs.len()` is a trusted precondition (not checked + // here), so the 4-byte read is in bounds for well-formed input. unsafe { next_block = ptr::read_unaligned(compressed_strs.as_ptr().add(in_curr) as *const u32); @@ -983,6 +1008,10 @@ fn decompress_bulk( if in_curr < in_end { // last code cannot be an escape code let code = compressed_strs[in_curr] as usize; + // SAFETY: see the closure-level invariant. This is the final write and has no + // subsequent write to cover its slack, so it is the tightest case: `out_curr` is at + // most 8*(consumed_input_bytes - 1), and the 8-byte store lands within `out.len()` + // precisely because the caller sized `out` to 8x the input. unsafe { let src = symbols[code]; ptr::write_unaligned(out.as_mut_ptr().add(*out_curr) as *mut u64, src); @@ -1188,8 +1217,19 @@ impl FsstDecoder { } self.decoder_switch_on = (st_info & (1 << 24)) != 0; - // when decoder_switch_on is true, we make sure the out_buf is at least 3 times the size of the in_buf, - if self.decoder_switch_on && in_buf.len() * 3 > out_buf.len() { + // A single 1-byte code can decode to a symbol of up to MAX_SYMBOL_LENGTH (8) bytes, so the + // decoded output can be up to 8x the input. `decompress_bulk` also relies on this bound: it + // writes a full 8-byte word per code (advancing only by the symbol length), so the output + // buffer must be large enough that even the final write stays in bounds. Require out_buf to + // be at least 8x in_buf. `checked_mul` guards against `in_buf.len() * 8` wrapping on 32-bit + // targets (an input >= 512 MiB would otherwise bypass the check); treat overflow as too + // small. + if self.decoder_switch_on + && in_buf + .len() + .checked_mul(8) + .is_none_or(|needed| needed > out_buf.len()) + { return Err(io::Error::new( io::ErrorKind::InvalidInput, "output buffer too small for FSST decoder", @@ -1288,8 +1328,10 @@ pub fn compress( // the following 32 bits after FSST_MAGIC contains information about FSST encoding, such as decoder_switch_on, suffix_lim, terminator, n_symbols // when the decoder_switch_on is off in the in_buf header, `decompress` first make sure the out_buf is at least the same size as the in_buf, then simply copy the // input data to the output -// when the decoder_switch_on is on, `decompress` first make sure the out_buf is at least 3 times the size of the in_buf, then start decoding the -// data using the symbol table +// when the decoder_switch_on is on, `decompress` first make sure the out_buf is at least 8 times the size of the in_buf, then start decoding the +// data using the symbol table. The 8x bound is required for correctness: a 1-byte code can expand to +// an 8-byte symbol, and the decode loop writes a full 8-byte word per code, so a smaller buffer can +// be written out of bounds. // the out_offsets_buf should be at least the same size as the in_offsets_buf, otherwise an error is returned // the symbol_table is the same symbol table created by `compression` pub fn decompress( @@ -1644,4 +1686,59 @@ But exactly how the acquaintance and friendship came about, we cannot say."; ); } } + + // Build a genuinely FSST-compressed (decoder_switch_on) buffer to exercise the decode-side + // output-buffer size contract. Returns (symbol_table, compressed_bytes, compressed_offsets). + fn compress_paragraph() -> ([u8; FSST_SYMBOL_TABLE_SIZE], Vec, Vec) { + let test_input = TEST_PARAGRAPH.repeat((1024 * 1024) / TEST_PARAGRAPH.len()); + let lines_vec = test_input.lines().collect::>(); + let string_array = StringArray::from(lines_vec); + let mut compress_output_buf: Vec = vec![0; string_array.value_data().len()]; + let mut compress_offset_buf: Vec = vec![0; string_array.value_offsets().len()]; + let mut symbol_table = [0; FSST_SYMBOL_TABLE_SIZE]; + compress( + symbol_table.as_mut(), + string_array.value_data(), + string_array.value_offsets(), + &mut compress_output_buf, + &mut compress_offset_buf, + ) + .unwrap(); + (symbol_table, compress_output_buf, compress_offset_buf) + } + + // The decoder writes a full 8-byte word per code, so the output buffer must be at least 8x the + // compressed input. A buffer sized below 8x must be rejected rather than written out of bounds. + #[test_log::test(tokio::test)] + async fn test_decompress_rejects_undersized_output_buffer() { + let (symbol_table, compressed, compressed_offsets) = compress_paragraph(); + // Sanity check: this input actually engaged FSST compression (decoder_switch_on). + let st_info = u64::from_ne_bytes(symbol_table[..8].try_into().unwrap()); + assert!(st_info & (1 << 24) != 0, "expected decoder_switch_on input"); + + // One byte short of the 8x requirement must be rejected. + let mut too_small = vec![0u8; compressed.len() * 8 - 1]; + let mut out_offsets = vec![0i32; compressed_offsets.len()]; + let err = decompress( + &symbol_table, + &compressed, + &compressed_offsets, + &mut too_small, + &mut out_offsets, + ) + .unwrap_err(); + assert_eq!(err.kind(), std::io::ErrorKind::InvalidInput); + + // Exactly 8x is the tight bound and must succeed. + let mut exact = vec![0u8; compressed.len() * 8]; + let mut out_offsets = vec![0i32; compressed_offsets.len()]; + decompress( + &symbol_table, + &compressed, + &compressed_offsets, + &mut exact, + &mut out_offsets, + ) + .unwrap(); + } } diff --git a/rust/examples/src/hnsw.rs b/rust/examples/src/hnsw.rs index 5be0debf6e1..52c15bd0851 100644 --- a/rust/examples/src/hnsw.rs +++ b/rust/examples/src/hnsw.rs @@ -129,6 +129,7 @@ async fn main() { lower_bound: None, upper_bound: None, dist_q_c: 0.0, + use_acorn: false, }; let results: HashSet = hnsw .search_basic(q.clone(), k, ¶ms, None, vector_store.as_ref()) diff --git a/rust/lance-arrow/Cargo.toml b/rust/lance-arrow/Cargo.toml index afbc9c26ed3..21513e638d8 100644 --- a/rust/lance-arrow/Cargo.toml +++ b/rust/lance-arrow/Cargo.toml @@ -21,6 +21,7 @@ arrow-ipc = { workspace = true } arrow-ord = { workspace = true } arrow-schema = { workspace = true } arrow-select = { workspace = true } +bytemuck = { workspace = true } bytes = { workspace = true } futures = { workspace = true } half = { workspace = true } diff --git a/rust/lance-arrow/README.md b/rust/lance-arrow/README.md index 8c3bedac3f3..f5feccaa191 100644 --- a/rust/lance-arrow/README.md +++ b/rust/lance-arrow/README.md @@ -1,7 +1,7 @@ # lance-arrow -`lance-arrow` is a internal sub-crate, containing [Apache-Arrow](https://github.com/apache/arrow-rs) -extensions used by [Lance](https://github.com/lancedb/lance). +`lance-arrow` is an internal sub-crate, containing [Apache-Arrow](https://github.com/apache/arrow-rs) +extensions used by [Lance](https://github.com/lance-format/lance). **Important Note**: This crate is **not intended for external usage**. diff --git a/rust/lance-arrow/src/bfloat16.rs b/rust/lance-arrow/src/bfloat16.rs index 2f59de51317..74c7f259a40 100644 --- a/rust/lance-arrow/src/bfloat16.rs +++ b/rust/lance-arrow/src/bfloat16.rs @@ -7,7 +7,7 @@ use std::fmt::Formatter; use std::slice; use arrow_array::{Array, FixedSizeBinaryArray, builder::BooleanBufferBuilder}; -use arrow_buffer::MutableBuffer; +use arrow_buffer::{Buffer, MutableBuffer}; use arrow_data::ArrayData; use arrow_schema::{ArrowError, DataType, Field as ArrowField}; use half::bf16; @@ -164,19 +164,18 @@ impl FromIterator for BFloat16Array { impl From> for BFloat16Array { fn from(data: Vec) -> Self { - let mut buffer = MutableBuffer::with_capacity(data.len() * 2); - - // Write each value's little-endian bytes straight into the buffer. Going - // through an intermediate `Vec` per element would allocate once per value. - for val in &data { - buffer.extend_from_slice(&val.to_bits().to_le_bytes()); - } - + let len = data.len(); + // Zero-copy: `bf16` is `#[repr(transparent)]` over `u16` and derives + // `bytemuck::Pod`, so `cast_vec` reinterprets the allocation in place — + // no per-element copy or heap alloc. The crate-root `compile_error!` + // pins `target_endian = "little"`, so the resulting bytes match the + // `FixedSizeBinary(2)` on-disk order Lance writes elsewhere. + let raw: Vec = bytemuck::cast_vec(data); let array_data = ArrayData::builder(DataType::FixedSizeBinary(2)) - .len(data.len()) - .add_buffer(buffer.into()); - // SAFETY: the buffer contains exactly `2 * data.len()` bytes — each - // `bf16` writes its two little-endian bytes once — matching the + .len(len) + .add_buffer(Buffer::from_vec(raw)); + // SAFETY: the value buffer contains exactly `2 * len` bytes — one + // `u16` per element after the layout-compatible cast — matching the // `FixedSizeBinary(2)` storage layout. No null buffer is attached, so // every element is logically valid. let array_data = unsafe { array_data.build_unchecked() }; @@ -284,9 +283,10 @@ impl FloatArray for FixedSizeBinaryArray { /// - `value_length()` must be 2 (the `FixedSizeBinary(2)` storage shape /// used by [`BFloat16Array`]). Asserted at entry. /// - The value buffer must be at least 2-byte aligned. Lance's in-tree - /// constructors always satisfy this (every value buffer goes through - /// `MutableBuffer`, which is aligned to arrow-buffer's `ALIGNMENT` - /// constant — ≥32 bytes on every supported target). Externally-built + /// constructors always satisfy this: value buffers are built either via + /// `MutableBuffer` (aligned to arrow-buffer's `ALIGNMENT` constant, ≥32 + /// bytes) or via `Buffer::from_vec::` (aligned to `align_of::()` + /// == 2); both meet `bf16`'s 2-byte requirement. Externally-built /// `FixedSizeBinaryArray`s arriving via FFI, IPC, or /// `Buffer::from_custom_allocation` are not required by arrow-rs to be /// aligned beyond a single byte; passing one to this method violates the @@ -329,8 +329,8 @@ impl FloatArray for FixedSizeBinaryArray { // (arrow-data `data.rs`), so arrow-rs alone does not guarantee // 2-byte alignment. Lance's in-tree construction paths build value // buffers via `MutableBuffer` (arrow-buffer `ALIGNMENT` constant, - // ≥32 bytes on every supported target), which trivially satisfies - // `bf16`'s 2-byte requirement. + // ≥32 bytes) or `Buffer::from_vec::` (2-byte aligned), both of + // which satisfy `bf16`'s 2-byte requirement. // - The returned slice borrows from `self`; the underlying ref-counted, // immutable Arrow buffer cannot be mutated or freed for the slice's // lifetime. @@ -361,6 +361,16 @@ mod tests { assert_eq!(array, array2); assert_eq!(array.len(), 3); + // Pin the raw little-endian bytes emitted by `From>` (rewritten to + // reinterpret the Vec via `bytemuck::cast_vec`), so a layout/byte-order + // regression is caught directly rather than only through Debug formatting. + // bf16 is the high 16 bits of the f32: 1.0->0x3F80, 2.0->0x4000, 3.0->0x4040. + let inner = array2.clone().into_inner(); + let raw_bytes: Vec = (0..inner.len()) + .flat_map(|i| inner.value(i).to_vec()) + .collect(); + assert_eq!(raw_bytes, vec![0x80, 0x3F, 0x00, 0x40, 0x40, 0x40]); + let expected_fmt = "BFloat16Array\n[\n 1.0,\n 2.0,\n 3.0,\n]"; assert_eq!(expected_fmt, format!("{:?}", array)); diff --git a/rust/lance-arrow/src/lib.rs b/rust/lance-arrow/src/lib.rs index a55b42cb6c0..942fdf7d9c0 100644 --- a/rust/lance-arrow/src/lib.rs +++ b/rust/lance-arrow/src/lib.rs @@ -1017,40 +1017,25 @@ fn merge_list_struct(left: &dyn Array, right: &dyn Array) -> Arc { } } -/// Helper function to normalize validity buffers -/// Returns None for all-null validity (placeholder structs) -fn normalize_validity( - validity: Option<&arrow_buffer::NullBuffer>, -) -> Option<&arrow_buffer::NullBuffer> { - validity.and_then(|v| { - if v.null_count() == v.len() { - None - } else { - Some(v) - } - }) -} - -/// Helper function to merge validity buffers from two struct arrays -/// Returns None only if both arrays are null at the same position +/// Helper function to merge validity buffers from two struct arrays. /// -/// Special handling for placeholder structs (all-null validity) +/// A row is valid if it is valid in either input. +/// An absent validity buffer means all rows are valid, an all-null buffer acts as the identity for this merge. fn merge_struct_validity( left_validity: Option<&arrow_buffer::NullBuffer>, right_validity: Option<&arrow_buffer::NullBuffer>, ) -> Option { - // Normalize both validity buffers (convert all-null to None) - let left_normalized = normalize_validity(left_validity); - let right_normalized = normalize_validity(right_validity); - - match (left_normalized, right_normalized) { + match (left_validity, right_validity) { // Fast paths: no computation needed - (None, None) => None, - (Some(left), None) => Some(left.clone()), - (None, Some(right)) => Some(right.clone()), + (None, _) | (_, None) => None, (Some(left), Some(right)) => { - // Fast path: if both have no nulls, can return either one - if left.null_count() == 0 && right.null_count() == 0 { + if left.null_count() == 0 || right.null_count() == 0 { + return None; + } + if left.null_count() == left.len() { + return Some(right.clone()); + } + if right.null_count() == right.len() { return Some(left.clone()); } @@ -1385,9 +1370,12 @@ fn merge_with_schema( ); let merged_validity = merge_struct_validity(left_list.nulls(), right_list.nulls()); + // `trimmed_values` starts at the first used value, so offsets + // must be shifted to match or `ListArray::new` panics when the + // input list was sliced (e.g. from a filtered batch). let merged_list = ListArray::new( child_field.clone(), - left_list.offsets().clone(), + left_list.trimmed_offsets(), merged_values, merged_validity, ); @@ -1412,7 +1400,7 @@ fn merge_with_schema( merge_struct_validity(left_list.nulls(), right_list.nulls()); let merged_list = LargeListArray::new( child_field.clone(), - left_list.offsets().clone(), + left_list.trimmed_offsets(), merged_values, merged_validity, ); @@ -2054,6 +2042,47 @@ mod tests { assert_eq!(width_values.value(0), 300); assert_eq!(width_values.value(1), 200); assert!(width_values.is_null(2)); // width is null when right struct was null + + // An all-null validity buffer is data, not a placeholder meaning "this side has no + // validity": merging two of them keeps the rows null. + let all_null_left = StructArray::new( + Fields::from(vec![Field::new("height", DataType::Int32, true)]), + vec![Arc::new(Int32Array::from(vec![None, None])) as ArrayRef], + Some(vec![false, false].into()), + ); + let all_null_right = StructArray::new( + Fields::from(vec![Field::new("width", DataType::Int32, true)]), + vec![Arc::new(Int32Array::from(vec![None, None])) as ArrayRef], + Some(vec![false, false].into()), + ); + + let merged = merge(&all_null_left, &all_null_right); + assert_eq!(merged.null_count(), 2); + + // An all-null side is the identity of the merge, so the other side decides each row. + let partial_left = StructArray::new( + Fields::from(vec![Field::new("height", DataType::Int32, true)]), + vec![Arc::new(Int32Array::from(vec![Some(1), None])) as ArrayRef], + Some(vec![true, false].into()), + ); + let merged = merge(&partial_left, &all_null_right); + assert!(!merged.is_null(0)); + assert!(merged.is_null(1)); + + // A missing validity buffer means all rows are valid, which absorbs an all-null side. + let all_valid_left = StructArray::new( + Fields::from(vec![Field::new("height", DataType::Int32, true)]), + vec![Arc::new(Int32Array::from(vec![1, 2])) as ArrayRef], + None, + ); + let merged = merge(&all_valid_left, &all_null_right); + assert_eq!(merged.null_count(), 0); + + // An explicit all-valid buffer has the same semantics as a missing buffer. + let all_valid: arrow_buffer::NullBuffer = vec![true, true].into(); + let partial: arrow_buffer::NullBuffer = vec![true, false].into(); + assert!(merge_struct_validity(Some(&all_valid), Some(&partial)).is_none()); + assert!(merge_struct_validity(Some(&partial), Some(&all_valid)).is_none()); } #[test] @@ -2380,6 +2409,118 @@ mod tests { assert_eq!(merged_array.len(), 2); } + #[test] + fn test_merge_with_schema_sliced_list_struct() { + test_merge_with_schema_sliced_list_struct_generic::(); + } + + #[test] + fn test_merge_with_schema_sliced_large_list_struct() { + test_merge_with_schema_sliced_list_struct_generic::(); + } + + // Regression for #6580: merge_with_schema panicked when the left list was a + // sliced view whose offsets did not start at zero (common after a filtered + // scan). Cloning those offsets alongside `trimmed_values` produced offsets + // larger than the trimmed child, panicking in `(Large)ListArray::new`. + fn test_merge_with_schema_sliced_list_struct_generic() { + let make_list_dtype = |item_field: Arc| { + if O::IS_LARGE { + DataType::LargeList(item_field) + } else { + DataType::List(item_field) + } + }; + + // Build a List with two rows of 5 items each, then slice away + // the first row so the remaining list's offsets start at 5, not 0. + let struct_fields_a = Fields::from(vec![Field::new("a", DataType::Int32, true)]); + let left_values = Arc::new(StructArray::new( + struct_fields_a.clone(), + vec![Arc::new(Int32Array::from_iter_values(0..10)) as ArrayRef], + None, + )); + let full_list = GenericListArray::::new( + Arc::new(Field::new("item", DataType::Struct(struct_fields_a), true)), + OffsetBuffer::::from_lengths([5, 5]), + left_values, + None, + ); + let sliced_left = full_list.slice(1, 1); + assert_eq!(sliced_left.offsets()[0].as_usize(), 5); + assert_eq!(sliced_left.offsets()[1].as_usize(), 10); + + let struct_fields_b = Fields::from(vec![Field::new("b", DataType::Int32, true)]); + let right_values = Arc::new(StructArray::new( + struct_fields_b.clone(), + vec![Arc::new(Int32Array::from_iter_values(100..105)) as ArrayRef], + None, + )); + let right_list = GenericListArray::::new( + Arc::new(Field::new("item", DataType::Struct(struct_fields_b), true)), + OffsetBuffer::::from_lengths([5]), + right_values, + None, + ); + + let target_item_field = Arc::new(Field::new( + "item", + DataType::Struct(Fields::from(vec![ + Field::new("a", DataType::Int32, true), + Field::new("b", DataType::Int32, true), + ])), + true, + )); + let target_fields = Fields::from(vec![Field::new( + "items", + make_list_dtype(target_item_field), + true, + )]); + + let left_batch = RecordBatch::try_new( + Arc::new(Schema::new(vec![Field::new( + "items", + sliced_left.data_type().clone(), + true, + )])), + vec![Arc::new(sliced_left) as ArrayRef], + ) + .unwrap(); + let right_batch = RecordBatch::try_new( + Arc::new(Schema::new(vec![Field::new( + "items", + right_list.data_type().clone(), + true, + )])), + vec![Arc::new(right_list) as ArrayRef], + ) + .unwrap(); + + let merged = left_batch + .merge_with_schema(&right_batch, &Schema::new(target_fields.to_vec())) + .unwrap(); + + let merged_list = merged + .column_by_name("items") + .unwrap() + .as_any() + .downcast_ref::>() + .unwrap(); + assert_eq!(merged_list.len(), 1); + assert_eq!(merged_list.value_length(0).as_usize(), 5); + let merged_struct = merged_list.values().as_struct(); + assert_eq!(merged_struct.num_columns(), 2); + let a = merged_struct + .column_by_name("a") + .unwrap() + .as_any() + .downcast_ref::() + .unwrap(); + // After shifting offsets to zero, values 5..10 should be first. + let a_vals: Vec = a.iter().map(|v| v.unwrap()).collect(); + assert_eq!(a_vals, vec![5, 6, 7, 8, 9]); + } + #[test] fn test_project_by_schema_list_struct_reorder() { // Test that project_by_schema correctly reorders fields inside List diff --git a/rust/lance-arrow/src/list.rs b/rust/lance-arrow/src/list.rs index 0c24fc579da..06b0fc592cf 100644 --- a/rust/lance-arrow/src/list.rs +++ b/rust/lance-arrow/src/list.rs @@ -23,6 +23,16 @@ pub trait ListArrayExt { /// behaves similarly to `values()` except it slices the array so that it starts at /// the first list offset and ends at the last list offset. fn trimmed_values(&self) -> Arc; + /// The offset type of the underlying list array. + type Offset: OffsetSizeTrait; + /// Returns offsets shifted so the first offset is zero, matching + /// [`Self::trimmed_values`]. + /// + /// Sliced list arrays (e.g. a filtered batch) keep offsets that reference the + /// original values buffer, so combining them with trimmed values produces + /// offsets that exceed the values length. Use this together with + /// `trimmed_values` when constructing a new list array. + fn trimmed_offsets(&self) -> OffsetBuffer; } impl ListArrayExt for GenericListArray { @@ -90,6 +100,20 @@ impl ListArrayExt for GenericListArray .unwrap_or(0); self.values().slice(first_value, last_value - first_value) } + + type Offset = OffsetSize; + + fn trimmed_offsets(&self) -> OffsetBuffer { + let offsets = self.offsets(); + let Some(&first) = offsets.first() else { + return offsets.clone(); + }; + if first == OffsetSize::zero() { + return offsets.clone(); + } + let shifted: Vec = offsets.iter().map(|&o| o - first).collect(); + OffsetBuffer::new(ScalarBuffer::from(shifted)) + } } #[cfg(test)] diff --git a/rust/lance-arrow/src/scalar.rs b/rust/lance-arrow/src/scalar.rs index e9fd2516f17..6003eb13497 100644 --- a/rust/lance-arrow/src/scalar.rs +++ b/rust/lance-arrow/src/scalar.rs @@ -1,10 +1,11 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright The Lance Authors -use arrow_array::{ArrayRef, make_array}; +use arrow_array::{ArrayRef, UInt64Array, make_array}; use arrow_buffer::Buffer; -use arrow_data::{ArrayDataBuilder, transform::MutableArrayData}; +use arrow_data::ArrayDataBuilder; use arrow_schema::{ArrowError, DataType}; +use arrow_select::take::take; use crate::DataTypeExt; @@ -19,10 +20,7 @@ pub fn extract_scalar_value(array: &ArrayRef, idx: usize) -> Result { )); } - let data = array.to_data(); - let mut mutable = MutableArrayData::new(vec![&data], /*use_nulls=*/ true, 1); - mutable.extend(0, idx, idx + 1); - Ok(make_array(mutable.freeze())) + take(array.as_ref(), &UInt64Array::from(vec![idx as u64]), None) } fn read_u32(buf: &[u8], offset: &mut usize) -> Result { @@ -195,7 +193,10 @@ pub fn try_inline_value(scalar: &ArrayRef) -> Option> { mod tests { use std::sync::Arc; - use arrow_array::{BooleanArray, FixedSizeBinaryArray, Int32Array, StringArray, cast::AsArray}; + use arrow_array::{ + BooleanArray, DictionaryArray, FixedSizeBinaryArray, Int8Array, Int32Array, StringArray, + cast::AsArray, types::Int8Type, + }; use super::*; @@ -212,6 +213,24 @@ mod tests { ); } + #[test] + fn test_extract_scalar_value_from_full_dictionary() { + let values = Arc::new(StringArray::from( + (0..=i8::MAX) + .map(|value| format!("value-{value}")) + .collect::>(), + )); + let keys = Int8Array::from((0..=i8::MAX).collect::>()); + let array: ArrayRef = Arc::new(DictionaryArray::::new(keys, values)); + + let scalar = extract_scalar_value(&array, i8::MAX as usize).unwrap(); + + let scalar = scalar.as_dictionary::(); + assert_eq!(scalar.len(), 1); + assert_eq!(scalar.key(0), Some(i8::MAX as usize)); + assert_eq!(scalar.values().len(), i8::MAX as usize + 1); + } + #[test] fn test_scalar_value_buffer_utf8_round_trip() { let scalar: ArrayRef = Arc::new(StringArray::from(vec!["hello"])); diff --git a/rust/lance-core/Cargo.toml b/rust/lance-core/Cargo.toml index 7f956c70430..8e7b99869cc 100644 --- a/rust/lance-core/Cargo.toml +++ b/rust/lance-core/Cargo.toml @@ -18,6 +18,7 @@ arrow-data.workspace = true arrow-schema.workspace = true async-trait.workspace = true lance-arrow.workspace = true +blake3.workspace = true byteorder.workspace = true bytes.workspace = true datafusion-common = { workspace = true, optional = true } @@ -28,6 +29,7 @@ itertools.workspace = true libc.workspace = true libm.workspace = true moka.workspace = true +quick_cache = "0.6" num_cpus = "1.0" object_store = { workspace = true } pin-project.workspace = true @@ -51,11 +53,20 @@ log.workspace = true libc = { version = "0.2" } [dev-dependencies] +criterion.workspace = true proptest.workspace = true rstest.workspace = true [features] +# Capture Rust backtraces in error types. When disabled (the default), +# the backtrace field is zero-sized with no overhead. At runtime, capture +# is still gated by RUST_BACKTRACE=1. +backtrace = [] datafusion = ["dep:datafusion-common", "dep:datafusion-sql"] +[[bench]] +name = "cache_keys" +harness = false + [lints] workspace = true diff --git a/rust/lance-core/benches/cache_keys.rs b/rust/lance-core/benches/cache_keys.rs new file mode 100644 index 00000000000..10f3917f10e --- /dev/null +++ b/rust/lance-core/benches/cache_keys.rs @@ -0,0 +1,365 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright The Lance Authors + +use std::any::Any; +use std::borrow::Cow; +use std::hash::{BuildHasher, RandomState}; +use std::hint::black_box; +use std::sync::atomic::{AtomicU64, Ordering}; +use std::sync::{Arc, Weak}; + +use async_trait::async_trait; +use criterion::{BenchmarkId, Criterion, criterion_group, criterion_main}; +use futures::FutureExt; +use lance_core::cache::{ + CacheKey, CacheKeySchema, CacheNamespace, KeyBuilder, LanceCache, WeakLanceCache, +}; + +struct PageKey { + column_index: u32, + page_index: u64, +} + +impl CacheKey for PageKey { + type ValueType = Vec; + + fn key(&self) -> Cow<'_, str> { + format!("{}-{}", self.column_index, self.page_index).into() + } + + fn type_name() -> &'static str { + "bench.Page" + } + + fn schema() -> CacheKeySchema { + CacheKeySchema::new("bench.page-key", 1) + } + + fn write_key(&self, builder: &mut KeyBuilder) { + builder.write_u32(self.column_index); + builder.write_u64(self.page_index); + } +} + +#[derive(Clone, Eq, Hash, PartialEq)] +struct LegacyPhysicalKey { + namespace: Arc, + logical_key: Arc, + type_name: &'static str, +} + +type LegacyEntryValue = Arc; + +#[derive(Clone)] +struct LegacyEntry { + value: LegacyEntryValue, + size_bytes: usize, +} + +#[async_trait] +trait LegacyBackend: Send + Sync { + async fn get(&self, key: &LegacyPhysicalKey) -> Option; + async fn insert(&self, key: &LegacyPhysicalKey, value: LegacyEntryValue, size_bytes: usize); +} + +struct LegacyMokaBackend { + cache: moka::future::Cache, +} + +impl LegacyMokaBackend { + fn with_capacity(capacity: usize) -> Self { + let cache = moka::future::Cache::builder() + .max_capacity(capacity as u64) + .weigher(|key: &LegacyPhysicalKey, entry: &LegacyEntry| { + std::mem::size_of::() + .saturating_add(key.logical_key.len()) + .saturating_add(entry.size_bytes) + .try_into() + .unwrap_or(u32::MAX) + }) + .support_invalidation_closures() + .build(); + Self { cache } + } +} + +#[async_trait] +impl LegacyBackend for LegacyMokaBackend { + async fn get(&self, key: &LegacyPhysicalKey) -> Option { + self.cache.get(key).await.map(|entry| entry.value) + } + + async fn insert(&self, key: &LegacyPhysicalKey, value: LegacyEntryValue, size_bytes: usize) { + self.cache + .insert(key.clone(), LegacyEntry { value, size_bytes }) + .await; + } +} + +#[derive(Clone)] +struct LegacyCache { + backend: Arc, + namespace: Arc, + hits: Arc, + misses: Arc, +} + +impl LegacyCache { + fn with_capacity(capacity: usize) -> Self { + Self { + backend: Arc::new(LegacyMokaBackend::with_capacity(capacity)), + namespace: Arc::from(""), + hits: Arc::new(AtomicU64::new(0)), + misses: Arc::new(AtomicU64::new(0)), + } + } + + fn with_key_prefix(&self, segment: &str) -> Self { + Self { + backend: self.backend.clone(), + namespace: Arc::from(format!("{}{segment}/", self.namespace)), + hits: self.hits.clone(), + misses: self.misses.clone(), + } + } + + fn physical_key(&self, key: &PageKey) -> LegacyPhysicalKey { + let logical_key = key.key(); + LegacyPhysicalKey { + namespace: self.namespace.clone(), + logical_key: Arc::from(logical_key.as_ref()), + type_name: PageKey::type_name(), + } + } + + async fn insert(&self, key: &PageKey, value: Arc>) { + let size_bytes = std::mem::size_of::>() + + value.capacity() + + std::mem::size_of::() * 2; + self.backend + .insert(&self.physical_key(key), value, size_bytes) + .boxed() + .await; + } + + async fn get(&self, key: &PageKey) -> Option>> { + async { + let Some(value) = self.backend.get(&self.physical_key(key)).await else { + self.misses.fetch_add(1, Ordering::Relaxed); + return None; + }; + match value.downcast::>() { + Ok(value) => { + self.hits.fetch_add(1, Ordering::Relaxed); + Some(value) + } + Err(_) => { + self.misses.fetch_add(1, Ordering::Relaxed); + None + } + } + } + .boxed() + .await + } +} + +struct LegacyWeakCache { + backend: Weak, + namespace: Arc, + hits: Arc, + misses: Arc, +} + +impl LegacyWeakCache { + fn from(cache: &LegacyCache) -> Self { + Self { + backend: Arc::downgrade(&cache.backend), + namespace: cache.namespace.clone(), + hits: cache.hits.clone(), + misses: cache.misses.clone(), + } + } + + async fn get(&self, key: &PageKey) -> Option>> { + let backend = self.backend.upgrade()?; + let logical_key = key.key(); + let physical_key = LegacyPhysicalKey { + namespace: self.namespace.clone(), + logical_key: Arc::from(logical_key.as_ref()), + type_name: PageKey::type_name(), + }; + let Some(value) = backend.get(&physical_key).await else { + self.misses.fetch_add(1, Ordering::Relaxed); + return None; + }; + match value.downcast::>() { + Ok(value) => { + self.hits.fetch_add(1, Ordering::Relaxed); + Some(value) + } + Err(_) => { + self.misses.fetch_add(1, Ordering::Relaxed); + None + } + } + } +} + +fn benchmark_key_preparation(c: &mut Criterion) { + let key = PageKey { + column_index: 17, + page_index: 42, + }; + let outer_hashes = RandomState::new(); + let prefixes: [(&str, Arc); 2] = [ + ("short", Arc::from("dataset")), + ("long", Arc::from("p".repeat(1024))), + ]; + let mut group = c.benchmark_group("cache_key_preparation"); + + for (case, prefix) in prefixes { + let namespace = CacheNamespace::root().child(&prefix); + + group.bench_with_input(BenchmarkId::new("legacy", case), &prefix, |b, prefix| { + b.iter(|| { + let logical_key = key.key(); + let physical_key = LegacyPhysicalKey { + namespace: Arc::clone(prefix), + logical_key: Arc::from(logical_key.as_ref()), + type_name: PageKey::type_name(), + }; + black_box(outer_hashes.hash_one(physical_key)) + }) + }); + + group.bench_function(BenchmarkId::new("blake3_typed", case), |b| { + b.iter(|| { + let mut builder = + KeyBuilder::new(namespace, PageKey::stable_type_id(), PageKey::schema()); + key.write_key(&mut builder); + black_box(outer_hashes.hash_one(builder.finish())) + }) + }); + } + + group.finish(); +} + +fn benchmark_namespace_derivation(c: &mut Criterion) { + let root = CacheNamespace::root(); + let long_segment = "p".repeat(160); + let mut group = c.benchmark_group("cache_namespace_derivation"); + + group.bench_function("root", |b| { + b.iter(|| black_box(CacheNamespace::root())); + }); + group.bench_with_input( + BenchmarkId::new("child", "short"), + &"dataset", + |b, segment| { + b.iter(|| black_box(root.child(black_box(segment)))); + }, + ); + group.bench_with_input( + BenchmarkId::new("child", "long"), + &long_segment, + |b, segment| { + b.iter(|| black_box(root.child(black_box(segment)))); + }, + ); + + group.finish(); +} + +fn benchmark_cache_operations(c: &mut Criterion) { + const CAPACITY: usize = 64 * 1024; + const ROTATING_KEYS: u64 = 512; + + let runtime = tokio::runtime::Runtime::new().unwrap(); + let prefix = "p".repeat(1024); + let legacy = LegacyCache::with_capacity(CAPACITY).with_key_prefix(&prefix); + let legacy_weak = LegacyWeakCache::from(&legacy); + let fixed = LanceCache::with_capacity(CAPACITY).with_key_prefix(&prefix); + let fixed_weak = WeakLanceCache::from(&fixed); + let hit_key = PageKey { + column_index: 17, + page_index: 42, + }; + runtime.block_on(async { + legacy.insert(&hit_key, Arc::new(vec![1_u8; 32])).await; + fixed + .insert_with_key(&hit_key, Arc::new(vec![1_u8; 32])) + .await; + }); + + let mut group = c.benchmark_group("cache_operations"); + group.bench_function(BenchmarkId::new("strong_warmed_hit", "legacy"), |b| { + b.to_async(&runtime) + .iter(|| legacy.get(black_box(&hit_key))); + }); + group.bench_function(BenchmarkId::new("strong_warmed_hit", "fixed"), |b| { + b.to_async(&runtime) + .iter(|| fixed.get_with_key(black_box(&hit_key))); + }); + group.bench_function(BenchmarkId::new("weak_warmed_hit", "legacy"), |b| { + b.to_async(&runtime) + .iter(|| legacy_weak.get(black_box(&hit_key))); + }); + group.bench_function(BenchmarkId::new("weak_warmed_hit", "fixed"), |b| { + b.to_async(&runtime) + .iter(|| fixed_weak.get_with_key(black_box(&hit_key))); + }); + + let values: Vec<_> = (0..16).map(|value| Arc::new(vec![value; 32])).collect(); + let next_legacy_insert = AtomicU64::new(0); + group.bench_function(BenchmarkId::new("bounded_rotating_insert", "legacy"), |b| { + b.to_async(&runtime).iter(|| { + let sequence = next_legacy_insert.fetch_add(1, Ordering::Relaxed); + let page_index = sequence % ROTATING_KEYS; + let value = Arc::clone(&values[sequence as usize % values.len()]); + let legacy = &legacy; + async move { + legacy + .insert( + &PageKey { + column_index: 17, + page_index, + }, + value, + ) + .await; + } + }); + }); + let next_fixed_insert = AtomicU64::new(0); + group.bench_function(BenchmarkId::new("bounded_rotating_insert", "fixed"), |b| { + b.to_async(&runtime).iter(|| { + let sequence = next_fixed_insert.fetch_add(1, Ordering::Relaxed); + let page_index = sequence % ROTATING_KEYS; + let value = Arc::clone(&values[sequence as usize % values.len()]); + let fixed = &fixed; + async move { + fixed + .insert_with_key( + &PageKey { + column_index: 17, + page_index, + }, + value, + ) + .await; + } + }); + }); + + group.finish(); +} + +criterion_group!( + benches, + benchmark_key_preparation, + benchmark_namespace_derivation, + benchmark_cache_operations +); +criterion_main!(benches); diff --git a/rust/lance-core/src/cache/backend.rs b/rust/lance-core/src/cache/backend.rs index 9307868f399..fdd612f1046 100644 --- a/rust/lance-core/src/cache/backend.rs +++ b/rust/lance-core/src/cache/backend.rs @@ -4,9 +4,31 @@ //! Backend interface for cache implementors. //! //! This module defines the trait that custom cache backends must implement, -//! along with the key and entry types they operate on. Most callers should +//! along with the entry type they operate on. Most callers should //! use [`LanceCache`](super::LanceCache) instead of interacting with //! backends directly. +//! +//! # Migrating custom backends +//! +//! Cache keys are opaque 16-byte values. Store +//! [`InternalCacheKey::as_bytes`] directly instead of decomposing a logical +//! prefix, key string, and Rust type name. The physical namespace must also +//! include [`CACHE_KEY_FORMAT`](super::CACHE_KEY_FORMAT), so a future key +//! protocol produces cold misses instead of aliases. Persistent or tiered +//! backends can route serializable values with [`CacheCodec::type_id`]. +//! +//! Prefix invalidation and key inventory are intentionally not part of this +//! interface: one-way digests cannot support either operation without +//! retaining the logical strings that fixed-size keys are designed to remove. +//! Existing callers should migrate removed symbols as follows: +//! - replace `with_backend_and_prefix(backend, prefix)` with +//! [`LanceCache::with_backend`](super::LanceCache::with_backend) followed by +//! [`LanceCache::with_key_prefix`](super::LanceCache::with_key_prefix); +//! - replace `invalidate_prefix` with [`LanceCache::clear`](super::LanceCache::clear) +//! when clearing the shared backend is acceptable, or rotate a versioned +//! namespace to leave older entries to age out; +//! - remove uses of `prefix`, `keys`, and session key-inventory methods; opaque +//! keys have no readable or enumerable equivalent. use std::any::Any; use std::pin::Pin; @@ -16,57 +38,13 @@ use async_trait::async_trait; use futures::Future; use crate::Result; +use crate::deepsize::Context; -use super::CacheCodec; +use super::{CacheCodec, InternalCacheKey}; /// A type-erased cache entry. pub type CacheEntry = Arc; -/// Iterator over cache keys currently known to a backend. -pub type CacheKeyIterator<'a> = Box + Send + 'a>; - -/// Structured cache key passed to [`CacheBackend`] methods. -/// -/// CacheBackend impls receive these ready-made from [`LanceCache`](super::LanceCache) -/// — you do not construct them yourself. Composed of three parts: -/// - **prefix**: scopes the key to a dataset or index (e.g. `"s3://bucket/dataset/"`) -/// - **key**: identifies the specific entry (e.g. `"42"` for a version number) -/// - **type_name**: distinguishes different value types stored under the same -/// user key (e.g. `"Vec"`) -#[derive(Clone, Debug, Hash, PartialEq, Eq)] -pub struct InternalCacheKey { - prefix: Arc, - key: Arc, - type_name: &'static str, -} - -impl InternalCacheKey { - pub fn new(prefix: Arc, key: Arc, type_name: &'static str) -> Self { - Self { - prefix, - key, - type_name, - } - } - - pub fn prefix(&self) -> &str { - &self.prefix - } - - pub fn key(&self) -> &str { - &self.key - } - - pub fn type_name(&self) -> &'static str { - self.type_name - } - - /// Returns true if this key's prefix starts with the given string. - pub fn starts_with(&self, prefix: &str) -> bool { - self.prefix.starts_with(prefix) - } -} - /// Low-level pluggable cache backend. /// /// Implementations store entries keyed by [`InternalCacheKey`] and return @@ -113,21 +91,9 @@ pub trait CacheBackend: Send + Sync + std::fmt::Debug { codec: Option, ) -> Result<(CacheEntry, bool)>; - /// Remove all entries whose prefix starts with the given string. - async fn invalidate_prefix(&self, prefix: &str); - /// Remove all entries. async fn clear(&self); - /// Return an iterator over cache keys currently known to this backend. - /// - /// Backends that cannot enumerate keys cheaply or accurately should return - /// `None`. An empty iterator means key inventory is supported and the - /// cache currently has no entries. - async fn keys(&self) -> Option> { - None - } - /// Number of entries currently stored (may flush pending operations). async fn num_entries(&self) -> usize; @@ -141,7 +107,7 @@ pub trait CacheBackend: Send + Sync + std::fmt::Debug { } /// Approximate weighted size in bytes, callable from synchronous contexts. - /// Used by `DeepSizeOf` to report cache memory usage. + /// Used as a `DeepSizeOf` fallback when exact entry traversal is unavailable. /// Backends that cannot provide this cheaply should return 0. /// /// Assumes entries do not share underlying buffers; if they do, the @@ -149,4 +115,23 @@ pub trait CacheBackend: Send + Sync + std::fmt::Debug { fn approx_size_bytes(&self) -> usize { 0 } + + /// Computes the size of the entries currently held in memory. + /// + /// `size_of_entry` threads a shared [`Context`] through each value so + /// allocations shared by multiple entries are counted once. It returns + /// `None` when the value's concrete type was not registered by + /// [`LanceCache`](super::LanceCache); implementations should use the + /// entry's declared eviction size as a fallback in that case. + /// + /// Backends that can enumerate their in-memory entries should include the + /// physical key footprint in the returned total. The default returns + /// `None`, causing `LanceCache` to use [`approx_size_bytes`](Self::approx_size_bytes). + fn deep_size_of_entries( + &self, + _context: &mut Context, + _size_of_entry: &dyn Fn(&CacheEntry, &mut Context) -> Option, + ) -> Option { + None + } } diff --git a/rust/lance-core/src/cache/backend_uri.rs b/rust/lance-core/src/cache/backend_uri.rs new file mode 100644 index 00000000000..33daa6adbe3 --- /dev/null +++ b/rust/lance-core/src/cache/backend_uri.rs @@ -0,0 +1,265 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright The Lance Authors + +//! URI-based configuration for cache backends. +//! +//! [`build_from_uri`] parses a compact string form such as +//! `moka://?capacity=1073741824` into a [`BackendConfig`] and hands it to +//! the registry. This gives Python/Java bindings and configuration files a +//! single-string representation of a backend without having to expose a +//! typed builder for every backend. +//! +//! Grammar (intentionally a subset of RFC 3986 — Lance only needs a +//! predictable, unambiguous form): +//! +//! ```text +//! uri ::= scheme ":" hier ( "?" query )? +//! scheme ::= ALPHA ( ALPHA | DIGIT | "+" | "-" | "." )* +//! hier ::= "//" authority path? -- e.g. moka://?..., other:///path?... +//! | path -- e.g. moka:capacity=... (rare) +//! authority ::= *( any char except "/" | "?" ) +//! path ::= *( any char except "?" ) +//! query ::= pair ( "&" pair )* +//! pair ::= key "=" value -- both percent-decoded +//! ``` +//! +//! Mapping to [`BackendConfig`]: +//! +//! * `scheme` becomes `kind`. +//! * The joined `authority + path` (with any leading `//` stripped) is stored +//! under the option key `path` when non-empty. Empty-authority absolute +//! paths such as `backend:///tmp/cache` keep their leading `/`; host-style +//! paths such as `backend://localhost:6379/0` are stored as +//! `localhost:6379/0`. If the query already contains a `path` key, the +//! URI-supplied path wins and the parser errors on the conflict. +//! * Each `key=value` pair from the query becomes an entry in `options`. +//! Duplicate keys are rejected. + +use std::sync::Arc; + +use super::backend::CacheBackend; +use super::registry::{BackendConfig, build_from_config, normalize_backend_kind}; +use crate::{Error, Result}; + +/// Parse `uri` into a [`BackendConfig`] and build the backend registered +/// under its `scheme`. +/// +/// Returns an error if: +/// * the URI cannot be parsed, +/// * no backend is registered for that scheme, or +/// * the constructor itself fails. +pub fn build_from_uri(uri: &str) -> Result> { + let config = parse_backend_uri(uri)?; + build_from_config(&config) +} + +/// Parse `uri` into a [`BackendConfig`] without touching the registry. +/// +/// See the module docs for the accepted grammar. +pub fn parse_backend_uri(uri: &str) -> Result { + let (scheme, rest) = split_scheme(uri)?; + let (path, query) = split_path_query(rest); + + let mut config = BackendConfig::new(&scheme)?; + + let normalized_path = normalize_path(path); + if !normalized_path.is_empty() { + config.options.insert("path".to_string(), normalized_path); + } + + if let Some(query) = query { + for raw_pair in query.split('&') { + if raw_pair.is_empty() { + continue; + } + let (raw_key, raw_value) = raw_pair.split_once('=').ok_or_else(|| { + Error::invalid_input(format!( + "cache backend uri {:?}: query pair {:?} is missing '='", + uri, raw_pair + )) + })?; + let key = percent_decode(raw_key).map_err(|err| { + Error::invalid_input(format!( + "cache backend uri {:?}: cannot decode query key {:?}: {}", + uri, raw_key, err + )) + })?; + let value = percent_decode(raw_value).map_err(|err| { + Error::invalid_input(format!( + "cache backend uri {:?}: cannot decode query value {:?}: {}", + uri, raw_value, err + )) + })?; + if config.options.contains_key(&key) { + return Err(Error::invalid_input(format!( + "cache backend uri {:?}: option {:?} is set more than once", + uri, key + ))); + } + config.options.insert(key, value); + } + } + + Ok(config) +} + +fn split_scheme(uri: &str) -> Result<(String, &str)> { + let colon = uri.find(':').ok_or_else(|| { + Error::invalid_input(format!("cache backend uri {:?} is missing ':'", uri)) + })?; + let scheme = &uri[..colon]; + let scheme = normalize_backend_kind(scheme) + .map_err(|err| Error::invalid_input(format!("cache backend uri {:?}: {}", uri, err)))?; + Ok((scheme, &uri[colon + 1..])) +} + +fn split_path_query(rest: &str) -> (&str, Option<&str>) { + match rest.split_once('?') { + Some((path, query)) => (path, Some(query)), + None => (rest, None), + } +} + +/// Strip leading `//authority/` boilerplate and return the useful path +/// component. Empty authorities (e.g. `moka://`) yield an empty path, while +/// empty-authority absolute paths (e.g. `disk:///tmp/cache`) retain their +/// leading slash. +fn normalize_path(raw: &str) -> String { + let Some(without_marker) = raw.strip_prefix("//") else { + return raw.to_string(); + }; + if without_marker.is_empty() { + return String::new(); + } + without_marker.to_string() +} + +fn percent_decode(input: &str) -> std::result::Result { + let bytes = input.as_bytes(); + let mut out = Vec::with_capacity(bytes.len()); + let mut i = 0; + while i < bytes.len() { + match bytes[i] { + b'%' => { + if i + 2 >= bytes.len() { + return Err(format!("truncated percent-escape at offset {}", i)); + } + let hi = decode_hex_digit(bytes[i + 1])?; + let lo = decode_hex_digit(bytes[i + 2])?; + out.push((hi << 4) | lo); + i += 3; + } + b => { + out.push(b); + i += 1; + } + } + } + String::from_utf8(out).map_err(|err| err.to_string()) +} + +fn decode_hex_digit(b: u8) -> std::result::Result { + match b { + b'0'..=b'9' => Ok(b - b'0'), + b'a'..=b'f' => Ok(10 + b - b'a'), + b'A'..=b'F' => Ok(10 + b - b'A'), + _ => Err(format!( + "invalid hex digit {:?} in percent-escape", + b as char + )), + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_parse_authority_only() { + let cfg = parse_backend_uri("moka://?capacity=1073741824").unwrap(); + assert_eq!(cfg.kind, "moka"); + assert_eq!( + cfg.options.get("capacity").map(String::as_str), + Some("1073741824") + ); + assert!(!cfg.options.contains_key("path")); + } + + #[test] + fn test_parse_path_and_query() { + let cfg = parse_backend_uri("example:///var/lance/cache?capacity=10G").unwrap(); + assert_eq!(cfg.kind, "example"); + assert_eq!( + cfg.options.get("path").map(String::as_str), + Some("/var/lance/cache") + ); + assert_eq!(cfg.options.get("capacity").map(String::as_str), Some("10G")); + } + + #[test] + fn test_parse_host_style() { + // Redis-style URI with host:port + path segment. All of it lives + // under the "path" option; the backend is responsible for + // interpreting it. + let cfg = parse_backend_uri("redis://localhost:6379/0?prefix=lance").unwrap(); + assert_eq!(cfg.kind, "redis"); + assert_eq!( + cfg.options.get("path").map(String::as_str), + Some("localhost:6379/0"), + ); + assert_eq!(cfg.options.get("prefix").map(String::as_str), Some("lance")); + } + + #[test] + fn test_scheme_is_lowercased() { + // Different upper/lower cases must resolve to the same registry + // key, otherwise `Moka://` and `moka://` would look up different + // backends. + let cfg = parse_backend_uri("MOKA://?capacity=1").unwrap(); + assert_eq!(cfg.kind, "moka"); + } + + #[test] + fn test_percent_decoding() { + let cfg = parse_backend_uri("kv://?prefix=a%2Fb&name=hello%20world&token=a+b%2Bc").unwrap(); + assert_eq!(cfg.options.get("prefix").map(String::as_str), Some("a/b")); + assert_eq!( + cfg.options.get("name").map(String::as_str), + Some("hello world") + ); + assert_eq!(cfg.options.get("token").map(String::as_str), Some("a+b+c")); + } + + #[test] + fn test_empty_query_pair_is_skipped() { + // A trailing "&" should not cause a spurious "" pair to appear. + let cfg = parse_backend_uri("moka://?capacity=1&").unwrap(); + assert_eq!(cfg.options.len(), 1); + } + + #[test] + fn test_missing_scheme_errors() { + let err = parse_backend_uri("no-scheme-here").unwrap_err(); + assert!(err.to_string().contains("missing ':'")); + } + + #[test] + fn test_invalid_scheme_errors() { + // Digit-leading schemes are invalid per RFC 3986 and would clash + // with URI-like values elsewhere in the config. + let err = parse_backend_uri("1moka://").unwrap_err(); + assert!(err.to_string().contains("must start with an ASCII letter")); + } + + #[test] + fn test_duplicate_option_errors() { + let err = parse_backend_uri("moka://?capacity=1&capacity=2").unwrap_err(); + assert!(err.to_string().contains("more than once")); + } + + #[test] + fn test_query_pair_without_equals_errors() { + let err = parse_backend_uri("moka://?capacity").unwrap_err(); + assert!(err.to_string().contains("missing '='")); + } +} diff --git a/rust/lance-core/src/cache/codec.rs b/rust/lance-core/src/cache/codec.rs index bba54840829..eff1b5e2b31 100644 --- a/rust/lance-core/src/cache/codec.rs +++ b/rust/lance-core/src/cache/codec.rs @@ -306,6 +306,14 @@ impl CacheCodec { } } + /// Return the stable entry type identity. + /// + /// Persistent and tiered backends can use this metadata to route encoded + /// values without retaining a readable logical cache key. + pub const fn type_id(&self) -> &'static str { + self.type_id + } + /// Serialize `value` into `writer`: envelope first, then the body. pub fn serialize(&self, value: &ArcAny, writer: &mut dyn Write) -> Result<()> { let body_offset = write_envelope(writer, self.type_id, self.version)?; diff --git a/rust/lance-core/src/cache/key.rs b/rust/lance-core/src/cache/key.rs new file mode 100644 index 00000000000..b7687d26459 --- /dev/null +++ b/rust/lance-core/src/cache/key.rs @@ -0,0 +1,476 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright The Lance Authors + +//! Canonical fixed-size cache key construction. +//! +//! Cache keys are BLAKE3 digests truncated to 128 bits. Logical key fields are +//! encoded with explicit type tags, fixed-width little-endian integers, and +//! length framing for variable-width values. This makes the pre-hash encoding +//! unambiguous and stable across processes, platforms, and builds. +//! +//! The digest is a cache identity, not an authentication or access-control +//! primitive: namespace derivation keys are deterministic and not secret. +//! Truncating to 128 bits gives generic birthday resistance of approximately +//! 64 bits. This protocol does not introduce a FIPS mode; BLAKE3 is the +//! repository's selected cache-key algorithm. + +use std::fmt; + +/// Storage namespace identifier for canonical cache keys. +/// +/// Persistent backends should include this identifier in their physical +/// namespace so future algorithm or framing changes produce cold misses. +pub const CACHE_KEY_FORMAT: &str = "blake3-128-v1"; + +const KEY_FORMAT_VERSION: u32 = 1; +const NAMESPACE_CONTEXT: &str = "lance-format/lance 2026-07-17 cache namespace v1"; +const NAMESPACE_DOMAIN: &[u8] = b"lance-cache-namespace\0"; +const ENTRY_DOMAIN: &[u8] = b"lance-cache-entry\0"; + +/// One-byte type discriminants in the stable key encoding. +#[derive(Clone, Copy)] +#[repr(u8)] +enum FieldTag { + U8 = 1, + U16 = 2, + U32 = 3, + U64 = 4, + I32 = 5, + I64 = 6, + Bool = 7, + Str = 8, + Bytes = 9, + FixedBytes = 10, + None = 11, + Some = 12, + Variant = 13, + Sequence = 14, +} + +impl FieldTag { + const fn as_u8(self) -> u8 { + self as u8 + } +} + +/// Versioned schema identity for fields emitted by a cache key. +/// +/// Change the version whenever the encoded fields or their meaning changes. +/// The identifier must be stable and globally unique to the logical layout. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub struct CacheKeySchema { + id: &'static str, + version: u32, +} + +impl CacheKeySchema { + /// Compatibility schema used by the default string-key bridge. + pub const LEGACY_TEXT: Self = Self::new("lance.cache.legacy-text", 1); + + /// Create a stable schema identifier and encoding version. + pub const fn new(id: &'static str, version: u32) -> Self { + Self { id, version } + } + + /// Return the author-assigned schema identifier. + pub const fn id(self) -> &'static str { + self.id + } + + /// Return the schema encoding version. + pub const fn version(self) -> u32 { + self.version + } +} + +/// Opaque 128-bit key passed to cache backends. +/// +/// The byte representation is canonical. It can be persisted directly and is +/// independent of the host's native integer endianness. +#[derive(Clone, Copy, Eq, Hash, Ord, PartialEq, PartialOrd)] +pub struct InternalCacheKey([u8; 16]); + +impl InternalCacheKey { + /// Reconstruct a key from its canonical bytes. + pub const fn from_bytes(bytes: [u8; 16]) -> Self { + Self(bytes) + } + + /// Borrow the canonical byte representation. + pub const fn as_bytes(&self) -> &[u8; 16] { + &self.0 + } + + /// Consume the key and return its canonical bytes. + pub const fn into_bytes(self) -> [u8; 16] { + self.0 + } +} + +impl fmt::Debug for InternalCacheKey { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str("InternalCacheKey(")?; + for byte in self.0 { + write!(f, "{byte:02x}")?; + } + f.write_str(")") + } +} + +/// Pre-derived namespace key shared by entries in one logical cache scope. +#[derive(Clone, Copy, Debug)] +pub struct CacheNamespace([u8; 32]); + +impl CacheNamespace { + /// Construct the stable root namespace. + pub fn root() -> Self { + Self(blake3::derive_key(NAMESPACE_CONTEXT, b"")) + } + + /// Derive a child namespace from one framed hierarchy segment. + pub fn child(self, segment: &str) -> Self { + let mut hasher = blake3::Hasher::new_keyed(&self.0); + write_framed(&mut hasher, NAMESPACE_DOMAIN); + hasher.update(&KEY_FORMAT_VERSION.to_le_bytes()); + write_framed(&mut hasher, segment.as_bytes()); + Self(hasher.finalize().into()) + } +} + +/// Streams typed logical fields into a canonical cache key. +/// +/// Integer methods use little-endian fixed-width encoding. Variable-width +/// strings and bytes are type-tagged and length-prefixed. There is deliberately +/// no `usize` method because cache identities must not depend on target width. +/// +/// # Examples +/// +/// ``` +/// use lance_core::cache::{CacheKeySchema, CacheNamespace, KeyBuilder}; +/// +/// let namespace = CacheNamespace::root().child("dataset"); +/// let mut builder = KeyBuilder::new( +/// namespace, +/// "example.Page", +/// CacheKeySchema::new("example.page-key", 1), +/// ); +/// builder.write_u32(7); +/// builder.write_str("values"); +/// let key = builder.finish(); +/// assert_eq!(key.as_bytes().len(), 16); +/// ``` +pub struct KeyBuilder { + hasher: blake3::Hasher, +} + +impl KeyBuilder { + /// Start a key in a namespace with a stable value type and key schema. + pub fn new( + namespace: CacheNamespace, + stable_type_id: &'static str, + schema: CacheKeySchema, + ) -> Self { + let mut hasher = blake3::Hasher::new_keyed(&namespace.0); + write_framed(&mut hasher, ENTRY_DOMAIN); + hasher.update(&KEY_FORMAT_VERSION.to_le_bytes()); + write_framed(&mut hasher, stable_type_id.as_bytes()); + write_framed(&mut hasher, schema.id().as_bytes()); + hasher.update(&schema.version().to_le_bytes()); + Self { hasher } + } + + /// Append a tagged, fixed-width `u8`. + #[inline] + pub fn write_u8(&mut self, value: u8) { + self.hasher.update(&[FieldTag::U8.as_u8(), value]); + } + + /// Append a tagged, little-endian `u16`. + #[inline] + pub fn write_u16(&mut self, value: u16) { + let mut encoded = [0; 3]; + encoded[0] = FieldTag::U16.as_u8(); + encoded[1..].copy_from_slice(&value.to_le_bytes()); + self.hasher.update(&encoded); + } + + /// Append a tagged, little-endian `u32`. + #[inline] + pub fn write_u32(&mut self, value: u32) { + let mut encoded = [0; 5]; + encoded[0] = FieldTag::U32.as_u8(); + encoded[1..].copy_from_slice(&value.to_le_bytes()); + self.hasher.update(&encoded); + } + + /// Append a tagged, little-endian `u64`. + #[inline] + pub fn write_u64(&mut self, value: u64) { + let mut encoded = [0; 9]; + encoded[0] = FieldTag::U64.as_u8(); + encoded[1..].copy_from_slice(&value.to_le_bytes()); + self.hasher.update(&encoded); + } + + /// Append a tagged, little-endian `i32`. + #[inline] + pub fn write_i32(&mut self, value: i32) { + let mut encoded = [0; 5]; + encoded[0] = FieldTag::I32.as_u8(); + encoded[1..].copy_from_slice(&value.to_le_bytes()); + self.hasher.update(&encoded); + } + + /// Append a tagged, little-endian `i64`. + #[inline] + pub fn write_i64(&mut self, value: i64) { + let mut encoded = [0; 9]; + encoded[0] = FieldTag::I64.as_u8(); + encoded[1..].copy_from_slice(&value.to_le_bytes()); + self.hasher.update(&encoded); + } + + /// Append a tagged boolean. + #[inline] + pub fn write_bool(&mut self, value: bool) { + self.hasher + .update(&[FieldTag::Bool.as_u8(), u8::from(value)]); + } + + /// Append a tagged, length-prefixed UTF-8 string. + #[inline] + pub fn write_str(&mut self, value: &str) { + self.write_variable(FieldTag::Str, value.as_bytes()); + } + + /// Append tagged, length-prefixed bytes. + #[inline] + pub fn write_bytes(&mut self, value: &[u8]) { + self.write_variable(FieldTag::Bytes, value); + } + + /// Append a tagged fixed-size byte array, including its length. + #[inline] + pub fn write_fixed_bytes(&mut self, value: &[u8; N]) { + self.write_variable(FieldTag::FixedBytes, value); + } + + /// Append the canonical marker for an absent optional value. + #[inline] + pub fn write_none(&mut self) { + self.hasher.update(&[FieldTag::None.as_u8()]); + } + + /// Append the canonical marker for a present optional value. + #[inline] + pub fn write_some(&mut self) { + self.hasher.update(&[FieldTag::Some.as_u8()]); + } + + /// Append a tagged enum variant ordinal. + #[inline] + pub fn write_variant(&mut self, variant: u32) { + let mut encoded = [0; 5]; + encoded[0] = FieldTag::Variant.as_u8(); + encoded[1..].copy_from_slice(&variant.to_le_bytes()); + self.hasher.update(&encoded); + } + + /// Append the length of a following sequence. + #[inline] + pub fn write_sequence_len(&mut self, len: u64) { + let mut encoded = [0; 9]; + encoded[0] = FieldTag::Sequence.as_u8(); + encoded[1..].copy_from_slice(&len.to_le_bytes()); + self.hasher.update(&encoded); + } + + /// Finalize and return the canonical 128-bit key. + #[inline] + pub fn finish(self) -> InternalCacheKey { + let hash = self.hasher.finalize(); + let mut bytes = [0; 16]; + bytes.copy_from_slice(&hash.as_bytes()[..16]); + InternalCacheKey(bytes) + } + + #[inline] + fn write_variable(&mut self, tag: FieldTag, value: &[u8]) { + self.hasher.update(&[tag.as_u8()]); + self.hasher.update(&encoded_len(value)); + self.hasher.update(value); + } +} + +#[inline] +fn write_framed(hasher: &mut blake3::Hasher, value: &[u8]) { + hasher.update(&encoded_len(value)); + hasher.update(value); +} + +#[inline] +fn encoded_len(value: &[u8]) -> [u8; 8] { + (value.len() as u64).to_le_bytes() +} + +#[cfg(test)] +mod tests { + use super::*; + + const SCHEMA: CacheKeySchema = CacheKeySchema::new("test.key", 1); + + fn builder() -> KeyBuilder { + KeyBuilder::new( + CacheNamespace::root().child("s3://bucket/dataset"), + "test.Value", + SCHEMA, + ) + } + + fn key_with(write: impl FnOnce(&mut KeyBuilder)) -> InternalCacheKey { + let mut key = builder(); + write(&mut key); + key.finish() + } + + #[test] + fn key_and_namespace_have_fixed_sizes() { + assert_eq!(std::mem::size_of::(), 16); + assert_eq!(std::mem::size_of::(), 32); + assert_eq!(std::mem::size_of::(), 1); + } + + #[test] + fn blake3_matches_official_empty_keyed_hash_vector() { + let key = *b"whats the Elvish word for friend"; + assert_eq!( + blake3::keyed_hash(&key, b"").as_bytes(), + &[ + 0x92, 0xb2, 0xb7, 0x56, 0x04, 0xed, 0x3c, 0x76, 0x1f, 0x9d, 0x6f, 0x62, 0x39, 0x2c, + 0x8a, 0x92, 0x27, 0xad, 0x0e, 0xa3, 0xf0, 0x95, 0x73, 0xe7, 0x83, 0xf1, 0x49, 0x8a, + 0x4e, 0xd6, 0x0d, 0x26, + ] + ); + } + + #[test] + fn typed_fields_and_boundaries_are_unambiguous() { + let cases = [ + key_with(|key| { + key.write_str("ab"); + key.write_str("c"); + }), + key_with(|key| { + key.write_str("a"); + key.write_str("bc"); + }), + key_with(|key| key.write_str("")), + key_with(|key| key.write_bytes(b"")), + key_with(|key| key.write_fixed_bytes(b"")), + key_with(|key| key.write_u8(1)), + key_with(|key| key.write_u16(1)), + key_with(|key| key.write_u32(1)), + key_with(|key| key.write_u64(1)), + key_with(|key| key.write_i32(1)), + key_with(|key| key.write_i64(1)), + key_with(|key| key.write_bool(false)), + key_with(|key| key.write_bool(true)), + key_with(KeyBuilder::write_none), + key_with(KeyBuilder::write_some), + key_with(|key| key.write_variant(0)), + key_with(|key| key.write_variant(1)), + ]; + assert_eq!(std::collections::BTreeSet::from(cases).len(), cases.len()); + + assert_ne!( + key_with(|key| { + key.write_sequence_len(2); + key.write_u32(1); + key.write_u32(2); + }), + key_with(|key| { + key.write_u32(1); + key.write_u32(2); + }) + ); + } + + #[test] + fn namespace_type_schema_and_version_are_domain_separated() { + let root = CacheNamespace::root(); + let namespace = root.child("dataset").child("index"); + let nested = KeyBuilder::new(namespace, "test.Value", SCHEMA).finish(); + let combined = KeyBuilder::new(root.child("dataset/index"), "test.Value", SCHEMA).finish(); + assert_ne!(nested, combined); + + assert_ne!( + nested, + KeyBuilder::new(namespace, "test.OtherValue", SCHEMA).finish() + ); + assert_ne!( + nested, + KeyBuilder::new( + namespace, + "test.Value", + CacheKeySchema::new("test.other-key", 1), + ) + .finish() + ); + assert_ne!( + nested, + KeyBuilder::new(namespace, "test.Value", CacheKeySchema::new("test.key", 2),).finish() + ); + + let tenant_a_memory = + KeyBuilder::new(root.child("tenant-a").child("memory"), "test.Value", SCHEMA).finish(); + assert_ne!( + tenant_a_memory, + KeyBuilder::new(root.child("tenant-b").child("memory"), "test.Value", SCHEMA,).finish() + ); + assert_ne!( + tenant_a_memory, + KeyBuilder::new( + root.child("tenant-a").child("persistent"), + "test.Value", + SCHEMA, + ) + .finish() + ); + } + + #[test] + fn integers_use_fixed_width_little_endian_encoding() { + let namespace = CacheNamespace::root().child("endianness"); + let mut key = KeyBuilder::new(namespace, "test.Value", SCHEMA); + key.write_u32(0x0102_0304); + let actual = key.finish(); + + let mut reference = blake3::Hasher::new_keyed(&namespace.0); + write_framed(&mut reference, ENTRY_DOMAIN); + reference.update(&KEY_FORMAT_VERSION.to_le_bytes()); + write_framed(&mut reference, b"test.Value"); + write_framed(&mut reference, SCHEMA.id().as_bytes()); + reference.update(&SCHEMA.version().to_le_bytes()); + reference.update(&[FieldTag::U32.as_u8(), 0x04, 0x03, 0x02, 0x01]); + let mut expected = [0; 16]; + expected.copy_from_slice(&reference.finalize().as_bytes()[..16]); + + assert_eq!(actual, InternalCacheKey::from_bytes(expected)); + } + + #[test] + fn key_has_stable_golden_vector() { + let mut key = builder(); + key.write_u32(7); + key.write_str("page"); + key.write_some(); + key.write_fixed_bytes(&[0xAB; 16]); + assert_eq!( + key.finish().into_bytes(), + [ + 0xc4, 0x38, 0xff, 0x22, 0x30, 0x55, 0x30, 0xfc, 0x74, 0x16, 0x38, 0xe9, 0x7d, 0x45, + 0xa5, 0x68, + ] + ); + } +} diff --git a/rust/lance-core/src/cache/mod.rs b/rust/lance-core/src/cache/mod.rs index 4f93f261bea..44b5604bd1a 100644 --- a/rust/lance-core/src/cache/mod.rs +++ b/rust/lance-core/src/cache/mod.rs @@ -31,9 +31,9 @@ //! ## For backend implementors //! //! Implement [`CacheBackend`] to provide a custom storage layer (disk, Redis, -//! etc.). Backends receive [`InternalCacheKey`] keys and type-erased -//! [`CacheEntry`] values — the typed wrapping is handled by [`LanceCache`]. -//! See the [`backend`] module for details. +//! etc.). Backends receive opaque, fixed-size [`InternalCacheKey`] values and +//! type-erased [`CacheEntry`] values. The typed wrapping is handled by +//! [`LanceCache`]. See the [`backend`] module for migration details. //! //! ## Serialization flow //! @@ -46,26 +46,36 @@ //! `codec.deserialize(reader)` on get to persist entries across restarts. pub mod backend; +mod backend_uri; pub mod codec; mod entry_io; +mod key; mod moka; +mod quick; +mod registry; -pub use backend::{CacheBackend, CacheEntry, CacheKeyIterator, InternalCacheKey}; +pub use backend::{CacheBackend, CacheEntry}; +pub use backend_uri::{build_from_uri, parse_backend_uri}; pub use codec::{ CacheCodec, CacheCodecImpl, CacheDecode, CacheMissReason, MAGIC, has_cache_envelope, }; pub use entry_io::{CacheEntryReader, CacheEntryWriter}; +pub use key::{CACHE_KEY_FORMAT, CacheKeySchema, CacheNamespace, InternalCacheKey, KeyBuilder}; pub use moka::MokaCacheBackend; +pub use quick::{QuickCacheBackend, recommended_cache_shards}; +pub use registry::{BackendBuildFn, BackendConfig, build_from_config, register_backend}; +use std::any::TypeId; use std::borrow::Cow; +use std::collections::HashMap; use std::sync::{ - Arc, + Arc, RwLock, Weak, atomic::{AtomicU64, Ordering}, }; -use futures::{Future, FutureExt}; +use futures::Future; -use crate::Result; +use crate::{Error, Result}; pub use crate::deepsize::{Context, DeepSizeOf}; @@ -75,9 +85,10 @@ pub use crate::deepsize::{Context, DeepSizeOf}; /// Typed cache key for sized value types. /// -/// Implement this trait to define a new type of cached entry. [`LanceCache`] -/// uses the key string and type name to construct an [`InternalCacheKey`] -/// for the backend. +/// Existing implementations can continue returning a logical string from +/// [`key`](Self::key). Performance-sensitive implementations should also +/// provide a stable schema and stream typed fields through +/// [`write_key`](Self::write_key), avoiding construction of that string. /// /// # Example /// @@ -98,14 +109,34 @@ pub trait CacheKey { /// Short, stable string identifying this value type. /// /// Two `CacheKey` impls that store different `ValueType`s **must** return - /// different type names; if they collide, gets will silently return `None` - /// due to failed downcasts. + /// different type names. /// /// Use a short literal (e.g. `"Vec"`), not /// `std::any::type_name` — the latter is not guaranteed stable across /// compiler versions or build configurations. fn type_name() -> &'static str; + /// Stable identity included in the physical key. + /// + /// The compatibility default preserves existing implementations by using + /// their author-assigned [`type_name`](Self::type_name). + fn stable_type_id() -> &'static str { + Self::type_name() + } + + /// Versioned schema for the logical key fields. + fn schema() -> CacheKeySchema { + CacheKeySchema::LEGACY_TEXT + } + + /// Stream the logical key fields into the canonical key builder. + /// + /// The compatibility default hashes the existing string key. In-tree hot + /// paths override this with typed, allocation-free field encoding. + fn write_key(&self, builder: &mut KeyBuilder) { + builder.write_str(self.key().as_ref()); + } + /// Optional codec for serializing/deserializing this key's value type. /// /// Returns `None` by default. Cache backends that support persistence @@ -134,6 +165,21 @@ pub trait UnsizedCacheKey { /// Short, stable string identifying this value type. /// See [`CacheKey::type_name`] for requirements. fn type_name() -> &'static str; + + /// Stable identity included in the physical key. + fn stable_type_id() -> &'static str { + Self::type_name() + } + + /// Versioned schema for the logical key fields. + fn schema() -> CacheKeySchema { + CacheKeySchema::LEGACY_TEXT + } + + /// Stream the logical key fields into the canonical key builder. + fn write_key(&self, builder: &mut KeyBuilder) { + builder.write_str(self.key().as_ref()); + } } // --------------------------------------------------------------------------- @@ -145,10 +191,61 @@ fn cache_entry_size(value: &T) -> usize { value.deep_size_of() + std::mem::size_of::() * 2 } -/// Build an [`InternalCacheKey`] from a cache's prefix, a user key string, -/// and a type name. -fn build_key(prefix: &Arc, key: &str, type_name: &'static str) -> InternalCacheKey { - InternalCacheKey::new(prefix.clone(), Arc::from(key), type_name) +type CacheEntrySizeAccessor = fn(&CacheEntry, &mut Context) -> Option; + +fn cache_entry_size_with_context(entry: &CacheEntry, context: &mut Context) -> Option +where + T: DeepSizeOf + Send + Sync + 'static, +{ + let value = entry.downcast_ref::()?; + let entry_ptr = Arc::as_ptr(entry) as *const () as usize; + if !context.mark_seen(entry_ptr) { + return Some(0); + } + Some( + std::mem::size_of_val(value) + + value.deep_size_of_children(context) + + std::mem::size_of::() * 2, + ) +} + +#[derive(Debug)] +struct CacheState { + backend: Arc, + hits: AtomicU64, + misses: AtomicU64, + entry_size_accessors: RwLock>, +} + +impl CacheState { + fn new(backend: Arc) -> Self { + Self { + backend, + hits: AtomicU64::new(0), + misses: AtomicU64::new(0), + entry_size_accessors: RwLock::new(HashMap::new()), + } + } + + fn entry_size(&self, value: &T) -> usize + where + T: DeepSizeOf + Send + Sync + 'static, + { + let type_id = TypeId::of::(); + let is_registered = self + .entry_size_accessors + .read() + .unwrap_or_else(|poisoned| poisoned.into_inner()) + .contains_key(&type_id); + if !is_registered { + self.entry_size_accessors + .write() + .unwrap_or_else(|poisoned| poisoned.into_inner()) + .entry(type_id) + .or_insert(cache_entry_size_with_context::); + } + cache_entry_size(value) + } } // --------------------------------------------------------------------------- @@ -161,184 +258,98 @@ fn build_key(prefix: &Arc, key: &str, type_name: &'static str) -> InternalC /// [`MokaCacheBackend`]; pass a custom backend via [`LanceCache::with_backend`]. #[derive(Clone)] pub struct LanceCache { - cache: Arc, - prefix: Arc, - hits: Arc, - misses: Arc, + state: Arc, + namespace: key::CacheNamespace, } impl std::fmt::Debug for LanceCache { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { f.debug_struct("LanceCache") - .field("cache", &self.cache) - .finish() + .field("backend", &self.state.backend) + .finish_non_exhaustive() } } impl DeepSizeOf for LanceCache { - fn deep_size_of_children(&self, _: &mut Context) -> usize { - self.cache.approx_size_bytes() + fn deep_size_of_children(&self, context: &mut Context) -> usize { + let state_ptr = Arc::as_ptr(&self.state) as usize; + if !context.mark_seen(state_ptr) { + return 0; + } + + let accessors = self + .state + .entry_size_accessors + .read() + .unwrap_or_else(|poisoned| poisoned.into_inner()) + .clone(); + self.state + .backend + .deep_size_of_entries(context, &|entry, context| { + accessors + .get(&entry.as_ref().type_id()) + .and_then(|size_of_entry| size_of_entry(entry, context)) + }) + .unwrap_or_else(|| self.state.backend.approx_size_bytes()) } } impl LanceCache { pub fn with_capacity(capacity: usize) -> Self { - Self { - cache: Arc::new(MokaCacheBackend::with_capacity(capacity)), - prefix: Arc::from(""), - hits: Arc::new(AtomicU64::new(0)), - misses: Arc::new(AtomicU64::new(0)), - } + Self::with_backend(Arc::new(MokaCacheBackend::with_capacity(capacity))) } /// Create a cache backed by a custom [`CacheBackend`]. pub fn with_backend(backend: Arc) -> Self { Self { - cache: backend, - prefix: Arc::from(""), - hits: Arc::new(AtomicU64::new(0)), - misses: Arc::new(AtomicU64::new(0)), + state: Arc::new(CacheState::new(backend)), + namespace: key::CacheNamespace::root(), } } pub fn no_cache() -> Self { - Self { - cache: Arc::new(MokaCacheBackend::no_cache()), - prefix: Arc::from(""), - hits: Arc::new(AtomicU64::new(0)), - misses: Arc::new(AtomicU64::new(0)), - } + Self::with_backend(Arc::new(MokaCacheBackend::no_cache())) } - /// Create a cache with the given backend and an exact prefix string. - /// Unlike `with_key_prefix`, this sets the prefix verbatim (no trailing slash added). - pub fn with_backend_and_prefix(backend: Arc, prefix: String) -> Self { - Self { - cache: backend, - prefix: Arc::from(prefix), - hits: Arc::new(AtomicU64::new(0)), - misses: Arc::new(AtomicU64::new(0)), - } - } - - /// Appends a prefix to the cache key. + /// Derive a child namespace for all keys in the returned cache handle. + /// + /// Each call adds one framed hierarchy segment. Consequently, + /// `cache.with_key_prefix("a").with_key_prefix("b")` is deliberately + /// distinct from `cache.with_key_prefix("a/b")`. pub fn with_key_prefix(&self, prefix: &str) -> Self { Self { - cache: self.cache.clone(), - prefix: Arc::from(format!("{}{}/", self.prefix, prefix)), - hits: self.hits.clone(), - misses: self.misses.clone(), + state: self.state.clone(), + namespace: self.namespace.child(prefix), } } - /// Invalidate all entries whose prefix starts with the given string. - pub async fn invalidate_prefix(&self, prefix: &str) { - let full_prefix = format!("{}{}", self.prefix, prefix); - self.cache.invalidate_prefix(&full_prefix).await; - } - pub async fn size(&self) -> usize { - self.cache.num_entries().await + self.state.backend.num_entries().await } pub fn approx_size(&self) -> usize { - self.cache.approx_num_entries() + self.state.backend.approx_num_entries() } pub async fn size_bytes(&self) -> usize { - self.cache.size_bytes().await - } - - /// Return an iterator over keys currently stored under this cache's prefix. - /// - /// Returns `None` when the backend does not support key inventory. The - /// iterator is intended for diagnostics and may be weakly consistent with - /// concurrent cache mutations. - /// - /// # Examples - /// - /// ``` - /// # use std::{borrow::Cow, sync::Arc}; - /// # use lance_core::cache::{CacheKey, LanceCache}; - /// # struct MyKey; - /// # impl CacheKey for MyKey { - /// # type ValueType = Vec; - /// # fn key(&self) -> Cow<'_, str> { Cow::Borrowed("my-key") } - /// # fn type_name() -> &'static str { "VecI32" } - /// # } - /// # async fn example() { - /// let cache = LanceCache::with_capacity(1024); - /// cache.insert_with_key(&MyKey, Arc::new(vec![1, 2, 3])).await; - /// - /// let mut keys = cache.keys().await.expect("Moka supports key inventory"); - /// assert_eq!(keys.next().unwrap().key(), "my-key"); - /// # } - /// ``` - pub async fn keys(&self) -> Option> { - Some(Box::new( - self.cache - .keys() - .await? - .filter(|key| key.starts_with(&self.prefix)), - )) - } - - // -- Sized insert/get (internal, shared by sized and unsized paths) -------- - - async fn insert_with_id( - &self, - key: &str, - type_name: &'static str, - codec: Option, - metadata: Arc, - ) { - let size = cache_entry_size(&*metadata); - let cache_key = build_key(&self.prefix, key, type_name); - self.cache.insert(&cache_key, metadata, size, codec).await; - } - - async fn get_with_id( - &self, - key: &str, - type_name: &'static str, - codec: Option, - ) -> Option> { - let cache_key = build_key(&self.prefix, key, type_name); - if let Some(entry) = self.cache.get(&cache_key, codec).await { - match entry.downcast::() { - Ok(val) => { - self.hits.fetch_add(1, Ordering::Relaxed); - Some(val) - } - Err(_) => { - // Type mismatch: the backend returned a different concrete - // type than expected (e.g. a disk cache may store - // intermediate state). Treat as a miss. - self.misses.fetch_add(1, Ordering::Relaxed); - None - } - } - } else { - self.misses.fetch_add(1, Ordering::Relaxed); - None - } + self.state.backend.size_bytes().await } // -- Stats / clear -------------------------------------------------------- pub async fn stats(&self) -> CacheStats { CacheStats { - hits: self.hits.load(Ordering::Relaxed), - misses: self.misses.load(Ordering::Relaxed), - num_entries: self.cache.num_entries().await, - size_bytes: self.cache.size_bytes().await, + hits: self.state.hits.load(Ordering::Relaxed), + misses: self.state.misses.load(Ordering::Relaxed), + num_entries: self.state.backend.num_entries().await, + size_bytes: self.state.backend.size_bytes().await, } } pub async fn clear(&self) { - self.cache.clear().await; - self.hits.store(0, Ordering::Relaxed); - self.misses.store(0, Ordering::Relaxed); + self.state.backend.clear().await; + self.state.hits.store(0, Ordering::Relaxed); + self.state.misses.store(0, Ordering::Relaxed); } // -- CacheKey-based methods ----------------------------------------------- @@ -348,9 +359,12 @@ impl LanceCache { K: CacheKey, K::ValueType: DeepSizeOf + Send + Sync + 'static, { - self.insert_with_id(&cache_key.key(), K::type_name(), K::codec(), metadata) - .boxed() - .await + let size = self.state.entry_size(metadata.as_ref()); + let key = self.sized_key(cache_key); + self.state + .backend + .insert(&key, metadata, size, K::codec()) + .await; } pub async fn get_with_key(&self, cache_key: &K) -> Option> @@ -358,9 +372,28 @@ impl LanceCache { K: CacheKey, K::ValueType: DeepSizeOf + Send + Sync + 'static, { - self.get_with_id::(&cache_key.key(), K::type_name(), K::codec()) - .boxed() - .await + let key = self.sized_key(cache_key); + let Some(entry) = self.state.backend.get(&key, K::codec()).await else { + self.state.misses.fetch_add(1, Ordering::Relaxed); + return None; + }; + match entry.downcast::() { + Ok(value) => { + self.state.hits.fetch_add(1, Ordering::Relaxed); + Some(value) + } + Err(_) => { + // Type mismatch: the backend returned a different concrete + // type than expected (e.g. a disk cache may store + // intermediate state). Treat as a miss. + log::warn!( + "cache backend returned a value with the wrong concrete type for key type {:?}", + K::stable_type_id() + ); + self.state.misses.fetch_add(1, Ordering::Relaxed); + None + } + } } pub async fn get_or_insert_with_key( @@ -374,27 +407,63 @@ impl LanceCache { F: FnOnce() -> Fut + Send, Fut: Future> + Send, { - let key = build_key(&self.prefix, &cache_key.key(), K::type_name()); + self.get_or_insert_with_key_hit(cache_key, loader) + .await + .map(|(value, _)| value) + } + /// Same as [`get_or_insert_with_key`](Self::get_or_insert_with_key), but + /// also returns a boolean indicating whether the loader was skipped for + /// this call. + /// + /// - `true` means this call did **not** execute the loader. That covers + /// both a true cache hit on an already-populated entry and a coalesced + /// concurrent load where an in-flight loader started by a different + /// caller produced the value. + /// - `false` means the loader ran on this call (a real cache miss). + /// + /// Callers that want strict "served from cache" semantics should treat + /// coalesced loads as misses; the current backend does not distinguish the + /// two cases. Prefer this over rolling a caller-side `Arc` + /// when the caller needs per-query hit/miss counters — the backend already + /// tracks this bit internally and this method just exposes it. + pub async fn get_or_insert_with_key_hit( + &self, + cache_key: K, + loader: F, + ) -> Result<(Arc, bool)> + where + K: CacheKey, + K::ValueType: DeepSizeOf + Send + Sync + 'static, + F: FnOnce() -> Fut + Send, + Fut: Future> + Send, + { + let key = self.sized_key(&cache_key); + let state = self.state.clone(); let typed_loader = Box::pin(async move { - let value = loader().await?; - let arc = Arc::new(value); - let size = cache_entry_size(&*arc); - Ok((arc as CacheEntry, size)) + let value = Arc::new(loader().await?); + let size = state.entry_size(value.as_ref()); + Ok((value as CacheEntry, size)) }); let (entry, was_cached) = self - .cache + .state + .backend .get_or_insert(&key, typed_loader, K::codec()) .await?; - + let entry = entry.downcast::().map_err(|_| { + self.state.misses.fetch_add(1, Ordering::Relaxed); + Error::io(format!( + "cache backend returned a value with the wrong concrete type for key type {:?}", + K::stable_type_id() + )) + })?; if was_cached { - self.hits.fetch_add(1, Ordering::Relaxed); + self.state.hits.fetch_add(1, Ordering::Relaxed); } else { - self.misses.fetch_add(1, Ordering::Relaxed); + self.state.misses.fetch_add(1, Ordering::Relaxed); } - - Ok(entry.downcast::().unwrap()) + Ok((entry, was_cached)) } pub async fn insert_unsized_with_key(&self, cache_key: &K, metadata: Arc) @@ -402,21 +471,90 @@ impl LanceCache { K: UnsizedCacheKey, K::ValueType: DeepSizeOf + Send + Sync + 'static, { - self.insert_with_id(&cache_key.key(), K::type_name(), None, Arc::new(metadata)) - .boxed() - .await + let metadata = Arc::new(metadata); + let size = self.state.entry_size(metadata.as_ref()); + let key = self.unsized_key(cache_key); + self.state.backend.insert(&key, metadata, size, None).await; } - pub async fn get_unsized_with_key(&self, cache_key: &K) -> Option> + pub async fn get_or_insert_unsized_with_key( + &self, + cache_key: K, + loader: F, + ) -> Result> where K: UnsizedCacheKey, K::ValueType: DeepSizeOf + Send + Sync + 'static, + F: FnOnce() -> Fut + Send, + Fut: Future>> + Send, { - let outer = self - .get_with_id::>(&cache_key.key(), K::type_name(), None) - .boxed() + let key = self.unsized_key(&cache_key); + let state = self.state.clone(); + let typed_loader = Box::pin(async move { + let value = loader().await?; + let size = state.entry_size(&value); + Ok((Arc::new(value) as CacheEntry, size)) + }); + + let (entry, was_cached) = self + .state + .backend + .get_or_insert(&key, typed_loader, None) .await?; - Some(outer.as_ref().clone()) + let entry = entry.downcast::>().map_err(|_| { + self.state.misses.fetch_add(1, Ordering::Relaxed); + Error::io(format!( + "cache backend returned a value with the wrong concrete type for unsized key type {:?}", + K::stable_type_id() + )) + })?; + if was_cached { + self.state.hits.fetch_add(1, Ordering::Relaxed); + } else { + self.state.misses.fetch_add(1, Ordering::Relaxed); + } + Ok(entry.as_ref().clone()) + } + + pub async fn get_unsized_with_key(&self, cache_key: &K) -> Option> + where + K: UnsizedCacheKey, + K::ValueType: DeepSizeOf + Send + Sync + 'static, + { + let key = self.unsized_key(cache_key); + let Some(entry) = self.state.backend.get(&key, None).await else { + self.state.misses.fetch_add(1, Ordering::Relaxed); + return None; + }; + match entry.downcast::>() { + Ok(value) => { + self.state.hits.fetch_add(1, Ordering::Relaxed); + Some(value.as_ref().clone()) + } + Err(_) => { + // Type mismatch: the backend returned a different concrete + // type than expected (e.g. a disk cache may store + // intermediate state). Treat as a miss. + log::warn!( + "cache backend returned a value with the wrong concrete type for unsized key type {:?}", + K::stable_type_id() + ); + self.state.misses.fetch_add(1, Ordering::Relaxed); + None + } + } + } + + fn sized_key(&self, cache_key: &K) -> InternalCacheKey { + let mut builder = KeyBuilder::new(self.namespace, K::stable_type_id(), K::schema()); + cache_key.write_key(&mut builder); + builder.finish() + } + + fn unsized_key(&self, cache_key: &K) -> InternalCacheKey { + let mut builder = KeyBuilder::new(self.namespace, K::stable_type_id(), K::schema()); + cache_key.write_key(&mut builder); + builder.finish() } } @@ -428,50 +566,31 @@ impl LanceCache { /// When the original cache is dropped, operations on this will gracefully no-op. #[derive(Clone, Debug)] pub struct WeakLanceCache { - inner: std::sync::Weak, - prefix: Arc, - hits: Arc, - misses: Arc, + state: Weak, + namespace: key::CacheNamespace, } impl WeakLanceCache { pub fn from(cache: &LanceCache) -> Self { Self { - inner: Arc::downgrade(&cache.cache), - prefix: cache.prefix.clone(), - hits: cache.hits.clone(), - misses: cache.misses.clone(), + state: Arc::downgrade(&cache.state), + namespace: cache.namespace, } } pub fn with_key_prefix(&self, prefix: &str) -> Self { Self { - inner: self.inner.clone(), - prefix: Arc::from(format!("{}{}/", self.prefix, prefix)), - hits: self.hits.clone(), - misses: self.misses.clone(), + state: self.state.clone(), + namespace: self.namespace.child(prefix), } } - /// The key prefix used for all entries in this cache. - pub fn prefix(&self) -> &str { - &self.prefix - } - pub async fn get_with_key(&self, cache_key: &K) -> Option> where K: CacheKey, K::ValueType: DeepSizeOf + Send + Sync + 'static, { - let cache = self.inner.upgrade()?; - let key = build_key(&self.prefix, &cache_key.key(), K::type_name()); - if let Some(entry) = cache.get(&key, K::codec()).await { - self.hits.fetch_add(1, Ordering::Relaxed); - Some(entry.downcast::().unwrap()) - } else { - self.misses.fetch_add(1, Ordering::Relaxed); - None - } + self.upgrade()?.get_with_key(cache_key).await } pub async fn insert_with_key(&self, cache_key: &K, value: Arc) -> bool @@ -479,15 +598,12 @@ impl WeakLanceCache { K: CacheKey, K::ValueType: DeepSizeOf + Send + Sync + 'static, { - if let Some(cache) = self.inner.upgrade() { - let size = cache_entry_size(&*value); - let key = build_key(&self.prefix, &cache_key.key(), K::type_name()); - cache.insert(&key, value, size, K::codec()).await; - true - } else { + let Some(cache) = self.upgrade() else { log::warn!("WeakLanceCache: cache no longer available, unable to insert item"); - false - } + return false; + }; + cache.insert_with_key(cache_key, value).await; + true } /// Get or insert an item, computing it if necessary. @@ -504,25 +620,31 @@ impl WeakLanceCache { F: FnOnce() -> Fut + Send, Fut: Future> + Send, { - if let Some(cache) = self.inner.upgrade() { - let key = build_key(&self.prefix, &cache_key.key(), K::type_name()); - let typed_loader = Box::pin(async move { - let value = loader().await?; - let arc = Arc::new(value); - let size = cache_entry_size(&*arc); - Ok((arc as CacheEntry, size)) - }); - let (entry, was_cached) = cache.get_or_insert(&key, typed_loader, K::codec()).await?; - if was_cached { - self.hits.fetch_add(1, Ordering::Relaxed); - } else { - self.misses.fetch_add(1, Ordering::Relaxed); - } - Ok(entry.downcast::().unwrap()) - } else { + self.get_or_insert_with_key_hit(cache_key, loader) + .await + .map(|(value, _)| value) + } + + /// Same as [`get_or_insert_with_key`](Self::get_or_insert_with_key), but + /// also returns a boolean indicating whether the loader was skipped for + /// this call. See [`LanceCache::get_or_insert_with_key_hit`] for the + /// coalesced-load caveat. + pub async fn get_or_insert_with_key_hit( + &self, + cache_key: K, + loader: F, + ) -> Result<(Arc, bool)> + where + K: CacheKey, + K::ValueType: DeepSizeOf + Send + Sync + 'static, + F: FnOnce() -> Fut + Send, + Fut: Future> + Send, + { + let Some(cache) = self.upgrade() else { log::warn!("WeakLanceCache: cache no longer available, computing without caching"); - loader().await.map(Arc::new) - } + return loader().await.map(|value| (Arc::new(value), false)); + }; + cache.get_or_insert_with_key_hit(cache_key, loader).await } pub async fn get_unsized_with_key(&self, cache_key: &K) -> Option> @@ -530,16 +652,7 @@ impl WeakLanceCache { K: UnsizedCacheKey, K::ValueType: DeepSizeOf + Send + Sync + 'static, { - let cache = self.inner.upgrade()?; - let key = build_key(&self.prefix, &cache_key.key(), K::type_name()); - if let Some(entry) = cache.get(&key, None).await { - entry - .downcast::>() - .ok() - .map(|arc| arc.as_ref().clone()) - } else { - None - } + self.upgrade()?.get_unsized_with_key(cache_key).await } pub async fn insert_unsized_with_key(&self, cache_key: &K, value: Arc) @@ -547,14 +660,18 @@ impl WeakLanceCache { K: UnsizedCacheKey, K::ValueType: DeepSizeOf + Send + Sync + 'static, { - if let Some(cache) = self.inner.upgrade() { - let wrapper = Arc::new(value); - let size = cache_entry_size(&*wrapper); - let key = build_key(&self.prefix, &cache_key.key(), K::type_name()); - cache.insert(&key, wrapper, size, None).await; - } else { + let Some(cache) = self.upgrade() else { log::warn!("WeakLanceCache: cache no longer available, unable to insert unsized item"); - } + return; + }; + cache.insert_unsized_with_key(cache_key, value).await; + } + + fn upgrade(&self) -> Option { + Some(LanceCache { + state: self.state.upgrade()?, + namespace: self.namespace, + }) } } @@ -594,457 +711,764 @@ impl CacheStats { #[cfg(test)] mod tests { + use std::collections::HashMap; + use std::pin::Pin; + use std::sync::{ + atomic::{AtomicUsize, Ordering}, + mpsc, + }; + use std::task::Poll; + use std::thread; + use std::time::Duration; + use super::*; - use std::collections::{BTreeSet, HashMap}; - use std::marker::PhantomData; - struct TestKey { - key: String, - _phantom: PhantomData, + async fn report_first_pending( + future: F, + parked: tokio::sync::oneshot::Sender<()>, + ) -> F::Output + where + F: Future, + { + tokio::pin!(future); + let mut parked = Some(parked); + futures::future::poll_fn(|cx| match future.as_mut().poll(cx) { + Poll::Pending => { + if let Some(parked) = parked.take() { + let _ = parked.send(()); + } + Poll::Pending + } + Poll::Ready(output) => Poll::Ready(output), + }) + .await } - impl TestKey { - fn new(key: &str) -> Self { - Self { - key: key.to_string(), - _phantom: PhantomData, - } + #[derive(Clone)] + struct VersionedTestKey { + id: u64, + } + + type TestKey = VersionedTestKey<1>; + type TestKeyV2 = VersionedTestKey<2>; + + impl VersionedTestKey { + fn new(id: u64) -> Self { + Self { id } } } - impl CacheKey for TestKey { - type ValueType = T; - fn key(&self) -> std::borrow::Cow<'_, str> { - std::borrow::Cow::Borrowed(&self.key) + impl CacheKey for VersionedTestKey { + type ValueType = Vec; + + fn key(&self) -> Cow<'_, str> { + self.id.to_string().into() } + fn type_name() -> &'static str { - std::any::type_name::() + "test.VecU32" + } + + fn schema() -> CacheKeySchema { + CacheKeySchema::new("test.vec-u32-key", SCHEMA_VERSION) + } + + fn write_key(&self, builder: &mut KeyBuilder) { + builder.write_u64(self.id); } } - /// Test helper: an UnsizedCacheKey for trait object values. - struct TestUnsizedKey { - key: String, - _phantom: PhantomData, + struct SharedTestValue { + data: Arc>, } - impl TestUnsizedKey { - fn new(key: &str) -> Self { - Self { - key: key.to_string(), - _phantom: PhantomData, - } + impl DeepSizeOf for SharedTestValue { + fn deep_size_of_children(&self, context: &mut Context) -> usize { + self.data.deep_size_of_children(context) } } - impl UnsizedCacheKey for TestUnsizedKey { - type ValueType = T; - fn key(&self) -> std::borrow::Cow<'_, str> { - std::borrow::Cow::Borrowed(&self.key) + struct SharedTestKey(u64); + + impl CacheKey for SharedTestKey { + type ValueType = SharedTestValue; + + fn key(&self) -> Cow<'_, str> { + self.0.to_string().into() } + fn type_name() -> &'static str { - std::any::type_name::() + "test.SharedValue" + } + + fn schema() -> CacheKeySchema { + CacheKeySchema::new("test.shared-value-key", 1) + } + + fn write_key(&self, builder: &mut KeyBuilder) { + builder.write_u64(self.0); } } - fn key_fields(keys: &[InternalCacheKey]) -> BTreeSet<(String, String, &'static str)> { - keys.iter() - .map(|key| { - ( - key.prefix().to_string(), - key.key().to_string(), - key.type_name(), - ) - }) - .collect() + struct ReentrantValue(LanceCache); + + impl DeepSizeOf for ReentrantValue { + fn deep_size_of_children(&self, context: &mut Context) -> usize { + self.0.deep_size_of_children(context) + } } - #[tokio::test] - async fn test_cache_bytes() { - let item = Arc::new(vec![1, 2, 3]); - let item_size = item.deep_size_of(); - let capacity = 10 * item_size; - let cache = LanceCache::with_capacity(capacity); + struct ReentrantKey; - cache - .insert_with_key(&TestKey::>::new("key"), item.clone()) - .await; - assert_eq!(cache.size().await, 1); + impl CacheKey for ReentrantKey { + type ValueType = ReentrantValue; - let retrieved = cache - .get_with_key(&TestKey::>::new("key")) - .await - .unwrap(); - assert_eq!(*retrieved, *item); + fn key(&self) -> Cow<'_, str> { + Cow::Borrowed("reentrant") + } - for i in 0..20 { - cache - .insert_with_key( - &TestKey::>::new(&format!("key_{}", i)), - Arc::new(vec![i, i, i]), - ) - .await; + fn type_name() -> &'static str { + "test.ReentrantValue" } - assert!(cache.size_bytes().await <= capacity); } - #[tokio::test] - async fn test_cache_weighs_key_footprint() { - // Weighted size charges the key's unique bytes, not just the value. - let cache = LanceCache::with_capacity(usize::MAX); - let key = "k".repeat(10_000); - let value = Arc::new(vec![1_i32]); - let expected = - std::mem::size_of::() + key.len() + cache_entry_size(&*value); - cache - .insert_with_key(&TestKey::>::new(&key), value) - .await; - assert_eq!(cache.size_bytes().await, expected); + #[derive(Clone, Copy, Debug)] + enum TestBackendKind { + Moka, + Quick, } - #[tokio::test] - async fn test_cache_shared_prefix_not_charged_per_entry() { - // The shared prefix contributes nothing per entry (it isn't freed on a - // single eviction); only struct + unique key + value are charged. - let cache = LanceCache::with_capacity(usize::MAX).with_key_prefix(&"p".repeat(10_000)); - for i in 0..100 { - cache - .insert_with_key( - &TestKey::>::new(&i.to_string()), - Arc::new(vec![1_i32]), - ) - .await; + impl TestBackendKind { + fn cache(self, capacity: usize) -> LanceCache { + match self { + Self::Moka => LanceCache::with_capacity(capacity), + Self::Quick => { + LanceCache::with_backend(Arc::new(QuickCacheBackend::with_capacity(capacity))) + } + } } - let value_cost = cache_entry_size(&vec![1_i32]); - let key_bytes: usize = (0..100).map(|i| i.to_string().len()).sum(); - let expected = 100 * (std::mem::size_of::() + value_cost) + key_bytes; - assert_eq!(cache.size_bytes().await, expected); } - #[tokio::test] - async fn test_cache_trait_objects() { - #[derive(Debug, DeepSizeOf)] - struct MyType(i32); + struct LegacyBridgeKey(&'static str); + + impl CacheKey for LegacyBridgeKey { + type ValueType = Vec; - trait MyTrait: DeepSizeOf + Send + Sync + std::any::Any { - fn as_any(&self) -> &dyn std::any::Any; + fn key(&self) -> Cow<'_, str> { + Cow::Borrowed(self.0) } - impl MyTrait for MyType { - fn as_any(&self) -> &dyn std::any::Any { - self - } + fn type_name() -> &'static str { + "test.LegacyBridge" } + } - let item: Arc = Arc::new(MyType(42)); - let cache = LanceCache::with_capacity(1000); - cache - .insert_unsized_with_key(&TestUnsizedKey::::new("test"), item) - .await; + struct ExplicitBridgeKey(&'static str); - let retrieved = cache - .get_unsized_with_key(&TestUnsizedKey::::new("test")) - .await - .unwrap(); - assert_eq!(retrieved.as_any().downcast_ref::().unwrap().0, 42); + impl CacheKey for ExplicitBridgeKey { + type ValueType = Vec; + + fn key(&self) -> Cow<'_, str> { + Cow::Borrowed(self.0) + } + + fn type_name() -> &'static str { + "test.LegacyBridge" + } + + fn write_key(&self, builder: &mut KeyBuilder) { + builder.write_str(self.0); + } } - #[tokio::test] - async fn test_cache_stats_basic() { - let cache = LanceCache::with_capacity(1000); - assert_eq!(cache.stats().await.hits, 0); + trait TestDynValue: DeepSizeOf + Send + Sync { + fn values(&self) -> &[u32]; + } - // Miss - assert!( - cache - .get_with_key(&TestKey::>::new("x")) + impl TestDynValue for Vec { + fn values(&self) -> &[u32] { + self + } + } + + struct LegacyUnsizedBridgeKey(&'static str); + + impl UnsizedCacheKey for LegacyUnsizedBridgeKey { + type ValueType = dyn TestDynValue; + + fn key(&self) -> Cow<'_, str> { + Cow::Borrowed(self.0) + } + + fn type_name() -> &'static str { + "test.LegacyUnsizedBridge" + } + } + + struct ExplicitUnsizedBridgeKey(&'static str); + + impl UnsizedCacheKey for ExplicitUnsizedBridgeKey { + type ValueType = dyn TestDynValue; + + fn key(&self) -> Cow<'_, str> { + Cow::Borrowed(self.0) + } + + fn type_name() -> &'static str { + "test.LegacyUnsizedBridge" + } + + fn write_key(&self, builder: &mut KeyBuilder) { + builder.write_str(self.0); + } + } + + #[derive(Debug, Default)] + struct HashMapBackend { + entries: tokio::sync::Mutex>, + } + + #[async_trait::async_trait] + impl CacheBackend for HashMapBackend { + async fn get( + &self, + key: &InternalCacheKey, + _codec: Option, + ) -> Option { + self.entries + .lock() .await - .is_none() - ); - assert_eq!(cache.stats().await.misses, 1); + .get(key) + .map(|(entry, _)| entry.clone()) + } - // Insert then hit - cache - .insert_with_key(&TestKey::new("k"), Arc::new(vec![1, 2, 3])) - .await; - assert!( - cache - .get_with_key(&TestKey::>::new("k")) + async fn insert( + &self, + key: &InternalCacheKey, + entry: CacheEntry, + size_bytes: usize, + _codec: Option, + ) { + self.entries.lock().await.insert(*key, (entry, size_bytes)); + } + + async fn get_or_insert<'a>( + &self, + key: &InternalCacheKey, + loader: Pin> + Send + 'a>>, + codec: Option, + ) -> Result<(CacheEntry, bool)> { + if let Some(entry) = self.get(key, codec).await { + return Ok((entry, true)); + } + let (entry, size_bytes) = loader.await?; + self.insert(key, entry.clone(), size_bytes, codec).await; + Ok((entry, false)) + } + + async fn clear(&self) { + self.entries.lock().await.clear(); + } + + async fn num_entries(&self) -> usize { + self.entries.lock().await.len() + } + + async fn size_bytes(&self) -> usize { + self.entries + .lock() .await - .is_some() + .values() + .map(|(_, size_bytes)| size_bytes) + .sum() + } + } + + #[derive(Debug)] + struct WrongTypeBackend; + + #[async_trait::async_trait] + impl CacheBackend for WrongTypeBackend { + async fn get( + &self, + _key: &InternalCacheKey, + _codec: Option, + ) -> Option { + Some(Arc::new(String::from("wrong type"))) + } + + async fn insert( + &self, + _key: &InternalCacheKey, + _entry: CacheEntry, + _size_bytes: usize, + _codec: Option, + ) { + } + + async fn get_or_insert<'a>( + &self, + _key: &InternalCacheKey, + _loader: Pin> + Send + 'a>>, + _codec: Option, + ) -> Result<(CacheEntry, bool)> { + Ok((Arc::new(String::from("wrong type")), true)) + } + + async fn clear(&self) {} + + async fn num_entries(&self) -> usize { + 0 + } + + async fn size_bytes(&self) -> usize { + 0 + } + } + + #[tokio::test] + async fn typed_roundtrip_stats_clear_and_namespace_isolation() { + let cache = LanceCache::with_capacity(4096); + let left = cache.with_key_prefix("left"); + let right = cache.with_key_prefix("right"); + left.insert_with_key(&TestKey::new(7), Arc::new(vec![1, 2, 3])) + .await; + + assert_eq!( + left.get_with_key(&TestKey::new(7)).await.as_deref(), + Some(&vec![1, 2, 3]) ); - assert_eq!(cache.stats().await.hits, 1); + assert!(right.get_with_key(&TestKey::new(7)).await.is_none()); + let stats = cache.stats().await; + assert_eq!((stats.hits, stats.misses, stats.num_entries), (1, 1, 1)); + + cache.clear().await; + let stats = left.stats().await; + assert_eq!((stats.hits, stats.misses, stats.num_entries), (0, 0, 0)); } #[tokio::test] - async fn test_cache_stats_with_prefixes() { - let base = LanceCache::with_capacity(1000); - let prefixed = base.with_key_prefix("ns"); + async fn strong_and_weak_handles_share_state_and_namespace() { + let cache = LanceCache::with_capacity(4096); + let child = cache.with_key_prefix("child"); + let weak = WeakLanceCache::from(&child); assert!( - prefixed - .get_with_key(&TestKey::>::new("k")) + weak.insert_with_key(&TestKey::new(1), Arc::new(vec![1])) .await - .is_none() ); - assert_eq!(base.stats().await.misses, 1); - - prefixed - .insert_with_key(&TestKey::new("k"), Arc::new(vec![1])) + assert_eq!( + child.get_with_key(&TestKey::new(1)).await.as_deref(), + Some(&vec![1]) + ); + child + .insert_with_key(&TestKey::new(2), Arc::new(vec![2])) .await; - assert!( - prefixed - .get_with_key(&TestKey::>::new("k")) - .await - .is_some() + assert_eq!( + weak.get_with_key(&TestKey::new(2)).await.as_deref(), + Some(&vec![2]) ); - assert_eq!(base.stats().await.hits, 1); + assert_eq!((cache.stats().await.hits, cache.size().await), (2, 2)); } #[tokio::test] - async fn test_cache_keys_with_prefixes() { - let base = LanceCache::with_capacity(1000); - let prefixed = base.with_key_prefix("ns"); - let nested = prefixed.with_key_prefix("index"); - let other = base.with_key_prefix("ns-other"); - - base.insert_with_key(&TestKey::new("root"), Arc::new(vec![0])) - .await; - prefixed - .insert_with_key(&TestKey::new("child"), Arc::new(vec![1])) - .await; + async fn nested_namespace_segments_do_not_alias_combined_segments() { + let cache = LanceCache::with_capacity(4096); + let nested = cache.with_key_prefix("a").with_key_prefix("b"); + let combined = cache.with_key_prefix("a/b"); nested - .insert_with_key(&TestKey::new("nested"), Arc::new(vec![2])) + .insert_with_key(&TestKey::new(1), Arc::new(vec![10])) .await; - other - .insert_with_key(&TestKey::new("other"), Arc::new(vec![3])) + assert!(combined.get_with_key(&TestKey::new(1)).await.is_none()); + } + + #[tokio::test] + async fn schema_change_produces_a_cold_miss() { + let cache = LanceCache::with_capacity(4096); + cache + .insert_with_key(&TestKey::new(1), Arc::new(vec![10])) .await; + assert!(cache.get_with_key(&TestKeyV2::new(1)).await.is_none()); + } - let base_keys = base.keys().await.unwrap().collect::>(); - assert_eq!( - key_fields(&base_keys), - BTreeSet::from([ - ( - "".to_string(), - "root".to_string(), - TestKey::>::type_name() - ), - ( - "ns/".to_string(), - "child".to_string(), - TestKey::>::type_name() - ), - ( - "ns/index/".to_string(), - "nested".to_string(), - TestKey::>::type_name() - ), - ( - "ns-other/".to_string(), - "other".to_string(), - TestKey::>::type_name() - ), - ]) - ); + #[tokio::test] + async fn get_or_insert_with_key_hit_reports_loader_execution() { + let cache = LanceCache::with_capacity(4096); + + // Cold: loader runs, was_cached = false. + let (value, was_cached) = cache + .get_or_insert_with_key_hit(TestKey::new(1), || async { Ok(vec![1, 2, 3]) }) + .await + .unwrap(); + assert_eq!(*value, vec![1, 2, 3]); + assert!(!was_cached); + + // Warm: loader must not run and was_cached = true. + let (value, was_cached) = cache + .get_or_insert_with_key_hit(TestKey::new(1), || async { + panic!("should not be called") + }) + .await + .unwrap(); + assert_eq!(*value, vec![1, 2, 3]); + assert!(was_cached); + } - let prefixed_keys = prefixed.keys().await.unwrap().collect::>(); + #[tokio::test] + async fn default_string_bridge_matches_explicit_legacy_encoding() { + let cache = LanceCache::with_capacity(4096); + cache + .insert_with_key(&LegacyBridgeKey("same"), Arc::new(vec![10])) + .await; assert_eq!( - key_fields(&prefixed_keys), - BTreeSet::from([ - ( - "ns/".to_string(), - "child".to_string(), - TestKey::>::type_name() - ), - ( - "ns/index/".to_string(), - "nested".to_string(), - TestKey::>::type_name() - ), - ]) + cache + .get_with_key(&ExplicitBridgeKey("same")) + .await + .as_deref(), + Some(&vec![10]) ); } #[tokio::test] - async fn test_cache_keys_reflect_invalidation_and_clear() { - let base = LanceCache::with_capacity(1000); - let prefixed = base.with_key_prefix("ns"); - let other = base.with_key_prefix("other"); - - prefixed - .insert_with_key(&TestKey::new("child"), Arc::new(vec![1])) - .await; - other - .insert_with_key(&TestKey::new("other"), Arc::new(vec![2])) + async fn unsized_default_string_bridge_matches_explicit_legacy_encoding() { + let cache = LanceCache::with_capacity(4096); + let value: Arc = Arc::new(vec![10, 20]); + cache + .insert_unsized_with_key(&LegacyUnsizedBridgeKey("same"), value) .await; - assert_eq!(base.keys().await.unwrap().count(), 2); - prefixed.invalidate_prefix("").await; - let keys = base.keys().await.unwrap().collect::>(); + let cached = cache + .get_unsized_with_key(&ExplicitUnsizedBridgeKey("same")) + .await + .unwrap(); + assert_eq!(cached.values(), &[10, 20]); + } + + #[tokio::test] + async fn custom_backend_receives_opaque_keys_and_shared_clear() { + let backend = Arc::new(HashMapBackend::default()); + let cache = LanceCache::with_backend(backend.clone()); + let child = cache.with_key_prefix("child"); + let value = Arc::new(vec![1, 2, 3]); + let value_size = cache_entry_size(value.as_ref()); + + child.insert_with_key(&TestKey::new(7), value).await; assert_eq!( - key_fields(&keys), - BTreeSet::from([( - "other/".to_string(), - "other".to_string(), - TestKey::>::type_name() - )]) + child.get_with_key(&TestKey::new(7)).await.as_deref(), + Some(&vec![1, 2, 3]) ); + assert_eq!(backend.entries.lock().await.len(), 1); + assert_eq!(cache.size_bytes().await, value_size); - base.clear().await; - assert_eq!(base.keys().await.unwrap().count(), 0); + cache.clear().await; + assert!(backend.entries.lock().await.is_empty()); + assert_eq!(child.stats().await.hits, 0); } #[tokio::test] - async fn test_cache_get_or_insert() { - let cache = LanceCache::with_capacity(1000); + async fn backend_type_collisions_are_contextual_misses_or_errors() { + let cache = LanceCache::with_backend(Arc::new(WrongTypeBackend)); - let v: Arc> = cache - .get_or_insert_with_key(TestKey::>::new("k"), || async { - Ok(vec![1, 2, 3]) - }) + assert!(cache.get_with_key(&TestKey::new(1)).await.is_none()); + let error = cache + .get_or_insert_with_key(TestKey::new(2), || async { Ok(vec![2]) }) .await - .unwrap(); - assert_eq!(*v, vec![1, 2, 3]); - assert_eq!(cache.stats().await.misses, 1); - assert_eq!(cache.stats().await.hits, 0); + .unwrap_err(); + assert!(error.to_string().contains("test.VecU32")); + let stats = cache.stats().await; + assert_eq!((stats.hits, stats.misses), (0, 2)); + } - // Second call should not invoke loader and should be a hit - let v: Arc> = cache - .get_or_insert_with_key(TestKey::>::new("k"), || async { - panic!("should not be called") - }) - .await + #[tokio::test] + async fn moka_weight_includes_the_fixed_physical_key() { + let value = Arc::new(vec![0_u32; 3]); + let expected = cache_entry_size(value.as_ref()) + .checked_add(std::mem::size_of::()) .unwrap(); - assert_eq!(*v, vec![1, 2, 3]); - assert_eq!(cache.stats().await.hits, 1); + let cache = LanceCache::with_capacity(expected * 2); + cache.insert_with_key(&TestKey::new(1), value).await; + assert_eq!(cache.size_bytes().await, expected); } + #[rstest::rstest] + #[case::moka(TestBackendKind::Moka)] + #[case::quick(TestBackendKind::Quick)] #[tokio::test] - async fn test_custom_backend() { - use async_trait::async_trait; - use tokio::sync::Mutex; + async fn deep_size_deduplicates_shared_entry_allocations( + #[case] backend_kind: TestBackendKind, + ) { + let cache = backend_kind.cache(1 << 20); + let shared_data = Arc::new(vec![0_u8; 1024]); - #[derive(Debug)] - struct HashMapBackend { - map: Mutex>, + for id in 0..2 { + let data = shared_data.clone(); + cache + .get_or_insert_with_key(SharedTestKey(id), || async move { + Ok(SharedTestValue { data }) + }) + .await + .unwrap(); } - impl HashMapBackend { - fn new() -> Self { - Self { - map: Mutex::new(HashMap::new()), - } - } - } + let arc_overhead = std::mem::size_of::() * 2; + let shared_allocation = std::mem::size_of::>() + shared_data.capacity(); + let expected_entries = 2 * std::mem::size_of::() + + 2 * (std::mem::size_of::() + arc_overhead) + + shared_allocation; - #[async_trait] - impl CacheBackend for HashMapBackend { - async fn get( - &self, - key: &InternalCacheKey, - _codec: Option, - ) -> Option { - self.map.lock().await.get(key).map(|(e, _)| e.clone()) - } - async fn insert( - &self, - key: &InternalCacheKey, - entry: CacheEntry, - size_bytes: usize, - _codec: Option, - ) { - self.map - .lock() - .await - .insert(key.clone(), (entry, size_bytes)); - } - async fn get_or_insert<'a>( - &self, - key: &InternalCacheKey, - loader: std::pin::Pin< - Box> + Send + 'a>, - >, - _codec: Option, - ) -> Result<(CacheEntry, bool)> { - if let Some((entry, _)) = self.map.lock().await.get(key) { - Ok((entry.clone(), true)) - } else { - let (entry, size) = loader.await?; - self.map - .lock() - .await - .insert(key.clone(), (entry.clone(), size)); - Ok((entry, false)) - } - } - async fn invalidate_prefix(&self, prefix: &str) { - self.map.lock().await.retain(|k, _| !k.starts_with(prefix)); - } - async fn clear(&self) { - self.map.lock().await.clear(); - } - async fn num_entries(&self) -> usize { - self.map.lock().await.len() - } - async fn size_bytes(&self) -> usize { - self.map.lock().await.values().map(|(_, s)| *s).sum() - } + let weighted_size = cache.size_bytes().await; + assert_eq!(weighted_size, expected_entries + shared_allocation); + assert_eq!( + cache.deep_size_of(), + std::mem::size_of::() + expected_entries + ); + + let mut context = Context::new(); + assert_eq!(cache.deep_size_of_children(&mut context), expected_entries); + assert_eq!( + cache + .with_key_prefix("another-handle") + .deep_size_of_children(&mut context), + 0 + ); + } + + #[test] + fn sizing_can_reenter_the_same_cache() { + let (done_tx, done_rx) = mpsc::channel(); + let worker = thread::spawn(move || { + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .unwrap(); + runtime.block_on(async move { + let cache = LanceCache::with_capacity(4096); + cache + .insert_with_key(&ReentrantKey, Arc::new(ReentrantValue(cache.clone()))) + .await; + done_tx.send(()).unwrap(); + }); + }); + + done_rx + .recv_timeout(Duration::from_secs(1)) + .expect("cache insertion deadlocked during sizing"); + worker.join().unwrap(); + } + + #[tokio::test] + async fn no_cache_computes_each_time() { + let cache = LanceCache::no_cache(); + let loads = Arc::new(AtomicUsize::new(0)); + for _ in 0..2 { + let loads = loads.clone(); + let value = cache + .get_or_insert_with_key(TestKey::new(1), move || async move { + loads.fetch_add(1, Ordering::SeqCst); + Ok(vec![42]) + }) + .await + .unwrap(); + assert_eq!(value.as_slice(), &[42]); } + assert_eq!(loads.load(Ordering::SeqCst), 2); + assert_eq!(cache.size().await, 0); + } - let cache = LanceCache::with_backend(Arc::new(HashMapBackend::new())); + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn single_flight_coalesces_success_after_contenders_are_parked() { + const CONTENDERS: usize = 4; - cache - .insert_with_key(&TestKey::new("k"), Arc::new(vec![1, 2, 3])) - .await; - assert!( - cache - .get_with_key(&TestKey::>::new("k")) + let cache = Arc::new(LanceCache::with_capacity(4096)); + let loader_calls = Arc::new(AtomicUsize::new(0)); + let release = Arc::new(tokio::sync::Notify::new()); + let (started_tx, started_rx) = tokio::sync::oneshot::channel(); + + let owner = { + let cache = cache.clone(); + let loader_calls = loader_calls.clone(); + let release = release.clone(); + tokio::spawn(async move { + cache + .get_or_insert_with_key(TestKey::new(10), move || async move { + loader_calls.fetch_add(1, Ordering::SeqCst); + let _ = started_tx.send(()); + release.notified().await; + Ok(vec![10]) + }) + .await + }) + }; + started_rx.await.unwrap(); + + let mut contenders = Vec::new(); + let mut parked = Vec::new(); + for _ in 0..CONTENDERS { + let cache = cache.clone(); + let loader_calls = loader_calls.clone(); + let (parked_tx, parked_rx) = tokio::sync::oneshot::channel(); + parked.push(parked_rx); + contenders.push(tokio::spawn(async move { + report_first_pending( + cache.get_or_insert_with_key(TestKey::new(10), move || async move { + loader_calls.fetch_add(1, Ordering::SeqCst); + Ok(vec![99]) + }), + parked_tx, + ) .await - .is_some() - ); - // Different type at same key = miss - assert!( - cache - .get_with_key(&TestKey::>::new("k")) + })); + } + for parked in parked { + parked .await - .is_none() - ); - assert!(cache.keys().await.is_none()); + .expect("contender completed instead of parking behind owner"); + } + assert_eq!(loader_calls.load(Ordering::SeqCst), 1); + assert!(contenders.iter().all(|handle| !handle.is_finished())); + + release.notify_one(); + assert_eq!(owner.await.unwrap().unwrap().as_slice(), &[10]); + for contender in contenders { + assert_eq!(contender.await.unwrap().unwrap().as_slice(), &[10]); + } + let stats = cache.stats().await; + assert_eq!((stats.hits, stats.misses), (CONTENDERS as u64, 1),); } - #[tokio::test] - async fn test_get_or_insert_dedup() { - use std::sync::atomic::AtomicUsize; + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn single_flight_coalesces_errors_after_contenders_are_parked() { + const CONTENDERS: usize = 4; - let load_count = Arc::new(AtomicUsize::new(0)); - let cache = LanceCache::with_capacity(10000); + let cache = Arc::new(LanceCache::with_capacity(4096)); + let loader_calls = Arc::new(AtomicUsize::new(0)); + let release = Arc::new(tokio::sync::Notify::new()); + let (started_tx, started_rx) = tokio::sync::oneshot::channel(); - let (barrier_tx, _) = tokio::sync::broadcast::channel::<()>(1); - let mut handles = Vec::new(); - for _ in 0..5 { + let owner = { let cache = cache.clone(); - let load_count = load_count.clone(); - let mut barrier_rx = barrier_tx.subscribe(); - handles.push(tokio::spawn(async move { - barrier_rx.recv().await.ok(); + let loader_calls = loader_calls.clone(); + let release = release.clone(); + tokio::spawn(async move { cache - .get_or_insert_with_key(TestKey::>::new("key"), || { - let load_count = load_count.clone(); - async move { - load_count.fetch_add(1, Ordering::SeqCst); - tokio::task::yield_now().await; - Ok(vec![1, 2, 3]) - } + .get_or_insert_with_key(TestKey::new(20), move || async move { + loader_calls.fetch_add(1, Ordering::SeqCst); + let _ = started_tx.send(()); + release.notified().await; + Err(Error::timeout("owner loader timed out")) }) .await + }) + }; + started_rx.await.unwrap(); + + let mut contenders = Vec::new(); + let mut parked = Vec::new(); + for _ in 0..CONTENDERS { + let cache = cache.clone(); + let loader_calls = loader_calls.clone(); + let (parked_tx, parked_rx) = tokio::sync::oneshot::channel(); + parked.push(parked_rx); + contenders.push(tokio::spawn(async move { + report_first_pending( + cache.get_or_insert_with_key(TestKey::new(20), move || async move { + loader_calls.fetch_add(1, Ordering::SeqCst); + Err(Error::timeout("contender loader timed out")) + }), + parked_tx, + ) + .await })); } - barrier_tx.send(()).unwrap(); - for h in handles { - let result: Arc> = h.await.unwrap().unwrap(); - assert_eq!(*result, vec![1, 2, 3]); + for parked in parked { + parked + .await + .expect("contender completed instead of parking behind owner"); + } + assert_eq!(loader_calls.load(Ordering::SeqCst), 1); + assert!(contenders.iter().all(|handle| !handle.is_finished())); + + release.notify_one(); + assert!(matches!(owner.await.unwrap(), Err(Error::Timeout { .. }))); + for contender in contenders { + assert!(matches!( + contender.await.unwrap(), + Err(Error::Timeout { .. }) + )); } + assert_eq!(loader_calls.load(Ordering::SeqCst), 1); + } - assert_eq!(load_count.load(Ordering::SeqCst), 1); + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn single_flight_retries_after_the_owner_is_cancelled() { + let cache = Arc::new(LanceCache::with_capacity(4096)); + let loader_calls = Arc::new(AtomicUsize::new(0)); + let (started_tx, started_rx) = tokio::sync::oneshot::channel(); + + let owner = { + let cache = cache.clone(); + let loader_calls = loader_calls.clone(); + tokio::spawn(async move { + cache + .get_or_insert_with_key(TestKey::new(30), move || async move { + loader_calls.fetch_add(1, Ordering::SeqCst); + let _ = started_tx.send(()); + std::future::pending::<()>().await; + Ok(vec![30]) + }) + .await + }) + }; + started_rx.await.unwrap(); + + let (parked_tx, parked_rx) = tokio::sync::oneshot::channel(); + let contender = { + let cache = cache.clone(); + let loader_calls = loader_calls.clone(); + tokio::spawn(async move { + report_first_pending( + cache.get_or_insert_with_key(TestKey::new(30), move || async move { + loader_calls.fetch_add(1, Ordering::SeqCst); + Ok(vec![31]) + }), + parked_tx, + ) + .await + }) + }; + parked_rx + .await + .expect("contender completed instead of parking behind owner"); + assert_eq!(loader_calls.load(Ordering::SeqCst), 1); + assert!(!contender.is_finished()); + + owner.abort(); + assert!(owner.await.unwrap_err().is_cancelled()); + let value = tokio::time::timeout(std::time::Duration::from_secs(5), contender) + .await + .expect("contender remained parked after owner cancellation") + .unwrap() + .unwrap(); + assert_eq!(value.as_slice(), &[31]); + assert_eq!(loader_calls.load(Ordering::SeqCst), 2); + } + + #[tokio::test] + async fn expired_weak_cache_degrades_without_retaining_state() { + let cache = LanceCache::with_capacity(4096); + let weak = WeakLanceCache::from(&cache); + drop(cache); + + assert!(weak.get_with_key(&TestKey::new(1)).await.is_none()); + assert!( + !weak + .insert_with_key(&TestKey::new(1), Arc::new(vec![1])) + .await + ); + let value = weak + .get_or_insert_with_key(TestKey::new(1), || async { Ok(vec![7]) }) + .await + .unwrap(); + assert_eq!(value.as_slice(), &[7]); } } diff --git a/rust/lance-core/src/cache/moka.rs b/rust/lance-core/src/cache/moka.rs index fd86b064f6b..2a93ba6c2ca 100644 --- a/rust/lance-core/src/cache/moka.rs +++ b/rust/lance-core/src/cache/moka.rs @@ -9,9 +9,11 @@ use async_trait::async_trait; use futures::Future; use crate::Result; +use crate::deepsize::Context; +use crate::error::CloneableError; -use super::CacheCodec; -use super::backend::{CacheBackend, CacheEntry, CacheKeyIterator, InternalCacheKey}; +use super::backend::{CacheBackend, CacheEntry}; +use super::{CacheCodec, InternalCacheKey}; /// Internal record stored in the moka cache. #[derive(Clone, Debug)] @@ -20,10 +22,28 @@ struct MokaCacheEntry { size_bytes: usize, } -/// Per-entry key cost for eviction: the struct plus the unique `key` bytes. -/// Excludes the shared `prefix` `Arc`, which isn't freed per eviction. -fn key_footprint(key: &InternalCacheKey) -> usize { - std::mem::size_of::() + key.key().len() +/// Per-entry key cost for eviction. +pub(super) fn key_footprint(_key: &InternalCacheKey) -> usize { + std::mem::size_of::() +} + +fn physical_size(key: &InternalCacheKey, size_bytes: usize) -> usize { + key_footprint(key).saturating_add(size_bytes) +} + +/// Number of physical bytes represented by one Moka weight unit. +/// +/// Moka limits each entry's weight to `u32`, so capacities above 4 GiB need +/// coarser units to account for a single large entry without undercharging it. +fn weight_unit(capacity: usize) -> usize { + capacity.div_ceil(u32::MAX as usize).max(1) +} + +fn entry_weight(key: &InternalCacheKey, size_bytes: usize, weight_unit: usize) -> u32 { + physical_size(key, size_bytes) + .div_ceil(weight_unit) + .try_into() + .unwrap_or(u32::MAX) } /// Default [`CacheBackend`] backed by a [moka](https://crates.io/crates/moka) cache. @@ -32,6 +52,8 @@ fn key_footprint(key: &InternalCacheKey) -> usize { /// via moka's built-in `optionally_get_with`. pub struct MokaCacheBackend { cache: moka::future::Cache, + capacity: usize, + weight_unit: usize, } impl std::fmt::Debug for MokaCacheBackend { @@ -44,24 +66,41 @@ impl std::fmt::Debug for MokaCacheBackend { impl MokaCacheBackend { pub fn with_capacity(capacity: usize) -> Self { + let weight_unit = weight_unit(capacity); + let capacity_weight = capacity.div_ceil(weight_unit) as u64; let cache = moka::future::Cache::builder() - .max_capacity(capacity as u64) - .weigher(|key: &InternalCacheKey, entry: &MokaCacheEntry| { - key_footprint(key) - .saturating_add(entry.size_bytes) - .try_into() - .unwrap_or(u32::MAX) + .max_capacity(capacity_weight) + .weigher(move |key: &InternalCacheKey, entry: &MokaCacheEntry| { + entry_weight(key, entry.size_bytes, weight_unit) }) - .support_invalidation_closures() .build(); - Self { cache } + Self { + cache, + capacity, + weight_unit, + } } pub fn no_cache() -> Self { Self { cache: moka::future::Cache::new(0), + capacity: 0, + weight_unit: 1, } } + + /// Configured weighted capacity in bytes. + pub fn capacity(&self) -> usize { + self.capacity + } + + fn weighted_size_bytes(&self) -> usize { + self.cache + .weighted_size() + .saturating_mul(self.weight_unit as u64) + .try_into() + .unwrap_or(usize::MAX) + } } #[async_trait] @@ -78,7 +117,7 @@ impl CacheBackend for MokaCacheBackend { _codec: Option, ) { self.cache - .insert(key.clone(), MokaCacheEntry { entry, size_bytes }) + .insert(*key, MokaCacheEntry { entry, size_bytes }) .await; } @@ -88,59 +127,33 @@ impl CacheBackend for MokaCacheBackend { loader: Pin> + Send + 'a>>, _codec: Option, ) -> Result<(CacheEntry, bool)> { - // Use moka's built-in dedup: optionally_get_with runs the init future - // at most once per key, even under concurrent access. - let (error_tx, error_rx) = tokio::sync::oneshot::channel(); - // Track whether the loader actually ran (= cache miss). let was_miss = Arc::new(AtomicBool::new(false)); let was_miss_clone = was_miss.clone(); let init = async move { was_miss_clone.store(true, Ordering::Relaxed); - match loader.await { - Ok((entry, size_bytes)) => Some(MokaCacheEntry { entry, size_bytes }), - Err(e) => { - let _ = error_tx.send(e); - None - } - } + loader + .await + .map(|(entry, size_bytes)| MokaCacheEntry { entry, size_bytes }) + .map_err(CloneableError) }; - let owned_key = key.clone(); - match self.cache.optionally_get_with(owned_key, init).await { - Some(record) => { + let owned_key = *key; + match self.cache.try_get_with(owned_key, init).await { + Ok(record) => { let was_cached = !was_miss.load(Ordering::Relaxed); Ok((record.entry, was_cached)) } - None => match error_rx.await { - Ok(err) => Err(err), - Err(_) => Err(crate::Error::internal( - "Failed to retrieve error from cache loader", - )), - }, + Err(error) => Err(Arc::unwrap_or_clone(error).0), } } - async fn invalidate_prefix(&self, prefix: &str) { - let prefix = prefix.to_owned(); - self.cache - .invalidate_entries_if(move |key, _value| key.starts_with(&prefix)) - .expect("Cache configured correctly"); - } - async fn clear(&self) { self.cache.invalidate_all(); self.cache.run_pending_tasks().await; } - async fn keys(&self) -> Option> { - self.cache.run_pending_tasks().await; - Some(Box::new( - self.cache.iter().map(|(key, _)| key.as_ref().clone()), - )) - } - async fn num_entries(&self) -> usize { self.cache.run_pending_tasks().await; self.cache.entry_count() as usize @@ -148,7 +161,7 @@ impl CacheBackend for MokaCacheBackend { async fn size_bytes(&self) -> usize { self.cache.run_pending_tasks().await; - self.cache.weighted_size() as usize + self.weighted_size_bytes() } fn approx_num_entries(&self) -> usize { @@ -156,12 +169,181 @@ impl CacheBackend for MokaCacheBackend { } fn approx_size_bytes(&self) -> usize { - // Iterate rather than using `weighted_size()` because moka's - // weighted_size can be stale without `run_pending_tasks()`, which + // `weighted_size()` can be stale without `run_pending_tasks()`, which // is async and can't be called from this synchronous context. - self.cache - .iter() - .map(|(key, entry)| key_footprint(key.as_ref()) + entry.size_bytes) - .sum() + self.weighted_size_bytes() + } + + fn deep_size_of_entries( + &self, + context: &mut Context, + size_of_entry: &dyn Fn(&CacheEntry, &mut Context) -> Option, + ) -> Option { + Some( + self.cache + .iter() + .map(|(key, record)| { + key_footprint(key.as_ref()) + + size_of_entry(&record.entry, context).unwrap_or(record.size_bytes) + }) + .sum(), + ) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn entry_weights_are_exact_at_byte_granularity() { + let key = InternalCacheKey::from_bytes([0; 16]); + assert_eq!(weight_unit(4096), 1); + assert_eq!(entry_weight(&key, 7, 1), 23); + } + + #[tokio::test] + async fn size_methods_use_constant_time_weighted_accounting() { + let backend = MokaCacheBackend::with_capacity(4096); + let key = InternalCacheKey::from_bytes([0; 16]); + let entry: CacheEntry = Arc::new(()); + let value_size = 7; + let expected = physical_size(&key, value_size); + + backend.insert(&key, entry, value_size, None).await; + + assert_eq!(backend.size_bytes().await, expected); + assert_eq!(backend.approx_size_bytes(), expected); + } + + #[cfg(target_pointer_width = "64")] + #[test] + fn entry_weights_scale_for_capacities_above_four_gibibytes() { + let key = InternalCacheKey::from_bytes([0; 16]); + let capacity = 6 * 1024 * 1024 * 1024; + let weight_unit = weight_unit(capacity); + assert_eq!(weight_unit, 2); + + let size_bytes = u32::MAX as usize + 1024; + let expected = physical_size(&key, size_bytes).div_ceil(weight_unit); + let weight = entry_weight(&key, size_bytes, weight_unit); + assert_eq!(weight as usize, expected); + assert_ne!(weight, u32::MAX); + } +} + +/// Registry identifier for the built-in Moka backend. +pub const MOKA_BACKEND_KIND: &str = "moka"; + +/// [`BackendBuildFn`](super::registry::BackendBuildFn) for [`MokaCacheBackend`]. +/// +/// Recognized options: +/// * `capacity` — total weighted capacity in bytes (`usize`). +/// This must be present and non-empty. +/// +/// Unknown options are rejected so typos surface immediately instead of +/// silently falling through to the default capacity. +pub(super) fn build_moka_backend( + config: &super::registry::BackendConfig, +) -> Result { + let mut capacity: Option = None; + for (key, value) in &config.options { + match key.as_str() { + "capacity" => { + if value.is_empty() { + return Err(crate::Error::invalid_input( + "moka cache backend: capacity must not be empty", + )); + } else { + capacity = Some(value.parse::().map_err(|err| { + crate::Error::invalid_input(format!( + "moka cache backend: cannot parse capacity {:?}: {}", + value, err + )) + })?); + } + } + other => { + return Err(crate::Error::invalid_input(format!( + "moka cache backend: unknown option {:?}", + other + ))); + } + } + } + let capacity = capacity.ok_or_else(|| { + crate::Error::invalid_input( + "moka cache backend: capacity is required; use moka://?capacity=", + ) + })?; + Ok(MokaCacheBackend::with_capacity(capacity)) +} + +pub(super) fn build_moka(config: &super::registry::BackendConfig) -> Result> { + Ok(Arc::new(build_moka_backend(config)?)) +} + +#[cfg(test)] +mod moka_registry_tests { + use super::super::backend_uri::{build_from_uri, parse_backend_uri}; + use super::super::registry::{BackendConfig, build_from_config, registry_test_lock}; + use super::*; + + #[test] + fn test_moka_builds_from_config() { + let _lock = registry_test_lock(); + let cfg = BackendConfig::new("moka") + .unwrap() + .with_option("capacity", "1048576"); + let backend = build_moka_backend(&cfg).unwrap(); + assert_eq!(backend.capacity(), 1048576); + let _backend = build_from_config(&cfg).unwrap(); + } + + #[test] + fn test_moka_builds_from_uri() { + let _lock = registry_test_lock(); + let cfg = parse_backend_uri("moka://?capacity=1048576").unwrap(); + let backend = build_moka_backend(&cfg).unwrap(); + assert_eq!(backend.capacity(), 1048576); + let _backend = build_from_uri("moka://?capacity=1048576").unwrap(); + } + + #[test] + fn test_moka_rejects_unknown_option() { + let _lock = registry_test_lock(); + let cfg = BackendConfig::new("moka") + .unwrap() + .with_option("mystery", "1"); + let err = build_from_config(&cfg).unwrap_err(); + assert!(err.to_string().contains("unknown option")); + } + + #[test] + fn test_moka_rejects_bad_capacity() { + let _lock = registry_test_lock(); + let cfg = BackendConfig::new("moka") + .unwrap() + .with_option("capacity", "not-a-number"); + let err = build_from_config(&cfg).unwrap_err(); + assert!(err.to_string().contains("cannot parse capacity")); + } + + #[test] + fn test_moka_rejects_missing_capacity() { + let _lock = registry_test_lock(); + let cfg = BackendConfig::new("moka").unwrap(); + let err = build_from_config(&cfg).unwrap_err(); + assert!(err.to_string().contains("capacity is required")); + } + + #[test] + fn test_moka_rejects_empty_capacity() { + let _lock = registry_test_lock(); + let cfg = BackendConfig::new("moka") + .unwrap() + .with_option("capacity", ""); + let err = build_from_config(&cfg).unwrap_err(); + assert!(err.to_string().contains("capacity must not be empty")); } } diff --git a/rust/lance-core/src/cache/quick.rs b/rust/lance-core/src/cache/quick.rs new file mode 100644 index 00000000000..53e0d0e8870 --- /dev/null +++ b/rust/lance-core/src/cache/quick.rs @@ -0,0 +1,298 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright The Lance Authors + +//! [`CacheBackend`] backed by [quick_cache](https://crates.io/crates/quick_cache), +//! whose hit path is one atomic bit — no read-op channel or inline +//! housekeeping. Used for the session index and metadata caches; the index +//! cache sees thousands of cache reads per query. + +use std::pin::Pin; + +use async_trait::async_trait; +use futures::Future; + +use super::backend::{CacheBackend, CacheEntry}; +use super::moka::key_footprint; +use super::{CacheCodec, InternalCacheKey}; +use crate::Result; +use crate::deepsize::Context; + +#[derive(Clone)] +struct QuickEntry { + entry: CacheEntry, + size_bytes: usize, +} + +#[derive(Clone)] +struct EntryWeighter; + +impl quick_cache::Weighter for EntryWeighter { + fn weight(&self, key: &InternalCacheKey, value: &QuickEntry) -> u64 { + // Same accounting as the moka backend. + key_footprint(key).saturating_add(value.size_bytes).max(1) as u64 + } +} + +pub struct QuickCacheBackend { + cache: quick_cache::sync::Cache, +} + +impl std::fmt::Debug for QuickCacheBackend { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("QuickCacheBackend") + .field("entry_count", &self.cache.len()) + .finish() + } +} + +/// Minimum weight budget (4 GiB) per shard: shards don't borrow capacity, and +/// an entry heavier than ~its shard's budget is silently refused admission. +const MIN_SHARD_SHARE: usize = 4 << 30; + +/// Recommended shard count: `min(cpus / 2, capacity / 4 GiB)`, power of two +/// in `[1, 1024]`. The cpu term bounds lock contention; the capacity term +/// keeps each shard's budget >= 4 GiB so large entries stay admissible. +/// Rounded down because quick_cache rounds requests up. +pub fn recommended_cache_shards(capacity: usize) -> usize { + let by_cpu = std::thread::available_parallelism() + .map(|n| n.get()) + .unwrap_or(2) + / 2; + let shards = (capacity / MIN_SHARD_SHARE).min(by_cpu).max(1); + let shards = if shards.is_power_of_two() { + shards + } else { + shards.next_power_of_two() / 2 + }; + shards.clamp(1, 1024) +} + +/// Assumed average entry size for pre-allocation sizing. +const ESTIMATED_AVG_ENTRY_BYTES: usize = 64 << 10; + +impl QuickCacheBackend { + /// Create a backend holding up to `capacity` bytes of weighted entries + /// (weight = key footprint + declared size), sharded per + /// [`recommended_cache_shards`]. + pub fn with_capacity(capacity: usize) -> Self { + let shards = recommended_cache_shards(capacity); + // Floor protects the shard count from quick_cache's items-per-shard + // heuristic; ceiling bounds pre-allocation. + let estimated_items = (capacity / ESTIMATED_AVG_ENTRY_BYTES).clamp(shards * 32, 1_000_000); + let options = quick_cache::OptionsBuilder::new() + .estimated_items_capacity(estimated_items) + .weight_capacity(capacity as u64) + .shards(shards) + .build() + // Only errors when weight/item capacity is missing; both are set. + .expect("quick_cache options"); + let cache = quick_cache::sync::Cache::with_options( + options, + EntryWeighter, + Default::default(), + Default::default(), + ); + Self { cache } + } +} + +#[async_trait] +impl CacheBackend for QuickCacheBackend { + async fn get(&self, key: &InternalCacheKey, _codec: Option) -> Option { + self.cache.get(key).map(|v| v.entry) + } + + async fn insert( + &self, + key: &InternalCacheKey, + entry: CacheEntry, + size_bytes: usize, + _codec: Option, + ) { + self.cache.insert(*key, QuickEntry { entry, size_bytes }); + } + + async fn get_or_insert<'a>( + &self, + key: &InternalCacheKey, + loader: Pin> + Send + 'a>>, + _codec: Option, + ) -> Result<(CacheEntry, bool)> { + match self.cache.get_value_or_guard_async(key).await { + Ok(value) => Ok((value.entry, true)), + Err(guard) => { + let (entry, size_bytes) = loader.await?; + let _ = guard.insert(QuickEntry { + entry: entry.clone(), + size_bytes, + }); + Ok((entry, false)) + } + } + } + + async fn clear(&self) { + self.cache.clear(); + } + + async fn num_entries(&self) -> usize { + self.cache.len() + } + + async fn size_bytes(&self) -> usize { + self.cache.weight() as usize + } + + fn approx_num_entries(&self) -> usize { + self.cache.len() + } + + fn approx_size_bytes(&self) -> usize { + self.cache.weight() as usize + } + + fn deep_size_of_entries( + &self, + context: &mut Context, + size_of_entry: &dyn Fn(&CacheEntry, &mut Context) -> Option, + ) -> Option { + Some( + self.cache + .iter() + .map(|(key, record)| { + key_footprint(&key) + + size_of_entry(&record.entry, context).unwrap_or(record.size_bytes) + }) + .sum(), + ) + } +} + +#[cfg(test)] +mod tests { + use std::marker::PhantomData; + use std::sync::Arc; + use std::sync::atomic::{AtomicUsize, Ordering}; + + use super::*; + use crate::cache::{CacheKey, LanceCache}; + + struct TestKey { + key: String, + _phantom: PhantomData, + } + + impl TestKey { + fn new(key: &str) -> Self { + Self { + key: key.to_string(), + _phantom: PhantomData, + } + } + } + + impl CacheKey for TestKey { + type ValueType = T; + fn key(&self) -> std::borrow::Cow<'_, str> { + std::borrow::Cow::Borrowed(&self.key) + } + fn type_name() -> &'static str { + std::any::type_name::() + } + } + + #[test] + fn entry_weight_includes_fixed_key() { + let key = InternalCacheKey::from_bytes([0; 16]); + let entry = QuickEntry { + entry: Arc::new(()), + size_bytes: 7, + }; + assert_eq!( + quick_cache::Weighter::weight(&EntryWeighter, &key, &entry), + 23 + ); + } + + #[tokio::test] + async fn test_quick_backend_roundtrip_singleflight_and_eviction() { + // Capacity must be large relative to one entry: quick_cache shards + // its weight budget, and an entry heavier than its shard's share is + // not admitted at all. + const CAPACITY: usize = 1 << 20; + let item = Arc::new(vec![1u8, 2, 3]); + let cache = LanceCache::with_backend(Arc::new(QuickCacheBackend::with_capacity(CAPACITY))); + + // insert + get roundtrip and weighted accounting + cache + .insert_with_key(&TestKey::>::new("a"), item.clone()) + .await; + assert_eq!( + cache + .get_with_key(&TestKey::>::new("a")) + .await + .as_deref(), + Some(&vec![1u8, 2, 3]) + ); + assert_eq!(cache.approx_size(), 1); + assert!(cache.size_bytes().await > 0); + + // get_or_insert runs the loader only on a miss + let loads = Arc::new(AtomicUsize::new(0)); + for _ in 0..2 { + let loads = loads.clone(); + let value = cache + .get_or_insert_with_key(TestKey::>::new("b"), || async move { + loads.fetch_add(1, Ordering::SeqCst); + Ok(vec![7u8]) + }) + .await + .unwrap(); + assert_eq!(value.as_ref(), &vec![7u8]); + } + assert_eq!(loads.load(Ordering::SeqCst), 1); + + // capacity is enforced: overfill with 4x capacity of 16KiB entries + // and confirm eviction kept the weighted size within budget + for i in 0..256 { + cache + .insert_with_key( + &TestKey::>::new(&format!("fill-{i}")), + Arc::new(vec![0u8; 16 << 10]), + ) + .await; + } + assert!(cache.size_bytes().await <= CAPACITY); + assert!(cache.size().await < 258); + + cache.clear().await; + assert_eq!(cache.size().await, 0); + } + + #[tokio::test] + async fn test_quick_backend_tiny_capacity() { + // A tiny cache must not over-provision item metadata and must still + // admit and evict correctly within its weight budget. + const CAPACITY: usize = 64 << 10; + let cache = LanceCache::with_backend(Arc::new(QuickCacheBackend::with_capacity(CAPACITY))); + for i in 0..64 { + cache + .insert_with_key( + &TestKey::>::new(&format!("k-{i}")), + Arc::new(vec![0u8; 4 << 10]), + ) + .await; + } + assert!(cache.size_bytes().await <= CAPACITY); + assert!(cache.size().await >= 1); + let hit = cache + .get_with_key(&TestKey::>::new("k-63")) + .await + .is_some() + || cache + .get_with_key(&TestKey::>::new("k-62")) + .await + .is_some(); + assert!(hit, "recently inserted entries should be resident"); + } +} diff --git a/rust/lance-core/src/cache/registry.rs b/rust/lance-core/src/cache/registry.rs new file mode 100644 index 00000000000..f370bdd3f8f --- /dev/null +++ b/rust/lance-core/src/cache/registry.rs @@ -0,0 +1,372 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright The Lance Authors + +//! Pluggable cache-backend registry. +//! +//! A [`BackendConfig`] identifies which backend to build (`kind`) and carries +//! backend-specific string options. Backends are constructed through a +//! [`BackendBuildFn`] registered under a unique `kind`. Third-party crates +//! integrate by calling [`register_backend`] once at application startup; +//! [`build_from_config`] then locates the constructor and hands it the +//! config. +//! +//! The registry uses `HashMap` for options so it can be +//! represented naturally across FFI (Python `dict[str, str]`, Java +//! `Map`, etc.). + +use std::collections::HashMap; +use std::sync::{Arc, Mutex, MutexGuard, OnceLock}; + +use super::backend::CacheBackend; +use super::moka::{MOKA_BACKEND_KIND, build_moka}; +use crate::{Error, Result}; + +/// Backend-independent configuration passed to a [`BackendBuildFn`]. +/// +/// `kind` selects which registered backend to construct; `options` carries +/// backend-specific key/value settings (e.g. `capacity`, `path`). +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct BackendConfig { + /// Registered backend identifier, e.g. `"moka"`. + pub kind: String, + /// Backend-specific string options. + pub options: HashMap, +} + +impl BackendConfig { + /// Build a config with no options. + pub fn new(kind: impl AsRef) -> Result { + Ok(Self { + kind: normalize_backend_kind(kind.as_ref())?, + options: HashMap::new(), + }) + } + + /// Insert a single option and return `self`, enabling chaining. + pub fn with_option(mut self, key: impl Into, value: impl Into) -> Self { + self.options.insert(key.into(), value.into()); + self + } +} + +/// Normalize and validate a cache backend kind. +/// +/// Backend kinds share the same syntax as URI schemes. They are matched +/// case-insensitively and stored as lowercase ASCII so registry lookups, +/// config dictionaries, and URI parsing all address the same key. +pub fn normalize_backend_kind(kind: &str) -> Result { + let mut chars = kind.chars(); + match chars.next() { + Some(c) if c.is_ascii_alphabetic() => {} + _ => { + return Err(Error::invalid_input(format!( + "cache backend kind {:?}: must start with an ASCII letter", + kind + ))); + } + } + for c in chars { + let ok = c.is_ascii_alphanumeric() || matches!(c, '+' | '-' | '.'); + if !ok { + return Err(Error::invalid_input(format!( + "cache backend kind {:?}: invalid character {:?}", + kind, c + ))); + } + } + Ok(kind.to_ascii_lowercase()) +} + +/// Constructor signature for a cache backend. +/// +/// Constructors are synchronous. Backends that need async initialization +/// should surface a `try_new_blocking` shim (or equivalent) and call it here. +pub type BackendBuildFn = fn(&BackendConfig) -> Result>; + +fn registry() -> &'static Mutex> { + static REGISTRY: OnceLock>> = OnceLock::new(); + REGISTRY.get_or_init(|| Mutex::new(HashMap::new())) +} + +fn registry_lock() -> Result>> { + registry() + .lock() + .map_err(|_| Error::internal("cache backend registry mutex is poisoned")) +} + +#[cfg(test)] +fn registry_lock_for_test() -> MutexGuard<'static, HashMap> { + registry() + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()) +} + +/// Register a constructor for a cache backend under `kind`. +/// +/// Returns `Err` if a non-built-in `kind` is already registered. Built-in +/// backends may be replaced so callers can mask a built-in implementation +/// (for example, a patched `"moka"` backend) without changing URI/config +/// strings elsewhere. +/// +/// Typical usage from a backend crate: +/// +/// ``` +/// # use std::sync::Arc; +/// # use lance_core::Result; +/// # use lance_core::cache::{BackendConfig, CacheBackend, MokaCacheBackend, register_backend}; +/// fn build_my_backend(_config: &BackendConfig) -> Result> { +/// Ok(Arc::new(MokaCacheBackend::with_capacity(1024))) +/// } +/// +/// # fn main() -> Result<()> { +/// register_backend("my-backend", build_my_backend)?; +/// # Ok(()) +/// # } +/// ``` +pub fn register_backend(kind: &str, build: BackendBuildFn) -> Result<()> { + let kind = normalize_backend_kind(kind)?; + insert_backend(&kind, build, builtin_backend(&kind).is_some()) +} + +fn insert_backend(kind: &str, build: BackendBuildFn, allow_replace: bool) -> Result<()> { + let mut map = registry_lock()?; + if map.contains_key(kind) && !allow_replace { + return Err(Error::invalid_input(format!( + "cache backend {:?} is already registered", + kind + ))); + } + map.insert(kind.to_string(), build); + Ok(()) +} + +fn builtin_backend(kind: &str) -> Option { + match kind { + MOKA_BACKEND_KIND => Some(build_moka), + _ => None, + } +} + +/// Look up the constructor for `config.kind` and build a backend. +/// +/// Returns `Err` if no backend has been registered under that identifier. +pub fn build_from_config(config: &BackendConfig) -> Result> { + ensure_builtin_backends()?; + let kind = normalize_backend_kind(&config.kind)?; + let config = BackendConfig { + kind: kind.clone(), + options: config.options.clone(), + }; + let build = { + let map = registry_lock()?; + map.get(&kind).copied() + }; + match build { + Some(build) => build(&config), + None => Err(Error::invalid_input(format!( + "unknown cache backend kind: {:?}", + kind + ))), + } +} + +/// Idempotently register the backends that ship with `lance-core`. +/// +/// Called by [`build_from_config`] (and, transitively, by +/// [`build_from_uri`](super::backend_uri::build_from_uri)) so a bare Lance +/// installation can build a Moka backend without the caller having to +/// register it. Third-party backends still have to opt in with their own +/// `register()` call. +/// +/// The check is against the current registry contents rather than a +/// process-once flag so that `#[cfg(test)]` helpers which snapshot and +/// restore the registry still see the built-in backend after they take +/// ownership. +fn ensure_builtin_backends() -> Result<()> { + let mut map = registry_lock()?; + if !map.contains_key(MOKA_BACKEND_KIND) + && let Some(build) = builtin_backend(MOKA_BACKEND_KIND) + { + map.insert(MOKA_BACKEND_KIND.to_string(), build); + } + Ok(()) +} + +/// Test-only helper: replace the registry with an empty map so tests can +/// exercise duplicate-registration logic without polluting the global one. +#[cfg(test)] +pub(super) fn take_registry_for_test() -> HashMap { + let mut map = registry_lock_for_test(); + std::mem::take(&mut *map) +} + +/// Test-only helper: restore a previously captured registry state. +#[cfg(test)] +pub(super) fn restore_registry_for_test(saved: HashMap) { + let mut map = registry_lock_for_test(); + *map = saved; +} + +#[cfg(test)] +pub(super) fn registry_test_lock() -> std::sync::MutexGuard<'static, ()> { + static M: OnceLock> = OnceLock::new(); + M.get_or_init(|| std::sync::Mutex::new(())) + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()) +} + +#[cfg(test)] +mod tests { + use super::*; + use async_trait::async_trait; + use std::pin::Pin; + + use crate::cache::InternalCacheKey; + use crate::cache::backend::CacheEntry; + use crate::cache::codec::CacheCodec; + use futures::Future; + + // A trivial no-op backend so tests do not depend on Moka or any other + // real backend. Every method returns "empty" / does nothing. + #[derive(Debug, Default)] + struct NullBackend; + + #[async_trait] + impl CacheBackend for NullBackend { + async fn get( + &self, + _key: &InternalCacheKey, + _codec: Option, + ) -> Option { + None + } + + async fn insert( + &self, + _key: &InternalCacheKey, + _entry: CacheEntry, + _size_bytes: usize, + _codec: Option, + ) { + } + + async fn get_or_insert<'a>( + &self, + _key: &InternalCacheKey, + loader: Pin> + Send + 'a>>, + _codec: Option, + ) -> crate::Result<(CacheEntry, bool)> { + let (entry, _size) = loader.await?; + Ok((entry, false)) + } + + async fn clear(&self) {} + async fn num_entries(&self) -> usize { + 0 + } + async fn size_bytes(&self) -> usize { + 0 + } + } + + fn build_null(_cfg: &BackendConfig) -> Result> { + Ok(Arc::new(NullBackend)) + } + + struct RegistryGuard { + // Hold the serialization lock for the full test. + _lock: std::sync::MutexGuard<'static, ()>, + saved: HashMap, + } + impl RegistryGuard { + fn new() -> Self { + Self { + _lock: registry_test_lock(), + saved: take_registry_for_test(), + } + } + } + impl Drop for RegistryGuard { + fn drop(&mut self) { + restore_registry_for_test(std::mem::take(&mut self.saved)); + } + } + + #[test] + fn test_register_and_build() { + let _guard = RegistryGuard::new(); + register_backend("null", build_null).unwrap(); + let backend = build_from_config(&BackendConfig::new("null").unwrap()).unwrap(); + // Backend is opaque; we just check that the constructor ran and + // gave us an Arc. + assert_eq!(Arc::strong_count(&backend), 1); + } + + #[test] + fn test_duplicate_registration_errors() { + let _guard = RegistryGuard::new(); + register_backend("dup", build_null).unwrap(); + let err = register_backend("dup", build_null).unwrap_err(); + assert!(err.to_string().contains("already registered")); + } + + #[test] + fn test_builtin_kind_can_be_overridden() { + let _guard = RegistryGuard::new(); + register_backend("moka", build_null).unwrap(); + let backend = build_from_config(&BackendConfig::new("moka").unwrap()).unwrap(); + assert_eq!(Arc::strong_count(&backend), 1); + } + + #[test] + fn test_unknown_kind_errors() { + let _guard = RegistryGuard::new(); + let err = build_from_config(&BackendConfig::new("missing").unwrap()).unwrap_err(); + assert!(err.to_string().contains("unknown cache backend kind")); + } + + #[test] + fn test_backend_kind_is_normalized() { + let _guard = RegistryGuard::new(); + register_backend("Echo.Backend", build_null).unwrap(); + let backend = build_from_config(&BackendConfig::new("echo.backend").unwrap()).unwrap(); + assert_eq!(Arc::strong_count(&backend), 1); + } + + #[test] + fn test_config_lookup_normalizes_direct_config() { + let _guard = RegistryGuard::new(); + fn build_echo(cfg: &BackendConfig) -> Result> { + assert_eq!(cfg.kind, "echo.backend"); + Ok(Arc::new(NullBackend)) + } + register_backend("echo.backend", build_echo).unwrap(); + let cfg = BackendConfig { + kind: "ECHO.Backend".to_string(), + options: HashMap::new(), + }; + build_from_config(&cfg).unwrap(); + } + + #[test] + fn test_invalid_backend_kind_errors() { + let err = register_backend("not a scheme", build_null).unwrap_err(); + assert!(err.to_string().contains("invalid character")); + let err = BackendConfig::new("1moka").unwrap_err(); + assert!(err.to_string().contains("must start with an ASCII letter")); + } + + #[test] + fn test_options_are_passed_through() { + let _guard = RegistryGuard::new(); + fn build_echo(cfg: &BackendConfig) -> Result> { + assert_eq!(cfg.options.get("capacity").map(String::as_str), Some("42")); + Ok(Arc::new(NullBackend)) + } + register_backend("echo", build_echo).unwrap(); + let cfg = BackendConfig::new("echo") + .unwrap() + .with_option("capacity", "42"); + build_from_config(&cfg).unwrap(); + } +} diff --git a/rust/lance-core/src/datatypes.rs b/rust/lance-core/src/datatypes.rs index 2f5c7b0680e..98eeebde54f 100644 --- a/rust/lance-core/src/datatypes.rs +++ b/rust/lance-core/src/datatypes.rs @@ -24,7 +24,7 @@ pub use field::{ }; pub use schema::{ BlobHandling, FieldRef, OnMissing, Projectable, Projection, Schema, - escape_field_path_for_project, format_field_path, parse_field_path, + escape_field_path_for_project, format_field_path, format_field_path_minimal, parse_field_path, validate_fixed_size_list_dimensions, }; @@ -48,6 +48,83 @@ pub static BLOB_DESC_FIELD: LazyLock = LazyLock::new(|| { pub static BLOB_DESC_LANCE_FIELD: LazyLock = LazyLock::new(|| Field::try_from(&*BLOB_DESC_FIELD).unwrap()); +/// The minimal logical blob v2 fields accepted from writers. +/// +/// Logical values may also use [`BLOB_V2_LOGICAL_FIELDS`] when an external +/// object range is present. +pub static BLOB_V2_LOGICAL_MINIMAL_FIELDS: LazyLock = LazyLock::new(|| { + Fields::from(vec![ + ArrowField::new("data", DataType::LargeBinary, true), + ArrowField::new("uri", DataType::Utf8, true), + ]) +}); + +/// The complete logical blob v2 fields used for writer input and rewrite output. +/// +/// `position` and `size` are an optional range within the external object named +/// by `uri`. They do not describe Lance-managed data, packed, or dedicated +/// storage. +pub static BLOB_V2_LOGICAL_FIELDS: LazyLock = LazyLock::new(|| { + let mut fields = BLOB_V2_LOGICAL_MINIMAL_FIELDS + .iter() + .cloned() + .collect::>(); + fields.extend([ + Arc::new(ArrowField::new("position", DataType::UInt64, true)), + Arc::new(ArrowField::new("size", DataType::UInt64, true)), + ]); + Fields::from(fields) +}); + +/// The complete logical blob v2 struct type. +pub static BLOB_V2_LOGICAL_TYPE: LazyLock = + LazyLock::new(|| DataType::Struct(BLOB_V2_LOGICAL_FIELDS.clone())); + +/// Writer-prepared blob v2 fields consumed by the structural encoder. +/// +/// The populated fields depend on [`BlobKind`]: +/// +/// - [`BlobKind::Inline`] carries `data`; the encoder derives the stored +/// `position` and `size` from the out-of-line buffer it creates. +/// - [`BlobKind::Packed`] carries `blob_id`, `position`, and `blob_size`. +/// - [`BlobKind::Dedicated`] carries `blob_id` and `blob_size`; its stored +/// `position` is zero. +/// - [`BlobKind::External`] carries `uri`, optional `blob_id`, `position`, and +/// `blob_size`. A zero `blob_size` is resolved to the complete external object +/// length when read. +/// +/// `blob_size` is distinct from the logical `size`, which is only an optional +/// external-object range before preparation. For external blobs, `uri` is +/// normalized into the stable stored `blob_uri` field. +pub static BLOB_V2_PREPARED_FIELDS: LazyLock = LazyLock::new(|| { + Fields::from(vec![ + ArrowField::new("kind", DataType::UInt8, true), + ArrowField::new("data", DataType::LargeBinary, true), + ArrowField::new("uri", DataType::Utf8, true), + ArrowField::new("blob_id", DataType::UInt32, true), + ArrowField::new("blob_size", DataType::UInt64, true), + ArrowField::new("position", DataType::UInt64, true), + ]) +}); + +/// The writer-prepared blob v2 struct type. +pub static BLOB_V2_PREPARED_TYPE: LazyLock = + LazyLock::new(|| DataType::Struct(BLOB_V2_PREPARED_FIELDS.clone())); + +/// Stored blob v2 descriptor fields. +/// +/// These field names are part of the stable file format. Their meaning depends +/// on `kind`: +/// +/// - [`BlobKind::Inline`]: `position` and `size` locate an out-of-line buffer in +/// the Lance data file. +/// - [`BlobKind::Packed`]: `blob_id` identifies a shared packed blob file, and +/// `position` and `size` locate a range within it. +/// - [`BlobKind::Dedicated`]: `blob_id` identifies a dedicated raw blob file, +/// `position` is zero, and `size` is the complete file length. +/// - [`BlobKind::External`]: `blob_uri` and `blob_id` identify the object, while +/// `position` and `size` select a range. A zero `size` is resolved to the +/// object's complete length when read. pub static BLOB_V2_DESC_FIELDS: LazyLock = LazyLock::new(|| { Fields::from(vec![ ArrowField::new("kind", DataType::UInt8, false), @@ -71,25 +148,95 @@ pub static BLOB_V2_DESC_FIELD: LazyLock = LazyLock::new(|| { pub static BLOB_V2_DESC_LANCE_FIELD: LazyLock = LazyLock::new(|| Field::try_from(&*BLOB_V2_DESC_FIELD).unwrap()); -/// Blob v2 user-view struct fields used by internal rewrite paths. -/// -/// This schema converts the descriptor view back into the write-side view used -/// by blob compaction. -pub static BLOB_V2_USER_FIELDS: LazyLock = LazyLock::new(|| { - Fields::from(vec![ - ArrowField::new("data", DataType::LargeBinary, true), - ArrowField::new("uri", DataType::Utf8, true), - ArrowField::new("position", DataType::UInt64, true), - ArrowField::new("size", DataType::UInt64, true), - ]) -}); +/// The in-memory representation of a blob v2 struct. +#[non_exhaustive] +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum BlobV2Layout { + /// Writer input or rewrite output. + /// + /// Both the minimal `data, uri` fields and the complete + /// `data, uri, position, size` fields have this layout. + Logical, + /// Kind-aware writer intermediate consumed by the structural encoder. + Prepared, + /// Stable descriptor stored in Lance files and returned by descriptor scans. + Descriptor, +} -/// Blob v2 user-view struct type used by internal rewrite paths. -/// -/// This schema converts the descriptor view back into the write-side view used -/// by blob compaction. -pub static BLOB_V2_USER_TYPE: LazyLock = - LazyLock::new(|| DataType::Struct(BLOB_V2_USER_FIELDS.clone())); +impl BlobV2Layout { + /// Classify blob v2 child fields by name, type, order, and layout-specific + /// nullability requirements. + /// + /// Child metadata is not part of the representation. The complete logical + /// layout also accepts non-nullable `position` and `size` fields, matching + /// the existing writer-input contract. Descriptor child nullability is + /// ignored because it changed across released schemas; row nullness has + /// been represented by either parent struct validity or a nullable `kind` + /// child. + pub fn classify(fields: &Fields) -> Option { + if logical_blob_v2_fields_match(fields) { + Some(Self::Logical) + } else if blob_v2_fields_match(fields, &BLOB_V2_PREPARED_FIELDS, true) { + Some(Self::Prepared) + } else if blob_v2_fields_match(fields, &BLOB_V2_DESC_FIELDS, false) { + Some(Self::Descriptor) + } else { + None + } + } +} + +impl fmt::Display for BlobV2Layout { + fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { + match self { + Self::Logical => write!(f, "logical"), + Self::Prepared => write!(f, "prepared"), + Self::Descriptor => write!(f, "descriptor"), + } + } +} + +fn blob_v2_field_matches( + actual: &ArrowField, + expected: &ArrowField, + compare_nullability: bool, +) -> bool { + actual.name() == expected.name() + && actual.data_type() == expected.data_type() + && (!compare_nullability || actual.is_nullable() == expected.is_nullable()) +} + +fn blob_v2_fields_match(actual: &Fields, expected: &Fields, compare_nullability: bool) -> bool { + actual.len() == expected.len() + && actual + .iter() + .zip(expected.iter()) + .all(|(actual, expected)| { + blob_v2_field_matches(actual.as_ref(), expected.as_ref(), compare_nullability) + }) +} + +fn logical_blob_v2_fields_match(fields: &Fields) -> bool { + if blob_v2_fields_match(fields, &BLOB_V2_LOGICAL_MINIMAL_FIELDS, true) { + return true; + } + fields.len() == BLOB_V2_LOGICAL_FIELDS.len() + && fields + .iter() + .zip(BLOB_V2_LOGICAL_FIELDS.iter()) + .enumerate() + .all(|(index, (actual, expected))| { + blob_v2_field_matches(actual.as_ref(), expected.as_ref(), index < 2) + }) +} + +/// Deprecated name for [`BLOB_V2_LOGICAL_FIELDS`]. +#[deprecated(note = "use BLOB_V2_LOGICAL_FIELDS")] +pub use self::BLOB_V2_LOGICAL_FIELDS as BLOB_V2_USER_FIELDS; + +/// Deprecated name for [`BLOB_V2_LOGICAL_TYPE`]. +#[deprecated(note = "use BLOB_V2_LOGICAL_TYPE")] +pub use self::BLOB_V2_LOGICAL_TYPE as BLOB_V2_USER_TYPE; pub const BLOB_LOGICAL_TYPE: &str = "blob"; @@ -462,3 +609,79 @@ impl TryFrom for BlobKind { } } } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_classify_blob_v2_layouts() { + assert_eq!( + BlobV2Layout::classify(&BLOB_V2_LOGICAL_MINIMAL_FIELDS), + Some(BlobV2Layout::Logical) + ); + assert_eq!( + BlobV2Layout::classify(&BLOB_V2_LOGICAL_FIELDS), + Some(BlobV2Layout::Logical) + ); + assert_eq!( + BlobV2Layout::classify(&BLOB_V2_PREPARED_FIELDS), + Some(BlobV2Layout::Prepared) + ); + assert_eq!( + BlobV2Layout::classify(&BLOB_V2_DESC_FIELDS), + Some(BlobV2Layout::Descriptor) + ); + } + + #[test] + fn test_classify_blob_v2_layout_uses_structural_contract() { + let logical_with_required_range = Fields::from(vec![ + ArrowField::new("data", DataType::LargeBinary, true), + ArrowField::new("uri", DataType::Utf8, true), + ArrowField::new("position", DataType::UInt64, false), + ArrowField::new("size", DataType::UInt64, false), + ]); + assert_eq!( + BlobV2Layout::classify(&logical_with_required_range), + Some(BlobV2Layout::Logical) + ); + + let prepared_with_child_metadata = + Fields::from( + BLOB_V2_PREPARED_FIELDS + .iter() + .map(|field| { + Arc::new(field.as_ref().clone().with_metadata(HashMap::from([( + "source".to_string(), + "test".to_string(), + )]))) + }) + .collect::>(), + ); + assert_eq!( + BlobV2Layout::classify(&prepared_with_child_metadata), + Some(BlobV2Layout::Prepared) + ); + + let nullable_descriptor = Fields::from( + BLOB_V2_DESC_FIELDS + .iter() + .map(|field| Arc::new(field.as_ref().clone().with_nullable(true))) + .collect::>(), + ); + assert_eq!( + BlobV2Layout::classify(&nullable_descriptor), + Some(BlobV2Layout::Descriptor) + ); + + let malformed_descriptor = Fields::from(vec![ + ArrowField::new("kind", DataType::UInt8, false), + ArrowField::new("position", DataType::UInt64, false), + ArrowField::new("size", DataType::UInt32, false), + ArrowField::new("blob_id", DataType::UInt32, false), + ArrowField::new("blob_uri", DataType::Utf8, false), + ]); + assert_eq!(BlobV2Layout::classify(&malformed_descriptor), None); + } +} diff --git a/rust/lance-core/src/datatypes/field.rs b/rust/lance-core/src/datatypes/field.rs index 9f06d421949..d5eb89dccb0 100644 --- a/rust/lance-core/src/datatypes/field.rs +++ b/rust/lance-core/src/datatypes/field.rs @@ -58,6 +58,8 @@ pub const LANCE_UNENFORCED_CLUSTERING_KEY_POSITION: &str = /// The value should be non-negative i32 value. Any negative value will be seen as -1. pub const LANCE_FIELD_ID_KEY: &str = "lance:field_id"; +const PACKED_KEYS: [&str; 2] = ["packed", "lance-encoding:packed"]; + fn has_blob_v2_extension(field: &ArrowField) -> bool { field .metadata() @@ -269,24 +271,15 @@ impl Field { } pub fn apply_projection(&self, projection: &Projection) -> Option { - // Map fields encode their physical layout as a single child entries - // struct (`Struct`) whose presence is required for the - // parent to be readable — we never want to filter into that subtree. - // But the parent field itself is still subject to selection: if the - // caller didn't ask for this Map column, drop it like any other - // non-selected leaf. Without this early return the unconditional - // children clone would keep `children.is_empty() == false` forever - // and every Map column in the schema would survive every projection, - // pulling tens-of-bytes-per-row of unrelated data through downstream - // operators (notably `SortExec` in scalar-index training, where it - // was responsible for >100 GiB external-sort spills on real-world - // tables). - if self.logical_type.is_map() && !projection.contains_field_id(self.id) { + // Maps and blob descriptors are atomic physical layouts. Map children + // must remain together, while projected blob descriptor children may + // have synthetic IDs that cannot be selected independently. + let is_atomic_layout = self.logical_type.is_map() || self.is_blob(); + if is_atomic_layout && !projection.contains_field_id(self.id) { return None; } - let children = if self.logical_type.is_map() { - // Map field is selected: keep all children intact. + let children = if is_atomic_layout { self.children.clone() } else { self.children @@ -549,6 +542,20 @@ impl Field { .get(ARROW_EXT_NAME_KEY) .map(|name| name == BLOB_V2_EXT_NAME) .unwrap_or(false) + || self.is_blob_v2_descriptor() + } + + fn is_blob_v2_descriptor(&self) -> bool { + self.metadata.contains_key(BLOB_META_KEY) + && self.logical_type == BLOB_V2_DESC_LANCE_FIELD.logical_type + && self.children.len() == BLOB_V2_DESC_LANCE_FIELD.children.len() + && self + .children + .iter() + .zip(BLOB_V2_DESC_LANCE_FIELD.children.iter()) + .all(|(child, expected)| { + child.name == expected.name && child.data_type() == expected.data_type() + }) } // Blob columns intentionally have two schema representations: @@ -575,6 +582,31 @@ impl Field { } } + /// Convert a blob field to the materialized binary payload view. + /// + /// The field keeps its name and id but uses `LargeBinary` with no children. + /// Blob v2 fields retain their extension marker internally so scan planning + /// can recognize the binary view before exposing a plain Arrow binary field. + pub fn binary_blob_mut(&mut self) { + if !self.is_blob() { + return; + } + let is_blob_v2 = self.is_blob_v2(); + + self.logical_type = LogicalType::try_from(&DataType::LargeBinary) + .expect("LargeBinary is always a valid logical type"); + self.children.clear(); + self.encoding = Some(Encoding::VarBinary); + if is_blob_v2 { + self.metadata.remove(BLOB_META_KEY); + for key in PACKED_KEYS { + self.metadata.remove(key); + } + self.metadata + .insert(ARROW_EXT_NAME_KEY.to_string(), BLOB_V2_EXT_NAME.to_string()); + } + } + /// Convert blob v2 fields in this field tree to their descriptor view. pub fn unload_blobs_recursive(&mut self) { if self.is_blob_v2() { @@ -812,6 +844,13 @@ impl Field { } if self.is_blob() != other.is_blob() { + if ignore_types { + return Ok(if self.id >= 0 { + self.clone() + } else { + other.clone() + }); + } return Err(Error::arrow(format!( "Attempt to intersect blob and non-blob field: {}", self.name @@ -847,7 +886,7 @@ impl Field { .iter() .filter_map(|c| { if let Some(other_child) = other.child(&c.name) { - let intersection = c.intersection(other_child).ok()?; + let intersection = c.do_intersection(other_child, ignore_types).ok()?; Some(intersection) } else { None @@ -1038,7 +1077,6 @@ impl Field { // Check if field has metadata `packed` set to true, this check is case insensitive. pub fn is_packed_struct(&self) -> bool { - const PACKED_KEYS: [&str; 2] = ["packed", "lance-encoding:packed"]; PACKED_KEYS.iter().any(|key| { self.metadata .get(*key) @@ -1847,6 +1885,14 @@ mod tests { #[test] fn blob_unloaded_mut_selects_layout_from_metadata() { let metadata = HashMap::from([(BLOB_META_KEY.to_string(), "true".to_string())]); + let mut binary_field: Field = ArrowField::new("blob", DataType::LargeBinary, true) + .with_metadata(metadata.clone()) + .try_into() + .unwrap(); + binary_field.binary_blob_mut(); + assert!(binary_field.metadata.contains_key(BLOB_META_KEY)); + assert!(!binary_field.is_blob_v2()); + let mut field: Field = ArrowField::new("blob", DataType::LargeBinary, true) .with_metadata(metadata) .try_into() @@ -1854,6 +1900,12 @@ mod tests { field.unloaded_mut(); assert_eq!(field.children.len(), 2); assert_eq!(field.logical_type, BLOB_DESC_LANCE_FIELD.logical_type); + assert!(field.is_blob()); + assert!(!field.is_blob_v2()); + field.unloaded_mut(); + assert_eq!(field.children.len(), 2); + assert_eq!(field.logical_type, BLOB_DESC_LANCE_FIELD.logical_type); + assert!(!field.is_blob_v2()); let metadata = HashMap::from([(ARROW_EXT_NAME_KEY.to_string(), BLOB_V2_EXT_NAME.to_string())]); @@ -1874,6 +1926,12 @@ mod tests { field.unloaded_mut(); assert_eq!(field.children.len(), 5); assert_eq!(field.logical_type, BLOB_V2_DESC_LANCE_FIELD.logical_type); + assert!(!field.metadata.contains_key(ARROW_EXT_NAME_KEY)); + assert!(field.is_blob_v2()); + field.unloaded_mut(); + assert_eq!(field.children.len(), 5); + assert_eq!(field.logical_type, BLOB_V2_DESC_LANCE_FIELD.logical_type); + assert!(!field.metadata.contains_key(ARROW_EXT_NAME_KEY)); } #[test] @@ -1944,4 +2002,43 @@ mod tests { .unwrap(); assert_eq!(unloaded_projected, unloaded); } + + #[test] + fn blob_descriptor_projection_preserves_synthetic_children() { + let metadata = + HashMap::from([(ARROW_EXT_NAME_KEY.to_string(), BLOB_V2_EXT_NAME.to_string())]); + let mut blob: Field = ArrowField::new( + "blob", + DataType::Struct( + vec![ + ArrowField::new("data", DataType::LargeBinary, true), + ArrowField::new("uri", DataType::Utf8, true), + ] + .into(), + ), + true, + ) + .with_metadata(metadata) + .try_into() + .unwrap(); + let mut next_id = 0; + blob.set_id(-1, &mut next_id); + + let schema = Arc::new(crate::datatypes::Schema { + fields: vec![blob], + metadata: HashMap::new(), + }); + let descriptor_schema = Projection::full(schema) + .with_blob_handling(crate::datatypes::BlobHandling::BlobsDescriptions) + .to_bare_schema(); + assert!( + descriptor_schema.fields[0] + .children + .iter() + .all(|child| child.id == -1) + ); + + let projected = Projection::full(Arc::new(descriptor_schema)).to_bare_schema(); + assert_eq!(projected.fields[0].children.len(), 5); + } } diff --git a/rust/lance-core/src/datatypes/schema.rs b/rust/lance-core/src/datatypes/schema.rs index 7f2cbc02f07..2e328c88d24 100644 --- a/rust/lance-core/src/datatypes/schema.rs +++ b/rust/lance-core/src/datatypes/schema.rs @@ -339,17 +339,10 @@ impl Schema { ))); } - let column_path = self - .field_ancestry_by_id(field.id) - .unwrap() - .iter() - .map(|f| f.name.as_str()) - .collect::>() - .join("."); - if !seen_names.insert(column_path.clone()) { + if !seen_names.insert(field.name.as_str()) { return Err(Error::schema(format!( "Duplicate field name \"{}\" in schema:\n {:#?}", - column_path, self + field.name, self ))); } } @@ -726,7 +719,8 @@ impl Schema { /// Merge this schema from the other schema. /// /// After merging, the field IDs from `other` schema will be reassigned, - /// following the fields in `self`. + /// following the fields in `self`. Schema metadata is combined, with values + /// from `self` taking precedence when both schemas contain the same key. pub fn merge>(&self, other: S) -> Result { let mut other: Self = other.try_into()?; other.reset_id(); @@ -747,12 +741,12 @@ impl Schema { merged_fields.push(field.clone()); } } - let metadata = self - .metadata - .iter() - .chain(other.metadata.iter()) - .map(|(k, v)| (k.clone(), v.clone())) - .collect(); + let mut metadata = other.metadata; + metadata.extend( + self.metadata + .iter() + .map(|(key, value)| (key.clone(), value.clone())), + ); let schema = Self { fields: merged_fields, metadata, @@ -762,6 +756,9 @@ impl Schema { /// Returns the properly formatted path from root to the field. /// Field names containing dots are quoted (e.g., struct.`field.with.dot`) + /// + /// The result is suitable for SQL parsing. For a human-readable path + /// (e.g. for display in index metadata), use [`Self::field_path_minimal`]. pub fn field_path(&self, field_id: i32) -> Result { self.field_ancestry_by_id(field_id) .map(|ancestry| { @@ -773,6 +770,57 @@ impl Schema { }) } + /// Returns the path from root to the field using *minimal* quoting. + /// + /// A segment is wrapped in backticks only when it contains a character that + /// [`parse_field_path`] treats specially (a `.` separator or a `` ` `` quote); + /// any other character — including hyphens — is left bare. Unlike + /// [`Self::field_path`] (which quotes for SQL-expression safety and so wraps + /// e.g. `my-col` in backticks), the result here is both human-readable and + /// round-trips back through `parse_field_path`, so it is safe to feed into + /// field-path APIs such as `drop_columns` / `update_field_metadata`. + /// + /// This is what should be exposed as the column name in index metadata. + /// + /// ``` + /// use arrow_schema::{DataType, Field, Fields, Schema as ArrowSchema}; + /// use lance_core::datatypes::{parse_field_path, Schema}; + /// + /// let arrow = ArrowSchema::new(vec![ + /// Field::new("my-col", DataType::Int32, false), + /// Field::new( + /// "parent", + /// DataType::Struct(Fields::from(vec![Field::new("child.x", DataType::Int32, true)])), + /// true, + /// ), + /// ]); + /// let schema = Schema::try_from(&arrow).unwrap(); + /// + /// // A hyphen is not special to `parse_field_path`, so it is left bare + /// // (unlike `field_path`, which would quote it as `` `my-col` ``). + /// let hyphen_id = schema.field("my-col").unwrap().id; + /// assert_eq!(schema.field_path_minimal(hyphen_id).unwrap(), "my-col"); + /// + /// // A `.` in a segment forces quoting so the path still round-trips. + /// let dotted_id = schema.field("parent").unwrap().children[0].id; + /// let path = schema.field_path_minimal(dotted_id).unwrap(); + /// assert_eq!(path, "parent.`child.x`"); + /// assert_eq!( + /// parse_field_path(&path).unwrap(), + /// vec!["parent".to_string(), "child.x".to_string()], + /// ); + /// ``` + pub fn field_path_minimal(&self, field_id: i32) -> Result { + self.field_ancestry_by_id(field_id) + .map(|ancestry| { + let field_refs: Vec<&str> = ancestry.iter().map(|f| f.name.as_str()).collect(); + format_field_path_minimal(&field_refs) + }) + .ok_or_else(|| { + Error::index(format!("Could not find field ancestry for id {}", field_id)) + }) + } + pub fn verify_primary_key(&self) -> Result<()> { let pk = self.unenforced_primary_key(); for pk_col in pk.into_iter() { @@ -1071,6 +1119,17 @@ pub enum BlobHandling { } impl BlobHandling { + fn should_load_binary(&self, field: &Field) -> bool { + if !field.is_blob() { + return false; + } + match self { + Self::AllBinary => true, + Self::SomeBlobsBinary(set) | Self::SomeBinary(set) => set.contains(&(field.id as u32)), + Self::BlobsDescriptions | Self::AllDescriptions => false, + } + } + fn should_unload(&self, field: &Field) -> bool { // Blob v2 columns are Structs, so we need to treat any blob-marked field as unloadable // even if the physical data type is not binary-like. @@ -1086,10 +1145,34 @@ impl BlobHandling { } } + /// Whether `field` will be projected as a lightweight blob *description* + /// (offset + size) rather than its full binary value under this handling. + /// + /// A description is tiny and cheap to read eagerly; the full binary value is + /// not. Materialization heuristics use this to decide early vs late loading. + pub fn returns_description(&self, field: &Field) -> bool { + self.should_unload(field) + } + + /// Apply this blob handling policy to a projected field tree. + /// + /// Blob descriptor modes convert blob leaves to descriptor views. Binary + /// modes convert selected blob leaves to `LargeBinary`. Non-blob nested + /// fields are preserved while their children are handled recursively. pub fn unload_if_needed(&self, mut field: Field) -> Field { + if self.should_load_binary(&field) { + field.binary_blob_mut(); + return field; + } if self.should_unload(&field) { field.unloaded_mut(); + return field; } + field.children = field + .children + .into_iter() + .map(|child| self.unload_if_needed(child)) + .collect(); field } } @@ -1596,6 +1679,50 @@ pub fn format_field_path(fields: &[&str]) -> String { .join(".") } +/// Like [`format_field_path`], but quotes a segment only when strictly required +/// for the result to round-trip back through [`parse_field_path`]. +/// +/// `parse_field_path` only treats `.` (segment separator) and `` ` `` (quote) +/// specially, so those are the only characters that force quoting here. Notably +/// a hyphen does NOT force quoting (`my-col` stays `my-col`), unlike +/// `format_field_path` which quotes any non-identifier character for +/// SQL-expression safety. Use this for human-readable, round-trippable paths +/// (e.g. column names in index metadata); use `format_field_path` when the +/// result will be embedded in a SQL expression. +/// +/// ``` +/// use lance_core::datatypes::{format_field_path_minimal, parse_field_path}; +/// +/// // Plain identifiers and hyphenated names are left bare. +/// assert_eq!(format_field_path_minimal(&["parent", "my-col"]), "parent.my-col"); +/// // A `.` in a segment forces quoting. +/// assert_eq!(format_field_path_minimal(&["parent", "child.x"]), "parent.`child.x`"); +/// // Embedded backticks are escaped by doubling them. +/// assert_eq!(format_field_path_minimal(&["child`x"]), "`child``x`"); +/// +/// // Whatever it produces round-trips back through `parse_field_path`. +/// for segments in [vec!["parent", "my-col"], vec!["parent", "child.x"], vec!["child`x"]] { +/// let path = format_field_path_minimal(&segments); +/// assert_eq!(parse_field_path(&path).unwrap(), segments); +/// } +/// ``` +pub fn format_field_path_minimal(fields: &[&str]) -> String { + fields + .iter() + .map(|field| { + let needs_quoting = field.contains('.') || field.contains('`'); + if needs_quoting { + // Escape embedded backticks by doubling them, matching parse_field_path. + let escaped = field.replace('`', "``"); + format!("`{}`", escaped) + } else { + field.to_string() + } + }) + .collect::>() + .join(".") +} + /// Escape a field path for project /// /// Parses the field path and formats it for SQL usage. @@ -1736,6 +1863,11 @@ mod tests { vec!["simple".to_string()] ); + assert_eq!( + parse_field_path("tags[*]").unwrap(), + vec!["tags[*]".to_string()] + ); + // Quoted field at the end assert_eq!( parse_field_path("parent.`field.with.dot`").unwrap(), @@ -1772,6 +1904,22 @@ mod tests { ); } + #[test] + fn test_validate_top_level_names_without_field_id_lookup() { + let mut first = Field::new_arrow("first", DataType::Int32, false).unwrap(); + first.id = 0; + let mut second = Field::new_arrow("second", DataType::Int32, false).unwrap(); + second.id = 0; + let schema = Schema { + fields: vec![first, second], + metadata: HashMap::new(), + }; + + let error = schema.validate().unwrap_err(); + assert!(matches!(&error, Error::Schema { .. })); + assert!(error.to_string().contains("Duplicate field id 0")); + } + #[test] fn test_resolve_quoted_fields() { // Test that top-level fields with dots are rejected during validation @@ -2202,6 +2350,35 @@ mod tests { assert_eq!(merged.max_field_id(), Some(9)); } + #[test] + fn test_merge_schema_metadata_preserves_self_values() { + let schema = Schema { + metadata: HashMap::from([ + ("shared".to_string(), "left".to_string()), + ("left_only".to_string(), "left".to_string()), + ]), + ..Default::default() + }; + let other = Schema { + metadata: HashMap::from([ + ("shared".to_string(), "right".to_string()), + ("right_only".to_string(), "right".to_string()), + ]), + ..Default::default() + }; + + let merged = schema.merge(&other).unwrap(); + + assert_eq!( + merged.metadata, + HashMap::from([ + ("shared".to_string(), "left".to_string()), + ("left_only".to_string(), "left".to_string()), + ("right_only".to_string(), "right".to_string()), + ]) + ); + } + #[test] fn test_merge_arrow_schema() { let arrow_schema = ArrowSchema::new(vec![ @@ -2768,6 +2945,77 @@ mod tests { assert!(paths.contains(&"name".to_string())); } + #[test] + fn test_field_path_minimal() { + // A struct child whose own NAME contains a dot is the case that makes + // "just strip all backticks" wrong: it must stay quoted to round-trip. + let arrow_schema = ArrowSchema::new(vec![ + ArrowField::new("mycol", ArrowDataType::Int32, false), + ArrowField::new("my_col", ArrowDataType::Int32, false), + ArrowField::new("my-col", ArrowDataType::Int32, false), + ArrowField::new( + "parent", + ArrowDataType::Struct(ArrowFields::from(vec![ + ArrowField::new("child-field", ArrowDataType::Int32, true), + ArrowField::new("child.x", ArrowDataType::Int32, true), + ArrowField::new("child`x", ArrowDataType::Int32, true), + ])), + true, + ), + ]); + let schema = Schema::try_from(&arrow_schema).unwrap(); + let id_of = |path: &str| schema.field(path).unwrap().id; + let child_id = |name: &str| { + schema + .field("parent") + .unwrap() + .children + .iter() + .find(|c| c.name == name) + .unwrap() + .id + }; + + // Plain identifiers: unchanged by either method. + assert_eq!(schema.field_path_minimal(id_of("mycol")).unwrap(), "mycol"); + assert_eq!( + schema.field_path_minimal(id_of("my_col")).unwrap(), + "my_col" + ); + + // Hyphen is NOT special to parse_field_path, so minimal quoting leaves it + // bare (field_path would quote it for SQL safety). + assert_eq!(schema.field_path(id_of("my-col")).unwrap(), "`my-col`"); + assert_eq!( + schema.field_path_minimal(id_of("my-col")).unwrap(), + "my-col" + ); + + // Nested hyphenated leaf: bare under minimal quoting. + assert_eq!( + schema.field_path_minimal(child_id("child-field")).unwrap(), + "parent.child-field" + ); + + // Nested leaf whose NAME contains a dot: MUST stay quoted so it + // round-trips through parse_field_path (this is the regression guard). + let dotted = schema.field_path_minimal(child_id("child.x")).unwrap(); + assert_eq!(dotted, "parent.`child.x`"); + assert_eq!( + parse_field_path(&dotted).unwrap(), + vec!["parent".to_string(), "child.x".to_string()] + ); + + // Nested leaf whose NAME contains a backtick: it must be quoted AND the + // backtick doubled so it round-trips through parse_field_path. + let backticked = schema.field_path_minimal(child_id("child`x")).unwrap(); + assert_eq!(backticked, "parent.`child``x`"); + assert_eq!( + parse_field_path(&backticked).unwrap(), + vec!["parent".to_string(), "child`x".to_string()] + ); + } + #[test] fn test_validate_rejects_zero_dimension_fixed_size_list() { // A zero dimension divides-by-zero further down the write path (#5102) diff --git a/rust/lance-core/src/error.rs b/rust/lance-core/src/error.rs index 2a6340da492..648a4881f1d 100644 --- a/rust/lance-core/src/error.rs +++ b/rust/lance-core/src/error.rs @@ -8,6 +8,52 @@ use snafu::{IntoError as _, Location, Snafu}; type BoxedError = Box; +#[cfg(feature = "backtrace")] +mod backtrace_support { + use std::backtrace::Backtrace; + + use snafu::{AsBacktrace, GenerateImplicitData}; + + #[derive(Debug)] + pub struct MaybeBacktrace(pub Option); + + impl GenerateImplicitData for MaybeBacktrace { + fn generate() -> Self { + Self(>::generate()) + } + } + + impl AsBacktrace for MaybeBacktrace { + fn as_backtrace(&self) -> Option<&Backtrace> { + self.0.as_ref() + } + } +} + +#[cfg(not(feature = "backtrace"))] +mod backtrace_support { + use std::backtrace::Backtrace; + + use snafu::{AsBacktrace, GenerateImplicitData}; + + #[derive(Debug)] + pub struct MaybeBacktrace; + + impl GenerateImplicitData for MaybeBacktrace { + fn generate() -> Self { + Self + } + } + + impl AsBacktrace for MaybeBacktrace { + fn as_backtrace(&self) -> Option<&Backtrace> { + None + } + } +} + +use backtrace_support::MaybeBacktrace; + /// Error for when a requested field is not found in a schema. /// /// This error computes suggestions lazily (only when displayed) to avoid @@ -45,6 +91,41 @@ impl fmt::Display for FieldNotFoundError { impl std::error::Error for FieldNotFoundError {} +/// A manifest commit returned an error and its final outcome could not be +/// determined safely. +/// +/// This is wrapped in [`Error::Wrapped`] so Lance can expose a structured +/// source without adding a variant to the exhaustive public [`Error`] enum. +#[derive(Debug)] +pub struct CommitStatusUnknownError { + version: u64, + source: BoxedError, +} + +impl CommitStatusUnknownError { + /// Return the manifest version whose commit outcome is unknown. + pub fn version(&self) -> u64 { + self.version + } +} + +impl std::fmt::Display for CommitStatusUnknownError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!( + f, + "Commit result for version {} is unknown: the commit may or may not have been \ + applied; check the table state before retrying: {}", + self.version, self.source + ) + } +} + +impl std::error::Error for CommitStatusUnknownError { + fn source(&self) -> Option<&(dyn std::error::Error + 'static)> { + Some(self.source.as_ref()) + } +} + /// Allocates error on the heap and then places `e` into it. #[inline] pub fn box_error(e: impl std::error::Error + Send + Sync + 'static) -> BoxedError { @@ -81,18 +162,24 @@ pub enum Error { source: BoxedError, #[snafu(implicit)] location: Location, + #[snafu(implicit)] + backtrace: MaybeBacktrace, }, #[snafu(display("Dataset already exists: {uri}, {location}"))] DatasetAlreadyExists { uri: String, #[snafu(implicit)] location: Location, + #[snafu(implicit)] + backtrace: MaybeBacktrace, }, #[snafu(display("Append with different schema: {difference}, location: {location}"))] SchemaMismatch { difference: String, #[snafu(implicit)] location: Location, + #[snafu(implicit)] + backtrace: MaybeBacktrace, }, #[snafu(display("Dataset at path {path} was not found: {source}, {location}"))] DatasetNotFound { @@ -100,6 +187,8 @@ pub enum Error { source: BoxedError, #[snafu(implicit)] location: Location, + #[snafu(implicit)] + backtrace: MaybeBacktrace, }, #[snafu(display("Encountered corrupt file {path}: {source}, {location}"))] CorruptFile { @@ -107,13 +196,16 @@ pub enum Error { source: BoxedError, #[snafu(implicit)] location: Location, - // TODO: add backtrace? + #[snafu(implicit)] + backtrace: MaybeBacktrace, }, #[snafu(display("Not supported: {source}, {location}"))] NotSupported { source: BoxedError, #[snafu(implicit)] location: Location, + #[snafu(implicit)] + backtrace: MaybeBacktrace, }, #[snafu(display("Commit conflict for version {version}: {source}, {location}"))] CommitConflict { @@ -121,12 +213,16 @@ pub enum Error { source: BoxedError, #[snafu(implicit)] location: Location, + #[snafu(implicit)] + backtrace: MaybeBacktrace, }, #[snafu(display("Incompatible transaction: {source}, {location}"))] IncompatibleTransaction { source: BoxedError, #[snafu(implicit)] location: Location, + #[snafu(implicit)] + backtrace: MaybeBacktrace, }, #[snafu(display("Retryable commit conflict for version {version}: {source}, {location}"))] RetryableCommitConflict { @@ -134,12 +230,16 @@ pub enum Error { source: BoxedError, #[snafu(implicit)] location: Location, + #[snafu(implicit)] + backtrace: MaybeBacktrace, }, #[snafu(display("Too many concurrent writers. {message}, {location}"))] TooMuchWriteContention { message: String, #[snafu(implicit)] location: Location, + #[snafu(implicit)] + backtrace: MaybeBacktrace, }, #[snafu(display("Operation timed out: {message}, {location}"))] Timeout { @@ -154,54 +254,72 @@ pub enum Error { message: String, #[snafu(implicit)] location: Location, + #[snafu(implicit)] + backtrace: MaybeBacktrace, }, #[snafu(display("A prerequisite task failed: {message}, {location}"))] PrerequisiteFailed { message: String, #[snafu(implicit)] location: Location, + #[snafu(implicit)] + backtrace: MaybeBacktrace, }, #[snafu(display("Unprocessable: {message}, {location}"))] Unprocessable { message: String, #[snafu(implicit)] location: Location, + #[snafu(implicit)] + backtrace: MaybeBacktrace, }, #[snafu(display("LanceError(Arrow): {message}, {location}"))] Arrow { message: String, #[snafu(implicit)] location: Location, + #[snafu(implicit)] + backtrace: MaybeBacktrace, }, #[snafu(display("LanceError(Schema): {message}, {location}"))] Schema { message: String, #[snafu(implicit)] location: Location, + #[snafu(implicit)] + backtrace: MaybeBacktrace, }, #[snafu(display("Not found: {uri}, {location}"))] NotFound { uri: String, #[snafu(implicit)] location: Location, + #[snafu(implicit)] + backtrace: MaybeBacktrace, }, #[snafu(display("LanceError(IO): {source}, {location}"))] IO { source: BoxedError, #[snafu(implicit)] location: Location, + #[snafu(implicit)] + backtrace: MaybeBacktrace, }, #[snafu(display("LanceError(Index): {message}, {location}"))] Index { message: String, #[snafu(implicit)] location: Location, + #[snafu(implicit)] + backtrace: MaybeBacktrace, }, #[snafu(display("Lance index not found: {identity}, {location}"))] IndexNotFound { identity: String, #[snafu(implicit)] location: Location, + #[snafu(implicit)] + backtrace: MaybeBacktrace, }, #[snafu(display("Cannot infer storage location from: {message}"))] InvalidTableLocation { message: String }, @@ -209,21 +327,28 @@ pub enum Error { Stop, #[snafu(display("Wrapped error: {error}, {location}"))] Wrapped { + #[snafu(source)] error: BoxedError, #[snafu(implicit)] location: Location, + #[snafu(implicit)] + backtrace: MaybeBacktrace, }, #[snafu(display("Cloned error: {message}, {location}"))] Cloned { message: String, #[snafu(implicit)] location: Location, + #[snafu(implicit)] + backtrace: MaybeBacktrace, }, #[snafu(display("Query Execution error: {message}, {location}"))] Execution { message: String, #[snafu(implicit)] location: Location, + #[snafu(implicit)] + backtrace: MaybeBacktrace, }, #[snafu(display("Ref is invalid: {message}"))] InvalidRef { message: String }, @@ -242,12 +367,16 @@ pub enum Error { minor_version: u16, #[snafu(implicit)] location: Location, + #[snafu(implicit)] + backtrace: MaybeBacktrace, }, #[snafu(display("Namespace error: {source}, {location}"))] Namespace { source: BoxedError, #[snafu(implicit)] location: Location, + #[snafu(implicit)] + backtrace: MaybeBacktrace, }, /// External error passed through from user code. /// @@ -283,11 +412,82 @@ pub enum Error { } impl Error { + /// Returns the captured Rust backtrace, if available. + /// + /// Requires the `backtrace` feature to be enabled at compile time + /// and `RUST_BACKTRACE=1` at runtime. + #[cfg(feature = "backtrace")] + pub fn backtrace(&self) -> Option<&std::backtrace::Backtrace> { + match self { + Self::InvalidInput { backtrace, .. } + | Self::DatasetAlreadyExists { backtrace, .. } + | Self::SchemaMismatch { backtrace, .. } + | Self::DatasetNotFound { backtrace, .. } + | Self::CorruptFile { backtrace, .. } + | Self::NotSupported { backtrace, .. } + | Self::CommitConflict { backtrace, .. } + | Self::IncompatibleTransaction { backtrace, .. } + | Self::RetryableCommitConflict { backtrace, .. } + | Self::TooMuchWriteContention { backtrace, .. } + | Self::Internal { backtrace, .. } + | Self::PrerequisiteFailed { backtrace, .. } + | Self::Unprocessable { backtrace, .. } + | Self::Arrow { backtrace, .. } + | Self::Schema { backtrace, .. } + | Self::NotFound { backtrace, .. } + | Self::IO { backtrace, .. } + | Self::Index { backtrace, .. } + | Self::IndexNotFound { backtrace, .. } + | Self::Wrapped { backtrace, .. } + | Self::Cloned { backtrace, .. } + | Self::Execution { backtrace, .. } + | Self::VersionConflict { backtrace, .. } + | Self::Namespace { backtrace, .. } => { + use snafu::AsBacktrace; + backtrace.as_backtrace() + } + // Variants without a backtrace field — listed explicitly so that + // adding a new variant with a backtrace field triggers a compiler error. + Self::InvalidTableLocation { .. } + | Self::Stop + | Self::InvalidRef { .. } + | Self::RefConflict { .. } + | Self::RefNotFound { .. } + | Self::Cleanup { .. } + | Self::VersionNotFound { .. } + | Self::External { .. } + | Self::FieldNotFound { .. } + | Self::Timeout { .. } + | Self::DiskCapExceeded { .. } + | Self::Fenced { .. } => None, + } + } + + /// Returns the captured Rust backtrace, if available. + /// + /// Always returns `None` when the `backtrace` feature is not enabled. + #[cfg(not(feature = "backtrace"))] + pub fn backtrace(&self) -> Option<&std::backtrace::Backtrace> { + None + } + #[track_caller] pub fn corrupt_file(path: object_store::path::Path, message: impl Into) -> Self { CorruptFileSnafu { path }.into_error(message.into().into()) } + /// Reports a corrupt file when the caller only has a logical/section name + /// rather than the real file path (for example, a decoder that validates an + /// in-memory buffer and does not know where it came from). + /// + /// `name` is carried in the `path` field of the resulting [`Error::CorruptFile`] + /// variant and is NOT a filesystem path; callers that have the real path should + /// use [`Self::corrupt_file`] instead. + #[track_caller] + pub fn corrupt_file_named(name: &str, message: impl Into) -> Self { + Self::corrupt_file(object_store::path::Path::from(name), message) + } + #[track_caller] pub fn invalid_input(message: impl Into) -> Self { InvalidInputSnafu.into_error(message.into().into()) @@ -367,9 +567,25 @@ impl Error { NotFoundSnafu { uri: uri.into() }.build() } + /// Return whether this error or one of its typed sources is a missing object. + pub fn is_not_found(&self) -> bool { + match self { + Self::NotFound { .. } => true, + Self::Wrapped { error, .. } + if error.downcast_ref::().is_some() => + { + false + } + Self::IO { source, .. } | Self::Wrapped { error: source, .. } => { + error_source_is_not_found(source.as_ref()) + } + _ => false, + } + } + #[track_caller] pub fn wrapped(error: BoxedError) -> Self { - WrappedSnafu { error }.build() + WrappedSnafu.into_error(error) } #[track_caller] @@ -498,6 +714,21 @@ impl Error { RetryableCommitConflictSnafu { version }.into_error(source) } + #[track_caller] + pub fn commit_status_unknown_source(version: u64, source: BoxedError) -> Self { + Self::wrapped(box_error(CommitStatusUnknownError { version, source })) + } + + /// Return whether this error represents a commit whose final outcome could + /// not be determined safely. + pub fn is_commit_status_unknown(&self) -> bool { + matches!( + self, + Self::Wrapped { error, .. } + if error.downcast_ref::().is_some() + ) + } + #[track_caller] pub fn incompatible_transaction_source(source: BoxedError) -> Self { IncompatibleTransactionSnafu.into_error(source) @@ -548,6 +779,17 @@ impl Error { } } +fn error_source_is_not_found(source: &(dyn std::error::Error + 'static)) -> bool { + if let Some(error) = source.downcast_ref::() { + return error.is_not_found(); + } + if let Some(error) = source.downcast_ref::() { + return matches!(error, object_store::Error::NotFound { .. }) + || std::error::Error::source(error).is_some_and(error_source_is_not_found); + } + source.source().is_some_and(error_source_is_not_found) +} + pub trait LanceOptionExt { /// Unwraps an option, returning an internal error if the option is None. /// @@ -611,7 +853,11 @@ impl From for Error { impl From for Error { #[track_caller] fn from(e: object_store::Error) -> Self { - Self::io_source(box_error(e)) + match e { + // source intentionally dropped; Error::NotFound carries only the path + object_store::Error::NotFound { path, .. } => Self::not_found(path), + other => Self::io_source(box_error(other)), + } } } @@ -717,6 +963,17 @@ impl From for Error { Self::not_supported_source(box_error(e)) } datafusion_common::DataFusionError::Execution(..) => Self::execution(e.to_string()), + datafusion_common::DataFusionError::Shared(shared) => { + // DataFusion shares an error across consumers (e.g. a join's + // build-side error fanned out to every probe partition) behind an + // `Arc`. If we are the sole owner we can recurse for full fidelity; + // otherwise the inner error can't be moved out, so we preserve its + // message under the execution category (its concrete type is lost). + match std::sync::Arc::try_unwrap(shared) { + Ok(inner) => Self::from(inner), + Err(shared) => Self::execution(shared.to_string()), + } + } datafusion_common::DataFusionError::External(source) => { // Try to downcast to lance_core::Error first match source.downcast::() { @@ -749,14 +1006,46 @@ pub fn get_caller_location() -> &'static std::panic::Location<'static> { /// Wrap an error in a new error type that implements Clone /// /// This is useful when two threads/streams share a common fallible source -/// The base error will always have the full error. Any cloned results will -/// only have Error::Cloned with the to_string of the base error. +/// Definite not-found errors preserve typed source-chain detection and their +/// human-readable representation. Timeout and I/O errors preserve their error +/// categories. Other cloned results use Error::Cloned with the string +/// representation of the base error. pub struct CloneableError(pub Error); +struct DisplayError(Error); + +impl fmt::Debug for DisplayError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + fmt::Display::fmt(self, f) + } +} + +impl fmt::Display for DisplayError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + fmt::Display::fmt(&self.0, f) + } +} + +impl std::error::Error for DisplayError { + fn source(&self) -> Option<&(dyn std::error::Error + 'static)> { + Some(&self.0) + } +} + impl Clone for CloneableError { #[track_caller] fn clone(&self) -> Self { - Self(Error::cloned(self.0.to_string())) + match &self.0 { + Error::NotFound { uri, .. } => Self(Error::wrapped(Box::new(DisplayError( + Error::not_found(uri.clone()), + )))), + error if error.is_not_found() => Self(Error::wrapped(Box::new(DisplayError( + Error::not_found(error.to_string()), + )))), + Error::Timeout { message, .. } => Self(Error::timeout(message.clone())), + Error::IO { source, .. } => Self(Error::io(source.to_string())), + error => Self(Error::cloned(error.to_string())), + } } } @@ -772,8 +1061,56 @@ impl From> for CloneableResult { #[cfg(test)] mod test { use super::*; + use std::error::Error as _; use std::fmt; + #[test] + fn cloneable_error_preserves_not_found_contract() { + let original = CloneableError(Error::not_found("metadata.lance")); + let cloned = original.clone(); + let cloned_again = cloned.clone(); + assert!(matches!(original.0, Error::NotFound { .. })); + assert!(cloned.0.is_not_found()); + assert!(cloned_again.0.is_not_found()); + assert!(cloned.0.to_string().to_lowercase().contains("not found")); + assert!( + cloned_again + .0 + .to_string() + .to_lowercase() + .contains("not found") + ); + assert!( + format!("{:?}", cloned.0) + .to_lowercase() + .contains("not found") + ); + assert!(cloned.0.source().is_some_and(|source| source.is::() + || source.source().is_some_and(|source| source.is::()))); + let downstream_error = Error::wrapped(Box::new(Error::io_source(Box::new( + object_store::Error::Generic { + store: "N/A", + source: Box::new(cloned.0), + }, + )))); + assert!(downstream_error.is_not_found()); + assert!( + format!("{downstream_error:?}") + .to_lowercase() + .contains("not found") + ); + + let original = CloneableError(Error::timeout("metadata read timed out")); + let cloned = original.clone(); + assert!(matches!(original.0, Error::Timeout { .. })); + assert!(matches!(cloned.0, Error::Timeout { .. })); + + let original = CloneableError(Error::io("metadata read was denied")); + let cloned = original.clone(); + assert!(matches!(original.0, Error::IO { .. })); + assert!(matches!(cloned.0, Error::IO { .. })); + } + #[test] fn test_caller_location_capture() { let current_fn = get_caller_location(); @@ -796,6 +1133,41 @@ mod test { } } + #[test] + fn test_caller_location_capture_not_found() { + let current_fn = get_caller_location(); + let f: Box Result<()>> = Box::new(|| { + Err(object_store::Error::NotFound { + path: "some/path".to_string(), + source: "not found".into(), + })?; + Ok(()) + }); + match f().unwrap_err() { + Error::NotFound { location, .. } => { + // +2 is the beginning of object_store::Error::NotFound... + assert_eq!(location.line(), current_fn.line() + 2, "{}", location) + } + #[allow(unreachable_patterns)] + other => panic!("expected NotFound, got {:?}", other), + } + } + + #[test] + fn test_object_store_not_found_converts_to_not_found() { + let os_err = object_store::Error::NotFound { + path: "test/path".to_string(), + source: "no such file".into(), + }; + let lance_err: Error = os_err.into(); + match lance_err { + Error::NotFound { uri, .. } => { + assert_eq!(uri, "test/path"); + } + other => panic!("Expected NotFound, got {:?}", other), + } + } + #[derive(Debug)] struct MyCustomError { code: i32, @@ -837,6 +1209,25 @@ mod test { assert!(matches!(converted, Error::IO { .. })); } + #[test] + fn test_commit_status_unknown_is_structured_without_masking_as_not_found() { + let error = Error::commit_status_unknown_source( + 42, + box_error(Error::not_found("temporarily invisible manifest")), + ); + + assert!(error.is_commit_status_unknown()); + assert!(!error.is_not_found()); + assert!(error.to_string().contains("version 42 is unknown")); + let Error::Wrapped { error, .. } = error else { + panic!("commit-status-unknown must use the semver-compatible wrapper") + }; + let status = error + .downcast_ref::() + .expect("wrapper must retain the typed commit status"); + assert_eq!(status.version(), 42); + } + #[test] fn test_external_error_creation() { let custom_err = MyCustomError { @@ -1048,4 +1439,68 @@ mod test { _ => panic!("Expected InvalidInput variant, got {:?}", recovered), } } + + #[test] + fn test_backtrace_accessor() { + // Verify that backtrace() returns the expected result based on feature state + let err = Error::io("test backtrace"); + let bt = err.backtrace(); + #[cfg(feature = "backtrace")] + { + // With the backtrace feature enabled, whether a backtrace is captured + // depends on the RUST_BACKTRACE env var at runtime. We just verify + // the accessor doesn't panic and returns a valid Option. + let _ = bt; + } + #[cfg(not(feature = "backtrace"))] + { + // Without the backtrace feature, this must always be None. + assert!(bt.is_none()); + } + } + + #[test] + fn test_backtrace_captured_when_feature_enabled() { + // Test that backtrace is actually captured when the feature is on and + // RUST_BACKTRACE=1 is set in the environment before the process starts. + // + // NOTE: std::backtrace::Backtrace caches the RUST_BACKTRACE env check, + // so set_var at runtime does not reliably enable capture. This test + // verifies the accessor works correctly in both cases: + // - If RUST_BACKTRACE=1 was set before the test binary started, we get Some. + // - If not, we get None (even with the feature on), which is expected. + #[cfg(feature = "backtrace")] + { + let err = Error::io("backtrace capture test"); + if std::env::var("RUST_BACKTRACE").is_ok() { + assert!( + err.backtrace().is_some(), + "Expected a backtrace when RUST_BACKTRACE=1 and backtrace feature is enabled" + ); + } + // When RUST_BACKTRACE is not set, backtrace() may return None even + // with the feature enabled — this is correct runtime gating behavior. + } + #[cfg(not(feature = "backtrace"))] + { + let err = Error::io("backtrace capture test"); + assert!(err.backtrace().is_none()); + } + } + + #[test] + fn test_backtrace_returns_none_for_variants_without_location() { + let err = Error::InvalidTableLocation { + message: "test".to_string(), + }; + assert!(err.backtrace().is_none()); + + let err = Error::InvalidRef { + message: "test".to_string(), + }; + assert!(err.backtrace().is_none()); + + let err = Error::Stop; + assert!(err.backtrace().is_none()); + } } diff --git a/rust/lance-core/src/utils/backoff.rs b/rust/lance-core/src/utils/backoff.rs index b30c757bb23..2b1f81e16be 100644 --- a/rust/lance-core/src/utils/backoff.rs +++ b/rust/lance-core/src/utils/backoff.rs @@ -77,6 +77,15 @@ impl Backoff { } } +/// Upper bound on the number of retry slots. +/// +/// Slots double each attempt to spread contending writers apart, but a hundred +/// or so already exceeds any realistic number of concurrent committers, so +/// further doubling only inflates the wait without reducing collisions. Capping +/// the count also bounds a single backoff to `(MAX_SLOTS - 1) * unit` instead of +/// letting it grow without limit as `attempt` climbs. +const MAX_SLOTS: u32 = 128; + /// SlotBackoff is a backoff strategy that randomly chooses a time slot to retry. /// /// This is useful when you have multiple tasks that can't overlap, and each @@ -85,7 +94,7 @@ impl Backoff { /// The `unit` represents the time it takes to complete one attempt. Future attempts /// are divided into time slots, and a random slot is chosen for the retry. The number /// of slots increases exponentially with each attempt. Initially, there are 4 slots, -/// then 8, then 16, and so on. +/// then 8, then 16, and so on, up to a fixed cap. /// /// Example: /// Suppose you have 10 tasks that can't overlap, each taking 1 second. The tasks @@ -138,10 +147,15 @@ impl SlotBackoff { } pub fn next_backoff(&mut self) -> Duration { - let num_slots = self.base.saturating_pow(self.attempt + self.starting_i); + let num_slots = self + .base + .saturating_pow(self.attempt.saturating_add(self.starting_i)) + .min(MAX_SLOTS); let slot_i = self.rng.random_range(0..num_slots); - self.attempt += 1; - Duration::from_millis((slot_i * self.unit) as u64) + self.attempt = self.attempt.saturating_add(1); + // Widen before multiplying: `unit` is the first-attempt latency, which + // can be large enough that a `u32` slot * unit product would overflow. + Duration::from_millis(slot_i as u64 * self.unit as u64) } } @@ -228,4 +242,50 @@ mod tests { assert_eq!(backoff.attempt(), 3); } } + + #[test] + fn test_slot_backoff_high_attempt_is_bounded() { + // Without the slot cap the wait grows unbounded with `attempt`. The cap + // holds every backoff to `(MAX_SLOTS - 1) * unit`. + let unit = 100_000; // 100s first attempt + let mut backoff = SlotBackoff::default().with_unit(unit); + let max_backoff = Duration::from_millis((MAX_SLOTS - 1) as u64 * unit as u64); + for _ in 0..40 { + assert!(backoff.next_backoff() <= max_backoff); + } + assert_eq!(backoff.attempt(), 40); + } + + #[test] + fn test_slot_backoff_large_unit_does_not_overflow() { + // With unit = u32::MAX, any slot >= 2 makes the old u32 `slot_i * unit` + // product overflow: a debug panic, or in release a wrap to a value that + // is no longer a multiple of unit. The u64 widening keeps every backoff + // an exact multiple of unit. Seed the RNG so the drawn slots — and thus + // this check — are deterministic rather than dependent on random draws. + let unit = u32::MAX; + let mut backoff = SlotBackoff::default().with_unit(unit); + backoff.rng = rand::rngs::SmallRng::seed_from_u64(0); + let mut saw_high_slot = false; + for _ in 0..64 { + let backoff_ms = backoff.next_backoff().as_millis(); + // `slot_i * unit` is always a multiple of unit; a wrapped u32 + // product is not. + assert_eq!(backoff_ms % unit as u128, 0, "{backoff_ms} wrapped"); + saw_high_slot |= backoff_ms >= 2 * unit as u128; + } + assert!(saw_high_slot, "expected a slot >= 2 in 64 seeded draws"); + } + + #[test] + fn test_slot_backoff_attempt_saturates() { + // At u32::MAX the counter must stay put rather than panic (debug) or + // wrap to 0 (release), which would restart the low-slot distribution. + let mut backoff = SlotBackoff { + attempt: u32::MAX, + ..Default::default() + }; + let _ = backoff.next_backoff(); + assert_eq!(backoff.attempt(), u32::MAX); + } } diff --git a/rust/lance-core/src/utils/cpu.rs b/rust/lance-core/src/utils/cpu.rs index 4e7ab01871d..58adeb24adb 100644 --- a/rust/lance-core/src/utils/cpu.rs +++ b/rust/lance-core/src/utils/cpu.rs @@ -1,14 +1,29 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright The Lance Authors +use std::fmt; use std::sync::LazyLock; -/// A level of SIMD support for some feature +/// A level of SIMD support for some feature. +/// +/// `#[non_exhaustive]` so future tiers (e.g. AVX-512 BF16, AMX) can be added +/// without breaking external `match` consumers. #[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[non_exhaustive] pub enum SimdSupport { None, Neon, Sse, + /// AVX (256-bit float ops) but no FMA and no AVX2. + /// Intel Sandy Bridge / Ivy Bridge. + Avx, + /// AVX + FMA but no AVX2. + /// AMD Piledriver / Steamroller / FX-7500. + AvxFma, + /// AVX2 + FMA. Intel Haswell / AMD Excavator and later. + /// + /// Selecting this tier asserts FMA is present: the kernels it dispatches to + /// are `#[target_feature(enable = "avx,fma")]`. Avx2, Avx512, Avx512FP16, @@ -16,6 +31,132 @@ pub enum SimdSupport { Lasx, } +impl fmt::Display for SimdSupport { + /// Formats the tier name in lowercase, matching pyarrow's + /// `runtime_info().simd_level` convention. + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + let name = match self { + Self::None => "none", + Self::Neon => "neon", + Self::Sse => "sse", + Self::Avx => "avx", + Self::AvxFma => "avx_fma", + Self::Avx2 => "avx2", + Self::Avx512 => "avx512", + Self::Avx512FP16 => "avx512_fp16", + Self::Lsx => "lsx", + Self::Lasx => "lasx", + }; + f.write_str(name) + } +} + +/// Snapshot of the SIMD tier lance dispatches to on the current host, plus the +/// raw CPU features detected for diagnostic purposes. +/// +/// Mirrors the role of `pyarrow.runtime_info()`: a single, cheap call users can +/// make to verify which SIMD tier the runtime selected and what underlying +/// features the host advertises. Obtain one with [`simd_info()`]. +#[derive(Debug, Clone)] +pub struct SimdInfo { + /// The SIMD tier lance dispatches to at runtime on this host. + pub tier: SimdSupport, + /// The architecture name (e.g. "x86_64", "aarch64", "loongarch64"). + pub target_arch: &'static str, + /// Raw CPU feature flags detected on this host (x86_64 only; empty on + /// other architectures). Each entry is a feature name like "avx2", + /// "fma", "avx512f", "popcnt", etc. + pub host_features: Vec<&'static str>, +} + +/// Returns a snapshot of the SIMD tier lance is using on this host along with +/// the raw CPU feature flags that drove the decision. +/// +/// Useful for performance debugging and giving users a way to verify which +/// dispatch tier they are hitting without rebuilding lance. See [`SimdInfo`] +/// for the meaning of each field and [`SimdSupport`] for the tier values. +/// +/// # Examples +/// +/// ``` +/// use lance_core::utils::cpu::simd_info; +/// +/// let info = simd_info(); +/// println!("dispatching to {} on {}", info.tier, info.target_arch); +/// ``` +pub fn simd_info() -> SimdInfo { + SimdInfo { + tier: *SIMD_SUPPORT, + target_arch: std::env::consts::ARCH, + host_features: detect_host_features(), + } +} + +#[cfg(target_arch = "x86_64")] +fn detect_host_features() -> Vec<&'static str> { + // Each call must be inline: `is_x86_feature_detected!` does its own custom + // input parsing and rejects feature names received via a `macro_rules!` + // `:literal` metavariable on some toolchains. + let mut features = Vec::with_capacity(17); + if is_x86_feature_detected!("sse2") { + features.push("sse2"); + } + if is_x86_feature_detected!("sse3") { + features.push("sse3"); + } + if is_x86_feature_detected!("ssse3") { + features.push("ssse3"); + } + if is_x86_feature_detected!("sse4.1") { + features.push("sse4.1"); + } + if is_x86_feature_detected!("sse4.2") { + features.push("sse4.2"); + } + if is_x86_feature_detected!("popcnt") { + features.push("popcnt"); + } + if is_x86_feature_detected!("avx") { + features.push("avx"); + } + if is_x86_feature_detected!("avx2") { + features.push("avx2"); + } + if is_x86_feature_detected!("fma") { + features.push("fma"); + } + if is_x86_feature_detected!("f16c") { + features.push("f16c"); + } + if is_x86_feature_detected!("bmi1") { + features.push("bmi1"); + } + if is_x86_feature_detected!("bmi2") { + features.push("bmi2"); + } + if is_x86_feature_detected!("avx512f") { + features.push("avx512f"); + } + if is_x86_feature_detected!("avx512bw") { + features.push("avx512bw"); + } + if is_x86_feature_detected!("avx512cd") { + features.push("avx512cd"); + } + if is_x86_feature_detected!("avx512dq") { + features.push("avx512dq"); + } + if is_x86_feature_detected!("avx512vl") { + features.push("avx512vl"); + } + features +} + +#[cfg(not(target_arch = "x86_64"))] +fn detect_host_features() -> Vec<&'static str> { + Vec::new() +} + /// Support for SIMD operations pub static SIMD_SUPPORT: LazyLock = LazyLock::new(|| { #[cfg(all(target_arch = "aarch64", any(target_os = "ios", target_os = "tvos")))] @@ -42,8 +183,19 @@ pub static SIMD_SUPPORT: LazyLock = LazyLock::new(|| { } else { SimdSupport::Avx512 } - } else if is_x86_feature_detected!("avx2") { + } else if is_x86_feature_detected!("avx2") && is_x86_feature_detected!("fma") { + // FMA is checked explicitly: every kernel selected for this tier is + // `#[target_feature(enable = "avx,fma")]`, and AVX2 does not imply + // FMA in the ISA. Every shipping AVX2 part has FMA, so this only + // guards against a host that would otherwise take an FMA kernel + // without FMA. SimdSupport::Avx2 + } else if is_x86_feature_detected!("avx") && is_x86_feature_detected!("fma") { + // AMD Piledriver / Steamroller / FX-7500: 256-bit float ops + FMA but no AVX2. + SimdSupport::AvxFma + } else if is_x86_feature_detected!("avx") { + // Intel Sandy Bridge / Ivy Bridge: 256-bit float ops without FMA. + SimdSupport::Avx } else { SimdSupport::None } @@ -58,6 +210,14 @@ pub static SIMD_SUPPORT: LazyLock = LazyLock::new(|| { SimdSupport::None } } + #[cfg(not(any( + target_arch = "aarch64", + target_arch = "x86_64", + target_arch = "loongarch64" + )))] + { + SimdSupport::None + } }); #[cfg(target_arch = "x86_64")] @@ -138,3 +298,68 @@ mod aarch64 { false } } + +#[cfg(test)] +mod tests { + use super::*; + use rstest::rstest; + + #[test] + fn simd_info_exposes_tier() { + let info = simd_info(); + assert_eq!(info.target_arch, std::env::consts::ARCH); + // Tier should match the detected SIMD support. + assert_eq!(info.tier, *SIMD_SUPPORT); + } + + #[cfg(target_arch = "x86_64")] + #[test] + fn simd_info_features_include_baseline() { + let info = simd_info(); + // The x86_64 ABI mandates SSE2, so it must always be present on this + // architecture. + assert!(info.host_features.contains(&"sse2")); + } + + #[cfg(not(target_arch = "x86_64"))] + #[test] + fn simd_info_features_empty_off_x86_64() { + let info = simd_info(); + assert!(info.host_features.is_empty()); + } + + /// The `Avx2` and `AvxFma` tiers both dispatch to kernels declared + /// `#[target_feature(enable = "avx,fma")]`, so neither may be selected on a + /// host without FMA. AVX2 does not imply FMA in the ISA, so the detection + /// checks it explicitly. (`Avx512*` is excluded: its kernels declare + /// `avx512f`, which is what `has_avx512` verifies.) + #[cfg(target_arch = "x86_64")] + #[test] + fn avx_fma_tiers_are_only_selected_when_fma_is_detected() { + if matches!(*SIMD_SUPPORT, SimdSupport::Avx2 | SimdSupport::AvxFma) { + assert!( + is_x86_feature_detected!("fma"), + "tier {} dispatches to avx,fma kernels but the host has no FMA", + *SIMD_SUPPORT + ); + } + } + + #[rstest] + #[case::none(SimdSupport::None, "none")] + #[case::neon(SimdSupport::Neon, "neon")] + #[case::sse(SimdSupport::Sse, "sse")] + #[case::avx(SimdSupport::Avx, "avx")] + #[case::avx_fma(SimdSupport::AvxFma, "avx_fma")] + #[case::avx2(SimdSupport::Avx2, "avx2")] + #[case::avx512(SimdSupport::Avx512, "avx512")] + #[case::avx512_fp16(SimdSupport::Avx512FP16, "avx512_fp16")] + #[case::lsx(SimdSupport::Lsx, "lsx")] + #[case::lasx(SimdSupport::Lasx, "lasx")] + fn simd_support_display_matches_lowercase_convention( + #[case] tier: SimdSupport, + #[case] expected: &str, + ) { + assert_eq!(tier.to_string(), expected); + } +} diff --git a/rust/lance-core/src/utils/futures.rs b/rust/lance-core/src/utils/futures.rs index 95a1c39aaae..a7e174eb32e 100644 --- a/rust/lance-core/src/utils/futures.rs +++ b/rust/lance-core/src/utils/futures.rs @@ -382,6 +382,19 @@ mod tests { assert_eq!(right.next().await, None); } + #[test] + fn test_shared_stream_replaces_waiting_waker() { + let inner_stream = futures::stream::pending::(); + let (mut left, mut right) = inner_stream.boxed().share(Capacity::Unbounded); + + let mut left_fut = left.next(); + assert!(is_pending(&mut left_fut)); + + let mut right_fut = right.next(); + assert!(is_pending(&mut right_fut)); + assert!(is_pending(&mut right_fut)); + } + #[tokio::test] async fn test_unbounded_shared_stream() { let (tx, rx) = tokio::sync::mpsc::channel::(10); diff --git a/rust/lance-core/src/utils/tokio.rs b/rust/lance-core/src/utils/tokio.rs index 46c9475665b..aacce4ad247 100644 --- a/rust/lance-core/src/utils/tokio.rs +++ b/rust/lance-core/src/utils/tokio.rs @@ -21,8 +21,8 @@ pub fn get_num_compute_intensive_cpus() -> usize { } fn calculate_num_compute_intensive_cpus() -> usize { - if let Ok(user_specified) = std::env::var("LANCE_CPU_THREADS") { - return user_specified.parse().unwrap(); + if let Ok(raw) = std::env::var("LANCE_CPU_THREADS") { + return parse_env_usize("LANCE_CPU_THREADS", &raw, 1).unwrap_or_else(|e| panic!("{e}")); } let cpus = num_cpus::get(); @@ -42,12 +42,38 @@ fn calculate_num_compute_intensive_cpus() -> usize { num_cpus::get() - *IO_CORE_RESERVATION } -pub static IO_CORE_RESERVATION: LazyLock = LazyLock::new(|| { - std::env::var("LANCE_IO_CORE_RESERVATION") - .unwrap_or("2".to_string()) +/// Parse an integer environment variable, rejecting values below `min`. +/// +/// The error names the variable, so a bad value is diagnosable instead of +/// surfacing as a bare `ParseIntError` or, for `LANCE_CPU_THREADS=0`, a panic +/// deep inside tokio's `max_blocking_threads`. +fn parse_env_usize(name: &str, raw: &str, min: usize) -> Result { + let value: usize = raw + .trim() .parse() - .unwrap() -}); + .map_err(|e| format!("environment variable {name} must be an integer, got {raw:?}: {e}"))?; + if value < min { + return Err(format!( + "environment variable {name} must be at least {min}, got {value}" + )); + } + Ok(value) +} + +/// Number of CPU cores held back for I/O and control tasks. +/// +/// Overridable via the `LANCE_IO_CORE_RESERVATION` environment variable; +/// defaults to `2` when unset. `0` is allowed (reserve nothing); +/// [`get_num_compute_intensive_cpus`] subtracts this from the core count to +/// size the compute pool. A non-integer value panics on first access with an +/// error naming the variable. +pub static IO_CORE_RESERVATION: LazyLock = + LazyLock::new(|| match std::env::var("LANCE_IO_CORE_RESERVATION") { + Ok(raw) => { + parse_env_usize("LANCE_IO_CORE_RESERVATION", &raw, 0).unwrap_or_else(|e| panic!("{e}")) + } + Err(_) => 2, + }); fn create_runtime() -> Runtime { Builder::new_multi_thread() @@ -66,11 +92,15 @@ static RUNTIME_INSTALLED: atomic::AtomicBool = atomic::AtomicBool::new(false); static ATFORK_INSTALLED: atomic::AtomicBool = atomic::AtomicBool::new(false); -fn global_cpu_runtime() -> &'static mut Runtime { +fn global_cpu_runtime() -> &'static Runtime { loop { let ptr = CPU_RUNTIME.load(Ordering::SeqCst); if !ptr.is_null() { - return unsafe { &mut *ptr }; + // SAFETY: `ptr` was produced by `Box::into_raw` below and is only ever + // reset to null by `atfork_tokio_child` in the forked child (single- + // threaded, async-signal context). The `Box` is never reclaimed, so the + // `Runtime` lives for the rest of the process. + return unsafe { &*ptr }; } if !RUNTIME_INSTALLED.fetch_or(true, Ordering::SeqCst) { break; @@ -82,7 +112,9 @@ fn global_cpu_runtime() -> &'static mut Runtime { } let new_ptr = Box::into_raw(Box::new(create_runtime())); CPU_RUNTIME.store(new_ptr, Ordering::SeqCst); - unsafe { &mut *new_ptr } + // SAFETY: `new_ptr` was just obtained from `Box::into_raw`, so it is non-null, + // aligned, and points to a live `Runtime` that is never reclaimed. + unsafe { &*new_ptr } } /// After a fork() operation, force re-creation of the BackgroundExecutor. Note: this function @@ -108,6 +140,43 @@ fn install_atfork() {} /// /// This can also be used to convert a big chunk of synchronous work into a future /// so that it can be run in parallel with something like StreamExt::buffered() +/// +/// # Only hand over substantial CPU work +/// +/// Dispatching to the pool has real overhead (a `spawn_blocking` hop plus a oneshot +/// channel round trip). As a rule of thumb the closure should be expected to do at +/// least ~100µs of CPU work; below that the thread-pool overhead is likely to +/// outweigh any parallelism benefit, and the work is better left inline. +/// +/// # The task must never wait on anything +/// +/// The CPU pool is sized to [`get_num_compute_intensive_cpus`], which is +/// `max(1, num_cpus - LANCE_IO_CORE_RESERVATION)`. On a big host that is plenty of +/// workers (e.g. 62 on a 64-core box), but in resource-constrained environments it can +/// collapse to a **single blocking thread** — on machines with `<= 3` visible CPUs +/// (1-vCPU VMs, CI runners, CPU-limited Kubernetes pods) the pool has exactly one +/// worker. A closure passed to `spawn_cpu` occupies one of these threads for its entire +/// lifetime, including any time it spends *parked*. So the closure must only consume +/// CPU and return; it must +/// **never** block, wait, or park. Concretely, the closure must not, directly or +/// transitively: +/// +/// * **No channels** — no blocking send/recv (`send_blocking`, blocking `recv`, etc.). +/// A full/empty channel parks the thread, and whatever would drain/fill the channel +/// may need the same pool to run. +/// * **No I/O** — no file, network, or object-store reads/writes, and no disk spills. +/// I/O parks the thread while making no progress on CPU work. +/// * **No locks** — no acquiring a contended lock (or any lock that is held across an +/// `.await` elsewhere). Waiting for the lock parks the thread. +/// * **No `block_on` / `.blocking_*`** — never drive or wait on another async task +/// from inside the closure. +/// +/// If any of these hold, the parked thread can starve the exact work that would +/// unblock it, deadlocking the whole pool with no timeout and no error — a silent +/// hang at 0% CPU. (See .) When work +/// needs to wait on a channel/lock/I/O, keep the waiting in an async task and only +/// hand the pure-CPU portion to `spawn_cpu`, e.g. build each batch with `spawn_cpu` +/// and dispatch it with `tx.send(batch).await` in the surrounding async code. pub fn spawn_cpu< E: std::error::Error + Send + 'static, F: FnOnce() -> std::result::Result + Send + 'static, @@ -125,3 +194,39 @@ pub fn spawn_cpu< }); recv.map(|res| res.unwrap()) } + +#[cfg(test)] +mod tests { + use super::*; + + // The env vars feed process-global `LazyLock`s that read once and are read + // in parallel by other tests, so the pure parser is tested directly rather + // than by mutating the environment. + + #[test] + fn parses_valid_value_and_trims_surrounding_whitespace() { + assert_eq!(parse_env_usize("VAR", "8", 1).unwrap(), 8); + assert_eq!(parse_env_usize("VAR", " 8 ", 1).unwrap(), 8); + } + + #[test] + fn rejects_non_integer_naming_the_variable() { + let err = parse_env_usize("LANCE_CPU_THREADS", "abc", 1).unwrap_err(); + assert!(err.contains("LANCE_CPU_THREADS"), "{err}"); + assert!(err.contains("must be an integer"), "{err}"); + } + + #[test] + fn rejects_value_below_minimum() { + // LANCE_CPU_THREADS=0 parses fine but would panic in tokio's + // max_blocking_threads(0); the minimum stops it at the boundary. + let err = parse_env_usize("LANCE_CPU_THREADS", "0", 1).unwrap_err(); + assert!(err.contains("at least 1"), "{err}"); + } + + #[test] + fn allows_zero_when_minimum_is_zero() { + // LANCE_IO_CORE_RESERVATION=0 is valid: no cores reserved for IO. + assert_eq!(parse_env_usize("VAR", "0", 0).unwrap(), 0); + } +} diff --git a/rust/lance-core/src/utils/tracing.rs b/rust/lance-core/src/utils/tracing.rs index 603a666e313..e1f19e3c6f4 100644 --- a/rust/lance-core/src/utils/tracing.rs +++ b/rust/lance-core/src/utils/tracing.rs @@ -66,6 +66,7 @@ pub const AUDIT_TYPE_DELETION: &str = "deletion"; pub const AUDIT_TYPE_MANIFEST: &str = "manifest"; pub const AUDIT_TYPE_INDEX: &str = "index"; pub const AUDIT_TYPE_DATA: &str = "data"; +pub const AUDIT_TYPE_TRANSACTION: &str = "transaction"; pub const TRACE_FILE_CREATE: &str = "create"; pub const TRACE_IO_EVENTS: &str = "lance::io_events"; pub const IO_TYPE_OPEN_SCALAR: &str = "open_scalar_index"; diff --git a/rust/lance-core/tests/cache_key_allocations.rs b/rust/lance-core/tests/cache_key_allocations.rs new file mode 100644 index 00000000000..ef73d485ef5 --- /dev/null +++ b/rust/lance-core/tests/cache_key_allocations.rs @@ -0,0 +1,166 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright The Lance Authors + +use std::alloc::{GlobalAlloc, Layout, System}; +use std::borrow::Cow; +use std::cell::Cell; +use std::hint::black_box; +use std::sync::atomic::{AtomicUsize, Ordering}; + +use lance_core::cache::{CacheKey, CacheKeySchema, CacheNamespace, InternalCacheKey, KeyBuilder}; + +struct TrackingAllocator; + +thread_local! { + static TRACK_ALLOCATIONS: Cell = const { Cell::new(false) }; +} + +static ALLOCATION_COUNT: AtomicUsize = AtomicUsize::new(0); + +fn record_allocation() { + if TRACK_ALLOCATIONS.try_with(Cell::get).unwrap_or(false) { + ALLOCATION_COUNT.fetch_add(1, Ordering::Relaxed); + } +} + +unsafe impl GlobalAlloc for TrackingAllocator { + unsafe fn alloc(&self, layout: Layout) -> *mut u8 { + record_allocation(); + unsafe { System.alloc(layout) } + } + + unsafe fn alloc_zeroed(&self, layout: Layout) -> *mut u8 { + record_allocation(); + unsafe { System.alloc_zeroed(layout) } + } + + unsafe fn dealloc(&self, ptr: *mut u8, layout: Layout) { + unsafe { System.dealloc(ptr, layout) } + } + + unsafe fn realloc(&self, ptr: *mut u8, layout: Layout, new_size: usize) -> *mut u8 { + record_allocation(); + unsafe { System.realloc(ptr, layout, new_size) } + } +} + +#[global_allocator] +static ALLOCATOR: TrackingAllocator = TrackingAllocator; + +fn measured_allocations(operation: impl FnOnce()) -> usize { + TRACK_ALLOCATIONS.with(|tracking| tracking.set(false)); + ALLOCATION_COUNT.store(0, Ordering::Relaxed); + TRACK_ALLOCATIONS.with(|tracking| tracking.set(true)); + operation(); + TRACK_ALLOCATIONS.with(|tracking| tracking.set(false)); + ALLOCATION_COUNT.load(Ordering::Relaxed) +} + +fn prepare_key(namespace: CacheNamespace, key: &K) -> InternalCacheKey { + let mut builder = KeyBuilder::new(namespace, K::stable_type_id(), K::schema()); + key.write_key(&mut builder); + builder.finish() +} + +struct PageKey { + path: &'static str, + column_index: u32, + page_index: u64, +} + +impl CacheKey for PageKey { + type ValueType = u64; + + fn key(&self) -> Cow<'_, str> { + Cow::Borrowed("unused") + } + + fn type_name() -> &'static str { + "allocation-test-page" + } + + fn schema() -> CacheKeySchema { + CacheKeySchema::new("allocation-test-page", 1) + } + + fn write_key(&self, builder: &mut KeyBuilder) { + builder.write_str(self.path); + builder.write_u32(self.column_index); + builder.write_u64(self.page_index); + } +} + +struct OptionalUuidKey { + generation: u64, + uuid: Option<[u8; 16]>, +} + +impl CacheKey for OptionalUuidKey { + type ValueType = u64; + + fn key(&self) -> Cow<'_, str> { + Cow::Borrowed("unused") + } + + fn type_name() -> &'static str { + "allocation-test-optional-uuid" + } + + fn schema() -> CacheKeySchema { + CacheKeySchema::new("allocation-test-optional-uuid", 1) + } + + fn write_key(&self, builder: &mut KeyBuilder) { + builder.write_u64(self.generation); + match self.uuid { + Some(uuid) => { + builder.write_some(); + builder.write_fixed_bytes(&uuid); + } + None => builder.write_none(), + } + } +} + +#[test] +fn production_shaped_typed_keys_allocate_nothing_after_warmup() { + let namespace = CacheNamespace::root() + .child("tenant-with-a-long-stable-identifier") + .child("index-with-a-long-stable-identifier"); + let page = PageKey { + path: "indices/01999f62-c3c2-7d6f-820d-22e7db948f31/pages/000000000042.lance", + column_index: 17, + page_index: 42, + }; + let uuid = OptionalUuidKey { + generation: 9, + uuid: Some(*b"0123456789abcdef"), + }; + let no_uuid = OptionalUuidKey { + generation: 10, + uuid: None, + }; + + black_box(prepare_key(namespace, &page)); + black_box(prepare_key(namespace, &uuid)); + black_box(prepare_key(namespace, &no_uuid)); + + assert_eq!( + measured_allocations(|| { + black_box(prepare_key(namespace, &page)); + }), + 0 + ); + assert_eq!( + measured_allocations(|| { + black_box(prepare_key(namespace, &uuid)); + }), + 0 + ); + assert_eq!( + measured_allocations(|| { + black_box(prepare_key(namespace, &no_uuid)); + }), + 0 + ); +} diff --git a/rust/lance-datafusion/src/exec.rs b/rust/lance-datafusion/src/exec.rs index 8f346f45612..994e16b783d 100644 --- a/rust/lance-datafusion/src/exec.rs +++ b/rust/lance-datafusion/src/exec.rs @@ -6,6 +6,7 @@ use std::{ collections::HashMap, fmt::{self, Formatter}, + num::NonZero, sync::{Arc, Mutex, OnceLock}, time::Duration, }; @@ -14,9 +15,8 @@ use chrono::{DateTime, Utc}; use arrow_array::RecordBatch; use arrow_schema::Schema as ArrowSchema; -use datafusion::physical_plan::metrics::MetricType; use datafusion::{ - catalog::streaming::StreamingTable, + catalog::{TableProvider, streaming::StreamingTable}, dataframe::DataFrame, execution::{ TaskContext, @@ -26,16 +26,19 @@ use datafusion::{ runtime_env::RuntimeEnvBuilder, }, physical_plan::{ - DisplayAs, DisplayFormatType, ExecutionPlan, PlanProperties, SendableRecordBatchStream, + DisplayAs, DisplayFormatType, ExecutionPlan, ExecutionPlanProperties, PlanProperties, + SendableRecordBatchStream, analyze::AnalyzeExec, coalesce_partitions::CoalescePartitionsExec, display::DisplayableExecutionPlan, execution_plan::{Boundedness, CardinalityEffect, EmissionType}, metrics::MetricValue, + sorts::sort_preserving_merge::SortPreservingMergeExec, stream::RecordBatchStreamAdapter, streaming::PartitionStream, }, }; +use datafusion::{execution::memory_pool::TrackConsumersPool, physical_plan::metrics::MetricType}; use datafusion_common::{DataFusionError, Statistics}; use datafusion_physical_expr::{EquivalenceProperties, Partitioning}; @@ -55,8 +58,9 @@ use crate::udf::register_functions; use crate::{ chunker::StrictBatchSizeStream, utils::{ - BYTES_READ_METRIC, INDEX_COMPARISONS_METRIC, INDICES_LOADED_METRIC, IOPS_METRIC, - MetricsExt, PARTS_LOADED_METRIC, REQUESTS_METRIC, + BYTES_READ_METRIC, INDEX_CACHE_HITS_METRIC, INDEX_CACHE_MISSES_METRIC, + INDEX_COMPARISONS_METRIC, INDICES_LOADED_METRIC, IOPS_METRIC, MetricsExt, + PARTS_LOADED_METRIC, REQUESTS_METRIC, }, }; @@ -152,10 +156,6 @@ impl ExecutionPlan for OneShotExec { "OneShotExec" } - fn as_any(&self) -> &dyn std::any::Any { - self - } - fn schema(&self) -> arrow_schema::SchemaRef { self.schema.clone() } @@ -243,10 +243,6 @@ impl ExecutionPlan for TracedExec { "TracedExec" } - fn as_any(&self) -> &dyn std::any::Any { - self - } - fn properties(&self) -> &Arc { &self.properties } @@ -310,7 +306,7 @@ impl std::fmt::Debug for LanceExecutionOptions { } } -const DEFAULT_LANCE_MEM_POOL_SIZE_PER_PARTITION: u64 = 100 * 1024 * 1024; +const DEFAULT_LANCE_MEM_POOL_SIZE_PER_PARTITION: u64 = 150 * 1024 * 1024; const DEFAULT_LANCE_MAX_TEMP_DIRECTORY_SIZE: u64 = 100 * 1024 * 1024 * 1024; // 100GB impl LanceExecutionOptions { @@ -366,12 +362,21 @@ pub fn new_session_context(options: &LanceExecutionOptions) -> SessionContext { session_config = session_config.with_target_partitions(target_partition); } if options.use_spilling() { + // The default 10MB sort spill reservation seems to be too small for many common cases. + // + // There currently is no reasonable guidance provided by DataFusion for setting this value. + // We bump this to 40MB but try a smaller value if the mem pool is small. + let sort_spill_reservation_bytes = + (options.mem_pool_size() / 3).min(40 * 1024 * 1024) as usize; + session_config = + session_config.with_sort_spill_reservation_bytes(sort_spill_reservation_bytes); let disk_manager_builder = DiskManagerBuilder::default() .with_max_temp_directory_size(options.max_temp_directory_size()); runtime_env_builder = runtime_env_builder .with_disk_manager_builder(disk_manager_builder) - .with_memory_pool(Arc::new(FairSpillPool::new( - options.mem_pool_size() as usize + .with_memory_pool(Arc::new(TrackConsumersPool::new( + FairSpillPool::new(options.mem_pool_size() as usize), + NonZero::try_from(16).unwrap(), ))); } let runtime_env = runtime_env_builder.build_arc().unwrap(); @@ -486,12 +491,88 @@ pub struct ExecutionSummaryCounts { pub index_comparisons: usize, /// Additional metrics for more detailed statistics. These are subject to change in the future /// and should only be used for debugging purposes. + /// + /// Newer metrics (e.g. [`INDEX_CACHE_HITS_METRIC`], [`INDEX_CACHE_MISSES_METRIC`]) are added + /// here rather than as `pub` fields, so this struct stays backwards compatible for callers + /// that construct or destructure it. Prefer the typed accessors below. pub all_counts: HashMap, /// Additional time metrics for more detailed statistics, stored in nanoseconds. /// These are subject to change in the future and should only be used for debugging purposes. pub all_times: HashMap, } +impl ExecutionSummaryCounts { + /// Number of index cache page lookups where the loader was not executed + /// (per-page granularity). + /// + /// A "hit" is any page-level lookup at an instrumented cache boundary that + /// did not run the loader on this call. That covers both a true cache hit + /// on an already-populated entry and a coalesced concurrent load where an + /// in-flight loader started by a different caller produced the value. + /// + /// Instrumented boundaries in this release: + /// BTree page, IVF partition (v2, `write_cache=true` scan path), inverted + /// posting list (grouped and per-token), inverted per-token metadata + /// (`PostingMetadataKey`), inverted phrase positions (`PositionKey`), + /// bitmap posting (Equals / Range / IsIn), ngram posting, and rtree page + /// / null slot. + /// + /// Caveats: + /// * IVF v2 streaming scans and legacy v1 IVF partitions run + /// `load_partition` with `write_cache=false`. Those loads always execute + /// the loader and never write the result back, so they are reported as a + /// miss on every call. See [`Self::index_cache_hit_ratio`]. + /// * A cold posting-list lookup on the grouped inverted layout can record + /// up to two misses (posting-list group + per-token metadata) for a + /// single term. + /// + /// Other index cache boundaries such as HNSW graph pages and quantizer + /// codebooks are not yet instrumented; a scan that only touches those + /// paths returns `0` here. + pub fn index_cache_hits(&self) -> usize { + self.all_counts + .get(INDEX_CACHE_HITS_METRIC) + .copied() + .unwrap_or(0) + } + + /// Number of index cache page lookups that had to execute the loader + /// (per-page granularity). + /// + /// A "miss" is any page-level lookup at an instrumented cache boundary + /// where the loader ran, i.e. the page was not resident and had to be + /// materialised (typically from storage). See + /// [`Self::index_cache_hits`] for the paired counter and the list of + /// instrumented boundaries. + pub fn index_cache_misses(&self) -> usize { + self.all_counts + .get(INDEX_CACHE_MISSES_METRIC) + .copied() + .unwrap_or(0) + } + + /// Ratio of index cache hits to total lookups. Returns `0.0` when no lookups + /// were recorded in this scan. + /// + /// This ratio only reflects paths that write their result back to the + /// index cache. Streaming scans (IVF v2 `write_cache=false` and legacy v1 + /// IVF `load_partition_stream`) intentionally bypass the cache and are + /// counted as misses on every call, so a workload dominated by streaming + /// vector scans will report a hit ratio near `0.0` regardless of cache + /// size. + pub fn index_cache_hit_ratio(&self) -> f32 { + // Widen to u128 before summing so a pathological (hits + misses) + // overflow can't panic in debug builds nor wrap in release builds. + let hits = self.index_cache_hits() as u128; + let total = hits + self.index_cache_misses() as u128; + if total == 0 { + 0.0 + } else { + hits as f32 / total as f32 + } + } +} + pub fn collect_execution_metrics(node: &dyn ExecutionPlan, counts: &mut ExecutionSummaryCounts) { if let Some(metrics) = node.metrics() { for (metric_name, count) in metrics.iter_counts() { @@ -552,6 +633,8 @@ fn report_plan_summary_metrics(plan: &dyn ExecutionPlan, options: &LanceExecutio indices_loaded = counts.indices_loaded, parts_loaded = counts.parts_loaded, index_comparisons = counts.index_comparisons, + index_cache_hits = counts.index_cache_hits(), + index_cache_misses = counts.index_cache_misses(), ); } if let Some(callback) = options.execution_stats_callback.as_ref() { @@ -610,8 +693,17 @@ pub fn execute_plan( // Coalesce to a single partition if the optimizer left more than one. // EnforceDistribution may remove RepartitionExec(1) nodes when the parent // declares UnspecifiedDistribution, leaving multi-partition plans here. + // + // If the plan carries an output ordering (e.g. a top-k `SortExec` whose + // result was later repartitioned to parallelize downstream operators), + // a plain `CoalescePartitionsExec` would scramble that order because it + // merges partitions in scheduling-dependent order. Use an order-preserving + // merge in that case instead, mirroring what `EnforceDistribution` itself + // does when it needs to merge an ordered, multi-partition plan. let plan: Arc = if plan.properties().partitioning.partition_count() == 1 { plan + } else if let Some(ordering) = plan.output_ordering() { + Arc::new(SortPreservingMergeExec::new(ordering.clone(), plan)) } else { Arc::new(CoalescePartitionsExec::new(plan)) }; @@ -640,7 +732,8 @@ pub async fn analyze_plan( let analyze = Arc::new(AnalyzeExec::new( true, true, - vec![MetricType::SUMMARY], + vec![MetricType::Summary], + None, plan, schema, )); @@ -867,6 +960,49 @@ impl SessionContextExt for SessionContext { } } +/// Scan a [`TableProvider`] into a single-partition [`SendableRecordBatchStream`]. +/// +/// Multi-partition providers are coalesced into a single partition. This adapts a +/// re-scannable provider back into the one stream the writer pipeline consumes; +/// re-scanning the same provider (e.g. on a write retry) yields a fresh stream. +/// +/// # Examples +/// +/// ``` +/// # use std::sync::Arc; +/// # use arrow_array::{Int32Array, RecordBatch}; +/// # use arrow_schema::{DataType, Field, Schema}; +/// # use datafusion::catalog::TableProvider; +/// # use datafusion::datasource::MemTable; +/// # use futures::TryStreamExt; +/// # use lance_datafusion::exec::provider_to_stream; +/// # #[tokio::main] +/// # async fn main() -> Result<(), Box> { +/// let schema = Arc::new(Schema::new(vec![Field::new("a", DataType::Int32, false)])); +/// let batch = +/// RecordBatch::try_new(schema.clone(), vec![Arc::new(Int32Array::from(vec![1, 2, 3]))])?; +/// let provider: Arc = Arc::new(MemTable::try_new(schema, vec![vec![batch]])?); +/// +/// // A re-scannable provider yields a fresh stream on each call. +/// let batches: Vec = provider_to_stream(provider).await?.try_collect().await?; +/// assert_eq!(batches.iter().map(|b| b.num_rows()).sum::(), 3); +/// # Ok(()) +/// # } +/// ``` +pub async fn provider_to_stream( + provider: Arc, +) -> Result { + let ctx = SessionContext::new(); + let plan = provider.scan(&ctx.state(), None, &[], None).await?; + let plan: Arc = + if plan.properties().output_partitioning().partition_count() > 1 { + Arc::new(CoalescePartitionsExec::new(plan)) + } else { + plan + }; + Ok(plan.execute(0, ctx.task_ctx())?) +} + #[derive(Clone, Debug)] pub struct StrictBatchSizeExec { input: Arc, @@ -894,10 +1030,6 @@ impl ExecutionPlan for StrictBatchSizeExec { "StrictBatchSizeExec" } - fn as_any(&self) -> &dyn std::any::Any { - self - } - fn properties(&self) -> &Arc { self.input.properties() } @@ -938,7 +1070,7 @@ impl ExecutionPlan for StrictBatchSizeExec { fn partition_statistics( &self, partition: Option, - ) -> datafusion_common::Result { + ) -> datafusion_common::Result> { self.input.partition_statistics(partition) } @@ -1000,10 +1132,6 @@ impl ExecutionPlan for HardCapBatchSizeExec { "HardCapBatchSizeExec" } - fn as_any(&self) -> &dyn std::any::Any { - self - } - fn properties(&self) -> &Arc { self.input.properties() } @@ -1065,7 +1193,7 @@ impl ExecutionPlan for HardCapBatchSizeExec { fn partition_statistics( &self, partition: Option, - ) -> datafusion_common::Result { + ) -> datafusion_common::Result> { self.input.partition_statistics(partition) } diff --git a/rust/lance-datafusion/src/logical_expr.rs b/rust/lance-datafusion/src/logical_expr.rs index 0eed438dae7..db9abd7e204 100644 --- a/rust/lance-datafusion/src/logical_expr.rs +++ b/rust/lance-datafusion/src/logical_expr.rs @@ -51,14 +51,10 @@ pub fn resolve_column_type(expr: &Expr, schema: &Schema) -> Option { field_path.push(c.name.as_str()); break; } - Expr::ScalarFunction(udf) => { - if udf.name() == GetFieldFunc::default().name() { - let name = get_as_string_scalar_opt(&udf.args[1])?; - field_path.push(name); - current_expr = &udf.args[0]; - } else { - return None; - } + Expr::ScalarFunction(udf) if udf.name() == GetFieldFunc::default().name() => { + let name = get_as_string_scalar_opt(&udf.args[1])?; + field_path.push(name); + current_expr = &udf.args[0]; } _ => return None, } @@ -285,8 +281,10 @@ pub fn field_path_to_expr(field_path: &str) -> Result { ))); } - // Build the column expression, handling nested fields - let mut expr = col(&parts[0]); + // Build the column expression, handling nested fields. + let mut expr = Expr::Column(datafusion::common::Column::new_unqualified( + parts[0].clone(), + )); for part in &parts[1..] { expr = expr.field_newstyle(part); } @@ -301,8 +299,26 @@ mod tests { use super::*; use arrow_schema::{Field, Schema as ArrowSchema}; + use datafusion::common::Column; use datafusion_functions::core::expr_ext::FieldAccessor; + #[test] + fn test_field_path_to_expr_preserves_case_sensitive_root_column() { + let expr = field_path_to_expr("VECTOR").unwrap(); + + assert_eq!(expr, Expr::Column(Column::new_unqualified("VECTOR"))); + } + + #[test] + fn test_field_path_to_expr_preserves_case_sensitive_escaped_nested_path() { + let expr = field_path_to_expr("Parent.`Child.With.Dot`").unwrap(); + + assert_eq!( + expr, + Expr::Column(Column::new_unqualified("Parent")).field_newstyle("Child.With.Dot") + ); + } + #[test] fn test_resolve_large_utf8() { let arrow_schema = ArrowSchema::new(vec![Field::new("a", DataType::LargeUtf8, false)]); diff --git a/rust/lance-datafusion/src/planner.rs b/rust/lance-datafusion/src/planner.rs index 1e62cba42d8..5ee19ee2f49 100644 --- a/rust/lance-datafusion/src/planner.rs +++ b/rust/lance-datafusion/src/planner.rs @@ -58,6 +58,31 @@ fn encode_jsonb(json_str: &str) -> Result { Ok(Expr::Literal(ScalarValue::LargeBinary(Some(bytes)), None)) } +// The escape in `LIKE/ILIKE ... ESCAPE ''` must be exactly one character. +// Reject empty or multi-character escape strings rather than silently treating +// them as "no escape" or truncating to the first character. +fn parse_like_escape_char(escape_char: &Option) -> Result> { + let Some(value) = escape_char else { + return Ok(None); + }; + let ValueWithSpan { + value: Value::SingleQuotedString(escape), + .. + } = value + else { + return Err(Error::invalid_input(format!( + "Invalid escape character in LIKE expression. Expected a single character wrapped with single quotes, got {value}" + ))); + }; + let mut chars = escape.chars(); + match (chars.next(), chars.next()) { + (Some(c), None) => Ok(Some(c)), + _ => Err(Error::invalid_input(format!( + "Invalid escape character in LIKE expression. Expected a single character, got '{escape}'" + ))), + } +} + #[derive(Debug, Clone, Eq, PartialEq, Hash)] struct CastListF16Udf { signature: Signature, @@ -72,10 +97,6 @@ impl CastListF16Udf { } impl ScalarUDFImpl for CastListF16Udf { - fn as_any(&self) -> &dyn std::any::Any { - self - } - fn name(&self) -> &str { "_cast_list_f16" } @@ -197,6 +218,13 @@ impl ContextProvider for LanceContextProvider { self.state.window_functions().get(name).cloned() } + fn get_higher_order_meta( + &self, + name: &str, + ) -> Option> { + self.state.higher_order_functions().get(name).cloned() + } + fn get_function_meta(&self, f: &str) -> Option> { match f { // TODO: cast should go thru CAST syntax instead of UDF @@ -227,6 +255,14 @@ impl ContextProvider for LanceContextProvider { self.state.window_functions().keys().cloned().collect() } + fn higher_order_function_names(&self) -> Vec { + self.state + .higher_order_functions() + .keys() + .cloned() + .collect() + } + fn get_expr_planners(&self) -> &[Arc] { &self.expr_planners } @@ -451,6 +487,8 @@ impl Planner { }; if let Ok(n) = value.parse::() { Ok(lit(n)) + } else if let Ok(n) = value.parse::() { + Ok(lit(n)) } else { value.parse::().map(lit).map_err(|_| { Error::invalid_input(format!("'{value}' is not supported number value.")) @@ -503,7 +541,7 @@ impl Planner { } _ => Err(Error::invalid_input(format!( "Unsupported function args: {:?}", - &func.args + func.args ))), } } @@ -747,10 +785,10 @@ impl Planner { data_type, value, .. }) => { let value = value.clone().into_string().expect_ok()?; - Ok(Expr::Cast(datafusion::logical_expr::Cast { - expr: Box::new(Expr::Literal(ScalarValue::Utf8(Some(value)), None)), - data_type: self.parse_type(data_type)?, - })) + Ok(Expr::Cast(datafusion::logical_expr::Cast::new( + Box::new(Expr::Literal(ScalarValue::Utf8(Some(value)), None)), + self.parse_type(data_type)?, + ))) } SQLExpr::IsFalse(expr) => Ok(Expr::IsFalse(Box::new(self.parse_sql_expr(expr)?))), SQLExpr::IsNotFalse(expr) => Ok(Expr::IsNotFalse(Box::new(self.parse_sql_expr(expr)?))), @@ -782,16 +820,7 @@ impl Planner { *negated, Box::new(self.parse_sql_expr(expr)?), Box::new(self.parse_sql_expr(pattern)?), - match escape_char { - Some(Value::SingleQuotedString(char)) => char.chars().next(), - Some(value) => { - return Err(Error::invalid_input(format!( - "Invalid escape character in LIKE expression. Expected a single character wrapped with single quotes, got {}", - value - ))); - } - None => None, - }, + parse_like_escape_char(escape_char)?, true, ))), SQLExpr::Like { @@ -804,16 +833,7 @@ impl Planner { *negated, Box::new(self.parse_sql_expr(expr)?), Box::new(self.parse_sql_expr(pattern)?), - match escape_char { - Some(Value::SingleQuotedString(char)) => char.chars().next(), - Some(value) => { - return Err(Error::invalid_input(format!( - "Invalid escape character in LIKE expression. Expected a single character wrapped with single quotes, got {}", - value - ))); - } - None => None, - }, + parse_like_escape_char(escape_char)?, false, ))), // JSONB cast: CAST('...' AS JSONB) or '...'::jsonb @@ -838,15 +858,15 @@ impl Planner { } => match kind { datafusion::sql::sqlparser::ast::CastKind::TryCast | datafusion::sql::sqlparser::ast::CastKind::SafeCast => { - Ok(Expr::TryCast(datafusion::logical_expr::TryCast { - expr: Box::new(self.parse_sql_expr(expr)?), - data_type: self.parse_type(data_type)?, - })) + Ok(Expr::TryCast(datafusion::logical_expr::TryCast::new( + Box::new(self.parse_sql_expr(expr)?), + self.parse_type(data_type)?, + ))) } - _ => Ok(Expr::Cast(datafusion::logical_expr::Cast { - expr: Box::new(self.parse_sql_expr(expr)?), - data_type: self.parse_type(data_type)?, - })), + _ => Ok(Expr::Cast(datafusion::logical_expr::Cast::new( + Box::new(self.parse_sql_expr(expr)?), + self.parse_type(data_type)?, + ))), }, SQLExpr::JsonAccess { .. } => Err(Error::invalid_input("JSON access is not supported")), SQLExpr::CompoundFieldAccess { root, access_chain } => { @@ -991,9 +1011,10 @@ impl Planner { // DataFusion needs the coerce and simplify passes to be applied before // expressions can be handled by the physical planner. - let simplify_context = SimplifyContext::default() + let simplify_context = SimplifyContext::builder() .with_schema(df_schema.clone()) - .with_query_execution_start_time(Some(Utc::now())); + .with_query_execution_start_time(Some(Utc::now())) + .build(); let simplifier = datafusion::optimizer::simplify_expressions::ExprSimplifier::new(simplify_context); @@ -1062,13 +1083,9 @@ impl TreeNodeVisitor<'_> for ColumnCapturingVisitor { self.columns.insert(path); self.current_path.clear(); } - Expr::ScalarFunction(udf) => { - if udf.name() == GetFieldFunc::default().name() { - if let Some(name) = get_as_string_scalar_opt(&udf.args[1]) { - self.current_path.push_front(name.to_string()) - } else { - self.current_path.clear(); - } + Expr::ScalarFunction(udf) if udf.name() == GetFieldFunc::default().name() => { + if let Some(name) = get_as_string_scalar_opt(&udf.args[1]) { + self.current_path.push_front(name.to_string()) } else { self.current_path.clear(); } @@ -1084,7 +1101,6 @@ impl TreeNodeVisitor<'_> for ColumnCapturingVisitor { #[cfg(test)] mod tests { - use std::any::Any; use crate::logical_expr::ExprExt; @@ -1174,6 +1190,23 @@ mod tests { ); } + #[test] + fn test_parse_filter_uint64_literal_above_i64_max() { + let value = u64::MAX - 1; + let batch = arrow_array::record_batch!(("id", UInt64, [1, value])).unwrap(); + let planner = Planner::new(batch.schema()); + + let expr = planner.parse_filter(&format!("id = {value}")).unwrap(); + assert_eq!(expr, col("id").eq(lit(value))); + + let physical_expr = planner.create_physical_expr(&expr).unwrap(); + let predicates = physical_expr.evaluate(&batch).unwrap(); + assert_eq!( + predicates.into_array(0).unwrap().as_ref(), + &BooleanArray::from(vec![false, true]) + ); + } + #[test] fn test_parse_deep_logical_filter() { let planner = Planner::new(Arc::new(Schema::empty())); @@ -1204,10 +1237,6 @@ mod tests { } impl ScalarUDFImpl for StrictFloat64Udf { - fn as_any(&self) -> &dyn Any { - self - } - fn name(&self) -> &str { "strict_float64" } @@ -1453,6 +1482,36 @@ mod tests { ); } + #[test] + fn test_like_escape_char() { + let schema = Arc::new(Schema::new(vec![Field::new("s", DataType::Utf8, true)])); + let planner = Planner::new(schema); + + // A valid single-character escape is captured for both LIKE and ILIKE. + for filter in ["s LIKE 'a!%' ESCAPE '!'", "s ILIKE 'a!%' ESCAPE '!'"] { + match planner.parse_filter(filter).unwrap() { + Expr::Like(like) => assert_eq!(like.escape_char, Some('!'), "{filter}"), + other => panic!("expected a LIKE expression for `{filter}`, got {other:?}"), + } + } + + // Empty and multi-character escapes are rejected rather than silently + // dropped or truncated to the first character. + for filter in [ + "s LIKE 'x' ESCAPE ''", + "s LIKE 'x' ESCAPE 'ab'", + "s ILIKE 'x' ESCAPE ''", + "s ILIKE 'x' ESCAPE 'ab'", + ] { + let err = planner.parse_filter(filter).unwrap_err(); + assert!( + err.to_string() + .contains("Invalid escape character in LIKE expression"), + "unexpected error for `{filter}`: {err}" + ); + } + } + #[test] fn test_sql_is_in() { let schema = Arc::new(Schema::new(vec![Field::new("s", DataType::Utf8, true)])); @@ -1609,7 +1668,7 @@ mod tests { match expr { Expr::BinaryExpr(BinaryExpr { right, .. }) => match right.as_ref() { - Expr::Cast(Cast { expr, data_type }) => { + Expr::Cast(Cast { expr, field }) => { match expr.as_ref() { Expr::Literal(ScalarValue::Utf8(Some(value_str)), _) => { assert_eq!(value_str, expected_value_str); @@ -1619,7 +1678,7 @@ mod tests { } _ => panic!("Expected cast to be applied to literal"), } - assert_eq!(data_type, expected_data_type); + assert_eq!(field.data_type(), expected_data_type); } _ => panic!("Expected right to be a cast"), }, @@ -1660,14 +1719,14 @@ mod tests { match expr { Expr::BinaryExpr(BinaryExpr { right, .. }) => match right.as_ref() { - Expr::Cast(Cast { expr, data_type }) => { + Expr::Cast(Cast { expr, field }) => { match expr.as_ref() { Expr::Literal(ScalarValue::Utf8(Some(value_str)), _) => { assert_eq!(value_str, expected_value_str); } _ => panic!("Expected cast to be applied to literal"), } - assert_eq!(data_type, expected_data_type); + assert_eq!(field.data_type(), expected_data_type); } _ => panic!("Expected right to be a cast"), }, diff --git a/rust/lance-datafusion/src/projection.rs b/rust/lance-datafusion/src/projection.rs index 8a15b321e49..463e83a3011 100644 --- a/rust/lance-datafusion/src/projection.rs +++ b/rust/lance-datafusion/src/projection.rs @@ -24,11 +24,20 @@ use crate::{ planner::Planner, }; +const SCORING_COLUMNS: [&str; 2] = ["_distance", "_score"]; + +fn canonical_scoring_column(name: &str) -> Option<&'static str> { + SCORING_COLUMNS + .into_iter() + .find(|scoring_column| name.eq_ignore_ascii_case(scoring_column)) +} + struct ProjectionBuilder { base: Arc, planner: Planner, output: HashMap, output_cols: Vec, + scoring_exprs: HashMap, physical_cols_set: HashSet, physical_cols: Vec, needs_row_id: bool, @@ -50,6 +59,7 @@ impl ProjectionBuilder { planner, output: HashMap::default(), output_cols: Vec::default(), + scoring_exprs: HashMap::default(), physical_cols_set: HashSet::default(), physical_cols: Vec::default(), needs_row_id: false, @@ -75,10 +85,18 @@ impl ProjectionBuilder { self.check_duplicate_column(output_name)?; let expr = self.planner.parse_expr(raw_expr)?; - // Run simplification + coercion so that expressions like `coalesce(...)` - // (which DataFusion's physical evaluator expects to have been rewritten - // into a `CASE` expression by the simplifier) work correctly. - let expr = self.planner.optimize_expr(expr)?; + let expr = if Self::references_scoring_column(&expr) { + // A scoring name can refer to either a stored column or a search-generated + // Float32 column. Reparse and coerce once the physical input schema disambiguates it. + self.scoring_exprs + .insert(output_name.to_string(), raw_expr.to_string()); + expr + } else { + // Run simplification + coercion so that expressions like `coalesce(...)` + // (which DataFusion's physical evaluator expects to have been rewritten + // into a `CASE` expression by the simplifier) work correctly. + self.planner.optimize_expr(expr)? + }; // If the expression is a bare column reference to a system column, mark that we need it if let Expr::Column(Column { @@ -101,11 +119,23 @@ impl ProjectionBuilder { } for col in Planner::column_names_in_expr(&expr) { - if self.physical_cols_set.contains(&col) { + // Discovery can bind an exact provisional scoring field beside a mixed-case stored + // field. Load the stored field too so final-schema replanning can select the stored + // or search-generated field from the physical input. + let physical_col = if canonical_scoring_column(&col).is_some() { + self.base + .schema() + .field_case_insensitive(&col) + .map(|field| field.name.clone()) + .unwrap_or(col) + } else { + col + }; + if self.physical_cols_set.contains(&physical_col) { continue; } - self.physical_cols.push(col.clone()); - self.physical_cols_set.insert(col); + self.physical_cols.push(physical_col.clone()); + self.physical_cols_set.insert(physical_col); } self.output.insert(output_name.to_string(), expr.clone()); @@ -117,6 +147,12 @@ impl ProjectionBuilder { Ok(()) } + fn references_scoring_column(expr: &Expr) -> bool { + Planner::column_names_in_expr(expr) + .iter() + .any(|name| canonical_scoring_column(name).is_some()) + } + fn add_columns(&mut self, columns: &[(impl AsRef, impl AsRef)]) -> Result<()> { for (output_name, raw_expr) in columns { if raw_expr.as_ref() == WILDCARD { @@ -159,6 +195,7 @@ impl ProjectionBuilder { physical_projection, must_add_row_offset: self.must_add_row_offset, requested_output_expr: self.output_cols, + scoring_exprs: self.scoring_exprs, }) } } @@ -181,6 +218,9 @@ pub struct ProjectionPlan { /// The desired output columns pub requested_output_expr: Vec, + + /// Original SQL for scoring expressions that must be replanned against the physical schema. + scoring_exprs: HashMap, } impl ProjectionPlan { @@ -199,6 +239,14 @@ impl ProjectionPlan { fields.push(Arc::new( (*lance_core::ROW_CREATED_AT_VERSION_FIELD).clone(), )); + // Exact scoring fields are needed for initial parsing of schema-dependent functions, even + // beside a mixed-case stored field. The stored field is carried into the physical + // projection separately, and scoring expressions are replanned against the final schema. + for name in SCORING_COLUMNS { + if schema.field_with_name(name).is_err() { + fields.push(Arc::new(ArrowField::new(name, DataType::Float32, true))); + } + } ArrowSchema::new(fields) } @@ -312,6 +360,7 @@ impl ProjectionPlan { physical_projection, requested_output_expr: exprs, must_add_row_offset, + scoring_exprs: HashMap::default(), }) } @@ -338,6 +387,7 @@ impl ProjectionPlan { physical_projection, must_add_row_offset: false, requested_output_expr, + scoring_exprs: HashMap::default(), }) } @@ -352,9 +402,16 @@ impl ProjectionPlan { self.requested_output_expr .iter() .map(|output_column| { + let expr = if let Some(raw_expr) = self.scoring_exprs.get(&output_column.name) { + let planner = Planner::new(Arc::new(current_schema.clone())); + let expr = planner.parse_expr(raw_expr)?; + planner.optimize_expr(expr)? + } else { + output_column.expr.clone() + }; Ok(( datafusion::physical_expr::create_physical_expr( - &output_column.expr, + &expr, physical_df_schema.as_ref(), &Default::default(), )?, @@ -459,9 +516,166 @@ impl ProjectionPlan { mod tests { use super::*; - use arrow_array::Int64Array; + use arrow_array::{ArrayRef, Float32Array, Int64Array}; use lance_arrow::json::{is_json_field, json_field}; + #[test] + fn test_scoring_column_expression() { + for scoring_column in ["_distance", "_score"] { + for has_stored_column in [false, true] { + let base = if has_stored_column { + Arc::new( + Schema::try_from(&ArrowSchema::new(vec![ArrowField::new( + scoring_column, + DataType::Float64, + true, + )])) + .unwrap(), + ) + } else { + Arc::new(Schema::default()) + }; + let expression = format!("1 - {scoring_column}"); + let plan = + ProjectionPlan::from_expressions(base, &[("inverted", expression.as_str())]) + .unwrap(); + + if has_stored_column { + let stored_output = plan.output_schema().unwrap(); + assert_eq!(stored_output.field(0).data_type(), &DataType::Float64); + } + + let batch = RecordBatch::try_from_iter([( + scoring_column, + Arc::new(Float32Array::from(vec![0.25, 0.75])) as ArrayRef, + )]) + .unwrap(); + + let physical_exprs = plan.to_physical_exprs(batch.schema().as_ref()).unwrap(); + let values = physical_exprs[0] + .0 + .evaluate(&batch) + .unwrap() + .into_array(batch.num_rows()) + .unwrap(); + + assert_eq!( + values.as_ref(), + &Float32Array::from(vec![0.75, 0.25]), + "unexpected result for {scoring_column}", + ); + } + } + } + + #[test] + fn test_stored_scoring_column_does_not_break_other_expressions() { + for scoring_column in ["_distance", "_score"] { + let base = Arc::new( + Schema::try_from(&ArrowSchema::new(vec![ + ArrowField::new("id", DataType::Int64, false), + ArrowField::new(scoring_column, DataType::Float64, true), + ])) + .unwrap(), + ); + + ProjectionPlan::from_expressions(base, &[("incremented", "id + 1")]).unwrap(); + } + } + + #[test] + fn test_stored_scoring_column_is_case_insensitive() { + for (stored_name, requested_name) in [("_Distance", "_distance"), ("_Score", "_score")] { + let base = Arc::new( + Schema::try_from(&ArrowSchema::new(vec![ArrowField::new( + stored_name, + DataType::Float64, + true, + )])) + .unwrap(), + ); + let plan = + ProjectionPlan::from_expressions(base, &[("stored", requested_name)]).unwrap(); + + assert_eq!( + plan.output_schema().unwrap().field(0).data_type(), + &DataType::Float64, + ); + + let batch = RecordBatch::try_from_iter([( + requested_name, + Arc::new(Float32Array::from(vec![0.25, 0.75])) as ArrayRef, + )]) + .unwrap(); + let physical_exprs = plan.to_physical_exprs(batch.schema().as_ref()).unwrap(); + assert_eq!( + physical_exprs[0] + .0 + .data_type(batch.schema().as_ref()) + .unwrap(), + DataType::Float32, + ); + } + } + + #[test] + fn test_generated_scoring_function_with_mixed_case_stored_column() { + for (stored_name, generated_name) in [("_Distance", "_distance"), ("_Score", "_score")] { + let base = Arc::new( + Schema::try_from(&ArrowSchema::new(vec![ArrowField::new( + stored_name, + DataType::Float64, + true, + )])) + .unwrap(), + ); + let expression = format!("coalesce(1 - {generated_name}, 0)"); + let plan = + ProjectionPlan::from_expressions(base, &[("normalized", expression.as_str())]) + .unwrap(); + let batch = RecordBatch::try_from_iter([( + generated_name, + Arc::new(Float32Array::from(vec![Some(0.25), None])) as ArrayRef, + )]) + .unwrap(); + + let physical_exprs = plan.to_physical_exprs(batch.schema().as_ref()).unwrap(); + let values = physical_exprs[0] + .0 + .evaluate(&batch) + .unwrap() + .into_array(batch.num_rows()) + .unwrap(); + assert_eq!(values.as_ref(), &Float32Array::from(vec![0.75, 0.0])); + } + } + + #[test] + fn test_scoring_column_function_expression() { + for scoring_column in ["_distance", "_score"] { + let expression = format!("coalesce(1 - {scoring_column}, 0)"); + let plan = ProjectionPlan::from_expressions( + Arc::new(Schema::default()), + &[("normalized", expression.as_str())], + ) + .unwrap(); + let batch = RecordBatch::try_from_iter([( + scoring_column, + Arc::new(Float32Array::from(vec![Some(0.25), None])) as ArrayRef, + )]) + .unwrap(); + + let physical_exprs = plan.to_physical_exprs(batch.schema().as_ref()).unwrap(); + let values = physical_exprs[0] + .0 + .evaluate(&batch) + .unwrap() + .into_array(batch.num_rows()) + .unwrap(); + assert_eq!(values.as_ref(), &Float32Array::from(vec![0.75, 0.0])); + } + } + #[tokio::test] async fn test_coalesce_in_column_map() { // Regression test: `coalesce` in a column-map expression used to fail with diff --git a/rust/lance-datafusion/src/spill.rs b/rust/lance-datafusion/src/spill.rs index 8fa60c93ab6..749a9637fc5 100644 --- a/rust/lance-datafusion/src/spill.rs +++ b/rust/lance-datafusion/src/spill.rs @@ -9,13 +9,17 @@ use std::{ use arrow::ipc::{reader::StreamReader, writer::StreamWriter}; use arrow_array::RecordBatch; -use arrow_schema::{ArrowError, Schema}; +use arrow_schema::{ArrowError, Schema, SchemaRef}; use datafusion::{ - execution::SendableRecordBatchStream, physical_plan::stream::RecordBatchStreamAdapter, + catalog::{TableProvider, streaming::StreamingTable}, + execution::{SendableRecordBatchStream, TaskContext}, + physical_plan::{stream::RecordBatchStreamAdapter, streaming::PartitionStream}, }; use datafusion_common::DataFusionError; +use futures::StreamExt; use lance_arrow::memory::MemoryAccumulator; use lance_core::error::LanceOptionExt; +use lance_core::utils::tempfile::TempDir; /// Start a spill of Arrow data to a file that can be read later multiple times. /// @@ -60,6 +64,141 @@ pub fn create_replay_spill( (sender, receiver) } +/// Wrap a one-shot [`SendableRecordBatchStream`] in a re-scannable [`TableProvider`]. +/// +/// The source is drained in the background into a replayable spill. Two properties +/// keep this cheap for the common case: +/// +/// - **Memory-first.** Up to `memory_limit` bytes are buffered in memory; the spill +/// only touches disk once that budget is exceeded. A source that fits under the +/// limit never hits the filesystem. +/// - **Streaming replay.** A scan can start consuming batches as soon as they land, +/// before the source has finished draining — the first reader is not blocked +/// waiting for the whole source to buffer. +/// +/// Each scan of the returned provider replays the full source, which is what makes a +/// one-shot stream usable in the write retry loop. +/// +/// The provider reports no statistics — the source size is not known until it has +/// been fully drained — so callers that need source statistics (e.g. to drive join +/// ordering) should prefer a materialized or file-backed provider instead. +/// +/// # Examples +/// +/// ``` +/// # use std::sync::Arc; +/// # use arrow_array::{Int32Array, RecordBatch}; +/// # use arrow_schema::{DataType, Field, Schema}; +/// # use datafusion::execution::SendableRecordBatchStream; +/// # use datafusion::physical_plan::stream::RecordBatchStreamAdapter; +/// # use futures::TryStreamExt; +/// # use lance_datafusion::exec::provider_to_stream; +/// # use lance_datafusion::spill::spilling_table_provider; +/// # #[tokio::main] +/// # async fn main() -> Result<(), Box> { +/// let schema = Arc::new(Schema::new(vec![Field::new("a", DataType::Int32, false)])); +/// let batch = +/// RecordBatch::try_new(schema.clone(), vec![Arc::new(Int32Array::from(vec![1, 2, 3]))])?; +/// // A one-shot stream can only be consumed once. +/// let source: SendableRecordBatchStream = Box::pin(RecordBatchStreamAdapter::new( +/// schema.clone(), +/// futures::stream::iter(vec![Ok(batch)]), +/// )); +/// +/// // Wrapping it makes it re-scannable: each scan replays the full source. +/// let provider = spilling_table_provider(source, 100 * 1024 * 1024).await?; +/// let first: Vec = provider_to_stream(provider.clone()).await?.try_collect().await?; +/// let second: Vec = provider_to_stream(provider).await?.try_collect().await?; +/// assert_eq!(first.iter().map(|b| b.num_rows()).sum::(), 3); +/// assert_eq!(second.iter().map(|b| b.num_rows()).sum::(), 3); +/// # Ok(()) +/// # } +/// ``` +pub async fn spilling_table_provider( + mut source: SendableRecordBatchStream, + memory_limit: usize, +) -> Result, DataFusionError> { + let schema = source.schema(); + let tmp_dir = tokio::task::spawn_blocking(TempDir::try_new) + .await + .map_err(|e| DataFusionError::Execution(format!("Failed to spawn temp dir task: {e}")))? + .map_err(|e| DataFusionError::Execution(format!("Failed to create temp dir: {e}")))?; + let tmp_path = tmp_dir.std_path().join("spill.arrows"); + let (mut sender, receiver) = create_replay_spill(tmp_path, schema.clone(), memory_limit); + + // Drain the one-shot source into the spill once, in the background. The spill + // tees to memory/disk so the first reader can consume batches as they arrive + // while later readers replay the complete source. + let drain_handle = tokio::task::spawn(async move { + let mut errored = false; + while let Some(res) = source.next().await { + match res { + Ok(batch) => { + if let Err(e) = sender.write(batch).await { + sender.send_error(e); + errored = true; + break; + } + } + Err(e) => { + sender.send_error(e); + errored = true; + break; + } + } + } + // Only finish on a clean drain. Calling finish() after an error would + // overwrite the original (replayable) error with a generic one, losing + // the source error's type (e.g. an external error from user code). + if !errored && let Err(err) = sender.finish().await { + sender.send_error(err); + } + sender + }); + + let partition = Arc::new(SpillPartition { + schema: schema.clone(), + receiver, + _tmp_dir: Arc::new(tmp_dir), + _drain_handle: Arc::new(drain_handle), + }); + Ok(Arc::new(StreamingTable::try_new(schema, vec![partition])?)) +} + +/// A [`PartitionStream`] backed by a replayable spill. +/// +/// Each call to [`PartitionStream::execute`] opens a fresh stream over the spill, +/// so the partition can be scanned repeatedly. The spill file and the background +/// task draining the source are kept alive for as long as this partition exists. +struct SpillPartition { + schema: SchemaRef, + receiver: SpillReceiver, + // The spilled data lives in this temp dir; dropping it deletes the spill file. + _tmp_dir: Arc, + // Keeps the background drain task (which owns the `SpillSender`) alive. The + // `SpillSender` must outlive the readers or they error out, so we hold the + // handle rather than detaching it. + _drain_handle: Arc>, +} + +impl std::fmt::Debug for SpillPartition { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("SpillPartition") + .field("schema", &self.schema) + .finish() + } +} + +impl PartitionStream for SpillPartition { + fn schema(&self) -> &SchemaRef { + &self.schema + } + + fn execute(&self, _ctx: Arc) -> SendableRecordBatchStream { + self.receiver.read() + } +} + #[derive(Clone)] pub struct SpillReceiver { status_receiver: tokio::sync::watch::Receiver, @@ -250,7 +389,7 @@ impl Default for DataLocation { } } -/// A DataFusion error that be be emitted multiple times. We provide the +/// A DataFusion error that can be emitted multiple times. We provide the /// Original error first, and subsequent conversions provide a copy with a /// string representation of the original error. #[derive(Debug)] diff --git a/rust/lance-datafusion/src/substrait.rs b/rust/lance-datafusion/src/substrait.rs index 1c465fcae4a..f38e6b79cf9 100644 --- a/rust/lance-datafusion/src/substrait.rs +++ b/rust/lance-datafusion/src/substrait.rs @@ -164,6 +164,9 @@ fn remap_expr_references(expr: &mut Expression, mapping: &HashMap) RexType::WindowFunction(_) | RexType::Subquery(_) => Err(Error::invalid_input( "Window functions or subqueries not allowed in filter expression", )), + RexType::Lambda(_) | RexType::LambdaInvocation(_) => Err(Error::invalid_input( + "Lambda expressions not allowed in filter expression", + )), // Pass through operators, nested children may have field references RexType::ScalarFunction(func) => { #[allow(deprecated)] @@ -551,7 +554,7 @@ mod tests { }, expression_reference::ExprType, extensions::{ - SimpleExtensionDeclaration, SimpleExtensionUri, SimpleExtensionUrn, + SimpleExtensionDeclaration, SimpleExtensionUrn, simple_extension_declaration::{ExtensionFunction, MappingType}, }, function_argument::ArgType, @@ -576,13 +579,6 @@ mod tests { git_hash: "".to_string(), producer: "unit-test".to_string(), }), - #[expect(deprecated)] - extension_uris: vec![ - SimpleExtensionUri { - extension_uri_anchor: 1, - uri: "https://github.com/substrait-io/substrait/blob/main/extensions/functions_comparison.yaml".to_string(), - } - ], extension_urns: vec![ SimpleExtensionUrn { extension_urn_anchor: 1, @@ -592,8 +588,6 @@ mod tests { extensions: vec![ SimpleExtensionDeclaration { mapping_type: Some(MappingType::ExtensionFunction(ExtensionFunction { - #[expect(deprecated)] - extension_uri_reference: 1, extension_urn_reference: 1, function_anchor: 1, name: "lt".to_string(), @@ -881,9 +875,7 @@ mod tests { fn agg_extension(anchor: u32, name: &str) -> SimpleExtensionDeclaration { SimpleExtensionDeclaration { mapping_type: Some(MappingType::ExtensionFunction(ExtensionFunction { - #[allow(deprecated)] - extension_uri_reference: 1, - extension_urn_reference: 0, + extension_urn_reference: 1, function_anchor: anchor, name: name.to_string(), })), @@ -919,10 +911,9 @@ mod tests { git_hash: String::new(), producer: "lance-test".to_string(), }), - #[allow(deprecated)] - extension_uris: vec![SimpleExtensionUri { - extension_uri_anchor: 1, - uri: "https://github.com/substrait-io/substrait/blob/main/extensions/functions_aggregate_generic.yaml".to_string(), + extension_urns: vec![SimpleExtensionUrn { + extension_urn_anchor: 1, + urn: "https://github.com/substrait-io/substrait/blob/main/extensions/functions_aggregate_generic.yaml".to_string(), }], extensions, relations: vec![PlanRel { @@ -935,7 +926,6 @@ mod tests { }], advanced_extensions: None, expected_type_urls: vec![], - extension_urns: vec![], parameter_bindings: vec![], type_aliases: vec![], }; diff --git a/rust/lance-datafusion/src/utils.rs b/rust/lance-datafusion/src/utils.rs index 470831bf0bc..e660c8ee44d 100644 --- a/rust/lance-datafusion/src/utils.rs +++ b/rust/lance-datafusion/src/utils.rs @@ -244,6 +244,8 @@ pub const INDICES_LOADED_METRIC: &str = "indices_loaded"; pub const PARTS_LOADED_METRIC: &str = "parts_loaded"; pub const PARTITIONS_RANKED_METRIC: &str = "partitions_ranked"; pub const INDEX_COMPARISONS_METRIC: &str = "index_comparisons"; +pub const INDEX_CACHE_HITS_METRIC: &str = "index_cache_hits"; +pub const INDEX_CACHE_MISSES_METRIC: &str = "index_cache_misses"; pub const FRAGMENTS_SCANNED_METRIC: &str = "fragments_scanned"; pub const RANGES_SCANNED_METRIC: &str = "ranges_scanned"; pub const ROWS_SCANNED_METRIC: &str = "rows_scanned"; diff --git a/rust/lance-datagen/src/generator.rs b/rust/lance-datagen/src/generator.rs index 39da4734619..34fe1202eb7 100644 --- a/rust/lance-datagen/src/generator.rs +++ b/rust/lance-datagen/src/generator.rs @@ -569,7 +569,19 @@ where }; self.leftover_count = ((self.leftover_count as u64 + length.0) % self.repeat as u64) as u32; self.leftover = values.last().copied().unwrap_or(T::default()); - Ok(Arc::new(ArrayType::from(values))) + let array = ArrayType::from(values); + // `ArrayType::from` uses the primitive type's default metadata. For + // timezone-aware timestamps this drops the timezone, so restore the + // generator's declared type when it differs. + if array.data_type() == &self.data_type { + return Ok(Arc::new(array)); + } + let data = array + .into_data() + .into_builder() + .data_type(self.data_type.clone()) + .build()?; + Ok(make_array(data)) } fn data_type(&self) -> &DataType { @@ -2818,14 +2830,29 @@ pub mod array { Box::new(RandomIntervalGenerator::new(unit)) } + /// The default sampling range for temporal generators: the 365 days ending at + /// 2024-01-01T00:00:00Z (exclusive) + /// + /// The range must be a fixed anchor and not derived from the wall clock + /// (e.g. `Utc::now()`), otherwise the same RNG seed would generate different + /// values depending on when the generator was created, breaking + /// reproducibility (e.g. of saved fuzz inputs). Callers that need a + /// time-relative range can use the `*_in_range` variants. + fn default_temporal_range() -> (chrono::DateTime, chrono::DateTime) { + let end = chrono::DateTime::::from_timestamp(1_704_067_200, 0) + .expect("2024-01-01T00:00:00Z is a valid timestamp"); + let start = end - chrono::TimeDelta::try_days(365).expect("TimeDelta try_days"); + (start, end) + } + /// Create a generator of randomly sampled date32 values /// - /// Instead of sampling the entire range, all values will be drawn from the last year as this - /// is a more common use pattern + /// Instead of sampling the entire range, all values will be drawn from a fixed + /// one-year range (the 365 days ending at 2024-01-01 UTC) as this is a more + /// common use pattern. Use [`rand_date32_in_range`] to control the range. pub fn rand_date32() -> Box { - let now = chrono::Utc::now(); - let one_year_ago = now - chrono::TimeDelta::try_days(365).expect("TimeDelta try days"); - rand_date32_in_range(one_year_ago, now) + let (start, end) = default_temporal_range(); + rand_date32_in_range(start, end) } /// Create a generator of randomly sampled date32 values in the given range @@ -2853,12 +2880,12 @@ pub mod array { /// Create a generator of randomly sampled date64 values /// - /// Instead of sampling the entire range, all values will be drawn from the last year as this - /// is a more common use pattern + /// Instead of sampling the entire range, all values will be drawn from a fixed + /// one-year range (the 365 days ending at 2024-01-01 UTC) as this is a more + /// common use pattern. Use [`rand_date64_in_range`] to control the range. pub fn rand_date64() -> Box { - let now = chrono::Utc::now(); - let one_year_ago = now - chrono::TimeDelta::try_days(365).expect("TimeDelta try_days"); - rand_date64_in_range(one_year_ago, now) + let (start, end) = default_temporal_range(); + rand_date64_in_range(start, end) } /// Create a generator of randomly sampled timestamp values in the given range @@ -2914,10 +2941,14 @@ pub mod array { } } + /// Create a generator of randomly sampled timestamp values + /// + /// Instead of sampling the entire range, all values will be drawn from a fixed + /// one-year range (the 365 days ending at 2024-01-01 UTC) as this is a more + /// common use pattern. Use [`rand_timestamp_in_range`] to control the range. pub fn rand_timestamp(data_type: &DataType) -> Box { - let now = chrono::Utc::now(); - let one_year_ago = now - chrono::Duration::try_days(365).unwrap(); - rand_timestamp_in_range(one_year_ago, now, data_type) + let (start, end) = default_temporal_range(); + rand_timestamp_in_range(start, end, data_type) } /// Create a generator of randomly sampled date64 values @@ -3214,11 +3245,40 @@ pub fn rand(schema: &Schema) -> BatchGeneratorBuilder { #[cfg(test)] mod tests { - use arrow::datatypes::{Float32Type, Int8Type, Int16Type, UInt32Type}; - use arrow_array::{BooleanArray, Float32Array, Int8Array, Int16Array, Int32Array, UInt32Array}; + use arrow::datatypes::{Float32Type, Int8Type, Int16Type, TimeUnit, UInt32Type}; + use arrow_array::{ + BooleanArray, Date32Array, Date64Array, Float32Array, Int8Array, Int16Array, Int32Array, + TimestampMicrosecondArray, TimestampMillisecondArray, TimestampNanosecondArray, + TimestampSecondArray, UInt32Array, + }; use super::*; + #[test] + fn test_timestamp_timezone_is_preserved() { + let data_type = DataType::Timestamp(TimeUnit::Millisecond, Some("UTC".into())); + let mut generator = array::rand_type(&data_type); + let generated = generator.generate_default(RowCount::from(2)).unwrap(); + assert_eq!(generated.data_type(), &data_type); + + let fields = Fields::from(vec![Field::new("timestamp", data_type, true)]); + let mut generator = array::rand_struct(fields.clone()); + let generated = generator.generate_default(RowCount::from(2)).unwrap(); + assert_eq!(generated.data_type(), &DataType::Struct(fields)); + } + + #[test] + fn test_fn_gen_propagates_array_data_build_error() { + // FnGen constructors are internal. Use an incompatible declared type to + // verify that ArrayDataBuilder validation failures are propagated. + let mut generator = FnGen::::new_unknown_size(DataType::Utf8, |_| 0, 1); + + assert!(matches!( + generator.generate_default(RowCount::from(1)), + Err(ArrowError::InvalidArgumentError(_)) + )); + } + #[test] fn test_step() { let mut rng = rand_xoshiro::Xoshiro256PlusPlus::seed_from_u64(DEFAULT_SEED.0); @@ -3395,6 +3455,72 @@ mod tests { ); } + #[test] + fn test_rng_temporal_deterministic() { + // The default temporal generators must not depend on the wall clock: the + // same seed must produce the same values no matter when the generator is + // created (https://github.com/lance-format/lance/issues/7913). These + // exact values pin both the RNG stream and the fixed default sampling + // range (the 365 days ending at 2024-01-01 UTC). + fn gen_values(mut genn: Box) -> Arc { + let mut rng = rand_xoshiro::Xoshiro256PlusPlus::seed_from_u64(DEFAULT_SEED.0); + genn.generate(RowCount::from(3), &mut rng).unwrap() + } + + assert_eq!( + *gen_values(array::rand_date32()), + Date32Array::from(vec![19655, 19474, 19717]) + ); + assert_eq!( + *gen_values(array::rand_date64()), + Date64Array::from(vec![ + 1_698_192_000_000, + 1_682_553_600_000, + 1_703_548_800_000 + ]) + ); + assert_eq!( + *gen_values(array::rand_timestamp(&DataType::Timestamp( + TimeUnit::Second, + None + ))), + TimestampSecondArray::from(vec![1_698_211_127, 1_682_585_540, 1_703_559_286]) + ); + assert_eq!( + *gen_values(array::rand_timestamp(&DataType::Timestamp( + TimeUnit::Millisecond, + None + ))), + TimestampMillisecondArray::from(vec![ + 1_698_211_127_056, + 1_682_585_540_319, + 1_703_559_286_487 + ]) + ); + assert_eq!( + *gen_values(array::rand_timestamp(&DataType::Timestamp( + TimeUnit::Microsecond, + None + ))), + TimestampMicrosecondArray::from(vec![ + 1_698_211_127_056_596, + 1_682_585_540_319_384, + 1_703_559_286_487_645 + ]) + ); + assert_eq!( + *gen_values(array::rand_timestamp(&DataType::Timestamp( + TimeUnit::Nanosecond, + None + ))), + TimestampNanosecondArray::from(vec![ + 1_698_211_127_056_596_085, + 1_682_585_540_319_384_548, + 1_703_559_286_487_645_287 + ]) + ); + } + #[test] fn test_rng_list() { // Note: these tests are heavily dependent on the default seed. diff --git a/rust/lance-encoding/Cargo.toml b/rust/lance-encoding/Cargo.toml index df5352262fe..d14227f5872 100644 --- a/rust/lance-encoding/Cargo.toml +++ b/rust/lance-encoding/Cargo.toml @@ -32,7 +32,6 @@ num-traits.workspace = true prost.workspace = true hyperloglogplus.workspace = true rand.workspace = true -strum = { workspace =true, features = ["derive"] } tokio.workspace = true tracing.workspace = true xxhash-rust = { version = "0.8.15", features = ["xxh3"] } diff --git a/rust/lance-encoding/benches/common/mod.rs b/rust/lance-encoding/benches/common/mod.rs new file mode 100644 index 00000000000..3539fdd90b4 --- /dev/null +++ b/rust/lance-encoding/benches/common/mod.rs @@ -0,0 +1,278 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright The Lance Authors + +use std::sync::Arc; + +use arrow_schema::DataType; +use lance_core::{Error, Result, datatypes::Field}; +use lance_encoding::{ + compression::{ + BlockCompressor, CompressionStrategy, field_metadata_params, finalize_miniblock_compressor, + reject_packed_struct_per_value, try_bitpacking_block, try_bitpacking_miniblock, + try_byte_stream_split_miniblock, try_fixed_packed_struct_miniblock, try_fixed_u8_rle_block, + try_fixed_u8_rle_miniblock, try_general_block, try_raw_block, + try_raw_fixed_size_list_miniblock, try_raw_fixed_width_miniblock, try_raw_per_value, + try_uncompressed_fixed_width_miniblock, try_variable_packed_struct_per_value, + try_variable_width_miniblock, try_variable_width_per_value, + }, + compression_config::{CompressionFieldParams, CompressionParams}, + data::DataBlock, + encoder::{ + ColumnIndexSequence, FieldEncoder, FieldEncodingContext, FieldEncodingStrategy, + structural::{ + PrimitiveFieldEncoding, PrimitivePageEncoding, try_create_binary_blob, try_create_list, + try_create_map, try_create_struct, try_create_structural_blob, + try_create_structural_fixed_size_list, + }, + }, + encodings::logical::primitive::{fullzip::PerValueCompressor, miniblock::MiniBlockCompressor}, + format::pb21::CompressiveEncoding, +}; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum BenchEncoding { + Array, + StructuralU16, + StructuralU32, +} + +impl std::fmt::Display for BenchEncoding { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str(match self { + Self::Array => "array", + Self::StructuralU16 => "structural-u16", + Self::StructuralU32 => "structural-u32", + }) + } +} + +#[derive(Debug, Clone)] +struct BenchCompressionStrategy { + encoding: BenchEncoding, + params: CompressionParams, +} + +impl BenchCompressionStrategy { + fn field_params(&self, field: &Field) -> CompressionFieldParams { + let mut params = self + .params + .get_field_params(&field.name, &field.data_type()); + let mut metadata = field_metadata_params(field); + if self.encoding == BenchEncoding::StructuralU16 + && metadata + .minichunk_size + .is_some_and(|size| size >= 32 * 1024) + { + metadata.minichunk_size = None; + } + params.merge(&metadata); + params + } +} + +impl CompressionStrategy for BenchCompressionStrategy { + fn create_miniblock_compressor( + &self, + field: &Field, + data: &DataBlock, + ) -> Result> { + let params = self.field_params(field); + let compressor = + if let Some(compressor) = try_uncompressed_fixed_width_miniblock(data, ¶ms) { + compressor + } else if let Some(compressor) = try_byte_stream_split_miniblock(data, ¶ms) { + compressor + } else if let Some(compressor) = try_fixed_u8_rle_miniblock(data, ¶ms) { + compressor + } else if let Some(compressor) = try_bitpacking_miniblock(data) { + compressor + } else if let Some(compressor) = try_raw_fixed_width_miniblock(data) { + compressor + } else if let Some(compressor) = try_variable_width_miniblock(field, data, ¶ms)? { + compressor + } else if let Some(compressor) = try_fixed_packed_struct_miniblock(data)? { + compressor + } else if let Some(compressor) = try_raw_fixed_size_list_miniblock(data) { + compressor + } else { + return Err(Error::not_supported_source( + format!( + "Mini-block compression not yet supported for block type {}", + data.name() + ) + .into(), + )); + }; + finalize_miniblock_compressor(data, compressor, ¶ms) + } + + fn create_per_value( + &self, + field: &Field, + data: &DataBlock, + ) -> Result> { + let params = self.field_params(field); + if let Some(compressor) = try_raw_per_value(data) { + return Ok(compressor); + } + let packed = match self.encoding { + BenchEncoding::StructuralU16 => reject_packed_struct_per_value(field, data)?, + BenchEncoding::StructuralU32 => { + try_variable_packed_struct_per_value(Arc::new(self.clone()), field, data)? + } + BenchEncoding::Array => unreachable!(), + }; + if let Some(compressor) = packed { + return Ok(compressor); + } + if let Some(compressor) = try_variable_width_per_value(field, data, ¶ms)? { + return Ok(compressor); + } + Err(Error::not_supported_source( + format!( + "Per-value compression not yet supported for block type {}", + data.name() + ) + .into(), + )) + } + + fn create_block_compressor( + &self, + field: &Field, + data: &DataBlock, + ) -> Result<(Box, CompressiveEncoding)> { + let params = self.field_params(field); + if self.encoding == BenchEncoding::StructuralU32 + && let Some(compressor) = try_fixed_u8_rle_block(data, ¶ms)? + { + return Ok(compressor); + } + if let Some(compressor) = try_bitpacking_block(data) { + return Ok(compressor); + } + if self.encoding == BenchEncoding::StructuralU32 + && let Some(compressor) = try_general_block(data, ¶ms)? + { + return Ok(compressor); + } + if let Some(compressor) = try_raw_block(data) { + return Ok(compressor); + } + Err(Error::not_supported_source( + format!( + "Block compression not yet supported for block type {}", + data.name() + ) + .into(), + )) + } +} + +#[derive(Debug)] +struct BenchFieldEncodingStrategy { + encoding: BenchEncoding, + primitive: PrimitiveFieldEncoding, +} + +impl FieldEncodingStrategy for BenchFieldEncodingStrategy { + fn create_field_encoder( + &self, + field: &Field, + column_index: &mut ColumnIndexSequence, + context: &FieldEncodingContext<'_>, + ) -> Result> { + if let Some(encoder) = + try_create_binary_blob(&self.primitive, field, column_index, context)? + { + return Ok(encoder); + } + if self.encoding == BenchEncoding::StructuralU32 + && let Some(encoder) = + try_create_structural_blob(&self.primitive, field, column_index, context)? + { + return Ok(encoder); + } + if field.is_blob() { + return Err(Error::invalid_input_source( + format!( + "Blob encoding is not available for field '{}' with data type {}", + field.name, + field.data_type() + ) + .into(), + )); + } + if self.encoding == BenchEncoding::StructuralU32 { + if let Some(encoder) = try_create_map(field, column_index, context)? { + return Ok(encoder); + } + if let Some(encoder) = + try_create_structural_fixed_size_list(field, column_index, context)? + { + return Ok(encoder); + } + } + if let Some(encoder) = self.primitive.try_create(field, column_index, context)? { + return Ok(encoder); + } + if self.encoding == BenchEncoding::StructuralU16 { + if matches!( + field.data_type(), + DataType::FixedSizeList(item, _) + if matches!(item.data_type(), DataType::Struct(_)) + ) { + return Err(Error::not_supported_source( + "FixedSizeList is not enabled by the selected file format".into(), + )); + } + if matches!(field.data_type(), DataType::Map(_, _)) { + return Err(Error::not_supported_source( + "Map data type is not enabled by the selected file format".into(), + )); + } + } + if let Some(encoder) = try_create_list(field, column_index, context)? { + return Ok(encoder); + } + if let Some(encoder) = try_create_struct(field, column_index, context)? { + return Ok(encoder); + } + Err(Error::not_supported_source( + format!( + "{} has no field encoding for '{}' with data type {}", + self.encoding, + field.name, + field.data_type() + ) + .into(), + )) + } +} + +pub fn encoding_strategy(encoding: BenchEncoding) -> Box { + if encoding == BenchEncoding::Array { + return Box::new(lance_encoding::encoder::ArrayFieldEncodingStrategy::new()); + } + + let compression = Arc::new(BenchCompressionStrategy { + encoding, + params: CompressionParams::default(), + }); + let page_encodings = match encoding { + BenchEncoding::StructuralU16 => vec![ + PrimitivePageEncoding::reject_sparse(), + PrimitivePageEncoding::dense_u16(compression), + ], + BenchEncoding::StructuralU32 => vec![ + PrimitivePageEncoding::reject_sparse(), + PrimitivePageEncoding::constant(), + PrimitivePageEncoding::dense_u32(compression), + ], + BenchEncoding::Array => unreachable!(), + }; + Box::new(BenchFieldEncodingStrategy { + encoding, + primitive: PrimitiveFieldEncoding::new(page_encodings), + }) +} diff --git a/rust/lance-encoding/benches/decoder.rs b/rust/lance-encoding/benches/decoder.rs index cc0404e1bb3..98d7caaf9e2 100644 --- a/rust/lance-encoding/benches/decoder.rs +++ b/rust/lance-encoding/benches/decoder.rs @@ -1,25 +1,42 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright The Lance Authors -use std::{collections::HashMap, sync::Arc}; +use std::{collections::HashMap, hint::black_box, sync::Arc}; use arrow_array::{RecordBatch, UInt32Array}; +#[cfg(feature = "bitpacking")] +use arrow_buffer::ArrowNativeType; use arrow_schema::{DataType, Field, Schema, TimeUnit}; use arrow_select::take::take; +#[cfg(feature = "bitpacking")] +use bytemuck::Pod; use criterion::{Criterion, criterion_group, criterion_main}; use futures::StreamExt; +#[cfg(feature = "bitpacking")] +use lance_bitpacking::BitPacking; use lance_core::cache::LanceCache; use lance_datagen::ArrayGeneratorExt; +#[cfg(feature = "bitpacking")] +use lance_encoding::buffer::LanceBuffer; +#[cfg(feature = "bitpacking")] +use lance_encoding::compression::BlockDecompressor; +#[cfg(feature = "bitpacking")] +use lance_encoding::data::{BlockInfo, DataBlock, FixedWidthDataBlock}; +#[cfg(feature = "bitpacking")] +use lance_encoding::encodings::physical::bitpacking::{ELEMS_PER_CHUNK, InlineBitpacking}; use lance_encoding::{ decoder::{ - DecodeBatchScheduler, DecoderConfig, DecoderPlugins, FilterExpression, create_decode_stream, + DecodeBatchScheduler, DecoderConfig, DecoderPlugins, EncodedBatchLayout, FilterExpression, + create_decode_stream, }, - encoder::{EncodingOptions, default_encoding_strategy, encode_batch}, - version::LanceFileVersion, + encoder::{EncodingOptions, encode_batch}, }; use tokio::sync::mpsc::unbounded_channel; use rand::Rng; +pub mod common; +use common::{BenchEncoding, encoding_strategy}; + const PRIMITIVE_TYPES: &[DataType] = &[ DataType::Date32, DataType::Date64, @@ -49,6 +66,15 @@ const PRIMITIVE_TYPES: &[DataType] = &[ // schema doesn't yet parse them in the context of a fixed size list. const PRIMITIVE_TYPES_FOR_FSL: &[DataType] = &[DataType::Int8, DataType::Float32]; +fn encoded_batch_layout(encoding: BenchEncoding) -> EncodedBatchLayout { + match encoding { + BenchEncoding::Array => EncodedBatchLayout::Array, + BenchEncoding::StructuralU16 | BenchEncoding::StructuralU32 => { + EncodedBatchLayout::Structural + } + } +} + fn bench_decode(c: &mut Criterion) { let rt = tokio::runtime::Runtime::new().unwrap(); let mut group = c.benchmark_group("decode_primitive"); @@ -64,7 +90,7 @@ fn bench_decode(c: &mut Criterion) { .unwrap(); let lance_schema = Arc::new(lance_core::datatypes::Schema::try_from(data.schema().as_ref()).unwrap()); - let encoding_strategy = default_encoding_strategy(LanceFileVersion::default()); + let encoding_strategy = encoding_strategy(BenchEncoding::StructuralU16); let encoded = rt .block_on(encode_batch( &data, @@ -81,7 +107,7 @@ fn bench_decode(c: &mut Criterion) { &FilterExpression::no_filter(), Arc::::default(), false, - LanceFileVersion::default(), + EncodedBatchLayout::Structural, Some(Arc::new(LanceCache::no_cache())), )) .unwrap(); @@ -95,14 +121,14 @@ fn bench_decode_fsl(c: &mut Criterion) { let rt = tokio::runtime::Runtime::new().unwrap(); let mut group = c.benchmark_group("decode_fsl"); const NUM_BYTES: u64 = 1024 * 1024 * 128; - for version in [ - LanceFileVersion::V2_0, - LanceFileVersion::V2_1, - LanceFileVersion::V2_2, + for encoding in [ + BenchEncoding::Array, + BenchEncoding::StructuralU16, + BenchEncoding::StructuralU32, ] { for data_type in PRIMITIVE_TYPES_FOR_FSL { for dimension in [4, 16, 32, 64, 128] { - let nullable_choices: &[bool] = if version == LanceFileVersion::V2_0 { + let nullable_choices: &[bool] = if encoding == BenchEncoding::Array { &[false] } else { &[false, true] @@ -110,7 +136,7 @@ fn bench_decode_fsl(c: &mut Criterion) { for nullable in nullable_choices { let func_name = format!( "{:?}_{}_v{}_null{}", - data_type, dimension, version, nullable + data_type, dimension, encoding, nullable ) .to_lowercase(); group.throughput(criterion::Throughput::Bytes(NUM_BYTES)); @@ -133,7 +159,7 @@ fn bench_decode_fsl(c: &mut Criterion) { lance_core::datatypes::Schema::try_from(data.schema().as_ref()) .unwrap(), ); - let encoding_strategy = default_encoding_strategy(version); + let encoding_strategy = encoding_strategy(encoding); let encoded = rt .block_on(encode_batch( &data, @@ -149,7 +175,7 @@ fn bench_decode_fsl(c: &mut Criterion) { &FilterExpression::no_filter(), Arc::::default(), false, - version, + encoded_batch_layout(encoding), Some(Arc::new(LanceCache::no_cache())), )) .unwrap(); @@ -199,7 +225,7 @@ fn bench_decode_str_with_dict_encoding(c: &mut Criterion) { let lance_schema = Arc::new(lance_core::datatypes::Schema::try_from(data.schema().as_ref()).unwrap()); - let encoding_strategy = default_encoding_strategy(LanceFileVersion::default()); + let encoding_strategy = encoding_strategy(BenchEncoding::StructuralU16); let encoded = rt .block_on(encode_batch( &data, @@ -215,7 +241,7 @@ fn bench_decode_str_with_dict_encoding(c: &mut Criterion) { &FilterExpression::no_filter(), Arc::::default(), false, - LanceFileVersion::default(), + EncodedBatchLayout::Structural, Some(Arc::new(LanceCache::no_cache())), )) .unwrap(); @@ -274,7 +300,7 @@ fn bench_decode_packed_struct(c: &mut Criterion) { RecordBatch::try_new(Arc::new(new_schema.clone()), data.columns().to_vec()).unwrap(); let lance_schema = Arc::new(lance_core::datatypes::Schema::try_from(&new_schema).unwrap()); - let encoding_strategy = default_encoding_strategy(LanceFileVersion::V2_2); + let encoding_strategy = encoding_strategy(BenchEncoding::StructuralU32); let encoded = rt .block_on(encode_batch( &data, @@ -291,7 +317,7 @@ fn bench_decode_packed_struct(c: &mut Criterion) { &FilterExpression::no_filter(), Arc::::default(), false, - LanceFileVersion::V2_2, + EncodedBatchLayout::Structural, Some(Arc::new(LanceCache::no_cache())), )) .unwrap(); @@ -331,7 +357,7 @@ fn bench_decode_str_with_fixed_size_binary_encoding(c: &mut Criterion) { let lance_schema = Arc::new(lance_core::datatypes::Schema::try_from(data.schema().as_ref()).unwrap()); - let encoding_strategy = default_encoding_strategy(LanceFileVersion::default()); + let encoding_strategy = encoding_strategy(BenchEncoding::StructuralU16); let encoded = rt .block_on(encode_batch( &data, @@ -347,7 +373,7 @@ fn bench_decode_str_with_fixed_size_binary_encoding(c: &mut Criterion) { &FilterExpression::no_filter(), Arc::::default(), false, - LanceFileVersion::default(), + EncodedBatchLayout::Structural, Some(Arc::new(LanceCache::no_cache())), )) .unwrap(); @@ -398,7 +424,7 @@ fn bench_decode_compressed(c: &mut Criterion) { let lance_schema = Arc::new(lance_core::datatypes::Schema::try_from(schema.as_ref()).unwrap()); // V2_2+ required for general compression - let encoding_strategy = default_encoding_strategy(LanceFileVersion::V2_2); + let encoding_strategy = encoding_strategy(BenchEncoding::StructuralU32); // Encode once during setup let encoded = rt @@ -423,7 +449,7 @@ fn bench_decode_compressed(c: &mut Criterion) { &FilterExpression::no_filter(), Arc::::default(), false, - LanceFileVersion::V2_2, + EncodedBatchLayout::Structural, Some(Arc::new(LanceCache::no_cache())), )) .unwrap(); @@ -476,7 +502,7 @@ fn bench_decode_compressed_parallel(c: &mut Criterion) { let lance_schema = Arc::new(lance_core::datatypes::Schema::try_from(schema.as_ref()).unwrap()); - let encoding_strategy = default_encoding_strategy(LanceFileVersion::V2_2); + let encoding_strategy = encoding_strategy(BenchEncoding::StructuralU32); let encoded = rt .block_on(encode_batch( @@ -563,6 +589,136 @@ fn bench_decode_compressed_parallel(c: &mut Criterion) { } } +#[cfg(feature = "bitpacking")] +fn make_inline_bitpacking_chunk(bit_width: usize) -> LanceBuffer +where + T: ArrowNativeType + BitPacking + Pod, +{ + let value_range = 1_usize << bit_width; + let values: Vec = (0..ELEMS_PER_CHUNK as usize) + .map(|i| T::from_usize((i * 31 + 7) % value_range).unwrap()) + .collect(); + let packed_words = ELEMS_PER_CHUNK as usize * bit_width / (std::mem::size_of::() * 8); + + let mut chunk = Vec::with_capacity(1 + packed_words); + chunk.push(T::from_usize(bit_width).unwrap()); + let payload_start = chunk.len(); + chunk.resize(payload_start + packed_words, T::from_usize(0).unwrap()); + unsafe { + BitPacking::unchecked_pack(bit_width, &values, &mut chunk[payload_start..]); + } + + LanceBuffer::reinterpret_vec(chunk) +} + +#[cfg(feature = "bitpacking")] +fn read_little_endian_header(bytes: &[u8]) -> usize { + bytes[..std::mem::size_of::()] + .iter() + .enumerate() + .fold(0_u64, |value, (idx, byte)| { + value | ((*byte as u64) << (idx * 8)) + }) as usize +} + +#[cfg(feature = "bitpacking")] +fn legacy_copy_unchunk(data: LanceBuffer, num_values: u64) -> DataBlock +where + T: ArrowNativeType + BitPacking + Pod, +{ + assert!(data.len() >= std::mem::size_of::()); + assert!(num_values <= ELEMS_PER_CHUNK); + + let chunk_in_u8 = data.to_vec(); + let bit_width_value = read_little_endian_header::(&chunk_in_u8); + let chunk = bytemuck::cast_slice(&chunk_in_u8[std::mem::size_of::()..]); + assert!(std::mem::size_of_val(chunk) == bit_width_value * ELEMS_PER_CHUNK as usize / 8); + + let mut decompressed = vec![T::from_usize(0).unwrap(); ELEMS_PER_CHUNK as usize]; + unsafe { + BitPacking::unchecked_unpack(bit_width_value, chunk, &mut decompressed); + } + + decompressed.truncate(num_values as usize); + DataBlock::FixedWidth(FixedWidthDataBlock { + data: LanceBuffer::reinterpret_vec(decompressed), + bits_per_value: (std::mem::size_of::() * 8) as u64, + num_values, + block_info: BlockInfo::new(), + }) +} + +#[cfg(feature = "bitpacking")] +fn typed_view_unchunk(buffer: LanceBuffer, uncompressed_bits: u64, num_values: u64) -> DataBlock { + InlineBitpacking::new(uncompressed_bits) + .decompress(buffer, num_values) + .unwrap() +} + +#[cfg(feature = "bitpacking")] +fn assert_same_fixed_width_payloads(legacy: &DataBlock, typed_view: &DataBlock) { + let legacy = legacy.as_fixed_width_ref().unwrap(); + let typed_view = typed_view.as_fixed_width_ref().unwrap(); + + assert_eq!(legacy.num_values, typed_view.num_values); + assert_eq!(legacy.bits_per_value, typed_view.bits_per_value); + assert_eq!(legacy.data.as_ref(), typed_view.data.as_ref()); +} + +#[cfg(feature = "bitpacking")] +fn bench_inline_bitpacking_case( + group: &mut criterion::BenchmarkGroup<'_, criterion::measurement::WallTime>, + name: &str, + bit_width: usize, +) where + T: ArrowNativeType + BitPacking + Pod, +{ + let buffer = make_inline_bitpacking_chunk::(bit_width); + let compressed_bytes = buffer.len() as u64; + let uncompressed_bits = (std::mem::size_of::() * 8) as u64; + group.throughput(criterion::Throughput::Bytes(compressed_bytes)); + + let legacy = legacy_copy_unchunk::(buffer.clone(), ELEMS_PER_CHUNK); + let typed_view = typed_view_unchunk(buffer.clone(), uncompressed_bits, ELEMS_PER_CHUNK); + assert_same_fixed_width_payloads(&legacy, &typed_view); + + group.bench_function(format!("{name}/legacy_copy/compressed_bytes"), |b| { + b.iter(|| { + let decoded = + legacy_copy_unchunk::(black_box(buffer.clone()), black_box(ELEMS_PER_CHUNK)); + let fixed = decoded.as_fixed_width().unwrap(); + black_box(fixed.data.as_ref()); + }) + }); + + group.bench_function(format!("{name}/typed_view/compressed_bytes"), |b| { + b.iter(|| { + let decoded = typed_view_unchunk( + black_box(buffer.clone()), + black_box(uncompressed_bits), + black_box(ELEMS_PER_CHUNK), + ); + let fixed = decoded.as_fixed_width().unwrap(); + black_box(fixed.data.as_ref()); + }) + }); +} + +#[cfg(feature = "bitpacking")] +fn bench_decode_inline_bitpacking_unchunk(c: &mut Criterion) { + let mut group = c.benchmark_group("decode_inline_bitpacking_unchunk"); + bench_inline_bitpacking_case::(&mut group, "u32_bw12_1024", 12); + bench_inline_bitpacking_case::(&mut group, "u64_bw23_1024", 23); + group.finish(); +} + +#[cfg(not(feature = "bitpacking"))] +fn bench_decode_inline_bitpacking_unchunk(c: &mut Criterion) { + let mut group = c.benchmark_group("decode_inline_bitpacking_unchunk"); + group.bench_function("bitpacking_feature_disabled", |b| b.iter(|| black_box(()))); + group.finish(); +} + #[cfg(target_os = "linux")] criterion_group!( name=benches; @@ -570,7 +726,7 @@ criterion_group!( .with_profiler(lance_testing::pprof::PProfProfiler::new(100, lance_testing::pprof::Output::Flamegraph(None))); targets = bench_decode, bench_decode_fsl, bench_decode_str_with_dict_encoding, bench_decode_packed_struct, bench_decode_str_with_fixed_size_binary_encoding, bench_decode_compressed, - bench_decode_compressed_parallel); + bench_decode_compressed_parallel, bench_decode_inline_bitpacking_unchunk); // Non-linux version does not support pprof. #[cfg(not(target_os = "linux"))] @@ -578,5 +734,5 @@ criterion_group!( name=benches; config = Criterion::default().significance_level(0.1).sample_size(10); targets = bench_decode, bench_decode_fsl, bench_decode_str_with_dict_encoding, bench_decode_packed_struct, - bench_decode_compressed, bench_decode_compressed_parallel); + bench_decode_compressed, bench_decode_compressed_parallel, bench_decode_inline_bitpacking_unchunk); criterion_main!(benches); diff --git a/rust/lance-encoding/benches/encoder.rs b/rust/lance-encoding/benches/encoder.rs index 08eb89d32fe..02ffd92083c 100644 --- a/rust/lance-encoding/benches/encoder.rs +++ b/rust/lance-encoding/benches/encoder.rs @@ -7,15 +7,15 @@ use arrow_array::{ArrayRef, BooleanArray, ListArray, RecordBatch}; use arrow_buffer::{OffsetBuffer, ScalarBuffer}; use arrow_schema::{DataType, Field, Schema}; use criterion::{Criterion, criterion_group, criterion_main}; -use lance_encoding::{ - encoder::{EncodingOptions, default_encoding_strategy, encode_batch}, - version::LanceFileVersion, -}; +use lance_encoding::encoder::{EncodingOptions, encode_batch}; + +pub mod common; +use common::{BenchEncoding, encoding_strategy}; fn encode_batch_sync(rt: &tokio::runtime::Runtime, data: &RecordBatch) { let lance_schema = Arc::new(lance_core::datatypes::Schema::try_from(data.schema().as_ref()).unwrap()); - let encoding_strategy = default_encoding_strategy(LanceFileVersion::V2_2); + let encoding_strategy = encoding_strategy(BenchEncoding::StructuralU32); rt.block_on(encode_batch( data, @@ -68,7 +68,7 @@ fn bench_encode_compressed(c: &mut Criterion) { let lance_schema = Arc::new(lance_core::datatypes::Schema::try_from(schema.as_ref()).unwrap()); // V2_2+ required for general compression - let encoding_strategy = default_encoding_strategy(LanceFileVersion::V2_2); + let encoding_strategy = encoding_strategy(BenchEncoding::StructuralU32); group.throughput(criterion::Throughput::Elements( (NUM_ROWS * NUM_COLUMNS) as u64, diff --git a/rust/lance-encoding/src/array_encoding.rs b/rust/lance-encoding/src/array_encoding.rs new file mode 100644 index 00000000000..a76c6a186ec --- /dev/null +++ b/rust/lance-encoding/src/array_encoding.rs @@ -0,0 +1,13 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright The Lance Authors + +//! Encoding and decoding mechanisms described by [`crate::format::pb::ArrayEncoding`]. +//! +//! File versions decide which mechanisms to compose and accept. This module +//! contains only the reusable implementation of that persisted grammar. + +pub mod logical; +pub mod physical; +mod strategy; + +pub use strategy::ArrayFieldEncodingStrategy; diff --git a/rust/lance-encoding/src/previous/encodings/logical.rs b/rust/lance-encoding/src/array_encoding/logical.rs similarity index 100% rename from rust/lance-encoding/src/previous/encodings/logical.rs rename to rust/lance-encoding/src/array_encoding/logical.rs diff --git a/rust/lance-encoding/src/previous/encodings/logical/binary.rs b/rust/lance-encoding/src/array_encoding/logical/binary.rs similarity index 97% rename from rust/lance-encoding/src/previous/encodings/logical/binary.rs rename to rust/lance-encoding/src/array_encoding/logical/binary.rs index 00715f64511..697c1503e0a 100644 --- a/rust/lance-encoding/src/previous/encodings/logical/binary.rs +++ b/rust/lance-encoding/src/array_encoding/logical/binary.rs @@ -19,7 +19,7 @@ use crate::{ DecodeArrayTask, FilterExpression, MessageType, NextDecodeTask, PriorityRange, ScheduledScanLine, SchedulerContext, }, - previous::decoder::{DecoderReady, FieldScheduler, LogicalPageDecoder, SchedulingJob}, + decoder::{DecoderReady, FieldScheduler, LogicalPageDecoder, SchedulingJob}, }; /// Wraps a varbin scheduler and uses a BinaryPageDecoder to cast @@ -41,7 +41,7 @@ impl SchedulingJob for BinarySchedulingJob<'_> { .decoders .into_iter() .map(|message| { - let decoder = message.into_legacy(); + let decoder = message.into_array(); MessageType::DecoderReady(DecoderReady { path: decoder.path, decoder: Box::new(BinaryPageDecoder { @@ -179,7 +179,7 @@ impl DecodeArrayTask for BinaryArrayDecoder { DataType::LargeUtf8 => Self::from_list_array::(arr.as_list::()), _ => panic!("Binary decoder does not support this data type"), }; - // data_size is only tracked in the v2.1 structural decode path; the legacy + // data_size is only tracked in the v2.1 structural decode path; the v2.0 array // v2.0 path does not need it so we return 0. Ok((result, 0)) } diff --git a/rust/lance-encoding/src/previous/encodings/logical/blob.rs b/rust/lance-encoding/src/array_encoding/logical/blob.rs similarity index 95% rename from rust/lance-encoding/src/previous/encodings/logical/blob.rs rename to rust/lance-encoding/src/array_encoding/logical/blob.rs index 13fa3b346cb..0bf3a39fca1 100644 --- a/rust/lance-encoding/src/previous/encodings/logical/blob.rs +++ b/rust/lance-encoding/src/array_encoding/logical/blob.rs @@ -23,9 +23,9 @@ use crate::{ DecodeArrayTask, FilterExpression, MessageType, NextDecodeTask, PriorityRange, ScheduledScanLine, SchedulerContext, }, + decoder::{DecoderReady, FieldScheduler, LogicalPageDecoder, SchedulingJob}, encoder::{EncodeTask, FieldEncoder, OutOfLineBuffers}, format::pb::{Blob, ColumnEncoding, column_encoding}, - previous::decoder::{DecoderReady, FieldScheduler, LogicalPageDecoder, SchedulingJob}, repdef::RepDefBuilder, }; @@ -67,7 +67,7 @@ impl SchedulingJob for BlobFieldSchedulingJob<'_> { let next_descriptions = self.descriptions_job.schedule_next(context, priority)?; let mut priority = priority.current_priority(); let decoders = next_descriptions.decoders.into_iter().map(|decoder| { - let decoder = decoder.into_legacy(); + let decoder = decoder.into_array(); let path = decoder.path; let mut decoder = decoder.decoder; let num_rows = decoder.num_rows(); @@ -285,7 +285,7 @@ impl DecodeArrayTask for BlobArrayDecodeTask { buffer.extend_from_slice(&bytes); } let data_buf = Buffer::from_vec(buffer); - // data_size is only tracked in the v2.1 structural decode path; the legacy + // data_size is only tracked in the v2.1 structural decode path; the v2.0 array // v2.0 path does not need it so we return 0. Ok(( Arc::new(LargeBinaryArray::new(offsets, data_buf, self.validity)), @@ -318,7 +318,7 @@ impl BlobFieldEncoder { .nulls() .cloned() .unwrap_or(NullBuffer::new_valid(binarray.len())); - for (w, is_valid) in binarray.value_offsets().windows(2).zip(nulls.into_iter()) { + for (w, is_valid) in binarray.value_offsets().windows(2).zip(&nulls) { if is_valid { let start = w[0] as u64; let end = w[1] as u64; @@ -417,10 +417,10 @@ mod tests { use super::BlobFieldDecoder; use crate::{ EncodingsIo, + decoder::LogicalPageDecoder, format::pb::column_encoding, - previous::decoder::LogicalPageDecoder, + testing::TestEncoding, testing::{TestCases, check_round_trip_encoding_of_data, check_specific_random}, - version::LanceFileVersion, }; static BLOB_META: LazyLock> = LazyLock::new(|| { @@ -433,11 +433,7 @@ mod tests { #[test_log::test(tokio::test)] async fn test_basic_blob() { let field = Field::new("", DataType::LargeBinary, false).with_metadata(BLOB_META.clone()); - check_specific_random( - field, - TestCases::basic().with_max_file_version(LanceFileVersion::V2_1), - ) - .await; + check_specific_random(field, TestCases::basic().with_array_and_u16_encodings()).await; } #[test_log::test(tokio::test)] @@ -446,10 +442,10 @@ mod tests { let val2: &[u8] = &[7, 8, 9]; let array = Arc::new(LargeBinaryArray::from(vec![Some(val1), None, Some(val2)])); let test_cases = TestCases::default() - .with_max_file_version(LanceFileVersion::V2_1) + .with_array_and_u16_encodings() .with_expected_encoding("packed_struct") - .with_verify_encoding(Arc::new(|cols, version| { - if version < &LanceFileVersion::V2_1 { + .with_verify_encoding(Arc::new(|cols, encoding| { + if *encoding == TestEncoding::Array { // In 2.0 we used a special "column encoding" to mark blob fields. In 2.1 we // don't do this and just rely on the regular page encoding. assert_eq!(cols.len(), 1); @@ -465,9 +461,9 @@ mod tests { .await; let test_cases = TestCases::default() - .with_min_file_version(LanceFileVersion::V2_1) - .with_verify_encoding(Arc::new(|cols, version| { - if version < &LanceFileVersion::V2_1 { + .with_structural_encodings() + .with_verify_encoding(Arc::new(|cols, encoding| { + if *encoding == TestEncoding::Array { assert_eq!(cols.len(), 1); let col = &cols[0]; assert!(!matches!( diff --git a/rust/lance-encoding/src/previous/encodings/logical/list.rs b/rust/lance-encoding/src/array_encoding/logical/list.rs similarity index 98% rename from rust/lance-encoding/src/previous/encodings/logical/list.rs rename to rust/lance-encoding/src/array_encoding/logical/list.rs index 3de886a21db..d76532ab3bb 100644 --- a/rust/lance-encoding/src/previous/encodings/logical/list.rs +++ b/rust/lance-encoding/src/array_encoding/logical/list.rs @@ -22,19 +22,19 @@ use tokio::task::JoinHandle; use crate::{ EncodingsIo, + array_encoding::logical::r#struct::{SimpleStructDecoder, SimpleStructScheduler}, buffer::LanceBuffer, data::{BlockInfo, DataBlock, FixedWidthDataBlock}, decoder::{ - DecodeArrayTask, DecodeBatchScheduler, FilterExpression, ListPriorityRange, MessageType, - NextDecodeTask, PageEncoding, PriorityRange, ScheduledScanLine, SchedulerContext, + DecodeArrayTask, DecodeBatchScheduler, FieldScheduler, FilterExpression, ListPriorityRange, + LogicalPageDecoder, MessageType, NextDecodeTask, PageEncoding, PriorityRange, + ScheduledScanLine, SchedulerContext, SchedulingJob, }, - encoder::{EncodeTask, EncodedColumn, EncodedPage, FieldEncoder, OutOfLineBuffers}, - format::pb, - previous::{ - decoder::{FieldScheduler, LogicalPageDecoder, SchedulingJob}, - encoder::{ArrayEncoder, EncodedArray}, - encodings::logical::r#struct::{SimpleStructDecoder, SimpleStructScheduler}, + encoder::{ + ArrayEncoder, EncodeTask, EncodedArray, EncodedColumn, EncodedPage, FieldEncoder, + OutOfLineBuffers, }, + format::pb, repdef::RepDefBuilder, utils::accumulation::AccumulationQueue, }; @@ -397,7 +397,7 @@ async fn indirect_schedule_task( for message in indirect_messages { for decoder in message.decoders { - let decoder = decoder.into_legacy(); + let decoder = decoder.into_array(); if !decoder.path.is_empty() { root_decoder.accept_child(decoder)?; } @@ -467,7 +467,7 @@ impl SchedulingJob for ListFieldSchedulingJob<'_> { .into_iter() .next() .unwrap() - .into_legacy() + .into_array() .decoder; let items_scheduler = self.scheduler.items_scheduler.clone(); @@ -690,7 +690,7 @@ impl DecodeArrayTask for ListDecodeTask { } _ => panic!("ListDecodeTask with data type that is not i32 or i64"), }; - // data_size is only tracked in the v2.1 structural decode path; the legacy + // data_size is only tracked in the v2.1 structural decode path; the v2.0 array // v2.0 path does not need it so we return 0. Ok((array, 0)) } @@ -961,7 +961,7 @@ impl ListOffsetsEncoder { description: PageEncoding::Legacy(description), num_rows, column_idx, - row_number: 0, // Legacy encoders do not use + row_number: 0, // V2.0 encoders do not use }) }) .map(|res_res| res_res.unwrap()) diff --git a/rust/lance-encoding/src/previous/encodings/logical/primitive.rs b/rust/lance-encoding/src/array_encoding/logical/primitive.rs similarity index 98% rename from rust/lance-encoding/src/previous/encodings/logical/primitive.rs rename to rust/lance-encoding/src/array_encoding/logical/primitive.rs index d1debf3ef33..3a8e9f73e2e 100644 --- a/rust/lance-encoding/src/previous/encodings/logical/primitive.rs +++ b/rust/lance-encoding/src/array_encoding/logical/primitive.rs @@ -10,10 +10,10 @@ use futures::{FutureExt, future::BoxFuture}; use log::trace; use crate::decoder::{ColumnBuffers, PageBuffers}; -use crate::previous::decoder::{FieldScheduler, LogicalPageDecoder, SchedulingJob}; -use crate::previous::encoder::ArrayEncodingStrategy; +use crate::decoder::{FieldScheduler, LogicalPageDecoder, SchedulingJob}; +use crate::encoder::ArrayEncodingStrategy; use crate::utils::accumulation::AccumulationQueue; -use crate::{data::DataBlock, previous::encodings::physical::decoder_from_array_encoding}; +use crate::{array_encoding::physical::decoder_from_array_encoding, data::DataBlock}; use lance_core::{Error, Result, datatypes::Field}; use crate::{ @@ -316,7 +316,7 @@ impl DecodeArrayTask for PrimitiveFieldDecodeTask { return Ok((new_array, 0)); } } - // data_size is only tracked in the v2.1 structural decode path; the legacy + // data_size is only tracked in the v2.1 structural decode path; the v2.0 array // v2.0 path does not need it so we return 0. Ok((array, 0)) } @@ -439,7 +439,7 @@ impl PrimitiveFieldEncoder { description: PageEncoding::Legacy(description), num_rows: num_values, column_idx, - row_number: 0, // legacy encoders do not use + row_number: 0, // v2.0 encoders do not use }) }) .map(|res_res| { diff --git a/rust/lance-encoding/src/previous/encodings/logical/struct.rs b/rust/lance-encoding/src/array_encoding/logical/struct.rs similarity index 99% rename from rust/lance-encoding/src/previous/encodings/logical/struct.rs rename to rust/lance-encoding/src/array_encoding/logical/struct.rs index b117f74ce2b..045b3bca71d 100644 --- a/rust/lance-encoding/src/previous/encodings/logical/struct.rs +++ b/rust/lance-encoding/src/array_encoding/logical/struct.rs @@ -12,7 +12,7 @@ use crate::{ DecodeArrayTask, FilterExpression, MessageType, NextDecodeTask, PriorityRange, ScheduledScanLine, SchedulerContext, }, - previous::decoder::{DecoderReady, FieldScheduler, LogicalPageDecoder, SchedulingJob}, + decoder::{DecoderReady, FieldScheduler, LogicalPageDecoder, SchedulingJob}, }; use arrow_array::{ArrayRef, StructArray}; use arrow_schema::{DataType, Field, Fields}; @@ -57,7 +57,7 @@ struct EmptyStructDecodeTask { impl DecodeArrayTask for EmptyStructDecodeTask { fn decode(self: Box) -> Result<(ArrayRef, u64)> { - // data_size is only tracked in the v2.1 structural decode path; the legacy + // data_size is only tracked in the v2.1 structural decode path; the v2.0 array // v2.0 path does not need it so we return 0. Ok(( Arc::new(StructArray::new_empty_fields(self.num_rows as usize, None)), @@ -607,7 +607,7 @@ impl DecodeArrayTask for SimpleStructDecodeTask { .into_iter() .map(|child| child.decode()) .collect::>>()?; - // data_size is only tracked in the v2.1 structural decode path; the legacy + // data_size is only tracked in the v2.1 structural decode path; the v2.0 array // v2.0 path does not need it so we return 0. Ok(( Arc::new(StructArray::try_new(self.child_fields, child_arrays, None)?), diff --git a/rust/lance-encoding/src/previous/encodings/physical.rs b/rust/lance-encoding/src/array_encoding/physical.rs similarity index 99% rename from rust/lance-encoding/src/previous/encodings/physical.rs rename to rust/lance-encoding/src/array_encoding/physical.rs index a3cb0adb4a7..66af9e806df 100644 --- a/rust/lance-encoding/src/previous/encodings/physical.rs +++ b/rust/lance-encoding/src/array_encoding/physical.rs @@ -5,16 +5,16 @@ use arrow_schema::DataType; use lance_arrow::DataTypeExt; use crate::{ - buffer::LanceBuffer, - decoder::{PageBuffers, PageScheduler}, - encodings::physical::block::{CompressionConfig, CompressionScheme}, - format::pb::{self, PackedStruct}, - previous::encodings::physical::{ + array_encoding::physical::{ basic::BasicPageScheduler, binary::BinaryPageScheduler, bitmap::DenseBitmapScheduler, dictionary::DictionaryPageScheduler, fixed_size_list::FixedListScheduler, fsst::FsstPageScheduler, packed_struct::PackedStructPageScheduler, value::ValuePageScheduler, }, + buffer::LanceBuffer, + decoder::{PageBuffers, PageScheduler}, + encodings::physical::block::{CompressionConfig, CompressionScheme}, + format::pb::{self, PackedStruct}, }; pub mod basic; @@ -293,9 +293,9 @@ pub fn decoder_from_array_encoding( #[cfg(test)] mod tests { + use crate::array_encoding::physical::get_buffer_decoder; use crate::decoder::{ColumnBuffers, FileBuffers, PageBuffers}; use crate::format::pb; - use crate::previous::encodings::physical::get_buffer_decoder; #[test] fn test_get_buffer_decoder_for_compressed_buffer() { diff --git a/rust/lance-encoding/src/previous/encodings/physical/basic.rs b/rust/lance-encoding/src/array_encoding/physical/basic.rs similarity index 99% rename from rust/lance-encoding/src/previous/encodings/physical/basic.rs rename to rust/lance-encoding/src/array_encoding/physical/basic.rs index ec098c3fff4..6dd7326a97f 100644 --- a/rust/lance-encoding/src/previous/encodings/physical/basic.rs +++ b/rust/lance-encoding/src/array_encoding/physical/basic.rs @@ -11,8 +11,8 @@ use crate::{ EncodingsIo, data::{AllNullDataBlock, BlockInfo, DataBlock, NullableDataBlock}, decoder::{PageScheduler, PrimitivePageDecoder}, + encoder::{ArrayEncoder, EncodedArray}, format::ProtobufUtils, - previous::encoder::{ArrayEncoder, EncodedArray}, }; use lance_core::Result; diff --git a/rust/lance-encoding/src/previous/encodings/physical/binary.rs b/rust/lance-encoding/src/array_encoding/physical/binary.rs similarity index 99% rename from rust/lance-encoding/src/previous/encodings/physical/binary.rs rename to rust/lance-encoding/src/array_encoding/physical/binary.rs index 0bd4d96b45e..294b295def3 100644 --- a/rust/lance-encoding/src/previous/encodings/physical/binary.rs +++ b/rust/lance-encoding/src/array_encoding/physical/binary.rs @@ -12,17 +12,17 @@ use futures::TryFutureExt; use futures::{FutureExt, future::BoxFuture}; +use crate::array_encoding::logical::primitive::PrimitiveFieldDecoder; use crate::buffer::LanceBuffer; use crate::data::{ BlockInfo, DataBlock, FixedWidthDataBlock, NullableDataBlock, VariableWidthBlock, }; +use crate::decoder::LogicalPageDecoder; +use crate::encoder::{ArrayEncoder, EncodedArray}; use crate::encodings::physical::block::{ BufferCompressor, CompressionConfig, GeneralBufferCompressor, }; use crate::format::ProtobufUtils; -use crate::previous::decoder::LogicalPageDecoder; -use crate::previous::encoder::{ArrayEncoder, EncodedArray}; -use crate::previous::encodings::logical::primitive::PrimitiveFieldDecoder; use crate::{ EncodingsIo, decoder::{PageScheduler, PrimitivePageDecoder}, diff --git a/rust/lance-encoding/src/previous/encodings/physical/bitmap.rs b/rust/lance-encoding/src/array_encoding/physical/bitmap.rs similarity index 99% rename from rust/lance-encoding/src/previous/encodings/physical/bitmap.rs rename to rust/lance-encoding/src/array_encoding/physical/bitmap.rs index fc6d295b0f9..2168aff24e2 100644 --- a/rust/lance-encoding/src/previous/encodings/physical/bitmap.rs +++ b/rust/lance-encoding/src/array_encoding/physical/bitmap.rs @@ -131,9 +131,9 @@ mod tests { use bytes::Bytes; use std::{collections::HashMap, sync::Arc}; + use crate::array_encoding::physical::bitmap::BitmapData; use crate::data::{DataBlock, FixedWidthDataBlock}; use crate::decoder::PrimitivePageDecoder; - use crate::previous::encodings::physical::bitmap::BitmapData; use crate::testing::{TestCases, check_basic_random, check_round_trip_encoding_of_data}; use super::BitmapDecoder; diff --git a/rust/lance-encoding/src/previous/encodings/physical/bitpack.rs b/rust/lance-encoding/src/array_encoding/physical/bitpack.rs similarity index 51% rename from rust/lance-encoding/src/previous/encodings/physical/bitpack.rs rename to rust/lance-encoding/src/array_encoding/physical/bitpack.rs index d80dec351d1..1d930af412b 100644 --- a/rust/lance-encoding/src/previous/encodings/physical/bitpack.rs +++ b/rust/lance-encoding/src/array_encoding/physical/bitpack.rs @@ -3,353 +3,23 @@ use std::sync::Arc; -use arrow_array::types::{ - Int8Type, Int16Type, Int32Type, Int64Type, UInt8Type, UInt16Type, UInt32Type, UInt64Type, -}; -use arrow_array::{Array, ArrayRef, ArrowPrimitiveType, PrimitiveArray, cast::AsArray}; -use arrow_buffer::ArrowNativeType; use arrow_buffer::bit_util::ceil; -use arrow_schema::DataType; use bytes::Bytes; use futures::future::{BoxFuture, FutureExt}; use log::trace; -use num_traits::{AsPrimitive, PrimInt}; -use lance_arrow::DataTypeExt; use lance_bitpacking::BitPacking; use lance_core::{Error, Result}; use crate::buffer::LanceBuffer; use crate::data::BlockInfo; -use crate::data::{DataBlock, FixedWidthDataBlock, NullableDataBlock}; +use crate::data::{DataBlock, FixedWidthDataBlock}; use crate::decoder::{PageScheduler, PrimitivePageDecoder}; -use crate::format::ProtobufUtils; -use crate::previous::encoder::{ArrayEncoder, EncodedArray}; use bytemuck::cast_slice; const LOG_ELEMS_PER_CHUNK: u8 = 10; const ELEMS_PER_CHUNK: u64 = 1 << LOG_ELEMS_PER_CHUNK; -// Compute the compressed_bit_width for a given array of integers -// todo: compute all statistics before encoding -// todo: see how to use rust macro to rewrite this function -pub fn compute_compressed_bit_width_for_non_neg(arrays: &[ArrayRef]) -> u64 { - debug_assert!(!arrays.is_empty()); - - let res; - - match arrays[0].data_type() { - DataType::UInt8 => { - let mut global_max: u8 = 0; - for array in arrays { - let primitive_array = array - .as_any() - .downcast_ref::>() - .unwrap(); - let array_max = arrow_arith::aggregate::bit_or(primitive_array); - global_max = global_max.max(array_max.unwrap_or(0)); - } - let num_bits = - arrays[0].data_type().byte_width() as u64 * 8 - global_max.leading_zeros() as u64; - // we will have constant encoding later - if num_bits == 0 { - res = 1; - } else { - res = num_bits; - } - } - - DataType::Int8 => { - let mut global_max_width: u64 = 0; - for array in arrays { - let primitive_array = array - .as_any() - .downcast_ref::>() - .unwrap(); - let array_max_width = arrow_arith::aggregate::bit_or(primitive_array).unwrap_or(0); - global_max_width = global_max_width.max(8 - array_max_width.leading_zeros() as u64); - } - if global_max_width == 0 { - res = 1; - } else { - res = global_max_width; - } - } - - DataType::UInt16 => { - let mut global_max: u16 = 0; - for array in arrays { - let primitive_array = array - .as_any() - .downcast_ref::>() - .unwrap(); - let array_max = arrow_arith::aggregate::bit_or(primitive_array).unwrap_or(0); - global_max = global_max.max(array_max); - } - let num_bits = - arrays[0].data_type().byte_width() as u64 * 8 - global_max.leading_zeros() as u64; - if num_bits == 0 { - res = 1; - } else { - res = num_bits; - } - } - - DataType::Int16 => { - let mut global_max_width: u64 = 0; - for array in arrays { - let primitive_array = array - .as_any() - .downcast_ref::>() - .unwrap(); - let array_max_width = arrow_arith::aggregate::bit_or(primitive_array).unwrap_or(0); - global_max_width = - global_max_width.max(16 - array_max_width.leading_zeros() as u64); - } - if global_max_width == 0 { - res = 1; - } else { - res = global_max_width; - } - } - - DataType::UInt32 => { - let mut global_max: u32 = 0; - for array in arrays { - let primitive_array = array - .as_any() - .downcast_ref::>() - .unwrap(); - let array_max = arrow_arith::aggregate::bit_or(primitive_array).unwrap_or(0); - global_max = global_max.max(array_max); - } - let num_bits = - arrays[0].data_type().byte_width() as u64 * 8 - global_max.leading_zeros() as u64; - if num_bits == 0 { - res = 1; - } else { - res = num_bits; - } - } - - DataType::Int32 => { - let mut global_max_width: u64 = 0; - for array in arrays { - let primitive_array = array - .as_any() - .downcast_ref::>() - .unwrap(); - let array_max_width = arrow_arith::aggregate::bit_or(primitive_array).unwrap_or(0); - global_max_width = - global_max_width.max(32 - array_max_width.leading_zeros() as u64); - } - if global_max_width == 0 { - res = 1; - } else { - res = global_max_width; - } - } - - DataType::UInt64 => { - let mut global_max: u64 = 0; - for array in arrays { - let primitive_array = array - .as_any() - .downcast_ref::>() - .unwrap(); - let array_max = arrow_arith::aggregate::bit_or(primitive_array).unwrap_or(0); - global_max = global_max.max(array_max); - } - let num_bits = - arrays[0].data_type().byte_width() as u64 * 8 - global_max.leading_zeros() as u64; - if num_bits == 0 { - res = 1; - } else { - res = num_bits; - } - } - - DataType::Int64 => { - let mut global_max_width: u64 = 0; - for array in arrays { - let primitive_array = array - .as_any() - .downcast_ref::>() - .unwrap(); - let array_max_width = arrow_arith::aggregate::bit_or(primitive_array).unwrap_or(0); - global_max_width = - global_max_width.max(64 - array_max_width.leading_zeros() as u64); - } - if global_max_width == 0 { - res = 1; - } else { - res = global_max_width; - } - } - _ => { - panic!( - "BitpackedForNonNegArrayEncoder only supports data types of UInt8, Int8, UInt16, Int16, UInt32, Int32, UInt64, Int64" - ); - } - }; - res -} - -// Bitpack integers using fastlanes algorithm, the input is sliced into chunks of 1024 integers, and bitpacked -// chunk by chunk. when the input is not a multiple of 1024, the last chunk is padded with zeros, this is fine because -// we also know the number of rows we have. -// Here self is a borrow of BitpackedForNonNegArrayEncoder, unpacked is a mutable borrow of FixedWidthDataBlock, -// data_type can be one of u8, u16, u32, or u64. -// buffer_index is a mutable borrow of u32, indicating the buffer index of the output EncodedArray. -// It outputs an fastlanes bitpacked EncodedArray -macro_rules! encode_fixed_width { - ($self:expr, $unpacked:expr, $data_type:ty, $buffer_index:expr) => {{ - let num_chunks = $unpacked.num_values.div_ceil(ELEMS_PER_CHUNK); - let num_full_chunks = $unpacked.num_values / ELEMS_PER_CHUNK; - let uncompressed_bit_width = std::mem::size_of::<$data_type>() as u64 * 8; - - // the output vector type is the same as the input type, for example, when input is u16, output is Vec - let packed_chunk_size = 1024 * $self.compressed_bit_width as usize / uncompressed_bit_width as usize; - - let input_slice = $unpacked.data.borrow_to_typed_slice::<$data_type>(); - let input = input_slice.as_ref(); - - let mut output = Vec::with_capacity(num_chunks as usize * packed_chunk_size); - - // Loop over all but the last chunk. - (0..num_full_chunks).for_each(|i| { - let start_elem = (i * ELEMS_PER_CHUNK) as usize; - - let output_len = output.len(); - unsafe { - output.set_len(output_len + packed_chunk_size); - BitPacking::unchecked_pack( - $self.compressed_bit_width, - &input[start_elem..][..ELEMS_PER_CHUNK as usize], - &mut output[output_len..][..packed_chunk_size], - ); - } - }); - - if num_chunks != num_full_chunks { - let last_chunk_elem_num = $unpacked.num_values % ELEMS_PER_CHUNK; - let mut last_chunk = vec![0 as $data_type; ELEMS_PER_CHUNK as usize]; - last_chunk[..last_chunk_elem_num as usize].clone_from_slice( - &input[$unpacked.num_values as usize - last_chunk_elem_num as usize..], - ); - - let output_len = output.len(); - unsafe { - output.set_len(output_len + packed_chunk_size); - BitPacking::unchecked_pack( - $self.compressed_bit_width, - &last_chunk, - &mut output[output_len..][..packed_chunk_size], - ); - } - } - - let bitpacked_for_non_neg_buffer_index = *$buffer_index; - *$buffer_index += 1; - - let encoding = ProtobufUtils::bitpacked_for_non_neg_encoding( - $self.compressed_bit_width as u64, - uncompressed_bit_width, - bitpacked_for_non_neg_buffer_index, - ); - let packed = DataBlock::FixedWidth(FixedWidthDataBlock { - bits_per_value: $self.compressed_bit_width as u64, - data: LanceBuffer::reinterpret_vec(output), - num_values: $unpacked.num_values, - block_info: BlockInfo::new(), - }); - - Result::Ok(EncodedArray { - data: packed, - encoding, - }) - }}; -} - -#[derive(Debug)] -pub struct BitpackedForNonNegArrayEncoder { - pub compressed_bit_width: usize, - pub original_data_type: DataType, -} - -impl BitpackedForNonNegArrayEncoder { - pub fn new(compressed_bit_width: usize, data_type: DataType) -> Self { - Self { - compressed_bit_width, - original_data_type: data_type, - } - } -} - -impl ArrayEncoder for BitpackedForNonNegArrayEncoder { - fn encode( - &self, - data: DataBlock, - data_type: &DataType, - buffer_index: &mut u32, - ) -> Result { - match data { - DataBlock::AllNull(_) => { - let encoding = ProtobufUtils::basic_all_null_encoding(); - Ok(EncodedArray { data, encoding }) - } - DataBlock::FixedWidth(unpacked) => { - match data_type { - DataType::UInt8 | DataType::Int8 => encode_fixed_width!(self, unpacked, u8, buffer_index), - DataType::UInt16 | DataType::Int16 => encode_fixed_width!(self, unpacked, u16, buffer_index), - DataType::UInt32 | DataType::Int32 => encode_fixed_width!(self, unpacked, u32, buffer_index), - DataType::UInt64 | DataType::Int64 => encode_fixed_width!(self, unpacked, u64, buffer_index), - _ => unreachable!("BitpackedForNonNegArrayEncoder only supports data types of UInt8, Int8, UInt16, Int16, UInt32, Int32, UInt64, Int64"), - } - } - DataBlock::Nullable(nullable) => { - let validity_buffer_index = *buffer_index; - *buffer_index += 1; - - let validity_desc = ProtobufUtils::flat_encoding( - 1, - validity_buffer_index, - /*compression=*/ None, - ); - let encoded_values: EncodedArray; - match *nullable.data { - DataBlock::FixedWidth(unpacked) => { - match data_type { - DataType::UInt8 | DataType::Int8 => encoded_values = encode_fixed_width!(self, unpacked, u8, buffer_index)?, - DataType::UInt16 | DataType::Int16 => encoded_values = encode_fixed_width!(self, unpacked, u16, buffer_index)?, - DataType::UInt32 | DataType::Int32 => encoded_values = encode_fixed_width!(self, unpacked, u32, buffer_index)?, - DataType::UInt64 | DataType::Int64 => encoded_values = encode_fixed_width!(self, unpacked, u64, buffer_index)?, - _ => unreachable!("BitpackedForNonNegArrayEncoder only supports data types of UInt8, Int8, UInt16, Int16, UInt32, Int32, UInt64, Int64"), - } - } - _ => { - return Err(Error::invalid_input_source("Bitpacking only supports fixed width data blocks or a nullable data block with fixed width data block inside or a all null data block".into())); - } - } - let encoding = - ProtobufUtils::basic_some_null_encoding(validity_desc, encoded_values.encoding); - let encoded = DataBlock::Nullable(NullableDataBlock { - data: Box::new(encoded_values.data), - nulls: nullable.nulls, - block_info: BlockInfo::new(), - }); - Ok(EncodedArray { - data: encoded, - encoding, - }) - } - _ => { - Err(Error::invalid_input_source("Bitpacking only supports fixed width data blocks or a nullable data block with fixed width data block inside or a all null data block".into())) - } - } - } -} - #[derive(Debug)] pub struct BitpackedForNonNegScheduler { compressed_bit_width: u64, @@ -589,219 +259,6 @@ fn bitpacked_for_non_neg_decode( } } -#[derive(Debug)] -pub struct BitpackParams { - pub num_bits: u64, - - pub signed: bool, -} - -// Compute the number of bits to use for each item, if this array can be encoded using -// bitpacking encoding. Returns `None` if the type or array data is not supported. -pub fn bitpack_params(arr: &dyn Array) -> Option { - match arr.data_type() { - DataType::UInt8 => bitpack_params_for_type::(arr.as_primitive()), - DataType::UInt16 => bitpack_params_for_type::(arr.as_primitive()), - DataType::UInt32 => bitpack_params_for_type::(arr.as_primitive()), - DataType::UInt64 => bitpack_params_for_type::(arr.as_primitive()), - DataType::Int8 => bitpack_params_for_signed_type::(arr.as_primitive()), - DataType::Int16 => bitpack_params_for_signed_type::(arr.as_primitive()), - DataType::Int32 => bitpack_params_for_signed_type::(arr.as_primitive()), - DataType::Int64 => bitpack_params_for_signed_type::(arr.as_primitive()), - // TODO -- eventually we could support temporal types as well - _ => None, - } -} - -// Compute the number bits to use for bitpacking generically. -// returns None if the array is empty or all nulls -fn bitpack_params_for_type(arr: &PrimitiveArray) -> Option -where - T: ArrowPrimitiveType, - T::Native: PrimInt + AsPrimitive, -{ - let max = arrow_arith::aggregate::bit_or(arr); - let num_bits = - max.map(|max| arr.data_type().byte_width() as u64 * 8 - max.leading_zeros() as u64); - - // we can't bitpack into 0 bits, so the minimum is 1 - num_bits - .map(|num_bits| num_bits.max(1)) - .map(|bits| BitpackParams { - num_bits: bits, - signed: false, - }) -} - -/// determine the minimum number of bits that can be used to represent -/// an array of signed values. It includes all the significant bits for -/// the value + plus 1 bit to represent the sign. If there are no negative values -/// then it will not add a signed bit -fn bitpack_params_for_signed_type(arr: &PrimitiveArray) -> Option -where - T: ArrowPrimitiveType, - T::Native: PrimInt + AsPrimitive, -{ - let mut add_signed_bit = false; - let mut min_leading_bits: Option = None; - for val in arr.iter() { - if val.is_none() { - continue; - } - let val = val.unwrap(); - if min_leading_bits.is_none() { - min_leading_bits = Some(u64::MAX); - } - - if val.to_i64().unwrap() < 0i64 { - min_leading_bits = min_leading_bits.map(|bits| bits.min(val.leading_ones() as u64)); - add_signed_bit = true; - } else { - min_leading_bits = min_leading_bits.map(|bits| bits.min(val.leading_zeros() as u64)); - } - } - - let mut min_leading_bits = arr.data_type().byte_width() as u64 * 8 - min_leading_bits?; - if add_signed_bit { - // Need extra sign bit - min_leading_bits += 1; - } - // cannot bitpack into <1 bit - let num_bits = min_leading_bits.max(1); - Some(BitpackParams { - num_bits, - signed: add_signed_bit, - }) -} -#[derive(Debug)] -pub struct BitpackedArrayEncoder { - num_bits: u64, - signed_type: bool, -} - -impl BitpackedArrayEncoder { - pub fn new(num_bits: u64, signed_type: bool) -> Self { - Self { - num_bits, - signed_type, - } - } -} - -impl ArrayEncoder for BitpackedArrayEncoder { - fn encode( - &self, - data: DataBlock, - _data_type: &DataType, - buffer_index: &mut u32, - ) -> Result { - // calculate the total number of bytes we need to allocate for the destination. - // this will be the number of items in the source array times the number of bits. - let dst_bytes_total = ceil(data.num_values() as usize * self.num_bits as usize, 8); - - let mut dst_buffer = vec![0u8; dst_bytes_total]; - let mut dst_idx = 0; - let mut dst_offset = 0; - - let DataBlock::FixedWidth(unpacked) = data else { - return Err(Error::invalid_input_source( - "Bitpacking only supports fixed width data blocks".into(), - )); - }; - - pack_bits( - &unpacked.data, - self.num_bits, - &mut dst_buffer, - &mut dst_idx, - &mut dst_offset, - ); - - let packed = DataBlock::FixedWidth(FixedWidthDataBlock { - bits_per_value: self.num_bits, - data: LanceBuffer::from(dst_buffer), - num_values: unpacked.num_values, - block_info: BlockInfo::new(), - }); - - let bitpacked_buffer_index = *buffer_index; - *buffer_index += 1; - - let encoding = ProtobufUtils::bitpacked_encoding( - self.num_bits, - unpacked.bits_per_value, - bitpacked_buffer_index, - self.signed_type, - ); - - Ok(EncodedArray { - data: packed, - encoding, - }) - } -} - -fn pack_bits( - src: &LanceBuffer, - num_bits: u64, - dst: &mut [u8], - dst_idx: &mut usize, - dst_offset: &mut u8, -) { - let bit_len = src.len() as u64 * 8; - - let mask = u64::MAX >> (64 - num_bits); - - let mut src_idx = 0; - while src_idx < src.len() { - let mut curr_mask = mask; - let mut curr_src = src[src_idx] & curr_mask as u8; - let mut src_offset = 0; - let mut src_bits_written = 0; - - while src_bits_written < num_bits { - dst[*dst_idx] += (curr_src >> src_offset) << *dst_offset as u64; - let bits_written = (num_bits - src_bits_written) - .min(8 - src_offset) - .min(8 - *dst_offset as u64); - src_bits_written += bits_written; - *dst_offset += bits_written as u8; - src_offset += bits_written; - - if *dst_offset == 8 { - *dst_idx += 1; - *dst_offset = 0; - } - - if src_offset == 8 { - src_idx += 1; - src_offset = 0; - curr_mask >>= 8; - if src_idx == src.len() { - break; - } - curr_src = src[src_idx] & curr_mask as u8; - } - } - - // advance source_offset to the next byte if we're not at the end.. - // note that we don't need to do this if we wrote the full number of bits - // because source index would have been advanced by the inner loop above - if bit_len != num_bits { - let partial_bytes_written = ceil(num_bits as usize, 8); - - // we also want to the next location in src, unless we wrote something - // byte-aligned in which case the logic above would have already advanced - let mut to_next_byte = 1; - if num_bits.is_multiple_of(8) { - to_next_byte = 0; - } - - src_idx += src.len() - partial_bytes_written + to_next_byte; - } - } -} - // A physical scheduler for bitpacked buffers #[derive(Debug, Clone, Copy)] pub struct BitpackedScheduler { @@ -1143,130 +600,20 @@ fn rows_in_buffer( #[cfg(test)] pub mod test { - use crate::{ - format::pb, - testing::{ArrayGeneratorProvider, TestCases, check_round_trip_encoding_generated}, - version::LanceFileVersion, - }; + use crate::testing::{ArrayGeneratorProvider, TestCases, check_round_trip_encoding_generated}; use super::*; - use std::{marker::PhantomData, sync::Arc}; + use std::marker::PhantomData; use arrow_array::{ - ArrayRef, Float32Array, Float64Array, Int8Array, Int16Array, Int32Array, Int64Array, - UInt8Array, UInt16Array, UInt32Array, UInt64Array, - types::{UInt8Type, UInt16Type}, + ArrowPrimitiveType, PrimitiveArray, + types::{Int16Type, Int32Type, Int64Type, UInt8Type, UInt32Type, UInt64Type}, }; - use arrow_schema::Field; - use lance_datagen::{ - ArrayGenerator, ArrayGeneratorExt, RowCount, - array::{fill, rand_with_distribution}, - gen_batch, - }; + use arrow_schema::{DataType, Field}; + use lance_datagen::{ArrayGenerator, array::rand_with_distribution}; use rand::distr::Uniform; - #[test] - fn test_bitpack_params() { - fn gen_array(generator: Box) -> ArrayRef { - gen_batch() - .anon_col(generator) - .into_batch_rows(RowCount::from(10000)) - .unwrap() - .column(0) - .clone() - } - - macro_rules! do_test { - ($num_bits:expr, $data_type:ident, $null_probability:expr) => { - let max = 1 << $num_bits - 1; - let mut arr = - gen_array(fill::<$data_type>(max).with_random_nulls($null_probability)); - - // ensure we don't randomly generate all nulls, that won't work - while arr.null_count() == arr.len() { - arr = gen_array(fill::<$data_type>(max).with_random_nulls($null_probability)); - } - let result = bitpack_params(arr.as_ref()); - assert!(result.is_some()); - assert_eq!($num_bits, result.unwrap().num_bits); - }; - } - - let test_cases = vec![ - (5u64, 0.0f64), - (5u64, 0.9f64), - (1u64, 0.0f64), - (1u64, 0.5f64), - (8u64, 0.0f64), - (8u64, 0.5f64), - ]; - - for (num_bits, null_probability) in &test_cases { - do_test!(*num_bits, UInt8Type, *null_probability); - do_test!(*num_bits, UInt16Type, *null_probability); - do_test!(*num_bits, UInt32Type, *null_probability); - do_test!(*num_bits, UInt64Type, *null_probability); - } - - // do some test cases that that will only work on larger types - let test_cases = vec![ - (13u64, 0.0f64), - (13u64, 0.5f64), - (16u64, 0.0f64), - (16u64, 0.5f64), - ]; - for (num_bits, null_probability) in &test_cases { - do_test!(*num_bits, UInt16Type, *null_probability); - do_test!(*num_bits, UInt32Type, *null_probability); - do_test!(*num_bits, UInt64Type, *null_probability); - } - let test_cases = vec![ - (25u64, 0.0f64), - (25u64, 0.5f64), - (32u64, 0.0f64), - (32u64, 0.5f64), - ]; - for (num_bits, null_probability) in &test_cases { - do_test!(*num_bits, UInt32Type, *null_probability); - do_test!(*num_bits, UInt64Type, *null_probability); - } - let test_cases = vec![ - (48u64, 0.0f64), - (48u64, 0.5f64), - (64u64, 0.0f64), - (64u64, 0.5f64), - ]; - for (num_bits, null_probability) in &test_cases { - do_test!(*num_bits, UInt64Type, *null_probability); - } - - // test that it returns None for datatypes that don't support bitpacking - let arr = Float64Array::from_iter_values(vec![0.1, 0.2, 0.3]); - let result = bitpack_params(&arr); - assert!(result.is_none()); - } - - #[test] - fn test_num_compressed_bits_signed_types() { - let values = Int32Array::from(vec![1, 2, -7]); - let arr = values; - let result = bitpack_params(&arr); - assert!(result.is_some()); - let result = result.unwrap(); - assert_eq!(4, result.num_bits); - assert!(result.signed); - - // check that it doesn't add a sign bit if it doesn't need to - let values = Int32Array::from(vec![1, 2, 7]); - let arr = values; - let result = bitpack_params(&arr); - assert!(result.is_some()); - let result = result.unwrap(); - assert_eq!(3, result.num_bits); - assert!(!result.signed); - } - #[test] fn test_rows_in_buffer() { let test_cases = vec![ @@ -1308,157 +655,6 @@ pub mod test { assert_eq!(StartOffset::SkipFull(8), result); } - #[test_log::test(test)] - fn test_will_bitpack_allowed_types_when_possible() { - let test_cases: Vec<(DataType, ArrayRef, u64)> = vec![ - ( - DataType::UInt8, - Arc::new(UInt8Array::from_iter_values(vec![0, 1, 2, 3, 4, 5])), - 3, // bits per value - ), - ( - DataType::UInt16, - Arc::new(UInt16Array::from_iter_values(vec![0, 1, 2, 3, 4, 5 << 8])), - 11, - ), - ( - DataType::UInt32, - Arc::new(UInt32Array::from_iter_values(vec![0, 1, 2, 3, 4, 5 << 16])), - 19, - ), - ( - DataType::UInt64, - Arc::new(UInt64Array::from_iter_values(vec![0, 1, 2, 3, 4, 5 << 32])), - 35, - ), - ( - DataType::Int8, - Arc::new(Int8Array::from_iter_values(vec![0, 2, 3, 4, -5])), - 4, - ), - ( - // check it will not pack with signed bit if all values of signed type are positive - DataType::Int8, - Arc::new(Int8Array::from_iter_values(vec![0, 2, 3, 4, 5])), - 3, - ), - ( - DataType::Int16, - Arc::new(Int16Array::from_iter_values(vec![0, 1, 2, 3, -4, 5 << 8])), - 12, - ), - ( - DataType::Int32, - Arc::new(Int32Array::from_iter_values(vec![0, 1, 2, 3, 4, -5 << 16])), - 20, - ), - ( - DataType::Int64, - Arc::new(Int64Array::from_iter_values(vec![ - 0, - 1, - 2, - -3, - -4, - -5 << 32, - ])), - 36, - ), - ]; - - for (data_type, arr, bits_per_value) in test_cases { - let mut buffed_index = 1; - let params = bitpack_params(arr.as_ref()).unwrap(); - let encoder = BitpackedArrayEncoder { - num_bits: params.num_bits, - signed_type: params.signed, - }; - let data = DataBlock::from_array(arr); - let result = encoder.encode(data, &data_type, &mut buffed_index).unwrap(); - - let data = result.data.as_fixed_width().unwrap(); - assert_eq!(bits_per_value, data.bits_per_value); - - let array_encoding = result.encoding.array_encoding.unwrap(); - - match array_encoding { - pb::array_encoding::ArrayEncoding::Bitpacked(bitpacked) => { - assert_eq!(bits_per_value, bitpacked.compressed_bits_per_value); - assert_eq!( - (data_type.byte_width() * 8) as u64, - bitpacked.uncompressed_bits_per_value - ); - } - _ => { - panic!("Array did not use bitpacking encoding") - } - } - } - - // check it will otherwise use flat encoding - let test_cases: Vec<(DataType, ArrayRef)> = vec![ - // it should use flat encoding for datatypes that don't support bitpacking - ( - DataType::Float32, - Arc::new(Float32Array::from_iter_values(vec![0.1, 0.2, 0.3])), - ), - // it should still use flat encoding if bitpacked encoding would be packed - // into the full byte range - ( - DataType::UInt8, - Arc::new(UInt8Array::from_iter_values(vec![0, 1, 2, 3, 4, 250])), - ), - ( - DataType::UInt16, - Arc::new(UInt16Array::from_iter_values(vec![0, 1, 2, 3, 4, 250 << 8])), - ), - ( - DataType::UInt32, - Arc::new(UInt32Array::from_iter_values(vec![ - 0, - 1, - 2, - 3, - 4, - 250 << 24, - ])), - ), - ( - DataType::UInt64, - Arc::new(UInt64Array::from_iter_values(vec![ - 0, - 1, - 2, - 3, - 4, - 250 << 56, - ])), - ), - ( - DataType::Int8, - Arc::new(Int8Array::from_iter_values(vec![-100])), - ), - ( - DataType::Int16, - Arc::new(Int16Array::from_iter_values(vec![-100 << 8])), - ), - ( - DataType::Int32, - Arc::new(Int32Array::from_iter_values(vec![-100 << 24])), - ), - ( - DataType::Int64, - Arc::new(Int64Array::from_iter_values(vec![-100 << 56])), - ), - ]; - - for (data_type, arr) in test_cases { - if let Some(params) = bitpack_params(arr.as_ref()) { - assert_eq!(params.num_bits, data_type.byte_width() as u64 * 8); - } - } - } - struct DistributionArrayGeneratorProvider< DataType, Dist: rand::distr::Distribution + Clone + Send + Sync + 'static, @@ -1677,7 +873,7 @@ pub mod test { for (data_type, array_gen_provider) in bitpacked_test_cases { let field = Field::new("", data_type.clone(), false); - let test_cases = TestCases::basic().with_min_file_version(LanceFileVersion::V2_1); + let test_cases = TestCases::basic().with_structural_encodings(); check_round_trip_encoding_generated(field, array_gen_provider.copy(), test_cases).await; } } diff --git a/rust/lance-encoding/src/previous/encodings/physical/block.rs b/rust/lance-encoding/src/array_encoding/physical/block.rs similarity index 97% rename from rust/lance-encoding/src/previous/encodings/physical/block.rs rename to rust/lance-encoding/src/array_encoding/physical/block.rs index 517bce4e3a7..3bbd966e24a 100644 --- a/rust/lance-encoding/src/previous/encodings/physical/block.rs +++ b/rust/lance-encoding/src/array_encoding/physical/block.rs @@ -5,9 +5,9 @@ use arrow_schema::DataType; use crate::{ data::{BlockInfo, DataBlock, OpaqueBlock}, + encoder::{ArrayEncoder, EncodedArray}, encodings::physical::block::{CompressedBufferEncoder, CompressionConfig, CompressionScheme}, format::ProtobufUtils, - previous::encoder::{ArrayEncoder, EncodedArray}, }; use lance_core::Result; diff --git a/rust/lance-encoding/src/previous/encodings/physical/dictionary.rs b/rust/lance-encoding/src/array_encoding/physical/dictionary.rs similarity index 96% rename from rust/lance-encoding/src/previous/encodings/physical/dictionary.rs rename to rust/lance-encoding/src/array_encoding/physical/dictionary.rs index a4c01ff9ef5..a59879bc5cb 100644 --- a/rust/lance-encoding/src/previous/encodings/physical/dictionary.rs +++ b/rust/lance-encoding/src/array_encoding/physical/dictionary.rs @@ -16,18 +16,18 @@ use lance_arrow::DataTypeExt; use lance_core::{Error, Result}; use std::collections::HashMap; +use crate::array_encoding::logical::primitive::PrimitiveFieldDecoder; use crate::buffer::LanceBuffer; use crate::data::{ BlockInfo, DataBlock, DictionaryDataBlock, FixedWidthDataBlock, NullableDataBlock, VariableWidthBlock, }; +use crate::decoder::LogicalPageDecoder; use crate::format::ProtobufUtils; -use crate::previous::decoder::LogicalPageDecoder; -use crate::previous::encodings::logical::primitive::PrimitiveFieldDecoder; use crate::{ EncodingsIo, decoder::{PageScheduler, PrimitivePageDecoder}, - previous::encoder::{ArrayEncoder, EncodedArray}, + encoder::{ArrayEncoder, EncodedArray}, }; #[derive(Debug)] @@ -259,6 +259,16 @@ impl ArrayEncoder for AlreadyDictionaryEncoder { } _ => panic!("Expected dictionary data"), }; + let declared_key_bits = key_type.byte_width() as u64 * 8; + if dict_data.indices.bits_per_value != declared_key_bits { + return Err(Error::invalid_input(format!( + "dictionary indices use {} bits but the declared {} key type uses {} bits; the normalized dictionary has {} values", + dict_data.indices.bits_per_value, + key_type, + declared_key_bits, + dict_data.dictionary.num_values() + ))); + } let num_dictionary_items = dict_data.dictionary.num_values() as u32; let encoded_indices = self.indices_encoder.encode( diff --git a/rust/lance-encoding/src/previous/encodings/physical/fixed_size_binary.rs b/rust/lance-encoding/src/array_encoding/physical/fixed_size_binary.rs similarity index 87% rename from rust/lance-encoding/src/previous/encodings/physical/fixed_size_binary.rs rename to rust/lance-encoding/src/array_encoding/physical/fixed_size_binary.rs index 696edde8c9b..cd5cfd70616 100644 --- a/rust/lance-encoding/src/previous/encodings/physical/fixed_size_binary.rs +++ b/rust/lance-encoding/src/array_encoding/physical/fixed_size_binary.rs @@ -4,17 +4,14 @@ use std::sync::Arc; use arrow_buffer::ScalarBuffer; -use arrow_schema::DataType; use futures::{FutureExt, future::BoxFuture}; use lance_core::Result; use crate::{ EncodingsIo, buffer::LanceBuffer, - data::{BlockInfo, DataBlock, FixedWidthDataBlock, VariableWidthBlock}, + data::{BlockInfo, DataBlock, VariableWidthBlock}, decoder::{PageScheduler, PrimitivePageDecoder}, - format::ProtobufUtils, - previous::encoder::{ArrayEncoder, EncodedArray}, }; /// A scheduler for fixed size binary data @@ -116,51 +113,6 @@ impl PrimitivePageDecoder for FixedSizeBinaryDecoder { } } -#[derive(Debug)] -pub struct FixedSizeBinaryEncoder { - bytes_encoder: Box, - byte_width: usize, -} - -impl FixedSizeBinaryEncoder { - pub fn new(bytes_encoder: Box, byte_width: usize) -> Self { - Self { - bytes_encoder, - byte_width, - } - } -} - -impl ArrayEncoder for FixedSizeBinaryEncoder { - fn encode( - &self, - data: DataBlock, - _data_type: &DataType, - buffer_index: &mut u32, - ) -> Result { - let bytes_data = data.as_variable_width().unwrap(); - let fixed_data = DataBlock::FixedWidth(FixedWidthDataBlock { - bits_per_value: 8 * self.byte_width as u64, - data: bytes_data.data, - num_values: bytes_data.num_values, - block_info: BlockInfo::new(), - }); - - let encoded_data = self.bytes_encoder.encode( - fixed_data, - &DataType::FixedSizeBinary(self.byte_width as i32), - buffer_index, - )?; - let encoding = - ProtobufUtils::fixed_size_binary(encoded_data.encoding, self.byte_width as u32); - - Ok(EncodedArray { - data: encoded_data.data, - encoding, - }) - } -} - #[cfg(test)] mod tests { use std::{collections::HashMap, sync::Arc}; @@ -173,9 +125,9 @@ mod tests { use arrow_data::ArrayData; use arrow_schema::{DataType, Field}; + use crate::array_encoding::physical::fixed_size_binary::FixedSizeBinaryDecoder; use crate::data::{DataBlock, FixedWidthDataBlock}; use crate::decoder::PrimitivePageDecoder; - use crate::previous::encodings::physical::fixed_size_binary::FixedSizeBinaryDecoder; use crate::testing::{TestCases, check_basic_random, check_round_trip_encoding_of_data}; #[test_log::test(tokio::test)] diff --git a/rust/lance-encoding/src/previous/encodings/physical/fixed_size_list.rs b/rust/lance-encoding/src/array_encoding/physical/fixed_size_list.rs similarity index 95% rename from rust/lance-encoding/src/previous/encodings/physical/fixed_size_list.rs rename to rust/lance-encoding/src/array_encoding/physical/fixed_size_list.rs index e980301d117..a0f596fd8cf 100644 --- a/rust/lance-encoding/src/previous/encodings/physical/fixed_size_list.rs +++ b/rust/lance-encoding/src/array_encoding/physical/fixed_size_list.rs @@ -12,8 +12,8 @@ use crate::{ EncodingsIo, data::{DataBlock, FixedSizeListBlock}, decoder::{PageScheduler, PrimitivePageDecoder}, + encoder::{ArrayEncoder, EncodedArray}, format::ProtobufUtils, - previous::encoder::{ArrayEncoder, EncodedArray}, }; /// A scheduler for fixed size lists of primitive values @@ -138,10 +138,7 @@ mod tests { use arrow_schema::{DataType, Field}; use lance_datagen::{ArrayGeneratorExt, RowCount, array, gen_array}; - use crate::{ - testing::{TestCases, check_basic_random, check_round_trip_encoding_of_data}, - version::LanceFileVersion, - }; + use crate::testing::{TestCases, check_basic_random, check_round_trip_encoding_of_data}; const PRIMITIVE_TYPES: &[DataType] = &[DataType::Int8, DataType::Float32, DataType::Float64]; @@ -182,7 +179,7 @@ mod tests { .with_indices(vec![0, 1, 2]) .with_indices(vec![1]) .with_indices(vec![2]) - .with_min_file_version(LanceFileVersion::V2_1); + .with_structural_encodings(); check_round_trip_encoding_of_data(vec![list], &test_cases, HashMap::default()).await; } @@ -209,7 +206,7 @@ mod tests { .with_indices(vec![0, 1, 2]) .with_indices(vec![1]) .with_indices(vec![2]) - .with_min_file_version(LanceFileVersion::V2_1); + .with_structural_encodings(); check_round_trip_encoding_of_data(vec![list], &test_cases, HashMap::default()).await; } @@ -260,7 +257,7 @@ mod tests { .with_range(1..3) .with_indices(vec![0, 1, 2]) .with_indices(vec![2]) - .with_min_file_version(LanceFileVersion::V2_1); + .with_structural_encodings(); check_round_trip_encoding_of_data(vec![outer_list], &test_cases, HashMap::default()).await; } diff --git a/rust/lance-encoding/src/previous/encodings/physical/fsst.rs b/rust/lance-encoding/src/array_encoding/physical/fsst.rs similarity index 99% rename from rust/lance-encoding/src/previous/encodings/physical/fsst.rs rename to rust/lance-encoding/src/array_encoding/physical/fsst.rs index e9bb585ed73..fd1f11b860e 100644 --- a/rust/lance-encoding/src/previous/encodings/physical/fsst.rs +++ b/rust/lance-encoding/src/array_encoding/physical/fsst.rs @@ -14,8 +14,8 @@ use crate::{ buffer::LanceBuffer, data::{BlockInfo, DataBlock, NullableDataBlock, VariableWidthBlock}, decoder::{PageScheduler, PrimitivePageDecoder}, + encoder::{ArrayEncoder, EncodedArray}, format::ProtobufUtils, - previous::encoder::{ArrayEncoder, EncodedArray}, }; #[derive(Debug)] diff --git a/rust/lance-encoding/src/previous/encodings/physical/packed_struct.rs b/rust/lance-encoding/src/array_encoding/physical/packed_struct.rs similarity index 99% rename from rust/lance-encoding/src/previous/encodings/physical/packed_struct.rs rename to rust/lance-encoding/src/array_encoding/physical/packed_struct.rs index 0f3a5fc3832..b608071e9b0 100644 --- a/rust/lance-encoding/src/previous/encodings/physical/packed_struct.rs +++ b/rust/lance-encoding/src/array_encoding/physical/packed_struct.rs @@ -18,7 +18,7 @@ use crate::{ buffer::LanceBuffer, data::{DataBlock, FixedWidthDataBlock, StructDataBlock}, decoder::{PageScheduler, PrimitivePageDecoder}, - previous::encoder::{ArrayEncoder, EncodedArray}, + encoder::{ArrayEncoder, EncodedArray}, }; #[derive(Debug)] diff --git a/rust/lance-encoding/src/previous/encodings/physical/value.rs b/rust/lance-encoding/src/array_encoding/physical/value.rs similarity index 99% rename from rust/lance-encoding/src/previous/encodings/physical/value.rs rename to rust/lance-encoding/src/array_encoding/physical/value.rs index 92ec3240a14..9f594a00252 100644 --- a/rust/lance-encoding/src/previous/encodings/physical/value.rs +++ b/rust/lance-encoding/src/array_encoding/physical/value.rs @@ -18,7 +18,7 @@ use crate::format::ProtobufUtils; use crate::{ EncodingsIo, decoder::{PageScheduler, PrimitivePageDecoder}, - previous::encoder::{ArrayEncoder, EncodedArray}, + encoder::{ArrayEncoder, EncodedArray}, }; use lance_core::{Error, Result}; diff --git a/rust/lance-encoding/src/previous/encoder.rs b/rust/lance-encoding/src/array_encoding/strategy.rs similarity index 64% rename from rust/lance-encoding/src/previous/encoder.rs rename to rust/lance-encoding/src/array_encoding/strategy.rs index 9c314ae97e9..1e80ebb7ed8 100644 --- a/rust/lance-encoding/src/previous/encoder.rs +++ b/rust/lance-encoding/src/array_encoding/strategy.rs @@ -3,23 +3,14 @@ use std::{collections::HashMap, env, hash::RandomState, sync::Arc}; -use arrow_array::{ArrayRef, UInt8Array, cast::AsArray}; +#[cfg(test)] +use arrow_array::cast::AsArray; +use arrow_array::{ArrayRef, UInt8Array}; use arrow_schema::DataType; use hyperloglogplus::{HyperLogLog, HyperLogLogPlus}; use crate::{ - buffer::LanceBuffer, - data::DataBlock, - encoder::{ColumnIndexSequence, EncodingOptions, FieldEncoder, FieldEncodingStrategy}, - encodings::{ - logical::r#struct::StructFieldEncoder, - physical::{ - block::{CompressionConfig, CompressionScheme}, - value::ValueEncoder, - }, - }, - format::pb, - previous::encodings::{ + array_encoding::{ logical::{ blob::BlobFieldEncoder, list::ListFieldEncoder, primitive::PrimitiveFieldEncoder, }, @@ -27,98 +18,49 @@ use crate::{ basic::BasicEncoder, binary::BinaryEncoder, dictionary::{AlreadyDictionaryEncoder, DictionaryEncoder}, - fixed_size_binary::FixedSizeBinaryEncoder, fixed_size_list::FslEncoder, fsst::FsstArrayEncoder, packed_struct::PackedStructEncoder, }, }, - version::LanceFileVersion, -}; - -#[cfg(feature = "bitpacking")] -use crate::previous::encodings::physical::bitpack::{ - BitpackedForNonNegArrayEncoder, compute_compressed_bit_width_for_non_neg, -}; - -use crate::constants::{ - COMPRESSION_LEVEL_META_KEY, COMPRESSION_META_KEY, PACKED_STRUCT_LEGACY_META_KEY, - PACKED_STRUCT_META_KEY, + constants::{ + COMPRESSION_LEVEL_META_KEY, COMPRESSION_META_KEY, PACKED_STRUCT_LEGACY_META_KEY, + PACKED_STRUCT_META_KEY, + }, + encoder::{ + ArrayEncoder, ArrayEncodingStrategy, ColumnIndexSequence, FieldEncoder, + FieldEncodingContext, FieldEncodingStrategy, + }, + encodings::{ + logical::r#struct::StructFieldEncoder, + physical::{ + block::{CompressionConfig, CompressionScheme}, + value::ValueEncoder, + }, + }, }; use lance_arrow::BLOB_META_KEY; use lance_core::datatypes::{BLOB_DESC_FIELD, Field}; use lance_core::{Error, Result}; -/// An encoded array -/// -/// Maps to a single Arrow array -/// -/// This contains the encoded data as well as a description of the encoding that was applied which -/// can be used to decode the data later. +/// Field-to-column composition for the `pb::ArrayEncoding` grammar. #[derive(Debug)] -pub struct EncodedArray { - /// The encoded buffers - pub data: DataBlock, - /// A description of the encoding used to encode the array - pub encoding: pb::ArrayEncoding, +pub struct ArrayFieldEncodingStrategy { + array_encoding_strategy: Arc, } -impl EncodedArray { - pub fn new(data: DataBlock, encoding: pb::ArrayEncoding) -> Self { - Self { data, encoding } - } - - pub fn into_buffers(self) -> (Vec, pb::ArrayEncoding) { - let buffers = self.data.into_buffers(); - (buffers, self.encoding) - } -} - -/// Encodes data from one format to another (hopefully more compact or useful) format -/// -/// The array encoder must be Send + Sync. Encoding is always done on its own -/// thread task in the background and there could potentially be multiple encode -/// tasks running for a column at once. -pub trait ArrayEncoder: std::fmt::Debug + Send + Sync { - /// Encode data +impl ArrayFieldEncodingStrategy { + /// Create the field strategy for the `pb::ArrayEncoding` grammar. /// - /// The result should contain a description of the encoding that was chosen. - /// This can be used to decode the data later. - fn encode( - &self, - data: DataBlock, - data_type: &DataType, - buffer_index: &mut u32, - ) -> Result; -} - -/// A trait to pick which encoding strategy to use for a single page -/// of data -/// -/// Presumably, implementations will make encoding decisions based on -/// array statistics. -pub trait ArrayEncodingStrategy: Send + Sync + std::fmt::Debug { - fn create_array_encoder( - &self, - arrays: &[ArrayRef], - field: &Field, - ) -> Result>; -} - -/// The core field encoding strategy is a set of basic encodings that -/// are generally applicable in most scenarios. -#[derive(Debug)] -pub struct CoreFieldEncodingStrategy { - pub array_encoding_strategy: Arc, - pub version: LanceFileVersion, -} - -impl CoreFieldEncodingStrategy { - pub fn new(version: LanceFileVersion) -> Self { + /// ``` + /// use lance_encoding::encoder::ArrayFieldEncodingStrategy; + /// + /// let strategy = ArrayFieldEncodingStrategy::new(); + /// ``` + pub fn new() -> Self { Self { - array_encoding_strategy: Arc::new(CoreArrayEncodingStrategy::new(version)), - version, + array_encoding_strategy: Arc::new(ArrayStrategy), } } @@ -157,14 +99,20 @@ impl CoreFieldEncodingStrategy { } } -impl FieldEncodingStrategy for CoreFieldEncodingStrategy { +impl Default for ArrayFieldEncodingStrategy { + fn default() -> Self { + Self::new() + } +} + +impl FieldEncodingStrategy for ArrayFieldEncodingStrategy { fn create_field_encoder( &self, - encoding_strategy_root: &dyn FieldEncodingStrategy, field: &Field, column_index: &mut ColumnIndexSequence, - options: &EncodingOptions, + context: &FieldEncodingContext<'_>, ) -> Result> { + let options = context.options; let data_type = field.data_type(); if Self::is_primitive_type(&data_type) { let column_index = column_index.next_column_index(field.id as u32); @@ -192,11 +140,10 @@ impl FieldEncodingStrategy for CoreFieldEncodingStrategy { match data_type { DataType::List(_child) | DataType::LargeList(_child) => { let list_idx = column_index.next_column_index(field.id as u32); - let inner_encoding = encoding_strategy_root.create_field_encoder( - encoding_strategy_root, + let inner_encoding = context.strategy.create_field_encoder( &field.children[0], column_index, - options, + context, )?; let offsets_encoder = Arc::new(BasicEncoder::new(Box::new(ValueEncoder::default()))); @@ -227,12 +174,9 @@ impl FieldEncodingStrategy for CoreFieldEncodingStrategy { .children .iter() .map(|field| { - self.create_field_encoder( - encoding_strategy_root, - field, - column_index, - options, - ) + context + .strategy + .create_field_encoder(field, column_index, context) }) .collect::>>()?; Ok(Box::new(StructFieldEncoder::new( @@ -259,39 +203,24 @@ impl FieldEncodingStrategy for CoreFieldEncodingStrategy { Err(Error::not_supported_source(format!("cannot encode a dictionary column whose value type is a logical type ({})", value_type).into())) } } - _ => todo!("Implement encoding for field {}", field), + _ => Err(Error::not_supported_source( + format!( + "Lance v2.0 has no field encoding for '{}' with data type {}", + field.name, + field.data_type() + ) + .into(), + )), } } } } -/// The core array encoding strategy is a set of basic encodings that -/// are generally applicable in most scenarios. +/// Page-encoding selection for the `pb::ArrayEncoding` grammar. #[derive(Debug)] -pub struct CoreArrayEncodingStrategy { - pub version: LanceFileVersion, -} - -const BINARY_DATATYPES: [DataType; 4] = [ - DataType::Binary, - DataType::LargeBinary, - DataType::Utf8, - DataType::LargeUtf8, -]; - -impl CoreArrayEncodingStrategy { - fn new(version: LanceFileVersion) -> Self { - Self { version } - } -} - -impl CoreArrayEncodingStrategy { - fn can_use_fsst(data_type: &DataType, data_size: u64, version: LanceFileVersion) -> bool { - version >= LanceFileVersion::V2_1 - && matches!(data_type, DataType::Utf8 | DataType::Binary) - && data_size > 4 * 1024 * 1024 - } +struct ArrayStrategy; +impl ArrayStrategy { fn get_field_compression(field_meta: &HashMap) -> Option { let compression = field_meta.get(COMPRESSION_META_KEY)?; let compression_scheme = compression.parse::(); @@ -308,16 +237,14 @@ impl CoreArrayEncodingStrategy { fn default_binary_encoder( arrays: &[ArrayRef], - data_type: &DataType, field_meta: Option<&HashMap>, data_size: u64, - version: LanceFileVersion, ) -> Result> { let bin_indices_encoder = - Self::choose_array_encoder(arrays, &DataType::UInt64, data_size, false, version, None)?; + Self::choose_array_encoder(arrays, &DataType::UInt64, data_size, false, None)?; if let Some(compression) = field_meta.and_then(Self::get_field_compression) { - if compression.scheme == CompressionScheme::Fsst { + if compression.scheme() == CompressionScheme::Fsst { // User requested FSST let raw_encoder = Box::new(BinaryEncoder::try_new(bin_indices_encoder, None)?); Ok(Box::new(FsstArrayEncoder::new(raw_encoder))) @@ -329,13 +256,7 @@ impl CoreArrayEncodingStrategy { )?)) } } else { - // No user-specified compression, use FSST if we can - let bin_encoder = Box::new(BinaryEncoder::try_new(bin_indices_encoder, None)?); - if Self::can_use_fsst(data_type, data_size, version) { - Ok(Box::new(FsstArrayEncoder::new(bin_encoder))) - } else { - Ok(bin_encoder) - } + Ok(Box::new(BinaryEncoder::try_new(bin_indices_encoder, None)?)) } } @@ -344,7 +265,6 @@ impl CoreArrayEncodingStrategy { data_type: &DataType, data_size: u64, use_dict_encoding: bool, - version: LanceFileVersion, field_meta: Option<&HashMap>, ) -> Result> { match data_type { @@ -355,7 +275,6 @@ impl CoreArrayEncodingStrategy { inner.data_type(), data_size, use_dict_encoding, - version, None, )?, *dimension as u32, @@ -363,10 +282,9 @@ impl CoreArrayEncodingStrategy { } DataType::Dictionary(key_type, value_type) => { let key_encoder = - Self::choose_array_encoder(arrays, key_type, data_size, false, version, None)?; - let value_encoder = Self::choose_array_encoder( - arrays, value_type, data_size, false, version, None, - )?; + Self::choose_array_encoder(arrays, key_type, data_size, false, None)?; + let value_encoder = + Self::choose_array_encoder(arrays, value_type, data_size, false, None)?; Ok(Box::new(AlreadyDictionaryEncoder::new( key_encoder, @@ -384,7 +302,6 @@ impl CoreArrayEncodingStrategy { &DataType::UInt8, data_size, false, - version, None, )?; let dict_items_encoder = Self::choose_array_encoder( @@ -392,7 +309,6 @@ impl CoreArrayEncodingStrategy { &DataType::Utf8, data_size, false, - version, None, )?; @@ -400,31 +316,8 @@ impl CoreArrayEncodingStrategy { dict_indices_encoder, dict_items_encoder, ))) - } - // The parent datatype should be binary or utf8 to use the fixed size encoding - // The variable 'data_type' is passed through recursion so comparing with it would be incorrect - else if BINARY_DATATYPES.contains(arrays[0].data_type()) { - if let Some(byte_width) = check_fixed_size_encoding(arrays, version) { - // use FixedSizeBinaryEncoder - let bytes_encoder = Self::choose_array_encoder( - arrays, - &DataType::UInt8, - data_size, - false, - version, - None, - )?; - - Ok(Box::new(BasicEncoder::new(Box::new( - FixedSizeBinaryEncoder::new(bytes_encoder, byte_width as usize), - )))) - } else { - Self::default_binary_encoder( - arrays, data_type, field_meta, data_size, version, - ) - } } else { - Self::default_binary_encoder(arrays, data_type, field_meta, data_size, version) + Self::default_binary_encoder(arrays, field_meta, data_size) } } DataType::Struct(fields) => { @@ -438,7 +331,6 @@ impl CoreArrayEncodingStrategy { inner_datatype, data_size, use_dict_encoding, - version, None, )?; inner_encoders.push(inner_encoder); @@ -446,54 +338,16 @@ impl CoreArrayEncodingStrategy { Ok(Box::new(PackedStructEncoder::new(inner_encoders))) } - DataType::UInt8 | DataType::UInt16 | DataType::UInt32 | DataType::UInt64 => { - if version >= LanceFileVersion::V2_1 && arrays[0].data_type() == data_type { - #[cfg(feature = "bitpacking")] - { - let compressed_bit_width = compute_compressed_bit_width_for_non_neg(arrays); - Ok(Box::new(BitpackedForNonNegArrayEncoder::new( - compressed_bit_width as usize, - data_type.clone(), - ))) - } - #[cfg(not(feature = "bitpacking"))] - { - Ok(Box::new(BasicEncoder::new(Box::new( - ValueEncoder::default(), - )))) - } - } else { - Ok(Box::new(BasicEncoder::new(Box::new( - ValueEncoder::default(), - )))) - } - } + DataType::UInt8 | DataType::UInt16 | DataType::UInt32 | DataType::UInt64 => Ok( + Box::new(BasicEncoder::new(Box::new(ValueEncoder::default()))), + ), // TODO: for signed integers, I intend to make it a cascaded encoding, a sparse array for the negative values and very wide(bit-width) values, // then a bitpacked array for the narrow(bit-width) values, I need `BitpackedForNeg` to be merged first, I am // thinking about putting this sparse array in the metadata so bitpacking remain using one page buffer only. - DataType::Int8 | DataType::Int16 | DataType::Int32 | DataType::Int64 => { - if version >= LanceFileVersion::V2_1 && arrays[0].data_type() == data_type { - #[cfg(feature = "bitpacking")] - { - let compressed_bit_width = compute_compressed_bit_width_for_non_neg(arrays); - Ok(Box::new(BitpackedForNonNegArrayEncoder::new( - compressed_bit_width as usize, - data_type.clone(), - ))) - } - #[cfg(not(feature = "bitpacking"))] - { - Ok(Box::new(BasicEncoder::new(Box::new( - ValueEncoder::default(), - )))) - } - } else { - Ok(Box::new(BasicEncoder::new(Box::new( - ValueEncoder::default(), - )))) - } - } + DataType::Int8 | DataType::Int16 | DataType::Int32 | DataType::Int64 => Ok(Box::new( + BasicEncoder::new(Box::new(ValueEncoder::default())), + )), _ => Ok(Box::new(BasicEncoder::new(Box::new( ValueEncoder::default(), )))), @@ -539,8 +393,9 @@ fn check_dict_encoding(arrays: &[ArrayRef], threshold: u64) -> bool { true } -fn check_fixed_size_encoding(arrays: &[ArrayRef], version: LanceFileVersion) -> Option { - if version < LanceFileVersion::V2_1 || arrays.is_empty() { +#[cfg(test)] +fn check_fixed_size_encoding(arrays: &[ArrayRef]) -> Option { + if arrays.is_empty() { return None; } @@ -612,7 +467,7 @@ fn check_fixed_size_encoding(arrays: &[ArrayRef], version: LanceFileVersion) -> } } -impl ArrayEncodingStrategy for CoreArrayEncodingStrategy { +impl ArrayEncodingStrategy for ArrayStrategy { fn create_array_encoder( &self, arrays: &[ArrayRef], @@ -632,7 +487,6 @@ impl ArrayEncodingStrategy for CoreArrayEncodingStrategy { data_type, data_size, use_dict_encoding, - self.version, Some(&field.metadata), ) } @@ -640,17 +494,48 @@ impl ArrayEncodingStrategy for CoreArrayEncodingStrategy { #[cfg(test)] mod tests { - use crate::constants::{COMPRESSION_LEVEL_META_KEY, COMPRESSION_META_KEY}; - use crate::previous::encoder::{ - ArrayEncodingStrategy, CoreArrayEncodingStrategy, check_dict_encoding, + use super::{ + ArrayEncodingStrategy, ArrayFieldEncodingStrategy, ArrayStrategy, check_dict_encoding, check_fixed_size_encoding, }; - use crate::version::LanceFileVersion; + use crate::constants::{COMPRESSION_LEVEL_META_KEY, COMPRESSION_META_KEY}; + use crate::encoder::{BatchEncoder, EncodingOptions}; use arrow_array::{ArrayRef, StringArray}; - use arrow_schema::Field; + use arrow_schema::{DataType, Field, Fields, Schema as ArrowSchema}; + use lance_core::{Error, datatypes::Schema}; use std::collections::HashMap; use std::sync::Arc; + #[test] + fn test_unsupported_field_type_returns_error() { + let entries = Field::new( + "entries", + DataType::Struct(Fields::from(vec![ + Field::new("key", DataType::Utf8, false), + Field::new("value", DataType::Int32, true), + ])), + false, + ); + let arrow_schema = ArrowSchema::new(vec![Field::new( + "attributes", + DataType::Map(Arc::new(entries), false), + true, + )]); + let schema = Schema::try_from(&arrow_schema).unwrap(); + + let error = BatchEncoder::try_new( + &schema, + &ArrayFieldEncodingStrategy::new(), + &EncodingOptions::default(), + ) + .err() + .unwrap(); + + assert!(matches!(error, Error::NotSupported { .. })); + assert!(error.to_string().contains("attributes")); + assert!(error.to_string().contains("Map")); + } + fn is_dict_encoding_applicable(arr: Vec>, threshold: u64) -> bool { let arr = StringArray::from(arr); let arr = Arc::new(arr) as ArrayRef; @@ -691,10 +576,7 @@ mod tests { assert!(!is_dict_encoding_applicable(vec![Some("a"), Some("a")], 3)); } - fn is_fixed_size_encoding_applicable( - arrays: Vec>>, - version: LanceFileVersion, - ) -> bool { + fn is_fixed_size_encoding_applicable(arrays: Vec>>) -> bool { let mut final_arrays = Vec::new(); for arr in arrays { let arr = StringArray::from(arr); @@ -702,82 +584,78 @@ mod tests { final_arrays.push(arr); } - check_fixed_size_encoding(&final_arrays.clone(), version).is_some() + check_fixed_size_encoding(&final_arrays).is_some() } #[test] fn test_fixed_size_binary_encoding_applicable() { - assert!(!is_fixed_size_encoding_applicable( - vec![vec![]], - LanceFileVersion::V2_1 - )); - - assert!(is_fixed_size_encoding_applicable( - vec![vec![Some("a"), Some("b")]], - LanceFileVersion::V2_1 - )); - - assert!(!is_fixed_size_encoding_applicable( - vec![vec![Some("abc"), Some("de")]], - LanceFileVersion::V2_1 - )); - - assert!(is_fixed_size_encoding_applicable( - vec![vec![Some("pqr"), None]], - LanceFileVersion::V2_1 - )); - - assert!(!is_fixed_size_encoding_applicable( - vec![vec![Some("pqr"), Some("")]], - LanceFileVersion::V2_1 - )); - - assert!(!is_fixed_size_encoding_applicable( - vec![vec![Some(""), Some("")]], - LanceFileVersion::V2_1 - )); + assert!(!is_fixed_size_encoding_applicable(vec![vec![]])); + + assert!(is_fixed_size_encoding_applicable(vec![vec![ + Some("a"), + Some("b") + ]])); + + assert!(!is_fixed_size_encoding_applicable(vec![vec![ + Some("abc"), + Some("de") + ]])); + + assert!(is_fixed_size_encoding_applicable(vec![vec![ + Some("pqr"), + None + ]])); + + assert!(!is_fixed_size_encoding_applicable(vec![vec![ + Some("pqr"), + Some("") + ]])); + + assert!(!is_fixed_size_encoding_applicable(vec![vec![ + Some(""), + Some("") + ]])); } #[test] fn test_fixed_size_binary_encoding_applicable_multiple_arrays() { - assert!(is_fixed_size_encoding_applicable( - vec![vec![Some("a"), Some("b")], vec![Some("c"), Some("d")]], - LanceFileVersion::V2_1 - )); - - assert!(!is_fixed_size_encoding_applicable( - vec![vec![Some("ab"), Some("bc")], vec![Some("c"), Some("d")]], - LanceFileVersion::V2_1 - )); - - assert!(!is_fixed_size_encoding_applicable( - vec![vec![Some("ab"), None], vec![None, Some("d")]], - LanceFileVersion::V2_1 - )); - - assert!(is_fixed_size_encoding_applicable( - vec![vec![Some("a"), None], vec![None, Some("d")]], - LanceFileVersion::V2_1 - )); - - assert!(!is_fixed_size_encoding_applicable( - vec![vec![Some(""), None], vec![None, Some("")]], - LanceFileVersion::V2_1 - )); - - assert!(!is_fixed_size_encoding_applicable( - vec![vec![None, None], vec![None, None]], - LanceFileVersion::V2_1 - )); + assert!(is_fixed_size_encoding_applicable(vec![ + vec![Some("a"), Some("b")], + vec![Some("c"), Some("d")] + ])); + + assert!(!is_fixed_size_encoding_applicable(vec![ + vec![Some("ab"), Some("bc")], + vec![Some("c"), Some("d")] + ])); + + assert!(!is_fixed_size_encoding_applicable(vec![ + vec![Some("ab"), None], + vec![None, Some("d")] + ])); + + assert!(is_fixed_size_encoding_applicable(vec![ + vec![Some("a"), None], + vec![None, Some("d")] + ])); + + assert!(!is_fixed_size_encoding_applicable(vec![ + vec![Some(""), None], + vec![None, Some("")] + ])); + + assert!(!is_fixed_size_encoding_applicable(vec![ + vec![None, None], + vec![None, None] + ])); } fn verify_array_encoder( array: ArrayRef, field_meta: Option>, - version: LanceFileVersion, expected_encoder: &str, ) { - let encoding_strategy = CoreArrayEncodingStrategy { version }; + let encoding_strategy = ArrayStrategy; let mut field = Field::new("test_field", array.data_type().clone(), true); if let Some(field_meta) = field_meta { field.set_metadata(field_meta); @@ -797,7 +675,6 @@ mod tests { COMPRESSION_META_KEY.to_string(), "zstd".to_string(), )])), - LanceFileVersion::V2_1, "BinaryEncoder { indices_encoder: BasicEncoder { values_encoder: ValueEncoder }, compression_config: Some(CompressionConfig { scheme: Zstd, level: None }), buffer_compressor: Some(ZstdBufferCompressor { compression_level: 0 }) }", ); } @@ -810,7 +687,6 @@ mod tests { (COMPRESSION_META_KEY.to_string(), "zstd".to_string()), (COMPRESSION_LEVEL_META_KEY.to_string(), "22".to_string()), ])), - LanceFileVersion::V2_1, "BinaryEncoder { indices_encoder: BasicEncoder { values_encoder: ValueEncoder }, compression_config: Some(CompressionConfig { scheme: Zstd, level: Some(22) }), buffer_compressor: Some(ZstdBufferCompressor { compression_level: 22 }) }", ); } diff --git a/rust/lance-encoding/src/compression.rs b/rust/lance-encoding/src/compression.rs index 9051e18e8b6..f20c8eb04ae 100644 --- a/rust/lance-encoding/src/compression.rs +++ b/rust/lance-encoding/src/compression.rs @@ -20,7 +20,7 @@ use crate::encodings::physical::bitpacking::{InlineBitpacking, OutOfLineBitpacking}; use crate::{ buffer::LanceBuffer, - compression_config::{BssMode, CompressionFieldParams, CompressionParams}, + compression_config::{BssMode, CompressionFieldParams}, constants::{ BSS_META_KEY, COMPRESSION_LEVEL_META_KEY, COMPRESSION_META_KEY, RLE_THRESHOLD_META_KEY, }, @@ -55,8 +55,8 @@ use crate::{ VariablePackedStructFieldKind, }, rle::{ - RleDecompressor, RleEncoder, RunLengthWidth, rle_encoded_size, - select_run_length_width, + RleChildDecompressor, RleDecompressor, RleEncoder, RunLengthWidth, + rle_encoded_size, select_run_length_width, }, value::{ValueDecompressor, ValueEncoder}, }, @@ -66,7 +66,6 @@ use crate::{ pb21::{CompressiveEncoding, compressive_encoding::Compression}, }, statistics::{GetStat, Stat}, - version::LanceFileVersion, }; use arrow_array::{cast::AsArray, types::UInt64Type}; @@ -141,14 +140,6 @@ pub trait CompressionStrategy: Send + Sync + std::fmt::Debug { ) -> Result>; } -#[derive(Debug, Default, Clone)] -pub struct DefaultCompressionStrategy { - /// User-configured compression parameters - params: CompressionParams, - /// The lance file version for compatibilities. - version: LanceFileVersion, -} - fn try_bss_for_mini_block( data: &FixedWidthDataBlock, params: &CompressionFieldParams, @@ -169,11 +160,7 @@ fn try_bss_for_mini_block( None } -fn try_rle_for_mini_block( - data: &FixedWidthDataBlock, - params: &CompressionFieldParams, - use_rle_v2: bool, -) -> Option> { +fn rle_is_applicable(data: &FixedWidthDataBlock, params: &CompressionFieldParams) -> Option { let bits = data.bits_per_value; if !matches!(bits, 8 | 16 | 32 | 64) { return None; @@ -196,48 +183,93 @@ fn try_rle_for_mini_block( return None; } - let num_values = data.num_values; - let raw_bytes = (num_values as u128) * (type_size as u128); - let (run_length_width, rle_bytes) = if use_rle_v2 { - estimate_rle_width_and_size_from_data(data, Some(*MAX_MINIBLOCK_VALUES)).ok()? - } else { - ( - RunLengthWidth::U8, - estimate_rle_size_for_width_from_data( - data, - Some(*MAX_MINIBLOCK_VALUES), - RunLengthWidth::U8, - ) - .ok()?, - ) - }; + Some((data.num_values as u128) * (type_size as u128)) +} + +fn rle_beats_raw_and_bitpacking( + data: &FixedWidthDataBlock, + encoded_bytes: u128, + raw_bytes: u128, +) -> bool { + if encoded_bytes >= raw_bytes { + return false; + } - if rle_bytes < raw_bytes { - #[cfg(feature = "bitpacking")] + #[cfg(feature = "bitpacking")] + { + if let Some(bitpack_bytes) = estimate_inline_bitpacking_bytes(data).map(u128::from) + && bitpack_bytes < encoded_bytes { - if let Some(bitpack_bytes) = estimate_inline_bitpacking_bytes(data) - && (bitpack_bytes as u128) < rle_bytes - { - return None; - } + return false; } - return Some(Box::new(RleEncoder::with_run_length_width( - run_length_width, - ))); } - None + true } -fn try_rle_for_block( +fn try_fixed_u8_rle_for_mini_block( data: &FixedWidthDataBlock, - version: LanceFileVersion, params: &CompressionFieldParams, - use_rle_v2: bool, -) -> Result, CompressiveEncoding)>> { - if version < LanceFileVersion::V2_2 { - return Ok(None); +) -> Option> { + let raw_bytes = rle_is_applicable(data, params)?; + let rle_bytes = estimate_rle_size_for_width_from_data( + data, + Some(*MAX_MINIBLOCK_VALUES), + RunLengthWidth::U8, + ) + .ok()?; + rle_beats_raw_and_bitpacking(data, rle_bytes, raw_bytes) + .then(|| Box::new(RleEncoder::with_run_length_width(RunLengthWidth::U8)) as _) +} + +fn try_child_rle_for_mini_block( + data: &FixedWidthDataBlock, + params: &CompressionFieldParams, +) -> Option> { + let raw_bytes = rle_is_applicable(data, params)?; + let (run_length_width, estimated_bytes) = + estimate_rle_width_and_size_from_data(data, Some(*MAX_MINIBLOCK_VALUES)).ok()?; + let child_compression = rle_child_compression_config(params); + let encoder = || { + RleEncoder::with_child_encoding( + run_length_width, + child_compression, + child_compression, + true, + ) + }; + + #[cfg(feature = "bitpacking")] + let bitpack_bytes = estimate_inline_bitpacking_bytes(data).map(u128::from); + #[cfg(not(feature = "bitpacking"))] + let bitpack_bytes = None::; + + let should_measure_children = (child_compression.is_some() || cfg!(feature = "bitpacking")) + && (estimated_bytes >= raw_bytes + || bitpack_bytes.is_some_and(|bytes| bytes < estimated_bytes)); + let selected_bytes = if should_measure_children { + encoder().selected_payload_size(data).ok()? + } else { + estimated_bytes + }; + + rle_beats_raw_and_bitpacking(data, selected_bytes, raw_bytes).then(|| Box::new(encoder()) as _) +} + +fn rle_child_compression_config(params: &CompressionFieldParams) -> Option { + let raw = params.compression.as_deref()?; + if matches!(raw, "none" | "fsst") { + return None; } + let scheme = CompressionScheme::from_str(raw).ok()?; + Some(CompressionConfig::new(scheme, params.compression_level)) +} +fn try_rle_for_block_with_width( + data: &FixedWidthDataBlock, + params: &CompressionFieldParams, + run_length_width: RunLengthWidth, + rle_payload_bytes: u128, +) -> Result, CompressiveEncoding)>> { let bits = data.bits_per_value; if !matches!(bits, 8 | 16 | 32 | 64) { return Ok(None); @@ -258,14 +290,6 @@ fn try_rle_for_block( } let raw_bytes = (data.num_values as u128) * ((bits / 8) as u128); - let (run_length_width, rle_payload_bytes) = if use_rle_v2 { - estimate_rle_width_and_size_from_data(data, None)? - } else { - ( - RunLengthWidth::U8, - estimate_rle_size_for_width_from_data(data, None, RunLengthWidth::U8)?, - ) - }; let rle_bytes = rle_payload_bytes.saturating_add(RLE_BLOCK_HEADER_BYTES); if rle_bytes >= raw_bytes { @@ -289,6 +313,28 @@ fn try_rle_for_block( Ok(Some((compressor, encoding))) } +fn try_fixed_u8_rle_for_block( + data: &FixedWidthDataBlock, + params: &CompressionFieldParams, +) -> Result, CompressiveEncoding)>> { + if !matches!(data.bits_per_value, 8 | 16 | 32 | 64) { + return Ok(None); + } + let encoded_bytes = estimate_rle_size_for_width_from_data(data, None, RunLengthWidth::U8)?; + try_rle_for_block_with_width(data, params, RunLengthWidth::U8, encoded_bytes) +} + +fn try_variable_rle_for_block( + data: &FixedWidthDataBlock, + params: &CompressionFieldParams, +) -> Result, CompressiveEncoding)>> { + if !matches!(data.bits_per_value, 8 | 16 | 32 | 64) { + return Ok(None); + } + let (width, encoded_bytes) = estimate_rle_width_and_size_from_data(data, None)?; + try_rle_for_block_with_width(data, params, width, encoded_bytes) +} + fn estimate_rle_width_and_size_from_data( data: &FixedWidthDataBlock, max_segment_values: Option, @@ -474,7 +520,6 @@ fn maybe_wrap_general_for_mini_block( } fn try_general_compression( - version: LanceFileVersion, field_params: &CompressionFieldParams, data: &DataBlock, ) -> Result, CompressionConfig)>> { @@ -485,9 +530,7 @@ fn try_general_compression( // User-requested compression (unused today but perhaps still used // in the future someday) - if let Some(compression_scheme) = &field_params.compression - && version >= LanceFileVersion::V2_2 - { + if let Some(compression_scheme) = &field_params.compression { let scheme: CompressionScheme = compression_scheme.parse()?; let config = CompressionConfig::new(scheme, field_params.compression_level); let compressor = Box::new(CompressedBufferEncoder::try_new(config)?); @@ -495,9 +538,7 @@ fn try_general_compression( } // Automatic compression for large blocks - if data.data_size() > MIN_BLOCK_SIZE_FOR_GENERAL_COMPRESSION - && version >= LanceFileVersion::V2_2 - { + if data.data_size() > MIN_BLOCK_SIZE_FOR_GENERAL_COMPRESSION { let compressor = Box::new(CompressedBufferEncoder::default()); let config = compressor.compressor.config(); return Ok(Some((compressor, config))); @@ -506,353 +547,354 @@ fn try_general_compression( Ok(None) } -impl DefaultCompressionStrategy { - /// Create a new compression strategy with default behavior - pub fn new() -> Self { - Self::default() - } +/// Parse field-level compression metadata without applying format-specific constraints. +pub fn field_metadata_params(field: &Field) -> CompressionFieldParams { + let mut params = CompressionFieldParams::default(); - /// Create a new compression strategy with user-configured parameters - pub fn with_params(params: CompressionParams) -> Self { - Self { - params, - version: LanceFileVersion::default(), - } + if let Some(compression) = field.metadata.get(COMPRESSION_META_KEY) { + params.compression = Some(compression.clone()); } - - /// Override the file version used to make compression decisions - pub fn with_version(mut self, version: LanceFileVersion) -> Self { - self.version = version; - self + if let Some(level) = field.metadata.get(COMPRESSION_LEVEL_META_KEY) { + params.compression_level = level.parse().ok(); } - - fn use_rle_v2(&self) -> bool { - self.version.resolve() >= LanceFileVersion::V2_3 + if let Some(threshold) = field.metadata.get(RLE_THRESHOLD_META_KEY) { + params.rle_threshold = threshold.parse().ok(); } - - /// Parse compression parameters from field metadata - fn parse_field_metadata(field: &Field, version: &LanceFileVersion) -> CompressionFieldParams { - let mut params = CompressionFieldParams::default(); - - // Parse compression method - if let Some(compression) = field.metadata.get(COMPRESSION_META_KEY) { - params.compression = Some(compression.clone()); - } - - // Parse compression level - if let Some(level) = field.metadata.get(COMPRESSION_LEVEL_META_KEY) { - params.compression_level = level.parse().ok(); - } - - // Parse RLE threshold - if let Some(threshold) = field.metadata.get(RLE_THRESHOLD_META_KEY) { - params.rle_threshold = threshold.parse().ok(); - } - - // Parse BSS mode - if let Some(bss_str) = field.metadata.get(BSS_META_KEY) { - match BssMode::parse(bss_str) { - Some(mode) => params.bss = Some(mode), - None => { - log::warn!("Invalid BSS mode '{}', using default", bss_str); - } - } + if let Some(bss_str) = field.metadata.get(BSS_META_KEY) { + match BssMode::parse(bss_str) { + Some(mode) => params.bss = Some(mode), + None => log::warn!("Invalid BSS mode '{}', using default", bss_str), } - - // Parse minichunk size - if let Some(minichunk_size_str) = field - .metadata - .get(super::constants::MINICHUNK_SIZE_META_KEY) - { - if let Ok(minichunk_size) = minichunk_size_str.parse::() { - // for lance v2.1, only 32kb or smaller is supported - if minichunk_size >= 32 * 1024 && *version <= LanceFileVersion::V2_1 { - log::warn!( - "minichunk_size '{}' too large for version '{}', using default", - minichunk_size, - version - ); - } else { - params.minichunk_size = Some(minichunk_size); - } - } else { - log::warn!("Invalid minichunk_size '{}', skipping", minichunk_size_str); - } - } - - params } - - fn build_fixed_width_compressor( - &self, - params: &CompressionFieldParams, - data: &FixedWidthDataBlock, - ) -> Result> { - if params.compression.as_deref() == Some("none") { - return Ok(Box::new(ValueEncoder::default())); + if let Some(minichunk_size_str) = field + .metadata + .get(super::constants::MINICHUNK_SIZE_META_KEY) + { + if let Ok(minichunk_size) = minichunk_size_str.parse::() { + params.minichunk_size = Some(minichunk_size); + } else { + log::warn!("Invalid minichunk_size '{}', skipping", minichunk_size_str); } + } - let base = try_bss_for_mini_block(data, params) - .or_else(|| try_rle_for_mini_block(data, params, self.use_rle_v2())) - .or_else(|| try_bitpack_for_mini_block(data)) - .unwrap_or_else(|| Box::new(ValueEncoder::default())); + params +} - maybe_wrap_general_for_mini_block(base, params) +/// Apply general-purpose compression requested for a fixed-width miniblock. +pub fn finalize_miniblock_compressor( + data: &DataBlock, + compressor: Box, + params: &CompressionFieldParams, +) -> Result> { + if matches!(data, DataBlock::FixedWidth(_)) { + maybe_wrap_general_for_mini_block(compressor, params) + } else { + Ok(compressor) } +} - /// Build compressor based on parameters for variable-width data - fn build_variable_width_compressor( - &self, - field: &Field, - data: &VariableWidthBlock, - ) -> Result> { - let params = self.get_merged_field_params(field); - let compression = params.compression.as_deref(); - if data.bits_per_offset != 32 && data.bits_per_offset != 64 { - return Err(Error::invalid_input(format!( - "Variable width compression not supported for {} bit offsets", - data.bits_per_offset - ))); - } +/// Honor an explicit `compression = none` request for fixed-width miniblocks. +pub fn try_uncompressed_fixed_width_miniblock( + data: &DataBlock, + params: &CompressionFieldParams, +) -> Option> { + (matches!(data, DataBlock::FixedWidth(_)) && params.compression.as_deref() == Some("none")) + .then(|| Box::new(ValueEncoder::default()) as _) +} - // Get statistics - let data_size = data.expect_single_stat::(Stat::DataSize); - let max_len = data.expect_single_stat::(Stat::MaxLength); +/// Select byte-stream-split compression for an applicable fixed-width miniblock. +pub fn try_byte_stream_split_miniblock( + data: &DataBlock, + params: &CompressionFieldParams, +) -> Option> { + let DataBlock::FixedWidth(data) = data else { + return None; + }; + try_bss_for_mini_block(data, params) +} - // Explicitly disable all compression. - if compression == Some("none") { - return Ok(Box::new(BinaryMiniBlockEncoder::new(params.minichunk_size))); - } +/// Select the original fixed-u8 RLE miniblock grammar. +pub fn try_fixed_u8_rle_miniblock( + data: &DataBlock, + params: &CompressionFieldParams, +) -> Option> { + let DataBlock::FixedWidth(data) = data else { + return None; + }; + try_fixed_u8_rle_for_mini_block(data, params) +} - let use_fsst = compression == Some("fsst") - || (compression.is_none() - && !matches!(field.data_type(), DataType::Binary | DataType::LargeBinary) - && max_len >= FSST_LEAST_INPUT_MAX_LENGTH - && data_size >= FSST_LEAST_INPUT_SIZE as u64); +/// Select variable-width RLE with independently encoded children. +pub fn try_child_rle_miniblock( + data: &DataBlock, + params: &CompressionFieldParams, +) -> Option> { + let DataBlock::FixedWidth(data) = data else { + return None; + }; + try_child_rle_for_mini_block(data, params) +} - // Choose base encoder (FSST or Binary) once. - let mut base_encoder: Box = if use_fsst { - Box::new(FsstMiniBlockEncoder::new(params.minichunk_size)) - } else { - Box::new(BinaryMiniBlockEncoder::new(params.minichunk_size)) - }; +/// Select inline bitpacking for applicable fixed-width miniblocks. +pub fn try_bitpacking_miniblock(data: &DataBlock) -> Option> { + let DataBlock::FixedWidth(data) = data else { + return None; + }; + try_bitpack_for_mini_block(data) +} - // Wrap with general compression when configured (except FSST / none). - if let Some(compression_scheme) = compression.filter(|scheme| *scheme != "fsst") { - let scheme: CompressionScheme = compression_scheme.parse()?; - let config = CompressionConfig::new(scheme, params.compression_level); - base_encoder = Box::new(GeneralMiniBlockCompressor::new(base_encoder, config)); - } +/// Store fixed-width miniblock values without a value codec. +pub fn try_raw_fixed_width_miniblock(data: &DataBlock) -> Option> { + matches!(data, DataBlock::FixedWidth(_)).then(|| Box::new(ValueEncoder::default()) as _) +} - Ok(base_encoder) +/// Encode variable-width miniblocks with binary or FSST encoding. +pub fn try_variable_width_miniblock( + field: &Field, + data: &DataBlock, + params: &CompressionFieldParams, +) -> Result>> { + let DataBlock::VariableWidth(data) = data else { + return Ok(None); + }; + if data.bits_per_offset != 32 && data.bits_per_offset != 64 { + return Err(Error::invalid_input(format!( + "Variable width compression not supported for {} bit offsets", + data.bits_per_offset + ))); } - /// Merge user-configured parameters with field metadata - /// Field metadata has highest priority - fn get_merged_field_params(&self, field: &Field) -> CompressionFieldParams { - let mut field_params = self - .params - .get_field_params(&field.name, &field.data_type()); - - // Override with field metadata if present (highest priority) - let metadata_params = Self::parse_field_metadata(field, &self.version); - field_params.merge(&metadata_params); + let compression = params.compression.as_deref(); + let data_size = data.expect_single_stat::(Stat::DataSize); + let max_len = data.expect_single_stat::(Stat::MaxLength); + if compression == Some("none") { + return Ok(Some(Box::new(BinaryMiniBlockEncoder::new( + params.minichunk_size, + )))); + } - field_params + let use_fsst = compression == Some("fsst") + || (compression.is_none() + && !matches!(field.data_type(), DataType::Binary | DataType::LargeBinary) + && max_len >= FSST_LEAST_INPUT_MAX_LENGTH + && data_size >= FSST_LEAST_INPUT_SIZE as u64); + let mut encoder: Box = if use_fsst { + Box::new(FsstMiniBlockEncoder::new(params.minichunk_size)) + } else { + Box::new(BinaryMiniBlockEncoder::new(params.minichunk_size)) + }; + if let Some(compression_scheme) = compression.filter(|scheme| *scheme != "fsst") { + let scheme: CompressionScheme = compression_scheme.parse()?; + let config = CompressionConfig::new(scheme, params.compression_level); + encoder = Box::new(GeneralMiniBlockCompressor::new(encoder, config)); } + Ok(Some(encoder)) } -impl CompressionStrategy for DefaultCompressionStrategy { - fn create_miniblock_compressor( - &self, - field: &Field, - data: &DataBlock, - ) -> Result> { - match data { - DataBlock::FixedWidth(fixed_width_data) => { - let field_params = self.get_merged_field_params(field); - self.build_fixed_width_compressor(&field_params, fixed_width_data) - } - DataBlock::VariableWidth(variable_width_data) => { - self.build_variable_width_compressor(field, variable_width_data) - } - DataBlock::Struct(struct_data_block) => { - // this condition is actually checked at `PrimitiveStructuralEncoder::do_flush`, - // just being cautious here. - if struct_data_block.has_variable_width_child() { - return Err(Error::invalid_input( - "Packed struct mini-block encoding supports only fixed-width children", - )); - } - Ok(Box::new(PackedStructFixedWidthMiniBlockEncoder::default())) - } - DataBlock::FixedSizeList(_) => { - // Ideally we would compress the list items but this creates something of a challenge. - // We don't want to break lists across chunks and we need to worry about inner validity - // layers. If we try and use a compression scheme then it is unlikely to respect these - // constraints. - // - // For now, we just don't compress. In the future, we might want to consider a more - // sophisticated approach. - Ok(Box::new(ValueEncoder::default())) - } - _ => Err(Error::not_supported_source( - format!( - "Mini-block compression not yet supported for block type {}", - data.name() - ) - .into(), - )), - } +/// Encode fixed-width packed structs as miniblocks. +pub fn try_fixed_packed_struct_miniblock( + data: &DataBlock, +) -> Result>> { + let DataBlock::Struct(data) = data else { + return Ok(None); + }; + if data.has_variable_width_child() { + return Err(Error::invalid_input( + "Packed struct mini-block encoding supports only fixed-width children", + )); } + Ok(Some(Box::new( + PackedStructFixedWidthMiniBlockEncoder::default(), + ))) +} - fn create_per_value( - &self, - field: &Field, - data: &DataBlock, - ) -> Result> { - let field_params = self.get_merged_field_params(field); - - match data { - DataBlock::FixedWidth(_) => Ok(Box::new(ValueEncoder::default())), - DataBlock::FixedSizeList(_) => Ok(Box::new(ValueEncoder::default())), - DataBlock::Struct(struct_block) => { - if field.children.len() != struct_block.children.len() { - return Err(Error::invalid_input( - "Struct field metadata does not match data block children", - )); - } - let has_variable_child = struct_block.has_variable_width_child(); - if has_variable_child { - if self.version < LanceFileVersion::V2_2 { - return Err(Error::not_supported_source("Variable packed struct encoding requires Lance file version 2.2 or later".into())); - } - Ok(Box::new(PackedStructVariablePerValueEncoder::new( - self.clone(), - field.children.clone(), - ))) - } else { - Err(Error::invalid_input( - "Packed struct per-value compression should not be used for fixed-width-only structs", - )) - } - } - DataBlock::VariableWidth(variable_width) => { - let compression = field_params.compression.as_deref(); - // Check for explicit "none" compression - if compression == Some("none") { - return Ok(Box::new(VariableEncoder::default())); - } +/// Store fixed-size-list miniblocks without a value codec. +pub fn try_raw_fixed_size_list_miniblock(data: &DataBlock) -> Option> { + matches!(data, DataBlock::FixedSizeList(_)).then(|| Box::new(ValueEncoder::default()) as _) +} - let max_len = variable_width.expect_single_stat::(Stat::MaxLength); - let data_size = variable_width.expect_single_stat::(Stat::DataSize); +/// Store fixed-width and fixed-size-list values directly in full-zip pages. +pub fn try_raw_per_value(data: &DataBlock) -> Option> { + matches!(data, DataBlock::FixedWidth(_) | DataBlock::FixedSizeList(_)) + .then(|| Box::new(ValueEncoder::default()) as _) +} - // If values are very large then use block compression on a per-value basis - // - // TODO: Could maybe use median here +fn validate_packed_struct(field: &Field, data: &DataBlock) -> Result> { + let DataBlock::Struct(data) = data else { + return Ok(None); + }; + if field.children.len() != data.children.len() { + return Err(Error::invalid_input( + "Struct field metadata does not match data block children", + )); + } + Ok(Some(data.has_variable_width_child())) +} - let per_value_requested = - compression.is_some_and(|compression| compression != "fsst"); +/// Reject variable-width packed structs while preserving the fixed-width error. +pub fn reject_packed_struct_per_value( + field: &Field, + data: &DataBlock, +) -> Result>> { + let Some(has_variable_child) = validate_packed_struct(field, data)? else { + return Ok(None); + }; + if has_variable_child { + return Err(Error::not_supported_source( + "Variable packed struct encoding is not enabled by the selected file format".into(), + )); + } + Err(Error::invalid_input( + "Packed struct per-value compression should not be used for fixed-width-only structs", + )) +} - if (max_len > 32 * 1024 || per_value_requested) - && data_size >= FSST_LEAST_INPUT_SIZE as u64 - { - return Ok(Box::new(CompressedBufferEncoder::default())); - } +/// Encode variable-width packed structs with the exact strategy recursively. +pub fn try_variable_packed_struct_per_value( + strategy: Arc, + field: &Field, + data: &DataBlock, +) -> Result>> { + let Some(has_variable_child) = validate_packed_struct(field, data)? else { + return Ok(None); + }; + if !has_variable_child { + return Err(Error::invalid_input( + "Packed struct per-value compression should not be used for fixed-width-only structs", + )); + } + Ok(Some(Box::new(PackedStructVariablePerValueEncoder::new( + strategy, + field.children.clone(), + )))) +} - if variable_width.bits_per_offset == 32 || variable_width.bits_per_offset == 64 { - let variable_compression = Box::new(VariableEncoder::default()); - let use_fsst = compression == Some("fsst") - || (compression.is_none() - && !matches!( - field.data_type(), - DataType::Binary | DataType::LargeBinary - ) - && max_len >= FSST_LEAST_INPUT_MAX_LENGTH - && data_size >= FSST_LEAST_INPUT_SIZE as u64); - - // Use FSST if explicitly requested or if data characteristics warrant it. - if use_fsst { - Ok(Box::new(FsstPerValueEncoder::new(variable_compression))) - } else { - Ok(variable_compression) - } - } else { - panic!( - "Does not support MiniBlockCompression for VariableWidth DataBlock with {} bits offsets.", - variable_width.bits_per_offset - ); - } - } - _ => unreachable!( - "Per-value compression not yet supported for block type: {}", - data.name() - ), +/// Encode variable-width values directly, with FSST or per-value compression +/// when applicable. +pub fn try_variable_width_per_value( + field: &Field, + data: &DataBlock, + params: &CompressionFieldParams, +) -> Result>> { + let DataBlock::VariableWidth(data) = data else { + return Ok(None); + }; + let compression = params.compression.as_deref(); + if compression == Some("none") { + return Ok(Some(Box::new(VariableEncoder::default()))); + } + + let max_len = data.expect_single_stat::(Stat::MaxLength); + let data_size = data.expect_single_stat::(Stat::DataSize); + let per_value_requested = compression.is_some_and(|compression| compression != "fsst"); + if (max_len > 32 * 1024 || per_value_requested) && data_size >= FSST_LEAST_INPUT_SIZE as u64 { + if compression == Some("zstd") { + let config = CompressionConfig::new(CompressionScheme::Zstd, params.compression_level); + return Ok(Some(Box::new(CompressedBufferEncoder::try_new(config)?))); } + return Ok(Some(Box::new(CompressedBufferEncoder::default()))); } - fn create_block_compressor( - &self, - field: &Field, - data: &DataBlock, - ) -> Result<(Box, CompressiveEncoding)> { - let field_params = self.get_merged_field_params(field); + if data.bits_per_offset != 32 && data.bits_per_offset != 64 { + return Err(Error::invalid_input(format!( + "Per-value compression does not support variable-width data with {}-bit offsets", + data.bits_per_offset + ))); + } + let encoder = Box::new(VariableEncoder::default()); + let use_fsst = compression == Some("fsst") + || (compression.is_none() + && !matches!(field.data_type(), DataType::Binary | DataType::LargeBinary) + && max_len >= FSST_LEAST_INPUT_MAX_LENGTH + && data_size >= FSST_LEAST_INPUT_SIZE as u64); + Ok(Some(if use_fsst { + Box::new(FsstPerValueEncoder::new(encoder)) + } else { + encoder + })) +} - match data { - DataBlock::FixedWidth(fixed_width) => { - if let Some((compressor, encoding)) = - try_rle_for_block(fixed_width, self.version, &field_params, self.use_rle_v2())? - { - return Ok((compressor, encoding)); - } - if let Some((compressor, encoding)) = try_bitpack_for_block(fixed_width) { - return Ok((compressor, encoding)); - } +/// Select fixed-u8 RLE for block compression. +pub fn try_fixed_u8_rle_block( + data: &DataBlock, + params: &CompressionFieldParams, +) -> Result, CompressiveEncoding)>> { + let DataBlock::FixedWidth(data) = data else { + return Ok(None); + }; + try_fixed_u8_rle_for_block(data, params) +} - // Try general compression (user-requested or automatic over MIN_BLOCK_SIZE_FOR_GENERAL_COMPRESSION) - if let Some((compressor, config)) = - try_general_compression(self.version, &field_params, data)? - { - let encoding = ProtobufUtils21::wrapped( - config, - ProtobufUtils21::flat(fixed_width.bits_per_value, None), - )?; - return Ok((compressor, encoding)); - } +/// Select variable-width RLE for block compression. +pub fn try_variable_rle_block( + data: &DataBlock, + params: &CompressionFieldParams, +) -> Result, CompressiveEncoding)>> { + let DataBlock::FixedWidth(data) = data else { + return Ok(None); + }; + try_variable_rle_for_block(data, params) +} - let encoder = Box::new(ValueEncoder::default()); - let encoding = ProtobufUtils21::flat(fixed_width.bits_per_value, None); - Ok((encoder, encoding)) - } - DataBlock::VariableWidth(variable_width) => { - // Try general compression - if let Some((compressor, config)) = - try_general_compression(self.version, &field_params, data)? - { - let encoding = ProtobufUtils21::wrapped( - config, - ProtobufUtils21::variable( - ProtobufUtils21::flat(variable_width.bits_per_offset as u64, None), - None, - ), - )?; - return Ok((compressor, encoding)); - } +/// Select block bitpacking for applicable fixed-width values. +pub fn try_bitpacking_block( + data: &DataBlock, +) -> Option<(Box, CompressiveEncoding)> { + let DataBlock::FixedWidth(data) = data else { + return None; + }; + try_bitpack_for_block(data) +} - let encoder = Box::new(VariableEncoder::default()); - let encoding = ProtobufUtils21::variable( - ProtobufUtils21::flat(variable_width.bits_per_offset as u64, None), - None, - ); - Ok((encoder, encoding)) - } - _ => unreachable!(), - } +/// Select explicitly requested or automatic general-purpose block compression. +pub fn try_general_block( + data: &DataBlock, + params: &CompressionFieldParams, +) -> Result, CompressiveEncoding)>> { + let Some((compressor, config)) = try_general_compression(params, data)? else { + return Ok(None); + }; + let inner = match data { + DataBlock::FixedWidth(data) => ProtobufUtils21::flat(data.bits_per_value, None), + DataBlock::VariableWidth(data) => ProtobufUtils21::variable( + ProtobufUtils21::flat(data.bits_per_offset as u64, None), + None, + ), + _ => return Ok(None), + }; + Ok(Some((compressor, ProtobufUtils21::wrapped(config, inner)?))) +} + +/// Store fixed- and variable-width block values without block compression. +pub fn try_raw_block(data: &DataBlock) -> Option<(Box, CompressiveEncoding)> { + match data { + DataBlock::FixedWidth(data) => Some(( + Box::new(ValueEncoder::default()) as Box, + ProtobufUtils21::flat(data.bits_per_value, None), + )), + DataBlock::VariableWidth(data) => Some(( + Box::new(VariableEncoder::default()) as Box, + ProtobufUtils21::variable( + ProtobufUtils21::flat(data.bits_per_offset as u64, None), + None, + ), + )), + _ => None, } } pub trait MiniBlockDecompressor: std::fmt::Debug + Send + Sync { fn decompress(&self, data: Vec, num_values: u64) -> Result; + + /// Returns the exact aggregate decoded size when it is determined solely by the value count. + /// + /// Implementations should only return `Some` when this aggregate estimate can be used by + /// [`DataBlockBuilder`](crate::data::DataBlockBuilder) to preallocate the decoded output + /// exactly. Outputs with multiple buffers or whose layout-dependent allocation cannot be + /// represented by one aggregate estimate should return `None`. + fn decoded_size_bytes(&self, _num_values: u64) -> Option { + None + } } pub trait FixedPerValueDecompressor: std::fmt::Debug + Send + Sync { @@ -862,6 +904,16 @@ pub trait FixedPerValueDecompressor: std::fmt::Debug + Send + Sync { /// /// Currently (and probably long term) this must be a multiple of 8 fn bits_per_value(&self) -> u64; + + /// Returns the exact aggregate decoded size when it is determined solely by the value count. + /// + /// Implementations should only return `Some` when this aggregate estimate can be used by + /// [`DataBlockBuilder`](crate::data::DataBlockBuilder) to preallocate the decoded output + /// exactly. Outputs with multiple buffers or whose layout-dependent allocation cannot be + /// represented by one aggregate estimate should return `None`. + fn decoded_size_bytes(&self, _num_values: u64) -> Option { + None + } } pub trait VariablePerValueDecompressor: std::fmt::Debug + Send + Sync { @@ -951,13 +1003,10 @@ impl DecompressionStrategy for DefaultDecompressionStrategy { // compression. Ok(Box::new(ValueDecompressor::from_fsl(fsl))) } - Compression::Rle(rle) => { - let (bits_per_value, run_length_width) = validate_rle_compression(rle)?; - Ok(Box::new(RleDecompressor::with_run_length_width( - bits_per_value, - run_length_width, - ))) - } + Compression::Rle(rle) => Ok(Box::new(create_rle_decompressor( + rle, + decompression_strategy, + )?)), Compression::ByteStreamSplit(bss) => { let Compression::Flat(values) = bss.values.as_ref().unwrap().compression.as_ref().unwrap() @@ -1144,19 +1193,15 @@ impl DecompressionStrategy for DefaultDecompressionStrategy { Ok(Box::new(general_decompressor)) } - Compression::Rle(rle) => { - let (bits_per_value, run_length_width) = validate_rle_compression(rle)?; - Ok(Box::new(RleDecompressor::with_run_length_width( - bits_per_value, - run_length_width, - ))) - } + Compression::Rle(rle) => Ok(Box::new(create_rle_decompressor(rle, self)?)), _ => todo!(), } } } -/// Validates RLE compression format and extracts value and run length widths. -fn validate_rle_compression(rle: &crate::format::pb21::Rle) -> Result<(u64, RunLengthWidth)> { +pub(crate) fn create_rle_decompressor( + rle: &crate::format::pb21::Rle, + decompression_strategy: &dyn DecompressionStrategy, +) -> Result { let values = rle .values .as_ref() @@ -1166,54 +1211,188 @@ fn validate_rle_compression(rle: &crate::format::pb21::Rle) -> Result<(u64, RunL .as_ref() .ok_or_else(|| Error::invalid_input("RLE compression missing run lengths encoding"))?; - let values = values - .compression - .as_ref() - .ok_or_else(|| Error::invalid_input("RLE compression missing values compression"))?; - let Compression::Flat(values) = values else { - return Err(Error::invalid_input( - "RLE compression only supports flat values", - )); - }; + let values = create_rle_child_decompressor(values, "values", decompression_strategy)?; + let run_lengths = + create_rle_child_decompressor(run_lengths, "run lengths", decompression_strategy)?; - let run_lengths = run_lengths - .compression - .as_ref() - .ok_or_else(|| Error::invalid_input("RLE compression missing run lengths compression"))?; - let Compression::Flat(run_lengths) = run_lengths else { - return Err(Error::invalid_input( - "RLE compression only supports flat run lengths", - )); - }; - - if !matches!(values.bits_per_value, 8 | 16 | 32 | 64) { + if !matches!(values.bits_per_value(), 8 | 16 | 32 | 64) { return Err(Error::invalid_input(format!( "RLE compression only supports 8, 16, 32, or 64-bit values, got {}", - values.bits_per_value + values.bits_per_value() ))); } let run_length_width = - RunLengthWidth::from_bits(run_lengths.bits_per_value).ok_or_else(|| { + RunLengthWidth::from_bits(run_lengths.bits_per_value()).ok_or_else(|| { Error::invalid_input(format!( "RLE compression only supports 8, 16, or 32-bit run lengths, got {}", - run_lengths.bits_per_value + run_lengths.bits_per_value() )) })?; - Ok((values.bits_per_value, run_length_width)) + if values.requires_num_values() && run_lengths.requires_num_values() { + return Err(Error::invalid_input( + "RLE values and run lengths child encodings cannot both require the run count", + )); + } + + if values.is_identity() && run_lengths.is_identity() { + return Ok(RleDecompressor::with_run_length_width( + values.bits_per_value(), + run_length_width, + )); + } + + Ok(RleDecompressor::with_child_decompressors( + values.bits_per_value(), + run_length_width, + values, + run_lengths, + )) +} + +fn create_rle_child_decompressor( + encoding: &CompressiveEncoding, + role: &str, + decompression_strategy: &dyn DecompressionStrategy, +) -> Result { + let compression = encoding + .compression + .as_ref() + .ok_or_else(|| Error::invalid_input(format!("RLE {role} missing child compression")))?; + let (bits_per_value, requires_num_values, needs_decompressor) = + validate_rle_child_compression(compression, role)?; + + if needs_decompressor { + Ok(RleChildDecompressor::block( + bits_per_value, + decompression_strategy.create_block_decompressor(encoding)?, + requires_num_values, + )) + } else { + Ok(RleChildDecompressor::flat(bits_per_value)) + } +} + +fn validate_rle_child_compression( + compression: &Compression, + role: &str, +) -> Result<(u64, bool, bool)> { + match compression { + Compression::Flat(flat) => Ok((flat.bits_per_value, false, false)), + Compression::General(general) => { + general.compression.as_ref().ok_or_else(|| { + Error::invalid_input(format!( + "RLE {role} general child missing compression config" + )) + })?; + let values = general.values.as_ref().ok_or_else(|| { + Error::invalid_input(format!("RLE {role} general child missing inner encoding")) + })?; + let inner = values.compression.as_ref().ok_or_else(|| { + Error::invalid_input(format!( + "RLE {role} general child missing inner compression" + )) + })?; + let (bits_per_value, requires_num_values) = + validate_rle_block_child_inner(inner, role)?; + Ok((bits_per_value, requires_num_values, true)) + } + Compression::OutOfLineBitpacking(out_of_line) => { + let values = out_of_line.values.as_ref().ok_or_else(|| { + Error::invalid_input(format!( + "RLE {role} bitpacking child missing values encoding" + )) + })?; + let Compression::Flat(_) = values.compression.as_ref().ok_or_else(|| { + Error::invalid_input(format!( + "RLE {role} bitpacking child missing values compression" + )) + })? + else { + return Err(Error::invalid_input(format!( + "RLE {role} bitpacking child only supports flat values" + ))); + }; + Ok((out_of_line.uncompressed_bits_per_value, true, true)) + } + other => Err(Error::invalid_input(format!( + "RLE {role} only supports flat, general, or out-of-line bitpacking child encodings, got {}", + compression_name(other) + ))), + } +} + +fn validate_rle_block_child_inner(compression: &Compression, role: &str) -> Result<(u64, bool)> { + match compression { + Compression::Flat(flat) => Ok((flat.bits_per_value, false)), + Compression::OutOfLineBitpacking(out_of_line) => { + let values = out_of_line.values.as_ref().ok_or_else(|| { + Error::invalid_input(format!( + "RLE {role} bitpacking child missing values encoding" + )) + })?; + let Compression::Flat(_) = values.compression.as_ref().ok_or_else(|| { + Error::invalid_input(format!( + "RLE {role} bitpacking child missing values compression" + )) + })? + else { + return Err(Error::invalid_input(format!( + "RLE {role} bitpacking child only supports flat values" + ))); + }; + Ok((out_of_line.uncompressed_bits_per_value, true)) + } + other => Err(Error::invalid_input(format!( + "RLE {role} general child only supports flat or out-of-line bitpacking inner encodings, got {}", + compression_name(other) + ))), + } +} + +fn compression_name(compression: &Compression) -> &'static str { + match compression { + Compression::Flat(_) => "flat", + Compression::Variable(_) => "variable", + Compression::Fsst(_) => "fsst", + Compression::OutOfLineBitpacking(_) => "out-of-line bitpacking", + Compression::InlineBitpacking(_) => "inline bitpacking", + Compression::General(_) => "general", + Compression::Constant(_) => "constant", + Compression::Dictionary(_) => "dictionary", + Compression::ByteStreamSplit(_) => "byte stream split", + Compression::PackedStruct(_) => "packed struct", + Compression::FixedSizeList(_) => "fixed-size list", + Compression::VariablePackedStruct(_) => "variable packed struct", + Compression::Rle(_) => "rle", + } } #[cfg(test)] mod tests { use super::*; use crate::buffer::LanceBuffer; + use crate::compression_config::CompressionParams; use crate::data::{BlockInfo, DataBlock, FixedWidthDataBlock}; + use crate::encodings::logical::primitive::miniblock::MiniBlockCompressionContext; use crate::statistics::ComputeStat; - use crate::testing::extract_array_encoding_chain; + use crate::testing::{TestEncoding, extract_array_encoding_chain, test_compression_strategy}; use arrow_schema::{DataType, Field as ArrowField}; use std::collections::HashMap; + fn strategy(encoding: TestEncoding, params: CompressionParams) -> Arc { + test_compression_strategy(encoding, params) + } + + fn baseline_strategy(params: CompressionParams) -> Arc { + strategy(TestEncoding::StructuralU16, params) + } + + fn miniblock_context() -> MiniBlockCompressionContext { + MiniBlockCompressionContext::new(0, true, true) + } + fn create_test_field(name: &str, data_type: DataType) -> Field { let arrow_field = ArrowField::new(name, data_type, true); let mut field = Field::try_from(&arrow_field).unwrap(); @@ -1307,6 +1486,20 @@ mod tests { run_lengths.bits_per_value } + fn expect_rle_encoding(encoding: &CompressiveEncoding) -> &crate::format::pb21::Rle { + match encoding.compression.as_ref().unwrap() { + Compression::Rle(rle) => rle, + Compression::General(general) => { + let inner = general.values.as_ref().unwrap(); + let Compression::Rle(rle) = inner.compression.as_ref().unwrap() else { + panic!("expected wrapped RLE encoding"); + }; + rle + } + other => panic!("expected RLE encoding, got {}", compression_name(other)), + } + } + fn create_variable_width_block( bits_per_offset: u8, num_values: u64, @@ -1388,7 +1581,7 @@ mod tests { }, ); - let strategy = DefaultCompressionStrategy::with_params(params); + let strategy = baseline_strategy(params); let field = create_test_field("user_id", DataType::Int32); // Create data with low run count for RLE @@ -1420,7 +1613,7 @@ mod tests { }, ); - let strategy = DefaultCompressionStrategy::with_params(params); + let strategy = baseline_strategy(params); let field = create_test_field("some_column", DataType::Int32); // Create data with very low run count (50 runs for 1000 values = 0.05 ratio) let data = create_fixed_width_block_with_stats(32, 1000, 50); @@ -1435,7 +1628,7 @@ mod tests { #[test] #[cfg(feature = "bitpacking")] fn test_block_bitpacks_with_zero_segment() { - let strategy = DefaultCompressionStrategy::new(); + let strategy = baseline_strategy(CompressionParams::default()); let field = create_test_field("levels", DataType::UInt16); // First 1024 zeros, then 1024 ones; max bit width is 1. @@ -1460,7 +1653,7 @@ mod tests { #[test] fn test_rle_block_accounts_for_header_before_selecting() { - let strategy = DefaultCompressionStrategy::new().with_version(LanceFileVersion::V2_3); + let strategy = strategy(TestEncoding::StructuralSparse, CompressionParams::default()); let field = create_test_field("small_constant", DataType::Int32); let values = vec![42i32; 2]; let mut block = FixedWidthDataBlock { @@ -1484,7 +1677,7 @@ mod tests { #[test] #[cfg(feature = "bitpacking")] fn test_rle_block_prefers_bitpacking_when_smaller() { - let strategy = DefaultCompressionStrategy::new().with_version(LanceFileVersion::V2_3); + let strategy = strategy(TestEncoding::StructuralSparse, CompressionParams::default()); let field = create_test_field("levels", DataType::UInt16); let mut values = Vec::with_capacity(2048); @@ -1515,7 +1708,7 @@ mod tests { #[test] #[cfg(feature = "bitpacking")] fn test_low_cardinality_prefers_bitpacking_over_rle() { - let strategy = DefaultCompressionStrategy::new(); + let strategy = baseline_strategy(CompressionParams::default()); let field = create_test_field("int_score", DataType::Int64); // Low cardinality values (3/4/5) but with moderate run count: @@ -1578,7 +1771,7 @@ mod tests { }, ); - let strategy = DefaultCompressionStrategy::with_params(params); + let strategy = baseline_strategy(params); let field = create_test_field("embeddings", DataType::Float32); let fixed_data = create_fixed_width_block(32, 1000); let variable_data = create_variable_width_block(32, 10, 32 * 1024); @@ -1587,12 +1780,16 @@ mod tests { let compressor = strategy .create_miniblock_compressor(&field, &fixed_data) .unwrap(); - let (_block, encoding) = compressor.compress(fixed_data.clone()).unwrap(); + let (_block, encoding) = compressor + .compress(miniblock_context(), fixed_data.clone()) + .unwrap(); check_uncompressed_encoding(&encoding, false); let compressor = strategy .create_miniblock_compressor(&field, &variable_data) .unwrap(); - let (_block, encoding) = compressor.compress(variable_data.clone()).unwrap(); + let (_block, encoding) = compressor + .compress(miniblock_context(), variable_data.clone()) + .unwrap(); check_uncompressed_encoding(&encoding, true); // Test pervalue @@ -1613,7 +1810,7 @@ mod tests { arrow_field = arrow_field.with_metadata(metadata); let field = Field::try_from(&arrow_field).unwrap(); - let strategy = DefaultCompressionStrategy::with_params(CompressionParams::new()); + let strategy = baseline_strategy(CompressionParams::new()); // Test miniblock let fixed_data = create_fixed_width_block(32, 1000); @@ -1622,13 +1819,17 @@ mod tests { let compressor = strategy .create_miniblock_compressor(&field, &fixed_data) .unwrap(); - let (_block, encoding) = compressor.compress(fixed_data.clone()).unwrap(); + let (_block, encoding) = compressor + .compress(miniblock_context(), fixed_data.clone()) + .unwrap(); check_uncompressed_encoding(&encoding, false); let compressor = strategy .create_miniblock_compressor(&field, &variable_data) .unwrap(); - let (_block, encoding) = compressor.compress(variable_data.clone()).unwrap(); + let (_block, encoding) = compressor + .compress(miniblock_context(), variable_data.clone()) + .unwrap(); check_uncompressed_encoding(&encoding, true); // Test pervalue @@ -1643,7 +1844,7 @@ mod tests { #[test] fn test_auto_fsst_disabled_for_binary_fields() { - let strategy = DefaultCompressionStrategy::new(); + let strategy = baseline_strategy(CompressionParams::default()); let field = create_test_field("bytes", DataType::Binary); let variable_data = create_fsst_candidate_variable_width_block(); @@ -1674,7 +1875,7 @@ mod tests { #[test] fn test_auto_fsst_still_enabled_for_utf8_fields() { - let strategy = DefaultCompressionStrategy::new(); + let strategy = baseline_strategy(CompressionParams::default()); let field = create_test_field("text", DataType::Utf8); let variable_data = create_fsst_candidate_variable_width_block(); @@ -1706,7 +1907,7 @@ mod tests { }, ); - let strategy = DefaultCompressionStrategy::with_params(params); + let strategy = baseline_strategy(params); let field = create_test_field("bytes", DataType::Binary); let variable_data = create_fsst_candidate_variable_width_block(); @@ -1727,6 +1928,30 @@ mod tests { ); } + #[test] + #[cfg(feature = "zstd")] + fn test_compression_level_honored_for_large_per_value() { + let mut params = CompressionParams::new(); + params.columns.insert( + "html".to_string(), + CompressionFieldParams { + compression: Some("zstd".to_string()), + compression_level: Some(19), + ..Default::default() + }, + ); + let strategy = baseline_strategy(params); + let field = create_test_field("html", DataType::Utf8); + let large = create_variable_width_block(32, 64, 40 * 1024); + + let per_value = strategy.create_per_value(&field, &large).unwrap(); + let debug = format!("{per_value:?}"); + assert!( + debug.contains("ZstdBufferCompressor") && debug.contains("compression_level: 19"), + "expected zstd level 19 to reach the per-value compressor, got: {debug}" + ); + } + #[test] fn test_parameter_merge_priority() { let mut params = CompressionParams::new(); @@ -1753,12 +1978,8 @@ mod tests { }, ); - let strategy = DefaultCompressionStrategy::with_params(params); - // Get merged params - let merged = strategy - .params - .get_field_params("user_id", &DataType::Int32); + let merged = params.get_field_params("user_id", &DataType::Int32); // Column params should override type params assert_eq!(merged.rle_threshold, Some(0.2)); @@ -1766,9 +1987,7 @@ mod tests { assert_eq!(merged.compression_level, Some(6)); // Test field with only type params - let merged = strategy - .params - .get_field_params("other_field", &DataType::Int32); + let merged = params.get_field_params("other_field", &DataType::Int32); assert_eq!(merged.rle_threshold, Some(0.5)); assert_eq!(merged.compression, Some("lz4".to_string())); assert_eq!(merged.compression_level, None); @@ -1788,26 +2007,20 @@ mod tests { }, ); - let strategy = DefaultCompressionStrategy::with_params(params); - // Should match pattern - let merged = strategy - .params - .get_field_params("log_messages", &DataType::Utf8); + let merged = params.get_field_params("log_messages", &DataType::Utf8); assert_eq!(merged.compression, Some("zstd".to_string())); assert_eq!(merged.compression_level, Some(6)); // Should not match - let merged = strategy - .params - .get_field_params("messages_log", &DataType::Utf8); + let merged = params.get_field_params("messages_log", &DataType::Utf8); assert_eq!(merged.compression, None); } #[test] fn test_legacy_metadata_support() { let params = CompressionParams::new(); - let strategy = DefaultCompressionStrategy::with_params(params); + let strategy = baseline_strategy(params); // Test field with "none" compression metadata let mut metadata = HashMap::new(); @@ -1826,7 +2039,7 @@ mod tests { fn test_default_behavior() { // Empty params should fall back to default behavior let params = CompressionParams::new(); - let strategy = DefaultCompressionStrategy::with_params(params); + let strategy = baseline_strategy(params); let field = create_test_field("random_column", DataType::Int32); // Create data with high run count that won't trigger RLE (600 runs for 1000 values = 0.6 ratio) @@ -1841,7 +2054,7 @@ mod tests { #[test] fn test_field_metadata_compression() { let params = CompressionParams::new(); - let strategy = DefaultCompressionStrategy::with_params(params); + let strategy = baseline_strategy(params); // Test field with compression metadata let mut metadata = HashMap::new(); @@ -1861,7 +2074,7 @@ mod tests { #[test] fn test_field_metadata_rle_threshold() { let params = CompressionParams::new(); - let strategy = DefaultCompressionStrategy::with_params(params); + let strategy = baseline_strategy(params); // Test field with RLE threshold metadata let mut metadata = HashMap::new(); @@ -1899,15 +2112,15 @@ mod tests { data.compute_stat(); let data = DataBlock::FixedWidth(data); - let strategy = DefaultCompressionStrategy::new().with_version(LanceFileVersion::V2_3); + let strategy = strategy(TestEncoding::StructuralSparse, CompressionParams::default()); let compressor = strategy.create_miniblock_compressor(&field, &data).unwrap(); - let (_compressed, encoding) = compressor.compress(data).unwrap(); + let (_compressed, encoding) = compressor.compress(miniblock_context(), data).unwrap(); assert_eq!(rle_run_length_bits(&encoding), 16); } #[test] fn test_rle_v2_miniblock_keeps_u8_run_lengths_before_v2_3() { - for version in [LanceFileVersion::V2_1, LanceFileVersion::V2_2] { + for version in [TestEncoding::StructuralU16, TestEncoding::StructuralU32] { let mut metadata = HashMap::new(); metadata.insert(RLE_THRESHOLD_META_KEY.to_string(), "1.0".to_string()); metadata.insert(BSS_META_KEY.to_string(), "off".to_string()); @@ -1924,9 +2137,9 @@ mod tests { data.compute_stat(); let data = DataBlock::FixedWidth(data); - let strategy = DefaultCompressionStrategy::new().with_version(version); + let strategy = strategy(version, CompressionParams::default()); let compressor = strategy.create_miniblock_compressor(&field, &data).unwrap(); - let (_compressed, encoding) = compressor.compress(data).unwrap(); + let (_compressed, encoding) = compressor.compress(miniblock_context(), data).unwrap(); assert_eq!(rle_run_length_bits(&encoding), 8, "version={version}"); } } @@ -1949,12 +2162,12 @@ mod tests { data.compute_stat(); let data = DataBlock::FixedWidth(data); - let strategy = DefaultCompressionStrategy::new().with_version(LanceFileVersion::V2_3); + let strategy = strategy(TestEncoding::StructuralSparse, CompressionParams::default()); let compressor = strategy.create_miniblock_compressor(&field, &data).unwrap(); let debug_str = format!("{compressor:?}"); assert!(debug_str.contains("RleEncoder")); - let (_compressed, encoding) = compressor.compress(data).unwrap(); + let (_compressed, encoding) = compressor.compress(miniblock_context(), data).unwrap(); assert_eq!(rle_run_length_bits(&encoding), 16); } @@ -1975,9 +2188,9 @@ mod tests { data.compute_stat(); let data = DataBlock::FixedWidth(data); - let strategy = DefaultCompressionStrategy::new().with_version(LanceFileVersion::V2_3); + let strategy = strategy(TestEncoding::StructuralSparse, CompressionParams::default()); let compressor = strategy.create_miniblock_compressor(&field, &data).unwrap(); - let (_compressed, encoding) = compressor.compress(data).unwrap(); + let (_compressed, encoding) = compressor.compress(miniblock_context(), data).unwrap(); assert_eq!(rle_run_length_bits(&encoding), 16); } @@ -1998,12 +2211,160 @@ mod tests { data.compute_stat(); let data = DataBlock::FixedWidth(data); - let strategy = DefaultCompressionStrategy::new().with_version(LanceFileVersion::V2_3); + let strategy = strategy(TestEncoding::StructuralSparse, CompressionParams::default()); let compressor = strategy.create_miniblock_compressor(&field, &data).unwrap(); - let (_compressed, encoding) = compressor.compress(data).unwrap(); + let (_compressed, encoding) = compressor.compress(miniblock_context(), data).unwrap(); assert_eq!(rle_run_length_bits(&encoding), 8); } + #[test] + #[cfg(any(feature = "lz4", feature = "zstd"))] + fn test_rle_miniblock_released_versions_keep_flat_children_when_compression_requested() { + for version in [TestEncoding::StructuralU16, TestEncoding::StructuralU32] { + let mut params = CompressionParams::new(); + params.columns.insert( + "dict_indices".to_string(), + CompressionFieldParams { + compression: Some( + if cfg!(feature = "lz4") { "lz4" } else { "zstd" }.to_string(), + ), + rle_threshold: Some(1.0), + bss: Some(BssMode::Off), + ..Default::default() + }, + ); + let strategy = strategy(version, params); + let field = create_test_field("dict_indices", DataType::UInt32); + + let mut values = Vec::with_capacity(8192 * 4); + for value in 0..8192u32 { + values.extend(std::iter::repeat_n(value, 4)); + } + let mut data = FixedWidthDataBlock { + bits_per_value: 32, + data: LanceBuffer::reinterpret_vec(values), + num_values: 8192 * 4, + block_info: BlockInfo::default(), + }; + data.compute_stat(); + let data = DataBlock::FixedWidth(data); + + let compressor = strategy.create_miniblock_compressor(&field, &data).unwrap(); + let (_compressed, encoding) = compressor.compress(miniblock_context(), data).unwrap(); + let rle = expect_rle_encoding(&encoding); + + assert!( + matches!( + rle.values.as_ref().unwrap().compression.as_ref().unwrap(), + Compression::Flat(_) + ), + "version={version}" + ); + assert!( + matches!( + rle.run_lengths + .as_ref() + .unwrap() + .compression + .as_ref() + .unwrap(), + Compression::Flat(_) + ), + "version={version}" + ); + } + } + + #[test] + #[cfg(feature = "bitpacking")] + fn test_rle_miniblock_strategy_bitpacks_child_values_when_smaller() { + let field = create_test_field("dict_indices", DataType::Int32); + + let mut values = Vec::with_capacity(8192 * 4); + for value in 0..8192 { + values.extend(std::iter::repeat_n(value, 4)); + } + let mut data = FixedWidthDataBlock { + bits_per_value: 32, + data: LanceBuffer::reinterpret_vec(values), + num_values: 8192 * 4, + block_info: BlockInfo::default(), + }; + data.compute_stat(); + let data = DataBlock::FixedWidth(data); + + let strategy = strategy(TestEncoding::StructuralSparse, CompressionParams::default()); + let compressor = strategy.create_miniblock_compressor(&field, &data).unwrap(); + let debug_str = format!("{compressor:?}"); + assert!(debug_str.contains("RleEncoder")); + + let (_compressed, encoding) = compressor.compress(miniblock_context(), data).unwrap(); + let Compression::Rle(rle) = encoding.compression.as_ref().unwrap() else { + panic!("expected RLE encoding"); + }; + assert!(matches!( + rle.values.as_ref().unwrap().compression.as_ref().unwrap(), + Compression::OutOfLineBitpacking(_) + )); + assert!(matches!( + rle.run_lengths + .as_ref() + .unwrap() + .compression + .as_ref() + .unwrap(), + Compression::Flat(_) + )); + } + + #[test] + #[cfg(feature = "bitpacking")] + fn test_rle_miniblock_keeps_child_bitpacked_rle_when_smaller_than_inline_bitpacking() { + let field = create_test_field("int_score", DataType::UInt64); + + let mut values = Vec::with_capacity(8192 * 8); + for run_idx in 0..8192 { + let value = match run_idx % 3 { + 0 => 3u64, + 1 => 4u64, + _ => 5u64, + }; + values.extend(std::iter::repeat_n(value, 8)); + } + let mut data = FixedWidthDataBlock { + bits_per_value: 64, + data: LanceBuffer::reinterpret_vec(values), + num_values: 8192 * 8, + block_info: BlockInfo::default(), + }; + data.compute_stat(); + let data = DataBlock::FixedWidth(data); + + let strategy = strategy(TestEncoding::StructuralSparse, CompressionParams::default()); + let compressor = strategy.create_miniblock_compressor(&field, &data).unwrap(); + let debug_str = format!("{compressor:?}"); + assert!( + debug_str.contains("RleEncoder"), + "expected RLE to beat inline bitpacking after child selection, got: {debug_str}" + ); + + let (_compressed, encoding) = compressor.compress(miniblock_context(), data).unwrap(); + let rle = expect_rle_encoding(&encoding); + assert!(matches!( + rle.values.as_ref().unwrap().compression.as_ref().unwrap(), + Compression::OutOfLineBitpacking(_) + )); + assert!(matches!( + rle.run_lengths + .as_ref() + .unwrap() + .compression + .as_ref() + .unwrap(), + Compression::Flat(_) + )); + } + #[test] fn test_field_metadata_override_params() { // Set up params with one configuration @@ -2019,7 +2380,7 @@ mod tests { }, ); - let strategy = DefaultCompressionStrategy::with_params(params); + let strategy = baseline_strategy(params); // Field metadata should override params let mut metadata = HashMap::new(); @@ -2047,7 +2408,7 @@ mod tests { }, ); - let strategy = DefaultCompressionStrategy::with_params(params); + let strategy = baseline_strategy(params); // Field metadata provides partial override let mut metadata = HashMap::new(); @@ -2066,7 +2427,7 @@ mod tests { #[test] fn test_bss_field_metadata() { let params = CompressionParams::new(); - let strategy = DefaultCompressionStrategy::with_params(params); + let strategy = baseline_strategy(params); // Test BSS "on" mode with compression enabled (BSS requires compression to be effective) let mut metadata = HashMap::new(); @@ -2087,7 +2448,7 @@ mod tests { #[test] fn test_bss_with_compression() { let params = CompressionParams::new(); - let strategy = DefaultCompressionStrategy::with_params(params); + let strategy = baseline_strategy(params); // Test BSS with LZ4 compression let mut metadata = HashMap::new(); @@ -2120,8 +2481,7 @@ mod tests { }, ); - let mut strategy = DefaultCompressionStrategy::with_params(params); - strategy.version = LanceFileVersion::V2_2; + let strategy = strategy(TestEncoding::StructuralU32, params); let field = create_test_field("dict_values", DataType::FixedSizeBinary(3)); let data = create_fixed_width_block(24, 1024); @@ -2174,8 +2534,7 @@ mod tests { }, ); - let strategy = - DefaultCompressionStrategy::with_params(params).with_version(LanceFileVersion::V2_1); + let strategy = strategy(TestEncoding::StructuralU16, params); let field = create_test_field("dict_values", DataType::FixedSizeBinary(3)); let data = create_fixed_width_block(24, 1024); @@ -2200,8 +2559,7 @@ mod tests { }, ); - let strategy = - DefaultCompressionStrategy::with_params(params).with_version(LanceFileVersion::V2_2); + let strategy = strategy(TestEncoding::StructuralU32, params); let field = create_test_field("dict_values", DataType::FixedSizeBinary(3)); let data = create_fixed_width_block(24, 20_000); @@ -2233,8 +2591,7 @@ mod tests { block.compute_stat(); let data = DataBlock::FixedWidth(block); - let strategy = DefaultCompressionStrategy::with_params(CompressionParams::new()) - .with_version(LanceFileVersion::V2_3); + let strategy = strategy(TestEncoding::StructuralSparse, CompressionParams::new()); let (compressor, encoding) = strategy.create_block_compressor(&field, &data).unwrap(); assert_eq!(rle_run_length_bits(&encoding), 32); @@ -2268,8 +2625,7 @@ mod tests { block.compute_stat(); let data = DataBlock::FixedWidth(block); - let strategy = DefaultCompressionStrategy::with_params(CompressionParams::new()) - .with_version(LanceFileVersion::V2_2); + let strategy = strategy(TestEncoding::StructuralU32, CompressionParams::new()); let (_compressor, encoding) = strategy.create_block_compressor(&field, &data).unwrap(); assert_eq!(rle_run_length_bits(&encoding), 8); } @@ -2298,8 +2654,7 @@ mod tests { let data_block = DataBlock::FixedWidth(block); - let strategy = DefaultCompressionStrategy::with_params(CompressionParams::new()) - .with_version(LanceFileVersion::V2_2); + let strategy = strategy(TestEncoding::StructuralU32, CompressionParams::new()); let (compressor, _) = strategy .create_block_compressor(&field, &data_block) @@ -2333,8 +2688,7 @@ mod tests { let data_block = DataBlock::FixedWidth(block); - let strategy = DefaultCompressionStrategy::with_params(CompressionParams::new()) - .with_version(LanceFileVersion::V2_1); + let strategy = strategy(TestEncoding::StructuralU16, CompressionParams::new()); let (compressor, _) = strategy .create_block_compressor(&field, &data_block) diff --git a/rust/lance-encoding/src/constants.rs b/rust/lance-encoding/src/constants.rs index c95b587a532..0bd31d676cc 100644 --- a/rust/lance-encoding/src/constants.rs +++ b/rust/lance-encoding/src/constants.rs @@ -55,6 +55,8 @@ pub const STRUCTURAL_ENCODING_META_KEY: &str = "lance-encoding:structural-encodi pub const STRUCTURAL_ENCODING_MINIBLOCK: &str = "miniblock"; /// Value for fullzip structural encoding pub const STRUCTURAL_ENCODING_FULLZIP: &str = "fullzip"; +/// Value for sparse structural encoding +pub const STRUCTURAL_ENCODING_SPARSE: &str = "sparse"; // Byte stream split metadata keys /// Metadata key for byte stream split encoding configuration diff --git a/rust/lance-encoding/src/data.rs b/rust/lance-encoding/src/data.rs index c4539214706..9e0f29e2d42 100644 --- a/rust/lance-encoding/src/data.rs +++ b/rust/lance-encoding/src/data.rs @@ -221,7 +221,7 @@ impl FixedWidthDataBlock { } #[derive(Debug)] -pub struct VariableWidthDataBlockBuilder { +struct VariableWidthDataBlockBuilder { offsets: Vec, bytes: Vec, } @@ -236,28 +236,54 @@ impl VariableWidthDataBlockBuilder { } impl DataBlockBuilderImpl for VariableWidthDataBlockBuilder { - fn append(&mut self, data_block: &DataBlock, selection: Range) { + fn validate_append(&self, data_block: &DataBlock, selection: &Range) -> Result<()> { let block = data_block.as_variable_width_ref().unwrap(); - assert!(block.bits_per_offset == T::get_byte_width() as u8 * 8); + block.validate_offsets_for_append::(selection) + } + + fn append_validated(&mut self, data_block: &DataBlock, selection: Range) -> Result<()> { + let block = data_block.as_variable_width_ref().unwrap(); + debug_assert_eq!(block.bits_per_offset, T::get_byte_width() as u8 * 8); let offsets = block.offsets.borrow_to_typed_view::(); let start_offset = offsets[selection.start as usize]; let end_offset = offsets[selection.end as usize]; - let mut previous_len = self.bytes.len(); + let selected_data_len = end_offset.as_usize() - start_offset.as_usize(); + let new_data_len = self + .bytes + .len() + .checked_add(selected_data_len) + .ok_or_else(|| { + Error::not_supported_source( + "appending variable-width data would overflow usize".into(), + ) + })?; + if T::from_usize(new_data_len).is_none() { + return Err(Error::not_supported_source( + format!( + "appending variable-width data would require {} bytes, which exceeds the \ + capacity of {}-bit offsets", + new_data_len, + T::get_byte_width() * 8 + ) + .into(), + )); + } + let previous_len = self.bytes.len(); self.bytes .extend_from_slice(&block.data[start_offset.as_usize()..end_offset.as_usize()]); self.offsets.extend( - offsets[selection.start as usize..selection.end as usize] + offsets[selection.start as usize + 1..=selection.end as usize] .iter() - .zip(&offsets[selection.start as usize + 1..=selection.end as usize]) - .map(|(¤t, &next)| { - let this_value_len = next - current; - previous_len += this_value_len.as_usize(); - T::from_usize(previous_len).unwrap() + .map(|&offset| { + let rebased_offset = + previous_len + (offset.as_usize() - start_offset.as_usize()); + T::from_usize(rebased_offset).unwrap() }), ); + Ok(()) } fn finish(self: Box) -> DataBlock { @@ -286,12 +312,17 @@ impl BitmapDataBlockBuilder { } impl DataBlockBuilderImpl for BitmapDataBlockBuilder { - fn append(&mut self, data_block: &DataBlock, selection: Range) { + fn validate_append(&self, _data_block: &DataBlock, _selection: &Range) -> Result<()> { + Ok(()) + } + + fn append_validated(&mut self, data_block: &DataBlock, selection: Range) -> Result<()> { let bitmap_blk = data_block.as_fixed_width_ref().unwrap(); self.values.append_packed_range( selection.start as usize..selection.end as usize, &bitmap_blk.data, ); + Ok(()) } fn finish(mut self: Box) -> DataBlock { @@ -326,12 +357,17 @@ impl FixedWidthDataBlockBuilder { } impl DataBlockBuilderImpl for FixedWidthDataBlockBuilder { - fn append(&mut self, data_block: &DataBlock, selection: Range) { + fn validate_append(&self, _data_block: &DataBlock, _selection: &Range) -> Result<()> { + Ok(()) + } + + fn append_validated(&mut self, data_block: &DataBlock, selection: Range) -> Result<()> { let block = data_block.as_fixed_width_ref().unwrap(); assert_eq!(self.bits_per_value, block.bits_per_value); let start = selection.start as usize * self.bytes_per_value as usize; let end = selection.end as usize * self.bytes_per_value as usize; self.values.extend_from_slice(&block.data[start..end]); + Ok(()) } fn finish(self: Box) -> DataBlock { @@ -357,11 +393,20 @@ impl StructDataBlockBuilder { } impl DataBlockBuilderImpl for StructDataBlockBuilder { - fn append(&mut self, data_block: &DataBlock, selection: Range) { + fn validate_append(&self, data_block: &DataBlock, selection: &Range) -> Result<()> { let data_block = data_block.as_struct_ref().unwrap(); for i in 0..self.children.len() { - self.children[i].append(&data_block.children[i], selection.clone()); + self.children[i].validate_append(&data_block.children[i], selection)?; } + Ok(()) + } + + fn append_validated(&mut self, data_block: &DataBlock, selection: Range) -> Result<()> { + let data_block = data_block.as_struct_ref().unwrap(); + for i in 0..self.children.len() { + self.children[i].append_validated(&data_block.children[i], selection.clone())?; + } + Ok(()) } fn finish(self: Box) -> DataBlock { @@ -384,8 +429,13 @@ struct AllNullDataBlockBuilder { } impl DataBlockBuilderImpl for AllNullDataBlockBuilder { - fn append(&mut self, _data_block: &DataBlock, selection: Range) { + fn validate_append(&self, _data_block: &DataBlock, _selection: &Range) -> Result<()> { + Ok(()) + } + + fn append_validated(&mut self, _data_block: &DataBlock, selection: Range) -> Result<()> { self.num_values += selection.end - selection.start; + Ok(()) } fn finish(self: Box) -> DataBlock { @@ -503,10 +553,16 @@ impl FixedSizeListBlockBuilder { } impl DataBlockBuilderImpl for FixedSizeListBlockBuilder { - fn append(&mut self, data_block: &DataBlock, selection: Range) { + fn validate_append(&self, data_block: &DataBlock, selection: &Range) -> Result<()> { let selection = selection.start * self.dimension..selection.end * self.dimension; let fsl = data_block.as_fixed_size_list_ref().unwrap(); - self.inner.append(fsl.child.as_ref(), selection); + self.inner.validate_append(fsl.child.as_ref(), &selection) + } + + fn append_validated(&mut self, data_block: &DataBlock, selection: Range) -> Result<()> { + let selection = selection.start * self.dimension..selection.end * self.dimension; + let fsl = data_block.as_fixed_size_list_ref().unwrap(); + self.inner.append_validated(fsl.child.as_ref(), selection) } fn finish(self: Box) -> DataBlock { @@ -534,15 +590,23 @@ impl NullableDataBlockBuilder { } impl DataBlockBuilderImpl for NullableDataBlockBuilder { - fn append(&mut self, data_block: &DataBlock, selection: Range) { + fn validate_append(&self, data_block: &DataBlock, selection: &Range) -> Result<()> { let nullable = data_block.as_nullable_ref().unwrap(); + self.inner + .validate_append(nullable.data.as_ref(), selection) + } + + fn append_validated(&mut self, data_block: &DataBlock, selection: Range) -> Result<()> { + let nullable = data_block.as_nullable_ref().unwrap(); + self.inner + .append_validated(nullable.data.as_ref(), selection.clone())?; let bool_buf = BooleanBuffer::new( nullable.nulls.clone().into_buffer(), selection.start as usize, (selection.end - selection.start) as usize, ); self.validity.append_buffer(&bool_buf); - self.inner.append(nullable.data.as_ref(), selection); + Ok(()) } fn finish(mut self: Box) -> DataBlock { @@ -588,20 +652,315 @@ pub struct VariableWidthBlock { pub block_info: BlockInfo, } +/// Proof that a [`VariableWidthBlock`] satisfies the Arrow layout contract for +/// its target data type (offsets buffer long enough, offsets monotonic and +/// within the data buffer, values valid UTF-8 where required). +/// +/// Only [`VariableWidthBlock::validate_layout`] can construct it, which ties the +/// unchecked Arrow build below to an actual validation pass instead of a +/// caller-controlled flag. +struct ValidVariableWidthLayout; + impl VariableWidthBlock { - fn into_arrow(self, data_type: DataType, validate: bool) -> Result { + fn append_error(&self, selection: &Range, detail: impl std::fmt::Display) -> Error { + Error::corrupt_file_named( + "variable width data block", + format!( + "cannot append offsets for selection {}..{}: {} (num_values: {}, \ + bits_per_offset: {}, offsets buffer size: {} bytes, data buffer size: {} bytes)", + selection.start, + selection.end, + detail, + self.num_values, + self.bits_per_offset, + self.offsets.len(), + self.data.len(), + ), + ) + } + + fn validate_offsets_for_append(&self, selection: &Range) -> Result<()> + where + T: OffsetSizeTrait + bytemuck::Pod, + { + let expected_bits_per_offset = T::get_byte_width() as u8 * 8; + if self.bits_per_offset != expected_bits_per_offset { + return Err(self.append_error( + selection, + format!( + "expected {}-bit offsets but found {}-bit offsets", + expected_bits_per_offset, self.bits_per_offset + ), + )); + } + let offset_size = std::mem::size_of::(); + if !self.offsets.len().is_multiple_of(offset_size) { + return Err(self.append_error( + selection, + format!( + "offsets buffer length {} is not a multiple of the {}-byte offset width", + self.offsets.len(), + offset_size + ), + )); + } + if selection.start > selection.end || selection.end > self.num_values { + return Err( + self.append_error(selection, "selection is outside the block's value range") + ); + } + let selection_start = usize::try_from(selection.start) + .map_err(|_| self.append_error(selection, "selection start does not fit in usize"))?; + let selection_end = usize::try_from(selection.end) + .map_err(|_| self.append_error(selection, "selection end does not fit in usize"))?; + let offsets = self.offsets.borrow_to_typed_view::(); + if selection_end >= offsets.len() { + return Err(self.append_error( + selection, + format!( + "selection requires offset {} but the buffer holds {} offsets", + selection_end, + offsets.len() + ), + )); + } + let selected_offsets = &offsets[selection_start..=selection_end]; + if let Some(detail) = + Self::offset_violation_detail(selected_offsets, self.data.len(), selection_start) + { + return Err(self.append_error(selection, detail)); + } + Ok(()) + } + + // The offsets buffer comes straight from file bytes, so an unchecked build would + // let a corrupt file smuggle out-of-bounds offsets into an Arrow array whose + // consumers then read (or crash on) memory outside the data buffer. This + // boundary therefore always validates the layout, ignoring the optional + // `validate` flag. Lance validates the common layouts itself (a branchless + // scan, measurably cheaper than Arrow's element-wise checked build) and only + // falls back to Arrow's checked build for the cold cases. + fn into_arrow(self, data_type: DataType, _validate: bool) -> Result { + let Some(expected_bits_per_offset) = Self::expected_bits_per_offset(&data_type) else { + // Not an [offsets, bytes] layout we know how to prove; let Arrow + // check it. + return self.into_arrow_checked(data_type); + }; + if self.bits_per_offset != expected_bits_per_offset { + return Err(self.layout_error( + &data_type, + format!( + "expected {}-bit offsets but got {}-bit offsets", + expected_bits_per_offset, self.bits_per_offset + ), + )); + } + if self.num_values == 0 { + // Cold path; Arrow handles the empty-offsets special cases. + return self.into_arrow_checked(data_type); + } + let proof = self.validate_layout(&data_type)?; + Ok(self.into_arrow_unchecked(data_type, proof)) + } + + /// The offset width Arrow mandates for `data_type`, or `None` if the type + /// does not use the `[offsets, bytes]` layout this block represents. + fn expected_bits_per_offset(data_type: &DataType) -> Option { + match data_type { + DataType::Binary | DataType::Utf8 => Some(32), + DataType::LargeBinary | DataType::LargeUtf8 => Some(64), + _ => None, + } + } + + fn layout_error(&self, data_type: &DataType, detail: impl std::fmt::Display) -> Error { + Self::format_layout_error( + data_type, + detail, + self.num_values, + self.bits_per_offset, + self.offsets.len(), + self.data.len(), + ) + } + + fn format_layout_error( + data_type: &DataType, + detail: impl std::fmt::Display, + num_values: u64, + bits_per_offset: u8, + offsets_size: usize, + data_size: usize, + ) -> Error { + Error::corrupt_file_named( + "variable width data block", + format!( + "invalid variable-width layout for {}: {} (num_values: {}, bits_per_offset: {}, \ + offsets buffer size: {} bytes, data buffer size: {} bytes)", + data_type, detail, num_values, bits_per_offset, offsets_size, data_size, + ), + ) + } + + fn validate_layout(&self, data_type: &DataType) -> Result { + let bytes_per_offset = (self.bits_per_offset / 8) as u64; + let required_bytes = self + .num_values + .checked_add(1) + .and_then(|num_offsets| num_offsets.checked_mul(bytes_per_offset)) + .ok_or_else(|| self.layout_error(data_type, "offsets buffer size overflows"))?; + if (self.offsets.len() as u64) < required_bytes { + return Err(self.layout_error( + data_type, + format!( + "offsets buffer must hold at least {} offsets ({} bytes)", + self.num_values + 1, + required_bytes + ), + )); + } + let validate_utf8 = matches!(data_type, DataType::Utf8 | DataType::LargeUtf8); + match self.bits_per_offset { + 32 => self.validate_offsets_and_values::(data_type, validate_utf8), + 64 => self.validate_offsets_and_values::(data_type, validate_utf8), + other => Err(self.layout_error( + data_type, + format!("unsupported offset width: {} bits", other), + )), + } + } + + fn validate_offsets_and_values( + &self, + data_type: &DataType, + validate_utf8: bool, + ) -> Result { + let num_offsets = self.num_values as usize + 1; + // Slice before borrowing: the buffer may carry padding that is not a + // multiple of the offset width. + let offsets = self + .offsets + .slice_with_length(0, num_offsets * std::mem::size_of::()); + let offsets = offsets.borrow_to_typed_slice::(); + let offsets: &[T] = offsets.as_ref(); + let data = self.data.as_ref(); + + if let Some(detail) = Self::offset_violation_detail(offsets, data.len(), 0) { + return Err(self.layout_error(data_type, detail)); + } + + if validate_utf8 { + let (first, last) = (offsets[0].as_usize(), offsets[num_offsets - 1].as_usize()); + let values = std::str::from_utf8(&data[first..last]) + .map_err(|utf8_err| self.layout_error(data_type, utf8_err))?; + let mut on_char_boundaries = true; + for &offset in offsets { + on_char_boundaries &= values.is_char_boundary(offset.as_usize() - first); + } + if !on_char_boundaries { + // Cold path: rescan to pinpoint the offending offset. + let position = offsets + .iter() + .position(|offset| !values.is_char_boundary(offset.as_usize() - first)) + .expect("the fast scan found a non-boundary offset"); + return Err(self.layout_error( + data_type, + format!("offset at position {position} splits a UTF-8 character"), + )); + } + } + + Ok(ValidVariableWidthLayout) + } + + fn offset_violation_detail( + offsets: &[T], + data_size: usize, + position_base: usize, + ) -> Option { + // A monotonic sequence with a non-negative first offset and an + // in-bounds last offset is entirely within [0, data_size]. Keep this + // valid path branchless so it vectorizes, and only rescan on failure. + let mut is_monotonic = true; + for window in offsets.windows(2) { + is_monotonic &= window[0] <= window[1]; + } + let first = offsets[0]; + let last = offsets[offsets.len() - 1]; + let bounds_ok = + first >= T::usize_as(0) && last.to_usize().is_some_and(|last| last <= data_size); + if is_monotonic && bounds_ok { + return None; + } + + for (relative_position, window) in offsets.windows(2).enumerate() { + if window[0] > window[1] { + let position = position_base + relative_position + 1; + return Some(format!( + "non-monotonic offset at position {}: {:?} decreases from {:?}", + position, window[1], window[0] + )); + } + } + for (relative_position, offset) in offsets.iter().enumerate() { + let position = position_base + relative_position; + match offset.to_usize() { + None => { + return Some(format!( + "offset at position {} is negative: {:?}", + position, offset + )); + } + Some(offset) if offset > data_size => { + return Some(format!( + "offset at position {} is out of bounds: {} > {}", + position, offset, data_size + )); + } + Some(_) => {} + } + } + Some("offsets failed validation".to_string()) + } + + fn into_arrow_checked(self, data_type: DataType) -> Result { + let num_values = self.num_values; + let bits_per_offset = self.bits_per_offset; + let offsets_size = self.offsets.len(); + let data_size = self.data.len(); + let builder = self.into_arrow_builder(data_type.clone()); + builder.build().map_err(|arrow_err| { + Self::format_layout_error( + &data_type, + arrow_err, + num_values, + bits_per_offset, + offsets_size, + data_size, + ) + }) + } + + fn into_arrow_unchecked( + self, + data_type: DataType, + _proof: ValidVariableWidthLayout, + ) -> ArrayData { + let builder = self.into_arrow_builder(data_type); + // SAFETY: `_proof` witnesses that `validate_layout` proved this block + // satisfies the Arrow layout contract for `data_type`. + unsafe { builder.build_unchecked() } + } + + fn into_arrow_builder(self, data_type: DataType) -> ArrayDataBuilder { + let num_values = self.num_values; let data_buffer = self.data.into_buffer(); let offsets_buffer = self.offsets.into_buffer(); - let builder = ArrayDataBuilder::new(data_type) + ArrayDataBuilder::new(data_type) .add_buffer(offsets_buffer) .add_buffer(data_buffer) - .len(self.num_values as usize) - .null_count(0); - if validate { - Ok(builder.build()?) - } else { - Ok(unsafe { builder.build_unchecked() }) - } + .len(num_values as usize) + .null_count(0) } fn into_buffers(self) -> Vec { @@ -727,12 +1086,11 @@ impl DictionaryDataBlock { let indices = self.indices.data.borrow_to_typed_slice::(); let indices = indices.as_ref(); - indices - .iter() - .map(|idx| idx.to_usize().unwrap() as u64) - .for_each(|idx| { - data_builder.append(&self.dictionary, idx..idx + 1); - }); + let selections = indices.iter().map(|idx| { + let idx = idx.to_usize().unwrap() as u64; + idx..idx + 1 + }); + data_builder.append_ranges(&self.dictionary, selections)?; Ok(data_builder.finish()) } @@ -756,6 +1114,16 @@ impl DictionaryDataBlock { value_type: Box, validate: bool, ) -> Result { + let declared_key_bits = key_type.byte_width() as u64 * 8; + if self.indices.bits_per_value != declared_key_bits { + return Err(lance_core::Error::corrupt_file_named( + "dictionary", + format!( + "dictionary indices use {} bits but the declared {} key type uses {} bits", + self.indices.bits_per_value, key_type, declared_key_bits + ), + )); + } let indices = self.indices.into_arrow((*key_type).clone(), validate)?; let dictionary = self .dictionary @@ -1002,7 +1370,7 @@ impl DataBlock { } } - pub fn make_builder(&self, estimated_size_bytes: u64) -> Box { + fn make_builder(&self, estimated_size_bytes: u64) -> Box { match self { Self::FixedWidth(inner) => { if inner.bits_per_value == 1 { @@ -1335,8 +1703,6 @@ fn arrow_dictionary_to_data_block(arrays: &[ArrayRef], validity: Option max_index_val { - // Widen the index type - if max_index_val >= u32::MAX as u64 { - unimplemented!("Dictionary arrays with 2^32 unique value (or more) and a null") - } - upcast = Some(arrow_cast::cast(indices, &DataType::UInt32).unwrap()); - indices = upcast.as_ref().unwrap(); - } - null_index + values.len() - 1 }); + let max_index_val = max_index_val(indices.data_type()); + let upcast = if first_invalid_index as u64 > max_index_val { + // Widen the index type when the null dictionary value cannot be addressed by the + // declared key type, whether the value already existed or was appended above. + if max_index_val >= u32::MAX as u64 { + unimplemented!("Dictionary arrays with 2^32 unique value (or more) and a null") + } + Some(arrow_cast::cast(indices, &DataType::UInt32).unwrap()) + } else { + None + }; + if let Some(upcast) = upcast.as_ref() { + indices = upcast; + } // This can't fail since we already checked for fit let null_index_arr = arrow_cast::cast( &UInt64Array::from(vec![first_invalid_index as u64]), @@ -1611,8 +1981,16 @@ impl From for DataBlock { } } -pub trait DataBlockBuilderImpl: std::fmt::Debug { - fn append(&mut self, data_block: &DataBlock, selection: Range); +trait DataBlockBuilderImpl: std::fmt::Debug { + fn validate_append(&self, data_block: &DataBlock, selection: &Range) -> Result<()>; + + fn append_validated(&mut self, data_block: &DataBlock, selection: Range) -> Result<()>; + + fn append(&mut self, data_block: &DataBlock, selection: Range) -> Result<()> { + self.validate_append(data_block, &selection)?; + self.append_validated(data_block, selection) + } + fn finish(self: Box) -> DataBlock; } @@ -1637,8 +2015,31 @@ impl DataBlockBuilder { self.builder.as_mut().unwrap().as_mut() } - pub fn append(&mut self, data_block: &DataBlock, selection: Range) { - self.get_builder(data_block).append(data_block, selection); + pub fn append(&mut self, data_block: &DataBlock, selection: Range) -> Result<()> { + self.get_builder(data_block).append(data_block, selection) + } + + fn append_ranges( + &mut self, + data_block: &DataBlock, + selections: impl IntoIterator>, + ) -> Result<()> { + let full_selection = 0..data_block.num_values(); + let builder = self.get_builder(data_block); + builder.validate_append(data_block, &full_selection)?; + for selection in selections { + if selection.start > selection.end || selection.end > full_selection.end { + return Err(Error::corrupt_file_named( + "data block", + format!( + "cannot append selection {}..{} from a block with {} values", + selection.start, selection.end, full_selection.end + ), + )); + } + builder.append_validated(data_block, selection)?; + } + Ok(()) } pub fn finish(self) -> DataBlock { @@ -1652,19 +2053,25 @@ mod tests { use std::sync::Arc; use arrow_array::{ - ArrayRef, BinaryViewArray, DictionaryArray, Int8Array, LargeBinaryArray, StringArray, - StringViewArray, UInt8Array, UInt16Array, make_array, new_null_array, + ArrayRef, BinaryArray, BinaryViewArray, DictionaryArray, Int8Array, LargeBinaryArray, + LargeStringArray, StringArray, StringViewArray, UInt8Array, UInt16Array, make_array, + new_null_array, types::{Int8Type, Int32Type}, }; use arrow_buffer::{BooleanBuffer, NullBuffer}; use arrow_schema::{DataType, Field, Fields}; + use lance_core::Error; use lance_datagen::{ArrayGeneratorExt, DEFAULT_SEED, RowCount, array}; use rand::SeedableRng; + use rstest::rstest; use crate::buffer::LanceBuffer; - use super::{AllNullDataBlock, DataBlock}; + use super::{ + AllNullDataBlock, BlockInfo, DataBlock, DataBlockBuilder, DictionaryDataBlock, + FixedWidthDataBlock, VariableWidthBlock, + }; use arrow_array::Array; @@ -2079,4 +2486,260 @@ mod tests { let total_nulls_size_in_bytes = concatenated_array.nulls().unwrap().len().div_ceil(8); assert!(block.data_size() == (total_buffer_size + total_nulls_size_in_bytes) as u64); } + + #[test] + fn variable_width_rejects_out_of_bounds_offsets_without_optional_validation() { + let block = VariableWidthBlock { + data: LanceBuffer::copy_slice(b"alphabetagamma"), + offsets: LanceBuffer::reinterpret_vec(vec![0_i32, 5, 9, 100_000]), + bits_per_offset: 32, + num_values: 3, + block_info: BlockInfo::new(), + }; + + let error = block + .into_arrow(DataType::Binary, false) + .expect_err("out-of-bounds offsets must be rejected"); + assert!( + matches!(error, Error::CorruptFile { .. }), + "expected CorruptFile, got: {error}" + ); + let message = error.to_string(); + assert!( + message.contains("100000") && message.contains("data buffer size: 14 bytes"), + "error must report the offending offset and the data buffer size: {message}" + ); + } + + #[rstest] + #[case::i32_decreasing( + LanceBuffer::reinterpret_vec(vec![0_i32, 5, 2]), + 32, + 2, + "decreases" + )] + #[case::i64_decreasing( + LanceBuffer::reinterpret_vec(vec![0_i64, 5, 2]), + 64, + 2, + "decreases" + )] + #[case::i32_out_of_bounds( + LanceBuffer::reinterpret_vec(vec![0_i32, 6]), + 32, + 1, + "out of bounds" + )] + fn variable_width_builder_rejects_malformed_offsets( + #[case] offsets: LanceBuffer, + #[case] bits_per_offset: u8, + #[case] num_values: u64, + #[case] expected_message: &str, + ) { + let block = DataBlock::VariableWidth(VariableWidthBlock { + data: LanceBuffer::copy_slice(b"abcde"), + offsets, + bits_per_offset, + num_values, + block_info: BlockInfo::new(), + }); + let mut builder = DataBlockBuilder::with_capacity_estimate(5); + + let error = builder + .append(&block, 0..num_values) + .expect_err("malformed offsets must fail concatenation"); + assert!( + matches!(error, Error::CorruptFile { .. }), + "expected CorruptFile, got: {error}" + ); + assert!( + error.to_string().contains(expected_message), + "unexpected message: {error}" + ); + } + + #[rstest] + #[case::binary_i32_tail_out_of_bounds( + DataType::Binary, + LanceBuffer::reinterpret_vec(vec![0_i32, 5, 9, 100_000]), + 32, + 3, + b"alphabetagamma".as_slice() + )] + #[case::utf8_i32_tail_out_of_bounds( + DataType::Utf8, + LanceBuffer::reinterpret_vec(vec![0_i32, 5, 9, 100_000]), + 32, + 3, + b"alphabetagamma".as_slice() + )] + #[case::large_binary_i64_tail_out_of_bounds( + DataType::LargeBinary, + LanceBuffer::reinterpret_vec(vec![0_i64, 5, 9, 100_000]), + 64, + 3, + b"alphabetagamma".as_slice() + )] + #[case::large_utf8_i64_tail_out_of_bounds( + DataType::LargeUtf8, + LanceBuffer::reinterpret_vec(vec![0_i64, 5, 9, 100_000]), + 64, + 3, + b"alphabetagamma".as_slice() + )] + #[case::binary_negative_offset( + DataType::Binary, + LanceBuffer::reinterpret_vec(vec![0_i32, -1, 9, 14]), + 32, + 3, + b"alphabetagamma".as_slice() + )] + #[case::binary_non_monotonic_offsets( + DataType::Binary, + LanceBuffer::reinterpret_vec(vec![0_i32, 9, 5, 14]), + 32, + 3, + b"alphabetagamma".as_slice() + )] + #[case::binary_interior_offset_out_of_bounds( + DataType::Binary, + LanceBuffer::reinterpret_vec(vec![0_i32, 100_000, 100_000, 14]), + 32, + 3, + b"alphabetagamma".as_slice() + )] + #[case::binary_offsets_buffer_too_short( + DataType::Binary, + LanceBuffer::reinterpret_vec(vec![0_i32, 5, 9]), + 32, + 3, + b"alphabetagamma".as_slice() + )] + #[case::utf8_invalid_byte_sequence( + DataType::Utf8, + LanceBuffer::reinterpret_vec(vec![0_i32, 1, 2, 3]), + 32, + 3, + &[b'a', 0xFF, b'b'] + )] + #[case::utf8_offset_splits_multibyte_char( + DataType::Utf8, + LanceBuffer::reinterpret_vec(vec![0_i32, 1, 2]), + 32, + 2, + "é".as_bytes() + )] + #[case::large_utf8_invalid_byte_sequence( + DataType::LargeUtf8, + LanceBuffer::reinterpret_vec(vec![0_i64, 1, 2, 3]), + 64, + 3, + &[b'a', 0xFF, b'b'] + )] + fn variable_width_rejects_malformed_layout( + #[case] data_type: DataType, + #[case] offsets: LanceBuffer, + #[case] bits_per_offset: u8, + #[case] num_values: u64, + #[case] data: &[u8], + ) { + let block = VariableWidthBlock { + data: LanceBuffer::copy_slice(data), + offsets, + bits_per_offset, + num_values, + block_info: BlockInfo::new(), + }; + + // The malformed layout must be rejected regardless of the optional + // `validate` flag: the flag selects extra validation, not the memory + // safety proof required to construct an Arrow array. + for validate in [false, true] { + let error = DataBlock::VariableWidth(block.clone()) + .into_arrow(data_type.clone(), validate) + .expect_err("malformed variable-width layout must be rejected"); + assert!( + matches!(error, Error::CorruptFile { .. }), + "expected CorruptFile with validate={validate}, got: {error}" + ); + } + } + + #[test] + fn dictionary_rejects_malformed_variable_width_values_without_optional_validation() { + let values = VariableWidthBlock { + data: LanceBuffer::copy_slice(b"alphabetagamma"), + offsets: LanceBuffer::reinterpret_vec(vec![0_i32, 5, 9, 100_000]), + bits_per_offset: 32, + num_values: 3, + block_info: BlockInfo::new(), + }; + let dictionary = DataBlock::Dictionary(DictionaryDataBlock { + indices: FixedWidthDataBlock { + data: LanceBuffer::reinterpret_vec(vec![0_i32, 1, 2]), + bits_per_value: 32, + num_values: 3, + block_info: BlockInfo::new(), + }, + dictionary: Box::new(DataBlock::VariableWidth(values)), + }); + + let data_type = DataType::Dictionary(Box::new(DataType::Int32), Box::new(DataType::Binary)); + let error = dictionary + .into_arrow(data_type, false) + .expect_err("dictionary with out-of-bounds value offsets must be rejected"); + assert!( + matches!(error, Error::CorruptFile { .. }), + "expected CorruptFile, got: {error}" + ); + } + + #[test] + fn dictionary_rejects_indices_wider_than_declared_key_type() { + let dictionary = DataBlock::Dictionary(DictionaryDataBlock { + indices: FixedWidthDataBlock { + data: LanceBuffer::reinterpret_vec(vec![0_u32, 1, 128]), + bits_per_value: 32, + num_values: 3, + block_info: BlockInfo::new(), + }, + dictionary: Box::new(DataBlock::from_array(StringArray::from(vec![ + Some("zero"), + Some("one"), + None, + ]))), + }); + + let data_type = DataType::Dictionary(Box::new(DataType::Int8), Box::new(DataType::Utf8)); + let error = dictionary + .into_arrow(data_type, false) + .expect_err("mismatched dictionary index widths must be rejected"); + + assert!(matches!(error, Error::CorruptFile { .. })); + assert!( + error.to_string().contains( + "dictionary indices use 32 bits but the declared Int8 key type uses 8 bits" + ) + ); + } + + #[rstest] + #[case::binary(Arc::new(BinaryArray::from_vec(vec![b"alpha", b"", b"gamma"])) as ArrayRef)] + #[case::large_binary( + Arc::new(LargeBinaryArray::from_vec(vec![b"alpha", b"", b"gamma"])) as ArrayRef + )] + #[case::utf8(Arc::new(StringArray::from(vec!["héllo", "", "world"])) as ArrayRef)] + #[case::large_utf8(Arc::new(LargeStringArray::from(vec!["héllo", "", "world"])) as ArrayRef)] + fn variable_width_valid_data_survives_mandatory_validation(#[case] array: ArrayRef) { + let block = DataBlock::from_array(array.clone()); + for validate in [false, true] { + let round_tripped = make_array( + block + .clone() + .into_arrow(array.data_type().clone(), validate) + .unwrap(), + ); + assert_eq!(&round_tripped, &array); + } + } } diff --git a/rust/lance-encoding/src/decoder.rs b/rust/lance-encoding/src/decoder.rs index 55340d09c94..ff383113694 100644 --- a/rust/lance-encoding/src/decoder.rs +++ b/rust/lance-encoding/src/decoder.rs @@ -240,6 +240,12 @@ use lance_core::error::LanceOptionExt; use lance_core::{ArrowResult, Error, Result}; use tracing::instrument; +use crate::array_encoding::logical::list::OffsetPageInfo; +use crate::array_encoding::logical::r#struct::{SimpleStructDecoder, SimpleStructScheduler}; +use crate::array_encoding::logical::{ + binary::BinaryFieldScheduler, blob::BlobFieldScheduler, list::ListFieldScheduler, + primitive::PrimitiveFieldScheduler, +}; use crate::compression::{DecompressionStrategy, DefaultDecompressionStrategy}; use crate::data::DataBlock; use crate::encoder::EncodedBatch; @@ -250,17 +256,72 @@ use crate::encodings::logical::primitive::StructuralPrimitiveFieldScheduler; use crate::encodings::logical::r#struct::{StructuralStructDecoder, StructuralStructScheduler}; use crate::format::pb::{self, column_encoding}; use crate::format::pb21; -use crate::previous::decoder::LogicalPageDecoder; -use crate::previous::encodings::logical::list::OffsetPageInfo; -use crate::previous::encodings::logical::r#struct::{SimpleStructDecoder, SimpleStructScheduler}; -use crate::previous::encodings::logical::{ - binary::BinaryFieldScheduler, blob::BlobFieldScheduler, list::ListFieldScheduler, - primitive::PrimitiveFieldScheduler, -}; use crate::repdef::{CompositeRepDefUnraveler, RepDefUnraveler}; -use crate::version::LanceFileVersion; use crate::{BufferScheduler, EncodingsIo}; +pub trait SchedulingJob: std::fmt::Debug { + fn schedule_next( + &mut self, + context: &mut SchedulerContext, + priority: &dyn PriorityRange, + ) -> Result; + + fn num_rows(&self) -> u64; +} + +/// Schedules the I/O needed to decode one field. +pub trait FieldScheduler: Send + Sync + std::fmt::Debug { + fn initialize<'a>( + &'a self, + filter: &'a FilterExpression, + context: &'a SchedulerContext, + ) -> BoxFuture<'a, Result<()>>; + + fn schedule_ranges<'a>( + &'a self, + ranges: &[Range], + filter: &FilterExpression, + ) -> Result>; + + fn num_rows(&self) -> u64; +} + +#[derive(Debug)] +pub struct DecoderReady { + pub decoder: Box, + pub path: VecDeque, +} + +/// Stateful decoder for one logical page. +pub trait LogicalPageDecoder: std::fmt::Debug + Send { + fn accept_child(&mut self, _child: DecoderReady) -> Result<()> { + Err(Error::internal(format!( + "The decoder {:?} does not expect children but received a child", + self + ))) + } + + fn wait_for_loaded(&'_ mut self, loaded_need: u64) -> BoxFuture<'_, Result<()>>; + + fn rows_loaded(&self) -> u64; + + fn rows_unloaded(&self) -> u64 { + self.num_rows() - self.rows_loaded() + } + + fn num_rows(&self) -> u64; + + fn rows_drained(&self) -> u64; + + fn rows_left(&self) -> u64 { + self.num_rows() - self.rows_drained() + } + + fn drain(&mut self, num_rows: u64) -> Result; + + fn data_type(&self) -> &DataType; +} + // If users are getting batches over 10MiB large then it's time to reduce the batch size const BATCH_SIZE_BYTES_WARNING: u64 = 10 * 1024 * 1024; const ENV_LANCE_STRUCTURAL_BATCH_DECODE_SPAWN_MODE: &str = @@ -288,13 +349,13 @@ fn inline_scheduling_threshold() -> u64 { }) } -/// Top-level encoding message for a page. Wraps both the -/// legacy pb::ArrayEncoding and the newer pb::PageLayout +/// Top-level encoding message for a page. Wraps both the v2.0 +/// [`pb::ArrayEncoding`] grammar and the structural [`pb21::PageLayout`] grammar. /// /// A file should only use one or the other and never both. /// 2.0 decoders can always assume this is pb::ArrayEncoding /// and 2.1+ decoders can always assume this is pb::PageLayout -#[derive(Debug)] +#[derive(Debug, Clone)] pub enum PageEncoding { Legacy(pb::ArrayEncoding), Structural(pb21::PageLayout), @@ -402,21 +463,21 @@ impl ColumnInfo { enum RootScheduler { Structural(Box), - Legacy(Arc), + Array(Arc), } impl RootScheduler { - fn as_legacy(&self) -> &Arc { + fn as_array(&self) -> &Arc { match self { - Self::Structural(_) => panic!("Expected a legacy scheduler"), - Self::Legacy(s) => s, + Self::Structural(_) => panic!("Expected an array scheduler"), + Self::Array(s) => s, } } fn as_structural(&self) -> &dyn StructuralFieldScheduler { match self { Self::Structural(s) => s.as_ref(), - Self::Legacy(_) => panic!("Expected a structural scheduler"), + Self::Array(_) => panic!("Expected a structural scheduler"), } } } @@ -606,14 +667,14 @@ impl CoreFieldDecoderStrategy { } } - fn is_primitive_legacy(data_type: &DataType) -> bool { + fn is_array_primitive(data_type: &DataType) -> bool { if data_type.is_primitive() { true } else { match data_type { // DataType::is_primitive doesn't consider these primitive but we do DataType::Boolean | DataType::Null | DataType::FixedSizeBinary(_) => true, - DataType::FixedSizeList(inner, _) => Self::is_primitive_legacy(inner.data_type()), + DataType::FixedSizeList(inner, _) => Self::is_array_primitive(inner.data_type()), _ => false, } } @@ -624,7 +685,7 @@ impl CoreFieldDecoderStrategy { field: &Field, column: &ColumnInfo, buffers: FileBuffers, - ) -> Result> { + ) -> Result> { Self::ensure_values_encoded(column, &field.name)?; // Primitive fields map to a single column let column_buffers = ColumnBuffers { @@ -667,52 +728,57 @@ impl CoreFieldDecoderStrategy { column_infos: &mut ColumnInfoIter, buffers: FileBuffers, offsets_column: &ColumnInfo, - ) -> Result> { + ) -> Result> { Self::ensure_values_encoded(offsets_column, &list_field.name)?; let offsets_column_buffers = ColumnBuffers { file_buffers: buffers, positions_and_sizes: &offsets_column.buffer_offsets_and_sizes, }; let items_scheduler = - self.create_legacy_field_scheduler(&list_field.children[0], column_infos, buffers)?; + self.create_array_field_scheduler(&list_field.children[0], column_infos, buffers)?; - let (inner_infos, null_offset_adjustments): (Vec<_>, Vec<_>) = offsets_column + let mut inner_infos = Vec::with_capacity(offsets_column.page_infos.len()); + let mut null_offset_adjustments = Vec::with_capacity(offsets_column.page_infos.len()); + for (page_index, offsets_page) in offsets_column .page_infos .iter() - .filter(|offsets_page| offsets_page.num_rows > 0) - .map(|offsets_page| { - if let Some(pb::array_encoding::ArrayEncoding::List(list_encoding)) = - &offsets_page.encoding.as_legacy().array_encoding - { - let inner = PageInfo { - buffer_offsets_and_sizes: offsets_page.buffer_offsets_and_sizes.clone(), - encoding: PageEncoding::Legacy( - list_encoding.offsets.as_ref().unwrap().as_ref().clone(), - ), - num_rows: offsets_page.num_rows, - priority: 0, - }; - ( - inner, - OffsetPageInfo { - offsets_in_page: offsets_page.num_rows, - null_offset_adjustment: list_encoding.null_offset_adjustment, - num_items_referenced_by_page: list_encoding.num_items, - }, - ) - } else { - // TODO: Should probably return Err here - panic!("Expected a list column"); - } - }) - .unzip(); + .enumerate() + .filter(|(_, offsets_page)| offsets_page.num_rows > 0) + { + let PageEncoding::Legacy(pb::ArrayEncoding { + array_encoding: Some(pb::array_encoding::ArrayEncoding::List(list_encoding)), + }) = &offsets_page.encoding + else { + return Err(Error::invalid_input(format!( + "expected list encoding for field '{}' in column {}, page {} but got {:?}", + list_field.name, offsets_column.index, page_index, offsets_page.encoding + ))); + }; + let offsets_encoding = list_encoding.offsets.as_ref().ok_or_else(|| { + Error::invalid_input(format!( + "list encoding for field '{}' in column {}, page {} is missing its offsets encoding", + list_field.name, offsets_column.index, page_index + )) + })?; + inner_infos.push(PageInfo { + buffer_offsets_and_sizes: offsets_page.buffer_offsets_and_sizes.clone(), + encoding: PageEncoding::Legacy(offsets_encoding.as_ref().clone()), + num_rows: offsets_page.num_rows, + priority: 0, + }); + null_offset_adjustments.push(OffsetPageInfo { + offsets_in_page: offsets_page.num_rows, + null_offset_adjustment: list_encoding.null_offset_adjustment, + num_items_referenced_by_page: list_encoding.num_items, + }); + } let inner = Arc::new(PrimitiveFieldScheduler::new( offsets_column.index, DataType::UInt64, Arc::from(inner_infos.into_boxed_slice()), offsets_column_buffers, self.validate_data, - )) as Arc; + )) as Arc; let items_field = match list_field.data_type() { DataType::List(inner) => inner, DataType::LargeList(inner) => inner, @@ -853,15 +919,15 @@ impl CoreFieldDecoderStrategy { } } - fn create_legacy_field_scheduler( + fn create_array_field_scheduler( &self, field: &Field, column_infos: &mut ColumnInfoIter, buffers: FileBuffers, - ) -> Result> { + ) -> Result> { let data_type = field.data_type(); validate_fixed_size_list_dimensions(&field.name, &data_type)?; - if Self::is_primitive_legacy(&data_type) { + if Self::is_array_primitive(&data_type) { let column_info = column_infos.expect_next()?; let scheduler = self.create_primitive_scheduler(field, column_info, buffers)?; return Ok(scheduler); @@ -920,7 +986,7 @@ impl CoreFieldDecoderStrategy { DataType::FixedSizeList(inner, _dimension) => { // A fixed size list column could either be a physical or a logical decoder // depending on the child data type. - if Self::is_primitive_legacy(inner.data_type()) { + if Self::is_array_primitive(inner.data_type()) { let primitive_col = column_infos.expect_next()?; let scheduler = self.create_primitive_scheduler(field, primitive_col, buffers)?; @@ -930,7 +996,7 @@ impl CoreFieldDecoderStrategy { } } DataType::Dictionary(_key_type, value_type) => { - if Self::is_primitive_legacy(value_type) || value_type.is_binary_like() { + if Self::is_array_primitive(value_type) || value_type.is_binary_like() { let primitive_col = column_infos.expect_next()?; let scheduler = self.create_primitive_scheduler(field, primitive_col, buffers)?; @@ -974,7 +1040,7 @@ impl CoreFieldDecoderStrategy { for field in &field.children { column_infos.next_top_level(); let field_scheduler = - self.create_legacy_field_scheduler(field, column_infos, buffers)?; + self.create_array_field_scheduler(field, column_infos, buffers)?; child_schedulers.push(Arc::from(field_scheduler)); } @@ -1008,7 +1074,7 @@ fn root_column(num_rows: u64) -> ColumnInfo { pb::SimpleStruct {}, )), }), - priority: 0, // not used in legacy scheduler + priority: 0, // not used by the array scheduler buffer_offsets_and_sizes: Arc::new([]), }) .collect::>(); @@ -1024,21 +1090,21 @@ fn root_column(num_rows: u64) -> ColumnInfo { pub enum RootDecoder { Structural(StructuralStructDecoder), - Legacy(SimpleStructDecoder), + Array(SimpleStructDecoder), } impl RootDecoder { pub fn into_structural(self) -> StructuralStructDecoder { match self { Self::Structural(decoder) => decoder, - Self::Legacy(_) => panic!("Expected a structural decoder"), + Self::Array(_) => panic!("Expected a structural decoder"), } } - pub fn into_legacy(self) -> SimpleStructDecoder { + pub fn into_array(self) -> SimpleStructDecoder { match self { - Self::Legacy(decoder) => decoder, - Self::Structural(_) => panic!("Expected a legacy decoder"), + Self::Array(decoder) => decoder, + Self::Structural(_) => panic!("Expected an array decoder"), } } } @@ -1104,27 +1170,27 @@ impl DecodeBatchScheduler { let mut column_iter = ColumnInfoIter::new(columns, &adjusted_column_indices); let strategy = CoreFieldDecoderStrategy::from_decoder_config(decoder_config); let root_scheduler = - strategy.create_legacy_field_scheduler(&root_field, &mut column_iter, buffers)?; + strategy.create_array_field_scheduler(&root_field, &mut column_iter, buffers)?; let context = SchedulerContext::new(io, cache.clone()); root_scheduler.initialize(filter, &context).await?; Ok(Self { - root_scheduler: RootScheduler::Legacy(root_scheduler.into()), + root_scheduler: RootScheduler::Array(root_scheduler.into()), root_fields, cache, }) } } - #[deprecated(since = "0.29.1", note = "This is for legacy 2.0 paths")] + #[deprecated(since = "0.29.1", note = "This is for v2.0 array-encoding paths")] pub fn from_scheduler( - root_scheduler: Arc, + root_scheduler: Arc, root_fields: Fields, cache: Arc, ) -> Self { Self { - root_scheduler: RootScheduler::Legacy(root_scheduler), + root_scheduler: RootScheduler::Array(root_scheduler), root_fields, cache, } @@ -1174,7 +1240,7 @@ impl DecodeBatchScheduler { } } - fn do_schedule_ranges_legacy( + fn do_schedule_ranges_array( &mut self, ranges: &[Range], filter: &FilterExpression, @@ -1185,7 +1251,7 @@ impl DecodeBatchScheduler { // tasks are scheduled at the same top level row. priority: Option>, ) { - let root_scheduler = self.root_scheduler.as_legacy(); + let root_scheduler = self.root_scheduler.as_array(); let rows_requested = ranges.iter().map(|r| r.end - r.start).sum::(); trace!( "Scheduling {} ranges across {}..{} ({} rows){}", @@ -1249,8 +1315,8 @@ impl DecodeBatchScheduler { priority: Option>, ) { match &self.root_scheduler { - RootScheduler::Legacy(_) => { - self.do_schedule_ranges_legacy(ranges, filter, io, schedule_action, priority) + RootScheduler::Array(_) => { + self.do_schedule_ranges_array(ranges, filter, io, schedule_action, priority) } RootScheduler::Structural(_) => { self.do_schedule_ranges_structural(ranges, filter, io, schedule_action) @@ -1422,7 +1488,7 @@ impl BatchDecodeStream { } } - fn accept_decoder(&mut self, decoder: crate::previous::decoder::DecoderReady) -> Result<()> { + fn accept_decoder(&mut self, decoder: DecoderReady) -> Result<()> { if decoder.path.is_empty() { // The root decoder we can ignore Ok(()) @@ -1443,7 +1509,7 @@ impl BatchDecodeStream { let scan_line = scan_line?; self.rows_scheduled = scan_line.scheduled_so_far; for message in scan_line.decoders { - self.accept_decoder(message.into_legacy())?; + self.accept_decoder(message.into_array())?; } } None => { @@ -1552,7 +1618,7 @@ impl BatchDecodeStream { // we can have a single implementation of the batch decode iterator enum RootDecoderMessage { LoadedPage(LoadedPageShard), - LegacyPage(crate::previous::decoder::DecoderReady), + ArrayPage(DecoderReady), } trait RootDecoderType { fn accept_message(&mut self, message: RootDecoderMessage) -> Result<()>; @@ -1576,10 +1642,10 @@ impl RootDecoderType for StructuralStructDecoder { } impl RootDecoderType for SimpleStructDecoder { fn accept_message(&mut self, message: RootDecoderMessage) -> Result<()> { - let RootDecoderMessage::LegacyPage(legacy_page) = message else { + let RootDecoderMessage::ArrayPage(array_page) = message else { unreachable!() }; - self.accept_child(legacy_page) + self.accept_child(array_page) } fn drain_batch(&mut self, num_rows: u64) -> Result { self.drain(num_rows) @@ -1663,7 +1729,7 @@ impl BatchDecodeIterator { // The root decoder we can ignore if !decoder_ready.path.is_empty() { self.root_decoder - .accept_message(RootDecoderMessage::LegacyPage(decoder_ready))?; + .accept_message(RootDecoderMessage::ArrayPage(decoder_ready))?; } } } @@ -1789,7 +1855,8 @@ pub struct StructuralBatchDecodeStream { // - false: run `into_batch` inline, which avoids Tokio scheduling overhead and is // typically better for point lookups / small takes. spawn_batch_decode_tasks: bool, - /// If set, target this many bytes per batch instead of `rows_per_batch` rows. + /// If set, target this many bytes per batch while retaining `rows_per_batch` + /// as an independent upper bound. batch_size_bytes: Option, /// Schema-based estimate of bytes per row, computed once at construction. /// Only meaningful when `batch_size_bytes` is `Some`. @@ -1877,6 +1944,7 @@ impl StructuralBatchDecodeStream { return Ok(None); } + let row_limit = self.rows_remaining.min(self.rows_per_batch as u64); let mut to_take = if let Some(batch_size_bytes) = self.batch_size_bytes { let feedback = self.bytes_per_row_feedback.load(Ordering::Relaxed); let bpr = if feedback > 0 { @@ -1885,9 +1953,9 @@ impl StructuralBatchDecodeStream { self.schema_bytes_per_row }; let rows = (batch_size_bytes as f64 / bpr) as u64; - self.rows_remaining.min(rows.max(1)) + row_limit.min(rows.max(1)) } else { - self.rows_remaining.min(self.rows_per_batch as u64) + row_limit }; self.rows_remaining -= to_take; @@ -1949,8 +2017,7 @@ impl StructuralBatchDecodeStream { next_task.into_batch(emitted_batch_size_warning)? }; let num_rows = batch.num_rows() as u64; - if num_rows > 0 { - let bpr = data_size / num_rows; + if let Some(bpr) = data_size.checked_div(num_rows) { let prev = bytes_per_row_feedback.load(Ordering::Relaxed); let next = if prev == 0 || bpr >= prev { // First batch or actual size is larger than estimate: @@ -2052,7 +2119,8 @@ pub struct SchedulerDecoderConfig { pub cache: Arc, /// Decoder configuration pub decoder_config: DecoderConfig, - /// If set, target this many bytes per batch instead of using `batch_size` rows. + /// If set, target this many bytes per batch while retaining `batch_size` as + /// an independent row-count upper bound. /// /// Only supported for v2.1+ (structural) files. For v2.0 files this /// option is ignored and a warning is logged. @@ -2632,17 +2700,14 @@ impl SchedulerContext { VecDeque::from_iter(self.path.iter().copied()) } - #[deprecated(since = "0.29.1", note = "This is for legacy 2.0 paths")] - pub fn locate_decoder( - &mut self, - decoder: Box, - ) -> crate::previous::decoder::DecoderReady { + #[deprecated(since = "0.29.1", note = "This is for v2.0 array-encoding paths")] + pub fn locate_decoder(&mut self, decoder: Box) -> DecoderReady { trace!( "Scheduling decoder of type {:?} for {:?}", decoder.data_type(), self.path, ); - crate::previous::decoder::DecoderReady { + DecoderReady { decoder, path: self.current_path(), } @@ -2718,8 +2783,9 @@ pub trait DecodeArrayTask: Send { impl DecodeArrayTask for Box { fn decode(self: Box) -> Result<(ArrayRef, u64)> { - StructuralDecodeArrayTask::decode(*self) - .map(|decoded_array| (decoded_array.array, decoded_array.data_size)) + let decoded_array = StructuralDecodeArrayTask::decode(*self)?; + decoded_array.repdef.ensure_exhausted()?; + Ok((decoded_array.array, decoded_array.data_size)) } } @@ -2741,10 +2807,7 @@ impl NextDecodeTask { // suggesting the user try a smaller batch size. #[instrument(name = "task_to_batch", level = "debug", skip_all)] fn into_batch(self, emitted_batch_size_warning: Arc) -> Result<(RecordBatch, u64)> { - let (struct_arr, data_size) = self - .task - .decode() - .map_err(|e| Error::internal(format!("Error decoding batch: {}", e)))?; + let (struct_arr, data_size) = self.task.decode()?; let batch = RecordBatch::from(struct_arr.as_struct()); if data_size > BATCH_SIZE_BYTES_WARNING { emitted_batch_size_warning.call_once(|| { @@ -2765,7 +2828,7 @@ pub enum MessageType { // decoder itself. The messages were not sent in priority order and the decoder // had to wait for I/O, figuring out the correct priority. This was a lot of // complexity. - DecoderReady(crate::previous::decoder::DecoderReady), + DecoderReady(DecoderReady), // Starting in 2.1 we use a simpler scheme where the scheduling happens in priority // order and the message is an unloaded decoder. These can be awaited, in order, and // the decoder does not have to worry about waiting for I/O. @@ -2773,7 +2836,7 @@ pub enum MessageType { } impl MessageType { - pub fn into_legacy(self) -> crate::previous::decoder::DecoderReady { + pub fn into_array(self) -> DecoderReady { match self { Self::DecoderReady(decoder) => decoder, Self::UnloadedPage(_) => { @@ -2873,13 +2936,22 @@ pub trait StructuralFieldDecoder: std::fmt::Debug + Send { #[derive(Debug, Default)] pub struct DecoderPlugins {} +/// The top-level column layout used by an in-memory encoded batch. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum EncodedBatchLayout { + /// Array pages include structural columns. + Array, + /// Structural pages include only leaf columns. + Structural, +} + /// Decodes a batch of data from an in-memory structure created by [`crate::encoder::encode_batch`] pub async fn decode_batch( batch: &EncodedBatch, filter: &FilterExpression, decoder_plugins: Arc, should_validate: bool, - version: LanceFileVersion, + layout: EncodedBatchLayout, cache: Option>, ) -> Result { // The io is synchronous so it shouldn't be possible for any async stuff to still be in progress @@ -2909,7 +2981,7 @@ pub async fn decode_batch( .await?; let (tx, rx) = unbounded_channel(); decode_scheduler.schedule_range(0..batch.num_rows, filter, tx, io_scheduler); - let is_structural = version >= LanceFileVersion::V2_1; + let is_structural = layout == EncodedBatchLayout::Structural; let mode = std::env::var(ENV_LANCE_STRUCTURAL_BATCH_DECODE_SPAWN_MODE); let spawn_structural_batch_decode_tasks = !matches!(mode.ok().as_deref(), Some("never")); let mut decode_stream = create_decode_stream( @@ -2929,7 +3001,6 @@ pub async fn decode_batch( // test coalesce indices to ranges mod tests { use super::*; - use crate::previous::decoder::{DecoderReady, LogicalPageDecoder}; use std::collections::VecDeque; #[derive(Debug)] @@ -2984,6 +3055,25 @@ mod tests { } } + struct InvalidInputDecodeTask; + + impl DecodeArrayTask for InvalidInputDecodeTask { + fn decode(self: Box) -> Result<(ArrayRef, u64)> { + Err(Error::invalid_input_source("malformed sparse page".into())) + } + } + + #[test] + fn next_decode_task_preserves_invalid_input_errors() { + let err = NextDecodeTask { + task: Box::new(InvalidInputDecodeTask), + num_rows: 0, + } + .into_batch(Arc::new(Once::new())) + .unwrap_err(); + assert!(matches!(err, Error::InvalidInput { .. })); + } + #[test] fn test_read_zero_dimension_fsl_errors_instead_of_panicking() { // Simulates reading a column whose stored schema declares a @@ -3012,11 +3102,11 @@ mod tests { err ); - let mut legacy_columns = ColumnInfoIter::new(vec![], &[]); + let mut array_columns = ColumnInfoIter::new(vec![], &[]); let err = strategy - .create_legacy_field_scheduler( + .create_array_field_scheduler( &field, - &mut legacy_columns, + &mut array_columns, FileBuffers { positions_and_sizes: &[], }, @@ -3030,8 +3120,53 @@ mod tests { ); } + #[test] + fn test_list_page_with_non_list_encoding_returns_error() { + let item = Arc::new(ArrowField::new("item", DataType::Int32, true)); + let list = DataType::List(item); + let field = Field::try_from(&ArrowField::new("values", list, true)).unwrap(); + let values_encoding = pb::ColumnEncoding { + column_encoding: Some(pb::column_encoding::ColumnEncoding::Values(())), + }; + let offsets_column = Arc::new(ColumnInfo::new( + 0, + Arc::new([PageInfo { + num_rows: 1, + priority: 0, + encoding: PageEncoding::Legacy(pb::ArrayEncoding { + array_encoding: Some(pb::array_encoding::ArrayEncoding::Flat( + pb::Flat::default(), + )), + }), + buffer_offsets_and_sizes: Arc::new([]), + }]), + vec![], + values_encoding.clone(), + )); + let items_column = Arc::new(ColumnInfo::new(1, Arc::new([]), vec![], values_encoding)); + let column_indices = [0, 1]; + let mut columns = ColumnInfoIter::new(vec![offsets_column, items_column], &column_indices); + + let err = CoreFieldDecoderStrategy::default() + .create_array_field_scheduler( + &field, + &mut columns, + FileBuffers { + positions_and_sizes: &[], + }, + ) + .unwrap_err(); + + assert!(matches!(err, Error::InvalidInput { .. })); + assert!( + err.to_string() + .contains("expected list encoding for field 'values' in column 0, page 0 but got"), + "unexpected error: {err}" + ); + } + #[tokio::test] - async fn test_legacy_stream_stops_on_load_error() { + async fn test_array_stream_stops_on_load_error() { use arrow_schema::Field as ArrowField; let rows_per_batch = 1; @@ -3065,7 +3200,7 @@ mod tests { let err = batches .next() .await - .expect("stream should emit the legacy page-load error") + .expect("stream should emit the array page-load error") .unwrap_err(); assert!( err.to_string().contains(load_error_message), @@ -3074,7 +3209,7 @@ mod tests { ); assert!( batches.next().await.is_none(), - "stream should stop after the legacy page-load error" + "stream should stop after the array page-load error" ); } @@ -3170,15 +3305,14 @@ mod tests { batch_size: u32, batch_size_bytes: Option, ) -> Vec { - use crate::encoder::{EncodingOptions, default_encoding_strategy, encode_batch}; - use crate::version::LanceFileVersion; - - let version = LanceFileVersion::V2_1; - let options = EncodingOptions { - version, - ..Default::default() + use crate::{ + encoder::{EncodingOptions, encode_batch}, + testing::{TestEncoding, test_encoding_strategy}, }; - let strategy = default_encoding_strategy(version); + + let version = TestEncoding::StructuralU16; + let options = EncodingOptions::default(); + let strategy = test_encoding_strategy(version); let schema = Schema::try_from(batch.schema().as_ref()).unwrap(); let encoded = encode_batch(batch, Arc::new(schema.clone()), strategy.as_ref(), &options) .await @@ -3318,6 +3452,29 @@ mod tests { } } + #[tokio::test] + async fn test_byte_sized_batches_respect_row_limit() { + use arrow_array::Int32Array; + + let num_rows: i32 = 1000; + let schema = Arc::new(ArrowSchema::new(vec![ArrowField::new( + "x", + DataType::Int32, + false, + )])); + let input_batch = RecordBatch::try_new( + schema, + vec![Arc::new(Int32Array::from_iter_values(0..num_rows))], + ) + .unwrap(); + + // The byte limit can hold every row, so the 100-row limit must win. + let batches = + decode_batches_with_byte_limit(&input_batch, /*batch_size=*/ 100, Some(10_000)).await; + assert_eq!(batches.len(), 10); + assert!(batches.iter().all(|batch| batch.num_rows() == 100)); + } + #[tokio::test] async fn test_byte_sized_batches_feedback_convergence() { use arrow_array::StringArray; diff --git a/rust/lance-encoding/src/encoder.rs b/rust/lance-encoding/src/encoder.rs index cd0731fd06f..9d9f1194556 100644 --- a/rust/lance-encoding/src/encoder.rs +++ b/rust/lance-encoding/src/encoder.rs @@ -6,44 +6,72 @@ //! Lance files are encoded using a [`FieldEncodingStrategy`] which choose //! what encoder to use for each field. //! -//! The current strategy is the [`StructuralEncodingStrategy`] which uses "structural" -//! encoding. A tree of encoders is built up for each field. The struct & list encoders -//! simply pull off the validity and offsets and collect them. Then, in the primitive leaf -//! encoder the validity, offsets, and values are accumulated in an accumulation buffer. Once -//! enough data has been collected the primitive encoder will either use a miniblock encoding -//! or a full zip encoding to create a page of data from the accumulation buffer. +//! Structural strategies build a tree of encoders for each field from the +//! version-free builders in [`structural`]. Struct and list encoders collect +//! validity and offsets; primitive leaf encoders accumulate values and emit +//! miniblock or full-zip pages. use std::{collections::HashMap, sync::Arc}; use arrow_array::{Array, ArrayRef, RecordBatch}; -use arrow_schema::DataType; use bytes::{Bytes, BytesMut}; use futures::future::BoxFuture; use lance_core::datatypes::{Field, Schema}; -use lance_core::error::LanceOptionExt; use lance_core::utils::bit::{is_pwr_two, pad_bytes_to}; use lance_core::{Error, Result}; use crate::buffer::LanceBuffer; -use crate::compression::{CompressionStrategy, DefaultCompressionStrategy}; -use crate::compression_config::CompressionParams; +use crate::data::DataBlock; use crate::decoder::PageEncoding; -use crate::encodings::logical::blob::{BlobStructuralEncoder, BlobV2StructuralEncoder}; -use crate::encodings::logical::fixed_size_list::FixedSizeListStructuralEncoder; -use crate::encodings::logical::list::ListStructuralEncoder; -use crate::encodings::logical::map::MapStructuralEncoder; -use crate::encodings::logical::primitive::PrimitiveStructuralEncoder; -use crate::encodings::logical::r#struct::StructStructuralEncoder; use crate::repdef::RepDefBuilder; -use crate::version::LanceFileVersion; use crate::{ decoder::{ColumnInfo, PageInfo}, format::pb, }; +pub use crate::array_encoding::ArrayFieldEncodingStrategy; + +pub mod structural; + /// The minimum alignment for a page buffer. Writers must respect this. pub const MIN_PAGE_BUFFER_ALIGNMENT: u64 = 8; +/// An array encoded with the `pb::ArrayEncoding` grammar. +#[derive(Debug)] +pub struct EncodedArray { + pub data: DataBlock, + pub encoding: pb::ArrayEncoding, +} + +impl EncodedArray { + pub fn new(data: DataBlock, encoding: pb::ArrayEncoding) -> Self { + Self { data, encoding } + } + + pub fn into_buffers(self) -> (Vec, pb::ArrayEncoding) { + (self.data.into_buffers(), self.encoding) + } +} + +/// Encodes one data block and describes it with `pb::ArrayEncoding`. +pub trait ArrayEncoder: std::fmt::Debug + Send + Sync { + fn encode( + &self, + data: DataBlock, + data_type: &arrow_schema::DataType, + buffer_index: &mut u32, + ) -> Result; +} + +/// Selects an `ArrayEncoder` for one page. +pub trait ArrayEncodingStrategy: Send + Sync + std::fmt::Debug { + fn create_array_encoder( + &self, + arrays: &[ArrayRef], + field: &Field, + ) -> Result>; +} + /// An encoded page of data /// /// Maps to a top-level array @@ -235,9 +263,6 @@ pub struct EncodingOptions { /// The encoder needs to know this so it figures the position of out-of-line /// buffers correctly pub buffer_alignment: u64, - - /// The Lance file version being written - pub version: LanceFileVersion, } impl Default for EncodingOptions { @@ -247,20 +272,10 @@ impl Default for EncodingOptions { max_page_bytes: 32 * 1024 * 1024, keep_original_array: true, buffer_alignment: 64, - version: LanceFileVersion::default(), } } } -impl EncodingOptions { - /// If true (for Lance file version 2.2+), miniblock chunk sizes are u32, - /// to allow storing larger chunks and their sizes for better compression. - /// For Lance file version 2.1, miniblock chunk sizes are u16. - pub fn support_large_chunk(&self) -> bool { - self.version >= LanceFileVersion::V2_2 - } -} - /// A trait to pick which kind of field encoding to use for a field /// /// Unlike the ArrayEncodingStrategy, the field encoding strategy is @@ -273,324 +288,23 @@ pub trait FieldEncodingStrategy: Send + Sync + std::fmt::Debug { /// The field encoder can be chosen on the data type as well as /// any metadata that is attached to the field. /// - /// The `encoding_strategy_root` is the encoder that should be - /// used to encode any inner data in struct / list / etc. fields. - /// - /// Initially it is the same as `self` and generally should be - /// forwarded to any inner encoding strategy. fn create_field_encoder( &self, - encoding_strategy_root: &dyn FieldEncodingStrategy, field: &Field, column_index: &mut ColumnIndexSequence, - options: &EncodingOptions, + context: &FieldEncodingContext<'_>, ) -> Result>; } -pub fn default_encoding_strategy(version: LanceFileVersion) -> Box { - match version.resolve() { - LanceFileVersion::Legacy => panic!(), - LanceFileVersion::V2_0 => Box::new( - crate::previous::encoder::CoreFieldEncodingStrategy::new(version), - ), - _ => Box::new(StructuralEncodingStrategy::with_version(version)), - } -} - -/// Create an encoding strategy with user-configured compression parameters -pub fn default_encoding_strategy_with_params( - version: LanceFileVersion, - params: CompressionParams, -) -> Result> { - match version.resolve() { - LanceFileVersion::Legacy | LanceFileVersion::V2_0 => Err(Error::invalid_input( - "Compression parameters are only supported in Lance file version 2.1 and later", - )), - _ => { - let compression_strategy = - Arc::new(DefaultCompressionStrategy::with_params(params).with_version(version)); - Ok(Box::new(StructuralEncodingStrategy { - compression_strategy, - version, - })) - } - } -} - -/// An encoding strategy used for 2.1+ files -#[derive(Debug)] -pub struct StructuralEncodingStrategy { - pub compression_strategy: Arc, - pub version: LanceFileVersion, -} - -// For some reason, clippy thinks we can add Default to the above derive but -// rustc doesn't agree (no default for Arc) -#[allow(clippy::derivable_impls)] -impl Default for StructuralEncodingStrategy { - fn default() -> Self { - Self { - compression_strategy: Arc::new(DefaultCompressionStrategy::new()), - version: LanceFileVersion::default(), - } - } -} - -impl StructuralEncodingStrategy { - pub fn with_version(version: LanceFileVersion) -> Self { - Self { - compression_strategy: Arc::new(DefaultCompressionStrategy::new().with_version(version)), - version, - } - } - - fn is_primitive_type(data_type: &DataType) -> bool { - match data_type { - DataType::FixedSizeList(inner, _) => Self::is_primitive_type(inner.data_type()), - _ => matches!( - data_type, - DataType::Boolean - | DataType::Date32 - | DataType::Date64 - | DataType::Decimal128(_, _) - | DataType::Decimal256(_, _) - | DataType::Duration(_) - | DataType::Float16 - | DataType::Float32 - | DataType::Float64 - | DataType::Int16 - | DataType::Int32 - | DataType::Int64 - | DataType::Int8 - | DataType::Interval(_) - | DataType::Null - | DataType::Time32(_) - | DataType::Time64(_) - | DataType::Timestamp(_, _) - | DataType::UInt16 - | DataType::UInt32 - | DataType::UInt64 - | DataType::UInt8 - | DataType::FixedSizeBinary(_) - | DataType::Binary - | DataType::LargeBinary - | DataType::Utf8 - | DataType::LargeUtf8, - ), - } - } - - fn do_create_field_encoder( - &self, - _encoding_strategy_root: &dyn FieldEncodingStrategy, - field: &Field, - column_index: &mut ColumnIndexSequence, - options: &EncodingOptions, - root_field_metadata: &HashMap, - ) -> Result> { - let data_type = field.data_type(); - - // Check if field is marked as blob - if field.is_blob() { - match data_type { - DataType::Binary | DataType::LargeBinary => { - return Ok(Box::new(BlobStructuralEncoder::new( - field, - column_index.next_column_index(field.id as u32), - options, - self.compression_strategy.clone(), - )?)); - } - DataType::Struct(_) if self.version >= LanceFileVersion::V2_2 => { - return Ok(Box::new(BlobV2StructuralEncoder::new( - field, - column_index.next_column_index(field.id as u32), - options, - self.compression_strategy.clone(), - )?)); - } - DataType::Struct(_) => { - return Err(Error::invalid_input_source( - "Blob v2 struct input requires file version >= 2.2".into(), - )); - } - _ => { - return Err(Error::invalid_input_source( - format!( - "Blob encoding only supports Binary/LargeBinary or v2 Struct, got {}", - data_type - ) - .into(), - )); - } - } - } - - if Self::is_primitive_type(&data_type) { - Ok(Box::new(PrimitiveStructuralEncoder::try_new( - options, - self.compression_strategy.clone(), - column_index.next_column_index(field.id as u32), - field.clone(), - Arc::new(root_field_metadata.clone()), - )?)) - } else { - match data_type { - DataType::List(_) | DataType::LargeList(_) => { - let child = field.children.first().expect_ok()?; - let child_encoder = self.do_create_field_encoder( - _encoding_strategy_root, - child, - column_index, - options, - root_field_metadata, - )?; - Ok(Box::new(ListStructuralEncoder::new( - options.keep_original_array, - child_encoder, - ))) - } - DataType::FixedSizeList(inner, _) - if matches!(inner.data_type(), DataType::Struct(_)) => - { - if self.version < LanceFileVersion::V2_2 { - return Err(Error::not_supported_source(format!( - "FixedSizeList is only supported in Lance file format 2.2+, current version: {}", - self.version - ) - .into())); - } - // Complex FixedSizeList needs structural encoding - let child = field.children.first().expect_ok()?; - let child_encoder = self.do_create_field_encoder( - _encoding_strategy_root, - child, - column_index, - options, - root_field_metadata, - )?; - Ok(Box::new(FixedSizeListStructuralEncoder::new( - options.keep_original_array, - child_encoder, - ))) - } - DataType::Map(_, keys_sorted) => { - // TODO: We only support keys_sorted=false for now, - // because converting a rust arrow map field to the python arrow field will - // lose the keys_sorted property. - if keys_sorted { - return Err(Error::not_supported_source(format!("Map data type is not supported with keys_sorted=true now, current value is {}", keys_sorted).into())); - } - if self.version < LanceFileVersion::V2_2 { - return Err(Error::not_supported_source(format!( - "Map data type is only supported in Lance file format 2.2+, current version: {}", - self.version - ) - .into())); - } - let entries_child = field.children.first().ok_or_else(|| { - Error::schema("Map should have an entries child".to_string()) - })?; - let DataType::Struct(struct_fields) = entries_child.data_type() else { - return Err(Error::schema( - "Map entries field must be a Struct".to_string(), - )); - }; - if struct_fields.len() < 2 { - return Err(Error::schema( - "Map entries struct must contain both key and value fields".to_string(), - )); - } - let key_field = &struct_fields[0]; - if key_field.is_nullable() { - return Err(Error::schema(format!( - "Map key field '{}' must be non-nullable according to Arrow Map specification", - key_field.name() - ))); - } - let child_encoder = self.do_create_field_encoder( - _encoding_strategy_root, - entries_child, - column_index, - options, - root_field_metadata, - )?; - Ok(Box::new(MapStructuralEncoder::new( - options.keep_original_array, - child_encoder, - ))) - } - DataType::Struct(fields) => { - if field.is_packed_struct() || fields.is_empty() { - // Both packed structs and empty structs are encoded as primitive - Ok(Box::new(PrimitiveStructuralEncoder::try_new( - options, - self.compression_strategy.clone(), - column_index.next_column_index(field.id as u32), - field.clone(), - Arc::new(root_field_metadata.clone()), - )?)) - } else { - let children_encoders = field - .children - .iter() - .map(|field| { - self.do_create_field_encoder( - _encoding_strategy_root, - field, - column_index, - options, - root_field_metadata, - ) - }) - .collect::>>()?; - Ok(Box::new(StructStructuralEncoder::new( - options.keep_original_array, - children_encoders, - ))) - } - } - DataType::Dictionary(_, value_type) => { - // A dictionary of primitive is, itself, primitive - if Self::is_primitive_type(&value_type) { - Ok(Box::new(PrimitiveStructuralEncoder::try_new( - options, - self.compression_strategy.clone(), - column_index.next_column_index(field.id as u32), - field.clone(), - Arc::new(root_field_metadata.clone()), - )?)) - } else { - // A dictionary of logical is, itself, logical and we don't support that today - // It could be possible (e.g. store indices in one column and values in remaining columns) - // but would be a significant amount of work - // - // An easier fallback implementation would be to decode-on-write and encode-on-read - Err(Error::not_supported_source(format!("cannot encode a dictionary column whose value type is a logical type ({})", value_type).into())) - } - } - _ => todo!("Implement encoding for field {}", field), - } - } - } -} - -impl FieldEncodingStrategy for StructuralEncodingStrategy { - fn create_field_encoder( - &self, - encoding_strategy_root: &dyn FieldEncodingStrategy, - field: &Field, - column_index: &mut ColumnIndexSequence, - options: &EncodingOptions, - ) -> Result> { - self.do_create_field_encoder( - encoding_strategy_root, - field, - column_index, - options, - &field.metadata, - ) - } +/// Context shared while one top-level field and all of its children are mapped +/// to concrete field encoders. +pub struct FieldEncodingContext<'a> { + /// The complete strategy composition used for recursive child fields. + pub strategy: &'a dyn FieldEncodingStrategy, + /// Runtime-only writer options. + pub options: &'a EncodingOptions, + /// Metadata inherited from the top-level field. + pub root_field_metadata: &'a HashMap, } /// A batch encoder that encodes RecordBatch objects by delegating @@ -612,12 +326,13 @@ impl BatchEncoder { .fields .iter() .map(|field| { - let encoder = strategy.create_field_encoder( + let context = FieldEncodingContext { strategy, - field, - &mut col_idx_sequence, options, - )?; + root_field_metadata: &field.metadata, + }; + let encoder = + strategy.create_field_encoder(field, &mut col_idx_sequence, &context)?; col_idx += encoder.as_ref().num_columns(); Ok(encoder) }) @@ -768,50 +483,9 @@ pub async fn encode_batch( #[cfg(test)] mod tests { use super::*; - use crate::compression_config::{CompressionFieldParams, CompressionParams}; + use crate::testing::{TestEncoding, create_test_field_encoder, test_encoding_strategy}; use arrow_schema::{DataType as ArrowDataType, Field as ArrowField, Fields as ArrowFields}; - #[test] - fn test_configured_encoding_strategy() { - // Create test parameters - let mut params = CompressionParams::new(); - params.columns.insert( - "*_id".to_string(), - CompressionFieldParams { - rle_threshold: Some(0.5), - compression: Some("lz4".to_string()), - compression_level: None, - bss: None, - minichunk_size: None, - }, - ); - - // Test with V2.1 - should succeed - let strategy = - default_encoding_strategy_with_params(LanceFileVersion::V2_1, params.clone()) - .expect("Should succeed for V2.1"); - - // Verify it's a StructuralEncodingStrategy - assert!(format!("{:?}", strategy).contains("StructuralEncodingStrategy")); - assert!(format!("{:?}", strategy).contains("DefaultCompressionStrategy")); - - // Test with V2.0 - should fail - let err = default_encoding_strategy_with_params(LanceFileVersion::V2_0, params.clone()) - .expect_err("Should fail for V2.0"); - assert!( - err.to_string() - .contains("only supported in Lance file version 2.1") - ); - - // Test with Legacy - should fail - let err = default_encoding_strategy_with_params(LanceFileVersion::Legacy, params) - .expect_err("Should fail for Legacy"); - assert!( - err.to_string() - .contains("only supported in Lance file version 2.1") - ); - } - #[test] fn test_fixed_size_list_struct_requires_v2_2() { let list_item = ArrowField::new( @@ -830,11 +504,12 @@ mod tests { ); let field = Field::try_from(&arrow_field).unwrap(); - let strategy = StructuralEncodingStrategy::with_version(LanceFileVersion::V2_1); + let strategy = test_encoding_strategy(TestEncoding::StructuralU16); let mut column_index = ColumnIndexSequence::default(); let options = EncodingOptions::default(); - let result = strategy.create_field_encoder(&strategy, &field, &mut column_index, &options); + let result = + create_test_field_encoder(strategy.as_ref(), &field, &mut column_index, &options); assert!( result.is_err(), "FixedSizeList should be rejected for file version 2.1" @@ -843,7 +518,7 @@ mod tests { assert!( err.to_string() - .contains("FixedSizeList is only supported in Lance file format 2.2+") + .contains("FixedSizeList is not enabled by the selected file format") ); } } diff --git a/rust/lance-encoding/src/encoder/structural.rs b/rust/lance-encoding/src/encoder/structural.rs new file mode 100644 index 00000000000..3d4e49f42c8 --- /dev/null +++ b/rust/lance-encoding/src/encoder/structural.rs @@ -0,0 +1,285 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright The Lance Authors + +//! Version-free structural field encoder builders. + +use std::sync::Arc; + +use arrow_schema::DataType; +use lance_core::{Error, Result, datatypes::Field, error::LanceOptionExt}; + +pub use crate::encodings::logical::primitive::PrimitivePageEncoding; + +use crate::encodings::logical::{ + blob::{BlobStructuralEncoder, BlobV2StructuralEncoder}, + fixed_size_list::FixedSizeListStructuralEncoder, + list::ListStructuralEncoder, + map::MapStructuralEncoder, + primitive::PrimitiveStructuralEncoder, + r#struct::StructStructuralEncoder, +}; + +use super::{ColumnIndexSequence, FieldEncoder, FieldEncodingContext}; + +/// Encode primitive leaves, primitive fixed-size lists, dictionaries, and +/// packed or empty structs using one concrete primitive page grammar. +#[derive(Debug, Clone)] +pub struct PrimitiveFieldEncoding { + page_encodings: Arc<[PrimitivePageEncoding]>, +} + +impl PrimitiveFieldEncoding { + /// Create a primitive field mechanism from ordered executable page behaviors. + pub fn new(page_encodings: impl IntoIterator) -> Self { + Self { + page_encodings: page_encodings.into_iter().collect(), + } + } + + fn is_primitive_type(data_type: &DataType) -> bool { + match data_type { + DataType::FixedSizeList(inner, _) => Self::is_primitive_type(inner.data_type()), + _ => matches!( + data_type, + DataType::Boolean + | DataType::Date32 + | DataType::Date64 + | DataType::Decimal128(_, _) + | DataType::Decimal256(_, _) + | DataType::Duration(_) + | DataType::Float16 + | DataType::Float32 + | DataType::Float64 + | DataType::Int16 + | DataType::Int32 + | DataType::Int64 + | DataType::Int8 + | DataType::Interval(_) + | DataType::Null + | DataType::Time32(_) + | DataType::Time64(_) + | DataType::Timestamp(_, _) + | DataType::UInt16 + | DataType::UInt32 + | DataType::UInt64 + | DataType::UInt8 + | DataType::FixedSizeBinary(_) + | DataType::Binary + | DataType::LargeBinary + | DataType::Utf8 + | DataType::LargeUtf8, + ), + } + } + + fn create_at( + &self, + field: Field, + column_index: u32, + context: &FieldEncodingContext<'_>, + ) -> Result> { + Ok(Box::new(PrimitiveStructuralEncoder::try_new( + context.options, + self.page_encodings.clone(), + column_index, + field, + Arc::new(context.root_field_metadata.clone()), + )?)) + } + + /// Create a primitive field encoder when this mechanism recognizes `field`. + pub fn try_create( + &self, + field: &Field, + column_index: &mut ColumnIndexSequence, + context: &FieldEncodingContext<'_>, + ) -> Result>> { + if field.is_blob() { + return Ok(None); + } + + let data_type = field.data_type(); + let is_primitive = Self::is_primitive_type(&data_type); + let is_packed_or_empty_struct = matches!( + &data_type, + DataType::Struct(fields) if field.is_packed_struct() || fields.is_empty() + ); + let is_primitive_dictionary = matches!( + &data_type, + DataType::Dictionary(_, value_type) if Self::is_primitive_type(value_type) + ); + + if !is_primitive && !is_packed_or_empty_struct && !is_primitive_dictionary { + if let DataType::Dictionary(_, value_type) = data_type { + return Err(Error::not_supported_source( + format!( + "cannot encode a dictionary column whose value type is a logical type ({})", + value_type + ) + .into(), + )); + } + return Ok(None); + } + + Ok(Some(self.create_at( + field.clone(), + column_index.next_column_index(field.id as u32), + context, + )?)) + } +} + +/// Create the original binary blob descriptor when `field` matches. +pub fn try_create_binary_blob( + primitive: &PrimitiveFieldEncoding, + field: &Field, + column_index: &mut ColumnIndexSequence, + context: &FieldEncodingContext<'_>, +) -> Result>> { + if !field.is_blob() || !matches!(field.data_type(), DataType::Binary | DataType::LargeBinary) { + return Ok(None); + } + let descriptor_column_index = column_index.next_column_index(field.id as u32); + Ok(Some(Box::new(BlobStructuralEncoder::new( + field, + |descriptor_field| primitive.create_at(descriptor_field, descriptor_column_index, context), + )?))) +} + +/// Create the structural blob descriptor when `field` matches. +pub fn try_create_structural_blob( + primitive: &PrimitiveFieldEncoding, + field: &Field, + column_index: &mut ColumnIndexSequence, + context: &FieldEncodingContext<'_>, +) -> Result>> { + if !field.is_blob() || !matches!(field.data_type(), DataType::Struct(_)) { + return Ok(None); + } + let descriptor_column_index = column_index.next_column_index(field.id as u32); + Ok(Some(Box::new(BlobV2StructuralEncoder::new( + field, + |descriptor_field| primitive.create_at(descriptor_field, descriptor_column_index, context), + )?))) +} + +/// Create a variable-size list encoder when `field` matches. +pub fn try_create_list( + field: &Field, + column_index: &mut ColumnIndexSequence, + context: &FieldEncodingContext<'_>, +) -> Result>> { + if !matches!( + field.data_type(), + DataType::List(_) | DataType::LargeList(_) + ) { + return Ok(None); + } + let child = field.children.first().expect_ok()?; + let child_encoder = context + .strategy + .create_field_encoder(child, column_index, context)?; + Ok(Some(Box::new(ListStructuralEncoder::new( + context.options.keep_original_array, + child_encoder, + )))) +} + +/// Create a fixed-size-list encoder whose child is a struct when applicable. +pub fn try_create_structural_fixed_size_list( + field: &Field, + column_index: &mut ColumnIndexSequence, + context: &FieldEncodingContext<'_>, +) -> Result>> { + if !matches!( + field.data_type(), + DataType::FixedSizeList(inner, _) if matches!(inner.data_type(), DataType::Struct(_)) + ) { + return Ok(None); + } + let child = field.children.first().expect_ok()?; + let child_encoder = context + .strategy + .create_field_encoder(child, column_index, context)?; + Ok(Some(Box::new(FixedSizeListStructuralEncoder::new( + context.options.keep_original_array, + child_encoder, + )))) +} + +/// Create an Arrow map encoder when `field` matches. +pub fn try_create_map( + field: &Field, + column_index: &mut ColumnIndexSequence, + context: &FieldEncodingContext<'_>, +) -> Result>> { + let DataType::Map(_, keys_sorted) = field.data_type() else { + return Ok(None); + }; + if keys_sorted { + return Err(Error::not_supported_source( + format!( + "Map data type is not supported with keys_sorted=true now, current value is {}", + keys_sorted + ) + .into(), + )); + } + let entries_child = field + .children + .first() + .ok_or_else(|| Error::schema("Map should have an entries child".to_string()))?; + let DataType::Struct(struct_fields) = entries_child.data_type() else { + return Err(Error::schema( + "Map entries field must be a Struct".to_string(), + )); + }; + if struct_fields.len() < 2 { + return Err(Error::schema( + "Map entries struct must contain both key and value fields".to_string(), + )); + } + let key_field = &struct_fields[0]; + if key_field.is_nullable() { + return Err(Error::schema(format!( + "Map key field '{}' must be non-nullable according to Arrow Map specification", + key_field.name() + ))); + } + let child_encoder = + context + .strategy + .create_field_encoder(entries_child, column_index, context)?; + Ok(Some(Box::new(MapStructuralEncoder::new( + context.options.keep_original_array, + child_encoder, + )))) +} + +/// Create a non-packed, non-empty struct encoder when `field` matches. +pub fn try_create_struct( + field: &Field, + column_index: &mut ColumnIndexSequence, + context: &FieldEncodingContext<'_>, +) -> Result>> { + let DataType::Struct(fields) = field.data_type() else { + return Ok(None); + }; + if field.is_blob() || field.is_packed_struct() || fields.is_empty() { + return Ok(None); + } + let children_encoders = field + .children + .iter() + .map(|child| { + context + .strategy + .create_field_encoder(child, column_index, context) + }) + .collect::>>()?; + Ok(Some(Box::new(StructStructuralEncoder::new( + context.options.keep_original_array, + children_encoders, + )))) +} diff --git a/rust/lance-encoding/src/encodings/fuzz_tests.rs b/rust/lance-encoding/src/encodings/fuzz_tests.rs index b92bac09cca..3521d633ac3 100644 --- a/rust/lance-encoding/src/encodings/fuzz_tests.rs +++ b/rust/lance-encoding/src/encodings/fuzz_tests.rs @@ -16,7 +16,6 @@ use arrow_schema::{DataType, Field}; use proptest::prelude::*; use crate::testing::{TestCases, check_round_trip_encoding_of_data}; -use crate::version::LanceFileVersion; use lance_core::Result; use lance_datagen::{ArrayGenerator, ByteCount, Dimension, RowCount, Seed, array, gen_batch}; @@ -280,7 +279,7 @@ proptest! { } let test_cases = TestCases::default() - .with_min_file_version(LanceFileVersion::V2_1) + .with_structural_encodings() .with_batch_size(100) .with_range(0..num_rows.min(500) as u64) .with_indices(vec![0, num_rows as u64 / 2, (num_rows - 1) as u64]); @@ -301,7 +300,7 @@ async fn test_edge_cases_single_value() { let single_int32 = Arc::new(Int32Array::from(vec![42])) as Arc; let single_string = Arc::new(StringArray::from(vec!["test"])) as Arc; - let test_cases = TestCases::default().with_min_file_version(LanceFileVersion::V2_1); + let test_cases = TestCases::default().with_structural_encodings(); check_round_trip_encoding_of_data(vec![single_int32], &test_cases, HashMap::new()).await; @@ -317,7 +316,7 @@ async fn test_edge_cases_all_nulls() { vec![None, None, None] as Vec> )) as Arc; - let test_cases = TestCases::default().with_min_file_version(LanceFileVersion::V2_1); + let test_cases = TestCases::default().with_structural_encodings(); check_round_trip_encoding_of_data(vec![all_nulls_int32], &test_cases, HashMap::new()).await; @@ -347,7 +346,7 @@ proptest! { let list_array = Arc::new(list_builder.finish()) as Arc; let test_cases = TestCases::default() - .with_min_file_version(LanceFileVersion::V2_1) + .with_structural_encodings() .with_range(0..list_sizes.len().min(50) as u64); check_round_trip_encoding_of_data( @@ -381,7 +380,7 @@ proptest! { .expect("Failed to generate test data"); let test_cases = TestCases::default() - .with_min_file_version(LanceFileVersion::V2_1); + .with_structural_encodings(); check_round_trip_encoding_of_data( vec![test_data], @@ -424,7 +423,7 @@ async fn test_list_dict_empty_batch() { let list_array = Arc::new(list_builder.finish()); let test_cases = TestCases::default() - .with_min_file_version(LanceFileVersion::V2_1) + .with_structural_encodings() // Read only the empty/null lists (rows 50-99) // This batch will have 0 underlying values .with_range(50..100); @@ -539,7 +538,7 @@ async fn test_all_valid_combinations() { let test_data = generate_test_data_for_config(&config, 100, 42).expect("Failed to generate test data"); - let test_cases = TestCases::default().with_min_file_version(LanceFileVersion::V2_1); + let test_cases = TestCases::default().with_structural_encodings(); check_round_trip_encoding_of_data(vec![test_data], &test_cases, HashMap::new()).await; } diff --git a/rust/lance-encoding/src/encodings/logical/blob.rs b/rust/lance-encoding/src/encodings/logical/blob.rs index cad2112bafe..a5432fdc54c 100644 --- a/rust/lance-encoding/src/encodings/logical/blob.rs +++ b/rust/lance-encoding/src/encodings/logical/blob.rs @@ -13,7 +13,9 @@ use arrow_buffer::Buffer; use arrow_schema::{DataType, Field as ArrowField, Fields}; use futures::{FutureExt, future::BoxFuture}; use lance_core::{ - Error, Result, datatypes::BLOB_V2_DESC_FIELDS, datatypes::Field, error::LanceOptionExt, + Error, Result, + datatypes::{BLOB_V2_DESC_FIELDS, BlobV2Layout, Field}, + error::LanceOptionExt, }; use crate::{ @@ -21,7 +23,6 @@ use crate::{ constants::PACKED_STRUCT_META_KEY, decoder::PageEncoding, encoder::{EncodeTask, EncodedColumn, EncodedPage, FieldEncoder, OutOfLineBuffers}, - encodings::logical::primitive::PrimitiveStructuralEncoder, format::ProtobufUtils21, repdef::{DefinitionInterpretation, RepDefBuilder}, }; @@ -42,9 +43,7 @@ pub struct BlobStructuralEncoder { impl BlobStructuralEncoder { pub fn new( field: &Field, - column_index: u32, - options: &crate::encoder::EncodingOptions, - compression_strategy: Arc, + make_descriptor_encoder: impl FnOnce(Field) -> Result>, ) -> Result { // Create descriptor field: struct // Preserve the original field's metadata for packed struct @@ -63,13 +62,7 @@ impl BlobStructuralEncoder { )?; // Use PrimitiveStructuralEncoder to handle the descriptor - let descriptor_encoder = Box::new(PrimitiveStructuralEncoder::try_new( - options, - compression_strategy, - column_index, - descriptor_field, - Arc::new(HashMap::new()), - )?); + let descriptor_encoder = make_descriptor_encoder(descriptor_field)?; Ok(Self { descriptor_encoder, @@ -137,14 +130,18 @@ impl FieldEncoder for BlobStructuralEncoder { let def = repdef.definition_levels.as_ref(); let def_meaning: Arc<[DefinitionInterpretation]> = repdef.def_meaning.into(); - match self.def_meaning.as_ref() { - None => { - self.def_meaning = Some(def_meaning.clone()); + // A blob page stores one definition interpretation for all of its rows. + // The descriptor encoder can buffer multiple input arrays, so finish the + // pending page before a later array changes from all-valid to nullable (or + // vice versa). + let mut encode_tasks = match self.def_meaning.as_ref() { + Some(existing) if existing != &def_meaning => { + let existing = existing.clone(); + Self::wrap_tasks(self.descriptor_encoder.flush(external_buffers)?, existing) } - Some(existing) => { - debug_assert_eq!(existing, &def_meaning); - } - } + _ => Vec::new(), + }; + self.def_meaning = Some(def_meaning.clone()); // Collect positions and sizes let mut positions = Vec::with_capacity(binary_array.len()); @@ -192,15 +189,16 @@ impl FieldEncoder for BlobStructuralEncoder { )); // Delegate to descriptor encoder - let encode_tasks = self.descriptor_encoder.maybe_encode( + let descriptor_tasks = self.descriptor_encoder.maybe_encode( descriptor_array, external_buffers, RepDefBuilder::default(), row_number, num_rows, )?; + encode_tasks.extend(Self::wrap_tasks(descriptor_tasks, def_meaning)); - Ok(Self::wrap_tasks(encode_tasks, def_meaning)) + Ok(encode_tasks) } fn flush(&mut self, external_buffers: &mut OutOfLineBuffers) -> Result> { @@ -236,9 +234,7 @@ pub struct BlobV2StructuralEncoder { impl BlobV2StructuralEncoder { pub fn new( field: &Field, - column_index: u32, - options: &crate::encoder::EncodingOptions, - compression_strategy: Arc, + make_descriptor_encoder: impl FnOnce(Field) -> Result>, ) -> Result { let mut descriptor_metadata = HashMap::with_capacity(1); descriptor_metadata.insert(PACKED_STRUCT_META_KEY.to_string(), "true".to_string()); @@ -250,13 +246,7 @@ impl BlobV2StructuralEncoder { .with_metadata(descriptor_metadata), )?; - let descriptor_encoder = Box::new(PrimitiveStructuralEncoder::try_new( - options, - compression_strategy, - column_index, - descriptor_field, - Arc::new(HashMap::new()), - )?); + let descriptor_encoder = make_descriptor_encoder(descriptor_field)?; Ok(Self { descriptor_encoder }) } @@ -267,15 +257,30 @@ impl FieldEncoder for BlobV2StructuralEncoder { &mut self, array: ArrayRef, external_buffers: &mut OutOfLineBuffers, - mut repdef: RepDefBuilder, + repdef: RepDefBuilder, row_number: u64, num_rows: u64, ) -> Result> { - let struct_arr = array.as_struct(); - if let Some(validity) = struct_arr.nulls() { - repdef.add_validity_bitmap(validity.clone()); - } else { - repdef.add_no_null(struct_arr.len()); + let struct_arr = array + .as_any() + .downcast_ref::() + .ok_or_else(|| { + Error::invalid_input_source( + format!( + "Blob v2 encoder expected StructArray, got {}", + array.data_type() + ) + .into(), + ) + })?; + if BlobV2Layout::classify(struct_arr.fields()) != Some(BlobV2Layout::Prepared) { + let actual = BlobV2Layout::classify(struct_arr.fields()) + .map(|layout| layout.to_string()) + .unwrap_or_else(|| format!("unrecognized ({:?})", struct_arr.fields())); + return Err(Error::invalid_input_source( + format!("Blob v2 encoder expected prepared array layout, got {actual} layout") + .into(), + )); } let kind_col = struct_arr @@ -403,7 +408,7 @@ impl FieldEncoder for BlobV2StructuralEncoder { let descriptor_array = Arc::new(StructArray::try_new( BLOB_V2_DESC_FIELDS.clone(), children, - None, + struct_arr.nulls().cloned(), )?) as ArrayRef; self.descriptor_encoder.maybe_encode( @@ -435,33 +440,82 @@ impl FieldEncoder for BlobV2StructuralEncoder { mod tests { use super::*; use crate::{ - compression::DefaultCompressionStrategy, encoder::{ColumnIndexSequence, EncodingOptions}, testing::{ - TestCases, check_round_trip_encoding_of_data, - check_round_trip_encoding_of_data_with_expected, + TestCases, TestEncoding, check_round_trip_encoding_of_data, + check_round_trip_encoding_of_data_with_expected, create_test_field_encoder, + test_encoding_strategy, }, - version::LanceFileVersion, }; use arrow_array::{ ArrayRef, LargeBinaryArray, StringArray, StructArray, UInt8Array, UInt32Array, UInt64Array, }; use arrow_schema::{DataType, Field as ArrowField}; + use lance_core::datatypes::BLOB_V2_LOGICAL_MINIMAL_FIELDS; #[test] fn test_blob_encoder_creation() { - let field = - Field::try_from(ArrowField::new("blob_field", DataType::LargeBinary, true)).unwrap(); + let field = Field::try_from( + ArrowField::new("blob_field", DataType::LargeBinary, true).with_metadata( + HashMap::from([(lance_arrow::BLOB_META_KEY.to_string(), "true".to_string())]), + ), + ) + .unwrap(); let mut column_index = ColumnIndexSequence::default(); - let column_idx = column_index.next_column_index(0); let options = EncodingOptions::default(); - let compression = Arc::new(DefaultCompressionStrategy::new()); + let strategy = test_encoding_strategy(TestEncoding::StructuralU16); - let encoder = BlobStructuralEncoder::new(&field, column_idx, &options, compression); + let encoder = + create_test_field_encoder(strategy.as_ref(), &field, &mut column_index, &options); assert!(encoder.is_ok()); } + #[test] + fn test_blob_v2_encoder_rejects_logical_array_layout() { + let field = Field::try_from( + ArrowField::new( + "blob_field", + DataType::Struct(BLOB_V2_LOGICAL_MINIMAL_FIELDS.clone()), + true, + ) + .with_metadata(HashMap::from([( + lance_arrow::ARROW_EXT_NAME_KEY.to_string(), + lance_arrow::BLOB_V2_EXT_NAME.to_string(), + )])), + ) + .unwrap(); + let mut column_index = ColumnIndexSequence::default(); + let options = EncodingOptions::default(); + let strategy = test_encoding_strategy(TestEncoding::StructuralU32); + let mut encoder = + create_test_field_encoder(strategy.as_ref(), &field, &mut column_index, &options) + .unwrap(); + let array = Arc::new( + StructArray::try_new( + BLOB_V2_LOGICAL_MINIMAL_FIELDS.clone(), + vec![ + Arc::new(LargeBinaryArray::from(vec![Some(b"payload".as_ref())])) as ArrayRef, + Arc::new(StringArray::from(vec![None::<&str>])) as ArrayRef, + ], + None, + ) + .unwrap(), + ) as ArrayRef; + let mut external_buffers = OutOfLineBuffers::new(0, 8); + let Err(error) = + encoder.maybe_encode(array, &mut external_buffers, RepDefBuilder::default(), 0, 1) + else { + panic!("logical array layout unexpectedly reached the descriptor encoder"); + }; + assert!(matches!(error, Error::InvalidInput { .. })); + assert!( + error + .to_string() + .contains("expected prepared array layout, got logical layout") + ); + } + #[tokio::test] async fn test_blob_encoding_simple() { let field = Field::try_from( @@ -471,12 +525,12 @@ mod tests { ) .unwrap(); let mut column_index = ColumnIndexSequence::default(); - let column_idx = column_index.next_column_index(0); let options = EncodingOptions::default(); - let compression = Arc::new(DefaultCompressionStrategy::new()); + let strategy = test_encoding_strategy(TestEncoding::StructuralU16); let mut encoder = - BlobStructuralEncoder::new(&field, column_idx, &options, compression).unwrap(); + create_test_field_encoder(strategy.as_ref(), &field, &mut column_index, &options) + .unwrap(); // Create test data with larger blobs let large_data = vec![0u8; 1024 * 100]; // 100KB blob @@ -529,7 +583,53 @@ mod tests { // Use the standard test harness check_round_trip_encoding_of_data( vec![array], - &TestCases::default().with_max_file_version(LanceFileVersion::V2_1), + &TestCases::default().with_array_and_u16_encodings(), + blob_metadata, + ) + .await; + } + + #[tokio::test] + async fn test_blob_round_trip_empty_values() { + // Empty values share size == 0 with nulls in the descriptor layout + // and schedule no read; each must decode to zero-length bytes without + // consuming the read result of a following non-empty blob. Empties + // are placed before payloads so a misassignment corrupts the output + // instead of only exhausting the read iterator. + let blob_metadata = + HashMap::from([(lance_arrow::BLOB_META_KEY.to_string(), "true".to_string())]); + + let val1: &[u8] = &vec![1u8; 1024]; + let val2: &[u8] = &vec![2u8; 10240]; + let empty: &[u8] = &[]; + let array = Arc::new(LargeBinaryArray::from(vec![ + Some(empty), + Some(val1), + None, + Some(empty), + Some(val2), + None, + Some(empty), + ])); + + check_round_trip_encoding_of_data(vec![array], &TestCases::default(), blob_metadata).await; + } + + #[tokio::test] + async fn test_blob_round_trip_varying_chunk_nullability() { + let blob_metadata = + HashMap::from([(lance_arrow::BLOB_META_KEY.to_string(), "true".to_string())]); + let all_valid = Arc::new(LargeBinaryArray::from(vec![Some(b"first".as_ref())])); + let with_null = Arc::new(LargeBinaryArray::from(vec![ + Some(b"second".as_ref()), + None, + Some(b"".as_ref()), + ])); + let all_valid_again = Arc::new(LargeBinaryArray::from(vec![Some(b"last".as_ref())])); + + check_round_trip_encoding_of_data( + vec![all_valid, with_null, all_valid_again], + &TestCases::default().with_encoding(TestEncoding::StructuralU16), blob_metadata, ) .await; @@ -607,7 +707,7 @@ mod tests { check_round_trip_encoding_of_data_with_expected( vec![Arc::new(struct_array)], Some(Arc::new(expected_descriptor)), - &TestCases::default().with_min_file_version(LanceFileVersion::V2_2), + &TestCases::default().with_u32_structural_encodings(), blob_metadata, ) .await; @@ -672,7 +772,7 @@ mod tests { check_round_trip_encoding_of_data_with_expected( vec![Arc::new(struct_array)], Some(Arc::new(expected_descriptor)), - &TestCases::default().with_min_file_version(LanceFileVersion::V2_2), + &TestCases::default().with_u32_structural_encodings(), blob_metadata, ) .await; @@ -734,7 +834,7 @@ mod tests { check_round_trip_encoding_of_data_with_expected( vec![Arc::new(struct_array)], Some(Arc::new(expected_descriptor)), - &TestCases::default().with_min_file_version(LanceFileVersion::V2_2), + &TestCases::default().with_u32_structural_encodings(), blob_metadata, ) .await; @@ -796,7 +896,7 @@ mod tests { check_round_trip_encoding_of_data_with_expected( vec![Arc::new(struct_array)], Some(Arc::new(expected_descriptor)), - &TestCases::default().with_min_file_version(LanceFileVersion::V2_2), + &TestCases::default().with_u32_structural_encodings(), blob_metadata, ) .await; diff --git a/rust/lance-encoding/src/encodings/logical/fixed_size_list.rs b/rust/lance-encoding/src/encodings/logical/fixed_size_list.rs index 9e8e3e109ea..94d3702fd66 100644 --- a/rust/lance-encoding/src/encodings/logical/fixed_size_list.rs +++ b/rust/lance-encoding/src/encodings/logical/fixed_size_list.rs @@ -65,7 +65,7 @@ impl FieldEncoder for FixedSizeListStructuralEncoder { } else { deep_copy_nulls(array.nulls()) }; - repdef.add_fsl(validity.clone(), dimension, num_rows as usize); + repdef.add_fsl(validity.clone(), dimension, fsl_arr.len()); // FSL forces child elements to exist even under null rows. Normalize any // nested lists under null FSL rows to null empty lists. @@ -255,7 +255,7 @@ impl StructuralDecodeArrayTask for StructuralFixedSizeListDecodeTask { match &self.data_type { DataType::FixedSizeList(child_field, dimension) => { let num_rows = self.num_rows as usize; - let validity = repdef.unravel_fsl_validity(num_rows, *dimension as usize); + let validity = repdef.unravel_fsl_validity(num_rows, *dimension as usize)?; let fsl_array = arrow_array::FixedSizeListArray::try_new( child_field.clone(), *dimension, @@ -534,7 +534,6 @@ mod tests { STRUCTURAL_ENCODING_MINIBLOCK, }, testing::{TestCases, check_specific_random}, - version::LanceFileVersion, }; fn make_fsl_struct_type(struct_fields: Fields, dimension: i32) -> DataType { @@ -688,18 +687,17 @@ mod tests { } #[rstest] - #[case::simple(simple_struct_fields(), 2, LanceFileVersion::V2_2)] - #[case::nested_struct(nested_struct_fields(), 2, LanceFileVersion::V2_2)] - #[case::struct_with_list(struct_with_list_fields(), 2, LanceFileVersion::V2_2)] - #[case::struct_with_large_list(struct_with_large_list_fields(), 2, LanceFileVersion::V2_2)] - #[case::nested_struct_with_list(nested_struct_with_list_fields(), 2, LanceFileVersion::V2_2)] - #[case::struct_with_nested_fsl(struct_with_nested_fsl_fields(), 2, LanceFileVersion::V2_2)] - #[case::struct_with_map(struct_with_map_fields(), 2, LanceFileVersion::V2_2)] + #[case::simple(simple_struct_fields(), 2)] + #[case::nested_struct(nested_struct_fields(), 2)] + #[case::struct_with_list(struct_with_list_fields(), 2)] + #[case::struct_with_large_list(struct_with_large_list_fields(), 2)] + #[case::nested_struct_with_list(nested_struct_with_list_fields(), 2)] + #[case::struct_with_nested_fsl(struct_with_nested_fsl_fields(), 2)] + #[case::struct_with_map(struct_with_map_fields(), 2)] #[test_log::test(tokio::test)] async fn test_fsl_struct_random( #[case] struct_fields: Fields, #[case] dimension: i32, - #[case] min_version: LanceFileVersion, #[values(STRUCTURAL_ENCODING_MINIBLOCK, STRUCTURAL_ENCODING_FULLZIP)] structural_encoding: &str, ) { @@ -710,7 +708,7 @@ mod tests { structural_encoding.into(), ); let field = Field::new("", data_type, true).with_metadata(field_metadata); - let test_cases = TestCases::basic().with_min_file_version(min_version); + let test_cases = TestCases::basic().with_u32_structural_encodings(); check_specific_random(field, test_cases).await; } diff --git a/rust/lance-encoding/src/encodings/logical/list.rs b/rust/lance-encoding/src/encodings/logical/list.rs index 250eb476671..13d6754e6e3 100644 --- a/rust/lance-encoding/src/encodings/logical/list.rs +++ b/rust/lance-encoding/src/encodings/logical/list.rs @@ -251,9 +251,9 @@ mod tests { use arrow_schema::{DataType, Field, Fields}; use rstest::rstest; - use crate::{ - testing::{TestCases, check_basic_random, check_round_trip_encoding_of_data}, - version::LanceFileVersion, + use crate::testing::{ + TestCases, TestEncoding, check_basic_random, check_round_trip_encoding_of_data, + create_test_field_encoder, test_encoding_strategy, }; fn make_list_type(inner_type: DataType) -> DataType { @@ -277,20 +277,16 @@ mod tests { let arrow_field = Field::new("", array.data_type().clone(), true).with_metadata(field_metadata); let lance_field = lance_core::datatypes::Field::try_from(&arrow_field).unwrap(); - let encoding_strategy = crate::encoder::default_encoding_strategy(LanceFileVersion::V2_2); + let encoding_strategy = test_encoding_strategy(TestEncoding::StructuralU32); let mut column_index_seq = crate::encoder::ColumnIndexSequence::default(); - let encoding_options = crate::encoder::EncodingOptions { - version: LanceFileVersion::V2_2, - ..Default::default() - }; - let mut encoder = encoding_strategy - .create_field_encoder( - encoding_strategy.as_ref(), - &lance_field, - &mut column_index_seq, - &encoding_options, - ) - .unwrap(); + let encoding_options = crate::encoder::EncodingOptions::default(); + let mut encoder = create_test_field_encoder( + encoding_strategy.as_ref(), + &lance_field, + &mut column_index_seq, + &encoding_options, + ) + .unwrap(); let mut external_buffers = crate::encoder::OutOfLineBuffers::new(0, crate::encoder::MIN_PAGE_BUFFER_ALIGNMENT); let num_rows = array.len() as u64; @@ -344,6 +340,7 @@ mod tests { } } crate::format::pb21::page_layout::Layout::BlobLayout(_) => {} + crate::format::pb21::page_layout::Layout::SparseLayout(_) => {} } } @@ -514,7 +511,7 @@ mod tests { structural_encoding.into(), ); - let test_cases = TestCases::default().with_min_file_version(LanceFileVersion::V2_1); + let test_cases = TestCases::default().with_structural_encodings(); check_round_trip_encoding_of_data(chunks, &test_cases, field_metadata).await; } @@ -624,7 +621,7 @@ mod tests { .with_range(5..7) .with_indices(vec![1, 6]) .with_indices(vec![6]) - .with_min_file_version(LanceFileVersion::V2_1); + .with_structural_encodings(); check_round_trip_encoding_of_data(vec![Arc::new(outer_list)], &test_cases, field_metadata) .await; } @@ -655,7 +652,7 @@ mod tests { .with_range(1..3) .with_indices(vec![1, 3]) .with_indices(vec![2]) - .with_min_file_version(LanceFileVersion::V2_1); + .with_structural_encodings(); check_round_trip_encoding_of_data(vec![Arc::new(list_array)], &test_cases, field_metadata) .await; } @@ -686,7 +683,7 @@ mod tests { .with_range(1..3) .with_indices(vec![1, 3]) .with_indices(vec![2]) - .with_min_file_version(LanceFileVersion::V2_1); + .with_structural_encodings(); check_round_trip_encoding_of_data(vec![Arc::new(list_array)], &test_cases, field_metadata) .await; } @@ -718,7 +715,7 @@ mod tests { .with_range(1..2) .with_indices(vec![0]) .with_indices(vec![1]) - .with_min_file_version(LanceFileVersion::V2_1); + .with_structural_encodings(); check_round_trip_encoding_of_data(vec![Arc::new(list_array)], &test_cases, field_metadata) .await; } @@ -770,7 +767,7 @@ mod tests { .with_range(1..2) .with_indices(vec![1]) .with_indices(vec![2]) - .with_min_file_version(LanceFileVersion::V2_1); + .with_structural_encodings(); check_round_trip_encoding_of_data( vec![Arc::new(list_arr)], &test_cases, @@ -809,7 +806,7 @@ mod tests { .with_range(1..2) .with_indices(vec![1]) .with_indices(vec![2]) - .with_min_file_version(LanceFileVersion::V2_1); + .with_structural_encodings(); check_round_trip_encoding_of_data(vec![Arc::new(list_arr)], &test_cases, field_metadata) .await; } @@ -845,7 +842,7 @@ mod tests { ); let test_cases = TestCases::default() - .with_min_file_version(LanceFileVersion::V2_1) + .with_structural_encodings() .with_page_sizes(vec![100]) .with_range(800..900); check_round_trip_encoding_of_data( @@ -1005,7 +1002,7 @@ mod tests { let test_cases = TestCases::default() .with_range(0..2) .with_indices(vec![1]) - .with_min_file_version(LanceFileVersion::V2_1); + .with_structural_encodings(); check_round_trip_encoding_of_data( vec![struct_array.clone()], &test_cases, @@ -1027,7 +1024,7 @@ mod tests { outer_list_builder.append_null(); let list_array = Arc::new(outer_list_builder.finish()); - let test_cases = TestCases::default().with_min_file_version(LanceFileVersion::V2_1); + let test_cases = TestCases::default().with_structural_encodings(); check_round_trip_encoding_of_data(vec![list_array], &test_cases, HashMap::new()).await; } @@ -1092,7 +1089,7 @@ mod tests { // This should trigger the assertion failure at primitive.rs:1362 // debug_assert!(rows_avail > 0) - let test_cases = TestCases::default().with_min_file_version(LanceFileVersion::V2_1); + let test_cases = TestCases::default().with_structural_encodings(); // The bug manifests when encoding this specific pattern // Expected: successful round-trip encoding @@ -1106,8 +1103,11 @@ mod tests { #[values(STRUCTURAL_ENCODING_MINIBLOCK, STRUCTURAL_ENCODING_FULLZIP)] structural_encoding: &str, ) { - // 2.5 million rows, mostly empty lists. ~100 lists have 10 short strings each. - let num_rows = 2_500_000u32; + // Three chunks' worth of rep/def levels (1 rep bit + 1 def bit each), so the + // planner must split the page. See #6184. + let levels_per_chunk = + crate::encodings::logical::primitive::miniblock::max_repdef_levels_per_chunk(2); + let num_rows = (levels_per_chunk * 3) as u32; let num_non_empty = 100u32; let strings_per_list = 10; @@ -1146,7 +1146,7 @@ mod tests { .with_range(0..1000) .with_range(0..num_rows as u64) .with_indices(vec![0, (step / 2) as u64, num_rows as u64 - 1]) - .with_max_file_version(LanceFileVersion::V2_2); + .with_dense_encodings(); check_round_trip_encoding_of_data(vec![Arc::new(list_array)], &test_cases, field_metadata) .await; } @@ -1186,7 +1186,7 @@ mod tests { .with_range(0..1000) .with_range(0..num_rows as u64) .with_indices(vec![0, (step / 2) as u64, num_rows as u64 - 1]) - .with_max_file_version(LanceFileVersion::V2_2); + .with_dense_encodings(); let list_array = Arc::new(list_array) as ArrayRef; let pages = encode_v22_pages(list_array.clone()).await; assert_split_miniblock_layout(&pages, false); @@ -1222,7 +1222,7 @@ mod tests { let test_cases = TestCases::default() .with_range(0..num_rows as u64) .with_indices(vec![0, empty_prefix_rows as u64, num_rows as u64 - 1]) - .with_max_file_version(LanceFileVersion::V2_2); + .with_dense_encodings(); let list_array = Arc::new(list_array) as ArrayRef; let pages = encode_v22_pages(list_array.clone()).await; assert_split_miniblock_layout(&pages, true); @@ -1259,7 +1259,7 @@ mod tests { let test_cases = TestCases::default() .with_range(0..num_rows as u64) .with_indices(vec![0, null_prefix_rows as u64, num_rows as u64 - 1]) - .with_max_file_version(LanceFileVersion::V2_2); + .with_dense_encodings(); let list_array = Arc::new(list_array) as ArrayRef; let pages = encode_v22_pages(list_array.clone()).await; assert_split_miniblock_layout(&pages, true); @@ -1290,7 +1290,7 @@ mod tests { let test_cases = TestCases::default() .with_range(0..num_rows as u64) .with_indices(vec![0, empty_prefix_rows as u64]) - .with_max_file_version(LanceFileVersion::V2_2); + .with_dense_encodings(); let list_array = Arc::new(list_array) as ArrayRef; let pages = encode_v22_pages(list_array.clone()).await; assert_split_miniblock_layout(&pages, true); @@ -1363,7 +1363,7 @@ mod tests { let test_cases = TestCases::default() .with_range(0..1) .with_indices(vec![0]) - .with_max_file_version(LanceFileVersion::V2_2); + .with_dense_encodings(); check_round_trip_encoding_of_data(vec![outer_list], &test_cases, HashMap::new()).await; } @@ -1411,8 +1411,7 @@ mod tests { .with_range(dense_rows.saturating_sub(8)..(dense_rows + 8)) .with_range(0..total_rows) .with_indices(vec![0, dense_rows - 1, dense_rows, total_rows - 1]) - .with_min_file_version(LanceFileVersion::V2_2) - .with_max_file_version(LanceFileVersion::V2_2); + .with_encoding(TestEncoding::StructuralU32); let list_array = Arc::new(list_array) as ArrayRef; let pages = encode_v22_pages(list_array.clone()).await; assert_split_miniblock_layout(&pages, true); @@ -1437,8 +1436,7 @@ mod tests { let test_cases = TestCases::default() .with_range(0..total_rows) - .with_min_file_version(LanceFileVersion::V2_2) - .with_max_file_version(LanceFileVersion::V2_2); + .with_encoding(TestEncoding::StructuralU32); let list_array = Arc::new(list_array) as ArrayRef; let pages = try_encode_v22_pages_with_metadata(list_array.clone(), field_metadata.clone()) .await diff --git a/rust/lance-encoding/src/encodings/logical/map.rs b/rust/lance-encoding/src/encodings/logical/map.rs index b5172d8c189..d07c267bd2a 100644 --- a/rust/lance-encoding/src/encodings/logical/map.rs +++ b/rust/lance-encoding/src/encodings/logical/map.rs @@ -223,7 +223,8 @@ impl StructuralDecodeArrayTask for StructuralMapDecodeTask { .clone(); // Build the MapArray from offsets, entries, validity, and keys_sorted - let map_array = MapArray::new(entries_field, offsets, entries, validity, keys_sorted); + let map_array = MapArray::try_new(entries_field, offsets, entries, validity, keys_sorted) + .map_err(|error| Error::invalid_input_source(error.to_string().into()))?; Ok(DecodedArray { array: Arc::new(map_array), @@ -244,14 +245,21 @@ mod tests { use arrow_buffer::{NullBuffer, OffsetBuffer, ScalarBuffer}; use arrow_schema::{DataType, Field, Fields}; - use crate::encoder::{ColumnIndexSequence, EncodingOptions, default_encoding_strategy}; - use crate::{ - testing::{TestCases, check_round_trip_encoding_of_data}, - version::LanceFileVersion, + use crate::decoder::{DecodedArray, StructuralDecodeArrayTask}; + use crate::encoder::{ColumnIndexSequence, EncodingOptions}; + use crate::encodings::logical::primitive::sparse::{ + SparseCountSet, SparsePositionSet, SparseStructuralLayerPlan, SparseStructuralPlan, + SparseValidityMeaning, SparseValiditySet, + }; + use crate::repdef::{CompositeRepDefUnraveler, RepDefUnraveler}; + use crate::testing::{ + TestCases, TestEncoding, check_round_trip_encoding_of_data, test_encoding_strategy, }; use arrow_schema::Field as ArrowField; use lance_core::datatypes::Field as LanceField; + use super::StructuralMapDecodeTask; + fn make_map_type(key_type: DataType, value_type: DataType) -> DataType { // Note: Arrow MapBuilder uses "keys" and "values" as field names (plural) let entries = Field::new( @@ -265,6 +273,70 @@ mod tests { DataType::Map(Arc::new(entries), false) } + #[derive(Debug)] + struct StaticMapEntriesTask { + entries: StructArray, + repdef: CompositeRepDefUnraveler, + } + + impl StructuralDecodeArrayTask for StaticMapEntriesTask { + fn decode(self: Box) -> lance_core::Result { + let Self { entries, repdef } = *self; + Ok(DecodedArray { + array: Arc::new(entries), + repdef, + data_size: 0, + }) + } + } + + #[test] + fn malformed_sparse_map_entries_return_invalid_input() { + let entry_fields = Fields::from(vec![ + Field::new("keys", DataType::Int32, false), + Field::new("values", DataType::Int32, true), + ]); + let entries = StructArray::try_new( + entry_fields.clone(), + vec![ + Arc::new(Int32Array::from(vec![1])), + Arc::new(Int32Array::from(vec![2])), + ], + Some(NullBuffer::from(vec![false])), + ) + .unwrap(); + let validity = SparseValiditySet { + meaning: SparseValidityMeaning::NullPositions, + positions: SparsePositionSet::Empty, + }; + let plan = SparseStructuralPlan { + layers: vec![SparseStructuralLayerPlan::List { + num_slots: 1, + num_child_slots: 1, + non_empty_positions: SparsePositionSet::All { len: 1 }, + counts: SparseCountSet::Constant { value: 1, len: 1 }, + validity, + }], + num_items: 1, + num_visible_items: 1, + }; + let child_task = StaticMapEntriesTask { + entries, + repdef: CompositeRepDefUnraveler::new(vec![RepDefUnraveler::new_sparse(plan)]), + }; + let map_type = DataType::Map( + Arc::new(Field::new("entries", DataType::Struct(entry_fields), false)), + false, + ); + + let Err(err) = + Box::new(StructuralMapDecodeTask::new(Box::new(child_task), map_type)).decode() + else { + panic!("expected malformed map entries to be rejected"); + }; + assert!(matches!(err, lance_core::Error::InvalidInput { .. })); + } + #[test_log::test(tokio::test)] async fn test_simple_map() { // Create a simple Map @@ -288,7 +360,7 @@ mod tests { let test_cases = TestCases::default() .with_range(0..2) - .with_min_file_version(LanceFileVersion::V2_2); + .with_u32_structural_encodings(); check_round_trip_encoding_of_data(vec![Arc::new(map_array)], &test_cases, HashMap::new()) .await; @@ -321,7 +393,7 @@ mod tests { .with_range(0..4) .with_indices(vec![1]) .with_indices(vec![2]) - .with_min_file_version(LanceFileVersion::V2_2); + .with_u32_structural_encodings(); check_round_trip_encoding_of_data(vec![Arc::new(map_array)], &test_cases, HashMap::new()) .await; @@ -352,7 +424,7 @@ mod tests { .with_range(0..2) .with_indices(vec![0]) .with_indices(vec![1]) - .with_min_file_version(LanceFileVersion::V2_2); + .with_u32_structural_encodings(); check_round_trip_encoding_of_data(vec![Arc::new(map_array)], &test_cases, HashMap::new()) .await; @@ -401,7 +473,7 @@ mod tests { let test_cases = TestCases::default() .with_range(0..3) .with_indices(vec![0, 2]) - .with_min_file_version(LanceFileVersion::V2_2); + .with_u32_structural_encodings(); check_round_trip_encoding_of_data( vec![Arc::new(struct_array)], @@ -455,7 +527,7 @@ mod tests { let test_cases = TestCases::default() .with_range(0..3) - .with_min_file_version(LanceFileVersion::V2_2); + .with_u32_structural_encodings(); check_round_trip_encoding_of_data( vec![Arc::new(struct_array)], @@ -501,7 +573,7 @@ mod tests { let test_cases = TestCases::default() .with_range(0..3) .with_indices(vec![0, 2]) - .with_min_file_version(LanceFileVersion::V2_2); + .with_u32_structural_encodings(); check_round_trip_encoding_of_data(vec![Arc::new(list_array)], &test_cases, HashMap::new()) .await; @@ -561,7 +633,7 @@ mod tests { let test_cases = TestCases::default() .with_range(0..1) - .with_min_file_version(LanceFileVersion::V2_2); + .with_u32_structural_encodings(); check_round_trip_encoding_of_data(vec![Arc::new(outer_map)], &test_cases, HashMap::new()) .await; @@ -591,7 +663,7 @@ mod tests { let test_cases = TestCases::default() .with_range(0..2) .with_indices(vec![0, 1]) - .with_min_file_version(LanceFileVersion::V2_2); + .with_u32_structural_encodings(); check_round_trip_encoding_of_data(vec![Arc::new(map_array)], &test_cases, HashMap::new()) .await; @@ -618,7 +690,7 @@ mod tests { let test_cases = TestCases::default() .with_range(0..2) - .with_min_file_version(LanceFileVersion::V2_2); + .with_u32_structural_encodings(); check_round_trip_encoding_of_data(vec![Arc::new(map_array)], &test_cases, HashMap::new()) .await; @@ -639,7 +711,7 @@ mod tests { let test_cases = TestCases::default() .with_range(0..2) - .with_min_file_version(LanceFileVersion::V2_2); + .with_u32_structural_encodings(); check_round_trip_encoding_of_data(vec![Arc::new(map_array)], &test_cases, HashMap::new()) .await; @@ -674,7 +746,7 @@ mod tests { let test_cases = TestCases::default() .with_range(0..3) .with_indices(vec![0, 1, 2]) - .with_min_file_version(LanceFileVersion::V2_2); + .with_u32_structural_encodings(); // This test ensures that regardless of the internal keep_original_array setting, // the end-to-end behavior produces equivalent results @@ -693,11 +765,11 @@ mod tests { let map_field = LanceField::try_from(&map_arrow_field).unwrap(); // Test encoder: Try to create encoder with V2_1 version - should fail - let encoder_strategy = default_encoding_strategy(LanceFileVersion::V2_1); + let encoder_strategy = test_encoding_strategy(TestEncoding::StructuralU16); let mut column_index = ColumnIndexSequence::default(); let options = EncodingOptions::default(); - let encoder_result = encoder_strategy.create_field_encoder( + let encoder_result = crate::testing::create_test_field_encoder( encoder_strategy.as_ref(), &map_field, &mut column_index, @@ -714,9 +786,8 @@ mod tests { let encoder_err_msg = format!("{}", encoder_err); assert!( - encoder_err_msg.contains("2.2"), - "Encoder error message should mention version 2.2, got: {}", - encoder_err_msg + encoder_err_msg.contains("not enabled by the selected file format"), + "unexpected encoder error: {encoder_err_msg}" ); assert!( encoder_err_msg.contains("Map data type"), diff --git a/rust/lance-encoding/src/encodings/logical/primitive.rs b/rust/lance-encoding/src/encodings/logical/primitive.rs index 8b6eed5d757..15b39d62b26 100644 --- a/rust/lance-encoding/src/encodings/logical/primitive.rs +++ b/rust/lance-encoding/src/encodings/logical/primitive.rs @@ -15,6 +15,7 @@ use std::{ use crate::{ constants::{ STRUCTURAL_ENCODING_FULLZIP, STRUCTURAL_ENCODING_META_KEY, STRUCTURAL_ENCODING_MINIBLOCK, + STRUCTURAL_ENCODING_SPARSE, }, data::DictionaryDataBlock, encodings::logical::primitive::blob::{BlobDescriptionPageScheduler, BlobPageScheduler}, @@ -32,17 +33,19 @@ use itertools::Itertools; use lance_arrow::DataTypeExt; use lance_arrow::deepcopy::deep_copy_nulls; use lance_core::{ - cache::{CacheKey, Context, DeepSizeOf}, + cache::{CacheKey, CacheKeySchema, Context, DeepSizeOf, KeyBuilder}, error::{Error, LanceOptionExt}, utils::bit::pad_bytes, }; -use log::trace; +use log::{debug, trace}; use crate::encodings::logical::primitive::miniblock::MiniBlockChunk; +use crate::encodings::physical::rle::{RleDecompressor, RleRuns}; use crate::utils::bytepack::ByteUnpacker; use crate::{ compression::{ BlockDecompressor, CompressionStrategy, DecompressionStrategy, MiniBlockDecompressor, + create_rle_decompressor, }, data::{AllNullDataBlock, DataBlock, VariableWidthBlock}, utils::bytepack::BytepackedIntegerEncoder, @@ -52,13 +55,14 @@ use crate::{ encodings::logical::primitive::fullzip::PerValueDataBlock, }; use crate::{ - encodings::logical::primitive::miniblock::MiniBlockCompressed, + encodings::logical::primitive::miniblock::{MiniBlockCompressed, MiniBlockCompressionContext}, statistics::{ComputeStat, GetStat, Stat}, }; use crate::{ repdef::{ CompositeRepDefUnraveler, ControlWordIterator, ControlWordParser, DefinitionInterpretation, - RepDefSlicer, SerializedRepDefs, StructuralPagePlan, build_control_word_iterator, + MiniBlockRepDefBudget, NormalizedStructuralPlan, RepDefSlicer, SerializedRepDefs, + build_control_word_iterator, }, utils::accumulation::AccumulationQueue, }; @@ -70,7 +74,6 @@ use crate::constants::{ DICT_VALUES_COMPRESSION_LEVEL_ENV_VAR, DICT_VALUES_COMPRESSION_LEVEL_META_KEY, DICT_VALUES_COMPRESSION_META_KEY, }; -use crate::version::LanceFileVersion; use crate::{ EncodingsIo, buffer::LanceBuffer, @@ -88,10 +91,15 @@ use crate::{ }; pub mod blob; +mod chunk_index; pub mod constant; pub mod dict; pub mod fullzip; +mod layout; pub mod miniblock; +pub(crate) mod sparse; + +use chunk_index::{ItemCounts, MiniBlockChunkIndex, PrefixSums, RowMapping, parse_nested_rep}; const FILL_BYTE: u8 = 0xFE; const DEFAULT_DICT_DIVISOR: u64 = 2; @@ -166,6 +174,19 @@ struct DecodeMiniBlockTask { } impl DecodeMiniBlockTask { + fn decoded_size_bytes(&self) -> Option { + if self.rep_decompressor.is_some() || self.def_decompressor.is_some() { + return None; + } + let num_values = self + .instructions + .iter() + .try_fold(0_u64, |total, (instruction, _)| { + total.checked_add(instruction.rows_to_take) + })?; + self.value_decompressor.decoded_size_bytes(num_values) + } + fn decode_levels( rep_decompressor: &dyn BlockDecompressor, levels: LanceBuffer, @@ -548,15 +569,15 @@ impl DecodePageTask for DecodeMiniBlockTask { let max_rep = self.def_meaning.iter().filter(|l| l.is_list()).count() as u16; - // This is probably an over-estimate but it's quick and easy to calculate - let estimated_size_bytes = self - .instructions - .iter() - .map(|(_, chunk)| chunk.data.len()) - .sum::() - * 2; - let mut data_builder = - DataBlockBuilder::with_capacity_estimate(estimated_size_bytes as u64); + let estimated_size_bytes = self.decoded_size_bytes().unwrap_or_else(|| { + // Variable-width and rep/def encoded output sizes are not known before decoding. + self.instructions + .iter() + .map(|(_, chunk)| chunk.data.len() as u64) + .sum::() + * 2 + }); + let mut data_builder = DataBlockBuilder::with_capacity_estimate(estimated_size_bytes); // We need to keep track of the offset into repbuf/defbuf that we are building up let mut level_offset = 0; @@ -621,7 +642,7 @@ impl DecodePageTask for DecodeMiniBlockTask { Self::extend_levels(level_range.clone(), &mut repbuf, &rep, level_offset); Self::extend_levels(level_range.clone(), &mut defbuf, &def, level_offset); level_offset += (level_range.end - level_range.start) as usize; - data_builder.append(&values, item_range); + data_builder.append(&values, item_range)?; } let mut data = data_builder.finish(); @@ -743,16 +764,669 @@ impl StructuralPageDecoder for MiniBlockDecoder { } } +/// How a complex-all-null page's rep/def level buffer is compressed on disk. +/// Captured at scheduler construction so `initialize` can keep RLE levels in run +/// form instead of expanding them. +#[derive(Debug, Clone)] +pub(crate) enum LevelCodec { + /// Raw little-endian u16 levels (no block compression). + Uncompressed, + /// RLE-compressed levels; the validated physical runs select their cached representation. + Rle(Arc), + /// Any other block compression; decoded eagerly into [`LazyLevels::Dense`] + /// (these encodings don't expand, so laziness buys nothing). + Block(Arc), +} + +impl LevelCodec { + fn try_new( + encoding: Option<&CompressiveEncoding>, + decompression_strategy: &dyn DecompressionStrategy, + ) -> Result { + match encoding { + None => Ok(Self::Uncompressed), + Some(encoding) => match encoding.compression.as_ref() { + Some(Compression::Rle(rle)) => Ok(Self::Rle(Arc::new(create_rle_decompressor( + rle, + decompression_strategy, + )?))), + _ => Ok(Self::Block(Arc::from( + decompression_strategy.create_block_decompressor(encoding)?, + ))), + }, + } + } +} + +#[derive(Debug)] +enum RunEnds { + U16(Box<[u16]>), + U32(Box<[u32]>), + U64(Box<[u64]>), +} + +impl RunEnds { + fn width_for(num_values: usize) -> usize { + if u16::try_from(num_values).is_ok() { + std::mem::size_of::() + } else if u32::try_from(num_values).is_ok() { + std::mem::size_of::() + } else { + std::mem::size_of::() + } + } + + fn len(&self) -> usize { + match self { + Self::U16(ends) => ends.len(), + Self::U32(ends) => ends.len(), + Self::U64(ends) => ends.len(), + } + } + + fn get(&self, run: usize) -> usize { + match self { + Self::U16(ends) => ends[run] as usize, + Self::U32(ends) => ends[run] as usize, + Self::U64(ends) => ends[run] as usize, + } + } + + fn partition_point(&self, logical_index: usize) -> usize { + match self { + Self::U16(ends) => ends.partition_point(|&end| end as usize <= logical_index), + Self::U32(ends) => ends.partition_point(|&end| end as usize <= logical_index), + Self::U64(ends) => ends.partition_point(|&end| end as usize <= logical_index), + } + } + + fn deep_size(&self) -> usize { + match self { + Self::U16(ends) => std::mem::size_of_val(ends.as_ref()), + Self::U32(ends) => std::mem::size_of_val(ends.as_ref()), + Self::U64(ends) => std::mem::size_of_val(ends.as_ref()), + } + } +} + +enum RunEndsBuilder { + U16(Vec), + U32(Vec), + U64(Vec), +} + +impl RunEndsBuilder { + fn with_capacity(num_values: usize, capacity: usize) -> Self { + if u16::try_from(num_values).is_ok() { + Self::U16(Vec::with_capacity(capacity)) + } else if u32::try_from(num_values).is_ok() { + Self::U32(Vec::with_capacity(capacity)) + } else { + Self::U64(Vec::with_capacity(capacity)) + } + } + + fn push(&mut self, end: usize) -> Result<()> { + match self { + Self::U16(ends) => ends.push( + u16::try_from(end) + .map_err(|_| Error::internal(format!("Run end {end} does not fit in u16")))?, + ), + Self::U32(ends) => ends.push( + u32::try_from(end) + .map_err(|_| Error::internal(format!("Run end {end} does not fit in u32")))?, + ), + Self::U64(ends) => ends.push(end as u64), + } + Ok(()) + } + + fn set_last(&mut self, end: usize) -> Result<()> { + match self { + Self::U16(ends) => { + let last = ends.last_mut().ok_or_else(|| { + Error::internal("Cannot extend an empty coalesced run buffer") + })?; + *last = u16::try_from(end) + .map_err(|_| Error::internal(format!("Run end {end} does not fit in u16")))?; + } + Self::U32(ends) => { + let last = ends.last_mut().ok_or_else(|| { + Error::internal("Cannot extend an empty coalesced run buffer") + })?; + *last = u32::try_from(end) + .map_err(|_| Error::internal(format!("Run end {end} does not fit in u32")))?; + } + Self::U64(ends) => { + let last = ends.last_mut().ok_or_else(|| { + Error::internal("Cannot extend an empty coalesced run buffer") + })?; + *last = end as u64; + } + } + Ok(()) + } + + fn finish(self) -> RunEnds { + match self { + Self::U16(ends) => RunEnds::U16(ends.into_boxed_slice()), + Self::U32(ends) => RunEnds::U32(ends.into_boxed_slice()), + Self::U64(ends) => RunEnds::U64(ends.into_boxed_slice()), + } + } +} + +#[derive(Debug)] +enum RunStorage { + Physical(RleRuns), + Coalesced { values: Box<[u16]>, ends: RunEnds }, +} + +impl RunStorage { + fn len(&self) -> usize { + match self { + Self::Physical(runs) => runs.num_values(), + Self::Coalesced { ends, .. } => ends.get(ends.len() - 1), + } + } + + fn num_runs(&self) -> usize { + match self { + Self::Physical(runs) => runs.num_runs(), + Self::Coalesced { values, .. } => values.len(), + } + } + + fn value(&self, run: usize) -> u16 { + match self { + Self::Physical(runs) => runs.value(run), + Self::Coalesced { values, .. } => values[run], + } + } + + fn first_value_above(&self, max: u16) -> Option<(usize, u16)> { + (0..self.num_runs()).find_map(|run| { + let value = self.value(run); + (value > max).then_some((run, value)) + }) + } + + fn seek(&self, position: &mut RunPosition, logical_index: usize) { + if logical_index >= self.len() { + *position = RunPosition { + run: self.num_runs(), + start: self.len(), + end: self.len(), + }; + return; + } + + match self { + Self::Physical(runs) => { + if position.run >= runs.num_runs() + || position.end == 0 + || logical_index < position.start + { + *position = RunPosition { + run: 0, + start: 0, + end: runs.length(0), + }; + } + while position.end <= logical_index { + self.advance(position); + } + } + Self::Coalesced { ends, .. } => { + if logical_index < position.start || logical_index >= position.end { + let run = ends.partition_point(logical_index); + *position = RunPosition { + run, + start: if run == 0 { 0 } else { ends.get(run - 1) }, + end: ends.get(run), + }; + } + } + } + } + + fn advance(&self, position: &mut RunPosition) { + let next_run = position.run + 1; + if next_run >= self.num_runs() { + *position = RunPosition { + run: self.num_runs(), + start: self.len(), + end: self.len(), + }; + return; + } + + let start = position.end; + position.run = next_run; + position.start = start; + position.end = match self { + Self::Physical(runs) => start + runs.length(next_run), + Self::Coalesced { ends, .. } => ends.get(next_run), + }; + } + + fn deep_size(&self) -> usize { + match self { + Self::Physical(runs) => runs.deep_size(), + Self::Coalesced { values, ends } => { + std::mem::size_of_val(values.as_ref()) + ends.deep_size() + } + } + } +} + +/// Rep/def levels for a complex-all-null page. +/// +/// RLE pages retain the smallest of their validated physical runs, coalesced +/// runs, and dense values. The decoder materializes only the per-drain slices +/// it touches. +#[derive(Debug, Clone)] +enum LazyLevels { + Dense(ScalarBuffer), + Runs(Arc), +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum LevelPlan { + Physical, + Coalesced, + Dense, +} + +#[derive(Debug, Default, Clone, Copy)] +struct RunPosition { + run: usize, + start: usize, + end: usize, +} + +/// Monotonic forward cursor into a [`LazyLevels`] sequence. +/// +/// Drains seek to strictly increasing rows, so each [`LazyLevels::seek_row_start`] +/// resumes from the last position instead of rescanning — every run is visited at +/// most once per page while locating and counting level ranges. +#[derive(Debug, Default, Clone, Copy)] +struct LevelCursor { + /// Logical level index where the current row begins. + level: usize, + /// Row index at `level` (the number of `max_rep` occurrences before it). + row: u64, + /// Run containing `level`. Unused for [`LazyLevels::Dense`]. + run: RunPosition, +} + +impl LazyLevels { + fn from_rle_runs(runs: RleRuns) -> Result { + let plan = Self::select_plan(&runs); + match plan { + LevelPlan::Physical => Ok(Self::Runs(Arc::new(RunStorage::Physical( + runs.into_owned(), + )))), + LevelPlan::Coalesced => Self::build_coalesced(runs), + LevelPlan::Dense => Self::build_dense(runs), + } + } + + /// Minimize retained payload bytes first, then expected traversal work. + /// If both are equal, keep the physical runs and avoid another allocation. + fn select_plan(runs: &RleRuns) -> LevelPlan { + if runs.num_values() == 0 { + return LevelPlan::Dense; + } + + let run_storage_size = std::mem::size_of::() as u128; + let physical_size = run_storage_size + runs.owned_size() as u128; + let coalesced_size = run_storage_size + + (runs.coalesced_runs() as u128) + * (std::mem::size_of::() + RunEnds::width_for(runs.num_values())) as u128; + let dense_size = (runs.num_values() as u128) * std::mem::size_of::() as u128; + [ + (physical_size, runs.num_runs(), 0usize, LevelPlan::Physical), + ( + coalesced_size, + runs.coalesced_runs(), + 1usize, + LevelPlan::Coalesced, + ), + (dense_size, runs.num_values(), 2usize, LevelPlan::Dense), + ] + .into_iter() + .min_by_key(|(size, traversal, priority, _)| (*size, *traversal, *priority)) + .map(|(_, _, _, plan)| plan) + .unwrap_or(LevelPlan::Dense) + } + + fn build_coalesced(runs: RleRuns) -> Result { + let mut values = Vec::with_capacity(runs.coalesced_runs()); + let mut ends = RunEndsBuilder::with_capacity(runs.num_values(), runs.coalesced_runs()); + let mut logical_end = 0usize; + for (value, length) in runs.iter() { + logical_end = logical_end + .checked_add(length) + .ok_or_else(|| Error::internal("Validated RLE run length sum overflowed usize"))?; + if values.last().copied() == Some(value) { + ends.set_last(logical_end)?; + } else { + values.push(value); + ends.push(logical_end)?; + } + } + Ok(Self::Runs(Arc::new(RunStorage::Coalesced { + values: values.into_boxed_slice(), + ends: ends.finish(), + }))) + } + + fn build_dense(runs: RleRuns) -> Result { + let mut values = Vec::new(); + values.try_reserve_exact(runs.num_values()).map_err(|_| { + Error::internal(format!( + "Cannot allocate {} dense repetition/definition levels", + runs.num_values() + )) + })?; + for (value, length) in runs.iter() { + values.resize(values.len() + length, value); + } + Ok(Self::Dense(ScalarBuffer::from(values))) + } + + fn len(&self) -> usize { + match self { + Self::Dense(buf) => buf.len(), + Self::Runs(runs) => runs.len(), + } + } + + fn validate_max(&self, level_type: &str, max: u16) -> Result<()> { + let invalid = match self { + Self::Dense(levels) => levels + .iter() + .enumerate() + .find_map(|(index, &value)| (value > max).then_some(("index", index, value))), + Self::Runs(runs) => runs + .first_value_above(max) + .map(|(run, value)| ("run", run, value)), + }; + if let Some((position_type, position, value)) = invalid { + return Err(Error::invalid_input_source( + format!( + "Invalid {level_type} level {value} at {position_type} {position}: maximum is {max}" + ) + .into(), + )); + } + Ok(()) + } + + /// Advance `cursor` to the start of row `target_row`, returning that row's + /// starting level index. + /// + /// Rows begin at `max_rep` positions, so this finds the `target_row`-th one. + /// `target_row` must be `>= cursor.row`: the cursor only moves forward, which + /// is what keeps a full page decode O(runs) rather than O(rows). + fn seek_row_start( + &self, + cursor: &mut LevelCursor, + target_row: u64, + max_rep: u16, + ) -> Result { + let mut need = target_row.checked_sub(cursor.row).ok_or_else(|| { + Error::internal(format!( + "Complex all-null row ranges are not sorted: target row {target_row} follows {}", + cursor.row + )) + })?; + if need == 0 { + return Ok(cursor.level); + } + match self { + Self::Dense(buf) => { + let mut level = cursor.level; + while need > 0 { + if level >= buf.len() { + return Err(Error::internal( + "Invalid complex all-null layout: repetition buffer too short", + )); + } + if buf[level] != max_rep { + return Err(Error::internal( + "Invalid complex all-null layout: row did not start at max repetition level", + )); + } + level += 1; + while level < buf.len() && buf[level] != max_rep { + level += 1; + } + need -= 1; + } + cursor.level = level; + cursor.row = target_row; + Ok(level) + } + Self::Runs(runs) => { + let mut level = cursor.level; + let mut run = cursor.run; + runs.seek(&mut run, level); + while need > 0 { + if run.run >= runs.num_runs() { + return Err(Error::internal( + "Invalid complex all-null layout: repetition buffer too short", + )); + } + if runs.value(run.run) != max_rep { + return Err(Error::internal( + "Invalid complex all-null layout: row did not start at max repetition level", + )); + } + let avail = (run.end - level) as u64; + if need < avail { + // Target lands inside this max-rep run. + level += need as usize; + need = 0; + } else { + // Consume every row start in this run, then skip the + // trailing non-max-rep runs to reach the next row start. + need -= avail; + runs.advance(&mut run); + while run.run < runs.num_runs() && runs.value(run.run) != max_rep { + runs.advance(&mut run); + } + level = if run.run < runs.num_runs() { + run.start + } else { + self.len() + }; + } + } + cursor.level = level; + cursor.row = target_row; + cursor.run = run; + Ok(level) + } + } + } + + /// Count of levels in `range` that are `<= max`, resuming from `*run_cursor` + /// and leaving it on the last run that overlaps `range`. + /// + /// Successive calls must pass ascending, non-overlapping ranges (`range.start + /// >=` the previous `range.end`) so runs are swept at most once per page. + fn count_le_cursor( + &self, + run_cursor: &mut RunPosition, + range: Range, + max: u16, + ) -> (u64, RunPosition) { + if range.is_empty() { + return (0, *run_cursor); + } + match self { + Self::Dense(buf) => ( + buf[range].iter().filter(|&&d| d <= max).count() as u64, + RunPosition::default(), + ), + Self::Runs(runs) => { + // Advance to the first run overlapping the range. + runs.seek(run_cursor, range.start); + let start = *run_cursor; + let mut count = 0u64; + let mut current = *run_cursor; + while current.run < runs.num_runs() && current.start < range.end { + if runs.value(current.run) <= max { + let lo = current.start.max(range.start); + let hi = current.end.min(range.end); + count += (hi - lo) as u64; + } + if current.end >= range.end { + break; + } + runs.advance(&mut current); + } + // Resume the next (ascending) range from the last overlapping run; + // `current` remains valid because `range` is non-empty. + *run_cursor = current; + (count, start) + } + } + } + + fn extend_into(&self, range: Range, run: RunPosition, out: &mut Vec) { + if range.is_empty() { + return; + } + match self { + Self::Dense(buf) => out.extend_from_slice(&buf[range]), + Self::Runs(runs) => { + let mut current = run; + runs.seek(&mut current, range.start); + while current.run < runs.num_runs() && current.start < range.end { + let lo = current.start.max(range.start); + let hi = current.end.min(range.end); + if hi > lo { + out.resize(out.len() + (hi - lo), runs.value(current.run)); + } + runs.advance(&mut current); + } + } + } + } + + #[cfg(test)] + fn deep_size(&self) -> usize { + self.deep_size_of_children(&mut Context::new()) + } +} + +impl DeepSizeOf for LazyLevels { + fn deep_size_of_children(&self, ctx: &mut Context) -> usize { + match self { + Self::Dense(buf) => buf.deep_size_of_children(ctx), + Self::Runs(runs) => { + let pointer = Arc::as_ptr(runs) as *const () as usize; + if ctx.mark_seen(pointer) { + std::mem::size_of_val(runs.as_ref()) + runs.deep_size() + } else { + 0 + } + } + } + } +} + +fn validate_complex_all_null_levels( + rep: &Option, + def: &Option, + max_rep: u16, + max_def: u16, +) -> Result<()> { + if let Some(rep) = rep { + rep.validate_max("repetition", max_rep)?; + } + if let Some(def) = def { + def.validate_max("definition", max_def)?; + } + if let (Some(rep), Some(def)) = (rep, def) + && rep.len() != def.len() + { + return Err(Error::invalid_input_source( + format!( + "Mismatched complex all-null level counts: repetition has {}, definition has {}", + rep.len(), + def.len() + ) + .into(), + )); + } + Ok(()) +} + +fn expected_level_bytes(num_values: u64, level_type: &str) -> Result { + usize::try_from(num_values) + .ok() + .and_then(|num_values| num_values.checked_mul(std::mem::size_of::())) + .ok_or_else(|| { + Error::invalid_input_source( + format!("{level_type} level count {num_values} does not fit in memory").into(), + ) + }) +} + +fn dense_levels_from_block( + decompressed: DataBlock, + num_values: u64, + level_type: &str, +) -> Result { + let DataBlock::FixedWidth(block) = decompressed else { + return Err(Error::invalid_input_source( + format!("Expected fixed-width data block for {level_type} levels").into(), + )); + }; + if block.num_values != num_values { + return Err(Error::invalid_input_source( + format!( + "Unexpected {level_type} level count after decompression: expected {num_values}, got {}", + block.num_values + ) + .into(), + )); + } + if block.bits_per_value != 16 { + return Err(Error::invalid_input_source( + format!( + "Unexpected {level_type} level bit width after decompression: expected 16, got {}", + block.bits_per_value + ) + .into(), + )); + } + let expected_bytes = expected_level_bytes(num_values, level_type)?; + if block.data.len() != expected_bytes { + return Err(Error::invalid_input_source( + format!( + "Unexpected decompressed {level_type} level size: expected {expected_bytes} bytes for {num_values} values, got {}", + block.data.len() + ) + .into(), + )); + } + Ok(LazyLevels::Dense(block.data.borrow_to_typed_slice::())) +} + #[derive(Debug)] struct CachedComplexAllNullState { - rep: Option>, - def: Option>, + rep: Option, + def: Option, } impl DeepSizeOf for CachedComplexAllNullState { - fn deep_size_of_children(&self, _ctx: &mut Context) -> usize { - self.rep.as_ref().map(|buf| buf.len() * 2).unwrap_or(0) - + self.def.as_ref().map(|buf| buf.len() * 2).unwrap_or(0) + fn deep_size_of_children(&self, ctx: &mut Context) -> usize { + self.rep.deep_size_of_children(ctx) + self.def.deep_size_of_children(ctx) } } @@ -777,23 +1451,28 @@ pub struct ComplexAllNullScheduler { def_meaning: Arc<[DefinitionInterpretation]>, repdef: Option>, max_rep: u16, + max_def: u16, max_visible_level: u16, - rep_decompressor: Option>, - def_decompressor: Option>, + rep_codec: LevelCodec, + def_codec: LevelCodec, num_rep_values: u64, num_def_values: u64, } impl ComplexAllNullScheduler { - pub fn new( + pub(crate) fn new( buffer_offsets_and_sizes: Arc<[(u64, u64)]>, def_meaning: Arc<[DefinitionInterpretation]>, - rep_decompressor: Option>, - def_decompressor: Option>, + rep_codec: LevelCodec, + def_codec: LevelCodec, num_rep_values: u64, num_def_values: u64, ) -> Self { let max_rep = def_meaning.iter().filter(|l| l.is_list()).count() as u16; + let max_def = def_meaning + .iter() + .map(|meaning| meaning.num_def_levels()) + .sum::(); let max_visible_level = def_meaning .iter() .take_while(|l| !l.is_list()) @@ -804,9 +1483,10 @@ impl ComplexAllNullScheduler { def_meaning, repdef: None, max_rep, + max_def, max_visible_level, - rep_decompressor, - def_decompressor, + rep_codec, + def_codec, num_rep_values, num_def_values, } @@ -833,84 +1513,83 @@ impl StructuralPageScheduler for ComplexAllNullScheduler { } let data = io.submit_request(reads, 0); - let rep_decompressor = self.rep_decompressor.clone(); - let def_decompressor = self.def_decompressor.clone(); + let rep_codec = self.rep_codec.clone(); + let def_codec = self.def_codec.clone(); let num_rep_values = self.num_rep_values; let num_def_values = self.num_def_values; + let max_rep = self.max_rep; + let max_def = self.max_def; async move { let data = data.await?; let mut data_iter = data.into_iter(); - let decompress_levels = |compressed_bytes: Bytes, - decompressor: &Arc, - num_values: u64, - level_type: &str| - -> Result> { - let compressed_buffer = LanceBuffer::from_bytes(compressed_bytes, 1); - let decompressed = decompressor.decompress(compressed_buffer, num_values)?; - match decompressed { - DataBlock::FixedWidth(block) => { - if block.num_values != num_values { - return Err(Error::invalid_input_source(format!( - "Unexpected {} level count after decompression: expected {}, got {}", - level_type, num_values, block.num_values - ) - .into())); - } - if block.bits_per_value != 16 { - return Err(Error::invalid_input_source(format!( - "Unexpected {} level bit width after decompression: expected 16, got {}", - level_type, block.bits_per_value - ) - .into())); + // RLE levels select the smallest validated cache representation; + // everything else expands eagerly to `LazyLevels::Dense`. + let build_levels = |compressed_bytes: Bytes, + codec: &LevelCodec, + num_values: u64, + level_type: &str| + -> Result { + match codec { + LevelCodec::Uncompressed => { + if num_values == 0 { + if !compressed_bytes + .len() + .is_multiple_of(std::mem::size_of::()) + { + return Err(Error::invalid_input_source( + format!( + "Unexpected uncompressed {level_type} level size: {} bytes is not divisible by {}", + compressed_bytes.len(), + std::mem::size_of::() + ) + .into(), + )); + } + } else { + let expected_bytes = expected_level_bytes(num_values, level_type)?; + if compressed_bytes.len() != expected_bytes { + return Err(Error::invalid_input_source( + format!( + "Unexpected uncompressed {level_type} level size: expected {expected_bytes} bytes for {num_values} values, got {}", + compressed_bytes.len() + ) + .into(), + )); + } } - Ok(block.data.borrow_to_typed_slice::()) + let buffer = LanceBuffer::from_bytes(compressed_bytes, 2); + Ok(LazyLevels::Dense(buffer.borrow_to_typed_slice::())) + } + LevelCodec::Rle(decompressor) => { + let frame = LanceBuffer::from_bytes(compressed_bytes, 1); + let runs = decompressor.decode_u16_runs(frame, num_values)?; + LazyLevels::from_rle_runs(runs) + } + LevelCodec::Block(decompressor) => { + let frame = LanceBuffer::from_bytes(compressed_bytes, 1); + let decompressed = decompressor.decompress(frame, num_values)?; + dense_levels_from_block(decompressed, num_values, level_type) } - _ => Err(Error::invalid_input_source(format!( - "Expected fixed-width data block for {} levels", - level_type - ) - .into())), } }; let rep = if has_rep { let rep = data_iter.next().unwrap(); - if let Some(rep_decompressor) = rep_decompressor.as_ref() { - Some(decompress_levels( - rep, - rep_decompressor, - num_rep_values, - "repetition", - )?) - } else { - let rep = LanceBuffer::from_bytes(rep, 2); - let rep = rep.borrow_to_typed_slice::(); - Some(rep) - } + Some(build_levels(rep, &rep_codec, num_rep_values, "repetition")?) } else { None }; let def = if has_def { let def = data_iter.next().unwrap(); - if let Some(def_decompressor) = def_decompressor.as_ref() { - Some(decompress_levels( - def, - def_decompressor, - num_def_values, - "definition", - )?) - } else { - let def = LanceBuffer::from_bytes(def, 2); - let def = def.borrow_to_typed_slice::(); - Some(def) - } + Some(build_levels(def, &def_codec, num_def_values, "definition")?) } else { None }; + validate_complex_all_null_levels(&rep, &def, max_rep, max_def)?; let repdef = Arc::new(CachedComplexAllNullState { rep, def }); self.repdef = Some(repdef.clone()); @@ -944,8 +1623,8 @@ impl StructuralPageScheduler for ComplexAllNullScheduler { def_meaning: self.def_meaning.clone(), max_rep: self.max_rep, max_visible_level: self.max_visible_level, - cursor_row: 0, - cursor_level: 0, + rep_cursor: LevelCursor::default(), + def_run_cursor: RunPosition::default(), }) as Box; let page_load_task = PageLoadTask { decoder_fut: std::future::ready(Ok(decoder)).boxed(), @@ -958,14 +1637,16 @@ impl StructuralPageScheduler for ComplexAllNullScheduler { #[derive(Debug)] pub struct ComplexAllNullPageDecoder { ranges: VecDeque>, - rep: Option>, - def: Option>, + rep: Option, + def: Option, num_rows: u64, def_meaning: Arc<[DefinitionInterpretation]>, max_rep: u16, max_visible_level: u16, - cursor_row: u64, - cursor_level: usize, + /// Monotonic cursor into `rep` tracking the current row's level start. + rep_cursor: LevelCursor, + /// Monotonic run cursor into `def` for `count_le_cursor`. + def_run_cursor: RunPosition, } impl ComplexAllNullPageDecoder { @@ -987,73 +1668,63 @@ impl ComplexAllNullPageDecoder { ranges } - fn take_row(&mut self) -> Result<(Range, u64)> { - let start = self.cursor_level; - let end = if let Some(rep) = &self.rep { - if start >= rep.len() { - return Err(Error::internal( - "Invalid complex all-null layout: repetition buffer too short", - )); - } - if rep[start] != self.max_rep { - return Err(Error::internal( - "Invalid complex all-null layout: row did not start at max repetition level", - )); - } - let mut end = start + 1; - while end < rep.len() && rep[end] != self.max_rep { - end += 1; - } - end - } else { - start + 1 - }; - - let visible = if let Some(def) = &self.def { - if end > def.len() { - return Err(Error::internal( - "Invalid complex all-null layout: definition buffer too short", - )); + /// Level index at which row `target_row` starts, advancing the monotonic + /// repetition cursor. Callers must request non-decreasing `target_row`. + fn seek_row_start(&mut self, target_row: u64) -> Result { + match &self.rep { + Some(rep) => rep.seek_row_start(&mut self.rep_cursor, target_row, self.max_rep), + None => { + // Without repetition every level is its own row. + self.rep_cursor.row = target_row; + self.rep_cursor.level = target_row as usize; + Ok(target_row as usize) } - def[start..end] - .iter() - .filter(|d| **d <= self.max_visible_level) - .count() as u64 - } else { - (end - start) as u64 - }; - - self.cursor_level = end; - self.cursor_row += 1; - Ok((start..end, visible)) + } } - fn skip_to_row(&mut self, target_row: u64) -> Result<()> { - while self.cursor_row < target_row { - self.take_row()?; + /// Number of visible items in the level range `levels` (definition levels + /// `<= max_visible_level`), advancing the monotonic definition cursor. + fn count_visible(&mut self, levels: Range) -> Result<(u64, RunPosition)> { + match &self.def { + Some(def) => { + if levels.end > def.len() { + return Err(Error::internal( + "Invalid complex all-null layout: definition buffer too short", + )); + } + Ok(def.count_le_cursor(&mut self.def_run_cursor, levels, self.max_visible_level)) + } + None => Ok(((levels.end - levels.start) as u64, RunPosition::default())), } - Ok(()) } } impl StructuralPageDecoder for ComplexAllNullPageDecoder { fn drain(&mut self, num_rows: u64) -> Result> { let drained_ranges = self.drain_ranges(num_rows); - let mut level_slices: Vec> = Vec::new(); + let mut level_slices: Vec = Vec::with_capacity(drained_ranges.len()); let mut visible_items_total = 0; + // Each row range is one contiguous level slice `[start_row_level, + // end_row_level)`, so we seek both boundaries and count its visibility at + // once rather than per row. The cursors only move forward, so locating and + // counting all requested ranges visits each intervening run at most once. for range in drained_ranges { - self.skip_to_row(range.start)?; - for _ in range.start..range.end { - let (level_range, visible) = self.take_row()?; - visible_items_total += visible; - if let Some(last) = level_slices.last_mut() - && last.end == level_range.start - { - last.end = level_range.end; - continue; - } - level_slices.push(level_range); + let level_start = self.seek_row_start(range.start)?; + let rep_run = self.rep_cursor.run; + let level_end = self.seek_row_start(range.end)?; + let (visible_items, def_run) = self.count_visible(level_start..level_end)?; + visible_items_total += visible_items; + if let Some(last) = level_slices.last_mut() + && last.range.end == level_start + { + last.range.end = level_end; + } else { + level_slices.push(LevelSlice { + range: level_start..level_end, + rep_run, + def_run, + }); } } @@ -1074,27 +1745,49 @@ impl StructuralPageDecoder for ComplexAllNullPageDecoder { /// We use `level_slices` to slice into `rep` and `def` and create rep/def buffers /// for the null data. +#[derive(Debug, Clone)] +struct LevelSlice { + range: Range, + rep_run: RunPosition, + def_run: RunPosition, +} + +#[derive(Clone, Copy)] +enum LevelKind { + Repetition, + Definition, +} + +impl LevelSlice { + fn run(&self, kind: LevelKind) -> RunPosition { + match kind { + LevelKind::Repetition => self.rep_run, + LevelKind::Definition => self.def_run, + } + } +} + #[derive(Debug)] pub struct DecodeComplexAllNullTask { - level_slices: Vec>, + level_slices: Vec, visible_items_total: u64, - rep: Option>, - def: Option>, + rep: Option, + def: Option, def_meaning: Arc<[DefinitionInterpretation]>, max_visible_level: u16, } impl DecodeComplexAllNullTask { - fn decode_level(&self, levels: &Option>) -> Option> { + fn decode_level(&self, levels: &Option, kind: LevelKind) -> Option> { levels.as_ref().map(|levels| { let num_levels = self .level_slices .iter() - .map(|range| range.end - range.start) + .map(|slice| slice.range.end - slice.range.start) .sum(); let mut referenced_levels = Vec::with_capacity(num_levels); - for range in &self.level_slices { - referenced_levels.extend(levels[range.start..range.end].iter().copied()); + for slice in &self.level_slices { + levels.extend_into(slice.range.clone(), slice.run(kind), &mut referenced_levels); } referenced_levels }) @@ -1103,8 +1796,8 @@ impl DecodeComplexAllNullTask { impl DecodePageTask for DecodeComplexAllNullTask { fn decode(self: Box) -> Result { - let rep = self.decode_level(&self.rep); - let def = self.decode_level(&self.def); + let rep = self.decode_level(&self.rep, LevelKind::Repetition); + let def = self.decode_level(&self.def, LevelKind::Definition); // If there are definition levels there may be empty / null lists which are not visible // in the items array. We need to account for that here to figure out how many values @@ -1206,121 +1899,18 @@ struct MiniBlockSchedulerDictionary { num_dictionary_items: u64, } -/// Individual block metadata within a MiniBlock repetition index. -#[derive(Debug)] -struct MiniBlockRepIndexBlock { - // The index of the first row that starts after the beginning of this block. If the block - // has a preamble this will be the row after the preamble. If the block is entirely preamble - // then this will be a row that starts in some future block. - first_row: u64, - // The number of rows in the block, including the trailer but not the preamble. - // Can be 0 if the block is entirely preamble - starts_including_trailer: u64, - // Whether the block has a preamble - has_preamble: bool, - // Whether the block has a trailer - has_trailer: bool, -} - -impl DeepSizeOf for MiniBlockRepIndexBlock { - fn deep_size_of_children(&self, _context: &mut Context) -> usize { - 0 - } -} - -/// Repetition index for MiniBlock encoding. -/// -/// Stores block-level offset information to enable efficient random -/// access to nested data structures within mini-blocks. -#[derive(Debug)] -struct MiniBlockRepIndex { - blocks: Vec, -} - -impl DeepSizeOf for MiniBlockRepIndex { - fn deep_size_of_children(&self, context: &mut Context) -> usize { - self.blocks.deep_size_of_children(context) - } -} - -impl MiniBlockRepIndex { - /// Decode repetition index from chunk metadata using default values. - /// - /// This creates a repetition index where each chunk has no partial values - /// and no trailers, suitable for simple sequential data layouts. - pub fn default_from_chunks(chunks: &[ChunkMeta]) -> Self { - let mut blocks = Vec::with_capacity(chunks.len()); - let mut offset: u64 = 0; - - for c in chunks { - blocks.push(MiniBlockRepIndexBlock { - first_row: offset, - starts_including_trailer: c.num_values, - has_preamble: false, - has_trailer: false, - }); - - offset += c.num_values; - } - - Self { blocks } - } - - /// Decode repetition index from raw bytes in little-endian format. - /// - /// The bytes should contain u64 values arranged in groups of `stride` elements, - /// where the first two values of each group represent ends_count and partial_count. - /// Returns an empty index if no bytes are provided. - pub fn decode_from_bytes(rep_bytes: &[u8], stride: usize) -> Self { - // Convert bytes to u64 slice, handling alignment automatically - let buffer = crate::buffer::LanceBuffer::from(rep_bytes.to_vec()); - let u64_slice = buffer.borrow_to_typed_slice::(); - let n = u64_slice.len() / stride; - - let mut blocks = Vec::with_capacity(n); - let mut chunk_has_preamble = false; - let mut offset: u64 = 0; - - // Extract first two values from each block: ends_count and partial_count - for i in 0..n { - let base_idx = i * stride; - let ends = u64_slice[base_idx]; - let partial = u64_slice[base_idx + 1]; - - let has_trailer = partial > 0; - // Convert branches to arithmetic for better compiler optimization - let starts_including_trailer = - ends + (has_trailer as u64) - (chunk_has_preamble as u64); - - blocks.push(MiniBlockRepIndexBlock { - first_row: offset, - starts_including_trailer, - has_preamble: chunk_has_preamble, - has_trailer, - }); - - chunk_has_preamble = has_trailer; - offset += starts_including_trailer; - } - - Self { blocks } - } -} - /// State that is loaded once and cached for future lookups #[derive(Debug)] struct MiniBlockCacheableState { - /// Metadata that describes each chunk in the page - chunk_meta: Vec, - /// The decoded repetition index - rep_index: MiniBlockRepIndex, + /// Compact per-chunk index (byte ranges + row/item mapping) for the page + chunk_index: MiniBlockChunkIndex, /// The dictionary for the page, if any dictionary: Option>, } impl DeepSizeOf for MiniBlockCacheableState { fn deep_size_of_children(&self, context: &mut Context) -> usize { - self.rep_index.deep_size_of_children(context) + self.chunk_index.deep_size_of_children(context) + self .dictionary .as_ref() @@ -1465,19 +2055,14 @@ impl MiniBlockScheduler { } fn lookup_chunks(&self, chunk_indices: &[usize]) -> Vec { - let page_meta = self.page_meta.as_ref().unwrap(); + let chunk_index = &self.page_meta.as_ref().unwrap().chunk_index; chunk_indices .iter() - .map(|&chunk_idx| { - let chunk_meta = &page_meta.chunk_meta[chunk_idx]; - let bytes_start = chunk_meta.offset_bytes; - let bytes_end = bytes_start + chunk_meta.chunk_size_bytes; - LoadedChunk { - byte_range: bytes_start..bytes_end, - items_in_chunk: chunk_meta.num_values, - chunk_idx, - data: LanceBuffer::empty(), - } + .map(|&chunk_idx| LoadedChunk { + byte_range: chunk_index.byte_range(chunk_idx), + items_in_chunk: chunk_index.items_in_chunk(chunk_idx), + chunk_idx, + data: LanceBuffer::empty(), }) .collect() } @@ -1582,9 +2167,12 @@ impl ChunkInstructions { // // The output will be a set of `ChunkInstructions` which tell us how to read from the chunks fn schedule_instructions( - rep_index: &MiniBlockRepIndex, + chunk_index: &MiniBlockChunkIndex, user_ranges: &[Range], ) -> Vec { + // Bind the per-page chunk count once; re-deriving it each iteration + // costs a width match plus a length read. + let num_chunks = chunk_index.num_chunks(); // This is an in-exact capacity guess but pretty good. The actual capacity can be // smaller if instructions are merged. It can be larger if there are multiple instructions // per row which can happen with lists. @@ -1596,43 +2184,30 @@ impl ChunkInstructions { // Need to find the first chunk with a first row >= user_range.start. If there are // multiple chunks with the same first row we need to take the first one. - let mut block_index = match rep_index - .blocks - .binary_search_by_key(&user_range.start, |block| block.first_row) - { - Ok(idx) => { - // Slightly tricky case, we may need to walk backwards a bit to make sure we - // are grabbing first eligible chunk - let mut idx = idx; - while idx > 0 && rep_index.blocks[idx - 1].first_row == user_range.start { - idx -= 1; - } - idx - } - // Easy case. idx is greater, and idx - 1 is smaller, so idx - 1 contains the start - Err(idx) => idx - 1, - }; + let mut block_index = chunk_index.find_chunk(user_range.start); - let mut to_skip = user_range.start - rep_index.blocks[block_index].first_row; + let mut to_skip = user_range.start - chunk_index.first_row(block_index); while rows_needed > 0 || need_preamble { // Check if we've gone past the last block (should not happen) - if block_index >= rep_index.blocks.len() { + if block_index >= num_chunks { log::warn!( - "schedule_instructions inconsistency: block_index >= rep_index.blocks.len(), exiting early" + "schedule_instructions inconsistency: block_index >= num_chunks, exiting early" ); break; } - let chunk = &rep_index.blocks[block_index]; - let rows_avail = chunk.starts_including_trailer.saturating_sub(to_skip); + let starts_including_trailer = chunk_index.rows_in_chunk(block_index); + let has_preamble = chunk_index.has_preamble(block_index); + let has_trailer = chunk_index.has_trailer(block_index); + let rows_avail = starts_including_trailer.saturating_sub(to_skip); // Handle blocks that are entirely preamble (rows_avail = 0) // These blocks have no rows to take but may have a preamble we need // We only look for preamble if to_skip == 0 (we're not skipping rows) if rows_avail == 0 && to_skip == 0 { // Only process if this chunk has a preamble we need - if chunk.has_preamble && need_preamble { + if has_preamble && need_preamble { chunk_instructions.push(Self { chunk_idx: block_index, preamble: PreambleAction::Take, @@ -1641,14 +2216,12 @@ impl ChunkInstructions { // We still need to look at has_trailer to distinguish between "all preamble // and row ends at end of chunk" and "all preamble and row bleeds into next // chunk". Both cases will have 0 rows available. - take_trailer: chunk.has_trailer, + take_trailer: has_trailer, }); // Only set need_preamble = false if the chunk has at least one row, // Or we are reaching the last block, // Otherwise, the chunk is entirely preamble and we need the next chunk's preamble too - if chunk.starts_including_trailer > 0 - || block_index == rep_index.blocks.len() - 1 - { + if starts_including_trailer > 0 || block_index == num_chunks - 1 { need_preamble = false; } } @@ -1663,7 +2236,7 @@ impl ChunkInstructions { if rows_avail == 0 && to_skip > 0 { // This block doesn't have enough rows to skip, move to next block // Adjust to_skip by the number of rows in this block - to_skip -= chunk.starts_including_trailer; + to_skip -= starts_including_trailer; block_index += 1; continue; } @@ -1672,7 +2245,7 @@ impl ChunkInstructions { rows_needed -= rows_to_take; let mut take_trailer = false; - let preamble = if chunk.has_preamble { + let preamble = if has_preamble { if need_preamble { PreambleAction::Take } else { @@ -1683,7 +2256,7 @@ impl ChunkInstructions { }; // Are we taking the trailer? If so, make sure we mark that we need the preamble - if rows_to_take == rows_avail && chunk.has_trailer { + if rows_to_take == rows_avail && has_trailer { take_trailer = true; need_preamble = true; } else { @@ -1707,25 +2280,33 @@ impl ChunkInstructions { // are _adjacent_ (i.e. don't merge "take first row of chunk 0" and "take third row of chunk 0" into "take 2 // rows of chunk 0 starting at 0") if user_ranges.len() > 1 { - // TODO: Could probably optimize this allocation away - let mut merged_instructions = Vec::with_capacity(chunk_instructions.len()); - let mut instructions_iter = chunk_instructions.into_iter(); - merged_instructions.push(instructions_iter.next().unwrap()); - for instruction in instructions_iter { - let last = merged_instructions.last_mut().unwrap(); - if last.chunk_idx == instruction.chunk_idx - && last.rows_to_take + last.rows_to_skip == instruction.rows_to_skip - { - last.rows_to_take += instruction.rows_to_take; - last.take_trailer |= instruction.take_trailer; + // Merge adjacent instructions in place. `write` indexes the last + // retained instruction; each following instruction is either folded + // into it (contiguous within the same chunk) or compacted forward. + let mut write = 0; + for read in 1..chunk_instructions.len() { + let merges = { + let last = &chunk_instructions[write]; + let candidate = &chunk_instructions[read]; + last.chunk_idx == candidate.chunk_idx + && last.rows_to_take + last.rows_to_skip == candidate.rows_to_skip + }; + if merges { + let rows_to_take = chunk_instructions[read].rows_to_take; + let take_trailer = chunk_instructions[read].take_trailer; + let last = &mut chunk_instructions[write]; + last.rows_to_take += rows_to_take; + last.take_trailer |= take_trailer; } else { - merged_instructions.push(instruction); + write += 1; + if write != read { + chunk_instructions.swap(write, read); + } } } - merged_instructions - } else { - chunk_instructions + chunk_instructions.truncate(write + 1); } + chunk_instructions } fn drain_from_instruction( @@ -1836,6 +2417,160 @@ impl<'a> Iterator for WordsIter<'a> { } } +/// Per-chunk leaf value-count analysis derived from the metadata words. +/// +/// `values_per_chunk` is the count shared by every non-last chunk (meaningful +/// when `uniform`), and `last_chunk_values` is the final chunk's count. +struct FlatValueCounts { + logs: Vec, + uniform: bool, + values_per_chunk: u64, + last_chunk_values: u64, +} + +fn analyze_value_counts(words: &Words, items_in_page: u64) -> Result { + let num_chunks = words.len(); + let logs = words.iter().map(|w| (w & 0x0F) as u8).collect::>(); + let mut counted = 0u64; + for (chunk_index, &log) in logs.iter().take(num_chunks.saturating_sub(1)).enumerate() { + if log == 0 { + return Err(Error::corrupt_file_named( + "miniblock_metadata", + format!( + "non-final chunk {chunk_index} of {num_chunks} has invalid log_num_values=0" + ), + )); + } + counted = counted.checked_add(1u64 << log).ok_or_else(|| { + Error::corrupt_file_named( + "miniblock_metadata", + format!( + "value count overflow at chunk {chunk_index}: counted_values={counted}, \ + log_num_values={log}, items_in_page={items_in_page}" + ), + ) + })?; + } + let last_chunk_values = items_in_page.checked_sub(counted).ok_or_else(|| { + Error::corrupt_file_named( + "miniblock_metadata", + format!( + "non-final chunks account for counted_values={counted}, exceeding \ + items_in_page={items_in_page}" + ), + ) + })?; + if let Some(&last_log) = logs.last() + && last_log != 0 + && (1u64 << last_log) != last_chunk_values + { + return Err(Error::corrupt_file_named( + "miniblock_metadata", + format!( + "final chunk log_num_values={last_log} does not match \ + last_chunk_values={last_chunk_values}: counted_values={counted}, \ + items_in_page={items_in_page}" + ), + )); + } + let uniform = num_chunks <= 1 || logs[..num_chunks - 1].iter().all(|&log| log == logs[0]); + // A single-chunk page has no "non-last" chunk to derive a stride from; use the + // page item count (min 1 so it stays a valid divisor in `find_chunk`). + let values_per_chunk = if num_chunks <= 1 { + items_in_page.max(1) + } else { + 1u64 << logs[0] + }; + Ok(FlatValueCounts { + logs, + uniform, + values_per_chunk, + last_chunk_values, + }) +} + +/// Iterator over per-chunk value counts for a non-uniform flat page. Non-last +/// chunks yield `1 << log`; the last yields the validated remaining item count. +fn flat_value_counts_iter(logs: &[u8], last_chunk_values: u64) -> impl Iterator + '_ { + let num_chunks = logs.len(); + (0..num_chunks).map(move |i| { + if i + 1 < num_chunks { + 1u64 << logs[i] + } else { + last_chunk_values + } + }) +} + +/// Builds the compact per-chunk index from the metadata words and, for nested +/// pages, the raw repetition-index bytes. The row axis is picked by page shape: +/// `UniformFlat` when all non-last chunks share a value count (fixed-width / +/// bitpacking), `Flat` for non-uniform flat pages (RLE / FSST), else `Nested`. +fn build_chunk_index( + words: &Words, + items_in_page: u64, + base: u64, + data_buf_size: u64, + rep_index_bytes: Option<&[u8]>, + repetition_index_depth: u16, +) -> Result { + let num_chunks = words.len(); + // Validate item counts before byte sizes because both share a metadata word, + // and an invalid count must not reach the final-chunk subtraction. + let value_counts = analyze_value_counts(words, items_in_page)?; + + // Each chunk stores `(divided_bytes + 1) * MINIBLOCK_ALIGNMENT` bytes, so the + // deltas are the chunk sizes and their grand total is the data buffer size. + let byte_starts = PrefixSums::from_deltas( + words + .iter() + .map(|word| ((word >> 4) as u64 + 1) * MINIBLOCK_ALIGNMENT as u64), + num_chunks, + data_buf_size, + ); + + // Nested pages track rows via the repetition index and keep leaf item counts + // separately; flat pages have row == value index, so value counts are rows. + let rows = if let Some(rep_index_data) = rep_index_bytes { + assert!(rep_index_data.len() % 8 == 0); + let stride = repetition_index_depth as usize + 1; + let (row_starts, has_trailer) = parse_nested_rep(rep_index_data, stride); + let item_counts = if value_counts.uniform { + ItemCounts::Uniform { + values_per_chunk: value_counts.values_per_chunk, + last_chunk_values: value_counts.last_chunk_values, + } + } else { + ItemCounts::PerChunkLog { + logs: value_counts.logs, + last_chunk_values: value_counts.last_chunk_values, + } + }; + RowMapping::Nested { + row_starts, + has_trailer, + item_counts, + } + } else { + if value_counts.uniform { + RowMapping::UniformFlat { + values_per_chunk: value_counts.values_per_chunk, + last_chunk_values: value_counts.last_chunk_values, + num_chunks, + } + } else { + let value_starts = PrefixSums::from_deltas( + flat_value_counts_iter(&value_counts.logs, value_counts.last_chunk_values), + num_chunks, + items_in_page, + ); + RowMapping::Flat { value_starts } + } + }; + + Ok(MiniBlockChunkIndex::new(base, byte_starts, rows)) +} + impl StructuralPageScheduler for MiniBlockScheduler { fn initialize<'a>( &'a mut self, @@ -1845,7 +2580,8 @@ impl StructuralPageScheduler for MiniBlockScheduler { // we may also need to fetch the repetition index. Here, we gather what buffers we // need. let (meta_buf_position, meta_buf_size) = self.buffer_offsets_and_sizes[0]; - let value_buf_position = self.buffer_offsets_and_sizes[1].0; + let base = self.buffer_offsets_and_sizes[1].0; + let data_buf_size = self.buffer_offsets_and_sizes[1].1; let mut bufs_needed = 1; if self.dictionary.is_some() { bufs_needed += 1; @@ -1874,65 +2610,31 @@ impl StructuralPageScheduler for MiniBlockScheduler { let dictionary_bytes = self.dictionary.as_ref().and_then(|_| buffers.next()); let rep_index_bytes = buffers.next(); - // Parse the metadata and build the chunk meta let words = Words::from_bytes(meta_bytes, self.has_large_chunk)?; - let mut chunk_meta = Vec::with_capacity(words.len()); - - let mut rows_counter = 0; - let mut offset_bytes = value_buf_position; - for (word_idx, word) in words.iter().enumerate() { - let log_num_values = word & 0x0F; - let divided_bytes = word >> 4; - let num_bytes = (divided_bytes as usize + 1) * MINIBLOCK_ALIGNMENT; - debug_assert!(num_bytes > 0); - let num_values = if word_idx < words.len() - 1 { - debug_assert!(log_num_values > 0); - 1 << log_num_values - } else { - debug_assert!( - log_num_values == 0 - || (1 << log_num_values) == (self.items_in_page - rows_counter) - ); - self.items_in_page - rows_counter - }; - rows_counter += num_values; - - chunk_meta.push(ChunkMeta { - num_values, - chunk_size_bytes: num_bytes as u64, - offset_bytes, - }); - offset_bytes += num_bytes as u64; - } - - // Build the repetition index - let rep_index = if let Some(rep_index_data) = rep_index_bytes { - assert!(rep_index_data.len() % 8 == 0); - let stride = self.repetition_index_depth as usize + 1; - MiniBlockRepIndex::decode_from_bytes(&rep_index_data, stride) - } else { - MiniBlockRepIndex::default_from_chunks(&chunk_meta) - }; - - let mut page_meta = MiniBlockCacheableState { - chunk_meta, - rep_index, - dictionary: None, - }; + let chunk_index = build_chunk_index( + &words, + self.items_in_page, + base, + data_buf_size, + rep_index_bytes.as_deref(), + self.repetition_index_depth, + )?; // decode dictionary - if let Some(ref mut dictionary) = self.dictionary { + let dictionary = if let Some(ref mut dictionary) = self.dictionary { let dictionary_data = dictionary_bytes.unwrap(); - page_meta.dictionary = - Some(Arc::new(dictionary.dictionary_decompressor.decompress( - LanceBuffer::from_bytes( - dictionary_data, - dictionary.dictionary_data_alignment, - ), - dictionary.num_dictionary_items, - )?)); + Some(Arc::new(dictionary.dictionary_decompressor.decompress( + LanceBuffer::from_bytes(dictionary_data, dictionary.dictionary_data_alignment), + dictionary.num_dictionary_items, + )?)) + } else { + None }; - let page_meta = Arc::new(page_meta); + + let page_meta = Arc::new(MiniBlockCacheableState { + chunk_index, + dictionary, + }); self.page_meta = Some(page_meta.clone()); Ok(page_meta as Arc) } @@ -1958,7 +2660,7 @@ impl StructuralPageScheduler for MiniBlockScheduler { let page_meta = self.page_meta.as_ref().unwrap(); let chunk_instructions = - ChunkInstructions::schedule_instructions(&page_meta.rep_index, ranges); + ChunkInstructions::schedule_instructions(&page_meta.chunk_index, ranges); debug_assert_eq!( num_rows, @@ -2294,7 +2996,7 @@ impl FullZipScheduler { num_rows, bits_per_offset, bits_per_offset, - ))) + )?)) } } } @@ -2709,7 +3411,7 @@ impl VariableFullZipDecoder { num_rows: u64, in_bits_per_length: u8, out_bits_per_offset: u8, - ) -> Self { + ) -> Result { let decompressor = match details.value_decompressor { PerValueDecompressor::Variable(ref d) => d.clone(), _ => unreachable!(), @@ -2754,9 +3456,9 @@ impl VariableFullZipDecoder { // - We could force each decode task to do a full unzip of all the data. Each decode task now // has to do more work but the work is all fused. // - We could just try doing this work on the decode thread and see if it is a problem. - decoder.unzip(data, in_bits_per_length, out_bits_per_offset, num_rows); + decoder.unzip(data, in_bits_per_length, out_bits_per_offset, num_rows)?; - decoder + Ok(decoder) } fn slice_batch_data_and_rebase_offsets_typed( @@ -2831,28 +3533,33 @@ impl VariableFullZipDecoder { } } - unsafe fn parse_length(data: &[u8], bits_per_offset: u8) -> u64 { - match bits_per_offset { - 8 => *data.get_unchecked(0) as u64, - 16 => u16::from_le_bytes([*data.get_unchecked(0), *data.get_unchecked(1)]) as u64, - 32 => u32::from_le_bytes([ - *data.get_unchecked(0), - *data.get_unchecked(1), - *data.get_unchecked(2), - *data.get_unchecked(3), - ]) as u64, - 64 => u64::from_le_bytes([ - *data.get_unchecked(0), - *data.get_unchecked(1), - *data.get_unchecked(2), - *data.get_unchecked(3), - *data.get_unchecked(4), - *data.get_unchecked(5), - *data.get_unchecked(6), - *data.get_unchecked(7), - ]), - _ => unreachable!(), + /// Reads a single length prefix from the front of `data`. + /// + /// The bytes come from the file. A page whose item walk ends with a partial + /// trailing item leaves fewer than `bits_per_offset / 8` bytes here, so this + /// is bounds checked and reports a corrupt file rather than reading past the + /// end of the buffer. + fn parse_length(data: &[u8], bits_per_offset: u8) -> Result { + let width = bits_per_offset as usize / 8; + if data.len() < width { + return Err(Error::corrupt_file_named( + "variable_full_zip", + format!( + "truncated length prefix: {} byte(s) remain in the page buffer but a \ + {}-bit length prefix requires {}", + data.len(), + bits_per_offset, + width + ), + )); } + Ok(match bits_per_offset { + 8 => data[0] as u64, + 16 => u16::from_le_bytes(data[..2].try_into().unwrap()) as u64, + 32 => u32::from_le_bytes(data[..4].try_into().unwrap()) as u64, + 64 => u64::from_le_bytes(data[..8].try_into().unwrap()), + _ => unreachable!(), + }) } fn unzip( @@ -2861,7 +3568,7 @@ impl VariableFullZipDecoder { in_bits_per_length: u8, out_bits_per_offset: u8, num_rows: u64, - ) { + ) -> Result<()> { // This undercounts if there are lists but, at this point, we don't really know how many items we have let mut rep = Vec::with_capacity(num_rows as usize); let mut def = Vec::with_capacity(num_rows as usize); @@ -2912,9 +3619,7 @@ impl VariableFullZipDecoder { if ctrl_desc.is_visible { visible_item_count += 1; if ctrl_desc.is_valid_item { - // Safety: Data should have at least bytes_per_length bytes remaining - debug_assert!(databuf.len() >= bytes_per_length); - let length = unsafe { Self::parse_length(databuf, in_bits_per_length) }; + let length = Self::parse_length(databuf, in_bits_per_length)?; match out_bits_per_offset { 32 => offsets_data .extend_from_slice(&(current_offset as u32).to_le_bytes()), @@ -2950,6 +3655,7 @@ impl VariableFullZipDecoder { self.def = ScalarBuffer::from(def); self.data = LanceBuffer::from(unzipped_data); self.offsets = LanceBuffer::from(offsets_data); + Ok(()) } } @@ -3066,15 +3772,31 @@ struct FixedFullZipDecodeTask { impl DecodePageTask for FixedFullZipDecodeTask { fn decode(self: Box) -> Result { - // Multiply by 2 to make a stab at the size of the output buffer (which will be decompressed and thus bigger) - let estimated_size_bytes = self - .data - .iter() - .map(|task_item| task_item.data.data_size() as usize) - .sum::() - * 2; - let mut data_builder = - DataBlockBuilder::with_capacity_estimate(estimated_size_bytes as u64); + let estimated_size_bytes = if self.details.ctrl_word_parser.bytes_per_word() == 0 { + let PerValueDecompressor::Fixed(decompressor) = &self.details.value_decompressor else { + return Err(Error::internal( + "FixedFullZipDecodeTask requires a fixed-width decompressor", + )); + }; + decompressor + .decoded_size_bytes(self.num_rows as u64) + .unwrap_or_else(|| { + self.data + .iter() + .map(|task_item| task_item.data.data_size()) + .sum::() + * 2 + }) + } else { + // Rep/def levels can suppress values, so the exact output size is not known + // until they are decoded. Keep the existing conservative estimate. + self.data + .iter() + .map(|task_item| task_item.data.data_size()) + .sum::() + * 2 + }; + let mut data_builder = DataBlockBuilder::with_capacity_estimate(estimated_size_bytes); if self.details.ctrl_word_parser.bytes_per_word() == 0 { // Fast path, no need to unzip because there is no rep/def @@ -3090,7 +3812,7 @@ impl DecodePageTask for FixedFullZipDecodeTask { }; debug_assert_eq!(fixed_data.num_values, task_item.rows_in_buf); let decompressed = decompressor.decompress(fixed_data, task_item.rows_in_buf)?; - data_builder.append(&decompressed, 0..task_item.rows_in_buf); + data_builder.append(&decompressed, 0..task_item.rows_in_buf)?; } let unraveler = RepDefUnraveler::new( @@ -3154,7 +3876,7 @@ impl DecodePageTask for FixedFullZipDecodeTask { unreachable!() }; let decompressed = decompressor.decompress(fixed_data, visible_items)?; - data_builder.append(&decompressed, 0..visible_items); + data_builder.append(&decompressed, 0..visible_items)?; } let repetition = if rep.is_empty() { None } else { Some(rep) }; @@ -3349,6 +4071,16 @@ impl StructuralPrimitiveFieldScheduler { mini_block, decompressors, )?), + Layout::SparseLayout(sparse_layout) => { + Box::new(sparse::SparseStructuralScheduler::try_new( + &page_info.buffer_offsets_and_sizes, + page_info.priority, + page_info.num_rows, + target_field.data_type(), + sparse_layout, + decompressors, + )?) + } Layout::FullZipLayout(full_zip) => { let mut scheduler = FullZipScheduler::try_new( &page_info.buffer_offsets_and_sizes, @@ -3381,25 +4113,22 @@ impl StructuralPrimitiveFieldScheduler { { Box::new(SimpleAllNullScheduler::default()) as Box } else { - let rep_decompressor = constant_layout - .rep_compression - .as_ref() - .map(|encoding| decompressors.create_block_decompressor(encoding)) - .transpose()? - .map(Arc::from); - - let def_decompressor = constant_layout - .def_compression - .as_ref() - .map(|encoding| decompressors.create_block_decompressor(encoding)) - .transpose()? - .map(Arc::from); + // RLE levels select a validated cache representation; other + // block compressions keep flowing through the eager decompressor. + let rep_codec = LevelCodec::try_new( + constant_layout.rep_compression.as_ref(), + decompressors, + )?; + let def_codec = LevelCodec::try_new( + constant_layout.def_compression.as_ref(), + decompressors, + )?; Box::new(ComplexAllNullScheduler::new( page_info.buffer_offsets_and_sizes.clone(), def_meaning.into(), - rep_decompressor, - def_decompressor, + rep_codec, + def_codec, constant_layout.num_rep_values, constant_layout.num_def_values, )) as Box @@ -3512,6 +4241,15 @@ impl CacheKey for FieldDataCacheKey { fn type_name() -> &'static str { "FieldData" } + + fn schema() -> CacheKeySchema { + CacheKeySchema::new("lance.encoding.logical.primitive.field-data-key", 1) + } + + fn write_key(&self, builder: &mut KeyBuilder) { + builder.write_u32(self.column_index); + builder.write_str(&self.view_tag); + } } impl StructuralFieldScheduler for StructuralPrimitiveFieldScheduler { @@ -3576,25 +4314,34 @@ impl StructuralCompositeDecodeArrayTask { fn restore_validity( array: Arc, unraveler: &mut CompositeRepDefUnraveler, - ) -> Arc { - let validity = unraveler.unravel_validity(array.len()); + ) -> Result> { + let validity = unraveler.unravel_validity(array.len())?; let Some(validity) = validity else { - return array; + return Ok(array); }; if array.data_type() == &DataType::Null { // We unravel from a null array but we don't add the null buffer because arrow-rs doesn't like it - return array; + return Ok(array); + } + if validity.len() != array.len() { + return Err(Error::invalid_input_source( + format!( + "Structural validity has {} entries for an array with {} values", + validity.len(), + array.len() + ) + .into(), + )); } - assert_eq!(validity.len(), array.len()); - // SAFETY: We've should have already asserted the buffers are all valid, we are just - // adding null buffers to the array here - make_array(unsafe { + // SAFETY: The array buffers have already been validated and the null buffer length + // matches the array. We are only attaching the null buffer here. + Ok(make_array(unsafe { array .to_data() .into_builder() .nulls(Some(validity)) .build_unchecked() - }) + })) } } @@ -3620,7 +4367,7 @@ impl StructuralDecodeArrayTask for StructuralCompositeDecodeArrayTask { let array = arrow_select::concat::concat(&array_refs)?; let mut repdef = CompositeRepDefUnraveler::new(unravelers); - let array = Self::restore_validity(array, &mut repdef); + let array = Self::restore_validity(array, &mut repdef)?; Ok(DecodedArray { array, @@ -3749,19 +4496,125 @@ const MINIBLOCK_ALIGNMENT: usize = 8; /// TODO: We should concatenate metadata buffers from all pages into a single buffer /// at (roughly) the end of the file so there is, at most, one read per column of /// metadata per file. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum MiniblockChunkSize { + U16, + U32, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum ComplexNullEncoding { + RawLevels, + CompressedLevels, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum FixedWidthDictionaryEncoding { + Exclude64Bit, + Include64Bit, +} + +trait PrimitivePageEncodingBehavior: Send + Sync + Debug { + fn validate_field(&self, _field: &Field, _metadata: &HashMap) -> Result<()> { + Ok(()) + } + + fn try_plan_pages( + &self, + _ctx: &PrimitivePlanContext<'_>, + _arrays: &[ArrayRef], + _normalized: &NormalizedStructuralPlan, + _row_number: u64, + _num_rows: u64, + _num_values: u64, + ) -> Result>> { + Ok(None) + } + + fn try_encode_page( + &self, + _ctx: &PrimitiveEncodeContext, + page: PrimitivePageData, + ) -> Result { + Ok(PrimitiveEncodeAttempt::Unhandled(page)) + } +} + +/// One executable primitive-page behavior selected by an exact file +/// composition. +#[derive(Debug, Clone)] +pub struct PrimitivePageEncoding { + behavior: Arc, +} + +impl PrimitivePageEncoding { + /// Reject an explicit request for sparse structural encoding. + pub fn reject_sparse() -> Self { + Self { + behavior: Arc::new(RejectSparsePrimitiveEncoding), + } + } + + /// Encode constant non-null values as a constant page when applicable. + pub fn constant() -> Self { + Self { + behavior: Arc::new(ConstantPrimitiveEncoding), + } + } + + /// Plan and encode sparse structural pages when applicable. + pub fn sparse(compression: Arc) -> Self { + Self { + behavior: Arc::new(SparsePrimitiveEncoding { compression }), + } + } + + /// Encode dense pages with the original u16 miniblock grammar. + pub fn dense_u16(compression: Arc) -> Self { + Self { + behavior: Arc::new(DenseU16PrimitiveEncoding { compression }), + } + } + + /// Encode dense pages with the u32 miniblock grammar. + pub fn dense_u32(compression: Arc) -> Self { + Self { + behavior: Arc::new(DenseU32PrimitiveEncoding { compression }), + } + } +} + +#[derive(Debug)] +struct RejectSparsePrimitiveEncoding; + +#[derive(Debug)] +struct ConstantPrimitiveEncoding; + +#[derive(Debug)] +struct SparsePrimitiveEncoding { + compression: Arc, +} + +#[derive(Debug)] +struct DenseU16PrimitiveEncoding { + compression: Arc, +} + +#[derive(Debug)] +struct DenseU32PrimitiveEncoding { + compression: Arc, +} + pub struct PrimitiveStructuralEncoder { // Accumulates arrays until we have enough data to justify a disk page accumulation_queue: AccumulationQueue, keep_original_array: bool, - support_large_chunk: bool, accumulated_repdefs: Vec, - // The compression strategy we will use to compress the data - compression_strategy: Arc, + page_encodings: Arc<[PrimitivePageEncoding]>, column_index: u32, field: Field, encoding_metadata: Arc>, - version: LanceFileVersion, } struct CompressedLevelsChunk { @@ -3787,18 +4640,38 @@ struct DictEncodingBudget { max_encoded_size: usize, } -// A primitive page after optional structural splitting. +enum PrimitivePageStructure { + Dense { + repdef: SerializedRepDefs, + single_row_miniblock_repdef_levels: Option, + }, + Sparse { + plan: sparse::SparseStructuralPlan, + prepared_values: Option, + }, +} + +// A primitive page after structural encoding selection and optional dense splitting. struct PrimitivePageData { // Arrow leaf arrays that contain this page's visible values. arrays: Vec, - // Repetition / definition levels aligned to this page. - repdef: SerializedRepDefs, + // Structural representation aligned to this page. + structure: PrimitivePageStructure, // Top-level row number of the first row in this page. row_number: u64, // Number of top-level rows in this page. num_rows: u64, - // Present when one top-level row is too large for one miniblock rep/def chunk. - unsplittable_miniblock_levels: Option, +} + +struct PrimitivePlanContext<'a> { + column_idx: u32, + field: &'a Field, + encoding_metadata: &'a HashMap, +} + +enum PrimitiveEncodeAttempt { + Encoded(EncodedPage), + Unhandled(PrimitivePageData), } // Immutable encoder state shared by per-page encode tasks. @@ -3809,48 +4682,61 @@ struct PrimitivePageData { struct PrimitiveEncodeContext { // Column being encoded. column_idx: u32, - // Logical field metadata for compression/layout selection. field: Field, - // Compression strategy shared across pages. - compression_strategy: Arc, - // Field-level encoding metadata such as structural encoding overrides. encoding_metadata: Arc>, - // Whether miniblock chunks may use the v2.2 large-chunk metadata. - support_large_chunk: bool, - // Lance file version selected by the writer. - version: LanceFileVersion, - // True when the only rep/def information is simple nullable validity. is_simple_validity: bool, - // True when the field has any non-empty rep/def information. has_repdef_info: bool, } impl PrimitiveStructuralEncoder { pub fn try_new( options: &EncodingOptions, - compression_strategy: Arc, + page_encodings: Arc<[PrimitivePageEncoding]>, column_index: u32, field: Field, encoding_metadata: Arc>, ) -> Result { + for page_encoding in page_encodings.iter() { + page_encoding + .behavior + .validate_field(&field, &encoding_metadata)?; + } Ok(Self { accumulation_queue: AccumulationQueue::new( options.cache_bytes_per_column, column_index, options.keep_original_array, ), - support_large_chunk: options.support_large_chunk(), keep_original_array: options.keep_original_array, accumulated_repdefs: Vec::new(), column_index, - compression_strategy, + page_encodings, field, encoding_metadata, - version: options.version, }) } - // TODO: This is a heuristic we may need to tune at some point + fn encode_page( + page_encodings: &[PrimitivePageEncoding], + ctx: &PrimitiveEncodeContext, + mut page: PrimitivePageData, + ) -> Result { + for page_encoding in page_encodings { + match page_encoding.behavior.try_encode_page(ctx, page)? { + PrimitiveEncodeAttempt::Encoded(page) => return Ok(page), + PrimitiveEncodeAttempt::Unhandled(unhandled) => page = unhandled, + } + } + Err(Error::invalid_input_source( + format!( + "No primitive page encoding atom supports field '{}'", + ctx.field.name + ) + .into(), + )) + } + + // TODO: This is a heuristic we may need to tune at some point // // As data gets narrow then the "zipping" process gets too expensive // and we prefer mini-block @@ -3944,7 +4830,7 @@ impl PrimitiveStructuralEncoder { miniblocks: MiniBlockCompressed, rep: Option>, def: Option>, - support_large_chunk: bool, + miniblock_chunk_size: MiniblockChunkSize, ) -> Result { let bytes_rep = rep .as_ref() @@ -3965,7 +4851,10 @@ impl PrimitiveStructuralEncoder { // 2 bytes for the length of each buffer and up to 7 bytes of padding per buffer let max_extra = 9 * num_buffers; let mut data_buffer = Vec::with_capacity(bytes_rep + bytes_def + bytes_data + max_extra); - let chunk_size_bytes = if support_large_chunk { 4 } else { 2 }; + let chunk_size_bytes = match miniblock_chunk_size { + MiniblockChunkSize::U16 => 2, + MiniblockChunkSize::U32 => 4, + }; let mut meta_buffer = Vec::with_capacity(miniblocks.chunks.len() * chunk_size_bytes); let mut rep_iter = rep.map(|r| r.into_iter()); @@ -4007,7 +4896,7 @@ impl PrimitiveStructuralEncoder { data_buffer.extend_from_slice(&bytes_def.to_le_bytes()); } - if support_large_chunk { + if miniblock_chunk_size == MiniblockChunkSize::U32 { for &buffer_size in &chunk.buffer_sizes { data_buffer.extend_from_slice(&buffer_size.to_le_bytes()); } @@ -4052,10 +4941,9 @@ impl PrimitiveStructuralEncoder { } let chunk_bytes = data_buffer.len() - start_pos; - let max_chunk_size = if support_large_chunk { - 1_u64 << 31 // 28 bits of 8-byte words in u32 metadata - } else { - 32 * 1024 // 32KiB limit with u16 metadata + let max_chunk_size = match miniblock_chunk_size { + MiniblockChunkSize::U16 => 32 * 1024, + MiniblockChunkSize::U32 => 1_u64 << 31, }; if chunk_bytes == 0 || chunk_bytes as u64 > max_chunk_size { return Err(Error::internal(format!( @@ -4082,7 +4970,7 @@ impl PrimitiveStructuralEncoder { let divided_bytes_minus_one = (divided_bytes - 1) as u64; let metadata = (divided_bytes_minus_one << 4) | chunk.log_num_values as u64; - if support_large_chunk { + if miniblock_chunk_size == MiniblockChunkSize::U32 { meta_buffer.extend_from_slice(&(metadata as u32).to_le_bytes()); } else { meta_buffer.extend_from_slice(&(metadata as u16).to_le_bytes()); @@ -4278,10 +5166,10 @@ impl PrimitiveStructuralEncoder { repdef: crate::repdef::SerializedRepDefs, row_number: u64, num_rows: u64, - version: LanceFileVersion, + complex_null_encoding: ComplexNullEncoding, compression_strategy: &dyn CompressionStrategy, ) -> Result { - if version.resolve() < LanceFileVersion::V2_2 { + if complex_null_encoding == ComplexNullEncoding::RawLevels { let rep_bytes = if let Some(rep) = repdef.repetition_levels.as_ref() { LanceBuffer::reinterpret_slice(rep.clone()) } else { @@ -4362,7 +5250,7 @@ impl PrimitiveStructuralEncoder { return Ok(None); } let mut validity = BooleanBufferBuilder::new(num_values); - unraveler.unravel_validity(&mut validity); + unraveler.unravel_validity(&mut validity)?; Ok(Some(validity.finish())) } @@ -4575,7 +5463,7 @@ impl PrimitiveStructuralEncoder { row_number: u64, dictionary_data: Option, num_rows: u64, - support_large_chunk: bool, + miniblock_chunk_size: MiniblockChunkSize, ) -> Result { if let DataBlock::AllNull(_null_block) = data { // We should not be using mini-block for all-null. There are other structural @@ -4586,7 +5474,12 @@ impl PrimitiveStructuralEncoder { let num_items = data.num_values(); let compressor = compression_strategy.create_miniblock_compressor(field, &data)?; - let (compressed_data, value_encoding) = compressor.compress(data)?; + let common_chunk_buffers = + u64::from(repdef.rep_slicer().is_some()) + u64::from(repdef.def_slicer().is_some()); + let support_large_chunk = miniblock_chunk_size == MiniblockChunkSize::U32; + let compression_context = + MiniBlockCompressionContext::new(common_chunk_buffers, support_large_chunk, true); + let (compressed_data, value_encoding) = compressor.compress(compression_context, data)?; let max_rep = repdef.def_meaning.iter().filter(|l| l.is_list()).count() as u16; @@ -4635,7 +5528,8 @@ impl PrimitiveStructuralEncoder { .map(|cd| std::mem::take(&mut cd.data)); let serialized = - Self::serialize_miniblocks(compressed_data, rep_data, def_data, support_large_chunk)?; + Self::serialize_miniblocks(compressed_data, rep_data, def_data, miniblock_chunk_size)?; + let has_large_chunk = miniblock_chunk_size == MiniblockChunkSize::U32; // Metadata, Data, Dictionary, (maybe) Repetition Index let mut data = Vec::with_capacity(4); @@ -4664,7 +5558,7 @@ impl PrimitiveStructuralEncoder { Some((dictionary_encoding, num_dictionary_items)), &repdef.def_meaning, num_items, - support_large_chunk, + has_large_chunk, ); Ok(EncodedPage { num_rows, @@ -4683,7 +5577,7 @@ impl PrimitiveStructuralEncoder { None, &repdef.def_meaning, num_items, - support_large_chunk, + has_large_chunk, ); if let Some(rep_index) = rep_index { @@ -5015,7 +5909,7 @@ impl PrimitiveStructuralEncoder { fn should_dictionary_encode( data_block: &DataBlock, field: &Field, - version: LanceFileVersion, + fixed_width_dictionary_encoding: FixedWidthDictionaryEncoding, ) -> Option { const DEFAULT_SAMPLE_SIZE: usize = 4096; const DEFAULT_SAMPLE_UNIQUE_RATIO: f64 = 0.98; @@ -5024,7 +5918,9 @@ impl PrimitiveStructuralEncoder { // estimating the size for other types. match data_block { DataBlock::FixedWidth(fixed) => { - if fixed.bits_per_value == 64 && version < LanceFileVersion::V2_2 { + if fixed.bits_per_value == 64 + && fixed_width_dictionary_encoding == FixedWidthDictionaryEncoding::Exclude64Bit + { return None; } if fixed.bits_per_value != 64 && fixed.bits_per_value != 128 { @@ -5292,33 +6188,37 @@ impl PrimitiveStructuralEncoder { Ok(sliced) } - fn split_structural_pages_for_miniblock_budget( + fn split_pages_for_miniblock_repdef_budget( arrays: Vec, repdef: SerializedRepDefs, - plan: StructuralPagePlan, + budget: MiniBlockRepDefBudget, row_number: u64, num_rows: u64, ) -> Result> { - if plan == StructuralPagePlan::Fits { + if budget == MiniBlockRepDefBudget::WithinBudget { return Ok(vec![PrimitivePageData { arrays, - repdef, + structure: PrimitivePageStructure::Dense { + repdef, + single_row_miniblock_repdef_levels: None, + }, row_number, num_rows, - unsplittable_miniblock_levels: None, }]); } - if let StructuralPagePlan::UnsplittableOverBudget(num_levels) = plan { + if let MiniBlockRepDefBudget::SingleRowOverBudget(num_levels) = budget { return Ok(vec![PrimitivePageData { arrays, - repdef, + structure: PrimitivePageStructure::Dense { + repdef, + single_row_miniblock_repdef_levels: Some(num_levels), + }, row_number, num_rows, - unsplittable_miniblock_levels: Some(num_levels), }]); } - let StructuralPagePlan::Split(splits) = plan else { + let MiniBlockRepDefBudget::RequiresPageSplit(splits) = budget else { unreachable!(); }; @@ -5328,35 +6228,50 @@ impl PrimitiveStructuralEncoder { let repdef = Self::slice_repdef(&repdef, split.level_range); pages.push(PrimitivePageData { arrays, - repdef, + structure: PrimitivePageStructure::Dense { + repdef, + single_row_miniblock_repdef_levels: None, + }, row_number: row_number + split.row_start, num_rows: split.num_rows, - unsplittable_miniblock_levels: None, }); } Ok(pages) } - fn encode_page(ctx: PrimitiveEncodeContext, page: PrimitivePageData) -> Result { + fn encode_dense_page( + ctx: PrimitiveEncodeContext, + page: PrimitivePageData, + compression_strategy: Arc, + miniblock_chunk_size: MiniblockChunkSize, + complex_null_encoding: ComplexNullEncoding, + fixed_width_dictionary_encoding: FixedWidthDictionaryEncoding, + ) -> Result { let PrimitiveEncodeContext { column_idx, field, - compression_strategy, encoding_metadata, - support_large_chunk, - version, is_simple_validity, has_repdef_info, } = ctx; let PrimitivePageData { arrays, - repdef, + structure, row_number, num_rows, - unsplittable_miniblock_levels, } = page; let num_values = arrays.iter().map(|arr| arr.len() as u64).sum(); + let (repdef, single_row_miniblock_repdef_levels) = match structure { + PrimitivePageStructure::Dense { + repdef, + single_row_miniblock_repdef_levels, + } => (repdef, single_row_miniblock_repdef_levels), + PrimitivePageStructure::Sparse { .. } => { + unreachable!("dense atom received sparse page") + } + }; + if num_values == 0 { // This page contains only structural events, such as empty/null list rows. // The existing complex-null layout stores the rep/def stream without value buffers. @@ -5371,7 +6286,7 @@ impl PrimitiveStructuralEncoder { repdef, row_number, num_rows, - version, + complex_null_encoding, compression_strategy.as_ref(), ); } @@ -5403,7 +6318,7 @@ impl PrimitiveStructuralEncoder { repdef, row_number, num_rows, - version, + complex_null_encoding, compression_strategy.as_ref(), ) }; @@ -5422,21 +6337,7 @@ impl PrimitiveStructuralEncoder { let data_block = DataBlock::from_arrays(&arrays, num_values); - if version.resolve() >= LanceFileVersion::V2_2 - && let Some(scalar) = Self::find_constant_scalar(&arrays, leaf_validity.as_ref())? - { - log::debug!( - "Encoding column {} with {} items ({} rows) using constant layout", - column_idx, - num_values, - num_rows - ); - return constant::encode_constant_page( - column_idx, scalar, repdef, row_number, num_rows, - ); - } - - if let Some(num_levels) = unsplittable_miniblock_levels { + if let Some(num_levels) = single_row_miniblock_repdef_levels { let requested_encoding = encoding_metadata .get(STRUCTURAL_ENCODING_META_KEY) .map(|requested| requested.to_lowercase()); @@ -5584,24 +6485,29 @@ impl PrimitiveStructuralEncoder { row_number, Some(dictionary_data_block), num_rows, - support_large_chunk, + miniblock_chunk_size, ); } // Try dictionary encoding first if applicable. If encoding aborts, fall back to the // preferred structural encoding. - let dict_result = Self::should_dictionary_encode(&data_block, &field, version).and_then(|budget| { - log::debug!( - "Encoding column {} with {} items using dictionary encoding (mini-block layout)", - column_idx, - num_values - ); - dict::dictionary_encode( - &data_block, - budget.max_dict_entries, - budget.max_encoded_size, - ) - }); + let dict_result = Self::should_dictionary_encode( + &data_block, + &field, + fixed_width_dictionary_encoding, + ) + .and_then(|budget| { + log::debug!( + "Encoding column {} with {} items using dictionary encoding (mini-block layout)", + column_idx, + num_values + ); + dict::dictionary_encode( + &data_block, + budget.max_dict_entries, + budget.max_encoded_size, + ) + }); if let Some((indices_data_block, dictionary_data_block)) = dict_result { Self::encode_miniblock( @@ -5613,7 +6519,7 @@ impl PrimitiveStructuralEncoder { row_number, Some(dictionary_data_block), num_rows, - support_large_chunk, + miniblock_chunk_size, ) } else if Self::prefers_miniblock(&data_block, encoding_metadata.as_ref()) { log::debug!( @@ -5630,7 +6536,7 @@ impl PrimitiveStructuralEncoder { row_number, None, num_rows, - support_large_chunk, + miniblock_chunk_size, ) } else if Self::prefers_fullzip(encoding_metadata.as_ref()) { log::debug!( @@ -5663,34 +6569,49 @@ impl PrimitiveStructuralEncoder { let num_values = arrays.iter().map(|arr| arr.len() as u64).sum(); let is_simple_validity = repdefs.iter().all(|rd| rd.is_simple_validity()); let has_repdef_info = repdefs.iter().any(|rd| !rd.is_empty()); - let (repdef, structural_plan) = RepDefBuilder::serialize_with_structural_plan( - repdefs, - miniblock::max_repdef_levels_per_chunk, - num_rows, - num_values, - )?; - let pages = Self::split_structural_pages_for_miniblock_budget( - arrays, - repdef, - structural_plan, - row_number, - num_rows, - )?; + let normalized = RepDefBuilder::normalize(repdefs); + let plan_ctx = PrimitivePlanContext { + column_idx: self.column_index, + field: &self.field, + encoding_metadata: &self.encoding_metadata, + }; + let mut pages = None; + for page_encoding in self.page_encodings.iter() { + if let Some(planned) = page_encoding.behavior.try_plan_pages( + &plan_ctx, + &arrays, + &normalized, + row_number, + num_rows, + num_values, + )? { + pages = Some(planned); + break; + } + } + let pages = pages.ok_or_else(|| { + Error::invalid_input_source( + format!( + "No primitive page planner supports field '{}'", + self.field.name + ) + .into(), + ) + })?; let mut tasks = Vec::with_capacity(pages.len()); let ctx = PrimitiveEncodeContext { column_idx: self.column_index, field: self.field.clone(), - compression_strategy: self.compression_strategy.clone(), encoding_metadata: self.encoding_metadata.clone(), - support_large_chunk: self.support_large_chunk, - version: self.version, is_simple_validity, has_repdef_info, }; for page in pages { let ctx = ctx.clone(); - let task = spawn_cpu(move || Self::encode_page(ctx, page)).boxed(); + let page_encodings = self.page_encodings.clone(); + let task = + spawn_cpu(move || Self::encode_page(page_encodings.as_ref(), &ctx, page)).boxed(); tasks.push(task); } Ok(tasks) @@ -5742,6 +6663,291 @@ impl PrimitiveStructuralEncoder { } } +impl PrimitivePageEncodingBehavior for RejectSparsePrimitiveEncoding { + fn validate_field(&self, field: &Field, metadata: &HashMap) -> Result<()> { + if metadata + .get(STRUCTURAL_ENCODING_META_KEY) + .is_some_and(|requested| requested.eq_ignore_ascii_case(STRUCTURAL_ENCODING_SPARSE)) + { + return Err(Error::invalid_input_source( + format!( + "Field '{}' requests sparse structural encoding, which is not enabled by the selected file format", + field.name + ) + .into(), + )); + } + Ok(()) + } +} + +fn plan_dense_primitive_pages( + arrays: &[ArrayRef], + normalized: &NormalizedStructuralPlan, + row_number: u64, + num_rows: u64, + num_values: u64, +) -> Result> { + let (repdef, miniblock_repdef_budget) = normalized.serialize_with_miniblock_repdef_budget( + miniblock::max_repdef_levels_per_chunk, + num_rows, + num_values, + )?; + PrimitiveStructuralEncoder::split_pages_for_miniblock_repdef_budget( + arrays.to_vec(), + repdef, + miniblock_repdef_budget, + row_number, + num_rows, + ) +} + +impl PrimitivePageEncodingBehavior for DenseU16PrimitiveEncoding { + fn try_plan_pages( + &self, + _ctx: &PrimitivePlanContext<'_>, + arrays: &[ArrayRef], + normalized: &NormalizedStructuralPlan, + row_number: u64, + num_rows: u64, + num_values: u64, + ) -> Result>> { + Ok(Some(plan_dense_primitive_pages( + arrays, normalized, row_number, num_rows, num_values, + )?)) + } + + fn try_encode_page( + &self, + ctx: &PrimitiveEncodeContext, + page: PrimitivePageData, + ) -> Result { + if !matches!(&page.structure, PrimitivePageStructure::Dense { .. }) { + return Ok(PrimitiveEncodeAttempt::Unhandled(page)); + } + Ok(PrimitiveEncodeAttempt::Encoded( + PrimitiveStructuralEncoder::encode_dense_page( + ctx.clone(), + page, + self.compression.clone(), + MiniblockChunkSize::U16, + ComplexNullEncoding::RawLevels, + FixedWidthDictionaryEncoding::Exclude64Bit, + )?, + )) + } +} + +impl PrimitivePageEncodingBehavior for DenseU32PrimitiveEncoding { + fn try_plan_pages( + &self, + _ctx: &PrimitivePlanContext<'_>, + arrays: &[ArrayRef], + normalized: &NormalizedStructuralPlan, + row_number: u64, + num_rows: u64, + num_values: u64, + ) -> Result>> { + Ok(Some(plan_dense_primitive_pages( + arrays, normalized, row_number, num_rows, num_values, + )?)) + } + + fn try_encode_page( + &self, + ctx: &PrimitiveEncodeContext, + page: PrimitivePageData, + ) -> Result { + if !matches!(&page.structure, PrimitivePageStructure::Dense { .. }) { + return Ok(PrimitiveEncodeAttempt::Unhandled(page)); + } + Ok(PrimitiveEncodeAttempt::Encoded( + PrimitiveStructuralEncoder::encode_dense_page( + ctx.clone(), + page, + self.compression.clone(), + MiniblockChunkSize::U32, + ComplexNullEncoding::CompressedLevels, + FixedWidthDictionaryEncoding::Include64Bit, + )?, + )) + } +} + +impl PrimitivePageEncodingBehavior for SparsePrimitiveEncoding { + fn try_plan_pages( + &self, + ctx: &PrimitivePlanContext<'_>, + arrays: &[ArrayRef], + normalized: &NormalizedStructuralPlan, + row_number: u64, + num_rows: u64, + num_values: u64, + ) -> Result>> { + let requested_encoding = ctx.encoding_metadata.get(STRUCTURAL_ENCODING_META_KEY); + let requests_sparse = requested_encoding + .is_some_and(|requested| requested.eq_ignore_ascii_case(STRUCTURAL_ENCODING_SPARSE)); + if requests_sparse { + let plan = sparse::writer::plan(normalized, num_values)?; + if sparse::writer::uses_constant_layout(&plan, ctx.field) { + return Ok(None); + } + return Ok(Some(vec![PrimitivePageData { + arrays: arrays.to_vec(), + structure: PrimitivePageStructure::Sparse { + plan, + prepared_values: None, + }, + row_number, + num_rows, + }])); + } + + let (_, miniblock_repdef_budget) = normalized.serialize_with_miniblock_repdef_budget( + miniblock::max_repdef_levels_per_chunk, + num_rows, + num_values, + )?; + let automatic_sparse = layout::select_automatic_sparse( + requested_encoding.map(String::as_str), + &miniblock_repdef_budget, + || { + let data = DataBlock::from_arrays(arrays, num_values); + if !sparse::writer::supports_value_block(&data) { + return Ok(None); + } + let prepared_values = match sparse::writer::prepare_values( + ctx.field, + self.compression.as_ref(), + data, + MiniblockChunkSize::U32, + ) { + Ok(prepared_values) => prepared_values, + Err(error) => { + debug!( + "Keeping column {} on its dense structural path because sparse value preparation is unavailable: {}", + ctx.column_idx, error + ); + return Ok(None); + } + }; + let plan = sparse::writer::plan(normalized, num_values)?; + if sparse::writer::uses_constant_layout(&plan, ctx.field) { + return Ok(None); + } + Ok(Some((plan, prepared_values))) + }, + )?; + Ok(automatic_sparse.map(|(plan, prepared_values)| { + vec![PrimitivePageData { + arrays: arrays.to_vec(), + structure: PrimitivePageStructure::Sparse { + plan, + prepared_values: Some(prepared_values), + }, + row_number, + num_rows, + }] + })) + } + + fn try_encode_page( + &self, + ctx: &PrimitiveEncodeContext, + page: PrimitivePageData, + ) -> Result { + if !matches!(&page.structure, PrimitivePageStructure::Sparse { .. }) { + return Ok(PrimitiveEncodeAttempt::Unhandled(page)); + } + let PrimitivePageData { + arrays, + structure: + PrimitivePageStructure::Sparse { + plan, + prepared_values, + }, + row_number, + num_rows, + } = page + else { + unreachable!() + }; + let num_values = arrays.iter().map(|array| array.len() as u64).sum(); + log::debug!( + "Encoding column {} with {} visible items ({} rows) using sparse layout", + ctx.column_idx, + num_values, + num_rows + ); + Ok(PrimitiveEncodeAttempt::Encoded( + sparse::writer::encode_page( + ctx.column_idx, + &ctx.field, + self.compression.as_ref(), + prepared_values.map_or_else( + || { + sparse::writer::SparseValueInput::Unprepared(DataBlock::from_arrays( + &arrays, num_values, + )) + }, + sparse::writer::SparseValueInput::Prepared, + ), + plan, + row_number, + num_rows, + MiniblockChunkSize::U32, + )?, + )) + } +} + +impl PrimitivePageEncodingBehavior for ConstantPrimitiveEncoding { + fn try_encode_page( + &self, + ctx: &PrimitiveEncodeContext, + page: PrimitivePageData, + ) -> Result { + let PrimitivePageStructure::Dense { repdef, .. } = &page.structure else { + return Ok(PrimitiveEncodeAttempt::Unhandled(page)); + }; + let num_values: u64 = page.arrays.iter().map(|array| array.len() as u64).sum(); + if num_values == 0 { + return Ok(PrimitiveEncodeAttempt::Unhandled(page)); + } + let leaf_validity = PrimitiveStructuralEncoder::leaf_validity(repdef, num_values as usize)?; + if leaf_validity + .as_ref() + .is_some_and(|validity| validity.count_set_bits() == 0) + || matches!(ctx.field.data_type(), DataType::Struct(fields) if fields.is_empty()) + { + return Ok(PrimitiveEncodeAttempt::Unhandled(page)); + } + let Some(scalar) = + PrimitiveStructuralEncoder::find_constant_scalar(&page.arrays, leaf_validity.as_ref())? + else { + return Ok(PrimitiveEncodeAttempt::Unhandled(page)); + }; + let PrimitivePageData { + structure: PrimitivePageStructure::Dense { repdef, .. }, + row_number, + num_rows, + .. + } = page + else { + unreachable!() + }; + log::debug!( + "Encoding column {} with {} items ({} rows) using constant layout", + ctx.column_idx, + num_values, + num_rows + ); + Ok(PrimitiveEncodeAttempt::Encoded( + constant::encode_constant_page(ctx.column_idx, scalar, repdef, row_number, num_rows)?, + )) + } +} + impl FieldEncoder for PrimitiveStructuralEncoder { // Buffers data, if there is enough to write a page then we create an encode task fn maybe_encode( @@ -5791,14 +6997,19 @@ impl FieldEncoder for PrimitiveStructuralEncoder { #[allow(clippy::single_range_in_vec_init)] mod tests { use super::{ - ChunkInstructions, DataBlock, DecodeMiniBlockTask, FixedPerValueDecompressor, - FixedWidthDataBlock, FullZipCacheableState, FullZipDecodeDetails, FullZipReadSource, - FullZipRepIndexDetails, FullZipScheduler, MiniBlockChunk, MiniBlockCompressed, - MiniBlockRepIndex, PerValueDecompressor, PreambleAction, StructuralPageScheduler, - VariableFullZipDecoder, + ChunkInstructions, DataBlock, DecodeMiniBlockTask, DecodePageTask, FixedFullZipDecodeTask, + FixedPerValueDecompressor, FixedWidthDataBlock, FixedWidthDictionaryEncoding, + FullZipCacheableState, FullZipDecodeDetails, FullZipDecodeTaskItem, FullZipReadSource, + FullZipRepIndexDetails, FullZipScheduler, LazyLevels, LevelCodec, LevelCursor, LevelPlan, + MiniBlockChunk, MiniBlockChunkIndex, MiniBlockCompressed, MiniblockChunkSize, + PerValueDataBlock, PerValueDecompressor, PreambleAction, RunEndsBuilder, RunPosition, + RunStorage, StructuralPageScheduler, VariableFullZipDecoder, dense_levels_from_block, + validate_complex_all_null_levels, }; use crate::buffer::LanceBuffer; - use crate::compression::DefaultDecompressionStrategy; + use crate::compression::{ + BlockCompressor, DefaultDecompressionStrategy, MiniBlockDecompressor, + }; use crate::constants::{ COMPRESSION_LEVEL_META_KEY, COMPRESSION_META_KEY, DICT_VALUES_COMPRESSION_LEVEL_META_KEY, DICT_VALUES_COMPRESSION_META_KEY, STRUCTURAL_ENCODING_META_KEY, @@ -5806,16 +7017,24 @@ mod tests { }; use crate::data::BlockInfo; use crate::decoder::{PageEncoding, StructuralFieldDecoder}; + use crate::encodings::logical::primitive::fullzip::PerValueCompressor; use crate::encodings::logical::primitive::{ - ChunkDrainInstructions, PrimitiveStructuralEncoder, StructuralPrimitiveFieldDecoder, + ChunkDrainInstructions, LoadedChunk, PrimitiveStructuralEncoder, + StructuralPrimitiveFieldDecoder, }; + use crate::encodings::physical::rle::{RleDecompressor, RleEncoder, RleRuns, RunLengthWidth}; + use crate::encodings::physical::value::{ValueDecompressor, ValueEncoder}; use crate::format::ProtobufUtils21; use crate::format::pb21; use crate::format::pb21::compressive_encoding::Compression; use crate::repdef::build_control_word_iterator; + use crate::testing::TestEncoding; use crate::testing::{TestCases, check_round_trip_encoding_of_data}; - use crate::version::LanceFileVersion; - use arrow_array::{ArrayRef, Int8Array, StringArray}; + use arrow_array::{ + Array, ArrayRef, FixedSizeListArray, Float32Array, Int8Array, StringArray, UInt8Array, + make_array, + }; + use arrow_buffer::ScalarBuffer; use arrow_schema::{DataType, Field as ArrowField}; use std::collections::HashMap; use std::{collections::VecDeque, sync::Arc}; @@ -5868,22 +7087,328 @@ mod tests { } #[test] - fn test_fullzip_fixed_rejects_non_byte_aligned_values() { - let fixed = FixedWidthDataBlock { - data: LanceBuffer::from(vec![0_u8]), - bits_per_value: 1, - num_values: 8, - block_info: BlockInfo::new(), - }; - let repdef = build_control_word_iterator(None, 0, None, 0, u16::MAX, 8); + fn test_fullzip_fixed_rejects_non_byte_aligned_values() { + let fixed = FixedWidthDataBlock { + data: LanceBuffer::from(vec![0_u8]), + bits_per_value: 1, + num_values: 8, + block_info: BlockInfo::new(), + }; + let repdef = build_control_word_iterator(None, 0, None, 0, u16::MAX, 8); + + let Err(err) = PrimitiveStructuralEncoder::serialize_full_zip_fixed(fixed, repdef, 8) + else { + panic!("expected full-zip to reject 1-bit fixed-width values"); + }; + assert!( + err.to_string().contains("byte aligned"), + "unexpected error: {err}" + ); + } + + fn decode_fixed_fullzip_no_levels( + decompressor: Arc, + data: Vec, + num_rows: usize, + bytes_per_value: usize, + ) -> DataBlock { + Box::new(FixedFullZipDecodeTask { + details: Arc::new(FullZipDecodeDetails { + value_decompressor: PerValueDecompressor::Fixed(decompressor), + def_meaning: Arc::from([]), + ctrl_word_parser: crate::repdef::ControlWordParser::new(0, 0), + max_rep: 0, + max_visible_def: u16::MAX, + }), + data, + num_rows, + bytes_per_value, + }) + .decode() + .unwrap() + .data + } + + #[test] + fn test_fixed_fullzip_decode_preallocates_exact_output_size() { + #[derive(Debug)] + struct IdentityFixedDecompressor; + + impl FixedPerValueDecompressor for IdentityFixedDecompressor { + fn decompress( + &self, + data: FixedWidthDataBlock, + num_rows: u64, + ) -> crate::Result { + assert_eq!(data.num_values, num_rows); + Ok(DataBlock::FixedWidth(data)) + } + + fn bits_per_value(&self) -> u64 { + 32 + } + + fn decoded_size_bytes(&self, num_values: u64) -> Option { + num_values.checked_mul(4) + } + } + + let make_item = |num_rows: u64| FullZipDecodeTaskItem { + data: PerValueDataBlock::Fixed(FixedWidthDataBlock { + data: LanceBuffer::from(vec![7_u8; num_rows as usize * 4]), + bits_per_value: 32, + num_values: num_rows, + block_info: BlockInfo::new(), + }), + rows_in_buf: num_rows, + }; + + let num_rows = 512; + let decoded = decode_fixed_fullzip_no_levels( + Arc::new(IdentityFixedDecompressor), + vec![make_item(128), make_item(384)], + num_rows, + 4, + ); + let values = decoded.as_fixed_width_ref().unwrap(); + let expected_size = num_rows * 4; + assert_eq!(values.data.len(), expected_size); + assert_eq!(values.data.clone().into_buffer().capacity(), expected_size); + } + + #[test] + fn test_fixed_fullzip_decode_falls_back_when_output_size_is_not_exact() { + #[derive(Debug)] + struct FallbackFixedDecompressor; + + impl FixedPerValueDecompressor for FallbackFixedDecompressor { + fn decompress( + &self, + data: FixedWidthDataBlock, + num_rows: u64, + ) -> crate::Result { + assert_eq!(data.num_values, num_rows); + Ok(DataBlock::FixedWidth(FixedWidthDataBlock { + data: LanceBuffer::from(vec![7_u8; num_rows as usize * 4]), + bits_per_value: 32, + num_values: num_rows, + block_info: BlockInfo::new(), + })) + } + + fn bits_per_value(&self) -> u64 { + // This deliberately cannot be multiplied by num_rows. If FullZip treats + // bits_per_value as an exact decoded-size estimate, decoding will fail. + u64::MAX - 7 + } + } + + let num_rows = 2; + let decoded = decode_fixed_fullzip_no_levels( + Arc::new(FallbackFixedDecompressor), + vec![FullZipDecodeTaskItem { + data: PerValueDataBlock::Fixed(FixedWidthDataBlock { + data: LanceBuffer::from(vec![0_u8; num_rows * 4]), + bits_per_value: 32, + num_values: num_rows as u64, + block_info: BlockInfo::new(), + }), + rows_in_buf: num_rows as u64, + }], + num_rows, + 4, + ); + let values = decoded.as_fixed_width_ref().unwrap(); + assert_eq!(values.num_values, num_rows as u64); + assert_eq!(values.data.len(), num_rows * 4); + } + + #[test] + fn test_fixed_fullzip_real_fsl_preallocates_exact_output_size() { + let num_rows = 64; + let dimension = 32; + let items = Arc::new(Float32Array::from_iter_values( + (0..num_rows * dimension).map(|value| value as f32), + )); + let item_field = Arc::new(ArrowField::new("item", DataType::Float32, false)); + let sample = FixedSizeListArray::new(item_field, dimension as i32, items, None); + + let (data, compression) = PerValueCompressor::compress( + &ValueEncoder::default(), + DataBlock::from_array(sample.clone()), + ) + .unwrap(); + let Compression::FixedSizeList(fsl) = compression.compression.unwrap() else { + panic!("expected fixed-size-list compression"); + }; + let decompressor = ValueDecompressor::from_fsl(fsl.as_ref()); + let expected_size = num_rows * dimension * size_of::(); + assert_eq!( + FixedPerValueDecompressor::decoded_size_bytes(&decompressor, num_rows as u64), + Some(expected_size as u64) + ); + + let decoded = decode_fixed_fullzip_no_levels( + Arc::new(decompressor), + vec![FullZipDecodeTaskItem { + data, + rows_in_buf: num_rows as u64, + }], + num_rows, + dimension * size_of::(), + ); + let fsl = decoded.as_fixed_size_list_ref().unwrap(); + let values = fsl.child.as_fixed_width_ref().unwrap(); + assert_eq!(values.data.len(), expected_size); + assert_eq!(values.data.clone().into_buffer().capacity(), expected_size); + + let decoded_array = make_array( + decoded + .into_arrow(sample.data_type().clone(), true) + .unwrap(), + ); + assert_eq!(decoded_array.as_ref(), &sample); + } + + #[test] + fn test_fixed_fullzip_nullable_fsl_uses_fallback_end_to_end() { + #[derive(Debug)] + struct NullableFslDecompressor { + inner: ValueDecompressor, + } + + impl FixedPerValueDecompressor for NullableFslDecompressor { + fn decompress( + &self, + data: FixedWidthDataBlock, + num_rows: u64, + ) -> crate::Result { + FixedPerValueDecompressor::decompress(&self.inner, data, num_rows) + } + + fn bits_per_value(&self) -> u64 { + // FullZip must not use this physical row width as an exact decoded-size + // estimate for the nullable, multi-buffer Arrow output. + u64::MAX - 7 + } + + fn decoded_size_bytes(&self, num_values: u64) -> Option { + FixedPerValueDecompressor::decoded_size_bytes(&self.inner, num_values) + } + } + + let num_rows = 64; + let items = Arc::new(UInt8Array::from_iter( + (0..num_rows).map(|value| (value % 3 != 0).then_some(value as u8)), + )); + let item_field = Arc::new(ArrowField::new("item", DataType::UInt8, true)); + let sample = FixedSizeListArray::new(item_field, 1, items, None); + + let (data, compression) = PerValueCompressor::compress( + &ValueEncoder::default(), + DataBlock::from_array(sample.clone()), + ) + .unwrap(); + let Compression::FixedSizeList(fsl) = compression.compression.unwrap() else { + panic!("expected fixed-size-list compression"); + }; + let decompressor = NullableFslDecompressor { + inner: ValueDecompressor::from_fsl(fsl.as_ref()), + }; + assert_eq!( + FixedPerValueDecompressor::decoded_size_bytes(&decompressor, num_rows as u64), + None + ); + + let decoded = decode_fixed_fullzip_no_levels( + Arc::new(decompressor), + vec![FullZipDecodeTaskItem { + data, + rows_in_buf: num_rows as u64, + }], + num_rows, + 2, + ); + let decoded_array = make_array( + decoded + .into_arrow(sample.data_type().clone(), true) + .unwrap(), + ); + assert_eq!(decoded_array.as_ref(), &sample); + } + + #[test] + fn test_miniblock_decode_uses_exact_fixed_width_output_size() { + #[derive(Debug)] + struct FixedWidthMiniBlockDecompressor; - let Err(err) = PrimitiveStructuralEncoder::serialize_full_zip_fixed(fixed, repdef, 8) - else { - panic!("expected full-zip to reject 1-bit fixed-width values"); + impl MiniBlockDecompressor for FixedWidthMiniBlockDecompressor { + fn decompress( + &self, + data: Vec, + num_values: u64, + ) -> crate::Result { + assert_eq!(data.len(), 1); + Ok(DataBlock::FixedWidth(FixedWidthDataBlock { + data: data.into_iter().next().unwrap(), + bits_per_value: 32, + num_values, + block_info: BlockInfo::new(), + })) + } + + fn decoded_size_bytes(&self, num_values: u64) -> Option { + num_values.checked_mul(4) + } + } + + let num_rows = 512; + let expected_size = num_rows * 4; + let mut chunk_data = Vec::new(); + chunk_data.extend_from_slice(&0_u16.to_le_bytes()); + chunk_data.extend_from_slice(&(expected_size as u16).to_le_bytes()); + let header_padding = + lance_core::utils::bit::pad_bytes::<{ super::MINIBLOCK_ALIGNMENT }>(chunk_data.len()); + chunk_data.resize(chunk_data.len() + header_padding, 0); + chunk_data.resize(chunk_data.len() + expected_size as usize, 7); + + let task = DecodeMiniBlockTask { + rep_decompressor: None, + def_decompressor: None, + value_decompressor: Arc::new(FixedWidthMiniBlockDecompressor), + dictionary_data: None, + def_meaning: Arc::from([]), + num_buffers: 1, + max_visible_level: 0, + instructions: vec![( + ChunkDrainInstructions { + chunk_instructions: ChunkInstructions { + chunk_idx: 0, + preamble: PreambleAction::Absent, + rows_to_skip: 0, + rows_to_take: num_rows, + take_trailer: false, + }, + rows_to_skip: 0, + rows_to_take: num_rows, + preamble_action: PreambleAction::Absent, + }, + LoadedChunk { + byte_range: 0..chunk_data.len() as u64, + data: LanceBuffer::from(chunk_data), + items_in_chunk: num_rows, + chunk_idx: 0, + }, + )], + has_large_chunk: false, }; - assert!( - err.to_string().contains("byte aligned"), - "unexpected error: {err}" + + let decoded = Box::new(task).decode().unwrap(); + let values = decoded.data.as_fixed_width_ref().unwrap(); + assert_eq!(values.data.len(), expected_size as usize); + assert_eq!( + values.data.clone().into_buffer().capacity(), + expected_size as usize ); } @@ -6289,11 +7814,10 @@ mod tests { // Convert repetition index to bytes for testing let rep_data: Vec = vec![5, 2, 3, 0, 4, 7, 2, 0]; let rep_bytes: Vec = rep_data.iter().flat_map(|v| v.to_le_bytes()).collect(); - let repetition_index = MiniBlockRepIndex::decode_from_bytes(&rep_bytes, 2); + let chunk_index = MiniBlockChunkIndex::new_nested_for_test(&rep_bytes, 2); let check = |user_ranges, expected_instructions| { - let instructions = - ChunkInstructions::schedule_instructions(&repetition_index, user_ranges); + let instructions = ChunkInstructions::schedule_instructions(&chunk_index, user_ranges); assert_eq!(instructions, expected_instructions); }; @@ -6445,11 +7969,11 @@ mod tests { // Convert repetition index to bytes for testing let rep_data: Vec = vec![5, 2, 3, 0, 4, 7, 2, 0]; let rep_bytes: Vec = rep_data.iter().flat_map(|v| v.to_le_bytes()).collect(); - let repetition_index = MiniBlockRepIndex::decode_from_bytes(&rep_bytes, 2); + let chunk_index = MiniBlockChunkIndex::new_nested_for_test(&rep_bytes, 2); let user_ranges = vec![1..7, 10..14]; // First, schedule the ranges - let scheduled = ChunkInstructions::schedule_instructions(&repetition_index, &user_ranges); + let scheduled = ChunkInstructions::schedule_instructions(&chunk_index, &user_ranges); let mut to_drain = VecDeque::from(scheduled.clone()); @@ -6530,11 +8054,11 @@ mod tests { // Regression case. Need a chunk with preamble, rows, and trailer (the middle chunk here) let rep_data: Vec = vec![5, 2, 3, 3, 20, 0]; let rep_bytes: Vec = rep_data.iter().flat_map(|v| v.to_le_bytes()).collect(); - let repetition_index = MiniBlockRepIndex::decode_from_bytes(&rep_bytes, 2); + let chunk_index = MiniBlockChunkIndex::new_nested_for_test(&rep_bytes, 2); let user_ranges = vec![0..28]; // First, schedule the ranges - let scheduled = ChunkInstructions::schedule_instructions(&repetition_index, &user_ranges); + let scheduled = ChunkInstructions::schedule_instructions(&chunk_index, &user_ranges); let mut to_drain = VecDeque::from(scheduled.clone()); @@ -6594,6 +8118,181 @@ mod tests { assert_eq!(skip_in_chunk, 0); } + use super::chunk_index::{PrefixSums, RowMapping}; + use super::{MINIBLOCK_ALIGNMENT, Words, build_chunk_index}; + use bytes::Bytes; + use lance_core::cache::{Context, DeepSizeOf}; + use rstest::rstest; + + /// Builds a `Words` metadata buffer (u16 words) from `(log_num_values, num_bytes)` + /// pairs, returning the words and the total data-buffer size. + fn words_from(entries: &[(u32, u32)]) -> (Words, u64) { + let mut raw = Vec::with_capacity(entries.len() * 2); + let mut total = 0u64; + for &(log, num_bytes) in entries { + assert!(num_bytes > 0 && num_bytes % MINIBLOCK_ALIGNMENT as u32 == 0); + let divided = num_bytes / MINIBLOCK_ALIGNMENT as u32 - 1; + let word = (divided << 4) | log; + assert!(word <= u16::MAX as u32, "test word {word} exceeds u16"); + raw.extend_from_slice(&(word as u16).to_le_bytes()); + total += num_bytes as u64; + } + (Words::from_bytes(Bytes::from(raw), false).unwrap(), total) + } + + fn rep_bytes_from(values: &[u64]) -> Vec { + values.iter().flat_map(|v| v.to_le_bytes()).collect() + } + + #[rstest] + // Two full chunks of 8 values (log 3) plus a partial last chunk; byte sizes vary + // independently of value counts. + #[case::uniform_partial_last(&[(3, 16), (3, 24), (0, 8)], 19, "uniform_flat", 8, 3)] + // Single chunk covers the whole page. + #[case::single_chunk(&[(0, 24)], 5, "uniform_flat", 5, 5)] + // Last chunk is also full (exact multiple). + #[case::exact_multiple(&[(3, 16), (3, 16)], 16, "uniform_flat", 8, 8)] + // Non-last chunks differ in size, so this is a non-uniform flat page. + #[case::non_uniform(&[(4, 16), (2, 16), (0, 8)], 21, "flat", 16, 1)] + fn test_flat_detection( + #[case] entries: &[(u32, u32)], + #[case] items_in_page: u64, + #[case] expected_kind: &str, + #[case] expected_first_items: u64, + #[case] expected_last_items: u64, + ) { + let base = 100u64; + let (words, data_buf_size) = words_from(entries); + let index = build_chunk_index(&words, items_in_page, base, data_buf_size, None, 0).unwrap(); + + assert_eq!(index.row_mapping_debug(), expected_kind); + assert_eq!(index.num_chunks(), entries.len()); + assert_eq!(index.items_in_chunk(0), expected_first_items); + assert_eq!(index.items_in_chunk(entries.len() - 1), expected_last_items); + + // Byte ranges are absolute, contiguous, and exactly cover the data buffer. + let mut expected_start = base; + for (i, &(_, num_bytes)) in entries.iter().enumerate() { + let range = index.byte_range(i); + assert_eq!(range.start, expected_start); + assert_eq!(range.end - range.start, num_bytes as u64); + expected_start = range.end; + } + assert_eq!(expected_start, base + data_buf_size); + + // For flat pages rows == items, so the per-chunk items sum to the page total. + let total_items: u64 = (0..index.num_chunks()) + .map(|i| index.items_in_chunk(i)) + .sum(); + assert_eq!(total_items, items_in_page); + } + + #[test] + fn test_nested_detection_and_axes() { + // Repetition index (stride 2): three chunks holding 5, 4, 3 rows, no trailers. + let rep = rep_bytes_from(&[5, 0, 4, 0, 3, 0]); + + // Uniform leaf chunking: value counts 4, 4, 2. + let (words, data_buf_size) = words_from(&[(2, 8), (2, 8), (0, 8)]); + let index = build_chunk_index(&words, 10, 0, data_buf_size, Some(&rep), 1).unwrap(); + assert_eq!(index.row_mapping_debug(), "nested"); + assert_eq!(index.num_chunks(), 3); + // Rows come from the repetition index, not the value counts. + assert_eq!(index.first_row(0), 0); + assert_eq!(index.rows_in_chunk(0), 5); + assert_eq!(index.first_row(1), 5); + assert_eq!(index.rows_in_chunk(1), 4); + assert_eq!(index.first_row(2), 9); + assert_eq!(index.rows_in_chunk(2), 3); + // Items come from the value words. + assert_eq!(index.items_in_chunk(0), 4); + assert_eq!(index.items_in_chunk(1), 4); + assert_eq!(index.items_in_chunk(2), 2); + + // Non-uniform leaf chunking: value counts 8, 2, 5. + let (words_nu, dbs_nu) = words_from(&[(3, 8), (1, 8), (0, 8)]); + let index_nu = build_chunk_index(&words_nu, 15, 0, dbs_nu, Some(&rep), 1).unwrap(); + assert_eq!(index_nu.row_mapping_debug(), "nested"); + assert_eq!(index_nu.items_in_chunk(0), 8); + assert_eq!(index_nu.items_in_chunk(1), 2); + assert_eq!(index_nu.items_in_chunk(2), 5); + // The row axis is unchanged by the leaf chunking. + assert_eq!(index_nu.rows_in_chunk(0), 5); + } + + #[test] + fn test_uniform_flat_matches_prefix_sum_flat() { + // Distribution: 4 chunks of 4 values, last chunk 3 (15 items total). + let (words, data_buf_size) = words_from(&[(2, 8), (2, 8), (2, 8), (0, 8)]); + let uniform = build_chunk_index(&words, 15, 0, data_buf_size, None, 0).unwrap(); + assert_eq!(uniform.row_mapping_debug(), "uniform_flat"); + + // The same distribution expressed as a non-uniform Flat prefix-sum index. + let byte_starts = PrefixSums::from_deltas([8u64, 8, 8, 8].into_iter(), 4, 32); + let value_starts = PrefixSums::from_deltas([4u64, 4, 4, 3].into_iter(), 4, 15); + let flat = MiniBlockChunkIndex::new(0, byte_starts, RowMapping::Flat { value_starts }); + assert_eq!(flat.row_mapping_debug(), "flat"); + + // Lookup parity: identical byte ranges and item counts. + for i in 0..4 { + assert_eq!(uniform.byte_range(i), flat.byte_range(i)); + assert_eq!(uniform.items_in_chunk(i), flat.items_in_chunk(i)); + } + + // Scheduler parity across scan / single-row / partial / scattered multi-range. + let range_sets: Vec>> = vec![ + vec![0..15], + vec![0..1], + vec![7..8], + vec![14..15], + vec![3..10], + vec![0..2, 5..6, 12..15], + ]; + for ranges in &range_sets { + let from_uniform = ChunkInstructions::schedule_instructions(&uniform, ranges); + let from_flat = ChunkInstructions::schedule_instructions(&flat, ranges); + assert_eq!(from_uniform, from_flat, "mismatch for ranges {ranges:?}"); + } + + // A full scan yields one Absent, no-trailer instruction per chunk. + let full = ChunkInstructions::schedule_instructions(&uniform, &[0..15]); + assert_eq!(full.len(), 4); + for (i, inst) in full.iter().enumerate() { + assert_eq!(inst.chunk_idx, i); + assert_eq!(inst.preamble, PreambleAction::Absent); + assert_eq!(inst.rows_to_skip, 0); + assert!(!inst.take_trailer); + } + assert_eq!(full.iter().map(|i| i.rows_to_take).sum::(), 15); + } + + #[test] + fn test_deep_size_per_variant_below_legacy() { + // The previous representation cached 48 bytes per chunk (24 for ChunkMeta plus + // 24 for a rep-index block); every variant's heap must be well below that. + const LEGACY_PER_CHUNK: usize = 48; + let num_chunks = 3; + let heap = |index: &MiniBlockChunkIndex| index.deep_size_of_children(&mut Context::new()); + + let (uniform_words, uniform_dbs) = words_from(&[(2, 8), (2, 8), (0, 8)]); + let uniform = build_chunk_index(&uniform_words, 10, 0, uniform_dbs, None, 0).unwrap(); + assert_eq!(uniform.row_mapping_debug(), "uniform_flat"); + assert!(heap(&uniform) < LEGACY_PER_CHUNK * num_chunks); + + let (flat_words, flat_dbs) = words_from(&[(3, 8), (1, 8), (0, 8)]); + let flat = build_chunk_index(&flat_words, 11, 0, flat_dbs, None, 0).unwrap(); + assert_eq!(flat.row_mapping_debug(), "flat"); + assert!(heap(&flat) < LEGACY_PER_CHUNK * num_chunks); + // Flat carries a value-starts array that UniformFlat derives arithmetically. + assert!(heap(&flat) > heap(&uniform)); + + let rep = rep_bytes_from(&[4, 0, 3, 0, 3, 0]); + let (nested_words, nested_dbs) = words_from(&[(2, 8), (2, 8), (0, 8)]); + let nested = build_chunk_index(&nested_words, 10, 0, nested_dbs, Some(&rep), 1).unwrap(); + assert_eq!(nested.row_mapping_debug(), "nested"); + assert!(heap(&nested) < LEGACY_PER_CHUNK * num_chunks); + } + #[tokio::test] async fn test_fullzip_initialize_is_lazy() { use futures::{FutureExt, future::BoxFuture}; @@ -6958,7 +8657,7 @@ mod tests { ); let test_cases = TestCases::default() - .with_min_file_version(LanceFileVersion::V2_1) + .with_structural_encodings() .with_batch_size(100) .with_range(0..num_rows.min(500) as u64) .with_indices(vec![0, num_rows as u64 / 2, (num_rows - 1) as u64]); @@ -6969,7 +8668,7 @@ mod tests { async fn test_minichunk_size_helper( string_data: Vec>, minichunk_size: u64, - file_version: LanceFileVersion, + encodings: &[TestEncoding], ) { use crate::constants::MINICHUNK_SIZE_META_KEY; use crate::testing::{TestCases, check_round_trip_encoding_of_data}; @@ -6989,7 +8688,7 @@ mod tests { ); let test_cases = TestCases::default() - .with_min_file_version(file_version) + .with_encodings(encodings.iter().copied()) .with_batch_size(1000); check_round_trip_encoding_of_data(vec![string_array], &test_cases, metadata).await; @@ -7003,7 +8702,16 @@ mod tests { string_data.push(Some(format!("test_string_{}", i).repeat(50))); } // configure minichunk size to 64 bytes (smaller than the default 4kb) for Lance 2.1 - test_minichunk_size_helper(string_data, 64, LanceFileVersion::V2_1).await; + test_minichunk_size_helper( + string_data, + 64, + &[ + TestEncoding::StructuralU16, + TestEncoding::StructuralU32, + TestEncoding::StructuralSparse, + ], + ) + .await; } #[tokio::test] @@ -7014,7 +8722,12 @@ mod tests { for i in 0..10000 { string_data.push(Some(format!("test_string_{}", i).repeat(50))); } - test_minichunk_size_helper(string_data, 128 * 1024, LanceFileVersion::V2_2).await; + test_minichunk_size_helper( + string_data, + 128 * 1024, + &[TestEncoding::StructuralU32, TestEncoding::StructuralSparse], + ) + .await; } #[tokio::test] @@ -7024,7 +8737,12 @@ mod tests { for i in 0..10000 { string_data.push(Some(format!("t_{}", i))); } - test_minichunk_size_helper(string_data, 128 * 1024, LanceFileVersion::V2_2).await; + test_minichunk_size_helper( + string_data, + 128 * 1024, + &[TestEncoding::StructuralU32, TestEncoding::StructuralSparse], + ) + .await; } #[tokio::test] @@ -7051,7 +8769,7 @@ mod tests { // Configure test to use V2_2 and verify encoding let test_cases = TestCases::default() - .with_min_file_version(LanceFileVersion::V2_2) + .with_u32_structural_encodings() .with_verify_encoding(Arc::new(|cols: &[crate::encoder::EncodedColumn], _| { assert_eq!(cols.len(), 1); let col = &cols[0]; @@ -7121,7 +8839,7 @@ mod tests { ) .with_metadata(metadata); - encode_first_page(field, dict_array, LanceFileVersion::V2_2).await + encode_first_page(field, dict_array, TestEncoding::StructuralU32).await } async fn encode_auto_fixed_dict_page( @@ -7151,7 +8869,7 @@ mod tests { let field = arrow_schema::Field::new("fixed_col", DataType::Decimal128(38, 0), false) .with_metadata(field_metadata); - encode_first_page(field, decimal, LanceFileVersion::V2_2).await + encode_first_page(field, decimal, TestEncoding::StructuralU32).await } #[tokio::test] @@ -7278,7 +8996,6 @@ mod tests { async fn test_dictionary_encode_int64() { use crate::constants::{DICT_SIZE_RATIO_META_KEY, STRUCTURAL_ENCODING_META_KEY}; use crate::testing::{TestCases, check_round_trip_encoding_of_data}; - use crate::version::LanceFileVersion; use arrow_array::{ArrayRef, Int64Array}; use std::collections::HashMap; use std::sync::Arc; @@ -7301,7 +9018,7 @@ mod tests { metadata.insert(DICT_SIZE_RATIO_META_KEY.to_string(), "0.99".to_string()); let test_cases = TestCases::default() - .with_min_file_version(LanceFileVersion::V2_2) + .with_u32_structural_encodings() .with_batch_size(1000) .with_range(0..1000) .with_indices(vec![0, 1, 10, 999]) @@ -7314,7 +9031,6 @@ mod tests { async fn test_dictionary_encode_float64() { use crate::constants::{DICT_SIZE_RATIO_META_KEY, STRUCTURAL_ENCODING_META_KEY}; use crate::testing::{TestCases, check_round_trip_encoding_of_data}; - use crate::version::LanceFileVersion; use arrow_array::{ArrayRef, Float64Array}; use std::collections::HashMap; use std::sync::Arc; @@ -7337,7 +9053,7 @@ mod tests { metadata.insert(DICT_SIZE_RATIO_META_KEY.to_string(), "0.99".to_string()); let test_cases = TestCases::default() - .with_min_file_version(LanceFileVersion::V2_2) + .with_u32_structural_encodings() .with_batch_size(1000) .with_range(0..1000) .with_indices(vec![0, 1, 10, 999]) @@ -7472,7 +9188,7 @@ mod tests { let result = PrimitiveStructuralEncoder::should_dictionary_encode( &block, &field, - LanceFileVersion::V2_1, + FixedWidthDictionaryEncoding::Exclude64Bit, ); assert!( @@ -7519,7 +9235,7 @@ mod tests { let result = PrimitiveStructuralEncoder::should_dictionary_encode( &block, &field, - LanceFileVersion::V2_2, + FixedWidthDictionaryEncoding::Include64Bit, ); assert!( @@ -7546,7 +9262,7 @@ mod tests { let result = PrimitiveStructuralEncoder::should_dictionary_encode( &block, &field, - LanceFileVersion::V2_2, + FixedWidthDictionaryEncoding::Include64Bit, ); assert!( @@ -7564,7 +9280,7 @@ mod tests { let field = arrow_schema::Field::new("test", DataType::Utf8, false).with_metadata(metadata); let array = create_sorted_string_array(200_000, 8_000); - let page = encode_first_page(field, array, LanceFileVersion::V2_2).await; + let page = encode_first_page(field, array, TestEncoding::StructuralU32).await; let _ = dictionary_encoding_from_page(&page); } @@ -7584,7 +9300,7 @@ mod tests { let result = PrimitiveStructuralEncoder::should_dictionary_encode( &block, &field, - LanceFileVersion::V2_1, + FixedWidthDictionaryEncoding::Exclude64Bit, ); assert!( @@ -7610,7 +9326,7 @@ mod tests { let result = PrimitiveStructuralEncoder::should_dictionary_encode( &block, &field, - LanceFileVersion::V2_1, + FixedWidthDictionaryEncoding::Exclude64Bit, ); assert!( @@ -7636,9 +9352,13 @@ mod tests { num_values: 32_769, }; - let serialized = - PrimitiveStructuralEncoder::serialize_miniblocks(miniblocks, None, None, false) - .unwrap(); + let serialized = PrimitiveStructuralEncoder::serialize_miniblocks( + miniblocks, + None, + None, + MiniblockChunkSize::U16, + ) + .unwrap(); let chunk_metadata = serialized.metadata.borrow_to_typed_slice::(); assert_eq!(chunk_metadata.len(), 2); @@ -7652,33 +9372,33 @@ mod tests { async fn encode_first_page( field: arrow_schema::Field, array: ArrayRef, - version: LanceFileVersion, + version: TestEncoding, ) -> crate::encoder::EncodedPage { - use crate::encoder::{ - ColumnIndexSequence, EncodingOptions, MIN_PAGE_BUFFER_ALIGNMENT, OutOfLineBuffers, - default_encoding_strategy, - }; use crate::repdef::RepDefBuilder; + use crate::{ + encoder::{ + ColumnIndexSequence, EncodingOptions, MIN_PAGE_BUFFER_ALIGNMENT, OutOfLineBuffers, + }, + testing::{create_test_field_encoder, test_encoding_strategy}, + }; let lance_field = lance_core::datatypes::Field::try_from(&field).unwrap(); - let encoding_strategy = default_encoding_strategy(version); + let encoding_strategy = test_encoding_strategy(version); let mut column_index_seq = ColumnIndexSequence::default(); let encoding_options = EncodingOptions { cache_bytes_per_column: 1, max_page_bytes: 32 * 1024 * 1024, keep_original_array: true, buffer_alignment: MIN_PAGE_BUFFER_ALIGNMENT, - version, }; - let mut encoder = encoding_strategy - .create_field_encoder( - encoding_strategy.as_ref(), - &lance_field, - &mut column_index_seq, - &encoding_options, - ) - .unwrap(); + let mut encoder = create_test_field_encoder( + encoding_strategy.as_ref(), + &lance_field, + &mut column_index_seq, + &encoding_options, + ) + .unwrap(); let mut external_buffers = OutOfLineBuffers::new(0, MIN_PAGE_BUFFER_ALIGNMENT); let repdef = RepDefBuilder::default(); @@ -7709,7 +9429,7 @@ mod tests { .unwrap(), ); let field = arrow_schema::Field::new("c", DataType::FixedSizeBinary(33), true); - let page = encode_first_page(field, arr.clone(), LanceFileVersion::V2_2).await; + let page = encode_first_page(field, arr.clone(), TestEncoding::StructuralU32).await; let PageEncoding::Structural(layout) = &page.description else { panic!("Expected structural encoding"); @@ -7721,8 +9441,7 @@ mod tests { assert_eq!(page.data.len(), 1); let test_cases = TestCases::default() - .with_min_file_version(LanceFileVersion::V2_2) - .with_max_file_version(LanceFileVersion::V2_2) + .with_encoding(TestEncoding::StructuralU32) .with_page_sizes(vec![4096]); check_round_trip_encoding_of_data(vec![arr], &test_cases, HashMap::new()).await; } @@ -7735,7 +9454,7 @@ mod tests { std::iter::repeat_n("hello", 512), )); let field = arrow_schema::Field::new("c", DataType::Utf8, true); - let page = encode_first_page(field, arr.clone(), LanceFileVersion::V2_2).await; + let page = encode_first_page(field, arr.clone(), TestEncoding::StructuralU32).await; let PageEncoding::Structural(layout) = &page.description else { panic!("Expected structural encoding"); @@ -7747,8 +9466,7 @@ mod tests { assert_eq!(page.data.len(), 1); let test_cases = TestCases::default() - .with_min_file_version(LanceFileVersion::V2_2) - .with_max_file_version(LanceFileVersion::V2_2) + .with_encoding(TestEncoding::StructuralU32) .with_page_sizes(vec![4096]); check_round_trip_encoding_of_data(vec![arr], &test_cases, HashMap::new()).await; } @@ -7765,7 +9483,7 @@ mod tests { Some(7), ])); let field = arrow_schema::Field::new("c", DataType::Int32, true); - let page = encode_first_page(field, arr.clone(), LanceFileVersion::V2_2).await; + let page = encode_first_page(field, arr.clone(), TestEncoding::StructuralU32).await; let PageEncoding::Structural(layout) = &page.description else { panic!("Expected structural encoding"); @@ -7777,8 +9495,7 @@ mod tests { assert_eq!(page.data.len(), 2); let test_cases = TestCases::default() - .with_min_file_version(LanceFileVersion::V2_2) - .with_max_file_version(LanceFileVersion::V2_2) + .with_encoding(TestEncoding::StructuralU32) .with_page_sizes(vec![4096]); check_round_trip_encoding_of_data(vec![arr], &test_cases, HashMap::new()).await; } @@ -7811,7 +9528,7 @@ mod tests { ))), true, ); - let page = encode_first_page(field, arr.clone(), LanceFileVersion::V2_2).await; + let page = encode_first_page(field, arr.clone(), TestEncoding::StructuralU32).await; let PageEncoding::Structural(layout) = &page.description else { panic!("Expected structural encoding"); @@ -7823,8 +9540,7 @@ mod tests { assert_eq!(page.data.len(), 2); let test_cases = TestCases::default() - .with_min_file_version(LanceFileVersion::V2_2) - .with_max_file_version(LanceFileVersion::V2_2) + .with_encoding(TestEncoding::StructuralU32) .with_page_sizes(vec![4096]); check_round_trip_encoding_of_data(vec![arr], &test_cases, HashMap::new()).await; } @@ -7850,7 +9566,7 @@ mod tests { ), true, ); - let page = encode_first_page(field, arr.clone(), LanceFileVersion::V2_2).await; + let page = encode_first_page(field, arr.clone(), TestEncoding::StructuralU32).await; if let PageEncoding::Structural(layout) = &page.description { assert!( @@ -7860,8 +9576,7 @@ mod tests { } let test_cases = TestCases::default() - .with_min_file_version(LanceFileVersion::V2_2) - .with_max_file_version(LanceFileVersion::V2_2) + .with_encoding(TestEncoding::StructuralU32) .with_page_sizes(vec![4096]); check_round_trip_encoding_of_data(vec![arr], &test_cases, HashMap::new()).await; } @@ -7872,7 +9587,7 @@ mod tests { let arr: ArrayRef = Arc::new(arrow_array::Int32Array::from(vec![7; 1024])); let field = arrow_schema::Field::new("c", DataType::Int32, true); - let page = encode_first_page(field, arr.clone(), LanceFileVersion::V2_1).await; + let page = encode_first_page(field, arr.clone(), TestEncoding::StructuralU16).await; let PageEncoding::Structural(layout) = &page.description else { return; @@ -7883,8 +9598,7 @@ mod tests { ); let test_cases = TestCases::default() - .with_min_file_version(LanceFileVersion::V2_1) - .with_max_file_version(LanceFileVersion::V2_1) + .with_encoding(TestEncoding::StructuralU16) .with_page_sizes(vec![4096]); check_round_trip_encoding_of_data(vec![arr], &test_cases, HashMap::new()).await; } @@ -7895,7 +9609,7 @@ mod tests { let arr: ArrayRef = Arc::new(arrow_array::Int32Array::from(vec![None, None, None])); let field = arrow_schema::Field::new("c", DataType::Int32, true); - let page = encode_first_page(field, arr.clone(), LanceFileVersion::V2_2).await; + let page = encode_first_page(field, arr.clone(), TestEncoding::StructuralU32).await; let PageEncoding::Structural(layout) = &page.description else { panic!("Expected structural encoding"); @@ -7907,26 +9621,26 @@ mod tests { assert_eq!(page.data.len(), 0); let test_cases = TestCases::default() - .with_min_file_version(LanceFileVersion::V2_2) - .with_max_file_version(LanceFileVersion::V2_2) + .with_encoding(TestEncoding::StructuralU32) .with_page_sizes(vec![4096]); check_round_trip_encoding_of_data(vec![arr], &test_cases, HashMap::new()).await; } #[test] fn test_encode_decode_complex_all_null_vals_roundtrip() { - use crate::compression::{ - DecompressionStrategy, DefaultCompressionStrategy, DefaultDecompressionStrategy, - }; + use crate::compression::{DecompressionStrategy, DefaultDecompressionStrategy}; let values: Arc<[u16]> = Arc::from((0..2048).map(|i| (i % 5) as u16).collect::>()); - let compression_strategy = DefaultCompressionStrategy::default(); + let compression_strategy = crate::testing::test_compression_strategy( + TestEncoding::StructuralU16, + crate::compression_config::CompressionParams::default(), + ); let decompression_strategy = DefaultDecompressionStrategy::default(); let (compressed_buf, encoding) = PrimitiveStructuralEncoder::encode_complex_all_null_vals( &values, - &compression_strategy, + compression_strategy.as_ref(), ) .unwrap(); @@ -7962,7 +9676,8 @@ mod tests { true, ); - let page_v21 = encode_first_page(field.clone(), arr.clone(), LanceFileVersion::V2_1).await; + let page_v21 = + encode_first_page(field.clone(), arr.clone(), TestEncoding::StructuralU16).await; let PageEncoding::Structural(layout_v21) = &page_v21.description else { panic!("Expected structural encoding"); }; @@ -7974,7 +9689,7 @@ mod tests { assert_eq!(layout_v21.num_rep_values, 0); assert_eq!(layout_v21.num_def_values, 0); - let page_v22 = encode_first_page(field, arr, LanceFileVersion::V2_2).await; + let page_v22 = encode_first_page(field, arr, TestEncoding::StructuralU32).await; let PageEncoding::Structural(layout_v22) = &page_v22.description else { panic!("Expected structural encoding"); }; @@ -7993,11 +9708,512 @@ mod tests { (0..1000).map(|i| if i % 2 == 0 { None } else { Some(vec![]) }), ); - let test_cases = TestCases::default().with_min_file_version(LanceFileVersion::V2_2); + let test_cases = TestCases::default().with_u32_structural_encodings(); + check_round_trip_encoding_of_data(vec![Arc::new(list_array)], &test_cases, HashMap::new()) + .await; + } + + #[tokio::test] + async fn test_complex_all_null_constant_def_round_trip() { + use arrow_array::ListArray; + + // Every row is a null list => constant def levels => a single RLE run, + // exercising the lazy run-form decode end to end. + let list_array = ListArray::from_iter_primitive::( + (0..5000).map(|_| None::>>), + ); + + let test_cases = TestCases::default().with_u32_structural_encodings(); check_round_trip_encoding_of_data(vec![Arc::new(list_array)], &test_cases, HashMap::new()) .await; } + fn encoded_u16_frame(levels: &[u16], run_length_width: RunLengthWidth) -> LanceBuffer { + let block = DataBlock::FixedWidth(FixedWidthDataBlock { + data: LanceBuffer::reinterpret_slice(Arc::from(levels)), + bits_per_value: 16, + num_values: levels.len() as u64, + block_info: BlockInfo::new(), + }); + BlockCompressor::compress(&RleEncoder::with_run_length_width(run_length_width), block) + .unwrap() + } + + fn encoded_u16_runs(levels: &[u16], run_length_width: RunLengthWidth) -> RleRuns { + let frame = encoded_u16_frame(levels, run_length_width); + RleDecompressor::with_run_length_width(16, run_length_width) + .decode_u16_runs(frame, levels.len() as u64) + .unwrap() + } + + fn physical_levels(levels: &[u16]) -> LazyLevels { + LazyLevels::Runs(Arc::new(RunStorage::Physical( + encoded_u16_runs(levels, RunLengthWidth::U8).into_owned(), + ))) + } + + fn coalesced_levels(levels: &[u16]) -> LazyLevels { + let mut values = Vec::new(); + let mut ends = RunEndsBuilder::with_capacity(levels.len(), levels.len()); + for (index, &value) in levels.iter().enumerate() { + if values.last() == Some(&value) { + ends.set_last(index + 1).unwrap(); + } else { + values.push(value); + ends.push(index + 1).unwrap(); + } + } + LazyLevels::Runs(Arc::new(RunStorage::Coalesced { + values: values.into_boxed_slice(), + ends: ends.finish(), + })) + } + + #[test] + fn lazy_levels_runs_match_dense() { + // Runs: 3x2, 1x1, 3x3, 0x2 => [3,3,1,3,3,3,0,0] + let expanded: Vec = vec![3, 3, 1, 3, 3, 3, 0, 0]; + let coalesced = coalesced_levels(&expanded); + let physical = physical_levels(&expanded); + let dense = LazyLevels::Dense(ScalarBuffer::::from(expanded.clone())); + let n = expanded.len(); + + assert_eq!(coalesced.len(), n); + assert_eq!(physical.len(), n); + assert_eq!(dense.len(), n); + + // Rows begin at each `max_rep` (3) position; row `num_rows` maps to `len`. + let max_rep = 3u16; + let row_starts: Vec = (0..n).filter(|&i| expanded[i] == max_rep).collect(); + for target in 0..=row_starts.len() as u64 { + let want = row_starts.get(target as usize).copied().unwrap_or(n); + for runs in [&coalesced, &physical] { + let mut cursor = LevelCursor::default(); + assert_eq!( + runs.seek_row_start(&mut cursor, target, max_rep).unwrap(), + want, + "seek_row_start({target})" + ); + } + let mut c_dense = LevelCursor::default(); + assert_eq!( + dense.seek_row_start(&mut c_dense, target, max_rep).unwrap(), + want + ); + } + + // `count_le_cursor` (fresh cursor per range) and `extend_into` agree with + // the dense reference on every sub-range. + for start in 0..=n { + for end in start..=n { + for max in [0u16, 1, 2, 3] { + let want = expanded[start..end].iter().filter(|&&d| d <= max).count() as u64; + for runs in [&coalesced, &physical] { + let mut cursor = RunPosition::default(); + assert_eq!( + runs.count_le_cursor(&mut cursor, start..end, max).0, + want, + "count_le_cursor({start}..{end}, {max})" + ); + } + let mut d_cur = RunPosition::default(); + assert_eq!(dense.count_le_cursor(&mut d_cur, start..end, max).0, want); + } + for runs in [&coalesced, &physical] { + let mut got = Vec::new(); + runs.extend_into(start..end, RunPosition::default(), &mut got); + assert_eq!( + got, + expanded[start..end].to_vec(), + "extend_into({start}..{end})" + ); + } + let mut got_dense = Vec::new(); + dense.extend_into(start..end, RunPosition::default(), &mut got_dense); + assert_eq!(got_dense, expanded[start..end].to_vec()); + } + } + } + + #[test] + fn physical_run_hints_support_deferred_materialization() { + let expanded: Vec = vec![3, 3, 1, 1, 2, 2, 0, 0]; + let physical = physical_levels(&expanded); + let LazyLevels::Runs(runs) = &physical else { + panic!("expected physical runs"); + }; + let mut first_hint = RunPosition::default(); + runs.seek(&mut first_hint, 2); + let mut second_hint = RunPosition::default(); + runs.seek(&mut second_hint, 6); + + let mut second = Vec::new(); + physical.extend_into(6..8, second_hint, &mut second); + let mut first = Vec::new(); + physical.extend_into(2..4, first_hint, &mut first); + assert_eq!(second, expanded[6..8]); + assert_eq!(first, expanded[2..4]); + } + + /// Fuzz parity for the run-oriented complex-all-null drain: the cursor walk + /// over `LazyLevels` must yield the exact level slices and visible + /// count that a brute-force reference over the fully expanded levels does, for + /// dense, physical-run, and coalesced-run forms and arbitrarily shaped range requests. + mod complex_all_null_drain_parity { + use std::ops::Range; + + use arrow_buffer::ScalarBuffer; + use proptest::prelude::*; + + use super::super::{LazyLevels, LevelCursor, RunPosition}; + use super::{coalesced_levels, physical_levels}; + use crate::Result; + + #[derive(Debug, Clone)] + struct DrainInput { + max_rep: u16, + max_visible: u16, + rep: Option>, + def: Option>, + ranges: Vec>, + } + + fn dense_levels(levels: &[u16]) -> LazyLevels { + LazyLevels::Dense(ScalarBuffer::from(levels.to_vec())) + } + + fn rle_levels(levels: &[u16]) -> LazyLevels { + coalesced_levels(levels) + } + + fn seek( + rep: Option<&LazyLevels>, + cursor: &mut LevelCursor, + row: u64, + max_rep: u16, + ) -> Result { + match rep { + Some(rep) => rep.seek_row_start(cursor, row, max_rep), + None => { + cursor.row = row; + cursor.level = row as usize; + Ok(row as usize) + } + } + } + + /// Mirror of `ComplexAllNullPageDecoder::drain`, driving the real + /// `seek_row_start` / `count_le_cursor` with monotonic cursors. + fn simulate_drain( + rep: Option<&LazyLevels>, + def: Option<&LazyLevels>, + max_rep: u16, + max_visible: u16, + ranges: &[Range], + ) -> Result<(Vec>, u64)> { + let mut rep_cursor = LevelCursor::default(); + let mut def_run_cursor = RunPosition::default(); + let mut slices: Vec> = Vec::new(); + let mut visible = 0u64; + for range in ranges { + let level_start = seek(rep, &mut rep_cursor, range.start, max_rep)?; + let level_end = seek(rep, &mut rep_cursor, range.end, max_rep)?; + visible += match def { + Some(def) => { + def.count_le_cursor( + &mut def_run_cursor, + level_start..level_end, + max_visible, + ) + .0 + } + None => (level_end - level_start) as u64, + }; + match slices.last_mut() { + Some(last) if last.end == level_start => last.end = level_end, + _ => slices.push(level_start..level_end), + } + } + Ok((slices, visible)) + } + + /// Independent brute-force reference over fully expanded levels. + fn reference_drain( + rep: Option<&[u16]>, + def: Option<&[u16]>, + max_rep: u16, + max_visible: u16, + ranges: &[Range], + ) -> (Vec>, u64) { + let total_levels = rep + .map(|r| r.len()) + .or_else(|| def.map(|d| d.len())) + .unwrap_or(0); + // Level index where each row starts (or `total_levels` for the end row). + let row_starts: Vec = match rep { + Some(rep) => (0..rep.len()).filter(|&i| rep[i] == max_rep).collect(), + None => (0..total_levels).collect(), + }; + let level_of_row = |row: u64| { + row_starts + .get(row as usize) + .copied() + .unwrap_or(total_levels) + }; + + let mut slices: Vec> = Vec::new(); + let mut visible = 0u64; + for range in ranges { + let ls = level_of_row(range.start); + let le = level_of_row(range.end); + visible += match def { + Some(def) => def[ls..le].iter().filter(|&&d| d <= max_visible).count() as u64, + None => (le - ls) as u64, + }; + match slices.last_mut() { + Some(last) if last.end == ls => last.end = le, + _ => slices.push(ls..le), + } + } + (slices, visible) + } + + fn ranges_strategy(num_rows: u64) -> BoxedStrategy>> { + if num_rows == 0 { + return Just(Vec::new()).boxed(); + } + // (gap, len) pairs; a zero gap yields ranges adjacent in row space, + // which exercises the level-slice coalescing path. + proptest::collection::vec((0u64..=3, 1u64..=4), 0..=8) + .prop_map(move |pairs| { + let mut ranges = Vec::new(); + let mut pos = 0u64; + for (gap, len) in pairs { + pos = pos.saturating_add(gap); + if pos >= num_rows { + break; + } + let end = (pos + len).min(num_rows); + ranges.push(pos..end); + pos = end; + } + ranges + }) + .boxed() + } + + fn drain_input() -> impl Strategy { + ( + 1u16..=3, + 0u16..=3, + any::(), + any::(), + 1usize..=48, + ) + .prop_flat_map(|(max_rep, max_visible, has_rep, has_def, len)| { + // Complex-all-null always has definition levels when there is + // no repetition, so force `def` present in that case. + let has_def = has_def || !has_rep; + let rep = if has_rep { + proptest::collection::vec(0u16..=max_rep, len) + .prop_map(move |mut v| { + // Row 0 must start at a max-rep boundary. + v[0] = max_rep; + Some(v) + }) + .boxed() + } else { + Just(None).boxed() + }; + let def = if has_def { + proptest::collection::vec(0u16..=(max_visible + 2), len) + .prop_map(Some) + .boxed() + } else { + Just(None).boxed() + }; + (Just(max_rep), Just(max_visible), rep, def) + }) + .prop_flat_map(|(max_rep, max_visible, rep, def)| { + let num_rows = match &rep { + Some(rep) => rep.iter().filter(|&&v| v == max_rep).count() as u64, + None => def.as_ref().map(|d| d.len() as u64).unwrap_or(0), + }; + ranges_strategy(num_rows).prop_map(move |ranges| DrainInput { + max_rep, + max_visible, + rep: rep.clone(), + def: def.clone(), + ranges, + }) + }) + } + + proptest! { + #![proptest_config(ProptestConfig::with_cases(512))] + + #[test] + fn drain_matches_reference(input in drain_input()) { + let DrainInput { max_rep, max_visible, rep, def, ranges } = input; + + let reference = + reference_drain(rep.as_deref(), def.as_deref(), max_rep, max_visible, &ranges); + + let rep_dense = rep.as_deref().map(dense_levels); + let def_dense = def.as_deref().map(dense_levels); + let got_dense = + simulate_drain(rep_dense.as_ref(), def_dense.as_ref(), max_rep, max_visible, &ranges) + .unwrap(); + prop_assert_eq!(&got_dense, &reference, "dense form diverged from reference"); + + let rep_rle = rep.as_deref().map(rle_levels); + let def_rle = def.as_deref().map(rle_levels); + let got_rle = + simulate_drain(rep_rle.as_ref(), def_rle.as_ref(), max_rep, max_visible, &ranges) + .unwrap(); + prop_assert_eq!(&got_rle, &reference, "rle form diverged from reference"); + + let rep_physical = rep.as_deref().map(physical_levels); + let def_physical = def.as_deref().map(physical_levels); + let got_physical = + simulate_drain(rep_physical.as_ref(), def_physical.as_ref(), max_rep, max_visible, &ranges) + .unwrap(); + prop_assert_eq!(&got_physical, &reference, "physical form diverged from reference"); + } + } + } + + #[test] + fn lazy_levels_runs_are_compact() { + let single_run = |n: usize| { + let mut ends = RunEndsBuilder::with_capacity(n, 1); + ends.push(n).unwrap(); + LazyLevels::Runs(Arc::new(RunStorage::Coalesced { + values: vec![1u16].into_boxed_slice(), + ends: ends.finish(), + })) + }; + // Run-form footprint is independent of the logical length within an end width... + assert_eq!(single_run(100).deep_size(), single_run(10_000).deep_size()); + assert!(single_run(10_000_000).deep_size() < 100); + assert_eq!(single_run(10_000_000).len(), 10_000_000); + // ...while Dense pays 2 bytes per value. + assert_eq!( + LazyLevels::Dense(ScalarBuffer::::from(vec![1u16; 1000])).deep_size(), + 2000 + ); + } + + #[test] + fn lazy_levels_selects_smallest_representation() { + let runs = encoded_u16_runs(&[7u16; 10], RunLengthWidth::U8); + assert_eq!(LazyLevels::select_plan(&runs), LevelPlan::Dense); + + let equal_size: Vec = std::iter::repeat_n(0, 256) + .chain(std::iter::repeat_n(1, 100)) + .chain(std::iter::repeat_n(2, 100)) + .collect(); + let runs = encoded_u16_runs(&equal_size, RunLengthWidth::U8); + assert_eq!(LazyLevels::select_plan(&runs), LevelPlan::Coalesced); + + let moderate_runs: Vec = (0..250) + .flat_map(|run| std::iter::repeat_n((run % 2) as u16, 4)) + .collect(); + let runs = encoded_u16_runs(&moderate_runs, RunLengthWidth::U8); + assert_eq!(LazyLevels::select_plan(&runs), LevelPlan::Physical); + + let split_constant = vec![7u16; 5000]; + let runs = encoded_u16_runs(&split_constant, RunLengthWidth::U8); + assert_eq!(LazyLevels::select_plan(&runs), LevelPlan::Coalesced); + + let high_density: Vec = (0..70_000).map(|index| (index % 2) as u16).collect(); + let runs = encoded_u16_runs(&high_density, RunLengthWidth::U32); + assert_eq!(LazyLevels::select_plan(&runs), LevelPlan::Dense); + } + + #[test] + fn physical_runs_detach_from_large_encoded_frame() { + let levels: Vec = (0..250) + .flat_map(|run| std::iter::repeat_n((run % 2) as u16, 4)) + .collect(); + let frame = encoded_u16_frame(&levels, RunLengthWidth::U8); + let frame_offset = 4096; + let mut allocation = vec![0; frame_offset + frame.len() + 1_000_000]; + allocation[frame_offset..frame_offset + frame.len()].copy_from_slice(frame.as_ref()); + let frame = LanceBuffer::from(allocation).slice_with_length(frame_offset, frame.len()); + let runs = RleDecompressor::with_run_length_width(16, RunLengthWidth::U8) + .decode_u16_runs(frame, levels.len() as u64) + .unwrap(); + + assert_eq!(LazyLevels::select_plan(&runs), LevelPlan::Physical); + let cached = LazyLevels::from_rle_runs(runs).unwrap(); + assert!( + matches!(cached, LazyLevels::Runs(ref runs) if matches!(runs.as_ref(), RunStorage::Physical(_))) + ); + assert_eq!(cached.len(), levels.len()); + assert!(cached.deep_size() < 4096); + } + + #[test] + fn complex_all_null_levels_reject_invalid_values_and_lengths() { + let invalid_levels = vec![0u16, 3]; + for levels in [ + LazyLevels::Dense(ScalarBuffer::from(invalid_levels.clone())), + physical_levels(&invalid_levels), + coalesced_levels(&invalid_levels), + ] { + let error = validate_complex_all_null_levels(&None, &Some(levels), 0, 2).unwrap_err(); + assert!(matches!(error, lance_core::Error::InvalidInput { .. })); + assert!(error.to_string().contains("Invalid definition level 3")); + } + + let rep = Some(LazyLevels::Dense(ScalarBuffer::from(vec![0u16; 2]))); + let def = Some(LazyLevels::Dense(ScalarBuffer::from(vec![0u16]))); + let error = validate_complex_all_null_levels(&rep, &def, 0, 0).unwrap_err(); + assert!(matches!(error, lance_core::Error::InvalidInput { .. })); + assert!( + error + .to_string() + .contains("repetition has 2, definition has 1") + ); + } + + #[test] + fn block_levels_reject_malformed_payload_size() { + let block = DataBlock::FixedWidth(FixedWidthDataBlock { + data: LanceBuffer::from(vec![0]), + bits_per_value: 16, + num_values: 1, + block_info: BlockInfo::new(), + }); + let error = dense_levels_from_block(block, 1, "definition").unwrap_err(); + assert!(matches!(error, lance_core::Error::InvalidInput { .. })); + assert!( + error + .to_string() + .contains("expected 2 bytes for 1 values, got 1") + ); + } + + #[test] + fn complex_all_null_level_codec_validates_rle_metadata() { + let encoding = pb21::CompressiveEncoding { + compression: Some(Compression::Rle(Box::new(pb21::Rle { + values: None, + run_lengths: Some(Box::new(ProtobufUtils21::flat(8, None))), + }))), + }; + + let error = LevelCodec::try_new(Some(&encoding), &DefaultDecompressionStrategy::default()) + .unwrap_err(); + assert!(matches!(error, lance_core::Error::InvalidInput { .. })); + assert!( + error + .to_string() + .contains("RLE compression missing values encoding") + ); + } + // https://github.com/lance-format/lance/issues/6681 #[tokio::test] async fn test_sparse_boolean_list_roundtrip() { @@ -8015,7 +10231,77 @@ mod tests { } let list_array = Arc::new(list_builder.finish()); - let test_cases = TestCases::default().with_min_file_version(LanceFileVersion::V2_1); + let test_cases = TestCases::default().with_structural_encodings(); check_round_trip_encoding_of_data(vec![list_array], &test_cases, HashMap::new()).await; } + + fn truncated_tail_details() -> std::sync::Arc { + use crate::compression::VariablePerValueDecompressor; + use crate::encodings::physical::binary::VariableDecoder; + use crate::repdef::{ControlWordParser, DefinitionInterpretation}; + use std::sync::Arc; + Arc::new(super::FullZipDecodeDetails { + value_decompressor: super::PerValueDecompressor::Variable(Arc::new( + VariableDecoder::default(), + ) + as Arc), + def_meaning: vec![DefinitionInterpretation::NullableItem].into(), + ctrl_word_parser: ControlWordParser::new(0, 0), + max_rep: 0, + max_visible_def: 0, + }) + } + + fn decode_variable_full_zip( + buf: Vec, + bits_per_offset: u8, + ) -> lance_core::Result { + use std::collections::VecDeque; + let mut data = VecDeque::new(); + data.push_back(crate::buffer::LanceBuffer::from(buf)); + super::VariableFullZipDecoder::new( + truncated_tail_details(), + data, + 1, + bits_per_offset, + bits_per_offset, + ) + } + + /// A well-formed length prefix decodes without incident, for both widths. + #[test] + fn variable_full_zip_wellformed_length_prefix() { + assert!(decode_variable_full_zip(0u32.to_le_bytes().to_vec(), 32).is_ok()); + assert!(decode_variable_full_zip(0u64.to_le_bytes().to_vec(), 64).is_ok()); + } + + /// A page whose item walk ends with a partial length prefix must surface a + /// corrupt-file error rather than read past the end of the buffer. + /// + /// This asserts the error variant and message rather than merely expecting a + /// panic: before the length prefix was bounds checked, the read was + /// `get_unchecked` behind a `debug_assert!`, so a debug build panicked here + /// (which a `#[should_panic]` test would have accepted as a pass) while a + /// release build read up to 8 bytes out of a 4 byte allocation. + #[test] + fn variable_full_zip_truncated_length_prefix_is_corrupt_file() { + use lance_core::Error; + + for (bits, buf_len) in [(32u8, 3usize), (64u8, 4usize)] { + let err = decode_variable_full_zip(vec![0xAA; buf_len], bits) + .expect_err("a truncated length prefix must not decode"); + assert!( + matches!(err, Error::CorruptFile { .. }), + "expected CorruptFile for a {}-bit prefix with {} byte(s), got: {:?}", + bits, + buf_len, + err + ); + let msg = err.to_string(); + assert!( + msg.contains("truncated length prefix"), + "error should say what is wrong, got: {msg}" + ); + } + } } diff --git a/rust/lance-encoding/src/encodings/logical/primitive/blob.rs b/rust/lance-encoding/src/encodings/logical/primitive/blob.rs index eed3e584b7e..52a7039b2b6 100644 --- a/rust/lance-encoding/src/encodings/logical/primitive/blob.rs +++ b/rust/lance-encoding/src/encodings/logical/primitive/blob.rs @@ -258,7 +258,9 @@ impl BlobPageScheduler { let bytes = read_fut.await?; let mut bytes_iter = bytes.into_iter(); for blob in loaded_blobs.iter_mut() { - if blob.def == 0 { + // Empty values have def == 0 too but scheduled no read; their + // bytes were set at scheduling time. + if blob.def == 0 && blob.bytes.is_none() { blob.set_bytes(bytes_iter.next().expect_ok()?); } } @@ -364,7 +366,17 @@ impl StructuralPageScheduler for BlobPageScheduler { if size == 0 { let rep = (position & 0xFFFF) as u16; let def = ((position >> 16) & 0xFFFF) as u16; - loaded_blobs.push(LoadedBlob::new(rep, def)); + let mut blob = LoadedBlob::new(rep, def); + if def == 0 { + // A size-0 descriptor with definition level 0 is a + // valid, empty value (nulls carry their non-zero + // packed rep/def levels in `position`). No read is + // scheduled for it, so it gets its zero-length bytes + // here rather than consuming another blob's read + // result in the load task. + blob.set_bytes(Bytes::new()); + } + loaded_blobs.push(blob); } else { loaded_blobs.push(LoadedBlob::new(0, 0)); ranges_to_read.push(position..(position + size)); diff --git a/rust/lance-encoding/src/encodings/logical/primitive/chunk_index.rs b/rust/lance-encoding/src/encodings/logical/primitive/chunk_index.rs new file mode 100644 index 00000000000..19fb0fdb041 --- /dev/null +++ b/rust/lance-encoding/src/encodings/logical/primitive/chunk_index.rs @@ -0,0 +1,536 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright The Lance Authors + +//! Compact per-page chunk index for the mini-block structural encoding. +//! +//! The chunk index is stored on disk in an extremely compressed form that +//! requires a lot of CPU to work with. However, extracting it out to its full +//! width can be RAM-intensive. As a compromise we extract into a prefix-sum +//! array that we fit into `u32` if possible and we avoid storing per-block row +//! counts when those are redundant. The scheduler looks chunks up by index +//! (byte range, leaf value count) and by row (which chunk holds a row). +//! +//! ```text +//! MiniBlockChunkIndex +//! |- base: u64 absolute file position of the value buffer +//! |- byte_starts: PrefixSums cumulative chunk byte sizes (all pages) +//! `- rows: RowMapping +//! |- UniformFlat { .. } flat page, uniform leaf chunking (arithmetic) +//! |- Flat { value_starts } flat page, non-uniform leaf chunking +//! `- Nested { row_starts, .. } repetition present; rows tracked as prefix sums +//! ``` + +use std::ops::Range; + +use arrow_buffer::{BooleanBuffer, BooleanBufferBuilder}; +use lance_core::cache::{Context, DeepSizeOf}; + +/// Cumulative (prefix-sum) array of length `num_chunks + 1` (entry `0` is `0`, +/// the last entry is the grand total). Stored as `u32` when the total fits, +/// else `u64`. +#[derive(Debug, DeepSizeOf)] +pub enum PrefixSums { + U32(Vec), + U64(Vec), +} + +impl PrefixSums { + /// Builds the cumulative array from per-chunk `deltas`. `total` selects the + /// storage width (callers must pass the true sum); `num_chunks` only pre-sizes. + pub fn from_deltas(deltas: impl Iterator, num_chunks: usize, total: u64) -> Self { + if total <= u32::MAX as u64 { + let mut values = Vec::with_capacity(num_chunks + 1); + let mut acc = 0u32; + values.push(0); + for delta in deltas { + acc += delta as u32; + values.push(acc); + } + debug_assert_eq!(values.len(), num_chunks + 1); + debug_assert_eq!(acc as u64, total); + Self::U32(values) + } else { + let mut values = Vec::with_capacity(num_chunks + 1); + let mut acc = 0u64; + values.push(0); + for delta in deltas { + acc += delta; + values.push(acc); + } + debug_assert_eq!(values.len(), num_chunks + 1); + debug_assert_eq!(acc, total); + Self::U64(values) + } + } + + /// Builds a `PrefixSums` from an already-cumulative array (`[0, .., total]`), + /// narrowing to `u32` when the total fits. Avoids the deltas buffer + /// [`Self::from_deltas`] would need. + fn from_prefix(prefix: Vec) -> Self { + debug_assert!(!prefix.is_empty()); + debug_assert_eq!(prefix[0], 0); + let total = prefix.last().copied().unwrap_or(0); + if total <= u32::MAX as u64 { + Self::U32(prefix.into_iter().map(|v| v as u32).collect()) + } else { + Self::U64(prefix) + } + } + + /// Cumulative value at position `i` (i.e. the start of chunk `i`). + pub fn get(&self, i: usize) -> u64 { + match self { + Self::U32(values) => values[i] as u64, + Self::U64(values) => values[i], + } + } + + /// Start and end of chunk `i` (positions `i`, `i + 1`) behind one width + /// match -- halves the branching of two `get` calls on the hot per-chunk path. + pub fn get_pair(&self, i: usize) -> (u64, u64) { + match self { + Self::U32(values) => (values[i] as u64, values[i + 1] as u64), + Self::U64(values) => (values[i], values[i + 1]), + } + } + + /// Number of chunks (array length minus the trailing total). + pub fn num_chunks(&self) -> usize { + match self { + Self::U32(values) => values.len() - 1, + Self::U64(values) => values.len() - 1, + } + } + + /// Size of chunk `i` (the delta between consecutive cumulative values). + pub fn delta(&self, i: usize) -> u64 { + let (start, end) = self.get_pair(i); + end - start + } + + /// Index of the chunk whose half-open span `[get(i), get(i+1))` contains + /// `value`. On an exact hit against a chunk start, returns the *first* chunk + /// with that start (chunks can share a start row). + pub fn find(&self, value: u64) -> usize { + // Match the width once, then binary-search only the starts (not the + // trailing total). `partition_point` already yields the first of any + // duplicated starts; the `idx - 1` fallback is safe since `get(0) == 0`. + match self { + Self::U32(values) => { + let starts = &values[..values.len() - 1]; + let idx = starts.partition_point(|&start| (start as u64) < value); + if idx < starts.len() && starts[idx] as u64 == value { + idx + } else { + idx - 1 + } + } + Self::U64(values) => { + let starts = &values[..values.len() - 1]; + let idx = starts.partition_point(|&start| start < value); + if idx < starts.len() && starts[idx] == value { + idx + } else { + idx - 1 + } + } + } + } +} + +/// Leaf value counts per chunk, needed to decode. Tracked only for nested +/// pages; flat pages read items off the row mapping (rows == items). +#[derive(Debug, DeepSizeOf)] +pub enum ItemCounts { + /// Every non-last chunk holds the same number of values. + Uniform { + values_per_chunk: u64, + last_chunk_values: u64, + }, + /// `log2` of each chunk's value count, stored as one byte per chunk rather + /// than the full count because this index stays cached in RAM; the last + /// chunk is handled via `last_chunk_values`. + PerChunkLog { + logs: Vec, + last_chunk_values: u64, + }, +} + +impl ItemCounts { + fn get(&self, i: usize, num_chunks: usize) -> u64 { + match self { + Self::Uniform { + values_per_chunk, + last_chunk_values, + } => { + if i == num_chunks - 1 { + *last_chunk_values + } else { + *values_per_chunk + } + } + Self::PerChunkLog { + logs, + last_chunk_values, + } => { + if i == num_chunks - 1 { + *last_chunk_values + } else { + 1u64 << logs[i] + } + } + } + } +} + +/// How row ranges map onto chunks. +/// +/// Flat pages have row == value index and no preamble/trailer. Nested pages +/// track rows separately from leaf items and store a trailer bit per chunk; a +/// chunk's preamble is the previous chunk's trailer. +#[derive(Debug)] +pub enum RowMapping { + /// Flat page whose non-last chunks all hold `values_per_chunk` values, so + /// row->chunk is pure arithmetic. + UniformFlat { + values_per_chunk: u64, + last_chunk_values: u64, + num_chunks: usize, + }, + /// Flat page with non-uniform chunk sizes; `value_starts` are the cumulative + /// value counts (final entry == number of items in the page). + Flat { value_starts: PrefixSums }, + /// Nested page. `row_starts` are cumulative row counts (final entry == + /// number of rows in the page); `has_trailer[i]` is set when chunk `i` ends + /// with a partial list. + Nested { + row_starts: PrefixSums, + has_trailer: BooleanBuffer, + item_counts: ItemCounts, + }, +} + +impl DeepSizeOf for RowMapping { + fn deep_size_of_children(&self, context: &mut Context) -> usize { + match self { + Self::UniformFlat { .. } => 0, + Self::Flat { value_starts } => value_starts.deep_size_of_children(context), + Self::Nested { + row_starts, + has_trailer, + item_counts, + } => { + row_starts.deep_size_of_children(context) + + has_trailer.len().div_ceil(8) + + item_counts.deep_size_of_children(context) + } + } + } +} + +/// Compact per-page chunk index that avoids fully materializing the repetition +/// index into u64's to save RAM. See the module docs for the layout. +#[derive(Debug)] +pub struct MiniBlockChunkIndex { + base: u64, + byte_starts: PrefixSums, + rows: RowMapping, +} + +impl MiniBlockChunkIndex { + pub fn new(base: u64, byte_starts: PrefixSums, rows: RowMapping) -> Self { + Self { + base, + byte_starts, + rows, + } + } + + /// Number of chunks in the page. + pub fn num_chunks(&self) -> usize { + self.byte_starts.num_chunks() + } + + /// Absolute byte range of chunk `i` within the file. + pub fn byte_range(&self, i: usize) -> Range { + let (start, end) = self.byte_starts.get_pair(i); + (self.base + start)..(self.base + end) + } + + /// Number of leaf values in chunk `i` (passed to the value decompressor). + pub fn items_in_chunk(&self, i: usize) -> u64 { + let num_chunks = self.num_chunks(); + match &self.rows { + RowMapping::UniformFlat { + values_per_chunk, + last_chunk_values, + .. + } => { + if i == num_chunks - 1 { + *last_chunk_values + } else { + *values_per_chunk + } + } + RowMapping::Flat { value_starts } => value_starts.delta(i), + RowMapping::Nested { item_counts, .. } => item_counts.get(i, num_chunks), + } + } + + /// Index of the chunk that contains `row`. + pub fn find_chunk(&self, row: u64) -> usize { + match &self.rows { + RowMapping::UniformFlat { + values_per_chunk, + num_chunks, + .. + } => ((row / values_per_chunk) as usize).min(num_chunks - 1), + RowMapping::Flat { value_starts } => value_starts.find(row), + RowMapping::Nested { row_starts, .. } => row_starts.find(row), + } + } + + /// First row (relative to the page) that begins in chunk `i`. + pub fn first_row(&self, i: usize) -> u64 { + match &self.rows { + RowMapping::UniformFlat { + values_per_chunk, .. + } => i as u64 * values_per_chunk, + RowMapping::Flat { value_starts } => value_starts.get(i), + RowMapping::Nested { row_starts, .. } => row_starts.get(i), + } + } + + /// Number of rows that start in chunk `i`, including a trailer but not a + /// preamble (the previous `starts_including_trailer`). + pub fn rows_in_chunk(&self, i: usize) -> u64 { + let num_chunks = self.num_chunks(); + match &self.rows { + RowMapping::UniformFlat { + values_per_chunk, + last_chunk_values, + .. + } => { + if i == num_chunks - 1 { + *last_chunk_values + } else { + *values_per_chunk + } + } + RowMapping::Flat { value_starts } => value_starts.delta(i), + RowMapping::Nested { row_starts, .. } => row_starts.delta(i), + } + } + + /// Whether chunk `i` begins with a preamble (a continuation of the previous + /// chunk's list). Always false for flat pages; for nested pages this is the + /// previous chunk's trailer. + pub fn has_preamble(&self, i: usize) -> bool { + match &self.rows { + RowMapping::Nested { has_trailer, .. } => i > 0 && has_trailer.value(i - 1), + _ => false, + } + } + + /// Whether chunk `i` ends with a trailer (a partial list continued in the + /// next chunk). Always false for flat pages. + pub fn has_trailer(&self, i: usize) -> bool { + match &self.rows { + RowMapping::Nested { has_trailer, .. } => has_trailer.value(i), + _ => false, + } + } + + /// Name of the active row-mapping variant, used to assert detection in tests. + #[cfg(test)] + pub fn row_mapping_debug(&self) -> &'static str { + match &self.rows { + RowMapping::UniformFlat { .. } => "uniform_flat", + RowMapping::Flat { .. } => "flat", + RowMapping::Nested { .. } => "nested", + } + } + + /// Builds a nested index from raw repetition-index bytes, using placeholder + /// byte offsets and item counts. Only the row axis is populated, which is + /// all the scheduler exercises. + #[cfg(test)] + pub fn new_nested_for_test(rep_bytes: &[u8], stride: usize) -> Self { + let (row_starts, has_trailer) = parse_nested_rep(rep_bytes, stride); + let num_chunks = row_starts.num_chunks(); + let byte_starts = PrefixSums::from_deltas( + std::iter::repeat_n(8u64, num_chunks), + num_chunks, + 8 * num_chunks as u64, + ); + Self { + base: 0, + byte_starts, + rows: RowMapping::Nested { + row_starts, + has_trailer, + item_counts: ItemCounts::Uniform { + values_per_chunk: 1, + last_chunk_values: 1, + }, + }, + } + } +} + +impl DeepSizeOf for MiniBlockChunkIndex { + fn deep_size_of_children(&self, context: &mut Context) -> usize { + self.byte_starts.deep_size_of_children(context) + self.rows.deep_size_of_children(context) + } +} + +/// Parses a mini-block repetition index into the compact nested row mapping. +/// +/// Bytes are `u64`s in groups of `stride`; the first two are `ends` (lists +/// finishing in the chunk) and `partial` (leftover items). Only cumulative row +/// starts and a trailer bit are kept: `has_preamble[i] = has_trailer[i-1]` and +/// `starts_including_trailer = ends + has_trailer - has_preamble`. +pub fn parse_nested_rep(rep_bytes: &[u8], stride: usize) -> (PrefixSums, BooleanBuffer) { + // Read the two `u64`s per group straight from the little-endian bytes rather + // than copying the buffer to reinterpret it. The caller guarantees + // `rep_bytes.len() % 8 == 0`, so the 8-byte windows stay in bounds. + const WORD: usize = std::mem::size_of::(); + let read_word = |word_idx: usize| -> u64 { + let byte = word_idx * WORD; + u64::from_le_bytes(rep_bytes[byte..byte + WORD].try_into().unwrap()) + }; + let num_chunks = (rep_bytes.len() / WORD) / stride; + + let mut has_trailer_builder = BooleanBufferBuilder::new(num_chunks); + // Accumulate the cumulative row starts in a single pass (entry 0 is 0, the + // trailing entry is the total) so there is no separate deltas buffer. + let mut row_starts = Vec::with_capacity(num_chunks + 1); + row_starts.push(0u64); + let mut acc = 0u64; + let mut chunk_has_preamble = false; + + for i in 0..num_chunks { + let base_idx = i * stride; + let ends = read_word(base_idx); + let partial = read_word(base_idx + 1); + + let has_trailer = partial > 0; + let starts_including_trailer = ends + (has_trailer as u64) - (chunk_has_preamble as u64); + + has_trailer_builder.append(has_trailer); + acc += starts_including_trailer; + row_starts.push(acc); + + chunk_has_preamble = has_trailer; + } + + ( + PrefixSums::from_prefix(row_starts), + has_trailer_builder.finish(), + ) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::buffer::LanceBuffer; + + /// Reference decode: the previous per-block repetition index, used as an + /// oracle for the compact `parse_nested_rep`. + struct RefBlock { + first_row: u64, + starts_including_trailer: u64, + has_preamble: bool, + has_trailer: bool, + } + + fn reference_decode(rep_bytes: &[u8], stride: usize) -> Vec { + let buffer = LanceBuffer::from(rep_bytes.to_vec()); + let u64_slice = buffer.borrow_to_typed_slice::(); + let n = u64_slice.len() / stride; + let mut blocks = Vec::with_capacity(n); + let mut chunk_has_preamble = false; + let mut offset = 0u64; + for i in 0..n { + let base_idx = i * stride; + let ends = u64_slice[base_idx]; + let partial = u64_slice[base_idx + 1]; + let has_trailer = partial > 0; + let starts_including_trailer = + ends + (has_trailer as u64) - (chunk_has_preamble as u64); + blocks.push(RefBlock { + first_row: offset, + starts_including_trailer, + has_preamble: chunk_has_preamble, + has_trailer, + }); + chunk_has_preamble = has_trailer; + offset += starts_including_trailer; + } + blocks + } + + fn rep_bytes(values: &[u64]) -> Vec { + values.iter().flat_map(|v| v.to_le_bytes()).collect() + } + + #[test] + fn test_prefix_sums_u32() { + let sums = PrefixSums::from_deltas([2u64, 3, 5].into_iter(), 3, 10); + assert!(matches!(sums, PrefixSums::U32(_))); + assert_eq!(sums.num_chunks(), 3); + assert_eq!(sums.get(0), 0); + assert_eq!(sums.get(1), 2); + assert_eq!(sums.get(3), 10); + assert_eq!(sums.delta(1), 3); + } + + #[test] + fn test_prefix_sums_u64_selected_by_total() { + let big = u32::MAX as u64 + 1; + let sums = PrefixSums::from_deltas([big].into_iter(), 1, big); + assert!(matches!(sums, PrefixSums::U64(_))); + assert_eq!(sums.get(1), big); + } + + #[test] + fn test_prefix_sums_find() { + // Chunk starts: 0, 5, 5, 12 (a zero-width chunk creates a duplicate start) + let sums = PrefixSums::from_deltas([5u64, 0, 7].into_iter(), 3, 12); + // Inside the first chunk + assert_eq!(sums.find(3), 0); + // Exact match on a duplicated start returns the first such chunk + assert_eq!(sums.find(5), 1); + // Inside the last chunk + assert_eq!(sums.find(11), 2); + // Start of the first chunk + assert_eq!(sums.find(0), 0); + } + + #[test] + fn test_parse_nested_rep_matches_reference() { + let cases: Vec> = vec![ + vec![5, 2, 3, 0, 4, 7, 2, 0], + vec![5, 2, 3, 3, 20, 0], + vec![0, 5, 0, 3, 10, 0], + vec![1, 0], + ]; + for values in cases { + let bytes = rep_bytes(&values); + let reference = reference_decode(&bytes, 2); + let (row_starts, has_trailer) = parse_nested_rep(&bytes, 2); + assert_eq!(row_starts.num_chunks(), reference.len()); + for (i, block) in reference.iter().enumerate() { + assert_eq!(row_starts.get(i), block.first_row, "first_row[{i}]"); + assert_eq!( + row_starts.delta(i), + block.starts_including_trailer, + "starts_including_trailer[{i}]" + ); + assert_eq!(has_trailer.value(i), block.has_trailer, "has_trailer[{i}]"); + let derived_preamble = i > 0 && has_trailer.value(i - 1); + assert_eq!(derived_preamble, block.has_preamble, "has_preamble[{i}]"); + } + } + } +} diff --git a/rust/lance-encoding/src/encodings/logical/primitive/layout.rs b/rust/lance-encoding/src/encodings/logical/primitive/layout.rs new file mode 100644 index 00000000000..95df12ede6b --- /dev/null +++ b/rust/lance-encoding/src/encodings/logical/primitive/layout.rs @@ -0,0 +1,72 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright The Lance Authors + +use lance_core::Result; + +use crate::repdef::MiniBlockRepDefBudget; + +/// Runs automatic sparse planning only after the dense mini-block budget makes it useful. +pub(super) fn select_automatic_sparse( + requested_encoding: Option<&str>, + dense_budget: &MiniBlockRepDefBudget, + candidate: impl FnOnce() -> Result>, +) -> Result> { + if requested_encoding.is_some() + || !matches!( + dense_budget, + MiniBlockRepDefBudget::RequiresPageSplit(_) + | MiniBlockRepDefBudget::SingleRowOverBudget(_) + ) + { + return Ok(None); + } + candidate() +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::constants::{ + STRUCTURAL_ENCODING_FULLZIP, STRUCTURAL_ENCODING_MINIBLOCK, STRUCTURAL_ENCODING_SPARSE, + }; + + #[test] + fn within_budget_does_not_construct_sparse_candidate() { + let selected = + select_automatic_sparse::<()>(None, &MiniBlockRepDefBudget::WithinBudget, || { + panic!("within-budget pages must not construct sparse candidates") + }) + .unwrap(); + assert!(selected.is_none()); + } + + #[test] + fn over_budget_selects_only_eligible_candidates() { + let split = MiniBlockRepDefBudget::RequiresPageSplit(Vec::new()); + let selected = select_automatic_sparse(None, &split, || Ok(Some(42))).unwrap(); + assert_eq!(selected, Some(42)); + + let ineligible = select_automatic_sparse::<()>(None, &split, || Ok(None)).unwrap(); + assert!(ineligible.is_none()); + + let unsplittable = MiniBlockRepDefBudget::SingleRowOverBudget(70_000); + let selected = select_automatic_sparse(None, &unsplittable, || Ok(Some(7))).unwrap(); + assert_eq!(selected, Some(7)); + } + + #[test] + fn explicit_modes_and_lance_2_2_do_not_auto_select() { + let split = MiniBlockRepDefBudget::RequiresPageSplit(Vec::new()); + for requested in [ + STRUCTURAL_ENCODING_MINIBLOCK, + STRUCTURAL_ENCODING_FULLZIP, + STRUCTURAL_ENCODING_SPARSE, + ] { + let selected = select_automatic_sparse::<()>(Some(requested), &split, || { + panic!("explicit modes must not invoke automatic sparse planning") + }) + .unwrap(); + assert!(selected.is_none()); + } + } +} diff --git a/rust/lance-encoding/src/encodings/logical/primitive/miniblock.rs b/rust/lance-encoding/src/encodings/logical/primitive/miniblock.rs index 1cf3b9bf581..0ee408a8510 100644 --- a/rust/lance-encoding/src/encodings/logical/primitive/miniblock.rs +++ b/rust/lance-encoding/src/encodings/logical/primitive/miniblock.rs @@ -19,7 +19,8 @@ use lance_core::Result; pub const MAX_MINIBLOCK_BYTES: u64 = 8 * 1024 - 6; const DEFAULT_MAX_MINIBLOCK_VALUES: u64 = 4096; -const MAX_CONFIGURABLE_MINIBLOCK_VALUES: u64 = 32768; +/// Maximum number of values that any mini-block decoder accepts from page metadata. +pub(crate) const MAX_CONFIGURABLE_MINIBLOCK_VALUES: u64 = 32768; fn parse_max_miniblock_values() -> u64 { let val = std::env::var("LANCE_MINIBLOCK_MAX_VALUES") @@ -53,6 +54,29 @@ pub struct MiniBlockCompressed { pub num_values: u64, } +/// Per-page framing details that can affect a mini-block compressor's choice. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct MiniBlockCompressionContext { + common_chunk_buffers: u64, + support_large_chunk: bool, + allow_generic_offsets: bool, +} + +impl MiniBlockCompressionContext { + /// Creates the framing context supplied by the owning mini-block page. + pub fn new( + common_chunk_buffers: u64, + support_large_chunk: bool, + allow_generic_offsets: bool, + ) -> Self { + Self { + common_chunk_buffers, + support_large_chunk, + allow_generic_offsets, + } + } +} + /// Describes the size of a mini-block chunk of data /// /// Mini-block chunks are designed to be small (just a few disk sectors) @@ -112,7 +136,11 @@ pub trait MiniBlockCompressor: std::fmt::Debug + Send + Sync { /// /// This method also returns a description of the encoding applied that will be /// used at decode time to read the data. - fn compress(&self, page: DataBlock) -> Result<(MiniBlockCompressed, CompressiveEncoding)>; + fn compress( + &self, + context: MiniBlockCompressionContext, + page: DataBlock, + ) -> Result<(MiniBlockCompressed, CompressiveEncoding)>; } #[cfg(test)] diff --git a/rust/lance-encoding/src/encodings/logical/primitive/sparse.rs b/rust/lance-encoding/src/encodings/logical/primitive/sparse.rs new file mode 100644 index 00000000000..25cb85f49ed --- /dev/null +++ b/rust/lance-encoding/src/encodings/logical/primitive/sparse.rs @@ -0,0 +1,5614 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright The Lance Authors + +use super::*; +use arrow_array::new_empty_array; +use arrow_buffer::ArrowNativeType; + +fn invalid_enum( + value: std::result::Result, + label: &str, +) -> Result { + value.map_err(|error| { + Error::invalid_input_source( + format!("Sparse structural {label} has an invalid enum value: {error}").into(), + ) + }) +} + +pub(super) mod writer; + +fn usize_from_u64(value: u64, label: &str) -> Result { + usize::try_from(value).map_err(|_| { + Error::invalid_input_source( + format!("Sparse structural {label} {value} exceeds usize::MAX").into(), + ) + }) +} + +/// Native sparse structural representation used by the 2.3 sparse layout. +/// +/// Layers are stored from outer-most to inner-most, matching the order Arrow structural +/// encoders record offsets and validity. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct SparseStructuralPlan { + pub(crate) layers: Vec, + pub(crate) num_items: u64, + pub(crate) num_visible_items: u64, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum SparsePositionSet { + Empty, + All { len: u64 }, + Range { start: u64, len: u64 }, + Explicit(Vec), +} + +impl SparsePositionSet { + pub(crate) fn from_positions( + positions: Vec, + domain_len: u64, + label: &str, + ) -> Result { + if positions.is_empty() { + return Ok(Self::Empty); + } + for window in positions.windows(2) { + let [previous, current] = window else { + continue; + }; + if previous >= current { + return Err(Error::invalid_input_source( + format!("Sparse structural {label} positions must be strictly increasing") + .into(), + )); + } + } + if let Some(position) = positions.iter().find(|position| **position >= domain_len) { + return Err(Error::invalid_input_source( + format!( + "Sparse structural {label} position {} is outside layer with {} slots", + position, domain_len + ) + .into(), + )); + } + + let len = u64::try_from(positions.len()).map_err(|_| { + Error::invalid_input_source( + format!("Sparse structural {label} position count exceeds u64::MAX").into(), + ) + })?; + let first = positions.first().copied().ok_or_else(|| { + Error::invalid_input_source( + format!("Sparse structural {label} positions are unexpectedly empty").into(), + ) + })?; + let last = positions.last().copied().ok_or_else(|| { + Error::invalid_input_source( + format!("Sparse structural {label} positions are unexpectedly empty").into(), + ) + })?; + if first == 0 && len == domain_len && domain_len > 0 && last == domain_len - 1 { + return Ok(Self::All { len: domain_len }); + } + if last - first + 1 == len { + return Ok(Self::Range { start: first, len }); + } + Ok(Self::Explicit(positions)) + } + + pub(crate) fn empty() -> Self { + Self::Empty + } + + pub(crate) fn all(len: u64) -> Self { + if len == 0 { + Self::Empty + } else { + Self::All { len } + } + } + + pub(crate) fn range(start: u64, len: u64) -> Self { + if len == 0 { + Self::Empty + } else { + Self::Range { start, len } + } + } + + pub(crate) fn len(&self) -> u64 { + match self { + Self::Empty => 0, + Self::All { len } | Self::Range { len, .. } => *len, + Self::Explicit(positions) => positions.len() as u64, + } + } + + pub(crate) fn is_empty(&self) -> bool { + self.len() == 0 + } + + pub(crate) fn deep_size(&self) -> usize { + match self { + Self::Explicit(positions) => positions.len() * std::mem::size_of::(), + Self::Empty | Self::All { .. } | Self::Range { .. } => 0, + } + } + + pub(crate) fn materialize(&self) -> Result> { + match self { + Self::Empty => Ok(Vec::new()), + Self::All { len } => Self::materialize_range(0, *len), + Self::Range { start, len } => Self::materialize_range(*start, *len), + Self::Explicit(positions) => Ok(positions.clone()), + } + } + + fn materialize_range(start: u64, len: u64) -> Result> { + let len_usize = usize::try_from(len).map_err(|_| { + Error::invalid_input_source( + format!( + "Sparse structural position range length {} exceeds usize::MAX", + len + ) + .into(), + ) + })?; + let end = start.checked_add(len).ok_or_else(|| { + Error::invalid_input_source("Sparse structural position range overflows".into()) + })?; + let mut positions = Vec::with_capacity(len_usize); + positions.extend(start..end); + Ok(positions) + } + + fn contains(&self, position: u64) -> bool { + match self { + Self::Empty => false, + Self::All { len } => position < *len, + Self::Range { start, len } => { + position >= *start && position < start.saturating_add(*len) + } + Self::Explicit(positions) => positions.binary_search(&position).is_ok(), + } + } + + fn is_subset_of(&self, other: &Self, domain_len: u64) -> Result { + self.validate_domain(domain_len, "subset")?; + other.validate_domain(domain_len, "superset")?; + Ok(match self { + Self::Empty => true, + Self::All { .. } => other.len() == domain_len, + Self::Range { start, len } => { + let end = start.checked_add(*len).ok_or_else(|| { + Error::invalid_input_source("Sparse structural subset range overflows".into()) + })?; + match other { + Self::All { .. } => true, + Self::Range { + start: other_start, + len: other_len, + } => { + let other_end = other_start.saturating_add(*other_len); + *start >= *other_start && end <= other_end + } + Self::Explicit(positions) => { + let first = positions.partition_point(|position| *position < *start); + let last = positions.partition_point(|position| *position < end); + u64::try_from(last.saturating_sub(first)).ok() == Some(*len) + } + Self::Empty => false, + } + } + Self::Explicit(positions) => positions.iter().all(|position| other.contains(*position)), + }) + } + + fn is_disjoint(&self, other: &Self, domain_len: u64) -> Result { + self.validate_domain(domain_len, "first disjoint set")?; + other.validate_domain(domain_len, "second disjoint set")?; + let (smaller, larger) = if self.len() <= other.len() { + (self, other) + } else { + (other, self) + }; + Ok(match smaller { + Self::Empty => true, + Self::All { .. } => larger.is_empty(), + Self::Range { start, len } => { + let end = start.saturating_add(*len); + match larger { + Self::Empty => true, + Self::All { .. } => false, + Self::Range { + start: other_start, + len: other_len, + } => end <= *other_start || other_start.saturating_add(*other_len) <= *start, + Self::Explicit(positions) => { + let index = positions.partition_point(|position| *position < *start); + positions.get(index).is_none_or(|position| *position >= end) + } + } + } + Self::Explicit(positions) => { + positions.iter().all(|position| !larger.contains(*position)) + } + }) + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum SparseValidityMeaning { + NullPositions, + ValidPositions, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct SparseValiditySet { + pub(crate) meaning: SparseValidityMeaning, + pub(crate) positions: SparsePositionSet, +} + +impl SparseValiditySet { + pub(crate) fn deep_size(&self) -> usize { + self.positions.deep_size() + } + + fn contains_only_valid_positions( + &self, + positions: &SparsePositionSet, + num_slots: u64, + ) -> Result { + match self.meaning { + SparseValidityMeaning::NullPositions => { + positions.is_disjoint(&self.positions, num_slots) + } + SparseValidityMeaning::ValidPositions => { + positions.is_subset_of(&self.positions, num_slots) + } + } + } + + fn append_to(&self, validity: &mut BooleanBufferBuilder, num_slots: u64) -> Result<()> { + self.positions.validate_domain(num_slots, "validity")?; + let num_slots_usize = usize_from_u64(num_slots, "validity slot count")?; + match (self.meaning, &self.positions) { + (SparseValidityMeaning::NullPositions, SparsePositionSet::Empty) => { + validity.append_n(num_slots_usize, true); + } + (SparseValidityMeaning::ValidPositions, SparsePositionSet::Empty) => { + validity.append_n(num_slots_usize, false); + } + (SparseValidityMeaning::NullPositions, SparsePositionSet::All { .. }) => { + validity.append_n(num_slots_usize, false); + } + (SparseValidityMeaning::ValidPositions, SparsePositionSet::All { .. }) => { + validity.append_n(num_slots_usize, true); + } + (meaning, SparsePositionSet::Range { start, len }) => { + let range_end = start.checked_add(*len).ok_or_else(|| { + Error::invalid_input_source("Sparse structural validity range overflows".into()) + })?; + let default_valid = matches!(meaning, SparseValidityMeaning::NullPositions); + let range_valid = !default_valid; + validity.append_n( + usize_from_u64(*start, "validity range start")?, + default_valid, + ); + validity.append_n(usize_from_u64(*len, "validity range length")?, range_valid); + validity.append_n( + usize_from_u64(num_slots - range_end, "validity range tail")?, + default_valid, + ); + } + (_, SparsePositionSet::Explicit(_)) => { + let mut cursor = SparseValidityCursor::new(self, num_slots, "validity")?; + for slot in 0..num_slots { + validity.append(cursor.is_valid(slot)?); + } + cursor.finish()?; + } + } + Ok(()) + } +} + +impl SparsePositionSet { + fn validate_domain(&self, domain_len: u64, label: &str) -> Result<()> { + match self { + Self::Empty => {} + Self::All { len } => { + if *len != domain_len { + return Err(Error::invalid_input_source( + format!( + "Sparse structural {label} all set length {} does not match domain {}", + len, domain_len + ) + .into(), + )); + } + } + Self::Range { start, len } => { + let end = start.checked_add(*len).ok_or_else(|| { + Error::invalid_input_source( + format!("Sparse structural {label} range overflows").into(), + ) + })?; + if end > domain_len { + return Err(Error::invalid_input_source( + format!( + "Sparse structural {label} range {}..{} is outside domain {}", + start, end, domain_len + ) + .into(), + )); + } + } + Self::Explicit(positions) => { + for window in positions.windows(2) { + let [previous, current] = window else { + continue; + }; + if previous >= current { + return Err(Error::invalid_input_source( + format!( + "Sparse structural {label} positions must be strictly increasing" + ) + .into(), + )); + } + } + if let Some(position) = positions.last() + && *position >= domain_len + { + return Err(Error::invalid_input_source( + format!( + "Sparse structural {label} position {} is outside layer with {} slots", + position, domain_len + ) + .into(), + )); + } + } + } + Ok(()) + } +} + +struct SparsePositionSetCursor<'a> { + set: &'a SparsePositionSet, + explicit: Option>>, +} + +impl<'a> SparsePositionSetCursor<'a> { + fn new(set: &'a SparsePositionSet, domain_len: u64, label: &str) -> Result { + set.validate_domain(domain_len, label)?; + let explicit = match set { + SparsePositionSet::Explicit(positions) => Some(positions.iter().peekable()), + _ => None, + }; + Ok(Self { set, explicit }) + } + + fn contains(&mut self, slot: u64) -> Result { + Ok(match self.set { + SparsePositionSet::Empty => false, + SparsePositionSet::All { .. } => true, + SparsePositionSet::Range { start, len } => { + slot >= *start && slot < start.saturating_add(*len) + } + SparsePositionSet::Explicit(_) => { + let iter = self.explicit.as_mut().ok_or_else(|| { + Error::internal("Sparse structural explicit cursor is missing".to_string()) + })?; + if let Some(position) = iter.peek() + && **position < slot + { + return Err(Error::invalid_input_source( + format!( + "Sparse structural explicit position {} was skipped before slot {}", + **position, slot + ) + .into(), + )); + } + if iter.peek().is_some_and(|position| **position == slot) { + iter.next(); + true + } else { + false + } + } + }) + } + + fn finish(&mut self) -> Result<()> { + if let Some(iter) = self.explicit.as_mut() + && let Some(position) = iter.next() + { + return Err(Error::invalid_input_source( + format!( + "Sparse structural explicit position {} was not consumed", + position + ) + .into(), + )); + } + Ok(()) + } +} + +struct SparseValidityCursor<'a> { + meaning: SparseValidityMeaning, + positions: SparsePositionSetCursor<'a>, +} + +impl<'a> SparseValidityCursor<'a> { + fn new(validity: &'a SparseValiditySet, domain_len: u64, label: &str) -> Result { + Ok(Self { + meaning: validity.meaning, + positions: SparsePositionSetCursor::new(&validity.positions, domain_len, label)?, + }) + } + + fn is_valid(&mut self, slot: u64) -> Result { + let is_stored = self.positions.contains(slot)?; + Ok(match self.meaning { + SparseValidityMeaning::NullPositions => !is_stored, + SparseValidityMeaning::ValidPositions => is_stored, + }) + } + + fn finish(&mut self) -> Result<()> { + self.positions.finish() + } +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum SparseCountSet { + Empty, + Constant { + value: u64, + len: u64, + }, + Explicit { + counts: Arc<[u64]>, + offsets: Arc<[u64]>, + }, +} + +impl SparseCountSet { + pub(crate) fn from_counts(counts: Vec) -> Result { + if counts.is_empty() { + return Ok(Self::Empty); + } + if let Some(first) = counts.first().copied() + && counts.iter().all(|count| *count == first) + { + return Ok(Self::Constant { + value: first, + len: counts.len() as u64, + }); + } + let offsets = offsets_from_counts(&counts)?; + Ok(Self::Explicit { + counts: counts.into(), + offsets: offsets.into(), + }) + } + + pub(crate) fn constant(value: u64, len: u64) -> Self { + if len == 0 { + Self::Empty + } else { + Self::Constant { value, len } + } + } + + pub(crate) fn len(&self) -> u64 { + match self { + Self::Empty => 0, + Self::Constant { len, .. } => *len, + Self::Explicit { counts, .. } => counts.len() as u64, + } + } + + pub(crate) fn deep_size(&self) -> usize { + match self { + Self::Explicit { counts, offsets } => { + counts.len() * std::mem::size_of::() + + offsets.len() * std::mem::size_of::() + } + Self::Empty | Self::Constant { .. } => 0, + } + } + + pub(crate) fn materialize(&self) -> Result> { + match self { + Self::Empty => Ok(Vec::new()), + Self::Constant { value, len } => { + let len = usize::try_from(*len).map_err(|_| { + Error::invalid_input_source( + "Sparse structural count set length exceeds usize::MAX".into(), + ) + })?; + Ok(vec![*value; len]) + } + Self::Explicit { counts, .. } => Ok(counts.to_vec()), + } + } + + pub(crate) fn sum(&self) -> Result { + match self { + Self::Empty => Ok(0), + Self::Constant { value, len } => value.checked_mul(*len).ok_or_else(|| { + Error::invalid_input_source( + format!( + "Sparse structural constant count sum overflows: value={}, len={}", + value, len + ) + .into(), + ) + }), + Self::Explicit { offsets, .. } => offsets.last().copied().ok_or_else(|| { + Error::invalid_input_source( + "Sparse structural explicit count offsets are empty".into(), + ) + }), + } + } + + fn validate_positive(&self) -> Result<()> { + let has_zero = match self { + Self::Empty => false, + Self::Constant { value, .. } => *value == 0, + Self::Explicit { counts, .. } => counts.contains(&0), + }; + if has_zero { + return Err(Error::invalid_input_source( + "Sparse structural non-empty list count is zero".into(), + )); + } + Ok(()) + } +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum SparseStructuralLayerPlan { + Validity { + num_slots: u64, + validity: SparseValiditySet, + }, + List { + num_slots: u64, + num_child_slots: u64, + non_empty_positions: SparsePositionSet, + counts: SparseCountSet, + validity: SparseValiditySet, + }, + FixedSizeList { + num_slots: u64, + dimension: u64, + validity: SparseValiditySet, + }, +} + +impl SparseStructuralPlan { + fn expected_num_items( + layers: &[SparseStructuralLayerPlan], + num_visible_items: u64, + ) -> Result { + layers.iter().try_fold(num_visible_items, |items, layer| { + let additional = match layer { + SparseStructuralLayerPlan::List { + num_slots, + non_empty_positions, + .. + } => num_slots + .checked_sub(non_empty_positions.len()) + .ok_or_else(|| { + Error::invalid_input_source( + "Sparse structural list has more non-empty positions than slots".into(), + ) + })?, + SparseStructuralLayerPlan::Validity { .. } + | SparseStructuralLayerPlan::FixedSizeList { .. } => 0, + }; + items.checked_add(additional).ok_or_else(|| { + Error::invalid_input_source("Sparse structural item count overflows".into()) + }) + }) + } + + fn validate(&self, row_domain: u64) -> Result<()> { + usize_from_u64(self.num_visible_items, "visible item count")?; + let expected_num_items = Self::expected_num_items(&self.layers, self.num_visible_items)?; + if self.num_items != expected_num_items { + return Err(Error::invalid_input_source( + format!( + "Sparse structural item count {} does not match the {} items implied by its layers", + self.num_items, expected_num_items + ) + .into(), + )); + } + let mut expected_slots = row_domain; + for (layer_index, layer) in self.layers.iter().enumerate() { + let (num_slots, num_child_slots, validity) = match layer { + SparseStructuralLayerPlan::Validity { + num_slots, + validity, + } => (*num_slots, *num_slots, validity), + SparseStructuralLayerPlan::List { + num_slots, + num_child_slots, + non_empty_positions, + counts, + validity, + } => { + counts.validate_positive()?; + if non_empty_positions.len() != counts.len() { + return Err(Error::invalid_input_source( + format!( + "Sparse structural list layer {} has {} non-empty positions but {} counts", + layer_index, + non_empty_positions.len(), + counts.len() + ) + .into(), + )); + } + if counts.sum()? != *num_child_slots { + return Err(Error::invalid_input_source( + format!( + "Sparse structural list layer {} count sum does not match {} child slots", + layer_index, num_child_slots + ) + .into(), + )); + } + if !validity.contains_only_valid_positions(non_empty_positions, *num_slots)? { + return Err(Error::invalid_input_source( + format!( + "Sparse structural list layer {} contains a non-empty null slot", + layer_index + ) + .into(), + )); + } + (*num_slots, *num_child_slots, validity) + } + SparseStructuralLayerPlan::FixedSizeList { + num_slots, + dimension, + validity, + } => { + if *dimension == 0 { + return Err(Error::invalid_input_source( + format!( + "Sparse structural fixed-size-list layer {} has dimension zero", + layer_index + ) + .into(), + )); + } + let num_child_slots = num_slots.checked_mul(*dimension).ok_or_else(|| { + Error::invalid_input_source( + "Sparse structural fixed-size-list child domain overflows".into(), + ) + })?; + (*num_slots, num_child_slots, validity) + } + }; + usize_from_u64(num_slots, "layer slot count")?; + usize_from_u64(num_child_slots, "layer child slot count")?; + if num_slots != expected_slots { + return Err(Error::invalid_input_source( + format!( + "Sparse structural layer {} has {} slots, expected {}", + layer_index, num_slots, expected_slots + ) + .into(), + )); + } + validity.positions.validate_domain(num_slots, "validity")?; + expected_slots = num_child_slots; + } + if expected_slots != self.num_visible_items { + return Err(Error::invalid_input_source( + format!( + "Sparse structural terminal domain has {} slots, expected {} visible items", + expected_slots, self.num_visible_items + ) + .into(), + )); + } + Ok(()) + } +} + +#[derive(Debug)] +pub struct SparseStructuralUnraveler { + layers: Vec, + next_layer: usize, + pending_fixed_size_list: bool, +} + +impl SparseStructuralUnraveler { + pub(crate) fn new(plan: SparseStructuralPlan) -> Self { + let next_layer = plan.layers.len(); + Self { + layers: plan.layers, + next_layer, + pending_fixed_size_list: false, + } + } + + fn current_layer(&self) -> Option<&SparseStructuralLayerPlan> { + self.next_layer + .checked_sub(1) + .and_then(|idx| self.layers.get(idx)) + } + + fn consume_current_layer(&mut self) -> Result<()> { + self.next_layer = self.next_layer.checked_sub(1).ok_or_else(|| { + Error::invalid_input_source( + "Sparse structural metadata has fewer layers than the Arrow schema".into(), + ) + })?; + Ok(()) + } + + pub(crate) fn ensure_exhausted(&self) -> Result<()> { + if self.pending_fixed_size_list { + return Err(Error::invalid_input_source( + "Sparse structural metadata has an unconsumed fixed-size-list layer".into(), + )); + } + if self.next_layer != 0 { + return Err(Error::invalid_input_source( + format!( + "Sparse structural metadata has {} unconsumed layer(s)", + self.next_layer + ) + .into(), + )); + } + Ok(()) + } + + pub(crate) fn is_all_valid(&self) -> bool { + match self.current_layer() { + Some(SparseStructuralLayerPlan::Validity { + num_slots, + validity, + }) + | Some(SparseStructuralLayerPlan::FixedSizeList { + num_slots, + validity, + .. + }) + | Some(SparseStructuralLayerPlan::List { + num_slots, + validity, + .. + }) => match validity.meaning { + SparseValidityMeaning::NullPositions => validity.positions.is_empty(), + SparseValidityMeaning::ValidPositions => validity.positions.len() == *num_slots, + }, + None => true, + } + } + + pub(crate) fn max_lists(&self) -> Result { + match self.current_layer() { + Some(SparseStructuralLayerPlan::List { num_slots, .. }) => { + usize_from_u64(*num_slots, "list slot count") + } + _ => Ok(0), + } + } + + pub(crate) fn skip_validity(&mut self) -> Result<()> { + match self.current_layer() { + Some(SparseStructuralLayerPlan::Validity { .. }) => { + if self.pending_fixed_size_list { + return Err(Error::invalid_input_source( + "Sparse fixed-size-list schema does not match a validity layer".into(), + )); + } + self.consume_current_layer()?; + } + Some(SparseStructuralLayerPlan::FixedSizeList { .. }) => { + if !self.pending_fixed_size_list { + return Err(Error::invalid_input_source( + "Sparse fixed-size-list layer does not match the Arrow schema".into(), + )); + } + self.pending_fixed_size_list = false; + self.consume_current_layer()?; + } + None => { + return Err(Error::invalid_input_source( + "Sparse structural metadata has fewer layers than the Arrow schema".into(), + )); + } + Some(SparseStructuralLayerPlan::List { .. }) => { + return Err(Error::invalid_input_source( + "Sparse structural list layer does not match an Arrow validity layer".into(), + )); + } + } + Ok(()) + } + + pub(crate) fn unravel_validity(&mut self, validity: &mut BooleanBufferBuilder) -> Result<()> { + match self.current_layer() { + Some(SparseStructuralLayerPlan::Validity { + num_slots, + validity: layer_validity, + }) => { + if self.pending_fixed_size_list { + return Err(Error::invalid_input_source( + "Sparse fixed-size-list schema does not match a validity layer".into(), + )); + } + layer_validity.append_to(validity, *num_slots)?; + self.consume_current_layer()?; + } + Some(SparseStructuralLayerPlan::FixedSizeList { + num_slots, + validity: layer_validity, + .. + }) => { + if !self.pending_fixed_size_list { + return Err(Error::invalid_input_source( + "Sparse fixed-size-list layer does not match the Arrow schema".into(), + )); + } + layer_validity.append_to(validity, *num_slots)?; + self.pending_fixed_size_list = false; + self.consume_current_layer()?; + } + None => { + return Err(Error::invalid_input_source( + "Sparse structural metadata has fewer layers than the Arrow schema".into(), + )); + } + Some(SparseStructuralLayerPlan::List { .. }) => { + return Err(Error::invalid_input_source( + "Sparse structural list layer does not match an Arrow validity layer".into(), + )); + } + } + Ok(()) + } + + pub(crate) fn decimate(&mut self, dimension: usize) -> Result<()> { + if self.pending_fixed_size_list { + return Err(Error::invalid_input_source( + "Sparse fixed-size-list layer was decimated more than once".into(), + )); + } + let Some(SparseStructuralLayerPlan::FixedSizeList { + dimension: actual_dimension, + .. + }) = self.current_layer() + else { + return Err(Error::invalid_input_source( + "Sparse structural layer does not match an Arrow fixed-size-list layer".into(), + )); + }; + if usize_from_u64(*actual_dimension, "fixed-size-list dimension")? != dimension { + return Err(Error::invalid_input_source( + format!( + "Sparse structural fixed-size-list dimension {} does not match Arrow dimension {}", + actual_dimension, dimension + ) + .into(), + )); + } + self.pending_fixed_size_list = true; + Ok(()) + } + + fn to_offset(value: u64) -> Result { + let value = usize::try_from(value).map_err(|_| { + Error::invalid_input_source( + format!("Sparse structural offset {} exceeds usize::MAX", value).into(), + ) + })?; + T::from_usize(value).ok_or_else(|| { + Error::invalid_input_source( + "Sparse structural offset does not fit the Arrow offset type".into(), + ) + }) + } + + pub(crate) fn unravel_offsets( + &mut self, + offsets: &mut Vec, + validity: Option<&mut BooleanBufferBuilder>, + ) -> Result<()> { + let Some(SparseStructuralLayerPlan::List { + num_slots, + num_child_slots, + non_empty_positions, + counts, + validity: layer_validity, + .. + }) = self.current_layer() + else { + return Err(Error::invalid_input_source( + "Sparse structural layer does not match an Arrow list layer".into(), + )); + }; + if self.pending_fixed_size_list { + return Err(Error::invalid_input_source( + "Sparse fixed-size-list schema does not match an Arrow list layer".into(), + )); + } + + if non_empty_positions.len() != counts.len() { + return Err(Error::invalid_input_source( + format!( + "Sparse structural list has {} non-empty positions but {} counts", + non_empty_positions.len(), + counts.len() + ) + .into(), + )); + } + let actual_child_slots = counts.sum()?; + if actual_child_slots != *num_child_slots { + return Err(Error::invalid_input_source( + format!( + "Sparse structural list count sum {} does not match child slots {}", + actual_child_slots, num_child_slots + ) + .into(), + )); + } + + let mut current_offset = offsets + .last() + .map(|offset| offset.as_usize() as u64) + .unwrap_or(0); + if offsets.is_empty() { + offsets.push(Self::to_offset(current_offset)?); + } + + if non_empty_positions.is_empty() { + if let Some(validity) = validity { + layer_validity.append_to(validity, *num_slots)?; + } + let offset = Self::to_offset(current_offset)?; + let new_len = offsets + .len() + .checked_add(usize_from_u64(*num_slots, "list slot count")?) + .ok_or_else(|| { + Error::invalid_input_source( + "Sparse structural list offset length overflows usize".into(), + ) + })?; + offsets.resize(new_len, offset); + self.consume_current_layer()?; + return Ok(()); + } + + let non_empty_positions = non_empty_positions.materialize()?; + let counts = counts.materialize()?; + let mut non_empty_iter = non_empty_positions + .iter() + .copied() + .zip(counts.iter().copied()) + .peekable(); + let mut validity_cursor = + SparseValidityCursor::new(layer_validity, *num_slots, "list validity")?; + let mut validity = validity; + for slot in 0..*num_slots { + let is_valid = validity_cursor.is_valid(slot)?; + if let Some(validity) = validity.as_mut() { + validity.append(is_valid); + } + + if non_empty_iter + .peek() + .is_some_and(|(non_empty_pos, _)| *non_empty_pos == slot) + { + let (_, count) = non_empty_iter.next().ok_or_else(|| { + Error::invalid_input_source( + "Sparse structural list position is missing its child count".into(), + ) + })?; + if !is_valid { + return Err(Error::invalid_input_source( + format!( + "Sparse structural list slot {} is both invalid and non-empty", + slot + ) + .into(), + )); + } + current_offset = current_offset.checked_add(count).ok_or_else(|| { + Error::invalid_input_source( + "Sparse structural list offsets overflow u64".into(), + ) + })?; + } + offsets.push(Self::to_offset(current_offset)?); + } + validity_cursor.finish()?; + if let Some((extra_pos, _)) = non_empty_iter.next() { + return Err(Error::invalid_input_source( + format!( + "Sparse structural non-empty position {} is outside layer with {} slots", + extra_pos, num_slots + ) + .into(), + )); + } + + self.consume_current_layer()?; + Ok(()) + } +} + +#[derive(Debug, Clone)] +enum SparsePositionSetDecoder { + Empty, + All { + len: u64, + }, + Range { + start: u64, + len: u64, + }, + Explicit { + decompressor: Arc, + encoding: CompressiveEncoding, + count: u64, + domain_len: u64, + }, +} + +#[derive(Debug, Clone)] +enum SparseCountSetDecoder { + Empty, + Constant { + value: u64, + len: u64, + }, + Explicit { + decompressor: Arc, + encoding: CompressiveEncoding, + count: u64, + }, +} + +#[derive(Debug, Clone)] +enum SparseLayerDecompressors { + Validity { + num_slots: u64, + validity: SparseValiditySetDecoder, + }, + List { + num_slots: u64, + num_child_slots: u64, + non_empty_positions: SparsePositionSetDecoder, + counts: SparseCountSetDecoder, + validity: SparseValiditySetDecoder, + }, + FixedSizeList { + num_slots: u64, + num_child_slots: u64, + dimension: u64, + validity: SparseValiditySetDecoder, + }, +} + +#[derive(Debug, Clone)] +struct SparseValiditySetDecoder { + meaning: SparseValidityMeaning, + positions: SparsePositionSetDecoder, +} + +#[derive(Debug)] +struct SparseStructuralCacheableState { + chunk_meta: Vec, + chunk_value_offsets: Arc<[u64]>, + plan: SparseStructuralPlan, + row_domain: u64, +} + +impl DeepSizeOf for SparseStructuralCacheableState { + fn deep_size_of_children(&self, _context: &mut Context) -> usize { + let structural_size = self + .plan + .layers + .iter() + .map(|layer| match layer { + SparseStructuralLayerPlan::Validity { validity, .. } => validity.deep_size(), + SparseStructuralLayerPlan::List { + non_empty_positions, + counts, + validity, + .. + } => non_empty_positions.deep_size() + counts.deep_size() + validity.deep_size(), + SparseStructuralLayerPlan::FixedSizeList { validity, .. } => validity.deep_size(), + }) + .sum::(); + self.chunk_meta.len() * std::mem::size_of::() + + self.chunk_value_offsets.len() * std::mem::size_of::() + + structural_size + } +} + +impl CachedPageData for SparseStructuralCacheableState { + fn as_arc_any(self: Arc) -> Arc { + self + } +} + +#[derive(Debug)] +pub(super) struct SparseStructuralScheduler { + buffer_offsets_and_sizes: Vec<(u64, u64)>, + priority: u64, + row_domain: u64, + row_scale: u64, + num_items: u64, + num_visible_items: u64, + num_buffers: u64, + value_encoding: CompressiveEncoding, + value_decompressor: Arc, + layer_decompressors: Vec, + data_type: DataType, + page_meta: Option>, + has_large_chunk: bool, +} + +impl SparseStructuralScheduler { + fn require_layer( + layer: &pb21::SparseStructuralLayer, + ) -> Result<&pb21::sparse_structural_layer::Layer> { + layer.layer.as_ref().ok_or_else(|| { + Error::invalid_input_source( + "Sparse structural layer is missing its layer variant".into(), + ) + }) + } + + fn layer_num_slots(layer: &pb21::SparseStructuralLayer) -> Result { + Ok(match Self::require_layer(layer)? { + pb21::sparse_structural_layer::Layer::Validity(layer) => layer.num_slots, + pb21::sparse_structural_layer::Layer::List(layer) => layer.num_slots, + pb21::sparse_structural_layer::Layer::FixedSizeList(layer) => layer.num_slots, + }) + } + + fn layer_num_child_slots(layer: &pb21::SparseStructuralLayer) -> Result { + Ok(match Self::require_layer(layer)? { + pb21::sparse_structural_layer::Layer::Validity(layer) => layer.num_slots, + pb21::sparse_structural_layer::Layer::List(layer) => layer.num_child_slots, + pb21::sparse_structural_layer::Layer::FixedSizeList(layer) => layer + .num_slots + .checked_mul(layer.dimension) + .ok_or_else(|| { + Error::invalid_input_source( + format!( + "Sparse structural fixed-size-list child slot count overflows: slots={}, dimension={}", + layer.num_slots, layer.dimension + ) + .into(), + ) + })?, + }) + } + + pub(super) fn try_new( + buffer_offsets_and_sizes: &[(u64, u64)], + priority: u64, + encoded_row_domain: u64, + data_type: DataType, + layout: &pb21::SparseLayout, + decompressors: &dyn DecompressionStrategy, + ) -> Result { + let value_compression = layout.value_compression.as_ref().ok_or_else(|| { + Error::invalid_input_source("Sparse layout is missing value compression".into()) + })?; + let value_buffer_count = Self::validate_value_encoding(value_compression)?; + if layout.num_buffers != value_buffer_count { + return Err(Error::invalid_input_source( + format!( + "Sparse layout declares {} value buffers, but its compression descriptor requires {}", + layout.num_buffers, value_buffer_count + ) + .into(), + )); + } + let row_domain = match layout.structural_layers.first() { + Some(layer) => Self::layer_num_slots(layer)?, + None => encoded_row_domain, + }; + Self::validate_domain_chain( + &layout.structural_layers, + row_domain, + layout.num_items, + layout.num_visible_items, + )?; + let expected_buffers = 2 + Self::structural_buffer_count(&layout.structural_layers)?; + if buffer_offsets_and_sizes.len() != expected_buffers { + return Err(Error::invalid_input_source( + format!( + "Sparse layout has {} buffers, expected {}", + buffer_offsets_and_sizes.len(), + expected_buffers + ) + .into(), + )); + } + Self::validate_page_buffers(buffer_offsets_and_sizes, layout.num_visible_items)?; + let row_scale = + layout + .structural_layers + .iter() + .try_fold(1_u64, |scale, layer| -> Result { + match Self::require_layer(layer)? { + pb21::sparse_structural_layer::Layer::FixedSizeList(layer) => { + scale.checked_mul(layer.dimension).ok_or_else(|| { + Error::invalid_input_source( + "Sparse structural fixed-size-list row scale overflows".into(), + ) + }) + } + pb21::sparse_structural_layer::Layer::Validity(_) + | pb21::sparse_structural_layer::Layer::List(_) => Ok(scale), + } + })?; + let expected_encoded_row_domain = row_domain.checked_mul(row_scale).ok_or_else(|| { + Error::invalid_input_source("Sparse structural encoded row domain overflows".into()) + })?; + if encoded_row_domain != expected_encoded_row_domain { + return Err(Error::invalid_input_source( + format!( + "Sparse structural encoded row domain {} does not match outer domain {} * fixed-size-list scale {}", + encoded_row_domain, row_domain, row_scale + ) + .into(), + )); + } + let value_decompressor = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + decompressors.create_miniblock_decompressor(value_compression, decompressors) + })) + .map_err(|_| { + Error::invalid_input_source( + "Sparse value compression descriptor caused decompressor construction to panic" + .into(), + ) + })? + .map_err(|error| { + Error::invalid_input_source( + format!("Sparse value decompressor construction failed: {error}").into(), + ) + })?; + let layer_decompressors = layout + .structural_layers + .iter() + .map(|layer| Self::layer_decompressors(layer, decompressors)) + .collect::>>()?; + + Ok(Self { + buffer_offsets_and_sizes: buffer_offsets_and_sizes.to_vec(), + priority, + row_domain, + row_scale, + num_items: layout.num_items, + num_visible_items: layout.num_visible_items, + num_buffers: layout.num_buffers, + value_encoding: value_compression.clone(), + value_decompressor: value_decompressor.into(), + layer_decompressors, + data_type, + page_meta: None, + has_large_chunk: layout.has_large_chunk, + }) + } + + fn validate_compression<'a>( + compression: &'a CompressiveEncoding, + label: &str, + ) -> Result<&'a CompressiveEncoding> { + compression.compression.as_ref().ok_or_else(|| { + Error::invalid_input_source( + format!("Sparse structural {label} is missing compression details").into(), + ) + })?; + Ok(compression) + } + + fn validate_buffer_compression( + compression: Option<&pb21::BufferCompression>, + label: &str, + ) -> Result<()> { + let Some(compression) = compression else { + return Ok(()); + }; + match invalid_enum( + pb21::CompressionScheme::try_from(compression.scheme), + "compression scheme", + )? { + pb21::CompressionScheme::CompressionAlgorithmUnspecified => { + Err(Error::invalid_input_source( + format!("Sparse structural {label} buffer compression is unspecified").into(), + )) + } + pb21::CompressionScheme::CompressionAlgorithmLz4 + | pb21::CompressionScheme::CompressionAlgorithmZstd => Ok(()), + } + } + + fn encoding_contains_general(root: &CompressiveEncoding) -> bool { + use pb21::compressive_encoding::Compression; + + let mut stack = vec![root]; + while let Some(encoding) = stack.pop() { + let Some(compression) = encoding.compression.as_ref() else { + continue; + }; + match compression { + Compression::General(_) => return true, + Compression::Variable(variable) => { + stack.extend(variable.offsets.as_deref()); + } + Compression::OutOfLineBitpacking(bitpacking) => { + stack.extend(bitpacking.values.as_deref()); + } + Compression::Fsst(fsst) => stack.extend(fsst.values.as_deref()), + Compression::Dictionary(dictionary) => { + stack.extend(dictionary.indices.as_deref()); + stack.extend(dictionary.items.as_deref()); + } + Compression::Rle(rle) => { + stack.extend(rle.values.as_deref()); + stack.extend(rle.run_lengths.as_deref()); + } + Compression::ByteStreamSplit(split) => { + stack.extend(split.values.as_deref()); + } + Compression::FixedSizeList(fsl) => stack.extend(fsl.values.as_deref()), + Compression::PackedStruct(packed) => stack.extend(packed.values.as_deref()), + Compression::VariablePackedStruct(packed) => { + stack.extend( + packed + .fields + .iter() + .filter_map(|field| field.value.as_ref()), + ); + } + Compression::Flat(_) + | Compression::Constant(_) + | Compression::InlineBitpacking(_) => {} + } + } + false + } + + fn require_encoding<'a>( + encoding: &'a Option>, + label: &str, + ) -> Result<&'a CompressiveEncoding> { + encoding.as_deref().ok_or_else(|| { + Error::invalid_input_source( + format!("Sparse structural {label} encoding is required").into(), + ) + }) + } + + fn validate_flat(flat: &pb21::Flat, label: &str) -> Result<()> { + if flat.bits_per_value == 0 { + return Err(Error::invalid_input_source( + format!("Sparse structural {label} flat bit width is zero").into(), + )); + } + if flat.data.is_some() { + return Err(Error::invalid_input_source( + format!("Sparse structural {label} uses unsupported leaf buffer compression") + .into(), + )); + } + Ok(()) + } + + fn validate_value_encoding(encoding: &CompressiveEncoding) -> Result { + use pb21::compressive_encoding::Compression; + + let compression = encoding.compression.as_ref().ok_or_else(|| { + Error::invalid_input_source( + "Sparse value compression is missing compression details".into(), + ) + })?; + match compression { + Compression::Flat(flat) => { + Self::validate_flat(flat, "value")?; + Ok(1) + } + Compression::InlineBitpacking(bitpacking) => { + if !matches!(bitpacking.uncompressed_bits_per_value, 8 | 16 | 32 | 64) { + return Err(Error::invalid_input_source( + format!( + "Sparse inline bitpacking width {} is not supported", + bitpacking.uncompressed_bits_per_value + ) + .into(), + )); + } + if bitpacking.values.is_some() { + return Err(Error::invalid_input_source( + "Sparse inline bitpacking uses unsupported leaf buffer compression".into(), + )); + } + Ok(1) + } + Compression::Variable(variable) => { + let offsets = variable.offsets.as_deref().ok_or_else(|| { + Error::invalid_input_source( + "Sparse variable compression is missing offsets".into(), + ) + })?; + let Some(Compression::Flat(offsets)) = offsets.compression.as_ref() else { + return Err(Error::invalid_input_source( + "Sparse variable offsets must use flat compression".into(), + )); + }; + Self::validate_flat(offsets, "variable offsets")?; + if !matches!(offsets.bits_per_value, 32 | 64) { + return Err(Error::invalid_input_source( + format!( + "Sparse variable offset width {} is not supported", + offsets.bits_per_value + ) + .into(), + )); + } + if variable.values.is_some() { + return Err(Error::invalid_input_source( + "Sparse variable values use unsupported leaf buffer compression".into(), + )); + } + Ok(1) + } + Compression::Fsst(fsst) => { + if fsst.symbol_table.is_empty() { + return Err(Error::invalid_input_source( + "Sparse FSST compression has an empty symbol table".into(), + )); + } + let values = Self::require_encoding(&fsst.values, "FSST values")?; + if !matches!(values.compression.as_ref(), Some(Compression::Variable(_))) { + return Err(Error::invalid_input_source( + "Sparse FSST values must use variable compression".into(), + )); + } + Self::validate_value_encoding(values) + } + Compression::ByteStreamSplit(split) => { + let values = Self::require_encoding(&split.values, "byte-stream-split values")?; + let Some(Compression::Flat(flat)) = values.compression.as_ref() else { + return Err(Error::invalid_input_source( + "Sparse byte-stream-split values must use flat compression".into(), + )); + }; + Self::validate_flat(flat, "byte-stream-split values")?; + if !matches!(flat.bits_per_value, 32 | 64) { + return Err(Error::invalid_input_source( + format!( + "Sparse byte-stream-split width {} is not supported", + flat.bits_per_value + ) + .into(), + )); + } + Ok(1) + } + Compression::FixedSizeList(fsl) => Self::validate_fsl_value_encoding(fsl), + Compression::PackedStruct(packed) => Self::validate_packed_value_encoding(packed), + Compression::Rle(rle) => { + let values = Self::require_encoding(&rle.values, "RLE values")?; + let lengths = Self::require_encoding(&rle.run_lengths, "RLE run lengths")?; + Self::validate_block_encoding(values, "RLE values")?; + Self::validate_block_encoding(lengths, "RLE run lengths")?; + Ok(2) + } + Compression::General(general) => { + let compression = general.compression.as_ref().ok_or_else(|| { + Error::invalid_input_source( + "Sparse general compression is missing its buffer compression".into(), + ) + })?; + Self::validate_buffer_compression(Some(compression), "general")?; + Self::validate_value_encoding(Self::require_encoding( + &general.values, + "general values", + )?) + } + Compression::Constant(_) + | Compression::OutOfLineBitpacking(_) + | Compression::Dictionary(_) + | Compression::VariablePackedStruct(_) => Err(Error::invalid_input_source( + "Sparse value compression uses an unsupported mini-block encoding".into(), + )), + } + } + + fn validate_fsl_value_encoding(fsl: &pb21::FixedSizeList) -> Result { + use pb21::compressive_encoding::Compression; + + if fsl.items_per_value == 0 { + return Err(Error::invalid_input_source( + "Sparse fixed-size-list value compression has dimension zero".into(), + )); + } + let values = Self::require_encoding(&fsl.values, "fixed-size-list values")?; + let child_buffers = match values.compression.as_ref() { + Some(Compression::Flat(flat)) => { + Self::validate_flat(flat, "fixed-size-list values")?; + 1_u64 + } + Some(Compression::FixedSizeList(inner)) => Self::validate_fsl_value_encoding(inner)?, + _ => { + return Err(Error::invalid_input_source( + "Sparse fixed-size-list values must use fixed-size-list or flat compression" + .into(), + )); + } + }; + child_buffers + .checked_add(u64::from(fsl.has_validity)) + .ok_or_else(|| { + Error::invalid_input_source( + "Sparse fixed-size-list value buffer count overflows".into(), + ) + }) + } + + fn validate_packed_value_encoding(packed: &pb21::PackedStruct) -> Result { + use pb21::compressive_encoding::Compression; + + if packed.bits_per_value.is_empty() + || packed + .bits_per_value + .iter() + .any(|bits| *bits == 0 || !bits.is_multiple_of(8)) + { + return Err(Error::invalid_input_source( + "Sparse packed-struct widths must be non-empty positive byte widths".into(), + )); + } + let values = Self::require_encoding(&packed.values, "packed-struct values")?; + let Some(Compression::Flat(flat)) = values.compression.as_ref() else { + return Err(Error::invalid_input_source( + "Sparse packed-struct values must use flat compression".into(), + )); + }; + Self::validate_flat(flat, "packed-struct values")?; + let total_bits = packed.bits_per_value.iter().try_fold(0_u64, |sum, bits| { + sum.checked_add(*bits).ok_or_else(|| { + Error::invalid_input_source("Sparse packed-struct bit width sum overflows".into()) + }) + })?; + if total_bits != flat.bits_per_value { + return Err(Error::invalid_input_source( + format!( + "Sparse packed-struct child widths sum to {}, but values use {} bits", + total_bits, flat.bits_per_value + ) + .into(), + )); + } + Ok(1) + } + + fn validate_block_encoding(encoding: &CompressiveEncoding, label: &str) -> Result<()> { + use pb21::compressive_encoding::Compression; + + match encoding.compression.as_ref().ok_or_else(|| { + Error::invalid_input_source( + format!("Sparse structural {label} is missing compression details").into(), + ) + })? { + Compression::Flat(flat) => Self::validate_flat(flat, label), + Compression::InlineBitpacking(bitpacking) => { + if !matches!(bitpacking.uncompressed_bits_per_value, 8 | 16 | 32 | 64) { + return Err(Error::invalid_input_source( + format!( + "Sparse structural {label} inline bitpacking width {} is unsupported", + bitpacking.uncompressed_bits_per_value + ) + .into(), + )); + } + if bitpacking.values.is_some() { + return Err(Error::invalid_input_source( + format!( + "Sparse structural {label} uses unsupported leaf buffer compression" + ) + .into(), + )); + } + Ok(()) + } + Compression::OutOfLineBitpacking(bitpacking) => { + if !matches!(bitpacking.uncompressed_bits_per_value, 8 | 16 | 32 | 64) { + return Err(Error::invalid_input_source( + format!( + "Sparse structural {label} out-of-line bitpacking width {} is unsupported", + bitpacking.uncompressed_bits_per_value + ) + .into(), + )); + } + let values = Self::require_encoding(&bitpacking.values, label)?; + let Some(Compression::Flat(flat)) = values.compression.as_ref() else { + return Err(Error::invalid_input_source( + format!("Sparse structural {label} bitpacked values must be flat").into(), + )); + }; + Self::validate_flat(flat, label) + } + Compression::Constant(constant) => { + if constant + .value + .as_ref() + .is_some_and(|value| value.len() != 8) + { + return Err(Error::invalid_input_source( + format!("Sparse structural {label} constant must be 64 bits").into(), + )); + } + Ok(()) + } + Compression::General(general) => { + let compression = general.compression.as_ref().ok_or_else(|| { + Error::invalid_input_source( + format!("Sparse structural {label} general compression is missing config") + .into(), + ) + })?; + Self::validate_buffer_compression(Some(compression), label)?; + Self::validate_block_encoding( + Self::require_encoding(&general.values, label)?, + label, + ) + } + Compression::Rle(rle) => { + Self::validate_block_encoding( + Self::require_encoding(&rle.values, "RLE values")?, + "RLE values", + )?; + Self::validate_block_encoding( + Self::require_encoding(&rle.run_lengths, "RLE run lengths")?, + "RLE run lengths", + ) + } + _ => Err(Error::invalid_input_source( + format!("Sparse structural {label} uses an unsupported block encoding").into(), + )), + } + } + + fn validate_domain_chain( + layers: &[pb21::SparseStructuralLayer], + row_domain: u64, + num_items: u64, + num_visible_items: u64, + ) -> Result<()> { + let expected_num_items = + layers + .iter() + .try_fold(num_visible_items, |items, layer| -> Result { + let additional = match Self::require_layer(layer)? { + pb21::sparse_structural_layer::Layer::List(layer) => { + let positions = Self::require_position_set( + &layer.non_empty_positions, + "list non-empty", + )?; + let num_non_empty = Self::position_cardinality( + positions, + layer.num_slots, + "list non-empty positions", + )?; + layer.num_slots.checked_sub(num_non_empty).ok_or_else(|| { + Error::invalid_input_source( + "Sparse structural list has more non-empty positions than slots" + .into(), + ) + })? + } + pb21::sparse_structural_layer::Layer::Validity(_) + | pb21::sparse_structural_layer::Layer::FixedSizeList(_) => 0, + }; + items.checked_add(additional).ok_or_else(|| { + Error::invalid_input_source("Sparse structural item count overflows".into()) + }) + })?; + if num_items != expected_num_items { + return Err(Error::invalid_input_source( + format!( + "Sparse layout has {} structural items, but its layers imply {}", + num_items, expected_num_items + ) + .into(), + )); + } + let mut expected_slots = row_domain; + for (layer_index, layer) in layers.iter().enumerate() { + let num_slots = Self::layer_num_slots(layer)?; + let num_child_slots = Self::layer_num_child_slots(layer)?; + usize_from_u64(num_slots, "layer slot count")?; + usize_from_u64(num_child_slots, "layer child slot count")?; + if num_slots != expected_slots { + return Err(Error::invalid_input_source( + format!( + "Sparse structural layer {} has {} slots, expected {} from the outer domain", + layer_index, num_slots, expected_slots + ) + .into(), + )); + } + expected_slots = num_child_slots; + } + if expected_slots != num_visible_items { + return Err(Error::invalid_input_source( + format!( + "Sparse structural terminal domain has {} slots, expected {} visible items", + expected_slots, num_visible_items + ) + .into(), + )); + } + Ok(()) + } + + fn metadata_buffer(&self) -> Result<(u64, u64)> { + self.buffer_offsets_and_sizes + .first() + .copied() + .ok_or_else(|| { + Error::invalid_input_source("Sparse layout is missing metadata buffer".into()) + }) + } + + fn value_buffer(&self) -> Result<(u64, u64)> { + self.buffer_offsets_and_sizes + .get(1) + .copied() + .ok_or_else(|| { + Error::invalid_input_source("Sparse layout is missing value buffer".into()) + }) + } + + fn checked_buffer_range(position: u64, size: u64, label: &str) -> Result> { + let end = position.checked_add(size).ok_or_else(|| { + Error::invalid_input_source( + format!("Sparse structural {label} buffer range overflows").into(), + ) + })?; + Ok(position..end) + } + + fn validate_page_buffers( + buffer_offsets_and_sizes: &[(u64, u64)], + num_visible_items: u64, + ) -> Result<()> { + let (_, metadata_size) = buffer_offsets_and_sizes.first().copied().ok_or_else(|| { + Error::invalid_input_source("Sparse layout is missing metadata buffer".into()) + })?; + if !metadata_size.is_multiple_of(8) { + return Err(Error::invalid_input_source( + format!( + "Sparse layout metadata buffer has {metadata_size} bytes, which is not a multiple of 8" + ) + .into(), + )); + } + let chunk_count = metadata_size / 8; + if num_visible_items == 0 { + if chunk_count != 0 { + return Err(Error::invalid_input_source( + "Sparse layout with no visible items must have empty chunk metadata".into(), + )); + } + } else { + let min_chunks = + num_visible_items.div_ceil(miniblock::MAX_CONFIGURABLE_MINIBLOCK_VALUES); + if chunk_count < min_chunks || chunk_count > num_visible_items { + return Err(Error::invalid_input_source( + format!( + "Sparse layout metadata declares {chunk_count} chunks for {num_visible_items} visible items, expected {min_chunks}..={num_visible_items}" + ) + .into(), + )); + } + } + + let (_, value_size) = buffer_offsets_and_sizes.get(1).copied().ok_or_else(|| { + Error::invalid_input_source("Sparse layout is missing value buffer".into()) + })?; + if (num_visible_items == 0) != (value_size == 0) { + return Err(Error::invalid_input_source( + format!( + "Sparse layout value buffer has {value_size} bytes for {num_visible_items} visible items" + ) + .into(), + )); + } + + for (position, size) in buffer_offsets_and_sizes.iter().skip(2) { + Self::checked_buffer_range(*position, *size, "structural")?; + } + + for (position, size) in buffer_offsets_and_sizes.iter().take(2) { + Self::checked_buffer_range(*position, *size, "page")?; + } + Ok(()) + } + + fn require_position_set<'a>( + set: &'a Option, + label: &str, + ) -> Result<&'a pb21::SparsePositionSet> { + set.as_ref().ok_or_else(|| { + Error::invalid_input_source( + format!("Sparse structural {label} position set is required").into(), + ) + }) + } + + fn require_count_set<'a>( + set: &'a Option, + label: &str, + ) -> Result<&'a pb21::SparseCountSet> { + set.as_ref().ok_or_else(|| { + Error::invalid_input_source( + format!("Sparse structural {label} count set is required").into(), + ) + }) + } + + fn require_validity_set<'a>( + set: &'a Option, + label: &str, + ) -> Result<&'a pb21::SparseValiditySet> { + set.as_ref().ok_or_else(|| { + Error::invalid_input_source( + format!("Sparse structural {label} validity set is required").into(), + ) + }) + } + + fn validity_meaning( + validity_set: &pb21::SparseValiditySet, + label: &str, + ) -> Result { + match invalid_enum( + pb21::sparse_validity_set::Meaning::try_from(validity_set.meaning), + "validity meaning", + )? { + pb21::sparse_validity_set::Meaning::SparseValidityUnspecified => { + Err(Error::invalid_input_source( + format!("Sparse structural {label} meaning is unspecified").into(), + )) + } + pb21::sparse_validity_set::Meaning::SparseValidityNullPositions => { + Ok(SparseValidityMeaning::NullPositions) + } + pb21::sparse_validity_set::Meaning::SparseValidityValidPositions => { + Ok(SparseValidityMeaning::ValidPositions) + } + } + } + + fn validity_buffer_count( + validity_set: &pb21::SparseValiditySet, + domain_len: u64, + label: &str, + ) -> Result<(usize, SparseValidityMeaning, u64)> { + let meaning = Self::validity_meaning(validity_set, label)?; + let position_set = validity_set.positions.as_ref().ok_or_else(|| { + Error::invalid_input_source( + format!("Sparse structural {label} positions are required").into(), + ) + })?; + let cardinality = Self::position_cardinality(position_set, domain_len, label)?; + let buffer_count = Self::position_buffer_count(position_set, domain_len, label)?; + Ok((buffer_count, meaning, cardinality)) + } + + fn position_cardinality( + position_set: &pb21::SparsePositionSet, + domain_len: u64, + label: &str, + ) -> Result { + let positions = position_set.positions.as_ref().ok_or_else(|| { + Error::invalid_input_source( + format!("Sparse structural {label} position set is missing its variant").into(), + ) + })?; + let cardinality = position_set.num_positions; + if cardinality > domain_len { + return Err(Error::invalid_input_source( + format!( + "Sparse structural {label} cardinality {} exceeds domain {}", + cardinality, domain_len + ) + .into(), + )); + } + match positions { + pb21::sparse_position_set::Positions::Empty(_) => { + if cardinality != 0 { + return Err(Error::invalid_input_source( + format!( + "Sparse structural {label} empty set has cardinality {}", + cardinality + ) + .into(), + )); + } + } + pb21::sparse_position_set::Positions::All(_) => { + if domain_len == 0 || cardinality != domain_len { + return Err(Error::invalid_input_source( + format!( + "Sparse structural {label} all set has cardinality {}, expected {}", + cardinality, domain_len + ) + .into(), + )); + } + } + pb21::sparse_position_set::Positions::Range(range) => { + let end = range.start.checked_add(range.length).ok_or_else(|| { + Error::invalid_input_source( + format!("Sparse structural {label} range overflows").into(), + ) + })?; + if range.length == 0 || range.length != cardinality || end > domain_len { + return Err(Error::invalid_input_source( + format!( + "Sparse structural {label} range {}..{} does not match cardinality {} in domain {}", + range.start, end, cardinality, domain_len + ) + .into(), + )); + } + } + pb21::sparse_position_set::Positions::Explicit(compression) => { + if cardinality == 0 { + return Err(Error::invalid_input_source( + format!("Sparse structural {label} has compression but no values").into(), + )); + } + Self::validate_compression(compression, label)?; + } + } + Ok(cardinality) + } + + fn position_buffer_count( + position_set: &pb21::SparsePositionSet, + domain_len: u64, + label: &str, + ) -> Result { + Self::position_cardinality(position_set, domain_len, label)?; + let positions = position_set.positions.as_ref().ok_or_else(|| { + Error::invalid_input_source( + format!("Sparse structural {label} position set is missing its variant").into(), + ) + })?; + Ok(usize::from(matches!( + positions, + pb21::sparse_position_set::Positions::Explicit(_) + ))) + } + + fn count_buffer_count( + count_set: &pb21::SparseCountSet, + cardinality: u64, + label: &str, + ) -> Result { + let counts = count_set.counts.as_ref().ok_or_else(|| { + Error::invalid_input_source( + format!("Sparse structural {label} count set is missing its variant").into(), + ) + })?; + match counts { + pb21::sparse_count_set::Counts::Empty(_) => { + if cardinality != 0 { + return Err(Error::invalid_input_source( + format!( + "Sparse structural {label} empty count set has cardinality {}", + cardinality + ) + .into(), + )); + } + Ok(0) + } + pb21::sparse_count_set::Counts::Constant(constant) => { + if cardinality == 0 { + return Err(Error::invalid_input_source( + format!("Sparse structural {label} constant count has no values").into(), + )); + } + if constant.value == 0 { + return Err(Error::invalid_input_source( + format!("Sparse structural {label} constant count is zero").into(), + )); + } + Ok(0) + } + pb21::sparse_count_set::Counts::Explicit(compression) => { + if cardinality == 0 { + return Err(Error::invalid_input_source( + format!("Sparse structural {label} has compression but no values").into(), + )); + } + Self::validate_compression(compression, label)?; + Ok(1) + } + } + } + + fn count_set_child_slots( + count_set: &pb21::SparseCountSet, + cardinality: u64, + label: &str, + ) -> Result> { + let counts = count_set.counts.as_ref().ok_or_else(|| { + Error::invalid_input_source( + format!("Sparse structural {label} count set is missing its variant").into(), + ) + })?; + match counts { + pb21::sparse_count_set::Counts::Empty(_) => Ok(Some(0)), + pb21::sparse_count_set::Counts::Constant(constant) => constant + .value + .checked_mul(cardinality) + .map(Some) + .ok_or_else(|| { + Error::invalid_input_source( + format!( + "Sparse structural {label} constant count sum overflows: value={}, len={}", + constant.value, cardinality + ) + .into(), + ) + }), + pb21::sparse_count_set::Counts::Explicit(_) => Ok(None), + } + } + + fn create_position_decompressor( + compression: &CompressiveEncoding, + label: &str, + decompressors: &dyn DecompressionStrategy, + ) -> Result> { + let compression = Self::validate_compression(compression, label)?; + Self::validate_block_encoding(compression, label)?; + std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + decompressors.create_block_decompressor(compression) + })) + .map_err(|_| { + Error::invalid_input_source( + format!( + "Sparse structural {label} descriptor caused decompressor construction to panic" + ) + .into(), + ) + })? + .map(Arc::from) + .map_err(|error| { + Error::invalid_input_source( + format!("Sparse structural {label} decompressor construction failed: {error}") + .into(), + ) + }) + } + + fn position_set_decoder( + position_set: &pb21::SparsePositionSet, + domain_len: u64, + label: &str, + decompressors: &dyn DecompressionStrategy, + ) -> Result<(SparsePositionSetDecoder, u64)> { + let cardinality = Self::position_cardinality(position_set, domain_len, label)?; + let positions = position_set.positions.as_ref().ok_or_else(|| { + Error::invalid_input_source( + format!("Sparse structural {label} position set is missing its variant").into(), + ) + })?; + Ok(( + match positions { + pb21::sparse_position_set::Positions::Empty(_) => SparsePositionSetDecoder::Empty, + pb21::sparse_position_set::Positions::All(_) => { + SparsePositionSetDecoder::All { len: domain_len } + } + pb21::sparse_position_set::Positions::Range(range) => { + SparsePositionSetDecoder::Range { + start: range.start, + len: range.length, + } + } + pb21::sparse_position_set::Positions::Explicit(compression) => { + SparsePositionSetDecoder::Explicit { + decompressor: Self::create_position_decompressor( + compression, + label, + decompressors, + )?, + encoding: compression.clone(), + count: cardinality, + domain_len, + } + } + }, + cardinality, + )) + } + + fn validity_set_decoder( + validity_set: &pb21::SparseValiditySet, + domain_len: u64, + label: &str, + decompressors: &dyn DecompressionStrategy, + ) -> Result<(SparseValiditySetDecoder, u64)> { + let meaning = Self::validity_meaning(validity_set, label)?; + let position_set = validity_set.positions.as_ref().ok_or_else(|| { + Error::invalid_input_source( + format!("Sparse structural {label} positions are required").into(), + ) + })?; + let (positions, cardinality) = + Self::position_set_decoder(position_set, domain_len, label, decompressors)?; + Ok((SparseValiditySetDecoder { meaning, positions }, cardinality)) + } + + fn num_valid_slots( + meaning: SparseValidityMeaning, + cardinality: u64, + num_slots: u64, + label: &str, + ) -> Result { + match meaning { + SparseValidityMeaning::NullPositions => { + num_slots.checked_sub(cardinality).ok_or_else(|| { + Error::invalid_input_source( + format!( + "Sparse structural {label} null cardinality {} exceeds slots {}", + cardinality, num_slots + ) + .into(), + ) + }) + } + SparseValidityMeaning::ValidPositions => Ok(cardinality), + } + } + + fn count_set_decoder( + count_set: &pb21::SparseCountSet, + cardinality: u64, + label: &str, + decompressors: &dyn DecompressionStrategy, + ) -> Result { + Self::count_buffer_count(count_set, cardinality, label)?; + let counts = count_set.counts.as_ref().ok_or_else(|| { + Error::invalid_input_source( + format!("Sparse structural {label} count set is missing its variant").into(), + ) + })?; + Ok(match counts { + pb21::sparse_count_set::Counts::Empty(_) => SparseCountSetDecoder::Empty, + pb21::sparse_count_set::Counts::Constant(constant) => SparseCountSetDecoder::Constant { + value: constant.value, + len: cardinality, + }, + pb21::sparse_count_set::Counts::Explicit(compression) => { + SparseCountSetDecoder::Explicit { + decompressor: Self::create_position_decompressor( + compression, + label, + decompressors, + )?, + encoding: compression.clone(), + count: cardinality, + } + } + }) + } + + fn add_buffer_count(count: &mut usize, additional: usize) -> Result<()> { + *count = count.checked_add(additional).ok_or_else(|| { + Error::invalid_input_source("Sparse structural buffer count overflows".into()) + })?; + Ok(()) + } + + fn structural_buffer_count(layers: &[pb21::SparseStructuralLayer]) -> Result { + layers + .iter() + .try_fold(0_usize, |mut count, layer| -> Result { + match Self::require_layer(layer)? { + pb21::sparse_structural_layer::Layer::Validity(layer) => { + let (validity_buffers, _, _) = Self::validity_buffer_count( + Self::require_validity_set(&layer.validity, "validity")?, + layer.num_slots, + "validity positions", + )?; + Self::add_buffer_count(&mut count, validity_buffers)?; + } + pb21::sparse_structural_layer::Layer::List(layer) => { + let non_empty_positions = Self::require_position_set( + &layer.non_empty_positions, + "list non-empty", + )?; + let num_non_empty = Self::position_cardinality( + non_empty_positions, + layer.num_slots, + "list non-empty positions", + )?; + let non_empty_buffers = Self::position_buffer_count( + non_empty_positions, + layer.num_slots, + "list non-empty positions", + )?; + Self::add_buffer_count(&mut count, non_empty_buffers)?; + let (validity_buffers, validity_meaning, validity_cardinality) = + Self::validity_buffer_count( + Self::require_validity_set(&layer.validity, "list")?, + layer.num_slots, + "list validity positions", + )?; + Self::add_buffer_count(&mut count, validity_buffers)?; + let num_valid_slots = Self::num_valid_slots( + validity_meaning, + validity_cardinality, + layer.num_slots, + "list validity", + )?; + if num_non_empty > num_valid_slots { + return Err(Error::invalid_input_source( + format!( + "Sparse structural list has {} non-empty slots but only {} valid slots", + num_non_empty, num_valid_slots + ) + .into(), + )); + } + let counts = Self::require_count_set(&layer.counts, "list counts")?; + let count_buffers = + Self::count_buffer_count(counts, num_non_empty, "list counts")?; + Self::add_buffer_count(&mut count, count_buffers)?; + if let Some(child_slots) = + Self::count_set_child_slots(counts, num_non_empty, "list counts")? + && child_slots != layer.num_child_slots + { + return Err(Error::invalid_input_source( + format!( + "Sparse structural list count sum {} does not match child slots {}", + child_slots, layer.num_child_slots + ) + .into(), + )); + } + } + pb21::sparse_structural_layer::Layer::FixedSizeList(layer) => { + if layer.dimension == 0 { + return Err(Error::invalid_input_source( + "Sparse structural fixed-size-list dimension is zero".into(), + )); + } + layer.num_slots.checked_mul(layer.dimension).ok_or_else(|| { + Error::invalid_input_source( + format!( + "Sparse structural fixed-size-list child slot count overflows: slots={}, dimension={}", + layer.num_slots, layer.dimension + ) + .into(), + ) + })?; + let (validity_buffers, _, _) = Self::validity_buffer_count( + Self::require_validity_set(&layer.validity, "fixed-size-list")?, + layer.num_slots, + "fixed-size-list validity positions", + )?; + Self::add_buffer_count(&mut count, validity_buffers)?; + } + } + Ok(count) + }) + } + + fn layer_decompressors( + layer: &pb21::SparseStructuralLayer, + decompressors: &dyn DecompressionStrategy, + ) -> Result { + Ok(match Self::require_layer(layer)? { + pb21::sparse_structural_layer::Layer::Validity(layer) => { + let (validity, _) = Self::validity_set_decoder( + Self::require_validity_set(&layer.validity, "validity")?, + layer.num_slots, + "validity positions", + decompressors, + )?; + SparseLayerDecompressors::Validity { + num_slots: layer.num_slots, + validity, + } + } + pb21::sparse_structural_layer::Layer::List(layer) => { + let non_empty_positions = + Self::require_position_set(&layer.non_empty_positions, "list non-empty")?; + let (non_empty_positions, num_non_empty) = Self::position_set_decoder( + non_empty_positions, + layer.num_slots, + "list non-empty positions", + decompressors, + )?; + let counts = Self::count_set_decoder( + Self::require_count_set(&layer.counts, "list counts")?, + num_non_empty, + "list counts", + decompressors, + )?; + let (validity, _) = Self::validity_set_decoder( + Self::require_validity_set(&layer.validity, "list")?, + layer.num_slots, + "list validity positions", + decompressors, + )?; + SparseLayerDecompressors::List { + num_slots: layer.num_slots, + num_child_slots: layer.num_child_slots, + non_empty_positions, + counts, + validity, + } + } + pb21::sparse_structural_layer::Layer::FixedSizeList(layer) => { + let (validity, _) = Self::validity_set_decoder( + Self::require_validity_set(&layer.validity, "fixed-size-list")?, + layer.num_slots, + "fixed-size-list validity positions", + decompressors, + )?; + SparseLayerDecompressors::FixedSizeList { + num_slots: layer.num_slots, + num_child_slots: layer.num_slots.checked_mul(layer.dimension).ok_or_else( + || { + Error::invalid_input_source( + "Sparse structural fixed-size-list child slot count overflows" + .into(), + ) + }, + )?, + dimension: layer.dimension, + validity, + } + } + }) + } + + fn parse_chunk_meta(&self, meta_bytes: Bytes) -> Result> { + if !meta_bytes.len().is_multiple_of(8) { + return Err(Error::invalid_input_source( + format!( + "Sparse layout metadata length {} is not a multiple of 8", + meta_bytes.len() + ) + .into(), + )); + } + + let (value_buf_position, value_buf_size) = self.value_buffer()?; + let value_buf_end = value_buf_position + .checked_add(value_buf_size) + .ok_or_else(|| { + Error::invalid_input_source("Sparse layout value buffer range overflows".into()) + })?; + let mut rows_counter = 0_u64; + let mut offset_bytes = value_buf_position; + let mut chunk_meta = Vec::with_capacity(meta_bytes.len() / 8); + for chunk in meta_bytes.chunks_exact(8) { + let entry: [u8; 8] = chunk.try_into().map_err(|_| { + Error::invalid_input_source( + "Sparse layout chunk metadata entry is not 8 bytes".into(), + ) + })?; + let divided_bytes_minus_one = u32::from_le_bytes( + entry + .get(..4) + .and_then(|bytes| bytes.try_into().ok()) + .ok_or_else(|| { + Error::invalid_input_source( + "Sparse layout chunk byte-size field is malformed".into(), + ) + })?, + ); + let num_values = u64::from(u32::from_le_bytes( + entry + .get(4..) + .and_then(|bytes| bytes.try_into().ok()) + .ok_or_else(|| { + Error::invalid_input_source( + "Sparse layout chunk value-count field is malformed".into(), + ) + })?, + )); + if num_values == 0 { + return Err(Error::invalid_input_source( + "Sparse layout contains an empty value chunk".into(), + )); + } + if num_values > miniblock::MAX_CONFIGURABLE_MINIBLOCK_VALUES { + return Err(Error::invalid_input_source( + format!( + "Sparse layout value chunk has {} values, exceeding the mini-block limit {}", + num_values, + miniblock::MAX_CONFIGURABLE_MINIBLOCK_VALUES + ) + .into(), + )); + } + let num_bytes = u64::from(divided_bytes_minus_one) + .checked_add(1) + .and_then(|units| units.checked_mul(MINIBLOCK_ALIGNMENT as u64)) + .ok_or_else(|| { + Error::invalid_input_source( + "Sparse layout value chunk byte size overflows".into(), + ) + })?; + rows_counter = rows_counter.checked_add(num_values).ok_or_else(|| { + Error::invalid_input_source("Sparse layout visible item count overflows".into()) + })?; + chunk_meta.push(ChunkMeta { + num_values, + chunk_size_bytes: num_bytes, + offset_bytes, + }); + offset_bytes = offset_bytes.checked_add(num_bytes).ok_or_else(|| { + Error::invalid_input_source("Sparse layout value chunk byte range overflows".into()) + })?; + } + if rows_counter != self.num_visible_items { + return Err(Error::invalid_input_source( + format!( + "Sparse layout visible item count mismatch: metadata has {}, layout has {}", + rows_counter, self.num_visible_items + ) + .into(), + )); + } + if offset_bytes != value_buf_end { + return Err(Error::invalid_input_source( + format!( + "Sparse layout chunk metadata describes {} value bytes, but value buffer has {} bytes", + offset_bytes - value_buf_position, + value_buf_size + ) + .into(), + )); + } + Ok(chunk_meta) + } + + fn validate_general_buffer_header( + general: &pb21::General, + data: &[u8], + label: &str, + ) -> Result<()> { + let compression = general.compression.as_ref().ok_or_else(|| { + Error::invalid_input_source( + format!("Sparse structural {label} general compression is missing config").into(), + ) + })?; + Self::validate_buffer_compression(Some(compression), label)?; + let values = Self::require_encoding(&general.values, label)?; + if Self::encoding_contains_general(values) { + return Err(Error::invalid_input_source( + format!( + "Sparse structural {label} contains nested general compression, which is unsupported" + ) + .into(), + )); + } + + let scheme = invalid_enum( + pb21::CompressionScheme::try_from(compression.scheme), + "compression scheme", + )?; + match scheme { + pb21::CompressionScheme::CompressionAlgorithmLz4 => { + data.get(..4).ok_or_else(|| { + Error::invalid_input_source( + format!( + "Sparse structural {label} LZ4 buffer is missing its length prefix" + ) + .into(), + ) + })?; + } + pb21::CompressionScheme::CompressionAlgorithmZstd => { + data.get(..8).ok_or_else(|| { + Error::invalid_input_source( + format!( + "Sparse structural {label} Zstd buffer is missing its length prefix" + ) + .into(), + ) + })?; + } + pb21::CompressionScheme::CompressionAlgorithmUnspecified => { + return Err(Error::invalid_input_source( + format!("Sparse structural {label} general compression scheme is unspecified") + .into(), + )); + } + } + Ok(()) + } + + fn validate_general_child_buffer( + encoding: &CompressiveEncoding, + data: &[u8], + label: &str, + ) -> Result<()> { + if let Some(pb21::compressive_encoding::Compression::General(general)) = + encoding.compression.as_ref() + { + Self::validate_general_buffer_header(general, data, label)?; + } + Ok(()) + } + + fn validate_structural_buffer_headers( + encoding: &CompressiveEncoding, + data: &[u8], + label: &str, + ) -> Result<()> { + use pb21::compressive_encoding::Compression; + + match encoding.compression.as_ref() { + Some(Compression::General(general)) => { + Self::validate_general_buffer_header(general, data, label) + } + Some(Compression::Rle(rle)) => { + let values_size = u64::from_le_bytes( + data.get(..8) + .ok_or_else(|| { + Error::invalid_input_source( + format!( + "Sparse structural {label} RLE buffer is missing its header" + ) + .into(), + ) + })? + .try_into() + .map_err(|_| { + Error::invalid_input_source( + format!("Sparse structural {label} RLE header is malformed").into(), + ) + })?, + ); + let values_size = usize_from_u64(values_size, "RLE values buffer size")?; + let values_end = 8_usize.checked_add(values_size).ok_or_else(|| { + Error::invalid_input_source( + format!("Sparse structural {label} RLE values range overflows").into(), + ) + })?; + let values_data = data.get(8..values_end).ok_or_else(|| { + Error::invalid_input_source( + format!("Sparse structural {label} RLE values buffer is truncated").into(), + ) + })?; + let lengths_data = data.get(values_end..).ok_or_else(|| { + Error::invalid_input_source( + format!("Sparse structural {label} RLE run-length buffer is missing") + .into(), + ) + })?; + Self::validate_general_child_buffer( + Self::require_encoding(&rle.values, "RLE values")?, + values_data, + "RLE values", + )?; + Self::validate_general_child_buffer( + Self::require_encoding(&rle.run_lengths, "RLE run lengths")?, + lengths_data, + "RLE run lengths", + ) + } + _ => Ok(()), + } + } + + fn decode_u64_values( + decompressor: &dyn BlockDecompressor, + encoding: &CompressiveEncoding, + data: Bytes, + num_values: u64, + label: &str, + ) -> Result> { + Self::validate_structural_buffer_headers(encoding, &data, label)?; + let decoded = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + decompressor.decompress(LanceBuffer::from_bytes(data, 1), num_values) + })) + .map_err(|_| { + Error::invalid_input_source( + format!("Sparse structural {label} decompression panicked").into(), + ) + })? + .map_err(|error| { + Error::invalid_input_source( + format!("Sparse structural {label} decompression failed: {error}").into(), + ) + })?; + let fixed = decoded.as_fixed_width().ok_or_else(|| { + Error::invalid_input_source( + format!("Sparse structural {label} did not decode to fixed width data").into(), + ) + })?; + if fixed.bits_per_value != 64 { + return Err(Error::invalid_input_source( + format!( + "Sparse structural {label} decoded to {} bits per value, expected 64", + fixed.bits_per_value + ) + .into(), + )); + } + if fixed.num_values != num_values { + return Err(Error::invalid_input_source( + format!( + "Sparse structural {label} decoded {} values, expected {}", + fixed.num_values, num_values + ) + .into(), + )); + } + let num_values_usize = usize::try_from(num_values).map_err(|_| { + Error::invalid_input_source( + format!("Sparse structural {label} value count exceeds usize::MAX").into(), + ) + })?; + let expected_len = num_values_usize + .checked_mul(std::mem::size_of::()) + .ok_or_else(|| { + Error::invalid_input_source( + format!("Sparse structural {label} decoded byte length overflows").into(), + ) + })?; + if fixed.data.len() != expected_len { + return Err(Error::invalid_input_source( + format!( + "Sparse structural {label} decoded {} bytes, expected {}", + fixed.data.len(), + expected_len + ) + .into(), + )); + } + let values = fixed.data.borrow_to_typed_slice::(); + if values.len() != num_values_usize { + return Err(Error::invalid_input_source( + format!( + "Sparse structural {label} decoded {} u64 values, expected {}", + values.len(), + num_values + ) + .into(), + )); + } + Ok(values.to_vec()) + } + + fn decode_explicit_positions( + decompressor: &Arc, + encoding: &CompressiveEncoding, + data: Bytes, + num_positions: u64, + num_slots: u64, + label: &str, + ) -> Result { + let deltas = + Self::decode_u64_values(decompressor.as_ref(), encoding, data, num_positions, label)?; + let mut positions = Vec::with_capacity(deltas.len()); + let mut current = 0_u64; + for (idx, delta) in deltas.into_iter().enumerate() { + if idx == 0 { + current = delta; + } else { + if delta == 0 { + return Err(Error::invalid_input_source( + format!("Sparse structural {label} positions must be strictly increasing") + .into(), + )); + } + current = current.checked_add(delta).ok_or_else(|| { + Error::invalid_input_source( + format!("Sparse structural {label} position overflow").into(), + ) + })?; + } + if current >= num_slots { + return Err(Error::invalid_input_source( + format!( + "Sparse structural {label} position {} is outside layer with {} slots", + current, num_slots + ) + .into(), + )); + } + positions.push(current); + } + SparsePositionSet::from_positions(positions, num_slots, label) + } + + fn decode_position_set( + decoder: &SparsePositionSetDecoder, + buffers: &mut impl Iterator, + label: &str, + ) -> Result { + match decoder { + SparsePositionSetDecoder::Empty => Ok(SparsePositionSet::empty()), + SparsePositionSetDecoder::All { len } => Ok(SparsePositionSet::all(*len)), + SparsePositionSetDecoder::Range { start, len } => { + Ok(SparsePositionSet::range(*start, *len)) + } + SparsePositionSetDecoder::Explicit { + decompressor, + encoding, + count, + domain_len, + } => Self::decode_explicit_positions( + decompressor, + encoding, + Self::next_structural_buffer(buffers, label)?, + *count, + *domain_len, + label, + ), + } + } + + fn decode_validity_set( + decoder: &SparseValiditySetDecoder, + buffers: &mut impl Iterator, + label: &str, + ) -> Result { + Ok(SparseValiditySet { + meaning: decoder.meaning, + positions: Self::decode_position_set(&decoder.positions, buffers, label)?, + }) + } + + fn decode_count_set( + decoder: &SparseCountSetDecoder, + buffers: &mut impl Iterator, + label: &str, + ) -> Result { + match decoder { + SparseCountSetDecoder::Empty => Ok(SparseCountSet::Empty), + SparseCountSetDecoder::Constant { value, len } => { + Ok(SparseCountSet::constant(*value, *len)) + } + SparseCountSetDecoder::Explicit { + decompressor, + encoding, + count, + } => { + let counts = Self::decode_u64_values( + decompressor.as_ref(), + encoding, + Self::next_structural_buffer(buffers, label)?, + *count, + label, + )?; + SparseCountSet::from_counts(counts) + } + } + } + + fn next_structural_buffer( + buffers: &mut impl Iterator, + label: &str, + ) -> Result { + buffers.next().ok_or_else(|| { + Error::invalid_input_source( + format!("Sparse structural {label} is missing its buffer").into(), + ) + }) + } + + fn decode_layer( + layer: &SparseLayerDecompressors, + buffers: &mut impl Iterator, + ) -> Result { + Ok(match layer { + SparseLayerDecompressors::Validity { + num_slots, + validity, + } => SparseStructuralLayerPlan::Validity { + num_slots: *num_slots, + validity: Self::decode_validity_set(validity, buffers, "validity positions")?, + }, + SparseLayerDecompressors::List { + num_slots, + num_child_slots, + non_empty_positions, + counts, + validity, + } => { + let non_empty_positions = Self::decode_position_set( + non_empty_positions, + buffers, + "list non-empty positions", + )?; + let counts = Self::decode_count_set(counts, buffers, "list counts")?; + let validity = + Self::decode_validity_set(validity, buffers, "list validity positions")?; + let actual_child_slots = counts.sum()?; + if actual_child_slots != *num_child_slots { + return Err(Error::invalid_input_source( + format!( + "Sparse structural list count sum {} does not match declared child slots {}", + actual_child_slots, num_child_slots + ) + .into(), + )); + } + SparseStructuralLayerPlan::List { + num_slots: *num_slots, + num_child_slots: *num_child_slots, + non_empty_positions, + counts, + validity, + } + } + SparseLayerDecompressors::FixedSizeList { + num_slots, + num_child_slots, + dimension, + validity, + } => { + let expected_child_slots = num_slots.checked_mul(*dimension).ok_or_else(|| { + Error::invalid_input_source( + format!( + "Sparse structural fixed-size-list child slot count overflows: slots={}, dimension={}", + num_slots, dimension + ) + .into(), + ) + })?; + if expected_child_slots != *num_child_slots { + return Err(Error::invalid_input_source( + format!( + "Sparse structural fixed-size-list child slot count {} does not match slots {} * dimension {}", + num_child_slots, num_slots, dimension + ) + .into(), + )); + } + SparseStructuralLayerPlan::FixedSizeList { + num_slots: *num_slots, + dimension: *dimension, + validity: Self::decode_validity_set( + validity, + buffers, + "fixed-size-list validity positions", + )?, + } + } + }) + } + + fn lookup_value_chunks(&self, chunk_indices: &[usize]) -> Result> { + let page_meta = self.page_meta.as_ref().ok_or_else(|| { + Error::internal("Sparse page scheduler has not been initialized".to_string()) + })?; + chunk_indices + .iter() + .map(|&chunk_idx| { + let chunk_meta = page_meta.chunk_meta.get(chunk_idx).ok_or_else(|| { + Error::invalid_input_source( + format!("Sparse layout missing value chunk metadata for chunk {chunk_idx}") + .into(), + ) + })?; + let bytes_start = chunk_meta.offset_bytes; + let bytes_end = bytes_start + .checked_add(chunk_meta.chunk_size_bytes) + .ok_or_else(|| { + Error::invalid_input_source( + format!( + "Sparse layout value chunk {} byte range overflows", + chunk_idx + ) + .into(), + ) + })?; + Ok(LoadedChunk { + byte_range: bytes_start..bytes_end, + items_in_chunk: chunk_meta.num_values, + chunk_idx, + data: LanceBuffer::empty(), + }) + }) + .collect() + } + + fn value_chunk_index(chunk_value_offsets: &[u64], value: u64) -> Result { + let total_values = chunk_value_offsets.last().copied().ok_or_else(|| { + Error::invalid_input_source("Sparse layout has no value chunk offsets".into()) + })?; + if chunk_value_offsets.len() < 2 || value >= total_values { + return Err(Error::invalid_input_source( + format!( + "Sparse layout value index {} is outside {} visible items", + value, total_values + ) + .into(), + )); + } + chunk_value_offsets + .partition_point(|&offset| offset <= value) + .checked_sub(1) + .ok_or_else(|| { + Error::invalid_input_source( + format!("Sparse layout value index {value} is before the first chunk").into(), + ) + }) + } + + fn value_chunk_range( + chunk_value_offsets: &[u64], + value_range: Range, + ) -> Result> { + if value_range.is_empty() { + return Ok(0..0); + } + let total_values = chunk_value_offsets.last().copied().ok_or_else(|| { + Error::invalid_input_source("Sparse layout has no value chunk offsets".into()) + })?; + if value_range.start > value_range.end || value_range.end > total_values { + return Err(Error::invalid_input_source( + format!( + "Sparse layout value range {}..{} is outside {} visible items", + value_range.start, value_range.end, total_values + ) + .into(), + )); + } + let start = Self::value_chunk_index(chunk_value_offsets, value_range.start)?; + let end = chunk_value_offsets + .partition_point(|&offset| offset < value_range.end) + .max(start + 1); + Ok(start..end) + } +} + +impl StructuralPageScheduler for SparseStructuralScheduler { + fn initialize<'a>( + &'a mut self, + io: &Arc, + ) -> BoxFuture<'a, Result>> { + let (meta_buf_position, meta_buf_size) = match self.metadata_buffer() { + Ok(buffer) => buffer, + Err(err) => return std::future::ready(Err(err)).boxed(), + }; + let required_ranges = match (|| -> Result>> { + let mut required_ranges = Vec::new(); + required_ranges.push(Self::checked_buffer_range( + meta_buf_position, + meta_buf_size, + "metadata", + )?); + for (position, size) in self.buffer_offsets_and_sizes.iter().skip(2) { + required_ranges.push(Self::checked_buffer_range(*position, *size, "structural")?); + } + Ok(required_ranges) + })() { + Ok(ranges) => ranges, + Err(err) => return std::future::ready(Err(err)).boxed(), + }; + let io_req = io.submit_request(required_ranges, 0); + + async move { + let mut buffers = io_req.await?.into_iter(); + let meta_bytes = buffers.next().ok_or_else(|| { + Error::invalid_input_source("Sparse layout is missing chunk metadata buffer".into()) + })?; + + let chunk_meta = self.parse_chunk_meta(meta_bytes)?; + let mut chunk_value_offsets = Vec::with_capacity(chunk_meta.len() + 1); + let mut value_offset = 0_u64; + chunk_value_offsets.push(value_offset); + for chunk in &chunk_meta { + value_offset = value_offset.checked_add(chunk.num_values).ok_or_else(|| { + Error::invalid_input_source( + "Sparse layout visible item offset overflows".into(), + ) + })?; + chunk_value_offsets.push(value_offset); + } + + let layers = self + .layer_decompressors + .iter() + .map(|layer| Self::decode_layer(layer, &mut buffers)) + .collect::>>()?; + let plan = SparseStructuralPlan { + layers, + num_items: self.num_items, + num_visible_items: self.num_visible_items, + }; + plan.validate(self.row_domain)?; + if buffers.next().is_some() { + return Err(Error::invalid_input_source( + "Sparse layout has unused structural buffers".into(), + )); + } + + let page_meta = Arc::new(SparseStructuralCacheableState { + chunk_meta, + chunk_value_offsets: chunk_value_offsets.into(), + plan, + row_domain: self.row_domain, + }); + self.page_meta = Some(page_meta.clone()); + Ok(page_meta as Arc) + } + .boxed() + } + + fn load(&mut self, data: &Arc) { + self.page_meta = data + .clone() + .as_arc_any() + .downcast::() + .ok(); + } + + fn schedule_ranges( + &self, + ranges: &[Range], + io: &Arc, + ) -> Result> { + let page_meta = self.page_meta.as_ref().ok_or_else(|| { + Error::internal("Sparse page scheduler has not been initialized".to_string()) + })?; + let encoded_row_domain = page_meta + .row_domain + .checked_mul(self.row_scale) + .ok_or_else(|| { + Error::invalid_input_source("Sparse structural encoded row domain overflows".into()) + })?; + let num_rows = validate_slice_ranges(ranges, encoded_row_domain, "row")?; + let ranges = ranges + .iter() + .map(|range| { + if !range.start.is_multiple_of(self.row_scale) + || !range.end.is_multiple_of(self.row_scale) + { + return Err(Error::invalid_input_source( + format!( + "Sparse structural encoded row range {}..{} is not aligned to fixed-size-list scale {}", + range.start, range.end, self.row_scale + ) + .into(), + )); + } + Ok((range.start / self.row_scale)..(range.end / self.row_scale)) + }) + .collect::>>()?; + + let mut chunks_needed = Vec::new(); + let selection = slice_sparse_plan(&page_meta.plan, &ranges, page_meta.row_domain)?; + for value_range in &selection.leaf_ranges { + chunks_needed.extend(Self::value_chunk_range( + &page_meta.chunk_value_offsets, + value_range.clone(), + )?); + } + chunks_needed.sort_unstable(); + chunks_needed.dedup(); + + let mut loaded_chunks = self.lookup_value_chunks(&chunks_needed)?; + let chunk_ranges = loaded_chunks + .iter() + .map(|chunk| chunk.byte_range.clone()) + .collect::>(); + let loaded_chunk_data = io.submit_request(chunk_ranges, self.priority); + let ranges = VecDeque::from(ranges); + let value_decompressor = self.value_decompressor.clone(); + let value_encoding = self.value_encoding.clone(); + let data_type = self.data_type.clone(); + let page_meta = page_meta.clone(); + let num_buffers = self.num_buffers; + let has_large_chunk = self.has_large_chunk; + let row_scale = self.row_scale; + + let res = async move { + let loaded_chunk_data = loaded_chunk_data.await?; + for (loaded_chunk, chunk_data) in loaded_chunks.iter_mut().zip(loaded_chunk_data) { + loaded_chunk.data = LanceBuffer::from_bytes(chunk_data, 1); + } + + Ok(Box::new(SparseStructuralDecoder { + value_decompressor, + value_encoding, + data_type, + page_meta, + loaded_chunks: Arc::new(loaded_chunks), + ranges, + offset_in_current_range: 0, + num_rows, + row_scale, + num_buffers, + has_large_chunk, + }) as Box) + } + .boxed(); + Ok(vec![PageLoadTask { + decoder_fut: res, + num_rows, + }]) + } +} + +#[derive(Debug)] +struct SparseStructuralDecoder { + value_decompressor: Arc, + value_encoding: CompressiveEncoding, + data_type: DataType, + page_meta: Arc, + loaded_chunks: Arc>, + ranges: VecDeque>, + offset_in_current_range: u64, + num_rows: u64, + row_scale: u64, + num_buffers: u64, + has_large_chunk: bool, +} + +impl SparseStructuralDecoder { + fn drain_ranges(&mut self, mut rows_desired: u64) -> Result>> { + if !rows_desired.is_multiple_of(self.row_scale) { + return Err(Error::invalid_input_source( + format!( + "Sparse page decoder drain of {} encoded rows is not aligned to fixed-size-list scale {}", + rows_desired, self.row_scale + ) + .into(), + )); + } + rows_desired /= self.row_scale; + let mut ranges = Vec::new(); + while rows_desired > 0 { + let range = self.ranges.front().ok_or_else(|| { + Error::invalid_input_source( + "Sparse page decoder was asked to drain more rows than were scheduled".into(), + ) + })?; + let start = range + .start + .checked_add(self.offset_in_current_range) + .ok_or_else(|| { + Error::invalid_input_source("Sparse page decoder row offset overflows".into()) + })?; + let rows_available = range.end.checked_sub(start).ok_or_else(|| { + Error::invalid_input_source( + "Sparse page decoder range offset exceeds the scheduled range".into(), + ) + })?; + let rows_to_take = rows_available.min(rows_desired); + let end = start.checked_add(rows_to_take).ok_or_else(|| { + Error::invalid_input_source("Sparse page decoder row range overflows".into()) + })?; + ranges.push(start..end); + rows_desired -= rows_to_take; + self.offset_in_current_range += rows_to_take; + if self.offset_in_current_range == range.end - range.start { + self.offset_in_current_range = 0; + self.ranges.pop_front(); + } + } + Ok(ranges) + } +} + +impl StructuralPageDecoder for SparseStructuralDecoder { + fn drain(&mut self, num_rows: u64) -> Result> { + Ok(Box::new(DecodeSparseStructuralTask { + row_ranges: self.drain_ranges(num_rows)?, + value_decompressor: self.value_decompressor.clone(), + value_encoding: self.value_encoding.clone(), + data_type: self.data_type.clone(), + page_meta: self.page_meta.clone(), + loaded_chunks: self.loaded_chunks.clone(), + num_buffers: self.num_buffers, + has_large_chunk: self.has_large_chunk, + })) + } + + fn num_rows(&self) -> u64 { + self.num_rows + } +} + +#[derive(Debug)] +struct DecodeSparseStructuralTask { + row_ranges: Vec>, + value_decompressor: Arc, + value_encoding: CompressiveEncoding, + data_type: DataType, + page_meta: Arc, + loaded_chunks: Arc>, + num_buffers: u64, + has_large_chunk: bool, +} + +impl DecodeSparseStructuralTask { + fn read_chunk_size( + buf: &[u8], + offset: &mut usize, + width: usize, + chunk_idx: usize, + ) -> Result { + let end = offset.checked_add(width).ok_or_else(|| { + Error::invalid_input_source( + format!("Sparse structural value chunk {chunk_idx} size header overflows").into(), + ) + })?; + let bytes = buf.get(*offset..end).ok_or_else(|| { + Error::invalid_input_source( + format!("Sparse structural value chunk {chunk_idx} has a truncated size header") + .into(), + ) + })?; + let size = match width { + 2 => u32::from(u16::from_le_bytes(bytes.try_into().map_err(|_| { + Error::invalid_input_source( + format!("Sparse structural value chunk {chunk_idx} has a malformed u16 size") + .into(), + ) + })?)), + 4 => u32::from_le_bytes(bytes.try_into().map_err(|_| { + Error::invalid_input_source( + format!("Sparse structural value chunk {chunk_idx} has a malformed u32 size") + .into(), + ) + })?), + _ => { + return Err(Error::internal(format!( + "Unsupported sparse value chunk size width {width}" + ))); + } + }; + *offset = end; + Ok(size) + } + + fn expected_fixed_bytes(num_values: u64, bits_per_value: u64, label: &str) -> Result { + let bits = num_values.checked_mul(bits_per_value).ok_or_else(|| { + Error::invalid_input_source( + format!("Sparse structural {label} decoded bit length overflows").into(), + ) + })?; + usize_from_u64(bits.div_ceil(8), label) + } + + fn validate_fixed_buffer( + buffer: &LanceBuffer, + num_values: u64, + bits_per_value: u64, + label: &str, + ) -> Result<()> { + let expected = Self::expected_fixed_bytes(num_values, bits_per_value, label)?; + if buffer.len() != expected { + return Err(Error::invalid_input_source( + format!( + "Sparse structural {label} buffer has {} bytes, expected {}", + buffer.len(), + expected + ) + .into(), + )); + } + Ok(()) + } + + fn validate_variable_buffer( + buffer: &LanceBuffer, + num_values: u64, + bits_per_offset: u64, + ) -> Result<()> { + let width = usize_from_u64(bits_per_offset / 8, "variable offset width")?; + let offset_count = num_values.checked_add(1).ok_or_else(|| { + Error::invalid_input_source("Sparse variable offset count overflows".into()) + })?; + let table_len = usize_from_u64(offset_count, "variable offset count")? + .checked_mul(width) + .ok_or_else(|| { + Error::invalid_input_source("Sparse variable offset table size overflows".into()) + })?; + if buffer.len() < table_len { + return Err(Error::invalid_input_source( + format!( + "Sparse variable buffer has {} bytes, smaller than its {}-byte offset table", + buffer.len(), + table_len + ) + .into(), + )); + } + let mut previous = None; + for index in 0..usize_from_u64(offset_count, "variable offset count")? { + let start = index.checked_mul(width).ok_or_else(|| { + Error::invalid_input_source("Sparse variable offset index overflows".into()) + })?; + let end = start.checked_add(width).ok_or_else(|| { + Error::invalid_input_source("Sparse variable offset range overflows".into()) + })?; + let bytes = buffer.as_ref().get(start..end).ok_or_else(|| { + Error::invalid_input_source("Sparse variable offset table is truncated".into()) + })?; + let offset = match width { + 4 => u64::from(u32::from_le_bytes(bytes.try_into().map_err(|_| { + Error::invalid_input_source("Sparse variable u32 offset is malformed".into()) + })?)), + 8 => u64::from_le_bytes(bytes.try_into().map_err(|_| { + Error::invalid_input_source("Sparse variable u64 offset is malformed".into()) + })?), + _ => { + return Err(Error::invalid_input_source( + format!( + "Sparse variable offset width {} is unsupported", + bits_per_offset + ) + .into(), + )); + } + }; + if offset < table_len as u64 || offset > buffer.len() as u64 { + return Err(Error::invalid_input_source( + format!( + "Sparse variable offset {} is outside payload range {}..{}", + offset, + table_len, + buffer.len() + ) + .into(), + )); + } + if previous.is_some_and(|previous| offset < previous) { + return Err(Error::invalid_input_source( + "Sparse variable offsets are not monotonically increasing".into(), + )); + } + previous = Some(offset); + } + Ok(()) + } + + fn validate_fsl_buffers( + fsl: &pb21::FixedSizeList, + buffers: &[LanceBuffer], + num_values: u64, + buffer_index: &mut usize, + ) -> Result<()> { + use pb21::compressive_encoding::Compression; + + let child_values = num_values.checked_mul(fsl.items_per_value).ok_or_else(|| { + Error::invalid_input_source("Sparse fixed-size-list value count overflows".into()) + })?; + if fsl.has_validity { + let validity = buffers.get(*buffer_index).ok_or_else(|| { + Error::invalid_input_source( + "Sparse fixed-size-list value validity buffer is missing".into(), + ) + })?; + Self::validate_fixed_buffer(validity, child_values, 1, "fixed-size-list validity")?; + *buffer_index = buffer_index.checked_add(1).ok_or_else(|| { + Error::invalid_input_source("Sparse fixed-size-list buffer index overflows".into()) + })?; + } + let values = fsl.values.as_deref().ok_or_else(|| { + Error::invalid_input_source("Sparse fixed-size-list value encoding is missing".into()) + })?; + match values.compression.as_ref() { + Some(Compression::FixedSizeList(inner)) => { + Self::validate_fsl_buffers(inner, buffers, child_values, buffer_index) + } + Some(Compression::Flat(flat)) => { + let values = buffers.get(*buffer_index).ok_or_else(|| { + Error::invalid_input_source( + "Sparse fixed-size-list leaf value buffer is missing".into(), + ) + })?; + Self::validate_fixed_buffer( + values, + child_values, + flat.bits_per_value, + "fixed-size-list leaf values", + )?; + *buffer_index = buffer_index.checked_add(1).ok_or_else(|| { + Error::invalid_input_source( + "Sparse fixed-size-list buffer index overflows".into(), + ) + })?; + Ok(()) + } + _ => Err(Error::invalid_input_source( + "Sparse fixed-size-list value encoding is malformed".into(), + )), + } + } + + fn validate_value_buffers(&self, buffers: &[LanceBuffer], num_values: u64) -> Result<()> { + use pb21::compressive_encoding::Compression; + + let compression = self.value_encoding.compression.as_ref().ok_or_else(|| { + Error::invalid_input_source("Sparse value compression is missing".into()) + })?; + match compression { + Compression::Flat(flat) => Self::validate_fixed_buffer( + buffers.first().ok_or_else(|| { + Error::invalid_input_source("Sparse flat value buffer is missing".into()) + })?, + num_values, + flat.bits_per_value, + "flat values", + ), + Compression::InlineBitpacking(bitpacking) => { + let buffer = buffers.first().ok_or_else(|| { + Error::invalid_input_source( + "Sparse inline-bitpacked value buffer is missing".into(), + ) + })?; + if num_values > 1024 { + return Err(Error::invalid_input_source( + format!( + "Sparse inline-bitpacked chunk has {} values, exceeding 1024", + num_values + ) + .into(), + )); + } + let word_bytes = usize_from_u64( + bitpacking.uncompressed_bits_per_value / 8, + "inline bitpacking word width", + )?; + let header = buffer.as_ref().get(..word_bytes).ok_or_else(|| { + Error::invalid_input_source( + "Sparse inline-bitpacked buffer is missing its bit-width header".into(), + ) + })?; + let bit_width = + header + .iter() + .enumerate() + .try_fold(0_u64, |value, (idx, byte)| { + let shift = u32::try_from(idx.checked_mul(8).ok_or_else(|| { + Error::invalid_input_source( + "Sparse inline bit-width shift overflows".into(), + ) + })?) + .map_err(|_| { + Error::invalid_input_source( + "Sparse inline bit-width shift exceeds u32".into(), + ) + })?; + Ok::<_, Error>(value | (u64::from(*byte) << shift)) + })?; + if bit_width > bitpacking.uncompressed_bits_per_value { + return Err(Error::invalid_input_source( + format!( + "Sparse inline bit width {} exceeds uncompressed width {}", + bit_width, bitpacking.uncompressed_bits_per_value + ) + .into(), + )); + } + let payload_bytes = usize_from_u64( + bit_width.checked_mul(1024).ok_or_else(|| { + Error::invalid_input_source( + "Sparse inline-bitpacked payload size overflows".into(), + ) + })? / 8, + "inline-bitpacked payload size", + )?; + let expected = word_bytes.checked_add(payload_bytes).ok_or_else(|| { + Error::invalid_input_source( + "Sparse inline-bitpacked buffer size overflows".into(), + ) + })?; + if buffer.len() != expected { + return Err(Error::invalid_input_source( + format!( + "Sparse inline-bitpacked buffer has {} bytes, expected {}", + buffer.len(), + expected + ) + .into(), + )); + } + Ok(()) + } + Compression::Variable(variable) => { + let offsets = variable + .offsets + .as_deref() + .and_then(|encoding| encoding.compression.as_ref()) + .and_then(|compression| match compression { + Compression::Flat(flat) => Some(flat), + _ => None, + }) + .ok_or_else(|| { + Error::invalid_input_source( + "Sparse variable offset encoding is malformed".into(), + ) + })?; + Self::validate_variable_buffer( + buffers.first().ok_or_else(|| { + Error::invalid_input_source( + "Sparse variable value buffer is missing".into(), + ) + })?, + num_values, + offsets.bits_per_value, + ) + } + Compression::Fsst(fsst) => { + let variable = fsst + .values + .as_deref() + .and_then(|encoding| encoding.compression.as_ref()) + .and_then(|compression| match compression { + Compression::Variable(variable) => Some(variable), + _ => None, + }) + .ok_or_else(|| { + Error::invalid_input_source( + "Sparse FSST value encoding is malformed".into(), + ) + })?; + let offsets = variable + .offsets + .as_deref() + .and_then(|encoding| encoding.compression.as_ref()) + .and_then(|compression| match compression { + Compression::Flat(flat) => Some(flat), + _ => None, + }) + .ok_or_else(|| { + Error::invalid_input_source( + "Sparse FSST offset encoding is malformed".into(), + ) + })?; + Self::validate_variable_buffer( + buffers.first().ok_or_else(|| { + Error::invalid_input_source("Sparse FSST value buffer is missing".into()) + })?, + num_values, + offsets.bits_per_value, + ) + } + Compression::ByteStreamSplit(split) => { + let bits = split + .values + .as_deref() + .and_then(|encoding| encoding.compression.as_ref()) + .and_then(|compression| match compression { + Compression::Flat(flat) => Some(flat.bits_per_value), + _ => None, + }) + .ok_or_else(|| { + Error::invalid_input_source( + "Sparse byte-stream-split encoding is malformed".into(), + ) + })?; + Self::validate_fixed_buffer( + buffers.first().ok_or_else(|| { + Error::invalid_input_source( + "Sparse byte-stream-split buffer is missing".into(), + ) + })?, + num_values, + bits, + "byte-stream-split values", + ) + } + Compression::FixedSizeList(fsl) => { + let mut buffer_index = 0; + Self::validate_fsl_buffers(fsl, buffers, num_values, &mut buffer_index)?; + if buffer_index != buffers.len() { + return Err(Error::invalid_input_source( + format!( + "Sparse fixed-size-list descriptor consumed {} of {} buffers", + buffer_index, + buffers.len() + ) + .into(), + )); + } + Ok(()) + } + Compression::PackedStruct(packed) => { + let bits = packed.bits_per_value.iter().try_fold(0_u64, |sum, bits| { + sum.checked_add(*bits).ok_or_else(|| { + Error::invalid_input_source( + "Sparse packed-struct bit width sum overflows".into(), + ) + }) + })?; + Self::validate_fixed_buffer( + buffers.first().ok_or_else(|| { + Error::invalid_input_source( + "Sparse packed-struct value buffer is missing".into(), + ) + })?, + num_values, + bits, + "packed-struct values", + ) + } + Compression::Rle(rle) => { + if buffers.len() != 2 { + return Err(Error::invalid_input_source( + format!( + "Sparse RLE value chunk has {} buffers, expected 2", + buffers.len() + ) + .into(), + )); + } + SparseStructuralScheduler::validate_general_child_buffer( + SparseStructuralScheduler::require_encoding(&rle.values, "RLE values")?, + buffers + .first() + .ok_or_else(|| { + Error::invalid_input_source( + "Sparse RLE value buffer is missing after count validation".into(), + ) + })? + .as_ref(), + "value chunk RLE values", + )?; + SparseStructuralScheduler::validate_general_child_buffer( + SparseStructuralScheduler::require_encoding( + &rle.run_lengths, + "RLE run lengths", + )?, + buffers + .get(1) + .ok_or_else(|| { + Error::invalid_input_source( + "Sparse RLE run-length buffer is missing after count validation" + .into(), + ) + })? + .as_ref(), + "value chunk RLE run lengths", + ) + } + Compression::General(general) => { + let buffer = buffers.first().ok_or_else(|| { + Error::invalid_input_source( + "Sparse general-compressed value chunk is missing its first buffer".into(), + ) + })?; + SparseStructuralScheduler::validate_general_buffer_header( + general, + buffer.as_ref(), + "value chunk", + ) + } + _ => Err(Error::invalid_input_source( + "Sparse value chunk uses an unsupported compression descriptor".into(), + )), + } + } + + fn loaded_chunk(&self, chunk_idx: usize) -> Result<&LoadedChunk> { + let index = self + .loaded_chunks + .binary_search_by_key(&chunk_idx, |chunk| chunk.chunk_idx) + .map_err(|_| { + Error::internal(format!( + "Sparse structural decode missing loaded value chunk {}", + chunk_idx + )) + })?; + self.loaded_chunks.get(index).ok_or_else(|| { + Error::internal(format!( + "Sparse structural loaded chunk index {} is missing", + index + )) + }) + } + + fn decode_value_chunk(&self, chunk: &LoadedChunk) -> Result { + let buf = &chunk.data; + if buf.len() < 2 { + return Err(Error::invalid_input_source( + format!( + "Sparse structural value chunk {} is too small for its header: {} bytes", + chunk.chunk_idx, + buf.len() + ) + .into(), + )); + } + let num_levels = u16::from_le_bytes( + buf.as_ref() + .get(..2) + .and_then(|bytes| bytes.try_into().ok()) + .ok_or_else(|| { + Error::invalid_input_source( + format!( + "Sparse structural value chunk {} has a malformed level header", + chunk.chunk_idx + ) + .into(), + ) + })?, + ); + let mut offset: usize = 2; + if num_levels != 0 { + return Err(Error::invalid_input_source( + format!( + "Sparse structural value chunk unexpectedly contains {} rep/def levels", + num_levels + ) + .into(), + )); + } + + let size_width = if self.has_large_chunk { 4 } else { 2 }; + let num_buffers = usize::try_from(self.num_buffers).map_err(|_| { + Error::invalid_input_source( + format!( + "Sparse structural value chunk has too many buffers: {}", + self.num_buffers + ) + .into(), + ) + })?; + let sizes_len = num_buffers.checked_mul(size_width).ok_or_else(|| { + Error::invalid_input_source( + "Sparse structural value chunk buffer-size header overflows".into(), + ) + })?; + let header_len = offset.checked_add(sizes_len).ok_or_else(|| { + Error::invalid_input_source( + "Sparse structural value chunk header length overflows".into(), + ) + })?; + if buf.len() < header_len { + return Err(Error::invalid_input_source( + format!( + "Sparse structural value chunk {} is too small for {} buffer sizes: {} bytes", + chunk.chunk_idx, + self.num_buffers, + buf.len() + ) + .into(), + )); + } + let buffer_sizes = (0..num_buffers) + .map(|_| Self::read_chunk_size(buf, &mut offset, size_width, chunk.chunk_idx)) + .collect::>>()?; + + offset = offset + .checked_add(pad_bytes::(offset)) + .ok_or_else(|| { + Error::invalid_input_source( + format!( + "Sparse structural value chunk {} padded header overflows", + chunk.chunk_idx + ) + .into(), + ) + })?; + if offset > buf.len() { + return Err(Error::invalid_input_source( + format!( + "Sparse structural value chunk {} is missing padding after its header", + chunk.chunk_idx + ) + .into(), + )); + } + let buffers = buffer_sizes + .into_iter() + .map(|buf_size| { + let buf_size = buf_size as usize; + let end = offset.checked_add(buf_size).ok_or_else(|| { + Error::invalid_input_source( + format!( + "Sparse structural value chunk {} buffer size overflows", + chunk.chunk_idx + ) + .into(), + ) + })?; + if end > buf.len() { + return Err(Error::invalid_input_source( + format!( + "Sparse structural value chunk {} buffer extends past chunk end", + chunk.chunk_idx + ) + .into(), + )); + } + let buffer = buf.slice_with_length(offset, buf_size); + offset = end; + offset = offset + .checked_add(pad_bytes::(offset)) + .ok_or_else(|| { + Error::invalid_input_source( + format!( + "Sparse structural value chunk {} padded buffer range overflows", + chunk.chunk_idx + ) + .into(), + ) + })?; + if offset > buf.len() { + return Err(Error::invalid_input_source( + format!( + "Sparse structural value chunk {} padding extends past chunk end", + chunk.chunk_idx + ) + .into(), + )); + } + Ok(buffer) + }) + .collect::>>()?; + + if offset != buf.len() { + return Err(Error::invalid_input_source( + format!( + "Sparse structural value chunk {} consumed {} of {} bytes", + chunk.chunk_idx, + offset, + buf.len() + ) + .into(), + )); + } + + self.validate_value_buffers(&buffers, chunk.items_in_chunk)?; + + let decoded = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + self.value_decompressor + .decompress(buffers, chunk.items_in_chunk) + })) + .map_err(|_| { + Error::invalid_input_source( + format!( + "Sparse structural value chunk {} decompression panicked", + chunk.chunk_idx + ) + .into(), + ) + })? + .map_err(|error| { + Error::invalid_input_source( + format!( + "Sparse structural value chunk {} decompression failed: {error}", + chunk.chunk_idx + ) + .into(), + ) + })?; + if decoded.num_values() != chunk.items_in_chunk { + return Err(Error::invalid_input_source( + format!( + "Sparse structural value chunk {} decoded {} values, expected {}", + chunk.chunk_idx, + decoded.num_values(), + chunk.items_in_chunk + ) + .into(), + )); + } + Ok(decoded) + } + + fn append_value_range( + &self, + value_range: Range, + data_builder: &mut DataBlockBuilder, + chunk_cache: &mut Option<(usize, DataBlock)>, + ) -> Result<()> { + let mut value_start = value_range.start; + while value_start < value_range.end { + let chunk_idx = SparseStructuralScheduler::value_chunk_index( + &self.page_meta.chunk_value_offsets, + value_start, + )?; + let chunk_value_start = *self + .page_meta + .chunk_value_offsets + .get(chunk_idx) + .ok_or_else(|| { + Error::invalid_input_source( + format!("Sparse value chunk {} has no start offset", chunk_idx).into(), + ) + })?; + let chunk_value_end = *self + .page_meta + .chunk_value_offsets + .get(chunk_idx.checked_add(1).ok_or_else(|| { + Error::invalid_input_source("Sparse value chunk index overflows".into()) + })?) + .ok_or_else(|| { + Error::invalid_input_source( + format!("Sparse value chunk {} has no end offset", chunk_idx).into(), + ) + })?; + let take_end = value_range.end.min(chunk_value_end); + if value_start < chunk_value_start || take_end <= value_start { + return Err(Error::invalid_input_source( + format!( + "Sparse value range {}..{} does not make progress in chunk {} covering {}..{}", + value_range.start, + value_range.end, + chunk_idx, + chunk_value_start, + chunk_value_end + ) + .into(), + )); + } + + if !matches!(chunk_cache, Some((cached_idx, _)) if *cached_idx == chunk_idx) { + let chunk = self.loaded_chunk(chunk_idx)?; + *chunk_cache = Some((chunk_idx, self.decode_value_chunk(chunk)?)); + } + let values = &chunk_cache + .as_ref() + .ok_or_else(|| Error::internal("Sparse structural chunk cache is empty"))? + .1; + data_builder.append( + values, + value_start - chunk_value_start..take_end - chunk_value_start, + )?; + value_start = take_end; + } + Ok(()) + } + + fn decode_checked(self) -> Result { + let selection = slice_sparse_plan( + &self.page_meta.plan, + &self.row_ranges, + self.page_meta.row_domain, + )?; + let estimated_size_bytes = self + .loaded_chunks + .iter() + .map(|chunk| chunk.data.len()) + .try_fold(0_usize, |total, len| total.checked_add(len)) + .and_then(|total| total.checked_mul(2)) + .ok_or_else(|| { + Error::invalid_input_source( + "Sparse structural decode size estimate overflows".into(), + ) + })?; + let mut data_builder = DataBlockBuilder::with_capacity_estimate( + u64::try_from(estimated_size_bytes).map_err(|_| { + Error::invalid_input_source("Sparse structural decode size exceeds u64::MAX".into()) + })?, + ); + let mut chunk_cache: Option<(usize, DataBlock)> = None; + let mut appended_values = false; + for value_range in &selection.leaf_ranges { + self.append_value_range(value_range.clone(), &mut data_builder, &mut chunk_cache)?; + appended_values = true; + } + + let data = if appended_values { + data_builder.finish() + } else { + DataBlock::from_array(new_empty_array(&self.data_type)) + }; + let unraveler = RepDefUnraveler::new_sparse(selection.plan); + Ok(DecodedPage { + data, + repdef: unraveler, + }) + } +} + +impl DecodePageTask for DecodeSparseStructuralTask { + fn decode(self: Box) -> Result { + std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| (*self).decode_checked())) + .map_err(|_| { + Error::invalid_input_source( + "Sparse structural page decoding panicked on malformed input".into(), + ) + })? + } +} + +struct SparseStructuralSelection { + plan: SparseStructuralPlan, + leaf_ranges: Vec>, +} + +struct SparsePositionSelection { + positions: SparsePositionSet, + ordinal_ranges: Vec>, +} + +fn validate_slice_ranges(ranges: &[Range], domain_len: u64, label: &str) -> Result { + let mut total = 0_u64; + for range in ranges { + if range.start > range.end || range.end > domain_len { + return Err(Error::invalid_input_source( + format!( + "Sparse structural {label} slice {}..{} is outside domain {}", + range.start, range.end, domain_len + ) + .into(), + )); + } + total = total.checked_add(range.end - range.start).ok_or_else(|| { + Error::invalid_input_source( + format!("Sparse structural {label} slice length overflows").into(), + ) + })?; + } + Ok(total) +} + +fn push_coalesced_range(ranges: &mut Vec>, range: Range) { + if range.is_empty() { + return; + } + if let Some(last) = ranges.last_mut() + && last.end == range.start + { + last.end = range.end; + return; + } + ranges.push(range); +} + +fn position_segments_to_set( + segments: Vec>, + output_domain: u64, + label: &str, +) -> Result { + let segments = coalesce_ranges(segments); + if segments.is_empty() { + return Ok(SparsePositionSet::empty()); + } + if segments.len() == 1 { + let segment = segments.first().ok_or_else(|| { + Error::internal("Sparse structural segment unexpectedly missing".to_string()) + })?; + if segment.start == 0 && segment.end == output_domain { + return Ok(SparsePositionSet::all(output_domain)); + } + return Ok(SparsePositionSet::range( + segment.start, + segment.end - segment.start, + )); + } + + let total_len = segments + .iter() + .map(|range| range.end - range.start) + .try_fold(0_u64, |sum, len| { + sum.checked_add(len).ok_or_else(|| { + Error::invalid_input_source( + format!("Sparse structural {label} segment length overflows").into(), + ) + }) + })?; + let total_len = usize::try_from(total_len).map_err(|_| { + Error::invalid_input_source( + format!("Sparse structural {label} segment length exceeds usize::MAX").into(), + ) + })?; + let mut positions = Vec::with_capacity(total_len); + for segment in segments { + positions.extend(segment); + } + SparsePositionSet::from_positions(positions, output_domain, label) +} + +fn select_position_set( + positions: &SparsePositionSet, + ranges: &[Range], + domain_len: u64, + label: &str, +) -> Result { + let output_domain = validate_slice_ranges(ranges, domain_len, label)?; + let mut segments = Vec::new(); + let mut ordinal_ranges = Vec::new(); + let mut output_base = 0_u64; + + match positions { + SparsePositionSet::Empty => {} + SparsePositionSet::All { len } => { + if *len != domain_len { + return Err(Error::invalid_input_source( + format!( + "Sparse structural {label} all set length {} does not match domain {}", + len, domain_len + ) + .into(), + )); + } + for range in ranges { + let range_len = range.end - range.start; + let output_end = output_base.checked_add(range_len).ok_or_else(|| { + Error::invalid_input_source( + format!("Sparse structural {label} output domain overflows").into(), + ) + })?; + push_coalesced_range(&mut segments, output_base..output_end); + push_coalesced_range(&mut ordinal_ranges, range.clone()); + output_base = output_end; + } + } + SparsePositionSet::Range { start, len } => { + let source_end = start.checked_add(*len).ok_or_else(|| { + Error::invalid_input_source( + format!("Sparse structural {label} range overflows").into(), + ) + })?; + if source_end > domain_len { + return Err(Error::invalid_input_source( + format!( + "Sparse structural {label} range {}..{} is outside domain {}", + start, source_end, domain_len + ) + .into(), + )); + } + for range in ranges { + let intersect_start = range.start.max(*start); + let intersect_end = range.end.min(source_end); + if intersect_start < intersect_end { + let out_start = output_base + .checked_add(intersect_start - range.start) + .ok_or_else(|| { + Error::invalid_input_source( + format!("Sparse structural {label} output position overflows") + .into(), + ) + })?; + let out_end = output_base + .checked_add(intersect_end - range.start) + .ok_or_else(|| { + Error::invalid_input_source( + format!("Sparse structural {label} output position overflows") + .into(), + ) + })?; + push_coalesced_range(&mut segments, out_start..out_end); + push_coalesced_range( + &mut ordinal_ranges, + intersect_start - *start..intersect_end - *start, + ); + } + output_base = output_base + .checked_add(range.end - range.start) + .ok_or_else(|| { + Error::invalid_input_source( + format!("Sparse structural {label} output domain overflows").into(), + ) + })?; + } + } + SparsePositionSet::Explicit(source_positions) => { + let mut out_positions = Vec::new(); + for range in ranges { + let idx_start = + source_positions.partition_point(|position| *position < range.start); + let idx_end = source_positions.partition_point(|position| *position < range.end); + if idx_start < idx_end { + let selected_positions = + source_positions.get(idx_start..idx_end).ok_or_else(|| { + Error::invalid_input_source( + format!( + "Sparse structural {label} explicit position slice is invalid" + ) + .into(), + ) + })?; + for position in selected_positions { + out_positions.push( + output_base + .checked_add(*position - range.start) + .ok_or_else(|| { + Error::invalid_input_source( + format!( + "Sparse structural {label} output position overflows" + ) + .into(), + ) + })?, + ); + } + push_coalesced_range(&mut ordinal_ranges, idx_start as u64..idx_end as u64); + } + output_base = output_base + .checked_add(range.end - range.start) + .ok_or_else(|| { + Error::invalid_input_source( + format!("Sparse structural {label} output domain overflows").into(), + ) + })?; + } + let positions = SparsePositionSet::from_positions(out_positions, output_domain, label)?; + return Ok(SparsePositionSelection { + positions, + ordinal_ranges, + }); + } + } + + Ok(SparsePositionSelection { + positions: position_segments_to_set(segments, output_domain, label)?, + ordinal_ranges, + }) +} + +fn select_validity_set( + validity: &SparseValiditySet, + ranges: &[Range], + domain_len: u64, + label: &str, +) -> Result { + Ok(SparseValiditySet { + meaning: validity.meaning, + positions: select_position_set(&validity.positions, ranges, domain_len, label)?.positions, + }) +} + +fn offsets_from_counts(counts: &[u64]) -> Result> { + let mut offsets = Vec::with_capacity(counts.len() + 1); + let mut offset = 0_u64; + offsets.push(offset); + for count in counts { + offset = offset.checked_add(*count).ok_or_else(|| { + Error::invalid_input_source("Sparse structural list count offsets overflow".into()) + })?; + offsets.push(offset); + } + Ok(offsets) +} + +fn coalesce_ranges(ranges: Vec>) -> Vec> { + let mut coalesced: Vec> = Vec::with_capacity(ranges.len()); + for range in ranges { + if range.is_empty() { + continue; + } + if let Some(last) = coalesced.last_mut() + && last.end == range.start + { + last.end = range.end; + continue; + } + coalesced.push(range); + } + coalesced +} + +fn slice_list_layer( + num_slots: u64, + num_child_slots: u64, + non_empty_positions: &SparsePositionSet, + counts: &SparseCountSet, + validity: &SparseValiditySet, + ranges: &[Range], +) -> Result<(SparseStructuralLayerPlan, Vec>)> { + if non_empty_positions.len() != counts.len() { + return Err(Error::invalid_input_source( + "Sparse structural list has mismatched non-empty positions and counts".into(), + )); + } + let count_sum = counts.sum()?; + if count_sum != num_child_slots { + return Err(Error::invalid_input_source( + format!( + "Sparse structural list count sum {} does not match child slots {}", + count_sum, num_child_slots + ) + .into(), + )); + } + + let non_empty_selection = + select_position_set(non_empty_positions, ranges, num_slots, "list non-empty")?; + let out_counts = select_count_set( + counts, + &non_empty_selection.ordinal_ranges, + non_empty_selection.positions.len(), + )?; + let child_ranges = + child_ranges_from_counts(counts, num_child_slots, &non_empty_selection.ordinal_ranges)?; + let out_validity = select_validity_set(validity, ranges, num_slots, "list validity")?; + let out_num_slots = validate_slice_ranges(ranges, num_slots, "list")?; + let out_num_child_slots = out_counts.sum()?; + Ok(( + SparseStructuralLayerPlan::List { + num_slots: out_num_slots, + num_child_slots: out_num_child_slots, + non_empty_positions: non_empty_selection.positions, + counts: out_counts, + validity: out_validity, + }, + child_ranges, + )) +} + +fn select_count_set( + counts: &SparseCountSet, + ordinal_ranges: &[Range], + selected_len: u64, +) -> Result { + match counts { + SparseCountSet::Empty => { + if selected_len != 0 { + return Err(Error::invalid_input_source( + "Sparse structural selected non-empty positions but counts are empty".into(), + )); + } + Ok(SparseCountSet::Empty) + } + SparseCountSet::Constant { value, len } => { + validate_ordinal_ranges(ordinal_ranges, *len, "constant list counts")?; + Ok(SparseCountSet::constant(*value, selected_len)) + } + SparseCountSet::Explicit { + counts: source_counts, + .. + } => { + validate_ordinal_ranges( + ordinal_ranges, + source_counts.len() as u64, + "explicit list counts", + )?; + let selected_len = usize::try_from(selected_len).map_err(|_| { + Error::invalid_input_source( + "Sparse structural selected count length exceeds usize::MAX".into(), + ) + })?; + let mut out_counts = Vec::with_capacity(selected_len); + for range in ordinal_ranges { + let start = usize::try_from(range.start).map_err(|_| { + Error::invalid_input_source( + "Sparse structural count ordinal exceeds usize::MAX".into(), + ) + })?; + let end = usize::try_from(range.end).map_err(|_| { + Error::invalid_input_source( + "Sparse structural count ordinal exceeds usize::MAX".into(), + ) + })?; + out_counts.extend_from_slice(source_counts.get(start..end).ok_or_else(|| { + Error::invalid_input_source( + "Sparse structural explicit count slice is invalid".into(), + ) + })?); + } + SparseCountSet::from_counts(out_counts) + } + } +} + +fn validate_ordinal_ranges(ranges: &[Range], len: u64, label: &str) -> Result<()> { + for range in ranges { + if range.start > range.end || range.end > len { + return Err(Error::invalid_input_source( + format!( + "Sparse structural {label} ordinal range {}..{} is outside {} values", + range.start, range.end, len + ) + .into(), + )); + } + } + Ok(()) +} + +fn child_ranges_from_counts( + counts: &SparseCountSet, + num_child_slots: u64, + ordinal_ranges: &[Range], +) -> Result>> { + if ordinal_ranges.is_empty() { + return Ok(Vec::new()); + } + let child_ranges = match counts { + SparseCountSet::Empty => { + return Err(Error::invalid_input_source( + "Sparse structural selected non-empty positions but counts are empty".into(), + )); + } + SparseCountSet::Constant { value, len } => { + let expected_child_slots = value.checked_mul(*len).ok_or_else(|| { + Error::invalid_input_source( + "Sparse structural list constant count sum overflows child slots".into(), + ) + })?; + if expected_child_slots != num_child_slots { + return Err(Error::invalid_input_source( + format!( + "Sparse structural list constant count sum {} does not match child slots {}", + expected_child_slots, num_child_slots + ) + .into(), + )); + } + validate_ordinal_ranges(ordinal_ranges, *len, "constant list counts")?; + ordinal_ranges + .iter() + .map(|range| { + let start = range.start.checked_mul(*value).ok_or_else(|| { + Error::invalid_input_source( + "Sparse structural list child range start overflows".into(), + ) + })?; + let end = range.end.checked_mul(*value).ok_or_else(|| { + Error::invalid_input_source( + "Sparse structural list child range end overflows".into(), + ) + })?; + Ok(start..end) + }) + .collect::>>()? + } + SparseCountSet::Explicit { + counts, + offsets: value_offsets, + } => { + let last_offset = *value_offsets.last().ok_or_else(|| { + Error::invalid_input_source( + "Sparse structural list count offsets are unexpectedly empty".into(), + ) + })?; + if last_offset != num_child_slots { + return Err(Error::invalid_input_source( + format!( + "Sparse structural list count sum {} does not match child slots {}", + last_offset, num_child_slots + ) + .into(), + )); + } + validate_ordinal_ranges(ordinal_ranges, counts.len() as u64, "explicit list counts")?; + ordinal_ranges + .iter() + .map(|range| { + let start = usize::try_from(range.start).map_err(|_| { + Error::invalid_input_source( + "Sparse structural count ordinal exceeds usize::MAX".into(), + ) + })?; + let end = usize::try_from(range.end).map_err(|_| { + Error::invalid_input_source( + "Sparse structural count ordinal exceeds usize::MAX".into(), + ) + })?; + let start_offset = *value_offsets.get(start).ok_or_else(|| { + Error::invalid_input_source( + "Sparse structural list start offset is missing".into(), + ) + })?; + let end_offset = *value_offsets.get(end).ok_or_else(|| { + Error::invalid_input_source( + "Sparse structural list end offset is missing".into(), + ) + })?; + Ok(start_offset..end_offset) + }) + .collect::>>()? + } + }; + Ok(coalesce_ranges(child_ranges)) +} + +fn slice_sparse_plan( + plan: &SparseStructuralPlan, + row_ranges: &[Range], + row_domain: u64, +) -> Result { + plan.validate(row_domain)?; + let mut selected_ranges = row_ranges.to_vec(); + let mut selected_domain = row_domain; + let mut sliced_layers = Vec::with_capacity(plan.layers.len()); + + for layer in &plan.layers { + match layer { + SparseStructuralLayerPlan::Validity { + num_slots, + validity, + } => { + if selected_domain != *num_slots { + return Err(Error::invalid_input_source( + format!( + "Sparse structural validity slice domain {} does not match {} slots", + selected_domain, num_slots + ) + .into(), + )); + } + let out_num_slots = + validate_slice_ranges(&selected_ranges, *num_slots, "validity")?; + sliced_layers.push(SparseStructuralLayerPlan::Validity { + num_slots: out_num_slots, + validity: select_validity_set( + validity, + &selected_ranges, + *num_slots, + "validity", + )?, + }); + } + SparseStructuralLayerPlan::List { + num_slots, + num_child_slots, + non_empty_positions, + counts, + validity, + .. + } => { + if selected_domain != *num_slots { + return Err(Error::invalid_input_source( + format!( + "Sparse structural list slice domain {} does not match {} slots", + selected_domain, num_slots + ) + .into(), + )); + } + let (sliced_layer, child_ranges) = slice_list_layer( + *num_slots, + *num_child_slots, + non_empty_positions, + counts, + validity, + &selected_ranges, + )?; + sliced_layers.push(sliced_layer); + selected_ranges = child_ranges; + selected_domain = *num_child_slots; + } + SparseStructuralLayerPlan::FixedSizeList { + num_slots, + dimension, + validity, + } => { + if selected_domain != *num_slots { + return Err(Error::invalid_input_source( + format!( + "Sparse structural fixed-size-list slice domain {} does not match {} slots", + selected_domain, num_slots + ) + .into(), + )); + } + let num_child_slots = num_slots.checked_mul(*dimension).ok_or_else(|| { + Error::invalid_input_source( + format!( + "Sparse structural fixed-size-list child slot count overflows: slots={}, dimension={}", + num_slots, dimension + ) + .into(), + ) + })?; + let out_num_slots = + validate_slice_ranges(&selected_ranges, *num_slots, "fixed-size-list")?; + let child_ranges = selected_ranges + .iter() + .map(|range| { + let start = range.start.checked_mul(*dimension).ok_or_else(|| { + Error::invalid_input_source( + "Sparse structural fixed-size-list child range start overflows" + .into(), + ) + })?; + let end = range.end.checked_mul(*dimension).ok_or_else(|| { + Error::invalid_input_source( + "Sparse structural fixed-size-list child range end overflows" + .into(), + ) + })?; + Ok(start..end) + }) + .collect::>>()?; + sliced_layers.push(SparseStructuralLayerPlan::FixedSizeList { + num_slots: out_num_slots, + dimension: *dimension, + validity: select_validity_set( + validity, + &selected_ranges, + *num_slots, + "fixed-size-list validity", + )?, + }); + selected_ranges = child_ranges; + selected_domain = num_child_slots; + } + } + } + + if selected_domain != plan.num_visible_items { + return Err(Error::invalid_input_source( + format!( + "Sparse structural selected terminal domain {} does not match {} visible items", + selected_domain, plan.num_visible_items + ) + .into(), + )); + } + let num_visible_items = + validate_slice_ranges(&selected_ranges, plan.num_visible_items, "visible value")?; + let num_items = SparseStructuralPlan::expected_num_items(&sliced_layers, num_visible_items)?; + Ok(SparseStructuralSelection { + plan: SparseStructuralPlan { + layers: sliced_layers, + num_items, + num_visible_items, + }, + leaf_ranges: coalesce_ranges(selected_ranges), + }) +} + +#[cfg(test)] +mod tests { + use std::sync::Mutex; + + use crate::{ + compression::DefaultDecompressionStrategy, + encodings::physical::block::{CompressionConfig, CompressionScheme}, + testing::SimulatedScheduler, + }; + + use super::*; + + fn position_set( + positions: pb21::sparse_position_set::Positions, + num_positions: u64, + ) -> Option { + Some(pb21::SparsePositionSet { + positions: Some(positions), + num_positions, + }) + } + + fn position_empty() -> Option { + position_set( + pb21::sparse_position_set::Positions::Empty(pb21::SparsePositionEmpty {}), + 0, + ) + } + + fn position_all(num_positions: u64) -> Option { + position_set( + pb21::sparse_position_set::Positions::All(pb21::SparsePositionAll {}), + num_positions, + ) + } + + fn position_explicit(num_positions: u64) -> Option { + position_set( + pb21::sparse_position_set::Positions::Explicit(ProtobufUtils21::flat(64, None)), + num_positions, + ) + } + + fn general_lz4(values: CompressiveEncoding) -> CompressiveEncoding { + ProtobufUtils21::wrapped(CompressionConfig::new(CompressionScheme::Lz4, None), values) + .unwrap() + } + + fn validity( + meaning: pb21::sparse_validity_set::Meaning, + positions: Option, + ) -> Option { + Some(pb21::SparseValiditySet { + meaning: meaning as i32, + positions, + }) + } + + fn null_positions( + positions: Option, + ) -> Option { + validity( + pb21::sparse_validity_set::Meaning::SparseValidityNullPositions, + positions, + ) + } + + fn count_empty() -> Option { + Some(pb21::SparseCountSet { + counts: Some(pb21::sparse_count_set::Counts::Empty( + pb21::SparseCountEmpty {}, + )), + }) + } + + fn count_constant(value: u64) -> Option { + Some(pb21::SparseCountSet { + counts: Some(pb21::sparse_count_set::Counts::Constant( + pb21::SparseCountConstant { value }, + )), + }) + } + + fn sparse_layout() -> pb21::SparseLayout { + pb21::SparseLayout { + value_compression: Some(ProtobufUtils21::flat(32, None)), + num_buffers: 1, + num_items: 1, + num_visible_items: 1, + has_large_chunk: false, + structural_layers: Vec::new(), + } + } + + fn validity_layer( + num_slots: u64, + validity: Option, + ) -> pb21::SparseStructuralLayer { + pb21::SparseStructuralLayer { + layer: Some(pb21::sparse_structural_layer::Layer::Validity( + pb21::SparseValidityLayer { + num_slots, + validity, + }, + )), + } + } + + fn list_layer( + num_slots: u64, + num_child_slots: u64, + non_empty_positions: Option, + counts: Option, + validity: Option, + ) -> pb21::SparseStructuralLayer { + pb21::SparseStructuralLayer { + layer: Some(pb21::sparse_structural_layer::Layer::List( + pb21::SparseListLayer { + num_slots, + num_child_slots, + non_empty_positions, + counts, + validity, + }, + )), + } + } + + fn fixed_size_list_layer( + num_slots: u64, + dimension: u64, + validity: Option, + ) -> pb21::SparseStructuralLayer { + pb21::SparseStructuralLayer { + layer: Some(pb21::sparse_structural_layer::Layer::FixedSizeList( + pb21::SparseFixedSizeListLayer { + num_slots, + dimension, + validity, + }, + )), + } + } + + fn assert_invalid_input_contains(err: Error, expected: &str) { + let message = err.to_string(); + assert!( + matches!(&err, Error::InvalidInput { .. }), + "expected InvalidInput, got {err:?}" + ); + assert!( + message.contains(expected), + "expected error to contain {expected:?}, got {message}" + ); + } + + #[test] + fn rejects_missing_layer_variants_and_item_count_mismatches() { + let decompressors = DefaultDecompressionStrategy::default(); + + let mut layout = sparse_layout(); + layout + .structural_layers + .push(pb21::SparseStructuralLayer { layer: None }); + let err = SparseStructuralScheduler::try_new( + &[(0, 0), (0, 0)], + 0, + 1, + DataType::Int32, + &layout, + &decompressors, + ) + .unwrap_err(); + assert_invalid_input_contains(err, "missing its layer variant"); + + let mut layout = sparse_layout(); + layout.num_items = 2; + layout + .structural_layers + .push(validity_layer(1, null_positions(position_empty()))); + let err = SparseStructuralScheduler::try_new( + &[(0, 0), (0, 0)], + 0, + 1, + DataType::Int32, + &layout, + &decompressors, + ) + .unwrap_err(); + assert_invalid_input_contains(err, "layers imply 1"); + } + + #[test] + fn accepts_large_structural_descriptors() { + let explicit_values = 8_u64 * 1024 * 1024 + 1; + let metadata_size = explicit_values * 8; + let value_position = metadata_size; + let value_size = explicit_values * 16; + let structural_position = value_position + value_size; + let structural_size = explicit_values * std::mem::size_of::() as u64; + let mut layout = sparse_layout(); + layout.num_items = explicit_values; + layout.num_visible_items = explicit_values; + layout.structural_layers.push(validity_layer( + explicit_values, + null_positions(position_explicit(explicit_values)), + )); + + SparseStructuralScheduler::try_new( + &[ + (0, metadata_size), + (value_position, value_size), + (structural_position, structural_size), + ], + 0, + explicit_values, + DataType::Int32, + &layout, + &DefaultDecompressionStrategy::default(), + ) + .unwrap(); + } + + #[test] + fn accepts_deep_supported_value_encodings() { + let encoding = (0..300).fold(ProtobufUtils21::flat(32, None), |values, _| { + ProtobufUtils21::fsl(1, false, values) + }); + + assert_eq!( + SparseStructuralScheduler::validate_value_encoding(&encoding).unwrap(), + 1 + ); + } + + fn null_set(positions: SparsePositionSet) -> SparseValiditySet { + SparseValiditySet { + meaning: SparseValidityMeaning::NullPositions, + positions, + } + } + + fn valid_set(positions: SparsePositionSet) -> SparseValiditySet { + SparseValiditySet { + meaning: SparseValidityMeaning::ValidPositions, + positions, + } + } + + #[test] + fn semantic_position_and_count_sets_project_without_materializing_ranges() { + let ranges = [1..4]; + assert_eq!( + select_position_set(&SparsePositionSet::Empty, &ranges, 5, "empty") + .unwrap() + .positions, + SparsePositionSet::Empty + ); + assert_eq!( + select_position_set(&SparsePositionSet::all(5), &ranges, 5, "all") + .unwrap() + .positions, + SparsePositionSet::all(3) + ); + assert_eq!( + select_position_set(&SparsePositionSet::range(1, 3), &ranges, 5, "range") + .unwrap() + .positions, + SparsePositionSet::all(3) + ); + assert_eq!( + select_position_set( + &SparsePositionSet::Explicit(vec![0, 2, 4]), + &[0..1, 4..5], + 5, + "explicit", + ) + .unwrap() + .positions, + SparsePositionSet::all(2) + ); + + assert_eq!( + select_count_set(&SparseCountSet::Empty, &[], 0).unwrap(), + SparseCountSet::Empty + ); + assert_eq!( + select_count_set(&SparseCountSet::constant(2, 3), &[0..1, 2..3], 2,).unwrap(), + SparseCountSet::constant(2, 2) + ); + assert_eq!( + select_count_set( + &SparseCountSet::from_counts(vec![1, 2, 3]).unwrap(), + &[0..1, 2..3], + 2, + ) + .unwrap(), + SparseCountSet::from_counts(vec![1, 3]).unwrap() + ); + } + + #[test] + fn validity_polarities_rebuild_the_same_arrow_domain() { + let mut null_builder = BooleanBufferBuilder::new(4); + null_set(SparsePositionSet::Explicit(vec![1, 3])) + .append_to(&mut null_builder, 4) + .unwrap(); + assert_eq!( + null_builder.finish().iter().collect::>(), + vec![true, false, true, false] + ); + + let mut valid_builder = BooleanBufferBuilder::new(4); + valid_set(SparsePositionSet::range(1, 2)) + .append_to(&mut valid_builder, 4) + .unwrap(); + assert_eq!( + valid_builder.finish().iter().collect::>(), + vec![false, true, true, false] + ); + } + + #[test] + fn schema_layer_mismatches_are_invalid_input() { + let mut missing = SparseStructuralUnraveler::new(SparseStructuralPlan { + layers: Vec::new(), + num_items: 1, + num_visible_items: 1, + }); + let mut validity = BooleanBufferBuilder::new(1); + let err = missing.unravel_validity(&mut validity).unwrap_err(); + assert_invalid_input_contains(err, "fewer layers than the Arrow schema"); + + let mut fixed_size_list = SparseStructuralUnraveler::new(SparseStructuralPlan { + layers: vec![SparseStructuralLayerPlan::FixedSizeList { + num_slots: 2, + dimension: 2, + validity: null_set(SparsePositionSet::empty()), + }], + num_items: 4, + num_visible_items: 4, + }); + let mut validity = BooleanBufferBuilder::new(2); + let err = fixed_size_list.unravel_validity(&mut validity).unwrap_err(); + assert_invalid_input_contains(err, "does not match the Arrow schema"); + + let extra = SparseStructuralUnraveler::new(SparseStructuralPlan { + layers: vec![SparseStructuralLayerPlan::Validity { + num_slots: 1, + validity: null_set(SparsePositionSet::empty()), + }], + num_items: 1, + num_visible_items: 1, + }); + let err = extra.ensure_exhausted().unwrap_err(); + assert_invalid_input_contains(err, "1 unconsumed layer"); + } + + fn nested_plan() -> SparseStructuralPlan { + SparseStructuralPlan { + layers: vec![ + SparseStructuralLayerPlan::Validity { + num_slots: 6, + validity: null_set(SparsePositionSet::Explicit(vec![1, 4])), + }, + SparseStructuralLayerPlan::List { + num_slots: 6, + num_child_slots: 5, + non_empty_positions: SparsePositionSet::Explicit(vec![0, 2, 5]), + counts: SparseCountSet::from_counts(vec![2, 1, 2]).unwrap(), + validity: null_set(SparsePositionSet::Explicit(vec![1, 4])), + }, + SparseStructuralLayerPlan::FixedSizeList { + num_slots: 5, + dimension: 2, + validity: valid_set(SparsePositionSet::range(1, 3)), + }, + ], + num_items: 13, + num_visible_items: 10, + } + } + + #[test] + fn discontiguous_projection_preserves_outer_to_inner_order() { + let selection = slice_sparse_plan(&nested_plan(), &[5..6, 0..1], 6).unwrap(); + assert_eq!(selection.leaf_ranges, vec![6..10, 0..4]); + assert_eq!(selection.plan.num_visible_items, 8); + selection.plan.validate(2).unwrap(); + + let SparseStructuralLayerPlan::List { + non_empty_positions, + counts, + .. + } = &selection.plan.layers[1] + else { + panic!("expected projected list layer"); + }; + assert_eq!(*non_empty_positions, SparsePositionSet::all(2)); + assert_eq!(*counts, SparseCountSet::constant(2, 2)); + } + + #[test] + fn no_value_projection_keeps_empty_and_null_list_structure() { + let selection = slice_sparse_plan(&nested_plan(), &[1..2, 3..4], 6).unwrap(); + assert!(selection.leaf_ranges.is_empty()); + assert_eq!(selection.plan.num_visible_items, 0); + selection.plan.validate(2).unwrap(); + + let SparseStructuralLayerPlan::List { + num_slots, + num_child_slots, + non_empty_positions, + counts, + validity, + } = &selection.plan.layers[1] + else { + panic!("expected projected list layer"); + }; + assert_eq!((*num_slots, *num_child_slots), (2, 0)); + assert_eq!(*non_empty_positions, SparsePositionSet::Empty); + assert_eq!(*counts, SparseCountSet::Empty); + assert_eq!(*validity, null_set(SparsePositionSet::range(0, 1))); + } + + #[test] + fn rejects_missing_value_compression_and_buffer_count_mismatch() { + let decompressors = DefaultDecompressionStrategy::default(); + let mut layout = sparse_layout(); + layout.value_compression = None; + let err = SparseStructuralScheduler::try_new( + &[(0, 0), (0, 0)], + 0, + 1, + DataType::Int32, + &layout, + &decompressors, + ) + .unwrap_err(); + assert_invalid_input_contains(err, "missing value compression"); + + let mut layout = sparse_layout(); + layout.num_buffers = 2; + let err = SparseStructuralScheduler::try_new( + &[(0, 0), (0, 0)], + 0, + 1, + DataType::Int32, + &layout, + &decompressors, + ) + .unwrap_err(); + assert_invalid_input_contains(err, "declares 2 value buffers"); + } + + #[test] + fn rejects_inconsistent_chunk_count_before_io() { + let decompressors = DefaultDecompressionStrategy::default(); + + let layout = sparse_layout(); + let err = SparseStructuralScheduler::try_new( + &[(0, 16), (16, 8)], + 0, + 1, + DataType::Int32, + &layout, + &decompressors, + ) + .unwrap_err(); + assert_invalid_input_contains(err, "declares 2 chunks for 1 visible items"); + } + + #[cfg(feature = "lz4")] + #[test] + fn accepts_large_general_decompression_headers() { + let encoding = general_lz4(ProtobufUtils21::flat(64, None)); + let Some(pb21::compressive_encoding::Compression::General(general)) = + encoding.compression.as_ref() + else { + panic!("expected General compression"); + }; + let declared_size = 65_u32 * 1024 * 1024; + + SparseStructuralScheduler::validate_general_buffer_header( + general, + &declared_size.to_le_bytes(), + "test", + ) + .unwrap(); + } + + #[cfg(feature = "lz4")] + #[tokio::test] + async fn rejects_malformed_general_structural_buffers_as_invalid_input() { + let mut layout = sparse_layout(); + layout.structural_layers.push(validity_layer( + 1, + null_positions(position_set( + pb21::sparse_position_set::Positions::Explicit(general_lz4(ProtobufUtils21::flat( + 64, None, + ))), + 1, + )), + )); + let decompressors = DefaultDecompressionStrategy::default(); + + let mut scheduler = SparseStructuralScheduler::try_new( + &[(0, 8), (8, 8), (16, 8)], + 0, + 1, + DataType::Int32, + &layout, + &decompressors, + ) + .unwrap(); + let mut data = Vec::new(); + data.extend_from_slice(&0_u32.to_le_bytes()); + data.extend_from_slice(&1_u32.to_le_bytes()); + data.extend_from_slice(&[0; 8]); + data.extend_from_slice(&8_u32.to_le_bytes()); + data.extend_from_slice(&[0xff; 4]); + let io: Arc = Arc::new(SimulatedScheduler::new(Bytes::from(data))); + let Err(err) = scheduler.initialize(&io).await else { + panic!("expected malformed General buffer to be rejected"); + }; + assert_invalid_input_contains(err, "decompression failed"); + } + + #[cfg(feature = "lz4")] + #[tokio::test] + async fn rejects_malformed_general_value_buffer() { + let mut layout = sparse_layout(); + layout.value_compression = Some(general_lz4(ProtobufUtils21::flat(32, None))); + let decompressors = DefaultDecompressionStrategy::default(); + let mut scheduler = SparseStructuralScheduler::try_new( + &[(0, 8), (8, 8)], + 0, + 1, + DataType::Int32, + &layout, + &decompressors, + ) + .unwrap(); + + let mut data = Vec::new(); + data.extend_from_slice(&0_u32.to_le_bytes()); + data.extend_from_slice(&1_u32.to_le_bytes()); + data.extend_from_slice(&0_u16.to_le_bytes()); + data.extend_from_slice(&0_u16.to_le_bytes()); + data.extend_from_slice(&[0; 4]); + let io: Arc = Arc::new(SimulatedScheduler::new(Bytes::from(data))); + scheduler.initialize(&io).await.unwrap(); + let mut page_tasks = scheduler.schedule_ranges(&[0..1], &io).unwrap(); + let mut decoder = page_tasks.pop().unwrap().decoder_fut.await.unwrap(); + let Err(err) = decoder.drain(1).unwrap().decode() else { + panic!("expected malformed General value buffer to be rejected"); + }; + assert_invalid_input_contains(err, "missing its length prefix"); + } + + #[test] + fn rejects_layer_domain_and_fixed_size_list_mismatches() { + let decompressors = DefaultDecompressionStrategy::default(); + let mut layout = sparse_layout(); + layout.num_items = 2; + layout.num_visible_items = 2; + layout + .structural_layers + .push(validity_layer(1, null_positions(position_empty()))); + layout + .structural_layers + .push(validity_layer(2, null_positions(position_empty()))); + let err = SparseStructuralScheduler::try_new( + &[(0, 0), (0, 0)], + 0, + 1, + DataType::Int32, + &layout, + &decompressors, + ) + .unwrap_err(); + assert_invalid_input_contains(err, "layer 1 has 2 slots, expected 1"); + + let mut layout = sparse_layout(); + layout.num_items = 4; + layout.num_visible_items = 4; + layout.structural_layers.push(fixed_size_list_layer( + 2, + 3, + null_positions(position_empty()), + )); + let err = SparseStructuralScheduler::try_new( + &[(0, 0), (0, 0)], + 0, + 2, + DataType::Int32, + &layout, + &decompressors, + ) + .unwrap_err(); + assert_invalid_input_contains(err, "terminal domain has 6 slots"); + } + + #[test] + fn rejects_invalid_validity_and_list_count_semantics() { + let decompressors = DefaultDecompressionStrategy::default(); + let mut layout = sparse_layout(); + layout.structural_layers.push(validity_layer( + 1, + validity( + pb21::sparse_validity_set::Meaning::SparseValidityUnspecified, + position_empty(), + ), + )); + let err = SparseStructuralScheduler::try_new( + &[(0, 0), (0, 0)], + 0, + 1, + DataType::Int32, + &layout, + &decompressors, + ) + .unwrap_err(); + assert_invalid_input_contains(err, "meaning is unspecified"); + + let mut layout = sparse_layout(); + layout.num_items = 3; + layout.num_visible_items = 3; + layout.structural_layers.push(list_layer( + 2, + 3, + position_all(2), + count_constant(2), + null_positions(position_empty()), + )); + let err = SparseStructuralScheduler::try_new( + &[(0, 0), (0, 0)], + 0, + 2, + DataType::Int32, + &layout, + &decompressors, + ) + .unwrap_err(); + assert_invalid_input_contains(err, "count sum 4 does not match child slots 3"); + } + + #[tokio::test] + async fn rejects_unordered_explicit_positions_after_decompression() { + let mut layout = sparse_layout(); + layout.num_items = 4; + layout.num_visible_items = 4; + layout + .structural_layers + .push(validity_layer(4, null_positions(position_explicit(2)))); + let decompressors = DefaultDecompressionStrategy::default(); + let mut scheduler = SparseStructuralScheduler::try_new( + &[(0, 8), (8, 8), (16, 16)], + 0, + 4, + DataType::Int32, + &layout, + &decompressors, + ) + .unwrap(); + + let mut data = Vec::new(); + data.extend_from_slice(&0_u32.to_le_bytes()); + data.extend_from_slice(&4_u32.to_le_bytes()); + data.extend_from_slice(&[0; 8]); + data.extend_from_slice(&3_u64.to_le_bytes()); + data.extend_from_slice(&0_u64.to_le_bytes()); + let io: Arc = Arc::new(SimulatedScheduler::new(Bytes::from(data))); + + let Err(err) = scheduler.initialize(&io).await else { + panic!("expected unordered sparse positions to be rejected"); + }; + assert_invalid_input_contains(err, "positions must be strictly increasing"); + } + + #[tokio::test] + async fn rejects_chunk_metadata_value_and_byte_sum_mismatches() { + let layout = sparse_layout(); + let decompressors = DefaultDecompressionStrategy::default(); + let mut scheduler = SparseStructuralScheduler::try_new( + &[(0, 8), (8, 8)], + 0, + 1, + DataType::Int32, + &layout, + &decompressors, + ) + .unwrap(); + let mut data = Vec::new(); + data.extend_from_slice(&1_u32.to_le_bytes()); + data.extend_from_slice(&1_u32.to_le_bytes()); + data.extend_from_slice(&[0; 8]); + let io: Arc = Arc::new(SimulatedScheduler::new(Bytes::from(data))); + let Err(err) = scheduler.initialize(&io).await else { + panic!("expected chunk byte sum mismatch"); + }; + assert_invalid_input_contains(err, "describes 16 value bytes"); + + let mut layout = sparse_layout(); + layout.num_items = 2; + layout.num_visible_items = 2; + let mut scheduler = SparseStructuralScheduler::try_new( + &[(0, 8), (8, 8)], + 0, + 2, + DataType::Int32, + &layout, + &decompressors, + ) + .unwrap(); + let mut data = Vec::new(); + data.extend_from_slice(&0_u32.to_le_bytes()); + data.extend_from_slice(&1_u32.to_le_bytes()); + data.extend_from_slice(&[0; 8]); + let io: Arc = Arc::new(SimulatedScheduler::new(Bytes::from(data))); + let Err(err) = scheduler.initialize(&io).await else { + panic!("expected chunk value sum mismatch"); + }; + assert_invalid_input_contains(err, "metadata has 1, layout has 2"); + } + + #[tokio::test] + async fn rejects_malformed_value_chunk_before_decompression() { + let layout = sparse_layout(); + let decompressors = DefaultDecompressionStrategy::default(); + let mut scheduler = SparseStructuralScheduler::try_new( + &[(0, 8), (8, 8)], + 0, + 1, + DataType::Int32, + &layout, + &decompressors, + ) + .unwrap(); + let mut data = Vec::new(); + data.extend_from_slice(&0_u32.to_le_bytes()); + data.extend_from_slice(&1_u32.to_le_bytes()); + data.extend_from_slice(&[0; 8]); + let io: Arc = Arc::new(SimulatedScheduler::new(Bytes::from(data))); + scheduler.initialize(&io).await.unwrap(); + let mut page_tasks = scheduler.schedule_ranges(&[0..1], &io).unwrap(); + let page_task = page_tasks.pop().unwrap(); + let mut decoder = page_task.decoder_fut.await.unwrap(); + let decode_task = decoder.drain(1).unwrap(); + let Err(err) = decode_task.decode() else { + panic!("expected malformed value chunk to be rejected"); + }; + assert_invalid_input_contains(err, "flat values buffer has 0 bytes, expected 4"); + } + + #[tokio::test] + async fn accepts_value_chunks_larger_than_64_mib() { + let chunk_size = 65_u64 * 1024 * 1024; + let layout = sparse_layout(); + let mut scheduler = SparseStructuralScheduler::try_new( + &[(0, 8), (8, chunk_size)], + 0, + 1, + DataType::Int32, + &layout, + &DefaultDecompressionStrategy::default(), + ) + .unwrap(); + let words_minus_one = u32::try_from(chunk_size / MINIBLOCK_ALIGNMENT as u64 - 1).unwrap(); + let mut metadata = Vec::new(); + metadata.extend_from_slice(&words_minus_one.to_le_bytes()); + metadata.extend_from_slice(&1_u32.to_le_bytes()); + let io: Arc = Arc::new(SimulatedScheduler::new(Bytes::from(metadata))); + + scheduler.initialize(&io).await.unwrap(); + } + + #[tokio::test] + async fn rejects_value_chunks_above_the_miniblock_limit() { + let num_values = miniblock::MAX_CONFIGURABLE_MINIBLOCK_VALUES + 1; + let mut layout = sparse_layout(); + layout.num_items = num_values; + layout.num_visible_items = num_values; + let decompressors = DefaultDecompressionStrategy::default(); + let mut scheduler = SparseStructuralScheduler::try_new( + &[(0, 16), (16, 16)], + 0, + num_values, + DataType::Int32, + &layout, + &decompressors, + ) + .unwrap(); + let mut data = Vec::new(); + data.extend_from_slice(&0_u32.to_le_bytes()); + data.extend_from_slice(&(num_values as u32).to_le_bytes()); + data.extend_from_slice(&0_u32.to_le_bytes()); + data.extend_from_slice(&0_u32.to_le_bytes()); + data.extend_from_slice(&[0; 16]); + let io: Arc = Arc::new(SimulatedScheduler::new(Bytes::from(data))); + + let Err(err) = scheduler.initialize(&io).await else { + panic!("expected oversized value chunk to be rejected"); + }; + assert_invalid_input_contains(err, "exceeding the mini-block limit"); + } + + #[derive(Debug, Clone)] + struct RecordingIo { + data: Bytes, + calls: Arc>>>>, + } + + impl RecordingIo { + fn new(data: Bytes) -> Self { + Self { + data, + calls: Arc::new(Mutex::new(Vec::new())), + } + } + } + + impl EncodingsIo for RecordingIo { + fn submit_request( + &self, + ranges: Vec>, + _priority: u64, + ) -> BoxFuture<'static, Result>> { + self.calls.lock().unwrap().push(ranges.clone()); + let data = self.data.clone(); + async move { + ranges + .into_iter() + .map(|range| { + let start = usize_from_u64(range.start, "test range start")?; + let end = usize_from_u64(range.end, "test range end")?; + if start > end || end > data.len() { + return Err(Error::invalid_input_source( + "Test I/O range is outside fixture data".into(), + )); + } + Ok(data.slice(start..end)) + }) + .collect() + } + .boxed() + } + } + + #[tokio::test] + async fn selective_read_requests_only_intersecting_value_chunk() { + let mut layout = sparse_layout(); + layout.num_items = 4; + layout.num_visible_items = 4; + let decompressors = DefaultDecompressionStrategy::default(); + let mut scheduler = SparseStructuralScheduler::try_new( + &[(0, 16), (16, 32)], + 0, + 4, + DataType::Int32, + &layout, + &decompressors, + ) + .unwrap(); + + let mut data = Vec::new(); + for _ in 0..2 { + data.extend_from_slice(&1_u32.to_le_bytes()); + data.extend_from_slice(&2_u32.to_le_bytes()); + } + for values in [[10_i32, 20], [30, 40]] { + data.extend_from_slice(&0_u16.to_le_bytes()); + data.extend_from_slice(&8_u16.to_le_bytes()); + data.extend_from_slice(&[0; 4]); + for value in values { + data.extend_from_slice(&value.to_le_bytes()); + } + } + + let io = Arc::new(RecordingIo::new(Bytes::from(data))); + let trait_io: Arc = io.clone(); + scheduler.initialize(&trait_io).await.unwrap(); + let mut page_tasks = scheduler.schedule_ranges(&[2..3], &trait_io).unwrap(); + let page_task = page_tasks.pop().unwrap(); + let mut decoder = page_task.decoder_fut.await.unwrap(); + let decoded = decoder.drain(1).unwrap().decode().unwrap(); + assert_eq!(decoded.data.num_values(), 1); + + let calls = io.calls.lock().unwrap(); + assert_eq!(calls.as_slice(), &[vec![0..16], vec![32..48]]); + } + + #[tokio::test] + async fn empty_leaf_selection_rebuilds_offsets_without_value_io() { + let mut layout = sparse_layout(); + layout.num_visible_items = 0; + layout.structural_layers.push(list_layer( + 1, + 0, + position_empty(), + count_empty(), + null_positions(position_empty()), + )); + let decompressors = DefaultDecompressionStrategy::default(); + let mut scheduler = SparseStructuralScheduler::try_new( + &[(0, 0), (0, 0)], + 0, + 1, + DataType::Int32, + &layout, + &decompressors, + ) + .unwrap(); + let io = Arc::new(RecordingIo::new(Bytes::new())); + let trait_io: Arc = io.clone(); + scheduler.initialize(&trait_io).await.unwrap(); + let mut page_tasks = scheduler.schedule_ranges(&[0..1], &trait_io).unwrap(); + let page_task = page_tasks.pop().unwrap(); + let mut decoder = page_task.decoder_fut.await.unwrap(); + let decoded = decoder.drain(1).unwrap().decode().unwrap(); + assert_eq!(decoded.data.num_values(), 0); + + let mut repdef = CompositeRepDefUnraveler::new(vec![decoded.repdef]); + let (offsets, validity) = repdef.unravel_offsets::().unwrap(); + assert_eq!(offsets.as_ref(), &[0, 0]); + assert!(validity.is_none()); + + let calls = io.calls.lock().unwrap(); + assert_eq!(calls.len(), 2); + assert!(calls[1].is_empty(), "value payload must not be requested"); + } +} diff --git a/rust/lance-encoding/src/encodings/logical/primitive/sparse/writer.rs b/rust/lance-encoding/src/encodings/logical/primitive/sparse/writer.rs new file mode 100644 index 00000000000..ad910c6d8f9 --- /dev/null +++ b/rust/lance-encoding/src/encodings/logical/primitive/sparse/writer.rs @@ -0,0 +1,2408 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright The Lance Authors + +//! Sparse structural planning and serialization. + +use std::iter; + +use arrow_buffer::BooleanBuffer; +use lance_core::{Error, Result, datatypes::Field, utils::bit::pad_bytes}; + +use crate::{ + buffer::LanceBuffer, + compression::CompressionStrategy, + data::{BlockInfo, DataBlock, FixedWidthDataBlock}, + decoder::PageEncoding, + encoder::EncodedPage, + format::pb21::{self, CompressiveEncoding}, + repdef::{NormalizedStructuralLayer, NormalizedStructuralPlan}, + statistics::ComputeStat, +}; + +use super::super::MiniblockChunkSize; + +use super::{ + SparseCountSet, SparsePositionSet, SparseStructuralLayerPlan, SparseStructuralPlan, + SparseValidityMeaning, SparseValiditySet, +}; +use crate::encodings::logical::primitive::{ + FILL_BYTE, MINIBLOCK_ALIGNMENT, + miniblock::{MiniBlockCompressed, MiniBlockCompressionContext}, +}; + +#[derive(Clone, Copy, Default)] +struct PositionSetStats { + count: u64, + first: u64, + last: u64, + is_contiguous: bool, +} + +impl PositionSetStats { + fn observe(&mut self, position: u64) { + if self.count == 0 { + self.first = position; + self.is_contiguous = true; + } else if self.last.checked_add(1) != Some(position) { + self.is_contiguous = false; + } + self.last = position; + self.count += 1; + } + + fn encoded_cost(&self, domain_len: u64) -> u64 { + if self.count == 0 || self.count == domain_len || self.is_contiguous { + 0 + } else { + self.count + } + } + + fn to_set( + self, + validity: &BooleanBuffer, + want_valid: bool, + domain_len: u64, + label: &str, + ) -> Result { + if self.count == 0 { + return Ok(SparsePositionSet::empty()); + } + if self.count == domain_len { + return Ok(SparsePositionSet::all(domain_len)); + } + if self.is_contiguous { + return Ok(SparsePositionSet::range(self.first, self.count)); + } + + let capacity = usize::try_from(self.count).map_err(|_| { + Error::invalid_input_source( + format!("Sparse structural {label} positions exceed usize::MAX").into(), + ) + })?; + let mut positions = Vec::with_capacity(capacity); + for (index, is_valid) in validity.iter().enumerate() { + if is_valid == want_valid { + positions.push(u64::try_from(index).map_err(|_| { + Error::invalid_input_source( + format!("Sparse structural {label} position exceeds u64::MAX").into(), + ) + })?); + } + } + SparsePositionSet::from_positions(positions, domain_len, label) + } +} + +fn usize_to_u64(value: usize, label: &str) -> Result { + u64::try_from(value).map_err(|_| { + Error::invalid_input_source( + format!("Sparse structural {label} {value} exceeds u64::MAX").into(), + ) + }) +} + +fn validity_set( + validity: Option<&BooleanBuffer>, + num_slots: usize, + label: &str, +) -> Result { + let Some(validity) = validity else { + return Ok(SparseValiditySet { + meaning: SparseValidityMeaning::NullPositions, + positions: SparsePositionSet::empty(), + }); + }; + if validity.len() != num_slots { + return Err(Error::invalid_input_source( + format!( + "Sparse structural {label} validity length {} does not match {} slots", + validity.len(), + num_slots + ) + .into(), + )); + } + + let domain_len = usize_to_u64(num_slots, "validity domain")?; + let mut valid_stats = PositionSetStats::default(); + let mut null_stats = PositionSetStats::default(); + for (index, is_valid) in validity.iter().enumerate() { + let index = usize_to_u64(index, "validity position")?; + if is_valid { + valid_stats.observe(index); + } else { + null_stats.observe(index); + } + } + + if null_stats.count == 0 { + return Ok(SparseValiditySet { + meaning: SparseValidityMeaning::NullPositions, + positions: SparsePositionSet::empty(), + }); + } + if valid_stats.count == 0 { + return Ok(SparseValiditySet { + meaning: SparseValidityMeaning::ValidPositions, + positions: SparsePositionSet::empty(), + }); + } + + let valid_cost = valid_stats.encoded_cost(domain_len); + let null_cost = null_stats.encoded_cost(domain_len); + if valid_cost < null_cost { + Ok(SparseValiditySet { + meaning: SparseValidityMeaning::ValidPositions, + positions: valid_stats.to_set(validity, true, domain_len, label)?, + }) + } else { + Ok(SparseValiditySet { + meaning: SparseValidityMeaning::NullPositions, + positions: null_stats.to_set(validity, false, domain_len, label)?, + }) + } +} + +/// Builds the semantic sparse plan directly from the once-normalized Arrow layers. +pub(in crate::encodings::logical::primitive) fn plan( + normalized: &NormalizedStructuralPlan, + num_visible_items: u64, +) -> Result { + let mut layers = Vec::with_capacity(normalized.layers().len()); + let mut num_items = num_visible_items; + + for layer in normalized.layers() { + match layer { + NormalizedStructuralLayer::Validity { + validity, + num_slots, + } => { + layers.push(SparseStructuralLayerPlan::Validity { + num_slots: usize_to_u64(num_slots, "validity slot count")?, + validity: validity_set(validity, num_slots, "validity")?, + }); + } + NormalizedStructuralLayer::FixedSizeList { + validity, + dimension, + num_slots, + } => { + if dimension == 0 { + return Err(Error::invalid_input_source( + "Sparse structural fixed-size-list dimension is zero".into(), + )); + } + layers.push(SparseStructuralLayerPlan::FixedSizeList { + num_slots: usize_to_u64(num_slots, "fixed-size-list slot count")?, + dimension: usize_to_u64(dimension, "fixed-size-list dimension")?, + validity: validity_set(validity, num_slots, "fixed-size-list validity")?, + }); + } + NormalizedStructuralLayer::List { + offsets, + validity, + num_slots, + } => { + let expected_offsets = num_slots.checked_add(1).ok_or_else(|| { + Error::invalid_input_source( + "Sparse structural list offset count overflows".into(), + ) + })?; + if offsets.len() != expected_offsets { + return Err(Error::invalid_input_source( + format!( + "Sparse structural list has {} offsets for {} slots", + offsets.len(), + num_slots + ) + .into(), + )); + } + if offsets.first().copied() != Some(0) { + return Err(Error::invalid_input_source( + "Sparse structural list offsets must start at zero".into(), + )); + } + + let mut non_empty_positions = Vec::new(); + let mut counts = Vec::new(); + for slot in 0..num_slots { + let start = offsets[slot]; + let end = offsets[slot + 1]; + let count = end.checked_sub(start).ok_or_else(|| { + Error::invalid_input_source( + format!( + "Sparse structural list offsets decrease at slot {slot}: {start}..{end}" + ) + .into(), + ) + })?; + let is_valid = validity.is_none_or(|validity| validity.value(slot)); + if !is_valid && count != 0 { + return Err(Error::invalid_input_source( + format!( + "Sparse structural null list slot {slot} has {count} child slots" + ) + .into(), + )); + } + if is_valid && count > 0 { + non_empty_positions.push(usize_to_u64(slot, "list position")?); + counts.push(u64::try_from(count).map_err(|_| { + Error::invalid_input_source( + format!("Sparse structural list count {count} exceeds u64::MAX") + .into(), + ) + })?); + } + } + + let num_slots_u64 = usize_to_u64(num_slots, "list slot count")?; + let num_non_empty = usize_to_u64(non_empty_positions.len(), "list position count")?; + num_items = num_items + .checked_add(num_slots_u64 - num_non_empty) + .ok_or_else(|| { + Error::invalid_input_source( + "Sparse structural item count overflows u64".into(), + ) + })?; + let num_child_slots = offsets.last().copied().ok_or_else(|| { + Error::invalid_input_source("Sparse structural list has no offsets".into()) + })?; + let num_child_slots = u64::try_from(num_child_slots).map_err(|_| { + Error::invalid_input_source( + format!( + "Sparse structural list child slot count {num_child_slots} is negative" + ) + .into(), + ) + })?; + layers.push(SparseStructuralLayerPlan::List { + num_slots: num_slots_u64, + num_child_slots, + non_empty_positions: SparsePositionSet::from_positions( + non_empty_positions, + num_slots_u64, + "list non-empty", + )?, + counts: SparseCountSet::from_counts(counts)?, + validity: validity_set(validity, num_slots, "list validity")?, + }); + } + } + } + + let row_domain = match layers.first() { + Some(SparseStructuralLayerPlan::Validity { num_slots, .. }) + | Some(SparseStructuralLayerPlan::List { num_slots, .. }) + | Some(SparseStructuralLayerPlan::FixedSizeList { num_slots, .. }) => *num_slots, + None => { + return Err(Error::invalid_input_source( + "Sparse structural encoding requires at least one Arrow structural layer".into(), + )); + } + }; + let plan = SparseStructuralPlan { + layers, + num_items, + num_visible_items, + }; + plan.validate(row_domain)?; + Ok(plan) +} + +/// ConstantLayout remains the canonical representation when no value payload exists. +pub(in crate::encodings::logical::primitive) fn uses_constant_layout( + plan: &SparseStructuralPlan, + field: &Field, +) -> bool { + if plan.num_visible_items == 0 { + return true; + } + if matches!(field.data_type(), arrow_schema::DataType::Struct(fields) if fields.is_empty()) { + return true; + } + + let Some(layer) = plan.layers.last() else { + return false; + }; + let (num_slots, validity) = match layer { + SparseStructuralLayerPlan::Validity { + num_slots, + validity, + } + | SparseStructuralLayerPlan::List { + num_slots, + validity, + .. + } + | SparseStructuralLayerPlan::FixedSizeList { + num_slots, + validity, + .. + } => (*num_slots, validity), + }; + match validity.meaning { + SparseValidityMeaning::NullPositions => validity.positions.len() == num_slots, + SparseValidityMeaning::ValidPositions => validity.positions.is_empty(), + } +} + +fn supports_fixed_size_list_values(data: &DataBlock) -> bool { + match data { + DataBlock::FixedWidth(_) => true, + DataBlock::FixedSizeList(list) => supports_fixed_size_list_values(list.child.as_ref()), + DataBlock::Nullable(nullable) => supports_fixed_size_list_values(nullable.data.as_ref()), + _ => false, + } +} + +/// Whether the sparse writer can encode this value block without changing its value path. +pub fn supports_value_block(data: &DataBlock) -> bool { + match data { + DataBlock::FixedWidth(_) | DataBlock::VariableWidth(_) => true, + DataBlock::Struct(data) => !data.has_variable_width_child(), + DataBlock::FixedSizeList(data) => supports_fixed_size_list_values(data.child.as_ref()), + DataBlock::Empty() + | DataBlock::Constant(_) + | DataBlock::AllNull(_) + | DataBlock::Nullable(_) + | DataBlock::Opaque(_) + | DataBlock::Dictionary(_) => false, + } +} + +struct SparseMiniBlockChunk { + buffer_sizes: Vec, + num_values: u32, +} + +struct SparseMiniBlockCompressed { + data: Vec, + chunks: Vec, +} + +struct SerializedValuePage { + num_buffers: u64, + data: LanceBuffer, + metadata: LanceBuffer, +} + +pub struct PreparedSparseValues { + num_values: u64, + value_compression: CompressiveEncoding, + values: SerializedValuePage, +} + +pub enum SparseValueInput { + Unprepared(DataBlock), + Prepared(PreparedSparseValues), +} + +struct EncodedStructuralPlan { + layers: Vec, + buffers: Vec, +} + +fn with_explicit_value_counts( + compressed: MiniBlockCompressed, +) -> Result { + let mut values_in_previous_chunks = 0_u64; + let mut chunks = Vec::with_capacity(compressed.chunks.len()); + for chunk in compressed.chunks { + let num_values = chunk.num_values(values_in_previous_chunks, compressed.num_values); + values_in_previous_chunks = values_in_previous_chunks + .checked_add(num_values) + .ok_or_else(|| Error::internal("Sparse value count overflows u64".to_string()))?; + chunks.push(SparseMiniBlockChunk { + buffer_sizes: chunk.buffer_sizes, + num_values: u32::try_from(num_values).map_err(|_| { + Error::invalid_input_source( + format!( + "Sparse value chunk has {num_values} visible values, which exceeds the u32 metadata limit" + ) + .into(), + ) + })?, + }); + } + if values_in_previous_chunks != compressed.num_values { + return Err(Error::internal(format!( + "Sparse value chunks describe {values_in_previous_chunks} values, expected {}", + compressed.num_values + ))); + } + Ok(SparseMiniBlockCompressed { + data: compressed.data, + chunks, + }) +} + +fn serialize_value_chunks( + compressed: SparseMiniBlockCompressed, + miniblock_chunk_size: MiniblockChunkSize, +) -> Result { + let bytes_data = compressed.data.iter().map(LanceBuffer::len).sum::(); + let num_buffers = compressed.data.len(); + let mut data_buffer = Vec::with_capacity(bytes_data + 9 * num_buffers); + let mut metadata = Vec::with_capacity(compressed.chunks.len() * 8); + let mut buffer_offsets = vec![0_usize; num_buffers]; + + for chunk in compressed.chunks { + if chunk.buffer_sizes.len() != num_buffers { + return Err(Error::internal(format!( + "Sparse chunk has {} value buffer sizes, expected {num_buffers}", + chunk.buffer_sizes.len() + ))); + } + + let chunk_start = data_buffer.len(); + debug_assert_eq!(chunk_start % MINIBLOCK_ALIGNMENT, 0); + data_buffer.extend_from_slice(&0_u16.to_le_bytes()); + if miniblock_chunk_size == MiniblockChunkSize::U32 { + for buffer_size in &chunk.buffer_sizes { + data_buffer.extend_from_slice(&buffer_size.to_le_bytes()); + } + } else { + for buffer_size in &chunk.buffer_sizes { + let buffer_size = u16::try_from(*buffer_size).map_err(|_| { + Error::internal(format!( + "Sparse value buffer size ({buffer_size} bytes) exceeds 16-bit metadata" + )) + })?; + data_buffer.extend_from_slice(&buffer_size.to_le_bytes()); + } + } + let add_padding = |buffer: &mut Vec| { + let padding = pad_bytes::(buffer.len()); + buffer.extend(iter::repeat_n(FILL_BYTE, padding)); + }; + add_padding(&mut data_buffer); + + for (buffer_size, (buffer, buffer_offset)) in chunk + .buffer_sizes + .iter() + .zip(compressed.data.iter().zip(buffer_offsets.iter_mut())) + { + let start = *buffer_offset; + let end = start.checked_add(*buffer_size as usize).ok_or_else(|| { + Error::internal("Sparse value buffer range overflows".to_string()) + })?; + let bytes = buffer.as_ref().get(start..end).ok_or_else(|| { + Error::internal(format!( + "Sparse value chunk requests bytes {start}..{end} from a {}-byte buffer", + buffer.len() + )) + })?; + *buffer_offset = end; + data_buffer.extend_from_slice(bytes); + add_padding(&mut data_buffer); + } + + let chunk_bytes = data_buffer.len() - chunk_start; + if chunk_bytes == 0 || !chunk_bytes.is_multiple_of(MINIBLOCK_ALIGNMENT) { + return Err(Error::internal(format!( + "Sparse value chunk size {chunk_bytes} is not a positive multiple of {MINIBLOCK_ALIGNMENT}" + ))); + } + let words_minus_one = chunk_bytes / MINIBLOCK_ALIGNMENT - 1; + metadata.extend_from_slice( + &u32::try_from(words_minus_one) + .map_err(|_| { + Error::internal(format!( + "Sparse value chunk size {chunk_bytes} exceeds the metadata limit" + )) + })? + .to_le_bytes(), + ); + metadata.extend_from_slice(&chunk.num_values.to_le_bytes()); + } + + for (index, (consumed, buffer)) in buffer_offsets + .iter() + .zip(compressed.data.iter()) + .enumerate() + { + if *consumed != buffer.len() { + return Err(Error::internal(format!( + "Sparse value buffer {index} consumed {consumed} bytes, expected {}", + buffer.len() + ))); + } + } + + Ok(SerializedValuePage { + num_buffers: usize_to_u64(num_buffers, "value buffer count")?, + data: LanceBuffer::from(data_buffer), + metadata: LanceBuffer::from(metadata), + }) +} + +pub fn prepare_values( + field: &Field, + compression_strategy: &dyn CompressionStrategy, + data: DataBlock, + miniblock_chunk_size: MiniblockChunkSize, +) -> Result { + match &data { + DataBlock::AllNull(_) => { + return Err(Error::internal( + "All-null values must use ConstantLayout".to_string(), + )); + } + DataBlock::Dictionary(_) => { + return Err(Error::not_supported_source( + "Sparse layout does not support dictionary data blocks".into(), + )); + } + DataBlock::Struct(data) if data.has_variable_width_child() => { + return Err(Error::not_supported_source( + "Sparse layout does not support variable-width packed struct data blocks".into(), + )); + } + _ => {} + } + + let num_values = data.num_values(); + let compressor = compression_strategy.create_miniblock_compressor(field, &data)?; + let support_large_chunk = miniblock_chunk_size == MiniblockChunkSize::U32; + let compression_context = MiniBlockCompressionContext::new(0, support_large_chunk, false); + let (compressed, value_compression) = compressor.compress(compression_context, data)?; + let values = serialize_value_chunks( + with_explicit_value_counts(compressed)?, + miniblock_chunk_size, + )?; + Ok(PreparedSparseValues { + num_values, + value_compression, + values, + }) +} + +fn encode_u64_values( + values: Vec, + compression_strategy: &dyn CompressionStrategy, +) -> Result<(LanceBuffer, CompressiveEncoding)> { + let num_values = usize_to_u64(values.len(), "u64 value count")?; + let mut block = DataBlock::FixedWidth(FixedWidthDataBlock { + data: LanceBuffer::reinterpret_vec(values), + bits_per_value: 64, + num_values, + block_info: BlockInfo::new(), + }); + block.compute_stat(); + let field = Field::new_arrow("", arrow_schema::DataType::UInt64, false)?; + let (compressor, encoding) = compression_strategy.create_block_compressor(&field, &block)?; + Ok((compressor.compress(block)?, encoding)) +} + +fn positions_to_deltas(positions: &[u64], label: &str) -> Result> { + let mut previous = 0_u64; + positions + .iter() + .copied() + .enumerate() + .map(|(index, position)| { + if index > 0 && position <= previous { + return Err(Error::invalid_input_source( + format!("Sparse structural {label} positions must be strictly increasing") + .into(), + )); + } + let delta = if index == 0 { + position + } else { + position - previous + }; + previous = position; + Ok(delta) + }) + .collect() +} + +fn encode_position_set( + positions: &SparsePositionSet, + compression_strategy: &dyn CompressionStrategy, + label: &str, +) -> Result<(Option, pb21::SparsePositionSet)> { + let (buffer, positions_pb) = match positions { + SparsePositionSet::Empty => ( + None, + pb21::sparse_position_set::Positions::Empty(pb21::SparsePositionEmpty {}), + ), + SparsePositionSet::All { .. } => ( + None, + pb21::sparse_position_set::Positions::All(pb21::SparsePositionAll {}), + ), + SparsePositionSet::Range { start, len } => ( + None, + pb21::sparse_position_set::Positions::Range(pb21::SparsePositionRange { + start: *start, + length: *len, + }), + ), + SparsePositionSet::Explicit(positions) => { + if positions.is_empty() { + return Err(Error::internal(format!( + "Sparse structural {label} explicit set is empty" + ))); + } + let (buffer, encoding) = + encode_u64_values(positions_to_deltas(positions, label)?, compression_strategy)?; + ( + Some(buffer), + pb21::sparse_position_set::Positions::Explicit(encoding), + ) + } + }; + Ok(( + buffer, + pb21::SparsePositionSet { + positions: Some(positions_pb), + num_positions: positions.len(), + }, + )) +} + +fn encode_count_set( + counts: &SparseCountSet, + compression_strategy: &dyn CompressionStrategy, +) -> Result<(Option, pb21::SparseCountSet)> { + let (buffer, counts_pb) = match counts { + SparseCountSet::Empty => ( + None, + pb21::sparse_count_set::Counts::Empty(pb21::SparseCountEmpty {}), + ), + SparseCountSet::Constant { value, .. } => ( + None, + pb21::sparse_count_set::Counts::Constant(pb21::SparseCountConstant { value: *value }), + ), + SparseCountSet::Explicit { counts, .. } => { + if counts.is_empty() { + return Err(Error::internal( + "Sparse structural explicit count set is empty".to_string(), + )); + } + let (buffer, encoding) = encode_u64_values(counts.to_vec(), compression_strategy)?; + ( + Some(buffer), + pb21::sparse_count_set::Counts::Explicit(encoding), + ) + } + }; + Ok(( + buffer, + pb21::SparseCountSet { + counts: Some(counts_pb), + }, + )) +} + +fn encode_validity_set( + validity: &SparseValiditySet, + compression_strategy: &dyn CompressionStrategy, + label: &str, +) -> Result<(Option, pb21::SparseValiditySet)> { + let (buffer, positions) = + encode_position_set(&validity.positions, compression_strategy, label)?; + let meaning = match validity.meaning { + SparseValidityMeaning::NullPositions => { + pb21::sparse_validity_set::Meaning::SparseValidityNullPositions + } + SparseValidityMeaning::ValidPositions => { + pb21::sparse_validity_set::Meaning::SparseValidityValidPositions + } + }; + Ok(( + buffer, + pb21::SparseValiditySet { + meaning: meaning as i32, + positions: Some(positions), + }, + )) +} + +fn encode_structural_plan( + plan: &SparseStructuralPlan, + compression_strategy: &dyn CompressionStrategy, +) -> Result { + let mut layers = Vec::with_capacity(plan.layers.len()); + let mut buffers = Vec::new(); + + for layer in &plan.layers { + match layer { + SparseStructuralLayerPlan::Validity { + num_slots, + validity, + } => { + let (validity_buffer, validity) = + encode_validity_set(validity, compression_strategy, "validity")?; + buffers.extend(validity_buffer); + layers.push(pb21::SparseStructuralLayer { + layer: Some(pb21::sparse_structural_layer::Layer::Validity( + pb21::SparseValidityLayer { + num_slots: *num_slots, + validity: Some(validity), + }, + )), + }); + } + SparseStructuralLayerPlan::List { + num_slots, + num_child_slots, + non_empty_positions, + counts, + validity, + } => { + if non_empty_positions.len() != counts.len() { + return Err(Error::invalid_input_source( + format!( + "Sparse structural list has {} non-empty positions but {} counts", + non_empty_positions.len(), + counts.len() + ) + .into(), + )); + } + let (position_buffer, non_empty_positions) = encode_position_set( + non_empty_positions, + compression_strategy, + "list non-empty", + )?; + buffers.extend(position_buffer); + let (count_buffer, counts) = encode_count_set(counts, compression_strategy)?; + buffers.extend(count_buffer); + let (validity_buffer, validity) = + encode_validity_set(validity, compression_strategy, "list validity")?; + buffers.extend(validity_buffer); + layers.push(pb21::SparseStructuralLayer { + layer: Some(pb21::sparse_structural_layer::Layer::List( + pb21::SparseListLayer { + num_slots: *num_slots, + num_child_slots: *num_child_slots, + non_empty_positions: Some(non_empty_positions), + counts: Some(counts), + validity: Some(validity), + }, + )), + }); + } + SparseStructuralLayerPlan::FixedSizeList { + num_slots, + dimension, + validity, + } => { + let (validity_buffer, validity) = encode_validity_set( + validity, + compression_strategy, + "fixed-size-list validity", + )?; + buffers.extend(validity_buffer); + num_slots.checked_mul(*dimension).ok_or_else(|| { + Error::invalid_input_source( + format!( + "Sparse structural fixed-size-list child slot count overflows: slots={num_slots}, dimension={dimension}" + ) + .into(), + ) + })?; + layers.push(pb21::SparseStructuralLayer { + layer: Some(pb21::sparse_structural_layer::Layer::FixedSizeList( + pb21::SparseFixedSizeListLayer { + num_slots: *num_slots, + dimension: *dimension, + validity: Some(validity), + }, + )), + }); + } + } + } + + Ok(EncodedStructuralPlan { layers, buffers }) +} + +#[allow(clippy::too_many_arguments)] +pub(in crate::encodings::logical::primitive) fn encode_page( + column_idx: u32, + field: &Field, + compression_strategy: &dyn CompressionStrategy, + values: SparseValueInput, + plan: SparseStructuralPlan, + row_number: u64, + num_rows: u64, + miniblock_chunk_size: MiniblockChunkSize, +) -> Result { + let PreparedSparseValues { + num_values, + value_compression, + values, + } = match values { + SparseValueInput::Unprepared(data) => { + prepare_values(field, compression_strategy, data, miniblock_chunk_size)? + } + SparseValueInput::Prepared(prepared) => prepared, + }; + if plan.num_visible_items != num_values { + return Err(Error::internal(format!( + "Sparse structural plan has {} visible items but data has {} values", + plan.num_visible_items, num_values + ))); + } + let structural = encode_structural_plan(&plan, compression_strategy)?; + let description = pb21::PageLayout { + layout: Some(pb21::page_layout::Layout::SparseLayout( + pb21::SparseLayout { + value_compression: Some(value_compression), + num_buffers: values.num_buffers, + num_items: plan.num_items, + num_visible_items: plan.num_visible_items, + has_large_chunk: miniblock_chunk_size == MiniblockChunkSize::U32, + structural_layers: structural.layers, + }, + )), + }; + + let mut page_data = Vec::with_capacity(2 + structural.buffers.len()); + page_data.push(values.metadata); + page_data.push(values.data); + page_data.extend(structural.buffers); + Ok(EncodedPage { + data: page_data, + description: PageEncoding::Structural(description), + num_rows, + row_number, + column_idx, + }) +} + +#[cfg(test)] +mod tests { + use std::{collections::HashMap, sync::Arc}; + + use arrow_array::{ + Array, ArrayRef, DictionaryArray, FixedSizeBinaryArray, FixedSizeListArray, Int8Array, + Int32Array, LargeListArray, ListArray, StringArray, StructArray, + builder::{Int32Builder, MapBuilder, StringBuilder}, + types::Int8Type, + }; + use arrow_buffer::{BooleanBuffer, NullBuffer, OffsetBuffer, ScalarBuffer}; + use arrow_schema::{DataType, Field as ArrowField, Fields}; + + use crate::{ + constants::{ + PACKED_STRUCT_META_KEY, STRUCTURAL_ENCODING_FULLZIP, STRUCTURAL_ENCODING_META_KEY, + STRUCTURAL_ENCODING_MINIBLOCK, STRUCTURAL_ENCODING_SPARSE, + }, + data::FixedSizeListBlock, + encoder::{ + ColumnIndexSequence, EncodingOptions, FieldEncoder, MIN_PAGE_BUFFER_ALIGNMENT, + OutOfLineBuffers, + }, + testing::{ + TestCases, TestEncoding, check_round_trip_encoding_of_data, test_encoding_strategy, + }, + }; + + use super::*; + + fn sparse_metadata() -> HashMap { + HashMap::from([( + STRUCTURAL_ENCODING_META_KEY.to_string(), + STRUCTURAL_ENCODING_SPARSE.to_string(), + )]) + } + + fn structural_metadata(value: &str) -> HashMap { + HashMap::from([(STRUCTURAL_ENCODING_META_KEY.to_string(), value.to_string())]) + } + + fn null_buffer(validity: impl IntoIterator) -> NullBuffer { + NullBuffer::new(BooleanBuffer::from_iter(validity)) + } + + fn list_i32(offsets: Vec, validity: Option>) -> ArrayRef { + let num_values = offsets.last().copied().unwrap_or_default(); + let values = Arc::new(Int32Array::from_iter_values(0..num_values)) as ArrayRef; + Arc::new( + ListArray::try_new( + Arc::new(ArrowField::new("item", DataType::Int32, true)), + OffsetBuffer::new(ScalarBuffer::from(offsets)), + values, + validity.map(null_buffer), + ) + .unwrap(), + ) + } + + fn sparse_list_values( + num_rows: usize, + stride: usize, + values: ArrayRef, + item_field: Arc, + ) -> ArrayRef { + let mut offsets = Vec::with_capacity(num_rows + 1); + let mut num_values = 0_i32; + offsets.push(num_values); + for row in 0..num_rows { + if (row + 1).is_multiple_of(stride) || row + 1 == num_rows { + num_values += 1; + } + offsets.push(num_values); + } + assert_eq!(values.len(), num_values as usize); + Arc::new( + ListArray::try_new( + item_field, + OffsetBuffer::new(ScalarBuffer::from(offsets)), + values, + None, + ) + .unwrap(), + ) + } + + fn sparse_i32_list(num_rows: usize, stride: usize) -> ArrayRef { + let num_values = num_rows.div_ceil(stride); + sparse_list_values( + num_rows, + stride, + Arc::new(Int32Array::from_iter_values(0..num_values as i32)), + Arc::new(ArrowField::new("item", DataType::Int32, true)), + ) + } + + fn unsplittable_nested_list(values: ArrayRef, item_field: Arc) -> ArrayRef { + const NUM_INNER_LISTS: usize = 70_000; + + let mut inner_offsets = vec![0_i32; NUM_INNER_LISTS + 1]; + assert!(!values.is_empty()); + assert!(values.len() <= NUM_INNER_LISTS); + for value_index in 1..=values.len() { + inner_offsets[NUM_INNER_LISTS - values.len() + value_index] = value_index as i32; + } + let inner = Arc::new( + ListArray::try_new( + item_field, + OffsetBuffer::new(ScalarBuffer::from(inner_offsets)), + values, + None, + ) + .unwrap(), + ) as ArrayRef; + Arc::new( + ListArray::try_new( + Arc::new(ArrowField::new("item", inner.data_type().clone(), true)), + OffsetBuffer::new(ScalarBuffer::from(vec![0_i32, NUM_INNER_LISTS as i32])), + inner, + None, + ) + .unwrap(), + ) + } + + fn variable_packed_struct_values(num_values: usize) -> (ArrayRef, Arc) { + let fields = Fields::from(vec![ArrowField::new("value", DataType::Utf8, false)]); + let values = Arc::new(StructArray::new( + fields.clone(), + vec![Arc::new(StringArray::from_iter_values( + (0..num_values).map(|index| format!("value-{index}")), + ))], + None, + )) as ArrayRef; + let item_field = Arc::new( + ArrowField::new("item", DataType::Struct(fields), true).with_metadata(HashMap::from([ + (PACKED_STRUCT_META_KEY.to_string(), "true".to_string()), + ])), + ); + (values, item_field) + } + + fn dictionary_values(num_values: usize) -> (ArrayRef, Arc) { + let keys = Int8Array::from_iter_values((0..num_values).map(|index| (index % 2) as i8)); + let values = Arc::new(StringArray::from(vec!["value-0", "value-1"])); + let dictionary = Arc::new(DictionaryArray::::try_new(keys, values).unwrap()); + let item_field = Arc::new(ArrowField::new( + "item", + dictionary.data_type().clone(), + true, + )); + (dictionary, item_field) + } + + fn large_list_i32(offsets: Vec, validity: Option>) -> ArrayRef { + let num_values = offsets.last().copied().unwrap_or_default(); + let values = Arc::new(Int32Array::from_iter_values(0..num_values as i32)) as ArrayRef; + Arc::new( + LargeListArray::try_new( + Arc::new(ArrowField::new("item", DataType::Int32, true)), + OffsetBuffer::new(ScalarBuffer::from(offsets)), + values, + validity.map(null_buffer), + ) + .unwrap(), + ) + } + + fn map_i32() -> ArrayRef { + let mut builder = MapBuilder::new(None, StringBuilder::new(), Int32Builder::new()); + builder.keys().append_value("a"); + builder.values().append_value(1); + builder.append(true).unwrap(); + builder.append(false).unwrap(); + builder.append(true).unwrap(); + builder.keys().append_value("b"); + builder.values().append_null(); + builder.keys().append_value("c"); + builder.values().append_value(3); + builder.append(true).unwrap(); + Arc::new(builder.finish()) + } + + fn fixed_size_list_struct() -> ArrayRef { + let fields = Fields::from(vec![ArrowField::new("value", DataType::Int32, true)]); + let child = Arc::new(StructArray::new( + fields.clone(), + vec![Arc::new(Int32Array::from(vec![ + Some(0), + Some(1), + None, + Some(3), + Some(4), + Some(5), + None, + Some(7), + Some(8), + Some(9), + Some(10), + None, + ]))], + Some(null_buffer([ + true, true, false, true, true, true, true, false, true, true, true, true, + ])), + )) as ArrayRef; + Arc::new( + FixedSizeListArray::try_new( + Arc::new(ArrowField::new("item", DataType::Struct(fields), true)), + 2, + child, + Some(null_buffer([true, false, true, true, false, true])), + ) + .unwrap(), + ) + } + + fn list_fixed_size_list_struct() -> ArrayRef { + let struct_fields = Fields::from(vec![ArrowField::new("value", DataType::Int32, true)]); + let structs = Arc::new(StructArray::new( + struct_fields.clone(), + vec![Arc::new(Int32Array::from(vec![ + Some(0), + None, + Some(2), + Some(3), + None, + Some(5), + ]))], + Some(null_buffer([true, true, false, true, true, true])), + )) as ArrayRef; + let fixed_size_list = Arc::new( + FixedSizeListArray::try_new( + Arc::new(ArrowField::new( + "item", + DataType::Struct(struct_fields), + true, + )), + 2, + structs, + Some(null_buffer([true, false, true])), + ) + .unwrap(), + ) as ArrayRef; + Arc::new( + ListArray::try_new( + Arc::new(ArrowField::new( + "item", + fixed_size_list.data_type().clone(), + true, + )), + OffsetBuffer::new(ScalarBuffer::from(vec![0_i32, 1, 1, 3, 3])), + fixed_size_list, + Some(null_buffer([true, false, true, true])), + ) + .unwrap(), + ) + } + + fn nullable_struct() -> ArrayRef { + let fields = Fields::from(vec![ArrowField::new("value", DataType::Int32, true)]); + Arc::new(StructArray::new( + fields, + vec![Arc::new(Int32Array::from(vec![ + Some(10), + Some(20), + None, + Some(40), + Some(50), + ]))], + Some(null_buffer([true, false, true, true, false])), + )) + } + + fn deeply_nested() -> ArrayRef { + let leaf = Arc::new(Int32Array::from(vec![ + Some(0), + None, + Some(2), + Some(3), + None, + Some(5), + Some(6), + Some(7), + ])) as ArrayRef; + let inner = Arc::new( + LargeListArray::try_new( + Arc::new(ArrowField::new("item", DataType::Int32, true)), + OffsetBuffer::new(ScalarBuffer::from(vec![0_i64, 2, 2, 3, 5, 5, 8])), + leaf, + Some(null_buffer([true, false, true, true, true, true])), + ) + .unwrap(), + ) as ArrayRef; + let struct_fields = Fields::from(vec![ArrowField::new( + "inner", + inner.data_type().clone(), + true, + )]); + let structs = Arc::new(StructArray::new( + struct_fields.clone(), + vec![inner], + Some(null_buffer([true, true, false, true, true, true])), + )) as ArrayRef; + Arc::new( + ListArray::try_new( + Arc::new(ArrowField::new( + "item", + DataType::Struct(struct_fields), + true, + )), + OffsetBuffer::new(ScalarBuffer::from(vec![0_i32, 2, 2, 2, 4, 6])), + structs, + Some(null_buffer([true, false, true, true, true])), + ) + .unwrap(), + ) + } + + fn page_layout(page: &EncodedPage) -> &pb21::page_layout::Layout { + let PageEncoding::Structural(layout) = &page.description else { + panic!("expected structural page encoding"); + }; + layout.layout.as_ref().expect("page layout must be present") + } + + fn create_encoder( + array: &ArrayRef, + version: TestEncoding, + metadata: HashMap, + ) -> Result> { + let arrow_field = + ArrowField::new("values", array.data_type().clone(), true).with_metadata(metadata); + let field = Field::try_from(&arrow_field)?; + let strategy = test_encoding_strategy(version); + let options = EncodingOptions { + cache_bytes_per_column: 1, + ..Default::default() + }; + crate::testing::create_test_field_encoder( + strategy.as_ref(), + &field, + &mut ColumnIndexSequence::default(), + &options, + ) + } + + async fn encode_pages( + array: ArrayRef, + version: TestEncoding, + metadata: HashMap, + ) -> Result> { + encode_chunks(vec![array], version, metadata).await + } + + async fn encode_chunks( + arrays: Vec, + version: TestEncoding, + metadata: HashMap, + ) -> Result> { + let first = arrays + .first() + .ok_or_else(|| Error::internal("test input has no arrays".to_string()))?; + let mut encoder = create_encoder(first, version, metadata)?; + let mut external_buffers = OutOfLineBuffers::new(0, MIN_PAGE_BUFFER_ALIGNMENT); + let mut pages = Vec::new(); + let mut row_number = 0_u64; + for array in arrays { + let num_rows = array.len() as u64; + for task in encoder.maybe_encode( + array, + &mut external_buffers, + crate::repdef::RepDefBuilder::default(), + row_number, + num_rows, + )? { + pages.push(task.await?); + } + row_number += num_rows; + } + for task in encoder.flush(&mut external_buffers)? { + pages.push(task.await?); + } + for column in encoder.finish(&mut external_buffers).await? { + pages.extend(column.final_pages); + } + Ok(pages) + } + + fn sparse_layout(page: &EncodedPage) -> &pb21::SparseLayout { + let pb21::page_layout::Layout::SparseLayout(sparse) = page_layout(page) else { + panic!("expected SparseLayout, got {:?}", page_layout(page)); + }; + sparse + } + + fn list_layer(sparse: &pb21::SparseLayout) -> &pb21::SparseListLayer { + sparse + .structural_layers + .iter() + .find_map(|layer| match layer.layer.as_ref() { + Some(pb21::sparse_structural_layer::Layer::List(layer)) => Some(layer), + _ => None, + }) + .expect("expected sparse list layer") + } + + fn validity_layer(layer: &pb21::SparseStructuralLayer) -> Option<&pb21::SparseValidityLayer> { + match layer.layer.as_ref() { + Some(pb21::sparse_structural_layer::Layer::Validity(layer)) => Some(layer), + _ => None, + } + } + + fn layer_num_slots(layer: &pb21::SparseStructuralLayer) -> u64 { + match layer.layer.as_ref().expect("expected sparse layer variant") { + pb21::sparse_structural_layer::Layer::Validity(layer) => layer.num_slots, + pb21::sparse_structural_layer::Layer::List(layer) => layer.num_slots, + pb21::sparse_structural_layer::Layer::FixedSizeList(layer) => layer.num_slots, + } + } + + fn fixed_size_list_dimension(layer: &pb21::SparseStructuralLayer) -> Option { + match layer.layer.as_ref() { + Some(pb21::sparse_structural_layer::Layer::FixedSizeList(layer)) => { + Some(layer.dimension) + } + _ => None, + } + } + + #[test] + fn test_sparse_value_block_eligibility() { + let fixed = DataBlock::from_array(Int32Array::from_iter_values(0..4)); + assert!(supports_value_block(&fixed)); + + let variable = DataBlock::from_array(StringArray::from(vec!["a", "b"])); + assert!(supports_value_block(&variable)); + + let nullable = DataBlock::from_array(Int32Array::from(vec![Some(1), None])); + assert!(!supports_value_block(&nullable)); + + let all_null = DataBlock::from_array(Int32Array::from(vec![None, None])); + assert!(!supports_value_block(&all_null)); + + let (dictionary, _) = dictionary_values(4); + assert!(!supports_value_block(&DataBlock::from_arrays( + &[dictionary], + 4 + ))); + + let fixed_fields = Fields::from(vec![ArrowField::new("value", DataType::Int32, false)]); + let fixed_struct = StructArray::new( + fixed_fields, + vec![Arc::new(Int32Array::from_iter_values(0..4))], + None, + ); + assert!(supports_value_block(&DataBlock::from_array(fixed_struct))); + + let variable_fields = Fields::from(vec![ArrowField::new("value", DataType::Utf8, false)]); + let variable_struct = StructArray::new( + variable_fields, + vec![Arc::new(StringArray::from(vec!["a", "b"]))], + None, + ); + assert!(!supports_value_block(&DataBlock::from_array( + variable_struct + ))); + + let fixed_size_list = FixedSizeListArray::try_new( + Arc::new(ArrowField::new("item", DataType::Int32, false)), + 2, + Arc::new(Int32Array::from_iter_values(0..4)), + None, + ) + .unwrap(); + assert!(supports_value_block(&DataBlock::from_array( + fixed_size_list + ))); + + let unsupported_fixed_size_list = DataBlock::FixedSizeList(FixedSizeListBlock { + child: Box::new(DataBlock::from_array(StringArray::from(vec![ + "a", "b", "c", "d", + ]))), + dimension: 2, + }); + assert!(!supports_value_block(&unsupported_fixed_size_list)); + } + + fn planned_list( + offsets: Vec, + list_validity: Option>, + leaf_validity: Option>, + ) -> SparseStructuralPlan { + let num_values = u64::try_from(*offsets.last().unwrap()).unwrap(); + let mut builder = crate::repdef::RepDefBuilder::default(); + assert!(!builder.add_offsets( + OffsetBuffer::new(ScalarBuffer::from(offsets)), + list_validity.map(null_buffer), + )); + if let Some(leaf_validity) = leaf_validity { + builder.add_validity_bitmap(null_buffer(leaf_validity)); + } else { + builder.add_no_null(num_values as usize); + } + let normalized = crate::repdef::RepDefBuilder::normalize(vec![builder]); + plan(&normalized, num_values).unwrap() + } + + fn planned_list_layer(plan: &SparseStructuralPlan) -> &SparseStructuralLayerPlan { + plan.layers + .iter() + .find(|layer| matches!(layer, SparseStructuralLayerPlan::List { .. })) + .expect("expected planned list layer") + } + + #[test] + fn test_semantic_position_and_count_forms() { + let empty = planned_list(vec![0, 0, 0, 0], None, None); + assert_eq!(empty.num_items, 3); + assert_eq!(empty.num_visible_items, 0); + assert!(matches!( + planned_list_layer(&empty), + SparseStructuralLayerPlan::List { + non_empty_positions: SparsePositionSet::Empty, + counts: SparseCountSet::Empty, + .. + } + )); + + let all = planned_list(vec![0, 2, 4, 6], None, None); + assert_eq!(all.num_items, 6); + assert!(matches!( + planned_list_layer(&all), + SparseStructuralLayerPlan::List { + non_empty_positions: SparsePositionSet::All { len: 3 }, + counts: SparseCountSet::Constant { value: 2, len: 3 }, + .. + } + )); + + let range = planned_list(vec![0, 2, 4, 4, 4], None, None); + assert_eq!(range.num_items, 6); + assert!(matches!( + planned_list_layer(&range), + SparseStructuralLayerPlan::List { + non_empty_positions: SparsePositionSet::Range { start: 0, len: 2 }, + counts: SparseCountSet::Constant { value: 2, len: 2 }, + .. + } + )); + + let explicit = planned_list(vec![0, 1, 1, 4, 4, 6], None, None); + assert_eq!(explicit.num_items, 8); + assert!(matches!( + planned_list_layer(&explicit), + SparseStructuralLayerPlan::List { + non_empty_positions: SparsePositionSet::Explicit(positions), + counts: SparseCountSet::Explicit { counts, .. }, + .. + } if positions == &vec![0, 2, 4] && counts.as_ref() == [1, 3, 2] + )); + } + + #[test] + fn test_validity_polarity_uses_semantic_encoded_cost() { + let mostly_valid = BooleanBuffer::from_iter([true, false, true, true, false, true]); + let validity = validity_set(Some(&mostly_valid), mostly_valid.len(), "test").unwrap(); + assert_eq!(validity.meaning, SparseValidityMeaning::NullPositions); + assert!(matches!(validity.positions, SparsePositionSet::Explicit(ref p) if p == &[1, 4])); + + let mostly_null = BooleanBuffer::from_iter([false, true, false, false, true, false]); + let validity = validity_set(Some(&mostly_null), mostly_null.len(), "test").unwrap(); + assert_eq!(validity.meaning, SparseValidityMeaning::ValidPositions); + assert!(matches!(validity.positions, SparsePositionSet::Explicit(ref p) if p == &[1, 4])); + + let valid_island = BooleanBuffer::from_iter([false, false, true, true, false]); + let validity = validity_set(Some(&valid_island), valid_island.len(), "test").unwrap(); + assert_eq!(validity.meaning, SparseValidityMeaning::ValidPositions); + assert!(matches!( + validity.positions, + SparsePositionSet::Range { start: 2, len: 2 } + )); + + let all_valid = BooleanBuffer::from_iter([true, true, true]); + let validity = validity_set(Some(&all_valid), all_valid.len(), "test").unwrap(); + assert_eq!(validity.meaning, SparseValidityMeaning::NullPositions); + assert!(matches!(validity.positions, SparsePositionSet::Empty)); + + let all_null = BooleanBuffer::from_iter([false, false, false]); + let validity = validity_set(Some(&all_null), all_null.len(), "test").unwrap(); + assert_eq!(validity.meaning, SparseValidityMeaning::ValidPositions); + assert!(matches!(validity.positions, SparsePositionSet::Empty)); + } + + #[tokio::test] + async fn test_explicit_sparse_nullable_primitive_roundtrip() { + let array = Arc::new(Int32Array::from(vec![ + Some(10), + None, + Some(20), + Some(30), + None, + Some(40), + ])) as ArrayRef; + let pages = encode_pages( + array.clone(), + TestEncoding::StructuralSparse, + sparse_metadata(), + ) + .await + .unwrap(); + assert_eq!(pages.len(), 1); + let sparse = sparse_layout(&pages[0]); + assert_eq!(sparse.num_items, 6); + assert_eq!(sparse.num_visible_items, 6); + assert_eq!(sparse.structural_layers.len(), 1); + let validity = validity_layer(&sparse.structural_layers[0]) + .unwrap() + .validity + .as_ref() + .unwrap(); + assert_eq!( + validity.meaning, + pb21::sparse_validity_set::Meaning::SparseValidityNullPositions as i32 + ); + assert!(matches!( + validity.positions.as_ref().unwrap().positions, + Some(pb21::sparse_position_set::Positions::Explicit(_)) + )); + + let cases = TestCases::default() + .with_encoding(TestEncoding::StructuralSparse) + .with_page_sizes(vec![1]) + .with_range(1..5) + .with_indices(vec![0, 2, 5]); + check_round_trip_encoding_of_data(vec![array], &cases, sparse_metadata()).await; + } + + #[tokio::test] + async fn test_explicit_sparse_nullable_struct_roundtrip() { + let array = nullable_struct(); + let pages = encode_pages( + array.clone(), + TestEncoding::StructuralSparse, + sparse_metadata(), + ) + .await + .unwrap(); + assert_eq!(pages.len(), 1); + let sparse = sparse_layout(&pages[0]); + assert_eq!(sparse.structural_layers.len(), 2); + assert!( + sparse + .structural_layers + .iter() + .all(|layer| validity_layer(layer).is_some()) + ); + let struct_validity = validity_layer(&sparse.structural_layers[0]) + .unwrap() + .validity + .as_ref() + .unwrap(); + assert_eq!( + struct_validity.meaning, + pb21::sparse_validity_set::Meaning::SparseValidityNullPositions as i32 + ); + assert!(matches!( + struct_validity.positions.as_ref().unwrap().positions, + Some(pb21::sparse_position_set::Positions::Explicit(_)) + )); + + let cases = TestCases::default() + .with_encoding(TestEncoding::StructuralSparse) + .with_page_sizes(vec![1]) + .with_range(1..5) + .with_indices(vec![0, 2, 4]); + check_round_trip_encoding_of_data(vec![array], &cases, sparse_metadata()).await; + } + + #[tokio::test] + async fn test_explicit_sparse_struct_with_constant_and_sparse_children() { + let fields = Fields::from(vec![ + ArrowField::new("constant", DataType::Int32, true), + ArrowField::new("sparse", DataType::Int32, true), + ]); + let array = Arc::new(StructArray::new( + fields, + vec![ + Arc::new(Int32Array::from(vec![None::; 5])), + Arc::new(Int32Array::from(vec![ + Some(10), + Some(20), + None, + Some(40), + Some(50), + ])), + ], + Some(null_buffer([true, false, true, true, false])), + )) as ArrayRef; + let pages = encode_pages( + array.clone(), + TestEncoding::StructuralSparse, + sparse_metadata(), + ) + .await + .unwrap(); + assert!(pages.iter().any(|page| matches!( + page_layout(page), + pb21::page_layout::Layout::ConstantLayout(_) + ))); + assert!(pages.iter().any(|page| matches!( + page_layout(page), + pb21::page_layout::Layout::SparseLayout(_) + ))); + + let cases = TestCases::default() + .with_encoding(TestEncoding::StructuralSparse) + .with_page_sizes(vec![1]) + .with_range(1..5) + .with_indices(vec![0, 2, 4]); + check_round_trip_encoding_of_data(vec![array], &cases, sparse_metadata()).await; + } + + #[tokio::test] + async fn test_explicit_sparse_emits_both_validity_polarities() { + let mostly_valid = Arc::new(Int32Array::from(vec![ + Some(0), + None, + Some(2), + Some(3), + None, + Some(5), + ])) as ArrayRef; + let mostly_null = Arc::new(Int32Array::from(vec![ + None, + Some(1), + None, + None, + Some(4), + None, + ])) as ArrayRef; + + let null_positions = encode_pages( + mostly_valid.clone(), + TestEncoding::StructuralSparse, + sparse_metadata(), + ) + .await + .unwrap(); + let validity = validity_layer(&sparse_layout(&null_positions[0]).structural_layers[0]) + .unwrap() + .validity + .as_ref() + .unwrap(); + assert_eq!( + validity.meaning, + pb21::sparse_validity_set::Meaning::SparseValidityNullPositions as i32 + ); + + let valid_positions = encode_pages( + mostly_null.clone(), + TestEncoding::StructuralSparse, + sparse_metadata(), + ) + .await + .unwrap(); + let validity = validity_layer(&sparse_layout(&valid_positions[0]).structural_layers[0]) + .unwrap() + .validity + .as_ref() + .unwrap(); + assert_eq!( + validity.meaning, + pb21::sparse_validity_set::Meaning::SparseValidityValidPositions as i32 + ); + + let cases = TestCases::default() + .with_encoding(TestEncoding::StructuralSparse) + .with_page_sizes(vec![1]) + .with_range(1..5) + .with_indices(vec![0, 2, 5]); + for array in [mostly_valid, mostly_null] { + check_round_trip_encoding_of_data(vec![array], &cases, sparse_metadata()).await; + } + } + + #[tokio::test] + async fn test_explicit_sparse_nested_page_boundaries_range_and_take() { + let nested = deeply_nested(); + let chunks = vec![nested.slice(0, 2), nested.slice(2, 3)]; + let pages = encode_chunks( + chunks.clone(), + TestEncoding::StructuralSparse, + sparse_metadata(), + ) + .await + .unwrap(); + assert!(pages.len() >= 2, "expected multiple sparse pages"); + let sparse = pages + .iter() + .map(sparse_layout) + .collect::>(); + assert!(sparse.iter().any(|layout| { + layout + .structural_layers + .iter() + .filter(|layer| { + matches!( + layer.layer.as_ref(), + Some(pb21::sparse_structural_layer::Layer::List(_)) + ) + }) + .count() + >= 2 + })); + assert!(sparse.iter().any(|layout| { + layout + .structural_layers + .iter() + .any(|layer| validity_layer(layer).is_some()) + })); + + let cases = TestCases::default() + .with_encoding(TestEncoding::StructuralSparse) + .with_page_sizes(vec![1]) + .with_batch_size(2) + .with_range(1..5) + .with_range(2..4) + .with_indices(vec![0, 2, 4]) + .with_indices(vec![1, 3]); + check_round_trip_encoding_of_data(chunks, &cases, sparse_metadata()).await; + } + + #[tokio::test] + async fn test_explicit_sparse_list_and_large_list_null_empty_roundtrip() { + let list = list_i32( + vec![0, 0, 0, 2, 3, 3], + Some(vec![false, true, true, true, true]), + ); + let large_list = large_list_i32( + vec![0, 0, 0, 2, 3, 3], + Some(vec![false, true, true, true, true]), + ); + let cases = TestCases::default() + .with_encoding(TestEncoding::StructuralSparse) + .with_page_sizes(vec![1]) + .with_range(0..5) + .with_range(1..4) + .with_indices(vec![0, 1, 4]) + .with_indices(vec![2, 3]); + + for array in [list, large_list] { + let pages = encode_pages( + array.clone(), + TestEncoding::StructuralSparse, + sparse_metadata(), + ) + .await + .unwrap(); + assert!(pages.iter().map(sparse_layout).all(|layout| { + layout.structural_layers.iter().any(|layer| { + matches!( + layer.layer.as_ref(), + Some(pb21::sparse_structural_layer::Layer::List(_)) + ) + }) + })); + check_round_trip_encoding_of_data(vec![array], &cases, sparse_metadata()).await; + } + } + + #[tokio::test] + async fn test_explicit_sparse_map_and_fixed_size_list_roundtrip() { + let map = map_i32(); + let map_pages = encode_pages( + map.clone(), + TestEncoding::StructuralSparse, + sparse_metadata(), + ) + .await + .unwrap(); + assert!(map_pages.iter().map(sparse_layout).any(|layout| { + layout.structural_layers.iter().any(|layer| { + matches!( + layer.layer.as_ref(), + Some(pb21::sparse_structural_layer::Layer::List(_)) + ) + }) + })); + let map_cases = TestCases::default() + .with_encoding(TestEncoding::StructuralSparse) + .with_page_sizes(vec![1]) + .with_range(1..4) + .with_indices(vec![0, 2, 3]); + check_round_trip_encoding_of_data(vec![map], &map_cases, sparse_metadata()).await; + + let fsl = fixed_size_list_struct(); + let fsl_pages = encode_pages( + fsl.clone(), + TestEncoding::StructuralSparse, + sparse_metadata(), + ) + .await + .unwrap(); + for page in &fsl_pages { + let layout = sparse_layout(page); + let outer_slots = layer_num_slots(layout.structural_layers.first().unwrap()); + let fixed_size_scale = layout + .structural_layers + .iter() + .filter_map(fixed_size_list_dimension) + .product::(); + assert_eq!(page.num_rows, outer_slots * fixed_size_scale); + } + assert!(fsl_pages.iter().map(sparse_layout).any(|layout| { + layout + .structural_layers + .iter() + .any(|layer| fixed_size_list_dimension(layer) == Some(2)) + })); + let fsl_cases = TestCases::default() + .with_encoding(TestEncoding::StructuralSparse) + .with_page_sizes(vec![1]) + .with_range(1..6) + .with_indices(vec![0, 3, 5]); + check_round_trip_encoding_of_data(vec![fsl], &fsl_cases, sparse_metadata()).await; + } + + #[tokio::test] + async fn test_explicit_sparse_list_fixed_size_list_struct_roundtrip() { + let array = list_fixed_size_list_struct(); + let pages = encode_pages( + array.clone(), + TestEncoding::StructuralSparse, + sparse_metadata(), + ) + .await + .unwrap(); + assert!(pages.iter().map(sparse_layout).any(|layout| { + let kinds = layout + .structural_layers + .iter() + .map(|layer| match layer.layer.as_ref().unwrap() { + pb21::sparse_structural_layer::Layer::Validity(_) => "validity", + pb21::sparse_structural_layer::Layer::List(_) => "list", + pb21::sparse_structural_layer::Layer::FixedSizeList(_) => "fixed-size-list", + }) + .collect::>(); + kinds.starts_with(&["list", "fixed-size-list"]) + })); + for page in &pages { + let layout = sparse_layout(page); + let outer_slots = layer_num_slots(layout.structural_layers.first().unwrap()); + assert_eq!(page.num_rows, outer_slots * 2); + } + + let cases = TestCases::default() + .with_encoding(TestEncoding::StructuralSparse) + .with_page_sizes(vec![1]) + .with_range(1..4) + .with_indices(vec![0, 2, 3]) + .with_indices(vec![1, 3]); + check_round_trip_encoding_of_data(vec![array], &cases, sparse_metadata()).await; + } + + #[tokio::test] + async fn test_explicit_sparse_serializes_semantic_list_forms() { + let all_array = list_i32(vec![0, 2, 4, 6], None); + let all = encode_pages( + all_array.clone(), + TestEncoding::StructuralSparse, + sparse_metadata(), + ) + .await + .unwrap(); + let all_layer = list_layer(sparse_layout(&all[0])); + assert!(matches!( + all_layer.non_empty_positions.as_ref().unwrap().positions, + Some(pb21::sparse_position_set::Positions::All(_)) + )); + assert!(matches!( + all_layer.counts.as_ref().unwrap().counts, + Some(pb21::sparse_count_set::Counts::Constant( + pb21::SparseCountConstant { value: 2 } + )) + )); + assert_eq!(all[0].data.len(), 2); + + let range_array = list_i32(vec![0, 2, 4, 4, 4], None); + let range = encode_pages( + range_array.clone(), + TestEncoding::StructuralSparse, + sparse_metadata(), + ) + .await + .unwrap(); + let range_layer = list_layer(sparse_layout(&range[0])); + assert!(matches!( + range_layer.non_empty_positions.as_ref().unwrap().positions, + Some(pb21::sparse_position_set::Positions::Range( + pb21::SparsePositionRange { + start: 0, + length: 2 + } + )) + )); + assert!(matches!( + range_layer.counts.as_ref().unwrap().counts, + Some(pb21::sparse_count_set::Counts::Constant( + pb21::SparseCountConstant { value: 2 } + )) + )); + assert_eq!(range[0].data.len(), 2); + + let explicit_array = list_i32(vec![0, 1, 1, 4, 4, 6], None); + let explicit = encode_pages( + explicit_array.clone(), + TestEncoding::StructuralSparse, + sparse_metadata(), + ) + .await + .unwrap(); + let explicit_layer = list_layer(sparse_layout(&explicit[0])); + assert!(matches!( + explicit_layer + .non_empty_positions + .as_ref() + .unwrap() + .positions, + Some(pb21::sparse_position_set::Positions::Explicit(_)) + )); + assert!(matches!( + explicit_layer.counts.as_ref().unwrap().counts, + Some(pb21::sparse_count_set::Counts::Explicit(_)) + )); + assert_eq!(explicit[0].data.len(), 4); + + let cases = TestCases::default() + .with_encoding(TestEncoding::StructuralSparse) + .with_page_sizes(vec![1]) + .with_range(1..3) + .with_indices(vec![0, 2]); + for array in [all_array, range_array, explicit_array] { + check_round_trip_encoding_of_data(vec![array], &cases, sparse_metadata()).await; + } + } + + #[tokio::test] + async fn test_constant_layout_boundary_is_explicit() { + let structural_only = encode_pages( + list_i32(vec![0, 0, 0, 0], None), + TestEncoding::StructuralSparse, + sparse_metadata(), + ) + .await + .unwrap(); + assert!(matches!( + page_layout(&structural_only[0]), + pb21::page_layout::Layout::ConstantLayout(_) + )); + + let empty_struct = Arc::new(StructArray::new_empty_fields(3, None)) as ArrayRef; + let empty_struct_pages = encode_pages( + empty_struct, + TestEncoding::StructuralSparse, + sparse_metadata(), + ) + .await + .unwrap(); + assert!(matches!( + page_layout(&empty_struct_pages[0]), + pb21::page_layout::Layout::ConstantLayout(_) + )); + + let all_null = Arc::new(Int32Array::from(vec![None, None, None])) as ArrayRef; + let all_null_pages = + encode_pages(all_null, TestEncoding::StructuralSparse, sparse_metadata()) + .await + .unwrap(); + assert!(matches!( + page_layout(&all_null_pages[0]), + pb21::page_layout::Layout::ConstantLayout(_) + )); + + let constant = Arc::new(Int32Array::from(vec![7, 7, 7])) as ArrayRef; + let constant_pages = + encode_pages(constant, TestEncoding::StructuralSparse, sparse_metadata()) + .await + .unwrap(); + assert!(matches!( + page_layout(&constant_pages[0]), + pb21::page_layout::Layout::SparseLayout(_) + )); + } + + #[tokio::test] + async fn test_default_and_explicit_dense_layouts_are_unchanged() { + let array = Arc::new(Int32Array::from_iter_values(0..16)) as ArrayRef; + let v2_2_default = encode_pages(array.clone(), TestEncoding::StructuralU32, HashMap::new()) + .await + .unwrap(); + let v2_2_miniblock = encode_pages( + array.clone(), + TestEncoding::StructuralU32, + structural_metadata(STRUCTURAL_ENCODING_MINIBLOCK), + ) + .await + .unwrap(); + assert_eq!(v2_2_default.len(), v2_2_miniblock.len()); + for (default, explicit) in v2_2_default.iter().zip(v2_2_miniblock.iter()) { + assert!(matches!( + page_layout(default), + pb21::page_layout::Layout::MiniBlockLayout(_) + )); + assert_eq!(page_layout(default), page_layout(explicit)); + assert_eq!(default.data, explicit.data); + } + + let v2_3_default = encode_pages( + array.clone(), + TestEncoding::StructuralSparse, + HashMap::new(), + ) + .await + .unwrap(); + assert!(matches!( + page_layout(&v2_3_default[0]), + pb21::page_layout::Layout::MiniBlockLayout(_) + )); + + let v2_3_miniblock = encode_pages( + array.clone(), + TestEncoding::StructuralSparse, + structural_metadata(STRUCTURAL_ENCODING_MINIBLOCK), + ) + .await + .unwrap(); + assert!(matches!( + page_layout(&v2_3_miniblock[0]), + pb21::page_layout::Layout::MiniBlockLayout(_) + )); + + let v2_3_fullzip = encode_pages( + array.clone(), + TestEncoding::StructuralSparse, + structural_metadata(STRUCTURAL_ENCODING_FULLZIP), + ) + .await + .unwrap(); + assert!(matches!( + page_layout(&v2_3_fullzip[0]), + pb21::page_layout::Layout::FullZipLayout(_) + )); + + let v2_3_sparse = encode_pages(array, TestEncoding::StructuralSparse, sparse_metadata()) + .await + .unwrap(); + assert!(matches!( + page_layout(&v2_3_sparse[0]), + pb21::page_layout::Layout::SparseLayout(_) + )); + } + + #[tokio::test] + async fn test_auto_sparse_for_split_required_page() { + const NUM_ROWS: usize = 70_000; + let array = sparse_i32_list(NUM_ROWS, 2_000); + + let within_budget = encode_pages( + sparse_i32_list(4_096, 1_024), + TestEncoding::StructuralSparse, + HashMap::new(), + ) + .await + .unwrap(); + assert_eq!(within_budget.len(), 1); + assert!(matches!( + page_layout(&within_budget[0]), + pb21::page_layout::Layout::MiniBlockLayout(_) + )); + + let automatic = encode_pages( + array.clone(), + TestEncoding::StructuralSparse, + HashMap::new(), + ) + .await + .unwrap(); + assert_eq!(automatic.len(), 1); + assert!(matches!( + page_layout(&automatic[0]), + pb21::page_layout::Layout::SparseLayout(_) + )); + + let miniblock = encode_pages( + array.clone(), + TestEncoding::StructuralSparse, + structural_metadata(STRUCTURAL_ENCODING_MINIBLOCK), + ) + .await + .unwrap(); + assert!(miniblock.len() > 1); + assert!(miniblock.iter().all(|page| matches!( + page_layout(page), + pb21::page_layout::Layout::MiniBlockLayout(_) + ))); + + let fullzip = encode_pages( + array.clone(), + TestEncoding::StructuralSparse, + structural_metadata(STRUCTURAL_ENCODING_FULLZIP), + ) + .await + .unwrap(); + assert_eq!(fullzip.len(), miniblock.len()); + assert!(fullzip.iter().all(|page| matches!( + page_layout(page), + pb21::page_layout::Layout::FullZipLayout(_) + ))); + + let sparse = encode_pages( + array.clone(), + TestEncoding::StructuralSparse, + sparse_metadata(), + ) + .await + .unwrap(); + assert_eq!(sparse.len(), 1); + assert!(matches!( + page_layout(&sparse[0]), + pb21::page_layout::Layout::SparseLayout(_) + )); + + let v2_2_default = encode_pages(array.clone(), TestEncoding::StructuralU32, HashMap::new()) + .await + .unwrap(); + let v2_2_miniblock = encode_pages( + array.clone(), + TestEncoding::StructuralU32, + structural_metadata(STRUCTURAL_ENCODING_MINIBLOCK), + ) + .await + .unwrap(); + assert_eq!(v2_2_default.len(), v2_2_miniblock.len()); + for (default, explicit) in v2_2_default.iter().zip(v2_2_miniblock.iter()) { + assert_eq!(page_layout(default), page_layout(explicit)); + assert_eq!(default.data, explicit.data); + } + + let cases = TestCases::default() + .with_encoding(TestEncoding::StructuralSparse) + .with_page_sizes(vec![1024 * 1024]) + .with_batch_size(NUM_ROWS as u32) + .with_range(1_999..2_002) + .with_indices(vec![0, 1_999, 2_000, NUM_ROWS as u64 - 1]); + check_round_trip_encoding_of_data(vec![array], &cases, HashMap::new()).await; + } + + #[tokio::test] + async fn test_auto_sparse_for_single_row_over_budget() { + let array = unsplittable_nested_list( + Arc::new(Int32Array::from(vec![42, 43])), + Arc::new(ArrowField::new("item", DataType::Int32, true)), + ); + + let automatic = encode_pages( + array.clone(), + TestEncoding::StructuralSparse, + HashMap::new(), + ) + .await + .unwrap(); + assert_eq!(automatic.len(), 1); + assert!(matches!( + page_layout(&automatic[0]), + pb21::page_layout::Layout::SparseLayout(_) + )); + + let v2_2 = encode_pages(array.clone(), TestEncoding::StructuralU32, HashMap::new()) + .await + .unwrap(); + assert_eq!(v2_2.len(), 1); + assert!(matches!( + page_layout(&v2_2[0]), + pb21::page_layout::Layout::FullZipLayout(_) + )); + + let miniblock_error = encode_pages( + array.clone(), + TestEncoding::StructuralSparse, + structural_metadata(STRUCTURAL_ENCODING_MINIBLOCK), + ) + .await + .unwrap_err(); + assert!( + miniblock_error + .to_string() + .contains("Mini-block cannot encode 70000 rep/def levels") + ); + + let fullzip = encode_pages( + array.clone(), + TestEncoding::StructuralSparse, + structural_metadata(STRUCTURAL_ENCODING_FULLZIP), + ) + .await + .unwrap(); + assert!(matches!( + page_layout(&fullzip[0]), + pb21::page_layout::Layout::FullZipLayout(_) + )); + + let explicit_sparse = encode_pages( + array.clone(), + TestEncoding::StructuralSparse, + sparse_metadata(), + ) + .await + .unwrap(); + assert!(matches!( + page_layout(&explicit_sparse[0]), + pb21::page_layout::Layout::SparseLayout(_) + )); + + let cases = TestCases::default() + .with_encoding(TestEncoding::StructuralSparse) + .with_range(0..1) + .with_indices(vec![0]); + check_round_trip_encoding_of_data(vec![array], &cases, HashMap::new()).await; + } + + #[tokio::test] + async fn test_auto_sparse_keeps_unsupported_values_dense() { + const NUM_ROWS: usize = 70_000; + let num_values = NUM_ROWS.div_ceil(2_000); + + let (dictionary, dictionary_field) = dictionary_values(num_values); + let dictionary_array = sparse_list_values(NUM_ROWS, 2_000, dictionary, dictionary_field); + let dictionary_pages = encode_pages( + dictionary_array.clone(), + TestEncoding::StructuralSparse, + HashMap::new(), + ) + .await + .unwrap(); + assert!(dictionary_pages.len() > 1); + assert!(dictionary_pages.iter().all(|page| matches!( + page_layout(page), + pb21::page_layout::Layout::MiniBlockLayout(_) + ))); + + let (packed_values, packed_field) = variable_packed_struct_values(num_values); + let packed_array = sparse_list_values(NUM_ROWS, 2_000, packed_values, packed_field); + let packed_pages = encode_pages( + packed_array.clone(), + TestEncoding::StructuralSparse, + HashMap::new(), + ) + .await + .unwrap(); + assert!(packed_pages.len() > 1); + assert!(packed_pages.iter().all(|page| matches!( + page_layout(page), + pb21::page_layout::Layout::FullZipLayout(_) + ))); + + let (packed_values, packed_field) = variable_packed_struct_values(2); + let unsplittable_packed = unsplittable_nested_list(packed_values, packed_field); + let packed_fallback = encode_pages( + unsplittable_packed.clone(), + TestEncoding::StructuralSparse, + HashMap::new(), + ) + .await + .unwrap(); + assert_eq!(packed_fallback.len(), 1); + assert!(matches!( + page_layout(&packed_fallback[0]), + pb21::page_layout::Layout::FullZipLayout(_) + )); + + let (dictionary, dictionary_field) = dictionary_values(2); + let unsplittable_dictionary = unsplittable_nested_list(dictionary, dictionary_field); + let v2_2_error = encode_pages( + unsplittable_dictionary.clone(), + TestEncoding::StructuralU32, + HashMap::new(), + ) + .await + .unwrap_err(); + let v2_3_error = encode_pages( + unsplittable_dictionary, + TestEncoding::StructuralSparse, + HashMap::new(), + ) + .await + .unwrap_err(); + assert_eq!(v2_3_error.to_string(), v2_2_error.to_string()); + assert!( + v2_3_error + .to_string() + .contains("Mini-block cannot encode 70000 rep/def levels") + ); + + let cases = TestCases::default() + .with_encoding(TestEncoding::StructuralSparse) + .with_page_sizes(vec![1024 * 1024]) + .with_batch_size(NUM_ROWS as u32) + .with_range(1_999..2_002) + .with_indices(vec![0, 2_000, NUM_ROWS as u64 - 1]); + check_round_trip_encoding_of_data(vec![dictionary_array], &cases, HashMap::new()).await; + let packed_cases = TestCases::default() + .with_encoding(TestEncoding::StructuralSparse) + .with_page_sizes(vec![1024 * 1024]) + .with_batch_size(NUM_ROWS as u32); + check_round_trip_encoding_of_data(vec![packed_array], &packed_cases, HashMap::new()).await; + + let single_row_cases = TestCases::default() + .with_encoding(TestEncoding::StructuralSparse) + .with_page_sizes(vec![1024 * 1024]) + .with_range(0..1) + .with_indices(vec![0]); + check_round_trip_encoding_of_data( + vec![unsplittable_packed], + &single_row_cases, + HashMap::new(), + ) + .await; + } + + #[tokio::test] + async fn test_auto_sparse_wide_values_keep_dense_fallback() { + let first = vec![0xAB_u8; 5_000]; + let second = vec![0xCD_u8; 5_000]; + let fixed_size_binary = Arc::new( + FixedSizeBinaryArray::try_from_sparse_iter_with_size( + [Some(first.as_slice()), Some(second.as_slice())].into_iter(), + 5_000, + ) + .unwrap(), + ) as ArrayRef; + + const FSL_DIMENSION: i32 = 2_048; + let fixed_size_list = Arc::new( + FixedSizeListArray::try_new( + Arc::new(ArrowField::new("item", DataType::Int32, true)), + FSL_DIMENSION, + Arc::new(Int32Array::from_iter_values(0..(FSL_DIMENSION * 2))), + None, + ) + .unwrap(), + ) as ArrayRef; + + for (label, values) in [ + ("fixed-size binary", fixed_size_binary), + ("fixed-size list", fixed_size_list), + ] { + let item_field = Arc::new(ArrowField::new("item", values.data_type().clone(), true)); + let array = unsplittable_nested_list(values, item_field); + + let v2_2 = encode_pages(array.clone(), TestEncoding::StructuralU32, HashMap::new()) + .await + .unwrap(); + let v2_3 = encode_pages( + array.clone(), + TestEncoding::StructuralSparse, + HashMap::new(), + ) + .await + .unwrap(); + for pages in [&v2_2, &v2_3] { + assert_eq!(pages.len(), 1, "unexpected {label} page count"); + assert!( + matches!( + page_layout(&pages[0]), + pb21::page_layout::Layout::FullZipLayout(_) + ), + "{label} should retain the dense full-zip fallback" + ); + } + + let explicit_error = encode_pages( + array.clone(), + TestEncoding::StructuralSparse, + sparse_metadata(), + ) + .await + .unwrap_err(); + assert!( + explicit_error.to_string().contains("too wide"), + "explicit sparse should preserve the {label} value error: {explicit_error}" + ); + + let cases = TestCases::default() + .with_u32_structural_encodings() + .with_range(0..1) + .with_indices(vec![0]); + check_round_trip_encoding_of_data(vec![array], &cases, HashMap::new()).await; + } + } + + #[test] + fn test_explicit_sparse_rejects_lance_2_2() { + let array = Arc::new(Int32Array::from(vec![Some(1), None])) as ArrayRef; + let Err(error) = create_encoder(&array, TestEncoding::StructuralU32, sparse_metadata()) + else { + panic!("expected Lance 2.2 to reject explicit sparse encoding"); + }; + assert!( + error + .to_string() + .contains("not enabled by the selected file format") + ); + + let structural_only = list_i32(vec![0, 0, 0], None); + let Err(error) = create_encoder( + &structural_only, + TestEncoding::StructuralU32, + sparse_metadata(), + ) else { + panic!("expected Lance 2.2 structural-only input to reject explicit sparse encoding"); + }; + assert!( + error + .to_string() + .contains("not enabled by the selected file format") + ); + } +} diff --git a/rust/lance-encoding/src/encodings/logical/struct.rs b/rust/lance-encoding/src/encodings/logical/struct.rs index e8c6289275a..cc3e88d4468 100644 --- a/rust/lance-encoding/src/encodings/logical/struct.rs +++ b/rust/lance-encoding/src/encodings/logical/struct.rs @@ -368,26 +368,57 @@ impl StructuralDecodeArrayTask for RepDefStructDecodeTask { .map(|task| task.decode()) .collect::>>()?; let mut children = Vec::with_capacity(arrays.len()); + let mut repdefs = Vec::with_capacity(arrays.len()); let mut data_size = 0u64; let mut arrays_iter = arrays.into_iter(); - let first_array = arrays_iter.next().unwrap(); + let first_array = arrays_iter.next().ok_or_else(|| { + Error::internal("Struct decoder unexpectedly has no child arrays".to_string()) + })?; let length = first_array.array.len(); // The repdef should be identical across all children at this point - let mut repdef = first_array.repdef; + repdefs.push(first_array.repdef); data_size += first_array.data_size; children.push(first_array.array); for array in arrays_iter { - debug_assert_eq!(length, array.array.len()); + if length != array.array.len() { + return Err(Error::invalid_input_source( + format!( + "Struct child array length {} does not match sibling length {}", + array.array.len(), + length + ) + .into(), + )); + } data_size += array.data_size; children.push(array.array); + repdefs.push(array.repdef); + } + + // Dense rep/def state can retain child-specific repetition information after a child + // decoder finishes, so comparing dense siblings is not meaningful. If any child carries + // sparse state, keep a sparse child as the canonical structural plan and compare it with + // every other sparse sibling so sparse metadata is never silently discarded. + let primary_repdef = repdefs + .iter() + .position(CompositeRepDefUnraveler::has_sparse) + .unwrap_or(0); + let mut repdef = repdefs.swap_remove(primary_repdef); + if repdef.has_sparse() { + for sibling in repdefs { + if sibling.has_sparse() { + repdef.add_compatibility_check(sibling); + } + } } let validity = if self.is_root { + repdef.ensure_exhausted()?; None } else { - repdef.unravel_validity(length) + repdef.unravel_validity(length)? }; let array = StructArray::try_new(self.child_fields, children, validity) @@ -601,10 +632,7 @@ mod tests { use arrow_buffer::{BooleanBuffer, NullBuffer, OffsetBuffer, ScalarBuffer}; use arrow_schema::{DataType, Field, Fields}; - use crate::{ - testing::{TestCases, check_basic_random, check_round_trip_encoding_of_data}, - version::LanceFileVersion, - }; + use crate::testing::{TestCases, check_basic_random, check_round_trip_encoding_of_data}; #[test_log::test(tokio::test)] async fn test_simple_struct() { @@ -663,7 +691,7 @@ mod tests { Some(rows_validity), ); - let test_cases = TestCases::default().with_min_file_version(LanceFileVersion::V2_1); + let test_cases = TestCases::default().with_structural_encodings(); check_round_trip_encoding_of_data(vec![Arc::new(rows)], &test_cases, HashMap::new()).await; } @@ -693,7 +721,7 @@ mod tests { ); check_round_trip_encoding_of_data( vec![Arc::new(struct_array)], - &TestCases::default().with_min_file_version(LanceFileVersion::V2_1), + &TestCases::default().with_structural_encodings(), HashMap::new(), ) .await; @@ -724,7 +752,7 @@ mod tests { ); check_round_trip_encoding_of_data( vec![Arc::new(struct_array)], - &TestCases::default().with_min_file_version(LanceFileVersion::V2_1), + &TestCases::default().with_structural_encodings(), HashMap::new(), ) .await; @@ -803,7 +831,7 @@ mod tests { check_round_trip_encoding_of_data( vec![Arc::new(list_array)], - &TestCases::default().with_min_file_version(LanceFileVersion::V2_2), + &TestCases::default().with_u32_structural_encodings(), HashMap::new(), ) .await; @@ -837,7 +865,7 @@ mod tests { .with_range(1..2) .with_indices(vec![0]) .with_indices(vec![1]) - .with_min_file_version(LanceFileVersion::V2_1), + .with_structural_encodings(), HashMap::new(), ) .await; @@ -885,7 +913,7 @@ mod tests { check_round_trip_encoding_of_data( vec![Arc::new(row_array)], - &TestCases::default().with_min_file_version(LanceFileVersion::V2_1), + &TestCases::default().with_structural_encodings(), HashMap::new(), ) .await; diff --git a/rust/lance-encoding/src/encodings/physical/binary.rs b/rust/lance-encoding/src/encodings/physical/binary.rs index d02cf2da693..06df60434e9 100644 --- a/rust/lance-encoding/src/encodings/physical/binary.rs +++ b/rust/lance-encoding/src/encodings/physical/binary.rs @@ -21,7 +21,8 @@ use crate::buffer::LanceBuffer; use crate::data::{BlockInfo, DataBlock, VariableWidthBlock}; use crate::encodings::logical::primitive::fullzip::{PerValueCompressor, PerValueDataBlock}; use crate::encodings::logical::primitive::miniblock::{ - MAX_MINIBLOCK_VALUES, MiniBlockChunk, MiniBlockCompressed, MiniBlockCompressor, + MAX_MINIBLOCK_VALUES, MiniBlockChunk, MiniBlockCompressed, MiniBlockCompressionContext, + MiniBlockCompressor, }; use crate::format::pb21::CompressiveEncoding; use crate::format::pb21::compressive_encoding::Compression; @@ -245,7 +246,11 @@ impl BinaryMiniBlockEncoder { } impl MiniBlockCompressor for BinaryMiniBlockEncoder { - fn compress(&self, data: DataBlock) -> Result<(MiniBlockCompressed, CompressiveEncoding)> { + fn compress( + &self, + _context: MiniBlockCompressionContext, + data: DataBlock, + ) -> Result<(MiniBlockCompressed, CompressiveEncoding)> { match data { DataBlock::VariableWidth(variable_width) => Ok(self.chunk_data(variable_width)), _ => Err(Error::invalid_input_source( @@ -288,52 +293,144 @@ impl BinaryMiniBlockDecompressor { } } +/// Cold path: pinpoint why the chunk-relative offsets of a binary mini-block +/// chunk failed validation. +fn chunk_offset_violation_error>(offsets: &[T], chunk_len: usize) -> Error { + let mut previous: u64 = offsets[0].into(); + for (position, &offset) in offsets.iter().enumerate().skip(1) { + let offset: u64 = offset.into(); + if offset < previous { + return Error::corrupt_file_named( + "binary mini-block", + format!( + "value offset at position {position} decreases: {offset} < {previous} \ + (chunk is {chunk_len} bytes)" + ), + ); + } + previous = offset; + } + Error::corrupt_file_named( + "binary mini-block", + format!("value offset {previous} is out of bounds for a chunk of {chunk_len} bytes"), + ) +} + impl MiniBlockDecompressor for BinaryMiniBlockDecompressor { // decompress a MiniBlock of binary data, the num_values must be less than or equal // to the number of values this MiniBlock has, BinaryMiniBlock doesn't store `the number of values` // it has so assertion can not be done here and the caller of `decompress` must ensure // `num_values` <= number of values in the chunk. + // + // The chunk-relative value offsets at the front of the chunk come straight + // from the file and are used to slice the chunk buffer, so corrupt values + // must surface as a typed error instead of a panic or an out-of-bounds + // read. The monotonicity check rides along the existing rebase loop (the + // `&=` accumulation keeps it branchless) so validation adds no extra pass. fn decompress(&self, data: Vec, num_values: u64) -> Result { assert_eq!(data.len(), 1); let data = data.into_iter().next().unwrap(); - if self.bits_per_offset == 64 { - // offset and at least one value - assert!(data.len() >= 16); + let bytes_per_offset = self.bits_per_offset as usize / 8; + if !data.len().is_multiple_of(bytes_per_offset) { + return Err(Error::corrupt_file_named( + "binary mini-block", + format!( + "chunk size {} is not a multiple of the {}-byte offset width", + data.len(), + bytes_per_offset + ), + )); + } + let num_offsets = (num_values as usize).checked_add(1).ok_or_else(|| { + Error::corrupt_file_named( + "binary mini-block", + format!("cannot decode {num_values} values from a single chunk"), + ) + })?; + if data.len() / bytes_per_offset < num_offsets { + return Err(Error::corrupt_file_named( + "binary mini-block", + format!( + "chunk of {} bytes holds {} offsets but decoding {} values requires {}", + data.len(), + data.len() / bytes_per_offset, + num_values, + num_offsets + ), + )); + } + // The value region must start past the offsets being decoded, otherwise + // the offset table itself aliases into the value bytes. A lower bound + // (not equality) because a prefix read of the chunk legitimately leaves + // unrequested offsets between the requested prefix and the values. + let min_value_region_start = num_offsets * bytes_per_offset; + let value_region_overlap_error = |first: u64| { + Error::corrupt_file_named( + "binary mini-block", + format!( + "value region starts at offset {first} which overlaps the {num_offsets} \ + requested offsets ({min_value_region_start} bytes)" + ), + ) + }; + + if self.bits_per_offset == 64 { let offsets_buffer = data.borrow_to_typed_slice::(); - let offsets = offsets_buffer.as_ref(); + let offsets = &offsets_buffer.as_ref()[..num_offsets]; - let result_offsets = offsets[0..(num_values + 1) as usize] + let first = offsets[0]; + if first < min_value_region_start as u64 { + return Err(value_region_overlap_error(first)); + } + let mut previous = first; + let mut is_monotonic = true; + let result_offsets = offsets .iter() - .map(|offset| offset - offsets[0]) + .map(|&offset| { + is_monotonic &= previous <= offset; + previous = offset; + offset.wrapping_sub(first) + }) .collect::>(); + let last = offsets[num_offsets - 1]; + if !is_monotonic || last as usize > data.len() { + return Err(chunk_offset_violation_error(offsets, data.len())); + } Ok(DataBlock::VariableWidth(VariableWidthBlock { - data: LanceBuffer::from( - data[offsets[0] as usize..offsets[num_values as usize] as usize].to_vec(), - ), + data: LanceBuffer::from(data[first as usize..last as usize].to_vec()), offsets: LanceBuffer::reinterpret_vec(result_offsets), bits_per_offset: 64, num_values, block_info: BlockInfo::new(), })) } else { - // offset and at least one value - assert!(data.len() >= 8); - let offsets_buffer = data.borrow_to_typed_slice::(); - let offsets = offsets_buffer.as_ref(); + let offsets = &offsets_buffer.as_ref()[..num_offsets]; - let result_offsets = offsets[0..(num_values + 1) as usize] + let first = offsets[0]; + if (first as u64) < min_value_region_start as u64 { + return Err(value_region_overlap_error(first as u64)); + } + let mut previous = first; + let mut is_monotonic = true; + let result_offsets = offsets .iter() - .map(|offset| offset - offsets[0]) + .map(|&offset| { + is_monotonic &= previous <= offset; + previous = offset; + offset.wrapping_sub(first) + }) .collect::>(); + let last = offsets[num_offsets - 1]; + if !is_monotonic || last as usize > data.len() { + return Err(chunk_offset_violation_error(offsets, data.len())); + } Ok(DataBlock::VariableWidth(VariableWidthBlock { - data: LanceBuffer::from( - data[offsets[0] as usize..offsets[num_values as usize] as usize].to_vec(), - ), + data: LanceBuffer::from(data[first as usize..last as usize].to_vec()), offsets: LanceBuffer::reinterpret_vec(result_offsets), bits_per_offset: 32, num_values, @@ -462,18 +559,48 @@ impl BlockDecompressor for BinaryBlockDecompressor { // never be more than 255 and it's little endian so the last 3 bytes will always be 0. These will be the least // significant 3 bytes of the number of values in the old scheme. It's pretty unlikely these are all 0 (that would // mean there are at least 16M values in a single page) so we'll use this to determine if the old scheme is used. + // + // The header fields and the offsets themselves come straight from the file. + // The structural checks below (all O(1)) reject blocks whose regions do not + // line up; the offset *values* are validated later, by the mandatory layout + // validation in `VariableWidthBlock::into_arrow`, so they are not rescanned + // here. + if data.len() < 4 { + return Err(Error::corrupt_file_named( + "variable-width block", + format!( + "block of {} bytes is too small to hold a header", + data.len() + ), + )); + } let is_old_scheme = data[1] != 0 || data[2] != 0 || data[3] != 0; + let ensure_header = |header_len: usize| { + if data.len() < header_len { + return Err(Error::corrupt_file_named( + "variable-width block", + format!( + "block of {} bytes is too small for a {} byte header", + data.len(), + header_len + ), + )); + } + Ok(()) + }; let (bits_per_offset, bytes_start_offset, offset_start) = if is_old_scheme { // Old scheme let bits_per_offset = data[0]; match bits_per_offset { 32 => { + ensure_header(9)?; debug_assert_eq!(LittleEndian::read_u32(&data[1..5]), num_values as u32); let bytes_start_offset = LittleEndian::read_u32(&data[5..9]); - (bits_per_offset, bytes_start_offset as u64, 9) + (bits_per_offset, bytes_start_offset as u64, 9_u64) } 64 => { + ensure_header(17)?; debug_assert_eq!(LittleEndian::read_u64(&data[1..9]), num_values); let bytes_start_offset = LittleEndian::read_u64(&data[9..17]); (bits_per_offset, bytes_start_offset, 17) @@ -489,10 +616,12 @@ impl BlockDecompressor for BinaryBlockDecompressor { let bits_per_offset = LittleEndian::read_u32(&data[0..4]) as u8; match bits_per_offset { 32 => { + ensure_header(8)?; let bytes_start_offset = LittleEndian::read_u32(&data[4..8]); (bits_per_offset, bytes_start_offset as u64, 8) } 64 => { + ensure_header(16)?; let bytes_start_offset = LittleEndian::read_u64(&data[8..16]); (bits_per_offset, bytes_start_offset, 16) } @@ -504,9 +633,55 @@ impl BlockDecompressor for BinaryBlockDecompressor { } }; + // The offsets region sits between the header and `bytes_start_offset` + // and must hold exactly `num_values + 1` offsets starting at zero. + let expected_offsets_bytes = num_values + .checked_add(1) + .and_then(|num_offsets| num_offsets.checked_mul(bits_per_offset as u64 / 8)) + .ok_or_else(|| { + Error::corrupt_file_named( + "variable-width block", + format!("offsets region size overflows for {num_values} values"), + ) + })?; + if bytes_start_offset < offset_start || bytes_start_offset > data.len() as u64 { + return Err(Error::corrupt_file_named( + "variable-width block", + format!( + "bytes start offset {} is outside the block (header: {} bytes, block: {} bytes)", + bytes_start_offset, + offset_start, + data.len() + ), + )); + } + if bytes_start_offset - offset_start != expected_offsets_bytes { + return Err(Error::corrupt_file_named( + "variable-width block", + format!( + "expected {} offset bytes for {} values but found {}", + expected_offsets_bytes, + num_values, + bytes_start_offset - offset_start + ), + )); + } + // the next `bytes_start_offset - offset_start` stores the offsets. - let offsets = - data.slice_with_length(offset_start, bytes_start_offset as usize - offset_start); + let offsets = data.slice_with_length( + offset_start as usize, + (bytes_start_offset - offset_start) as usize, + ); + let first_offset = match bits_per_offset { + 32 => LittleEndian::read_u32(&offsets[0..4]) as u64, + _ => LittleEndian::read_u64(&offsets[0..8]), + }; + if first_offset != 0 { + return Err(Error::corrupt_file_named( + "variable-width block", + format!("first offset must be 0 but found {first_offset}"), + )); + } // the rest are the binary bytes. let data = data.slice_with_length( @@ -533,31 +708,25 @@ mod tests { use arrow_schema::{DataType, Field}; use crate::{ + buffer::LanceBuffer, constants::{ COMPRESSION_META_KEY, STRUCTURAL_ENCODING_FULLZIP, STRUCTURAL_ENCODING_META_KEY, STRUCTURAL_ENCODING_MINIBLOCK, }, + data::{BlockInfo, DataBlock, VariableWidthBlock}, testing::check_specific_random, }; use rstest::rstest; use std::{collections::HashMap, sync::Arc, vec}; - use crate::{ - testing::{ - FnArrayGeneratorProvider, TestCases, check_basic_random, - check_round_trip_encoding_of_data, - }, - version::LanceFileVersion, + use crate::testing::{ + FnArrayGeneratorProvider, TestCases, check_basic_random, check_round_trip_encoding_of_data, }; #[test_log::test(tokio::test)] async fn test_utf8_binary() { let field = Field::new("", DataType::Utf8, false); - check_specific_random( - field, - TestCases::basic().with_min_file_version(LanceFileVersion::V2_1), - ) - .await; + check_specific_random(field, TestCases::basic().with_structural_encodings()).await; } #[rstest] @@ -592,7 +761,7 @@ mod tests { field_metadata.insert(COMPRESSION_META_KEY.to_string(), "fsst".into()); let field = Field::new("", data_type, true).with_metadata(field_metadata); // TODO (https://github.com/lance-format/lance/issues/4783) - let test_cases = TestCases::default().with_min_file_version(LanceFileVersion::V2_1); + let test_cases = TestCases::default().with_structural_encodings(); check_specific_random(field, test_cases).await; } @@ -610,11 +779,7 @@ mod tests { ); field_metadata.insert(COMPRESSION_META_KEY.to_string(), "fsst".into()); let field = Field::new("", data_type, true).with_metadata(field_metadata); - check_specific_random( - field, - TestCases::basic().with_min_file_version(LanceFileVersion::V2_1), - ) - .await; + check_specific_random(field, TestCases::basic().with_structural_encodings()).await; } #[test_log::test(tokio::test)] @@ -801,7 +966,7 @@ mod tests { #[values(true, false)] with_nulls: bool, #[values(100, 500, 35000)] dict_size: u32, ) { - let test_cases = TestCases::default().with_min_file_version(LanceFileVersion::V2_1); + let test_cases = TestCases::default().with_structural_encodings(); let strings = (0..dict_size) .map(|i| i.to_string()) .collect::>(); @@ -829,7 +994,7 @@ mod tests { let test_cases = TestCases::default() .with_expected_encoding("variable") - .with_min_file_version(LanceFileVersion::V2_1); + .with_structural_encodings(); // Test both automatic selection and explicit configuration // 1. Test automatic binary encoding selection (small strings that won't trigger FSST) @@ -968,4 +1133,193 @@ mod tests { } } } + + #[test] + fn test_binary_miniblock_rejects_corrupt_offsets() { + use super::BinaryMiniBlockDecompressor; + use crate::compression::MiniBlockDecompressor; + use lance_core::Error; + + // Chunk layout mirrors the on-disk format for ["alpha", "beta", "gamma"]: + // LE u32 offsets [16, 21, 25, 30] followed by the value bytes, padded to + // a multiple of 8 bytes. + fn chunk_u32(offsets: &[u32], values: &[u8]) -> LanceBuffer { + let mut chunk = offsets + .iter() + .flat_map(|offset| offset.to_le_bytes()) + .collect::>(); + chunk.extend_from_slice(values); + chunk.resize(chunk.len().next_multiple_of(8), 0); + LanceBuffer::from(chunk) + } + + let decompressor = BinaryMiniBlockDecompressor::new(32); + + // The tail offset points past the end of the 32-byte chunk. + let err = decompressor + .decompress( + vec![chunk_u32(&[16, 21, 25, 100_000], b"alphabetagamma")], + 3, + ) + .unwrap_err(); + assert!(matches!(err, Error::CorruptFile { .. }), "{err:?}"); + assert!(err.to_string().contains("out of bounds"), "{err}"); + + // Offsets go backwards, which would underflow the rebase subtraction. + let err = decompressor + .decompress(vec![chunk_u32(&[16, 25, 21, 30], b"alphabetagamma")], 3) + .unwrap_err(); + assert!(matches!(err, Error::CorruptFile { .. }), "{err:?}"); + assert!(err.to_string().contains("decreases"), "{err}"); + + // The first offset points inside the offset table, which would alias + // the serialized offsets into the value bytes. + let err = decompressor + .decompress(vec![chunk_u32(&[0, 21, 25, 30], b"alphabetagamma")], 3) + .unwrap_err(); + assert!(matches!(err, Error::CorruptFile { .. }), "{err:?}"); + assert!(err.to_string().contains("overlaps"), "{err}"); + + // The chunk stores fewer offsets than the requested value count needs. + let err = decompressor + .decompress(vec![chunk_u32(&[8, 8], &[])], 3) + .unwrap_err(); + assert!(matches!(err, Error::CorruptFile { .. }), "{err:?}"); + assert!(err.to_string().contains("requires 4"), "{err}"); + + // The chunk size is not a multiple of the offset width. + let err = decompressor + .decompress(vec![LanceBuffer::from(vec![0u8; 10])], 1) + .unwrap_err(); + assert!(matches!(err, Error::CorruptFile { .. }), "{err:?}"); + assert!(err.to_string().contains("multiple"), "{err}"); + + // 64-bit offsets take the same validation path. + fn chunk_u64(offsets: &[u64], values: &[u8]) -> LanceBuffer { + let mut chunk = offsets + .iter() + .flat_map(|offset| offset.to_le_bytes()) + .collect::>(); + chunk.extend_from_slice(values); + chunk.resize(chunk.len().next_multiple_of(8), 0); + LanceBuffer::from(chunk) + } + let decompressor = BinaryMiniBlockDecompressor::new(64); + let err = decompressor + .decompress( + vec![chunk_u64(&[32, 37, 41, 100_000], b"alphabetagamma")], + 3, + ) + .unwrap_err(); + assert!(matches!(err, Error::CorruptFile { .. }), "{err:?}"); + assert!(err.to_string().contains("out of bounds"), "{err}"); + let err = decompressor + .decompress(vec![chunk_u64(&[0, 37, 41, 46], b"alphabetagamma")], 3) + .unwrap_err(); + assert!(matches!(err, Error::CorruptFile { .. }), "{err:?}"); + assert!(err.to_string().contains("overlaps"), "{err}"); + + // A valid chunk still decodes: offsets rebase to [0, 5, 9, 14]. + let decompressor = BinaryMiniBlockDecompressor::new(32); + let block = decompressor + .decompress(vec![chunk_u32(&[16, 21, 25, 30], b"alphabetagamma")], 3) + .unwrap(); + let DataBlock::VariableWidth(block) = block else { + panic!("expected a variable-width block"); + }; + assert_eq!(block.data.as_ref(), b"alphabetagamma"); + assert_eq!( + block.offsets, + LanceBuffer::reinterpret_vec(vec![0_u32, 5, 9, 14]) + ); + } + + fn encoded_binary_block(bits_per_offset: u8) -> Vec { + use crate::compression::BlockCompressor; + + let offsets = match bits_per_offset { + 32 => LanceBuffer::reinterpret_vec(vec![0_i32, 5, 9, 14]), + 64 => LanceBuffer::reinterpret_vec(vec![0_i64, 5, 9, 14]), + _ => unreachable!(), + }; + let block = DataBlock::VariableWidth(VariableWidthBlock { + data: LanceBuffer::copy_slice(b"alphabetagamma"), + offsets, + bits_per_offset, + num_values: 3, + block_info: BlockInfo::new(), + }); + BlockCompressor::compress(&super::VariableEncoder::default(), block) + .unwrap() + .as_ref() + .to_vec() + } + + /// The block decompressor only checks the block structure (all O(1)); bad + /// offset values inside a structurally-sound block are rejected by the + /// mandatory layout validation when the block is converted to Arrow. + #[rstest] + #[case::i32_tail_out_of_bounds(32, 3, 100_000, "out of bounds")] + #[case::i64_tail_out_of_bounds(64, 3, 15, "out of bounds")] + #[case::i32_non_monotonic(32, 2, 4, "non-monotonic")] + #[case::i64_non_monotonic(64, 2, 4, "non-monotonic")] + fn test_binary_block_bad_offsets_rejected_at_arrow_conversion( + #[case] bits_per_offset: u8, + #[case] mutated_offset_index: usize, + #[case] mutated_offset_value: u64, + #[case] expected_message: &str, + ) { + use crate::compression::BlockDecompressor; + use lance_core::Error; + + let mut encoded = encoded_binary_block(bits_per_offset); + let bytes_per_offset = (bits_per_offset / 8) as usize; + // The standard scheme header is two offset-width fields. + let mutated_offset_start = bytes_per_offset * (2 + mutated_offset_index); + encoded[mutated_offset_start..mutated_offset_start + bytes_per_offset] + .copy_from_slice(&mutated_offset_value.to_le_bytes()[..bytes_per_offset]); + + let block = super::BinaryBlockDecompressor::default() + .decompress(LanceBuffer::from(encoded), 3) + .unwrap(); + let data_type = match bits_per_offset { + 32 => DataType::Binary, + _ => DataType::LargeBinary, + }; + let err = block.into_arrow(data_type, false).unwrap_err(); + assert!(matches!(err, Error::CorruptFile { .. }), "{err:?}"); + assert!(err.to_string().contains(expected_message), "{err}"); + } + + #[test] + fn test_binary_block_rejects_corrupt_structure() { + use crate::compression::BlockDecompressor; + use lance_core::Error; + + let decompressor = super::BinaryBlockDecompressor::default(); + + // The first offset must be zero. + let mut encoded = encoded_binary_block(32); + encoded[8..12].copy_from_slice(&5_u32.to_le_bytes()); + let err = decompressor + .decompress(LanceBuffer::from(encoded), 3) + .unwrap_err(); + assert!(matches!(err, Error::CorruptFile { .. }), "{err:?}"); + assert!(err.to_string().contains("first offset"), "{err}"); + + // The offsets region must hold exactly num_values + 1 offsets. + let encoded = encoded_binary_block(32); + let err = decompressor + .decompress(LanceBuffer::from(encoded), 4) + .unwrap_err(); + assert!(matches!(err, Error::CorruptFile { .. }), "{err:?}"); + assert!(err.to_string().contains("offset bytes"), "{err}"); + + // A block too small to hold its header is rejected, not a panic. + let err = decompressor + .decompress(LanceBuffer::from(vec![0_u8; 2]), 1) + .unwrap_err(); + assert!(matches!(err, Error::CorruptFile { .. }), "{err:?}"); + assert!(err.to_string().contains("too small"), "{err}"); + } } diff --git a/rust/lance-encoding/src/encodings/physical/bitpacking.rs b/rust/lance-encoding/src/encodings/physical/bitpacking.rs index 8ebdcc13c56..9b775911005 100644 --- a/rust/lance-encoding/src/encodings/physical/bitpacking.rs +++ b/rust/lance-encoding/src/encodings/physical/bitpacking.rs @@ -18,7 +18,6 @@ use arrow_array::types::UInt64Type; use arrow_array::{Array, PrimitiveArray}; use arrow_buffer::ArrowNativeType; -use byteorder::{ByteOrder, LittleEndian}; use lance_bitpacking::BitPacking; use lance_core::{Error, Result}; @@ -28,15 +27,16 @@ use crate::compression::{BlockCompressor, BlockDecompressor, MiniBlockDecompress use crate::data::BlockInfo; use crate::data::{DataBlock, FixedWidthDataBlock}; use crate::encodings::logical::primitive::miniblock::{ - MiniBlockChunk, MiniBlockCompressed, MiniBlockCompressor, + MiniBlockChunk, MiniBlockCompressed, MiniBlockCompressionContext, MiniBlockCompressor, }; use crate::format::pb21::CompressiveEncoding; use crate::format::{ProtobufUtils21, pb21}; use crate::statistics::{GetStat, Stat}; -use bytemuck::{AnyBitPattern, cast_slice}; +use bytemuck::Pod; -const LOG_ELEMS_PER_CHUNK: u8 = 10; -const ELEMS_PER_CHUNK: u64 = 1 << LOG_ELEMS_PER_CHUNK; +pub(crate) const LOG_ELEMS_PER_CHUNK: u8 = 10; +/// Number of values encoded in each inline bitpacking chunk. +pub const ELEMS_PER_CHUNK: u64 = 1 << LOG_ELEMS_PER_CHUNK; #[derive(Debug, Default)] pub struct InlineBitpacking { @@ -181,27 +181,85 @@ impl InlineBitpacking { ) } - fn unchunk( + fn unchunk( data: LanceBuffer, num_values: u64, ) -> Result { - // Ensure at least the header is present - assert!(data.len() >= std::mem::size_of::()); - assert!(num_values <= ELEMS_PER_CHUNK); - // This macro decompresses a chunk(1024 values) of bitpacked values. let uncompressed_bit_width = std::mem::size_of::() * 8; - let mut decompressed = vec![T::from_usize(0).unwrap(); ELEMS_PER_CHUNK as usize]; - - // Copy for memory alignment - let chunk_in_u8: Vec = data.to_vec(); - let bit_width_bytes = &chunk_in_u8[..std::mem::size_of::()]; - let bit_width_value = LittleEndian::read_uint(bit_width_bytes, std::mem::size_of::()); - let chunk = cast_slice(&chunk_in_u8[std::mem::size_of::()..]); - // The bit-packed chunk should have number of bytes (bit_width_value * ELEMS_PER_CHUNK / 8) - assert!(std::mem::size_of_val(chunk) == (bit_width_value * ELEMS_PER_CHUNK) as usize / 8); + let word_size = std::mem::size_of::(); + + if data.len() < word_size { + return Err(Error::corrupt_file_named( + "inline_bitpacking", + format!( + "Inline bitpacking chunk is too small for {}-byte header: {} bytes", + word_size, + data.len() + ), + )); + } + if !data.len().is_multiple_of(word_size) { + return Err(Error::corrupt_file_named( + "inline_bitpacking", + format!( + "Inline bitpacking chunk size must be a multiple of {} bytes, got {} bytes", + word_size, + data.len() + ), + )); + } + if num_values > ELEMS_PER_CHUNK { + return Err(Error::corrupt_file_named( + "inline_bitpacking", + format!( + "Inline bitpacking chunk has {} values, expected at most {}", + num_values, ELEMS_PER_CHUNK + ), + )); + } + + let chunk_words = data.borrow_to_typed_view::(); + let bit_width_value = chunk_words[0].as_usize(); + if bit_width_value > uncompressed_bit_width { + return Err(Error::corrupt_file_named( + "inline_bitpacking", + format!( + "Inline bitpacking width {} exceeds {}-bit values", + bit_width_value, uncompressed_bit_width + ), + )); + } + let chunk = &chunk_words[1..]; + // bit_width_value has already been verified to be <= uncompressed_bit_width + // (8/16/32/64), so bit_width_value * ELEMS_PER_CHUNK (1024) can never + // overflow usize on supported targets. Keep checked_mul as defense in depth. + let expected_num_bits = bit_width_value + .checked_mul(ELEMS_PER_CHUNK as usize) + .ok_or_else(|| { + Error::corrupt_file_named( + "inline_bitpacking", + format!( + "Inline bitpacking width {} overflows chunk bit count", + bit_width_value + ), + ) + })?; + let expected_num_bytes = expected_num_bits / 8; + let actual_num_bytes = std::mem::size_of_val(chunk); + if actual_num_bytes != expected_num_bytes { + return Err(Error::corrupt_file_named( + "inline_bitpacking", + format!( + "Inline bitpacking payload has {} bytes, expected {} bytes for bit width {}", + actual_num_bytes, expected_num_bytes, bit_width_value + ), + )); + } + + let mut decompressed = vec![T::default(); ELEMS_PER_CHUNK as usize]; unsafe { - BitPacking::unchecked_unpack(bit_width_value as usize, chunk, &mut decompressed); + BitPacking::unchecked_unpack(bit_width_value, chunk, &mut decompressed); } decompressed.truncate(num_values as usize); @@ -212,10 +270,25 @@ impl InlineBitpacking { block_info: BlockInfo::new(), })) } + + /// An empty fixed-width block, used for the `num_values == 0` short-circuit in + /// both decompressor entry points so empty blocks skip chunk validation entirely. + fn empty_block(&self) -> DataBlock { + DataBlock::FixedWidth(FixedWidthDataBlock { + data: LanceBuffer::empty(), + bits_per_value: self.uncompressed_bit_width, + num_values: 0, + block_info: BlockInfo::new(), + }) + } } impl MiniBlockCompressor for InlineBitpacking { - fn compress(&self, chunk: DataBlock) -> Result<(MiniBlockCompressed, CompressiveEncoding)> { + fn compress( + &self, + _context: MiniBlockCompressionContext, + chunk: DataBlock, + ) -> Result<(MiniBlockCompressed, CompressiveEncoding)> { match chunk { DataBlock::FixedWidth(fixed_width) => Ok(self.chunk_data(fixed_width)), _ => Err(Error::invalid_input_source( @@ -241,6 +314,10 @@ impl MiniBlockDecompressor for InlineBitpacking { fn decompress(&self, data: Vec, num_values: u64) -> Result { assert_eq!(data.len(), 1); let data = data.into_iter().next().unwrap(); + if num_values == 0 { + // Empty mini-blocks have no inline bit-width header to decode. + return Ok(self.empty_block()); + } match self.uncompressed_bit_width { 8 => Self::unchunk::(data, num_values), 16 => Self::unchunk::(data, num_values), @@ -249,10 +326,22 @@ impl MiniBlockDecompressor for InlineBitpacking { _ => unimplemented!("Bitpacking word size must be 8, 16, 32, or 64"), } } + + fn decoded_size_bytes(&self, num_values: u64) -> Option { + num_values + .checked_mul(self.uncompressed_bit_width) + .map(|bits| bits.div_ceil(8)) + } } impl BlockDecompressor for InlineBitpacking { fn decompress(&self, data: LanceBuffer, num_values: u64) -> Result { + if num_values == 0 { + // Empty blocks carry no inline bit-width header to decode; avoid + // spurious "too small for header" corrupt-file errors and mirror + // the MiniBlockDecompressor path. See #7794. + return Ok(self.empty_block()); + } match self.uncompressed_bit_width { 8 => Self::unchunk::(data, num_values), 16 => Self::unchunk::(data, num_values), @@ -527,19 +616,134 @@ mod test { use std::{collections::HashMap, sync::Arc}; use arrow_array::{Array, Int8Array, Int64Array}; + use arrow_buffer::ArrowNativeType; use arrow_schema::DataType; + use bytemuck::Pod; + use lance_bitpacking::BitPacking; + use rstest::rstest; - use super::{ELEMS_PER_CHUNK, bitpack_out_of_line, unpack_out_of_line}; + use super::{ELEMS_PER_CHUNK, InlineBitpacking, bitpack_out_of_line, unpack_out_of_line}; use crate::{ buffer::LanceBuffer, - data::{BlockInfo, FixedWidthDataBlock}, + compression::{BlockDecompressor, MiniBlockDecompressor}, + data::{BlockInfo, DataBlock, FixedWidthDataBlock}, testing::{TestCases, check_round_trip_encoding_of_data}, - version::LanceFileVersion, }; + #[rstest] + #[case::u8(8)] + #[case::u16(16)] + #[case::u32(32)] + #[case::u64(64)] + fn test_inline_bitpacking_decompress_empty_miniblock(#[case] bit_width: u64) { + let decompressor = InlineBitpacking::new(bit_width); + let decompressed = + MiniBlockDecompressor::decompress(&decompressor, vec![LanceBuffer::empty()], 0) + .unwrap(); + + let DataBlock::FixedWidth(block) = decompressed else { + panic!("Expected FixedWidth block"); + }; + assert_eq!(block.bits_per_value, bit_width); + assert_eq!(block.num_values, 0); + assert_eq!(block.data.len(), 0); + } + + // Regression test for #7794: the block-level decompressor must short-circuit + // on num_values == 0 the same way the mini-block decompressor does, instead + // of reporting a spurious "too small for header" corrupt-file error. + #[rstest] + #[case::u8(8)] + #[case::u16(16)] + #[case::u32(32)] + #[case::u64(64)] + fn test_inline_bitpacking_decompress_empty_block(#[case] bit_width: u64) { + let decompressor = InlineBitpacking::new(bit_width); + let decompressed = + BlockDecompressor::decompress(&decompressor, LanceBuffer::empty(), 0).unwrap(); + + let DataBlock::FixedWidth(block) = decompressed else { + panic!("Expected FixedWidth block"); + }; + assert_eq!(block.bits_per_value, bit_width); + assert_eq!(block.num_values, 0); + assert_eq!(block.data.len(), 0); + } + + fn roundtrip_unchunk(values: &[T], bit_width: usize) + where + T: ArrowNativeType + BitPacking + Pod, + { + assert!(values.len() <= ELEMS_PER_CHUNK as usize); + let num_values = values.len() as u64; + + let mut padded = vec![T::from_usize(0).unwrap(); ELEMS_PER_CHUNK as usize]; + padded[..values.len()].copy_from_slice(values); + + let packed_words = ELEMS_PER_CHUNK as usize * bit_width / (std::mem::size_of::() * 8); + let mut chunk: Vec = Vec::with_capacity(1 + packed_words); + chunk.push(T::from_usize(bit_width).unwrap()); + let out_len = chunk.len(); + chunk.resize(out_len + packed_words, T::from_usize(0).unwrap()); + unsafe { + BitPacking::unchecked_pack(bit_width, &padded, &mut chunk[out_len..]); + } + + let data = LanceBuffer::reinterpret_vec(chunk); + let decoded = InlineBitpacking::unchunk::(data, num_values).unwrap(); + let DataBlock::FixedWidth(fixed) = decoded else { + panic!("expected FixedWidth DataBlock"); + }; + let decoded_values = fixed.data.borrow_to_typed_view::(); + assert_eq!(decoded_values.as_ref(), values); + } + + fn assert_corrupt_unchunk(data: LanceBuffer, num_values: u64, expected_message: &str) + where + T: ArrowNativeType + BitPacking + Pod, + { + let err = InlineBitpacking::unchunk::(data, num_values).unwrap_err(); + assert!(matches!(err, lance_core::Error::CorruptFile { .. })); + let err = err.to_string(); + assert!( + err.contains(expected_message), + "expected error containing {expected_message:?}, got {err:?}" + ); + } + + #[test] + fn unchunk_u32_bw12_tail() { + let values: Vec = (0..500).map(|i| ((i * 7) % (1 << 12)) as u32).collect(); + roundtrip_unchunk(&values, 12); + } + + #[test] + fn unchunk_u64_bw23_full() { + let values: Vec = (0..1024).map(|i| ((i * 3) % (1 << 23)) as u64).collect(); + roundtrip_unchunk(&values, 23); + } + + #[rstest] + #[case::too_small_header(LanceBuffer::from(vec![1, 2, 3]), 1, "too small")] + #[case::misaligned_chunk_size(LanceBuffer::from(vec![0, 0, 0, 0, 0]), 1, "multiple")] + #[case::too_many_values( + LanceBuffer::reinterpret_vec(vec![0_u32]), + ELEMS_PER_CHUNK + 1, + "expected at most" + )] + #[case::payload_size_mismatch(LanceBuffer::reinterpret_vec(vec![12_u32]), 1, "payload")] + #[case::invalid_bit_width(LanceBuffer::reinterpret_vec(vec![33_u32]), 1, "exceeds")] + fn unchunk_rejects( + #[case] data: LanceBuffer, + #[case] num_values: u64, + #[case] expected_message: &str, + ) { + assert_corrupt_unchunk::(data, num_values, expected_message); + } + #[test_log::test(tokio::test)] async fn test_miniblock_bitpack() { - let test_cases = TestCases::default().with_min_file_version(LanceFileVersion::V2_1); + let test_cases = TestCases::default().with_structural_encodings(); let arrays = vec![ Arc::new(Int8Array::from(vec![100; 1024])) as Arc, @@ -578,7 +782,7 @@ mod test { // Test bitpacking encoding verification with varied small values that should trigger bitpacking let test_cases = TestCases::default() .with_expected_encoding("inline_bitpacking") - .with_min_file_version(LanceFileVersion::V2_1); + .with_structural_encodings(); // Generate data with varied small values to avoid RLE // Mix different values but keep them small to trigger bitpacking @@ -602,7 +806,7 @@ mod test { let test_cases = TestCases::default() .with_expected_encoding("inline_bitpacking") - .with_min_file_version(LanceFileVersion::V2_1); + .with_structural_encodings(); // Build 2048 values: first 1024 all zeros (bit_width=0), // next 1024 small varied values to avoid RLE and trigger bitpacking. diff --git a/rust/lance-encoding/src/encodings/physical/block.rs b/rust/lance-encoding/src/encodings/physical/block.rs index a1f5bdb3fdd..188af50c8a0 100644 --- a/rust/lance-encoding/src/encodings/physical/block.rs +++ b/rust/lance-encoding/src/encodings/physical/block.rs @@ -46,9 +46,20 @@ pub struct CompressionConfig { } impl CompressionConfig { - pub(crate) fn new(scheme: CompressionScheme, level: Option) -> Self { + /// Create a compression configuration for an encoding mechanism. + pub fn new(scheme: CompressionScheme, level: Option) -> Self { Self { scheme, level } } + + /// Return the selected compression scheme. + pub fn scheme(&self) -> CompressionScheme { + self.scheme + } + + /// Return the optional compression level. + pub fn level(&self) -> Option { + self.level + } } impl Default for CompressionConfig { @@ -765,7 +776,6 @@ mod tests { }, encodings::physical::block::lz4::Lz4BufferCompressor, testing::{FnArrayGeneratorProvider, TestCases, check_round_trip_encoding_generated}, - version::LanceFileVersion, }; #[test] @@ -809,7 +819,7 @@ mod tests { // Need to use large pages as small pages might be too small to compress .with_page_sizes(vec![1024 * 1024]) .with_expected_encoding("zstd") - .with_min_file_version(LanceFileVersion::V2_1); + .with_structural_encodings(); // Can't use the default random provider because random data isn't compressible // and we will fallback to uncompressed encoding diff --git a/rust/lance-encoding/src/encodings/physical/byte_stream_split.rs b/rust/lance-encoding/src/encodings/physical/byte_stream_split.rs index c2b7aac9b9c..d4dbd1045b3 100644 --- a/rust/lance-encoding/src/encodings/physical/byte_stream_split.rs +++ b/rust/lance-encoding/src/encodings/physical/byte_stream_split.rs @@ -62,7 +62,7 @@ use crate::compression::MiniBlockDecompressor; use crate::compression_config::BssMode; use crate::data::{BlockInfo, DataBlock, FixedWidthDataBlock}; use crate::encodings::logical::primitive::miniblock::{ - MiniBlockChunk, MiniBlockCompressed, MiniBlockCompressor, + MiniBlockChunk, MiniBlockCompressed, MiniBlockCompressionContext, MiniBlockCompressor, }; use crate::format::ProtobufUtils21; use crate::format::pb21::CompressiveEncoding; @@ -107,7 +107,11 @@ impl ByteStreamSplitEncoder { } impl MiniBlockCompressor for ByteStreamSplitEncoder { - fn compress(&self, page: DataBlock) -> Result<(MiniBlockCompressed, CompressiveEncoding)> { + fn compress( + &self, + _context: MiniBlockCompressionContext, + page: DataBlock, + ) -> Result<(MiniBlockCompressed, CompressiveEncoding)> { match page { DataBlock::FixedWidth(data) => { let num_values = data.num_values; @@ -263,6 +267,10 @@ impl MiniBlockDecompressor for ByteStreamSplitDecompressor { block_info: BlockInfo::new(), })) } + + fn decoded_size_bytes(&self, num_values: u64) -> Option { + num_values.checked_mul(self.bytes_per_value() as u64) + } } /// Determine if BSS should be used based on mode and data characteristics @@ -345,7 +353,9 @@ mod tests { }); // Compress - let (compressed, _encoding) = encoder.compress(data_block).unwrap(); + let (compressed, _encoding) = encoder + .compress(MiniBlockCompressionContext::new(0, true, true), data_block) + .unwrap(); // Decompress let decompressed = decompressor @@ -391,7 +401,9 @@ mod tests { }); // Compress - let (compressed, _encoding) = encoder.compress(data_block).unwrap(); + let (compressed, _encoding) = encoder + .compress(MiniBlockCompressionContext::new(0, true, true), data_block) + .unwrap(); // Decompress let decompressed = decompressor @@ -424,7 +436,9 @@ mod tests { }); // Compress empty data - let (compressed, _encoding) = encoder.compress(data_block).unwrap(); + let (compressed, _encoding) = encoder + .compress(MiniBlockCompressionContext::new(0, true, true), data_block) + .unwrap(); // Decompress empty data let decompressed = decompressor.decompress(compressed.data, 0).unwrap(); diff --git a/rust/lance-encoding/src/encodings/physical/fsst.rs b/rust/lance-encoding/src/encodings/physical/fsst.rs index 8c1fe4141df..eaa8c911236 100644 --- a/rust/lance-encoding/src/encodings/physical/fsst.rs +++ b/rust/lance-encoding/src/encodings/physical/fsst.rs @@ -23,7 +23,7 @@ use crate::{ data::{BlockInfo, DataBlock, VariableWidthBlock}, encodings::logical::primitive::{ fullzip::{PerValueCompressor, PerValueDataBlock}, - miniblock::{MiniBlockCompressed, MiniBlockCompressor}, + miniblock::{MiniBlockCompressed, MiniBlockCompressionContext, MiniBlockCompressor}, }, format::{ ProtobufUtils21, @@ -138,7 +138,11 @@ impl FsstMiniBlockEncoder { } impl MiniBlockCompressor for FsstMiniBlockEncoder { - fn compress(&self, data: DataBlock) -> Result<(MiniBlockCompressed, CompressiveEncoding)> { + fn compress( + &self, + context: MiniBlockCompressionContext, + data: DataBlock, + ) -> Result<(MiniBlockCompressed, CompressiveEncoding)> { let compressed = FsstCompressed::fsst_compress(data)?; let data_block = DataBlock::VariableWidth(compressed.data); @@ -148,7 +152,7 @@ impl MiniBlockCompressor for FsstMiniBlockEncoder { as Box; let (binary_miniblock_compressed, binary_array_encoding) = - binary_compressor.compress(data_block)?; + binary_compressor.compress(context, data_block)?; Ok(( binary_miniblock_compressed, @@ -377,16 +381,13 @@ mod tests { use lance_datagen::{ByteCount, RowCount}; - use crate::{ - testing::{TestCases, check_round_trip_encoding_of_data}, - version::LanceFileVersion, - }; + use crate::testing::{TestCases, check_round_trip_encoding_of_data}; #[test_log::test(tokio::test)] async fn test_fsst() { let test_cases = TestCases::default() .with_expected_encoding("fsst") - .with_min_file_version(LanceFileVersion::V2_1); + .with_structural_encodings(); // Generate data suitable for FSST (large strings, total size > 32KB) let arr = lance_datagen::gen_batch() diff --git a/rust/lance-encoding/src/encodings/physical/general.rs b/rust/lance-encoding/src/encodings/physical/general.rs index 53c61928870..bea6a85cb01 100644 --- a/rust/lance-encoding/src/encodings/physical/general.rs +++ b/rust/lance-encoding/src/encodings/physical/general.rs @@ -9,7 +9,9 @@ use crate::{ compression::MiniBlockDecompressor, data::DataBlock, encodings::{ - logical::primitive::miniblock::{MiniBlockCompressed, MiniBlockCompressor}, + logical::primitive::miniblock::{ + MiniBlockCompressed, MiniBlockCompressionContext, MiniBlockCompressor, + }, physical::block::{CompressionConfig, GeneralBufferCompressor}, }, format::{ProtobufUtils21, pb21::CompressiveEncoding}, @@ -35,9 +37,13 @@ const MIN_BUFFER_SIZE_FOR_COMPRESSION: usize = 4 * 1024; use super::super::logical::primitive::miniblock::MiniBlockChunk; impl MiniBlockCompressor for GeneralMiniBlockCompressor { - fn compress(&self, page: DataBlock) -> Result<(MiniBlockCompressed, CompressiveEncoding)> { + fn compress( + &self, + context: MiniBlockCompressionContext, + page: DataBlock, + ) -> Result<(MiniBlockCompressed, CompressiveEncoding)> { // First, compress with the inner compressor - let (inner_compressed, inner_encoding) = self.inner.compress(page)?; + let (inner_compressed, inner_encoding) = self.inner.compress(context, page)?; // Return the original encoding without compression if there's no data or // the first buffer is not large enough @@ -132,6 +138,10 @@ impl MiniBlockDecompressor for GeneralMiniBlockDecompressor { self.inner.decompress(data, num_values) } + + fn decoded_size_bytes(&self, num_values: u64) -> Option { + self.inner.decoded_size_bytes(num_values) + } } #[cfg(test)] @@ -146,6 +156,10 @@ mod tests { use crate::format::pb21::compressive_encoding::Compression; use arrow_array::{Float64Array, Int32Array}; + fn miniblock_context() -> MiniBlockCompressionContext { + MiniBlockCompressionContext::new(0, true, true) + } + #[derive(Debug)] struct TestCase { name: &'static str, @@ -249,7 +263,9 @@ mod tests { GeneralMiniBlockCompressor::new(test_case.inner_encoder, test_case.compression); // Compress the data - let (compressed, encoding) = compressor.compress(test_case.data).unwrap(); + let (compressed, encoding) = compressor + .compress(miniblock_context(), test_case.data) + .unwrap(); // Check if compression was applied as expected match &encoding.compression { @@ -461,7 +477,7 @@ mod tests { let compressor = GeneralMiniBlockCompressor::new(inner, compression); // Compress the data - let (compressed, encoding) = compressor.compress(block).unwrap(); + let (compressed, encoding) = compressor.compress(miniblock_context(), block).unwrap(); // Should get GeneralMiniBlock encoding since buffer is 4KB match &encoding.compression { @@ -503,7 +519,7 @@ mod tests { }, ); - let (compressed, _) = compressor.compress(data).unwrap(); + let (compressed, _) = compressor.compress(miniblock_context(), data).unwrap(); // RLE produces 2 buffers, but only the first one is compressed assert_eq!(compressed.data.len(), 2); } @@ -539,7 +555,9 @@ mod tests { }, ); - let (_compressed, encoding) = compressor.compress(test_32.data).unwrap(); + let (_compressed, encoding) = compressor + .compress(miniblock_context(), test_32.data) + .unwrap(); // Verify the encoding structure match &encoding.compression { @@ -596,7 +614,9 @@ mod tests { }, ); - let (_compressed_64, encoding_64) = compressor_64.compress(block_64).unwrap(); + let (_compressed_64, encoding_64) = compressor_64 + .compress(miniblock_context(), block_64) + .unwrap(); // Verify the encoding structure for 64-bit match &encoding_64.compression { @@ -650,7 +670,7 @@ mod tests { }, ); - let result = compressor.compress(empty_block); + let result = compressor.compress(miniblock_context(), empty_block); match result { Ok((compressed, _)) => { assert_eq!(compressed.num_values, 0); diff --git a/rust/lance-encoding/src/encodings/physical/packed.rs b/rust/lance-encoding/src/encodings/physical/packed.rs index 3ade6a70818..81879f9c4fe 100644 --- a/rust/lance-encoding/src/encodings/physical/packed.rs +++ b/rust/lance-encoding/src/encodings/physical/packed.rs @@ -18,7 +18,7 @@ use lance_core::{Error, Result, datatypes::Field}; use crate::{ buffer::LanceBuffer, compression::{ - DefaultCompressionStrategy, FixedPerValueDecompressor, MiniBlockDecompressor, + CompressionStrategy, FixedPerValueDecompressor, MiniBlockDecompressor, VariablePerValueDecompressor, }, data::{ @@ -27,7 +27,7 @@ use crate::{ }, encodings::logical::primitive::{ fullzip::{PerValueCompressor, PerValueDataBlock}, - miniblock::{MiniBlockCompressed, MiniBlockCompressor}, + miniblock::{MiniBlockCompressed, MiniBlockCompressionContext, MiniBlockCompressor}, }, format::{ ProtobufUtils21, @@ -73,7 +73,11 @@ fn struct_data_block_to_fixed_width_data_block( pub struct PackedStructFixedWidthMiniBlockEncoder {} impl MiniBlockCompressor for PackedStructFixedWidthMiniBlockEncoder { - fn compress(&self, data: DataBlock) -> Result<(MiniBlockCompressed, CompressiveEncoding)> { + fn compress( + &self, + context: MiniBlockCompressionContext, + data: DataBlock, + ) -> Result<(MiniBlockCompressed, CompressiveEncoding)> { match data { DataBlock::Struct(struct_data_block) => { let bits_per_values = struct_data_block.children.iter().map(|data_block| data_block.as_fixed_width_ref().unwrap().bits_per_value).collect::>(); @@ -84,7 +88,7 @@ impl MiniBlockCompressor for PackedStructFixedWidthMiniBlockEncoder { // store and transformed fixed-width data block. let value_miniblock_compressor = Box::new(ValueEncoder::default()) as Box; let (value_miniblock_compressed, value_array_encoding) = - value_miniblock_compressor.compress(data_block)?; + value_miniblock_compressor.compress(context, data_block)?; Ok(( value_miniblock_compressed, @@ -282,12 +286,12 @@ impl VariablePackedFieldData { #[derive(Debug)] pub struct PackedStructVariablePerValueEncoder { - strategy: DefaultCompressionStrategy, + strategy: Arc, fields: Vec, } impl PackedStructVariablePerValueEncoder { - pub fn new(strategy: DefaultCompressionStrategy, fields: Vec) -> Self { + pub fn new(strategy: Arc, fields: Vec) -> Self { Self { strategy, fields } } } @@ -323,12 +327,8 @@ impl PerValueCompressor for PackedStructVariablePerValueEncoder { let mut field_data = Vec::with_capacity(self.fields.len()); let mut field_metadata = Vec::with_capacity(self.fields.len()); - for (field, child_block) in self.fields.iter().zip(struct_block.children.into_iter()) { - let compressor = crate::compression::CompressionStrategy::create_per_value( - &self.strategy, - field, - &child_block, - )?; + for (field, child_block) in self.fields.iter().zip(struct_block.children) { + let compressor = self.strategy.create_per_value(field, &child_block)?; let (compressed, encoding) = compressor.compress(child_block)?; match compressed { PerValueDataBlock::Fixed(block) => { @@ -447,7 +447,7 @@ impl FieldAccumulator { // In full-zip variable packed decoding, rep/def may produce a visible row // with an empty payload (e.g. null/invalid item). We still need to append // one placeholder per child so child row counts remain aligned. - fn append_empty(&mut self) { + fn append_empty(&mut self) -> Result<()> { match self { Self::Fixed { builder, @@ -563,7 +563,7 @@ impl VariablePerValueDecompressor for PackedStructVariablePerValueDecompressor { } if row_start == row_end { for accumulator in accumulators.iter_mut() { - accumulator.append_empty(); + accumulator.append_empty()?; } continue; } @@ -592,7 +592,7 @@ impl VariablePerValueDecompressor for PackedStructVariablePerValueDecompressor { num_values: 1, block_info: BlockInfo::new(), }); - builder.append(&value_block, 0..1); + builder.append(&value_block, 0..1)?; cursor = end; } ( @@ -631,7 +631,7 @@ impl VariablePerValueDecompressor for PackedStructVariablePerValueDecompressor { num_values: 1, block_info: BlockInfo::new(), }); - builder.append(&value_block, 0..1); + builder.append(&value_block, 0..1)?; cursor = value_end; } ( @@ -670,7 +670,7 @@ impl VariablePerValueDecompressor for PackedStructVariablePerValueDecompressor { num_values: 1, block_info: BlockInfo::new(), }); - builder.append(&value_block, 0..1); + builder.append(&value_block, 0..1)?; cursor = value_end; } _ => { @@ -688,7 +688,7 @@ impl VariablePerValueDecompressor for PackedStructVariablePerValueDecompressor { } let mut children = Vec::with_capacity(self.fields.len()); - for (field, accumulator) in self.fields.iter().zip(accumulators.into_iter()) { + for (field, accumulator) in self.fields.iter().zip(accumulators) { match (field, accumulator) { ( VariablePackedStructFieldDecoder { @@ -758,12 +758,13 @@ impl VariablePerValueDecompressor for PackedStructVariablePerValueDecompressor { mod tests { use super::*; use crate::{ - compression::CompressionStrategy, - compression::{DefaultCompressionStrategy, DefaultDecompressionStrategy}, + compression::DefaultDecompressionStrategy, + compression_config::CompressionParams, constants::PACKED_STRUCT_META_KEY, statistics::ComputeStat, - testing::{TestCases, check_round_trip_encoding_of_data}, - version::LanceFileVersion, + testing::{ + TestCases, TestEncoding, check_round_trip_encoding_of_data, test_compression_strategy, + }, }; use arrow_array::{ Array, ArrayRef, BinaryArray, Int32Array, Int64Array, LargeStringArray, StringArray, @@ -863,9 +864,9 @@ mod tests { let data_block = DataBlock::Struct(struct_block); let compression_strategy = - DefaultCompressionStrategy::new().with_version(LanceFileVersion::V2_2); + test_compression_strategy(TestEncoding::StructuralU32, CompressionParams::default()); let compressor = crate::compression::CompressionStrategy::create_per_value( - &compression_strategy, + compression_strategy.as_ref(), &struct_field, &data_block, )?; @@ -931,9 +932,9 @@ mod tests { let data_block = DataBlock::Struct(struct_block); let compression_strategy = - DefaultCompressionStrategy::new().with_version(LanceFileVersion::V2_2); + test_compression_strategy(TestEncoding::StructuralU32, CompressionParams::default()); let compressor = crate::compression::CompressionStrategy::create_per_value( - &compression_strategy, + compression_strategy.as_ref(), &struct_field, &data_block, )?; @@ -1013,7 +1014,7 @@ mod tests { ])); let test_cases = TestCases::default() - .with_min_file_version(LanceFileVersion::V2_2) + .with_u32_structural_encodings() .with_expected_encoding("variable_packed_struct"); check_round_trip_encoding_of_data(vec![array], &test_cases, meta).await; @@ -1052,9 +1053,9 @@ mod tests { let data_block = DataBlock::Struct(struct_block); let compression_strategy = - DefaultCompressionStrategy::new().with_version(LanceFileVersion::V2_2); + test_compression_strategy(TestEncoding::StructuralU32, CompressionParams::default()); let compressor = crate::compression::CompressionStrategy::create_per_value( - &compression_strategy, + compression_strategy.as_ref(), &struct_field, &data_block, )?; @@ -1135,7 +1136,7 @@ mod tests { }; let compression_strategy = - DefaultCompressionStrategy::new().with_version(LanceFileVersion::V2_1); + test_compression_strategy(TestEncoding::StructuralU16, CompressionParams::default()); let result = compression_strategy.create_per_value(&struct_field, &DataBlock::Struct(struct_block)); diff --git a/rust/lance-encoding/src/encodings/physical/rle.rs b/rust/lance-encoding/src/encodings/physical/rle.rs index da758b05bfe..0f3e330a3ba 100644 --- a/rust/lance-encoding/src/encodings/physical/rle.rs +++ b/rust/lance-encoding/src/encodings/physical/rle.rs @@ -54,7 +54,7 @@ //! When used in the block compression path, the encoded output is a single buffer: //! `[8-byte header: values buffer size][values buffer][run_lengths buffer]`. -use arrow_buffer::ArrowNativeType; +use arrow_buffer::{ArrowNativeType, ScalarBuffer}; use log::trace; use crate::buffer::LanceBuffer; @@ -63,8 +63,9 @@ use crate::data::DataBlock; use crate::data::{BlockInfo, FixedWidthDataBlock}; use crate::encodings::logical::primitive::miniblock::{ MAX_MINIBLOCK_BYTES, MAX_MINIBLOCK_VALUES, MiniBlockChunk, MiniBlockCompressed, - MiniBlockCompressor, + MiniBlockCompressionContext, MiniBlockCompressor, }; +use crate::encodings::physical::block::{CompressionConfig, GeneralBufferCompressor}; use crate::format::ProtobufUtils21; use crate::format::pb21::CompressiveEncoding; @@ -326,6 +327,18 @@ pub(crate) fn accumulate_run_length_entries( #[derive(Debug)] pub struct RleEncoder { run_length_width: RunLengthWidth, + values_compression: Option, + run_lengths_compression: Option, + use_child_bitpacking: bool, +} + +#[derive(Clone)] +struct RleChildCandidate { + encoding: CompressiveEncoding, + data: LanceBuffer, + chunk_sizes: Vec, + size: usize, + requires_num_values: bool, } impl Default for RleEncoder { @@ -338,11 +351,33 @@ impl RleEncoder { pub fn new() -> Self { Self { run_length_width: RunLengthWidth::U8, + values_compression: None, + run_lengths_compression: None, + use_child_bitpacking: false, } } pub(crate) fn with_run_length_width(run_length_width: RunLengthWidth) -> Self { - Self { run_length_width } + Self { + run_length_width, + values_compression: None, + run_lengths_compression: None, + use_child_bitpacking: false, + } + } + + pub(crate) fn with_child_encoding( + run_length_width: RunLengthWidth, + values_compression: Option, + run_lengths_compression: Option, + use_child_bitpacking: bool, + ) -> Self { + Self { + run_length_width, + values_compression, + run_lengths_compression, + use_child_bitpacking, + } } fn encode_data( @@ -417,7 +452,15 @@ impl RleEncoder { }; if values_processed == 0 { - break; + // A non-final chunk needs at least two values because log_num_values == 0 + // identifies the final chunk. Report an error instead of returning partial data. + return Err(Error::internal(format!( + "RLE encoder made no progress: values_remaining={values_remaining}, \ + offset={offset}, data_len={}, bits_per_value={bits_per_value}, \ + max_miniblock_values={}", + data.len(), + *MAX_MINIBLOCK_VALUES + ))); } let log_num_values = if is_last_chunk { @@ -678,10 +721,326 @@ impl RleEncoder { total_chunks * (type_size + self.run_length_width.bytes_per_value()) } + + fn flat_child_candidate( + buffers: &[LanceBuffer], + chunks: &[MiniBlockChunk], + buffer_index: usize, + bits_per_value: u64, + ) -> RleChildCandidate { + RleChildCandidate { + encoding: ProtobufUtils21::flat(bits_per_value, None), + data: buffers[buffer_index].clone(), + chunk_sizes: chunks + .iter() + .map(|chunk| chunk.buffer_sizes[buffer_index]) + .collect(), + size: buffers[buffer_index].len(), + requires_num_values: false, + } + } + + fn general_child_candidate( + buffers: &[LanceBuffer], + chunks: &[MiniBlockChunk], + buffer_index: usize, + bits_per_value: u64, + compression: CompressionConfig, + ) -> Result> { + if buffers.is_empty() || buffers[buffer_index].is_empty() { + return Ok(None); + }; + + let compressor = GeneralBufferCompressor::get_compressor(compression)?; + let original = &buffers[buffer_index]; + let mut compressed = Vec::new(); + let mut offset = 0usize; + let mut total_original_size = 0usize; + let mut compressed_sizes = Vec::with_capacity(chunks.len()); + + for chunk in chunks.iter() { + let chunk_size = chunk.buffer_sizes[buffer_index] as usize; + let end = offset.checked_add(chunk_size).ok_or_else(|| { + Error::invalid_input_source("RLE child buffer offset overflow".into()) + })?; + if end > original.len() { + return Err(Error::invalid_input_source( + format!( + "RLE child buffer {} chunk size exceeds buffer length: end {}, len {}", + buffer_index, + end, + original.len() + ) + .into(), + )); + } + + let start = compressed.len(); + compressor.compress(&original.as_ref()[offset..end], &mut compressed)?; + let compressed_size = compressed.len() - start; + let compressed_size = u32::try_from(compressed_size).map_err(|_| { + Error::invalid_input_source( + format!( + "RLE child buffer {} compressed chunk is too large: {} bytes", + buffer_index, compressed_size + ) + .into(), + ) + })?; + compressed_sizes.push(compressed_size); + total_original_size += chunk_size; + offset = end; + } + + if compressed.len() >= total_original_size { + return Ok(None); + } + + let encoding = + ProtobufUtils21::wrapped(compression, ProtobufUtils21::flat(bits_per_value, None))?; + Ok(Some( + RleChildCandidate { + encoding, + data: LanceBuffer::from(compressed), + chunk_sizes: compressed_sizes, + size: 0, + requires_num_values: false, + } + .with_size_from_data(), + )) + } + + #[cfg(feature = "bitpacking")] + fn bitpacked_child_candidate( + buffers: &[LanceBuffer], + chunks: &[MiniBlockChunk], + buffer_index: usize, + bits_per_value: u64, + ) -> Result> { + let original = &buffers[buffer_index]; + if original.is_empty() { + return Ok(None); + } + let packed_bits = Self::required_bits(original, bits_per_value)?; + if packed_bits >= bits_per_value { + return Ok(None); + } + + let compressor = crate::encodings::physical::bitpacking::OutOfLineBitpacking::new( + packed_bits, + bits_per_value, + ); + let mut packed = Vec::new(); + let mut offset = 0usize; + let mut packed_sizes = Vec::with_capacity(chunks.len()); + let bytes_per_value = usize::try_from(bits_per_value / 8).map_err(|_| { + Error::invalid_input_source( + format!("RLE child bit width is too large: {bits_per_value}").into(), + ) + })?; + + for chunk in chunks { + let chunk_size = chunk.buffer_sizes[buffer_index] as usize; + let end = offset.checked_add(chunk_size).ok_or_else(|| { + Error::invalid_input_source("RLE child buffer offset overflow".into()) + })?; + if end > original.len() { + return Err(Error::invalid_input_source( + format!( + "RLE child buffer {} chunk size exceeds buffer length: end {}, len {}", + buffer_index, + end, + original.len() + ) + .into(), + )); + } + if bytes_per_value == 0 || !chunk_size.is_multiple_of(bytes_per_value) { + return Err(Error::invalid_input_source( + format!( + "RLE child buffer {} chunk has invalid size {} for {} bits per value", + buffer_index, chunk_size, bits_per_value + ) + .into(), + )); + } + + let child_values = (chunk_size / bytes_per_value) as u64; + let block = DataBlock::FixedWidth(FixedWidthDataBlock { + bits_per_value, + data: original.slice_with_length(offset, chunk_size), + num_values: child_values, + block_info: BlockInfo::default(), + }); + let chunk_packed = BlockCompressor::compress(&compressor, block)?; + let packed_size = u32::try_from(chunk_packed.len()).map_err(|_| { + Error::invalid_input_source( + format!( + "RLE child buffer {} bitpacked chunk is too large: {} bytes", + buffer_index, + chunk_packed.len() + ) + .into(), + ) + })?; + packed_sizes.push(packed_size); + packed.extend_from_slice(chunk_packed.as_ref()); + offset = end; + } + + if packed.len() >= original.len() { + return Ok(None); + } + + Ok(Some( + RleChildCandidate { + encoding: ProtobufUtils21::out_of_line_bitpacking( + bits_per_value, + ProtobufUtils21::flat(packed_bits, None), + ), + data: LanceBuffer::from(packed), + chunk_sizes: packed_sizes, + size: 0, + requires_num_values: true, + } + .with_size_from_data(), + )) + } + + #[cfg(feature = "bitpacking")] + fn required_bits(buffer: &LanceBuffer, bits_per_value: u64) -> Result { + let max_value = match bits_per_value { + 8 => buffer.as_ref().iter().map(|value| *value as u64).max(), + 16 => buffer + .as_ref() + .chunks_exact(2) + .map(|value| u16::from_le_bytes(value.try_into().unwrap()) as u64) + .max(), + 32 => buffer + .as_ref() + .chunks_exact(4) + .map(|value| u32::from_le_bytes(value.try_into().unwrap()) as u64) + .max(), + 64 => buffer + .as_ref() + .chunks_exact(8) + .map(|value| u64::from_le_bytes(value.try_into().unwrap())) + .max(), + _ => { + return Err(Error::invalid_input_source( + format!( + "RLE child bitpacking only supports 8, 16, 32, or 64-bit values, got {bits_per_value}" + ) + .into(), + )); + } + } + .unwrap_or(0); + Ok((u64::BITS - max_value.leading_zeros()).max(1) as u64) + } + + fn child_candidates( + buffers: &[LanceBuffer], + chunks: &[MiniBlockChunk], + buffer_index: usize, + bits_per_value: u64, + compression: Option, + use_child_bitpacking: bool, + ) -> Result> { + #[cfg(not(feature = "bitpacking"))] + let _ = use_child_bitpacking; + let mut candidates = vec![Self::flat_child_candidate( + buffers, + chunks, + buffer_index, + bits_per_value, + )]; + if let Some(compression) = compression + && let Some(candidate) = Self::general_child_candidate( + buffers, + chunks, + buffer_index, + bits_per_value, + compression, + )? + { + candidates.push(candidate); + } + #[cfg(feature = "bitpacking")] + { + if use_child_bitpacking + && let Some(candidate) = + Self::bitpacked_child_candidate(buffers, chunks, buffer_index, bits_per_value)? + { + candidates.push(candidate); + } + } + Ok(candidates) + } + + fn select_child_candidates( + values: Vec, + run_lengths: Vec, + ) -> (RleChildCandidate, RleChildCandidate) { + let mut best: Option<(usize, usize, usize)> = None; + for (value_idx, value) in values.iter().enumerate() { + for (length_idx, length) in run_lengths.iter().enumerate() { + if value.requires_num_values && length.requires_num_values { + continue; + } + let size = value.size + length.size; + if best.is_none_or(|(_, _, best_size)| size < best_size) { + best = Some((value_idx, length_idx, size)); + } + } + } + let (value_idx, length_idx, _) = + best.expect("flat RLE child candidates should always be selectable"); + (values[value_idx].clone(), run_lengths[length_idx].clone()) + } + + pub(crate) fn selected_payload_size(&self, data: &FixedWidthDataBlock) -> Result { + let (all_buffers, chunks) = + self.encode_data(&data.data, data.num_values, data.bits_per_value)?; + if all_buffers.is_empty() { + return Ok(0); + } + + let values_candidates = Self::child_candidates( + &all_buffers, + &chunks, + 0, + data.bits_per_value, + self.values_compression, + self.use_child_bitpacking, + )?; + let run_lengths_candidates = Self::child_candidates( + &all_buffers, + &chunks, + 1, + self.run_length_width.bits_per_value(), + self.run_lengths_compression, + self.use_child_bitpacking, + )?; + let (values, run_lengths) = + Self::select_child_candidates(values_candidates, run_lengths_candidates); + Ok((values.size as u128).saturating_add(run_lengths.size as u128)) + } +} + +impl RleChildCandidate { + fn with_size_from_data(mut self) -> Self { + self.size = self.data.len(); + self + } } impl MiniBlockCompressor for RleEncoder { - fn compress(&self, data: DataBlock) -> Result<(MiniBlockCompressed, CompressiveEncoding)> { + fn compress( + &self, + _context: MiniBlockCompressionContext, + data: DataBlock, + ) -> Result<(MiniBlockCompressed, CompressiveEncoding)> { match data { DataBlock::FixedWidth(fixed_width) => { let num_values = fixed_width.num_values; @@ -689,17 +1048,53 @@ impl MiniBlockCompressor for RleEncoder { let (all_buffers, chunks) = self.encode_data(&fixed_width.data, num_values, bits_per_value)?; + if all_buffers.is_empty() { + let compressed = MiniBlockCompressed { + data: all_buffers, + chunks, + num_values, + }; + let encoding = ProtobufUtils21::rle( + ProtobufUtils21::flat(bits_per_value, None), + ProtobufUtils21::flat(self.run_length_width.bits_per_value(), None), + ); + return Ok((compressed, encoding)); + } + + let values_candidates = Self::child_candidates( + &all_buffers, + &chunks, + 0, + bits_per_value, + self.values_compression, + self.use_child_bitpacking, + )?; + let run_lengths_candidates = Self::child_candidates( + &all_buffers, + &chunks, + 1, + self.run_length_width.bits_per_value(), + self.run_lengths_compression, + self.use_child_bitpacking, + )?; + let (values, run_lengths) = + Self::select_child_candidates(values_candidates, run_lengths_candidates); + let chunks = chunks + .into_iter() + .enumerate() + .map(|(idx, chunk)| MiniBlockChunk { + buffer_sizes: vec![values.chunk_sizes[idx], run_lengths.chunk_sizes[idx]], + log_num_values: chunk.log_num_values, + }) + .collect(); let compressed = MiniBlockCompressed { - data: all_buffers, + data: vec![values.data, run_lengths.data], chunks, num_values, }; - let encoding = ProtobufUtils21::rle( - ProtobufUtils21::flat(bits_per_value, None), - ProtobufUtils21::flat(self.run_length_width.bits_per_value(), None), - ); + let encoding = ProtobufUtils21::rle(values.encoding, run_lengths.encoding); Ok((compressed, encoding)) } @@ -741,6 +1136,125 @@ impl BlockCompressor for RleEncoder { pub struct RleDecompressor { bits_per_value: u64, run_length_width: RunLengthWidth, + values: RleChildDecompressor, + run_lengths: RleChildDecompressor, +} + +#[derive(Debug)] +pub(crate) struct RleChildDecompressor { + bits_per_value: u64, + inner: RleChildDecompressorInner, +} + +#[derive(Debug)] +enum RleChildDecompressorInner { + Flat, + Block { + decompressor: Box, + requires_num_values: bool, + }, +} + +impl RleChildDecompressor { + pub(crate) fn flat(bits_per_value: u64) -> Self { + Self { + bits_per_value, + inner: RleChildDecompressorInner::Flat, + } + } + + pub(crate) fn block( + bits_per_value: u64, + decompressor: Box, + requires_num_values: bool, + ) -> Self { + Self { + bits_per_value, + inner: RleChildDecompressorInner::Block { + decompressor, + requires_num_values, + }, + } + } + + pub(crate) fn bits_per_value(&self) -> u64 { + self.bits_per_value + } + + pub(crate) fn requires_num_values(&self) -> bool { + match &self.inner { + RleChildDecompressorInner::Flat => false, + RleChildDecompressorInner::Block { + requires_num_values, + .. + } => *requires_num_values, + } + } + + pub(crate) fn is_identity(&self) -> bool { + matches!(self.inner, RleChildDecompressorInner::Flat) + } + + fn decode( + &self, + data: LanceBuffer, + num_values: Option, + label: &str, + ) -> Result { + match &self.inner { + RleChildDecompressorInner::Flat => Ok(data), + RleChildDecompressorInner::Block { + decompressor, + requires_num_values, + } => { + let num_values = if *requires_num_values { + num_values.ok_or_else(|| { + Error::invalid_input_source( + format!("RLE {label} child compression requires the run count").into(), + ) + })? + } else { + num_values.unwrap_or(0) + }; + let decoded = decompressor.decompress(data, num_values)?; + self.extract_fixed_width(decoded, num_values, label) + } + } + } + + fn extract_fixed_width( + &self, + data: DataBlock, + expected_num_values: u64, + label: &str, + ) -> Result { + match data { + DataBlock::FixedWidth(block) => { + if block.bits_per_value != self.bits_per_value { + return Err(Error::invalid_input_source( + format!( + "RLE {label} child decoded {}-bit values, expected {}", + block.bits_per_value, self.bits_per_value + ) + .into(), + )); + } + if expected_num_values != 0 && block.num_values != expected_num_values { + return Err(Error::invalid_input_source( + format!( + "RLE {label} child decoded {} values, expected {}", + block.num_values, expected_num_values + ) + .into(), + )); + } + Ok(block.data) + } + _ => Err(Error::invalid_input_source( + format!("RLE {label} child decoded to a non fixed-width block").into(), + )), + } + } } impl RleDecompressor { @@ -748,6 +1262,8 @@ impl RleDecompressor { Self { bits_per_value, run_length_width: RunLengthWidth::U8, + values: RleChildDecompressor::flat(bits_per_value), + run_lengths: RleChildDecompressor::flat(RunLengthWidth::U8.bits_per_value()), } } @@ -758,10 +1274,31 @@ impl RleDecompressor { Self { bits_per_value, run_length_width, + values: RleChildDecompressor::flat(bits_per_value), + run_lengths: RleChildDecompressor::flat(run_length_width.bits_per_value()), + } + } + + pub(crate) fn with_child_decompressors( + bits_per_value: u64, + run_length_width: RunLengthWidth, + values: RleChildDecompressor, + run_lengths: RleChildDecompressor, + ) -> Self { + Self { + bits_per_value, + run_length_width, + values, + run_lengths, } } - fn decode_data(&self, data: Vec, num_values: u64) -> Result { + fn decode_data( + &self, + data: Vec, + num_values: u64, + clamp_overflow: bool, + ) -> Result { if num_values == 0 { return Ok(DataBlock::FixedWidth(FixedWidthDataBlock { bits_per_value: self.bits_per_value, @@ -781,14 +1318,37 @@ impl RleDecompressor { )); } - let values_buffer = &data[0]; - let lengths_buffer = &data[1]; + let mut data_iter = data.into_iter(); + let values_buffer = data_iter.next().unwrap(); + let lengths_buffer = data_iter.next().unwrap(); + let (values_buffer, lengths_buffer) = + self.decode_child_buffers(values_buffer, lengths_buffer)?; let decoded_data = match self.bits_per_value { - 8 => self.decode_generic::(values_buffer, lengths_buffer, num_values)?, - 16 => self.decode_generic::(values_buffer, lengths_buffer, num_values)?, - 32 => self.decode_generic::(values_buffer, lengths_buffer, num_values)?, - 64 => self.decode_generic::(values_buffer, lengths_buffer, num_values)?, + 8 => self.decode_generic::( + &values_buffer, + &lengths_buffer, + num_values, + clamp_overflow, + )?, + 16 => self.decode_generic::( + &values_buffer, + &lengths_buffer, + num_values, + clamp_overflow, + )?, + 32 => self.decode_generic::( + &values_buffer, + &lengths_buffer, + num_values, + clamp_overflow, + )?, + 64 => self.decode_generic::( + &values_buffer, + &lengths_buffer, + num_values, + clamp_overflow, + )?, _ => { return Err(Error::invalid_input_source( format!( @@ -808,11 +1368,74 @@ impl RleDecompressor { })) } + fn decode_child_buffers( + &self, + values_buffer: LanceBuffer, + lengths_buffer: LanceBuffer, + ) -> Result<(LanceBuffer, LanceBuffer)> { + let values_requires_num_runs = self.values.requires_num_values(); + let lengths_requires_num_runs = self.run_lengths.requires_num_values(); + if values_requires_num_runs && lengths_requires_num_runs { + return Err(Error::invalid_input_source( + "RLE values and run lengths child compression both require the run count".into(), + )); + } + + if values_requires_num_runs { + let lengths_buffer = self + .run_lengths + .decode(lengths_buffer, None, "run lengths")?; + let num_runs = Self::num_child_values( + &lengths_buffer, + self.run_lengths.bits_per_value(), + "run lengths", + )?; + let values_buffer = self + .values + .decode(values_buffer, Some(num_runs), "values")?; + Ok((values_buffer, lengths_buffer)) + } else if lengths_requires_num_runs { + let values_buffer = self.values.decode(values_buffer, None, "values")?; + let num_runs = + Self::num_child_values(&values_buffer, self.values.bits_per_value(), "values")?; + let lengths_buffer = + self.run_lengths + .decode(lengths_buffer, Some(num_runs), "run lengths")?; + Ok((values_buffer, lengths_buffer)) + } else { + let values_buffer = self.values.decode(values_buffer, None, "values")?; + let lengths_buffer = self + .run_lengths + .decode(lengths_buffer, None, "run lengths")?; + Ok((values_buffer, lengths_buffer)) + } + } + + fn num_child_values(buffer: &LanceBuffer, bits_per_value: u64, label: &str) -> Result { + let bytes_per_value = usize::try_from(bits_per_value / 8).map_err(|_| { + Error::invalid_input_source( + format!("RLE {label} child bit width is too large: {bits_per_value}").into(), + ) + })?; + if bytes_per_value == 0 || !buffer.len().is_multiple_of(bytes_per_value) { + return Err(Error::invalid_input_source( + format!( + "RLE {label} child decoded to {} bytes, not divisible by {}", + buffer.len(), + bytes_per_value + ) + .into(), + )); + } + Ok((buffer.len() / bytes_per_value) as u64) + } + fn decode_generic( &self, values_buffer: &LanceBuffer, lengths_buffer: &LanceBuffer, num_values: u64, + clamp_overflow: bool, ) -> Result where T: bytemuck::Pod + Copy + std::fmt::Debug + ArrowNativeType, @@ -865,8 +1488,21 @@ impl RleDecompressor { format!("RLE num_values does not fit in usize: {num_values}").into(), ) })?; - let mut decoded_value_count = 0usize; - for length_bytes in lengths.chunks_exact(length_size) { + // Legacy miniblock encoders rolled back to a power-of-2 checkpoint after a run + // had already crossed it, so a chunk's run lengths can sum past its declared + // value count (the excess values are re-encoded at the start of the next chunk). + // The pre-run-length-width decoder truncated the excess, so miniblock decoding + // clamps rather than rejects to keep those files readable. Block payloads never + // legitimately overflow, so they decode strictly. + let mut decoded: Vec = Vec::new(); + decoded + .try_reserve_exact(expected_value_count) + .map_err(|_| { + Error::invalid_input_source( + format!("RLE decoding cannot allocate {expected_value_count} values").into(), + ) + })?; + for (value, length_bytes) in values.iter().zip(lengths.chunks_exact(length_size)) { let length = self.run_length_width.read_length(length_bytes); if length == 0 { return Err(Error::invalid_input_source( @@ -878,95 +1514,534 @@ impl RleDecompressor { format!("RLE run length does not fit in usize: {length}").into(), ) })?; - decoded_value_count = decoded_value_count.checked_add(length).ok_or_else(|| { + let remaining = expected_value_count - decoded.len(); + if length > remaining { + if !clamp_overflow { + return Err(Error::invalid_input_source( + format!( + "RLE decoding overflowed expected value count: produced at least {}, expected {}", + decoded.len() + length, + expected_value_count + ) + .into(), + )); + } + decoded.resize(expected_value_count, *value); + break; + } + decoded.resize(decoded.len() + length, *value); + } + + if decoded.len() != expected_value_count { + return Err(Error::invalid_input_source( + format!( + "RLE decoding produced {} values, expected {}", + decoded.len(), + expected_value_count + ) + .into(), + )); + } + + trace!( + "RLE decoded {} {} values", + num_values, + std::any::type_name::() + ); + Ok(LanceBuffer::reinterpret_vec(decoded)) + } +} + +impl MiniBlockDecompressor for RleDecompressor { + fn decompress(&self, data: Vec, num_values: u64) -> Result { + self.decode_data(data, num_values, true) + } + + fn decoded_size_bytes(&self, num_values: u64) -> Option { + num_values + .checked_mul(self.bits_per_value) + .map(|bits| bits.div_ceil(8)) + } +} + +impl BlockDecompressor for RleDecompressor { + fn decompress(&self, data: LanceBuffer, num_values: u64) -> Result { + let (values_buffer, lengths_buffer) = parse_rle_block_frame(&data)?; + self.decode_data(vec![values_buffer, lengths_buffer], num_values, false) + } +} + +/// Split an RLE block-format buffer into its `(values, lengths)` sub-buffers. +/// Frame: `[values_size: u64-le][values bytes][run-length bytes]`. +fn parse_rle_block_frame(data: &LanceBuffer) -> Result<(LanceBuffer, LanceBuffer)> { + // fetch the values_size + if data.len() < 8 { + return Err(Error::invalid_input_source( + format!("Insufficient data size: {}", data.len()).into(), + )); + } + + let values_size_bytes: [u8; 8] = data[..8].try_into().expect("slice length already checked"); + let values_size: usize = u64::from_le_bytes(values_size_bytes) + .try_into() + .map_err(|_| { + Error::invalid_input_source( + format!( + "Invalid values buffer size: {}", + u64::from_le_bytes(values_size_bytes) + ) + .into(), + ) + })?; + + // parse values + let values_start: usize = 8; + let lengths_start = values_start + .checked_add(values_size) + .ok_or_else(|| Error::invalid_input_source("Invalid RLE values buffer size".into()))?; + + if data.len() < lengths_start { + return Err(Error::invalid_input_source( + format!("Insufficient data size: {}", data.len()).into(), + )); + } + + let values_buffer = data.slice_with_length(values_start, values_size); + let lengths_buffer = data.slice_with_length(lengths_start, data.len() - lengths_start); + Ok((values_buffer, lengths_buffer)) +} + +#[derive(Clone, Debug)] +enum RleRunLengths { + U8(ScalarBuffer), + U16(ScalarBuffer), + U32(ScalarBuffer), +} + +impl RleRunLengths { + fn try_new(buffer: LanceBuffer, width: RunLengthWidth) -> Result { + let width_bytes = width.bytes_per_value(); + if !buffer.len().is_multiple_of(width_bytes) { + return Err(Error::invalid_input_source( + format!( + "Invalid RLE run lengths buffer: {} bytes (not divisible by {})", + buffer.len(), + width_bytes + ) + .into(), + )); + } + Ok(match width { + RunLengthWidth::U8 => Self::U8(buffer.borrow_to_typed_slice()), + RunLengthWidth::U16 => Self::U16(buffer.borrow_to_typed_slice()), + RunLengthWidth::U32 => Self::U32(buffer.borrow_to_typed_slice()), + }) + } + + fn len(&self) -> usize { + match self { + Self::U8(lengths) => lengths.len(), + Self::U16(lengths) => lengths.len(), + Self::U32(lengths) => lengths.len(), + } + } + + fn get(&self, index: usize) -> usize { + match self { + Self::U8(lengths) => lengths[index] as usize, + Self::U16(lengths) => lengths[index] as usize, + Self::U32(lengths) => lengths[index] as usize, + } + } + + fn owned_size(&self) -> usize { + match self { + Self::U8(lengths) => std::mem::size_of_val(lengths.as_ref()), + Self::U16(lengths) => std::mem::size_of_val(lengths.as_ref()), + Self::U32(lengths) => std::mem::size_of_val(lengths.as_ref()), + } + } + + fn into_owned(self) -> Self { + match self { + Self::U8(lengths) => Self::U8(ScalarBuffer::from(lengths.as_ref().to_vec())), + Self::U16(lengths) => Self::U16(ScalarBuffer::from(lengths.as_ref().to_vec())), + Self::U32(lengths) => Self::U32(ScalarBuffer::from(lengths.as_ref().to_vec())), + } + } + + fn deep_size(&self) -> usize { + match self { + Self::U8(lengths) => lengths.inner().capacity(), + Self::U16(lengths) => lengths.inner().capacity(), + Self::U32(lengths) => lengths.inner().capacity(), + } + } +} + +/// Validated physical RLE runs for `u16` values. +/// +/// The values and original-width lengths remain unexpanded. The constructor +/// verifies that every length is non-zero and that the runs cover exactly +/// `num_values` logical values. +#[derive(Clone, Debug)] +pub(crate) struct RleRuns { + values: ScalarBuffer, + lengths: RleRunLengths, + num_values: usize, + coalesced_runs: usize, +} + +impl RleRuns { + fn try_new( + values_buffer: LanceBuffer, + lengths_buffer: LanceBuffer, + run_length_width: RunLengthWidth, + num_values: u64, + ) -> Result { + let num_values = usize::try_from(num_values).map_err(|_| { + Error::invalid_input_source( + format!("RLE num_values does not fit in usize: {num_values}").into(), + ) + })?; + let type_size = std::mem::size_of::(); + if !values_buffer.len().is_multiple_of(type_size) { + return Err(Error::invalid_input_source( + format!( + "Invalid RLE u16 values buffer: {} bytes (not divisible by {})", + values_buffer.len(), + type_size + ) + .into(), + )); + } + + let values = values_buffer.borrow_to_typed_slice::(); + let lengths = RleRunLengths::try_new(lengths_buffer, run_length_width)?; + if values.len() != lengths.len() { + return Err(Error::invalid_input_source( + format!( + "Inconsistent RLE buffers: {} runs but {} length entries", + values.len(), + lengths.len() + ) + .into(), + )); + } + if values.is_empty() && num_values != 0 { + return Err(Error::invalid_input_source( + format!("Empty RLE buffers but expected {num_values} values").into(), + )); + } + + let mut decoded_values = 0usize; + let mut coalesced_runs = 0usize; + let mut previous_value = None; + for run in 0..values.len() { + let length = lengths.get(run); + if length == 0 { + return Err(Error::invalid_input_source( + "RLE decoding encountered a zero run length".into(), + )); + } + decoded_values = decoded_values.checked_add(length).ok_or_else(|| { Error::invalid_input_source("RLE run length sum overflowed usize".into()) })?; - if decoded_value_count > expected_value_count { + if decoded_values > num_values { return Err(Error::invalid_input_source( format!( "RLE decoding overflowed expected value count: produced at least {}, expected {}", - decoded_value_count, expected_value_count + decoded_values, num_values ) .into(), )); } + if previous_value != Some(values[run]) { + coalesced_runs += 1; + previous_value = Some(values[run]); + } } - - if decoded_value_count != expected_value_count { + if decoded_values != num_values { return Err(Error::invalid_input_source( format!( "RLE decoding produced {} values, expected {}", - decoded_value_count, expected_value_count + decoded_values, num_values ) .into(), )); } - let mut decoded: Vec = Vec::with_capacity(expected_value_count); - for (value, length_bytes) in values.iter().zip(lengths.chunks_exact(length_size)) { - let length = self.run_length_width.read_length(length_bytes) as usize; - decoded.resize(decoded.len() + length, *value); - } - - trace!( - "RLE decoded {} {} values", + Ok(Self { + values, + lengths, num_values, - std::any::type_name::() - ); - Ok(LanceBuffer::reinterpret_vec(decoded)) + coalesced_runs, + }) } -} -impl MiniBlockDecompressor for RleDecompressor { - fn decompress(&self, data: Vec, num_values: u64) -> Result { - self.decode_data(data, num_values) + pub(crate) fn num_values(&self) -> usize { + self.num_values } -} -impl BlockDecompressor for RleDecompressor { - fn decompress(&self, data: LanceBuffer, num_values: u64) -> Result { - // fetch the values_size - if data.len() < 8 { - return Err(Error::invalid_input_source( - format!("Insufficient data size: {}", data.len()).into(), - )); + pub(crate) fn num_runs(&self) -> usize { + self.values.len() + } + + pub(crate) fn coalesced_runs(&self) -> usize { + self.coalesced_runs + } + + pub(crate) fn owned_size(&self) -> usize { + std::mem::size_of_val(self.values.as_ref()) + self.lengths.owned_size() + } + + pub(crate) fn into_owned(self) -> Self { + Self { + values: ScalarBuffer::from(self.values.as_ref().to_vec()), + lengths: self.lengths.into_owned(), + num_values: self.num_values, + coalesced_runs: self.coalesced_runs, } + } - let values_size_bytes: [u8; 8] = - data[..8].try_into().expect("slice length already checked"); - let values_size: u64 = u64::from_le_bytes(values_size_bytes); + pub(crate) fn deep_size(&self) -> usize { + self.values.inner().capacity() + self.lengths.deep_size() + } - // parse values - let values_start: usize = 8; - let values_size: usize = values_size.try_into().map_err(|_| { - Error::invalid_input_source( - format!("Invalid values buffer size: {}", values_size).into(), - ) - })?; - let lengths_start = values_start - .checked_add(values_size) - .ok_or_else(|| Error::invalid_input_source("Invalid RLE values buffer size".into()))?; + pub(crate) fn value(&self, run: usize) -> u16 { + self.values[run] + } + + pub(crate) fn length(&self, run: usize) -> usize { + self.lengths.get(run) + } + + pub(crate) fn iter(&self) -> impl ExactSizeIterator + '_ { + (0..self.num_runs()).map(|run| (self.value(run), self.length(run))) + } +} - if data.len() < lengths_start { +impl RleDecompressor { + /// Decode and validate a block frame while preserving its physical runs. + pub(crate) fn decode_u16_runs(&self, data: LanceBuffer, num_values: u64) -> Result { + if self.bits_per_value != 16 { return Err(Error::invalid_input_source( - format!("Insufficient data size: {}", data.len()).into(), + format!( + "RLE level values must be 16 bits, got {}", + self.bits_per_value + ) + .into(), )); } - - let values_buffer = data.slice_with_length(values_start, values_size); - let lengths_buffer = data.slice_with_length(lengths_start, data.len() - lengths_start); - - self.decode_data(vec![values_buffer, lengths_buffer], num_values) + let (values_buffer, lengths_buffer) = parse_rle_block_frame(&data)?; + let (values_buffer, lengths_buffer) = + self.decode_child_buffers(values_buffer, lengths_buffer)?; + RleRuns::try_new( + values_buffer, + lengths_buffer, + self.run_length_width, + num_values, + ) } } #[cfg(test)] mod tests { + use std::sync::Arc; + use super::*; + use crate::compression::{ + DecompressionStrategy, DefaultDecompressionStrategy, create_rle_decompressor, + }; use crate::data::DataBlock; use crate::encodings::logical::primitive::miniblock::MAX_MINIBLOCK_VALUES; - use crate::{buffer::LanceBuffer, compression::BlockDecompressor}; + use crate::encodings::physical::block::{CompressionConfig, CompressionScheme}; + use crate::{ + buffer::LanceBuffer, + compression::{BlockCompressor, BlockDecompressor}, + }; use arrow_array::Int32Array; + use rstest::rstest; + + fn compress_miniblock( + compressor: &dyn MiniBlockCompressor, + data: DataBlock, + ) -> Result<(MiniBlockCompressed, CompressiveEncoding)> { + compressor.compress(MiniBlockCompressionContext::new(0, true, true), data) + } + + fn expand_u16_runs(runs: &RleRuns) -> Vec { + let mut expanded = Vec::with_capacity(runs.num_values()); + for (value, length) in runs.iter() { + expanded.extend(std::iter::repeat_n(value, length)); + } + expanded + } + + #[test] + fn decode_u16_runs_matches_eager() { + // Near-constant u16 levels (the all-null shape): a few long runs. + let mut levels: Vec = vec![1u16; 1000]; + levels.extend(std::iter::repeat_n(0u16, 500)); + levels.extend(std::iter::repeat_n(2u16, 300)); + let num_values = levels.len() as u64; + + let block = DataBlock::FixedWidth(FixedWidthDataBlock { + data: LanceBuffer::reinterpret_slice(Arc::from(levels.clone())), + bits_per_value: 16, + num_values, + block_info: BlockInfo::new(), + }); + let frame = BlockCompressor::compress(&RleEncoder::new(), block).unwrap(); + + let eager = + BlockDecompressor::decompress(&RleDecompressor::new(16), frame.clone(), num_values) + .unwrap(); + let DataBlock::FixedWidth(eager) = eager else { + panic!("expected fixed-width block"); + }; + assert_eq!( + eager.data.borrow_to_typed_slice::().as_ref(), + levels.as_slice() + ); + + // Lazy run form preserves boundaries and expands identically. + let runs = RleDecompressor::new(16) + .decode_u16_runs(frame, num_values) + .unwrap(); + assert_eq!(runs.num_values(), num_values as usize); + // The encoder splits each value into <=255-length runs (4 + 2 + 2 = 8 + // on-disk runs here); the scan identifies 3 coalesced logical runs. + assert_eq!(runs.num_runs(), 8); + assert_eq!(runs.coalesced_runs(), 3); + assert_eq!(expand_u16_runs(&runs), levels); + } + + #[test] + fn decode_u16_runs_empty() { + let mut empty_frame = Vec::new(); + empty_frame.extend_from_slice(&0u64.to_le_bytes()); + let runs = RleDecompressor::new(16) + .decode_u16_runs(LanceBuffer::from(empty_frame), 0) + .unwrap(); + assert_eq!(runs.num_values(), 0); + assert_eq!(runs.num_runs(), 0); + assert_eq!(runs.coalesced_runs(), 0); + + let error = RleDecompressor::new(16) + .decode_u16_runs(LanceBuffer::empty(), 0) + .unwrap_err(); + assert!(error.to_string().contains("Insufficient data size: 0")); + } + + #[rstest] + #[case::zero(0, 1, "zero run length")] + #[case::underflow(1, 2, "produced 1 values, expected 2")] + #[case::overflow(2, 1, "overflowed expected value count")] + #[case::nonempty_for_empty(1, 0, "overflowed expected value count")] + fn decode_u16_runs_rejects_invalid_coverage( + #[case] run_length: u8, + #[case] num_values: u64, + #[case] expected_message: &str, + ) { + let mut frame = Vec::new(); + frame.extend_from_slice(&2u64.to_le_bytes()); + frame.extend_from_slice(&7u16.to_le_bytes()); + frame.push(run_length); + + let error = RleDecompressor::new(16) + .decode_u16_runs(LanceBuffer::from(frame), num_values) + .unwrap_err(); + assert!(matches!(error, lance_core::Error::InvalidInput { .. })); + assert!(error.to_string().contains(expected_message)); + } + + #[test] + #[cfg(any(feature = "lz4", feature = "zstd"))] + fn decode_u16_runs_supports_compressed_values_child() { + let levels: Vec = (0..1024) + .flat_map(|run| std::iter::repeat_n((run % 8) as u16, 4)) + .collect(); + let num_values = levels.len() as u64; + let block = DataBlock::FixedWidth(FixedWidthDataBlock { + data: LanceBuffer::reinterpret_slice(Arc::from(levels.clone())), + bits_per_value: 16, + num_values, + block_info: BlockInfo::new(), + }); + let frame = BlockCompressor::compress(&RleEncoder::new(), block).unwrap(); + let (values, lengths) = parse_rle_block_frame(&frame).unwrap(); + + let compression = test_general_compression(); + let compressor = GeneralBufferCompressor::get_compressor(compression).unwrap(); + let mut compressed_values = Vec::new(); + compressor + .compress(values.as_ref(), &mut compressed_values) + .unwrap(); + let mut compressed_frame = Vec::new(); + compressed_frame.extend_from_slice(&(compressed_values.len() as u64).to_le_bytes()); + compressed_frame.extend_from_slice(&compressed_values); + compressed_frame.extend_from_slice(lengths.as_ref()); + + let encoding = ProtobufUtils21::rle( + ProtobufUtils21::wrapped(compression, ProtobufUtils21::flat(16, None)).unwrap(), + ProtobufUtils21::flat(8, None), + ); + let decompressor = create_rle_decompressor( + expect_rle(&encoding), + &DefaultDecompressionStrategy::default(), + ) + .unwrap(); + let runs = decompressor + .decode_u16_runs(LanceBuffer::from(compressed_frame), num_values) + .unwrap(); + assert_eq!(expand_u16_runs(&runs), levels); + } + + #[test] + fn decode_u16_runs_counts_coalesced_runs() { + // A logically constant page is emitted as ceil(N / 255) equal-valued + // runs (the encoder caps run lengths at 255); the validated view records + // that they can collapse to a single logical run. + let num_values = 5000u64; + let constant: Vec = vec![7u16; num_values as usize]; + let block = DataBlock::FixedWidth(FixedWidthDataBlock { + data: LanceBuffer::reinterpret_slice(Arc::from(constant)), + bits_per_value: 16, + num_values, + block_info: BlockInfo::new(), + }); + let frame = BlockCompressor::compress(&RleEncoder::new(), block).unwrap(); + let runs = RleDecompressor::new(16) + .decode_u16_runs(frame, num_values) + .unwrap(); + assert_eq!( + runs.num_runs(), + num_values.div_ceil(u8::MAX as u64) as usize + ); + assert_eq!(runs.coalesced_runs(), 1); + assert_eq!(expand_u16_runs(&runs), vec![7u16; num_values as usize]); + + // Distinct adjacent values must not be merged: alternating single-value + // runs stay separate and still expand to the original. + let alternating: Vec = (0..200u16).map(|i| i % 2).collect(); + let n = alternating.len() as u64; + let block = DataBlock::FixedWidth(FixedWidthDataBlock { + data: LanceBuffer::reinterpret_slice(Arc::from(alternating.clone())), + bits_per_value: 16, + num_values: n, + block_info: BlockInfo::new(), + }); + let frame = BlockCompressor::compress(&RleEncoder::new(), block).unwrap(); + let runs = RleDecompressor::new(16).decode_u16_runs(frame, n).unwrap(); + assert_eq!( + runs.coalesced_runs() as u64, + n, + "no two adjacent values are equal" + ); + assert_eq!(expand_u16_runs(&runs), alternating); + } + // ========== Core Functionality Tests ========== #[test] @@ -977,7 +2052,7 @@ mod tests { let array = Int32Array::from(vec![1, 1, 1, 2, 2, 3, 3, 3, 3]); let data_block = DataBlock::from_array(array); - let (compressed, _) = MiniBlockCompressor::compress(&encoder, data_block).unwrap(); + let (compressed, _) = compress_miniblock(&encoder, data_block).unwrap(); assert_eq!(compressed.num_values, 9); assert_eq!(compressed.chunks.len(), 1); @@ -998,8 +2073,7 @@ mod tests { data.extend(&[100i32; 300]); // Will be split into 255+45 let array = Int32Array::from(data); - let (compressed, _) = - MiniBlockCompressor::compress(&encoder, DataBlock::from_array(array)).unwrap(); + let (compressed, _) = compress_miniblock(&encoder, DataBlock::from_array(array)).unwrap(); // Should have 6 runs total (4 for first value, 2 for second) let lengths_buffer = &compressed.data[1]; @@ -1013,7 +2087,7 @@ mod tests { let data = vec![42i32; 1000]; let array = Int32Array::from(data); let (compressed, encoding) = - MiniBlockCompressor::compress(&encoder, DataBlock::from_array(array)).unwrap(); + compress_miniblock(&encoder, DataBlock::from_array(array)).unwrap(); assert_eq!(compressed.data[0].len(), 4); assert_eq!(compressed.data[1].len(), 2); @@ -1046,6 +2120,323 @@ mod tests { } } + #[test] + #[cfg(any(feature = "lz4", feature = "zstd"))] + fn test_rle_miniblock_compressed_values_child() { + let compression = test_general_compression(); + let encoder = + RleEncoder::with_child_encoding(RunLengthWidth::U8, Some(compression), None, false); + let array = Int32Array::from(repeating_runs(1024, 4)); + let (compressed, encoding) = + compress_miniblock(&encoder, DataBlock::from_array(array)).unwrap(); + + let rle = expect_rle(&encoding); + assert!(matches!( + rle.values.as_ref().unwrap().compression.as_ref().unwrap(), + crate::format::pb21::compressive_encoding::Compression::General(_) + )); + assert!(matches!( + rle.run_lengths + .as_ref() + .unwrap() + .compression + .as_ref() + .unwrap(), + crate::format::pb21::compressive_encoding::Compression::Flat(_) + )); + + let decompressor = DefaultDecompressionStrategy::default() + .create_miniblock_decompressor(&encoding, &DefaultDecompressionStrategy::default()) + .unwrap(); + let decoded = + MiniBlockDecompressor::decompress(decompressor.as_ref(), compressed.data, 1024 * 4) + .unwrap(); + assert_decoded_i32_eq(decoded, &repeating_runs(1024, 4)); + } + + #[test] + #[cfg(any(feature = "lz4", feature = "zstd"))] + fn test_rle_miniblock_compressed_run_lengths_child() { + let compression = test_general_compression(); + let encoder = + RleEncoder::with_child_encoding(RunLengthWidth::U8, None, Some(compression), false); + let expected = repeating_runs(1024, 4); + let (compressed, encoding) = compress_miniblock( + &encoder, + DataBlock::from_array(Int32Array::from(expected.clone())), + ) + .unwrap(); + + let rle = expect_rle(&encoding); + assert!(matches!( + rle.values.as_ref().unwrap().compression.as_ref().unwrap(), + crate::format::pb21::compressive_encoding::Compression::Flat(_) + )); + assert!(matches!( + rle.run_lengths + .as_ref() + .unwrap() + .compression + .as_ref() + .unwrap(), + crate::format::pb21::compressive_encoding::Compression::General(_) + )); + + let decompressor = DefaultDecompressionStrategy::default() + .create_miniblock_decompressor(&encoding, &DefaultDecompressionStrategy::default()) + .unwrap(); + let decoded = + MiniBlockDecompressor::decompress(decompressor.as_ref(), compressed.data, 1024 * 4) + .unwrap(); + assert_decoded_i32_eq(decoded, &expected); + } + + #[test] + #[cfg(feature = "bitpacking")] + fn test_rle_miniblock_bitpacked_run_lengths_child() { + use crate::encodings::physical::bitpacking::OutOfLineBitpacking; + + let expected = repeating_runs(1024, 4); + let (compressed, _) = compress_miniblock( + &RleEncoder::new(), + DataBlock::from_array(Int32Array::from(expected.clone())), + ) + .unwrap(); + let run_lengths = compressed.data[1].clone(); + let num_runs = run_lengths.len() as u64; + let run_lengths_block = DataBlock::FixedWidth(FixedWidthDataBlock { + bits_per_value: 8, + data: run_lengths, + num_values: num_runs, + block_info: BlockInfo::default(), + }); + let bitpacked_run_lengths = + BlockCompressor::compress(&OutOfLineBitpacking::new(3, 8), run_lengths_block).unwrap(); + let encoding = ProtobufUtils21::rle( + ProtobufUtils21::flat(32, None), + ProtobufUtils21::out_of_line_bitpacking(8, ProtobufUtils21::flat(3, None)), + ); + + let decompressor = DefaultDecompressionStrategy::default() + .create_miniblock_decompressor(&encoding, &DefaultDecompressionStrategy::default()) + .unwrap(); + let decoded = MiniBlockDecompressor::decompress( + decompressor.as_ref(), + vec![compressed.data[0].clone(), bitpacked_run_lengths], + expected.len() as u64, + ) + .unwrap(); + assert_decoded_i32_eq(decoded, &expected); + } + + #[test] + #[cfg(feature = "bitpacking")] + fn test_rle_rejects_two_count_dependent_child_encodings() { + let encoding = ProtobufUtils21::rle( + ProtobufUtils21::out_of_line_bitpacking(32, ProtobufUtils21::flat(3, None)), + ProtobufUtils21::out_of_line_bitpacking(8, ProtobufUtils21::flat(3, None)), + ); + + let err = DefaultDecompressionStrategy::default() + .create_miniblock_decompressor(&encoding, &DefaultDecompressionStrategy::default()) + .unwrap_err(); + assert!( + err.to_string() + .contains("cannot both require the run count") + ); + } + + #[cfg(any(feature = "lz4", feature = "zstd"))] + fn test_general_compression() -> CompressionConfig { + if cfg!(feature = "zstd") { + CompressionConfig::new(CompressionScheme::Zstd, Some(3)) + } else { + CompressionConfig::new(CompressionScheme::Lz4, None) + } + } + + fn repeating_runs(num_runs: usize, run_length: usize) -> Vec { + let mut values = Vec::with_capacity(num_runs * run_length); + for run in 0..num_runs { + values.extend(std::iter::repeat_n((run % 8) as i32, run_length)); + } + values + } + + fn expect_rle(encoding: &CompressiveEncoding) -> &crate::format::pb21::Rle { + match encoding.compression.as_ref().unwrap() { + crate::format::pb21::compressive_encoding::Compression::Rle(rle) => rle, + other => panic!("expected RLE encoding, got {other:?}"), + } + } + + fn assert_decoded_i32_eq(decoded: DataBlock, expected: &[i32]) { + match decoded { + DataBlock::FixedWidth(block) => { + let values = block.data.borrow_to_typed_slice::(); + assert_eq!(values.as_ref(), expected); + } + _ => panic!("Expected FixedWidth block"), + } + } + + #[test] + #[cfg(any(feature = "lz4", feature = "zstd"))] + fn test_rle_miniblock_compressed_children_multiple_chunks() { + let compression = test_general_compression(); + let encoder = RleEncoder::with_child_encoding( + RunLengthWidth::U8, + Some(compression), + Some(compression), + false, + ); + let expected = repeating_runs(8192, 4); + let (compressed, encoding) = compress_miniblock( + &encoder, + DataBlock::from_array(Int32Array::from(expected.clone())), + ) + .unwrap(); + + assert!(compressed.chunks.len() > 1); + let rle = expect_rle(&encoding); + assert!(matches!( + rle.values.as_ref().unwrap().compression.as_ref().unwrap(), + crate::format::pb21::compressive_encoding::Compression::General(_) + )); + assert!(matches!( + rle.run_lengths + .as_ref() + .unwrap() + .compression + .as_ref() + .unwrap(), + crate::format::pb21::compressive_encoding::Compression::General(_) + )); + + let decoded = decompress_i32_chunks(&compressed, &encoding); + assert_eq!(decoded, expected); + } + + #[test] + #[cfg(feature = "bitpacking")] + fn test_rle_miniblock_bitpacks_values_child_when_smaller() { + let encoder = RleEncoder::with_child_encoding(RunLengthWidth::U8, None, None, true); + let expected = monotonic_runs(2048, 4); + let (compressed, encoding) = compress_miniblock( + &encoder, + DataBlock::from_array(Int32Array::from(expected.clone())), + ) + .unwrap(); + + let rle = expect_rle(&encoding); + assert!(matches!( + rle.values.as_ref().unwrap().compression.as_ref().unwrap(), + crate::format::pb21::compressive_encoding::Compression::OutOfLineBitpacking(_) + )); + assert!(matches!( + rle.run_lengths + .as_ref() + .unwrap() + .compression + .as_ref() + .unwrap(), + crate::format::pb21::compressive_encoding::Compression::Flat(_) + )); + + let decoded = decompress_i32_chunks(&compressed, &encoding); + assert_eq!(decoded, expected); + } + + #[test] + #[cfg(feature = "bitpacking")] + fn test_rle_miniblock_bitpacks_run_lengths_when_values_do_not_shrink() { + let encoder = RleEncoder::with_child_encoding(RunLengthWidth::U8, None, None, true); + let expected = high_entropy_runs(2048, 4); + let (compressed, encoding) = compress_miniblock( + &encoder, + DataBlock::from_array(Int32Array::from(expected.clone())), + ) + .unwrap(); + + let rle = expect_rle(&encoding); + assert!(matches!( + rle.values.as_ref().unwrap().compression.as_ref().unwrap(), + crate::format::pb21::compressive_encoding::Compression::Flat(_) + )); + assert!(matches!( + rle.run_lengths + .as_ref() + .unwrap() + .compression + .as_ref() + .unwrap(), + crate::format::pb21::compressive_encoding::Compression::OutOfLineBitpacking(_) + )); + + let decoded = decompress_i32_chunks(&compressed, &encoding); + assert_eq!(decoded, expected); + } + + fn decompress_i32_chunks( + compressed: &MiniBlockCompressed, + encoding: &CompressiveEncoding, + ) -> Vec { + let strategy = DefaultDecompressionStrategy::default(); + let decompressor = strategy + .create_miniblock_decompressor(encoding, &strategy) + .unwrap(); + let mut offsets = vec![0usize; compressed.data.len()]; + let mut values_processed = 0u64; + let mut decoded_values = Vec::new(); + + for chunk in &compressed.chunks { + let chunk_values = chunk.num_values(values_processed, compressed.num_values); + let mut chunk_buffers = Vec::with_capacity(chunk.buffer_sizes.len()); + for (idx, size) in chunk.buffer_sizes.iter().enumerate() { + let size = *size as usize; + chunk_buffers.push(compressed.data[idx].slice_with_length(offsets[idx], size)); + offsets[idx] += size; + } + + let decoded = decompressor + .decompress(chunk_buffers, chunk_values) + .unwrap(); + match decoded { + DataBlock::FixedWidth(block) => { + let values = block.data.borrow_to_typed_slice::(); + decoded_values.extend_from_slice(values.as_ref()); + } + _ => panic!("Expected FixedWidth block"), + } + values_processed += chunk_values; + } + + assert_eq!(values_processed, compressed.num_values); + decoded_values + } + + #[cfg(feature = "bitpacking")] + fn monotonic_runs(num_runs: usize, run_length: usize) -> Vec { + let mut values = Vec::with_capacity(num_runs * run_length); + for run in 0..num_runs { + values.extend(std::iter::repeat_n(run as i32, run_length)); + } + values + } + + #[cfg(feature = "bitpacking")] + fn high_entropy_runs(num_runs: usize, run_length: usize) -> Vec { + let mut values = Vec::with_capacity(num_runs * run_length); + let mut state = 7u64; + for _ in 0..num_runs { + state = state + .wrapping_mul(6364136223846793005) + .wrapping_add(1442695040888963407); + values.extend(std::iter::repeat_n((state >> 32) as i32, run_length)); + } + values + } + #[test] fn test_select_run_length_width_prefers_u16_for_long_runs() { let mut entries = [0u64; 3]; @@ -1089,7 +2480,7 @@ mod tests { block_info: BlockInfo::default(), }); - let (compressed, _) = MiniBlockCompressor::compress(&encoder, block).unwrap(); + let (compressed, _) = compress_miniblock(&encoder, block).unwrap(); let decompressor = RleDecompressor::new(bits_per_value); let decompressed = MiniBlockDecompressor::decompress( &decompressor, @@ -1123,7 +2514,7 @@ mod tests { let array = Int32Array::from(data); let (compressed, _) = - MiniBlockCompressor::compress(&encoder, DataBlock::from_array(array)).unwrap(); + compress_miniblock(&encoder, DataBlock::from_array(array)).unwrap(); // Verify all non-last chunks have power-of-2 values for (i, chunk) in compressed.chunks.iter().enumerate() { @@ -1139,8 +2530,73 @@ mod tests { } } + #[rstest] + #[case::u8_lengths(RunLengthWidth::U8)] + #[case::u16_lengths(RunLengthWidth::U16)] + #[case::u32_lengths(RunLengthWidth::U32)] + fn test_miniblock_chunk_counts_match_encoded_runs(#[case] run_length_width: RunLengthWidth) { + // This pattern crosses the 2,048-value boundary in the middle of a two-value run. + let levels = (0..4098) + .map(|index| if index % 3 == 0 { 1u16 } else { 0u16 }) + .collect::>(); + let num_values = levels.len() as u64; + let encoder = RleEncoder::with_run_length_width(run_length_width); + let (buffers, chunks) = encoder + .encode_data( + &LanceBuffer::reinterpret_vec(levels), + num_values, + u16::BITS as u64, + ) + .unwrap(); + + assert_eq!(buffers.len(), 2); + let bytes_per_length = run_length_width.bytes_per_value(); + let mut values_offset = 0usize; + let mut lengths_offset = 0usize; + let mut values_processed = 0u64; + + for chunk in &chunks { + let values_size = chunk.buffer_sizes[0] as usize; + let lengths_size = chunk.buffer_sizes[1] as usize; + let lengths_end = lengths_offset + lengths_size; + let chunk_lengths = &buffers[1].as_ref()[lengths_offset..lengths_end]; + let length_chunks = chunk_lengths.chunks_exact(bytes_per_length); + assert!(length_chunks.remainder().is_empty()); + let num_runs = length_chunks.len(); + let encoded_values = length_chunks + .map(|bytes| run_length_width.read_length(bytes)) + .sum::(); + let declared_values = chunk.num_values(values_processed, num_values); + + assert_eq!(values_size, num_runs * size_of::()); + assert_eq!(encoded_values, declared_values); + + values_offset += values_size; + lengths_offset = lengths_end; + values_processed += declared_values; + } + + assert_eq!(values_processed, num_values); + assert_eq!(values_offset, buffers[0].len()); + assert_eq!(lengths_offset, buffers[1].len()); + } + // ========== Error Handling Tests ========== + #[test] + fn test_encoder_rejects_zero_progress() { + let error = RleEncoder::new() + .encode_data(&LanceBuffer::empty(), 1, u16::BITS as u64) + .unwrap_err(); + + assert!( + matches!(&error, Error::Internal { .. }), + "expected internal error, got: {error:?}" + ); + assert!(error.to_string().contains("made no progress")); + assert!(error.to_string().contains("values_remaining=1")); + } + #[test] fn test_invalid_buffer_count() { let decompressor = RleDecompressor::new(32); @@ -1189,7 +2645,7 @@ mod tests { } #[test] - fn test_rle_rejects_underflow_overflow_and_zero_lengths() { + fn test_rle_rejects_underflow_and_zero_lengths_and_clamps_overflow() { let decompressor = RleDecompressor::with_run_length_width(32, RunLengthWidth::U16); let value = LanceBuffer::from(1i32.to_le_bytes().to_vec()); @@ -1212,12 +2668,15 @@ mod tests { ], 5, ) - .unwrap_err(); - assert!( - overflow - .to_string() - .contains("overflowed expected value count") - ); + .unwrap(); + match overflow { + DataBlock::FixedWidth(block) => { + assert_eq!(block.num_values, 5); + let decoded = block.data.borrow_to_typed_slice::(); + assert_eq!(decoded.as_ref(), &[1i32; 5]); + } + _ => panic!("Expected FixedWidth block"), + } let zero = MiniBlockDecompressor::decompress( &decompressor, @@ -1228,6 +2687,60 @@ mod tests { assert!(zero.to_string().contains("zero run length")); } + #[test] + fn test_block_rle_rejects_overflow() { + // Block payloads have no chunk boundaries, so run lengths summing past + // num_values can only be corruption and must stay a hard error. + let decompressor = RleDecompressor::with_run_length_width(32, RunLengthWidth::U16); + let values = 1i32.to_le_bytes(); + let lengths = 6u16.to_le_bytes(); + let mut payload = Vec::new(); + payload.extend_from_slice(&(values.len() as u64).to_le_bytes()); + payload.extend_from_slice(&values); + payload.extend_from_slice(&lengths); + + let error = BlockDecompressor::decompress(&decompressor, LanceBuffer::from(payload), 5) + .unwrap_err(); + assert!(matches!(&error, Error::InvalidInput { .. })); + assert!( + error + .to_string() + .contains("overflowed expected value count") + ); + } + + #[test] + fn test_rle_truncates_legacy_chunk_boundary_overflow() { + // Legacy encoders emitted chunks declaring 2048 values whose final run crossed + // the checkpoint boundary (e.g. run lengths summing to 2080); the excess values + // are duplicated at the start of the next chunk and must be ignored here. + let decompressor = RleDecompressor::with_run_length_width(32, RunLengthWidth::U16); + let mut values = Vec::new(); + values.extend_from_slice(&7i32.to_le_bytes()); + values.extend_from_slice(&8i32.to_le_bytes()); + let mut lengths = Vec::new(); + lengths.extend_from_slice(&2000u16.to_le_bytes()); + lengths.extend_from_slice(&80u16.to_le_bytes()); + + let decoded = MiniBlockDecompressor::decompress( + &decompressor, + vec![LanceBuffer::from(values), LanceBuffer::from(lengths)], + 2048, + ) + .unwrap(); + match decoded { + DataBlock::FixedWidth(block) => { + assert_eq!(block.num_values, 2048); + let decoded = block.data.borrow_to_typed_slice::(); + let decoded = decoded.as_ref(); + assert_eq!(decoded.len(), 2048); + assert!(decoded[..2000].iter().all(|&v| v == 7)); + assert!(decoded[2000..].iter().all(|&v| v == 8)); + } + _ => panic!("Expected FixedWidth block"), + } + } + #[test] fn test_empty_data_handling() { let encoder = RleEncoder::new(); @@ -1240,7 +2753,7 @@ mod tests { block_info: BlockInfo::default(), }); - let (compressed, _) = MiniBlockCompressor::compress(&encoder, empty_block).unwrap(); + let (compressed, _) = compress_miniblock(&encoder, empty_block).unwrap(); assert_eq!(compressed.num_values, 0); assert!(compressed.data.is_empty()); @@ -1274,8 +2787,7 @@ mod tests { data.extend(vec![777i32; 2000]); let array = Int32Array::from(data.clone()); - let (compressed, _) = - MiniBlockCompressor::compress(&encoder, DataBlock::from_array(array)).unwrap(); + let (compressed, _) = compress_miniblock(&encoder, DataBlock::from_array(array)).unwrap(); // Manually decompress all chunks let mut reconstructed = Vec::new(); @@ -1388,7 +2900,7 @@ mod tests { // Compress the data let array = Int32Array::from(data.clone()); let (compressed, _) = - MiniBlockCompressor::compress(&encoder, DataBlock::from_array(array)).unwrap(); + compress_miniblock(&encoder, DataBlock::from_array(array)).unwrap(); // Decompress and verify match MiniBlockDecompressor::decompress( @@ -1458,7 +2970,7 @@ mod tests { block_info: BlockInfo::default(), }); - let (compressed, _) = MiniBlockCompressor::compress(&encoder, block).unwrap(); + let (compressed, _) = compress_miniblock(&encoder, block).unwrap(); // Debug first few chunks for (i, chunk) in compressed.chunks.iter().take(5).enumerate() { @@ -1508,7 +3020,6 @@ mod tests { #[test_log::test(tokio::test)] async fn test_rle_encoding_verification() { use crate::testing::{TestCases, check_round_trip_encoding_of_data}; - use crate::version::LanceFileVersion; use arrow_array::{Array, Int32Array}; use lance_datagen::{ArrayGenerator, RowCount}; use std::collections::HashMap; @@ -1516,7 +3027,7 @@ mod tests { let test_cases = TestCases::default() .with_expected_encoding("rle") - .with_min_file_version(LanceFileVersion::V2_1); + .with_structural_encodings(); // Test both explicit metadata and automatic selection // 1. Test with explicit RLE threshold metadata (also disable BSS) @@ -1566,6 +3077,30 @@ mod tests { ); // 20% variety let arr = Arc::new(Int32Array::from(values)) as Arc; check_round_trip_encoding_of_data(vec![arr], &test_cases, metadata).await; + + #[cfg(any(feature = "lz4", feature = "zstd"))] + { + let mut metadata = HashMap::new(); + metadata.insert( + "lance-encoding:rle-threshold".to_string(), + "0.8".to_string(), + ); + metadata.insert("lance-encoding:bss".to_string(), "off".to_string()); + metadata.insert( + "lance-encoding:compression".to_string(), + if cfg!(feature = "zstd") { + "zstd".to_string() + } else { + "lz4".to_string() + }, + ); + let mut values = Vec::with_capacity(2048 * 4); + for run in 0..2048 { + values.extend(std::iter::repeat_n(i32::MIN + (run % 8), 4)); + } + let arr = Arc::new(Int32Array::from(values)) as Arc; + check_round_trip_encoding_of_data(vec![arr], &test_cases, metadata).await; + } } /// Generator that produces repetitive patterns suitable for RLE diff --git a/rust/lance-encoding/src/encodings/physical/value.rs b/rust/lance-encoding/src/encodings/physical/value.rs index c49bbd3efbd..0284e9784c6 100644 --- a/rust/lance-encoding/src/encodings/physical/value.rs +++ b/rust/lance-encoding/src/encodings/physical/value.rs @@ -13,7 +13,7 @@ use crate::data::{ use crate::encodings::logical::primitive::fullzip::{PerValueCompressor, PerValueDataBlock}; use crate::encodings::logical::primitive::miniblock::{ MAX_MINIBLOCK_BYTES, MAX_MINIBLOCK_VALUES, MiniBlockChunk, MiniBlockCompressed, - MiniBlockCompressor, + MiniBlockCompressionContext, MiniBlockCompressor, }; use crate::format::ProtobufUtils21; use crate::format::pb21::compressive_encoding::Compression; @@ -27,7 +27,7 @@ pub struct ValueEncoder {} impl ValueEncoder { /// Use the largest chunk we can smaller than 4KiB - fn find_log_vals_per_chunk(bytes_per_word: u64, values_per_word: u64) -> (u64, u64) { + fn find_log_vals_per_chunk(bytes_per_word: u64, values_per_word: u64) -> Result<(u64, u64)> { let mut size_bytes = 2 * bytes_per_word; let (mut log_num_vals, mut num_vals) = match values_per_word { 1 => (1, 2), @@ -35,8 +35,14 @@ impl ValueEncoder { _ => unreachable!(), }; - // If the type is so wide that we can't even fit 2 values we shouldn't be here - assert!(size_bytes < MAX_MINIBLOCK_BYTES); + if size_bytes >= MAX_MINIBLOCK_BYTES { + let num_values = 2 * values_per_word; + return Err(Error::invalid_input(format!( + "Value is too wide for miniblock encoding: {} values require {} bytes but a \ + miniblock chunk is limited to {} bytes.", + num_values, size_bytes, MAX_MINIBLOCK_BYTES + ))); + } while 2 * size_bytes < MAX_MINIBLOCK_BYTES && 2 * num_vals <= *MAX_MINIBLOCK_VALUES { log_num_vals += 1; @@ -44,10 +50,10 @@ impl ValueEncoder { num_vals *= 2; } - (log_num_vals, num_vals) + Ok((log_num_vals, num_vals)) } - fn chunk_data(data: FixedWidthDataBlock) -> MiniBlockCompressed { + fn chunk_data(data: FixedWidthDataBlock) -> Result { // Usually there are X bytes per value. However, when working with boolean // or FSL we might have some number of bits per value that isn't // divisible by 8. In this case, to avoid chunking in the middle of a byte @@ -60,7 +66,7 @@ impl ValueEncoder { // Aim for 4KiB chunks let (log_vals_per_chunk, vals_per_chunk) = - Self::find_log_vals_per_chunk(bytes_per_word, values_per_word); + Self::find_log_vals_per_chunk(bytes_per_word, values_per_word)?; let num_chunks = bit_util::ceil(data.num_values as usize, vals_per_chunk as usize); debug_assert_eq!(vals_per_chunk % values_per_word, 0); let bytes_per_chunk = bytes_per_word * (vals_per_chunk / values_per_word); @@ -99,11 +105,11 @@ impl ValueEncoder { debug_assert_eq!(chunks.len(), num_chunks); - MiniBlockCompressed { + Ok(MiniBlockCompressed { chunks, data: vec![data_buffer], num_values: data.num_values, - } + }) } } @@ -177,7 +183,7 @@ impl ValueEncoder { data: FixedWidthDataBlock, layers: Vec, num_rows: u64, - ) -> (MiniBlockCompressed, CompressiveEncoding) { + ) -> Result<(MiniBlockCompressed, CompressiveEncoding)> { // Count size to calculate rows per chunk let mut ceil_bytes_validity = 0; let mut cum_dim = 1; @@ -198,7 +204,7 @@ impl ValueEncoder { }; let est_bytes_per_word = (ceil_bytes_validity * vals_per_word) + cum_bytes_per_word; let (log_rows_per_chunk, rows_per_chunk) = - Self::find_log_vals_per_chunk(est_bytes_per_word, vals_per_word); + Self::find_log_vals_per_chunk(est_bytes_per_word, vals_per_word)?; let num_chunks = num_rows.div_ceil(rows_per_chunk) as usize; @@ -258,17 +264,17 @@ impl ValueEncoder { .chain(std::iter::once(data.data)) .collect::>(); - ( + Ok(( MiniBlockCompressed { chunks, data: buffers, num_values: num_rows, }, encoding, - ) + )) } - fn miniblock_fsl(data: DataBlock) -> (MiniBlockCompressed, CompressiveEncoding) { + fn miniblock_fsl(data: DataBlock) -> Result<(MiniBlockCompressed, CompressiveEncoding)> { let num_rows = data.num_values(); let fsl = data.as_fixed_size_list().unwrap(); let mut layers = Vec::new(); @@ -465,13 +471,17 @@ impl BlockCompressor for ValueEncoder { } impl MiniBlockCompressor for ValueEncoder { - fn compress(&self, chunk: DataBlock) -> Result<(MiniBlockCompressed, CompressiveEncoding)> { + fn compress( + &self, + _context: MiniBlockCompressionContext, + chunk: DataBlock, + ) -> Result<(MiniBlockCompressed, CompressiveEncoding)> { match chunk { DataBlock::FixedWidth(fixed_width) => { let encoding = ProtobufUtils21::flat(fixed_width.bits_per_value, None); - Ok((Self::chunk_data(fixed_width), encoding)) + Ok((Self::chunk_data(fixed_width)?, encoding)) } - DataBlock::FixedSizeList(_) => Ok(Self::miniblock_fsl(chunk)), + DataBlock::FixedSizeList(_) => Self::miniblock_fsl(chunk), _ => Err(Error::invalid_input_source( format!( "Cannot compress a data block of type {} with ValueEncoder", @@ -600,6 +610,15 @@ impl MiniBlockDecompressor for ValueDecompressor { assert_eq!(lists.num_values(), num_values); Ok(lists) } + + fn decoded_size_bytes(&self, num_values: u64) -> Option { + if self.has_validity() { + return None; + } + num_values + .checked_mul(self.bits_per_value) + .map(|bits| bits.div_ceil(8)) + } } struct FslDecompressorValidityBuilder { @@ -728,6 +747,15 @@ impl FixedPerValueDecompressor for ValueDecompressor { fn bits_per_value(&self) -> u64 { self.bits_per_value } + + fn decoded_size_bytes(&self, num_values: u64) -> Option { + if self.has_validity() { + return None; + } + num_values + .checked_mul(self.bits_per_value) + .map(|bits| bits.div_ceil(8)) + } } impl PerValueCompressor for ValueEncoder { @@ -769,7 +797,7 @@ mod tests { encodings::{ logical::primitive::{ fullzip::{PerValueCompressor, PerValueDataBlock}, - miniblock::MiniBlockCompressor, + miniblock::{MiniBlockCompressionContext, MiniBlockCompressor}, }, physical::value::ValueDecompressor, }, @@ -778,11 +806,14 @@ mod tests { FnArrayGeneratorProvider, TestCases, check_basic_random, check_round_trip_encoding_generated, check_round_trip_encoding_of_data, }, - version::LanceFileVersion, }; use super::ValueEncoder; + fn miniblock_context() -> MiniBlockCompressionContext { + MiniBlockCompressionContext::new(0, true, true) + } + const PRIMITIVE_TYPES: &[DataType] = &[ DataType::Null, DataType::FixedSizeBinary(2), @@ -828,7 +859,7 @@ mod tests { .with_indices(vec![0, 1, 2]) .with_indices(vec![1]) .with_indices(vec![2]) - .with_min_file_version(LanceFileVersion::V2_1); + .with_structural_encodings(); check_round_trip_encoding_of_data(vec![items], &test_cases, HashMap::default()).await; } @@ -839,7 +870,7 @@ mod tests { (0..5000).map(|i| if i % 2 == 0 { Some(i) } else { None }), )); - let test_cases = TestCases::default().with_min_file_version(LanceFileVersion::V2_1); + let test_cases = TestCases::default().with_structural_encodings(); check_round_trip_encoding_of_data(vec![items], &test_cases, HashMap::default()).await; } @@ -871,7 +902,7 @@ mod tests { #[test_log::test(tokio::test)] async fn test_decimal128_dictionary_encoding() { - let test_cases = TestCases::default().with_min_file_version(LanceFileVersion::V2_1); + let test_cases = TestCases::default().with_structural_encodings(); let decimals: Vec = (0..100).collect(); let repeated_strings: Vec<_> = decimals .iter() @@ -917,7 +948,7 @@ mod tests { let test_cases = TestCases::default() .with_page_sizes(vec![1000, 2000, 3000, 60000]) .with_batch_size(batch_size) - .with_min_file_version(LanceFileVersion::V2_1); + .with_structural_encodings(); check_round_trip_encoding_of_data(data.clone(), &test_cases, HashMap::new()).await; } @@ -963,7 +994,8 @@ mod tests { let starting_data = DataBlock::from_array(sample_list.clone()); let encoder = ValueEncoder::default(); - let (data, compression) = MiniBlockCompressor::compress(&encoder, starting_data).unwrap(); + let (data, compression) = + MiniBlockCompressor::compress(&encoder, miniblock_context(), starting_data).unwrap(); assert_eq!(data.num_values, 3); assert_eq!(data.data.len(), 3); @@ -989,6 +1021,59 @@ mod tests { assert_eq!(decompressed.as_ref(), &sample_list); } + fn wide_fixed_size_binary() -> ArrayRef { + let wide_value = vec![0xABu8; 5000]; + Arc::new( + arrow_array::FixedSizeBinaryArray::try_from_sparse_iter_with_size( + std::iter::repeat_n(Some(wide_value.as_slice()), 4), + 5000, + ) + .unwrap(), + ) + } + + fn wide_fixed_size_list_bool() -> ArrayRef { + // A wide FSL is sub-byte, so it chunks eight values per word and the + // smallest unit is 16 values rather than 2. + let dimension = 4095; + let values = arrow_array::BooleanArray::from(vec![false; dimension * 2]); + let field = Arc::new(Field::new("item", DataType::Boolean, true)); + Arc::new(FixedSizeListArray::new( + field, + dimension as i32, + Arc::new(values), + None, + )) + } + + #[rstest::rstest] + #[case::fixed_size_binary(wide_fixed_size_binary(), 2)] + #[case::fixed_size_list_bool(wide_fixed_size_list_bool(), 16)] + fn test_wide_value_miniblock_returns_error( + #[case] array: ArrayRef, + #[case] expected_min_values: u64, + ) { + let starting_data = DataBlock::from_array(array); + + let encoder = ValueEncoder::default(); + let result = MiniBlockCompressor::compress(&encoder, miniblock_context(), starting_data); + + let err = result.expect_err("wide values should not be encodable as miniblock"); + assert!( + matches!(err, lance_core::Error::InvalidInput { .. }), + "expected InvalidInput, got {err:?}" + ); + let msg = err.to_string(); + assert!( + msg.contains("too wide for miniblock encoding"), + "unexpected error message: {msg}" + ); + assert!( + msg.contains(&format!("{expected_min_values} values require")), + "unexpected error message: {msg}" + ); + } + #[test] fn test_fsl_value_compression_per_value() { let sample_list = create_simple_fsl(); @@ -1013,6 +1098,11 @@ mod tests { let decompressor = ValueDecompressor::from_fsl(fsl.as_ref()); let num_values = data.num_values; + assert_eq!( + FixedPerValueDecompressor::decoded_size_bytes(&decompressor, num_values), + None, + "nullable FSL output uses multiple buffers and requires the fallback estimate" + ); let decompressed = FixedPerValueDecompressor::decompress(&decompressor, data, num_values).unwrap(); @@ -1033,7 +1123,7 @@ mod tests { let list_array = FixedSizeListArray::new(items_field, 2, items, Some(NullBuffer::new(list_nulls))); - let test_cases = TestCases::default().with_min_file_version(LanceFileVersion::V2_1); + let test_cases = TestCases::default().with_structural_encodings(); check_round_trip_encoding_of_data(vec![Arc::new(list_array)], &test_cases, HashMap::new()) .await; @@ -1051,7 +1141,7 @@ mod tests { let list_arr = ListArray::new(list_field, OffsetBuffer::new(offsets), Arc::new(fsl), None); let test_cases = TestCases::default() - .with_min_file_version(LanceFileVersion::V2_1) + .with_structural_encodings() .with_batch_size(1); check_round_trip_encoding_of_data(vec![Arc::new(list_arr)], &test_cases, HashMap::new()) @@ -1083,7 +1173,8 @@ mod tests { ); let encoder = ValueEncoder::default(); - let (data, compression) = MiniBlockCompressor::compress(&encoder, starting_data).unwrap(); + let (data, compression) = + MiniBlockCompressor::compress(&encoder, miniblock_context(), starting_data).unwrap(); let Compression::FixedSizeList(fsl) = compression.compression.unwrap() else { panic!() @@ -1158,7 +1249,7 @@ mod tests { let test_cases = TestCases::default() .with_expected_encoding("flat") - .with_min_file_version(LanceFileVersion::V2_1); + .with_structural_encodings(); // Test both explicit configuration and automatic fallback scenarios // 1. Test explicit "none" compression to force flat encoding diff --git a/rust/lance-encoding/src/lib.rs b/rust/lance-encoding/src/lib.rs index a58e0a14c59..338028b79d0 100644 --- a/rust/lance-encoding/src/lib.rs +++ b/rust/lance-encoding/src/lib.rs @@ -8,6 +8,7 @@ use futures::{FutureExt, TryFutureExt, future::BoxFuture}; use lance_core::Result; +mod array_encoding; pub mod buffer; pub mod compression; pub mod compression_config; @@ -17,13 +18,11 @@ pub mod decoder; pub mod encoder; pub mod encodings; pub mod format; -pub mod previous; pub mod repdef; pub mod statistics; #[cfg(test)] pub mod testing; pub mod utils; -pub mod version; // We can definitely add support for big-endian machines someday. However, it's not a priority and // would involve extensive testing (probably through emulation) to ensure that the encodings are diff --git a/rust/lance-encoding/src/previous.rs b/rust/lance-encoding/src/previous.rs deleted file mode 100644 index eb4e1bcdb0d..00000000000 --- a/rust/lance-encoding/src/previous.rs +++ /dev/null @@ -1,8 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// SPDX-FileCopyrightText: Copyright The Lance Authors - -//! Legacy code for the 2.0 format that is no longer used in 2.1+ - -pub mod decoder; -pub mod encoder; -pub mod encodings; diff --git a/rust/lance-encoding/src/previous/decoder.rs b/rust/lance-encoding/src/previous/decoder.rs deleted file mode 100644 index bf32bea3d7c..00000000000 --- a/rust/lance-encoding/src/previous/decoder.rs +++ /dev/null @@ -1,132 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// SPDX-FileCopyrightText: Copyright The Lance Authors - -use std::{collections::VecDeque, ops::Range}; - -use crate::decoder::{ - FilterExpression, NextDecodeTask, PriorityRange, ScheduledScanLine, SchedulerContext, -}; - -use arrow_schema::DataType; -use futures::future::BoxFuture; -use lance_core::{Error, Result}; - -pub trait SchedulingJob: std::fmt::Debug { - fn schedule_next( - &mut self, - context: &mut SchedulerContext, - priority: &dyn PriorityRange, - ) -> Result; - - fn num_rows(&self) -> u64; -} - -/// A scheduler for a field's worth of data -/// -/// Each field in a reader's output schema maps to one field scheduler. This scheduler may -/// map to more than one column. For example, one field of struct data may -/// cover many columns of child data. In fact, the entire file is treated as one -/// top-level struct field. -/// -/// The scheduler is responsible for calculating the necessary I/O. One schedule_range -/// request could trigger multiple batches of I/O across multiple columns. The scheduler -/// should emit decoders into the sink as quickly as possible. -/// -/// As soon as the scheduler encounters a batch of data that can decoded then the scheduler -/// should emit a decoder in the "unloaded" state. The decode stream will pull the decoder -/// and start decoding. -/// -/// The order in which decoders are emitted is important. Pages should be emitted in -/// row-major order allowing decode of complete rows as quickly as possible. -/// -/// The `FieldScheduler` should be stateless and `Send` and `Sync`. This is -/// because it might need to be shared. For example, a list page has a reference to -/// the field schedulers for its items column. This is shared with the follow-up I/O -/// task created when the offsets are loaded. -/// -/// See [`crate::decoder`] for more information -pub trait FieldScheduler: Send + Sync + std::fmt::Debug { - /// Called at the beginning of scheduling to initialize the scheduler - fn initialize<'a>( - &'a self, - filter: &'a FilterExpression, - context: &'a SchedulerContext, - ) -> BoxFuture<'a, Result<()>>; - /// Schedules I/O for the requested portions of the field. - /// - /// Note: `ranges` must be ordered and non-overlapping - /// TODO: Support unordered or overlapping ranges in file scheduler - fn schedule_ranges<'a>( - &'a self, - ranges: &[Range], - filter: &FilterExpression, - ) -> Result>; - /// The number of rows in this field - fn num_rows(&self) -> u64; -} - -#[derive(Debug)] -pub struct DecoderReady { - // The decoder that is ready to be decoded - pub decoder: Box, - // The path to the decoder, the first value is the column index - // following values, if present, are nested child indices - // - // For example, a path of [1, 1, 0] would mean to grab the second - // column, then the second child, and then the first child. - // - // It could represent x in the following schema: - // - // score: float64 - // points: struct - // color: string - // location: struct - // x: float64 - // - // Currently, only struct decoders have "children" although other - // decoders may at some point as well. List children are only - // handled through indirect I/O at the moment and so they don't - // need to be represented (yet) - pub path: VecDeque, -} - -/// A decoder for a field's worth of data -/// -/// The decoder is initially "unloaded" (doesn't have all its data). The [`Self::wait_for_loaded`] -/// method should be called to wait for the needed I/O data before attempting to decode -/// any further. -/// -/// Unlike the other decoder types it is assumed that `LogicalPageDecoder` is stateful -/// and only `Send`. This is why we don't need a `rows_to_skip` argument in [`Self::drain`] -pub trait LogicalPageDecoder: std::fmt::Debug + Send { - /// Add a newly scheduled child decoder - /// - /// The default implementation does not expect children and returns - /// an error. - fn accept_child(&mut self, _child: DecoderReady) -> Result<()> { - Err(Error::internal(format!( - "The decoder {:?} does not expect children but received a child", - self - ))) - } - /// Waits until at least `num_rows` have been loaded - fn wait_for_loaded(&'_ mut self, loaded_need: u64) -> BoxFuture<'_, Result<()>>; - /// The number of rows loaded so far - fn rows_loaded(&self) -> u64; - /// The number of rows that still need loading - fn rows_unloaded(&self) -> u64 { - self.num_rows() - self.rows_loaded() - } - /// The total number of rows in the field - fn num_rows(&self) -> u64; - /// The number of rows that have been drained so far - fn rows_drained(&self) -> u64; - /// The number of rows that are still available to drain - fn rows_left(&self) -> u64 { - self.num_rows() - self.rows_drained() - } - /// Creates a task to decode `num_rows` of data into an array - fn drain(&mut self, num_rows: u64) -> Result; - /// The data type of the decoded data - fn data_type(&self) -> &DataType; -} diff --git a/rust/lance-encoding/src/previous/encodings.rs b/rust/lance-encoding/src/previous/encodings.rs deleted file mode 100644 index 67a43993508..00000000000 --- a/rust/lance-encoding/src/previous/encodings.rs +++ /dev/null @@ -1,7 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// SPDX-FileCopyrightText: Copyright The Lance Authors - -//! Legacy code for the 2.0 format that is no longer used in 2.1+ - -pub mod logical; -pub mod physical; diff --git a/rust/lance-encoding/src/repdef.rs b/rust/lance-encoding/src/repdef.rs index b418b906de8..bb5d9cca3af 100644 --- a/rust/lance-encoding/src/repdef.rs +++ b/rust/lance-encoding/src/repdef.rs @@ -118,13 +118,16 @@ use arrow_buffer::{ }; use lance_core::{Error, Result, utils::bit::log_2_ceil}; -use crate::buffer::LanceBuffer; +use crate::{ + buffer::LanceBuffer, + encodings::logical::primitive::sparse::{SparseStructuralPlan, SparseStructuralUnraveler}, +}; pub type LevelBuffer = Vec; -/// A contiguous top-level-row range that can be encoded as one structural page. +/// A top-level-row range whose dense rep/def stream fits one mini-block page. #[derive(Debug, Clone, PartialEq, Eq)] -pub(crate) struct StructuralPageSplit { +pub(crate) struct MiniBlockRepDefSplit { /// Top-level row offset, relative to the original unsplit page. pub(crate) row_start: u64, /// Number of top-level rows in this split. @@ -137,15 +140,15 @@ pub(crate) struct StructuralPageSplit { pub(crate) num_values: u64, } -/// Planner result for structural page budget handling. +/// Dense mini-block rep/def budget result for one accumulated page. #[derive(Debug, Clone, PartialEq, Eq)] -pub(crate) enum StructuralPagePlan { - /// The original page can be encoded as-is. - Fits, - /// The original page should be split on top-level row boundaries. - Split(Vec), - /// One top-level row is larger than the requested structural page budget. - UnsplittableOverBudget(u64), +pub(crate) enum MiniBlockRepDefBudget { + /// The dense rep/def stream fits one mini-block structural page. + WithinBudget, + /// The dense rep/def stream fits after splitting on top-level row boundaries. + RequiresPageSplit(Vec), + /// A single top-level row has this many rep/def levels and exceeds the budget. + SingleRowOverBudget(u64), } // As we build def levels we add this to special values to indicate that they @@ -199,6 +202,143 @@ enum RawRepDef { Fsl(FslDesc), } +/// A normalized Arrow structural layer shared by dense and sparse serializers. +#[derive(Clone, Copy, Debug)] +pub(crate) enum NormalizedStructuralLayer<'a> { + List { + offsets: &'a [i64], + validity: Option<&'a BooleanBuffer>, + num_slots: usize, + }, + Validity { + validity: Option<&'a BooleanBuffer>, + num_slots: usize, + }, + FixedSizeList { + validity: Option<&'a BooleanBuffer>, + dimension: usize, + num_slots: usize, + }, +} + +/// Structural layers concatenated across input batches exactly once. +/// +/// Dense rep/def serialization and sparse metadata planning both consume this +/// representation so the Arrow nesting is not independently reconstructed. +#[derive(Debug)] +pub(crate) struct NormalizedStructuralPlan { + layers: Vec, + dense_all_valid: bool, +} + +impl NormalizedStructuralPlan { + pub(crate) fn layers(&self) -> impl ExactSizeIterator> { + self.layers.iter().map(|layer| match layer { + RawRepDef::Offsets(OffsetDesc { + offsets, + validity, + num_values, + .. + }) => NormalizedStructuralLayer::List { + offsets, + validity: validity.as_ref(), + num_slots: *num_values, + }, + RawRepDef::Validity(ValidityDesc { + validity, + num_values, + }) => NormalizedStructuralLayer::Validity { + validity: validity.as_ref(), + num_slots: *num_values, + }, + RawRepDef::Fsl(FslDesc { + validity, + dimension, + num_values, + }) => NormalizedStructuralLayer::FixedSizeList { + validity: validity.as_ref(), + dimension: *dimension, + num_slots: *num_values, + }, + }) + } + + fn to_serializer(&self) -> (SerializerContext, Option) { + if self.dense_all_valid { + let def_meaning = self + .layers + .iter() + .map(|_| DefinitionInterpretation::AllValidItem) + .collect::>(); + return ( + SerializerContext { + def_meaning, + rep_levels: LevelBuffer::default(), + spare_rep: LevelBuffer::default(), + def_levels: LevelBuffer::default(), + spare_def: LevelBuffer::default(), + current_rep: 0, + current_def: 0, + current_len: 0, + current_num_specials: 0, + has_fsl: false, + }, + None, + ); + } + + let total_len = self.layers.last().map_or(0, RawRepDef::num_values) + + self + .layers + .iter() + .map(RawRepDef::num_specials) + .sum::(); + let max_rep = self.layers.iter().map(RawRepDef::max_rep).sum::(); + let max_def = self.layers.iter().map(RawRepDef::max_def).sum::(); + let bits_per_rep = if max_rep > 0 { + u64::from(u16::BITS - max_rep.leading_zeros()) + } else { + 0 + }; + let bits_per_def = if max_def > 0 { + u64::from(u16::BITS - max_def.leading_zeros()) + } else { + 0 + }; + let bits_per_level = + (bits_per_rep + bits_per_def > 0).then_some(bits_per_rep + bits_per_def); + + let num_layers = self.layers.len(); + let mut context = SerializerContext::new(total_len, num_layers, max_rep, max_def); + for layer in &self.layers { + match layer { + RawRepDef::Validity(def) => context.record_validity(def), + RawRepDef::Offsets(rep) => context.record_offsets(rep), + RawRepDef::Fsl(fsl) => context.record_fsl(fsl), + } + } + (context, bits_per_level) + } + + pub(crate) fn serialize(&self) -> SerializedRepDefs { + self.to_serializer().0.build() + } + + pub(crate) fn serialize_with_miniblock_repdef_budget( + &self, + max_levels_for_bits: impl FnOnce(u64) -> u64, + num_rows: u64, + num_values: u64, + ) -> Result<(SerializedRepDefs, MiniBlockRepDefBudget)> { + let (context, bits_per_level) = self.to_serializer(); + context.build_with_miniblock_repdef_budget( + bits_per_level.map(max_levels_for_bits), + num_rows, + num_values, + ) + } +} + impl RawRepDef { // Are there any nulls in this layer fn has_nulls(&self) -> bool { @@ -806,21 +946,20 @@ impl SerializerContext { max_levels_per_page: Option, num_rows: u64, num_values: u64, - ) -> Result { + ) -> Result { // Extremely sparse lists can have many rep/def levels for very few // visible leaf values. If this ratio becomes too skewed then a - // miniblock structural chunk can exceed its packed rep/def metadata - // budget even though the value buffers are small. We detect that case - // while normalizing special def levels and split the structural page on - // top-level row boundaries so each emitted page stays within the - // miniblock structural budget. + // mini-block rep/def chunk can exceed its packed metadata budget even + // though the value buffers are small. We detect that case while + // normalizing special def levels and split on top-level row boundaries + // so each emitted dense mini-block page stays within the budget. if self.def_levels.is_empty() { - return Ok(StructuralPagePlan::Fits); + return Ok(MiniBlockRepDefBudget::WithinBudget); } if self.rep_levels.is_empty() { self.normalize_specials(); - return Ok(StructuralPagePlan::Fits); + return Ok(MiniBlockRepDefBudget::WithinBudget); } if self.rep_levels.len() != self.def_levels.len() { @@ -833,12 +972,12 @@ impl SerializerContext { let Some(max_levels_per_page) = max_levels_per_page else { self.normalize_specials(); - return Ok(StructuralPagePlan::Fits); + return Ok(MiniBlockRepDefBudget::WithinBudget); }; if num_values == 0 { self.normalize_specials(); - return Ok(StructuralPagePlan::Fits); + return Ok(MiniBlockRepDefBudget::WithinBudget); } let max_schema_rep = def_meaning.iter().filter(|level| level.is_list()).count() as u16; @@ -847,7 +986,7 @@ impl SerializerContext { if !should_plan { self.normalize_specials(); - return Ok(StructuralPagePlan::Fits); + return Ok(MiniBlockRepDefBudget::WithinBudget); } let max_visible_level = max_visible_level.unwrap(); @@ -855,7 +994,7 @@ impl SerializerContext { let mut counted_rows = 0u64; let mut counted_values = 0u64; let mut saw_structural_overhead = false; - let mut unsplittable_over_budget = None; + let mut single_row_over_budget_levels = None; let mut current_row_level_start = None; let mut current_row_num_values = 0u64; @@ -876,14 +1015,14 @@ impl SerializerContext { saw_structural_overhead |= row_has_structural_overhead; if row_has_structural_overhead && row_num_levels > max_levels_per_page { - unsplittable_over_budget = Some(row_num_levels); + single_row_over_budget_levels = Some(row_num_levels); } if current_page_num_rows > 0 && (current_page_has_structural_overhead || row_has_structural_overhead) && current_page_num_levels + row_num_levels > max_levels_per_page { - splits.push(StructuralPageSplit { + splits.push(MiniBlockRepDefSplit { row_start: current_page_row_start, num_rows: current_page_num_rows, level_range: current_page_level_start..current_page_level_end, @@ -966,14 +1105,14 @@ impl SerializerContext { ))); } if !saw_structural_overhead { - return Ok(StructuralPagePlan::Fits); + return Ok(MiniBlockRepDefBudget::WithinBudget); } - if let Some(row_num_levels) = unsplittable_over_budget { - return Ok(StructuralPagePlan::UnsplittableOverBudget(row_num_levels)); + if let Some(row_num_levels) = single_row_over_budget_levels { + return Ok(MiniBlockRepDefBudget::SingleRowOverBudget(row_num_levels)); } if current_page_num_rows > 0 { - splits.push(StructuralPageSplit { + splits.push(MiniBlockRepDefSplit { row_start: current_page_row_start, num_rows: current_page_num_rows, level_range: current_page_level_start..current_page_level_end, @@ -983,9 +1122,9 @@ impl SerializerContext { } if splits.len() > 1 { - Ok(StructuralPagePlan::Split(splits)) + Ok(MiniBlockRepDefBudget::RequiresPageSplit(splits)) } else { - Ok(StructuralPagePlan::Fits) + Ok(MiniBlockRepDefBudget::WithinBudget) } } @@ -1023,12 +1162,12 @@ impl SerializerContext { ) } - fn build_with_structural_plan( + fn build_with_miniblock_repdef_budget( mut self, max_levels_per_page: Option, num_rows: u64, num_values: u64, - ) -> Result<(SerializedRepDefs, StructuralPagePlan)> { + ) -> Result<(SerializedRepDefs, MiniBlockRepDefBudget)> { if self.current_len == 0 { return Ok(( SerializedRepDefs::new_with_fixed_size_list_levels( @@ -1037,7 +1176,7 @@ impl SerializerContext { self.def_meaning, self.has_fsl, ), - StructuralPagePlan::Fits, + MiniBlockRepDefBudget::WithinBudget, )); } @@ -1046,7 +1185,7 @@ impl SerializerContext { .into_iter() .rev() .collect::>(); - let plan = self.normalize_specials_and_plan_splits( + let budget = self.normalize_specials_and_plan_splits( &def_meaning, max_levels_per_page, num_rows, @@ -1071,7 +1210,7 @@ impl SerializerContext { def_meaning, self.has_fsl, ), - plan, + budget, )) } } @@ -1409,54 +1548,18 @@ impl RepDefBuilder { /// Converts the validity / offsets buffers that have been gathered so far /// into repetition and definition levels pub fn serialize(builders: Vec) -> SerializedRepDefs { - Self::serialize_builders(builders).0.build() + Self::normalize(builders).serialize() } - /// Converts gathered structural buffers into rep/def levels and an encode-time plan. - pub(crate) fn serialize_with_structural_plan( - builders: Vec, - max_levels_for_bits: impl FnOnce(u64) -> u64, - num_rows: u64, - num_values: u64, - ) -> Result<(SerializedRepDefs, StructuralPagePlan)> { - let (context, bits_per_level) = Self::serialize_builders(builders); - context.build_with_structural_plan( - bits_per_level.map(max_levels_for_bits), - num_rows, - num_values, - ) - } - - fn serialize_builders(builders: Vec) -> (SerializerContext, Option) { + pub(crate) fn normalize(builders: Vec) -> NormalizedStructuralPlan { assert!(!builders.is_empty()); - if builders.iter().all(|b| b.is_empty()) { - // No repetition, all-valid - let def_meaning = builders - .first() - .unwrap() - .repdefs - .iter() - .map(|_| DefinitionInterpretation::AllValidItem) - .collect::>(); - return ( - SerializerContext { - def_meaning, - rep_levels: LevelBuffer::default(), - spare_rep: LevelBuffer::default(), - def_levels: LevelBuffer::default(), - spare_def: LevelBuffer::default(), - current_rep: 0, - current_def: 0, - current_len: 0, - current_num_specials: 0, - has_fsl: false, - }, - None, - ); - } - let num_layers = builders[0].num_layers(); - let combined_layers = (0..num_layers) + debug_assert!( + builders + .iter() + .all(|builder| builder.num_layers() == num_layers) + ); + let layers = (0..num_layers) .map(|layer_index| { Self::concat_layers( builders.iter().map(|b| &b.repdefs[layer_index]), @@ -1464,47 +1567,10 @@ impl RepDefBuilder { ) }) .collect::>(); - debug_assert!( - builders - .iter() - .all(|b| b.num_layers() == builders[0].num_layers()) - ); - - let total_len = combined_layers.last().unwrap().num_values() - + combined_layers - .iter() - .map(|l| l.num_specials()) - .sum::(); - let max_rep = combined_layers.iter().map(|l| l.max_rep()).sum::(); - let max_def = combined_layers.iter().map(|l| l.max_def()).sum::(); - let bits_per_rep = if max_rep > 0 { - u64::from(u16::BITS - max_rep.leading_zeros()) - } else { - 0 - }; - let bits_per_def = if max_def > 0 { - u64::from(u16::BITS - max_def.leading_zeros()) - } else { - 0 - }; - let bits_per_level = - (bits_per_rep + bits_per_def > 0).then_some(bits_per_rep + bits_per_def); - - let mut context = SerializerContext::new(total_len, num_layers, max_rep, max_def); - for layer in combined_layers.into_iter() { - match layer { - RawRepDef::Validity(def) => { - context.record_validity(&def); - } - RawRepDef::Offsets(rep) => { - context.record_offsets(&rep); - } - RawRepDef::Fsl(fsl) => { - context.record_fsl(&fsl); - } - } + NormalizedStructuralPlan { + layers, + dense_all_valid: builders.iter().all(Self::is_empty), } - (context, bits_per_level) } } @@ -1514,6 +1580,7 @@ impl RepDefBuilder { /// This is used during decoding to create the necessary arrow structures #[derive(Debug)] pub struct RepDefUnraveler { + sparse: Option, rep_levels: Option, def_levels: Option, // Maps from definition level to the rep level at which that definition level is visible @@ -1567,6 +1634,7 @@ impl RepDefUnraveler { } } Self { + sparse: None, rep_levels, def_levels, current_def_cmp: 0, @@ -1578,7 +1646,35 @@ impl RepDefUnraveler { } } + pub(crate) fn new_sparse(plan: SparseStructuralPlan) -> Self { + Self { + sparse: Some(SparseStructuralUnraveler::new(plan)), + rep_levels: None, + def_levels: None, + levels_to_rep: Vec::new(), + def_meaning: Arc::new([]), + current_def_cmp: 0, + current_rep_cmp: 0, + current_layer: 0, + num_items: 0, + } + } + + fn ensure_exhausted(&self) -> Result<()> { + if let Some(sparse) = &self.sparse { + sparse.ensure_exhausted()?; + } + Ok(()) + } + + fn is_sparse(&self) -> bool { + self.sparse.is_some() + } + pub fn is_all_valid(&self) -> bool { + if let Some(sparse) = &self.sparse { + return sparse.is_all_valid(); + } self.def_levels.is_none() || self.def_meaning[self.current_layer].is_all_valid() } @@ -1587,15 +1683,19 @@ impl RepDefUnraveler { /// /// This is not valid to call when the current level is a struct/primitive layer because /// in some cases there may be no rep or def information to know this. - pub fn max_lists(&self) -> usize { + pub fn max_lists(&self) -> Result { + if let Some(sparse) = &self.sparse { + return sparse.max_lists(); + } debug_assert!( self.def_meaning[self.current_layer] != DefinitionInterpretation::NullableItem ); - self.rep_levels + Ok(self + .rep_levels .as_ref() // Worst case every rep item is max_rep and a new list .map(|levels| levels.len()) - .unwrap_or(0) + .unwrap_or(0)) } /// Unravels a layer of offsets from the unraveler into the given offset width @@ -1607,6 +1707,9 @@ impl RepDefUnraveler { offsets: &mut Vec, validity: Option<&mut BooleanBufferBuilder>, ) -> Result<()> { + if let Some(sparse) = self.sparse.as_mut() { + return sparse.unravel_offsets(offsets, validity); + } let rep_levels = self .rep_levels .as_mut() @@ -1757,18 +1860,25 @@ impl RepDefUnraveler { } } - pub fn skip_validity(&mut self) { + pub fn skip_validity(&mut self) -> Result<()> { + if let Some(sparse) = self.sparse.as_mut() { + return sparse.skip_validity(); + } debug_assert!(self.is_all_valid()); self.current_layer += 1; + Ok(()) } /// Unravels a layer of validity from the definition levels - pub fn unravel_validity(&mut self, validity: &mut BooleanBufferBuilder) { + pub fn unravel_validity(&mut self, validity: &mut BooleanBufferBuilder) -> Result<()> { + if let Some(sparse) = self.sparse.as_mut() { + return sparse.unravel_validity(validity); + } let meaning = self.def_meaning[self.current_layer]; if meaning == DefinitionInterpretation::AllValidItem || self.def_levels.is_none() { self.current_layer += 1; validity.append_n(self.num_items as usize, true); - return; + return Ok(()); } self.current_layer += 1; @@ -1786,9 +1896,13 @@ impl RepDefUnraveler { }) { validity.append(is_valid); } + Ok(()) } - pub fn decimate(&mut self, dimension: usize) { + pub fn decimate(&mut self, dimension: usize) -> Result<()> { + if let Some(sparse) = self.sparse.as_mut() { + return sparse.decimate(dimension); + } if self.rep_levels.is_some() { // If we need to support this then I think we need to walk through the rep def levels to find // the spots at which we keep. E.g. if we have: @@ -1804,7 +1918,7 @@ impl RepDefUnraveler { todo!("Not yet supported FSL<...List<...>>"); } let Some(def_levels) = self.def_levels.as_mut() else { - return; + return Ok(()); }; let mut read_idx = 0; let mut write_idx = 0; @@ -1816,6 +1930,7 @@ impl RepDefUnraveler { read_idx += dimension; } def_levels.truncate(write_idx); + Ok(()) } } @@ -1835,44 +1950,104 @@ impl RepDefUnraveler { #[derive(Debug)] pub struct CompositeRepDefUnraveler { unravelers: Vec, + comparisons: Vec, } impl CompositeRepDefUnraveler { pub fn new(unravelers: Vec) -> Self { - Self { unravelers } + Self { + unravelers, + comparisons: Vec::new(), + } + } + + pub(crate) fn add_compatibility_check(&mut self, other: Self) { + self.comparisons.push(other); + } + + pub(crate) fn has_sparse(&self) -> bool { + self.unravelers.iter().any(RepDefUnraveler::is_sparse) + || self.comparisons.iter().any(Self::has_sparse) + } + + pub(crate) fn ensure_exhausted(&self) -> Result<()> { + for unraveler in &self.unravelers { + unraveler.ensure_exhausted()?; + } + for comparison in &self.comparisons { + comparison.ensure_exhausted()?; + } + Ok(()) + } + + fn null_buffers_equal( + left: &Option, + right: &Option, + expected_len: usize, + ) -> bool { + match (left, right) { + (None, None) => true, + (Some(left), Some(right)) => { + left.len() == expected_len + && right.len() == expected_len + && left.iter().eq(right.iter()) + } + (None, Some(right)) => right.len() == expected_len && right.null_count() == 0, + (Some(left), None) => left.len() == expected_len && left.null_count() == 0, + } + } + + fn decimate(&mut self, dimension: usize) -> Result<()> { + for unraveler in &mut self.unravelers { + unraveler.decimate(dimension)?; + } + for comparison in &mut self.comparisons { + comparison.decimate(dimension)?; + } + Ok(()) } /// Unravels a layer of validity /// /// Returns None if there are no null items in this layer - pub fn unravel_validity(&mut self, num_values: usize) -> Option { + pub fn unravel_validity(&mut self, num_values: usize) -> Result> { let is_all_valid = self .unravelers .iter() .all(|unraveler| unraveler.is_all_valid()); - if is_all_valid { + let validity = if is_all_valid { for unraveler in self.unravelers.iter_mut() { - unraveler.skip_validity(); + unraveler.skip_validity()?; } None } else { let mut validity = BooleanBufferBuilder::new(num_values); for unraveler in self.unravelers.iter_mut() { - unraveler.unravel_validity(&mut validity); + unraveler.unravel_validity(&mut validity)?; } Some(NullBuffer::new(validity.finish())) + }; + for comparison in &mut self.comparisons { + let other = comparison.unravel_validity(num_values)?; + if !Self::null_buffers_equal(&validity, &other, num_values) { + return Err(Error::invalid_input_source( + format!( + "Structural sibling fields have incompatible validity metadata for {num_values} values" + ) + .into(), + )); + } } + Ok(validity) } pub fn unravel_fsl_validity( &mut self, num_values: usize, dimension: usize, - ) -> Option { - for unraveler in self.unravelers.iter_mut() { - unraveler.decimate(dimension); - } + ) -> Result> { + self.decimate(dimension)?; self.unravel_validity(num_values) } @@ -1881,10 +2056,16 @@ impl CompositeRepDefUnraveler { &mut self, ) -> Result<(OffsetBuffer, Option)> { let mut is_all_valid = true; - let mut max_num_lists = 0; + let mut max_num_lists: usize = 0; for unraveler in self.unravelers.iter() { is_all_valid &= unraveler.is_all_valid(); - max_num_lists += unraveler.max_lists(); + max_num_lists = max_num_lists + .checked_add(unraveler.max_lists()?) + .ok_or_else(|| { + Error::invalid_input_source( + "Combined repetition/definition list count exceeds usize::MAX".into(), + ) + })?; } let mut validity = if is_all_valid { @@ -1901,10 +2082,28 @@ impl CompositeRepDefUnraveler { unraveler.unravel_offsets(&mut offsets, validity.as_mut())?; } - Ok(( - OffsetBuffer::new(ScalarBuffer::from(offsets)), - validity.map(|mut v| NullBuffer::new(v.finish())), - )) + let offsets = OffsetBuffer::new(ScalarBuffer::from(offsets)); + let validity = validity.map(|mut v| NullBuffer::new(v.finish())); + for comparison in &mut self.comparisons { + let (other_offsets, other_validity) = comparison.unravel_offsets::()?; + if offsets.as_ref() != other_offsets.as_ref() + || !Self::null_buffers_equal( + &validity, + &other_validity, + offsets.len().saturating_sub(1), + ) + { + return Err(Error::invalid_input_source( + format!( + "Structural sibling fields have incompatible list metadata for {} slots", + offsets.len().saturating_sub(1) + ) + .into(), + )); + } + } + + Ok((offsets, validity)) } } @@ -2603,6 +2802,10 @@ impl ControlWordParser { mod tests { use arrow_buffer::{NullBuffer, OffsetBuffer, ScalarBuffer}; + use crate::encodings::logical::primitive::sparse::{ + SparsePositionSet, SparseStructuralLayerPlan, SparseStructuralPlan, SparseValidityMeaning, + SparseValiditySet, + }; use crate::repdef::{ CompositeRepDefUnraveler, DefinitionInterpretation, RepDefUnraveler, SerializedRepDefs, }; @@ -2621,6 +2824,31 @@ mod tests { OffsetBuffer::::new(ScalarBuffer::from_iter(values.iter().copied())) } + #[test] + fn sparse_sibling_validity_mismatch_is_invalid_input() { + let sparse = |positions| { + RepDefUnraveler::new_sparse(SparseStructuralPlan { + layers: vec![SparseStructuralLayerPlan::Validity { + num_slots: 2, + validity: SparseValiditySet { + meaning: SparseValidityMeaning::NullPositions, + positions, + }, + }], + num_items: 2, + num_visible_items: 2, + }) + }; + let mut repdef = CompositeRepDefUnraveler::new(vec![sparse(SparsePositionSet::Empty)]); + repdef.add_compatibility_check(CompositeRepDefUnraveler::new(vec![sparse( + SparsePositionSet::Explicit(vec![0]), + )])); + + let err = repdef.unravel_validity(2).unwrap_err(); + assert!(matches!(err, lance_core::Error::InvalidInput { .. })); + assert!(err.to_string().contains("incompatible validity metadata")); + } + #[test] fn test_repdef_empty_offsets() { // Empty offsets should serialize without panicking. @@ -2666,7 +2894,7 @@ mod tests { // Note: validity doesn't exactly round-trip because repdef normalizes some of the // redundant validity values assert_eq!( - unraveler.unravel_validity(9), + unraveler.unravel_validity(9).unwrap(), Some(validity(&[ true, true, true, false, false, false, true, true, false ])) @@ -2804,14 +3032,14 @@ mod tests { )]); assert_eq!( - unraveler.unravel_validity(8), + unraveler.unravel_validity(8).unwrap(), Some(validity(&[ true, false, true, false, false, false, false, false ])) ); - assert_eq!(unraveler.unravel_fsl_validity(4, 2), None); + assert_eq!(unraveler.unravel_fsl_validity(4, 2).unwrap(), None); assert_eq!( - unraveler.unravel_fsl_validity(2, 2), + unraveler.unravel_fsl_validity(2, 2).unwrap(), Some(validity(&[true, false])) ); } @@ -2847,10 +3075,10 @@ mod tests { 8, )]); - assert_eq!(unraveler.unravel_validity(8), None); - assert_eq!(unraveler.unravel_fsl_validity(4, 2), None); + assert_eq!(unraveler.unravel_validity(8).unwrap(), None); + assert_eq!(unraveler.unravel_fsl_validity(4, 2).unwrap(), None); assert_eq!( - unraveler.unravel_fsl_validity(2, 2), + unraveler.unravel_fsl_validity(2, 2).unwrap(), Some(validity(&[true, false])) ); } @@ -2928,7 +3156,7 @@ mod tests { 8, )]); - assert_eq!(unraveler.unravel_validity(6), None); + assert_eq!(unraveler.unravel_validity(6).unwrap(), None); let (off, val) = unraveler.unravel_offsets::().unwrap(); assert_eq!(off.inner(), offsets_32(&[0, 4, 4, 4, 6]).inner()); assert_eq!(val, None); @@ -2954,7 +3182,7 @@ mod tests { 9, )]); - assert_eq!(unraveler.unravel_validity(9), None); + assert_eq!(unraveler.unravel_validity(9).unwrap(), None); let (off, val) = unraveler.unravel_offsets::().unwrap(); assert_eq!(off.inner(), offsets_32(&[0, 1, 3, 5, 7, 9]).inner()); assert_eq!(val, None); @@ -3018,7 +3246,7 @@ mod tests { 8, )]); - assert_eq!(unraveler.unravel_validity(6), None); + assert_eq!(unraveler.unravel_validity(6).unwrap(), None); let (off, val) = unraveler.unravel_offsets::().unwrap(); assert_eq!(off.inner(), offsets_32(&[0, 4, 4, 4, 6]).inner()); assert_eq!(val, None); @@ -3048,7 +3276,7 @@ mod tests { 8, )]); - assert_eq!(unraveler.unravel_validity(6), None); + assert_eq!(unraveler.unravel_validity(6).unwrap(), None); let (off, val) = unraveler.unravel_offsets::().unwrap(); assert_eq!(off.inner(), offsets_32(&[0, 4, 4, 4, 6]).inner()); assert_eq!(val, Some(validity(&[true, false, false, true]))); @@ -3078,7 +3306,7 @@ mod tests { 8, )]); - assert_eq!(unraveler.unravel_validity(6), None); + assert_eq!(unraveler.unravel_validity(6).unwrap(), None); let (off, val) = unraveler.unravel_offsets::().unwrap(); assert_eq!(off.inner(), offsets_32(&[0, 4, 4, 4, 6]).inner()); assert_eq!(val, Some(validity(&[true, false, true, true]))); @@ -3106,11 +3334,11 @@ mod tests { )]); assert_eq!( - unraveler.unravel_validity(4), + unraveler.unravel_validity(4).unwrap(), Some(validity(&[false, true, false, false])) ); assert_eq!( - unraveler.unravel_validity(4), + unraveler.unravel_validity(4).unwrap(), Some(validity(&[false, true, false, false])) ); let (off, val) = unraveler.unravel_offsets::().unwrap(); @@ -3139,14 +3367,14 @@ mod tests { )]); assert_eq!( - unraveler.unravel_validity(5), + unraveler.unravel_validity(5).unwrap(), Some(validity(&[false, false, true, true, false])) ); assert_eq!( - unraveler.unravel_validity(5), + unraveler.unravel_validity(5).unwrap(), Some(validity(&[false, false, true, true, true])) ); - assert_eq!(unraveler.unravel_validity(5), None); + assert_eq!(unraveler.unravel_validity(5).unwrap(), None); } #[test] @@ -3188,7 +3416,7 @@ mod tests { let mut unraveler = CompositeRepDefUnraveler::new(vec![unravel1, unravel2]); - assert!(unraveler.unravel_validity(9).is_none()); + assert!(unraveler.unravel_validity(9).unwrap().is_none()); let (off, val) = unraveler.unravel_offsets::().unwrap(); assert_eq!( off.inner(), @@ -3484,11 +3712,11 @@ mod tests { 0, )]); - assert_eq!(unraveler.unravel_validity(0), None); + assert_eq!(unraveler.unravel_validity(0).unwrap(), None); let (off, val) = unraveler.unravel_offsets::().unwrap(); assert_eq!(off.inner(), offsets_32(&[0, 0, 0, 0]).inner()); assert_eq!(val, Some(validity(&[false, false, false]))); - let val = unraveler.unravel_validity(3).unwrap(); + let val = unraveler.unravel_validity(3).unwrap().unwrap(); assert_eq!(val.inner(), validity(&[true, false, true]).inner()); } @@ -3516,7 +3744,7 @@ mod tests { 1, )]); - assert_eq!(unraveler.unravel_validity(1), None); + assert_eq!(unraveler.unravel_validity(1).unwrap(), None); let (off, val) = unraveler.unravel_offsets::().unwrap(); assert_eq!(off.inner(), offsets_32(&[0, 1, 1]).inner()); assert_eq!(val, Some(validity(&[true, false]))); @@ -3547,7 +3775,7 @@ mod tests { ]); assert_eq!( - unraveler.unravel_validity(8), + unraveler.unravel_validity(8).unwrap(), Some(validity(&[ true, false, true, false, true, true, true, true ])) @@ -3584,7 +3812,7 @@ mod tests { ]); assert_eq!( - unraveler.unravel_validity(4), + unraveler.unravel_validity(4).unwrap(), Some(validity(&[true, false, true, true])) ); assert_eq!( @@ -3616,7 +3844,7 @@ mod tests { ]); assert_eq!( - unraveler.unravel_validity(8), + unraveler.unravel_validity(8).unwrap(), Some(validity(&[ true, false, true, false, true, true, true, true ])) diff --git a/rust/lance-encoding/src/testing.rs b/rust/lance-encoding/src/testing.rs index 176083d6d64..90c5c0bb61b 100644 --- a/rust/lance-encoding/src/testing.rs +++ b/rust/lance-encoding/src/testing.rs @@ -21,27 +21,344 @@ use futures::{FutureExt, StreamExt, future::BoxFuture}; use log::{debug, info, trace}; use tokio::sync::mpsc::{self, UnboundedSender}; -use lance_core::{Result, utils::bit::pad_bytes}; +use lance_core::{Error, Result, datatypes::Field as LanceField, utils::bit::pad_bytes}; use lance_datagen::{ArrayGenerator, RowCount, Seed, array, gen_batch}; use crate::{ EncodingsIo, buffer::LanceBuffer, + compression::{ + BlockCompressor, CompressionStrategy, field_metadata_params, finalize_miniblock_compressor, + reject_packed_struct_per_value, try_bitpacking_block, try_bitpacking_miniblock, + try_byte_stream_split_miniblock, try_child_rle_miniblock, + try_fixed_packed_struct_miniblock, try_fixed_u8_rle_block, try_fixed_u8_rle_miniblock, + try_general_block, try_raw_block, try_raw_fixed_size_list_miniblock, + try_raw_fixed_width_miniblock, try_raw_per_value, try_uncompressed_fixed_width_miniblock, + try_variable_packed_struct_per_value, try_variable_rle_block, try_variable_width_miniblock, + try_variable_width_per_value, + }, + compression_config::{CompressionFieldParams, CompressionParams}, + data::DataBlock, decoder::{ ColumnInfo, DecodeBatchScheduler, DecoderMessage, DecoderPlugins, FilterExpression, PageInfo, create_decode_stream, }, encoder::{ ColumnIndexSequence, EncodedColumn, EncodedPage, EncodingOptions, FieldEncoder, - MIN_PAGE_BUFFER_ALIGNMENT, OutOfLineBuffers, default_encoding_strategy, + FieldEncodingContext, FieldEncodingStrategy, MIN_PAGE_BUFFER_ALIGNMENT, OutOfLineBuffers, + structural::{ + PrimitiveFieldEncoding, PrimitivePageEncoding, try_create_binary_blob, try_create_list, + try_create_map, try_create_struct, try_create_structural_blob, + try_create_structural_fixed_size_list, + }, }, + encodings::logical::primitive::{fullzip::PerValueCompressor, miniblock::MiniBlockCompressor}, repdef::RepDefBuilder, - version::LanceFileVersion, }; const MAX_PAGE_BYTES: u64 = 32 * 1024 * 1024; const TEST_ALIGNMENT: usize = MIN_PAGE_BUFFER_ALIGNMENT as usize; +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum TestEncoding { + Array, + StructuralU16, + StructuralU32, + StructuralSparse, +} + +impl TestEncoding { + fn all() -> impl Iterator { + [ + Self::Array, + Self::StructuralU16, + Self::StructuralU32, + Self::StructuralSparse, + ] + .into_iter() + } + + fn is_structural(self) -> bool { + self != Self::Array + } +} + +impl std::fmt::Display for TestEncoding { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::Array => write!(f, "array"), + Self::StructuralU16 => write!(f, "structural-u16"), + Self::StructuralU32 => write!(f, "structural-u32"), + Self::StructuralSparse => write!(f, "structural-sparse"), + } + } +} + +#[derive(Debug, Clone)] +struct TestCompressionStrategy { + encoding: TestEncoding, + params: CompressionParams, +} + +impl TestCompressionStrategy { + fn field_params(&self, field: &LanceField) -> CompressionFieldParams { + let mut params = self + .params + .get_field_params(&field.name, &field.data_type()); + let mut metadata = field_metadata_params(field); + if self.encoding == TestEncoding::StructuralU16 + && metadata + .minichunk_size + .is_some_and(|size| size >= 32 * 1024) + { + metadata.minichunk_size = None; + } + params.merge(&metadata); + params + } +} + +impl CompressionStrategy for TestCompressionStrategy { + fn create_miniblock_compressor( + &self, + field: &LanceField, + data: &DataBlock, + ) -> Result> { + let params = self.field_params(field); + let compressor = + if let Some(compressor) = try_uncompressed_fixed_width_miniblock(data, ¶ms) { + compressor + } else if let Some(compressor) = try_byte_stream_split_miniblock(data, ¶ms) { + compressor + } else if let Some(compressor) = match self.encoding { + TestEncoding::StructuralSparse => try_child_rle_miniblock(data, ¶ms), + TestEncoding::Array | TestEncoding::StructuralU16 | TestEncoding::StructuralU32 => { + try_fixed_u8_rle_miniblock(data, ¶ms) + } + } { + compressor + } else if let Some(compressor) = try_bitpacking_miniblock(data) { + compressor + } else if let Some(compressor) = try_raw_fixed_width_miniblock(data) { + compressor + } else if let Some(compressor) = try_variable_width_miniblock(field, data, ¶ms)? { + compressor + } else if let Some(compressor) = try_fixed_packed_struct_miniblock(data)? { + compressor + } else if let Some(compressor) = try_raw_fixed_size_list_miniblock(data) { + compressor + } else { + return Err(lance_core::Error::not_supported_source( + format!( + "Mini-block compression not yet supported for block type {}", + data.name() + ) + .into(), + )); + }; + finalize_miniblock_compressor(data, compressor, ¶ms) + } + + fn create_per_value( + &self, + field: &LanceField, + data: &DataBlock, + ) -> Result> { + let params = self.field_params(field); + if let Some(compressor) = try_raw_per_value(data) { + return Ok(compressor); + } + let packed = match self.encoding { + TestEncoding::Array | TestEncoding::StructuralU16 => { + reject_packed_struct_per_value(field, data)? + } + TestEncoding::StructuralU32 | TestEncoding::StructuralSparse => { + try_variable_packed_struct_per_value(Arc::new(self.clone()), field, data)? + } + }; + if let Some(compressor) = packed { + return Ok(compressor); + } + if let Some(compressor) = try_variable_width_per_value(field, data, ¶ms)? { + return Ok(compressor); + } + Err(lance_core::Error::not_supported_source( + format!( + "Per-value compression not yet supported for block type {}", + data.name() + ) + .into(), + )) + } + + fn create_block_compressor( + &self, + field: &LanceField, + data: &DataBlock, + ) -> Result<(Box, CompressiveEncoding)> { + let params = self.field_params(field); + let rle = match self.encoding { + TestEncoding::Array | TestEncoding::StructuralU16 => None, + TestEncoding::StructuralU32 => try_fixed_u8_rle_block(data, ¶ms)?, + TestEncoding::StructuralSparse => try_variable_rle_block(data, ¶ms)?, + }; + if let Some(compressor) = rle { + return Ok(compressor); + } + if let Some(compressor) = try_bitpacking_block(data) { + return Ok(compressor); + } + if matches!( + self.encoding, + TestEncoding::StructuralU32 | TestEncoding::StructuralSparse + ) && let Some(compressor) = try_general_block(data, ¶ms)? + { + return Ok(compressor); + } + if let Some(compressor) = try_raw_block(data) { + return Ok(compressor); + } + Err(lance_core::Error::not_supported_source( + format!( + "Block compression not yet supported for block type {}", + data.name() + ) + .into(), + )) + } +} + +pub fn test_compression_strategy( + encoding: TestEncoding, + params: CompressionParams, +) -> Arc { + Arc::new(TestCompressionStrategy { encoding, params }) +} + +#[derive(Debug)] +struct TestFieldEncodingStrategy { + encoding: TestEncoding, + primitive: PrimitiveFieldEncoding, +} + +impl FieldEncodingStrategy for TestFieldEncodingStrategy { + fn create_field_encoder( + &self, + field: &LanceField, + column_index: &mut ColumnIndexSequence, + context: &FieldEncodingContext<'_>, + ) -> Result> { + if let Some(encoder) = + try_create_binary_blob(&self.primitive, field, column_index, context)? + { + return Ok(encoder); + } + if self.encoding != TestEncoding::StructuralU16 + && let Some(encoder) = + try_create_structural_blob(&self.primitive, field, column_index, context)? + { + return Ok(encoder); + } + if field.is_blob() { + return Err(Error::invalid_input_source( + format!( + "Blob encoding is not available for field '{}' with data type {}", + field.name, + field.data_type() + ) + .into(), + )); + } + if self.encoding != TestEncoding::StructuralU16 { + if let Some(encoder) = try_create_map(field, column_index, context)? { + return Ok(encoder); + } + if let Some(encoder) = + try_create_structural_fixed_size_list(field, column_index, context)? + { + return Ok(encoder); + } + } + if let Some(encoder) = self.primitive.try_create(field, column_index, context)? { + return Ok(encoder); + } + if self.encoding == TestEncoding::StructuralU16 { + if matches!( + field.data_type(), + DataType::FixedSizeList(item, _) + if matches!(item.data_type(), DataType::Struct(_)) + ) { + return Err(Error::not_supported_source( + "FixedSizeList is not enabled by the selected file format".into(), + )); + } + if matches!(field.data_type(), DataType::Map(_, _)) { + return Err(Error::not_supported_source( + "Map data type is not enabled by the selected file format".into(), + )); + } + } + if let Some(encoder) = try_create_list(field, column_index, context)? { + return Ok(encoder); + } + if let Some(encoder) = try_create_struct(field, column_index, context)? { + return Ok(encoder); + } + Err(Error::not_supported_source( + format!( + "{} has no field encoding for '{}' with data type {}", + self.encoding, + field.name, + field.data_type() + ) + .into(), + )) + } +} + +pub fn test_encoding_strategy(encoding: TestEncoding) -> Box { + if encoding == TestEncoding::Array { + return Box::new(crate::array_encoding::ArrayFieldEncodingStrategy::new()); + } + + let compression = test_compression_strategy(encoding, CompressionParams::default()); + let page_encodings = match encoding { + TestEncoding::StructuralU16 => vec![ + PrimitivePageEncoding::reject_sparse(), + PrimitivePageEncoding::dense_u16(compression), + ], + TestEncoding::StructuralU32 => vec![ + PrimitivePageEncoding::reject_sparse(), + PrimitivePageEncoding::constant(), + PrimitivePageEncoding::dense_u32(compression), + ], + TestEncoding::StructuralSparse => vec![ + PrimitivePageEncoding::sparse(compression.clone()), + PrimitivePageEncoding::constant(), + PrimitivePageEncoding::dense_u32(compression), + ], + TestEncoding::Array => unreachable!(), + }; + Box::new(TestFieldEncodingStrategy { + encoding, + primitive: PrimitiveFieldEncoding::new(page_encodings), + }) +} + +pub fn create_test_field_encoder( + strategy: &dyn FieldEncodingStrategy, + field: &lance_core::datatypes::Field, + column_index: &mut ColumnIndexSequence, + options: &EncodingOptions, +) -> Result> { + let context = FieldEncodingContext { + strategy, + options, + root_field_metadata: &field.metadata, + }; + strategy.create_field_encoder(field, column_index, &context) +} + #[derive(Debug)] pub(crate) struct SimulatedScheduler { data: Bytes, @@ -342,24 +659,22 @@ pub async fn check_round_trip_encoding_generated( let lance_field = lance_core::datatypes::Field::try_from(&field).unwrap(); for page_size in test_cases.page_sizes.iter().copied() { debug!("Testing random data with a page size of {}", page_size); - let encoder_factory = |version: LanceFileVersion| { - let encoding_strategy = default_encoding_strategy(version); + let encoder_factory = |encoding: TestEncoding| { + let encoding_strategy = test_encoding_strategy(encoding); let mut column_index_seq = ColumnIndexSequence::default(); let encoding_options = EncodingOptions { max_page_bytes: MAX_PAGE_BYTES, cache_bytes_per_column: page_size, keep_original_array: true, buffer_alignment: MIN_PAGE_BUFFER_ALIGNMENT, - version, }; - encoding_strategy - .create_field_encoder( - encoding_strategy.as_ref(), - &lance_field, - &mut column_index_seq, - &encoding_options, - ) - .unwrap() + create_test_field_encoder( + encoding_strategy.as_ref(), + &lance_field, + &mut column_index_seq, + &encoding_options, + ) + .unwrap() }; check_round_trip_random( @@ -372,9 +687,9 @@ pub async fn check_round_trip_encoding_generated( } } -fn supports_nulls(data_type: &DataType, version: LanceFileVersion) -> bool { +fn supports_nulls(data_type: &DataType, encoding: TestEncoding) -> bool { if let DataType::Struct(fields) = data_type { - if version == LanceFileVersion::V2_0 { + if encoding == TestEncoding::Array { // 2.0 doesn't support nullability for structs false } else if fields.is_empty() { @@ -389,7 +704,7 @@ fn supports_nulls(data_type: &DataType, version: LanceFileVersion) -> bool { } } -type EncodingVerificationFn = dyn Fn(&[EncodedColumn], &LanceFileVersion); +type EncodingVerificationFn = dyn Fn(&[EncodedColumn], &TestEncoding); // The default will just test the full read #[derive(Clone)] @@ -400,8 +715,7 @@ pub struct TestCases { skip_validation: bool, max_page_size: Option, page_sizes: Vec, - min_file_version: Option, - max_file_version: Option, + encodings: Vec, verify_encoding: Option>, expected_encoding: Option>, } @@ -415,8 +729,7 @@ impl Default for TestCases { skip_validation: false, max_page_size: None, page_sizes: vec![4096, 1024 * 1024], - min_file_version: None, - max_file_version: None, + encodings: TestEncoding::all().collect(), verify_encoding: None, expected_encoding: None, } @@ -459,16 +772,40 @@ impl TestCases { self } - pub fn with_min_file_version(mut self, version: LanceFileVersion) -> Self { - self.min_file_version = Some(version); + pub fn with_encoding(mut self, encoding: TestEncoding) -> Self { + self.encodings = vec![encoding]; self } - pub fn with_max_file_version(mut self, version: LanceFileVersion) -> Self { - self.max_file_version = Some(version); + pub fn with_encodings(mut self, encodings: impl IntoIterator) -> Self { + self.encodings = encodings.into_iter().collect(); self } + pub fn with_structural_encodings(self) -> Self { + self.with_encodings([ + TestEncoding::StructuralU16, + TestEncoding::StructuralU32, + TestEncoding::StructuralSparse, + ]) + } + + pub fn with_u32_structural_encodings(self) -> Self { + self.with_encodings([TestEncoding::StructuralU32, TestEncoding::StructuralSparse]) + } + + pub fn with_dense_encodings(self) -> Self { + self.with_encodings([ + TestEncoding::Array, + TestEncoding::StructuralU16, + TestEncoding::StructuralU32, + ]) + } + + pub fn with_array_and_u16_encodings(self) -> Self { + self.with_encodings([TestEncoding::Array, TestEncoding::StructuralU16]) + } + pub fn with_page_sizes(mut self, page_sizes: Vec) -> Self { self.page_sizes = page_sizes; self @@ -483,22 +820,8 @@ impl TestCases { self.max_page_size.unwrap_or(MAX_PAGE_BYTES) } - fn get_versions(&self) -> Vec { - LanceFileVersion::iter_non_legacy() - .filter(|v| { - if let Some(min_file_version) = &self.min_file_version - && v < min_file_version - { - return false; - } - if let Some(max_file_version) = &self.max_file_version - && v > max_file_version - { - return false; - } - true - }) - .collect() + fn encodings(&self) -> impl Iterator + '_ { + self.encodings.iter().copied() } pub fn with_verify_encoding(mut self, verify_encoding: Arc) -> Self { @@ -506,9 +829,9 @@ impl TestCases { self } - fn verify_encoding(&self, encoding: &[EncodedColumn], version: &LanceFileVersion) { + fn verify_encoding(&self, columns: &[EncodedColumn], encoding: &TestEncoding) { if let Some(verify_encoding) = self.verify_encoding.as_ref() { - verify_encoding(encoding, version); + verify_encoding(columns, encoding); } } @@ -666,6 +989,11 @@ fn collect_page_encoding(layout: &PageLayout, actual_chain: &mut Vec) -> collect_page_encoding(inner_layout.as_ref(), actual_chain)? } } + Layout::SparseLayout(sparse) => { + if let Some(value_compression) = &sparse.value_compression { + actual_chain.extend(extract_array_encoding_chain(value_compression)); + } + } } } @@ -700,7 +1028,7 @@ fn verify_page_encoding( } } PageEncoding::Legacy(_) => { - // We don't need to care about legacy. + // We don't need to care about the v2.0 array encoding. } } @@ -743,28 +1071,26 @@ pub async fn check_round_trip_encoding_of_data_with_expected( let mut field = Field::new("", example_data.data_type().clone(), true); field = field.with_metadata(metadata); let lance_field = lance_core::datatypes::Field::try_from(&field).unwrap(); - for file_version in test_cases.get_versions() { + for encoding in test_cases.encodings() { for page_size in test_cases.page_sizes.iter() { - let encoding_strategy = default_encoding_strategy(file_version); + let encoding_strategy = test_encoding_strategy(encoding); let mut column_index_seq = ColumnIndexSequence::default(); let encoding_options = EncodingOptions { cache_bytes_per_column: *page_size, max_page_bytes: test_cases.get_max_page_size(), keep_original_array: true, buffer_alignment: MIN_PAGE_BUFFER_ALIGNMENT, - version: file_version, }; - let encoder = encoding_strategy - .create_field_encoder( - encoding_strategy.as_ref(), - &lance_field, - &mut column_index_seq, - &encoding_options, - ) - .unwrap(); + let encoder = create_test_field_encoder( + encoding_strategy.as_ref(), + &lance_field, + &mut column_index_seq, + &encoding_options, + ) + .unwrap(); info!( - "Testing round trip encoding of data with file version {} and page size {}", - file_version, page_size + "Testing round trip encoding of data with test encoding {} and page size {}", + encoding, page_size ); check_round_trip_encoding_inner( encoder, @@ -772,7 +1098,7 @@ pub async fn check_round_trip_encoding_of_data_with_expected( data.clone(), expected_override.clone(), test_cases, - file_version, + encoding, ) .await } @@ -845,7 +1171,7 @@ async fn check_round_trip_encoding_inner( data: Vec>, expected_override: Option>, test_cases: &TestCases, - file_version: LanceFileVersion, + encoding: TestEncoding, ) { let mut writer = SimulatedWriter::new(encoder.num_columns()); @@ -886,7 +1212,7 @@ async fn check_round_trip_encoding_inner( log_page(&encoded_page); // For V2.1, verify encoding in the page if expected - if file_version >= LanceFileVersion::V2_1 + if encoding.is_structural() && let Some(ref expected) = test_cases.expected_encoding { verify_page_encoding(&encoded_page, expected, encoded_page.column_idx as usize) @@ -908,7 +1234,7 @@ async fn check_round_trip_encoding_inner( log_page(&encoded_page); // For V2.1, verify encoding in the page if expected - if file_version >= LanceFileVersion::V2_1 + if encoding.is_structural() && let Some(ref expected) = test_cases.expected_encoding { verify_page_encoding(&encoded_page, expected, encoded_page.column_idx as usize) @@ -920,7 +1246,7 @@ async fn check_round_trip_encoding_inner( let mut external_buffers = writer.new_external_buffers(); let encoded_columns = encoder.finish(&mut external_buffers).await.unwrap(); - test_cases.verify_encoding(&encoded_columns, &file_version); + test_cases.verify_encoding(&encoded_columns, &encoding); for buffer in external_buffers.take_buffers() { writer.write_lance_buffer(buffer); } @@ -973,7 +1299,7 @@ async fn check_round_trip_encoding_inner( let expected_data = expected_override.clone().or_else(|| concat_data.clone()); - let is_structural_encoding = file_version >= LanceFileVersion::V2_1; + let is_structural_encoding = encoding.is_structural(); let decode_field = if is_structural_encoding { let mut lance_field = lance_core::datatypes::Field::try_from(field).unwrap(); @@ -1113,20 +1439,20 @@ const NUM_RANDOM_ROWS: u32 = 10000; /// /// To test specific test cases use the async fn check_round_trip_random( - encoder_factory: impl Fn(LanceFileVersion) -> Box, + encoder_factory: impl Fn(TestEncoding) -> Box, field: Field, array_generator_provider: Box, test_cases: &TestCases, ) { for null_rate in [None, Some(0.5), Some(1.0)] { for use_slicing in [false, true] { - for file_version in test_cases.get_versions() { + for encoding in test_cases.encodings() { if null_rate != Some(1.0) && matches!(field.data_type(), DataType::Null) { continue; } let field = if null_rate.is_some() { - if !supports_nulls(field.data_type(), file_version) { + if !supports_nulls(field.data_type(), encoding) { continue; } field.clone().with_nullable(true) @@ -1184,8 +1510,8 @@ async fn check_round_trip_random( } info!( - "Testing version {} with {} rows divided across {} batches for {} rows per batch with null_rate={:?} and use_slicing={}", - file_version, + "Testing encoding {} with {} rows divided across {} batches for {} rows per batch with null_rate={:?} and use_slicing={}", + encoding, NUM_RANDOM_ROWS, num_ingest_batches, rows_per_batch, @@ -1193,12 +1519,12 @@ async fn check_round_trip_random( use_slicing ); check_round_trip_encoding_inner( - encoder_factory(file_version), + encoder_factory(encoding), &field, data, None, test_cases, - file_version, + encoding, ) .await } diff --git a/rust/lance-encoding/src/version.rs b/rust/lance-encoding/src/version.rs deleted file mode 100644 index cd8f09b011b..00000000000 --- a/rust/lance-encoding/src/version.rs +++ /dev/null @@ -1,147 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// SPDX-FileCopyrightText: Copyright The Lance Authors - -use std::str::FromStr; - -use lance_arrow::DataTypeExt; -use lance_core::datatypes::Field; -use lance_core::deepsize::{Context, DeepSizeOf}; -use lance_core::{Error, Result}; - -pub const LEGACY_FORMAT_VERSION: &str = "0.1"; -pub const V2_FORMAT_2_0: &str = "2.0"; -pub const V2_FORMAT_2_1: &str = "2.1"; -pub const V2_FORMAT_2_2: &str = "2.2"; -pub const V2_FORMAT_2_3: &str = "2.3"; - -/// Lance file version -#[derive(Debug, Default, PartialEq, Eq, Clone, Copy, Ord, PartialOrd, strum::EnumIter)] -pub enum LanceFileVersion { - // This is a little confusing but we rely on the following facts: - // - // Any version <= Next is stable - // The latest version before Stable is the default version for new datasets - // Any version >= Next is unstable - // - // As a result, 'Stable' is not the divider between stable and unstable (Next does this) - // but only serves to mark the default version for new datasets. - // - /// The legacy (0.1) format - Legacy, - V2_0, - #[default] - V2_1, - /// The latest stable release (also the default version for new datasets) - Stable, - V2_2, - /// The latest unstable release - Next, - V2_3, -} - -impl DeepSizeOf for LanceFileVersion { - fn deep_size_of_children(&self, _context: &mut Context) -> usize { - 0 - } -} - -impl LanceFileVersion { - /// Convert Stable or Next to the actual version - pub fn resolve(&self) -> Self { - match self { - Self::Stable => Self::default(), - Self::Next => Self::V2_3, - _ => *self, - } - } - - pub fn is_unstable(&self) -> bool { - self >= &Self::Next - } - - pub fn try_from_major_minor(major: u32, minor: u32) -> Result { - match (major, minor) { - (0, 0) => Ok(Self::Legacy), - (0, 1) => Ok(Self::Legacy), - (0, 2) => Ok(Self::Legacy), - (0, 3) => Ok(Self::V2_0), - (2, 0) => Ok(Self::V2_0), - (2, 1) => Ok(Self::V2_1), - (2, 2) => Ok(Self::V2_2), - (2, 3) => Ok(Self::V2_3), - _ => Err(Error::invalid_input_source( - format!("Unknown Lance storage version: {}.{}", major, minor).into(), - )), - } - } - - pub fn to_numbers(&self) -> (u32, u32) { - match self { - Self::Legacy => (0, 2), - Self::V2_0 => (2, 0), - Self::V2_1 => (2, 1), - Self::V2_2 => (2, 2), - Self::V2_3 => (2, 3), - Self::Stable => self.resolve().to_numbers(), - Self::Next => self.resolve().to_numbers(), - } - } - - pub fn iter_non_legacy() -> impl Iterator { - use strum::IntoEnumIterator; - - Self::iter().filter(|&v| v != Self::Stable && v != Self::Next && v != Self::Legacy) - } - - pub fn support_add_sub_column(&self) -> bool { - self > &Self::V2_1 - } - - pub fn support_remove_sub_column(&self, field: &Field) -> bool { - if self <= &Self::V2_1 { - field.data_type().is_struct() - } else { - field.data_type().is_nested() - } - } -} - -impl std::fmt::Display for LanceFileVersion { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - write!( - f, - "{}", - match self { - Self::Legacy => LEGACY_FORMAT_VERSION, - Self::V2_0 => V2_FORMAT_2_0, - Self::V2_1 => V2_FORMAT_2_1, - Self::V2_2 => V2_FORMAT_2_2, - Self::V2_3 => V2_FORMAT_2_3, - Self::Stable => "stable", - Self::Next => "next", - } - ) - } -} - -impl FromStr for LanceFileVersion { - type Err = Error; - - fn from_str(value: &str) -> Result { - match value.to_lowercase().as_str() { - LEGACY_FORMAT_VERSION => Ok(Self::Legacy), - V2_FORMAT_2_0 => Ok(Self::V2_0), - V2_FORMAT_2_1 => Ok(Self::V2_1), - V2_FORMAT_2_2 => Ok(Self::V2_2), - V2_FORMAT_2_3 => Ok(Self::V2_3), - "stable" => Ok(Self::Stable), - "legacy" => Ok(Self::Legacy), - "next" => Ok(Self::Next), - // Version 0.3 is an alias of 2.0 - "0.3" => Ok(Self::V2_0), - _ => Err(Error::invalid_input_source( - format!("Unknown Lance storage version: {}", value).into(), - )), - } - } -} diff --git a/rust/lance-file/Cargo.toml b/rust/lance-file/Cargo.toml index f08cd3457aa..f9e98f60cca 100644 --- a/rust/lance-file/Cargo.toml +++ b/rust/lance-file/Cargo.toml @@ -19,6 +19,7 @@ lance-io.workspace = true arrow-arith.workspace = true arrow-array.workspace = true arrow-buffer.workspace = true +arrow-cast.workspace = true arrow-data.workspace = true arrow-schema.workspace = true arrow-select.workspace = true @@ -43,6 +44,7 @@ criterion.workspace = true rstest.workspace = true proptest.workspace = true pretty_assertions.workspace = true +rand.workspace = true test-log.workspace = true libc.workspace = true @@ -61,5 +63,9 @@ features = ["protoc"] name = "reader" harness = false +[[bench]] +name = "schema" +harness = false + [lints] workspace = true diff --git a/rust/lance-file/benches/reader.rs b/rust/lance-file/benches/reader.rs index 7ee36e0b6f4..5d280f00d26 100644 --- a/rust/lance-file/benches/reader.rs +++ b/rust/lance-file/benches/reader.rs @@ -14,8 +14,9 @@ use lance_encoding::decoder::{DecoderConfig, DecoderPlugins, FilterExpression}; use lance_file::{ reader::{FileReader, FileReaderOptions}, testing::test_cache, - version::LanceFileVersion, - writer::{FileWriter, FileWriterOptions}, + version::ConcreteFileVersion, + versions as file_versions, + writer::FileWriterOptions, }; use lance_io::{ object_store::ObjectStore, @@ -28,9 +29,9 @@ use tokio::runtime::Runtime; fn bench_reader(c: &mut Criterion) { for version in [ - LanceFileVersion::V2_0, - LanceFileVersion::V2_1, - LanceFileVersion::V2_2, + ConcreteFileVersion::V2_0, + ConcreteFileVersion::V2_1, + ConcreteFileVersion::V2_2, ] { let mut group = c.benchmark_group(format!("reader_{}", version)); let data = lance_datagen::gen_batch() @@ -47,13 +48,11 @@ fn bench_reader(c: &mut Criterion) { let file_path = base_path.clone().join("foo.lance"); let object_writer = rt.block_on(object_store.create(&file_path)).unwrap(); - let mut writer = FileWriter::try_new( + let mut writer = file_versions::create_writer( + version, object_writer, data.schema().as_ref().try_into().unwrap(), - FileWriterOptions { - format_version: Some(version), - ..Default::default() - }, + FileWriterOptions::default(), ) .unwrap(); rt.block_on(writer.write_batch(&data)).unwrap(); @@ -175,7 +174,7 @@ fn get_cached_readers( tmpdir: &TempDir, filesystem: &str, rt: &Runtime, - version: LanceFileVersion, + version: ConcreteFileVersion, ) -> Arc { use std::sync::{LazyLock, Mutex}; @@ -213,13 +212,11 @@ fn get_cached_readers( // Write file let object_writer = rt.block_on(object_store.create(&file_path)).unwrap(); - let mut writer = FileWriter::try_new( + let mut writer = file_versions::create_writer( + version, object_writer, data.schema().as_ref().try_into().unwrap(), - FileWriterOptions { - format_version: Some(version), - ..Default::default() - }, + FileWriterOptions::default(), ) .unwrap(); rt.block_on(writer.write_batch(&data)).unwrap(); @@ -366,9 +363,9 @@ fn bench_random_access(c: &mut Criterion) { let mut group = c.benchmark_group("take"); let versions = [ - LanceFileVersion::V2_0, - LanceFileVersion::V2_1, - LanceFileVersion::V2_2, + ConcreteFileVersion::V2_0, + ConcreteFileVersion::V2_1, + ConcreteFileVersion::V2_2, ]; for filesystem in filesystems { @@ -379,7 +376,8 @@ fn bench_random_access(c: &mut Criterion) { for multithreaded in [false, true] { for rows_at_a_time in [1, 100] { for cached in [true, false] { - if !cached && (filesystem == "mem" || version == LanceFileVersion::V2_0) { + if !cached && (filesystem == "mem" || version == ConcreteFileVersion::V2_0) + { continue; } diff --git a/rust/lance-file/benches/schema.rs b/rust/lance-file/benches/schema.rs new file mode 100644 index 00000000000..b8da23d9777 --- /dev/null +++ b/rust/lance-file/benches/schema.rs @@ -0,0 +1,73 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright The Lance Authors + +use std::hint::black_box; + +use criterion::{BenchmarkId, Criterion, Throughput, criterion_group, criterion_main}; +use lance_core::datatypes::Schema; +use lance_file::{datatypes::Fields, format::pb}; + +fn proto_field(id: i32, parent_id: i32, name: String, logical_type: &str) -> pb::Field { + pb::Field { + id, + parent_id, + name, + logical_type: logical_type.to_owned(), + ..Default::default() + } +} + +/// Builds a pre-order flat schema with `num_physical_columns` physical leaves. +/// +/// Each struct contributes one parent and two `int32` leaves. Root fields use +/// `-1` as `parent_id`, and each struct consumes a three-ID block. +fn wide_two_leaf_structs(num_physical_columns: usize) -> Fields { + assert_eq!(num_physical_columns % 2, 0); + let num_structs = num_physical_columns / 2; + let mut fields = Vec::with_capacity(num_structs + num_physical_columns); + + for struct_index in 0..num_structs { + let parent_id = (struct_index * 3) as i32; + fields.push(proto_field( + parent_id, + -1, + format!("struct_{struct_index}"), + "struct", + )); + fields.push(proto_field( + parent_id + 1, + parent_id, + format!("left_{struct_index}"), + "int32", + )); + fields.push(proto_field( + parent_id + 2, + parent_id, + format!("right_{struct_index}"), + "int32", + )); + } + + Fields(fields) +} + +fn bench_schema_reconstruction(c: &mut Criterion) { + let mut group = c.benchmark_group("schema_from_flat_fields"); + + for num_physical_columns in [1024, 4096, 16_384, 65_536] { + let fields = wide_two_leaf_structs(num_physical_columns); + group.throughput(Throughput::Elements(fields.0.len() as u64)); + group.bench_with_input( + BenchmarkId::new("physical_columns", num_physical_columns), + &fields, + |bencher, fields| { + bencher.iter(|| Schema::try_from(black_box(fields)).unwrap()); + }, + ); + } + + group.finish(); +} + +criterion_group!(benches, bench_schema_reconstruction); +criterion_main!(benches); diff --git a/rust/lance-file/src/compatibility_tests.rs b/rust/lance-file/src/compatibility_tests.rs new file mode 100644 index 00000000000..c63137ec6e2 --- /dev/null +++ b/rust/lance-file/src/compatibility_tests.rs @@ -0,0 +1,524 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright The Lance Authors + +use std::collections::HashMap; +use std::sync::Arc; + +use arrow_array::builder::StringDictionaryBuilder; +use arrow_array::cast::AsArray; +use arrow_array::types::{Int8Type, Int32Type}; +use arrow_array::{ + Array, ArrayRef, Int32Array, LargeBinaryArray, ListArray, RecordBatch, StringArray, +}; +use arrow_schema::{DataType, Field, Schema as ArrowSchema}; +use bytes::Bytes; +use futures::TryStreamExt; +use lance_core::cache::LanceCache; +use lance_core::datatypes::Schema as LanceSchema; +use lance_encoding::decoder::{DecoderPlugins, EncodedBatchLayout, FilterExpression, decode_batch}; +use lance_encoding::encoder::{EncodedBatch, EncodingOptions, encode_batch}; +use lance_io::ReadBatchParams; +use lance_io::traits::Writer; +use lance_io::utils::CachedFileSize; +use rstest::rstest; +use tokio::io::AsyncWriteExt; + +use crate::reader::{EncodedBatchReaderExt, FileReader, FileReaderOptions}; +use crate::testing::FsFixture; +use crate::version::ConcreteFileVersion; +use crate::versions; +use crate::versions::v1::reader::FileReader as V1Reader; +use crate::versions::v1::writer::{ + FileWriter as V1Writer, FileWriterOptions as V1WriterOptions, NotSelfDescribing, +}; +use crate::writer::FileWriterOptions; + +fn compatibility_fixture_batch() -> RecordBatch { + let row_count = 4097; + let ids = Arc::new(Int32Array::from_iter_values(0..row_count)) as ArrayRef; + let names = Arc::new(StringArray::from_iter((0..row_count).map(|index| { + (index % 7 != 0).then(|| format!("value-{index:04}-deterministic-fixture")) + }))) as ArrayRef; + let items = Arc::new(ListArray::from_iter_primitive::( + (0..row_count).map(|index| { + (index % 11 != 0).then(|| { + vec![ + Some(index), + (index % 5 != 0).then_some(index * 2), + Some(index * 3), + ] + }) + }), + )) as ArrayRef; + let mut categories = StringDictionaryBuilder::::new(); + for index in 0..row_count { + if index % 13 == 0 { + categories.append_null(); + } else { + categories + .append(match index % 3 { + 0 => "red", + 1 => "green", + _ => "blue", + }) + .unwrap(); + } + } + let categories = Arc::new(categories.finish()) as ArrayRef; + let blobs = Arc::new(LargeBinaryArray::from_iter_values( + (0..row_count).map(|index| format!("blob-{index:04}-deterministic-payload").into_bytes()), + )) as ArrayRef; + + let schema = Arc::new(ArrowSchema::new(vec![ + Field::new("id", DataType::Int32, false), + Field::new("name", DataType::Utf8, true).with_metadata(HashMap::from([( + "lance-encoding:compression".to_string(), + "none".to_string(), + )])), + Field::new( + "items", + DataType::List(Arc::new(Field::new("item", DataType::Int32, true))), + true, + ), + Field::new( + "category", + DataType::Dictionary(Box::new(DataType::Int8), Box::new(DataType::Utf8)), + true, + ) + .with_metadata(HashMap::from([( + "lance-encoding:dict-values-compression".to_string(), + "none".to_string(), + )])), + Field::new("blob", DataType::LargeBinary, true).with_metadata(HashMap::from([( + "lance-encoding:blob".to_string(), + "true".to_string(), + )])), + ])); + RecordBatch::try_new(schema, vec![ids, names, items, categories, blobs]).unwrap() +} + +fn v1_reader_expected_batch(batch: &RecordBatch) -> RecordBatch { + // The V1 reader historically materializes null lists as empty, null child integers as zero, + // and null dictionary keys as the first dictionary value. + let items = Arc::new(ListArray::from_iter_primitive::( + (0..batch.num_rows() as i32).map(|index| { + Some(if index % 11 == 0 { + Vec::new() + } else { + vec![ + Some(index), + Some(if index % 5 == 0 { 0 } else { index * 2 }), + Some(index * 3), + ] + }) + }), + )) as ArrayRef; + let mut categories = StringDictionaryBuilder::::new(); + for index in 0..batch.num_rows() { + categories + .append(if index % 13 == 0 { + "green" + } else { + match index % 3 { + 0 => "red", + 1 => "green", + _ => "blue", + } + }) + .unwrap(); + } + let categories = Arc::new(categories.finish()) as ArrayRef; + let mut columns = batch.columns().to_vec(); + columns[2] = items; + columns[3] = categories; + RecordBatch::try_new(batch.schema(), columns).unwrap() +} + +fn stable_fixture(version: ConcreteFileVersion) -> &'static [u8] { + match version { + ConcreteFileVersion::V1 => include_bytes!("../test_data/exact_versions/v1.lance"), + ConcreteFileVersion::V2_0 => { + include_bytes!("../test_data/exact_versions/v2_0.lance") + } + ConcreteFileVersion::V2_1 => { + include_bytes!("../test_data/exact_versions/v2_1.lance") + } + ConcreteFileVersion::V2_2 => { + include_bytes!("../test_data/exact_versions/v2_2.lance") + } + ConcreteFileVersion::V2_3 => { + unreachable!("v2.3 is unstable and has no compatibility fixture") + } + } +} + +fn assert_blob_column_eq(actual: &dyn Array, expected: &dyn Array) { + let actual = actual.as_binary::(); + let expected = expected.as_binary::(); + assert_eq!(actual.len(), expected.len()); + for index in 0..actual.len() { + assert_eq!( + actual.is_null(index), + expected.is_null(index), + "blob validity differs at row {index}" + ); + if actual.is_valid(index) { + assert_eq!( + actual.value(index), + expected.value(index), + "blob payload differs at row {index}" + ); + } + } +} + +fn assert_record_batch_eq(actual: &RecordBatch, expected: &RecordBatch) { + assert_eq!(actual.schema_ref(), expected.schema_ref()); + assert_eq!(actual.num_rows(), expected.num_rows()); + assert_eq!(actual.num_columns(), expected.num_columns()); + + for column_index in 0..actual.num_columns() { + if expected.schema().field(column_index).name() == "blob" { + assert_blob_column_eq( + actual.column(column_index).as_ref(), + expected.column(column_index).as_ref(), + ); + } else if actual.column(column_index).to_data() != expected.column(column_index).to_data() { + let row_index = (0..actual.num_rows()) + .find(|row_index| { + actual.column(column_index).slice(*row_index, 1).to_data() + != expected.column(column_index).slice(*row_index, 1).to_data() + }) + .unwrap(); + panic!( + "column {} ({}) differs at row {}: actual={:?}, expected={:?}", + column_index, + expected.schema().field(column_index).name(), + row_index, + actual.column(column_index).slice(row_index, 1), + expected.column(column_index).slice(row_index, 1) + ); + } + } +} + +fn footer_version(bytes: &[u8]) -> (u16, u16) { + let version_start = bytes.len() - 8; + ( + u16::from_le_bytes([bytes[version_start], bytes[version_start + 1]]), + u16::from_le_bytes([bytes[version_start + 2], bytes[version_start + 3]]), + ) +} + +fn assert_wire_bytes_equal(actual: &[u8], expected: &[u8]) { + if let Some(offset) = actual + .iter() + .zip(expected) + .position(|(actual, expected)| actual != expected) + { + panic!( + "wire fixture first differs at byte {offset}: actual={}, expected={}", + actual[offset], expected[offset] + ); + } + assert_eq!( + actual.len(), + expected.len(), + "wire fixture length changed after a common {}-byte prefix", + actual.len().min(expected.len()) + ); +} + +async fn write_current_fixture( + version: ConcreteFileVersion, + batch: &RecordBatch, + schema: &LanceSchema, +) -> Vec { + let fs = FsFixture::default(); + let object_writer = fs.object_store.create(&fs.tmp_path).await.unwrap(); + let options = FileWriterOptions { + data_cache_bytes: Some(1), + max_page_bytes: Some(1024), + ..Default::default() + }; + let summary = match version { + ConcreteFileVersion::V1 => { + unreachable!("legacy fixtures use the legacy writer") + } + ConcreteFileVersion::V2_0 => { + let mut writer = + versions::v2_0::create_writer(object_writer, schema.clone(), options).unwrap(); + for offset in (0..batch.num_rows()).step_by(1024) { + let slice = batch.slice(offset, (batch.num_rows() - offset).min(1024)); + writer.write_batch(&slice).await.unwrap(); + } + writer.finish().await.unwrap() + } + ConcreteFileVersion::V2_1 => { + let mut writer = + versions::v2_1::create_writer(object_writer, schema.clone(), options).unwrap(); + for offset in (0..batch.num_rows()).step_by(1024) { + let slice = batch.slice(offset, (batch.num_rows() - offset).min(1024)); + writer.write_batch(&slice).await.unwrap(); + } + writer.finish().await.unwrap() + } + ConcreteFileVersion::V2_2 => { + let mut writer = + versions::v2_2::create_writer(object_writer, schema.clone(), options).unwrap(); + for offset in (0..batch.num_rows()).step_by(1024) { + let slice = batch.slice(offset, (batch.num_rows() - offset).min(1024)); + writer.write_batch(&slice).await.unwrap(); + } + writer.finish().await.unwrap() + } + ConcreteFileVersion::V2_3 => { + let mut writer = + versions::v2_3::create_writer(object_writer, schema.clone(), options).unwrap(); + for offset in (0..batch.num_rows()).step_by(1024) { + let slice = batch.slice(offset, (batch.num_rows() - offset).min(1024)); + writer.write_batch(&slice).await.unwrap(); + } + writer.finish().await.unwrap() + } + }; + fs.object_store + .open(&fs.tmp_path) + .await + .unwrap() + .get_range(0..summary.size_bytes as usize) + .await + .unwrap() + .to_vec() +} + +async fn write_v2_0_embedded_fixtures(batch: &RecordBatch, schema: &LanceSchema) -> (Bytes, Bytes) { + let options = EncodingOptions { + cache_bytes_per_column: 1, + max_page_bytes: 1024, + keep_original_array: true, + buffer_alignment: 64, + }; + let encoding_strategy = crate::versions::v2_0::encoding_strategy(); + let encoded_batch = encode_batch( + batch, + Arc::new(schema.clone()), + encoding_strategy.as_ref(), + &options, + ) + .await + .unwrap(); + + ( + versions::v2_0::encode_self_described_batch(&encoded_batch).unwrap(), + versions::v2_0::encode_mini_batch(&encoded_batch).unwrap(), + ) +} + +async fn assert_current_reader_roundtrip( + fixture: &[u8], + version: ConcreteFileVersion, + expected: &RecordBatch, +) { + let fs = FsFixture::default(); + let mut fixture_writer = fs.object_store.create(&fs.tmp_path).await.unwrap(); + fixture_writer.write_all(fixture).await.unwrap(); + Writer::shutdown(fixture_writer.as_mut()).await.unwrap(); + let scheduler = fs + .scheduler + .open_file(&fs.tmp_path, &CachedFileSize::new(fixture.len() as u64)) + .await + .unwrap(); + let reader = FileReader::try_open( + scheduler, + None, + Arc::::default(), + &LanceCache::no_cache(), + FileReaderOptions::default(), + ) + .await + .unwrap(); + assert_eq!(reader.metadata().version(), version); + assert!( + reader + .metadata() + .column_metadatas + .iter() + .any(|metadata| metadata.pages.len() > 1) + ); + let batches = reader + .read_stream( + ReadBatchParams::RangeFull, + 1024, + 16, + FilterExpression::no_filter(), + ) + .await + .unwrap() + .try_collect::>() + .await + .unwrap(); + assert_eq!( + batches.iter().map(RecordBatch::num_rows).sum::(), + expected.num_rows() + ); + assert!( + batches + .iter() + .all(|actual| actual.schema_ref() == expected.schema_ref()) + ); + let mut row_offset = 0; + for actual in &batches { + let expected = expected.slice(row_offset, actual.num_rows()); + assert_record_batch_eq(actual, &expected); + row_offset += actual.num_rows(); + } + assert_eq!(row_offset, expected.num_rows()); +} + +#[rstest] +#[case::v2_0(ConcreteFileVersion::V2_0)] +#[case::v2_1(ConcreteFileVersion::V2_1)] +#[case::v2_2(ConcreteFileVersion::V2_2)] +#[tokio::test] +async fn stable_current_writer_and_reader_are_wire_compatible( + #[case] version: ConcreteFileVersion, +) { + let batch = compatibility_fixture_batch(); + let mut schema = LanceSchema::try_from(batch.schema().as_ref()).unwrap(); + schema.set_dictionary(&batch).unwrap(); + + let actual = write_current_fixture(version, &batch, &schema).await; + let expected = stable_fixture(version); + assert_wire_bytes_equal(&actual, expected); + assert_eq!( + footer_version(expected), + version.to_standard_footer_numbers() + ); + assert_current_reader_roundtrip(expected, version, &batch).await; +} + +#[tokio::test] +async fn v2_0_embedded_writer_and_reader_are_wire_compatible() { + let batch = compatibility_fixture_batch() + .project(&[0, 1]) + .unwrap() + .slice(0, 257); + let mut schema = LanceSchema::try_from(batch.schema().as_ref()).unwrap(); + schema.set_dictionary(&batch).unwrap(); + + let (actual_self_described, actual_mini) = write_v2_0_embedded_fixtures(&batch, &schema).await; + let expected_self_described = + include_bytes!("../test_data/exact_versions/v2_0_self_described.lance"); + let expected_mini = include_bytes!("../test_data/exact_versions/v2_0_mini.lance"); + assert_wire_bytes_equal(&actual_self_described, expected_self_described); + assert_wire_bytes_equal(&actual_mini, expected_mini); + + let expected_footer = ConcreteFileVersion::V2_0.to_embedded_footer_numbers(); + assert_eq!(footer_version(expected_self_described), expected_footer); + assert_eq!(footer_version(expected_mini), expected_footer); + + let self_described = + EncodedBatch::try_from_self_described_lance(Bytes::from_static(expected_self_described)) + .unwrap(); + let decoded = decode_batch( + &self_described, + &FilterExpression::no_filter(), + Arc::::default(), + false, + EncodedBatchLayout::Array, + None, + ) + .await + .unwrap(); + assert_record_batch_eq(&decoded, &batch); + + let mini = + EncodedBatch::try_from_mini_lance(Bytes::from_static(expected_mini), &schema).unwrap(); + let decoded = decode_batch( + &mini, + &FilterExpression::no_filter(), + Arc::::default(), + false, + EncodedBatchLayout::Array, + None, + ) + .await + .unwrap(); + assert_record_batch_eq(&decoded, &batch); +} + +#[tokio::test] +async fn v2_3_output_is_deterministic_within_the_current_revision() { + let batch = compatibility_fixture_batch(); + let mut schema = LanceSchema::try_from(batch.schema().as_ref()).unwrap(); + schema.set_dictionary(&batch).unwrap(); + + let first = write_current_fixture(ConcreteFileVersion::V2_3, &batch, &schema).await; + let second = write_current_fixture(ConcreteFileVersion::V2_3, &batch, &schema).await; + assert_eq!(first, second); + assert_eq!( + footer_version(&first), + ConcreteFileVersion::V2_3.to_standard_footer_numbers() + ); + assert_current_reader_roundtrip(&first, ConcreteFileVersion::V2_3, &batch).await; +} + +#[tokio::test] +async fn v1_writer_and_reader_are_wire_compatible() { + let expected = stable_fixture(ConcreteFileVersion::V1); + let batch = compatibility_fixture_batch(); + let mut schema = LanceSchema::try_from(batch.schema().as_ref()).unwrap(); + schema.set_dictionary(&batch).unwrap(); + let fs = FsFixture::default(); + let mut writer = V1Writer::::try_new( + fs.object_store.as_ref(), + &fs.tmp_path, + schema.clone(), + &V1WriterOptions { + collect_stats_for_fields: Some(Vec::new()), + }, + ) + .await + .unwrap(); + for offset in (0..batch.num_rows()).step_by(1024) { + let slice = batch.slice(offset, (batch.num_rows() - offset).min(1024)); + writer.write(std::slice::from_ref(&slice)).await.unwrap(); + } + let summary = writer.finish().await.unwrap(); + let actual = fs + .object_store + .open(&fs.tmp_path) + .await + .unwrap() + .get_range(0..summary.size_bytes as usize) + .await + .unwrap(); + assert_wire_bytes_equal(actual.as_ref(), expected); + assert_eq!( + footer_version(expected), + ConcreteFileVersion::V1.to_standard_footer_numbers() + ); + + let fixture_fs = FsFixture::default(); + let mut fixture_writer = fixture_fs + .object_store + .create(&fixture_fs.tmp_path) + .await + .unwrap(); + fixture_writer.write_all(expected).await.unwrap(); + Writer::shutdown(fixture_writer.as_mut()).await.unwrap(); + let reader = V1Reader::try_new( + fixture_fs.object_store.as_ref(), + &fixture_fs.tmp_path, + schema.clone(), + ) + .await + .unwrap(); + let actual_batch = reader + .read_range(0..batch.num_rows(), &schema) + .await + .unwrap(); + assert_eq!(reader.num_batches(), 5); + assert_record_batch_eq(&actual_batch, &v1_reader_expected_batch(&batch)); +} diff --git a/rust/lance-file/src/datatypes.rs b/rust/lance-file/src/datatypes.rs index ac6a8d7b293..3d84e99267c 100644 --- a/rust/lance-file/src/datatypes.rs +++ b/rust/lance-file/src/datatypes.rs @@ -1,14 +1,9 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright The Lance Authors -use arrow_schema::DataType; -use async_recursion::async_recursion; use lance_arrow::ARROW_EXT_NAME_KEY; -use lance_arrow::DataTypeExt; use lance_core::datatypes::{Dictionary, Encoding, Field, LogicalType, Schema}; use lance_core::{Error, Result}; -use lance_io::traits::Reader; -use lance_io::utils::{read_binary_array, read_fixed_stride_array}; use std::collections::HashMap; use crate::format::pb; @@ -99,6 +94,32 @@ impl From<&Field> for pb::Field { pub struct Fields(pub Vec); +struct FieldNode { + field: Field, + child_indices: Vec, +} + +/// Searches in pre-order depth-first order and returns the first matching node, +/// preserving the legacy parent tie-break for duplicate field IDs. +fn first_field_index_by_id( + nodes: &[FieldNode], + root_indices: &[usize], + field_id: i32, +) -> Option { + let mut to_visit = Vec::with_capacity(nodes.len()); + to_visit.extend(root_indices.iter().rev().copied()); + + while let Some(node_index) = to_visit.pop() { + let node = &nodes[node_index]; + if node.field.id == field_id { + return Some(node_index); + } + to_visit.extend(node.child_indices.iter().rev().copied()); + } + + None +} + impl From<&Field> for Fields { fn from(field: &Field) -> Self { let mut protos = vec![pb::Field::from(field)]; @@ -107,24 +128,119 @@ impl From<&Field> for Fields { } } -/// Convert list of protobuf `Field` to a Schema. -impl From<&Fields> for Schema { - fn from(fields: &Fields) -> Self { - let mut schema = Self { - fields: vec![], - metadata: HashMap::default(), - }; - - fields.0.iter().for_each(|f| { - if f.parent_id == -1 { - schema.fields.push(Field::from(f)); +/// Reconstruct a schema from a flat, pre-order protobuf field list. +/// +/// Parent fields must appear before their children. Historical manifests may +/// contain duplicate field IDs, so an ID may not identify a unique parent. For +/// those references, reconstruction preserves the legacy +/// [`Schema::mut_field_by_id`] tie-break by selecting the first matching field +/// in pre-order depth-first traversal. +/// +/// # Examples +/// +/// ``` +/// use lance_core::datatypes::Schema; +/// use lance_file::{datatypes::Fields, format::pb}; +/// +/// let field = pb::Field { +/// id: 0, +/// parent_id: -1, +/// name: "value".to_owned(), +/// logical_type: "int32".to_owned(), +/// ..Default::default() +/// }; +/// let fields = Fields(vec![field]); +/// let schema = Schema::try_from(&fields)?; +/// assert_eq!(schema.fields[0].name, "value"); +/// # Ok::<(), lance_core::Error>(()) +/// ``` +impl TryFrom<&Fields> for Schema { + type Error = Error; + + fn try_from(fields: &Fields) -> Result { + let mut nodes: Vec = Vec::with_capacity(fields.0.len()); + let mut root_indices = Vec::with_capacity(fields.0.len()); + let mut field_indices: HashMap> = HashMap::with_capacity(fields.0.len()); + + for proto_field in &fields.0 { + let parent_index = if proto_field.parent_id == -1 { + None } else { - let parent = schema.mut_field_by_id(f.parent_id).unwrap(); - parent.children.push(Field::from(f)); + let parent_index = match field_indices.get(&proto_field.parent_id) { + Some(Some(parent_index)) => *parent_index, + Some(None) => { + // Duplicate IDs are invalid but occur in historical + // manifests. Match the legacy tree traversal only for + // these ambiguous parent references so valid schemas + // retain the linear fast path. + first_field_index_by_id(&nodes, &root_indices, proto_field.parent_id) + .ok_or_else(|| { + Error::internal(format!( + "Duplicate field id {} has no existing arena node", + proto_field.parent_id + )) + })? + } + None => { + return Err(Error::schema(format!( + "Field '{}' (id={}) references parent id {}, which must appear earlier in the protobuf field list", + proto_field.name, proto_field.id, proto_field.parent_id + ))); + } + }; + Some(parent_index) + }; + + let node_index = nodes.len(); + if let Some(parent_index) = parent_index { + nodes[parent_index].child_indices.push(node_index); + } else { + root_indices.push(node_index); } - }); + nodes.push(FieldNode { + field: Field::from(proto_field), + child_indices: Vec::new(), + }); + + field_indices + .entry(proto_field.id) + .and_modify(|field_index| *field_index = None) + .or_insert(Some(node_index)); + } + + let mut fields_by_node = Vec::with_capacity(nodes.len()); + fields_by_node.resize_with(nodes.len(), || None); + for (node_index, mut node) in nodes.into_iter().enumerate().rev() { + node.field.children.reserve(node.child_indices.len()); + for child_index in node.child_indices { + let child = fields_by_node + .get_mut(child_index) + .and_then(Option::take) + .ok_or_else(|| { + Error::internal(format!( + "Schema field arena node {child_index} was not materialized before its parent" + )) + })?; + node.field.children.push(child); + } + fields_by_node[node_index] = Some(node.field); + } + + let fields = root_indices + .into_iter() + .map(|root_index| { + fields_by_node[root_index].take().ok_or_else(|| { + Error::internal(format!( + "Schema field arena root node {root_index} was not materialized" + )) + }) + }) + .collect::>>()?; - schema + Ok(Self { + fields, + metadata: HashMap::default(), + }) } } @@ -133,9 +249,28 @@ pub struct FieldsWithMeta { pub metadata: HashMap>, } -/// Convert list of protobuf `Field` and Metadata to a Schema. -impl From for Schema { - fn from(fields_with_meta: FieldsWithMeta) -> Self { +/// Reconstruct a schema from flat protobuf fields and schema metadata. +/// +/// # Examples +/// +/// ``` +/// use std::collections::HashMap; +/// +/// use lance_core::datatypes::Schema; +/// use lance_file::datatypes::{Fields, FieldsWithMeta}; +/// +/// let fields = FieldsWithMeta { +/// fields: Fields(Vec::new()), +/// metadata: HashMap::from([("owner".to_owned(), b"lance".to_vec())]), +/// }; +/// let schema = Schema::try_from(fields)?; +/// assert_eq!(schema.metadata["owner"], "lance"); +/// # Ok::<(), lance_core::Error>(()) +/// ``` +impl TryFrom for Schema { + type Error = Error; + + fn try_from(fields_with_meta: FieldsWithMeta) -> Result { let lance_metadata = fields_with_meta .metadata .into_iter() @@ -145,11 +280,11 @@ impl From for Schema { }) .collect(); - let schema_with_fields = Self::from(&fields_with_meta.fields); - Self { + let schema_with_fields = Self::try_from(&fields_with_meta.fields)?; + Ok(Self { fields: schema_with_fields.fields, metadata: lance_metadata, - } + }) } } @@ -208,76 +343,29 @@ impl From for pb::Encoding { } } -#[async_recursion] -async fn load_field_dictionary<'a>(field: &mut Field, reader: &dyn Reader) -> Result<()> { - if let DataType::Dictionary(_, value_type) = field.data_type() { - assert!(field.dictionary.is_some()); - if let Some(dict_info) = field.dictionary.as_mut() { - use DataType::*; - match value_type.as_ref() { - _ if value_type.is_binary_like() => { - dict_info.values = Some( - read_binary_array( - reader, - value_type.as_ref(), - true, // Empty values are null - dict_info.offset, - dict_info.length, - .., - ) - .await?, - ); - } - Int8 | Int16 | Int32 | Int64 | UInt8 | UInt16 | UInt32 | UInt64 => { - dict_info.values = Some( - read_fixed_stride_array( - reader, - value_type.as_ref(), - dict_info.offset, - dict_info.length, - .., - ) - .await?, - ); - } - _ => { - return Err(Error::schema(format!( - "Does not support {} as dictionary value type", - value_type - ))); - } - } - } else { - panic!("Should not reach here: dictionary field does not load dictionary info") - } - Ok(()) - } else { - for child in field.children.as_mut_slice() { - load_field_dictionary(child, reader).await?; - } - Ok(()) - } -} - -/// Load dictionary value array from manifest files. -// TODO: pub(crate) -pub async fn populate_schema_dictionary(schema: &mut Schema, reader: &dyn Reader) -> Result<()> { - for field in schema.fields.as_mut_slice() { - load_field_dictionary(field, reader).await?; - } - Ok(()) -} - #[cfg(test)] mod tests { + use std::collections::HashMap; + use arrow_schema::DataType; use arrow_schema::Field as ArrowField; use arrow_schema::Fields as ArrowFields; use arrow_schema::Schema as ArrowSchema; + use lance_core::Error; use lance_core::datatypes::Schema; - use std::collections::HashMap; use super::{Fields, FieldsWithMeta}; + use crate::format::pb; + + fn proto_field(id: i32, parent_id: i32, name: String, logical_type: &str) -> pb::Field { + pb::Field { + id, + parent_id, + name, + logical_type: logical_type.to_owned(), + ..Default::default() + } + } #[test] fn test_schema_set_ids() { @@ -317,10 +405,120 @@ mod tests { let expected_schema = Schema::try_from(&arrow_schema).unwrap(); let fields_with_meta: FieldsWithMeta = (&expected_schema).into(); - let schema = Schema::from(fields_with_meta); + let schema = Schema::try_from(fields_with_meta).unwrap(); assert_eq!(expected_schema, schema); } + #[test] + fn test_reconstruct_wide_nested_schema() { + const NUM_STRUCTS: usize = 4096; + + let mut proto_fields = Vec::with_capacity(NUM_STRUCTS * 3); + for struct_index in 0..NUM_STRUCTS { + let parent_id = (struct_index * 3) as i32; + proto_fields.push(proto_field( + parent_id, + -1, + format!("struct_{struct_index}"), + "struct", + )); + proto_fields.push(proto_field( + parent_id + 1, + parent_id, + format!("left_{struct_index}"), + "int32", + )); + proto_fields.push(proto_field( + parent_id + 2, + parent_id, + format!("right_{struct_index}"), + "int32", + )); + } + + let fields = Fields(proto_fields); + let schema = Schema::try_from(&fields).unwrap(); + assert_eq!(schema.fields.len(), NUM_STRUCTS); + for (struct_index, field) in schema.fields.iter().enumerate() { + let parent_id = (struct_index * 3) as i32; + assert_eq!(field.id, parent_id); + assert_eq!(field.name, format!("struct_{struct_index}")); + assert_eq!(field.children.len(), 2); + assert_eq!(field.children[0].id, parent_id + 1); + assert_eq!(field.children[0].name, format!("left_{struct_index}")); + assert_eq!(field.children[1].id, parent_id + 2); + assert_eq!(field.children[1].name, format!("right_{struct_index}")); + } + } + + #[test] + fn test_reconstruct_deep_nested_schema() { + const DEPTH: usize = 1024; + + let proto_fields = (0..DEPTH) + .map(|depth| { + proto_field( + depth as i32, + if depth == 0 { -1 } else { depth as i32 - 1 }, + format!("level_{depth}"), + if depth + 1 == DEPTH { + "int32" + } else { + "struct" + }, + ) + }) + .collect(); + + let fields = Fields(proto_fields); + let schema = Schema::try_from(&fields).unwrap(); + assert_eq!(schema.fields.len(), 1); + let mut field = &schema.fields[0]; + for depth in 0..DEPTH { + assert_eq!(field.id, depth as i32); + assert_eq!(field.name, format!("level_{depth}")); + if depth + 1 == DEPTH { + assert!(field.children.is_empty()); + } else { + assert_eq!(field.children.len(), 1); + field = &field.children[0]; + } + } + } + + #[test] + fn test_reconstruct_schema_reports_missing_parent() { + let fields = Fields(vec![proto_field(7, 42, "child".to_owned(), "int32")]); + + let error = Schema::try_from(&fields).unwrap_err(); + assert!(matches!(&error, Error::Schema { .. })); + assert!( + error.to_string().contains( + "Field 'child' (id=7) references parent id 42, which must appear earlier" + ) + ); + } + + #[test] + fn test_reconstruct_schema_preserves_legacy_duplicate_id_match() { + let fields = Fields(vec![ + proto_field(1, -1, "root_a".to_owned(), "struct"), + proto_field(2, -1, "root_b".to_owned(), "struct"), + proto_field(2, 1, "nested_duplicate".to_owned(), "struct"), + proto_field(3, 2, "child".to_owned(), "int32"), + ]); + + let schema = Schema::try_from(&fields).unwrap(); + assert_eq!(schema.fields.len(), 2); + assert_eq!(schema.fields[0].name, "root_a"); + assert_eq!(schema.fields[0].children.len(), 1); + assert_eq!(schema.fields[0].children[0].name, "nested_duplicate"); + assert_eq!(schema.fields[0].children[0].children.len(), 1); + assert_eq!(schema.fields[0].children[0].children[0].name, "child"); + assert_eq!(schema.fields[1].name, "root_b"); + assert!(schema.fields[1].children.is_empty()); + } + #[test] fn test_clustering_key_roundtrip() { let arrow_schema = ArrowSchema::new(vec![ @@ -351,7 +549,7 @@ mod tests { // Round-trip through protobuf let fields_with_meta: FieldsWithMeta = (&schema).into(); - let restored = Schema::from(fields_with_meta); + let restored = Schema::try_from(fields_with_meta).unwrap(); let ck2 = restored.unenforced_clustering_key(); assert_eq!(ck2.len(), 2); diff --git a/rust/lance-file/src/io.rs b/rust/lance-file/src/io.rs index 1a8edf92b08..86e5189a81d 100644 --- a/rust/lance-file/src/io.rs +++ b/rust/lance-file/src/io.rs @@ -55,14 +55,27 @@ impl EncodingsIo for LanceEncodingsIo { ) -> BoxFuture<'static, lance_core::Result>> { let mut split_ranges = Vec::new(); let mut split_indices = Vec::new(); // Track which original range each split came from + // Large ranges (above read_chunk_size) will be split into + // multiple reads. Empty ranges will skip the I/O layer + // entirely. If we have either of these we will need to + // reassemble our results, inserting empties and merging parts + let mut needs_reassembly = false; // Split large ranges into smaller chunks // // TODO: consider read_chunk_size before submitting requests. for (idx, range) in ranges.iter().enumerate() { + if range.start == range.end { + // EncodingsIo requires one result per input range. Zero-length + // ranges schedule no I/O, so their empty results are restored + // after the non-empty requests complete. + needs_reassembly = true; + continue; + } let range_size = range.end - range.start; if range_size > self.read_chunk_size { + needs_reassembly = true; let num_chunks = range_size.div_ceil(self.read_chunk_size); let chunk_size = range_size / num_chunks; @@ -87,34 +100,48 @@ impl EncodingsIo for LanceEncodingsIo { async move { let split_results = fut.await?; - // Fast path: if no splitting occurred, return results directly - if split_results.len() == ranges.len() { + if split_results.len() != split_indices.len() { + return Err(lance_core::Error::internal(format!( + "Encoding I/O returned {} results for {} requested range chunks", + split_results.len(), + split_indices.len() + ))); + } + if !needs_reassembly { return Ok(split_results); } - // Slow path: reassemble split results let mut results = vec![Vec::new(); ranges.len()]; - for (split_result, &orig_idx) in split_results.iter().zip(split_indices.iter()) { - results[orig_idx].push(split_result.clone()); + for (split_result, orig_idx) in split_results.into_iter().zip(split_indices) { + results[orig_idx].push(split_result); } - Ok(results - .into_iter() - .map(|chunks| { - if chunks.len() == 1 { - chunks.into_iter().next().unwrap() - } else { - // Concatenate multiple chunks - let total_size: usize = chunks.iter().map(|c| c.len()).sum(); - let mut combined = Vec::with_capacity(total_size); - for chunk in chunks { - combined.extend_from_slice(&chunk); - } - bytes::Bytes::from(combined) + let mut reassembled = Vec::with_capacity(ranges.len()); + for (range, chunks) in ranges.iter().zip(results) { + if chunks.is_empty() { + if range.start == range.end { + reassembled.push(bytes::Bytes::new()); + continue; } - }) - .collect()) + return Err(lance_core::Error::internal(format!( + "Encoding I/O returned no data for non-empty range {}..{}", + range.start, range.end + ))); + } + if chunks.len() == 1 { + reassembled.push(chunks[0].clone()); + continue; + } + + let total_size: usize = chunks.iter().map(|c| c.len()).sum(); + let mut combined = Vec::with_capacity(total_size); + for chunk in chunks { + combined.extend_from_slice(&chunk); + } + reassembled.push(bytes::Bytes::from(combined)); + } + Ok(reassembled) } .boxed() } diff --git a/rust/lance-file/src/lib.rs b/rust/lance-file/src/lib.rs index c89f7c7b5bf..32dfdb89f80 100644 --- a/rust/lance-file/src/lib.rs +++ b/rust/lance-file/src/lib.rs @@ -4,28 +4,32 @@ pub mod datatypes; pub mod format; pub(crate) mod io; -pub mod previous; pub mod reader; pub mod testing; +pub mod version; +pub mod versions; pub mod writer; +#[cfg(test)] +mod compatibility_tests; + pub use io::LanceEncodingsIo; use format::MAGIC; -pub use lance_encoding::version; - use lance_core::{Error, Result}; -use lance_encoding::version::LanceFileVersion; use lance_io::object_store::ObjectStore; use object_store::path::Path; +use version::ConcreteFileVersion; pub async fn determine_file_version( store: &ObjectStore, path: &Path, known_size: Option, -) -> Result { +) -> Result { let size = match known_size { - None => store.size(path).await.unwrap() as usize, + None => usize::try_from(store.size(path).await?).map_err(|_| { + Error::invalid_input(format!("file {} is too large for this platform", path)) + })?, Some(size) => size, }; if size < 8 { @@ -51,5 +55,5 @@ pub async fn determine_file_version( let major_version = u16::from_le_bytes([footer[0], footer[1]]); let minor_version = u16::from_le_bytes([footer[2], footer[3]]); - LanceFileVersion::try_from_major_minor(major_version as u32, minor_version as u32) + ConcreteFileVersion::from_footer_numbers(major_version, minor_version) } diff --git a/rust/lance-file/src/previous/mod.rs b/rust/lance-file/src/previous/mod.rs deleted file mode 100644 index 9031d2b4992..00000000000 --- a/rust/lance-file/src/previous/mod.rs +++ /dev/null @@ -1,9 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// SPDX-FileCopyrightText: Copyright The Lance Authors - -//! Legacy Lance file v1 implementation kept for backwards compatibility. - -pub mod format; -pub mod page_table; -pub mod reader; -pub mod writer; diff --git a/rust/lance-file/src/reader.rs b/rust/lance-file/src/reader.rs index 048cf550d66..a8e4619e4d0 100644 --- a/rust/lance-file/src/reader.rs +++ b/rust/lance-file/src/reader.rs @@ -4,6 +4,7 @@ use std::{ borrow::Cow, collections::{BTreeMap, BTreeSet}, + fmt::Debug, io::Cursor, ops::Range, pin::Pin, @@ -12,6 +13,7 @@ use std::{ use arrow_array::RecordBatchReader; use arrow_schema::Schema as ArrowSchema; +use async_trait::async_trait; use byteorder::{ByteOrder, LittleEndian, ReadBytesExt}; use bytes::{Bytes, BytesMut}; use futures::{Stream, StreamExt, stream::BoxStream}; @@ -19,20 +21,18 @@ use lance_core::deepsize::{Context, DeepSizeOf}; use lance_encoding::{ EncodingsIo, decoder::{ - ColumnInfo, DecoderConfig, DecoderPlugins, FilterExpression, PageEncoding, PageInfo, - ReadBatchTask, RequestedRows, SchedulerDecoderConfig, schedule_and_decode, - schedule_and_decode_blocking, + ColumnInfo, DecoderConfig, DecoderPlugins, FilterExpression, PageEncoding, ReadBatchTask, + RequestedRows, SchedulerDecoderConfig, schedule_and_decode, schedule_and_decode_blocking, }, encoder::EncodedBatch, - version::LanceFileVersion, }; use log::debug; use object_store::path::Path; -use prost::{Message, Name}; +use prost::Message; use lance_core::{ Error, Result, - cache::{CacheKey, LanceCache}, + cache::{CacheKey, CacheKeySchema, KeyBuilder, LanceCache}, datatypes::{Field, Schema}, }; use lance_encoding::format::pb as pbenc; @@ -45,11 +45,14 @@ use lance_io::{ use crate::{ datatypes::{Fields, FieldsWithMeta}, - format::{MAGIC, MAJOR_VERSION, MINOR_VERSION, pb, pbfile}, + format::{MAGIC, pb, pbfile}, io::LanceEncodingsIo, - writer::PAGE_BUFFER_ALIGNMENT, + version::ConcreteFileVersion, + versions, }; +pub(crate) mod structural; + /// Default chunk size for reading large pages (8MiB) /// Pages larger than this will be split into multiple chunks during read pub const DEFAULT_READ_CHUNK_SIZE: u64 = 8 * 1024 * 1024; @@ -64,6 +67,39 @@ pub struct BufferDescriptor { pub size: u64, } +impl BufferDescriptor { + fn checked_range(&self, buffer_index: usize, file_len: u64) -> Result> { + let end = self.position.checked_add(self.size).ok_or_else(|| { + Error::invalid_input_source( + format!( + "Global buffer {} range overflows: position={}, size={}", + buffer_index, self.position, self.size + ) + .into(), + ) + })?; + if self.position > file_len { + return Err(Error::invalid_input_source( + format!( + "Global buffer {} position {} is outside file of size {}", + buffer_index, self.position, file_len + ) + .into(), + )); + } + if end > file_len { + return Err(Error::invalid_input_source( + format!( + "Global buffer {} range {}..{} is outside file of size {}", + buffer_index, self.position, end, file_len + ) + .into(), + )); + } + Ok(self.position..end) + } +} + /// Statistics summarize some of the file metadata for quick summary info #[derive(Debug)] pub struct FileStatistics { @@ -102,8 +138,11 @@ pub struct CachedFileMetadata { pub num_global_buffer_bytes: u64, /// The number of bytes contained in the CMO and GBO tables pub num_footer_bytes: u64, + /// The major version number stored in the file footer. pub major_version: u16, + /// The minor version number stored in the file footer. pub minor_version: u16, + pub version: ConcreteFileVersion, /// The actual total file size in bytes, as reported by the object store. pub file_size_bytes: u64, /// User global buffers (index >= 1) whose bytes were already captured by the @@ -176,14 +215,14 @@ impl DeepSizeOf for CachedFileMetadata { /// hold decoded metadata for every column. #[derive(Debug, DeepSizeOf)] pub struct FileMetadataIndex { - file_schema: Arc, - num_rows: u64, - file_buffers: Vec, - column_metadata_offsets: Arc<[(u64, u64)]>, - num_columns: u32, - version: LanceFileVersion, - file_size_bytes: u64, - retained_global_buffers: BTreeMap, + pub(crate) file_schema: Arc, + pub(crate) num_rows: u64, + pub(crate) file_buffers: Vec, + pub(crate) column_metadata_offsets: Arc<[(u64, u64)]>, + pub(crate) num_columns: u32, + pub(crate) version: ConcreteFileVersion, + pub(crate) file_size_bytes: u64, + pub(crate) retained_global_buffers: BTreeMap, } impl FileMetadataIndex { @@ -226,21 +265,19 @@ impl CacheKey for ColumnMetadataCacheKey { fn type_name() -> &'static str { "ColumnMetadata" } + + fn schema() -> CacheKeySchema { + CacheKeySchema::new("lance.file.column-metadata-key", 1) + } + + fn write_key(&self, builder: &mut KeyBuilder) { + builder.write_u32(self.column_index); + } } impl CachedFileMetadata { - pub fn version(&self) -> LanceFileVersion { - match (self.major_version, self.minor_version) { - (0, 3) => LanceFileVersion::V2_0, - (2, 0) => LanceFileVersion::V2_0, - (2, 1) => LanceFileVersion::V2_1, - (2, 2) => LanceFileVersion::V2_2, - (2, 3) => LanceFileVersion::V2_3, - _ => panic!( - "Unsupported version: {}.{}", - self.major_version, self.minor_version - ), - } + pub fn version(&self) -> ConcreteFileVersion { + self.version } } @@ -313,123 +350,14 @@ pub struct ReaderProjection { } impl ReaderProjection { - fn from_field_ids_helper<'a>( - file_version: LanceFileVersion, - fields: impl Iterator, - field_id_to_column_index: &BTreeMap, - column_indices: &mut Vec, - ) -> Result<()> { - for field in fields { - let is_structural = file_version >= LanceFileVersion::V2_1; - let (contributes, recurse) = field_column_shape(field, is_structural); - // In the 2.0 system we needed ids for intermediate fields. In 2.1+ - // we only need ids for leaf fields. - if contributes - && let Some(column_idx) = field_id_to_column_index.get(&(field.id as u32)).copied() - { - column_indices.push(column_idx); - } - if recurse { - Self::from_field_ids_helper( - file_version, - field.children.iter(), - field_id_to_column_index, - column_indices, - )?; - } - } - Ok(()) - } - - /// Creates a projection using a mapping from field IDs to column indices - /// - /// You can obtain such a mapping when the file is written using the - /// [`crate::writer::FileWriter::field_id_to_column_indices`] method. - pub fn from_field_ids( - file_version: LanceFileVersion, - schema: &Schema, - field_id_to_column_index: &BTreeMap, - ) -> Result { - let mut column_indices = Vec::new(); - Self::from_field_ids_helper( - file_version, - schema.fields.iter(), - field_id_to_column_index, - &mut column_indices, - )?; - let projection = Self { - schema: Arc::new(schema.clone()), - column_indices, - }; - Ok(projection) - } - - /// Creates a projection that reads the entire file - /// - /// If the schema provided is not the schema of the entire file then - /// the projection will be invalid and the read will fail. - /// If the field is a `struct datatype` with `packed` set to true in the field metadata, - /// the whole struct has one column index. - /// To support nested `packed-struct encoding`, this method need to be further adjusted. - pub fn from_whole_schema(schema: &Schema, version: LanceFileVersion) -> Self { - let schema = Arc::new(schema.clone()); - let is_structural = version >= LanceFileVersion::V2_1; - let mut column_indices = vec![]; - let mut curr_column_idx = 0; - let mut packed_struct_fields_num = 0; - for field in schema.fields_pre_order() { - if packed_struct_fields_num > 0 { - packed_struct_fields_num -= 1; - continue; - } - if field.is_packed_struct() { - column_indices.push(curr_column_idx); - curr_column_idx += 1; - packed_struct_fields_num = field.children.len(); - } else if field.children.is_empty() || !is_structural { - column_indices.push(curr_column_idx); - curr_column_idx += 1; - } - } - Self { - schema, - column_indices, - } - } - - /// Creates a projection that reads the specified columns provided by name + /// Returns whether this projection is selective enough to benefit from + /// loading column metadata through the file's metadata index. /// - /// The syntax for column names is the same as [`lance_core::datatypes::Schema::project`] - /// - /// If the schema provided is not the schema of the entire file then - /// the projection will be invalid and the read will fail. - pub fn from_column_names( - file_version: LanceFileVersion, - schema: &Schema, - column_names: &[&str], - ) -> Result { - let field_id_to_column_index = schema - .fields_pre_order() - // In the 2.0 system we needed ids for intermediate fields. In 2.1+ - // we only need ids for leaf fields. - .filter(|field| { - file_version < LanceFileVersion::V2_1 || field.is_leaf() || field.is_packed_struct() - }) - .enumerate() - .map(|(idx, field)| (field.id as u32, idx as u32)) - .collect::>(); - let projected = schema.project(column_names)?; - let mut column_indices = Vec::new(); - Self::from_field_ids_helper( - file_version, - projected.fields.iter(), - &field_id_to_column_index, - &mut column_indices, - )?; - Ok(Self { - schema: Arc::new(projected), - column_indices, - }) + /// The caller must already have selected a file format that supports indexed + /// metadata. This method only evaluates the projection shape and selectivity. + pub fn prefers_indexed_metadata(&self, total_columns: usize) -> bool { + FileMetadataProvider::projection_matches_indexed_metadata(self) + && self.column_indices.len().saturating_mul(4) < total_columns } } @@ -442,7 +370,8 @@ pub struct FileReaderOptions { /// Default: 8MB (DEFAULT_READ_CHUNK_SIZE) pub read_chunk_size: u64, /// If set, the reader will produce batches whose total size in bytes - /// is approximately this value, overriding the row-based `batch_size`. + /// is approximately this value. The row-based `batch_size` remains an + /// independent upper bound, and the limit reached first determines the batch size. /// /// This can be set at the dataset level (via `ReadParams::file_reader_options`) /// to provide a default for all scans, or at the scanner level (via @@ -461,28 +390,52 @@ impl Default for FileReaderOptions { } #[derive(Debug, Clone)] -struct PreparedProjection { - column_infos: Vec>, - decoder_projection: ReaderProjection, +pub(crate) struct PreparedProjection { + pub column_infos: Vec>, + pub decoder_projection: ReaderProjection, } #[derive(Debug, Clone)] -enum FileMetadataProvider { +pub(crate) enum FileMetadataProvider { Full(Arc), Indexed(Arc), } +/// Executable projection behavior selected by an exact file-version module. +/// +/// The shared reader invokes this behavior but never interprets a version or +/// accepted-grammar profile. +#[async_trait] +pub(crate) trait ReadProjection: Debug + Send + Sync { + fn validate_indexed( + &self, + projection: &ReaderProjection, + metadata_index: &FileMetadataIndex, + ) -> Result<()>; + + fn read_length(&self, prepared: &PreparedProjection) -> Result; + + async fn prepare( + &self, + metadata_provider: &FileMetadataProvider, + projection: &ReaderProjection, + io: &Arc, + cache: &Arc, + ) -> Result<(PreparedProjection, u64)>; +} + #[derive(Debug, Clone)] -struct FileReadCore { - scheduler: Arc, - base_projection: ReaderProjection, - metadata_provider: FileMetadataProvider, - decoder_plugins: Arc, - cache: Arc, - options: FileReaderOptions, +pub(crate) struct DecodeEngine { + pub scheduler: Arc, + pub base_projection: ReaderProjection, + pub metadata_provider: FileMetadataProvider, + pub read_projection: Arc, + pub decoder_plugins: Arc, + pub cache: Arc, + pub options: FileReaderOptions, } -/// A projection-scoped reader for Lance files. +/// A projection-scoped reader for a current-format Lance file. /// /// This reader fixes a base projection at construction time. All later reads /// must stay within that projection, which lets the reader load only the column @@ -491,125 +444,90 @@ struct FileReadCore { /// file metadata. #[derive(Debug, Clone)] pub struct ProjectedFileReader { - core: FileReadCore, + core: DecodeEngine, } -/// A Lance file reader backed by fully decoded file metadata. +/// A current-format Lance file reader backed by fully decoded metadata. #[derive(Debug, Clone)] pub struct FileReader { - core: FileReadCore, - metadata: Arc, + pub(crate) core: DecodeEngine, + pub(crate) metadata: Arc, +} + +pub(crate) fn tasks_to_record_batch_stream( + schema: Arc, + tasks: Pin + Send>>, + batch_readahead: u32, +) -> Pin> { + let arrow_schema = Arc::new(ArrowSchema::from(schema.as_ref())); + let batches = tasks + .map(|task| task.task) + .buffered(batch_readahead as usize) + .boxed(); + Box::pin(RecordBatchStreamAdapter::new(arrow_schema, batches)) +} + +pub(crate) enum RawFileMetadataOpen { + Legacy { + major_version: u16, + minor_version: u16, + }, + Current { + version: ConcreteFileVersion, + metadata: RawFileMetadata, + }, +} + +pub(crate) struct RawFileMetadata { + pub file_schema: Arc, + pub column_metadatas: Vec, + pub num_rows: u64, + pub file_buffers: Vec, + pub num_data_bytes: u64, + pub num_column_metadata_bytes: u64, + pub num_global_buffer_bytes: u64, + pub num_footer_bytes: u64, + pub footer: Footer, + pub file_size_bytes: u64, + pub retained_global_buffers: BTreeMap, } + #[derive(Debug)] -struct Footer { +pub(crate) struct Footer { #[allow(dead_code)] - column_meta_start: u64, + pub column_meta_start: u64, // We don't use this today because we always load metadata for every column // and don't yet support "metadata projection" #[allow(dead_code)] - column_meta_offsets_start: u64, - global_buff_offsets_start: u64, - num_global_buffers: u32, - num_columns: u32, - major_version: u16, - minor_version: u16, + pub column_meta_offsets_start: u64, + pub global_buff_offsets_start: u64, + pub num_global_buffers: u32, + pub num_columns: u32, + pub major_version: u16, + pub minor_version: u16, } const FOOTER_LEN: usize = 40; -// How a field maps onto physical columns, shared by the projection-building and -// projection-validation walks so they stay in lockstep. In the 2.0 layout every -// ordinary field (including structs and lists) has its own column; in 2.1 only -// leaves do. Blob/packed-struct fields are opaque in all versions: they are a -// single column with no descent, including unloaded blob descriptor schemas. -// Returns `(contributes, recurse)`: whether the field has its own column and -// whether to walk into its children. The DFS order is the field's own column (if -// any) followed by its children, so a field's root (first) column is always the -// first entry of its sub-slice. -fn field_column_shape(field: &Field, is_structural: bool) -> (bool, bool) { +// Count the V2.1 physical columns required to reconstruct a projected field. +// This is the same DFS shape consumed by `ColumnInfoIter`: ordinary structural +// nodes are transparent and leaves contribute columns. Indexed metadata loading +// can therefore compact any ordinary structural projection into 0..N while +// preserving this order. +// +// Blob and packed-struct fields remain unsupported by indexed projection. Their +// opaque decode semantics are handled by the existing full-metadata reader. +fn indexed_projection_column_count(field: &Field) -> Option { if field.is_blob() || field.is_packed_struct() { - return (true, false); + return None; } - let contributes = !is_structural || field.children.is_empty(); - let recurse = !field.children.is_empty(); - (contributes, recurse) -} - -// Whether a field's children each cover the same rows as the field itself. Struct -// children do (one value per parent row), so they must share its length. List, -// map, and fixed-size-list items have an independent cardinality (item count, not -// row count) and are validated only against themselves. -fn children_share_parent_length(field: &Field) -> bool { - field.logical_type.is_struct() -} - -// Validate one field's slice of a projection's flat `column_indices`, returning -// the field's top-level row count (the page-row sum of its root column). Walks the -// same DFS order as `from_field_ids_helper`, advancing `cursor` past every column -// the field contributes. -// -// `comparable` tracks whether the field's row count shares the read's top-level -// cardinality. A struct's children must all match that count -- the decoders -// combine them assuming equal lengths and would otherwise panic or read past a -// shorter child -- so the equality check runs only while `comparable` holds. Once -// the walk descends through a list/map/fixed-size-list its items have an -// independent cardinality (item count, not row count), so `comparable` turns off -// for that whole subtree and a nested struct's children are no longer compared. -fn validate_field_length Result>( - field: &Field, - is_structural: bool, - comparable: bool, - column_indices: &[u32], - cursor: &mut usize, - column_len: &F, -) -> Result { - let (contributes, recurse) = field_column_shape(field, is_structural); - let mut field_rows: Option = None; - if contributes { - let column = *column_indices.get(*cursor).ok_or_else(|| { - Error::invalid_input(format!( - "projection supplied fewer column indices than its fields require \ - (ran out at field '{}')", - field.name - )) - })?; - *cursor += 1; - field_rows = Some(column_len(column as usize)?); - } - if recurse { - // Only enforce equal-length children for a struct whose own count is still - // at the top-level cardinality; below a list/map/fixed-size-list the items - // have an independent cardinality, so neither this field nor its - // descendants are comparable. - let enforce_children = comparable && children_share_parent_length(field); - for child in &field.children { - let child_rows = validate_field_length( - child, - is_structural, - enforce_children, - column_indices, - cursor, - column_len, - )?; - // A struct that contributes no column of its own (the 2.1 layout) - // takes its row count from its first child. - let expected = *field_rows.get_or_insert(child_rows); - if enforce_children && child_rows != expected { - return Err(Error::invalid_input(format!( - "cannot read field '{}': its children have differing lengths \ - (child '{}' has {} rows, but the field has {}); a struct's \ - children must all have the same length", - field.name, child.name, child_rows, expected - ))); - } - } + if field.children.is_empty() { + return Some(1); } - field_rows.ok_or_else(|| { - Error::invalid_input(format!( - "projected field '{}' maps to no columns", - field.name - )) + + field.children.iter().try_fold(0usize, |count, child| { + count.checked_add(indexed_projection_column_count(child)?) }) } @@ -618,7 +536,31 @@ fn validate_field_length Result>( // error (naming each column's length) when they differ. Ordinary files always // pass; only files written with `FileWriter::write_column` whose columns ended up // unequal can fail, and those must be read separately. -fn verify_uniform_lengths(field_lengths: &[(&str, u64)]) -> Result { +pub(crate) fn normalized_column_num_rows(info: &ColumnInfo) -> Result { + info.page_infos.iter().try_fold(0_u64, |rows, page| { + let page_rows = match &page.encoding { + PageEncoding::Structural(layout) => match &layout.layout { + Some(pbenc21::page_layout::Layout::SparseLayout(sparse)) => sparse + .structural_layers + .first() + .and_then(|layer| layer.layer.as_ref()) + .map_or(page.num_rows, |layer| match layer { + pbenc21::sparse_structural_layer::Layer::Validity(layer) => layer.num_slots, + pbenc21::sparse_structural_layer::Layer::List(layer) => layer.num_slots, + pbenc21::sparse_structural_layer::Layer::FixedSizeList(layer) => { + layer.num_slots + } + }), + _ => page.num_rows, + }, + _ => page.num_rows, + }; + rows.checked_add(page_rows) + .ok_or_else(|| Error::invalid_input_source("Column row count overflows u64".into())) + }) +} + +pub(crate) fn verify_uniform_lengths(field_lengths: &[(&str, u64)]) -> Result { let first = field_lengths.first().map_or(0, |&(_, len)| len); if field_lengths.iter().all(|&(_, len)| len == first) { return Ok(first); @@ -635,6 +577,30 @@ fn verify_uniform_lengths(field_lengths: &[(&str, u64)]) -> Result { } impl FileReader { + pub(crate) fn base_projection(&self) -> &ReaderProjection { + &self.core.base_projection + } + + pub(crate) fn full_projection(&self, projection: ReaderProjection) -> PreparedProjection { + PreparedProjection { + column_infos: self.metadata.column_infos.clone(), + decoder_projection: projection, + } + } + + pub(crate) async fn read_prepared_tasks( + &self, + params: ReadBatchParams, + batch_size: u32, + prepared: PreparedProjection, + read_len: u64, + filter: FilterExpression, + ) -> Result + Send>>> { + self.core + .read_prepared_tasks(params, batch_size, prepared, read_len, filter) + .await + } + pub fn with_scheduler(&self, scheduler: Arc) -> Self { Self { core: self.core.with_scheduler(scheduler), @@ -753,25 +719,22 @@ impl FileReader { gbo_table: &[BufferDescriptor], tail_bytes: &Bytes, tail_offset: u64, - ) -> BTreeMap { - let tail_end = tail_offset + tail_bytes.len() as u64; - gbo_table - .iter() - .enumerate() - .skip(1) - .filter_map(|(index, buffer)| { - let start = buffer.position; - let end = buffer.position + buffer.size; - if start >= tail_offset && end <= tail_end { - let rel_start = (start - tail_offset) as usize; - let rel_end = (end - tail_offset) as usize; - let bytes = Bytes::copy_from_slice(&tail_bytes[rel_start..rel_end]); - Some((index as u32, bytes)) - } else { - None - } - }) - .collect() + file_len: u64, + ) -> Result> { + let tail_end = tail_offset + .checked_add(tail_bytes.len() as u64) + .ok_or_else(|| Error::invalid_input_source("Tail byte range overflows".into()))?; + let mut retained_buffers = BTreeMap::new(); + for (index, buffer) in gbo_table.iter().enumerate().skip(1) { + let range = buffer.checked_range(index, file_len)?; + if range.start >= tail_offset && range.end <= tail_end { + let rel_start = (range.start - tail_offset) as usize; + let rel_end = (range.end - tail_offset) as usize; + let bytes = Bytes::copy_from_slice(&tail_bytes[rel_start..rel_end]); + retained_buffers.insert(index as u32, bytes); + } + } + Ok(retained_buffers) } // Checks to make sure the footer is written correctly and returns the @@ -794,14 +757,6 @@ impl FileReader { let major_version = cursor.read_u16::()?; let minor_version = cursor.read_u16::()?; - if major_version == MAJOR_VERSION as u16 && minor_version == MINOR_VERSION as u16 { - return Err(Error::version_conflict( - "Attempt to use the lance v2 reader to read a legacy file".to_string(), - major_version, - minor_version, - )); - } - let magic_bytes = footer_bytes.slice(len - 4..); if magic_bytes.as_ref() != MAGIC { return Err(Error::invalid_input(format!( @@ -820,6 +775,22 @@ impl FileReader { }) } + fn current_file_version(footer: &Footer) -> Result { + let version = + ConcreteFileVersion::from_footer_numbers(footer.major_version, footer.minor_version)?; + match version { + ConcreteFileVersion::V1 => Err(Error::version_conflict( + "Attempt to use the lance v2 reader to read a legacy file".to_string(), + footer.major_version, + footer.minor_version, + )), + ConcreteFileVersion::V2_0 + | ConcreteFileVersion::V2_1 + | ConcreteFileVersion::V2_2 + | ConcreteFileVersion::V2_3 => Ok(version), + } + } + // TODO: Once we have coalesced I/O we should only read the column metadatas that we need fn read_all_column_metadata( column_metadata_bytes: Bytes, @@ -895,7 +866,15 @@ impl FileReader { scheduler: &FileScheduler, file_len: u64, ) -> Result { - let num_bytes_needed = (file_len - start_pos) as usize; + let num_bytes_needed = file_len.checked_sub(start_pos).ok_or_else(|| { + Error::invalid_input_source( + format!( + "Tail read position {} is outside file of size {}", + start_pos, file_len + ) + .into(), + ) + })? as usize; if data.len() >= num_bytes_needed { Ok(data.slice((data.len() - num_bytes_needed)..)) } else { @@ -911,19 +890,12 @@ impl FileReader { } } - fn do_decode_gbo_table( - gbo_bytes: &Bytes, - footer: &Footer, - version: LanceFileVersion, - ) -> Result> { + fn do_decode_gbo_table(gbo_bytes: &Bytes, footer: &Footer) -> Result> { let mut global_bufs_cursor = Cursor::new(gbo_bytes); let mut global_buffers = Vec::with_capacity(footer.num_global_buffers as usize); for _ in 0..footer.num_global_buffers { let buf_pos = global_bufs_cursor.read_u64::()?; - assert!( - version < LanceFileVersion::V2_1 || buf_pos % PAGE_BUFFER_ALIGNMENT as u64 == 0 - ); let buf_size = global_bufs_cursor.read_u64::()?; global_buffers.push(BufferDescriptor { position: buf_pos, @@ -934,12 +906,24 @@ impl FileReader { Ok(global_buffers) } + fn validate_gbo_table( + gbo_table: &[BufferDescriptor], + file_len: u64, + version: ConcreteFileVersion, + ) -> Result<()> { + versions::validate_global_buffers(version, gbo_table)?; + for (buffer_index, buffer) in gbo_table.iter().enumerate() { + buffer.checked_range(buffer_index, file_len)?; + } + Ok(()) + } + async fn decode_gbo_table( tail_bytes: &Bytes, file_len: u64, scheduler: &FileScheduler, footer: &Footer, - version: LanceFileVersion, + version: ConcreteFileVersion, ) -> Result> { // This could, in theory, trigger another IOP but the GBO table should never be large // enough for that to happen @@ -950,7 +934,9 @@ impl FileReader { file_len, ) .await?; - Self::do_decode_gbo_table(&gbo_bytes, footer, version) + let gbo_table = Self::do_decode_gbo_table(&gbo_bytes, footer)?; + Self::validate_gbo_table(&gbo_table, file_len, version)?; + Ok(gbo_table) } fn decode_schema(schema_bytes: Bytes) -> Result<(u64, lance_core::datatypes::Schema)> { @@ -961,36 +947,27 @@ impl FileReader { fields: Fields(pb_schema.fields), metadata: pb_schema.metadata, }; - let schema = lance_core::datatypes::Schema::from(fields_with_meta); + let schema = Schema::try_from(fields_with_meta)?; Ok((num_rows, schema)) } - // TODO: Support late projection. Currently, if we want to perform a - // projected read of a file, we load all of the column metadata, and then - // only read the column data that is requested. This is fine for most cases. - // - // However, if there are many columns then loading all of the column metadata - // may be expensive. We should support a mode where we only load the column - // metadata for the columns that are requested (the file format supports this). - // - // The main challenge is that we either need to ignore the column metadata cache - // or have a more sophisticated cache that can cache per-column metadata. - // - // Also, if the number of columns is fairly small, it's faster to read them as a - // single IOP, but we can fix this through coalescing. - pub async fn read_all_metadata(scheduler: &FileScheduler) -> Result { - // 1. read the footer + pub(crate) async fn read_raw_metadata_for_dispatch( + scheduler: &FileScheduler, + ) -> Result { let (tail_bytes, file_len) = Self::read_tail(scheduler).await?; let tail_offset = file_len - tail_bytes.len() as u64; let footer = Self::decode_footer(&tail_bytes)?; - - let file_version = LanceFileVersion::try_from_major_minor( - footer.major_version as u32, - footer.minor_version as u32, - )?; + let version = + ConcreteFileVersion::from_footer_numbers(footer.major_version, footer.minor_version)?; + if version == ConcreteFileVersion::V1 { + return Ok(RawFileMetadataOpen::Legacy { + major_version: footer.major_version, + minor_version: footer.minor_version, + }); + } let gbo_table = - Self::decode_gbo_table(&tail_bytes, file_len, scheduler, &footer, file_version).await?; + Self::decode_gbo_table(&tail_bytes, file_len, scheduler, &footer, version).await?; if gbo_table.is_empty() { return Err(Error::internal( "File did not contain any global buffers, schema expected".to_string(), @@ -998,19 +975,20 @@ impl FileReader { } let schema_start = gbo_table[0].position; let schema_size = gbo_table[0].size; - - let num_footer_bytes = file_len - schema_start; - - // By default we read all column metadatas. We do NOT read the column metadata buffers - // at this point. We only want to read the column metadata for columns we are actually loading. + let num_footer_bytes = file_len.checked_sub(schema_start).ok_or_else(|| { + Error::invalid_input_source( + format!( + "Schema position {} is outside file of size {}", + schema_start, file_len + ) + .into(), + ) + })?; let all_metadata_bytes = Self::optimistic_tail_read(&tail_bytes, schema_start, scheduler, file_len).await?; - let schema_bytes = all_metadata_bytes.slice(0..schema_size as usize); let (num_rows, schema) = Self::decode_schema(schema_bytes)?; - // Next, read the metadata for the columns - // This is both the column metadata and the CMO table let column_metadata_start = (footer.column_meta_start - schema_start) as usize; let column_metadata_end = (footer.global_buff_offsets_start - schema_start) as usize; let column_metadata_bytes = @@ -1020,35 +998,37 @@ impl FileReader { let num_global_buffer_bytes = gbo_table.iter().map(|buf| buf.size).sum::(); let num_data_bytes = footer.column_meta_start - num_global_buffer_bytes; let num_column_metadata_bytes = footer.global_buff_offsets_start - footer.column_meta_start; - - let column_infos = Self::meta_to_col_infos(column_metadatas.as_slice(), file_version); - // The tail read above already pulled in any global buffer that lives within // the captured window. Copy those user buffers (index >= 1; the schema at 0 // is decoded above and never fetched via read_global_buffer) out of the tail // so read_global_buffer can serve them without I/O. We copy rather than slice // so the much larger tail allocation can be released once decoding is done. - let retained_global_buffers = - Self::retained_global_buffers_from_tail(&gbo_table, &tail_bytes, tail_offset); + let retained_global_buffers = Self::retained_global_buffers_from_tail( + &gbo_table, + &tail_bytes, + tail_offset, + file_len, + )?; - Ok(CachedFileMetadata { - file_schema: Arc::new(schema), - column_metadatas, - column_infos, - num_rows, - num_data_bytes, - num_column_metadata_bytes, - num_global_buffer_bytes, - num_footer_bytes, - file_buffers: gbo_table, - major_version: footer.major_version, - minor_version: footer.minor_version, - file_size_bytes: file_len, - retained_global_buffers, + Ok(RawFileMetadataOpen::Current { + version, + metadata: RawFileMetadata { + file_schema: Arc::new(schema), + column_metadatas, + num_rows, + file_buffers: gbo_table, + num_data_bytes, + num_column_metadata_bytes, + num_global_buffer_bytes, + num_footer_bytes, + footer, + file_size_bytes: file_len, + retained_global_buffers, + }, }) } - async fn read_metadata_index_with_known_schema( + async fn read_raw_metadata_index_with_known_schema( scheduler: &FileScheduler, known_schema: Option<(Arc, u64)>, ) -> Result { @@ -1056,10 +1036,7 @@ impl FileReader { let tail_offset = file_len - tail_bytes.len() as u64; let footer = Self::decode_footer(&tail_bytes)?; - let file_version = LanceFileVersion::try_from_major_minor( - footer.major_version as u32, - footer.minor_version as u32, - )?; + let file_version = Self::current_file_version(&footer)?; let gbo_table = Self::decode_gbo_table(&tail_bytes, file_len, scheduler, &footer, file_version).await?; @@ -1072,11 +1049,12 @@ impl FileReader { Some((file_schema, num_rows)) => (file_schema, num_rows), None => { let schema_buffer = &gbo_table[0]; + let schema_range = schema_buffer.checked_range(0, file_len)?; let schema_bytes = Self::read_range_from_tail_or_scheduler( &tail_bytes, tail_offset, scheduler, - schema_buffer.position..schema_buffer.position + schema_buffer.size, + schema_range, ) .await?; let (num_rows, schema) = Self::decode_schema(schema_bytes)?; @@ -1093,8 +1071,12 @@ impl FileReader { .await?; let column_metadata_offsets = Self::decode_cmo_table(cmo_table, &footer)?; - let retained_global_buffers = - Self::retained_global_buffers_from_tail(&gbo_table, &tail_bytes, tail_offset); + let retained_global_buffers = Self::retained_global_buffers_from_tail( + &gbo_table, + &tail_bytes, + tail_offset, + file_len, + )?; Ok(FileMetadataIndex { file_schema, @@ -1113,107 +1095,26 @@ impl FileReader { /// This reads the file schema from the schema global buffer. Use /// [`Self::read_metadata_index_with_schema`] when the caller already has /// the schema and row count from a higher-level metadata source. - pub async fn read_metadata_index(scheduler: &FileScheduler) -> Result { - Self::read_metadata_index_with_known_schema(scheduler, None).await + pub(crate) async fn read_raw_metadata_index( + scheduler: &FileScheduler, + ) -> Result { + Self::read_raw_metadata_index_with_known_schema(scheduler, None).await } /// Reads the metadata index without fetching the schema global buffer. /// /// Use this when the caller already has the file schema and physical row /// count from an enclosing metadata layer, such as a dataset manifest. - pub async fn read_metadata_index_with_schema( + pub(crate) async fn read_raw_metadata_index_with_schema( scheduler: &FileScheduler, file_schema: Arc, num_rows: u64, ) -> Result { - Self::read_metadata_index_with_known_schema(scheduler, Some((file_schema, num_rows))).await - } - - fn fetch_encoding(encoding: &pbfile::Encoding) -> M { - match &encoding.location { - Some(pbfile::encoding::Location::Indirect(_)) => todo!(), - Some(pbfile::encoding::Location::Direct(encoding)) => { - let encoding_buf = Bytes::from(encoding.encoding.clone()); - let encoding_any = prost_types::Any::decode(encoding_buf).unwrap(); - encoding_any.to_msg::().unwrap() - } - Some(pbfile::encoding::Location::None(_)) => panic!(), - None => panic!(), - } - } - - fn meta_to_col_infos( - column_metadatas: &[pbfile::ColumnMetadata], - file_version: LanceFileVersion, - ) -> Vec> { - column_metadatas - .iter() - .enumerate() - .map(|(col_idx, col_meta)| { - Self::meta_to_col_info(col_idx as u32, col_meta, file_version) - }) - .collect::>() - } - - fn meta_to_col_info( - col_idx: u32, - col_meta: &pbfile::ColumnMetadata, - file_version: LanceFileVersion, - ) -> Arc { - let page_infos = col_meta - .pages - .iter() - .map(|page| { - let num_rows = page.length; - let encoding = match file_version { - LanceFileVersion::V2_0 => { - PageEncoding::Legacy(Self::fetch_encoding::( - page.encoding.as_ref().unwrap(), - )) - } - _ => PageEncoding::Structural(Self::fetch_encoding::( - page.encoding.as_ref().unwrap(), - )), - }; - let buffer_offsets_and_sizes = Arc::from( - page.buffer_offsets - .iter() - .zip(page.buffer_sizes.iter()) - .map(|(offset, size)| { - // Starting with version 2.1 we can assert that page buffers are aligned - assert!( - file_version < LanceFileVersion::V2_1 - || offset % PAGE_BUFFER_ALIGNMENT as u64 == 0 - ); - (*offset, *size) - }) - .collect::>(), - ); - PageInfo { - buffer_offsets_and_sizes, - encoding, - num_rows, - priority: page.priority, - } - }) - .collect::>(); - let buffer_offsets_and_sizes = Arc::from( - col_meta - .buffer_offsets - .iter() - .zip(col_meta.buffer_sizes.iter()) - .map(|(offset, size)| (*offset, *size)) - .collect::>(), - ); - Arc::new(ColumnInfo { - index: col_idx, - page_infos: Arc::from(page_infos), - buffer_offsets_and_sizes, - encoding: Self::fetch_encoding(col_meta.encoding.as_ref().unwrap()), - }) + Self::read_raw_metadata_index_with_known_schema(scheduler, Some((file_schema, num_rows))) + .await } - fn validate_projection( + pub(crate) fn validate_projection( projection: &ReaderProjection, metadata: &CachedFileMetadata, ) -> Result<()> { @@ -1242,87 +1143,25 @@ impl FileReader { Ok(()) } - /// Opens a new file reader without any pre-existing knowledge - /// - /// This will read the file schema from the file itself and thus requires a bit more I/O - /// - /// A `base_projection` can also be provided. If provided, then the projection will apply - /// to all reads from the file that do not specify their own projection. - pub async fn try_open( - scheduler: FileScheduler, - base_projection: Option, - decoder_plugins: Arc, - cache: &LanceCache, - options: FileReaderOptions, - ) -> Result { - let file_metadata = Arc::new(Self::read_all_metadata(&scheduler).await?); - let path = scheduler.reader().path().clone(); - - // Create LanceEncodingsIo with read chunk size from options - let encodings_io = - LanceEncodingsIo::new(scheduler).with_read_chunk_size(options.read_chunk_size); - - Self::try_open_with_file_metadata( - Arc::new(encodings_io), - path, - base_projection, - decoder_plugins, - file_metadata, - cache, - options, - ) - .await - } - - /// Same as `try_open` but with the file metadata already loaded. - /// - /// This method also can accept any kind of `EncodingsIo` implementation allowing - /// for custom strategies to be used for I/O scheduling (e.g. for takes on fast - /// disks it may be better to avoid asynchronous overhead). - /// Opens a data reader backed by fully decoded file metadata. - pub async fn try_open_with_file_metadata( - scheduler: Arc, - path: Path, - base_projection: Option, - decoder_plugins: Arc, - file_metadata: Arc, - cache: &LanceCache, - options: FileReaderOptions, - ) -> Result { - let cache = Arc::new(cache.with_key_prefix(path.as_ref())); - let core = FileReadCore::try_new( - scheduler, - base_projection, - decoder_plugins, - FileMetadataProvider::Full(file_metadata.clone()), - cache, - options, - )?; - Ok(Self { - core, - metadata: file_metadata, - }) - } - - // The actual decoder needs all the column infos that make up a type. In other words, if - // the first type in the schema is Struct then the decoder will need 3 column infos. - // - // This is a file reader concern because the file reader needs to support late projection of columns - // and so it will need to figure this out anyways. - // - // It's a bit of a tricky process though because the number of column infos may depend on the - // encoding. Considering the above example, if we wrote it with a packed encoding, then there would - // only be a single column in the file (and not 3). - // - // At the moment this method words because our rules are simple and we just repeat them here. See - // Self::default_projection for a similar problem. In the future this is something the encodings - // registry will need to figure out. - fn collect_columns_from_projection( - &self, - _projection: &ReaderProjection, - ) -> Result>> { - Ok(self.metadata.column_infos.clone()) - } + // The actual decoder needs all the column infos that make up a type. In other words, if + // the first type in the schema is Struct then the decoder will need 3 column infos. + // + // This is a file reader concern because the file reader needs to support late projection of columns + // and so it will need to figure this out anyways. + // + // It's a bit of a tricky process though because the number of column infos may depend on the + // encoding. Considering the above example, if we wrote it with a packed encoding, then there would + // only be a single column in the file (and not 3). + // + // At the moment this method words because our rules are simple and we just repeat them here. See + // Self::default_projection for a similar problem. In the future this is something the encodings + // registry will need to figure out. + fn collect_columns_from_projection( + &self, + _projection: &ReaderProjection, + ) -> Result>> { + Ok(self.metadata.column_infos.clone()) + } #[allow(clippy::too_many_arguments)] async fn do_read_range( @@ -1459,92 +1298,6 @@ impl FileReader { .await } - /// Creates a stream of "read tasks" to read the data from the file - /// - /// The arguments are similar to [`Self::read_stream_projected`] but instead of returning a stream - /// of record batches it returns a stream of "read tasks". - /// - /// The tasks should be consumed with some kind of `buffered` argument if CPU parallelism is desired. - /// - /// Note that "read task" is probably a bit imprecise. The tasks are actually "decode tasks". The - /// reading happens asynchronously in the background. In other words, a single read task may map to - /// multiple I/O operations or a single I/O operation may map to multiple read tasks. - /// - /// # Why is this async? - /// - /// Constructing the read stream requires running the decode scheduler's - /// `initialize` step, which performs the metadata I/O (chunk metadata, - /// dictionaries, repetition index, ...) needed to plan the read. We - /// drive that I/O on the awaiting task rather than smuggling it into - /// the stream's first poll. This way callers control where the - /// scheduling I/O runs (typically inside a per-fragment - /// `tokio::spawn`), planning errors surface from the await instead of - /// from the first stream item, and small reads can also complete the - /// synchronous scheduling step before returning (see - /// [`DecoderConfig::inline_scheduling`]). - pub async fn read_tasks( - &self, - params: ReadBatchParams, - batch_size: u32, - projection: Option, - filter: FilterExpression, - ) -> Result + Send>>> { - self.core - .read_tasks(params, batch_size, projection, filter) - .await - } - - /// Reads data from the file as a stream of record batches - /// - /// * `params` - Specifies the range (or indices) of data to read - /// * `batch_size` - The maximum size of a single batch. A batch may be smaller - /// if it is the last batch or if it is not possible to create a batch of the - /// requested size. - /// - /// For example, if the batch size is 1024 and one of the columns is a string - /// column then there may be some ranges of 1024 rows that contain more than - /// 2^31 bytes of string data (which is the maximum size of a string column - /// in Arrow). In this case smaller batches may be emitted. - /// * `batch_readahead` - The number of batches to read ahead. This controls the - /// amount of CPU parallelism of the read. In other words it controls how many - /// batches will be decoded in parallel. It has no effect on the I/O parallelism - /// of the read (how many I/O requests are in flight at once). - /// - /// This parameter also is also related to backpressure. If the consumer of the - /// stream is slow then the reader will build up RAM. - /// * `projection` - A projection to apply to the read. This controls which columns - /// are read from the file. The projection is NOT applied on top of the base - /// projection. The projection is applied directly to the file schema. - /// - /// # Why is this async? - /// - /// This delegates to [`Self::read_tasks`], which awaits the decode - /// scheduler's `initialize` step (and, for small reads, the synchronous - /// scheduling that follows) before returning. See `read_tasks` for - /// details on why this work is performed up front rather than on the - /// stream's first poll. - pub async fn read_stream_projected( - &self, - params: ReadBatchParams, - batch_size: u32, - batch_readahead: u32, - projection: ReaderProjection, - filter: FilterExpression, - ) -> Result>> { - let arrow_schema = Arc::new(ArrowSchema::from(projection.schema.as_ref())); - let tasks_stream = self - .read_tasks(params, batch_size, Some(projection), filter) - .await?; - let batch_stream = tasks_stream - .map(|task| task.task) - .buffered(batch_readahead as usize) - .boxed(); - Ok(Box::pin(RecordBatchStreamAdapter::new( - arrow_schema, - batch_stream, - ))) - } - fn take_rows_blocking( &self, indices: Vec, @@ -1663,36 +1416,15 @@ impl FileReader { ) } - /// Read data from the file as an iterator of record batches - /// - /// This is a blocking variant of [`Self::read_stream_projected`] that runs entirely in the - /// calling thread. It will block on I/O if the decode is faster than the I/O. It is useful - /// for benchmarking and potentially from "take"ing small batches from fast disks. - /// - /// Large scans of in-memory data will still benefit from threading (and should therefore not - /// use this method) because we can parallelize the decode. - /// - /// Note: calling this from within a tokio runtime will panic. It is acceptable to call this - /// from a spawn_blocking context. - pub fn read_stream_projected_blocking( + pub(crate) fn read_prepared_blocking( &self, params: ReadBatchParams, batch_size: u32, - projection: Option, + prepared: PreparedProjection, + read_len: u64, filter: FilterExpression, ) -> Result> { - let projection = projection.unwrap_or_else(|| self.core.base_projection.clone()); - Self::validate_projection(&projection, &self.metadata)?; - // Apply the same projection-length validation as the async path. This - // reader is always backed by full metadata, so we can build the prepared - // projection synchronously (no column-metadata I/O) and reuse the shared - // check. `read_len` is the projection's common column length, which - // `RangeFull`/`RangeFrom` resolve against rather than `num_rows`. - let prepared = PreparedProjection { - column_infos: self.metadata.column_infos.clone(), - decoder_projection: projection.clone(), - }; - let read_len = self.core.prepared_read_length(&prepared)?; + let projection = prepared.decoder_projection; let verify_bound = |params: &ReadBatchParams, bound: u64, inclusive: bool| { if bound > read_len || (bound == read_len && inclusive) { Err(Error::invalid_input(format!( @@ -1704,17 +1436,13 @@ impl FileReader { }; match ¶ms { ReadBatchParams::Indices(indices) => { - for idx in indices { - match idx { - None => { - return Err(Error::invalid_input("Null value in indices array")); - } - Some(idx) => { - verify_bound(¶ms, idx as u64, true)?; - } + for index in indices { + match index { + None => return Err(Error::invalid_input("Null value in indices array")), + Some(index) => verify_bound(¶ms, index as u64, true)?, } } - let indices = indices.iter().map(|idx| idx.unwrap() as u64).collect(); + let indices = indices.iter().map(|index| index.unwrap() as u64).collect(); self.take_rows_blocking(indices, batch_size, projection, filter) } ReadBatchParams::Range(range) => { @@ -1753,76 +1481,55 @@ impl FileReader { } } - /// Reads data from the file as a stream of record batches - /// - /// This is similar to [`Self::read_stream_projected`] but uses the base projection - /// provided when the file was opened (or reads all columns if the file was - /// opened without a base projection) - /// - /// # Why is this async? - /// - /// This delegates to [`Self::read_stream_projected`], which awaits the - /// decode scheduler's `initialize` step before returning the stream. - /// See [`Self::read_tasks`] for the rationale. - pub async fn read_stream( - &self, - params: ReadBatchParams, - batch_size: u32, - batch_readahead: u32, - filter: FilterExpression, - ) -> Result>> { - self.read_stream_projected( - params, - batch_size, - batch_readahead, - self.core.base_projection.clone(), - filter, - ) - .await - } - pub fn schema(&self) -> &Arc { self.core.schema() } } impl FileMetadataProvider { - fn version(&self) -> LanceFileVersion { + pub(crate) fn version(&self) -> ConcreteFileVersion { match self { - Self::Full(metadata) => metadata.version(), + Self::Full(metadata) => metadata.version, Self::Indexed(metadata_index) => metadata_index.version, } } - fn num_rows(&self) -> u64 { + pub(crate) fn num_rows(&self) -> u64 { match self { Self::Full(metadata) => metadata.num_rows, Self::Indexed(metadata_index) => metadata_index.num_rows, } } - fn schema(&self) -> &Arc { + pub(crate) fn schema(&self) -> &Arc { match self { Self::Full(metadata) => &metadata.file_schema, Self::Indexed(metadata_index) => &metadata_index.file_schema, } } - fn file_buffers(&self) -> &Vec { + pub(crate) fn file_buffers(&self) -> &Vec { match self { Self::Full(metadata) => &metadata.file_buffers, Self::Indexed(metadata_index) => &metadata_index.file_buffers, } } - fn retained_global_buffers(&self) -> &BTreeMap { + pub(crate) fn retained_global_buffers(&self) -> &BTreeMap { match self { Self::Full(metadata) => &metadata.retained_global_buffers, Self::Indexed(metadata_index) => &metadata_index.retained_global_buffers, } } - fn file_statistics(&self) -> Option { + fn file_size(&self) -> u64 { + match self { + Self::Full(metadata) => metadata.file_size_bytes, + Self::Indexed(metadata_index) => metadata_index.file_size_bytes, + } + } + + pub(crate) fn file_statistics(&self) -> Option { let metadata = match self { Self::Full(metadata) => metadata, Self::Indexed(_) => return None, @@ -1832,19 +1539,22 @@ impl FileMetadataProvider { )) } - fn supports_indexed_projection( - projection: &ReaderProjection, - version: LanceFileVersion, - ) -> bool { - version >= LanceFileVersion::V2_1 - && !projection.schema.fields.is_empty() - && projection.schema.fields.len() == projection.column_indices.len() - && projection.schema.fields.iter().all(|field| { - field.children.is_empty() && !field.is_blob() && !field.is_packed_struct() + pub(crate) fn projection_matches_indexed_metadata(projection: &ReaderProjection) -> bool { + if projection.schema.fields.is_empty() { + return false; + } + + projection + .schema + .fields + .iter() + .try_fold(0usize, |count, field| { + count.checked_add(indexed_projection_column_count(field)?) }) + == Some(projection.column_indices.len()) } - fn validate_indexed_projection( + pub(crate) fn validate_indexed_projection_structure( projection: &ReaderProjection, metadata_index: &FileMetadataIndex, ) -> Result<()> { @@ -1869,24 +1579,19 @@ impl FileMetadataProvider { ))); } } - if !Self::supports_indexed_projection(projection, metadata_index.version) { - return Err(Error::not_supported(format!( - "lazy column metadata loading only supports direct V2.1+ top-level physical column projections; got file version {:?}, {} schema fields, and {} column indices", - metadata_index.version, - projection.schema.fields.len(), - projection.column_indices.len() - ))); - } Ok(()) } - fn validate_projection(&self, projection: &ReaderProjection) -> Result<()> { - match self { - Self::Full(metadata) => FileReader::validate_projection(projection, metadata), - Self::Indexed(metadata_index) => { - Self::validate_indexed_projection(projection, metadata_index) - } - } + pub(crate) fn indexed_projection_error( + projection: &ReaderProjection, + metadata_index: &FileMetadataIndex, + ) -> Error { + Error::not_supported(format!( + "lazy column metadata loading requires a V2.1+ ordinary structural projection without blob or packed-struct fields whose physical-column count matches the projection; got file version {:?}, {} schema fields, and {} column indices", + metadata_index.version, + projection.schema.fields.len(), + projection.column_indices.len() + )) } fn column_metadata_range( @@ -1912,12 +1617,16 @@ impl FileMetadataProvider { Ok(position..end) } - async fn load_indexed_column_infos( + pub(crate) async fn load_indexed_column_infos( metadata_index: &FileMetadataIndex, io: &Arc, cache: &Arc, column_indices: &[u32], - ) -> Result>> { + decode_column: F, + ) -> Result>> + where + F: Fn(u32, &pbfile::ColumnMetadata) -> Result>, + { let mut column_infos = vec![None; column_indices.len()]; let mut missing_columns = Vec::new(); @@ -1939,14 +1648,10 @@ impl FileMetadataProvider { .collect::>(); let metadata_bytes = io.submit_request(ranges, 0).await?; for ((result_index, column_index, _), bytes) in - missing_columns.into_iter().zip(metadata_bytes.into_iter()) + missing_columns.into_iter().zip(metadata_bytes) { let column_metadata = pbfile::ColumnMetadata::decode(bytes)?; - let column_info = FileReader::meta_to_col_info( - column_index, - &column_metadata, - metadata_index.version, - ); + let column_info = decode_column(column_index, &column_metadata)?; let cached = Arc::new(CachedColumnMetadata { column_metadata, column_info: column_info.clone(), @@ -1970,83 +1675,41 @@ impl FileMetadataProvider { }) .collect() } - - async fn prepare_projection( - &self, - projection: &ReaderProjection, - io: &Arc, - cache: &Arc, - ) -> Result { - self.validate_projection(projection)?; - match self { - Self::Full(metadata) => Ok(PreparedProjection { - column_infos: metadata.column_infos.clone(), - decoder_projection: projection.clone(), - }), - Self::Indexed(metadata_index) => { - let column_infos = Self::load_indexed_column_infos( - metadata_index, - io, - cache, - &projection.column_indices, - ) - .await?; - let decoder_projection = ReaderProjection { - schema: projection.schema.clone(), - column_indices: (0..projection.column_indices.len()) - .map(|idx| idx as u32) - .collect(), - }; - Ok(PreparedProjection { - column_infos, - decoder_projection, - }) - } - } - } } -impl FileReadCore { - fn try_new( +impl DecodeEngine { + pub(crate) fn try_new( scheduler: Arc, - base_projection: Option, + base_projection: ReaderProjection, decoder_plugins: Arc, metadata_provider: FileMetadataProvider, + read_projection: Arc, cache: Arc, options: FileReaderOptions, ) -> Result { - if let Some(base_projection) = base_projection.as_ref() { - metadata_provider.validate_projection(base_projection)?; - } - let base_projection = base_projection.unwrap_or(ReaderProjection::from_whole_schema( - metadata_provider.schema().as_ref(), - metadata_provider.version(), - )); Ok(Self { scheduler, base_projection, metadata_provider, + read_projection, decoder_plugins, cache, options, }) } - fn with_scheduler(&self, scheduler: Arc) -> Self { + pub(crate) fn with_scheduler(&self, scheduler: Arc) -> Self { Self { scheduler, base_projection: self.base_projection.clone(), metadata_provider: self.metadata_provider.clone(), + read_projection: self.read_projection.clone(), decoder_plugins: self.decoder_plugins.clone(), cache: self.cache.clone(), options: self.options.clone(), } } - fn version(&self) -> LanceFileVersion { - self.metadata_provider.version() - } - fn num_rows(&self) -> u64 { self.metadata_provider.num_rows() } @@ -2055,7 +1718,7 @@ impl FileReadCore { self.metadata_provider.schema() } - async fn read_global_buffer(&self, index: u32) -> Result { + pub(crate) async fn read_global_buffer(&self, index: u32) -> Result { let file_buffers = self.metadata_provider.file_buffers(); let buffer_desc = file_buffers.get(index as usize).ok_or_else(|| { Error::invalid_input(format!( @@ -2072,7 +1735,10 @@ impl FileReadCore { let bytes = self .scheduler .submit_request( - vec![buffer_desc.position..buffer_desc.position + buffer_desc.size], + vec![ + buffer_desc + .checked_range(index as usize, self.metadata_provider.file_size())?, + ], 0, ) .await?; @@ -2084,57 +1750,6 @@ impl FileReadCore { }) } - // The common length to read across a prepared projection, after validating - // its columns can be combined into rectangular batches. Each top-level field - // is checked for internal consistency (see `validate_field_length`); the - // top-level fields must then share a length, since one read combines them. - // Ordinary files always pass (every column has `num_rows` rows); files - // written with `FileWriter::write_column` whose columns ended up unequal are - // rejected here and must be read separately. - // - // `column_infos` and `decoder_projection.column_indices` line up for both - // metadata providers: the full provider keeps absolute indices into the whole - // file, while the indexed (lazy) provider loads only the projected columns and - // renumbers them 0..N -- in either case `column_infos[column_index]` is the - // requested column. - fn prepared_read_length(&self, prepared: &PreparedProjection) -> Result { - let is_structural = self.version() >= LanceFileVersion::V2_1; - let column_infos = &prepared.column_infos; - let column_len = |column: usize| -> Result { - let info = column_infos.get(column).ok_or_else(|| { - Error::invalid_input(format!( - "projection references column index {} but only {} columns are available", - column, - column_infos.len() - )) - })?; - Ok(info.page_infos.iter().map(|page| page.num_rows).sum()) - }; - let column_indices = &prepared.decoder_projection.column_indices; - let fields = &prepared.decoder_projection.schema.fields; - let mut cursor = 0usize; - let mut field_lengths = Vec::with_capacity(fields.len()); - for field in fields { - let rows = validate_field_length( - field, - is_structural, - true, - column_indices, - &mut cursor, - &column_len, - )?; - field_lengths.push((field.name.as_str(), rows)); - } - if cursor != column_indices.len() { - return Err(Error::invalid_input(format!( - "projection supplied {} column indices but its fields require {}", - column_indices.len(), - cursor - ))); - } - verify_uniform_lengths(&field_lengths) - } - async fn read_range( &self, range: Range, @@ -2201,25 +1816,14 @@ impl FileReadCore { .await } - async fn read_tasks( + pub(crate) async fn read_prepared_tasks( &self, params: ReadBatchParams, batch_size: u32, - projection: Option, + prepared: PreparedProjection, + read_len: u64, filter: FilterExpression, ) -> Result + Send>>> { - let projection = projection.unwrap_or_else(|| self.base_projection.clone()); - let prepared = self - .metadata_provider - .prepare_projection(&projection, &self.scheduler, &self.cache) - .await?; - // All projected columns must share a length: the reader combines them - // into rectangular batches. Ordinary files satisfy this (every column - // has `num_rows` rows); files written with `FileWriter::write_column` - // may not, and such columns must be read separately. `read_len` is that - // common length, which `RangeFull`/`RangeFrom` resolve against (rather - // than `num_rows`, the file's longest column). - let read_len = self.prepared_read_length(&prepared)?; let verify_bound = |params: &ReadBatchParams, bound: u64, inclusive: bool| { if bound > read_len || (bound == read_len && inclusive) { Err(Error::invalid_input(format!( @@ -2282,93 +1886,21 @@ impl FileReadCore { } impl ProjectedFileReader { - /// Opens a data reader backed by indexed column metadata. - /// - /// `base_projection` must be a supported indexed projection. Reads that do - /// not pass an explicit projection use this base projection. - pub async fn try_open( - scheduler: FileScheduler, - base_projection: Option, - decoder_plugins: Arc, - cache: &LanceCache, - options: FileReaderOptions, - ) -> Result { - let base_projection = Self::require_indexed_base_projection(base_projection)?; - let metadata_index = Arc::new(FileReader::read_metadata_index(&scheduler).await?); - let path = scheduler.reader().path().clone(); - let encodings_io = - LanceEncodingsIo::new(scheduler).with_read_chunk_size(options.read_chunk_size); - Self::try_open_with_metadata_index( - Arc::new(encodings_io), - path, - Some(base_projection), - decoder_plugins, - metadata_index, - cache, - options, - ) - .await - } - - /// Opens a data reader from a previously loaded metadata index. - /// - /// `base_projection` must be a supported indexed projection. Use - /// [`Self::try_open_with_file_metadata`] when the default read should cover - /// the whole file schema. - pub async fn try_open_with_metadata_index( - scheduler: Arc, - path: Path, - base_projection: Option, - decoder_plugins: Arc, - metadata_index: Arc, - cache: &LanceCache, - options: FileReaderOptions, - ) -> Result { - let base_projection = Self::require_indexed_base_projection(base_projection)?; - let cache = Arc::new(cache.with_key_prefix(path.as_ref())); - let core = FileReadCore::try_new( - scheduler, - Some(base_projection), - decoder_plugins, - FileMetadataProvider::Indexed(metadata_index), - cache, - options, - )?; - Ok(Self { core }) - } - - fn require_indexed_base_projection( - base_projection: Option, - ) -> Result { - base_projection.ok_or_else(|| { - Error::invalid_input("ProjectedFileReader requires an explicit base projection") - }) - } - - pub async fn try_open_with_file_metadata( - scheduler: Arc, - path: Path, - base_projection: Option, - decoder_plugins: Arc, - file_metadata: Arc, - cache: &LanceCache, - options: FileReaderOptions, - ) -> Result { - let cache = Arc::new(cache.with_key_prefix(path.as_ref())); - let core = FileReadCore::try_new( - scheduler, - base_projection, - decoder_plugins, - FileMetadataProvider::Full(file_metadata), - cache, - options, - )?; - Ok(Self { core }) + pub(crate) fn base_projection(&self) -> &ReaderProjection { + &self.core.base_projection } - /// Returns whether a projection can be served by indexed column metadata. - pub fn supports_projection(projection: &ReaderProjection, version: LanceFileVersion) -> bool { - FileMetadataProvider::supports_indexed_projection(projection, version) + pub(crate) async fn read_prepared_tasks( + &self, + params: ReadBatchParams, + batch_size: u32, + prepared: PreparedProjection, + read_len: u64, + filter: FilterExpression, + ) -> Result + Send>>> { + self.core + .read_prepared_tasks(params, batch_size, prepared, read_len, filter) + .await } /// Returns a clone of this reader using a different scheduler. @@ -2378,11 +1910,6 @@ impl ProjectedFileReader { } } - /// Returns the Lance file version. - pub fn version(&self) -> LanceFileVersion { - self.core.version() - } - /// Returns the number of rows in the file. pub fn num_rows(&self) -> u64 { self.core.num_rows() @@ -2399,7 +1926,7 @@ impl ProjectedFileReader { } #[cfg(test)] - fn metadata_index(&self) -> Option<&Arc> { + pub(crate) fn metadata_index(&self) -> Option<&Arc> { match &self.core.metadata_provider { FileMetadataProvider::Indexed(metadata_index) => Some(metadata_index), FileMetadataProvider::Full(_) => None, @@ -2410,17 +1937,348 @@ impl ProjectedFileReader { pub async fn read_global_buffer(&self, index: u32) -> Result { self.core.read_global_buffer(index).await } +} - /// Creates a stream of read tasks for the requested rows and projection. - pub async fn read_tasks( - &self, - params: ReadBatchParams, - batch_size: u32, - projection: Option, - filter: FilterExpression, - ) -> Result + Send>>> { - self.core - .read_tasks(params, batch_size, projection, filter) +impl FileReader { + #[cfg(test)] + fn scheduler(&self) -> Arc { + self.core.scheduler.clone() + } + + pub async fn try_open( + scheduler: FileScheduler, + base_projection: Option, + decoder_plugins: Arc, + cache: &LanceCache, + options: FileReaderOptions, + ) -> Result { + match Self::try_open_for_dispatch( + scheduler, + base_projection, + decoder_plugins, + cache, + options, + ) + .await? + { + versions::OpenedFileReader::V1 { + major_version, + minor_version, + } => Err(Error::version_conflict( + "Attempt to use the Lance current-format reader to read a v1 file".to_string(), + major_version, + minor_version, + )), + versions::OpenedFileReader::Current(reader) => Ok(reader), + } + } + + pub(crate) async fn try_open_for_dispatch( + scheduler: FileScheduler, + base_projection: Option, + decoder_plugins: Arc, + cache: &LanceCache, + options: FileReaderOptions, + ) -> Result { + let metadata = match Self::read_raw_metadata_for_dispatch(&scheduler).await? { + RawFileMetadataOpen::Legacy { + major_version, + minor_version, + } => { + return Ok(versions::OpenedFileReader::V1 { + major_version, + minor_version, + }); + } + RawFileMetadataOpen::Current { version, metadata } => { + Arc::new(versions::finish_metadata(version, metadata)?) + } + }; + let path = scheduler.reader().path().clone(); + let io = Arc::new( + LanceEncodingsIo::new(scheduler).with_read_chunk_size(options.read_chunk_size), + ); + Self::try_open_with_file_metadata( + io, + path, + base_projection, + decoder_plugins, + metadata, + cache, + options, + ) + .await + .map(versions::OpenedFileReader::Current) + } + + pub async fn try_open_with_file_metadata( + scheduler: Arc, + path: Path, + base_projection: Option, + decoder_plugins: Arc, + metadata: Arc, + cache: &LanceCache, + options: FileReaderOptions, + ) -> Result { + if metadata.version == ConcreteFileVersion::V1 { + return Err(Error::version_conflict( + "Attempt to use the Lance current-format reader with v1 metadata".to_string(), + metadata.major_version, + metadata.minor_version, + )); + } + let read_projection = versions::read_projection(metadata.version)?; + let has_explicit_projection = base_projection.is_some(); + let base_projection = base_projection.unwrap_or_else(|| { + versions::reader_projection_from_whole_schema(&metadata.file_schema, metadata.version) + }); + if has_explicit_projection { + Self::validate_projection(&base_projection, &metadata)?; + } + let cache = Arc::new(cache.with_key_prefix(path.as_ref())); + let core = DecodeEngine::try_new( + scheduler, + base_projection, + decoder_plugins, + FileMetadataProvider::Full(metadata.clone()), + read_projection, + cache, + options, + )?; + Ok(Self { core, metadata }) + } + + pub async fn read_all_metadata(scheduler: &FileScheduler) -> Result { + match Self::read_raw_metadata_for_dispatch(scheduler).await? { + RawFileMetadataOpen::Legacy { + major_version, + minor_version, + } => Err(Error::version_conflict( + "Attempt to use the Lance current-format reader to read v1 metadata".to_string(), + major_version, + minor_version, + )), + RawFileMetadataOpen::Current { version, metadata } => { + versions::finish_metadata(version, metadata) + } + } + } + + pub async fn read_metadata_index(scheduler: &FileScheduler) -> Result { + let index = Self::read_raw_metadata_index(scheduler).await?; + versions::finish_metadata_index(index) + } + + pub async fn read_metadata_index_with_schema( + scheduler: &FileScheduler, + file_schema: Arc, + num_rows: u64, + ) -> Result { + let index = + Self::read_raw_metadata_index_with_schema(scheduler, file_schema, num_rows).await?; + versions::finish_metadata_index(index) + } + + pub fn version(&self) -> ConcreteFileVersion { + self.metadata.version + } + + async fn prepare(&self, projection: ReaderProjection) -> Result<(PreparedProjection, u64)> { + self.core + .read_projection + .prepare( + &self.core.metadata_provider, + &projection, + &self.core.scheduler, + &self.core.cache, + ) + .await + } + + pub async fn read_tasks( + &self, + params: ReadBatchParams, + batch_size: u32, + projection: Option, + filter: FilterExpression, + ) -> Result + Send>>> { + let projection = projection.unwrap_or_else(|| self.base_projection().clone()); + let (prepared, read_len) = self.prepare(projection).await?; + self.read_prepared_tasks(params, batch_size, prepared, read_len, filter) + .await + } + + pub async fn read_stream_projected( + &self, + params: ReadBatchParams, + batch_size: u32, + batch_readahead: u32, + projection: ReaderProjection, + filter: FilterExpression, + ) -> Result>> { + let schema = projection.schema.clone(); + let tasks = self + .read_tasks(params, batch_size, Some(projection), filter) + .await?; + Ok(tasks_to_record_batch_stream(schema, tasks, batch_readahead)) + } + + pub fn read_stream_projected_blocking( + &self, + params: ReadBatchParams, + batch_size: u32, + projection: Option, + filter: FilterExpression, + ) -> Result> { + let projection = projection.unwrap_or_else(|| self.base_projection().clone()); + Self::validate_projection(&projection, self.metadata())?; + let prepared = self.full_projection(projection); + let read_len = self.core.read_projection.read_length(&prepared)?; + self.read_prepared_blocking(params, batch_size, prepared, read_len, filter) + } + + pub async fn read_stream( + &self, + params: ReadBatchParams, + batch_size: u32, + batch_readahead: u32, + filter: FilterExpression, + ) -> Result>> { + self.read_stream_projected( + params, + batch_size, + batch_readahead, + self.base_projection().clone(), + filter, + ) + .await + } +} + +impl ProjectedFileReader { + pub async fn try_open( + scheduler: FileScheduler, + base_projection: Option, + decoder_plugins: Arc, + cache: &LanceCache, + options: FileReaderOptions, + ) -> Result { + let base_projection = base_projection.ok_or_else(|| { + Error::invalid_input("ProjectedReader requires an explicit base projection") + })?; + let metadata_index = Arc::new(FileReader::read_metadata_index(&scheduler).await?); + let path = scheduler.reader().path().clone(); + let io = Arc::new( + LanceEncodingsIo::new(scheduler).with_read_chunk_size(options.read_chunk_size), + ); + Self::try_open_with_metadata_index( + io, + path, + Some(base_projection), + decoder_plugins, + metadata_index, + cache, + options, + ) + .await + } + + pub async fn try_open_with_metadata_index( + scheduler: Arc, + path: Path, + base_projection: Option, + decoder_plugins: Arc, + metadata_index: Arc, + cache: &LanceCache, + options: FileReaderOptions, + ) -> Result { + if metadata_index.version == ConcreteFileVersion::V1 { + return Err(Error::version_conflict( + "Attempt to use the Lance projected current-format reader with v1 metadata" + .to_string(), + 0, + 2, + )); + } + let base_projection = base_projection.ok_or_else(|| { + Error::invalid_input("ProjectedReader requires an explicit base projection") + })?; + let read_projection = versions::read_projection(metadata_index.version)?; + read_projection.validate_indexed(&base_projection, &metadata_index)?; + let cache = Arc::new(cache.with_key_prefix(path.as_ref())); + let core = DecodeEngine::try_new( + scheduler, + base_projection, + decoder_plugins, + FileMetadataProvider::Indexed(metadata_index), + read_projection, + cache, + options, + )?; + Ok(Self { core }) + } + + pub async fn try_open_with_file_metadata( + scheduler: Arc, + path: Path, + base_projection: Option, + decoder_plugins: Arc, + metadata: Arc, + cache: &LanceCache, + options: FileReaderOptions, + ) -> Result { + if metadata.version == ConcreteFileVersion::V1 { + return Err(Error::version_conflict( + "Attempt to use the Lance projected current-format reader with v1 metadata" + .to_string(), + metadata.major_version, + metadata.minor_version, + )); + } + let read_projection = versions::read_projection(metadata.version)?; + let has_explicit_projection = base_projection.is_some(); + let base_projection = base_projection.unwrap_or_else(|| { + versions::reader_projection_from_whole_schema(&metadata.file_schema, metadata.version) + }); + if has_explicit_projection { + FileReader::validate_projection(&base_projection, &metadata)?; + } + let cache = Arc::new(cache.with_key_prefix(path.as_ref())); + let core = DecodeEngine::try_new( + scheduler, + base_projection, + decoder_plugins, + FileMetadataProvider::Full(metadata), + read_projection, + cache, + options, + )?; + Ok(Self { core }) + } + + pub fn version(&self) -> ConcreteFileVersion { + self.core.metadata_provider.version() + } + + pub async fn read_tasks( + &self, + params: ReadBatchParams, + batch_size: u32, + projection: Option, + filter: FilterExpression, + ) -> Result + Send>>> { + let projection = projection.unwrap_or_else(|| self.base_projection().clone()); + let (prepared, read_len) = self + .core + .read_projection + .prepare( + &self.core.metadata_provider, + &projection, + &self.core.scheduler, + &self.core.cache, + ) + .await?; + self.read_prepared_tasks(params, batch_size, prepared, read_len, filter) .await } } @@ -2475,11 +2333,7 @@ pub fn describe_encoding(page: &pbfile::column_metadata::Page) -> String { } pub trait EncodedBatchReaderExt { - fn try_from_mini_lance( - bytes: Bytes, - schema: &Schema, - version: LanceFileVersion, - ) -> Result + fn try_from_mini_lance(bytes: Bytes, schema: &Schema) -> Result where Self: Sized; fn try_from_self_described_lance(bytes: Bytes) -> Result @@ -2488,16 +2342,13 @@ pub trait EncodedBatchReaderExt { } impl EncodedBatchReaderExt for EncodedBatch { - fn try_from_mini_lance( - bytes: Bytes, - schema: &Schema, - file_version: LanceFileVersion, - ) -> Result + fn try_from_mini_lance(bytes: Bytes, schema: &Schema) -> Result where Self: Sized, { - let projection = ReaderProjection::from_whole_schema(schema, file_version); let footer = FileReader::decode_footer(&bytes)?; + let file_version = FileReader::current_file_version(&footer)?; + let projection = versions::reader_projection_from_whole_schema(schema, file_version); // Next, read the metadata for the columns // This is both the column metadata and the CMO table @@ -2507,12 +2358,7 @@ impl EncodedBatchReaderExt for EncodedBatch { let column_metadatas = FileReader::read_all_column_metadata(column_metadata_bytes, &footer)?; - let file_version = LanceFileVersion::try_from_major_minor( - footer.major_version as u32, - footer.minor_version as u32, - )?; - - let page_table = FileReader::meta_to_col_infos(&column_metadatas, file_version); + let page_table = versions::decode_column_metadata(file_version, &column_metadatas)?; Ok(Self { data: bytes, @@ -2531,27 +2377,26 @@ impl EncodedBatchReaderExt for EncodedBatch { Self: Sized, { let footer = FileReader::decode_footer(&bytes)?; - let file_version = LanceFileVersion::try_from_major_minor( - footer.major_version as u32, - footer.minor_version as u32, - )?; + let file_version = FileReader::current_file_version(&footer)?; + let file_len = bytes.len() as u64; let gbo_table = FileReader::do_decode_gbo_table( &bytes.slice(footer.global_buff_offsets_start as usize..), &footer, - file_version, )?; + FileReader::validate_gbo_table(&gbo_table, file_len, file_version)?; if gbo_table.is_empty() { return Err(Error::internal( "File did not contain any global buffers, schema expected".to_string(), )); } - let schema_start = gbo_table[0].position as usize; - let schema_size = gbo_table[0].size as usize; + let schema_range = gbo_table[0].checked_range(0, file_len)?; + let schema_start = schema_range.start as usize; + let schema_end = schema_range.end as usize; - let schema_bytes = bytes.slice(schema_start..(schema_start + schema_size)); + let schema_bytes = bytes.slice(schema_start..schema_end); let (_, schema) = FileReader::decode_schema(schema_bytes)?; - let projection = ReaderProjection::from_whole_schema(&schema, file_version); + let projection = versions::reader_projection_from_whole_schema(&schema, file_version); // Next, read the metadata for the columns // This is both the column metadata and the CMO table @@ -2561,7 +2406,7 @@ impl EncodedBatchReaderExt for EncodedBatch { let column_metadatas = FileReader::read_all_column_metadata(column_metadata_bytes, &footer)?; - let page_table = FileReader::meta_to_col_infos(&column_metadatas, file_version); + let page_table = versions::decode_column_metadata(file_version, &column_metadatas)?; Ok(Self { data: bytes, @@ -2585,21 +2430,25 @@ mod tests { }; use arrow_array::{ - RecordBatch, UInt32Array, - types::{Float64Type, Int32Type}, + DictionaryArray, Int8Array, Int32Array, ListArray, RecordBatch, RecordBatchIterator, + StringArray, UInt32Array, + types::{Float64Type, Int8Type, Int32Type}, }; + use arrow_buffer::{NullBuffer, OffsetBuffer, ScalarBuffer}; use arrow_schema::{DataType, Field, Fields, Schema as ArrowSchema}; use bytes::Bytes; use futures::{StreamExt, prelude::stream::TryStreamExt}; use lance_arrow::{BLOB_META_KEY, RecordBatchExt}; use lance_core::{ArrowResult, datatypes::Schema}; - use lance_datagen::{BatchCount, ByteCount, RowCount, array, gen_batch}; + use lance_datagen::{ArrayGeneratorExt, BatchCount, ByteCount, RowCount, array, gen_batch}; use lance_encoding::{ + constants::{STRUCTURAL_ENCODING_META_KEY, STRUCTURAL_ENCODING_SPARSE}, decoder::{ - DecodeBatchScheduler, DecoderPlugins, FilterExpression, ReadBatchTask, decode_batch, + DecodeBatchScheduler, DecoderPlugins, EncodedBatchLayout, FilterExpression, + PageEncoding, ReadBatchTask, decode_batch, }, - encoder::{EncodedBatch, EncodingOptions, default_encoding_strategy, encode_batch}, - version::LanceFileVersion, + encoder::{EncodedBatch, EncodingOptions, encode_batch}, + format::pb21, }; use lance_io::{stream::RecordBatchStream, utils::CachedFileSize}; use log::debug; @@ -2607,14 +2456,205 @@ mod tests { use tokio::sync::mpsc; use crate::reader::{ - EncodedBatchReaderExt, FileReader, FileReaderOptions, ProjectedFileReader, - ReaderProjection, validate_field_length, verify_uniform_lengths, + EncodedBatchReaderExt, FileReader, FileReaderOptions, ProjectedFileReader, ReaderProjection, }; use crate::testing::{FsFixture, WrittenFile, test_cache, write_lance_file}; - use crate::writer::{EncodedBatchWriteExt, FileWriter, FileWriterOptions}; + use crate::version::{ConcreteFileVersion, LanceFileVersion}; + use crate::versions; + use crate::writer::{FileWriterOptions, PAGE_BUFFER_ALIGNMENT}; use lance_encoding::decoder::DecoderConfig; - async fn create_some_file(fs: &FsFixture, version: LanceFileVersion) -> WrittenFile { + fn footer_version(bytes: &[u8]) -> (u16, u16) { + let version_start = bytes.len() - 8; + ( + u16::from_le_bytes([bytes[version_start], bytes[version_start + 1]]), + u16::from_le_bytes([bytes[version_start + 2], bytes[version_start + 3]]), + ) + } + + #[tokio::test] + async fn sparse_file_writer_reader_scan_range_and_take_roundtrip() { + let fs = FsFixture::default(); + let sparse_metadata = HashMap::from([( + STRUCTURAL_ENCODING_META_KEY.to_string(), + STRUCTURAL_ENCODING_SPARSE.to_string(), + )]); + let value_field = + Field::new("values", DataType::Int32, true).with_metadata(sparse_metadata.clone()); + let item_field = Arc::new(Field::new("item", DataType::Int32, true)); + let list_field = Field::new("items", DataType::List(item_field.clone()), true) + .with_metadata(sparse_metadata); + let arrow_schema = Arc::new(ArrowSchema::new(vec![value_field, list_field])); + let list = ListArray::try_new( + item_field, + OffsetBuffer::new(ScalarBuffer::from(vec![0_i32, 2, 2, 2, 3, 3, 5])), + Arc::new(Int32Array::from(vec![ + Some(1), + None, + Some(3), + Some(4), + Some(5), + ])), + Some(NullBuffer::from(vec![true, false, true, true, true, true])), + ) + .unwrap(); + let batch = RecordBatch::try_new( + arrow_schema.clone(), + vec![ + Arc::new(Int32Array::from(vec![ + Some(10), + None, + Some(30), + Some(40), + None, + Some(60), + ])), + Arc::new(list), + ], + ) + .unwrap(); + let input = RecordBatchIterator::new(vec![Ok(batch.clone())], arrow_schema); + write_lance_file( + input, + &fs, + ConcreteFileVersion::V2_3, + FileWriterOptions::default(), + ) + .await; + + let file_scheduler = fs + .scheduler + .open_file(&fs.tmp_path, &CachedFileSize::unknown()) + .await + .unwrap(); + let file_reader = FileReader::try_open( + file_scheduler, + None, + Arc::::default(), + &test_cache(), + FileReaderOptions::default(), + ) + .await + .unwrap(); + assert_eq!(file_reader.metadata().column_infos.len(), 2); + assert!( + file_reader + .metadata() + .column_infos + .iter() + .flat_map(|column| column.page_infos.iter()) + .all(|page| { + matches!( + &page.encoding, + PageEncoding::Structural(layout) + if matches!( + layout.layout, + Some(pb21::page_layout::Layout::SparseLayout(_)) + ) + ) + }) + ); + + let scan = file_reader + .read_stream( + lance_io::ReadBatchParams::RangeFull, + 1024, + 1, + FilterExpression::no_filter(), + ) + .await + .unwrap() + .try_collect::>() + .await + .unwrap(); + assert_eq!(scan, vec![batch.clone()]); + + let range = file_reader + .read_stream( + lance_io::ReadBatchParams::Range(1..5), + 1024, + 1, + FilterExpression::no_filter(), + ) + .await + .unwrap() + .try_collect::>() + .await + .unwrap(); + assert_eq!(range, vec![batch.slice(1, 4)]); + + let indices = UInt32Array::from(vec![0, 3, 5]); + let take = file_reader + .read_stream( + lance_io::ReadBatchParams::Indices(indices.clone()), + 1024, + 1, + FilterExpression::no_filter(), + ) + .await + .unwrap() + .try_collect::>() + .await + .unwrap(); + assert_eq!(take, vec![batch.take(&indices).unwrap()]); + } + + #[tokio::test] + async fn full_int8_dictionary_v2_2_roundtrip() { + let fs = FsFixture::default(); + let values = Arc::new(StringArray::from( + (0..=i8::MAX) + .map(|value| format!("value-{value}")) + .collect::>(), + )); + let keys = Int8Array::from((0..=i8::MAX).collect::>()); + let dictionary = Arc::new(DictionaryArray::::new(keys, values)); + let arrow_schema = Arc::new(ArrowSchema::new(vec![Field::new( + "dictionary", + DataType::Dictionary(Box::new(DataType::Int8), Box::new(DataType::Utf8)), + true, + )])); + let batch = RecordBatch::try_new(arrow_schema.clone(), vec![dictionary]).unwrap(); + + write_lance_file( + RecordBatchIterator::new([Ok(batch.clone())], arrow_schema), + &fs, + ConcreteFileVersion::V2_2, + FileWriterOptions::default(), + ) + .await; + + let file_scheduler = fs + .scheduler + .open_file(&fs.tmp_path, &CachedFileSize::unknown()) + .await + .unwrap(); + let file_reader = FileReader::try_open( + file_scheduler, + None, + Arc::::default(), + &test_cache(), + FileReaderOptions::default(), + ) + .await + .unwrap(); + let actual = file_reader + .read_stream( + lance_io::ReadBatchParams::RangeFull, + 1024, + 1, + FilterExpression::no_filter(), + ) + .await + .unwrap() + .try_collect::>() + .await + .unwrap(); + + assert_eq!(actual, vec![batch]); + } + + async fn create_some_file(fs: &FsFixture, version: ConcreteFileVersion) -> WrittenFile { let location_type = DataType::Struct(Fields::from(vec![ Field::new("x", DataType::Float64, true), Field::new("y", DataType::Float64, true), @@ -2626,36 +2666,76 @@ mod tests { .col("location", array::rand_type(&location_type)) .col("categories", array::rand_type(&categories_type)) .col("binary", array::rand_type(&DataType::Binary)); - if version <= LanceFileVersion::V2_0 { + if version == ConcreteFileVersion::V2_0 { reader = reader.col("large_bin", array::rand_type(&DataType::LargeBinary)); } let reader = reader.into_reader_rows(RowCount::from(1000), BatchCount::from(100)); + write_lance_file(reader, fs, version, FileWriterOptions::default()).await + } + + async fn create_wide_direct_file(fs: &FsFixture, num_columns: usize) -> WrittenFile { + let mut reader = gen_batch(); + for column_idx in 0..num_columns { + reader = reader.col(format!("c{column_idx}"), array::step::()); + } + let reader = reader.into_reader_rows(RowCount::from(1000), BatchCount::from(100)); + write_lance_file( reader, fs, - FileWriterOptions { - format_version: Some(version), - ..Default::default() - }, + ConcreteFileVersion::V2_1, + FileWriterOptions::default(), ) .await } - async fn create_wide_direct_file(fs: &FsFixture, num_columns: usize) -> WrittenFile { + async fn create_wide_fixed_size_list_file(fs: &FsFixture, num_columns: usize) -> WrittenFile { + let data_type = + DataType::FixedSizeList(Arc::new(Field::new("item", DataType::Float32, true)), 4); let mut reader = gen_batch(); for column_idx in 0..num_columns { - reader = reader.col(format!("c{column_idx}"), array::step::()); + reader = reader.col( + format!("c{column_idx}"), + array::rand_type(&data_type).with_random_nulls(0.1), + ); } - let reader = reader.into_reader_rows(RowCount::from(1000), BatchCount::from(100)); + let reader = reader.into_reader_rows(RowCount::from(64), BatchCount::from(4)); write_lance_file( reader, fs, - FileWriterOptions { - format_version: Some(LanceFileVersion::V2_1), - ..Default::default() - }, + ConcreteFileVersion::V2_1, + FileWriterOptions::default(), + ) + .await + } + + async fn create_wide_structural_file(fs: &FsFixture, num_groups: usize) -> WrittenFile { + let struct_type = DataType::Struct(Fields::from(vec![ + Field::new("x", DataType::Int32, true), + Field::new("y", DataType::Int32, true), + ])); + let list_type = DataType::List(Arc::new(Field::new("item", DataType::Int32, true))); + let mut reader = gen_batch(); + for group_idx in 0..num_groups { + reader = reader + .col( + format!("s{group_idx}"), + array::rand_type(&struct_type).with_random_nulls(0.5), + ) + .col( + format!("l{group_idx}"), + array::rand_type(&list_type).with_random_nulls(0.5), + ); + } + let reader = reader.into_reader_rows(RowCount::from(64), BatchCount::from(4)); + + write_lance_file( + reader, + fs, + ConcreteFileVersion::V2_1, + FileWriterOptions::default(), ) .await } @@ -2712,23 +2792,333 @@ mod tests { assert_eq!(remaining, 0); } - async fn collect_read_tasks( - tasks: Pin + Send>>, - readahead: usize, - ) -> Vec { - tasks - .map(|task| task.task) - .buffered(readahead) - .try_collect::>() - .await - .unwrap() + async fn collect_read_tasks( + tasks: Pin + Send>>, + readahead: usize, + ) -> Vec { + tasks + .map(|task| task.task) + .buffered(readahead) + .try_collect::>() + .await + .unwrap() + } + + /// Writes `batch` to a fresh file, overwrites `patch` bytes at `patch_offset` + /// into the single occurrence of `pattern`, and reads the file back with the + /// default reader configuration. + async fn read_file_with_mutated_bytes( + version: LanceFileVersion, + batch: RecordBatch, + pattern: &[u8], + patch_offset: usize, + patch: &[u8], + ) -> lance_core::Result> { + let fs = FsFixture::default(); + let schema = batch.schema(); + write_lance_file( + RecordBatchIterator::new(vec![Ok(batch)], schema), + &fs, + version.resolve(), + FileWriterOptions::default(), + ) + .await; + + let mut bytes = fs + .object_store + .read_one_all(&fs.tmp_path) + .await + .unwrap() + .to_vec(); + let matches = bytes + .windows(pattern.len()) + .enumerate() + .filter_map(|(position, window)| (window == pattern).then_some(position)) + .collect::>(); + assert_eq!( + matches.len(), + 1, + "expected the byte pattern to appear exactly once in the file" + ); + let patch_start = matches[0] + patch_offset; + bytes[patch_start..patch_start + patch.len()].copy_from_slice(patch); + fs.object_store.put(&fs.tmp_path, &bytes).await.unwrap(); + + let file_scheduler = fs + .scheduler + .open_file(&fs.tmp_path, &CachedFileSize::unknown()) + .await + .unwrap(); + let file_reader = FileReader::try_open( + file_scheduler, + None, + Arc::::default(), + &test_cache(), + FileReaderOptions::default(), + ) + .await + .unwrap(); + file_reader + .read_stream( + lance_io::ReadBatchParams::RangeFull, + 1024, + 16, + FilterExpression::no_filter(), + ) + .await? + .try_collect::>() + .await + } + + #[tokio::test] + async fn test_reader_rejects_excess_miniblock_row_counts() { + let batch = + arrow_array::record_batch!(("id", UInt64, (0..2048_u64).collect::>())).unwrap(); + let fs = FsFixture::default(); + write_lance_file( + RecordBatchIterator::new(vec![Ok(batch.clone())], batch.schema()), + &fs, + ConcreteFileVersion::V2_1, + FileWriterOptions::default(), + ) + .await; + + let mut bytes = fs + .object_store + .read_one_all(&fs.tmp_path) + .await + .unwrap() + .to_vec(); + // V2.1 places this column's first mini-block metadata word at byte zero. + // The issue's mutation makes its non-final item count exceed the page total. + bytes[0] ^= 0xf7; + fs.object_store.put(&fs.tmp_path, &bytes).await.unwrap(); + + let file_scheduler = fs + .scheduler + .open_file(&fs.tmp_path, &CachedFileSize::unknown()) + .await + .unwrap(); + let file_reader = FileReader::try_open( + file_scheduler, + None, + Arc::::default(), + &test_cache(), + FileReaderOptions::default(), + ) + .await + .unwrap(); + let result = file_reader + .read_stream( + lance_io::ReadBatchParams::RangeFull, + 1024, + 16, + FilterExpression::no_filter(), + ) + .await; + let error = match result { + Ok(stream) => stream + .try_collect::>() + .await + .expect_err("excess mini-block row counts must fail the read"), + Err(error) => error, + }; + assert!( + matches!(error, lance_core::Error::CorruptFile { .. }), + "expected CorruptFile, got: {error}" + ); + assert!( + error.to_string().contains("exceeding items_in_page"), + "unexpected message: {error}" + ); + } + + /// A corrupt file whose variable-width offsets point outside the value bytes + /// must fail with a typed error under the default reader configuration + /// (`validate_on_decode` disabled) instead of materializing values outside + /// the data buffer. + /// + /// Uses a dictionary-encoded string column because its values page stores + /// the offsets verbatim, so flipping the tail offset in the file reaches the + /// Arrow conversion boundary without being rejected by an intermediate + /// decompressor. + #[rstest] + #[tokio::test] + async fn test_default_reader_rejects_out_of_bounds_variable_width_offsets( + #[values(LanceFileVersion::V2_1, LanceFileVersion::V2_2, LanceFileVersion::V2_3)] + version: LanceFileVersion, + ) { + use arrow_array::{Array, DictionaryArray, Int32Array, StringArray}; + + let values = StringArray::from(vec!["alpha", "beta", "gamma"]); + let indices = Int32Array::from((0..300).map(|i| i % 3).collect::>()); + let dictionary = DictionaryArray::new(indices, Arc::new(values)); + let arrow_schema = Arc::new(ArrowSchema::new(vec![Field::new( + "category", + dictionary.data_type().clone(), + false, + )])); + let batch = RecordBatch::try_new(arrow_schema, vec![Arc::new(dictionary)]).unwrap(); + + // The dictionary values page stores the value offsets as plain + // little-endian i32s ending with [5, 9, 14] (2.1 also stores the leading + // zero, 2.2+ omits it). If a future encoding change stops storing these + // offsets verbatim this lookup fails loudly and the test needs a new + // byte pattern. The patch rewrites the tail offset so it points far + // beyond the value bytes. + let offsets_tail_pattern = [5_i32, 9, 14] + .iter() + .flat_map(|value| value.to_le_bytes()) + .collect::>(); + let error = read_file_with_mutated_bytes( + version, + batch, + &offsets_tail_pattern, + 8, + &100_000_i32.to_le_bytes(), + ) + .await + .expect_err("out-of-bounds offsets must fail the read"); + assert!( + matches!(error, lance_core::Error::CorruptFile { .. }), + "expected CorruptFile, got: {error}" + ); + assert!( + error.to_string().contains("out of bounds"), + "unexpected message: {error}" + ); + } + + /// Storage dictionaries expand their values through `DataBlockBuilder` + /// before the final Arrow layout validation. Corrupt dictionary offsets + /// must therefore fail at the append boundary instead of reaching a slice + /// operation with a decreasing range. + #[rstest] + #[tokio::test] + async fn test_default_reader_rejects_non_monotonic_storage_dictionary_offsets( + #[values(LanceFileVersion::V2_1, LanceFileVersion::V2_2, LanceFileVersion::V2_3)] + version: LanceFileVersion, + ) { + use arrow_array::StringArray; + + let metadata = HashMap::from([ + ( + "lance-encoding:dict-size-ratio".to_string(), + "0.99".to_string(), + ), + ( + "lance-encoding:dict-values-compression".to_string(), + "none".to_string(), + ), + ]); + let arrow_schema = Arc::new(ArrowSchema::new(vec![ + Field::new("category", DataType::Utf8, false).with_metadata(metadata), + ])); + let values = (0..300) + .map(|index| match index % 3 { + 0 => "alpha", + 1 => "beta", + _ => "gamma", + }) + .collect::>(); + let batch = + RecordBatch::try_new(arrow_schema, vec![Arc::new(StringArray::from(values))]).unwrap(); + + let offsets_tail_pattern = [5_i32, 9, 14] + .iter() + .flat_map(|value| value.to_le_bytes()) + .collect::>(); + let error = read_file_with_mutated_bytes( + version, + batch, + &offsets_tail_pattern, + 4, + &2_i32.to_le_bytes(), + ) + .await + .expect_err("non-monotonic dictionary offsets must fail the read"); + assert!( + matches!(error, lance_core::Error::CorruptFile { .. }), + "expected CorruptFile, got: {error}" + ); + assert!( + error.to_string().contains("decreases"), + "unexpected message: {error}" + ); + } + + /// Same contract as the test above, but for a plain (non-dictionary) string + /// column: the mini-block chunk stores chunk-relative value offsets that are + /// used to slice the chunk, so a corrupt tail offset must surface as a typed + /// error from the chunk decompressor instead of a panic in the decode task. + #[rstest] + #[tokio::test] + async fn test_default_reader_rejects_out_of_bounds_miniblock_offsets( + #[values(LanceFileVersion::V2_1, LanceFileVersion::V2_2, LanceFileVersion::V2_3)] + version: LanceFileVersion, + ) { + use arrow_array::StringArray; + + let arrow_schema = Arc::new(ArrowSchema::new(vec![Field::new( + "strings", + DataType::Utf8, + false, + )])); + let batch = RecordBatch::try_new( + arrow_schema, + vec![Arc::new(StringArray::from(vec!["alpha", "beta", "gamma"]))], + ) + .unwrap(); + + // For ["alpha", "beta", "gamma"] the chunk stores LE i32 offsets + // [16, 21, 25, 30] (chunk-relative: a 16-byte offsets region precedes + // the value bytes). The patch rewrites the tail offset to point far + // past the chunk. + let chunk_offsets_pattern = [16_i32, 21, 25, 30] + .iter() + .flat_map(|value| value.to_le_bytes()) + .collect::>(); + let error = read_file_with_mutated_bytes( + version, + batch, + &chunk_offsets_pattern, + 12, + &100_000_i32.to_le_bytes(), + ) + .await + .expect_err("an out-of-bounds chunk offset must fail the read"); + assert!( + matches!(error, lance_core::Error::CorruptFile { .. }), + "expected CorruptFile, got: {error}" + ); + assert!( + error.to_string().contains("out of bounds"), + "unexpected message: {error}" + ); } #[tokio::test] async fn test_round_trip() { let fs = FsFixture::default(); - let WrittenFile { data, .. } = create_some_file(&fs, LanceFileVersion::V2_0).await; + let WrittenFile { data, .. } = create_some_file(&fs, ConcreteFileVersion::V2_0).await; + + let file_size = fs.object_store.size(&fs.tmp_path).await.unwrap() as usize; + let footer = fs + .object_store + .open(&fs.tmp_path) + .await + .unwrap() + .get_range(file_size - 8..file_size) + .await + .unwrap(); + assert_eq!(footer_version(&footer), (0, 3)); + assert_eq!( + crate::determine_file_version(&fs.object_store, &fs.tmp_path, Some(file_size)) + .await + .unwrap(), + ConcreteFileVersion::V2_0 + ); for read_size in [32, 1024, 1024 * 1024] { let file_scheduler = fs @@ -2746,6 +3136,13 @@ mod tests { .await .unwrap(); + assert_eq!( + ( + file_reader.metadata().major_version, + file_reader.metadata().minor_version + ), + (0, 3) + ); let schema = file_reader.schema(); assert_eq!(schema.metadata.get("foo").unwrap(), "bar"); @@ -2767,7 +3164,7 @@ mod tests { #[test_log::test(tokio::test)] async fn test_encoded_batch_round_trip( // TODO: Add V2_1 (currently fails) - #[values(LanceFileVersion::V2_0)] version: LanceFileVersion, + #[values(ConcreteFileVersion::V2_0)] version: ConcreteFileVersion, ) { let data = gen_batch() .col("x", array::rand::()) @@ -2782,10 +3179,9 @@ mod tests { max_page_bytes: 32 * 1024 * 1024, keep_original_array: true, buffer_alignment: 64, - version, }; - let encoding_strategy = default_encoding_strategy(version); + let encoding_strategy = crate::versions::v2_0::encoding_strategy(); let encoded_batch = encode_batch( &data, @@ -2797,7 +3193,8 @@ mod tests { .unwrap(); // Test self described - let bytes = encoded_batch.try_to_self_described_lance(version).unwrap(); + let bytes = versions::encode_self_described_batch(version, &encoded_batch).unwrap(); + assert_eq!(footer_version(&bytes), (2, 0)); let decoded_batch = EncodedBatch::try_from_self_described_lance(bytes).unwrap(); @@ -2806,7 +3203,7 @@ mod tests { &FilterExpression::no_filter(), Arc::::default(), false, - version, + EncodedBatchLayout::Array, None, ) .await @@ -2815,16 +3212,16 @@ mod tests { assert_eq!(data, decoded); // Test mini - let bytes = encoded_batch.try_to_mini_lance(version).unwrap(); + let bytes = versions::encode_mini_batch(version, &encoded_batch).unwrap(); + assert_eq!(footer_version(&bytes), (2, 0)); let decoded_batch = - EncodedBatch::try_from_mini_lance(bytes, lance_schema.as_ref(), LanceFileVersion::V2_0) - .unwrap(); + EncodedBatch::try_from_mini_lance(bytes, lance_schema.as_ref()).unwrap(); let decoded = decode_batch( &decoded_batch, &FilterExpression::no_filter(), Arc::::default(), false, - version, + EncodedBatchLayout::Array, None, ) .await @@ -2836,8 +3233,12 @@ mod tests { #[rstest] #[test_log::test(tokio::test)] async fn test_projection( - #[values(LanceFileVersion::V2_0, LanceFileVersion::V2_1, LanceFileVersion::V2_2)] - version: LanceFileVersion, + #[values( + ConcreteFileVersion::V2_0, + ConcreteFileVersion::V2_1, + ConcreteFileVersion::V2_2 + )] + version: ConcreteFileVersion, ) { let fs = FsFixture::default(); @@ -2884,15 +3285,15 @@ mod tests { let projected_schema = written_file.schema.project(&columns).unwrap(); let projection = if use_field_ids { - ReaderProjection::from_field_ids( - file_reader.metadata.version(), + versions::reader_projection_from_field_ids( + file_reader.metadata().version(), &projected_schema, &field_id_mapping, ) .unwrap() } else { - ReaderProjection::from_column_names( - file_reader.metadata.version(), + versions::reader_projection_from_column_names( + file_reader.metadata().version(), &written_file.schema, &columns, ) @@ -3014,8 +3415,8 @@ mod tests { .open_file(&fs.tmp_path, &CachedFileSize::unknown()) .await .unwrap(); - let projection = ReaderProjection::from_column_names( - LanceFileVersion::V2_1, + let projection = versions::reader_projection_from_column_names( + ConcreteFileVersion::V2_1, &written_file.schema, &["c10"], ) @@ -3073,8 +3474,8 @@ mod tests { let fs = FsFixture::default(); let written_file = create_wide_direct_file(&fs, 512).await; - let projection = ReaderProjection::from_column_names( - LanceFileVersion::V2_1, + let projection = versions::reader_projection_from_column_names( + ConcreteFileVersion::V2_1, &written_file.schema, &["c0"], ) @@ -3157,6 +3558,192 @@ mod tests { ); } + async fn assert_lazy_projection_matches_eager_and_reads_metadata_subset( + fs: &FsFixture, + projection: ReaderProjection, + shape: &str, + ) -> Vec { + let file_scheduler = fs + .scheduler + .open_file(&fs.tmp_path, &CachedFileSize::unknown()) + .await + .unwrap(); + let eager_reader = FileReader::try_open( + file_scheduler.clone(), + None, + Arc::::default(), + &test_cache(), + FileReaderOptions::default(), + ) + .await + .unwrap(); + let expected = eager_reader + .read_stream_projected( + lance_io::ReadBatchParams::RangeFull, + 127, + 16, + projection.clone(), + FilterExpression::no_filter(), + ) + .await + .unwrap() + .try_collect::>() + .await + .unwrap(); + + let cache = test_cache(); + let lazy_reader = ProjectedFileReader::try_open( + file_scheduler, + Some(projection.clone()), + Arc::::default(), + &cache, + FileReaderOptions::default(), + ) + .await + .unwrap(); + let metadata_index = lazy_reader.metadata_index().unwrap(); + let requested_metadata_bytes = projection + .column_indices + .iter() + .map(|column_index| metadata_index.column_metadata_offsets[*column_index as usize].1) + .sum::(); + let total_metadata_bytes = metadata_index + .column_metadata_offsets + .iter() + .map(|(_, length)| *length) + .sum::(); + assert!(total_metadata_bytes > requested_metadata_bytes * 8); + + fs.object_store.io_stats_incremental(); + let tasks = lazy_reader + .read_tasks( + lance_io::ReadBatchParams::Range(0..0), + 127, + None, + FilterExpression::no_filter(), + ) + .await + .unwrap(); + assert!(collect_read_tasks(tasks, 1).await.is_empty()); + let metadata_stats = fs.object_store.io_stats_incremental(); + assert!( + metadata_stats.read_bytes < total_metadata_bytes / 2, + "lazy {shape} read fetched too much metadata: read {} bytes, requested column metadata is {} bytes, total column metadata is {} bytes", + metadata_stats.read_bytes, + requested_metadata_bytes, + total_metadata_bytes + ); + + let tasks = lazy_reader + .read_tasks( + lance_io::ReadBatchParams::RangeFull, + 127, + None, + FilterExpression::no_filter(), + ) + .await + .unwrap(); + let actual = collect_read_tasks(tasks, 16).await; + assert_eq!(expected, actual); + actual + } + + #[tokio::test] + async fn test_lazy_reader_fixed_size_list_projection_matches_eager_reader() { + let fs = FsFixture::default(); + let written_file = create_wide_fixed_size_list_file(&fs, 512).await; + let projection = versions::reader_projection_from_column_names( + ConcreteFileVersion::V2_1, + &written_file.schema, + &["c17", "c509"], + ) + .unwrap(); + assert!(projection.prefers_indexed_metadata(512)); + assert_lazy_projection_matches_eager_and_reads_metadata_subset( + &fs, + projection, + "fixed-size-list", + ) + .await; + } + + #[tokio::test] + async fn test_v2_0_rejects_indexed_metadata_reader() { + let fs = FsFixture::default(); + let written_file = create_some_file(&fs, ConcreteFileVersion::V2_0).await; + let projection = versions::reader_projection_from_column_names( + ConcreteFileVersion::V2_0, + &written_file.schema, + &["score"], + ) + .unwrap(); + assert!(projection.prefers_indexed_metadata(100)); + + let file_scheduler = fs + .scheduler + .open_file(&fs.tmp_path, &CachedFileSize::unknown()) + .await + .unwrap(); + let err = ProjectedFileReader::try_open( + file_scheduler, + Some(projection), + Arc::::default(), + &test_cache(), + FileReaderOptions::default(), + ) + .await + .unwrap_err(); + assert!( + matches!(err, lance_core::Error::NotSupported { .. }), + "expected V2.0 indexed metadata open to fail, got {err:?}" + ); + } + + #[tokio::test] + async fn test_lazy_reader_nested_projection_compacts_physical_columns() { + let fs = FsFixture::default(); + let written_file = create_wide_structural_file(&fs, 128).await; + let projection = versions::reader_projection_from_column_names( + ConcreteFileVersion::V2_1, + &written_file.schema, + &["s97.y", "l4", "s3"], + ) + .unwrap(); + + assert_eq!( + projection + .schema + .fields + .iter() + .map(|field| field.name.as_str()) + .collect::>(), + vec!["s97", "l4", "s3"] + ); + assert_eq!(projection.schema.fields[0].children.len(), 1); + assert_eq!(projection.schema.fields[0].children[0].name, "y"); + assert_eq!(projection.schema.fields[2].children.len(), 2); + assert_eq!(projection.column_indices.len(), 4); + assert!( + projection + .column_indices + .windows(2) + .any(|indices| indices[0] > indices[1]), + "the projection must reorder physical columns to exercise compact remapping" + ); + assert!(projection.prefers_indexed_metadata(128 * 4)); + let actual = assert_lazy_projection_matches_eager_and_reads_metadata_subset( + &fs, projection, "nested", + ) + .await; + assert!( + actual + .iter() + .flat_map(|batch| batch.columns()) + .any(|column| column.null_count() > 0), + "the structural projection must exercise nullable arrays" + ); + } + #[rstest] #[case::before_metadata_region(90, 5)] #[case::after_metadata_region(190, 20)] @@ -3185,21 +3772,22 @@ mod tests { ); } + #[rstest] + #[case::blob(BLOB_META_KEY)] + #[case::packed_struct("lance-encoding:packed")] #[tokio::test] - async fn test_lazy_reader_rejects_unsupported_projection() { + async fn test_lazy_reader_rejects_opaque_projection(#[case] metadata_key: &str) { let fs = FsFixture::default(); - let written_file = create_some_file(&fs, LanceFileVersion::V2_1).await; + let written_file = create_some_file(&fs, ConcreteFileVersion::V2_1).await; - let projection = ReaderProjection::from_column_names( - LanceFileVersion::V2_1, + let ordinary_projection = versions::reader_projection_from_column_names( + ConcreteFileVersion::V2_1, &written_file.schema, - &["location"], + &["location.x"], ) .unwrap(); - assert!(!ProjectedFileReader::supports_projection( - &projection, - LanceFileVersion::V2_1 - )); + assert_eq!(ordinary_projection.schema.fields[0].children.len(), 1); + assert!(ordinary_projection.prefers_indexed_metadata(100)); let file_scheduler = fs .scheduler @@ -3220,6 +3808,12 @@ mod tests { "expected InvalidInput, got {err:?}" ); + let mut projection = ordinary_projection; + Arc::make_mut(&mut projection.schema).fields[0] + .metadata + .insert(metadata_key.to_string(), "true".to_string()); + assert!(!projection.prefers_indexed_metadata(100)); + let err = ProjectedFileReader::try_open( file_scheduler, Some(projection), @@ -3231,11 +3825,11 @@ mod tests { .unwrap_err(); assert!( matches!(err, lance_core::Error::NotSupported { .. }), - "expected NotSupported, got {err:?}" + "expected NotSupported for {metadata_key}, got {err:?}" ); } - // The projection-length validation lives in `FileReadCore`, shared by the + // The projection-length validation lives in `DecodeEngine`, shared by the // eager and the lazy (indexed) metadata providers. The indexed provider loads // only the projected columns and renumbers them 0..N, so this checks that the // renumbered `column_infos`/`column_indices` still line up for the length @@ -3253,14 +3847,10 @@ mod tests { let lance_schema = Schema::try_from(arrow_schema.as_ref()).unwrap(); let fs = FsFixture::default(); - let options = FileWriterOptions { - format_version: Some(LanceFileVersion::V2_1), - ..Default::default() - }; - let mut writer = FileWriter::try_new( + let mut writer = versions::v2_1::create_writer( fs.object_store.create(&fs.tmp_path).await.unwrap(), lance_schema.clone(), - options, + FileWriterOptions::default(), ) .unwrap(); // "a" has 5 rows, "c" has 1 -- an unequal-length file. @@ -3281,9 +3871,12 @@ mod tests { .unwrap(); let cache = test_cache(); let open_indexed = |names: &[&str]| { - let projection = - ReaderProjection::from_column_names(LanceFileVersion::V2_1, &lance_schema, names) - .unwrap(); + let projection = versions::reader_projection_from_column_names( + ConcreteFileVersion::V2_1, + &lance_schema, + names, + ) + .unwrap(); ProjectedFileReader::try_open( file_scheduler.clone(), Some(projection), @@ -3346,7 +3939,7 @@ mod tests { async fn test_compressing_buffer() { let fs = FsFixture::default(); - let written_file = create_some_file(&fs, LanceFileVersion::V2_0).await; + let written_file = create_some_file(&fs, ConcreteFileVersion::V2_0).await; let file_scheduler = fs .scheduler .open_file(&fs.tmp_path, &CachedFileSize::unknown()) @@ -3401,7 +3994,7 @@ mod tests { #[tokio::test] async fn test_read_all() { let fs = FsFixture::default(); - let WrittenFile { data, .. } = create_some_file(&fs, LanceFileVersion::V2_0).await; + let WrittenFile { data, .. } = create_some_file(&fs, ConcreteFileVersion::V2_0).await; let total_rows = data.iter().map(|batch| batch.num_rows()).sum::(); let file_scheduler = fs @@ -3438,8 +4031,12 @@ mod tests { #[rstest] #[tokio::test] async fn test_blocking_take( - #[values(LanceFileVersion::V2_0, LanceFileVersion::V2_1, LanceFileVersion::V2_2)] - version: LanceFileVersion, + #[values( + ConcreteFileVersion::V2_0, + ConcreteFileVersion::V2_1, + ConcreteFileVersion::V2_2 + )] + version: ConcreteFileVersion, ) { let fs = FsFixture::default(); let WrittenFile { data, schema, .. } = create_some_file(&fs, version).await; @@ -3452,7 +4049,10 @@ mod tests { .unwrap(); let file_reader = FileReader::try_open( file_scheduler.clone(), - Some(ReaderProjection::from_column_names(version, &schema, &["score"]).unwrap()), + Some( + versions::reader_projection_from_column_names(version, &schema, &["score"]) + .unwrap(), + ), Arc::::default(), &test_cache(), FileReaderOptions::default(), @@ -3483,7 +4083,7 @@ mod tests { #[tokio::test(flavor = "multi_thread")] async fn test_drop_in_progress() { let fs = FsFixture::default(); - let WrittenFile { data, .. } = create_some_file(&fs, LanceFileVersion::V2_0).await; + let WrittenFile { data, .. } = create_some_file(&fs, ConcreteFileVersion::V2_0).await; let total_rows = data.iter().map(|batch| batch.num_rows()).sum::(); let file_scheduler = fs @@ -3532,7 +4132,7 @@ mod tests { // if the stream was dropped before it finished. let fs = FsFixture::default(); - let written_file = create_some_file(&fs, LanceFileVersion::V2_0).await; + let written_file = create_some_file(&fs, ConcreteFileVersion::V2_0).await; let total_rows = written_file .data .iter() @@ -3554,11 +4154,11 @@ mod tests { .await .unwrap(); - let projection = - ReaderProjection::from_whole_schema(&written_file.schema, LanceFileVersion::V2_0); - let column_infos = file_reader - .collect_columns_from_projection(&projection) - .unwrap(); + let projection = versions::reader_projection_from_whole_schema( + &written_file.schema, + ConcreteFileVersion::V2_0, + ); + let column_infos = file_reader.metadata().column_infos.clone(); let mut decode_scheduler = DecodeBatchScheduler::try_new( &projection.schema, &projection.column_indices, @@ -3566,7 +4166,7 @@ mod tests { &vec![], total_rows as u64, Arc::::default(), - file_reader.core.scheduler.clone(), + file_reader.scheduler(), test_cache(), &FilterExpression::no_filter(), &DecoderConfig::default(), @@ -3586,14 +4186,14 @@ mod tests { range, &FilterExpression::no_filter(), tx, - file_reader.core.scheduler.clone(), + file_reader.scheduler(), ) } #[tokio::test] async fn test_read_empty_range() { let fs = FsFixture::default(); - create_some_file(&fs, LanceFileVersion::V2_0).await; + create_some_file(&fs, ConcreteFileVersion::V2_0).await; let file_scheduler = fs .scheduler @@ -3651,7 +4251,7 @@ mod tests { )])) .unwrap(); - let mut file_writer = FileWriter::try_new( + let mut file_writer = versions::v2_1::create_writer( fs.object_store.create(&fs.tmp_path).await.unwrap(), lance_schema, FileWriterOptions::default(), @@ -3664,6 +4264,96 @@ mod tests { file_writer.finish().await.unwrap(); } + #[derive(Clone, Copy, Debug)] + enum MetadataReadPath { + Full, + Indexed, + } + + #[derive(Clone, Copy, Debug)] + enum InvalidGboDescriptor { + Unaligned, + PastEof, + Overflowing, + } + + #[rstest] + #[case::full_unaligned(MetadataReadPath::Full, InvalidGboDescriptor::Unaligned, "not aligned")] + #[case::full_past_eof(MetadataReadPath::Full, InvalidGboDescriptor::PastEof, "outside file")] + #[case::full_overflowing( + MetadataReadPath::Full, + InvalidGboDescriptor::Overflowing, + "overflows" + )] + #[case::indexed_unaligned( + MetadataReadPath::Indexed, + InvalidGboDescriptor::Unaligned, + "not aligned" + )] + #[case::indexed_past_eof( + MetadataReadPath::Indexed, + InvalidGboDescriptor::PastEof, + "outside file" + )] + #[case::indexed_overflowing( + MetadataReadPath::Indexed, + InvalidGboDescriptor::Overflowing, + "overflows" + )] + #[tokio::test] + async fn test_metadata_rejects_invalid_gbo_descriptor( + #[case] read_path: MetadataReadPath, + #[case] invalid_descriptor: InvalidGboDescriptor, + #[case] expected_message: &str, + ) { + let fs = FsFixture::default(); + write_file_with_global_buffer(&fs, Bytes::from_static(b"hello")).await; + + let mut file_bytes = fs + .object_store + .read_one_all(&fs.tmp_path) + .await + .unwrap() + .to_vec(); + let file_len = file_bytes.len() as u64; + let footer = FileReader::decode_footer(&Bytes::copy_from_slice(&file_bytes)).unwrap(); + let gbo_table_start = usize::try_from(footer.global_buff_offsets_start).unwrap(); + let alignment = PAGE_BUFFER_ALIGNMENT as u64; + let (position, size) = match invalid_descriptor { + InvalidGboDescriptor::Unaligned => (1, 0), + InvalidGboDescriptor::PastEof => (((file_len + alignment) / alignment) * alignment, 0), + InvalidGboDescriptor::Overflowing => (u64::MAX - (u64::MAX % alignment), alignment), + }; + file_bytes[gbo_table_start..gbo_table_start + 8].copy_from_slice(&position.to_le_bytes()); + file_bytes[gbo_table_start + 8..gbo_table_start + 16].copy_from_slice(&size.to_le_bytes()); + fs.object_store + .put(&fs.tmp_path, &file_bytes) + .await + .unwrap(); + + let scheduler = fs + .scheduler + .open_file(&fs.tmp_path, &CachedFileSize::unknown()) + .await + .unwrap(); + let error = match read_path { + MetadataReadPath::Full => FileReader::read_all_metadata(&scheduler).await.map(|_| ()), + MetadataReadPath::Indexed => FileReader::read_metadata_index(&scheduler) + .await + .map(|_| ()), + } + .expect_err("invalid GBO descriptor must fail before metadata I/O"); + + assert!( + matches!(error, lance_core::Error::InvalidInput { .. }), + "expected InvalidInput, got {error:?}" + ); + assert!( + error.to_string().contains(expected_message), + "unexpected error: {error}" + ); + } + /// A global buffer that fits inside the tail region captured at open is served /// from memory with no additional I/O. A buffer larger than that window cannot /// fit and falls back to a dedicated read. Both must round-trip correctly. @@ -3720,7 +4410,7 @@ mod tests { #[tokio::test] async fn test_read_global_buffer_no_user_buffers() { let fs = FsFixture::default(); - create_some_file(&fs, LanceFileVersion::V2_1).await; + create_some_file(&fs, ConcreteFileVersion::V2_1).await; let file_scheduler = fs .scheduler @@ -3749,12 +4439,12 @@ mod tests { #[tokio::test] async fn test_deep_size_of_includes_column_metadata( #[values( - LanceFileVersion::V2_0, - LanceFileVersion::V2_1, - LanceFileVersion::V2_2, - LanceFileVersion::V2_3 + ConcreteFileVersion::V2_0, + ConcreteFileVersion::V2_1, + ConcreteFileVersion::V2_2, + ConcreteFileVersion::V2_3 )] - version: LanceFileVersion, + version: ConcreteFileVersion, ) { // Regression test: CachedFileMetadata::deep_size_of must account for // column_metadatas and column_infos, otherwise the moka cache weigher @@ -3850,21 +4540,11 @@ mod tests { let run = |dt: DataType, indices: &[u32], lengths: Vec| -> lance_core::Result { let arrow = ArrowSchema::new(vec![Field::new("s", dt, true)]); let schema = Schema::try_from(&arrow).unwrap(); - let column_len = |c: usize| Ok(lengths[c]); - let mut cursor = 0usize; - let mut field_lengths = Vec::new(); - for field in &schema.fields { - let rows = validate_field_length( - field, - is_structural, - true, - indices, - &mut cursor, - &column_len, - )?; - field_lengths.push((field.name.as_str(), rows)); + if is_structural { + versions::v2_1::test_projection_length(&schema, indices, &lengths) + } else { + versions::v2_0::test_projection_length(&schema, indices, &lengths) } - verify_uniform_lengths(&field_lengths) }; let struct_ty = || { @@ -3904,24 +4584,14 @@ mod tests { schema: Arc::new(schema), column_indices: vec![0], }; - let column_len = |column: usize| { - assert_eq!(column, 0); - Ok(3) - }; - let mut cursor = 0usize; - - let rows = validate_field_length( - &projection.schema.fields[0], - false, - true, + let rows = versions::v2_0::test_projection_length( + &projection.schema, &projection.column_indices, - &mut cursor, - &column_len, + &[3], ) .unwrap(); assert_eq!(rows, 3); - assert_eq!(cursor, 1); } #[test] @@ -3933,16 +4603,11 @@ mod tests { -> lance_core::Result { let arrow = ArrowSchema::new(vec![Field::new("f", dt, true)]); let schema = Schema::try_from(&arrow).unwrap(); - let column_len = |c: usize| Ok(lengths[c]); - let mut cursor = 0usize; - validate_field_length( - &schema.fields[0], - is_structural, - true, - indices, - &mut cursor, - &column_len, - ) + if is_structural { + versions::v2_1::test_projection_length(&schema, indices, &lengths) + } else { + versions::v2_0::test_projection_length(&schema, indices, &lengths) + } }; // A list's items have a different cardinality than its rows; that gap diff --git a/rust/lance-file/src/reader/structural.rs b/rust/lance-file/src/reader/structural.rs new file mode 100644 index 00000000000..18bda6eb933 --- /dev/null +++ b/rust/lance-file/src/reader/structural.rs @@ -0,0 +1,445 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright The Lance Authors + +use std::{collections::BTreeMap, sync::Arc}; + +use async_trait::async_trait; +use bytes::Bytes; +use lance_core::{ + Error, Result, + cache::LanceCache, + datatypes::{Field, Schema}, +}; +use lance_encoding::{ + EncodingsIo, + decoder::{ColumnInfo, PageEncoding, PageInfo}, + format::{pb, pb21}, +}; +use prost::{Message, Name}; + +use crate::{ + format::pbfile, + reader::{ + BufferDescriptor, FileMetadataIndex, FileMetadataProvider, FileReader, PreparedProjection, + ReadProjection, ReaderProjection, normalized_column_num_rows, verify_uniform_lengths, + }, + writer::PAGE_BUFFER_ALIGNMENT, +}; + +fn fetch_encoding(encoding: &pbfile::Encoding) -> Result { + match &encoding.location { + Some(pbfile::encoding::Location::Indirect(_)) => Err(Error::invalid_input_source( + "Indirect file encodings are not supported".into(), + )), + Some(pbfile::encoding::Location::Direct(encoding)) => { + let envelope = prost_types::Any::decode(Bytes::from(encoding.encoding.clone())) + .map_err(|error| { + Error::invalid_input_source( + format!("Invalid direct {} encoding envelope: {error}", M::NAME).into(), + ) + })?; + envelope.to_msg::().map_err(|error| { + Error::invalid_input_source( + format!("Invalid direct {} encoding: {error}", M::NAME).into(), + ) + }) + } + Some(pbfile::encoding::Location::None(_)) => Err(Error::invalid_input_source( + format!("Missing {} encoding description", M::NAME).into(), + )), + None => Err(Error::invalid_input_source( + format!("Missing {} encoding location", M::NAME).into(), + )), + } +} + +/// Decode structural page syntax before an exact reader validates its grammar. +pub fn decode_page_layout( + column_index: u32, + page_index: usize, + page: &pbfile::column_metadata::Page, +) -> Result { + fetch_encoding(page.encoding.as_ref().ok_or_else(|| { + Error::invalid_input_source( + format!( + "Column {} page {} is missing its encoding", + column_index, page_index + ) + .into(), + ) + })?) +} + +/// Build normalized page metadata after exact grammar validation. +pub fn build_page_info( + column_index: u32, + page_index: usize, + page: &pbfile::column_metadata::Page, + page_layout: pb21::PageLayout, +) -> Result { + if page.buffer_offsets.len() != page.buffer_sizes.len() { + return Err(Error::invalid_input_source( + format!( + "Column {} page {} has {} buffer offsets but {} buffer sizes", + column_index, + page_index, + page.buffer_offsets.len(), + page.buffer_sizes.len() + ) + .into(), + )); + } + let buffer_offsets_and_sizes = Arc::from( + page.buffer_offsets + .iter() + .zip(&page.buffer_sizes) + .map(|(offset, size)| { + if offset % PAGE_BUFFER_ALIGNMENT as u64 != 0 { + return Err(Error::invalid_input_source( + format!( + "Column {} page {} buffer offset {} is not aligned to {} bytes", + column_index, page_index, offset, PAGE_BUFFER_ALIGNMENT + ) + .into(), + )); + } + Ok((*offset, *size)) + }) + .collect::>>()?, + ); + Ok(PageInfo { + buffer_offsets_and_sizes, + encoding: PageEncoding::Structural(page_layout), + num_rows: page.length, + priority: page.priority, + }) +} + +/// Finish normalized column metadata after all pages have been validated. +pub fn build_column_info( + column_index: u32, + metadata: &pbfile::ColumnMetadata, + page_infos: Vec, +) -> Result> { + if metadata.buffer_offsets.len() != metadata.buffer_sizes.len() { + return Err(Error::invalid_input_source( + format!( + "Column {} has {} buffer offsets but {} buffer sizes", + column_index, + metadata.buffer_offsets.len(), + metadata.buffer_sizes.len() + ) + .into(), + )); + } + let buffer_offsets_and_sizes = Arc::from( + metadata + .buffer_offsets + .iter() + .zip(&metadata.buffer_sizes) + .map(|(offset, size)| (*offset, *size)) + .collect::>(), + ); + let encoding: pb::ColumnEncoding = + fetch_encoding(metadata.encoding.as_ref().ok_or_else(|| { + Error::invalid_input_source( + format!("Column {} is missing its encoding", column_index).into(), + ) + })?)?; + Ok(Arc::new(ColumnInfo { + index: column_index, + page_infos: Arc::from(page_infos), + buffer_offsets_and_sizes, + encoding, + })) +} + +/// Validate the structural global-buffer alignment contract. +pub fn validate_global_buffers(buffers: &[BufferDescriptor]) -> Result<()> { + for (buffer_index, buffer) in buffers.iter().enumerate() { + if buffer.position % PAGE_BUFFER_ALIGNMENT as u64 != 0 { + return Err(Error::invalid_input_source( + format!( + "Global buffer {} position {} is not aligned to {} bytes", + buffer_index, buffer.position, PAGE_BUFFER_ALIGNMENT + ) + .into(), + )); + } + } + Ok(()) +} + +fn field_column_shape(field: &Field) -> (bool, bool) { + if field.is_blob() || field.is_packed_struct() { + return (true, false); + } + (field.children.is_empty(), !field.children.is_empty()) +} + +/// Count physical columns in the leaf-column layout used by v2.1+. +pub fn physical_column_count(field: &Field) -> usize { + if field.children.is_empty() || field.is_blob() || field.is_packed_struct() { + 1 + } else { + field.children.iter().map(physical_column_count).sum() + } +} + +fn append_physical_fields( + fields: &[Field], + field_ids: &mut Vec, + column_indices: &mut Vec, + next_column: &mut i32, +) { + for field in fields { + if field.children.is_empty() || field.is_blob() || field.is_packed_struct() { + field_ids.push(field.id); + column_indices.push(*next_column); + *next_column += 1; + } else { + append_physical_fields(&field.children, field_ids, column_indices, next_column); + } + } +} + +/// Build persisted field-to-column entries for the v2.1+ leaf layout. +pub fn data_file_columns(schema: &Schema) -> (Vec, Vec) { + let mut field_ids = Vec::new(); + let mut column_indices = Vec::new(); + append_physical_fields(&schema.fields, &mut field_ids, &mut column_indices, &mut 0); + (field_ids, column_indices) +} + +/// Build the field-id lookup for the v2.1+ leaf layout. +pub fn field_id_to_column_index(schema: &Schema) -> BTreeMap { + let (field_ids, column_indices) = data_file_columns(schema); + field_ids + .into_iter() + .zip(column_indices) + .filter_map(|(field_id, column_index)| { + (column_index >= 0).then_some((field_id as u32, column_index as u32)) + }) + .collect() +} + +fn append_field_ids( + fields: &[Field], + field_id_to_column_index: &BTreeMap, + column_indices: &mut Vec, +) { + for field in fields { + let (contributes, recurse) = field_column_shape(field); + if contributes + && let Some(column_index) = field_id_to_column_index.get(&(field.id as u32)).copied() + { + column_indices.push(column_index); + } + if recurse { + append_field_ids(&field.children, field_id_to_column_index, column_indices); + } + } +} + +/// Build the leaf-column projection selected by a v2.1+ exact reader. +pub fn projection_from_field_ids( + schema: &Schema, + field_id_to_column_index: &BTreeMap, +) -> ReaderProjection { + let mut column_indices = Vec::new(); + append_field_ids( + &schema.fields, + field_id_to_column_index, + &mut column_indices, + ); + ReaderProjection { + schema: Arc::new(schema.clone()), + column_indices, + } +} + +/// Project names using a caller-selected leaf-column mapping. +pub fn projection_from_column_names( + schema: &Schema, + column_names: &[&str], + field_id_to_column_index: &BTreeMap, +) -> Result { + let projected = schema.project(column_names)?; + Ok(projection_from_field_ids( + &projected, + field_id_to_column_index, + )) +} + +fn children_share_parent_length(field: &Field) -> bool { + field.logical_type.is_struct() +} + +fn validate_field_length Result>( + field: &Field, + comparable: bool, + column_indices: &[u32], + cursor: &mut usize, + column_len: &F, +) -> Result { + let (contributes, recurse) = field_column_shape(field); + let mut field_rows = None; + if contributes { + let column = *column_indices.get(*cursor).ok_or_else(|| { + Error::invalid_input(format!( + "projection supplied fewer column indices than its fields require (ran out at field '{}')", + field.name + )) + })?; + *cursor += 1; + field_rows = Some(column_len(column as usize)?); + } + if recurse { + let enforce_children = comparable && children_share_parent_length(field); + for child in &field.children { + let child_rows = + validate_field_length(child, enforce_children, column_indices, cursor, column_len)?; + let expected = *field_rows.get_or_insert(child_rows); + if enforce_children && child_rows != expected { + return Err(Error::invalid_input(format!( + "cannot read field '{}': its children have differing lengths (child '{}' has {} rows, but the field has {}); a struct's children must all have the same length", + field.name, child.name, child_rows, expected + ))); + } + } + } + field_rows.ok_or_else(|| { + Error::invalid_input(format!( + "projected field '{}' maps to no columns", + field.name + )) + }) +} + +/// Determine the normalized logical read length for a structural projection. +pub fn prepared_read_length(prepared: &PreparedProjection) -> Result { + let column_len = |column: usize| { + let info = prepared.column_infos.get(column).ok_or_else(|| { + Error::invalid_input(format!( + "projection references column index {} but only {} columns are available", + column, + prepared.column_infos.len() + )) + })?; + normalized_column_num_rows(info) + }; + projection_length( + &prepared.decoder_projection.schema, + &prepared.decoder_projection.column_indices, + &column_len, + ) +} + +fn projection_length Result>( + schema: &Schema, + column_indices: &[u32], + column_len: &F, +) -> Result { + let mut cursor = 0; + let mut field_lengths = Vec::with_capacity(schema.fields.len()); + for field in &schema.fields { + let rows = validate_field_length(field, true, column_indices, &mut cursor, column_len)?; + field_lengths.push((field.name.as_str(), rows)); + } + if cursor != column_indices.len() { + return Err(Error::invalid_input(format!( + "projection supplied {} column indices but its fields require {}", + column_indices.len(), + cursor + ))); + } + verify_uniform_lengths(&field_lengths) +} + +pub type DecodeColumn = fn(u32, &pbfile::ColumnMetadata) -> Result>; + +#[derive(Debug)] +struct StructuralReadProjection { + decode_column: DecodeColumn, +} + +/// Compose structural projection execution with an exact column grammar. +pub fn read_projection(decode_column: DecodeColumn) -> Arc { + Arc::new(StructuralReadProjection { decode_column }) +} + +#[async_trait] +impl ReadProjection for StructuralReadProjection { + fn validate_indexed( + &self, + projection: &ReaderProjection, + metadata_index: &FileMetadataIndex, + ) -> Result<()> { + FileMetadataProvider::validate_indexed_projection_structure(projection, metadata_index)?; + if FileMetadataProvider::projection_matches_indexed_metadata(projection) { + Ok(()) + } else { + Err(FileMetadataProvider::indexed_projection_error( + projection, + metadata_index, + )) + } + } + + fn read_length(&self, prepared: &PreparedProjection) -> Result { + prepared_read_length(prepared) + } + + async fn prepare( + &self, + metadata_provider: &FileMetadataProvider, + projection: &ReaderProjection, + io: &Arc, + cache: &Arc, + ) -> Result<(PreparedProjection, u64)> { + let prepared = match metadata_provider { + FileMetadataProvider::Full(metadata) => { + FileReader::validate_projection(projection, metadata)?; + PreparedProjection { + column_infos: metadata.column_infos.clone(), + decoder_projection: projection.clone(), + } + } + FileMetadataProvider::Indexed(metadata_index) => { + self.validate_indexed(projection, metadata_index)?; + let column_infos = FileMetadataProvider::load_indexed_column_infos( + metadata_index, + io, + cache, + &projection.column_indices, + self.decode_column, + ) + .await?; + PreparedProjection { + column_infos, + decoder_projection: ReaderProjection { + schema: projection.schema.clone(), + column_indices: (0..projection.column_indices.len()) + .map(|index| index as u32) + .collect(), + }, + } + } + }; + let read_len = self.read_length(&prepared)?; + Ok((prepared, read_len)) + } +} + +#[cfg(test)] +pub fn test_projection_length( + schema: &Schema, + column_indices: &[u32], + column_lengths: &[u64], +) -> Result { + projection_length(schema, column_indices, &|column| { + column_lengths.get(column).copied().ok_or_else(|| { + Error::invalid_input(format!("missing synthetic length for column {column}")) + }) + }) +} diff --git a/rust/lance-file/src/testing.rs b/rust/lance-file/src/testing.rs index 7f554e39d31..65b340e7641 100644 --- a/rust/lance-file/src/testing.rs +++ b/rust/lance-file/src/testing.rs @@ -16,7 +16,8 @@ use lance_io::{ }; use crate::reader::{FileReader, FileReaderOptions}; -use crate::writer::{FileWriter, FileWriterOptions}; +use crate::version::ConcreteFileVersion; +use crate::{versions, writer::FileWriterOptions}; pub struct FsFixture { pub tmp_path: TempObjFile, @@ -47,13 +48,15 @@ pub struct WrittenFile { pub async fn write_lance_file( data: impl RecordBatchReader, fs: &FsFixture, + version: ConcreteFileVersion, options: FileWriterOptions, ) -> WrittenFile { let writer = fs.object_store.create(&fs.tmp_path).await.unwrap(); let lance_schema = lance_core::datatypes::Schema::try_from(data.schema().as_ref()).unwrap(); - let mut file_writer = FileWriter::try_new(writer, lance_schema.clone(), options).unwrap(); + let mut file_writer = + versions::create_writer(version, writer, lance_schema.clone(), options).unwrap(); let data = data .collect::, ArrowError>>() diff --git a/rust/lance-file/src/version.rs b/rust/lance-file/src/version.rs new file mode 100644 index 00000000000..da29b52d576 --- /dev/null +++ b/rust/lance-file/src/version.rs @@ -0,0 +1,409 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright The Lance Authors + +use std::{ + fmt::{Display, Formatter}, + str::FromStr, +}; + +use lance_core::deepsize::{Context, DeepSizeOf}; +use lance_core::{Error, Result}; + +pub const LEGACY_FORMAT_VERSION: &str = "0.1"; +pub const V2_FORMAT_2_0: &str = "2.0"; +pub const V2_FORMAT_2_1: &str = "2.1"; +pub const V2_FORMAT_2_2: &str = "2.2"; +pub const V2_FORMAT_2_3: &str = "2.3"; + +/// Resolve the current stable release policy to an exact file version. +pub const fn stable_file_version() -> ConcreteFileVersion { + ConcreteFileVersion::V2_1 +} + +/// Resolve the current next release policy to an exact file version. +pub const fn next_file_version() -> ConcreteFileVersion { + ConcreteFileVersion::V2_3 +} + +/// A caller-facing Lance file-version request. +/// +/// `Stable` and `Next` are release selectors. They resolve to an exact +/// [`ConcreteFileVersion`] before file or dataset dispatch and are never persisted. +#[derive(Debug, Default, Clone, Copy, PartialEq, Eq)] +pub enum LanceFileVersion { + /// The legacy v1 format. + Legacy, + /// Exact v2.0. + V2_0, + /// Exact v2.1 and the current default. + #[default] + V2_1, + /// The latest stable release. + Stable, + /// Exact v2.2. + V2_2, + /// The latest unstable release. + Next, + /// Exact v2.3. + V2_3, +} + +impl DeepSizeOf for LanceFileVersion { + fn deep_size_of_children(&self, _context: &mut Context) -> usize { + 0 + } +} + +impl LanceFileVersion { + /// Resolve this request through the current release policy. + pub const fn resolve(self) -> ConcreteFileVersion { + match self { + Self::Legacy => ConcreteFileVersion::V1, + Self::V2_0 => ConcreteFileVersion::V2_0, + Self::V2_1 => ConcreteFileVersion::V2_1, + Self::Stable => stable_file_version(), + Self::V2_2 => ConcreteFileVersion::V2_2, + Self::Next => next_file_version(), + Self::V2_3 => ConcreteFileVersion::V2_3, + } + } + + /// Whether this request resolves to an unstable exact format. + pub const fn is_unstable(self) -> bool { + self.resolve().is_unstable() + } +} + +impl Display for LanceFileVersion { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + f.write_str(match self { + Self::Legacy => LEGACY_FORMAT_VERSION, + Self::V2_0 => V2_FORMAT_2_0, + Self::V2_1 => V2_FORMAT_2_1, + Self::V2_2 => V2_FORMAT_2_2, + Self::V2_3 => V2_FORMAT_2_3, + Self::Stable => "stable", + Self::Next => "next", + }) + } +} + +impl FromStr for LanceFileVersion { + type Err = Error; + + fn from_str(value: &str) -> Result { + match value.to_lowercase().as_str() { + LEGACY_FORMAT_VERSION | "legacy" => Ok(Self::Legacy), + V2_FORMAT_2_0 | "0.3" => Ok(Self::V2_0), + V2_FORMAT_2_1 => Ok(Self::V2_1), + V2_FORMAT_2_2 => Ok(Self::V2_2), + V2_FORMAT_2_3 => Ok(Self::V2_3), + "stable" => Ok(Self::Stable), + "next" => Ok(Self::Next), + _ => Err(unknown_version(value)), + } + } +} + +/// The exact persisted identity of a Lance file format. +/// +/// Unlike [`LanceFileVersion`], this type cannot represent release selectors such as +/// `stable` or `next`. Exact versions deliberately have no ordering because format +/// capabilities are not implied by release order. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub enum ConcreteFileVersion { + /// The legacy v1 file format. + V1, + /// The v2.0 file format. + V2_0, + /// The v2.1 file format. + V2_1, + /// The v2.2 file format. + V2_2, + /// The v2.3 file format. + V2_3, +} + +impl DeepSizeOf for ConcreteFileVersion { + fn deep_size_of_children(&self, _context: &mut Context) -> usize { + 0 + } +} + +impl ConcreteFileVersion { + /// Convert this exact identity to the corresponding exact public selector. + /// + /// This never produces the release selectors `stable` or `next`. + pub const fn to_selector(self) -> LanceFileVersion { + match self { + Self::V1 => LanceFileVersion::Legacy, + Self::V2_0 => LanceFileVersion::V2_0, + Self::V2_1 => LanceFileVersion::V2_1, + Self::V2_2 => LanceFileVersion::V2_2, + Self::V2_3 => LanceFileVersion::V2_3, + } + } + + /// Whether this exact format is covered only by the unstable release policy. + pub const fn is_unstable(self) -> bool { + matches!(self, Self::V2_3) + } + + /// Decode the exact version string stored in a dataset manifest. + /// + /// Public selector aliases such as `legacy`, `0.3`, `stable`, and `next` are + /// intentionally rejected because manifests only store canonical exact versions. + pub fn from_manifest_string(value: &str) -> Result { + match value { + LEGACY_FORMAT_VERSION => Ok(Self::V1), + V2_FORMAT_2_0 => Ok(Self::V2_0), + V2_FORMAT_2_1 => Ok(Self::V2_1), + V2_FORMAT_2_2 => Ok(Self::V2_2), + V2_FORMAT_2_3 => Ok(Self::V2_3), + _ => Err(unknown_version(value)), + } + } + + /// Encode this exact version as the canonical string stored in a dataset manifest. + pub const fn to_manifest_string(self) -> &'static str { + match self { + Self::V1 => LEGACY_FORMAT_VERSION, + Self::V2_0 => V2_FORMAT_2_0, + Self::V2_1 => V2_FORMAT_2_1, + Self::V2_2 => V2_FORMAT_2_2, + Self::V2_3 => V2_FORMAT_2_3, + } + } + + /// Decode the major/minor version stored in `DataFile` metadata. + /// + /// Legacy manifests may omit these fields and decode to `(0, 0)`, so all legacy + /// v1 number pairs accepted by the historical decoder remain valid inputs. The + /// historical generic decoder also accepted the standard v2.0 footer pair `(0, 3)`; + /// decoding retains that compatibility while encoding always emits `(2, 0)`. + pub fn from_data_file_numbers(major: u32, minor: u32) -> Result { + match (major, minor) { + (0, 0..=2) => Ok(Self::V1), + (0, 3) | (2, 0) => Ok(Self::V2_0), + (2, 1) => Ok(Self::V2_1), + (2, 2) => Ok(Self::V2_2), + (2, 3) => Ok(Self::V2_3), + _ => Err(unknown_version(format_args!("{}.{}", major, minor))), + } + } + + /// Encode the canonical major/minor pair stored in `DataFile` metadata. + pub const fn to_data_file_numbers(self) -> (u32, u32) { + match self { + Self::V1 => (0, 2), + Self::V2_0 => (2, 0), + Self::V2_1 => (2, 1), + Self::V2_2 => (2, 2), + Self::V2_3 => (2, 3), + } + } + + /// Decode the major/minor version stored in a Lance file footer. + /// + /// V2.0 has two accepted representations: `(0, 3)` from the standard file writer + /// and `(2, 0)` from self-described and mini-lance writers. + pub fn from_footer_numbers(major: u16, minor: u16) -> Result { + match (major, minor) { + (0, 0..=2) => Ok(Self::V1), + (0, 3) | (2, 0) => Ok(Self::V2_0), + (2, 1) => Ok(Self::V2_1), + (2, 2) => Ok(Self::V2_2), + (2, 3) => Ok(Self::V2_3), + _ => Err(unknown_version(format_args!("{}.{}", major, minor))), + } + } + + /// Encode the footer numbers emitted by the standard Lance file writer. + pub const fn to_standard_footer_numbers(self) -> (u16, u16) { + match self { + Self::V1 => (0, 2), + Self::V2_0 => (0, 3), + Self::V2_1 => (2, 1), + Self::V2_2 => (2, 2), + Self::V2_3 => (2, 3), + } + } + + /// Encode the footer numbers emitted by self-described and mini-lance writers. + pub const fn to_embedded_footer_numbers(self) -> (u16, u16) { + match self { + Self::V1 => (0, 2), + Self::V2_0 => (2, 0), + Self::V2_1 => (2, 1), + Self::V2_2 => (2, 2), + Self::V2_3 => (2, 3), + } + } +} + +impl Display for ConcreteFileVersion { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + f.write_str(self.to_manifest_string()) + } +} + +fn unknown_version(value: impl Display) -> Error { + Error::invalid_input_source(format!("Unknown Lance storage version: {}", value).into()) +} + +#[cfg(test)] +mod tests { + use std::str::FromStr; + + use lance_io::object_store::ObjectStore; + use object_store::path::Path; + + use super::*; + + const EXACT_VERSIONS: [ConcreteFileVersion; 5] = [ + ConcreteFileVersion::V1, + ConcreteFileVersion::V2_0, + ConcreteFileVersion::V2_1, + ConcreteFileVersion::V2_2, + ConcreteFileVersion::V2_3, + ]; + + #[test] + fn selector_resolution_is_exact() { + let cases = [ + (LanceFileVersion::Legacy, ConcreteFileVersion::V1), + (LanceFileVersion::V2_0, ConcreteFileVersion::V2_0), + (LanceFileVersion::V2_1, ConcreteFileVersion::V2_1), + (LanceFileVersion::Stable, ConcreteFileVersion::V2_1), + (LanceFileVersion::V2_2, ConcreteFileVersion::V2_2), + (LanceFileVersion::Next, ConcreteFileVersion::V2_3), + (LanceFileVersion::V2_3, ConcreteFileVersion::V2_3), + ]; + + for (selector, expected) in cases { + assert_eq!(selector.resolve(), expected); + } + } + + #[test] + fn public_selector_aliases_remain_unchanged() { + let cases = [ + ("0.1", LanceFileVersion::Legacy), + ("legacy", LanceFileVersion::Legacy), + ("2.0", LanceFileVersion::V2_0), + ("0.3", LanceFileVersion::V2_0), + ("2.1", LanceFileVersion::V2_1), + ("stable", LanceFileVersion::Stable), + ("2.2", LanceFileVersion::V2_2), + ("next", LanceFileVersion::Next), + ("2.3", LanceFileVersion::V2_3), + ]; + + for (value, expected) in cases { + assert_eq!(LanceFileVersion::from_str(value).unwrap(), expected); + } + } + + #[test] + fn manifest_codec_only_accepts_canonical_exact_versions() { + for version in EXACT_VERSIONS { + let encoded = version.to_manifest_string(); + assert_eq!( + ConcreteFileVersion::from_manifest_string(encoded).unwrap(), + version + ); + } + + for selector_or_alias in ["legacy", "0.3", "stable", "next"] { + assert!(ConcreteFileVersion::from_manifest_string(selector_or_alias).is_err()); + } + } + + #[test] + fn data_file_codec_preserves_wire_numbers() { + let cases = [ + (ConcreteFileVersion::V1, (0, 2)), + (ConcreteFileVersion::V2_0, (2, 0)), + (ConcreteFileVersion::V2_1, (2, 1)), + (ConcreteFileVersion::V2_2, (2, 2)), + (ConcreteFileVersion::V2_3, (2, 3)), + ]; + + for (version, encoded) in cases { + assert_eq!(version.to_data_file_numbers(), encoded); + assert_eq!( + ConcreteFileVersion::from_data_file_numbers(encoded.0, encoded.1).unwrap(), + version + ); + } + for minor in 0..=2 { + assert_eq!( + ConcreteFileVersion::from_data_file_numbers(0, minor).unwrap(), + ConcreteFileVersion::V1 + ); + } + assert_eq!( + ConcreteFileVersion::from_data_file_numbers(0, 3).unwrap(), + ConcreteFileVersion::V2_0 + ); + } + + #[test] + fn footer_codec_preserves_both_v2_0_writer_representations() { + let standard_cases = [ + (ConcreteFileVersion::V1, (0, 2)), + (ConcreteFileVersion::V2_0, (0, 3)), + (ConcreteFileVersion::V2_1, (2, 1)), + (ConcreteFileVersion::V2_2, (2, 2)), + (ConcreteFileVersion::V2_3, (2, 3)), + ]; + let embedded_cases = [ + (ConcreteFileVersion::V1, (0, 2)), + (ConcreteFileVersion::V2_0, (2, 0)), + (ConcreteFileVersion::V2_1, (2, 1)), + (ConcreteFileVersion::V2_2, (2, 2)), + (ConcreteFileVersion::V2_3, (2, 3)), + ]; + + for (version, encoded) in standard_cases { + assert_eq!(version.to_standard_footer_numbers(), encoded); + assert_eq!( + ConcreteFileVersion::from_footer_numbers(encoded.0, encoded.1).unwrap(), + version + ); + } + for (version, encoded) in embedded_cases { + assert_eq!(version.to_embedded_footer_numbers(), encoded); + assert_eq!( + ConcreteFileVersion::from_footer_numbers(encoded.0, encoded.1).unwrap(), + version + ); + } + for minor in 0..=2 { + assert_eq!( + ConcreteFileVersion::from_footer_numbers(0, minor).unwrap(), + ConcreteFileVersion::V1 + ); + } + } + + #[tokio::test] + async fn file_version_detection_accepts_all_legacy_footer_aliases() { + let object_store = ObjectStore::memory(); + for minor in 0u16..=2 { + let path = Path::from(format!("legacy-{minor}.lance")); + let mut footer = Vec::with_capacity(8); + footer.extend_from_slice(&0u16.to_le_bytes()); + footer.extend_from_slice(&minor.to_le_bytes()); + footer.extend_from_slice(crate::format::MAGIC); + object_store.put(&path, &footer).await.unwrap(); + + assert_eq!( + crate::determine_file_version(&object_store, &path, Some(footer.len())) + .await + .unwrap(), + ConcreteFileVersion::V1 + ); + } + } +} diff --git a/rust/lance-file/src/versions/mod.rs b/rust/lance-file/src/versions/mod.rs new file mode 100644 index 00000000000..95547baa514 --- /dev/null +++ b/rust/lance-file/src/versions/mod.rs @@ -0,0 +1,396 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright The Lance Authors + +//! Exact file-format composition roots. +//! +//! Each version module lists the mechanisms used by that file format. Callers +//! resolve release selectors before entering this module. Prefer APIs under a +//! concrete module such as [`v2_1`] when the version is statically known. The +//! root functions perform the single exhaustive dispatch for runtime versions. + +use bytes::Bytes; +use lance_core::{Error, Result, datatypes::Schema}; +use lance_encoding::{ + decoder::{ColumnInfo, DecoderPlugins, PageInfo}, + encoder::EncodedBatch, +}; +use lance_io::{scheduler::FileScheduler, traits::Writer}; +use std::{collections::BTreeMap, future::Future, sync::Arc}; + +use crate::{ + format::pbfile, + reader::{ + BufferDescriptor, CachedFileMetadata, FileMetadataIndex, FileMetadataProvider, FileReader, + FileReaderOptions, ProjectedFileReader, RawFileMetadata, ReadProjection, ReaderProjection, + }, + version::ConcreteFileVersion, + writer::{FileWriter, FileWriterOptions}, +}; +use lance_core::cache::LanceCache; + +pub mod v1; +pub mod v2_0; +pub mod v2_1; +pub mod v2_2; +pub mod v2_3; + +pub(crate) fn read_projection(version: ConcreteFileVersion) -> Result> { + match version { + ConcreteFileVersion::V1 => Err(Error::internal( + "current reader composition received Lance v1".to_string(), + )), + ConcreteFileVersion::V2_0 => Ok(v2_0::read_projection()), + ConcreteFileVersion::V2_1 => Ok(v2_1::read_projection()), + ConcreteFileVersion::V2_2 => Ok(v2_2::read_projection()), + ConcreteFileVersion::V2_3 => Ok(v2_3::read_projection()), + } +} + +/// A self-described file reader selected by the exact footer version. +pub enum OpenedFileReader { + /// A v1 file. The persisted footer numbers are retained for diagnostics. + V1 { + /// The major version stored in the footer. + major_version: u16, + /// The minor version stored in the footer. + minor_version: u16, + }, + /// A current-format reader selected from the exact footer identity. + Current(FileReader), +} + +pub(crate) fn finish_metadata( + version: ConcreteFileVersion, + metadata: RawFileMetadata, +) -> Result { + match version { + ConcreteFileVersion::V1 => Err(Error::internal( + "current metadata dispatch received a Lance v1 file".to_string(), + )), + ConcreteFileVersion::V2_0 => v2_0::finish_metadata(metadata), + ConcreteFileVersion::V2_1 => v2_1::finish_metadata(metadata), + ConcreteFileVersion::V2_2 => v2_2::finish_metadata(metadata), + ConcreteFileVersion::V2_3 => v2_3::finish_metadata(metadata), + } +} + +pub(crate) fn finish_metadata_index(index: FileMetadataIndex) -> Result { + match index.version { + ConcreteFileVersion::V1 => Err(Error::version_conflict( + "Attempt to use the Lance current-format reader with a v1 metadata index".to_string(), + 0, + 2, + )), + ConcreteFileVersion::V2_0 => v2_0::finish_metadata_index(index), + ConcreteFileVersion::V2_1 => v2_1::finish_metadata_index(index), + ConcreteFileVersion::V2_2 => v2_2::finish_metadata_index(index), + ConcreteFileVersion::V2_3 => v2_3::finish_metadata_index(index), + } +} + +pub(crate) fn decode_column_metadata( + version: ConcreteFileVersion, + column_metadatas: &[pbfile::ColumnMetadata], +) -> Result>> { + match version { + ConcreteFileVersion::V1 => Err(Error::not_supported( + "self-described batches are not part of the Lance v1 grammar".to_string(), + )), + ConcreteFileVersion::V2_0 => v2_0::decode_column_metadata(column_metadatas), + ConcreteFileVersion::V2_1 => v2_1::decode_column_metadata(column_metadatas), + ConcreteFileVersion::V2_2 => v2_2::decode_column_metadata(column_metadatas), + ConcreteFileVersion::V2_3 => v2_3::decode_column_metadata(column_metadatas), + } +} + +pub(crate) fn validate_global_buffers( + version: ConcreteFileVersion, + buffers: &[BufferDescriptor], +) -> Result<()> { + match version { + ConcreteFileVersion::V1 => Ok(()), + ConcreteFileVersion::V2_0 => v2_0::validate_global_buffers(buffers), + ConcreteFileVersion::V2_1 => v2_1::validate_global_buffers(buffers), + ConcreteFileVersion::V2_2 => v2_2::validate_global_buffers(buffers), + ConcreteFileVersion::V2_3 => v2_3::validate_global_buffers(buffers), + } +} + +pub fn reader_projection_from_field_ids( + version: ConcreteFileVersion, + schema: &Schema, + field_id_to_column_index: &BTreeMap, +) -> Result { + Ok(match version { + ConcreteFileVersion::V1 => v1::projection_from_field_ids(schema, field_id_to_column_index), + ConcreteFileVersion::V2_0 => { + v2_0::projection_from_field_ids(schema, field_id_to_column_index) + } + ConcreteFileVersion::V2_1 => { + v2_1::projection_from_field_ids(schema, field_id_to_column_index) + } + ConcreteFileVersion::V2_2 => { + v2_2::projection_from_field_ids(schema, field_id_to_column_index) + } + ConcreteFileVersion::V2_3 => { + v2_3::projection_from_field_ids(schema, field_id_to_column_index) + } + }) +} + +pub fn reader_projection_from_whole_schema( + schema: &Schema, + version: ConcreteFileVersion, +) -> ReaderProjection { + match version { + ConcreteFileVersion::V1 => v1::projection_from_whole_schema(schema), + ConcreteFileVersion::V2_0 => v2_0::projection_from_whole_schema(schema), + ConcreteFileVersion::V2_1 => v2_1::projection_from_whole_schema(schema), + ConcreteFileVersion::V2_2 => v2_2::projection_from_whole_schema(schema), + ConcreteFileVersion::V2_3 => v2_3::projection_from_whole_schema(schema), + } +} + +pub fn reader_projection_from_column_names( + version: ConcreteFileVersion, + schema: &Schema, + column_names: &[&str], +) -> Result { + match version { + ConcreteFileVersion::V1 => v1::projection_from_column_names(schema, column_names), + ConcreteFileVersion::V2_0 => v2_0::projection_from_column_names(schema, column_names), + ConcreteFileVersion::V2_1 => v2_1::projection_from_column_names(schema, column_names), + ConcreteFileVersion::V2_2 => v2_2::projection_from_column_names(schema, column_names), + ConcreteFileVersion::V2_3 => v2_3::projection_from_column_names(schema, column_names), + } +} + +/// Count the physical columns represented by one field in an exact grammar. +pub fn physical_column_count( + version: ConcreteFileVersion, + field: &lance_core::datatypes::Field, +) -> usize { + match version { + ConcreteFileVersion::V1 => v1::physical_column_count(field), + ConcreteFileVersion::V2_0 => v2_0::physical_column_count(field), + ConcreteFileVersion::V2_1 => v2_1::physical_column_count(field), + ConcreteFileVersion::V2_2 => v2_2::physical_column_count(field), + ConcreteFileVersion::V2_3 => v2_3::physical_column_count(field), + } +} + +/// Build persisted field-to-column entries for an exact grammar. +pub fn data_file_columns(version: ConcreteFileVersion, schema: &Schema) -> (Vec, Vec) { + match version { + ConcreteFileVersion::V1 => v1::data_file_columns(schema), + ConcreteFileVersion::V2_0 => v2_0::data_file_columns(schema), + ConcreteFileVersion::V2_1 => v2_1::data_file_columns(schema), + ConcreteFileVersion::V2_2 => v2_2::data_file_columns(schema), + ConcreteFileVersion::V2_3 => v2_3::data_file_columns(schema), + } +} + +/// Copy one column's external metadata and buffers according to the exact file +/// grammar. +/// +/// The caller supplies the version-free I/O operation. V2.0 may suppress that +/// operation when a structural header page has already been copied. +pub async fn copy_external_metadata_column( + version: ConcreteFileVersion, + schema: &Schema, + column_index: usize, + has_existing_pages: bool, + copy: Copy, +) -> Result<()> +where + Copy: FnOnce() -> CopyFuture + Send, + CopyFuture: Future> + Send, +{ + match version { + ConcreteFileVersion::V1 => Err(Error::not_supported( + "binary-copy metadata operations are not supported for Lance v1".to_string(), + )), + ConcreteFileVersion::V2_0 => { + if v2_0::should_copy_external_metadata_column(schema, column_index, has_existing_pages) + { + copy().await + } else { + Ok(()) + } + } + ConcreteFileVersion::V2_1 | ConcreteFileVersion::V2_2 | ConcreteFileVersion::V2_3 => { + copy().await + } + } +} + +/// Normalize one copied column before an exact-version footer is written. +pub fn finalize_external_metadata_column( + version: ConcreteFileVersion, + schema: &Schema, + column_index: usize, + pages: &mut Vec, + num_rows: u64, +) -> Result<()> { + match version { + ConcreteFileVersion::V1 => Err(Error::not_supported( + "binary-copy metadata operations are not supported for Lance v1".to_string(), + )), + ConcreteFileVersion::V2_0 => { + v2_0::finalize_external_metadata_column(schema, column_index, pages, num_rows); + Ok(()) + } + ConcreteFileVersion::V2_1 | ConcreteFileVersion::V2_2 | ConcreteFileVersion::V2_3 => Ok(()), + } +} + +/// Open a projected reader while keeping exact metadata-form selection in the +/// file layer. +/// +/// `open_indexed` returns `None` when the loaded index is not selective enough +/// to justify a projected reader. V2.0 never invokes it because indexed +/// metadata is not part of that reader's accepted grammar. +pub async fn open_projected_reader( + version: ConcreteFileVersion, + projection: &ReaderProjection, + prefer_indexed: bool, + open_indexed: OpenIndexed, + open_full: OpenFull, +) -> Result +where + OpenIndexed: FnOnce() -> IndexedFuture + Send, + IndexedFuture: Future>> + Send, + OpenFull: FnOnce() -> FullFuture + Send, + FullFuture: Future> + Send, +{ + match version { + ConcreteFileVersion::V1 => Err(Error::not_supported( + "projected current-format readers cannot open Lance v1 files".to_string(), + )), + ConcreteFileVersion::V2_0 => open_full().await, + ConcreteFileVersion::V2_1 | ConcreteFileVersion::V2_2 | ConcreteFileVersion::V2_3 => { + if prefer_indexed + && FileMetadataProvider::projection_matches_indexed_metadata(projection) + && let Some(reader) = open_indexed().await? + { + return Ok(reader); + } + open_full().await + } + } +} + +/// Open a self-described file and dispatch to the matching reader. +/// +/// The current-format reader's optimistic tail read is also used for exact +/// version detection, so current files do not pay for a separate footer probe. +pub async fn open_self_described_reader( + scheduler: FileScheduler, + decoder_plugins: Arc, + cache: &LanceCache, + options: FileReaderOptions, +) -> Result { + FileReader::try_open_for_dispatch(scheduler, None, decoder_plugins, cache, options).await +} + +/// Create a current-format writer for an exact file version. +/// +/// V1 uses [`v1::writer::FileWriter`] directly because its manifest provider is +/// part of the writer type. +pub fn create_writer( + version: ConcreteFileVersion, + object_writer: Box, + schema: Schema, + options: FileWriterOptions, +) -> Result { + match version { + ConcreteFileVersion::V1 => Err(Error::not_supported( + "Lance v1 files must be created with versions::v1::writer::FileWriter".to_string(), + )), + ConcreteFileVersion::V2_0 => { + v2_0::create_writer(object_writer, schema, options).map(Into::into) + } + ConcreteFileVersion::V2_1 => { + v2_1::create_writer(object_writer, schema, options).map(Into::into) + } + ConcreteFileVersion::V2_2 => { + v2_2::create_writer(object_writer, schema, options).map(Into::into) + } + ConcreteFileVersion::V2_3 => { + v2_3::create_writer(object_writer, schema, options).map(Into::into) + } + } +} + +/// Create a lazy current-format writer for an exact file version. +pub fn create_lazy_writer( + version: ConcreteFileVersion, + object_writer: Box, + options: FileWriterOptions, +) -> Result { + match version { + ConcreteFileVersion::V1 => Err(Error::not_supported( + "legacy v1 files require an explicit schema and manifest provider".to_string(), + )), + ConcreteFileVersion::V2_0 => Ok(v2_0::create_lazy_writer(object_writer, options).into()), + ConcreteFileVersion::V2_1 => Ok(v2_1::create_lazy_writer(object_writer, options).into()), + ConcreteFileVersion::V2_2 => Ok(v2_2::create_lazy_writer(object_writer, options).into()), + ConcreteFileVersion::V2_3 => Ok(v2_3::create_lazy_writer(object_writer, options).into()), + } +} + +/// Encode a self-described batch for an exact file version. +pub fn encode_self_described_batch( + version: ConcreteFileVersion, + batch: &EncodedBatch, +) -> Result { + match version { + ConcreteFileVersion::V1 => Err(Error::not_supported( + "Lance v1 does not support self-described current-format batches".to_string(), + )), + ConcreteFileVersion::V2_0 => v2_0::encode_self_described_batch(batch), + ConcreteFileVersion::V2_1 => v2_1::encode_self_described_batch(batch), + ConcreteFileVersion::V2_2 => v2_2::encode_self_described_batch(batch), + ConcreteFileVersion::V2_3 => v2_3::encode_self_described_batch(batch), + } +} + +/// Encode a mini-lance batch for an exact file version. +pub fn encode_mini_batch(version: ConcreteFileVersion, batch: &EncodedBatch) -> Result { + match version { + ConcreteFileVersion::V1 => Err(Error::not_supported( + "Lance v1 does not support mini-lance current-format batches".to_string(), + )), + ConcreteFileVersion::V2_0 => v2_0::encode_mini_batch(batch), + ConcreteFileVersion::V2_1 => v2_1::encode_mini_batch(batch), + ConcreteFileVersion::V2_2 => v2_2::encode_mini_batch(batch), + ConcreteFileVersion::V2_3 => v2_3::encode_mini_batch(batch), + } +} + +#[cfg(test)] +mod tests { + use std::sync::Arc; + + use super::*; + + #[test] + fn v1_rejects_current_format_embedded_batches() { + let batch = EncodedBatch { + data: Bytes::new(), + page_table: Vec::new(), + schema: Arc::new(Schema::default()), + top_level_columns: Vec::new(), + num_rows: 0, + }; + + assert!(matches!( + encode_self_described_batch(ConcreteFileVersion::V1, &batch), + Err(Error::NotSupported { .. }) + )); + assert!(matches!( + encode_mini_batch(ConcreteFileVersion::V1, &batch), + Err(Error::NotSupported { .. }) + )); + } +} diff --git a/rust/lance-file/src/versions/v1/encoding.rs b/rust/lance-file/src/versions/v1/encoding.rs new file mode 100644 index 00000000000..674fd357fad --- /dev/null +++ b/rust/lance-file/src/versions/v1/encoding.rs @@ -0,0 +1,192 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright The Lance Authors + +//! Lance v1 column encoding grammar. +//! +//! These codecs own the v1 wire layout, including schema dictionary payloads. +//! They are intentionally not exposed through the version-free I/O layer. + +use arrow_array::{ + Array, ArrayRef, + types::{BinaryType, LargeBinaryType, LargeUtf8Type, Utf8Type}, +}; +use arrow_schema::DataType; +use async_recursion::async_recursion; + +pub mod binary; +pub mod dictionary; +pub mod plain; + +use lance_arrow::DataTypeExt; +use lance_core::{ + Error, Result, + datatypes::{Field, Schema}, +}; +use lance_io::{ + ReadBatchParams, + traits::{Reader, Writer}, +}; + +use self::{ + binary::{BinaryDecoder, BinaryEncoder}, + plain::{PlainDecoder, PlainEncoder}, +}; + +/// Decode a binary-like array from a v1 values region. +pub async fn read_binary_array( + reader: &dyn Reader, + data_type: &DataType, + nullable: bool, + position: usize, + length: usize, + params: impl Into, +) -> Result { + use arrow_schema::DataType::*; + + let params = params.into(); + match data_type { + Utf8 => { + BinaryDecoder::::new(reader, position, length, nullable) + .get(params) + .await + } + Binary => { + BinaryDecoder::::new(reader, position, length, nullable) + .get(params) + .await + } + LargeUtf8 => { + BinaryDecoder::::new(reader, position, length, nullable) + .get(params) + .await + } + LargeBinary => { + BinaryDecoder::::new(reader, position, length, nullable) + .get(params) + .await + } + _ => Err(lance_core::Error::invalid_input(format!( + "unsupported v1 binary data type: {data_type}" + ))), + } +} + +/// Decode a fixed-stride array from a v1 values region. +pub async fn read_fixed_stride_array( + reader: &dyn Reader, + data_type: &DataType, + position: usize, + length: usize, + params: impl Into, +) -> Result { + if !lance_arrow::DataTypeExt::is_fixed_stride(data_type) { + return Err(lance_core::Error::schema(format!( + "{data_type} is not a fixed stride type" + ))); + } + PlainDecoder::new(reader, data_type, position, length)? + .get(params.into()) + .await +} + +/// Persist every schema dictionary using the v1 value codecs. +pub async fn write_schema_dictionaries(writer: &mut dyn Writer, schema: &mut Schema) -> Result<()> { + let max_field_id = schema.max_field_id().unwrap_or(-1); + for field_id in 0..=max_field_id { + let Some(field) = schema.mut_field_by_id(field_id) else { + continue; + }; + if !field.data_type().is_dictionary() { + continue; + } + + let dict_info = field.dictionary.as_mut().ok_or_else(|| { + Error::io(format!( + "v1 dictionary field '{}' is missing dictionary metadata", + field.name + )) + })?; + let values = dict_info.values.as_ref().ok_or_else(|| { + Error::invalid_input(format!( + "v1 dictionary field '{}' is missing dictionary values", + field.name + )) + })?; + + let data_type = values.data_type(); + let position = if data_type.is_numeric() { + PlainEncoder::new(writer, data_type) + .encode(&[values]) + .await? + } else if data_type.is_binary_like() { + BinaryEncoder::new(writer).encode(&[values]).await? + } else { + return Err(Error::schema(format!( + "v1 dictionary values do not support data type {data_type}" + ))); + }; + dict_info.offset = position; + dict_info.length = values.len(); + } + Ok(()) +} + +#[async_recursion] +async fn populate_field_dictionary(field: &mut Field, reader: &dyn Reader) -> Result<()> { + if let DataType::Dictionary(_, value_type) = field.data_type() { + let dict_info = field.dictionary.as_mut().ok_or_else(|| { + Error::io(format!( + "v1 dictionary field '{}' is missing dictionary metadata", + field.name + )) + })?; + let values = if value_type.is_binary_like() { + read_binary_array( + reader, + value_type.as_ref(), + true, + dict_info.offset, + dict_info.length, + .., + ) + .await? + } else if matches!( + value_type.as_ref(), + DataType::Int8 + | DataType::Int16 + | DataType::Int32 + | DataType::Int64 + | DataType::UInt8 + | DataType::UInt16 + | DataType::UInt32 + | DataType::UInt64 + ) { + read_fixed_stride_array( + reader, + value_type.as_ref(), + dict_info.offset, + dict_info.length, + .., + ) + .await? + } else { + return Err(Error::schema(format!( + "v1 dictionary values do not support data type {value_type}" + ))); + }; + dict_info.values = Some(values); + } else { + for child in &mut field.children { + populate_field_dictionary(child, reader).await?; + } + } + Ok(()) +} + +/// Load every persisted v1 schema dictionary into its in-memory field. +pub async fn populate_schema_dictionaries(schema: &mut Schema, reader: &dyn Reader) -> Result<()> { + for field in &mut schema.fields { + populate_field_dictionary(field, reader).await?; + } + Ok(()) +} diff --git a/rust/lance-io/src/encodings/binary.rs b/rust/lance-file/src/versions/v1/encoding/binary.rs similarity index 90% rename from rust/lance-io/src/encodings/binary.rs rename to rust/lance-file/src/versions/v1/encoding/binary.rs index ecd14f0d462..38aaa0de72d 100644 --- a/rust/lance-io/src/encodings/binary.rs +++ b/rust/lance-file/src/versions/v1/encoding/binary.rs @@ -5,7 +5,7 @@ //! use std::marker::PhantomData; -use std::ops::{Range, RangeFrom, RangeFull, RangeTo}; +use std::ops::Range; use std::sync::Arc; use arrow_arith::numeric::sub; @@ -23,16 +23,17 @@ use arrow_buffer::{ArrowNativeType, Buffer, MutableBuffer, ScalarBuffer, bit_uti use arrow_cast::cast::cast; use arrow_data::ArrayDataBuilder; use arrow_schema::DataType; -use async_trait::async_trait; use bytes::Bytes; use futures::{StreamExt, TryStreamExt}; use lance_arrow::BufferExt; use tokio::io::AsyncWriteExt; -use super::ReadBatchParams; -use super::{AsyncIndex, Decoder, Encoder, plain::PlainDecoder}; -use crate::traits::{Reader, Writer}; +use super::plain::PlainDecoder; use lance_core::Result; +use lance_io::{ + ReadBatchParams, + traits::{Reader, Writer}, +}; /// Encoder for Var-binary encoding. pub struct BinaryEncoder<'a> { @@ -87,9 +88,8 @@ impl<'a> BinaryEncoder<'a> { } } -#[async_trait] -impl Encoder for BinaryEncoder<'_> { - async fn encode(&mut self, arrs: &[&dyn Array]) -> Result { +impl BinaryEncoder<'_> { + pub async fn encode(&mut self, arrs: &[&dyn Array]) -> Result { assert!(!arrs.is_empty()); let data_type = arrs[0].data_type(); match data_type { @@ -97,12 +97,10 @@ impl Encoder for BinaryEncoder<'_> { DataType::Binary => self.encode_typed_arr::(arrs).await, DataType::LargeUtf8 => self.encode_typed_arr::(arrs).await, DataType::LargeBinary => self.encode_typed_arr::(arrs).await, - _ => { - return Err(lance_core::Error::invalid_input(format!( - "Unsupported data type for binary encoding: {}", - data_type - ))); - } + _ => Err(lance_core::Error::invalid_input(format!( + "Unsupported data type for binary encoding: {}", + data_type + ))), } } } @@ -134,7 +132,8 @@ impl<'a, T: ByteArrayType> BinaryDecoder<'a, T> { /// ```rust /// use arrow_array::types::Utf8Type; /// use object_store::path::Path; - /// use lance_io::{local::LocalObjectReader, encodings::binary::BinaryDecoder, traits::Reader}; + /// use lance_file::versions::v1::encoding::binary::BinaryDecoder; + /// use lance_io::{local::LocalObjectReader, traits::Reader}; /// /// async { /// let reader = LocalObjectReader::open_local_path("/tmp/foo.lance", 2048, None).await.unwrap(); @@ -285,16 +284,15 @@ fn plan_take_chunks( Ok(chunks) } -#[async_trait] -impl Decoder for BinaryDecoder<'_, T> { - async fn decode(&self) -> Result { +impl BinaryDecoder<'_, T> { + pub async fn decode(&self) -> Result { self.get(..).await } /// Take the values at the given indices. /// /// This function assumes indices are sorted. - async fn take(&self, indices: &UInt32Array) -> Result { + pub async fn take(&self, indices: &UInt32Array) -> Result { if indices.is_empty() { return Ok(new_empty_array(&T::DATA_TYPE)); } @@ -393,64 +391,17 @@ impl Decoder for BinaryDecoder<'_, T> { } } -#[async_trait] -impl AsyncIndex for BinaryDecoder<'_, T> { - type Output = Result; - - async fn get(&self, index: usize) -> Self::Output { - self.get(index..index + 1).await - } -} - -#[async_trait] -impl AsyncIndex> for BinaryDecoder<'_, T> { - type Output = Result; - - async fn get(&self, index: RangeFrom) -> Self::Output { - self.get(index.start..self.length).await - } -} - -#[async_trait] -impl AsyncIndex> for BinaryDecoder<'_, T> { - type Output = Result; - - async fn get(&self, index: RangeTo) -> Self::Output { - self.get(0..index.end).await - } -} - -#[async_trait] -impl AsyncIndex for BinaryDecoder<'_, T> { - type Output = Result; - - async fn get(&self, _: RangeFull) -> Self::Output { - self.get(0..self.length).await - } -} - -#[async_trait] -impl AsyncIndex for BinaryDecoder<'_, T> { - type Output = Result; - - async fn get(&self, params: ReadBatchParams) -> Self::Output { - match params { - ReadBatchParams::Range(r) => self.get(r).await, - // Ranges not supported in v1 files - ReadBatchParams::Ranges(_) => unimplemented!(), - ReadBatchParams::RangeFull => self.get(..).await, - ReadBatchParams::RangeTo(r) => self.get(r).await, - ReadBatchParams::RangeFrom(r) => self.get(r).await, - ReadBatchParams::Indices(indices) => self.take(&indices).await, +impl BinaryDecoder<'_, T> { + async fn decode_range(&self, index: Range) -> Result { + if index.end > self.length { + return Err(lance_core::Error::invalid_input(format!( + "v1 binary row range {}..{} exceeds length {}", + index.start, index.end, self.length + ))); + } + if index.is_empty() { + return Ok(new_empty_array(&T::DATA_TYPE)); } - } -} - -#[async_trait] -impl AsyncIndex> for BinaryDecoder<'_, T> { - type Output = Result; - - async fn get(&self, index: Range) -> Self::Output { let position_decoder = PlainDecoder::new( self.reader, &DataType::Int64, @@ -462,6 +413,19 @@ impl AsyncIndex> for BinaryDecoder<'_, T> { self.get_range(int64_positions, 0..index.len()).await } + + pub async fn get(&self, params: impl Into) -> Result { + match params.into() { + ReadBatchParams::Range(range) => self.decode_range(range).await, + ReadBatchParams::Ranges(_) => Err(lance_core::Error::invalid_input( + "multiple ranges are not supported by v1 binary encoding", + )), + ReadBatchParams::RangeFull => self.decode_range(0..self.length).await, + ReadBatchParams::RangeTo(range) => self.decode_range(0..range.end).await, + ReadBatchParams::RangeFrom(range) => self.decode_range(range.start..self.length).await, + ReadBatchParams::Indices(indices) => self.take(&indices).await, + } + } } #[cfg(test)] @@ -474,7 +438,7 @@ mod tests { use arrow_select::concat::concat; use lance_core::utils::tempfile::TempStdFile; - use crate::local::LocalObjectReader; + use lance_io::local::LocalObjectReader; async fn write_test_data( path: impl AsRef, diff --git a/rust/lance-io/src/encodings/dictionary.rs b/rust/lance-file/src/versions/v1/encoding/dictionary.rs similarity index 88% rename from rust/lance-io/src/encodings/dictionary.rs rename to rust/lance-file/src/versions/v1/encoding/dictionary.rs index b51adf66a59..0056632f725 100644 --- a/rust/lance-io/src/encodings/dictionary.rs +++ b/rust/lance-file/src/versions/v1/encoding/dictionary.rs @@ -14,18 +14,14 @@ use arrow_array::types::{ }; use arrow_array::{Array, ArrayRef, DictionaryArray, PrimitiveArray, UInt32Array}; use arrow_schema::DataType; -use async_trait::async_trait; -use crate::{ +use lance_core::{Error, Result}; +use lance_io::{ ReadBatchParams, traits::{Reader, Writer}, }; -use lance_core::{Error, Result}; -use super::AsyncIndex; -use super::plain::PlainEncoder; -use crate::encodings::plain::PlainDecoder; -use crate::encodings::{Decoder, Encoder}; +use super::plain::{PlainDecoder, PlainEncoder}; /// Encoder for Dictionary encoding. pub struct DictionaryEncoder<'a> { @@ -60,9 +56,8 @@ impl<'a> DictionaryEncoder<'a> { } } -#[async_trait] -impl Encoder for DictionaryEncoder<'_> { - async fn encode(&mut self, array: &[&dyn Array]) -> Result { +impl DictionaryEncoder<'_> { + pub async fn encode(&mut self, array: &[&dyn Array]) -> Result { use DataType::*; match self.key_type { @@ -165,36 +160,17 @@ impl<'a> DictionaryDecoder<'a> { } } -#[async_trait] -impl Decoder for DictionaryDecoder<'_> { - async fn decode(&self) -> Result { +impl DictionaryDecoder<'_> { + pub async fn decode(&self) -> Result { self.decode_impl(..).await } - async fn take(&self, indices: &UInt32Array) -> Result { + pub async fn take(&self, indices: &UInt32Array) -> Result { self.decode_impl(indices.clone()).await } -} - -#[async_trait] -impl AsyncIndex for DictionaryDecoder<'_> { - type Output = Result; - - async fn get(&self, _index: usize) -> Self::Output { - Err(Error::not_supported_source( - "DictionaryDecoder does not support get()" - .to_string() - .into(), - )) - } -} - -#[async_trait] -impl AsyncIndex for DictionaryDecoder<'_> { - type Output = Result; - async fn get(&self, params: ReadBatchParams) -> Self::Output { - self.decode_impl(params.clone()).await + pub async fn get(&self, params: impl Into) -> Result { + self.decode_impl(params).await } } @@ -202,10 +178,10 @@ impl AsyncIndex for DictionaryDecoder<'_> { mod tests { use super::*; - use crate::local::LocalObjectReader; use arrow_array::StringArray; use arrow_buffer::ArrowNativeType; use lance_core::utils::tempfile::TempStdFile; + use lance_io::local::LocalObjectReader; use tokio::io::AsyncWriteExt; async fn test_dict_decoder_for_type() { diff --git a/rust/lance-io/src/encodings/plain.rs b/rust/lance-file/src/versions/v1/encoding/plain.rs similarity index 91% rename from rust/lance-io/src/encodings/plain.rs rename to rust/lance-file/src/versions/v1/encoding/plain.rs index a5ec97c7beb..18910e9251e 100644 --- a/rust/lance-io/src/encodings/plain.rs +++ b/rust/lance-file/src/versions/v1/encoding/plain.rs @@ -6,14 +6,10 @@ //! Plain encoding works with fixed stride types, i.e., `boolean`, `i8...i64`, `f16...f64`, //! it stores the array directly in the file. It offers O(1) read access. -use std::ops::{Range, RangeFrom, RangeFull, RangeTo}; +use std::ops::Range; use std::slice::from_raw_parts; use std::sync::Arc; -use crate::{ - ReadBatchParams, - traits::{Reader, Writer}, -}; use arrow_arith::numeric::sub; use arrow_array::{ Array, ArrayRef, BooleanArray, FixedSizeBinaryArray, FixedSizeListArray, UInt8Array, @@ -24,15 +20,15 @@ use arrow_data::{ArrayDataBuilder, BufferSpec, layout}; use arrow_schema::{DataType, Field}; use arrow_select::{concat::concat, take::take}; use async_recursion::async_recursion; -use async_trait::async_trait; -use bytes::Bytes; use futures::stream::{self, StreamExt, TryStreamExt}; use lance_arrow::*; use lance_core::{Error, Result}; +use lance_io::{ + ReadBatchParams, + traits::{Reader, Writer}, +}; use tokio::io::AsyncWriteExt; -use crate::encodings::{AsyncIndex, Decoder}; - /// Encoder for plain encoding. /// pub struct PlainEncoder<'a> { @@ -175,49 +171,41 @@ fn get_byte_range(data_type: &DataType, row_range: Range) -> Range } } -pub fn bytes_to_array( +fn bytes_to_array( data_type: &DataType, - bytes: Bytes, + bytes: bytes::Bytes, len: usize, offset: usize, ) -> Result { let layout = layout(data_type); - if layout.buffers.len() != 1 { return Err(Error::internal(format!( - "Can only convert datatypes that require one buffer, found {:?}", - data_type + "v1 plain encoding requires one value buffer, found {data_type}" ))); } - let buf: Buffer = if let BufferSpec::FixedWidth { + let buffer = if let BufferSpec::FixedWidth { byte_width, alignment, } = &layout.buffers[0] { - // this code is taken from - // https://github.com/apache/arrow-rs/blob/master/arrow-data/src/data.rs#L748-L768 - let len_plus_offset = len + offset; - let min_buffer_size = len_plus_offset.saturating_mul(*byte_width); - - // alignment or size isn't right -- just make a copy + let min_buffer_size = (len + offset).saturating_mul(*byte_width); if bytes.len() < min_buffer_size { Buffer::copy_bytes_bytes(bytes, min_buffer_size) } else { Buffer::from_bytes_bytes(bytes, *alignment as u64) } } else { - // cases we don't handle, just copy Buffer::from_slice_ref(bytes) }; - let array_data = ArrayDataBuilder::new(data_type.clone()) + let data = ArrayDataBuilder::new(data_type.clone()) .len(len) .offset(offset) .null_count(0) - .add_buffer(buf) + .add_buffer(buffer) .build()?; - Ok(make_array(array_data)) + Ok(make_array(data)) } impl<'a> PlainDecoder<'a> { @@ -383,13 +371,12 @@ fn make_chunked_requests( chunked_ranges } -#[async_trait] -impl Decoder for PlainDecoder<'_> { - async fn decode(&self) -> Result { +impl PlainDecoder<'_> { + pub async fn decode(&self) -> Result { self.get(0..self.length).await } - async fn take(&self, indices: &UInt32Array) -> Result { + pub async fn take(&self, indices: &UInt32Array) -> Result { if indices.is_empty() { return Ok(new_empty_array(self.data_type)); } @@ -419,23 +406,9 @@ impl Decoder for PlainDecoder<'_> { let references = arrays.iter().map(|a| a.as_ref()).collect::>(); Ok(concat(&references)?) } -} - -#[async_trait] -impl AsyncIndex for PlainDecoder<'_> { - // TODO: should this return a Scalar value? - type Output = Result; - - async fn get(&self, index: usize) -> Self::Output { - self.get(index..index + 1).await - } -} - -#[async_trait] -impl AsyncIndex> for PlainDecoder<'_> { - type Output = Result; - async fn get(&self, index: Range) -> Self::Output { + #[async_recursion] + async fn decode_range(&self, index: Range) -> Result { if index.is_empty() { return Ok(new_empty_array(self.data_type)); } @@ -451,47 +424,16 @@ impl AsyncIndex> for PlainDecoder<'_> { _ => self.decode_primitive(index.start, index.end).await, } } -} - -#[async_trait] -impl AsyncIndex> for PlainDecoder<'_> { - type Output = Result; - - async fn get(&self, index: RangeFrom) -> Self::Output { - self.get(index.start..self.length).await - } -} - -#[async_trait] -impl AsyncIndex> for PlainDecoder<'_> { - type Output = Result; - - async fn get(&self, index: RangeTo) -> Self::Output { - self.get(0..index.end).await - } -} - -#[async_trait] -impl AsyncIndex for PlainDecoder<'_> { - type Output = Result; - - async fn get(&self, _: RangeFull) -> Self::Output { - self.get(0..self.length).await - } -} -#[async_trait] -impl AsyncIndex for PlainDecoder<'_> { - type Output = Result; - - async fn get(&self, params: ReadBatchParams) -> Self::Output { - match params { - ReadBatchParams::Range(r) => self.get(r).await, - // Ranges not supported in v1 files - ReadBatchParams::Ranges(_) => unimplemented!(), - ReadBatchParams::RangeFull => self.get(..).await, - ReadBatchParams::RangeTo(r) => self.get(r).await, - ReadBatchParams::RangeFrom(r) => self.get(r).await, + pub async fn get(&self, params: impl Into) -> Result { + match params.into() { + ReadBatchParams::Range(range) => self.decode_range(range).await, + ReadBatchParams::Ranges(_) => Err(Error::invalid_input( + "multiple ranges are not supported by v1 plain encoding", + )), + ReadBatchParams::RangeFull => self.decode_range(0..self.length).await, + ReadBatchParams::RangeTo(range) => self.decode_range(0..range.end).await, + ReadBatchParams::RangeFrom(range) => self.decode_range(range.start..self.length).await, ReadBatchParams::Indices(indices) => self.take(&indices).await, } } @@ -502,11 +444,14 @@ mod tests { use std::ops::Deref; use arrow_array::*; + use arrow_buffer::Buffer; + use arrow_data::ArrayDataBuilder; + use bytes::Bytes; use lance_core::utils::tempfile::TempStdFile; + use lance_io::local::LocalObjectReader; use rand::prelude::*; use super::*; - use crate::local::LocalObjectReader; #[tokio::test] async fn test_encode_decode_primitive_array() { @@ -682,7 +627,7 @@ mod tests { } async fn make_array_(data_type: &DataType, buffer: &Buffer) -> ArrayRef { - make_array( + arrow_array::make_array( ArrayDataBuilder::new(data_type.clone()) .len(126) .add_buffer(buffer.clone()) diff --git a/rust/lance-file/src/previous/format/metadata.rs b/rust/lance-file/src/versions/v1/format/metadata.rs similarity index 98% rename from rust/lance-file/src/previous/format/metadata.rs rename to rust/lance-file/src/versions/v1/format/metadata.rs index 11ba00c3243..209d0733cec 100644 --- a/rust/lance-file/src/previous/format/metadata.rs +++ b/rust/lance-file/src/versions/v1/format/metadata.rs @@ -62,10 +62,10 @@ impl TryFrom for Metadata { manifest_position: Some(m.manifest_position as usize), stats_metadata: if let Some(stats_meta) = m.statistics { Some(StatisticsMetadata { - schema: Schema::from(FieldsWithMeta { + schema: Schema::try_from(FieldsWithMeta { fields: Fields(stats_meta.schema), metadata: Default::default(), - }), + })?, leaf_field_ids: stats_meta.fields, page_table_position: stats_meta.page_table_position as usize, }) diff --git a/rust/lance-file/src/previous/format/mod.rs b/rust/lance-file/src/versions/v1/format/mod.rs similarity index 100% rename from rust/lance-file/src/previous/format/mod.rs rename to rust/lance-file/src/versions/v1/format/mod.rs diff --git a/rust/lance-file/src/versions/v1/mod.rs b/rust/lance-file/src/versions/v1/mod.rs new file mode 100644 index 00000000000..a7da08de11e --- /dev/null +++ b/rust/lance-file/src/versions/v1/mod.rs @@ -0,0 +1,116 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright The Lance Authors + +//! Lance v1 file implementation. +//! +//! This module is the canonical home of the v1 reader, writer, metadata, and +//! page-table grammar. V1 accepts footer versions `(0, 0)` through `(0, 2)` and +//! writes the `(0, 2)` identity used by [`writer::FileWriter`]. + +pub mod encoding; +pub mod format; +pub mod page_table; +pub mod reader; +pub mod writer; + +use std::{collections::BTreeMap, sync::Arc}; + +use lance_core::{ + Result, + datatypes::{Field, Schema}, +}; + +use crate::reader::ReaderProjection; + +fn append_field_ids( + fields: &[Field], + field_id_to_column_index: &BTreeMap, + column_indices: &mut Vec, +) { + for field in fields { + if let Some(column_index) = field_id_to_column_index.get(&(field.id as u32)).copied() { + column_indices.push(column_index); + } + if !field.is_blob() && !field.is_packed_struct() { + append_field_ids(&field.children, field_id_to_column_index, column_indices); + } + } +} + +pub fn projection_from_field_ids( + schema: &Schema, + field_id_to_column_index: &BTreeMap, +) -> ReaderProjection { + let mut column_indices = Vec::new(); + append_field_ids( + &schema.fields, + field_id_to_column_index, + &mut column_indices, + ); + ReaderProjection { + schema: Arc::new(schema.clone()), + column_indices, + } +} + +pub fn projection_from_whole_schema(schema: &Schema) -> ReaderProjection { + projection_from_field_ids(schema, &field_id_to_column_index(schema)) +} + +pub fn projection_from_column_names( + schema: &Schema, + column_names: &[&str], +) -> Result { + let field_id_to_column_index = field_id_to_column_index(schema); + let projected = schema.project(column_names)?; + Ok(projection_from_field_ids( + &projected, + &field_id_to_column_index, + )) +} + +/// Count physical columns represented by a field in a v1 footer. +pub fn physical_column_count(field: &Field) -> usize { + if field.is_blob() || field.is_packed_struct() { + 1 + } else { + 1 + field + .children + .iter() + .map(physical_column_count) + .sum::() + } +} + +/// Build persisted field-to-column entries for a v1 data file. +pub fn data_file_columns(schema: &Schema) -> (Vec, Vec) { + let mut field_ids = Vec::new(); + let mut column_indices = Vec::new(); + append_physical_fields(&schema.fields, &mut field_ids, &mut column_indices, &mut 0); + (field_ids, column_indices) +} + +fn field_id_to_column_index(schema: &Schema) -> BTreeMap { + let (field_ids, column_indices) = data_file_columns(schema); + field_ids + .into_iter() + .zip(column_indices) + .map(|(field_id, column_index)| (field_id as u32, column_index as u32)) + .collect() +} + +fn append_physical_fields( + fields: &[Field], + field_ids: &mut Vec, + column_indices: &mut Vec, + next_column: &mut i32, +) { + for field in fields { + field_ids.push(field.id); + column_indices.push(*next_column); + *next_column += 1; + if !field.is_blob() && !field.is_packed_struct() { + append_physical_fields(&field.children, field_ids, column_indices, next_column); + } + } +} diff --git a/rust/lance-file/src/previous/page_table.rs b/rust/lance-file/src/versions/v1/page_table.rs similarity index 99% rename from rust/lance-file/src/previous/page_table.rs rename to rust/lance-file/src/versions/v1/page_table.rs index cc246caa585..b2ff750c13a 100644 --- a/rust/lance-file/src/previous/page_table.rs +++ b/rust/lance-file/src/versions/v1/page_table.rs @@ -5,14 +5,14 @@ use arrow_array::builder::Int64Builder; use arrow_array::{Array, Int64Array}; use arrow_schema::DataType; use lance_core::deepsize::DeepSizeOf; -use lance_io::encodings::Decoder; -use lance_io::encodings::plain::PlainDecoder; use std::collections::BTreeMap; use tokio::io::AsyncWriteExt; use lance_core::{Error, Result}; use lance_io::traits::{Reader, Writer}; +use super::encoding::plain::PlainDecoder; + #[derive(Clone, Debug, PartialEq, DeepSizeOf)] pub struct PageInfo { pub position: usize, diff --git a/rust/lance-file/src/previous/reader.rs b/rust/lance-file/src/versions/v1/reader.rs similarity index 93% rename from rust/lance-file/src/previous/reader.rs rename to rust/lance-file/src/versions/v1/reader.rs index 1ab861985e1..0c1d490d5a6 100644 --- a/rust/lance-file/src/previous/reader.rs +++ b/rust/lance-file/src/versions/v1/reader.rs @@ -21,25 +21,22 @@ use arrow_select::concat::{self, concat_batches}; use async_recursion::async_recursion; use futures::{Future, FutureExt, StreamExt, TryStreamExt, stream}; use lance_arrow::*; -use lance_core::cache::{CacheKey, LanceCache}; +use lance_core::cache::{CacheKey, CacheKeySchema, KeyBuilder, LanceCache}; use lance_core::datatypes::{Field, Schema}; use lance_core::deepsize::DeepSizeOf; use lance_core::{Error, Result}; -use lance_io::encodings::AsyncIndex; -use lance_io::encodings::dictionary::DictionaryDecoder; use lance_io::stream::{RecordBatchStream, RecordBatchStreamAdapter}; use lance_io::traits::Reader; -use lance_io::utils::{ - read_fixed_stride_array, read_metadata_offset, read_struct, read_struct_from_buf, -}; +use lance_io::utils::{read_metadata_offset, read_struct, read_struct_from_buf}; use lance_io::{ReadBatchParams, object_store::ObjectStore}; use std::borrow::Cow; use object_store::path::Path; use tracing::instrument; -use crate::previous::format::metadata::Metadata; -use crate::previous::page_table::{PageInfo, PageTable}; +use crate::versions::v1::encoding::{dictionary::DictionaryDecoder, read_fixed_stride_array}; +use crate::versions::v1::format::metadata::Metadata; +use crate::versions::v1::page_table::{PageInfo, PageTable}; /// Lance File Reader. /// @@ -83,7 +80,23 @@ impl<'a, T> StringCacheKey<'a, T> { } } -impl CacheKey for StringCacheKey<'_, T> { +trait StableStringCacheValue: 'static { + const STABLE_TYPE_ID: &'static str; +} + +impl StableStringCacheValue for Metadata { + const STABLE_TYPE_ID: &'static str = "lance.file.previous.Metadata"; +} + +impl StableStringCacheValue for PageTable { + const STABLE_TYPE_ID: &'static str = "lance.file.previous.PageTable"; +} + +impl StableStringCacheValue for Option { + const STABLE_TYPE_ID: &'static str = "lance.file.previous.OptionalPageTable"; +} + +impl CacheKey for StringCacheKey<'_, T> { type ValueType = T; fn key(&self) -> Cow<'_, str> { @@ -91,11 +104,22 @@ impl CacheKey for StringCacheKey<'_, T> { } fn type_name() -> &'static str { - // This is a private, crate-internal key that is only instantiated with - // a single concrete T within one build, so std::any::type_name is fine - // here — there is no cross-crate collision risk. + // Keep the legacy diagnostic name for compatibility. `stable_type_id` + // provides the compiler-independent identity used by physical keys. std::any::type_name::() } + + fn stable_type_id() -> &'static str { + T::STABLE_TYPE_ID + } + + fn schema() -> CacheKeySchema { + CacheKeySchema::new("lance.file.previous.string-key", 1) + } + + fn write_key(&self, builder: &mut KeyBuilder) { + builder.write_str(self.key); + } } impl FileReader { @@ -244,7 +268,7 @@ impl FileReader { } /// Load some metadata about the fragment from the cache, if there is one. - async fn load_from_cache( + async fn load_from_cache( cache: Option<&LanceCache>, key: String, loader: F, @@ -607,7 +631,7 @@ async fn read_binary_array( ) -> Result { let page_info = get_page_info(page_table, field, batch_id)?; - lance_io::utils::read_binary_array( + crate::versions::v1::encoding::read_binary_array( reader.object_reader.as_ref(), &field.data_type(), field.nullable, @@ -785,7 +809,7 @@ where #[cfg(test)] mod tests { - use crate::previous::writer::{FileWriter as PreviousFileWriter, NotSelfDescribing}; + use crate::versions::v1::writer::{FileWriter as V1FileWriter, NotSelfDescribing}; use super::*; @@ -800,6 +824,34 @@ mod tests { use arrow_schema::{Field as ArrowField, Fields as ArrowFields, Schema as ArrowSchema}; use lance_io::object_store::ObjectStoreParams; + #[test] + fn string_cache_key_discriminators_are_stable_and_type_scoped() { + assert_eq!( + [ + as CacheKey>::stable_type_id(), + as CacheKey>::stable_type_id(), + > as CacheKey>::stable_type_id(), + ], + [ + "lance.file.previous.Metadata", + "lance.file.previous.PageTable", + "lance.file.previous.OptionalPageTable", + ] + ); + assert_eq!( + [ + as CacheKey>::type_name(), + as CacheKey>::type_name(), + > as CacheKey>::type_name(), + ], + [ + std::any::type_name::(), + std::any::type_name::(), + std::any::type_name::>(), + ] + ); + } + #[tokio::test] async fn test_take() { let arrow_schema = ArrowSchema::new(vec![ @@ -840,7 +892,7 @@ mod tests { } schema.set_dictionary(&batches[0]).unwrap(); - let mut file_writer = PreviousFileWriter::::try_new( + let mut file_writer = V1FileWriter::::try_new( &store, &path, schema.clone(), @@ -904,7 +956,7 @@ mod tests { )])); let batch = RecordBatch::try_new(arrow_schema.clone(), vec![struct_arr]).unwrap(); - let mut file_writer = PreviousFileWriter::::try_new( + let mut file_writer = V1FileWriter::::try_new( &store, &path, schema.clone(), @@ -958,7 +1010,7 @@ mod tests { .collect::>(); let batches_ref = batches.iter().collect::>(); - let mut file_writer = PreviousFileWriter::::try_new( + let mut file_writer = V1FileWriter::::try_new( &store, &path, schema.clone(), @@ -985,7 +1037,7 @@ mod tests { let schema: Schema = Schema::try_from(arrow_schema.as_ref()).unwrap(); let batch = RecordBatch::try_new(arrow_schema.clone(), vec![struct_array.clone()]).unwrap(); - let mut file_writer = PreviousFileWriter::::try_new( + let mut file_writer = V1FileWriter::::try_new( &store, &path, schema.clone(), @@ -1110,7 +1162,7 @@ mod tests { // write to a lance file let store = ObjectStore::memory(); let path = Path::from("/takes"); - let mut file_writer = PreviousFileWriter::::try_new( + let mut file_writer = V1FileWriter::::try_new( &store, &path, schema.clone(), @@ -1227,7 +1279,7 @@ mod tests { let store = ObjectStore::memory(); let path = Path::from("/take_list"); let schema: Schema = (&arrow_schema).try_into().unwrap(); - let mut file_writer = PreviousFileWriter::::try_new( + let mut file_writer = V1FileWriter::::try_new( &store, &path, schema.clone(), @@ -1304,7 +1356,7 @@ mod tests { .unwrap(); let schema: Schema = (&arrow_schema).try_into().unwrap(); - let mut file_writer = PreviousFileWriter::::try_new( + let mut file_writer = V1FileWriter::::try_new( &store, &path, schema.clone(), @@ -1338,7 +1390,7 @@ mod tests { // write to a lance file let store = ObjectStore::memory(); let path = Path::from("/read_range"); - let mut file_writer = PreviousFileWriter::::try_new( + let mut file_writer = V1FileWriter::::try_new( &store, &path, schema.clone(), @@ -1365,7 +1417,7 @@ mod tests { let arrow_schema = ArrowSchema::new(vec![ArrowField::new("i", DataType::Int32, true)]); let schema = Schema::try_from(&arrow_schema).unwrap(); - let mut writer = PreviousFileWriter::::try_new( + let mut writer = V1FileWriter::::try_new( &store, &path, schema.clone(), @@ -1425,7 +1477,7 @@ mod tests { false, )])); let schema = Schema::try_from(arrow_schema.as_ref()).unwrap(); - let mut file_writer = PreviousFileWriter::::try_new( + let mut file_writer = V1FileWriter::::try_new( &store, &path, schema.clone(), @@ -1470,7 +1522,7 @@ mod tests { let partial_schema = schema.project(&["f50"]).unwrap(); let partial_arrow: ArrowSchema = (&partial_schema).into(); - let mut file_writer = PreviousFileWriter::::try_new( + let mut file_writer = V1FileWriter::::try_new( &store, &path, partial_schema.clone(), diff --git a/rust/lance-file/src/previous/writer/mod.rs b/rust/lance-file/src/versions/v1/writer/mod.rs similarity index 95% rename from rust/lance-file/src/previous/writer/mod.rs rename to rust/lance-file/src/versions/v1/writer/mod.rs index ab13e782367..e3188e0d6a7 100644 --- a/rust/lance-file/src/previous/writer/mod.rs +++ b/rust/lance-file/src/versions/v1/writer/mod.rs @@ -18,17 +18,18 @@ use async_trait::async_trait; use lance_arrow::*; use lance_core::datatypes::{Encoding, Field, NullabilityComparison, Schema, SchemaCompareOptions}; use lance_core::{Error, Result}; -use lance_io::encodings::{ - Encoder, binary::BinaryEncoder, dictionary::DictionaryEncoder, plain::PlainEncoder, -}; use lance_io::object_store::ObjectStore; use lance_io::traits::{WriteExt, Writer}; use object_store::path::Path; use tokio::io::AsyncWriteExt; use crate::format::{MAGIC, MAJOR_VERSION, MINOR_VERSION}; -use crate::previous::format::metadata::{Metadata, StatisticsMetadata}; -use crate::previous::page_table::{PageInfo, PageTable}; +use crate::versions::v1::encoding::{ + binary::BinaryEncoder, dictionary::DictionaryEncoder, plain::PlainEncoder, + write_schema_dictionaries, +}; +use crate::versions::v1::format::metadata::{Metadata, StatisticsMetadata}; +use crate::versions::v1::page_table::{PageInfo, PageTable}; use crate::writer::FileWriteSummary; /// The file format currently includes a "manifest" where it stores the schema for @@ -623,53 +624,6 @@ impl FileWriter { } } - /// Writes the dictionaries (using plain/binary encoding) into the file - /// - /// The offsets and lengths of the written buffers are stored in the given - /// schema so that the dictionaries can be loaded in the future. - async fn write_dictionaries(writer: &mut dyn Writer, schema: &mut Schema) -> Result<()> { - // Write dictionary values. - let max_field_id = schema.max_field_id().unwrap_or(-1); - for field_id in 0..max_field_id + 1 { - if let Some(field) = schema.mut_field_by_id(field_id) - && field.data_type().is_dictionary() - { - let dict_info = field.dictionary.as_mut().ok_or_else(|| { - // and wrap it in here. - Error::io(format!("Lance field {} misses dictionary info", field.name)) - })?; - - let value_arr = dict_info.values.as_ref().ok_or_else(|| { - Error::invalid_input(format!( - "Lance field {} is dictionary type, but misses the dictionary value array", - field.name - )) - })?; - - let data_type = value_arr.data_type(); - let pos = match data_type { - dt if dt.is_numeric() => { - let mut encoder = PlainEncoder::new(writer, dt); - encoder.encode(&[value_arr]).await? - } - dt if dt.is_binary_like() => { - let mut encoder = BinaryEncoder::new(writer); - encoder.encode(&[value_arr]).await? - } - _ => { - return Err(Error::schema(format!( - "Does not support {} as dictionary value type", - value_arr.data_type() - ))); - } - }; - dict_info.offset = pos; - dict_info.length = value_arr.len(); - } - } - Ok(()) - } - async fn write_footer(&mut self) -> Result<()> { // Step 1. Write page table. let field_id_offset = *self.schema.field_ids().iter().min().unwrap(); @@ -683,7 +637,7 @@ impl FileWriter { self.metadata.stats_metadata = self.write_statistics().await?; // Step 3. Write manifest and dictionary values. - Self::write_dictionaries(self.object_writer.as_mut(), &mut self.schema).await?; + write_schema_dictionaries(self.object_writer.as_mut(), &mut self.schema).await?; let pos = M::store_schema(self.object_writer.as_mut(), &self.schema).await?; // Step 4. Write metadata. @@ -756,7 +710,7 @@ mod tests { }; use arrow_select::concat::concat_batches; - use crate::previous::reader::FileReader; + use crate::versions::v1::reader::FileReader; #[tokio::test] async fn test_write_file() { @@ -1028,6 +982,37 @@ mod tests { assert_eq!(actual, batch); } + #[tokio::test] + async fn test_write_empty_non_nullable_string() { + let arrow_schema = Arc::new(ArrowSchema::new(vec![ArrowField::new( + "value", + DataType::Utf8, + false, + )])); + let schema = Schema::try_from(arrow_schema.as_ref()).unwrap(); + let store = ObjectStore::memory(); + let path = Path::from("/empty"); + let mut file_writer = FileWriter::::try_new( + &store, + &path, + schema.clone(), + &Default::default(), + ) + .await + .unwrap(); + + file_writer + .write(&[RecordBatch::new_empty(arrow_schema)]) + .await + .unwrap(); + let summary = file_writer.finish().await.unwrap(); + assert_eq!(summary.num_rows, 0); + + let reader = FileReader::try_new(&store, &path, schema).await.unwrap(); + let actual = reader.read_batch(0, .., reader.schema()).await.unwrap(); + assert_eq!(actual.num_rows(), 0); + } + #[tokio::test] async fn test_collect_stats() { // Validate: diff --git a/rust/lance-file/src/previous/writer/statistics.rs b/rust/lance-file/src/versions/v1/writer/statistics.rs similarity index 99% rename from rust/lance-file/src/previous/writer/statistics.rs rename to rust/lance-file/src/versions/v1/writer/statistics.rs index 1ccc38ca43d..7eeb3db12bc 100644 --- a/rust/lance-file/src/previous/writer/statistics.rs +++ b/rust/lance-file/src/versions/v1/writer/statistics.rs @@ -642,8 +642,10 @@ impl StatisticsCollector { let max_value = Arc::new(builder.max_value.finish()); let struct_fields = vec![ ArrowField::new("null_count", DataType::Int64, false), - ArrowField::new("min_value", field.data_type(), field.nullable), - ArrowField::new("max_value", field.data_type(), field.nullable), + // Bounds can be absent for empty pages regardless of the data field's + // nullability. + ArrowField::new("min_value", field.data_type(), true), + ArrowField::new("max_value", field.data_type(), true), ]; let stats = StructArray::new( diff --git a/rust/lance-file/src/versions/v2_0/mod.rs b/rust/lance-file/src/versions/v2_0/mod.rs new file mode 100644 index 00000000000..83efef58289 --- /dev/null +++ b/rust/lance-file/src/versions/v2_0/mod.rs @@ -0,0 +1,164 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright The Lance Authors + +//! Lance v2.0 file composition. + +use std::{collections::BTreeMap, sync::Arc}; + +use bytes::Bytes; +use lance_core::{ + Result, + datatypes::{Field, Schema}, +}; +use lance_encoding::{ + decoder::PageInfo, + encoder::{ArrayFieldEncodingStrategy, EncodedBatch, FieldEncodingStrategy}, +}; +use lance_io::traits::Writer as ObjectWriter; + +use crate::{reader::ReadProjection, writer::FileWriterOptions}; + +mod reader; +mod writer; + +#[cfg(test)] +pub(crate) use reader::test_projection_length; +pub(crate) use reader::{ + decode_column_metadata, finish_metadata, finish_metadata_index, validate_global_buffers, +}; +pub use reader::{ + projection_from_column_names, projection_from_field_ids, projection_from_whole_schema, +}; +pub use writer::Writer; + +pub(crate) fn read_projection() -> Arc { + reader::read_projection() +} + +/// Count physical columns represented by a field in a v2.0 footer. +pub fn physical_column_count(field: &Field) -> usize { + if field.is_blob() || field.is_packed_struct() { + 1 + } else { + 1 + field + .children + .iter() + .map(physical_column_count) + .sum::() + } +} + +/// Build persisted field-to-column entries for a v2.0 data file. +pub fn data_file_columns(schema: &Schema) -> (Vec, Vec) { + let mut field_ids = Vec::new(); + let mut column_indices = Vec::new(); + append_physical_fields(&schema.fields, &mut field_ids, &mut column_indices, &mut 0); + (field_ids, column_indices) +} + +pub(super) fn field_id_to_column_index(schema: &Schema) -> BTreeMap { + let (field_ids, column_indices) = data_file_columns(schema); + field_ids + .into_iter() + .zip(column_indices) + .map(|(field_id, column_index)| (field_id as u32, column_index as u32)) + .collect() +} + +fn append_physical_fields( + fields: &[Field], + field_ids: &mut Vec, + column_indices: &mut Vec, + next_column: &mut i32, +) { + for field in fields { + field_ids.push(field.id); + column_indices.push(*next_column); + *next_column += 1; + if !field.is_blob() && !field.is_packed_struct() { + append_physical_fields(&field.children, field_ids, column_indices, next_column); + } + } +} + +fn is_external_metadata_structural_header( + fields: &[Field], + target_column: usize, + next_column: &mut usize, +) -> Option { + for field in fields { + if *next_column == target_column { + return Some(field.logical_type.is_struct() && !field.is_packed_struct()); + } + *next_column += 1; + if !field.is_blob() + && !field.is_packed_struct() + && let Some(is_header) = + is_external_metadata_structural_header(&field.children, target_column, next_column) + { + return Some(is_header); + } + } + None +} + +pub(super) fn should_copy_external_metadata_column( + schema: &Schema, + column_index: usize, + has_existing_pages: bool, +) -> bool { + let mut next_column = 0; + let is_header = + is_external_metadata_structural_header(&schema.fields, column_index, &mut next_column) + .unwrap_or(false); + !is_header || !has_existing_pages +} + +pub(super) fn finalize_external_metadata_column( + schema: &Schema, + column_index: usize, + pages: &mut Vec, + num_rows: u64, +) { + let mut next_column = 0; + let is_header = + is_external_metadata_structural_header(&schema.fields, column_index, &mut next_column) + .unwrap_or(false); + if is_header && !pages.is_empty() { + pages[0].num_rows = num_rows; + pages[0].priority = 0; + pages.truncate(1); + } +} + +/// Compose the v2.0 field encoding mechanisms. +pub fn encoding_strategy() -> Arc { + Arc::new(ArrayFieldEncodingStrategy::new()) +} + +/// Create a v2.0 writer with an explicit schema. +pub fn create_writer( + object_writer: Box, + schema: Schema, + options: FileWriterOptions, +) -> Result { + Writer::try_new(object_writer, schema, options) +} + +/// Create a v2.0 writer whose schema is inferred from the first batch. +pub fn create_lazy_writer( + object_writer: Box, + options: FileWriterOptions, +) -> Writer { + Writer::new_lazy(object_writer, options) +} + +/// Encode a self-described v2.0 batch. +pub fn encode_self_described_batch(batch: &EncodedBatch) -> Result { + writer::concat_lance_footer(batch, true) +} + +/// Encode a mini-lance v2.0 batch. +pub fn encode_mini_batch(batch: &EncodedBatch) -> Result { + writer::concat_lance_footer(batch, false) +} diff --git a/rust/lance-file/src/versions/v2_0/reader.rs b/rust/lance-file/src/versions/v2_0/reader.rs new file mode 100644 index 00000000000..f54c972f43b --- /dev/null +++ b/rust/lance-file/src/versions/v2_0/reader.rs @@ -0,0 +1,415 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright The Lance Authors + +use std::{collections::BTreeMap, sync::Arc}; + +use async_trait::async_trait; +use bytes::Bytes; +use lance_core::{ + Error, Result, + cache::LanceCache, + datatypes::{Field, Schema}, +}; +use lance_encoding::{ + EncodingsIo, + decoder::{ColumnInfo, PageEncoding, PageInfo}, + format::pb, +}; +use prost::{Message, Name}; + +use crate::{ + format::pbfile, + reader::{ + BufferDescriptor, CachedFileMetadata, FileMetadataIndex, FileMetadataProvider, FileReader, + PreparedProjection, RawFileMetadata, ReadProjection, ReaderProjection, + normalized_column_num_rows, verify_uniform_lengths, + }, + version::ConcreteFileVersion, +}; + +fn fetch_encoding(encoding: &pbfile::Encoding) -> Result { + match &encoding.location { + Some(pbfile::encoding::Location::Indirect(_)) => Err(Error::invalid_input_source( + "Indirect file encodings are not supported".into(), + )), + Some(pbfile::encoding::Location::Direct(encoding)) => { + let envelope = prost_types::Any::decode(Bytes::from(encoding.encoding.clone())) + .map_err(|error| { + Error::invalid_input_source( + format!("Invalid direct {} encoding envelope: {error}", M::NAME).into(), + ) + })?; + envelope.to_msg::().map_err(|error| { + Error::invalid_input_source( + format!("Invalid direct {} encoding: {error}", M::NAME).into(), + ) + }) + } + Some(pbfile::encoding::Location::None(_)) => Err(Error::invalid_input_source( + format!("Missing {} encoding description", M::NAME).into(), + )), + None => Err(Error::invalid_input_source( + format!("Missing {} encoding location", M::NAME).into(), + )), + } +} + +pub fn decode_column( + column_index: u32, + metadata: &pbfile::ColumnMetadata, +) -> Result> { + let page_infos = metadata + .pages + .iter() + .enumerate() + .map(|(page_index, page)| { + let array_encoding = + fetch_encoding::(page.encoding.as_ref().ok_or_else(|| { + Error::invalid_input_source( + format!( + "Column {} page {} is missing its encoding", + column_index, page_index + ) + .into(), + ) + })?)?; + if page.buffer_offsets.len() != page.buffer_sizes.len() { + return Err(Error::invalid_input_source( + format!( + "Column {} page {} has {} buffer offsets but {} buffer sizes", + column_index, + page_index, + page.buffer_offsets.len(), + page.buffer_sizes.len() + ) + .into(), + )); + } + let buffer_offsets_and_sizes = Arc::from( + page.buffer_offsets + .iter() + .zip(&page.buffer_sizes) + .map(|(offset, size)| (*offset, *size)) + .collect::>(), + ); + Ok(PageInfo { + buffer_offsets_and_sizes, + encoding: PageEncoding::Legacy(array_encoding), + num_rows: page.length, + priority: page.priority, + }) + }) + .collect::>>()?; + + if metadata.buffer_offsets.len() != metadata.buffer_sizes.len() { + return Err(Error::invalid_input_source( + format!( + "Column {} has {} buffer offsets but {} buffer sizes", + column_index, + metadata.buffer_offsets.len(), + metadata.buffer_sizes.len() + ) + .into(), + )); + } + let buffer_offsets_and_sizes = Arc::from( + metadata + .buffer_offsets + .iter() + .zip(&metadata.buffer_sizes) + .map(|(offset, size)| (*offset, *size)) + .collect::>(), + ); + Ok(Arc::new(ColumnInfo { + index: column_index, + page_infos: Arc::from(page_infos), + buffer_offsets_and_sizes, + encoding: fetch_encoding(metadata.encoding.as_ref().ok_or_else(|| { + Error::invalid_input_source( + format!("Column {} is missing its encoding", column_index).into(), + ) + })?)?, + })) +} + +pub fn decode_column_metadata( + column_metadatas: &[pbfile::ColumnMetadata], +) -> Result>> { + column_metadatas + .iter() + .enumerate() + .map(|(column_index, metadata)| { + let column_index = u32::try_from(column_index).map_err(|_| { + Error::invalid_input_source("File has more than u32::MAX columns".into()) + })?; + decode_column(column_index, metadata) + }) + .collect() +} + +pub async fn prepare_projection( + metadata_provider: &FileMetadataProvider, + projection: &ReaderProjection, + _io: &Arc, + _cache: &Arc, +) -> Result { + match metadata_provider { + FileMetadataProvider::Full(metadata) => { + FileReader::validate_projection(projection, metadata)?; + Ok(PreparedProjection { + column_infos: metadata.column_infos.clone(), + decoder_projection: projection.clone(), + }) + } + FileMetadataProvider::Indexed(metadata_index) => { + FileMetadataProvider::validate_indexed_projection_structure( + projection, + metadata_index, + )?; + Err(FileMetadataProvider::indexed_projection_error( + projection, + metadata_index, + )) + } + } +} + +fn field_column_shape(field: &Field) -> (bool, bool) { + if field.is_blob() || field.is_packed_struct() { + return (true, false); + } + (true, !field.children.is_empty()) +} + +fn append_field_ids( + fields: &[Field], + field_id_to_column_index: &BTreeMap, + column_indices: &mut Vec, +) { + for field in fields { + let (contributes, recurse) = field_column_shape(field); + if contributes + && let Some(column_index) = field_id_to_column_index.get(&(field.id as u32)).copied() + { + column_indices.push(column_index); + } + if recurse { + append_field_ids(&field.children, field_id_to_column_index, column_indices); + } + } +} + +pub fn projection_from_field_ids( + schema: &Schema, + field_id_to_column_index: &BTreeMap, +) -> ReaderProjection { + let mut column_indices = Vec::new(); + append_field_ids( + &schema.fields, + field_id_to_column_index, + &mut column_indices, + ); + ReaderProjection { + schema: Arc::new(schema.clone()), + column_indices, + } +} + +pub fn projection_from_whole_schema(schema: &Schema) -> ReaderProjection { + projection_from_field_ids(schema, &super::field_id_to_column_index(schema)) +} + +pub fn projection_from_column_names( + schema: &Schema, + column_names: &[&str], +) -> Result { + let field_id_to_column_index = super::field_id_to_column_index(schema); + let projected = schema.project(column_names)?; + Ok(projection_from_field_ids( + &projected, + &field_id_to_column_index, + )) +} + +fn children_share_parent_length(field: &Field) -> bool { + field.logical_type.is_struct() +} + +fn validate_field_length Result>( + field: &Field, + comparable: bool, + column_indices: &[u32], + cursor: &mut usize, + column_len: &F, +) -> Result { + let (contributes, recurse) = field_column_shape(field); + let mut field_rows = None; + if contributes { + let column = *column_indices.get(*cursor).ok_or_else(|| { + Error::invalid_input(format!( + "projection supplied fewer column indices than its fields require (ran out at field '{}')", + field.name + )) + })?; + *cursor += 1; + field_rows = Some(column_len(column as usize)?); + } + if recurse { + let enforce_children = comparable && children_share_parent_length(field); + for child in &field.children { + let child_rows = + validate_field_length(child, enforce_children, column_indices, cursor, column_len)?; + let expected = *field_rows.get_or_insert(child_rows); + if enforce_children && child_rows != expected { + return Err(Error::invalid_input(format!( + "cannot read field '{}': its children have differing lengths (child '{}' has {} rows, but the field has {}); a struct's children must all have the same length", + field.name, child.name, child_rows, expected + ))); + } + } + } + field_rows.ok_or_else(|| { + Error::invalid_input(format!( + "projected field '{}' maps to no columns", + field.name + )) + }) +} + +pub fn prepared_read_length(prepared: &PreparedProjection) -> Result { + let column_len = |column: usize| { + let info = prepared.column_infos.get(column).ok_or_else(|| { + Error::invalid_input(format!( + "projection references column index {} but only {} columns are available", + column, + prepared.column_infos.len() + )) + })?; + normalized_column_num_rows(info) + }; + let mut cursor = 0; + let mut field_lengths = Vec::with_capacity(prepared.decoder_projection.schema.fields.len()); + for field in &prepared.decoder_projection.schema.fields { + let rows = validate_field_length( + field, + true, + &prepared.decoder_projection.column_indices, + &mut cursor, + &column_len, + )?; + field_lengths.push((field.name.as_str(), rows)); + } + if cursor != prepared.decoder_projection.column_indices.len() { + return Err(Error::invalid_input(format!( + "projection supplied {} column indices but its fields require {}", + prepared.decoder_projection.column_indices.len(), + cursor + ))); + } + verify_uniform_lengths(&field_lengths) +} + +#[derive(Debug)] +struct V20ReadProjection; + +pub(super) fn read_projection() -> Arc { + Arc::new(V20ReadProjection) +} + +#[async_trait] +impl ReadProjection for V20ReadProjection { + fn validate_indexed( + &self, + projection: &ReaderProjection, + metadata_index: &FileMetadataIndex, + ) -> Result<()> { + FileMetadataProvider::validate_indexed_projection_structure(projection, metadata_index)?; + Err(FileMetadataProvider::indexed_projection_error( + projection, + metadata_index, + )) + } + + fn read_length(&self, prepared: &PreparedProjection) -> Result { + prepared_read_length(prepared) + } + + async fn prepare( + &self, + metadata_provider: &FileMetadataProvider, + projection: &ReaderProjection, + io: &Arc, + cache: &Arc, + ) -> Result<(PreparedProjection, u64)> { + let prepared = prepare_projection(metadata_provider, projection, io, cache).await?; + let read_len = self.read_length(&prepared)?; + Ok((prepared, read_len)) + } +} + +pub fn finish_metadata(raw: RawFileMetadata) -> Result { + if !matches!( + (raw.footer.major_version, raw.footer.minor_version), + (0, 3) | (2, 0) + ) { + return Err(Error::version_conflict( + "Attempt to use the Lance v2.0 reader for a different file version".to_string(), + raw.footer.major_version, + raw.footer.minor_version, + )); + } + let column_infos = decode_column_metadata(&raw.column_metadatas)?; + Ok(CachedFileMetadata { + file_schema: raw.file_schema, + column_metadatas: raw.column_metadatas, + column_infos, + num_rows: raw.num_rows, + file_buffers: raw.file_buffers, + num_data_bytes: raw.num_data_bytes, + num_column_metadata_bytes: raw.num_column_metadata_bytes, + num_global_buffer_bytes: raw.num_global_buffer_bytes, + num_footer_bytes: raw.num_footer_bytes, + major_version: raw.footer.major_version, + minor_version: raw.footer.minor_version, + version: ConcreteFileVersion::V2_0, + file_size_bytes: raw.file_size_bytes, + retained_global_buffers: raw.retained_global_buffers, + }) +} + +pub fn validate_global_buffers(_buffers: &[BufferDescriptor]) -> Result<()> { + Ok(()) +} + +pub fn finish_metadata_index(index: FileMetadataIndex) -> Result { + if index.version == ConcreteFileVersion::V2_0 { + Ok(index) + } else { + let (major, minor) = index.version.to_standard_footer_numbers(); + Err(Error::version_conflict( + "Attempt to use the Lance v2.0 reader for a different metadata index".to_string(), + major, + minor, + )) + } +} + +#[cfg(test)] +pub fn test_projection_length( + schema: &Schema, + column_indices: &[u32], + column_lengths: &[u64], +) -> Result { + let column_len = |column: usize| { + column_lengths.get(column).copied().ok_or_else(|| { + Error::invalid_input(format!("missing synthetic length for column {column}")) + }) + }; + let mut cursor = 0; + let mut field_lengths = Vec::with_capacity(schema.fields.len()); + for field in &schema.fields { + let rows = validate_field_length(field, true, column_indices, &mut cursor, &column_len)?; + field_lengths.push((field.name.as_str(), rows)); + } + verify_uniform_lengths(&field_lengths) +} diff --git a/rust/lance-file/src/versions/v2_0/writer.rs b/rust/lance-file/src/versions/v2_0/writer.rs new file mode 100644 index 00000000000..9a6cb7846ec --- /dev/null +++ b/rust/lance-file/src/versions/v2_0/writer.rs @@ -0,0 +1,957 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright The Lance Authors + +use core::panic; +use std::collections::HashMap; +use std::sync::Arc; + +use arrow_array::{ArrayRef, RecordBatch}; + +use arrow_data::ArrayData; +use bytes::{Buf, BufMut, Bytes, BytesMut}; +use futures::StreamExt; +use futures::stream::FuturesOrdered; +use lance_core::datatypes::{Field, Schema as LanceSchema}; +use lance_core::utils::bit::pad_bytes; +use lance_core::{Error, Result}; +use lance_encoding::decoder::PageEncoding; +use lance_encoding::encoder::{ + ArrayFieldEncodingStrategy, BatchEncoder, EncodeTask, EncodedBatch, EncodedPage, + EncodingOptions, FieldEncoder, FieldEncodingStrategy, OutOfLineBuffers, +}; +use lance_encoding::repdef::RepDefBuilder; +use lance_io::object_store::ObjectStore; +use lance_io::traits::Writer as ObjectWriter; +use log::{debug, warn}; +use object_store::path::Path; +use prost::Message; +use prost_types::Any; +use tokio::io::AsyncWrite; +use tokio::io::AsyncWriteExt; +use tracing::instrument; + +use crate::datatypes::FieldsWithMeta; +use crate::format::MAGIC; +use crate::format::pb; +use crate::format::pbfile; +use crate::format::pbfile::DirectEncoding; +use crate::writer::{ + ENV_LANCE_FILE_WRITER_MAX_PAGE_BYTES, FileWriteSummary, FileWriterOptions, + PAGE_BUFFER_ALIGNMENT, +}; + +const PAD_BUFFER: [u8; PAGE_BUFFER_ALIGNMENT] = [72; PAGE_BUFFER_ALIGNMENT]; +// In 2.1+, we split large pages on read instead of write to avoid empty pages +// and small pages issues. However, we keep the write-time limit at 32MB to avoid +// potential regressions in 2.0 format readers. +// +// This limit is not applied in the 2.1 writer +const MAX_PAGE_BYTES: usize = 32 * 1024 * 1024; +// Total in-memory budget for buffering serialized page metadata before flushing +// to the spill file. Divided evenly across columns (with a floor of 64 bytes). +const DEFAULT_SPILL_BUFFER_LIMIT: usize = 256 * 1024; + +/// Spills serialized page metadata to a temporary file to bound memory usage. +/// +/// The spill file is an unstructured sequence of "chunks". Each chunk is a +/// contiguous run of length-delimited protobuf `Page` messages belonging to a +/// single column. Chunks from different columns are interleaved in the order +/// they are flushed (i.e. whenever a column's in-memory buffer exceeds +/// `per_column_limit`). The `column_chunks` index records the (offset, length) +/// of every chunk so each column's pages can be read back and reassembled in +/// order. +struct PageMetadataSpill { + writer: Box, + object_store: Arc, + path: Path, + /// Current write position in the spill file. + position: u64, + /// Per-column buffer of serialized (length-delimited protobuf) page metadata + /// that has not yet been flushed to the spill file. + column_buffers: Vec>, + /// Per-column list of chunks that have been flushed to the spill file. + /// Each entry is (offset, length) pointing into the spill file. + column_chunks: Vec>, + /// Maximum bytes to buffer per column before flushing to the spill file. + per_column_limit: usize, +} + +impl PageMetadataSpill { + async fn new(object_store: Arc, path: Path, num_columns: usize) -> Result { + let writer = object_store.create(&path).await?; + let per_column_limit = (DEFAULT_SPILL_BUFFER_LIMIT / num_columns.max(1)).max(64); + Ok(Self { + writer, + object_store, + path, + position: 0, + column_buffers: vec![Vec::new(); num_columns], + column_chunks: vec![Vec::new(); num_columns], + per_column_limit, + }) + } + + async fn append_page( + &mut self, + column_idx: usize, + page: &pbfile::column_metadata::Page, + ) -> Result<()> { + page.encode_length_delimited(&mut self.column_buffers[column_idx]) + .map_err(|e| { + Error::io_source(Box::new(std::io::Error::new( + std::io::ErrorKind::InvalidData, + e, + ))) + })?; + if self.column_buffers[column_idx].len() >= self.per_column_limit { + self.flush_column(column_idx).await?; + } + Ok(()) + } + + async fn flush_column(&mut self, column_idx: usize) -> Result<()> { + let buf = &self.column_buffers[column_idx]; + if buf.is_empty() { + return Ok(()); + } + let len = buf.len(); + self.writer.write_all(buf).await?; + self.column_chunks[column_idx].push((self.position, len as u32)); + self.position += len as u64; + self.column_buffers[column_idx].clear(); + Ok(()) + } + + async fn shutdown_writer(&mut self) -> Result<()> { + for col_idx in 0..self.column_buffers.len() { + self.flush_column(col_idx).await?; + } + ObjectWriter::shutdown(self.writer.as_mut()).await?; + Ok(()) + } +} + +fn decode_spilled_chunk(data: &Bytes) -> Result> { + let mut pages = Vec::new(); + let mut cursor = data.clone(); + while cursor.has_remaining() { + let page = + pbfile::column_metadata::Page::decode_length_delimited(&mut cursor).map_err(|e| { + Error::io_source(Box::new(std::io::Error::new( + std::io::ErrorKind::InvalidData, + e, + ))) + })?; + pages.push(page); + } + Ok(pages) +} + +enum PageSpillState { + Pending(Arc, Path), + Active(PageMetadataSpill), +} + +/// A writer for the Lance v2.0 file grammar. +pub struct Writer { + writer: Box, + schema: Option, + column_writers: Vec>, + column_metadata: Vec, + field_id_to_column_indices: Vec<(u32, u32)>, + num_columns: u32, + rows_written: u64, + // The number of rows written for each top-level field (i.e. each entry in + // `column_writers`). With `write_batch` every field advances together and + // these are all equal, but `write_column` advances one field at a time, so + // a single file may end up with columns of differing item counts. + field_rows_written: Vec, + global_buffers: Vec<(u64, u64)>, + schema_metadata: HashMap, + encoding_strategy: Box, + options: FileWriterOptions, + page_spill: Option, +} + +fn initial_column_metadata() -> pbfile::ColumnMetadata { + pbfile::ColumnMetadata { + pages: Vec::new(), + buffer_offsets: Vec::new(), + buffer_sizes: Vec::new(), + encoding: None, + } +} + +impl Writer { + /// Create a new v2.0 writer with a desired output schema. + pub fn try_new( + object_writer: Box, + schema: LanceSchema, + options: FileWriterOptions, + ) -> Result { + let mut writer = Self::new_lazy(object_writer, options); + writer.initialize(schema)?; + Ok(writer) + } + + /// Create a new v2.0 writer without a desired output schema. + /// + /// The output schema will be set based on the first batch of data to arrive. + /// If no data arrives and the writer is finished then the write will fail. + pub fn new_lazy(object_writer: Box, options: FileWriterOptions) -> Self { + Self { + writer: object_writer, + schema: None, + column_writers: Vec::new(), + column_metadata: Vec::new(), + num_columns: 0, + rows_written: 0, + field_rows_written: Vec::new(), + field_id_to_column_indices: Vec::new(), + global_buffers: Vec::new(), + schema_metadata: HashMap::new(), + page_spill: None, + encoding_strategy: Box::new(ArrayFieldEncodingStrategy::new()), + options, + } + } + + /// Spill page metadata to a sidecar file instead of accumulating in memory. + /// + /// This can dramatically reduce memory usage when many writers are open + /// concurrently (e.g. IVF shuffle with thousands of partition writers). + /// The sidecar file is created lazily on the first page write. The caller + /// is responsible for cleaning up `path` (e.g. by placing it in a temp + /// directory that is removed via RAII). + pub fn with_page_metadata_spill(mut self, object_store: Arc, path: Path) -> Self { + self.page_spill = Some(PageSpillState::Pending(object_store, path)); + self + } + + async fn do_write_buffer(writer: &mut (impl AsyncWrite + Unpin), buf: &[u8]) -> Result<()> { + writer.write_all(buf).await?; + let pad_bytes = pad_bytes::(buf.len()); + writer.write_all(&PAD_BUFFER[..pad_bytes]).await?; + Ok(()) + } + + async fn write_page(&mut self, encoded_page: EncodedPage) -> Result<()> { + let buffers = encoded_page.data; + let mut buffer_offsets = Vec::with_capacity(buffers.len()); + let mut buffer_sizes = Vec::with_capacity(buffers.len()); + for buffer in buffers { + buffer_offsets.push(self.writer.tell().await? as u64); + buffer_sizes.push(buffer.len() as u64); + Self::do_write_buffer(&mut self.writer, &buffer).await?; + } + let encoded_encoding = match encoded_page.description { + PageEncoding::Legacy(array_encoding) => Any::from_msg(&array_encoding)?.encode_to_vec(), + PageEncoding::Structural(page_layout) => Any::from_msg(&page_layout)?.encode_to_vec(), + }; + let page = pbfile::column_metadata::Page { + buffer_offsets, + buffer_sizes, + encoding: Some(pbfile::Encoding { + location: Some(pbfile::encoding::Location::Direct(DirectEncoding { + encoding: encoded_encoding, + })), + }), + length: encoded_page.num_rows, + priority: encoded_page.row_number, + }; + let col_idx = encoded_page.column_idx as usize; + if matches!(&self.page_spill, Some(PageSpillState::Pending(..))) { + let Some(PageSpillState::Pending(store, path)) = self.page_spill.take() else { + unreachable!() + }; + self.page_spill = Some(PageSpillState::Active( + PageMetadataSpill::new(store, path, self.num_columns as usize).await?, + )); + } + match &mut self.page_spill { + Some(PageSpillState::Active(spill)) => spill.append_page(col_idx, &page).await?, + None => self.column_metadata[col_idx].pages.push(page), + Some(PageSpillState::Pending(..)) => unreachable!(), + } + Ok(()) + } + + #[instrument(skip_all, level = "debug")] + async fn write_pages(&mut self, mut encoding_tasks: FuturesOrdered) -> Result<()> { + // As soon as an encoding task is done we write it. There is no parallelism + // needed here because "writing" is really just submitting the buffer to the + // underlying write scheduler (either the OS or object_store's scheduler for + // cloud writes). The only time we might truly await on write_page is if the + // scheduler's write queue is full. + // + // Also, there is no point in trying to make write_page parallel anyways + // because we wouldn't want buffers getting mixed up across pages. + while let Some(encoding_task) = encoding_tasks.next().await { + let encoded_page = encoding_task?; + self.write_page(encoded_page).await?; + } + // It's important to flush here, we don't know when the next batch will arrive + // and the underlying cloud store could have writes in progress that won't advance + // until we interact with the writer again. These in-progress writes will time out + // if we don't flush. + self.writer.flush().await?; + Ok(()) + } + + /// Schedule batches of data to be written to the file + pub async fn write_batches( + &mut self, + batches: impl Iterator, + ) -> Result<()> { + for batch in batches { + self.write_batch(batch).await?; + } + Ok(()) + } + + fn verify_field_nullability(arr: &ArrayData, field: &Field) -> Result<()> { + if !field.nullable && arr.null_count() > 0 { + return Err(Error::invalid_input(format!( + "The field `{}` contained null values even though the field is marked non-null in the schema", + field.name + ))); + } + + for (child_field, child_arr) in field.children.iter().zip(arr.child_data()) { + Self::verify_field_nullability(child_arr, child_field)?; + } + + Ok(()) + } + + fn verify_nullability_constraints(&self, batch: &RecordBatch) -> Result<()> { + for (col, field) in batch + .columns() + .iter() + .zip(self.schema.as_ref().unwrap().fields.iter()) + { + Self::verify_field_nullability(&col.to_data(), field)?; + } + Ok(()) + } + + fn initialize(&mut self, mut schema: LanceSchema) -> Result<()> { + let cache_bytes_per_column = if let Some(data_cache_bytes) = self.options.data_cache_bytes { + data_cache_bytes / schema.fields.len() as u64 + } else { + 8 * 1024 * 1024 + }; + + let max_page_bytes = self.options.max_page_bytes.unwrap_or_else(|| { + std::env::var(ENV_LANCE_FILE_WRITER_MAX_PAGE_BYTES) + .map(|s| { + s.parse::().unwrap_or_else(|e| { + warn!( + "Failed to parse {}: {}, using default", + ENV_LANCE_FILE_WRITER_MAX_PAGE_BYTES, e + ); + MAX_PAGE_BYTES as u64 + }) + }) + .unwrap_or(MAX_PAGE_BYTES as u64) + }); + + schema.validate()?; + + let keep_original_array = self.options.keep_original_array.unwrap_or(false); + let encoding_options = EncodingOptions { + cache_bytes_per_column, + max_page_bytes, + keep_original_array, + buffer_alignment: PAGE_BUFFER_ALIGNMENT as u64, + }; + let encoder = + BatchEncoder::try_new(&schema, self.encoding_strategy.as_ref(), &encoding_options)?; + self.num_columns = encoder.num_columns(); + + self.field_rows_written = vec![0; encoder.field_encoders.len()]; + self.column_writers = encoder.field_encoders; + self.column_metadata = vec![initial_column_metadata(); self.num_columns as usize]; + self.field_id_to_column_indices = encoder.field_id_to_column_index; + self.schema_metadata + .extend(std::mem::take(&mut schema.metadata)); + self.schema = Some(schema); + Ok(()) + } + + fn ensure_initialized(&mut self, batch: &RecordBatch) -> Result<&LanceSchema> { + if self.schema.is_none() { + let schema = LanceSchema::try_from(batch.schema().as_ref())?; + self.initialize(schema)?; + } + Ok(self.schema.as_ref().unwrap()) + } + + #[instrument(skip_all, level = "debug")] + fn encode_batch( + &mut self, + batch: &RecordBatch, + external_buffers: &mut OutOfLineBuffers, + ) -> Result>> { + let field_arrays = self + .schema + .as_ref() + .unwrap() + .fields + .iter() + .enumerate() + .map(|(field_idx, field)| { + let array = + batch + .column_by_name(&field.name) + .ok_or(Error::invalid_input_source( + format!( + "Cannot write batch. The batch was missing the column `{}`", + field.name + ) + .into(), + ))?; + Ok((field_idx, array.clone())) + }) + .collect::>>()?; + self.encode_columns(&field_arrays, external_buffers) + } + + // Encode a set of `(field index, array)` pairs, each advancing only its own + // column. Each task captures its field's current row offset at encode time, + // so `advance_columns` must run after this call (never before); the order of + // the returned tasks relative to `write_pages` does not matter. + fn encode_columns( + &mut self, + field_arrays: &[(usize, ArrayRef)], + external_buffers: &mut OutOfLineBuffers, + ) -> Result>> { + // Snapshot the starting row number of each field before borrowing the + // column writers mutably below. + let row_numbers = field_arrays + .iter() + .map(|(field_idx, _)| self.field_rows_written[*field_idx]) + .collect::>(); + field_arrays + .iter() + .zip(row_numbers) + .map(|((field_idx, array), row_number)| { + let repdef = RepDefBuilder::default(); + let num_rows = array.len() as u64; + self.column_writers[*field_idx].maybe_encode( + array.clone(), + external_buffers, + repdef, + row_number, + num_rows, + ) + }) + .collect::>>() + } + + // Advance the per-field row counters after a set of columns has been + // written, keeping `rows_written` (the file's logical length) in sync as the + // longest column. Only the written fields move, so their new totals fold into + // `rows_written` directly without rescanning every field. (`write_batch` + // advances every field uniformly and tracks this inline instead.) + fn advance_columns(&mut self, field_arrays: &[(usize, ArrayRef)]) { + for (field_idx, array) in field_arrays { + let new_total = self.field_rows_written[*field_idx] + array.len() as u64; + self.field_rows_written[*field_idx] = new_total; + self.rows_written = self.rows_written.max(new_total); + } + } + + /// Schedule a batch of data to be written to the file + /// + /// Note: the future returned by this method may complete before the data has been fully + /// flushed to the file (some data may be in the data cache or the I/O cache) + pub async fn write_batch(&mut self, batch: &RecordBatch) -> Result<()> { + debug!( + "write_batch called with {} rows, {} columns, and {} bytes of data", + batch.num_rows(), + batch.num_columns(), + batch.get_array_memory_size() + ); + self.ensure_initialized(batch)?; + self.verify_nullability_constraints(batch)?; + let num_rows = batch.num_rows() as u64; + if num_rows == 0 { + return Ok(()); + } + if num_rows > u32::MAX as u64 { + return Err(Error::invalid_input_source( + "cannot write Lance files with more than 2^32 rows".into(), + )); + } + // First we push each array into its column writer. This may or may not generate enough + // data to trigger an encoding task. We collect any encoding tasks into a queue. + let mut external_buffers = + OutOfLineBuffers::new(self.tell().await?, PAGE_BUFFER_ALIGNMENT as u64); + let encoding_tasks = self.encode_batch(batch, &mut external_buffers)?; + // Next, write external buffers + for external_buffer in external_buffers.take_buffers() { + Self::do_write_buffer(&mut self.writer, &external_buffer).await?; + } + + let encoding_tasks = encoding_tasks + .into_iter() + .flatten() + .collect::>(); + + // `write_batch` advances every field by the same amount, so the longest + // column simply grows by `num_rows`. Guard against overflowing the row + // counter. + if self.rows_written.checked_add(num_rows).is_none() { + return Err(Error::invalid_input_source(format!("cannot write batch with {} rows because {} rows have already been written and Lance files cannot contain more than 2^64 rows", num_rows, self.rows_written).into())); + } + for field_rows in self.field_rows_written.iter_mut() { + *field_rows += num_rows; + } + self.rows_written += num_rows; + + self.write_pages(encoding_tasks).await?; + + Ok(()) + } + + /// Write a single column, advancing only that column's row counter. + /// + /// Unlike [`write_batch`](Self::write_batch), which advances every column + /// from a single shared row counter, this method advances one column + /// independently. Used across calls it produces a single file whose columns + /// may have different item counts. + /// + /// `column_index` refers to a top-level field in the writer's schema (the + /// same order as the schema's fields); a nested child cannot be targeted on + /// its own. Because each call writes the whole field from a single array, the + /// children of a struct field always advance together and stay equal-length; + /// only different top-level fields can diverge in length. A column may be + /// written across multiple calls; its values are appended. A field that is + /// never written ends up as a zero-length column. The writer must have been + /// created with an explicit schema (via [`try_new`](Self::try_new)); a lazy + /// schema cannot be inferred here because individual calls need not cover + /// every field. + /// + /// ``` + /// # use arrow_array::{ArrayRef, Int32Array}; + /// # use std::sync::Arc; + /// # use lance_file::writer::FileWriter; + /// # async fn example(writer: &mut FileWriter) -> lance_core::Result<()> { + /// // Field 0 gets three values, field 1 gets one — a non-rectangular file. + /// writer.write_column(0, Arc::new(Int32Array::from(vec![1, 2, 3]))).await?; + /// writer.write_column(1, Arc::new(Int32Array::from(vec![10]))).await?; + /// # Ok(()) + /// # } + /// ``` + pub async fn write_column(&mut self, column_index: usize, array: ArrayRef) -> Result<()> { + let schema = self.schema.as_ref().ok_or_else(|| { + Error::invalid_input_source( + "write_column requires the writer to be created with an explicit schema".into(), + ) + })?; + let field = schema.fields.get(column_index).ok_or_else(|| { + Error::invalid_input_source( + format!( + "write_column: field index {} is out of bounds (schema has {} fields)", + column_index, + schema.fields.len() + ) + .into(), + ) + })?; + if array.len() as u64 > u32::MAX as u64 { + return Err(Error::invalid_input_source( + "cannot write Lance files with more than 2^32 rows".into(), + )); + } + Self::verify_field_nullability(&array.to_data(), field)?; + + // A never-advanced field simply remains a zero-length column, which the + // encoders handle at `finish` time. + if array.is_empty() { + return Ok(()); + } + + let columns = [(column_index, array)]; + let mut external_buffers = + OutOfLineBuffers::new(self.tell().await?, PAGE_BUFFER_ALIGNMENT as u64); + let encoding_tasks = self.encode_columns(&columns, &mut external_buffers)?; + for external_buffer in external_buffers.take_buffers() { + Self::do_write_buffer(&mut self.writer, &external_buffer).await?; + } + let encoding_tasks = encoding_tasks + .into_iter() + .flatten() + .collect::>(); + + self.advance_columns(&columns); + self.write_pages(encoding_tasks).await?; + Ok(()) + } + + async fn write_column_metadata( + &mut self, + metadata: pbfile::ColumnMetadata, + ) -> Result<(u64, u64)> { + let metadata_bytes = metadata.encode_to_vec(); + let position = self.writer.tell().await? as u64; + let len = metadata_bytes.len() as u64; + self.writer.write_all(&metadata_bytes).await?; + Ok((position, len)) + } + + async fn write_column_metadatas(&mut self) -> Result> { + let metadatas = std::mem::take(&mut self.column_metadata); + + // If spilling, finalize the spill writer and reopen for reading. + // The spill file itself is cleaned up by the caller (it lives in a + // temp directory managed by the caller's RAII guard). + let spill_state = self.page_spill.take(); + let (spill_chunks, spill_reader) = + if let Some(PageSpillState::Active(mut spill)) = spill_state { + spill.shutdown_writer().await?; + let reader = spill.object_store.open(&spill.path).await?; + let chunks = std::mem::take(&mut spill.column_chunks); + (chunks, Some(reader)) + } else { + (Vec::new(), None) + }; + + let mut metadata_positions = Vec::with_capacity(metadatas.len()); + for (col_idx, mut metadata) in metadatas.into_iter().enumerate() { + if let Some(reader) = &spill_reader { + let mut pages = Vec::new(); + for &(offset, len) in &spill_chunks[col_idx] { + let data = reader + .get_range(offset as usize..(offset as usize + len as usize)) + .await + .map_err(|e| Error::io_source(Box::new(e)))?; + pages.extend(decode_spilled_chunk(&data)?); + } + metadata.pages = pages; + } + metadata_positions.push(self.write_column_metadata(metadata).await?); + } + + Ok(metadata_positions) + } + + fn make_file_descriptor( + schema: &lance_core::datatypes::Schema, + num_rows: u64, + ) -> Result { + let fields_with_meta = FieldsWithMeta::from(schema); + Ok(pb::FileDescriptor { + schema: Some(pb::Schema { + fields: fields_with_meta.fields.0, + metadata: fields_with_meta.metadata, + }), + length: num_rows, + }) + } + + async fn write_global_buffers(&mut self) -> Result> { + let schema = self.schema.as_mut().ok_or(Error::invalid_input("No schema provided on writer open and no data provided. Schema is unknown and file cannot be created"))?; + schema.metadata = std::mem::take(&mut self.schema_metadata); + // Use descriptor layout for blob v2 fields in the footer to avoid exposing logical child fields. + schema + .fields + .iter_mut() + .for_each(|f| f.unload_blobs_recursive()); + + let file_descriptor = Self::make_file_descriptor(schema, self.rows_written)?; + let file_descriptor_bytes = file_descriptor.encode_to_vec(); + let file_descriptor_len = file_descriptor_bytes.len() as u64; + let file_descriptor_position = self.writer.tell().await? as u64; + self.writer.write_all(&file_descriptor_bytes).await?; + let mut gbo_table = Vec::with_capacity(1 + self.global_buffers.len()); + gbo_table.push((file_descriptor_position, file_descriptor_len)); + gbo_table.append(&mut self.global_buffers); + Ok(gbo_table) + } + + /// Add a metadata entry to the schema + /// + /// This method is useful because sometimes the metadata is not known until after the + /// data has been written. This method allows you to alter the schema metadata. It + /// must be called before `finish` is called. + pub fn add_schema_metadata(&mut self, key: impl Into, value: impl Into) { + self.schema_metadata.insert(key.into(), value.into()); + } + + /// Prepare the writer when column data and metadata were produced externally. + /// + /// This is useful for flows that copy already-encoded pages (e.g., binary copy + /// during compaction) where the column buffers have been written directly and we + /// only need to write the footer and schema metadata. The provided + /// `column_metadata` must describe the buffers already persisted by the + /// underlying `ObjectWriter`, and `rows_written` should reflect the total number + /// of rows in those buffers. + pub fn initialize_with_external_metadata( + &mut self, + schema: lance_core::datatypes::Schema, + column_metadata: Vec, + rows_written: u64, + ) { + self.schema = Some(schema); + self.num_columns = column_metadata.len() as u32; + self.column_metadata = column_metadata; + self.rows_written = rows_written; + } + + /// Adds a global buffer to the file + /// + /// The global buffer can contain any arbitrary bytes. It will be written to the disk + /// immediately. This method returns the index of the global buffer (this will always + /// start at 1 and increment by 1 each time this method is called) + pub async fn add_global_buffer(&mut self, buffer: Bytes) -> Result { + let position = self.writer.tell().await? as u64; + let len = buffer.len() as u64; + Self::do_write_buffer(&mut self.writer, &buffer).await?; + self.global_buffers.push((position, len)); + Ok(self.global_buffers.len() as u32) + } + + async fn finish_writers(&mut self) -> Result<()> { + let mut col_idx = 0; + for mut writer in std::mem::take(&mut self.column_writers) { + let mut external_buffers = + OutOfLineBuffers::new(self.tell().await?, PAGE_BUFFER_ALIGNMENT as u64); + let columns = writer.finish(&mut external_buffers).await?; + for buffer in external_buffers.take_buffers() { + self.writer.write_all(&buffer).await?; + } + debug_assert_eq!( + columns.len(), + writer.num_columns() as usize, + "Expected {} columns from column at index {} and got {}", + writer.num_columns(), + col_idx, + columns.len() + ); + for column in columns { + for page in column.final_pages { + self.write_page(page).await?; + } + let column_metadata = &mut self.column_metadata[col_idx]; + let mut buffer_pos = self.writer.tell().await? as u64; + for buffer in column.column_buffers { + column_metadata.buffer_offsets.push(buffer_pos); + let mut size = 0; + Self::do_write_buffer(&mut self.writer, &buffer).await?; + size += buffer.len() as u64; + buffer_pos += size; + column_metadata.buffer_sizes.push(size); + } + let encoded_encoding = Any::from_msg(&column.encoding)?.encode_to_vec(); + column_metadata.encoding = Some(pbfile::Encoding { + location: Some(pbfile::encoding::Location::Direct(pbfile::DirectEncoding { + encoding: encoded_encoding, + })), + }); + col_idx += 1; + } + } + if col_idx != self.column_metadata.len() { + panic!( + "Column writers finished with {} columns but we expected {}", + col_idx, + self.column_metadata.len() + ); + } + Ok(()) + } + + /// Finishes writing the file + /// + /// This method will wait until all data has been flushed to the file. Then it + /// will write the file metadata and the footer. It will not return until all + /// data has been flushed and the file has been closed. + /// + /// Returns a summary of the completed file write. + pub async fn finish(&mut self) -> Result { + // 1. flush any remaining data and write out those pages + let mut external_buffers = + OutOfLineBuffers::new(self.tell().await?, PAGE_BUFFER_ALIGNMENT as u64); + let encoding_tasks = self + .column_writers + .iter_mut() + .map(|writer| writer.flush(&mut external_buffers)) + .collect::>>()?; + for external_buffer in external_buffers.take_buffers() { + Self::do_write_buffer(&mut self.writer, &external_buffer).await?; + } + let encoding_tasks = encoding_tasks + .into_iter() + .flatten() + .collect::>(); + self.write_pages(encoding_tasks).await?; + + if !self.column_writers.is_empty() { + self.finish_writers().await?; + } + + // 3. write global buffers (we write the schema here) + let global_buffer_offsets = self.write_global_buffers().await?; + let num_global_buffers = global_buffer_offsets.len() as u32; + + // 4. write the column metadatas + let column_metadata_start = self.writer.tell().await? as u64; + let metadata_positions = self.write_column_metadatas().await?; + + // 5. write the column metadata offset table + let cmo_table_start = self.writer.tell().await? as u64; + for (meta_pos, meta_len) in metadata_positions { + self.writer.write_u64_le(meta_pos).await?; + self.writer.write_u64_le(meta_len).await?; + } + + // 6. write global buffers offset table + let gbo_table_start = self.writer.tell().await? as u64; + for (gbo_pos, gbo_len) in global_buffer_offsets { + self.writer.write_u64_le(gbo_pos).await?; + self.writer.write_u64_le(gbo_len).await?; + } + + // 7. write the footer + self.writer.write_u64_le(column_metadata_start).await?; + self.writer.write_u64_le(cmo_table_start).await?; + self.writer.write_u64_le(gbo_table_start).await?; + self.writer.write_u32_le(num_global_buffers).await?; + self.writer.write_u32_le(self.num_columns).await?; + self.writer.write_u16_le(0).await?; + self.writer.write_u16_le(3).await?; + self.writer.write_all(MAGIC).await?; + + // 7. close the writer + let write_result = ObjectWriter::shutdown(self.writer.as_mut()).await?; + + Ok(FileWriteSummary { + num_rows: self.rows_written, + size_bytes: write_result.size as u64, + }) + } + + pub async fn abort(&mut self) { + // For multipart uploads, ObjectWriter's Drop impl will abort + // the upload when the writer is dropped. + } + + pub async fn tell(&mut self) -> Result { + Ok(self.writer.tell().await? as u64) + } + + /// Append a buffer whose metadata is supplied by the caller. + pub async fn write_external_buffer(&mut self, bytes: &[u8]) -> Result<(u64, u64)> { + let start = self.tell().await?; + self.writer.write_all(bytes).await?; + Ok((start, bytes.len() as u64)) + } + + pub fn field_id_to_column_indices(&self) -> &[(u32, u32)] { + &self.field_id_to_column_indices + } +} + +// Creates a lance footer and appends it to the encoded data +// +// The logic here is very similar to logic in the FileWriter except we +// are using BufMut (put_xyz) instead of AsyncWrite (write_xyz). +pub fn concat_lance_footer(batch: &EncodedBatch, write_schema: bool) -> Result { + // Estimating 1MiB for file footer + let mut data = BytesMut::with_capacity(batch.data.len() + 1024 * 1024); + data.put(batch.data.clone()); + // write global buffers (we write the schema here) + let global_buffers = if write_schema { + let schema_start = data.len() as u64; + let lance_schema = lance_core::datatypes::Schema::try_from(batch.schema.as_ref())?; + let descriptor = Writer::make_file_descriptor(&lance_schema, batch.num_rows)?; + let descriptor_bytes = descriptor.encode_to_vec(); + let descriptor_len = descriptor_bytes.len() as u64; + data.put(descriptor_bytes.as_slice()); + + vec![(schema_start, descriptor_len)] + } else { + vec![] + }; + let col_metadata_start = data.len() as u64; + + let mut col_metadata_positions = Vec::new(); + // Write column metadata + for col in &batch.page_table { + let position = data.len() as u64; + let pages = col + .page_infos + .iter() + .map(|page_info| { + let encoded_encoding = match &page_info.encoding { + PageEncoding::Legacy(array_encoding) => { + Any::from_msg(array_encoding)?.encode_to_vec() + } + PageEncoding::Structural(page_layout) => { + Any::from_msg(page_layout)?.encode_to_vec() + } + }; + let (buffer_offsets, buffer_sizes): (Vec<_>, Vec<_>) = page_info + .buffer_offsets_and_sizes + .as_ref() + .iter() + .cloned() + .unzip(); + Ok(pbfile::column_metadata::Page { + buffer_offsets, + buffer_sizes, + encoding: Some(pbfile::Encoding { + location: Some(pbfile::encoding::Location::Direct(DirectEncoding { + encoding: encoded_encoding, + })), + }), + length: page_info.num_rows, + priority: page_info.priority, + }) + }) + .collect::>>()?; + let (buffer_offsets, buffer_sizes): (Vec<_>, Vec<_>) = + col.buffer_offsets_and_sizes.iter().cloned().unzip(); + let encoded_col_encoding = Any::from_msg(&col.encoding)?.encode_to_vec(); + let column = pbfile::ColumnMetadata { + pages, + buffer_offsets, + buffer_sizes, + encoding: Some(pbfile::Encoding { + location: Some(pbfile::encoding::Location::Direct(pbfile::DirectEncoding { + encoding: encoded_col_encoding, + })), + }), + }; + let column_bytes = column.encode_to_vec(); + col_metadata_positions.push((position, column_bytes.len() as u64)); + data.put(column_bytes.as_slice()); + } + // Write column metadata offsets table + let cmo_table_start = data.len() as u64; + for (meta_pos, meta_len) in col_metadata_positions { + data.put_u64_le(meta_pos); + data.put_u64_le(meta_len); + } + // Write global buffers offsets table + let gbo_table_start = data.len() as u64; + let num_global_buffers = global_buffers.len() as u32; + for (gbo_pos, gbo_len) in global_buffers { + data.put_u64_le(gbo_pos); + data.put_u64_le(gbo_len); + } + + // write the footer + data.put_u64_le(col_metadata_start); + data.put_u64_le(cmo_table_start); + data.put_u64_le(gbo_table_start); + data.put_u32_le(num_global_buffers); + data.put_u32_le(batch.page_table.len() as u32); + data.put_u16_le(2); + data.put_u16_le(0); + data.put(MAGIC.as_slice()); + + Ok(data.freeze()) +} diff --git a/rust/lance-file/src/versions/v2_1/compression.rs b/rust/lance-file/src/versions/v2_1/compression.rs new file mode 100644 index 00000000000..5be59bf8e53 --- /dev/null +++ b/rust/lance-file/src/versions/v2_1/compression.rs @@ -0,0 +1,130 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright The Lance Authors + +use lance_core::{Error, Result, datatypes::Field}; +use lance_encoding::{ + compression::{ + BlockCompressor, CompressionStrategy, field_metadata_params, finalize_miniblock_compressor, + reject_packed_struct_per_value, try_bitpacking_block, try_bitpacking_miniblock, + try_byte_stream_split_miniblock, try_fixed_packed_struct_miniblock, + try_fixed_u8_rle_miniblock, try_raw_block, try_raw_fixed_size_list_miniblock, + try_raw_fixed_width_miniblock, try_raw_per_value, try_uncompressed_fixed_width_miniblock, + try_variable_width_miniblock, try_variable_width_per_value, + }, + compression_config::{CompressionFieldParams, CompressionParams}, + data::DataBlock, + encodings::logical::primitive::{fullzip::PerValueCompressor, miniblock::MiniBlockCompressor}, + format::pb21::CompressiveEncoding, +}; + +#[derive(Debug, Clone)] +pub(super) struct Strategy { + params: CompressionParams, +} + +impl Strategy { + pub(super) fn new(params: CompressionParams) -> Self { + Self { params } + } + + fn field_params(&self, field: &Field) -> CompressionFieldParams { + let mut params = self + .params + .get_field_params(&field.name, &field.data_type()); + let mut metadata = field_metadata_params(field); + if metadata + .minichunk_size + .is_some_and(|size| size >= 32 * 1024) + { + log::warn!( + "minichunk_size '{}' is too large for the selected u16 miniblock layout, using default", + metadata.minichunk_size.unwrap() + ); + metadata.minichunk_size = None; + } + params.merge(&metadata); + params + } +} + +impl CompressionStrategy for Strategy { + fn create_miniblock_compressor( + &self, + field: &Field, + data: &DataBlock, + ) -> Result> { + let params = self.field_params(field); + let compressor = + if let Some(compressor) = try_uncompressed_fixed_width_miniblock(data, ¶ms) { + compressor + } else if let Some(compressor) = try_byte_stream_split_miniblock(data, ¶ms) { + compressor + } else if let Some(compressor) = try_fixed_u8_rle_miniblock(data, ¶ms) { + compressor + } else if let Some(compressor) = try_bitpacking_miniblock(data) { + compressor + } else if let Some(compressor) = try_raw_fixed_width_miniblock(data) { + compressor + } else if let Some(compressor) = try_variable_width_miniblock(field, data, ¶ms)? { + compressor + } else if let Some(compressor) = try_fixed_packed_struct_miniblock(data)? { + compressor + } else if let Some(compressor) = try_raw_fixed_size_list_miniblock(data) { + compressor + } else { + return Err(Error::not_supported_source( + format!( + "Mini-block compression not yet supported for block type {}", + data.name() + ) + .into(), + )); + }; + finalize_miniblock_compressor(data, compressor, ¶ms) + } + + fn create_per_value( + &self, + field: &Field, + data: &DataBlock, + ) -> Result> { + let params = self.field_params(field); + if let Some(compressor) = try_raw_per_value(data) { + return Ok(compressor); + } + if let Some(compressor) = reject_packed_struct_per_value(field, data)? { + return Ok(compressor); + } + if let Some(compressor) = try_variable_width_per_value(field, data, ¶ms)? { + return Ok(compressor); + } + Err(Error::not_supported_source( + format!( + "Per-value compression not yet supported for block type {}", + data.name() + ) + .into(), + )) + } + + fn create_block_compressor( + &self, + field: &Field, + data: &DataBlock, + ) -> Result<(Box, CompressiveEncoding)> { + let _params = self.field_params(field); + if let Some(compressor) = try_bitpacking_block(data) { + return Ok(compressor); + } + if let Some(compressor) = try_raw_block(data) { + return Ok(compressor); + } + Err(Error::not_supported_source( + format!( + "Block compression not yet supported for block type {}", + data.name() + ) + .into(), + )) + } +} diff --git a/rust/lance-file/src/versions/v2_1/mod.rs b/rust/lance-file/src/versions/v2_1/mod.rs new file mode 100644 index 00000000000..0555f3d0264 --- /dev/null +++ b/rust/lance-file/src/versions/v2_1/mod.rs @@ -0,0 +1,180 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright The Lance Authors + +//! Lance v2.1 file composition. + +use std::{collections::BTreeMap, sync::Arc}; + +use bytes::Bytes; +use lance_core::{ + Error, Result, + datatypes::{Field, Schema}, +}; +use lance_encoding::{ + compression_config::CompressionParams, + encoder::{ + ColumnIndexSequence, EncodedBatch, FieldEncoder, FieldEncodingContext, + FieldEncodingStrategy, + structural::{ + PrimitiveFieldEncoding, PrimitivePageEncoding, try_create_binary_blob, try_create_list, + try_create_struct, + }, + }, +}; +use lance_io::traits::Writer as ObjectWriter; + +use crate::{ + reader::{ReadProjection, structural}, + writer::FileWriterOptions, +}; + +mod compression; +mod reader; +mod writer; + +pub use reader::{ + projection_from_column_names, projection_from_field_ids, projection_from_whole_schema, +}; +pub use writer::Writer; + +#[cfg(test)] +pub(crate) use reader::test_projection_length; +pub(crate) use reader::{ + decode_column_metadata, finish_metadata, finish_metadata_index, validate_global_buffers, +}; + +pub(crate) fn read_projection() -> Arc { + structural::read_projection(reader::decode_column) +} + +/// Count physical columns represented by a field in a v2.1 footer. +pub fn physical_column_count(field: &Field) -> usize { + structural::physical_column_count(field) +} + +/// Build persisted field-to-column entries for a v2.1 data file. +pub fn data_file_columns(schema: &Schema) -> (Vec, Vec) { + structural::data_file_columns(schema) +} + +pub(super) fn field_id_to_column_index(schema: &Schema) -> BTreeMap { + structural::field_id_to_column_index(schema) +} + +#[derive(Debug)] +struct FieldStrategy { + primitive: PrimitiveFieldEncoding, +} + +impl FieldEncodingStrategy for FieldStrategy { + fn create_field_encoder( + &self, + field: &Field, + column_index: &mut ColumnIndexSequence, + context: &FieldEncodingContext<'_>, + ) -> Result> { + if let Some(encoder) = + try_create_binary_blob(&self.primitive, field, column_index, context)? + { + return Ok(encoder); + } + if field.is_blob() { + return Err(Error::invalid_input_source( + format!( + "Blob encoding is not available for field '{}' with data type {}", + field.name, + field.data_type() + ) + .into(), + )); + } + if let Some(encoder) = self.primitive.try_create(field, column_index, context)? { + return Ok(encoder); + } + if matches!( + field.data_type(), + arrow_schema::DataType::FixedSizeList(item, _) + if matches!(item.data_type(), arrow_schema::DataType::Struct(_)) + ) { + return Err(Error::not_supported_source( + "FixedSizeList is not enabled by the selected file format".into(), + )); + } + if matches!(field.data_type(), arrow_schema::DataType::Map(_, _)) { + return Err(Error::not_supported_source( + "Map data type is not enabled by the selected file format".into(), + )); + } + if let Some(encoder) = try_create_list(field, column_index, context)? { + return Ok(encoder); + } + if let Some(encoder) = try_create_struct(field, column_index, context)? { + return Ok(encoder); + } + Err(Error::not_supported_source( + format!( + "Lance v2.1 has no field encoding for '{}' with data type {}", + field.name, + field.data_type() + ) + .into(), + )) + } +} + +/// Compose the v2.1 field encoding mechanisms. +pub fn encoding_strategy(params: CompressionParams) -> Arc { + let compression = Arc::new(compression::Strategy::new(params)); + Arc::new(FieldStrategy { + primitive: PrimitiveFieldEncoding::new([ + PrimitivePageEncoding::reject_sparse(), + PrimitivePageEncoding::dense_u16(compression), + ]), + }) +} + +/// Create a v2.1 writer with an explicit schema. +pub fn create_writer( + object_writer: Box, + schema: Schema, + options: FileWriterOptions, +) -> Result { + Writer::try_new(object_writer, schema, options) +} + +/// Create a v2.1 writer with explicit compression tuning. +pub fn create_writer_with_compression( + object_writer: Box, + schema: Schema, + options: FileWriterOptions, + compression: CompressionParams, +) -> Result { + Writer::try_new_with_compression(object_writer, schema, options, compression) +} + +/// Create a v2.1 writer whose schema is inferred from the first batch. +pub fn create_lazy_writer( + object_writer: Box, + options: FileWriterOptions, +) -> Writer { + Writer::new_lazy(object_writer, options) +} + +/// Create a lazy v2.1 writer with explicit compression tuning. +pub fn create_lazy_writer_with_compression( + object_writer: Box, + options: FileWriterOptions, + compression: CompressionParams, +) -> Writer { + Writer::new_lazy_with_compression(object_writer, options, compression) +} + +/// Encode a self-described v2.1 batch. +pub fn encode_self_described_batch(batch: &EncodedBatch) -> Result { + writer::concat_lance_footer(batch, true) +} + +/// Encode a mini-lance v2.1 batch. +pub fn encode_mini_batch(batch: &EncodedBatch) -> Result { + writer::concat_lance_footer(batch, false) +} diff --git a/rust/lance-file/src/versions/v2_1/reader.rs b/rust/lance-file/src/versions/v2_1/reader.rs new file mode 100644 index 00000000000..0c11e5f8976 --- /dev/null +++ b/rust/lance-file/src/versions/v2_1/reader.rs @@ -0,0 +1,304 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright The Lance Authors + +use std::{collections::BTreeMap, sync::Arc}; + +use lance_core::{Error, Result, datatypes::Schema}; +use lance_encoding::{decoder::ColumnInfo, format::pb21}; + +use crate::{ + format::pbfile, + reader::{ + BufferDescriptor, CachedFileMetadata, FileMetadataIndex, RawFileMetadata, ReaderProjection, + structural, + }, + version::ConcreteFileVersion, +}; + +fn required<'a, T>(value: Option<&'a T>, label: &str) -> Result<&'a T> { + value.ok_or_else(|| { + Error::invalid_input_source( + format!("Lance v2.1 {label} is missing its nested encoding").into(), + ) + }) +} + +fn validate_compressive_encoding(encoding: &pb21::CompressiveEncoding) -> Result<()> { + use pb21::compressive_encoding::Compression; + + match encoding.compression.as_ref() { + Some(Compression::Flat(_)) + | Some(Compression::InlineBitpacking(_)) + | Some(Compression::Constant(_)) => Ok(()), + Some(Compression::Variable(variable)) => validate_compressive_encoding(required( + variable.offsets.as_deref(), + "variable offsets", + )?), + Some(Compression::OutOfLineBitpacking(bitpacking)) => { + validate_compressive_encoding(required( + bitpacking.values.as_deref(), + "out-of-line bitpacking values", + )?) + } + Some(Compression::Fsst(fsst)) => { + validate_compressive_encoding(required(fsst.values.as_deref(), "FSST values")?) + } + Some(Compression::Dictionary(dictionary)) => { + validate_compressive_encoding(required( + dictionary.indices.as_deref(), + "dictionary indices", + )?)?; + validate_compressive_encoding(required( + dictionary.items.as_deref(), + "dictionary items", + )?) + } + Some(Compression::Rle(rle)) => { + let values = required(rle.values.as_deref(), "RLE values")?; + let run_lengths = required(rle.run_lengths.as_deref(), "RLE run lengths")?; + let fixed_values = matches!( + values.compression.as_ref(), + Some(Compression::Flat(flat)) + if matches!(flat.bits_per_value, 8 | 16 | 32 | 64) + && flat.data.is_none() + ); + let fixed_u8_lengths = matches!( + run_lengths.compression.as_ref(), + Some(Compression::Flat(flat)) + if flat.bits_per_value == 8 && flat.data.is_none() + ); + if !fixed_values || !fixed_u8_lengths { + return Err(Error::invalid_input_source( + "Lance v2.1 RLE requires flat values and flat u8 run lengths".into(), + )); + } + Ok(()) + } + Some(Compression::ByteStreamSplit(split)) => validate_compressive_encoding(required( + split.values.as_deref(), + "byte-stream-split values", + )?), + Some(Compression::General(general)) => validate_compressive_encoding(required( + general.values.as_deref(), + "general-compression values", + )?), + Some(Compression::FixedSizeList(list)) => validate_compressive_encoding(required( + list.values.as_deref(), + "fixed-size-list values", + )?), + Some(Compression::PackedStruct(packed)) => validate_compressive_encoding(required( + packed.values.as_deref(), + "packed-struct values", + )?), + Some(Compression::VariablePackedStruct(_)) => Err(Error::invalid_input_source( + "Variable packed struct compression is not part of the Lance v2.1 grammar".into(), + )), + None => Err(Error::invalid_input_source( + "Lance v2.1 compressive encoding is missing its compression variant".into(), + )), + } +} + +fn validate_page_layout(layout: &pb21::PageLayout) -> Result<()> { + use pb21::page_layout::Layout; + + match layout.layout.as_ref() { + Some(Layout::MiniBlockLayout(miniblock)) => { + if miniblock.has_large_chunk { + return Err(Error::invalid_input_source( + "Large miniblock chunks are not part of the Lance v2.1 grammar".into(), + )); + } + if let Some(rep) = miniblock.rep_compression.as_ref() { + validate_compressive_encoding(rep)?; + } + if let Some(def) = miniblock.def_compression.as_ref() { + validate_compressive_encoding(def)?; + } + validate_compressive_encoding(required( + miniblock.value_compression.as_ref(), + "miniblock values", + )?)?; + if let Some(dictionary) = miniblock.dictionary.as_ref() { + validate_compressive_encoding(dictionary)?; + } + Ok(()) + } + Some(Layout::FullZipLayout(fullzip)) => validate_compressive_encoding(required( + fullzip.value_compression.as_ref(), + "full-zip values", + )?), + Some(Layout::ConstantLayout(constant)) => { + if constant.inline_value.is_some() { + Err(Error::invalid_input_source( + "Lance v2.1 only accepts the all-null form of constant page layout".into(), + )) + } else if constant.rep_compression.is_some() || constant.def_compression.is_some() { + Err(Error::invalid_input_source( + "Compressed constant-page levels are not part of the Lance v2.1 grammar".into(), + )) + } else { + Ok(()) + } + } + Some(Layout::BlobLayout(blob)) => { + let inner = blob.inner_layout.as_deref().ok_or_else(|| { + Error::invalid_input_source( + "Lance v2.1 blob page layout is missing its inner layout".into(), + ) + })?; + validate_page_layout(inner) + } + Some(Layout::SparseLayout(_)) => Err(Error::invalid_input_source( + "Sparse page layout is not part of the Lance v2.1 grammar".into(), + )), + None => Err(Error::invalid_input_source( + "Lance v2.1 page is missing its page layout".into(), + )), + } +} + +pub fn decode_column( + column_index: u32, + metadata: &pbfile::ColumnMetadata, +) -> Result> { + let page_infos = metadata + .pages + .iter() + .enumerate() + .map(|(page_index, page)| { + let page_layout = structural::decode_page_layout(column_index, page_index, page)?; + validate_page_layout(&page_layout)?; + structural::build_page_info(column_index, page_index, page, page_layout) + }) + .collect::>>()?; + structural::build_column_info(column_index, metadata, page_infos) +} + +pub fn decode_column_metadata( + column_metadatas: &[pbfile::ColumnMetadata], +) -> Result>> { + column_metadatas + .iter() + .enumerate() + .map(|(column_index, metadata)| { + let column_index = u32::try_from(column_index).map_err(|_| { + Error::invalid_input_source("File has more than u32::MAX columns".into()) + })?; + decode_column(column_index, metadata) + }) + .collect() +} + +pub fn projection_from_field_ids( + schema: &Schema, + field_id_to_column_index: &BTreeMap, +) -> ReaderProjection { + structural::projection_from_field_ids(schema, field_id_to_column_index) +} + +pub fn projection_from_whole_schema(schema: &Schema) -> ReaderProjection { + structural::projection_from_field_ids(schema, &super::field_id_to_column_index(schema)) +} + +pub fn projection_from_column_names( + schema: &Schema, + column_names: &[&str], +) -> Result { + structural::projection_from_column_names( + schema, + column_names, + &super::field_id_to_column_index(schema), + ) +} + +pub fn finish_metadata(raw: RawFileMetadata) -> Result { + if (raw.footer.major_version, raw.footer.minor_version) != (2, 1) { + return Err(Error::version_conflict( + "Attempt to use the Lance v2.1 reader for a different file version".to_string(), + raw.footer.major_version, + raw.footer.minor_version, + )); + } + validate_global_buffers(&raw.file_buffers)?; + let column_infos = decode_column_metadata(&raw.column_metadatas)?; + Ok(CachedFileMetadata { + file_schema: raw.file_schema, + column_metadatas: raw.column_metadatas, + column_infos, + num_rows: raw.num_rows, + file_buffers: raw.file_buffers, + num_data_bytes: raw.num_data_bytes, + num_column_metadata_bytes: raw.num_column_metadata_bytes, + num_global_buffer_bytes: raw.num_global_buffer_bytes, + num_footer_bytes: raw.num_footer_bytes, + major_version: raw.footer.major_version, + minor_version: raw.footer.minor_version, + version: ConcreteFileVersion::V2_1, + file_size_bytes: raw.file_size_bytes, + retained_global_buffers: raw.retained_global_buffers, + }) +} + +pub fn validate_global_buffers(buffers: &[BufferDescriptor]) -> Result<()> { + structural::validate_global_buffers(buffers) +} + +pub fn finish_metadata_index(index: FileMetadataIndex) -> Result { + if index.version != ConcreteFileVersion::V2_1 { + let (major, minor) = index.version.to_standard_footer_numbers(); + return Err(Error::version_conflict( + "Attempt to use the Lance v2.1 reader for a different metadata index".to_string(), + major, + minor, + )); + } + validate_global_buffers(&index.file_buffers)?; + Ok(index) +} + +#[cfg(test)] +pub fn test_projection_length( + schema: &Schema, + column_indices: &[u32], + column_lengths: &[u64], +) -> Result { + structural::test_projection_length(schema, column_indices, column_lengths) +} + +#[cfg(test)] +mod grammar_tests { + use super::*; + use pb21::{ + CompressiveEncoding, FullZipLayout, General, PageLayout, VariablePackedStruct, + compressive_encoding::Compression, page_layout::Layout, + }; + + #[test] + fn rejects_nested_variable_packed_struct() { + let variable_packed = CompressiveEncoding { + compression: Some(Compression::VariablePackedStruct(VariablePackedStruct { + fields: Vec::new(), + })), + }; + let wrapped = CompressiveEncoding { + compression: Some(Compression::General(Box::new(General { + compression: None, + values: Some(Box::new(variable_packed)), + }))), + }; + let layout = PageLayout { + layout: Some(Layout::FullZipLayout(FullZipLayout { + value_compression: Some(wrapped), + ..Default::default() + })), + }; + + let error = validate_page_layout(&layout).unwrap_err(); + assert!( + error + .to_string() + .contains("Variable packed struct compression is not part") + ); + } +} diff --git a/rust/lance-file/src/versions/v2_1/writer.rs b/rust/lance-file/src/versions/v2_1/writer.rs new file mode 100644 index 00000000000..1599a9fd6e1 --- /dev/null +++ b/rust/lance-file/src/versions/v2_1/writer.rs @@ -0,0 +1,223 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright The Lance Authors + +use std::sync::Arc; + +use arrow_array::{ArrayRef, RecordBatch}; +use bytes::{BufMut, Bytes}; +use lance_core::{Result, datatypes::Schema}; +use lance_encoding::{ + compression_config::CompressionParams, + encoder::{BatchEncoder, EncodedBatch}, +}; +use lance_io::{object_store::ObjectStore, traits::Writer as ObjectWriter}; +use object_store::path::Path; +use tokio::io::AsyncWriteExt; + +use crate::{ + format::{MAGIC, pbfile}, + writer::{ + FileWriteSummary, FileWriterOptions, + structural::{EncodedBatchBody, EncodingPipeline, StructuralFileSink, encode_batch_body}, + }, +}; + +use super::encoding_strategy; + +/// A writer for the Lance v2.1 file grammar. +/// +/// The concrete writer owns the v2.1 encoding composition, finish ordering, +/// and exact footer identity. Shared components only execute the structural +/// encoding and I/O mechanisms selected here. +pub struct Writer { + sink: StructuralFileSink, + encoding: EncodingPipeline, + compression: CompressionParams, +} + +impl Writer { + /// Create a v2.1 writer with an explicit schema. + pub fn try_new( + object_writer: Box, + schema: Schema, + options: FileWriterOptions, + ) -> Result { + Self::try_new_with_compression(object_writer, schema, options, Default::default()) + } + + /// Create a v2.1 writer with explicit compression tuning. + pub fn try_new_with_compression( + object_writer: Box, + schema: Schema, + options: FileWriterOptions, + compression: CompressionParams, + ) -> Result { + let mut writer = Self::new_lazy_with_compression(object_writer, options, compression); + writer.initialize(schema)?; + Ok(writer) + } + + /// Create a v2.1 writer whose schema is inferred from the first batch. + pub fn new_lazy(object_writer: Box, options: FileWriterOptions) -> Self { + Self::new_lazy_with_compression(object_writer, options, Default::default()) + } + + /// Create a lazy v2.1 writer with explicit compression tuning. + pub fn new_lazy_with_compression( + object_writer: Box, + options: FileWriterOptions, + compression: CompressionParams, + ) -> Self { + Self { + sink: StructuralFileSink::new(object_writer), + encoding: EncodingPipeline::new(options), + compression, + } + } + + fn initialize(&mut self, schema: Schema) -> Result<()> { + let encoding_options = self.encoding.encoding_options(&schema); + schema.validate()?; + let strategy = encoding_strategy(self.compression.clone()); + let encoder = BatchEncoder::try_new(&schema, strategy.as_ref(), &encoding_options)?; + self.encoding.initialize(schema, encoder, &mut self.sink); + Ok(()) + } + + fn ensure_initialized(&mut self, batch: &RecordBatch) -> Result<()> { + if !self.encoding.is_initialized() { + self.initialize(Schema::try_from(batch.schema().as_ref())?)?; + } + Ok(()) + } + + /// Spill page metadata to a sidecar file instead of retaining it in memory. + pub fn with_page_metadata_spill(mut self, object_store: Arc, path: Path) -> Self { + self.sink.with_page_metadata_spill(object_store, path); + self + } + + /// Schedule batches to be written in iteration order. + pub async fn write_batches( + &mut self, + batches: impl Iterator, + ) -> Result<()> { + for batch in batches { + self.write_batch(batch).await?; + } + Ok(()) + } + + /// Schedule one record batch for writing. + pub async fn write_batch(&mut self, batch: &RecordBatch) -> Result<()> { + self.ensure_initialized(batch)?; + self.encoding.write_batch(batch, &mut self.sink).await + } + + /// Write one top-level field, advancing only that field's row count. + pub async fn write_column(&mut self, column_index: usize, array: ArrayRef) -> Result<()> { + self.encoding + .write_column(column_index, array, &mut self.sink) + .await + } + + /// Append a buffer whose page or column metadata is supplied externally. + pub async fn write_external_buffer(&mut self, bytes: &[u8]) -> Result<(u64, u64)> { + self.sink.write_external_buffer(bytes).await + } + + /// Add an entry to the schema metadata written in the file descriptor. + pub fn add_schema_metadata(&mut self, key: impl Into, value: impl Into) { + self.encoding.add_schema_metadata(key, value); + } + + /// Prepare the writer for encoded column data produced externally. + pub fn initialize_with_external_metadata( + &mut self, + schema: Schema, + column_metadata: Vec, + rows_written: u64, + ) { + self.encoding + .initialize_with_external_metadata(schema, rows_written); + self.sink.initialize_with_external_metadata(column_metadata); + } + + /// Add an arbitrary global buffer and return its one-based index. + pub async fn add_global_buffer(&mut self, buffer: Bytes) -> Result { + self.sink.add_global_buffer(buffer).await + } + + /// Finish the v2.1 file and close its object writer. + pub async fn finish(&mut self) -> Result { + // The order below is the v2.1 wire contract. + self.encoding.flush(&mut self.sink).await?; + self.encoding.finish_encoders(&mut self.sink).await?; + + let descriptor = self.encoding.make_file_descriptor()?; + let global_buffer_offsets = self.sink.write_global_buffers(descriptor).await?; + let num_global_buffers = global_buffer_offsets.len() as u32; + + let column_metadata_start = self.sink.tell().await?; + let column_metadata_offsets = self.sink.write_column_metadatas().await?; + let column_metadata_offsets_start = self + .sink + .write_offset_table(&column_metadata_offsets) + .await?; + let global_buffer_offsets_start = + self.sink.write_offset_table(&global_buffer_offsets).await?; + let num_columns = self.sink.num_columns(); + + let output = self.sink.output_mut(); + output.write_u64_le(column_metadata_start).await?; + output.write_u64_le(column_metadata_offsets_start).await?; + output.write_u64_le(global_buffer_offsets_start).await?; + output.write_u32_le(num_global_buffers).await?; + output.write_u32_le(num_columns).await?; + output.write_u16_le(2).await?; + output.write_u16_le(1).await?; + output.write_all(MAGIC).await?; + + Ok(FileWriteSummary { + num_rows: self.encoding.rows_written(), + size_bytes: self.sink.shutdown().await?, + }) + } + + /// Abandon this write. + pub async fn abort(&mut self) { + // Dropping a multipart ObjectWriter aborts the upload. + } + + /// Return the current object-writer position. + pub async fn tell(&mut self) -> Result { + self.sink.tell().await + } + + /// Return the field-id to physical-column mapping selected by v2.1. + pub fn field_id_to_column_indices(&self) -> &[(u32, u32)] { + self.encoding.field_id_to_column_indices() + } +} + +/// Append a self-described or mini-lance v2.1 footer to an encoded batch. +pub fn concat_lance_footer(batch: &EncodedBatch, write_schema: bool) -> Result { + let EncodedBatchBody { + mut data, + column_metadata_start, + column_metadata_offsets_start, + global_buffer_offsets_start, + num_global_buffers, + num_columns, + } = encode_batch_body(batch, write_schema)?; + + data.put_u64_le(column_metadata_start); + data.put_u64_le(column_metadata_offsets_start); + data.put_u64_le(global_buffer_offsets_start); + data.put_u32_le(num_global_buffers); + data.put_u32_le(num_columns); + data.put_u16_le(2); + data.put_u16_le(1); + data.extend_from_slice(MAGIC); + Ok(data.freeze()) +} diff --git a/rust/lance-file/src/versions/v2_2/compression.rs b/rust/lance-file/src/versions/v2_2/compression.rs new file mode 100644 index 00000000000..a965d5afc64 --- /dev/null +++ b/rust/lance-file/src/versions/v2_2/compression.rs @@ -0,0 +1,130 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright The Lance Authors + +use std::sync::Arc; + +use lance_core::{Error, Result, datatypes::Field}; +use lance_encoding::{ + compression::{ + BlockCompressor, CompressionStrategy, field_metadata_params, finalize_miniblock_compressor, + try_bitpacking_block, try_bitpacking_miniblock, try_byte_stream_split_miniblock, + try_fixed_packed_struct_miniblock, try_fixed_u8_rle_block, try_fixed_u8_rle_miniblock, + try_general_block, try_raw_block, try_raw_fixed_size_list_miniblock, + try_raw_fixed_width_miniblock, try_raw_per_value, try_uncompressed_fixed_width_miniblock, + try_variable_packed_struct_per_value, try_variable_width_miniblock, + try_variable_width_per_value, + }, + compression_config::{CompressionFieldParams, CompressionParams}, + data::DataBlock, + encodings::logical::primitive::{fullzip::PerValueCompressor, miniblock::MiniBlockCompressor}, + format::pb21::CompressiveEncoding, +}; + +#[derive(Debug, Clone)] +pub(super) struct Strategy { + params: CompressionParams, +} + +impl Strategy { + pub(super) fn new(params: CompressionParams) -> Self { + Self { params } + } + + fn field_params(&self, field: &Field) -> CompressionFieldParams { + let mut params = self + .params + .get_field_params(&field.name, &field.data_type()); + params.merge(&field_metadata_params(field)); + params + } +} + +impl CompressionStrategy for Strategy { + fn create_miniblock_compressor( + &self, + field: &Field, + data: &DataBlock, + ) -> Result> { + let params = self.field_params(field); + let compressor = + if let Some(compressor) = try_uncompressed_fixed_width_miniblock(data, ¶ms) { + compressor + } else if let Some(compressor) = try_byte_stream_split_miniblock(data, ¶ms) { + compressor + } else if let Some(compressor) = try_fixed_u8_rle_miniblock(data, ¶ms) { + compressor + } else if let Some(compressor) = try_bitpacking_miniblock(data) { + compressor + } else if let Some(compressor) = try_raw_fixed_width_miniblock(data) { + compressor + } else if let Some(compressor) = try_variable_width_miniblock(field, data, ¶ms)? { + compressor + } else if let Some(compressor) = try_fixed_packed_struct_miniblock(data)? { + compressor + } else if let Some(compressor) = try_raw_fixed_size_list_miniblock(data) { + compressor + } else { + return Err(Error::not_supported_source( + format!( + "Mini-block compression not yet supported for block type {}", + data.name() + ) + .into(), + )); + }; + finalize_miniblock_compressor(data, compressor, ¶ms) + } + + fn create_per_value( + &self, + field: &Field, + data: &DataBlock, + ) -> Result> { + let params = self.field_params(field); + if let Some(compressor) = try_raw_per_value(data) { + return Ok(compressor); + } + if let Some(compressor) = + try_variable_packed_struct_per_value(Arc::new(self.clone()), field, data)? + { + return Ok(compressor); + } + if let Some(compressor) = try_variable_width_per_value(field, data, ¶ms)? { + return Ok(compressor); + } + Err(Error::not_supported_source( + format!( + "Per-value compression not yet supported for block type {}", + data.name() + ) + .into(), + )) + } + + fn create_block_compressor( + &self, + field: &Field, + data: &DataBlock, + ) -> Result<(Box, CompressiveEncoding)> { + let params = self.field_params(field); + if let Some(compressor) = try_fixed_u8_rle_block(data, ¶ms)? { + return Ok(compressor); + } + if let Some(compressor) = try_bitpacking_block(data) { + return Ok(compressor); + } + if let Some(compressor) = try_general_block(data, ¶ms)? { + return Ok(compressor); + } + if let Some(compressor) = try_raw_block(data) { + return Ok(compressor); + } + Err(Error::not_supported_source( + format!( + "Block compression not yet supported for block type {}", + data.name() + ) + .into(), + )) + } +} diff --git a/rust/lance-file/src/versions/v2_2/mod.rs b/rust/lance-file/src/versions/v2_2/mod.rs new file mode 100644 index 00000000000..b56f78a5c4d --- /dev/null +++ b/rust/lance-file/src/versions/v2_2/mod.rs @@ -0,0 +1,177 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright The Lance Authors + +//! Lance v2.2 file composition. + +use std::{collections::BTreeMap, sync::Arc}; + +use bytes::Bytes; +use lance_core::{ + Error, Result, + datatypes::{Field, Schema}, +}; +use lance_encoding::{ + compression_config::CompressionParams, + encoder::{ + ColumnIndexSequence, EncodedBatch, FieldEncoder, FieldEncodingContext, + FieldEncodingStrategy, + structural::{ + PrimitiveFieldEncoding, PrimitivePageEncoding, try_create_binary_blob, try_create_list, + try_create_map, try_create_struct, try_create_structural_blob, + try_create_structural_fixed_size_list, + }, + }, +}; +use lance_io::traits::Writer as ObjectWriter; + +use crate::{ + reader::{ReadProjection, structural}, + writer::FileWriterOptions, +}; + +mod compression; +mod reader; +mod writer; + +pub(crate) use reader::{ + decode_column_metadata, finish_metadata, finish_metadata_index, validate_global_buffers, +}; +pub use reader::{ + projection_from_column_names, projection_from_field_ids, projection_from_whole_schema, +}; + +pub(crate) fn read_projection() -> Arc { + structural::read_projection(reader::decode_column) +} +pub use writer::Writer; + +/// Count physical columns represented by a field in a v2.2 footer. +pub fn physical_column_count(field: &Field) -> usize { + structural::physical_column_count(field) +} + +/// Build persisted field-to-column entries for a v2.2 data file. +pub fn data_file_columns(schema: &Schema) -> (Vec, Vec) { + structural::data_file_columns(schema) +} + +pub(super) fn field_id_to_column_index(schema: &Schema) -> BTreeMap { + structural::field_id_to_column_index(schema) +} + +#[derive(Debug)] +struct FieldStrategy { + primitive: PrimitiveFieldEncoding, +} + +impl FieldEncodingStrategy for FieldStrategy { + fn create_field_encoder( + &self, + field: &Field, + column_index: &mut ColumnIndexSequence, + context: &FieldEncodingContext<'_>, + ) -> Result> { + if let Some(encoder) = + try_create_binary_blob(&self.primitive, field, column_index, context)? + { + return Ok(encoder); + } + if let Some(encoder) = + try_create_structural_blob(&self.primitive, field, column_index, context)? + { + return Ok(encoder); + } + if field.is_blob() { + return Err(Error::invalid_input_source( + format!( + "Blob encoding is not available for field '{}' with data type {}", + field.name, + field.data_type() + ) + .into(), + )); + } + if let Some(encoder) = try_create_map(field, column_index, context)? { + return Ok(encoder); + } + if let Some(encoder) = try_create_structural_fixed_size_list(field, column_index, context)? + { + return Ok(encoder); + } + if let Some(encoder) = self.primitive.try_create(field, column_index, context)? { + return Ok(encoder); + } + if let Some(encoder) = try_create_list(field, column_index, context)? { + return Ok(encoder); + } + if let Some(encoder) = try_create_struct(field, column_index, context)? { + return Ok(encoder); + } + Err(Error::not_supported_source( + format!( + "Lance v2.2 has no field encoding for '{}' with data type {}", + field.name, + field.data_type() + ) + .into(), + )) + } +} + +/// Compose the v2.2 field encoding mechanisms. +pub fn encoding_strategy(params: CompressionParams) -> Arc { + let compression = Arc::new(compression::Strategy::new(params)); + Arc::new(FieldStrategy { + primitive: PrimitiveFieldEncoding::new([ + PrimitivePageEncoding::reject_sparse(), + PrimitivePageEncoding::constant(), + PrimitivePageEncoding::dense_u32(compression), + ]), + }) +} + +/// Create a v2.2 writer with an explicit schema. +pub fn create_writer( + object_writer: Box, + schema: Schema, + options: FileWriterOptions, +) -> Result { + Writer::try_new(object_writer, schema, options) +} + +/// Create a v2.2 writer with explicit compression tuning. +pub fn create_writer_with_compression( + object_writer: Box, + schema: Schema, + options: FileWriterOptions, + compression: CompressionParams, +) -> Result { + Writer::try_new_with_compression(object_writer, schema, options, compression) +} + +/// Create a v2.2 writer whose schema is inferred from the first batch. +pub fn create_lazy_writer( + object_writer: Box, + options: FileWriterOptions, +) -> Writer { + Writer::new_lazy(object_writer, options) +} + +/// Create a lazy v2.2 writer with explicit compression tuning. +pub fn create_lazy_writer_with_compression( + object_writer: Box, + options: FileWriterOptions, + compression: CompressionParams, +) -> Writer { + Writer::new_lazy_with_compression(object_writer, options, compression) +} + +/// Encode a self-described v2.2 batch. +pub fn encode_self_described_batch(batch: &EncodedBatch) -> Result { + writer::concat_lance_footer(batch, true) +} + +/// Encode a mini-lance v2.2 batch. +pub fn encode_mini_batch(batch: &EncodedBatch) -> Result { + writer::concat_lance_footer(batch, false) +} diff --git a/rust/lance-file/src/versions/v2_2/reader.rs b/rust/lance-file/src/versions/v2_2/reader.rs new file mode 100644 index 00000000000..81a88720c77 --- /dev/null +++ b/rust/lance-file/src/versions/v2_2/reader.rs @@ -0,0 +1,308 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright The Lance Authors + +use std::{collections::BTreeMap, sync::Arc}; + +use lance_core::{Error, Result, datatypes::Schema}; +use lance_encoding::{decoder::ColumnInfo, format::pb21}; + +use crate::{ + format::pbfile, + reader::{ + BufferDescriptor, CachedFileMetadata, FileMetadataIndex, RawFileMetadata, ReaderProjection, + structural, + }, + version::ConcreteFileVersion, +}; + +fn required<'a, T>(value: Option<&'a T>, label: &str) -> Result<&'a T> { + value.ok_or_else(|| { + Error::invalid_input_source( + format!("Lance v2.2 {label} is missing its nested encoding").into(), + ) + }) +} + +fn validate_compressive_encoding(encoding: &pb21::CompressiveEncoding) -> Result<()> { + use pb21::compressive_encoding::Compression; + + match encoding.compression.as_ref() { + Some(Compression::Flat(_)) + | Some(Compression::InlineBitpacking(_)) + | Some(Compression::Constant(_)) => Ok(()), + Some(Compression::Variable(variable)) => validate_compressive_encoding(required( + variable.offsets.as_deref(), + "variable offsets", + )?), + Some(Compression::OutOfLineBitpacking(bitpacking)) => { + validate_compressive_encoding(required( + bitpacking.values.as_deref(), + "out-of-line bitpacking values", + )?) + } + Some(Compression::Fsst(fsst)) => { + validate_compressive_encoding(required(fsst.values.as_deref(), "FSST values")?) + } + Some(Compression::Dictionary(dictionary)) => { + validate_compressive_encoding(required( + dictionary.indices.as_deref(), + "dictionary indices", + )?)?; + validate_compressive_encoding(required( + dictionary.items.as_deref(), + "dictionary items", + )?) + } + Some(Compression::Rle(rle)) => { + let values = required(rle.values.as_deref(), "RLE values")?; + let run_lengths = required(rle.run_lengths.as_deref(), "RLE run lengths")?; + let fixed_values = matches!( + values.compression.as_ref(), + Some(Compression::Flat(flat)) + if matches!(flat.bits_per_value, 8 | 16 | 32 | 64) + && flat.data.is_none() + ); + let fixed_u8_lengths = matches!( + run_lengths.compression.as_ref(), + Some(Compression::Flat(flat)) + if flat.bits_per_value == 8 && flat.data.is_none() + ); + if !fixed_values || !fixed_u8_lengths { + return Err(Error::invalid_input_source( + "Lance v2.2 RLE requires flat values and flat u8 run lengths".into(), + )); + } + Ok(()) + } + Some(Compression::ByteStreamSplit(split)) => validate_compressive_encoding(required( + split.values.as_deref(), + "byte-stream-split values", + )?), + Some(Compression::General(general)) => validate_compressive_encoding(required( + general.values.as_deref(), + "general-compression values", + )?), + Some(Compression::FixedSizeList(list)) => validate_compressive_encoding(required( + list.values.as_deref(), + "fixed-size-list values", + )?), + Some(Compression::PackedStruct(packed)) => validate_compressive_encoding(required( + packed.values.as_deref(), + "packed-struct values", + )?), + Some(Compression::VariablePackedStruct(packed)) => { + for field in &packed.fields { + validate_compressive_encoding(required( + field.value.as_ref(), + "variable packed-struct field", + )?)?; + } + Ok(()) + } + None => Err(Error::invalid_input_source( + "Lance v2.2 compressive encoding is missing its compression variant".into(), + )), + } +} + +fn validate_page_layout(layout: &pb21::PageLayout) -> Result<()> { + use pb21::page_layout::Layout; + + match layout.layout.as_ref() { + Some(Layout::MiniBlockLayout(miniblock)) => { + if !miniblock.has_large_chunk { + return Err(Error::invalid_input_source( + "Lance v2.2 miniblock pages require the u32 chunk grammar".into(), + )); + } + if let Some(rep) = miniblock.rep_compression.as_ref() { + validate_compressive_encoding(rep)?; + } + if let Some(def) = miniblock.def_compression.as_ref() { + validate_compressive_encoding(def)?; + } + validate_compressive_encoding(required( + miniblock.value_compression.as_ref(), + "miniblock values", + )?)?; + if let Some(dictionary) = miniblock.dictionary.as_ref() { + validate_compressive_encoding(dictionary)?; + } + Ok(()) + } + Some(Layout::FullZipLayout(fullzip)) => validate_compressive_encoding(required( + fullzip.value_compression.as_ref(), + "full-zip values", + )?), + Some(Layout::ConstantLayout(constant)) => { + if let Some(rep) = constant.rep_compression.as_ref() { + validate_compressive_encoding(rep)?; + } + if let Some(def) = constant.def_compression.as_ref() { + validate_compressive_encoding(def)?; + } + Ok(()) + } + Some(Layout::BlobLayout(blob)) => { + let inner = blob.inner_layout.as_deref().ok_or_else(|| { + Error::invalid_input_source( + "Lance v2.2 blob page layout is missing its inner layout".into(), + ) + })?; + validate_page_layout(inner) + } + Some(Layout::SparseLayout(_)) => Err(Error::invalid_input_source( + "Sparse page layout is not part of the Lance v2.2 grammar".into(), + )), + None => Err(Error::invalid_input_source( + "Lance v2.2 page is missing its page layout".into(), + )), + } +} + +pub fn decode_column( + column_index: u32, + metadata: &pbfile::ColumnMetadata, +) -> Result> { + let page_infos = metadata + .pages + .iter() + .enumerate() + .map(|(page_index, page)| { + let page_layout = structural::decode_page_layout(column_index, page_index, page)?; + validate_page_layout(&page_layout)?; + structural::build_page_info(column_index, page_index, page, page_layout) + }) + .collect::>>()?; + structural::build_column_info(column_index, metadata, page_infos) +} + +pub fn decode_column_metadata( + column_metadatas: &[pbfile::ColumnMetadata], +) -> Result>> { + column_metadatas + .iter() + .enumerate() + .map(|(column_index, metadata)| { + let column_index = u32::try_from(column_index).map_err(|_| { + Error::invalid_input_source("File has more than u32::MAX columns".into()) + })?; + decode_column(column_index, metadata) + }) + .collect() +} + +pub fn projection_from_field_ids( + schema: &Schema, + field_id_to_column_index: &BTreeMap, +) -> ReaderProjection { + structural::projection_from_field_ids(schema, field_id_to_column_index) +} + +pub fn projection_from_whole_schema(schema: &Schema) -> ReaderProjection { + structural::projection_from_field_ids(schema, &super::field_id_to_column_index(schema)) +} + +pub fn projection_from_column_names( + schema: &Schema, + column_names: &[&str], +) -> Result { + structural::projection_from_column_names( + schema, + column_names, + &super::field_id_to_column_index(schema), + ) +} + +pub fn finish_metadata(raw: RawFileMetadata) -> Result { + if (raw.footer.major_version, raw.footer.minor_version) != (2, 2) { + return Err(Error::version_conflict( + "Attempt to use the Lance v2.2 reader for a different file version".to_string(), + raw.footer.major_version, + raw.footer.minor_version, + )); + } + validate_global_buffers(&raw.file_buffers)?; + let column_infos = decode_column_metadata(&raw.column_metadatas)?; + Ok(CachedFileMetadata { + file_schema: raw.file_schema, + column_metadatas: raw.column_metadatas, + column_infos, + num_rows: raw.num_rows, + file_buffers: raw.file_buffers, + num_data_bytes: raw.num_data_bytes, + num_column_metadata_bytes: raw.num_column_metadata_bytes, + num_global_buffer_bytes: raw.num_global_buffer_bytes, + num_footer_bytes: raw.num_footer_bytes, + major_version: raw.footer.major_version, + minor_version: raw.footer.minor_version, + version: ConcreteFileVersion::V2_2, + file_size_bytes: raw.file_size_bytes, + retained_global_buffers: raw.retained_global_buffers, + }) +} + +pub fn validate_global_buffers(buffers: &[BufferDescriptor]) -> Result<()> { + structural::validate_global_buffers(buffers) +} + +pub fn finish_metadata_index(index: FileMetadataIndex) -> Result { + if index.version != ConcreteFileVersion::V2_2 { + let (major, minor) = index.version.to_standard_footer_numbers(); + return Err(Error::version_conflict( + "Attempt to use the Lance v2.2 reader for a different metadata index".to_string(), + major, + minor, + )); + } + validate_global_buffers(&index.file_buffers)?; + Ok(index) +} + +#[cfg(test)] +mod grammar_tests { + use super::*; + use pb21::{ + CompressiveEncoding, Dictionary, Flat, FullZipLayout, PageLayout, Rle, + compressive_encoding::Compression, page_layout::Layout, + }; + + fn flat(bits_per_value: u64) -> CompressiveEncoding { + CompressiveEncoding { + compression: Some(Compression::Flat(Flat { + bits_per_value, + data: None, + })), + } + } + + #[test] + fn rejects_nested_variable_width_rle() { + let rle = CompressiveEncoding { + compression: Some(Compression::Rle(Box::new(Rle { + values: Some(Box::new(flat(32))), + run_lengths: Some(Box::new(flat(16))), + }))), + }; + let dictionary = CompressiveEncoding { + compression: Some(Compression::Dictionary(Box::new(Dictionary { + indices: Some(Box::new(rle)), + items: Some(Box::new(flat(32))), + num_dictionary_items: 1, + }))), + }; + let layout = PageLayout { + layout: Some(Layout::FullZipLayout(FullZipLayout { + value_compression: Some(dictionary), + ..Default::default() + })), + }; + + let error = validate_page_layout(&layout).unwrap_err(); + assert!( + error + .to_string() + .contains("flat values and flat u8 run lengths") + ); + } +} diff --git a/rust/lance-file/src/versions/v2_2/writer.rs b/rust/lance-file/src/versions/v2_2/writer.rs new file mode 100644 index 00000000000..bfe5494cc1f --- /dev/null +++ b/rust/lance-file/src/versions/v2_2/writer.rs @@ -0,0 +1,223 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright The Lance Authors + +use std::sync::Arc; + +use arrow_array::{ArrayRef, RecordBatch}; +use bytes::{BufMut, Bytes}; +use lance_core::{Result, datatypes::Schema}; +use lance_encoding::{ + compression_config::CompressionParams, + encoder::{BatchEncoder, EncodedBatch}, +}; +use lance_io::{object_store::ObjectStore, traits::Writer as ObjectWriter}; +use object_store::path::Path; +use tokio::io::AsyncWriteExt; + +use crate::{ + format::{MAGIC, pbfile}, + writer::{ + FileWriteSummary, FileWriterOptions, + structural::{EncodedBatchBody, EncodingPipeline, StructuralFileSink, encode_batch_body}, + }, +}; + +use super::encoding_strategy; + +/// A writer for the Lance v2.2 file grammar. +/// +/// The concrete writer owns the v2.2 encoding composition, finish ordering, +/// and exact footer identity. Shared components only execute the structural +/// encoding and I/O mechanisms selected here. +pub struct Writer { + sink: StructuralFileSink, + encoding: EncodingPipeline, + compression: CompressionParams, +} + +impl Writer { + /// Create a v2.2 writer with an explicit schema. + pub fn try_new( + object_writer: Box, + schema: Schema, + options: FileWriterOptions, + ) -> Result { + Self::try_new_with_compression(object_writer, schema, options, Default::default()) + } + + /// Create a v2.2 writer with explicit compression tuning. + pub fn try_new_with_compression( + object_writer: Box, + schema: Schema, + options: FileWriterOptions, + compression: CompressionParams, + ) -> Result { + let mut writer = Self::new_lazy_with_compression(object_writer, options, compression); + writer.initialize(schema)?; + Ok(writer) + } + + /// Create a v2.2 writer whose schema is inferred from the first batch. + pub fn new_lazy(object_writer: Box, options: FileWriterOptions) -> Self { + Self::new_lazy_with_compression(object_writer, options, Default::default()) + } + + /// Create a lazy v2.2 writer with explicit compression tuning. + pub fn new_lazy_with_compression( + object_writer: Box, + options: FileWriterOptions, + compression: CompressionParams, + ) -> Self { + Self { + sink: StructuralFileSink::new(object_writer), + encoding: EncodingPipeline::new(options), + compression, + } + } + + fn initialize(&mut self, schema: Schema) -> Result<()> { + let encoding_options = self.encoding.encoding_options(&schema); + schema.validate()?; + let strategy = encoding_strategy(self.compression.clone()); + let encoder = BatchEncoder::try_new(&schema, strategy.as_ref(), &encoding_options)?; + self.encoding.initialize(schema, encoder, &mut self.sink); + Ok(()) + } + + fn ensure_initialized(&mut self, batch: &RecordBatch) -> Result<()> { + if !self.encoding.is_initialized() { + self.initialize(Schema::try_from(batch.schema().as_ref())?)?; + } + Ok(()) + } + + /// Spill page metadata to a sidecar file instead of retaining it in memory. + pub fn with_page_metadata_spill(mut self, object_store: Arc, path: Path) -> Self { + self.sink.with_page_metadata_spill(object_store, path); + self + } + + /// Schedule batches to be written in iteration order. + pub async fn write_batches( + &mut self, + batches: impl Iterator, + ) -> Result<()> { + for batch in batches { + self.write_batch(batch).await?; + } + Ok(()) + } + + /// Schedule one record batch for writing. + pub async fn write_batch(&mut self, batch: &RecordBatch) -> Result<()> { + self.ensure_initialized(batch)?; + self.encoding.write_batch(batch, &mut self.sink).await + } + + /// Write one top-level field, advancing only that field's row count. + pub async fn write_column(&mut self, column_index: usize, array: ArrayRef) -> Result<()> { + self.encoding + .write_column(column_index, array, &mut self.sink) + .await + } + + /// Append a buffer whose page or column metadata is supplied externally. + pub async fn write_external_buffer(&mut self, bytes: &[u8]) -> Result<(u64, u64)> { + self.sink.write_external_buffer(bytes).await + } + + /// Add an entry to the schema metadata written in the file descriptor. + pub fn add_schema_metadata(&mut self, key: impl Into, value: impl Into) { + self.encoding.add_schema_metadata(key, value); + } + + /// Prepare the writer for encoded column data produced externally. + pub fn initialize_with_external_metadata( + &mut self, + schema: Schema, + column_metadata: Vec, + rows_written: u64, + ) { + self.encoding + .initialize_with_external_metadata(schema, rows_written); + self.sink.initialize_with_external_metadata(column_metadata); + } + + /// Add an arbitrary global buffer and return its one-based index. + pub async fn add_global_buffer(&mut self, buffer: Bytes) -> Result { + self.sink.add_global_buffer(buffer).await + } + + /// Finish the v2.2 file and close its object writer. + pub async fn finish(&mut self) -> Result { + // The order below is the v2.2 wire contract. + self.encoding.flush(&mut self.sink).await?; + self.encoding.finish_encoders(&mut self.sink).await?; + + let descriptor = self.encoding.make_file_descriptor()?; + let global_buffer_offsets = self.sink.write_global_buffers(descriptor).await?; + let num_global_buffers = global_buffer_offsets.len() as u32; + + let column_metadata_start = self.sink.tell().await?; + let column_metadata_offsets = self.sink.write_column_metadatas().await?; + let column_metadata_offsets_start = self + .sink + .write_offset_table(&column_metadata_offsets) + .await?; + let global_buffer_offsets_start = + self.sink.write_offset_table(&global_buffer_offsets).await?; + let num_columns = self.sink.num_columns(); + + let output = self.sink.output_mut(); + output.write_u64_le(column_metadata_start).await?; + output.write_u64_le(column_metadata_offsets_start).await?; + output.write_u64_le(global_buffer_offsets_start).await?; + output.write_u32_le(num_global_buffers).await?; + output.write_u32_le(num_columns).await?; + output.write_u16_le(2).await?; + output.write_u16_le(2).await?; + output.write_all(MAGIC).await?; + + Ok(FileWriteSummary { + num_rows: self.encoding.rows_written(), + size_bytes: self.sink.shutdown().await?, + }) + } + + /// Abandon this write. + pub async fn abort(&mut self) { + // Dropping a multipart ObjectWriter aborts the upload. + } + + /// Return the current object-writer position. + pub async fn tell(&mut self) -> Result { + self.sink.tell().await + } + + /// Return the field-id to physical-column mapping selected by v2.2. + pub fn field_id_to_column_indices(&self) -> &[(u32, u32)] { + self.encoding.field_id_to_column_indices() + } +} + +/// Append a self-described or mini-lance v2.2 footer to an encoded batch. +pub fn concat_lance_footer(batch: &EncodedBatch, write_schema: bool) -> Result { + let EncodedBatchBody { + mut data, + column_metadata_start, + column_metadata_offsets_start, + global_buffer_offsets_start, + num_global_buffers, + num_columns, + } = encode_batch_body(batch, write_schema)?; + + data.put_u64_le(column_metadata_start); + data.put_u64_le(column_metadata_offsets_start); + data.put_u64_le(global_buffer_offsets_start); + data.put_u32_le(num_global_buffers); + data.put_u32_le(num_columns); + data.put_u16_le(2); + data.put_u16_le(2); + data.extend_from_slice(MAGIC); + Ok(data.freeze()) +} diff --git a/rust/lance-file/src/versions/v2_3/compression.rs b/rust/lance-file/src/versions/v2_3/compression.rs new file mode 100644 index 00000000000..731e4a8c084 --- /dev/null +++ b/rust/lance-file/src/versions/v2_3/compression.rs @@ -0,0 +1,130 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright The Lance Authors + +use std::sync::Arc; + +use lance_core::{Error, Result, datatypes::Field}; +use lance_encoding::{ + compression::{ + BlockCompressor, CompressionStrategy, field_metadata_params, finalize_miniblock_compressor, + try_bitpacking_block, try_bitpacking_miniblock, try_byte_stream_split_miniblock, + try_child_rle_miniblock, try_fixed_packed_struct_miniblock, try_general_block, + try_raw_block, try_raw_fixed_size_list_miniblock, try_raw_fixed_width_miniblock, + try_raw_per_value, try_uncompressed_fixed_width_miniblock, + try_variable_packed_struct_per_value, try_variable_rle_block, try_variable_width_miniblock, + try_variable_width_per_value, + }, + compression_config::{CompressionFieldParams, CompressionParams}, + data::DataBlock, + encodings::logical::primitive::{fullzip::PerValueCompressor, miniblock::MiniBlockCompressor}, + format::pb21::CompressiveEncoding, +}; + +#[derive(Debug, Clone)] +pub(super) struct Strategy { + params: CompressionParams, +} + +impl Strategy { + pub(super) fn new(params: CompressionParams) -> Self { + Self { params } + } + + fn field_params(&self, field: &Field) -> CompressionFieldParams { + let mut params = self + .params + .get_field_params(&field.name, &field.data_type()); + params.merge(&field_metadata_params(field)); + params + } +} + +impl CompressionStrategy for Strategy { + fn create_miniblock_compressor( + &self, + field: &Field, + data: &DataBlock, + ) -> Result> { + let params = self.field_params(field); + let compressor = + if let Some(compressor) = try_uncompressed_fixed_width_miniblock(data, ¶ms) { + compressor + } else if let Some(compressor) = try_byte_stream_split_miniblock(data, ¶ms) { + compressor + } else if let Some(compressor) = try_child_rle_miniblock(data, ¶ms) { + compressor + } else if let Some(compressor) = try_bitpacking_miniblock(data) { + compressor + } else if let Some(compressor) = try_raw_fixed_width_miniblock(data) { + compressor + } else if let Some(compressor) = try_variable_width_miniblock(field, data, ¶ms)? { + compressor + } else if let Some(compressor) = try_fixed_packed_struct_miniblock(data)? { + compressor + } else if let Some(compressor) = try_raw_fixed_size_list_miniblock(data) { + compressor + } else { + return Err(Error::not_supported_source( + format!( + "Mini-block compression not yet supported for block type {}", + data.name() + ) + .into(), + )); + }; + finalize_miniblock_compressor(data, compressor, ¶ms) + } + + fn create_per_value( + &self, + field: &Field, + data: &DataBlock, + ) -> Result> { + let params = self.field_params(field); + if let Some(compressor) = try_raw_per_value(data) { + return Ok(compressor); + } + if let Some(compressor) = + try_variable_packed_struct_per_value(Arc::new(self.clone()), field, data)? + { + return Ok(compressor); + } + if let Some(compressor) = try_variable_width_per_value(field, data, ¶ms)? { + return Ok(compressor); + } + Err(Error::not_supported_source( + format!( + "Per-value compression not yet supported for block type {}", + data.name() + ) + .into(), + )) + } + + fn create_block_compressor( + &self, + field: &Field, + data: &DataBlock, + ) -> Result<(Box, CompressiveEncoding)> { + let params = self.field_params(field); + if let Some(compressor) = try_variable_rle_block(data, ¶ms)? { + return Ok(compressor); + } + if let Some(compressor) = try_bitpacking_block(data) { + return Ok(compressor); + } + if let Some(compressor) = try_general_block(data, ¶ms)? { + return Ok(compressor); + } + if let Some(compressor) = try_raw_block(data) { + return Ok(compressor); + } + Err(Error::not_supported_source( + format!( + "Block compression not yet supported for block type {}", + data.name() + ) + .into(), + )) + } +} diff --git a/rust/lance-file/src/versions/v2_3/mod.rs b/rust/lance-file/src/versions/v2_3/mod.rs new file mode 100644 index 00000000000..efbd5008ef6 --- /dev/null +++ b/rust/lance-file/src/versions/v2_3/mod.rs @@ -0,0 +1,200 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright The Lance Authors + +//! Lance v2.3 file composition. + +use std::{ + collections::BTreeMap, + sync::{ + Arc, + atomic::{AtomicBool, Ordering}, + }, +}; + +use bytes::Bytes; +use lance_core::{ + Error, Result, + datatypes::{Field, Schema}, +}; +use lance_encoding::{ + compression_config::CompressionParams, + encoder::{ + ColumnIndexSequence, EncodedBatch, FieldEncoder, FieldEncodingContext, + FieldEncodingStrategy, + structural::{ + PrimitiveFieldEncoding, PrimitivePageEncoding, try_create_binary_blob, try_create_list, + try_create_map, try_create_struct, try_create_structural_blob, + try_create_structural_fixed_size_list, + }, + }, +}; +use lance_io::traits::Writer as ObjectWriter; + +use crate::{ + reader::{ReadProjection, structural}, + writer::FileWriterOptions, +}; + +mod compression; +mod reader; +mod writer; + +pub(crate) use reader::{ + decode_column_metadata, finish_metadata, finish_metadata_index, validate_global_buffers, +}; +pub use reader::{ + projection_from_column_names, projection_from_field_ids, projection_from_whole_schema, +}; + +pub(crate) fn read_projection() -> Arc { + structural::read_projection(reader::decode_column) +} +pub use writer::Writer; + +static WARNED_ON_UNSTABLE_FORMAT: AtomicBool = AtomicBool::new(false); + +/// Count physical columns represented by a field in a v2.3 footer. +pub fn physical_column_count(field: &Field) -> usize { + structural::physical_column_count(field) +} + +/// Build persisted field-to-column entries for a v2.3 data file. +pub fn data_file_columns(schema: &Schema) -> (Vec, Vec) { + structural::data_file_columns(schema) +} + +pub(super) fn field_id_to_column_index(schema: &Schema) -> BTreeMap { + structural::field_id_to_column_index(schema) +} + +#[derive(Debug)] +struct FieldStrategy { + primitive: PrimitiveFieldEncoding, +} + +impl FieldEncodingStrategy for FieldStrategy { + fn create_field_encoder( + &self, + field: &Field, + column_index: &mut ColumnIndexSequence, + context: &FieldEncodingContext<'_>, + ) -> Result> { + if let Some(encoder) = + try_create_binary_blob(&self.primitive, field, column_index, context)? + { + return Ok(encoder); + } + if let Some(encoder) = + try_create_structural_blob(&self.primitive, field, column_index, context)? + { + return Ok(encoder); + } + if field.is_blob() { + return Err(Error::invalid_input_source( + format!( + "Blob encoding is not available for field '{}' with data type {}", + field.name, + field.data_type() + ) + .into(), + )); + } + if let Some(encoder) = try_create_map(field, column_index, context)? { + return Ok(encoder); + } + if let Some(encoder) = try_create_structural_fixed_size_list(field, column_index, context)? + { + return Ok(encoder); + } + if let Some(encoder) = self.primitive.try_create(field, column_index, context)? { + return Ok(encoder); + } + if let Some(encoder) = try_create_list(field, column_index, context)? { + return Ok(encoder); + } + if let Some(encoder) = try_create_struct(field, column_index, context)? { + return Ok(encoder); + } + Err(Error::not_supported_source( + format!( + "Lance v2.3 has no field encoding for '{}' with data type {}", + field.name, + field.data_type() + ) + .into(), + )) + } +} + +/// Compose the v2.3 field encoding mechanisms. +pub fn encoding_strategy(params: CompressionParams) -> Arc { + let compression = Arc::new(compression::Strategy::new(params)); + Arc::new(FieldStrategy { + primitive: PrimitiveFieldEncoding::new([ + PrimitivePageEncoding::sparse(compression.clone()), + PrimitivePageEncoding::constant(), + PrimitivePageEncoding::dense_u32(compression), + ]), + }) +} + +fn warn_unstable_format() { + if WARNED_ON_UNSTABLE_FORMAT + .compare_exchange(false, true, Ordering::Relaxed, Ordering::Relaxed) + .is_ok() + { + log::warn!( + "You have requested an unstable format version. Files written with this format version may not be readable in the future! This is a development feature and should only be used for experimentation and never for production data." + ); + } +} + +/// Create a v2.3 writer with an explicit schema. +pub fn create_writer( + object_writer: Box, + schema: Schema, + options: FileWriterOptions, +) -> Result { + warn_unstable_format(); + Writer::try_new(object_writer, schema, options) +} + +/// Create a v2.3 writer with explicit compression tuning. +pub fn create_writer_with_compression( + object_writer: Box, + schema: Schema, + options: FileWriterOptions, + compression: CompressionParams, +) -> Result { + warn_unstable_format(); + Writer::try_new_with_compression(object_writer, schema, options, compression) +} + +/// Create a v2.3 writer whose schema is inferred from the first batch. +pub fn create_lazy_writer( + object_writer: Box, + options: FileWriterOptions, +) -> Writer { + warn_unstable_format(); + Writer::new_lazy(object_writer, options) +} + +/// Create a lazy v2.3 writer with explicit compression tuning. +pub fn create_lazy_writer_with_compression( + object_writer: Box, + options: FileWriterOptions, + compression: CompressionParams, +) -> Writer { + warn_unstable_format(); + Writer::new_lazy_with_compression(object_writer, options, compression) +} + +/// Encode a self-described v2.3 batch. +pub fn encode_self_described_batch(batch: &EncodedBatch) -> Result { + writer::concat_lance_footer(batch, true) +} + +/// Encode a mini-lance v2.3 batch. +pub fn encode_mini_batch(batch: &EncodedBatch) -> Result { + writer::concat_lance_footer(batch, false) +} diff --git a/rust/lance-file/src/versions/v2_3/reader.rs b/rust/lance-file/src/versions/v2_3/reader.rs new file mode 100644 index 00000000000..cdafbd0b9db --- /dev/null +++ b/rust/lance-file/src/versions/v2_3/reader.rs @@ -0,0 +1,341 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright The Lance Authors + +use std::{collections::BTreeMap, sync::Arc}; + +use lance_core::{Error, Result, datatypes::Schema}; +use lance_encoding::{decoder::ColumnInfo, format::pb21}; + +use crate::{ + format::pbfile, + reader::{ + BufferDescriptor, CachedFileMetadata, FileMetadataIndex, RawFileMetadata, ReaderProjection, + structural, + }, + version::ConcreteFileVersion, +}; + +fn required<'a, T>(value: Option<&'a T>, label: &str) -> Result<&'a T> { + value.ok_or_else(|| { + Error::invalid_input_source( + format!("Lance v2.3 {label} is missing its nested encoding").into(), + ) + }) +} + +fn validate_compressive_encoding(encoding: &pb21::CompressiveEncoding) -> Result<()> { + use pb21::compressive_encoding::Compression; + + match encoding.compression.as_ref() { + Some(Compression::Flat(_)) + | Some(Compression::InlineBitpacking(_)) + | Some(Compression::Constant(_)) => Ok(()), + Some(Compression::Variable(variable)) => validate_compressive_encoding(required( + variable.offsets.as_deref(), + "variable offsets", + )?), + Some(Compression::OutOfLineBitpacking(bitpacking)) => { + validate_compressive_encoding(required( + bitpacking.values.as_deref(), + "out-of-line bitpacking values", + )?) + } + Some(Compression::Fsst(fsst)) => { + validate_compressive_encoding(required(fsst.values.as_deref(), "FSST values")?) + } + Some(Compression::Dictionary(dictionary)) => { + validate_compressive_encoding(required( + dictionary.indices.as_deref(), + "dictionary indices", + )?)?; + validate_compressive_encoding(required( + dictionary.items.as_deref(), + "dictionary items", + )?) + } + Some(Compression::Rle(rle)) => { + validate_compressive_encoding(required(rle.values.as_deref(), "RLE values")?)?; + validate_compressive_encoding(required(rle.run_lengths.as_deref(), "RLE run lengths")?) + } + Some(Compression::ByteStreamSplit(split)) => validate_compressive_encoding(required( + split.values.as_deref(), + "byte-stream-split values", + )?), + Some(Compression::General(general)) => validate_compressive_encoding(required( + general.values.as_deref(), + "general-compression values", + )?), + Some(Compression::FixedSizeList(list)) => validate_compressive_encoding(required( + list.values.as_deref(), + "fixed-size-list values", + )?), + Some(Compression::PackedStruct(packed)) => validate_compressive_encoding(required( + packed.values.as_deref(), + "packed-struct values", + )?), + Some(Compression::VariablePackedStruct(packed)) => { + for field in &packed.fields { + validate_compressive_encoding(required( + field.value.as_ref(), + "variable packed-struct field", + )?)?; + } + Ok(()) + } + None => Err(Error::invalid_input_source( + "Lance v2.3 compressive encoding is missing its compression variant".into(), + )), + } +} + +fn validate_sparse_positions(positions: Option<&pb21::SparsePositionSet>) -> Result<()> { + use pb21::sparse_position_set::Positions; + + if let Some(Positions::Explicit(encoding)) = + positions.and_then(|positions| positions.positions.as_ref()) + { + validate_compressive_encoding(encoding)?; + } + Ok(()) +} + +fn validate_sparse_validity(validity: Option<&pb21::SparseValiditySet>) -> Result<()> { + if let Some(validity) = validity { + validate_sparse_positions(validity.positions.as_ref())?; + } + Ok(()) +} + +fn validate_sparse_layout(layout: &pb21::SparseLayout) -> Result<()> { + use pb21::{sparse_count_set::Counts, sparse_structural_layer::Layer}; + + if !layout.has_large_chunk { + return Err(Error::invalid_input_source( + "Lance v2.3 sparse pages require the u32 chunk grammar".into(), + )); + } + validate_compressive_encoding(required( + layout.value_compression.as_ref(), + "sparse values", + )?)?; + for layer in &layout.structural_layers { + match layer.layer.as_ref() { + Some(Layer::Validity(validity)) => { + validate_sparse_validity(validity.validity.as_ref())? + } + Some(Layer::List(list)) => { + validate_sparse_positions(list.non_empty_positions.as_ref())?; + if let Some(Counts::Explicit(encoding)) = list + .counts + .as_ref() + .and_then(|counts| counts.counts.as_ref()) + { + validate_compressive_encoding(encoding)?; + } + validate_sparse_validity(list.validity.as_ref())?; + } + Some(Layer::FixedSizeList(list)) => validate_sparse_validity(list.validity.as_ref())?, + None => { + return Err(Error::invalid_input_source( + "Lance v2.3 sparse structural layer is missing its layer variant".into(), + )); + } + } + } + Ok(()) +} + +fn validate_page_layout(layout: &pb21::PageLayout) -> Result<()> { + use pb21::page_layout::Layout; + + match layout.layout.as_ref() { + Some(Layout::MiniBlockLayout(miniblock)) => { + if !miniblock.has_large_chunk { + return Err(Error::invalid_input_source( + "Lance v2.3 miniblock pages require the u32 chunk grammar".into(), + )); + } + if let Some(rep) = miniblock.rep_compression.as_ref() { + validate_compressive_encoding(rep)?; + } + if let Some(def) = miniblock.def_compression.as_ref() { + validate_compressive_encoding(def)?; + } + validate_compressive_encoding(required( + miniblock.value_compression.as_ref(), + "miniblock values", + )?)?; + if let Some(dictionary) = miniblock.dictionary.as_ref() { + validate_compressive_encoding(dictionary)?; + } + Ok(()) + } + Some(Layout::FullZipLayout(fullzip)) => validate_compressive_encoding(required( + fullzip.value_compression.as_ref(), + "full-zip values", + )?), + Some(Layout::ConstantLayout(constant)) => { + if let Some(rep) = constant.rep_compression.as_ref() { + validate_compressive_encoding(rep)?; + } + if let Some(def) = constant.def_compression.as_ref() { + validate_compressive_encoding(def)?; + } + Ok(()) + } + Some(Layout::SparseLayout(sparse)) => validate_sparse_layout(sparse), + Some(Layout::BlobLayout(blob)) => { + let inner = blob.inner_layout.as_deref().ok_or_else(|| { + Error::invalid_input_source( + "Lance v2.3 blob page layout is missing its inner layout".into(), + ) + })?; + validate_page_layout(inner) + } + None => Err(Error::invalid_input_source( + "Lance v2.3 page is missing its page layout".into(), + )), + } +} + +pub fn decode_column( + column_index: u32, + metadata: &pbfile::ColumnMetadata, +) -> Result> { + let page_infos = metadata + .pages + .iter() + .enumerate() + .map(|(page_index, page)| { + let page_layout = structural::decode_page_layout(column_index, page_index, page)?; + validate_page_layout(&page_layout)?; + structural::build_page_info(column_index, page_index, page, page_layout) + }) + .collect::>>()?; + structural::build_column_info(column_index, metadata, page_infos) +} + +pub fn decode_column_metadata( + column_metadatas: &[pbfile::ColumnMetadata], +) -> Result>> { + column_metadatas + .iter() + .enumerate() + .map(|(column_index, metadata)| { + let column_index = u32::try_from(column_index).map_err(|_| { + Error::invalid_input_source("File has more than u32::MAX columns".into()) + })?; + decode_column(column_index, metadata) + }) + .collect() +} + +pub fn projection_from_field_ids( + schema: &Schema, + field_id_to_column_index: &BTreeMap, +) -> ReaderProjection { + structural::projection_from_field_ids(schema, field_id_to_column_index) +} + +pub fn projection_from_whole_schema(schema: &Schema) -> ReaderProjection { + structural::projection_from_field_ids(schema, &super::field_id_to_column_index(schema)) +} + +pub fn projection_from_column_names( + schema: &Schema, + column_names: &[&str], +) -> Result { + structural::projection_from_column_names( + schema, + column_names, + &super::field_id_to_column_index(schema), + ) +} + +pub fn finish_metadata(raw: RawFileMetadata) -> Result { + if (raw.footer.major_version, raw.footer.minor_version) != (2, 3) { + return Err(Error::version_conflict( + "Attempt to use the Lance v2.3 reader for a different file version".to_string(), + raw.footer.major_version, + raw.footer.minor_version, + )); + } + validate_global_buffers(&raw.file_buffers)?; + let column_infos = decode_column_metadata(&raw.column_metadatas)?; + Ok(CachedFileMetadata { + file_schema: raw.file_schema, + column_metadatas: raw.column_metadatas, + column_infos, + num_rows: raw.num_rows, + file_buffers: raw.file_buffers, + num_data_bytes: raw.num_data_bytes, + num_column_metadata_bytes: raw.num_column_metadata_bytes, + num_global_buffer_bytes: raw.num_global_buffer_bytes, + num_footer_bytes: raw.num_footer_bytes, + major_version: raw.footer.major_version, + minor_version: raw.footer.minor_version, + version: ConcreteFileVersion::V2_3, + file_size_bytes: raw.file_size_bytes, + retained_global_buffers: raw.retained_global_buffers, + }) +} + +pub fn validate_global_buffers(buffers: &[BufferDescriptor]) -> Result<()> { + structural::validate_global_buffers(buffers) +} + +pub fn finish_metadata_index(index: FileMetadataIndex) -> Result { + if index.version != ConcreteFileVersion::V2_3 { + let (major, minor) = index.version.to_standard_footer_numbers(); + return Err(Error::version_conflict( + "Attempt to use the Lance v2.3 reader for a different metadata index".to_string(), + major, + minor, + )); + } + validate_global_buffers(&index.file_buffers)?; + Ok(index) +} + +#[cfg(test)] +mod grammar_tests { + use super::*; + use pb21::{ + CompressiveEncoding, Dictionary, Flat, FullZipLayout, PageLayout, Rle, + compressive_encoding::Compression, page_layout::Layout, + }; + + fn flat(bits_per_value: u64) -> CompressiveEncoding { + CompressiveEncoding { + compression: Some(Compression::Flat(Flat { + bits_per_value, + data: None, + })), + } + } + + #[test] + fn accepts_nested_variable_width_rle() { + let rle = CompressiveEncoding { + compression: Some(Compression::Rle(Box::new(Rle { + values: Some(Box::new(flat(32))), + run_lengths: Some(Box::new(flat(16))), + }))), + }; + let dictionary = CompressiveEncoding { + compression: Some(Compression::Dictionary(Box::new(Dictionary { + indices: Some(Box::new(rle)), + items: Some(Box::new(flat(32))), + num_dictionary_items: 1, + }))), + }; + let layout = PageLayout { + layout: Some(Layout::FullZipLayout(FullZipLayout { + value_compression: Some(dictionary), + ..Default::default() + })), + }; + + validate_page_layout(&layout).unwrap(); + } +} diff --git a/rust/lance-file/src/versions/v2_3/writer.rs b/rust/lance-file/src/versions/v2_3/writer.rs new file mode 100644 index 00000000000..c18c7aa6e18 --- /dev/null +++ b/rust/lance-file/src/versions/v2_3/writer.rs @@ -0,0 +1,223 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright The Lance Authors + +use std::sync::Arc; + +use arrow_array::{ArrayRef, RecordBatch}; +use bytes::{BufMut, Bytes}; +use lance_core::{Result, datatypes::Schema}; +use lance_encoding::{ + compression_config::CompressionParams, + encoder::{BatchEncoder, EncodedBatch}, +}; +use lance_io::{object_store::ObjectStore, traits::Writer as ObjectWriter}; +use object_store::path::Path; +use tokio::io::AsyncWriteExt; + +use crate::{ + format::{MAGIC, pbfile}, + writer::{ + FileWriteSummary, FileWriterOptions, + structural::{EncodedBatchBody, EncodingPipeline, StructuralFileSink, encode_batch_body}, + }, +}; + +use super::encoding_strategy; + +/// A writer for the Lance v2.3 file grammar. +/// +/// The concrete writer owns the v2.3 encoding composition, finish ordering, +/// and exact footer identity. Shared components only execute the structural +/// encoding and I/O mechanisms selected here. +pub struct Writer { + sink: StructuralFileSink, + encoding: EncodingPipeline, + compression: CompressionParams, +} + +impl Writer { + /// Create a v2.3 writer with an explicit schema. + pub fn try_new( + object_writer: Box, + schema: Schema, + options: FileWriterOptions, + ) -> Result { + Self::try_new_with_compression(object_writer, schema, options, Default::default()) + } + + /// Create a v2.3 writer with explicit compression tuning. + pub fn try_new_with_compression( + object_writer: Box, + schema: Schema, + options: FileWriterOptions, + compression: CompressionParams, + ) -> Result { + let mut writer = Self::new_lazy_with_compression(object_writer, options, compression); + writer.initialize(schema)?; + Ok(writer) + } + + /// Create a v2.3 writer whose schema is inferred from the first batch. + pub fn new_lazy(object_writer: Box, options: FileWriterOptions) -> Self { + Self::new_lazy_with_compression(object_writer, options, Default::default()) + } + + /// Create a lazy v2.3 writer with explicit compression tuning. + pub fn new_lazy_with_compression( + object_writer: Box, + options: FileWriterOptions, + compression: CompressionParams, + ) -> Self { + Self { + sink: StructuralFileSink::new(object_writer), + encoding: EncodingPipeline::new(options), + compression, + } + } + + fn initialize(&mut self, schema: Schema) -> Result<()> { + let encoding_options = self.encoding.encoding_options(&schema); + schema.validate()?; + let strategy = encoding_strategy(self.compression.clone()); + let encoder = BatchEncoder::try_new(&schema, strategy.as_ref(), &encoding_options)?; + self.encoding.initialize(schema, encoder, &mut self.sink); + Ok(()) + } + + fn ensure_initialized(&mut self, batch: &RecordBatch) -> Result<()> { + if !self.encoding.is_initialized() { + self.initialize(Schema::try_from(batch.schema().as_ref())?)?; + } + Ok(()) + } + + /// Spill page metadata to a sidecar file instead of retaining it in memory. + pub fn with_page_metadata_spill(mut self, object_store: Arc, path: Path) -> Self { + self.sink.with_page_metadata_spill(object_store, path); + self + } + + /// Schedule batches to be written in iteration order. + pub async fn write_batches( + &mut self, + batches: impl Iterator, + ) -> Result<()> { + for batch in batches { + self.write_batch(batch).await?; + } + Ok(()) + } + + /// Schedule one record batch for writing. + pub async fn write_batch(&mut self, batch: &RecordBatch) -> Result<()> { + self.ensure_initialized(batch)?; + self.encoding.write_batch(batch, &mut self.sink).await + } + + /// Write one top-level field, advancing only that field's row count. + pub async fn write_column(&mut self, column_index: usize, array: ArrayRef) -> Result<()> { + self.encoding + .write_column(column_index, array, &mut self.sink) + .await + } + + /// Append a buffer whose page or column metadata is supplied externally. + pub async fn write_external_buffer(&mut self, bytes: &[u8]) -> Result<(u64, u64)> { + self.sink.write_external_buffer(bytes).await + } + + /// Add an entry to the schema metadata written in the file descriptor. + pub fn add_schema_metadata(&mut self, key: impl Into, value: impl Into) { + self.encoding.add_schema_metadata(key, value); + } + + /// Prepare the writer for encoded column data produced externally. + pub fn initialize_with_external_metadata( + &mut self, + schema: Schema, + column_metadata: Vec, + rows_written: u64, + ) { + self.encoding + .initialize_with_external_metadata(schema, rows_written); + self.sink.initialize_with_external_metadata(column_metadata); + } + + /// Add an arbitrary global buffer and return its one-based index. + pub async fn add_global_buffer(&mut self, buffer: Bytes) -> Result { + self.sink.add_global_buffer(buffer).await + } + + /// Finish the v2.3 file and close its object writer. + pub async fn finish(&mut self) -> Result { + // The order below is the v2.3 wire contract. + self.encoding.flush(&mut self.sink).await?; + self.encoding.finish_encoders(&mut self.sink).await?; + + let descriptor = self.encoding.make_file_descriptor()?; + let global_buffer_offsets = self.sink.write_global_buffers(descriptor).await?; + let num_global_buffers = global_buffer_offsets.len() as u32; + + let column_metadata_start = self.sink.tell().await?; + let column_metadata_offsets = self.sink.write_column_metadatas().await?; + let column_metadata_offsets_start = self + .sink + .write_offset_table(&column_metadata_offsets) + .await?; + let global_buffer_offsets_start = + self.sink.write_offset_table(&global_buffer_offsets).await?; + let num_columns = self.sink.num_columns(); + + let output = self.sink.output_mut(); + output.write_u64_le(column_metadata_start).await?; + output.write_u64_le(column_metadata_offsets_start).await?; + output.write_u64_le(global_buffer_offsets_start).await?; + output.write_u32_le(num_global_buffers).await?; + output.write_u32_le(num_columns).await?; + output.write_u16_le(2).await?; + output.write_u16_le(3).await?; + output.write_all(MAGIC).await?; + + Ok(FileWriteSummary { + num_rows: self.encoding.rows_written(), + size_bytes: self.sink.shutdown().await?, + }) + } + + /// Abandon this write. + pub async fn abort(&mut self) { + // Dropping a multipart ObjectWriter aborts the upload. + } + + /// Return the current object-writer position. + pub async fn tell(&mut self) -> Result { + self.sink.tell().await + } + + /// Return the field-id to physical-column mapping selected by v2.3. + pub fn field_id_to_column_indices(&self) -> &[(u32, u32)] { + self.encoding.field_id_to_column_indices() + } +} + +/// Append a self-described or mini-lance v2.3 footer to an encoded batch. +pub fn concat_lance_footer(batch: &EncodedBatch, write_schema: bool) -> Result { + let EncodedBatchBody { + mut data, + column_metadata_start, + column_metadata_offsets_start, + global_buffer_offsets_start, + num_global_buffers, + num_columns, + } = encode_batch_body(batch, write_schema)?; + + data.put_u64_le(column_metadata_start); + data.put_u64_le(column_metadata_offsets_start); + data.put_u64_le(global_buffer_offsets_start); + data.put_u32_le(num_global_buffers); + data.put_u32_le(num_columns); + data.put_u16_le(2); + data.put_u16_le(3); + data.extend_from_slice(MAGIC); + Ok(data.freeze()) +} diff --git a/rust/lance-file/src/writer.rs b/rust/lance-file/src/writer.rs index f7042b6b99c..dcba2658225 100644 --- a/rust/lance-file/src/writer.rs +++ b/rust/lance-file/src/writer.rs @@ -1,53 +1,24 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright The Lance Authors -use core::panic; -use std::collections::HashMap; use std::sync::Arc; -use std::sync::atomic::AtomicBool; use arrow_array::{ArrayRef, RecordBatch}; - -use arrow_data::ArrayData; -use bytes::{Buf, BufMut, Bytes, BytesMut}; -use futures::StreamExt; -use futures::stream::FuturesOrdered; -use lance_core::datatypes::{Field, Schema as LanceSchema}; -use lance_core::utils::bit::pad_bytes; -use lance_core::{Error, Result}; -use lance_encoding::decoder::PageEncoding; -use lance_encoding::encoder::{ - BatchEncoder, EncodeTask, EncodedBatch, EncodedPage, EncodingOptions, FieldEncoder, - FieldEncodingStrategy, OutOfLineBuffers, default_encoding_strategy, -}; -use lance_encoding::repdef::RepDefBuilder; -use lance_encoding::version::LanceFileVersion; +use bytes::Bytes; +use lance_core::{Result, datatypes::Schema}; +use lance_encoding::decoder::{ColumnInfo, PageEncoding}; use lance_io::object_store::ObjectStore; -use lance_io::traits::Writer; -use log::{debug, warn}; use object_store::path::Path; use prost::Message; use prost_types::Any; -use tokio::io::AsyncWrite; -use tokio::io::AsyncWriteExt; -use tracing::instrument; -use crate::datatypes::FieldsWithMeta; -use crate::format::MAGIC; -use crate::format::pb; -use crate::format::pbfile; -use crate::format::pbfile::DirectEncoding; +use crate::{format::pbfile, versions}; + +pub(crate) mod structural; -/// Pages buffers are aligned to 64 bytes +/// Page buffers in current Lance files are aligned to 64 bytes. pub(crate) const PAGE_BUFFER_ALIGNMENT: usize = 64; -const PAD_BUFFER: [u8; PAGE_BUFFER_ALIGNMENT] = [72; PAGE_BUFFER_ALIGNMENT]; -// In 2.1+, we split large pages on read instead of write to avoid empty pages -// and small pages issues. However, we keep the write-time limit at 32MB to avoid -// potential regressions in 2.0 format readers. -// -// This limit is not applied in the 2.1 writer -const MAX_PAGE_BYTES: usize = 32 * 1024 * 1024; -const ENV_LANCE_FILE_WRITER_MAX_PAGE_BYTES: &str = "LANCE_FILE_WRITER_MAX_PAGE_BYTES"; +pub(crate) const ENV_LANCE_FILE_WRITER_MAX_PAGE_BYTES: &str = "LANCE_FILE_WRITER_MAX_PAGE_BYTES"; /// Summary of a completed Lance file write. #[derive(Debug, Clone, Copy, PartialEq, Eq)] @@ -58,345 +29,120 @@ pub struct FileWriteSummary { pub size_bytes: u64, } +/// Runtime options shared by all current-format writers. +/// +/// These options control buffering and execution only. Select the file grammar +/// by constructing a writer under [`crate::versions`]. #[derive(Debug, Clone, Default)] pub struct FileWriterOptions { - /// How many bytes to use for buffering column data + /// How many bytes to use for buffering column data. /// - /// When data comes in small batches the writer will buffer column data so that - /// larger pages can be created. This value will be divided evenly across all of the - /// columns. Generally you want this to be at least large enough to match your - /// filesystem's ideal read size per column. - /// - /// In some cases you might want this value to be even larger if you have highly - /// compressible data. However, if this is too large, then the writer could require - /// a lot of memory and write performance may suffer if the CPU-expensive encoding - /// falls behind and can't be interleaved with the I/O expensive flushing. - /// - /// The default will use 8MiB per column which should be reasonable for most cases. - // TODO: Do we need to be able to set this on a per-column basis? + /// The budget is divided evenly across columns. The default is 8 MiB per + /// column. pub data_cache_bytes: Option, - /// A hint to indicate the max size of a page - /// - /// This hint can't always be respected. A single value could be larger than this value - /// and we never slice single values. In addition, there are some cases where it can be - /// difficult to know size up-front and so we might not be able to respect this value. + /// A best-effort maximum encoded page size. pub max_page_bytes: Option, - /// The file writer buffers columns until enough data has arrived to flush a page - /// to disk. + /// Keep input arrays instead of copying buffered slices. /// - /// Some columns with small data types may not flush very often. These arrays can - /// stick around for a long time. These arrays might also be keeping larger data - /// structures alive. By default, the writer will make a deep copy of this array - /// to avoid any potential memory leaks. However, this can be disabled for a - /// (probably minor) performance boost if you are sure that arrays are not keeping - /// any sibling structures alive (this typically means the array was allocated in - /// the same language / runtime as the writer) - /// - /// Do not enable this if your data is arriving from the C data interface. - /// Data typically arrives one "batch" at a time (encoded in the C data interface - /// as a struct array). Each array in that batch keeps the entire batch alive. - /// This means a small boolean array (which we will buffer in memory for quite a - /// while) might keep a much larger record batch around in memory (even though most - /// of that batch's data has been written to disk) + /// Do not enable this for arrays arriving through the Arrow C data + /// interface because a small child array can keep an entire batch alive. pub keep_original_array: Option, - pub encoding_strategy: Option>, - /// The format version to use when writing the file - /// - /// This controls which encodings will be used when encoding the data. Newer - /// versions may have more efficient encodings. However, newer format versions will - /// require more up-to-date readers to read the data. - pub format_version: Option, } -// Total in-memory budget for buffering serialized page metadata before flushing -// to the spill file. Divided evenly across columns (with a floor of 64 bytes). -const DEFAULT_SPILL_BUFFER_LIMIT: usize = 256 * 1024; - -/// Spills serialized page metadata to a temporary file to bound memory usage. +/// A type-erased current-format file writer. /// -/// The spill file is an unstructured sequence of "chunks". Each chunk is a -/// contiguous run of length-delimited protobuf `Page` messages belonging to a -/// single column. Chunks from different columns are interleaved in the order -/// they are flushed (i.e. whenever a column's in-memory buffer exceeds -/// `per_column_limit`). The `column_chunks` index records the (offset, length) -/// of every chunk so each column's pages can be read back and reassembled in -/// order. -struct PageMetadataSpill { - writer: Box, - object_store: Arc, - path: Path, - /// Current write position in the spill file. - position: u64, - /// Per-column buffer of serialized (length-delimited protobuf) page metadata - /// that has not yet been flushed to the spill file. - column_buffers: Vec>, - /// Per-column list of chunks that have been flushed to the spill file. - /// Each entry is (offset, length) pointing into the spill file. - column_chunks: Vec>, - /// Maximum bytes to buffer per column before flushing to the spill file. - per_column_limit: usize, +/// This enum exists for callers that select a concrete file version at +/// runtime. Each variant owns the complete implementation for exactly one file +/// grammar; this type only forwards operations without adding format policy. +pub enum FileWriter { + V2_0(Box), + V2_1(Box), + V2_2(Box), + V2_3(Box), } -impl PageMetadataSpill { - async fn new(object_store: Arc, path: Path, num_columns: usize) -> Result { - let writer = object_store.create(&path).await?; - let per_column_limit = (DEFAULT_SPILL_BUFFER_LIMIT / num_columns.max(1)).max(64); - Ok(Self { - writer, - object_store, - path, - position: 0, - column_buffers: vec![Vec::new(); num_columns], - column_chunks: vec![Vec::new(); num_columns], - per_column_limit, +fn column_info_to_metadata(column: &ColumnInfo) -> Result { + let pages = column + .page_infos + .iter() + .map(|page| { + let encoding = match &page.encoding { + PageEncoding::Legacy(encoding) => Any::from_msg(encoding)?.encode_to_vec(), + PageEncoding::Structural(encoding) => Any::from_msg(encoding)?.encode_to_vec(), + }; + let (buffer_offsets, buffer_sizes) = + page.buffer_offsets_and_sizes.iter().copied().unzip(); + Ok(pbfile::column_metadata::Page { + buffer_offsets, + buffer_sizes, + encoding: Some(pbfile::Encoding { + location: Some(pbfile::encoding::Location::Direct(pbfile::DirectEncoding { + encoding, + })), + }), + length: page.num_rows, + priority: page.priority, + }) }) - } - - async fn append_page( - &mut self, - column_idx: usize, - page: &pbfile::column_metadata::Page, - ) -> Result<()> { - page.encode_length_delimited(&mut self.column_buffers[column_idx]) - .map_err(|e| { - Error::io_source(Box::new(std::io::Error::new( - std::io::ErrorKind::InvalidData, - e, - ))) - })?; - if self.column_buffers[column_idx].len() >= self.per_column_limit { - self.flush_column(column_idx).await?; - } - Ok(()) - } - - async fn flush_column(&mut self, column_idx: usize) -> Result<()> { - let buf = &self.column_buffers[column_idx]; - if buf.is_empty() { - return Ok(()); - } - let len = buf.len(); - self.writer.write_all(buf).await?; - self.column_chunks[column_idx].push((self.position, len as u32)); - self.position += len as u64; - self.column_buffers[column_idx].clear(); - Ok(()) - } - - async fn shutdown_writer(&mut self) -> Result<()> { - for col_idx in 0..self.column_buffers.len() { - self.flush_column(col_idx).await?; - } - Writer::shutdown(self.writer.as_mut()).await?; - Ok(()) - } + .collect::>>()?; + let (buffer_offsets, buffer_sizes) = column.buffer_offsets_and_sizes.iter().copied().unzip(); + let encoding = Any::from_msg(&column.encoding)?.encode_to_vec(); + Ok(pbfile::ColumnMetadata { + pages, + buffer_offsets, + buffer_sizes, + encoding: Some(pbfile::Encoding { + location: Some(pbfile::encoding::Location::Direct(pbfile::DirectEncoding { + encoding, + })), + }), + }) } -fn decode_spilled_chunk(data: &Bytes) -> Result> { - let mut pages = Vec::new(); - let mut cursor = data.clone(); - while cursor.has_remaining() { - let page = - pbfile::column_metadata::Page::decode_length_delimited(&mut cursor).map_err(|e| { - Error::io_source(Box::new(std::io::Error::new( - std::io::ErrorKind::InvalidData, - e, - ))) - })?; - pages.push(page); +impl From for FileWriter { + fn from(writer: versions::v2_0::Writer) -> Self { + Self::V2_0(Box::new(writer)) } - Ok(pages) } -enum PageSpillState { - Pending(Arc, Path), - Active(PageMetadataSpill), +impl From for FileWriter { + fn from(writer: versions::v2_1::Writer) -> Self { + Self::V2_1(Box::new(writer)) + } } -pub struct FileWriter { - writer: Box, - schema: Option, - column_writers: Vec>, - column_metadata: Vec, - field_id_to_column_indices: Vec<(u32, u32)>, - num_columns: u32, - rows_written: u64, - // The number of rows written for each top-level field (i.e. each entry in - // `column_writers`). With `write_batch` every field advances together and - // these are all equal, but `write_column` advances one field at a time, so - // a single file may end up with columns of differing item counts. - field_rows_written: Vec, - global_buffers: Vec<(u64, u64)>, - schema_metadata: HashMap, - options: FileWriterOptions, - page_spill: Option, +impl From for FileWriter { + fn from(writer: versions::v2_2::Writer) -> Self { + Self::V2_2(Box::new(writer)) + } } -fn initial_column_metadata() -> pbfile::ColumnMetadata { - pbfile::ColumnMetadata { - pages: Vec::new(), - buffer_offsets: Vec::new(), - buffer_sizes: Vec::new(), - encoding: None, +impl From for FileWriter { + fn from(writer: versions::v2_3::Writer) -> Self { + Self::V2_3(Box::new(writer)) } } -static WARNED_ON_UNSTABLE_API: AtomicBool = AtomicBool::new(false); - impl FileWriter { - /// Create a new FileWriter with a desired output schema - pub fn try_new( - object_writer: Box, - schema: LanceSchema, - options: FileWriterOptions, - ) -> Result { - let mut writer = Self::new_lazy(object_writer, options); - writer.initialize(schema)?; - Ok(writer) - } - - /// Create a new FileWriter without a desired output schema - /// - /// The output schema will be set based on the first batch of data to arrive. - /// If no data arrives and the writer is finished then the write will fail. - pub fn new_lazy(object_writer: Box, options: FileWriterOptions) -> Self { - if let Some(format_version) = options.format_version - && format_version.is_unstable() - && WARNED_ON_UNSTABLE_API - .compare_exchange( - false, - true, - std::sync::atomic::Ordering::Relaxed, - std::sync::atomic::Ordering::Relaxed, - ) - .is_ok() - { - warn!( - "You have requested an unstable format version. Files written with this format version may not be readable in the future! This is a development feature and should only be used for experimentation and never for production data." - ); - } - Self { - writer: object_writer, - schema: None, - column_writers: Vec::new(), - column_metadata: Vec::new(), - num_columns: 0, - rows_written: 0, - field_rows_written: Vec::new(), - field_id_to_column_indices: Vec::new(), - global_buffers: Vec::new(), - schema_metadata: HashMap::new(), - page_spill: None, - options, - } - } - - /// Spill page metadata to a sidecar file instead of accumulating in memory. - /// - /// This can dramatically reduce memory usage when many writers are open - /// concurrently (e.g. IVF shuffle with thousands of partition writers). - /// The sidecar file is created lazily on the first page write. The caller - /// is responsible for cleaning up `path` (e.g. by placing it in a temp - /// directory that is removed via RAII). - pub fn with_page_metadata_spill(mut self, object_store: Arc, path: Path) -> Self { - self.page_spill = Some(PageSpillState::Pending(object_store, path)); - self - } - - /// Write a series of record batches to a new file - /// - /// Returns the number of rows written - pub async fn create_file_with_batches( - store: &ObjectStore, - path: &Path, - schema: lance_core::datatypes::Schema, - batches: impl Iterator + Send, - options: FileWriterOptions, - ) -> Result { - let writer = store.create(path).await?; - let mut writer = Self::try_new(writer, schema, options)?; - for batch in batches { - writer.write_batch(&batch).await?; - } - Ok(writer.finish().await?.num_rows as usize) - } - - async fn do_write_buffer(writer: &mut (impl AsyncWrite + Unpin), buf: &[u8]) -> Result<()> { - writer.write_all(buf).await?; - let pad_bytes = pad_bytes::(buf.len()); - writer.write_all(&PAD_BUFFER[..pad_bytes]).await?; - Ok(()) - } - - /// Returns the format version that will be used when writing the file - pub fn version(&self) -> LanceFileVersion { - self.options.format_version.unwrap_or_default() - } - - async fn write_page(&mut self, encoded_page: EncodedPage) -> Result<()> { - let buffers = encoded_page.data; - let mut buffer_offsets = Vec::with_capacity(buffers.len()); - let mut buffer_sizes = Vec::with_capacity(buffers.len()); - for buffer in buffers { - buffer_offsets.push(self.writer.tell().await? as u64); - buffer_sizes.push(buffer.len() as u64); - Self::do_write_buffer(&mut self.writer, &buffer).await?; - } - let encoded_encoding = match encoded_page.description { - PageEncoding::Legacy(array_encoding) => Any::from_msg(&array_encoding)?.encode_to_vec(), - PageEncoding::Structural(page_layout) => Any::from_msg(&page_layout)?.encode_to_vec(), - }; - let page = pbfile::column_metadata::Page { - buffer_offsets, - buffer_sizes, - encoding: Some(pbfile::Encoding { - location: Some(pbfile::encoding::Location::Direct(DirectEncoding { - encoding: encoded_encoding, - })), - }), - length: encoded_page.num_rows, - priority: encoded_page.row_number, - }; - let col_idx = encoded_page.column_idx as usize; - if matches!(&self.page_spill, Some(PageSpillState::Pending(..))) { - let Some(PageSpillState::Pending(store, path)) = self.page_spill.take() else { - unreachable!() - }; - self.page_spill = Some(PageSpillState::Active( - PageMetadataSpill::new(store, path, self.num_columns as usize).await?, - )); - } - match &mut self.page_spill { - Some(PageSpillState::Active(spill)) => spill.append_page(col_idx, &page).await?, - None => self.column_metadata[col_idx].pages.push(page), - Some(PageSpillState::Pending(..)) => unreachable!(), - } - Ok(()) - } - - #[instrument(skip_all, level = "debug")] - async fn write_pages(&mut self, mut encoding_tasks: FuturesOrdered) -> Result<()> { - // As soon as an encoding task is done we write it. There is no parallelism - // needed here because "writing" is really just submitting the buffer to the - // underlying write scheduler (either the OS or object_store's scheduler for - // cloud writes). The only time we might truly await on write_page is if the - // scheduler's write queue is full. - // - // Also, there is no point in trying to make write_page parallel anyways - // because we wouldn't want buffers getting mixed up across pages. - while let Some(encoding_task) = encoding_tasks.next().await { - let encoded_page = encoding_task?; - self.write_page(encoded_page).await?; - } - // It's important to flush here, we don't know when the next batch will arrive - // and the underlying cloud store could have writes in progress that won't advance - // until we interact with the writer again. These in-progress writes will time out - // if we don't flush. - self.writer.flush().await?; - Ok(()) - } - - /// Schedule batches of data to be written to the file + /// Spill page metadata to a sidecar file. + pub fn with_page_metadata_spill(self, object_store: Arc, path: Path) -> Self { + match self { + Self::V2_0(writer) => Self::V2_0(Box::new( + (*writer).with_page_metadata_spill(object_store, path), + )), + Self::V2_1(writer) => Self::V2_1(Box::new( + (*writer).with_page_metadata_spill(object_store, path), + )), + Self::V2_2(writer) => Self::V2_2(Box::new( + (*writer).with_page_metadata_spill(object_store, path), + )), + Self::V2_3(writer) => Self::V2_3(Box::new( + (*writer).with_page_metadata_spill(object_store, path), + )), + } + } + + /// Schedule batches of data to be written to the file. pub async fn write_batches( &mut self, batches: impl Iterator, @@ -407,2056 +153,127 @@ impl FileWriter { Ok(()) } - fn verify_field_nullability(arr: &ArrayData, field: &Field) -> Result<()> { - if !field.nullable && arr.null_count() > 0 { - return Err(Error::invalid_input(format!( - "The field `{}` contained null values even though the field is marked non-null in the schema", - field.name - ))); - } - - for (child_field, child_arr) in field.children.iter().zip(arr.child_data()) { - Self::verify_field_nullability(child_arr, child_field)?; - } - - Ok(()) - } - - fn verify_nullability_constraints(&self, batch: &RecordBatch) -> Result<()> { - for (col, field) in batch - .columns() - .iter() - .zip(self.schema.as_ref().unwrap().fields.iter()) - { - Self::verify_field_nullability(&col.to_data(), field)?; - } - Ok(()) - } - - fn initialize(&mut self, mut schema: LanceSchema) -> Result<()> { - let cache_bytes_per_column = if let Some(data_cache_bytes) = self.options.data_cache_bytes { - data_cache_bytes / schema.fields.len() as u64 - } else { - 8 * 1024 * 1024 - }; - - let max_page_bytes = self.options.max_page_bytes.unwrap_or_else(|| { - std::env::var(ENV_LANCE_FILE_WRITER_MAX_PAGE_BYTES) - .map(|s| { - s.parse::().unwrap_or_else(|e| { - warn!( - "Failed to parse {}: {}, using default", - ENV_LANCE_FILE_WRITER_MAX_PAGE_BYTES, e - ); - MAX_PAGE_BYTES as u64 - }) - }) - .unwrap_or(MAX_PAGE_BYTES as u64) - }); - - schema.validate()?; - - let keep_original_array = self.options.keep_original_array.unwrap_or(false); - let encoding_strategy = self.options.encoding_strategy.clone().unwrap_or_else(|| { - let version = self.version(); - default_encoding_strategy(version).into() - }); - - let encoding_options = EncodingOptions { - cache_bytes_per_column, - max_page_bytes, - keep_original_array, - buffer_alignment: PAGE_BUFFER_ALIGNMENT as u64, - version: self.version(), - }; - let encoder = - BatchEncoder::try_new(&schema, encoding_strategy.as_ref(), &encoding_options)?; - self.num_columns = encoder.num_columns(); - - self.field_rows_written = vec![0; encoder.field_encoders.len()]; - self.column_writers = encoder.field_encoders; - self.column_metadata = vec![initial_column_metadata(); self.num_columns as usize]; - self.field_id_to_column_indices = encoder.field_id_to_column_index; - self.schema_metadata - .extend(std::mem::take(&mut schema.metadata)); - self.schema = Some(schema); - Ok(()) - } - - fn ensure_initialized(&mut self, batch: &RecordBatch) -> Result<&LanceSchema> { - if self.schema.is_none() { - let schema = LanceSchema::try_from(batch.schema().as_ref())?; - self.initialize(schema)?; - } - Ok(self.schema.as_ref().unwrap()) - } - - #[instrument(skip_all, level = "debug")] - fn encode_batch( - &mut self, - batch: &RecordBatch, - external_buffers: &mut OutOfLineBuffers, - ) -> Result>> { - let field_arrays = self - .schema - .as_ref() - .unwrap() - .fields - .iter() - .enumerate() - .map(|(field_idx, field)| { - let array = - batch - .column_by_name(&field.name) - .ok_or(Error::invalid_input_source( - format!( - "Cannot write batch. The batch was missing the column `{}`", - field.name - ) - .into(), - ))?; - Ok((field_idx, array.clone())) - }) - .collect::>>()?; - self.encode_columns(&field_arrays, external_buffers) - } - - // Encode a set of `(field index, array)` pairs, each advancing only its own - // column. Each task captures its field's current row offset at encode time, - // so `advance_columns` must run after this call (never before); the order of - // the returned tasks relative to `write_pages` does not matter. - fn encode_columns( - &mut self, - field_arrays: &[(usize, ArrayRef)], - external_buffers: &mut OutOfLineBuffers, - ) -> Result>> { - // Snapshot the starting row number of each field before borrowing the - // column writers mutably below. - let row_numbers = field_arrays - .iter() - .map(|(field_idx, _)| self.field_rows_written[*field_idx]) - .collect::>(); - field_arrays - .iter() - .zip(row_numbers) - .map(|((field_idx, array), row_number)| { - let repdef = RepDefBuilder::default(); - let num_rows = array.len() as u64; - self.column_writers[*field_idx].maybe_encode( - array.clone(), - external_buffers, - repdef, - row_number, - num_rows, - ) - }) - .collect::>>() - } - - // Advance the per-field row counters after a set of columns has been - // written, keeping `rows_written` (the file's logical length) in sync as the - // longest column. Only the written fields move, so their new totals fold into - // `rows_written` directly without rescanning every field. (`write_batch` - // advances every field uniformly and tracks this inline instead.) - fn advance_columns(&mut self, field_arrays: &[(usize, ArrayRef)]) { - for (field_idx, array) in field_arrays { - let new_total = self.field_rows_written[*field_idx] + array.len() as u64; - self.field_rows_written[*field_idx] = new_total; - self.rows_written = self.rows_written.max(new_total); - } - } - - /// Schedule a batch of data to be written to the file - /// - /// Note: the future returned by this method may complete before the data has been fully - /// flushed to the file (some data may be in the data cache or the I/O cache) + /// Schedule a batch of data to be written to the file. pub async fn write_batch(&mut self, batch: &RecordBatch) -> Result<()> { - debug!( - "write_batch called with {} rows, {} columns, and {} bytes of data", - batch.num_rows(), - batch.num_columns(), - batch.get_array_memory_size() - ); - self.ensure_initialized(batch)?; - self.verify_nullability_constraints(batch)?; - let num_rows = batch.num_rows() as u64; - if num_rows == 0 { - return Ok(()); - } - if num_rows > u32::MAX as u64 { - return Err(Error::invalid_input_source( - "cannot write Lance files with more than 2^32 rows".into(), - )); - } - // First we push each array into its column writer. This may or may not generate enough - // data to trigger an encoding task. We collect any encoding tasks into a queue. - let mut external_buffers = - OutOfLineBuffers::new(self.tell().await?, PAGE_BUFFER_ALIGNMENT as u64); - let encoding_tasks = self.encode_batch(batch, &mut external_buffers)?; - // Next, write external buffers - for external_buffer in external_buffers.take_buffers() { - Self::do_write_buffer(&mut self.writer, &external_buffer).await?; - } - - let encoding_tasks = encoding_tasks - .into_iter() - .flatten() - .collect::>(); - - // `write_batch` advances every field by the same amount, so the longest - // column simply grows by `num_rows`. Guard against overflowing the row - // counter. - if self.rows_written.checked_add(num_rows).is_none() { - return Err(Error::invalid_input_source(format!("cannot write batch with {} rows because {} rows have already been written and Lance files cannot contain more than 2^64 rows", num_rows, self.rows_written).into())); - } - for field_rows in self.field_rows_written.iter_mut() { - *field_rows += num_rows; + match self { + Self::V2_0(writer) => writer.write_batch(batch).await, + Self::V2_1(writer) => writer.write_batch(batch).await, + Self::V2_2(writer) => writer.write_batch(batch).await, + Self::V2_3(writer) => writer.write_batch(batch).await, } - self.rows_written += num_rows; - - self.write_pages(encoding_tasks).await?; - - Ok(()) } - /// Write a single column, advancing only that column's row counter. - /// - /// Unlike [`write_batch`](Self::write_batch), which advances every column - /// from a single shared row counter, this method advances one column - /// independently. Used across calls it produces a single file whose columns - /// may have different item counts. - /// - /// `column_index` refers to a top-level field in the writer's schema (the - /// same order as the schema's fields); a nested child cannot be targeted on - /// its own. Because each call writes the whole field from a single array, the - /// children of a struct field always advance together and stay equal-length; - /// only different top-level fields can diverge in length. A column may be - /// written across multiple calls; its values are appended. A field that is - /// never written ends up as a zero-length column. The writer must have been - /// created with an explicit schema (via [`try_new`](Self::try_new)); a lazy - /// schema cannot be inferred here because individual calls need not cover - /// every field. - /// - /// ``` - /// # use arrow_array::{ArrayRef, Int32Array}; - /// # use std::sync::Arc; - /// # use lance_file::writer::FileWriter; - /// # async fn example(writer: &mut FileWriter) -> lance_core::Result<()> { - /// // Field 0 gets three values, field 1 gets one — a non-rectangular file. - /// writer.write_column(0, Arc::new(Int32Array::from(vec![1, 2, 3]))).await?; - /// writer.write_column(1, Arc::new(Int32Array::from(vec![10]))).await?; - /// # Ok(()) - /// # } - /// ``` + /// Write one top-level column. pub async fn write_column(&mut self, column_index: usize, array: ArrayRef) -> Result<()> { - let schema = self.schema.as_ref().ok_or_else(|| { - Error::invalid_input_source( - "write_column requires the writer to be created with an explicit schema".into(), - ) - })?; - let field = schema.fields.get(column_index).ok_or_else(|| { - Error::invalid_input_source( - format!( - "write_column: field index {} is out of bounds (schema has {} fields)", - column_index, - schema.fields.len() - ) - .into(), - ) - })?; - if array.len() as u64 > u32::MAX as u64 { - return Err(Error::invalid_input_source( - "cannot write Lance files with more than 2^32 rows".into(), - )); - } - Self::verify_field_nullability(&array.to_data(), field)?; - - // A never-advanced field simply remains a zero-length column, which the - // encoders handle at `finish` time. - if array.is_empty() { - return Ok(()); - } - - let columns = [(column_index, array)]; - let mut external_buffers = - OutOfLineBuffers::new(self.tell().await?, PAGE_BUFFER_ALIGNMENT as u64); - let encoding_tasks = self.encode_columns(&columns, &mut external_buffers)?; - for external_buffer in external_buffers.take_buffers() { - Self::do_write_buffer(&mut self.writer, &external_buffer).await?; + match self { + Self::V2_0(writer) => writer.write_column(column_index, array).await, + Self::V2_1(writer) => writer.write_column(column_index, array).await, + Self::V2_2(writer) => writer.write_column(column_index, array).await, + Self::V2_3(writer) => writer.write_column(column_index, array).await, } - let encoding_tasks = encoding_tasks - .into_iter() - .flatten() - .collect::>(); - - self.advance_columns(&columns); - self.write_pages(encoding_tasks).await?; - Ok(()) - } - - async fn write_column_metadata( - &mut self, - metadata: pbfile::ColumnMetadata, - ) -> Result<(u64, u64)> { - let metadata_bytes = metadata.encode_to_vec(); - let position = self.writer.tell().await? as u64; - let len = metadata_bytes.len() as u64; - self.writer.write_all(&metadata_bytes).await?; - Ok((position, len)) } - async fn write_column_metadatas(&mut self) -> Result> { - let metadatas = std::mem::take(&mut self.column_metadata); - - // If spilling, finalize the spill writer and reopen for reading. - // The spill file itself is cleaned up by the caller (it lives in a - // temp directory managed by the caller's RAII guard). - let spill_state = self.page_spill.take(); - let (spill_chunks, spill_reader) = - if let Some(PageSpillState::Active(mut spill)) = spill_state { - spill.shutdown_writer().await?; - let reader = spill.object_store.open(&spill.path).await?; - let chunks = std::mem::take(&mut spill.column_chunks); - (chunks, Some(reader)) - } else { - (Vec::new(), None) - }; - - let mut metadata_positions = Vec::with_capacity(metadatas.len()); - for (col_idx, mut metadata) in metadatas.into_iter().enumerate() { - if let Some(reader) = &spill_reader { - let mut pages = Vec::new(); - for &(offset, len) in &spill_chunks[col_idx] { - let data = reader - .get_range(offset as usize..(offset as usize + len as usize)) - .await - .map_err(|e| Error::io_source(Box::new(e)))?; - pages.extend(decode_spilled_chunk(&data)?); - } - metadata.pages = pages; - } - metadata_positions.push(self.write_column_metadata(metadata).await?); + /// Append a buffer whose page or column metadata is supplied externally. + pub async fn write_external_buffer(&mut self, bytes: &[u8]) -> Result<(u64, u64)> { + match self { + Self::V2_0(writer) => writer.write_external_buffer(bytes).await, + Self::V2_1(writer) => writer.write_external_buffer(bytes).await, + Self::V2_2(writer) => writer.write_external_buffer(bytes).await, + Self::V2_3(writer) => writer.write_external_buffer(bytes).await, } - - Ok(metadata_positions) } - fn make_file_descriptor( - schema: &lance_core::datatypes::Schema, - num_rows: u64, - ) -> Result { - let fields_with_meta = FieldsWithMeta::from(schema); - Ok(pb::FileDescriptor { - schema: Some(pb::Schema { - fields: fields_with_meta.fields.0, - metadata: fields_with_meta.metadata, - }), - length: num_rows, - }) - } - - async fn write_global_buffers(&mut self) -> Result> { - let schema = self.schema.as_mut().ok_or(Error::invalid_input("No schema provided on writer open and no data provided. Schema is unknown and file cannot be created"))?; - schema.metadata = std::mem::take(&mut self.schema_metadata); - // Use descriptor layout for blob v2 fields in the footer to avoid exposing logical child fields. - schema - .fields - .iter_mut() - .for_each(|f| f.unload_blobs_recursive()); - - let file_descriptor = Self::make_file_descriptor(schema, self.rows_written)?; - let file_descriptor_bytes = file_descriptor.encode_to_vec(); - let file_descriptor_len = file_descriptor_bytes.len() as u64; - let file_descriptor_position = self.writer.tell().await? as u64; - self.writer.write_all(&file_descriptor_bytes).await?; - let mut gbo_table = Vec::with_capacity(1 + self.global_buffers.len()); - gbo_table.push((file_descriptor_position, file_descriptor_len)); - gbo_table.append(&mut self.global_buffers); - Ok(gbo_table) - } - - /// Add a metadata entry to the schema - /// - /// This method is useful because sometimes the metadata is not known until after the - /// data has been written. This method allows you to alter the schema metadata. It - /// must be called before `finish` is called. + /// Add a metadata entry to the schema. pub fn add_schema_metadata(&mut self, key: impl Into, value: impl Into) { - self.schema_metadata.insert(key.into(), value.into()); + let key = key.into(); + let value = value.into(); + match self { + Self::V2_0(writer) => writer.add_schema_metadata(key, value), + Self::V2_1(writer) => writer.add_schema_metadata(key, value), + Self::V2_2(writer) => writer.add_schema_metadata(key, value), + Self::V2_3(writer) => writer.add_schema_metadata(key, value), + } } - /// Prepare the writer when column data and metadata were produced externally. - /// - /// This is useful for flows that copy already-encoded pages (e.g., binary copy - /// during compaction) where the column buffers have been written directly and we - /// only need to write the footer and schema metadata. The provided - /// `column_metadata` must describe the buffers already persisted by the - /// underlying `ObjectWriter`, and `rows_written` should reflect the total number - /// of rows in those buffers. - pub fn initialize_with_external_metadata( + /// Prepare a writer from encoded columns whose buffers were produced externally. + pub fn initialize_with_external_columns( &mut self, - schema: lance_core::datatypes::Schema, - column_metadata: Vec, + schema: Schema, + columns: &[Arc], rows_written: u64, - ) { - self.schema = Some(schema); - self.num_columns = column_metadata.len() as u32; - self.column_metadata = column_metadata; - self.rows_written = rows_written; - } - - /// Adds a global buffer to the file - /// - /// The global buffer can contain any arbitrary bytes. It will be written to the disk - /// immediately. This method returns the index of the global buffer (this will always - /// start at 1 and increment by 1 each time this method is called) - pub async fn add_global_buffer(&mut self, buffer: Bytes) -> Result { - let position = self.writer.tell().await? as u64; - let len = buffer.len() as u64; - Self::do_write_buffer(&mut self.writer, &buffer).await?; - self.global_buffers.push((position, len)); - Ok(self.global_buffers.len() as u32) - } - - async fn finish_writers(&mut self) -> Result<()> { - let mut col_idx = 0; - for mut writer in std::mem::take(&mut self.column_writers) { - let mut external_buffers = - OutOfLineBuffers::new(self.tell().await?, PAGE_BUFFER_ALIGNMENT as u64); - let columns = writer.finish(&mut external_buffers).await?; - for buffer in external_buffers.take_buffers() { - self.writer.write_all(&buffer).await?; + ) -> Result<()> { + let column_metadata = columns + .iter() + .map(|column| column_info_to_metadata(column)) + .collect::>>()?; + match self { + Self::V2_0(writer) => { + writer.initialize_with_external_metadata(schema, column_metadata, rows_written) } - debug_assert_eq!( - columns.len(), - writer.num_columns() as usize, - "Expected {} columns from column at index {} and got {}", - writer.num_columns(), - col_idx, - columns.len() - ); - for column in columns { - for page in column.final_pages { - self.write_page(page).await?; - } - let column_metadata = &mut self.column_metadata[col_idx]; - let mut buffer_pos = self.writer.tell().await? as u64; - for buffer in column.column_buffers { - column_metadata.buffer_offsets.push(buffer_pos); - let mut size = 0; - Self::do_write_buffer(&mut self.writer, &buffer).await?; - size += buffer.len() as u64; - buffer_pos += size; - column_metadata.buffer_sizes.push(size); - } - let encoded_encoding = Any::from_msg(&column.encoding)?.encode_to_vec(); - column_metadata.encoding = Some(pbfile::Encoding { - location: Some(pbfile::encoding::Location::Direct(pbfile::DirectEncoding { - encoding: encoded_encoding, - })), - }); - col_idx += 1; + Self::V2_1(writer) => { + writer.initialize_with_external_metadata(schema, column_metadata, rows_written) + } + Self::V2_2(writer) => { + writer.initialize_with_external_metadata(schema, column_metadata, rows_written) + } + Self::V2_3(writer) => { + writer.initialize_with_external_metadata(schema, column_metadata, rows_written) } - } - if col_idx != self.column_metadata.len() { - panic!( - "Column writers finished with {} columns but we expected {}", - col_idx, - self.column_metadata.len() - ); } Ok(()) } - /// Converts self.version (which is a mix of "software version" and - /// "format version" into a format version) - fn version_to_numbers(&self) -> (u16, u16) { - let version = self.options.format_version.unwrap_or_default(); - match version.resolve() { - LanceFileVersion::V2_0 => (0, 3), - LanceFileVersion::V2_1 => (2, 1), - LanceFileVersion::V2_2 => (2, 2), - LanceFileVersion::V2_3 => (2, 3), - _ => panic!("Unsupported version: {}", version), + /// Add an arbitrary global buffer and return its one-based index. + pub async fn add_global_buffer(&mut self, buffer: Bytes) -> Result { + match self { + Self::V2_0(writer) => writer.add_global_buffer(buffer).await, + Self::V2_1(writer) => writer.add_global_buffer(buffer).await, + Self::V2_2(writer) => writer.add_global_buffer(buffer).await, + Self::V2_3(writer) => writer.add_global_buffer(buffer).await, } } - /// Finishes writing the file - /// - /// This method will wait until all data has been flushed to the file. Then it - /// will write the file metadata and the footer. It will not return until all - /// data has been flushed and the file has been closed. - /// - /// Returns a summary of the completed file write. + /// Finish the file and close its object writer. pub async fn finish(&mut self) -> Result { - // 1. flush any remaining data and write out those pages - let mut external_buffers = - OutOfLineBuffers::new(self.tell().await?, PAGE_BUFFER_ALIGNMENT as u64); - let encoding_tasks = self - .column_writers - .iter_mut() - .map(|writer| writer.flush(&mut external_buffers)) - .collect::>>()?; - for external_buffer in external_buffers.take_buffers() { - Self::do_write_buffer(&mut self.writer, &external_buffer).await?; - } - let encoding_tasks = encoding_tasks - .into_iter() - .flatten() - .collect::>(); - self.write_pages(encoding_tasks).await?; - - if !self.column_writers.is_empty() { - self.finish_writers().await?; + match self { + Self::V2_0(writer) => writer.finish().await, + Self::V2_1(writer) => writer.finish().await, + Self::V2_2(writer) => writer.finish().await, + Self::V2_3(writer) => writer.finish().await, } - - // 3. write global buffers (we write the schema here) - let global_buffer_offsets = self.write_global_buffers().await?; - let num_global_buffers = global_buffer_offsets.len() as u32; - - // 4. write the column metadatas - let column_metadata_start = self.writer.tell().await? as u64; - let metadata_positions = self.write_column_metadatas().await?; - - // 5. write the column metadata offset table - let cmo_table_start = self.writer.tell().await? as u64; - for (meta_pos, meta_len) in metadata_positions { - self.writer.write_u64_le(meta_pos).await?; - self.writer.write_u64_le(meta_len).await?; - } - - // 6. write global buffers offset table - let gbo_table_start = self.writer.tell().await? as u64; - for (gbo_pos, gbo_len) in global_buffer_offsets { - self.writer.write_u64_le(gbo_pos).await?; - self.writer.write_u64_le(gbo_len).await?; - } - - let (major, minor) = self.version_to_numbers(); - // 7. write the footer - self.writer.write_u64_le(column_metadata_start).await?; - self.writer.write_u64_le(cmo_table_start).await?; - self.writer.write_u64_le(gbo_table_start).await?; - self.writer.write_u32_le(num_global_buffers).await?; - self.writer.write_u32_le(self.num_columns).await?; - self.writer.write_u16_le(major).await?; - self.writer.write_u16_le(minor).await?; - self.writer.write_all(MAGIC).await?; - - // 7. close the writer - let write_result = Writer::shutdown(self.writer.as_mut()).await?; - - Ok(FileWriteSummary { - num_rows: self.rows_written, - size_bytes: write_result.size as u64, - }) } + /// Abandon the file write. pub async fn abort(&mut self) { - // For multipart uploads, ObjectWriter's Drop impl will abort - // the upload when the writer is dropped. + match self { + Self::V2_0(writer) => writer.abort().await, + Self::V2_1(writer) => writer.abort().await, + Self::V2_2(writer) => writer.abort().await, + Self::V2_3(writer) => writer.abort().await, + } } + /// Return the current object-writer position. pub async fn tell(&mut self) -> Result { - Ok(self.writer.tell().await? as u64) + match self { + Self::V2_0(writer) => writer.tell().await, + Self::V2_1(writer) => writer.tell().await, + Self::V2_2(writer) => writer.tell().await, + Self::V2_3(writer) => writer.tell().await, + } } + /// Return the field-id to physical-column mapping. pub fn field_id_to_column_indices(&self) -> &[(u32, u32)] { - &self.field_id_to_column_indices - } -} - -/// Utility trait for converting EncodedBatch to Bytes using the -/// lance file format -pub trait EncodedBatchWriteExt { - /// Serializes into a lance file, including the schema - fn try_to_self_described_lance(&self, version: LanceFileVersion) -> Result; - /// Serializes into a lance file, without the schema. - /// - /// The schema must be provided to deserialize the buffer - fn try_to_mini_lance(&self, version: LanceFileVersion) -> Result; -} - -// Creates a lance footer and appends it to the encoded data -// -// The logic here is very similar to logic in the FileWriter except we -// are using BufMut (put_xyz) instead of AsyncWrite (write_xyz). -fn concat_lance_footer( - batch: &EncodedBatch, - write_schema: bool, - version: LanceFileVersion, -) -> Result { - // Estimating 1MiB for file footer - let mut data = BytesMut::with_capacity(batch.data.len() + 1024 * 1024); - data.put(batch.data.clone()); - // write global buffers (we write the schema here) - let global_buffers = if write_schema { - let schema_start = data.len() as u64; - let lance_schema = lance_core::datatypes::Schema::try_from(batch.schema.as_ref())?; - let descriptor = FileWriter::make_file_descriptor(&lance_schema, batch.num_rows)?; - let descriptor_bytes = descriptor.encode_to_vec(); - let descriptor_len = descriptor_bytes.len() as u64; - data.put(descriptor_bytes.as_slice()); - - vec![(schema_start, descriptor_len)] - } else { - vec![] - }; - let col_metadata_start = data.len() as u64; - - let mut col_metadata_positions = Vec::new(); - // Write column metadata - for col in &batch.page_table { - let position = data.len() as u64; - let pages = col - .page_infos - .iter() - .map(|page_info| { - let encoded_encoding = match &page_info.encoding { - PageEncoding::Legacy(array_encoding) => { - Any::from_msg(array_encoding)?.encode_to_vec() - } - PageEncoding::Structural(page_layout) => { - Any::from_msg(page_layout)?.encode_to_vec() - } - }; - let (buffer_offsets, buffer_sizes): (Vec<_>, Vec<_>) = page_info - .buffer_offsets_and_sizes - .as_ref() - .iter() - .cloned() - .unzip(); - Ok(pbfile::column_metadata::Page { - buffer_offsets, - buffer_sizes, - encoding: Some(pbfile::Encoding { - location: Some(pbfile::encoding::Location::Direct(DirectEncoding { - encoding: encoded_encoding, - })), - }), - length: page_info.num_rows, - priority: page_info.priority, - }) - }) - .collect::>>()?; - let (buffer_offsets, buffer_sizes): (Vec<_>, Vec<_>) = - col.buffer_offsets_and_sizes.iter().cloned().unzip(); - let encoded_col_encoding = Any::from_msg(&col.encoding)?.encode_to_vec(); - let column = pbfile::ColumnMetadata { - pages, - buffer_offsets, - buffer_sizes, - encoding: Some(pbfile::Encoding { - location: Some(pbfile::encoding::Location::Direct(pbfile::DirectEncoding { - encoding: encoded_col_encoding, - })), - }), - }; - let column_bytes = column.encode_to_vec(); - col_metadata_positions.push((position, column_bytes.len() as u64)); - data.put(column_bytes.as_slice()); - } - // Write column metadata offsets table - let cmo_table_start = data.len() as u64; - for (meta_pos, meta_len) in col_metadata_positions { - data.put_u64_le(meta_pos); - data.put_u64_le(meta_len); - } - // Write global buffers offsets table - let gbo_table_start = data.len() as u64; - let num_global_buffers = global_buffers.len() as u32; - for (gbo_pos, gbo_len) in global_buffers { - data.put_u64_le(gbo_pos); - data.put_u64_le(gbo_len); - } - - let (major, minor) = version.to_numbers(); - - // write the footer - data.put_u64_le(col_metadata_start); - data.put_u64_le(cmo_table_start); - data.put_u64_le(gbo_table_start); - data.put_u32_le(num_global_buffers); - data.put_u32_le(batch.page_table.len() as u32); - data.put_u16_le(major as u16); - data.put_u16_le(minor as u16); - data.put(MAGIC.as_slice()); - - Ok(data.freeze()) -} - -impl EncodedBatchWriteExt for EncodedBatch { - fn try_to_self_described_lance(&self, version: LanceFileVersion) -> Result { - concat_lance_footer(self, true, version) - } - - fn try_to_mini_lance(&self, version: LanceFileVersion) -> Result { - concat_lance_footer(self, false, version) + match self { + Self::V2_0(writer) => writer.field_id_to_column_indices(), + Self::V2_1(writer) => writer.field_id_to_column_indices(), + Self::V2_2(writer) => writer.field_id_to_column_indices(), + Self::V2_3(writer) => writer.field_id_to_column_indices(), + } } } #[cfg(test)] -mod tests { - use std::collections::HashMap; - use std::sync::Arc; - - use crate::reader::{FileReader, FileReaderOptions, ReaderProjection, describe_encoding}; - use crate::testing::FsFixture; - use crate::writer::{ENV_LANCE_FILE_WRITER_MAX_PAGE_BYTES, FileWriter, FileWriterOptions}; - use arrow_array::builder::{Float32Builder, Int32Builder}; - use arrow_array::{ArrayRef, Int32Array, RecordBatch, UInt64Array}; - use arrow_array::{RecordBatchReader, StringArray, types::Float64Type}; - use arrow_schema::{DataType, Field, Field as ArrowField, Schema, Schema as ArrowSchema}; - use lance_core::cache::LanceCache; - use lance_core::datatypes::Schema as LanceSchema; - use lance_core::utils::tempfile::TempObjFile; - use lance_datagen::{BatchCount, RowCount, array, gen_batch}; - use lance_encoding::compression_config::{CompressionFieldParams, CompressionParams}; - use lance_encoding::decoder::DecoderPlugins; - use lance_encoding::version::LanceFileVersion; - use lance_io::object_store::ObjectStore; - use lance_io::utils::CachedFileSize; - use rstest::rstest; - - #[tokio::test] - async fn test_basic_write() { - let tmp_path = TempObjFile::default(); - let obj_store = Arc::new(ObjectStore::local()); - - let reader = gen_batch() - .col("score", array::rand::()) - .into_reader_rows(RowCount::from(1000), BatchCount::from(10)); - - let writer = obj_store.create(&tmp_path).await.unwrap(); - - let lance_schema = - lance_core::datatypes::Schema::try_from(reader.schema().as_ref()).unwrap(); - - let mut file_writer = - FileWriter::try_new(writer, lance_schema, FileWriterOptions::default()).unwrap(); - - for batch in reader { - file_writer.write_batch(&batch.unwrap()).await.unwrap(); - } - file_writer.add_schema_metadata("foo", "bar"); - file_writer.finish().await.unwrap(); - // Tests asserting the contents of the written file are in reader.rs - } - - #[tokio::test] - async fn test_write_empty() { - let tmp_path = TempObjFile::default(); - let obj_store = Arc::new(ObjectStore::local()); - - let reader = gen_batch() - .col("score", array::rand::()) - .into_reader_rows(RowCount::from(0), BatchCount::from(0)); - - let writer = obj_store.create(&tmp_path).await.unwrap(); - - let lance_schema = - lance_core::datatypes::Schema::try_from(reader.schema().as_ref()).unwrap(); - - let mut file_writer = - FileWriter::try_new(writer, lance_schema, FileWriterOptions::default()).unwrap(); - - for batch in reader { - file_writer.write_batch(&batch.unwrap()).await.unwrap(); - } - file_writer.add_schema_metadata("foo", "bar"); - file_writer.finish().await.unwrap(); - } - - // Read a single column back at an explicit range/index set, returning its - // `Int32` values. Reading one column (or an equal-length group) at a time is - // how unequal-length files are consumed: a full scan across columns of - // differing lengths cannot form a single rectangular batch. - async fn read_int32_column( - reader: &FileReader, - schema: &LanceSchema, - version: LanceFileVersion, - name: &str, - params: lance_io::ReadBatchParams, - ) -> Vec> { - use futures::TryStreamExt; - use lance_encoding::decoder::FilterExpression; - - let projection = ReaderProjection::from_column_names(version, schema, &[name]).unwrap(); - let batches: Vec = reader - .read_stream_projected(params, 1024, 16, projection, FilterExpression::no_filter()) - .await - .unwrap() - .try_collect() - .await - .unwrap(); - batches - .iter() - .flat_map(|b| { - b.column(0) - .as_any() - .downcast_ref::() - .unwrap() - .iter() - .collect::>() - }) - .collect() - } - - /// A single file may hold columns of differing item counts, written by - /// advancing each column's row counter independently (no shared global - /// counter). - #[rstest] - #[tokio::test] - async fn test_write_columns_unequal_lengths( - #[values(LanceFileVersion::V2_0, LanceFileVersion::V2_1)] version: LanceFileVersion, - ) { - use lance_io::ReadBatchParams; - - let arrow_schema = Arc::new(ArrowSchema::new(vec![ - ArrowField::new("a", DataType::Int32, true), - ArrowField::new("b", DataType::Int32, true), - ArrowField::new("c", DataType::Int32, true), - ])); - let lance_schema = LanceSchema::try_from(arrow_schema.as_ref()).unwrap(); - - let fs = FsFixture::default(); - let options = FileWriterOptions { - format_version: Some(version), - ..Default::default() - }; - let mut writer = FileWriter::try_new( - fs.object_store.create(&fs.tmp_path).await.unwrap(), - lance_schema.clone(), - options, - ) - .unwrap(); - - // Field "a" gets 5 values across two calls (appending), field "b" gets a - // single value, and field "c" is never written (a zero-length column). - let a1: ArrayRef = Arc::new(Int32Array::from(vec![1, 2, 3])); - let b: ArrayRef = Arc::new(Int32Array::from(vec![10])); - writer.write_column(0, a1).await.unwrap(); - writer.write_column(1, b).await.unwrap(); - let a2: ArrayRef = Arc::new(Int32Array::from(vec![4, 5])); - writer.write_column(0, a2).await.unwrap(); - // An empty array is a no-op whether or not the field already has rows: - // field "a" keeps its 5 rows, field "c" stays a zero-length column. - let empty: ArrayRef = Arc::new(Int32Array::from(Vec::::new())); - writer.write_column(0, empty.clone()).await.unwrap(); - writer.write_column(2, empty).await.unwrap(); - - let summary = writer.finish().await.unwrap(); - // The file's logical length is the longest column. - assert_eq!(summary.num_rows, 5); - - let file_scheduler = fs - .scheduler - .open_file(&fs.tmp_path, &CachedFileSize::unknown()) - .await - .unwrap(); - let reader = FileReader::try_open( - file_scheduler, - None, - Arc::::default(), - &LanceCache::no_cache(), - FileReaderOptions::default(), - ) - .await - .unwrap(); - - // Per-column row counts are recorded in / derivable from file metadata. - assert_eq!(reader.num_rows(), 5); - assert_eq!(reader.column_num_rows(0).unwrap(), 5); - assert_eq!(reader.column_num_rows(1).unwrap(), 1); - assert_eq!(reader.column_num_rows(2).unwrap(), 0); - assert!(reader.column_num_rows(3).is_err()); - - // Each column reads back independently at its own length. - assert_eq!( - read_int32_column( - &reader, - &lance_schema, - version, - "a", - ReadBatchParams::Range(0..5) - ) - .await, - vec![Some(1), Some(2), Some(3), Some(4), Some(5)], - ); - assert_eq!( - read_int32_column( - &reader, - &lance_schema, - version, - "b", - ReadBatchParams::Range(0..1) - ) - .await, - vec![Some(10)], - ); - - // Random access by position within the longer column returns the right - // value even though other columns are shorter. (The take path requires - // strictly increasing indices.) - assert_eq!( - read_int32_column( - &reader, - &lance_schema, - version, - "a", - ReadBatchParams::Indices(arrow_array::UInt32Array::from(vec![0, 2, 4])), - ) - .await, - vec![Some(1), Some(3), Some(5)], - ); - } - - /// Reading an unequal-length file: - /// - a projection whose columns are equal length full-scans normally; - /// - a full scan across columns of differing length is rejected up front, - /// before any batch is produced (even though a prefix would be rectangular); - /// - a bounded read is valid as long as every projected column covers it; - /// - a single-column `RangeFull` resolves to that column's own length, not - /// the file's (maximum) length. - #[rstest] - #[tokio::test] - async fn test_read_unequal_length_projection( - #[values(LanceFileVersion::V2_0, LanceFileVersion::V2_1)] version: LanceFileVersion, - ) { - use futures::TryStreamExt; - use lance_encoding::decoder::FilterExpression; - use lance_io::ReadBatchParams; - - let arrow_schema = Arc::new(ArrowSchema::new(vec![ - ArrowField::new("a", DataType::Int32, true), - ArrowField::new("b", DataType::Int32, true), - ArrowField::new("c", DataType::Int32, true), - ])); - let lance_schema = LanceSchema::try_from(arrow_schema.as_ref()).unwrap(); - let fs = FsFixture::default(); - let options = FileWriterOptions { - format_version: Some(version), - ..Default::default() - }; - let mut writer = FileWriter::try_new( - fs.object_store.create(&fs.tmp_path).await.unwrap(), - lance_schema.clone(), - options, - ) - .unwrap(); - // "a" and "b" are equal length (5); "c" is shorter (1). - writer - .write_column(0, Arc::new(Int32Array::from(vec![1, 2, 3, 4, 5]))) - .await - .unwrap(); - writer - .write_column(1, Arc::new(Int32Array::from(vec![6, 7, 8, 9, 10]))) - .await - .unwrap(); - writer - .write_column(2, Arc::new(Int32Array::from(vec![100]))) - .await - .unwrap(); - writer.finish().await.unwrap(); - - let file_scheduler = fs - .scheduler - .open_file(&fs.tmp_path, &CachedFileSize::unknown()) - .await - .unwrap(); - let reader = FileReader::try_open( - file_scheduler, - None, - Arc::::default(), - &LanceCache::no_cache(), - FileReaderOptions::default(), - ) - .await - .unwrap(); - - let read = |names: &'static [&'static str], params: ReadBatchParams| { - let projection = - ReaderProjection::from_column_names(version, &lance_schema, names).unwrap(); - async { - match reader - .read_stream_projected( - params, - 1024, - 16, - projection, - FilterExpression::no_filter(), - ) - .await - { - Ok(stream) => stream.try_collect::>().await, - Err(e) => Err(e), - } - } - }; - let col_values = |batches: &[RecordBatch], idx: usize| -> Vec> { - batches - .iter() - .flat_map(|b| { - b.column(idx) - .as_any() - .downcast_ref::() - .unwrap() - .iter() - .collect::>() - }) - .collect() - }; - - // Equal-length projection [a, b] full-scans into rectangular batches. - let batches = read(&["a", "b"], ReadBatchParams::RangeFull).await.unwrap(); - assert_eq!( - col_values(&batches, 0), - vec![Some(1), Some(2), Some(3), Some(4), Some(5)] - ); - assert_eq!( - col_values(&batches, 1), - vec![Some(6), Some(7), Some(8), Some(9), Some(10)] - ); - - // A mismatched-length projection [a, c] (5 vs 1) is rejected before any - // batch is yielded, regardless of the read params — its columns cannot - // be combined into rectangular batches. The error names each column's - // length so the caller can see which column is the odd one out. - let err = read(&["a", "c"], ReadBatchParams::RangeFull) - .await - .unwrap_err() - .to_string(); - assert!( - err.contains("a=5") && err.contains("c=1"), - "error should name each column's length, got: {err}" - ); - assert!( - read(&["a", "c"], ReadBatchParams::Range(0..1)) - .await - .is_err(), - "even a common-prefix read of unequal-length columns must error" - ); - - // A single-column RangeFull resolves to that column's own length. - let batches = read(&["c"], ReadBatchParams::RangeFull).await.unwrap(); - assert_eq!(col_values(&batches, 0), vec![Some(100)]); - let batches = read(&["a"], ReadBatchParams::RangeFull).await.unwrap(); - assert_eq!( - col_values(&batches, 0), - vec![Some(1), Some(2), Some(3), Some(4), Some(5)] - ); - - // RangeFrom/RangeTo likewise resolve against the projected column's own - // length rather than the file's longest column. - let batches = read(&["a"], ReadBatchParams::RangeFrom(2..)).await.unwrap(); - assert_eq!(col_values(&batches, 0), vec![Some(3), Some(4), Some(5)]); - // RangeFrom on the short column "c" resolves to length 1, not 5. - let batches = read(&["c"], ReadBatchParams::RangeFrom(0..)).await.unwrap(); - assert_eq!(col_values(&batches, 0), vec![Some(100)]); - let batches = read(&["a"], ReadBatchParams::RangeTo(..3)).await.unwrap(); - assert_eq!(col_values(&batches, 0), vec![Some(1), Some(2), Some(3)]); - // A bound past the projected column's length errors. - assert!( - read(&["a"], ReadBatchParams::RangeTo(..6)).await.is_err(), - "RangeTo past the column length must error" - ); - assert!( - read(&["c"], ReadBatchParams::RangeFrom(2..)).await.is_err(), - "RangeFrom past the column length must error" - ); - } - - /// A struct and a list column each map to multiple physical columns, and a - /// list's item column is longer than its top-level row count. The - /// projection-length check must partition `column_indices` by top-level - /// field and use each field's root column, so an ordinary (rectangular) file - /// with nested columns still reads under the new validation path. - #[rstest] - #[tokio::test] - async fn test_read_nested_columns_under_validation( - #[values(LanceFileVersion::V2_0, LanceFileVersion::V2_1)] version: LanceFileVersion, - ) { - use arrow_array::types::Int32Type; - use arrow_array::{ListArray, StructArray}; - use futures::TryStreamExt; - use lance_encoding::decoder::FilterExpression; - use lance_io::ReadBatchParams; - - let struct_type = DataType::Struct( - vec![ - ArrowField::new("x", DataType::Int32, true), - ArrowField::new("y", DataType::Int32, true), - ] - .into(), - ); - let list_type = DataType::List(Arc::new(ArrowField::new("item", DataType::Int32, true))); - let arrow_schema = Arc::new(ArrowSchema::new(vec![ - ArrowField::new("a", DataType::Int32, true), - ArrowField::new("s", struct_type, true), - ArrowField::new("lst", list_type, true), - ])); - let lance_schema = LanceSchema::try_from(arrow_schema.as_ref()).unwrap(); - - let a: ArrayRef = Arc::new(Int32Array::from(vec![1, 2, 3])); - let s: ArrayRef = Arc::new(StructArray::from(vec![ - ( - Arc::new(ArrowField::new("x", DataType::Int32, true)), - Arc::new(Int32Array::from(vec![10, 20, 30])) as ArrayRef, - ), - ( - Arc::new(ArrowField::new("y", DataType::Int32, true)), - Arc::new(Int32Array::from(vec![11, 21, 31])) as ArrayRef, - ), - ])); - // 3 lists, 6 items: the item column is longer than the top-level rows. - let lst: ArrayRef = Arc::new(ListArray::from_iter_primitive::(vec![ - Some(vec![Some(1), Some(2)]), - Some(vec![Some(3)]), - Some(vec![Some(4), Some(5), Some(6)]), - ])); - let batch = RecordBatch::try_new(arrow_schema.clone(), vec![a, s, lst]).unwrap(); - - let fs = FsFixture::default(); - let options = FileWriterOptions { - format_version: Some(version), - ..Default::default() - }; - let mut writer = FileWriter::try_new( - fs.object_store.create(&fs.tmp_path).await.unwrap(), - lance_schema.clone(), - options, - ) - .unwrap(); - writer.write_batch(&batch).await.unwrap(); - writer.finish().await.unwrap(); - - let file_scheduler = fs - .scheduler - .open_file(&fs.tmp_path, &CachedFileSize::unknown()) - .await - .unwrap(); - let reader = FileReader::try_open( - file_scheduler, - None, - Arc::::default(), - &LanceCache::no_cache(), - FileReaderOptions::default(), - ) - .await - .unwrap(); - - // If `validate_field_length` mispartitioned the physical columns, the - // length check would read the wrong root column (e.g. the list's item - // column, length 6) and spuriously reject this rectangular file. - for names in [&["a", "s", "lst"][..], &["a", "lst"][..], &["a", "s"][..]] { - let projection = - ReaderProjection::from_column_names(version, &lance_schema, names).unwrap(); - let batches: Vec = reader - .read_stream_projected( - ReadBatchParams::RangeFull, - 1024, - 16, - projection, - FilterExpression::no_filter(), - ) - .await - .unwrap() - .try_collect() - .await - .unwrap(); - let total_rows: usize = batches.iter().map(|b| b.num_rows()).sum(); - assert_eq!( - total_rows, 3, - "projection {names:?} should read 3 top-level rows" - ); - } - } - - /// `write_column` rejects invalid inputs at the API boundary with - /// descriptive errors: a writer without an explicit schema, an - /// out-of-bounds field index, and a null written into a non-nullable field. - #[tokio::test] - async fn test_write_column_validation_errors() { - // A lazy-schema writer cannot infer the schema from a single column. - let fs = FsFixture::default(); - let mut lazy_writer = FileWriter::new_lazy( - fs.object_store.create(&fs.tmp_path).await.unwrap(), - FileWriterOptions::default(), - ); - let err = lazy_writer - .write_column(0, Arc::new(Int32Array::from(vec![1, 2, 3]))) - .await - .unwrap_err() - .to_string(); - assert!( - err.contains("explicit schema"), - "expected explicit-schema error, got: {err}" - ); - - let arrow_schema = Arc::new(ArrowSchema::new(vec![ - ArrowField::new("a", DataType::Int32, false), - ArrowField::new("b", DataType::Int32, true), - ])); - let lance_schema = LanceSchema::try_from(arrow_schema.as_ref()).unwrap(); - - // An out-of-bounds field index is rejected, naming the index and count. - let fs = FsFixture::default(); - let mut writer = FileWriter::try_new( - fs.object_store.create(&fs.tmp_path).await.unwrap(), - lance_schema.clone(), - FileWriterOptions::default(), - ) - .unwrap(); - let err = writer - .write_column(5, Arc::new(Int32Array::from(vec![1]))) - .await - .unwrap_err() - .to_string(); - assert!( - err.contains('5') && err.contains('2'), - "expected out-of-bounds error naming index 5 and 2 fields, got: {err}" - ); - - // A null in a non-nullable field ("a") is rejected. - let err = writer - .write_column(0, Arc::new(Int32Array::from(vec![Some(1), None, Some(3)]))) - .await - .unwrap_err() - .to_string(); - assert!( - err.contains("non-null"), - "expected nullability error, got: {err}" - ); - } - - /// The blocking read path applies the same projection-length validation as - /// the async path: a short single column resolves to its own length, and a - /// mismatched-length projection errors up front. - #[rstest] - #[tokio::test] - async fn test_blocking_read_unequal_length( - #[values(LanceFileVersion::V2_0, LanceFileVersion::V2_1)] version: LanceFileVersion, - ) { - use lance_encoding::decoder::FilterExpression; - use lance_io::ReadBatchParams; - - let arrow_schema = Arc::new(ArrowSchema::new(vec![ - ArrowField::new("a", DataType::Int32, true), - ArrowField::new("c", DataType::Int32, true), - ])); - let lance_schema = LanceSchema::try_from(arrow_schema.as_ref()).unwrap(); - let fs = FsFixture::default(); - let options = FileWriterOptions { - format_version: Some(version), - ..Default::default() - }; - let mut writer = FileWriter::try_new( - fs.object_store.create(&fs.tmp_path).await.unwrap(), - lance_schema.clone(), - options, - ) - .unwrap(); - writer - .write_column(0, Arc::new(Int32Array::from(vec![1, 2, 3, 4, 5]))) - .await - .unwrap(); - writer - .write_column(1, Arc::new(Int32Array::from(vec![100]))) - .await - .unwrap(); - writer.finish().await.unwrap(); - - let file_scheduler = fs - .scheduler - .open_file(&fs.tmp_path, &CachedFileSize::unknown()) - .await - .unwrap(); - let reader = Arc::new( - FileReader::try_open( - file_scheduler, - None, - Arc::::default(), - &LanceCache::no_cache(), - FileReaderOptions::default(), - ) - .await - .unwrap(), - ); - - // Single short column: RangeFull resolves to its own length (1). - let proj_c = ReaderProjection::from_column_names(version, &lance_schema, &["c"]).unwrap(); - let reader_c = reader.clone(); - let batches = tokio::task::spawn_blocking(move || { - reader_c - .read_stream_projected_blocking( - ReadBatchParams::RangeFull, - 1024, - Some(proj_c), - FilterExpression::no_filter(), - ) - .unwrap() - .collect::, _>>() - .unwrap() - }) - .await - .unwrap(); - let total_rows: usize = batches.iter().map(|b| b.num_rows()).sum(); - assert_eq!(total_rows, 1); - - // A mismatched projection [a, c] errors on the blocking path too. - let proj_ac = - ReaderProjection::from_column_names(version, &lance_schema, &["a", "c"]).unwrap(); - let reader_ac = reader.clone(); - let is_err = tokio::task::spawn_blocking(move || { - reader_ac - .read_stream_projected_blocking( - ReadBatchParams::RangeFull, - 1024, - Some(proj_ac), - FilterExpression::no_filter(), - ) - .is_err() - }) - .await - .unwrap(); - assert!( - is_err, - "blocking full scan across unequal-length columns must error" - ); - } - - /// Files written the ordinary (rectangular) way keep equal column lengths, - /// so the unequal-length support is backwards compatible. - #[tokio::test] - async fn test_write_batch_keeps_equal_lengths() { - let arrow_schema = Arc::new(ArrowSchema::new(vec![ - ArrowField::new("a", DataType::Int32, true), - ArrowField::new("b", DataType::Int32, true), - ])); - let lance_schema = LanceSchema::try_from(arrow_schema.as_ref()).unwrap(); - - let fs = FsFixture::default(); - let mut writer = FileWriter::try_new( - fs.object_store.create(&fs.tmp_path).await.unwrap(), - lance_schema, - FileWriterOptions::default(), - ) - .unwrap(); - let batch = RecordBatch::try_new( - arrow_schema.clone(), - vec![ - Arc::new(Int32Array::from(vec![1, 2, 3])), - Arc::new(Int32Array::from(vec![4, 5, 6])), - ], - ) - .unwrap(); - writer.write_batch(&batch).await.unwrap(); - let summary = writer.finish().await.unwrap(); - assert_eq!(summary.num_rows, 3); - - let file_scheduler = fs - .scheduler - .open_file(&fs.tmp_path, &CachedFileSize::unknown()) - .await - .unwrap(); - let reader = FileReader::try_open( - file_scheduler, - None, - Arc::::default(), - &LanceCache::no_cache(), - FileReaderOptions::default(), - ) - .await - .unwrap(); - assert_eq!(reader.column_num_rows(0).unwrap(), 3); - assert_eq!(reader.column_num_rows(1).unwrap(), 3); - } - - #[tokio::test] - async fn test_max_page_bytes_enforced() { - let arrow_field = Field::new("data", DataType::UInt64, false); - let arrow_schema = Schema::new(vec![arrow_field]); - let lance_schema = LanceSchema::try_from(&arrow_schema).unwrap(); - - // 8MiB - let data: Vec = (0..1_000_000).collect(); - let array = UInt64Array::from(data); - let batch = - RecordBatch::try_new(arrow_schema.clone().into(), vec![Arc::new(array)]).unwrap(); - - let options = FileWriterOptions { - max_page_bytes: Some(1024 * 1024), // 1MB - // This is a 2.0 only test because 2.1+ splits large pages on read instead of write - format_version: Some(LanceFileVersion::V2_0), - ..Default::default() - }; - - let path = TempObjFile::default(); - let object_store = ObjectStore::local(); - let mut writer = FileWriter::try_new( - object_store.create(&path).await.unwrap(), - lance_schema, - options, - ) - .unwrap(); - - writer.write_batch(&batch).await.unwrap(); - writer.finish().await.unwrap(); - - let fs = FsFixture::default(); - let file_scheduler = fs - .scheduler - .open_file(&path, &CachedFileSize::unknown()) - .await - .unwrap(); - let file_reader = FileReader::try_open( - file_scheduler, - None, - Arc::::default(), - &LanceCache::no_cache(), - FileReaderOptions::default(), - ) - .await - .unwrap(); - - let column_meta = file_reader.metadata(); - - let mut total_page_num: u32 = 0; - for (col_idx, col_metadata) in column_meta.column_metadatas.iter().enumerate() { - assert!( - !col_metadata.pages.is_empty(), - "Column {} has no pages", - col_idx - ); - - for (page_idx, page) in col_metadata.pages.iter().enumerate() { - total_page_num += 1; - let total_size: u64 = page.buffer_sizes.iter().sum(); - assert!( - total_size <= 1024 * 1024, - "Column {} Page {} size {} exceeds 1MB limit", - col_idx, - page_idx, - total_size - ); - } - } - - assert_eq!(total_page_num, 8) - } - - #[tokio::test(flavor = "current_thread")] - async fn test_max_page_bytes_env_var() { - let arrow_field = Field::new("data", DataType::UInt64, false); - let arrow_schema = Schema::new(vec![arrow_field]); - let lance_schema = LanceSchema::try_from(&arrow_schema).unwrap(); - // 4MiB - let data: Vec = (0..500_000).collect(); - let array = UInt64Array::from(data); - let batch = - RecordBatch::try_new(arrow_schema.clone().into(), vec![Arc::new(array)]).unwrap(); - - // 2MiB - unsafe { - std::env::set_var(ENV_LANCE_FILE_WRITER_MAX_PAGE_BYTES, "2097152"); - } - - let options = FileWriterOptions { - max_page_bytes: None, // enforce env - ..Default::default() - }; - - let path = TempObjFile::default(); - let object_store = ObjectStore::local(); - let mut writer = FileWriter::try_new( - object_store.create(&path).await.unwrap(), - lance_schema.clone(), - options, - ) - .unwrap(); - - writer.write_batch(&batch).await.unwrap(); - writer.finish().await.unwrap(); - - let fs = FsFixture::default(); - let file_scheduler = fs - .scheduler - .open_file(&path, &CachedFileSize::unknown()) - .await - .unwrap(); - let file_reader = FileReader::try_open( - file_scheduler, - None, - Arc::::default(), - &LanceCache::no_cache(), - FileReaderOptions::default(), - ) - .await - .unwrap(); - - for col_metadata in file_reader.metadata().column_metadatas.iter() { - for page in col_metadata.pages.iter() { - let total_size: u64 = page.buffer_sizes.iter().sum(); - assert!( - total_size <= 2 * 1024 * 1024, - "Page size {} exceeds 2MB limit", - total_size - ); - } - } - - unsafe { - std::env::set_var(ENV_LANCE_FILE_WRITER_MAX_PAGE_BYTES, ""); - } - } - - #[tokio::test] - async fn test_compression_overrides_end_to_end() { - // Create test schema with different column types - let arrow_schema = Arc::new(ArrowSchema::new(vec![ - ArrowField::new("customer_id", DataType::Int32, false), - ArrowField::new("product_id", DataType::Int32, false), - ArrowField::new("quantity", DataType::Int32, false), - ArrowField::new("price", DataType::Float32, false), - ArrowField::new("description", DataType::Utf8, false), - ])); - - let lance_schema = LanceSchema::try_from(arrow_schema.as_ref()).unwrap(); - - // Create test data with patterns suitable for different compression - let mut customer_ids = Int32Builder::new(); - let mut product_ids = Int32Builder::new(); - let mut quantities = Int32Builder::new(); - let mut prices = Float32Builder::new(); - let mut descriptions = Vec::new(); - - // Generate data with specific patterns: - // - customer_id: highly repetitive (good for RLE) - // - product_id: moderately repetitive (good for RLE) - // - quantity: random values (not good for RLE) - // - price: some repetition - // - description: long strings (good for Zstd) - for i in 0..10000 { - // Customer ID repeats every 100 rows (100 unique customers) - // This creates runs of 100 identical values - customer_ids.append_value(i / 100); - - // Product ID has only 5 unique values with long runs - product_ids.append_value(i / 2000); - - // Quantity is mostly 1 with occasional other values - quantities.append_value(if i % 10 == 0 { 5 } else { 1 }); - - // Price has only 3 unique values - prices.append_value(match i % 3 { - 0 => 9.99, - 1 => 19.99, - _ => 29.99, - }); - - // Descriptions are repetitive but we'll keep them simple - descriptions.push(format!("Product {}", i / 2000)); - } - - let batch = RecordBatch::try_new( - arrow_schema.clone(), - vec![ - Arc::new(customer_ids.finish()), - Arc::new(product_ids.finish()), - Arc::new(quantities.finish()), - Arc::new(prices.finish()), - Arc::new(StringArray::from(descriptions)), - ], - ) - .unwrap(); - - // Configure compression parameters - let mut params = CompressionParams::new(); - - // RLE for ID columns (ends with _id) - params.columns.insert( - "*_id".to_string(), - CompressionFieldParams { - rle_threshold: Some(0.5), // Lower threshold to trigger RLE more easily - compression: None, // Will use default compression if any - compression_level: None, - bss: Some(lance_encoding::compression_config::BssMode::Off), // Explicitly disable BSS to ensure RLE is used - minichunk_size: None, - }, - ); - - // For now, we'll skip Zstd compression since it's not imported - // In a real implementation, you could add other compression types here - - // Build encoding strategy with compression parameters - let encoding_strategy = lance_encoding::encoder::default_encoding_strategy_with_params( - LanceFileVersion::V2_1, - params, - ) - .unwrap(); - - // Configure file writer options - let options = FileWriterOptions { - encoding_strategy: Some(Arc::from(encoding_strategy)), - format_version: Some(LanceFileVersion::V2_1), - max_page_bytes: Some(64 * 1024), // 64KB pages - ..Default::default() - }; - - // Write the file - let path = TempObjFile::default(); - let object_store = ObjectStore::local(); - - let mut writer = FileWriter::try_new( - object_store.create(&path).await.unwrap(), - lance_schema.clone(), - options, - ) - .unwrap(); - - writer.write_batch(&batch).await.unwrap(); - writer.add_schema_metadata("compression_test", "configured_compression"); - writer.finish().await.unwrap(); - - // Now write the same data without compression overrides for comparison - let path_no_compression = TempObjFile::default(); - let default_options = FileWriterOptions { - format_version: Some(LanceFileVersion::V2_1), - max_page_bytes: Some(64 * 1024), - ..Default::default() - }; - - let mut writer_no_compression = FileWriter::try_new( - object_store.create(&path_no_compression).await.unwrap(), - lance_schema.clone(), - default_options, - ) - .unwrap(); - - writer_no_compression.write_batch(&batch).await.unwrap(); - writer_no_compression.finish().await.unwrap(); - - // Note: With our current data patterns and RLE compression, the compressed file - // might actually be slightly larger due to compression metadata overhead. - // This is expected and the test is mainly to verify the system works end-to-end. - - // Read back the compressed file and verify data integrity - let fs = FsFixture::default(); - let file_scheduler = fs - .scheduler - .open_file(&path, &CachedFileSize::unknown()) - .await - .unwrap(); - - let file_reader = FileReader::try_open( - file_scheduler, - None, - Arc::::default(), - &LanceCache::no_cache(), - FileReaderOptions::default(), - ) - .await - .unwrap(); - - // Verify metadata - let metadata = file_reader.metadata(); - assert_eq!(metadata.major_version, 2); - assert_eq!(metadata.minor_version, 1); - - let schema = file_reader.schema(); - assert_eq!( - schema.metadata.get("compression_test"), - Some(&"configured_compression".to_string()) - ); - - // Verify the actual encodings used - let column_metadatas = &metadata.column_metadatas; - - // Check customer_id column (index 0) - should use RLE due to our configuration - assert!(!column_metadatas[0].pages.is_empty()); - let customer_id_encoding = describe_encoding(&column_metadatas[0].pages[0]); - assert!( - customer_id_encoding.contains("RLE") || customer_id_encoding.contains("Rle"), - "customer_id column should use RLE encoding due to '*_id' pattern match, but got: {}", - customer_id_encoding - ); - - // Check product_id column (index 1) - should use RLE due to our configuration - assert!(!column_metadatas[1].pages.is_empty()); - let product_id_encoding = describe_encoding(&column_metadatas[1].pages[0]); - assert!( - product_id_encoding.contains("RLE") || product_id_encoding.contains("Rle"), - "product_id column should use RLE encoding due to '*_id' pattern match, but got: {}", - product_id_encoding - ); - } - - #[tokio::test] - async fn test_field_metadata_compression() { - // Test that field metadata compression settings are respected - let mut metadata = HashMap::new(); - metadata.insert( - lance_encoding::constants::COMPRESSION_META_KEY.to_string(), - "zstd".to_string(), - ); - metadata.insert( - lance_encoding::constants::COMPRESSION_LEVEL_META_KEY.to_string(), - "6".to_string(), - ); - - let arrow_schema = Arc::new(ArrowSchema::new(vec![ - ArrowField::new("id", DataType::Int32, false), - ArrowField::new("text", DataType::Utf8, false).with_metadata(metadata.clone()), - ArrowField::new("data", DataType::Int32, false).with_metadata(HashMap::from([( - lance_encoding::constants::COMPRESSION_META_KEY.to_string(), - "none".to_string(), - )])), - ])); - - let lance_schema = LanceSchema::try_from(arrow_schema.as_ref()).unwrap(); - - // Create test data - let id_array = Int32Array::from_iter_values(0..1000); - let text_array = StringArray::from_iter_values( - (0..1000).map(|i| format!("test string {} repeated text", i)), - ); - let data_array = Int32Array::from_iter_values((0..1000).map(|i| i * 2)); - - let batch = RecordBatch::try_new( - arrow_schema.clone(), - vec![ - Arc::new(id_array), - Arc::new(text_array), - Arc::new(data_array), - ], - ) - .unwrap(); - - let path = TempObjFile::default(); - let object_store = ObjectStore::local(); - - // Create encoding strategy that will read from field metadata - let params = CompressionParams::new(); - let encoding_strategy = lance_encoding::encoder::default_encoding_strategy_with_params( - LanceFileVersion::V2_1, - params, - ) - .unwrap(); - - let options = FileWriterOptions { - encoding_strategy: Some(Arc::from(encoding_strategy)), - format_version: Some(LanceFileVersion::V2_1), - ..Default::default() - }; - let mut writer = FileWriter::try_new( - object_store.create(&path).await.unwrap(), - lance_schema.clone(), - options, - ) - .unwrap(); - - writer.write_batch(&batch).await.unwrap(); - writer.finish().await.unwrap(); - - // Read back metadata - let fs = FsFixture::default(); - let file_scheduler = fs - .scheduler - .open_file(&path, &CachedFileSize::unknown()) - .await - .unwrap(); - let file_reader = FileReader::try_open( - file_scheduler, - None, - Arc::::default(), - &LanceCache::no_cache(), - FileReaderOptions::default(), - ) - .await - .unwrap(); - - let column_metadatas = &file_reader.metadata().column_metadatas; - - // The text column (index 1) should use zstd compression based on metadata - let text_encoding = describe_encoding(&column_metadatas[1].pages[0]); - // For string columns, we expect Binary encoding with zstd compression - assert!( - text_encoding.contains("Zstd"), - "text column should use zstd compression from field metadata, but got: {}", - text_encoding - ); - - // The data column (index 2) should use no compression based on metadata - let data_encoding = describe_encoding(&column_metadatas[2].pages[0]); - // For Int32 columns with "none" compression, we expect Flat encoding without compression - assert!( - data_encoding.contains("Flat") && data_encoding.contains("compression: None"), - "data column should use no compression from field metadata, but got: {}", - data_encoding - ); - } - - #[tokio::test] - async fn test_field_metadata_rle_threshold() { - // Test that RLE threshold from field metadata is respected - let mut metadata = HashMap::new(); - metadata.insert( - lance_encoding::constants::RLE_THRESHOLD_META_KEY.to_string(), - "0.9".to_string(), - ); - // Also set compression to ensure RLE is used - metadata.insert( - lance_encoding::constants::COMPRESSION_META_KEY.to_string(), - "lz4".to_string(), - ); - // Explicitly disable BSS to ensure RLE is tested - metadata.insert( - lance_encoding::constants::BSS_META_KEY.to_string(), - "off".to_string(), - ); - - let arrow_schema = Arc::new(ArrowSchema::new(vec![ - ArrowField::new("status", DataType::Int32, false).with_metadata(metadata), - ])); - - let lance_schema = LanceSchema::try_from(arrow_schema.as_ref()).unwrap(); - - // Create data with very high repetition (3 runs for 10000 values = 0.0003 ratio) - let status_array = Int32Array::from_iter_values( - std::iter::repeat_n(200, 8000) - .chain(std::iter::repeat_n(404, 1500)) - .chain(std::iter::repeat_n(500, 500)), - ); - - let batch = - RecordBatch::try_new(arrow_schema.clone(), vec![Arc::new(status_array)]).unwrap(); - - let path = TempObjFile::default(); - let object_store = ObjectStore::local(); - - // Create encoding strategy that will read from field metadata - let params = CompressionParams::new(); - let encoding_strategy = lance_encoding::encoder::default_encoding_strategy_with_params( - LanceFileVersion::V2_1, - params, - ) - .unwrap(); - - let options = FileWriterOptions { - encoding_strategy: Some(Arc::from(encoding_strategy)), - format_version: Some(LanceFileVersion::V2_1), - ..Default::default() - }; - let mut writer = FileWriter::try_new( - object_store.create(&path).await.unwrap(), - lance_schema.clone(), - options, - ) - .unwrap(); - - writer.write_batch(&batch).await.unwrap(); - writer.finish().await.unwrap(); - - // Read back and check encoding - let fs = FsFixture::default(); - let file_scheduler = fs - .scheduler - .open_file(&path, &CachedFileSize::unknown()) - .await - .unwrap(); - let file_reader = FileReader::try_open( - file_scheduler, - None, - Arc::::default(), - &LanceCache::no_cache(), - FileReaderOptions::default(), - ) - .await - .unwrap(); - - let column_metadatas = &file_reader.metadata().column_metadatas; - let status_encoding = describe_encoding(&column_metadatas[0].pages[0]); - assert!( - status_encoding.contains("RLE") || status_encoding.contains("Rle"), - "status column should use RLE encoding due to metadata threshold, but got: {}", - status_encoding - ); - } - - #[tokio::test] - async fn test_large_page_split_on_read() { - use arrow_array::Array; - use futures::TryStreamExt; - use lance_encoding::decoder::FilterExpression; - use lance_io::ReadBatchParams; - - // Test that large pages written with relaxed limits can be split during read - - let arrow_field = ArrowField::new("data", DataType::Binary, false); - let arrow_schema = ArrowSchema::new(vec![arrow_field]); - let lance_schema = LanceSchema::try_from(&arrow_schema).unwrap(); - - // Create a large binary value (40MB) to trigger large page creation - let large_value = vec![42u8; 40 * 1024 * 1024]; - let array = arrow_array::BinaryArray::from(vec![ - Some(large_value.as_slice()), - Some(b"small value"), - ]); - let batch = RecordBatch::try_new(Arc::new(arrow_schema), vec![Arc::new(array)]).unwrap(); - - // Write with relaxed page size limit (128MB) - let options = FileWriterOptions { - max_page_bytes: Some(128 * 1024 * 1024), - format_version: Some(LanceFileVersion::V2_1), - ..Default::default() - }; - - let fs = FsFixture::default(); - let path = fs.tmp_path; - - let mut writer = FileWriter::try_new( - fs.object_store.create(&path).await.unwrap(), - lance_schema.clone(), - options, - ) - .unwrap(); - - writer.write_batch(&batch).await.unwrap(); - let write_summary = writer.finish().await.unwrap(); - assert_eq!(write_summary.num_rows, 2); - assert_eq!( - write_summary.size_bytes, - fs.object_store.size(&path).await.unwrap() - ); - - // Read back with split configuration - let file_scheduler = fs - .scheduler - .open_file(&path, &CachedFileSize::unknown()) - .await - .unwrap(); - - // Configure reader to split pages larger than 10MB into chunks - let reader_options = FileReaderOptions { - read_chunk_size: 10 * 1024 * 1024, // 10MB chunks - ..Default::default() - }; - - let file_reader = FileReader::try_open( - file_scheduler, - None, - Arc::::default(), - &LanceCache::no_cache(), - reader_options, - ) - .await - .unwrap(); - - // Read the data back - let stream = file_reader - .read_stream( - ReadBatchParams::RangeFull, - 1024, - 10, // batch_readahead - FilterExpression::no_filter(), - ) - .await - .unwrap(); - - let batches: Vec = stream.try_collect().await.unwrap(); - assert_eq!(batches.len(), 1); - - // Verify the data is correctly read despite splitting - let read_array = batches[0].column(0); - let read_binary = read_array - .as_any() - .downcast_ref::() - .unwrap(); - - assert_eq!(read_binary.len(), 2); - assert_eq!(read_binary.value(0).len(), 40 * 1024 * 1024); - assert_eq!(read_binary.value(1), b"small value"); - - // Verify first value matches what we wrote - assert!(read_binary.value(0).iter().all(|&b| b == 42u8)); - } - - fn spill_config() -> (TempObjFile, Arc) { - let spill_path = TempObjFile::default(); - (spill_path, Arc::new(ObjectStore::local())) - } - - fn make_batches(num_batches: i32, num_cols: usize, rows_per_batch: i32) -> Vec { - let fields: Vec<_> = (0..num_cols) - .map(|c| ArrowField::new(format!("c{c}"), DataType::Int32, false)) - .collect(); - let schema = Arc::new(ArrowSchema::new(fields)); - (0..num_batches) - .map(|i| { - let cols: Vec> = (0..num_cols) - .map(|c| { - let start = (i * rows_per_batch + c as i32) * 100; - Arc::new(Int32Array::from_iter_values(start..start + rows_per_batch)) - as Arc - }) - .collect(); - RecordBatch::try_new(schema.clone(), cols).unwrap() - }) - .collect() - } - - async fn write_and_read_batches( - batches: &[RecordBatch], - spill: Option<(Arc, object_store::path::Path)>, - ) -> Vec { - let fs = FsFixture::default(); - let lance_schema = LanceSchema::try_from(batches[0].schema().as_ref()).unwrap(); - let writer = fs.object_store.create(&fs.tmp_path).await.unwrap(); - let mut file_writer = - FileWriter::try_new(writer, lance_schema, FileWriterOptions::default()).unwrap(); - if let Some((store, path)) = spill { - file_writer = file_writer.with_page_metadata_spill(store, path); - } - for batch in batches { - file_writer.write_batch(batch).await.unwrap(); - } - file_writer.add_schema_metadata("foo", "bar"); - file_writer.finish().await.unwrap(); - - crate::testing::read_lance_file( - &fs, - Arc::::default(), - lance_encoding::decoder::FilterExpression::no_filter(), - ) - .await - } - - #[rstest::rstest] - #[case::multi_col(20, 2, 100)] - #[case::many_batches(50, 2, 100)] - #[tokio::test] - async fn test_page_metadata_spill_roundtrip( - #[case] num_batches: i32, - #[case] num_cols: usize, - #[case] rows_per_batch: i32, - ) { - let batches = make_batches(num_batches, num_cols, rows_per_batch); - let baseline = write_and_read_batches(&batches, None).await; - let (spill_path, spill_store) = spill_config(); - let spilled = - write_and_read_batches(&batches, Some((spill_store, spill_path.as_ref().clone()))) - .await; - assert_eq!(baseline, spilled); - } - - #[tokio::test] - async fn test_page_metadata_spill_many_columns() { - // Many columns forces small per-column buffer limits, exercising mid-write flushing. - let batches = make_batches(10, 500, 100); - let baseline = write_and_read_batches(&batches, None).await; - let (spill_path, spill_store) = spill_config(); - let spilled = - write_and_read_batches(&batches, Some((spill_store, spill_path.as_ref().clone()))) - .await; - assert_eq!(baseline, spilled); - } -} +#[path = "writer_tests.rs"] +mod writer_tests; diff --git a/rust/lance-file/src/writer/structural.rs b/rust/lance-file/src/writer/structural.rs new file mode 100644 index 00000000000..3d180b83dca --- /dev/null +++ b/rust/lance-file/src/writer/structural.rs @@ -0,0 +1,837 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright The Lance Authors + +use core::panic; +use std::{collections::HashMap, sync::Arc}; + +use arrow_array::{ArrayRef, RecordBatch}; +use arrow_data::ArrayData; +use bytes::{Buf, Bytes, BytesMut}; +use futures::{StreamExt, stream::FuturesOrdered}; +use lance_core::{ + Error, Result, + datatypes::{Field, Schema}, + utils::bit::pad_bytes, +}; +use lance_encoding::{ + decoder::PageEncoding, + encoder::{ + BatchEncoder, EncodeTask, EncodedBatch, EncodedPage, EncodingOptions, FieldEncoder, + OutOfLineBuffers, + }, + repdef::RepDefBuilder, +}; +use lance_io::{object_store::ObjectStore, traits::Writer as ObjectWriter}; +use log::{debug, warn}; +use object_store::path::Path; +use prost::Message; +use prost_types::Any; +use tokio::io::{AsyncWrite, AsyncWriteExt}; +use tracing::instrument; + +use crate::{ + datatypes::FieldsWithMeta, + format::{pb, pbfile}, + writer::{ENV_LANCE_FILE_WRITER_MAX_PAGE_BYTES, FileWriterOptions, PAGE_BUFFER_ALIGNMENT}, +}; + +const PAD_BUFFER: [u8; PAGE_BUFFER_ALIGNMENT] = [72; PAGE_BUFFER_ALIGNMENT]; + +// V2.1+ splits large pages on read instead of write. This write-time limit is +// retained as a best-effort memory bound and is not a grammar decision. +const MAX_PAGE_BYTES: usize = 32 * 1024 * 1024; + +// Total in-memory budget for serialized page metadata before spill. +const DEFAULT_SPILL_BUFFER_LIMIT: usize = 256 * 1024; + +struct PageMetadataSpill { + writer: Box, + object_store: Arc, + path: Path, + position: u64, + column_buffers: Vec>, + column_chunks: Vec>, + per_column_limit: usize, +} + +impl PageMetadataSpill { + async fn new(object_store: Arc, path: Path, num_columns: usize) -> Result { + let writer = object_store.create(&path).await?; + let per_column_limit = (DEFAULT_SPILL_BUFFER_LIMIT / num_columns.max(1)).max(64); + Ok(Self { + writer, + object_store, + path, + position: 0, + column_buffers: vec![Vec::new(); num_columns], + column_chunks: vec![Vec::new(); num_columns], + per_column_limit, + }) + } + + async fn append_page( + &mut self, + column_index: usize, + page: &pbfile::column_metadata::Page, + ) -> Result<()> { + page.encode_length_delimited(&mut self.column_buffers[column_index]) + .map_err(|error| { + Error::io_source(Box::new(std::io::Error::new( + std::io::ErrorKind::InvalidData, + error, + ))) + })?; + if self.column_buffers[column_index].len() >= self.per_column_limit { + self.flush_column(column_index).await?; + } + Ok(()) + } + + async fn flush_column(&mut self, column_index: usize) -> Result<()> { + let buffer = &self.column_buffers[column_index]; + if buffer.is_empty() { + return Ok(()); + } + let len = buffer.len(); + self.writer.write_all(buffer).await?; + self.column_chunks[column_index].push((self.position, len as u32)); + self.position += len as u64; + self.column_buffers[column_index].clear(); + Ok(()) + } + + async fn shutdown_writer(&mut self) -> Result<()> { + for column_index in 0..self.column_buffers.len() { + self.flush_column(column_index).await?; + } + ObjectWriter::shutdown(self.writer.as_mut()).await?; + Ok(()) + } +} + +fn decode_spilled_chunk(data: &Bytes) -> Result> { + let mut pages = Vec::new(); + let mut cursor = data.clone(); + while cursor.has_remaining() { + let page = pbfile::column_metadata::Page::decode_length_delimited(&mut cursor).map_err( + |error| { + Error::io_source(Box::new(std::io::Error::new( + std::io::ErrorKind::InvalidData, + error, + ))) + }, + )?; + pages.push(page); + } + Ok(pages) +} + +enum PageSpillState { + Pending(Arc, Path), + Active(PageMetadataSpill), +} + +fn initial_column_metadata() -> pbfile::ColumnMetadata { + pbfile::ColumnMetadata { + pages: Vec::new(), + buffer_offsets: Vec::new(), + buffer_sizes: Vec::new(), + encoding: None, + } +} + +/// Writes the structural page and metadata representation used by v2.1+. +/// +/// This component does not choose a file version, field encoding strategy, or +/// footer identity. Exact version writers opt into this representation and own +/// the order in which its operations are invoked. +pub struct StructuralFileSink { + writer: Box, + column_metadata: Vec, + num_columns: u32, + global_buffers: Vec<(u64, u64)>, + page_spill: Option, +} + +impl StructuralFileSink { + pub fn new(writer: Box) -> Self { + Self { + writer, + column_metadata: Vec::new(), + num_columns: 0, + global_buffers: Vec::new(), + page_spill: None, + } + } + + pub fn with_page_metadata_spill(&mut self, object_store: Arc, path: Path) { + self.page_spill = Some(PageSpillState::Pending(object_store, path)); + } + + pub fn initialize_columns(&mut self, num_columns: u32) { + self.num_columns = num_columns; + self.column_metadata = vec![initial_column_metadata(); num_columns as usize]; + } + + pub fn initialize_with_external_metadata( + &mut self, + column_metadata: Vec, + ) { + self.num_columns = column_metadata.len() as u32; + self.column_metadata = column_metadata; + } + + async fn write_aligned_buffer_to( + writer: &mut (impl AsyncWrite + Unpin), + buffer: &[u8], + ) -> Result<()> { + writer.write_all(buffer).await?; + let padding = pad_bytes::(buffer.len()); + writer.write_all(&PAD_BUFFER[..padding]).await?; + Ok(()) + } + + pub async fn write_aligned_buffer(&mut self, buffer: &[u8]) -> Result<()> { + Self::write_aligned_buffer_to(&mut self.writer, buffer).await + } + + pub async fn write_raw(&mut self, buffer: &[u8]) -> Result<()> { + self.writer.write_all(buffer).await?; + Ok(()) + } + + pub async fn write_page(&mut self, encoded_page: EncodedPage) -> Result<()> { + let buffers = encoded_page.data; + let mut buffer_offsets = Vec::with_capacity(buffers.len()); + let mut buffer_sizes = Vec::with_capacity(buffers.len()); + for buffer in buffers { + buffer_offsets.push(self.tell().await?); + buffer_sizes.push(buffer.len() as u64); + self.write_aligned_buffer(&buffer).await?; + } + let encoded_encoding = match encoded_page.description { + PageEncoding::Legacy(array_encoding) => Any::from_msg(&array_encoding)?.encode_to_vec(), + PageEncoding::Structural(page_layout) => Any::from_msg(&page_layout)?.encode_to_vec(), + }; + let page = pbfile::column_metadata::Page { + buffer_offsets, + buffer_sizes, + encoding: Some(pbfile::Encoding { + location: Some(pbfile::encoding::Location::Direct(pbfile::DirectEncoding { + encoding: encoded_encoding, + })), + }), + length: encoded_page.num_rows, + priority: encoded_page.row_number, + }; + let column_index = encoded_page.column_idx as usize; + if matches!(&self.page_spill, Some(PageSpillState::Pending(..))) { + let Some(PageSpillState::Pending(store, path)) = self.page_spill.take() else { + unreachable!() + }; + self.page_spill = Some(PageSpillState::Active( + PageMetadataSpill::new(store, path, self.num_columns as usize).await?, + )); + } + match &mut self.page_spill { + Some(PageSpillState::Active(spill)) => spill.append_page(column_index, &page).await?, + None => self.column_metadata[column_index].pages.push(page), + Some(PageSpillState::Pending(..)) => unreachable!(), + } + Ok(()) + } + + #[instrument(skip_all, level = "debug")] + pub async fn write_pages( + &mut self, + mut encoding_tasks: FuturesOrdered, + ) -> Result<()> { + while let Some(encoding_task) = encoding_tasks.next().await { + self.write_page(encoding_task?).await?; + } + self.writer.flush().await?; + Ok(()) + } + + pub async fn write_column_buffer_at( + &mut self, + column_index: usize, + position: u64, + buffer: &[u8], + ) -> Result<()> { + self.write_aligned_buffer(buffer).await?; + let metadata = &mut self.column_metadata[column_index]; + metadata.buffer_offsets.push(position); + metadata.buffer_sizes.push(buffer.len() as u64); + Ok(()) + } + + pub fn set_column_encoding(&mut self, column_index: usize, encoding: pbfile::Encoding) { + self.column_metadata[column_index].encoding = Some(encoding); + } + + pub async fn add_global_buffer(&mut self, buffer: Bytes) -> Result { + let position = self.tell().await?; + let len = buffer.len() as u64; + self.write_aligned_buffer(&buffer).await?; + self.global_buffers.push((position, len)); + Ok(self.global_buffers.len() as u32) + } + + pub async fn write_global_buffers( + &mut self, + descriptor: pb::FileDescriptor, + ) -> Result> { + let descriptor_bytes = descriptor.encode_to_vec(); + let descriptor_len = descriptor_bytes.len() as u64; + let descriptor_position = self.tell().await?; + self.writer.write_all(&descriptor_bytes).await?; + let mut offsets = Vec::with_capacity(1 + self.global_buffers.len()); + offsets.push((descriptor_position, descriptor_len)); + offsets.append(&mut self.global_buffers); + Ok(offsets) + } + + async fn write_column_metadata( + &mut self, + metadata: pbfile::ColumnMetadata, + ) -> Result<(u64, u64)> { + let metadata_bytes = metadata.encode_to_vec(); + let position = self.tell().await?; + let len = metadata_bytes.len() as u64; + self.writer.write_all(&metadata_bytes).await?; + Ok((position, len)) + } + + pub async fn write_column_metadatas(&mut self) -> Result> { + let metadatas = std::mem::take(&mut self.column_metadata); + let spill_state = self.page_spill.take(); + let (spill_chunks, spill_reader) = + if let Some(PageSpillState::Active(mut spill)) = spill_state { + spill.shutdown_writer().await?; + let reader = spill.object_store.open(&spill.path).await?; + let chunks = std::mem::take(&mut spill.column_chunks); + (chunks, Some(reader)) + } else { + (Vec::new(), None) + }; + + let mut metadata_positions = Vec::with_capacity(metadatas.len()); + for (column_index, mut metadata) in metadatas.into_iter().enumerate() { + if let Some(reader) = &spill_reader { + let mut pages = Vec::new(); + for &(offset, len) in &spill_chunks[column_index] { + let data = reader + .get_range(offset as usize..(offset as usize + len as usize)) + .await + .map_err(|error| Error::io_source(Box::new(error)))?; + pages.extend(decode_spilled_chunk(&data)?); + } + metadata.pages = pages; + } + metadata_positions.push(self.write_column_metadata(metadata).await?); + } + Ok(metadata_positions) + } + + pub async fn write_offset_table(&mut self, offsets: &[(u64, u64)]) -> Result { + let start = self.tell().await?; + for (position, len) in offsets { + self.writer.write_u64_le(*position).await?; + self.writer.write_u64_le(*len).await?; + } + Ok(start) + } + + pub async fn write_external_buffer(&mut self, bytes: &[u8]) -> Result<(u64, u64)> { + const ZERO_PADDING: [u8; PAGE_BUFFER_ALIGNMENT] = [0; PAGE_BUFFER_ALIGNMENT]; + let position = self.tell().await?; + let padding = (PAGE_BUFFER_ALIGNMENT - position as usize % PAGE_BUFFER_ALIGNMENT) + % PAGE_BUFFER_ALIGNMENT; + self.writer.write_all(&ZERO_PADDING[..padding]).await?; + let start = position + padding as u64; + self.writer.write_all(bytes).await?; + Ok((start, bytes.len() as u64)) + } + + pub fn output_mut(&mut self) -> &mut dyn ObjectWriter { + self.writer.as_mut() + } + + pub async fn tell(&mut self) -> Result { + Ok(self.writer.tell().await? as u64) + } + + pub fn num_columns(&self) -> u32 { + self.num_columns + } + + pub async fn shutdown(&mut self) -> Result { + let result = ObjectWriter::shutdown(self.writer.as_mut()).await?; + Ok(result.size as u64) + } +} + +/// Runs field encoders and row accounting without selecting a file grammar. +/// +/// The exact version writer constructs a [`BatchEncoder`] and supplies it when +/// a schema is initialized. This component only executes that decision. +pub struct EncodingPipeline { + schema: Option, + field_encoders: Vec>, + field_id_to_column_indices: Vec<(u32, u32)>, + rows_written: u64, + field_rows_written: Vec, + schema_metadata: HashMap, + options: FileWriterOptions, +} + +impl EncodingPipeline { + pub fn new(options: FileWriterOptions) -> Self { + Self { + schema: None, + field_encoders: Vec::new(), + field_id_to_column_indices: Vec::new(), + rows_written: 0, + field_rows_written: Vec::new(), + schema_metadata: HashMap::new(), + options, + } + } + + pub fn encoding_options(&self, schema: &Schema) -> EncodingOptions { + let cache_bytes_per_column = if let Some(data_cache_bytes) = self.options.data_cache_bytes { + data_cache_bytes / schema.fields.len() as u64 + } else { + 8 * 1024 * 1024 + }; + let max_page_bytes = self.options.max_page_bytes.unwrap_or_else(|| { + std::env::var(ENV_LANCE_FILE_WRITER_MAX_PAGE_BYTES) + .map(|value| { + value.parse::().unwrap_or_else(|error| { + warn!( + "Failed to parse {}: {}, using default", + ENV_LANCE_FILE_WRITER_MAX_PAGE_BYTES, error + ); + MAX_PAGE_BYTES as u64 + }) + }) + .unwrap_or(MAX_PAGE_BYTES as u64) + }); + + EncodingOptions { + cache_bytes_per_column, + max_page_bytes, + keep_original_array: self.options.keep_original_array.unwrap_or(false), + buffer_alignment: PAGE_BUFFER_ALIGNMENT as u64, + } + } + + pub fn initialize( + &mut self, + mut schema: Schema, + encoder: BatchEncoder, + sink: &mut StructuralFileSink, + ) { + sink.initialize_columns(encoder.num_columns()); + self.field_rows_written = vec![0; encoder.field_encoders.len()]; + self.field_encoders = encoder.field_encoders; + self.field_id_to_column_indices = encoder.field_id_to_column_index; + self.schema_metadata + .extend(std::mem::take(&mut schema.metadata)); + self.schema = Some(schema); + } + + pub fn is_initialized(&self) -> bool { + self.schema.is_some() + } + + fn verify_field_nullability(array: &ArrayData, field: &Field) -> Result<()> { + if !field.nullable && array.null_count() > 0 { + return Err(Error::invalid_input(format!( + "The field `{}` contained null values even though the field is marked non-null in the schema", + field.name + ))); + } + for (child_field, child_array) in field.children.iter().zip(array.child_data()) { + Self::verify_field_nullability(child_array, child_field)?; + } + Ok(()) + } + + fn verify_nullability_constraints(&self, batch: &RecordBatch) -> Result<()> { + for (column, field) in batch + .columns() + .iter() + .zip(self.schema.as_ref().unwrap().fields.iter()) + { + Self::verify_field_nullability(&column.to_data(), field)?; + } + Ok(()) + } + + fn encode_columns( + &mut self, + fields: &[(usize, ArrayRef)], + external_buffers: &mut OutOfLineBuffers, + ) -> Result>> { + let row_numbers = fields + .iter() + .map(|(field_index, _)| self.field_rows_written[*field_index]) + .collect::>(); + fields + .iter() + .zip(row_numbers) + .map(|((field_index, array), row_number)| { + self.field_encoders[*field_index].maybe_encode( + array.clone(), + external_buffers, + RepDefBuilder::default(), + row_number, + array.len() as u64, + ) + }) + .collect() + } + + fn encode_batch( + &mut self, + batch: &RecordBatch, + external_buffers: &mut OutOfLineBuffers, + ) -> Result>> { + let field_arrays = self + .schema + .as_ref() + .unwrap() + .fields + .iter() + .enumerate() + .map(|(field_index, field)| { + let array = batch.column_by_name(&field.name).ok_or_else(|| { + Error::invalid_input_source( + format!( + "Cannot write batch. The batch was missing the column `{}`", + field.name + ) + .into(), + ) + })?; + Ok((field_index, array.clone())) + }) + .collect::>>()?; + self.encode_columns(&field_arrays, external_buffers) + } + + #[instrument(skip_all, level = "debug")] + pub async fn write_batch( + &mut self, + batch: &RecordBatch, + sink: &mut StructuralFileSink, + ) -> Result<()> { + debug!( + "write_batch called with {} rows, {} columns, and {} bytes of data", + batch.num_rows(), + batch.num_columns(), + batch.get_array_memory_size() + ); + self.verify_nullability_constraints(batch)?; + let num_rows = batch.num_rows() as u64; + if num_rows == 0 { + return Ok(()); + } + if num_rows > u32::MAX as u64 { + return Err(Error::invalid_input_source( + "cannot write Lance files with more than 2^32 rows".into(), + )); + } + + let mut external_buffers = + OutOfLineBuffers::new(sink.tell().await?, PAGE_BUFFER_ALIGNMENT as u64); + let encoding_tasks = self.encode_batch(batch, &mut external_buffers)?; + for external_buffer in external_buffers.take_buffers() { + sink.write_aligned_buffer(&external_buffer).await?; + } + let encoding_tasks = encoding_tasks + .into_iter() + .flatten() + .collect::>(); + + if self.rows_written.checked_add(num_rows).is_none() { + return Err(Error::invalid_input_source(format!( + "cannot write batch with {} rows because {} rows have already been written and Lance files cannot contain more than 2^64 rows", + num_rows, self.rows_written + ).into())); + } + for field_rows in &mut self.field_rows_written { + *field_rows += num_rows; + } + self.rows_written += num_rows; + sink.write_pages(encoding_tasks).await + } + + pub async fn write_column( + &mut self, + column_index: usize, + array: ArrayRef, + sink: &mut StructuralFileSink, + ) -> Result<()> { + let schema = self.schema.as_ref().ok_or_else(|| { + Error::invalid_input_source( + "write_column requires the writer to be created with an explicit schema".into(), + ) + })?; + let field = schema.fields.get(column_index).ok_or_else(|| { + Error::invalid_input_source( + format!( + "write_column: field index {} is out of bounds (schema has {} fields)", + column_index, + schema.fields.len() + ) + .into(), + ) + })?; + if array.len() as u64 > u32::MAX as u64 { + return Err(Error::invalid_input_source( + "cannot write Lance files with more than 2^32 rows".into(), + )); + } + Self::verify_field_nullability(&array.to_data(), field)?; + if array.is_empty() { + return Ok(()); + } + + let fields = [(column_index, array)]; + let mut external_buffers = + OutOfLineBuffers::new(sink.tell().await?, PAGE_BUFFER_ALIGNMENT as u64); + let encoding_tasks = self.encode_columns(&fields, &mut external_buffers)?; + for external_buffer in external_buffers.take_buffers() { + sink.write_aligned_buffer(&external_buffer).await?; + } + let encoding_tasks = encoding_tasks + .into_iter() + .flatten() + .collect::>(); + for (field_index, array) in &fields { + let new_total = self.field_rows_written[*field_index] + array.len() as u64; + self.field_rows_written[*field_index] = new_total; + self.rows_written = self.rows_written.max(new_total); + } + sink.write_pages(encoding_tasks).await + } + + pub async fn flush(&mut self, sink: &mut StructuralFileSink) -> Result<()> { + let mut external_buffers = + OutOfLineBuffers::new(sink.tell().await?, PAGE_BUFFER_ALIGNMENT as u64); + let encoding_tasks = self + .field_encoders + .iter_mut() + .map(|writer| writer.flush(&mut external_buffers)) + .collect::>>()?; + for external_buffer in external_buffers.take_buffers() { + sink.write_aligned_buffer(&external_buffer).await?; + } + sink.write_pages( + encoding_tasks + .into_iter() + .flatten() + .collect::>(), + ) + .await + } + + pub async fn finish_encoders(&mut self, sink: &mut StructuralFileSink) -> Result<()> { + if self.field_encoders.is_empty() { + return Ok(()); + } + let mut column_index = 0; + for mut writer in std::mem::take(&mut self.field_encoders) { + let mut external_buffers = + OutOfLineBuffers::new(sink.tell().await?, PAGE_BUFFER_ALIGNMENT as u64); + let columns = writer.finish(&mut external_buffers).await?; + for buffer in external_buffers.take_buffers() { + sink.write_raw(&buffer).await?; + } + debug_assert_eq!( + columns.len(), + writer.num_columns() as usize, + "Expected {} columns from column at index {} and got {}", + writer.num_columns(), + column_index, + columns.len() + ); + for column in columns { + for page in column.final_pages { + sink.write_page(page).await?; + } + let mut buffer_position = sink.tell().await?; + for buffer in column.column_buffers { + sink.write_column_buffer_at(column_index, buffer_position, &buffer) + .await?; + buffer_position += buffer.len() as u64; + } + let encoded_encoding = Any::from_msg(&column.encoding)?.encode_to_vec(); + sink.set_column_encoding( + column_index, + pbfile::Encoding { + location: Some(pbfile::encoding::Location::Direct( + pbfile::DirectEncoding { + encoding: encoded_encoding, + }, + )), + }, + ); + column_index += 1; + } + } + if column_index != sink.num_columns() as usize { + panic!( + "Column writers finished with {} columns but we expected {}", + column_index, + sink.num_columns() + ); + } + Ok(()) + } + + pub fn add_schema_metadata(&mut self, key: impl Into, value: impl Into) { + self.schema_metadata.insert(key.into(), value.into()); + } + + pub fn initialize_with_external_metadata(&mut self, schema: Schema, rows_written: u64) { + self.schema = Some(schema); + self.rows_written = rows_written; + } + + pub fn make_file_descriptor(&mut self) -> Result { + let schema = self.schema.as_mut().ok_or_else(|| { + Error::invalid_input( + "No schema provided on writer open and no data provided. Schema is unknown and file cannot be created", + ) + })?; + schema.metadata = std::mem::take(&mut self.schema_metadata); + schema + .fields + .iter_mut() + .for_each(|field| field.unload_blobs_recursive()); + make_file_descriptor(schema, self.rows_written) + } + + pub fn rows_written(&self) -> u64 { + self.rows_written + } + + pub fn field_id_to_column_indices(&self) -> &[(u32, u32)] { + &self.field_id_to_column_indices + } +} + +fn make_file_descriptor(schema: &Schema, num_rows: u64) -> Result { + let fields_with_meta = FieldsWithMeta::from(schema); + Ok(pb::FileDescriptor { + schema: Some(pb::Schema { + fields: fields_with_meta.fields.0, + metadata: fields_with_meta.metadata, + }), + length: num_rows, + }) +} + +/// Structural file body produced before an exact version appends its footer. +pub struct EncodedBatchBody { + pub data: BytesMut, + pub column_metadata_start: u64, + pub column_metadata_offsets_start: u64, + pub global_buffer_offsets_start: u64, + pub num_global_buffers: u32, + pub num_columns: u32, +} + +pub fn encode_batch_body(batch: &EncodedBatch, write_schema: bool) -> Result { + use bytes::BufMut; + + let mut data = BytesMut::with_capacity(batch.data.len() + 1024 * 1024); + data.extend_from_slice(&batch.data); + let global_buffers = if write_schema { + let schema_start = data.len() as u64; + let schema = Schema::try_from(batch.schema.as_ref())?; + let descriptor = make_file_descriptor(&schema, batch.num_rows)?; + let descriptor_bytes = descriptor.encode_to_vec(); + let descriptor_len = descriptor_bytes.len() as u64; + data.extend_from_slice(&descriptor_bytes); + vec![(schema_start, descriptor_len)] + } else { + Vec::new() + }; + let column_metadata_start = data.len() as u64; + + let mut column_metadata_positions = Vec::with_capacity(batch.page_table.len()); + for column in &batch.page_table { + let position = data.len() as u64; + let pages = column + .page_infos + .iter() + .map(|page_info| { + let encoded_encoding = match &page_info.encoding { + PageEncoding::Legacy(array_encoding) => { + Any::from_msg(array_encoding)?.encode_to_vec() + } + PageEncoding::Structural(page_layout) => { + Any::from_msg(page_layout)?.encode_to_vec() + } + }; + let (buffer_offsets, buffer_sizes): (Vec<_>, Vec<_>) = + page_info.buffer_offsets_and_sizes.iter().copied().unzip(); + Ok(pbfile::column_metadata::Page { + buffer_offsets, + buffer_sizes, + encoding: Some(pbfile::Encoding { + location: Some(pbfile::encoding::Location::Direct( + pbfile::DirectEncoding { + encoding: encoded_encoding, + }, + )), + }), + length: page_info.num_rows, + priority: page_info.priority, + }) + }) + .collect::>>()?; + let (buffer_offsets, buffer_sizes): (Vec<_>, Vec<_>) = + column.buffer_offsets_and_sizes.iter().copied().unzip(); + let encoded_column_encoding = Any::from_msg(&column.encoding)?.encode_to_vec(); + let metadata = pbfile::ColumnMetadata { + pages, + buffer_offsets, + buffer_sizes, + encoding: Some(pbfile::Encoding { + location: Some(pbfile::encoding::Location::Direct(pbfile::DirectEncoding { + encoding: encoded_column_encoding, + })), + }), + }; + let metadata_bytes = metadata.encode_to_vec(); + column_metadata_positions.push((position, metadata_bytes.len() as u64)); + data.extend_from_slice(&metadata_bytes); + } + + let column_metadata_offsets_start = data.len() as u64; + for (position, len) in column_metadata_positions { + data.put_u64_le(position); + data.put_u64_le(len); + } + let global_buffer_offsets_start = data.len() as u64; + let num_global_buffers = global_buffers.len() as u32; + for (position, len) in global_buffers { + data.put_u64_le(position); + data.put_u64_le(len); + } + + Ok(EncodedBatchBody { + data, + column_metadata_start, + column_metadata_offsets_start, + global_buffer_offsets_start, + num_global_buffers, + num_columns: batch.page_table.len() as u32, + }) +} diff --git a/rust/lance-file/src/writer_tests.rs b/rust/lance-file/src/writer_tests.rs new file mode 100644 index 00000000000..49e40c01185 --- /dev/null +++ b/rust/lance-file/src/writer_tests.rs @@ -0,0 +1,1382 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright The Lance Authors + +#[cfg(test)] +mod tests { + use std::collections::HashMap; + use std::sync::Arc; + + use crate::reader::{FileReader, FileReaderOptions, ReaderProjection, describe_encoding}; + use crate::testing::FsFixture; + use crate::version::ConcreteFileVersion; + use crate::versions; + use crate::writer::{ENV_LANCE_FILE_WRITER_MAX_PAGE_BYTES, FileWriter, FileWriterOptions}; + use arrow_array::builder::{Float32Builder, Int32Builder}; + use arrow_array::types::Float64Type; + use arrow_array::{ + Array, ArrayRef, Int32Array, RecordBatch, RecordBatchReader, StringArray, UInt64Array, + }; + use arrow_schema::{DataType, Field, Field as ArrowField, Schema, Schema as ArrowSchema}; + use lance_core::cache::LanceCache; + use lance_core::datatypes::Schema as LanceSchema; + use lance_core::utils::tempfile::TempObjFile; + use lance_datagen::{BatchCount, RowCount, array, gen_batch}; + use lance_encoding::compression_config::{CompressionFieldParams, CompressionParams}; + use lance_encoding::decoder::DecoderPlugins; + use lance_io::object_store::ObjectStore; + use lance_io::traits::Writer; + use lance_io::utils::CachedFileSize; + use rstest::rstest; + + fn create_writer( + object_writer: Box, + schema: LanceSchema, + version: ConcreteFileVersion, + options: FileWriterOptions, + ) -> lance_core::Result { + versions::create_writer(version, object_writer, schema, options) + } + + fn create_v2_1_writer_with_compression( + object_writer: Box, + schema: LanceSchema, + options: FileWriterOptions, + compression: CompressionParams, + ) -> lance_core::Result { + versions::v2_1::create_writer_with_compression(object_writer, schema, options, compression) + .map(Into::into) + } + + fn reader_projection_from_column_names( + version: ConcreteFileVersion, + schema: &LanceSchema, + column_names: &[&str], + ) -> lance_core::Result { + versions::reader_projection_from_column_names(version, schema, column_names) + } + + #[tokio::test] + async fn test_basic_write() { + let tmp_path = TempObjFile::default(); + let obj_store = Arc::new(ObjectStore::local()); + + let reader = gen_batch() + .col("score", array::rand::()) + .into_reader_rows(RowCount::from(1000), BatchCount::from(10)); + + let writer = obj_store.create(&tmp_path).await.unwrap(); + + let lance_schema = + lance_core::datatypes::Schema::try_from(reader.schema().as_ref()).unwrap(); + + let mut file_writer = create_writer( + writer, + lance_schema, + ConcreteFileVersion::V2_1, + FileWriterOptions::default(), + ) + .unwrap(); + + for batch in reader { + file_writer.write_batch(&batch.unwrap()).await.unwrap(); + } + file_writer.add_schema_metadata("foo", "bar"); + file_writer.finish().await.unwrap(); + // Tests asserting the contents of the written file are in reader.rs + } + + #[tokio::test] + async fn test_write_empty() { + let tmp_path = TempObjFile::default(); + let obj_store = Arc::new(ObjectStore::local()); + + let reader = gen_batch() + .col("score", array::rand::()) + .into_reader_rows(RowCount::from(0), BatchCount::from(0)); + + let writer = obj_store.create(&tmp_path).await.unwrap(); + + let lance_schema = + lance_core::datatypes::Schema::try_from(reader.schema().as_ref()).unwrap(); + + let mut file_writer = create_writer( + writer, + lance_schema, + ConcreteFileVersion::V2_1, + FileWriterOptions::default(), + ) + .unwrap(); + + for batch in reader { + file_writer.write_batch(&batch.unwrap()).await.unwrap(); + } + file_writer.add_schema_metadata("foo", "bar"); + file_writer.finish().await.unwrap(); + } + + // Read a single column back at an explicit range/index set, returning its + // `Int32` values. Reading one column (or an equal-length group) at a time is + // how unequal-length files are consumed: a full scan across columns of + // differing lengths cannot form a single rectangular batch. + async fn read_int32_column( + reader: &FileReader, + schema: &LanceSchema, + version: ConcreteFileVersion, + name: &str, + params: lance_io::ReadBatchParams, + ) -> Vec> { + use futures::TryStreamExt; + use lance_encoding::decoder::FilterExpression; + + let projection = reader_projection_from_column_names(version, schema, &[name]).unwrap(); + let batches: Vec = reader + .read_stream_projected(params, 1024, 16, projection, FilterExpression::no_filter()) + .await + .unwrap() + .try_collect() + .await + .unwrap(); + batches + .iter() + .flat_map(|b| { + b.column(0) + .as_any() + .downcast_ref::() + .unwrap() + .iter() + .collect::>() + }) + .collect() + } + + /// A single file may hold columns of differing item counts, written by + /// advancing each column's row counter independently (no shared global + /// counter). + #[rstest] + #[tokio::test] + async fn test_write_columns_unequal_lengths( + #[values(ConcreteFileVersion::V2_0, ConcreteFileVersion::V2_1)] + version: ConcreteFileVersion, + ) { + use lance_io::ReadBatchParams; + + let arrow_schema = Arc::new(ArrowSchema::new(vec![ + ArrowField::new("a", DataType::Int32, true), + ArrowField::new("b", DataType::Int32, true), + ArrowField::new("c", DataType::Int32, true), + ])); + let lance_schema = LanceSchema::try_from(arrow_schema.as_ref()).unwrap(); + + let fs = FsFixture::default(); + let mut writer = create_writer( + fs.object_store.create(&fs.tmp_path).await.unwrap(), + lance_schema.clone(), + version, + FileWriterOptions::default(), + ) + .unwrap(); + + // Field "a" gets 5 values across two calls (appending), field "b" gets a + // single value, and field "c" is never written (a zero-length column). + let a1: ArrayRef = Arc::new(Int32Array::from(vec![1, 2, 3])); + let b: ArrayRef = Arc::new(Int32Array::from(vec![10])); + writer.write_column(0, a1).await.unwrap(); + writer.write_column(1, b).await.unwrap(); + let a2: ArrayRef = Arc::new(Int32Array::from(vec![4, 5])); + writer.write_column(0, a2).await.unwrap(); + // An empty array is a no-op whether or not the field already has rows: + // field "a" keeps its 5 rows, field "c" stays a zero-length column. + let empty: ArrayRef = Arc::new(Int32Array::from(Vec::::new())); + writer.write_column(0, empty.clone()).await.unwrap(); + writer.write_column(2, empty).await.unwrap(); + + let summary = writer.finish().await.unwrap(); + // The file's logical length is the longest column. + assert_eq!(summary.num_rows, 5); + + let file_scheduler = fs + .scheduler + .open_file(&fs.tmp_path, &CachedFileSize::unknown()) + .await + .unwrap(); + let reader = FileReader::try_open( + file_scheduler, + None, + Arc::::default(), + &LanceCache::no_cache(), + FileReaderOptions::default(), + ) + .await + .unwrap(); + + // Per-column row counts are recorded in / derivable from file metadata. + assert_eq!(reader.num_rows(), 5); + assert_eq!(reader.column_num_rows(0).unwrap(), 5); + assert_eq!(reader.column_num_rows(1).unwrap(), 1); + assert_eq!(reader.column_num_rows(2).unwrap(), 0); + assert!(reader.column_num_rows(3).is_err()); + + // Each column reads back independently at its own length. + assert_eq!( + read_int32_column( + &reader, + &lance_schema, + version, + "a", + ReadBatchParams::Range(0..5) + ) + .await, + vec![Some(1), Some(2), Some(3), Some(4), Some(5)], + ); + assert_eq!( + read_int32_column( + &reader, + &lance_schema, + version, + "b", + ReadBatchParams::Range(0..1) + ) + .await, + vec![Some(10)], + ); + + // Random access by position within the longer column returns the right + // value even though other columns are shorter. (The take path requires + // strictly increasing indices.) + assert_eq!( + read_int32_column( + &reader, + &lance_schema, + version, + "a", + ReadBatchParams::Indices(arrow_array::UInt32Array::from(vec![0, 2, 4])), + ) + .await, + vec![Some(1), Some(3), Some(5)], + ); + } + + /// Reading an unequal-length file: + /// - a projection whose columns are equal length full-scans normally; + /// - a full scan across columns of differing length is rejected up front, + /// before any batch is produced (even though a prefix would be rectangular); + /// - a bounded read is valid as long as every projected column covers it; + /// - a single-column `RangeFull` resolves to that column's own length, not + /// the file's (maximum) length. + #[rstest] + #[tokio::test] + async fn test_read_unequal_length_projection( + #[values(ConcreteFileVersion::V2_0, ConcreteFileVersion::V2_1)] + version: ConcreteFileVersion, + ) { + use futures::TryStreamExt; + use lance_encoding::decoder::FilterExpression; + use lance_io::ReadBatchParams; + + let arrow_schema = Arc::new(ArrowSchema::new(vec![ + ArrowField::new("a", DataType::Int32, true), + ArrowField::new("b", DataType::Int32, true), + ArrowField::new("c", DataType::Int32, true), + ])); + let lance_schema = LanceSchema::try_from(arrow_schema.as_ref()).unwrap(); + let fs = FsFixture::default(); + let mut writer = create_writer( + fs.object_store.create(&fs.tmp_path).await.unwrap(), + lance_schema.clone(), + version, + FileWriterOptions::default(), + ) + .unwrap(); + // "a" and "b" are equal length (5); "c" is shorter (1). + writer + .write_column(0, Arc::new(Int32Array::from(vec![1, 2, 3, 4, 5]))) + .await + .unwrap(); + writer + .write_column(1, Arc::new(Int32Array::from(vec![6, 7, 8, 9, 10]))) + .await + .unwrap(); + writer + .write_column(2, Arc::new(Int32Array::from(vec![100]))) + .await + .unwrap(); + writer.finish().await.unwrap(); + + let file_scheduler = fs + .scheduler + .open_file(&fs.tmp_path, &CachedFileSize::unknown()) + .await + .unwrap(); + let reader = FileReader::try_open( + file_scheduler, + None, + Arc::::default(), + &LanceCache::no_cache(), + FileReaderOptions::default(), + ) + .await + .unwrap(); + + let read = |names: &'static [&'static str], params: ReadBatchParams| { + let projection = + reader_projection_from_column_names(version, &lance_schema, names).unwrap(); + async { + match reader + .read_stream_projected( + params, + 1024, + 16, + projection, + FilterExpression::no_filter(), + ) + .await + { + Ok(stream) => stream.try_collect::>().await, + Err(e) => Err(e), + } + } + }; + let col_values = |batches: &[RecordBatch], idx: usize| -> Vec> { + batches + .iter() + .flat_map(|b| { + b.column(idx) + .as_any() + .downcast_ref::() + .unwrap() + .iter() + .collect::>() + }) + .collect() + }; + + // Equal-length projection [a, b] full-scans into rectangular batches. + let batches = read(&["a", "b"], ReadBatchParams::RangeFull).await.unwrap(); + assert_eq!( + col_values(&batches, 0), + vec![Some(1), Some(2), Some(3), Some(4), Some(5)] + ); + assert_eq!( + col_values(&batches, 1), + vec![Some(6), Some(7), Some(8), Some(9), Some(10)] + ); + + // A mismatched-length projection [a, c] (5 vs 1) is rejected before any + // batch is yielded, regardless of the read params — its columns cannot + // be combined into rectangular batches. The error names each column's + // length so the caller can see which column is the odd one out. + let err = read(&["a", "c"], ReadBatchParams::RangeFull) + .await + .unwrap_err() + .to_string(); + assert!( + err.contains("a=5") && err.contains("c=1"), + "error should name each column's length, got: {err}" + ); + assert!( + read(&["a", "c"], ReadBatchParams::Range(0..1)) + .await + .is_err(), + "even a common-prefix read of unequal-length columns must error" + ); + + // A single-column RangeFull resolves to that column's own length. + let batches = read(&["c"], ReadBatchParams::RangeFull).await.unwrap(); + assert_eq!(col_values(&batches, 0), vec![Some(100)]); + let batches = read(&["a"], ReadBatchParams::RangeFull).await.unwrap(); + assert_eq!( + col_values(&batches, 0), + vec![Some(1), Some(2), Some(3), Some(4), Some(5)] + ); + + // RangeFrom/RangeTo likewise resolve against the projected column's own + // length rather than the file's longest column. + let batches = read(&["a"], ReadBatchParams::RangeFrom(2..)).await.unwrap(); + assert_eq!(col_values(&batches, 0), vec![Some(3), Some(4), Some(5)]); + // RangeFrom on the short column "c" resolves to length 1, not 5. + let batches = read(&["c"], ReadBatchParams::RangeFrom(0..)).await.unwrap(); + assert_eq!(col_values(&batches, 0), vec![Some(100)]); + let batches = read(&["a"], ReadBatchParams::RangeTo(..3)).await.unwrap(); + assert_eq!(col_values(&batches, 0), vec![Some(1), Some(2), Some(3)]); + // A bound past the projected column's length errors. + assert!( + read(&["a"], ReadBatchParams::RangeTo(..6)).await.is_err(), + "RangeTo past the column length must error" + ); + assert!( + read(&["c"], ReadBatchParams::RangeFrom(2..)).await.is_err(), + "RangeFrom past the column length must error" + ); + } + + /// A struct and a list column each map to multiple physical columns, and a + /// list's item column is longer than its top-level row count. The + /// projection-length check must partition `column_indices` by top-level + /// field and use each field's root column, so an ordinary (rectangular) file + /// with nested columns still reads under the new validation path. + #[rstest] + #[tokio::test] + async fn test_read_nested_columns_under_validation( + #[values(ConcreteFileVersion::V2_0, ConcreteFileVersion::V2_1)] + version: ConcreteFileVersion, + ) { + use arrow_array::types::Int32Type; + use arrow_array::{ListArray, StructArray}; + use futures::TryStreamExt; + use lance_encoding::decoder::FilterExpression; + use lance_io::ReadBatchParams; + + let struct_type = DataType::Struct( + vec![ + ArrowField::new("x", DataType::Int32, true), + ArrowField::new("y", DataType::Int32, true), + ] + .into(), + ); + let list_type = DataType::List(Arc::new(ArrowField::new("item", DataType::Int32, true))); + let arrow_schema = Arc::new(ArrowSchema::new(vec![ + ArrowField::new("a", DataType::Int32, true), + ArrowField::new("s", struct_type, true), + ArrowField::new("lst", list_type, true), + ])); + let lance_schema = LanceSchema::try_from(arrow_schema.as_ref()).unwrap(); + + let a: ArrayRef = Arc::new(Int32Array::from(vec![1, 2, 3])); + let s: ArrayRef = Arc::new(StructArray::from(vec![ + ( + Arc::new(ArrowField::new("x", DataType::Int32, true)), + Arc::new(Int32Array::from(vec![10, 20, 30])) as ArrayRef, + ), + ( + Arc::new(ArrowField::new("y", DataType::Int32, true)), + Arc::new(Int32Array::from(vec![11, 21, 31])) as ArrayRef, + ), + ])); + // 3 lists, 6 items: the item column is longer than the top-level rows. + let lst: ArrayRef = Arc::new(ListArray::from_iter_primitive::(vec![ + Some(vec![Some(1), Some(2)]), + Some(vec![Some(3)]), + Some(vec![Some(4), Some(5), Some(6)]), + ])); + let batch = RecordBatch::try_new(arrow_schema.clone(), vec![a, s, lst]).unwrap(); + + let fs = FsFixture::default(); + let mut writer = create_writer( + fs.object_store.create(&fs.tmp_path).await.unwrap(), + lance_schema.clone(), + version, + FileWriterOptions::default(), + ) + .unwrap(); + writer.write_batch(&batch).await.unwrap(); + writer.finish().await.unwrap(); + + let file_scheduler = fs + .scheduler + .open_file(&fs.tmp_path, &CachedFileSize::unknown()) + .await + .unwrap(); + let reader = FileReader::try_open( + file_scheduler, + None, + Arc::::default(), + &LanceCache::no_cache(), + FileReaderOptions::default(), + ) + .await + .unwrap(); + + // If `validate_field_length` mispartitioned the physical columns, the + // length check would read the wrong root column (e.g. the list's item + // column, length 6) and spuriously reject this rectangular file. + for names in [&["a", "s", "lst"][..], &["a", "lst"][..], &["a", "s"][..]] { + let projection = + reader_projection_from_column_names(version, &lance_schema, names).unwrap(); + let batches: Vec = reader + .read_stream_projected( + ReadBatchParams::RangeFull, + 1024, + 16, + projection, + FilterExpression::no_filter(), + ) + .await + .unwrap() + .try_collect() + .await + .unwrap(); + let total_rows: usize = batches.iter().map(|b| b.num_rows()).sum(); + assert_eq!( + total_rows, 3, + "projection {names:?} should read 3 top-level rows" + ); + } + } + + /// `write_column` rejects invalid inputs at the API boundary with + /// descriptive errors: a writer without an explicit schema, an + /// out-of-bounds field index, and a null written into a non-nullable field. + #[tokio::test] + async fn test_write_column_validation_errors() { + // A lazy-schema writer cannot infer the schema from a single column. + let fs = FsFixture::default(); + let mut lazy_writer = versions::v2_1::create_lazy_writer( + fs.object_store.create(&fs.tmp_path).await.unwrap(), + FileWriterOptions::default(), + ); + let err = lazy_writer + .write_column(0, Arc::new(Int32Array::from(vec![1, 2, 3]))) + .await + .unwrap_err() + .to_string(); + assert!( + err.contains("explicit schema"), + "expected explicit-schema error, got: {err}" + ); + + let arrow_schema = Arc::new(ArrowSchema::new(vec![ + ArrowField::new("a", DataType::Int32, false), + ArrowField::new("b", DataType::Int32, true), + ])); + let lance_schema = LanceSchema::try_from(arrow_schema.as_ref()).unwrap(); + + // An out-of-bounds field index is rejected, naming the index and count. + let fs = FsFixture::default(); + let mut writer = create_writer( + fs.object_store.create(&fs.tmp_path).await.unwrap(), + lance_schema.clone(), + ConcreteFileVersion::V2_1, + FileWriterOptions::default(), + ) + .unwrap(); + let err = writer + .write_column(5, Arc::new(Int32Array::from(vec![1]))) + .await + .unwrap_err() + .to_string(); + assert!( + err.contains('5') && err.contains('2'), + "expected out-of-bounds error naming index 5 and 2 fields, got: {err}" + ); + + // A null in a non-nullable field ("a") is rejected. + let err = writer + .write_column(0, Arc::new(Int32Array::from(vec![Some(1), None, Some(3)]))) + .await + .unwrap_err() + .to_string(); + assert!( + err.contains("non-null"), + "expected nullability error, got: {err}" + ); + } + + /// The blocking read path applies the same projection-length validation as + /// the async path: a short single column resolves to its own length, and a + /// mismatched-length projection errors up front. + #[rstest] + #[tokio::test] + async fn test_blocking_read_unequal_length( + #[values(ConcreteFileVersion::V2_0, ConcreteFileVersion::V2_1)] + version: ConcreteFileVersion, + ) { + use lance_encoding::decoder::FilterExpression; + use lance_io::ReadBatchParams; + + let arrow_schema = Arc::new(ArrowSchema::new(vec![ + ArrowField::new("a", DataType::Int32, true), + ArrowField::new("c", DataType::Int32, true), + ])); + let lance_schema = LanceSchema::try_from(arrow_schema.as_ref()).unwrap(); + let fs = FsFixture::default(); + let mut writer = create_writer( + fs.object_store.create(&fs.tmp_path).await.unwrap(), + lance_schema.clone(), + version, + FileWriterOptions::default(), + ) + .unwrap(); + writer + .write_column(0, Arc::new(Int32Array::from(vec![1, 2, 3, 4, 5]))) + .await + .unwrap(); + writer + .write_column(1, Arc::new(Int32Array::from(vec![100]))) + .await + .unwrap(); + writer.finish().await.unwrap(); + + let file_scheduler = fs + .scheduler + .open_file(&fs.tmp_path, &CachedFileSize::unknown()) + .await + .unwrap(); + let reader = Arc::new( + FileReader::try_open( + file_scheduler, + None, + Arc::::default(), + &LanceCache::no_cache(), + FileReaderOptions::default(), + ) + .await + .unwrap(), + ); + + // Single short column: RangeFull resolves to its own length (1). + let proj_c = reader_projection_from_column_names(version, &lance_schema, &["c"]).unwrap(); + let reader_c = reader.clone(); + let batches = tokio::task::spawn_blocking(move || { + reader_c + .read_stream_projected_blocking( + ReadBatchParams::RangeFull, + 1024, + Some(proj_c), + FilterExpression::no_filter(), + ) + .unwrap() + .collect::, _>>() + .unwrap() + }) + .await + .unwrap(); + let total_rows: usize = batches.iter().map(|b| b.num_rows()).sum(); + assert_eq!(total_rows, 1); + + // A mismatched projection [a, c] errors on the blocking path too. + let proj_ac = + reader_projection_from_column_names(version, &lance_schema, &["a", "c"]).unwrap(); + let reader_ac = reader.clone(); + let is_err = tokio::task::spawn_blocking(move || { + reader_ac + .read_stream_projected_blocking( + ReadBatchParams::RangeFull, + 1024, + Some(proj_ac), + FilterExpression::no_filter(), + ) + .is_err() + }) + .await + .unwrap(); + assert!( + is_err, + "blocking full scan across unequal-length columns must error" + ); + } + + /// Files written the ordinary (rectangular) way keep equal column lengths, + /// so the unequal-length support is backwards compatible. + #[tokio::test] + async fn test_write_batch_keeps_equal_lengths() { + let arrow_schema = Arc::new(ArrowSchema::new(vec![ + ArrowField::new("a", DataType::Int32, true), + ArrowField::new("b", DataType::Int32, true), + ])); + let lance_schema = LanceSchema::try_from(arrow_schema.as_ref()).unwrap(); + + let fs = FsFixture::default(); + let mut writer = create_writer( + fs.object_store.create(&fs.tmp_path).await.unwrap(), + lance_schema, + ConcreteFileVersion::V2_1, + FileWriterOptions::default(), + ) + .unwrap(); + let batch = RecordBatch::try_new( + arrow_schema.clone(), + vec![ + Arc::new(Int32Array::from(vec![1, 2, 3])), + Arc::new(Int32Array::from(vec![4, 5, 6])), + ], + ) + .unwrap(); + writer.write_batch(&batch).await.unwrap(); + let summary = writer.finish().await.unwrap(); + assert_eq!(summary.num_rows, 3); + + let file_scheduler = fs + .scheduler + .open_file(&fs.tmp_path, &CachedFileSize::unknown()) + .await + .unwrap(); + let reader = FileReader::try_open( + file_scheduler, + None, + Arc::::default(), + &LanceCache::no_cache(), + FileReaderOptions::default(), + ) + .await + .unwrap(); + assert_eq!(reader.column_num_rows(0).unwrap(), 3); + assert_eq!(reader.column_num_rows(1).unwrap(), 3); + } + + #[tokio::test] + async fn test_max_page_bytes_enforced() { + let arrow_field = Field::new("data", DataType::UInt64, false); + let arrow_schema = Schema::new(vec![arrow_field]); + let lance_schema = LanceSchema::try_from(&arrow_schema).unwrap(); + + // 8MiB + let data: Vec = (0..1_000_000).collect(); + let array = UInt64Array::from(data); + let batch = + RecordBatch::try_new(arrow_schema.clone().into(), vec![Arc::new(array)]).unwrap(); + + let options = FileWriterOptions { + max_page_bytes: Some(1024 * 1024), // 1MB + ..Default::default() + }; + + let path = TempObjFile::default(); + let object_store = ObjectStore::local(); + let mut writer = create_writer( + object_store.create(&path).await.unwrap(), + lance_schema, + ConcreteFileVersion::V2_0, + options, + ) + .unwrap(); + + writer.write_batch(&batch).await.unwrap(); + writer.finish().await.unwrap(); + + let fs = FsFixture::default(); + let file_scheduler = fs + .scheduler + .open_file(&path, &CachedFileSize::unknown()) + .await + .unwrap(); + let file_reader = FileReader::try_open( + file_scheduler, + None, + Arc::::default(), + &LanceCache::no_cache(), + FileReaderOptions::default(), + ) + .await + .unwrap(); + + let column_meta = file_reader.metadata(); + + let mut total_page_num: u32 = 0; + for (col_idx, col_metadata) in column_meta.column_metadatas.iter().enumerate() { + assert!( + !col_metadata.pages.is_empty(), + "Column {} has no pages", + col_idx + ); + + for (page_idx, page) in col_metadata.pages.iter().enumerate() { + total_page_num += 1; + let total_size: u64 = page.buffer_sizes.iter().sum(); + assert!( + total_size <= 1024 * 1024, + "Column {} Page {} size {} exceeds 1MB limit", + col_idx, + page_idx, + total_size + ); + } + } + + assert_eq!(total_page_num, 8) + } + + #[tokio::test(flavor = "current_thread")] + async fn test_max_page_bytes_env_var() { + let arrow_field = Field::new("data", DataType::UInt64, false); + let arrow_schema = Schema::new(vec![arrow_field]); + let lance_schema = LanceSchema::try_from(&arrow_schema).unwrap(); + // 4MiB + let data: Vec = (0..500_000).collect(); + let array = UInt64Array::from(data); + let batch = + RecordBatch::try_new(arrow_schema.clone().into(), vec![Arc::new(array)]).unwrap(); + + // 2MiB + unsafe { + std::env::set_var(ENV_LANCE_FILE_WRITER_MAX_PAGE_BYTES, "2097152"); + } + + let options = FileWriterOptions { + max_page_bytes: None, // enforce env + ..Default::default() + }; + + let path = TempObjFile::default(); + let object_store = ObjectStore::local(); + let mut writer = create_writer( + object_store.create(&path).await.unwrap(), + lance_schema.clone(), + ConcreteFileVersion::V2_1, + options, + ) + .unwrap(); + + writer.write_batch(&batch).await.unwrap(); + writer.finish().await.unwrap(); + + let fs = FsFixture::default(); + let file_scheduler = fs + .scheduler + .open_file(&path, &CachedFileSize::unknown()) + .await + .unwrap(); + let file_reader = FileReader::try_open( + file_scheduler, + None, + Arc::::default(), + &LanceCache::no_cache(), + FileReaderOptions::default(), + ) + .await + .unwrap(); + + for col_metadata in file_reader.metadata().column_metadatas.iter() { + for page in col_metadata.pages.iter() { + let total_size: u64 = page.buffer_sizes.iter().sum(); + assert!( + total_size <= 2 * 1024 * 1024, + "Page size {} exceeds 2MB limit", + total_size + ); + } + } + + unsafe { + std::env::set_var(ENV_LANCE_FILE_WRITER_MAX_PAGE_BYTES, ""); + } + } + + #[tokio::test] + async fn test_compression_overrides_end_to_end() { + // Create test schema with different column types + let arrow_schema = Arc::new(ArrowSchema::new(vec![ + ArrowField::new("customer_id", DataType::Int32, false), + ArrowField::new("product_id", DataType::Int32, false), + ArrowField::new("quantity", DataType::Int32, false), + ArrowField::new("price", DataType::Float32, false), + ArrowField::new("description", DataType::Utf8, false), + ])); + + let lance_schema = LanceSchema::try_from(arrow_schema.as_ref()).unwrap(); + + // Create test data with patterns suitable for different compression + let mut customer_ids = Int32Builder::new(); + let mut product_ids = Int32Builder::new(); + let mut quantities = Int32Builder::new(); + let mut prices = Float32Builder::new(); + let mut descriptions = Vec::new(); + + // Generate data with specific patterns: + // - customer_id: highly repetitive (good for RLE) + // - product_id: moderately repetitive (good for RLE) + // - quantity: random values (not good for RLE) + // - price: some repetition + // - description: long strings (good for Zstd) + for i in 0..10000 { + // Customer ID repeats every 100 rows (100 unique customers) + // This creates runs of 100 identical values + customer_ids.append_value(i / 100); + + // Product ID has only 5 unique values with long runs + product_ids.append_value(i / 2000); + + // Quantity is mostly 1 with occasional other values + quantities.append_value(if i % 10 == 0 { 5 } else { 1 }); + + // Price has only 3 unique values + prices.append_value(match i % 3 { + 0 => 9.99, + 1 => 19.99, + _ => 29.99, + }); + + // Descriptions are repetitive but we'll keep them simple + descriptions.push(format!("Product {}", i / 2000)); + } + + let batch = RecordBatch::try_new( + arrow_schema.clone(), + vec![ + Arc::new(customer_ids.finish()), + Arc::new(product_ids.finish()), + Arc::new(quantities.finish()), + Arc::new(prices.finish()), + Arc::new(StringArray::from(descriptions)), + ], + ) + .unwrap(); + + // Configure compression parameters + let mut params = CompressionParams::new(); + + // RLE for ID columns (ends with _id) + params.columns.insert( + "*_id".to_string(), + CompressionFieldParams { + rle_threshold: Some(0.5), // Lower threshold to trigger RLE more easily + compression: None, // Will use default compression if any + compression_level: None, + bss: Some(lance_encoding::compression_config::BssMode::Off), // Explicitly disable BSS to ensure RLE is used + minichunk_size: None, + }, + ); + + // For now, we'll skip Zstd compression since it's not imported + // In a real implementation, you could add other compression types here + + // Configure file writer options + let options = FileWriterOptions { + max_page_bytes: Some(64 * 1024), // 64KB pages + ..Default::default() + }; + + // Write the file + let path = TempObjFile::default(); + let object_store = ObjectStore::local(); + + let mut writer = create_v2_1_writer_with_compression( + object_store.create(&path).await.unwrap(), + lance_schema.clone(), + options, + params, + ) + .unwrap(); + + writer.write_batch(&batch).await.unwrap(); + writer.add_schema_metadata("compression_test", "configured_compression"); + writer.finish().await.unwrap(); + + // Now write the same data without compression overrides for comparison + let path_no_compression = TempObjFile::default(); + let default_options = FileWriterOptions { + max_page_bytes: Some(64 * 1024), + ..Default::default() + }; + + let mut writer_no_compression = create_writer( + object_store.create(&path_no_compression).await.unwrap(), + lance_schema.clone(), + ConcreteFileVersion::V2_1, + default_options, + ) + .unwrap(); + + writer_no_compression.write_batch(&batch).await.unwrap(); + writer_no_compression.finish().await.unwrap(); + + // Note: With our current data patterns and RLE compression, the compressed file + // might actually be slightly larger due to compression metadata overhead. + // This is expected and the test is mainly to verify the system works end-to-end. + + // Read back the compressed file and verify data integrity + let fs = FsFixture::default(); + let file_scheduler = fs + .scheduler + .open_file(&path, &CachedFileSize::unknown()) + .await + .unwrap(); + + let file_reader = FileReader::try_open( + file_scheduler, + None, + Arc::::default(), + &LanceCache::no_cache(), + FileReaderOptions::default(), + ) + .await + .unwrap(); + + // Verify metadata + let metadata = file_reader.metadata(); + assert_eq!(metadata.version(), ConcreteFileVersion::V2_1); + + let schema = file_reader.schema(); + assert_eq!( + schema.metadata.get("compression_test"), + Some(&"configured_compression".to_string()) + ); + + // Verify the actual encodings used + let column_metadatas = &metadata.column_metadatas; + + // Check customer_id column (index 0) - should use RLE due to our configuration + assert!(!column_metadatas[0].pages.is_empty()); + let customer_id_encoding = describe_encoding(&column_metadatas[0].pages[0]); + assert!( + customer_id_encoding.contains("RLE") || customer_id_encoding.contains("Rle"), + "customer_id column should use RLE encoding due to '*_id' pattern match, but got: {}", + customer_id_encoding + ); + + // Check product_id column (index 1) - should use RLE due to our configuration + assert!(!column_metadatas[1].pages.is_empty()); + let product_id_encoding = describe_encoding(&column_metadatas[1].pages[0]); + assert!( + product_id_encoding.contains("RLE") || product_id_encoding.contains("Rle"), + "product_id column should use RLE encoding due to '*_id' pattern match, but got: {}", + product_id_encoding + ); + } + + #[tokio::test] + async fn test_field_metadata_compression() { + // Test that field metadata compression settings are respected + let mut metadata = HashMap::new(); + metadata.insert( + lance_encoding::constants::COMPRESSION_META_KEY.to_string(), + "zstd".to_string(), + ); + metadata.insert( + lance_encoding::constants::COMPRESSION_LEVEL_META_KEY.to_string(), + "6".to_string(), + ); + + let arrow_schema = Arc::new(ArrowSchema::new(vec![ + ArrowField::new("id", DataType::Int32, false), + ArrowField::new("text", DataType::Utf8, false).with_metadata(metadata.clone()), + ArrowField::new("data", DataType::Int32, false).with_metadata(HashMap::from([( + lance_encoding::constants::COMPRESSION_META_KEY.to_string(), + "none".to_string(), + )])), + ])); + + let lance_schema = LanceSchema::try_from(arrow_schema.as_ref()).unwrap(); + + // Create test data + let id_array = Int32Array::from_iter_values(0..1000); + let text_array = StringArray::from_iter_values( + (0..1000).map(|i| format!("test string {} repeated text", i)), + ); + let data_array = Int32Array::from_iter_values((0..1000).map(|i| i * 2)); + + let batch = RecordBatch::try_new( + arrow_schema.clone(), + vec![ + Arc::new(id_array), + Arc::new(text_array), + Arc::new(data_array), + ], + ) + .unwrap(); + + let path = TempObjFile::default(); + let object_store = ObjectStore::local(); + + // Create encoding strategy that will read from field metadata + let params = CompressionParams::new(); + let options = FileWriterOptions::default(); + let mut writer = create_v2_1_writer_with_compression( + object_store.create(&path).await.unwrap(), + lance_schema.clone(), + options, + params, + ) + .unwrap(); + + writer.write_batch(&batch).await.unwrap(); + writer.finish().await.unwrap(); + + // Read back metadata + let fs = FsFixture::default(); + let file_scheduler = fs + .scheduler + .open_file(&path, &CachedFileSize::unknown()) + .await + .unwrap(); + let file_reader = FileReader::try_open( + file_scheduler, + None, + Arc::::default(), + &LanceCache::no_cache(), + FileReaderOptions::default(), + ) + .await + .unwrap(); + + let column_metadatas = &file_reader.metadata().column_metadatas; + + // The text column (index 1) should use zstd compression based on metadata + let text_encoding = describe_encoding(&column_metadatas[1].pages[0]); + // For string columns, we expect Binary encoding with zstd compression + assert!( + text_encoding.contains("Zstd"), + "text column should use zstd compression from field metadata, but got: {}", + text_encoding + ); + + // The data column (index 2) should use no compression based on metadata + let data_encoding = describe_encoding(&column_metadatas[2].pages[0]); + // For Int32 columns with "none" compression, we expect Flat encoding without compression + assert!( + data_encoding.contains("Flat") && data_encoding.contains("compression: None"), + "data column should use no compression from field metadata, but got: {}", + data_encoding + ); + } + + #[tokio::test] + async fn test_field_metadata_rle_threshold() { + // Test that RLE threshold from field metadata is respected + let mut metadata = HashMap::new(); + metadata.insert( + lance_encoding::constants::RLE_THRESHOLD_META_KEY.to_string(), + "0.9".to_string(), + ); + // Also set compression to ensure RLE is used + metadata.insert( + lance_encoding::constants::COMPRESSION_META_KEY.to_string(), + "lz4".to_string(), + ); + // Explicitly disable BSS to ensure RLE is tested + metadata.insert( + lance_encoding::constants::BSS_META_KEY.to_string(), + "off".to_string(), + ); + + let arrow_schema = Arc::new(ArrowSchema::new(vec![ + ArrowField::new("status", DataType::Int32, false).with_metadata(metadata), + ])); + + let lance_schema = LanceSchema::try_from(arrow_schema.as_ref()).unwrap(); + + // Create data with very high repetition (3 runs for 10000 values = 0.0003 ratio) + let status_array = Int32Array::from_iter_values( + std::iter::repeat_n(200, 8000) + .chain(std::iter::repeat_n(404, 1500)) + .chain(std::iter::repeat_n(500, 500)), + ); + + let batch = + RecordBatch::try_new(arrow_schema.clone(), vec![Arc::new(status_array)]).unwrap(); + + let path = TempObjFile::default(); + let object_store = ObjectStore::local(); + + // Create encoding strategy that will read from field metadata + let params = CompressionParams::new(); + let options = FileWriterOptions::default(); + let mut writer = create_v2_1_writer_with_compression( + object_store.create(&path).await.unwrap(), + lance_schema.clone(), + options, + params, + ) + .unwrap(); + + writer.write_batch(&batch).await.unwrap(); + writer.finish().await.unwrap(); + + // Read back and check encoding + let fs = FsFixture::default(); + let file_scheduler = fs + .scheduler + .open_file(&path, &CachedFileSize::unknown()) + .await + .unwrap(); + let file_reader = FileReader::try_open( + file_scheduler, + None, + Arc::::default(), + &LanceCache::no_cache(), + FileReaderOptions::default(), + ) + .await + .unwrap(); + + let column_metadatas = &file_reader.metadata().column_metadatas; + let status_encoding = describe_encoding(&column_metadatas[0].pages[0]); + assert!( + status_encoding.contains("RLE") || status_encoding.contains("Rle"), + "status column should use RLE encoding due to metadata threshold, but got: {}", + status_encoding + ); + } + + #[tokio::test] + async fn test_large_page_split_on_read() { + use arrow_array::Array; + use futures::TryStreamExt; + use lance_encoding::decoder::FilterExpression; + use lance_io::ReadBatchParams; + + // Test that large pages written with relaxed limits can be split during read + + let arrow_field = ArrowField::new("data", DataType::Binary, false); + let arrow_schema = ArrowSchema::new(vec![arrow_field]); + let lance_schema = LanceSchema::try_from(&arrow_schema).unwrap(); + + // Create a large binary value (40MB) to trigger large page creation + let large_value = vec![42u8; 40 * 1024 * 1024]; + let array = arrow_array::BinaryArray::from(vec![ + Some(large_value.as_slice()), + Some(b"small value"), + ]); + let batch = RecordBatch::try_new(Arc::new(arrow_schema), vec![Arc::new(array)]).unwrap(); + + // Write with relaxed page size limit (128MB) + let options = FileWriterOptions { + max_page_bytes: Some(128 * 1024 * 1024), + ..Default::default() + }; + + let fs = FsFixture::default(); + let path = fs.tmp_path; + + let mut writer = create_writer( + fs.object_store.create(&path).await.unwrap(), + lance_schema.clone(), + ConcreteFileVersion::V2_1, + options, + ) + .unwrap(); + + writer.write_batch(&batch).await.unwrap(); + let write_summary = writer.finish().await.unwrap(); + assert_eq!(write_summary.num_rows, 2); + assert_eq!( + write_summary.size_bytes, + fs.object_store.size(&path).await.unwrap() + ); + + // Read back with split configuration + let file_scheduler = fs + .scheduler + .open_file(&path, &CachedFileSize::unknown()) + .await + .unwrap(); + + // Configure reader to split pages larger than 10MB into chunks + let reader_options = FileReaderOptions { + read_chunk_size: 10 * 1024 * 1024, // 10MB chunks + ..Default::default() + }; + + let file_reader = FileReader::try_open( + file_scheduler, + None, + Arc::::default(), + &LanceCache::no_cache(), + reader_options, + ) + .await + .unwrap(); + + // Read the data back + let stream = file_reader + .read_stream( + ReadBatchParams::RangeFull, + 1024, + 10, // batch_readahead + FilterExpression::no_filter(), + ) + .await + .unwrap(); + + let batches: Vec = stream.try_collect().await.unwrap(); + assert_eq!(batches.len(), 1); + + // Verify the data is correctly read despite splitting + let read_array = batches[0].column(0); + let read_binary = read_array + .as_any() + .downcast_ref::() + .unwrap(); + + assert_eq!(read_binary.len(), 2); + assert_eq!(read_binary.value(0).len(), 40 * 1024 * 1024); + assert_eq!(read_binary.value(1), b"small value"); + + // Verify first value matches what we wrote + assert!(read_binary.value(0).iter().all(|&b| b == 42u8)); + } + + fn spill_config() -> (TempObjFile, Arc) { + let spill_path = TempObjFile::default(); + (spill_path, Arc::new(ObjectStore::local())) + } + + fn make_batches(num_batches: i32, num_cols: usize, rows_per_batch: i32) -> Vec { + let fields: Vec<_> = (0..num_cols) + .map(|c| ArrowField::new(format!("c{c}"), DataType::Int32, false)) + .collect(); + let schema = Arc::new(ArrowSchema::new(fields)); + (0..num_batches) + .map(|i| { + let cols: Vec> = (0..num_cols) + .map(|c| { + let start = (i * rows_per_batch + c as i32) * 100; + Arc::new(Int32Array::from_iter_values(start..start + rows_per_batch)) + as Arc + }) + .collect(); + RecordBatch::try_new(schema.clone(), cols).unwrap() + }) + .collect() + } + + async fn write_and_read_batches( + batches: &[RecordBatch], + spill: Option<(Arc, object_store::path::Path)>, + ) -> Vec { + let fs = FsFixture::default(); + let lance_schema = LanceSchema::try_from(batches[0].schema().as_ref()).unwrap(); + let writer = fs.object_store.create(&fs.tmp_path).await.unwrap(); + let mut file_writer = create_writer( + writer, + lance_schema, + ConcreteFileVersion::V2_1, + FileWriterOptions::default(), + ) + .unwrap(); + if let Some((store, path)) = spill { + file_writer = file_writer.with_page_metadata_spill(store, path); + } + for batch in batches { + file_writer.write_batch(batch).await.unwrap(); + } + file_writer.add_schema_metadata("foo", "bar"); + file_writer.finish().await.unwrap(); + + crate::testing::read_lance_file( + &fs, + Arc::::default(), + lance_encoding::decoder::FilterExpression::no_filter(), + ) + .await + } + + #[rstest::rstest] + #[case::multi_col(20, 2, 100)] + #[case::many_batches(50, 2, 100)] + #[tokio::test] + async fn test_page_metadata_spill_roundtrip( + #[case] num_batches: i32, + #[case] num_cols: usize, + #[case] rows_per_batch: i32, + ) { + let batches = make_batches(num_batches, num_cols, rows_per_batch); + let baseline = write_and_read_batches(&batches, None).await; + let (spill_path, spill_store) = spill_config(); + let spilled = + write_and_read_batches(&batches, Some((spill_store, spill_path.as_ref().clone()))) + .await; + assert_eq!(baseline, spilled); + } + + #[tokio::test] + async fn test_page_metadata_spill_many_columns() { + // Many columns forces small per-column buffer limits, exercising mid-write flushing. + let batches = make_batches(10, 500, 100); + let baseline = write_and_read_batches(&batches, None).await; + let (spill_path, spill_store) = spill_config(); + let spilled = + write_and_read_batches(&batches, Some((spill_store, spill_path.as_ref().clone()))) + .await; + assert_eq!(baseline, spilled); + } +} diff --git a/rust/lance-file/test_data/exact_versions/README.md b/rust/lance-file/test_data/exact_versions/README.md new file mode 100644 index 00000000000..72b8edcd86d --- /dev/null +++ b/rust/lance-file/test_data/exact_versions/README.md @@ -0,0 +1,41 @@ +# Exact File-Version Compatibility Fixtures + +These files were generated with the writers at baseline commit +`3a72f8a61e14613f517dded6816d4bfc77817c93`. The deterministic input batch is +defined by `compatibility_fixture_batch` in `src/compatibility_tests.rs` and +covers primitive, nullable UTF-8, nullable list, nullable dictionary, blob, +multiple input batches, and multiple pages. The V2.0 embedded fixtures use the +primitive and nullable UTF-8 columns from the first 257 rows of the same batch. + +`datagen.py` copies the baseline-compatible `datagen.rs` into a clean checkout +of that commit, runs it twice in separate processes with an isolated Cargo +target directory, and verifies that both runs produce identical bytes: + +```shell +git worktree add --detach /tmp/lance-exact-version-baseline \ + 3a72f8a61e14613f517dded6816d4bfc77817c93 +python3 rust/lance-file/test_data/exact_versions/datagen.py \ + --source /tmp/lance-exact-version-baseline +git worktree remove /tmp/lance-exact-version-baseline +``` + +Pass `--write` only when intentionally restoring these files from the locked +baseline. + +| File | SHA-256 | +| --- | --- | +| `v1.lance` | `fa8b3d81b9d4fd4ade5a7c3d077ebf2155664e12b9335e26fac1c0d0774e916c` | +| `v2_0.lance` | `073c8c24eb4433b83d0dda95bf7a731a9f5d8f32d78440f2f391474e99b9c49a` | +| `v2_1.lance` | `3af97ba176b72c7e00a248b4a270a53402a72e594631950f76eb3daab45c50ce` | +| `v2_2.lance` | `8298cd9301e657417b0725461345c27cf46515529d2a8b35824be139e3466a14` | +| `v2_0_self_described.lance` | `6a3a9ce8ef56f058d1d105e7f4494ce35a9026479e2f76fc6f26c04b3201a406` | +| `v2_0_mini.lance` | `5e3fc99b01a4d2f5d16a2fb051dacb49b4a736428b1494715cd83633ad142a63` | + +The compatibility tests require each stable writer to reproduce its fixture +byte-for-byte and each reader to open and read the baseline file. The V2.0 +standard fixture preserves footer `(0, 3)` while its self-described and +mini-Lance fixtures preserve `(2, 0)`. V2.3 is unstable, so it has deterministic +current-revision tests instead of a checked-in compatibility fixture. + +Regenerate these fixtures only from the baseline writer APIs. Files generated +with the implementation under test are not independent compatibility evidence. diff --git a/rust/lance-file/test_data/exact_versions/datagen.py b/rust/lance-file/test_data/exact_versions/datagen.py new file mode 100644 index 00000000000..5543d9fd190 --- /dev/null +++ b/rust/lance-file/test_data/exact_versions/datagen.py @@ -0,0 +1,129 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright The Lance Authors + +import argparse +import hashlib +import os +import shutil +import subprocess +import tempfile +from pathlib import Path + +BASELINE_COMMIT = "3a72f8a61e14613f517dded6816d4bfc77817c93" +FIXTURE_NAMES = ( + "v1.lance", + "v2_0.lance", + "v2_1.lance", + "v2_2.lance", + "v2_0_self_described.lance", + "v2_0_mini.lance", +) + + +def git_output(source: Path, *args: str) -> str: + return subprocess.run( + ["git", "-C", source, *args], + check=True, + capture_output=True, + text=True, + ).stdout.strip() + + +def generate(source: Path, generator: Path, output: Path, target: Path) -> None: + examples = source / "rust/lance-file/examples" + target_generator = examples / "exact_version_fixture_generator.rs" + created_examples = not examples.exists() + examples.mkdir(parents=True, exist_ok=True) + if target_generator.exists(): + raise RuntimeError(f"refusing to replace existing {target_generator}") + + shutil.copyfile(generator, target_generator) + try: + env = os.environ.copy() + env["CARGO_TARGET_DIR"] = str(target) + subprocess.run( + [ + "cargo", + "run", + "-p", + "lance-file", + "--example", + "exact_version_fixture_generator", + "--", + str(output), + ], + cwd=source, + env=env, + check=True, + ) + finally: + target_generator.unlink(missing_ok=True) + if created_examples: + examples.rmdir() + + +def digest(path: Path) -> str: + return hashlib.sha256(path.read_bytes()).hexdigest() + + +def main() -> None: + parser = argparse.ArgumentParser( + description="Reproduce exact-version fixtures with the locked baseline writers." + ) + parser.add_argument( + "--source", + type=Path, + required=True, + help=f"clean Lance checkout at {BASELINE_COMMIT}", + ) + parser.add_argument( + "--write", + action="store_true", + help="replace the checked-in fixtures after both baseline runs agree", + ) + args = parser.parse_args() + + source = args.source.resolve() + if git_output(source, "rev-parse", "HEAD") != BASELINE_COMMIT: + raise RuntimeError( + f"{source} must be checked out at baseline commit {BASELINE_COMMIT}" + ) + if git_output(source, "status", "--porcelain"): + raise RuntimeError(f"{source} must be clean before fixture generation") + + fixture_dir = Path(__file__).resolve().parent + generator = fixture_dir / "datagen.rs" + with ( + tempfile.TemporaryDirectory(prefix="lance-exact-fixtures-a-") as first_dir, + tempfile.TemporaryDirectory(prefix="lance-exact-fixtures-b-") as second_dir, + tempfile.TemporaryDirectory( + prefix="lance-exact-fixtures-target-" + ) as target_dir, + ): + first = Path(first_dir) + second = Path(second_dir) + target = Path(target_dir) + generate(source, generator, first, target) + generate(source, generator, second, target) + + for name in FIXTURE_NAMES: + first_path = first / name + second_path = second / name + if first_path.read_bytes() != second_path.read_bytes(): + raise RuntimeError(f"separate baseline runs disagree for {name}") + + checked_path = fixture_dir / name + if args.write: + shutil.copyfile(first_path, checked_path) + elif ( + not checked_path.exists() + or checked_path.read_bytes() != first_path.read_bytes() + ): + raise RuntimeError( + f"{name} differs from the reproducible baseline; rerun with --write" + ) + print(f"{name}: {digest(first_path)}") + + +if __name__ == "__main__": + main() diff --git a/rust/lance-file/test_data/exact_versions/datagen.rs b/rust/lance-file/test_data/exact_versions/datagen.rs new file mode 100644 index 00000000000..ebbefdc417b --- /dev/null +++ b/rust/lance-file/test_data/exact_versions/datagen.rs @@ -0,0 +1,228 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright The Lance Authors + +use std::collections::HashMap; +use std::path::Path; +use std::sync::Arc; + +use arrow_array::builder::StringDictionaryBuilder; +use arrow_array::types::{Int8Type, Int32Type}; +use arrow_array::{ArrayRef, Int32Array, LargeBinaryArray, ListArray, RecordBatch, StringArray}; +use arrow_schema::{DataType, Field, Schema as ArrowSchema}; +use async_trait::async_trait; +use lance_core::datatypes::Schema as LanceSchema; +use lance_encoding::encoder::{EncodingOptions, default_encoding_strategy, encode_batch}; +use lance_file::previous::writer::{ + FileWriter as V1Writer, FileWriterOptions as V1WriterOptions, ManifestProvider, +}; +use lance_file::testing::FsFixture; +use lance_file::version::LanceFileVersion; +use lance_file::writer::{EncodedBatchWriteExt, FileWriter, FileWriterOptions}; +use lance_io::traits::Writer; + +type DynError = Box; + +struct NoManifest; + +#[async_trait] +impl ManifestProvider for NoManifest { + async fn store_schema( + _: &mut dyn Writer, + _: &LanceSchema, + ) -> lance_core::Result> { + Ok(None) + } +} + +fn compatibility_fixture_batch() -> RecordBatch { + let row_count = 4097; + let ids = Arc::new(Int32Array::from_iter_values(0..row_count)) as ArrayRef; + let names = Arc::new(StringArray::from_iter((0..row_count).map(|index| { + (index % 7 != 0).then(|| format!("value-{index:04}-deterministic-fixture")) + }))) as ArrayRef; + let items = Arc::new(ListArray::from_iter_primitive::( + (0..row_count).map(|index| { + (index % 11 != 0).then(|| { + vec![ + Some(index), + (index % 5 != 0).then_some(index * 2), + Some(index * 3), + ] + }) + }), + )) as ArrayRef; + let mut categories = StringDictionaryBuilder::::new(); + for index in 0..row_count { + if index % 13 == 0 { + categories.append_null(); + } else { + categories + .append(match index % 3 { + 0 => "red", + 1 => "green", + _ => "blue", + }) + .expect("fixture dictionary values must fit"); + } + } + let categories = Arc::new(categories.finish()) as ArrayRef; + let blobs = Arc::new(LargeBinaryArray::from_iter_values( + (0..row_count).map(|index| format!("blob-{index:04}-deterministic-payload").into_bytes()), + )) as ArrayRef; + + let schema = Arc::new(ArrowSchema::new(vec![ + Field::new("id", DataType::Int32, false), + Field::new("name", DataType::Utf8, true).with_metadata(HashMap::from([( + "lance-encoding:compression".to_string(), + "none".to_string(), + )])), + Field::new( + "items", + DataType::List(Arc::new(Field::new("item", DataType::Int32, true))), + true, + ), + Field::new( + "category", + DataType::Dictionary(Box::new(DataType::Int8), Box::new(DataType::Utf8)), + true, + ) + .with_metadata(HashMap::from([( + "lance-encoding:dict-values-compression".to_string(), + "none".to_string(), + )])), + Field::new("blob", DataType::LargeBinary, true).with_metadata(HashMap::from([( + "lance-encoding:blob".to_string(), + "true".to_string(), + )])), + ])); + RecordBatch::try_new(schema, vec![ids, names, items, categories, blobs]) + .expect("fixture schema and arrays must agree") +} + +async fn write_current_fixture( + version: LanceFileVersion, + batch: &RecordBatch, + schema: &LanceSchema, +) -> Result, DynError> { + let fs = FsFixture::default(); + let object_writer = fs.object_store.create(&fs.tmp_path).await?; + let mut writer = FileWriter::try_new( + object_writer, + schema.clone(), + FileWriterOptions { + data_cache_bytes: Some(1), + max_page_bytes: Some(1024), + format_version: Some(version), + ..Default::default() + }, + )?; + for offset in (0..batch.num_rows()).step_by(1024) { + let slice = batch.slice(offset, (batch.num_rows() - offset).min(1024)); + writer.write_batch(&slice).await?; + } + let summary = writer.finish().await?; + Ok(fs + .object_store + .open(&fs.tmp_path) + .await? + .get_range(0..summary.size_bytes as usize) + .await? + .to_vec()) +} + +async fn write_v1_fixture(batch: &RecordBatch, schema: &LanceSchema) -> Result, DynError> { + let fs = FsFixture::default(); + let mut writer = V1Writer::::try_new( + fs.object_store.as_ref(), + &fs.tmp_path, + schema.clone(), + &V1WriterOptions { + collect_stats_for_fields: Some(Vec::new()), + }, + ) + .await?; + for offset in (0..batch.num_rows()).step_by(1024) { + let slice = batch.slice(offset, (batch.num_rows() - offset).min(1024)); + writer.write(std::slice::from_ref(&slice)).await?; + } + let summary = writer.finish().await?; + Ok(fs + .object_store + .open(&fs.tmp_path) + .await? + .get_range(0..summary.size_bytes as usize) + .await? + .to_vec()) +} + +async fn write_v2_0_embedded_fixtures( + batch: &RecordBatch, + schema: &LanceSchema, +) -> Result<(Vec, Vec), DynError> { + let version = LanceFileVersion::V2_0; + let options = EncodingOptions { + cache_bytes_per_column: 1, + max_page_bytes: 1024, + keep_original_array: true, + buffer_alignment: 64, + version, + }; + let encoding_strategy = default_encoding_strategy(version); + let encoded_batch = encode_batch( + batch, + Arc::new(schema.clone()), + encoding_strategy.as_ref(), + &options, + ) + .await?; + + Ok(( + encoded_batch.try_to_self_described_lance(version)?.to_vec(), + encoded_batch.try_to_mini_lance(version)?.to_vec(), + )) +} + +fn write_fixture(output_dir: &Path, name: &str, bytes: &[u8]) -> Result<(), DynError> { + std::fs::write(output_dir.join(name), bytes)?; + Ok(()) +} + +#[tokio::main] +async fn main() -> Result<(), DynError> { + let output_dir = std::env::args_os() + .nth(1) + .map(std::path::PathBuf::from) + .ok_or("usage: exact_version_fixture_generator ")?; + std::fs::create_dir_all(&output_dir)?; + + let batch = compatibility_fixture_batch(); + let mut schema = LanceSchema::try_from(batch.schema().as_ref())?; + schema.set_dictionary(&batch)?; + + write_fixture( + &output_dir, + "v1.lance", + &write_v1_fixture(&batch, &schema).await?, + )?; + for (version, name) in [ + (LanceFileVersion::V2_0, "v2_0.lance"), + (LanceFileVersion::V2_1, "v2_1.lance"), + (LanceFileVersion::V2_2, "v2_2.lance"), + ] { + write_fixture( + &output_dir, + name, + &write_current_fixture(version, &batch, &schema).await?, + )?; + } + + let embedded_batch = batch.project(&[0, 1])?.slice(0, 257); + let mut embedded_schema = LanceSchema::try_from(embedded_batch.schema().as_ref())?; + embedded_schema.set_dictionary(&embedded_batch)?; + let (self_described, mini) = + write_v2_0_embedded_fixtures(&embedded_batch, &embedded_schema).await?; + write_fixture(&output_dir, "v2_0_self_described.lance", &self_described)?; + write_fixture(&output_dir, "v2_0_mini.lance", &mini)?; + + Ok(()) +} diff --git a/rust/lance-file/test_data/exact_versions/v1.lance b/rust/lance-file/test_data/exact_versions/v1.lance new file mode 100644 index 00000000000..267835ffb2f Binary files /dev/null and b/rust/lance-file/test_data/exact_versions/v1.lance differ diff --git a/rust/lance-file/test_data/exact_versions/v2_0.lance b/rust/lance-file/test_data/exact_versions/v2_0.lance new file mode 100644 index 00000000000..5c713fc36b3 --- /dev/null +++ b/rust/lance-file/test_data/exact_versions/v2_0.lance @@ -0,0 +1,10040 @@ +blob-0000-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0001-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0002-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0003-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0004-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0005-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0006-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0007-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0008-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0009-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0010-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0011-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0012-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0013-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0014-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0015-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0016-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0017-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0018-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0019-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0020-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0021-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0022-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0023-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0024-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0025-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0026-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0027-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0028-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0029-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0030-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0031-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0032-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0033-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0034-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0035-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0036-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0037-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0038-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0039-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0040-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0041-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0042-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0043-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0044-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0045-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0046-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0047-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0048-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0049-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0050-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0051-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0052-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0053-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0054-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0055-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0056-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0057-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0058-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0059-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0060-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0061-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0062-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0063-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0064-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0065-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0066-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0067-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0068-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0069-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0070-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0071-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0072-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0073-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0074-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0075-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0076-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0077-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0078-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0079-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0080-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0081-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0082-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0083-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0084-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0085-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0086-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0087-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0088-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0089-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0090-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0091-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0092-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0093-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0094-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0095-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0096-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0097-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0098-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0099-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0100-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0101-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0102-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0103-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0104-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0105-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0106-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0107-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0108-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0109-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0110-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0111-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0112-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0113-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0114-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0115-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0116-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0117-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0118-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0119-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0120-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0121-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0122-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0123-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0124-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0125-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0126-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0127-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0128-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0129-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0130-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0131-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0132-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0133-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0134-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0135-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0136-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0137-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0138-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0139-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0140-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0141-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0142-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0143-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0144-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0145-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0146-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0147-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0148-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0149-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0150-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0151-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0152-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0153-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0154-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0155-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0156-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0157-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0158-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0159-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0160-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0161-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0162-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0163-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0164-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0165-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0166-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0167-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0168-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0169-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0170-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0171-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0172-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0173-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0174-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0175-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0176-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0177-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0178-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0179-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0180-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0181-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0182-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0183-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0184-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0185-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0186-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0187-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0188-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0189-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0190-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0191-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0192-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0193-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0194-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0195-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0196-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0197-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0198-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0199-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0200-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0201-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0202-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0203-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0204-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0205-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0206-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0207-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0208-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0209-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0210-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0211-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0212-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0213-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0214-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0215-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0216-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0217-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0218-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0219-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0220-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0221-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0222-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0223-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0224-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0225-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0226-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0227-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0228-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0229-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0230-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0231-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0232-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0233-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0234-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0235-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0236-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0237-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0238-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0239-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0240-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0241-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0242-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0243-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0244-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0245-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0246-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0247-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0248-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0249-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0250-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0251-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0252-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0253-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0254-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0255-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0256-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0257-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0258-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0259-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0260-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0261-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0262-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0263-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0264-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0265-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0266-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0267-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0268-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0269-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0270-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0271-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0272-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0273-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0274-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0275-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0276-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0277-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0278-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0279-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0280-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0281-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0282-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0283-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0284-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0285-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0286-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0287-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0288-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0289-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0290-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0291-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0292-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0293-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0294-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0295-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0296-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0297-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0298-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0299-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0300-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0301-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0302-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0303-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0304-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0305-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0306-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0307-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0308-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0309-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0310-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0311-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0312-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0313-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0314-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0315-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0316-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0317-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0318-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0319-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0320-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0321-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0322-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0323-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0324-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0325-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0326-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0327-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0328-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0329-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0330-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0331-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0332-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0333-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0334-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0335-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0336-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0337-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0338-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0339-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0340-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0341-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0342-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0343-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0344-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0345-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0346-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0347-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0348-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0349-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0350-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0351-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0352-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0353-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0354-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0355-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0356-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0357-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0358-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0359-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0360-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0361-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0362-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0363-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0364-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0365-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0366-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0367-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0368-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0369-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0370-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0371-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0372-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0373-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0374-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0375-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0376-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0377-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0378-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0379-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0380-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0381-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0382-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0383-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0384-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0385-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0386-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0387-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0388-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0389-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0390-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0391-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0392-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0393-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0394-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0395-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0396-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0397-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0398-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0399-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0400-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0401-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0402-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0403-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0404-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0405-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0406-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0407-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0408-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0409-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0410-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0411-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0412-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0413-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0414-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0415-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0416-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0417-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0418-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0419-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0420-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0421-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0422-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0423-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0424-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0425-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0426-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0427-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0428-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0429-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0430-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0431-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0432-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0433-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0434-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0435-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0436-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0437-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0438-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0439-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0440-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0441-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0442-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0443-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0444-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0445-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0446-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0447-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0448-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0449-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0450-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0451-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0452-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0453-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0454-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0455-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0456-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0457-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0458-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0459-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0460-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0461-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0462-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0463-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0464-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0465-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0466-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0467-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0468-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0469-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0470-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0471-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0472-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0473-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0474-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0475-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0476-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0477-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0478-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0479-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0480-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0481-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0482-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0483-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0484-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0485-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0486-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0487-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0488-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0489-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0490-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0491-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0492-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0493-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0494-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0495-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0496-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0497-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0498-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0499-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0500-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0501-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0502-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0503-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0504-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0505-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0506-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0507-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0508-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0509-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0510-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0511-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0512-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0513-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0514-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0515-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0516-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0517-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0518-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0519-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0520-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0521-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0522-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0523-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0524-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0525-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0526-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0527-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0528-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0529-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0530-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0531-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0532-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0533-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0534-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0535-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0536-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0537-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0538-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0539-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0540-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0541-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0542-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0543-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0544-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0545-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0546-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0547-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0548-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0549-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0550-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0551-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0552-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0553-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0554-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0555-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0556-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0557-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0558-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0559-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0560-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0561-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0562-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0563-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0564-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0565-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0566-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0567-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0568-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0569-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0570-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0571-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0572-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0573-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0574-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0575-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0576-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0577-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0578-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0579-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0580-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0581-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0582-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0583-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0584-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0585-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0586-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0587-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0588-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0589-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0590-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0591-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0592-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0593-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0594-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0595-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0596-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0597-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0598-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0599-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0600-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0601-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0602-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0603-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0604-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0605-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0606-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0607-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0608-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0609-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0610-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0611-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0612-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0613-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0614-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0615-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0616-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0617-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0618-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0619-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0620-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0621-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0622-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0623-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0624-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0625-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0626-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0627-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0628-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0629-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0630-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0631-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0632-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0633-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0634-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0635-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0636-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0637-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0638-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0639-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0640-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0641-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0642-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0643-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0644-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0645-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0646-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0647-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0648-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0649-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0650-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0651-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0652-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0653-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0654-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0655-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0656-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0657-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0658-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0659-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0660-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0661-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0662-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0663-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0664-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0665-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0666-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0667-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0668-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0669-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0670-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0671-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0672-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0673-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0674-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0675-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0676-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0677-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0678-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0679-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0680-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0681-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0682-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0683-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0684-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0685-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0686-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0687-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0688-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0689-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0690-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0691-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0692-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0693-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0694-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0695-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0696-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0697-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0698-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0699-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0700-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0701-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0702-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0703-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0704-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0705-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0706-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0707-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0708-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0709-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0710-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0711-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0712-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0713-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0714-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0715-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0716-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0717-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0718-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0719-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0720-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0721-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0722-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0723-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0724-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0725-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0726-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0727-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0728-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0729-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0730-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0731-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0732-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0733-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0734-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0735-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0736-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0737-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0738-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0739-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0740-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0741-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0742-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0743-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0744-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0745-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0746-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0747-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0748-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0749-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0750-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0751-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0752-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0753-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0754-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0755-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0756-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0757-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0758-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0759-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0760-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0761-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0762-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0763-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0764-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0765-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0766-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0767-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0768-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0769-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0770-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0771-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0772-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0773-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0774-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0775-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0776-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0777-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0778-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0779-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0780-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0781-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0782-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0783-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0784-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0785-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0786-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0787-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0788-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0789-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0790-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0791-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0792-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0793-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0794-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0795-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0796-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0797-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0798-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0799-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0800-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0801-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0802-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0803-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0804-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0805-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0806-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0807-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0808-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0809-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0810-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0811-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0812-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0813-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0814-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0815-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0816-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0817-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0818-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0819-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0820-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0821-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0822-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0823-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0824-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0825-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0826-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0827-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0828-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0829-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0830-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0831-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0832-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0833-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0834-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0835-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0836-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0837-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0838-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0839-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0840-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0841-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0842-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0843-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0844-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0845-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0846-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0847-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0848-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0849-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0850-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0851-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0852-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0853-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0854-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0855-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0856-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0857-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0858-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0859-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0860-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0861-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0862-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0863-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0864-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0865-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0866-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0867-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0868-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0869-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0870-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0871-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0872-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0873-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0874-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0875-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0876-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0877-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0878-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0879-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0880-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0881-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0882-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0883-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0884-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0885-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0886-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0887-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0888-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0889-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0890-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0891-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0892-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0893-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0894-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0895-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0896-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0897-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0898-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0899-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0900-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0901-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0902-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0903-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0904-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0905-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0906-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0907-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0908-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0909-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0910-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0911-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0912-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0913-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0914-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0915-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0916-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0917-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0918-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0919-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0920-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0921-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0922-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0923-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0924-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0925-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0926-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0927-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0928-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0929-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0930-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0931-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0932-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0933-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0934-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0935-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0936-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0937-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0938-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0939-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0940-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0941-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0942-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0943-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0944-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0945-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0946-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0947-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0948-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0949-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0950-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0951-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0952-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0953-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0954-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0955-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0956-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0957-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0958-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0959-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0960-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0961-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0962-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0963-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0964-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0965-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0966-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0967-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0968-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0969-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0970-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0971-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0972-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0973-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0974-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0975-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0976-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0977-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0978-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0979-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0980-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0981-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0982-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0983-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0984-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0985-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0986-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0987-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0988-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0989-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0990-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0991-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0992-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0993-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0994-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0995-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0996-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0997-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0998-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0999-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1000-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1001-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1002-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1003-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1004-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1005-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1006-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1007-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1008-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1009-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1010-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1011-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1012-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1013-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1014-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1015-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1016-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1017-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1018-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1019-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1020-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1021-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1022-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1023-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH +  !"#$%&'()*+,-./0123456789:;<HHHHHHHHHHHH=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\]^_`abcdefghijklmnopqrstuvwxyHHHHHHHHHHHHz{|}~HHHHHHHHHHHHHHHHHHHHHHHH  +    !"#$%&'()*+,-./0HHHHHHHHHHHH123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\]^_`abcdefghijklmHHHHHHHHHHHHnopqrstuvwxyz{|}~HHHHHHHHHHHHHHHHHHHHHHHH  +    !"#$HHHHHHHHHHHH%&'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\]^_`aHHHHHHHHHHHHbcdefghijklmnopqrstuvwxyz{|}~HHHHHHHHHHHHHHHHHHHHHHHH  +   HHHHHHHHHHHH !"#$%&'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUHHHHHHHHHHHHVWXYZ[\]^_`abcdefghijklmnopqrstuvwxyz{|}~HHHHHHHHHHHHHHHHHHHHHHHH @`value-0001-deterministic-fixturevalue-0002-deterministic-fixturevalue-0003-deterministic-fixturevalue-0004-deterministic-fixturevalue-0005-deterministic-fixturevalue-0006-deterministic-fixture @`value-0008-deterministic-fixturevalue-0009-deterministic-fixturevalue-0010-deterministic-fixturevalue-0011-deterministic-fixturevalue-0012-deterministic-fixturevalue-0013-deterministic-fixturevalue-0015-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-0016-deterministic-fixturevalue-0017-deterministic-fixturevalue-0018-deterministic-fixturevalue-0019-deterministic-fixturevalue-0020-deterministic-fixturevalue-0022-deterministic-fixturevalue-0023-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`avalue-0024-deterministic-fixturevalue-0025-deterministic-fixturevalue-0026-deterministic-fixturevalue-0027-deterministic-fixturevalue-0029-deterministic-fixturevalue-0030-deterministic-fixturevalue-0031-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`Avalue-0032-deterministic-fixturevalue-0033-deterministic-fixturevalue-0034-deterministic-fixturevalue-0036-deterministic-fixturevalue-0037-deterministic-fixturevalue-0038-deterministic-fixturevalue-0039-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @!`value-0040-deterministic-fixturevalue-0041-deterministic-fixturevalue-0043-deterministic-fixturevalue-0044-deterministic-fixturevalue-0045-deterministic-fixturevalue-0046-deterministic-fixturevalue-0047-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-0048-deterministic-fixturevalue-0050-deterministic-fixturevalue-0051-deterministic-fixturevalue-0052-deterministic-fixturevalue-0053-deterministic-fixturevalue-0054-deterministic-fixturevalue-0055-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-0057-deterministic-fixturevalue-0058-deterministic-fixturevalue-0059-deterministic-fixturevalue-0060-deterministic-fixturevalue-0061-deterministic-fixturevalue-0062-deterministic-fixture @`value-0064-deterministic-fixturevalue-0065-deterministic-fixturevalue-0066-deterministic-fixturevalue-0067-deterministic-fixturevalue-0068-deterministic-fixturevalue-0069-deterministic-fixturevalue-0071-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-0072-deterministic-fixturevalue-0073-deterministic-fixturevalue-0074-deterministic-fixturevalue-0075-deterministic-fixturevalue-0076-deterministic-fixturevalue-0078-deterministic-fixturevalue-0079-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`avalue-0080-deterministic-fixturevalue-0081-deterministic-fixturevalue-0082-deterministic-fixturevalue-0083-deterministic-fixturevalue-0085-deterministic-fixturevalue-0086-deterministic-fixturevalue-0087-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`Avalue-0088-deterministic-fixturevalue-0089-deterministic-fixturevalue-0090-deterministic-fixturevalue-0092-deterministic-fixturevalue-0093-deterministic-fixturevalue-0094-deterministic-fixturevalue-0095-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @!`value-0096-deterministic-fixturevalue-0097-deterministic-fixturevalue-0099-deterministic-fixturevalue-0100-deterministic-fixturevalue-0101-deterministic-fixturevalue-0102-deterministic-fixturevalue-0103-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-0104-deterministic-fixturevalue-0106-deterministic-fixturevalue-0107-deterministic-fixturevalue-0108-deterministic-fixturevalue-0109-deterministic-fixturevalue-0110-deterministic-fixturevalue-0111-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-0113-deterministic-fixturevalue-0114-deterministic-fixturevalue-0115-deterministic-fixturevalue-0116-deterministic-fixturevalue-0117-deterministic-fixturevalue-0118-deterministic-fixture @`value-0120-deterministic-fixturevalue-0121-deterministic-fixturevalue-0122-deterministic-fixturevalue-0123-deterministic-fixturevalue-0124-deterministic-fixturevalue-0125-deterministic-fixturevalue-0127-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-0128-deterministic-fixturevalue-0129-deterministic-fixturevalue-0130-deterministic-fixturevalue-0131-deterministic-fixturevalue-0132-deterministic-fixturevalue-0134-deterministic-fixturevalue-0135-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`avalue-0136-deterministic-fixturevalue-0137-deterministic-fixturevalue-0138-deterministic-fixturevalue-0139-deterministic-fixturevalue-0141-deterministic-fixturevalue-0142-deterministic-fixturevalue-0143-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`Avalue-0144-deterministic-fixturevalue-0145-deterministic-fixturevalue-0146-deterministic-fixturevalue-0148-deterministic-fixturevalue-0149-deterministic-fixturevalue-0150-deterministic-fixturevalue-0151-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @!`value-0152-deterministic-fixturevalue-0153-deterministic-fixturevalue-0155-deterministic-fixturevalue-0156-deterministic-fixturevalue-0157-deterministic-fixturevalue-0158-deterministic-fixturevalue-0159-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-0160-deterministic-fixturevalue-0162-deterministic-fixturevalue-0163-deterministic-fixturevalue-0164-deterministic-fixturevalue-0165-deterministic-fixturevalue-0166-deterministic-fixturevalue-0167-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-0169-deterministic-fixturevalue-0170-deterministic-fixturevalue-0171-deterministic-fixturevalue-0172-deterministic-fixturevalue-0173-deterministic-fixturevalue-0174-deterministic-fixture @`value-0176-deterministic-fixturevalue-0177-deterministic-fixturevalue-0178-deterministic-fixturevalue-0179-deterministic-fixturevalue-0180-deterministic-fixturevalue-0181-deterministic-fixturevalue-0183-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-0184-deterministic-fixturevalue-0185-deterministic-fixturevalue-0186-deterministic-fixturevalue-0187-deterministic-fixturevalue-0188-deterministic-fixturevalue-0190-deterministic-fixturevalue-0191-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`avalue-0192-deterministic-fixturevalue-0193-deterministic-fixturevalue-0194-deterministic-fixturevalue-0195-deterministic-fixturevalue-0197-deterministic-fixturevalue-0198-deterministic-fixturevalue-0199-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`Avalue-0200-deterministic-fixturevalue-0201-deterministic-fixturevalue-0202-deterministic-fixturevalue-0204-deterministic-fixturevalue-0205-deterministic-fixturevalue-0206-deterministic-fixturevalue-0207-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @!`value-0208-deterministic-fixturevalue-0209-deterministic-fixturevalue-0211-deterministic-fixturevalue-0212-deterministic-fixturevalue-0213-deterministic-fixturevalue-0214-deterministic-fixturevalue-0215-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-0216-deterministic-fixturevalue-0218-deterministic-fixturevalue-0219-deterministic-fixturevalue-0220-deterministic-fixturevalue-0221-deterministic-fixturevalue-0222-deterministic-fixturevalue-0223-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-0225-deterministic-fixturevalue-0226-deterministic-fixturevalue-0227-deterministic-fixturevalue-0228-deterministic-fixturevalue-0229-deterministic-fixturevalue-0230-deterministic-fixture @`value-0232-deterministic-fixturevalue-0233-deterministic-fixturevalue-0234-deterministic-fixturevalue-0235-deterministic-fixturevalue-0236-deterministic-fixturevalue-0237-deterministic-fixturevalue-0239-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-0240-deterministic-fixturevalue-0241-deterministic-fixturevalue-0242-deterministic-fixturevalue-0243-deterministic-fixturevalue-0244-deterministic-fixturevalue-0246-deterministic-fixturevalue-0247-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`avalue-0248-deterministic-fixturevalue-0249-deterministic-fixturevalue-0250-deterministic-fixturevalue-0251-deterministic-fixturevalue-0253-deterministic-fixturevalue-0254-deterministic-fixturevalue-0255-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`Avalue-0256-deterministic-fixturevalue-0257-deterministic-fixturevalue-0258-deterministic-fixturevalue-0260-deterministic-fixturevalue-0261-deterministic-fixturevalue-0262-deterministic-fixturevalue-0263-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @!`value-0264-deterministic-fixturevalue-0265-deterministic-fixturevalue-0267-deterministic-fixturevalue-0268-deterministic-fixturevalue-0269-deterministic-fixturevalue-0270-deterministic-fixturevalue-0271-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-0272-deterministic-fixturevalue-0274-deterministic-fixturevalue-0275-deterministic-fixturevalue-0276-deterministic-fixturevalue-0277-deterministic-fixturevalue-0278-deterministic-fixturevalue-0279-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-0281-deterministic-fixturevalue-0282-deterministic-fixturevalue-0283-deterministic-fixturevalue-0284-deterministic-fixturevalue-0285-deterministic-fixturevalue-0286-deterministic-fixture @`value-0288-deterministic-fixturevalue-0289-deterministic-fixturevalue-0290-deterministic-fixturevalue-0291-deterministic-fixturevalue-0292-deterministic-fixturevalue-0293-deterministic-fixturevalue-0295-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-0296-deterministic-fixturevalue-0297-deterministic-fixturevalue-0298-deterministic-fixturevalue-0299-deterministic-fixturevalue-0300-deterministic-fixturevalue-0302-deterministic-fixturevalue-0303-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`avalue-0304-deterministic-fixturevalue-0305-deterministic-fixturevalue-0306-deterministic-fixturevalue-0307-deterministic-fixturevalue-0309-deterministic-fixturevalue-0310-deterministic-fixturevalue-0311-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`Avalue-0312-deterministic-fixturevalue-0313-deterministic-fixturevalue-0314-deterministic-fixturevalue-0316-deterministic-fixturevalue-0317-deterministic-fixturevalue-0318-deterministic-fixturevalue-0319-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @!`value-0320-deterministic-fixturevalue-0321-deterministic-fixturevalue-0323-deterministic-fixturevalue-0324-deterministic-fixturevalue-0325-deterministic-fixturevalue-0326-deterministic-fixturevalue-0327-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-0328-deterministic-fixturevalue-0330-deterministic-fixturevalue-0331-deterministic-fixturevalue-0332-deterministic-fixturevalue-0333-deterministic-fixturevalue-0334-deterministic-fixturevalue-0335-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-0337-deterministic-fixturevalue-0338-deterministic-fixturevalue-0339-deterministic-fixturevalue-0340-deterministic-fixturevalue-0341-deterministic-fixturevalue-0342-deterministic-fixture @`value-0344-deterministic-fixturevalue-0345-deterministic-fixturevalue-0346-deterministic-fixturevalue-0347-deterministic-fixturevalue-0348-deterministic-fixturevalue-0349-deterministic-fixturevalue-0351-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-0352-deterministic-fixturevalue-0353-deterministic-fixturevalue-0354-deterministic-fixturevalue-0355-deterministic-fixturevalue-0356-deterministic-fixturevalue-0358-deterministic-fixturevalue-0359-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`avalue-0360-deterministic-fixturevalue-0361-deterministic-fixturevalue-0362-deterministic-fixturevalue-0363-deterministic-fixturevalue-0365-deterministic-fixturevalue-0366-deterministic-fixturevalue-0367-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`Avalue-0368-deterministic-fixturevalue-0369-deterministic-fixturevalue-0370-deterministic-fixturevalue-0372-deterministic-fixturevalue-0373-deterministic-fixturevalue-0374-deterministic-fixturevalue-0375-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @!`value-0376-deterministic-fixturevalue-0377-deterministic-fixturevalue-0379-deterministic-fixturevalue-0380-deterministic-fixturevalue-0381-deterministic-fixturevalue-0382-deterministic-fixturevalue-0383-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-0384-deterministic-fixturevalue-0386-deterministic-fixturevalue-0387-deterministic-fixturevalue-0388-deterministic-fixturevalue-0389-deterministic-fixturevalue-0390-deterministic-fixturevalue-0391-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-0393-deterministic-fixturevalue-0394-deterministic-fixturevalue-0395-deterministic-fixturevalue-0396-deterministic-fixturevalue-0397-deterministic-fixturevalue-0398-deterministic-fixture @`value-0400-deterministic-fixturevalue-0401-deterministic-fixturevalue-0402-deterministic-fixturevalue-0403-deterministic-fixturevalue-0404-deterministic-fixturevalue-0405-deterministic-fixturevalue-0407-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-0408-deterministic-fixturevalue-0409-deterministic-fixturevalue-0410-deterministic-fixturevalue-0411-deterministic-fixturevalue-0412-deterministic-fixturevalue-0414-deterministic-fixturevalue-0415-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`avalue-0416-deterministic-fixturevalue-0417-deterministic-fixturevalue-0418-deterministic-fixturevalue-0419-deterministic-fixturevalue-0421-deterministic-fixturevalue-0422-deterministic-fixturevalue-0423-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`Avalue-0424-deterministic-fixturevalue-0425-deterministic-fixturevalue-0426-deterministic-fixturevalue-0428-deterministic-fixturevalue-0429-deterministic-fixturevalue-0430-deterministic-fixturevalue-0431-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @!`value-0432-deterministic-fixturevalue-0433-deterministic-fixturevalue-0435-deterministic-fixturevalue-0436-deterministic-fixturevalue-0437-deterministic-fixturevalue-0438-deterministic-fixturevalue-0439-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-0440-deterministic-fixturevalue-0442-deterministic-fixturevalue-0443-deterministic-fixturevalue-0444-deterministic-fixturevalue-0445-deterministic-fixturevalue-0446-deterministic-fixturevalue-0447-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-0449-deterministic-fixturevalue-0450-deterministic-fixturevalue-0451-deterministic-fixturevalue-0452-deterministic-fixturevalue-0453-deterministic-fixturevalue-0454-deterministic-fixture @`value-0456-deterministic-fixturevalue-0457-deterministic-fixturevalue-0458-deterministic-fixturevalue-0459-deterministic-fixturevalue-0460-deterministic-fixturevalue-0461-deterministic-fixturevalue-0463-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-0464-deterministic-fixturevalue-0465-deterministic-fixturevalue-0466-deterministic-fixturevalue-0467-deterministic-fixturevalue-0468-deterministic-fixturevalue-0470-deterministic-fixturevalue-0471-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`avalue-0472-deterministic-fixturevalue-0473-deterministic-fixturevalue-0474-deterministic-fixturevalue-0475-deterministic-fixturevalue-0477-deterministic-fixturevalue-0478-deterministic-fixturevalue-0479-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`Avalue-0480-deterministic-fixturevalue-0481-deterministic-fixturevalue-0482-deterministic-fixturevalue-0484-deterministic-fixturevalue-0485-deterministic-fixturevalue-0486-deterministic-fixturevalue-0487-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @!`value-0488-deterministic-fixturevalue-0489-deterministic-fixturevalue-0491-deterministic-fixturevalue-0492-deterministic-fixturevalue-0493-deterministic-fixturevalue-0494-deterministic-fixturevalue-0495-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-0496-deterministic-fixturevalue-0498-deterministic-fixturevalue-0499-deterministic-fixturevalue-0500-deterministic-fixturevalue-0501-deterministic-fixturevalue-0502-deterministic-fixturevalue-0503-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-0505-deterministic-fixturevalue-0506-deterministic-fixturevalue-0507-deterministic-fixturevalue-0508-deterministic-fixturevalue-0509-deterministic-fixturevalue-0510-deterministic-fixture @`value-0512-deterministic-fixturevalue-0513-deterministic-fixturevalue-0514-deterministic-fixturevalue-0515-deterministic-fixturevalue-0516-deterministic-fixturevalue-0517-deterministic-fixturevalue-0519-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-0520-deterministic-fixturevalue-0521-deterministic-fixturevalue-0522-deterministic-fixturevalue-0523-deterministic-fixturevalue-0524-deterministic-fixturevalue-0526-deterministic-fixturevalue-0527-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`avalue-0528-deterministic-fixturevalue-0529-deterministic-fixturevalue-0530-deterministic-fixturevalue-0531-deterministic-fixturevalue-0533-deterministic-fixturevalue-0534-deterministic-fixturevalue-0535-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`Avalue-0536-deterministic-fixturevalue-0537-deterministic-fixturevalue-0538-deterministic-fixturevalue-0540-deterministic-fixturevalue-0541-deterministic-fixturevalue-0542-deterministic-fixturevalue-0543-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @!`value-0544-deterministic-fixturevalue-0545-deterministic-fixturevalue-0547-deterministic-fixturevalue-0548-deterministic-fixturevalue-0549-deterministic-fixturevalue-0550-deterministic-fixturevalue-0551-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-0552-deterministic-fixturevalue-0554-deterministic-fixturevalue-0555-deterministic-fixturevalue-0556-deterministic-fixturevalue-0557-deterministic-fixturevalue-0558-deterministic-fixturevalue-0559-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-0561-deterministic-fixturevalue-0562-deterministic-fixturevalue-0563-deterministic-fixturevalue-0564-deterministic-fixturevalue-0565-deterministic-fixturevalue-0566-deterministic-fixture @`value-0568-deterministic-fixturevalue-0569-deterministic-fixturevalue-0570-deterministic-fixturevalue-0571-deterministic-fixturevalue-0572-deterministic-fixturevalue-0573-deterministic-fixturevalue-0575-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-0576-deterministic-fixturevalue-0577-deterministic-fixturevalue-0578-deterministic-fixturevalue-0579-deterministic-fixturevalue-0580-deterministic-fixturevalue-0582-deterministic-fixturevalue-0583-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`avalue-0584-deterministic-fixturevalue-0585-deterministic-fixturevalue-0586-deterministic-fixturevalue-0587-deterministic-fixturevalue-0589-deterministic-fixturevalue-0590-deterministic-fixturevalue-0591-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`Avalue-0592-deterministic-fixturevalue-0593-deterministic-fixturevalue-0594-deterministic-fixturevalue-0596-deterministic-fixturevalue-0597-deterministic-fixturevalue-0598-deterministic-fixturevalue-0599-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @!`value-0600-deterministic-fixturevalue-0601-deterministic-fixturevalue-0603-deterministic-fixturevalue-0604-deterministic-fixturevalue-0605-deterministic-fixturevalue-0606-deterministic-fixturevalue-0607-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-0608-deterministic-fixturevalue-0610-deterministic-fixturevalue-0611-deterministic-fixturevalue-0612-deterministic-fixturevalue-0613-deterministic-fixturevalue-0614-deterministic-fixturevalue-0615-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-0617-deterministic-fixturevalue-0618-deterministic-fixturevalue-0619-deterministic-fixturevalue-0620-deterministic-fixturevalue-0621-deterministic-fixturevalue-0622-deterministic-fixture @`value-0624-deterministic-fixturevalue-0625-deterministic-fixturevalue-0626-deterministic-fixturevalue-0627-deterministic-fixturevalue-0628-deterministic-fixturevalue-0629-deterministic-fixturevalue-0631-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-0632-deterministic-fixturevalue-0633-deterministic-fixturevalue-0634-deterministic-fixturevalue-0635-deterministic-fixturevalue-0636-deterministic-fixturevalue-0638-deterministic-fixturevalue-0639-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`avalue-0640-deterministic-fixturevalue-0641-deterministic-fixturevalue-0642-deterministic-fixturevalue-0643-deterministic-fixturevalue-0645-deterministic-fixturevalue-0646-deterministic-fixturevalue-0647-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`Avalue-0648-deterministic-fixturevalue-0649-deterministic-fixturevalue-0650-deterministic-fixturevalue-0652-deterministic-fixturevalue-0653-deterministic-fixturevalue-0654-deterministic-fixturevalue-0655-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @!`value-0656-deterministic-fixturevalue-0657-deterministic-fixturevalue-0659-deterministic-fixturevalue-0660-deterministic-fixturevalue-0661-deterministic-fixturevalue-0662-deterministic-fixturevalue-0663-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-0664-deterministic-fixturevalue-0666-deterministic-fixturevalue-0667-deterministic-fixturevalue-0668-deterministic-fixturevalue-0669-deterministic-fixturevalue-0670-deterministic-fixturevalue-0671-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-0673-deterministic-fixturevalue-0674-deterministic-fixturevalue-0675-deterministic-fixturevalue-0676-deterministic-fixturevalue-0677-deterministic-fixturevalue-0678-deterministic-fixture @`value-0680-deterministic-fixturevalue-0681-deterministic-fixturevalue-0682-deterministic-fixturevalue-0683-deterministic-fixturevalue-0684-deterministic-fixturevalue-0685-deterministic-fixturevalue-0687-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-0688-deterministic-fixturevalue-0689-deterministic-fixturevalue-0690-deterministic-fixturevalue-0691-deterministic-fixturevalue-0692-deterministic-fixturevalue-0694-deterministic-fixturevalue-0695-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`avalue-0696-deterministic-fixturevalue-0697-deterministic-fixturevalue-0698-deterministic-fixturevalue-0699-deterministic-fixturevalue-0701-deterministic-fixturevalue-0702-deterministic-fixturevalue-0703-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`Avalue-0704-deterministic-fixturevalue-0705-deterministic-fixturevalue-0706-deterministic-fixturevalue-0708-deterministic-fixturevalue-0709-deterministic-fixturevalue-0710-deterministic-fixturevalue-0711-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @!`value-0712-deterministic-fixturevalue-0713-deterministic-fixturevalue-0715-deterministic-fixturevalue-0716-deterministic-fixturevalue-0717-deterministic-fixturevalue-0718-deterministic-fixturevalue-0719-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-0720-deterministic-fixturevalue-0722-deterministic-fixturevalue-0723-deterministic-fixturevalue-0724-deterministic-fixturevalue-0725-deterministic-fixturevalue-0726-deterministic-fixturevalue-0727-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-0729-deterministic-fixturevalue-0730-deterministic-fixturevalue-0731-deterministic-fixturevalue-0732-deterministic-fixturevalue-0733-deterministic-fixturevalue-0734-deterministic-fixture @`value-0736-deterministic-fixturevalue-0737-deterministic-fixturevalue-0738-deterministic-fixturevalue-0739-deterministic-fixturevalue-0740-deterministic-fixturevalue-0741-deterministic-fixturevalue-0743-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-0744-deterministic-fixturevalue-0745-deterministic-fixturevalue-0746-deterministic-fixturevalue-0747-deterministic-fixturevalue-0748-deterministic-fixturevalue-0750-deterministic-fixturevalue-0751-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`avalue-0752-deterministic-fixturevalue-0753-deterministic-fixturevalue-0754-deterministic-fixturevalue-0755-deterministic-fixturevalue-0757-deterministic-fixturevalue-0758-deterministic-fixturevalue-0759-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`Avalue-0760-deterministic-fixturevalue-0761-deterministic-fixturevalue-0762-deterministic-fixturevalue-0764-deterministic-fixturevalue-0765-deterministic-fixturevalue-0766-deterministic-fixturevalue-0767-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @!`value-0768-deterministic-fixturevalue-0769-deterministic-fixturevalue-0771-deterministic-fixturevalue-0772-deterministic-fixturevalue-0773-deterministic-fixturevalue-0774-deterministic-fixturevalue-0775-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-0776-deterministic-fixturevalue-0778-deterministic-fixturevalue-0779-deterministic-fixturevalue-0780-deterministic-fixturevalue-0781-deterministic-fixturevalue-0782-deterministic-fixturevalue-0783-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-0785-deterministic-fixturevalue-0786-deterministic-fixturevalue-0787-deterministic-fixturevalue-0788-deterministic-fixturevalue-0789-deterministic-fixturevalue-0790-deterministic-fixture @`value-0792-deterministic-fixturevalue-0793-deterministic-fixturevalue-0794-deterministic-fixturevalue-0795-deterministic-fixturevalue-0796-deterministic-fixturevalue-0797-deterministic-fixturevalue-0799-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-0800-deterministic-fixturevalue-0801-deterministic-fixturevalue-0802-deterministic-fixturevalue-0803-deterministic-fixturevalue-0804-deterministic-fixturevalue-0806-deterministic-fixturevalue-0807-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`avalue-0808-deterministic-fixturevalue-0809-deterministic-fixturevalue-0810-deterministic-fixturevalue-0811-deterministic-fixturevalue-0813-deterministic-fixturevalue-0814-deterministic-fixturevalue-0815-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`Avalue-0816-deterministic-fixturevalue-0817-deterministic-fixturevalue-0818-deterministic-fixturevalue-0820-deterministic-fixturevalue-0821-deterministic-fixturevalue-0822-deterministic-fixturevalue-0823-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @!`value-0824-deterministic-fixturevalue-0825-deterministic-fixturevalue-0827-deterministic-fixturevalue-0828-deterministic-fixturevalue-0829-deterministic-fixturevalue-0830-deterministic-fixturevalue-0831-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-0832-deterministic-fixturevalue-0834-deterministic-fixturevalue-0835-deterministic-fixturevalue-0836-deterministic-fixturevalue-0837-deterministic-fixturevalue-0838-deterministic-fixturevalue-0839-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-0841-deterministic-fixturevalue-0842-deterministic-fixturevalue-0843-deterministic-fixturevalue-0844-deterministic-fixturevalue-0845-deterministic-fixturevalue-0846-deterministic-fixture @`value-0848-deterministic-fixturevalue-0849-deterministic-fixturevalue-0850-deterministic-fixturevalue-0851-deterministic-fixturevalue-0852-deterministic-fixturevalue-0853-deterministic-fixturevalue-0855-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-0856-deterministic-fixturevalue-0857-deterministic-fixturevalue-0858-deterministic-fixturevalue-0859-deterministic-fixturevalue-0860-deterministic-fixturevalue-0862-deterministic-fixturevalue-0863-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`avalue-0864-deterministic-fixturevalue-0865-deterministic-fixturevalue-0866-deterministic-fixturevalue-0867-deterministic-fixturevalue-0869-deterministic-fixturevalue-0870-deterministic-fixturevalue-0871-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`Avalue-0872-deterministic-fixturevalue-0873-deterministic-fixturevalue-0874-deterministic-fixturevalue-0876-deterministic-fixturevalue-0877-deterministic-fixturevalue-0878-deterministic-fixturevalue-0879-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @!`value-0880-deterministic-fixturevalue-0881-deterministic-fixturevalue-0883-deterministic-fixturevalue-0884-deterministic-fixturevalue-0885-deterministic-fixturevalue-0886-deterministic-fixturevalue-0887-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-0888-deterministic-fixturevalue-0890-deterministic-fixturevalue-0891-deterministic-fixturevalue-0892-deterministic-fixturevalue-0893-deterministic-fixturevalue-0894-deterministic-fixturevalue-0895-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-0897-deterministic-fixturevalue-0898-deterministic-fixturevalue-0899-deterministic-fixturevalue-0900-deterministic-fixturevalue-0901-deterministic-fixturevalue-0902-deterministic-fixture @`value-0904-deterministic-fixturevalue-0905-deterministic-fixturevalue-0906-deterministic-fixturevalue-0907-deterministic-fixturevalue-0908-deterministic-fixturevalue-0909-deterministic-fixturevalue-0911-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-0912-deterministic-fixturevalue-0913-deterministic-fixturevalue-0914-deterministic-fixturevalue-0915-deterministic-fixturevalue-0916-deterministic-fixturevalue-0918-deterministic-fixturevalue-0919-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`avalue-0920-deterministic-fixturevalue-0921-deterministic-fixturevalue-0922-deterministic-fixturevalue-0923-deterministic-fixturevalue-0925-deterministic-fixturevalue-0926-deterministic-fixturevalue-0927-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`Avalue-0928-deterministic-fixturevalue-0929-deterministic-fixturevalue-0930-deterministic-fixturevalue-0932-deterministic-fixturevalue-0933-deterministic-fixturevalue-0934-deterministic-fixturevalue-0935-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @!`value-0936-deterministic-fixturevalue-0937-deterministic-fixturevalue-0939-deterministic-fixturevalue-0940-deterministic-fixturevalue-0941-deterministic-fixturevalue-0942-deterministic-fixturevalue-0943-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-0944-deterministic-fixturevalue-0946-deterministic-fixturevalue-0947-deterministic-fixturevalue-0948-deterministic-fixturevalue-0949-deterministic-fixturevalue-0950-deterministic-fixturevalue-0951-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-0953-deterministic-fixturevalue-0954-deterministic-fixturevalue-0955-deterministic-fixturevalue-0956-deterministic-fixturevalue-0957-deterministic-fixturevalue-0958-deterministic-fixture @`value-0960-deterministic-fixturevalue-0961-deterministic-fixturevalue-0962-deterministic-fixturevalue-0963-deterministic-fixturevalue-0964-deterministic-fixturevalue-0965-deterministic-fixturevalue-0967-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-0968-deterministic-fixturevalue-0969-deterministic-fixturevalue-0970-deterministic-fixturevalue-0971-deterministic-fixturevalue-0972-deterministic-fixturevalue-0974-deterministic-fixturevalue-0975-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`avalue-0976-deterministic-fixturevalue-0977-deterministic-fixturevalue-0978-deterministic-fixturevalue-0979-deterministic-fixturevalue-0981-deterministic-fixturevalue-0982-deterministic-fixturevalue-0983-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`Avalue-0984-deterministic-fixturevalue-0985-deterministic-fixturevalue-0986-deterministic-fixturevalue-0988-deterministic-fixturevalue-0989-deterministic-fixturevalue-0990-deterministic-fixturevalue-0991-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @!`value-0992-deterministic-fixturevalue-0993-deterministic-fixturevalue-0995-deterministic-fixturevalue-0996-deterministic-fixturevalue-0997-deterministic-fixturevalue-0998-deterministic-fixturevalue-0999-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-1000-deterministic-fixturevalue-1002-deterministic-fixturevalue-1003-deterministic-fixturevalue-1004-deterministic-fixturevalue-1005-deterministic-fixturevalue-1006-deterministic-fixturevalue-1007-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-1009-deterministic-fixturevalue-1010-deterministic-fixturevalue-1011-deterministic-fixturevalue-1012-deterministic-fixturevalue-1013-deterministic-fixturevalue-1014-deterministic-fixture @`value-1016-deterministic-fixturevalue-1017-deterministic-fixturevalue-1018-deterministic-fixturevalue-1019-deterministic-fixturevalue-1020-deterministic-fixturevalue-1021-deterministic-fixturevalue-1023-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH +  !$'*-0369<# ?BEHKNQTWZA ]`cfilorux_ {~}    #&), /258;>ADGJ1 MPSVY\_behO knqtwz}m      +  "%(+.147:! =@CFILORUX? [^adgjmpsv] y|{       !$'*-0369<?BEH/KNQTWZ]`cfMilorux{~k  #&),/258;>ADGJMPSV=Y\_behknqt[wz}y + "%(+.147:=@CF-ILORUX[^adKgjmpsvy|i  !$'*-0369<?BEHKNQT;WZ]`cfilorYux{~w  #& ),/258;>AD+GJMPSVY\_bIehknqtwz}g + "%(+.147:=@CFILOR9UX[^adgjmpWsvy|u          ! $ ' * - 0 3 6 9 < ? B )E H K N Q T W Z ] ` Gc f i l o r u x { ~ e      + + + + + + + + + + +# +& +) +, +/ +2 +5 +8 +; +> +A +D +G +J +M +P +7S +V +Y +\ +_ +b +e +h +k +n +Uq +t +w +z +} + + + + + +s + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH     + $ '*-HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH 0"3$6&9<*?.E0HK4N6Q8T:WZ>HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH] @`"Df#i$Hl%Jo&Lr'Nu(x)R{*T~+V-.\/^HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH0`1b23f4h5j6l8p9r:t;v<=z>|?HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH~@ACDEFGHIJKLNOHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHPQRSTUVWY Z[\]^HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH_` a#b&d,e/f2g5h8i;j>kAlDmGoHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHMpPqSrVsYt\u_vbwexhzn{q|t}w~HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHz} + HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH $&(*.028:HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH<>BDFHLNPRVXZHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH\ + bdfjl"n%p(+t.x4z7HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH:~=@CFILOUX[^adHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHgjmpvy|HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH   +   + ! HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH$ '*- 0"3$6(<*?,B.EH2K4N6QHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH8TW>] @`!Bc"f#Fi$Hl%Jo&Lr'u(Px*T~+V,HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH-Z.\/^0`12d3f5j67n8p9r:t;HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH<x=z>|@ABCDEFGHIKHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHLMNOPQRSTVWXY Z[HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH\]^_a#b&c)d,e/f2g5h8i;jHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH>lDmGnJoMpPqSrVsYt\u_wexhykznHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH{q|t}w~z} +HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH"$&(,02HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH68:<@BFJLNPHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHTVX +^ `bdhjl"n%r+HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHt.v1x47|:~=@CFLORUXHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH[^adgmpsvy|HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH   HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH  + ! $ '*-"36&9(<*?,BEHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH0H2K4NT:W<Z>] @`!c"Df#Fi$Hl%Jo'Nu(PHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHx)R{*T~+,X-Z.\/^02d3f4h56l7nHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH8p9r:;v=z>|?@ABCDEFHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHIJKLMNOPQSTUVWHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHXY Z[\^_` a#b&c)d,e/f2HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHg5i;j>kAlDmGnJoMpPqSrVt\u_vbwHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHexhykzn{q|t}w} HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH "$&*,HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH.0468:@BDHJHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHLNRVX\ +^ `bfhlHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH"%p(r+t.v14z7|:~=CFILOHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHRUX[^dgjmpsvy|HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH    HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH +      + ! $ * -  0 3 $6 &9 (HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH< *? B .E 2K 4N Q 8T :W <Z >] ` !Bc "Df $Hl HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH%o &Lr 'Nu (Px )R{ *~ +V ,X -Z / 0` 1b 2d 3f 4HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH 5j 6l 7n 8p :t ;v <x =z > ?~ @ A B CHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH E F G H I J K L M N P Q R S HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHT U V +W +X +Y +[ +\ +] +^ +_ +` +a# +b& +cHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH) +d, +f2 +g5 +h8 +i; +j> +kA +lD +mG +nJ +oM +qS +rV +sHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHY +t\ +u_ +vb +we +xh +yk +zn +|t +}w +~z +} + + + +HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH + + + + + + + + + + + + +  +$ +HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH +( +* +, +. + +2 +4 +6 + +< +> +@ +B +HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH +F +H +J +L +P +R +T +V  Z \ + ^ `  HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHf h j " n% p( r+ t. 1 x4 |: ~= @ C HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHF I L O R U [ ^ a d g j m p HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHs v |             HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH              HHHHHHHHHHHHHHHHHHHH?HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH             HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHgreenblueredHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHgreenblueredHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHgreenblueredHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHgreenblueredHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHgreenblueredHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHgreenblueredHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHgreenblueredHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHgreenblueredHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHgreenblueredHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHgreenblueredHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHgreenblueredHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHgreenblueredHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHgreenblueredHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHgreenblueredHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHgreenblueredHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH@@@@@@@@@ @    +@ + + + @    @    @   @@@@@@@@@@@@@@@@@@ @   !@!!!"@"""#@###$@$$$%@%%%&@&&&'@'''(@((()@)))*@***+@+++,@,,,-@---.@.../@///0@0001@1112@2223@3334@4445@5556@6667@7778@8889@999:@:::;@;;;<@<<<=@===>@>>>?@???@@@@@A@AAAB@BBBC@CCCD@DDDE@EEEF@FFFG@GGGH@HHHI@IIIJ@JJJK@KKKL@LLLM@MMMN@NNNO@OOOP@PPPQ@QQQR@RRRS@SSST@TTTU@UUUV@VVVW@WWWX@XXXY@YYYZ@ZZZ[@[[[\@\\\]@]]]^@^^^_@___`@```a@aaab@bbbc@cccd@ddde@eeef@fffg@gggh@hhhi@iiij@jjjk@kkkl@lllm@mmmn@nnno@ooop@pppq@qqqr@rrrs@ssst@tttu@uuuv@vvvw@wwwx@xxxy@yyyz@zzz{@{{{|@|||}@}}}~@~~~@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@blob-1024-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1025-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1026-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1027-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1028-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1029-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1030-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1031-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1032-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1033-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1034-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1035-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1036-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1037-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1038-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1039-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1040-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1041-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1042-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1043-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1044-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1045-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1046-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1047-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1048-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1049-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1050-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1051-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1052-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1053-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1054-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1055-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1056-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1057-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1058-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1059-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1060-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1061-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1062-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1063-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1064-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1065-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1066-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1067-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1068-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1069-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1070-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1071-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1072-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1073-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1074-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1075-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1076-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1077-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1078-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1079-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1080-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1081-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1082-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1083-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1084-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1085-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1086-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1087-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1088-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1089-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1090-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1091-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1092-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1093-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1094-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1095-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1096-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1097-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1098-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1099-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1100-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1101-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1102-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1103-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1104-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1105-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1106-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1107-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1108-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1109-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1110-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1111-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1112-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1113-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1114-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1115-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1116-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1117-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1118-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1119-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1120-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1121-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1122-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1123-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1124-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1125-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1126-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1127-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1128-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1129-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1130-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1131-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1132-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1133-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1134-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1135-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1136-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1137-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1138-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1139-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1140-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1141-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1142-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1143-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1144-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1145-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1146-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1147-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1148-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1149-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1150-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1151-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1152-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1153-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1154-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1155-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1156-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1157-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1158-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1159-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1160-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1161-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1162-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1163-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1164-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1165-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1166-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1167-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1168-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1169-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1170-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1171-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1172-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1173-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1174-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1175-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1176-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1177-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1178-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1179-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1180-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1181-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1182-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1183-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1184-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1185-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1186-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1187-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1188-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1189-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1190-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1191-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1192-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1193-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1194-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1195-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1196-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1197-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1198-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1199-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1200-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1201-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1202-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1203-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1204-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1205-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1206-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1207-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1208-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1209-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1210-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1211-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1212-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1213-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1214-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1215-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1216-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1217-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1218-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1219-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1220-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1221-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1222-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1223-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1224-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1225-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1226-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1227-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1228-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1229-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1230-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1231-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1232-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1233-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1234-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1235-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1236-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1237-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1238-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1239-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1240-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1241-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1242-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1243-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1244-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1245-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1246-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1247-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1248-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1249-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1250-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1251-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1252-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1253-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1254-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1255-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1256-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1257-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1258-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1259-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1260-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1261-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1262-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1263-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1264-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1265-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1266-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1267-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1268-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1269-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1270-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1271-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1272-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1273-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1274-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1275-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1276-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1277-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1278-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1279-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1280-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1281-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1282-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1283-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1284-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1285-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1286-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1287-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1288-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1289-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1290-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1291-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1292-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1293-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1294-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1295-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1296-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1297-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1298-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1299-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1300-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1301-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1302-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1303-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1304-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1305-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1306-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1307-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1308-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1309-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1310-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1311-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1312-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1313-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1314-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1315-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1316-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1317-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1318-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1319-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1320-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1321-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1322-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1323-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1324-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1325-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1326-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1327-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1328-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1329-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1330-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1331-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1332-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1333-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1334-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1335-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1336-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1337-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1338-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1339-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1340-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1341-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1342-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1343-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1344-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1345-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1346-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1347-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1348-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1349-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1350-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1351-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1352-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1353-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1354-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1355-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1356-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1357-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1358-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1359-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1360-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1361-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1362-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1363-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1364-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1365-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1366-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1367-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1368-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1369-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1370-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1371-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1372-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1373-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1374-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1375-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1376-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1377-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1378-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1379-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1380-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1381-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1382-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1383-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1384-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1385-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1386-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1387-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1388-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1389-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1390-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1391-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1392-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1393-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1394-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1395-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1396-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1397-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1398-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1399-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1400-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1401-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1402-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1403-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1404-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1405-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1406-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1407-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1408-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1409-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1410-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1411-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1412-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1413-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1414-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1415-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1416-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1417-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1418-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1419-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1420-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1421-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1422-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1423-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1424-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1425-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1426-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1427-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1428-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1429-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1430-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1431-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1432-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1433-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1434-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1435-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1436-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1437-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1438-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1439-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1440-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1441-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1442-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1443-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1444-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1445-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1446-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1447-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1448-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1449-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1450-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1451-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1452-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1453-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1454-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1455-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1456-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1457-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1458-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1459-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1460-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1461-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1462-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1463-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1464-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1465-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1466-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1467-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1468-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1469-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1470-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1471-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1472-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1473-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1474-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1475-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1476-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1477-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1478-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1479-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1480-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1481-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1482-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1483-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1484-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1485-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1486-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1487-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1488-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1489-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1490-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1491-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1492-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1493-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1494-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1495-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1496-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1497-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1498-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1499-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1500-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1501-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1502-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1503-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1504-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1505-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1506-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1507-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1508-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1509-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1510-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1511-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1512-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1513-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1514-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1515-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1516-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1517-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1518-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1519-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1520-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1521-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1522-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1523-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1524-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1525-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1526-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1527-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1528-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1529-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1530-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1531-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1532-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1533-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1534-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1535-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1536-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1537-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1538-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1539-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1540-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1541-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1542-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1543-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1544-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1545-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1546-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1547-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1548-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1549-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1550-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1551-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1552-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1553-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1554-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1555-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1556-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1557-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1558-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1559-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1560-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1561-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1562-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1563-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1564-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1565-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1566-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1567-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1568-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1569-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1570-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1571-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1572-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1573-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1574-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1575-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1576-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1577-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1578-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1579-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1580-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1581-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1582-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1583-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1584-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1585-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1586-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1587-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1588-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1589-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1590-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1591-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1592-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1593-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1594-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1595-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1596-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1597-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1598-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1599-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1600-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1601-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1602-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1603-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1604-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1605-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1606-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1607-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1608-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1609-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1610-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1611-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1612-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1613-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1614-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1615-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1616-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1617-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1618-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1619-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1620-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1621-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1622-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1623-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1624-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1625-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1626-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1627-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1628-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1629-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1630-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1631-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1632-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1633-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1634-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1635-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1636-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1637-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1638-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1639-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1640-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1641-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1642-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1643-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1644-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1645-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1646-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1647-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1648-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1649-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1650-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1651-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1652-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1653-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1654-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1655-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1656-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1657-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1658-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1659-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1660-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1661-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1662-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1663-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1664-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1665-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1666-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1667-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1668-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1669-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1670-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1671-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1672-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1673-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1674-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1675-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1676-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1677-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1678-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1679-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1680-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1681-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1682-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1683-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1684-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1685-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1686-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1687-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1688-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1689-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1690-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1691-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1692-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1693-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1694-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1695-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1696-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1697-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1698-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1699-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1700-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1701-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1702-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1703-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1704-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1705-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1706-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1707-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1708-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1709-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1710-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1711-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1712-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1713-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1714-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1715-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1716-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1717-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1718-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1719-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1720-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1721-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1722-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1723-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1724-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1725-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1726-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1727-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1728-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1729-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1730-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1731-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1732-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1733-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1734-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1735-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1736-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1737-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1738-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1739-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1740-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1741-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1742-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1743-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1744-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1745-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1746-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1747-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1748-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1749-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1750-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1751-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1752-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1753-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1754-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1755-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1756-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1757-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1758-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1759-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1760-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1761-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1762-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1763-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1764-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1765-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1766-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1767-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1768-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1769-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1770-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1771-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1772-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1773-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1774-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1775-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1776-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1777-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1778-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1779-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1780-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1781-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1782-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1783-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1784-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1785-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1786-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1787-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1788-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1789-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1790-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1791-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1792-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1793-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1794-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1795-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1796-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1797-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1798-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1799-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1800-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1801-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1802-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1803-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1804-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1805-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1806-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1807-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1808-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1809-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1810-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1811-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1812-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1813-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1814-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1815-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1816-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1817-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1818-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1819-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1820-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1821-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1822-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1823-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1824-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1825-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1826-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1827-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1828-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1829-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1830-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1831-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1832-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1833-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1834-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1835-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1836-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1837-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1838-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1839-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1840-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1841-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1842-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1843-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1844-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1845-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1846-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1847-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1848-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1849-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1850-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1851-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1852-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1853-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1854-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1855-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1856-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1857-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1858-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1859-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1860-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1861-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1862-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1863-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1864-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1865-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1866-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1867-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1868-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1869-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1870-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1871-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1872-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1873-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1874-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1875-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1876-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1877-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1878-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1879-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1880-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1881-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1882-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1883-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1884-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1885-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1886-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1887-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1888-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1889-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1890-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1891-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1892-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1893-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1894-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1895-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1896-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1897-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1898-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1899-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1900-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1901-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1902-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1903-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1904-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1905-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1906-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1907-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1908-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1909-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1910-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1911-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1912-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1913-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1914-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1915-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1916-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1917-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1918-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1919-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1920-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1921-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1922-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1923-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1924-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1925-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1926-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1927-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1928-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1929-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1930-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1931-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1932-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1933-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1934-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1935-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1936-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1937-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1938-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1939-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1940-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1941-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1942-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1943-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1944-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1945-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1946-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1947-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1948-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1949-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1950-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1951-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1952-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1953-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1954-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1955-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1956-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1957-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1958-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1959-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1960-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1961-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1962-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1963-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1964-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1965-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1966-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1967-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1968-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1969-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1970-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1971-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1972-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1973-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1974-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1975-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1976-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1977-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1978-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1979-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1980-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1981-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1982-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1983-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1984-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1985-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1986-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1987-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1988-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1989-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1990-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1991-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1992-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1993-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1994-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1995-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1996-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1997-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1998-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1999-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2000-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2001-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2002-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2003-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2004-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2005-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2006-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2007-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2008-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2009-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2010-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2011-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2012-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2013-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2014-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2015-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2016-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2017-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2018-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2019-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2020-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2021-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2022-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2023-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2024-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2025-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2026-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2027-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2028-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2029-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2030-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2031-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2032-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2033-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2034-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2035-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2036-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2037-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2038-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2039-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2040-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2041-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2042-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2043-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2044-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2045-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2046-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2047-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH  +    !"#$%&'()*+,-./0123456789:;<HHHHHHHHHHHH=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\]^_`abcdefghijklmnopqrstuvwxyHHHHHHHHHHHHz{|}~HHHHHHHHHHHHHHHHHHHHHHHH  +    !"#$%&'()*+,-./0HHHHHHHHHHHH123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\]^_`abcdefghijklmHHHHHHHHHHHHnopqrstuvwxyz{|}~HHHHHHHHHHHHHHHHHHHHHHHH  +    !"#$HHHHHHHHHHHH%&'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\]^_`aHHHHHHHHHHHHbcdefghijklmnopqrstuvwxyz{|}~HHHHHHHHHHHHHHHHHHHHHHHH  +   HHHHHHHHHHHH !"#$%&'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUHHHHHHHHHHHHVWXYZ[\]^_`abcdefghijklmnopqrstuvwxyz{|}~HHHHHHHHHHHHHHHHHHHHHHHH @`value-1024-deterministic-fixturevalue-1025-deterministic-fixturevalue-1026-deterministic-fixturevalue-1027-deterministic-fixturevalue-1028-deterministic-fixturevalue-1030-deterministic-fixturevalue-1031-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`avalue-1032-deterministic-fixturevalue-1033-deterministic-fixturevalue-1034-deterministic-fixturevalue-1035-deterministic-fixturevalue-1037-deterministic-fixturevalue-1038-deterministic-fixturevalue-1039-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`Avalue-1040-deterministic-fixturevalue-1041-deterministic-fixturevalue-1042-deterministic-fixturevalue-1044-deterministic-fixturevalue-1045-deterministic-fixturevalue-1046-deterministic-fixturevalue-1047-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @!`value-1048-deterministic-fixturevalue-1049-deterministic-fixturevalue-1051-deterministic-fixturevalue-1052-deterministic-fixturevalue-1053-deterministic-fixturevalue-1054-deterministic-fixturevalue-1055-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-1056-deterministic-fixturevalue-1058-deterministic-fixturevalue-1059-deterministic-fixturevalue-1060-deterministic-fixturevalue-1061-deterministic-fixturevalue-1062-deterministic-fixturevalue-1063-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-1065-deterministic-fixturevalue-1066-deterministic-fixturevalue-1067-deterministic-fixturevalue-1068-deterministic-fixturevalue-1069-deterministic-fixturevalue-1070-deterministic-fixture @`value-1072-deterministic-fixturevalue-1073-deterministic-fixturevalue-1074-deterministic-fixturevalue-1075-deterministic-fixturevalue-1076-deterministic-fixturevalue-1077-deterministic-fixturevalue-1079-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-1080-deterministic-fixturevalue-1081-deterministic-fixturevalue-1082-deterministic-fixturevalue-1083-deterministic-fixturevalue-1084-deterministic-fixturevalue-1086-deterministic-fixturevalue-1087-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`avalue-1088-deterministic-fixturevalue-1089-deterministic-fixturevalue-1090-deterministic-fixturevalue-1091-deterministic-fixturevalue-1093-deterministic-fixturevalue-1094-deterministic-fixturevalue-1095-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`Avalue-1096-deterministic-fixturevalue-1097-deterministic-fixturevalue-1098-deterministic-fixturevalue-1100-deterministic-fixturevalue-1101-deterministic-fixturevalue-1102-deterministic-fixturevalue-1103-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @!`value-1104-deterministic-fixturevalue-1105-deterministic-fixturevalue-1107-deterministic-fixturevalue-1108-deterministic-fixturevalue-1109-deterministic-fixturevalue-1110-deterministic-fixturevalue-1111-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-1112-deterministic-fixturevalue-1114-deterministic-fixturevalue-1115-deterministic-fixturevalue-1116-deterministic-fixturevalue-1117-deterministic-fixturevalue-1118-deterministic-fixturevalue-1119-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-1121-deterministic-fixturevalue-1122-deterministic-fixturevalue-1123-deterministic-fixturevalue-1124-deterministic-fixturevalue-1125-deterministic-fixturevalue-1126-deterministic-fixture @`value-1128-deterministic-fixturevalue-1129-deterministic-fixturevalue-1130-deterministic-fixturevalue-1131-deterministic-fixturevalue-1132-deterministic-fixturevalue-1133-deterministic-fixturevalue-1135-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-1136-deterministic-fixturevalue-1137-deterministic-fixturevalue-1138-deterministic-fixturevalue-1139-deterministic-fixturevalue-1140-deterministic-fixturevalue-1142-deterministic-fixturevalue-1143-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`avalue-1144-deterministic-fixturevalue-1145-deterministic-fixturevalue-1146-deterministic-fixturevalue-1147-deterministic-fixturevalue-1149-deterministic-fixturevalue-1150-deterministic-fixturevalue-1151-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`Avalue-1152-deterministic-fixturevalue-1153-deterministic-fixturevalue-1154-deterministic-fixturevalue-1156-deterministic-fixturevalue-1157-deterministic-fixturevalue-1158-deterministic-fixturevalue-1159-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @!`value-1160-deterministic-fixturevalue-1161-deterministic-fixturevalue-1163-deterministic-fixturevalue-1164-deterministic-fixturevalue-1165-deterministic-fixturevalue-1166-deterministic-fixturevalue-1167-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-1168-deterministic-fixturevalue-1170-deterministic-fixturevalue-1171-deterministic-fixturevalue-1172-deterministic-fixturevalue-1173-deterministic-fixturevalue-1174-deterministic-fixturevalue-1175-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-1177-deterministic-fixturevalue-1178-deterministic-fixturevalue-1179-deterministic-fixturevalue-1180-deterministic-fixturevalue-1181-deterministic-fixturevalue-1182-deterministic-fixture @`value-1184-deterministic-fixturevalue-1185-deterministic-fixturevalue-1186-deterministic-fixturevalue-1187-deterministic-fixturevalue-1188-deterministic-fixturevalue-1189-deterministic-fixturevalue-1191-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-1192-deterministic-fixturevalue-1193-deterministic-fixturevalue-1194-deterministic-fixturevalue-1195-deterministic-fixturevalue-1196-deterministic-fixturevalue-1198-deterministic-fixturevalue-1199-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`avalue-1200-deterministic-fixturevalue-1201-deterministic-fixturevalue-1202-deterministic-fixturevalue-1203-deterministic-fixturevalue-1205-deterministic-fixturevalue-1206-deterministic-fixturevalue-1207-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`Avalue-1208-deterministic-fixturevalue-1209-deterministic-fixturevalue-1210-deterministic-fixturevalue-1212-deterministic-fixturevalue-1213-deterministic-fixturevalue-1214-deterministic-fixturevalue-1215-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @!`value-1216-deterministic-fixturevalue-1217-deterministic-fixturevalue-1219-deterministic-fixturevalue-1220-deterministic-fixturevalue-1221-deterministic-fixturevalue-1222-deterministic-fixturevalue-1223-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-1224-deterministic-fixturevalue-1226-deterministic-fixturevalue-1227-deterministic-fixturevalue-1228-deterministic-fixturevalue-1229-deterministic-fixturevalue-1230-deterministic-fixturevalue-1231-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-1233-deterministic-fixturevalue-1234-deterministic-fixturevalue-1235-deterministic-fixturevalue-1236-deterministic-fixturevalue-1237-deterministic-fixturevalue-1238-deterministic-fixture @`value-1240-deterministic-fixturevalue-1241-deterministic-fixturevalue-1242-deterministic-fixturevalue-1243-deterministic-fixturevalue-1244-deterministic-fixturevalue-1245-deterministic-fixturevalue-1247-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-1248-deterministic-fixturevalue-1249-deterministic-fixturevalue-1250-deterministic-fixturevalue-1251-deterministic-fixturevalue-1252-deterministic-fixturevalue-1254-deterministic-fixturevalue-1255-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`avalue-1256-deterministic-fixturevalue-1257-deterministic-fixturevalue-1258-deterministic-fixturevalue-1259-deterministic-fixturevalue-1261-deterministic-fixturevalue-1262-deterministic-fixturevalue-1263-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`Avalue-1264-deterministic-fixturevalue-1265-deterministic-fixturevalue-1266-deterministic-fixturevalue-1268-deterministic-fixturevalue-1269-deterministic-fixturevalue-1270-deterministic-fixturevalue-1271-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @!`value-1272-deterministic-fixturevalue-1273-deterministic-fixturevalue-1275-deterministic-fixturevalue-1276-deterministic-fixturevalue-1277-deterministic-fixturevalue-1278-deterministic-fixturevalue-1279-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-1280-deterministic-fixturevalue-1282-deterministic-fixturevalue-1283-deterministic-fixturevalue-1284-deterministic-fixturevalue-1285-deterministic-fixturevalue-1286-deterministic-fixturevalue-1287-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-1289-deterministic-fixturevalue-1290-deterministic-fixturevalue-1291-deterministic-fixturevalue-1292-deterministic-fixturevalue-1293-deterministic-fixturevalue-1294-deterministic-fixture @`value-1296-deterministic-fixturevalue-1297-deterministic-fixturevalue-1298-deterministic-fixturevalue-1299-deterministic-fixturevalue-1300-deterministic-fixturevalue-1301-deterministic-fixturevalue-1303-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-1304-deterministic-fixturevalue-1305-deterministic-fixturevalue-1306-deterministic-fixturevalue-1307-deterministic-fixturevalue-1308-deterministic-fixturevalue-1310-deterministic-fixturevalue-1311-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`avalue-1312-deterministic-fixturevalue-1313-deterministic-fixturevalue-1314-deterministic-fixturevalue-1315-deterministic-fixturevalue-1317-deterministic-fixturevalue-1318-deterministic-fixturevalue-1319-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`Avalue-1320-deterministic-fixturevalue-1321-deterministic-fixturevalue-1322-deterministic-fixturevalue-1324-deterministic-fixturevalue-1325-deterministic-fixturevalue-1326-deterministic-fixturevalue-1327-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @!`value-1328-deterministic-fixturevalue-1329-deterministic-fixturevalue-1331-deterministic-fixturevalue-1332-deterministic-fixturevalue-1333-deterministic-fixturevalue-1334-deterministic-fixturevalue-1335-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-1336-deterministic-fixturevalue-1338-deterministic-fixturevalue-1339-deterministic-fixturevalue-1340-deterministic-fixturevalue-1341-deterministic-fixturevalue-1342-deterministic-fixturevalue-1343-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-1345-deterministic-fixturevalue-1346-deterministic-fixturevalue-1347-deterministic-fixturevalue-1348-deterministic-fixturevalue-1349-deterministic-fixturevalue-1350-deterministic-fixture @`value-1352-deterministic-fixturevalue-1353-deterministic-fixturevalue-1354-deterministic-fixturevalue-1355-deterministic-fixturevalue-1356-deterministic-fixturevalue-1357-deterministic-fixturevalue-1359-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-1360-deterministic-fixturevalue-1361-deterministic-fixturevalue-1362-deterministic-fixturevalue-1363-deterministic-fixturevalue-1364-deterministic-fixturevalue-1366-deterministic-fixturevalue-1367-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`avalue-1368-deterministic-fixturevalue-1369-deterministic-fixturevalue-1370-deterministic-fixturevalue-1371-deterministic-fixturevalue-1373-deterministic-fixturevalue-1374-deterministic-fixturevalue-1375-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`Avalue-1376-deterministic-fixturevalue-1377-deterministic-fixturevalue-1378-deterministic-fixturevalue-1380-deterministic-fixturevalue-1381-deterministic-fixturevalue-1382-deterministic-fixturevalue-1383-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @!`value-1384-deterministic-fixturevalue-1385-deterministic-fixturevalue-1387-deterministic-fixturevalue-1388-deterministic-fixturevalue-1389-deterministic-fixturevalue-1390-deterministic-fixturevalue-1391-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-1392-deterministic-fixturevalue-1394-deterministic-fixturevalue-1395-deterministic-fixturevalue-1396-deterministic-fixturevalue-1397-deterministic-fixturevalue-1398-deterministic-fixturevalue-1399-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-1401-deterministic-fixturevalue-1402-deterministic-fixturevalue-1403-deterministic-fixturevalue-1404-deterministic-fixturevalue-1405-deterministic-fixturevalue-1406-deterministic-fixture @`value-1408-deterministic-fixturevalue-1409-deterministic-fixturevalue-1410-deterministic-fixturevalue-1411-deterministic-fixturevalue-1412-deterministic-fixturevalue-1413-deterministic-fixturevalue-1415-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-1416-deterministic-fixturevalue-1417-deterministic-fixturevalue-1418-deterministic-fixturevalue-1419-deterministic-fixturevalue-1420-deterministic-fixturevalue-1422-deterministic-fixturevalue-1423-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`avalue-1424-deterministic-fixturevalue-1425-deterministic-fixturevalue-1426-deterministic-fixturevalue-1427-deterministic-fixturevalue-1429-deterministic-fixturevalue-1430-deterministic-fixturevalue-1431-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`Avalue-1432-deterministic-fixturevalue-1433-deterministic-fixturevalue-1434-deterministic-fixturevalue-1436-deterministic-fixturevalue-1437-deterministic-fixturevalue-1438-deterministic-fixturevalue-1439-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @!`value-1440-deterministic-fixturevalue-1441-deterministic-fixturevalue-1443-deterministic-fixturevalue-1444-deterministic-fixturevalue-1445-deterministic-fixturevalue-1446-deterministic-fixturevalue-1447-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-1448-deterministic-fixturevalue-1450-deterministic-fixturevalue-1451-deterministic-fixturevalue-1452-deterministic-fixturevalue-1453-deterministic-fixturevalue-1454-deterministic-fixturevalue-1455-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-1457-deterministic-fixturevalue-1458-deterministic-fixturevalue-1459-deterministic-fixturevalue-1460-deterministic-fixturevalue-1461-deterministic-fixturevalue-1462-deterministic-fixture @`value-1464-deterministic-fixturevalue-1465-deterministic-fixturevalue-1466-deterministic-fixturevalue-1467-deterministic-fixturevalue-1468-deterministic-fixturevalue-1469-deterministic-fixturevalue-1471-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-1472-deterministic-fixturevalue-1473-deterministic-fixturevalue-1474-deterministic-fixturevalue-1475-deterministic-fixturevalue-1476-deterministic-fixturevalue-1478-deterministic-fixturevalue-1479-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`avalue-1480-deterministic-fixturevalue-1481-deterministic-fixturevalue-1482-deterministic-fixturevalue-1483-deterministic-fixturevalue-1485-deterministic-fixturevalue-1486-deterministic-fixturevalue-1487-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`Avalue-1488-deterministic-fixturevalue-1489-deterministic-fixturevalue-1490-deterministic-fixturevalue-1492-deterministic-fixturevalue-1493-deterministic-fixturevalue-1494-deterministic-fixturevalue-1495-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @!`value-1496-deterministic-fixturevalue-1497-deterministic-fixturevalue-1499-deterministic-fixturevalue-1500-deterministic-fixturevalue-1501-deterministic-fixturevalue-1502-deterministic-fixturevalue-1503-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-1504-deterministic-fixturevalue-1506-deterministic-fixturevalue-1507-deterministic-fixturevalue-1508-deterministic-fixturevalue-1509-deterministic-fixturevalue-1510-deterministic-fixturevalue-1511-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-1513-deterministic-fixturevalue-1514-deterministic-fixturevalue-1515-deterministic-fixturevalue-1516-deterministic-fixturevalue-1517-deterministic-fixturevalue-1518-deterministic-fixture @`value-1520-deterministic-fixturevalue-1521-deterministic-fixturevalue-1522-deterministic-fixturevalue-1523-deterministic-fixturevalue-1524-deterministic-fixturevalue-1525-deterministic-fixturevalue-1527-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-1528-deterministic-fixturevalue-1529-deterministic-fixturevalue-1530-deterministic-fixturevalue-1531-deterministic-fixturevalue-1532-deterministic-fixturevalue-1534-deterministic-fixturevalue-1535-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`avalue-1536-deterministic-fixturevalue-1537-deterministic-fixturevalue-1538-deterministic-fixturevalue-1539-deterministic-fixturevalue-1541-deterministic-fixturevalue-1542-deterministic-fixturevalue-1543-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`Avalue-1544-deterministic-fixturevalue-1545-deterministic-fixturevalue-1546-deterministic-fixturevalue-1548-deterministic-fixturevalue-1549-deterministic-fixturevalue-1550-deterministic-fixturevalue-1551-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @!`value-1552-deterministic-fixturevalue-1553-deterministic-fixturevalue-1555-deterministic-fixturevalue-1556-deterministic-fixturevalue-1557-deterministic-fixturevalue-1558-deterministic-fixturevalue-1559-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-1560-deterministic-fixturevalue-1562-deterministic-fixturevalue-1563-deterministic-fixturevalue-1564-deterministic-fixturevalue-1565-deterministic-fixturevalue-1566-deterministic-fixturevalue-1567-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-1569-deterministic-fixturevalue-1570-deterministic-fixturevalue-1571-deterministic-fixturevalue-1572-deterministic-fixturevalue-1573-deterministic-fixturevalue-1574-deterministic-fixture @`value-1576-deterministic-fixturevalue-1577-deterministic-fixturevalue-1578-deterministic-fixturevalue-1579-deterministic-fixturevalue-1580-deterministic-fixturevalue-1581-deterministic-fixturevalue-1583-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-1584-deterministic-fixturevalue-1585-deterministic-fixturevalue-1586-deterministic-fixturevalue-1587-deterministic-fixturevalue-1588-deterministic-fixturevalue-1590-deterministic-fixturevalue-1591-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`avalue-1592-deterministic-fixturevalue-1593-deterministic-fixturevalue-1594-deterministic-fixturevalue-1595-deterministic-fixturevalue-1597-deterministic-fixturevalue-1598-deterministic-fixturevalue-1599-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`Avalue-1600-deterministic-fixturevalue-1601-deterministic-fixturevalue-1602-deterministic-fixturevalue-1604-deterministic-fixturevalue-1605-deterministic-fixturevalue-1606-deterministic-fixturevalue-1607-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @!`value-1608-deterministic-fixturevalue-1609-deterministic-fixturevalue-1611-deterministic-fixturevalue-1612-deterministic-fixturevalue-1613-deterministic-fixturevalue-1614-deterministic-fixturevalue-1615-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-1616-deterministic-fixturevalue-1618-deterministic-fixturevalue-1619-deterministic-fixturevalue-1620-deterministic-fixturevalue-1621-deterministic-fixturevalue-1622-deterministic-fixturevalue-1623-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-1625-deterministic-fixturevalue-1626-deterministic-fixturevalue-1627-deterministic-fixturevalue-1628-deterministic-fixturevalue-1629-deterministic-fixturevalue-1630-deterministic-fixture @`value-1632-deterministic-fixturevalue-1633-deterministic-fixturevalue-1634-deterministic-fixturevalue-1635-deterministic-fixturevalue-1636-deterministic-fixturevalue-1637-deterministic-fixturevalue-1639-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-1640-deterministic-fixturevalue-1641-deterministic-fixturevalue-1642-deterministic-fixturevalue-1643-deterministic-fixturevalue-1644-deterministic-fixturevalue-1646-deterministic-fixturevalue-1647-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`avalue-1648-deterministic-fixturevalue-1649-deterministic-fixturevalue-1650-deterministic-fixturevalue-1651-deterministic-fixturevalue-1653-deterministic-fixturevalue-1654-deterministic-fixturevalue-1655-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`Avalue-1656-deterministic-fixturevalue-1657-deterministic-fixturevalue-1658-deterministic-fixturevalue-1660-deterministic-fixturevalue-1661-deterministic-fixturevalue-1662-deterministic-fixturevalue-1663-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @!`value-1664-deterministic-fixturevalue-1665-deterministic-fixturevalue-1667-deterministic-fixturevalue-1668-deterministic-fixturevalue-1669-deterministic-fixturevalue-1670-deterministic-fixturevalue-1671-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-1672-deterministic-fixturevalue-1674-deterministic-fixturevalue-1675-deterministic-fixturevalue-1676-deterministic-fixturevalue-1677-deterministic-fixturevalue-1678-deterministic-fixturevalue-1679-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-1681-deterministic-fixturevalue-1682-deterministic-fixturevalue-1683-deterministic-fixturevalue-1684-deterministic-fixturevalue-1685-deterministic-fixturevalue-1686-deterministic-fixture @`value-1688-deterministic-fixturevalue-1689-deterministic-fixturevalue-1690-deterministic-fixturevalue-1691-deterministic-fixturevalue-1692-deterministic-fixturevalue-1693-deterministic-fixturevalue-1695-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-1696-deterministic-fixturevalue-1697-deterministic-fixturevalue-1698-deterministic-fixturevalue-1699-deterministic-fixturevalue-1700-deterministic-fixturevalue-1702-deterministic-fixturevalue-1703-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`avalue-1704-deterministic-fixturevalue-1705-deterministic-fixturevalue-1706-deterministic-fixturevalue-1707-deterministic-fixturevalue-1709-deterministic-fixturevalue-1710-deterministic-fixturevalue-1711-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`Avalue-1712-deterministic-fixturevalue-1713-deterministic-fixturevalue-1714-deterministic-fixturevalue-1716-deterministic-fixturevalue-1717-deterministic-fixturevalue-1718-deterministic-fixturevalue-1719-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @!`value-1720-deterministic-fixturevalue-1721-deterministic-fixturevalue-1723-deterministic-fixturevalue-1724-deterministic-fixturevalue-1725-deterministic-fixturevalue-1726-deterministic-fixturevalue-1727-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-1728-deterministic-fixturevalue-1730-deterministic-fixturevalue-1731-deterministic-fixturevalue-1732-deterministic-fixturevalue-1733-deterministic-fixturevalue-1734-deterministic-fixturevalue-1735-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-1737-deterministic-fixturevalue-1738-deterministic-fixturevalue-1739-deterministic-fixturevalue-1740-deterministic-fixturevalue-1741-deterministic-fixturevalue-1742-deterministic-fixture @`value-1744-deterministic-fixturevalue-1745-deterministic-fixturevalue-1746-deterministic-fixturevalue-1747-deterministic-fixturevalue-1748-deterministic-fixturevalue-1749-deterministic-fixturevalue-1751-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-1752-deterministic-fixturevalue-1753-deterministic-fixturevalue-1754-deterministic-fixturevalue-1755-deterministic-fixturevalue-1756-deterministic-fixturevalue-1758-deterministic-fixturevalue-1759-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`avalue-1760-deterministic-fixturevalue-1761-deterministic-fixturevalue-1762-deterministic-fixturevalue-1763-deterministic-fixturevalue-1765-deterministic-fixturevalue-1766-deterministic-fixturevalue-1767-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`Avalue-1768-deterministic-fixturevalue-1769-deterministic-fixturevalue-1770-deterministic-fixturevalue-1772-deterministic-fixturevalue-1773-deterministic-fixturevalue-1774-deterministic-fixturevalue-1775-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @!`value-1776-deterministic-fixturevalue-1777-deterministic-fixturevalue-1779-deterministic-fixturevalue-1780-deterministic-fixturevalue-1781-deterministic-fixturevalue-1782-deterministic-fixturevalue-1783-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-1784-deterministic-fixturevalue-1786-deterministic-fixturevalue-1787-deterministic-fixturevalue-1788-deterministic-fixturevalue-1789-deterministic-fixturevalue-1790-deterministic-fixturevalue-1791-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-1793-deterministic-fixturevalue-1794-deterministic-fixturevalue-1795-deterministic-fixturevalue-1796-deterministic-fixturevalue-1797-deterministic-fixturevalue-1798-deterministic-fixture @`value-1800-deterministic-fixturevalue-1801-deterministic-fixturevalue-1802-deterministic-fixturevalue-1803-deterministic-fixturevalue-1804-deterministic-fixturevalue-1805-deterministic-fixturevalue-1807-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-1808-deterministic-fixturevalue-1809-deterministic-fixturevalue-1810-deterministic-fixturevalue-1811-deterministic-fixturevalue-1812-deterministic-fixturevalue-1814-deterministic-fixturevalue-1815-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`avalue-1816-deterministic-fixturevalue-1817-deterministic-fixturevalue-1818-deterministic-fixturevalue-1819-deterministic-fixturevalue-1821-deterministic-fixturevalue-1822-deterministic-fixturevalue-1823-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`Avalue-1824-deterministic-fixturevalue-1825-deterministic-fixturevalue-1826-deterministic-fixturevalue-1828-deterministic-fixturevalue-1829-deterministic-fixturevalue-1830-deterministic-fixturevalue-1831-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @!`value-1832-deterministic-fixturevalue-1833-deterministic-fixturevalue-1835-deterministic-fixturevalue-1836-deterministic-fixturevalue-1837-deterministic-fixturevalue-1838-deterministic-fixturevalue-1839-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-1840-deterministic-fixturevalue-1842-deterministic-fixturevalue-1843-deterministic-fixturevalue-1844-deterministic-fixturevalue-1845-deterministic-fixturevalue-1846-deterministic-fixturevalue-1847-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-1849-deterministic-fixturevalue-1850-deterministic-fixturevalue-1851-deterministic-fixturevalue-1852-deterministic-fixturevalue-1853-deterministic-fixturevalue-1854-deterministic-fixture @`value-1856-deterministic-fixturevalue-1857-deterministic-fixturevalue-1858-deterministic-fixturevalue-1859-deterministic-fixturevalue-1860-deterministic-fixturevalue-1861-deterministic-fixturevalue-1863-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-1864-deterministic-fixturevalue-1865-deterministic-fixturevalue-1866-deterministic-fixturevalue-1867-deterministic-fixturevalue-1868-deterministic-fixturevalue-1870-deterministic-fixturevalue-1871-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`avalue-1872-deterministic-fixturevalue-1873-deterministic-fixturevalue-1874-deterministic-fixturevalue-1875-deterministic-fixturevalue-1877-deterministic-fixturevalue-1878-deterministic-fixturevalue-1879-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`Avalue-1880-deterministic-fixturevalue-1881-deterministic-fixturevalue-1882-deterministic-fixturevalue-1884-deterministic-fixturevalue-1885-deterministic-fixturevalue-1886-deterministic-fixturevalue-1887-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @!`value-1888-deterministic-fixturevalue-1889-deterministic-fixturevalue-1891-deterministic-fixturevalue-1892-deterministic-fixturevalue-1893-deterministic-fixturevalue-1894-deterministic-fixturevalue-1895-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-1896-deterministic-fixturevalue-1898-deterministic-fixturevalue-1899-deterministic-fixturevalue-1900-deterministic-fixturevalue-1901-deterministic-fixturevalue-1902-deterministic-fixturevalue-1903-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-1905-deterministic-fixturevalue-1906-deterministic-fixturevalue-1907-deterministic-fixturevalue-1908-deterministic-fixturevalue-1909-deterministic-fixturevalue-1910-deterministic-fixture @`value-1912-deterministic-fixturevalue-1913-deterministic-fixturevalue-1914-deterministic-fixturevalue-1915-deterministic-fixturevalue-1916-deterministic-fixturevalue-1917-deterministic-fixturevalue-1919-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-1920-deterministic-fixturevalue-1921-deterministic-fixturevalue-1922-deterministic-fixturevalue-1923-deterministic-fixturevalue-1924-deterministic-fixturevalue-1926-deterministic-fixturevalue-1927-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`avalue-1928-deterministic-fixturevalue-1929-deterministic-fixturevalue-1930-deterministic-fixturevalue-1931-deterministic-fixturevalue-1933-deterministic-fixturevalue-1934-deterministic-fixturevalue-1935-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`Avalue-1936-deterministic-fixturevalue-1937-deterministic-fixturevalue-1938-deterministic-fixturevalue-1940-deterministic-fixturevalue-1941-deterministic-fixturevalue-1942-deterministic-fixturevalue-1943-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @!`value-1944-deterministic-fixturevalue-1945-deterministic-fixturevalue-1947-deterministic-fixturevalue-1948-deterministic-fixturevalue-1949-deterministic-fixturevalue-1950-deterministic-fixturevalue-1951-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-1952-deterministic-fixturevalue-1954-deterministic-fixturevalue-1955-deterministic-fixturevalue-1956-deterministic-fixturevalue-1957-deterministic-fixturevalue-1958-deterministic-fixturevalue-1959-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-1961-deterministic-fixturevalue-1962-deterministic-fixturevalue-1963-deterministic-fixturevalue-1964-deterministic-fixturevalue-1965-deterministic-fixturevalue-1966-deterministic-fixture @`value-1968-deterministic-fixturevalue-1969-deterministic-fixturevalue-1970-deterministic-fixturevalue-1971-deterministic-fixturevalue-1972-deterministic-fixturevalue-1973-deterministic-fixturevalue-1975-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-1976-deterministic-fixturevalue-1977-deterministic-fixturevalue-1978-deterministic-fixturevalue-1979-deterministic-fixturevalue-1980-deterministic-fixturevalue-1982-deterministic-fixturevalue-1983-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`avalue-1984-deterministic-fixturevalue-1985-deterministic-fixturevalue-1986-deterministic-fixturevalue-1987-deterministic-fixturevalue-1989-deterministic-fixturevalue-1990-deterministic-fixturevalue-1991-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`Avalue-1992-deterministic-fixturevalue-1993-deterministic-fixturevalue-1994-deterministic-fixturevalue-1996-deterministic-fixturevalue-1997-deterministic-fixturevalue-1998-deterministic-fixturevalue-1999-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @!`value-2000-deterministic-fixturevalue-2001-deterministic-fixturevalue-2003-deterministic-fixturevalue-2004-deterministic-fixturevalue-2005-deterministic-fixturevalue-2006-deterministic-fixturevalue-2007-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-2008-deterministic-fixturevalue-2010-deterministic-fixturevalue-2011-deterministic-fixturevalue-2012-deterministic-fixturevalue-2013-deterministic-fixturevalue-2014-deterministic-fixturevalue-2015-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-2017-deterministic-fixturevalue-2018-deterministic-fixturevalue-2019-deterministic-fixturevalue-2020-deterministic-fixturevalue-2021-deterministic-fixturevalue-2022-deterministic-fixture @`value-2024-deterministic-fixturevalue-2025-deterministic-fixturevalue-2026-deterministic-fixturevalue-2027-deterministic-fixturevalue-2028-deterministic-fixturevalue-2029-deterministic-fixturevalue-2031-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-2032-deterministic-fixturevalue-2033-deterministic-fixturevalue-2034-deterministic-fixturevalue-2035-deterministic-fixturevalue-2036-deterministic-fixturevalue-2038-deterministic-fixturevalue-2039-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`avalue-2040-deterministic-fixturevalue-2041-deterministic-fixturevalue-2042-deterministic-fixturevalue-2043-deterministic-fixturevalue-2045-deterministic-fixturevalue-2046-deterministic-fixturevalue-2047-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH  !$'*-0369<& ?BEHKNQTWZD ]`cfiloruxb {~    #&), /258;>ADGJ4 MPSVY\_behR knqtwz}p      +  "%(+.147:$ =@CFILORUXB [^adgjmpsv` y|~       !$'*-0369<?BEH2KNQTWZ]`cfPilorux{~n  #&),/258";>ADGJMPSV@Y\_behknqt^wz}| + "%(+.147:=@CF0ILORUX[^adNgjmpsvy|l  !$'*-036 9<?BEHKNQT>WZ]`cfilor\ux{~z  #&),/258;>AD.GJMPSVY\_bLehknqtwz}j + "%(+.147:=@CFILOR<UX[^adgjmpZsvy|x          ! $ ' * - 0 3 6 9 < ? B ,E H K N Q T W Z ] ` Jc f i l o r u x { ~ h      + + + + + + + + + + +# +& +) +, +/ +2 +5 +8 +; +> +A +D +G +J +M +P +:S +V +Y +\ +_ +b +e +h +k +n +Xq +t +w +z +} + + + + + +v + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH      +     ! $ ' * HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH- 0 "3 $6 &9 (< ,B .E 0H 2K N 6Q 8T :W <HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHZ ] !Bc "Df #Fi $l %Jo &Lr 'Nu (Px ){ *T~ ,X -Z . HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH/^ 0` 1b 2d 3 4h 5j 7n 8 9r :t ;v <x = >HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH| ?~ @ B C D E F G H I J K M NHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH O P Q R S T U V X Y Z [ \ ] HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH^ _ ` a# c) d, e/ f2 g5 h8 i; j> kA lD nHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHJ oM pP qS rV sY t\ u_ vb we yk zn {q |t }HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHw ~z }      +        HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH   "  & ( * ,  0 4 6  HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH: < > @  D F J  N P R T X HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHZ \ +b d f h l "n %p (r +v 1x 4HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHz 7| := @ C F IL R U X[ ^ aHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH d gj m s vy |      HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH           HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH            HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH    + + +  +  + +  + +  +HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH!  +$  +' +*- +0" +3& +9<* +?, +B. +E0 +HK4 +NHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH6 +Q8 +TZ> +] @ +`!B +c"D +f#i$H +l%J +o&L +r'N +u)R +{*T +~+HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHV +,X +-.\ +/^ +0` +1b +24h +5j +6l +78p +9r +:t +HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH;v +<=z +?~ +@ +AB +C +D +E +FG +H +J +HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHKL +M +N +O +PQ +R +S +UV +W +X +Y + ZHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH[ +\ +] +^ +` + a +#b +&c +)d,e +/f +2g +5h +8iHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH;k +Al +Dm +GnJo +Mp +Pq +Sr +VsYt +\v +bw +exhy +kHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHz +n{ +q| +t}w~ +z +}    +   HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH      $ & ( * . 0 2 HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH4 8 : < > D F H L N P HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHR V Z \ + ` b d f j l "p (HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH+t .v 1x 4z 7:~ = @ CI L O R UHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHX [ ^ a d j m p sv y |  HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH           HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH            HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH          +   HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH   +   !  $  '* 0" 3$ 69( <* ?, BHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH. EH2 K6 Q8 TW< Z> ] @ `!B c"f#F i$H l&L r'HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHu(P x)R {*T ~+V ,-Z .\ /^ 12d 3f 4h 5j 6HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH7n 8p 9r :t <x =z >| ?~ @A B C D EGHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH H I JK L M N OP R S TU V HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHW X Y Z [ ] ^_ ` a #b &c)d ,e /HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHf 2h8i ;j >k Al DmGn Jo Mp Pq Ss Yt \u _vHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH bwex hy kz n{ q|t~ z }     + HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH       " $ ( HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH, . 0 2 6 8 : @ B D F HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHJ L N P T V X Z  +^ ` b d j HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHl "n %(r +t .v 1x 47| : @ CF I LHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH O RU X [ ad g j m ps v yHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH |            HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH           HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH         HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH  +   + ! '*- 0"36&HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH9(<*?,B0H2K4N6QT:W<Z>] @`!c#FiHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH$Hl%Jo&r'Nu(Px)R{*T~+,X.\/^01b2d3HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHf4h56l7n9r:;v<x=z>|?@ABHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHDEFGHIJKLMOPQRHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHSTUVWXZ[\]^_` a#bHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH&c)e/f2g5h8i;j>kAlDmGnJpPqSrHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHVsYt\u_vbwexhyk{q|t}w~z}HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH "HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH$&*,.048:>@BHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHDHJNRTVX\ +^ `HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHfhjl"%p(r+t.v1z7|:~=@HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHCFILORX[^adgjmHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHpsy|HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHgreenblueredHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHgreenblueredHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHgreenblueredHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHgreenblueredHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHgreenblueredHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHgreenblueredHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHgreenblueredHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHgreenblueredHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHgreenblueredHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHgreenblueredHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHgreenblueredHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHgreenblueredHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHgreenblueredHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHgreenblueredHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHgreenblueredHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH[\@\\\]@]]]^@^^^_@___`@```a@aaab@bbbc@cccd@ddde@eeef@fffg@gggh@hhhi@iiij@jjjk@kkkl@lllm@mmmn@nnno@ooop@pppq@qqqr@rrrs@ssst@tttu@uuuv@vvvw@wwwx@xxxy@yyyz@zzz{@{{{|@|||}@}}}~@~~~@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ @    +@ + + + @    @    @   @@@@@@@@@@@@@@@@@@ @   !@!!!"@"""#@###$@$$$%@%%%&@&&&'@'''(@((()@)))*@***+@+++,@,,,-@---.@.../@///0@0001@1112@2223@3334@4445@5556@6667@7778@8889@999:@:::;@;;;<@<<<=@===>@>>>?@???@@@@@A@AAAB@BBBC@CCCD@DDDE@EEEF@FFFG@GGGH@HHHI@IIIJ@JJJK@KKKL@LLLM@MMMN@NNNO@OOOP@PPPQ@QQQR@RRRS@SSST@TTTU@UUUV@VVVW@WWWX@XXXY@YYYZ@ZZZ[@[[blob-2048-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2049-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2050-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2051-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2052-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2053-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2054-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2055-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2056-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2057-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2058-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2059-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2060-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2061-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2062-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2063-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2064-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2065-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2066-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2067-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2068-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2069-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2070-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2071-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2072-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2073-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2074-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2075-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2076-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2077-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2078-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2079-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2080-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2081-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2082-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2083-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2084-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2085-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2086-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2087-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2088-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2089-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2090-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2091-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2092-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2093-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2094-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2095-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2096-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2097-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2098-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2099-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2100-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2101-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2102-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2103-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2104-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2105-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2106-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2107-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2108-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2109-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2110-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2111-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2112-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2113-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2114-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2115-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2116-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2117-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2118-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2119-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2120-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2121-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2122-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2123-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2124-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2125-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2126-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2127-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2128-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2129-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2130-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2131-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2132-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2133-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2134-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2135-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2136-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2137-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2138-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2139-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2140-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2141-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2142-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2143-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2144-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2145-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2146-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2147-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2148-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2149-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2150-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2151-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2152-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2153-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2154-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2155-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2156-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2157-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2158-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2159-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2160-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2161-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2162-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2163-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2164-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2165-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2166-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2167-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2168-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2169-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2170-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2171-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2172-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2173-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2174-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2175-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2176-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2177-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2178-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2179-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2180-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2181-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2182-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2183-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2184-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2185-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2186-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2187-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2188-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2189-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2190-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2191-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2192-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2193-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2194-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2195-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2196-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2197-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2198-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2199-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2200-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2201-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2202-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2203-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2204-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2205-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2206-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2207-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2208-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2209-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2210-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2211-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2212-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2213-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2214-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2215-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2216-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2217-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2218-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2219-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2220-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2221-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2222-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2223-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2224-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2225-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2226-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2227-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2228-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2229-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2230-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2231-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2232-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2233-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2234-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2235-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2236-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2237-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2238-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2239-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2240-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2241-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2242-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2243-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2244-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2245-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2246-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2247-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2248-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2249-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2250-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2251-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2252-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2253-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2254-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2255-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2256-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2257-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2258-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2259-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2260-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2261-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2262-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2263-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2264-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2265-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2266-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2267-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2268-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2269-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2270-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2271-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2272-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2273-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2274-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2275-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2276-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2277-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2278-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2279-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2280-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2281-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2282-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2283-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2284-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2285-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2286-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2287-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2288-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2289-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2290-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2291-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2292-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2293-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2294-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2295-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2296-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2297-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2298-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2299-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2300-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2301-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2302-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2303-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2304-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2305-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2306-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2307-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2308-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2309-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2310-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2311-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2312-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2313-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2314-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2315-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2316-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2317-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2318-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2319-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2320-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2321-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2322-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2323-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2324-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2325-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2326-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2327-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2328-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2329-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2330-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2331-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2332-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2333-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2334-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2335-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2336-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2337-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2338-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2339-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2340-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2341-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2342-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2343-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2344-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2345-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2346-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2347-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2348-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2349-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2350-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2351-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2352-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2353-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2354-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2355-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2356-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2357-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2358-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2359-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2360-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2361-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2362-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2363-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2364-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2365-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2366-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2367-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2368-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2369-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2370-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2371-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2372-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2373-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2374-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2375-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2376-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2377-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2378-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2379-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2380-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2381-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2382-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2383-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2384-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2385-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2386-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2387-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2388-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2389-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2390-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2391-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2392-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2393-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2394-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2395-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2396-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2397-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2398-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2399-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2400-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2401-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2402-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2403-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2404-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2405-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2406-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2407-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2408-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2409-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2410-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2411-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2412-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2413-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2414-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2415-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2416-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2417-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2418-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2419-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2420-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2421-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2422-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2423-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2424-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2425-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2426-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2427-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2428-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2429-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2430-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2431-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2432-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2433-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2434-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2435-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2436-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2437-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2438-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2439-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2440-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2441-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2442-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2443-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2444-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2445-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2446-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2447-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2448-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2449-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2450-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2451-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2452-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2453-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2454-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2455-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2456-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2457-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2458-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2459-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2460-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2461-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2462-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2463-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2464-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2465-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2466-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2467-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2468-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2469-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2470-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2471-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2472-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2473-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2474-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2475-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2476-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2477-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2478-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2479-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2480-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2481-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2482-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2483-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2484-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2485-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2486-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2487-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2488-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2489-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2490-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2491-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2492-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2493-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2494-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2495-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2496-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2497-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2498-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2499-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2500-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2501-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2502-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2503-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2504-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2505-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2506-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2507-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2508-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2509-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2510-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2511-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2512-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2513-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2514-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2515-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2516-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2517-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2518-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2519-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2520-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2521-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2522-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2523-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2524-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2525-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2526-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2527-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2528-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2529-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2530-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2531-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2532-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2533-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2534-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2535-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2536-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2537-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2538-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2539-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2540-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2541-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2542-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2543-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2544-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2545-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2546-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2547-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2548-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2549-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2550-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2551-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2552-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2553-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2554-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2555-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2556-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2557-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2558-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2559-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2560-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2561-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2562-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2563-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2564-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2565-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2566-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2567-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2568-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2569-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2570-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2571-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2572-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2573-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2574-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2575-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2576-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2577-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2578-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2579-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2580-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2581-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2582-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2583-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2584-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2585-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2586-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2587-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2588-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2589-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2590-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2591-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2592-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2593-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2594-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2595-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2596-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2597-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2598-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2599-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2600-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2601-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2602-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2603-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2604-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2605-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2606-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2607-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2608-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2609-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2610-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2611-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2612-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2613-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2614-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2615-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2616-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2617-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2618-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2619-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2620-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2621-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2622-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2623-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2624-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2625-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2626-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2627-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2628-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2629-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2630-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2631-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2632-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2633-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2634-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2635-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2636-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2637-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2638-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2639-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2640-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2641-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2642-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2643-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2644-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2645-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2646-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2647-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2648-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2649-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2650-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2651-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2652-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2653-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2654-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2655-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2656-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2657-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2658-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2659-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2660-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2661-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2662-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2663-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2664-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2665-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2666-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2667-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2668-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2669-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2670-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2671-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2672-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2673-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2674-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2675-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2676-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2677-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2678-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2679-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2680-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2681-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2682-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2683-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2684-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2685-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2686-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2687-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2688-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2689-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2690-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2691-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2692-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2693-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2694-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2695-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2696-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2697-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2698-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2699-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2700-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2701-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2702-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2703-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2704-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2705-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2706-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2707-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2708-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2709-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2710-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2711-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2712-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2713-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2714-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2715-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2716-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2717-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2718-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2719-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2720-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2721-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2722-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2723-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2724-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2725-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2726-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2727-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2728-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2729-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2730-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2731-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2732-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2733-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2734-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2735-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2736-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2737-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2738-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2739-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2740-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2741-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2742-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2743-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2744-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2745-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2746-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2747-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2748-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2749-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2750-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2751-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2752-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2753-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2754-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2755-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2756-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2757-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2758-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2759-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2760-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2761-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2762-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2763-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2764-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2765-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2766-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2767-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2768-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2769-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2770-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2771-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2772-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2773-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2774-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2775-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2776-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2777-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2778-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2779-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2780-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2781-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2782-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2783-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2784-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2785-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2786-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2787-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2788-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2789-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2790-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2791-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2792-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2793-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2794-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2795-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2796-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2797-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2798-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2799-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2800-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2801-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2802-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2803-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2804-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2805-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2806-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2807-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2808-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2809-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2810-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2811-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2812-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2813-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2814-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2815-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2816-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2817-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2818-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2819-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2820-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2821-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2822-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2823-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2824-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2825-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2826-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2827-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2828-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2829-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2830-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2831-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2832-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2833-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2834-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2835-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2836-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2837-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2838-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2839-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2840-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2841-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2842-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2843-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2844-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2845-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2846-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2847-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2848-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2849-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2850-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2851-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2852-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2853-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2854-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2855-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2856-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2857-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2858-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2859-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2860-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2861-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2862-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2863-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2864-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2865-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2866-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2867-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2868-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2869-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2870-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2871-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2872-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2873-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2874-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2875-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2876-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2877-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2878-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2879-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2880-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2881-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2882-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2883-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2884-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2885-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2886-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2887-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2888-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2889-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2890-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2891-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2892-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2893-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2894-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2895-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2896-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2897-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2898-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2899-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2900-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2901-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2902-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2903-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2904-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2905-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2906-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2907-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2908-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2909-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2910-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2911-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2912-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2913-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2914-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2915-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2916-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2917-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2918-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2919-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2920-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2921-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2922-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2923-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2924-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2925-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2926-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2927-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2928-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2929-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2930-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2931-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2932-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2933-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2934-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2935-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2936-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2937-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2938-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2939-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2940-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2941-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2942-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2943-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2944-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2945-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2946-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2947-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2948-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2949-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2950-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2951-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2952-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2953-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2954-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2955-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2956-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2957-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2958-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2959-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2960-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2961-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2962-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2963-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2964-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2965-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2966-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2967-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2968-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2969-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2970-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2971-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2972-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2973-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2974-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2975-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2976-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2977-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2978-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2979-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2980-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2981-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2982-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2983-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2984-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2985-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2986-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2987-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2988-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2989-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2990-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2991-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2992-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2993-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2994-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2995-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2996-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2997-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2998-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2999-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3000-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3001-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3002-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3003-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3004-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3005-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3006-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3007-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3008-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3009-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3010-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3011-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3012-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3013-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3014-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3015-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3016-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3017-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3018-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3019-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3020-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3021-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3022-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3023-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3024-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3025-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3026-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3027-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3028-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3029-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3030-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3031-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3032-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3033-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3034-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3035-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3036-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3037-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3038-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3039-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3040-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3041-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3042-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3043-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3044-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3045-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3046-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3047-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3048-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3049-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3050-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3051-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3052-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3053-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3054-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3055-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3056-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3057-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3058-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3059-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3060-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3061-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3062-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3063-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3064-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3065-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3066-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3067-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3068-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3069-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3070-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3071-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH  +    !"#$%&'()*+,-./0123456789:;<HHHHHHHHHHHH=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\]^_`abcdefghijklmnopqrstuvwxyHHHHHHHHHHHHz{|}~HHHHHHHHHHHHHHHHHHHHHHHH         +                   ! " # $ % & ' ( ) * + , - . / 0 HHHHHHHHHHHH1 2 3 4 5 6 7 8 9 : ; < = > ? @ A B C D E F G H I J K L M N O P Q R S T U V W X Y Z [ \ ] ^ _ ` a b c d e f g h i j k l m HHHHHHHHHHHHn o p q r s t u v w x y z { | } ~  HHHHHHHHHHHH HHHHHHHHHHHH + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +! +" +# +$ +HHHHHHHHHHHH% +& +' +( +) +* ++ +, +- +. +/ +0 +1 +2 +3 +4 +5 +6 +7 +8 +9 +: +; +< += +> +? +@ +A +B +C +D +E +F +G +H +I +J +K +L +M +N +O +P +Q +R +S +T +U +V +W +X +Y +Z +[ +\ +] +^ +_ +` +a +HHHHHHHHHHHHb +c +d +e +f +g +h +i +j +k +l +m +n +o +p +q +r +s +t +u +v +w +x +y +z +{ +| +} +~ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +HHHHHHHHHHHH + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +HHHHHHHHHHHH + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +         +            HHHHHHHHHHHH       ! " # $ % & ' ( ) * + , - . / 0 1 2 3 4 5 6 7 8 9 : ; < = > ? @ A B C D E F G H I J K L M N O P Q R S T U HHHHHHHHHHHHV W X Y Z [ \ ] ^ _ ` a b c d e f g h i j k l m n o p q r s t u v w x y z { | } ~  HHHHHHHHHHHH HHHHHHHHHHHH @`Avalue-2048-deterministic-fixturevalue-2049-deterministic-fixturevalue-2050-deterministic-fixturevalue-2052-deterministic-fixturevalue-2053-deterministic-fixturevalue-2054-deterministic-fixturevalue-2055-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @!`value-2056-deterministic-fixturevalue-2057-deterministic-fixturevalue-2059-deterministic-fixturevalue-2060-deterministic-fixturevalue-2061-deterministic-fixturevalue-2062-deterministic-fixturevalue-2063-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-2064-deterministic-fixturevalue-2066-deterministic-fixturevalue-2067-deterministic-fixturevalue-2068-deterministic-fixturevalue-2069-deterministic-fixturevalue-2070-deterministic-fixturevalue-2071-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-2073-deterministic-fixturevalue-2074-deterministic-fixturevalue-2075-deterministic-fixturevalue-2076-deterministic-fixturevalue-2077-deterministic-fixturevalue-2078-deterministic-fixture @`value-2080-deterministic-fixturevalue-2081-deterministic-fixturevalue-2082-deterministic-fixturevalue-2083-deterministic-fixturevalue-2084-deterministic-fixturevalue-2085-deterministic-fixturevalue-2087-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-2088-deterministic-fixturevalue-2089-deterministic-fixturevalue-2090-deterministic-fixturevalue-2091-deterministic-fixturevalue-2092-deterministic-fixturevalue-2094-deterministic-fixturevalue-2095-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`avalue-2096-deterministic-fixturevalue-2097-deterministic-fixturevalue-2098-deterministic-fixturevalue-2099-deterministic-fixturevalue-2101-deterministic-fixturevalue-2102-deterministic-fixturevalue-2103-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`Avalue-2104-deterministic-fixturevalue-2105-deterministic-fixturevalue-2106-deterministic-fixturevalue-2108-deterministic-fixturevalue-2109-deterministic-fixturevalue-2110-deterministic-fixturevalue-2111-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @!`value-2112-deterministic-fixturevalue-2113-deterministic-fixturevalue-2115-deterministic-fixturevalue-2116-deterministic-fixturevalue-2117-deterministic-fixturevalue-2118-deterministic-fixturevalue-2119-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-2120-deterministic-fixturevalue-2122-deterministic-fixturevalue-2123-deterministic-fixturevalue-2124-deterministic-fixturevalue-2125-deterministic-fixturevalue-2126-deterministic-fixturevalue-2127-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-2129-deterministic-fixturevalue-2130-deterministic-fixturevalue-2131-deterministic-fixturevalue-2132-deterministic-fixturevalue-2133-deterministic-fixturevalue-2134-deterministic-fixture @`value-2136-deterministic-fixturevalue-2137-deterministic-fixturevalue-2138-deterministic-fixturevalue-2139-deterministic-fixturevalue-2140-deterministic-fixturevalue-2141-deterministic-fixturevalue-2143-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-2144-deterministic-fixturevalue-2145-deterministic-fixturevalue-2146-deterministic-fixturevalue-2147-deterministic-fixturevalue-2148-deterministic-fixturevalue-2150-deterministic-fixturevalue-2151-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`avalue-2152-deterministic-fixturevalue-2153-deterministic-fixturevalue-2154-deterministic-fixturevalue-2155-deterministic-fixturevalue-2157-deterministic-fixturevalue-2158-deterministic-fixturevalue-2159-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`Avalue-2160-deterministic-fixturevalue-2161-deterministic-fixturevalue-2162-deterministic-fixturevalue-2164-deterministic-fixturevalue-2165-deterministic-fixturevalue-2166-deterministic-fixturevalue-2167-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @!`value-2168-deterministic-fixturevalue-2169-deterministic-fixturevalue-2171-deterministic-fixturevalue-2172-deterministic-fixturevalue-2173-deterministic-fixturevalue-2174-deterministic-fixturevalue-2175-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-2176-deterministic-fixturevalue-2178-deterministic-fixturevalue-2179-deterministic-fixturevalue-2180-deterministic-fixturevalue-2181-deterministic-fixturevalue-2182-deterministic-fixturevalue-2183-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-2185-deterministic-fixturevalue-2186-deterministic-fixturevalue-2187-deterministic-fixturevalue-2188-deterministic-fixturevalue-2189-deterministic-fixturevalue-2190-deterministic-fixture @`value-2192-deterministic-fixturevalue-2193-deterministic-fixturevalue-2194-deterministic-fixturevalue-2195-deterministic-fixturevalue-2196-deterministic-fixturevalue-2197-deterministic-fixturevalue-2199-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-2200-deterministic-fixturevalue-2201-deterministic-fixturevalue-2202-deterministic-fixturevalue-2203-deterministic-fixturevalue-2204-deterministic-fixturevalue-2206-deterministic-fixturevalue-2207-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`avalue-2208-deterministic-fixturevalue-2209-deterministic-fixturevalue-2210-deterministic-fixturevalue-2211-deterministic-fixturevalue-2213-deterministic-fixturevalue-2214-deterministic-fixturevalue-2215-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`Avalue-2216-deterministic-fixturevalue-2217-deterministic-fixturevalue-2218-deterministic-fixturevalue-2220-deterministic-fixturevalue-2221-deterministic-fixturevalue-2222-deterministic-fixturevalue-2223-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @!`value-2224-deterministic-fixturevalue-2225-deterministic-fixturevalue-2227-deterministic-fixturevalue-2228-deterministic-fixturevalue-2229-deterministic-fixturevalue-2230-deterministic-fixturevalue-2231-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-2232-deterministic-fixturevalue-2234-deterministic-fixturevalue-2235-deterministic-fixturevalue-2236-deterministic-fixturevalue-2237-deterministic-fixturevalue-2238-deterministic-fixturevalue-2239-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-2241-deterministic-fixturevalue-2242-deterministic-fixturevalue-2243-deterministic-fixturevalue-2244-deterministic-fixturevalue-2245-deterministic-fixturevalue-2246-deterministic-fixture @`value-2248-deterministic-fixturevalue-2249-deterministic-fixturevalue-2250-deterministic-fixturevalue-2251-deterministic-fixturevalue-2252-deterministic-fixturevalue-2253-deterministic-fixturevalue-2255-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-2256-deterministic-fixturevalue-2257-deterministic-fixturevalue-2258-deterministic-fixturevalue-2259-deterministic-fixturevalue-2260-deterministic-fixturevalue-2262-deterministic-fixturevalue-2263-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`avalue-2264-deterministic-fixturevalue-2265-deterministic-fixturevalue-2266-deterministic-fixturevalue-2267-deterministic-fixturevalue-2269-deterministic-fixturevalue-2270-deterministic-fixturevalue-2271-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`Avalue-2272-deterministic-fixturevalue-2273-deterministic-fixturevalue-2274-deterministic-fixturevalue-2276-deterministic-fixturevalue-2277-deterministic-fixturevalue-2278-deterministic-fixturevalue-2279-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @!`value-2280-deterministic-fixturevalue-2281-deterministic-fixturevalue-2283-deterministic-fixturevalue-2284-deterministic-fixturevalue-2285-deterministic-fixturevalue-2286-deterministic-fixturevalue-2287-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-2288-deterministic-fixturevalue-2290-deterministic-fixturevalue-2291-deterministic-fixturevalue-2292-deterministic-fixturevalue-2293-deterministic-fixturevalue-2294-deterministic-fixturevalue-2295-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-2297-deterministic-fixturevalue-2298-deterministic-fixturevalue-2299-deterministic-fixturevalue-2300-deterministic-fixturevalue-2301-deterministic-fixturevalue-2302-deterministic-fixture @`value-2304-deterministic-fixturevalue-2305-deterministic-fixturevalue-2306-deterministic-fixturevalue-2307-deterministic-fixturevalue-2308-deterministic-fixturevalue-2309-deterministic-fixturevalue-2311-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-2312-deterministic-fixturevalue-2313-deterministic-fixturevalue-2314-deterministic-fixturevalue-2315-deterministic-fixturevalue-2316-deterministic-fixturevalue-2318-deterministic-fixturevalue-2319-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`avalue-2320-deterministic-fixturevalue-2321-deterministic-fixturevalue-2322-deterministic-fixturevalue-2323-deterministic-fixturevalue-2325-deterministic-fixturevalue-2326-deterministic-fixturevalue-2327-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`Avalue-2328-deterministic-fixturevalue-2329-deterministic-fixturevalue-2330-deterministic-fixturevalue-2332-deterministic-fixturevalue-2333-deterministic-fixturevalue-2334-deterministic-fixturevalue-2335-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @!`value-2336-deterministic-fixturevalue-2337-deterministic-fixturevalue-2339-deterministic-fixturevalue-2340-deterministic-fixturevalue-2341-deterministic-fixturevalue-2342-deterministic-fixturevalue-2343-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-2344-deterministic-fixturevalue-2346-deterministic-fixturevalue-2347-deterministic-fixturevalue-2348-deterministic-fixturevalue-2349-deterministic-fixturevalue-2350-deterministic-fixturevalue-2351-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-2353-deterministic-fixturevalue-2354-deterministic-fixturevalue-2355-deterministic-fixturevalue-2356-deterministic-fixturevalue-2357-deterministic-fixturevalue-2358-deterministic-fixture @`value-2360-deterministic-fixturevalue-2361-deterministic-fixturevalue-2362-deterministic-fixturevalue-2363-deterministic-fixturevalue-2364-deterministic-fixturevalue-2365-deterministic-fixturevalue-2367-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-2368-deterministic-fixturevalue-2369-deterministic-fixturevalue-2370-deterministic-fixturevalue-2371-deterministic-fixturevalue-2372-deterministic-fixturevalue-2374-deterministic-fixturevalue-2375-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`avalue-2376-deterministic-fixturevalue-2377-deterministic-fixturevalue-2378-deterministic-fixturevalue-2379-deterministic-fixturevalue-2381-deterministic-fixturevalue-2382-deterministic-fixturevalue-2383-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`Avalue-2384-deterministic-fixturevalue-2385-deterministic-fixturevalue-2386-deterministic-fixturevalue-2388-deterministic-fixturevalue-2389-deterministic-fixturevalue-2390-deterministic-fixturevalue-2391-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @!`value-2392-deterministic-fixturevalue-2393-deterministic-fixturevalue-2395-deterministic-fixturevalue-2396-deterministic-fixturevalue-2397-deterministic-fixturevalue-2398-deterministic-fixturevalue-2399-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-2400-deterministic-fixturevalue-2402-deterministic-fixturevalue-2403-deterministic-fixturevalue-2404-deterministic-fixturevalue-2405-deterministic-fixturevalue-2406-deterministic-fixturevalue-2407-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-2409-deterministic-fixturevalue-2410-deterministic-fixturevalue-2411-deterministic-fixturevalue-2412-deterministic-fixturevalue-2413-deterministic-fixturevalue-2414-deterministic-fixture @`value-2416-deterministic-fixturevalue-2417-deterministic-fixturevalue-2418-deterministic-fixturevalue-2419-deterministic-fixturevalue-2420-deterministic-fixturevalue-2421-deterministic-fixturevalue-2423-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-2424-deterministic-fixturevalue-2425-deterministic-fixturevalue-2426-deterministic-fixturevalue-2427-deterministic-fixturevalue-2428-deterministic-fixturevalue-2430-deterministic-fixturevalue-2431-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`avalue-2432-deterministic-fixturevalue-2433-deterministic-fixturevalue-2434-deterministic-fixturevalue-2435-deterministic-fixturevalue-2437-deterministic-fixturevalue-2438-deterministic-fixturevalue-2439-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`Avalue-2440-deterministic-fixturevalue-2441-deterministic-fixturevalue-2442-deterministic-fixturevalue-2444-deterministic-fixturevalue-2445-deterministic-fixturevalue-2446-deterministic-fixturevalue-2447-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @!`value-2448-deterministic-fixturevalue-2449-deterministic-fixturevalue-2451-deterministic-fixturevalue-2452-deterministic-fixturevalue-2453-deterministic-fixturevalue-2454-deterministic-fixturevalue-2455-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-2456-deterministic-fixturevalue-2458-deterministic-fixturevalue-2459-deterministic-fixturevalue-2460-deterministic-fixturevalue-2461-deterministic-fixturevalue-2462-deterministic-fixturevalue-2463-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-2465-deterministic-fixturevalue-2466-deterministic-fixturevalue-2467-deterministic-fixturevalue-2468-deterministic-fixturevalue-2469-deterministic-fixturevalue-2470-deterministic-fixture @`value-2472-deterministic-fixturevalue-2473-deterministic-fixturevalue-2474-deterministic-fixturevalue-2475-deterministic-fixturevalue-2476-deterministic-fixturevalue-2477-deterministic-fixturevalue-2479-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-2480-deterministic-fixturevalue-2481-deterministic-fixturevalue-2482-deterministic-fixturevalue-2483-deterministic-fixturevalue-2484-deterministic-fixturevalue-2486-deterministic-fixturevalue-2487-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`avalue-2488-deterministic-fixturevalue-2489-deterministic-fixturevalue-2490-deterministic-fixturevalue-2491-deterministic-fixturevalue-2493-deterministic-fixturevalue-2494-deterministic-fixturevalue-2495-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`Avalue-2496-deterministic-fixturevalue-2497-deterministic-fixturevalue-2498-deterministic-fixturevalue-2500-deterministic-fixturevalue-2501-deterministic-fixturevalue-2502-deterministic-fixturevalue-2503-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @!`value-2504-deterministic-fixturevalue-2505-deterministic-fixturevalue-2507-deterministic-fixturevalue-2508-deterministic-fixturevalue-2509-deterministic-fixturevalue-2510-deterministic-fixturevalue-2511-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-2512-deterministic-fixturevalue-2514-deterministic-fixturevalue-2515-deterministic-fixturevalue-2516-deterministic-fixturevalue-2517-deterministic-fixturevalue-2518-deterministic-fixturevalue-2519-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-2521-deterministic-fixturevalue-2522-deterministic-fixturevalue-2523-deterministic-fixturevalue-2524-deterministic-fixturevalue-2525-deterministic-fixturevalue-2526-deterministic-fixture @`value-2528-deterministic-fixturevalue-2529-deterministic-fixturevalue-2530-deterministic-fixturevalue-2531-deterministic-fixturevalue-2532-deterministic-fixturevalue-2533-deterministic-fixturevalue-2535-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-2536-deterministic-fixturevalue-2537-deterministic-fixturevalue-2538-deterministic-fixturevalue-2539-deterministic-fixturevalue-2540-deterministic-fixturevalue-2542-deterministic-fixturevalue-2543-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`avalue-2544-deterministic-fixturevalue-2545-deterministic-fixturevalue-2546-deterministic-fixturevalue-2547-deterministic-fixturevalue-2549-deterministic-fixturevalue-2550-deterministic-fixturevalue-2551-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`Avalue-2552-deterministic-fixturevalue-2553-deterministic-fixturevalue-2554-deterministic-fixturevalue-2556-deterministic-fixturevalue-2557-deterministic-fixturevalue-2558-deterministic-fixturevalue-2559-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @!`value-2560-deterministic-fixturevalue-2561-deterministic-fixturevalue-2563-deterministic-fixturevalue-2564-deterministic-fixturevalue-2565-deterministic-fixturevalue-2566-deterministic-fixturevalue-2567-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-2568-deterministic-fixturevalue-2570-deterministic-fixturevalue-2571-deterministic-fixturevalue-2572-deterministic-fixturevalue-2573-deterministic-fixturevalue-2574-deterministic-fixturevalue-2575-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-2577-deterministic-fixturevalue-2578-deterministic-fixturevalue-2579-deterministic-fixturevalue-2580-deterministic-fixturevalue-2581-deterministic-fixturevalue-2582-deterministic-fixture @`value-2584-deterministic-fixturevalue-2585-deterministic-fixturevalue-2586-deterministic-fixturevalue-2587-deterministic-fixturevalue-2588-deterministic-fixturevalue-2589-deterministic-fixturevalue-2591-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-2592-deterministic-fixturevalue-2593-deterministic-fixturevalue-2594-deterministic-fixturevalue-2595-deterministic-fixturevalue-2596-deterministic-fixturevalue-2598-deterministic-fixturevalue-2599-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`avalue-2600-deterministic-fixturevalue-2601-deterministic-fixturevalue-2602-deterministic-fixturevalue-2603-deterministic-fixturevalue-2605-deterministic-fixturevalue-2606-deterministic-fixturevalue-2607-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`Avalue-2608-deterministic-fixturevalue-2609-deterministic-fixturevalue-2610-deterministic-fixturevalue-2612-deterministic-fixturevalue-2613-deterministic-fixturevalue-2614-deterministic-fixturevalue-2615-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @!`value-2616-deterministic-fixturevalue-2617-deterministic-fixturevalue-2619-deterministic-fixturevalue-2620-deterministic-fixturevalue-2621-deterministic-fixturevalue-2622-deterministic-fixturevalue-2623-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-2624-deterministic-fixturevalue-2626-deterministic-fixturevalue-2627-deterministic-fixturevalue-2628-deterministic-fixturevalue-2629-deterministic-fixturevalue-2630-deterministic-fixturevalue-2631-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-2633-deterministic-fixturevalue-2634-deterministic-fixturevalue-2635-deterministic-fixturevalue-2636-deterministic-fixturevalue-2637-deterministic-fixturevalue-2638-deterministic-fixture @`value-2640-deterministic-fixturevalue-2641-deterministic-fixturevalue-2642-deterministic-fixturevalue-2643-deterministic-fixturevalue-2644-deterministic-fixturevalue-2645-deterministic-fixturevalue-2647-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-2648-deterministic-fixturevalue-2649-deterministic-fixturevalue-2650-deterministic-fixturevalue-2651-deterministic-fixturevalue-2652-deterministic-fixturevalue-2654-deterministic-fixturevalue-2655-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`avalue-2656-deterministic-fixturevalue-2657-deterministic-fixturevalue-2658-deterministic-fixturevalue-2659-deterministic-fixturevalue-2661-deterministic-fixturevalue-2662-deterministic-fixturevalue-2663-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`Avalue-2664-deterministic-fixturevalue-2665-deterministic-fixturevalue-2666-deterministic-fixturevalue-2668-deterministic-fixturevalue-2669-deterministic-fixturevalue-2670-deterministic-fixturevalue-2671-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @!`value-2672-deterministic-fixturevalue-2673-deterministic-fixturevalue-2675-deterministic-fixturevalue-2676-deterministic-fixturevalue-2677-deterministic-fixturevalue-2678-deterministic-fixturevalue-2679-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-2680-deterministic-fixturevalue-2682-deterministic-fixturevalue-2683-deterministic-fixturevalue-2684-deterministic-fixturevalue-2685-deterministic-fixturevalue-2686-deterministic-fixturevalue-2687-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-2689-deterministic-fixturevalue-2690-deterministic-fixturevalue-2691-deterministic-fixturevalue-2692-deterministic-fixturevalue-2693-deterministic-fixturevalue-2694-deterministic-fixture @`value-2696-deterministic-fixturevalue-2697-deterministic-fixturevalue-2698-deterministic-fixturevalue-2699-deterministic-fixturevalue-2700-deterministic-fixturevalue-2701-deterministic-fixturevalue-2703-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-2704-deterministic-fixturevalue-2705-deterministic-fixturevalue-2706-deterministic-fixturevalue-2707-deterministic-fixturevalue-2708-deterministic-fixturevalue-2710-deterministic-fixturevalue-2711-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`avalue-2712-deterministic-fixturevalue-2713-deterministic-fixturevalue-2714-deterministic-fixturevalue-2715-deterministic-fixturevalue-2717-deterministic-fixturevalue-2718-deterministic-fixturevalue-2719-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`Avalue-2720-deterministic-fixturevalue-2721-deterministic-fixturevalue-2722-deterministic-fixturevalue-2724-deterministic-fixturevalue-2725-deterministic-fixturevalue-2726-deterministic-fixturevalue-2727-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @!`value-2728-deterministic-fixturevalue-2729-deterministic-fixturevalue-2731-deterministic-fixturevalue-2732-deterministic-fixturevalue-2733-deterministic-fixturevalue-2734-deterministic-fixturevalue-2735-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-2736-deterministic-fixturevalue-2738-deterministic-fixturevalue-2739-deterministic-fixturevalue-2740-deterministic-fixturevalue-2741-deterministic-fixturevalue-2742-deterministic-fixturevalue-2743-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-2745-deterministic-fixturevalue-2746-deterministic-fixturevalue-2747-deterministic-fixturevalue-2748-deterministic-fixturevalue-2749-deterministic-fixturevalue-2750-deterministic-fixture @`value-2752-deterministic-fixturevalue-2753-deterministic-fixturevalue-2754-deterministic-fixturevalue-2755-deterministic-fixturevalue-2756-deterministic-fixturevalue-2757-deterministic-fixturevalue-2759-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-2760-deterministic-fixturevalue-2761-deterministic-fixturevalue-2762-deterministic-fixturevalue-2763-deterministic-fixturevalue-2764-deterministic-fixturevalue-2766-deterministic-fixturevalue-2767-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`avalue-2768-deterministic-fixturevalue-2769-deterministic-fixturevalue-2770-deterministic-fixturevalue-2771-deterministic-fixturevalue-2773-deterministic-fixturevalue-2774-deterministic-fixturevalue-2775-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`Avalue-2776-deterministic-fixturevalue-2777-deterministic-fixturevalue-2778-deterministic-fixturevalue-2780-deterministic-fixturevalue-2781-deterministic-fixturevalue-2782-deterministic-fixturevalue-2783-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @!`value-2784-deterministic-fixturevalue-2785-deterministic-fixturevalue-2787-deterministic-fixturevalue-2788-deterministic-fixturevalue-2789-deterministic-fixturevalue-2790-deterministic-fixturevalue-2791-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-2792-deterministic-fixturevalue-2794-deterministic-fixturevalue-2795-deterministic-fixturevalue-2796-deterministic-fixturevalue-2797-deterministic-fixturevalue-2798-deterministic-fixturevalue-2799-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-2801-deterministic-fixturevalue-2802-deterministic-fixturevalue-2803-deterministic-fixturevalue-2804-deterministic-fixturevalue-2805-deterministic-fixturevalue-2806-deterministic-fixture @`value-2808-deterministic-fixturevalue-2809-deterministic-fixturevalue-2810-deterministic-fixturevalue-2811-deterministic-fixturevalue-2812-deterministic-fixturevalue-2813-deterministic-fixturevalue-2815-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-2816-deterministic-fixturevalue-2817-deterministic-fixturevalue-2818-deterministic-fixturevalue-2819-deterministic-fixturevalue-2820-deterministic-fixturevalue-2822-deterministic-fixturevalue-2823-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`avalue-2824-deterministic-fixturevalue-2825-deterministic-fixturevalue-2826-deterministic-fixturevalue-2827-deterministic-fixturevalue-2829-deterministic-fixturevalue-2830-deterministic-fixturevalue-2831-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`Avalue-2832-deterministic-fixturevalue-2833-deterministic-fixturevalue-2834-deterministic-fixturevalue-2836-deterministic-fixturevalue-2837-deterministic-fixturevalue-2838-deterministic-fixturevalue-2839-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @!`value-2840-deterministic-fixturevalue-2841-deterministic-fixturevalue-2843-deterministic-fixturevalue-2844-deterministic-fixturevalue-2845-deterministic-fixturevalue-2846-deterministic-fixturevalue-2847-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-2848-deterministic-fixturevalue-2850-deterministic-fixturevalue-2851-deterministic-fixturevalue-2852-deterministic-fixturevalue-2853-deterministic-fixturevalue-2854-deterministic-fixturevalue-2855-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-2857-deterministic-fixturevalue-2858-deterministic-fixturevalue-2859-deterministic-fixturevalue-2860-deterministic-fixturevalue-2861-deterministic-fixturevalue-2862-deterministic-fixture @`value-2864-deterministic-fixturevalue-2865-deterministic-fixturevalue-2866-deterministic-fixturevalue-2867-deterministic-fixturevalue-2868-deterministic-fixturevalue-2869-deterministic-fixturevalue-2871-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-2872-deterministic-fixturevalue-2873-deterministic-fixturevalue-2874-deterministic-fixturevalue-2875-deterministic-fixturevalue-2876-deterministic-fixturevalue-2878-deterministic-fixturevalue-2879-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`avalue-2880-deterministic-fixturevalue-2881-deterministic-fixturevalue-2882-deterministic-fixturevalue-2883-deterministic-fixturevalue-2885-deterministic-fixturevalue-2886-deterministic-fixturevalue-2887-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`Avalue-2888-deterministic-fixturevalue-2889-deterministic-fixturevalue-2890-deterministic-fixturevalue-2892-deterministic-fixturevalue-2893-deterministic-fixturevalue-2894-deterministic-fixturevalue-2895-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @!`value-2896-deterministic-fixturevalue-2897-deterministic-fixturevalue-2899-deterministic-fixturevalue-2900-deterministic-fixturevalue-2901-deterministic-fixturevalue-2902-deterministic-fixturevalue-2903-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-2904-deterministic-fixturevalue-2906-deterministic-fixturevalue-2907-deterministic-fixturevalue-2908-deterministic-fixturevalue-2909-deterministic-fixturevalue-2910-deterministic-fixturevalue-2911-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-2913-deterministic-fixturevalue-2914-deterministic-fixturevalue-2915-deterministic-fixturevalue-2916-deterministic-fixturevalue-2917-deterministic-fixturevalue-2918-deterministic-fixture @`value-2920-deterministic-fixturevalue-2921-deterministic-fixturevalue-2922-deterministic-fixturevalue-2923-deterministic-fixturevalue-2924-deterministic-fixturevalue-2925-deterministic-fixturevalue-2927-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-2928-deterministic-fixturevalue-2929-deterministic-fixturevalue-2930-deterministic-fixturevalue-2931-deterministic-fixturevalue-2932-deterministic-fixturevalue-2934-deterministic-fixturevalue-2935-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`avalue-2936-deterministic-fixturevalue-2937-deterministic-fixturevalue-2938-deterministic-fixturevalue-2939-deterministic-fixturevalue-2941-deterministic-fixturevalue-2942-deterministic-fixturevalue-2943-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`Avalue-2944-deterministic-fixturevalue-2945-deterministic-fixturevalue-2946-deterministic-fixturevalue-2948-deterministic-fixturevalue-2949-deterministic-fixturevalue-2950-deterministic-fixturevalue-2951-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @!`value-2952-deterministic-fixturevalue-2953-deterministic-fixturevalue-2955-deterministic-fixturevalue-2956-deterministic-fixturevalue-2957-deterministic-fixturevalue-2958-deterministic-fixturevalue-2959-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-2960-deterministic-fixturevalue-2962-deterministic-fixturevalue-2963-deterministic-fixturevalue-2964-deterministic-fixturevalue-2965-deterministic-fixturevalue-2966-deterministic-fixturevalue-2967-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-2969-deterministic-fixturevalue-2970-deterministic-fixturevalue-2971-deterministic-fixturevalue-2972-deterministic-fixturevalue-2973-deterministic-fixturevalue-2974-deterministic-fixture @`value-2976-deterministic-fixturevalue-2977-deterministic-fixturevalue-2978-deterministic-fixturevalue-2979-deterministic-fixturevalue-2980-deterministic-fixturevalue-2981-deterministic-fixturevalue-2983-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-2984-deterministic-fixturevalue-2985-deterministic-fixturevalue-2986-deterministic-fixturevalue-2987-deterministic-fixturevalue-2988-deterministic-fixturevalue-2990-deterministic-fixturevalue-2991-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`avalue-2992-deterministic-fixturevalue-2993-deterministic-fixturevalue-2994-deterministic-fixturevalue-2995-deterministic-fixturevalue-2997-deterministic-fixturevalue-2998-deterministic-fixturevalue-2999-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`Avalue-3000-deterministic-fixturevalue-3001-deterministic-fixturevalue-3002-deterministic-fixturevalue-3004-deterministic-fixturevalue-3005-deterministic-fixturevalue-3006-deterministic-fixturevalue-3007-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @!`value-3008-deterministic-fixturevalue-3009-deterministic-fixturevalue-3011-deterministic-fixturevalue-3012-deterministic-fixturevalue-3013-deterministic-fixturevalue-3014-deterministic-fixturevalue-3015-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-3016-deterministic-fixturevalue-3018-deterministic-fixturevalue-3019-deterministic-fixturevalue-3020-deterministic-fixturevalue-3021-deterministic-fixturevalue-3022-deterministic-fixturevalue-3023-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-3025-deterministic-fixturevalue-3026-deterministic-fixturevalue-3027-deterministic-fixturevalue-3028-deterministic-fixturevalue-3029-deterministic-fixturevalue-3030-deterministic-fixture @`value-3032-deterministic-fixturevalue-3033-deterministic-fixturevalue-3034-deterministic-fixturevalue-3035-deterministic-fixturevalue-3036-deterministic-fixturevalue-3037-deterministic-fixturevalue-3039-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-3040-deterministic-fixturevalue-3041-deterministic-fixturevalue-3042-deterministic-fixturevalue-3043-deterministic-fixturevalue-3044-deterministic-fixturevalue-3046-deterministic-fixturevalue-3047-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`avalue-3048-deterministic-fixturevalue-3049-deterministic-fixturevalue-3050-deterministic-fixturevalue-3051-deterministic-fixturevalue-3053-deterministic-fixturevalue-3054-deterministic-fixturevalue-3055-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`Avalue-3056-deterministic-fixturevalue-3057-deterministic-fixturevalue-3058-deterministic-fixturevalue-3060-deterministic-fixturevalue-3061-deterministic-fixturevalue-3062-deterministic-fixturevalue-3063-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @!`value-3064-deterministic-fixturevalue-3065-deterministic-fixturevalue-3067-deterministic-fixturevalue-3068-deterministic-fixturevalue-3069-deterministic-fixturevalue-3070-deterministic-fixturevalue-3071-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH  !$'*-0369# <?BEHKNQTWA Z]`cfiloru_ x{~}    #&) ,/258;>ADG1 JMPSVY\_beO hknqtwz}m      +  "%(+.147! :=@CFILORU? X[^adgjmps] vy|{      !$'*-0369<?BE/HKNQTWZ]`cMfilorux{~k  #&),/258;>ADGJMPS=VY\_behknq[twz}y + "%(+.147:=@C-FILORUX[^aKdgjmpsvy|i  !$'*-0369<?BEHKNQ;TWZ]`cfiloYrux{~w  # &),/258;>A+DGJMPSVY\_Ibehknqtwz}g + "%(+.147:=@CFILO9RUX[^adgjmWpsvy|u         ! $ ' * - 0 3 6 9 < ? )B E H K N Q T W Z ] G` c f i l o r u x { e~      + + + + + + + + + + +# +& +) +, +/ +2 +5 +8 +; +> +A +D +G +J +M +7P +S +V +Y +\ +_ +b +e +h +k +Un +q +t +w +z +} + + + + +s + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH   +  + ! $ '*HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH- 03$6&9*?B.E0H2K4NQ8T:W<HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHZ `!Bc"Df#Fi$Hl%o&Lr'Nu(Px)R{+V,X-Z.\HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH/0`1b2d3f46l7n8p9:t;v<x=z>HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH?~ABCDEFGHIJLMNHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHOPQRSTUWXY Z[\]HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH^_` b&c)d,e/f2g5h8i;j>kAmGnHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHJoMpPqSrVsYt\u_vbxhykzn{q|t}HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHw~z} + HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH "$(*,.2468HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH<>@BHJLPRTVHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHZ^ `dfhj"n%p(t.1x4HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHz7|:~=@CFIORUX[^aHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHdgjpsvy|HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH        +    +  HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH! $ ' * - 0 $6 &9 (< ? ,B .E 0H 2K NHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH 6Q :W <Z ] @`! Bc" Df# Fi$ l% Jo& Lr( Px) {* T~+ HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHV, X- Z. / ^0 `1 b3 4 h5 j6 l7 n8 9 r: tHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH; v< x> |? ~@ A B C D E F G I J HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHK L M N O P Q R T U V W X Y  Z HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH[ \ ] _ ` a #b &c )d ,e /f 2g 5h 8j HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH>k Al Dm Gn Jo Mp Pq Sr Vs Yu _v bw ex hy kHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHz n{ q| t} w~ z      +    HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH       "  & ( ,  0 2HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH 4 6  : < >  D F H J  N PHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH R T X Z \ + ^   b d f h  n% p( HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHr+ . v1 x4 z7 |: = @ F I L O R U HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHX [ ^ a g j m p s v y |   HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH               HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH              HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH        + + + +  + +  +HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH + + + + +! +$ +' +- + 0 +"3 +$6 +&9 +< +*? +,B +HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH.E +0H +4N +6Q +8T +:W +Z +>] +@`! +Bc" +Df# +i% +Jo& +Lr' +NHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHu( +x) +R{* +T~+ +V, +X- +. +\0 +`1 +b2 +3 +f4 +h5 +j6 +lHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH7 +8 +p9 +r; +v< += +z> +|? +~@ +A +B +C +D +F +G +HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH +I +J +K +L +M +N +O +Q +R +S +T +U +V +HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHW +X +Y + Z +\ +] +^ +_ +` + a +#b +&c +)d +,e +/HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHg +5h +8i +;j +>k +Al +Dm +Gn +Jo +Mp +Pr +Vs +Yt +\u +_v +HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHbw +ex +hy +kz +n{ +q} +w~ +z +} + + + + + + +HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH +  + + + + + + + + +  + +& +( +*HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH + +. +0 +2 +4 + +8 +< +> + +B +D +F +H +HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH +L +N +R + +V +X +Z +\ + + +` +b +d + +jHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH +l" +n% +p( ++ +t. +v1 +x4 +z7 +~= +@ +C +F +I +L HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH +O +R +U +X +^ +a +d +g +j +m +p +s +v +y +HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH + + + + + + + + + + + + + +HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH + + + + + + + + + + + + + + HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH + + + + + + + + + + +  ! !  ! HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH ! +! ! ! ! ! + ! $! '! *! -! 0! "3! $6! HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH9! (]! @`!" f!# Fi!HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH$ Hl!% Jo!& Lr!' u!( Px!) R{!* T~!+ V!- Z!. \!/ ^!0 `!1 !2 d!3 HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHf!4 h!5 j!6 !8 p!9 r!: t!; !< x!= z!> |!? ~!@ !A !C HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH!D !E !F !G !H !I !J !K !L !N !O !P !Q !R !HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHS !T !U !V "W "Y "Z "[ "\ "] "^ "_ "`  "a #"b HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH&"d ,"e /"f 2"g 5"h 8"i ;"j >"k A"l D"m G"o M"p P"q S"r HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHV"s Y"t \"u _"v b"w e"x h"z n"{ q"| t"} w"~ z" }" " "HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH " " +" " " " " " " " " " " "" HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH$" &" (" " ," ." 0" 2" 6" 8" :" <" " @" BHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH" D" F" " L" N" P" " T" V# X# Z# +# ^ # b#HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH d# # h# j# l"# n%# (# r+# t.# x4# 7# |:# ~=# @# HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHC# F# I# L# O# U# X# [# ^# a# d# g# j# m# HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHp# v# y# |# # # # # # # # # # # #HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH # # # # # # # # # # # # # # HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH# # # # # # # # # # # # # #HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHgreenblueredHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHgreenblueredHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHgreenblueredHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHgreenblueredHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHgreenblueredHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHgreenblueredHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHgreenblueredHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHgreenblueredHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHgreenblueredHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHgreenblueredHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHgreenblueredHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHgreenblueredHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHgreenblueredHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHgreenblueredHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHgreenblueredHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ @    +@ + + + @    @    @   @@@@@@@@@@@@@@@@@@ @   !@!!!"@"""#@###$@$$$%@%%%&@&&&'@'''(@((()@)))*@***+@+++,@,,,-@---.@.../@///0@0001@1112@2223@3334@4445@5556@6667@7778@8889@999:@:::;@;;;<@<<<=@===>@>>>?@???@@@@@A@AAAB@BBBC@CCCD@DDDE@EEEF@FFFG@GGGH@HHHI@IIIJ@JJJK@KKKL@LLLM@MMMN@NNNO@OOOP@PPPQ@QQQR@RRRS@SSST@TTTU@UUUV@VVVW@WWWX@XXXY@YYYZ@ZZZ[@[[[\@\\\]@]]]^@^^^_@___`@```a@aaab@bbbc@cccd@ddde@eeef@fffg@gggh@hhhi@iiij@jjjk@kkkl@lllm@mmmn@nnno@ooop@pppq@qqqr@rrrs@ssst@tttu@uuuv@vvvw@wwwx@xxxy@yyyz@zzz{@{{{|@|||}@}}}~@~~~@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@blob-3072-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3073-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3074-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3075-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3076-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3077-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3078-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3079-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3080-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3081-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3082-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3083-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3084-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3085-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3086-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3087-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3088-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3089-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3090-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3091-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3092-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3093-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3094-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3095-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3096-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3097-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3098-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3099-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3100-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3101-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3102-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3103-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3104-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3105-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3106-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3107-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3108-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3109-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3110-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3111-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3112-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3113-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3114-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3115-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3116-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3117-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3118-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3119-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3120-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3121-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3122-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3123-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3124-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3125-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3126-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3127-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3128-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3129-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3130-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3131-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3132-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3133-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3134-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3135-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3136-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3137-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3138-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3139-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3140-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3141-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3142-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3143-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3144-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3145-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3146-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3147-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3148-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3149-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3150-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3151-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3152-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3153-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3154-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3155-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3156-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3157-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3158-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3159-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3160-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3161-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3162-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3163-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3164-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3165-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3166-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3167-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3168-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3169-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3170-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3171-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3172-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3173-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3174-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3175-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3176-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3177-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3178-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3179-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3180-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3181-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3182-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3183-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3184-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3185-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3186-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3187-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3188-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3189-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3190-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3191-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3192-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3193-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3194-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3195-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3196-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3197-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3198-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3199-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3200-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3201-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3202-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3203-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3204-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3205-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3206-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3207-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3208-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3209-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3210-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3211-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3212-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3213-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3214-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3215-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3216-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3217-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3218-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3219-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3220-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3221-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3222-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3223-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3224-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3225-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3226-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3227-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3228-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3229-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3230-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3231-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3232-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3233-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3234-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3235-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3236-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3237-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3238-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3239-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3240-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3241-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3242-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3243-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3244-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3245-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3246-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3247-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3248-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3249-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3250-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3251-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3252-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3253-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3254-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3255-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3256-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3257-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3258-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3259-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3260-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3261-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3262-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3263-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3264-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3265-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3266-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3267-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3268-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3269-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3270-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3271-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3272-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3273-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3274-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3275-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3276-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3277-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3278-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3279-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3280-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3281-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3282-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3283-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3284-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3285-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3286-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3287-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3288-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3289-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3290-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3291-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3292-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3293-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3294-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3295-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3296-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3297-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3298-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3299-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3300-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3301-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3302-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3303-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3304-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3305-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3306-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3307-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3308-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3309-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3310-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3311-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3312-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3313-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3314-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3315-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3316-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3317-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3318-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3319-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3320-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3321-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3322-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3323-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3324-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3325-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3326-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3327-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3328-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3329-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3330-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3331-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3332-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3333-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3334-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3335-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3336-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3337-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3338-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3339-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3340-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3341-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3342-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3343-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3344-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3345-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3346-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3347-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3348-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3349-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3350-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3351-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3352-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3353-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3354-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3355-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3356-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3357-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3358-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3359-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3360-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3361-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3362-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3363-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3364-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3365-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3366-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3367-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3368-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3369-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3370-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3371-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3372-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3373-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3374-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3375-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3376-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3377-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3378-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3379-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3380-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3381-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3382-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3383-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3384-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3385-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3386-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3387-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3388-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3389-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3390-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3391-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3392-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3393-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3394-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3395-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3396-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3397-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3398-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3399-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3400-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3401-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3402-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3403-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3404-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3405-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3406-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3407-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3408-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3409-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3410-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3411-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3412-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3413-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3414-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3415-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3416-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3417-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3418-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3419-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3420-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3421-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3422-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3423-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3424-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3425-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3426-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3427-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3428-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3429-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3430-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3431-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3432-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3433-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3434-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3435-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3436-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3437-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3438-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3439-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3440-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3441-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3442-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3443-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3444-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3445-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3446-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3447-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3448-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3449-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3450-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3451-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3452-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3453-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3454-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3455-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3456-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3457-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3458-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3459-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3460-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3461-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3462-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3463-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3464-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3465-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3466-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3467-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3468-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3469-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3470-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3471-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3472-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3473-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3474-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3475-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3476-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3477-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3478-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3479-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3480-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3481-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3482-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3483-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3484-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3485-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3486-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3487-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3488-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3489-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3490-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3491-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3492-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3493-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3494-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3495-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3496-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3497-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3498-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3499-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3500-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3501-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3502-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3503-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3504-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3505-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3506-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3507-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3508-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3509-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3510-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3511-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3512-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3513-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3514-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3515-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3516-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3517-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3518-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3519-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3520-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3521-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3522-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3523-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3524-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3525-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3526-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3527-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3528-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3529-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3530-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3531-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3532-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3533-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3534-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3535-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3536-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3537-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3538-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3539-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3540-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3541-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3542-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3543-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3544-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3545-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3546-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3547-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3548-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3549-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3550-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3551-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3552-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3553-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3554-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3555-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3556-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3557-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3558-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3559-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3560-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3561-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3562-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3563-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3564-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3565-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3566-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3567-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3568-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3569-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3570-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3571-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3572-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3573-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3574-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3575-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3576-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3577-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3578-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3579-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3580-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3581-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3582-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3583-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3584-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3585-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3586-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3587-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3588-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3589-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3590-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3591-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3592-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3593-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3594-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3595-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3596-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3597-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3598-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3599-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3600-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3601-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3602-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3603-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3604-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3605-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3606-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3607-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3608-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3609-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3610-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3611-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3612-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3613-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3614-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3615-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3616-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3617-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3618-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3619-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3620-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3621-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3622-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3623-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3624-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3625-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3626-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3627-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3628-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3629-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3630-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3631-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3632-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3633-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3634-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3635-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3636-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3637-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3638-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3639-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3640-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3641-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3642-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3643-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3644-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3645-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3646-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3647-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3648-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3649-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3650-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3651-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3652-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3653-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3654-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3655-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3656-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3657-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3658-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3659-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3660-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3661-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3662-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3663-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3664-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3665-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3666-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3667-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3668-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3669-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3670-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3671-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3672-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3673-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3674-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3675-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3676-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3677-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3678-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3679-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3680-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3681-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3682-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3683-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3684-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3685-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3686-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3687-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3688-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3689-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3690-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3691-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3692-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3693-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3694-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3695-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3696-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3697-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3698-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3699-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3700-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3701-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3702-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3703-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3704-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3705-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3706-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3707-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3708-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3709-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3710-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3711-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3712-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3713-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3714-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3715-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3716-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3717-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3718-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3719-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3720-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3721-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3722-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3723-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3724-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3725-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3726-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3727-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3728-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3729-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3730-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3731-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3732-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3733-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3734-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3735-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3736-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3737-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3738-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3739-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3740-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3741-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3742-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3743-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3744-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3745-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3746-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3747-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3748-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3749-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3750-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3751-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3752-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3753-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3754-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3755-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3756-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3757-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3758-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3759-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3760-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3761-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3762-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3763-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3764-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3765-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3766-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3767-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3768-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3769-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3770-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3771-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3772-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3773-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3774-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3775-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3776-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3777-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3778-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3779-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3780-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3781-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3782-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3783-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3784-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3785-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3786-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3787-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3788-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3789-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3790-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3791-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3792-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3793-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3794-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3795-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3796-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3797-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3798-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3799-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3800-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3801-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3802-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3803-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3804-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3805-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3806-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3807-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3808-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3809-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3810-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3811-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3812-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3813-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3814-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3815-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3816-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3817-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3818-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3819-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3820-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3821-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3822-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3823-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3824-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3825-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3826-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3827-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3828-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3829-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3830-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3831-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3832-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3833-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3834-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3835-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3836-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3837-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3838-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3839-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3840-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3841-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3842-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3843-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3844-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3845-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3846-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3847-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3848-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3849-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3850-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3851-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3852-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3853-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3854-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3855-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3856-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3857-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3858-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3859-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3860-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3861-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3862-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3863-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3864-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3865-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3866-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3867-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3868-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3869-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3870-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3871-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3872-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3873-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3874-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3875-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3876-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3877-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3878-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3879-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3880-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3881-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3882-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3883-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3884-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3885-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3886-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3887-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3888-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3889-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3890-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3891-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3892-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3893-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3894-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3895-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3896-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3897-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3898-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3899-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3900-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3901-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3902-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3903-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3904-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3905-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3906-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3907-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3908-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3909-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3910-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3911-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3912-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3913-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3914-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3915-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3916-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3917-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3918-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3919-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3920-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3921-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3922-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3923-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3924-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3925-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3926-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3927-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3928-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3929-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3930-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3931-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3932-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3933-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3934-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3935-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3936-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3937-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3938-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3939-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3940-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3941-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3942-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3943-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3944-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3945-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3946-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3947-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3948-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3949-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3950-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3951-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3952-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3953-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3954-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3955-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3956-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3957-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3958-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3959-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3960-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3961-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3962-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3963-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3964-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3965-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3966-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3967-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3968-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3969-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3970-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3971-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3972-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3973-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3974-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3975-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3976-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3977-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3978-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3979-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3980-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3981-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3982-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3983-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3984-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3985-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3986-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3987-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3988-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3989-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3990-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3991-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3992-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3993-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3994-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3995-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3996-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3997-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3998-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3999-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4000-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4001-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4002-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4003-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4004-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4005-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4006-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4007-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4008-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4009-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4010-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4011-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4012-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4013-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4014-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4015-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4016-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4017-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4018-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4019-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4020-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4021-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4022-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4023-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4024-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4025-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4026-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4027-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4028-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4029-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4030-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4031-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4032-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4033-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4034-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4035-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4036-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4037-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4038-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4039-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4040-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4041-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4042-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4043-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4044-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4045-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4046-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4047-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4048-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4049-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4050-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4051-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4052-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4053-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4054-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4055-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4056-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4057-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4058-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4059-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4060-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4061-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4062-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4063-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4064-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4065-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4066-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4067-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4068-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4069-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4070-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4071-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4072-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4073-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4074-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4075-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4076-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4077-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4078-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4079-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4080-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4081-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4082-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4083-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4084-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4085-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4086-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4087-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4088-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4089-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4090-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4091-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4092-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4093-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4094-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4095-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH         +                   ! " # $ % & ' ( ) * + , - . / 0 1 2 3 4 5 6 7 8 9 : ; < HHHHHHHHHHHH= > ? @ A B C D E F G H I J K L M N O P Q R S T U V W X Y Z [ \ ] ^ _ ` a b c d e f g h i j k l m n o p q r s t u v w x y HHHHHHHHHHHHz { | } ~  HHHHHHHHHHHH HHHHHHHHHHHH         +                   ! " # $ % & ' ( ) * + , - . / 0 HHHHHHHHHHHH1 2 3 4 5 6 7 8 9 : ; < = > ? @ A B C D E F G H I J K L M N O P Q R S T U V W X Y Z [ \ ] ^ _ ` a b c d e f g h i j k l m HHHHHHHHHHHHn o p q r s t u v w x y z { | } ~  HHHHHHHHHHHH HHHHHHHHHHHH   +    !"#$HHHHHHHHHHHH%&'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\]^_`aHHHHHHHHHHHHbcdefghijklmnopqrstuvwxyz{|}~HHHHHHHHHHHHHHHHHHHHHHHH  +   HHHHHHHHHHHH !"#$%&'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUHHHHHHHHHHHHVWXYZ[\]^_`abcdefghijklmnopqrstuvwxyz{|}~HHHHHHHHHHHHHHHHHHHHHHHH @`value-3072-deterministic-fixturevalue-3074-deterministic-fixturevalue-3075-deterministic-fixturevalue-3076-deterministic-fixturevalue-3077-deterministic-fixturevalue-3078-deterministic-fixturevalue-3079-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-3081-deterministic-fixturevalue-3082-deterministic-fixturevalue-3083-deterministic-fixturevalue-3084-deterministic-fixturevalue-3085-deterministic-fixturevalue-3086-deterministic-fixture @`value-3088-deterministic-fixturevalue-3089-deterministic-fixturevalue-3090-deterministic-fixturevalue-3091-deterministic-fixturevalue-3092-deterministic-fixturevalue-3093-deterministic-fixturevalue-3095-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-3096-deterministic-fixturevalue-3097-deterministic-fixturevalue-3098-deterministic-fixturevalue-3099-deterministic-fixturevalue-3100-deterministic-fixturevalue-3102-deterministic-fixturevalue-3103-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`avalue-3104-deterministic-fixturevalue-3105-deterministic-fixturevalue-3106-deterministic-fixturevalue-3107-deterministic-fixturevalue-3109-deterministic-fixturevalue-3110-deterministic-fixturevalue-3111-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`Avalue-3112-deterministic-fixturevalue-3113-deterministic-fixturevalue-3114-deterministic-fixturevalue-3116-deterministic-fixturevalue-3117-deterministic-fixturevalue-3118-deterministic-fixturevalue-3119-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @!`value-3120-deterministic-fixturevalue-3121-deterministic-fixturevalue-3123-deterministic-fixturevalue-3124-deterministic-fixturevalue-3125-deterministic-fixturevalue-3126-deterministic-fixturevalue-3127-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-3128-deterministic-fixturevalue-3130-deterministic-fixturevalue-3131-deterministic-fixturevalue-3132-deterministic-fixturevalue-3133-deterministic-fixturevalue-3134-deterministic-fixturevalue-3135-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-3137-deterministic-fixturevalue-3138-deterministic-fixturevalue-3139-deterministic-fixturevalue-3140-deterministic-fixturevalue-3141-deterministic-fixturevalue-3142-deterministic-fixture @`value-3144-deterministic-fixturevalue-3145-deterministic-fixturevalue-3146-deterministic-fixturevalue-3147-deterministic-fixturevalue-3148-deterministic-fixturevalue-3149-deterministic-fixturevalue-3151-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-3152-deterministic-fixturevalue-3153-deterministic-fixturevalue-3154-deterministic-fixturevalue-3155-deterministic-fixturevalue-3156-deterministic-fixturevalue-3158-deterministic-fixturevalue-3159-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`avalue-3160-deterministic-fixturevalue-3161-deterministic-fixturevalue-3162-deterministic-fixturevalue-3163-deterministic-fixturevalue-3165-deterministic-fixturevalue-3166-deterministic-fixturevalue-3167-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`Avalue-3168-deterministic-fixturevalue-3169-deterministic-fixturevalue-3170-deterministic-fixturevalue-3172-deterministic-fixturevalue-3173-deterministic-fixturevalue-3174-deterministic-fixturevalue-3175-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @!`value-3176-deterministic-fixturevalue-3177-deterministic-fixturevalue-3179-deterministic-fixturevalue-3180-deterministic-fixturevalue-3181-deterministic-fixturevalue-3182-deterministic-fixturevalue-3183-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-3184-deterministic-fixturevalue-3186-deterministic-fixturevalue-3187-deterministic-fixturevalue-3188-deterministic-fixturevalue-3189-deterministic-fixturevalue-3190-deterministic-fixturevalue-3191-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-3193-deterministic-fixturevalue-3194-deterministic-fixturevalue-3195-deterministic-fixturevalue-3196-deterministic-fixturevalue-3197-deterministic-fixturevalue-3198-deterministic-fixture @`value-3200-deterministic-fixturevalue-3201-deterministic-fixturevalue-3202-deterministic-fixturevalue-3203-deterministic-fixturevalue-3204-deterministic-fixturevalue-3205-deterministic-fixturevalue-3207-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-3208-deterministic-fixturevalue-3209-deterministic-fixturevalue-3210-deterministic-fixturevalue-3211-deterministic-fixturevalue-3212-deterministic-fixturevalue-3214-deterministic-fixturevalue-3215-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`avalue-3216-deterministic-fixturevalue-3217-deterministic-fixturevalue-3218-deterministic-fixturevalue-3219-deterministic-fixturevalue-3221-deterministic-fixturevalue-3222-deterministic-fixturevalue-3223-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`Avalue-3224-deterministic-fixturevalue-3225-deterministic-fixturevalue-3226-deterministic-fixturevalue-3228-deterministic-fixturevalue-3229-deterministic-fixturevalue-3230-deterministic-fixturevalue-3231-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @!`value-3232-deterministic-fixturevalue-3233-deterministic-fixturevalue-3235-deterministic-fixturevalue-3236-deterministic-fixturevalue-3237-deterministic-fixturevalue-3238-deterministic-fixturevalue-3239-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-3240-deterministic-fixturevalue-3242-deterministic-fixturevalue-3243-deterministic-fixturevalue-3244-deterministic-fixturevalue-3245-deterministic-fixturevalue-3246-deterministic-fixturevalue-3247-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-3249-deterministic-fixturevalue-3250-deterministic-fixturevalue-3251-deterministic-fixturevalue-3252-deterministic-fixturevalue-3253-deterministic-fixturevalue-3254-deterministic-fixture @`value-3256-deterministic-fixturevalue-3257-deterministic-fixturevalue-3258-deterministic-fixturevalue-3259-deterministic-fixturevalue-3260-deterministic-fixturevalue-3261-deterministic-fixturevalue-3263-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-3264-deterministic-fixturevalue-3265-deterministic-fixturevalue-3266-deterministic-fixturevalue-3267-deterministic-fixturevalue-3268-deterministic-fixturevalue-3270-deterministic-fixturevalue-3271-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`avalue-3272-deterministic-fixturevalue-3273-deterministic-fixturevalue-3274-deterministic-fixturevalue-3275-deterministic-fixturevalue-3277-deterministic-fixturevalue-3278-deterministic-fixturevalue-3279-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`Avalue-3280-deterministic-fixturevalue-3281-deterministic-fixturevalue-3282-deterministic-fixturevalue-3284-deterministic-fixturevalue-3285-deterministic-fixturevalue-3286-deterministic-fixturevalue-3287-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @!`value-3288-deterministic-fixturevalue-3289-deterministic-fixturevalue-3291-deterministic-fixturevalue-3292-deterministic-fixturevalue-3293-deterministic-fixturevalue-3294-deterministic-fixturevalue-3295-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-3296-deterministic-fixturevalue-3298-deterministic-fixturevalue-3299-deterministic-fixturevalue-3300-deterministic-fixturevalue-3301-deterministic-fixturevalue-3302-deterministic-fixturevalue-3303-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-3305-deterministic-fixturevalue-3306-deterministic-fixturevalue-3307-deterministic-fixturevalue-3308-deterministic-fixturevalue-3309-deterministic-fixturevalue-3310-deterministic-fixture @`value-3312-deterministic-fixturevalue-3313-deterministic-fixturevalue-3314-deterministic-fixturevalue-3315-deterministic-fixturevalue-3316-deterministic-fixturevalue-3317-deterministic-fixturevalue-3319-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-3320-deterministic-fixturevalue-3321-deterministic-fixturevalue-3322-deterministic-fixturevalue-3323-deterministic-fixturevalue-3324-deterministic-fixturevalue-3326-deterministic-fixturevalue-3327-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`avalue-3328-deterministic-fixturevalue-3329-deterministic-fixturevalue-3330-deterministic-fixturevalue-3331-deterministic-fixturevalue-3333-deterministic-fixturevalue-3334-deterministic-fixturevalue-3335-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`Avalue-3336-deterministic-fixturevalue-3337-deterministic-fixturevalue-3338-deterministic-fixturevalue-3340-deterministic-fixturevalue-3341-deterministic-fixturevalue-3342-deterministic-fixturevalue-3343-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @!`value-3344-deterministic-fixturevalue-3345-deterministic-fixturevalue-3347-deterministic-fixturevalue-3348-deterministic-fixturevalue-3349-deterministic-fixturevalue-3350-deterministic-fixturevalue-3351-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-3352-deterministic-fixturevalue-3354-deterministic-fixturevalue-3355-deterministic-fixturevalue-3356-deterministic-fixturevalue-3357-deterministic-fixturevalue-3358-deterministic-fixturevalue-3359-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-3361-deterministic-fixturevalue-3362-deterministic-fixturevalue-3363-deterministic-fixturevalue-3364-deterministic-fixturevalue-3365-deterministic-fixturevalue-3366-deterministic-fixture @`value-3368-deterministic-fixturevalue-3369-deterministic-fixturevalue-3370-deterministic-fixturevalue-3371-deterministic-fixturevalue-3372-deterministic-fixturevalue-3373-deterministic-fixturevalue-3375-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-3376-deterministic-fixturevalue-3377-deterministic-fixturevalue-3378-deterministic-fixturevalue-3379-deterministic-fixturevalue-3380-deterministic-fixturevalue-3382-deterministic-fixturevalue-3383-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`avalue-3384-deterministic-fixturevalue-3385-deterministic-fixturevalue-3386-deterministic-fixturevalue-3387-deterministic-fixturevalue-3389-deterministic-fixturevalue-3390-deterministic-fixturevalue-3391-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`Avalue-3392-deterministic-fixturevalue-3393-deterministic-fixturevalue-3394-deterministic-fixturevalue-3396-deterministic-fixturevalue-3397-deterministic-fixturevalue-3398-deterministic-fixturevalue-3399-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @!`value-3400-deterministic-fixturevalue-3401-deterministic-fixturevalue-3403-deterministic-fixturevalue-3404-deterministic-fixturevalue-3405-deterministic-fixturevalue-3406-deterministic-fixturevalue-3407-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-3408-deterministic-fixturevalue-3410-deterministic-fixturevalue-3411-deterministic-fixturevalue-3412-deterministic-fixturevalue-3413-deterministic-fixturevalue-3414-deterministic-fixturevalue-3415-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-3417-deterministic-fixturevalue-3418-deterministic-fixturevalue-3419-deterministic-fixturevalue-3420-deterministic-fixturevalue-3421-deterministic-fixturevalue-3422-deterministic-fixture @`value-3424-deterministic-fixturevalue-3425-deterministic-fixturevalue-3426-deterministic-fixturevalue-3427-deterministic-fixturevalue-3428-deterministic-fixturevalue-3429-deterministic-fixturevalue-3431-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-3432-deterministic-fixturevalue-3433-deterministic-fixturevalue-3434-deterministic-fixturevalue-3435-deterministic-fixturevalue-3436-deterministic-fixturevalue-3438-deterministic-fixturevalue-3439-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`avalue-3440-deterministic-fixturevalue-3441-deterministic-fixturevalue-3442-deterministic-fixturevalue-3443-deterministic-fixturevalue-3445-deterministic-fixturevalue-3446-deterministic-fixturevalue-3447-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`Avalue-3448-deterministic-fixturevalue-3449-deterministic-fixturevalue-3450-deterministic-fixturevalue-3452-deterministic-fixturevalue-3453-deterministic-fixturevalue-3454-deterministic-fixturevalue-3455-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @!`value-3456-deterministic-fixturevalue-3457-deterministic-fixturevalue-3459-deterministic-fixturevalue-3460-deterministic-fixturevalue-3461-deterministic-fixturevalue-3462-deterministic-fixturevalue-3463-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-3464-deterministic-fixturevalue-3466-deterministic-fixturevalue-3467-deterministic-fixturevalue-3468-deterministic-fixturevalue-3469-deterministic-fixturevalue-3470-deterministic-fixturevalue-3471-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-3473-deterministic-fixturevalue-3474-deterministic-fixturevalue-3475-deterministic-fixturevalue-3476-deterministic-fixturevalue-3477-deterministic-fixturevalue-3478-deterministic-fixture @`value-3480-deterministic-fixturevalue-3481-deterministic-fixturevalue-3482-deterministic-fixturevalue-3483-deterministic-fixturevalue-3484-deterministic-fixturevalue-3485-deterministic-fixturevalue-3487-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-3488-deterministic-fixturevalue-3489-deterministic-fixturevalue-3490-deterministic-fixturevalue-3491-deterministic-fixturevalue-3492-deterministic-fixturevalue-3494-deterministic-fixturevalue-3495-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`avalue-3496-deterministic-fixturevalue-3497-deterministic-fixturevalue-3498-deterministic-fixturevalue-3499-deterministic-fixturevalue-3501-deterministic-fixturevalue-3502-deterministic-fixturevalue-3503-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`Avalue-3504-deterministic-fixturevalue-3505-deterministic-fixturevalue-3506-deterministic-fixturevalue-3508-deterministic-fixturevalue-3509-deterministic-fixturevalue-3510-deterministic-fixturevalue-3511-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @!`value-3512-deterministic-fixturevalue-3513-deterministic-fixturevalue-3515-deterministic-fixturevalue-3516-deterministic-fixturevalue-3517-deterministic-fixturevalue-3518-deterministic-fixturevalue-3519-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-3520-deterministic-fixturevalue-3522-deterministic-fixturevalue-3523-deterministic-fixturevalue-3524-deterministic-fixturevalue-3525-deterministic-fixturevalue-3526-deterministic-fixturevalue-3527-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-3529-deterministic-fixturevalue-3530-deterministic-fixturevalue-3531-deterministic-fixturevalue-3532-deterministic-fixturevalue-3533-deterministic-fixturevalue-3534-deterministic-fixture @`value-3536-deterministic-fixturevalue-3537-deterministic-fixturevalue-3538-deterministic-fixturevalue-3539-deterministic-fixturevalue-3540-deterministic-fixturevalue-3541-deterministic-fixturevalue-3543-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-3544-deterministic-fixturevalue-3545-deterministic-fixturevalue-3546-deterministic-fixturevalue-3547-deterministic-fixturevalue-3548-deterministic-fixturevalue-3550-deterministic-fixturevalue-3551-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`avalue-3552-deterministic-fixturevalue-3553-deterministic-fixturevalue-3554-deterministic-fixturevalue-3555-deterministic-fixturevalue-3557-deterministic-fixturevalue-3558-deterministic-fixturevalue-3559-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`Avalue-3560-deterministic-fixturevalue-3561-deterministic-fixturevalue-3562-deterministic-fixturevalue-3564-deterministic-fixturevalue-3565-deterministic-fixturevalue-3566-deterministic-fixturevalue-3567-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @!`value-3568-deterministic-fixturevalue-3569-deterministic-fixturevalue-3571-deterministic-fixturevalue-3572-deterministic-fixturevalue-3573-deterministic-fixturevalue-3574-deterministic-fixturevalue-3575-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-3576-deterministic-fixturevalue-3578-deterministic-fixturevalue-3579-deterministic-fixturevalue-3580-deterministic-fixturevalue-3581-deterministic-fixturevalue-3582-deterministic-fixturevalue-3583-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-3585-deterministic-fixturevalue-3586-deterministic-fixturevalue-3587-deterministic-fixturevalue-3588-deterministic-fixturevalue-3589-deterministic-fixturevalue-3590-deterministic-fixture @`value-3592-deterministic-fixturevalue-3593-deterministic-fixturevalue-3594-deterministic-fixturevalue-3595-deterministic-fixturevalue-3596-deterministic-fixturevalue-3597-deterministic-fixturevalue-3599-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-3600-deterministic-fixturevalue-3601-deterministic-fixturevalue-3602-deterministic-fixturevalue-3603-deterministic-fixturevalue-3604-deterministic-fixturevalue-3606-deterministic-fixturevalue-3607-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`avalue-3608-deterministic-fixturevalue-3609-deterministic-fixturevalue-3610-deterministic-fixturevalue-3611-deterministic-fixturevalue-3613-deterministic-fixturevalue-3614-deterministic-fixturevalue-3615-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`Avalue-3616-deterministic-fixturevalue-3617-deterministic-fixturevalue-3618-deterministic-fixturevalue-3620-deterministic-fixturevalue-3621-deterministic-fixturevalue-3622-deterministic-fixturevalue-3623-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @!`value-3624-deterministic-fixturevalue-3625-deterministic-fixturevalue-3627-deterministic-fixturevalue-3628-deterministic-fixturevalue-3629-deterministic-fixturevalue-3630-deterministic-fixturevalue-3631-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-3632-deterministic-fixturevalue-3634-deterministic-fixturevalue-3635-deterministic-fixturevalue-3636-deterministic-fixturevalue-3637-deterministic-fixturevalue-3638-deterministic-fixturevalue-3639-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-3641-deterministic-fixturevalue-3642-deterministic-fixturevalue-3643-deterministic-fixturevalue-3644-deterministic-fixturevalue-3645-deterministic-fixturevalue-3646-deterministic-fixture @`value-3648-deterministic-fixturevalue-3649-deterministic-fixturevalue-3650-deterministic-fixturevalue-3651-deterministic-fixturevalue-3652-deterministic-fixturevalue-3653-deterministic-fixturevalue-3655-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-3656-deterministic-fixturevalue-3657-deterministic-fixturevalue-3658-deterministic-fixturevalue-3659-deterministic-fixturevalue-3660-deterministic-fixturevalue-3662-deterministic-fixturevalue-3663-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`avalue-3664-deterministic-fixturevalue-3665-deterministic-fixturevalue-3666-deterministic-fixturevalue-3667-deterministic-fixturevalue-3669-deterministic-fixturevalue-3670-deterministic-fixturevalue-3671-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`Avalue-3672-deterministic-fixturevalue-3673-deterministic-fixturevalue-3674-deterministic-fixturevalue-3676-deterministic-fixturevalue-3677-deterministic-fixturevalue-3678-deterministic-fixturevalue-3679-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @!`value-3680-deterministic-fixturevalue-3681-deterministic-fixturevalue-3683-deterministic-fixturevalue-3684-deterministic-fixturevalue-3685-deterministic-fixturevalue-3686-deterministic-fixturevalue-3687-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-3688-deterministic-fixturevalue-3690-deterministic-fixturevalue-3691-deterministic-fixturevalue-3692-deterministic-fixturevalue-3693-deterministic-fixturevalue-3694-deterministic-fixturevalue-3695-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-3697-deterministic-fixturevalue-3698-deterministic-fixturevalue-3699-deterministic-fixturevalue-3700-deterministic-fixturevalue-3701-deterministic-fixturevalue-3702-deterministic-fixture @`value-3704-deterministic-fixturevalue-3705-deterministic-fixturevalue-3706-deterministic-fixturevalue-3707-deterministic-fixturevalue-3708-deterministic-fixturevalue-3709-deterministic-fixturevalue-3711-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-3712-deterministic-fixturevalue-3713-deterministic-fixturevalue-3714-deterministic-fixturevalue-3715-deterministic-fixturevalue-3716-deterministic-fixturevalue-3718-deterministic-fixturevalue-3719-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`avalue-3720-deterministic-fixturevalue-3721-deterministic-fixturevalue-3722-deterministic-fixturevalue-3723-deterministic-fixturevalue-3725-deterministic-fixturevalue-3726-deterministic-fixturevalue-3727-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`Avalue-3728-deterministic-fixturevalue-3729-deterministic-fixturevalue-3730-deterministic-fixturevalue-3732-deterministic-fixturevalue-3733-deterministic-fixturevalue-3734-deterministic-fixturevalue-3735-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @!`value-3736-deterministic-fixturevalue-3737-deterministic-fixturevalue-3739-deterministic-fixturevalue-3740-deterministic-fixturevalue-3741-deterministic-fixturevalue-3742-deterministic-fixturevalue-3743-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-3744-deterministic-fixturevalue-3746-deterministic-fixturevalue-3747-deterministic-fixturevalue-3748-deterministic-fixturevalue-3749-deterministic-fixturevalue-3750-deterministic-fixturevalue-3751-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-3753-deterministic-fixturevalue-3754-deterministic-fixturevalue-3755-deterministic-fixturevalue-3756-deterministic-fixturevalue-3757-deterministic-fixturevalue-3758-deterministic-fixture @`value-3760-deterministic-fixturevalue-3761-deterministic-fixturevalue-3762-deterministic-fixturevalue-3763-deterministic-fixturevalue-3764-deterministic-fixturevalue-3765-deterministic-fixturevalue-3767-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-3768-deterministic-fixturevalue-3769-deterministic-fixturevalue-3770-deterministic-fixturevalue-3771-deterministic-fixturevalue-3772-deterministic-fixturevalue-3774-deterministic-fixturevalue-3775-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`avalue-3776-deterministic-fixturevalue-3777-deterministic-fixturevalue-3778-deterministic-fixturevalue-3779-deterministic-fixturevalue-3781-deterministic-fixturevalue-3782-deterministic-fixturevalue-3783-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`Avalue-3784-deterministic-fixturevalue-3785-deterministic-fixturevalue-3786-deterministic-fixturevalue-3788-deterministic-fixturevalue-3789-deterministic-fixturevalue-3790-deterministic-fixturevalue-3791-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @!`value-3792-deterministic-fixturevalue-3793-deterministic-fixturevalue-3795-deterministic-fixturevalue-3796-deterministic-fixturevalue-3797-deterministic-fixturevalue-3798-deterministic-fixturevalue-3799-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-3800-deterministic-fixturevalue-3802-deterministic-fixturevalue-3803-deterministic-fixturevalue-3804-deterministic-fixturevalue-3805-deterministic-fixturevalue-3806-deterministic-fixturevalue-3807-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-3809-deterministic-fixturevalue-3810-deterministic-fixturevalue-3811-deterministic-fixturevalue-3812-deterministic-fixturevalue-3813-deterministic-fixturevalue-3814-deterministic-fixture @`value-3816-deterministic-fixturevalue-3817-deterministic-fixturevalue-3818-deterministic-fixturevalue-3819-deterministic-fixturevalue-3820-deterministic-fixturevalue-3821-deterministic-fixturevalue-3823-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-3824-deterministic-fixturevalue-3825-deterministic-fixturevalue-3826-deterministic-fixturevalue-3827-deterministic-fixturevalue-3828-deterministic-fixturevalue-3830-deterministic-fixturevalue-3831-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`avalue-3832-deterministic-fixturevalue-3833-deterministic-fixturevalue-3834-deterministic-fixturevalue-3835-deterministic-fixturevalue-3837-deterministic-fixturevalue-3838-deterministic-fixturevalue-3839-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`Avalue-3840-deterministic-fixturevalue-3841-deterministic-fixturevalue-3842-deterministic-fixturevalue-3844-deterministic-fixturevalue-3845-deterministic-fixturevalue-3846-deterministic-fixturevalue-3847-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @!`value-3848-deterministic-fixturevalue-3849-deterministic-fixturevalue-3851-deterministic-fixturevalue-3852-deterministic-fixturevalue-3853-deterministic-fixturevalue-3854-deterministic-fixturevalue-3855-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-3856-deterministic-fixturevalue-3858-deterministic-fixturevalue-3859-deterministic-fixturevalue-3860-deterministic-fixturevalue-3861-deterministic-fixturevalue-3862-deterministic-fixturevalue-3863-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-3865-deterministic-fixturevalue-3866-deterministic-fixturevalue-3867-deterministic-fixturevalue-3868-deterministic-fixturevalue-3869-deterministic-fixturevalue-3870-deterministic-fixture @`value-3872-deterministic-fixturevalue-3873-deterministic-fixturevalue-3874-deterministic-fixturevalue-3875-deterministic-fixturevalue-3876-deterministic-fixturevalue-3877-deterministic-fixturevalue-3879-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-3880-deterministic-fixturevalue-3881-deterministic-fixturevalue-3882-deterministic-fixturevalue-3883-deterministic-fixturevalue-3884-deterministic-fixturevalue-3886-deterministic-fixturevalue-3887-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`avalue-3888-deterministic-fixturevalue-3889-deterministic-fixturevalue-3890-deterministic-fixturevalue-3891-deterministic-fixturevalue-3893-deterministic-fixturevalue-3894-deterministic-fixturevalue-3895-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`Avalue-3896-deterministic-fixturevalue-3897-deterministic-fixturevalue-3898-deterministic-fixturevalue-3900-deterministic-fixturevalue-3901-deterministic-fixturevalue-3902-deterministic-fixturevalue-3903-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @!`value-3904-deterministic-fixturevalue-3905-deterministic-fixturevalue-3907-deterministic-fixturevalue-3908-deterministic-fixturevalue-3909-deterministic-fixturevalue-3910-deterministic-fixturevalue-3911-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-3912-deterministic-fixturevalue-3914-deterministic-fixturevalue-3915-deterministic-fixturevalue-3916-deterministic-fixturevalue-3917-deterministic-fixturevalue-3918-deterministic-fixturevalue-3919-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-3921-deterministic-fixturevalue-3922-deterministic-fixturevalue-3923-deterministic-fixturevalue-3924-deterministic-fixturevalue-3925-deterministic-fixturevalue-3926-deterministic-fixture @`value-3928-deterministic-fixturevalue-3929-deterministic-fixturevalue-3930-deterministic-fixturevalue-3931-deterministic-fixturevalue-3932-deterministic-fixturevalue-3933-deterministic-fixturevalue-3935-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-3936-deterministic-fixturevalue-3937-deterministic-fixturevalue-3938-deterministic-fixturevalue-3939-deterministic-fixturevalue-3940-deterministic-fixturevalue-3942-deterministic-fixturevalue-3943-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`avalue-3944-deterministic-fixturevalue-3945-deterministic-fixturevalue-3946-deterministic-fixturevalue-3947-deterministic-fixturevalue-3949-deterministic-fixturevalue-3950-deterministic-fixturevalue-3951-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`Avalue-3952-deterministic-fixturevalue-3953-deterministic-fixturevalue-3954-deterministic-fixturevalue-3956-deterministic-fixturevalue-3957-deterministic-fixturevalue-3958-deterministic-fixturevalue-3959-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @!`value-3960-deterministic-fixturevalue-3961-deterministic-fixturevalue-3963-deterministic-fixturevalue-3964-deterministic-fixturevalue-3965-deterministic-fixturevalue-3966-deterministic-fixturevalue-3967-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-3968-deterministic-fixturevalue-3970-deterministic-fixturevalue-3971-deterministic-fixturevalue-3972-deterministic-fixturevalue-3973-deterministic-fixturevalue-3974-deterministic-fixturevalue-3975-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-3977-deterministic-fixturevalue-3978-deterministic-fixturevalue-3979-deterministic-fixturevalue-3980-deterministic-fixturevalue-3981-deterministic-fixturevalue-3982-deterministic-fixture @`value-3984-deterministic-fixturevalue-3985-deterministic-fixturevalue-3986-deterministic-fixturevalue-3987-deterministic-fixturevalue-3988-deterministic-fixturevalue-3989-deterministic-fixturevalue-3991-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-3992-deterministic-fixturevalue-3993-deterministic-fixturevalue-3994-deterministic-fixturevalue-3995-deterministic-fixturevalue-3996-deterministic-fixturevalue-3998-deterministic-fixturevalue-3999-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`avalue-4000-deterministic-fixturevalue-4001-deterministic-fixturevalue-4002-deterministic-fixturevalue-4003-deterministic-fixturevalue-4005-deterministic-fixturevalue-4006-deterministic-fixturevalue-4007-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`Avalue-4008-deterministic-fixturevalue-4009-deterministic-fixturevalue-4010-deterministic-fixturevalue-4012-deterministic-fixturevalue-4013-deterministic-fixturevalue-4014-deterministic-fixturevalue-4015-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @!`value-4016-deterministic-fixturevalue-4017-deterministic-fixturevalue-4019-deterministic-fixturevalue-4020-deterministic-fixturevalue-4021-deterministic-fixturevalue-4022-deterministic-fixturevalue-4023-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-4024-deterministic-fixturevalue-4026-deterministic-fixturevalue-4027-deterministic-fixturevalue-4028-deterministic-fixturevalue-4029-deterministic-fixturevalue-4030-deterministic-fixturevalue-4031-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-4033-deterministic-fixturevalue-4034-deterministic-fixturevalue-4035-deterministic-fixturevalue-4036-deterministic-fixturevalue-4037-deterministic-fixturevalue-4038-deterministic-fixture @`value-4040-deterministic-fixturevalue-4041-deterministic-fixturevalue-4042-deterministic-fixturevalue-4043-deterministic-fixturevalue-4044-deterministic-fixturevalue-4045-deterministic-fixturevalue-4047-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-4048-deterministic-fixturevalue-4049-deterministic-fixturevalue-4050-deterministic-fixturevalue-4051-deterministic-fixturevalue-4052-deterministic-fixturevalue-4054-deterministic-fixturevalue-4055-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`avalue-4056-deterministic-fixturevalue-4057-deterministic-fixturevalue-4058-deterministic-fixturevalue-4059-deterministic-fixturevalue-4061-deterministic-fixturevalue-4062-deterministic-fixturevalue-4063-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`Avalue-4064-deterministic-fixturevalue-4065-deterministic-fixturevalue-4066-deterministic-fixturevalue-4068-deterministic-fixturevalue-4069-deterministic-fixturevalue-4070-deterministic-fixturevalue-4071-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @!`value-4072-deterministic-fixturevalue-4073-deterministic-fixturevalue-4075-deterministic-fixturevalue-4076-deterministic-fixturevalue-4077-deterministic-fixturevalue-4078-deterministic-fixturevalue-4079-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-4080-deterministic-fixturevalue-4082-deterministic-fixturevalue-4083-deterministic-fixturevalue-4084-deterministic-fixturevalue-4085-deterministic-fixturevalue-4086-deterministic-fixturevalue-4087-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-4089-deterministic-fixturevalue-4090-deterministic-fixturevalue-4091-deterministic-fixturevalue-4092-deterministic-fixturevalue-4093-deterministic-fixturevalue-4094-deterministic-fixture  !$'*-036 9<?BEHKNQT> WZ]`cfilor\ ux{~z   #& ),/258;>AD. GJMPSVY\_bL ehknqtwz}j      +  "%(+.14 7:=@CFILOR< UX[^adgjmpZ svy|x      !$'*-0369<?B,EHKNQTWZ]`Jcfilorux{~h  #&),/258;>ADGJMP:SVY\_behknXqtwz}v + " %(+.147:=@*CFILORUX[^Hadgjmpsvy|f  !$'*-0369<?BEHKN8QTWZ]`cfilVorux{~t   +#&),/258;>(ADGJMPSVY\F_behknqtwzd} + "%(+.147:=@CFIL6ORUX[^adgjTmpsvy|r         ! $ ' * - 0 3 6 9 < &? B E H K N Q T W Z D] ` c f i l o r u x b{ ~      + + + + + + + + + + +# +& +) +, +/ +2 +5 +8 +; +> +A +D +G +J +4M +P +S +V +Y +\ +_ +b +e +h +Rk +n +q +t +w +z +} + + + +p + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH $ $ $ $  $ +$ $ $ $ + $ !$ $$ '$ *$ HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH-$ 0$ "3$ 6$ (<$ *?$ ,B$ E$ 0H$ 2K$ 4N$ 6Q$ T$ :W$ >HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH]$ @`$! c$" Df$# Fi$$ Hl$% Jo$& r$' Nu$( Px$* T~$+ $, X$- Z$. \$HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH/ ^$0 $1 b$2 d$3 f$5 $6 l$7 n$8 p$9 r$: $; v$< x$= z$> HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH|$@ $A $B $C $D $E $F $G $H $I $K $L $M $N HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH$O $P $Q $R $S $T $V %W %X %Y  %Z %[ %\ %] %HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH^ %_ %a #%b &%c )%d ,%e /%f 2%g 5%h 8%i ;%j >%l D%m G%n HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHJ%o M%p P%q S%r V%s Y%t \%u _%w e%x h%y k%z n%{ q%| t%} HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHw%~ z% }% % % % % % % % % % % % %HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH % % % "% $% &% % *% ,% 0% % 4% 6% 8% HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH:% % >% @% B% % H% J% L% N% % R% T% V& XHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH& \ +& ^ & `& b& & f& h& j& l"& %& r+& t.& v1& 4&HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH z7& |:& ~=& @& C& F& L& O& R& U& X& [& ^& a& HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHd& g& m& p& s& v& y& |& & & & & & & HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH& & & & & & & & & & & & & & &HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH & & & & & & & & & & & & & & HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH& & & & ' ' '  '  ' ' ' ' ' + ' HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH!' $' '' *' -' 3' $6' &9' (<' *?' B' .E' 0H' 2K' 4N'HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH 8T' :W' <Z' >]' `'! Bc'" Df'# Fi'$ Hl'% o'' Nu'( Px') R{'* ~'+ HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHV', X'- Z'. \'/ '0 `'2 d'3 f'4 '5 j'6 l'7 n'8 p'9 ': tHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH'; v'= z'> '? ~'@ 'A 'B 'C 'D 'E 'F 'H 'I 'J 'HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHK 'L 'M 'N 'O 'P 'Q 'S 'T 'U 'V (W (X (Y  (Z HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH([ (\ (^ (_ (`  (a #(b &(c )(d ,(e /(f 2(g 5(i ;(j HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH>(k A(l D(m G(n J(o M(p P(q S(r V(t \(u _(v b(w e(x h(y k(HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHz n({ q(| t(} w( }( ( ( ( ( ( +( ( ( ( HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH( ( ( ( ( ( ( "( $( ( *( ,( .( ( 2HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH( 4( 6( 8( ( <( @( B( ( F( H( J( L( ( P(HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH R( V) ) Z) \ +) ^ ) `) ) d) f) h) ") n%) p() HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHr+) t.) 1) x4) z7) |:) ~=) C) F) I) L) O) R) U) HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHX) [) ^) d) g) j) m) p) s) v) y) |) ) ) )HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH ) ) ) ) ) ) ) ) ) ) ) ) ) ) HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH) ) ) ) ) ) ) ) ) ) ) ) ) ) HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH) ) ) ) ) ) ) )** * * +***HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH* * +* !* $***-*0*"3*$6*&9*(<*?*,B*HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH.E*2K*N*6Q*8T*:W*<Z*]* @`*!Bc*"Df*$l*%Jo*&Lr*'NHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHu*(Px*){**T~*+V*,X*-Z*/^*0`*1b*2d*3*4h*5j*6l*HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH7n*8*:t*;v*<x*=*>|*?~*@*A*B*C*E*F*GHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH*H*I*J*K*L*M*N*P*Q*R*S*T*U*VHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH+W+X+Y +[+\+]+^+_+` +a#+b&+c)+d,+f2+HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHg5+h8+i;+j>+kA+lD+mG+nJ+oM+qS+rV+sY+t\+u_+vHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHb+we+xh+yk+zn+|t+}w+~z+}++++++ +HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH++++++++++ ++&+(+*+HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH,++0+2+4+6+:+<+>+@++D+F+H+HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHJ++P+R+T+,X,Z,\ +,^ ,,b,f,h,HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH,l",n%,p(,r+,.,v1,x4,|:,=,@,C,F,I,L,HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHO,R,U,[,^,a,d,g,j,m,p,s,v,|,HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH,,,,,,,,,,,,,,HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH,,,,,,,,,,,,,,,HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH,,,,,,,,,,--- -HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH -- --- - !- $- '-*--- 0-"3-$6-&HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH9-<-,B-.E-0H-K-4N-6Q-8T-:W-Z->]-!Bc-"Df-#i-HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH$Hl-%Jo-&Lr-'Nu-(x-)R{-*T~-,X---.\-/^-0`-1b-2-3HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHf-4h-5j-7-8p-9r-:t-;v-<-=z->|-?~-@-B-CHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH-D-E-F-G-H-I-J-K-M-N-O-P-Q-R-HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHS-T-U-V.X.Y .Z.[.\.].^._.` .a#.cHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH).d,.e/.f2.g5.h8.i;.j>.kA.lD.nJ.oM.pP.qS.rHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHV.sY.t\.u_.vb.we.yk.zn.{q.|t.}w.~z.}...HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH.. +. ......... ..HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH$.&.(.*....0.4..8.:.<.>..BHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH.D.F..L.N.P.R..V/X/Z/\ +/`/b/HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHd/f//j/l"/n%/p(/+/v1/x4/z7/:/~=/@/HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHC/F/I/L/R/U/X/[/^/a/d/g/j/m/HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHs/v/y/|////////////HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH//////////////HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH//////////////HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHgreenblueredHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHgreenblueredHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHgreenblueredHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHgreenblueredHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHgreenblueredHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHgreenblueredHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHgreenblueredHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHgreenblueredHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHgreenblueredHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHgreenblueredHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHgreenblueredHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHgreenblueredHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHgreenblueredHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHgreenblueredHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHgreenblueredHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH@@@@@@@@@@@@ @   !@!!!"@"""#@###$@$$$%@%%%&@&&&'@'''(@((()@)))*@***+@+++,@,,,-@---.@.../@///0@0001@1112@2223@3334@4445@5556@6667@7778@8889@999:@:::;@;;;<@<<<=@===>@>>>?@???@@@@@A@AAAB@BBBC@CCCD@DDDE@EEEF@FFFG@GGGH@HHHI@IIIJ@JJJK@KKKL@LLLM@MMMN@NNNO@OOOP@PPPQ@QQQR@RRRS@SSST@TTTU@UUUV@VVVW@WWWX@XXXY@YYYZ@ZZZ[@[[[\@\\\]@]]]^@^^^_@___`@```a@aaab@bbbc@cccd@ddde@eeef@fffg@gggh@hhhi@iiij@jjjk@kkkl@lllm@mmmn@nnno@ooop@pppq@qqqr@rrrs@ssst@tttu@uuuv@vvvw@wwwx@xxxy@yyyz@zzz{@{{{|@|||}@}}}~@~~~@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ @    +@ + + + @    @    @   @@@@@@blob-4096-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHvalue-4096-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH0HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHgreenblueredHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHo HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH + +id *int328 +Cname *string08R" +lance-encoding:compressionnone +items *list08 +item *int3208 +ecategory *dict:string:int8:false08BR. +&lance-encoding:dict-values-compressionnone +Bblob * large_binary08R +lance-encoding:blobtrue +)' +% +/lance.encodings.ColumnEncoding +? +="20 +. +/lance.encodings.ArrayEncoding  + + + + ? +="20 +. +/lance.encodings.ArrayEncoding  + + + + ? +="20 +. +/lance.encodings.ArrayEncoding  + + + + ? +="20 +. +/lance.encodings.ArrayEncoding  + + + + ? +="20 +. +/lance.encodings.ArrayEncoding  + + + + ? +="20 +. +/lance.encodings.ArrayEncoding  + + + + ? +="20 +. +/lance.encodings.ArrayEncoding  + + + + ? +="20 +. +/lance.encodings.ArrayEncoding  + + + + ? +="20 +. +/lance.encodings.ArrayEncoding  + + + + ? +="20 +. +/lance.encodings.ArrayEncoding  + + + + ? +="20 +. +/lance.encodings.ArrayEncoding  + + + + ? +="20 +. +/lance.encodings.ArrayEncoding  + + + + ? +="20 +. +/lance.encodings.ArrayEncoding  + + + + ? +="20 +. +/lance.encodings.ArrayEncoding  + + + + ? +="20 +. +/lance.encodings.ArrayEncoding  + + + + ? +="20 +. +/lance.encodings.ArrayEncoding  + + + + ? +0"20 +. +/lance.encodings.ArrayEncoding  + + + + ? + ="20 +. +/lance.encodings.ArrayEncoding  + + + + ? + ="20 +. +/lance.encodings.ArrayEncoding  + + + + ? + ="20 +. +/lance.encodings.ArrayEncoding  + + + + ? + ="20 +. +/lance.encodings.ArrayEncoding  + + + + ? + ="20 +. +/lance.encodings.ArrayEncoding  + + + + ? + ="20 +. +/lance.encodings.ArrayEncoding  + + + + ? + ="20 +. +/lance.encodings.ArrayEncoding  + + + + ? + ="20 +. +/lance.encodings.ArrayEncoding  + + + + ? + ="20 +. +/lance.encodings.ArrayEncoding  + + + + ? + ="20 +. +/lance.encodings.ArrayEncoding  + + + + ? + ="20 +. +/lance.encodings.ArrayEncoding  + + + + ? + ="20 +. +/lance.encodings.ArrayEncoding  + + + + ? + ="20 +. +/lance.encodings.ArrayEncoding  + + + + ? + ="20 +. +/lance.encodings.ArrayEncoding  + + + + ? + ="20 +. +/lance.encodings.ArrayEncoding  + + + + ? + ="20 +. +/lance.encodings.ArrayEncoding  + + + + ? + 0"20 +. +/lance.encodings.ArrayEncoding  + + + + ? +="20 +. +/lance.encodings.ArrayEncoding  + + + + ? +="20 +. +/lance.encodings.ArrayEncoding  + + + + ? +="20 +. +/lance.encodings.ArrayEncoding  + + + + ? +="20 +. +/lance.encodings.ArrayEncoding  + + + + ? +="20 +. +/lance.encodings.ArrayEncoding  + + + + ? +="20 +. +/lance.encodings.ArrayEncoding  + + + + ? +="20 +. +/lance.encodings.ArrayEncoding  + + + + ? +="20 +. +/lance.encodings.ArrayEncoding  + + + + ? +="20 +. +/lance.encodings.ArrayEncoding  + + + + ? +="20 +. +/lance.encodings.ArrayEncoding  + + + + ? +="20 +. +/lance.encodings.ArrayEncoding  + + + + ? +="20 +. +/lance.encodings.ArrayEncoding  + + + + ? +="20 +. +/lance.encodings.ArrayEncoding  + + + + ? +="20 +. +/lance.encodings.ArrayEncoding  + + + + ? +="20 +. +/lance.encodings.ArrayEncoding  + + + + ? +="20 +. +/lance.encodings.ArrayEncoding  + + + + ? +0"20 +. +/lance.encodings.ArrayEncoding  + + + + ? + ="20 +. +/lance.encodings.ArrayEncoding  + + + + ? + ="20 +. +/lance.encodings.ArrayEncoding  + + + + ? + ="20 +. +/lance.encodings.ArrayEncoding  + + + + ? + ="20 +. +/lance.encodings.ArrayEncoding  + + + + ? + ="20 +. +/lance.encodings.ArrayEncoding  + + + + ? + ="20 +. +/lance.encodings.ArrayEncoding  + + + + ? + ="20 +. +/lance.encodings.ArrayEncoding  + + + + ? + ="20 +. +/lance.encodings.ArrayEncoding  + + + + ? + ="20 +. +/lance.encodings.ArrayEncoding  + + + + ? + ="20 +. +/lance.encodings.ArrayEncoding  + + + + ? + ="20 +. +/lance.encodings.ArrayEncoding  + + + + ? + ="20 +. +/lance.encodings.ArrayEncoding  + + + + ? + ="20 +. +/lance.encodings.ArrayEncoding  + + + + ? + ="20 +. +/lance.encodings.ArrayEncoding  + + + + ? + ="20 +. +/lance.encodings.ArrayEncoding  + + + + ? + ="20 +. +/lance.encodings.ArrayEncoding  + + + + ? + 0"20 +. +/lance.encodings.ArrayEncoding  + + + + > +%"20 +. +/lance.encodings.ArrayEncoding  + + + +  +)' +% +/lance.encodings.ColumnEncoding +\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ + @"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ + @"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ + @"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ + @"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ + @"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ + @"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ + @"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ + @"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ + @"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ + @"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ + @"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ + @"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ + @"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ + @"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ + @"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ + @"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ + @"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ + @"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ + @"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ + @"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ + @"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ + @"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ + @"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ + @"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ + @"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ + @"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ + @"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ + @"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ + @"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ + @"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ + @"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ + @"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ + @"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ + @"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ + @"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ + @"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ + @"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ + @"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ + @"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +!!@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +!!@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +!!@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +!!@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +!!@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +!!@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +!!@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +!!@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +!!@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +!!@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +!!@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +!!@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +!!@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +!!@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +!!@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +!!@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +!!@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +!!@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +!!@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +!!@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +!!@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +!!@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +!!@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +!!@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +!!@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +!!@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +!!@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +!!@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +!!@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +!!@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +!!@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +!!@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +!!@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +!!@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +!!@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +!!@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +!!@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +!!@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +!!@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +!!@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +!!@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +!!@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +!!@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +!!@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +!!@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +!!@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +!!@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +!!@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +!!@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +!!@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +!!@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +!!@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +!!@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +""@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +""@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +""@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +""@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +""@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +""@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +""@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +""@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +""@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +""@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +""@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +""@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +""@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +""@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +""@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +""@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +""@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +""@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +""@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +""@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +""@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +""@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +""@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +""@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +""@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +""@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +""@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +""@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +""@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +""@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +""@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +""@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +""@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +""@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +""@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +""@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +""@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +""@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +""@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +""@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +""@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +""@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +""@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +""@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +""@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +""@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +""@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +""@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +""@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +""@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +""@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +""@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +noneZ +%% "JH +F +/lance.encodings.ArrayEncoding$2" +  + + + +@ + +none! +)' +% +/lance.encodings.ColumnEncoding +J +@"<: +8 +/lance.encodings.ArrayEncoding" +  + + + +@J +@"<: +8 +/lance.encodings.ArrayEncoding" +  + + + +@J +@"<: +8 +/lance.encodings.ArrayEncoding" +  + + + +@J +"@"<: +8 +/lance.encodings.ArrayEncoding" +  + + + +@F +%":8 +6 +/lance.encodings.ArrayEncoding" +  + + + +@ +)' +% +/lance.encodings.ColumnEncoding +M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M +&"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M +)"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M +)"<: +8 +/lance.encodings.ArrayEncoding + + + M +##+"<: +8 +/lance.encodings.ArrayEncoding + + + M +##+"<: +8 +/lance.encodings.ArrayEncoding + + + M +##+"<: +8 +/lance.encodings.ArrayEncoding + + + M +##+"<: +8 +/lance.encodings.ArrayEncoding + + + M +##+"<: +8 +/lance.encodings.ArrayEncoding + + + M +##+"<: +8 +/lance.encodings.ArrayEncoding + + + M +##+"<: +8 +/lance.encodings.ArrayEncoding + + + M +##+"<: +8 +/lance.encodings.ArrayEncoding + + + M +##+"<: +8 +/lance.encodings.ArrayEncoding + + + M +##+"<: +8 +/lance.encodings.ArrayEncoding + + + M +##+"<: +8 +/lance.encodings.ArrayEncoding + + + M +##+"<: +8 +/lance.encodings.ArrayEncoding + + + M +##+"<: +8 +/lance.encodings.ArrayEncoding + + + M +##+"<: +8 +/lance.encodings.ArrayEncoding + + + M +##+"<: +8 +/lance.encodings.ArrayEncoding + + + M +##+"<: +8 +/lance.encodings.ArrayEncoding + + + M +##+"<: +8 +/lance.encodings.ArrayEncoding + + + M +##+"<: +8 +/lance.encodings.ArrayEncoding + + + M +##+"<: +8 +/lance.encodings.ArrayEncoding + + + M +##+"<: +8 +/lance.encodings.ArrayEncoding + + + M +##+"<: +8 +/lance.encodings.ArrayEncoding + + + M +##+"<: +8 +/lance.encodings.ArrayEncoding + + + M +##+"<: +8 +/lance.encodings.ArrayEncoding + + + M +##+"<: +8 +/lance.encodings.ArrayEncoding + + + M +##+"<: +8 +/lance.encodings.ArrayEncoding + + + M +##+"<: +8 +/lance.encodings.ArrayEncoding + + + M +##+"<: +8 +/lance.encodings.ArrayEncoding + + + M +##+"<: +8 +/lance.encodings.ArrayEncoding + + + M +##+"<: +8 +/lance.encodings.ArrayEncoding + + + M +##+"<: +8 +/lance.encodings.ArrayEncoding + + + M +##+"<: +8 +/lance.encodings.ArrayEncoding + + + M +##+"<: +8 +/lance.encodings.ArrayEncoding + + + M +#$+"<: +8 +/lance.encodings.ArrayEncoding + + + M +$$+"<: +8 +/lance.encodings.ArrayEncoding + + + M +$$+"<: +8 +/lance.encodings.ArrayEncoding + + + M +$$+"<: +8 +/lance.encodings.ArrayEncoding + + + M +$$+"<: +8 +/lance.encodings.ArrayEncoding + + + M +$$+"<: +8 +/lance.encodings.ArrayEncoding + + + M +$$+"<: +8 +/lance.encodings.ArrayEncoding + + + M +$$+"<: +8 +/lance.encodings.ArrayEncoding + + + M +$$+"<: +8 +/lance.encodings.ArrayEncoding + + + M +$$+"<: +8 +/lance.encodings.ArrayEncoding + + + M +$$+"<: +8 +/lance.encodings.ArrayEncoding + + + M +$$+"<: +8 +/lance.encodings.ArrayEncoding + + + M +$$+"<: +8 +/lance.encodings.ArrayEncoding + + + M +$$+"<: +8 +/lance.encodings.ArrayEncoding + + + M +$$+"<: +8 +/lance.encodings.ArrayEncoding + + + M +$$+"<: +8 +/lance.encodings.ArrayEncoding + + + M +$$+"<: +8 +/lance.encodings.ArrayEncoding + + + M +$$+"<: +8 +/lance.encodings.ArrayEncoding + + + M +$$+"<: +8 +/lance.encodings.ArrayEncoding + + + M +$$+"<: +8 +/lance.encodings.ArrayEncoding + + + M +$$+"<: +8 +/lance.encodings.ArrayEncoding + + + M +$$+"<: +8 +/lance.encodings.ArrayEncoding + + + M +$$+"<: +8 +/lance.encodings.ArrayEncoding + + + M +$$+"<: +8 +/lance.encodings.ArrayEncoding + + + M +$$+"<: +8 +/lance.encodings.ArrayEncoding + + + M +$$+"<: +8 +/lance.encodings.ArrayEncoding + + + M +$$+"<: +8 +/lance.encodings.ArrayEncoding + + + M +$$+"<: +8 +/lance.encodings.ArrayEncoding + + + M +$$+"<: +8 +/lance.encodings.ArrayEncoding + + + M +$$+"<: +8 +/lance.encodings.ArrayEncoding + + + M +$$+"<: +8 +/lance.encodings.ArrayEncoding + + + M +$$+"<: +8 +/lance.encodings.ArrayEncoding + + + M +$$)"<: +8 +/lance.encodings.ArrayEncoding + + + L +%%"<: +8 +/lance.encodings.ArrayEncoding + + + L +%%"<: +8 +/lance.encodings.ArrayEncoding + + + L +%%"<: +8 +/lance.encodings.ArrayEncoding + + +  +)' +% +/lance.encodings.ColumnEncoding +l + E E"XV +T +/lance.encodings.ArrayEncoding2:0 +  + + + +2 + + + + +@ + l + E E"XV +T +/lance.encodings.ArrayEncoding2:0 +  + + + +2 + + + + +@ + l + E E"XV +T +/lance.encodings.ArrayEncoding2:0 +  + + + +2 + + + + +@ + l + E E"XV +T +/lance.encodings.ArrayEncoding2:0 +  + + + +2 + + + + +@ + l + E E"XV +T +/lance.encodings.ArrayEncoding2:0 +  + + + +2 + + + + +@ + l + E E"XV +T +/lance.encodings.ArrayEncoding2:0 +  + + + +2 + + + + +@ + l + E E"XV +T +/lance.encodings.ArrayEncoding2:0 +  + + + +2 + + + + +@ + l + E E"XV +T +/lance.encodings.ArrayEncoding2:0 +  + + + +2 + + + + +@ + l + E E"XV +T +/lance.encodings.ArrayEncoding2:0 +  + + + +2 + + + + +@ + l + E E"XV +T +/lance.encodings.ArrayEncoding2:0 +  + + + +2 + + + + +@ + l + E E"XV +T +/lance.encodings.ArrayEncoding2:0 +  + + + +2 + + + + +@ + l + E E"XV +T +/lance.encodings.ArrayEncoding2:0 +  + + + +2 + + + + +@ + l + E E"XV +T +/lance.encodings.ArrayEncoding2:0 +  + + + +2 + + + + +@ + l + E E"XV +T +/lance.encodings.ArrayEncoding2:0 +  + + + +2 + + + + +@ + l + : :"XV +T +/lance.encodings.ArrayEncoding2:0 +  + + + +2 + + + + +@ + l + E E"XV +T +/lance.encodings.ArrayEncoding2:0 +  + + + +2 + + + + +@ + l + E E"XV +T +/lance.encodings.ArrayEncoding2:0 +  + + + +2 + + + + +@ + l + E E"XV +T +/lance.encodings.ArrayEncoding2:0 +  + + + +2 + + + + +@ + l + E E"XV +T +/lance.encodings.ArrayEncoding2:0 +  + + + +2 + + + + +@ + l + E E"XV +T +/lance.encodings.ArrayEncoding2:0 +  + + + +2 + + + + +@ + l + E E"XV +T +/lance.encodings.ArrayEncoding2:0 +  + + + +2 + + + + +@ + l + E E"XV +T +/lance.encodings.ArrayEncoding2:0 +  + + + +2 + + + + +@ + l + E E"XV +T +/lance.encodings.ArrayEncoding2:0 +  + + + +2 + + + + +@ + l + E E"XV +T +/lance.encodings.ArrayEncoding2:0 +  + + + +2 + + + + +@ + l + E E"XV +T +/lance.encodings.ArrayEncoding2:0 +  + + + +2 + + + + +@ + l + E E"XV +T +/lance.encodings.ArrayEncoding2:0 +  + + + +2 + + + + +@ + l + E E"XV +T +/lance.encodings.ArrayEncoding2:0 +  + + + +2 + + + + +@ + l + E E"XV +T +/lance.encodings.ArrayEncoding2:0 +  + + + +2 + + + + +@ + l + E E"XV +T +/lance.encodings.ArrayEncoding2:0 +  + + + +2 + + + + +@ + l + : :"XV +T +/lance.encodings.ArrayEncoding2:0 +  + + + +2 + + + + +@ + l + E E"XV +T +/lance.encodings.ArrayEncoding2:0 +  + + + +2 + + + + +@ + l + E E"XV +T +/lance.encodings.ArrayEncoding2:0 +  + + + +2 + + + + +@ + l + E E"XV +T +/lance.encodings.ArrayEncoding2:0 +  + + + +2 + + + + +@ + l + E E"XV +T +/lance.encodings.ArrayEncoding2:0 +  + + + +2 + + + + +@ + l + E E"XV +T +/lance.encodings.ArrayEncoding2:0 +  + + + +2 + + + + +@ + l + E E"XV +T +/lance.encodings.ArrayEncoding2:0 +  + + + +2 + + + + +@ + l + E E"XV +T +/lance.encodings.ArrayEncoding2:0 +  + + + +2 + + + + +@ + l + E E"XV +T +/lance.encodings.ArrayEncoding2:0 +  + + + +2 + + + + +@ + l + E E"XV +T +/lance.encodings.ArrayEncoding2:0 +  + + + +2 + + + + +@ + l + E E"XV +T +/lance.encodings.ArrayEncoding2:0 +  + + + +2 + + + + +@ + l + E E"XV +T +/lance.encodings.ArrayEncoding2:0 +  + + + +2 + + + + +@ + l + E E"XV +T +/lance.encodings.ArrayEncoding2:0 +  + + + +2 + + + + +@ + l + E E"XV +T +/lance.encodings.ArrayEncoding2:0 +  + + + +2 + + + + +@ + l + E E"XV +T +/lance.encodings.ArrayEncoding2:0 +  + + + +2 + + + + +@ + l + : :"XV +T +/lance.encodings.ArrayEncoding2:0 +  + + + +2 + + + + +@ + l + $$$E E"XV +T +/lance.encodings.ArrayEncoding2:0 +  + + + +2 + + + + +@ + l + $$$E E"XV +T +/lance.encodings.ArrayEncoding2:0 +  + + + +2 + + + + +@ + l + $$$E E"XV +T +/lance.encodings.ArrayEncoding2:0 +  + + + +2 + + + + +@ + l + $$$E E"XV +T +/lance.encodings.ArrayEncoding2:0 +  + + + +2 + + + + +@ + l + $$$E E"XV +T +/lance.encodings.ArrayEncoding2:0 +  + + + +2 + + + + +@ + l + $$$E E"XV +T +/lance.encodings.ArrayEncoding2:0 +  + + + +2 + + + + +@ + l + $$$E E"XV +T +/lance.encodings.ArrayEncoding2:0 +  + + + +2 + + + + +@ + l + $$$E E"XV +T +/lance.encodings.ArrayEncoding2:0 +  + + + +2 + + + + +@ + l + $$$E E"XV +T +/lance.encodings.ArrayEncoding2:0 +  + + + +2 + + + + +@ + l + $$$E E"XV +T +/lance.encodings.ArrayEncoding2:0 +  + + + +2 + + + + +@ + l + $$$E E"XV +T +/lance.encodings.ArrayEncoding2:0 +  + + + +2 + + + + +@ + l + $$$E E"XV +T +/lance.encodings.ArrayEncoding2:0 +  + + + +2 + + + + +@ + l + $$$E E"XV +T +/lance.encodings.ArrayEncoding2:0 +  + + + +2 + + + + +@ + l + $$$E E"XV +T +/lance.encodings.ArrayEncoding2:0 +  + + + +2 + + + + +@ + l + $$$: :"XV +T +/lance.encodings.ArrayEncoding2:0 +  + + + +2 + + + + +@ + l + %%% "XV +T +/lance.encodings.ArrayEncoding2:0 +  + + + +2 + + + + +@ +  +-+ +) +/lance.encodings.ColumnEncoding + +S +@"FD +B +/lance.encodings.ArrayEncoding J +  + + + +@ +  + + + +@S +@"FD +B +/lance.encodings.ArrayEncoding J +  + + + +@ +  + + + +@S +@"FD +B +/lance.encodings.ArrayEncoding J +  + + + +@ +  + + + +@S +@"FD +B +/lance.encodings.ArrayEncoding J +  + + + +@ +  + + + +@S +@"FD +B +/lance.encodings.ArrayEncoding J +  + + + +@ +  + + + +@S +@"FD +B +/lance.encodings.ArrayEncoding J +  + + + +@ +  + + + +@S +@"FD +B +/lance.encodings.ArrayEncoding J +  + + + +@ +  + + + +@S +@"FD +B +/lance.encodings.ArrayEncoding J +  + + + +@ +  + + + +@S +@"FD +B +/lance.encodings.ArrayEncoding J +  + + + +@ +  + + + +@S +@"FD +B +/lance.encodings.ArrayEncoding J +  + + + +@ +  + + + +@S + @"FD +B +/lance.encodings.ArrayEncoding J +  + + + +@ +  + + + +@S + @"FD +B +/lance.encodings.ArrayEncoding J +  + + + +@ +  + + + +@S + @"FD +B +/lance.encodings.ArrayEncoding J +  + + + +@ +  + + + +@S + @"FD +B +/lance.encodings.ArrayEncoding J +  + + + +@ +  + + + +@S + @"FD +B +/lance.encodings.ArrayEncoding J +  + + + +@ +  + + + +@S + @"FD +B +/lance.encodings.ArrayEncoding J +  + + + +@ +  + + + +@S +@"FD +B +/lance.encodings.ArrayEncoding J +  + + + +@ +  + + + +@S +@"FD +B +/lance.encodings.ArrayEncoding J +  + + + +@ +  + + + +@S +@"FD +B +/lance.encodings.ArrayEncoding J +  + + + +@ +  + + + +@S +@"FD +B +/lance.encodings.ArrayEncoding J +  + + + +@ +  + + + +@S +@"FD +B +/lance.encodings.ArrayEncoding J +  + + + +@ +  + + + +@S +@"FD +B +/lance.encodings.ArrayEncoding J +  + + + +@ +  + + + +@S +@"FD +B +/lance.encodings.ArrayEncoding J +  + + + +@ +  + + + +@S +@"FD +B +/lance.encodings.ArrayEncoding J +  + + + +@ +  + + + +@S +@"FD +B +/lance.encodings.ArrayEncoding J +  + + + +@ +  + + + +@S +@"FD +B +/lance.encodings.ArrayEncoding J +  + + + +@ +  + + + +@S +@"FD +B +/lance.encodings.ArrayEncoding J +  + + + +@ +  + + + +@S +@"FD +B +/lance.encodings.ArrayEncoding J +  + + + +@ +  + + + +@S +@"FD +B +/lance.encodings.ArrayEncoding J +  + + + +@ +  + + + +@S +@"FD +B +/lance.encodings.ArrayEncoding J +  + + + +@ +  + + + +@S +@"FD +B +/lance.encodings.ArrayEncoding J +  + + + +@ +  + + + +@S +@"FD +B +/lance.encodings.ArrayEncoding J +  + + + +@ +  + + + +@S +@"FD +B +/lance.encodings.ArrayEncoding J +  + + + +@ +  + + + +@S +@"FD +B +/lance.encodings.ArrayEncoding J +  + + + +@ +  + + + +@S +@"FD +B +/lance.encodings.ArrayEncoding J +  + + + +@ +  + + + +@S +@"FD +B +/lance.encodings.ArrayEncoding J +  + + + +@ +  + + + +@S +@"FD +B +/lance.encodings.ArrayEncoding J +  + + + +@ +  + + + +@S +@"FD +B +/lance.encodings.ArrayEncoding J +  + + + +@ +  + + + +@S +@"FD +B +/lance.encodings.ArrayEncoding J +  + + + +@ +  + + + +@S +@"FD +B +/lance.encodings.ArrayEncoding J +  + + + +@ +  + + + +@S +@"FD +B +/lance.encodings.ArrayEncoding J +  + + + +@ +  + + + +@S +@"FD +B +/lance.encodings.ArrayEncoding J +  + + + +@ +  + + + +@S +@"FD +B +/lance.encodings.ArrayEncoding J +  + + + +@ +  + + + +@S +@"FD +B +/lance.encodings.ArrayEncoding J +  + + + +@ +  + + + +@S +@"FD +B +/lance.encodings.ArrayEncoding J +  + + + +@ +  + + + +@S +@"FD +B +/lance.encodings.ArrayEncoding J +  + + + +@ +  + + + +@S +@"FD +B +/lance.encodings.ArrayEncoding J +  + + + +@ +  + + + +@S +@"FD +B +/lance.encodings.ArrayEncoding J +  + + + +@ +  + + + +@S +$@"FD +B +/lance.encodings.ArrayEncoding J +  + + + +@ +  + + + +@S +$@"FD +B +/lance.encodings.ArrayEncoding J +  + + + +@ +  + + + +@S +$@"FD +B +/lance.encodings.ArrayEncoding J +  + + + +@ +  + + + +@S +$@"FD +B +/lance.encodings.ArrayEncoding J +  + + + +@ +  + + + +@S +$@"FD +B +/lance.encodings.ArrayEncoding J +  + + + +@ +  + + + +@S +%@"FD +B +/lance.encodings.ArrayEncoding J +  + + + +@ +  + + + +@S +%@"FD +B +/lance.encodings.ArrayEncoding J +  + + + +@ +  + + + +@S +%@"FD +B +/lance.encodings.ArrayEncoding J +  + + + +@ +  + + + +@S +%@"FD +B +/lance.encodings.ArrayEncoding J +  + + + +@ +  + + + +@S +%@"FD +B +/lance.encodings.ArrayEncoding J +  + + + +@ +  + + + +@S +%@"FD +B +/lance.encodings.ArrayEncoding J +  + + + +@ +  + + + +@S +%@"FD +B +/lance.encodings.ArrayEncoding J +  + + + +@ +  + + + +@S +%@"FD +B +/lance.encodings.ArrayEncoding J +  + + + +@ +  + + + +@S +%@"FD +B +/lance.encodings.ArrayEncoding J +  + + + +@ +  + + + +@S +%@"FD +B +/lance.encodings.ArrayEncoding J +  + + + +@ +  + + + +@S +%@"FD +B +/lance.encodings.ArrayEncoding J +  + + + +@ +  + + + +@R +%"FD +B +/lance.encodings.ArrayEncoding J +  + + + +@ +  + + + +@t 6 B +`D +QQ +a +@s Gt +5 +LANC \ No newline at end of file diff --git a/rust/lance-file/test_data/exact_versions/v2_0_mini.lance b/rust/lance-file/test_data/exact_versions/v2_0_mini.lance new file mode 100644 index 00000000000..9de22610e08 Binary files /dev/null and b/rust/lance-file/test_data/exact_versions/v2_0_mini.lance differ diff --git a/rust/lance-file/test_data/exact_versions/v2_0_self_described.lance b/rust/lance-file/test_data/exact_versions/v2_0_self_described.lance new file mode 100644 index 00000000000..13b103c7761 Binary files /dev/null and b/rust/lance-file/test_data/exact_versions/v2_0_self_described.lance differ diff --git a/rust/lance-file/test_data/exact_versions/v2_1.lance b/rust/lance-file/test_data/exact_versions/v2_1.lance new file mode 100644 index 00000000000..7ac8ae0c845 --- /dev/null +++ b/rust/lance-file/test_data/exact_versions/v2_1.lance @@ -0,0 +1,639 @@ +blob-0000-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0001-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0002-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0003-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0004-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0005-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0006-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0007-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0008-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0009-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0010-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0011-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0012-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0013-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0014-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0015-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0016-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0017-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0018-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0019-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0020-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0021-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0022-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0023-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0024-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0025-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0026-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0027-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0028-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0029-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0030-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0031-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0032-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0033-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0034-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0035-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0036-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0037-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0038-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0039-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0040-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0041-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0042-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0043-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0044-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0045-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0046-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0047-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0048-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0049-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0050-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0051-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0052-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0053-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0054-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0055-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0056-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0057-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0058-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0059-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0060-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0061-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0062-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0063-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0064-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0065-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0066-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0067-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0068-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0069-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0070-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0071-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0072-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0073-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0074-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0075-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0076-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0077-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0078-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0079-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0080-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0081-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0082-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0083-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0084-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0085-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0086-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0087-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0088-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0089-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0090-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0091-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0092-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0093-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0094-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0095-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0096-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0097-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0098-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0099-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0100-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0101-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0102-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0103-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0104-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0105-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0106-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0107-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0108-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0109-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0110-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0111-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0112-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0113-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0114-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0115-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0116-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0117-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0118-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0119-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0120-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0121-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0122-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0123-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0124-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0125-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0126-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0127-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0128-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0129-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0130-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0131-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0132-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0133-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0134-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0135-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0136-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0137-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0138-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0139-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0140-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0141-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0142-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0143-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0144-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0145-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0146-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0147-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0148-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0149-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0150-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0151-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0152-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0153-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0154-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0155-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0156-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0157-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0158-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0159-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0160-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0161-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0162-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0163-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0164-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0165-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0166-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0167-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0168-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0169-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0170-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0171-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0172-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0173-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0174-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0175-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0176-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0177-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0178-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0179-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0180-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0181-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0182-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0183-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0184-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0185-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0186-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0187-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0188-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0189-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0190-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0191-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0192-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0193-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0194-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0195-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0196-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0197-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0198-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0199-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0200-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0201-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0202-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0203-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0204-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0205-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0206-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0207-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0208-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0209-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0210-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0211-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0212-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0213-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0214-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0215-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0216-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0217-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0218-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0219-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0220-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0221-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0222-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0223-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0224-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0225-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0226-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0227-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0228-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0229-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0230-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0231-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0232-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0233-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0234-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0235-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0236-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0237-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0238-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0239-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0240-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0241-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0242-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0243-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0244-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0245-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0246-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0247-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0248-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0249-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0250-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0251-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0252-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0253-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0254-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0255-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0256-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0257-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0258-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0259-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0260-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0261-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0262-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0263-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0264-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0265-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0266-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0267-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0268-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0269-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0270-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0271-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0272-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0273-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0274-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0275-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0276-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0277-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0278-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0279-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0280-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0281-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0282-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0283-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0284-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0285-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0286-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0287-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0288-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0289-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0290-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0291-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0292-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0293-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0294-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0295-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0296-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0297-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0298-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0299-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0300-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0301-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0302-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0303-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0304-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0305-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0306-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0307-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0308-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0309-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0310-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0311-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0312-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0313-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0314-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0315-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0316-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0317-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0318-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0319-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0320-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0321-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0322-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0323-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0324-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0325-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0326-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0327-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0328-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0329-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0330-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0331-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0332-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0333-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0334-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0335-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0336-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0337-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0338-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0339-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0340-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0341-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0342-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0343-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0344-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0345-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0346-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0347-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0348-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0349-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0350-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0351-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0352-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0353-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0354-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0355-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0356-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0357-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0358-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0359-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0360-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0361-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0362-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0363-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0364-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0365-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0366-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0367-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0368-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0369-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0370-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0371-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0372-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0373-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0374-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0375-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0376-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0377-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0378-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0379-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0380-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0381-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0382-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0383-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0384-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0385-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0386-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0387-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0388-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0389-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0390-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0391-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0392-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0393-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0394-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0395-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0396-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0397-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0398-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0399-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0400-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0401-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0402-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0403-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0404-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0405-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0406-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0407-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0408-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0409-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0410-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0411-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0412-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0413-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0414-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0415-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0416-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0417-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0418-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0419-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0420-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0421-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0422-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0423-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0424-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0425-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0426-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0427-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0428-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0429-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0430-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0431-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0432-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0433-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0434-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0435-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0436-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0437-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0438-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0439-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0440-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0441-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0442-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0443-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0444-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0445-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0446-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0447-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0448-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0449-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0450-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0451-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0452-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0453-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0454-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0455-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0456-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0457-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0458-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0459-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0460-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0461-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0462-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0463-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0464-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0465-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0466-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0467-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0468-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0469-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0470-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0471-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0472-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0473-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0474-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0475-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0476-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0477-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0478-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0479-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0480-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0481-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0482-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0483-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0484-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0485-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0486-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0487-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0488-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0489-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0490-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0491-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0492-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0493-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0494-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0495-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0496-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0497-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0498-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0499-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0500-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0501-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0502-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0503-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0504-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0505-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0506-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0507-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0508-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0509-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0510-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0511-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0512-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0513-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0514-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0515-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0516-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0517-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0518-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0519-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0520-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0521-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0522-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0523-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0524-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0525-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0526-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0527-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0528-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0529-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0530-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0531-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0532-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0533-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0534-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0535-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0536-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0537-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0538-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0539-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0540-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0541-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0542-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0543-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0544-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0545-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0546-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0547-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0548-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0549-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0550-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0551-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0552-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0553-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0554-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0555-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0556-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0557-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0558-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0559-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0560-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0561-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0562-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0563-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0564-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0565-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0566-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0567-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0568-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0569-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0570-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0571-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0572-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0573-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0574-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0575-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0576-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0577-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0578-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0579-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0580-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0581-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0582-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0583-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0584-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0585-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0586-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0587-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0588-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0589-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0590-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0591-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0592-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0593-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0594-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0595-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0596-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0597-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0598-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0599-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0600-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0601-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0602-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0603-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0604-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0605-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0606-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0607-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0608-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0609-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0610-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0611-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0612-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0613-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0614-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0615-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0616-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0617-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0618-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0619-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0620-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0621-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0622-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0623-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0624-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0625-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0626-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0627-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0628-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0629-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0630-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0631-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0632-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0633-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0634-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0635-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0636-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0637-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0638-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0639-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0640-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0641-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0642-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0643-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0644-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0645-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0646-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0647-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0648-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0649-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0650-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0651-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0652-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0653-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0654-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0655-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0656-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0657-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0658-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0659-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0660-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0661-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0662-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0663-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0664-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0665-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0666-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0667-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0668-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0669-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0670-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0671-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0672-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0673-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0674-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0675-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0676-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0677-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0678-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0679-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0680-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0681-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0682-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0683-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0684-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0685-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0686-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0687-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0688-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0689-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0690-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0691-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0692-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0693-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0694-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0695-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0696-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0697-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0698-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0699-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0700-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0701-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0702-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0703-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0704-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0705-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0706-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0707-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0708-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0709-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0710-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0711-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0712-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0713-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0714-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0715-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0716-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0717-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0718-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0719-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0720-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0721-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0722-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0723-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0724-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0725-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0726-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0727-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0728-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0729-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0730-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0731-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0732-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0733-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0734-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0735-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0736-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0737-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0738-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0739-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0740-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0741-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0742-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0743-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0744-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0745-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0746-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0747-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0748-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0749-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0750-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0751-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0752-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0753-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0754-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0755-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0756-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0757-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0758-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0759-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0760-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0761-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0762-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0763-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0764-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0765-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0766-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0767-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0768-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0769-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0770-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0771-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0772-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0773-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0774-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0775-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0776-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0777-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0778-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0779-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0780-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0781-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0782-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0783-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0784-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0785-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0786-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0787-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0788-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0789-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0790-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0791-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0792-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0793-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0794-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0795-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0796-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0797-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0798-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0799-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0800-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0801-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0802-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0803-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0804-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0805-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0806-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0807-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0808-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0809-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0810-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0811-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0812-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0813-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0814-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0815-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0816-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0817-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0818-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0819-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0820-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0821-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0822-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0823-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0824-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0825-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0826-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0827-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0828-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0829-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0830-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0831-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0832-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0833-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0834-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0835-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0836-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0837-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0838-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0839-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0840-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0841-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0842-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0843-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0844-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0845-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0846-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0847-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0848-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0849-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0850-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0851-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0852-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0853-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0854-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0855-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0856-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0857-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0858-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0859-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0860-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0861-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0862-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0863-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0864-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0865-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0866-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0867-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0868-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0869-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0870-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0871-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0872-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0873-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0874-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0875-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0876-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0877-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0878-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0879-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0880-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0881-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0882-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0883-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0884-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0885-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0886-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0887-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0888-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0889-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0890-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0891-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0892-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0893-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0894-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0895-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0896-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0897-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0898-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0899-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0900-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0901-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0902-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0903-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0904-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0905-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0906-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0907-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0908-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0909-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0910-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0911-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0912-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0913-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0914-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0915-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0916-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0917-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0918-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0919-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0920-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0921-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0922-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0923-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0924-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0925-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0926-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0927-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0928-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0929-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0930-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0931-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0932-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0933-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0934-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0935-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0936-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0937-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0938-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0939-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0940-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0941-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0942-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0943-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0944-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0945-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0946-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0947-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0948-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0949-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0950-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0951-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0952-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0953-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0954-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0955-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0956-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0957-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0958-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0959-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0960-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0961-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0962-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0963-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0964-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0965-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0966-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0967-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0968-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0969-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0970-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0971-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0972-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0973-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0974-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0975-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0976-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0977-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0978-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0979-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0980-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0981-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0982-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0983-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0984-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0985-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0986-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0987-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0988-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0989-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0990-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0991-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0992-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0993-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0994-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0995-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0996-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0997-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0998-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0999-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1000-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1001-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1002-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1003-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1004-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1005-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1006-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1007-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1008-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1009-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1010-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1011-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1012-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1013-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1014-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1015-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1016-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1017-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1018-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1019-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1020-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1021-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1022-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1023-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH +HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH +P" 2BRPbr  $P +( , 0 4P8<@DQH"L2PBTRQXb\r`dQhlptQx|` +`` +*`:aJaZajazb"b &b +*b .c 2c 6c:c>dB +dFdJ*dN:eRJeVZeZje^zfbfffjfngrgvgzg~0@pABC 0DpEFG0H pI$J(K,0L0pM4N8O<1P@qQDRHSL1TPqUTVXW\1X`qYdZh[l1\pq]t^x_|p@TpA#pB3pCCqDSTqEcqFsqGrHTrIrJrKsLTsMsNsOtPUtQ#tR3tSCuTSUuUcuVsuWvXUvYvZv[w\Uw]w^w_ 4t ++;K4[tk{"4&t*.246t:>B 5FuJ+N;RK5V[uZk^{b5fujnr5vuz~ !R""#2$B%RR&b'r()R*+,-R./01S2"324B5RS6b7r89S:袓;<=S>?h +h!h"*h#:i$Ji%Zi&ji'zj(j)j*j+k,k-k.k/l0 +l1l2*l3:m4Jm5Zm6jm7zn8n9n:n;oo?2`rabc2drefg2hrijk2lrmno3psqijrs3tsuԳvw3xsyz{3|s}~x`Vxa#xb3xcCydSVyecyfsygzhVzizjzk{lV{m{n{o|pW|q#|r3|sC}tSW}uc}vs}w~xW~y~z~{|W}~ 6v+;K6[vk{6v6v 7w+;K7[wk{7w꫷7wHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHg g ` HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH$Dd$Dd$DDd$Dd$Dd$DDd$Dd $ D d  +$ +D +D +d + + + + +  $ D d  $ D d  $ D D d $Dd$Ddvalue-0001-deterministic-fixturevalue-0002-deterministic-fixturevalue-0003-deterministic-fixturevalue-0004-deterministic-fixturevalue-0005-deterministic-fixturevalue-0006-deterministic-fixturevalue-0008-deterministic-fixturevalue-0009-deterministic-fixturevalue-0010-deterministic-fixturevalue-0011-deterministic-fixturevalue-0012-deterministic-fixturevalue-0013-deterministic-fixturevalue-0015-deterministic-fixturevalue-0016-deterministic-fixturevalue-0017-deterministic-fixturevalue-0018-deterministic-fixturevalue-0019-deterministic-fixturevalue-0020-deterministic-fixturevalue-0022-deterministic-fixturevalue-0023-deterministic-fixturevalue-0024-deterministic-fixturevalue-0025-deterministic-fixturevalue-0026-deterministic-fixturevalue-0027-deterministic-fixturevalue-0029-deterministic-fixturevalue-0030-deterministic-fixturevalue-0031-deterministic-fixturevalue-0032-deterministic-fixturevalue-0033-deterministic-fixturevalue-0034-deterministic-fixturevalue-0036-deterministic-fixturevalue-0037-deterministic-fixturevalue-0038-deterministic-fixturevalue-0039-deterministic-fixturevalue-0040-deterministic-fixturevalue-0041-deterministic-fixturevalue-0043-deterministic-fixturevalue-0044-deterministic-fixturevalue-0045-deterministic-fixturevalue-0046-deterministic-fixturevalue-0047-deterministic-fixturevalue-0048-deterministic-fixturevalue-0050-deterministic-fixturevalue-0051-deterministic-fixturevalue-0052-deterministic-fixturevalue-0053-deterministic-fixturevalue-0054-deterministic-fixturevalue-0055-deterministic-fixturevalue-0057-deterministic-fixturevalue-0058-deterministic-fixturevalue-0059-deterministic-fixturevalue-0060-deterministic-fixturevalue-0061-deterministic-fixturevalue-0062-deterministic-fixturevalue-0064-deterministic-fixturevalue-0065-deterministic-fixturevalue-0066-deterministic-fixturevalue-0067-deterministic-fixturevalue-0068-deterministic-fixturevalue-0069-deterministic-fixturevalue-0071-deterministic-fixturevalue-0072-deterministic-fixturevalue-0073-deterministic-fixturevalue-0074-deterministic-fixturevalue-0075-deterministic-fixturevalue-0076-deterministic-fixturevalue-0078-deterministic-fixturevalue-0079-deterministic-fixturevalue-0080-deterministic-fixturevalue-0081-deterministic-fixturevalue-0082-deterministic-fixturevalue-0083-deterministic-fixturevalue-0085-deterministic-fixturevalue-0086-deterministic-fixturevalue-0087-deterministic-fixturevalue-0088-deterministic-fixturevalue-0089-deterministic-fixturevalue-0090-deterministic-fixturevalue-0092-deterministic-fixturevalue-0093-deterministic-fixturevalue-0094-deterministic-fixturevalue-0095-deterministic-fixturevalue-0096-deterministic-fixturevalue-0097-deterministic-fixturevalue-0099-deterministic-fixturevalue-0100-deterministic-fixturevalue-0101-deterministic-fixturevalue-0102-deterministic-fixturevalue-0103-deterministic-fixturevalue-0104-deterministic-fixturevalue-0106-deterministic-fixturevalue-0107-deterministic-fixturevalue-0108-deterministic-fixturevalue-0109-deterministic-fixturevalue-0110-deterministic-fixturevalue-0111-deterministic-fixturevalue-0113-deterministic-fixturevalue-0114-deterministic-fixturevalue-0115-deterministic-fixturevalue-0116-deterministic-fixturevalue-0117-deterministic-fixturevalue-0118-deterministic-fixturevalue-0120-deterministic-fixturevalue-0121-deterministic-fixturevalue-0122-deterministic-fixturevalue-0123-deterministic-fixturevalue-0124-deterministic-fixturevalue-0125-deterministic-fixturevalue-0127-deterministic-fixture$Dd$Ddd$$Dd$Dd$Ddd$$Dd$Dd $ D d d  +$ +$ +D +d + + + + + + $ D d  $ D d d  $ $ D d $Dd$Dddvalue-0128-deterministic-fixturevalue-0129-deterministic-fixturevalue-0130-deterministic-fixturevalue-0131-deterministic-fixturevalue-0132-deterministic-fixturevalue-0134-deterministic-fixturevalue-0135-deterministic-fixturevalue-0136-deterministic-fixturevalue-0137-deterministic-fixturevalue-0138-deterministic-fixturevalue-0139-deterministic-fixturevalue-0141-deterministic-fixturevalue-0142-deterministic-fixturevalue-0143-deterministic-fixturevalue-0144-deterministic-fixturevalue-0145-deterministic-fixturevalue-0146-deterministic-fixturevalue-0148-deterministic-fixturevalue-0149-deterministic-fixturevalue-0150-deterministic-fixturevalue-0151-deterministic-fixturevalue-0152-deterministic-fixturevalue-0153-deterministic-fixturevalue-0155-deterministic-fixturevalue-0156-deterministic-fixturevalue-0157-deterministic-fixturevalue-0158-deterministic-fixturevalue-0159-deterministic-fixturevalue-0160-deterministic-fixturevalue-0162-deterministic-fixturevalue-0163-deterministic-fixturevalue-0164-deterministic-fixturevalue-0165-deterministic-fixturevalue-0166-deterministic-fixturevalue-0167-deterministic-fixturevalue-0169-deterministic-fixturevalue-0170-deterministic-fixturevalue-0171-deterministic-fixturevalue-0172-deterministic-fixturevalue-0173-deterministic-fixturevalue-0174-deterministic-fixturevalue-0176-deterministic-fixturevalue-0177-deterministic-fixturevalue-0178-deterministic-fixturevalue-0179-deterministic-fixturevalue-0180-deterministic-fixturevalue-0181-deterministic-fixturevalue-0183-deterministic-fixturevalue-0184-deterministic-fixturevalue-0185-deterministic-fixturevalue-0186-deterministic-fixturevalue-0187-deterministic-fixturevalue-0188-deterministic-fixturevalue-0190-deterministic-fixturevalue-0191-deterministic-fixturevalue-0192-deterministic-fixturevalue-0193-deterministic-fixturevalue-0194-deterministic-fixturevalue-0195-deterministic-fixturevalue-0197-deterministic-fixturevalue-0198-deterministic-fixturevalue-0199-deterministic-fixturevalue-0200-deterministic-fixturevalue-0201-deterministic-fixturevalue-0202-deterministic-fixturevalue-0204-deterministic-fixturevalue-0205-deterministic-fixturevalue-0206-deterministic-fixturevalue-0207-deterministic-fixturevalue-0208-deterministic-fixturevalue-0209-deterministic-fixturevalue-0211-deterministic-fixturevalue-0212-deterministic-fixturevalue-0213-deterministic-fixturevalue-0214-deterministic-fixturevalue-0215-deterministic-fixturevalue-0216-deterministic-fixturevalue-0218-deterministic-fixturevalue-0219-deterministic-fixturevalue-0220-deterministic-fixturevalue-0221-deterministic-fixturevalue-0222-deterministic-fixturevalue-0223-deterministic-fixturevalue-0225-deterministic-fixturevalue-0226-deterministic-fixturevalue-0227-deterministic-fixturevalue-0228-deterministic-fixturevalue-0229-deterministic-fixturevalue-0230-deterministic-fixturevalue-0232-deterministic-fixturevalue-0233-deterministic-fixturevalue-0234-deterministic-fixturevalue-0235-deterministic-fixturevalue-0236-deterministic-fixturevalue-0237-deterministic-fixturevalue-0239-deterministic-fixturevalue-0240-deterministic-fixturevalue-0241-deterministic-fixturevalue-0242-deterministic-fixturevalue-0243-deterministic-fixturevalue-0244-deterministic-fixturevalue-0246-deterministic-fixturevalue-0247-deterministic-fixturevalue-0248-deterministic-fixturevalue-0249-deterministic-fixturevalue-0250-deterministic-fixturevalue-0251-deterministic-fixturevalue-0253-deterministic-fixturevalue-0254-deterministic-fixturevalue-0255-deterministic-fixture$Ddd$$Dd$Dd$Ddd$$Dd$Dd$Ddd $ $ D d  +$ +D +d + + + + + + $ D d d  $ $ D d  $ D d $Ddd$$Ddvalue-0256-deterministic-fixturevalue-0257-deterministic-fixturevalue-0258-deterministic-fixturevalue-0260-deterministic-fixturevalue-0261-deterministic-fixturevalue-0262-deterministic-fixturevalue-0263-deterministic-fixturevalue-0264-deterministic-fixturevalue-0265-deterministic-fixturevalue-0267-deterministic-fixturevalue-0268-deterministic-fixturevalue-0269-deterministic-fixturevalue-0270-deterministic-fixturevalue-0271-deterministic-fixturevalue-0272-deterministic-fixturevalue-0274-deterministic-fixturevalue-0275-deterministic-fixturevalue-0276-deterministic-fixturevalue-0277-deterministic-fixturevalue-0278-deterministic-fixturevalue-0279-deterministic-fixturevalue-0281-deterministic-fixturevalue-0282-deterministic-fixturevalue-0283-deterministic-fixturevalue-0284-deterministic-fixturevalue-0285-deterministic-fixturevalue-0286-deterministic-fixturevalue-0288-deterministic-fixturevalue-0289-deterministic-fixturevalue-0290-deterministic-fixturevalue-0291-deterministic-fixturevalue-0292-deterministic-fixturevalue-0293-deterministic-fixturevalue-0295-deterministic-fixturevalue-0296-deterministic-fixturevalue-0297-deterministic-fixturevalue-0298-deterministic-fixturevalue-0299-deterministic-fixturevalue-0300-deterministic-fixturevalue-0302-deterministic-fixturevalue-0303-deterministic-fixturevalue-0304-deterministic-fixturevalue-0305-deterministic-fixturevalue-0306-deterministic-fixturevalue-0307-deterministic-fixturevalue-0309-deterministic-fixturevalue-0310-deterministic-fixturevalue-0311-deterministic-fixturevalue-0312-deterministic-fixturevalue-0313-deterministic-fixturevalue-0314-deterministic-fixturevalue-0316-deterministic-fixturevalue-0317-deterministic-fixturevalue-0318-deterministic-fixturevalue-0319-deterministic-fixturevalue-0320-deterministic-fixturevalue-0321-deterministic-fixturevalue-0323-deterministic-fixturevalue-0324-deterministic-fixturevalue-0325-deterministic-fixturevalue-0326-deterministic-fixturevalue-0327-deterministic-fixturevalue-0328-deterministic-fixturevalue-0330-deterministic-fixturevalue-0331-deterministic-fixturevalue-0332-deterministic-fixturevalue-0333-deterministic-fixturevalue-0334-deterministic-fixturevalue-0335-deterministic-fixturevalue-0337-deterministic-fixturevalue-0338-deterministic-fixturevalue-0339-deterministic-fixturevalue-0340-deterministic-fixturevalue-0341-deterministic-fixturevalue-0342-deterministic-fixturevalue-0344-deterministic-fixturevalue-0345-deterministic-fixturevalue-0346-deterministic-fixturevalue-0347-deterministic-fixturevalue-0348-deterministic-fixturevalue-0349-deterministic-fixturevalue-0351-deterministic-fixturevalue-0352-deterministic-fixturevalue-0353-deterministic-fixturevalue-0354-deterministic-fixturevalue-0355-deterministic-fixturevalue-0356-deterministic-fixturevalue-0358-deterministic-fixturevalue-0359-deterministic-fixturevalue-0360-deterministic-fixturevalue-0361-deterministic-fixturevalue-0362-deterministic-fixturevalue-0363-deterministic-fixturevalue-0365-deterministic-fixturevalue-0366-deterministic-fixturevalue-0367-deterministic-fixturevalue-0368-deterministic-fixturevalue-0369-deterministic-fixturevalue-0370-deterministic-fixturevalue-0372-deterministic-fixturevalue-0373-deterministic-fixturevalue-0374-deterministic-fixturevalue-0375-deterministic-fixturevalue-0376-deterministic-fixturevalue-0377-deterministic-fixturevalue-0379-deterministic-fixturevalue-0380-deterministic-fixturevalue-0381-deterministic-fixturevalue-0382-deterministic-fixturevalue-0383-deterministic-fixture$$Dd$Dd$Ddd$$Dd$Dd$Ddd$$Dd $ D d  +$ +D +d +d + + + + + $ $ D d  $ D d  $ D d d $$Dd$Ddvalue-0384-deterministic-fixturevalue-0386-deterministic-fixturevalue-0387-deterministic-fixturevalue-0388-deterministic-fixturevalue-0389-deterministic-fixturevalue-0390-deterministic-fixturevalue-0391-deterministic-fixturevalue-0393-deterministic-fixturevalue-0394-deterministic-fixturevalue-0395-deterministic-fixturevalue-0396-deterministic-fixturevalue-0397-deterministic-fixturevalue-0398-deterministic-fixturevalue-0400-deterministic-fixturevalue-0401-deterministic-fixturevalue-0402-deterministic-fixturevalue-0403-deterministic-fixturevalue-0404-deterministic-fixturevalue-0405-deterministic-fixturevalue-0407-deterministic-fixturevalue-0408-deterministic-fixturevalue-0409-deterministic-fixturevalue-0410-deterministic-fixturevalue-0411-deterministic-fixturevalue-0412-deterministic-fixturevalue-0414-deterministic-fixturevalue-0415-deterministic-fixturevalue-0416-deterministic-fixturevalue-0417-deterministic-fixturevalue-0418-deterministic-fixturevalue-0419-deterministic-fixturevalue-0421-deterministic-fixturevalue-0422-deterministic-fixturevalue-0423-deterministic-fixturevalue-0424-deterministic-fixturevalue-0425-deterministic-fixturevalue-0426-deterministic-fixturevalue-0428-deterministic-fixturevalue-0429-deterministic-fixturevalue-0430-deterministic-fixturevalue-0431-deterministic-fixturevalue-0432-deterministic-fixturevalue-0433-deterministic-fixturevalue-0435-deterministic-fixturevalue-0436-deterministic-fixturevalue-0437-deterministic-fixturevalue-0438-deterministic-fixturevalue-0439-deterministic-fixturevalue-0440-deterministic-fixturevalue-0442-deterministic-fixturevalue-0443-deterministic-fixturevalue-0444-deterministic-fixturevalue-0445-deterministic-fixturevalue-0446-deterministic-fixturevalue-0447-deterministic-fixturevalue-0449-deterministic-fixturevalue-0450-deterministic-fixturevalue-0451-deterministic-fixturevalue-0452-deterministic-fixturevalue-0453-deterministic-fixturevalue-0454-deterministic-fixturevalue-0456-deterministic-fixturevalue-0457-deterministic-fixturevalue-0458-deterministic-fixturevalue-0459-deterministic-fixturevalue-0460-deterministic-fixturevalue-0461-deterministic-fixturevalue-0463-deterministic-fixturevalue-0464-deterministic-fixturevalue-0465-deterministic-fixturevalue-0466-deterministic-fixturevalue-0467-deterministic-fixturevalue-0468-deterministic-fixturevalue-0470-deterministic-fixturevalue-0471-deterministic-fixturevalue-0472-deterministic-fixturevalue-0473-deterministic-fixturevalue-0474-deterministic-fixturevalue-0475-deterministic-fixturevalue-0477-deterministic-fixturevalue-0478-deterministic-fixturevalue-0479-deterministic-fixturevalue-0480-deterministic-fixturevalue-0481-deterministic-fixturevalue-0482-deterministic-fixturevalue-0484-deterministic-fixturevalue-0485-deterministic-fixturevalue-0486-deterministic-fixturevalue-0487-deterministic-fixturevalue-0488-deterministic-fixturevalue-0489-deterministic-fixturevalue-0491-deterministic-fixturevalue-0492-deterministic-fixturevalue-0493-deterministic-fixturevalue-0494-deterministic-fixturevalue-0495-deterministic-fixturevalue-0496-deterministic-fixturevalue-0498-deterministic-fixturevalue-0499-deterministic-fixturevalue-0500-deterministic-fixturevalue-0501-deterministic-fixturevalue-0502-deterministic-fixturevalue-0503-deterministic-fixturevalue-0505-deterministic-fixturevalue-0506-deterministic-fixturevalue-0507-deterministic-fixturevalue-0508-deterministic-fixturevalue-0509-deterministic-fixturevalue-0510-deterministic-fixture$Dd$Dd$DDd$Dd$Dd$DDd$Dd $ D d  +$ +D +D +d + + + + +  $ D d  $ D d  $ D D d $Dd$Ddvalue-0512-deterministic-fixturevalue-0513-deterministic-fixturevalue-0514-deterministic-fixturevalue-0515-deterministic-fixturevalue-0516-deterministic-fixturevalue-0517-deterministic-fixturevalue-0519-deterministic-fixturevalue-0520-deterministic-fixturevalue-0521-deterministic-fixturevalue-0522-deterministic-fixturevalue-0523-deterministic-fixturevalue-0524-deterministic-fixturevalue-0526-deterministic-fixturevalue-0527-deterministic-fixturevalue-0528-deterministic-fixturevalue-0529-deterministic-fixturevalue-0530-deterministic-fixturevalue-0531-deterministic-fixturevalue-0533-deterministic-fixturevalue-0534-deterministic-fixturevalue-0535-deterministic-fixturevalue-0536-deterministic-fixturevalue-0537-deterministic-fixturevalue-0538-deterministic-fixturevalue-0540-deterministic-fixturevalue-0541-deterministic-fixturevalue-0542-deterministic-fixturevalue-0543-deterministic-fixturevalue-0544-deterministic-fixturevalue-0545-deterministic-fixturevalue-0547-deterministic-fixturevalue-0548-deterministic-fixturevalue-0549-deterministic-fixturevalue-0550-deterministic-fixturevalue-0551-deterministic-fixturevalue-0552-deterministic-fixturevalue-0554-deterministic-fixturevalue-0555-deterministic-fixturevalue-0556-deterministic-fixturevalue-0557-deterministic-fixturevalue-0558-deterministic-fixturevalue-0559-deterministic-fixturevalue-0561-deterministic-fixturevalue-0562-deterministic-fixturevalue-0563-deterministic-fixturevalue-0564-deterministic-fixturevalue-0565-deterministic-fixturevalue-0566-deterministic-fixturevalue-0568-deterministic-fixturevalue-0569-deterministic-fixturevalue-0570-deterministic-fixturevalue-0571-deterministic-fixturevalue-0572-deterministic-fixturevalue-0573-deterministic-fixturevalue-0575-deterministic-fixturevalue-0576-deterministic-fixturevalue-0577-deterministic-fixturevalue-0578-deterministic-fixturevalue-0579-deterministic-fixturevalue-0580-deterministic-fixturevalue-0582-deterministic-fixturevalue-0583-deterministic-fixturevalue-0584-deterministic-fixturevalue-0585-deterministic-fixturevalue-0586-deterministic-fixturevalue-0587-deterministic-fixturevalue-0589-deterministic-fixturevalue-0590-deterministic-fixturevalue-0591-deterministic-fixturevalue-0592-deterministic-fixturevalue-0593-deterministic-fixturevalue-0594-deterministic-fixturevalue-0596-deterministic-fixturevalue-0597-deterministic-fixturevalue-0598-deterministic-fixturevalue-0599-deterministic-fixturevalue-0600-deterministic-fixturevalue-0601-deterministic-fixturevalue-0603-deterministic-fixturevalue-0604-deterministic-fixturevalue-0605-deterministic-fixturevalue-0606-deterministic-fixturevalue-0607-deterministic-fixturevalue-0608-deterministic-fixturevalue-0610-deterministic-fixturevalue-0611-deterministic-fixturevalue-0612-deterministic-fixturevalue-0613-deterministic-fixturevalue-0614-deterministic-fixturevalue-0615-deterministic-fixturevalue-0617-deterministic-fixturevalue-0618-deterministic-fixturevalue-0619-deterministic-fixturevalue-0620-deterministic-fixturevalue-0621-deterministic-fixturevalue-0622-deterministic-fixturevalue-0624-deterministic-fixturevalue-0625-deterministic-fixturevalue-0626-deterministic-fixturevalue-0627-deterministic-fixturevalue-0628-deterministic-fixturevalue-0629-deterministic-fixturevalue-0631-deterministic-fixturevalue-0632-deterministic-fixturevalue-0633-deterministic-fixturevalue-0634-deterministic-fixturevalue-0635-deterministic-fixturevalue-0636-deterministic-fixturevalue-0638-deterministic-fixturevalue-0639-deterministic-fixture$Dd$DDd$Dd$Dd$DDd$Dd$Dd $ D D d  + +$ +D +d + + + + + + $ D d  $ D D d   $ D d $Dd$DDdvalue-0640-deterministic-fixturevalue-0641-deterministic-fixturevalue-0642-deterministic-fixturevalue-0643-deterministic-fixturevalue-0645-deterministic-fixturevalue-0646-deterministic-fixturevalue-0647-deterministic-fixturevalue-0648-deterministic-fixturevalue-0649-deterministic-fixturevalue-0650-deterministic-fixturevalue-0652-deterministic-fixturevalue-0653-deterministic-fixturevalue-0654-deterministic-fixturevalue-0655-deterministic-fixturevalue-0656-deterministic-fixturevalue-0657-deterministic-fixturevalue-0659-deterministic-fixturevalue-0660-deterministic-fixturevalue-0661-deterministic-fixturevalue-0662-deterministic-fixturevalue-0663-deterministic-fixturevalue-0664-deterministic-fixturevalue-0666-deterministic-fixturevalue-0667-deterministic-fixturevalue-0668-deterministic-fixturevalue-0669-deterministic-fixturevalue-0670-deterministic-fixturevalue-0671-deterministic-fixturevalue-0673-deterministic-fixturevalue-0674-deterministic-fixturevalue-0675-deterministic-fixturevalue-0676-deterministic-fixturevalue-0677-deterministic-fixturevalue-0678-deterministic-fixturevalue-0680-deterministic-fixturevalue-0681-deterministic-fixturevalue-0682-deterministic-fixturevalue-0683-deterministic-fixturevalue-0684-deterministic-fixturevalue-0685-deterministic-fixturevalue-0687-deterministic-fixturevalue-0688-deterministic-fixturevalue-0689-deterministic-fixturevalue-0690-deterministic-fixturevalue-0691-deterministic-fixturevalue-0692-deterministic-fixturevalue-0694-deterministic-fixturevalue-0695-deterministic-fixturevalue-0696-deterministic-fixturevalue-0697-deterministic-fixturevalue-0698-deterministic-fixturevalue-0699-deterministic-fixturevalue-0701-deterministic-fixturevalue-0702-deterministic-fixturevalue-0703-deterministic-fixturevalue-0704-deterministic-fixturevalue-0705-deterministic-fixturevalue-0706-deterministic-fixturevalue-0708-deterministic-fixturevalue-0709-deterministic-fixturevalue-0710-deterministic-fixturevalue-0711-deterministic-fixturevalue-0712-deterministic-fixturevalue-0713-deterministic-fixturevalue-0715-deterministic-fixturevalue-0716-deterministic-fixturevalue-0717-deterministic-fixturevalue-0718-deterministic-fixturevalue-0719-deterministic-fixturevalue-0720-deterministic-fixturevalue-0722-deterministic-fixturevalue-0723-deterministic-fixturevalue-0724-deterministic-fixturevalue-0725-deterministic-fixturevalue-0726-deterministic-fixturevalue-0727-deterministic-fixturevalue-0729-deterministic-fixturevalue-0730-deterministic-fixturevalue-0731-deterministic-fixturevalue-0732-deterministic-fixturevalue-0733-deterministic-fixturevalue-0734-deterministic-fixturevalue-0736-deterministic-fixturevalue-0737-deterministic-fixturevalue-0738-deterministic-fixturevalue-0739-deterministic-fixturevalue-0740-deterministic-fixturevalue-0741-deterministic-fixturevalue-0743-deterministic-fixturevalue-0744-deterministic-fixturevalue-0745-deterministic-fixturevalue-0746-deterministic-fixturevalue-0747-deterministic-fixturevalue-0748-deterministic-fixturevalue-0750-deterministic-fixturevalue-0751-deterministic-fixturevalue-0752-deterministic-fixturevalue-0753-deterministic-fixturevalue-0754-deterministic-fixturevalue-0755-deterministic-fixturevalue-0757-deterministic-fixturevalue-0758-deterministic-fixturevalue-0759-deterministic-fixturevalue-0760-deterministic-fixturevalue-0761-deterministic-fixturevalue-0762-deterministic-fixturevalue-0764-deterministic-fixturevalue-0765-deterministic-fixturevalue-0766-deterministic-fixturevalue-0767-deterministic-fixture$DDd$Dd$Dd$DDd$Dd$Dd$DDd  $ D d  +$ +D +d + + + + + + $ D D d   $ D d  $ D d $DDd$Ddvalue-0768-deterministic-fixturevalue-0769-deterministic-fixturevalue-0771-deterministic-fixturevalue-0772-deterministic-fixturevalue-0773-deterministic-fixturevalue-0774-deterministic-fixturevalue-0775-deterministic-fixturevalue-0776-deterministic-fixturevalue-0778-deterministic-fixturevalue-0779-deterministic-fixturevalue-0780-deterministic-fixturevalue-0781-deterministic-fixturevalue-0782-deterministic-fixturevalue-0783-deterministic-fixturevalue-0785-deterministic-fixturevalue-0786-deterministic-fixturevalue-0787-deterministic-fixturevalue-0788-deterministic-fixturevalue-0789-deterministic-fixturevalue-0790-deterministic-fixturevalue-0792-deterministic-fixturevalue-0793-deterministic-fixturevalue-0794-deterministic-fixturevalue-0795-deterministic-fixturevalue-0796-deterministic-fixturevalue-0797-deterministic-fixturevalue-0799-deterministic-fixturevalue-0800-deterministic-fixturevalue-0801-deterministic-fixturevalue-0802-deterministic-fixturevalue-0803-deterministic-fixturevalue-0804-deterministic-fixturevalue-0806-deterministic-fixturevalue-0807-deterministic-fixturevalue-0808-deterministic-fixturevalue-0809-deterministic-fixturevalue-0810-deterministic-fixturevalue-0811-deterministic-fixturevalue-0813-deterministic-fixturevalue-0814-deterministic-fixturevalue-0815-deterministic-fixturevalue-0816-deterministic-fixturevalue-0817-deterministic-fixturevalue-0818-deterministic-fixturevalue-0820-deterministic-fixturevalue-0821-deterministic-fixturevalue-0822-deterministic-fixturevalue-0823-deterministic-fixturevalue-0824-deterministic-fixturevalue-0825-deterministic-fixturevalue-0827-deterministic-fixturevalue-0828-deterministic-fixturevalue-0829-deterministic-fixturevalue-0830-deterministic-fixturevalue-0831-deterministic-fixturevalue-0832-deterministic-fixturevalue-0834-deterministic-fixturevalue-0835-deterministic-fixturevalue-0836-deterministic-fixturevalue-0837-deterministic-fixturevalue-0838-deterministic-fixturevalue-0839-deterministic-fixturevalue-0841-deterministic-fixturevalue-0842-deterministic-fixturevalue-0843-deterministic-fixturevalue-0844-deterministic-fixturevalue-0845-deterministic-fixturevalue-0846-deterministic-fixturevalue-0848-deterministic-fixturevalue-0849-deterministic-fixturevalue-0850-deterministic-fixturevalue-0851-deterministic-fixturevalue-0852-deterministic-fixturevalue-0853-deterministic-fixturevalue-0855-deterministic-fixturevalue-0856-deterministic-fixturevalue-0857-deterministic-fixturevalue-0858-deterministic-fixturevalue-0859-deterministic-fixturevalue-0860-deterministic-fixturevalue-0862-deterministic-fixturevalue-0863-deterministic-fixturevalue-0864-deterministic-fixturevalue-0865-deterministic-fixturevalue-0866-deterministic-fixturevalue-0867-deterministic-fixturevalue-0869-deterministic-fixturevalue-0870-deterministic-fixturevalue-0871-deterministic-fixturevalue-0872-deterministic-fixturevalue-0873-deterministic-fixturevalue-0874-deterministic-fixturevalue-0876-deterministic-fixturevalue-0877-deterministic-fixturevalue-0878-deterministic-fixturevalue-0879-deterministic-fixturevalue-0880-deterministic-fixturevalue-0881-deterministic-fixturevalue-0883-deterministic-fixturevalue-0884-deterministic-fixturevalue-0885-deterministic-fixturevalue-0886-deterministic-fixturevalue-0887-deterministic-fixturevalue-0888-deterministic-fixturevalue-0890-deterministic-fixturevalue-0891-deterministic-fixturevalue-0892-deterministic-fixturevalue-0893-deterministic-fixturevalue-0894-deterministic-fixturevalue-0895-deterministic-fixture$Dd$Dd$DDd$Dd$Dd$DDd$Dd $ D d  +$ +D +D +d + + + + +  $ D d  $ D d  $ D D d $Dd$Ddvalue-0897-deterministic-fixturevalue-0898-deterministic-fixturevalue-0899-deterministic-fixturevalue-0900-deterministic-fixturevalue-0901-deterministic-fixturevalue-0902-deterministic-fixturevalue-0904-deterministic-fixturevalue-0905-deterministic-fixturevalue-0906-deterministic-fixturevalue-0907-deterministic-fixturevalue-0908-deterministic-fixturevalue-0909-deterministic-fixturevalue-0911-deterministic-fixturevalue-0912-deterministic-fixturevalue-0913-deterministic-fixturevalue-0914-deterministic-fixturevalue-0915-deterministic-fixturevalue-0916-deterministic-fixturevalue-0918-deterministic-fixturevalue-0919-deterministic-fixturevalue-0920-deterministic-fixturevalue-0921-deterministic-fixturevalue-0922-deterministic-fixturevalue-0923-deterministic-fixturevalue-0925-deterministic-fixturevalue-0926-deterministic-fixturevalue-0927-deterministic-fixturevalue-0928-deterministic-fixturevalue-0929-deterministic-fixturevalue-0930-deterministic-fixturevalue-0932-deterministic-fixturevalue-0933-deterministic-fixturevalue-0934-deterministic-fixturevalue-0935-deterministic-fixturevalue-0936-deterministic-fixturevalue-0937-deterministic-fixturevalue-0939-deterministic-fixturevalue-0940-deterministic-fixturevalue-0941-deterministic-fixturevalue-0942-deterministic-fixturevalue-0943-deterministic-fixturevalue-0944-deterministic-fixturevalue-0946-deterministic-fixturevalue-0947-deterministic-fixturevalue-0948-deterministic-fixturevalue-0949-deterministic-fixturevalue-0950-deterministic-fixturevalue-0951-deterministic-fixturevalue-0953-deterministic-fixturevalue-0954-deterministic-fixturevalue-0955-deterministic-fixturevalue-0956-deterministic-fixturevalue-0957-deterministic-fixturevalue-0958-deterministic-fixturevalue-0960-deterministic-fixturevalue-0961-deterministic-fixturevalue-0962-deterministic-fixturevalue-0963-deterministic-fixturevalue-0964-deterministic-fixturevalue-0965-deterministic-fixturevalue-0967-deterministic-fixturevalue-0968-deterministic-fixturevalue-0969-deterministic-fixturevalue-0970-deterministic-fixturevalue-0971-deterministic-fixturevalue-0972-deterministic-fixturevalue-0974-deterministic-fixturevalue-0975-deterministic-fixturevalue-0976-deterministic-fixturevalue-0977-deterministic-fixturevalue-0978-deterministic-fixturevalue-0979-deterministic-fixturevalue-0981-deterministic-fixturevalue-0982-deterministic-fixturevalue-0983-deterministic-fixturevalue-0984-deterministic-fixturevalue-0985-deterministic-fixturevalue-0986-deterministic-fixturevalue-0988-deterministic-fixturevalue-0989-deterministic-fixturevalue-0990-deterministic-fixturevalue-0991-deterministic-fixturevalue-0992-deterministic-fixturevalue-0993-deterministic-fixturevalue-0995-deterministic-fixturevalue-0996-deterministic-fixturevalue-0997-deterministic-fixturevalue-0998-deterministic-fixturevalue-0999-deterministic-fixturevalue-1000-deterministic-fixturevalue-1002-deterministic-fixturevalue-1003-deterministic-fixturevalue-1004-deterministic-fixturevalue-1005-deterministic-fixturevalue-1006-deterministic-fixturevalue-1007-deterministic-fixturevalue-1009-deterministic-fixturevalue-1010-deterministic-fixturevalue-1011-deterministic-fixturevalue-1012-deterministic-fixturevalue-1013-deterministic-fixturevalue-1014-deterministic-fixturevalue-1016-deterministic-fixturevalue-1017-deterministic-fixturevalue-1018-deterministic-fixturevalue-1019-deterministic-fixturevalue-1020-deterministic-fixturevalue-1021-deterministic-fixturevalue-1023-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH::HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH#F$I$ɒ$$ddd2d2D2D2D"D"L"L"L&L&H&H&H$H$I$I$ɒ$$ddd2d2D2D2D"D"L"L"L&L&H&H&H$H$I$I@ @@ @!@@@ @@ @!@@ H@ @@ h/FAG0H @0H 01I@ KPC2K`3L @3AM +X4NC pNA#4ҋ`Nvq#8 bTv #@c`w"# efw$$Hfl#x&1$L" hrCx(a$P2ixy*$TBl~z,$P nz.$\ro{0Q%` qh{0()H8MRld{h)KDRp$|4)NP St|6)Q\MS||8)TtSD}:)W } *Z T~> *]MT d~@I*`T"~D!i*fT#$)*i8Wpب;L@X;NHBYPPZ=RX[=T`B\ >Vp]DPXx]h?Z^?\B_ @^` E B L'RX'b Úd!(r|Q(ق (ݐ#(ࢍD( + d) ҍѤq) ěġ)  ,$NX dI,'XĄi,*$,-YЄ ,0NY",3YD$,6Y& -90H )-<@NZ`d*(@-?PZxć.0-E[52Od#56APC68QpCc6:Qt7qT|@Æ8@T9BтUDBV":FaW2@c:H{ %r1|ȍ%D t|%uA } & wQ)} &dxYI~& z`&$ }i&~q(N' yɀ4'D@'Ť$H*l0U%J*o@U&LB+r@V'N+u`(+xpV)(RB+{V*XT+~@W+pV+W-+.\B+W/^@b)` &d*a ,Af1*b"!2DBha*c0!8dCj*dB!>l*eR"DDn*fb"JDp!+f"PEr+g"V$Et+h#\Dx+i􄈕-䈖-[E-[ +. \ *.O\ eJ.$Ŋ!j.0\%.<]1.HO]勤9.T!A +/"D$ddd2d2D2D2D"D"L"L"L&L&H&H&H$H$I$I$ɒ$$ddd2d2D2D2D"D"L"L"L&L&H&H&H$H$I$I$ɒ$$d @@ @H@ @@ @!@@@ @@ @!@@  rh4yQOk5zOn{OqD5|Otd5}Pw5~APzP}5ހP6߂Q$61Q#D6@4b`h4` f@5!a@s @6QalT 7and %A8"apt EA9Rar &eA:b FA;Abv f=qbz A>Bb| A?rbr",&L2r$%,&GXLbr&5,&gXLr*E,&XL„s,U,&§Xs.e,&M"s0u,&XMRs,&YMt4,&'Yt6,&GYMt8,&ԇY7퐡Sǰ7S"ʳ8T$ж$81T&#ӹaT(Cּ8TPٿ8T,c8!U0sQU29U$9UAe!FCLqe!CMe!CNBe"DOrf"P1f#"EDQaf0"&eDS2f C"FDTbf4S"fDUfDs"DVQgT"'[OvR-''[wV-'OwX-'h[Ow-'[OBw\-( +[rx^-( [Px`-(P҉xb.(\Px .(H\Pbxf5.(h\aQ3⅑Q +C6QP6Qc6!Rs6QR$7花RD7Rd7S7ASqS!%B@1c#!Aac3!eBBc@!BC2c$S!FBDbc@c!̆BE!dTs!ϦFQdd!CH"dt!%CIRd!ECJd!&KMBt:,&קYMrt,'YNu@,'Y҆uB-'NuD-''ZN2u -'GZNbuH5-'gZvJE-'ZN‡vLU-'OvNu-'ZORv-6D9U8d9V:AV<ڤ9qV9V@:WB1WF#D:aW0d:WJC:WLS :EWgd"%EX"gp"EEYRg" Zh"E[Ah"gE\qh"E^Bh"E_rh#`i$# FFabi0##fFbixhE.(PŠylU.("\Py`.(%\Q"ypu.((\Ryr.(+]Qzt.(.Qzv.(1H]Qz.(4h]Qzz.(7]rz|.):R{~.)=]$ 2$ 2 22"" " " & &&&$$ $ $ $ $ $ $ $ 2$ 2 22"" " " & &&&$$ $ $ $ $ $ $ $ @ @@ H@ @@@ @ cM1cNҕ2P2(dQ2 2HdRbE2܈dS–U2Te2dU"p2dVR2eW2(eXCV7Kos7koZ7 Cӧ\8 + ps^8 )p3`&8 Ipc68+ip3f'F8K cèh7V8 pjGp8pSP8"={={={={={>|>|>|>>|>|`3,ff#v32fg S35 gh +38Igi 3;!igj 3>$gkC3A*gls3D- m 3G0gnӛ4J3 ho$4MF9F+r3ë'P9I rc7f9Lkrœ#Pv9OrSg9R +#w9U *sS9[ Jsʃ9^+js˳9aK +s9dks9g sC~??~?~?~??????2Y◲2 he[2e\B2e]r2^Ә3 f_3)f`3&3 Ifac63# ifb +F3& cÙ V3)fd +qng8% +#pw8(JqSr8+ jqt8.KqC81kqsx84 +C|8: qsӪ~9= +r&9@Jrc69C +>|>|>>}>}>}>>}>}?~?6)hq3&&4S9Ihrc(64V< s*V4Y?ht`4\Bhu#.v4_EhvS24bK w44eN)ix4hQIiy84kTiiz:4nWi|Cӭ:j +ts:m*tУ3&:p +c6:s jt@:v tîV:| tf: t#: +u: *u: Ju: ??HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHxwHHHHHHHHHHHHHHHH HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH@ @@ @@ @@ @@ aaaaaAaa`a!aaaaaaaAaa`a!aaaaaaaAaa`a!aaaaaaaaaAaa`a!aaaaaaaAaa`a!aaaaaaaAaa`a!aaaaaaaHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH  greenblueredHHHHHHHHHHHHHHHHHHHHHHHHHHHH    HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH@@@@@@@@@ @    +@ + + + @    @    @   @@@@@@@@@@@@@@@@@@ @   !@!!!"@"""#@###$@$$$%@%%%&@&&&'@'''(@((()@)))*@***+@+++,@,,,-@---.@.../@///0@0001@1112@2223@3334@4445@5556@6667@7778@8889@999:@:::;@;;;<@<<<=@===>@>>>?@???@@@@@A@AAAB@BBBC@CCCD@DDDE@EEEF@FFFG@GGGH@HHHI@IIIJ@JJJK@KKKL@LLLM@MMMN@NNNO@OOOP@PPPQ@QQQR@RRRS@SSST@TTTU@UUUV@VVVW@WWWX@XXXY@YYYZ@ZZZ[@[[[\@\\\]@]]]^@^^^_@___`@```a@aaab@bbbc@cccd@ddde@eeef@fffg@gggh@hhhi@iiij@jjjk@kkkl@lllm@mmmn@nnno@ooop@pppq@qqqr@rrrs@ssst@tttu@uuuv@vvvw@wwwx@xxxy@yyyz@zzz{@{{{|@|||}@}}}~@~~~@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1024-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1025-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1026-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1027-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1028-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1029-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1030-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1031-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1032-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1033-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1034-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1035-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1036-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1037-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1038-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1039-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1040-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1041-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1042-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1043-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1044-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1045-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1046-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1047-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1048-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1049-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1050-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1051-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1052-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1053-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1054-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1055-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1056-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1057-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1058-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1059-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1060-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1061-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1062-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1063-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1064-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1065-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1066-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1067-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1068-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1069-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1070-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1071-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1072-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1073-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1074-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1075-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1076-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1077-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1078-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1079-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1080-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1081-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1082-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1083-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1084-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1085-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1086-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1087-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1088-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1089-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1090-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1091-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1092-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1093-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1094-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1095-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1096-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1097-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1098-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1099-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1100-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1101-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1102-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1103-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1104-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1105-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1106-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1107-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1108-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1109-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1110-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1111-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1112-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1113-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1114-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1115-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1116-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1117-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1118-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1119-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1120-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1121-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1122-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1123-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1124-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1125-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1126-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1127-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1128-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1129-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1130-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1131-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1132-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1133-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1134-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1135-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1136-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1137-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1138-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1139-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1140-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1141-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1142-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1143-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1144-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1145-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1146-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1147-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1148-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1149-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1150-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1151-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1152-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1153-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1154-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1155-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1156-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1157-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1158-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1159-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1160-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1161-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1162-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1163-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1164-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1165-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1166-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1167-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1168-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1169-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1170-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1171-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1172-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1173-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1174-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1175-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1176-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1177-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1178-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1179-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1180-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1181-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1182-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1183-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1184-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1185-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1186-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1187-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1188-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1189-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1190-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1191-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1192-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1193-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1194-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1195-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1196-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1197-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1198-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1199-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1200-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1201-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1202-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1203-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1204-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1205-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1206-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1207-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1208-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1209-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1210-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1211-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1212-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1213-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1214-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1215-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1216-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1217-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1218-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1219-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1220-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1221-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1222-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1223-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1224-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1225-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1226-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1227-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1228-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1229-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1230-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1231-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1232-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1233-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1234-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1235-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1236-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1237-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1238-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1239-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1240-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1241-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1242-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1243-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1244-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1245-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1246-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1247-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1248-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1249-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1250-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1251-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1252-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1253-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1254-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1255-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1256-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1257-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1258-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1259-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1260-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1261-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1262-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1263-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1264-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1265-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1266-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1267-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1268-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1269-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1270-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1271-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1272-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1273-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1274-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1275-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1276-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1277-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1278-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1279-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1280-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1281-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1282-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1283-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1284-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1285-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1286-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1287-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1288-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1289-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1290-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1291-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1292-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1293-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1294-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1295-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1296-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1297-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1298-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1299-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1300-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1301-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1302-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1303-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1304-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1305-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1306-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1307-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1308-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1309-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1310-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1311-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1312-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1313-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1314-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1315-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1316-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1317-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1318-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1319-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1320-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1321-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1322-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1323-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1324-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1325-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1326-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1327-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1328-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1329-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1330-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1331-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1332-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1333-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1334-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1335-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1336-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1337-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1338-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1339-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1340-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1341-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1342-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1343-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1344-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1345-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1346-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1347-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1348-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1349-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1350-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1351-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1352-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1353-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1354-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1355-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1356-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1357-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1358-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1359-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1360-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1361-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1362-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1363-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1364-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1365-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1366-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1367-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1368-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1369-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1370-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1371-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1372-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1373-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1374-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1375-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1376-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1377-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1378-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1379-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1380-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1381-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1382-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1383-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1384-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1385-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1386-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1387-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1388-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1389-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1390-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1391-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1392-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1393-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1394-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1395-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1396-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1397-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1398-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1399-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1400-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1401-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1402-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1403-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1404-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1405-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1406-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1407-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1408-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1409-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1410-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1411-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1412-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1413-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1414-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1415-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1416-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1417-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1418-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1419-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1420-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1421-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1422-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1423-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1424-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1425-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1426-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1427-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1428-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1429-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1430-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1431-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1432-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1433-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1434-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1435-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1436-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1437-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1438-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1439-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1440-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1441-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1442-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1443-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1444-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1445-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1446-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1447-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1448-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1449-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1450-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1451-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1452-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1453-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1454-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1455-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1456-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1457-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1458-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1459-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1460-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1461-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1462-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1463-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1464-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1465-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1466-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1467-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1468-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1469-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1470-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1471-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1472-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1473-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1474-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1475-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1476-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1477-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1478-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1479-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1480-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1481-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1482-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1483-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1484-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1485-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1486-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1487-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1488-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1489-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1490-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1491-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1492-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1493-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1494-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1495-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1496-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1497-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1498-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1499-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1500-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1501-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1502-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1503-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1504-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1505-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1506-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1507-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1508-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1509-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1510-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1511-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1512-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1513-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1514-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1515-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1516-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1517-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1518-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1519-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1520-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1521-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1522-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1523-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1524-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1525-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1526-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1527-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1528-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1529-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1530-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1531-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1532-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1533-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1534-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1535-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1536-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1537-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1538-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1539-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1540-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1541-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1542-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1543-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1544-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1545-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1546-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1547-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1548-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1549-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1550-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1551-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1552-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1553-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1554-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1555-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1556-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1557-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1558-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1559-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1560-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1561-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1562-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1563-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1564-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1565-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1566-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1567-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1568-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1569-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1570-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1571-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1572-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1573-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1574-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1575-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1576-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1577-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1578-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1579-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1580-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1581-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1582-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1583-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1584-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1585-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1586-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1587-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1588-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1589-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1590-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1591-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1592-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1593-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1594-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1595-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1596-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1597-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1598-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1599-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1600-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1601-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1602-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1603-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1604-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1605-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1606-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1607-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1608-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1609-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1610-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1611-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1612-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1613-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1614-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1615-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1616-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1617-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1618-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1619-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1620-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1621-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1622-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1623-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1624-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1625-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1626-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1627-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1628-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1629-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1630-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1631-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1632-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1633-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1634-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1635-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1636-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1637-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1638-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1639-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1640-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1641-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1642-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1643-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1644-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1645-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1646-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1647-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1648-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1649-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1650-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1651-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1652-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1653-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1654-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1655-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1656-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1657-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1658-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1659-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1660-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1661-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1662-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1663-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1664-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1665-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1666-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1667-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1668-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1669-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1670-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1671-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1672-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1673-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1674-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1675-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1676-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1677-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1678-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1679-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1680-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1681-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1682-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1683-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1684-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1685-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1686-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1687-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1688-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1689-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1690-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1691-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1692-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1693-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1694-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1695-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1696-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1697-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1698-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1699-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1700-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1701-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1702-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1703-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1704-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1705-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1706-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1707-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1708-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1709-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1710-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1711-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1712-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1713-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1714-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1715-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1716-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1717-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1718-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1719-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1720-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1721-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1722-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1723-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1724-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1725-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1726-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1727-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1728-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1729-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1730-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1731-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1732-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1733-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1734-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1735-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1736-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1737-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1738-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1739-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1740-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1741-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1742-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1743-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1744-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1745-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1746-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1747-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1748-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1749-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1750-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1751-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1752-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1753-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1754-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1755-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1756-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1757-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1758-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1759-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1760-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1761-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1762-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1763-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1764-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1765-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1766-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1767-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1768-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1769-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1770-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1771-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1772-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1773-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1774-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1775-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1776-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1777-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1778-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1779-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1780-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1781-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1782-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1783-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1784-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1785-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1786-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1787-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1788-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1789-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1790-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1791-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1792-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1793-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1794-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1795-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1796-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1797-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1798-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1799-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1800-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1801-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1802-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1803-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1804-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1805-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1806-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1807-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1808-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1809-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1810-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1811-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1812-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1813-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1814-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1815-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1816-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1817-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1818-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1819-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1820-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1821-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1822-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1823-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1824-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1825-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1826-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1827-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1828-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1829-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1830-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1831-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1832-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1833-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1834-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1835-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1836-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1837-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1838-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1839-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1840-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1841-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1842-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1843-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1844-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1845-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1846-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1847-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1848-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1849-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1850-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1851-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1852-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1853-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1854-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1855-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1856-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1857-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1858-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1859-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1860-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1861-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1862-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1863-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1864-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1865-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1866-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1867-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1868-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1869-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1870-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1871-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1872-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1873-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1874-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1875-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1876-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1877-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1878-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1879-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1880-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1881-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1882-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1883-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1884-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1885-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1886-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1887-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1888-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1889-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1890-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1891-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1892-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1893-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1894-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1895-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1896-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1897-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1898-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1899-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1900-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1901-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1902-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1903-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1904-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1905-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1906-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1907-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1908-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1909-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1910-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1911-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1912-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1913-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1914-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1915-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1916-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1917-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1918-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1919-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1920-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1921-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1922-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1923-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1924-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1925-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1926-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1927-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1928-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1929-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1930-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1931-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1932-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1933-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1934-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1935-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1936-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1937-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1938-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1939-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1940-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1941-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1942-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1943-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1944-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1945-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1946-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1947-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1948-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1949-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1950-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1951-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1952-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1953-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1954-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1955-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1956-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1957-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1958-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1959-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1960-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1961-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1962-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1963-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1964-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1965-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1966-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1967-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1968-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1969-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1970-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1971-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1972-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1973-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1974-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1975-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1976-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1977-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1978-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1979-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1980-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1981-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1982-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1983-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1984-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1985-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1986-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1987-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1988-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1989-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1990-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1991-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1992-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1993-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1994-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1995-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1996-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1997-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1998-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1999-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2000-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2001-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2002-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2003-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2004-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2005-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2006-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2007-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2008-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2009-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2010-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2011-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2012-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2013-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2014-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2015-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2016-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2017-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2018-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2019-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2020-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2021-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2022-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2023-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2024-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2025-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2026-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2027-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2028-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2029-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2030-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2031-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2032-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2033-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2034-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2035-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2036-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2037-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2038-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2039-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2040-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2041-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2042-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2043-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2044-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2045-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2046-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2047-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH $@ d@@@$$A,dA4A?%HA eHCHE+HG;%IIKeIK[IMkIO{%JQeJSJUJW%KYeK[ۥK]K_%La eLcLe+Lg;%MiKeMk[MmkMo{%NqeNsNuNw%OyeO{ۥO}ObPPQ6Q6$mD64mɣFR6DmHS6S6+GPre+/@Gr+kGr%,/Gs,˹GsE-/Gs-Gt.+GPte.KH. +k Ht %/0Hu `e: +#0(ve< S0)e$= 0*e*? 0%+e6@I0)e:pWz@pW{A qW'{C cqW7{Dk:“qWG{F:qXW{Iqe//m"/q(/y4/:/@/F/L/R/ G .U +gN ǵn! ''*U-G 0"3X'\K*;cX0^k2ʓXG_XWaBXgb J;#Xd R̃Xe+Z;ͳYgKYj jYǸkr;C(Yt|/t|/t|/t|/t|/u|/u|/u|/u|/v|/v|/-TfMm&F.Ug@ n ' +XaJ XiLҺ8XlM:DXqOPXuP\XyRhX|S:tX́UX̅V +X̉YX͍[g{X/#rw{^/ Sr{d/ r{j/+r{v/Kr{|/ks{/ss{/s{/s|/ t|/ 36&9(<*?,B0H2K4N6QT @YUŝgNLYǸXY̝;dYU繢pY՝G;|YYݝ.Yg;NYǻʻ`Y'msv|nУv|pv}q w}s+3w0}tkғwG}vwW}w wg}z#xp}|Sx}} Ճx///HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHwwHHHHHHHHHHHHHHHH HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @@ @@ @@ @@ @@aaAaa`a!aaaaaaaAaa`a!aaaaaaaAaa`a!aaaaaaaAaAaa`a!aaaaaaaAaa`a!aaaaaaaAaa`a!aaaaaaaAaaHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH  greenblueredHHHHHHHHHHHHHHHHHHHHHHHHHHHH    HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ @    +@ + + + @    @    @   @@@@@@@@@@@@@@@@@@ @   !@!!!"@"""#@###$@$$$%@%%%&@&&&'@'''(@((()@)))*@***+@+++,@,,,-@---.@.../@///0@0001@1112@2223@3334@4445@5556@6667@7778@8889@999:@:::;@;;;<@<<<=@===>@>>>?@???@@@@@A@AAAB@BBBC@CCCD@DDDE@EEEF@FFFG@GGGH@HHHI@IIIJ@JJJK@KKKL@LLLM@MMMN@NNNO@OOOP@PPPQ@QQQR@RRRS@SSST@TTTU@UUUV@VVVW@WWWX@XXXY@YYYZ@ZZZ[@[[[\@\\\]@]]]^@^^^_@___`@```a@aaab@bbbc@cccd@ddde@eeef@fffg@gggh@hhhi@iiij@jjjk@kkkl@lllm@mmmn@nnno@ooop@pppq@qqqr@rrrs@ssst@tttu@uuuv@vvvw@wwwx@xxxy@yyyz@zzz{@{{{|@|||}@}}}~@~~~@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2048-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2049-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2050-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2051-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2052-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2053-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2054-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2055-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2056-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2057-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2058-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2059-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2060-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2061-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2062-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2063-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2064-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2065-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2066-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2067-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2068-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2069-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2070-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2071-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2072-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2073-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2074-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2075-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2076-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2077-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2078-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2079-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2080-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2081-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2082-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2083-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2084-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2085-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2086-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2087-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2088-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2089-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2090-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2091-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2092-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2093-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2094-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2095-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2096-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2097-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2098-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2099-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2100-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2101-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2102-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2103-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2104-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2105-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2106-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2107-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2108-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2109-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2110-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2111-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2112-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2113-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2114-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2115-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2116-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2117-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2118-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2119-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2120-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2121-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2122-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2123-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2124-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2125-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2126-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2127-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2128-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2129-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2130-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2131-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2132-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2133-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2134-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2135-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2136-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2137-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2138-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2139-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2140-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2141-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2142-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2143-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2144-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2145-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2146-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2147-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2148-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2149-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2150-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2151-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2152-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2153-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2154-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2155-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2156-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2157-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2158-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2159-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2160-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2161-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2162-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2163-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2164-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2165-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2166-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2167-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2168-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2169-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2170-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2171-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2172-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2173-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2174-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2175-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2176-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2177-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2178-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2179-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2180-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2181-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2182-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2183-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2184-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2185-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2186-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2187-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2188-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2189-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2190-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2191-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2192-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2193-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2194-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2195-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2196-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2197-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2198-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2199-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2200-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2201-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2202-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2203-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2204-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2205-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2206-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2207-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2208-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2209-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2210-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2211-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2212-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2213-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2214-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2215-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2216-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2217-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2218-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2219-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2220-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2221-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2222-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2223-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2224-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2225-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2226-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2227-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2228-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2229-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2230-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2231-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2232-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2233-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2234-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2235-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2236-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2237-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2238-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2239-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2240-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2241-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2242-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2243-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2244-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2245-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2246-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2247-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2248-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2249-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2250-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2251-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2252-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2253-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2254-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2255-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2256-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2257-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2258-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2259-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2260-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2261-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2262-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2263-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2264-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2265-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2266-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2267-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2268-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2269-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2270-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2271-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2272-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2273-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2274-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2275-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2276-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2277-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2278-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2279-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2280-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2281-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2282-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2283-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2284-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2285-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2286-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2287-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2288-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2289-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2290-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2291-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2292-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2293-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2294-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2295-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2296-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2297-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2298-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2299-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2300-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2301-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2302-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2303-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2304-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2305-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2306-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2307-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2308-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2309-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2310-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2311-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2312-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2313-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2314-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2315-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2316-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2317-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2318-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2319-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2320-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2321-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2322-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2323-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2324-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2325-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2326-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2327-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2328-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2329-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2330-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2331-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2332-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2333-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2334-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2335-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2336-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2337-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2338-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2339-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2340-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2341-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2342-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2343-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2344-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2345-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2346-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2347-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2348-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2349-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2350-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2351-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2352-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2353-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2354-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2355-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2356-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2357-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2358-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2359-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2360-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2361-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2362-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2363-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2364-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2365-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2366-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2367-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2368-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2369-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2370-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2371-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2372-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2373-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2374-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2375-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2376-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2377-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2378-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2379-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2380-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2381-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2382-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2383-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2384-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2385-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2386-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2387-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2388-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2389-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2390-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2391-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2392-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2393-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2394-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2395-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2396-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2397-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2398-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2399-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2400-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2401-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2402-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2403-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2404-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2405-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2406-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2407-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2408-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2409-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2410-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2411-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2412-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2413-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2414-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2415-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2416-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2417-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2418-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2419-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2420-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2421-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2422-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2423-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2424-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2425-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2426-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2427-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2428-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2429-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2430-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2431-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2432-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2433-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2434-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2435-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2436-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2437-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2438-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2439-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2440-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2441-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2442-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2443-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2444-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2445-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2446-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2447-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2448-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2449-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2450-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2451-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2452-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2453-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2454-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2455-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2456-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2457-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2458-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2459-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2460-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2461-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2462-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2463-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2464-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2465-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2466-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2467-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2468-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2469-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2470-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2471-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2472-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2473-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2474-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2475-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2476-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2477-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2478-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2479-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2480-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2481-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2482-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2483-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2484-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2485-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2486-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2487-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2488-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2489-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2490-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2491-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2492-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2493-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2494-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2495-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2496-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2497-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2498-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2499-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2500-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2501-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2502-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2503-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2504-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2505-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2506-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2507-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2508-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2509-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2510-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2511-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2512-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2513-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2514-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2515-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2516-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2517-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2518-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2519-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2520-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2521-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2522-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2523-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2524-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2525-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2526-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2527-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2528-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2529-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2530-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2531-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2532-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2533-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2534-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2535-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2536-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2537-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2538-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2539-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2540-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2541-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2542-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2543-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2544-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2545-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2546-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2547-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2548-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2549-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2550-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2551-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2552-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2553-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2554-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2555-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2556-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2557-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2558-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2559-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2560-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2561-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2562-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2563-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2564-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2565-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2566-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2567-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2568-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2569-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2570-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2571-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2572-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2573-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2574-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2575-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2576-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2577-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2578-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2579-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2580-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2581-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2582-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2583-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2584-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2585-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2586-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2587-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2588-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2589-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2590-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2591-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2592-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2593-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2594-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2595-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2596-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2597-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2598-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2599-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2600-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2601-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2602-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2603-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2604-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2605-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2606-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2607-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2608-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2609-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2610-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2611-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2612-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2613-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2614-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2615-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2616-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2617-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2618-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2619-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2620-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2621-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2622-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2623-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2624-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2625-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2626-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2627-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2628-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2629-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2630-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2631-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2632-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2633-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2634-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2635-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2636-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2637-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2638-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2639-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2640-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2641-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2642-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2643-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2644-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2645-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2646-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2647-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2648-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2649-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2650-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2651-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2652-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2653-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2654-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2655-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2656-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2657-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2658-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2659-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2660-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2661-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2662-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2663-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2664-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2665-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2666-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2667-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2668-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2669-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2670-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2671-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2672-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2673-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2674-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2675-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2676-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2677-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2678-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2679-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2680-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2681-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2682-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2683-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2684-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2685-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2686-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2687-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2688-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2689-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2690-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2691-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2692-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2693-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2694-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2695-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2696-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2697-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2698-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2699-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2700-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2701-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2702-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2703-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2704-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2705-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2706-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2707-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2708-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2709-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2710-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2711-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2712-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2713-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2714-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2715-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2716-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2717-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2718-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2719-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2720-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2721-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2722-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2723-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2724-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2725-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2726-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2727-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2728-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2729-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2730-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2731-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2732-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2733-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2734-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2735-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2736-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2737-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2738-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2739-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2740-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2741-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2742-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2743-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2744-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2745-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2746-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2747-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2748-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2749-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2750-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2751-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2752-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2753-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2754-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2755-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2756-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2757-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2758-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2759-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2760-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2761-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2762-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2763-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2764-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2765-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2766-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2767-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2768-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2769-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2770-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2771-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2772-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2773-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2774-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2775-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2776-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2777-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2778-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2779-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2780-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2781-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2782-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2783-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2784-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2785-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2786-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2787-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2788-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2789-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2790-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2791-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2792-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2793-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2794-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2795-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2796-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2797-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2798-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2799-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2800-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2801-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2802-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2803-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2804-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2805-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2806-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2807-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2808-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2809-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2810-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2811-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2812-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2813-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2814-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2815-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2816-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2817-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2818-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2819-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2820-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2821-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2822-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2823-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2824-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2825-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2826-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2827-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2828-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2829-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2830-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2831-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2832-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2833-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2834-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2835-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2836-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2837-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2838-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2839-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2840-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2841-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2842-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2843-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2844-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2845-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2846-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2847-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2848-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2849-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2850-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2851-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2852-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2853-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2854-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2855-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2856-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2857-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2858-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2859-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2860-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2861-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2862-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2863-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2864-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2865-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2866-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2867-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2868-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2869-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2870-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2871-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2872-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2873-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2874-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2875-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2876-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2877-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2878-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2879-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2880-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2881-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2882-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2883-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2884-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2885-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2886-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2887-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2888-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2889-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2890-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2891-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2892-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2893-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2894-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2895-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2896-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2897-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2898-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2899-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2900-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2901-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2902-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2903-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2904-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2905-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2906-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2907-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2908-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2909-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2910-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2911-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2912-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2913-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2914-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2915-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2916-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2917-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2918-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2919-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2920-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2921-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2922-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2923-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2924-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2925-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2926-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2927-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2928-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2929-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2930-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2931-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2932-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2933-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2934-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2935-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2936-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2937-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2938-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2939-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2940-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2941-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2942-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2943-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2944-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2945-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2946-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2947-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2948-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2949-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2950-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2951-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2952-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2953-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2954-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2955-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2956-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2957-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2958-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2959-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2960-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2961-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2962-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2963-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2964-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2965-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2966-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2967-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2968-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2969-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2970-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2971-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2972-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2973-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2974-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2975-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2976-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2977-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2978-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2979-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2980-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2981-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2982-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2983-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2984-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2985-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2986-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2987-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2988-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2989-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2990-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2991-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2992-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2993-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2994-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2995-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2996-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2997-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2998-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2999-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3000-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3001-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3002-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3003-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3004-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3005-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3006-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3007-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3008-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3009-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3010-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3011-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3012-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3013-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3014-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3015-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3016-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3017-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3018-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3019-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3020-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3021-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3022-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3023-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3024-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3025-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3026-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3027-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3028-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3029-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3030-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3031-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3032-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3033-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3034-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3035-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3036-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3037-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3038-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3039-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3040-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3041-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3042-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3043-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3044-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3045-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3046-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3047-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3048-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3049-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3050-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3051-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3052-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3053-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3054-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3055-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3056-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3057-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3058-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3059-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3060-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3061-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3062-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3063-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3064-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3065-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3066-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3067-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3068-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3069-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3070-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3071-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH (8HXhx + + Ȉ ؈ (8HXhxȉ؉ +)*9:IJYZijyz + ɘ ٘   +)*9:IJYZijyzəٙ +;K[k{ + ˸ ۸븨 +;K[k{˹۹빩@@AAB(BC8CDHDEXEFhFGxGHHIIJJKKLȌLM،MNNOOPPQQR(RS8STHTUXUVhVWxWXXYYZZ[[\ȍ\]؍]^^__ @ +A)B*9C:IDJYEZiFjyGzHIJKɜLٜMNO P +Q)R*9S:ITJYUZiVjyWzXYZ[ɝ\ٝ]^_@ AB+C;DKE[FkG{HIJKL˼MۼN뼬OP QR+S;TKU[VkW{XYZ[\˽]۽^뽭_  !!"("#8#$H$%X%&h&'x'(())**++,Ȋ,-؊-..//00112(23834H45X56h67x78899::;;<ȋ<=؋=>>?? +!)"*9#:I$JY%Zi&jy'z()*+ɚ,ٚ-./ 0 +1)2*93:I4JY5Zi6jy7z89:;ɛ<ٛ=>? !"+#;$K%[&k'{()*+,˺-ۺ.뺪/0 12+3;4K5[6k7{89:;<˻=ۻ>뻫?``aab(bc8cdHdeXefhfgxghhiijjkklȎlm؎mnnooppqqr(rs8stHtuXuvhvwxwxxyyzz{{|ȏ|}؏}~~ ` +a)b*9c:IdJYeZifjygzhijkɞlٞmno p +q)r*9s:ItJYuZivjywzxyz{ɟ|ٟ}~` ab+c;dKe[fkg{hijkl˾m۾n뾮op qr+s;tKu[vkw{xyz{|˿}ۿ~뿯HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH g g HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH$Ddd$$Dd$Dd$Ddd$$Dd$Dd$Ddd $ $ D d  +$ +D +d + + + + + + $ D d d  $ $ D d  $ D d $Ddd$$Ddvalue-2048-deterministic-fixturevalue-2049-deterministic-fixturevalue-2050-deterministic-fixturevalue-2052-deterministic-fixturevalue-2053-deterministic-fixturevalue-2054-deterministic-fixturevalue-2055-deterministic-fixturevalue-2056-deterministic-fixturevalue-2057-deterministic-fixturevalue-2059-deterministic-fixturevalue-2060-deterministic-fixturevalue-2061-deterministic-fixturevalue-2062-deterministic-fixturevalue-2063-deterministic-fixturevalue-2064-deterministic-fixturevalue-2066-deterministic-fixturevalue-2067-deterministic-fixturevalue-2068-deterministic-fixturevalue-2069-deterministic-fixturevalue-2070-deterministic-fixturevalue-2071-deterministic-fixturevalue-2073-deterministic-fixturevalue-2074-deterministic-fixturevalue-2075-deterministic-fixturevalue-2076-deterministic-fixturevalue-2077-deterministic-fixturevalue-2078-deterministic-fixturevalue-2080-deterministic-fixturevalue-2081-deterministic-fixturevalue-2082-deterministic-fixturevalue-2083-deterministic-fixturevalue-2084-deterministic-fixturevalue-2085-deterministic-fixturevalue-2087-deterministic-fixturevalue-2088-deterministic-fixturevalue-2089-deterministic-fixturevalue-2090-deterministic-fixturevalue-2091-deterministic-fixturevalue-2092-deterministic-fixturevalue-2094-deterministic-fixturevalue-2095-deterministic-fixturevalue-2096-deterministic-fixturevalue-2097-deterministic-fixturevalue-2098-deterministic-fixturevalue-2099-deterministic-fixturevalue-2101-deterministic-fixturevalue-2102-deterministic-fixturevalue-2103-deterministic-fixturevalue-2104-deterministic-fixturevalue-2105-deterministic-fixturevalue-2106-deterministic-fixturevalue-2108-deterministic-fixturevalue-2109-deterministic-fixturevalue-2110-deterministic-fixturevalue-2111-deterministic-fixturevalue-2112-deterministic-fixturevalue-2113-deterministic-fixturevalue-2115-deterministic-fixturevalue-2116-deterministic-fixturevalue-2117-deterministic-fixturevalue-2118-deterministic-fixturevalue-2119-deterministic-fixturevalue-2120-deterministic-fixturevalue-2122-deterministic-fixturevalue-2123-deterministic-fixturevalue-2124-deterministic-fixturevalue-2125-deterministic-fixturevalue-2126-deterministic-fixturevalue-2127-deterministic-fixturevalue-2129-deterministic-fixturevalue-2130-deterministic-fixturevalue-2131-deterministic-fixturevalue-2132-deterministic-fixturevalue-2133-deterministic-fixturevalue-2134-deterministic-fixturevalue-2136-deterministic-fixturevalue-2137-deterministic-fixturevalue-2138-deterministic-fixturevalue-2139-deterministic-fixturevalue-2140-deterministic-fixturevalue-2141-deterministic-fixturevalue-2143-deterministic-fixturevalue-2144-deterministic-fixturevalue-2145-deterministic-fixturevalue-2146-deterministic-fixturevalue-2147-deterministic-fixturevalue-2148-deterministic-fixturevalue-2150-deterministic-fixturevalue-2151-deterministic-fixturevalue-2152-deterministic-fixturevalue-2153-deterministic-fixturevalue-2154-deterministic-fixturevalue-2155-deterministic-fixturevalue-2157-deterministic-fixturevalue-2158-deterministic-fixturevalue-2159-deterministic-fixturevalue-2160-deterministic-fixturevalue-2161-deterministic-fixturevalue-2162-deterministic-fixturevalue-2164-deterministic-fixturevalue-2165-deterministic-fixturevalue-2166-deterministic-fixturevalue-2167-deterministic-fixturevalue-2168-deterministic-fixturevalue-2169-deterministic-fixturevalue-2171-deterministic-fixturevalue-2172-deterministic-fixturevalue-2173-deterministic-fixturevalue-2174-deterministic-fixturevalue-2175-deterministic-fixture$$Dd$Dd$Ddd$$Dd$Dd$Ddd$$Dd $ D d  +$ +D +d +d + + + + + $ $ D d  $ D d  $ D d d $$Dd$Ddvalue-2176-deterministic-fixturevalue-2178-deterministic-fixturevalue-2179-deterministic-fixturevalue-2180-deterministic-fixturevalue-2181-deterministic-fixturevalue-2182-deterministic-fixturevalue-2183-deterministic-fixturevalue-2185-deterministic-fixturevalue-2186-deterministic-fixturevalue-2187-deterministic-fixturevalue-2188-deterministic-fixturevalue-2189-deterministic-fixturevalue-2190-deterministic-fixturevalue-2192-deterministic-fixturevalue-2193-deterministic-fixturevalue-2194-deterministic-fixturevalue-2195-deterministic-fixturevalue-2196-deterministic-fixturevalue-2197-deterministic-fixturevalue-2199-deterministic-fixturevalue-2200-deterministic-fixturevalue-2201-deterministic-fixturevalue-2202-deterministic-fixturevalue-2203-deterministic-fixturevalue-2204-deterministic-fixturevalue-2206-deterministic-fixturevalue-2207-deterministic-fixturevalue-2208-deterministic-fixturevalue-2209-deterministic-fixturevalue-2210-deterministic-fixturevalue-2211-deterministic-fixturevalue-2213-deterministic-fixturevalue-2214-deterministic-fixturevalue-2215-deterministic-fixturevalue-2216-deterministic-fixturevalue-2217-deterministic-fixturevalue-2218-deterministic-fixturevalue-2220-deterministic-fixturevalue-2221-deterministic-fixturevalue-2222-deterministic-fixturevalue-2223-deterministic-fixturevalue-2224-deterministic-fixturevalue-2225-deterministic-fixturevalue-2227-deterministic-fixturevalue-2228-deterministic-fixturevalue-2229-deterministic-fixturevalue-2230-deterministic-fixturevalue-2231-deterministic-fixturevalue-2232-deterministic-fixturevalue-2234-deterministic-fixturevalue-2235-deterministic-fixturevalue-2236-deterministic-fixturevalue-2237-deterministic-fixturevalue-2238-deterministic-fixturevalue-2239-deterministic-fixturevalue-2241-deterministic-fixturevalue-2242-deterministic-fixturevalue-2243-deterministic-fixturevalue-2244-deterministic-fixturevalue-2245-deterministic-fixturevalue-2246-deterministic-fixturevalue-2248-deterministic-fixturevalue-2249-deterministic-fixturevalue-2250-deterministic-fixturevalue-2251-deterministic-fixturevalue-2252-deterministic-fixturevalue-2253-deterministic-fixturevalue-2255-deterministic-fixturevalue-2256-deterministic-fixturevalue-2257-deterministic-fixturevalue-2258-deterministic-fixturevalue-2259-deterministic-fixturevalue-2260-deterministic-fixturevalue-2262-deterministic-fixturevalue-2263-deterministic-fixturevalue-2264-deterministic-fixturevalue-2265-deterministic-fixturevalue-2266-deterministic-fixturevalue-2267-deterministic-fixturevalue-2269-deterministic-fixturevalue-2270-deterministic-fixturevalue-2271-deterministic-fixturevalue-2272-deterministic-fixturevalue-2273-deterministic-fixturevalue-2274-deterministic-fixturevalue-2276-deterministic-fixturevalue-2277-deterministic-fixturevalue-2278-deterministic-fixturevalue-2279-deterministic-fixturevalue-2280-deterministic-fixturevalue-2281-deterministic-fixturevalue-2283-deterministic-fixturevalue-2284-deterministic-fixturevalue-2285-deterministic-fixturevalue-2286-deterministic-fixturevalue-2287-deterministic-fixturevalue-2288-deterministic-fixturevalue-2290-deterministic-fixturevalue-2291-deterministic-fixturevalue-2292-deterministic-fixturevalue-2293-deterministic-fixturevalue-2294-deterministic-fixturevalue-2295-deterministic-fixturevalue-2297-deterministic-fixturevalue-2298-deterministic-fixturevalue-2299-deterministic-fixturevalue-2300-deterministic-fixturevalue-2301-deterministic-fixturevalue-2302-deterministic-fixture$Dd$Dd$DDd$Dd$Dd$DDd$Dd $ D d  +$ +D +D +d + + + + +  $ D d  $ D d  $ D D d $Dd$Ddvalue-2304-deterministic-fixturevalue-2305-deterministic-fixturevalue-2306-deterministic-fixturevalue-2307-deterministic-fixturevalue-2308-deterministic-fixturevalue-2309-deterministic-fixturevalue-2311-deterministic-fixturevalue-2312-deterministic-fixturevalue-2313-deterministic-fixturevalue-2314-deterministic-fixturevalue-2315-deterministic-fixturevalue-2316-deterministic-fixturevalue-2318-deterministic-fixturevalue-2319-deterministic-fixturevalue-2320-deterministic-fixturevalue-2321-deterministic-fixturevalue-2322-deterministic-fixturevalue-2323-deterministic-fixturevalue-2325-deterministic-fixturevalue-2326-deterministic-fixturevalue-2327-deterministic-fixturevalue-2328-deterministic-fixturevalue-2329-deterministic-fixturevalue-2330-deterministic-fixturevalue-2332-deterministic-fixturevalue-2333-deterministic-fixturevalue-2334-deterministic-fixturevalue-2335-deterministic-fixturevalue-2336-deterministic-fixturevalue-2337-deterministic-fixturevalue-2339-deterministic-fixturevalue-2340-deterministic-fixturevalue-2341-deterministic-fixturevalue-2342-deterministic-fixturevalue-2343-deterministic-fixturevalue-2344-deterministic-fixturevalue-2346-deterministic-fixturevalue-2347-deterministic-fixturevalue-2348-deterministic-fixturevalue-2349-deterministic-fixturevalue-2350-deterministic-fixturevalue-2351-deterministic-fixturevalue-2353-deterministic-fixturevalue-2354-deterministic-fixturevalue-2355-deterministic-fixturevalue-2356-deterministic-fixturevalue-2357-deterministic-fixturevalue-2358-deterministic-fixturevalue-2360-deterministic-fixturevalue-2361-deterministic-fixturevalue-2362-deterministic-fixturevalue-2363-deterministic-fixturevalue-2364-deterministic-fixturevalue-2365-deterministic-fixturevalue-2367-deterministic-fixturevalue-2368-deterministic-fixturevalue-2369-deterministic-fixturevalue-2370-deterministic-fixturevalue-2371-deterministic-fixturevalue-2372-deterministic-fixturevalue-2374-deterministic-fixturevalue-2375-deterministic-fixturevalue-2376-deterministic-fixturevalue-2377-deterministic-fixturevalue-2378-deterministic-fixturevalue-2379-deterministic-fixturevalue-2381-deterministic-fixturevalue-2382-deterministic-fixturevalue-2383-deterministic-fixturevalue-2384-deterministic-fixturevalue-2385-deterministic-fixturevalue-2386-deterministic-fixturevalue-2388-deterministic-fixturevalue-2389-deterministic-fixturevalue-2390-deterministic-fixturevalue-2391-deterministic-fixturevalue-2392-deterministic-fixturevalue-2393-deterministic-fixturevalue-2395-deterministic-fixturevalue-2396-deterministic-fixturevalue-2397-deterministic-fixturevalue-2398-deterministic-fixturevalue-2399-deterministic-fixturevalue-2400-deterministic-fixturevalue-2402-deterministic-fixturevalue-2403-deterministic-fixturevalue-2404-deterministic-fixturevalue-2405-deterministic-fixturevalue-2406-deterministic-fixturevalue-2407-deterministic-fixturevalue-2409-deterministic-fixturevalue-2410-deterministic-fixturevalue-2411-deterministic-fixturevalue-2412-deterministic-fixturevalue-2413-deterministic-fixturevalue-2414-deterministic-fixturevalue-2416-deterministic-fixturevalue-2417-deterministic-fixturevalue-2418-deterministic-fixturevalue-2419-deterministic-fixturevalue-2420-deterministic-fixturevalue-2421-deterministic-fixturevalue-2423-deterministic-fixturevalue-2424-deterministic-fixturevalue-2425-deterministic-fixturevalue-2426-deterministic-fixturevalue-2427-deterministic-fixturevalue-2428-deterministic-fixturevalue-2430-deterministic-fixturevalue-2431-deterministic-fixture$Dd$DDd$Dd$Dd$DDd$Dd$Dd $ D D d  + +$ +D +d + + + + + + $ D d  $ D D d   $ D d $Dd$DDdvalue-2432-deterministic-fixturevalue-2433-deterministic-fixturevalue-2434-deterministic-fixturevalue-2435-deterministic-fixturevalue-2437-deterministic-fixturevalue-2438-deterministic-fixturevalue-2439-deterministic-fixturevalue-2440-deterministic-fixturevalue-2441-deterministic-fixturevalue-2442-deterministic-fixturevalue-2444-deterministic-fixturevalue-2445-deterministic-fixturevalue-2446-deterministic-fixturevalue-2447-deterministic-fixturevalue-2448-deterministic-fixturevalue-2449-deterministic-fixturevalue-2451-deterministic-fixturevalue-2452-deterministic-fixturevalue-2453-deterministic-fixturevalue-2454-deterministic-fixturevalue-2455-deterministic-fixturevalue-2456-deterministic-fixturevalue-2458-deterministic-fixturevalue-2459-deterministic-fixturevalue-2460-deterministic-fixturevalue-2461-deterministic-fixturevalue-2462-deterministic-fixturevalue-2463-deterministic-fixturevalue-2465-deterministic-fixturevalue-2466-deterministic-fixturevalue-2467-deterministic-fixturevalue-2468-deterministic-fixturevalue-2469-deterministic-fixturevalue-2470-deterministic-fixturevalue-2472-deterministic-fixturevalue-2473-deterministic-fixturevalue-2474-deterministic-fixturevalue-2475-deterministic-fixturevalue-2476-deterministic-fixturevalue-2477-deterministic-fixturevalue-2479-deterministic-fixturevalue-2480-deterministic-fixturevalue-2481-deterministic-fixturevalue-2482-deterministic-fixturevalue-2483-deterministic-fixturevalue-2484-deterministic-fixturevalue-2486-deterministic-fixturevalue-2487-deterministic-fixturevalue-2488-deterministic-fixturevalue-2489-deterministic-fixturevalue-2490-deterministic-fixturevalue-2491-deterministic-fixturevalue-2493-deterministic-fixturevalue-2494-deterministic-fixturevalue-2495-deterministic-fixturevalue-2496-deterministic-fixturevalue-2497-deterministic-fixturevalue-2498-deterministic-fixturevalue-2500-deterministic-fixturevalue-2501-deterministic-fixturevalue-2502-deterministic-fixturevalue-2503-deterministic-fixturevalue-2504-deterministic-fixturevalue-2505-deterministic-fixturevalue-2507-deterministic-fixturevalue-2508-deterministic-fixturevalue-2509-deterministic-fixturevalue-2510-deterministic-fixturevalue-2511-deterministic-fixturevalue-2512-deterministic-fixturevalue-2514-deterministic-fixturevalue-2515-deterministic-fixturevalue-2516-deterministic-fixturevalue-2517-deterministic-fixturevalue-2518-deterministic-fixturevalue-2519-deterministic-fixturevalue-2521-deterministic-fixturevalue-2522-deterministic-fixturevalue-2523-deterministic-fixturevalue-2524-deterministic-fixturevalue-2525-deterministic-fixturevalue-2526-deterministic-fixturevalue-2528-deterministic-fixturevalue-2529-deterministic-fixturevalue-2530-deterministic-fixturevalue-2531-deterministic-fixturevalue-2532-deterministic-fixturevalue-2533-deterministic-fixturevalue-2535-deterministic-fixturevalue-2536-deterministic-fixturevalue-2537-deterministic-fixturevalue-2538-deterministic-fixturevalue-2539-deterministic-fixturevalue-2540-deterministic-fixturevalue-2542-deterministic-fixturevalue-2543-deterministic-fixturevalue-2544-deterministic-fixturevalue-2545-deterministic-fixturevalue-2546-deterministic-fixturevalue-2547-deterministic-fixturevalue-2549-deterministic-fixturevalue-2550-deterministic-fixturevalue-2551-deterministic-fixturevalue-2552-deterministic-fixturevalue-2553-deterministic-fixturevalue-2554-deterministic-fixturevalue-2556-deterministic-fixturevalue-2557-deterministic-fixturevalue-2558-deterministic-fixturevalue-2559-deterministic-fixture$DDd$Dd$Dd$DDd$Dd$Dd$DDd  $ D d  +$ +D +d + + + + + + $ D D d   $ D d  $ D d $DDd$Ddvalue-2560-deterministic-fixturevalue-2561-deterministic-fixturevalue-2563-deterministic-fixturevalue-2564-deterministic-fixturevalue-2565-deterministic-fixturevalue-2566-deterministic-fixturevalue-2567-deterministic-fixturevalue-2568-deterministic-fixturevalue-2570-deterministic-fixturevalue-2571-deterministic-fixturevalue-2572-deterministic-fixturevalue-2573-deterministic-fixturevalue-2574-deterministic-fixturevalue-2575-deterministic-fixturevalue-2577-deterministic-fixturevalue-2578-deterministic-fixturevalue-2579-deterministic-fixturevalue-2580-deterministic-fixturevalue-2581-deterministic-fixturevalue-2582-deterministic-fixturevalue-2584-deterministic-fixturevalue-2585-deterministic-fixturevalue-2586-deterministic-fixturevalue-2587-deterministic-fixturevalue-2588-deterministic-fixturevalue-2589-deterministic-fixturevalue-2591-deterministic-fixturevalue-2592-deterministic-fixturevalue-2593-deterministic-fixturevalue-2594-deterministic-fixturevalue-2595-deterministic-fixturevalue-2596-deterministic-fixturevalue-2598-deterministic-fixturevalue-2599-deterministic-fixturevalue-2600-deterministic-fixturevalue-2601-deterministic-fixturevalue-2602-deterministic-fixturevalue-2603-deterministic-fixturevalue-2605-deterministic-fixturevalue-2606-deterministic-fixturevalue-2607-deterministic-fixturevalue-2608-deterministic-fixturevalue-2609-deterministic-fixturevalue-2610-deterministic-fixturevalue-2612-deterministic-fixturevalue-2613-deterministic-fixturevalue-2614-deterministic-fixturevalue-2615-deterministic-fixturevalue-2616-deterministic-fixturevalue-2617-deterministic-fixturevalue-2619-deterministic-fixturevalue-2620-deterministic-fixturevalue-2621-deterministic-fixturevalue-2622-deterministic-fixturevalue-2623-deterministic-fixturevalue-2624-deterministic-fixturevalue-2626-deterministic-fixturevalue-2627-deterministic-fixturevalue-2628-deterministic-fixturevalue-2629-deterministic-fixturevalue-2630-deterministic-fixturevalue-2631-deterministic-fixturevalue-2633-deterministic-fixturevalue-2634-deterministic-fixturevalue-2635-deterministic-fixturevalue-2636-deterministic-fixturevalue-2637-deterministic-fixturevalue-2638-deterministic-fixturevalue-2640-deterministic-fixturevalue-2641-deterministic-fixturevalue-2642-deterministic-fixturevalue-2643-deterministic-fixturevalue-2644-deterministic-fixturevalue-2645-deterministic-fixturevalue-2647-deterministic-fixturevalue-2648-deterministic-fixturevalue-2649-deterministic-fixturevalue-2650-deterministic-fixturevalue-2651-deterministic-fixturevalue-2652-deterministic-fixturevalue-2654-deterministic-fixturevalue-2655-deterministic-fixturevalue-2656-deterministic-fixturevalue-2657-deterministic-fixturevalue-2658-deterministic-fixturevalue-2659-deterministic-fixturevalue-2661-deterministic-fixturevalue-2662-deterministic-fixturevalue-2663-deterministic-fixturevalue-2664-deterministic-fixturevalue-2665-deterministic-fixturevalue-2666-deterministic-fixturevalue-2668-deterministic-fixturevalue-2669-deterministic-fixturevalue-2670-deterministic-fixturevalue-2671-deterministic-fixturevalue-2672-deterministic-fixturevalue-2673-deterministic-fixturevalue-2675-deterministic-fixturevalue-2676-deterministic-fixturevalue-2677-deterministic-fixturevalue-2678-deterministic-fixturevalue-2679-deterministic-fixturevalue-2680-deterministic-fixturevalue-2682-deterministic-fixturevalue-2683-deterministic-fixturevalue-2684-deterministic-fixturevalue-2685-deterministic-fixturevalue-2686-deterministic-fixturevalue-2687-deterministic-fixture$Dd$Dd$DDd$Dd$Dd$DDd$Dd $ D d  +$ +D +D +d + + + + +  $ D d  $ D d  $ D D d $Dd$Ddvalue-2689-deterministic-fixturevalue-2690-deterministic-fixturevalue-2691-deterministic-fixturevalue-2692-deterministic-fixturevalue-2693-deterministic-fixturevalue-2694-deterministic-fixturevalue-2696-deterministic-fixturevalue-2697-deterministic-fixturevalue-2698-deterministic-fixturevalue-2699-deterministic-fixturevalue-2700-deterministic-fixturevalue-2701-deterministic-fixturevalue-2703-deterministic-fixturevalue-2704-deterministic-fixturevalue-2705-deterministic-fixturevalue-2706-deterministic-fixturevalue-2707-deterministic-fixturevalue-2708-deterministic-fixturevalue-2710-deterministic-fixturevalue-2711-deterministic-fixturevalue-2712-deterministic-fixturevalue-2713-deterministic-fixturevalue-2714-deterministic-fixturevalue-2715-deterministic-fixturevalue-2717-deterministic-fixturevalue-2718-deterministic-fixturevalue-2719-deterministic-fixturevalue-2720-deterministic-fixturevalue-2721-deterministic-fixturevalue-2722-deterministic-fixturevalue-2724-deterministic-fixturevalue-2725-deterministic-fixturevalue-2726-deterministic-fixturevalue-2727-deterministic-fixturevalue-2728-deterministic-fixturevalue-2729-deterministic-fixturevalue-2731-deterministic-fixturevalue-2732-deterministic-fixturevalue-2733-deterministic-fixturevalue-2734-deterministic-fixturevalue-2735-deterministic-fixturevalue-2736-deterministic-fixturevalue-2738-deterministic-fixturevalue-2739-deterministic-fixturevalue-2740-deterministic-fixturevalue-2741-deterministic-fixturevalue-2742-deterministic-fixturevalue-2743-deterministic-fixturevalue-2745-deterministic-fixturevalue-2746-deterministic-fixturevalue-2747-deterministic-fixturevalue-2748-deterministic-fixturevalue-2749-deterministic-fixturevalue-2750-deterministic-fixturevalue-2752-deterministic-fixturevalue-2753-deterministic-fixturevalue-2754-deterministic-fixturevalue-2755-deterministic-fixturevalue-2756-deterministic-fixturevalue-2757-deterministic-fixturevalue-2759-deterministic-fixturevalue-2760-deterministic-fixturevalue-2761-deterministic-fixturevalue-2762-deterministic-fixturevalue-2763-deterministic-fixturevalue-2764-deterministic-fixturevalue-2766-deterministic-fixturevalue-2767-deterministic-fixturevalue-2768-deterministic-fixturevalue-2769-deterministic-fixturevalue-2770-deterministic-fixturevalue-2771-deterministic-fixturevalue-2773-deterministic-fixturevalue-2774-deterministic-fixturevalue-2775-deterministic-fixturevalue-2776-deterministic-fixturevalue-2777-deterministic-fixturevalue-2778-deterministic-fixturevalue-2780-deterministic-fixturevalue-2781-deterministic-fixturevalue-2782-deterministic-fixturevalue-2783-deterministic-fixturevalue-2784-deterministic-fixturevalue-2785-deterministic-fixturevalue-2787-deterministic-fixturevalue-2788-deterministic-fixturevalue-2789-deterministic-fixturevalue-2790-deterministic-fixturevalue-2791-deterministic-fixturevalue-2792-deterministic-fixturevalue-2794-deterministic-fixturevalue-2795-deterministic-fixturevalue-2796-deterministic-fixturevalue-2797-deterministic-fixturevalue-2798-deterministic-fixturevalue-2799-deterministic-fixturevalue-2801-deterministic-fixturevalue-2802-deterministic-fixturevalue-2803-deterministic-fixturevalue-2804-deterministic-fixturevalue-2805-deterministic-fixturevalue-2806-deterministic-fixturevalue-2808-deterministic-fixturevalue-2809-deterministic-fixturevalue-2810-deterministic-fixturevalue-2811-deterministic-fixturevalue-2812-deterministic-fixturevalue-2813-deterministic-fixturevalue-2815-deterministic-fixture$Dd$Ddd$$Dd$Dd$Ddd$$Dd$Dd $ D d d  +$ +$ +D +d + + + + + + $ D d  $ D d d  $ $ D d $Dd$Dddvalue-2816-deterministic-fixturevalue-2817-deterministic-fixturevalue-2818-deterministic-fixturevalue-2819-deterministic-fixturevalue-2820-deterministic-fixturevalue-2822-deterministic-fixturevalue-2823-deterministic-fixturevalue-2824-deterministic-fixturevalue-2825-deterministic-fixturevalue-2826-deterministic-fixturevalue-2827-deterministic-fixturevalue-2829-deterministic-fixturevalue-2830-deterministic-fixturevalue-2831-deterministic-fixturevalue-2832-deterministic-fixturevalue-2833-deterministic-fixturevalue-2834-deterministic-fixturevalue-2836-deterministic-fixturevalue-2837-deterministic-fixturevalue-2838-deterministic-fixturevalue-2839-deterministic-fixturevalue-2840-deterministic-fixturevalue-2841-deterministic-fixturevalue-2843-deterministic-fixturevalue-2844-deterministic-fixturevalue-2845-deterministic-fixturevalue-2846-deterministic-fixturevalue-2847-deterministic-fixturevalue-2848-deterministic-fixturevalue-2850-deterministic-fixturevalue-2851-deterministic-fixturevalue-2852-deterministic-fixturevalue-2853-deterministic-fixturevalue-2854-deterministic-fixturevalue-2855-deterministic-fixturevalue-2857-deterministic-fixturevalue-2858-deterministic-fixturevalue-2859-deterministic-fixturevalue-2860-deterministic-fixturevalue-2861-deterministic-fixturevalue-2862-deterministic-fixturevalue-2864-deterministic-fixturevalue-2865-deterministic-fixturevalue-2866-deterministic-fixturevalue-2867-deterministic-fixturevalue-2868-deterministic-fixturevalue-2869-deterministic-fixturevalue-2871-deterministic-fixturevalue-2872-deterministic-fixturevalue-2873-deterministic-fixturevalue-2874-deterministic-fixturevalue-2875-deterministic-fixturevalue-2876-deterministic-fixturevalue-2878-deterministic-fixturevalue-2879-deterministic-fixturevalue-2880-deterministic-fixturevalue-2881-deterministic-fixturevalue-2882-deterministic-fixturevalue-2883-deterministic-fixturevalue-2885-deterministic-fixturevalue-2886-deterministic-fixturevalue-2887-deterministic-fixturevalue-2888-deterministic-fixturevalue-2889-deterministic-fixturevalue-2890-deterministic-fixturevalue-2892-deterministic-fixturevalue-2893-deterministic-fixturevalue-2894-deterministic-fixturevalue-2895-deterministic-fixturevalue-2896-deterministic-fixturevalue-2897-deterministic-fixturevalue-2899-deterministic-fixturevalue-2900-deterministic-fixturevalue-2901-deterministic-fixturevalue-2902-deterministic-fixturevalue-2903-deterministic-fixturevalue-2904-deterministic-fixturevalue-2906-deterministic-fixturevalue-2907-deterministic-fixturevalue-2908-deterministic-fixturevalue-2909-deterministic-fixturevalue-2910-deterministic-fixturevalue-2911-deterministic-fixturevalue-2913-deterministic-fixturevalue-2914-deterministic-fixturevalue-2915-deterministic-fixturevalue-2916-deterministic-fixturevalue-2917-deterministic-fixturevalue-2918-deterministic-fixturevalue-2920-deterministic-fixturevalue-2921-deterministic-fixturevalue-2922-deterministic-fixturevalue-2923-deterministic-fixturevalue-2924-deterministic-fixturevalue-2925-deterministic-fixturevalue-2927-deterministic-fixturevalue-2928-deterministic-fixturevalue-2929-deterministic-fixturevalue-2930-deterministic-fixturevalue-2931-deterministic-fixturevalue-2932-deterministic-fixturevalue-2934-deterministic-fixturevalue-2935-deterministic-fixturevalue-2936-deterministic-fixturevalue-2937-deterministic-fixturevalue-2938-deterministic-fixturevalue-2939-deterministic-fixturevalue-2941-deterministic-fixturevalue-2942-deterministic-fixturevalue-2943-deterministic-fixture$Ddd$$Dd$Dd$Ddd$$Dd$Dd$Ddd $ $ D d  +$ +D +d + + + + + + $ D d d  $ $ D d  $ D d $Ddd$$Ddvalue-2944-deterministic-fixturevalue-2945-deterministic-fixturevalue-2946-deterministic-fixturevalue-2948-deterministic-fixturevalue-2949-deterministic-fixturevalue-2950-deterministic-fixturevalue-2951-deterministic-fixturevalue-2952-deterministic-fixturevalue-2953-deterministic-fixturevalue-2955-deterministic-fixturevalue-2956-deterministic-fixturevalue-2957-deterministic-fixturevalue-2958-deterministic-fixturevalue-2959-deterministic-fixturevalue-2960-deterministic-fixturevalue-2962-deterministic-fixturevalue-2963-deterministic-fixturevalue-2964-deterministic-fixturevalue-2965-deterministic-fixturevalue-2966-deterministic-fixturevalue-2967-deterministic-fixturevalue-2969-deterministic-fixturevalue-2970-deterministic-fixturevalue-2971-deterministic-fixturevalue-2972-deterministic-fixturevalue-2973-deterministic-fixturevalue-2974-deterministic-fixturevalue-2976-deterministic-fixturevalue-2977-deterministic-fixturevalue-2978-deterministic-fixturevalue-2979-deterministic-fixturevalue-2980-deterministic-fixturevalue-2981-deterministic-fixturevalue-2983-deterministic-fixturevalue-2984-deterministic-fixturevalue-2985-deterministic-fixturevalue-2986-deterministic-fixturevalue-2987-deterministic-fixturevalue-2988-deterministic-fixturevalue-2990-deterministic-fixturevalue-2991-deterministic-fixturevalue-2992-deterministic-fixturevalue-2993-deterministic-fixturevalue-2994-deterministic-fixturevalue-2995-deterministic-fixturevalue-2997-deterministic-fixturevalue-2998-deterministic-fixturevalue-2999-deterministic-fixturevalue-3000-deterministic-fixturevalue-3001-deterministic-fixturevalue-3002-deterministic-fixturevalue-3004-deterministic-fixturevalue-3005-deterministic-fixturevalue-3006-deterministic-fixturevalue-3007-deterministic-fixturevalue-3008-deterministic-fixturevalue-3009-deterministic-fixturevalue-3011-deterministic-fixturevalue-3012-deterministic-fixturevalue-3013-deterministic-fixturevalue-3014-deterministic-fixturevalue-3015-deterministic-fixturevalue-3016-deterministic-fixturevalue-3018-deterministic-fixturevalue-3019-deterministic-fixturevalue-3020-deterministic-fixturevalue-3021-deterministic-fixturevalue-3022-deterministic-fixturevalue-3023-deterministic-fixturevalue-3025-deterministic-fixturevalue-3026-deterministic-fixturevalue-3027-deterministic-fixturevalue-3028-deterministic-fixturevalue-3029-deterministic-fixturevalue-3030-deterministic-fixturevalue-3032-deterministic-fixturevalue-3033-deterministic-fixturevalue-3034-deterministic-fixturevalue-3035-deterministic-fixturevalue-3036-deterministic-fixturevalue-3037-deterministic-fixturevalue-3039-deterministic-fixturevalue-3040-deterministic-fixturevalue-3041-deterministic-fixturevalue-3042-deterministic-fixturevalue-3043-deterministic-fixturevalue-3044-deterministic-fixturevalue-3046-deterministic-fixturevalue-3047-deterministic-fixturevalue-3048-deterministic-fixturevalue-3049-deterministic-fixturevalue-3050-deterministic-fixturevalue-3051-deterministic-fixturevalue-3053-deterministic-fixturevalue-3054-deterministic-fixturevalue-3055-deterministic-fixturevalue-3056-deterministic-fixturevalue-3057-deterministic-fixturevalue-3058-deterministic-fixturevalue-3060-deterministic-fixturevalue-3061-deterministic-fixturevalue-3062-deterministic-fixturevalue-3063-deterministic-fixturevalue-3064-deterministic-fixturevalue-3065-deterministic-fixturevalue-3067-deterministic-fixturevalue-3068-deterministic-fixturevalue-3069-deterministic-fixturevalue-3070-deterministic-fixturevalue-3071-deterministic-fixture::HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH"D$ddd2d2D2D2D"D"L"L"L&L&H&H&H$H$I$I$ɒ$$ddd2d2D2D2D"D"L"L"L&L&H&H&H$H$I$I$ɒ$$!@@@ @@ @!@@ H@ @@ @@ @H@  H]xi ~ uX hP p ( +#  H+  +;0 hCPBFdȋ!ҌBGDdЋ!լGtd苡CHd!Hԣd CId!,I4d LCJdd8!lJdX CKdh!K$dxcF|LҖFڂ5PّӦF򈵍\QԶ@G +dG"5lޑֽG:5pQ#HR5|@HjͣH5HQ&#I5.JEH0JH2 JK40 JNXh@ JQ8 JT: JW< JZ H J`!B +Jc"D0 +JfXqe8!lR}eHڡ CSĉeX!Se`CT$ex!TTe CUe! UemCVe!VDeWte 6 dL  L6 R$M6 dMM*6!M0R%$N66)dN<NB61NHRK! ` K$ (K'&2K*X6J$K-FHb ,K0z 4K3Xfh$P8K6vȪ&pDK9(*TK?\KBXKLdSCMd!MekCNe!sN)e{CO5e!OAeơCPMe!PYeΡCQee!CRTcĵ ф5,񑴥#5 c5l5D،赬Qt#5c 5,4 LRdF#IFJiV$IHJl%JJov&J*LJrކ'JBNJuߖ(JZPJxক)KrRJ{+KVJƕ,KXJ֕-KZJ.L\hDP +f" p + f(D +%f" +1fX+D +=f" 3 If!;D0 UfX""h!CP af&!p mf*"("SD yf,""[İ fX5NXԩ9T Y=Z -!Y4(` M%ZdI8l &ZMHr)[ĪQ`x+[$x~-\TY .]]-1]a$O +6dKƖO"6lK֖O:⶟tKRPR6|KdPjⶠKP6KPⶡKR&Q6K6dQⶢKVQ6KfQ6K"Ddd2d2D2D2D"D"L"L"L&L&H&H&H$H$I$I$ɒ$$ddd2d2D2D2D"D"L"L"L&L&H&H&H$H$I$I$ɒ$$ddd@@ @!@@@ @@ @!@@ H@ @@ @iePxRjnԔ{>'jqդ|?Gktִ} AgRkw~AkBzԀBkrICl DSl҆!P30xf:(4x|R1Ԙ215JgR@ +ˇRT2U˧R3eJ"4u +RTRS5'S5JGS6 +T+V+ V +VI" + $+!V0&+)VI<(+1V H,S+œ9VT!+`0)+ȜQVIx2,+YV ĞDMpoBPN=uo\O=yphP>}ptQp2$RG>qb 0Sg>qDS>qTT>q)dUr"1tV>yT(ޞE+ݜVÍI+ ؁DSM+朙VFT+VHY+VIJ]+rR9V'?r够WG?rI崟XsQğY?sBYПZ?sra[?t\?tq(\t@]'@t2 L^G@ubDXQV +yT(BWy(CQXy(DQY +yT( DQZz(EQ\zԢ(E]Jz(Q^ +zT(FQ_z(GQ`zԣ( GaJz+WUV+"WeV+"*WuV+RȂW+:WԂ+BWGW+JWgW+ūW+BbWիW+rjWW+rW2 22"" " " & &&&$$ $ $ $ $ $ $ $ 2$ 2 22"" " " & &&&$$ $ $ $ $ $ $ $ 2$ 2 2@ H@ @@@ @!@ UޕB VǢРJW  +X Y(2٠%JZHb 5 +[hUɒ⠒E[ U\ `J]" u +^ĉTi-ЉVi-[܉X-[Zj-[j. +\^j.  bj .\d .\$j.*\@>>>?@???@@@@@A@AAAB@BBBC@CCCD@DDDE@EEEF@FFFG@GGGH@HHHI@IIIJ@JJJK@KKKL@LLLM@MMMN@NNNO@OOOP@PPPQ@QQQR@RRRS@SSST@TTTU@UUUV@VVVW@WWWX@XXXY@YYYZ@ZZZ[@[[[\@\\\]@]]]^@^^^_@___`@```a@aaab@bbbc@cccd@ddde@eeef@fffg@gggh@hhhi@iiij@jjjk@kkkl@lllm@mmmn@nnno@ooop@pppq@qqqr@rrrs@ssst@tttu@uuuv@vvvw@wwwx@xxxy@yyyz@zzz{@{{{|@|||}@}}}~@~~~@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3072-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3073-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3074-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3075-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3076-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3077-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3078-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3079-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3080-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3081-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3082-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3083-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3084-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3085-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3086-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3087-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3088-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3089-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3090-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3091-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3092-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3093-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3094-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3095-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3096-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3097-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3098-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3099-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3100-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3101-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3102-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3103-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3104-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3105-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3106-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3107-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3108-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3109-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3110-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3111-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3112-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3113-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3114-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3115-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3116-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3117-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3118-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3119-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3120-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3121-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3122-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3123-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3124-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3125-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3126-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3127-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3128-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3129-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3130-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3131-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3132-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3133-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3134-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3135-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3136-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3137-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3138-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3139-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3140-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3141-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3142-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3143-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3144-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3145-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3146-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3147-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3148-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3149-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3150-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3151-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3152-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3153-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3154-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3155-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3156-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3157-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3158-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3159-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3160-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3161-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3162-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3163-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3164-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3165-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3166-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3167-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3168-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3169-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3170-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3171-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3172-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3173-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3174-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3175-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3176-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3177-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3178-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3179-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3180-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3181-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3182-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3183-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3184-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3185-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3186-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3187-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3188-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3189-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3190-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3191-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3192-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3193-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3194-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3195-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3196-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3197-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3198-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3199-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3200-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3201-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3202-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3203-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3204-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3205-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3206-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3207-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3208-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3209-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3210-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3211-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3212-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3213-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3214-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3215-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3216-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3217-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3218-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3219-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3220-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3221-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3222-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3223-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3224-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3225-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3226-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3227-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3228-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3229-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3230-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3231-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3232-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3233-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3234-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3235-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3236-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3237-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3238-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3239-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3240-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3241-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3242-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3243-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3244-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3245-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3246-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3247-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3248-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3249-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3250-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3251-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3252-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3253-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3254-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3255-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3256-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3257-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3258-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3259-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3260-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3261-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3262-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3263-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3264-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3265-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3266-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3267-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3268-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3269-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3270-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3271-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3272-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3273-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3274-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3275-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3276-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3277-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3278-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3279-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3280-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3281-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3282-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3283-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3284-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3285-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3286-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3287-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3288-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3289-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3290-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3291-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3292-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3293-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3294-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3295-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3296-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3297-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3298-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3299-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3300-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3301-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3302-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3303-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3304-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3305-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3306-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3307-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3308-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3309-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3310-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3311-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3312-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3313-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3314-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3315-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3316-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3317-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3318-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3319-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3320-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3321-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3322-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3323-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3324-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3325-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3326-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3327-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3328-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3329-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3330-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3331-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3332-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3333-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3334-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3335-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3336-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3337-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3338-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3339-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3340-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3341-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3342-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3343-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3344-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3345-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3346-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3347-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3348-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3349-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3350-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3351-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3352-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3353-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3354-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3355-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3356-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3357-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3358-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3359-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3360-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3361-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3362-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3363-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3364-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3365-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3366-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3367-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3368-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3369-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3370-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3371-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3372-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3373-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3374-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3375-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3376-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3377-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3378-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3379-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3380-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3381-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3382-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3383-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3384-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3385-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3386-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3387-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3388-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3389-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3390-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3391-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3392-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3393-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3394-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3395-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3396-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3397-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3398-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3399-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3400-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3401-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3402-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3403-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3404-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3405-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3406-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3407-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3408-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3409-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3410-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3411-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3412-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3413-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3414-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3415-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3416-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3417-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3418-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3419-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3420-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3421-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3422-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3423-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3424-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3425-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3426-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3427-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3428-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3429-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3430-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3431-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3432-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3433-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3434-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3435-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3436-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3437-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3438-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3439-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3440-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3441-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3442-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3443-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3444-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3445-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3446-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3447-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3448-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3449-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3450-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3451-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3452-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3453-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3454-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3455-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3456-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3457-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3458-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3459-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3460-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3461-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3462-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3463-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3464-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3465-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3466-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3467-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3468-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3469-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3470-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3471-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3472-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3473-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3474-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3475-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3476-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3477-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3478-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3479-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3480-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3481-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3482-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3483-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3484-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3485-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3486-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3487-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3488-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3489-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3490-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3491-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3492-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3493-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3494-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3495-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3496-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3497-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3498-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3499-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3500-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3501-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3502-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3503-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3504-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3505-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3506-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3507-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3508-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3509-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3510-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3511-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3512-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3513-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3514-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3515-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3516-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3517-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3518-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3519-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3520-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3521-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3522-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3523-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3524-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3525-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3526-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3527-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3528-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3529-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3530-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3531-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3532-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3533-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3534-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3535-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3536-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3537-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3538-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3539-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3540-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3541-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3542-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3543-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3544-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3545-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3546-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3547-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3548-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3549-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3550-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3551-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3552-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3553-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3554-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3555-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3556-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3557-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3558-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3559-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3560-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3561-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3562-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3563-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3564-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3565-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3566-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3567-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3568-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3569-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3570-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3571-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3572-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3573-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3574-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3575-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3576-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3577-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3578-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3579-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3580-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3581-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3582-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3583-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3584-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3585-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3586-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3587-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3588-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3589-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3590-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3591-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3592-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3593-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3594-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3595-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3596-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3597-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3598-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3599-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3600-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3601-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3602-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3603-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3604-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3605-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3606-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3607-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3608-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3609-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3610-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3611-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3612-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3613-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3614-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3615-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3616-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3617-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3618-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3619-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3620-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3621-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3622-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3623-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3624-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3625-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3626-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3627-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3628-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3629-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3630-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3631-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3632-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3633-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3634-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3635-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3636-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3637-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3638-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3639-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3640-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3641-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3642-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3643-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3644-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3645-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3646-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3647-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3648-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3649-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3650-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3651-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3652-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3653-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3654-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3655-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3656-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3657-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3658-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3659-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3660-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3661-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3662-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3663-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3664-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3665-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3666-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3667-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3668-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3669-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3670-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3671-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3672-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3673-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3674-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3675-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3676-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3677-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3678-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3679-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3680-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3681-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3682-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3683-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3684-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3685-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3686-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3687-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3688-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3689-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3690-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3691-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3692-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3693-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3694-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3695-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3696-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3697-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3698-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3699-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3700-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3701-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3702-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3703-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3704-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3705-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3706-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3707-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3708-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3709-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3710-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3711-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3712-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3713-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3714-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3715-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3716-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3717-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3718-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3719-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3720-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3721-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3722-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3723-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3724-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3725-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3726-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3727-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3728-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3729-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3730-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3731-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3732-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3733-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3734-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3735-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3736-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3737-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3738-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3739-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3740-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3741-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3742-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3743-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3744-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3745-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3746-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3747-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3748-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3749-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3750-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3751-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3752-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3753-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3754-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3755-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3756-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3757-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3758-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3759-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3760-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3761-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3762-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3763-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3764-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3765-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3766-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3767-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3768-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3769-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3770-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3771-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3772-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3773-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3774-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3775-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3776-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3777-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3778-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3779-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3780-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3781-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3782-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3783-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3784-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3785-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3786-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3787-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3788-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3789-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3790-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3791-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3792-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3793-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3794-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3795-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3796-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3797-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3798-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3799-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3800-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3801-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3802-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3803-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3804-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3805-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3806-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3807-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3808-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3809-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3810-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3811-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3812-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3813-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3814-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3815-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3816-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3817-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3818-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3819-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3820-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3821-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3822-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3823-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3824-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3825-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3826-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3827-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3828-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3829-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3830-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3831-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3832-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3833-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3834-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3835-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3836-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3837-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3838-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3839-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3840-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3841-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3842-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3843-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3844-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3845-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3846-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3847-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3848-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3849-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3850-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3851-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3852-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3853-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3854-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3855-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3856-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3857-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3858-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3859-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3860-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3861-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3862-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3863-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3864-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3865-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3866-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3867-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3868-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3869-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3870-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3871-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3872-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3873-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3874-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3875-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3876-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3877-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3878-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3879-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3880-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3881-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3882-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3883-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3884-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3885-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3886-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3887-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3888-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3889-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3890-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3891-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3892-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3893-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3894-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3895-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3896-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3897-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3898-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3899-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3900-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3901-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3902-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3903-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3904-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3905-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3906-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3907-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3908-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3909-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3910-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3911-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3912-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3913-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3914-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3915-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3916-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3917-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3918-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3919-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3920-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3921-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3922-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3923-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3924-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3925-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3926-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3927-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3928-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3929-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3930-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3931-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3932-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3933-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3934-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3935-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3936-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3937-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3938-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3939-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3940-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3941-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3942-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3943-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3944-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3945-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3946-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3947-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3948-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3949-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3950-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3951-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3952-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3953-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3954-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3955-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3956-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3957-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3958-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3959-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3960-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3961-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3962-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3963-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3964-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3965-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3966-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3967-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3968-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3969-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3970-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3971-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3972-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3973-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3974-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3975-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3976-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3977-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3978-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3979-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3980-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3981-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3982-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3983-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3984-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3985-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3986-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3987-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3988-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3989-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3990-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3991-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3992-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3993-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3994-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3995-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3996-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3997-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3998-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3999-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4000-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4001-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4002-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4003-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4004-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4005-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4006-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4007-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4008-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4009-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4010-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4011-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4012-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4013-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4014-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4015-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4016-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4017-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4018-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4019-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4020-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4021-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4022-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4023-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4024-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4025-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4026-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4027-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4028-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4029-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4030-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4031-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4032-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4033-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4034-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4035-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4036-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4037-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4038-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4039-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4040-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4041-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4042-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4043-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4044-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4045-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4046-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4047-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4048-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4049-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4050-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4051-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4052-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4053-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4054-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4055-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4056-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4057-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4058-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4059-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4060-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4061-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4062-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4063-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4064-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4065-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4066-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4067-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4068-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4069-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4070-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4071-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4072-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4073-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4074-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4075-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4076-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4077-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4078-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4079-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4080-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4081-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4082-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4083-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4084-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4085-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4086-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4087-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4088-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4089-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4090-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4091-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4092-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4093-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4094-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4095-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH ,<L\l| + +  ,<L\l| -.=>MN]^mn}~ +  -.=>MN]^mn}~/?O_o + /?O_o@ @AAB,BCMDN]E^mFn}G~HIJKLMNO PQ-R.=S>MTN]U^mVn}W~XYZ[\]^_@AB/C?DOE_FoGHIJKLMNOPQR/S?TOU_VoWXYZ[\]^_ !!","#<#$L$%\%&l&'|'(())**++,,--..//0 0112,23<34L45\56l67|78899::;;<<==>>?? !-".=#>M$N]%^m&n}'~()*+,-./ 01-2.=3>M4N]5^m6n}7~89:;<=>? !"/#?$O%_&o'()*+,-./012/3?4O5_6o789:;<=>?` `aab,bcMdN]e^mfn}g~hijklmno pq-r.=s>MtN]u^mvn}w~xyz{|}~`ab/c?dOe_foghijklmnopqr/s?tOu_vowxyz{|}~HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHg g ` HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH$$Dd$Dd$Ddd$$Dd$Dd$Ddd$$Dd $ D d  +$ +D +d +d + + + + + $ $ D d  $ D d  $ D d d $$Dd$Ddvalue-3072-deterministic-fixturevalue-3074-deterministic-fixturevalue-3075-deterministic-fixturevalue-3076-deterministic-fixturevalue-3077-deterministic-fixturevalue-3078-deterministic-fixturevalue-3079-deterministic-fixturevalue-3081-deterministic-fixturevalue-3082-deterministic-fixturevalue-3083-deterministic-fixturevalue-3084-deterministic-fixturevalue-3085-deterministic-fixturevalue-3086-deterministic-fixturevalue-3088-deterministic-fixturevalue-3089-deterministic-fixturevalue-3090-deterministic-fixturevalue-3091-deterministic-fixturevalue-3092-deterministic-fixturevalue-3093-deterministic-fixturevalue-3095-deterministic-fixturevalue-3096-deterministic-fixturevalue-3097-deterministic-fixturevalue-3098-deterministic-fixturevalue-3099-deterministic-fixturevalue-3100-deterministic-fixturevalue-3102-deterministic-fixturevalue-3103-deterministic-fixturevalue-3104-deterministic-fixturevalue-3105-deterministic-fixturevalue-3106-deterministic-fixturevalue-3107-deterministic-fixturevalue-3109-deterministic-fixturevalue-3110-deterministic-fixturevalue-3111-deterministic-fixturevalue-3112-deterministic-fixturevalue-3113-deterministic-fixturevalue-3114-deterministic-fixturevalue-3116-deterministic-fixturevalue-3117-deterministic-fixturevalue-3118-deterministic-fixturevalue-3119-deterministic-fixturevalue-3120-deterministic-fixturevalue-3121-deterministic-fixturevalue-3123-deterministic-fixturevalue-3124-deterministic-fixturevalue-3125-deterministic-fixturevalue-3126-deterministic-fixturevalue-3127-deterministic-fixturevalue-3128-deterministic-fixturevalue-3130-deterministic-fixturevalue-3131-deterministic-fixturevalue-3132-deterministic-fixturevalue-3133-deterministic-fixturevalue-3134-deterministic-fixturevalue-3135-deterministic-fixturevalue-3137-deterministic-fixturevalue-3138-deterministic-fixturevalue-3139-deterministic-fixturevalue-3140-deterministic-fixturevalue-3141-deterministic-fixturevalue-3142-deterministic-fixturevalue-3144-deterministic-fixturevalue-3145-deterministic-fixturevalue-3146-deterministic-fixturevalue-3147-deterministic-fixturevalue-3148-deterministic-fixturevalue-3149-deterministic-fixturevalue-3151-deterministic-fixturevalue-3152-deterministic-fixturevalue-3153-deterministic-fixturevalue-3154-deterministic-fixturevalue-3155-deterministic-fixturevalue-3156-deterministic-fixturevalue-3158-deterministic-fixturevalue-3159-deterministic-fixturevalue-3160-deterministic-fixturevalue-3161-deterministic-fixturevalue-3162-deterministic-fixturevalue-3163-deterministic-fixturevalue-3165-deterministic-fixturevalue-3166-deterministic-fixturevalue-3167-deterministic-fixturevalue-3168-deterministic-fixturevalue-3169-deterministic-fixturevalue-3170-deterministic-fixturevalue-3172-deterministic-fixturevalue-3173-deterministic-fixturevalue-3174-deterministic-fixturevalue-3175-deterministic-fixturevalue-3176-deterministic-fixturevalue-3177-deterministic-fixturevalue-3179-deterministic-fixturevalue-3180-deterministic-fixturevalue-3181-deterministic-fixturevalue-3182-deterministic-fixturevalue-3183-deterministic-fixturevalue-3184-deterministic-fixturevalue-3186-deterministic-fixturevalue-3187-deterministic-fixturevalue-3188-deterministic-fixturevalue-3189-deterministic-fixturevalue-3190-deterministic-fixturevalue-3191-deterministic-fixturevalue-3193-deterministic-fixturevalue-3194-deterministic-fixturevalue-3195-deterministic-fixturevalue-3196-deterministic-fixturevalue-3197-deterministic-fixturevalue-3198-deterministic-fixture$Dd$Dd$DDd$Dd$Dd$DDd$Dd $ D d  +$ +D +D +d + + + + +  $ D d  $ D d  $ D D d $Dd$Ddvalue-3200-deterministic-fixturevalue-3201-deterministic-fixturevalue-3202-deterministic-fixturevalue-3203-deterministic-fixturevalue-3204-deterministic-fixturevalue-3205-deterministic-fixturevalue-3207-deterministic-fixturevalue-3208-deterministic-fixturevalue-3209-deterministic-fixturevalue-3210-deterministic-fixturevalue-3211-deterministic-fixturevalue-3212-deterministic-fixturevalue-3214-deterministic-fixturevalue-3215-deterministic-fixturevalue-3216-deterministic-fixturevalue-3217-deterministic-fixturevalue-3218-deterministic-fixturevalue-3219-deterministic-fixturevalue-3221-deterministic-fixturevalue-3222-deterministic-fixturevalue-3223-deterministic-fixturevalue-3224-deterministic-fixturevalue-3225-deterministic-fixturevalue-3226-deterministic-fixturevalue-3228-deterministic-fixturevalue-3229-deterministic-fixturevalue-3230-deterministic-fixturevalue-3231-deterministic-fixturevalue-3232-deterministic-fixturevalue-3233-deterministic-fixturevalue-3235-deterministic-fixturevalue-3236-deterministic-fixturevalue-3237-deterministic-fixturevalue-3238-deterministic-fixturevalue-3239-deterministic-fixturevalue-3240-deterministic-fixturevalue-3242-deterministic-fixturevalue-3243-deterministic-fixturevalue-3244-deterministic-fixturevalue-3245-deterministic-fixturevalue-3246-deterministic-fixturevalue-3247-deterministic-fixturevalue-3249-deterministic-fixturevalue-3250-deterministic-fixturevalue-3251-deterministic-fixturevalue-3252-deterministic-fixturevalue-3253-deterministic-fixturevalue-3254-deterministic-fixturevalue-3256-deterministic-fixturevalue-3257-deterministic-fixturevalue-3258-deterministic-fixturevalue-3259-deterministic-fixturevalue-3260-deterministic-fixturevalue-3261-deterministic-fixturevalue-3263-deterministic-fixturevalue-3264-deterministic-fixturevalue-3265-deterministic-fixturevalue-3266-deterministic-fixturevalue-3267-deterministic-fixturevalue-3268-deterministic-fixturevalue-3270-deterministic-fixturevalue-3271-deterministic-fixturevalue-3272-deterministic-fixturevalue-3273-deterministic-fixturevalue-3274-deterministic-fixturevalue-3275-deterministic-fixturevalue-3277-deterministic-fixturevalue-3278-deterministic-fixturevalue-3279-deterministic-fixturevalue-3280-deterministic-fixturevalue-3281-deterministic-fixturevalue-3282-deterministic-fixturevalue-3284-deterministic-fixturevalue-3285-deterministic-fixturevalue-3286-deterministic-fixturevalue-3287-deterministic-fixturevalue-3288-deterministic-fixturevalue-3289-deterministic-fixturevalue-3291-deterministic-fixturevalue-3292-deterministic-fixturevalue-3293-deterministic-fixturevalue-3294-deterministic-fixturevalue-3295-deterministic-fixturevalue-3296-deterministic-fixturevalue-3298-deterministic-fixturevalue-3299-deterministic-fixturevalue-3300-deterministic-fixturevalue-3301-deterministic-fixturevalue-3302-deterministic-fixturevalue-3303-deterministic-fixturevalue-3305-deterministic-fixturevalue-3306-deterministic-fixturevalue-3307-deterministic-fixturevalue-3308-deterministic-fixturevalue-3309-deterministic-fixturevalue-3310-deterministic-fixturevalue-3312-deterministic-fixturevalue-3313-deterministic-fixturevalue-3314-deterministic-fixturevalue-3315-deterministic-fixturevalue-3316-deterministic-fixturevalue-3317-deterministic-fixturevalue-3319-deterministic-fixturevalue-3320-deterministic-fixturevalue-3321-deterministic-fixturevalue-3322-deterministic-fixturevalue-3323-deterministic-fixturevalue-3324-deterministic-fixturevalue-3326-deterministic-fixturevalue-3327-deterministic-fixture$Dd$DDd$Dd$Dd$DDd$Dd$Dd $ D D d  + +$ +D +d + + + + + + $ D d  $ D D d   $ D d $Dd$DDdvalue-3328-deterministic-fixturevalue-3329-deterministic-fixturevalue-3330-deterministic-fixturevalue-3331-deterministic-fixturevalue-3333-deterministic-fixturevalue-3334-deterministic-fixturevalue-3335-deterministic-fixturevalue-3336-deterministic-fixturevalue-3337-deterministic-fixturevalue-3338-deterministic-fixturevalue-3340-deterministic-fixturevalue-3341-deterministic-fixturevalue-3342-deterministic-fixturevalue-3343-deterministic-fixturevalue-3344-deterministic-fixturevalue-3345-deterministic-fixturevalue-3347-deterministic-fixturevalue-3348-deterministic-fixturevalue-3349-deterministic-fixturevalue-3350-deterministic-fixturevalue-3351-deterministic-fixturevalue-3352-deterministic-fixturevalue-3354-deterministic-fixturevalue-3355-deterministic-fixturevalue-3356-deterministic-fixturevalue-3357-deterministic-fixturevalue-3358-deterministic-fixturevalue-3359-deterministic-fixturevalue-3361-deterministic-fixturevalue-3362-deterministic-fixturevalue-3363-deterministic-fixturevalue-3364-deterministic-fixturevalue-3365-deterministic-fixturevalue-3366-deterministic-fixturevalue-3368-deterministic-fixturevalue-3369-deterministic-fixturevalue-3370-deterministic-fixturevalue-3371-deterministic-fixturevalue-3372-deterministic-fixturevalue-3373-deterministic-fixturevalue-3375-deterministic-fixturevalue-3376-deterministic-fixturevalue-3377-deterministic-fixturevalue-3378-deterministic-fixturevalue-3379-deterministic-fixturevalue-3380-deterministic-fixturevalue-3382-deterministic-fixturevalue-3383-deterministic-fixturevalue-3384-deterministic-fixturevalue-3385-deterministic-fixturevalue-3386-deterministic-fixturevalue-3387-deterministic-fixturevalue-3389-deterministic-fixturevalue-3390-deterministic-fixturevalue-3391-deterministic-fixturevalue-3392-deterministic-fixturevalue-3393-deterministic-fixturevalue-3394-deterministic-fixturevalue-3396-deterministic-fixturevalue-3397-deterministic-fixturevalue-3398-deterministic-fixturevalue-3399-deterministic-fixturevalue-3400-deterministic-fixturevalue-3401-deterministic-fixturevalue-3403-deterministic-fixturevalue-3404-deterministic-fixturevalue-3405-deterministic-fixturevalue-3406-deterministic-fixturevalue-3407-deterministic-fixturevalue-3408-deterministic-fixturevalue-3410-deterministic-fixturevalue-3411-deterministic-fixturevalue-3412-deterministic-fixturevalue-3413-deterministic-fixturevalue-3414-deterministic-fixturevalue-3415-deterministic-fixturevalue-3417-deterministic-fixturevalue-3418-deterministic-fixturevalue-3419-deterministic-fixturevalue-3420-deterministic-fixturevalue-3421-deterministic-fixturevalue-3422-deterministic-fixturevalue-3424-deterministic-fixturevalue-3425-deterministic-fixturevalue-3426-deterministic-fixturevalue-3427-deterministic-fixturevalue-3428-deterministic-fixturevalue-3429-deterministic-fixturevalue-3431-deterministic-fixturevalue-3432-deterministic-fixturevalue-3433-deterministic-fixturevalue-3434-deterministic-fixturevalue-3435-deterministic-fixturevalue-3436-deterministic-fixturevalue-3438-deterministic-fixturevalue-3439-deterministic-fixturevalue-3440-deterministic-fixturevalue-3441-deterministic-fixturevalue-3442-deterministic-fixturevalue-3443-deterministic-fixturevalue-3445-deterministic-fixturevalue-3446-deterministic-fixturevalue-3447-deterministic-fixturevalue-3448-deterministic-fixturevalue-3449-deterministic-fixturevalue-3450-deterministic-fixturevalue-3452-deterministic-fixturevalue-3453-deterministic-fixturevalue-3454-deterministic-fixturevalue-3455-deterministic-fixture$DDd$Dd$Dd$DDd$Dd$Dd$DDd  $ D d  +$ +D +d + + + + + + $ D D d   $ D d  $ D d $DDd$Ddvalue-3456-deterministic-fixturevalue-3457-deterministic-fixturevalue-3459-deterministic-fixturevalue-3460-deterministic-fixturevalue-3461-deterministic-fixturevalue-3462-deterministic-fixturevalue-3463-deterministic-fixturevalue-3464-deterministic-fixturevalue-3466-deterministic-fixturevalue-3467-deterministic-fixturevalue-3468-deterministic-fixturevalue-3469-deterministic-fixturevalue-3470-deterministic-fixturevalue-3471-deterministic-fixturevalue-3473-deterministic-fixturevalue-3474-deterministic-fixturevalue-3475-deterministic-fixturevalue-3476-deterministic-fixturevalue-3477-deterministic-fixturevalue-3478-deterministic-fixturevalue-3480-deterministic-fixturevalue-3481-deterministic-fixturevalue-3482-deterministic-fixturevalue-3483-deterministic-fixturevalue-3484-deterministic-fixturevalue-3485-deterministic-fixturevalue-3487-deterministic-fixturevalue-3488-deterministic-fixturevalue-3489-deterministic-fixturevalue-3490-deterministic-fixturevalue-3491-deterministic-fixturevalue-3492-deterministic-fixturevalue-3494-deterministic-fixturevalue-3495-deterministic-fixturevalue-3496-deterministic-fixturevalue-3497-deterministic-fixturevalue-3498-deterministic-fixturevalue-3499-deterministic-fixturevalue-3501-deterministic-fixturevalue-3502-deterministic-fixturevalue-3503-deterministic-fixturevalue-3504-deterministic-fixturevalue-3505-deterministic-fixturevalue-3506-deterministic-fixturevalue-3508-deterministic-fixturevalue-3509-deterministic-fixturevalue-3510-deterministic-fixturevalue-3511-deterministic-fixturevalue-3512-deterministic-fixturevalue-3513-deterministic-fixturevalue-3515-deterministic-fixturevalue-3516-deterministic-fixturevalue-3517-deterministic-fixturevalue-3518-deterministic-fixturevalue-3519-deterministic-fixturevalue-3520-deterministic-fixturevalue-3522-deterministic-fixturevalue-3523-deterministic-fixturevalue-3524-deterministic-fixturevalue-3525-deterministic-fixturevalue-3526-deterministic-fixturevalue-3527-deterministic-fixturevalue-3529-deterministic-fixturevalue-3530-deterministic-fixturevalue-3531-deterministic-fixturevalue-3532-deterministic-fixturevalue-3533-deterministic-fixturevalue-3534-deterministic-fixturevalue-3536-deterministic-fixturevalue-3537-deterministic-fixturevalue-3538-deterministic-fixturevalue-3539-deterministic-fixturevalue-3540-deterministic-fixturevalue-3541-deterministic-fixturevalue-3543-deterministic-fixturevalue-3544-deterministic-fixturevalue-3545-deterministic-fixturevalue-3546-deterministic-fixturevalue-3547-deterministic-fixturevalue-3548-deterministic-fixturevalue-3550-deterministic-fixturevalue-3551-deterministic-fixturevalue-3552-deterministic-fixturevalue-3553-deterministic-fixturevalue-3554-deterministic-fixturevalue-3555-deterministic-fixturevalue-3557-deterministic-fixturevalue-3558-deterministic-fixturevalue-3559-deterministic-fixturevalue-3560-deterministic-fixturevalue-3561-deterministic-fixturevalue-3562-deterministic-fixturevalue-3564-deterministic-fixturevalue-3565-deterministic-fixturevalue-3566-deterministic-fixturevalue-3567-deterministic-fixturevalue-3568-deterministic-fixturevalue-3569-deterministic-fixturevalue-3571-deterministic-fixturevalue-3572-deterministic-fixturevalue-3573-deterministic-fixturevalue-3574-deterministic-fixturevalue-3575-deterministic-fixturevalue-3576-deterministic-fixturevalue-3578-deterministic-fixturevalue-3579-deterministic-fixturevalue-3580-deterministic-fixturevalue-3581-deterministic-fixturevalue-3582-deterministic-fixturevalue-3583-deterministic-fixture$Dd$Dd$DDd$Dd$Dd$DDd$Dd $ D d  +$ +D +D +d + + + + +  $ D d  $ D d  $ D D d $Dd$Ddvalue-3585-deterministic-fixturevalue-3586-deterministic-fixturevalue-3587-deterministic-fixturevalue-3588-deterministic-fixturevalue-3589-deterministic-fixturevalue-3590-deterministic-fixturevalue-3592-deterministic-fixturevalue-3593-deterministic-fixturevalue-3594-deterministic-fixturevalue-3595-deterministic-fixturevalue-3596-deterministic-fixturevalue-3597-deterministic-fixturevalue-3599-deterministic-fixturevalue-3600-deterministic-fixturevalue-3601-deterministic-fixturevalue-3602-deterministic-fixturevalue-3603-deterministic-fixturevalue-3604-deterministic-fixturevalue-3606-deterministic-fixturevalue-3607-deterministic-fixturevalue-3608-deterministic-fixturevalue-3609-deterministic-fixturevalue-3610-deterministic-fixturevalue-3611-deterministic-fixturevalue-3613-deterministic-fixturevalue-3614-deterministic-fixturevalue-3615-deterministic-fixturevalue-3616-deterministic-fixturevalue-3617-deterministic-fixturevalue-3618-deterministic-fixturevalue-3620-deterministic-fixturevalue-3621-deterministic-fixturevalue-3622-deterministic-fixturevalue-3623-deterministic-fixturevalue-3624-deterministic-fixturevalue-3625-deterministic-fixturevalue-3627-deterministic-fixturevalue-3628-deterministic-fixturevalue-3629-deterministic-fixturevalue-3630-deterministic-fixturevalue-3631-deterministic-fixturevalue-3632-deterministic-fixturevalue-3634-deterministic-fixturevalue-3635-deterministic-fixturevalue-3636-deterministic-fixturevalue-3637-deterministic-fixturevalue-3638-deterministic-fixturevalue-3639-deterministic-fixturevalue-3641-deterministic-fixturevalue-3642-deterministic-fixturevalue-3643-deterministic-fixturevalue-3644-deterministic-fixturevalue-3645-deterministic-fixturevalue-3646-deterministic-fixturevalue-3648-deterministic-fixturevalue-3649-deterministic-fixturevalue-3650-deterministic-fixturevalue-3651-deterministic-fixturevalue-3652-deterministic-fixturevalue-3653-deterministic-fixturevalue-3655-deterministic-fixturevalue-3656-deterministic-fixturevalue-3657-deterministic-fixturevalue-3658-deterministic-fixturevalue-3659-deterministic-fixturevalue-3660-deterministic-fixturevalue-3662-deterministic-fixturevalue-3663-deterministic-fixturevalue-3664-deterministic-fixturevalue-3665-deterministic-fixturevalue-3666-deterministic-fixturevalue-3667-deterministic-fixturevalue-3669-deterministic-fixturevalue-3670-deterministic-fixturevalue-3671-deterministic-fixturevalue-3672-deterministic-fixturevalue-3673-deterministic-fixturevalue-3674-deterministic-fixturevalue-3676-deterministic-fixturevalue-3677-deterministic-fixturevalue-3678-deterministic-fixturevalue-3679-deterministic-fixturevalue-3680-deterministic-fixturevalue-3681-deterministic-fixturevalue-3683-deterministic-fixturevalue-3684-deterministic-fixturevalue-3685-deterministic-fixturevalue-3686-deterministic-fixturevalue-3687-deterministic-fixturevalue-3688-deterministic-fixturevalue-3690-deterministic-fixturevalue-3691-deterministic-fixturevalue-3692-deterministic-fixturevalue-3693-deterministic-fixturevalue-3694-deterministic-fixturevalue-3695-deterministic-fixturevalue-3697-deterministic-fixturevalue-3698-deterministic-fixturevalue-3699-deterministic-fixturevalue-3700-deterministic-fixturevalue-3701-deterministic-fixturevalue-3702-deterministic-fixturevalue-3704-deterministic-fixturevalue-3705-deterministic-fixturevalue-3706-deterministic-fixturevalue-3707-deterministic-fixturevalue-3708-deterministic-fixturevalue-3709-deterministic-fixturevalue-3711-deterministic-fixture$Dd$Ddd$$Dd$Dd$Ddd$$Dd$Dd $ D d d  +$ +$ +D +d + + + + + + $ D d  $ D d d  $ $ D d $Dd$Dddvalue-3712-deterministic-fixturevalue-3713-deterministic-fixturevalue-3714-deterministic-fixturevalue-3715-deterministic-fixturevalue-3716-deterministic-fixturevalue-3718-deterministic-fixturevalue-3719-deterministic-fixturevalue-3720-deterministic-fixturevalue-3721-deterministic-fixturevalue-3722-deterministic-fixturevalue-3723-deterministic-fixturevalue-3725-deterministic-fixturevalue-3726-deterministic-fixturevalue-3727-deterministic-fixturevalue-3728-deterministic-fixturevalue-3729-deterministic-fixturevalue-3730-deterministic-fixturevalue-3732-deterministic-fixturevalue-3733-deterministic-fixturevalue-3734-deterministic-fixturevalue-3735-deterministic-fixturevalue-3736-deterministic-fixturevalue-3737-deterministic-fixturevalue-3739-deterministic-fixturevalue-3740-deterministic-fixturevalue-3741-deterministic-fixturevalue-3742-deterministic-fixturevalue-3743-deterministic-fixturevalue-3744-deterministic-fixturevalue-3746-deterministic-fixturevalue-3747-deterministic-fixturevalue-3748-deterministic-fixturevalue-3749-deterministic-fixturevalue-3750-deterministic-fixturevalue-3751-deterministic-fixturevalue-3753-deterministic-fixturevalue-3754-deterministic-fixturevalue-3755-deterministic-fixturevalue-3756-deterministic-fixturevalue-3757-deterministic-fixturevalue-3758-deterministic-fixturevalue-3760-deterministic-fixturevalue-3761-deterministic-fixturevalue-3762-deterministic-fixturevalue-3763-deterministic-fixturevalue-3764-deterministic-fixturevalue-3765-deterministic-fixturevalue-3767-deterministic-fixturevalue-3768-deterministic-fixturevalue-3769-deterministic-fixturevalue-3770-deterministic-fixturevalue-3771-deterministic-fixturevalue-3772-deterministic-fixturevalue-3774-deterministic-fixturevalue-3775-deterministic-fixturevalue-3776-deterministic-fixturevalue-3777-deterministic-fixturevalue-3778-deterministic-fixturevalue-3779-deterministic-fixturevalue-3781-deterministic-fixturevalue-3782-deterministic-fixturevalue-3783-deterministic-fixturevalue-3784-deterministic-fixturevalue-3785-deterministic-fixturevalue-3786-deterministic-fixturevalue-3788-deterministic-fixturevalue-3789-deterministic-fixturevalue-3790-deterministic-fixturevalue-3791-deterministic-fixturevalue-3792-deterministic-fixturevalue-3793-deterministic-fixturevalue-3795-deterministic-fixturevalue-3796-deterministic-fixturevalue-3797-deterministic-fixturevalue-3798-deterministic-fixturevalue-3799-deterministic-fixturevalue-3800-deterministic-fixturevalue-3802-deterministic-fixturevalue-3803-deterministic-fixturevalue-3804-deterministic-fixturevalue-3805-deterministic-fixturevalue-3806-deterministic-fixturevalue-3807-deterministic-fixturevalue-3809-deterministic-fixturevalue-3810-deterministic-fixturevalue-3811-deterministic-fixturevalue-3812-deterministic-fixturevalue-3813-deterministic-fixturevalue-3814-deterministic-fixturevalue-3816-deterministic-fixturevalue-3817-deterministic-fixturevalue-3818-deterministic-fixturevalue-3819-deterministic-fixturevalue-3820-deterministic-fixturevalue-3821-deterministic-fixturevalue-3823-deterministic-fixturevalue-3824-deterministic-fixturevalue-3825-deterministic-fixturevalue-3826-deterministic-fixturevalue-3827-deterministic-fixturevalue-3828-deterministic-fixturevalue-3830-deterministic-fixturevalue-3831-deterministic-fixturevalue-3832-deterministic-fixturevalue-3833-deterministic-fixturevalue-3834-deterministic-fixturevalue-3835-deterministic-fixturevalue-3837-deterministic-fixturevalue-3838-deterministic-fixturevalue-3839-deterministic-fixture$Ddd$$Dd$Dd$Ddd$$Dd$Dd$Ddd $ $ D d  +$ +D +d + + + + + + $ D d d  $ $ D d  $ D d $Ddd$$Ddvalue-3840-deterministic-fixturevalue-3841-deterministic-fixturevalue-3842-deterministic-fixturevalue-3844-deterministic-fixturevalue-3845-deterministic-fixturevalue-3846-deterministic-fixturevalue-3847-deterministic-fixturevalue-3848-deterministic-fixturevalue-3849-deterministic-fixturevalue-3851-deterministic-fixturevalue-3852-deterministic-fixturevalue-3853-deterministic-fixturevalue-3854-deterministic-fixturevalue-3855-deterministic-fixturevalue-3856-deterministic-fixturevalue-3858-deterministic-fixturevalue-3859-deterministic-fixturevalue-3860-deterministic-fixturevalue-3861-deterministic-fixturevalue-3862-deterministic-fixturevalue-3863-deterministic-fixturevalue-3865-deterministic-fixturevalue-3866-deterministic-fixturevalue-3867-deterministic-fixturevalue-3868-deterministic-fixturevalue-3869-deterministic-fixturevalue-3870-deterministic-fixturevalue-3872-deterministic-fixturevalue-3873-deterministic-fixturevalue-3874-deterministic-fixturevalue-3875-deterministic-fixturevalue-3876-deterministic-fixturevalue-3877-deterministic-fixturevalue-3879-deterministic-fixturevalue-3880-deterministic-fixturevalue-3881-deterministic-fixturevalue-3882-deterministic-fixturevalue-3883-deterministic-fixturevalue-3884-deterministic-fixturevalue-3886-deterministic-fixturevalue-3887-deterministic-fixturevalue-3888-deterministic-fixturevalue-3889-deterministic-fixturevalue-3890-deterministic-fixturevalue-3891-deterministic-fixturevalue-3893-deterministic-fixturevalue-3894-deterministic-fixturevalue-3895-deterministic-fixturevalue-3896-deterministic-fixturevalue-3897-deterministic-fixturevalue-3898-deterministic-fixturevalue-3900-deterministic-fixturevalue-3901-deterministic-fixturevalue-3902-deterministic-fixturevalue-3903-deterministic-fixturevalue-3904-deterministic-fixturevalue-3905-deterministic-fixturevalue-3907-deterministic-fixturevalue-3908-deterministic-fixturevalue-3909-deterministic-fixturevalue-3910-deterministic-fixturevalue-3911-deterministic-fixturevalue-3912-deterministic-fixturevalue-3914-deterministic-fixturevalue-3915-deterministic-fixturevalue-3916-deterministic-fixturevalue-3917-deterministic-fixturevalue-3918-deterministic-fixturevalue-3919-deterministic-fixturevalue-3921-deterministic-fixturevalue-3922-deterministic-fixturevalue-3923-deterministic-fixturevalue-3924-deterministic-fixturevalue-3925-deterministic-fixturevalue-3926-deterministic-fixturevalue-3928-deterministic-fixturevalue-3929-deterministic-fixturevalue-3930-deterministic-fixturevalue-3931-deterministic-fixturevalue-3932-deterministic-fixturevalue-3933-deterministic-fixturevalue-3935-deterministic-fixturevalue-3936-deterministic-fixturevalue-3937-deterministic-fixturevalue-3938-deterministic-fixturevalue-3939-deterministic-fixturevalue-3940-deterministic-fixturevalue-3942-deterministic-fixturevalue-3943-deterministic-fixturevalue-3944-deterministic-fixturevalue-3945-deterministic-fixturevalue-3946-deterministic-fixturevalue-3947-deterministic-fixturevalue-3949-deterministic-fixturevalue-3950-deterministic-fixturevalue-3951-deterministic-fixturevalue-3952-deterministic-fixturevalue-3953-deterministic-fixturevalue-3954-deterministic-fixturevalue-3956-deterministic-fixturevalue-3957-deterministic-fixturevalue-3958-deterministic-fixturevalue-3959-deterministic-fixturevalue-3960-deterministic-fixturevalue-3961-deterministic-fixturevalue-3963-deterministic-fixturevalue-3964-deterministic-fixturevalue-3965-deterministic-fixturevalue-3966-deterministic-fixturevalue-3967-deterministic-fixture$$Dd$Dd$Ddd$$Dd$Dd$Ddd$$Dd $ D d  +$ +D +d +d + + + + + $ $ D d  $ D d  $ D d d $$Dd$Ddvalue-3968-deterministic-fixturevalue-3970-deterministic-fixturevalue-3971-deterministic-fixturevalue-3972-deterministic-fixturevalue-3973-deterministic-fixturevalue-3974-deterministic-fixturevalue-3975-deterministic-fixturevalue-3977-deterministic-fixturevalue-3978-deterministic-fixturevalue-3979-deterministic-fixturevalue-3980-deterministic-fixturevalue-3981-deterministic-fixturevalue-3982-deterministic-fixturevalue-3984-deterministic-fixturevalue-3985-deterministic-fixturevalue-3986-deterministic-fixturevalue-3987-deterministic-fixturevalue-3988-deterministic-fixturevalue-3989-deterministic-fixturevalue-3991-deterministic-fixturevalue-3992-deterministic-fixturevalue-3993-deterministic-fixturevalue-3994-deterministic-fixturevalue-3995-deterministic-fixturevalue-3996-deterministic-fixturevalue-3998-deterministic-fixturevalue-3999-deterministic-fixturevalue-4000-deterministic-fixturevalue-4001-deterministic-fixturevalue-4002-deterministic-fixturevalue-4003-deterministic-fixturevalue-4005-deterministic-fixturevalue-4006-deterministic-fixturevalue-4007-deterministic-fixturevalue-4008-deterministic-fixturevalue-4009-deterministic-fixturevalue-4010-deterministic-fixturevalue-4012-deterministic-fixturevalue-4013-deterministic-fixturevalue-4014-deterministic-fixturevalue-4015-deterministic-fixturevalue-4016-deterministic-fixturevalue-4017-deterministic-fixturevalue-4019-deterministic-fixturevalue-4020-deterministic-fixturevalue-4021-deterministic-fixturevalue-4022-deterministic-fixturevalue-4023-deterministic-fixturevalue-4024-deterministic-fixturevalue-4026-deterministic-fixturevalue-4027-deterministic-fixturevalue-4028-deterministic-fixturevalue-4029-deterministic-fixturevalue-4030-deterministic-fixturevalue-4031-deterministic-fixturevalue-4033-deterministic-fixturevalue-4034-deterministic-fixturevalue-4035-deterministic-fixturevalue-4036-deterministic-fixturevalue-4037-deterministic-fixturevalue-4038-deterministic-fixturevalue-4040-deterministic-fixturevalue-4041-deterministic-fixturevalue-4042-deterministic-fixturevalue-4043-deterministic-fixturevalue-4044-deterministic-fixturevalue-4045-deterministic-fixturevalue-4047-deterministic-fixturevalue-4048-deterministic-fixturevalue-4049-deterministic-fixturevalue-4050-deterministic-fixturevalue-4051-deterministic-fixturevalue-4052-deterministic-fixturevalue-4054-deterministic-fixturevalue-4055-deterministic-fixturevalue-4056-deterministic-fixturevalue-4057-deterministic-fixturevalue-4058-deterministic-fixturevalue-4059-deterministic-fixturevalue-4061-deterministic-fixturevalue-4062-deterministic-fixturevalue-4063-deterministic-fixturevalue-4064-deterministic-fixturevalue-4065-deterministic-fixturevalue-4066-deterministic-fixturevalue-4068-deterministic-fixturevalue-4069-deterministic-fixturevalue-4070-deterministic-fixturevalue-4071-deterministic-fixturevalue-4072-deterministic-fixturevalue-4073-deterministic-fixturevalue-4075-deterministic-fixturevalue-4076-deterministic-fixturevalue-4077-deterministic-fixturevalue-4078-deterministic-fixturevalue-4079-deterministic-fixturevalue-4080-deterministic-fixturevalue-4082-deterministic-fixturevalue-4083-deterministic-fixturevalue-4084-deterministic-fixturevalue-4085-deterministic-fixturevalue-4086-deterministic-fixturevalue-4087-deterministic-fixturevalue-4089-deterministic-fixturevalue-4090-deterministic-fixturevalue-4091-deterministic-fixturevalue-4092-deterministic-fixturevalue-4093-deterministic-fixturevalue-4094-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH::HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH"Ddd2d2D2D2D"D"L"L"L&L&H&H&H$H$I$I$ɒ$$ddd2d2D2D2D"D"L"L"L&L&H&H&H$H$I$I$ɒ$$dd @@ @H@ @@ @!@@@ @@ @!@@ " sL# $ $X $) 3 $&%  c 6L&i +X $@' $V( f L)#$v + * XS * 421QjdŜ924QrdŨz=27Qƴ|A2:Rd~D2=Rd̖I2@RdؖM2CRd䖄Q2FR𖈌U2LSdX2OSda2RSd&uь):tuLI;t&v Y;Cuv̱;su&ьɣA5]$j @E5`j!L̟M5c$j" DXQ5fU"#Vǰ4U*#vǼ5鎙SV2$6 V:6)V%8IVJ&9 WR&9CWZ:sWb';ɏWr((<鏽Wz(4e@2y&eԌ@2|eL—A2&e ×A2&׌ėA2&eLŗB2&e Ɨ@B2eƘB2&e܌ǘB2eLȘC2&e ɘYC 5yk ӀI5cy6ӌi5ykFӘ 5ykVӤ5y"k`Ӱɦ5#z*kvӼ香5SzȠ 5z:kԠ 5zJkI5zRki5{!$ +)$ L,I'$ - XC*-s-$. 0$  0"X3 0 6$!1 (3Sw& Y>w|̺)>w}L ?w&~ iY?x&̽?Cx&Ҍ?sxLɥ@x& Y@xӢj#FT5i$j$HY5l$j%LJ]5o$& a5r$j' Nd5uj(L,Pi5x$j* 8TXm5~j+Dq5$,PXy5j-LhZ|5$j. t\X< X)&@=IcX0L>iX*FX?X+Vd@ X+fpAɐ#Yp|BSY,ȈB Y-ȔC)Y-ȬD Y.ȸEiZ2eɘC2&eʘC2L˘YD2&e̘ęD2&e͘D2eLΘE2&e ϘYE2ϘΙE2&eИE2eLјF2&e 5s{bk 5{jk駹5{rk( 5|zk4)53|&@I5c|k6L 5|kFX5|kPp5|f|ɨ5#}kԈ 5"Dd2D2D2D"D"L"L"L&L&H&H&H$H$I$I$ɒ$$ddd2d2D2D2D"D"L"L"L&L&H&H&H$H$I$I$ɒ$$ddd2d2DH@ @@ @@ @H@ @@ @!@@@ @@ @ie(Vx +@jnԖ{> +jqզ|M@Jkt(} AjZkwAkC}(րBksMC +l( D[lӆ("p308*p4޴| +Ҁ*1' +ܘc17Nj*2GˊҜû*3W̪ܙ3gN#*4w +\S4 +Ӝ**ܚ5NJ*6j\;v;( +v ;vM"[ ;v$$;(0;*vM<*;(2v H,[;(:vT.%;(l);ȨRvMx2-;(Zv DM +posPNʽuo\OypӜhP +}ptQ*p3&߀R +qc +6ߌSjq@ߘSqÝV߰Tq*f߼Uʾr#2V +\8*pKک8*=pL۩8*>pMݩ\8>Nݩ8*qPީܟ8?qQNߩ8*@qR੤\8@S੦8*AqTᩨܠ8*qUN⩪8*Bv*a; Je;v, +h;v8m;CvDq;svPu;\};vt +;w뀲*օ;3 +w0댲j։;G똲D +l(&EJlc6MF + +m(F G mÒ(PHmfMIn#(v J[nS܆J +n(ݖK*n(MLJo(޶ Mj@oA8?~E8p@̨H8pAبM8ÚpBQ8ƚpC𨆼U8ɛEY8ϛpF\8қpGe8՛pH,i8؛I8m8ۜpJ6ѪӜs*7ܛ*N\ӽ8 +Ԁ*9 +ܜ39'NJc*0j\:G׊Ԁþ*;Wتܝ;gN +#*< +Մ4[1;Ψbv͐65;(jv89;( @=;רzvʹ<@;(v@E;(vM̱BI;(v ر@Q;樢vFU;(vHX;쨲vMJ];(r:V*rBWJsYjsRY +sCZZssb[ʿtj\꿽tӟz4\ +t@] +t36L^ju@XqV@8BqW㩮ܡ8CXN婰8+qY橲\8 Dq[8+Eq\窸ܢ8Eq]N誺8+Fq^骼\8+q_骾8+Gq`8 GqaN֍;wW뤲 +;"wg밲֕;#*wp뼲֙;S2wȲ +ם;Բ*ס;Bw +;Rwj׭;Zwױ;C׵;jw +;rw2"" " " & &&&$$ $ $ $ $ $ $ $ 2$ 2 22"" " " & &&&$$ $ $ $ $ $ $ $ 2$ 2 22"!@@ @@ H@ @@ʬUޝs,VǣЬNW@ӬX ,Y+3, NZKc,7[k@ɓ⬒G[,W\ 謓gN]#,^ Ĺ={йV=ܹX={Z={\> +|`> +| b >d>"|0f>*|<>2|Hj>@ / / _ / O / _/ / O / ^-O- ^ !- $ O'-^*-- 0O"g L⚚WgQ`iU#㪜wjYSj ]΃㺝k `㝠lKi8mkmDmqCPn tϣ\oy>C/|>F/|Ï>I|O>L/_>R/|>U/|ȏ>X|O>[/_>^|>a/|̏>d|O           ]ʃ^+ʳ,_ ,ak],aC,bsOc̣-d ^d  -e+3 7Ofk͓-GTl!>"B|`n$>%J|lp)>(R|x->+Z|v1>1x5>4j|z<>7z|A>:|~E>=̺I>@|غ _ O / @ / /O / @ / O 3-$6-&9O<-,^B.E-0HOK-4^N6Q-8T-Ohp }tq+3'rKc +7s ѓGsPtgu#2wvS:v+ҳBwKJxk嗢@>g|>j/Ϗ>m/}O>s/}>v/}>y/>|/}>/}>/}>/}>/   HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHwwHHHHHHHHHHHHHHHH HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @@ @@ @@ @@ @@!aaaaaaaAaa`a!aaaaaaaAaa`a!aaaaaaaAaa`a!aaaaaaaaaAaa`a!aaaaaaaAaa`a!aaaaaaaAaa`a!aaaaHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH  greenblueredHHHHHHHHHHHHHHHHHHHHHHHHHHHH    HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ @    +@ + + + @    @    @   @@@@@@@@@@@@@@@@@@ @   !@!!!"@"""#@###$@$$$%@%%%&@&&&'@'''(@((()@)))*@***+@+++,@,,,-@---.@.../@///0@0001@1112@2223@3334@4445@5556@6667@7778@8889@999:@:::;@;;;<@<<<=@===>@>>>?@???@@@@@A@AAAB@BBBC@CCCD@DDDE@EEEF@FFFG@GGGH@HHHI@IIIJ@JJJK@KKKL@LLLM@MMMN@NNNO@OOOP@PPPQ@QQQR@RRRS@SSST@TTTU@UUUV@VVVW@WWWX@XXXY@YYYZ@ZZZ[@[[[\@\\\]@]]]^@^^^_@___`@```a@aaab@bbbc@cccd@ddde@eeef@fffg@gggh@hhhi@iiij@jjjk@kkkl@lllm@mmmn@nnno@ooop@pppq@qqqr@rrrs@ssst@tttu@uuuv@vvvw@wwwx@xxxy@yyyz@zzz{@{{{|@|||}@}}}~@~~~@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4096-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHPHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH((value-4096-deterministic-fixtureHHHHHHHHHHHHHHHH@HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH  0HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH  greenblueredHHHHHHHHHHHHHHHHHHHHHHHHHHHH HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH + +id *int328 +Cname *string08R" +lance-encoding:compressionnone +items *list08 +item *int3208 +ecategory *dict:string:int8:false08BR. +&lance-encoding:dict-values-compressionnone +Bblob * large_binary08R +lance-encoding:blobtrue +)' +% +/lance.encodings.ColumnEncoding +G + +"53 +1 +/lance.encodings21.PageLayout +* 28HJ +  "53 +1 +/lance.encodings21.PageLayout +* 28H(J + "53 +1 +/lance.encodings21.PageLayout +* 28H(J + "53 +1 +/lance.encodings21.PageLayout +* 28H(G +"42 +0 +/lance.encodings21.PageLayout +  + 28H( +)' +% +/lance.encodings.ColumnEncoding +R +"?= +; +/lance.encodings21.PageLayout +* + + 28HU + "?= +; +/lance.encodings21.PageLayout +* + + 28H(U +"?= +; +/lance.encodings21.PageLayout +* + + 28H(U +"?= +; +/lance.encodings21.PageLayout +* + + 28H(K +0"86 +4 +/lance.encodings21.PageLayout + + + 28H( +)' +% +/lance.encodings.ColumnEncoding +f + .0"PN +L +/lance.encodings21.PageLayout+ +) + +" + +" +* 28@Hi + 20"PN +L +/lance.encodings21.PageLayout+ +) + +" + +" +* 28@H(i + 40"PN +L +/lance.encodings21.PageLayout+ +) + +" + +" +* 28@H(i + 50"PN +L +/lance.encodings21.PageLayout+ +) + +" + +" +* 28@H(Z + ("CA +? +/lance.encodings21.PageLayout + + +* + 28@H( +)' +% +/lance.encodings.ColumnEncoding +] + $"GE +C +/lance.encodings21.PageLayout" + **" + + (28H` + $"GE +C +/lance.encodings21.PageLayout" + **" + + (28H(` + $"GE +C +/lance.encodings21.PageLayout" + **" + + (28H(` + $"GE +C +/lance.encodings21.PageLayout" + **" + + (28H(W + $"@> +< +/lance.encodings21.PageLayout + +" + + (28H( +)' +% +/lance.encodings.ColumnEncoding +X +"EC +A +/lance.encodings21.PageLayout " + + b +@@ +28H[ +"EC +A +/lance.encodings21.PageLayout " + + b +@@ +28H([ +"EC +A +/lance.encodings21.PageLayout " + + b +@@ +28H([ +"EC +A +/lance.encodings21.PageLayout " + + b +@@ +28H(W +"DB +@ +/lance.encodings21.PageLayout" + + b +@@ +28H( (0) 2@G'wLANC \ No newline at end of file diff --git a/rust/lance-file/test_data/exact_versions/v2_2.lance b/rust/lance-file/test_data/exact_versions/v2_2.lance new file mode 100644 index 00000000000..683c06cd2fe --- /dev/null +++ b/rust/lance-file/test_data/exact_versions/v2_2.lance @@ -0,0 +1,633 @@ +blob-0000-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0001-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0002-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0003-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0004-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0005-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0006-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0007-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0008-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0009-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0010-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0011-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0012-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0013-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0014-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0015-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0016-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0017-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0018-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0019-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0020-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0021-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0022-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0023-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0024-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0025-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0026-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0027-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0028-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0029-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0030-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0031-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0032-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0033-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0034-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0035-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0036-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0037-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0038-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0039-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0040-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0041-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0042-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0043-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0044-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0045-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0046-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0047-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0048-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0049-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0050-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0051-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0052-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0053-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0054-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0055-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0056-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0057-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0058-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0059-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0060-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0061-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0062-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0063-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0064-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0065-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0066-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0067-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0068-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0069-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0070-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0071-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0072-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0073-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0074-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0075-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0076-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0077-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0078-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0079-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0080-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0081-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0082-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0083-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0084-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0085-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0086-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0087-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0088-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0089-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0090-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0091-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0092-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0093-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0094-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0095-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0096-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0097-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0098-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0099-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0100-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0101-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0102-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0103-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0104-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0105-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0106-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0107-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0108-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0109-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0110-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0111-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0112-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0113-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0114-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0115-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0116-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0117-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0118-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0119-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0120-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0121-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0122-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0123-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0124-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0125-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0126-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0127-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0128-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0129-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0130-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0131-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0132-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0133-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0134-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0135-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0136-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0137-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0138-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0139-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0140-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0141-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0142-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0143-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0144-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0145-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0146-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0147-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0148-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0149-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0150-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0151-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0152-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0153-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0154-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0155-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0156-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0157-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0158-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0159-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0160-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0161-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0162-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0163-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0164-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0165-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0166-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0167-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0168-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0169-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0170-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0171-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0172-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0173-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0174-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0175-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0176-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0177-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0178-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0179-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0180-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0181-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0182-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0183-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0184-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0185-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0186-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0187-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0188-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0189-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0190-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0191-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0192-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0193-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0194-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0195-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0196-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0197-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0198-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0199-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0200-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0201-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0202-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0203-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0204-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0205-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0206-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0207-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0208-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0209-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0210-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0211-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0212-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0213-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0214-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0215-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0216-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0217-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0218-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0219-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0220-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0221-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0222-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0223-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0224-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0225-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0226-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0227-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0228-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0229-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0230-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0231-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0232-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0233-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0234-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0235-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0236-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0237-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0238-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0239-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0240-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0241-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0242-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0243-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0244-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0245-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0246-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0247-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0248-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0249-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0250-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0251-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0252-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0253-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0254-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0255-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0256-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0257-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0258-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0259-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0260-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0261-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0262-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0263-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0264-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0265-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0266-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0267-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0268-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0269-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0270-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0271-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0272-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0273-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0274-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0275-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0276-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0277-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0278-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0279-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0280-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0281-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0282-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0283-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0284-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0285-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0286-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0287-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0288-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0289-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0290-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0291-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0292-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0293-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0294-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0295-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0296-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0297-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0298-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0299-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0300-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0301-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0302-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0303-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0304-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0305-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0306-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0307-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0308-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0309-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0310-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0311-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0312-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0313-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0314-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0315-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0316-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0317-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0318-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0319-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0320-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0321-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0322-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0323-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0324-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0325-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0326-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0327-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0328-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0329-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0330-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0331-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0332-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0333-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0334-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0335-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0336-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0337-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0338-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0339-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0340-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0341-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0342-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0343-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0344-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0345-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0346-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0347-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0348-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0349-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0350-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0351-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0352-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0353-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0354-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0355-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0356-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0357-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0358-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0359-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0360-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0361-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0362-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0363-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0364-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0365-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0366-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0367-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0368-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0369-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0370-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0371-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0372-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0373-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0374-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0375-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0376-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0377-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0378-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0379-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0380-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0381-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0382-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0383-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0384-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0385-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0386-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0387-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0388-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0389-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0390-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0391-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0392-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0393-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0394-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0395-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0396-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0397-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0398-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0399-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0400-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0401-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0402-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0403-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0404-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0405-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0406-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0407-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0408-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0409-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0410-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0411-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0412-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0413-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0414-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0415-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0416-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0417-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0418-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0419-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0420-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0421-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0422-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0423-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0424-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0425-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0426-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0427-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0428-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0429-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0430-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0431-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0432-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0433-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0434-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0435-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0436-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0437-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0438-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0439-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0440-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0441-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0442-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0443-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0444-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0445-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0446-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0447-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0448-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0449-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0450-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0451-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0452-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0453-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0454-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0455-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0456-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0457-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0458-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0459-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0460-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0461-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0462-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0463-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0464-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0465-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0466-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0467-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0468-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0469-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0470-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0471-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0472-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0473-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0474-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0475-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0476-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0477-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0478-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0479-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0480-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0481-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0482-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0483-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0484-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0485-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0486-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0487-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0488-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0489-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0490-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0491-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0492-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0493-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0494-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0495-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0496-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0497-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0498-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0499-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0500-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0501-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0502-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0503-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0504-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0505-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0506-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0507-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0508-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0509-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0510-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0511-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0512-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0513-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0514-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0515-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0516-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0517-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0518-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0519-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0520-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0521-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0522-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0523-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0524-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0525-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0526-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0527-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0528-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0529-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0530-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0531-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0532-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0533-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0534-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0535-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0536-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0537-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0538-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0539-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0540-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0541-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0542-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0543-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0544-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0545-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0546-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0547-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0548-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0549-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0550-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0551-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0552-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0553-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0554-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0555-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0556-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0557-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0558-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0559-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0560-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0561-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0562-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0563-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0564-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0565-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0566-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0567-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0568-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0569-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0570-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0571-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0572-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0573-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0574-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0575-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0576-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0577-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0578-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0579-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0580-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0581-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0582-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0583-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0584-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0585-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0586-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0587-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0588-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0589-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0590-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0591-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0592-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0593-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0594-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0595-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0596-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0597-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0598-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0599-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0600-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0601-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0602-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0603-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0604-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0605-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0606-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0607-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0608-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0609-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0610-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0611-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0612-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0613-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0614-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0615-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0616-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0617-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0618-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0619-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0620-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0621-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0622-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0623-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0624-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0625-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0626-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0627-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0628-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0629-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0630-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0631-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0632-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0633-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0634-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0635-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0636-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0637-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0638-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0639-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0640-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0641-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0642-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0643-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0644-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0645-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0646-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0647-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0648-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0649-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0650-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0651-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0652-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0653-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0654-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0655-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0656-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0657-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0658-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0659-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0660-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0661-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0662-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0663-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0664-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0665-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0666-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0667-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0668-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0669-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0670-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0671-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0672-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0673-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0674-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0675-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0676-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0677-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0678-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0679-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0680-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0681-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0682-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0683-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0684-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0685-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0686-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0687-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0688-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0689-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0690-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0691-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0692-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0693-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0694-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0695-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0696-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0697-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0698-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0699-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0700-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0701-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0702-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0703-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0704-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0705-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0706-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0707-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0708-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0709-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0710-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0711-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0712-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0713-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0714-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0715-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0716-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0717-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0718-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0719-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0720-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0721-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0722-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0723-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0724-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0725-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0726-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0727-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0728-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0729-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0730-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0731-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0732-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0733-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0734-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0735-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0736-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0737-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0738-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0739-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0740-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0741-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0742-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0743-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0744-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0745-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0746-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0747-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0748-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0749-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0750-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0751-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0752-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0753-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0754-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0755-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0756-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0757-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0758-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0759-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0760-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0761-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0762-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0763-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0764-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0765-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0766-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0767-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0768-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0769-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0770-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0771-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0772-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0773-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0774-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0775-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0776-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0777-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0778-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0779-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0780-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0781-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0782-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0783-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0784-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0785-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0786-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0787-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0788-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0789-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0790-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0791-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0792-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0793-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0794-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0795-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0796-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0797-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0798-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0799-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0800-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0801-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0802-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0803-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0804-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0805-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0806-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0807-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0808-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0809-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0810-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0811-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0812-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0813-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0814-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0815-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0816-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0817-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0818-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0819-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0820-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0821-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0822-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0823-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0824-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0825-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0826-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0827-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0828-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0829-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0830-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0831-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0832-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0833-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0834-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0835-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0836-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0837-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0838-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0839-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0840-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0841-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0842-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0843-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0844-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0845-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0846-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0847-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0848-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0849-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0850-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0851-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0852-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0853-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0854-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0855-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0856-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0857-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0858-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0859-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0860-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0861-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0862-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0863-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0864-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0865-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0866-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0867-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0868-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0869-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0870-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0871-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0872-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0873-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0874-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0875-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0876-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0877-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0878-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0879-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0880-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0881-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0882-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0883-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0884-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0885-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0886-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0887-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0888-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0889-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0890-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0891-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0892-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0893-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0894-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0895-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0896-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0897-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0898-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0899-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0900-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0901-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0902-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0903-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0904-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0905-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0906-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0907-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0908-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0909-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0910-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0911-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0912-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0913-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0914-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0915-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0916-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0917-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0918-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0919-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0920-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0921-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0922-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0923-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0924-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0925-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0926-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0927-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0928-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0929-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0930-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0931-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0932-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0933-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0934-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0935-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0936-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0937-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0938-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0939-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0940-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0941-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0942-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0943-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0944-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0945-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0946-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0947-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0948-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0949-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0950-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0951-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0952-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0953-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0954-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0955-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0956-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0957-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0958-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0959-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0960-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0961-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0962-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0963-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0964-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0965-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0966-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0967-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0968-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0969-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0970-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0971-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0972-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0973-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0974-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0975-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0976-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0977-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0978-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0979-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0980-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0981-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0982-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0983-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0984-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0985-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0986-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0987-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0988-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0989-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0990-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0991-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0992-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0993-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0994-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0995-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0996-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0997-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0998-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0999-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1000-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1001-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1002-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1003-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1004-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1005-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1006-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1007-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1008-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1009-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1010-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1011-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1012-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1013-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1014-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1015-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1016-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1017-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1018-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1019-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1020-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1021-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1022-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1023-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH +HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH +P" 2BRPbr  $P +( , 0 4P8<@DQH"L2PBTRQXb\r`dQhlptQx|` +`` +*`:aJaZajazb"b &b +*b .c 2c 6c:c>dB +dFdJ*dN:eRJeVZeZje^zfbfffjfngrgvgzg~0@pABC 0DpEFG0H pI$J(K,0L0pM4N8O<1P@qQDRHSL1TPqUTVXW\1X`qYdZh[l1\pq]t^x_|p@TpA#pB3pCCqDSTqEcqFsqGrHTrIrJrKsLTsMsNsOtPUtQ#tR3tSCuTSUuUcuVsuWvXUvYvZv[w\Uw]w^w_ 4t ++;K4[tk{"4&t*.246t:>B 5FuJ+N;RK5V[uZk^{b5fujnr5vuz~ !R""#2$B%RR&b'r()R*+,-R./01S2"324B5RS6b7r89S:袓;<=S>?h +h!h"*h#:i$Ji%Zi&ji'zj(j)j*j+k,k-k.k/l0 +l1l2*l3:m4Jm5Zm6jm7zn8n9n:n;oo?2`rabc2drefg2hrijk2lrmno3psqijrs3tsuԳvw3xsyz{3|s}~x`Vxa#xb3xcCydSVyecyfsygzhVzizjzk{lV{m{n{o|pW|q#|r3|sC}tSW}uc}vs}w~xW~y~z~{|W}~ 6v+;K6[vk{6v6v 7w+;K7[wk{7w꫷7wHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHg g ` HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH$Dd$Dd$DDd$Dd$Dd$DDd$Dd $ D d  +$ +D +D +d + + + + +  $ D d  $ D d  $ D D d $Dd$Ddvalue-0001-deterministic-fixturevalue-0002-deterministic-fixturevalue-0003-deterministic-fixturevalue-0004-deterministic-fixturevalue-0005-deterministic-fixturevalue-0006-deterministic-fixturevalue-0008-deterministic-fixturevalue-0009-deterministic-fixturevalue-0010-deterministic-fixturevalue-0011-deterministic-fixturevalue-0012-deterministic-fixturevalue-0013-deterministic-fixturevalue-0015-deterministic-fixturevalue-0016-deterministic-fixturevalue-0017-deterministic-fixturevalue-0018-deterministic-fixturevalue-0019-deterministic-fixturevalue-0020-deterministic-fixturevalue-0022-deterministic-fixturevalue-0023-deterministic-fixturevalue-0024-deterministic-fixturevalue-0025-deterministic-fixturevalue-0026-deterministic-fixturevalue-0027-deterministic-fixturevalue-0029-deterministic-fixturevalue-0030-deterministic-fixturevalue-0031-deterministic-fixturevalue-0032-deterministic-fixturevalue-0033-deterministic-fixturevalue-0034-deterministic-fixturevalue-0036-deterministic-fixturevalue-0037-deterministic-fixturevalue-0038-deterministic-fixturevalue-0039-deterministic-fixturevalue-0040-deterministic-fixturevalue-0041-deterministic-fixturevalue-0043-deterministic-fixturevalue-0044-deterministic-fixturevalue-0045-deterministic-fixturevalue-0046-deterministic-fixturevalue-0047-deterministic-fixturevalue-0048-deterministic-fixturevalue-0050-deterministic-fixturevalue-0051-deterministic-fixturevalue-0052-deterministic-fixturevalue-0053-deterministic-fixturevalue-0054-deterministic-fixturevalue-0055-deterministic-fixturevalue-0057-deterministic-fixturevalue-0058-deterministic-fixturevalue-0059-deterministic-fixturevalue-0060-deterministic-fixturevalue-0061-deterministic-fixturevalue-0062-deterministic-fixturevalue-0064-deterministic-fixturevalue-0065-deterministic-fixturevalue-0066-deterministic-fixturevalue-0067-deterministic-fixturevalue-0068-deterministic-fixturevalue-0069-deterministic-fixturevalue-0071-deterministic-fixturevalue-0072-deterministic-fixturevalue-0073-deterministic-fixturevalue-0074-deterministic-fixturevalue-0075-deterministic-fixturevalue-0076-deterministic-fixturevalue-0078-deterministic-fixturevalue-0079-deterministic-fixturevalue-0080-deterministic-fixturevalue-0081-deterministic-fixturevalue-0082-deterministic-fixturevalue-0083-deterministic-fixturevalue-0085-deterministic-fixturevalue-0086-deterministic-fixturevalue-0087-deterministic-fixturevalue-0088-deterministic-fixturevalue-0089-deterministic-fixturevalue-0090-deterministic-fixturevalue-0092-deterministic-fixturevalue-0093-deterministic-fixturevalue-0094-deterministic-fixturevalue-0095-deterministic-fixturevalue-0096-deterministic-fixturevalue-0097-deterministic-fixturevalue-0099-deterministic-fixturevalue-0100-deterministic-fixturevalue-0101-deterministic-fixturevalue-0102-deterministic-fixturevalue-0103-deterministic-fixturevalue-0104-deterministic-fixturevalue-0106-deterministic-fixturevalue-0107-deterministic-fixturevalue-0108-deterministic-fixturevalue-0109-deterministic-fixturevalue-0110-deterministic-fixturevalue-0111-deterministic-fixturevalue-0113-deterministic-fixturevalue-0114-deterministic-fixturevalue-0115-deterministic-fixturevalue-0116-deterministic-fixturevalue-0117-deterministic-fixturevalue-0118-deterministic-fixturevalue-0120-deterministic-fixturevalue-0121-deterministic-fixturevalue-0122-deterministic-fixturevalue-0123-deterministic-fixturevalue-0124-deterministic-fixturevalue-0125-deterministic-fixturevalue-0127-deterministic-fixture$Dd$Ddd$$Dd$Dd$Ddd$$Dd$Dd $ D d d  +$ +$ +D +d + + + + + + $ D d  $ D d d  $ $ D d $Dd$Dddvalue-0128-deterministic-fixturevalue-0129-deterministic-fixturevalue-0130-deterministic-fixturevalue-0131-deterministic-fixturevalue-0132-deterministic-fixturevalue-0134-deterministic-fixturevalue-0135-deterministic-fixturevalue-0136-deterministic-fixturevalue-0137-deterministic-fixturevalue-0138-deterministic-fixturevalue-0139-deterministic-fixturevalue-0141-deterministic-fixturevalue-0142-deterministic-fixturevalue-0143-deterministic-fixturevalue-0144-deterministic-fixturevalue-0145-deterministic-fixturevalue-0146-deterministic-fixturevalue-0148-deterministic-fixturevalue-0149-deterministic-fixturevalue-0150-deterministic-fixturevalue-0151-deterministic-fixturevalue-0152-deterministic-fixturevalue-0153-deterministic-fixturevalue-0155-deterministic-fixturevalue-0156-deterministic-fixturevalue-0157-deterministic-fixturevalue-0158-deterministic-fixturevalue-0159-deterministic-fixturevalue-0160-deterministic-fixturevalue-0162-deterministic-fixturevalue-0163-deterministic-fixturevalue-0164-deterministic-fixturevalue-0165-deterministic-fixturevalue-0166-deterministic-fixturevalue-0167-deterministic-fixturevalue-0169-deterministic-fixturevalue-0170-deterministic-fixturevalue-0171-deterministic-fixturevalue-0172-deterministic-fixturevalue-0173-deterministic-fixturevalue-0174-deterministic-fixturevalue-0176-deterministic-fixturevalue-0177-deterministic-fixturevalue-0178-deterministic-fixturevalue-0179-deterministic-fixturevalue-0180-deterministic-fixturevalue-0181-deterministic-fixturevalue-0183-deterministic-fixturevalue-0184-deterministic-fixturevalue-0185-deterministic-fixturevalue-0186-deterministic-fixturevalue-0187-deterministic-fixturevalue-0188-deterministic-fixturevalue-0190-deterministic-fixturevalue-0191-deterministic-fixturevalue-0192-deterministic-fixturevalue-0193-deterministic-fixturevalue-0194-deterministic-fixturevalue-0195-deterministic-fixturevalue-0197-deterministic-fixturevalue-0198-deterministic-fixturevalue-0199-deterministic-fixturevalue-0200-deterministic-fixturevalue-0201-deterministic-fixturevalue-0202-deterministic-fixturevalue-0204-deterministic-fixturevalue-0205-deterministic-fixturevalue-0206-deterministic-fixturevalue-0207-deterministic-fixturevalue-0208-deterministic-fixturevalue-0209-deterministic-fixturevalue-0211-deterministic-fixturevalue-0212-deterministic-fixturevalue-0213-deterministic-fixturevalue-0214-deterministic-fixturevalue-0215-deterministic-fixturevalue-0216-deterministic-fixturevalue-0218-deterministic-fixturevalue-0219-deterministic-fixturevalue-0220-deterministic-fixturevalue-0221-deterministic-fixturevalue-0222-deterministic-fixturevalue-0223-deterministic-fixturevalue-0225-deterministic-fixturevalue-0226-deterministic-fixturevalue-0227-deterministic-fixturevalue-0228-deterministic-fixturevalue-0229-deterministic-fixturevalue-0230-deterministic-fixturevalue-0232-deterministic-fixturevalue-0233-deterministic-fixturevalue-0234-deterministic-fixturevalue-0235-deterministic-fixturevalue-0236-deterministic-fixturevalue-0237-deterministic-fixturevalue-0239-deterministic-fixturevalue-0240-deterministic-fixturevalue-0241-deterministic-fixturevalue-0242-deterministic-fixturevalue-0243-deterministic-fixturevalue-0244-deterministic-fixturevalue-0246-deterministic-fixturevalue-0247-deterministic-fixturevalue-0248-deterministic-fixturevalue-0249-deterministic-fixturevalue-0250-deterministic-fixturevalue-0251-deterministic-fixturevalue-0253-deterministic-fixturevalue-0254-deterministic-fixturevalue-0255-deterministic-fixture$Ddd$$Dd$Dd$Ddd$$Dd$Dd$Ddd $ $ D d  +$ +D +d + + + + + + $ D d d  $ $ D d  $ D d $Ddd$$Ddvalue-0256-deterministic-fixturevalue-0257-deterministic-fixturevalue-0258-deterministic-fixturevalue-0260-deterministic-fixturevalue-0261-deterministic-fixturevalue-0262-deterministic-fixturevalue-0263-deterministic-fixturevalue-0264-deterministic-fixturevalue-0265-deterministic-fixturevalue-0267-deterministic-fixturevalue-0268-deterministic-fixturevalue-0269-deterministic-fixturevalue-0270-deterministic-fixturevalue-0271-deterministic-fixturevalue-0272-deterministic-fixturevalue-0274-deterministic-fixturevalue-0275-deterministic-fixturevalue-0276-deterministic-fixturevalue-0277-deterministic-fixturevalue-0278-deterministic-fixturevalue-0279-deterministic-fixturevalue-0281-deterministic-fixturevalue-0282-deterministic-fixturevalue-0283-deterministic-fixturevalue-0284-deterministic-fixturevalue-0285-deterministic-fixturevalue-0286-deterministic-fixturevalue-0288-deterministic-fixturevalue-0289-deterministic-fixturevalue-0290-deterministic-fixturevalue-0291-deterministic-fixturevalue-0292-deterministic-fixturevalue-0293-deterministic-fixturevalue-0295-deterministic-fixturevalue-0296-deterministic-fixturevalue-0297-deterministic-fixturevalue-0298-deterministic-fixturevalue-0299-deterministic-fixturevalue-0300-deterministic-fixturevalue-0302-deterministic-fixturevalue-0303-deterministic-fixturevalue-0304-deterministic-fixturevalue-0305-deterministic-fixturevalue-0306-deterministic-fixturevalue-0307-deterministic-fixturevalue-0309-deterministic-fixturevalue-0310-deterministic-fixturevalue-0311-deterministic-fixturevalue-0312-deterministic-fixturevalue-0313-deterministic-fixturevalue-0314-deterministic-fixturevalue-0316-deterministic-fixturevalue-0317-deterministic-fixturevalue-0318-deterministic-fixturevalue-0319-deterministic-fixturevalue-0320-deterministic-fixturevalue-0321-deterministic-fixturevalue-0323-deterministic-fixturevalue-0324-deterministic-fixturevalue-0325-deterministic-fixturevalue-0326-deterministic-fixturevalue-0327-deterministic-fixturevalue-0328-deterministic-fixturevalue-0330-deterministic-fixturevalue-0331-deterministic-fixturevalue-0332-deterministic-fixturevalue-0333-deterministic-fixturevalue-0334-deterministic-fixturevalue-0335-deterministic-fixturevalue-0337-deterministic-fixturevalue-0338-deterministic-fixturevalue-0339-deterministic-fixturevalue-0340-deterministic-fixturevalue-0341-deterministic-fixturevalue-0342-deterministic-fixturevalue-0344-deterministic-fixturevalue-0345-deterministic-fixturevalue-0346-deterministic-fixturevalue-0347-deterministic-fixturevalue-0348-deterministic-fixturevalue-0349-deterministic-fixturevalue-0351-deterministic-fixturevalue-0352-deterministic-fixturevalue-0353-deterministic-fixturevalue-0354-deterministic-fixturevalue-0355-deterministic-fixturevalue-0356-deterministic-fixturevalue-0358-deterministic-fixturevalue-0359-deterministic-fixturevalue-0360-deterministic-fixturevalue-0361-deterministic-fixturevalue-0362-deterministic-fixturevalue-0363-deterministic-fixturevalue-0365-deterministic-fixturevalue-0366-deterministic-fixturevalue-0367-deterministic-fixturevalue-0368-deterministic-fixturevalue-0369-deterministic-fixturevalue-0370-deterministic-fixturevalue-0372-deterministic-fixturevalue-0373-deterministic-fixturevalue-0374-deterministic-fixturevalue-0375-deterministic-fixturevalue-0376-deterministic-fixturevalue-0377-deterministic-fixturevalue-0379-deterministic-fixturevalue-0380-deterministic-fixturevalue-0381-deterministic-fixturevalue-0382-deterministic-fixturevalue-0383-deterministic-fixture$$Dd$Dd$Ddd$$Dd$Dd$Ddd$$Dd $ D d  +$ +D +d +d + + + + + $ $ D d  $ D d  $ D d d $$Dd$Ddvalue-0384-deterministic-fixturevalue-0386-deterministic-fixturevalue-0387-deterministic-fixturevalue-0388-deterministic-fixturevalue-0389-deterministic-fixturevalue-0390-deterministic-fixturevalue-0391-deterministic-fixturevalue-0393-deterministic-fixturevalue-0394-deterministic-fixturevalue-0395-deterministic-fixturevalue-0396-deterministic-fixturevalue-0397-deterministic-fixturevalue-0398-deterministic-fixturevalue-0400-deterministic-fixturevalue-0401-deterministic-fixturevalue-0402-deterministic-fixturevalue-0403-deterministic-fixturevalue-0404-deterministic-fixturevalue-0405-deterministic-fixturevalue-0407-deterministic-fixturevalue-0408-deterministic-fixturevalue-0409-deterministic-fixturevalue-0410-deterministic-fixturevalue-0411-deterministic-fixturevalue-0412-deterministic-fixturevalue-0414-deterministic-fixturevalue-0415-deterministic-fixturevalue-0416-deterministic-fixturevalue-0417-deterministic-fixturevalue-0418-deterministic-fixturevalue-0419-deterministic-fixturevalue-0421-deterministic-fixturevalue-0422-deterministic-fixturevalue-0423-deterministic-fixturevalue-0424-deterministic-fixturevalue-0425-deterministic-fixturevalue-0426-deterministic-fixturevalue-0428-deterministic-fixturevalue-0429-deterministic-fixturevalue-0430-deterministic-fixturevalue-0431-deterministic-fixturevalue-0432-deterministic-fixturevalue-0433-deterministic-fixturevalue-0435-deterministic-fixturevalue-0436-deterministic-fixturevalue-0437-deterministic-fixturevalue-0438-deterministic-fixturevalue-0439-deterministic-fixturevalue-0440-deterministic-fixturevalue-0442-deterministic-fixturevalue-0443-deterministic-fixturevalue-0444-deterministic-fixturevalue-0445-deterministic-fixturevalue-0446-deterministic-fixturevalue-0447-deterministic-fixturevalue-0449-deterministic-fixturevalue-0450-deterministic-fixturevalue-0451-deterministic-fixturevalue-0452-deterministic-fixturevalue-0453-deterministic-fixturevalue-0454-deterministic-fixturevalue-0456-deterministic-fixturevalue-0457-deterministic-fixturevalue-0458-deterministic-fixturevalue-0459-deterministic-fixturevalue-0460-deterministic-fixturevalue-0461-deterministic-fixturevalue-0463-deterministic-fixturevalue-0464-deterministic-fixturevalue-0465-deterministic-fixturevalue-0466-deterministic-fixturevalue-0467-deterministic-fixturevalue-0468-deterministic-fixturevalue-0470-deterministic-fixturevalue-0471-deterministic-fixturevalue-0472-deterministic-fixturevalue-0473-deterministic-fixturevalue-0474-deterministic-fixturevalue-0475-deterministic-fixturevalue-0477-deterministic-fixturevalue-0478-deterministic-fixturevalue-0479-deterministic-fixturevalue-0480-deterministic-fixturevalue-0481-deterministic-fixturevalue-0482-deterministic-fixturevalue-0484-deterministic-fixturevalue-0485-deterministic-fixturevalue-0486-deterministic-fixturevalue-0487-deterministic-fixturevalue-0488-deterministic-fixturevalue-0489-deterministic-fixturevalue-0491-deterministic-fixturevalue-0492-deterministic-fixturevalue-0493-deterministic-fixturevalue-0494-deterministic-fixturevalue-0495-deterministic-fixturevalue-0496-deterministic-fixturevalue-0498-deterministic-fixturevalue-0499-deterministic-fixturevalue-0500-deterministic-fixturevalue-0501-deterministic-fixturevalue-0502-deterministic-fixturevalue-0503-deterministic-fixturevalue-0505-deterministic-fixturevalue-0506-deterministic-fixturevalue-0507-deterministic-fixturevalue-0508-deterministic-fixturevalue-0509-deterministic-fixturevalue-0510-deterministic-fixture$Dd$Dd$DDd$Dd$Dd$DDd$Dd $ D d  +$ +D +D +d + + + + +  $ D d  $ D d  $ D D d $Dd$Ddvalue-0512-deterministic-fixturevalue-0513-deterministic-fixturevalue-0514-deterministic-fixturevalue-0515-deterministic-fixturevalue-0516-deterministic-fixturevalue-0517-deterministic-fixturevalue-0519-deterministic-fixturevalue-0520-deterministic-fixturevalue-0521-deterministic-fixturevalue-0522-deterministic-fixturevalue-0523-deterministic-fixturevalue-0524-deterministic-fixturevalue-0526-deterministic-fixturevalue-0527-deterministic-fixturevalue-0528-deterministic-fixturevalue-0529-deterministic-fixturevalue-0530-deterministic-fixturevalue-0531-deterministic-fixturevalue-0533-deterministic-fixturevalue-0534-deterministic-fixturevalue-0535-deterministic-fixturevalue-0536-deterministic-fixturevalue-0537-deterministic-fixturevalue-0538-deterministic-fixturevalue-0540-deterministic-fixturevalue-0541-deterministic-fixturevalue-0542-deterministic-fixturevalue-0543-deterministic-fixturevalue-0544-deterministic-fixturevalue-0545-deterministic-fixturevalue-0547-deterministic-fixturevalue-0548-deterministic-fixturevalue-0549-deterministic-fixturevalue-0550-deterministic-fixturevalue-0551-deterministic-fixturevalue-0552-deterministic-fixturevalue-0554-deterministic-fixturevalue-0555-deterministic-fixturevalue-0556-deterministic-fixturevalue-0557-deterministic-fixturevalue-0558-deterministic-fixturevalue-0559-deterministic-fixturevalue-0561-deterministic-fixturevalue-0562-deterministic-fixturevalue-0563-deterministic-fixturevalue-0564-deterministic-fixturevalue-0565-deterministic-fixturevalue-0566-deterministic-fixturevalue-0568-deterministic-fixturevalue-0569-deterministic-fixturevalue-0570-deterministic-fixturevalue-0571-deterministic-fixturevalue-0572-deterministic-fixturevalue-0573-deterministic-fixturevalue-0575-deterministic-fixturevalue-0576-deterministic-fixturevalue-0577-deterministic-fixturevalue-0578-deterministic-fixturevalue-0579-deterministic-fixturevalue-0580-deterministic-fixturevalue-0582-deterministic-fixturevalue-0583-deterministic-fixturevalue-0584-deterministic-fixturevalue-0585-deterministic-fixturevalue-0586-deterministic-fixturevalue-0587-deterministic-fixturevalue-0589-deterministic-fixturevalue-0590-deterministic-fixturevalue-0591-deterministic-fixturevalue-0592-deterministic-fixturevalue-0593-deterministic-fixturevalue-0594-deterministic-fixturevalue-0596-deterministic-fixturevalue-0597-deterministic-fixturevalue-0598-deterministic-fixturevalue-0599-deterministic-fixturevalue-0600-deterministic-fixturevalue-0601-deterministic-fixturevalue-0603-deterministic-fixturevalue-0604-deterministic-fixturevalue-0605-deterministic-fixturevalue-0606-deterministic-fixturevalue-0607-deterministic-fixturevalue-0608-deterministic-fixturevalue-0610-deterministic-fixturevalue-0611-deterministic-fixturevalue-0612-deterministic-fixturevalue-0613-deterministic-fixturevalue-0614-deterministic-fixturevalue-0615-deterministic-fixturevalue-0617-deterministic-fixturevalue-0618-deterministic-fixturevalue-0619-deterministic-fixturevalue-0620-deterministic-fixturevalue-0621-deterministic-fixturevalue-0622-deterministic-fixturevalue-0624-deterministic-fixturevalue-0625-deterministic-fixturevalue-0626-deterministic-fixturevalue-0627-deterministic-fixturevalue-0628-deterministic-fixturevalue-0629-deterministic-fixturevalue-0631-deterministic-fixturevalue-0632-deterministic-fixturevalue-0633-deterministic-fixturevalue-0634-deterministic-fixturevalue-0635-deterministic-fixturevalue-0636-deterministic-fixturevalue-0638-deterministic-fixturevalue-0639-deterministic-fixture$Dd$DDd$Dd$Dd$DDd$Dd$Dd $ D D d  + +$ +D +d + + + + + + $ D d  $ D D d   $ D d $Dd$DDdvalue-0640-deterministic-fixturevalue-0641-deterministic-fixturevalue-0642-deterministic-fixturevalue-0643-deterministic-fixturevalue-0645-deterministic-fixturevalue-0646-deterministic-fixturevalue-0647-deterministic-fixturevalue-0648-deterministic-fixturevalue-0649-deterministic-fixturevalue-0650-deterministic-fixturevalue-0652-deterministic-fixturevalue-0653-deterministic-fixturevalue-0654-deterministic-fixturevalue-0655-deterministic-fixturevalue-0656-deterministic-fixturevalue-0657-deterministic-fixturevalue-0659-deterministic-fixturevalue-0660-deterministic-fixturevalue-0661-deterministic-fixturevalue-0662-deterministic-fixturevalue-0663-deterministic-fixturevalue-0664-deterministic-fixturevalue-0666-deterministic-fixturevalue-0667-deterministic-fixturevalue-0668-deterministic-fixturevalue-0669-deterministic-fixturevalue-0670-deterministic-fixturevalue-0671-deterministic-fixturevalue-0673-deterministic-fixturevalue-0674-deterministic-fixturevalue-0675-deterministic-fixturevalue-0676-deterministic-fixturevalue-0677-deterministic-fixturevalue-0678-deterministic-fixturevalue-0680-deterministic-fixturevalue-0681-deterministic-fixturevalue-0682-deterministic-fixturevalue-0683-deterministic-fixturevalue-0684-deterministic-fixturevalue-0685-deterministic-fixturevalue-0687-deterministic-fixturevalue-0688-deterministic-fixturevalue-0689-deterministic-fixturevalue-0690-deterministic-fixturevalue-0691-deterministic-fixturevalue-0692-deterministic-fixturevalue-0694-deterministic-fixturevalue-0695-deterministic-fixturevalue-0696-deterministic-fixturevalue-0697-deterministic-fixturevalue-0698-deterministic-fixturevalue-0699-deterministic-fixturevalue-0701-deterministic-fixturevalue-0702-deterministic-fixturevalue-0703-deterministic-fixturevalue-0704-deterministic-fixturevalue-0705-deterministic-fixturevalue-0706-deterministic-fixturevalue-0708-deterministic-fixturevalue-0709-deterministic-fixturevalue-0710-deterministic-fixturevalue-0711-deterministic-fixturevalue-0712-deterministic-fixturevalue-0713-deterministic-fixturevalue-0715-deterministic-fixturevalue-0716-deterministic-fixturevalue-0717-deterministic-fixturevalue-0718-deterministic-fixturevalue-0719-deterministic-fixturevalue-0720-deterministic-fixturevalue-0722-deterministic-fixturevalue-0723-deterministic-fixturevalue-0724-deterministic-fixturevalue-0725-deterministic-fixturevalue-0726-deterministic-fixturevalue-0727-deterministic-fixturevalue-0729-deterministic-fixturevalue-0730-deterministic-fixturevalue-0731-deterministic-fixturevalue-0732-deterministic-fixturevalue-0733-deterministic-fixturevalue-0734-deterministic-fixturevalue-0736-deterministic-fixturevalue-0737-deterministic-fixturevalue-0738-deterministic-fixturevalue-0739-deterministic-fixturevalue-0740-deterministic-fixturevalue-0741-deterministic-fixturevalue-0743-deterministic-fixturevalue-0744-deterministic-fixturevalue-0745-deterministic-fixturevalue-0746-deterministic-fixturevalue-0747-deterministic-fixturevalue-0748-deterministic-fixturevalue-0750-deterministic-fixturevalue-0751-deterministic-fixturevalue-0752-deterministic-fixturevalue-0753-deterministic-fixturevalue-0754-deterministic-fixturevalue-0755-deterministic-fixturevalue-0757-deterministic-fixturevalue-0758-deterministic-fixturevalue-0759-deterministic-fixturevalue-0760-deterministic-fixturevalue-0761-deterministic-fixturevalue-0762-deterministic-fixturevalue-0764-deterministic-fixturevalue-0765-deterministic-fixturevalue-0766-deterministic-fixturevalue-0767-deterministic-fixture$DDd$Dd$Dd$DDd$Dd$Dd$DDd  $ D d  +$ +D +d + + + + + + $ D D d   $ D d  $ D d $DDd$Ddvalue-0768-deterministic-fixturevalue-0769-deterministic-fixturevalue-0771-deterministic-fixturevalue-0772-deterministic-fixturevalue-0773-deterministic-fixturevalue-0774-deterministic-fixturevalue-0775-deterministic-fixturevalue-0776-deterministic-fixturevalue-0778-deterministic-fixturevalue-0779-deterministic-fixturevalue-0780-deterministic-fixturevalue-0781-deterministic-fixturevalue-0782-deterministic-fixturevalue-0783-deterministic-fixturevalue-0785-deterministic-fixturevalue-0786-deterministic-fixturevalue-0787-deterministic-fixturevalue-0788-deterministic-fixturevalue-0789-deterministic-fixturevalue-0790-deterministic-fixturevalue-0792-deterministic-fixturevalue-0793-deterministic-fixturevalue-0794-deterministic-fixturevalue-0795-deterministic-fixturevalue-0796-deterministic-fixturevalue-0797-deterministic-fixturevalue-0799-deterministic-fixturevalue-0800-deterministic-fixturevalue-0801-deterministic-fixturevalue-0802-deterministic-fixturevalue-0803-deterministic-fixturevalue-0804-deterministic-fixturevalue-0806-deterministic-fixturevalue-0807-deterministic-fixturevalue-0808-deterministic-fixturevalue-0809-deterministic-fixturevalue-0810-deterministic-fixturevalue-0811-deterministic-fixturevalue-0813-deterministic-fixturevalue-0814-deterministic-fixturevalue-0815-deterministic-fixturevalue-0816-deterministic-fixturevalue-0817-deterministic-fixturevalue-0818-deterministic-fixturevalue-0820-deterministic-fixturevalue-0821-deterministic-fixturevalue-0822-deterministic-fixturevalue-0823-deterministic-fixturevalue-0824-deterministic-fixturevalue-0825-deterministic-fixturevalue-0827-deterministic-fixturevalue-0828-deterministic-fixturevalue-0829-deterministic-fixturevalue-0830-deterministic-fixturevalue-0831-deterministic-fixturevalue-0832-deterministic-fixturevalue-0834-deterministic-fixturevalue-0835-deterministic-fixturevalue-0836-deterministic-fixturevalue-0837-deterministic-fixturevalue-0838-deterministic-fixturevalue-0839-deterministic-fixturevalue-0841-deterministic-fixturevalue-0842-deterministic-fixturevalue-0843-deterministic-fixturevalue-0844-deterministic-fixturevalue-0845-deterministic-fixturevalue-0846-deterministic-fixturevalue-0848-deterministic-fixturevalue-0849-deterministic-fixturevalue-0850-deterministic-fixturevalue-0851-deterministic-fixturevalue-0852-deterministic-fixturevalue-0853-deterministic-fixturevalue-0855-deterministic-fixturevalue-0856-deterministic-fixturevalue-0857-deterministic-fixturevalue-0858-deterministic-fixturevalue-0859-deterministic-fixturevalue-0860-deterministic-fixturevalue-0862-deterministic-fixturevalue-0863-deterministic-fixturevalue-0864-deterministic-fixturevalue-0865-deterministic-fixturevalue-0866-deterministic-fixturevalue-0867-deterministic-fixturevalue-0869-deterministic-fixturevalue-0870-deterministic-fixturevalue-0871-deterministic-fixturevalue-0872-deterministic-fixturevalue-0873-deterministic-fixturevalue-0874-deterministic-fixturevalue-0876-deterministic-fixturevalue-0877-deterministic-fixturevalue-0878-deterministic-fixturevalue-0879-deterministic-fixturevalue-0880-deterministic-fixturevalue-0881-deterministic-fixturevalue-0883-deterministic-fixturevalue-0884-deterministic-fixturevalue-0885-deterministic-fixturevalue-0886-deterministic-fixturevalue-0887-deterministic-fixturevalue-0888-deterministic-fixturevalue-0890-deterministic-fixturevalue-0891-deterministic-fixturevalue-0892-deterministic-fixturevalue-0893-deterministic-fixturevalue-0894-deterministic-fixturevalue-0895-deterministic-fixture$Dd$Dd$DDd$Dd$Dd$DDd$Dd $ D d  +$ +D +D +d + + + + +  $ D d  $ D d  $ D D d $Dd$Ddvalue-0897-deterministic-fixturevalue-0898-deterministic-fixturevalue-0899-deterministic-fixturevalue-0900-deterministic-fixturevalue-0901-deterministic-fixturevalue-0902-deterministic-fixturevalue-0904-deterministic-fixturevalue-0905-deterministic-fixturevalue-0906-deterministic-fixturevalue-0907-deterministic-fixturevalue-0908-deterministic-fixturevalue-0909-deterministic-fixturevalue-0911-deterministic-fixturevalue-0912-deterministic-fixturevalue-0913-deterministic-fixturevalue-0914-deterministic-fixturevalue-0915-deterministic-fixturevalue-0916-deterministic-fixturevalue-0918-deterministic-fixturevalue-0919-deterministic-fixturevalue-0920-deterministic-fixturevalue-0921-deterministic-fixturevalue-0922-deterministic-fixturevalue-0923-deterministic-fixturevalue-0925-deterministic-fixturevalue-0926-deterministic-fixturevalue-0927-deterministic-fixturevalue-0928-deterministic-fixturevalue-0929-deterministic-fixturevalue-0930-deterministic-fixturevalue-0932-deterministic-fixturevalue-0933-deterministic-fixturevalue-0934-deterministic-fixturevalue-0935-deterministic-fixturevalue-0936-deterministic-fixturevalue-0937-deterministic-fixturevalue-0939-deterministic-fixturevalue-0940-deterministic-fixturevalue-0941-deterministic-fixturevalue-0942-deterministic-fixturevalue-0943-deterministic-fixturevalue-0944-deterministic-fixturevalue-0946-deterministic-fixturevalue-0947-deterministic-fixturevalue-0948-deterministic-fixturevalue-0949-deterministic-fixturevalue-0950-deterministic-fixturevalue-0951-deterministic-fixturevalue-0953-deterministic-fixturevalue-0954-deterministic-fixturevalue-0955-deterministic-fixturevalue-0956-deterministic-fixturevalue-0957-deterministic-fixturevalue-0958-deterministic-fixturevalue-0960-deterministic-fixturevalue-0961-deterministic-fixturevalue-0962-deterministic-fixturevalue-0963-deterministic-fixturevalue-0964-deterministic-fixturevalue-0965-deterministic-fixturevalue-0967-deterministic-fixturevalue-0968-deterministic-fixturevalue-0969-deterministic-fixturevalue-0970-deterministic-fixturevalue-0971-deterministic-fixturevalue-0972-deterministic-fixturevalue-0974-deterministic-fixturevalue-0975-deterministic-fixturevalue-0976-deterministic-fixturevalue-0977-deterministic-fixturevalue-0978-deterministic-fixturevalue-0979-deterministic-fixturevalue-0981-deterministic-fixturevalue-0982-deterministic-fixturevalue-0983-deterministic-fixturevalue-0984-deterministic-fixturevalue-0985-deterministic-fixturevalue-0986-deterministic-fixturevalue-0988-deterministic-fixturevalue-0989-deterministic-fixturevalue-0990-deterministic-fixturevalue-0991-deterministic-fixturevalue-0992-deterministic-fixturevalue-0993-deterministic-fixturevalue-0995-deterministic-fixturevalue-0996-deterministic-fixturevalue-0997-deterministic-fixturevalue-0998-deterministic-fixturevalue-0999-deterministic-fixturevalue-1000-deterministic-fixturevalue-1002-deterministic-fixturevalue-1003-deterministic-fixturevalue-1004-deterministic-fixturevalue-1005-deterministic-fixturevalue-1006-deterministic-fixturevalue-1007-deterministic-fixturevalue-1009-deterministic-fixturevalue-1010-deterministic-fixturevalue-1011-deterministic-fixturevalue-1012-deterministic-fixturevalue-1013-deterministic-fixturevalue-1014-deterministic-fixturevalue-1016-deterministic-fixturevalue-1017-deterministic-fixturevalue-1018-deterministic-fixturevalue-1019-deterministic-fixturevalue-1020-deterministic-fixturevalue-1021-deterministic-fixturevalue-1023-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHJJ HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH#F$I$ɒ$$ddd2d2D2D2D"D"L"L"L&L&H&H&H$H$I$I$ɒ$$ddd2d2D2D2D"D"L"L"L&L&H&H&H$H$I$I@ @@ @!@@@ @@ @!@@ H@ @@ h/FAG0H @0H 01I@ KPC2K`3L @3AM +X4NC pNA#4ҋ`Nvq#8 bTv #@c`w"# efw$$Hfl#x&1$L" hrCx(a$P2ixy*$TBl~z,$P nz.$\ro{0Q%` qh{0()H8MRld{h)KDRp$|4)NP St|6)Q\MS||8)TtSD}:)W } *Z T~> *]MT d~@I*`T"~D!i*fT#$)*i8Wpب;L@X;NHBYPPZ=RX[=T`B\ >Vp]DPXx]h?Z^?\B_ @^` E B L'RX'b Úd!(r|Q(ق (ݐ#(ࢍD( + d) ҍѤq) ěġ)  ,$NX dI,'XĄi,*$,-YЄ ,0NY",3YD$,6Y& -90H )-<@NZ`d*(@-?PZxć.0-E[52Od#56APC68QpCc6:Qt7qT|@Æ8@T9BтUDBV":FaW2@c:H{ %r1|ȍ%D t|%uA } & wQ)} &dxYI~& z`&$ }i&~q(N' yɀ4'D@'Ť$H*l0U%J*o@U&LB+r@V'N+u`(+xpV)(RB+{V*XT+~@W+pV+W-+.\B+W/^@b)` &d*a ,Af1*b"!2DBha*c0!8dCj*dB!>l*eR"DDn*fb"JDp!+f"PEr+g"V$Et+h#\Dx+i􄈕-䈖-[E-[ +. \ *.O\ eJ.$Ŋ!j.0\%.<]1.HO]勤9.T!A +/"D$ddd2d2D2D2D"D"L"L"L&L&H&H&H$H$I$I$ɒ$$ddd2d2D2D2D"D"L"L"L&L&H&H&H$H$I$I$ɒ$$d @@ @H@ @@ @!@@@ @@ @!@@  rh4yQOk5zOn{OqD5|Otd5}Pw5~APzP}5ހP6߂Q$61Q#D6@4b`h4` f@5!a@s @6QalT 7and %A8"apt EA9Rar &eA:b FA;Abv f=qbz A>Bb| A?rbr",&L2r$%,&GXLbr&5,&gXLr*E,&XL„s,U,&§Xs.e,&M"s0u,&XMRs,&YMt4,&'Yt6,&GYMt8,&ԇY7퐡Sǰ7S"ʳ8T$ж$81T&#ӹaT(Cּ8TPٿ8T,c8!U0sQU29U$9UAe!FCLqe!CMe!CNBe"DOrf"P1f#"EDQaf0"&eDS2f C"FDTbf4S"fDUfDs"DVQgT"'[OvR-''[wV-'OwX-'h[Ow-'[OBw\-( +[rx^-( [Px`-(P҉xb.(\Px .(H\Pbxf5.(h\aQ3⅑Q +C6QP6Qc6!Rs6QR$7花RD7Rd7S7ASqS!%B@1c#!Aac3!eBBc@!BC2c$S!FBDbc@c!̆BE!dTs!ϦFQdd!CH"dt!%CIRd!ECJd!&KMBt:,&קYMrt,'YNu@,'Y҆uB-'NuD-''ZN2u -'GZNbuH5-'gZvJE-'ZN‡vLU-'OvNu-'ZORv-6D9U8d9V:AV<ڤ9qV9V@:WB1WF#D:aW0d:WJC:WLS :EWgd"%EX"gp"EEYRg" Zh"E[Ah"gE\qh"E^Bh"E_rh#`i$# FFabi0##fFbixhE.(PŠylU.("\Py`.(%\Q"ypu.((\Ryr.(+]Qzt.(.Qzv.(1H]Qz.(4h]Qzz.(7]rz|.):R{~.)=]$ 2$ 2 22"" " " & &&&$$ $ $ $ $ $ $ $ 2$ 2 22"" " " & &&&$$ $ $ $ $ $ $ $ @ @@ H@ @@@ @ cM1cNҕ2P2(dQ2 2HdRbE2܈dS–U2Te2dU"p2dVR2eW2(eXCV7Kos7koZ7 Cӧ\8 + ps^8 )p3`&8 Ipc68+ip3f'F8K cèh7V8 pjGp8pSP8"={={={={={>|>|>|>>|>|`3,ff#v32fg S35 gh +38Igi 3;!igj 3>$gkC3A*gls3D- m 3G0gnӛ4J3 ho$4MF9F+r3ë'P9I rc7f9Lkrœ#Pv9OrSg9R +#w9U *sS9[ Jsʃ9^+js˳9aK +s9dks9g sC~??~?~?~??????2Y◲2 he[2e\B2e]r2^Ә3 f_3)f`3&3 Ifac63# ifb +F3& cÙ V3)fd +qng8% +#pw8(JqSr8+ jqt8.KqC81kqsx84 +C|8: qsӪ~9= +r&9@Jrc69C +>|>|>>}>}>}>>}>}?~?6)hq3&&4S9Ihrc(64V< s*V4Y?ht`4\Bhu#.v4_EhvS24bK w44eN)ix4hQIiy84kTiiz:4nWi|Cӭ:j +ts:m*tУ3&:p +c6:s jt@:v tîV:| tf: t#: +u: *u: Ju: ??HHHHHHHHHHHHHHHHHHHHHHHHxwHHHHHHHHHHHHHHHH HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH@ @@ @@ @@ @@ aaaaaAaa`a!aaaaaaaAaa`a!aaaaaaaAaa`a!aaaaaaaaaAaa`a!aaaaaaaAaa`a!aaaaaaaAaa`a!aaaaaaaHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH  greenblueredHHHHHHHHHHHHHHHHHHHHHHHHHHHH    HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH@@@@@@@@@ @    +@ + + + @    @    @   @@@@@@@@@@@@@@@@@@ @   !@!!!"@"""#@###$@$$$%@%%%&@&&&'@'''(@((()@)))*@***+@+++,@,,,-@---.@.../@///0@0001@1112@2223@3334@4445@5556@6667@7778@8889@999:@:::;@;;;<@<<<=@===>@>>>?@???@@@@@A@AAAB@BBBC@CCCD@DDDE@EEEF@FFFG@GGGH@HHHI@IIIJ@JJJK@KKKL@LLLM@MMMN@NNNO@OOOP@PPPQ@QQQR@RRRS@SSST@TTTU@UUUV@VVVW@WWWX@XXXY@YYYZ@ZZZ[@[[[\@\\\]@]]]^@^^^_@___`@```a@aaab@bbbc@cccd@ddde@eeef@fffg@gggh@hhhi@iiij@jjjk@kkkl@lllm@mmmn@nnno@ooop@pppq@qqqr@rrrs@ssst@tttu@uuuv@vvvw@wwwx@xxxy@yyyz@zzz{@{{{|@|||}@}}}~@~~~@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1024-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1025-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1026-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1027-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1028-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1029-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1030-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1031-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1032-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1033-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1034-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1035-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1036-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1037-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1038-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1039-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1040-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1041-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1042-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1043-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1044-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1045-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1046-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1047-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1048-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1049-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1050-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1051-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1052-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1053-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1054-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1055-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1056-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1057-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1058-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1059-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1060-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1061-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1062-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1063-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1064-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1065-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1066-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1067-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1068-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1069-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1070-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1071-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1072-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1073-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1074-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1075-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1076-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1077-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1078-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1079-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1080-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1081-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1082-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1083-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1084-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1085-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1086-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1087-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1088-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1089-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1090-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1091-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1092-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1093-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1094-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1095-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1096-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1097-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1098-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1099-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1100-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1101-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1102-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1103-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1104-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1105-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1106-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1107-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1108-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1109-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1110-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1111-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1112-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1113-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1114-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1115-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1116-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1117-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1118-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1119-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1120-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1121-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1122-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1123-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1124-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1125-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1126-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1127-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1128-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1129-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1130-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1131-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1132-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1133-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1134-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1135-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1136-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1137-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1138-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1139-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1140-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1141-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1142-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1143-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1144-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1145-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1146-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1147-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1148-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1149-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1150-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1151-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1152-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1153-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1154-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1155-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1156-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1157-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1158-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1159-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1160-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1161-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1162-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1163-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1164-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1165-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1166-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1167-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1168-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1169-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1170-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1171-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1172-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1173-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1174-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1175-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1176-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1177-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1178-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1179-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1180-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1181-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1182-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1183-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1184-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1185-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1186-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1187-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1188-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1189-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1190-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1191-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1192-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1193-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1194-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1195-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1196-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1197-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1198-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1199-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1200-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1201-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1202-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1203-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1204-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1205-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1206-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1207-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1208-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1209-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1210-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1211-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1212-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1213-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1214-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1215-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1216-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1217-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1218-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1219-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1220-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1221-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1222-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1223-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1224-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1225-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1226-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1227-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1228-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1229-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1230-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1231-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1232-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1233-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1234-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1235-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1236-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1237-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1238-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1239-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1240-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1241-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1242-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1243-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1244-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1245-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1246-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1247-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1248-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1249-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1250-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1251-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1252-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1253-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1254-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1255-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1256-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1257-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1258-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1259-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1260-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1261-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1262-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1263-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1264-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1265-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1266-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1267-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1268-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1269-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1270-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1271-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1272-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1273-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1274-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1275-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1276-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1277-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1278-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1279-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1280-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1281-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1282-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1283-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1284-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1285-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1286-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1287-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1288-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1289-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1290-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1291-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1292-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1293-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1294-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1295-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1296-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1297-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1298-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1299-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1300-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1301-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1302-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1303-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1304-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1305-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1306-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1307-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1308-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1309-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1310-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1311-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1312-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1313-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1314-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1315-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1316-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1317-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1318-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1319-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1320-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1321-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1322-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1323-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1324-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1325-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1326-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1327-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1328-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1329-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1330-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1331-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1332-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1333-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1334-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1335-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1336-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1337-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1338-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1339-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1340-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1341-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1342-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1343-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1344-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1345-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1346-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1347-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1348-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1349-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1350-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1351-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1352-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1353-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1354-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1355-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1356-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1357-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1358-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1359-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1360-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1361-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1362-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1363-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1364-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1365-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1366-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1367-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1368-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1369-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1370-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1371-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1372-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1373-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1374-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1375-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1376-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1377-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1378-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1379-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1380-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1381-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1382-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1383-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1384-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1385-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1386-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1387-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1388-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1389-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1390-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1391-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1392-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1393-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1394-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1395-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1396-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1397-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1398-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1399-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1400-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1401-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1402-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1403-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1404-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1405-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1406-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1407-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1408-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1409-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1410-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1411-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1412-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1413-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1414-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1415-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1416-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1417-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1418-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1419-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1420-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1421-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1422-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1423-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1424-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1425-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1426-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1427-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1428-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1429-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1430-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1431-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1432-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1433-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1434-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1435-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1436-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1437-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1438-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1439-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1440-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1441-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1442-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1443-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1444-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1445-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1446-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1447-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1448-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1449-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1450-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1451-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1452-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1453-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1454-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1455-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1456-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1457-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1458-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1459-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1460-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1461-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1462-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1463-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1464-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1465-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1466-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1467-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1468-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1469-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1470-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1471-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1472-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1473-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1474-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1475-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1476-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1477-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1478-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1479-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1480-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1481-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1482-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1483-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1484-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1485-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1486-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1487-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1488-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1489-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1490-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1491-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1492-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1493-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1494-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1495-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1496-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1497-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1498-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1499-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1500-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1501-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1502-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1503-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1504-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1505-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1506-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1507-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1508-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1509-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1510-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1511-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1512-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1513-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1514-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1515-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1516-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1517-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1518-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1519-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1520-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1521-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1522-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1523-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1524-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1525-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1526-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1527-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1528-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1529-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1530-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1531-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1532-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1533-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1534-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1535-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1536-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1537-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1538-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1539-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1540-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1541-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1542-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1543-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1544-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1545-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1546-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1547-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1548-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1549-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1550-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1551-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1552-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1553-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1554-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1555-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1556-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1557-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1558-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1559-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1560-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1561-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1562-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1563-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1564-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1565-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1566-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1567-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1568-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1569-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1570-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1571-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1572-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1573-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1574-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1575-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1576-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1577-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1578-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1579-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1580-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1581-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1582-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1583-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1584-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1585-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1586-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1587-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1588-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1589-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1590-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1591-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1592-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1593-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1594-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1595-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1596-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1597-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1598-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1599-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1600-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1601-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1602-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1603-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1604-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1605-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1606-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1607-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1608-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1609-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1610-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1611-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1612-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1613-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1614-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1615-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1616-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1617-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1618-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1619-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1620-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1621-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1622-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1623-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1624-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1625-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1626-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1627-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1628-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1629-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1630-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1631-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1632-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1633-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1634-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1635-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1636-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1637-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1638-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1639-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1640-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1641-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1642-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1643-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1644-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1645-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1646-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1647-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1648-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1649-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1650-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1651-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1652-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1653-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1654-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1655-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1656-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1657-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1658-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1659-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1660-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1661-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1662-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1663-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1664-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1665-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1666-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1667-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1668-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1669-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1670-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1671-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1672-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1673-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1674-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1675-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1676-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1677-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1678-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1679-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1680-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1681-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1682-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1683-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1684-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1685-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1686-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1687-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1688-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1689-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1690-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1691-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1692-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1693-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1694-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1695-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1696-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1697-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1698-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1699-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1700-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1701-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1702-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1703-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1704-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1705-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1706-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1707-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1708-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1709-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1710-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1711-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1712-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1713-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1714-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1715-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1716-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1717-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1718-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1719-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1720-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1721-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1722-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1723-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1724-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1725-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1726-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1727-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1728-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1729-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1730-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1731-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1732-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1733-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1734-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1735-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1736-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1737-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1738-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1739-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1740-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1741-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1742-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1743-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1744-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1745-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1746-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1747-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1748-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1749-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1750-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1751-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1752-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1753-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1754-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1755-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1756-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1757-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1758-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1759-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1760-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1761-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1762-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1763-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1764-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1765-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1766-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1767-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1768-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1769-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1770-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1771-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1772-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1773-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1774-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1775-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1776-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1777-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1778-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1779-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1780-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1781-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1782-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1783-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1784-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1785-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1786-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1787-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1788-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1789-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1790-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1791-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1792-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1793-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1794-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1795-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1796-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1797-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1798-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1799-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1800-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1801-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1802-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1803-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1804-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1805-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1806-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1807-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1808-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1809-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1810-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1811-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1812-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1813-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1814-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1815-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1816-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1817-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1818-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1819-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1820-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1821-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1822-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1823-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1824-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1825-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1826-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1827-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1828-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1829-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1830-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1831-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1832-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1833-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1834-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1835-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1836-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1837-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1838-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1839-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1840-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1841-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1842-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1843-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1844-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1845-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1846-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1847-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1848-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1849-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1850-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1851-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1852-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1853-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1854-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1855-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1856-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1857-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1858-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1859-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1860-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1861-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1862-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1863-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1864-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1865-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1866-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1867-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1868-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1869-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1870-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1871-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1872-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1873-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1874-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1875-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1876-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1877-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1878-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1879-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1880-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1881-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1882-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1883-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1884-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1885-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1886-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1887-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1888-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1889-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1890-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1891-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1892-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1893-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1894-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1895-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1896-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1897-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1898-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1899-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1900-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1901-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1902-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1903-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1904-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1905-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1906-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1907-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1908-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1909-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1910-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1911-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1912-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1913-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1914-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1915-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1916-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1917-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1918-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1919-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1920-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1921-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1922-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1923-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1924-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1925-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1926-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1927-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1928-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1929-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1930-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1931-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1932-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1933-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1934-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1935-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1936-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1937-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1938-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1939-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1940-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1941-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1942-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1943-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1944-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1945-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1946-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1947-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1948-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1949-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1950-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1951-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1952-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1953-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1954-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1955-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1956-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1957-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1958-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1959-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1960-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1961-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1962-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1963-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1964-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1965-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1966-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1967-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1968-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1969-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1970-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1971-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1972-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1973-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1974-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1975-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1976-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1977-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1978-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1979-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1980-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1981-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1982-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1983-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1984-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1985-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1986-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1987-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1988-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1989-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1990-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1991-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1992-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1993-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1994-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1995-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1996-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1997-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1998-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1999-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2000-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2001-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2002-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2003-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2004-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2005-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2006-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2007-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2008-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2009-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2010-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2011-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2012-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2013-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2014-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2015-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2016-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2017-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2018-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2019-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2020-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2021-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2022-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2023-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2024-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2025-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2026-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2027-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2028-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2029-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2030-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2031-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2032-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2033-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2034-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2035-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2036-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2037-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2038-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2039-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2040-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2041-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2042-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2043-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2044-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2045-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2046-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2047-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH $@ d@@@$$A,dA4A?%HA eHCHE+HG;%IIKeIK[IMkIO{%JQeJSJUJW%KYeK[ۥK]K_%La eLcLe+Lg;%MiKeMk[MmkMo{%NqeNsNuNw%OyeO{ۥO}ObPPQ6Q6$mD64mɣFR6DmHS6S6+GPre+/@Gr+kGr%,/Gs,˹GsE-/Gs-Gt.+GPte.KH. +k Ht %/0Hu `e: +#0(ve< S0)e$= 0*e*? 0%+e6@I0)e:pWz@pW{A qW'{C cqW7{Dk:“qWG{F:qXW{Iqe//m"/q(/y4/:/@/F/L/R/ G .U +gN ǵn! ''*U-G 0"3X'\K*;cX0^k2ʓXG_XWaBXgb J;#Xd R̃Xe+Z;ͳYgKYj jYǸkr;C(Yt|/t|/t|/t|/t|/u|/u|/u|/u|/v|/v|/-TfMm&F.Ug@ n ' +XaJ XiLҺ8XlM:DXqOPXuP\XyRhX|S:tX́UX̅V +X̉YX͍[g{X/#rw{^/ Sr{d/ r{j/+r{v/Kr{|/ks{/ss{/s{/s|/ t|/ 36&9(<*?,B0H2K4N6QT @YUŝgNLYǸXY̝;dYU繢pY՝G;|YYݝ.Yg;NYǻʻ`Y'msv|nУv|pv}q w}s+3w0}tkғwG}vwW}w wg}z#xp}|Sx}} Ճx///HHHHHHHHHHHHHHHHHHHHHHHHwwHHHHHHHHHHHHHHHH HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @@ @@ @@ @@ @@aaAaa`a!aaaaaaaAaa`a!aaaaaaaAaa`a!aaaaaaaAaAaa`a!aaaaaaaAaa`a!aaaaaaaAaa`a!aaaaaaaAaaHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH  greenblueredHHHHHHHHHHHHHHHHHHHHHHHHHHHH    HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ @    +@ + + + @    @    @   @@@@@@@@@@@@@@@@@@ @   !@!!!"@"""#@###$@$$$%@%%%&@&&&'@'''(@((()@)))*@***+@+++,@,,,-@---.@.../@///0@0001@1112@2223@3334@4445@5556@6667@7778@8889@999:@:::;@;;;<@<<<=@===>@>>>?@???@@@@@A@AAAB@BBBC@CCCD@DDDE@EEEF@FFFG@GGGH@HHHI@IIIJ@JJJK@KKKL@LLLM@MMMN@NNNO@OOOP@PPPQ@QQQR@RRRS@SSST@TTTU@UUUV@VVVW@WWWX@XXXY@YYYZ@ZZZ[@[[[\@\\\]@]]]^@^^^_@___`@```a@aaab@bbbc@cccd@ddde@eeef@fffg@gggh@hhhi@iiij@jjjk@kkkl@lllm@mmmn@nnno@ooop@pppq@qqqr@rrrs@ssst@tttu@uuuv@vvvw@wwwx@xxxy@yyyz@zzz{@{{{|@|||}@}}}~@~~~@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2048-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2049-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2050-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2051-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2052-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2053-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2054-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2055-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2056-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2057-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2058-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2059-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2060-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2061-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2062-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2063-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2064-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2065-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2066-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2067-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2068-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2069-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2070-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2071-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2072-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2073-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2074-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2075-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2076-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2077-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2078-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2079-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2080-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2081-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2082-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2083-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2084-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2085-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2086-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2087-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2088-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2089-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2090-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2091-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2092-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2093-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2094-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2095-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2096-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2097-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2098-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2099-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2100-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2101-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2102-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2103-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2104-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2105-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2106-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2107-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2108-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2109-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2110-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2111-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2112-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2113-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2114-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2115-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2116-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2117-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2118-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2119-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2120-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2121-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2122-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2123-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2124-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2125-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2126-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2127-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2128-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2129-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2130-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2131-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2132-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2133-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2134-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2135-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2136-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2137-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2138-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2139-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2140-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2141-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2142-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2143-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2144-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2145-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2146-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2147-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2148-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2149-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2150-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2151-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2152-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2153-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2154-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2155-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2156-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2157-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2158-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2159-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2160-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2161-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2162-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2163-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2164-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2165-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2166-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2167-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2168-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2169-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2170-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2171-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2172-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2173-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2174-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2175-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2176-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2177-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2178-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2179-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2180-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2181-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2182-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2183-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2184-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2185-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2186-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2187-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2188-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2189-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2190-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2191-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2192-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2193-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2194-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2195-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2196-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2197-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2198-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2199-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2200-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2201-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2202-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2203-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2204-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2205-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2206-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2207-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2208-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2209-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2210-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2211-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2212-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2213-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2214-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2215-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2216-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2217-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2218-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2219-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2220-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2221-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2222-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2223-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2224-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2225-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2226-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2227-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2228-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2229-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2230-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2231-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2232-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2233-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2234-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2235-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2236-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2237-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2238-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2239-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2240-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2241-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2242-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2243-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2244-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2245-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2246-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2247-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2248-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2249-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2250-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2251-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2252-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2253-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2254-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2255-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2256-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2257-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2258-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2259-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2260-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2261-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2262-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2263-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2264-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2265-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2266-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2267-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2268-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2269-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2270-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2271-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2272-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2273-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2274-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2275-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2276-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2277-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2278-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2279-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2280-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2281-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2282-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2283-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2284-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2285-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2286-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2287-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2288-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2289-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2290-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2291-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2292-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2293-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2294-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2295-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2296-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2297-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2298-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2299-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2300-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2301-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2302-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2303-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2304-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2305-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2306-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2307-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2308-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2309-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2310-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2311-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2312-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2313-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2314-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2315-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2316-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2317-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2318-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2319-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2320-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2321-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2322-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2323-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2324-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2325-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2326-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2327-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2328-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2329-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2330-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2331-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2332-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2333-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2334-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2335-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2336-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2337-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2338-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2339-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2340-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2341-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2342-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2343-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2344-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2345-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2346-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2347-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2348-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2349-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2350-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2351-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2352-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2353-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2354-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2355-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2356-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2357-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2358-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2359-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2360-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2361-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2362-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2363-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2364-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2365-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2366-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2367-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2368-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2369-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2370-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2371-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2372-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2373-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2374-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2375-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2376-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2377-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2378-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2379-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2380-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2381-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2382-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2383-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2384-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2385-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2386-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2387-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2388-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2389-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2390-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2391-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2392-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2393-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2394-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2395-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2396-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2397-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2398-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2399-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2400-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2401-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2402-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2403-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2404-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2405-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2406-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2407-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2408-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2409-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2410-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2411-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2412-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2413-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2414-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2415-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2416-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2417-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2418-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2419-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2420-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2421-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2422-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2423-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2424-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2425-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2426-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2427-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2428-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2429-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2430-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2431-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2432-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2433-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2434-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2435-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2436-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2437-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2438-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2439-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2440-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2441-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2442-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2443-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2444-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2445-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2446-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2447-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2448-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2449-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2450-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2451-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2452-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2453-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2454-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2455-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2456-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2457-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2458-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2459-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2460-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2461-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2462-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2463-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2464-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2465-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2466-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2467-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2468-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2469-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2470-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2471-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2472-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2473-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2474-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2475-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2476-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2477-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2478-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2479-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2480-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2481-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2482-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2483-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2484-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2485-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2486-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2487-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2488-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2489-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2490-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2491-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2492-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2493-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2494-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2495-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2496-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2497-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2498-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2499-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2500-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2501-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2502-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2503-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2504-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2505-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2506-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2507-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2508-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2509-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2510-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2511-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2512-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2513-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2514-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2515-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2516-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2517-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2518-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2519-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2520-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2521-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2522-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2523-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2524-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2525-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2526-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2527-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2528-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2529-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2530-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2531-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2532-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2533-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2534-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2535-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2536-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2537-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2538-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2539-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2540-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2541-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2542-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2543-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2544-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2545-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2546-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2547-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2548-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2549-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2550-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2551-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2552-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2553-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2554-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2555-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2556-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2557-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2558-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2559-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2560-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2561-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2562-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2563-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2564-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2565-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2566-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2567-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2568-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2569-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2570-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2571-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2572-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2573-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2574-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2575-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2576-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2577-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2578-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2579-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2580-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2581-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2582-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2583-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2584-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2585-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2586-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2587-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2588-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2589-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2590-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2591-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2592-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2593-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2594-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2595-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2596-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2597-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2598-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2599-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2600-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2601-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2602-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2603-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2604-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2605-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2606-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2607-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2608-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2609-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2610-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2611-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2612-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2613-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2614-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2615-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2616-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2617-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2618-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2619-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2620-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2621-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2622-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2623-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2624-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2625-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2626-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2627-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2628-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2629-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2630-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2631-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2632-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2633-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2634-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2635-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2636-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2637-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2638-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2639-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2640-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2641-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2642-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2643-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2644-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2645-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2646-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2647-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2648-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2649-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2650-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2651-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2652-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2653-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2654-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2655-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2656-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2657-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2658-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2659-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2660-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2661-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2662-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2663-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2664-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2665-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2666-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2667-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2668-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2669-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2670-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2671-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2672-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2673-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2674-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2675-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2676-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2677-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2678-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2679-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2680-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2681-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2682-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2683-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2684-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2685-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2686-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2687-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2688-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2689-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2690-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2691-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2692-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2693-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2694-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2695-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2696-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2697-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2698-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2699-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2700-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2701-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2702-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2703-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2704-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2705-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2706-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2707-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2708-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2709-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2710-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2711-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2712-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2713-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2714-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2715-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2716-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2717-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2718-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2719-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2720-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2721-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2722-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2723-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2724-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2725-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2726-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2727-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2728-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2729-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2730-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2731-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2732-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2733-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2734-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2735-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2736-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2737-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2738-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2739-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2740-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2741-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2742-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2743-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2744-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2745-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2746-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2747-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2748-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2749-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2750-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2751-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2752-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2753-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2754-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2755-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2756-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2757-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2758-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2759-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2760-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2761-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2762-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2763-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2764-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2765-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2766-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2767-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2768-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2769-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2770-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2771-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2772-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2773-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2774-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2775-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2776-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2777-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2778-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2779-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2780-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2781-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2782-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2783-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2784-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2785-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2786-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2787-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2788-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2789-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2790-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2791-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2792-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2793-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2794-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2795-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2796-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2797-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2798-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2799-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2800-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2801-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2802-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2803-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2804-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2805-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2806-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2807-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2808-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2809-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2810-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2811-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2812-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2813-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2814-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2815-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2816-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2817-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2818-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2819-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2820-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2821-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2822-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2823-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2824-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2825-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2826-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2827-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2828-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2829-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2830-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2831-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2832-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2833-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2834-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2835-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2836-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2837-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2838-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2839-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2840-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2841-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2842-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2843-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2844-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2845-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2846-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2847-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2848-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2849-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2850-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2851-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2852-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2853-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2854-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2855-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2856-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2857-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2858-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2859-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2860-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2861-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2862-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2863-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2864-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2865-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2866-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2867-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2868-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2869-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2870-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2871-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2872-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2873-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2874-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2875-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2876-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2877-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2878-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2879-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2880-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2881-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2882-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2883-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2884-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2885-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2886-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2887-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2888-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2889-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2890-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2891-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2892-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2893-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2894-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2895-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2896-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2897-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2898-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2899-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2900-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2901-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2902-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2903-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2904-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2905-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2906-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2907-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2908-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2909-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2910-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2911-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2912-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2913-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2914-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2915-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2916-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2917-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2918-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2919-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2920-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2921-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2922-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2923-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2924-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2925-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2926-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2927-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2928-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2929-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2930-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2931-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2932-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2933-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2934-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2935-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2936-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2937-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2938-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2939-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2940-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2941-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2942-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2943-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2944-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2945-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2946-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2947-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2948-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2949-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2950-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2951-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2952-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2953-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2954-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2955-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2956-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2957-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2958-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2959-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2960-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2961-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2962-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2963-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2964-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2965-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2966-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2967-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2968-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2969-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2970-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2971-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2972-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2973-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2974-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2975-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2976-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2977-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2978-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2979-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2980-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2981-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2982-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2983-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2984-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2985-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2986-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2987-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2988-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2989-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2990-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2991-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2992-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2993-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2994-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2995-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2996-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2997-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2998-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2999-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3000-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3001-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3002-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3003-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3004-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3005-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3006-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3007-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3008-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3009-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3010-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3011-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3012-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3013-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3014-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3015-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3016-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3017-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3018-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3019-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3020-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3021-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3022-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3023-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3024-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3025-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3026-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3027-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3028-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3029-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3030-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3031-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3032-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3033-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3034-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3035-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3036-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3037-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3038-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3039-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3040-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3041-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3042-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3043-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3044-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3045-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3046-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3047-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3048-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3049-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3050-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3051-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3052-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3053-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3054-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3055-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3056-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3057-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3058-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3059-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3060-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3061-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3062-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3063-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3064-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3065-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3066-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3067-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3068-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3069-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3070-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3071-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH (8HXhx + + Ȉ ؈ (8HXhxȉ؉ +)*9:IJYZijyz + ɘ ٘   +)*9:IJYZijyzəٙ +;K[k{ + ˸ ۸븨 +;K[k{˹۹빩@@AAB(BC8CDHDEXEFhFGxGHHIIJJKKLȌLM،MNNOOPPQQR(RS8STHTUXUVhVWxWXXYYZZ[[\ȍ\]؍]^^__ @ +A)B*9C:IDJYEZiFjyGzHIJKɜLٜMNO P +Q)R*9S:ITJYUZiVjyWzXYZ[ɝ\ٝ]^_@ AB+C;DKE[FkG{HIJKL˼MۼN뼬OP QR+S;TKU[VkW{XYZ[\˽]۽^뽭_  !!"("#8#$H$%X%&h&'x'(())**++,Ȋ,-؊-..//00112(23834H45X56h67x78899::;;<ȋ<=؋=>>?? +!)"*9#:I$JY%Zi&jy'z()*+ɚ,ٚ-./ 0 +1)2*93:I4JY5Zi6jy7z89:;ɛ<ٛ=>? !"+#;$K%[&k'{()*+,˺-ۺ.뺪/0 12+3;4K5[6k7{89:;<˻=ۻ>뻫?``aab(bc8cdHdeXefhfgxghhiijjkklȎlm؎mnnooppqqr(rs8stHtuXuvhvwxwxxyyzz{{|ȏ|}؏}~~ ` +a)b*9c:IdJYeZifjygzhijkɞlٞmno p +q)r*9s:ItJYuZivjywzxyz{ɟ|ٟ}~` ab+c;dKe[fkg{hijkl˾m۾n뾮op qr+s;tKu[vkw{xyz{|˿}ۿ~뿯HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH g g HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH$Ddd$$Dd$Dd$Ddd$$Dd$Dd$Ddd $ $ D d  +$ +D +d + + + + + + $ D d d  $ $ D d  $ D d $Ddd$$Ddvalue-2048-deterministic-fixturevalue-2049-deterministic-fixturevalue-2050-deterministic-fixturevalue-2052-deterministic-fixturevalue-2053-deterministic-fixturevalue-2054-deterministic-fixturevalue-2055-deterministic-fixturevalue-2056-deterministic-fixturevalue-2057-deterministic-fixturevalue-2059-deterministic-fixturevalue-2060-deterministic-fixturevalue-2061-deterministic-fixturevalue-2062-deterministic-fixturevalue-2063-deterministic-fixturevalue-2064-deterministic-fixturevalue-2066-deterministic-fixturevalue-2067-deterministic-fixturevalue-2068-deterministic-fixturevalue-2069-deterministic-fixturevalue-2070-deterministic-fixturevalue-2071-deterministic-fixturevalue-2073-deterministic-fixturevalue-2074-deterministic-fixturevalue-2075-deterministic-fixturevalue-2076-deterministic-fixturevalue-2077-deterministic-fixturevalue-2078-deterministic-fixturevalue-2080-deterministic-fixturevalue-2081-deterministic-fixturevalue-2082-deterministic-fixturevalue-2083-deterministic-fixturevalue-2084-deterministic-fixturevalue-2085-deterministic-fixturevalue-2087-deterministic-fixturevalue-2088-deterministic-fixturevalue-2089-deterministic-fixturevalue-2090-deterministic-fixturevalue-2091-deterministic-fixturevalue-2092-deterministic-fixturevalue-2094-deterministic-fixturevalue-2095-deterministic-fixturevalue-2096-deterministic-fixturevalue-2097-deterministic-fixturevalue-2098-deterministic-fixturevalue-2099-deterministic-fixturevalue-2101-deterministic-fixturevalue-2102-deterministic-fixturevalue-2103-deterministic-fixturevalue-2104-deterministic-fixturevalue-2105-deterministic-fixturevalue-2106-deterministic-fixturevalue-2108-deterministic-fixturevalue-2109-deterministic-fixturevalue-2110-deterministic-fixturevalue-2111-deterministic-fixturevalue-2112-deterministic-fixturevalue-2113-deterministic-fixturevalue-2115-deterministic-fixturevalue-2116-deterministic-fixturevalue-2117-deterministic-fixturevalue-2118-deterministic-fixturevalue-2119-deterministic-fixturevalue-2120-deterministic-fixturevalue-2122-deterministic-fixturevalue-2123-deterministic-fixturevalue-2124-deterministic-fixturevalue-2125-deterministic-fixturevalue-2126-deterministic-fixturevalue-2127-deterministic-fixturevalue-2129-deterministic-fixturevalue-2130-deterministic-fixturevalue-2131-deterministic-fixturevalue-2132-deterministic-fixturevalue-2133-deterministic-fixturevalue-2134-deterministic-fixturevalue-2136-deterministic-fixturevalue-2137-deterministic-fixturevalue-2138-deterministic-fixturevalue-2139-deterministic-fixturevalue-2140-deterministic-fixturevalue-2141-deterministic-fixturevalue-2143-deterministic-fixturevalue-2144-deterministic-fixturevalue-2145-deterministic-fixturevalue-2146-deterministic-fixturevalue-2147-deterministic-fixturevalue-2148-deterministic-fixturevalue-2150-deterministic-fixturevalue-2151-deterministic-fixturevalue-2152-deterministic-fixturevalue-2153-deterministic-fixturevalue-2154-deterministic-fixturevalue-2155-deterministic-fixturevalue-2157-deterministic-fixturevalue-2158-deterministic-fixturevalue-2159-deterministic-fixturevalue-2160-deterministic-fixturevalue-2161-deterministic-fixturevalue-2162-deterministic-fixturevalue-2164-deterministic-fixturevalue-2165-deterministic-fixturevalue-2166-deterministic-fixturevalue-2167-deterministic-fixturevalue-2168-deterministic-fixturevalue-2169-deterministic-fixturevalue-2171-deterministic-fixturevalue-2172-deterministic-fixturevalue-2173-deterministic-fixturevalue-2174-deterministic-fixturevalue-2175-deterministic-fixture$$Dd$Dd$Ddd$$Dd$Dd$Ddd$$Dd $ D d  +$ +D +d +d + + + + + $ $ D d  $ D d  $ D d d $$Dd$Ddvalue-2176-deterministic-fixturevalue-2178-deterministic-fixturevalue-2179-deterministic-fixturevalue-2180-deterministic-fixturevalue-2181-deterministic-fixturevalue-2182-deterministic-fixturevalue-2183-deterministic-fixturevalue-2185-deterministic-fixturevalue-2186-deterministic-fixturevalue-2187-deterministic-fixturevalue-2188-deterministic-fixturevalue-2189-deterministic-fixturevalue-2190-deterministic-fixturevalue-2192-deterministic-fixturevalue-2193-deterministic-fixturevalue-2194-deterministic-fixturevalue-2195-deterministic-fixturevalue-2196-deterministic-fixturevalue-2197-deterministic-fixturevalue-2199-deterministic-fixturevalue-2200-deterministic-fixturevalue-2201-deterministic-fixturevalue-2202-deterministic-fixturevalue-2203-deterministic-fixturevalue-2204-deterministic-fixturevalue-2206-deterministic-fixturevalue-2207-deterministic-fixturevalue-2208-deterministic-fixturevalue-2209-deterministic-fixturevalue-2210-deterministic-fixturevalue-2211-deterministic-fixturevalue-2213-deterministic-fixturevalue-2214-deterministic-fixturevalue-2215-deterministic-fixturevalue-2216-deterministic-fixturevalue-2217-deterministic-fixturevalue-2218-deterministic-fixturevalue-2220-deterministic-fixturevalue-2221-deterministic-fixturevalue-2222-deterministic-fixturevalue-2223-deterministic-fixturevalue-2224-deterministic-fixturevalue-2225-deterministic-fixturevalue-2227-deterministic-fixturevalue-2228-deterministic-fixturevalue-2229-deterministic-fixturevalue-2230-deterministic-fixturevalue-2231-deterministic-fixturevalue-2232-deterministic-fixturevalue-2234-deterministic-fixturevalue-2235-deterministic-fixturevalue-2236-deterministic-fixturevalue-2237-deterministic-fixturevalue-2238-deterministic-fixturevalue-2239-deterministic-fixturevalue-2241-deterministic-fixturevalue-2242-deterministic-fixturevalue-2243-deterministic-fixturevalue-2244-deterministic-fixturevalue-2245-deterministic-fixturevalue-2246-deterministic-fixturevalue-2248-deterministic-fixturevalue-2249-deterministic-fixturevalue-2250-deterministic-fixturevalue-2251-deterministic-fixturevalue-2252-deterministic-fixturevalue-2253-deterministic-fixturevalue-2255-deterministic-fixturevalue-2256-deterministic-fixturevalue-2257-deterministic-fixturevalue-2258-deterministic-fixturevalue-2259-deterministic-fixturevalue-2260-deterministic-fixturevalue-2262-deterministic-fixturevalue-2263-deterministic-fixturevalue-2264-deterministic-fixturevalue-2265-deterministic-fixturevalue-2266-deterministic-fixturevalue-2267-deterministic-fixturevalue-2269-deterministic-fixturevalue-2270-deterministic-fixturevalue-2271-deterministic-fixturevalue-2272-deterministic-fixturevalue-2273-deterministic-fixturevalue-2274-deterministic-fixturevalue-2276-deterministic-fixturevalue-2277-deterministic-fixturevalue-2278-deterministic-fixturevalue-2279-deterministic-fixturevalue-2280-deterministic-fixturevalue-2281-deterministic-fixturevalue-2283-deterministic-fixturevalue-2284-deterministic-fixturevalue-2285-deterministic-fixturevalue-2286-deterministic-fixturevalue-2287-deterministic-fixturevalue-2288-deterministic-fixturevalue-2290-deterministic-fixturevalue-2291-deterministic-fixturevalue-2292-deterministic-fixturevalue-2293-deterministic-fixturevalue-2294-deterministic-fixturevalue-2295-deterministic-fixturevalue-2297-deterministic-fixturevalue-2298-deterministic-fixturevalue-2299-deterministic-fixturevalue-2300-deterministic-fixturevalue-2301-deterministic-fixturevalue-2302-deterministic-fixture$Dd$Dd$DDd$Dd$Dd$DDd$Dd $ D d  +$ +D +D +d + + + + +  $ D d  $ D d  $ D D d $Dd$Ddvalue-2304-deterministic-fixturevalue-2305-deterministic-fixturevalue-2306-deterministic-fixturevalue-2307-deterministic-fixturevalue-2308-deterministic-fixturevalue-2309-deterministic-fixturevalue-2311-deterministic-fixturevalue-2312-deterministic-fixturevalue-2313-deterministic-fixturevalue-2314-deterministic-fixturevalue-2315-deterministic-fixturevalue-2316-deterministic-fixturevalue-2318-deterministic-fixturevalue-2319-deterministic-fixturevalue-2320-deterministic-fixturevalue-2321-deterministic-fixturevalue-2322-deterministic-fixturevalue-2323-deterministic-fixturevalue-2325-deterministic-fixturevalue-2326-deterministic-fixturevalue-2327-deterministic-fixturevalue-2328-deterministic-fixturevalue-2329-deterministic-fixturevalue-2330-deterministic-fixturevalue-2332-deterministic-fixturevalue-2333-deterministic-fixturevalue-2334-deterministic-fixturevalue-2335-deterministic-fixturevalue-2336-deterministic-fixturevalue-2337-deterministic-fixturevalue-2339-deterministic-fixturevalue-2340-deterministic-fixturevalue-2341-deterministic-fixturevalue-2342-deterministic-fixturevalue-2343-deterministic-fixturevalue-2344-deterministic-fixturevalue-2346-deterministic-fixturevalue-2347-deterministic-fixturevalue-2348-deterministic-fixturevalue-2349-deterministic-fixturevalue-2350-deterministic-fixturevalue-2351-deterministic-fixturevalue-2353-deterministic-fixturevalue-2354-deterministic-fixturevalue-2355-deterministic-fixturevalue-2356-deterministic-fixturevalue-2357-deterministic-fixturevalue-2358-deterministic-fixturevalue-2360-deterministic-fixturevalue-2361-deterministic-fixturevalue-2362-deterministic-fixturevalue-2363-deterministic-fixturevalue-2364-deterministic-fixturevalue-2365-deterministic-fixturevalue-2367-deterministic-fixturevalue-2368-deterministic-fixturevalue-2369-deterministic-fixturevalue-2370-deterministic-fixturevalue-2371-deterministic-fixturevalue-2372-deterministic-fixturevalue-2374-deterministic-fixturevalue-2375-deterministic-fixturevalue-2376-deterministic-fixturevalue-2377-deterministic-fixturevalue-2378-deterministic-fixturevalue-2379-deterministic-fixturevalue-2381-deterministic-fixturevalue-2382-deterministic-fixturevalue-2383-deterministic-fixturevalue-2384-deterministic-fixturevalue-2385-deterministic-fixturevalue-2386-deterministic-fixturevalue-2388-deterministic-fixturevalue-2389-deterministic-fixturevalue-2390-deterministic-fixturevalue-2391-deterministic-fixturevalue-2392-deterministic-fixturevalue-2393-deterministic-fixturevalue-2395-deterministic-fixturevalue-2396-deterministic-fixturevalue-2397-deterministic-fixturevalue-2398-deterministic-fixturevalue-2399-deterministic-fixturevalue-2400-deterministic-fixturevalue-2402-deterministic-fixturevalue-2403-deterministic-fixturevalue-2404-deterministic-fixturevalue-2405-deterministic-fixturevalue-2406-deterministic-fixturevalue-2407-deterministic-fixturevalue-2409-deterministic-fixturevalue-2410-deterministic-fixturevalue-2411-deterministic-fixturevalue-2412-deterministic-fixturevalue-2413-deterministic-fixturevalue-2414-deterministic-fixturevalue-2416-deterministic-fixturevalue-2417-deterministic-fixturevalue-2418-deterministic-fixturevalue-2419-deterministic-fixturevalue-2420-deterministic-fixturevalue-2421-deterministic-fixturevalue-2423-deterministic-fixturevalue-2424-deterministic-fixturevalue-2425-deterministic-fixturevalue-2426-deterministic-fixturevalue-2427-deterministic-fixturevalue-2428-deterministic-fixturevalue-2430-deterministic-fixturevalue-2431-deterministic-fixture$Dd$DDd$Dd$Dd$DDd$Dd$Dd $ D D d  + +$ +D +d + + + + + + $ D d  $ D D d   $ D d $Dd$DDdvalue-2432-deterministic-fixturevalue-2433-deterministic-fixturevalue-2434-deterministic-fixturevalue-2435-deterministic-fixturevalue-2437-deterministic-fixturevalue-2438-deterministic-fixturevalue-2439-deterministic-fixturevalue-2440-deterministic-fixturevalue-2441-deterministic-fixturevalue-2442-deterministic-fixturevalue-2444-deterministic-fixturevalue-2445-deterministic-fixturevalue-2446-deterministic-fixturevalue-2447-deterministic-fixturevalue-2448-deterministic-fixturevalue-2449-deterministic-fixturevalue-2451-deterministic-fixturevalue-2452-deterministic-fixturevalue-2453-deterministic-fixturevalue-2454-deterministic-fixturevalue-2455-deterministic-fixturevalue-2456-deterministic-fixturevalue-2458-deterministic-fixturevalue-2459-deterministic-fixturevalue-2460-deterministic-fixturevalue-2461-deterministic-fixturevalue-2462-deterministic-fixturevalue-2463-deterministic-fixturevalue-2465-deterministic-fixturevalue-2466-deterministic-fixturevalue-2467-deterministic-fixturevalue-2468-deterministic-fixturevalue-2469-deterministic-fixturevalue-2470-deterministic-fixturevalue-2472-deterministic-fixturevalue-2473-deterministic-fixturevalue-2474-deterministic-fixturevalue-2475-deterministic-fixturevalue-2476-deterministic-fixturevalue-2477-deterministic-fixturevalue-2479-deterministic-fixturevalue-2480-deterministic-fixturevalue-2481-deterministic-fixturevalue-2482-deterministic-fixturevalue-2483-deterministic-fixturevalue-2484-deterministic-fixturevalue-2486-deterministic-fixturevalue-2487-deterministic-fixturevalue-2488-deterministic-fixturevalue-2489-deterministic-fixturevalue-2490-deterministic-fixturevalue-2491-deterministic-fixturevalue-2493-deterministic-fixturevalue-2494-deterministic-fixturevalue-2495-deterministic-fixturevalue-2496-deterministic-fixturevalue-2497-deterministic-fixturevalue-2498-deterministic-fixturevalue-2500-deterministic-fixturevalue-2501-deterministic-fixturevalue-2502-deterministic-fixturevalue-2503-deterministic-fixturevalue-2504-deterministic-fixturevalue-2505-deterministic-fixturevalue-2507-deterministic-fixturevalue-2508-deterministic-fixturevalue-2509-deterministic-fixturevalue-2510-deterministic-fixturevalue-2511-deterministic-fixturevalue-2512-deterministic-fixturevalue-2514-deterministic-fixturevalue-2515-deterministic-fixturevalue-2516-deterministic-fixturevalue-2517-deterministic-fixturevalue-2518-deterministic-fixturevalue-2519-deterministic-fixturevalue-2521-deterministic-fixturevalue-2522-deterministic-fixturevalue-2523-deterministic-fixturevalue-2524-deterministic-fixturevalue-2525-deterministic-fixturevalue-2526-deterministic-fixturevalue-2528-deterministic-fixturevalue-2529-deterministic-fixturevalue-2530-deterministic-fixturevalue-2531-deterministic-fixturevalue-2532-deterministic-fixturevalue-2533-deterministic-fixturevalue-2535-deterministic-fixturevalue-2536-deterministic-fixturevalue-2537-deterministic-fixturevalue-2538-deterministic-fixturevalue-2539-deterministic-fixturevalue-2540-deterministic-fixturevalue-2542-deterministic-fixturevalue-2543-deterministic-fixturevalue-2544-deterministic-fixturevalue-2545-deterministic-fixturevalue-2546-deterministic-fixturevalue-2547-deterministic-fixturevalue-2549-deterministic-fixturevalue-2550-deterministic-fixturevalue-2551-deterministic-fixturevalue-2552-deterministic-fixturevalue-2553-deterministic-fixturevalue-2554-deterministic-fixturevalue-2556-deterministic-fixturevalue-2557-deterministic-fixturevalue-2558-deterministic-fixturevalue-2559-deterministic-fixture$DDd$Dd$Dd$DDd$Dd$Dd$DDd  $ D d  +$ +D +d + + + + + + $ D D d   $ D d  $ D d $DDd$Ddvalue-2560-deterministic-fixturevalue-2561-deterministic-fixturevalue-2563-deterministic-fixturevalue-2564-deterministic-fixturevalue-2565-deterministic-fixturevalue-2566-deterministic-fixturevalue-2567-deterministic-fixturevalue-2568-deterministic-fixturevalue-2570-deterministic-fixturevalue-2571-deterministic-fixturevalue-2572-deterministic-fixturevalue-2573-deterministic-fixturevalue-2574-deterministic-fixturevalue-2575-deterministic-fixturevalue-2577-deterministic-fixturevalue-2578-deterministic-fixturevalue-2579-deterministic-fixturevalue-2580-deterministic-fixturevalue-2581-deterministic-fixturevalue-2582-deterministic-fixturevalue-2584-deterministic-fixturevalue-2585-deterministic-fixturevalue-2586-deterministic-fixturevalue-2587-deterministic-fixturevalue-2588-deterministic-fixturevalue-2589-deterministic-fixturevalue-2591-deterministic-fixturevalue-2592-deterministic-fixturevalue-2593-deterministic-fixturevalue-2594-deterministic-fixturevalue-2595-deterministic-fixturevalue-2596-deterministic-fixturevalue-2598-deterministic-fixturevalue-2599-deterministic-fixturevalue-2600-deterministic-fixturevalue-2601-deterministic-fixturevalue-2602-deterministic-fixturevalue-2603-deterministic-fixturevalue-2605-deterministic-fixturevalue-2606-deterministic-fixturevalue-2607-deterministic-fixturevalue-2608-deterministic-fixturevalue-2609-deterministic-fixturevalue-2610-deterministic-fixturevalue-2612-deterministic-fixturevalue-2613-deterministic-fixturevalue-2614-deterministic-fixturevalue-2615-deterministic-fixturevalue-2616-deterministic-fixturevalue-2617-deterministic-fixturevalue-2619-deterministic-fixturevalue-2620-deterministic-fixturevalue-2621-deterministic-fixturevalue-2622-deterministic-fixturevalue-2623-deterministic-fixturevalue-2624-deterministic-fixturevalue-2626-deterministic-fixturevalue-2627-deterministic-fixturevalue-2628-deterministic-fixturevalue-2629-deterministic-fixturevalue-2630-deterministic-fixturevalue-2631-deterministic-fixturevalue-2633-deterministic-fixturevalue-2634-deterministic-fixturevalue-2635-deterministic-fixturevalue-2636-deterministic-fixturevalue-2637-deterministic-fixturevalue-2638-deterministic-fixturevalue-2640-deterministic-fixturevalue-2641-deterministic-fixturevalue-2642-deterministic-fixturevalue-2643-deterministic-fixturevalue-2644-deterministic-fixturevalue-2645-deterministic-fixturevalue-2647-deterministic-fixturevalue-2648-deterministic-fixturevalue-2649-deterministic-fixturevalue-2650-deterministic-fixturevalue-2651-deterministic-fixturevalue-2652-deterministic-fixturevalue-2654-deterministic-fixturevalue-2655-deterministic-fixturevalue-2656-deterministic-fixturevalue-2657-deterministic-fixturevalue-2658-deterministic-fixturevalue-2659-deterministic-fixturevalue-2661-deterministic-fixturevalue-2662-deterministic-fixturevalue-2663-deterministic-fixturevalue-2664-deterministic-fixturevalue-2665-deterministic-fixturevalue-2666-deterministic-fixturevalue-2668-deterministic-fixturevalue-2669-deterministic-fixturevalue-2670-deterministic-fixturevalue-2671-deterministic-fixturevalue-2672-deterministic-fixturevalue-2673-deterministic-fixturevalue-2675-deterministic-fixturevalue-2676-deterministic-fixturevalue-2677-deterministic-fixturevalue-2678-deterministic-fixturevalue-2679-deterministic-fixturevalue-2680-deterministic-fixturevalue-2682-deterministic-fixturevalue-2683-deterministic-fixturevalue-2684-deterministic-fixturevalue-2685-deterministic-fixturevalue-2686-deterministic-fixturevalue-2687-deterministic-fixture$Dd$Dd$DDd$Dd$Dd$DDd$Dd $ D d  +$ +D +D +d + + + + +  $ D d  $ D d  $ D D d $Dd$Ddvalue-2689-deterministic-fixturevalue-2690-deterministic-fixturevalue-2691-deterministic-fixturevalue-2692-deterministic-fixturevalue-2693-deterministic-fixturevalue-2694-deterministic-fixturevalue-2696-deterministic-fixturevalue-2697-deterministic-fixturevalue-2698-deterministic-fixturevalue-2699-deterministic-fixturevalue-2700-deterministic-fixturevalue-2701-deterministic-fixturevalue-2703-deterministic-fixturevalue-2704-deterministic-fixturevalue-2705-deterministic-fixturevalue-2706-deterministic-fixturevalue-2707-deterministic-fixturevalue-2708-deterministic-fixturevalue-2710-deterministic-fixturevalue-2711-deterministic-fixturevalue-2712-deterministic-fixturevalue-2713-deterministic-fixturevalue-2714-deterministic-fixturevalue-2715-deterministic-fixturevalue-2717-deterministic-fixturevalue-2718-deterministic-fixturevalue-2719-deterministic-fixturevalue-2720-deterministic-fixturevalue-2721-deterministic-fixturevalue-2722-deterministic-fixturevalue-2724-deterministic-fixturevalue-2725-deterministic-fixturevalue-2726-deterministic-fixturevalue-2727-deterministic-fixturevalue-2728-deterministic-fixturevalue-2729-deterministic-fixturevalue-2731-deterministic-fixturevalue-2732-deterministic-fixturevalue-2733-deterministic-fixturevalue-2734-deterministic-fixturevalue-2735-deterministic-fixturevalue-2736-deterministic-fixturevalue-2738-deterministic-fixturevalue-2739-deterministic-fixturevalue-2740-deterministic-fixturevalue-2741-deterministic-fixturevalue-2742-deterministic-fixturevalue-2743-deterministic-fixturevalue-2745-deterministic-fixturevalue-2746-deterministic-fixturevalue-2747-deterministic-fixturevalue-2748-deterministic-fixturevalue-2749-deterministic-fixturevalue-2750-deterministic-fixturevalue-2752-deterministic-fixturevalue-2753-deterministic-fixturevalue-2754-deterministic-fixturevalue-2755-deterministic-fixturevalue-2756-deterministic-fixturevalue-2757-deterministic-fixturevalue-2759-deterministic-fixturevalue-2760-deterministic-fixturevalue-2761-deterministic-fixturevalue-2762-deterministic-fixturevalue-2763-deterministic-fixturevalue-2764-deterministic-fixturevalue-2766-deterministic-fixturevalue-2767-deterministic-fixturevalue-2768-deterministic-fixturevalue-2769-deterministic-fixturevalue-2770-deterministic-fixturevalue-2771-deterministic-fixturevalue-2773-deterministic-fixturevalue-2774-deterministic-fixturevalue-2775-deterministic-fixturevalue-2776-deterministic-fixturevalue-2777-deterministic-fixturevalue-2778-deterministic-fixturevalue-2780-deterministic-fixturevalue-2781-deterministic-fixturevalue-2782-deterministic-fixturevalue-2783-deterministic-fixturevalue-2784-deterministic-fixturevalue-2785-deterministic-fixturevalue-2787-deterministic-fixturevalue-2788-deterministic-fixturevalue-2789-deterministic-fixturevalue-2790-deterministic-fixturevalue-2791-deterministic-fixturevalue-2792-deterministic-fixturevalue-2794-deterministic-fixturevalue-2795-deterministic-fixturevalue-2796-deterministic-fixturevalue-2797-deterministic-fixturevalue-2798-deterministic-fixturevalue-2799-deterministic-fixturevalue-2801-deterministic-fixturevalue-2802-deterministic-fixturevalue-2803-deterministic-fixturevalue-2804-deterministic-fixturevalue-2805-deterministic-fixturevalue-2806-deterministic-fixturevalue-2808-deterministic-fixturevalue-2809-deterministic-fixturevalue-2810-deterministic-fixturevalue-2811-deterministic-fixturevalue-2812-deterministic-fixturevalue-2813-deterministic-fixturevalue-2815-deterministic-fixture$Dd$Ddd$$Dd$Dd$Ddd$$Dd$Dd $ D d d  +$ +$ +D +d + + + + + + $ D d  $ D d d  $ $ D d $Dd$Dddvalue-2816-deterministic-fixturevalue-2817-deterministic-fixturevalue-2818-deterministic-fixturevalue-2819-deterministic-fixturevalue-2820-deterministic-fixturevalue-2822-deterministic-fixturevalue-2823-deterministic-fixturevalue-2824-deterministic-fixturevalue-2825-deterministic-fixturevalue-2826-deterministic-fixturevalue-2827-deterministic-fixturevalue-2829-deterministic-fixturevalue-2830-deterministic-fixturevalue-2831-deterministic-fixturevalue-2832-deterministic-fixturevalue-2833-deterministic-fixturevalue-2834-deterministic-fixturevalue-2836-deterministic-fixturevalue-2837-deterministic-fixturevalue-2838-deterministic-fixturevalue-2839-deterministic-fixturevalue-2840-deterministic-fixturevalue-2841-deterministic-fixturevalue-2843-deterministic-fixturevalue-2844-deterministic-fixturevalue-2845-deterministic-fixturevalue-2846-deterministic-fixturevalue-2847-deterministic-fixturevalue-2848-deterministic-fixturevalue-2850-deterministic-fixturevalue-2851-deterministic-fixturevalue-2852-deterministic-fixturevalue-2853-deterministic-fixturevalue-2854-deterministic-fixturevalue-2855-deterministic-fixturevalue-2857-deterministic-fixturevalue-2858-deterministic-fixturevalue-2859-deterministic-fixturevalue-2860-deterministic-fixturevalue-2861-deterministic-fixturevalue-2862-deterministic-fixturevalue-2864-deterministic-fixturevalue-2865-deterministic-fixturevalue-2866-deterministic-fixturevalue-2867-deterministic-fixturevalue-2868-deterministic-fixturevalue-2869-deterministic-fixturevalue-2871-deterministic-fixturevalue-2872-deterministic-fixturevalue-2873-deterministic-fixturevalue-2874-deterministic-fixturevalue-2875-deterministic-fixturevalue-2876-deterministic-fixturevalue-2878-deterministic-fixturevalue-2879-deterministic-fixturevalue-2880-deterministic-fixturevalue-2881-deterministic-fixturevalue-2882-deterministic-fixturevalue-2883-deterministic-fixturevalue-2885-deterministic-fixturevalue-2886-deterministic-fixturevalue-2887-deterministic-fixturevalue-2888-deterministic-fixturevalue-2889-deterministic-fixturevalue-2890-deterministic-fixturevalue-2892-deterministic-fixturevalue-2893-deterministic-fixturevalue-2894-deterministic-fixturevalue-2895-deterministic-fixturevalue-2896-deterministic-fixturevalue-2897-deterministic-fixturevalue-2899-deterministic-fixturevalue-2900-deterministic-fixturevalue-2901-deterministic-fixturevalue-2902-deterministic-fixturevalue-2903-deterministic-fixturevalue-2904-deterministic-fixturevalue-2906-deterministic-fixturevalue-2907-deterministic-fixturevalue-2908-deterministic-fixturevalue-2909-deterministic-fixturevalue-2910-deterministic-fixturevalue-2911-deterministic-fixturevalue-2913-deterministic-fixturevalue-2914-deterministic-fixturevalue-2915-deterministic-fixturevalue-2916-deterministic-fixturevalue-2917-deterministic-fixturevalue-2918-deterministic-fixturevalue-2920-deterministic-fixturevalue-2921-deterministic-fixturevalue-2922-deterministic-fixturevalue-2923-deterministic-fixturevalue-2924-deterministic-fixturevalue-2925-deterministic-fixturevalue-2927-deterministic-fixturevalue-2928-deterministic-fixturevalue-2929-deterministic-fixturevalue-2930-deterministic-fixturevalue-2931-deterministic-fixturevalue-2932-deterministic-fixturevalue-2934-deterministic-fixturevalue-2935-deterministic-fixturevalue-2936-deterministic-fixturevalue-2937-deterministic-fixturevalue-2938-deterministic-fixturevalue-2939-deterministic-fixturevalue-2941-deterministic-fixturevalue-2942-deterministic-fixturevalue-2943-deterministic-fixture$Ddd$$Dd$Dd$Ddd$$Dd$Dd$Ddd $ $ D d  +$ +D +d + + + + + + $ D d d  $ $ D d  $ D d $Ddd$$Ddvalue-2944-deterministic-fixturevalue-2945-deterministic-fixturevalue-2946-deterministic-fixturevalue-2948-deterministic-fixturevalue-2949-deterministic-fixturevalue-2950-deterministic-fixturevalue-2951-deterministic-fixturevalue-2952-deterministic-fixturevalue-2953-deterministic-fixturevalue-2955-deterministic-fixturevalue-2956-deterministic-fixturevalue-2957-deterministic-fixturevalue-2958-deterministic-fixturevalue-2959-deterministic-fixturevalue-2960-deterministic-fixturevalue-2962-deterministic-fixturevalue-2963-deterministic-fixturevalue-2964-deterministic-fixturevalue-2965-deterministic-fixturevalue-2966-deterministic-fixturevalue-2967-deterministic-fixturevalue-2969-deterministic-fixturevalue-2970-deterministic-fixturevalue-2971-deterministic-fixturevalue-2972-deterministic-fixturevalue-2973-deterministic-fixturevalue-2974-deterministic-fixturevalue-2976-deterministic-fixturevalue-2977-deterministic-fixturevalue-2978-deterministic-fixturevalue-2979-deterministic-fixturevalue-2980-deterministic-fixturevalue-2981-deterministic-fixturevalue-2983-deterministic-fixturevalue-2984-deterministic-fixturevalue-2985-deterministic-fixturevalue-2986-deterministic-fixturevalue-2987-deterministic-fixturevalue-2988-deterministic-fixturevalue-2990-deterministic-fixturevalue-2991-deterministic-fixturevalue-2992-deterministic-fixturevalue-2993-deterministic-fixturevalue-2994-deterministic-fixturevalue-2995-deterministic-fixturevalue-2997-deterministic-fixturevalue-2998-deterministic-fixturevalue-2999-deterministic-fixturevalue-3000-deterministic-fixturevalue-3001-deterministic-fixturevalue-3002-deterministic-fixturevalue-3004-deterministic-fixturevalue-3005-deterministic-fixturevalue-3006-deterministic-fixturevalue-3007-deterministic-fixturevalue-3008-deterministic-fixturevalue-3009-deterministic-fixturevalue-3011-deterministic-fixturevalue-3012-deterministic-fixturevalue-3013-deterministic-fixturevalue-3014-deterministic-fixturevalue-3015-deterministic-fixturevalue-3016-deterministic-fixturevalue-3018-deterministic-fixturevalue-3019-deterministic-fixturevalue-3020-deterministic-fixturevalue-3021-deterministic-fixturevalue-3022-deterministic-fixturevalue-3023-deterministic-fixturevalue-3025-deterministic-fixturevalue-3026-deterministic-fixturevalue-3027-deterministic-fixturevalue-3028-deterministic-fixturevalue-3029-deterministic-fixturevalue-3030-deterministic-fixturevalue-3032-deterministic-fixturevalue-3033-deterministic-fixturevalue-3034-deterministic-fixturevalue-3035-deterministic-fixturevalue-3036-deterministic-fixturevalue-3037-deterministic-fixturevalue-3039-deterministic-fixturevalue-3040-deterministic-fixturevalue-3041-deterministic-fixturevalue-3042-deterministic-fixturevalue-3043-deterministic-fixturevalue-3044-deterministic-fixturevalue-3046-deterministic-fixturevalue-3047-deterministic-fixturevalue-3048-deterministic-fixturevalue-3049-deterministic-fixturevalue-3050-deterministic-fixturevalue-3051-deterministic-fixturevalue-3053-deterministic-fixturevalue-3054-deterministic-fixturevalue-3055-deterministic-fixturevalue-3056-deterministic-fixturevalue-3057-deterministic-fixturevalue-3058-deterministic-fixturevalue-3060-deterministic-fixturevalue-3061-deterministic-fixturevalue-3062-deterministic-fixturevalue-3063-deterministic-fixturevalue-3064-deterministic-fixturevalue-3065-deterministic-fixturevalue-3067-deterministic-fixturevalue-3068-deterministic-fixturevalue-3069-deterministic-fixturevalue-3070-deterministic-fixturevalue-3071-deterministic-fixtureJJ HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH"D$ddd2d2D2D2D"D"L"L"L&L&H&H&H$H$I$I$ɒ$$ddd2d2D2D2D"D"L"L"L&L&H&H&H$H$I$I$ɒ$$!@@@ @@ @!@@ H@ @@ @@ @H@  H]xi ~ uX hP p ( +#  H+  +;0 hCPBFdȋ!ҌBGDdЋ!լGtd苡CHd!Hԣd CId!,I4d LCJdd8!lJdX CKdh!K$dxcF|LҖFڂ5PّӦF򈵍\QԶ@G +dG"5lޑֽG:5pQ#HR5|@HjͣH5HQ&#I5.JEH0JH2 JK40 JNXh@ JQ8 JT: JW< JZ H J`!B +Jc"D0 +JfXqe8!lR}eHڡ CSĉeX!Se`CT$ex!TTe CUe! UemCVe!VDeWte 6 dL  L6 R$M6 dMM*6!M0R%$N66)dN<NB61NHRK! ` K$ (K'&2K*X6J$K-FHb ,K0z 4K3Xfh$P8K6vȪ&pDK9(*TK?\KBXKLdSCMd!MekCNe!sN)e{CO5e!OAeơCPMe!PYeΡCQee!CRTcĵ ф5,񑴥#5 c5l5D،赬Qt#5c 5,4 LRdF#IFJiV$IHJl%JJov&J*LJrކ'JBNJuߖ(JZPJxক)KrRJ{+KVJƕ,KXJ֕-KZJ.L\hDP +f" p + f(D +%f" +1fX+D +=f" 3 If!;D0 UfX""h!CP af&!p mf*"("SD yf,""[İ fX5NXԩ9T Y=Z -!Y4(` M%ZdI8l &ZMHr)[ĪQ`x+[$x~-\TY .]]-1]a$O +6dKƖO"6lK֖O:⶟tKRPR6|KdPjⶠKP6KPⶡKR&Q6K6dQⶢKVQ6KfQ6K"Ddd2d2D2D2D"D"L"L"L&L&H&H&H$H$I$I$ɒ$$ddd2d2D2D2D"D"L"L"L&L&H&H&H$H$I$I$ɒ$$ddd@@ @!@@@ @@ @!@@ H@ @@ @iePxRjnԔ{>'jqդ|?Gktִ} AgRkw~AkBzԀBkrICl DSl҆!P30xf:(4x|R1Ԙ215JgR@ +ˇRT2U˧R3eJ"4u +RTRS5'S5JGS6 +T+V+ V +VI" + $+!V0&+)VI<(+1V H,S+œ9VT!+`0)+ȜQVIx2,+YV ĞDMpoBPN=uo\O=yphP>}ptQp2$RG>qb 0Sg>qDS>qTT>q)dUr"1tV>yT(ޞE+ݜVÍI+ ؁DSM+朙VFT+VHY+VIJ]+rR9V'?r够WG?rI崟XsQğY?sBYПZ?sra[?t\?tq(\t@]'@t2 L^G@ubDXQV +yT(BWy(CQXy(DQY +yT( DQZz(EQ\zԢ(E]Jz(Q^ +zT(FQ_z(GQ`zԣ( GaJz+WUV+"WeV+"*WuV+RȂW+:WԂ+BWGW+JWgW+ūW+BbWիW+rjWW+rW2 22"" " " & &&&$$ $ $ $ $ $ $ $ 2$ 2 22"" " " & &&&$$ $ $ $ $ $ $ $ 2$ 2 2@ H@ @@@ @!@ UޕB VǢРJW  +X Y(2٠%JZHb 5 +[hUɒ⠒E[ U\ `J]" u +^ĉTi-ЉVi-[܉X-[Zj-[j. +\^j.  bj .\d .\$j.*\@>>>?@???@@@@@A@AAAB@BBBC@CCCD@DDDE@EEEF@FFFG@GGGH@HHHI@IIIJ@JJJK@KKKL@LLLM@MMMN@NNNO@OOOP@PPPQ@QQQR@RRRS@SSST@TTTU@UUUV@VVVW@WWWX@XXXY@YYYZ@ZZZ[@[[[\@\\\]@]]]^@^^^_@___`@```a@aaab@bbbc@cccd@ddde@eeef@fffg@gggh@hhhi@iiij@jjjk@kkkl@lllm@mmmn@nnno@ooop@pppq@qqqr@rrrs@ssst@tttu@uuuv@vvvw@wwwx@xxxy@yyyz@zzz{@{{{|@|||}@}}}~@~~~@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3072-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3073-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3074-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3075-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3076-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3077-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3078-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3079-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3080-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3081-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3082-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3083-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3084-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3085-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3086-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3087-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3088-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3089-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3090-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3091-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3092-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3093-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3094-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3095-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3096-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3097-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3098-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3099-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3100-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3101-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3102-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3103-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3104-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3105-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3106-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3107-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3108-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3109-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3110-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3111-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3112-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3113-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3114-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3115-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3116-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3117-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3118-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3119-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3120-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3121-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3122-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3123-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3124-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3125-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3126-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3127-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3128-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3129-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3130-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3131-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3132-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3133-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3134-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3135-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3136-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3137-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3138-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3139-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3140-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3141-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3142-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3143-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3144-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3145-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3146-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3147-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3148-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3149-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3150-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3151-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3152-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3153-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3154-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3155-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3156-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3157-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3158-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3159-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3160-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3161-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3162-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3163-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3164-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3165-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3166-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3167-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3168-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3169-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3170-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3171-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3172-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3173-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3174-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3175-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3176-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3177-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3178-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3179-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3180-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3181-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3182-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3183-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3184-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3185-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3186-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3187-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3188-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3189-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3190-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3191-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3192-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3193-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3194-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3195-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3196-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3197-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3198-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3199-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3200-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3201-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3202-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3203-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3204-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3205-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3206-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3207-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3208-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3209-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3210-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3211-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3212-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3213-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3214-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3215-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3216-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3217-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3218-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3219-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3220-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3221-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3222-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3223-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3224-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3225-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3226-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3227-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3228-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3229-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3230-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3231-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3232-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3233-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3234-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3235-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3236-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3237-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3238-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3239-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3240-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3241-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3242-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3243-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3244-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3245-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3246-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3247-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3248-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3249-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3250-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3251-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3252-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3253-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3254-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3255-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3256-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3257-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3258-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3259-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3260-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3261-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3262-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3263-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3264-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3265-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3266-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3267-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3268-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3269-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3270-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3271-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3272-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3273-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3274-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3275-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3276-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3277-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3278-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3279-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3280-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3281-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3282-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3283-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3284-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3285-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3286-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3287-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3288-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3289-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3290-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3291-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3292-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3293-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3294-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3295-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3296-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3297-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3298-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3299-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3300-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3301-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3302-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3303-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3304-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3305-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3306-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3307-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3308-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3309-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3310-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3311-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3312-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3313-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3314-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3315-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3316-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3317-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3318-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3319-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3320-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3321-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3322-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3323-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3324-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3325-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3326-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3327-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3328-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3329-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3330-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3331-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3332-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3333-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3334-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3335-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3336-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3337-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3338-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3339-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3340-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3341-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3342-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3343-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3344-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3345-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3346-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3347-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3348-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3349-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3350-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3351-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3352-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3353-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3354-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3355-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3356-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3357-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3358-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3359-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3360-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3361-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3362-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3363-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3364-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3365-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3366-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3367-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3368-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3369-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3370-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3371-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3372-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3373-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3374-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3375-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3376-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3377-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3378-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3379-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3380-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3381-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3382-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3383-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3384-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3385-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3386-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3387-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3388-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3389-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3390-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3391-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3392-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3393-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3394-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3395-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3396-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3397-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3398-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3399-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3400-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3401-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3402-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3403-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3404-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3405-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3406-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3407-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3408-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3409-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3410-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3411-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3412-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3413-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3414-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3415-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3416-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3417-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3418-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3419-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3420-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3421-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3422-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3423-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3424-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3425-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3426-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3427-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3428-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3429-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3430-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3431-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3432-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3433-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3434-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3435-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3436-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3437-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3438-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3439-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3440-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3441-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3442-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3443-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3444-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3445-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3446-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3447-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3448-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3449-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3450-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3451-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3452-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3453-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3454-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3455-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3456-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3457-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3458-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3459-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3460-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3461-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3462-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3463-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3464-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3465-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3466-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3467-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3468-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3469-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3470-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3471-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3472-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3473-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3474-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3475-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3476-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3477-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3478-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3479-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3480-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3481-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3482-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3483-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3484-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3485-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3486-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3487-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3488-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3489-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3490-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3491-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3492-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3493-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3494-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3495-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3496-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3497-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3498-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3499-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3500-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3501-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3502-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3503-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3504-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3505-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3506-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3507-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3508-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3509-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3510-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3511-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3512-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3513-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3514-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3515-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3516-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3517-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3518-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3519-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3520-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3521-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3522-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3523-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3524-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3525-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3526-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3527-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3528-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3529-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3530-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3531-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3532-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3533-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3534-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3535-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3536-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3537-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3538-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3539-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3540-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3541-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3542-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3543-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3544-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3545-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3546-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3547-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3548-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3549-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3550-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3551-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3552-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3553-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3554-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3555-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3556-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3557-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3558-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3559-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3560-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3561-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3562-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3563-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3564-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3565-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3566-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3567-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3568-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3569-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3570-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3571-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3572-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3573-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3574-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3575-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3576-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3577-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3578-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3579-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3580-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3581-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3582-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3583-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3584-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3585-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3586-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3587-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3588-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3589-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3590-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3591-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3592-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3593-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3594-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3595-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3596-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3597-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3598-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3599-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3600-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3601-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3602-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3603-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3604-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3605-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3606-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3607-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3608-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3609-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3610-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3611-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3612-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3613-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3614-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3615-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3616-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3617-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3618-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3619-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3620-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3621-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3622-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3623-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3624-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3625-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3626-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3627-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3628-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3629-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3630-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3631-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3632-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3633-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3634-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3635-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3636-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3637-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3638-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3639-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3640-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3641-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3642-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3643-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3644-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3645-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3646-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3647-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3648-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3649-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3650-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3651-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3652-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3653-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3654-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3655-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3656-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3657-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3658-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3659-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3660-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3661-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3662-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3663-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3664-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3665-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3666-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3667-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3668-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3669-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3670-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3671-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3672-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3673-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3674-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3675-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3676-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3677-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3678-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3679-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3680-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3681-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3682-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3683-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3684-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3685-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3686-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3687-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3688-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3689-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3690-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3691-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3692-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3693-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3694-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3695-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3696-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3697-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3698-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3699-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3700-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3701-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3702-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3703-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3704-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3705-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3706-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3707-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3708-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3709-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3710-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3711-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3712-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3713-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3714-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3715-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3716-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3717-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3718-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3719-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3720-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3721-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3722-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3723-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3724-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3725-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3726-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3727-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3728-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3729-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3730-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3731-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3732-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3733-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3734-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3735-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3736-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3737-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3738-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3739-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3740-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3741-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3742-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3743-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3744-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3745-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3746-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3747-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3748-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3749-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3750-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3751-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3752-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3753-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3754-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3755-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3756-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3757-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3758-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3759-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3760-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3761-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3762-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3763-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3764-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3765-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3766-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3767-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3768-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3769-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3770-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3771-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3772-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3773-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3774-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3775-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3776-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3777-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3778-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3779-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3780-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3781-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3782-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3783-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3784-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3785-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3786-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3787-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3788-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3789-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3790-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3791-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3792-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3793-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3794-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3795-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3796-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3797-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3798-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3799-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3800-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3801-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3802-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3803-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3804-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3805-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3806-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3807-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3808-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3809-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3810-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3811-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3812-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3813-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3814-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3815-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3816-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3817-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3818-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3819-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3820-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3821-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3822-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3823-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3824-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3825-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3826-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3827-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3828-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3829-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3830-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3831-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3832-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3833-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3834-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3835-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3836-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3837-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3838-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3839-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3840-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3841-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3842-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3843-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3844-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3845-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3846-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3847-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3848-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3849-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3850-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3851-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3852-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3853-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3854-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3855-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3856-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3857-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3858-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3859-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3860-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3861-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3862-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3863-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3864-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3865-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3866-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3867-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3868-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3869-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3870-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3871-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3872-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3873-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3874-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3875-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3876-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3877-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3878-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3879-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3880-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3881-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3882-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3883-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3884-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3885-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3886-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3887-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3888-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3889-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3890-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3891-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3892-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3893-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3894-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3895-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3896-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3897-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3898-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3899-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3900-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3901-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3902-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3903-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3904-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3905-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3906-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3907-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3908-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3909-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3910-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3911-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3912-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3913-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3914-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3915-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3916-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3917-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3918-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3919-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3920-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3921-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3922-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3923-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3924-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3925-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3926-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3927-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3928-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3929-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3930-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3931-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3932-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3933-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3934-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3935-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3936-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3937-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3938-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3939-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3940-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3941-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3942-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3943-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3944-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3945-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3946-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3947-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3948-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3949-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3950-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3951-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3952-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3953-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3954-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3955-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3956-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3957-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3958-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3959-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3960-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3961-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3962-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3963-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3964-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3965-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3966-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3967-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3968-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3969-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3970-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3971-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3972-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3973-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3974-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3975-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3976-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3977-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3978-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3979-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3980-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3981-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3982-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3983-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3984-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3985-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3986-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3987-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3988-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3989-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3990-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3991-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3992-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3993-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3994-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3995-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3996-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3997-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3998-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3999-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4000-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4001-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4002-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4003-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4004-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4005-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4006-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4007-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4008-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4009-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4010-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4011-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4012-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4013-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4014-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4015-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4016-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4017-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4018-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4019-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4020-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4021-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4022-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4023-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4024-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4025-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4026-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4027-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4028-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4029-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4030-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4031-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4032-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4033-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4034-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4035-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4036-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4037-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4038-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4039-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4040-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4041-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4042-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4043-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4044-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4045-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4046-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4047-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4048-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4049-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4050-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4051-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4052-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4053-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4054-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4055-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4056-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4057-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4058-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4059-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4060-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4061-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4062-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4063-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4064-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4065-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4066-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4067-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4068-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4069-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4070-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4071-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4072-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4073-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4074-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4075-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4076-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4077-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4078-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4079-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4080-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4081-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4082-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4083-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4084-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4085-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4086-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4087-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4088-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4089-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4090-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4091-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4092-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4093-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4094-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4095-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH ,<L\l| + +  ,<L\l| -.=>MN]^mn}~ +  -.=>MN]^mn}~/?O_o + /?O_o@ @AAB,BCMDN]E^mFn}G~HIJKLMNO PQ-R.=S>MTN]U^mVn}W~XYZ[\]^_@AB/C?DOE_FoGHIJKLMNOPQR/S?TOU_VoWXYZ[\]^_ !!","#<#$L$%\%&l&'|'(())**++,,--..//0 0112,23<34L45\56l67|78899::;;<<==>>?? !-".=#>M$N]%^m&n}'~()*+,-./ 01-2.=3>M4N]5^m6n}7~89:;<=>? !"/#?$O%_&o'()*+,-./012/3?4O5_6o789:;<=>?` `aab,bcMdN]e^mfn}g~hijklmno pq-r.=s>MtN]u^mvn}w~xyz{|}~`ab/c?dOe_foghijklmnopqr/s?tOu_vowxyz{|}~HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHg g ` HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH$$Dd$Dd$Ddd$$Dd$Dd$Ddd$$Dd $ D d  +$ +D +d +d + + + + + $ $ D d  $ D d  $ D d d $$Dd$Ddvalue-3072-deterministic-fixturevalue-3074-deterministic-fixturevalue-3075-deterministic-fixturevalue-3076-deterministic-fixturevalue-3077-deterministic-fixturevalue-3078-deterministic-fixturevalue-3079-deterministic-fixturevalue-3081-deterministic-fixturevalue-3082-deterministic-fixturevalue-3083-deterministic-fixturevalue-3084-deterministic-fixturevalue-3085-deterministic-fixturevalue-3086-deterministic-fixturevalue-3088-deterministic-fixturevalue-3089-deterministic-fixturevalue-3090-deterministic-fixturevalue-3091-deterministic-fixturevalue-3092-deterministic-fixturevalue-3093-deterministic-fixturevalue-3095-deterministic-fixturevalue-3096-deterministic-fixturevalue-3097-deterministic-fixturevalue-3098-deterministic-fixturevalue-3099-deterministic-fixturevalue-3100-deterministic-fixturevalue-3102-deterministic-fixturevalue-3103-deterministic-fixturevalue-3104-deterministic-fixturevalue-3105-deterministic-fixturevalue-3106-deterministic-fixturevalue-3107-deterministic-fixturevalue-3109-deterministic-fixturevalue-3110-deterministic-fixturevalue-3111-deterministic-fixturevalue-3112-deterministic-fixturevalue-3113-deterministic-fixturevalue-3114-deterministic-fixturevalue-3116-deterministic-fixturevalue-3117-deterministic-fixturevalue-3118-deterministic-fixturevalue-3119-deterministic-fixturevalue-3120-deterministic-fixturevalue-3121-deterministic-fixturevalue-3123-deterministic-fixturevalue-3124-deterministic-fixturevalue-3125-deterministic-fixturevalue-3126-deterministic-fixturevalue-3127-deterministic-fixturevalue-3128-deterministic-fixturevalue-3130-deterministic-fixturevalue-3131-deterministic-fixturevalue-3132-deterministic-fixturevalue-3133-deterministic-fixturevalue-3134-deterministic-fixturevalue-3135-deterministic-fixturevalue-3137-deterministic-fixturevalue-3138-deterministic-fixturevalue-3139-deterministic-fixturevalue-3140-deterministic-fixturevalue-3141-deterministic-fixturevalue-3142-deterministic-fixturevalue-3144-deterministic-fixturevalue-3145-deterministic-fixturevalue-3146-deterministic-fixturevalue-3147-deterministic-fixturevalue-3148-deterministic-fixturevalue-3149-deterministic-fixturevalue-3151-deterministic-fixturevalue-3152-deterministic-fixturevalue-3153-deterministic-fixturevalue-3154-deterministic-fixturevalue-3155-deterministic-fixturevalue-3156-deterministic-fixturevalue-3158-deterministic-fixturevalue-3159-deterministic-fixturevalue-3160-deterministic-fixturevalue-3161-deterministic-fixturevalue-3162-deterministic-fixturevalue-3163-deterministic-fixturevalue-3165-deterministic-fixturevalue-3166-deterministic-fixturevalue-3167-deterministic-fixturevalue-3168-deterministic-fixturevalue-3169-deterministic-fixturevalue-3170-deterministic-fixturevalue-3172-deterministic-fixturevalue-3173-deterministic-fixturevalue-3174-deterministic-fixturevalue-3175-deterministic-fixturevalue-3176-deterministic-fixturevalue-3177-deterministic-fixturevalue-3179-deterministic-fixturevalue-3180-deterministic-fixturevalue-3181-deterministic-fixturevalue-3182-deterministic-fixturevalue-3183-deterministic-fixturevalue-3184-deterministic-fixturevalue-3186-deterministic-fixturevalue-3187-deterministic-fixturevalue-3188-deterministic-fixturevalue-3189-deterministic-fixturevalue-3190-deterministic-fixturevalue-3191-deterministic-fixturevalue-3193-deterministic-fixturevalue-3194-deterministic-fixturevalue-3195-deterministic-fixturevalue-3196-deterministic-fixturevalue-3197-deterministic-fixturevalue-3198-deterministic-fixture$Dd$Dd$DDd$Dd$Dd$DDd$Dd $ D d  +$ +D +D +d + + + + +  $ D d  $ D d  $ D D d $Dd$Ddvalue-3200-deterministic-fixturevalue-3201-deterministic-fixturevalue-3202-deterministic-fixturevalue-3203-deterministic-fixturevalue-3204-deterministic-fixturevalue-3205-deterministic-fixturevalue-3207-deterministic-fixturevalue-3208-deterministic-fixturevalue-3209-deterministic-fixturevalue-3210-deterministic-fixturevalue-3211-deterministic-fixturevalue-3212-deterministic-fixturevalue-3214-deterministic-fixturevalue-3215-deterministic-fixturevalue-3216-deterministic-fixturevalue-3217-deterministic-fixturevalue-3218-deterministic-fixturevalue-3219-deterministic-fixturevalue-3221-deterministic-fixturevalue-3222-deterministic-fixturevalue-3223-deterministic-fixturevalue-3224-deterministic-fixturevalue-3225-deterministic-fixturevalue-3226-deterministic-fixturevalue-3228-deterministic-fixturevalue-3229-deterministic-fixturevalue-3230-deterministic-fixturevalue-3231-deterministic-fixturevalue-3232-deterministic-fixturevalue-3233-deterministic-fixturevalue-3235-deterministic-fixturevalue-3236-deterministic-fixturevalue-3237-deterministic-fixturevalue-3238-deterministic-fixturevalue-3239-deterministic-fixturevalue-3240-deterministic-fixturevalue-3242-deterministic-fixturevalue-3243-deterministic-fixturevalue-3244-deterministic-fixturevalue-3245-deterministic-fixturevalue-3246-deterministic-fixturevalue-3247-deterministic-fixturevalue-3249-deterministic-fixturevalue-3250-deterministic-fixturevalue-3251-deterministic-fixturevalue-3252-deterministic-fixturevalue-3253-deterministic-fixturevalue-3254-deterministic-fixturevalue-3256-deterministic-fixturevalue-3257-deterministic-fixturevalue-3258-deterministic-fixturevalue-3259-deterministic-fixturevalue-3260-deterministic-fixturevalue-3261-deterministic-fixturevalue-3263-deterministic-fixturevalue-3264-deterministic-fixturevalue-3265-deterministic-fixturevalue-3266-deterministic-fixturevalue-3267-deterministic-fixturevalue-3268-deterministic-fixturevalue-3270-deterministic-fixturevalue-3271-deterministic-fixturevalue-3272-deterministic-fixturevalue-3273-deterministic-fixturevalue-3274-deterministic-fixturevalue-3275-deterministic-fixturevalue-3277-deterministic-fixturevalue-3278-deterministic-fixturevalue-3279-deterministic-fixturevalue-3280-deterministic-fixturevalue-3281-deterministic-fixturevalue-3282-deterministic-fixturevalue-3284-deterministic-fixturevalue-3285-deterministic-fixturevalue-3286-deterministic-fixturevalue-3287-deterministic-fixturevalue-3288-deterministic-fixturevalue-3289-deterministic-fixturevalue-3291-deterministic-fixturevalue-3292-deterministic-fixturevalue-3293-deterministic-fixturevalue-3294-deterministic-fixturevalue-3295-deterministic-fixturevalue-3296-deterministic-fixturevalue-3298-deterministic-fixturevalue-3299-deterministic-fixturevalue-3300-deterministic-fixturevalue-3301-deterministic-fixturevalue-3302-deterministic-fixturevalue-3303-deterministic-fixturevalue-3305-deterministic-fixturevalue-3306-deterministic-fixturevalue-3307-deterministic-fixturevalue-3308-deterministic-fixturevalue-3309-deterministic-fixturevalue-3310-deterministic-fixturevalue-3312-deterministic-fixturevalue-3313-deterministic-fixturevalue-3314-deterministic-fixturevalue-3315-deterministic-fixturevalue-3316-deterministic-fixturevalue-3317-deterministic-fixturevalue-3319-deterministic-fixturevalue-3320-deterministic-fixturevalue-3321-deterministic-fixturevalue-3322-deterministic-fixturevalue-3323-deterministic-fixturevalue-3324-deterministic-fixturevalue-3326-deterministic-fixturevalue-3327-deterministic-fixture$Dd$DDd$Dd$Dd$DDd$Dd$Dd $ D D d  + +$ +D +d + + + + + + $ D d  $ D D d   $ D d $Dd$DDdvalue-3328-deterministic-fixturevalue-3329-deterministic-fixturevalue-3330-deterministic-fixturevalue-3331-deterministic-fixturevalue-3333-deterministic-fixturevalue-3334-deterministic-fixturevalue-3335-deterministic-fixturevalue-3336-deterministic-fixturevalue-3337-deterministic-fixturevalue-3338-deterministic-fixturevalue-3340-deterministic-fixturevalue-3341-deterministic-fixturevalue-3342-deterministic-fixturevalue-3343-deterministic-fixturevalue-3344-deterministic-fixturevalue-3345-deterministic-fixturevalue-3347-deterministic-fixturevalue-3348-deterministic-fixturevalue-3349-deterministic-fixturevalue-3350-deterministic-fixturevalue-3351-deterministic-fixturevalue-3352-deterministic-fixturevalue-3354-deterministic-fixturevalue-3355-deterministic-fixturevalue-3356-deterministic-fixturevalue-3357-deterministic-fixturevalue-3358-deterministic-fixturevalue-3359-deterministic-fixturevalue-3361-deterministic-fixturevalue-3362-deterministic-fixturevalue-3363-deterministic-fixturevalue-3364-deterministic-fixturevalue-3365-deterministic-fixturevalue-3366-deterministic-fixturevalue-3368-deterministic-fixturevalue-3369-deterministic-fixturevalue-3370-deterministic-fixturevalue-3371-deterministic-fixturevalue-3372-deterministic-fixturevalue-3373-deterministic-fixturevalue-3375-deterministic-fixturevalue-3376-deterministic-fixturevalue-3377-deterministic-fixturevalue-3378-deterministic-fixturevalue-3379-deterministic-fixturevalue-3380-deterministic-fixturevalue-3382-deterministic-fixturevalue-3383-deterministic-fixturevalue-3384-deterministic-fixturevalue-3385-deterministic-fixturevalue-3386-deterministic-fixturevalue-3387-deterministic-fixturevalue-3389-deterministic-fixturevalue-3390-deterministic-fixturevalue-3391-deterministic-fixturevalue-3392-deterministic-fixturevalue-3393-deterministic-fixturevalue-3394-deterministic-fixturevalue-3396-deterministic-fixturevalue-3397-deterministic-fixturevalue-3398-deterministic-fixturevalue-3399-deterministic-fixturevalue-3400-deterministic-fixturevalue-3401-deterministic-fixturevalue-3403-deterministic-fixturevalue-3404-deterministic-fixturevalue-3405-deterministic-fixturevalue-3406-deterministic-fixturevalue-3407-deterministic-fixturevalue-3408-deterministic-fixturevalue-3410-deterministic-fixturevalue-3411-deterministic-fixturevalue-3412-deterministic-fixturevalue-3413-deterministic-fixturevalue-3414-deterministic-fixturevalue-3415-deterministic-fixturevalue-3417-deterministic-fixturevalue-3418-deterministic-fixturevalue-3419-deterministic-fixturevalue-3420-deterministic-fixturevalue-3421-deterministic-fixturevalue-3422-deterministic-fixturevalue-3424-deterministic-fixturevalue-3425-deterministic-fixturevalue-3426-deterministic-fixturevalue-3427-deterministic-fixturevalue-3428-deterministic-fixturevalue-3429-deterministic-fixturevalue-3431-deterministic-fixturevalue-3432-deterministic-fixturevalue-3433-deterministic-fixturevalue-3434-deterministic-fixturevalue-3435-deterministic-fixturevalue-3436-deterministic-fixturevalue-3438-deterministic-fixturevalue-3439-deterministic-fixturevalue-3440-deterministic-fixturevalue-3441-deterministic-fixturevalue-3442-deterministic-fixturevalue-3443-deterministic-fixturevalue-3445-deterministic-fixturevalue-3446-deterministic-fixturevalue-3447-deterministic-fixturevalue-3448-deterministic-fixturevalue-3449-deterministic-fixturevalue-3450-deterministic-fixturevalue-3452-deterministic-fixturevalue-3453-deterministic-fixturevalue-3454-deterministic-fixturevalue-3455-deterministic-fixture$DDd$Dd$Dd$DDd$Dd$Dd$DDd  $ D d  +$ +D +d + + + + + + $ D D d   $ D d  $ D d $DDd$Ddvalue-3456-deterministic-fixturevalue-3457-deterministic-fixturevalue-3459-deterministic-fixturevalue-3460-deterministic-fixturevalue-3461-deterministic-fixturevalue-3462-deterministic-fixturevalue-3463-deterministic-fixturevalue-3464-deterministic-fixturevalue-3466-deterministic-fixturevalue-3467-deterministic-fixturevalue-3468-deterministic-fixturevalue-3469-deterministic-fixturevalue-3470-deterministic-fixturevalue-3471-deterministic-fixturevalue-3473-deterministic-fixturevalue-3474-deterministic-fixturevalue-3475-deterministic-fixturevalue-3476-deterministic-fixturevalue-3477-deterministic-fixturevalue-3478-deterministic-fixturevalue-3480-deterministic-fixturevalue-3481-deterministic-fixturevalue-3482-deterministic-fixturevalue-3483-deterministic-fixturevalue-3484-deterministic-fixturevalue-3485-deterministic-fixturevalue-3487-deterministic-fixturevalue-3488-deterministic-fixturevalue-3489-deterministic-fixturevalue-3490-deterministic-fixturevalue-3491-deterministic-fixturevalue-3492-deterministic-fixturevalue-3494-deterministic-fixturevalue-3495-deterministic-fixturevalue-3496-deterministic-fixturevalue-3497-deterministic-fixturevalue-3498-deterministic-fixturevalue-3499-deterministic-fixturevalue-3501-deterministic-fixturevalue-3502-deterministic-fixturevalue-3503-deterministic-fixturevalue-3504-deterministic-fixturevalue-3505-deterministic-fixturevalue-3506-deterministic-fixturevalue-3508-deterministic-fixturevalue-3509-deterministic-fixturevalue-3510-deterministic-fixturevalue-3511-deterministic-fixturevalue-3512-deterministic-fixturevalue-3513-deterministic-fixturevalue-3515-deterministic-fixturevalue-3516-deterministic-fixturevalue-3517-deterministic-fixturevalue-3518-deterministic-fixturevalue-3519-deterministic-fixturevalue-3520-deterministic-fixturevalue-3522-deterministic-fixturevalue-3523-deterministic-fixturevalue-3524-deterministic-fixturevalue-3525-deterministic-fixturevalue-3526-deterministic-fixturevalue-3527-deterministic-fixturevalue-3529-deterministic-fixturevalue-3530-deterministic-fixturevalue-3531-deterministic-fixturevalue-3532-deterministic-fixturevalue-3533-deterministic-fixturevalue-3534-deterministic-fixturevalue-3536-deterministic-fixturevalue-3537-deterministic-fixturevalue-3538-deterministic-fixturevalue-3539-deterministic-fixturevalue-3540-deterministic-fixturevalue-3541-deterministic-fixturevalue-3543-deterministic-fixturevalue-3544-deterministic-fixturevalue-3545-deterministic-fixturevalue-3546-deterministic-fixturevalue-3547-deterministic-fixturevalue-3548-deterministic-fixturevalue-3550-deterministic-fixturevalue-3551-deterministic-fixturevalue-3552-deterministic-fixturevalue-3553-deterministic-fixturevalue-3554-deterministic-fixturevalue-3555-deterministic-fixturevalue-3557-deterministic-fixturevalue-3558-deterministic-fixturevalue-3559-deterministic-fixturevalue-3560-deterministic-fixturevalue-3561-deterministic-fixturevalue-3562-deterministic-fixturevalue-3564-deterministic-fixturevalue-3565-deterministic-fixturevalue-3566-deterministic-fixturevalue-3567-deterministic-fixturevalue-3568-deterministic-fixturevalue-3569-deterministic-fixturevalue-3571-deterministic-fixturevalue-3572-deterministic-fixturevalue-3573-deterministic-fixturevalue-3574-deterministic-fixturevalue-3575-deterministic-fixturevalue-3576-deterministic-fixturevalue-3578-deterministic-fixturevalue-3579-deterministic-fixturevalue-3580-deterministic-fixturevalue-3581-deterministic-fixturevalue-3582-deterministic-fixturevalue-3583-deterministic-fixture$Dd$Dd$DDd$Dd$Dd$DDd$Dd $ D d  +$ +D +D +d + + + + +  $ D d  $ D d  $ D D d $Dd$Ddvalue-3585-deterministic-fixturevalue-3586-deterministic-fixturevalue-3587-deterministic-fixturevalue-3588-deterministic-fixturevalue-3589-deterministic-fixturevalue-3590-deterministic-fixturevalue-3592-deterministic-fixturevalue-3593-deterministic-fixturevalue-3594-deterministic-fixturevalue-3595-deterministic-fixturevalue-3596-deterministic-fixturevalue-3597-deterministic-fixturevalue-3599-deterministic-fixturevalue-3600-deterministic-fixturevalue-3601-deterministic-fixturevalue-3602-deterministic-fixturevalue-3603-deterministic-fixturevalue-3604-deterministic-fixturevalue-3606-deterministic-fixturevalue-3607-deterministic-fixturevalue-3608-deterministic-fixturevalue-3609-deterministic-fixturevalue-3610-deterministic-fixturevalue-3611-deterministic-fixturevalue-3613-deterministic-fixturevalue-3614-deterministic-fixturevalue-3615-deterministic-fixturevalue-3616-deterministic-fixturevalue-3617-deterministic-fixturevalue-3618-deterministic-fixturevalue-3620-deterministic-fixturevalue-3621-deterministic-fixturevalue-3622-deterministic-fixturevalue-3623-deterministic-fixturevalue-3624-deterministic-fixturevalue-3625-deterministic-fixturevalue-3627-deterministic-fixturevalue-3628-deterministic-fixturevalue-3629-deterministic-fixturevalue-3630-deterministic-fixturevalue-3631-deterministic-fixturevalue-3632-deterministic-fixturevalue-3634-deterministic-fixturevalue-3635-deterministic-fixturevalue-3636-deterministic-fixturevalue-3637-deterministic-fixturevalue-3638-deterministic-fixturevalue-3639-deterministic-fixturevalue-3641-deterministic-fixturevalue-3642-deterministic-fixturevalue-3643-deterministic-fixturevalue-3644-deterministic-fixturevalue-3645-deterministic-fixturevalue-3646-deterministic-fixturevalue-3648-deterministic-fixturevalue-3649-deterministic-fixturevalue-3650-deterministic-fixturevalue-3651-deterministic-fixturevalue-3652-deterministic-fixturevalue-3653-deterministic-fixturevalue-3655-deterministic-fixturevalue-3656-deterministic-fixturevalue-3657-deterministic-fixturevalue-3658-deterministic-fixturevalue-3659-deterministic-fixturevalue-3660-deterministic-fixturevalue-3662-deterministic-fixturevalue-3663-deterministic-fixturevalue-3664-deterministic-fixturevalue-3665-deterministic-fixturevalue-3666-deterministic-fixturevalue-3667-deterministic-fixturevalue-3669-deterministic-fixturevalue-3670-deterministic-fixturevalue-3671-deterministic-fixturevalue-3672-deterministic-fixturevalue-3673-deterministic-fixturevalue-3674-deterministic-fixturevalue-3676-deterministic-fixturevalue-3677-deterministic-fixturevalue-3678-deterministic-fixturevalue-3679-deterministic-fixturevalue-3680-deterministic-fixturevalue-3681-deterministic-fixturevalue-3683-deterministic-fixturevalue-3684-deterministic-fixturevalue-3685-deterministic-fixturevalue-3686-deterministic-fixturevalue-3687-deterministic-fixturevalue-3688-deterministic-fixturevalue-3690-deterministic-fixturevalue-3691-deterministic-fixturevalue-3692-deterministic-fixturevalue-3693-deterministic-fixturevalue-3694-deterministic-fixturevalue-3695-deterministic-fixturevalue-3697-deterministic-fixturevalue-3698-deterministic-fixturevalue-3699-deterministic-fixturevalue-3700-deterministic-fixturevalue-3701-deterministic-fixturevalue-3702-deterministic-fixturevalue-3704-deterministic-fixturevalue-3705-deterministic-fixturevalue-3706-deterministic-fixturevalue-3707-deterministic-fixturevalue-3708-deterministic-fixturevalue-3709-deterministic-fixturevalue-3711-deterministic-fixture$Dd$Ddd$$Dd$Dd$Ddd$$Dd$Dd $ D d d  +$ +$ +D +d + + + + + + $ D d  $ D d d  $ $ D d $Dd$Dddvalue-3712-deterministic-fixturevalue-3713-deterministic-fixturevalue-3714-deterministic-fixturevalue-3715-deterministic-fixturevalue-3716-deterministic-fixturevalue-3718-deterministic-fixturevalue-3719-deterministic-fixturevalue-3720-deterministic-fixturevalue-3721-deterministic-fixturevalue-3722-deterministic-fixturevalue-3723-deterministic-fixturevalue-3725-deterministic-fixturevalue-3726-deterministic-fixturevalue-3727-deterministic-fixturevalue-3728-deterministic-fixturevalue-3729-deterministic-fixturevalue-3730-deterministic-fixturevalue-3732-deterministic-fixturevalue-3733-deterministic-fixturevalue-3734-deterministic-fixturevalue-3735-deterministic-fixturevalue-3736-deterministic-fixturevalue-3737-deterministic-fixturevalue-3739-deterministic-fixturevalue-3740-deterministic-fixturevalue-3741-deterministic-fixturevalue-3742-deterministic-fixturevalue-3743-deterministic-fixturevalue-3744-deterministic-fixturevalue-3746-deterministic-fixturevalue-3747-deterministic-fixturevalue-3748-deterministic-fixturevalue-3749-deterministic-fixturevalue-3750-deterministic-fixturevalue-3751-deterministic-fixturevalue-3753-deterministic-fixturevalue-3754-deterministic-fixturevalue-3755-deterministic-fixturevalue-3756-deterministic-fixturevalue-3757-deterministic-fixturevalue-3758-deterministic-fixturevalue-3760-deterministic-fixturevalue-3761-deterministic-fixturevalue-3762-deterministic-fixturevalue-3763-deterministic-fixturevalue-3764-deterministic-fixturevalue-3765-deterministic-fixturevalue-3767-deterministic-fixturevalue-3768-deterministic-fixturevalue-3769-deterministic-fixturevalue-3770-deterministic-fixturevalue-3771-deterministic-fixturevalue-3772-deterministic-fixturevalue-3774-deterministic-fixturevalue-3775-deterministic-fixturevalue-3776-deterministic-fixturevalue-3777-deterministic-fixturevalue-3778-deterministic-fixturevalue-3779-deterministic-fixturevalue-3781-deterministic-fixturevalue-3782-deterministic-fixturevalue-3783-deterministic-fixturevalue-3784-deterministic-fixturevalue-3785-deterministic-fixturevalue-3786-deterministic-fixturevalue-3788-deterministic-fixturevalue-3789-deterministic-fixturevalue-3790-deterministic-fixturevalue-3791-deterministic-fixturevalue-3792-deterministic-fixturevalue-3793-deterministic-fixturevalue-3795-deterministic-fixturevalue-3796-deterministic-fixturevalue-3797-deterministic-fixturevalue-3798-deterministic-fixturevalue-3799-deterministic-fixturevalue-3800-deterministic-fixturevalue-3802-deterministic-fixturevalue-3803-deterministic-fixturevalue-3804-deterministic-fixturevalue-3805-deterministic-fixturevalue-3806-deterministic-fixturevalue-3807-deterministic-fixturevalue-3809-deterministic-fixturevalue-3810-deterministic-fixturevalue-3811-deterministic-fixturevalue-3812-deterministic-fixturevalue-3813-deterministic-fixturevalue-3814-deterministic-fixturevalue-3816-deterministic-fixturevalue-3817-deterministic-fixturevalue-3818-deterministic-fixturevalue-3819-deterministic-fixturevalue-3820-deterministic-fixturevalue-3821-deterministic-fixturevalue-3823-deterministic-fixturevalue-3824-deterministic-fixturevalue-3825-deterministic-fixturevalue-3826-deterministic-fixturevalue-3827-deterministic-fixturevalue-3828-deterministic-fixturevalue-3830-deterministic-fixturevalue-3831-deterministic-fixturevalue-3832-deterministic-fixturevalue-3833-deterministic-fixturevalue-3834-deterministic-fixturevalue-3835-deterministic-fixturevalue-3837-deterministic-fixturevalue-3838-deterministic-fixturevalue-3839-deterministic-fixture$Ddd$$Dd$Dd$Ddd$$Dd$Dd$Ddd $ $ D d  +$ +D +d + + + + + + $ D d d  $ $ D d  $ D d $Ddd$$Ddvalue-3840-deterministic-fixturevalue-3841-deterministic-fixturevalue-3842-deterministic-fixturevalue-3844-deterministic-fixturevalue-3845-deterministic-fixturevalue-3846-deterministic-fixturevalue-3847-deterministic-fixturevalue-3848-deterministic-fixturevalue-3849-deterministic-fixturevalue-3851-deterministic-fixturevalue-3852-deterministic-fixturevalue-3853-deterministic-fixturevalue-3854-deterministic-fixturevalue-3855-deterministic-fixturevalue-3856-deterministic-fixturevalue-3858-deterministic-fixturevalue-3859-deterministic-fixturevalue-3860-deterministic-fixturevalue-3861-deterministic-fixturevalue-3862-deterministic-fixturevalue-3863-deterministic-fixturevalue-3865-deterministic-fixturevalue-3866-deterministic-fixturevalue-3867-deterministic-fixturevalue-3868-deterministic-fixturevalue-3869-deterministic-fixturevalue-3870-deterministic-fixturevalue-3872-deterministic-fixturevalue-3873-deterministic-fixturevalue-3874-deterministic-fixturevalue-3875-deterministic-fixturevalue-3876-deterministic-fixturevalue-3877-deterministic-fixturevalue-3879-deterministic-fixturevalue-3880-deterministic-fixturevalue-3881-deterministic-fixturevalue-3882-deterministic-fixturevalue-3883-deterministic-fixturevalue-3884-deterministic-fixturevalue-3886-deterministic-fixturevalue-3887-deterministic-fixturevalue-3888-deterministic-fixturevalue-3889-deterministic-fixturevalue-3890-deterministic-fixturevalue-3891-deterministic-fixturevalue-3893-deterministic-fixturevalue-3894-deterministic-fixturevalue-3895-deterministic-fixturevalue-3896-deterministic-fixturevalue-3897-deterministic-fixturevalue-3898-deterministic-fixturevalue-3900-deterministic-fixturevalue-3901-deterministic-fixturevalue-3902-deterministic-fixturevalue-3903-deterministic-fixturevalue-3904-deterministic-fixturevalue-3905-deterministic-fixturevalue-3907-deterministic-fixturevalue-3908-deterministic-fixturevalue-3909-deterministic-fixturevalue-3910-deterministic-fixturevalue-3911-deterministic-fixturevalue-3912-deterministic-fixturevalue-3914-deterministic-fixturevalue-3915-deterministic-fixturevalue-3916-deterministic-fixturevalue-3917-deterministic-fixturevalue-3918-deterministic-fixturevalue-3919-deterministic-fixturevalue-3921-deterministic-fixturevalue-3922-deterministic-fixturevalue-3923-deterministic-fixturevalue-3924-deterministic-fixturevalue-3925-deterministic-fixturevalue-3926-deterministic-fixturevalue-3928-deterministic-fixturevalue-3929-deterministic-fixturevalue-3930-deterministic-fixturevalue-3931-deterministic-fixturevalue-3932-deterministic-fixturevalue-3933-deterministic-fixturevalue-3935-deterministic-fixturevalue-3936-deterministic-fixturevalue-3937-deterministic-fixturevalue-3938-deterministic-fixturevalue-3939-deterministic-fixturevalue-3940-deterministic-fixturevalue-3942-deterministic-fixturevalue-3943-deterministic-fixturevalue-3944-deterministic-fixturevalue-3945-deterministic-fixturevalue-3946-deterministic-fixturevalue-3947-deterministic-fixturevalue-3949-deterministic-fixturevalue-3950-deterministic-fixturevalue-3951-deterministic-fixturevalue-3952-deterministic-fixturevalue-3953-deterministic-fixturevalue-3954-deterministic-fixturevalue-3956-deterministic-fixturevalue-3957-deterministic-fixturevalue-3958-deterministic-fixturevalue-3959-deterministic-fixturevalue-3960-deterministic-fixturevalue-3961-deterministic-fixturevalue-3963-deterministic-fixturevalue-3964-deterministic-fixturevalue-3965-deterministic-fixturevalue-3966-deterministic-fixturevalue-3967-deterministic-fixture$$Dd$Dd$Ddd$$Dd$Dd$Ddd$$Dd $ D d  +$ +D +d +d + + + + + $ $ D d  $ D d  $ D d d $$Dd$Ddvalue-3968-deterministic-fixturevalue-3970-deterministic-fixturevalue-3971-deterministic-fixturevalue-3972-deterministic-fixturevalue-3973-deterministic-fixturevalue-3974-deterministic-fixturevalue-3975-deterministic-fixturevalue-3977-deterministic-fixturevalue-3978-deterministic-fixturevalue-3979-deterministic-fixturevalue-3980-deterministic-fixturevalue-3981-deterministic-fixturevalue-3982-deterministic-fixturevalue-3984-deterministic-fixturevalue-3985-deterministic-fixturevalue-3986-deterministic-fixturevalue-3987-deterministic-fixturevalue-3988-deterministic-fixturevalue-3989-deterministic-fixturevalue-3991-deterministic-fixturevalue-3992-deterministic-fixturevalue-3993-deterministic-fixturevalue-3994-deterministic-fixturevalue-3995-deterministic-fixturevalue-3996-deterministic-fixturevalue-3998-deterministic-fixturevalue-3999-deterministic-fixturevalue-4000-deterministic-fixturevalue-4001-deterministic-fixturevalue-4002-deterministic-fixturevalue-4003-deterministic-fixturevalue-4005-deterministic-fixturevalue-4006-deterministic-fixturevalue-4007-deterministic-fixturevalue-4008-deterministic-fixturevalue-4009-deterministic-fixturevalue-4010-deterministic-fixturevalue-4012-deterministic-fixturevalue-4013-deterministic-fixturevalue-4014-deterministic-fixturevalue-4015-deterministic-fixturevalue-4016-deterministic-fixturevalue-4017-deterministic-fixturevalue-4019-deterministic-fixturevalue-4020-deterministic-fixturevalue-4021-deterministic-fixturevalue-4022-deterministic-fixturevalue-4023-deterministic-fixturevalue-4024-deterministic-fixturevalue-4026-deterministic-fixturevalue-4027-deterministic-fixturevalue-4028-deterministic-fixturevalue-4029-deterministic-fixturevalue-4030-deterministic-fixturevalue-4031-deterministic-fixturevalue-4033-deterministic-fixturevalue-4034-deterministic-fixturevalue-4035-deterministic-fixturevalue-4036-deterministic-fixturevalue-4037-deterministic-fixturevalue-4038-deterministic-fixturevalue-4040-deterministic-fixturevalue-4041-deterministic-fixturevalue-4042-deterministic-fixturevalue-4043-deterministic-fixturevalue-4044-deterministic-fixturevalue-4045-deterministic-fixturevalue-4047-deterministic-fixturevalue-4048-deterministic-fixturevalue-4049-deterministic-fixturevalue-4050-deterministic-fixturevalue-4051-deterministic-fixturevalue-4052-deterministic-fixturevalue-4054-deterministic-fixturevalue-4055-deterministic-fixturevalue-4056-deterministic-fixturevalue-4057-deterministic-fixturevalue-4058-deterministic-fixturevalue-4059-deterministic-fixturevalue-4061-deterministic-fixturevalue-4062-deterministic-fixturevalue-4063-deterministic-fixturevalue-4064-deterministic-fixturevalue-4065-deterministic-fixturevalue-4066-deterministic-fixturevalue-4068-deterministic-fixturevalue-4069-deterministic-fixturevalue-4070-deterministic-fixturevalue-4071-deterministic-fixturevalue-4072-deterministic-fixturevalue-4073-deterministic-fixturevalue-4075-deterministic-fixturevalue-4076-deterministic-fixturevalue-4077-deterministic-fixturevalue-4078-deterministic-fixturevalue-4079-deterministic-fixturevalue-4080-deterministic-fixturevalue-4082-deterministic-fixturevalue-4083-deterministic-fixturevalue-4084-deterministic-fixturevalue-4085-deterministic-fixturevalue-4086-deterministic-fixturevalue-4087-deterministic-fixturevalue-4089-deterministic-fixturevalue-4090-deterministic-fixturevalue-4091-deterministic-fixturevalue-4092-deterministic-fixturevalue-4093-deterministic-fixturevalue-4094-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHJJ HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH"Ddd2d2D2D2D"D"L"L"L&L&H&H&H$H$I$I$ɒ$$ddd2d2D2D2D"D"L"L"L&L&H&H&H$H$I$I$ɒ$$dd @@ @H@ @@ @!@@@ @@ @!@@ " sL# $ $X $) 3 $&%  c 6L&i +X $@' $V( f L)#$v + * XS * 421QjdŜ924QrdŨz=27Qƴ|A2:Rd~D2=Rd̖I2@RdؖM2CRd䖄Q2FR𖈌U2LSdX2OSda2RSd&uь):tuLI;t&v Y;Cuv̱;su&ьɣA5]$j @E5`j!L̟M5c$j" DXQ5fU"#Vǰ4U*#vǼ5鎙SV2$6 V:6)V%8IVJ&9 WR&9CWZ:sWb';ɏWr((<鏽Wz(4e@2y&eԌ@2|eL—A2&e ×A2&׌ėA2&eLŗB2&e Ɨ@B2eƘB2&e܌ǘB2eLȘC2&e ɘYC 5yk ӀI5cy6ӌi5ykFӘ 5ykVӤ5y"k`Ӱɦ5#z*kvӼ香5SzȠ 5z:kԠ 5zJkI5zRki5{!$ +)$ L,I'$ - XC*-s-$. 0$  0"X3 0 6$!1 (3Sw& Y>w|̺)>w}L ?w&~ iY?x&̽?Cx&Ҍ?sxLɥ@x& Y@xӢj#FT5i$j$HY5l$j%LJ]5o$& a5r$j' Nd5uj(L,Pi5x$j* 8TXm5~j+Dq5$,PXy5j-LhZ|5$j. t\X< X)&@=IcX0L>iX*FX?X+Vd@ X+fpAɐ#Yp|BSY,ȈB Y-ȔC)Y-ȬD Y.ȸEiZ2eɘC2&eʘC2L˘YD2&e̘ęD2&e͘D2eLΘE2&e ϘYE2ϘΙE2&eИE2eLјF2&e 5s{bk 5{jk駹5{rk( 5|zk4)53|&@I5c|k6L 5|kFX5|kPp5|f|ɨ5#}kԈ 5"Dd2D2D2D"D"L"L"L&L&H&H&H$H$I$I$ɒ$$ddd2d2D2D2D"D"L"L"L&L&H&H&H$H$I$I$ɒ$$ddd2d2DH@ @@ @@ @H@ @@ @!@@@ @@ @ie(Vx +@jnԖ{> +jqզ|M@Jkt(} AjZkwAkC}(րBksMC +l( D[lӆ("p308*p4޴| +Ҁ*1' +ܘc17Nj*2GˊҜû*3W̪ܙ3gN#*4w +\S4 +Ӝ**ܚ5NJ*6j\;v;( +v ;vM"[ ;v$$;(0;*vM<*;(2v H,[;(:vT.%;(l);ȨRvMx2-;(Zv DM +posPNʽuo\OypӜhP +}ptQ*p3&߀R +qc +6ߌSjq@ߘSqÝV߰Tq*f߼Uʾr#2V +\8*pKک8*=pL۩8*>pMݩ\8>Nݩ8*qPީܟ8?qQNߩ8*@qR੤\8@S੦8*AqTᩨܠ8*qUN⩪8*Bv*a; Je;v, +h;v8m;CvDq;svPu;\};vt +;w뀲*օ;3 +w0댲j։;G똲D +l(&EJlc6MF + +m(F G mÒ(PHmfMIn#(v J[nS܆J +n(ݖK*n(MLJo(޶ Mj@oA8?~E8p@̨H8pAبM8ÚpBQ8ƚpC𨆼U8ɛEY8ϛpF\8қpGe8՛pH,i8؛I8m8ۜpJ6ѪӜs*7ܛ*N\ӽ8 +Ԁ*9 +ܜ39'NJc*0j\:G׊Ԁþ*;Wتܝ;gN +#*< +Մ4[1;Ψbv͐65;(jv89;( @=;רzvʹ<@;(v@E;(vM̱BI;(v ر@Q;樢vFU;(vHX;쨲vMJ];(r:V*rBWJsYjsRY +sCZZssb[ʿtj\꿽tӟz4\ +t@] +t36L^ju@XqV@8BqW㩮ܡ8CXN婰8+qY橲\8 Dq[8+Eq\窸ܢ8Eq]N誺8+Fq^骼\8+q_骾8+Gq`8 GqaN֍;wW뤲 +;"wg밲֕;#*wp뼲֙;S2wȲ +ם;Բ*ס;Bw +;Rwj׭;Zwױ;C׵;jw +;rw2"" " " & &&&$$ $ $ $ $ $ $ $ 2$ 2 22"" " " & &&&$$ $ $ $ $ $ $ $ 2$ 2 22"!@@ @@ H@ @@ʬUޝs,VǣЬNW@ӬX ,Y+3, NZKc,7[k@ɓ⬒G[,W\ 謓gN]#,^ Ĺ={йV=ܹX={Z={\> +|`> +| b >d>"|0f>*|<>2|Hj>@ / / _ / O / _/ / O / ^-O- ^ !- $ O'-^*-- 0O"g L⚚WgQ`iU#㪜wjYSj ]΃㺝k `㝠lKi8mkmDmqCPn tϣ\oy>C/|>F/|Ï>I|O>L/_>R/|>U/|ȏ>X|O>[/_>^|>a/|̏>d|O           ]ʃ^+ʳ,_ ,ak],aC,bsOc̣-d ^d  -e+3 7Ofk͓-GTl!>"B|`n$>%J|lp)>(R|x->+Z|v1>1x5>4j|z<>7z|A>:|~E>=̺I>@|غ _ O / @ / /O / @ / O 3-$6-&9O<-,^B.E-0HOK-4^N6Q-8T-Ohp }tq+3'rKc +7s ѓGsPtgu#2wvS:v+ҳBwKJxk嗢@>g|>j/Ϗ>m/}O>s/}>v/}>y/>|/}>/}>/}>/}>/   HHHHHHHHHHHHHHHHHHHHHHHHwwHHHHHHHHHHHHHHHH HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @@ @@ @@ @@ @@!aaaaaaaAaa`a!aaaaaaaAaa`a!aaaaaaaAaa`a!aaaaaaaaaAaa`a!aaaaaaaAaa`a!aaaaaaaAaa`a!aaaaHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH  greenblueredHHHHHHHHHHHHHHHHHHHHHHHHHHHH    HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ @    +@ + + + @    @    @   @@@@@@@@@@@@@@@@@@ @   !@!!!"@"""#@###$@$$$%@%%%&@&&&'@'''(@((()@)))*@***+@+++,@,,,-@---.@.../@///0@0001@1112@2223@3334@4445@5556@6667@7778@8889@999:@:::;@;;;<@<<<=@===>@>>>?@???@@@@@A@AAAB@BBBC@CCCD@DDDE@EEEF@FFFG@GGGH@HHHI@IIIJ@JJJK@KKKL@LLLM@MMMN@NNNO@OOOP@PPPQ@QQQR@RRRS@SSST@TTTU@UUUV@VVVW@WWWX@XXXY@YYYZ@ZZZ[@[[[\@\\\]@]]]^@^^^_@___`@```a@aaab@bbbc@cccd@ddde@eeef@fffg@gggh@hhhi@iiij@jjjk@kkkl@lllm@mmmn@nnno@ooop@pppq@qqqr@rrrs@ssst@tttu@uuuv@vvvw@wwwx@xxxy@yyyz@zzz{@{{{|@|||}@}}}~@~~~@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4096-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH value-4096-deterministic-fixtureHHHHHHHHHHHHPHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH  0HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH  greenblueredHHHHHHHHHHHHHHHHHHHHHHHHHHHH HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH + +id *int328 +Cname *string08R" +lance-encoding:compressionnone +items *list08 +item *int3208 +ecategory *dict:string:int8:false08BR. +&lance-encoding:dict-values-compressionnone +Bblob * large_binary08R +lance-encoding:blobtrue +)' +% +/lance.encodings.ColumnEncoding +I + +"75 +3 +/lance.encodings21.PageLayout +* 28HPL +  "75 +3 +/lance.encodings21.PageLayout +* 28HP(L + "75 +3 +/lance.encodings21.PageLayout +* 28HP(L + "75 +3 +/lance.encodings21.PageLayout +* 28HP(7"0. +, +/lance.encodings21.PageLayout  *2( +)' +% +/lance.encodings.ColumnEncoding +T + "A? += +/lance.encodings21.PageLayout +* + + 28HPW +  "A? += +/lance.encodings21.PageLayout +* + + 28HP(W + "A? += +/lance.encodings21.PageLayout +* + + 28HP(W + "A? += +/lance.encodings21.PageLayout +* + + 28HP(9 +4"*( +& +/lance.encodings21.PageLayout*( +)' +% +/lance.encodings.ColumnEncoding +h +  .0"RP +N +/lance.encodings21.PageLayout- ++ + +" + +" +* 28@HPk +  20"RP +N +/lance.encodings21.PageLayout- ++ + +" + +" +* 28@HP(k +  40"RP +N +/lance.encodings21.PageLayout- ++ + +" + +" +* 28@HP(k +  50"RP +N +/lance.encodings21.PageLayout- ++ + +" + +" +* 28@HP(\ + 0"EC +A +/lance.encodings21.PageLayout + + +* + 28@HP( +)' +% +/lance.encodings.ColumnEncoding +_ + $"IG +E +/lance.encodings21.PageLayout$ +"**" + + (28HPb + $"IG +E +/lance.encodings21.PageLayout$ +"**" + + (28HP(b + $"IG +E +/lance.encodings21.PageLayout$ +"**" + + (28HP(b + $"IG +E +/lance.encodings21.PageLayout$ +"**" + + (28HP(Y + $"B@ +> +/lance.encodings21.PageLayout + +" + + (28HP( +)' +% +/lance.encodings.ColumnEncoding +Z +"GE +C +/lance.encodings21.PageLayout"" + + b +@@ +28HP] +"GE +C +/lance.encodings21.PageLayout"" + + b +@@ +28HP(] +"GE +C +/lance.encodings21.PageLayout"" + + b +@@ +28HP(] +"GE +C +/lance.encodings21.PageLayout"" + + b +@@ +28HP(Y +"FD +B +/lance.encodings21.PageLayout!" + + b +@@ +28HP( ǚ`':atGǚsäLANC \ No newline at end of file diff --git a/rust/lance-geo/src/bbox.rs b/rust/lance-geo/src/bbox.rs index 71537683bf6..9d4fb7871f0 100644 --- a/rust/lance-geo/src/bbox.rs +++ b/rust/lance-geo/src/bbox.rs @@ -97,8 +97,16 @@ impl BoundingBox { } pub fn add_rect(&mut self, rect: &impl RectTrait) { - self.add_coord(&rect.min()); - self.add_coord(&rect.max()); + let min = rect.min(); + let max = rect.max(); + + // Empty bounding boxes use inverted bounds, so they are the identity for a union. + if min.x() > max.x() || min.y() > max.y() { + return; + } + + self.add_coord(&min); + self.add_coord(&max); } pub fn add_polygon(&mut self, polygon: &impl PolygonTrait) { @@ -331,3 +339,30 @@ fn impl_total_bounds<'a>(arr: &'a impl GeoArrowArrayAccessor<'a>) -> ArrowResult Ok(bbox) } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_add_empty_rect_is_noop() { + let valid = BoundingBox::new_with_coords(&[ + geo_types::coord! { x: -1.0, y: -2.0 }, + geo_types::coord! { x: 3.0, y: 4.0 }, + ]); + let empty = BoundingBox::new(); + + let mut valid_then_empty = valid; + valid_then_empty.add_rect(&empty); + + let mut empty_then_valid = empty; + empty_then_valid.add_rect(&valid); + + for bbox in [valid_then_empty, empty_then_valid] { + assert_eq!(bbox.minx(), -1.0); + assert_eq!(bbox.miny(), -2.0); + assert_eq!(bbox.maxx(), 3.0); + assert_eq!(bbox.maxy(), 4.0); + } + } +} diff --git a/rust/lance-index-core/Cargo.toml b/rust/lance-index-core/Cargo.toml new file mode 100644 index 00000000000..5024f2eee0b --- /dev/null +++ b/rust/lance-index-core/Cargo.toml @@ -0,0 +1,33 @@ +[package] +name = "lance-index-core" +version.workspace = true +edition.workspace = true +authors.workspace = true +license.workspace = true +repository.workspace = true +readme = "README.md" +description = "Core traits and types for Lance index plugins" +keywords.workspace = true +categories.workspace = true +rust-version.workspace = true + +[dependencies] +async-trait.workspace = true +arrow-array.workspace = true +arrow-schema.workspace = true +arrow-select.workspace = true +bytes.workspace = true +datafusion-common.workspace = true +datafusion-expr.workspace = true +datafusion.workspace = true +futures.workspace = true +lance-core.workspace = true +lance-io.workspace = true +lance-select.workspace = true +prost-types.workspace = true +roaring.workspace = true +serde.workspace = true +serde_json.workspace = true + +[lints] +workspace = true diff --git a/rust/lance-index-core/README.md b/rust/lance-index-core/README.md new file mode 100644 index 00000000000..a954522a283 --- /dev/null +++ b/rust/lance-index-core/README.md @@ -0,0 +1,6 @@ +# lance-index-core + +`lance-index-core` is an internal sub-crate, containing the core traits and types used to +implement index plugins for [Lance](https://github.com/lance-format/lance). + +**Important Note**: This crate is **not intended for external usage**. diff --git a/rust/lance-index-core/src/lib.rs b/rust/lance-index-core/src/lib.rs new file mode 100644 index 00000000000..c949387b0b2 --- /dev/null +++ b/rust/lance-index-core/src/lib.rs @@ -0,0 +1,313 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright The Lance Authors + +use std::{any::Any, sync::Arc}; + +use async_trait::async_trait; +use lance_core::deepsize::DeepSizeOf; +use lance_core::{Error, Result}; +use roaring::RoaringBitmap; +use serde::{Deserialize, Serialize}; +use std::convert::TryFrom; + +pub mod metrics; +pub mod scalar; + +/// Generic methods common across all types of secondary indices +/// +#[async_trait] +pub trait Index: Send + Sync + DeepSizeOf { + /// Cast to [Any]. + fn as_any(&self) -> &dyn Any; + + /// Cast to [Index] + fn as_index(self: Arc) -> Arc; + + /// Retrieve index statistics as a JSON Value + fn statistics(&self) -> Result; + + /// Prewarm the index. + /// + /// This will load the index into memory and cache it. + async fn prewarm(&self) -> Result<()>; + + /// Get the type of the index + fn index_type(&self) -> IndexType; + + /// Read through the index and determine which fragment ids are covered by the index + /// + /// This is a kind of slow operation. It's better to use the fragment_bitmap. This + /// only exists for cases where the fragment_bitmap has become corrupted or missing. + async fn calculate_included_frags(&self) -> Result; +} + +/// Index Type +#[derive(Debug, PartialEq, Eq, Copy, Hash, Clone, DeepSizeOf, Serialize, Deserialize)] +pub enum IndexType { + // Preserve 0-100 for simple indices. + Scalar = 0, // Legacy scalar index, alias to BTree + + BTree = 1, // BTree + + Bitmap = 2, // Bitmap + + LabelList = 3, // LabelList + + Inverted = 4, // Inverted + + NGram = 5, // NGram + + FragmentReuse = 6, + + MemWal = 7, + + ZoneMap = 8, // ZoneMap + + BloomFilter = 9, // Bloom filter + + RTree = 10, // RTree + + Fm = 11, // FM-Index + + // 100+ and up for vector index. + /// Flat vector index. + Vector = 100, // Legacy vector index, alias to IvfPq + IvfFlat = 101, + IvfSq = 102, + IvfPq = 103, + IvfHnswSq = 104, + IvfHnswPq = 105, + IvfHnswFlat = 106, + IvfRq = 107, +} + +impl std::fmt::Display for IndexType { + fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + match self { + Self::Scalar | Self::BTree => write!(f, "BTree"), + Self::Bitmap => write!(f, "Bitmap"), + Self::LabelList => write!(f, "LabelList"), + Self::Inverted => write!(f, "Inverted"), + Self::NGram => write!(f, "NGram"), + Self::FragmentReuse => write!(f, "FragmentReuse"), + Self::MemWal => write!(f, "MemWal"), + Self::ZoneMap => write!(f, "ZoneMap"), + Self::BloomFilter => write!(f, "BloomFilter"), + Self::RTree => write!(f, "RTree"), + Self::Fm => write!(f, "Fm"), + Self::Vector | Self::IvfPq => write!(f, "IVF_PQ"), + Self::IvfFlat => write!(f, "IVF_FLAT"), + Self::IvfSq => write!(f, "IVF_SQ"), + Self::IvfHnswSq => write!(f, "IVF_HNSW_SQ"), + Self::IvfHnswPq => write!(f, "IVF_HNSW_PQ"), + Self::IvfHnswFlat => write!(f, "IVF_HNSW_FLAT"), + Self::IvfRq => write!(f, "IVF_RQ"), + } + } +} + +impl TryFrom for IndexType { + type Error = Error; + + fn try_from(value: i32) -> Result { + match value { + v if v == Self::Scalar as i32 => Ok(Self::Scalar), + v if v == Self::BTree as i32 => Ok(Self::BTree), + v if v == Self::Bitmap as i32 => Ok(Self::Bitmap), + v if v == Self::LabelList as i32 => Ok(Self::LabelList), + v if v == Self::NGram as i32 => Ok(Self::NGram), + v if v == Self::Inverted as i32 => Ok(Self::Inverted), + v if v == Self::FragmentReuse as i32 => Ok(Self::FragmentReuse), + v if v == Self::MemWal as i32 => Ok(Self::MemWal), + v if v == Self::ZoneMap as i32 => Ok(Self::ZoneMap), + v if v == Self::BloomFilter as i32 => Ok(Self::BloomFilter), + v if v == Self::RTree as i32 => Ok(Self::RTree), + v if v == Self::Fm as i32 => Ok(Self::Fm), + v if v == Self::Vector as i32 => Ok(Self::Vector), + v if v == Self::IvfFlat as i32 => Ok(Self::IvfFlat), + v if v == Self::IvfSq as i32 => Ok(Self::IvfSq), + v if v == Self::IvfPq as i32 => Ok(Self::IvfPq), + v if v == Self::IvfHnswSq as i32 => Ok(Self::IvfHnswSq), + v if v == Self::IvfHnswPq as i32 => Ok(Self::IvfHnswPq), + v if v == Self::IvfHnswFlat as i32 => Ok(Self::IvfHnswFlat), + v if v == Self::IvfRq as i32 => Ok(Self::IvfRq), + _ => Err(Error::invalid_input_source( + format!("the input value {} is not a valid IndexType", value).into(), + )), + } + } +} + +impl TryFrom<&str> for IndexType { + type Error = Error; + + fn try_from(value: &str) -> Result { + match value { + "BTree" | "BTREE" => Ok(Self::BTree), + "Bitmap" | "BITMAP" => Ok(Self::Bitmap), + "LabelList" | "LABELLIST" => Ok(Self::LabelList), + "Inverted" | "INVERTED" => Ok(Self::Inverted), + "NGram" | "NGRAM" => Ok(Self::NGram), + "ZoneMap" | "ZONEMAP" => Ok(Self::ZoneMap), + "BloomFilter" | "BLOOMFILTER" | "BLOOM_FILTER" => Ok(Self::BloomFilter), + "RTree" | "RTREE" | "R_TREE" => Ok(Self::RTree), + "Fm" | "FM" => Ok(Self::Fm), + "Vector" | "VECTOR" => Ok(Self::Vector), + "IVF_FLAT" => Ok(Self::IvfFlat), + "IVF_SQ" => Ok(Self::IvfSq), + "IVF_PQ" => Ok(Self::IvfPq), + "IVF_RQ" => Ok(Self::IvfRq), + "IVF_HNSW_FLAT" => Ok(Self::IvfHnswFlat), + "IVF_HNSW_SQ" => Ok(Self::IvfHnswSq), + "IVF_HNSW_PQ" => Ok(Self::IvfHnswPq), + "FragmentReuse" => Ok(Self::FragmentReuse), + "MemWal" => Ok(Self::MemWal), + _ => Err(Error::invalid_input(format!( + "invalid index type: {}", + value + ))), + } + } +} + +impl IndexType { + pub fn is_scalar(&self) -> bool { + matches!( + self, + Self::Scalar + | Self::BTree + | Self::Bitmap + | Self::LabelList + | Self::Inverted + | Self::NGram + | Self::ZoneMap + | Self::BloomFilter + | Self::RTree + | Self::Fm, + ) + } + + pub fn is_vector(&self) -> bool { + matches!( + self, + Self::Vector + | Self::IvfPq + | Self::IvfHnswSq + | Self::IvfHnswPq + | Self::IvfHnswFlat + | Self::IvfFlat + | Self::IvfSq + | Self::IvfRq + ) + } + + pub fn is_system(&self) -> bool { + matches!(self, Self::FragmentReuse | Self::MemWal) + } + + /// Returns the current format version of the index type, + /// bump this when the index format changes. + /// Indices which higher version than these will be ignored for compatibility, + /// This would happen when creating index in a newer version of Lance, + /// but then opening the index in older version of Lance + pub fn version(&self) -> i32 { + match self { + Self::Scalar => 0, + Self::BTree => 0, + Self::Bitmap => 0, + Self::LabelList => 0, + Self::Inverted => 0, + Self::NGram => 0, + Self::FragmentReuse => 0, + Self::MemWal => 0, + Self::ZoneMap => 0, + Self::BloomFilter => 0, + Self::RTree => 0, + Self::Fm => 0, + + // IMPORTANT: if any vector index subtype needs a format bump that is + // not backward compatible, its new version must be set to + // (current max vector index version + 1), even if only one subtype + // changed. Compatibility filtering currently cannot distinguish vector + // subtypes from details-only metadata, so vector versions effectively + // share one global monotonic compatibility level. + Self::Vector + | Self::IvfFlat + | Self::IvfSq + | Self::IvfPq + | Self::IvfHnswSq + | Self::IvfHnswPq + | Self::IvfHnswFlat => 1, + Self::IvfRq => 2, + } + } + + /// Returns the target partition size for the index type. + /// + /// This is used to compute the number of partitions for the index. + /// The partition size is optimized for the best performance of the index. + /// + /// This is for vector indices only. + pub fn target_partition_size(&self) -> usize { + match self { + Self::Vector => 8192, + Self::IvfFlat => 4096, + Self::IvfSq => 8192, + Self::IvfPq => 8192, + Self::IvfRq => 4096, + Self::IvfHnswFlat => 1 << 20, + Self::IvfHnswSq => 1 << 20, + Self::IvfHnswPq => 1 << 20, + _ => 8192, + } + } + + /// Returns the highest supported vector index version in this Lance build. + pub fn max_vector_version() -> u32 { + [ + Self::Vector, + Self::IvfFlat, + Self::IvfSq, + Self::IvfPq, + Self::IvfHnswSq, + Self::IvfHnswPq, + Self::IvfHnswFlat, + Self::IvfRq, + ] + .into_iter() + .map(|index_type| index_type.version() as u32) + .max() + .unwrap_or(1) + } + + pub fn matches_details(&self, details: &prost_types::Any) -> bool { + let url = &details.type_url; + match self { + Self::Scalar | Self::BTree => url.ends_with("BTreeIndexDetails"), + Self::Bitmap => url.ends_with("BitmapIndexDetails"), + Self::LabelList => url.ends_with("LabelListIndexDetails"), + Self::Inverted => url.ends_with("InvertedIndexDetails"), + Self::NGram => url.ends_with("NGramIndexDetails"), + Self::ZoneMap => url.ends_with("ZoneMapIndexDetails"), + Self::BloomFilter => url.ends_with("BloomFilterIndexDetails"), + Self::RTree => url.ends_with("RTreeIndexDetails"), + Self::Fm => url.ends_with("FMIndexDetails"), + Self::FragmentReuse => url.ends_with("FragmentReuseIndexDetails"), + Self::MemWal => url.ends_with("MemWalIndexDetails"), + Self::Vector + | Self::IvfFlat + | Self::IvfSq + | Self::IvfPq + | Self::IvfHnswSq + | Self::IvfHnswPq + | Self::IvfHnswFlat + | Self::IvfRq => url.ends_with("VectorIndexDetails"), + } + } +} + +pub trait IndexParams: Send + Sync { + fn as_any(&self) -> &dyn Any; + + fn index_name(&self) -> &str; +} diff --git a/rust/lance-index-core/src/metrics.rs b/rust/lance-index-core/src/metrics.rs new file mode 100644 index 00000000000..8259ace5299 --- /dev/null +++ b/rust/lance-index-core/src/metrics.rs @@ -0,0 +1,255 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright The Lance Authors + +use std::sync::atomic::{AtomicUsize, Ordering}; + +pub const AND_CANDIDATES_SEEN_METRIC: &str = "and_candidates_seen"; +pub const AND_CANDIDATES_PRUNED_BEFORE_RETURN_METRIC: &str = "and_candidates_pruned_before_return"; +pub const AND_FULL_SCORES_METRIC: &str = "and_full_scores"; +pub const FREQS_COLLECTED_METRIC: &str = "freqs_collected"; +pub const COMPOUND_ADDRESSES_RESOLVED_METRIC: &str = "compound_addresses_resolved"; +pub const COMPOUND_ADDRESS_RESOLUTION_BATCHES_METRIC: &str = "compound_address_resolution_batches"; +pub const COMPOUND_PEAK_ADDRESS_RESOLUTION_BATCH_SIZE_METRIC: &str = + "compound_peak_address_resolution_batch_size"; +pub const COMPOUND_SCORE_FLOOR_OVERFLOWS_METRIC: &str = "compound_score_floor_overflows"; +pub const COMPOUND_PEAK_BUFFERED_CANDIDATES_METRIC: &str = "compound_peak_buffered_candidates"; + +/// A trait used by the index to report metrics +/// +/// Callers can implement this trait to collect metrics +pub trait MetricsCollector: Send + Sync { + /// Record partition loads + /// + /// Many indices consist of partitions that may need to be loaded + /// into cache. For example, an inverted index or ngram index has a + /// posting list for each token. + /// + /// In the ideal case, these shards are in the cache and will not need + /// to be loaded from disk. This method should not be called if the + /// shard is in the cache. + fn record_parts_loaded(&self, num_parts: usize); + + /// Record a shard load + fn record_part_load(&self) { + self.record_parts_loaded(1); + } + + /// Record an index load + /// + /// This should be called when a scalar index is loaded from storage. + /// It should not be called if the index is already in memory. + fn record_index_loads(&self, num_indexes: usize); + + /// Record an index load + fn record_index_load(&self) { + self.record_index_loads(1); + } + + /// Record the number of "comparisons" made by the index + /// + /// What exactly constitutes a comparison depends on the index type. + /// For example, a B-tree index may make comparisons while searching for a value. + /// On the other hand, a bitmap index makes comparisons when computing the intersection + /// of two bitmaps. + /// + /// The goal is to provide some visibility into the compute cost of the search + fn record_comparisons(&self, num_comparisons: usize); + + /// Record index cache hits observed while serving this query. + /// + /// A "hit" is one page-level lookup (partition, posting list, BTree page, etc.) + /// that was served from the in-memory index cache without touching storage. + fn record_index_cache_hits(&self, _num_hits: usize) {} + + /// Convenience for a single cache hit. + fn record_index_cache_hit(&self) { + self.record_index_cache_hits(1); + } + + /// Record index cache misses observed while serving this query. + /// + /// A "miss" is one page-level lookup that had to be loaded from storage + /// because it was not present in the cache. + fn record_index_cache_misses(&self, _num_misses: usize) {} + + /// Convenience for a single cache miss. + fn record_index_cache_miss(&self) { + self.record_index_cache_misses(1); + } + + /// Record AND candidates returned from WAND alignment to the scoring loop. + /// + /// This excludes candidates pruned before `next()` returns. Use this with + /// `record_and_candidates_pruned_before_return` to recover total aligned + /// AND candidates. + fn record_and_candidates_seen(&self, _num_candidates: usize) {} + + /// Record AND candidates pruned during WAND alignment before `next()` returns. + fn record_and_candidates_pruned_before_return(&self, _num_candidates: usize) {} + + fn record_and_full_scores(&self, _num_scores: usize) {} + + fn record_freqs_collected(&self, _num_collections: usize) {} + + /// Record compound FTS document addresses resolved for final row-ID ties. + fn record_compound_addresses_resolved(&self, _num_addresses: usize) {} + + /// Record bounded compound FTS address-resolution batches. + fn record_compound_address_resolution_batches(&self, _num_batches: usize) {} + + /// Record the largest compound FTS address-resolution batch. + fn record_compound_peak_address_resolution_batch_size(&self, _num_addresses: usize) {} + + /// Record unresolved score floors that required a resolved-key retry. + fn record_compound_score_floor_overflows(&self, _num_overflows: usize) {} + + /// Record a candidate-buffer high-water mark for compound FTS. + fn record_compound_peak_buffered_candidates(&self, _num_candidates: usize) {} + + /// Returns an optional sink for recording exact I/O statistics (bytes read, + /// IOPS, and requests) performed on behalf of this collector. + /// + /// Index implementations that read from a + /// [`lance_io::scheduler::ScanScheduler`] can attach the returned handle to + /// their file readers so the I/O performed for a single query is measured + /// and attributed here. The default returns `None`, meaning the caller does + /// not want I/O measured (and index implementations should then take their + /// normal, uninstrumented read path). + fn io_stats(&self) -> Option { + None + } +} + +/// A no-op metrics collector that does nothing +pub struct NoOpMetricsCollector; + +impl MetricsCollector for NoOpMetricsCollector { + fn record_parts_loaded(&self, _num_parts: usize) {} + fn record_index_loads(&self, _num_indexes: usize) {} + fn record_comparisons(&self, _num_comparisons: usize) {} +} + +#[derive(Default)] +pub struct LocalMetricsCollector { + pub parts_loaded: AtomicUsize, + pub index_loads: AtomicUsize, + pub comparisons: AtomicUsize, + // Kept `pub(crate)` so that adding new metric fields to this public struct + // does not break downstream callers that construct or destructure the + // existing three fields. Callers can still read cumulative values via + // [`Self::index_cache_hits`] / [`Self::index_cache_misses`]. + pub(crate) index_cache_hits: AtomicUsize, + pub(crate) index_cache_misses: AtomicUsize, +} + +impl LocalMetricsCollector { + pub fn dump_into(self, other: &dyn MetricsCollector) { + other.record_parts_loaded(self.parts_loaded.load(Ordering::Relaxed)); + other.record_index_loads(self.index_loads.load(Ordering::Relaxed)); + other.record_comparisons(self.comparisons.load(Ordering::Relaxed)); + other.record_index_cache_hits(self.index_cache_hits.load(Ordering::Relaxed)); + other.record_index_cache_misses(self.index_cache_misses.load(Ordering::Relaxed)); + } + + /// Cumulative index cache hits recorded so far. + pub fn index_cache_hits(&self) -> usize { + self.index_cache_hits.load(Ordering::Relaxed) + } + + /// Cumulative index cache misses recorded so far. + pub fn index_cache_misses(&self) -> usize { + self.index_cache_misses.load(Ordering::Relaxed) + } +} + +impl MetricsCollector for LocalMetricsCollector { + fn record_parts_loaded(&self, num_parts: usize) { + self.parts_loaded.fetch_add(num_parts, Ordering::Relaxed); + } + + fn record_index_loads(&self, num_indexes: usize) { + self.index_loads.fetch_add(num_indexes, Ordering::Relaxed); + } + + fn record_comparisons(&self, num_comparisons: usize) { + self.comparisons + .fetch_add(num_comparisons, Ordering::Relaxed); + } + + fn record_index_cache_hits(&self, num_hits: usize) { + self.index_cache_hits.fetch_add(num_hits, Ordering::Relaxed); + } + + fn record_index_cache_misses(&self, num_misses: usize) { + self.index_cache_misses + .fetch_add(num_misses, Ordering::Relaxed); + } +} + +#[cfg(test)] +mod tests { + use super::*; + + struct SumSink { + parts: AtomicUsize, + loads: AtomicUsize, + comparisons: AtomicUsize, + hits: AtomicUsize, + misses: AtomicUsize, + } + + impl MetricsCollector for SumSink { + fn record_parts_loaded(&self, n: usize) { + self.parts.fetch_add(n, Ordering::Relaxed); + } + fn record_index_loads(&self, n: usize) { + self.loads.fetch_add(n, Ordering::Relaxed); + } + fn record_comparisons(&self, n: usize) { + self.comparisons.fetch_add(n, Ordering::Relaxed); + } + fn record_index_cache_hits(&self, n: usize) { + self.hits.fetch_add(n, Ordering::Relaxed); + } + fn record_index_cache_misses(&self, n: usize) { + self.misses.fetch_add(n, Ordering::Relaxed); + } + } + + #[test] + fn local_metrics_collector_forwards_cache_counts() { + let local = LocalMetricsCollector::default(); + local.record_index_cache_hit(); + local.record_index_cache_hit(); + local.record_index_cache_misses(3); + local.record_part_load(); + local.record_index_load(); + local.record_comparisons(5); + + let sink = SumSink { + parts: AtomicUsize::new(0), + loads: AtomicUsize::new(0), + comparisons: AtomicUsize::new(0), + hits: AtomicUsize::new(0), + misses: AtomicUsize::new(0), + }; + local.dump_into(&sink); + + assert_eq!(sink.parts.load(Ordering::Relaxed), 1); + assert_eq!(sink.loads.load(Ordering::Relaxed), 1); + assert_eq!(sink.comparisons.load(Ordering::Relaxed), 5); + assert_eq!(sink.hits.load(Ordering::Relaxed), 2); + assert_eq!(sink.misses.load(Ordering::Relaxed), 3); + } + + #[test] + fn no_op_metrics_collector_ignores_cache_counts() { + // Ensures existing implementors that do not override cache-count methods + // remain sound (default impl is a no-op). + let collector = NoOpMetricsCollector; + collector.record_index_cache_hit(); + collector.record_index_cache_miss(); + collector.record_index_cache_hits(10); + collector.record_index_cache_misses(20); + } +} diff --git a/rust/lance-index-core/src/scalar.rs b/rust/lance-index-core/src/scalar.rs new file mode 100644 index 00000000000..49b8e949905 --- /dev/null +++ b/rust/lance-index-core/src/scalar.rs @@ -0,0 +1,595 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright The Lance Authors + +//! Abstract scalar index traits and types for Lance index plugins + +use arrow_array::{BooleanArray, RecordBatch, UInt64Array}; +use arrow_schema::{DataType, Schema}; +use async_trait::async_trait; +use bytes::Bytes; +use datafusion::physical_plan::SendableRecordBatchStream; +use datafusion_common::scalar::ScalarValue; +use datafusion_expr::Expr; +use lance_core::deepsize::DeepSizeOf; +use lance_core::utils::row_addr_remap::RowAddrRemap; +use lance_core::{Error, Result}; +use lance_io::stream::{RecordBatchStream, RecordBatchStreamAdapter}; +use lance_select::{NullableRowAddrSet, RowAddrTreeMap, RowSetOps}; +use roaring::{RoaringBitmap, RoaringTreemap}; +use serde::Serialize; +use std::collections::HashMap; +use std::fmt::Debug; +use std::pin::Pin; +use std::{any::Any, sync::Arc}; + +use crate::metrics::MetricsCollector; +use crate::{Index, IndexParams, IndexType}; + +/// Metadata about a single file within an index segment. +#[derive(Debug, Clone, PartialEq, DeepSizeOf)] +pub struct IndexFile { + /// Path relative to the index directory (e.g., "index.idx", "auxiliary.idx") + pub path: String, + /// Size of the file in bytes + pub size_bytes: u64, +} + +pub const LANCE_SCALAR_INDEX: &str = "__lance_scalar_index"; + +/// Builtin index types supported by the Lance library +/// +/// This is primarily for convenience to avoid a bunch of string +/// constants and provide some auto-complete. This type should not +/// be used in the manifest as plugins cannot add new entries. +#[derive(Debug, Clone, PartialEq, Eq, DeepSizeOf)] +pub enum BuiltinIndexType { + BTree, + Bitmap, + LabelList, + NGram, + ZoneMap, + BloomFilter, + RTree, + Inverted, + Fm, +} + +impl BuiltinIndexType { + pub fn as_str(&self) -> &str { + match self { + Self::BTree => "btree", + Self::Bitmap => "bitmap", + Self::LabelList => "labellist", + Self::NGram => "ngram", + Self::ZoneMap => "zonemap", + Self::Inverted => "inverted", + Self::BloomFilter => "bloomfilter", + Self::RTree => "rtree", + Self::Fm => "fm", + } + } +} + +impl TryFrom for BuiltinIndexType { + type Error = Error; + + fn try_from(value: IndexType) -> Result { + match value { + IndexType::BTree => Ok(Self::BTree), + IndexType::Bitmap => Ok(Self::Bitmap), + IndexType::LabelList => Ok(Self::LabelList), + IndexType::NGram => Ok(Self::NGram), + IndexType::ZoneMap => Ok(Self::ZoneMap), + IndexType::Inverted => Ok(Self::Inverted), + IndexType::BloomFilter => Ok(Self::BloomFilter), + IndexType::RTree => Ok(Self::RTree), + IndexType::Fm => Ok(Self::Fm), + _ => Err(Error::index("Invalid index type".to_string())), + } + } +} + +#[derive(Debug, Clone, PartialEq)] +pub struct ScalarIndexParams { + /// The type of index to create + /// + /// Plugins may add additional index types. Index type lookup is case-insensitive. + pub index_type: String, + /// The parameters to train the index + /// + /// This should be a JSON string. The contents of the JSON string will be specific to the + /// index type. If not set, then default parameters will be used for the index type. + pub params: Option, +} + +impl Default for ScalarIndexParams { + fn default() -> Self { + Self { + index_type: BuiltinIndexType::BTree.as_str().to_string(), + params: None, + } + } +} + +impl ScalarIndexParams { + /// Creates a new ScalarIndexParams from one of the builtin index types + pub fn for_builtin(index_type: BuiltinIndexType) -> Self { + Self { + index_type: index_type.as_str().to_string(), + params: None, + } + } + + /// Create a new ScalarIndexParams with the given index type + pub fn new(index_type: String) -> Self { + Self { + index_type, + params: None, + } + } + + /// Set the parameters for the index + pub fn with_params(mut self, params: &ParamsType) -> Self { + self.params = Some(serde_json::to_string(params).unwrap()); + self + } +} + +impl IndexParams for ScalarIndexParams { + fn as_any(&self) -> &dyn std::any::Any { + self + } + + fn index_name(&self) -> &str { + LANCE_SCALAR_INDEX + } +} + +/// Trait for storing an index (or parts of an index) into storage +#[async_trait] +pub trait IndexWriter: Send { + /// Writes a record batch into the file, returning the 0-based index of the batch in the file + /// + /// E.g. if this is the third time this is called this method will return 2 + async fn write_record_batch(&mut self, batch: RecordBatch) -> Result; + /// Adds a global buffer and returns its index. + async fn add_global_buffer(&mut self, _data: Bytes) -> Result { + Err(Error::not_supported( + "global buffers are not supported by this index writer", + )) + } + /// Finishes writing the file and closes the file + async fn finish(&mut self) -> Result; + /// Finishes writing the file and closes the file with additional metadata + async fn finish_with_metadata( + &mut self, + metadata: HashMap, + ) -> Result; +} + +/// Trait for reading an index (or parts of an index) from storage +#[async_trait] +pub trait IndexReader: Send + Sync { + /// Read the n-th record batch from the file + async fn read_record_batch(&self, n: u64, batch_size: u64) -> Result; + /// Reads a global buffer by index. + async fn read_global_buffer(&self, _index: u32) -> Result { + Err(Error::not_supported( + "global buffers are not supported by this index reader", + )) + } + /// Read the range of rows from the file. + /// If projection is Some, only return the columns in the projection, + /// nested columns like Some(&["x.y"]) are not supported. + /// If projection is None, return all columns. + async fn read_range( + &self, + range: std::ops::Range, + projection: Option<&[&str]>, + ) -> Result; + /// Read multiple ranges and concatenate into a single batch. + /// Default impl runs `read_range`s in parallel via `try_join_all`. + async fn read_ranges( + &self, + ranges: &[std::ops::Range], + projection: Option<&[&str]>, + ) -> Result { + if ranges.is_empty() { + return self.read_range(0..0, projection).await; + } + let futures = ranges + .iter() + .map(|r| self.read_range(r.clone(), projection)); + let batches = futures::future::try_join_all(futures).await?; + let schema = batches[0].schema(); + Ok(arrow_select::concat::concat_batches(&schema, &batches)?) + } + /// Read a range of rows as a stream of record batches. + /// + /// This allows the caller to process rows incrementally without loading the + /// entire range into memory at once. + /// + /// The default implementation falls back to [`Self::read_range`] and wraps + /// the result in a single-item stream. + async fn read_range_stream( + &self, + range: std::ops::Range, + projection: Option<&[&str]>, + ) -> Result>> { + let batch = self.read_range(range, projection).await?; + let schema = batch.schema(); + Ok(Box::pin(RecordBatchStreamAdapter::new( + schema, + futures::stream::once(async move { Ok(batch) }), + ))) + } + /// Return the number of batches in the file + async fn num_batches(&self, batch_size: u64) -> u32; + /// Return the number of rows in the file + fn num_rows(&self) -> usize; + /// Return the metadata of the file + fn schema(&self) -> &lance_core::datatypes::Schema; + /// Best-effort on-disk byte size of the file when the reader already knows it + /// without extra I/O, else `None`. Used to size prewarm chunks. + fn file_size_bytes(&self) -> Option { + None + } +} + +/// Trait abstracting I/O away from index logic +/// +/// Scalar indices are currently serialized as indexable arrow record batches stored in +/// named "files". The index store is responsible for serializing and deserializing +/// these batches into file data (e.g. as .lance files or .parquet files, etc.) +#[async_trait] +pub trait IndexStore: std::fmt::Debug + Send + Sync + DeepSizeOf { + fn as_any(&self) -> &dyn Any; + fn clone_arc(&self) -> Arc; + + /// Suggested I/O parallelism for the store + fn io_parallelism(&self) -> usize; + + /// Create a new file and return a writer to store data in the file + async fn new_index_file(&self, name: &str, schema: Arc) + -> Result>; + + /// Open an existing file for retrieval + async fn open_index_file(&self, name: &str) -> Result>; + + /// Return a store that submits its I/O at the given base priority. + fn with_io_priority(&self, io_priority: u64) -> Arc; + + /// Copy a range of batches from an index file from this store to another + /// + /// This is often useful when remapping or updating + async fn copy_index_file(&self, name: &str, dest_store: &dyn IndexStore) -> Result; + + /// Copy an index file from this store to a new name in another store, leaving the source intact + async fn copy_index_file_to( + &self, + name: &str, + new_name: &str, + dest_store: &dyn IndexStore, + ) -> Result { + if name == new_name { + self.copy_index_file(name, dest_store).await + } else { + Err(Error::not_supported(format!( + "copying index file {name} to {new_name} is not supported by this index store" + ))) + } + } + + /// Rename an index file + async fn rename_index_file(&self, name: &str, new_name: &str) -> Result; + + /// Delete an index file (used in the tmp spill store to keep tmp size down) + async fn delete_index_file(&self, name: &str) -> Result<()>; + + /// List all files in the index directory with their sizes. + /// + /// Returns a list of (relative_path, size_bytes) tuples. + /// Used to capture file metadata after index creation/modification. + async fn list_files_with_sizes(&self) -> Result>; +} + +/// Different scalar indices may support different kinds of queries +/// +/// For example, a btree index can support a wide range of queries (e.g. x > 7) +/// while an index based on FTS only supports queries like "x LIKE 'foo'" +/// +/// This trait is used when we need an object that can represent any kind of query +/// +/// Note: if you are implementing this trait for a query type then you probably also +/// need to implement the scalar query parser trait to create instances of your query at parse time. +pub trait AnyQuery: std::fmt::Debug + Any + Send + Sync { + /// Cast the query as Any to allow for downcasting + fn as_any(&self) -> &dyn Any; + /// Format the query as a string for display purposes + fn format(&self, col: &str) -> String; + /// Convert the query to a datafusion expression + fn to_expr(&self, col: String) -> Expr; + /// Compare this query to another query + fn dyn_eq(&self, other: &dyn AnyQuery) -> bool; +} + +impl PartialEq for dyn AnyQuery { + fn eq(&self, other: &Self) -> bool { + self.dyn_eq(other) + } +} + +/// The result of a search operation against a scalar index +#[derive(Debug, PartialEq)] +pub enum SearchResult { + /// The exact row ids that satisfy the query + Exact(NullableRowAddrSet), + /// Any row id satisfying the query will be in this set but not every + /// row id in this set will satisfy the query, a further recheck step + /// is needed + AtMost(NullableRowAddrSet), + /// All of the given row ids satisfy the query but there may be more + /// + /// No scalar index actually returns this today but it can arise from + /// boolean operations (e.g. NOT(AtMost(x)) == AtLeast(NOT(x))) + AtLeast(NullableRowAddrSet), +} + +impl SearchResult { + pub fn exact(row_ids: impl Into) -> Self { + Self::Exact(NullableRowAddrSet::new(row_ids.into(), Default::default())) + } + + pub fn at_most(row_ids: impl Into) -> Self { + Self::AtMost(NullableRowAddrSet::new(row_ids.into(), Default::default())) + } + + pub fn at_least(row_ids: impl Into) -> Self { + Self::AtLeast(NullableRowAddrSet::new(row_ids.into(), Default::default())) + } + + pub fn with_nulls(self, nulls: impl Into) -> Self { + match self { + Self::Exact(row_ids) => Self::Exact(row_ids.with_nulls(nulls.into())), + Self::AtMost(row_ids) => Self::AtMost(row_ids.with_nulls(nulls.into())), + Self::AtLeast(row_ids) => Self::AtLeast(row_ids.with_nulls(nulls.into())), + } + } + + pub fn row_addrs(&self) -> &NullableRowAddrSet { + match self { + Self::Exact(row_addrs) => row_addrs, + Self::AtMost(row_addrs) => row_addrs, + Self::AtLeast(row_addrs) => row_addrs, + } + } + + pub fn is_exact(&self) -> bool { + matches!(self, Self::Exact(_)) + } +} + +/// Brief information about an index that was created +pub struct CreatedIndex { + /// The details of the index that was created + /// + /// These should be stored somewhere as they will be needed to + /// load the index later. + pub index_details: prost_types::Any, + /// The version of the index that was created + /// + /// This can be used to determine if a reader is able to load the index. + pub index_version: u32, + /// List of files and their sizes for this index + /// + /// This enables skipping HEAD calls when opening indices and provides + /// visibility into index storage size via describe_indices(). + pub files: Vec, +} + +/// The ordering that training data must satisfy +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum TrainingOrdering { + /// The input will arrive sorted by the value column in ascending order + Values, + /// The input will arrive sorted by the address column in ascending order + Addresses, + /// The input will arrive in an arbitrary order + None, +} + +#[derive(Debug, Clone)] +pub struct TrainingCriteria { + pub ordering: TrainingOrdering, + pub needs_row_ids: bool, + pub needs_row_addrs: bool, +} + +impl TrainingCriteria { + pub fn new(ordering: TrainingOrdering) -> Self { + Self { + ordering, + needs_row_ids: false, + needs_row_addrs: false, + } + } + + pub fn with_row_id(mut self) -> Self { + self.needs_row_ids = true; + self + } + + pub fn with_row_addr(mut self) -> Self { + self.needs_row_addrs = true; + self + } +} + +/// The criteria that specifies how to update an index +pub struct UpdateCriteria { + /// If true, then we need to read the old data to update the index + /// + /// This should be avoided if possible but is left in for some legacy paths + pub requires_old_data: bool, + /// The criteria required for data (both old and new) + pub data_criteria: TrainingCriteria, +} + +/// Filter used when merging existing scalar-index rows during update. +/// +/// The caller must pick a filter mode that matches the row-id semantics of the +/// dataset: +/// - address-style row IDs: fragment filtering is valid +/// - stable row IDs: use exact row-id membership instead +#[derive(Debug, Clone)] +pub enum OldIndexDataFilter { + /// Keeps track of which fragments are still valid and which are no longer valid. + /// + /// This is valid for address-style row IDs. + Fragments { + to_keep: RoaringBitmap, + to_remove: RoaringBitmap, + }, + /// Keep old rows whose row IDs are in this exact allow-list. + /// + /// This is required for stable row IDs, where row IDs are opaque and + /// should not be interpreted as encoded row addresses. + RowIds(RowAddrTreeMap), +} + +impl OldIndexDataFilter { + /// Build a boolean mask that keeps only row IDs selected by this filter. + pub fn filter_row_ids(&self, row_ids: &UInt64Array) -> BooleanArray { + match self { + Self::Fragments { to_keep, .. } => row_ids + .iter() + .map(|id| id.map(|id| to_keep.contains((id >> 32) as u32))) + .collect(), + Self::RowIds(valid_row_ids) => row_ids + .iter() + .map(|id| id.map(|id| valid_row_ids.contains(id))) + .collect(), + } + } + + /// Apply this filter in place to a set of existing (old) row ids/addresses, + /// retaining only the rows the filter selects to keep. Used by index types + /// that merge old postings directly (e.g. bitmap) instead of re-scanning a + /// row-id array through [`Self::filter_row_ids`]. + pub fn retain_old_rows(&self, rows: &mut RowAddrTreeMap) { + match self { + Self::Fragments { to_keep, .. } => rows.retain_fragments(to_keep.iter()), + Self::RowIds(valid_row_ids) => *rows &= valid_row_ids, + } + } +} + +impl UpdateCriteria { + pub fn requires_old_data(data_criteria: TrainingCriteria) -> Self { + Self { + requires_old_data: true, + data_criteria, + } + } + + pub fn only_new_data(data_criteria: TrainingCriteria) -> Self { + Self { + requires_old_data: false, + data_criteria, + } + } +} + +/// A trait for a scalar index, a structure that can determine row ids that satisfy scalar queries +#[async_trait] +pub trait ScalarIndex: Send + Sync + std::fmt::Debug + Index + DeepSizeOf { + /// Search the scalar index + /// + /// Returns all row ids that satisfy the query, these row ids are not necessarily ordered + async fn search( + &self, + query: &dyn AnyQuery, + metrics: &dyn MetricsCollector, + ) -> Result; + + /// Returns true if this index reports matches as physical row addresses + /// (`fragment_id << 32 | offset`) rather than row ids + /// + /// Address-domain indices (e.g. zone map, bloom filter) are built over the + /// `_rowaddr` column. On a dataset with stable row ids the address and + /// row-id domains diverge, so these results must be translated back to row + /// ids (via the per-fragment row-id sequences, known only at the dataset + /// layer) before they are combined with row-id results or handed to the + /// scan. The default (row-id domain) needs no translation. + fn results_are_row_addresses(&self) -> bool { + false + } + + /// Returns true if the remap operation is supported + fn can_remap(&self) -> bool; + + /// Remap the row ids, creating a new remapped version of this index in `dest_store` + async fn remap( + &self, + mapping: &RowAddrRemap, + dest_store: &dyn IndexStore, + ) -> Result; + + /// Add the new data into the index, creating an updated version of the index in `dest_store` + /// + /// If `old_data_filter` is provided, old index data will be filtered before + /// merge according to the chosen filter mode. + async fn update( + &self, + new_data: SendableRecordBatchStream, + dest_store: &dyn IndexStore, + old_data_filter: Option, + ) -> Result; + + /// Returns the criteria that will be used to update the index + fn update_criteria(&self) -> UpdateCriteria; + + /// Derive the index parameters from the current index + /// + /// This returns a ScalarIndexParams that can be used to recreate an index + /// with the same configuration on another dataset. + fn derive_index_params(&self) -> Result; + + /// Returns the value type expected by [`Self::update`], when the index has + /// a durable type contract for its training data. + /// + /// Wrapper indices use this to transform new data to the same type as the + /// loaded index instead of inferring a potentially different type from an + /// update batch. Index types without such a contract may return `None`. + fn training_data_type(&self) -> Option { + None + } + + /// Global `[min, max]` of the indexed column from index metadata, without a + /// scan, or `None` if this index type cannot supply a sound bound. When + /// `Some`, the range is a superset of live values (conservative under + /// deletes): safe to prune with, not guaranteed tight. + fn value_range(&self) -> Option<(ScalarValue, ScalarValue)> { + None + } +} + +/// Abstraction over any type that can remap row IDs during index loading. +/// +/// This decouples scalar index plugins from the table-level frag reuse index type. +/// The frag reuse index implements this trait, but callers may also supply custom +/// implementations for testing or other remapping strategies. +pub trait RowIdRemapper: Send + Sync + std::fmt::Debug { + /// Remap a single row id. Returns `None` if the row was deleted. + fn remap_row_id(&self, row_id: u64) -> Option; + /// Remap all addresses in a [`RowAddrTreeMap`], dropping deleted rows. + fn remap_row_addrs_tree_map(&self, row_addrs: &RowAddrTreeMap) -> RowAddrTreeMap; + /// Remap all row ids in a [`RoaringTreemap`], dropping deleted rows. + fn remap_row_ids_roaring_tree_map(&self, row_ids: &RoaringTreemap) -> RoaringTreemap; + /// Remap the row-id column at `row_id_idx` inside `batch`, dropping deleted rows. + fn remap_row_ids_record_batch( + &self, + batch: RecordBatch, + row_id_idx: usize, + ) -> Result; +} diff --git a/rust/lance-index/Cargo.toml b/rust/lance-index/Cargo.toml index b9d1a5fde29..ec128ee1b85 100644 --- a/rust/lance-index/Cargo.toml +++ b/rust/lance-index/Cargo.toml @@ -12,9 +12,10 @@ categories.workspace = true rust-version.workspace = true [dependencies] -arc-swap.workspace = true +arc-swap = { workspace = true, features = ["weak"] } arrow.workspace = true arrow-array.workspace = true +arrow-ipc.workspace = true arrow-ord.workspace = true arrow-schema.workspace = true arrow-select.workspace = true @@ -39,6 +40,7 @@ jsonb.workspace = true lance-arrow.workspace = true lance-arrow-stats.workspace = true lance-core.workspace = true +lance-index-core.workspace = true lance-datafusion.workspace = true lance-encoding.workspace = true lance-file.workspace = true diff --git a/rust/lance-index/README.md b/rust/lance-index/README.md index 12f1f824936..fc7cf0e9517 100644 --- a/rust/lance-index/README.md +++ b/rust/lance-index/README.md @@ -1,7 +1,7 @@ # lance-index `lance-index` is an internal sub-crate, containing various vector index implementations -used in [Lance](https://github.com/lancedb/lance). +used in [Lance](https://github.com/lance-format/lance). **Important Note**: This crate is **not intended for external usage**. diff --git a/rust/lance-index/benches/hnsw.rs b/rust/lance-index/benches/hnsw.rs index 0a9b10bf42c..5884410ad9a 100644 --- a/rust/lance-index/benches/hnsw.rs +++ b/rust/lance-index/benches/hnsw.rs @@ -53,6 +53,7 @@ fn bench_hnsw(c: &mut Criterion) { lower_bound: None, upper_bound: None, dist_q_c: 0.0, + use_acorn: false, }, None, vectors.as_ref(), @@ -81,6 +82,7 @@ fn bench_hnsw(c: &mut Criterion) { lower_bound: None, upper_bound: None, dist_q_c: 0.0, + use_acorn: false, }, None, vectors.as_ref(), @@ -141,6 +143,7 @@ fn bench_hnsw_load(c: &mut Criterion) { lower_bound: None, upper_bound: None, dist_q_c: 0.0, + use_acorn: false, }, None, vectors.as_ref(), @@ -206,6 +209,7 @@ fn bench_hnsw_sq(c: &mut Criterion) { lower_bound: None, upper_bound: None, dist_q_c: 0.0, + use_acorn: false, }, None, vectors.as_ref(), @@ -235,6 +239,7 @@ fn bench_hnsw_sq(c: &mut Criterion) { lower_bound: None, upper_bound: None, dist_q_c: 0.0, + use_acorn: false, }, None, vectors.as_ref(), @@ -302,6 +307,7 @@ fn bench_hnsw_pq(c: &mut Criterion) { lower_bound: None, upper_bound: None, dist_q_c: 0.0, + use_acorn: false, }, None, vectors.as_ref(), @@ -331,6 +337,7 @@ fn bench_hnsw_pq(c: &mut Criterion) { lower_bound: None, upper_bound: None, dist_q_c: 0.0, + use_acorn: false, }, None, vectors.as_ref(), diff --git a/rust/lance-index/examples/acorn_bench.rs b/rust/lance-index/examples/acorn_bench.rs new file mode 100644 index 00000000000..8a24088e89b --- /dev/null +++ b/rust/lance-index/examples/acorn_bench.rs @@ -0,0 +1,259 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright The Lance Authors + +//! Compare the current mask-aware HNSW traversal (`search_basic`) against an +//! ACORN-1 style traversal (`search_acorn`) and a flat scan over matching +//! rows, across filter selectivities and mask shapes: +//! +//! - `corr-in`: mask is a cluster in embedding space, query is inside it +//! (e.g. "filter to courses" with a course query) +//! - `corr-out`: mask is a cluster, query is unrelated +//! - `random`: uniform random mask (the easy case) +//! +//! Run: cargo run --release -p lance-index --example acorn_bench + +#![allow(clippy::print_stdout)] + +use std::sync::Arc; +use std::time::Instant; + +use arrow_array::{Array, FixedSizeListArray, types::Float32Type}; +use lance_arrow::FixedSizeListArrayExt; +use lance_index::vector::flat::storage::FlatFloatStorage; +use lance_index::vector::graph::VisitedGenerator; +use lance_index::vector::hnsw::builder::{HNSW, HnswBuildParams, HnswQueryParams}; +use lance_index::vector::storage::{DistCalculator, VectorStore}; +use lance_index::vector::v3::subindex::IvfSubIndex; +use lance_linalg::distance::DistanceType; +use lance_testing::datagen::generate_random_array_with_seed; +use rand::rngs::SmallRng; +use rand::seq::{IndexedRandom, SliceRandom}; +use rand::{Rng, SeedableRng}; + +const TOTAL: usize = 100_000; +const DIMENSION: usize = 768; +const K: usize = 10; +const EF: usize = 100; +const EF_HI: usize = 400; +const QUERIES_PER_CASE: usize = 20; +const SEED: [u8; 32] = [42; 32]; + +fn all_distances(storage: &FlatFloatStorage, query: Arc) -> Vec { + let dist_calc = storage.dist_calculator(query, 0.0); + (0..TOTAL as u32).map(|i| dist_calc.distance(i)).collect() +} + +/// Node ids sorted ascending by distance to `query`, restricted to `mask`. +fn ground_truth(storage: &FlatFloatStorage, query: Arc, mask: &[bool]) -> Vec { + let dists = all_distances(storage, query); + let mut ids: Vec = (0..TOTAL as u32).filter(|&i| mask[i as usize]).collect(); + ids.sort_by(|&a, &b| dists[a as usize].partial_cmp(&dists[b as usize]).unwrap()); + ids.truncate(K); + ids +} + +fn recall(got: &[u32], truth: &[u32]) -> f64 { + let hits = truth.iter().filter(|id| got.contains(id)).count(); + hits as f64 / truth.len().max(1) as f64 +} + +struct CaseResult { + latency_us: Vec, + recalls: Vec, +} + +impl CaseResult { + fn new() -> Self { + Self { + latency_us: vec![], + recalls: vec![], + } + } + fn median_latency_ms(&mut self) -> f64 { + self.latency_us.sort_by(|a, b| a.partial_cmp(b).unwrap()); + self.latency_us[self.latency_us.len() / 2] / 1000.0 + } + fn mean_recall(&self) -> f64 { + self.recalls.iter().sum::() / self.recalls.len() as f64 + } +} + +fn main() { + println!("generating {TOTAL} x {DIMENSION} vectors..."); + let data = generate_random_array_with_seed::(TOTAL * DIMENSION, SEED); + let fsl = FixedSizeListArray::try_new_from_values(data, DIMENSION as i32).unwrap(); + let storage = Arc::new(FlatFloatStorage::new(fsl.clone(), DistanceType::L2)); + + println!("building HNSW graph..."); + let build_start = Instant::now(); + let hnsw = HNSW::index_vectors(storage.as_ref(), HnswBuildParams::default()).unwrap(); + println!("built in {:.1}s", build_start.elapsed().as_secs_f32()); + + let mut rng = SmallRng::seed_from_u64(7); + let params = HnswQueryParams { + ef: EF, + lower_bound: None, + upper_bound: None, + dist_q_c: 0.0, + use_acorn: false, + }; + let params_hi = HnswQueryParams { + ef: EF_HI, + ..params + }; + + // unfiltered control: this graph's recall ceiling at ef=EF + { + let mut control = CaseResult::new(); + let all_mask = vec![true; TOTAL]; + for _ in 0..QUERIES_PER_CASE { + let query_id = rng.random_range(0..TOTAL as u32); + let query = fsl.value(query_id as usize); + let truth = ground_truth(&storage, query.clone(), &all_mask); + let t = Instant::now(); + let nodes = hnsw + .search_basic(query.clone(), K, ¶ms, None, storage.as_ref()) + .unwrap(); + control.latency_us.push(t.elapsed().as_secs_f64() * 1e6); + let got: Vec = nodes.iter().map(|n| n.id).collect(); + control.recalls.push(recall(&got, &truth)); + } + println!( + "\nunfiltered control (ef={EF}): {:.2} ms, recall@{K} {:.3}", + control.median_latency_ms(), + control.mean_recall() + ); + } + + println!( + "\n{:<10} {:>5} | {:>9} {:>7} | {:>10} {:>7} | {:>9} {:>7} | {:>9} {:>7}", + "mask", + "sel%", + "basic ms", + "recall", + "basic4x ms", + "recall", + "acorn ms", + "recall", + "flat ms", + "recall" + ); + + for mask_kind in ["corr-in", "corr-out", "random"] { + for selectivity in [0.02f64, 0.05, 0.10, 0.25, 0.50] { + let mask_size = (TOTAL as f64 * selectivity) as usize; + let mut basic = CaseResult::new(); + let mut basic_hi = CaseResult::new(); + let mut acorn = CaseResult::new(); + let mut flat = CaseResult::new(); + let mut mask_generator = VisitedGenerator::new(TOTAL); + + for _ in 0..QUERIES_PER_CASE { + let mut mask = vec![false; TOTAL]; + let anchor_id = rng.random_range(0..TOTAL as u32); + let member_ids: Vec = match mask_kind { + "random" => { + let mut ids: Vec = (0..TOTAL as u32).collect(); + ids.shuffle(&mut rng); + ids.truncate(mask_size); + ids + } + _ => { + // cluster: the nearest nodes to a random anchor + let dists = all_distances(&storage, fsl.value(anchor_id as usize)); + let mut ids: Vec = (0..TOTAL as u32).collect(); + ids.sort_by(|&a, &b| { + dists[a as usize].partial_cmp(&dists[b as usize]).unwrap() + }); + ids.truncate(mask_size); + ids + } + }; + for &id in &member_ids { + mask[id as usize] = true; + } + + let query_id = match mask_kind { + "corr-in" => *member_ids.choose(&mut rng).unwrap(), + _ => rng.random_range(0..TOTAL as u32), + }; + let query = fsl.value(query_id as usize); + let truth = ground_truth(&storage, query.clone(), &mask); + + // search_basic (current traversal) + { + let mut bitset = mask_generator.generate(TOTAL); + for &id in &member_ids { + bitset.insert(id); + } + let t = Instant::now(); + let nodes = hnsw + .search_basic(query.clone(), K, ¶ms, Some(bitset), storage.as_ref()) + .unwrap(); + basic.latency_us.push(t.elapsed().as_secs_f64() * 1e6); + let got: Vec = nodes.iter().map(|n| n.id).collect(); + basic.recalls.push(recall(&got, &truth)); + } + + // search_basic at 4x ef (recall-equalizing baseline) + { + let mut bitset = mask_generator.generate(TOTAL); + for &id in &member_ids { + bitset.insert(id); + } + let t = Instant::now(); + let nodes = hnsw + .search_basic(query.clone(), K, ¶ms_hi, Some(bitset), storage.as_ref()) + .unwrap(); + basic_hi.latency_us.push(t.elapsed().as_secs_f64() * 1e6); + let got: Vec = nodes.iter().map(|n| n.id).collect(); + basic_hi.recalls.push(recall(&got, &truth)); + } + + // search_acorn + { + let mut bitset = mask_generator.generate(TOTAL); + for &id in &member_ids { + bitset.insert(id); + } + let t = Instant::now(); + let nodes = hnsw + .search_acorn(query.clone(), K, ¶ms, &bitset, storage.as_ref()) + .unwrap(); + acorn.latency_us.push(t.elapsed().as_secs_f64() * 1e6); + let got: Vec = nodes.iter().map(|n| n.id).collect(); + acorn.recalls.push(recall(&got, &truth)); + } + + // flat scan over matching rows + { + let t = Instant::now(); + let dist_calc = storage.dist_calculator(query.clone(), 0.0); + let mut scored: Vec<(f32, u32)> = member_ids + .iter() + .map(|&id| (dist_calc.distance(id), id)) + .collect(); + scored.sort_by(|a, b| a.0.partial_cmp(&b.0).unwrap()); + scored.truncate(K); + flat.latency_us.push(t.elapsed().as_secs_f64() * 1e6); + let got: Vec = scored.iter().map(|&(_, id)| id).collect(); + flat.recalls.push(recall(&got, &truth)); + } + } + + println!( + "{:<10} {:>5.0} | {:>9.2} {:>7.3} | {:>10.2} {:>7.3} | {:>9.2} {:>7.3} | {:>9.2} {:>7.3}", + mask_kind, + selectivity * 100.0, + basic.median_latency_ms(), + basic.mean_recall(), + basic_hi.median_latency_ms(), + basic_hi.mean_recall(), + acorn.median_latency_ms(), + acorn.mean_recall(), + flat.median_latency_ms(), + flat.mean_recall(), + ); + } + } +} diff --git a/rust/lance-index/examples/acorn_bench_sift.rs b/rust/lance-index/examples/acorn_bench_sift.rs new file mode 100644 index 00000000000..c781ae12577 --- /dev/null +++ b/rust/lance-index/examples/acorn_bench_sift.rs @@ -0,0 +1,329 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright The Lance Authors + +//! SIFT1M variant of `acorn_bench`: compares `search_basic`, `search_acorn`, +//! and a flat scan on the ANN_SIFT1M dataset (1M x 128, L2) with synthetic +//! filter masks. The unfiltered control is checked against the official +//! ground truth. +//! +//! Run: SIFT_DIR=/path/to/sift cargo run --release -p lance-index --example acorn_bench_sift +//! Works with any texmex-format dataset, e.g. SIFT_DIR=/path/to/gist for GIST1M. + +#![allow(clippy::print_stdout)] + +use std::sync::Arc; +use std::time::Instant; + +use arrow_array::{Array, FixedSizeListArray, Float32Array}; +use lance_arrow::FixedSizeListArrayExt; +use lance_index::vector::flat::storage::FlatFloatStorage; +use lance_index::vector::graph::VisitedGenerator; +use lance_index::vector::hnsw::builder::{HNSW, HnswBuildParams, HnswQueryParams}; +use lance_index::vector::storage::{DistCalculator, VectorStore}; +use lance_index::vector::v3::subindex::IvfSubIndex; +use lance_linalg::distance::DistanceType; +use rand::rngs::SmallRng; +use rand::seq::SliceRandom; +use rand::{Rng, SeedableRng}; + +const K: usize = 10; +const EF: usize = 100; +const EF_HI: usize = 400; +const QUERIES_PER_CASE: usize = 20; + +/// (flat values, dim, count) +fn read_fvecs(path: &str) -> (Vec, usize, usize) { + let bytes = std::fs::read(path).unwrap_or_else(|e| panic!("read {path}: {e}")); + let dim = i32::from_le_bytes(bytes[0..4].try_into().unwrap()) as usize; + let record = 4 + dim * 4; + assert_eq!(bytes.len() % record, 0); + let count = bytes.len() / record; + let mut values = Vec::with_capacity(count * dim); + for row in 0..count { + let start = row * record + 4; + for i in 0..dim { + let offset = start + i * 4; + values.push(f32::from_le_bytes( + bytes[offset..offset + 4].try_into().unwrap(), + )); + } + } + (values, dim, count) +} + +fn read_ivecs(path: &str) -> Vec> { + let bytes = std::fs::read(path).unwrap_or_else(|e| panic!("read {path}: {e}")); + let dim = i32::from_le_bytes(bytes[0..4].try_into().unwrap()) as usize; + let record = 4 + dim * 4; + assert_eq!(bytes.len() % record, 0); + (0..bytes.len() / record) + .map(|row| { + let start = row * record + 4; + (0..dim) + .map(|i| { + let offset = start + i * 4; + u32::from_le_bytes(bytes[offset..offset + 4].try_into().unwrap()) + }) + .collect() + }) + .collect() +} + +fn all_distances(storage: &FlatFloatStorage, total: usize, query: Arc) -> Vec { + let dist_calc = storage.dist_calculator(query, 0.0); + (0..total as u32).map(|i| dist_calc.distance(i)).collect() +} + +fn ground_truth( + storage: &FlatFloatStorage, + total: usize, + query: Arc, + mask: &[bool], +) -> Vec { + let dists = all_distances(storage, total, query); + let mut ids: Vec = (0..total as u32).filter(|&i| mask[i as usize]).collect(); + ids.sort_by(|&a, &b| dists[a as usize].partial_cmp(&dists[b as usize]).unwrap()); + ids.truncate(K); + ids +} + +fn recall(got: &[u32], truth: &[u32]) -> f64 { + let hits = truth.iter().filter(|id| got.contains(id)).count(); + hits as f64 / truth.len().max(1) as f64 +} + +struct CaseResult { + latency_us: Vec, + recalls: Vec, +} + +impl CaseResult { + fn new() -> Self { + Self { + latency_us: vec![], + recalls: vec![], + } + } + fn median_latency_ms(&mut self) -> f64 { + self.latency_us.sort_by(|a, b| a.partial_cmp(b).unwrap()); + self.latency_us[self.latency_us.len() / 2] / 1000.0 + } + fn mean_recall(&self) -> f64 { + self.recalls.iter().sum::() / self.recalls.len() as f64 + } +} + +fn main() { + let sift_dir = std::env::var("SIFT_DIR").expect("set SIFT_DIR to the extracted dataset dir"); + let prefix = std::path::Path::new(&sift_dir) + .file_name() + .and_then(|name| name.to_str()) + .expect("SIFT_DIR must end in the dataset name, e.g. sift or gist") + .to_string(); + println!("loading {prefix} from {sift_dir}..."); + let (base, dim, total) = read_fvecs(&format!("{sift_dir}/{prefix}_base.fvecs")); + let (queries, query_dim, num_queries) = read_fvecs(&format!("{sift_dir}/{prefix}_query.fvecs")); + let official_gt = read_ivecs(&format!("{sift_dir}/{prefix}_groundtruth.ivecs")); + assert_eq!(dim, query_dim); + println!("base {total} x {dim}, {num_queries} queries"); + + let fsl = + FixedSizeListArray::try_new_from_values(Float32Array::from(base), dim as i32).unwrap(); + let storage = Arc::new(FlatFloatStorage::new(fsl.clone(), DistanceType::L2)); + let query_vec = |i: usize| -> Arc { + Arc::new(Float32Array::from(queries[i * dim..(i + 1) * dim].to_vec())) + }; + + println!("building HNSW graph..."); + let build_start = Instant::now(); + let hnsw = HNSW::index_vectors(storage.as_ref(), HnswBuildParams::default()).unwrap(); + println!("built in {:.1}s", build_start.elapsed().as_secs_f32()); + + let mut rng = SmallRng::seed_from_u64(7); + let params = HnswQueryParams { + ef: EF, + lower_bound: None, + upper_bound: None, + dist_q_c: 0.0, + use_acorn: false, + }; + let params_hi = HnswQueryParams { + ef: EF_HI, + ..params + }; + + // unfiltered control against the official ground truth + { + let mut control = CaseResult::new(); + for _ in 0..QUERIES_PER_CASE { + let query_id = rng.random_range(0..num_queries); + let query = query_vec(query_id); + let truth: Vec = official_gt[query_id][..K].to_vec(); + let t = Instant::now(); + let nodes = hnsw + .search_basic(query, K, ¶ms, None, storage.as_ref()) + .unwrap(); + control.latency_us.push(t.elapsed().as_secs_f64() * 1e6); + let got: Vec = nodes.iter().map(|n| n.id).collect(); + control.recalls.push(recall(&got, &truth)); + } + println!( + "\nunfiltered control (ef={EF}, official GT): {:.2} ms, recall@{K} {:.3}", + control.median_latency_ms(), + control.mean_recall() + ); + } + + println!( + "\n{:<10} {:>5} | {:>9} {:>7} | {:>10} {:>7} | {:>9} {:>7} | {:>10} {:>7} | {:>9} {:>7}", + "mask", + "sel%", + "basic ms", + "recall", + "basic4x ms", + "recall", + "acorn ms", + "recall", + "acorn4x ms", + "recall", + "flat ms", + "recall" + ); + + for mask_kind in ["corr-in", "corr-out", "random"] { + for selectivity in [0.02f64, 0.05, 0.10, 0.25, 0.50] { + let mask_size = (total as f64 * selectivity) as usize; + let mut basic = CaseResult::new(); + let mut basic_hi = CaseResult::new(); + let mut acorn = CaseResult::new(); + let mut acorn_hi = CaseResult::new(); + let mut flat = CaseResult::new(); + let mut mask_generator = VisitedGenerator::new(total); + + for _ in 0..QUERIES_PER_CASE { + let query_id = rng.random_range(0..num_queries); + let query = query_vec(query_id); + + // corr-in: cluster around the query's own neighborhood + // corr-out: cluster around an unrelated base vector + let mut mask = vec![false; total]; + let member_ids: Vec = match mask_kind { + "random" => { + let mut ids: Vec = (0..total as u32).collect(); + ids.shuffle(&mut rng); + ids.truncate(mask_size); + ids + } + _ => { + let anchor: Arc = match mask_kind { + "corr-in" => query.clone(), + _ => fsl.value(rng.random_range(0..total)), + }; + let dists = all_distances(&storage, total, anchor); + let mut ids: Vec = (0..total as u32).collect(); + ids.sort_by(|&a, &b| { + dists[a as usize].partial_cmp(&dists[b as usize]).unwrap() + }); + ids.truncate(mask_size); + ids + } + }; + for &id in &member_ids { + mask[id as usize] = true; + } + let truth = ground_truth(&storage, total, query.clone(), &mask); + + // search_basic (current traversal) + { + let mut bitset = mask_generator.generate(total); + for &id in &member_ids { + bitset.insert(id); + } + let t = Instant::now(); + let nodes = hnsw + .search_basic(query.clone(), K, ¶ms, Some(bitset), storage.as_ref()) + .unwrap(); + basic.latency_us.push(t.elapsed().as_secs_f64() * 1e6); + let got: Vec = nodes.iter().map(|n| n.id).collect(); + basic.recalls.push(recall(&got, &truth)); + } + + // search_basic at 4x ef (recall-equalizing baseline) + { + let mut bitset = mask_generator.generate(total); + for &id in &member_ids { + bitset.insert(id); + } + let t = Instant::now(); + let nodes = hnsw + .search_basic(query.clone(), K, ¶ms_hi, Some(bitset), storage.as_ref()) + .unwrap(); + basic_hi.latency_us.push(t.elapsed().as_secs_f64() * 1e6); + let got: Vec = nodes.iter().map(|n| n.id).collect(); + basic_hi.recalls.push(recall(&got, &truth)); + } + + // search_acorn + { + let mut bitset = mask_generator.generate(total); + for &id in &member_ids { + bitset.insert(id); + } + let t = Instant::now(); + let nodes = hnsw + .search_acorn(query.clone(), K, ¶ms, &bitset, storage.as_ref()) + .unwrap(); + acorn.latency_us.push(t.elapsed().as_secs_f64() * 1e6); + let got: Vec = nodes.iter().map(|n| n.id).collect(); + acorn.recalls.push(recall(&got, &truth)); + } + + // search_acorn at 4x ef + { + let mut bitset = mask_generator.generate(total); + for &id in &member_ids { + bitset.insert(id); + } + let t = Instant::now(); + let nodes = hnsw + .search_acorn(query.clone(), K, ¶ms_hi, &bitset, storage.as_ref()) + .unwrap(); + acorn_hi.latency_us.push(t.elapsed().as_secs_f64() * 1e6); + let got: Vec = nodes.iter().map(|n| n.id).collect(); + acorn_hi.recalls.push(recall(&got, &truth)); + } + + // flat scan over matching rows + { + let t = Instant::now(); + let dist_calc = storage.dist_calculator(query.clone(), 0.0); + let mut scored: Vec<(f32, u32)> = member_ids + .iter() + .map(|&id| (dist_calc.distance(id), id)) + .collect(); + scored.sort_by(|a, b| a.0.partial_cmp(&b.0).unwrap()); + scored.truncate(K); + flat.latency_us.push(t.elapsed().as_secs_f64() * 1e6); + let got: Vec = scored.iter().map(|&(_, id)| id).collect(); + flat.recalls.push(recall(&got, &truth)); + } + } + + println!( + "{:<10} {:>5.0} | {:>9.2} {:>7.3} | {:>10.2} {:>7.3} | {:>9.2} {:>7.3} | {:>10.2} {:>7.3} | {:>9.2} {:>7.3}", + mask_kind, + selectivity * 100.0, + basic.median_latency_ms(), + basic.mean_recall(), + basic_hi.median_latency_ms(), + basic_hi.mean_recall(), + acorn.median_latency_ms(), + acorn.mean_recall(), + acorn_hi.median_latency_ms(), + acorn_hi.mean_recall(), + flat.median_latency_ms(), + flat.mean_recall(), + ); + } + } +} diff --git a/rust/lance-index/protos-cache/cache.proto b/rust/lance-index/protos-cache/cache.proto index b24a27055d7..efe16e92ea0 100644 --- a/rust/lance-index/protos-cache/cache.proto +++ b/rust/lance-index/protos-cache/cache.proto @@ -28,6 +28,11 @@ message CompressedPostingHeader { PositionStorage position_storage = 4; // Only meaningful when position_storage == POSITION_STORAGE_SHARED. PositionStreamCodec position_stream_codec = 5; + // Number of documents in each compressed posting block. Older cache entries + // omit this field and decode as the legacy 128-doc block size. + uint32 block_size = 6; + // Whether an impact IPC section follows the posting/position sections. + bool has_impacts = 7; } // Header for a serialized `PlainPostingList` cache entry. Followed by an Arrow @@ -164,6 +169,9 @@ message SqPartitionHeader { // three raw blobs: the IVF model protobuf, the quantizer's extra-metadata // buffer (may be empty), and the auxiliary IVF model protobuf. message IvfStateHeader { + reserved 8; + reserved "cache_key_prefix"; + string index_file_path = 1; string uuid = 2; string distance_type = 3; @@ -174,7 +182,6 @@ message IvfStateHeader { // type is generic over the quantizer; the proto envelope still provides // additive evolution for the surrounding fields. string quantizer_metadata_json = 7; - string cache_key_prefix = 8; uint64 index_file_size = 9; uint64 aux_file_size = 10; } diff --git a/rust/lance-index/src/frag_reuse.rs b/rust/lance-index/src/frag_reuse.rs index 4c70db44094..7072cdbe446 100644 --- a/rust/lance-index/src/frag_reuse.rs +++ b/rust/lance-index/src/frag_reuse.rs @@ -5,14 +5,16 @@ //! //! The data structures and table-format logic live in //! [`lance_table::system_index::frag_reuse`]; this module re-exports them and -//! implements the local [`Index`] trait for [`FragReuseIndex`]. +//! provides newtype wrappers that implement the [`Index`] and [`RowIdRemapper`] +//! traits. use std::any::Any; use std::sync::Arc; use arrow_array::RecordBatch; use async_trait::async_trait; -use lance_core::{Error, Result}; +use lance_core::Result; +use lance_core::deepsize::DeepSizeOf; use lance_select::RowAddrTreeMap; use roaring::{RoaringBitmap, RoaringTreemap}; use serde::Serialize; @@ -22,25 +24,22 @@ pub use lance_table::system_index::frag_reuse::*; use crate::scalar::RowIdRemapper; use crate::{Index, IndexType}; -impl RowIdRemapper for FragReuseIndex { - fn remap_row_id(&self, row_id: u64) -> Option { - self.remap_row_id(row_id) - } +/// Newtype wrapping [`FragReuseIndex`] so that `lance-index` can implement +/// the `Index` and `RowIdRemapper` traits (orphan rules prevent implementing +/// them directly in `lance-table`). +pub struct FragReuseIndexHandle(pub Arc); - fn remap_row_addrs_tree_map(&self, row_addrs: &RowAddrTreeMap) -> RowAddrTreeMap { - self.remap_row_addrs_tree_map(row_addrs) +impl std::fmt::Debug for FragReuseIndexHandle { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_tuple("FragReuseIndexHandle") + .field(&self.0) + .finish() } +} - fn remap_row_ids_roaring_tree_map(&self, row_ids: &RoaringTreemap) -> RoaringTreemap { - self.remap_row_ids_roaring_tree_map(row_ids) - } - - fn remap_row_ids_record_batch( - &self, - batch: RecordBatch, - row_id_idx: usize, - ) -> Result { - self.remap_row_ids_record_batch(batch, row_id_idx) +impl DeepSizeOf for FragReuseIndexHandle { + fn deep_size_of_children(&self, context: &mut lance_core::deepsize::Context) -> usize { + self.0.deep_size_of_children(context) } } @@ -50,7 +49,7 @@ struct FragReuseStatistics { } #[async_trait] -impl Index for FragReuseIndex { +impl Index for FragReuseIndexHandle { fn as_any(&self) -> &dyn Any { self } @@ -61,10 +60,10 @@ impl Index for FragReuseIndex { fn statistics(&self) -> Result { let stats = FragReuseStatistics { - num_versions: self.details.versions.len(), + num_versions: self.0.details.versions.len(), }; serde_json::to_value(stats).map_err(|e| { - Error::internal(format!( + lance_core::Error::internal(format!( "failed to serialize fragment reuse index statistics: {}", e )) @@ -83,3 +82,25 @@ impl Index for FragReuseIndex { unimplemented!() } } + +impl RowIdRemapper for FragReuseIndexHandle { + fn remap_row_id(&self, row_id: u64) -> Option { + self.0.remap_row_id(row_id) + } + + fn remap_row_addrs_tree_map(&self, row_addrs: &RowAddrTreeMap) -> RowAddrTreeMap { + self.0.remap_row_addrs_tree_map(row_addrs) + } + + fn remap_row_ids_roaring_tree_map(&self, row_ids: &RoaringTreemap) -> RoaringTreemap { + self.0.remap_row_ids_roaring_tree_map(row_ids) + } + + fn remap_row_ids_record_batch( + &self, + batch: RecordBatch, + row_id_idx: usize, + ) -> Result { + self.0.remap_row_ids_record_batch(batch, row_id_idx) + } +} diff --git a/rust/lance-index/src/lib.rs b/rust/lance-index/src/lib.rs index 61b45550367..c95e2ce609c 100644 --- a/rust/lance-index/src/lib.rs +++ b/rust/lance-index/src/lib.rs @@ -1,5 +1,6 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright The Lance Authors +#![cfg_attr(coverage, feature(coverage_attribute))] //! Lance secondary index library //! @@ -9,16 +10,9 @@ //! API stability is not guaranteed. //! -use std::{any::Any, sync::Arc}; - use crate::frag_reuse::FRAG_REUSE_INDEX_NAME; use crate::mem_wal::MEM_WAL_INDEX_NAME; -use async_trait::async_trait; -use lance_core::deepsize::DeepSizeOf; -use lance_core::{Error, Result}; -use roaring::RoaringBitmap; use serde::{Deserialize, Serialize}; -use std::convert::TryFrom; pub mod frag_reuse; pub mod mem_wal; @@ -33,6 +27,9 @@ pub mod vector; pub use crate::traits::*; +// Re-export core traits from lance-index-core +pub use lance_index_core::{Index, IndexParams, IndexType}; + pub const INDEX_FILE_NAME: &str = "index.idx"; /// The name of the auxiliary index file. /// @@ -75,280 +72,6 @@ pub mod cache_pb { include!(concat!(env!("OUT_DIR"), "/lance.index.cache.rs")); } -/// Generic methods common across all types of secondary indices -/// -#[async_trait] -pub trait Index: Send + Sync + DeepSizeOf { - /// Cast to [Any]. - fn as_any(&self) -> &dyn Any; - - /// Cast to [Index] - fn as_index(self: Arc) -> Arc; - - /// Retrieve index statistics as a JSON Value - fn statistics(&self) -> Result; - - /// Prewarm the index. - /// - /// This will load the index into memory and cache it. - async fn prewarm(&self) -> Result<()>; - - /// Get the type of the index - fn index_type(&self) -> IndexType; - - /// Read through the index and determine which fragment ids are covered by the index - /// - /// This is a kind of slow operation. It's better to use the fragment_bitmap. This - /// only exists for cases where the fragment_bitmap has become corrupted or missing. - async fn calculate_included_frags(&self) -> Result; -} - -/// Index Type -#[derive(Debug, PartialEq, Eq, Copy, Hash, Clone, DeepSizeOf)] -pub enum IndexType { - // Preserve 0-100 for simple indices. - Scalar = 0, // Legacy scalar index, alias to BTree - - BTree = 1, // BTree - - Bitmap = 2, // Bitmap - - LabelList = 3, // LabelList - - Inverted = 4, // Inverted - - NGram = 5, // NGram - - FragmentReuse = 6, - - MemWal = 7, - - ZoneMap = 8, // ZoneMap - - BloomFilter = 9, // Bloom filter - - RTree = 10, // RTree - - Fm = 11, // FM-Index - - // 100+ and up for vector index. - /// Flat vector index. - Vector = 100, // Legacy vector index, alias to IvfPq - IvfFlat = 101, - IvfSq = 102, - IvfPq = 103, - IvfHnswSq = 104, - IvfHnswPq = 105, - IvfHnswFlat = 106, - IvfRq = 107, -} - -impl std::fmt::Display for IndexType { - fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { - match self { - Self::Scalar | Self::BTree => write!(f, "BTree"), - Self::Bitmap => write!(f, "Bitmap"), - Self::LabelList => write!(f, "LabelList"), - Self::Inverted => write!(f, "Inverted"), - Self::NGram => write!(f, "NGram"), - Self::FragmentReuse => write!(f, "FragmentReuse"), - Self::MemWal => write!(f, "MemWal"), - Self::ZoneMap => write!(f, "ZoneMap"), - Self::BloomFilter => write!(f, "BloomFilter"), - Self::RTree => write!(f, "RTree"), - Self::Fm => write!(f, "Fm"), - Self::Vector | Self::IvfPq => write!(f, "IVF_PQ"), - Self::IvfFlat => write!(f, "IVF_FLAT"), - Self::IvfSq => write!(f, "IVF_SQ"), - Self::IvfHnswSq => write!(f, "IVF_HNSW_SQ"), - Self::IvfHnswPq => write!(f, "IVF_HNSW_PQ"), - Self::IvfHnswFlat => write!(f, "IVF_HNSW_FLAT"), - Self::IvfRq => write!(f, "IVF_RQ"), - } - } -} - -impl TryFrom for IndexType { - type Error = Error; - - fn try_from(value: i32) -> Result { - match value { - v if v == Self::Scalar as i32 => Ok(Self::Scalar), - v if v == Self::BTree as i32 => Ok(Self::BTree), - v if v == Self::Bitmap as i32 => Ok(Self::Bitmap), - v if v == Self::LabelList as i32 => Ok(Self::LabelList), - v if v == Self::NGram as i32 => Ok(Self::NGram), - v if v == Self::Inverted as i32 => Ok(Self::Inverted), - v if v == Self::FragmentReuse as i32 => Ok(Self::FragmentReuse), - v if v == Self::MemWal as i32 => Ok(Self::MemWal), - v if v == Self::ZoneMap as i32 => Ok(Self::ZoneMap), - v if v == Self::BloomFilter as i32 => Ok(Self::BloomFilter), - v if v == Self::RTree as i32 => Ok(Self::RTree), - v if v == Self::Fm as i32 => Ok(Self::Fm), - v if v == Self::Vector as i32 => Ok(Self::Vector), - v if v == Self::IvfFlat as i32 => Ok(Self::IvfFlat), - v if v == Self::IvfSq as i32 => Ok(Self::IvfSq), - v if v == Self::IvfPq as i32 => Ok(Self::IvfPq), - v if v == Self::IvfHnswSq as i32 => Ok(Self::IvfHnswSq), - v if v == Self::IvfHnswPq as i32 => Ok(Self::IvfHnswPq), - v if v == Self::IvfHnswFlat as i32 => Ok(Self::IvfHnswFlat), - v if v == Self::IvfRq as i32 => Ok(Self::IvfRq), - _ => Err(Error::invalid_input_source( - format!("the input value {} is not a valid IndexType", value).into(), - )), - } - } -} - -impl TryFrom<&str> for IndexType { - type Error = Error; - - fn try_from(value: &str) -> Result { - match value { - "BTree" | "BTREE" => Ok(Self::BTree), - "Bitmap" | "BITMAP" => Ok(Self::Bitmap), - "LabelList" | "LABELLIST" => Ok(Self::LabelList), - "Inverted" | "INVERTED" => Ok(Self::Inverted), - "NGram" | "NGRAM" => Ok(Self::NGram), - "ZoneMap" | "ZONEMAP" => Ok(Self::ZoneMap), - "BloomFilter" | "BLOOMFILTER" | "BLOOM_FILTER" => Ok(Self::BloomFilter), - "RTree" | "RTREE" | "R_TREE" => Ok(Self::RTree), - "Fm" | "FM" => Ok(Self::Fm), - "Vector" | "VECTOR" => Ok(Self::Vector), - "IVF_FLAT" => Ok(Self::IvfFlat), - "IVF_SQ" => Ok(Self::IvfSq), - "IVF_PQ" => Ok(Self::IvfPq), - "IVF_RQ" => Ok(Self::IvfRq), - "IVF_HNSW_FLAT" => Ok(Self::IvfHnswFlat), - "IVF_HNSW_SQ" => Ok(Self::IvfHnswSq), - "IVF_HNSW_PQ" => Ok(Self::IvfHnswPq), - "FragmentReuse" => Ok(Self::FragmentReuse), - "MemWal" => Ok(Self::MemWal), - _ => Err(Error::invalid_input(format!( - "invalid index type: {}", - value - ))), - } - } -} - -impl IndexType { - pub fn is_scalar(&self) -> bool { - matches!( - self, - Self::Scalar - | Self::BTree - | Self::Bitmap - | Self::LabelList - | Self::Inverted - | Self::NGram - | Self::ZoneMap - | Self::BloomFilter - | Self::RTree - | Self::Fm, - ) - } - - pub fn is_vector(&self) -> bool { - matches!( - self, - Self::Vector - | Self::IvfPq - | Self::IvfHnswSq - | Self::IvfHnswPq - | Self::IvfHnswFlat - | Self::IvfFlat - | Self::IvfSq - | Self::IvfRq - ) - } - - pub fn is_system(&self) -> bool { - matches!(self, Self::FragmentReuse | Self::MemWal) - } - - /// Returns the current format version of the index type, - /// bump this when the index format changes. - /// Indices which higher version than these will be ignored for compatibility, - /// This would happen when creating index in a newer version of Lance, - /// but then opening the index in older version of Lance - pub fn version(&self) -> i32 { - match self { - Self::Scalar => 0, - Self::BTree => 0, - Self::Bitmap => 0, - Self::LabelList => 0, - Self::Inverted => 0, - Self::NGram => 0, - Self::FragmentReuse => 0, - Self::MemWal => 0, - Self::ZoneMap => 0, - Self::BloomFilter => 0, - Self::RTree => 0, - Self::Fm => 0, - - // IMPORTANT: if any vector index subtype needs a format bump that is - // not backward compatible, its new version must be set to - // (current max vector index version + 1), even if only one subtype - // changed. Compatibility filtering currently cannot distinguish vector - // subtypes from details-only metadata, so vector versions effectively - // share one global monotonic compatibility level. - Self::Vector - | Self::IvfFlat - | Self::IvfSq - | Self::IvfPq - | Self::IvfHnswSq - | Self::IvfHnswPq - | Self::IvfHnswFlat => VECTOR_INDEX_VERSION as i32, - Self::IvfRq => IVF_RQ_INDEX_VERSION as i32, - } - } - - /// Returns the target partition size for the index type. - /// - /// This is used to compute the number of partitions for the index. - /// The partition size is optimized for the best performance of the index. - /// - /// This is for vector indices only. - pub fn target_partition_size(&self) -> usize { - match self { - Self::Vector => 8192, - Self::IvfFlat => 4096, - Self::IvfSq => 8192, - Self::IvfPq => 8192, - Self::IvfRq => 4096, - Self::IvfHnswFlat => 1 << 20, - Self::IvfHnswSq => 1 << 20, - Self::IvfHnswPq => 1 << 20, - _ => 8192, - } - } - - /// Returns the highest supported vector index version in this Lance build. - pub fn max_vector_version() -> u32 { - [ - Self::Vector, - Self::IvfFlat, - Self::IvfSq, - Self::IvfPq, - Self::IvfHnswSq, - Self::IvfHnswPq, - Self::IvfHnswFlat, - Self::IvfRq, - ] - .into_iter() - .map(|index_type| index_type.version() as u32) - .max() - .unwrap_or(VECTOR_INDEX_VERSION) - } -} - -pub trait IndexParams: Send + Sync { - fn as_any(&self) -> &dyn Any; - - fn index_name(&self) -> &str; -} - #[derive(Serialize, Deserialize, Debug)] pub struct IndexMetadata { #[serde(rename = "type")] diff --git a/rust/lance-index/src/mem_wal.rs b/rust/lance-index/src/mem_wal.rs index 9bd72ff7866..879f4757919 100644 --- a/rust/lance-index/src/mem_wal.rs +++ b/rust/lance-index/src/mem_wal.rs @@ -5,13 +5,14 @@ //! //! The data structures and table-format logic live in //! [`lance_table::system_index::mem_wal`]; this module re-exports them and -//! implements the local [`Index`] trait for [`MemWalIndex`]. +//! provides a newtype wrapper that implements the [`Index`] trait. use std::any::Any; use std::sync::Arc; use async_trait::async_trait; -use lance_core::Error; +use lance_core::Result; +use lance_core::deepsize::DeepSizeOf; use roaring::RoaringBitmap; use serde::Serialize; @@ -19,17 +20,28 @@ pub use lance_table::system_index::mem_wal::*; use crate::{Index, IndexType}; +/// Newtype wrapping [`MemWalIndex`] so that `lance-index` can implement +/// the `Index` trait (orphan rules prevent implementing it directly in +/// `lance-table`). +pub struct MemWalIndexHandle(pub Arc); + +impl DeepSizeOf for MemWalIndexHandle { + fn deep_size_of_children(&self, context: &mut lance_core::deepsize::Context) -> usize { + self.0.deep_size_of_children(context) + } +} + #[derive(Serialize)] struct MemWalStatistics { num_shards: u32, - num_merged_generations: usize, + num_compacted_sstables: usize, num_shard_specs: usize, num_maintained_indexes: usize, num_index_catchup_entries: usize, } #[async_trait] -impl Index for MemWalIndex { +impl Index for MemWalIndexHandle { fn as_any(&self) -> &dyn Any { self } @@ -38,23 +50,23 @@ impl Index for MemWalIndex { self } - fn statistics(&self) -> lance_core::Result { + fn statistics(&self) -> Result { let stats = MemWalStatistics { - num_shards: self.details.num_shards, - num_merged_generations: self.details.merged_generations.len(), - num_shard_specs: self.details.sharding_specs.len(), - num_maintained_indexes: self.details.maintained_indexes.len(), - num_index_catchup_entries: self.details.index_catchup.len(), + num_shards: self.0.details.num_shards, + num_compacted_sstables: self.0.details.compacted_sstables.len(), + num_shard_specs: self.0.details.sharding_specs.len(), + num_maintained_indexes: self.0.details.maintained_indexes.len(), + num_index_catchup_entries: self.0.details.index_catchup.len(), }; serde_json::to_value(stats).map_err(|e| { - Error::internal(format!( + lance_core::Error::internal(format!( "failed to serialize MemWAL index statistics: {}", e )) }) } - async fn prewarm(&self) -> lance_core::Result<()> { + async fn prewarm(&self) -> Result<()> { Ok(()) } @@ -62,7 +74,7 @@ impl Index for MemWalIndex { IndexType::MemWal } - async fn calculate_included_frags(&self) -> lance_core::Result { + async fn calculate_included_frags(&self) -> Result { Ok(RoaringBitmap::new()) } } diff --git a/rust/lance-index/src/metrics.rs b/rust/lance-index/src/metrics.rs index 8c0c119a3c3..7d1ea2964c6 100644 --- a/rust/lance-index/src/metrics.rs +++ b/rust/lance-index/src/metrics.rs @@ -1,117 +1,4 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright The Lance Authors -use std::sync::atomic::{AtomicUsize, Ordering}; - -pub const AND_CANDIDATES_SEEN_METRIC: &str = "and_candidates_seen"; -pub const AND_CANDIDATES_PRUNED_BEFORE_RETURN_METRIC: &str = "and_candidates_pruned_before_return"; -pub const AND_FULL_SCORES_METRIC: &str = "and_full_scores"; -pub const FREQS_COLLECTED_METRIC: &str = "freqs_collected"; - -/// A trait used by the index to report metrics -/// -/// Callers can implement this trait to collect metrics -pub trait MetricsCollector: Send + Sync { - /// Record partition loads - /// - /// Many indices consist of partitions that may need to be loaded - /// into cache. For example, an inverted index or ngram index has a - /// posting list for each token. - /// - /// In the ideal case, these shards are in the cache and will not need - /// to be loaded from disk. This method should not be called if the - /// shard is in the cache. - fn record_parts_loaded(&self, num_parts: usize); - - /// Record a shard load - fn record_part_load(&self) { - self.record_parts_loaded(1); - } - - /// Record an index load - /// - /// This should be called when a scalar index is loaded from storage. - /// It should not be called if the index is already in memory. - fn record_index_loads(&self, num_indexes: usize); - - /// Record an index load - fn record_index_load(&self) { - self.record_index_loads(1); - } - - /// Record the number of "comparisons" made by the index - /// - /// What exactly constitutes a comparison depends on the index type. - /// For example, a B-tree index may make comparisons while searching for a value. - /// On the other hand, a bitmap index makes comparisons when computing the intersection - /// of two bitmaps. - /// - /// The goal is to provide some visibility into the compute cost of the search - fn record_comparisons(&self, num_comparisons: usize); - - /// Record AND candidates returned from WAND alignment to the scoring loop. - /// - /// This excludes candidates pruned before `next()` returns. Use this with - /// `record_and_candidates_pruned_before_return` to recover total aligned - /// AND candidates. - fn record_and_candidates_seen(&self, _num_candidates: usize) {} - - /// Record AND candidates pruned during WAND alignment before `next()` returns. - fn record_and_candidates_pruned_before_return(&self, _num_candidates: usize) {} - - fn record_and_full_scores(&self, _num_scores: usize) {} - - fn record_freqs_collected(&self, _num_collections: usize) {} - - /// Returns an optional sink for recording exact I/O statistics (bytes read, - /// IOPS, and requests) performed on behalf of this collector. - /// - /// Index implementations that read from a - /// [`lance_io::scheduler::ScanScheduler`] can attach the returned handle to - /// their file readers so the I/O performed for a single query is measured - /// and attributed here. The default returns `None`, meaning the caller does - /// not want I/O measured (and index implementations should then take their - /// normal, uninstrumented read path). - fn io_stats(&self) -> Option { - None - } -} - -/// A no-op metrics collector that does nothing -pub struct NoOpMetricsCollector; - -impl MetricsCollector for NoOpMetricsCollector { - fn record_parts_loaded(&self, _num_parts: usize) {} - fn record_index_loads(&self, _num_indexes: usize) {} - fn record_comparisons(&self, _num_comparisons: usize) {} -} - -#[derive(Default)] -pub struct LocalMetricsCollector { - pub parts_loaded: AtomicUsize, - pub index_loads: AtomicUsize, - pub comparisons: AtomicUsize, -} - -impl LocalMetricsCollector { - pub fn dump_into(self, other: &dyn MetricsCollector) { - other.record_parts_loaded(self.parts_loaded.load(Ordering::Relaxed)); - other.record_index_loads(self.index_loads.load(Ordering::Relaxed)); - other.record_comparisons(self.comparisons.load(Ordering::Relaxed)); - } -} - -impl MetricsCollector for LocalMetricsCollector { - fn record_parts_loaded(&self, num_parts: usize) { - self.parts_loaded.fetch_add(num_parts, Ordering::Relaxed); - } - - fn record_index_loads(&self, num_indexes: usize) { - self.index_loads.fetch_add(num_indexes, Ordering::Relaxed); - } - - fn record_comparisons(&self, num_comparisons: usize) { - self.comparisons - .fetch_add(num_comparisons, Ordering::Relaxed); - } -} +pub use lance_index_core::metrics::*; diff --git a/rust/lance-index/src/registry.rs b/rust/lance-index/src/registry.rs index bd7448240fe..753f32afafd 100644 --- a/rust/lance-index/src/registry.rs +++ b/rust/lance-index/src/registry.rs @@ -16,6 +16,21 @@ use crate::{ }, }; +/// Derive the scalar index plugin name from a details type URL. +/// +/// Takes the last `.`-separated segment, lowercases it, and strips any trailing +/// `"indexdetails"` suffix so the result matches the plugin name used in +/// [`IndexPluginRegistry`]. For example, `/lance.index.pb.ZoneMapIndexDetails` +/// yields `"zonemap"`. +pub fn plugin_name_from_details_url(type_url: &str) -> String { + let segment = type_url.split('.').next_back().unwrap_or(type_url); + let lower = segment.to_lowercase(); + lower + .strip_suffix("indexdetails") + .map(|s| s.to_string()) + .unwrap_or(lower) +} + /// Derive a human-readable index type name from a details type URL. /// /// The display name is the final `.`-separated segment of the type URL with any @@ -42,12 +57,7 @@ impl IndexPluginRegistry { } fn get_plugin_name_from_details_name(&self, details_name: &str) -> String { - let details_name = Self::normalize_plugin_name(details_name); - if details_name.ends_with("indexdetails") { - details_name.replace("indexdetails", "") - } else { - details_name - } + plugin_name_from_details_url(details_name) } /// Adds a plugin to the registry, using the name of the details message to determine diff --git a/rust/lance-index/src/scalar.rs b/rust/lance-index/src/scalar.rs index f7b158ed558..92549e289d8 100644 --- a/rust/lance-index/src/scalar.rs +++ b/rust/lance-index/src/scalar.rs @@ -4,19 +4,13 @@ //! Scalar indices for metadata search & filtering use arrow::buffer::{OffsetBuffer, ScalarBuffer}; -use arrow_array::{BooleanArray, ListArray, RecordBatch, UInt64Array}; -use arrow_schema::{Field, Schema}; -use async_trait::async_trait; -use bytes::Bytes; +use arrow_array::ListArray; +use arrow_schema::Field; use datafusion::functions::regex::regexplike::RegexpLikeFunc; use datafusion::functions::string::contains::ContainsFunc; use datafusion::functions_nested::array_has; -use datafusion::physical_plan::SendableRecordBatchStream; use datafusion_common::{Column, scalar::ScalarValue}; -use lance_core::utils::row_addr_remap::RowAddrRemap; -use std::collections::{HashMap, HashSet}; -use std::fmt::Debug; -use std::pin::Pin; +use std::collections::HashSet; use std::{any::Any, ops::Bound, sync::Arc}; use datafusion_expr::{ @@ -24,17 +18,17 @@ use datafusion_expr::{ expr::{Like, ScalarFunction}, }; use inverted::query::{FtsQuery, FtsQueryNode, FtsSearchParams, MatchQuery, fill_fts_query_column}; -use lance_core::deepsize::DeepSizeOf; -use lance_core::{Error, Result}; -use lance_io::stream::{RecordBatchStream, RecordBatchStreamAdapter}; -use lance_select::{NullableRowAddrSet, RowAddrTreeMap, RowSetOps}; -use roaring::{RoaringBitmap, RoaringTreemap}; -use serde::Serialize; - -use crate::metrics::MetricsCollector; -use crate::scalar::registry::TrainingCriteria; -use crate::{Index, IndexParams, IndexType}; -pub use lance_table::format::IndexFile; +use lance_core::Result; + +use lance_datafusion::udf::CONTAINS_TOKENS_UDF; + +use crate::IndexParams; +pub use crate::metrics::MetricsCollector; +pub use lance_index_core::scalar::{ + AnyQuery, BuiltinIndexType, CreatedIndex, IndexFile, IndexReader, IndexStore, IndexWriter, + LANCE_SCALAR_INDEX, OldIndexDataFilter, RowIdRemapper, ScalarIndex, ScalarIndexParams, + SearchResult, TrainingCriteria, TrainingOrdering, UpdateCriteria, +}; pub mod bitmap; pub mod bloomfilter; @@ -49,121 +43,44 @@ pub mod ngram; pub mod registry; #[cfg(feature = "geo")] pub mod rtree; +pub mod seed; pub mod zoned; pub mod zonemap; pub use inverted::tokenizer::InvertedIndexParams; -use lance_datafusion::udf::CONTAINS_TOKENS_UDF; -pub const LANCE_SCALAR_INDEX: &str = "__lance_scalar_index"; - -/// Builtin index types supported by the Lance library +/// Convert a `Vec<`[`lance_index_core::scalar::IndexFile`]`>` to a +/// `Vec<`[`lance_table::format::IndexFile`]`>`. /// -/// This is primarily for convenience to avoid a bunch of string -/// constants and provide some auto-complete. This type should not -/// be used in the manifest as plugins cannot add new entries. -#[derive(Debug, Clone, PartialEq, Eq, DeepSizeOf)] -pub enum BuiltinIndexType { - BTree, - Bitmap, - LabelList, - NGram, - ZoneMap, - BloomFilter, - RTree, - Inverted, - Fm, -} - -impl BuiltinIndexType { - pub fn as_str(&self) -> &str { - match self { - Self::BTree => "btree", - Self::Bitmap => "bitmap", - Self::LabelList => "labellist", - Self::NGram => "ngram", - Self::ZoneMap => "zonemap", - Self::Inverted => "inverted", - Self::BloomFilter => "bloomfilter", - Self::RTree => "rtree", - Self::Fm => "fm", - } - } -} - -impl TryFrom for BuiltinIndexType { - type Error = Error; - - fn try_from(value: IndexType) -> Result { - match value { - IndexType::BTree => Ok(Self::BTree), - IndexType::Bitmap => Ok(Self::Bitmap), - IndexType::LabelList => Ok(Self::LabelList), - IndexType::NGram => Ok(Self::NGram), - IndexType::ZoneMap => Ok(Self::ZoneMap), - IndexType::Inverted => Ok(Self::Inverted), - IndexType::BloomFilter => Ok(Self::BloomFilter), - IndexType::RTree => Ok(Self::RTree), - IndexType::Fm => Ok(Self::Fm), - _ => Err(Error::index("Invalid index type".to_string())), - } - } -} - -#[derive(Debug, Clone, PartialEq)] -pub struct ScalarIndexParams { - /// The type of index to create - /// - /// Plugins may add additional index types. Index type lookup is case-insensitive. - pub index_type: String, - /// The parameters to train the index - /// - /// This should be a JSON string. The contents of the JSON string will be specific to the - /// index type. If not set, then default parameters will be used for the index type. - pub params: Option, -} - -impl Default for ScalarIndexParams { - fn default() -> Self { - Self { - index_type: BuiltinIndexType::BTree.as_str().to_string(), - params: None, - } - } -} - -impl ScalarIndexParams { - /// Creates a new ScalarIndexParams from one of the builtin index types - pub fn for_builtin(index_type: BuiltinIndexType) -> Self { - Self { - index_type: index_type.as_str().to_string(), - params: None, - } - } - - /// Create a new ScalarIndexParams with the given index type - pub fn new(index_type: String) -> Self { - Self { - index_type, - params: None, - } - } - - /// Set the parameters for the index - pub fn with_params(mut self, params: &ParamsType) -> Self { - self.params = Some(serde_json::to_string(params).unwrap()); - self - } -} - -impl IndexParams for ScalarIndexParams { - fn as_any(&self) -> &dyn std::any::Any { - self - } - - fn index_name(&self) -> &str { - LANCE_SCALAR_INDEX - } +/// These two structs have identical fields; this helper bridges the crate +/// boundary without relying on orphan-rule–violating `From` impls. +pub fn index_files_to_table( + files: Vec, +) -> Vec { + files + .into_iter() + .map(|f| lance_table::format::IndexFile { + path: f.path, + size_bytes: f.size_bytes, + }) + .collect() +} + +/// Convert a `Vec<`[`lance_table::format::IndexFile`]`>` to a +/// `Vec<`[`lance_index_core::scalar::IndexFile`]`>`. +/// +/// These two structs have identical fields; this helper bridges the crate +/// boundary without relying on orphan-rule–violating `From` impls. +pub fn table_files_to_index( + files: Vec, +) -> Vec { + files + .into_iter() + .map(|f| lance_index_core::scalar::IndexFile { + path: f.path, + size_bytes: f.size_bytes, + }) + .collect() } impl IndexParams for InvertedIndexParams { @@ -176,180 +93,6 @@ impl IndexParams for InvertedIndexParams { } } -/// Trait for storing an index (or parts of an index) into storage -#[async_trait] -pub trait IndexWriter: Send { - /// Writes a record batch into the file, returning the 0-based index of the batch in the file - /// - /// E.g. if this is the third time this is called this method will return 2 - async fn write_record_batch(&mut self, batch: RecordBatch) -> Result; - /// Adds a global buffer and returns its index. - async fn add_global_buffer(&mut self, _data: Bytes) -> Result { - Err(Error::not_supported( - "global buffers are not supported by this index writer", - )) - } - /// Finishes writing the file and closes the file - async fn finish(&mut self) -> Result; - /// Finishes writing the file and closes the file with additional metadata - async fn finish_with_metadata( - &mut self, - metadata: HashMap, - ) -> Result; -} - -/// Trait for reading an index (or parts of an index) from storage -#[async_trait] -pub trait IndexReader: Send + Sync { - /// Read the n-th record batch from the file - async fn read_record_batch(&self, n: u64, batch_size: u64) -> Result; - /// Reads a global buffer by index. - async fn read_global_buffer(&self, _index: u32) -> Result { - Err(Error::not_supported( - "global buffers are not supported by this index reader", - )) - } - /// Read the range of rows from the file. - /// If projection is Some, only return the columns in the projection, - /// nested columns like Some(&["x.y"]) are not supported. - /// If projection is None, return all columns. - async fn read_range( - &self, - range: std::ops::Range, - projection: Option<&[&str]>, - ) -> Result; - /// Read multiple ranges and concatenate into a single batch. - /// Default impl runs `read_range`s in parallel via `try_join_all`. - async fn read_ranges( - &self, - ranges: &[std::ops::Range], - projection: Option<&[&str]>, - ) -> Result { - if ranges.is_empty() { - return self.read_range(0..0, projection).await; - } - let futures = ranges - .iter() - .map(|r| self.read_range(r.clone(), projection)); - let batches = futures::future::try_join_all(futures).await?; - let schema = batches[0].schema(); - Ok(arrow_select::concat::concat_batches(&schema, &batches)?) - } - /// Read a range of rows as a stream of record batches. - /// - /// This allows the caller to process rows incrementally without loading the - /// entire range into memory at once. - /// - /// The default implementation falls back to [`Self::read_range`] and wraps - /// the result in a single-item stream. - async fn read_range_stream( - &self, - range: std::ops::Range, - projection: Option<&[&str]>, - ) -> Result>> { - let batch = self.read_range(range, projection).await?; - let schema = batch.schema(); - Ok(Box::pin(RecordBatchStreamAdapter::new( - schema, - futures::stream::once(async move { Ok(batch) }), - ))) - } - /// Return the number of batches in the file - async fn num_batches(&self, batch_size: u64) -> u32; - /// Return the number of rows in the file - fn num_rows(&self) -> usize; - /// Return the metadata of the file - fn schema(&self) -> &lance_core::datatypes::Schema; - /// Best-effort on-disk byte size of the file when the reader already knows it - /// without extra I/O, else `None`. Used to size prewarm chunks. - fn file_size_bytes(&self) -> Option { - None - } -} - -/// Trait abstracting I/O away from index logic -/// -/// Scalar indices are currently serialized as indexable arrow record batches stored in -/// named "files". The index store is responsible for serializing and deserializing -/// these batches into file data (e.g. as .lance files or .parquet files, etc.) -#[async_trait] -pub trait IndexStore: std::fmt::Debug + Send + Sync + DeepSizeOf { - fn as_any(&self) -> &dyn Any; - fn clone_arc(&self) -> Arc; - - /// Suggested I/O parallelism for the store - fn io_parallelism(&self) -> usize; - - /// Create a new file and return a writer to store data in the file - async fn new_index_file(&self, name: &str, schema: Arc) - -> Result>; - - /// Open an existing file for retrieval - async fn open_index_file(&self, name: &str) -> Result>; - - /// Return a store that submits its I/O at the given base priority. - fn with_io_priority(&self, io_priority: u64) -> Arc; - - /// Copy a range of batches from an index file from this store to another - /// - /// This is often useful when remapping or updating - async fn copy_index_file(&self, name: &str, dest_store: &dyn IndexStore) -> Result; - - /// Copy an index file from this store to a new name in another store, leaving the source intact - async fn copy_index_file_to( - &self, - name: &str, - new_name: &str, - dest_store: &dyn IndexStore, - ) -> Result { - if name == new_name { - self.copy_index_file(name, dest_store).await - } else { - Err(Error::not_supported(format!( - "copying index file {name} to {new_name} is not supported by this index store" - ))) - } - } - - /// Rename an index file - async fn rename_index_file(&self, name: &str, new_name: &str) -> Result; - - /// Delete an index file (used in the tmp spill store to keep tmp size down) - async fn delete_index_file(&self, name: &str) -> Result<()>; - - /// List all files in the index directory with their sizes. - /// - /// Returns a list of (relative_path, size_bytes) tuples. - /// Used to capture file metadata after index creation/modification. - async fn list_files_with_sizes(&self) -> Result>; -} - -/// Different scalar indices may support different kinds of queries -/// -/// For example, a btree index can support a wide range of queries (e.g. x > 7) -/// while an index based on FTS only supports queries like "x LIKE 'foo'" -/// -/// This trait is used when we need an object that can represent any kind of query -/// -/// Note: if you are implementing this trait for a query type then you probably also -/// need to implement the [crate::scalar::expression::ScalarQueryParser] trait to -/// create instances of your query at parse time. -pub trait AnyQuery: std::fmt::Debug + Any + Send + Sync { - /// Cast the query as Any to allow for downcasting - fn as_any(&self) -> &dyn Any; - /// Format the query as a string for display purposes - fn format(&self, col: &str) -> String; - /// Convert the query to a datafusion expression - fn to_expr(&self, col: String) -> Expr; - /// Compare this query to another query - fn dyn_eq(&self, other: &dyn AnyQuery) -> bool; -} - -impl PartialEq for dyn AnyQuery { - fn eq(&self, other: &Self) -> bool { - self.dyn_eq(other) - } -} /// A full text search query #[derive(Debug, Clone, PartialEq)] pub struct FullTextSearchQuery { @@ -892,149 +635,6 @@ impl AnyQuery for GeoQuery { } } -/// The result of a search operation against a scalar index -#[derive(Debug, PartialEq)] -pub enum SearchResult { - /// The exact row ids that satisfy the query - Exact(NullableRowAddrSet), - /// Any row id satisfying the query will be in this set but not every - /// row id in this set will satisfy the query, a further recheck step - /// is needed - AtMost(NullableRowAddrSet), - /// All of the given row ids satisfy the query but there may be more - /// - /// No scalar index actually returns this today but it can arise from - /// boolean operations (e.g. NOT(AtMost(x)) == AtLeast(NOT(x))) - AtLeast(NullableRowAddrSet), -} - -impl SearchResult { - pub fn exact(row_ids: impl Into) -> Self { - Self::Exact(NullableRowAddrSet::new(row_ids.into(), Default::default())) - } - - pub fn at_most(row_ids: impl Into) -> Self { - Self::AtMost(NullableRowAddrSet::new(row_ids.into(), Default::default())) - } - - pub fn at_least(row_ids: impl Into) -> Self { - Self::AtLeast(NullableRowAddrSet::new(row_ids.into(), Default::default())) - } - - pub fn with_nulls(self, nulls: impl Into) -> Self { - match self { - Self::Exact(row_ids) => Self::Exact(row_ids.with_nulls(nulls.into())), - Self::AtMost(row_ids) => Self::AtMost(row_ids.with_nulls(nulls.into())), - Self::AtLeast(row_ids) => Self::AtLeast(row_ids.with_nulls(nulls.into())), - } - } - - pub fn row_addrs(&self) -> &NullableRowAddrSet { - match self { - Self::Exact(row_addrs) => row_addrs, - Self::AtMost(row_addrs) => row_addrs, - Self::AtLeast(row_addrs) => row_addrs, - } - } - - pub fn is_exact(&self) -> bool { - matches!(self, Self::Exact(_)) - } -} - -/// Brief information about an index that was created -pub struct CreatedIndex { - /// The details of the index that was created - /// - /// These should be stored somewhere as they will be needed to - /// load the index later. - pub index_details: prost_types::Any, - /// The version of the index that was created - /// - /// This can be used to determine if a reader is able to load the index. - pub index_version: u32, - /// List of files and their sizes for this index - /// - /// This enables skipping HEAD calls when opening indices and provides - /// visibility into index storage size via describe_indices(). - pub files: Vec, -} - -/// The criteria that specifies how to update an index -pub struct UpdateCriteria { - /// If true, then we need to read the old data to update the index - /// - /// This should be avoided if possible but is left in for some legacy paths - pub requires_old_data: bool, - /// The criteria required for data (both old and new) - pub data_criteria: TrainingCriteria, -} - -/// Filter used when merging existing scalar-index rows during update. -/// -/// The caller must pick a filter mode that matches the row-id semantics of the -/// dataset: -/// - address-style row IDs: fragment filtering is valid -/// - stable row IDs: use exact row-id membership instead -#[derive(Debug, Clone)] -pub enum OldIndexDataFilter { - /// Keeps track of which fragments are still valid and which are no longer valid. - /// - /// This is valid for address-style row IDs. - Fragments { - to_keep: RoaringBitmap, - to_remove: RoaringBitmap, - }, - /// Keep old rows whose row IDs are in this exact allow-list. - /// - /// This is required for stable row IDs, where row IDs are opaque and - /// should not be interpreted as encoded row addresses. - RowIds(RowAddrTreeMap), -} - -impl OldIndexDataFilter { - /// Build a boolean mask that keeps only row IDs selected by this filter. - pub fn filter_row_ids(&self, row_ids: &UInt64Array) -> BooleanArray { - match self { - Self::Fragments { to_keep, .. } => row_ids - .iter() - .map(|id| id.map(|id| to_keep.contains((id >> 32) as u32))) - .collect(), - Self::RowIds(valid_row_ids) => row_ids - .iter() - .map(|id| id.map(|id| valid_row_ids.contains(id))) - .collect(), - } - } - - /// Apply this filter in place to a set of existing (old) row ids/addresses, - /// retaining only the rows the filter selects to keep. Used by index types - /// that merge old postings directly (e.g. bitmap) instead of re-scanning a - /// row-id array through [`Self::filter_row_ids`]. - pub fn retain_old_rows(&self, rows: &mut RowAddrTreeMap) { - match self { - Self::Fragments { to_keep, .. } => rows.retain_fragments(to_keep.iter()), - Self::RowIds(valid_row_ids) => *rows &= valid_row_ids, - } - } -} - -impl UpdateCriteria { - pub fn requires_old_data(data_criteria: TrainingCriteria) -> Self { - Self { - requires_old_data: true, - data_criteria, - } - } - - pub fn only_new_data(data_criteria: TrainingCriteria) -> Self { - Self { - requires_old_data: false, - data_criteria, - } - } -} - /// Compute the lexicographically next prefix by incrementing the last character's code point. /// Returns None if no valid upper bound exists. /// @@ -1091,77 +691,6 @@ fn next_unicode_char(c: char) -> Option { char::from_u32(next_cp) } -/// A trait for a scalar index, a structure that can determine row ids that satisfy scalar queries -#[async_trait] -pub trait ScalarIndex: Send + Sync + std::fmt::Debug + Index + DeepSizeOf { - /// Search the scalar index - /// - /// Returns all row ids that satisfy the query, these row ids are not necessarily ordered - async fn search( - &self, - query: &dyn AnyQuery, - metrics: &dyn MetricsCollector, - ) -> Result; - - /// Returns true if the remap operation is supported - fn can_remap(&self) -> bool; - - /// Remap the row ids, creating a new remapped version of this index in `dest_store` - async fn remap( - &self, - mapping: &RowAddrRemap, - dest_store: &dyn IndexStore, - ) -> Result; - - /// Add the new data into the index, creating an updated version of the index in `dest_store` - /// - /// If `old_data_filter` is provided, old index data will be filtered before - /// merge according to the chosen filter mode. - async fn update( - &self, - new_data: SendableRecordBatchStream, - dest_store: &dyn IndexStore, - old_data_filter: Option, - ) -> Result; - - /// Returns the criteria that will be used to update the index - fn update_criteria(&self) -> UpdateCriteria; - - /// Derive the index parameters from the current index - /// - /// This returns a ScalarIndexParams that can be used to recreate an index - /// with the same configuration on another dataset. - fn derive_index_params(&self) -> Result; - - /// Global `[min, max]` of the indexed column from index metadata, without a - /// scan, or `None` if this index type cannot supply a sound bound. When - /// `Some`, the range is a superset of live values (conservative under - /// deletes): safe to prune with, not guaranteed tight. - fn value_range(&self) -> Option<(ScalarValue, ScalarValue)> { - None - } -} - -/// Abstraction over any type that can remap row IDs during index loading. -/// -/// This decouples scalar index plugins from the table-level [`crate::frag_reuse::FragReuseIndex`] -/// type. [`crate::frag_reuse::FragReuseIndex`] implements this trait, but callers may also -/// supply custom implementations for testing or other remapping strategies. -pub trait RowIdRemapper: Send + Sync + std::fmt::Debug { - /// Remap a single row id. Returns `None` if the row was deleted. - fn remap_row_id(&self, row_id: u64) -> Option; - /// Remap all addresses in a [`RowAddrTreeMap`], dropping deleted rows. - fn remap_row_addrs_tree_map(&self, row_addrs: &RowAddrTreeMap) -> RowAddrTreeMap; - /// Remap all row ids in a [`RoaringTreemap`], dropping deleted rows. - fn remap_row_ids_roaring_tree_map(&self, row_ids: &RoaringTreemap) -> RoaringTreemap; - /// Remap the row-id column at `row_id_idx` inside `batch`, dropping deleted rows. - fn remap_row_ids_record_batch( - &self, - batch: RecordBatch, - row_id_idx: usize, - ) -> Result; -} - #[cfg(test)] mod tests { use super::*; diff --git a/rust/lance-index/src/scalar/bitmap.rs b/rust/lance-index/src/scalar/bitmap.rs index 98a07ff827d..6a614b0f11d 100644 --- a/rust/lance-index/src/scalar/bitmap.rs +++ b/rust/lance-index/src/scalar/bitmap.rs @@ -23,8 +23,8 @@ use lance_core::deepsize::DeepSizeOf; use lance_core::{ Error, ROW_ID, Result, cache::{ - CacheCodec, CacheCodecImpl, CacheEntryReader, CacheEntryWriter, CacheKey, LanceCache, - WeakLanceCache, + CacheCodec, CacheCodecImpl, CacheEntryReader, CacheEntryWriter, CacheKey, CacheKeySchema, + KeyBuilder, LanceCache, WeakLanceCache, }, error::LanceOptionExt, utils::tokio::get_num_compute_intensive_cpus, @@ -132,20 +132,39 @@ pub struct BitmapIndex { #[derive(Debug, Clone)] pub struct BitmapKey { - value: OrderableScalarValue, + row_offset: u64, +} + +impl BitmapKey { + fn try_new(row_offset: usize) -> Result { + let row_offset = u64::try_from(row_offset).map_err(|_| { + Error::internal(format!( + "bitmap row offset {row_offset} does not fit in u64" + )) + })?; + Ok(Self { row_offset }) + } } impl CacheKey for BitmapKey { type ValueType = RowAddrTreeMap; fn key(&self) -> std::borrow::Cow<'_, str> { - format!("{}", self.value.0).into() + self.row_offset.to_string().into() } fn type_name() -> &'static str { "Bitmap" } + fn schema() -> CacheKeySchema { + CacheKeySchema::new("lance.scalar.bitmap-row-offset-key", 1) + } + + fn write_key(&self, builder: &mut KeyBuilder) { + builder.write_u64(self.row_offset); + } + fn codec() -> Option { Some(CacheCodec::from_impl::()) } @@ -335,6 +354,14 @@ impl CacheKey for BitmapIndexStateKey { "BitmapIndexState" } + fn schema() -> CacheKeySchema { + CacheKeySchema::new("lance.scalar.bitmap-index-state-key", 1) + } + + fn write_key(&self, builder: &mut KeyBuilder) { + builder.write_variant(0); + } + fn codec() -> Option { Some(CacheCodec::from_impl::()) } @@ -446,22 +473,30 @@ impl BitmapIndex { return Ok(self.null_map.clone()); } - let cache_key = BitmapKey { value: key.clone() }; + // A value that isn't in `index_map` never reaches the loader or the + // cache, so it should not touch the per-query cache counters either. + // Checking here (before the cached-lookup fast path) also avoids + // returning an unmapped-value response as a spurious cache hit if a + // prior insert somehow ended up under `cache_key`. + let row_offset = match self.index_map.get(key) { + Some(loc) => *loc, + None => return Ok(Arc::new(RowAddrTreeMap::default())), + }; + let cache_key = BitmapKey::try_new(row_offset)?; if let Some(cached) = self.index_cache.get_with_key(&cache_key).await { + if let Some(metrics) = metrics { + metrics.record_index_cache_hit(); + } return Ok(cached); } // Record that we're loading a partition from disk if let Some(metrics) = metrics { + metrics.record_index_cache_miss(); metrics.record_part_load(); } - let row_offset = match self.index_map.get(key) { - Some(loc) => *loc, - None => return Ok(Arc::new(RowAddrTreeMap::default())), - }; - let page_lookup_file = self.lazy_reader.get().await?; let batch = page_lookup_file .read_range(row_offset..row_offset + 1, Some(&["bitmaps"])) @@ -602,7 +637,12 @@ impl Index for BitmapIndex { bitmap = frag_reuse_index_ref.remap_row_addrs_tree_map(&bitmap); } - let cache_key = BitmapKey { value: key }; + let row_offset = start_row.checked_add(idx).ok_or_else(|| { + Error::internal(format!( + "bitmap row offset overflow: start_row={start_row}, idx={idx}" + )) + })?; + let cache_key = BitmapKey::try_new(row_offset)?; self.index_cache .insert_with_key(&cache_key, Arc::new(bitmap)) .await; @@ -698,7 +738,7 @@ impl ScalarIndex for BitmapIndex { } else { let bitmaps: Vec<_> = stream::iter( keys.into_iter() - .map(|key| async move { self.load_bitmap(&key, None).await }), + .map(|key| async move { self.load_bitmap(&key, Some(metrics)).await }), ) .buffer_unordered(get_num_compute_intensive_cpus()) .try_collect() @@ -740,7 +780,7 @@ impl ScalarIndex for BitmapIndex { // Load bitmaps in parallel let mut bitmaps: Vec<_> = stream::iter( keys.into_iter() - .map(|key| async move { self.load_bitmap(&key, None).await }), + .map(|key| async move { self.load_bitmap(&key, Some(metrics)).await }), ) .buffer_unordered(get_num_compute_intensive_cpus()) .try_collect() @@ -891,8 +931,7 @@ impl BitmapBatchWriter { return Ok(()); } let keys_array = - ScalarValue::iter_to_array(self.keys.drain(..).collect::>().into_iter()) - .unwrap(); + ScalarValue::iter_to_array(self.keys.drain(..).collect::>()).unwrap(); let total_size: usize = self.serialized.iter().map(|b| b.len()).sum(); let mut binary_builder = BinaryBuilder::with_capacity(self.serialized.len(), total_size); for b in self.serialized.drain(..) { @@ -1123,10 +1162,7 @@ async fn drain_same_key_bitmaps( let merged_key = OrderableScalarValue(key); advance_cursor_and_push(cursors, heap, item.shard_idx).await?; - loop { - let Some(Reverse(next_item)) = heap.peek() else { - break; - }; + while let Some(Reverse(next_item)) = heap.peek() { if next_item.key != merged_key { break; } @@ -1280,7 +1316,7 @@ impl BitmapIndexPlugin { let bitmap_size = bytes.len(); if cur_bytes + bitmap_size > MAX_BITMAP_ARRAY_LENGTH { - let keys_array = ScalarValue::iter_to_array(cur_keys.clone().into_iter()).unwrap(); + let keys_array = ScalarValue::iter_to_array(cur_keys.clone()).unwrap(); let mut binary_builder = BinaryBuilder::new(); for b in &cur_bitmaps { binary_builder.append_value(b); @@ -1882,7 +1918,7 @@ impl ScalarIndexPlugin for BitmapIndexPlugin { #[cfg(test)] mod tests { use super::*; - use crate::metrics::NoOpMetricsCollector; + use crate::metrics::{LocalMetricsCollector, NoOpMetricsCollector}; use crate::scalar::lance_format::LanceIndexStore; use arrow_array::{RecordBatch, StringArray, UInt64Array, record_batch}; use arrow_schema::{DataType, Field, Schema}; @@ -1997,6 +2033,19 @@ mod tests { } } + #[tokio::test] + async fn test_bitmap_cache_key_uses_row_offset_identity() { + let cache = LanceCache::with_capacity(1024); + let first = BitmapKey::try_new(3).unwrap(); + let second = BitmapKey::try_new(4).unwrap(); + + cache + .insert_with_key(&first, Arc::new(RowAddrTreeMap::default())) + .await; + + assert!(cache.get_with_key(&second).await.is_none()); + } + #[tokio::test] async fn test_bitmap_lazy_loading_and_cache() { // Create a temporary directory for the index @@ -2132,6 +2181,95 @@ mod tests { } } + /// Regression test for the review fix that gates `load_bitmap` on + /// `index_map.contains_key` before recording a miss: a value that is + /// not present in the index must short-circuit before touching the + /// per-query cache counters. Previously an Equals query for a missing + /// value would silently bump `index_cache_misses` and `parts_loaded` + /// on every call even though no bitmap page was actually loaded. + #[tokio::test] + async fn test_bitmap_absent_value_records_no_cache_activity() { + let tmpdir = TempObjDir::default(); + let store = Arc::new(LanceIndexStore::new( + Arc::new(ObjectStore::local()), + tmpdir.clone(), + Arc::new(LanceCache::no_cache()), + )); + + let colors = vec!["red", "blue", "green", "yellow"]; + let row_ids = (0u64..4u64).collect::>(); + let schema = Arc::new(Schema::new(vec![ + Field::new("value", DataType::Utf8, false), + Field::new("_rowid", DataType::UInt64, false), + ])); + let batch = RecordBatch::try_new( + schema.clone(), + vec![ + Arc::new(StringArray::from(colors)), + Arc::new(UInt64Array::from(row_ids)), + ], + ) + .unwrap(); + let batch = sort_batch_by_value(&batch); + let stream = stream::once(async move { Ok(batch) }); + let stream = Box::pin(RecordBatchStreamAdapter::new(schema, stream)); + BitmapIndexPlugin::train_bitmap_index(stream, store.as_ref()) + .await + .unwrap(); + + // Keep the `LanceCache` alive in test scope so the `WeakLanceCache` + // inside `BitmapIndex` can upgrade during search. + let cache = LanceCache::with_capacity(1024 * 1024); + let index = BitmapIndex::load(store.clone(), None, &cache) + .await + .unwrap(); + + // Equals on a value that is not in `index_map` must not touch + // the cache counters and must not report a part load. + let metrics = LocalMetricsCollector::default(); + let query = SargableQuery::Equals(ScalarValue::Utf8(Some("purple".to_string()))); + let result = index.search(&query, &metrics).await.unwrap(); + if let SearchResult::Exact(row_ids) = result { + assert!(row_ids.true_rows().is_empty()); + } else { + panic!("Expected exact search result"); + } + assert_eq!( + metrics.index_cache_hits(), + 0, + "absent value must not record any cache hits", + ); + assert_eq!( + metrics.index_cache_misses(), + 0, + "absent value must not record a cache miss (no loader ran)", + ); + + // IsIn covering only absent values also stays at 0/0. + let metrics = LocalMetricsCollector::default(); + let query = SargableQuery::IsIn(vec![ + ScalarValue::Utf8(Some("purple".to_string())), + ScalarValue::Utf8(Some("teal".to_string())), + ]); + let result = index.search(&query, &metrics).await.unwrap(); + if let SearchResult::Exact(row_ids) = result { + assert!(row_ids.true_rows().is_empty()); + } else { + panic!("Expected exact search result"); + } + assert_eq!(metrics.index_cache_hits(), 0); + assert_eq!(metrics.index_cache_misses(), 0); + + // Sanity: a present value on the same cold cache still records + // exactly one miss, proving the counters are wired up and the + // absent-value path above is not silently no-op. + let metrics = LocalMetricsCollector::default(); + let query = SargableQuery::Equals(ScalarValue::Utf8(Some("red".to_string()))); + index.search(&query, &metrics).await.unwrap(); + assert_eq!(metrics.index_cache_hits(), 0); + assert_eq!(metrics.index_cache_misses(), 1); + } + // Regression test for the O(N log N) warm-cache rebuild introduced in // commit 4de5ce67d. BitmapIndexState now caches the parsed Arc // so that get_from_cache skips parse_lookup_batch on warm hits. @@ -2401,12 +2539,10 @@ mod tests { .unwrap(); // Verify no bitmaps are cached yet - let cache_key_red = BitmapKey { - value: OrderableScalarValue(ScalarValue::Utf8(Some("red".to_string()))), - }; - let cache_key_blue = BitmapKey { - value: OrderableScalarValue(ScalarValue::Utf8(Some("blue".to_string()))), - }; + let red = OrderableScalarValue(ScalarValue::Utf8(Some("red".to_string()))); + let blue = OrderableScalarValue(ScalarValue::Utf8(Some("blue".to_string()))); + let cache_key_red = BitmapKey::try_new(*index.index_map.get(&red).unwrap()).unwrap(); + let cache_key_blue = BitmapKey::try_new(*index.index_map.get(&blue).unwrap()).unwrap(); assert!( cache @@ -2700,6 +2836,7 @@ mod tests { } _ => panic!("Expected Exact search result"), } + let entries_after_value_lookup = cache.size().await; // Test 2: Search for null values - should return allow=[2], null=None let query = SargableQuery::IsNull(); @@ -2728,6 +2865,11 @@ mod tests { } _ => panic!("Expected Exact search result"), } + assert_eq!( + cache.size().await, + entries_after_value_lookup, + "null bitmap lookup should bypass the per-value cache" + ); // Test 3: Range query - should return matching rows and null_list let query = SargableQuery::Range( diff --git a/rust/lance-index/src/scalar/bloomfilter.rs b/rust/lance-index/src/scalar/bloomfilter.rs index 41bcb5a8b11..940cabe8f02 100644 --- a/rust/lance-index/src/scalar/bloomfilter.rs +++ b/rust/lance-index/src/scalar/bloomfilter.rs @@ -7,6 +7,7 @@ //! It is a space-efficient data structure that can be used to test whether an element is a member of a set. //! It's an inexact filter - they may include false positives that require rechecking. +use crate::pb; use crate::scalar::expression::{BloomFilterQueryParser, ScalarQueryParser}; use crate::scalar::registry::{ BasicTrainer, ScalarIndexPlugin, TrainingCriteria, TrainingOrdering, TrainingRequest, @@ -14,15 +15,18 @@ use crate::scalar::registry::{ use crate::scalar::{ BloomFilterQuery, BuiltinIndexType, CreatedIndex, IndexFile, ScalarIndexParams, UpdateCriteria, }; -use crate::{Any, pb}; use arrow_array::{Array, UInt64Array}; -use arrow_schema::{DataType, Field}; +use arrow_schema::{DataType, Field, Schema}; +use futures::TryStreamExt; use lance_arrow_stats::StatisticsAccumulator; use lance_core::utils::bloomfilter::as_bytes; use lance_core::utils::bloomfilter::sbbf::{Sbbf, SbbfBuilder}; use lance_core::utils::row_addr_remap::RowAddrRemap; +use lance_select::RowAddrTreeMap; use serde::{Deserialize, Serialize}; +use std::any::Any; +use std::collections::HashMap; use std::sync::LazyLock; use datafusion::execution::SendableRecordBatchStream; @@ -44,7 +48,13 @@ use super::zoned::{ZoneBound, ZoneProcessor, ZoneTrainer, rebuild_zones, search_ const BLOOMFILTER_FILENAME: &str = "bloomfilter.lance"; const BLOOMFILTER_ITEM_META_KEY: &str = "bloomfilter_item"; +const NULL_BITMAP_META_KEY: &str = "null_bitmap"; const BLOOMFILTER_PROBABILITY_META_KEY: &str = "bloomfilter_probability"; +/// Upper bound on the total serialized bytes packed into a single bloom filter +/// `BinaryArray`. Its offsets are `i32`, so the concatenated payload cannot exceed +/// `i32::MAX`. We reserve a 1 MiB margin below that hard limit so per-row Arrow +/// bookkeeping (offset and validity buffers) cannot push a batch over the edge. +const MAX_BLOOMFILTER_ARRAY_LENGTH: usize = i32::MAX as usize - 1024 * 1024; const BLOOMFILTER_INDEX_VERSION: u32 = 0; #[derive(Debug, Clone)] @@ -80,24 +90,34 @@ pub struct BloomFilterIndex { number_of_items: u64, // Probability of false positives, fraction between 0 and 1 probability: f64, + // Exact set of null row addresses; None for older indices without this bitmap. + null_rows: Option, + frag_reuse_index: Option>, } impl DeepSizeOf for BloomFilterIndex { fn deep_size_of_children(&self, context: &mut lance_core::deepsize::Context) -> usize { - self.zones.deep_size_of_children(context) + self.zones.deep_size_of_children(context) + self.null_rows.deep_size_of_children(context) } } impl BloomFilterIndex { async fn load( store: Arc, - _fri: Option>, + fri: Option>, + index_cache: &LanceCache, + ) -> Result> { + Self::load_with_max_array_length(store, fri, index_cache, MAX_BLOOMFILTER_ARRAY_LENGTH) + .await + } + + async fn load_with_max_array_length( + store: Arc, + fri: Option>, _index_cache: &LanceCache, + max_array_length: usize, ) -> Result> { let index_file = store.open_index_file(BLOOMFILTER_FILENAME).await?; - let bloom_data = index_file - .read_range(0..index_file.num_rows(), None) - .await?; let file_schema = index_file.schema(); let number_of_items: u64 = file_schema @@ -112,25 +132,65 @@ impl BloomFilterIndex { .and_then(|bs| bs.parse().ok()) .unwrap_or(*DEFAULT_PROBABILITY); - Ok(Arc::new(Self::try_from_serialized( - bloom_data, + let null_rows = if let Some(idx_str) = file_schema.metadata.get(NULL_BITMAP_META_KEY) { + let idx = idx_str.parse::().map_err(|e| { + Error::invalid_input(format!("invalid null bitmap buffer index: {e}")) + })?; + let bytes = index_file.read_global_buffer(idx).await?; + Some(RowAddrTreeMap::deserialize_from(bytes.as_ref())?) + } else { + None + }; + + let read_batch_size = + Self::read_batch_size(number_of_items, probability, max_array_length)?; + + let mut zones = Vec::with_capacity(index_file.num_rows()); + for start in (0..index_file.num_rows()).step_by(read_batch_size) { + let end = (start + read_batch_size).min(index_file.num_rows()); + let mut bloom_data = index_file.read_range_stream(start..end, None).await?; + while let Some(batch) = bloom_data.try_next().await? { + zones.extend(Self::try_from_serialized(batch, max_array_length)?); + } + } + + Ok(Arc::new(Self { + zones, number_of_items, probability, - )?)) + null_rows, + frag_reuse_index: fri, + })) } - fn try_from_serialized( - data: RecordBatch, + fn read_batch_size( number_of_items: u64, probability: f64, - ) -> Result { + max_array_length: usize, + ) -> Result { + // Bloom filters are stored in an Arrow BinaryArray, whose offsets are i32. + // The serialized filter size is fixed by the index parameters, so bound + // reads by total serialized bytes instead of row count alone. + let params = BloomFilterIndexBuilderParams { + number_of_items, + probability, + }; + let filter_size = BloomFilterProcessor::build_filter(¶ms)?.size_bytes(); + if filter_size > max_array_length { + return Err(Error::invalid_input(format!( + "Serialized bloom filter size {} exceeds max supported batch bytes {}", + filter_size, max_array_length + ))); + } + Ok((max_array_length / filter_size).max(1)) + } + + fn try_from_serialized( + data: RecordBatch, + max_array_length: usize, + ) -> Result> { if data.num_rows() == 0 { - // Return empty index for empty data - return Ok(Self { - zones: Vec::new(), - number_of_items, - probability, - }); + return Ok(Vec::new()); } let fragment_id_col = data @@ -171,6 +231,18 @@ impl BloomFilterIndex { Error::invalid_input("BloomFilterIndex: 'bloom_filter_data' column is not Binary") })?; + // Enforce the i32-offset cap on read, symmetric to the write side. A batch this + // large means the read chunking was bypassed; reject it before it overflows the + // BinaryArray offsets instead of panicking deep inside Arrow. + let offsets = bloom_filter_data_col.value_offsets(); + let batch_bytes = (offsets[offsets.len() - 1] - offsets[0]) as usize; + if batch_bytes > max_array_length { + return Err(Error::invalid_input(format!( + "Serialized bloom filter batch size {} exceeds max supported batch bytes {}", + batch_bytes, max_array_length + ))); + } + let has_null_col = data .column_by_name("has_null") .ok_or_else(|| Error::invalid_input("BloomFilterIndex: missing 'has_null' column"))? @@ -190,7 +262,6 @@ impl BloomFilterIndex { Vec::new() }; - // Convert bytes back to Sbbf let bloom_filter = Sbbf::new(&bloom_filter_bytes).map_err(|e| { Error::invalid_input(format!("Failed to deserialize bloom filter: {:?}", e)) })?; @@ -206,11 +277,7 @@ impl BloomFilterIndex { }); } - Ok(Self { - zones: blocks, - number_of_items, - probability, - }) + Ok(blocks) } fn evaluate_block_against_query( @@ -415,11 +482,33 @@ impl ScalarIndex for BloomFilterIndex { metrics: &dyn MetricsCollector, ) -> Result { let query = query.as_any().downcast_ref::().unwrap(); - search_zones(&self.zones, metrics, |block| { - self.evaluate_block_against_query(block, query) + let result = if let BloomFilterQuery::IsNull() = query + && let Some(null_rows) = &self.null_rows + { + SearchResult::exact(null_rows.clone()) + } else { + search_zones(&self.zones, metrics, |block| { + self.evaluate_block_against_query(block, query) + })? + }; + + let Some(remapper) = &self.frag_reuse_index else { + return Ok(result); + }; + let selected = remapper.remap_row_addrs_tree_map(result.row_addrs().selected_rows()); + let nulls = remapper.remap_row_addrs_tree_map(result.row_addrs().null_rows()); + + Ok(match result { + SearchResult::Exact(_) => SearchResult::exact(selected).with_nulls(nulls), + SearchResult::AtMost(_) => SearchResult::at_most(selected).with_nulls(nulls), + SearchResult::AtLeast(_) => SearchResult::at_least(selected).with_nulls(nulls), }) } + fn results_are_row_addresses(&self) -> bool { + true + } + fn can_remap(&self) -> bool { false } @@ -448,18 +537,28 @@ impl ScalarIndex for BloomFilterIndex { let processor = BloomFilterProcessor::new(params.clone())?; let trainer = ZoneTrainer::new(processor, params.number_of_items)?; - let updated_blocks = rebuild_zones(&self.zones, trainer, new_data).await?; + let (updated_blocks, new_null_rows) = rebuild_zones(&self.zones, trainer, new_data).await?; + + // Merge existing and new null rows. If the existing index had no null bitmap + // (legacy format — null positions unknown), preserve that None: updating cannot + // recover the missing information, and claiming the result has zero nulls would + // be a false negative. Only a full retrain produces a fresh, complete bitmap. + let merged_null_rows = self.null_rows.as_ref().map(|existing| { + let mut merged = existing.clone(); + merged |= &new_null_rows; + merged + }); // Write the combined zones back to storage let mut builder = BloomFilterIndexBuilder::try_new(params)?; builder.blocks = updated_blocks; - let file = builder.write_index(dest_store).await?; + builder.null_rows = merged_null_rows; + let files = builder.write_index(dest_store).await?; Ok(CreatedIndex { - index_details: prost_types::Any::from_msg(&pb::BloomFilterIndexDetails::default()) - .unwrap(), + index_details: prost_types::Any::from_msg(&pb::BloomFilterIndexDetails::default())?, index_version: BLOOMFILTER_INDEX_VERSION, - files: vec![file], + files, }) } @@ -478,6 +577,126 @@ impl ScalarIndex for BloomFilterIndex { } } +fn remap_zone( + zone: &BloomFilterStatistics, + remapper: &dyn RowIdRemapper, +) -> Vec { + let zone_start = (zone.bound.fragment_id << 32).saturating_add(zone.bound.start); + let mut remapped = (0..zone.bound.length as u64) + .filter_map(|offset| remapper.remap_row_id(zone_start.saturating_add(offset))) + .collect::>(); + remapped.sort_unstable(); + remapped.dedup(); + + let mut zones = Vec::new(); + let mut run_start = None; + let mut previous = 0u64; + for row_id in remapped { + if run_start.is_none() { + run_start = Some(row_id); + } else if row_id != previous.saturating_add(1) || row_id >> 32 != previous >> 32 { + let start = run_start.take().unwrap(); + zones.push(BloomFilterStatistics { + bound: ZoneBound { + fragment_id: start >> 32, + start: start & u64::from(u32::MAX), + length: (previous - start + 1) as usize, + }, + has_null: zone.has_null, + bloom_filter: zone.bloom_filter.clone(), + }); + run_start = Some(row_id); + } + previous = row_id; + } + if let Some(start) = run_start { + zones.push(BloomFilterStatistics { + bound: ZoneBound { + fragment_id: start >> 32, + start: start & u64::from(u32::MAX), + length: (previous - start + 1) as usize, + }, + has_null: zone.has_null, + bloom_filter: zone.bloom_filter.clone(), + }); + } + zones +} + +/// Merge caller-selected BloomFilter segments into one self-contained segment. +pub async fn merge_bloomfilter_indices( + source_indices: &[(&BloomFilterIndex, &RoaringBitmap)], + dest_store: &dyn IndexStore, +) -> Result { + let first = source_indices + .iter() + .find(|(_, fragment_filter)| !fragment_filter.is_empty()) + .or_else(|| source_indices.first()) + .ok_or_else(|| { + Error::invalid_input("merge_bloomfilter_indices requires at least one source index") + })?; + let params = BloomFilterIndexBuilderParams { + number_of_items: first.0.number_of_items, + probability: first.0.probability, + }; + + let mut blocks = Vec::new(); + let mut merged_null_rows = RowAddrTreeMap::new(); + let mut has_missing_null_bitmap = false; + for (source, fragment_filter) in source_indices { + if fragment_filter.is_empty() { + continue; + } + if source.number_of_items != params.number_of_items + || source.probability != params.probability + { + return Err(Error::invalid_input(format!( + "cannot merge BloomFilter segments with different parameters: \ + number_of_items={}, probability={} and number_of_items={}, probability={}", + params.number_of_items, + params.probability, + source.number_of_items, + source.probability + ))); + } + let source_zones = source.zones.iter().flat_map(|block| { + source.frag_reuse_index.as_deref().map_or_else( + || vec![block.clone()], + |remapper| remap_zone(block, remapper), + ) + }); + blocks.extend(source_zones.filter(|block| { + u32::try_from(block.bound.fragment_id) + .is_ok_and(|fragment_id| fragment_filter.contains(fragment_id)) + })); + match &source.null_rows { + Some(null_rows) => { + let mut filtered = source.frag_reuse_index.as_deref().map_or_else( + || null_rows.clone(), + |remapper| remapper.remap_row_addrs_tree_map(null_rows), + ); + filtered.retain_fragments(fragment_filter.iter()); + merged_null_rows |= &filtered; + } + None => has_missing_null_bitmap = true, + } + } + blocks.sort_by_key(|block| (block.bound.fragment_id, block.bound.start)); + + let mut builder = BloomFilterIndexBuilder::try_new(params)?; + builder.blocks = blocks; + if !has_missing_null_bitmap { + builder.null_rows = Some(merged_null_rows); + } + let files = builder.write_index(dest_store).await?; + + Ok(CreatedIndex { + index_details: prost_types::Any::from_msg(&pb::BloomFilterIndexDetails::default())?, + index_version: BLOOMFILTER_INDEX_VERSION, + files, + }) +} + fn default_number_of_items() -> u64 { *DEFAULT_NUMBER_OF_ITEMS } @@ -542,6 +761,10 @@ impl BloomFilterIndexBuilderParams { pub struct BloomFilterIndexBuilder { params: BloomFilterIndexBuilderParams, blocks: Vec, + // None means "legacy index — null positions unknown"; Some means a complete bitmap. + // write_index omits the null-bitmap global buffer when this is None, preserving the + // legacy format so that downstream searches remain conservative. + null_rows: Option, } impl BloomFilterIndexBuilder { @@ -549,6 +772,7 @@ impl BloomFilterIndexBuilder { Ok(Self { params, blocks: Vec::new(), + null_rows: None, }) } @@ -558,37 +782,36 @@ impl BloomFilterIndexBuilder { pub async fn train(&mut self, batches_source: SendableRecordBatchStream) -> Result<()> { let processor = BloomFilterProcessor::new(self.params.clone())?; let trainer = ZoneTrainer::new(processor, self.params.number_of_items)?; - self.blocks = trainer.train(batches_source).await?; + let (blocks, null_rows) = trainer.train(batches_source).await?; + self.blocks = blocks; + self.null_rows = Some(null_rows); Ok(()) } - fn bloomfilter_stats_as_batch(&self) -> Result { - let fragment_ids = - UInt64Array::from_iter_values(self.blocks.iter().map(|block| block.bound.fragment_id)); - - let zone_starts = - UInt64Array::from_iter_values(self.blocks.iter().map(|block| block.bound.start)); - - let zone_lengths = UInt64Array::from_iter_values( - self.blocks.iter().map(|block| block.bound.length as u64), - ); - - let has_nulls = arrow_array::BooleanArray::from( - self.blocks - .iter() - .map(|block| block.has_null) - .collect::>(), - ); + fn bloomfilter_schema() -> Arc { + Arc::new(Schema::new(vec![ + Field::new("fragment_id", DataType::UInt64, false), + Field::new("zone_start", DataType::UInt64, false), + Field::new("zone_length", DataType::UInt64, false), + Field::new("has_null", DataType::Boolean, false), + Field::new("bloom_filter_data", DataType::Binary, false), + ])) + } - // Convert bloom filters to binary data for serialization - let bloom_filter_data = if self.blocks.is_empty() { + fn bloomfilter_stats_as_batch( + fragment_ids: Vec, + zone_starts: Vec, + zone_lengths: Vec, + has_nulls: Vec, + binary_data: Vec>, + ) -> Result { + let fragment_ids = UInt64Array::from(fragment_ids); + let zone_starts = UInt64Array::from(zone_starts); + let zone_lengths = UInt64Array::from(zone_lengths); + let has_nulls = arrow_array::BooleanArray::from(has_nulls); + let bloom_filter_data = if binary_data.is_empty() { Arc::new(arrow_array::BinaryArray::new_null(0)) as ArrayRef } else { - let binary_data: Vec> = self - .blocks - .iter() - .map(|block| block.bloom_filter.to_bytes()) - .collect(); let binary_refs: Vec> = binary_data .iter() .map(|bytes| Some(bytes.as_slice())) @@ -596,14 +819,6 @@ impl BloomFilterIndexBuilder { Arc::new(arrow_array::BinaryArray::from_opt_vec(binary_refs)) as ArrayRef }; - let schema = Arc::new(arrow_schema::Schema::new(vec![ - Field::new("fragment_id", DataType::UInt64, false), - Field::new("zone_start", DataType::UInt64, false), - Field::new("zone_length", DataType::UInt64, false), - Field::new("has_null", DataType::Boolean, false), - Field::new("bloom_filter_data", DataType::Binary, false), - ])); - let columns: Vec = vec![ Arc::new(fragment_ids) as ArrayRef, Arc::new(zone_starts) as ArrayRef, @@ -612,28 +827,160 @@ impl BloomFilterIndexBuilder { bloom_filter_data, ]; - Ok(RecordBatch::try_new(schema, columns)?) + Ok(RecordBatch::try_new(Self::bloomfilter_schema(), columns)?) } - pub async fn write_index(self, index_store: &dyn IndexStore) -> Result { - let record_batch = self.bloomfilter_stats_as_batch()?; + /// Serialize the trained bloom filter zone statistics into an index file in + /// `index_store`, returning the resulting [`IndexFile`]s. + /// + /// Zones are flushed as one or more record batches, each bounded by + /// `MAX_BLOOMFILTER_ARRAY_LENGTH` serialized bytes so the underlying Arrow + /// `BinaryArray` never overflows its `i32` offsets. Any optional null-row bitmap + /// is persisted as a global buffer on the same [`IndexFile`] via [`IndexStore`]. + pub async fn write_index(self, index_store: &dyn IndexStore) -> Result> { + self.write_index_with_max_array_length(index_store, MAX_BLOOMFILTER_ARRAY_LENGTH) + .await + } - let mut file_schema = record_batch.schema().as_ref().clone(); + async fn write_index_with_max_array_length( + self, + index_store: &dyn IndexStore, + max_array_length: usize, + ) -> Result> { + let mut file_schema = Self::bloomfilter_schema().as_ref().clone(); file_schema.metadata.insert( BLOOMFILTER_ITEM_META_KEY.to_string(), self.params.number_of_items.to_string(), ); - file_schema.metadata.insert( BLOOMFILTER_PROBABILITY_META_KEY.to_string(), self.params.probability.to_string(), ); - let mut index_file = index_store + let index_file = index_store .new_index_file(BLOOMFILTER_FILENAME, Arc::new(file_schema)) .await?; - index_file.write_record_batch(record_batch).await?; - index_file.finish().await + + let mut writer = BloomFilterBatchWriter::new(index_file, max_array_length); + for block in self.blocks { + writer.emit(block).await?; + } + let bloomfilter_file = writer.finish(self.null_rows).await?; + Ok(vec![bloomfilter_file]) + } +} + +/// Buffers serialized bloom filter zone statistics and flushes them as record batches +/// to the index file, respecting the `max_array_length` limit. +struct BloomFilterBatchWriter { + file: Box, + max_array_length: usize, + fragment_ids: Vec, + zone_starts: Vec, + zone_lengths: Vec, + has_nulls: Vec, + bloom_filter_data: Vec>, + current_bytes: usize, + has_written: bool, +} + +impl BloomFilterBatchWriter { + fn new(file: Box, max_array_length: usize) -> Self { + Self { + file, + max_array_length, + fragment_ids: Vec::new(), + zone_starts: Vec::new(), + zone_lengths: Vec::new(), + has_nulls: Vec::new(), + bloom_filter_data: Vec::new(), + current_bytes: 0, + has_written: false, + } + } + + async fn emit(&mut self, block: BloomFilterStatistics) -> Result<()> { + let serialized_filter = block.bloom_filter.to_bytes(); + let serialized_len = serialized_filter.len(); + + if serialized_len > self.max_array_length { + return Err(Error::invalid_input(format!( + "Serialized bloom filter size {} exceeds max supported batch bytes {}", + serialized_len, self.max_array_length + ))); + } + + let next_bytes = self + .current_bytes + .checked_add(serialized_len) + .ok_or_else(|| { + Error::invalid_input(format!( + "Bloom filter batch size overflow when adding {} bytes to {} bytes", + serialized_len, self.current_bytes + )) + })?; + + if !self.bloom_filter_data.is_empty() && next_bytes > self.max_array_length { + self.flush().await?; + } + + self.fragment_ids.push(block.bound.fragment_id); + self.zone_starts.push(block.bound.start); + self.zone_lengths.push(block.bound.length as u64); + self.has_nulls.push(block.has_null); + self.bloom_filter_data.push(serialized_filter); + self.current_bytes += serialized_len; + Ok(()) + } + + async fn flush(&mut self) -> Result<()> { + if self.bloom_filter_data.is_empty() { + return Ok(()); + } + + let batch = BloomFilterIndexBuilder::bloomfilter_stats_as_batch( + std::mem::take(&mut self.fragment_ids), + std::mem::take(&mut self.zone_starts), + std::mem::take(&mut self.zone_lengths), + std::mem::take(&mut self.has_nulls), + std::mem::take(&mut self.bloom_filter_data), + )?; + self.file.write_record_batch(batch).await?; + self.current_bytes = 0; + self.has_written = true; + Ok(()) + } + + async fn finish(mut self, null_rows: Option) -> Result { + self.flush().await?; + if !self.has_written { + self.file + .write_record_batch(BloomFilterIndexBuilder::bloomfilter_stats_as_batch( + Vec::new(), + Vec::new(), + Vec::new(), + Vec::new(), + Vec::new(), + )?) + .await?; + } + + if let Some(null_rows) = null_rows { + let mut null_bitmap_bytes = Vec::with_capacity(null_rows.serialized_size()); + null_rows.serialize_into(&mut null_bitmap_bytes)?; + let null_bitmap_idx = self + .file + .add_global_buffer(bytes::Bytes::from(null_bitmap_bytes)) + .await?; + self.file + .finish_with_metadata(HashMap::from([( + NULL_BITMAP_META_KEY.to_string(), + null_bitmap_idx.to_string(), + )])) + .await + } else { + self.file.finish_with_metadata(HashMap::new()).await + } } } @@ -980,7 +1327,7 @@ impl BloomFilterIndexPlugin { batches_source: SendableRecordBatchStream, index_store: &dyn IndexStore, options: Option, - ) -> Result { + ) -> Result> { let mut builder = BloomFilterIndexBuilder::try_new(options.unwrap_or_default())?; builder.train(batches_source).await?; @@ -1049,15 +1396,9 @@ impl BasicTrainer for BloomFilterIndexPlugin { data: SendableRecordBatchStream, index_store: &dyn IndexStore, request: Box, - fragment_ids: Option>, + _fragment_ids: Option>, _progress: Arc, ) -> Result { - if fragment_ids.is_some() { - return Err(Error::invalid_input_source( - "BloomFilter index does not support fragment training".into(), - )); - } - let request = (request as Box) .downcast::() .map_err(|_| { @@ -1065,12 +1406,12 @@ impl BasicTrainer for BloomFilterIndexPlugin { "must provide training request created by new_training_request".into(), ) })?; - let file = Self::train_bloomfilter_index(data, index_store, Some(request.params)).await?; + let files = Self::train_bloomfilter_index(data, index_store, Some(request.params)).await?; Ok(CreatedIndex { index_details: prost_types::Any::from_msg(&pb::BloomFilterIndexDetails::default()) .unwrap(), index_version: BLOOMFILTER_INDEX_VERSION, - files: vec![file], + files, }) } } @@ -1154,7 +1495,9 @@ impl TrainingRequest for BloomFilterIndexTrainingRequest { #[cfg(test)] mod tests { + use crate::frag_reuse::{FragReuseIndex, FragReuseIndexDetails, FragReuseIndexHandle}; use crate::scalar::registry::VALUE_COLUMN_NAME; + use std::collections::HashMap; use std::sync::Arc; use crate::scalar::bloomfilter::BloomFilterIndexPlugin; @@ -1164,15 +1507,19 @@ mod tests { use datafusion::physical_plan::stream::RecordBatchStreamAdapter; use datafusion_common::ScalarValue; use futures::{StreamExt, stream}; - use lance_core::{ROW_ADDR, cache::LanceCache, utils::tempfile::TempObjDir}; + use lance_core::{Error, ROW_ADDR, cache::LanceCache, utils::tempfile::TempObjDir}; use lance_io::object_store::ObjectStore; use lance_select::RowAddrTreeMap; use crate::scalar::{ - BloomFilterQuery, ScalarIndex, SearchResult, - bloomfilter::{BloomFilterIndex, BloomFilterIndexBuilderParams}, + BloomFilterQuery, IndexStore, ScalarIndex, SearchResult, + bloomfilter::{ + BloomFilterIndex, BloomFilterIndexBuilder, BloomFilterIndexBuilderParams, + merge_bloomfilter_indices, + }, lance_format::LanceIndexStore, }; + use lance_core::utils::bloomfilter::sbbf::Sbbf; use crate::Index; // Import Index trait to access calculate_included_frags use crate::metrics::NoOpMetricsCollector; @@ -2037,10 +2384,10 @@ mod tests { expected.insert_range(500..750); // Should match the zone containing 500 assert_eq!(result, SearchResult::at_most(expected)); - // Test IsNull query + // Test IsNull query (no nulls in data, should return exact empty set) let query = BloomFilterQuery::IsNull(); let result = index.search(&query, &NoOpMetricsCollector).await.unwrap(); - assert_eq!(result, SearchResult::at_most(RowAddrTreeMap::new())); // No nulls in the data + assert_eq!(result, SearchResult::exact(RowAddrTreeMap::new())); // Test IsIn query let query = BloomFilterQuery::IsIn(vec![ @@ -2133,4 +2480,523 @@ mod tests { _ => panic!("Expected AtMost search result from bloomfilter"), } } + + #[test] + fn test_bloomfilter_read_batch_size_is_byte_bounded() { + let number_of_items = 1; + let probability = 0.25; + let max_test_batch_bytes = 48; + let filter_bytes = Sbbf::with_ndv_fpp(number_of_items, probability) + .unwrap() + .to_bytes(); + + assert!(filter_bytes.len() <= max_test_batch_bytes); + assert!(filter_bytes.len() * 2 > max_test_batch_bytes); + + assert_eq!( + BloomFilterIndex::read_batch_size(number_of_items, probability, max_test_batch_bytes,) + .unwrap(), + 1 + ); + } + + #[test] + fn test_bloomfilter_read_batch_size_rejects_oversized_filter() { + let number_of_items = 1; + let probability = 0.25; + let filter_bytes = Sbbf::with_ndv_fpp(number_of_items, probability) + .unwrap() + .to_bytes(); + + let error = + BloomFilterIndex::read_batch_size(number_of_items, probability, filter_bytes.len() - 1) + .unwrap_err(); + assert!( + matches!(error, Error::InvalidInput { .. }), + "unexpected error variant: {error:?}" + ); + assert!( + error + .to_string() + .contains("exceeds max supported batch bytes"), + "unexpected error: {error}" + ); + } + + #[tokio::test] + async fn test_bloomfilter_chunked_write_and_load() { + let tmpdir = TempObjDir::default(); + let test_store = Arc::new(LanceIndexStore::new( + Arc::new(ObjectStore::local()), + tmpdir.clone(), + Arc::new(LanceCache::no_cache()), + )); + + let row_count = 5_000; + // Sprinkle nulls at known positions (none of which are asserted individually + // below) so the chunked write must carry a non-empty null-row bitmap through + // multiple flushes and reload it correctly via add_global_buffer. + let expected_null_count = (0..row_count).filter(|&i| i % 100 == 50).count(); + let values = (0..row_count) + .map(|i| (i % 100 != 50).then_some(i)) + .collect::>(); + let data = record_batch!((VALUE_COLUMN_NAME, Int32, values)).unwrap(); + let schema = data.schema(); + let data_stream: SendableRecordBatchStream = Box::pin(RecordBatchStreamAdapter::new( + schema, + stream::once(std::future::ready(Ok(data))), + )); + let data_stream = add_row_addr(data_stream); + + let mut builder = + BloomFilterIndexBuilder::try_new(BloomFilterIndexBuilderParams::new(1, 0.25)).unwrap(); + builder.train(data_stream).await.unwrap(); + assert_eq!(builder.blocks.len(), row_count as usize); + + // Small byte limit forces the writer to emit many on-disk batches and the + // reader to chunk its reads. The total payload is far larger than the limit, + // so a load that concatenated everything into one BinaryArray would trip the + // read-side cap; a correctly chunked load must still succeed. + let max_array_length = 64; + let filter_bytes = Sbbf::with_ndv_fpp(1, 0.25).unwrap().size_bytes(); + assert!(filter_bytes * row_count as usize > max_array_length); + + builder + .write_index_with_max_array_length(test_store.as_ref(), max_array_length) + .await + .unwrap(); + + let index = BloomFilterIndex::load_with_max_array_length( + test_store.clone(), + None, + &LanceCache::no_cache(), + max_array_length, + ) + .await + .expect("Failed to load chunked BloomFilterIndex"); + + assert_eq!(index.zones.len(), row_count as usize); + assert_eq!(index.zones[0].bound.start, 0); + assert_eq!(index.zones[4096].bound.start, 4096); + assert_eq!(index.zones[4999].bound.start, 4999); + assert_eq!(index.zones[4999].bound.length, 1); + assert!(!index.zones[4096].bloom_filter.to_bytes().is_empty()); + + // The null-row bitmap is stored as a global buffer, independent of the chunked + // zone batches. Verify it survives the multi-flush write and reloads intact. + let null_rows = index + .null_rows + .as_ref() + .expect("chunked write must preserve the null-row bitmap"); + let loaded_null_count = null_rows + .row_addrs() + .map(|addrs| addrs.count()) + .unwrap_or(0); + assert_eq!(loaded_null_count, expected_null_count); + } + + #[tokio::test] + async fn test_bloomfilter_load_rejects_unchunked_oversized_read() { + // Guards the read-side invariant directly: a batch whose concatenated filter + // payload exceeds the cap must be rejected rather than building an oversized + // BinaryArray. This is what protects `load` if its chunking ever regresses. + let filter_bytes = Sbbf::with_ndv_fpp(1, 0.25).unwrap().to_bytes(); + let batch = BloomFilterIndexBuilder::bloomfilter_stats_as_batch( + vec![0, 0], + vec![0, 1], + vec![1, 1], + vec![false, false], + vec![filter_bytes.clone(), filter_bytes.clone()], + ) + .unwrap(); + + // One filter fits, two together do not. + let max_array_length = filter_bytes.len(); + let error = BloomFilterIndex::try_from_serialized(batch, max_array_length).unwrap_err(); + assert!( + matches!(error, Error::InvalidInput { .. }), + "unexpected error variant: {error:?}" + ); + assert!( + error + .to_string() + .contains("exceeds max supported batch bytes"), + "unexpected error: {error}" + ); + } + + #[tokio::test] + async fn test_bloomfilter_chunked_write_rejects_oversized_filter() { + let tmpdir = TempObjDir::default(); + let test_store = Arc::new(LanceIndexStore::new( + Arc::new(ObjectStore::local()), + tmpdir.clone(), + Arc::new(LanceCache::no_cache()), + )); + + let data = record_batch!((VALUE_COLUMN_NAME, Int32, [0])).unwrap(); + let schema = data.schema(); + let data_stream: SendableRecordBatchStream = Box::pin(RecordBatchStreamAdapter::new( + schema, + stream::once(std::future::ready(Ok(data))), + )); + let data_stream = add_row_addr(data_stream); + + let mut builder = + BloomFilterIndexBuilder::try_new(BloomFilterIndexBuilderParams::new(1, 0.25)).unwrap(); + builder.train(data_stream).await.unwrap(); + + let error = builder + .write_index_with_max_array_length(test_store.as_ref(), 16) + .await + .unwrap_err(); + assert!( + matches!(error, Error::InvalidInput { .. }), + "unexpected error variant: {error:?}" + ); + assert!( + error + .to_string() + .contains("exceeds max supported batch bytes 16"), + "unexpected error: {error}" + ); + } + + // Writes a bloomfilter file in the legacy format (no null bitmap global buffer), + // simulating an index created before the null bitmap feature was added. + async fn write_legacy_bloomfilter(store: &dyn IndexStore, has_null: bool) { + use crate::scalar::bloomfilter::{ + BLOOMFILTER_FILENAME, BLOOMFILTER_ITEM_META_KEY, BLOOMFILTER_PROBABILITY_META_KEY, + }; + use arrow_array::BooleanArray; + let schema = Arc::new(Schema::new(vec![ + Field::new("fragment_id", DataType::UInt64, false), + Field::new("zone_start", DataType::UInt64, false), + Field::new("zone_length", DataType::UInt64, false), + Field::new("has_null", DataType::Boolean, false), + Field::new("bloom_filter_data", DataType::Binary, false), + ])); + let batch = RecordBatch::try_new( + schema.clone(), + vec![ + Arc::new(UInt64Array::from(vec![0u64])) as _, + Arc::new(UInt64Array::from(vec![0u64])) as _, + Arc::new(UInt64Array::from(vec![3u64])) as _, + Arc::new(BooleanArray::from(vec![has_null])) as _, + Arc::new(arrow_array::BinaryArray::from_vec(vec![b"".as_ref()])) as _, + ], + ) + .unwrap(); + let mut file_schema = schema.as_ref().clone(); + file_schema + .metadata + .insert(BLOOMFILTER_ITEM_META_KEY.to_string(), "1000".to_string()); + file_schema.metadata.insert( + BLOOMFILTER_PROBABILITY_META_KEY.to_string(), + "0.01".to_string(), + ); + let mut writer = store + .new_index_file(BLOOMFILTER_FILENAME, Arc::new(file_schema)) + .await + .unwrap(); + writer.write_record_batch(batch).await.unwrap(); + writer.finish().await.unwrap(); + } + + // Updating a legacy (null_rows = None) index must not silently treat None as + // "no nulls". The bug: `self.null_rows.clone().unwrap_or_default()` collapses + // None into an empty RowAddrTreeMap; after the merge the updated index has + // `null_rows = Some(empty)`, so an IsNull search returns `exact(empty)` — a + // false negative even though the legacy zone recorded has_null = true. + #[tokio::test] + async fn test_update_legacy_none_null_rows_not_treated_as_no_nulls() { + let tmpdir = TempObjDir::default(); + let store = Arc::new(LanceIndexStore::new( + Arc::new(ObjectStore::local()), + tmpdir.clone(), + Arc::new(LanceCache::no_cache()), + )); + + // Write a legacy-format index (no null bitmap) with has_null=true in its zone. + write_legacy_bloomfilter(store.as_ref(), true).await; + + let index = BloomFilterIndex::load(store.clone(), None, &LanceCache::no_cache()) + .await + .unwrap(); + assert!( + index.null_rows.is_none(), + "precondition: legacy null_rows is None" + ); + + // Update with new data from fragment 1 (no nulls). The destination is the + // same store so we can reload from it afterwards. + let new_schema = Arc::new(Schema::new(vec![ + Field::new(VALUE_COLUMN_NAME, DataType::Int32, true), + Field::new(ROW_ADDR, DataType::UInt64, false), + ])); + let new_batch = RecordBatch::try_new( + new_schema.clone(), + vec![ + Arc::new(arrow_array::Int32Array::from(vec![ + Some(10i32), + Some(20), + Some(30), + ])) as _, + Arc::new(UInt64Array::from_iter_values( + (0u64..3).map(|i| (1u64 << 32) | i), + )) as _, + ], + ) + .unwrap(); + let new_stream: SendableRecordBatchStream = Box::pin(RecordBatchStreamAdapter::new( + new_schema, + stream::once(std::future::ready(Ok(new_batch))), + )); + + index + .update(new_stream, store.as_ref(), None) + .await + .unwrap(); + + let updated_index = BloomFilterIndex::load(store.clone(), None, &LanceCache::no_cache()) + .await + .unwrap(); + + // The legacy zone had has_null=true, so there ARE nulls at unknown positions. + // An IsNull search on the updated index must NOT claim "no nulls" (exact empty). + // It must be conservative and return AtMost, falling back to the has_null scan. + let result = updated_index + .search(&BloomFilterQuery::IsNull(), &NoOpMetricsCollector) + .await + .unwrap(); + + // With the bug: null_rows = Some(empty) → returns exact(empty) ← FALSE NEGATIVE + // With the fix: null_rows = None → falls through to has_null scan → AtMost + assert!( + !result.is_exact(), + "IsNull on an updated legacy index must not return exact(empty); \ + the legacy zone had has_null=true so nulls exist at unknown positions" + ); + } + + #[tokio::test] + async fn test_legacy_bloomfilter_no_null_bitmap() { + let tmpdir = TempObjDir::default(); + let store = Arc::new(LanceIndexStore::new( + Arc::new(ObjectStore::local()), + tmpdir.clone(), + Arc::new(LanceCache::no_cache()), + )); + + write_legacy_bloomfilter(store.as_ref(), true).await; + + let index = BloomFilterIndex::load(store, None, &LanceCache::no_cache()) + .await + .expect("failed to load legacy bloomfilter"); + + assert!( + index.null_rows.is_none(), + "legacy index should have no null bitmap" + ); + + // IS NULL should fall back to the has_null zone scan and return AtMost, not Exact. + let result = index + .search(&BloomFilterQuery::IsNull(), &NoOpMetricsCollector) + .await + .unwrap(); + assert!( + !result.is_exact(), + "IS NULL on a legacy index should not be exact" + ); + } + + #[tokio::test] + async fn test_merge_bloomfilter_indices_preserves_exact_and_legacy_nulls() { + fn create_store() -> (TempObjDir, Arc) { + let tmpdir = TempObjDir::default(); + let store = Arc::new(LanceIndexStore::new( + Arc::new(ObjectStore::local()), + tmpdir.clone(), + Arc::new(LanceCache::no_cache()), + )); + (tmpdir, store) + } + + let (_first_tmpdir, first_store) = create_store(); + let (_second_tmpdir, second_store) = create_store(); + let (_merged_tmpdir, merged_store) = create_store(); + let params = BloomFilterIndexBuilderParams::new(1000, 0.01); + for (fragment_id, store) in [(0_u64, &first_store), (1_u64, &second_store)] { + let row_base = fragment_id << 32; + let batch = record_batch!( + (VALUE_COLUMN_NAME, Int64, [Some(10), None, Some(30)]), + (ROW_ADDR, UInt64, [row_base, row_base + 1, row_base + 2]) + ) + .unwrap(); + let schema = batch.schema(); + let stream = Box::pin(RecordBatchStreamAdapter::new( + schema, + stream::once(async move { Ok(batch) }), + )); + BloomFilterIndexPlugin::train_bloomfilter_index( + stream, + store.as_ref(), + Some(params.clone()), + ) + .await + .unwrap(); + } + + let first = BloomFilterIndex::load(first_store.clone(), None, &LanceCache::no_cache()) + .await + .unwrap(); + let second = BloomFilterIndex::load(second_store, None, &LanceCache::no_cache()) + .await + .unwrap(); + merge_bloomfilter_indices( + &[ + (first.as_ref(), &RoaringBitmap::from_iter([0])), + (second.as_ref(), &RoaringBitmap::from_iter([1])), + ], + merged_store.as_ref(), + ) + .await + .unwrap(); + let merged = BloomFilterIndex::load(merged_store.clone(), None, &LanceCache::no_cache()) + .await + .unwrap(); + let mut expected_nulls = RowAddrTreeMap::new(); + expected_nulls.insert(1); + expected_nulls.insert((1_u64 << 32) + 1); + assert_eq!( + merged + .search(&BloomFilterQuery::IsNull(), &NoOpMetricsCollector) + .await + .unwrap(), + SearchResult::exact(expected_nulls) + ); + + let remapped_base = 2_u64 << 32; + let remapper = FragReuseIndexHandle(Arc::new(FragReuseIndex::new( + uuid::Uuid::new_v4(), + vec![HashMap::from([ + (0, Some(remapped_base)), + (1, Some(remapped_base + 1)), + (2, Some(remapped_base + 2)), + ])], + FragReuseIndexDetails { versions: vec![] }, + ))); + let remapped_first = BloomFilterIndex::load( + first_store, + Some(Arc::new(remapper)), + &LanceCache::no_cache(), + ) + .await + .unwrap(); + let (_remapped_tmpdir, remapped_store) = create_store(); + merge_bloomfilter_indices( + &[(remapped_first.as_ref(), &RoaringBitmap::from_iter([2]))], + remapped_store.as_ref(), + ) + .await + .unwrap(); + let remapped = BloomFilterIndex::load(remapped_store, None, &LanceCache::no_cache()) + .await + .unwrap(); + let mut expected_remapped_nulls = RowAddrTreeMap::new(); + expected_remapped_nulls.insert(remapped_base + 1); + assert_eq!( + remapped + .search(&BloomFilterQuery::IsNull(), &NoOpMetricsCollector) + .await + .unwrap(), + SearchResult::exact(expected_remapped_nulls) + ); + let candidates = remapped + .search( + &BloomFilterQuery::Equals(ScalarValue::Int64(Some(10))), + &NoOpMetricsCollector, + ) + .await + .unwrap(); + assert!( + candidates + .row_addrs() + .true_rows() + .row_addrs() + .unwrap() + .map(u64::from) + .any(|row_id| row_id == remapped_base) + ); + + let (_legacy_tmpdir, legacy_store) = create_store(); + let (_legacy_merged_tmpdir, legacy_merged_store) = create_store(); + write_legacy_bloomfilter(legacy_store.as_ref(), true).await; + let legacy = BloomFilterIndex::load(legacy_store, None, &LanceCache::no_cache()) + .await + .unwrap(); + merge_bloomfilter_indices( + &[ + (legacy.as_ref(), &RoaringBitmap::from_iter([0])), + (second.as_ref(), &RoaringBitmap::from_iter([1])), + ], + legacy_merged_store.as_ref(), + ) + .await + .unwrap(); + let legacy_merged = + BloomFilterIndex::load(legacy_merged_store, None, &LanceCache::no_cache()) + .await + .unwrap(); + assert!( + !legacy_merged + .search(&BloomFilterQuery::IsNull(), &NoOpMetricsCollector) + .await + .unwrap() + .is_exact() + ); + + let (_trimmed_tmpdir, trimmed_store) = create_store(); + let mut ignored_legacy = legacy.as_ref().clone(); + ignored_legacy.probability = 0.5; + merge_bloomfilter_indices( + &[ + (&ignored_legacy, &RoaringBitmap::new()), + (second.as_ref(), &RoaringBitmap::from_iter([1])), + ], + trimmed_store.as_ref(), + ) + .await + .unwrap(); + let trimmed = BloomFilterIndex::load(trimmed_store, None, &LanceCache::no_cache()) + .await + .unwrap(); + let mut expected_nulls = RowAddrTreeMap::new(); + expected_nulls.insert((1_u64 << 32) + 1); + assert_eq!( + trimmed + .search(&BloomFilterQuery::IsNull(), &NoOpMetricsCollector) + .await + .unwrap(), + SearchResult::exact(expected_nulls) + ); + + let (_empty_tmpdir, empty_store) = create_store(); + merge_bloomfilter_indices( + &[(&ignored_legacy, &RoaringBitmap::new())], + empty_store.as_ref(), + ) + .await + .unwrap(); + let empty = BloomFilterIndex::load(empty_store, None, &LanceCache::no_cache()) + .await + .unwrap(); + assert_eq!( + empty + .search(&BloomFilterQuery::IsNull(), &NoOpMetricsCollector) + .await + .unwrap(), + SearchResult::exact(RowAddrTreeMap::new()) + ); + } } diff --git a/rust/lance-index/src/scalar/btree.rs b/rust/lance-index/src/scalar/btree.rs index 7201574d58a..305df6ebae1 100644 --- a/rust/lance-index/src/scalar/btree.rs +++ b/rust/lance-index/src/scalar/btree.rs @@ -63,8 +63,8 @@ use lance_core::deepsize::DeepSizeOf; use lance_core::{ Error, ROW_ID, Result, cache::{ - CacheCodec, CacheCodecImpl, CacheEntryReader, CacheEntryWriter, CacheKey, LanceCache, - WeakLanceCache, + CacheCodec, CacheCodecImpl, CacheEntryReader, CacheEntryWriter, CacheKey, CacheKeySchema, + KeyBuilder, LanceCache, WeakLanceCache, }, error::LanceOptionExt, utils::{ @@ -397,6 +397,8 @@ impl Ord for OrderableScalarValue { panic!("Attempt to compare List with non-List") } (LargeList(_), _) => todo!(), + (ListView(_), _) => todo!(), + (LargeListView(_), _) => todo!(), (Map(_), Map(_)) => todo!(), (Map(left), Null) => { if left.is_null(0) { @@ -1363,6 +1365,14 @@ impl CacheKey for BTreePageKey { "BTreePage" } + fn schema() -> CacheKeySchema { + CacheKeySchema::new("lance.scalar.btree-page-key", 1) + } + + fn write_key(&self, builder: &mut KeyBuilder) { + builder.write_u32(self.page_number); + } + fn codec() -> Option { // Pages are cached as `FlatIndex` values (see `ValueType` above). Some(CacheCodec::from_impl::()) @@ -1489,6 +1499,14 @@ impl CacheKey for BTreeIndexStateKey { "BTreeIndexState" } + fn schema() -> CacheKeySchema { + CacheKeySchema::new("lance.scalar.btree-index-state-key", 1) + } + + fn write_key(&self, builder: &mut KeyBuilder) { + builder.write_variant(0); + } + fn codec() -> Option { Some(CacheCodec::from_impl::()) } @@ -1633,11 +1651,17 @@ impl BTreeIndex { index_reader: LazyIndexReader, metrics: &dyn MetricsCollector, ) -> Result> { - self.index_cache - .get_or_insert_with_key(BTreePageKey { page_number }, move || async move { + let result = self + .index_cache + .get_or_insert_with_key_hit(BTreePageKey { page_number }, move || async move { self.read_page(page_number, index_reader, metrics).await }) - .await + .await; + match &result { + Ok((_, true)) => metrics.record_index_cache_hit(), + _ => metrics.record_index_cache_miss(), + } + result.map(|(page, _)| page) } #[instrument(level = "debug", skip_all)] @@ -2344,6 +2368,10 @@ impl ScalarIndex for BTreeIndex { })?; Ok(ScalarIndexParams::for_builtin(BuiltinIndexType::BTree).with_params(¶ms)) } + + fn training_data_type(&self) -> Option { + Some(self.data_type.clone()) + } } struct BatchStats { @@ -2562,9 +2590,7 @@ pub async fn train_btree_index( Ok(vec![pages_file, lookup_file]) } -fn find_single_partition_files( - files: &[lance_table::format::IndexFile], -) -> Result> { +fn find_single_partition_files(files: &[super::IndexFile]) -> Result> { let lookup_files = files .iter() .filter_map(|file| { @@ -3704,6 +3730,53 @@ mod tests { assert_eq!(metrics.parts_loaded.load(Ordering::Relaxed), 1); } + #[tokio::test] + async fn test_page_cache_hit_miss_counts() { + let tmpdir = TempObjDir::default(); + let test_store = Arc::new(LanceIndexStore::new( + Arc::new(ObjectStore::local()), + tmpdir.clone(), + Arc::new(LanceCache::no_cache()), + )); + + let data = gen_batch() + .col("value", array::step::()) + .col("_rowid", array::step::()) + .into_df_exec(RowCount::from(1000), BatchCount::from(10)); + let schema = data.schema(); + let sort_expr = PhysicalSortExpr::new_default(col("value", schema.as_ref()).unwrap()); + let plan = Arc::new(SortExec::new([sort_expr].into(), data)); + let stream = plan.execute(0, Arc::new(TaskContext::default())).unwrap(); + let stream = break_stream(stream, 64); + let stream = stream.map_err(DataFusionError::from); + let stream = + Box::pin(RecordBatchStreamAdapter::new(schema, stream)) as SendableRecordBatchStream; + + train_btree_index(stream, test_store.as_ref(), 64, None, None) + .await + .unwrap(); + + let cache = Arc::new(LanceCache::with_capacity(100 * 1024 * 1024)); + let index = BTreeIndex::load(test_store, None, cache.as_ref()) + .await + .unwrap(); + + // First search: cold cache — the page fetch must miss. + let query = SargableQuery::Equals(ScalarValue::Float32(Some(0.0))); + let cold = LocalMetricsCollector::default(); + index.search(&query, &cold).await.unwrap(); + assert_eq!(cold.index_cache_hits(), 0); + assert_eq!(cold.index_cache_misses(), 1); + assert_eq!(cold.parts_loaded.load(Ordering::Relaxed), 1); + + // Second search: same key, page must now be served from cache. + let warm = LocalMetricsCollector::default(); + index.search(&query, &warm).await.unwrap(); + assert_eq!(warm.index_cache_hits(), 1); + assert_eq!(warm.index_cache_misses(), 0); + assert_eq!(warm.parts_loaded.load(Ordering::Relaxed), 0); + } + #[tokio::test] async fn test_like_prefix_search() { use arrow::datatypes::DataType; @@ -5327,12 +5400,8 @@ mod tests { mapping.insert(old_id, None); } - let mut new_id_counter = 100_000; - // Remap all other rows - for old_id in (0..1000).chain(10000..15000) { - let new_id = new_id_counter; - new_id_counter += 1; + for (new_id, old_id) in (100_000..).zip((0..1000).chain(10000..15000)) { mapping.insert(old_id, Some(new_id)); } @@ -6289,7 +6358,7 @@ mod tests { #[tokio::test] async fn test_btree_index_state_reconstruct_applies_frag_reuse_index() { - use crate::frag_reuse::{FragReuseIndex, FragReuseIndexDetails}; + use crate::frag_reuse::{FragReuseIndex, FragReuseIndexDetails, FragReuseIndexHandle}; use std::collections::HashMap; use uuid::Uuid; @@ -6322,11 +6391,11 @@ mod tests { // Querying for value == 0 should now return row 5000, confirming reconstruct threaded // the FragReuseIndex through to the rebuilt BTreeIndex. let frag_reuse_index: Arc = - Arc::new(FragReuseIndex::new( + Arc::new(FragReuseIndexHandle(Arc::new(FragReuseIndex::new( Uuid::new_v4(), vec![HashMap::from([(0u64, Some(5000u64))])], FragReuseIndexDetails { versions: vec![] }, - )); + )))); let reconstructed = state .reconstruct( test_store.clone(), diff --git a/rust/lance-index/src/scalar/btree/flat.rs b/rust/lance-index/src/scalar/btree/flat.rs index 7eb9f0422f3..1af5a6a69e9 100644 --- a/rust/lance-index/src/scalar/btree/flat.rs +++ b/rust/lance-index/src/scalar/btree/flat.rs @@ -218,13 +218,13 @@ impl FlatIndex { )); } } - (Bound::Included(lower) | Bound::Excluded(lower), Bound::Unbounded) => { - if lower.is_null() { - return Ok(NullableRowAddrSet::new( - Default::default(), - self.all_addrs_map.clone(), - )); - } + (Bound::Included(lower) | Bound::Excluded(lower), Bound::Unbounded) + if lower.is_null() => + { + return Ok(NullableRowAddrSet::new( + Default::default(), + self.all_addrs_map.clone(), + )); } _ => {} }, diff --git a/rust/lance-index/src/scalar/expression.rs b/rust/lance-index/src/scalar/expression.rs index e495d5b701f..20d42abb8f8 100644 --- a/rust/lance-index/src/scalar/expression.rs +++ b/rust/lance-index/src/scalar/expression.rs @@ -1,7 +1,12 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright The Lance Authors -use std::{ops::Bound, sync::Arc}; +use std::{ + cmp::Ordering, + collections::{HashMap, HashSet, hash_map::Entry}, + ops::Bound, + sync::Arc, +}; use arrow_schema::{DataType, Field}; use async_recursion::async_recursion; @@ -15,7 +20,7 @@ use tokio::try_join; use super::{ AnyQuery, BloomFilterQuery, LabelListQuery, MetricsCollector, SargableQuery, ScalarIndex, - SearchResult, TextQuery, TokenQuery, + SearchResult, TextQuery, TokenQuery, label_list::validate_label_list_data_type, }; #[cfg(feature = "geo")] use super::{GeoQuery, RelationQuery}; @@ -269,6 +274,10 @@ pub struct SargableQueryParser { index_name: String, index_type: String, needs_recheck: bool, + /// Whether IS NULL queries need post-filter rechecking. May be false even + /// when `needs_recheck` is true for indexes that track exact null positions + /// (e.g. zone maps with a null bitmap). + is_null_needs_recheck: bool, supports_like_prefix: bool, } @@ -278,6 +287,7 @@ impl SargableQueryParser { index_name, index_type, needs_recheck, + is_null_needs_recheck: needs_recheck, supports_like_prefix: true, } } @@ -289,6 +299,14 @@ impl SargableQueryParser { self.supports_like_prefix = false; self } + + /// Mark IS NULL as exact so that IS NOT NULL can be served by the index + /// without a post-filter. Use this when the index tracks null row addresses + /// precisely (e.g. a zone map with a null bitmap). + pub fn with_exact_null_tracking(mut self) -> Self { + self.is_null_needs_recheck = false; + self + } } impl ScalarQueryParser for SargableQueryParser { @@ -357,7 +375,7 @@ impl ScalarQueryParser for SargableQueryParser { self.index_name.clone(), self.index_type.clone(), Arc::new(SargableQuery::IsNull()), - self.needs_recheck, + self.is_null_needs_recheck, )) } @@ -769,6 +787,11 @@ impl ScalarQueryParser for LabelListQueryParser { if args.len() != 2 { return None; } + // LABEL_LIST stores unnested items as bitmap keys. Nested items cannot be + // ordered by the bitmap lookup, so legacy indexes must fall back to a scan. + if validate_label_list_data_type(data_type).is_err() { + return None; + } // DataFusion normalizes array_contains to array_has if func.name() == "array_has" { let inner_type = match data_type { @@ -791,34 +814,34 @@ impl ScalarQueryParser for LabelListQueryParser { } let label_list = maybe_scalar(&args[1], data_type)?; - if let ScalarValue::List(list_arr) = label_list { - let list_values = list_arr.values(); - if list_values.is_empty() { - return None; - } - let mut scalars = Vec::with_capacity(list_values.len()); - for idx in 0..list_values.len() { - scalars.push(ScalarValue::try_from_array(list_values.as_ref(), idx).ok()?); - } - if func.name() == "array_has_all" { - let query = LabelListQuery::HasAllLabels(scalars); - Some(IndexedExpression::index_query( - column.to_string(), - self.index_name.clone(), - self.index_type.clone(), - Arc::new(query), - )) - } else if func.name() == "array_has_any" { - let query = LabelListQuery::HasAnyLabel(scalars); - Some(IndexedExpression::index_query( - column.to_string(), - self.index_name.clone(), - self.index_type.clone(), - Arc::new(query), - )) - } else { - None - } + let list_values = match label_list { + ScalarValue::List(list_arr) => list_arr.values().clone(), + ScalarValue::LargeList(list_arr) => list_arr.values().clone(), + _ => return None, + }; + if list_values.is_empty() { + return None; + } + let mut scalars = Vec::with_capacity(list_values.len()); + for idx in 0..list_values.len() { + scalars.push(ScalarValue::try_from_array(list_values.as_ref(), idx).ok()?); + } + if func.name() == "array_has_all" { + let query = LabelListQuery::HasAllLabels(scalars); + Some(IndexedExpression::index_query( + column.to_string(), + self.index_name.clone(), + self.index_type.clone(), + Arc::new(query), + )) + } else if func.name() == "array_has_any" { + let query = LabelListQuery::HasAnyLabel(scalars); + Some(IndexedExpression::index_query( + column.to_string(), + self.index_name.clone(), + self.index_type.clone(), + Arc::new(query), + )) } else { None } @@ -833,6 +856,10 @@ pub struct TextQueryParser { index_type: String, needs_recheck: bool, supports_regex: bool, + /// The shortest `contains` pattern (in characters, not bytes) the index can + /// say anything useful about. Shorter patterns bypass the index entirely so + /// they are answered by a full scan. Use 0 for indices with no such limit. + min_contains_chars: usize, } impl TextQueryParser { @@ -841,12 +868,14 @@ impl TextQueryParser { index_type: String, needs_recheck: bool, supports_regex: bool, + min_contains_chars: usize, ) -> Self { Self { index_name, index_type, needs_recheck, supports_regex, + min_contains_chars, } } } @@ -903,7 +932,16 @@ impl ScalarQueryParser for TextQueryParser { }; let query = match func.name() { - "contains" if args.len() == 2 => TextQuery::StringContains(pattern), + "contains" if args.len() == 2 => { + // A pattern shorter than the index's token width produces no + // tokens, so the index can only answer "recheck everything". + // Leave it to a full scan instead. The count is in characters + // because tokenizers split on characters, not bytes. + if pattern.chars().count() < self.min_contains_chars { + return None; + } + TextQuery::StringContains(pattern) + } "regexp_like" | "regexp_match" if self.supports_regex => { let pattern = match args.get(2) { Some(flags_expr) => apply_regex_flags(&pattern, flags_expr)?, @@ -1402,6 +1440,20 @@ pub trait ScalarIndexLoader: Send + Sync { index_name: &str, metrics: &dyn MetricsCollector, ) -> Result>; + + /// Translate an address-domain index result into the row-id domain + /// + /// Address-domain indices (see [`ScalarIndex::results_are_row_addresses`]) + /// report matches as physical row addresses. The default returns `result` + /// unchanged, which is correct when addresses and row ids coincide (no + /// stable row ids). A dataset with stable row ids overrides this to remap + /// addresses to stable row ids via its per-fragment row-id sequences. + async fn row_addr_result_to_row_ids( + &self, + result: NullableIndexExprResult, + ) -> Result { + Ok(result) + } } /// This represents a search into a scalar index @@ -1462,173 +1514,389 @@ impl PartialEq for ScalarIndexExpr { } } +/// Conservative grouping key for rewrites targeting the same parsed scalar index. +/// `index_type` is display metadata, but including it prevents rewrites across +/// index implementations that may not share query semantics. +type ScalarIndexQueryKey = (String, String, String); + /// Returns the tighter (more restrictive) lower bound. -/// Priority: Included/Excluded > Unbounded; Excluded > Included for same value. -fn tighter_lower_bound(a: &Bound, b: &Bound) -> Bound { +/// +/// Returns `None` if the bound values cannot be compared. In that case callers +/// keep the original predicates rather than inventing ordering semantics. +fn tighter_lower_bound( + a: &Bound, + b: &Bound, +) -> Option> { match (a, b) { - (Bound::Unbounded, other) | (other, Bound::Unbounded) => other.clone(), - (Bound::Included(va), Bound::Included(vb)) => { - if va >= vb { - Bound::Included(va.clone()) - } else { - Bound::Included(vb.clone()) - } - } - (Bound::Excluded(va), Bound::Excluded(vb)) => { - if va >= vb { - Bound::Excluded(va.clone()) - } else { - Bound::Excluded(vb.clone()) - } - } - (Bound::Excluded(va), Bound::Included(vb)) => { - if va >= vb { - Bound::Excluded(va.clone()) - } else { - Bound::Included(vb.clone()) - } - } - (Bound::Included(va), Bound::Excluded(vb)) => { - if vb >= va { - Bound::Excluded(vb.clone()) - } else { - Bound::Included(va.clone()) - } - } + (Bound::Unbounded, Bound::Unbounded) => Some(Bound::Unbounded), + (Bound::Unbounded, other) | (other, Bound::Unbounded) => Some(other.clone()), + ( + Bound::Included(a_value) | Bound::Excluded(a_value), + Bound::Included(b_value) | Bound::Excluded(b_value), + ) => match a_value.partial_cmp(b_value)? { + Ordering::Less => Some(b.clone()), + Ordering::Equal => Some(stricter_bound_for_equal_value(a_value, a, b)), + Ordering::Greater => Some(a.clone()), + }, } } /// Returns the tighter (more restrictive) upper bound. -/// Priority: Included/Excluded > Unbounded; Excluded > Included for same value. -fn tighter_upper_bound(a: &Bound, b: &Bound) -> Bound { +/// +/// Returns `None` if the bound values cannot be compared. In that case callers +/// keep the original predicates rather than inventing ordering semantics. +fn tighter_upper_bound( + a: &Bound, + b: &Bound, +) -> Option> { match (a, b) { - (Bound::Unbounded, other) | (other, Bound::Unbounded) => other.clone(), - (Bound::Included(va), Bound::Included(vb)) => { - if va <= vb { - Bound::Included(va.clone()) - } else { - Bound::Included(vb.clone()) - } - } - (Bound::Excluded(va), Bound::Excluded(vb)) => { - if va <= vb { - Bound::Excluded(va.clone()) - } else { - Bound::Excluded(vb.clone()) + (Bound::Unbounded, Bound::Unbounded) => Some(Bound::Unbounded), + (Bound::Unbounded, other) | (other, Bound::Unbounded) => Some(other.clone()), + ( + Bound::Included(a_value) | Bound::Excluded(a_value), + Bound::Included(b_value) | Bound::Excluded(b_value), + ) => match a_value.partial_cmp(b_value)? { + Ordering::Less => Some(a.clone()), + Ordering::Equal => Some(stricter_bound_for_equal_value(a_value, a, b)), + Ordering::Greater => Some(b.clone()), + }, + } +} + +fn is_excluded_bound(bound: &Bound) -> bool { + matches!(bound, Bound::Excluded(_)) +} + +/// For an equal scalar value, an excluded bound is stricter than an included +/// bound. This handles cases like `x >= 5 AND x > 5`. +fn stricter_bound_for_equal_value( + value: &ScalarValue, + lhs: &Bound, + rhs: &Bound, +) -> Bound { + if is_excluded_bound(lhs) || is_excluded_bound(rhs) { + Bound::Excluded(value.clone()) + } else { + Bound::Included(value.clone()) + } +} + +fn range_has_non_null_bound(lower: &Bound, upper: &Bound) -> bool { + let mut has_bound = false; + for bound in [lower, upper] { + match bound { + Bound::Included(value) | Bound::Excluded(value) => { + if value.is_null() { + return false; + } + has_bound = true; } + Bound::Unbounded => {} } - (Bound::Excluded(va), Bound::Included(vb)) => { - if va <= vb { - Bound::Excluded(va.clone()) - } else { - Bound::Included(vb.clone()) - } + } + has_bound +} + +/// Null bounds are skipped by range optimization. Comparisons with NULL should +/// already be rejected by normal parsing, but this keeps manually constructed +/// ScalarIndexExpr values from being rewritten into misleading ranges. +fn range_has_null_bound(lower: &Bound, upper: &Bound) -> bool { + [lower, upper].iter().any(|bound| match bound { + Bound::Included(value) | Bound::Excluded(value) => value.is_null(), + Bound::Unbounded => false, + }) +} + +impl ScalarIndexSearch { + /// Only SargableQuery participates in these expression-level rewrites. + /// Other AnyQuery implementations may have different null/range semantics. + fn sargable_query(&self) -> Option<&SargableQuery> { + self.query.as_any().downcast_ref::() + } + + /// Require a concrete SargableQuery before producing a key so callers do not + /// accidentally group unrelated AnyQuery implementations by metadata alone. + fn sargable_query_key(&self) -> Option { + self.sargable_query()?; + Some(( + self.column.clone(), + self.index_name.clone(), + self.index_type.clone(), + )) + } + + /// Only exact SargableQuery values may drive NULL-elimination rewrites. + /// Range merging also accepts inexact queries and preserves their recheck. + fn exact_sargable_query(&self) -> Option<&SargableQuery> { + if self.needs_recheck { + return None; } - (Bound::Included(va), Bound::Excluded(vb)) => { - if vb <= va { - Bound::Excluded(vb.clone()) - } else { - Bound::Included(va.clone()) + self.sargable_query() + } + + /// Return a scalar-index key only when the query is an exact SargableQuery. + fn exact_sargable_query_key(&self) -> Option { + self.exact_sargable_query()?; + self.sargable_query_key() + } + + /// A query is null-intolerant if the original predicate cannot match NULL. + /// Only exact queries can make `IS NOT NULL` redundant in the index expression. + fn is_null_intolerant_sargable_query(&self) -> bool { + match self.exact_sargable_query() { + Some(SargableQuery::Range(lower, upper)) => range_has_non_null_bound(lower, upper), + Some(SargableQuery::Equals(value)) => !value.is_null(), + Some(SargableQuery::IsIn(values)) => { + !values.is_empty() && values.iter().all(|value| !value.is_null()) } + _ => false, } } } impl ScalarIndexExpr { - /// Optimize the expression tree by merging range queries on the same index. + /// Apply scalar-index optimizer rules within each contiguous AND region. + /// + /// Range queries for the same parsed scalar index are intersected and retain + /// `needs_recheck` if any input range requires it. An exact `IS NOT NULL` + /// query is removed when another exact same-index query is null-intolerant. + /// OR branches are optimized independently. Range queries are still merged + /// under NOT, but `IS NOT NULL` elimination is disabled there to preserve + /// SQL NULL semantics. + /// + /// Compatible [`SargableQuery::Range`] values carried by + /// [`ScalarIndexSearch`] are merged into one query. + /// + /// # Example /// - /// This collects all leaf Range queries from the AND tree, groups them by - /// index name, merges overlapping ranges into a single closed-range query, - /// and rebuilds the tree. This handles the case where `log_time >= X` and - /// `log_time <= Y` end up in different branches of a nested AND tree. + /// ``` + /// # use std::{ops::Bound, sync::Arc}; + /// # use lance_index::scalar::{SargableQuery, expression::{ScalarIndexExpr, ScalarIndexSearch}}; + /// # let range = |lower, upper| ScalarIndexExpr::Query(ScalarIndexSearch { + /// # column: "x".into(), + /// # index_name: "x_idx".into(), + /// # index_type: "BTree".into(), + /// # query: Arc::new(SargableQuery::Range(lower, upper)), + /// # needs_recheck: false, + /// # fragment_bitmap: None, + /// # }); + /// let optimized = ScalarIndexExpr::And( + /// Box::new(range(Bound::Included(10_i64.into()), Bound::Unbounded)), + /// Box::new(range(Bound::Unbounded, Bound::Included(20_i64.into()))), + /// ) + /// .optimize(); + /// + /// assert!(matches!(optimized, ScalarIndexExpr::Query(_))); + /// ``` pub fn optimize(self) -> Self { + self.optimize_with_context(true) + } + + fn optimize_with_context(self, is_not_null_elidable: bool) -> Self { match self { - Self::And(_, _) => self.optimize_and_tree(), - Self::Or(lhs, rhs) => Self::Or(Box::new(lhs.optimize()), Box::new(rhs.optimize())), - Self::Not(inner) => Self::Not(Box::new(inner.optimize())), + Self::And(_, _) => self.optimize_and_tree(is_not_null_elidable), + Self::Or(lhs, rhs) => Self::Or( + Box::new(lhs.optimize_with_context(is_not_null_elidable)), + Box::new(rhs.optimize_with_context(is_not_null_elidable)), + ), + Self::Not(inner) => Self::Not(Box::new(inner.optimize_with_context(false))), other => other, } } - /// Flatten an AND tree, merge ranges on same index, rebuild. - fn optimize_and_tree(self) -> Self { + /// Flatten one contiguous AND region, apply local optimizer rules, and + /// rebuild a balanced AND tree. + fn optimize_and_tree(self, is_not_null_elidable: bool) -> Self { let mut leaves = Vec::new(); - self.collect_and_leaves(&mut leaves); + self.collect_and_leaves(&mut leaves, is_not_null_elidable); + let leaves = Self::optimize_and_leaves(leaves, is_not_null_elidable); + + Self::rebuild_and_tree(leaves).expect("AND tree optimization should keep at least one leaf") + } + + /// Apply optimizer rules within one AND region only. OR branches have already + /// been kept as opaque leaves, so range and null-intolerance rewrites cannot + /// cross boolean boundaries. + fn optimize_and_leaves(leaves: Vec, is_not_null_elidable: bool) -> Vec { + let null_intolerant_keys = if !is_not_null_elidable { + HashSet::new() + } else { + let is_not_null_keys = leaves + .iter() + .filter_map(Self::is_not_null_query_key) + .collect::>(); + if is_not_null_keys.is_empty() { + HashSet::new() + } else { + leaves + .iter() + .filter_map(Self::null_intolerant_query_key) + .filter(|key| is_not_null_keys.contains(key)) + .collect() + } + }; - // Try to merge Range queries on the same index - let mut merged_indices: Vec = vec![false; leaves.len()]; - let mut result_leaves: Vec = Vec::new(); + let mut optimized = Vec::with_capacity(leaves.len()); + let mut range_positions = HashMap::new(); - for i in 0..leaves.len() { - if merged_indices[i] { + for leaf in leaves { + if let Some(key) = leaf.is_not_null_query_key() + && null_intolerant_keys.contains(&key) + { continue; } - let mut current = leaves[i].clone(); - // Try to merge with subsequent leaves on the same index - for j in (i + 1)..leaves.len() { - if merged_indices[j] { - continue; - } - if let Some(merged) = try_merge_range_pair(¤t, &leaves[j]) { - current = merged; - merged_indices[j] = true; + if let Some(key) = leaf.range_query_key() { + match range_positions.entry(key) { + Entry::Vacant(entry) => { + entry.insert(optimized.len()); + optimized.push(leaf); + } + Entry::Occupied(entry) => { + if !optimized[*entry.get()].try_merge_range(&leaf) { + optimized.push(leaf); + } + } } + } else { + optimized.push(leaf); } - - result_leaves.push(current); } - // Rebuild the AND tree from remaining leaves - let mut iter = result_leaves.into_iter(); - let first = iter.next().expect("AND tree must have at least one leaf"); - iter.fold(first, |acc, leaf| Self::And(Box::new(acc), Box::new(leaf))) + optimized } - /// Recursively collect all leaf nodes from an AND tree. - fn collect_and_leaves(self, leaves: &mut Vec) { + /// Recursively collect all leaf nodes from an AND tree. OR branches are + /// optimized independently with the current null-elision context. NOT + /// branches remain leaves while their children are optimized with null + /// elimination disabled. + fn collect_and_leaves(self, leaves: &mut Vec, is_not_null_elidable: bool) { match self { Self::And(lhs, rhs) => { - lhs.collect_and_leaves(leaves); - rhs.collect_and_leaves(leaves); + lhs.collect_and_leaves(leaves, is_not_null_elidable); + rhs.collect_and_leaves(leaves, is_not_null_elidable); + } + Self::Or(lhs, rhs) => { + leaves.push(Self::Or( + Box::new(lhs.optimize_with_context(is_not_null_elidable)), + Box::new(rhs.optimize_with_context(is_not_null_elidable)), + )); + } + Self::Not(inner) => { + leaves.push(Self::Not(Box::new(inner.optimize_with_context(false)))) } other => leaves.push(other), } } -} -/// Try to merge two ScalarIndexExpr nodes if they are both Range queries on the same index. -fn try_merge_range_pair(lhs: &ScalarIndexExpr, rhs: &ScalarIndexExpr) -> Option { - let (ScalarIndexExpr::Query(l), ScalarIndexExpr::Query(r)) = (lhs, rhs) else { - return None; - }; - if l.index_name != r.index_name || l.column != r.column { - return None; + /// Rebuild as a balanced tree so large planner-generated conjunctions do not + /// become deep left-leaning trees after optimization. + fn rebuild_and_tree(leaves: Vec) -> Option { + let mut leaves = leaves; + if leaves.is_empty() { + return None; + } + + while leaves.len() > 1 { + let mut next = Vec::with_capacity(leaves.len().div_ceil(2)); + let mut iter = leaves.into_iter(); + while let Some(lhs) = iter.next() { + if let Some(rhs) = iter.next() { + next.push(Self::And(Box::new(lhs), Box::new(rhs))); + } else { + next.push(lhs); + } + } + leaves = next; + } + + leaves.pop() } - let l_query = l.query.as_any().downcast_ref::()?; - let r_query = r.query.as_any().downcast_ref::()?; + /// Detect the scalar-index representation of `col IS NOT NULL`, which is + /// stored as `NOT(col IS NULL)`. + fn is_not_null_query_key(&self) -> Option { + match self { + Self::Not(inner) => match inner.as_ref() { + Self::Query(search) + if matches!(search.sargable_query(), Some(SargableQuery::IsNull())) => + { + search.exact_sargable_query_key() + } + _ => None, + }, + _ => None, + } + } - let (SargableQuery::Range(l_low, l_high), SargableQuery::Range(r_low, r_high)) = - (l_query, r_query) - else { - return None; - }; + /// Return the key of a same-column predicate that makes `IS NOT NULL` + /// redundant in the same AND region. + fn null_intolerant_query_key(&self) -> Option { + match self { + Self::Query(search) if search.is_null_intolerant_sargable_query() => { + search.exact_sargable_query_key() + } + Self::Not(inner) => match inner.as_ref() { + Self::Query(search) + if matches!( + search.exact_sargable_query(), + Some(SargableQuery::Equals(value)) if !value.is_null() + ) => + { + search.exact_sargable_query_key() + } + _ => None, + }, + _ => None, + } + } + + /// Return the grouping key for an optimizable range query. Ranges with NULL + /// bounds are left unchanged. + fn range_query_key(&self) -> Option { + let Self::Query(search) = self else { + return None; + }; + let SargableQuery::Range(lower, upper) = search.sargable_query()? else { + return None; + }; + if range_has_null_bound(lower, upper) { + return None; + } + search.sargable_query_key() + } + + /// Merge another compatible range into this expression. + /// + /// The caller must first group both expressions by [`ScalarIndexQueryKey`]. + /// Different fragment coverage or incomparable bounds leave both predicates + /// unchanged. Empty intersections are retained as ranges. + fn try_merge_range(&mut self, other: &Self) -> bool { + let (Self::Query(search), Self::Query(other_search)) = (self, other) else { + return false; + }; + if search.fragment_bitmap != other_search.fragment_bitmap { + return false; + } + + let ( + Some(SargableQuery::Range(lower, upper)), + Some(SargableQuery::Range(other_lower, other_upper)), + ) = (search.sargable_query(), other_search.sargable_query()) + else { + return false; + }; + let Some(lower) = tighter_lower_bound(lower, other_lower) else { + return false; + }; + let Some(upper) = tighter_upper_bound(upper, other_upper) else { + return false; + }; - let merged_low = tighter_lower_bound(l_low, r_low); - let merged_high = tighter_upper_bound(l_high, r_high); - - Some(ScalarIndexExpr::Query(ScalarIndexSearch { - column: l.column.clone(), - index_name: l.index_name.clone(), - index_type: l.index_type.clone(), - query: Arc::new(SargableQuery::Range(merged_low, merged_high)), - needs_recheck: l.needs_recheck || r.needs_recheck, - // Both queries target the same index (checked above), so they share - // the same fragment coverage; carry it over to keep the merged query - // usable by coverage-dependent optimizer rules. - fragment_bitmap: l.fragment_bitmap.clone(), - })) + search.query = Arc::new(SargableQuery::Range(lower, upper)); + search.needs_recheck |= other_search.needs_recheck; + true + } } impl std::fmt::Display for ScalarIndexExpr { @@ -1648,12 +1916,16 @@ impl std::fmt::Display for ScalarIndexExpr { } } -impl From for NullableIndexExprResult { - fn from(result: SearchResult) -> Self { - match result { - SearchResult::Exact(mask) => Self::exact(NullableRowAddrMask::AllowList(mask)), - SearchResult::AtMost(mask) => Self::at_most(NullableRowAddrMask::AllowList(mask)), - SearchResult::AtLeast(mask) => Self::at_least(NullableRowAddrMask::AllowList(mask)), +fn search_result_to_nullable(result: SearchResult) -> NullableIndexExprResult { + match result { + SearchResult::Exact(mask) => { + NullableIndexExprResult::exact(NullableRowAddrMask::AllowList(mask)) + } + SearchResult::AtMost(mask) => { + NullableIndexExprResult::at_most(NullableRowAddrMask::AllowList(mask)) + } + SearchResult::AtLeast(mask) => { + NullableIndexExprResult::at_least(NullableRowAddrMask::AllowList(mask)) } } } @@ -1693,7 +1965,15 @@ impl ScalarIndexExpr { .load_index(&search.column, &search.index_name, metrics) .await?; let search_result = index.search(search.query.as_ref(), metrics).await?; - Ok(search_result.into()) + let result = search_result_to_nullable(search_result); + if index.results_are_row_addresses() { + // Translate address-domain results to the row-id domain + // before combining or scanning; otherwise stable-row-id + // datasets silently drop matches (issue #7434). + index_loader.row_addr_result_to_row_ids(result).await + } else { + Ok(result) + } } } } @@ -1744,27 +2024,30 @@ fn maybe_column(expr: &Expr) -> Option<&str> { } } -// Extract the full nested column path from a get_field expression chain -// For example: get_field(get_field(metadata, "status"), "code") -> "metadata.status.code" +// Extract the full nested column path from chained or variadic get_field expressions. +// For example, both get_field(get_field(metadata, "status"), "code") and +// get_field(metadata, "status", "code") produce "metadata.status.code". fn extract_nested_column_path(expr: &Expr) -> Option { let mut current_expr = expr; let mut parts = Vec::new(); - // Walk up the get_field chain + // Walk up the get_field chain. Add variadic arguments in reverse because all + // parts are reversed after reaching the base column. loop { match current_expr { Expr::ScalarFunction(udf) if udf.name() == "get_field" => { - if udf.args.len() != 2 { + if udf.args.len() < 2 { return None; } - // Extract the field name from the second argument - // The Literal now has two fields: ScalarValue and Option - if let Expr::Literal(ScalarValue::Utf8(Some(field_name)), _) = &udf.args[1] { - parts.push(field_name.clone()); - } else { - return None; + + for field_expr in udf.args[1..].iter().rev() { + if let Expr::Literal(ScalarValue::Utf8(Some(field_name)), _) = field_expr { + parts.push(field_name.clone()); + } else { + return None; + } } - // Move up to the parent expression + current_expr = &udf.args[0]; } Expr::Column(col) => { @@ -1838,7 +2121,7 @@ fn maybe_scalar(expr: &Expr, expected_type: &DataType) -> Option { // In this case we need to extract the value, apply the cast, and then test the casted value Expr::Cast(cast) => match cast.expr.as_ref() { Expr::Literal(value, _) => { - let casted = value.cast_to(&cast.data_type).ok()?; + let casted = value.cast_to(cast.field.data_type()).ok()?; safe_coerce_scalar(&casted, expected_type) } _ => None, @@ -2352,13 +2635,14 @@ mod tests { use std::collections::HashMap; use arrow_array::Array; - use arrow_schema::{Field, Schema}; + use arrow_schema::{Field, Fields, Schema}; use chrono::Utc; use datafusion_common::{Column, DFSchema}; use datafusion_expr::simplify::SimplifyContext; use lance_datafusion::exec::{LanceExecutionOptions, get_session_context}; use lance_select::result::IndexExprResultWireFormat; use roaring::RoaringBitmap; + use rstest::rstest; use crate::scalar::json::{JsonQuery, JsonQueryParser}; @@ -2420,15 +2704,26 @@ mod tests { Field::new("price", DataType::Float32, false), Field::new("json", DataType::LargeBinary, false), ]); + check_with_schema(index_info, expr, expected, optimize, schema); + } + + fn check_with_schema( + index_info: &dyn IndexInformationProvider, + expr: &str, + expected: Option, + optimize: bool, + schema: Schema, + ) { let df_schema: DFSchema = schema.try_into().unwrap(); let ctx = get_session_context(&LanceExecutionOptions::default()); let state = ctx.state(); let mut expr = state.create_logical_expr(expr, &df_schema).unwrap(); if optimize { - let simplify_context = SimplifyContext::default() + let simplify_context = SimplifyContext::builder() .with_schema(Arc::new(df_schema)) - .with_query_execution_start_time(Some(Utc::now())); + .with_query_execution_start_time(Some(Utc::now())) + .build(); let simplifier = datafusion::optimizer::simplify_expressions::ExprSimplifier::new(simplify_context); expr = simplifier.simplify(expr).unwrap(); @@ -2508,6 +2803,117 @@ mod tests { ) } + #[rstest] + #[case::list(DataType::List(Arc::new(Field::new("item", DataType::Utf8, true))))] + #[case::large_list(DataType::LargeList(Arc::new(Field::new("item", DataType::Utf8, true))))] + fn test_label_list_query_parser(#[case] label_type: DataType) { + let index_info = MockIndexInfoProvider::new(vec![( + "labels", + ColInfo::new( + label_type.clone(), + Box::new(LabelListQueryParser::new( + "labels_idx".to_string(), + "LabelList".to_string(), + )), + ), + )]); + let schema = Schema::new(vec![Field::new("labels", label_type, true)]); + + check_with_schema( + &index_info, + "array_has_any(labels, ['distributed'])", + Some(IndexedExpression::index_query( + "labels".to_string(), + "labels_idx".to_string(), + "LabelList".to_string(), + Arc::new(LabelListQuery::HasAnyLabel(vec![ScalarValue::Utf8(Some( + "distributed".to_string(), + ))])), + )), + true, + schema, + ); + } + + #[rstest] + #[case::list(DataType::List(Arc::new(Field::new( + "item", + DataType::List(Arc::new(Field::new("item", DataType::Int64, true))), + true, + ))))] + #[case::large_list(DataType::LargeList(Arc::new(Field::new( + "item", + DataType::List(Arc::new(Field::new("item", DataType::Int64, true))), + true, + ))))] + fn test_label_list_nested_item(#[case] label_type: DataType) { + let index_info = MockIndexInfoProvider::new(vec![( + "labels", + ColInfo::new( + label_type.clone(), + Box::new(LabelListQueryParser::new( + "labels_idx".to_string(), + "LabelList".to_string(), + )), + ), + )]); + let schema = Schema::new(vec![Field::new("labels", label_type, true)]); + + // Nested item types must not produce a scalar index query. + check_with_schema( + &index_info, + "array_has_any(labels, [[1]])", + None, + true, + schema, + ); + } + + #[test] + fn test_nested_column_index() { + let column = "user_profile.basic.age"; + let index_info = MockIndexInfoProvider::new(vec![( + column, + ColInfo::new( + DataType::Int32, + Box::new(SargableQueryParser::new( + format!("{column}_idx"), + "BTree".to_string(), + false, + )), + ), + )]); + let schema = Schema::new(vec![Field::new( + "user_profile", + DataType::Struct(Fields::from(vec![Field::new( + "basic", + DataType::Struct(Fields::from(vec![Field::new("age", DataType::Int32, true)])), + true, + )])), + true, + )]); + + let planner = Planner::new(Arc::new(schema)); + let filter = planner + .parse_filter("user_profile.basic.age > 900") + .unwrap(); + let plan = planner + .create_filter_plan(filter, &index_info, true) + .unwrap(); + let expected = IndexedExpression::index_query( + column.to_string(), + format!("{column}_idx"), + "BTree".to_string(), + Arc::new(SargableQuery::Range( + Bound::Excluded(ScalarValue::Int32(Some(900))), + Bound::Unbounded, + )), + ); + + assert_eq!(plan.index_query, expected.scalar_query); + assert!(plan.refine_expr.is_none()); + } + #[test] fn test_expressions() { let index_info = MockIndexInfoProvider::new(vec![ @@ -3302,9 +3708,10 @@ mod tests { .unwrap(); // Apply DataFusion simplification (this may convert starts_with to LIKE) - let simplify_context = SimplifyContext::default() + let simplify_context = SimplifyContext::builder() .with_schema(Arc::new(df_schema)) - .with_query_execution_start_time(Some(Utc::now())); + .with_query_execution_start_time(Some(Utc::now())) + .build(); let simplifier = datafusion::optimizer::simplify_expressions::ExprSimplifier::new(simplify_context); let simplified_expr = simplifier.simplify(expr).unwrap(); @@ -3871,7 +4278,7 @@ mod tests { // and the other queries as separate leaves // Let's verify by collecting leaves let mut leaves = Vec::new(); - optimized.collect_and_leaves(&mut leaves); + optimized.collect_and_leaves(&mut leaves, true); // Should have 3 leaves: fqdn, merged_time, channel assert_eq!( @@ -3943,7 +4350,7 @@ mod tests { // Should remain as two separate leaves (not merged) let mut leaves = Vec::new(); - optimized.collect_and_leaves(&mut leaves); + optimized.collect_and_leaves(&mut leaves, true); assert_eq!(leaves.len(), 2); } @@ -3980,7 +4387,7 @@ mod tests { let optimized = expr.optimize(); let mut leaves = Vec::new(); - optimized.collect_and_leaves(&mut leaves); + optimized.collect_and_leaves(&mut leaves, true); assert_eq!(leaves.len(), 2, "Equals + Range should not merge"); } @@ -4020,7 +4427,7 @@ mod tests { let optimized = expr.optimize(); let mut leaves = Vec::new(); - optimized.collect_and_leaves(&mut leaves); + optimized.collect_and_leaves(&mut leaves, true); assert_eq!(leaves.len(), 1); if let ScalarIndexExpr::Query(s) = &leaves[0] { @@ -4099,7 +4506,7 @@ mod tests { let optimized = expr.optimize(); let mut leaves = Vec::new(); - optimized.collect_and_leaves(&mut leaves); + optimized.collect_and_leaves(&mut leaves, true); // Should have 2 leaves: OR node (preserved) + merged range assert_eq!(leaves.len(), 2); @@ -4129,54 +4536,678 @@ mod tests { } #[test] - fn test_optimize_needs_recheck_preserved() { - use super::{ScalarIndexExpr, ScalarIndexSearch}; - use crate::scalar::SargableQuery; - use datafusion_common::ScalarValue; - use std::ops::Bound; - use std::sync::Arc; + fn test_optimize_respects_fragment_coverage_when_merging_ranges() { + let fragments = RoaringBitmap::from_iter([1, 3, 5]); + let lower = test_scalar_range_with_metadata( + Bound::Included(ScalarValue::Int64(Some(1))), + Bound::Unbounded, + true, + Some(fragments.clone()), + ); + let upper = test_scalar_range_with_metadata( + Bound::Unbounded, + Bound::Included(ScalarValue::Int64(Some(99))), + false, + Some(fragments.clone()), + ); - // If either range has needs_recheck=true, merged result should too - let range_a = ScalarIndexExpr::Query(ScalarIndexSearch { - column: "x".to_string(), - index_name: "idx_x".to_string(), - index_type: "".to_string(), - query: Arc::new(SargableQuery::Range( - Bound::Included(ScalarValue::Int64(Some(1))), - Bound::Unbounded, - )), - needs_recheck: true, - fragment_bitmap: None, - }); - let range_b = ScalarIndexExpr::Query(ScalarIndexSearch { - column: "x".to_string(), - index_name: "idx_x".to_string(), - index_type: "".to_string(), - query: Arc::new(SargableQuery::Range( + let leaves = collect_test_and_leaves(test_and_terms(vec![lower.clone(), upper]).optimize()); + assert_eq!(leaves.len(), 1); + assert!(matches!( + &leaves[0], + ScalarIndexExpr::Query(search) + if search.needs_recheck + && search.fragment_bitmap.as_ref() == Some(&fragments) + && matches!( + search.sargable_query(), + Some(SargableQuery::Range( + Bound::Included(ScalarValue::Int64(Some(1))), + Bound::Included(ScalarValue::Int64(Some(99))), + )) + ) + )); + + let upper_without_coverage = test_scalar_range_with_metadata( + Bound::Unbounded, + Bound::Included(ScalarValue::Int64(Some(99))), + false, + None, + ); + for (lhs, rhs) in [ + (lower.clone(), upper_without_coverage.clone()), + (upper_without_coverage, lower), + ] { + let optimized = ScalarIndexExpr::And(Box::new(lhs), Box::new(rhs)).optimize(); + let leaves = collect_test_and_leaves(optimized); + assert_eq!(leaves.len(), 2); + assert!(leaves.iter().any(|leaf| { + matches!(leaf, ScalarIndexExpr::Query(search) + if search.needs_recheck + && search.fragment_bitmap.as_ref() == Some(&fragments)) + })); + assert!(leaves.iter().any(|leaf| { + matches!(leaf, ScalarIndexExpr::Query(search) + if !search.needs_recheck && search.fragment_bitmap.is_none()) + })); + } + } + + #[test] + fn test_optimize_merges_recheck_ranges_into_empty_range() { + let lower = test_scalar_query_with_recheck( + "x", + "idx_x", + SargableQuery::Range( + Bound::Included(ScalarValue::Int64(Some(200))), Bound::Unbounded, - Bound::Included(ScalarValue::Int64(Some(99))), - )), + ), + ); + let upper = test_scalar_query_with_recheck( + "x", + "idx_x", + SargableQuery::Range( + Bound::Unbounded, + Bound::Included(ScalarValue::Int64(Some(100))), + ), + ); + + let leaves = collect_test_and_leaves(test_and_terms(vec![lower, upper]).optimize()); + + assert_eq!(leaves.len(), 1); + assert!(matches!( + &leaves[0], + ScalarIndexExpr::Query(search) + if search.needs_recheck + && matches!( + search.sargable_query(), + Some(SargableQuery::Range( + Bound::Included(ScalarValue::Int64(Some(200))), + Bound::Included(ScalarValue::Int64(Some(100))), + )) + ) + )); + } + + fn test_scalar_query(column: &str, index_name: &str, query: SargableQuery) -> ScalarIndexExpr { + ScalarIndexExpr::Query(ScalarIndexSearch { + column: column.to_string(), + index_name: index_name.to_string(), + index_type: "BTree".to_string(), + query: Arc::new(query), needs_recheck: false, fragment_bitmap: None, - }); + }) + } - let expr = ScalarIndexExpr::And(Box::new(range_a), Box::new(range_b)); - let optimized = expr.optimize(); + fn test_scalar_query_with_recheck( + column: &str, + index_name: &str, + query: SargableQuery, + ) -> ScalarIndexExpr { + ScalarIndexExpr::Query(ScalarIndexSearch { + column: column.to_string(), + index_name: index_name.to_string(), + index_type: "ZoneMap".to_string(), + query: Arc::new(query), + needs_recheck: true, + fragment_bitmap: None, + }) + } + + fn test_scalar_range( + column: &str, + index_name: &str, + lower: Bound, + upper: Bound, + ) -> ScalarIndexExpr { + test_scalar_query(column, index_name, SargableQuery::Range(lower, upper)) + } + fn test_scalar_range_with_metadata( + lower: Bound, + upper: Bound, + needs_recheck: bool, + fragment_bitmap: Option, + ) -> ScalarIndexExpr { + ScalarIndexExpr::Query(ScalarIndexSearch { + column: "x".to_string(), + index_name: "idx_x".to_string(), + index_type: "ZoneMap".to_string(), + query: Arc::new(SargableQuery::Range(lower, upper)), + needs_recheck, + fragment_bitmap, + }) + } + + fn test_and_terms(mut terms: Vec) -> ScalarIndexExpr { + assert!(!terms.is_empty()); + while terms.len() > 1 { + let mut next = Vec::with_capacity(terms.len().div_ceil(2)); + let mut iter = terms.into_iter(); + while let Some(lhs) = iter.next() { + if let Some(rhs) = iter.next() { + next.push(ScalarIndexExpr::And(Box::new(lhs), Box::new(rhs))); + } else { + next.push(lhs); + } + } + terms = next; + } + terms.pop().unwrap() + } + + fn collect_test_and_leaves(expr: ScalarIndexExpr) -> Vec { let mut leaves = Vec::new(); - optimized.collect_and_leaves(&mut leaves); - assert_eq!(leaves.len(), 1); + expr.collect_and_leaves(&mut leaves, true); + leaves + } - if let ScalarIndexExpr::Query(s) = &leaves[0] { - assert!( - s.needs_recheck, - "Merged query should preserve needs_recheck=true" - ); - } else { - panic!("Expected a Query leaf"); + fn test_and_depth(expr: &ScalarIndexExpr) -> usize { + match expr { + ScalarIndexExpr::And(lhs, rhs) => 1 + test_and_depth(lhs).max(test_and_depth(rhs)), + ScalarIndexExpr::Or(lhs, rhs) => test_and_depth(lhs).max(test_and_depth(rhs)), + ScalarIndexExpr::Not(inner) => test_and_depth(inner), + ScalarIndexExpr::Query(_) => 0, } } + fn balanced_depth_bound(mut term_count: usize) -> usize { + let mut depth = 0; + while term_count > 1 { + term_count = term_count.div_ceil(2); + depth += 1; + } + depth + } + + fn int64_index_info(index_type: &str, needs_recheck: bool) -> MockIndexInfoProvider { + let parser = |column: &str| { + ColInfo::new( + DataType::Int64, + Box::new(SargableQueryParser::new( + format!("{}_idx", column), + index_type.to_string(), + needs_recheck, + )), + ) + }; + MockIndexInfoProvider::new(vec![("x", parser("x")), ("y", parser("y"))]) + } + + fn parse_int64_filter( + expr: &str, + index_info: &dyn IndexInformationProvider, + ) -> IndexedExpression { + let schema = Schema::new(vec![ + Field::new("x", DataType::Int64, true), + Field::new("y", DataType::Int64, true), + ]); + let df_schema: DFSchema = schema.try_into().unwrap(); + let ctx = get_session_context(&LanceExecutionOptions::default()); + let state = ctx.state(); + let expr = state.create_logical_expr(expr, &df_schema).unwrap(); + apply_scalar_indices(expr, index_info).unwrap() + } + + fn optimize_parsed_scalar_filter( + expr: &str, + index_info: &dyn IndexInformationProvider, + ) -> Vec { + let indexed = parse_int64_filter(expr, index_info); + collect_test_and_leaves(indexed.scalar_query.unwrap().optimize()) + } + + #[test] + fn test_optimize_does_not_remove_is_not_null_for_recheck_range() { + let is_not_null = ScalarIndexExpr::Not(Box::new(test_scalar_query( + "x", + "idx_x", + SargableQuery::IsNull(), + ))); + let mut range = test_scalar_range( + "x", + "idx_x", + Bound::Included(ScalarValue::Int64(Some(10))), + Bound::Unbounded, + ); + let ScalarIndexExpr::Query(search) = &mut range else { + panic!("expected a range query"); + }; + search.needs_recheck = true; + + let leaves = collect_test_and_leaves(test_and_terms(vec![is_not_null, range]).optimize()); + + assert_eq!(leaves.len(), 2); + assert!(leaves.iter().any(|leaf| { + matches!( + leaf, + ScalarIndexExpr::Not(inner) + if matches!(inner.as_ref(), ScalarIndexExpr::Query(search) + if matches!(search.sargable_query(), Some(SargableQuery::IsNull())) + && !search.needs_recheck) + ) + })); + } + + #[test] + fn test_optimize_does_not_remove_is_not_null_across_or() { + let is_not_null = ScalarIndexExpr::Not(Box::new(test_scalar_query( + "x", + "idx_x", + SargableQuery::IsNull(), + ))); + let range = test_scalar_range( + "x", + "idx_x", + Bound::Included(ScalarValue::Int64(Some(10))), + Bound::Unbounded, + ); + let other = test_scalar_query( + "y", + "idx_y", + SargableQuery::Equals(ScalarValue::Int64(Some(1))), + ); + let disjunction = ScalarIndexExpr::Or(Box::new(range), Box::new(other)); + + let leaves = collect_test_and_leaves( + ScalarIndexExpr::And(Box::new(is_not_null), Box::new(disjunction)).optimize(), + ); + + assert_eq!(leaves.len(), 2); + assert!( + leaves + .iter() + .any(|leaf| matches!(leaf, ScalarIndexExpr::Not(_))) + ); + assert!( + leaves + .iter() + .any(|leaf| matches!(leaf, ScalarIndexExpr::Or(_, _))) + ); + } + + #[test] + fn test_optimize_does_not_remove_is_not_null_for_different_index() { + let is_not_null = ScalarIndexExpr::Not(Box::new(test_scalar_query( + "x", + "idx_x", + SargableQuery::IsNull(), + ))); + let range = test_scalar_range( + "x", + "idx_x_other", + Bound::Included(ScalarValue::Int64(Some(10))), + Bound::Unbounded, + ); + + let leaves = collect_test_and_leaves( + ScalarIndexExpr::And(Box::new(is_not_null), Box::new(range)).optimize(), + ); + + assert_eq!(leaves.len(), 2); + assert!( + leaves + .iter() + .any(|leaf| matches!(leaf, ScalarIndexExpr::Not(_))) + ); + } + + #[test] + fn test_optimize_does_not_merge_different_index_types() { + let range = |index_type: &str, lower, upper| { + ScalarIndexExpr::Query(ScalarIndexSearch { + column: "x".to_string(), + index_name: "idx_x".to_string(), + index_type: index_type.to_string(), + query: Arc::new(SargableQuery::Range(lower, upper)), + needs_recheck: false, + fragment_bitmap: None, + }) + }; + let btree = range( + "BTree", + Bound::Included(ScalarValue::Int64(Some(10))), + Bound::Unbounded, + ); + let zone_map = range( + "ZoneMap", + Bound::Unbounded, + Bound::Included(ScalarValue::Int64(Some(20))), + ); + + let leaves = collect_test_and_leaves(test_and_terms(vec![btree, zone_map]).optimize()); + + assert_eq!(leaves.len(), 2); + } + + #[test] + fn test_optimize_parser_exact_removes_is_not_null_and_merges_ranges() { + let index_info = int64_index_info("BTree", false); + + let leaves = optimize_parsed_scalar_filter( + "x IS NOT NULL AND y = 1 AND x >= 10 AND x <= 20", + &index_info, + ); + + assert_eq!(leaves.len(), 2); + assert!(leaves.iter().any(|leaf| { + matches!( + leaf, + ScalarIndexExpr::Query(search) + if search.column == "x" + && matches!( + search.sargable_query(), + Some(SargableQuery::Range( + Bound::Included(ScalarValue::Int64(Some(10))), + Bound::Included(ScalarValue::Int64(Some(20))), + )) + ) + ) + })); + assert!(leaves.iter().any(|leaf| { + matches!( + leaf, + ScalarIndexExpr::Query(search) + if search.column == "y" + && matches!( + search.sargable_query(), + Some(SargableQuery::Equals(ScalarValue::Int64(Some(1)))) + ) + ) + })); + assert!( + !leaves + .iter() + .any(|leaf| matches!(leaf, ScalarIndexExpr::Not(_))) + ); + } + + #[test] + fn test_optimize_parser_merges_ranges_for_multiple_columns() { + let index_info = int64_index_info("BTree", false); + + let leaves = + optimize_parsed_scalar_filter("x > 10 AND x < 20 AND y > 30 AND y < 40", &index_info); + + assert_eq!(leaves.len(), 2); + assert!(leaves.iter().any(|leaf| { + matches!( + leaf, + ScalarIndexExpr::Query(search) + if search.column == "x" + && matches!( + search.sargable_query(), + Some(SargableQuery::Range( + Bound::Excluded(ScalarValue::Int64(Some(10))), + Bound::Excluded(ScalarValue::Int64(Some(20))), + )) + ) + ) + })); + assert!(leaves.iter().any(|leaf| { + matches!( + leaf, + ScalarIndexExpr::Query(search) + if search.column == "y" + && matches!( + search.sargable_query(), + Some(SargableQuery::Range( + Bound::Excluded(ScalarValue::Int64(Some(30))), + Bound::Excluded(ScalarValue::Int64(Some(40))), + )) + ) + ) + })); + } + + #[test] + fn test_optimize_parser_preserves_standalone_null_checks() { + let index_info = int64_index_info("BTree", false); + + let is_not_null = optimize_parsed_scalar_filter("x IS NOT NULL", &index_info); + assert_eq!(is_not_null.len(), 1); + assert!(matches!(&is_not_null[0], ScalarIndexExpr::Not(_))); + + let is_null = optimize_parsed_scalar_filter("x IS NULL", &index_info); + assert_eq!(is_null.len(), 1); + assert!(matches!( + &is_null[0], + ScalarIndexExpr::Query(search) + if matches!(search.sargable_query(), Some(SargableQuery::IsNull())) + )); + } + + #[test] + fn test_optimize_parser_does_not_remove_is_not_null_for_different_column() { + let index_info = int64_index_info("BTree", false); + + let leaves = optimize_parsed_scalar_filter("x IS NOT NULL AND y >= 10", &index_info); + + assert_eq!(leaves.len(), 2); + assert!(leaves.iter().any(|leaf| { + matches!( + leaf, + ScalarIndexExpr::Not(inner) + if matches!(inner.as_ref(), ScalarIndexExpr::Query(search) + if search.column == "x" + && matches!(search.sargable_query(), Some(SargableQuery::IsNull()))) + ) + })); + } + + #[test] + fn test_optimize_parser_handles_in_list_null_semantics() { + let index_info = int64_index_info("BTree", false); + + let leaves = optimize_parsed_scalar_filter("x IS NOT NULL AND x IN (1, 2)", &index_info); + assert_eq!(leaves.len(), 1); + assert!(matches!( + &leaves[0], + ScalarIndexExpr::Query(search) + if matches!(search.sargable_query(), Some(SargableQuery::IsIn(values)) if values.len() == 2) + )); + + let indexed = parse_int64_filter("x IS NOT NULL AND x IN (1, NULL)", &index_info); + assert!(indexed.refine_expr.is_some()); + let leaves = collect_test_and_leaves(indexed.scalar_query.unwrap().optimize()); + assert_eq!(leaves.len(), 1); + assert!(matches!(&leaves[0], ScalarIndexExpr::Not(_))); + } + + #[test] + fn test_optimize_parser_removes_is_not_null_from_not_equal() { + let index_info = int64_index_info("BTree", false); + + let leaves = optimize_parsed_scalar_filter("x IS NOT NULL AND x != 5", &index_info); + + assert_eq!(leaves.len(), 1); + assert!(matches!( + &leaves[0], + ScalarIndexExpr::Not(inner) + if matches!(inner.as_ref(), ScalarIndexExpr::Query(search) + if matches!(search.sargable_query(), Some(SargableQuery::Equals(value)) + if *value == ScalarValue::Int64(Some(5)))) + )); + } + + #[test] + fn test_optimize_parser_merges_recheck_ranges() { + let index_info = int64_index_info("ZoneMap", true); + + let leaves = optimize_parsed_scalar_filter("x >= 10 AND y = 1 AND x <= 20", &index_info); + + assert_eq!(leaves.len(), 2); + assert!(leaves.iter().any(|leaf| { + matches!( + leaf, + ScalarIndexExpr::Query(search) + if search.column == "x" + && search.needs_recheck + && matches!( + search.sargable_query(), + Some(SargableQuery::Range( + Bound::Included(ScalarValue::Int64(Some(10))), + Bound::Included(ScalarValue::Int64(Some(20))), + )) + ) + ) + })); + } + + #[test] + fn test_optimize_parser_keeps_recheck_is_not_null_as_refine() { + let index_info = int64_index_info("ZoneMap", true); + + let indexed = parse_int64_filter("x IS NOT NULL AND x >= 10", &index_info); + + assert!(indexed.scalar_query.is_some()); + assert!(matches!( + indexed.refine_expr.as_ref(), + Some(Expr::IsNotNull(expr)) + if matches!(expr.as_ref(), Expr::Column(column) if column.name == "x") + )); + let leaves = collect_test_and_leaves(indexed.scalar_query.unwrap().optimize()); + assert_eq!(leaves.len(), 1); + assert!(matches!( + &leaves[0], + ScalarIndexExpr::Query(search) + if search.needs_recheck + && matches!(search.sargable_query(), Some(SargableQuery::Range(_, _))) + )); + } + + #[test] + fn test_optimize_preserves_balanced_depth_for_unmerged_terms() { + let term_count = 2048; + let terms = (0..term_count) + .map(|value| { + test_scalar_query( + "x", + "idx_x", + SargableQuery::Equals(ScalarValue::Int64(Some(value))), + ) + }) + .collect::>(); + + let optimized = test_and_terms(terms).optimize(); + + assert_eq!( + collect_test_and_leaves(optimized.clone()).len(), + term_count as usize + ); + assert!( + test_and_depth(&optimized) <= balanced_depth_bound(term_count as usize), + "optimized AND depth should stay balanced, got {} for {} terms", + test_and_depth(&optimized), + term_count + ); + } + + #[test] + fn test_optimize_merges_ranges_inside_not() { + let lower = test_scalar_range( + "x", + "idx_x", + Bound::Included(ScalarValue::Int64(Some(10))), + Bound::Unbounded, + ); + let upper = test_scalar_range( + "x", + "idx_x", + Bound::Unbounded, + Bound::Included(ScalarValue::Int64(Some(20))), + ); + let sibling = test_scalar_query( + "y", + "idx_y", + SargableQuery::Equals(ScalarValue::Int64(Some(1))), + ); + + let nested_not = ScalarIndexExpr::Not(Box::new(test_and_terms(vec![lower, upper]))); + let optimized = ScalarIndexExpr::And(Box::new(nested_not), Box::new(sibling)).optimize(); + + let ScalarIndexExpr::And(nested_not, _) = optimized else { + panic!("expected outer AND expression"); + }; + let ScalarIndexExpr::Not(inner) = *nested_not else { + panic!("expected NOT expression"); + }; + let leaves = collect_test_and_leaves(*inner); + + assert_eq!(leaves.len(), 1); + assert!(matches!( + &leaves[0], + ScalarIndexExpr::Query(search) + if matches!( + search.sargable_query(), + Some(SargableQuery::Range( + Bound::Included(ScalarValue::Int64(Some(10))), + Bound::Included(ScalarValue::Int64(Some(20))), + )) + ) + )); + } + + #[test] + fn test_optimize_does_not_remove_is_not_null_inside_not() { + let is_not_null = ScalarIndexExpr::Not(Box::new(test_scalar_query( + "x", + "idx_x", + SargableQuery::IsNull(), + ))); + let range = test_scalar_range( + "x", + "idx_x", + Bound::Included(ScalarValue::Int64(Some(10))), + Bound::Unbounded, + ); + let guarded_range = test_and_terms(vec![is_not_null, range]); + let alternative = test_scalar_query( + "y", + "idx_y", + SargableQuery::Equals(ScalarValue::Int64(Some(1))), + ); + let or = ScalarIndexExpr::Or(Box::new(guarded_range), Box::new(alternative)); + let sibling = test_scalar_query( + "z", + "idx_z", + SargableQuery::Equals(ScalarValue::Int64(Some(2))), + ); + let outer_sibling = test_scalar_query( + "w", + "idx_w", + SargableQuery::Equals(ScalarValue::Int64(Some(3))), + ); + + let nested_not = ScalarIndexExpr::Not(Box::new(test_and_terms(vec![or, sibling]))); + let optimized = + ScalarIndexExpr::And(Box::new(nested_not), Box::new(outer_sibling)).optimize(); + + let ScalarIndexExpr::And(nested_not, _) = optimized else { + panic!("expected outer AND expression"); + }; + let ScalarIndexExpr::Not(inner) = *nested_not else { + panic!("expected NOT expression"); + }; + let ScalarIndexExpr::And(or, _) = *inner else { + panic!("expected AND expression"); + }; + let ScalarIndexExpr::Or(lhs, _) = *or else { + panic!("expected OR expression"); + }; + let leaves = collect_test_and_leaves(*lhs); + + assert_eq!(leaves.len(), 2); + assert!(leaves.iter().any(|leaf| { + matches!( + leaf, + ScalarIndexExpr::Not(inner) + if matches!(inner.as_ref(), ScalarIndexExpr::Query(search) + if matches!(search.sargable_query(), Some(SargableQuery::IsNull()))) + ) + })); + } + #[test] fn test_optimize_single_query_passthrough() { use super::{ScalarIndexExpr, ScalarIndexSearch}; @@ -4268,7 +5299,7 @@ mod tests { let optimized = expr.optimize(); let mut leaves = Vec::new(); - optimized.collect_and_leaves(&mut leaves); + optimized.collect_and_leaves(&mut leaves, true); assert_eq!(leaves.len(), 1, "All 4 ranges should merge into 1"); if let ScalarIndexExpr::Query(s) = &leaves[0] { @@ -4305,7 +5336,7 @@ mod tests { let optimized = expr.optimize(); let mut leaves = Vec::new(); - optimized.collect_and_leaves(&mut leaves); + optimized.collect_and_leaves(&mut leaves, true); assert_eq!( leaves.len(), 2, @@ -4353,7 +5384,7 @@ mod tests { if let ScalarIndexExpr::Or(lhs, rhs) = &optimized { // Each branch should be a single merged range let mut left_leaves = Vec::new(); - lhs.clone().collect_and_leaves(&mut left_leaves); + lhs.clone().collect_and_leaves(&mut left_leaves, true); assert_eq!( left_leaves.len(), 1, @@ -4371,7 +5402,7 @@ mod tests { } let mut right_leaves = Vec::new(); - rhs.clone().collect_and_leaves(&mut right_leaves); + rhs.clone().collect_and_leaves(&mut right_leaves, true); assert_eq!( right_leaves.len(), 1, @@ -4429,7 +5460,7 @@ mod tests { let optimized = expr.optimize(); let mut leaves = Vec::new(); - optimized.collect_and_leaves(&mut leaves); + optimized.collect_and_leaves(&mut leaves, true); assert_eq!( leaves.len(), 2, @@ -4502,7 +5533,7 @@ mod tests { let optimized = expr.optimize(); let mut leaves = Vec::new(); - optimized.collect_and_leaves(&mut leaves); + optimized.collect_and_leaves(&mut leaves, true); assert_eq!(leaves.len(), 3, "fqdn + merged_time + channel = 3 leaves"); let time_leaf = leaves @@ -4541,7 +5572,7 @@ mod tests { let optimized = expr.optimize(); let mut leaves = Vec::new(); - optimized.collect_and_leaves(&mut leaves); + optimized.collect_and_leaves(&mut leaves, true); assert_eq!(leaves.len(), 1); if let ScalarIndexExpr::Query(s) = &leaves[0] { let range = s.query.as_any().downcast_ref::().unwrap(); @@ -4575,7 +5606,7 @@ mod tests { let optimized = expr.optimize(); let mut leaves = Vec::new(); - optimized.collect_and_leaves(&mut leaves); + optimized.collect_and_leaves(&mut leaves, true); assert_eq!(leaves.len(), 1); if let ScalarIndexExpr::Query(s) = &leaves[0] { let range = s.query.as_any().downcast_ref::().unwrap(); @@ -4610,7 +5641,7 @@ mod tests { let optimized = expr.optimize(); let mut leaves = Vec::new(); - optimized.collect_and_leaves(&mut leaves); + optimized.collect_and_leaves(&mut leaves, true); assert_eq!(leaves.len(), 1); if let ScalarIndexExpr::Query(s) = &leaves[0] { let range = s.query.as_any().downcast_ref::().unwrap(); diff --git a/rust/lance-index/src/scalar/fmindex.rs b/rust/lance-index/src/scalar/fmindex.rs index 7b87e492ec2..e0677043ed1 100644 --- a/rust/lance-index/src/scalar/fmindex.rs +++ b/rust/lance-index/src/scalar/fmindex.rs @@ -178,7 +178,6 @@ struct RankBitVec { len: usize, } -#[allow(dead_code)] impl RankBitVec { fn new(len: usize) -> Self { Self { @@ -193,11 +192,6 @@ impl RankBitVec { self.words[pos / 64] |= 1u64 << (pos % 64); } - #[inline] - fn get(&self, pos: usize) -> bool { - (self.words[pos / 64] >> (pos % 64)) & 1 != 0 - } - fn build_rank_index(&mut self) { let num_sb = self.words.len().div_ceil(WORDS_PER_SUPERBLOCK) + 1; self.superblocks = Vec::with_capacity(num_sb); @@ -211,29 +205,6 @@ impl RankBitVec { self.superblocks.push(cum); } - #[inline] - fn rank1(&self, pos: usize) -> usize { - if pos == 0 { - return 0; - } - let word_idx = pos / 64; - let bit_idx = pos % 64; - let sb_idx = word_idx / WORDS_PER_SUPERBLOCK; - let mut count = self.superblocks[sb_idx] as usize; - for i in (sb_idx * WORDS_PER_SUPERBLOCK)..word_idx { - count += self.words[i].count_ones() as usize; - } - if bit_idx > 0 { - count += (self.words[word_idx] & ((1u64 << bit_idx) - 1)).count_ones() as usize; - } - count - } - - #[inline] - fn rank0(&self, pos: usize) -> usize { - pos - self.rank1(pos) - } - fn deep_size(&self) -> usize { self.words.len() * 8 + self.superblocks.len() * 4 } @@ -285,7 +256,6 @@ impl Ord for HuffNode { } } -#[allow(dead_code)] impl HuffmanWaveletTree { fn build(data: &[u8]) -> Self { let n = data.len(); @@ -403,73 +373,6 @@ impl HuffmanWaveletTree { } } - /// Retrieve the byte at position `pos` in the original BWT. - #[inline] - fn access(&self, mut pos: usize) -> u8 { - if self.nodes.is_empty() { - return 0; - } - let mut node_idx = 0; - loop { - let bit = self.nodes[node_idx].get(pos); - let (ref left, ref right) = self.children[node_idx]; - if bit { - pos = self.nodes[node_idx].rank1(pos); - match right { - WaveletChild::Leaf(b) => return *b, - WaveletChild::Node(next) => node_idx = *next, - } - } else { - pos = self.nodes[node_idx].rank0(pos); - match left { - WaveletChild::Leaf(b) => return *b, - WaveletChild::Node(next) => node_idx = *next, - } - } - } - } - - /// Count occurrences of byte `c` in positions `[0, pos)`. - #[inline] - fn rank(&self, c: u8, pos: usize) -> usize { - let code = &self.codes[c as usize]; - if code.length == 0 { - return 0; - } - let (mut lo, mut hi) = (0, pos); - for (level, &nid) in code.node_path.iter().enumerate() { - if (code.bits >> (code.length - 1 - level as u8)) & 1 == 0 { - lo = self.nodes[nid].rank0(lo); - hi = self.nodes[nid].rank0(hi); - } else { - lo = self.nodes[nid].rank1(lo); - hi = self.nodes[nid].rank1(hi); - } - } - hi - lo - } - - #[inline] - fn rank_pair(&self, c: u8, lo: usize, hi: usize) -> (usize, usize) { - let code = &self.codes[c as usize]; - if code.length == 0 { - return (0, 0); - } - let (mut s, mut l, mut h) = (0, lo, hi); - for (level, &nid) in code.node_path.iter().enumerate() { - if (code.bits >> (code.length - 1 - level as u8)) & 1 == 0 { - s = self.nodes[nid].rank0(s); - l = self.nodes[nid].rank0(l); - h = self.nodes[nid].rank0(h); - } else { - s = self.nodes[nid].rank1(s); - l = self.nodes[nid].rank1(l); - h = self.nodes[nid].rank1(h); - } - } - (l - s, h - s) - } - fn deep_size(&self) -> usize { self.nodes.iter().map(|n| n.deep_size()).sum::() + self @@ -1008,7 +911,6 @@ impl DeepSizeOf for FMIndex { } } -#[allow(dead_code)] impl FMIndex { fn build(texts: &[(u64, &[u8])]) -> Result { if texts.is_empty() { @@ -1080,89 +982,6 @@ impl FMIndex { }) } - /// Locate: resolve SA[pos] by walking LF-mapping until hitting a sampled position. - /// For large data (N >> SA_SAMPLE_RATE), converges within SA_SAMPLE_RATE steps. - /// For small data with short LF cycles, may need up to N steps. - #[inline] - fn locate(&self, mut pos: usize) -> usize { - let mut steps = 0; - let n = self.wavelet.len; - loop { - if pos.is_multiple_of(SA_SAMPLE_RATE) && (pos / SA_SAMPLE_RATE) < self.sa_samples.len() - { - return (self.sa_samples[pos / SA_SAMPLE_RATE] as usize + steps) % n; - } - let c = self.wavelet.access(pos); - pos = self.c_table[c as usize] + self.wavelet.rank(c, pos); - steps += 1; - if steps >= n { - log::warn!("FM-Index SA locate exceeded {n} steps, possible index corruption"); - return 0; - } - } - } - - /// Map a text position to document index via binary search on doc_start_positions. - #[inline] - fn doc_for_position(&self, text_pos: usize) -> usize { - let tp = text_pos as u64; - match self.doc_start_positions.binary_search(&tp) { - Ok(idx) => idx, - Err(idx) => idx - 1, - } - } - - fn backward_search(&self, pattern: &[u8]) -> (usize, usize) { - if pattern.is_empty() || self.wavelet.len == 0 { - return (0, 0); - } - let (mut lo, mut hi) = (0, self.wavelet.len); - for &b in pattern.iter().rev() { - let c = self.c_table[b as usize]; - let (occ_lo, occ_hi) = self.wavelet.rank_pair(b, lo, hi); - lo = c + occ_lo; - hi = c + occ_hi; - if lo >= hi { - return (0, 0); - } - } - (lo, hi) - } - - #[cfg(test)] - fn search(&self, pattern: &[u8]) -> RoaringBitmap { - let (lo, hi) = self.backward_search(pattern); - if lo >= hi { - return RoaringBitmap::new(); - } - let mut result = RoaringBitmap::new(); - for i in lo..hi { - let text_pos = self.locate(i); - let doc_idx = self.doc_for_position(text_pos); - result.insert(self.row_ids[doc_idx] as u32); - } - result - } - - /// Search returning full u64 row addresses (preserving fragment ID in upper bits). - fn search_row_addrs(&self, pattern: &[u8]) -> Vec { - let (lo, hi) = self.backward_search(pattern); - if lo >= hi { - return Vec::new(); - } - let mut seen = std::collections::HashSet::new(); - let mut result = Vec::new(); - for i in lo..hi { - let text_pos = self.locate(i); - let doc_idx = self.doc_for_position(text_pos); - let row_addr = self.row_ids[doc_idx]; - if seen.insert(row_addr) { - result.push(row_addr); - } - } - result - } - fn serialize_huffman_codes(&self) -> Vec { let mut buf = Vec::new(); for code in &self.wavelet.codes { @@ -1701,7 +1520,7 @@ impl FMIndexScalarIndex { } pfiles.sort_by_key(|(id, _)| *id); let io_parallelism = store.io_parallelism().max(1); - let mut parts = futures::stream::iter(pfiles.into_iter()) + let mut parts = futures::stream::iter(pfiles) .map(|(id, name)| { let store = Arc::clone(&store); async move { @@ -2492,6 +2311,8 @@ impl ScalarIndexPlugin for FMIndexPlugin { false, // supports_regex: regex acceleration is only implemented for ngram. false, + // min_contains_chars: the FM-index can match a needle of any length. + 0, ))) } async fn load_index( @@ -2616,107 +2437,6 @@ mod tests { .sum() } - #[test] - fn test_fmindex_build_and_search() { - let texts: Vec<(u64, &[u8])> = vec![ - (0, b"hello world"), - (1, b"hello rust"), - (2, b"goodbye world"), - ]; - let fm = FMIndex::build(&texts).unwrap(); - - let r = fm.search(b"hello"); - assert!(r.contains(0)); - assert!(r.contains(1)); - assert!(!r.contains(2)); - - let r = fm.search(b"world"); - assert!(r.contains(0)); - assert!(!r.contains(1)); - assert!(r.contains(2)); - - let r = fm.search(b"goodbye"); - assert!(!r.contains(0)); - assert!(!r.contains(1)); - assert!(r.contains(2)); - - assert!(fm.search(b"xyz").is_empty()); - } - - #[test] - fn test_fmindex_empty() { - let fm = FMIndex::build(&[]).unwrap(); - assert!(fm.search(b"anything").is_empty()); - } - - #[test] - fn test_fmindex_single_char_search() { - let texts: Vec<(u64, &[u8])> = vec![(0, b"abc"), (1, b"def")]; - let fm = FMIndex::build(&texts).unwrap(); - assert!(fm.search(b"a").contains(0)); - assert!(!fm.search(b"a").contains(1)); - assert!(!fm.search(b"d").contains(0)); - assert!(fm.search(b"d").contains(1)); - } - - #[test] - fn test_fmindex_repeated_pattern() { - let texts: Vec<(u64, &[u8])> = vec![(0, b"ababab"), (1, b"cdcd")]; - let fm = FMIndex::build(&texts).unwrap(); - assert!(fm.search(b"ab").contains(0)); - assert!(!fm.search(b"ab").contains(1)); - assert!(!fm.search(b"cd").contains(0)); - assert!(fm.search(b"cd").contains(1)); - } - - #[test] - fn test_early_exit_all_docs_match() { - let texts: Vec<(u64, &[u8])> = vec![(0, b"the cat"), (1, b"the dog"), (2, b"the bird")]; - let fm = FMIndex::build(&texts).unwrap(); - assert_eq!(fm.search(b"the").len(), 3); - } - - #[test] - fn test_locate_correctness() { - let texts: Vec<(u64, &[u8])> = vec![ - (0, b"the quick brown fox jumps over the lazy dog"), - (1, b"pack my box with five dozen liquor jugs"), - (2, b"how vexingly quick daft zebras jump"), - ]; - let fm = FMIndex::build(&texts).unwrap(); - - let r = fm.search(b"quick"); - assert!(r.contains(0)); - assert!(!r.contains(1)); - assert!(r.contains(2)); - - let r = fm.search(b"the"); - assert!(r.contains(0)); - assert!(!r.contains(1)); - assert!(!r.contains(2)); - - let r = fm.search(b"jump"); - assert!(r.contains(0)); - assert!(r.contains(2)); - } - - #[test] - fn test_many_documents() { - let docs: Vec> = (0..100) - .map(|i| format!("document number {} with hello world data xyz", i).into_bytes()) - .collect(); - let texts: Vec<(u64, &[u8])> = docs - .iter() - .enumerate() - .map(|(i, d)| (i as u64, d.as_slice())) - .collect(); - let fm = FMIndex::build(&texts).unwrap(); - - assert_eq!(fm.search(b"hello world").len(), 100); - assert_eq!(fm.search(b"document number 42").len(), 1); - assert_eq!(fm.search(b"nonexistent").len(), 0); - } - #[test] fn test_index_size_ratio() { let docs: Vec> = (0..200) @@ -2747,42 +2467,6 @@ mod tests { ); } - #[test] - fn test_wavelet_access_consistency() { - let docs: Vec> = (0..50) - .map(|i| format!("document {i} hello world test").into_bytes()) - .collect(); - let texts: Vec<(u64, &[u8])> = docs - .iter() - .enumerate() - .map(|(i, d)| (i as u64, d.as_slice())) - .collect(); - - let mut concat = Vec::new(); - for (_, text) in &texts { - concat.extend_from_slice(text); - concat.push(SENTINEL_BYTE); - } - concat.push(0x00); - let sa = build_suffix_array(&concat); - let n = concat.len(); - let bwt: Vec = sa - .iter() - .map(|&pos| { - if pos == 0 { - concat[n - 1] - } else { - concat[pos - 1] - } - }) - .collect(); - let wavelet = HuffmanWaveletTree::build(&bwt); - - for (i, &expected) in bwt.iter().enumerate().take(n.min(500)) { - assert_eq!(wavelet.access(i), expected, "access mismatch at {i}"); - } - } - #[test] fn test_serialization_roundtrip() { let texts: Vec<(u64, &[u8])> = vec![ @@ -2827,43 +2511,6 @@ mod tests { assert!(!fmindex_partition_limit_reached(3, 99, 10, 100)); } - #[test] - fn test_sentinel_sanitization() { - // Text containing \xFF should be sanitized to space during training. - let texts: Vec<(u64, &[u8])> = vec![(0, b"hello\xFFworld")]; - let fm = FMIndex::build(&texts).unwrap(); - // Build itself does not sanitize, but search should still work. - let r = fm.search(b"hello"); - assert!(r.contains(0)); - } - - #[test] - fn test_wavelet_rank_pair_consistency() { - let docs: Vec> = (0..30) - .map(|i| format!("doc {i} with repeated words hello world test data").into_bytes()) - .collect(); - let texts: Vec<(u64, &[u8])> = docs - .iter() - .enumerate() - .map(|(i, d)| (i as u64, d.as_slice())) - .collect(); - let fm = FMIndex::build(&texts).unwrap(); - - let n = fm.wavelet.len; - for b in [b'a', b'e', b' ', SENTINEL_BYTE] { - for &(lo, hi) in &[(0usize, 1usize), (0, n), (n / 4, n / 2)] { - if lo >= n || hi > n || lo >= hi { - continue; - } - let (pl, ph) = fm.wavelet.rank_pair(b, lo, hi); - let rl = fm.wavelet.rank(b, lo); - let rh = fm.wavelet.rank(b, hi); - assert_eq!(pl, rl, "rank_pair lo mismatch for b={b} [{lo},{hi})"); - assert_eq!(ph, rh, "rank_pair hi mismatch for b={b} [{lo},{hi})"); - } - } - } - #[test] fn test_large_sa_sampling() { // Test with enough documents to have multiple SA sample points @@ -2884,9 +2531,6 @@ mod tests { let fm = FMIndex::build(&texts).unwrap(); assert!(fm.sa_samples.len() > 1, "should have multiple SA samples"); - assert_eq!(fm.search(b"document number 25").len(), 1); - assert_eq!(fm.search(b"document number").len(), 50); - assert_eq!(fm.search(b"nonexistent pattern").len(), 0); } #[tokio::test(flavor = "multi_thread")] diff --git a/rust/lance-index/src/scalar/inverted.rs b/rust/lance-index/src/scalar/inverted.rs index 6c9a5ad2947..b07ee2b5e73 100644 --- a/rust/lance-index/src/scalar/inverted.rs +++ b/rust/lance-index/src/scalar/inverted.rs @@ -3,11 +3,13 @@ pub mod builder; mod cache_codec; +mod compound; +mod documents; mod encoding; +mod impact; mod index; mod iter; pub mod json; -mod lazy_docset; pub mod parser; pub mod query; mod scorer; @@ -20,6 +22,7 @@ use std::sync::Arc; use arrow_schema::{DataType, Field}; use async_trait::async_trait; pub use builder::InvertedIndexBuilder; +pub use compound::{compound_search, compound_search_with_base_scorer}; use datafusion::execution::SendableRecordBatchStream; pub use index::*; use lance_core::{Result, cache::LanceCache}; @@ -73,6 +76,11 @@ fn scorer_terms( /// statistics rather than per-segment statistics. Computes the union of /// fuzzy-expanded terms when `params.fuzziness` is set. /// +/// `metrics`, when provided, is forwarded to the per-token metadata cache +/// boundary on each segment so callers running under an `ExecutionPlan` +/// (e.g. `MatchQueryExec`) see the reads triggered here in their per-query +/// `index_cache_hits`/`index_cache_misses` counters. +/// /// Public as the canonical producer paired with the `with_base_scorer` /// consumer on FTS exec types: callers holding `Arc` segment /// handles locally can construct an injectable scorer without reimplementing @@ -83,24 +91,25 @@ pub async fn build_global_bm25_scorer( indices: &[Arc], query_tokens: &Tokens, params: &FtsSearchParams, + metrics: Option<&dyn crate::scalar::MetricsCollector>, ) -> Result { let terms = scorer_terms(indices, query_tokens, params)?; let first_index = indices.first().ok_or_else(|| { lance_core::Error::invalid_input("FTS index requires at least one segment") })?; let (mut total_tokens, mut num_docs, first_token_docs) = - first_index.bm25_stats_for_terms(&terms).await?; + first_index.bm25_stats_for_terms(&terms, metrics).await?; let mut token_docs = HashMap::with_capacity(terms.len()); - for (term, count) in terms.iter().cloned().zip(first_token_docs.into_iter()) { + for (term, count) in terms.iter().cloned().zip(first_token_docs) { token_docs.insert(term, count); } for index in indices.iter().skip(1) { let (segment_total_tokens, segment_num_docs, segment_token_docs) = - index.bm25_stats_for_terms(&terms).await?; + index.bm25_stats_for_terms(&terms, metrics).await?; total_tokens += segment_total_tokens; num_docs += segment_num_docs; - for (term, count) in terms.iter().zip(segment_token_docs.into_iter()) { + for (term, count) in terms.iter().zip(segment_token_docs) { *token_docs .get_mut(term) .expect("global scorer terms should already be initialized") += count; @@ -118,7 +127,8 @@ use crate::scalar::{ CreatedIndex, RowIdRemapper, ScalarIndex, expression::{FtsQueryParser, ScalarQueryParser}, registry::{ - BasicTrainer, ScalarIndexPlugin, TrainingCriteria, TrainingOrdering, TrainingRequest, + BasicTrainer, ScalarIndexCacheKey, ScalarIndexLoad, ScalarIndexPlugin, TrainingCriteria, + TrainingOrdering, TrainingRequest, }, }; @@ -146,7 +156,9 @@ impl InvertedIndexPlugin { } }); + params.validate_format_version()?; let format_version = params.resolved_format_version(); + let is_element_document = params.get_document_granularity().is_list_element(); let details = pbold::InvertedIndexDetails::try_from(¶ms)?; let mut inverted_index = InvertedIndexBuilder::new_with_fragment_mask(params, fragment_mask) @@ -154,7 +166,11 @@ impl InvertedIndexPlugin { let files = inverted_index.update(data, index_store, None).await?; Ok(CreatedIndex { index_details: prost_types::Any::from_msg(&details).unwrap(), - index_version: format_version.index_version(), + index_version: if is_element_document { + INVERTED_INDEX_VERSION_V3 + } else { + format_version.index_version() + }, files, }) } @@ -211,7 +227,7 @@ impl BasicTrainer for InvertedIndexPlugin { .into())) } - let params = serde_json::from_str::(params)?; + let params = InvertedIndexParams::from_training_json(params)?; Ok(Box::new(InvertedIndexTrainingRequest::new(params))) } @@ -266,7 +282,7 @@ impl ScalarIndexPlugin for InvertedIndexPlugin { } fn version(&self) -> u32 { - max_supported_fts_format_version().index_version() + INVERTED_INDEX_VERSION_V3 } fn new_query_parser( @@ -295,33 +311,95 @@ impl ScalarIndexPlugin for InvertedIndexPlugin { async fn load_index( &self, index_store: Arc, - _index_details: &prost_types::Any, + index_details: &prost_types::Any, frag_reuse_index: Option>, cache: &LanceCache, ) -> Result> { - Ok( - InvertedIndex::load(index_store, frag_reuse_index, cache).await? - as Arc, - ) + let index = InvertedIndex::load(index_store, frag_reuse_index, cache).await?; + let details = index_details.to_msg::()?; + let expected_granularity = DocumentGranularity::try_from(details.document_granularity)?; + let physical_granularity = index.params().get_document_granularity(); + if physical_granularity != expected_granularity { + return Err(Error::index(format!( + "FTS document granularity in index details is {expected_granularity:?}, but the physical document schema implies {physical_granularity:?}" + ))); + } + Ok(index as Arc) + } + + async fn get_or_insert_in_cache( + &self, + _index_store: Arc, + _frag_reuse_index: Option>, + cache: &LanceCache, + load: ScalarIndexLoad<'_>, + ) -> Result> { + cache + .get_or_insert_unsized_with_key(ScalarIndexCacheKey, || load) + .await } fn details_as_json(&self, details: &prost_types::Any) -> Result { let index_details = details.to_msg::()?; let index_params = InvertedIndexParams::try_from(&index_details)?; - Ok(serde_json::json!(&index_params)) + Ok(index_params.to_details_json()?) } } #[cfg(test)] mod tests { use super::*; + use crate::scalar::{BuiltinIndexType, ScalarIndexParams}; #[test] - fn test_plugin_version_tracks_max_supported_format() { + fn test_plugin_version_tracks_v3_capability_gate() { let plugin = InvertedIndexPlugin; - assert_eq!( - plugin.version(), - max_supported_fts_format_version().index_version() - ); + assert_eq!(plugin.version(), INVERTED_INDEX_VERSION_V3); + } + + #[test] + fn test_details_json_includes_document_granularity() { + let details = pbold::InvertedIndexDetails { + document_granularity: pbold::inverted_index_details::DocumentGranularity::ListElement + as i32, + ..Default::default() + }; + let details = prost_types::Any::from_msg(&details).unwrap(); + + let json = InvertedIndexPlugin.details_as_json(&details).unwrap(); + + assert_eq!(json["document_granularity"], "list_element"); + } + + #[test] + fn test_new_training_request_defaults_missing_block_size_to_128() { + let plugin = InvertedIndexPlugin; + let field = Field::new("text", DataType::Utf8, true); + + let cases = [ + ( + ScalarIndexParams::for_builtin(BuiltinIndexType::Inverted), + false, + ), + (ScalarIndexParams::new("inverted".to_string()), false), + ( + ScalarIndexParams::new("inverted".to_string()) + .with_params(&serde_json::json!({ "with_position": true })), + true, + ), + ]; + + for (params, expected_with_position) in cases { + let request = plugin + .new_training_request(params.params.as_deref().unwrap_or("{}"), &field) + .unwrap(); + let request = request + .as_any() + .downcast_ref::() + .unwrap(); + + assert_eq!(request.parameters.posting_block_size(), DEFAULT_BLOCK_SIZE); + assert_eq!(request.parameters.has_positions(), expected_with_position); + } } } diff --git a/rust/lance-index/src/scalar/inverted/builder.rs b/rust/lance-index/src/scalar/inverted/builder.rs index ede00ad43ee..f50bd827e20 100644 --- a/rust/lance-index/src/scalar/inverted/builder.rs +++ b/rust/lance-index/src/scalar/inverted/builder.rs @@ -1,10 +1,10 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright The Lance Authors -use super::encoding::encode_group_starts; use super::{InvertedIndexParams, index::*}; use crate::scalar::inverted::document_tokenizer::DocType; use crate::scalar::inverted::json::JsonTextStream; +use crate::scalar::inverted::tokenizer::LEGACY_BLOCK_SIZE; use crate::scalar::inverted::tokenizer::document_tokenizer::LanceTokenizer; #[cfg(test)] use crate::scalar::lance_format::LanceIndexStore; @@ -13,12 +13,11 @@ use crate::vector::graph::OrderedFloat; use crate::{progress::IndexBuildProgress, progress::noop_progress}; use arrow::array::AsArray; use arrow::datatypes; -use arrow_array::{Array, BinaryArray, RecordBatch, UInt64Array}; +use arrow_array::{Array, BinaryArray, RecordBatch}; use arrow_schema::{DataType, Field, Schema, SchemaRef}; -use bytes::Bytes; -use datafusion::execution::{RecordBatchStream, SendableRecordBatchStream}; +use datafusion::execution::SendableRecordBatchStream; use fst::Streamer; -use futures::{Stream, StreamExt, TryStreamExt}; +use futures::{StreamExt, TryStreamExt}; use lance_arrow::json::JSON_EXT_NAME; use lance_arrow::{ARROW_EXT_NAME_KEY, iter_str_array}; use lance_bitpacking::{BitPacker, BitPacker4x}; @@ -27,24 +26,22 @@ use lance_core::deepsize::DeepSizeOf; use lance_core::error::LanceOptionExt; use lance_core::utils::row_addr_remap::RowAddrRemap; use lance_core::utils::tokio::{IO_CORE_RESERVATION, get_num_compute_intensive_cpus, spawn_cpu}; -use lance_core::{Error, ROW_ID, ROW_ID_FIELD, Result}; +use lance_core::{Error, ROW_ID, Result}; use lance_io::object_store::ObjectStore; use lance_select::RowSetOps; use object_store::path::Path; use roaring::RoaringBitmap; use smallvec::SmallVec; use std::collections::HashMap; -use std::pin::Pin; use std::str::FromStr; use std::sync::Arc; use std::sync::LazyLock; -use std::task::{Context, Poll}; use std::{fmt::Debug, sync::atomic::AtomicU64}; use tracing::instrument; -// the number of elements in each block -// each block contains 128 row ids and 128 frequencies -// WARNING: changing this value will break the compatibility with existing indexes +// The legacy bitpacking block size. Position streams still use this block size; +// FTS posting blocks choose their physical bitpacker from the configured +// InvertedIndexParams::block_size. pub const BLOCK_SIZE: usize = BitPacker4x::BLOCK_LEN; // The default number of workers to use for FTS builds. @@ -75,92 +72,8 @@ static LANCE_FTS_POSTING_BATCH_ROWS: LazyLock = LazyLock::new(|| { .parse() .expect("failed to parse LANCE_FTS_POSTING_BATCH_ROWS") }); -// Target serialized byte size of a posting-list cache group. Consecutive -// posting lists are grouped into a single cache entry until their combined -// serialized size reaches this target, amortizing per-entry overhead across -// small (Zipfian-rare) terms. See issue #7040. -static LANCE_FTS_POSTING_GROUP_TARGET_BYTES: LazyLock = LazyLock::new(|| { - std::env::var("LANCE_FTS_POSTING_GROUP_TARGET_BYTES") - .unwrap_or_else(|_| "4096".to_string()) - .parse() - .expect("failed to parse LANCE_FTS_POSTING_GROUP_TARGET_BYTES") -}); -// Maximum number of posting lists in a single cache group, regardless of byte -// size. Caps the work and memory of a single group read for corpora with many -// tiny terms. -static LANCE_FTS_POSTING_GROUP_MAX_TOKENS: LazyLock = LazyLock::new(|| { - std::env::var("LANCE_FTS_POSTING_GROUP_MAX_TOKENS") - .unwrap_or_else(|_| "256".to_string()) - .parse() - .expect("failed to parse LANCE_FTS_POSTING_GROUP_MAX_TOKENS") -}); const MAX_RETAINED_TOKEN_IDS: usize = 8 * 1024; -/// Write-time configuration controlling how consecutive posting lists are -/// grouped into a single read-path cache entry (issue #7040). Defaults come -/// from the `LANCE_FTS_POSTING_GROUP_*` environment variables. -#[derive(Debug, Clone, Copy)] -pub(crate) struct PostingGroupConfig { - pub(crate) target_bytes: usize, - pub(crate) max_tokens: usize, -} - -impl Default for PostingGroupConfig { - fn default() -> Self { - Self { - target_bytes: (*LANCE_FTS_POSTING_GROUP_TARGET_BYTES).max(1), - max_tokens: (*LANCE_FTS_POSTING_GROUP_MAX_TOKENS).max(1), - } - } -} - -/// Accumulates posting-list group boundaries at write time. Tokens are pushed -/// in row order; a group is cut once its serialized bytes reach -/// `target_bytes` or it holds `max_tokens` posting lists. A posting list -/// larger than the target that *starts* a group occupies that group alone (the -/// clamp case); one encountered mid-group is absorbed and closes that group, so -/// a single term is never split across groups. -#[derive(Debug)] -pub(crate) struct PostingGroupAccumulator { - config: PostingGroupConfig, - starts: Vec, - next_token: u32, - current_bytes: usize, - current_tokens: usize, -} - -impl PostingGroupAccumulator { - pub(crate) fn new(config: PostingGroupConfig) -> Self { - Self { - config, - starts: Vec::new(), - next_token: 0, - current_bytes: 0, - current_tokens: 0, - } - } - - /// Record the next posting list in row order, given its serialized byte size. - pub(crate) fn push(&mut self, posting_bytes: usize) { - if self.current_tokens == 0 { - self.starts.push(self.next_token); - } - self.current_bytes += posting_bytes; - self.current_tokens += 1; - self.next_token += 1; - if self.current_bytes >= self.config.target_bytes - || self.current_tokens >= self.config.max_tokens - { - self.current_bytes = 0; - self.current_tokens = 0; - } - } - - pub(crate) fn into_starts(self) -> Vec { - self.starts - } -} - fn default_num_workers() -> usize { let total_cpus = get_num_compute_intensive_cpus() + *IO_CORE_RESERVATION; std::cmp::max(1, total_cpus / 2) @@ -195,9 +108,13 @@ fn merge_all_tail_partitions( ) -> Result> { let mut merged_builders: Vec = Vec::new(); let mut merged: Option = None; + let mut empty_coordinate_builder: Option = None; for tail in tails { let builder = tail.builder; if builder.is_empty() { + if builder.docs.coordinate_rank() > 0 && empty_coordinate_builder.is_none() { + empty_coordinate_builder = Some(builder); + } continue; } match &mut merged { @@ -219,6 +136,11 @@ fn merge_all_tail_partitions( if let Some(builder) = merged { merged_builders.push(builder); } + if merged_builders.is_empty() + && let Some(builder) = empty_coordinate_builder + { + merged_builders.push(builder); + } Ok(merged_builders) } @@ -282,8 +204,11 @@ impl InvertedIndexBuilder { } pub fn with_posting_tail_codec(mut self, posting_tail_codec: PostingTailCodec) -> Self { - self.format_version = - InvertedListFormatVersion::from_posting_tail_codec(posting_tail_codec); + self.format_version = InvertedListFormatVersion::from_posting_tail_codec_and_block_size( + posting_tail_codec, + self.params.block_size, + ) + .expect("invalid posting tail codec for posting block size"); self.posting_tail_codec = posting_tail_codec; self } @@ -310,6 +235,7 @@ impl InvertedIndexBuilder { dest_store: &dyn IndexStore, old_data_filter: Option, ) -> Result> { + validate_format_version_block_size(self.format_version, self.params.block_size)?; let schema = new_data.schema(); let doc_col = schema.field(0).name(); @@ -344,6 +270,7 @@ impl InvertedIndexBuilder { old_segments: &[Arc], old_data_filter: Option, ) -> Result> { + validate_format_version_block_size(self.format_version, self.params.block_size)?; let schema = new_data.schema(); let doc_col = schema.field(0).name(); @@ -469,6 +396,8 @@ impl InvertedIndexBuilder { fragment_mask: self.fragment_mask, token_set_format: self.token_set_format, worker_memory_limit_bytes, + block_size: self.params.block_size, + coordinate_rank: document_coordinate_rank(&stream.schema()), }; let next_id = self.next_partition_id(); let id_alloc = Arc::new(AtomicU64::new(next_id)); @@ -619,6 +548,7 @@ impl InvertedIndexBuilder { dest_store: &dyn IndexStore, partitions: &[u64], ) -> Result { + validate_format_version_block_size(self.format_version, self.params.block_size)?; let mut serialized_deleted_fragments = Vec::with_capacity(self.deleted_fragments.serialized_size()); self.deleted_fragments @@ -635,6 +565,14 @@ impl InvertedIndexBuilder { POSTING_TAIL_CODEC_KEY.to_owned(), self.posting_tail_codec.as_str().to_owned(), ), + ( + FTS_FORMAT_VERSION_KEY.to_owned(), + self.format_version.index_version().to_string(), + ), + ( + POSTING_BLOCK_SIZE_KEY.to_owned(), + self.params.block_size.to_string(), + ), ]); if self.params.with_position && self.format_version.uses_shared_position_stream() { @@ -677,9 +615,19 @@ impl InvertedIndexBuilder { pub(crate) async fn write_part_metadata( &self, dest_store: &dyn IndexStore, - partition: u64, // Modify parameter type + partition: u64, + ) -> Result { + self.write_staged_metadata(dest_store, part_metadata_file_path(partition), &[partition]) + .await + } + + async fn write_staged_metadata( + &self, + dest_store: &dyn IndexStore, + file_name: String, + partitions: &[u64], ) -> Result { - let partitions = vec![partition]; + validate_format_version_block_size(self.format_version, self.params.block_size)?; let mut metadata = HashMap::from_iter(vec![ ("partitions".to_owned(), serde_json::to_string(&partitions)?), ("params".to_owned(), serde_json::to_string(&self.params)?), @@ -691,6 +639,14 @@ impl InvertedIndexBuilder { POSTING_TAIL_CODEC_KEY.to_owned(), self.posting_tail_codec.as_str().to_owned(), ), + ( + FTS_FORMAT_VERSION_KEY.to_owned(), + self.format_version.index_version().to_string(), + ), + ( + POSTING_BLOCK_SIZE_KEY.to_owned(), + self.params.block_size.to_string(), + ), ]); if self.params.with_position && self.format_version.uses_shared_position_stream() { metadata.insert( @@ -706,8 +662,6 @@ impl InvertedIndexBuilder { .to_owned(), ); } - // Use partition ID to generate a unique temporary filename - let file_name = part_metadata_file_path(partition); let mut writer = dest_store .new_index_file(&file_name, Arc::new(Schema::empty())) .await?; @@ -719,26 +673,39 @@ impl InvertedIndexBuilder { dest_store: &dyn IndexStore, partitions: &[u64], ) -> Result> { - let total = if self.fragment_mask.is_none() { - Some(1) - } else { - Some(partitions.len() as u64) - }; + let total = Some(partitions.len().max(1) as u64); let mut files = Vec::new(); self.progress .stage_start("write_metadata", total, "files") .await?; - if self.fragment_mask.is_none() { - files.push(self.write_metadata(dest_store, partitions).await?); - self.progress.stage_progress("write_metadata", 1).await?; - } else { - let mut completed = 0; - for &partition_id in partitions { - files.push(self.write_part_metadata(dest_store, partition_id).await?); - completed += 1; - self.progress - .stage_progress("write_metadata", completed) - .await?; + match self.fragment_mask { + None => { + files.push(self.write_metadata(dest_store, partitions).await?); + self.progress.stage_progress("write_metadata", 1).await?; + } + Some(fragment_mask) if partitions.is_empty() => { + // Root metadata is the finalization marker for the shared index directory. An + // empty shard must publish only staged metadata so sibling partitions are still + // finalized. + files.push( + self.write_staged_metadata( + dest_store, + empty_part_metadata_file_path(fragment_mask), + partitions, + ) + .await?, + ); + self.progress.stage_progress("write_metadata", 1).await?; + } + Some(_) => { + let mut completed = 0; + for &partition_id in partitions { + files.push(self.write_part_metadata(dest_store, partition_id).await?); + completed += 1; + self.progress + .stage_progress("write_metadata", completed) + .await?; + } } } self.progress.stage_complete("write_metadata").await?; @@ -827,10 +794,10 @@ pub struct InnerBuilder { token_set_format: TokenSetFormat, format_version: InvertedListFormatVersion, posting_tail_codec: PostingTailCodec, + block_size: usize, pub(crate) tokens: TokenSet, pub(crate) posting_lists: Vec, pub(crate) docs: DocSet, - pub(crate) group_config: PostingGroupConfig, } impl InnerBuilder { @@ -849,16 +816,51 @@ impl InnerBuilder { token_set_format: TokenSetFormat, format_version: InvertedListFormatVersion, ) -> Self { + Self::new_with_format_version_and_block_size( + id, + with_position, + token_set_format, + format_version, + LEGACY_BLOCK_SIZE, + ) + } + + pub fn new_with_block_size( + id: u64, + with_position: bool, + token_set_format: TokenSetFormat, + block_size: usize, + ) -> Self { + let format_version = default_fts_format_version_for_block_size(block_size) + .expect("invalid posting list block size"); + Self::new_with_format_version_and_block_size( + id, + with_position, + token_set_format, + format_version, + block_size, + ) + } + + pub fn new_with_format_version_and_block_size( + id: u64, + with_position: bool, + token_set_format: TokenSetFormat, + format_version: InvertedListFormatVersion, + block_size: usize, + ) -> Self { + validate_format_version_block_size(format_version, block_size) + .expect("invalid FTS format version for posting block size"); Self { id, with_position, token_set_format, format_version, posting_tail_codec: format_version.posting_tail_codec(), + block_size, tokens: TokenSet::default(), posting_lists: Vec::new(), docs: DocSet::default(), - group_config: PostingGroupConfig::default(), } } @@ -868,13 +870,34 @@ impl InnerBuilder { token_set_format: TokenSetFormat, posting_tail_codec: PostingTailCodec, ) -> Self { - let format_version = if posting_tail_codec == PostingTailCodec::Fixed32 { - InvertedListFormatVersion::V1 - } else { - InvertedListFormatVersion::V2 - }; - let mut builder = - Self::new_with_format_version(id, with_position, token_set_format, format_version); + Self::new_with_posting_tail_codec_and_block_size( + id, + with_position, + token_set_format, + posting_tail_codec, + LEGACY_BLOCK_SIZE, + ) + } + + pub fn new_with_posting_tail_codec_and_block_size( + id: u64, + with_position: bool, + token_set_format: TokenSetFormat, + posting_tail_codec: PostingTailCodec, + block_size: usize, + ) -> Self { + let format_version = InvertedListFormatVersion::from_posting_tail_codec_and_block_size( + posting_tail_codec, + block_size, + ) + .expect("invalid posting tail codec for posting block size"); + let mut builder = Self::new_with_format_version_and_block_size( + id, + with_position, + token_set_format, + format_version, + block_size, + ); builder.posting_tail_codec = posting_tail_codec; builder } @@ -955,10 +978,10 @@ impl InnerBuilder { token_set_format, format_version, posting_tail_codec, + block_size, tokens, posting_lists, docs, - group_config: _, } = other; if self.with_position != with_position { @@ -985,6 +1008,12 @@ impl InnerBuilder { self.posting_tail_codec, posting_tail_codec ))); } + if self.block_size != block_size { + return Err(Error::index(format!( + "cannot merge partitions with mismatched FTS block sizes: {} vs {}", + self.block_size, block_size + ))); + } let mut token_id_map = vec![u32::MAX; posting_lists.len()]; match tokens.tokens { @@ -1006,11 +1035,23 @@ impl InnerBuilder { } let doc_id_offset = self.docs.len() as u32; - for (row_id, num_tokens) in docs.iter() { - self.docs.append(*row_id, *num_tokens); + for doc_id in 0..docs.len() as u32 { + let row_id = docs.row_id(doc_id); + let num_tokens = docs.num_tokens(doc_id); + let doc_index = docs.doc_index(doc_id); + if doc_index.is_empty() { + self.docs.append(row_id, num_tokens); + } else { + self.docs + .append_with_doc_index(row_id, num_tokens, &doc_index)?; + } } self.posting_lists.resize_with(self.tokens.len(), || { - PostingListBuilder::new_with_posting_tail_codec(with_position, self.posting_tail_codec) + PostingListBuilder::new_with_posting_tail_codec_and_block_size( + with_position, + self.posting_tail_codec, + self.block_size, + ) }); for (token_id, posting_list) in posting_lists.into_iter().enumerate() { @@ -1077,7 +1118,11 @@ impl InnerBuilder { let mut writer = store .new_index_file( path, - inverted_list_schema_for_version(self.with_position, self.format_version), + inverted_list_schema_for_version_with_block_size( + self.with_position, + self.format_version, + self.block_size, + ), ) .await?; let posting_lists = std::mem::take(&mut self.posting_lists); @@ -1090,8 +1135,11 @@ impl InnerBuilder { ); let with_position = self.with_position; let format_version = self.format_version; - let group_config = self.group_config; - let schema = inverted_list_schema_for_version(self.with_position, self.format_version); + let schema = inverted_list_schema_for_version_with_block_size( + self.with_position, + self.format_version, + self.block_size, + ); let docs_for_batches = docs.clone(); let schema_for_batches = schema.clone(); let batch_rows = *LANCE_FTS_POSTING_BATCH_ROWS; @@ -1111,15 +1159,13 @@ impl InnerBuilder { with_position, format_version, batch_rows, - group_config, ); let mut posting_lists = posting_lists.into_iter(); loop { let docs_for_batches = docs_for_batches.clone(); // Build the next batch on the CPU pool. The builder and the // remaining posting lists are moved in and handed back so state - // persists across batches -- notably the cache-group accumulator, - // which spans every batch this builder produces. + // persists across batches. let (next_builder, next_posting_lists, batch) = spawn_cpu(move || { let mut batch_builder = batch_builder; let mut posting_lists = posting_lists; @@ -1155,7 +1201,7 @@ impl InnerBuilder { } } - Result::Ok(batch_builder.into_group_starts()) + Result::Ok(()) }); while let Ok(batch) = rx.recv().await { @@ -1167,22 +1213,8 @@ impl InnerBuilder { } } drop(rx); - let group_starts = producer.await??; - - // Persist the posting-list cache-group boundaries as a global buffer, - // recording its 1-indexed id in schema metadata so the reader can group - // small posting lists into a single cache entry (issue #7040). Empty - // partitions skip this entirely and fall back to the per-token path. - let mut extra_metadata = HashMap::new(); - if !group_starts.is_empty() { - let encoded = encode_group_starts(&group_starts); - let buffer_id = writer.add_global_buffer(Bytes::from(encoded)).await?; - extra_metadata.insert( - POSTING_GROUP_OFFSETS_BUF_KEY.to_owned(), - buffer_id.to_string(), - ); - } - writer.finish_with_metadata(extra_metadata).await + producer.await??; + writer.finish().await } #[instrument(level = "debug", skip_all)] @@ -1206,7 +1238,12 @@ impl InnerBuilder { let batch = docs.to_batch()?; let mut writer = store.new_index_file(path, batch.schema()).await?; writer.write_record_batch(batch).await?; - writer.finish().await + writer + .finish_with_metadata(HashMap::from([( + super::documents::TOTAL_TOKENS_KEY.to_owned(), + docs.total_tokens_num().to_string(), + )])) + .await } } @@ -1252,6 +1289,7 @@ struct IndexWorker { token_set_format: TokenSetFormat, token_ids: Vec, last_token_count: usize, + coordinate_rank: usize, } struct TailPartition { @@ -1264,6 +1302,11 @@ struct WorkerOutput { tail_partition: Option, } +enum DocumentSource<'a> { + Text(&'a str), + StringList(&'a dyn Array), +} + #[derive(Debug, Clone, Copy)] struct IndexWorkerConfig { with_position: bool, @@ -1271,6 +1314,8 @@ struct IndexWorkerConfig { fragment_mask: Option, token_set_format: TokenSetFormat, worker_memory_limit_bytes: u64, + block_size: usize, + coordinate_rank: usize, } impl IndexWorker { @@ -1314,18 +1359,26 @@ impl IndexWorker { id_alloc: Arc, config: IndexWorkerConfig, ) -> Result { - let schema = inverted_list_schema_for_version(config.with_position, config.format_version); + let schema = inverted_list_schema_for_version_with_block_size( + config.with_position, + config.format_version, + config.block_size, + ); + + let mut builder = InnerBuilder::new_with_format_version_and_block_size( + id_alloc.fetch_add(1, std::sync::atomic::Ordering::Relaxed) + | config.fragment_mask.unwrap_or(0), + config.with_position, + config.token_set_format, + config.format_version, + config.block_size, + ); + builder.docs = DocSet::with_coordinate_rank(config.coordinate_rank); Ok(Self { tokenizer, dest_store, - builder: InnerBuilder::new_with_format_version( - id_alloc.fetch_add(1, std::sync::atomic::Ordering::Relaxed) - | config.fragment_mask.unwrap_or(0), - config.with_position, - config.token_set_format, - config.format_version, - ), + builder, partitions: Vec::new(), files: Vec::new(), id_alloc, @@ -1337,6 +1390,7 @@ impl IndexWorker { token_set_format: config.token_set_format, token_ids: Vec::new(), last_token_count: 0, + coordinate_rank: config.coordinate_rank, }) } @@ -1349,147 +1403,301 @@ impl IndexWorker { async fn process_batch(&mut self, batch: RecordBatch) -> Result<()> { let doc_col = batch.column(0); - let doc_iter = iter_str_array(doc_col); let row_id_col = batch[ROW_ID].as_primitive::(); - let docs = doc_iter - .zip(row_id_col.values().iter()) - .filter_map(|(doc, row_id)| doc.map(|doc| (doc, *row_id))); + let doc_index_columns = (0..self.coordinate_rank) + .map(|rank| { + let column_name = doc_index_storage_column(rank); + batch + .column_by_name(&column_name) + .ok_or_else(|| { + Error::index(format!( + "FTS document input is missing coordinate column {column_name}" + )) + }) + .map(|column| column.as_primitive::()) + }) + .collect::>>()?; + match doc_col.data_type() { + DataType::Utf8 | DataType::LargeUtf8 | DataType::Utf8View => { + for (row_index, (doc, row_id)) in iter_str_array(doc_col.as_ref()) + .zip(row_id_col.values().iter()) + .enumerate() + { + let doc = match doc { + Some(doc) => doc, + None if self.coordinate_rank > 0 => "", + None => continue, + }; + let doc_index = doc_index_columns + .iter() + .map(|column| column.value(row_index)) + .collect::>(); + self.process_document(*row_id, DocumentSource::Text(doc), &doc_index) + .await?; + } + } + DataType::List(_) => { + if self.coordinate_rank > 0 { + return Err(Error::index( + "ListElement FTS input must be expanded to string documents before indexing" + .to_string(), + )); + } + self.process_string_list_batch::(doc_col, row_id_col) + .await?; + } + DataType::LargeList(_) => { + if self.coordinate_rank > 0 { + return Err(Error::index( + "ListElement FTS input must be expanded to string documents before indexing" + .to_string(), + )); + } + self.process_string_list_batch::(doc_col, row_id_col) + .await?; + } + data_type => { + return Err(Error::index(format!( + "expect data type String, LargeString, List(String), or LargeList(String) but got {}", + data_type + ))); + } + } + Ok(()) + } + + async fn process_string_list_batch( + &mut self, + doc_col: &Arc, + row_id_col: &arrow_array::PrimitiveArray, + ) -> Result<()> { + let docs = doc_col.as_list::(); + match docs.value_type() { + datatypes::DataType::Utf8 | datatypes::DataType::LargeUtf8 => {} + data_type => { + return Err(Error::index(format!( + "expect list item data type String or LargeString but got {}", + data_type + ))); + } + } + + for (doc, row_id) in docs.iter().zip(row_id_col.values().iter()) { + let Some(doc) = doc else { + continue; + }; + + self.process_document(*row_id, DocumentSource::StringList(doc.as_ref()), &[]) + .await?; + } + + Ok(()) + } + + fn checked_token_position(row_id: u64, token_position: usize) -> Result { + u32::try_from(token_position).map_err(|_| { + Error::invalid_input(format!( + "token position overflow for row_id={row_id}: token_position={token_position}" + )) + }) + } + + fn materialize_string_list(elements: &dyn Array) -> String { + let mut doc = String::new(); + for element in iter_str_array(elements).flatten() { + if !doc.is_empty() { + doc.push(' '); + } + doc.push_str(element); + } + doc + } + + async fn process_document( + &mut self, + row_id: u64, + document: DocumentSource<'_>, + doc_index: &[u32], + ) -> Result<()> { let with_position = self.has_position(); - for (doc, row_id) in docs { - let builder_was_empty = self.builder.docs.is_empty(); - let old_temporary_memory_size = self.temporary_memory_size(); - let old_token_memory_size = self.builder.tokens.memory_size() as u64; - let doc_id = self.builder.docs.len() as u32; - let mut token_num: u32 = 0; - let mut posting_memory_delta = 0i64; - if with_position { + let builder_was_empty = self.builder.docs.is_empty(); + let old_temporary_memory_size = self.temporary_memory_size(); + let old_token_memory_size = self.builder.tokens.memory_size() as u64; + let doc_id = self.builder.docs.len() as u32; + let mut token_num: u32 = 0; + let mut doc_length_bytes = 0usize; + let mut posting_memory_delta = 0i64; + if with_position { + { if self.token_ids.capacity() < self.last_token_count { self.token_ids .reserve(self.last_token_count - self.token_ids.capacity()); } self.token_ids.clear(); + let tokenizer = &mut self.tokenizer; let builder = &mut self.builder; let token_ids = &mut self.token_ids; let memory_size = &mut self.memory_size; let posting_tail_codec = builder.posting_tail_codec; - let mut token_stream = self.tokenizer.token_stream_for_doc(doc); - while token_stream.advance() { - let token = token_stream.token(); - let token_id = builder.tokens.get_or_add(&token.text); - if token_id as usize == builder.posting_lists.len() { - let old_posting_lists_overhead_size = (builder.posting_lists.capacity() - * std::mem::size_of::()) - as u64; - builder.posting_lists.push( - PostingListBuilder::new_with_posting_tail_codec( - true, - posting_tail_codec, - ), - ); - let new_posting_lists_overhead_size = (builder.posting_lists.capacity() - * std::mem::size_of::()) - as u64; - Self::adjust_tracked_value( - memory_size, - old_posting_lists_overhead_size, - new_posting_lists_overhead_size, - ); + let block_size = builder.block_size; + let mut process_text = |text: &str| -> Result<()> { + doc_length_bytes += text.len(); + let mut token_stream = tokenizer.token_stream_for_doc(text); + while token_stream.advance() { + let token = token_stream.token(); + let position = Self::checked_token_position(row_id, token.position)?; + let token_id = builder.tokens.get_or_add(&token.text); + if token_id as usize == builder.posting_lists.len() { + let old_posting_lists_overhead_size = (builder.posting_lists.capacity() + * std::mem::size_of::()) + as u64; + builder.posting_lists.push( + PostingListBuilder::new_with_posting_tail_codec_and_block_size( + true, + posting_tail_codec, + block_size, + ), + ); + let new_posting_lists_overhead_size = (builder.posting_lists.capacity() + * std::mem::size_of::()) + as u64; + Self::adjust_tracked_value( + memory_size, + old_posting_lists_overhead_size, + new_posting_lists_overhead_size, + ); + } + let posting_list = &mut builder.posting_lists[token_id as usize]; + let old_posting_memory_size = posting_list.size(); + if posting_list.add_occurrence(doc_id, position)? { + token_ids.push(token_id); + } + let new_posting_memory_size = posting_list.size(); + posting_memory_delta += + new_posting_memory_size as i64 - old_posting_memory_size as i64; + token_num += 1; + } + Ok(()) + }; + + match document { + DocumentSource::Text(doc) => { + process_text(doc)?; } - let posting_list = &mut builder.posting_lists[token_id as usize]; - let old_posting_memory_size = posting_list.size(); - if posting_list.add_occurrence(doc_id, token.position as u32)? { - token_ids.push(token_id); + DocumentSource::StringList(elements) => { + let doc = Self::materialize_string_list(elements); + process_text(&doc)?; } - let new_posting_memory_size = posting_list.size(); - posting_memory_delta += - new_posting_memory_size as i64 - old_posting_memory_size as i64; - token_num += 1; } - } else { + } + } else { + { if self.token_ids.capacity() < self.last_token_count { self.token_ids .reserve(self.last_token_count - self.token_ids.capacity()); } self.token_ids.clear(); - let mut token_stream = self.tokenizer.token_stream_for_doc(doc); - while token_stream.advance() { - let token_id = self.builder.tokens.get_or_add(&token_stream.token().text); - self.token_ids.push(token_id); - token_num += 1; - } - } - self.adjust_tracked_memory_size( - old_token_memory_size, - self.builder.tokens.memory_size() as u64, - ); + let tokenizer = &mut self.tokenizer; + let builder = &mut self.builder; + let token_ids = &mut self.token_ids; + let mut process_text = |text: &str| { + doc_length_bytes += text.len(); + let mut token_stream = tokenizer.token_stream_for_doc(text); + while token_stream.advance() { + let token_id = builder.tokens.get_or_add(&token_stream.token().text); + token_ids.push(token_id); + token_num += 1; + } + }; - if !with_position { - let old_posting_lists_overhead_size = self.posting_lists_overhead_size(); - self.builder - .posting_lists - .resize_with(self.builder.tokens.len(), || { - PostingListBuilder::new_with_posting_tail_codec( - false, - self.builder.posting_tail_codec, - ) - }); - let new_posting_lists_overhead_size = self.posting_lists_overhead_size(); - Self::adjust_tracked_value( - &mut self.memory_size, - old_posting_lists_overhead_size, - new_posting_lists_overhead_size, - ); + match document { + DocumentSource::Text(doc) => process_text(doc), + DocumentSource::StringList(elements) => { + let doc = Self::materialize_string_list(elements); + process_text(&doc); + } + } } + } + self.adjust_tracked_memory_size( + old_token_memory_size, + self.builder.tokens.memory_size() as u64, + ); - let old_doc_memory_size = self.builder.docs.memory_size() as u64; - let appended_doc_id = self.builder.docs.append(row_id, token_num); - debug_assert_eq!(appended_doc_id, doc_id); + // Row indexes omit zero-token documents from corpus statistics. + // ListElement indexes retain them so their physical coordinates remain + // part of the document corpus even when they cannot match a term. + if token_num == 0 && self.coordinate_rank == 0 { + self.last_token_count = 0; + self.trim_temporary_buffers(); self.adjust_tracked_memory_size( - old_doc_memory_size, - self.builder.docs.memory_size() as u64, + old_temporary_memory_size, + self.temporary_memory_size(), ); - self.total_doc_length += doc.len(); + return Ok(()); + } - if with_position { - for &token_id in &self.token_ids { - let (old_posting_memory_size, new_posting_memory_size) = { - let posting_list = &mut self.builder.posting_lists[token_id as usize]; - let old_posting_memory_size = posting_list.size(); - posting_list.finish_open_doc(doc_id)?; - let new_posting_memory_size = posting_list.size(); - (old_posting_memory_size, new_posting_memory_size) - }; - posting_memory_delta += - new_posting_memory_size as i64 - old_posting_memory_size as i64; - } - Self::apply_delta(&mut self.memory_size, posting_memory_delta); - } else if token_num > 0 { - self.token_ids.sort_unstable(); - let mut iter = self.token_ids.iter(); - let mut current = *iter.next().unwrap(); - let mut count = 1u32; - for &token_id in iter { - if token_id == current { - count += 1; - continue; - } + if !with_position { + let old_posting_lists_overhead_size = self.posting_lists_overhead_size(); + self.builder + .posting_lists + .resize_with(self.builder.tokens.len(), || { + PostingListBuilder::new_with_posting_tail_codec_and_block_size( + false, + self.builder.posting_tail_codec, + self.builder.block_size, + ) + }); + let new_posting_lists_overhead_size = self.posting_lists_overhead_size(); + Self::adjust_tracked_value( + &mut self.memory_size, + old_posting_lists_overhead_size, + new_posting_lists_overhead_size, + ); + } - let (old_posting_memory_size, new_posting_memory_size) = { - let posting_list = &mut self.builder.posting_lists[current as usize]; - let old_posting_memory_size = posting_list.size(); - posting_list.add(doc_id, PositionRecorder::Count(count)); - let new_posting_memory_size = posting_list.size(); - (old_posting_memory_size, new_posting_memory_size) - }; - posting_memory_delta += - new_posting_memory_size as i64 - old_posting_memory_size as i64; + let old_doc_memory_size = self.builder.docs.memory_size() as u64; + let appended_doc_id = if doc_index.is_empty() { + self.builder.docs.append(row_id, token_num) + } else { + self.builder + .docs + .append_with_doc_index(row_id, token_num, doc_index)? + }; + debug_assert_eq!(appended_doc_id, doc_id); + self.adjust_tracked_memory_size( + old_doc_memory_size, + self.builder.docs.memory_size() as u64, + ); + self.total_doc_length += doc_length_bytes; - current = token_id; - count = 1; + if with_position { + for &token_id in &self.token_ids { + let (old_posting_memory_size, new_posting_memory_size) = { + let posting_list = &mut self.builder.posting_lists[token_id as usize]; + let old_posting_memory_size = posting_list.size(); + posting_list.finish_open_doc(doc_id)?; + let new_posting_memory_size = posting_list.size(); + (old_posting_memory_size, new_posting_memory_size) + }; + posting_memory_delta += + new_posting_memory_size as i64 - old_posting_memory_size as i64; + } + Self::apply_delta(&mut self.memory_size, posting_memory_delta); + } else if token_num > 0 { + self.token_ids.sort_unstable(); + let mut iter = self.token_ids.iter(); + let mut current = *iter.next().unwrap(); + let mut count = 1u32; + for &token_id in iter { + if token_id == current { + count += 1; + continue; } + let (old_posting_memory_size, new_posting_memory_size) = { let posting_list = &mut self.builder.posting_lists[current as usize]; let old_posting_memory_size = posting_list.size(); @@ -1499,27 +1707,35 @@ impl IndexWorker { }; posting_memory_delta += new_posting_memory_size as i64 - old_posting_memory_size as i64; - Self::apply_delta(&mut self.memory_size, posting_memory_delta); - } - self.last_token_count = self.token_ids.len(); - self.trim_temporary_buffers(); - self.adjust_tracked_memory_size( - old_temporary_memory_size, - self.temporary_memory_size(), - ); - if self.builder.docs.len() == 1 && self.memory_size > self.worker_memory_limit_bytes { - return Err(Error::invalid_input(format!( - "single document row_id={} exceeds worker memory limit: {} > {} bytes", - row_id, self.memory_size, self.worker_memory_limit_bytes - ))); + current = token_id; + count = 1; } + let (old_posting_memory_size, new_posting_memory_size) = { + let posting_list = &mut self.builder.posting_lists[current as usize]; + let old_posting_memory_size = posting_list.size(); + posting_list.add(doc_id, PositionRecorder::Count(count)); + let new_posting_memory_size = posting_list.size(); + (old_posting_memory_size, new_posting_memory_size) + }; + posting_memory_delta += new_posting_memory_size as i64 - old_posting_memory_size as i64; + Self::apply_delta(&mut self.memory_size, posting_memory_delta); + } + self.last_token_count = self.token_ids.len(); + self.trim_temporary_buffers(); + self.adjust_tracked_memory_size(old_temporary_memory_size, self.temporary_memory_size()); - if self.builder.docs.len() as u32 == u32::MAX - || (!builder_was_empty && self.memory_size >= self.worker_memory_limit_bytes) - { - self.flush().await?; - } + if self.builder.docs.len() == 1 && self.memory_size > self.worker_memory_limit_bytes { + return Err(Error::invalid_input(format!( + "single document row_id={} exceeds worker memory limit: {} > {} bytes", + row_id, self.memory_size, self.worker_memory_limit_bytes + ))); + } + + if self.builder.docs.len() as u32 == u32::MAX + || (!builder_was_empty && self.memory_size >= self.worker_memory_limit_bytes) + { + self.flush().await?; } Ok(()) @@ -1527,7 +1743,7 @@ impl IndexWorker { #[instrument(level = "debug", skip_all)] async fn flush(&mut self) -> Result<()> { - if self.builder.tokens.is_empty() { + if self.builder.docs.is_empty() { return Ok(()); } @@ -1538,17 +1754,18 @@ impl IndexWorker { self.memory_size = self.temporary_memory_size(); let with_position = self.has_position(); let format_version = self.builder.format_version; - let builder = std::mem::replace( - &mut self.builder, - InnerBuilder::new_with_format_version( - self.id_alloc - .fetch_add(1, std::sync::atomic::Ordering::Relaxed) - | self.fragment_mask.unwrap_or(0), - with_position, - self.token_set_format, - format_version, - ), + let block_size = self.builder.block_size; + let mut replacement = InnerBuilder::new_with_format_version_and_block_size( + self.id_alloc + .fetch_add(1, std::sync::atomic::Ordering::Relaxed) + | self.fragment_mask.unwrap_or(0), + with_position, + self.token_set_format, + format_version, + block_size, ); + replacement.docs = DocSet::with_coordinate_rank(self.coordinate_rank); + let builder = std::mem::replace(&mut self.builder, replacement); let written_partition_id = builder.id(); let mut builder = builder; let target = if self.fragment_mask.is_some() { @@ -1571,7 +1788,7 @@ impl IndexWorker { } async fn finish(self) -> Result { - let tail_partition = if self.builder.tokens.is_empty() { + let tail_partition = if self.builder.docs.is_empty() && self.coordinate_rank == 0 { None } else { Some(TailPartition { @@ -1615,6 +1832,7 @@ impl PositionRecorder { #[derive(Debug, Eq, PartialEq, Clone, DeepSizeOf)] pub struct ScoredDoc { pub row_id: u64, + pub doc_index: Vec, pub score: OrderedFloat, } @@ -1622,6 +1840,15 @@ impl ScoredDoc { pub fn new(row_id: u64, score: f32) -> Self { Self { row_id, + doc_index: Vec::new(), + score: OrderedFloat(score), + } + } + + pub fn with_doc_index(row_id: u64, doc_index: Vec, score: f32) -> Self { + Self { + row_id, + doc_index, score: OrderedFloat(score), } } @@ -1666,17 +1893,56 @@ pub fn inverted_list_schema_for_version( with_position: bool, format_version: InvertedListFormatVersion, ) -> SchemaRef { + inverted_list_schema_for_version_with_block_size( + with_position, + format_version, + LEGACY_BLOCK_SIZE, + ) +} + +pub fn inverted_list_schema_for_version_with_block_size( + with_position: bool, + format_version: InvertedListFormatVersion, + block_size: usize, +) -> SchemaRef { + inverted_list_schema_for_version_with_block_size_and_impacts( + with_position, + format_version, + block_size, + true, + ) +} + +pub(crate) fn inverted_list_schema_for_version_with_block_size_and_impacts( + with_position: bool, + format_version: InvertedListFormatVersion, + block_size: usize, + with_impacts: bool, +) -> SchemaRef { + validate_format_version_block_size(format_version, block_size) + .expect("invalid FTS format version for posting block size"); match format_version { - InvertedListFormatVersion::V1 => inverted_list_schema_v1(with_position), - InvertedListFormatVersion::V2 => inverted_list_schema_with_tail_codec_and_position_codec( - with_position, - PostingTailCodec::VarintDelta, - Some(PositionStreamCodec::PackedDelta), - ), + InvertedListFormatVersion::V1 => { + inverted_list_schema_v1(with_position, block_size, with_impacts) + } + InvertedListFormatVersion::V2 | InvertedListFormatVersion::V3 => { + inverted_list_schema_with_tail_codec_and_position_codec( + with_position, + format_version, + PostingTailCodec::VarintDelta, + Some(PositionStreamCodec::PackedDelta), + block_size, + with_impacts, + ) + } } } -fn inverted_list_schema_v1(with_position: bool) -> SchemaRef { +fn inverted_list_schema_v1( + with_position: bool, + block_size: usize, + with_impacts: bool, +) -> SchemaRef { let mut fields = vec![ arrow_schema::Field::new( POSTING_COL, @@ -1690,6 +1956,17 @@ fn inverted_list_schema_v1(with_position: bool) -> SchemaRef { arrow_schema::Field::new(MAX_SCORE_COL, datatypes::DataType::Float32, false), arrow_schema::Field::new(LENGTH_COL, datatypes::DataType::UInt32, false), ]; + if with_impacts { + fields.push(arrow_schema::Field::new( + IMPACT_COL, + datatypes::DataType::List(Arc::new(Field::new( + "item", + datatypes::DataType::LargeBinary, + true, + ))), + false, + )); + } if with_position { fields.push(arrow_schema::Field::new( POSITION_COL, @@ -1705,24 +1982,44 @@ fn inverted_list_schema_v1(with_position: bool) -> SchemaRef { false, )); } - Arc::new(arrow_schema::Schema::new(fields)) + Arc::new(arrow_schema::Schema::new_with_metadata( + fields, + HashMap::from([ + (POSTING_BLOCK_SIZE_KEY.to_owned(), block_size.to_string()), + ( + FTS_FORMAT_VERSION_KEY.to_owned(), + InvertedListFormatVersion::V1.index_version().to_string(), + ), + ]), + )) } pub fn inverted_list_schema_with_tail_codec( with_position: bool, posting_tail_codec: PostingTailCodec, ) -> SchemaRef { + let format_version = InvertedListFormatVersion::from_posting_tail_codec_and_block_size( + posting_tail_codec, + LEGACY_BLOCK_SIZE, + ) + .expect("invalid posting tail codec for posting block size"); inverted_list_schema_with_tail_codec_and_position_codec( with_position, + format_version, posting_tail_codec, Some(PositionStreamCodec::PackedDelta), + LEGACY_BLOCK_SIZE, + false, ) } fn inverted_list_schema_with_tail_codec_and_position_codec( with_position: bool, + format_version: InvertedListFormatVersion, posting_tail_codec: PostingTailCodec, position_codec: Option, + block_size: usize, + with_impacts: bool, ) -> SchemaRef { let mut fields = vec![ // we compress the posting lists (including row ids and frequencies), @@ -1739,6 +2036,17 @@ fn inverted_list_schema_with_tail_codec_and_position_codec( arrow_schema::Field::new(MAX_SCORE_COL, datatypes::DataType::Float32, false), arrow_schema::Field::new(LENGTH_COL, datatypes::DataType::UInt32, false), ]; + if with_impacts { + fields.push(arrow_schema::Field::new( + IMPACT_COL, + datatypes::DataType::List(Arc::new(Field::new( + "item", + datatypes::DataType::LargeBinary, + true, + ))), + false, + )); + } if with_position { fields.push(arrow_schema::Field::new( COMPRESSED_POSITION_COL, @@ -1759,6 +2067,11 @@ fn inverted_list_schema_with_tail_codec_and_position_codec( POSTING_TAIL_CODEC_KEY.to_owned(), posting_tail_codec.as_str().to_owned(), )]); + metadata.insert( + FTS_FORMAT_VERSION_KEY.to_owned(), + format_version.index_version().to_string(), + ); + metadata.insert(POSTING_BLOCK_SIZE_KEY.to_owned(), block_size.to_string()); if let Some(position_codec) = position_codec.filter(|_| with_position) { metadata.insert( POSITIONS_LAYOUT_KEY.to_owned(), @@ -1772,129 +2085,6 @@ fn inverted_list_schema_with_tail_codec_and_position_codec( Arc::new(arrow_schema::Schema::new_with_metadata(fields, metadata)) } -/// Flatten the string list stream into a string stream -pub struct FlattenStream { - /// Inner record batch stream with 2 columns: - /// 1. doc_col: List(Utf8) or List(LargeUtf8) - /// 2. row_id_col: UInt64 - inner: SendableRecordBatchStream, - field_type: DataType, - data_type: DataType, -} - -impl FlattenStream { - pub fn new(input: SendableRecordBatchStream) -> Self { - let schema = input.schema(); - let field = schema.field(0); - let data_type = match field.data_type() { - DataType::List(f) if matches!(f.data_type(), DataType::Utf8) => DataType::Utf8, - DataType::List(f) if matches!(f.data_type(), DataType::LargeUtf8) => { - DataType::LargeUtf8 - } - DataType::LargeList(f) if matches!(f.data_type(), DataType::Utf8) => DataType::Utf8, - DataType::LargeList(f) if matches!(f.data_type(), DataType::LargeUtf8) => { - DataType::LargeUtf8 - } - _ => panic!( - "expect data type List(Utf8) or List(LargeUtf8) but got {:?}", - field.data_type() - ), - }; - Self { - inner: input, - field_type: field.data_type().clone(), - data_type, - } - } -} - -impl Stream for FlattenStream { - type Item = datafusion_common::Result; - - fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { - match Pin::new(&mut self.inner).poll_next(cx) { - Poll::Ready(Some(Ok(batch))) => { - let doc_col = batch.column(0); - let batch = match self.field_type { - DataType::List(_) => flatten_string_list::(&batch, doc_col).map_err(|e| { - datafusion_common::error::DataFusionError::Execution(format!( - "flatten string list error: {}", - e - )) - }), - DataType::LargeList(_) => { - flatten_string_list::(&batch, doc_col).map_err(|e| { - datafusion_common::error::DataFusionError::Execution(format!( - "flatten string list error: {}", - e - )) - }) - } - _ => unreachable!( - "expect data type List or LargeList but got {:?}", - self.field_type - ), - }; - Poll::Ready(Some(batch)) - } - Poll::Ready(Some(Err(e))) => Poll::Ready(Some(Err(e))), - Poll::Ready(None) => Poll::Ready(None), - Poll::Pending => Poll::Pending, - } - } -} - -impl RecordBatchStream for FlattenStream { - fn schema(&self) -> SchemaRef { - let schema = Schema::new(vec![ - Field::new( - self.inner.schema().field(0).name(), - self.data_type.clone(), - true, - ), - ROW_ID_FIELD.clone(), - ]); - - Arc::new(schema) - } -} - -fn flatten_string_list( - batch: &RecordBatch, - doc_col: &Arc, -) -> Result { - let docs = doc_col.as_list::(); - let row_ids = batch[ROW_ID].as_primitive::(); - - let row_ids = row_ids - .values() - .iter() - .zip(docs.iter()) - .flat_map(|(row_id, doc)| std::iter::repeat_n(*row_id, doc.map(|d| d.len()).unwrap_or(0))); - - let row_ids = Arc::new(UInt64Array::from_iter_values(row_ids)); - let docs = match docs.value_type() { - datatypes::DataType::Utf8 | datatypes::DataType::LargeUtf8 => docs.values().clone(), - _ => { - return Err(Error::index(format!( - "expect data type String or LargeString but got {}", - docs.value_type() - ))); - } - }; - - let schema = Schema::new(vec![ - Field::new( - batch.schema().field(0).name(), - docs.data_type().clone(), - true, - ), - ROW_ID_FIELD.clone(), - ]); - let batch = RecordBatch::try_new(Arc::new(schema), vec![docs, row_ids])?; - Ok(batch) -} - pub(crate) fn token_file_path(partition_id: u64) -> String { format!("part_{}_{}", partition_id, TOKENS_FILE) } @@ -1911,6 +2101,10 @@ pub(crate) fn part_metadata_file_path(partition_id: u64) -> String { staged_partition_file_path(partition_id, METADATA_FILE) } +fn empty_part_metadata_file_path(fragment_mask: u64) -> String { + format!("{STAGED_PARTITION_DIR}/part_empty_{fragment_mask}_{METADATA_FILE}") +} + const PARTITION_FILE_SUFFIXES: [&str; 3] = [TOKENS_FILE, INVERT_LIST_FILE, DOCS_FILE]; const STAGED_PARTITION_DIR: &str = "staging"; @@ -2015,7 +2209,6 @@ async fn merge_metadata_files( let mut params = None; let mut token_set_format = None; let mut format_version = None; - let mut posting_tail_codec = None; let mut deleted_fragments = RoaringBitmap::new(); progress .stage_start( @@ -2057,9 +2250,6 @@ async fn merge_metadata_files( if format_version.is_none() { format_version = Some(parse_format_version_from_metadata(metadata)?); } - if posting_tail_codec.is_none() { - posting_tail_codec = Some(parse_posting_tail_codec(metadata)?); - } if reader.num_rows() > 0 { let metadata_batch = reader.read_range(0..1, None).await?; @@ -2125,8 +2315,7 @@ async fn merge_metadata_files( None, deleted_fragments, ) - .with_format_version(format_version.unwrap_or(InvertedListFormatVersion::V1)) - .with_posting_tail_codec(posting_tail_codec.unwrap_or(PostingTailCodec::Fixed32)); + .with_format_version(format_version.unwrap_or(InvertedListFormatVersion::V1)); progress .stage_start("write_merged_metadata", Some(1), "files") .await?; @@ -2164,11 +2353,11 @@ pub fn document_input( let schema = input.schema(); let field = schema.column_with_name(column).expect_ok()?.1; match field.data_type() { - DataType::Utf8 | DataType::LargeUtf8 => Ok(input), + DataType::Utf8 | DataType::LargeUtf8 | DataType::Utf8View => Ok(input), DataType::List(field) | DataType::LargeList(field) if matches!(field.data_type(), DataType::Utf8 | DataType::LargeUtf8) => { - Ok(Box::pin(FlattenStream::new(input))) + Ok(input) } DataType::LargeBinary => match field.metadata().get(ARROW_EXT_NAME_KEY) { Some(name) if name.as_str() == JSON_EXT_NAME => { @@ -2192,10 +2381,12 @@ pub fn document_input( #[cfg(test)] mod tests { use super::*; + use crate::Index; use crate::metrics::NoOpMetricsCollector; use crate::progress::IndexBuildProgress; + use crate::scalar::inverted::{MemBM25Scorer, Scorer}; use crate::scalar::{IndexFile, IndexReader, IndexWriter, ScalarIndex}; - use arrow_array::{RecordBatch, StringArray, UInt64Array}; + use arrow_array::{RecordBatch, StringArray, UInt32Array, UInt64Array}; use arrow_schema::{DataType, Field, Schema}; use async_trait::async_trait; use bytes::Bytes; @@ -2226,6 +2417,17 @@ mod tests { RecordBatch::try_new(schema, vec![docs, row_ids]).unwrap() } + fn make_doc_batch_from_docs(docs: Vec>) -> RecordBatch { + let schema = Arc::new(Schema::new(vec![ + Field::new("doc", DataType::Utf8, true), + Field::new(ROW_ID, DataType::UInt64, false), + ])); + let num_rows = docs.len(); + let docs = Arc::new(StringArray::from(docs)); + let row_ids = Arc::new(UInt64Array::from_iter_values(0..num_rows as u64)); + RecordBatch::try_new(schema, vec![docs, row_ids]).unwrap() + } + struct FailingListObjectStore { inner: InMemory, } @@ -2613,8 +2815,7 @@ mod tests { } async fn add_global_buffer(&mut self, _data: Bytes) -> Result { - // The posting-list writer stores the group offsets as a global - // buffer; mirror the real writer's 1-indexed return value. + // Mirror the real writer's 1-indexed return value. Ok(1) } @@ -2699,63 +2900,6 @@ mod tests { } } - fn collect_group_starts(config: PostingGroupConfig, sizes: &[usize]) -> Vec { - let mut acc = PostingGroupAccumulator::new(config); - for &size in sizes { - acc.push(size); - } - acc.into_starts() - } - - #[test] - fn test_group_accumulator_cuts_on_target_bytes() { - let config = PostingGroupConfig { - target_bytes: 100, - max_tokens: 1000, - }; - // 40+40 -> cut at 80? no, 80 < 100; third 40 reaches 120 >= 100 -> cut. - // So group 0 = tokens [0,3), then a new group starts at token 3. - let starts = collect_group_starts(config, &[40, 40, 40, 10, 10]); - assert_eq!(starts, vec![0, 3]); - } - - #[test] - fn test_group_accumulator_cuts_on_max_tokens() { - let config = PostingGroupConfig { - target_bytes: 1_000_000, - max_tokens: 2, - }; - // Byte target never reached; cap of 2 forces a cut every 2 tokens. - let starts = collect_group_starts(config, &[1, 1, 1, 1, 1]); - assert_eq!(starts, vec![0, 2, 4]); - } - - #[test] - fn test_group_accumulator_clamps_oversized_term() { - let config = PostingGroupConfig { - target_bytes: 100, - max_tokens: 64, - }; - // A term larger than the target that *starts* a group occupies that - // group alone ([1, 2) here), so a single huge posting list is never - // forced to share a cache entry. Token 0 (==100) closes its own group - // first; the trailing small terms regroup after the big one. - let starts = collect_group_starts(config, &[100, 5000, 10, 10]); - assert_eq!(starts, vec![0, 1, 2]); - - // A huge term encountered mid-group is absorbed and closes that group; - // we never split one term across groups. - let starts = collect_group_starts(config, &[10, 10, 5000, 10, 10]); - assert_eq!(starts, vec![0, 3]); - } - - #[test] - fn test_group_accumulator_empty_and_single() { - let config = PostingGroupConfig::default(); - assert_eq!(collect_group_starts(config, &[]), Vec::::new()); - assert_eq!(collect_group_starts(config, &[10]), vec![0]); - } - #[tokio::test] async fn test_write_posting_lists_batches_multiple_rows() -> Result<()> { let mut builder = InnerBuilder::new(0, false, TokenSetFormat::default()); @@ -2890,6 +3034,10 @@ mod tests { expected_partitions.dedup(); let remapped_partitions = (0..expected_partitions.len() as u64).collect::>(); assert_eq!(written_partitions, remapped_partitions); + assert_eq!( + parse_format_version_from_metadata(metadata)?, + InvertedListFormatVersion::V2 + ); for (new_id, old_id) in expected_partitions.iter().enumerate() { assert_partition_file_markers(base_store.as_ref(), new_id as u64, *old_id).await?; @@ -3179,6 +3327,80 @@ mod tests { Ok(()) } + #[tokio::test] + async fn test_distributed_empty_build_does_not_finalize_shared_directory() -> Result<()> { + let index_dir = TempDir::default(); + let object_store = Arc::new(ObjectStore::local()); + let store = Arc::new(LanceIndexStore::new( + object_store.clone(), + index_dir.obj_path(), + Arc::new(LanceCache::no_cache()), + )); + + let empty_fragment_mask = 7_u64 << 32; + let batch = make_doc_batch_from_docs(vec![None, None]); + let stream = RecordBatchStreamAdapter::new(batch.schema(), stream::iter(vec![Ok(batch)])); + let params = InvertedIndexParams { + lance_tokenizer: Some("text".to_string()), + with_position: true, + ..Default::default() + }; + let mut builder = + InvertedIndexBuilder::new_with_fragment_mask(params.clone(), Some(empty_fragment_mask)); + + let files = builder + .update(Box::pin(stream), store.as_ref(), None) + .await?; + + assert_eq!(files.len(), 1); + let empty_metadata_path = empty_part_metadata_file_path(empty_fragment_mask); + assert_eq!(files[0].path, empty_metadata_path); + assert!( + store.open_index_file(METADATA_FILE).await.is_err(), + "an empty shard must not finalize the shared directory" + ); + let reader = store.open_index_file(&empty_metadata_path).await?; + let metadata = &reader.schema().metadata; + let partitions: Vec = serde_json::from_str( + metadata + .get("partitions") + .expect("partitions missing from metadata"), + )?; + assert!(partitions.is_empty()); + let written_params: InvertedIndexParams = serde_json::from_str( + metadata + .get("params") + .expect("params missing from metadata"), + )?; + assert_eq!(written_params, params); + + let non_empty_fragment_mask = 8_u64 << 32; + let batch = make_doc_batch("searchable text", non_empty_fragment_mask); + let stream = RecordBatchStreamAdapter::new(batch.schema(), stream::iter(vec![Ok(batch)])); + let mut builder = + InvertedIndexBuilder::new_with_fragment_mask(params, Some(non_empty_fragment_mask)); + builder + .update(Box::pin(stream), store.as_ref(), None) + .await?; + + let staged_metadata = + list_metadata_files(object_store.as_ref(), &index_dir.obj_path()).await?; + assert_eq!(staged_metadata.len(), 2); + + merge_index_files( + object_store.as_ref(), + &index_dir.obj_path(), + store.clone(), + noop_progress(), + ) + .await?; + + let index = InvertedIndex::load(store, None, &LanceCache::no_cache()).await?; + assert_eq!(index.partition_count(), 1); + + Ok(()) + } + #[tokio::test] async fn test_merge_index_files_is_noop_when_metadata_exists() -> Result<()> { let index_dir = TempDir::default(); @@ -3241,6 +3463,8 @@ mod tests { fragment_mask: None, token_set_format, worker_memory_limit_bytes: u64::MAX, + block_size: params.block_size, + coordinate_rank: 0, }, ) .await?; @@ -3264,6 +3488,8 @@ mod tests { fragment_mask: None, token_set_format, worker_memory_limit_bytes: u64::MAX, + block_size: params.block_size, + coordinate_rank: 0, }, ) .await?; @@ -3411,6 +3637,39 @@ mod tests { assert_eq!(builder.posting_tail_codec, PostingTailCodec::VarintDelta); } + #[test] + fn test_v3_128_reuses_v2_physical_layout() { + for with_position in [false, true] { + for with_impacts in [false, true] { + let v2 = inverted_list_schema_for_version_with_block_size_and_impacts( + with_position, + InvertedListFormatVersion::V2, + LEGACY_BLOCK_SIZE, + with_impacts, + ); + let v3 = inverted_list_schema_for_version_with_block_size_and_impacts( + with_position, + InvertedListFormatVersion::V3, + LEGACY_BLOCK_SIZE, + with_impacts, + ); + + assert_eq!(v2.fields(), v3.fields()); + let mut v2_metadata = v2.metadata.clone(); + let mut v3_metadata = v3.metadata.clone(); + assert_eq!( + v2_metadata.remove(FTS_FORMAT_VERSION_KEY).as_deref(), + Some("2") + ); + assert_eq!( + v3_metadata.remove(FTS_FORMAT_VERSION_KEY).as_deref(), + Some("3") + ); + assert_eq!(v2_metadata, v3_metadata); + } + } + } + #[tokio::test] async fn test_inverted_index_without_positions_tracks_frequency() -> Result<()> { let index_dir = TempDir::default(); @@ -3459,6 +3718,177 @@ mod tests { Ok(()) } + #[tokio::test] + async fn test_zero_token_string_documents_are_skipped_in_corpus_stats() -> Result<()> { + let index_dir = TempDir::default(); + let store = Arc::new(LanceIndexStore::new( + ObjectStore::local().into(), + index_dir.obj_path(), + Arc::new(LanceCache::no_cache()), + )); + + let batch = make_doc_batch_from_docs(vec![ + Some(""), + Some(" "), + Some("the"), + Some("overlength"), + None, + Some("hello"), + ]); + let stream = RecordBatchStreamAdapter::new(batch.schema(), stream::iter(vec![Ok(batch)])); + let params = + InvertedIndexParams::new("whitespace".to_string(), lance_tokenizer::Language::English) + .with_position(false) + .remove_stop_words(true) + .stem(false) + .max_token_length(Some(6)) + .num_workers(1); + + let mut builder = InvertedIndexBuilder::new(params); + builder + .update(Box::pin(stream), store.as_ref(), None) + .await?; + + let index = InvertedIndex::load(store, None, &LanceCache::no_cache()).await?; + let (total_tokens, num_docs, token_docs) = index + .bm25_stats_for_terms(&["hello".to_string()], None) + .await?; + assert_eq!(total_tokens, 1); + assert_eq!(num_docs, 1); + assert_eq!(token_docs, vec![1]); + + let actual_scorer = MemBM25Scorer::new( + total_tokens, + num_docs, + HashMap::from([("hello".to_string(), token_docs[0])]), + ); + let expected_scorer = MemBM25Scorer::new(1, 1, HashMap::from([("hello".to_string(), 1)])); + assert_eq!( + actual_scorer.avg_doc_length(), + expected_scorer.avg_doc_length() + ); + assert_eq!( + actual_scorer.query_weight("hello"), + expected_scorer.query_weight("hello") + ); + + Ok(()) + } + + #[tokio::test] + async fn test_all_empty_string_documents_build_empty_index() -> Result<()> { + let index_dir = TempDir::default(); + let store = Arc::new(LanceIndexStore::new( + ObjectStore::local().into(), + index_dir.obj_path(), + Arc::new(LanceCache::no_cache()), + )); + + let batch = make_doc_batch_from_docs(vec![Some(""), Some(" "), None]); + let stream = RecordBatchStreamAdapter::new(batch.schema(), stream::iter(vec![Ok(batch)])); + let params = + InvertedIndexParams::new("whitespace".to_string(), lance_tokenizer::Language::English) + .with_position(false) + .remove_stop_words(false) + .stem(false) + .max_token_length(None) + .num_workers(1); + + let mut builder = InvertedIndexBuilder::new(params); + builder + .update(Box::pin(stream), store.as_ref(), None) + .await?; + + let index = InvertedIndex::load(store, None, &LanceCache::no_cache()).await?; + assert!(index.partitions.is_empty()); + let statistics = index.statistics()?; + assert_eq!(statistics["num_tokens"], 0); + assert_eq!(statistics["num_docs"], 0); + + Ok(()) + } + + #[tokio::test] + async fn test_zero_token_coordinate_documents_are_preserved_in_corpus_stats() -> Result<()> { + let index_dir = TempDir::default(); + let store = Arc::new(LanceIndexStore::new( + ObjectStore::local().into(), + index_dir.obj_path(), + Arc::new(LanceCache::no_cache()), + )); + let schema = Arc::new(Schema::new(vec![ + Field::new("doc", DataType::Utf8, true), + Field::new(doc_index_storage_column(0), DataType::UInt32, false), + Field::new(ROW_ID, DataType::UInt64, false), + ])); + let batch = RecordBatch::try_new( + schema, + vec![ + Arc::new(StringArray::from(vec![ + None, + Some(""), + Some(" "), + Some("the"), + Some("overlength"), + ])), + Arc::new(UInt32Array::from(vec![0, 1, 2, 3, 4])), + Arc::new(UInt64Array::from(vec![7, 7, 7, 7, 7])), + ], + )?; + let stream = RecordBatchStreamAdapter::new(batch.schema(), stream::iter(vec![Ok(batch)])); + let params = + InvertedIndexParams::new("whitespace".to_string(), lance_tokenizer::Language::English) + .with_position(false) + .remove_stop_words(true) + .stem(false) + .max_token_length(Some(6)) + .num_workers(1); + + let mut builder = InvertedIndexBuilder::new(params); + builder + .update(Box::pin(stream), store.as_ref(), None) + .await?; + + let index = InvertedIndex::load(store, None, &LanceCache::no_cache()).await?; + let statistics = index.statistics()?; + assert_eq!(statistics["num_tokens"], 0); + assert_eq!(statistics["num_docs"], 5); + + Ok(()) + } + + #[tokio::test] + async fn test_all_empty_string_documents_do_not_create_tail_partition() -> Result<()> { + let tokenizer = InvertedIndexParams::default().build()?; + let store = Arc::new(CountingStore::new()); + let id_alloc = Arc::new(AtomicU64::new(0)); + let mut worker = IndexWorker::new( + tokenizer, + store, + id_alloc, + IndexWorkerConfig { + with_position: false, + format_version: InvertedListFormatVersion::V1, + fragment_mask: None, + token_set_format: TokenSetFormat::default(), + worker_memory_limit_bytes: u64::MAX, + block_size: InvertedIndexParams::default().block_size, + coordinate_rank: 0, + }, + ) + .await?; + + worker + .process_batch(make_doc_batch_from_docs(vec![Some(""), Some(" "), None])) + .await?; + let output = worker.finish().await?; + + assert!(output.partitions.is_empty()); + assert!(output.tail_partition.is_none()); + + Ok(()) + } + lance_testing::define_stage_event_progress!(RecordingProgress, IndexBuildProgress, Result<()>); #[derive(Debug, Default)] @@ -3746,6 +4176,8 @@ mod tests { fragment_mask: None, token_set_format: TokenSetFormat::default(), worker_memory_limit_bytes: u64::MAX, + block_size: InvertedIndexParams::default().block_size, + coordinate_rank: 0, }, ) .await?; @@ -3777,6 +4209,8 @@ mod tests { fragment_mask: None, token_set_format: TokenSetFormat::default(), worker_memory_limit_bytes: u64::MAX, + block_size: InvertedIndexParams::default().block_size, + coordinate_rank: 0, }, ) .await?; @@ -3815,6 +4249,8 @@ mod tests { fragment_mask: None, token_set_format: TokenSetFormat::default(), worker_memory_limit_bytes: u64::MAX, + block_size: InvertedIndexParams::default().block_size, + coordinate_rank: 0, }, ) .await?; diff --git a/rust/lance-index/src/scalar/inverted/cache_codec.rs b/rust/lance-index/src/scalar/inverted/cache_codec.rs index a676455d5c9..e59b22dc501 100644 --- a/rust/lance-index/src/scalar/inverted/cache_codec.rs +++ b/rust/lance-index/src/scalar/inverted/cache_codec.rs @@ -14,9 +14,13 @@ //! - the compressed posting list: an IPC section for `blocks`, then the //! position sections (legacy IPC, or shared block-offsets IPC + a raw blob of //! the [`SharedPositionStream`] byte buffer, which has its own portable -//! encoding); +//! encoding), then an optional impact IPC section; //! - the plain posting list: an IPC section of `(row_ids, frequencies)`, then //! an optional legacy position IPC section; +//! - a packed posting-list group: one IPC section containing the original +//! `List` posting rows and optional impact rows; prewarmed groups +//! omit score/length metadata and inject it from the posting reader into +//! query-local views; //! - the standalone [`Positions`] codec: the position sections alone. //! //! All sections read back zero-copy via [`lance_arrow::ipc`]. This is the FTS @@ -39,10 +43,13 @@ use crate::cache_pb::{ PostingTailCodec as PbPostingTailCodec, }; +use super::impact::ImpactSkipData; use super::index::{ CompressedPositionStorage, CompressedPostingList, PlainPostingList, PositionStreamCodec, - Positions, PostingList, PostingListGroup, PostingTailCodec, SharedPositionStream, + Positions, PostingList, PostingListGroup, PostingListGroupStorage, PostingTailCodec, + SharedPositionStream, }; +use super::tokenizer::{LEGACY_BLOCK_SIZE, validate_block_size}; // --------------------------------------------------------------------------- // Tags @@ -50,6 +57,8 @@ use super::index::{ const POSTING_VARIANT_PLAIN: u8 = 0; const POSTING_VARIANT_COMPRESSED: u8 = 1; +const GROUP_VARIANT_MATERIALIZED: u8 = 0; +const GROUP_VARIANT_PACKED: u8 = 1; // --------------------------------------------------------------------------- // Codec enum mappings @@ -72,6 +81,23 @@ fn proto_to_posting_tail_codec(c: PbPostingTailCodec) -> PostingTailCodec { } } +fn posting_tail_codec_to_tag(c: PostingTailCodec) -> u8 { + match c { + PostingTailCodec::Fixed32 => 0, + PostingTailCodec::VarintDelta => 1, + } +} + +fn posting_tail_codec_from_tag(tag: u8) -> Result { + match tag { + 0 => Ok(PostingTailCodec::Fixed32), + 1 => Ok(PostingTailCodec::VarintDelta), + other => Err(Error::io(format!( + "unknown packed posting tail codec: {other}" + ))), + } +} + fn position_stream_codec_to_proto(c: PositionStreamCodec) -> PbPositionStreamCodec { match c { PositionStreamCodec::VarintDocDelta => PbPositionStreamCodec::VarintDocDelta, @@ -95,6 +121,7 @@ const BLOCK_OFFSETS_COLUMN: &str = "block_offsets"; const ROW_IDS_COLUMN: &str = "row_ids"; const FREQUENCIES_COLUMN: &str = "frequencies"; const BLOCKS_COLUMN: &str = "blocks"; +const IMPACTS_COLUMN: &str = "impacts"; fn legacy_positions_batch(list: &ListArray) -> Result { let schema = Arc::new(Schema::new(vec![Field::new( @@ -108,7 +135,8 @@ fn legacy_positions_batch(list: &ListArray) -> Result { fn read_legacy_positions(r: &mut CacheEntryReader<'_>) -> Result { let batch = r.read_ipc()?; Ok(batch - .column(0) + .column_by_name(POSITION_LIST_COLUMN) + .ok_or_else(|| Error::io("legacy position column is missing".to_string()))? .as_any() .downcast_ref::() .ok_or_else(|| Error::io("legacy position column is not a ListArray".to_string()))? @@ -156,7 +184,8 @@ fn read_position_sections( PbPositionStorage::Shared => { let batch = r.read_ipc()?; let block_offsets = batch - .column(0) + .column_by_name(BLOCK_OFFSETS_COLUMN) + .ok_or_else(|| Error::io("block_offsets column is missing".to_string()))? .as_primitive_opt::() .ok_or_else(|| Error::io("block_offsets column is not UInt32".to_string()))? .values() @@ -178,7 +207,11 @@ fn read_position_sections( impl CacheCodecImpl for PostingList { const TYPE_ID: &'static str = "lance.fts.PostingList"; - const CURRENT_VERSION: u32 = 1; + // Version 3 adds the optional impact IPC section. Main already used v2 for + // configurable posting block sizes, so impact data needs a distinct + // version to keep older readers from accepting a body with an extra + // section they cannot consume. + const CURRENT_VERSION: u32 = 3; fn serialize(&self, w: &mut CacheEntryWriter<'_>) -> Result<()> { match self { @@ -194,15 +227,24 @@ impl CacheCodecImpl for PostingList { } fn deserialize(r: &mut CacheEntryReader<'_>) -> Result { - let variant = r.read_u8()?; - match variant { - POSTING_VARIANT_PLAIN => Ok(Self::Plain(deserialize_plain(r)?)), - POSTING_VARIANT_COMPRESSED => Ok(Self::Compressed(deserialize_compressed(r)?)), - other => Err(Error::io(format!("unknown PostingList variant: {other}"))), + match r.version() { + 1 | 2 | Self::CURRENT_VERSION => deserialize_posting_list_body(r), + other => Err(Error::io(format!( + "unsupported PostingList cache version: {other}" + ))), } } } +fn deserialize_posting_list_body(r: &mut CacheEntryReader<'_>) -> Result { + let variant = r.read_u8()?; + match variant { + POSTING_VARIANT_PLAIN => Ok(PostingList::Plain(deserialize_plain(r)?)), + POSTING_VARIANT_COMPRESSED => Ok(PostingList::Compressed(deserialize_compressed(r)?)), + other => Err(Error::io(format!("unknown PostingList variant: {other}"))), + } +} + fn serialize_plain(w: &mut CacheEntryWriter<'_>, plain: &PlainPostingList) -> Result<()> { // Plain postings carry only per-doc legacy positions (or none). let position_storage = if plain.positions.is_some() { @@ -236,13 +278,15 @@ fn deserialize_plain(r: &mut CacheEntryReader<'_>) -> Result { let batch = r.read_ipc()?; let row_ids = batch - .column(0) + .column_by_name(ROW_IDS_COLUMN) + .ok_or_else(|| Error::io("row_ids column is missing".to_string()))? .as_primitive_opt::() .ok_or_else(|| Error::io("row_ids column is not UInt64".to_string()))? .values() .clone(); let frequencies = batch - .column(1) + .column_by_name(FREQUENCIES_COLUMN) + .ok_or_else(|| Error::io("frequencies column is missing".to_string()))? .as_primitive_opt::() .ok_or_else(|| Error::io("frequencies column is not Float32".to_string()))? .values() @@ -291,6 +335,8 @@ fn serialize_compressed( posting_tail_codec: posting_tail_codec_to_proto(posting.posting_tail_codec) as i32, position_storage: position_storage as i32, position_stream_codec: position_stream_codec as i32, + block_size: posting.block_size as u32, + has_impacts: posting.impacts.is_some(), }; w.write_header(&header)?; @@ -305,6 +351,15 @@ fn serialize_compressed( if let Some(storage) = &posting.positions { write_position_sections(w, storage)?; } + if let Some(impacts) = &posting.impacts { + let schema = Arc::new(Schema::new(vec![Field::new( + IMPACTS_COLUMN, + DataType::LargeBinary, + false, + )])); + let batch = RecordBatch::try_new(schema, vec![Arc::new(impacts.entries().clone())])?; + w.write_ipc(&batch)?; + } Ok(()) } @@ -314,7 +369,8 @@ fn deserialize_compressed(r: &mut CacheEntryReader<'_>) -> Result() .ok_or_else(|| Error::io("blocks column is not a LargeBinaryArray".to_string()))? @@ -322,13 +378,33 @@ fn deserialize_compressed(r: &mut CacheEntryReader<'_>) -> Result= 3 && header.has_impacts { + let batch = r.read_ipc()?; + let entries = batch + .column_by_name(IMPACTS_COLUMN) + .ok_or_else(|| Error::io("impacts column is missing".to_string()))? + .as_any() + .downcast_ref::() + .ok_or_else(|| Error::io("impacts column is not a LargeBinaryArray".to_string()))? + .clone(); + Some(ImpactSkipData::new(entries, blocks.len())?) + } else { + None + }; Ok(CompressedPostingList::new( blocks, header.max_score, header.length, posting_tail_codec, + block_size, positions, + impacts, )) } @@ -336,34 +412,79 @@ fn deserialize_compressed(r: &mut CacheEntryReader<'_>) -> Result) -> Result<()> { - let count = u32::try_from(self.posting_lists.len()) + let count = u32::try_from(self.len()) .map_err(|_| Error::io("posting list group too large to serialize".to_string()))?; - w.write_header(&PostingListGroupHeader { count })?; - for posting in &self.posting_lists { - posting.serialize(w)?; + match &self.storage { + PostingListGroupStorage::Materialized(posting_lists) => { + w.write_u8(GROUP_VARIANT_MATERIALIZED)?; + w.write_header(&PostingListGroupHeader { count })?; + for posting in posting_lists { + posting.serialize(w)?; + } + } + PostingListGroupStorage::Packed(group) => { + w.write_u8(GROUP_VARIANT_PACKED)?; + w.write_header(&PostingListGroupHeader { count })?; + w.write_u8(posting_tail_codec_to_tag(group.posting_tail_codec))?; + w.write_ipc(&group.batch)?; + } } Ok(()) } fn deserialize(r: &mut CacheEntryReader<'_>) -> Result { - let header: PostingListGroupHeader = r.read_header()?; - let mut posting_lists = Vec::with_capacity(header.count as usize); - for _ in 0..header.count { - posting_lists.push(PostingList::deserialize(r)?); + match r.version() { + 1 => return deserialize_materialized_group(r), + 2 | 3 | Self::CURRENT_VERSION => {} + other => { + return Err(Error::io(format!( + "unsupported PostingListGroup cache version: {other}" + ))); + } } - Ok(Self::new(posting_lists)) + + let variant = r.read_u8()?; + match variant { + GROUP_VARIANT_MATERIALIZED => deserialize_materialized_group(r), + GROUP_VARIANT_PACKED => { + let header: PostingListGroupHeader = r.read_header()?; + let posting_tail_codec = posting_tail_codec_from_tag(r.read_u8()?)?; + let batch = r.read_ipc()?; + if batch.num_rows() != header.count as usize { + return Err(Error::io(format!( + "packed posting group row count {} does not match header count {}", + batch.num_rows(), + header.count + ))); + } + Self::new_packed(batch, posting_tail_codec) + } + other => Err(Error::io(format!( + "unknown PostingListGroup variant: {other}" + ))), + } + } +} + +fn deserialize_materialized_group(r: &mut CacheEntryReader<'_>) -> Result { + let header: PostingListGroupHeader = r.read_header()?; + let mut posting_lists = Vec::with_capacity(header.count as usize); + for _ in 0..header.count { + posting_lists.push(deserialize_posting_list_body(r)?); } + Ok(PostingListGroup::new(posting_lists)) } // --------------------------------------------------------------------------- @@ -409,17 +530,26 @@ impl CacheCodecImpl for Positions { #[cfg(test)] mod tests { + use std::collections::HashMap; + use std::sync::Arc; + use arrow::buffer::ScalarBuffer; - use arrow_array::LargeBinaryArray; - use arrow_array::builder::{Int32Builder, ListBuilder}; + use arrow_array::builder::{Int32Builder, LargeBinaryBuilder, ListBuilder}; + use arrow_array::{Array, LargeBinaryArray, RecordBatch}; + use arrow_schema::{Field, Schema}; use bytes::Bytes; use lance_core::Result; use lance_core::cache::{CacheCodecImpl, CacheEntryReader, CacheEntryWriter}; + use crate::cache_pb::{CompressedPostingHeader, PostingTailCodec as PbPostingTailCodec}; + + use super::super::impact::{ImpactSkipData, ImpactSkipDataBuilder}; use super::super::index::{ - CompressedPositionStorage, CompressedPostingList, PlainPostingList, PositionStreamCodec, - Positions, PostingList, PostingListGroup, PostingTailCodec, SharedPositionStream, + CompressedPositionStorage, CompressedPostingList, IMPACT_COL, POSTING_BLOCK_SIZE_KEY, + POSTING_COL, PlainPostingList, PositionStreamCodec, Positions, PostingList, + PostingListGroup, PostingTailCodec, SharedPositionStream, }; + use super::super::tokenizer::LEGACY_BLOCK_SIZE; fn legacy_positions(rows: &[&[i32]]) -> arrow_array::ListArray { let mut builder = ListBuilder::new(Int32Builder::new()); @@ -432,6 +562,73 @@ mod tests { builder.finish() } + fn packed_batch(postings: &[Vec>], block_size: Option) -> RecordBatch { + let mut builder = ListBuilder::new(LargeBinaryBuilder::new()); + for posting in postings { + for block in posting { + builder.values().append_value(block); + } + builder.append(true); + } + let postings = builder.finish(); + let fields = vec![Field::new(POSTING_COL, postings.data_type().clone(), false)]; + let schema = Arc::new(match block_size { + Some(block_size) => Schema::new_with_metadata( + fields, + HashMap::from([(POSTING_BLOCK_SIZE_KEY.to_owned(), block_size.to_string())]), + ), + None => Schema::new(fields), + }); + RecordBatch::try_new(schema, vec![Arc::new(postings)]).unwrap() + } + + fn packed_group( + postings: &[Vec>], + posting_tail_codec: PostingTailCodec, + block_size: Option, + ) -> PostingListGroup { + PostingListGroup::new_packed(packed_batch(postings, block_size), posting_tail_codec) + .unwrap() + } + + fn packed_group_with_impacts( + postings: &[Vec>], + impacts: &[ImpactSkipData], + posting_tail_codec: PostingTailCodec, + block_size: usize, + ) -> PostingListGroup { + assert_eq!(postings.len(), impacts.len()); + let posting_batch = packed_batch(postings, Some(block_size)); + let mut impacts_builder = ListBuilder::new(LargeBinaryBuilder::new()); + for impacts in impacts { + for entry_idx in 0..impacts.entries().len() { + impacts_builder + .values() + .append_value(impacts.entries().value(entry_idx)); + } + impacts_builder.append(true); + } + let impacts = impacts_builder.finish(); + let fields = vec![ + Field::new( + POSTING_COL, + posting_batch.column(0).data_type().clone(), + false, + ), + Field::new(IMPACT_COL, impacts.data_type().clone(), false), + ]; + let schema = Arc::new(Schema::new_with_metadata( + fields, + posting_batch.schema_ref().metadata().clone(), + )); + let batch = RecordBatch::try_new( + schema, + vec![posting_batch.column(0).clone(), Arc::new(impacts)], + ) + .unwrap(); + PostingListGroup::new_packed(batch, posting_tail_codec).unwrap() + } + fn assert_plain_eq(a: &PlainPostingList, b: &PlainPostingList) { assert_eq!(a.row_ids.as_ref(), b.row_ids.as_ref()); assert_eq!(a.frequencies.as_ref(), b.frequencies.as_ref()); @@ -461,6 +658,20 @@ mod tests { } } + fn impact_skip_data(level0_len: usize, block_size: usize) -> ImpactSkipData { + let mut builder = ImpactSkipDataBuilder::with_capacity(level0_len, block_size); + for block_idx in 0..level0_len { + let doc_base = block_idx as u32 * 10; + builder + .append_block(&[ + (doc_base + 1, block_idx as u32 + 1, 10), + (doc_base + 9, block_idx as u32 + 2, 8), + ]) + .unwrap(); + } + builder.finish().unwrap() + } + /// Serialize a codec body (no envelope) into a standalone buffer. fn body_bytes(entry: &T) -> Bytes { let mut buf = Vec::new(); @@ -475,6 +686,34 @@ mod tests { T::deserialize(&mut r) } + fn from_body_version(data: &Bytes, version: u32) -> Result { + let mut r = CacheEntryReader::new(data, 0, version); + T::deserialize(&mut r) + } + + fn compressed_body_with_ipc_sections( + blocks: &RecordBatch, + impacts: Option<&RecordBatch>, + ) -> Bytes { + let mut buf = Vec::new(); + let mut w = CacheEntryWriter::new(&mut buf); + w.write_u8(super::POSTING_VARIANT_COMPRESSED).unwrap(); + w.write_header(&CompressedPostingHeader { + max_score: 1.0, + length: 1, + posting_tail_codec: PbPostingTailCodec::VarintDelta as i32, + block_size: 256, + has_impacts: impacts.is_some(), + ..Default::default() + }) + .unwrap(); + w.write_ipc(blocks).unwrap(); + if let Some(impacts) = impacts { + w.write_ipc(impacts).unwrap(); + } + Bytes::from(buf) + } + fn roundtrip_posting_list(entry: &PostingList) -> PostingList { from_body::(&body_bytes(entry)).unwrap() } @@ -532,14 +771,22 @@ mod tests { Some(&[1u8, 2, 3, 4, 5][..]), Some(&[6, 7, 8, 9, 10][..]), ]); - let posting = - CompressedPostingList::new(blocks, 3.5, 42, PostingTailCodec::VarintDelta, None); + let posting = CompressedPostingList::new( + blocks, + 3.5, + 42, + PostingTailCodec::VarintDelta, + 256, + None, + None, + ); let entry = PostingList::Compressed(posting.clone()); match roundtrip_posting_list(&entry) { PostingList::Compressed(restored) => { assert_eq!(restored.max_score, posting.max_score); assert_eq!(restored.length, posting.length); assert_eq!(restored.posting_tail_codec, posting.posting_tail_codec); + assert_eq!(restored.block_size, posting.block_size); assert_eq!(restored.blocks, posting.blocks); assert!(restored.positions.is_none()); } @@ -547,6 +794,73 @@ mod tests { } } + #[test] + fn compressed_posting_list_impacts_roundtrip() { + let blocks = LargeBinaryArray::from_opt_vec(vec![ + Some(&[1u8, 2, 3, 4, 5][..]), + Some(&[6, 7, 8, 9, 10][..]), + ]); + let impacts = impact_skip_data(blocks.len(), 256); + let posting = CompressedPostingList::new( + blocks, + 3.5, + 42, + PostingTailCodec::VarintDelta, + 256, + None, + Some(impacts.clone()), + ); + let entry = PostingList::Compressed(posting); + match roundtrip_posting_list(&entry) { + PostingList::Compressed(restored) => { + let restored = restored.impacts.expect("impacts should roundtrip"); + assert_eq!(restored.level0_len(), impacts.level0_len()); + assert_eq!(restored.level1_len(), impacts.level1_len()); + assert_eq!(restored.entries(), impacts.entries()); + } + PostingList::Plain(_) => panic!("expected Compressed variant"), + } + } + + #[test] + fn compressed_posting_list_missing_ipc_columns_returns_error() { + let empty = RecordBatch::new_empty(Arc::new(Schema::empty())); + assert!( + from_body::(&compressed_body_with_ipc_sections(&empty, None)).is_err() + ); + + let blocks = LargeBinaryArray::from_opt_vec(vec![Some(&[1_u8, 2, 3][..])]); + let schema = Arc::new(Schema::new(vec![Field::new( + super::BLOCKS_COLUMN, + blocks.data_type().clone(), + false, + )])); + let blocks = RecordBatch::try_new(schema, vec![Arc::new(blocks)]).unwrap(); + assert!( + from_body::(&compressed_body_with_ipc_sections(&blocks, Some(&empty))) + .is_err() + ); + } + + #[test] + fn compressed_posting_list_v1_cache_without_impacts_decodes() { + let posting = CompressedPostingList::new( + LargeBinaryArray::from_opt_vec(vec![Some(&[1u8, 2, 3][..])]), + 1.25, + 5, + PostingTailCodec::Fixed32, + crate::scalar::inverted::LEGACY_BLOCK_SIZE, + None, + None, + ); + let data = body_bytes(&PostingList::Compressed(posting)); + let restored = from_body_version::(&data, 1).unwrap(); + let PostingList::Compressed(restored) = restored else { + panic!("expected Compressed variant"); + }; + assert!(restored.impacts.is_none()); + } + #[test] fn compressed_posting_list_legacy_positions_roundtrip() { let blocks = LargeBinaryArray::from_opt_vec(vec![Some(&[1u8, 2, 3][..])]); @@ -555,9 +869,11 @@ mod tests { 1.25, 5, PostingTailCodec::Fixed32, + crate::scalar::inverted::LEGACY_BLOCK_SIZE, Some(CompressedPositionStorage::LegacyPerDoc(legacy_positions( &[&[0, 4, 8]], ))), + None, ); let entry = PostingList::Compressed(posting.clone()); match roundtrip_posting_list(&entry) { @@ -589,7 +905,9 @@ mod tests { 7.0, 3, PostingTailCodec::VarintDelta, + 256, Some(CompressedPositionStorage::SharedStream(stream)), + None, ); let entry = PostingList::Compressed(posting.clone()); match roundtrip_posting_list(&entry) { @@ -617,9 +935,11 @@ mod tests { 7.0, 3, PostingTailCodec::VarintDelta, + 256, Some(CompressedPositionStorage::SharedStream( expected_stream.clone(), )), + None, ); let serialized = body_bytes(&PostingList::Compressed(posting)); @@ -651,6 +971,8 @@ mod tests { 2.5, 7, PostingTailCodec::VarintDelta, + 256, + None, None, )); @@ -661,9 +983,11 @@ mod tests { ] { let group = PostingListGroup::new(members.clone()); let restored = from_body::(&body_bytes(&group)).unwrap(); - assert_eq!(restored.posting_lists.len(), members.len()); - for (a, b) in members.iter().zip(restored.posting_lists.iter()) { - match (a, b) { + assert!(!restored.is_packed()); + assert_eq!(restored.len(), members.len()); + for (index, a) in members.iter().enumerate() { + let b = restored.posting_list(index, None, None).unwrap().unwrap(); + match (a, &b) { (PostingList::Plain(x), PostingList::Plain(y)) => assert_plain_eq(x, y), (PostingList::Compressed(x), PostingList::Compressed(y)) => { assert_eq!(x.blocks, y.blocks); @@ -676,6 +1000,167 @@ mod tests { } } + #[test] + fn packed_posting_list_group_roundtrip_and_v1_fallback() { + let group = packed_group( + &[vec![vec![1, 2, 3], vec![4, 5]], vec![vec![7; 16 * 1024]]], + PostingTailCodec::VarintDelta, + Some(256), + ); + let restored = from_body::(&body_bytes(&group)).unwrap(); + assert!(restored.is_packed()); + assert_eq!(restored.len(), 2); + for slot in 0..2 { + let max_score = [1.5, 3.25][slot]; + let length = [3, 4096][slot]; + let expected = group + .posting_list(slot, Some(max_score), Some(length)) + .unwrap() + .unwrap(); + let actual = restored + .posting_list(slot, Some(max_score), Some(length)) + .unwrap() + .unwrap(); + let (PostingList::Compressed(expected), PostingList::Compressed(actual)) = + (expected, actual) + else { + panic!("expected compressed packed posting views"); + }; + assert_eq!(actual.blocks, expected.blocks); + assert_eq!(actual.max_score, expected.max_score); + assert_eq!(actual.length, expected.length); + assert_eq!(actual.posting_tail_codec, expected.posting_tail_codec); + assert_eq!(actual.block_size, 256); + } + + let legacy_packed = + packed_group(&[vec![vec![9, 8, 7]]], PostingTailCodec::VarintDelta, None); + let restored = from_body::(&body_bytes(&legacy_packed)).unwrap(); + let PostingList::Compressed(posting) = restored + .posting_list(0, Some(2.0), Some(3)) + .unwrap() + .unwrap() + else { + panic!("expected compressed legacy packed posting"); + }; + assert_eq!(posting.block_size, LEGACY_BLOCK_SIZE); + + let legacy_member = PostingList::Compressed(CompressedPostingList::new( + LargeBinaryArray::from_opt_vec(vec![Some(&[9u8, 8, 7][..])]), + 2.0, + 3, + PostingTailCodec::VarintDelta, + LEGACY_BLOCK_SIZE, + None, + None, + )); + let mut legacy_body = Vec::new(); + let mut writer = CacheEntryWriter::new(&mut legacy_body); + writer + .write_header(&crate::cache_pb::PostingListGroupHeader { count: 1 }) + .unwrap(); + legacy_member.serialize(&mut writer).unwrap(); + let legacy_body = Bytes::from(legacy_body); + let mut reader = CacheEntryReader::new(&legacy_body, 0, 1); + let restored = PostingListGroup::deserialize(&mut reader).unwrap(); + assert!(!restored.is_packed()); + assert_eq!(restored.len(), 1); + } + + #[test] + fn posting_list_group_impacted_compressed_members_roundtrip() { + let first = CompressedPostingList::new( + LargeBinaryArray::from_opt_vec(vec![Some(&[1u8, 2, 3][..]), Some(&[4u8, 5, 6][..])]), + 3.0, + 256, + PostingTailCodec::VarintDelta, + LEGACY_BLOCK_SIZE, + None, + Some(impact_skip_data(2, LEGACY_BLOCK_SIZE)), + ); + let second = CompressedPostingList::new( + LargeBinaryArray::from_opt_vec(vec![Some(&[7u8, 8, 9][..])]), + 5.0, + 128, + PostingTailCodec::Fixed32, + 256, + Some(CompressedPositionStorage::SharedStream( + SharedPositionStream::new( + PositionStreamCodec::PackedDelta, + vec![0u32, 12], + Bytes::from(vec![0xABu8; 32]), + ), + )), + Some(impact_skip_data(1, 256)), + ); + let members = vec![ + PostingList::Compressed(first.clone()), + PostingList::Compressed(second.clone()), + ]; + let group = PostingListGroup::new(members); + let restored = from_body::(&body_bytes(&group)).unwrap(); + assert!(!restored.is_packed()); + assert_eq!(restored.len(), 2); + + let expected = [&first, &second]; + for (slot, expected) in expected.iter().enumerate() { + let restored = restored.posting_list(slot, None, None).unwrap().unwrap(); + let PostingList::Compressed(restored) = restored else { + panic!("expected compressed member"); + }; + assert_eq!(restored.blocks, expected.blocks); + assert_eq!(restored.length, expected.length); + assert_eq!(restored.max_score, expected.max_score); + assert_eq!(restored.posting_tail_codec, expected.posting_tail_codec); + assert_eq!(restored.block_size, expected.block_size); + assert_eq!( + restored.impacts.as_ref().unwrap().entries(), + expected.impacts.as_ref().unwrap().entries() + ); + match (&expected.positions, &restored.positions) { + (Some(expected), Some(restored)) => { + assert_position_storage_eq(expected, restored); + } + (None, None) => {} + _ => panic!("position storage mismatch"), + } + } + } + + #[test] + fn packed_posting_list_group_impacts_roundtrip() { + let postings = vec![vec![vec![1, 2, 3], vec![4, 5, 6]], vec![vec![7, 8, 9]]]; + let expected_impacts = vec![impact_skip_data(2, 256), impact_skip_data(1, 256)]; + let group = packed_group_with_impacts( + &postings, + &expected_impacts, + PostingTailCodec::VarintDelta, + 256, + ); + + let restored = from_body::(&body_bytes(&group)).unwrap(); + assert!(restored.is_packed()); + assert_eq!(restored.len(), expected_impacts.len()); + for (slot, expected) in expected_impacts.iter().enumerate() { + let posting = restored + .posting_list(slot, Some(3.0), Some(256)) + .unwrap() + .unwrap(); + let PostingList::Compressed(posting) = posting else { + panic!("expected compressed packed posting"); + }; + let actual = posting.impacts.as_ref().expect("impacts should roundtrip"); + assert_eq!(actual.entries(), expected.entries()); + assert_eq!(actual.level0_len(), expected.level0_len()); + assert_eq!( + actual.level1_doc_up_to(0), + expected.level1_doc_up_to(0), + "impact entries should remain decodable with the packed block size", + ); + assert!(actual.level1_doc_up_to(0).is_some()); + } + } + #[test] fn positions_legacy_roundtrip() { let positions = Positions(CompressedPositionStorage::LegacyPerDoc(legacy_positions( @@ -717,26 +1202,130 @@ mod tests { use std::sync::Arc; use arrow_array::Array; - use lance_core::cache::CacheCodec; + use arrow_schema::DataType; + use lance_core::cache::{ + CacheCodec, CacheCodecImpl, CacheDecode, CacheEntryReader, CacheEntryWriter, + CacheMissReason, + }; + use lance_core::{Error, Result}; use prost::Message; + use super::super::{ + BLOCKS_COLUMN, GROUP_VARIANT_PACKED, POSTING_VARIANT_COMPRESSED, + posting_tail_codec_to_tag, + }; use super::*; - use crate::cache_pb::{CompressedPostingHeader, PostingTailCodec as PbPostingTailCodec}; + use crate::cache_pb::{ + CompressedPostingHeader, PostingListGroupHeader, PostingTailCodec as PbPostingTailCodec, + }; type ArcAny = Arc; + struct PostingListV2Codec(PostingList); + + impl CacheCodecImpl for PostingListV2Codec { + const TYPE_ID: &'static str = ::TYPE_ID; + const CURRENT_VERSION: u32 = 2; + + fn serialize(&self, w: &mut CacheEntryWriter<'_>) -> Result<()> { + self.0.serialize(w) + } + + fn deserialize(r: &mut CacheEntryReader<'_>) -> Result { + PostingList::deserialize(r).map(Self) + } + } + + struct PostingListGroupV3Codec(PostingListGroup); + + impl CacheCodecImpl for PostingListGroupV3Codec { + const TYPE_ID: &'static str = ::TYPE_ID; + const CURRENT_VERSION: u32 = 3; + + fn serialize(&self, w: &mut CacheEntryWriter<'_>) -> Result<()> { + self.0.serialize(w) + } + + fn deserialize(r: &mut CacheEntryReader<'_>) -> Result { + PostingListGroup::deserialize(r).map(Self) + } + } + + struct LegacyCompressedPostingV1 { + blocks: LargeBinaryArray, + } + + impl CacheCodecImpl for LegacyCompressedPostingV1 { + const TYPE_ID: &'static str = ::TYPE_ID; + const CURRENT_VERSION: u32 = 1; + + fn serialize(&self, w: &mut CacheEntryWriter<'_>) -> Result<()> { + w.write_u8(POSTING_VARIANT_COMPRESSED)?; + w.write_header(&CompressedPostingHeader { + max_score: 2.0, + length: 3, + posting_tail_codec: PbPostingTailCodec::VarintDelta as i32, + ..Default::default() + })?; + let schema = Arc::new(Schema::new(vec![Field::new( + BLOCKS_COLUMN, + DataType::LargeBinary, + false, + )])); + let batch = RecordBatch::try_new(schema, vec![Arc::new(self.blocks.clone())])?; + w.write_ipc(&batch) + } + + fn deserialize(_r: &mut CacheEntryReader<'_>) -> Result { + Err(Error::io( + "LegacyCompressedPostingV1 is a writer-only test codec".to_string(), + )) + } + } + + struct LegacyPackedGroupV2 { + batch: RecordBatch, + posting_tail_codec: PostingTailCodec, + } + + impl CacheCodecImpl for LegacyPackedGroupV2 { + const TYPE_ID: &'static str = ::TYPE_ID; + const CURRENT_VERSION: u32 = 2; + + fn serialize(&self, w: &mut CacheEntryWriter<'_>) -> Result<()> { + w.write_u8(GROUP_VARIANT_PACKED)?; + let count = u32::try_from(self.batch.num_rows()) + .map_err(|_| Error::io("legacy packed group is too large".to_string()))?; + w.write_header(&PostingListGroupHeader { count })?; + w.write_u8(posting_tail_codec_to_tag(self.posting_tail_codec))?; + w.write_ipc(&self.batch) + } + + fn deserialize(_r: &mut CacheEntryReader<'_>) -> Result { + Err(Error::io( + "LegacyPackedGroupV2 is a writer-only test codec".to_string(), + )) + } + } + fn codec() -> CacheCodec { CacheCodec::from_impl::() } - /// Serialize an entry through the full codec (envelope + body). - fn serialize_entry(entry: PostingList) -> Vec { + fn serialize_typed_entry(entry: T) -> Vec { let any: ArcAny = Arc::new(entry); let mut buf = Vec::new(); - codec().serialize(&any, &mut buf).unwrap(); + CacheCodec::from_impl::() + .serialize(&any, &mut buf) + .unwrap(); buf } + /// Serialize an entry through the full codec (envelope + body). + fn serialize_entry(entry: PostingList) -> Vec { + serialize_typed_entry(entry) + } + /// A `Bytes` whose base address is 64-byte aligned, modelling a backend /// that reads cache entries into an aligned buffer. fn aligned_bytes(payload: &[u8]) -> Bytes { @@ -760,7 +1349,9 @@ mod tests { 7.0, 3, PostingTailCodec::VarintDelta, + 256, Some(CompressedPositionStorage::SharedStream(stream)), + None, )) } @@ -795,25 +1386,17 @@ mod tests { assert!(points_in(stream.bytes().as_ptr() as usize)); } - /// Every member of a `PostingListGroup` must also decode zero-copy. The - /// group writes its members inline so each member's IPC sections stay - /// 64-byte aligned within the entry; embedding members in per-member - /// sub-buffers would land them at arbitrary offsets and force a - /// realigning memcpy on load. + /// A packed group's single IPC batch and all posting views decoded from + /// it must borrow the cache entry's aligned input buffer. #[test] - fn group_member_sections_are_zero_copy_through_envelope() { - let make_member = |fill: u8| { - let blocks = - LargeBinaryArray::from_opt_vec(vec![Some(&[fill; 48][..]), Some(&[fill; 48])]); - PostingList::Compressed(CompressedPostingList::new( - blocks, - 7.0, - 3, - PostingTailCodec::VarintDelta, - None, - )) - }; - let group = PostingListGroup::new(vec![make_member(9), make_member(1)]); + fn packed_group_sections_are_zero_copy_through_envelope() { + let postings = vec![ + vec![vec![9; 48], vec![9; 48]], + vec![vec![1; 48], vec![1; 48]], + ]; + let impacts = vec![impact_skip_data(2, 256), impact_skip_data(2, 256)]; + let group = + packed_group_with_impacts(&postings, &impacts, PostingTailCodec::VarintDelta, 256); let group_codec = CacheCodec::from_impl::(); let any: ArcAny = Arc::new(group); @@ -828,8 +1411,13 @@ mod tests { let end = base + serialized.len(); let points_in = |ptr: usize| ptr >= base && ptr < end; - assert_eq!(restored.posting_lists.len(), 2); - for member in &restored.posting_lists { + assert!(restored.is_packed()); + assert_eq!(restored.len(), 2); + for slot in 0..restored.len() { + let member = restored + .posting_list(slot, Some(7.0), Some(3)) + .unwrap() + .unwrap(); let PostingList::Compressed(member) = member else { panic!("expected Compressed member"); }; @@ -837,7 +1425,17 @@ mod tests { assert!( points_in(buf.as_ptr() as usize), "group member blocks buffer was realigned out of the input — \ - misaligned IPC section", + misaligned IPC section", + ); + } + let impacts = member + .impacts + .as_ref() + .expect("packed impacts should decode"); + for buf in impacts.entries().to_data().buffers() { + assert!( + points_in(buf.as_ptr() as usize), + "group member impact buffer was realigned out of the input", ); } } @@ -915,6 +1513,110 @@ mod tests { assert!(codec().deserialize(&Bytes::from(buf)).hit().is_none()); } + #[test] + fn old_codecs_reject_new_impact_envelopes_as_version_too_new() { + let posting = Bytes::from(serialize_entry(compressed_with_shared_positions())); + match CacheCodec::from_impl::().deserialize(&posting) { + CacheDecode::Miss(reason) => { + assert_eq!(reason, CacheMissReason::VersionTooNew) + } + CacheDecode::Hit(_) => panic!("v2 PostingList codec accepted a v3 envelope"), + } + + let group = packed_group( + &[vec![vec![1, 2, 3]], vec![vec![4, 5, 6]]], + PostingTailCodec::VarintDelta, + Some(256), + ); + let group = Bytes::from(serialize_typed_entry(group)); + match CacheCodec::from_impl::().deserialize(&group) { + CacheDecode::Miss(reason) => { + assert_eq!(reason, CacheMissReason::VersionTooNew) + } + CacheDecode::Hit(_) => { + panic!("v3 PostingListGroup codec accepted a v4 envelope") + } + } + } + + #[test] + fn current_codecs_read_previous_main_versions() { + let previous_posting = PostingListV2Codec(compressed_with_shared_positions()); + let previous_posting = Bytes::from(serialize_typed_entry(previous_posting)); + let restored = codec().deserialize(&previous_posting).hit().unwrap(); + let restored = restored.downcast::().unwrap(); + let PostingList::Compressed(restored) = restored.as_ref() else { + panic!("expected compressed posting"); + }; + assert_eq!(restored.block_size, 256); + assert!(restored.impacts.is_none()); + + let previous_group = PostingListGroupV3Codec(packed_group( + &[vec![vec![1, 2, 3]], vec![vec![4, 5, 6]]], + PostingTailCodec::VarintDelta, + Some(256), + )); + let previous_group = Bytes::from(serialize_typed_entry(previous_group)); + let restored = CacheCodec::from_impl::() + .deserialize(&previous_group) + .hit() + .unwrap() + .downcast::() + .unwrap(); + assert!(restored.is_packed()); + assert_eq!(restored.len(), 2); + let PostingList::Compressed(restored) = restored + .posting_list(0, Some(2.0), Some(3)) + .unwrap() + .unwrap() + else { + panic!("expected compressed packed posting"); + }; + assert_eq!(restored.block_size, 256); + assert!(restored.impacts.is_none()); + } + + #[test] + fn current_codecs_read_legacy_payloads_without_block_size() { + let legacy_posting = LegacyCompressedPostingV1 { + blocks: LargeBinaryArray::from_opt_vec(vec![Some(&[9u8, 8, 7][..])]), + }; + let legacy_posting = Bytes::from(serialize_typed_entry(legacy_posting)); + let restored = codec().deserialize(&legacy_posting).hit().unwrap(); + let restored = restored.downcast::().unwrap(); + let PostingList::Compressed(restored) = restored.as_ref() else { + panic!("expected a compressed legacy posting"); + }; + assert_eq!(restored.block_size, LEGACY_BLOCK_SIZE); + + let legacy_batch = packed_batch(&[vec![vec![1, 2, 3]], vec![vec![4, 5, 6]]], None); + assert!( + !legacy_batch + .schema_ref() + .metadata() + .contains_key(POSTING_BLOCK_SIZE_KEY) + ); + let legacy_group = LegacyPackedGroupV2 { + batch: legacy_batch, + posting_tail_codec: PostingTailCodec::VarintDelta, + }; + let legacy_group = Bytes::from(serialize_typed_entry(legacy_group)); + let restored = CacheCodec::from_impl::() + .deserialize(&legacy_group) + .hit() + .unwrap() + .downcast::() + .unwrap(); + let PostingList::Compressed(restored) = restored + .posting_list(0, Some(2.0), Some(3)) + .unwrap() + .unwrap() + else { + panic!("expected a compressed legacy packed posting"); + }; + assert_eq!(restored.block_size, LEGACY_BLOCK_SIZE); + } + /// A pre-stabilization blob (no magic) self-heals to a miss. #[test] fn pre_stabilization_blob_is_miss() { diff --git a/rust/lance-index/src/scalar/inverted/compound.rs b/rust/lance-index/src/scalar/inverted/compound.rs new file mode 100644 index 00000000000..453c14db682 --- /dev/null +++ b/rust/lance-index/src/scalar/inverted/compound.rs @@ -0,0 +1,3411 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright The Lance Authors + +use std::cmp::Ordering; +use std::collections::{BinaryHeap, HashSet}; +use std::sync::Arc; +use std::sync::atomic::{AtomicU32, Ordering as AtomicOrdering}; + +use futures::{StreamExt, TryStreamExt, stream}; +use lance_core::utils::tokio::{get_num_compute_intensive_cpus, spawn_cpu}; +use lance_core::{Error, Result}; +use lance_select::RowAddrMask; +use lance_tokenizer::{SimpleTokenizer, TextAnalyzer}; + +use super::{ + InvertedIndex, build_global_bm25_scorer, + document_tokenizer::{DocType, JsonTokenizer, LanceTokenizer}, + documents::{DocId, DocLengths, DocVisibility, PartitionDocuments, ResidentAddressProjection}, + index::{DocSet, InvertedPartition}, + query::{ + FtsQuery, FtsSearchParams, MatchQuery, Operator, PhraseQuery, Tokens, collect_query_tokens, + }, + scorer::MemBM25Scorer, + tokenizer::document_tokenizer::TextTokenizer, + wand::{ + FLAT_SEARCH_PERCENT_THRESHOLD, LegacyWandDocuments, ModernWandDocuments, PostingIterator, + WandCursor, WandDocuments, + }, +}; +use crate::{metrics::MetricsCollector, prefilter::PreFilter}; + +const DEFAULT_BLOCK_SIZE: usize = 128; +const SCORE_FLOOR_RESOLUTION_BATCH_SIZE: usize = DEFAULT_BLOCK_SIZE; + +/// One exact FTS result in a compound collector's candidate domain. +#[derive(Debug, Clone, Copy, PartialEq)] +pub(super) struct ScoredRow { + pub row_id: K, + pub score: f32, +} + +#[cfg(test)] +impl ScoredRow { + pub(super) fn new(row_id: u64, score: f32) -> Result { + if !score.is_finite() { + return Err(Error::invalid_input(format!( + "FTS score for row_id={row_id} must be finite, got {score}" + ))); + } + Ok(Self { row_id, score }) + } +} + +/// Conservative score bounds for a document range. +/// +/// The lower bound is needed by signed compositions such as [`BoostScorer`]. +/// Arithmetic widens both sides by one representable `f32` so nested +/// operations cannot round an upper bound below an exact score. +#[derive(Debug, Clone, Copy, PartialEq)] +pub(super) struct ScoreBounds { + lower: f32, + upper: f32, +} + +impl ScoreBounds { + const ZERO: Self = Self { + lower: 0.0, + upper: 0.0, + }; + const UNBOUNDED: Self = Self { + lower: f32::NEG_INFINITY, + upper: f32::INFINITY, + }; + + #[cfg(test)] + fn point(score: f32) -> Result { + if !score.is_finite() { + return Err(Error::invalid_input(format!( + "FTS score bounds require a finite score, got {score}" + ))); + } + Ok(Self { + lower: score, + upper: score, + }) + } + + fn scale_non_negative(self, factor: f32) -> Self { + debug_assert!(factor.is_finite() && factor >= 0.0); + if factor == 0.0 { + return Self::ZERO; + } + if !self.lower.is_finite() || !self.upper.is_finite() { + return Self::UNBOUNDED; + } + Self { + lower: next_down(self.lower * factor), + upper: next_up(self.upper * factor), + } + } + + fn include_zero(self) -> Self { + Self { + lower: self.lower.min(0.0), + upper: self.upper.max(0.0), + } + } + + fn add(self, other: Self) -> Self { + if !self.lower.is_finite() + || !self.upper.is_finite() + || !other.lower.is_finite() + || !other.upper.is_finite() + { + return Self::UNBOUNDED; + } + Self { + lower: next_down(self.lower + other.lower), + upper: next_up(self.upper + other.upper), + } + } + + fn subtract_scaled(self, other: Self, factor: f32) -> Self { + let penalty = other.scale_non_negative(factor); + if !self.lower.is_finite() + || !self.upper.is_finite() + || !penalty.lower.is_finite() + || !penalty.upper.is_finite() + { + return Self::UNBOUNDED; + } + Self { + lower: next_down(self.lower - penalty.upper), + upper: next_up(self.upper - penalty.lower), + } + } +} + +fn next_up(value: f32) -> f32 { + if !value.is_finite() { + return value; + } + if value == 0.0 { + return f32::from_bits(1); + } + let bits = value.to_bits(); + if value > 0.0 { + f32::from_bits(bits + 1) + } else { + f32::from_bits(bits - 1) + } +} + +fn next_down(value: f32) -> f32 { + if !value.is_finite() { + return value; + } + if value == 0.0 { + return f32::from_bits((1_u32 << 31) | 1); + } + let bits = value.to_bits(); + if value > 0.0 { + f32::from_bits(bits - 1) + } else { + f32::from_bits(bits + 1) + } +} + +fn checked_score(score: f32, context: &str) -> Result { + if score.is_finite() { + Ok(score) + } else { + Err(Error::invalid_input(format!( + "{context} produced a non-finite FTS score: {score}" + ))) + } +} + +/// Internal document-at-a-time scorer protocol for compound FTS. +/// +/// Implementations iterate matching partition-local document ids in ascending +/// order and expose the corresponding candidate key separately. A collector +/// may shallow-advance independently of the exact iterator, inspect a +/// conservative range bound, and monotonically raise the competitive score. +/// `matches` is the optional two-phase confirmation hook: cheap approximations +/// return a candidate from `next` / `advance` and defer expensive checks such +/// as phrase positions until confirmation. +pub(super) trait ComposableScorer: Send { + fn doc(&self) -> Option; + fn document_key(&self) -> Option { + self.doc() + } + fn next(&mut self) -> Result>; + fn advance(&mut self, target: u64) -> Result>; + fn cost(&self) -> usize; + fn score(&mut self) -> Result; + fn advance_shallow(&mut self, target: u64) -> Result; + fn score_bounds(&mut self, up_to: u64) -> Result; + fn set_min_competitive_score(&mut self, min_score: f32) -> Result<()>; + + fn matches(&mut self) -> Result { + Ok(true) + } + + /// Estimated relative cost of [`Self::matches`], stable for this scorer's + /// lifetime. `None` means no ordering hint, not that confirmation may be skipped. + fn match_cost(&self) -> Option { + None + } + + fn scores_non_negative(&self) -> bool { + false + } +} + +type BoxScorer<'a> = Box; + +#[derive(Debug, Clone)] +enum CompoundScorerPlan { + Leaf { + index: usize, + boost: f32, + }, + Boost { + positive: Box, + negative: Box, + negative_boost: f32, + }, + MultiMatch(Vec), + Boolean { + should: Vec, + must: Vec, + must_not: Vec, + }, +} + +impl CompoundScorerPlan { + fn from_query(query: &FtsQuery, num_leaves: &mut usize) -> Result { + match query { + FtsQuery::Match(query) => { + let index = *num_leaves; + *num_leaves += 1; + Ok(Self::Leaf { + index, + boost: query.boost, + }) + } + FtsQuery::Phrase(_) => { + let index = *num_leaves; + *num_leaves += 1; + Ok(Self::Leaf { index, boost: 1.0 }) + } + FtsQuery::Boost(query) => Ok(Self::Boost { + positive: Box::new(Self::from_query(&query.positive, num_leaves)?), + negative: Box::new(Self::from_query(&query.negative, num_leaves)?), + negative_boost: query.negative_boost, + }), + FtsQuery::MultiMatch(query) => Ok(Self::MultiMatch( + query + .match_queries + .iter() + .map(|query| Self::from_query(&FtsQuery::Match(query.clone()), num_leaves)) + .collect::>>()?, + )), + FtsQuery::Boolean(query) => Ok(Self::Boolean { + should: query + .should + .iter() + .map(|query| Self::from_query(query, num_leaves)) + .collect::>>()?, + must: query + .must + .iter() + .map(|query| Self::from_query(query, num_leaves)) + .collect::>>()?, + must_not: query + .must_not + .iter() + .map(|query| Self::from_query(query, num_leaves)) + .collect::>>()?, + }), + } + } + + fn build<'a>(&self, leaves: &mut [Option>]) -> Result> { + match self { + Self::Leaf { index, boost } => { + let leaf = leaves + .get_mut(*index) + .and_then(Option::take) + .ok_or_else(|| { + Error::internal(format!( + "compound FTS scorer references missing leaf index {index}" + )) + })?; + Ok(Box::new(ScaleScorer::try_new(leaf, *boost)?)) + } + Self::Boost { + positive, + negative, + negative_boost, + } => Ok(Box::new(BoostScorer::try_new( + positive.build(leaves)?, + negative.build(leaves)?, + *negative_boost, + )?)), + Self::MultiMatch(children) => Ok(Box::new(DisjunctionScorer::try_new( + children + .iter() + .map(|child| child.build(leaves)) + .collect::>>()?, + DisjunctionScore::Max, + )?)), + Self::Boolean { + should, + must, + must_not, + } => Ok(Box::new(BooleanScorer::try_new( + should + .iter() + .map(|child| child.build(leaves)) + .collect::>>()?, + must.iter() + .map(|child| child.build(leaves)) + .collect::>>()?, + must_not + .iter() + .map(|child| child.build(leaves)) + .collect::>>()?, + )?)), + } + } +} + +impl ComposableScorer for WandCursor<'_, D> { + fn doc(&self) -> Option { + self.doc() + } + + fn document_key(&self) -> Option { + self.document_key() + } + + fn next(&mut self) -> Result> { + self.next() + } + + fn advance(&mut self, target: u64) -> Result> { + self.advance(target) + } + + fn cost(&self) -> usize { + self.cost() + } + + fn score(&mut self) -> Result { + self.current_score() + } + + fn advance_shallow(&mut self, target: u64) -> Result { + self.advance_shallow(target) + } + + fn score_bounds(&mut self, up_to: u64) -> Result { + Ok(ScoreBounds { + lower: 0.0, + upper: self.score_upper_bound(up_to)?, + }) + } + + fn set_min_competitive_score(&mut self, min_score: f32) -> Result<()> { + self.set_min_competitive_score(min_score) + } + + fn matches(&mut self) -> Result { + self.matches() + } + + fn match_cost(&self) -> Option { + self.match_cost() + } + + fn scores_non_negative(&self) -> bool { + true + } +} + +#[derive(Debug, Clone, Copy)] +#[cfg(test)] +struct ShallowRange { + target: u64, + up_to: u64, + start: usize, + end: usize, +} + +/// Exact in-memory scorer used to unit-test compound nodes and the collector. +#[cfg(test)] +struct MaterializedScorer { + rows: Vec, + block_size: usize, + index: Option, + shallow: Option, + min_competitive_score: f32, + scores_non_negative: bool, +} + +#[cfg(test)] +impl MaterializedScorer { + fn try_new(mut rows: Vec) -> Result { + rows.sort_unstable_by_key(|row| row.row_id); + for pair in rows.windows(2) { + if pair[0].row_id == pair[1].row_id { + return Err(Error::internal(format!( + "FTS leaf scorer produced duplicate row_id={}", + pair[0].row_id + ))); + } + } + let scores_non_negative = rows.iter().all(|row| row.score >= 0.0); + Ok(Self { + rows, + block_size: DEFAULT_BLOCK_SIZE, + index: None, + shallow: None, + min_competitive_score: f32::NEG_INFINITY, + scores_non_negative, + }) + } + + #[cfg(test)] + fn with_block_size(mut self, block_size: usize) -> Self { + assert!(block_size > 0); + self.block_size = block_size; + self + } + + fn block_bounds(&self, start: usize, end: usize) -> Result { + let Some(first) = self.rows.get(start) else { + return Ok(ScoreBounds::ZERO); + }; + let mut bounds = ScoreBounds::point(first.score)?; + for row in &self.rows[start + 1..end] { + bounds.lower = bounds.lower.min(row.score); + bounds.upper = bounds.upper.max(row.score); + } + Ok(bounds) + } + + fn position_at(&mut self, mut index: usize) -> Result> { + while index < self.rows.len() { + let block_start = (index / self.block_size) * self.block_size; + let block_end = (block_start + self.block_size).min(self.rows.len()); + if self.block_bounds(block_start, block_end)?.upper < self.min_competitive_score { + index = block_end; + continue; + } + self.index = Some(index); + self.shallow = None; + return Ok(Some(self.rows[index].row_id)); + } + self.index = None; + self.shallow = None; + Ok(None) + } +} + +#[cfg(test)] +impl ComposableScorer for MaterializedScorer { + fn doc(&self) -> Option { + self.index.map(|index| self.rows[index].row_id) + } + + fn next(&mut self) -> Result> { + self.position_at(self.index.map_or(0, |index| index + 1)) + } + + fn advance(&mut self, target: u64) -> Result> { + if self.doc().is_some_and(|doc| doc >= target) { + return Ok(self.doc()); + } + let start = self.index.map_or(0, |index| index + 1); + let offset = self.rows[start..].partition_point(|row| row.row_id < target); + self.position_at(start + offset) + } + + fn cost(&self) -> usize { + self.rows.len() + } + + fn score(&mut self) -> Result { + self.index + .map(|index| self.rows[index].score) + .ok_or_else(|| Error::internal("FTS scorer is not positioned on a document")) + } + + fn advance_shallow(&mut self, target: u64) -> Result { + let start = self.rows.partition_point(|row| row.row_id < target); + if start == self.rows.len() { + self.shallow = Some(ShallowRange { + target, + up_to: u64::MAX, + start, + end: start, + }); + return Ok(u64::MAX); + } + let block_start = (start / self.block_size) * self.block_size; + let end = (block_start + self.block_size).min(self.rows.len()); + let up_to = self + .rows + .get(end) + .map(|next| next.row_id.saturating_sub(1)) + .unwrap_or(u64::MAX); + self.shallow = Some(ShallowRange { + target, + up_to, + start, + end, + }); + Ok(up_to) + } + + fn score_bounds(&mut self, up_to: u64) -> Result { + let shallow = self.shallow.ok_or_else(|| { + Error::internal("score_bounds requires advance_shallow on the FTS scorer") + })?; + if up_to < shallow.target || up_to > shallow.up_to { + return Err(Error::internal(format!( + "FTS score bound up_to={up_to} is outside shallow range [{}, {}]", + shallow.target, shallow.up_to + ))); + } + let end = shallow.start + + self.rows[shallow.start..shallow.end].partition_point(|row| row.row_id <= up_to); + self.block_bounds(shallow.start, end) + } + + fn set_min_competitive_score(&mut self, min_score: f32) -> Result<()> { + if min_score.is_nan() { + return Err(Error::invalid_input( + "minimum competitive FTS score cannot be NaN", + )); + } + if min_score > self.min_competitive_score { + self.min_competitive_score = min_score; + } + Ok(()) + } + + fn scores_non_negative(&self) -> bool { + self.scores_non_negative + } +} + +/// Monotonic score-only floor shared by partition-local top-k collectors. +/// +/// Equal-score candidates are never pruned because final ordering also uses +/// row id. The score-only floor is therefore a safe lower bound even when +/// partitions encounter ties in different orders. +#[derive(Debug)] +pub(super) struct CompetitiveScore { + bits: AtomicU32, +} + +impl Default for CompetitiveScore { + fn default() -> Self { + Self { + bits: AtomicU32::new(f32::NEG_INFINITY.to_bits()), + } + } +} + +impl CompetitiveScore { + fn get(&self) -> f32 { + f32::from_bits(self.bits.load(AtomicOrdering::Relaxed)) + } + + fn raise(&self, score: f32) { + debug_assert!(!score.is_nan()); + let mut current = self.bits.load(AtomicOrdering::Relaxed); + while score > f32::from_bits(current) { + match self.bits.compare_exchange_weak( + current, + score.to_bits(), + AtomicOrdering::Relaxed, + AtomicOrdering::Relaxed, + ) { + Ok(_) => break, + Err(actual) => current = actual, + } + } + } +} + +#[derive(Debug, Clone, Copy)] +struct HeapRow(ScoredRow); + +impl PartialEq for HeapRow { + fn eq(&self, other: &Self) -> bool { + self.0.row_id == other.0.row_id && self.0.score.to_bits() == other.0.score.to_bits() + } +} + +impl Eq for HeapRow {} + +impl PartialOrd for HeapRow { + fn partial_cmp(&self, other: &Self) -> Option { + Some(self.cmp(other)) + } +} + +impl Ord for HeapRow { + fn cmp(&self, other: &Self) -> Ordering { + // The worst result is the heap maximum: lower score, then higher row id. + other + .0 + .score + .total_cmp(&self.0.score) + .then_with(|| self.0.row_id.cmp(&other.0.row_id)) + } +} + +fn compare_scored_rows(left: &ScoredRow, right: &ScoredRow) -> Ordering { + right + .score + .total_cmp(&left.score) + .then_with(|| left.row_id.cmp(&right.row_id)) +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(super) enum CollectionStatus { + Complete, + ScoreFloorOverflow, +} + +#[derive(Debug, Clone, Copy)] +enum TieHandling { + ResolveByKey, + RetainScoreFloor { max_buffered: usize }, +} + +/// The sole owner of top-k state for a compound scorer tree. +/// +/// Resolved document keys use the normal `(score DESC, row_id ASC)` ordering +/// and keep exactly `limit` rows. An unresolved partition temporarily retains +/// its kth-score floor, but stops once the bounded resolution buffer fills. +pub(super) struct TopKCollector { + limit: usize, + heap: BinaryHeap>, + competitive_score: Arc, + tie_handling: TieHandling, + peak_buffered: usize, +} + +impl TopKCollector { + pub(super) fn new(limit: usize) -> Self { + Self::with_competitive_score(limit, Arc::new(CompetitiveScore::default())) + } + + pub(super) fn with_competitive_score( + limit: usize, + competitive_score: Arc, + ) -> Self { + Self::with_tie_handling(limit, competitive_score, TieHandling::ResolveByKey) + } + + fn retaining_score_floor( + limit: usize, + competitive_score: Arc, + max_buffered: usize, + ) -> Self { + debug_assert!(max_buffered >= limit); + Self::with_tie_handling( + limit, + competitive_score, + TieHandling::RetainScoreFloor { max_buffered }, + ) + } + + fn with_tie_handling( + limit: usize, + competitive_score: Arc, + tie_handling: TieHandling, + ) -> Self { + Self { + limit, + heap: BinaryHeap::with_capacity(limit.min(DEFAULT_BLOCK_SIZE)), + competitive_score, + tie_handling, + peak_buffered: 0, + } + } + + fn insert(&mut self, row: ScoredRow) -> CollectionStatus { + if self.limit == 0 { + return CollectionStatus::Complete; + } + if self.heap.len() < self.limit { + self.heap.push(HeapRow(row)); + } else { + let worst = self.heap.peek().expect("a full top-k heap is non-empty").0; + match row.score.total_cmp(&worst.score) { + Ordering::Less => {} + Ordering::Equal => match self.tie_handling { + TieHandling::ResolveByKey => { + if row.row_id < worst.row_id { + self.heap.pop(); + self.heap.push(HeapRow(row)); + } + } + TieHandling::RetainScoreFloor { max_buffered } => { + if self.heap.len() >= max_buffered { + self.raise_competitive_score(); + return CollectionStatus::ScoreFloorOverflow; + } + self.heap.push(HeapRow(row)); + } + }, + Ordering::Greater => { + match self.tie_handling { + TieHandling::ResolveByKey => { + self.heap.pop(); + self.heap.push(HeapRow(row)); + } + TieHandling::RetainScoreFloor { max_buffered } => { + self.heap.push(HeapRow(row)); + self.prune_obsolete_score_floors(); + if self.heap.len() > max_buffered { + // This collector is discarded and the partition is + // retried with resolved keys. Keep its observable + // working set within the advertised bound meanwhile. + self.heap.pop(); + self.raise_competitive_score(); + return CollectionStatus::ScoreFloorOverflow; + } + } + } + } + } + } + self.peak_buffered = self.peak_buffered.max(self.heap.len()); + self.raise_competitive_score(); + CollectionStatus::Complete + } + + fn raise_competitive_score(&self) { + if self.heap.len() >= self.limit + && let Some(worst) = self.heap.peek() + { + self.competitive_score.raise(worst.0.score); + } + } + + fn prune_obsolete_score_floors(&mut self) { + while self.heap.len() > self.limit { + let floor = self + .heap + .peek() + .expect("a non-empty top-k heap has a score floor") + .0 + .score; + let floor_count = self + .heap + .iter() + .filter(|row| row.0.score.total_cmp(&floor) == Ordering::Equal) + .count(); + if self.heap.len() - floor_count < self.limit { + break; + } + self.heap + .retain(|row| row.0.score.total_cmp(&floor) != Ordering::Equal); + } + } + + pub(super) fn collect_mapped( + &mut self, + scorer: &mut dyn ComposableScorer, + mut map_document: impl FnMut(u64) -> Result, + ) -> Result { + if self.limit == 0 { + return Ok(CollectionStatus::Complete); + } + let capacity_limit = match self.tie_handling { + TieHandling::ResolveByKey => self.limit, + TieHandling::RetainScoreFloor { max_buffered } => max_buffered, + }; + let expected = scorer.cost().min(capacity_limit); + self.heap + .reserve(expected.saturating_sub(self.heap.capacity())); + + scorer.set_min_competitive_score(self.competitive_score.get())?; + let mut doc = scorer.next()?; + while let Some(doc_id) = doc { + let min_score = self.competitive_score.get(); + scorer.set_min_competitive_score(min_score)?; + let up_to = scorer.advance_shallow(doc_id)?; + let bounds = scorer.score_bounds(up_to)?; + if bounds.upper < min_score { + doc = if up_to == u64::MAX { + None + } else { + scorer.advance(up_to + 1)? + }; + continue; + } + + if let Some(match_cost) = scorer.match_cost() + && (!match_cost.is_finite() || match_cost < 0.0) + { + return Err(Error::internal(format!( + "FTS scorer reported invalid two-phase match cost: {match_cost}" + ))); + } + if scorer.matches()? { + let score = checked_score(scorer.score()?, "compound scorer")?; + // A shared partition floor is already known to be globally + // competitive. Scores strictly below it cannot enter final top-k. + if score >= self.competitive_score.get() { + let document_key = scorer.document_key().ok_or_else(|| { + Error::internal( + "compound FTS scorer did not expose its current document key", + ) + })?; + let status = self.insert(ScoredRow { + row_id: map_document(document_key)?, + score, + }); + if status == CollectionStatus::ScoreFloorOverflow { + return Ok(status); + } + } + } + doc = scorer.next()?; + } + + Ok(CollectionStatus::Complete) + } + + fn into_candidates(self) -> Vec> { + let mut rows = self.heap.into_iter().map(|row| row.0).collect::>(); + rows.sort_unstable_by(compare_scored_rows); + rows + } + + fn into_rows(self) -> Vec> { + let limit = self.limit; + let mut rows = self.into_candidates(); + rows.truncate(limit); + rows + } +} + +impl TopKCollector { + #[cfg(test)] + fn collect(mut self, scorer: &mut dyn ComposableScorer) -> Result> { + self.collect_mapped(scorer, Ok)?; + Ok(self.into_rows()) + } +} + +#[derive(Debug, Clone, Copy)] +pub(super) enum DisjunctionScore { + Sum, + Max, +} + +struct EmptyScorer; + +impl ComposableScorer for EmptyScorer { + fn doc(&self) -> Option { + None + } + + fn next(&mut self) -> Result> { + Ok(None) + } + + fn advance(&mut self, _target: u64) -> Result> { + Ok(None) + } + + fn cost(&self) -> usize { + 0 + } + + fn score(&mut self) -> Result { + Err(Error::internal( + "score requested from an empty compound FTS scorer", + )) + } + + fn advance_shallow(&mut self, _target: u64) -> Result { + Ok(u64::MAX) + } + + fn score_bounds(&mut self, _up_to: u64) -> Result { + Ok(ScoreBounds::ZERO) + } + + fn set_min_competitive_score(&mut self, _min_score: f32) -> Result<()> { + Ok(()) + } + + fn scores_non_negative(&self) -> bool { + true + } +} + +struct ScaleScorer<'a> { + child: BoxScorer<'a>, + factor: f32, +} + +impl<'a> ScaleScorer<'a> { + fn try_new(child: BoxScorer<'a>, factor: f32) -> Result { + if !factor.is_finite() || factor < 0.0 { + return Err(Error::invalid_input(format!( + "MatchQuery boost must be finite and non-negative, got {factor}" + ))); + } + Ok(Self { child, factor }) + } +} + +impl ComposableScorer for ScaleScorer<'_> { + fn doc(&self) -> Option { + self.child.doc() + } + + fn document_key(&self) -> Option { + self.child.document_key() + } + + fn next(&mut self) -> Result> { + self.child.next() + } + + fn advance(&mut self, target: u64) -> Result> { + self.child.advance(target) + } + + fn cost(&self) -> usize { + self.child.cost() + } + + fn score(&mut self) -> Result { + checked_score(self.child.score()? * self.factor, "MatchQuery boost") + } + + fn advance_shallow(&mut self, target: u64) -> Result { + self.child.advance_shallow(target) + } + + fn score_bounds(&mut self, up_to: u64) -> Result { + Ok(self + .child + .score_bounds(up_to)? + .scale_non_negative(self.factor)) + } + + fn set_min_competitive_score(&mut self, min_score: f32) -> Result<()> { + if self.factor > 0.0 { + self.child + .set_min_competitive_score(next_down(min_score / self.factor))?; + } + Ok(()) + } + + fn matches(&mut self) -> Result { + self.child.matches() + } + + fn match_cost(&self) -> Option { + self.child.match_cost() + } + + fn scores_non_negative(&self) -> bool { + self.child.scores_non_negative() + } +} + +/// Union scorer used for Boolean SHOULD sums and MultiMatch DisMax. +pub(super) struct DisjunctionScorer<'a> { + children: Vec>, + mode: DisjunctionScore, + current: Option, + confirmed_doc: Option, + confirmed: Vec, + min_competitive_score: f32, +} + +impl<'a> DisjunctionScorer<'a> { + pub(super) fn try_new(children: Vec>, mode: DisjunctionScore) -> Result { + if children.is_empty() { + return Err(Error::internal( + "FTS disjunction scorer requires at least one child", + )); + } + let confirmed = vec![false; children.len()]; + Ok(Self { + children, + mode, + current: None, + confirmed_doc: None, + confirmed, + min_competitive_score: f32::NEG_INFINITY, + }) + } + + fn set_current_from_children(&mut self) -> Option { + self.current = self.children.iter().filter_map(|child| child.doc()).min(); + self.confirmed_doc = None; + self.confirmed.fill(false); + self.current + } + + fn ensure_confirmed(&mut self) -> Result { + let Some(current) = self.current else { + return Ok(false); + }; + if self.confirmed_doc == Some(current) { + return Ok(self.confirmed.iter().any(|matched| *matched)); + } + self.confirmed.fill(false); + for (matched, child) in self.confirmed.iter_mut().zip(&mut self.children) { + if child.doc() == Some(current) { + *matched = child.matches()?; + } + } + self.confirmed_doc = Some(current); + Ok(self.confirmed.iter().any(|matched| *matched)) + } +} + +impl ComposableScorer for DisjunctionScorer<'_> { + fn doc(&self) -> Option { + self.current + } + + fn document_key(&self) -> Option { + let current = self.current?; + self.children + .iter() + .find(|child| child.doc() == Some(current)) + .and_then(|child| child.document_key()) + } + + fn next(&mut self) -> Result> { + match self.current { + None => { + for child in &mut self.children { + child.next()?; + } + } + Some(current) => { + for child in &mut self.children { + if child.doc() == Some(current) { + child.next()?; + } + } + } + } + Ok(self.set_current_from_children()) + } + + fn advance(&mut self, target: u64) -> Result> { + if self.current.is_some_and(|current| current >= target) { + return Ok(self.current); + } + for child in &mut self.children { + if child.doc().is_none_or(|doc| doc < target) { + child.advance(target)?; + } + } + Ok(self.set_current_from_children()) + } + + fn cost(&self) -> usize { + self.children + .iter() + .map(|child| child.cost()) + .fold(0, usize::saturating_add) + } + + fn score(&mut self) -> Result { + if !self.ensure_confirmed()? { + return Err(Error::internal( + "FTS disjunction score requested for an unconfirmed document", + )); + } + let mut score = match self.mode { + DisjunctionScore::Sum => 0.0_f32, + DisjunctionScore::Max => f32::NEG_INFINITY, + }; + for (matched, child) in self.confirmed.iter().zip(&mut self.children) { + if !matched { + continue; + } + let child_score = child.score()?; + score = match self.mode { + DisjunctionScore::Sum => score + child_score, + DisjunctionScore::Max => score.max(child_score), + }; + } + checked_score(score, "FTS disjunction") + } + + fn advance_shallow(&mut self, target: u64) -> Result { + let mut up_to = u64::MAX; + for child in &mut self.children { + if let Some(doc) = child.doc() { + up_to = up_to.min(child.advance_shallow(target.max(doc))?); + } + } + Ok(up_to) + } + + fn score_bounds(&mut self, up_to: u64) -> Result { + let mut bounds = match self.mode { + DisjunctionScore::Sum => ScoreBounds::ZERO, + DisjunctionScore::Max => ScoreBounds { + lower: f32::INFINITY, + upper: f32::NEG_INFINITY, + }, + }; + for child in &mut self.children { + let child_bounds = if child.doc().is_some_and(|doc| doc <= up_to) { + child.score_bounds(up_to)? + } else { + ScoreBounds::ZERO + }; + bounds = match self.mode { + DisjunctionScore::Sum => bounds.add(child_bounds.include_zero()), + DisjunctionScore::Max => ScoreBounds { + lower: bounds.lower.min(child_bounds.lower), + upper: bounds.upper.max(child_bounds.upper), + }, + }; + } + if bounds.lower == f32::INFINITY { + Ok(ScoreBounds::ZERO) + } else { + Ok(bounds) + } + } + + fn set_min_competitive_score(&mut self, min_score: f32) -> Result<()> { + if min_score.is_nan() { + return Err(Error::invalid_input( + "minimum competitive FTS score cannot be NaN", + )); + } + if min_score <= self.min_competitive_score { + return Ok(()); + } + self.min_competitive_score = min_score; + // A child below a DisMax threshold cannot affect a competitive max. + // Sum scorers need sibling-global bounds before translating the floor, + // so they keep it at this node and prune from their combined block bound. + if matches!(self.mode, DisjunctionScore::Max) { + for child in &mut self.children { + child.set_min_competitive_score(min_score)?; + } + } + Ok(()) + } + + fn matches(&mut self) -> Result { + self.ensure_confirmed() + } + + fn match_cost(&self) -> Option { + self.children + .iter() + .filter_map(|child| child.match_cost()) + .reduce(|left, right| left + right) + } + + fn scores_non_negative(&self) -> bool { + self.children + .iter() + .all(|child| child.scores_non_negative()) + } +} + +/// Intersection scorer that requires and scores every Boolean MUST child. +pub(super) struct RequiredConjunctionScorer<'a> { + children: Vec>, + /// Child indices sorted by approximation cost, omitted when query order is + /// already cheapest-first. `children` remains in query order so scoring and + /// score-bound arithmetic stay bit-for-bit stable. + approximation_order: Option>, + /// Child indices sorted by two-phase confirmation cost. Children without a + /// cost hint remain in query order after costed confirmations. + confirmation_order: Option>, + current: Option, + confirmed_doc: Option, + confirmed: bool, +} + +fn align_conjunction_children( + children: &mut [BoxScorer<'_>], + mut target: u64, + child_index: impl Fn(usize) -> usize, +) -> Result> { + loop { + for position in 0..children.len() { + let child = &mut children[child_index(position)]; + if child.doc().is_none_or(|doc| doc < target) { + let Some(doc) = child.advance(target)? else { + return Ok(None); + }; + target = target.max(doc); + } + } + let min_doc = children.iter().filter_map(|child| child.doc()).min(); + let max_doc = children.iter().filter_map(|child| child.doc()).max(); + if min_doc == max_doc { + return Ok(min_doc); + } + target = max_doc.ok_or_else(|| { + Error::internal("FTS conjunction lost a child while aligning scorers") + })?; + } +} + +fn compare_confirmation_cost( + left: &dyn ComposableScorer, + right: &dyn ComposableScorer, +) -> Ordering { + match (left.match_cost(), right.match_cost()) { + (Some(left), Some(right)) => left.total_cmp(&right), + (Some(_), None) => Ordering::Less, + (None, Some(_)) => Ordering::Greater, + (None, None) => Ordering::Equal, + } +} + +fn confirm_conjunction_children( + children: &mut [BoxScorer<'_>], + child_index: impl Fn(usize) -> usize, +) -> Result { + for position in 0..children.len() { + if !children[child_index(position)].matches()? { + return Ok(false); + } + } + Ok(true) +} + +impl<'a> RequiredConjunctionScorer<'a> { + pub(super) fn try_new(children: Vec>) -> Result { + if children.is_empty() { + return Err(Error::internal( + "FTS conjunction scorer requires at least one child", + )); + } + let approximation_order = if children + .windows(2) + .all(|pair| pair[0].cost() <= pair[1].cost()) + { + None + } else { + let mut order = (0..children.len()).collect::>(); + order.sort_by_key(|&index| (children[index].cost(), index)); + Some(order) + }; + for (index, child) in children.iter().enumerate() { + if let Some(match_cost) = child.match_cost() + && (!match_cost.is_finite() || match_cost < 0.0) + { + return Err(Error::internal(format!( + "FTS conjunction child {index} reported invalid two-phase match cost: {match_cost}" + ))); + } + } + let confirmation_order = if children.windows(2).all(|pair| { + compare_confirmation_cost(pair[0].as_ref(), pair[1].as_ref()) != Ordering::Greater + }) { + None + } else { + let mut order = (0..children.len()).collect::>(); + order.sort_by(|&left, &right| { + compare_confirmation_cost(children[left].as_ref(), children[right].as_ref()) + .then_with(|| left.cmp(&right)) + }); + Some(order) + }; + Ok(Self { + children, + approximation_order, + confirmation_order, + current: None, + confirmed_doc: None, + confirmed: false, + }) + } + + fn align(&mut self, target: u64) -> Result> { + self.current = if let Some(order) = &self.approximation_order { + align_conjunction_children(&mut self.children, target, |position| order[position])? + } else { + align_conjunction_children(&mut self.children, target, |position| position)? + }; + if self.current.is_some() { + self.confirmed_doc = None; + self.confirmed = false; + } + Ok(self.current) + } + + fn ensure_confirmed(&mut self) -> Result { + let Some(current) = self.current else { + return Ok(false); + }; + if self.confirmed_doc == Some(current) { + return Ok(self.confirmed); + } + self.confirmed = if let Some(order) = &self.confirmation_order { + confirm_conjunction_children(&mut self.children, |position| order[position])? + } else { + confirm_conjunction_children(&mut self.children, |position| position)? + }; + self.confirmed_doc = Some(current); + Ok(self.confirmed) + } +} + +impl ComposableScorer for RequiredConjunctionScorer<'_> { + fn doc(&self) -> Option { + self.current + } + + fn document_key(&self) -> Option { + self.children.first().and_then(|child| child.document_key()) + } + + fn next(&mut self) -> Result> { + let target = match self.current { + None => 0, + Some(u64::MAX) => return Ok(None), + Some(current) => current + 1, + }; + self.align(target) + } + + fn advance(&mut self, target: u64) -> Result> { + if self.current.is_some_and(|current| current >= target) { + return Ok(self.current); + } + self.align(target) + } + + fn cost(&self) -> usize { + self.children + .iter() + .map(|child| child.cost()) + .min() + .unwrap_or(0) + } + + fn score(&mut self) -> Result { + if !self.ensure_confirmed()? { + return Err(Error::internal( + "FTS conjunction score requested for an unconfirmed document", + )); + } + let mut score = 0.0_f32; + for child in &mut self.children { + score += child.score()?; + } + checked_score(score, "FTS conjunction") + } + + fn advance_shallow(&mut self, target: u64) -> Result { + let mut up_to = u64::MAX; + for child in &mut self.children { + let child_target = child.doc().map_or(target, |doc| target.max(doc)); + up_to = up_to.min(child.advance_shallow(child_target)?); + } + Ok(up_to) + } + + fn score_bounds(&mut self, up_to: u64) -> Result { + let mut bounds = ScoreBounds::ZERO; + for child in &mut self.children { + bounds = bounds.add(child.score_bounds(up_to)?); + } + Ok(bounds) + } + + fn set_min_competitive_score(&mut self, min_score: f32) -> Result<()> { + if min_score.is_nan() { + return Err(Error::invalid_input( + "minimum competitive FTS score cannot be NaN", + )); + } + // Propagating the full conjunction floor to one child is unsafe because + // individually sub-threshold MUST scores may sum to a competitive hit. + if self.children.len() == 1 { + self.children[0].set_min_competitive_score(min_score)?; + } + Ok(()) + } + + fn matches(&mut self) -> Result { + self.ensure_confirmed() + } + + fn match_cost(&self) -> Option { + self.children + .iter() + .filter_map(|child| child.match_cost()) + .reduce(|left, right| left + right) + } + + fn scores_non_negative(&self) -> bool { + self.children + .iter() + .all(|child| child.scores_non_negative()) + } +} + +/// Positive-driven Boost scorer with signed conservative bounds. +pub(super) struct BoostScorer<'a> { + positive: BoxScorer<'a>, + negative: BoxScorer<'a>, + negative_boost: f32, + negative_matches_doc: Option, + negative_matches: bool, +} + +impl<'a> BoostScorer<'a> { + pub(super) fn try_new( + positive: BoxScorer<'a>, + negative: BoxScorer<'a>, + negative_boost: f32, + ) -> Result { + if !negative_boost.is_finite() || negative_boost < 0.0 { + return Err(Error::invalid_input(format!( + "BoostQuery negative_boost must be finite and non-negative, got {negative_boost}" + ))); + } + Ok(Self { + positive, + negative, + negative_boost, + negative_matches_doc: None, + negative_matches: false, + }) + } + + fn reset_confirmation(&mut self) { + self.negative_matches_doc = None; + self.negative_matches = false; + } + + fn confirm_negative(&mut self) -> Result { + let Some(current) = self.positive.doc() else { + return Ok(false); + }; + if self.negative_matches_doc == Some(current) { + return Ok(self.negative_matches); + } + self.negative_matches = + self.negative.advance(current)? == Some(current) && self.negative.matches()?; + self.negative_matches_doc = Some(current); + Ok(self.negative_matches) + } +} + +impl ComposableScorer for BoostScorer<'_> { + fn doc(&self) -> Option { + self.positive.doc() + } + + fn document_key(&self) -> Option { + self.positive.document_key() + } + + fn next(&mut self) -> Result> { + self.reset_confirmation(); + self.positive.next() + } + + fn advance(&mut self, target: u64) -> Result> { + self.reset_confirmation(); + self.positive.advance(target) + } + + fn cost(&self) -> usize { + self.positive.cost() + } + + fn score(&mut self) -> Result { + let positive = self.positive.score()?; + let score = if self.confirm_negative()? { + positive - self.negative_boost * self.negative.score()? + } else { + positive + }; + checked_score(score, "BoostQuery scorer") + } + + fn advance_shallow(&mut self, target: u64) -> Result { + let mut up_to = self.positive.advance_shallow(target)?; + if self.negative.doc().is_none_or(|doc| doc < target) { + self.negative.advance(target)?; + } + if let Some(doc) = self.negative.doc() { + up_to = up_to.min(self.negative.advance_shallow(target.max(doc))?); + } + Ok(up_to) + } + + fn score_bounds(&mut self, up_to: u64) -> Result { + let positive = self.positive.score_bounds(up_to)?; + let negative = if self.negative.doc().is_some_and(|doc| doc <= up_to) { + self.negative.score_bounds(up_to)?.include_zero() + } else { + ScoreBounds::ZERO + }; + Ok(positive.subtract_scaled(negative, self.negative_boost)) + } + + fn set_min_competitive_score(&mut self, min_score: f32) -> Result<()> { + // With a non-negative negative scorer, Boost can only demote the + // positive score, so the parent's floor is safe for the positive side. + if self.negative.scores_non_negative() { + self.positive.set_min_competitive_score(min_score)?; + } + Ok(()) + } + + fn matches(&mut self) -> Result { + self.positive.matches() + } + + fn match_cost(&self) -> Option { + self.positive.match_cost() + } +} + +/// Required-plus-optional scorer that only touches the optional side when it +/// can change competitiveness or an exact score is requested. +/// +/// The required scorer drives iteration. Once a score floor is available, +/// block bounds may either skip the whole range or temporarily turn the +/// optional approximation into a required iterator when the required score +/// cannot reach the floor on its own. +#[derive(Clone, Copy)] +struct ReqOptBounds { + up_to: u64, + required: ScoreBounds, + combined: ScoreBounds, +} + +struct ReqOptScorer<'a> { + required: BoxScorer<'a>, + optional: BoxScorer<'a>, + current: Option, + exhausted: bool, + optional_initialized: bool, + optional_is_required: bool, + optional_checked_doc: Option, + optional_matches: bool, + confirmed_doc: Option, + confirmed: bool, + min_competitive_score: f32, + shallow_bounds: Option, +} + +impl<'a> ReqOptScorer<'a> { + fn new(required: BoxScorer<'a>, optional: BoxScorer<'a>) -> Self { + debug_assert!(required.scores_non_negative()); + debug_assert!(optional.scores_non_negative()); + Self { + required, + optional, + current: None, + exhausted: false, + optional_initialized: false, + optional_is_required: false, + optional_checked_doc: None, + optional_matches: false, + confirmed_doc: None, + confirmed: false, + min_competitive_score: f32::NEG_INFINITY, + shallow_bounds: None, + } + } + + fn set_current(&mut self, current: Option) { + if self.current != current { + self.optional_checked_doc = None; + self.optional_matches = false; + self.confirmed_doc = None; + self.confirmed = false; + } + self.current = current; + self.optional_is_required = false; + } + + fn set_optional_required(&mut self, required: bool) { + if self.optional_is_required != required { + self.confirmed_doc = None; + self.confirmed = false; + } + self.optional_is_required = required; + } + + fn exhaust(&mut self) -> Option { + self.exhausted = true; + self.set_current(None); + None + } + + fn ensure_optional_at_or_after(&mut self, target: u64) -> Result> { + if !self.optional_initialized || self.optional.doc().is_some_and(|doc| doc < target) { + self.optional.advance(target)?; + self.optional_initialized = true; + } + Ok(self.optional.doc()) + } + + fn optional_matches_current(&mut self) -> Result { + let Some(current) = self.current else { + return Ok(false); + }; + if self.optional_checked_doc == Some(current) { + return Ok(self.optional_matches); + } + self.optional_matches = self.ensure_optional_at_or_after(current)? == Some(current) + && self.optional.matches()?; + self.optional_checked_doc = Some(current); + Ok(self.optional_matches) + } + + fn usable_bounds(bounds: ScoreBounds) -> bool { + bounds.lower.is_finite() && bounds.upper.is_finite() && bounds.lower <= bounds.upper + } + + fn bounds(&mut self, up_to: u64) -> Result { + if let Some(bounds) = self.shallow_bounds + && bounds.up_to == up_to + { + return Ok(bounds); + } + + let required = self.required.score_bounds(up_to)?; + let optional = if self.optional.doc().is_some_and(|doc| doc <= up_to) { + let bounds = self.optional.score_bounds(up_to)?; + if Self::usable_bounds(bounds) { + bounds.include_zero() + } else { + ScoreBounds::UNBOUNDED + } + } else { + ScoreBounds::ZERO + }; + let combined = required.add(optional); + let bounds = ReqOptBounds { + up_to, + required, + combined, + }; + self.shallow_bounds = Some(bounds); + Ok(bounds) + } + + fn position(&mut self, mut target: u64) -> Result> { + if self.exhausted { + return Ok(None); + } + + 'search: loop { + let bounds = self.shallow_bounds.filter(|bounds| target <= bounds.up_to); + + if self.min_competitive_score.is_finite() + && self.min_competitive_score > 0.0 + && let Some(bounds) = bounds + && Self::usable_bounds(bounds.required) + && Self::usable_bounds(bounds.combined) + { + if bounds.combined.upper < self.min_competitive_score { + if bounds.up_to == u64::MAX { + return Ok(self.exhaust()); + } + target = bounds.up_to + 1; + self.shallow_bounds = None; + continue; + } + + if bounds.required.upper < self.min_competitive_score { + // The optional contribution is necessary throughout this + // cached shallow range. Intersect approximations until + // both sides agree or the range is exhausted. + let Some(mut required_doc) = self.required.advance(target)? else { + return Ok(self.exhaust()); + }; + if required_doc > bounds.up_to { + target = required_doc; + self.shallow_bounds = None; + continue; + } + self.set_current(Some(required_doc)); + self.set_optional_required(true); + loop { + self.set_current(Some(required_doc)); + self.set_optional_required(true); + let Some(optional_doc) = self.ensure_optional_at_or_after(required_doc)? + else { + if bounds.up_to == u64::MAX { + return Ok(self.exhaust()); + } + target = bounds.up_to + 1; + self.shallow_bounds = None; + continue 'search; + }; + if optional_doc > bounds.up_to { + if bounds.up_to == u64::MAX { + return Ok(self.exhaust()); + } + target = bounds.up_to + 1; + self.shallow_bounds = None; + continue 'search; + } + if optional_doc == required_doc { + return Ok(self.current); + } + let Some(next_required) = self.required.advance(optional_doc)? else { + return Ok(self.exhaust()); + }; + if next_required > bounds.up_to { + target = next_required; + self.shallow_bounds = None; + continue 'search; + } + required_doc = next_required; + } + } + } + + let Some(required_doc) = self.required.advance(target)? else { + return Ok(self.exhaust()); + }; + self.set_current(Some(required_doc)); + self.set_optional_required(false); + return Ok(self.current); + } + } + + fn ensure_confirmed(&mut self) -> Result { + let Some(current) = self.current else { + return Ok(false); + }; + if self.confirmed_doc == Some(current) { + return Ok(self.confirmed); + } + self.confirmed = self.required.matches()? + && (!self.optional_is_required || self.optional_matches_current()?); + self.confirmed_doc = Some(current); + Ok(self.confirmed) + } +} + +impl ComposableScorer for ReqOptScorer<'_> { + fn doc(&self) -> Option { + self.current + } + + fn document_key(&self) -> Option { + self.required.document_key() + } + + fn next(&mut self) -> Result> { + let target = match self.current { + None => 0, + Some(u64::MAX) => return Ok(self.exhaust()), + Some(current) => current + 1, + }; + self.position(target) + } + + fn advance(&mut self, target: u64) -> Result> { + if self.current.is_some_and(|current| current >= target) { + return Ok(self.current); + } + self.position(target) + } + + fn cost(&self) -> usize { + self.required.cost() + } + + fn score(&mut self) -> Result { + if !self.ensure_confirmed()? { + return Err(Error::internal( + "required-plus-optional FTS score requested for an unconfirmed document", + )); + } + let mut score = self.required.score()?; + if self.optional_matches_current()? { + score += self.optional.score()?; + } + checked_score(score, "required-plus-optional FTS scorer") + } + + fn advance_shallow(&mut self, target: u64) -> Result { + let target = self.current.map_or(target, |current| target.max(current)); + if let Some(bounds) = self.shallow_bounds + && target <= bounds.up_to + { + return Ok(bounds.up_to); + } + self.shallow_bounds = None; + let mut up_to = self.required.advance_shallow(target)?; + match self.ensure_optional_at_or_after(target)? { + Some(optional_doc) if optional_doc <= target => { + up_to = up_to.min(self.optional.advance_shallow(target)?); + } + Some(optional_doc) => { + up_to = up_to.min(optional_doc.saturating_sub(1)); + } + None => {} + } + Ok(up_to) + } + + fn score_bounds(&mut self, up_to: u64) -> Result { + Ok(self.bounds(up_to)?.combined) + } + + fn set_min_competitive_score(&mut self, min_score: f32) -> Result<()> { + if min_score.is_nan() { + return Err(Error::invalid_input( + "minimum competitive FTS score cannot be NaN", + )); + } + if min_score > self.min_competitive_score { + self.min_competitive_score = min_score; + } + Ok(()) + } + + fn matches(&mut self) -> Result { + self.ensure_confirmed() + } + + fn match_cost(&self) -> Option { + self.required + .match_cost() + .into_iter() + .chain(self.optional.match_cost()) + .reduce(|left, right| left + right) + } + + fn scores_non_negative(&self) -> bool { + true + } +} + +/// Boolean scorer preserving the current membership and score semantics. +pub(super) struct BooleanScorer<'a> { + driver: BoxScorer<'a>, + optional: Option>, + prohibited: Option>, + current: Option, + optional_matches: bool, +} + +impl<'a> BooleanScorer<'a> { + pub(super) fn try_new( + should: Vec>, + must: Vec>, + must_not: Vec>, + ) -> Result { + let mut optional = if should.is_empty() { + None + } else { + Some( + Box::new(DisjunctionScorer::try_new(should, DisjunctionScore::Sum)?) + as BoxScorer<'a>, + ) + }; + let driver = if must.is_empty() { + optional.take().ok_or_else(|| { + Error::invalid_input("boolean query must have at least one should/must query") + })? + } else { + let required = Box::new(RequiredConjunctionScorer::try_new(must)?) as BoxScorer<'a>; + if required.scores_non_negative() + && optional + .as_ref() + .is_some_and(|optional| optional.scores_non_negative()) + { + Box::new(ReqOptScorer::new( + required, + optional + .take() + .expect("checked that the optional scorer is present"), + )) as BoxScorer<'a> + } else { + required + } + }; + let prohibited = if must_not.is_empty() { + None + } else { + Some( + Box::new(DisjunctionScorer::try_new(must_not, DisjunctionScore::Max)?) + as BoxScorer<'a>, + ) + }; + Ok(Self { + driver, + optional, + prohibited, + current: None, + optional_matches: false, + }) + } + + fn accept_driver_doc(&mut self) -> Result { + let Some(current) = self.driver.doc() else { + return Ok(false); + }; + if !self.driver.matches()? { + return Ok(false); + } + if let Some(prohibited) = &mut self.prohibited + && prohibited.advance(current)? == Some(current) + && prohibited.matches()? + { + return Ok(false); + } + self.optional_matches = if let Some(optional) = &mut self.optional { + optional.advance(current)? == Some(current) && optional.matches()? + } else { + false + }; + self.current = Some(current); + Ok(true) + } + + fn next_accepted(&mut self, target: Option) -> Result> { + let mut doc = match target { + Some(target) => self.driver.advance(target)?, + None => self.driver.next()?, + }; + while doc.is_some() { + if self.accept_driver_doc()? { + return Ok(self.current); + } + doc = self.driver.next()?; + } + self.current = None; + self.optional_matches = false; + Ok(None) + } +} + +impl ComposableScorer for BooleanScorer<'_> { + fn doc(&self) -> Option { + self.current + } + + fn document_key(&self) -> Option { + self.driver.document_key() + } + + fn next(&mut self) -> Result> { + self.next_accepted(None) + } + + fn advance(&mut self, target: u64) -> Result> { + if self.current.is_some_and(|current| current >= target) { + return Ok(self.current); + } + self.next_accepted(Some(target)) + } + + fn cost(&self) -> usize { + self.driver.cost() + } + + fn score(&mut self) -> Result { + if self.current.is_none() { + return Err(Error::internal( + "Boolean FTS scorer is not positioned on a document", + )); + } + let mut score = self.driver.score()?; + if self.optional_matches + && let Some(optional) = &mut self.optional + { + score += optional.score()?; + } + checked_score(score, "BooleanQuery scorer") + } + + fn advance_shallow(&mut self, target: u64) -> Result { + let mut up_to = self.driver.advance_shallow(target)?; + if let Some(optional) = &mut self.optional + && let Some(doc) = optional.doc() + { + up_to = up_to.min(optional.advance_shallow(target.max(doc))?); + } + Ok(up_to) + } + + fn score_bounds(&mut self, up_to: u64) -> Result { + let mut bounds = self.driver.score_bounds(up_to)?; + if let Some(optional) = &mut self.optional + && optional.doc().is_some_and(|doc| doc <= up_to) + { + bounds = bounds.add(optional.score_bounds(up_to)?.include_zero()); + } + Ok(bounds) + } + + fn set_min_competitive_score(&mut self, min_score: f32) -> Result<()> { + // When SHOULD is also present, a global sibling bound is required to + // translate the parent threshold safely. The combined block bound still + // prunes at this node. Without SHOULD, driver score is the full score. + if self.optional.is_none() { + self.driver.set_min_competitive_score(min_score)?; + } + Ok(()) + } + + fn matches(&mut self) -> Result { + Ok(self.current.is_some()) + } + + fn scores_non_negative(&self) -> bool { + self.driver.scores_non_negative() + && self + .optional + .as_ref() + .is_none_or(|optional| optional.scores_non_negative()) + } +} + +#[derive(Clone)] +enum LeafQuery { + Match(MatchQuery), + Phrase(PhraseQuery), +} + +impl LeafQuery { + fn terms(&self) -> &str { + match self { + Self::Match(query) => &query.terms, + Self::Phrase(query) => &query.terms, + } + } + + fn operator(&self) -> Operator { + match self { + Self::Match(query) => query.operator, + Self::Phrase(_) => Operator::And, + } + } + + fn effective_params(&self, params: &FtsSearchParams) -> FtsSearchParams { + match self { + Self::Match(query) => params + .clone() + .with_limit(None) + .with_phrase_slop(None) + .with_fuzziness(query.fuzziness) + .with_max_expansions(query.max_expansions) + .with_prefix_length(query.prefix_length), + Self::Phrase(query) => params + .clone() + .with_limit(None) + .with_phrase_slop(Some(query.slop)), + } + } +} + +fn collect_leaf_queries(query: &FtsQuery, leaves: &mut Vec) -> Result<()> { + match query { + FtsQuery::Match(query) => leaves.push(LeafQuery::Match(query.clone())), + FtsQuery::Phrase(query) => leaves.push(LeafQuery::Phrase(query.clone())), + FtsQuery::Boost(query) => { + collect_leaf_queries(&query.positive, leaves)?; + collect_leaf_queries(&query.negative, leaves)?; + } + FtsQuery::MultiMatch(query) => { + leaves.extend(query.match_queries.iter().cloned().map(LeafQuery::Match)); + } + FtsQuery::Boolean(query) => { + for child in query + .should + .iter() + .chain(&query.must) + .chain(&query.must_not) + { + collect_leaf_queries(child, leaves)?; + } + } + } + Ok(()) +} + +struct PreparedLeaf { + tokens_by_segment: Vec>, + params: Arc, + operator: Operator, + scorer: Arc, +} + +fn tokenize_leaf(index: &InvertedIndex, leaf: &LeafQuery, params: &FtsSearchParams) -> Tokens { + let is_fuzzy_match = matches!(leaf, LeafQuery::Match(_)) + && matches!(params.fuzziness, Some(distance) if distance != 0); + let mut tokenizer = if is_fuzzy_match { + let analyzer = TextAnalyzer::from(SimpleTokenizer::default()); + match index.tokenizer().doc_type() { + DocType::Text => Box::new(TextTokenizer::new(analyzer)) as Box, + DocType::Json => Box::new(JsonTokenizer::new(analyzer)) as Box, + } + } else { + index.tokenizer() + }; + collect_query_tokens(leaf.terms(), &mut tokenizer) +} + +fn expanded_leaf_tokens( + index: &InvertedIndex, + tokens: &Tokens, + params: &FtsSearchParams, + operator: Operator, +) -> Result { + if !matches!(params.fuzziness, Some(distance) if distance != 0) { + return Ok(tokens.clone()); + } + let expanded = index.expand_fuzzy_tokens(tokens, params)?; + if operator == Operator::And || params.phrase_slop.is_some() { + let surviving = (0..expanded.len()) + .map(|index| expanded.position(index)) + .collect::>(); + if (0..tokens.len()).any(|index| !surviving.contains(&tokens.position(index))) { + return Ok(Tokens::with_positions( + Vec::new(), + Vec::new(), + tokens.token_type().clone(), + )); + } + } + Ok(expanded) +} + +fn validate_injected_scorer_tokens(scorer: &MemBM25Scorer, tokens: &Tokens) -> Result<()> { + for token in tokens { + if !scorer.token_docs.contains_key(token) { + return Err(Error::invalid_input(format!( + "injected BM25 scorer is missing compound FTS token '{token}'" + ))); + } + } + Ok(()) +} + +async fn prepare_compound_query( + indices: &[Arc], + query: &FtsQuery, + params: &FtsSearchParams, + metrics: &dyn MetricsCollector, + base_scorer: Option>, +) -> Result<(CompoundScorerPlan, Vec)> { + let first_index = indices + .first() + .ok_or_else(|| Error::invalid_input("compound FTS requires at least one index segment"))?; + let mut leaf_queries = Vec::new(); + collect_leaf_queries(query, &mut leaf_queries)?; + let mut num_plan_leaves = 0; + let plan = CompoundScorerPlan::from_query(query, &mut num_plan_leaves)?; + if num_plan_leaves != leaf_queries.len() { + return Err(Error::internal(format!( + "compound FTS planned {num_plan_leaves} leaves but prepared {}", + leaf_queries.len() + ))); + } + + let mut leaves = Vec::with_capacity(leaf_queries.len()); + for leaf in leaf_queries { + let effective_params = leaf.effective_params(params); + let tokens = tokenize_leaf(first_index, &leaf, &effective_params); + let scorer = match &base_scorer { + Some(scorer) => scorer.clone(), + None => Arc::new( + build_global_bm25_scorer(indices, &tokens, &effective_params, Some(metrics)) + .await?, + ), + }; + let mut tokens_by_segment = Vec::with_capacity(indices.len()); + for index in indices { + let expanded_tokens = + expanded_leaf_tokens(index, &tokens, &effective_params, leaf.operator())?; + if base_scorer.is_some() { + validate_injected_scorer_tokens(&scorer, &expanded_tokens)?; + } + tokens_by_segment.push(Arc::new(expanded_tokens)); + } + leaves.push(PreparedLeaf { + tokens_by_segment, + params: Arc::new(effective_params), + operator: leaf.operator(), + scorer, + }); + } + Ok((plan, leaves)) +} + +struct LoadedLeaf { + postings: Vec, + params: Arc, + operator: Operator, + scorer: Arc, +} + +enum LoadedDocuments { + Legacy(Arc), + Modern { + documents: Arc, + lengths: Arc, + visibility: DocVisibility, + projection: Option, + }, +} + +struct LoadedPartition { + segment_ordinal: usize, + partition_ordinal: usize, + partition: Arc, + documents: LoadedDocuments, + leaves: Vec, +} + +async fn load_compound_partition( + segment_ordinal: usize, + partition_ordinal: usize, + partition: Arc, + leaves: &[PreparedLeaf], + mask: Arc, + metrics: Arc, +) -> Result> { + let leaf_loads = leaves.iter().map(|leaf| { + let partition = partition.clone(); + let tokens = leaf.tokens_by_segment[segment_ordinal].clone(); + let params = leaf.params.clone(); + let scorer = leaf.scorer.clone(); + let metrics = metrics.clone(); + let operator = leaf.operator; + async move { + let postings = if tokens.is_empty() { + Vec::new() + } else { + partition + .load_posting_lists( + tokens.as_ref(), + params.as_ref(), + operator, + scorer.as_ref(), + metrics.as_ref(), + true, + ) + .await? + .postings + }; + Result::Ok(LoadedLeaf { + postings, + params, + operator, + scorer, + }) + } + }); + let leaves = futures::future::try_join_all(leaf_loads).await?; + + let documents = if let Some(docs) = partition.docs.legacy() { + LoadedDocuments::Legacy(docs.clone()) + } else { + let documents = partition.docs.modern().cloned().ok_or_else(|| { + Error::internal("FTS partition contains neither legacy nor modern documents") + })?; + let materialize_selected = mask.max_len().is_some_and(|selected| { + u128::from(selected).saturating_mul(100) + <= u128::from(*FLAT_SEARCH_PERCENT_THRESHOLD) + .saturating_mul(documents.len() as u128) + }); + let visibility = match documents.immediate_visibility(mask.clone(), materialize_selected) { + Some(visibility) => visibility, + None => { + documents + .visibility(mask.clone(), materialize_selected) + .await? + } + }; + if visibility.is_empty() { + return Ok(None); + } + let lengths = match documents.cached_lengths() { + Some(lengths) => lengths, + None => documents.lengths().await?, + }; + let projection = documents.resident_address_projection(); + LoadedDocuments::Modern { + documents, + lengths, + visibility, + projection, + } + }; + + Ok(Some(LoadedPartition { + segment_ordinal, + partition_ordinal, + partition, + documents, + leaves, + })) +} + +struct DeferredCompoundRows { + documents: Arc, + rows: Vec>, +} + +struct OverflowedCompoundPartition { + segment_ordinal: usize, + partition_ordinal: usize, + partition: Arc, + documents: Arc, +} + +enum PartitionCollectionBoundary { + Deferred(DeferredCompoundRows), + Overflow(OverflowedCompoundPartition), +} + +struct CollectedPartitions { + collector: TopKCollector, + remaining: Vec, + boundary: Option, +} + +fn collect_partition_with_documents( + documents: &D, + leaves: Vec, + plan: &CompoundScorerPlan, + metrics: &dyn MetricsCollector, + collector: &mut TopKCollector, + mut map_document: impl FnMut(u64) -> Result, +) -> Result +where + D: WandDocuments + Sync, + K: Copy + Ord, +{ + let mut leaf_scorers = leaves + .into_iter() + .map(|leaf| { + let scorer: BoxScorer<'_> = if leaf.postings.is_empty() { + Box::new(EmptyScorer) + } else { + Box::new(WandCursor::new( + leaf.operator, + leaf.postings, + documents, + leaf.scorer, + leaf.params.as_ref(), + metrics, + )) + }; + Some(scorer) + }) + .collect::>(); + let mut scorer = plan.build(&mut leaf_scorers)?; + if leaf_scorers.iter().any(Option::is_some) { + return Err(Error::internal( + "compound FTS scorer did not consume every prepared leaf", + )); + } + collector.collect_mapped(scorer.as_mut(), &mut map_document) +} + +fn collect_loaded_partitions( + partitions: Vec, + plan: &CompoundScorerPlan, + mask: &RowAddrMask, + metrics: &dyn MetricsCollector, + mut collector: TopKCollector, +) -> Result { + let mut partitions = partitions.into_iter(); + while let Some(partition) = partitions.next() { + let LoadedPartition { + segment_ordinal, + partition_ordinal, + partition: source, + documents, + leaves, + } = partition; + match documents { + LoadedDocuments::Legacy(docs) => { + let documents = LegacyWandDocuments::new(docs.as_ref(), mask); + let status = collect_partition_with_documents( + &documents, + leaves, + plan, + metrics, + &mut collector, + Ok, + )?; + debug_assert_eq!(status, CollectionStatus::Complete); + } + LoadedDocuments::Modern { + documents: partition_documents, + lengths, + visibility, + projection, + } => { + let documents = ModernWandDocuments::filtered(lengths.as_ref(), &visibility); + if let Some(projection) = projection { + let mut addresses_resolved = 0; + let status = collect_partition_with_documents( + &documents, + leaves, + plan, + metrics, + &mut collector, + |doc_id| { + let doc_id = DocId::new(u32::try_from(doc_id).map_err(|_| { + Error::index(format!( + "FTS DocId {doc_id} exceeds the modern u32 domain" + )) + })?); + let row_id = projection.address(doc_id).ok_or_else(|| { + Error::internal(format!( + "compound FTS scorer returned non-visible DocId {} in segment {segment_ordinal}, partition {partition_ordinal}", + doc_id.get() + )) + })?; + addresses_resolved += 1; + Ok(row_id) + }, + )?; + debug_assert_eq!(status, CollectionStatus::Complete); + metrics.record_compound_addresses_resolved(addresses_resolved); + } else { + let max_buffered = collector + .limit + .saturating_add(SCORE_FLOOR_RESOLUTION_BATCH_SIZE); + let mut local_collector = TopKCollector::retaining_score_floor( + collector.limit, + collector.competitive_score.clone(), + max_buffered, + ); + let status = collect_partition_with_documents( + &documents, + leaves, + plan, + metrics, + &mut local_collector, + |doc_id| { + Ok(DocId::new(u32::try_from(doc_id).map_err(|_| { + Error::index(format!( + "FTS DocId {doc_id} exceeds the modern u32 domain" + )) + })?)) + }, + )?; + metrics.record_compound_peak_buffered_candidates(local_collector.peak_buffered); + let boundary = match status { + CollectionStatus::Complete => { + PartitionCollectionBoundary::Deferred(DeferredCompoundRows { + documents: partition_documents, + rows: local_collector.into_candidates(), + }) + } + CollectionStatus::ScoreFloorOverflow => { + metrics.record_compound_score_floor_overflows(1); + PartitionCollectionBoundary::Overflow(OverflowedCompoundPartition { + segment_ordinal, + partition_ordinal, + partition: source, + documents: partition_documents, + }) + } + }; + metrics.record_compound_peak_buffered_candidates(collector.peak_buffered); + return Ok(CollectedPartitions { + collector, + remaining: partitions.collect(), + boundary: Some(boundary), + }); + } + } + } + } + metrics.record_compound_peak_buffered_candidates(collector.peak_buffered); + Ok(CollectedPartitions { + collector, + remaining: Vec::new(), + boundary: None, + }) +} + +async fn merge_resolved_compound_rows( + collector: &mut TopKCollector, + deferred: DeferredCompoundRows, + metrics: &dyn MetricsCollector, +) -> Result<()> { + for rows in deferred.rows.chunks(SCORE_FLOOR_RESOLUTION_BATCH_SIZE) { + let doc_ids = rows.iter().map(|row| row.row_id).collect::>(); + let addresses = deferred.documents.resolve_addresses(&doc_ids).await?; + if addresses.len() != rows.len() { + return Err(Error::internal(format!( + "compound FTS resolved {} addresses for {} DocIds", + addresses.len(), + rows.len() + ))); + } + metrics.record_compound_address_resolution_batches(1); + metrics.record_compound_peak_address_resolution_batch_size(rows.len()); + metrics.record_compound_addresses_resolved(addresses.len()); + for (row, row_id) in rows.iter().zip(addresses) { + let status = collector.insert(ScoredRow { + row_id, + score: row.score, + }); + debug_assert_eq!(status, CollectionStatus::Complete); + } + } + metrics.record_compound_peak_buffered_candidates(collector.peak_buffered); + Ok(()) +} + +async fn reload_compound_partition_with_projection( + overflow: OverflowedCompoundPartition, + leaves: &[PreparedLeaf], + mask: Arc, + metrics: Arc, +) -> Result { + let projection = overflow.documents.address_projection().await?; + let mut loaded = load_compound_partition( + overflow.segment_ordinal, + overflow.partition_ordinal, + overflow.partition, + leaves, + mask, + metrics, + ) + .await? + .ok_or_else(|| { + Error::internal(format!( + "compound FTS retry lost visible documents in segment {}, partition {}", + overflow.segment_ordinal, overflow.partition_ordinal + )) + })?; + match &mut loaded.documents { + LoadedDocuments::Modern { + projection: loaded_projection, + .. + } => *loaded_projection = Some(projection), + LoadedDocuments::Legacy(_) => { + return Err(Error::internal(format!( + "compound FTS retry changed segment {}, partition {} from modern to legacy documents", + overflow.segment_ordinal, overflow.partition_ordinal + ))); + } + } + Ok(loaded) +} + +/// Search one-column compound FTS directly over posting-backed scorers. +/// +/// The caller must provide all committed index segments for the column and a +/// ready prefilter. One collector owns the global top-k heap and propagates its +/// score floor through every partition-local scorer tree. Modern partitions +/// resolve candidates in bounded batches; an oversized kth-score tie is retried +/// against a resident row-address projection so final row-id ordering stays exact. +pub async fn compound_search( + indices: &[Arc], + query: &FtsQuery, + params: &FtsSearchParams, + prefilter: Arc, + metrics: Arc, +) -> Result<(Vec, Vec)> { + compound_search_impl(indices, query, params, prefilter, metrics, None).await +} + +/// Search one-column compound FTS with caller-supplied corpus-wide BM25 statistics. +/// +/// The scorer must contain an entry for every token used by every query leaf, +/// including terms produced by fuzzy expansion. An incomplete scorer is rejected +/// instead of treating missing token statistics as zero. +pub async fn compound_search_with_base_scorer( + indices: &[Arc], + query: &FtsQuery, + params: &FtsSearchParams, + prefilter: Arc, + metrics: Arc, + base_scorer: Arc, +) -> Result<(Vec, Vec)> { + compound_search_impl( + indices, + query, + params, + prefilter, + metrics, + Some(base_scorer), + ) + .await +} + +async fn compound_search_impl( + indices: &[Arc], + query: &FtsQuery, + params: &FtsSearchParams, + prefilter: Arc, + metrics: Arc, + base_scorer: Option>, +) -> Result<(Vec, Vec)> { + let limit = params.limit.unwrap_or(usize::MAX); + if limit == 0 { + return Ok((Vec::new(), Vec::new())); + } + let (plan, leaves) = + prepare_compound_query(indices, query, params, metrics.as_ref(), base_scorer).await?; + prefilter.wait_for_ready().await?; + let mask = prefilter.mask(); + let mut collector = TopKCollector::new(limit); + + for (segment_ordinal, index) in indices.iter().enumerate() { + let loads = + index + .partitions + .iter() + .cloned() + .enumerate() + .map(|(partition_ordinal, partition)| { + load_compound_partition( + segment_ordinal, + partition_ordinal, + partition, + &leaves, + mask.clone(), + metrics.clone(), + ) + }); + let mut partitions = stream::iter(loads) + .buffer_unordered(get_num_compute_intensive_cpus().clamp(1, 32)) + .try_collect::>() + .await? + .into_iter() + .flatten() + .collect::>(); + while !partitions.is_empty() { + let cpu_plan = plan.clone(); + let cpu_mask = mask.clone(); + let cpu_metrics = metrics.clone(); + let collected = spawn_cpu(move || { + collect_loaded_partitions( + partitions, + &cpu_plan, + cpu_mask.as_ref(), + cpu_metrics.as_ref(), + collector, + ) + }) + .await?; + collector = collected.collector; + partitions = collected.remaining; + match collected.boundary { + Some(PartitionCollectionBoundary::Deferred(deferred)) => { + merge_resolved_compound_rows(&mut collector, deferred, metrics.as_ref()) + .await?; + } + Some(PartitionCollectionBoundary::Overflow(overflow)) => { + let retry = reload_compound_partition_with_projection( + overflow, + &leaves, + mask.clone(), + metrics.clone(), + ) + .await?; + partitions.insert(0, retry); + } + None => debug_assert!(partitions.is_empty()), + } + } + } + + let rows = collector.into_rows(); + Ok(rows.into_iter().map(|row| (row.row_id, row.score)).unzip()) +} + +#[cfg(test)] +mod tests { + use std::sync::atomic::AtomicUsize; + + use super::*; + + fn rows(values: &[(u64, f32)]) -> Vec { + values + .iter() + .map(|(row_id, score)| ScoredRow::new(*row_id, *score).unwrap()) + .collect() + } + + fn materialized(values: &[(u64, f32)]) -> Box { + Box::new(MaterializedScorer::try_new(rows(values)).unwrap()) + } + + #[test] + fn score_bounds_are_conservative_under_nested_sum_and_boost() { + let should = DisjunctionScorer::try_new( + vec![ + materialized(&[(0, 0.1), (2, 2.0)]), + materialized(&[(0, 0.2)]), + ], + DisjunctionScore::Sum, + ) + .unwrap(); + let negative = materialized(&[(0, 0.3), (2, 5.0)]); + let mut scorer = BoostScorer::try_new(Box::new(should), negative, 0.5).unwrap(); + + assert_eq!(scorer.next().unwrap(), Some(0)); + let up_to = scorer.advance_shallow(0).unwrap(); + let bounds = scorer.score_bounds(up_to).unwrap(); + let first_score = scorer.score().unwrap(); + assert!(bounds.lower <= first_score); + assert!(bounds.upper >= first_score); + + assert_eq!(scorer.next().unwrap(), Some(2)); + let second_score = scorer.score().unwrap(); + assert!(second_score.is_sign_negative()); + let up_to = scorer.advance_shallow(2).unwrap(); + let bounds = scorer.score_bounds(up_to).unwrap(); + assert!(bounds.lower <= second_score); + assert!(bounds.upper >= second_score); + } + + #[test] + fn collector_propagates_threshold_across_partitions_and_keeps_ties() { + let mut collector = TopKCollector::new(2); + let mut first = MaterializedScorer::try_new(rows(&[(8, 9.0), (4, 10.0), (3, 9.0)])) + .unwrap() + .with_block_size(1); + collector.collect_mapped(&mut first, Ok).unwrap(); + assert_eq!(collector.competitive_score.get(), 9.0); + + let mut second = MaterializedScorer::try_new(rows(&[(1, 1.0), (2, 9.0)])) + .unwrap() + .with_block_size(1); + collector.collect_mapped(&mut second, Ok).unwrap(); + assert_eq!( + collector.into_rows(), + vec![ + ScoredRow { + row_id: 4, + score: 10.0 + }, + ScoredRow { + row_id: 2, + score: 9.0 + } + ] + ); + } + + #[test] + fn collector_bounds_equal_score_candidates() { + let limit = 1; + let num_candidates = DEFAULT_BLOCK_SIZE * 4; + let values = (0..num_candidates) + .map(|row_id| (row_id as u64, 1.0)) + .collect::>(); + let mut scorer = MaterializedScorer::try_new(rows(&values)).unwrap(); + let max_buffered = limit + SCORE_FLOOR_RESOLUTION_BATCH_SIZE; + let mut collector = TopKCollector::retaining_score_floor( + limit, + Arc::new(CompetitiveScore::default()), + max_buffered, + ); + + let status = collector.collect_mapped(&mut scorer, Ok).unwrap(); + + assert_eq!(status, CollectionStatus::ScoreFloorOverflow); + assert_eq!(collector.heap.len(), max_buffered); + } + + #[test] + fn collector_reclaims_obsolete_score_floor_before_overflowing() { + let limit = 2; + let max_buffered = 4; + let values = [ + (0, 1.0), + (1, 1.0), + (2, 1.0), + (3, 2.0), + (4, 2.0), + (5, 2.0), + (6, 2.0), + (7, 2.0), + ]; + let mut scorer = MaterializedScorer::try_new(rows(&values)).unwrap(); + let competitive_score = Arc::new(CompetitiveScore::default()); + let mut collector = + TopKCollector::retaining_score_floor(limit, competitive_score.clone(), max_buffered); + + let status = collector.collect_mapped(&mut scorer, Ok).unwrap(); + + assert_eq!(status, CollectionStatus::ScoreFloorOverflow); + assert_eq!(collector.heap.len(), max_buffered); + assert!(collector.heap.iter().all(|row| row.0.score == 2.0)); + assert_eq!(competitive_score.get(), 2.0); + } + + struct TwoPhaseScorer { + inner: MaterializedScorer, + accepted: Vec, + match_cost: Option, + approximations: Arc, + confirmations: Arc, + } + + impl ComposableScorer for TwoPhaseScorer { + fn doc(&self) -> Option { + self.inner.doc() + } + + fn next(&mut self) -> Result> { + let doc = self.inner.next()?; + if doc.is_some() { + self.approximations.fetch_add(1, AtomicOrdering::Relaxed); + } + Ok(doc) + } + + fn advance(&mut self, target: u64) -> Result> { + let doc = self.inner.advance(target)?; + if doc.is_some() { + self.approximations.fetch_add(1, AtomicOrdering::Relaxed); + } + Ok(doc) + } + + fn cost(&self) -> usize { + self.inner.cost() + } + + fn score(&mut self) -> Result { + self.inner.score() + } + + fn advance_shallow(&mut self, target: u64) -> Result { + self.inner.advance_shallow(target) + } + + fn score_bounds(&mut self, up_to: u64) -> Result { + self.inner.score_bounds(up_to) + } + + fn set_min_competitive_score(&mut self, min_score: f32) -> Result<()> { + self.inner.set_min_competitive_score(min_score) + } + + fn matches(&mut self) -> Result { + self.confirmations.fetch_add(1, AtomicOrdering::Relaxed); + Ok(self + .doc() + .is_some_and(|doc| self.accepted.binary_search(&doc).is_ok())) + } + + fn match_cost(&self) -> Option { + self.match_cost + } + + fn scores_non_negative(&self) -> bool { + true + } + } + + fn two_phase( + values: &[(u64, f32)], + accepted: Vec, + match_cost: Option, + ) -> ( + Box, + Arc, + Arc, + ) { + let approximations = Arc::new(AtomicUsize::new(0)); + let confirmations = Arc::new(AtomicUsize::new(0)); + let scorer = TwoPhaseScorer { + inner: MaterializedScorer::try_new(rows(values)).unwrap(), + accepted, + match_cost, + approximations: approximations.clone(), + confirmations: confirmations.clone(), + }; + (Box::new(scorer), approximations, confirmations) + } + + #[derive(Default)] + struct ScorerWork { + advances: AtomicUsize, + confirmations: AtomicUsize, + shallow_advances: AtomicUsize, + bounds: AtomicUsize, + } + + struct InstrumentedScorer<'a> { + inner: BoxScorer<'a>, + work: Arc, + } + + impl ComposableScorer for InstrumentedScorer<'_> { + fn doc(&self) -> Option { + self.inner.doc() + } + + fn document_key(&self) -> Option { + self.inner.document_key() + } + + fn next(&mut self) -> Result> { + let doc = self.inner.next()?; + if doc.is_some() { + self.work.advances.fetch_add(1, AtomicOrdering::Relaxed); + } + Ok(doc) + } + + fn advance(&mut self, target: u64) -> Result> { + let doc = self.inner.advance(target)?; + if doc.is_some() { + self.work.advances.fetch_add(1, AtomicOrdering::Relaxed); + } + Ok(doc) + } + + fn cost(&self) -> usize { + self.inner.cost() + } + + fn score(&mut self) -> Result { + self.inner.score() + } + + fn advance_shallow(&mut self, target: u64) -> Result { + self.work + .shallow_advances + .fetch_add(1, AtomicOrdering::Relaxed); + self.inner.advance_shallow(target) + } + + fn score_bounds(&mut self, up_to: u64) -> Result { + self.work.bounds.fetch_add(1, AtomicOrdering::Relaxed); + self.inner.score_bounds(up_to) + } + + fn set_min_competitive_score(&mut self, min_score: f32) -> Result<()> { + self.inner.set_min_competitive_score(min_score) + } + + fn matches(&mut self) -> Result { + self.work + .confirmations + .fetch_add(1, AtomicOrdering::Relaxed); + self.inner.matches() + } + + fn match_cost(&self) -> Option { + self.inner.match_cost() + } + + fn scores_non_negative(&self) -> bool { + self.inner.scores_non_negative() + } + } + + fn instrumented<'a>(inner: BoxScorer<'a>) -> (BoxScorer<'a>, Arc) { + let work = Arc::new(ScorerWork::default()); + ( + Box::new(InstrumentedScorer { + inner, + work: work.clone(), + }), + work, + ) + } + + struct UnboundedScorer { + inner: MaterializedScorer, + } + + impl ComposableScorer for UnboundedScorer { + fn doc(&self) -> Option { + self.inner.doc() + } + + fn next(&mut self) -> Result> { + self.inner.next() + } + + fn advance(&mut self, target: u64) -> Result> { + self.inner.advance(target) + } + + fn cost(&self) -> usize { + self.inner.cost() + } + + fn score(&mut self) -> Result { + self.inner.score() + } + + fn advance_shallow(&mut self, target: u64) -> Result { + self.inner.advance_shallow(target) + } + + fn score_bounds(&mut self, _up_to: u64) -> Result { + Ok(ScoreBounds::UNBOUNDED) + } + + fn set_min_competitive_score(&mut self, min_score: f32) -> Result<()> { + self.inner.set_min_competitive_score(min_score) + } + + fn scores_non_negative(&self) -> bool { + true + } + } + + struct CountingScorer { + inner: MaterializedScorer, + cost: usize, + advance_calls: Arc, + } + + impl ComposableScorer for CountingScorer { + fn doc(&self) -> Option { + self.inner.doc() + } + + fn next(&mut self) -> Result> { + self.inner.next() + } + + fn advance(&mut self, target: u64) -> Result> { + self.advance_calls.fetch_add(1, AtomicOrdering::Relaxed); + self.inner.advance(target) + } + + fn cost(&self) -> usize { + self.cost + } + + fn score(&mut self) -> Result { + self.inner.score() + } + + fn advance_shallow(&mut self, target: u64) -> Result { + self.inner.advance_shallow(target) + } + + fn score_bounds(&mut self, up_to: u64) -> Result { + self.inner.score_bounds(up_to) + } + + fn set_min_competitive_score(&mut self, min_score: f32) -> Result<()> { + self.inner.set_min_competitive_score(min_score) + } + + fn matches(&mut self) -> Result { + self.inner.matches() + } + + fn scores_non_negative(&self) -> bool { + self.inner.scores_non_negative() + } + } + + fn counting( + values: &[(u64, f32)], + cost: usize, + ) -> (Box, Arc) { + let advance_calls = Arc::new(AtomicUsize::new(0)); + let scorer = CountingScorer { + inner: MaterializedScorer::try_new(rows(values)).unwrap(), + cost, + advance_calls: advance_calls.clone(), + }; + (Box::new(scorer), advance_calls) + } + + #[test] + fn collector_confirms_two_phase_matches_without_a_cost_hint() { + let (mut scorer, approximations, confirmations) = + two_phase(&[(1, 100.0), (2, 2.0), (3, 1.0)], vec![2, 3], None); + let results = TopKCollector::new(2).collect(scorer.as_mut()).unwrap(); + assert_eq!(results, rows(&[(2, 2.0), (3, 1.0)])); + assert_eq!(approximations.load(AtomicOrdering::Relaxed), 3); + assert_eq!(confirmations.load(AtomicOrdering::Relaxed), 3); + assert_eq!(scorer.match_cost(), None); + } + + #[test] + fn required_conjunction_confirms_cheapest_first_and_short_circuits() { + let values = (0..100).map(|doc| (doc, 1.0)).collect::>(); + let accepted_by_cheap = (0..100).step_by(5).collect::>(); + + let (expensive, expensive_approximations, expensive_confirmations) = + two_phase(&values, (0..100).collect(), Some(10.0)); + let (cheap, cheap_approximations, cheap_confirmations) = + two_phase(&values, accepted_by_cheap.clone(), Some(1.0)); + let mut scorer = RequiredConjunctionScorer::try_new(vec![expensive, cheap]).unwrap(); + assert_eq!(scorer.confirmation_order.as_deref(), Some(&[1, 0][..])); + + let results = TopKCollector::new(100).collect(&mut scorer).unwrap(); + let expected = accepted_by_cheap + .iter() + .map(|doc| (*doc, 2.0)) + .collect::>(); + assert_eq!(results, rows(&expected)); + assert_eq!(cheap_confirmations.load(AtomicOrdering::Relaxed), 100); + assert_eq!(expensive_confirmations.load(AtomicOrdering::Relaxed), 20); + let approximations = cheap_approximations.load(AtomicOrdering::Relaxed) + + expensive_approximations.load(AtomicOrdering::Relaxed); + let confirmations = cheap_confirmations.load(AtomicOrdering::Relaxed) + + expensive_confirmations.load(AtomicOrdering::Relaxed); + assert_eq!(approximations, 200); + assert_eq!(confirmations, 120); + assert!( + confirmations * 5 <= approximations * 4, + "confirmation ordering should reduce work by at least 20%: {confirmations}/{approximations}" + ); + + let (cheap, _, _) = two_phase(&values, accepted_by_cheap, Some(1.0)); + let (expensive, _, _) = two_phase(&values, (0..100).collect(), Some(10.0)); + let scorer = RequiredConjunctionScorer::try_new(vec![cheap, expensive]).unwrap(); + assert!(scorer.confirmation_order.is_none()); + } + + #[test] + fn required_conjunction_confirms_children_without_cost_hints() { + let (unknown, _, unknown_confirmations) = two_phase(&[(0, 1.0)], Vec::new(), None); + let (costed, _, costed_confirmations) = two_phase(&[(0, 1.0)], vec![0], Some(1.0)); + let mut scorer = RequiredConjunctionScorer::try_new(vec![unknown, costed]).unwrap(); + + assert_eq!(scorer.confirmation_order.as_deref(), Some(&[1, 0][..])); + assert!( + TopKCollector::new(1) + .collect(&mut scorer) + .unwrap() + .is_empty() + ); + assert_eq!(costed_confirmations.load(AtomicOrdering::Relaxed), 1); + assert_eq!(unknown_confirmations.load(AtomicOrdering::Relaxed), 1); + } + + #[test] + fn required_conjunction_rejects_invalid_match_cost() { + let (invalid, _, _) = two_phase(&[(0, 1.0)], vec![0], Some(f32::NAN)); + + let error = RequiredConjunctionScorer::try_new(vec![invalid]) + .err() + .unwrap(); + assert!(matches!(error, Error::Internal { .. })); + assert!( + error + .to_string() + .contains("child 0 reported invalid two-phase match cost: NaN") + ); + } + + #[test] + fn required_conjunction_uses_all_must_scores_for_competitive_bounds() { + let left = Box::new( + MaterializedScorer::try_new(rows(&[(1, 3.0), (3, 1.0)])) + .unwrap() + .with_block_size(1), + ); + let right = Box::new( + MaterializedScorer::try_new(rows(&[(1, 30.0), (3, 10.0)])) + .unwrap() + .with_block_size(1), + ); + let mut scorer = RequiredConjunctionScorer::try_new(vec![left, right]).unwrap(); + let competitive_score = Arc::new(CompetitiveScore::default()); + competitive_score.raise(10.0); + + let results = TopKCollector::with_competitive_score(10, competitive_score) + .collect(&mut scorer) + .unwrap(); + + assert_eq!(results, rows(&[(1, 33.0), (3, 11.0)])); + } + + #[test] + fn required_conjunction_aligns_cheapest_clause_first() { + let dense_rows = (0..=100).map(|row_id| (row_id, 1.0)).collect::>(); + let (dense, dense_advance_calls) = counting(&dense_rows, 101); + let (rare, rare_advance_calls) = counting(&[(50, 1.0)], 1); + let mut scorer = RequiredConjunctionScorer::try_new(vec![dense, rare]).unwrap(); + assert_eq!(scorer.approximation_order.as_deref(), Some(&[1, 0][..])); + + assert_eq!(scorer.next().unwrap(), Some(50)); + assert_eq!(scorer.next().unwrap(), None); + assert_eq!(dense_advance_calls.load(AtomicOrdering::Relaxed), 1); + assert_eq!(rare_advance_calls.load(AtomicOrdering::Relaxed), 2); + + let (rare, _) = counting(&[(50, 1.0)], 1); + let (dense, _) = counting(&dense_rows, 101); + let scorer = RequiredConjunctionScorer::try_new(vec![rare, dense]).unwrap(); + assert!(scorer.approximation_order.is_none()); + } + + #[test] + fn required_conjunction_preserves_query_score_order() { + let (large, _) = counting(&[(0, 16_777_216.0)], 3); + let (first_small, _) = counting(&[(0, 1.0)], 1); + let (second_small, _) = counting(&[(0, 1.0)], 2); + let mut scorer = + RequiredConjunctionScorer::try_new(vec![large, first_small, second_small]).unwrap(); + + assert_eq!(scorer.next().unwrap(), Some(0)); + assert_eq!(scorer.score().unwrap(), 16_777_216.0); + } + + #[test] + fn boolean_sums_all_matching_clause_scores() { + let must = vec![ + materialized(&[(1, 3.0), (2, 2.0), (3, 1.0)]), + materialized(&[(1, 30.0), (3, 10.0)]), + ]; + let should = vec![ + materialized(&[(1, 0.5), (3, 4.0)]), + materialized(&[(3, 2.0)]), + ]; + let must_not = vec![materialized(&[(1, 9.0)])]; + let mut boolean = BooleanScorer::try_new(should, must, must_not).unwrap(); + let results = TopKCollector::new(10).collect(&mut boolean).unwrap(); + assert_eq!( + results, + vec![ScoredRow { + row_id: 3, + score: 17.0 + }] + ); + + let mut dismax = DisjunctionScorer::try_new( + vec![ + materialized(&[(1, 2.0), (3, 3.0)]), + materialized(&[(1, 4.0), (2, 4.0)]), + ], + DisjunctionScore::Max, + ) + .unwrap(); + let results = TopKCollector::new(2).collect(&mut dismax).unwrap(); + assert_eq!(results, rows(&[(1, 4.0), (2, 4.0)])); + } + + #[test] + fn reqopt_delays_sparse_optional_probes() { + let values = (0..100).map(|doc| (doc, 1.0)).collect::>(); + let build = || { + let required = + Box::new(RequiredConjunctionScorer::try_new(vec![materialized(&values)]).unwrap()); + let optional = Box::new( + DisjunctionScorer::try_new( + vec![materialized(&[(99, 100.0)])], + DisjunctionScore::Sum, + ) + .unwrap(), + ); + let (required, required_work) = instrumented(required); + let (optional, optional_work) = instrumented(optional); + (required, required_work, optional, optional_work) + }; + + let (required, _, optional, eager_optional_work) = build(); + let mut eager = BooleanScorer { + driver: required, + optional: Some(optional), + prohibited: None, + current: None, + optional_matches: false, + }; + let eager_results = TopKCollector::new(1).collect(&mut eager).unwrap(); + + let (required, required_work, optional, optional_work) = build(); + let mut scorer = ReqOptScorer::new(required, optional); + let results = TopKCollector::new(1).collect(&mut scorer).unwrap(); + + assert_eq!(eager_results, rows(&[(99, 101.0)])); + assert_eq!(results, rows(&[(99, 101.0)])); + let required_advances = required_work.advances.load(AtomicOrdering::Relaxed); + let eager_optional_probes = eager_optional_work.advances.load(AtomicOrdering::Relaxed); + let optional_probes = optional_work.advances.load(AtomicOrdering::Relaxed); + assert_eq!(required_advances, 100); + assert_eq!(eager_optional_probes, 100); + assert_eq!(optional_probes, 1); + assert_eq!( + required_work.shallow_advances.load(AtomicOrdering::Relaxed), + 2 + ); + assert_eq!(required_work.bounds.load(AtomicOrdering::Relaxed), 2); + assert_eq!( + required_work.confirmations.load(AtomicOrdering::Relaxed), + 100 + ); + assert_eq!(optional_work.confirmations.load(AtomicOrdering::Relaxed), 1); + assert!( + optional_probes * 5 <= eager_optional_probes * 4, + "lazy required-plus-optional scoring should reduce optional probes by at least 20%: \ + {optional_probes}/{eager_optional_probes}" + ); + } + + #[test] + fn reqopt_temporarily_requires_optional_contribution() { + let values = (0..100).map(|doc| (doc, 1.0)).collect::>(); + let (required, required_work) = instrumented(materialized(&values)); + let (optional, optional_work) = instrumented(materialized(&[(50, 10.0)])); + let mut scorer = ReqOptScorer::new(required, optional); + let competitive_score = Arc::new(CompetitiveScore::default()); + competitive_score.raise(10.0); + + let results = TopKCollector::with_competitive_score(1, competitive_score) + .collect(&mut scorer) + .unwrap(); + + assert_eq!(results, rows(&[(50, 11.0)])); + assert!(required_work.advances.load(AtomicOrdering::Relaxed) < 10); + assert_eq!(required_work.confirmations.load(AtomicOrdering::Relaxed), 1); + assert_eq!(optional_work.advances.load(AtomicOrdering::Relaxed), 1); + assert_eq!(optional_work.confirmations.load(AtomicOrdering::Relaxed), 1); + } + + #[test] + fn boolean_reqopt_keeps_current_confirmation_stable_after_bounds() { + let mut scorer = BooleanScorer::try_new( + vec![materialized(&[(1, 10.0)])], + vec![materialized(&[(0, 1.0), (1, 1.0)])], + Vec::new(), + ) + .unwrap(); + let competitive_score = Arc::new(CompetitiveScore::default()); + competitive_score.raise(5.0); + + let results = TopKCollector::with_competitive_score(1, competitive_score) + .collect(&mut scorer) + .unwrap(); + + assert_eq!(results, rows(&[(1, 11.0)])); + } + + #[test] + fn boolean_gates_reqopt_and_signed_boost_uses_exact_fallback() { + let supported = BooleanScorer::try_new( + vec![materialized(&[(1, 4.0)])], + vec![materialized(&[(1, 2.0)])], + Vec::new(), + ) + .unwrap(); + assert!(supported.optional.is_none()); + + let signed_optional = Box::new( + BoostScorer::try_new( + materialized(&[(1, 4.0), (2, 1.0)]), + materialized(&[(1, 1.0), (2, 4.0)]), + 0.5, + ) + .unwrap(), + ); + let mut fallback = BooleanScorer::try_new( + vec![signed_optional], + vec![materialized(&[(1, 2.0), (2, 2.0)])], + Vec::new(), + ) + .unwrap(); + assert!(fallback.optional.is_some()); + assert_eq!( + TopKCollector::new(2).collect(&mut fallback).unwrap(), + rows(&[(1, 5.5), (2, 1.0)]) + ); + } + + #[test] + fn reqopt_uses_exact_iteration_for_unbounded_scorers() { + let required = Box::new(UnboundedScorer { + inner: MaterializedScorer::try_new(rows(&[(1, 1.0), (2, 1.0)])).unwrap(), + }); + let optional = materialized(&[(2, 10.0)]); + let mut scorer = ReqOptScorer::new(required, optional); + let competitive_score = Arc::new(CompetitiveScore::default()); + competitive_score.raise(5.0); + + let results = TopKCollector::with_competitive_score(1, competitive_score) + .collect(&mut scorer) + .unwrap(); + + assert_eq!(results, rows(&[(2, 11.0)])); + } +} diff --git a/rust/lance-index/src/scalar/inverted/documents.rs b/rust/lance-index/src/scalar/inverted/documents.rs new file mode 100644 index 00000000000..03020d37f2a --- /dev/null +++ b/rust/lance-index/src/scalar/inverted/documents.rs @@ -0,0 +1,2275 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright The Lance Authors + +//! Typed document-side state for partitioned FTS indices. +//! +//! Posting lists use dense, partition-local document identifiers. This module +//! keeps that identity separate from dataset-version row addresses so scoring +//! never has to infer which value a numeric slot represents. + +use std::borrow::Cow; +use std::sync::{Arc, OnceLock, Weak}; + +use arc_swap::ArcSwapWeak; +use arrow::buffer::ScalarBuffer; +use arrow_array::{Array, RecordBatch, UInt32Array, UInt64Array}; +use lance_core::cache::{CacheKey, WeakLanceCache}; +use lance_core::deepsize::DeepSizeOf; +use lance_core::utils::address::RowAddress; +use lance_core::utils::tokio::spawn_cpu; +use lance_core::{Error, ROW_ID, Result}; +use lance_select::{RowAddrMask, RowAddrSelection, RowAddrTreeMap}; +use object_store::path::Path; +use roaring::RoaringBitmap; +use tokio::sync::OnceCell; + +use crate::FtsPrewarmDocumentStatus; +use crate::scalar::{IndexReader, IndexStore, RowIdRemapper}; + +use super::index::{ + DocSet, NUM_TOKEN_COL, dequantize_doc_length, doc_index_storage_column, + document_coordinate_rank, quantize_doc_length, +}; + +/// Schema metadata key persisted in every modern `docs.lance` partition. +pub(super) const TOTAL_TOKENS_KEY: &str = "total_tokens"; + +/// Dense, immutable document identity inside one FTS partition. +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub(super) struct DocId(u32); + +impl DocId { + pub(crate) fn new(value: u32) -> Self { + Self(value) + } + + pub(crate) fn get(self) -> u32 { + self.0 + } + + fn as_usize(self) -> usize { + self.0 as usize + } +} + +/// Immutable corpus statistics for one partition. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(super) struct PartitionStats { + pub(crate) num_docs: usize, + pub(crate) total_tokens: u64, +} + +/// One partition's immutable `DocId -> row address` column. +/// +/// The independently weighed cache entry is the only long-lived owner. The +/// address projection keeps a weak handle and query-local guards upgrade it, +/// so cache eviction releases the column once in-flight queries finish. +#[derive(Debug)] +pub(super) struct CachedDocRowIds { + pub(crate) row_ids: Arc, +} + +impl DeepSizeOf for CachedDocRowIds { + fn deep_size_of_children(&self, _context: &mut lance_core::deepsize::Context) -> usize { + self.row_ids.len() * std::mem::size_of::() + } +} + +/// Cache key for one partition's [`CachedDocRowIds`]. +#[derive(Debug, Clone)] +pub(super) struct DocRowIdsKey { + pub(crate) partition_id: u64, +} + +impl CacheKey for DocRowIdsKey { + type ValueType = CachedDocRowIds; + + fn key(&self) -> Cow<'_, str> { + format!("doc-row-ids-{}", self.partition_id).into() + } + + fn type_name() -> &'static str { + "DocRowIds" + } +} + +/// Exact document lengths plus the optional quantized scoring representation. +#[derive(Debug)] +pub(super) struct DocLengths { + values: ScalarBuffer, + total_tokens: u64, + quantized_scoring: bool, + norms: OnceLock>, +} + +impl DeepSizeOf for DocLengths { + fn deep_size_of_children(&self, context: &mut lance_core::deepsize::Context) -> usize { + self.values.deep_size_of_children(context) + + self + .norms + .get() + .map(|norms| std::mem::size_of_val(norms.as_ref())) + .unwrap_or(0) + } +} + +impl DocLengths { + fn try_new( + values: ScalarBuffer, + expected_num_docs: usize, + persisted_total_tokens: Option, + quantized_scoring: bool, + path: &str, + ) -> Result { + if values.len() != expected_num_docs { + return Err(corrupt_docs( + path, + format!( + "{NUM_TOKEN_COL} has {} rows but the file footer reports {expected_num_docs}", + values.len() + ), + )); + } + let total_tokens = values.iter().try_fold(0_u64, |total, &value| { + total + .checked_add(u64::from(value)) + .ok_or_else(|| corrupt_docs(path, format!("{NUM_TOKEN_COL} sum overflows u64"))) + })?; + if let Some(expected) = persisted_total_tokens + && expected != total_tokens + { + return Err(corrupt_docs( + path, + format!( + "{TOTAL_TOKENS_KEY} metadata is {expected}, but {NUM_TOKEN_COL} sums to {total_tokens}" + ), + )); + } + Ok(Self { + values, + total_tokens, + quantized_scoring, + norms: OnceLock::new(), + }) + } + + pub(crate) fn len(&self) -> usize { + self.values.len() + } + + pub(crate) fn total_tokens(&self) -> u64 { + self.total_tokens + } + + #[inline] + pub(crate) fn exact(&self, doc_id: DocId) -> u32 { + self.values[doc_id.as_usize()] + } + + pub(crate) fn scoring_norms(&self) -> Option<&[u8]> { + if !self.quantized_scoring { + return None; + } + Some( + self.norms + .get_or_init(|| { + self.values + .iter() + .map(|&length| quantize_doc_length(length)) + .collect() + }) + .as_ref(), + ) + } + + fn scoring_ready(&self) -> bool { + !self.quantized_scoring || self.norms.get().is_some() + } + + #[inline] + pub(crate) fn scoring(&self, doc_id: DocId) -> u32 { + match self.scoring_norms() { + Some(norms) => dequantize_doc_length(norms[doc_id.as_usize()]), + None => self.exact(doc_id), + } + } +} + +#[derive(Debug)] +enum AddressValues { + Shared { len: usize }, + Owned(Arc>), +} + +impl AddressValues { + fn len(&self) -> usize { + match self { + Self::Shared { len } => *len, + Self::Owned(values) => values.len(), + } + } +} + +impl DeepSizeOf for AddressValues { + fn deep_size_of_children(&self, context: &mut lance_core::deepsize::Context) -> usize { + match self { + Self::Shared { .. } => 0, + Self::Owned(values) => values.deep_size_of_children(context), + } + } +} + +/// A compact address-sorted view of the live DocIds in a projection. +/// +/// Most newly built partitions already store addresses in DocId order, so the +/// identity variant adds no per-document memory. Remapped or otherwise +/// unsorted projections keep only a u32 permutation instead of duplicating +/// the u64 address column. +#[derive(Debug)] +enum AddressDocIdLookup { + Identity, + Sorted(Box<[u32]>), +} + +impl DeepSizeOf for AddressDocIdLookup { + fn deep_size_of_children(&self, _context: &mut lance_core::deepsize::Context) -> usize { + match self { + Self::Identity => 0, + Self::Sorted(doc_ids) => std::mem::size_of_val(doc_ids.as_ref()), + } + } +} + +impl AddressDocIdLookup { + fn build(projection: &ResidentAddressProjection) -> Self { + if projection.projection.live_docs.is_none() + && (1..projection.len()).all(|index| { + projection.stored_address(index - 1) <= projection.stored_address(index) + }) + { + return Self::Identity; + } + + let mut doc_ids = match projection.projection.live_docs.as_ref() { + Some(live_docs) => live_docs.iter().collect::>(), + None => (0..projection.len() as u32).collect::>(), + }; + doc_ids.sort_unstable_by_key(|&doc_id| projection.stored_address(doc_id as usize)); + Self::Sorted(doc_ids.into_boxed_slice()) + } + + fn len(&self, projection: &ResidentAddressProjection) -> usize { + match self { + Self::Identity => projection.len(), + Self::Sorted(doc_ids) => doc_ids.len(), + } + } + + fn doc_id_at(&self, position: usize) -> u32 { + match self { + Self::Identity => position as u32, + Self::Sorted(doc_ids) => doc_ids[position], + } + } + + fn address_at(&self, projection: &ResidentAddressProjection, position: usize) -> u64 { + projection.stored_address(self.doc_id_at(position) as usize) + } + + fn partition_point( + &self, + projection: &ResidentAddressProjection, + mut predicate: impl FnMut(u64) -> bool, + ) -> usize { + let mut left = 0; + let mut right = self.len(projection); + while left < right { + let middle = left + (right - left) / 2; + if predicate(self.address_at(projection, middle)) { + left = middle + 1; + } else { + right = middle; + } + } + left + } + + fn insert_address_range( + &self, + projection: &ResidentAddressProjection, + start: u64, + end: u64, + selected: &mut RoaringBitmap, + ) { + let first = self.partition_point(projection, |address| address < start); + let after_last = self.partition_point(projection, |address| address <= end); + for position in first..after_last { + selected.insert(self.doc_id_at(position)); + } + } + + fn matching_doc_ids( + &self, + projection: &ResidentAddressProjection, + addresses: &RowAddrTreeMap, + ) -> RoaringBitmap { + let mut selected = RoaringBitmap::new(); + for (&fragment_id, selection) in addresses.iter() { + match selection { + RowAddrSelection::Full => { + let start = u64::from(RowAddress::new_from_parts(fragment_id, 0)); + let end = u64::from(RowAddress::new_from_parts(fragment_id, u32::MAX)); + self.insert_address_range(projection, start, end, &mut selected); + } + RowAddrSelection::Partial(offsets) => { + let mut offsets = offsets.iter(); + while let Some(range) = offsets.next_range() { + let start = + u64::from(RowAddress::new_from_parts(fragment_id, *range.start())); + let end = u64::from(RowAddress::new_from_parts(fragment_id, *range.end())); + self.insert_address_range(projection, start, end, &mut selected); + } + } + } + } + selected + } + + fn visibility( + &self, + projection: &ResidentAddressProjection, + mask: &RowAddrMask, + ) -> DocVisibility { + match mask { + RowAddrMask::AllowList(allowed) => { + DocVisibility::Selected(self.matching_doc_ids(projection, allowed)) + } + RowAddrMask::BlockList(blocked) => { + let blocked = self.matching_doc_ids(projection, blocked); + let mut selected = projection.live_doc_ids(); + selected -= &blocked; + DocVisibility::Selected(selected) + } + } + } +} + +/// Addresses projected into the dataset version that opened the index. +#[derive(Debug)] +pub(super) struct VersionAddressProjection { + addresses: AddressValues, + /// `None` means every slot is live. Deleted documents retain their DocId + /// slot but are absent from this bitmap. + live_docs: Option, + doc_ids_by_address: OnceCell>, +} + +/// A query-scoped projection guard. Shared addresses remain alive only while +/// this guard or their independently weighed cache entry owns the Arrow column. +#[derive(Debug, Clone)] +pub(super) struct ResidentAddressProjection { + projection: Arc, + addresses: ResidentAddressValues, +} + +#[derive(Debug, Clone)] +enum ResidentAddressValues { + Shared(Arc), + Owned(Arc>), +} + +impl DeepSizeOf for VersionAddressProjection { + fn deep_size_of_children(&self, context: &mut lance_core::deepsize::Context) -> usize { + self.addresses.deep_size_of_children(context) + + self + .live_docs + .as_ref() + .map(|live| live.serialized_size()) + .unwrap_or(0) + + self + .doc_ids_by_address + .get() + .map(|lookup| lookup.deep_size_of_children(context)) + .unwrap_or(0) + } +} + +impl VersionAddressProjection { + fn try_new( + raw: &UInt64Array, + expected_num_docs: usize, + remapper: Option<&dyn RowIdRemapper>, + path: &str, + ) -> Result { + if raw.len() != expected_num_docs { + return Err(corrupt_docs( + path, + format!( + "{ROW_ID} has {} rows but the file footer reports {expected_num_docs}", + raw.len() + ), + )); + } + if raw.null_count() != 0 { + return Err(corrupt_docs(path, format!("{ROW_ID} contains null values"))); + } + + let Some(remapper) = remapper else { + return Ok(Self { + addresses: AddressValues::Shared { len: raw.len() }, + live_docs: None, + doc_ids_by_address: OnceCell::new(), + }); + }; + + let mut addresses = Vec::with_capacity(raw.len()); + let mut live_docs = RoaringBitmap::new(); + for (doc_id, &address) in raw.values().iter().enumerate() { + match remapper.remap_row_id(address) { + Some(current) => { + addresses.push(current); + live_docs.insert(doc_id as u32); + } + None => { + // The value in a dead slot is intentionally meaningless; + // callers must consult `live_docs` before reading it. + addresses.push(0); + } + } + } + Ok(Self { + addresses: AddressValues::Owned(Arc::new(addresses)), + live_docs: Some(live_docs), + doc_ids_by_address: OnceCell::new(), + }) + } + + fn resident( + self: &Arc, + shared_addresses: Option>, + ) -> Option { + match &self.addresses { + AddressValues::Shared { len } => { + let shared_addresses = shared_addresses?; + debug_assert_eq!(shared_addresses.len(), *len); + Some(ResidentAddressProjection { + projection: self.clone(), + addresses: ResidentAddressValues::Shared(shared_addresses), + }) + } + AddressValues::Owned(values) => Some(ResidentAddressProjection { + projection: self.clone(), + addresses: ResidentAddressValues::Owned(values.clone()), + }), + } + } +} + +impl ResidentAddressProjection { + fn len(&self) -> usize { + self.projection.addresses.len() + } + + fn stored_address(&self, index: usize) -> u64 { + match &self.addresses { + ResidentAddressValues::Shared(values) => values.value(index), + ResidentAddressValues::Owned(values) => values[index], + } + } + + pub(super) fn address(&self, doc_id: DocId) -> Option { + if self + .projection + .live_docs + .as_ref() + .is_some_and(|live| !live.contains(doc_id.get())) + { + None + } else { + Some(self.stored_address(doc_id.as_usize())) + } + } + + fn live_doc_ids(&self) -> RoaringBitmap { + self.projection + .live_docs + .clone() + .unwrap_or_else(|| (0..self.len() as u32).collect()) + } + + async fn doc_ids_by_address(&self) -> Result> { + self.projection + .doc_ids_by_address + .get_or_try_init(|| { + let projection = self.clone(); + async move { + spawn_cpu(move || Result::Ok(Arc::new(AddressDocIdLookup::build(&projection)))) + .await + } + }) + .await + .cloned() + } + + async fn materialize_visibility(self, mask: Arc) -> Result { + let lookup = self.doc_ids_by_address().await?; + let projection = self; + spawn_cpu(move || Result::Ok(lookup.visibility(&projection, &mask))).await + } +} + +/// Query-local selection in the partition-local DocId domain. +#[derive(Debug, Clone)] +pub(super) enum DocVisibility { + All, + Selected(RoaringBitmap), + Filtered { + projection: ResidentAddressProjection, + mask: Arc, + }, +} + +impl DocVisibility { + pub(crate) fn is_all(&self) -> bool { + matches!(self, Self::All) + } + + #[inline] + pub(crate) fn selected(&self, doc_id: DocId) -> bool { + match self { + Self::All => true, + Self::Selected(selected) => selected.contains(doc_id.get()), + Self::Filtered { projection, mask } => projection + .address(doc_id) + .is_some_and(|address| mask.selected(address)), + } + } + + pub(crate) fn len(&self, total_docs: usize) -> usize { + match self { + Self::All => total_docs, + Self::Selected(selected) => selected.len() as usize, + Self::Filtered { .. } => total_docs, + } + } + + pub(crate) fn is_empty(&self) -> bool { + matches!(self, Self::Selected(selected) if selected.is_empty()) + } + + pub(crate) fn iter(&self) -> Option + '_> { + match self { + Self::Selected(selected) => Some(selected.iter().map(DocId::new)), + Self::All | Self::Filtered { .. } => None, + } + } +} + +/// Modern query-side document state for one partition. +pub(super) struct PartitionDocuments { + store: Arc, + path: String, + partition_id: u64, + index_cache: WeakLanceCache, + num_docs: usize, + coordinate_rank: usize, + persisted_total_tokens: Option, + quantized_scoring: bool, + remapper: Option>, + lengths: OnceCell>, + projection: OnceCell>, + shared_addresses: ArcSwapWeak, + prewarm_complete: OnceCell<()>, +} + +/// Load-boundary discriminator between the read-only legacy representation and +/// the typed partitioned representation. Query code dispatches on this enum +/// once; modern scoring never receives a partial legacy [`DocSet`]. +#[derive(Debug, Clone)] +pub(super) enum PartitionDocumentStore { + Legacy(Arc), + Modern(Arc), +} + +impl DeepSizeOf for PartitionDocumentStore { + fn deep_size_of_children(&self, context: &mut lance_core::deepsize::Context) -> usize { + match self { + Self::Legacy(docs) => docs.deep_size_of_children(context), + Self::Modern(docs) => docs.deep_size_of_children(context), + } + } +} + +impl PartitionDocumentStore { + pub(crate) fn len(&self) -> usize { + match self { + Self::Legacy(docs) => docs.len(), + Self::Modern(docs) => docs.len(), + } + } + + pub(crate) fn coordinate_rank(&self) -> usize { + match self { + Self::Legacy(docs) => docs.coordinate_rank(), + Self::Modern(docs) => docs.coordinate_rank(), + } + } + + pub(crate) fn legacy(&self) -> Option<&Arc> { + match self { + Self::Legacy(docs) => Some(docs), + Self::Modern(_) => None, + } + } + + pub(crate) fn modern(&self) -> Option<&Arc> { + match self { + Self::Legacy(_) => None, + Self::Modern(docs) => Some(docs), + } + } + + pub(crate) async fn stats(&self) -> Result { + match self { + Self::Legacy(docs) => Ok(PartitionStats { + num_docs: docs.len(), + total_tokens: docs.total_tokens_num(), + }), + Self::Modern(docs) => docs.stats().await, + } + } + + pub(crate) fn cached_stats(&self) -> Option { + match self { + Self::Legacy(docs) => Some(PartitionStats { + num_docs: docs.len(), + total_tokens: docs.total_tokens_num(), + }), + Self::Modern(docs) => docs.cached_stats(), + } + } + + pub(crate) async fn prewarm(&self) -> Result<()> { + match self { + Self::Legacy(_) => Ok(()), + Self::Modern(docs) => docs.prewarm().await, + } + } + + #[cfg(test)] + pub(crate) fn query_ready(&self) -> bool { + match self { + Self::Legacy(_) => true, + Self::Modern(docs) => docs.query_ready(), + } + } + + pub(crate) fn prewarm_status(&self) -> FtsPrewarmDocumentStatus { + match self { + Self::Legacy(_) => FtsPrewarmDocumentStatus { + prewarm_complete: true, + scoring_ready: true, + reverse_lookup_ready: true, + projection_resident: true, + }, + Self::Modern(docs) => docs.prewarm_status(), + } + } + + pub(crate) async fn load_build_docset(&self) -> Result { + match self { + Self::Legacy(docs) => Ok((**docs).clone()), + Self::Modern(docs) => docs.load_build_docset().await, + } + } +} + +impl std::fmt::Debug for PartitionDocuments { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("PartitionDocuments") + .field("path", &self.path) + .field("num_docs", &self.num_docs) + .field("coordinate_rank", &self.coordinate_rank) + .field("persisted_total_tokens", &self.persisted_total_tokens) + .field("lengths_loaded", &self.lengths.initialized()) + .field("projection_loaded", &self.projection.initialized()) + .finish() + } +} + +impl DeepSizeOf for PartitionDocuments { + fn deep_size_of_children(&self, context: &mut lance_core::deepsize::Context) -> usize { + self.lengths + .get() + .map(|lengths| lengths.deep_size_of_children(context)) + .unwrap_or(0) + + self + .projection + .get() + .map(|projection| projection.deep_size_of_children(context)) + .unwrap_or(0) + } +} + +impl PartitionDocuments { + pub(crate) fn try_new( + store: Arc, + path: String, + partition_id: u64, + index_cache: WeakLanceCache, + reader: &dyn IndexReader, + remapper: Option>, + quantized_scoring: bool, + ) -> Result { + let num_docs = reader.num_rows(); + if num_docs > u32::MAX as usize { + return Err(corrupt_docs( + &path, + format!("document count {num_docs} exceeds dense DocId capacity"), + )); + } + let persisted_total_tokens = reader + .schema() + .metadata + .get(TOTAL_TOKENS_KEY) + .map(|value| { + value.parse::().map_err(|error| { + corrupt_docs( + &path, + format!("invalid {TOTAL_TOKENS_KEY} metadata value {value:?}: {error}"), + ) + }) + }) + .transpose()?; + let coordinate_rank = + document_coordinate_rank(&arrow_schema::Schema::from(reader.schema())); + Ok(Self { + store, + path, + partition_id, + index_cache, + num_docs, + coordinate_rank, + persisted_total_tokens, + quantized_scoring, + remapper, + lengths: OnceCell::new(), + projection: OnceCell::new(), + shared_addresses: ArcSwapWeak::from(Weak::new()), + prewarm_complete: OnceCell::new(), + }) + } + + pub(crate) fn len(&self) -> usize { + self.num_docs + } + + pub(crate) fn coordinate_rank(&self) -> usize { + self.coordinate_rank + } + + #[cfg(test)] + pub(crate) fn lengths_loaded(&self) -> bool { + self.lengths.initialized() + } + + #[cfg(test)] + pub(crate) fn projection_loaded(&self) -> bool { + self.projection.initialized() + } + + #[cfg(test)] + pub(crate) fn address_buffer_handle(&self) -> Weak { + self.shared_addresses.load_full() + } + + pub(crate) fn projection_resident(&self) -> bool { + self.resident_address_projection().is_some() + } + + #[cfg(test)] + pub(crate) fn query_ready(&self) -> bool { + self.prewarm_status().query_ready() + } + + pub(crate) fn prewarm_status(&self) -> FtsPrewarmDocumentStatus { + FtsPrewarmDocumentStatus { + prewarm_complete: self.prewarm_complete.initialized(), + scoring_ready: self + .lengths + .get() + .is_some_and(|lengths| lengths.scoring_ready()), + reverse_lookup_ready: self + .projection + .get() + .is_some_and(|projection| projection.doc_ids_by_address.initialized()), + projection_resident: self.projection_resident(), + } + } + + async fn reader(&self) -> Result> { + self.store.open_index_file(&self.path).await + } + + async fn row_ids_column(&self) -> Result> { + let store = self.store.clone(); + let path = self.path.clone(); + let num_docs = self.num_docs; + let cached = self + .index_cache + .get_or_insert_with_key( + DocRowIdsKey { + partition_id: self.partition_id, + }, + || async move { + let reader = store.open_index_file(&path).await?; + let batch = reader.read_range(0..num_docs, Some(&[ROW_ID])).await?; + let row_ids = required_u64_column(&batch, ROW_ID, &path)?; + if row_ids.null_count() != 0 { + return Err(corrupt_docs( + &path, + format!("{ROW_ID} contains null values"), + )); + } + if row_ids.len() != num_docs { + return Err(corrupt_docs( + &path, + format!( + "{ROW_ID} has {} rows but the file footer reports {num_docs}", + row_ids.len() + ), + )); + } + Ok(CachedDocRowIds { + row_ids: Arc::new(row_ids.clone()), + }) + }, + ) + .await?; + let row_ids = cached.row_ids.clone(); + self.shared_addresses.store(Arc::downgrade(&row_ids)); + Ok(row_ids) + } + + fn lengths_from_batch(&self, batch: &RecordBatch) -> Result> { + let column = required_u32_column(batch, NUM_TOKEN_COL, &self.path)?; + if column.null_count() != 0 { + return Err(corrupt_docs( + &self.path, + format!("{NUM_TOKEN_COL} contains null values"), + )); + } + Ok(Arc::new(DocLengths::try_new( + column.values().clone(), + self.num_docs, + self.persisted_total_tokens, + self.quantized_scoring, + &self.path, + )?)) + } + + pub(crate) async fn stats(&self) -> Result { + let total_tokens = match self.persisted_total_tokens { + Some(total_tokens) => total_tokens, + None => self.lengths().await?.total_tokens(), + }; + Ok(PartitionStats { + num_docs: self.num_docs, + total_tokens, + }) + } + + pub(crate) fn cached_stats(&self) -> Option { + self.persisted_total_tokens + .or_else(|| self.lengths.get().map(|lengths| lengths.total_tokens())) + .map(|total_tokens| PartitionStats { + num_docs: self.num_docs, + total_tokens, + }) + } + + pub(crate) async fn lengths(&self) -> Result> { + self.lengths + .get_or_try_init(|| async { + let reader = self.reader().await?; + let batch = reader + .read_range(0..self.num_docs, Some(&[NUM_TOKEN_COL])) + .await?; + self.lengths_from_batch(&batch) + }) + .await + .cloned() + } + + /// Return resident lengths without entering the asynchronous singleflight path. + pub(crate) fn cached_lengths(&self) -> Option> { + self.lengths.get().cloned() + } + + pub(super) fn resident_address_projection(&self) -> Option { + let projection = self.projection.get()?.clone(); + let shared_addresses = match &projection.addresses { + AddressValues::Shared { .. } => self.shared_addresses.load().upgrade(), + AddressValues::Owned(_) => None, + }; + projection.resident(shared_addresses) + } + + pub(crate) async fn address_projection(&self) -> Result { + if let Some(projection) = self.resident_address_projection() { + return Ok(projection); + } + + let row_ids = self.row_ids_column().await?; + let projection = self + .projection + .get_or_try_init(|| async { + Result::Ok(Arc::new(VersionAddressProjection::try_new( + row_ids.as_ref(), + self.num_docs, + self.remapper.as_deref(), + &self.path, + )?)) + }) + .await + .cloned()?; + projection.resident(Some(row_ids)).ok_or_else(|| { + Error::internal(format!( + "address projection for {} could not bind its cache-managed ROW_ID column", + self.path + )) + }) + } + + pub(crate) async fn visibility( + &self, + mask: Arc, + materialize_selected: bool, + ) -> Result { + if let Some(visibility) = self.immediate_visibility(mask.clone(), materialize_selected) { + return Ok(visibility); + } + + let projection = self.address_projection().await?; + if mask.is_select_all() { + return Ok(DocVisibility::Selected(projection.live_doc_ids())); + } + if materialize_selected { + projection.materialize_visibility(mask).await + } else { + Ok(DocVisibility::Filtered { projection, mask }) + } + } + + /// Resolve visibility without I/O or CPU-pool work when all required state is resident. + pub(crate) fn immediate_visibility( + &self, + mask: Arc, + materialize_selected: bool, + ) -> Option { + if mask.max_len() == Some(0) { + return Some(DocVisibility::Selected(RoaringBitmap::new())); + } + if mask.is_select_all() && self.remapper.is_none() { + return Some(DocVisibility::All); + } + + let projection = self.resident_address_projection()?; + if mask.is_select_all() { + return Some(DocVisibility::Selected(projection.live_doc_ids())); + } + if materialize_selected { + None + } else { + Some(DocVisibility::Filtered { projection, mask }) + } + } + + /// Resolve final global top-k DocIds to current row addresses. + pub(crate) async fn resolve_addresses(&self, doc_ids: &[DocId]) -> Result> { + if doc_ids.is_empty() { + return Ok(Vec::new()); + } + self.validate_doc_ids(doc_ids)?; + if let Some(projection) = self.resident_address_projection() { + return self.resolve_projected_addresses(&projection, doc_ids); + } + if self.remapper.is_some() { + let projection = self.address_projection().await?; + return self.resolve_projected_addresses(&projection, doc_ids); + } + + let row_ids = self.row_ids_column().await?; + Ok(doc_ids + .iter() + .map(|doc_id| row_ids.value(doc_id.as_usize())) + .collect()) + } + + /// Resolve final global top-k DocIds to their logical FTS document keys. + pub(crate) async fn resolve_document_keys( + &self, + doc_ids: &[DocId], + ) -> Result)>> { + let row_ids = self.resolve_addresses(doc_ids).await?; + if self.coordinate_rank == 0 { + return Ok(row_ids + .into_iter() + .map(|row_id| (row_id, Vec::new())) + .collect()); + } + + let ranges = doc_ids + .iter() + .map(|doc_id| { + let index = doc_id.as_usize(); + index..index + 1 + }) + .collect::>(); + let coordinate_names = (0..self.coordinate_rank) + .map(doc_index_storage_column) + .collect::>(); + let projection = coordinate_names + .iter() + .map(String::as_str) + .collect::>(); + let batch = self + .reader() + .await? + .read_ranges(&ranges, Some(&projection)) + .await?; + if batch.num_rows() != row_ids.len() { + return Err(corrupt_docs( + &self.path, + format!( + "document coordinate projection returned {} rows for {} candidates", + batch.num_rows(), + row_ids.len() + ), + )); + } + let coordinate_columns = coordinate_names + .iter() + .map(|name| { + let column = required_u32_column(&batch, name, &self.path)?; + if column.null_count() != 0 { + return Err(corrupt_docs( + &self.path, + format!("document coordinate column {name} contains null values"), + )); + } + Ok(column) + }) + .collect::>>()?; + + Ok(row_ids + .into_iter() + .enumerate() + .map(|(index, row_id)| { + ( + row_id, + coordinate_columns + .iter() + .map(|column| column.value(index)) + .collect(), + ) + }) + .collect()) + } + + fn validate_doc_ids(&self, doc_ids: &[DocId]) -> Result<()> { + for doc_id in doc_ids { + if doc_id.as_usize() >= self.num_docs { + return Err(corrupt_docs( + &self.path, + format!( + "candidate DocId {} is outside [0, {})", + doc_id.get(), + self.num_docs + ), + )); + } + } + Ok(()) + } + + fn resolve_projected_addresses( + &self, + projection: &ResidentAddressProjection, + doc_ids: &[DocId], + ) -> Result> { + doc_ids + .iter() + .map(|&doc_id| { + projection.address(doc_id).ok_or_else(|| { + corrupt_docs( + &self.path, + format!("candidate DocId {} is not live", doc_id.get()), + ) + }) + }) + .collect() + } + + /// Resolve addresses synchronously when the cache-managed projection buffer + /// is resident. The returned `None` asks the caller to use the async reload path. + pub(crate) fn cached_row_addresses(&self, doc_ids: &[DocId]) -> Result>> { + self.validate_doc_ids(doc_ids)?; + let Some(projection) = self.resident_address_projection() else { + return Ok(None); + }; + self.resolve_projected_addresses(&projection, doc_ids) + .map(Some) + } + + /// Estimated Arrow payload retained while loading this partition's cached + /// row-address column. + /// The estimate is used to cap cross-partition read concurrency; a single + /// oversized partition is still allowed to make progress. + pub(crate) fn estimated_address_read_bytes(&self, doc_ids: &[DocId]) -> usize { + if doc_ids.is_empty() || self.projection_resident() { + return 0; + } + self.num_docs.saturating_mul(std::mem::size_of::()) + } + + /// Materialize the build-side table for rewrite/update operations. + pub(crate) async fn load_build_docset(&self) -> Result { + DocSet::load(self.reader().await?, false, self.remapper.clone()).await + } + + pub(crate) async fn prewarm(&self) -> Result<()> { + self.prewarm_complete + .get_or_try_init(|| async { + if self.lengths.get().is_none() && self.projection.get().is_none() { + let reader = self.reader().await?; + let batch = reader + .read_range(0..self.num_docs, Some(&[ROW_ID, NUM_TOKEN_COL])) + .await?; + let lengths = self.lengths_from_batch(&batch)?; + let row_ids = + Arc::new(required_u64_column(&batch, ROW_ID, &self.path)?.clone()); + if row_ids.null_count() != 0 { + return Err(corrupt_docs( + &self.path, + format!("{ROW_ID} contains null values"), + )); + } + let projection = Arc::new(VersionAddressProjection::try_new( + row_ids.as_ref(), + self.num_docs, + self.remapper.as_deref(), + &self.path, + )?); + let cached_row_ids = Arc::new(CachedDocRowIds { + row_ids: row_ids.clone(), + }); + self.index_cache + .insert_with_key( + &DocRowIdsKey { + partition_id: self.partition_id, + }, + cached_row_ids, + ) + .await; + self.shared_addresses.store(Arc::downgrade(&row_ids)); + + // A concurrent single-column request may win either OnceCell. + // Awaiting the accessors below joins that initialization without + // replacing the already-published value. + let _ = self.lengths.set(lengths); + let _ = self.projection.set(projection); + } + + let lengths = self.lengths().await?; + spawn_cpu(move || { + let _ = lengths.scoring_norms(); + Result::Ok(()) + }) + .await?; + self.address_projection() + .await? + .doc_ids_by_address() + .await?; + Result::Ok(()) + }) + .await?; + if !self.projection_resident() { + self.address_projection().await?; + } + Ok(()) + } +} + +fn required_u32_column<'a>( + batch: &'a RecordBatch, + name: &str, + path: &str, +) -> Result<&'a UInt32Array> { + let column = batch + .column_by_name(name) + .ok_or_else(|| corrupt_docs(path, format!("required column {name} is missing")))?; + column + .as_any() + .downcast_ref::() + .ok_or_else(|| { + corrupt_docs( + path, + format!( + "column {name} has type {}, expected UInt32", + column.data_type() + ), + ) + }) +} + +fn required_u64_column<'a>( + batch: &'a RecordBatch, + name: &str, + path: &str, +) -> Result<&'a UInt64Array> { + let column = batch + .column_by_name(name) + .ok_or_else(|| corrupt_docs(path, format!("required column {name} is missing")))?; + column + .as_any() + .downcast_ref::() + .ok_or_else(|| { + corrupt_docs( + path, + format!( + "column {name} has type {}, expected UInt64", + column.data_type() + ), + ) + }) +} + +fn corrupt_docs(path: &str, message: impl Into) -> Error { + Error::corrupt_file(Path::from(path), message) +} + +#[cfg(test)] +mod tests { + use std::collections::HashMap; + use std::ops::Range; + use std::sync::atomic::{AtomicUsize, Ordering}; + use std::time::Duration; + + use arrow_array::{ArrayRef, RecordBatch}; + use arrow_schema::{DataType, Field, Schema}; + use async_trait::async_trait; + use lance_core::cache::{LanceCache, QuickCacheBackend}; + use lance_core::utils::tempfile::TempObjDir; + use lance_io::object_store::ObjectStore; + use lance_select::RowAddrTreeMap; + use roaring::RoaringTreemap; + use tokio::sync::Notify; + + use crate::scalar::lance_format::LanceIndexStore; + use crate::scalar::{IndexFile, IndexWriter}; + + use super::*; + + #[derive(Debug, Default)] + struct DocumentReadCounts { + open_calls: AtomicUsize, + range_calls: AtomicUsize, + ranges_calls: AtomicUsize, + rows: AtomicUsize, + length_rows: AtomicUsize, + address_rows: AtomicUsize, + } + + impl DocumentReadCounts { + fn record(&self, rows: usize, projection: Option<&[&str]>) { + self.rows.fetch_add(rows, Ordering::Relaxed); + if projection.is_none_or(|columns| columns.contains(&NUM_TOKEN_COL)) { + self.length_rows.fetch_add(rows, Ordering::Relaxed); + } + if projection.is_none_or(|columns| columns.contains(&ROW_ID)) { + self.address_rows.fetch_add(rows, Ordering::Relaxed); + } + } + } + + const PAUSE_ONCE: usize = 1; + const FAIL_ONCE: usize = 2; + + #[derive(Debug, Default)] + struct ReadFault { + action: AtomicUsize, + started: Notify, + } + + impl ReadFault { + async fn apply(&self) -> Result<()> { + match self.action.swap(0, Ordering::AcqRel) { + PAUSE_ONCE => { + self.started.notify_one(); + std::future::pending::>().await + } + FAIL_ONCE => Err(Error::io("injected document read failure")), + _ => Ok(()), + } + } + } + + struct CountingReader { + inner: Arc, + counts: Arc, + fault: Option>, + } + + #[async_trait] + impl IndexReader for CountingReader { + async fn read_record_batch(&self, n: u64, batch_size: u64) -> Result { + self.inner.read_record_batch(n, batch_size).await + } + + async fn read_global_buffer(&self, index: u32) -> Result { + self.inner.read_global_buffer(index).await + } + + async fn read_range( + &self, + range: Range, + projection: Option<&[&str]>, + ) -> Result { + self.counts.range_calls.fetch_add(1, Ordering::Relaxed); + self.counts.record(range.len(), projection); + if let Some(fault) = &self.fault { + fault.apply().await?; + } + self.inner.read_range(range, projection).await + } + + async fn read_ranges( + &self, + ranges: &[Range], + projection: Option<&[&str]>, + ) -> Result { + self.counts.ranges_calls.fetch_add(1, Ordering::Relaxed); + self.counts + .record(ranges.iter().map(Range::len).sum(), projection); + if let Some(fault) = &self.fault { + fault.apply().await?; + } + self.inner.read_ranges(ranges, projection).await + } + + async fn num_batches(&self, batch_size: u64) -> u32 { + self.inner.num_batches(batch_size).await + } + + fn num_rows(&self) -> usize { + self.inner.num_rows() + } + + fn schema(&self) -> &lance_core::datatypes::Schema { + self.inner.schema() + } + + fn file_size_bytes(&self) -> Option { + self.inner.file_size_bytes() + } + } + + #[derive(Debug)] + struct CountingStore { + inner: Arc, + target: String, + counts: Arc, + fault: Option>, + } + + impl DeepSizeOf for CountingStore { + fn deep_size_of_children(&self, context: &mut lance_core::deepsize::Context) -> usize { + self.inner.deep_size_of_children(context) + } + } + + #[async_trait] + impl IndexStore for CountingStore { + fn as_any(&self) -> &dyn std::any::Any { + self + } + + fn clone_arc(&self) -> Arc { + Arc::new(Self { + inner: self.inner.clone(), + target: self.target.clone(), + counts: self.counts.clone(), + fault: self.fault.clone(), + }) + } + + fn io_parallelism(&self) -> usize { + self.inner.io_parallelism() + } + + async fn new_index_file( + &self, + name: &str, + schema: Arc, + ) -> Result> { + self.inner.new_index_file(name, schema).await + } + + async fn open_index_file(&self, name: &str) -> Result> { + let reader = self.inner.open_index_file(name).await?; + if name == self.target { + self.counts.open_calls.fetch_add(1, Ordering::Relaxed); + Ok(Arc::new(CountingReader { + inner: reader, + counts: self.counts.clone(), + fault: self.fault.clone(), + })) + } else { + Ok(reader) + } + } + + fn with_io_priority(&self, io_priority: u64) -> Arc { + Arc::new(Self { + inner: self.inner.with_io_priority(io_priority), + target: self.target.clone(), + counts: self.counts.clone(), + fault: self.fault.clone(), + }) + } + + async fn copy_index_file( + &self, + name: &str, + dest_store: &dyn IndexStore, + ) -> Result { + self.inner.copy_index_file(name, dest_store).await + } + + async fn copy_index_file_to( + &self, + name: &str, + new_name: &str, + dest_store: &dyn IndexStore, + ) -> Result { + self.inner + .copy_index_file_to(name, new_name, dest_store) + .await + } + + async fn rename_index_file(&self, name: &str, new_name: &str) -> Result { + self.inner.rename_index_file(name, new_name).await + } + + async fn delete_index_file(&self, name: &str) -> Result<()> { + self.inner.delete_index_file(name).await + } + + async fn list_files_with_sizes(&self) -> Result> { + self.inner.list_files_with_sizes().await + } + } + + fn test_store() -> (TempObjDir, Arc, Arc) { + let directory = TempObjDir::default(); + let cache = Arc::new(LanceCache::with_capacity(1024 * 1024)); + test_store_with_cache(directory, cache) + } + + fn eviction_test_store() -> (TempObjDir, Arc, Arc) { + let directory = TempObjDir::default(); + let cache = Arc::new(LanceCache::with_backend(Arc::new( + QuickCacheBackend::with_capacity(1024 * 1024), + ))); + test_store_with_cache(directory, cache) + } + + fn test_store_with_cache( + directory: TempObjDir, + cache: Arc, + ) -> (TempObjDir, Arc, Arc) { + let store = Arc::new(LanceIndexStore::new( + ObjectStore::local().into(), + directory.clone(), + cache.clone(), + )); + (directory, store, cache) + } + + async fn write_documents( + store: &dyn IndexStore, + path: &str, + addresses: UInt64Array, + lengths: UInt32Array, + total_tokens: Option<&str>, + ) { + let schema = Arc::new(Schema::new(vec![ + Field::new(ROW_ID, DataType::UInt64, addresses.null_count() != 0), + Field::new(NUM_TOKEN_COL, DataType::UInt32, lengths.null_count() != 0), + ])); + let batch = RecordBatch::try_new( + schema.clone(), + vec![ + Arc::new(addresses) as ArrayRef, + Arc::new(lengths) as ArrayRef, + ], + ) + .unwrap(); + let mut writer = store.new_index_file(path, schema).await.unwrap(); + writer.write_record_batch(batch).await.unwrap(); + if let Some(total_tokens) = total_tokens { + writer + .finish_with_metadata(HashMap::from([( + TOTAL_TOKENS_KEY.to_owned(), + total_tokens.to_owned(), + )])) + .await + .unwrap(); + } else { + writer.finish().await.unwrap(); + } + } + + async fn open_documents( + store: Arc, + path: &str, + index_cache: &LanceCache, + remapper: Option>, + ) -> Result { + let reader = store.open_index_file(path).await?; + PartitionDocuments::try_new( + store, + path.to_owned(), + 0, + WeakLanceCache::from(index_cache), + reader.as_ref(), + remapper, + false, + ) + } + + fn counted_store( + inner: Arc, + target: &str, + ) -> (Arc, Arc) { + let counts = Arc::new(DocumentReadCounts::default()); + ( + Arc::new(CountingStore { + inner, + target: target.to_owned(), + counts: counts.clone(), + fault: None, + }), + counts, + ) + } + + fn faulting_store( + inner: Arc, + target: &str, + action: usize, + ) -> (Arc, Arc) { + let fault = Arc::new(ReadFault { + action: AtomicUsize::new(action), + started: Notify::new(), + }); + ( + Arc::new(CountingStore { + inner, + target: target.to_owned(), + counts: Arc::new(DocumentReadCounts::default()), + fault: Some(fault.clone()), + }), + fault, + ) + } + + #[derive(Debug)] + struct TestRemapper { + mapping: HashMap>, + } + + impl RowIdRemapper for TestRemapper { + fn remap_row_id(&self, row_id: u64) -> Option { + self.mapping.get(&row_id).copied().unwrap_or(Some(row_id)) + } + + fn remap_row_addrs_tree_map(&self, _: &RowAddrTreeMap) -> RowAddrTreeMap { + unreachable!("not used by document projection tests") + } + + fn remap_row_ids_roaring_tree_map(&self, _: &RoaringTreemap) -> RoaringTreemap { + unreachable!("not used by document projection tests") + } + + fn remap_row_ids_record_batch(&self, _: RecordBatch, _: usize) -> Result { + unreachable!("not used by document projection tests") + } + } + + #[test] + fn required_document_columns_validate_name_and_type() { + let schema = Arc::new(Schema::new(vec![Field::new( + NUM_TOKEN_COL, + DataType::UInt64, + false, + )])); + let batch = RecordBatch::try_new( + schema, + vec![Arc::new(UInt64Array::from(vec![1])) as ArrayRef], + ) + .unwrap(); + + let missing = required_u64_column(&batch, ROW_ID, "docs.lance").unwrap_err(); + assert!( + missing + .to_string() + .contains("required column _rowid is missing") + ); + let wrong_type = required_u32_column(&batch, NUM_TOKEN_COL, "docs.lance").unwrap_err(); + assert!(wrong_type.to_string().contains("expected UInt32")); + } + + #[test] + fn live_documents_use_doc_ids() { + let projection = Arc::new(VersionAddressProjection { + addresses: AddressValues::Owned(Arc::new(vec![10, 20, 30])), + live_docs: Some(RoaringBitmap::from_iter([0, 2])), + doc_ids_by_address: OnceCell::new(), + }); + let projection = projection.resident(None).unwrap(); + let selected = projection.live_doc_ids(); + assert_eq!(selected.iter().collect::>(), vec![0, 2]); + } + + #[tokio::test] + async fn remap_preserves_doc_id_slots_and_filters_in_current_address_domain() { + let raw = UInt64Array::from(vec![10, 20, 30, 40]); + let remapper = TestRemapper { + mapping: HashMap::from([(10, Some(100)), (20, None), (30, Some(300))]), + }; + let projection = Arc::new( + VersionAddressProjection::try_new(&raw, 4, Some(&remapper), "docs") + .expect("valid projection"), + ); + let projection = projection.resident(None).unwrap(); + + assert_eq!(projection.address(DocId::new(0)), Some(100)); + assert_eq!(projection.address(DocId::new(1)), None); + assert_eq!(projection.address(DocId::new(2)), Some(300)); + assert_eq!(projection.address(DocId::new(3)), Some(40)); + + let all_live = projection.live_doc_ids(); + assert_eq!(all_live.iter().collect::>(), vec![0, 2, 3]); + + let allowed = Arc::new(RowAddrMask::from_allowed(RowAddrTreeMap::from_iter([ + 100, 40, + ]))); + let DocVisibility::Selected(selected) = projection + .clone() + .materialize_visibility(allowed) + .await + .expect("valid allow-list") + else { + panic!("allow-list must compile to DocIds") + }; + assert_eq!(selected.iter().collect::>(), vec![0, 3]); + + let first_lookup = projection + .doc_ids_by_address() + .await + .expect("cached address lookup"); + let blocked = Arc::new(RowAddrMask::from_block(RowAddrTreeMap::from_iter([300]))); + let DocVisibility::Selected(selected) = projection + .clone() + .materialize_visibility(blocked) + .await + .expect("valid block-list") + else { + panic!("block-list must compile to DocIds") + }; + assert_eq!(selected.iter().collect::>(), vec![0, 3]); + let second_lookup = projection + .doc_ids_by_address() + .await + .expect("cached address lookup"); + assert!(Arc::ptr_eq(&first_lookup, &second_lookup)); + } + + #[tokio::test] + async fn materialized_visibility_handles_unsorted_duplicate_addresses_and_full_fragments() { + let row_address = |fragment_id, row_offset| { + u64::from(RowAddress::new_from_parts(fragment_id, row_offset)) + }; + let projection = Arc::new(VersionAddressProjection { + addresses: AddressValues::Owned(Arc::new(vec![ + row_address(2, 5), + row_address(1, 2), + row_address(1, 2), + row_address(1, 4), + ])), + live_docs: None, + doc_ids_by_address: OnceCell::new(), + }); + let projection = projection.resident(None).unwrap(); + + let allowed = Arc::new(RowAddrMask::from_allowed(RowAddrTreeMap::from_iter([ + row_address(1, 2), + row_address(1, 3), + row_address(1, 4), + ]))); + let DocVisibility::Selected(selected) = projection + .clone() + .materialize_visibility(allowed) + .await + .expect("valid allow-list") + else { + panic!("allow-list must compile to DocIds") + }; + assert_eq!(selected.iter().collect::>(), vec![1, 2, 3]); + + let mut full_fragment = RowAddrTreeMap::new(); + full_fragment.insert_fragment(2); + let DocVisibility::Selected(selected) = projection + .materialize_visibility(Arc::new(RowAddrMask::from_allowed(full_fragment))) + .await + .expect("valid full-fragment allow-list") + else { + panic!("allow-list must compile to DocIds") + }; + assert_eq!(selected.iter().collect::>(), vec![0]); + } + + #[test] + fn lazy_visibility_projects_only_candidate_doc_ids() { + let projection = Arc::new(VersionAddressProjection { + addresses: AddressValues::Owned(Arc::new(vec![10, 20, 30])), + live_docs: Some(RoaringBitmap::from_iter([0, 2])), + doc_ids_by_address: OnceCell::new(), + }); + let resident = projection.resident(None).unwrap(); + let visibility = DocVisibility::Filtered { + projection: resident, + mask: Arc::new(RowAddrMask::from_allowed(RowAddrTreeMap::from_iter([ + 20, 30, + ]))), + }; + + assert!(!visibility.selected(DocId::new(0))); + assert!(!visibility.selected(DocId::new(1))); + assert!(visibility.selected(DocId::new(2))); + assert!(!projection.doc_ids_by_address.initialized()); + } + + #[test] + fn doc_lengths_validate_shape_total_and_memory() { + let mismatch = DocLengths::try_new(ScalarBuffer::from(vec![2, 3]), 3, None, false, "docs") + .unwrap_err(); + assert!(mismatch.to_string().contains("2 rows")); + + let mismatch = DocLengths::try_new( + ScalarBuffer::from(vec![2, 3, 5]), + 3, + Some(11), + false, + "docs", + ) + .unwrap_err(); + assert!(mismatch.to_string().contains("sums to 10")); + + let lengths = + DocLengths::try_new(ScalarBuffer::from(vec![2, 3, 5]), 3, Some(10), true, "docs") + .unwrap(); + let before_norms = lengths.deep_size_of(); + assert_eq!(lengths.total_tokens(), 10); + assert_eq!(lengths.scoring_norms().unwrap().len(), 3); + assert_eq!(lengths.deep_size_of() - before_norms, 3); + } + + #[tokio::test] + async fn persisted_stats_are_footer_only_and_compatible_with_full_docset_reader() { + let (_directory, store, cache) = test_store(); + let path = "docs.lance"; + write_documents( + store.as_ref(), + path, + UInt64Array::from(vec![10, 20, 30]), + UInt32Array::from(vec![2, 3, 5]), + Some("10"), + ) + .await; + + let reader = store.open_index_file(path).await.unwrap(); + assert_eq!( + reader.schema().metadata.get(TOTAL_TOKENS_KEY), + Some(&"10".to_owned()) + ); + let complete = DocSet::load(reader, false, None).await.unwrap(); + assert_eq!(complete.len(), 3); + assert_eq!(complete.row_id(1), 20); + assert_eq!(complete.total_tokens_num(), 10); + + let (counting, counts) = counted_store(store, path); + let documents = open_documents(counting, path, cache.as_ref(), None) + .await + .unwrap(); + assert_eq!( + documents.stats().await.unwrap(), + PartitionStats { + num_docs: 3, + total_tokens: 10, + } + ); + assert_eq!(counts.rows.load(Ordering::Relaxed), 0); + assert!(!documents.lengths_loaded()); + assert!(!documents.projection_loaded()); + } + + #[tokio::test] + async fn missing_stats_fall_back_once_to_lengths() { + let (_directory, store, cache) = test_store(); + let path = "docs.lance"; + write_documents( + store.as_ref(), + path, + UInt64Array::from(vec![10, 20, 30]), + UInt32Array::from(vec![2, 3, 5]), + None, + ) + .await; + let (counting, counts) = counted_store(store, path); + let documents = open_documents(counting, path, cache.as_ref(), None) + .await + .unwrap(); + + assert_eq!(documents.stats().await.unwrap().total_tokens, 10); + assert_eq!(documents.stats().await.unwrap().total_tokens, 10); + assert_eq!(counts.range_calls.load(Ordering::Relaxed), 1); + assert_eq!(counts.length_rows.load(Ordering::Relaxed), 3); + assert_eq!(counts.address_rows.load(Ordering::Relaxed), 0); + assert!(documents.lengths_loaded()); + assert!(!documents.projection_loaded()); + } + + #[tokio::test] + async fn prewarm_loads_document_columns_and_address_lookup_once() { + let (_directory, store, cache) = test_store(); + let path = "docs.lance"; + write_documents( + store.as_ref(), + path, + UInt64Array::from(vec![10, 20, 30]), + UInt32Array::from(vec![2, 3, 5]), + Some("10"), + ) + .await; + let (counting, counts) = counted_store(store, path); + let documents = open_documents(counting, path, cache.as_ref(), None) + .await + .unwrap(); + + futures::future::join_all((0..8).map(|_| documents.prewarm())) + .await + .into_iter() + .collect::>>() + .unwrap(); + let projection = documents.address_projection().await.unwrap(); + let first_lookup = projection.doc_ids_by_address().await.unwrap(); + documents.prewarm().await.unwrap(); + let second_lookup = documents + .address_projection() + .await + .unwrap() + .doc_ids_by_address() + .await + .unwrap(); + + assert!(documents.lengths_loaded()); + assert!(documents.projection_loaded()); + assert!(documents.query_ready()); + assert_eq!(documents.cached_lengths().unwrap().total_tokens(), 10); + assert!(matches!( + documents.immediate_visibility(Arc::new(RowAddrMask::all_rows()), false), + Some(DocVisibility::All) + )); + assert!(matches!( + first_lookup.as_ref(), + AddressDocIdLookup::Identity + )); + assert!(Arc::ptr_eq(&first_lookup, &second_lookup)); + assert_eq!( + documents.cached_row_addresses(&[DocId::new(1)]).unwrap(), + Some(vec![20]) + ); + assert_eq!(counts.open_calls.load(Ordering::Relaxed), 2); + assert_eq!(counts.range_calls.load(Ordering::Relaxed), 1); + assert_eq!(counts.length_rows.load(Ordering::Relaxed), 3); + assert_eq!(counts.address_rows.load(Ordering::Relaxed), 3); + assert!( + cache + .get_with_key(&DocRowIdsKey { partition_id: 0 }) + .await + .is_some() + ); + } + + #[tokio::test] + async fn filtered_visibility_releases_addresses_after_cache_eviction() { + let (_directory, store, cache) = eviction_test_store(); + let path = "docs.lance"; + write_documents( + store.as_ref(), + path, + UInt64Array::from(vec![10, 20, 30]), + UInt32Array::from(vec![2, 3, 5]), + Some("10"), + ) + .await; + let (counting, counts) = counted_store(store, path); + let documents = open_documents(counting, path, cache.as_ref(), None) + .await + .unwrap(); + let mask = Arc::new(RowAddrMask::from_allowed(RowAddrTreeMap::from_iter([20]))); + + let visibility = documents.visibility(mask.clone(), false).await.unwrap(); + assert!(!visibility.selected(DocId::new(0))); + assert!(visibility.selected(DocId::new(1))); + assert!(!visibility.selected(DocId::new(2))); + + let weak_addresses = documents.address_buffer_handle(); + + cache.clear().await; + assert!(weak_addresses.upgrade().is_some()); + assert!(documents.projection_resident()); + + drop(visibility); + assert!(weak_addresses.upgrade().is_none()); + assert!(!documents.projection_resident()); + + let reloaded = documents.visibility(mask, false).await.unwrap(); + assert!(reloaded.selected(DocId::new(1))); + assert_eq!(counts.address_rows.load(Ordering::Relaxed), 6); + } + + #[tokio::test] + async fn prewarm_reloads_addresses_after_cache_eviction() { + let (_directory, store, cache) = eviction_test_store(); + let path = "docs.lance"; + write_documents( + store.as_ref(), + path, + UInt64Array::from(vec![10, 20, 30]), + UInt32Array::from(vec![2, 3, 5]), + Some("10"), + ) + .await; + let (counting, counts) = counted_store(store, path); + let documents = open_documents(counting, path, cache.as_ref(), None) + .await + .unwrap(); + + documents.prewarm().await.unwrap(); + assert!(documents.query_ready()); + let weak_addresses = documents.address_buffer_handle(); + + cache.clear().await; + assert!(weak_addresses.upgrade().is_none()); + assert!(documents.projection_loaded()); + assert!(!documents.projection_resident()); + assert!(!documents.query_ready()); + + documents.prewarm().await.unwrap(); + assert!(documents.query_ready()); + assert_eq!( + documents + .cached_row_addresses(&[DocId::new(2), DocId::new(0)]) + .unwrap(), + Some(vec![30, 10]) + ); + assert_eq!(counts.length_rows.load(Ordering::Relaxed), 3); + assert_eq!(counts.address_rows.load(Ordering::Relaxed), 6); + } + + #[tokio::test] + async fn prewarm_materializes_quantized_norms_before_becoming_query_ready() { + let (_directory, store, cache) = test_store(); + let path = "docs.lance"; + write_documents( + store.as_ref(), + path, + UInt64Array::from(vec![10, 20, 30]), + UInt32Array::from(vec![2, 300, 5]), + Some("307"), + ) + .await; + let reader = store.open_index_file(path).await.unwrap(); + let documents = PartitionDocuments::try_new( + store, + path.to_owned(), + 0, + WeakLanceCache::from(cache.as_ref()), + reader.as_ref(), + None, + true, + ) + .unwrap(); + + assert!(!documents.query_ready()); + documents.prewarm().await.unwrap(); + let lengths = documents.lengths().await.unwrap(); + assert!(lengths.scoring_ready()); + assert_eq!(lengths.scoring_norms().unwrap().len(), 3); + assert!(documents.query_ready()); + } + + #[tokio::test] + async fn cancelled_or_failed_prewarm_can_retry_without_partial_publication() { + let (_directory, store, cache) = test_store(); + let path = "docs.lance"; + write_documents( + store.as_ref(), + path, + UInt64Array::from(vec![10, 20, 30]), + UInt32Array::from(vec![2, 3, 5]), + Some("10"), + ) + .await; + + let (pausing, pause) = faulting_store(store.clone(), path, PAUSE_ONCE); + let documents = Arc::new( + open_documents(pausing, path, cache.as_ref(), None) + .await + .unwrap(), + ); + let task = tokio::spawn({ + let documents = documents.clone(); + async move { documents.prewarm().await } + }); + tokio::time::timeout(Duration::from_secs(5), pause.started.notified()) + .await + .expect("prewarm should reach the injected pending read"); + task.abort(); + assert!(task.await.unwrap_err().is_cancelled()); + assert!(!documents.lengths_loaded()); + assert!(!documents.projection_loaded()); + assert!(!documents.query_ready()); + documents.prewarm().await.unwrap(); + assert!(documents.query_ready()); + + let (failing, _fault) = faulting_store(store, path, FAIL_ONCE); + let documents = open_documents(failing, path, cache.as_ref(), None) + .await + .unwrap(); + let error = documents.prewarm().await.unwrap_err(); + assert!(error.to_string().contains("injected document read failure")); + assert!(!documents.lengths_loaded()); + assert!(!documents.projection_loaded()); + assert!(!documents.query_ready()); + documents.prewarm().await.unwrap(); + assert!(documents.query_ready()); + } + + #[tokio::test] + async fn prewarm_reuses_an_already_loaded_document_column() { + let (_directory, store, cache) = test_store(); + let path = "docs.lance"; + write_documents( + store.as_ref(), + path, + UInt64Array::from(vec![10, 20, 30]), + UInt32Array::from(vec![2, 3, 5]), + Some("10"), + ) + .await; + let (counting, counts) = counted_store(store, path); + let documents = open_documents(counting, path, cache.as_ref(), None) + .await + .unwrap(); + + documents.lengths().await.unwrap(); + documents.prewarm().await.unwrap(); + documents.prewarm().await.unwrap(); + + assert_eq!(counts.open_calls.load(Ordering::Relaxed), 3); + assert_eq!(counts.range_calls.load(Ordering::Relaxed), 2); + assert_eq!(counts.length_rows.load(Ordering::Relaxed), 3); + assert_eq!(counts.address_rows.load(Ordering::Relaxed), 3); + } + + #[tokio::test] + async fn invalid_or_mismatched_stats_are_corruption_not_fallback() { + let (_directory, store, cache) = test_store(); + write_documents( + store.as_ref(), + "invalid.lance", + UInt64Array::from(vec![10]), + UInt32Array::from(vec![2]), + Some("not-a-u64"), + ) + .await; + let error = open_documents(store.clone(), "invalid.lance", cache.as_ref(), None) + .await + .unwrap_err(); + assert!(error.to_string().contains("invalid total_tokens")); + + write_documents( + store.as_ref(), + "mismatch.lance", + UInt64Array::from(vec![10, 20, 30]), + UInt32Array::from(vec![2, 3, 5]), + Some("11"), + ) + .await; + let (counting, counts) = counted_store(store, "mismatch.lance"); + let documents = open_documents(counting, "mismatch.lance", cache.as_ref(), None) + .await + .unwrap(); + assert_eq!(documents.stats().await.unwrap().total_tokens, 11); + let error = documents.lengths().await.unwrap_err(); + assert!(error.to_string().contains("sums to 10")); + assert_eq!(counts.length_rows.load(Ordering::Relaxed), 3); + assert!(!documents.lengths_loaded()); + } + + #[tokio::test] + async fn final_address_resolution_reuses_the_cached_row_id_column() { + let (_directory, store, cache) = test_store(); + let path = "docs.lance"; + let num_docs = 600_u64; + write_documents( + store.as_ref(), + path, + UInt64Array::from_iter_values((0..num_docs).map(|id| id + 1_000)), + UInt32Array::from_iter_values((0..num_docs).map(|_| 1)), + Some("600"), + ) + .await; + let (counting, counts) = counted_store(store, path); + let documents = open_documents(counting, path, cache.as_ref(), None) + .await + .unwrap(); + + assert!(documents.resolve_addresses(&[]).await.unwrap().is_empty()); + assert_eq!(counts.rows.load(Ordering::Relaxed), 0); + + let point_ids = [DocId::new(5), DocId::new(6), DocId::new(10), DocId::new(5)]; + assert_eq!( + documents.estimated_address_read_bytes(&point_ids), + num_docs as usize * std::mem::size_of::() + ); + assert_eq!( + documents.resolve_addresses(&point_ids).await.unwrap(), + vec![1005, 1006, 1010, 1005] + ); + assert_eq!(counts.ranges_calls.load(Ordering::Relaxed), 0); + assert_eq!(counts.range_calls.load(Ordering::Relaxed), 1); + assert_eq!(counts.address_rows.load(Ordering::Relaxed), 600); + assert!( + cache + .get_with_key(&DocRowIdsKey { partition_id: 0 }) + .await + .is_some() + ); + + let bulk_ids = (0..=512).step_by(2).map(DocId::new).collect::>(); + assert_eq!( + documents.estimated_address_read_bytes(&bulk_ids), + num_docs as usize * std::mem::size_of::() + ); + let resolved = documents.resolve_addresses(&bulk_ids).await.unwrap(); + assert_eq!(resolved.first(), Some(&1000)); + assert_eq!(resolved.last(), Some(&1512)); + assert_eq!(counts.ranges_calls.load(Ordering::Relaxed), 0); + assert_eq!(counts.range_calls.load(Ordering::Relaxed), 1); + assert_eq!(counts.address_rows.load(Ordering::Relaxed), 600); + assert!(!documents.projection_loaded()); + } + + #[tokio::test] + async fn final_address_resolution_reloads_after_cache_eviction() { + let (_directory, store, _cache) = test_store(); + let path = "docs.lance"; + write_documents( + store.as_ref(), + path, + UInt64Array::from(vec![10, 20, 30]), + UInt32Array::from(vec![2, 3, 5]), + Some("10"), + ) + .await; + let (counting, counts) = counted_store(store, path); + let no_retention_cache = LanceCache::no_cache(); + let documents = open_documents(counting, path, &no_retention_cache, None) + .await + .unwrap(); + + for _ in 0..2 { + assert_eq!( + documents + .resolve_addresses(&[DocId::new(2), DocId::new(0)]) + .await + .unwrap(), + vec![30, 10] + ); + } + assert_eq!(counts.ranges_calls.load(Ordering::Relaxed), 0); + assert_eq!(counts.range_calls.load(Ordering::Relaxed), 2); + assert_eq!(counts.address_rows.load(Ordering::Relaxed), 6); + assert!(!documents.projection_loaded()); + } + + #[tokio::test] + async fn document_column_nulls_are_reported_as_corruption() { + let (_directory, store, cache) = test_store(); + write_documents( + store.as_ref(), + "null-length.lance", + UInt64Array::from(vec![Some(10), Some(20)]), + UInt32Array::from(vec![Some(2), None]), + Some("2"), + ) + .await; + let documents = open_documents(store.clone(), "null-length.lance", cache.as_ref(), None) + .await + .unwrap(); + assert!( + documents + .lengths() + .await + .unwrap_err() + .to_string() + .contains("_num_tokens contains null") + ); + + write_documents( + store.as_ref(), + "null-address.lance", + UInt64Array::from(vec![Some(10), None]), + UInt32Array::from(vec![2, 3]), + Some("5"), + ) + .await; + let documents = open_documents(store, "null-address.lance", cache.as_ref(), None) + .await + .unwrap(); + assert!( + documents + .resolve_addresses(&[DocId::new(1)]) + .await + .unwrap_err() + .to_string() + .contains("_rowid contains null") + ); + assert!( + documents + .prewarm() + .await + .unwrap_err() + .to_string() + .contains("_rowid contains null") + ); + assert!(!documents.lengths_loaded()); + assert!(!documents.projection_loaded()); + } +} diff --git a/rust/lance-index/src/scalar/inverted/encoding.rs b/rust/lance-index/src/scalar/inverted/encoding.rs index d75fa742ee7..6d57b032650 100644 --- a/rust/lance-index/src/scalar/inverted/encoding.rs +++ b/rust/lance-index/src/scalar/inverted/encoding.rs @@ -5,10 +5,15 @@ use std::io::Write; use super::builder::BLOCK_SIZE; use super::index::{PositionStreamCodec, PostingTailCodec}; +#[cfg(test)] +use super::tokenizer::LEGACY_BLOCK_SIZE; +use super::tokenizer::validate_block_size; use arrow::array::LargeBinaryBuilder; -use lance_bitpacking::{BitPacker, BitPacker4x}; +use lance_bitpacking::{BitPacker, BitPacker4x, BitPacker8x}; use lance_core::{Error, Result}; +pub const MAX_POSTING_BLOCK_SIZE: usize = BitPacker8x::BLOCK_LEN; + // we compress the posting list to multiple blocks of fixed number of elements (BLOCK_SIZE), // returns a LargeBinaryArray, where each binary is a compressed block (128 row ids + 128 frequencies) // each block is: @@ -41,18 +46,40 @@ pub fn compress_posting_list<'a>( #[cfg(test)] pub fn compress_posting_list_with_tail_codec<'a>( + length: usize, + doc_ids: impl Iterator, + frequencies: impl Iterator, + block_max_scores: impl Iterator, + tail_codec: PostingTailCodec, +) -> Result { + compress_posting_list_with_tail_codec_and_block_size( + length, + doc_ids, + frequencies, + block_max_scores, + tail_codec, + LEGACY_BLOCK_SIZE, + ) +} + +#[cfg(test)] +pub fn compress_posting_list_with_tail_codec_and_block_size<'a>( length: usize, doc_ids: impl Iterator, frequencies: impl Iterator, mut block_max_scores: impl Iterator, tail_codec: PostingTailCodec, + block_size: usize, ) -> Result { - if length < BLOCK_SIZE { + let block_size = validate_block_size(block_size)?; + if length < block_size { // directly do remainder compression to avoid overhead of creating buffer let mut builder = LargeBinaryBuilder::with_capacity(1, length * 4 * 2 + 1); - // write the max score of the block - let max_score = block_max_scores.next().unwrap(); - let _ = builder.write(max_score.to_le_bytes().as_ref())?; + // write the max score of the block (128-doc blocks only) + if posting_block_score_prefix_len(block_size) > 0 { + let max_score = block_max_scores.next().unwrap(); + let _ = builder.write(max_score.to_le_bytes().as_ref())?; + } compress_posting_remainder( doc_ids.copied().collect::>().as_slice(), frequencies.copied().collect::>().as_slice(), @@ -63,27 +90,25 @@ pub fn compress_posting_list_with_tail_codec<'a>( return Ok(builder.finish()); } - let mut builder = LargeBinaryBuilder::with_capacity(length.div_ceil(BLOCK_SIZE), length * 3); - let mut buffer = [0u8; BLOCK_SIZE * 4 + 5]; - let mut doc_id_buffer = Vec::with_capacity(BLOCK_SIZE); - let mut freq_buffer = Vec::with_capacity(BLOCK_SIZE); + let mut builder = LargeBinaryBuilder::with_capacity(length.div_ceil(block_size), length * 3); + let mut doc_id_buffer = Vec::with_capacity(block_size); + let mut freq_buffer = Vec::with_capacity(block_size); for (doc_id, freq) in std::iter::zip(doc_ids, frequencies) { doc_id_buffer.push(*doc_id); freq_buffer.push(*freq); - if doc_id_buffer.len() < BLOCK_SIZE { + if doc_id_buffer.len() < block_size { continue; } - assert_eq!(doc_id_buffer.len(), BLOCK_SIZE); + assert_eq!(doc_id_buffer.len(), block_size); - // write the max score of the block - let max_score = block_max_scores.next().unwrap(); - let _ = builder.write(max_score.to_le_bytes().as_ref())?; - // delta encoding + bitpacking for doc ids - compress_sorted_block(&doc_id_buffer, &mut buffer, &mut builder)?; - // bitpacking for frequencies - compress_block(&freq_buffer, &mut buffer, &mut builder)?; + // write the max score of the block (128-doc blocks only) + if posting_block_score_prefix_len(block_size) > 0 { + let max_score = block_max_scores.next().unwrap(); + let _ = builder.write(max_score.to_le_bytes().as_ref())?; + } + encode_posting_block_payload(&doc_id_buffer, &freq_buffer, &mut builder)?; builder.append_value(""); doc_id_buffer.clear(); freq_buffer.clear(); @@ -91,44 +116,187 @@ pub fn compress_posting_list_with_tail_codec<'a>( // we don't compress the last block if it is not full if !doc_id_buffer.is_empty() { - // write the max score of the block - let max_score = block_max_scores.next().unwrap(); - let _ = builder.write(max_score.to_le_bytes().as_ref())?; + // write the max score of the block (128-doc blocks only) + if posting_block_score_prefix_len(block_size) > 0 { + let max_score = block_max_scores.next().unwrap(); + let _ = builder.write(max_score.to_le_bytes().as_ref())?; + } compress_posting_remainder(&doc_id_buffer, &freq_buffer, tail_codec, &mut builder)?; builder.append_value(""); } Ok(builder.finish()) } +/// Byte length of the block-max-score prefix on posting blocks. 128-doc +/// blocks store a per-block max score, patched in at build time; 256-document +/// blocks always carry impact skip data, which supersedes it, so they +/// store none. +#[inline] +pub fn posting_block_score_prefix_len(block_size: usize) -> usize { + if block_size == MAX_POSTING_BLOCK_SIZE { + 0 + } else { + 4 + } +} + pub fn encode_full_posting_block_into( doc_ids: &[u32], frequencies: &[u32], block: &mut Vec, ) -> Result<()> { - debug_assert_eq!(doc_ids.len(), BLOCK_SIZE); - debug_assert_eq!(frequencies.len(), BLOCK_SIZE); - block.extend_from_slice(&0f32.to_le_bytes()); - let mut buffer = [0u8; BLOCK_SIZE * 4 + 5]; - compress_sorted_block(doc_ids, &mut buffer, block)?; - compress_block(frequencies, &mut buffer, block)?; + validate_block_size(doc_ids.len())?; + debug_assert_eq!(doc_ids.len(), frequencies.len()); + if posting_block_score_prefix_len(doc_ids.len()) > 0 { + block.extend_from_slice(&0f32.to_le_bytes()); + } + encode_posting_block_payload(doc_ids, frequencies, block)?; Ok(()) } +fn encode_posting_block_payload( + doc_ids: &[u32], + frequencies: &[u32], + block: &mut impl Write, +) -> Result<()> { + debug_assert_eq!(doc_ids.len(), frequencies.len()); + validate_block_size(doc_ids.len())?; + let mut buffer = [0u8; MAX_POSTING_BLOCK_SIZE * 4 + 5]; + match doc_ids.len() { + BitPacker4x::BLOCK_LEN => { + compress_sorted_block_with::(doc_ids, &mut buffer, block)?; + compress_block_with::(frequencies, &mut buffer, block)?; + } + // 256-document blocks store frequencies with patched FOR: + // outliers no longer widen the whole block, which matters because one + // large tf per block otherwise doubles the frequency payload. + BitPacker8x::BLOCK_LEN => { + compress_sorted_block_with::(doc_ids, &mut buffer, block)?; + compress_pfor_block_with::(frequencies, &mut buffer, block)?; + } + _ => unreachable!("validated posting block size should be supported"), + } + Ok(()) +} + +/// Patched FOR (Lucene PForUtil style): pick the body bit width that +/// minimizes total bytes, pack all values masked to that width, and append +/// up to [`PFOR_MAX_EXCEPTIONS`] exceptions as (index u8, high-bits varint). +const PFOR_MAX_EXCEPTIONS: usize = 31; + +#[inline] +fn u32_bits(value: u32) -> usize { + (32 - value.leading_zeros()) as usize +} + +#[inline] +fn varint_u32_len(value: u32) -> usize { + u32_bits(value).max(1).div_ceil(7) +} + +fn compress_pfor_block_with( + data: &[u32], + buffer: &mut [u8], + builder: &mut impl Write, +) -> Result<()> { + debug_assert_eq!(data.len(), P::BLOCK_LEN); + let max_bits = data.iter().map(|&v| u32_bits(v)).max().unwrap_or(0); + let mut best_width = max_bits; + let mut best_cost = P::BLOCK_LEN * max_bits / 8; + for width in (0..max_bits).rev() { + let mut exceptions = 0usize; + let mut exception_bytes = 0usize; + for &value in data { + if u32_bits(value) > width { + exceptions += 1; + exception_bytes += 1 + varint_u32_len(value >> width); + } + } + if exceptions > PFOR_MAX_EXCEPTIONS { + break; + } + let cost = P::BLOCK_LEN * width / 8 + exception_bytes; + if cost < best_cost { + best_cost = cost; + best_width = width; + } + } + + let mask = if best_width >= 32 { + u32::MAX + } else { + (1u32 << best_width) - 1 + }; + let mut body = [0u32; MAX_POSTING_BLOCK_SIZE]; + let mut exception_buf = Vec::new(); + let mut exception_count = 0u8; + for (index, &value) in data.iter().enumerate() { + body[index] = value & mask; + if u32_bits(value) > best_width { + exception_buf.push(index as u8); + encode_varint_u32(&mut exception_buf, value >> best_width); + exception_count += 1; + } + } + let compressor = P::new(); + let num_bytes = compressor.compress(&body[..P::BLOCK_LEN], buffer, best_width as u8); + let _ = builder.write(&[best_width as u8, exception_count])?; + let _ = builder.write(&buffer[..num_bytes])?; + let _ = builder.write(&exception_buf)?; + Ok(()) +} + +fn decompress_pfor_block_with( + block: &[u8], + buffer: &mut [u32], + res: &mut Vec, +) -> usize { + debug_assert!(buffer.len() >= P::BLOCK_LEN); + let buffer = &mut buffer[..P::BLOCK_LEN]; + let width = block[0]; + let exception_count = block[1] as usize; + let compressor = P::new(); + let num_bytes = compressor.decompress(&block[2..], buffer, width); + let mut offset = 2 + num_bytes; + for _ in 0..exception_count { + let index = block[offset] as usize; + offset += 1; + let high = decode_varint_u32(block, &mut offset) + .expect("pfor exception high bits should be a valid varint"); + buffer[index] |= high << width; + } + res.extend_from_slice(buffer); + offset +} + pub fn encode_remainder_posting_block_into( doc_ids: &[u32], frequencies: &[u32], codec: PostingTailCodec, + block_size: usize, block: &mut Vec, ) -> Result<()> { debug_assert_eq!(doc_ids.len(), frequencies.len()); - block.extend_from_slice(&0f32.to_le_bytes()); + if posting_block_score_prefix_len(block_size) > 0 { + block.extend_from_slice(&0f32.to_le_bytes()); + } compress_posting_remainder(doc_ids, frequencies, codec, block)?; Ok(()) } #[inline] fn compress_sorted_block(data: &[u32], buffer: &mut [u8], builder: &mut impl Write) -> Result<()> { - let compressor = BitPacker4x::new(); + compress_sorted_block_with::(data, buffer, builder) +} + +#[inline] +fn compress_sorted_block_with( + data: &[u32], + buffer: &mut [u8], + builder: &mut impl Write, +) -> Result<()> { + debug_assert_eq!(data.len(), P::BLOCK_LEN); + let compressor = P::new(); let num_bits = compressor.num_bits_sorted(data[0], data); let num_bytes = compressor.compress_sorted(data[0], data, buffer, num_bits); let _ = builder.write(data[0].to_le_bytes().as_ref())?; @@ -139,7 +307,17 @@ fn compress_sorted_block(data: &[u32], buffer: &mut [u8], builder: &mut impl Wri #[inline] fn compress_block(data: &[u32], buffer: &mut [u8], builder: &mut impl Write) -> Result<()> { - let compressor = BitPacker4x::new(); + compress_block_with::(data, buffer, builder) +} + +#[inline] +fn compress_block_with( + data: &[u32], + buffer: &mut [u8], + builder: &mut impl Write, +) -> Result<()> { + debug_assert_eq!(data.len(), P::BLOCK_LEN); + let compressor = P::new(); let num_bits = compressor.num_bits(data); let num_bytes = compressor.compress(data, buffer, num_bits); let _ = builder.write(&[num_bits])?; @@ -236,7 +414,7 @@ pub fn compress_positions(positions: &[u32]) -> Result, mut value: u32) { +pub fn encode_varint_u32(dst: &mut Vec, mut value: u32) { while value >= 0x80 { dst.push((value as u8) | 0x80); value >>= 7; @@ -244,39 +422,6 @@ fn encode_varint_u32(dst: &mut Vec, mut value: u32) { dst.push(value as u8); } -/// Encode a monotonically increasing sequence of `group_starts` (the first -/// row of each posting-list cache group) as varint-encoded deltas. The first -/// value is stored as-is and each subsequent value as its delta from the -/// previous one; since deltas are group sizes (1..=cap) they fit in ~1 byte, -/// keeping the buffer tiny even for indexes with millions of tokens. See -/// issue #7040. -pub(super) fn encode_group_starts(group_starts: &[u32]) -> Vec { - let mut dst = Vec::with_capacity(group_starts.len()); - let mut previous = 0u32; - for &start in group_starts { - debug_assert!(start >= previous, "group_starts must be monotonic"); - encode_varint_u32(&mut dst, start - previous); - previous = start; - } - dst -} - -/// Decode the buffer produced by [`encode_group_starts`] back into the -/// absolute `group_starts` values. -pub(super) fn decode_group_starts(src: &[u8]) -> Result> { - let mut group_starts = Vec::new(); - let mut offset = 0; - let mut previous = 0u32; - while offset < src.len() { - let delta = decode_varint_u32(src, &mut offset)?; - previous = previous - .checked_add(delta) - .ok_or_else(|| Error::index("group_starts delta decode overflowed u32".to_owned()))?; - group_starts.push(previous); - } - Ok(group_starts) -} - #[derive(Debug, Clone, PartialEq, Eq)] pub(super) struct PositionBlockBuilder { codec: PositionStreamCodec, @@ -379,7 +524,7 @@ impl PositionBlockBuilder { } #[inline] -fn decode_varint_u32(src: &[u8], offset: &mut usize) -> Result { +pub fn decode_varint_u32(src: &[u8], offset: &mut usize) -> Result { let mut value = 0u32; let mut shift = 0u32; while *offset < src.len() { @@ -557,15 +702,10 @@ fn decode_position_stream_packed_block( let mut deltas = Vec::with_capacity(total_positions); for _ in 0..full_delta_blocks { - if packed_offset >= src.len() { - return Err(Error::index( - "unexpected EOF while decoding packed position stream".to_owned(), - )); - } - let num_bits = src[packed_offset]; - packed_offset += 1; - let consumed = compressor.decompress(&src[packed_offset..], &mut packed_values, num_bits); - packed_offset += consumed; + let (num_bits, payload, next_offset) = packed_position_group(src, packed_offset)?; + let consumed = compressor.decompress(payload, &mut packed_values, num_bits); + debug_assert_eq!(consumed, payload.len()); + packed_offset = next_offset; deltas.extend_from_slice(&packed_values); } @@ -601,6 +741,144 @@ fn decode_position_stream_packed_block( Ok(()) } +fn packed_position_group(src: &[u8], offset: usize) -> Result<(u8, &[u8], usize)> { + let num_bits = *src.get(offset).ok_or_else(|| { + Error::index(format!( + "unexpected EOF reading packed position group header at byte offset {offset}; \ + stream length is {}", + src.len() + )) + })?; + if num_bits > u32::BITS as u8 { + return Err(Error::index(format!( + "invalid packed position group bit width {num_bits} at byte offset {offset}; \ + expected at most {}", + u32::BITS + ))); + } + + let payload_start = offset + .checked_add(1) + .ok_or_else(|| Error::index("packed position group offset overflow".to_owned()))?; + let payload_len = usize::from(num_bits) * BLOCK_SIZE / 8; + let payload_end = payload_start + .checked_add(payload_len) + .ok_or_else(|| Error::index("packed position group length overflow".to_owned()))?; + let payload = src.get(payload_start..payload_end).ok_or_else(|| { + Error::index(format!( + "unexpected EOF reading packed position group payload at byte offset {offset}; \ + need {payload_len} bytes after the header but stream length is {}", + src.len() + )) + })?; + Ok((num_bits, payload, payload_end)) +} + +/// Decode one document's positions out of a PackedDelta position block +/// without decoding the rest of the block. Full 128-delta groups are +/// self-describing (`[num_bits u8][16 * num_bits packed bytes]`), so group +/// byte offsets are recovered by hopping headers — no format change is +/// involved. `delta_range` is the doc's range in the block-wide delta stream +/// (from the frequency prefix sums); per-doc deltas reset at document +/// boundaries, so decoding starts cleanly at `delta_range.start`. +/// +/// The caller passes per-block scratch state that this function maintains: +/// `group_offsets` (lazily extended header index, seeded with `[0]`), +/// `unpacked_group`/`unpacked_group_idx` (the last unpacked group), and +/// `tail_cache` (the varint tail, decoded in full on first touch). All are +/// reset by the caller when the block cursor moves. +#[allow(clippy::too_many_arguments)] +pub(super) fn seek_packed_doc_positions( + src: &[u8], + total_deltas: usize, + delta_range: std::ops::Range, + group_offsets: &mut Vec, + unpacked_group: &mut [u32; BLOCK_SIZE], + unpacked_group_idx: &mut Option, + tail_cache: &mut Vec, + dst: &mut Vec, +) -> Result<()> { + dst.clear(); + if delta_range.start > delta_range.end || delta_range.end > total_deltas { + return Err(Error::index(format!( + "invalid packed position delta range {}..{} for {total_deltas} total deltas", + delta_range.start, delta_range.end + ))); + } + if delta_range.is_empty() { + return Ok(()); + } + let num_full_groups = total_deltas / BLOCK_SIZE; + let packed_deltas_end = num_full_groups * BLOCK_SIZE; + + // Extend the header index far enough for this range (tail needs the + // offset one past the last full group). + let last_needed_group = if delta_range.end > packed_deltas_end { + num_full_groups + } else { + (delta_range.end - 1) / BLOCK_SIZE + }; + while group_offsets.len() <= last_needed_group { + let last = *group_offsets + .last() + .ok_or_else(|| Error::index("packed position group offsets are empty".to_owned()))?; + let (_, _, next_offset) = packed_position_group(src, last)?; + group_offsets.push(next_offset); + } + + let mut previous = 0u32; + let mut first = true; + let mut push_delta = |delta: u32, dst: &mut Vec| -> Result<()> { + let position = if first { + first = false; + delta + } else { + previous + .checked_add(delta) + .ok_or_else(|| Error::index("position stream overflow while decoding".to_owned()))? + }; + dst.push(position); + previous = position; + Ok(()) + }; + + for index in delta_range.start..delta_range.end.min(packed_deltas_end) { + let group = index / BLOCK_SIZE; + if *unpacked_group_idx != Some(group) { + let offset = *group_offsets.get(group).ok_or_else(|| { + Error::index(format!( + "missing packed position group offset for group {group}; have {} offsets", + group_offsets.len() + )) + })?; + let (num_bits, payload, _) = packed_position_group(src, offset)?; + BitPacker4x::new().decompress(payload, unpacked_group, num_bits); + *unpacked_group_idx = Some(group); + } + push_delta(unpacked_group[index % BLOCK_SIZE], dst)?; + } + + if delta_range.end > packed_deltas_end { + let tail_len = total_deltas - packed_deltas_end; + if tail_cache.len() != tail_len { + tail_cache.clear(); + tail_cache.reserve(tail_len); + let mut offset = *group_offsets.get(num_full_groups).ok_or_else(|| { + Error::index(format!( + "missing packed position tail offset after {num_full_groups} full groups" + )) + })?; + for _ in 0..tail_len { + tail_cache.push(decode_varint_u32(src, &mut offset)?); + } + } + for index in delta_range.start.max(packed_deltas_end)..delta_range.end { + push_delta(tail_cache[index - packed_deltas_end], dst)?; + } + } + Ok(()) +} + #[cfg(test)] pub fn encode_position_stream_block_into( positions: &[u32], @@ -650,23 +928,46 @@ pub fn decompress_posting_list_with_tail_codec( posting_list: &arrow::array::LargeBinaryArray, tail_codec: PostingTailCodec, ) -> Result<(Vec, Vec)> { + decompress_posting_list_with_tail_codec_and_block_size( + num_docs, + posting_list, + tail_codec, + LEGACY_BLOCK_SIZE, + ) +} + +#[cfg(test)] +pub fn decompress_posting_list_with_tail_codec_and_block_size( + num_docs: u32, + posting_list: &arrow::array::LargeBinaryArray, + tail_codec: PostingTailCodec, + block_size: usize, +) -> Result<(Vec, Vec)> { + let block_size = validate_block_size(block_size)?; let mut doc_ids: Vec = Vec::with_capacity(num_docs as usize); let mut frequencies: Vec = Vec::with_capacity(num_docs as usize); - let mut buffer = [0u32; BLOCK_SIZE]; - let bitpacking_blocks = num_docs as usize / BLOCK_SIZE; + let mut buffer = [0u32; MAX_POSTING_BLOCK_SIZE]; + let bitpacking_blocks = num_docs as usize / block_size; for compressed in posting_list.iter().take(bitpacking_blocks) { let compressed = compressed.unwrap(); - decompress_posting_block(compressed, &mut buffer, &mut doc_ids, &mut frequencies); + decompress_posting_block( + compressed, + &mut buffer, + &mut doc_ids, + &mut frequencies, + block_size, + ); } - let remainder = num_docs as usize % BLOCK_SIZE; + let remainder = num_docs as usize % block_size; if remainder > 0 { let compressed = posting_list.value(bitpacking_blocks); decompress_posting_remainder( compressed, remainder, tail_codec, + block_size, &mut doc_ids, &mut frequencies, ); @@ -701,24 +1002,46 @@ pub fn read_num_positions(compressed: &arrow::array::LargeBinaryArray) -> u32 { pub fn decompress_posting_block( block: &[u8], - buffer: &mut [u32; BLOCK_SIZE], + buffer: &mut [u32], doc_ids: &mut Vec, frequencies: &mut Vec, + block_size: usize, ) { - // skip the first 4 bytes for the max block score - let block = &block[4..]; - let num_bytes = decompress_sorted_block(block, buffer, doc_ids); - decompress_block(&block[num_bytes..], buffer, frequencies); + debug_assert!(validate_block_size(block_size).is_ok()); + debug_assert!(buffer.len() >= block_size); + // skip the block max score prefix (128-doc blocks only) + let mut block = &block[posting_block_score_prefix_len(block_size)..]; + match block_size { + BitPacker4x::BLOCK_LEN => { + let num_bytes = decompress_sorted_block_with::(block, buffer, doc_ids); + block = &block[num_bytes..]; + let num_bytes = decompress_block_with::(block, buffer, frequencies); + block = &block[num_bytes..]; + } + BitPacker8x::BLOCK_LEN => { + let num_bytes = decompress_sorted_block_with::(block, buffer, doc_ids); + block = &block[num_bytes..]; + let num_bytes = decompress_pfor_block_with::(block, buffer, frequencies); + block = &block[num_bytes..]; + } + _ => unreachable!("validated posting block size should be supported"), + } + debug_assert!( + block.is_empty(), + "posting block has {} trailing bytes after decoding", + block.len() + ); } pub fn decompress_posting_remainder( block: &[u8], n: usize, codec: PostingTailCodec, + block_size: usize, doc_ids: &mut Vec, frequencies: &mut Vec, ) { - let block = &block[4..]; + let block = &block[posting_block_score_prefix_len(block_size)..]; match codec { PostingTailCodec::Fixed32 => { decompress_raw_remainder(block, n, doc_ids); @@ -755,9 +1078,14 @@ pub fn decompress_posting_remainder( } } -pub fn decode_full_posting_block(block: &[u8], doc_ids: &mut Vec, frequencies: &mut Vec) { - let mut buffer = [0u32; BLOCK_SIZE]; - decompress_posting_block(block, &mut buffer, doc_ids, frequencies); +pub fn decode_full_posting_block( + block: &[u8], + doc_ids: &mut Vec, + frequencies: &mut Vec, + block_size: usize, +) { + let mut buffer = [0u32; MAX_POSTING_BLOCK_SIZE]; + decompress_posting_block(block, &mut buffer, doc_ids, frequencies, block_size); } pub fn decompress_sorted_block( @@ -765,7 +1093,17 @@ pub fn decompress_sorted_block( buffer: &mut [u32; BLOCK_SIZE], res: &mut Vec, ) -> usize { - let compressor = BitPacker4x::new(); + decompress_sorted_block_with::(block, buffer, res) +} + +fn decompress_sorted_block_with( + block: &[u8], + buffer: &mut [u32], + res: &mut Vec, +) -> usize { + debug_assert!(buffer.len() >= P::BLOCK_LEN); + let buffer = &mut buffer[..P::BLOCK_LEN]; + let compressor = P::new(); let initial = u32::from_le_bytes(block[0..4].try_into().unwrap()); let num_bits = block[4]; let num_bytes = compressor.decompress_sorted(initial, &block[5..], buffer, num_bits); @@ -773,11 +1111,18 @@ pub fn decompress_sorted_block( 5 + num_bytes } -fn decompress_block(block: &[u8], buffer: &mut [u32; BLOCK_SIZE], res: &mut Vec) { - let compressor = BitPacker4x::new(); +fn decompress_block_with( + block: &[u8], + buffer: &mut [u32], + res: &mut Vec, +) -> usize { + debug_assert!(buffer.len() >= P::BLOCK_LEN); + let buffer = &mut buffer[..P::BLOCK_LEN]; + let compressor = P::new(); let num_bits = block[0]; - compressor.decompress(&block[1..], buffer, num_bits); + let num_bytes = compressor.decompress(&block[1..], buffer, num_bits); res.extend_from_slice(&buffer[..]); + 1 + num_bytes } pub fn decompress_raw_remainder(compressed: &[u8], n: usize, dest: &mut Vec) { @@ -787,11 +1132,18 @@ pub fn decompress_raw_remainder(compressed: &[u8], n: usize, dest: &mut Vec } } -pub fn read_posting_tail_first_doc(block: &[u8], codec: PostingTailCodec) -> u32 { +pub fn read_posting_tail_first_doc( + block: &[u8], + codec: PostingTailCodec, + block_size: usize, +) -> u32 { + let prefix = posting_block_score_prefix_len(block_size); match codec { - PostingTailCodec::Fixed32 => u32::from_le_bytes(block[4..8].try_into().unwrap()), + PostingTailCodec::Fixed32 => { + u32::from_le_bytes(block[prefix..prefix + 4].try_into().unwrap()) + } PostingTailCodec::VarintDelta => { - let mut offset = 4usize; + let mut offset = prefix; decode_varint_u32(block, &mut offset) .expect("posting tail block should contain a valid first doc id") } @@ -805,31 +1157,6 @@ mod tests { use itertools::Itertools; use rand::Rng; - #[test] - fn test_group_starts_codec_roundtrip() { - for case in [ - vec![], - vec![0u32], - vec![0, 1, 2, 3], - // realistic: monotonic with varied group sizes and a large jump - vec![0, 64, 128, 129, 4096, 1_000_000], - ] { - let encoded = encode_group_starts(&case); - let decoded = decode_group_starts(&encoded).unwrap(); - assert_eq!(decoded, case, "roundtrip mismatch for {case:?}"); - } - } - - #[test] - fn test_decode_group_starts_rejects_overflow() { - // A crafted buffer whose deltas sum past u32::MAX must error rather - // than wrap. Encodes u32::MAX followed by a +1 delta. - let mut buf = Vec::new(); - encode_varint_u32(&mut buf, u32::MAX); - encode_varint_u32(&mut buf, 1); - assert!(decode_group_starts(&buf).is_err()); - } - #[test] fn test_compress_posting_list() -> Result<()> { let num_rows: usize = BLOCK_SIZE * 1024 - 7; @@ -867,6 +1194,84 @@ mod tests { Ok(()) } + #[test] + fn test_compress_posting_list_supported_block_sizes() -> Result<()> { + for block_size in [128, 256] { + let num_rows: usize = block_size * 2 + 7; + let doc_ids = (0..num_rows as u32).collect::>(); + let frequencies = (0..num_rows as u32) + .map(|value| value % 7 + 1) + .collect::>(); + let block_max_scores = + (0..num_rows.div_ceil(block_size)).map(|value| value as f32 + 1.0); + + let posting_list = compress_posting_list_with_tail_codec_and_block_size( + doc_ids.len(), + doc_ids.iter(), + frequencies.iter(), + block_max_scores, + PostingTailCodec::VarintDelta, + block_size, + )?; + assert_eq!(posting_list.len(), num_rows.div_ceil(block_size)); + + let (decoded_doc_ids, decoded_frequencies) = + decompress_posting_list_with_tail_codec_and_block_size( + num_rows as u32, + &posting_list, + PostingTailCodec::VarintDelta, + block_size, + )?; + assert_eq!(decoded_doc_ids, doc_ids); + assert_eq!(decoded_frequencies, frequencies); + } + Ok(()) + } + + #[test] + fn test_256_posting_block_uses_single_physical_bitpack_chunk() -> Result<()> { + let block_size = BitPacker8x::BLOCK_LEN; + let doc_ids = (0..block_size as u32).collect::>(); + let frequencies = (0..block_size as u32) + .map(|value| value % 13 + 1) + .collect::>(); + + let posting_list = compress_posting_list_with_tail_codec_and_block_size( + doc_ids.len(), + doc_ids.iter(), + frequencies.iter(), + std::iter::once(1.0), + PostingTailCodec::VarintDelta, + block_size, + )?; + assert_eq!(posting_list.len(), 1); + + let block = posting_list.value(0); + // 256-doc blocks carry no block-max-score prefix (impacts supply the + // per-block bound): [first_doc u32][doc num_bits u8][doc payload]... + let doc_num_bits = block[4]; + let doc_bytes = BitPacker8x::compressed_block_size(doc_num_bits); + let freq_header_offset = 5 + doc_bytes; + // 256-doc blocks use patched FOR for frequencies: + // [width u8][exception_count u8][body][exceptions...] + let freq_num_bits = block[freq_header_offset]; + let exception_count = block[freq_header_offset + 1] as usize; + assert_eq!(exception_count, 0, "uniform freqs need no exceptions"); + let freq_bytes = BitPacker8x::compressed_block_size(freq_num_bits); + assert_eq!(block.len(), freq_header_offset + 2 + freq_bytes); + + let (decoded_doc_ids, decoded_frequencies) = + decompress_posting_list_with_tail_codec_and_block_size( + doc_ids.len() as u32, + &posting_list, + PostingTailCodec::VarintDelta, + block_size, + )?; + assert_eq!(decoded_doc_ids, doc_ids); + assert_eq!(decoded_frequencies, frequencies); + Ok(()) + } + #[test] fn test_compress_posting_list_fixed32_tail_still_roundtrips() -> Result<()> { let doc_ids = vec![3_u32, 10_u32, 24_u32]; @@ -913,6 +1318,116 @@ mod tests { Ok(()) } + /// Per-doc seek decoding of a PackedDelta position block must return + /// exactly the same positions as decoding the whole block, for every doc, + /// across group-boundary-straddling docs and varint tails. + #[test] + fn test_packed_position_doc_seek_matches_block_decode() -> Result<()> { + let mut rng = rand::rng(); + // Frequency shapes: tiny blocks (tail only), exactly one group, a doc + // straddling group boundaries, and a large multi-group block. + let freq_shapes: Vec> = vec![ + vec![1], + vec![3, 1, 5], + vec![64, 64], + vec![100, 60, 40], + (0..256u32).map(|i| (i % 7) + 1).collect(), + vec![300, 2, 129, 1, 77], + ]; + for frequencies in freq_shapes { + let total: usize = frequencies.iter().map(|&f| f as usize).sum(); + // Positions ascend within each doc; docs are independent. + let mut positions = Vec::with_capacity(total); + for &freq in &frequencies { + let mut current = rng.random_range(0..1000u32); + for _ in 0..freq { + positions.push(current); + current += rng.random_range(1..50u32); + } + } + + let mut encoded = Vec::new(); + encode_position_stream_block_into( + &positions, + &frequencies, + PositionStreamCodec::PackedDelta, + &mut encoded, + )?; + + let mut whole = Vec::new(); + decode_position_stream_block( + &encoded, + &frequencies, + PositionStreamCodec::PackedDelta, + &mut whole, + )?; + assert_eq!(whole, positions); + + let mut group_offsets = vec![0usize]; + let mut unpacked_group = Box::new([0u32; BLOCK_SIZE]); + let mut unpacked_group_idx = None; + let mut tail_cache = Vec::new(); + let mut scratch = Vec::new(); + let mut delta_start = 0usize; + for &freq in &frequencies { + let delta_end = delta_start + freq as usize; + seek_packed_doc_positions( + &encoded, + total, + delta_start..delta_end, + &mut group_offsets, + &mut unpacked_group, + &mut unpacked_group_idx, + &mut tail_cache, + &mut scratch, + )?; + assert_eq!( + scratch, + whole[delta_start..delta_end], + "doc positions mismatch for range {delta_start}..{delta_end} freqs={frequencies:?}" + ); + delta_start = delta_end; + } + } + Ok(()) + } + + #[test] + fn test_packed_position_decoders_reject_malformed_groups() { + let frequencies = [BLOCK_SIZE as u32]; + for encoded in [&[1_u8][..], &[33_u8][..]] { + let mut decoded = Vec::new(); + assert!( + decode_position_stream_block( + encoded, + &frequencies, + PositionStreamCodec::PackedDelta, + &mut decoded, + ) + .is_err() + ); + + let mut group_offsets = vec![0usize]; + let mut unpacked_group = Box::new([0u32; BLOCK_SIZE]); + let mut unpacked_group_idx = None; + let mut tail_cache = Vec::new(); + let mut scratch = Vec::new(); + assert!( + seek_packed_doc_positions( + encoded, + BLOCK_SIZE, + 0..BLOCK_SIZE, + &mut group_offsets, + &mut unpacked_group, + &mut unpacked_group_idx, + &mut tail_cache, + &mut scratch, + ) + .is_err() + ); + } + } + #[test] fn test_encode_position_stream_block_roundtrip() -> Result<()> { let frequencies = vec![1, 3, 2, 4]; diff --git a/rust/lance-index/src/scalar/inverted/impact.rs b/rust/lance-index/src/scalar/inverted/impact.rs new file mode 100644 index 00000000000..848c2376459 --- /dev/null +++ b/rust/lance-index/src/scalar/inverted/impact.rs @@ -0,0 +1,1055 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright The Lance Authors + +use std::mem::size_of; +use std::sync::{Arc, Mutex, MutexGuard}; + +use arrow_array::builder::LargeBinaryBuilder; +use arrow_array::{Array, LargeBinaryArray}; +use lance_core::{Error, Result}; + +use super::scorer::Scorer; + +pub const IMPACT_LEVEL1_BLOCKS: usize = 32; +const SMALL_FRONTIER_FREQ_LIMIT: usize = 256; + +/// On-disk encoding of one impact entry, shared by every posting block size. +/// +/// Entries contain `[doc_up_to varint][pair_count varint][pairs...]`. Each +/// pair stores a varint whose high bits are `freq_delta - 1` and whose low bit +/// reports whether a one-byte norm delta follows. The norm itself is the +/// quantized `u8` document-length code; the common `norm_delta == 1` case needs +/// no norm byte. +#[derive(Debug, Clone)] +pub struct ImpactSkipData { + entries: LargeBinaryArray, + level0_len: usize, + // Last doc id covered by each entry (level0 entries then level1 entries), + // decoded once at construction. Level1 markers are fully validated because + // WAND may use them to skip a group; u32::MAX marks malformed entries. + entry_doc_up_tos: Arc<[u32]>, + // The most recently baked bounds with a stable scorer key. Each query holds + // its own Arc in ImpactScoreCache, so replacing this slot for another scorer + // cannot change bounds already in use. Scorers without a key never enter the + // shared slot. Malformed entries bake to INFINITY so pruning stays safe. + last_keyed_bounds: Arc>, +} + +impl PartialEq for ImpactSkipData { + fn eq(&self, other: &Self) -> bool { + self.entries == other.entries && self.level0_len == other.level0_len + } +} + +#[cfg(test)] +#[derive(Debug, Clone, Copy)] +pub struct ImpactScore { + pub score: f32, + pub entries_scanned: usize, +} + +#[derive(Debug)] +struct ImpactBounds { + per_entry: Box<[f32]>, + global: f32, +} + +type LastKeyedImpactBounds = Option<(u64, Arc)>; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum ImpactBoundsCacheKey { + Keyed(u64), + QueryLocal, +} + +#[derive(Debug, Default, Clone)] +pub struct ImpactScoreCache { + key: Option, + bounds: Option>, +} + +impl ImpactScoreCache { + fn bounds<'a, S: Scorer + ?Sized>( + &'a mut self, + impacts: &ImpactSkipData, + scorer: &S, + ) -> &'a ImpactBounds { + let scorer_key = scorer.doc_weight_cache_key(); + let cache_key = scorer_key + .map(ImpactBoundsCacheKey::Keyed) + .unwrap_or(ImpactBoundsCacheKey::QueryLocal); + if self.key != Some(cache_key) { + self.key = Some(cache_key); + self.bounds = None; + } + + self.bounds + .get_or_insert_with(|| impacts.bounds_for_scorer(scorer, scorer_key)) + } + + fn entry_score( + &mut self, + impacts: &ImpactSkipData, + entry_idx: usize, + query_weight: f32, + scorer: &S, + ) -> f32 { + if query_weight <= 0.0 { + return 0.0; + } + query_weight * self.bounds(impacts, scorer).per_entry[entry_idx] + } +} + +impl ImpactSkipData { + pub fn new(entries: LargeBinaryArray, level0_len: usize) -> Result { + let expected_len = level0_len + level1_len(level0_len); + if entries.len() != expected_len { + return Err(Error::index(format!( + "impact entry count mismatch: got {}, expected {} for {} level0 blocks", + entries.len(), + expected_len, + level0_len + ))); + } + let entry_doc_up_tos = (0..entries.len()) + .map(|entry_idx| { + if entries.is_null(entry_idx) { + return u32::MAX; + } + let bytes = entries.value(entry_idx); + let doc_up_to = if entry_idx < level0_len { + decode_level0_entry_doc_up_to(bytes) + } else { + decode_entry_doc_up_to(bytes) + }; + doc_up_to.unwrap_or(u32::MAX) + }) + .collect::>(); + Ok(Self { + entries, + level0_len, + entry_doc_up_tos, + last_keyed_bounds: Arc::new(Mutex::new(None)), + }) + } + + fn keyed_bounds_guard(&self) -> MutexGuard<'_, LastKeyedImpactBounds> { + match self.last_keyed_bounds.lock() { + Ok(guard) => guard, + Err(poisoned) => poisoned.into_inner(), + } + } + + fn bounds_for_scorer( + &self, + scorer: &S, + scorer_key: Option, + ) -> Arc { + let Some(scorer_key) = scorer_key else { + return Arc::new(self.compute_bounds(scorer)); + }; + + { + let cached = self.keyed_bounds_guard(); + if let Some((cached_key, bounds)) = cached.as_ref() + && *cached_key == scorer_key + { + return bounds.clone(); + } + } + + // Compute outside the mutex. Concurrent misses may duplicate this work, + // but only the short publication/check below holds the shared lock. + let computed = Arc::new(self.compute_bounds(scorer)); + let mut cached = self.keyed_bounds_guard(); + if let Some((cached_key, bounds)) = cached.as_ref() + && *cached_key == scorer_key + { + return bounds.clone(); + } + *cached = Some((scorer_key, computed.clone())); + computed + } + + fn compute_bounds(&self, scorer: &S) -> ImpactBounds { + let per_entry = (0..self.entries.len()) + .map(|entry_idx| { + if self.entries.is_null(entry_idx) { + return f32::INFINITY; + } + let bytes = self.entries.value(entry_idx); + let mut max_doc_weight = 0.0_f32; + match for_each_entry_pair(bytes, |freq, doc_len| { + max_doc_weight = max_doc_weight.max(scorer.doc_weight(freq, doc_len)); + }) { + Ok(()) => max_doc_weight, + Err(_) => f32::INFINITY, + } + }) + .collect::>(); + // The level1 entries cover every block, so their max is the list-wide + // max doc weight; zero-entry lists fall back to the empty level0 slab. + let global = if per_entry.len() > self.level0_len { + per_entry[self.level0_len..] + .iter() + .copied() + .fold(0.0_f32, f32::max) + } else { + per_entry.iter().copied().fold(0.0_f32, f32::max) + }; + ImpactBounds { per_entry, global } + } + + /// List-wide max doc weight, from the scorer-specific cached bounds. The + /// tightest valid global score bound is `query_weight * this`, matching what + /// the non-impact format stores as `max_score` at build time. + pub fn global_max_doc_weight_cached( + &self, + scorer: &S, + cache: &mut ImpactScoreCache, + ) -> f32 { + cache.bounds(self, scorer).global + } + + /// Cached per-block max doc weights (level0 entries only), for bulk skip + /// scans over dead ranges without per-block window bookkeeping. + pub(crate) fn level0_doc_weight_bounds_cached<'a, S: Scorer + ?Sized>( + &self, + scorer: &S, + cache: &'a mut ImpactScoreCache, + ) -> &'a [f32] { + &cache.bounds(self, scorer).per_entry[..self.level0_len] + } + + pub fn entries(&self) -> &LargeBinaryArray { + &self.entries + } + + /// Conservative heap charge for query-independent derived state and one + /// shared keyed-bound slab, whether or not that slab has been initialized + /// yet. The Arrow impact entries are owned by the enclosing batch and are + /// deliberately excluded so packed-group cache accounting counts them once. + pub(crate) fn derived_cache_bytes(&self) -> usize { + Self::derived_cache_bytes_for_entries(self.entries.len()) + } + + pub(crate) fn derived_cache_bytes_for_entries(entry_count: usize) -> usize { + entry_count * size_of::() + + size_of::>() + + size_of::() + + entry_count * size_of::() + } + + #[cfg(test)] + pub(crate) fn shares_derived_state_with(&self, other: &Self) -> bool { + Arc::ptr_eq(&self.entry_doc_up_tos, &other.entry_doc_up_tos) + && Arc::ptr_eq(&self.last_keyed_bounds, &other.last_keyed_bounds) + } + + #[cfg(test)] + pub fn level0_len(&self) -> usize { + self.level0_len + } + + #[cfg(test)] + pub fn level1_len(&self) -> usize { + level1_len(self.level0_len) + } + + pub(crate) fn level1_doc_up_to(&self, group_idx: usize) -> Option { + if group_idx >= level1_len(self.level0_len) { + return None; + } + match self.entry_doc_up_tos[self.level0_len + group_idx] { + u32::MAX => None, + doc_up_to => Some(doc_up_to), + } + } + + /// Last doc id covered by the level0 entry of `block_idx`, or `None` when + /// the entry is missing or malformed. + pub(crate) fn level0_doc_up_to(&self, block_idx: usize) -> Option { + if block_idx >= self.level0_len { + return None; + } + match self.entry_doc_up_tos[block_idx] { + u32::MAX => None, + doc_up_to => Some(doc_up_to), + } + } + + pub fn level0_score_cached( + &self, + block_idx: usize, + query_weight: f32, + scorer: &S, + cache: &mut ImpactScoreCache, + ) -> f32 { + if block_idx >= self.level0_len { + return 0.0; + } + cache.entry_score(self, block_idx, query_weight, scorer) + } + + /// Max score of the docs covered by the level1 entry of `group_idx`, + /// answered from the scorer-specific cached bounds slab. + pub(crate) fn level1_score_cached( + &self, + group_idx: usize, + query_weight: f32, + scorer: &S, + cache: &mut ImpactScoreCache, + ) -> f32 { + if group_idx >= level1_len(self.level0_len) { + return 0.0; + } + cache.entry_score(self, self.level0_len + group_idx, query_weight, scorer) + } + + #[cfg(test)] + pub fn max_score_up_to_cached( + &self, + start_block_idx: usize, + up_to: u64, + query_weight: f32, + scorer: &S, + cache: &mut ImpactScoreCache, + ) -> ImpactScore + where + S: Scorer + ?Sized, + { + let mut block_idx = start_block_idx; + let mut max_score = 0.0_f32; + let mut entries_scanned = 0usize; + + while block_idx < self.level0_len { + let group_idx = block_idx / IMPACT_LEVEL1_BLOCKS; + let group_start = group_idx * IMPACT_LEVEL1_BLOCKS; + let group_end = ((group_idx + 1) * IMPACT_LEVEL1_BLOCKS).min(self.level0_len); + if block_idx == group_start { + let level1_entry_idx = self.level0_len + group_idx; + match self.entry_doc_up_tos[level1_entry_idx] { + u32::MAX => { + return ImpactScore { + score: f32::INFINITY, + entries_scanned: entries_scanned + 1, + }; + } + doc_up_to if u64::from(doc_up_to) <= up_to => { + max_score = max_score.max(cache.entry_score( + self, + level1_entry_idx, + query_weight, + scorer, + )); + entries_scanned += 1; + block_idx = group_end; + continue; + } + _ => {} + } + } + + max_score = max_score.max(cache.entry_score(self, block_idx, query_weight, scorer)); + entries_scanned += 1; + match self.entry_doc_up_tos[block_idx] { + u32::MAX => { + return ImpactScore { + score: f32::INFINITY, + entries_scanned, + }; + } + doc_up_to if u64::from(doc_up_to) >= up_to => break, + _ => {} + } + block_idx += 1; + } + + ImpactScore { + score: max_score, + entries_scanned, + } + } +} + +pub struct ImpactSkipDataBuilder { + entries: LargeBinaryBuilder, + level0_len: usize, + level1_entries: Vec>, + level1_docs: Vec<(u32, u32, u32)>, +} + +impl ImpactSkipDataBuilder { + pub fn with_capacity(level0_blocks: usize, block_size: usize) -> Self { + Self { + entries: LargeBinaryBuilder::with_capacity( + level0_blocks + level1_len(level0_blocks), + 0, + ), + level0_len: 0, + level1_entries: Vec::with_capacity(level1_len(level0_blocks)), + level1_docs: Vec::with_capacity(IMPACT_LEVEL1_BLOCKS * block_size), + } + } + + pub fn append_block(&mut self, docs: &[(u32, u32, u32)]) -> Result<()> { + let bytes = encode_impact_entry(docs)?; + self.entries.append_value(bytes.as_slice()); + self.level0_len += 1; + self.level1_docs.extend_from_slice(docs); + if self.level0_len.is_multiple_of(IMPACT_LEVEL1_BLOCKS) { + self.flush_level1()?; + } + Ok(()) + } + + pub fn finish(mut self) -> Result { + if !self.level1_docs.is_empty() { + self.flush_level1()?; + } + for entry in self.level1_entries { + self.entries.append_value(entry.as_slice()); + } + ImpactSkipData::new(self.entries.finish(), self.level0_len) + } + + fn flush_level1(&mut self) -> Result<()> { + let bytes = encode_impact_entry(self.level1_docs.as_slice())?; + self.level1_entries.push(bytes); + self.level1_docs.clear(); + Ok(()) + } +} + +#[cfg(test)] +pub fn build_impact_skip_data(blocks: &[Vec<(u32, u32, u32)>]) -> Result { + let block_size = blocks.iter().map(Vec::len).max().unwrap_or(0).max(1); + let mut builder = ImpactSkipDataBuilder::with_capacity(blocks.len(), block_size); + for block in blocks { + builder.append_block(block)?; + } + builder.finish() +} + +fn encode_impact_entry(docs: &[(u32, u32, u32)]) -> Result> { + if docs.is_empty() { + return Err(Error::index( + "cannot encode an empty impact entry".to_owned(), + )); + } + let doc_up_to = docs + .last() + .map(|(doc_id, _, _)| *doc_id) + .expect("non-empty impact entry was validated above"); + let frontier = quantized_impact_frontier(docs); + let pair_count = u32::try_from(frontier.len()).map_err(|_| { + Error::index("impact frontier too large to encode as u32 pair count".to_string()) + })?; + let mut bytes = Vec::with_capacity(5 + frontier.len() * 2); + super::encoding::encode_varint_u32(&mut bytes, doc_up_to); + super::encoding::encode_varint_u32(&mut bytes, pair_count); + let mut previous_freq = 0u32; + let mut previous_norm = 0u8; + for (pair_idx, (freq, norm)) in frontier.into_iter().enumerate() { + let freq_delta_minus_one = freq + .checked_sub(previous_freq) + .and_then(|delta| delta.checked_sub(1)) + .ok_or_else(|| { + Error::index(format!( + "impact frequencies must be positive and strictly increasing: previous={previous_freq}, current={freq}" + )) + })?; + let norm_delta = norm.checked_sub(previous_norm).ok_or_else(|| { + Error::index(format!( + "impact norms must be non-decreasing: previous={previous_norm}, current={norm}" + )) + })?; + if pair_idx > 0 && norm_delta == 0 { + return Err(Error::index(format!( + "impact norms must be strictly increasing after quantization: norm={norm}" + ))); + } + + let has_explicit_norm_delta = norm_delta != 1; + let packed_freq_delta = + (u64::from(freq_delta_minus_one) << 1) | u64::from(has_explicit_norm_delta); + encode_varint_u64(&mut bytes, packed_freq_delta); + if has_explicit_norm_delta { + bytes.push(norm_delta); + } + previous_freq = freq; + previous_norm = norm; + } + Ok(bytes) +} + +fn decode_entry_doc_up_to(bytes: &[u8]) -> Result { + let mut offset = 0usize; + let doc_up_to = super::encoding::decode_varint_u32(bytes, &mut offset)?; + // Level-1 doc ids drive whole-group skips, so only publish a doc id after + // validating the complete entry. A truncated entry may still have a valid + // first varint. + for_each_entry_pair(bytes, |_, _| {})?; + Ok(doc_up_to) +} + +fn decode_level0_entry_doc_up_to(bytes: &[u8]) -> Result { + // A malformed level0 frontier bakes an INFINITY score before this marker + // can terminate a range scan, so avoid parsing every frontier twice on the + // query-load path. Level1 entries are fully validated. + let mut offset = 0usize; + super::encoding::decode_varint_u32(bytes, &mut offset) +} + +/// Walk an entry's (freq, doc_len) frontier pairs, validating the layout. +fn for_each_entry_pair(bytes: &[u8], mut visit: impl FnMut(u32, u32)) -> Result<()> { + let mut offset = 0usize; + let _doc_up_to = super::encoding::decode_varint_u32(bytes, &mut offset)?; + let pair_count = super::encoding::decode_varint_u32(bytes, &mut offset)?; + if pair_count == 0 { + return Err(Error::index( + "impact entry must contain at least one frontier pair".to_owned(), + )); + } + + let mut previous_freq = 0u32; + let mut previous_norm = 0u16; + for pair_idx in 0..pair_count { + let packed_freq_delta = decode_varint_u64(bytes, &mut offset)?; + let freq_delta_minus_one = u32::try_from(packed_freq_delta >> 1) + .map_err(|_| Error::index("impact freq delta exceeds u32".to_owned()))?; + let freq_delta = freq_delta_minus_one + .checked_add(1) + .ok_or_else(|| Error::index("impact freq delta overflow".to_owned()))?; + let freq = previous_freq + .checked_add(freq_delta) + .ok_or_else(|| Error::index("impact frequency overflow".to_owned()))?; + + let has_explicit_norm_delta = packed_freq_delta & 1 != 0; + let norm_delta = if has_explicit_norm_delta { + let norm_delta = bytes.get(offset).copied().ok_or_else(|| { + Error::index("unexpected EOF while decoding impact norm delta".to_owned()) + })?; + offset += 1; + norm_delta + } else { + 1 + }; + if pair_idx > 0 && norm_delta == 0 { + return Err(Error::index( + "impact norms must be strictly increasing".to_owned(), + )); + } + + let norm = previous_norm + .checked_add(u16::from(norm_delta)) + .filter(|norm| *norm <= u16::from(u8::MAX)) + .ok_or_else(|| Error::index("impact norm delta overflow".to_owned()))?; + let norm = norm as u8; + visit(freq, super::index::dequantize_doc_length(norm)); + previous_freq = freq; + previous_norm = u16::from(norm); + } + if offset != bytes.len() { + return Err(Error::index(format!( + "impact entry has {} trailing bytes", + bytes.len() - offset + ))); + } + Ok(()) +} + +#[inline] +fn encode_varint_u64(dst: &mut Vec, mut value: u64) { + while value >= 0x80 { + dst.push((value as u8) | 0x80); + value >>= 7; + } + dst.push(value as u8); +} + +#[inline] +fn decode_varint_u64(src: &[u8], offset: &mut usize) -> Result { + let mut value = 0u64; + let mut shift = 0u32; + while *offset < src.len() { + let byte = src[*offset]; + *offset += 1; + if shift == 63 && byte & 0xFE != 0 { + return Err(Error::index( + "invalid u64 varint in impact entry".to_owned(), + )); + } + value |= u64::from(byte & 0x7F) << shift; + if byte & 0x80 == 0 { + return Ok(value); + } + shift += 7; + if shift > 63 { + return Err(Error::index( + "invalid u64 varint in impact entry".to_owned(), + )); + } + } + Err(Error::index( + "unexpected EOF while decoding impact entry".to_owned(), + )) +} + +fn quantized_impact_frontier(docs: &[(u32, u32, u32)]) -> Vec<(u32, u8)> { + let raw_frontier = impact_frontier(docs); + let mut frontier: Vec<(u32, u8)> = Vec::with_capacity(raw_frontier.len()); + for (freq, doc_len) in raw_frontier { + let norm = super::index::quantize_doc_length(doc_len); + match frontier.last_mut() { + Some((last_freq, last_norm)) if *last_norm == norm => { + // At the same quantized norm, the larger frequency dominates. + *last_freq = freq; + } + Some((_, last_norm)) => { + debug_assert!( + *last_norm < norm, + "raw impact frontier document lengths must be increasing" + ); + frontier.push((freq, norm)); + } + None => frontier.push((freq, norm)), + } + } + frontier +} + +fn impact_frontier(docs: &[(u32, u32, u32)]) -> Vec<(u32, u32)> { + let max_freq = docs.iter().map(|(_, freq, _)| *freq).max().unwrap_or(0) as usize; + if max_freq <= SMALL_FRONTIER_FREQ_LIMIT { + return impact_frontier_small_freq(docs, max_freq); + } + + impact_frontier_sparse_freq(docs) +} + +fn impact_frontier_small_freq(docs: &[(u32, u32, u32)], max_freq: usize) -> Vec<(u32, u32)> { + let mut min_doc_len_by_freq = [u32::MAX; SMALL_FRONTIER_FREQ_LIMIT + 1]; + for (_, freq, doc_len) in docs { + min_doc_len_by_freq[*freq as usize] = min_doc_len_by_freq[*freq as usize].min(*doc_len); + } + + let min_doc_lens = min_doc_len_by_freq[..=max_freq] + .iter() + .enumerate() + .filter_map(|(freq, doc_len)| (*doc_len != u32::MAX).then_some((freq as u32, *doc_len))) + .collect::>(); + frontier_from_min_doc_lens(min_doc_lens) +} + +fn impact_frontier_sparse_freq(docs: &[(u32, u32, u32)]) -> Vec<(u32, u32)> { + let mut pairs = docs + .iter() + .map(|(_, freq, doc_len)| (*freq, *doc_len)) + .collect::>(); + pairs.sort_unstable_by_key(|(freq, _)| *freq); + + let mut min_doc_lens: Vec<(u32, u32)> = Vec::with_capacity(pairs.len()); + for (freq, doc_len) in pairs { + match min_doc_lens.last_mut() { + Some((last_freq, last_doc_len)) if *last_freq == freq => { + *last_doc_len = (*last_doc_len).min(doc_len); + } + _ => min_doc_lens.push((freq, doc_len)), + } + } + + frontier_from_min_doc_lens(min_doc_lens) +} + +fn frontier_from_min_doc_lens(min_doc_lens: Vec<(u32, u32)>) -> Vec<(u32, u32)> { + let mut best_doc_len = u32::MAX; + let mut frontier = Vec::with_capacity(min_doc_lens.len()); + for (freq, doc_len) in min_doc_lens.into_iter().rev() { + if doc_len < best_doc_len { + frontier.push((freq, doc_len)); + best_doc_len = doc_len; + } + } + frontier.reverse(); + frontier +} + +fn level1_len(level0_len: usize) -> usize { + level0_len.div_ceil(IMPACT_LEVEL1_BLOCKS) +} + +#[cfg(test)] +mod tests { + use std::collections::HashMap; + use std::sync::Arc; + use std::sync::atomic::{AtomicUsize, Ordering}; + + use arrow::buffer::{Buffer, NullBuffer, OffsetBuffer, ScalarBuffer}; + + use super::*; + use crate::scalar::inverted::scorer::{MemBM25Scorer, Scorer}; + + struct KeyedCountingScorer { + key: u64, + calls: Arc, + } + + impl Scorer for KeyedCountingScorer { + fn query_weight(&self, _token: &str) -> f32 { + 1.0 + } + + fn doc_weight(&self, freq: u32, doc_tokens: u32) -> f32 { + self.calls.fetch_add(1, Ordering::Relaxed); + freq as f32 / doc_tokens as f32 + } + + fn doc_weight_cache_key(&self) -> Option { + Some(self.key) + } + } + + #[test] + fn impact_entry_frontier_drops_dominated_pairs() { + let docs = vec![(0, 1, 10), (1, 1, 8), (2, 2, 9), (3, 3, 20)]; + assert_eq!(impact_frontier(&docs), vec![(1, 8), (2, 9), (3, 20)]); + } + + #[test] + fn impact_entry_frontier_handles_sparse_large_frequencies() { + let docs = vec![ + (0, 1, 100), + (1, 1, 80), + (2, 512, 90), + (3, 1_000, 120), + (4, 1_000, 110), + ]; + assert_eq!( + impact_frontier(&docs), + vec![(1, 80), (512, 90), (1_000, 110)] + ); + } + + #[test] + fn quantized_impact_frontier_drops_equal_norms() { + let docs = vec![(0, 1, 16), (1, 2, 17), (2, 3, 24)]; + assert_eq!( + quantized_impact_frontier(&docs), + vec![ + (2, super::super::index::quantize_doc_length(17)), + (3, super::super::index::quantize_doc_length(24)), + ] + ); + } + + #[test] + fn impact_max_score_can_use_level1_entry() { + let blocks = (0..40) + .map(|block| vec![(block as u32, 1 + block as u32 % 3, 10)]) + .collect::>(); + let impacts = build_impact_skip_data(&blocks).unwrap(); + assert_eq!(impacts.level0_len(), 40); + assert_eq!(impacts.level1_len(), 2); + let scorer = MemBM25Scorer::new(400, 40, HashMap::from([(String::from("token"), 40usize)])); + let mut cache = ImpactScoreCache::default(); + let score = impacts.max_score_up_to_cached(0, 31, 1.0, &scorer, &mut cache); + assert!(score.entries_scanned < IMPACT_LEVEL1_BLOCKS); + assert!(score.score > 0.0); + } + + #[test] + fn impact_level1_doc_up_to_reports_full_and_partial_groups() { + let blocks = (0..40) + .map(|block| vec![(block as u32, 1, 10)]) + .collect::>(); + let impacts = build_impact_skip_data(&blocks).unwrap(); + + assert_eq!( + impacts.level1_doc_up_to(0), + Some((IMPACT_LEVEL1_BLOCKS - 1) as u32) + ); + assert_eq!(impacts.level1_doc_up_to(1), Some(39)); + assert_eq!(impacts.level1_doc_up_to(2), None); + } + + #[test] + fn impact_level1_doc_up_to_returns_none_for_malformed_entry() { + let level0 = encode_impact_entry(&[(0, 1, 10)]).unwrap(); + let malformed_level1 = vec![1, 2, 3]; + let entries = LargeBinaryArray::from_opt_vec(vec![ + Some(level0.as_slice()), + Some(malformed_level1.as_slice()), + ]); + let impacts = ImpactSkipData::new(entries, 1).unwrap(); + + assert_eq!(impacts.level1_doc_up_to(0), None); + } + + #[test] + fn impact_level1_doc_up_to_validates_complete_entry() { + let level0 = encode_impact_entry(&[(0, 1, 10)]).unwrap(); + // A complete first varint is not enough: the pair count and frontier + // are required before this doc id can safely drive a group skip. + let truncated_level1 = [31_u8]; + let entries = LargeBinaryArray::from_opt_vec(vec![ + Some(level0.as_slice()), + Some(truncated_level1.as_slice()), + ]); + let impacts = ImpactSkipData::new(entries, 1).unwrap(); + + assert_eq!(impacts.level1_doc_up_to(0), None); + } + + #[test] + fn empty_impact_frontiers_are_malformed_bounds() { + let scorer = MemBM25Scorer::new(10, 1, HashMap::new()); + let level0 = encode_impact_entry(&[(0, 1, 10)]).unwrap(); + let empty_level1 = [0, 0]; + let entries = LargeBinaryArray::from_opt_vec(vec![ + Some(level0.as_slice()), + Some(empty_level1.as_slice()), + ]); + let impacts = ImpactSkipData::new(entries, 1).unwrap(); + let mut cache = ImpactScoreCache::default(); + + assert_eq!(impacts.level1_doc_up_to(0), None); + assert!( + impacts + .global_max_doc_weight_cached(&scorer, &mut cache) + .is_infinite() + ); + assert!( + ImpactSkipDataBuilder::with_capacity(1, 128) + .append_block(&[]) + .is_err() + ); + } + + #[test] + fn null_impact_entry_is_an_infinite_bound_even_with_hidden_bytes() { + let level0 = encode_impact_entry(&[(0, 1, 10)]).unwrap(); + let level1 = encode_impact_entry(&[(0, 2, 8)]).unwrap(); + let mut values = level0.clone(); + values.extend_from_slice(&level1); + let entries = LargeBinaryArray::new( + OffsetBuffer::new(ScalarBuffer::from(vec![ + 0_i64, + level0.len() as i64, + values.len() as i64, + ])), + Buffer::from_vec(values), + Some(NullBuffer::from(vec![true, false])), + ); + let impacts = ImpactSkipData::new(entries, 1).unwrap(); + let scorer = MemBM25Scorer::new(10, 1, HashMap::new()); + let mut cache = ImpactScoreCache::default(); + + assert_eq!(impacts.level1_doc_up_to(0), None); + assert!( + impacts + .global_max_doc_weight_cached(&scorer, &mut cache) + .is_infinite() + ); + } + + #[test] + fn impact_bounds_follow_changed_bm25_average_doc_length() { + let impacts = build_impact_skip_data(&[vec![(0, 1, 100)]]).unwrap(); + let low_avgdl = MemBM25Scorer::new(1, 1, HashMap::new()); + let high_avgdl = MemBM25Scorer::new(100, 1, HashMap::new()); + let mut low_cache = ImpactScoreCache::default(); + let mut high_cache = ImpactScoreCache::default(); + + let low_bound = impacts.global_max_doc_weight_cached(&low_avgdl, &mut low_cache); + let high_bound = impacts.global_max_doc_weight_cached(&high_avgdl, &mut high_cache); + let quantized_doc_length = super::super::index::dequantize_doc_length( + super::super::index::quantize_doc_length(100), + ); + + assert!((low_bound - low_avgdl.doc_weight(1, quantized_doc_length)).abs() < 1e-6); + assert!((high_bound - high_avgdl.doc_weight(1, quantized_doc_length)).abs() < 1e-6); + assert!(low_bound >= low_avgdl.doc_weight(1, 100)); + assert!(high_bound >= high_avgdl.doc_weight(1, 100)); + assert!( + high_bound > low_bound, + "larger avgdl must recompute a larger bound: low={low_bound}, high={high_bound}" + ); + } + + #[test] + fn impact_bounds_reuse_same_scorer_key_across_queries() { + let impacts = build_impact_skip_data(&[vec![(0, 2, 10)]]).unwrap(); + let cloned = impacts.clone(); + assert!(impacts.shares_derived_state_with(&cloned)); + let calls = Arc::new(AtomicUsize::new(0)); + let scorer = KeyedCountingScorer { + key: 7, + calls: calls.clone(), + }; + let mut first_query = ImpactScoreCache::default(); + let mut second_query = ImpactScoreCache::default(); + + let first = impacts.global_max_doc_weight_cached(&scorer, &mut first_query); + let baked_calls = calls.load(Ordering::Relaxed); + assert!(baked_calls > 0); + let second = cloned.global_max_doc_weight_cached(&scorer, &mut second_query); + + assert_eq!(second, first); + assert_eq!( + calls.load(Ordering::Relaxed), + baked_calls, + "the same keyed bounds should be shared across query caches" + ); + } + + #[test] + fn malformed_unscanned_entry_does_not_poison_range_score() { + let level0_0 = encode_impact_entry(&[(0, 1, 10)]).unwrap(); + let malformed_level0_1 = vec![1, 2, 3]; + let level1 = encode_impact_entry(&[(0, 1, 10), (1, 1, 10)]).unwrap(); + let entries = LargeBinaryArray::from_opt_vec(vec![ + Some(level0_0.as_slice()), + Some(malformed_level0_1.as_slice()), + Some(level1.as_slice()), + ]); + let impacts = ImpactSkipData::new(entries, 2).unwrap(); + let scorer = MemBM25Scorer::new(10, 10, HashMap::from([(String::from("token"), 2usize)])); + let mut cache = ImpactScoreCache::default(); + + let score = impacts.max_score_up_to_cached(0, 0, 1.0, &scorer, &mut cache); + assert!(score.score.is_finite()); + assert_eq!(score.entries_scanned, 1); + + assert_eq!( + impacts.level0_score_cached(1, 1.0, &scorer, &mut cache), + f32::INFINITY + ); + } + + #[test] + fn impact_entries_store_quantized_norm_deltas() { + let docs = vec![(7, 1, 1), (9, 2, 2), (12, 3, 5)]; + let encoded = encode_impact_entry(&docs).unwrap(); + + // doc_up_to=12, pair_count=3, two implicit +1 norm deltas, then an + // explicit +3 norm delta folded behind the frequency varint's low bit. + assert_eq!(encoded, vec![12, 3, 0, 0, 1, 3]); + + let mut pairs = Vec::new(); + for_each_entry_pair(&encoded, |freq, doc_len| pairs.push((freq, doc_len))).unwrap(); + assert_eq!(pairs, vec![(1, 1), (2, 2), (3, 5)]); + } + + #[test] + fn malformed_norm_deltas_are_rejected() { + // doc_up_to=0, pair_count=1, and the pair flag promises a norm byte + // that is not present. + let truncated = [0, 1, 1]; + let error = for_each_entry_pair(&truncated, |_, _| {}).unwrap_err(); + assert!(matches!(&error, Error::Index { .. })); + assert!(error.to_string().contains("impact norm delta")); + + // The first pair reaches norm 255, so an implicit +1 on the second + // pair must fail instead of wrapping back to zero. + let overflowing = [0, 2, 1, 255, 0]; + let error = for_each_entry_pair(&overflowing, |_, _| {}).unwrap_err(); + assert!(matches!(&error, Error::Index { .. })); + assert!(error.to_string().contains("impact norm delta overflow")); + } + + #[test] + fn impact_entries_roundtrip_quantized_frontier() { + let docs = vec![(3, 1, 100), (9, 2, 40), (200, 7, 80), (4095, 130, 900)]; + let encoded = encode_impact_entry(&docs).unwrap(); + assert_eq!(decode_entry_doc_up_to(&encoded).unwrap(), 4095); + let mut decoded_pairs = Vec::new(); + for_each_entry_pair(&encoded, |freq, doc_len| { + decoded_pairs.push((freq, doc_len)) + }) + .unwrap(); + let expected_pairs = quantized_impact_frontier(&docs) + .into_iter() + .map(|(freq, norm)| (freq, super::super::index::dequantize_doc_length(norm))) + .collect::>(); + assert_eq!(decoded_pairs, expected_pairs); + assert!(!decoded_pairs.is_empty()); + + // A 256-doc-block skip data goes through the shared codec end to end. + let blocks: Vec> = (0..3) + .map(|b| (0..256).map(|i| (b * 256 + i, 1 + i % 5, 10)).collect()) + .collect(); + let impacts = build_impact_skip_data(&blocks).unwrap(); + assert_eq!(impacts.level1_doc_up_to(0), Some(767)); + let scorer = MemBM25Scorer::new(400, 768, HashMap::from([(String::from("t"), 768usize)])); + let mut cache = ImpactScoreCache::default(); + assert!( + impacts + .level0_score_cached(0, 1.0, &scorer, &mut cache) + .is_finite() + ); + let level1 = impacts.max_score_up_to_cached(0, 767, 1.0, &scorer, &mut cache); + assert!(level1.score.is_finite() && level1.score > 0.0); + } + + #[test] + fn posting_block_sizes_use_identical_impact_encoding() { + let docs = vec![(3, 1, 100), (9, 2, 40), (200, 7, 80)]; + let mut v2_builder = ImpactSkipDataBuilder::with_capacity(1, 128); + v2_builder.append_block(&docs).unwrap(); + let v2 = v2_builder.finish().unwrap(); + let mut v3_builder = ImpactSkipDataBuilder::with_capacity(1, 256); + v3_builder.append_block(&docs).unwrap(); + let v3 = v3_builder.finish().unwrap(); + + assert_eq!(v2.entries(), v3.entries()); + } + + #[test] + fn impact_upper_bound_covers_real_scores() { + let blocks = vec![ + vec![(0, 1, 100), (3, 2, 40), (7, 4, 80)], + vec![(9, 3, 15), (10, 1, 5), (12, 5, 30)], + vec![(16, 2, 10), (18, 6, 70), (21, 3, 12)], + vec![(24, 1, 4), (28, 7, 100), (30, 2, 8)], + ]; + let impacts = build_impact_skip_data(&blocks).unwrap(); + let scorer = MemBM25Scorer::new(474, 31, HashMap::from([(String::from("token"), 4usize)])); + let query_weight = scorer.query_weight("token"); + let mut cache = ImpactScoreCache::default(); + + for start_block_idx in 0..blocks.len() { + let up_to = blocks + .iter() + .skip(start_block_idx) + .take(2) + .flatten() + .map(|(doc_id, _, _)| *doc_id) + .max() + .unwrap(); + let upper_bound = impacts.max_score_up_to_cached( + start_block_idx, + u64::from(up_to), + query_weight, + &scorer, + &mut cache, + ); + let exact_max = blocks + .iter() + .skip(start_block_idx) + .flatten() + .take_while(|(doc_id, _, _)| *doc_id <= up_to) + .map(|(_, freq, doc_len)| query_weight * scorer.doc_weight(*freq, *doc_len)) + .fold(0.0_f32, f32::max); + assert!( + upper_bound.score + 1e-6 >= exact_max, + "upper bound {} should cover exact max {} from block {} up to doc {}", + upper_bound.score, + exact_max, + start_block_idx, + up_to + ); + } + } +} diff --git a/rust/lance-index/src/scalar/inverted/index.rs b/rust/lance-index/src/scalar/inverted/index.rs index d5a513c7112..ab6549b0808 100644 --- a/rust/lance-index/src/scalar/inverted/index.rs +++ b/rust/lance-index/src/scalar/inverted/index.rs @@ -3,14 +3,14 @@ use lance_core::utils::row_addr_remap::RowAddrRemap; use std::fmt::{Debug, Display}; -use std::sync::Arc; use std::sync::atomic::{AtomicBool, AtomicU32, AtomicU64, Ordering}; +use std::sync::{Arc, OnceLock}; use std::{ cmp::{Reverse, min}, collections::BinaryHeap, }; use std::{ - collections::{BTreeMap, HashMap, HashSet}, + collections::{BTreeMap, BTreeSet, HashMap, HashSet}, ops::Range, time::Instant, }; @@ -18,7 +18,8 @@ use std::{ use crate::metrics::NoOpMetricsCollector; use crate::prefilter::NoFilter; use crate::scalar::registry::{TrainingCriteria, TrainingOrdering}; -use arrow::array::{FixedSizeListBuilder, Float32Builder}; +use crate::vector::graph::OrderedFloat; +use arrow::array::{BooleanBuilder, FixedSizeListBuilder, Float32Builder, Int32Builder}; use arrow::datatypes::{self, Float32Type, Int32Type, UInt64Type}; use arrow::{ array::{ @@ -28,8 +29,8 @@ use arrow::{ }; use arrow::{buffer::ScalarBuffer, datatypes::UInt32Type}; use arrow_array::{ - Array, ArrayRef, Float32Array, LargeBinaryArray, ListArray, OffsetSizeTrait, RecordBatch, - UInt32Array, UInt64Array, + Array, ArrayRef, BooleanArray, Float32Array, LargeBinaryArray, ListArray, OffsetSizeTrait, + RecordBatch, UInt32Array, UInt64Array, }; use arrow_schema::{DataType, Field, Schema, SchemaRef}; use async_trait::async_trait; @@ -40,7 +41,9 @@ use fst::{Automaton, IntoStreamer, Streamer}; use futures::{FutureExt, Stream, StreamExt, TryStreamExt, stream}; use itertools::{Either, Itertools}; use lance_arrow::{RecordBatchExt, iter_str_array}; -use lance_core::cache::{CacheCodec, CacheKey, LanceCache, WeakLanceCache}; +use lance_core::cache::{ + CacheCodec, CacheKey, CacheKeySchema, KeyBuilder, LanceCache, WeakLanceCache, +}; use lance_core::deepsize::DeepSizeOf; use lance_core::error::{DataFusionResult, LanceOptionExt}; use lance_core::utils::address::RowAddress; @@ -50,17 +53,25 @@ use lance_core::{Error, ROW_ID, ROW_ID_FIELD, Result}; use lance_select::{RowAddrMask, RowAddrTreeMap}; use roaring::RoaringBitmap; use std::sync::LazyLock; -use tokio::{sync::OnceCell, task::spawn_blocking}; -use tracing::{info, instrument}; +use tokio::{ + sync::{Mutex, OnceCell}, + task::spawn_blocking, +}; +use tracing::{info, instrument, warn}; -use super::encoding::{PositionBlockBuilder, decode_group_starts}; +use super::documents::{ + DocId, DocLengths, DocVisibility, PartitionDocumentStore, PartitionDocuments, +}; +use super::encoding::{MAX_POSTING_BLOCK_SIZE, PositionBlockBuilder}; +use super::impact::{IMPACT_LEVEL1_BLOCKS, ImpactSkipData, ImpactSkipDataBuilder}; use super::iter::PostingListIterator; -use super::lazy_docset::LazyDocSet; -use super::{InvertedIndexBuilder, InvertedIndexParams, wand::*}; +use super::tokenizer::{LEGACY_BLOCK_SIZE, validate_block_size}; +use super::{DocumentGranularity, InvertedIndexBuilder, InvertedIndexParams, wand::*}; use super::{ builder::{ - BLOCK_SIZE, PostingGroupAccumulator, PostingGroupConfig, ScoredDoc, doc_file_path, - inverted_list_schema_for_version, posting_file_path, token_file_path, + BLOCK_SIZE, ScoredDoc, doc_file_path, + inverted_list_schema_for_version_with_block_size_and_impacts, posting_file_path, + token_file_path, }, iter::PlainPostingListIterator, query::*, @@ -79,9204 +90,41 @@ use crate::scalar::{ OldIndexDataFilter, RowIdRemapper, ScalarIndex, ScalarIndexParams, SearchResult, TokenQuery, UpdateCriteria, }; -use crate::{FtsPrewarmOptions, Index}; +use crate::{ + FtsPrewarmDiagnostics, FtsPrewarmOptions, FtsPrewarmPartitionStatus, FtsPrewarmResult, Index, +}; use crate::{prefilter::PreFilter, scalar::inverted::iter::take_fst_keys}; use std::str::FromStr; -// Version 0: Arrow TokenSetFormat (legacy) -// Version 1: Fst TokenSetFormat with per-doc compressed positions -// Version 2: Fst TokenSetFormat with shared posting-list position streams. -pub const INVERTED_INDEX_VERSION_V1: u32 = 1; -pub const INVERTED_INDEX_VERSION_V2: u32 = 2; -pub const TOKENS_FILE: &str = "tokens.lance"; -pub const INVERT_LIST_FILE: &str = "invert.lance"; -pub const DOCS_FILE: &str = "docs.lance"; -pub const METADATA_FILE: &str = "metadata.lance"; - -pub const TOKEN_COL: &str = "_token"; -pub const TOKEN_ID_COL: &str = "_token_id"; -pub const TOKEN_FST_BYTES_COL: &str = "_token_fst_bytes"; -pub const TOKEN_NEXT_ID_COL: &str = "_token_next_id"; -pub const TOKEN_TOTAL_LENGTH_COL: &str = "_token_total_length"; -pub const FREQUENCY_COL: &str = "_frequency"; -pub const POSITION_COL: &str = "_position"; -pub const COMPRESSED_POSITION_COL: &str = "_compressed_position"; -pub const POSITION_BLOCK_OFFSET_COL: &str = "_position_block_offset"; -pub const POSTING_COL: &str = "_posting"; -pub const MAX_SCORE_COL: &str = "_max_score"; -pub const LENGTH_COL: &str = "_length"; -pub const BLOCK_MAX_SCORE_COL: &str = "_block_max_score"; -pub const NUM_TOKEN_COL: &str = "_num_tokens"; -pub const SCORE_COL: &str = "_score"; -pub const TOKEN_SET_FORMAT_KEY: &str = "token_set_format"; -pub const POSTING_TAIL_CODEC_KEY: &str = "posting_tail_codec"; -pub const POSITIONS_LAYOUT_KEY: &str = "positions_layout"; -pub const POSITIONS_CODEC_KEY: &str = "positions_codec"; -/// Schema-metadata key holding the 1-indexed global-buffer id of the -/// varint-delta-encoded posting-list cache-group boundaries (issue #7040). -/// Absent on indexes written before grouping was introduced, which fall back -/// to the per-token cache path. -pub const POSTING_GROUP_OFFSETS_BUF_KEY: &str = "posting_group_offsets_buf"; -pub const POSTING_TAIL_CODEC_FIXED32_V1: &str = "fixed32_v1"; -pub const POSTING_TAIL_CODEC_VARINT_DELTA_V1: &str = "varint_delta_v1"; -pub const POSITIONS_LAYOUT_SHARED_STREAM_V2: &str = "shared_stream_v2"; -pub const POSITIONS_CODEC_VARINT_DOC_DELTA_V2: &str = "varint_doc_delta_v2"; -pub const POSITIONS_CODEC_PACKED_DELTA_V1: &str = "packed_delta_v1"; -pub const DELETED_FRAGMENTS_COL: &str = "deleted_fragments"; - -// Just a heuristic when we need to pre-allocate memory for tokens -pub const ESTIMATED_MAX_TOKENS_PER_ROW: usize = 4 * 1024; - -pub static SCORE_FIELD: LazyLock = - LazyLock::new(|| Field::new(SCORE_COL, DataType::Float32, true)); -pub static FTS_SCHEMA: LazyLock = - LazyLock::new(|| Arc::new(Schema::new(vec![ROW_ID_FIELD.clone(), SCORE_FIELD.clone()]))); -static ROW_ID_SCHEMA: LazyLock = - LazyLock::new(|| Arc::new(Schema::new(vec![ROW_ID_FIELD.clone()]))); - -pub fn resolve_fts_format_version( - value: Option<&str>, -) -> std::result::Result { - match value { - Some(value) => value.parse(), - None => Ok(default_fts_format_version()), - } -} - -pub fn default_fts_format_version() -> InvertedListFormatVersion { - InvertedListFormatVersion::V2 -} - -pub fn current_fts_format_version() -> InvertedListFormatVersion { - default_fts_format_version() -} - -pub fn max_supported_fts_format_version() -> InvertedListFormatVersion { - InvertedListFormatVersion::V2 -} - -#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Default)] -pub enum InvertedListFormatVersion { - V1, - #[default] - V2, -} - -impl InvertedListFormatVersion { - pub fn from_posting_tail_codec(codec: PostingTailCodec) -> Self { - match codec { - PostingTailCodec::Fixed32 => Self::V1, - PostingTailCodec::VarintDelta => Self::V2, - } - } - - pub fn index_version(self) -> u32 { - match self { - Self::V1 => INVERTED_INDEX_VERSION_V1, - Self::V2 => INVERTED_INDEX_VERSION_V2, - } - } - - pub fn posting_tail_codec(self) -> PostingTailCodec { - match self { - Self::V1 => PostingTailCodec::Fixed32, - Self::V2 => PostingTailCodec::VarintDelta, - } - } - - pub fn position_codec(self) -> Option { - match self { - Self::V1 => None, - Self::V2 => Some(PositionStreamCodec::PackedDelta), - } - } - - pub fn uses_shared_position_stream(self) -> bool { - matches!(self, Self::V2) - } -} - -impl FromStr for InvertedListFormatVersion { - type Err = Error; - - fn from_str(s: &str) -> std::result::Result { - match s.trim() { - "1" | "v1" | "V1" => Ok(Self::V1), - "2" | "v2" | "V2" => Ok(Self::V2), - other => Err(Error::index(format!( - "unsupported FTS format version {}, expected 1 or 2", - other - ))), - } - } -} - -#[derive(Debug)] -struct PartitionCandidates { - tokens_by_position: Vec, - grouped_expansions: Vec, - candidates: Vec, -} - -impl PartitionCandidates { - fn empty() -> Self { - Self { - tokens_by_position: Vec::new(), - grouped_expansions: Vec::new(), - candidates: Vec::new(), - } - } -} - -#[derive(Debug)] -struct LoadedPostings { - postings: Vec, - grouped_expansions: Vec, -} - -impl LoadedPostings { - fn empty() -> Self { - Self { - postings: Vec::new(), - grouped_expansions: Vec::new(), - } - } -} - -#[derive(Debug)] -struct GroupedExpansionTerms { - position: u32, - terms: Vec, -} - -fn grouped_rescore_wand_limit( - limit: Option, - grouped_expansions: &[GroupedExpansionTerms], -) -> Option { - let limit = limit?; - // Grouped fuzzy AND rescoring needs a small candidate cushion because WAND - // ranks by the unioned group posting first and the exact expansion IDF later. - let expansion_terms = grouped_expansions - .iter() - .map(|group| group.terms.len()) - .sum::() - .max(1); - Some(limit.saturating_mul(expansion_terms)) -} - -#[derive(Debug)] -struct ExpansionTermFreqs { - token: String, - freqs_by_posting_doc_id: Vec<(u64, u32)>, -} - -impl ExpansionTermFreqs { - fn new(token: String, posting: &PostingList) -> Self { - let freqs_by_posting_doc_id = posting - .iter() - .map(|(posting_doc_id, freq, _)| (posting_doc_id, freq)) - .collect(); - Self { - token, - freqs_by_posting_doc_id, - } - } - - fn frequency(&self, posting_doc_id: u64) -> Option { - self.freqs_by_posting_doc_id - .binary_search_by_key(&posting_doc_id, |(doc_id, _)| *doc_id) - .ok() - .map(|idx| self.freqs_by_posting_doc_id[idx].1) - } -} - -#[derive(Copy, Clone, Debug, Eq, PartialEq, Hash, Default)] -pub enum TokenSetFormat { - Arrow, - #[default] - Fst, -} - -impl Display for TokenSetFormat { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - match self { - Self::Arrow => f.write_str("arrow"), - Self::Fst => f.write_str("fst"), - } - } -} - -impl FromStr for TokenSetFormat { - type Err = Error; - - fn from_str(s: &str) -> std::result::Result { - match s.trim() { - "" => Ok(Self::Arrow), - "arrow" => Ok(Self::Arrow), - "fst" => Ok(Self::Fst), - other => Err(Error::index(format!( - "unsupported token set format {}", - other - ))), - } - } -} - -impl DeepSizeOf for TokenSetFormat { - fn deep_size_of_children(&self, _: &mut lance_core::deepsize::Context) -> usize { - 0 - } -} - -#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Default)] -pub enum PositionStreamCodec { - VarintDocDelta, - #[default] - PackedDelta, -} - -#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Default)] -pub enum PostingTailCodec { - Fixed32, - #[default] - VarintDelta, -} - -impl PostingTailCodec { - pub fn as_str(self) -> &'static str { - match self { - Self::Fixed32 => POSTING_TAIL_CODEC_FIXED32_V1, - Self::VarintDelta => POSTING_TAIL_CODEC_VARINT_DELTA_V1, - } - } - - fn from_metadata_value(value: &str) -> Result { - match value.trim() { - POSTING_TAIL_CODEC_FIXED32_V1 => Ok(Self::Fixed32), - POSTING_TAIL_CODEC_VARINT_DELTA_V1 => Ok(Self::VarintDelta), - other => Err(Error::index(format!( - "unsupported posting tail codec {}", - other - ))), - } - } -} - -pub(super) fn parse_posting_tail_codec( - metadata: &HashMap, -) -> Result { - Ok(metadata - .get(POSTING_TAIL_CODEC_KEY) - .map(|codec| PostingTailCodec::from_metadata_value(codec)) - .transpose()? - .unwrap_or(PostingTailCodec::Fixed32)) -} - -impl PositionStreamCodec { - pub fn as_str(self) -> &'static str { - match self { - Self::VarintDocDelta => POSITIONS_CODEC_VARINT_DOC_DELTA_V2, - Self::PackedDelta => POSITIONS_CODEC_PACKED_DELTA_V1, - } - } - - fn from_metadata_value(value: &str) -> Result { - match value.trim() { - POSITIONS_CODEC_VARINT_DOC_DELTA_V2 => Ok(Self::VarintDocDelta), - POSITIONS_CODEC_PACKED_DELTA_V1 => Ok(Self::PackedDelta), - other => Err(Error::index(format!( - "unsupported positions codec {}", - other - ))), - } - } -} - -fn parse_shared_position_codec(metadata: &HashMap) -> Result { - if let Some(codec) = metadata.get(POSITIONS_CODEC_KEY) { - return PositionStreamCodec::from_metadata_value(codec); - } - - match metadata - .get(POSITIONS_LAYOUT_KEY) - .map(|layout| layout.as_str()) - { - Some(POSITIONS_LAYOUT_SHARED_STREAM_V2) => Ok(PositionStreamCodec::VarintDocDelta), - _ => Ok(PositionStreamCodec::VarintDocDelta), - } -} - -pub(super) fn parse_format_version_from_metadata( - metadata: &HashMap, -) -> Result { - if metadata.contains_key(POSITIONS_CODEC_KEY) || metadata.contains_key(POSITIONS_LAYOUT_KEY) { - return Ok(InvertedListFormatVersion::V2); - } - if parse_posting_tail_codec(metadata)? == PostingTailCodec::VarintDelta { - Ok(InvertedListFormatVersion::V2) - } else { - Ok(InvertedListFormatVersion::V1) - } -} - -#[derive(Clone)] -pub struct InvertedIndex { - params: InvertedIndexParams, - store: Arc, - tokenizer: Box, - token_set_format: TokenSetFormat, - format_version: InvertedListFormatVersion, - pub(crate) partitions: Vec>, - corpus_stats: Arc>, - // Fragments which are contained in the index, but no longer in the dataset. - // These should be pruned at search time since we don't prune them at update time. - deleted_fragments: RoaringBitmap, -} - -impl Debug for InvertedIndex { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - f.debug_struct("InvertedIndex") - .field("params", &self.params) - .field("token_set_format", &self.token_set_format) - .field("format_version", &self.format_version) - .field("partitions", &self.partitions) - .field("deleted_fragments", &self.deleted_fragments) - .finish() - } -} - -impl DeepSizeOf for InvertedIndex { - fn deep_size_of_children(&self, context: &mut lance_core::deepsize::Context) -> usize { - self.partitions.deep_size_of_children(context) - } -} - -/// Resolve any `Pending` candidates that wand emitted via the -/// deferred-row_id path. After this returns, every entry in -/// `candidates` carries a real row_id. -async fn resolve_deferred_candidates( - docs: &LazyDocSet, - candidates: &mut [DocCandidate], -) -> Result<()> { - let pending: Vec = candidates - .iter() - .filter_map(|c| match c.addr { - CandidateAddr::Pending(d) => Some(d), - CandidateAddr::RowId(_) => None, - }) - .collect(); - if pending.is_empty() { - return Ok(()); - } - let mut iter = docs.resolve_row_ids(&pending).await?.into_iter(); - for c in candidates { - if matches!(c.addr, CandidateAddr::Pending(_)) { - let r = iter.next().ok_or_else(|| { - Error::internal("resolve_row_ids returned fewer items than requested") - })?; - c.addr = CandidateAddr::RowId(r); - } - } - Ok(()) -} - -impl InvertedIndex { - fn format_version(&self) -> InvertedListFormatVersion { - self.format_version - } - - fn index_version(&self) -> u32 { - match self.token_set_format { - TokenSetFormat::Arrow => 0, - TokenSetFormat::Fst => self.format_version().index_version(), - } - } - - fn posting_tail_codec(&self) -> PostingTailCodec { - self.partitions - .first() - .map(|partition| partition.inverted_list.posting_tail_codec()) - .unwrap_or_default() - } - - fn to_builder(&self) -> InvertedIndexBuilder { - self.to_builder_with_offset(None) - } - - fn to_builder_with_offset(&self, fragment_mask: Option) -> InvertedIndexBuilder { - if self.is_legacy() { - // for legacy format, we re-create the index in the new format - InvertedIndexBuilder::from_existing_index( - self.params.clone(), - None, - Vec::new(), - self.token_set_format, - fragment_mask, - self.deleted_fragments.clone(), - ) - .with_posting_tail_codec(self.posting_tail_codec()) - } else { - let partitions = match fragment_mask { - Some(fragment_mask) => self - .partitions - .iter() - // Filter partitions that belong to the specified fragment - // The mask contains fragment_id in high 32 bits, we check if partition's - // fragment_id matches by comparing the masked result with the original mask - .filter(|part| part.belongs_to_fragment(fragment_mask)) - .map(|part| part.id()) - .collect(), - None => self.partitions.iter().map(|part| part.id()).collect(), - }; - - InvertedIndexBuilder::from_existing_index( - self.params.clone(), - Some(self.store.clone()), - partitions, - self.token_set_format, - fragment_mask, - self.deleted_fragments.clone(), - ) - .with_format_version(self.format_version()) - } - } - - pub fn tokenizer(&self) -> Box { - self.tokenizer.clone() - } - - pub fn params(&self) -> &InvertedIndexParams { - &self.params - } - - /// Returns the number of partitions in this inverted index. - pub fn partition_count(&self) -> usize { - self.partitions.len() - } - /// Returns the set of fragments which are contained in the index, but no longer in the dataset. - /// - /// Most other indices remove data from deleted fragments when the index updates (copy-on-write). - /// However, this would require an expensive copy of the FTS index. Instead, we track the deleted - /// fragments and prune them at search time (merge-on-read). - pub fn deleted_fragments(&self) -> &RoaringBitmap { - &self.deleted_fragments - } - - pub async fn merge_segments( - segments: &[Arc], - new_data: SendableRecordBatchStream, - dest_store: &dyn IndexStore, - old_data_filter: Option, - progress: Arc, - ) -> Result { - let Some(first) = segments.first() else { - return Err(Error::invalid_input( - "cannot merge inverted index without at least one source segment".to_string(), - )); - }; - - for segment in segments.iter().skip(1) { - if segment.params != first.params { - return Err(Error::index( - "cannot merge inverted index segments with different parameters".to_string(), - )); - } - if segment.token_set_format != first.token_set_format { - return Err(Error::index( - "cannot merge inverted index segments with different token set formats" - .to_string(), - )); - } - if segment.format_version() != first.format_version() { - return Err(Error::index( - "cannot merge inverted index segments with different format versions" - .to_string(), - )); - } - if segment.posting_tail_codec() != first.posting_tail_codec() { - return Err(Error::index( - "cannot merge inverted index segments with different posting tail codecs" - .to_string(), - )); - } - } - - let mut builder = InvertedIndexBuilder::new(first.params.clone()).with_progress(progress); - builder = builder - .with_token_set_format(first.token_set_format) - .with_format_version(first.format_version()) - .with_posting_tail_codec(first.posting_tail_codec()); - let files = builder - .update_from_segments(new_data, dest_store, segments, old_data_filter) - .await?; - - let details = pbold::InvertedIndexDetails::try_from(&first.params)?; - - Ok(CreatedIndex { - index_details: prost_types::Any::from_msg(&details).unwrap(), - index_version: first.index_version(), - files, - }) - } - - /// Build a single-segment [`MemBM25Scorer`] whose per-term IDF table - /// covers every token that the per-partition scoring loop will look - /// up. For fuzzy queries that means the union of Levenshtein - /// expansions, not just the raw query tokens — otherwise - /// `query_weight(expanded_token)` returns 0 and the BM25 contribution - /// of every expanded match is discarded. - pub async fn bm25_base_scorer( - &self, - query_tokens: &Tokens, - params: &FtsSearchParams, - ) -> Result { - let (total_tokens, num_docs) = self.aggregate_corpus_stats().await?; - let mut terms: Vec = Vec::new(); - let mut seen = HashSet::new(); - if matches!(params.fuzziness, Some(n) if n != 0) { - let expanded = self.expand_fuzzy_tokens(query_tokens, params)?; - for idx in 0..expanded.len() { - let token = expanded.get_token(idx); - if seen.insert(token.to_string()) { - terms.push(token.to_string()); - } - } - } else { - for token in query_tokens { - if seen.insert(token.to_string()) { - terms.push(token.to_string()); - } - } - } - let mut token_docs = HashMap::with_capacity(terms.len()); - for term in &terms { - let df = self.df_for_term(term).await?; - token_docs.insert(term.clone(), df); - } - Ok(MemBM25Scorer::new(total_tokens, num_docs, token_docs)) - } - - pub async fn bm25_stats_for_terms(&self, terms: &[String]) -> Result<(u64, usize, Vec)> { - let (total_tokens, num_docs) = self.aggregate_corpus_stats().await?; - let token_docs = - futures::future::try_join_all(terms.iter().map(|term| self.df_for_term(term))).await?; - Ok((total_tokens, num_docs, token_docs)) - } - - /// Aggregate per-partition `total_tokens` and `num_docs` across the - /// index. `len` is cheap (no IO); `total_tokens_num` reads only the - /// num_tokens column the first time per partition and caches it on - /// `LazyDocSet`. Avoids materializing the full DocSet just to get - /// these two scalars. - async fn aggregate_corpus_stats(&self) -> Result<(u64, usize)> { - self.corpus_stats - .get_or_try_init(|| async { - let io_parallelism = self.store.io_parallelism(); - let num_docs: usize = self.partitions.iter().map(|p| p.docs.len()).sum(); - let futures = self - .partitions - .iter() - .map(|p| { - let docs = p.docs.clone(); - async move { docs.total_tokens_num().await } - }) - .collect::>(); - let totals: Vec = stream::iter(futures) - .buffer_unordered(io_parallelism) - .try_collect() - .await?; - Ok((totals.into_iter().sum(), num_docs)) - }) - .await - .copied() - } - - /// Sum the posting-list length for `term` across this index's partitions - /// via single-row reads, with partition lookups bounded by the store's - /// `io_parallelism()`. - async fn df_for_term(&self, term: &str) -> Result { - let io_parallelism = self.store.io_parallelism(); - let futures = self - .partitions - .iter() - .map(|part| { - let part = part.clone(); - async move { - match part.tokens.get(term) { - Some(token_id) => part.inverted_list.posting_len_for_token(token_id).await, - None => Ok(0), - } - } - }) - .collect::>(); - let dfs: Vec = stream::iter(futures) - .buffer_unordered(io_parallelism) - .try_collect() - .await?; - Ok(dfs.into_iter().sum()) - } - - /// Expand fuzzy query tokens against all partitions in this segment. - pub fn expand_fuzzy_tokens(&self, tokens: &Tokens, params: &FtsSearchParams) -> Result { - let mut expanded_tokens = Vec::new(); - let mut expanded_positions = Vec::new(); - let mut seen = HashSet::new(); - for partition in &self.partitions { - let expanded = partition.expand_fuzzy(tokens, params)?; - for idx in 0..expanded.len() { - let token = expanded.get_token(idx); - let position = expanded.position(idx); - if seen.insert((token.to_string(), position)) { - expanded_tokens.push(token.to_string()); - expanded_positions.push(position); - } - } - } - Ok(Tokens::with_positions( - expanded_tokens, - expanded_positions, - tokens.token_type().clone(), - )) - } - - /// Search documents that match the query and return row ids sorted by BM25 score. - /// - /// When `base_scorer` is provided, search uses those corpus-level BM25 statistics - /// instead of deriving them from this segment alone. - #[instrument(level = "debug", skip_all)] - pub async fn bm25_search( - &self, - tokens: Arc, - params: Arc, - operator: Operator, - prefilter: Arc, - metrics: Arc, - base_scorer: Option<&MemBM25Scorer>, - ) -> Result<(Vec, Vec)> { - // The wand only consults `scorer.doc_weight`, which is metadata-free. - // The outer aggregation below consults `scorer.query_weight`, which - // hits per-token `posting_len`; building a `MemBM25Scorer` with - // precomputed per-term IDFs avoids the v2 bulk metadata pull. - let local_scorer; - let scorer: &dyn Scorer = if let Some(base_scorer) = base_scorer { - base_scorer - } else { - local_scorer = self - .bm25_base_scorer(tokens.as_ref(), params.as_ref()) - .await?; - &local_scorer - }; - - let limit = params.limit.unwrap_or(usize::MAX); - if limit == 0 { - return Ok((Vec::new(), Vec::new())); - } - - fn push_scored_candidate( - candidates: &mut BinaryHeap>, - limit: usize, - addr: CandidateAddr, - score: f32, - ) -> Result<()> { - // resolve_deferred_candidates ran upstream, so every candidate - // carries a real row_id at this point. - let row_id = match addr { - CandidateAddr::RowId(r) => r, - CandidateAddr::Pending(_) => { - return Err(Error::internal( - "bm25_search post-condition: deferred candidate left unresolved", - )); - } - }; - - if candidates.len() < limit { - candidates.push(Reverse(ScoredDoc::new(row_id, score))); - } else if candidates.peek().unwrap().0.score.0 < score { - candidates.pop(); - candidates.push(Reverse(ScoredDoc::new(row_id, score))); - } - Ok(()) - } - - let mask = prefilter.mask(); - - let mut candidates = BinaryHeap::new(); - // Shared top-k floor across this query's partitions. Seeded to -inf so - // the first real score wins; each partition publishes its local k-th - // and prunes against the running global k-th (a lower bound on the true - // global k-th — see `Wand::shared_threshold`). - let shared_threshold = Arc::new(AtomicU32::new(f32::NEG_INFINITY.to_bits())); - let parts = self - .partitions - .iter() - .map(|part| { - let part = part.clone(); - let tokens = tokens.clone(); - let params = params.clone(); - let mask = mask.clone(); - let metrics = metrics.clone(); - let shared_threshold = shared_threshold.clone(); - async move { - let loaded_postings = part - .load_posting_lists( - tokens.as_ref(), - params.as_ref(), - operator, - metrics.as_ref(), - ) - .await?; - let LoadedPostings { - postings, - grouped_expansions, - } = loaded_postings; - if postings.is_empty() { - // No hits in this partition; its DocSet stays - // unloaded, so we never pay the per-doc - // row_id/num_tokens download for it. - return Result::Ok(PartitionCandidates::empty()); - } - let docs_for_wand = part.docs.docs_for_wand(mask.as_ref()).await?; - let max_position = postings - .iter() - .map(|posting| posting.term_index() as usize) - .max() - .unwrap_or_default(); - let mut tokens_by_position = vec![String::new(); max_position + 1]; - for posting in &postings { - let idx = posting.term_index() as usize; - tokens_by_position[idx] = posting.token().to_owned(); - } - let params = params.clone(); - let mask = mask.clone(); - let metrics = metrics.clone(); - let part_for_wand = part.clone(); - let has_grouped_expansions = !grouped_expansions.is_empty(); - let wand_params = if has_grouped_expansions { - let mut rescoring_params = params.as_ref().clone(); - rescoring_params.limit = - grouped_rescore_wand_limit(params.limit, &grouped_expansions); - Arc::new(rescoring_params) - } else { - params.clone() - }; - let partition_threshold = if has_grouped_expansions { - Arc::new(AtomicU32::new(f32::NEG_INFINITY.to_bits())) - } else { - shared_threshold - }; - let candidates = spawn_cpu(move || { - let candidates = part_for_wand.bm25_search( - docs_for_wand.as_ref(), - wand_params.as_ref(), - operator, - mask, - postings, - metrics.as_ref(), - partition_threshold, - )?; - std::result::Result::<_, Error>::Ok(candidates) - }) - .await?; - let mut partition_result = PartitionCandidates { - tokens_by_position, - grouped_expansions, - candidates, - }; - resolve_deferred_candidates(&part.docs, &mut partition_result.candidates) - .await?; - Result::Ok(partition_result) - } - }) - .collect::>(); - let mut parts = stream::iter(parts).buffer_unordered(get_num_compute_intensive_cpus()); - let mut idf_cache: HashMap = HashMap::new(); - while let Some(res) = parts.try_next().await? { - if res.candidates.is_empty() { - continue; - } - let PartitionCandidates { - tokens_by_position, - grouped_expansions, - candidates: part_candidates, - } = res; - let mut idf_by_position = Vec::with_capacity(tokens_by_position.len()); - for token in &tokens_by_position { - let idf_weight = match idf_cache.get(token) { - Some(weight) => *weight, - None => { - let weight = scorer.query_weight(token); - idf_cache.insert(token.clone(), weight); - weight - } - }; - idf_by_position.push(idf_weight); - } - - if grouped_expansions.is_empty() { - for DocCandidate { - addr, - freqs, - doc_length, - .. - } in part_candidates - { - let mut score = 0.0; - for (term_index, freq) in freqs.into_iter() { - debug_assert!((term_index as usize) < idf_by_position.len()); - score += idf_by_position[term_index as usize] - * scorer.doc_weight(freq, doc_length); - } - push_scored_candidate(&mut candidates, limit, addr, score)?; - } - } else { - let grouped_positions = grouped_expansions - .iter() - .map(|group| group.position) - .collect::>(); - for DocCandidate { - addr, - posting_doc_id, - freqs, - doc_length, - } in part_candidates - { - let mut score = 0.0; - for (term_index, freq) in freqs.into_iter() { - if grouped_positions.contains(&term_index) { - continue; - } - debug_assert!((term_index as usize) < idf_by_position.len()); - score += idf_by_position[term_index as usize] - * scorer.doc_weight(freq, doc_length); - } - for group in &grouped_expansions { - for term in &group.terms { - let Some(freq) = term.frequency(posting_doc_id) else { - continue; - }; - let idf_weight = match idf_cache.get(&term.token) { - Some(weight) => *weight, - None => { - let weight = scorer.query_weight(&term.token); - idf_cache.insert(term.token.clone(), weight); - weight - } - }; - score += idf_weight * scorer.doc_weight(freq, doc_length); - } - } - push_scored_candidate(&mut candidates, limit, addr, score)?; - } - } - } - - Ok(candidates - .into_sorted_vec() - .into_iter() - .map(|Reverse(doc)| (doc.row_id, doc.score.0)) - .unzip()) - } - - async fn load_legacy_index( - store: Arc, - frag_reuse_index: Option>, - index_cache: &LanceCache, - ) -> Result> { - log::warn!("loading legacy FTS index"); - let tokens_fut = tokio::spawn({ - let store = store.clone(); - async move { - let token_reader = store.open_index_file(TOKENS_FILE).await?; - let tokenizer = token_reader - .schema() - .metadata - .get("tokenizer") - .map(|s| serde_json::from_str::(s)) - .transpose()? - .unwrap_or_default(); - let tokens = TokenSet::load(token_reader, TokenSetFormat::Arrow).await?; - Result::Ok((tokenizer, tokens)) - } - }); - let invert_list_fut = tokio::spawn({ - let store = store.clone(); - let index_cache_clone = index_cache.clone(); - async move { - let invert_list_reader = store.open_index_file(INVERT_LIST_FILE).await?; - let invert_list = - PostingListReader::try_new(invert_list_reader, &index_cache_clone).await?; - Result::Ok(Arc::new(invert_list)) - } - }); - let docs_fut = tokio::spawn({ - let store = store.clone(); - async move { - let docs_reader = store.open_index_file(DOCS_FILE).await?; - let docs = DocSet::load(docs_reader, true, frag_reuse_index).await?; - Result::Ok(docs) - } - }); - - let (tokenizer_config, tokens) = tokens_fut.await??; - let inverted_list = invert_list_fut.await??; - let docs = docs_fut.await??; - - let tokenizer = tokenizer_config.build()?; - - Ok(Arc::new(Self { - params: tokenizer_config, - store: store.clone(), - tokenizer, - token_set_format: TokenSetFormat::Arrow, - format_version: InvertedListFormatVersion::V1, - partitions: vec![Arc::new(InvertedPartition { - id: 0, - store, - tokens, - inverted_list, - docs: Arc::new(LazyDocSet::from_loaded(docs)), - token_set_format: TokenSetFormat::Arrow, - })], - corpus_stats: Arc::new(OnceCell::new()), - deleted_fragments: RoaringBitmap::new(), - })) - } - - pub fn is_legacy(&self) -> bool { - self.partitions.len() == 1 && self.partitions[0].is_legacy() - } - - pub async fn load( - store: Arc, - frag_reuse_index: Option>, - index_cache: &LanceCache, - ) -> Result> - where - Self: Sized, - { - // for new index format, there is a metadata file and multiple partitions, - // each partition is a separate index containing tokens, inverted list and docs. - // for old index format, there is no metadata file, and it's just like a single partition - - match store.open_index_file(METADATA_FILE).await { - Ok(reader) => { - let params = reader - .schema() - .metadata - .get("params") - .ok_or(Error::index("params not found in metadata".to_owned()))?; - let params = serde_json::from_str::(params)?; - let partitions = reader - .schema() - .metadata - .get("partitions") - .ok_or(Error::index("partitions not found in metadata".to_owned()))?; - let partitions: Vec = serde_json::from_str(partitions)?; - let token_set_format = reader - .schema() - .metadata - .get(TOKEN_SET_FORMAT_KEY) - .map(|name| TokenSetFormat::from_str(name)) - .transpose()? - .unwrap_or(TokenSetFormat::Arrow); - let format_version = parse_format_version_from_metadata(&reader.schema().metadata)?; - - // Load deleted_fragments if present (optional for backward compatibility) - let deleted_fragments = if reader.num_rows() > 0 { - let metadata_batch = reader.read_range(0..1, None).await?; - if let Some(col) = metadata_batch.column_by_name(DELETED_FRAGMENTS_COL) { - let arr = col.as_binary_opt::().expect_ok()?; - RoaringBitmap::deserialize_from(arr.value(0))? - } else { - RoaringBitmap::new() - } - } else { - RoaringBitmap::new() - }; - - let format = token_set_format; - let partitions = partitions.into_iter().enumerate().map(|(priority, id)| { - let store = store.with_io_priority(priority as u64); - let frag_reuse_index_clone = frag_reuse_index.clone(); - let index_cache_for_part = - index_cache.with_key_prefix(format!("part-{}", id).as_str()); - let token_set_format = format; - async move { - Result::Ok(Arc::new( - InvertedPartition::load( - store, - id, - frag_reuse_index_clone, - &index_cache_for_part, - token_set_format, - ) - .await?, - )) - } - }); - let partitions = stream::iter(partitions) - .buffer_unordered(store.io_parallelism()) - .try_collect::>() - .await?; - - let tokenizer = params.build()?; - Ok(Arc::new(Self { - params, - store, - tokenizer, - token_set_format, - format_version, - partitions, - corpus_stats: Arc::new(OnceCell::new()), - deleted_fragments, - })) - } - Err(_) => { - // old index format - Self::load_legacy_index(store, frag_reuse_index, index_cache).await - } - } - } -} - -#[async_trait] -impl Index for InvertedIndex { - fn as_any(&self) -> &dyn std::any::Any { - self - } - - fn as_index(self: Arc) -> Arc { - self - } - - fn statistics(&self) -> Result { - let num_tokens = self - .partitions - .iter() - .map(|part| part.tokens.len()) - .sum::(); - let num_docs = self - .partitions - .iter() - .map(|part| part.docs.len()) - .sum::(); - Ok(serde_json::json!({ - "params": self.params, - "num_tokens": num_tokens, - "num_docs": num_docs, - })) - } - - async fn prewarm(&self) -> Result<()> { - self.prewarm_with_options(&FtsPrewarmOptions::default()) - .await - } - - fn index_type(&self) -> crate::IndexType { - crate::IndexType::Inverted - } - - async fn calculate_included_frags(&self) -> Result { - unimplemented!() - } -} - -/// Target on-disk size of one prewarm chunk. Keep this large enough that cloud -/// stores do not spend prewarm time on thousands of tiny range reads, but still -/// bounded so one large partition is not materialized all at once. -const PREWARM_CHUNK_TARGET_BYTES: u64 = 128 << 20; - -/// Cap on token rows per chunk, bounding the built `Vec` when posting lists are tiny. -const PREWARM_MAX_CHUNK_TOKENS: usize = 256 * 1024; - -/// Floor on token rows per chunk, so a partition always makes progress. -const PREWARM_MIN_CHUNK_TOKENS: usize = 1; - -/// Token rows per chunk: byte target / average bytes-per-token, clamped to `[MIN, MAX]`. -fn prewarm_chunk_tokens(token_count: usize, file_size_bytes: u64) -> usize { - if token_count == 0 { - return PREWARM_MIN_CHUNK_TOKENS; - } - let bytes_per_token = (file_size_bytes / token_count as u64).max(1); // >= 1: no div-by-zero - let by_bytes = (PREWARM_CHUNK_TARGET_BYTES / bytes_per_token) as usize; - by_bytes.clamp(PREWARM_MIN_CHUNK_TOKENS, PREWARM_MAX_CHUNK_TOKENS) -} - -/// Snap a chunk's exclusive token end back to a posting-group boundary so no group -/// straddles chunks. Returns the largest group boundary in `(tok_start, desired_end]`, -/// or the next boundary past an oversized group so it runs as one solo chunk. -fn group_aligned_chunk_end( - starts: &[u32], - token_count: usize, - tok_start: usize, - desired_end: usize, -) -> usize { - if desired_end >= token_count { - return token_count; - } - - let first_after_start = starts.partition_point(|&start| start as usize <= tok_start); - let first_after_desired = starts.partition_point(|&start| start as usize <= desired_end); - if first_after_desired > first_after_start { - return starts[first_after_desired - 1] as usize; - } - - // Oversized group: extend to its end so it runs as one chunk. - starts - .get(first_after_start) - .map(|&start| start as usize) - .unwrap_or(token_count) -} - -fn prewarm_chunk_ranges( - group_starts: Option<&[u32]>, - token_count: usize, - chunk_tokens: usize, -) -> Vec<(usize, usize)> { - let mut ranges = Vec::new(); - let mut tok_start = 0usize; - while tok_start < token_count { - let mut tok_end = (tok_start + chunk_tokens).min(token_count); - // `tok_start` is always a group boundary; snap `tok_end` back to one too. - if let Some(starts) = group_starts { - tok_end = group_aligned_chunk_end(starts, token_count, tok_start, tok_end); - } - ranges.push((tok_start, tok_end)); - tok_start = tok_end; - } - ranges -} - -fn group_start_indices_for_chunk(starts: &[u32], tok_start: usize, tok_end: usize) -> Range { - let first = starts.partition_point(|&start| (start as usize) < tok_start); - let end = starts.partition_point(|&start| (start as usize) < tok_end); - first..end -} - -fn group_range_for_start_index(starts: &[u32], token_count: usize, group_idx: usize) -> (u32, u32) { - let start = starts[group_idx]; - let end = starts - .get(group_idx + 1) - .copied() - .unwrap_or(token_count as u32); - (start, end) -} - -impl InvertedIndex { - pub async fn prewarm_with_options(&self, options: &FtsPrewarmOptions) -> Result<()> { - let with_position = options.with_position; - let chunk_concurrency = self.store.io_parallelism().max(1); - for part in &self.partitions { - part.inverted_list - .prewarm_posting_lists(with_position, chunk_concurrency) - .await?; - // Materialize the deferred DocSet too: prewarm's contract is - // that subsequent queries do no IO, so the per-doc row_ids / - // num_tokens must be resident, not lazily faulted in at query - // time. `ensure_loaded` opens, reads, and drops the reader. - part.docs.ensure_loaded().await?; - } - Ok(()) - } - /// Search docs match the input text. - async fn do_search(&self, text: &str) -> Result { - let params = FtsSearchParams::new(); - let mut tokenizer = self.tokenizer.clone(); - let tokens = collect_query_tokens(text, &mut tokenizer); - - let (doc_ids, _) = self - .bm25_search( - Arc::new(tokens), - params.into(), - Operator::And, - Arc::new(NoFilter), - Arc::new(NoOpMetricsCollector), - None, - ) - .boxed() - .await?; - - Ok(RecordBatch::try_new( - ROW_ID_SCHEMA.clone(), - vec![Arc::new(UInt64Array::from(doc_ids))], - )?) - } -} - -#[async_trait] -impl ScalarIndex for InvertedIndex { - // return the row ids of the documents that contain the query - #[instrument(level = "debug", skip_all)] - async fn search( - &self, - query: &dyn AnyQuery, - _metrics: &dyn MetricsCollector, - ) -> Result { - let query = query.as_any().downcast_ref::().unwrap(); - - match query { - TokenQuery::TokensContains(text) => { - let records = self.do_search(text).await?; - let row_ids = records - .column(0) - .as_any() - .downcast_ref::() - .unwrap(); - let row_ids = row_ids.iter().flatten().collect_vec(); - Ok(SearchResult::at_most(RowAddrTreeMap::from_iter(row_ids))) - } - } - } - - fn can_remap(&self) -> bool { - true - } - - async fn remap( - &self, - mapping: &RowAddrRemap, - dest_store: &dyn IndexStore, - ) -> Result { - let files = self - .to_builder() - .remap(mapping, self.store.clone(), dest_store) - .await?; - - let details = pbold::InvertedIndexDetails::try_from(&self.params)?; - - Ok(CreatedIndex { - index_details: prost_types::Any::from_msg(&details).unwrap(), - index_version: self.index_version(), - files, - }) - } - - async fn update( - &self, - new_data: SendableRecordBatchStream, - dest_store: &dyn IndexStore, - old_data_filter: Option, - ) -> Result { - let files = self - .to_builder() - .update(new_data, dest_store, old_data_filter) - .await?; - - let details = pbold::InvertedIndexDetails::try_from(&self.params)?; - - Ok(CreatedIndex { - index_details: prost_types::Any::from_msg(&details).unwrap(), - index_version: self.index_version(), - files, - }) - } - - fn update_criteria(&self) -> UpdateCriteria { - let criteria = TrainingCriteria::new(TrainingOrdering::None).with_row_id(); - if self.is_legacy() { - UpdateCriteria::requires_old_data(criteria) - } else { - UpdateCriteria::only_new_data(criteria) - } - } - - fn derive_index_params(&self) -> Result { - let mut params = self.params.clone(); - if params.base_tokenizer.is_empty() { - // Empty tokenizer metadata only appears in legacy simple-tokenizer indexes. - params.base_tokenizer = "simple".to_string(); - } - params = params.format_version(self.format_version()); - - let params_json = params.to_training_json()?.to_string(); - - Ok(ScalarIndexParams { - index_type: BuiltinIndexType::Inverted.as_str().to_string(), - params: Some(params_json), - }) - } -} - -#[derive(Debug, Clone, DeepSizeOf)] -pub struct InvertedPartition { - // 0 for legacy format - id: u64, - store: Arc, - pub(crate) tokens: TokenSet, - pub(crate) inverted_list: Arc, - /// Per-doc row_id + num_tokens. Wrapped in `LazyDocSet` so partitions - /// that don't contribute hits to a query never pay the full-array - /// download. Scoring paths call `ensure_loaded` before walking wand. - pub(crate) docs: Arc, - token_set_format: TokenSetFormat, -} - -impl InvertedPartition { - /// Check if this partition belongs to the specified fragment. - /// - /// This method encapsulates the bit manipulation logic for fragment filtering - /// in distributed indexing scenarios. - /// - /// # Arguments - /// * `fragment_mask` - A mask with fragment_id in high 32 bits - /// - /// # Returns - /// * `true` if the partition belongs to the fragment, `false` otherwise - pub fn belongs_to_fragment(&self, fragment_mask: u64) -> bool { - (self.id() & fragment_mask) == fragment_mask - } - - pub fn id(&self) -> u64 { - self.id - } - - pub fn store(&self) -> &dyn IndexStore { - self.store.as_ref() - } - - pub fn is_legacy(&self) -> bool { - self.inverted_list.is_legacy_layout() - } - - pub async fn load( - store: Arc, - id: u64, - frag_reuse_index: Option>, - index_cache: &LanceCache, - token_set_format: TokenSetFormat, - ) -> Result { - let token_file = store.open_index_file(&token_file_path(id)).await?; - let tokens = TokenSet::load(token_file, token_set_format).await?; - let invert_list_file = store.open_index_file(&posting_file_path(id)).await?; - let inverted_list = PostingListReader::try_new(invert_list_file, index_cache).await?; - // Defer the per-doc row_id/num_tokens read. Construction reads only - // the doc count (one footer read) and then drops the reader; the bulk - // load happens on first scoring use, re-opening the docs file on - // demand, and partitions that never score skip it entirely. Storing - // the store + path instead of an open reader keeps a cached partition - // from pinning a docs-file handle for its whole lifetime. - let docs_path = doc_file_path(id); - let num_docs = store.open_index_file(&docs_path).await?.num_rows(); - let docs = Arc::new(LazyDocSet::new( - store.clone(), - docs_path, - num_docs, - false, - frag_reuse_index, - )); - - Ok(Self { - id, - store, - tokens, - inverted_list: Arc::new(inverted_list), - docs, - token_set_format, - }) - } - - fn map(&self, token: &str) -> Option { - self.tokens.get(token) - } - - pub fn expand_fuzzy(&self, tokens: &Tokens, params: &FtsSearchParams) -> Result { - let mut new_tokens = Vec::with_capacity(min(tokens.len(), params.max_expansions)); - let mut new_positions = Vec::with_capacity(new_tokens.capacity()); - let mut seen = HashSet::new(); - for token_idx in 0..tokens.len() { - if new_tokens.len() >= params.max_expansions { - break; - } - let token = tokens.get_token(token_idx); - let position = tokens.position(token_idx); - let fuzziness = match params.fuzziness { - Some(fuzziness) => fuzziness, - None => MatchQuery::auto_fuzziness(token), - }; - let lev = fst::automaton::Levenshtein::new(token, fuzziness) - .map_err(|e| Error::index(format!("failed to construct the fuzzy query: {}", e)))?; - - let base_len = tokens.token_type().prefix_len(token) as u32; - if let TokenMap::Fst(ref map) = self.tokens.tokens { - let mut expanded = Vec::new(); - let remaining = params.max_expansions - new_tokens.len(); - match base_len + params.prefix_length { - 0 => take_fst_keys(map.search(lev), &mut expanded, remaining), - prefix_length => { - let prefix = &token[..min(prefix_length as usize, token.len())]; - let prefix = fst::automaton::Str::new(prefix).starts_with(); - take_fst_keys( - map.search(lev.intersection(prefix)), - &mut expanded, - remaining, - ) - } - } - for token in expanded { - if seen.insert((token.clone(), position)) { - new_tokens.push(token); - new_positions.push(position); - if new_tokens.len() >= params.max_expansions { - break; - } - } - } - } else { - return Err(Error::index( - "tokens is not fst, which is not expected".to_owned(), - )); - } - } - Ok(Tokens::with_positions( - new_tokens, - new_positions, - tokens.token_type().clone(), - )) - } - - fn union_plain_posting_lists(postings: Vec) -> Result { - let mut freqs_by_row_id = BTreeMap::new(); - for posting in postings { - for (row_id, freq, _) in posting.iter() { - let entry = freqs_by_row_id.entry(row_id).or_insert(0u32); - *entry = entry.checked_add(freq).ok_or_else(|| { - Error::index(format!("posting frequency overflow for row id {}", row_id)) - })?; - } - } - let mut row_ids = Vec::with_capacity(freqs_by_row_id.len()); - let mut frequencies = Vec::with_capacity(freqs_by_row_id.len()); - for (row_id, freq) in freqs_by_row_id { - row_ids.push(row_id); - frequencies.push(freq as f32); - } - Ok(PostingList::Plain(PlainPostingList::new( - ScalarBuffer::from(row_ids), - ScalarBuffer::from(frequencies), - None, - None, - ))) - } - - fn union_compressed_posting_lists( - postings: Vec, - docs: &DocSet, - ) -> Result { - let mut freqs_by_doc_id = BTreeMap::new(); - for posting in postings { - for (doc_id, freq, _) in posting.iter() { - let doc_id = u32::try_from(doc_id).map_err(|_| { - Error::index(format!( - "compressed posting doc id {} exceeds u32::MAX", - doc_id - )) - })?; - let entry = freqs_by_doc_id.entry(doc_id).or_insert(0u32); - *entry = entry.checked_add(freq).ok_or_else(|| { - Error::index(format!("posting frequency overflow for doc id {}", doc_id)) - })?; - } - } - if freqs_by_doc_id.is_empty() { - return Ok(PostingList::Plain(PlainPostingList::new( - ScalarBuffer::from(Vec::::new()), - ScalarBuffer::from(Vec::::new()), - None, - None, - ))); - } - - let mut builder = PostingListBuilder::new(false); - let mut doc_ids = Vec::with_capacity(freqs_by_doc_id.len()); - let mut frequencies = Vec::with_capacity(freqs_by_doc_id.len()); - for (doc_id, freq) in freqs_by_doc_id { - builder.add(doc_id, PositionRecorder::Count(freq)); - doc_ids.push(doc_id); - frequencies.push(freq); - } - let block_max_scores = docs.calculate_block_max_scores(doc_ids.iter(), frequencies.iter()); - let batch = builder.to_batch(block_max_scores)?; - let max_score = batch[MAX_SCORE_COL].as_primitive::().value(0); - let length = batch[LENGTH_COL].as_primitive::().value(0); - PostingList::from_batch(&batch, Some(max_score), Some(length)) - } - - fn union_posting_lists(postings: Vec, docs: &DocSet) -> Result { - let has_plain = postings - .iter() - .any(|posting| matches!(posting, PostingList::Plain(_))); - let has_compressed = postings - .iter() - .any(|posting| matches!(posting, PostingList::Compressed(_))); - match (has_plain, has_compressed) { - (true, true) => Err(Error::index( - "cannot union mixed plain and compressed posting lists".to_owned(), - )), - (true, false) => Self::union_plain_posting_lists(postings), - (false, true) => Self::union_compressed_posting_lists(postings, docs), - (false, false) => Ok(PostingList::Plain(PlainPostingList::new( - ScalarBuffer::from(Vec::::new()), - ScalarBuffer::from(Vec::::new()), - None, - None, - ))), - } - } - - // search the documents that contain the query - // return the doc info and the doc length - // ref: https://en.wikipedia.org/wiki/Okapi_BM25 - #[instrument(level = "debug", skip_all)] - async fn load_posting_lists( - &self, - tokens: &Tokens, - params: &FtsSearchParams, - operator: Operator, - metrics: &dyn MetricsCollector, - ) -> Result { - let is_fuzzy = matches!(params.fuzziness, Some(n) if n != 0); - let is_phrase_query = params.phrase_slop.is_some(); - let is_and_query = operator == Operator::And; - let required_positions = (is_and_query || is_phrase_query).then(|| { - (0..tokens.len()) - .map(|index| tokens.position(index)) - .collect::>() - }); - let tokens = match is_fuzzy { - true => self.expand_fuzzy(tokens, params)?, - false => tokens.clone(), - }; - let token_positions = (0..tokens.len()) - .map(|index| tokens.position(index)) - .collect::>(); - let mut token_ids = Vec::with_capacity(tokens.len()); - let mut matched_positions = required_positions.as_ref().map(|_| HashSet::new()); - for (index, token) in tokens.into_iter().enumerate() { - let token_id = self.map(&token); - if let Some(token_id) = token_id { - let position = token_positions[index]; - if let Some(matched_positions) = matched_positions.as_mut() { - matched_positions.insert(position); - } - token_ids.push((token_id, token, position)); - } else if is_phrase_query || is_and_query { - // if the token is not found, we can't do phrase or AND query - return Ok(LoadedPostings::empty()); - } - } - if token_ids.is_empty() { - return Ok(LoadedPostings::empty()); - } - if let Some(required_positions) = required_positions.as_ref() - && let Some(matched_positions) = matched_positions.as_ref() - && !required_positions.is_subset(matched_positions) - { - return Ok(LoadedPostings::empty()); - } - - let is_fuzzy_and_query = is_fuzzy && is_and_query && !is_phrase_query; - if !is_phrase_query { - if is_fuzzy_and_query { - token_ids.sort_unstable_by_key(|(token_id, _, position)| (*position, *token_id)); - token_ids.dedup_by(|lhs, rhs| lhs.0 == rhs.0 && lhs.2 == rhs.2); - } else { - token_ids.sort_unstable_by_key(|(token_id, _, _)| *token_id); - token_ids.dedup_by_key(|(token_id, _, _)| *token_id); - } - } - - let num_docs = self.docs.len(); - let loaded_postings = stream::iter(token_ids) - .map(|(token_id, token, position)| async move { - let posting = self - .inverted_list - .posting_list(token_id, is_phrase_query, metrics) - .await?; - - Result::Ok((token_id, token, position, posting)) - }) - .buffered(self.store.io_parallelism()) - .try_collect::>() - .await?; - - if (is_and_query || is_phrase_query) - && !is_fuzzy_and_query - && loaded_postings - .iter() - .any(|(_, _, _, posting)| posting.is_empty()) - { - return Ok(LoadedPostings::empty()); - } - - if !is_fuzzy_and_query { - return Ok(LoadedPostings { - postings: loaded_postings - .into_iter() - .map(|(token_id, token, position, posting)| { - let query_weight = idf(posting.len(), num_docs); - PostingIterator::with_query_weight( - token, - token_id, - position, - query_weight, - posting, - num_docs, - ) - }) - .collect(), - grouped_expansions: Vec::new(), - }); - } - - let needs_union = loaded_postings - .windows(2) - .any(|window| window[0].2 == window[1].2); - let docs_for_union = if needs_union { - Some(self.docs.ensure_num_tokens_loaded().await?) - } else { - None - }; - - // WAND's AND mode treats every iterator as required, so expansions from - // one original query position must be merged before scoring. - let mut grouped_postings = Vec::new(); - let mut grouped_expansions = Vec::new(); - let mut iter = loaded_postings.into_iter().peekable(); - while let Some((token_id, token, position, posting)) = iter.next() { - let mut group = vec![(token_id, token, posting)]; - while matches!(iter.peek(), Some((_, _, next_position, _)) if *next_position == position) - { - let (token_id, token, _, posting) = iter.next().expect("peeked item must exist"); - group.push((token_id, token, posting)); - } - - let (token_id, token, posting) = if group.len() == 1 { - group.pop().expect("single-item group must exist") - } else { - let token_id = group[0].0; - let token = group[0].1.clone(); - grouped_expansions.push(GroupedExpansionTerms { - position, - terms: group - .iter() - .map(|(_, token, posting)| ExpansionTermFreqs::new(token.clone(), posting)) - .collect(), - }); - let postings = group - .into_iter() - .map(|(_, _, posting)| posting) - .collect::>(); - let posting = Self::union_posting_lists( - postings, - docs_for_union - .as_deref() - .expect("union docs must be loaded for grouped fuzzy AND"), - )?; - (token_id, token, posting) - }; - if posting.is_empty() { - return Ok(LoadedPostings::empty()); - } - - let query_weight = idf(posting.len(), num_docs); - grouped_postings.push(PostingIterator::with_query_weight( - token, - token_id, - position, - query_weight, - posting, - num_docs, - )); - } - - Ok(LoadedPostings { - postings: grouped_postings, - grouped_expansions, - }) - } - - #[instrument(level = "debug", skip_all)] - // Deferred-DocSet adds the `docs` param (caller materializes it) on top of - // the cross-partition `shared_threshold`, tipping this hot-path search fn - // one over the limit. Bundling args isn't worth the churn here. - #[allow(clippy::too_many_arguments)] - pub fn bm25_search( - &self, - docs: &DocSet, - params: &FtsSearchParams, - operator: Operator, - mask: Arc, - postings: Vec, - metrics: &dyn MetricsCollector, - shared_threshold: Arc, - ) -> Result> { - if postings.is_empty() { - return Ok(Vec::new()); - } - - // Caller selects the DocSet shape via `LazyDocSet::docs_for_wand` - // and passes it in here; wand uses `docs.has_row_ids()` to - // handle the num_tokens-only case. - let scorer = IndexBM25Scorer::new(std::iter::once(self)); - let mut wand = Wand::new(operator, postings.into_iter(), docs, scorer) - .with_shared_threshold(shared_threshold); - let hits = wand.search(params, mask, metrics)?; - Ok(hits) - } - - pub async fn into_builder(self) -> Result { - let mut builder = InnerBuilder::new_with_posting_tail_codec( - self.id, - self.inverted_list.has_positions(), - self.token_set_format, - self.inverted_list.posting_tail_codec(), - ); - builder.tokens = self.tokens.into_mutable(); - // into_builder rewrites every doc, so materialize the full - // DocSet now and clone it out of the Arc. - let docs_arc = self.docs.ensure_loaded().await?; - builder.docs = (*docs_arc).clone(); - - builder - .posting_lists - .reserve_exact(self.inverted_list.len()); - for posting_list in self - .inverted_list - .read_all(self.inverted_list.has_positions()) - .await? - { - let posting_list = posting_list?; - builder - .posting_lists - .push(posting_list.into_builder(&builder.docs)); - } - Ok(builder) - } -} - -// at indexing, we use HashMap because we need it to be mutable, -// at searching, we use fst::Map because it's more efficient -#[derive(Debug, Clone)] -pub enum TokenMap { - HashMap(HashMap), - Fst(fst::Map>), -} - -impl Default for TokenMap { - fn default() -> Self { - Self::HashMap(HashMap::new()) - } -} - -impl DeepSizeOf for TokenMap { - fn deep_size_of_children(&self, ctx: &mut lance_core::deepsize::Context) -> usize { - match self { - Self::HashMap(map) => map.deep_size_of_children(ctx), - Self::Fst(map) => map.as_fst().size(), - } - } -} - -impl TokenMap { - pub fn len(&self) -> usize { - match self { - Self::HashMap(map) => map.len(), - Self::Fst(map) => map.len(), - } - } - - pub fn is_empty(&self) -> bool { - self.len() == 0 - } -} - -// TokenSet is a mapping from tokens to token ids -#[derive(Debug, Clone, Default, DeepSizeOf)] -pub struct TokenSet { - // token -> token_id - pub(crate) tokens: TokenMap, - pub(crate) next_id: u32, - total_length: usize, -} - -impl TokenSet { - pub fn into_mut(self) -> Self { - let tokens = match self.tokens { - TokenMap::HashMap(map) => map, - TokenMap::Fst(map) => { - let mut new_map = HashMap::with_capacity(map.len()); - let mut stream = map.into_stream(); - while let Some((token, token_id)) = stream.next() { - new_map.insert(String::from_utf8_lossy(token).into_owned(), token_id as u32); - } - - new_map - } - }; - - Self { - tokens: TokenMap::HashMap(tokens), - next_id: self.next_id, - total_length: self.total_length, - } - } - - pub fn len(&self) -> usize { - self.tokens.len() - } - - pub fn is_empty(&self) -> bool { - self.len() == 0 - } - - pub fn to_batch(self, format: TokenSetFormat) -> Result { - match format { - TokenSetFormat::Arrow => self.into_arrow_batch(), - TokenSetFormat::Fst => self.into_fst_batch(), - } - } - - fn into_arrow_batch(self) -> Result { - let mut token_builder = StringBuilder::with_capacity(self.tokens.len(), self.total_length); - let mut token_id_builder = UInt32Builder::with_capacity(self.tokens.len()); - - match self.tokens { - TokenMap::Fst(map) => { - let mut stream = map.stream(); - while let Some((token, token_id)) = stream.next() { - token_builder.append_value(String::from_utf8_lossy(token)); - token_id_builder.append_value(token_id as u32); - } - } - TokenMap::HashMap(map) => { - for (token, token_id) in map.into_iter().sorted_unstable() { - token_builder.append_value(token); - token_id_builder.append_value(token_id); - } - } - } - - let token_col = token_builder.finish(); - let token_id_col = token_id_builder.finish(); - - let schema = arrow_schema::Schema::new(vec![ - arrow_schema::Field::new(TOKEN_COL, DataType::Utf8, false), - arrow_schema::Field::new(TOKEN_ID_COL, DataType::UInt32, false), - ]); - - let batch = RecordBatch::try_new( - Arc::new(schema), - vec![ - Arc::new(token_col) as ArrayRef, - Arc::new(token_id_col) as ArrayRef, - ], - )?; - Ok(batch) - } - - fn into_fst_batch(mut self) -> Result { - let fst_map = match std::mem::take(&mut self.tokens) { - TokenMap::Fst(map) => map, - TokenMap::HashMap(map) => Self::build_fst_from_map(map)?, - }; - let bytes = fst_map.into_fst().into_inner(); - - let mut fst_builder = LargeBinaryBuilder::with_capacity(1, bytes.len()); - fst_builder.append_value(bytes); - let fst_col = fst_builder.finish(); - - let mut next_id_builder = UInt32Builder::with_capacity(1); - next_id_builder.append_value(self.next_id); - let next_id_col = next_id_builder.finish(); - - let mut total_length_builder = UInt64Builder::with_capacity(1); - total_length_builder.append_value(self.total_length as u64); - let total_length_col = total_length_builder.finish(); - - let schema = arrow_schema::Schema::new(vec![ - arrow_schema::Field::new(TOKEN_FST_BYTES_COL, DataType::LargeBinary, false), - arrow_schema::Field::new(TOKEN_NEXT_ID_COL, DataType::UInt32, false), - arrow_schema::Field::new(TOKEN_TOTAL_LENGTH_COL, DataType::UInt64, false), - ]); - - let batch = RecordBatch::try_new( - Arc::new(schema), - vec![ - Arc::new(fst_col) as ArrayRef, - Arc::new(next_id_col) as ArrayRef, - Arc::new(total_length_col) as ArrayRef, - ], - )?; - Ok(batch) - } - - fn build_fst_from_map(map: HashMap) -> Result>> { - let mut entries: Vec<_> = map.into_iter().collect(); - entries.sort_unstable_by(|(lhs, _), (rhs, _)| lhs.cmp(rhs)); - let mut builder = fst::MapBuilder::memory(); - for (token, token_id) in entries { - builder - .insert(&token, token_id as u64) - .map_err(|e| Error::index(format!("failed to insert token {}: {}", token, e)))?; - } - Ok(builder.into_map()) - } - - pub async fn load(reader: Arc, format: TokenSetFormat) -> Result { - match format { - TokenSetFormat::Arrow => Self::load_arrow(reader).await, - TokenSetFormat::Fst => Self::load_fst(reader).await, - } - } - - async fn load_arrow(reader: Arc) -> Result { - let batch = reader.read_range(0..reader.num_rows(), None).await?; - - let (tokens, next_id, total_length) = spawn_blocking(move || { - let mut next_id = 0; - let mut total_length = 0; - let mut tokens = fst::MapBuilder::memory(); - - let token_col = batch[TOKEN_COL].as_string::(); - let token_id_col = batch[TOKEN_ID_COL].as_primitive::(); - - for (token, &token_id) in token_col.iter().zip(token_id_col.values().iter()) { - let token = - token.ok_or(Error::index("found null token in token set".to_owned()))?; - next_id = next_id.max(token_id + 1); - total_length += token.len(); - tokens.insert(token, token_id as u64).map_err(|e| { - Error::index(format!("failed to insert token {}: {}", token, e)) - })?; - } - - Ok::<_, Error>((tokens.into_map(), next_id, total_length)) - }) - .await - .map_err(|err| Error::execution(format!("failed to spawn blocking task: {}", err)))??; - - Ok(Self { - tokens: TokenMap::Fst(tokens), - next_id, - total_length, - }) - } - - async fn load_fst(reader: Arc) -> Result { - let batch = reader.read_range(0..reader.num_rows(), None).await?; - if batch.num_rows() == 0 { - return Err(Error::index("token set batch is empty".to_owned())); - } - - let fst_col = batch[TOKEN_FST_BYTES_COL].as_binary::(); - let bytes = fst_col.value(0); - let map = fst::Map::new(bytes.to_vec()) - .map_err(|e| Error::index(format!("failed to load fst tokens: {}", e)))?; - - let total_length_col = - batch[TOKEN_TOTAL_LENGTH_COL].as_primitive::(); - - // Token ids are dense `[0, len)`, so `next_id` must equal the token count. Recompute - // it instead of trusting the persisted value, which writers before #7115 could leave - // stale. Mirrors `load_arrow`. - let next_id = map.len() as u32; - - let total_length = total_length_col - .values() - .first() - .copied() - .ok_or(Error::index( - "token total length column is empty".to_owned(), - ))?; - - Ok(Self { - tokens: TokenMap::Fst(map), - next_id, - total_length: usize::try_from(total_length).map_err(|_| { - Error::index(format!( - "token total length {} overflows usize", - total_length - )) - })?, - }) - } - - pub fn add(&mut self, token: String) -> u32 { - let next_id = self.next_id(); - let len = token.len(); - let token_id = match self.tokens { - TokenMap::HashMap(ref mut map) => *map.entry(token).or_insert(next_id), - _ => unreachable!("tokens must be HashMap while indexing"), - }; - - // add token if it doesn't exist - if token_id == next_id { - self.next_id += 1; - self.total_length += len; - } - - token_id - } - - pub(crate) fn get_or_add(&mut self, token: &str) -> u32 { - let next_id = self.next_id; - match self.tokens { - TokenMap::HashMap(ref mut map) => { - if let Some(&token_id) = map.get(token) { - return token_id; - } - - map.insert(token.to_owned(), next_id); - } - _ => unreachable!("tokens must be HashMap while indexing"), - } - - self.next_id += 1; - self.total_length += token.len(); - next_id - } - - pub(crate) fn into_mutable(self) -> Self { - let Self { - tokens, - next_id, - total_length, - } = self; - match tokens { - TokenMap::HashMap(_) => Self { - tokens, - next_id, - total_length, - }, - TokenMap::Fst(map) => { - let mut mutable = HashMap::new(); - let mut stream = map.stream(); - while let Some((token, token_id)) = stream.next() { - mutable.insert(String::from_utf8_lossy(token).into_owned(), token_id as u32); - } - Self { - tokens: TokenMap::HashMap(mutable), - next_id, - total_length, - } - } - } - } - - pub fn get(&self, token: &str) -> Option { - match self.tokens { - TokenMap::HashMap(ref map) => map.get(token).copied(), - TokenMap::Fst(ref map) => map.get(token).map(|id| id as u32), - } - } - - // the `removed_token_ids` must be sorted - pub fn remap(&mut self, removed_token_ids: &[u32]) { - if removed_token_ids.is_empty() { - return; - } - - let mut map = match std::mem::take(&mut self.tokens) { - TokenMap::HashMap(map) => map, - TokenMap::Fst(map) => { - let mut new_map = HashMap::with_capacity(map.len()); - let mut stream = map.into_stream(); - while let Some((token, token_id)) = stream.next() { - new_map.insert(String::from_utf8_lossy(token).into_owned(), token_id as u32); - } - - new_map - } - }; - - let mut retained_length = 0; - map.retain( - |token, token_id| match removed_token_ids.binary_search(token_id) { - Ok(_) => false, - Err(index) => { - *token_id -= index as u32; - retained_length += token.len(); - true - } - }, - ); - - self.tokens = TokenMap::HashMap(map); - - // The retain above compacts the surviving token ids into a dense `[0, len)` - // range, so `next_id` (handed to the next new token) must follow them down. - // `total_length` likewise must drop the removed tokens' bytes; it is persisted - // and feeds memory accounting, so a stale value drifts across remap/merge cycles. - self.next_id = self.tokens.len() as u32; - self.total_length = retained_length; - } - - pub fn next_id(&self) -> u32 { - self.next_id - } - - pub(crate) fn memory_size(&self) -> usize { - match &self.tokens { - TokenMap::HashMap(map) => { - self.total_length - + map.capacity() - * (std::mem::size_of::() - + std::mem::size_of::() - + std::mem::size_of::()) - } - TokenMap::Fst(map) => map.as_fst().size(), - } - } -} - -pub struct PostingListReader { - reader: Arc, - - /// Layout-specific metadata. V2 keeps its per-token max-score and - /// length columns lazy so opening a partition doesn't drag O(num_tokens) - /// bytes off cold storage when the caller only needs `df` for a few terms. - metadata: PostingMetadata, - - has_position: bool, - posting_tail_codec: PostingTailCodec, - positions_layout: PositionsLayout, - - /// First row of each posting-list cache group, decoded at open from the - /// global buffer named by [`POSTING_GROUP_OFFSETS_BUF_KEY`] (issue #7040). - /// `None` for indexes written before grouping; those use the per-token - /// cache path. Always present for grouped v2 indexes with `>0` rows. - group_starts: Option>, - - index_cache: WeakLanceCache, -} - -/// Per-token metadata (max_score, length) needed by the BM25 query and stats -/// paths. The legacy and v2 formats store this metadata in different -/// places, with very different cost profiles for cold-load: the variants -/// surface that asymmetry so callers can choose a per-token or bulk access -/// pattern. -enum PostingMetadata { - /// Legacy v1: offsets and max_scores are encoded in the file's schema - /// metadata, so they are already in memory by the time `try_new` returns. - LegacyV1 { - offsets: Vec, - max_scores: Option>, - }, - /// V2: per-token `max_score` and `length` live as columns in the - /// posting file. The bulk vectors are filled lazily by - /// `ensure_metadata_loaded`, and the stats path can also fetch a single - /// token via `posting_len_for_token` without forcing the bulk load. - V2 { - metadata: OnceCell, - }, -} - -#[derive(Debug, Clone)] -struct LoadedPostingMetadata { - max_scores: Vec, - lengths: Vec, -} - -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -enum PositionsLayout { - None, - LegacyPerDoc, - SharedStream(PositionStreamCodec), -} - -impl std::fmt::Debug for PostingListReader { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - let mut s = f.debug_struct("InvertedListReader"); - match &self.metadata { - PostingMetadata::LegacyV1 { - offsets, - max_scores, - } => { - s.field("layout", &"legacy_v1") - .field("offsets", offsets) - .field("max_scores", max_scores); - } - PostingMetadata::V2 { metadata } => { - s.field("layout", &"v2") - .field("metadata_loaded", &metadata.initialized()); - } - } - s.finish() - } -} - -impl DeepSizeOf for PostingListReader { - fn deep_size_of_children(&self, context: &mut lance_core::deepsize::Context) -> usize { - let metadata_size = match &self.metadata { - PostingMetadata::LegacyV1 { - offsets, - max_scores, - } => offsets.deep_size_of_children(context) + max_scores.deep_size_of_children(context), - PostingMetadata::V2 { metadata } => metadata - .get() - .map(|loaded| { - loaded.max_scores.deep_size_of_children(context) - + loaded.lengths.deep_size_of_children(context) - }) - .unwrap_or(0), - }; - metadata_size + self.group_starts.deep_size_of_children(context) - } -} - -impl PostingListReader { - pub(crate) async fn try_new( - reader: Arc, - index_cache: &LanceCache, - ) -> Result { - let positions_layout = if reader.schema().field(COMPRESSED_POSITION_COL).is_some() { - PositionsLayout::SharedStream(parse_shared_position_codec(&reader.schema().metadata)?) - } else if reader.schema().field(POSITION_COL).is_some() { - PositionsLayout::LegacyPerDoc - } else { - PositionsLayout::None - }; - let posting_tail_codec = parse_posting_tail_codec(&reader.schema().metadata)?; - let has_position = positions_layout != PositionsLayout::None; - let metadata = if reader.schema().field(POSTING_COL).is_none() { - let (offsets, max_scores) = Self::load_metadata(reader.schema())?; - PostingMetadata::LegacyV1 { - offsets, - max_scores, - } - } else { - PostingMetadata::V2 { - metadata: OnceCell::new(), - } - }; - - let group_starts = Self::load_group_starts(reader.as_ref()).await?; - - Ok(Self { - reader, - metadata, - has_position, - posting_tail_codec, - positions_layout, - group_starts, - index_cache: WeakLanceCache::from(index_cache), - }) - } - - /// Decode the posting-list cache-group boundaries from the global buffer - /// recorded in schema metadata, if present (issue #7040). Returns `None` - /// for indexes written before grouping was introduced. - async fn load_group_starts(reader: &dyn IndexReader) -> Result>> { - let Some(buf_id) = reader.schema().metadata.get(POSTING_GROUP_OFFSETS_BUF_KEY) else { - return Ok(None); - }; - let buf_id: u32 = buf_id.parse().map_err(|e| { - Error::index(format!( - "invalid {POSTING_GROUP_OFFSETS_BUF_KEY} metadata value {buf_id:?}: {e}" - )) - })?; - let bytes = reader.read_global_buffer(buf_id).await?; - let group_starts = decode_group_starts(&bytes)?; - Ok(Some(group_starts)) - } - - // for legacy format - // returns the offsets and max scores - fn load_metadata( - schema: &lance_core::datatypes::Schema, - ) -> Result<(Vec, Option>)> { - let offsets = schema - .metadata - .get("offsets") - .ok_or(Error::index("offsets not found in metadata".to_owned()))?; - let offsets = serde_json::from_str(offsets)?; - - let max_scores = schema - .metadata - .get("max_scores") - .map(|max_scores| serde_json::from_str(max_scores)) - .transpose()?; - Ok((offsets, max_scores)) - } - - // the number of posting lists - pub fn len(&self) -> usize { - match &self.metadata { - PostingMetadata::LegacyV1 { offsets, .. } => offsets.len(), - PostingMetadata::V2 { .. } => self.reader.num_rows(), - } - } - - pub fn is_empty(&self) -> bool { - self.len() == 0 - } - - pub(crate) fn has_positions(&self) -> bool { - self.has_position - } - - pub(crate) fn posting_tail_codec(&self) -> PostingTailCodec { - self.posting_tail_codec - } - - fn is_legacy_layout(&self) -> bool { - matches!(self.metadata, PostingMetadata::LegacyV1 { .. }) - } - - /// Sync access to `posting_len`. Requires v2 metadata to already be - /// loaded via [`ensure_metadata_loaded`]; the bm25 scoring path enforces - /// that contract before kicking off wand. The stats path uses - /// [`Self::posting_len_for_token`] instead, which avoids the bulk load. - pub(crate) fn posting_len(&self, token_id: u32) -> usize { - let token_id = token_id as usize; - match &self.metadata { - PostingMetadata::LegacyV1 { offsets, .. } => { - let next_offset = offsets - .get(token_id + 1) - .copied() - .unwrap_or(self.reader.num_rows()); - next_offset - offsets[token_id] - } - PostingMetadata::V2 { metadata } => { - let metadata = metadata - .get() - .expect("v2 posting metadata must be bulk-loaded before sync posting_len; call ensure_metadata_loaded first"); - metadata.lengths[token_id] as usize - } - } - } - - /// Async access to a single token's posting list length. For v2 - /// indexes this reads one row of posting metadata if the bulk metadata has - /// not been loaded yet, and never triggers the bulk load itself. The stats - /// path uses this so a single-term `df` lookup costs O(1) bytes rather - /// than O(num_unique_tokens). - pub(crate) async fn posting_len_for_token(&self, token_id: u32) -> Result { - match &self.metadata { - PostingMetadata::LegacyV1 { .. } => Ok(self.posting_len(token_id)), - PostingMetadata::V2 { metadata } => { - if let Some(metadata) = metadata.get() { - return Ok(metadata.lengths[token_id as usize] as usize); - } - let (_, length) = self.posting_metadata_for_token(token_id).await?; - length - .map(|len| len as usize) - .ok_or_else(|| Error::index("posting length metadata missing".to_string())) - } - } - } - - /// Async access to a single token's `(max_score, length)` pair. Mirrors - /// [`Self::posting_len_for_token`] but covers both columns the scoring - /// path needs, in one read. For v2 indexes that have not been - /// bulk-loaded this issues one `read_range(token..token+1, [MAX_SCORE, - /// LENGTH])`; for legacy v1 the values come from in-memory schema - /// metadata. - pub(crate) async fn posting_metadata_for_token( - &self, - token_id: u32, - ) -> Result<(Option, Option)> { - match &self.metadata { - PostingMetadata::LegacyV1 { max_scores, .. } => { - Ok((max_scores.as_ref().map(|m| m[token_id as usize]), None)) - } - PostingMetadata::V2 { metadata } => { - if let Some(loaded) = metadata.get() { - return Ok(( - Some(loaded.max_scores[token_id as usize]), - Some(loaded.lengths[token_id as usize]), - )); - } - let metadata = self - .index_cache - .get_or_insert_with_key(PostingMetadataKey { token_id }, || async move { - let token_id = token_id as usize; - let batch = self - .reader - .read_range(token_id..token_id + 1, Some(&[MAX_SCORE_COL, LENGTH_COL])) - .await?; - let max_score = batch[MAX_SCORE_COL].as_primitive::().value(0); - let length = batch[LENGTH_COL].as_primitive::().value(0); - Ok(PostingMetadataValue { max_score, length }) - }) - .await?; - Ok((Some(metadata.max_score), Some(metadata.length))) - } - } - } - - /// Force the v2 bulk metadata (`max_scores`, `lengths`) into - /// memory. Cheap to call repeatedly; no-op for legacy v1 indexes whose - /// metadata is already populated from schema metadata at `try_new` time. - pub(crate) async fn ensure_metadata_loaded(&self) -> Result<()> { - let PostingMetadata::V2 { metadata } = &self.metadata else { - return Ok(()); - }; - metadata - .get_or_try_init(|| async { - let batch = self - .reader - .read_range( - 0..self.reader.num_rows(), - Some(&[MAX_SCORE_COL, LENGTH_COL]), - ) - .await?; - let max_scores = batch[MAX_SCORE_COL] - .as_primitive::() - .values() - .to_vec(); - let lengths = batch[LENGTH_COL] - .as_primitive::() - .values() - .to_vec(); - Ok::(LoadedPostingMetadata { - max_scores, - lengths, - }) - }) - .await?; - Ok(()) - } - - pub(crate) async fn posting_batch( - &self, - token_id: u32, - with_position: bool, - ) -> Result { - if self.is_legacy_layout() { - self.posting_batch_legacy(token_id, with_position).await - } else { - let token_id = token_id as usize; - let columns = if with_position { - match self.positions_layout { - PositionsLayout::SharedStream(_) => { - vec![ - POSTING_COL, - COMPRESSED_POSITION_COL, - POSITION_BLOCK_OFFSET_COL, - ] - } - PositionsLayout::LegacyPerDoc => vec![POSTING_COL, POSITION_COL], - PositionsLayout::None => vec![POSTING_COL], - } - } else { - vec![POSTING_COL] - }; - let batch = self - .reader - .read_range(token_id..token_id + 1, Some(&columns)) - .await?; - Ok(batch) - } - } - - async fn posting_batch_legacy( - &self, - token_id: u32, - with_position: bool, - ) -> Result { - let mut columns = vec![ROW_ID, FREQUENCY_COL]; - if with_position { - columns.push(POSITION_COL); - } - - let length = self.posting_len(token_id); - let PostingMetadata::LegacyV1 { offsets, .. } = &self.metadata else { - unreachable!("posting_batch_legacy is only reachable on legacy v1 layout"); - }; - let token_id = token_id as usize; - let offset = offsets[token_id]; - let batch = self - .reader - .read_range(offset..offset + length, Some(&columns)) - .await?; - Ok(batch) - } - - #[instrument(level = "debug", skip(self, metrics))] - pub(crate) async fn posting_list( - &self, - token_id: u32, - is_phrase_query: bool, - metrics: &dyn MetricsCollector, - ) -> Result { - let mut posting = match self.group_range_for_token(token_id) { - // Grouped path (issue #7040): one cache entry covers rows - // [start, end), so neighbouring rare terms share a single read. - Some((start, end)) => { - let group = self - .index_cache - .get_or_insert_with_key(PostingListGroupKey { start, end }, || async move { - metrics.record_part_load(); - info!(target: TRACE_IO_EVENTS, r#type=IO_TYPE_LOAD_SCALAR_PART, index_type="inverted", part_id=start); - self.load_posting_list_group(start, end).await - }) - .await?; - let slot = (token_id - start) as usize; - group - .get(slot) - .ok_or_else(|| { - Error::index(format!( - "token {token_id} maps to slot {slot} outside posting group [{start}, {end})" - )) - })? - .clone() - } - // Fallback for indexes written before grouping: one cache entry - // per token. - None => self - .index_cache - .get_or_insert_with_key(PostingListKey { token_id }, || async move { - metrics.record_part_load(); - info!(target: TRACE_IO_EVENTS, r#type=IO_TYPE_LOAD_SCALAR_PART, index_type="inverted", part_id=token_id); - // Fetch the posting batch and this token's (max_score, - // length) in parallel; for cold v2 partitions this is one - // single-row metadata read plus one posting-row read, - // instead of pulling the full per-token metadata table. - let (batch, (max_score, length)) = futures::try_join!( - self.posting_batch(token_id, false), - self.posting_metadata_for_token(token_id), - )?; - self.posting_list_from_batch(&batch, max_score, length) - }) - .await? - .as_ref() - .clone(), - }; - - if is_phrase_query && !posting.has_position() { - // hit the cache and when the cache was populated, the positions column was not loaded - let positions = self.read_positions(token_id).await?; - posting.set_positions(positions); - } - - Ok(posting) - } - - /// Map a token id to its cache group's row range `[start, end)`, or `None` - /// when grouping is not available (pre-grouping indexes) so the caller - /// falls back to the per-token path. In v2 the token id is the row offset, - /// so the group range is also the physical row range. - fn group_range_for_token(&self, token_id: u32) -> Option<(u32, u32)> { - let starts = self.group_starts.as_ref()?; - // partition_point returns the count of group starts <= token_id, so the - // owning group begins at index k - 1 and the next start (if any) is its - // exclusive end. - let k = starts.partition_point(|&s| s <= token_id); - // k == 0 means token_id precedes the first group start, which cannot - // happen for a valid token in a grouped index (the first group starts - // at row 0); guard anyway and fall back to the per-token path. - if k == 0 { - return None; - } - let start = starts[k - 1]; - // The last group runs to the final posting list. `self.len()` is the - // authoritative posting-list count (offsets length for v1, row count for - // v2), and prewarm derives the same `end` from it — so warm- and - // cold-cache group keys are identical by construction, not by the - // incidental v2 `num_rows == token_count` equality. - let end = starts.get(k).copied().unwrap_or(self.len() as u32); - Some((start, end)) - } - - /// Read rows `[start, end)` of the posting file and decode them into a - /// [`PostingListGroup`] cache value (issue #7040). Positions are excluded; - /// phrase queries load them on demand via [`Self::read_positions`]. - async fn load_posting_list_group(&self, start: u32, end: u32) -> Result { - let batch = self - .reader - .read_range( - start as usize..end as usize, - Some(&[POSTING_COL, MAX_SCORE_COL, LENGTH_COL]), - ) - .await?; - let max_scores = batch[MAX_SCORE_COL].as_primitive::(); - let lengths = batch[LENGTH_COL].as_primitive::(); - let mut posting_lists = Vec::with_capacity(batch.num_rows()); - for i in 0..batch.num_rows() { - let row = batch.slice(i, 1); - let posting = self.posting_list_from_batch( - &row, - Some(max_scores.value(i)), - Some(lengths.value(i)), - )?; - posting_lists.push(posting); - } - Ok(PostingListGroup::new(posting_lists)) - } - - fn posting_list_from_batch_parts( - batch: &RecordBatch, - max_score: Option, - length: Option, - posting_tail_codec: PostingTailCodec, - positions_layout: PositionsLayout, - ) -> Result { - let posting_list = PostingList::from_batch_with_tail_codec_and_positions_layout( - batch, - max_score, - length, - posting_tail_codec, - positions_layout, - )?; - Ok(posting_list) - } - - pub(crate) fn posting_list_from_batch( - &self, - batch: &RecordBatch, - max_score: Option, - length: Option, - ) -> Result { - Self::posting_list_from_batch_parts( - batch, - max_score, - length, - self.posting_tail_codec, - self.positions_layout, - ) - } - - /// Build posting lists for one chunk's token range from `chunk_batch`, rebasing - /// global offsets to chunk-local rows. Returns `(global token_id, PostingList)` - /// pairs identical to the whole-file path, only bounded to one chunk. - fn build_prewarm_posting_lists_chunk( - chunk_batch: RecordBatch, - chunk: PrewarmChunk<'_>, - ctx: &PrewarmBuildCtx<'_>, - ) -> Result> { - let mut posting_lists = Vec::with_capacity(chunk.token_count); - for local in 0..chunk.token_count { - let global = chunk.tok_start + local; - let row_batch = if let Some(chunk_offsets) = chunk.offsets { - // Legacy v1: rebase global offsets to chunk row 0; the last token - // ends at `chunk.end_row` (no trailing sentinel in chunk_offsets). - let base = chunk_offsets[0]; - let start = chunk_offsets[local] - base; - let end = if local + 1 < chunk_offsets.len() { - chunk_offsets[local + 1] - base - } else { - chunk.end_row - base - }; - chunk_batch.slice(start, end - start) - } else { - // V2: one posting row per token; row `local` within the chunk. - chunk_batch.slice(local, 1) - }; - let row_batch = row_batch.shrink_to_fit()?; - let posting_list = Self::posting_list_from_batch_parts( - &row_batch, - ctx.max_scores.map(|scores| scores[global]), - ctx.lengths.map(|lengths| lengths[global]), - ctx.posting_tail_codec, - ctx.positions_layout, - )?; - posting_lists.push((global as u32, posting_list)); - } - - Ok(posting_lists) - } - - /// Read the posting rows for token ids `[tok_start, tok_end)` into one RecordBatch. - /// For v2 the token range is the row range; for v1 it's derived from the offsets. - async fn read_chunk_batch( - &self, - tok_start: usize, - tok_end: usize, - with_position: bool, - ) -> Result { - let columns = self.posting_columns(with_position); - let row_range = match &self.metadata { - PostingMetadata::LegacyV1 { offsets, .. } => { - let start = offsets[tok_start]; - let end = offsets - .get(tok_end) - .copied() - .unwrap_or_else(|| self.reader.num_rows()); - start..end - } - PostingMetadata::V2 { .. } => tok_start..tok_end, - }; - let batch = self.reader.read_range(row_range, Some(&columns)).await?; - Ok(batch) - } - - async fn prewarm_posting_lists( - &self, - with_position: bool, - chunk_concurrency: usize, - ) -> Result<()> { - self.prewarm_posting_lists_chunked(with_position, None, chunk_concurrency) - .await?; - Ok(()) - } - - /// Stream the partition's posting lists into the cache in bounded token-row chunks - /// (read -> build -> insert -> drop), so peak resident set is ~one chunk. Returns - /// the chunk count (tests assert it split). `chunk_tokens_override` is test-only. - async fn prewarm_posting_lists_chunked( - &self, - with_position: bool, - chunk_tokens_override: Option, - chunk_concurrency: usize, - ) -> Result { - if with_position && !self.has_positions() { - return Err(Error::invalid_input( - "cannot prewarm positions for an inverted index that was built without positions; recreate the index with with_position=true".to_owned(), - )); - } - - // Make sure max_scores/lengths are populated before we clone them into - // the blocking task; otherwise the v2 branch would unwrap empty - // OnceCells. - self.ensure_metadata_loaded().await?; - - let state = self.chunk_build_state(); - // With grouping the cache stores one entry per group, so a group's posting - // lists must all be resident at once: align chunk boundaries to whole - // groups. Without grouping, chunks are plain token ranges. - let group_starts = self.group_starts.clone(); - let token_count = self.len(); - let posting_data_size_bytes = self.posting_data_size_bytes(); - let chunk_tokens = chunk_tokens_override - .unwrap_or_else(|| prewarm_chunk_tokens(token_count, posting_data_size_bytes)) - .max(1); - let chunk_ranges = prewarm_chunk_ranges(group_starts.as_deref(), token_count, chunk_tokens); - let chunk_count = chunk_ranges.len(); - let chunk_concurrency = chunk_concurrency.max(1); - - let read_build_start = Instant::now(); - stream::iter(chunk_ranges) - .map(|(tok_start, tok_end)| { - let state = &state; - let group_starts = group_starts.as_deref(); - async move { - let posting_lists = self - .build_chunk_postings(tok_start, tok_end, with_position, state) - .await?; - self.publish_chunk_postings( - posting_lists, - group_starts, - tok_start, - tok_end, - token_count, - with_position, - ) - .await; - Result::Ok(()) - } - }) - .buffer_unordered(chunk_concurrency) - .try_collect::<()>() - .await?; - let read_build_elapsed = read_build_start.elapsed(); - - info!( - legacy_layout = self.is_legacy_layout(), - with_position, - token_count, - chunk_count, - chunk_tokens, - chunk_concurrency, - posting_data_size_bytes, - read_build_ms = read_build_elapsed.as_secs_f64() * 1000.0, - "posting list prewarm timing" - ); - - Ok(chunk_count) - } - - /// Loop-invariant inputs shared by every chunk build: the metadata vecs - /// (`Arc`d so chunks share them without re-cloning) plus codec/layout. - fn chunk_build_state(&self) -> ChunkBuildState { - let (offsets, max_scores, lengths) = match &self.metadata { - PostingMetadata::LegacyV1 { - offsets, - max_scores, - } => (Some(offsets.clone()), max_scores.clone(), None), - PostingMetadata::V2 { metadata } => ( - None, - metadata.get().map(|loaded| loaded.max_scores.clone()), - metadata.get().map(|loaded| loaded.lengths.clone()), - ), - }; - ChunkBuildState { - offsets: offsets.map(Arc::new), - max_scores: max_scores.map(Arc::new), - lengths: lengths.map(Arc::new), - posting_tail_codec: self.posting_tail_codec, - positions_layout: self.positions_layout, - } - } - - /// Read one token-row chunk and build its posting lists off the runtime thread. - /// The large batch is dropped inside the blocking task once built, bounding - /// resident memory to one chunk. - async fn build_chunk_postings( - &self, - tok_start: usize, - tok_end: usize, - with_position: bool, - state: &ChunkBuildState, - ) -> Result> { - let chunk_token_count = tok_end - tok_start; - let chunk_batch = self - .read_chunk_batch(tok_start, tok_end, with_position) - .await?; - - let (chunk_offsets, chunk_end_row) = match state.offsets.as_ref() { - Some(offsets) => { - let end_row = offsets - .get(tok_end) - .copied() - .unwrap_or_else(|| self.reader.num_rows()); - (Some(offsets[tok_start..tok_end].to_vec()), end_row) - } - // V2 doesn't use chunk_end_row (one row per token); pass tok_end. - None => (None, tok_end), - }; - let max_scores = state.max_scores.clone(); - let lengths = state.lengths.clone(); - let posting_tail_codec = state.posting_tail_codec; - let positions_layout = state.positions_layout; - let posting_lists = spawn_blocking(move || { - let ctx = PrewarmBuildCtx { - max_scores: max_scores.as_deref().map(|v| v.as_slice()), - lengths: lengths.as_deref().map(|v| v.as_slice()), - posting_tail_codec, - positions_layout, - }; - let chunk = PrewarmChunk { - tok_start, - token_count: chunk_token_count, - offsets: chunk_offsets.as_deref(), - end_row: chunk_end_row, - }; - Self::build_prewarm_posting_lists_chunk(chunk_batch, chunk, &ctx) - }) - .await - .map_err(|err| { - Error::internal(format!( - "Failed to build prewarm posting lists in blocking task: {err}" - )) - })??; - // The chunk yields its token range as contiguous ascending ids from - // `tok_start`; the group publish path relies on this to index the lists. - debug_assert_eq!(posting_lists.len(), chunk_token_count); - debug_assert!( - posting_lists - .iter() - .enumerate() - .all(|(i, (token_id, _))| *token_id as usize == tok_start + i) - ); - Ok(posting_lists) - } - - /// Strip positions into their own per-token cache entries (the posting cache - /// holds positions-free lists), then populate the same cache keys the read - /// path uses: grouped entries when grouping is active, per-token entries - /// otherwise. Called once per chunk; the chunk's lists drop on return. - async fn publish_chunk_postings( - &self, - posting_lists: Vec<(u32, PostingList)>, - group_starts: Option<&[u32]>, - tok_start: usize, - tok_end: usize, - token_count: usize, - with_position: bool, - ) { - match group_starts { - Some(starts) => { - let mut chunk_postings = Vec::with_capacity(posting_lists.len()); - for (token_id, mut posting_list) in posting_lists { - self.cache_positions(&mut posting_list, token_id, with_position) - .await; - chunk_postings.push(posting_list); - } - // Chunk is group-aligned, so every group starting in it also ends - // in it; `chunk_postings[i]` is token `tok_start + i`. The last - // group's `end` derives from `token_count`, matching the read path - // so both produce identical `PostingListGroupKey`s. - for group_idx in group_start_indices_for_chunk(starts, tok_start, tok_end) { - let (start, end) = group_range_for_start_index(starts, token_count, group_idx); - let start_usize = start as usize; - let lo = start_usize - tok_start; - let hi = end as usize - tok_start; - let group = PostingListGroup::new(chunk_postings[lo..hi].to_vec()); - self.index_cache - .insert_with_key(&PostingListGroupKey { start, end }, Arc::new(group)) - .await; - } - } - None => { - for (token_id, mut posting_list) in posting_lists { - self.cache_positions(&mut posting_list, token_id, with_position) - .await; - self.index_cache - .insert_with_key(&PostingListKey { token_id }, Arc::new(posting_list)) - .await; - } - } - } - } - - /// Move a posting list's positions (when present and requested) into the - /// dedicated per-token position cache, leaving the posting list positions-free. - async fn cache_positions( - &self, - posting_list: &mut PostingList, - token_id: u32, - with_position: bool, - ) { - if with_position && let Some(positions) = posting_list.take_positions() { - self.index_cache - .insert_with_key(&PositionKey { token_id }, Arc::new(Positions(positions))) - .await; - } - } - - /// Cheap `invert.lance` size estimate (file length from object metadata, no - /// data read), used only to size prewarm chunks. Falls back to a row-count - /// proxy when the reader can't surface the length (legacy v1). - pub(crate) fn posting_data_size_bytes(&self) -> u64 { - if let Some(size) = self.reader.file_size_bytes() { - return size; - } - // Fallback proxy for readers that don't cache their file length: just needs - // to be monotonic in partition size. - const ESTIMATED_BYTES_PER_ROW: u64 = 16; - (self.reader.num_rows() as u64).saturating_mul(ESTIMATED_BYTES_PER_ROW) - } - - pub(crate) async fn read_batch(&self, with_position: bool) -> Result { - let columns = self.posting_columns(with_position); - let batch = self - .reader - .read_range(0..self.reader.num_rows(), Some(&columns)) - .await?; - Ok(batch) - } - - pub(crate) async fn read_all( - &self, - with_position: bool, - ) -> Result> + '_> { - // read_all walks every posting list; the bulk metadata is paid for - // unconditionally, so just load it once up front and index into it - // synchronously below. - self.ensure_metadata_loaded().await?; - let batch = self.read_batch(with_position).await?; - Ok((0..self.len()).map(move |i| { - let token_id = i as u32; - let range = self.posting_list_range(token_id); - let batch = batch.slice(i, range.end - range.start); - let (max_score, length) = self.bulk_metadata_for_token(token_id); - self.posting_list_from_batch(&batch, max_score, length) - })) - } - - /// Sync lookup of `(max_score, length)` from the bulk-loaded metadata. - /// Only safe after [`Self::ensure_metadata_loaded`]; callers that hold - /// the OnceCell-loaded reference (e.g. read_all, prewarm) use this to - /// avoid the per-token IO path. - fn bulk_metadata_for_token(&self, token_id: u32) -> (Option, Option) { - match &self.metadata { - PostingMetadata::LegacyV1 { max_scores, .. } => { - (max_scores.as_ref().map(|m| m[token_id as usize]), None) - } - PostingMetadata::V2 { metadata } => { - let loaded = metadata.get().expect( - "v2 metadata must be bulk-loaded before bulk_metadata_for_token; call ensure_metadata_loaded first", - ); - ( - Some(loaded.max_scores[token_id as usize]), - Some(loaded.lengths[token_id as usize]), - ) - } - } - } - - async fn read_positions(&self, token_id: u32) -> Result { - let positions = self.index_cache.get_or_insert_with_key(PositionKey { token_id }, || async move { - let positions = match self.positions_layout { - PositionsLayout::None => { - return Err(Error::invalid_input( - "position is not found but required for phrase queries, try recreating the index with position".to_owned(), - )); - } - PositionsLayout::LegacyPerDoc => { - let batch = self - .reader - .read_range(self.posting_list_range(token_id), Some(&[POSITION_COL])) - .await - .map_err(|e| match e { - Error::Schema { .. } => Error::invalid_input("position is not found but required for phrase queries, try recreating the index with position".to_owned()), - e => e, - })?; - CompressedPositionStorage::LegacyPerDoc( - batch[POSITION_COL].as_list::().value(0).as_list::().clone(), - ) - } - PositionsLayout::SharedStream(codec) => { - let batch = self - .reader - .read_range( - self.posting_list_range(token_id), - Some(&[COMPRESSED_POSITION_COL, POSITION_BLOCK_OFFSET_COL]), - ) - .await - .map_err(|e| match e { - Error::Schema { .. } => Error::invalid_input("position is not found but required for phrase queries, try recreating the index with position".to_owned()), - e => e, - })?; - let bytes = bytes::Bytes::from( - batch[COMPRESSED_POSITION_COL] - .as_binary::() - .value(0) - .to_vec(), - ); - let block_offsets = batch[POSITION_BLOCK_OFFSET_COL] - .as_list::() - .value(0) - .as_primitive::() - .values() - .to_vec(); - CompressedPositionStorage::SharedStream(SharedPositionStream::new( - codec, - block_offsets, - bytes, - )) - } - }; - Result::Ok(Positions(positions)) - }).await?; - Ok(positions.0.clone()) - } - - fn posting_list_range(&self, token_id: u32) -> Range { - match &self.metadata { - PostingMetadata::LegacyV1 { offsets, .. } => { - let offset = offsets[token_id as usize]; - let posting_len = self.posting_len(token_id); - offset..offset + posting_len - } - PostingMetadata::V2 { .. } => { - let token_id = token_id as usize; - token_id..token_id + 1 - } - } - } - - fn posting_columns(&self, with_position: bool) -> Vec<&'static str> { - let mut base_columns = if self.is_legacy_layout() { - vec![ROW_ID, FREQUENCY_COL] - } else { - vec![POSTING_COL] - }; - if with_position { - match self.positions_layout { - PositionsLayout::None => {} - PositionsLayout::LegacyPerDoc => base_columns.push(POSITION_COL), - PositionsLayout::SharedStream(_) => { - base_columns.push(COMPRESSED_POSITION_COL); - base_columns.push(POSITION_BLOCK_OFFSET_COL); - } - } - } - base_columns - } -} - -/// Loop-invariant state for [`InvertedPartition::build_chunk_postings`]. The -/// metadata vecs are `Arc`d so each chunk's blocking build shares them cheaply. -struct ChunkBuildState { - offsets: Option>>, - max_scores: Option>>, - lengths: Option>>, - posting_tail_codec: PostingTailCodec, - positions_layout: PositionsLayout, -} - -/// Chunk-invariant inputs to [`InvertedPartition::build_prewarm_posting_lists_chunk`]: -/// the per-partition codec/layout and the (shared, whole-partition) metadata -/// slices indexed by global token id. These don't change across chunks. -struct PrewarmBuildCtx<'a> { - max_scores: Option<&'a [f32]>, - lengths: Option<&'a [u32]>, - posting_tail_codec: PostingTailCodec, - positions_layout: PositionsLayout, -} - -/// Per-chunk inputs to [`InvertedPartition::build_prewarm_posting_lists_chunk`]: -/// the token sub-range `[tok_start, tok_start + token_count)` and, for legacy -/// v1, the rebased offset slice plus the chunk's end row. -struct PrewarmChunk<'a> { - tok_start: usize, - token_count: usize, - /// Legacy v1 only: `offsets[tok_start..tok_start+token_count]` (no sentinel). - offsets: Option<&'a [usize]>, - /// Legacy v1 only: global row at which this chunk's posting rows end. - end_row: usize, -} - -/// New type just to allow Positions implement DeepSizeOf so it can be put -/// in the cache. -#[derive(Clone)] -pub struct Positions(pub(super) CompressedPositionStorage); - -/// Slice-aware cache-size charge for the Arrow array shapes stored in posting -/// caches. [`Array::get_buffer_memory_size`] reports the full capacity of shared -/// backing buffers; cached posting lists often reference only a small slice of a -/// group read. Count the referenced span for the known posting-list types and -/// fall back to Arrow's full-buffer size for anything else. -fn sliced_cache_bytes(array: &dyn Array) -> usize { - let validity_bytes = array - .nulls() - .map(|nulls| nulls.len().div_ceil(8)) - .unwrap_or(0); - match array.data_type() { - DataType::LargeBinary => { - let array = array.as_binary::(); - let data_bytes = if array.is_empty() { - 0 - } else { - let offsets = array.value_offsets(); - (offsets[array.len()] - offsets[0]) as usize - }; - data_bytes + (array.len() + 1) * std::mem::size_of::() + validity_bytes - } - DataType::List(_) => { - let array = array.as_list::(); - let (child_start, child_end) = if array.is_empty() { - (0, 0) - } else { - let offsets = array.value_offsets(); - (offsets[0] as usize, offsets[array.len()] as usize) - }; - let offset_bytes = (array.len() + 1) * std::mem::size_of::(); - let child = array.values().slice(child_start, child_end - child_start); - offset_bytes + validity_bytes + sliced_cache_bytes(child.as_ref()) - } - // Fixed-width primitives hold exactly `len * width` bytes regardless of - // buffer capacity, so this is already slice-aware. Any other type falls - // back to the full-buffer size. - other => match other.primitive_width() { - Some(width) => array.len() * width + validity_bytes, - None => array.get_buffer_memory_size(), - }, - } -} - -impl DeepSizeOf for Positions { - fn deep_size_of_children(&self, context: &mut lance_core::deepsize::Context) -> usize { - self.0.deep_size_of_children(context) - } -} - -// Cache key implementations for type-safe cache access -#[derive(Debug, Clone)] -pub struct PostingListKey { - pub token_id: u32, -} - -impl CacheKey for PostingListKey { - type ValueType = PostingList; - - fn key(&self) -> std::borrow::Cow<'_, str> { - format!("postings-{}", self.token_id).into() - } - - fn type_name() -> &'static str { - "PostingList" - } - - fn codec() -> Option { - Some(CacheCodec::from_impl::()) - } -} - -/// Cache key for a group of consecutive posting lists stored as a single -/// entry, covering rows `[start, end)` (issue #7040). The range, not a token -/// id, is the key so that a write-time config change that reshapes groups -/// simply misses old entries instead of serving a differently-shaped group. -#[derive(Debug, Clone)] -pub struct PostingListGroupKey { - pub start: u32, - pub end: u32, -} - -impl CacheKey for PostingListGroupKey { - type ValueType = PostingListGroup; - - fn key(&self) -> std::borrow::Cow<'_, str> { - format!("postings-{}-{}", self.start, self.end).into() - } - - fn type_name() -> &'static str { - "PostingListGroup" - } - - fn codec() -> Option { - Some(CacheCodec::from_impl::()) - } -} - -#[derive(Debug, Clone, DeepSizeOf)] -struct PostingMetadataValue { - max_score: f32, - length: u32, -} - -#[derive(Debug, Clone)] -struct PostingMetadataKey { - token_id: u32, -} - -impl CacheKey for PostingMetadataKey { - type ValueType = PostingMetadataValue; - - fn key(&self) -> std::borrow::Cow<'_, str> { - format!("posting-metadata-{}", self.token_id).into() - } - - fn type_name() -> &'static str { - "PostingMetadata" - } -} - -#[derive(Debug, Clone)] -pub struct PositionKey { - pub token_id: u32, -} - -impl CacheKey for PositionKey { - type ValueType = Positions; - - fn key(&self) -> std::borrow::Cow<'_, str> { - format!("positions-{}", self.token_id).into() - } - - fn type_name() -> &'static str { - "Position" - } - - fn codec() -> Option { - Some(CacheCodec::from_impl::()) - } -} - -#[derive(Debug, Clone, PartialEq)] -pub enum CompressedPositionStorage { - LegacyPerDoc(ListArray), - SharedStream(SharedPositionStream), -} - -impl DeepSizeOf for CompressedPositionStorage { - fn deep_size_of_children(&self, _context: &mut lance_core::deepsize::Context) -> usize { - match self { - Self::LegacyPerDoc(positions) => sliced_cache_bytes(positions), - Self::SharedStream(stream) => stream.size(), - } - } -} - -#[derive(Debug, Clone, PartialEq, Eq, Default)] -pub struct SharedPositionStream { - codec: PositionStreamCodec, - block_offsets: Arc<[u32]>, - // Stored with shared ownership so cache hits can clone position streams - // without copying either offsets or bytes. - bytes: bytes::Bytes, -} - -impl SharedPositionStream { - pub fn new(codec: PositionStreamCodec, block_offsets: Vec, bytes: bytes::Bytes) -> Self { - Self { - codec, - block_offsets: Arc::from(block_offsets.into_boxed_slice()), - bytes, - } - } - - pub fn codec(&self) -> PositionStreamCodec { - self.codec - } - - pub fn block_count(&self) -> usize { - self.block_offsets.len() - } - - pub fn block_range(&self, index: usize) -> Range { - let start = self.block_offsets[index] as usize; - let end = self - .block_offsets - .get(index + 1) - .map(|offset| *offset as usize) - .unwrap_or(self.bytes.len()); - start..end - } - - pub fn block(&self, index: usize) -> &[u8] { - let range = self.block_range(index); - &self.bytes[range] - } - - pub fn bytes(&self) -> &[u8] { - &self.bytes - } - - pub fn block_offsets(&self) -> &[u32] { - self.block_offsets.as_ref() - } - - pub fn size(&self) -> usize { - self.block_offsets.len() * std::mem::size_of::() + self.bytes.len() - } -} - -/// A group of consecutive posting lists held in a single cache entry, in row -/// order (issue #7040). `posting_lists[i]` corresponds to row `start + i`, -/// where `start` is the group's first row from [`PostingListGroupKey`]. -#[derive(Debug, Clone, DeepSizeOf)] -pub struct PostingListGroup { - pub(super) posting_lists: Vec, -} - -impl PostingListGroup { - pub(super) fn new(posting_lists: Vec) -> Self { - Self { posting_lists } - } - - /// Borrow the posting list at offset `slot` within the group (i.e. - /// `token_id - start`). - pub(super) fn get(&self, slot: usize) -> Option<&PostingList> { - self.posting_lists.get(slot) - } -} - -#[derive(Debug, Clone, DeepSizeOf)] -pub enum PostingList { - Plain(PlainPostingList), - Compressed(CompressedPostingList), -} - -impl PostingList { - pub fn from_batch( - batch: &RecordBatch, - max_score: Option, - length: Option, - ) -> Result { - let posting_tail_codec = parse_posting_tail_codec(batch.schema_ref().metadata())?; - Self::from_batch_with_tail_codec(batch, max_score, length, posting_tail_codec) - } - - pub fn from_batch_with_tail_codec( - batch: &RecordBatch, - max_score: Option, - length: Option, - posting_tail_codec: PostingTailCodec, - ) -> Result { - let positions_layout = if batch.column_by_name(COMPRESSED_POSITION_COL).is_some() { - PositionsLayout::SharedStream(parse_shared_position_codec( - batch.schema_ref().metadata(), - )?) - } else if batch.column_by_name(POSITION_COL).is_some() { - PositionsLayout::LegacyPerDoc - } else { - PositionsLayout::None - }; - Self::from_batch_with_tail_codec_and_positions_layout( - batch, - max_score, - length, - posting_tail_codec, - positions_layout, - ) - } - - fn from_batch_with_tail_codec_and_positions_layout( - batch: &RecordBatch, - max_score: Option, - length: Option, - posting_tail_codec: PostingTailCodec, - positions_layout: PositionsLayout, - ) -> Result { - match batch.column_by_name(POSTING_COL) { - Some(_) => { - debug_assert!(max_score.is_some() && length.is_some()); - let shared_position_codec = match positions_layout { - PositionsLayout::SharedStream(codec) => Some(codec), - _ => None, - }; - let posting = CompressedPostingList::from_batch( - batch, - max_score.unwrap(), - length.unwrap(), - posting_tail_codec, - shared_position_codec, - ); - Ok(Self::Compressed(posting)) - } - None => { - let posting = PlainPostingList::from_batch(batch, max_score); - Ok(Self::Plain(posting)) - } - } - } - - pub fn iter(&self) -> PostingListIterator<'_> { - PostingListIterator::new(self) - } - - pub fn has_position(&self) -> bool { - match self { - Self::Plain(posting) => posting.positions.is_some(), - Self::Compressed(posting) => posting.positions.is_some(), - } - } - - pub fn set_positions(&mut self, positions: CompressedPositionStorage) { - match self { - Self::Plain(posting) => match positions { - CompressedPositionStorage::LegacyPerDoc(positions) => { - posting.positions = Some(positions) - } - CompressedPositionStorage::SharedStream(_) => { - unreachable!("shared position stream is not supported for plain postings") - } - }, - Self::Compressed(posting) => { - posting.positions = Some(positions); - } - } - } - - pub fn take_positions(&mut self) -> Option { - match self { - Self::Plain(posting) => posting - .positions - .take() - .map(CompressedPositionStorage::LegacyPerDoc), - Self::Compressed(posting) => posting.positions.take(), - } - } - - pub fn max_score(&self) -> Option { - match self { - Self::Plain(posting) => posting.max_score, - Self::Compressed(posting) => Some(posting.max_score), - } - } - - pub fn len(&self) -> usize { - match self { - Self::Plain(posting) => posting.len(), - Self::Compressed(posting) => posting.length as usize, - } - } - - pub fn is_empty(&self) -> bool { - self.len() == 0 - } - - pub fn into_builder(self, docs: &DocSet) -> PostingListBuilder { - let posting_tail_codec = match &self { - Self::Plain(_) => PostingTailCodec::Fixed32, - Self::Compressed(posting) => posting.posting_tail_codec, - }; - let mut builder = PostingListBuilder::new_with_posting_tail_codec( - self.has_position(), - posting_tail_codec, - ); - match self { - // legacy format - Self::Plain(posting) => { - // convert the posting list to the new format: - // 1. map row ids to doc ids - // 2. sort the posting list by doc ids - struct Item { - doc_id: u32, - positions: PositionRecorder, - } - let doc_ids = docs - .row_ids - .iter() - .enumerate() - .map(|(doc_id, row_id)| (*row_id, doc_id as u32)) - .collect::>(); - let mut items = Vec::with_capacity(posting.len()); - for (row_id, freq, positions) in posting.iter() { - let freq = freq as u32; - let positions = match positions { - Some(positions) => { - PositionRecorder::Position(positions.collect::>().into()) - } - None => PositionRecorder::Count(freq), - }; - items.push(Item { - doc_id: doc_ids[&row_id], - positions, - }); - } - items.sort_unstable_by_key(|item| item.doc_id); - for item in items { - builder.add(item.doc_id, item.positions); - } - } - Self::Compressed(posting) => { - posting.iter().for_each(|(doc_id, freq, positions)| { - let positions = match positions { - Some(positions) => { - PositionRecorder::Position(positions.collect::>().into()) - } - None => PositionRecorder::Count(freq), - }; - builder.add(doc_id, positions); - }); - } - } - builder - } -} - -#[derive(Debug, PartialEq, Clone)] -pub struct PlainPostingList { - pub row_ids: ScalarBuffer, - pub frequencies: ScalarBuffer, - pub max_score: Option, - pub positions: Option, // List of Int32 -} - -impl DeepSizeOf for PlainPostingList { - fn deep_size_of_children(&self, _context: &mut lance_core::deepsize::Context) -> usize { - self.row_ids.len() * std::mem::size_of::() - + self.frequencies.len() * std::mem::size_of::() - + self - .positions - .as_ref() - .map(|positions| sliced_cache_bytes(positions)) - .unwrap_or(0) - } -} - -impl PlainPostingList { - pub fn new( - row_ids: ScalarBuffer, - frequencies: ScalarBuffer, - max_score: Option, - positions: Option, - ) -> Self { - Self { - row_ids, - frequencies, - max_score, - positions, - } - } - - pub fn from_batch(batch: &RecordBatch, max_score: Option) -> Self { - let row_ids = batch[ROW_ID].as_primitive::().values().clone(); - let frequencies = batch[FREQUENCY_COL] - .as_primitive::() - .values() - .clone(); - let positions = batch - .column_by_name(POSITION_COL) - .map(|col| col.as_list::().clone()); - - Self::new(row_ids, frequencies, max_score, positions) - } - - pub fn len(&self) -> usize { - self.row_ids.len() - } - - pub fn is_empty(&self) -> bool { - self.len() == 0 - } - - pub fn iter(&self) -> PlainPostingListIterator<'_> { - Box::new( - self.row_ids - .iter() - .zip(self.frequencies.iter()) - .enumerate() - .map(|(idx, (doc_id, freq))| { - ( - *doc_id, - *freq, - self.positions.as_ref().map(|p| { - let start = p.value_offsets()[idx] as usize; - let end = p.value_offsets()[idx + 1] as usize; - Box::new( - p.values().as_primitive::().values()[start..end] - .iter() - .map(|pos| *pos as u32), - ) as _ - }), - ) - }), - ) - } - - #[inline] - pub fn doc(&self, i: usize) -> LocatedDocInfo { - LocatedDocInfo::new(self.row_ids[i], self.frequencies[i]) - } - - pub fn positions(&self, index: usize) -> Option> { - self.positions - .as_ref() - .map(|positions| positions.value(index)) - } - - pub fn max_score(&self) -> Option { - self.max_score - } - - pub fn row_id(&self, i: usize) -> u64 { - self.row_ids[i] - } -} - -#[derive(Debug, PartialEq, Clone)] -pub struct CompressedPostingList { - pub max_score: f32, - pub length: u32, - // each binary is a block of compressed data - // that contains `BLOCK_SIZE` doc ids and then `BLOCK_SIZE` frequencies - pub blocks: LargeBinaryArray, - pub posting_tail_codec: PostingTailCodec, - pub positions: Option, -} - -impl DeepSizeOf for CompressedPostingList { - fn deep_size_of_children(&self, context: &mut lance_core::deepsize::Context) -> usize { - sliced_cache_bytes(&self.blocks) - + self - .positions - .as_ref() - .map(|positions| positions.deep_size_of_children(context)) - .unwrap_or(0) - } -} - -impl CompressedPostingList { - pub fn new( - blocks: LargeBinaryArray, - max_score: f32, - length: u32, - posting_tail_codec: PostingTailCodec, - positions: Option, - ) -> Self { - Self { - max_score, - length, - blocks, - posting_tail_codec, - positions, - } - } - - pub fn from_batch( - batch: &RecordBatch, - max_score: f32, - length: u32, - posting_tail_codec: PostingTailCodec, - shared_position_codec: Option, - ) -> Self { - debug_assert_eq!(batch.num_rows(), 1); - let blocks = batch[POSTING_COL] - .as_list::() - .value(0) - .as_binary::() - .clone(); - let positions = if let Some(col) = batch.column_by_name(COMPRESSED_POSITION_COL) { - let bytes = bytes::Bytes::from(col.as_binary::().value(0).to_vec()); - let block_offsets = batch[POSITION_BLOCK_OFFSET_COL] - .as_list::() - .value(0) - .as_primitive::() - .values() - .to_vec(); - let codec = shared_position_codec.unwrap_or_else(|| { - parse_shared_position_codec(batch.schema_ref().metadata()) - .expect("shared position stream codec metadata should be valid") - }); - Some(CompressedPositionStorage::SharedStream( - SharedPositionStream::new(codec, block_offsets, bytes), - )) - } else { - batch.column_by_name(POSITION_COL).map(|col| { - CompressedPositionStorage::LegacyPerDoc( - col.as_list::().value(0).as_list::().clone(), - ) - }) - }; - - Self { - max_score, - length, - blocks, - posting_tail_codec, - positions, - } - } - - pub fn iter(&self) -> CompressedPostingListIterator { - CompressedPostingListIterator::new( - self.length as usize, - self.blocks.clone(), - self.posting_tail_codec, - self.positions.clone(), - ) - } - - pub fn block_max_score(&self, block_idx: usize) -> f32 { - let block = self.blocks.value(block_idx); - block[0..4].try_into().map(f32::from_le_bytes).unwrap() - } - - pub fn block_least_doc_id(&self, block_idx: usize) -> u32 { - let block = self.blocks.value(block_idx); - let remainder = self.length as usize % BLOCK_SIZE; - let is_remainder_block = remainder > 0 && block_idx + 1 == self.blocks.len(); - if is_remainder_block { - super::encoding::read_posting_tail_first_doc(block, self.posting_tail_codec) - } else { - block[4..8].try_into().map(u32::from_le_bytes).unwrap() - } - } -} - -#[derive(Debug, Clone, PartialEq, Eq, Default)] -struct EncodedBlocks { - offsets: Vec, - bytes: Vec, -} - -impl EncodedBlocks { - fn len(&self) -> usize { - self.offsets.len() - } - - fn size(&self) -> usize { - self.offsets.capacity() * std::mem::size_of::() + self.bytes.capacity() - } - - fn push_full_block(&mut self, doc_ids: &[u32], frequencies: &[u32]) -> Result { - let start = self.bytes.len(); - self.offsets.push(start as u32); - super::encoding::encode_full_posting_block_into(doc_ids, frequencies, &mut self.bytes)?; - Ok(self.bytes.len() - start) - } - - fn block(&self, index: usize) -> &[u8] { - let (start, end) = self.block_range(index); - &self.bytes[start..end] - } - - fn block_range(&self, index: usize) -> (usize, usize) { - let start = self.offsets[index] as usize; - let end = self - .offsets - .get(index + 1) - .map(|offset| *offset as usize) - .unwrap_or(self.bytes.len()); - (start, end) - } - - fn set_block_score(&mut self, index: usize, score: f32) { - let (start, _) = self.block_range(index); - self.bytes[start..start + 4].copy_from_slice(&score.to_le_bytes()); - } - - fn append_remainder_block_with_codec( - &mut self, - doc_ids: &[u32], - frequencies: &[u32], - codec: PostingTailCodec, - ) -> Result<()> { - self.offsets.push(self.bytes.len() as u32); - super::encoding::encode_remainder_posting_block_into( - doc_ids, - frequencies, - codec, - &mut self.bytes, - ) - } - - fn into_array(mut self) -> LargeBinaryArray { - let mut offsets = Vec::with_capacity(self.offsets.len() + 1); - offsets.extend(self.offsets.into_iter().map(i64::from)); - offsets.push(self.bytes.len() as i64); - LargeBinaryArray::new( - OffsetBuffer::new(ScalarBuffer::from(offsets)), - Buffer::from_vec(std::mem::take(&mut self.bytes)), - None, - ) - } - - fn iter(&self) -> impl Iterator { - (0..self.len()).map(|index| self.block(index)) - } -} - -#[derive(Debug, Clone, PartialEq, Eq, Default)] -struct EncodedPositionBlocks { - offsets: Vec, - bytes: Vec, -} - -impl EncodedPositionBlocks { - fn size(&self) -> usize { - self.offsets.capacity() * std::mem::size_of::() + self.bytes.capacity() - } - - fn block(&self, index: usize) -> &[u8] { - let start = self.offsets[index] as usize; - let end = self - .offsets - .get(index + 1) - .map(|offset| *offset as usize) - .unwrap_or(self.bytes.len()); - &self.bytes[start..end] - } - - fn push_encoded_block(&mut self, block: &[u8]) -> usize { - let start = self.bytes.len(); - self.offsets.push(start as u32); - self.bytes.extend_from_slice(block); - self.bytes.len() - start - } - - fn into_stream(self) -> SharedPositionStream { - SharedPositionStream::new( - PositionStreamCodec::PackedDelta, - self.offsets, - bytes::Bytes::from(self.bytes), - ) - } -} - -#[derive(Debug)] -pub struct PostingListBuilder { - with_positions: bool, - posting_tail_codec: PostingTailCodec, - encoded_blocks: Option>, - encoded_position_blocks: Option>, - tail_entries: Vec, - tail_positions: PositionBlockBuilder, - open_doc_id: Option, - open_doc_frequency: u32, - open_doc_last_position: Option, - memory_size_bytes: u32, - len: u32, -} - -pub(super) struct PostingListBatchBuilder { - schema: SchemaRef, - postings: ListBuilder, - max_scores: Float32Builder, - lengths: UInt32Builder, - positions: BatchPositionsBuilder, - len: usize, - /// Tracks posting-list cache-group boundaries in row order across all - /// batches this builder produces (issue #7040). Outlives `finish`, which - /// only resets the per-batch column builders. - group_accumulator: PostingGroupAccumulator, -} - -enum BatchPositionsBuilder { - None, - Legacy(ListBuilder>), - Shared { - bytes: LargeBinaryBuilder, - block_offsets: ListBuilder, - }, -} - -struct PostingListParts<'a> { - with_positions: bool, - posting_tail_codec: PostingTailCodec, - length: usize, - encoded_blocks: EncodedBlocks, - encoded_position_blocks: EncodedPositionBlocks, - tail_entries: &'a [RawDocInfo], - tail_position_block: Option>, -} - -impl PostingListBatchBuilder { - pub fn new( - schema: SchemaRef, - with_positions: bool, - format_version: InvertedListFormatVersion, - capacity: usize, - group_config: PostingGroupConfig, - ) -> Self { - let positions = if !with_positions { - BatchPositionsBuilder::None - } else if format_version.uses_shared_position_stream() { - BatchPositionsBuilder::Shared { - bytes: LargeBinaryBuilder::with_capacity(capacity, 0), - block_offsets: ListBuilder::with_capacity(UInt32Builder::new(), capacity), - } - } else { - BatchPositionsBuilder::Legacy(ListBuilder::with_capacity( - ListBuilder::new(LargeBinaryBuilder::new()), - capacity, - )) - }; - Self { - schema, - postings: ListBuilder::with_capacity(LargeBinaryBuilder::new(), capacity), - max_scores: Float32Builder::with_capacity(capacity), - lengths: UInt32Builder::with_capacity(capacity), - positions, - len: 0, - group_accumulator: PostingGroupAccumulator::new(group_config), - } - } - - pub fn len(&self) -> usize { - self.len - } - - pub fn is_empty(&self) -> bool { - self.len == 0 - } - - fn append( - &mut self, - compressed: LargeBinaryArray, - max_score: f32, - length: u32, - positions: Option<&CompressedPositionStorage>, - ) -> Result<()> { - let posting_bytes = compressed.value_data().len(); - { - let values = self.postings.values(); - for index in 0..compressed.len() { - values.append_value(compressed.value(index)); - } - } - self.postings.append(true); - self.group_accumulator.push(posting_bytes); - self.max_scores.append_value(max_score); - self.lengths.append_value(length); - - match &mut self.positions { - BatchPositionsBuilder::None => {} - BatchPositionsBuilder::Shared { - bytes, - block_offsets, - } => { - let positions = positions.ok_or_else(|| { - Error::index(format!( - "positions builder missing position data for posting length {}", - length - )) - })?; - let CompressedPositionStorage::SharedStream(positions) = positions else { - return Err(Error::index( - "shared positions builder received legacy positions".to_owned(), - )); - }; - bytes.append_value(positions.bytes()); - let offsets_builder = block_offsets.values(); - for &offset in positions.block_offsets() { - offsets_builder.append_value(offset); - } - block_offsets.append(true); - } - BatchPositionsBuilder::Legacy(position_lists) => { - let positions = positions.ok_or_else(|| { - Error::index(format!( - "positions builder missing position data for posting length {}", - length - )) - })?; - let CompressedPositionStorage::LegacyPerDoc(positions) = positions else { - return Err(Error::index( - "legacy positions builder received shared position stream".to_owned(), - )); - }; - let docs_builder = position_lists.values(); - for doc_idx in 0..positions.len() { - let doc_positions = positions.value(doc_idx); - let compressed_positions = doc_positions.as_binary::(); - for block_idx in 0..compressed_positions.len() { - docs_builder - .values() - .append_value(compressed_positions.value(block_idx)); - } - docs_builder.append(true); - } - position_lists.append(true); - } - } - - self.len += 1; - Ok(()) - } - - pub fn finish(&mut self) -> Result { - let mut columns = vec![ - Arc::new(self.postings.finish()) as ArrayRef, - Arc::new(self.max_scores.finish()) as ArrayRef, - Arc::new(self.lengths.finish()) as ArrayRef, - ]; - match &mut self.positions { - BatchPositionsBuilder::None => {} - BatchPositionsBuilder::Legacy(position_lists) => { - columns.push(Arc::new(position_lists.finish()) as ArrayRef); - } - BatchPositionsBuilder::Shared { - bytes, - block_offsets, - } => { - columns.push(Arc::new(bytes.finish()) as ArrayRef); - columns.push(Arc::new(block_offsets.finish()) as ArrayRef); - } - } - self.len = 0; - RecordBatch::try_new(self.schema.clone(), columns).map_err(Error::from) - } - - /// Consume the builder and return the posting-list cache-group boundaries - /// accumulated across all batches (issue #7040). Each entry is the first - /// row of a group; the sequence is monotonically increasing. - pub fn into_group_starts(self) -> Vec { - self.group_accumulator.into_starts() - } -} - -impl PostingListBuilder { - pub fn size(&self) -> u64 { - self.memory_size_bytes as u64 - } - - pub fn has_positions(&self) -> bool { - self.with_positions - } - - pub fn new(with_position: bool) -> Self { - Self::new_with_posting_tail_codec( - with_position, - current_fts_format_version().posting_tail_codec(), - ) - } - - pub fn new_with_posting_tail_codec( - with_position: bool, - posting_tail_codec: PostingTailCodec, - ) -> Self { - Self { - with_positions: with_position, - posting_tail_codec, - encoded_blocks: None, - encoded_position_blocks: None, - tail_entries: Vec::new(), - tail_positions: PositionBlockBuilder::default(), - open_doc_id: None, - open_doc_frequency: 0, - open_doc_last_position: None, - len: 0, - memory_size_bytes: 0, - } - } - - pub fn len(&self) -> usize { - self.len as usize - } - - pub fn is_empty(&self) -> bool { - self.len == 0 - } - - pub fn iter(&self) -> std::vec::IntoIter<(u32, u32, Option>)> { - self.collect_entries().into_iter() - } - - pub fn for_each_entry( - &self, - mut visit: impl FnMut(u32, u32, Option>) -> std::result::Result<(), E>, - ) -> std::result::Result<(), E> { - let mut doc_ids = Vec::with_capacity(BLOCK_SIZE); - let mut frequencies = Vec::with_capacity(BLOCK_SIZE); - let mut decoded_positions = Vec::new(); - let mut position_block_index = 0usize; - - if let Some(encoded_blocks) = self.encoded_blocks.as_deref() { - for block in encoded_blocks.iter() { - doc_ids.clear(); - frequencies.clear(); - super::encoding::decode_full_posting_block(block, &mut doc_ids, &mut frequencies); - decoded_positions.clear(); - if self.with_positions { - let position_blocks = self - .encoded_position_blocks - .as_deref() - .expect("positions must exist for posting list"); - super::encoding::decode_position_stream_block( - position_blocks.block(position_block_index), - &frequencies, - PositionStreamCodec::PackedDelta, - &mut decoded_positions, - ) - .expect("position stream decoding should succeed"); - position_block_index += 1; - } - let mut offset = 0usize; - for (doc_id, frequency) in doc_ids.iter().copied().zip(frequencies.iter().copied()) - { - let positions = self.with_positions.then(|| { - let end = offset + frequency as usize; - let doc_positions = decoded_positions[offset..end].to_vec(); - offset = end; - doc_positions - }); - visit(doc_id, frequency, positions)?; - } - } - } - - let mut decoded_tail_positions = Vec::new(); - if self.with_positions && !self.tail_entries.is_empty() { - let tail_frequencies = self - .tail_entries - .iter() - .map(|entry| entry.frequency) - .collect::>(); - self.tail_positions - .decode_into(tail_frequencies.as_slice(), &mut decoded_tail_positions) - .expect("tail position stream decoding should succeed"); - } - let mut tail_offset = 0usize; - for entry in &self.tail_entries { - let positions = self.with_positions.then(|| { - let end = tail_offset + entry.frequency as usize; - let doc_positions = decoded_tail_positions[tail_offset..end].to_vec(); - tail_offset = end; - doc_positions - }); - visit(entry.doc_id, entry.frequency, positions)?; - } - - Ok(()) - } - - pub fn add(&mut self, doc_id: u32, term_positions: PositionRecorder) { - debug_assert!( - self.open_doc_id.is_none(), - "cannot add closed doc while a positions doc is still open" - ); - let tail_entries_capacity_before = self.tail_entries.capacity(); - self.tail_entries - .push(RawDocInfo::new(doc_id, term_positions.len())); - let tail_entries_capacity_after = self.tail_entries.capacity(); - if tail_entries_capacity_after > tail_entries_capacity_before { - self.add_memory_bytes( - (tail_entries_capacity_after - tail_entries_capacity_before) - * std::mem::size_of::(), - ); - } - if let PositionRecorder::Position(positions_in_doc) = term_positions { - debug_assert!(self.with_positions); - let old_size = self.tail_positions.size(); - self.tail_positions - .append_doc_positions(positions_in_doc.as_slice()) - .expect("position stream encoding should succeed"); - self.adjust_tail_positions_size(old_size); - } - self.len += 1; - - if self.tail_entries.len() == BLOCK_SIZE { - self.flush_tail_block() - .expect("posting list block compression should succeed"); - } - } - - pub fn add_occurrence(&mut self, doc_id: u32, position: u32) -> Result { - if !self.with_positions { - return Err(Error::index( - "cannot append streamed positions to a posting list without positions".to_owned(), - )); - } - - match self.open_doc_id { - Some(open_doc_id) if open_doc_id == doc_id => { - let old_size = self.tail_positions.size(); - self.tail_positions - .append_position(position, self.open_doc_last_position)?; - self.adjust_tail_positions_size(old_size); - self.open_doc_frequency += 1; - self.open_doc_last_position = Some(position); - Ok(false) - } - Some(open_doc_id) => Err(Error::index(format!( - "posting list received doc {} before finishing open doc {}", - doc_id, open_doc_id - ))), - None => { - let old_size = self.tail_positions.size(); - self.tail_positions.append_position(position, None)?; - self.adjust_tail_positions_size(old_size); - self.open_doc_id = Some(doc_id); - self.open_doc_frequency = 1; - self.open_doc_last_position = Some(position); - self.len += 1; - Ok(true) - } - } - } - - pub fn finish_open_doc(&mut self, doc_id: u32) -> Result<()> { - if !self.with_positions { - return Ok(()); - } - match self.open_doc_id { - Some(open_doc_id) if open_doc_id == doc_id => { - let tail_entries_capacity_before = self.tail_entries.capacity(); - self.tail_entries - .push(RawDocInfo::new(doc_id, self.open_doc_frequency)); - let tail_entries_capacity_after = self.tail_entries.capacity(); - if tail_entries_capacity_after > tail_entries_capacity_before { - self.add_memory_bytes( - (tail_entries_capacity_after - tail_entries_capacity_before) - * std::mem::size_of::(), - ); - } - self.open_doc_id = None; - self.open_doc_frequency = 0; - self.open_doc_last_position = None; - if self.tail_entries.len() == BLOCK_SIZE { - self.flush_tail_block()?; - } - Ok(()) - } - Some(open_doc_id) => Err(Error::index(format!( - "attempted to finish doc {} while doc {} is still open", - doc_id, open_doc_id - ))), - None => Ok(()), - } - } - - fn collect_entries(&self) -> Vec<(u32, u32, Option>)> { - let mut entries = Vec::with_capacity(self.len()); - self.for_each_entry(|doc_id, frequency, positions| { - entries.push((doc_id, frequency, positions)); - Ok::<(), ()>(()) - }) - .expect("collecting posting list entries should not fail"); - entries - } - - fn encoded_blocks_mut(&mut self) -> &mut EncodedBlocks { - if self.encoded_blocks.is_none() { - self.encoded_blocks = Some(Box::default()); - self.add_memory_bytes(std::mem::size_of::()); - } - self.encoded_blocks - .as_deref_mut() - .expect("encoded blocks must exist") - } - - fn encoded_position_blocks_mut(&mut self) -> &mut EncodedPositionBlocks { - if self.encoded_position_blocks.is_none() { - self.encoded_position_blocks = Some(Box::default()); - self.add_memory_bytes(std::mem::size_of::()); - } - self.encoded_position_blocks - .as_deref_mut() - .expect("encoded position blocks must exist") - } - - fn flush_tail_block(&mut self) -> Result<()> { - if self.tail_entries.is_empty() { - return Ok(()); - } - debug_assert!( - self.open_doc_id.is_none(), - "cannot flush a posting block while a document is still open" - ); - debug_assert_eq!(self.tail_entries.len(), BLOCK_SIZE); - let mut doc_ids = [0u32; BLOCK_SIZE]; - let mut frequencies = [0u32; BLOCK_SIZE]; - for (index, entry) in self.tail_entries.iter().enumerate() { - doc_ids[index] = entry.doc_id; - frequencies[index] = entry.frequency; - } - let encoded_blocks_size_before = self - .encoded_blocks - .as_ref() - .map(|encoded_blocks| encoded_blocks.size()) - .unwrap_or(0usize); - self.encoded_blocks_mut() - .push_full_block(&doc_ids, &frequencies)?; - let encoded_blocks_size_after = self - .encoded_blocks - .as_ref() - .map(|encoded_blocks| encoded_blocks.size()) - .unwrap_or(0usize); - if encoded_blocks_size_after > encoded_blocks_size_before { - self.add_memory_bytes(encoded_blocks_size_after - encoded_blocks_size_before); - } - if self.with_positions { - let encoded_positions_size_before = self - .encoded_position_blocks - .as_ref() - .map(|encoded| encoded.size()) - .unwrap_or(0usize); - let released_tail_positions_bytes = self.tail_positions.size(); - let tail_position_block = std::mem::take(&mut self.tail_positions).finish(); - self.encoded_position_blocks_mut() - .push_encoded_block(tail_position_block.as_slice()); - let encoded_positions_size_after = self - .encoded_position_blocks - .as_ref() - .map(|encoded| encoded.size()) - .unwrap_or(0usize); - if released_tail_positions_bytes > 0 { - self.subtract_memory_bytes(released_tail_positions_bytes); - } - if encoded_positions_size_after > encoded_positions_size_before { - self.add_memory_bytes(encoded_positions_size_after - encoded_positions_size_before); - } - } - self.tail_entries.clear(); - Ok(()) - } - - fn adjust_tail_positions_size(&mut self, old_size: usize) { - let new_size = self.tail_positions.size(); - if new_size > old_size { - self.add_memory_bytes(new_size - old_size); - } else if old_size > new_size { - self.subtract_memory_bytes(old_size - new_size); - } - } - - fn add_memory_bytes(&mut self, bytes: usize) { - self.memory_size_bytes = self - .memory_size_bytes - .checked_add( - u32::try_from(bytes).expect("posting list memory size delta overflowed u32"), - ) - .expect("posting list memory size overflowed u32"); - } - - fn subtract_memory_bytes(&mut self, bytes: usize) { - self.memory_size_bytes = self - .memory_size_bytes - .checked_sub( - u32::try_from(bytes).expect("posting list memory size delta overflowed u32"), - ) - .expect("posting list memory size underflowed u32"); - } - - fn build_position_columns( - positions: Option, - ) -> Result> { - let Some(positions) = positions else { - return Ok(Vec::new()); - }; - match positions { - CompressedPositionStorage::LegacyPerDoc(positions) => { - Ok(vec![Arc::new(ListArray::try_new( - Arc::new(Field::new("item", positions.data_type().clone(), true)), - OffsetBuffer::new(ScalarBuffer::from(vec![0_i32, positions.len() as i32])), - Arc::new(positions) as ArrayRef, - None, - )?) as ArrayRef]) - } - CompressedPositionStorage::SharedStream(positions) => { - let mut columns = Vec::with_capacity(2); - columns.push( - Arc::new(LargeBinaryArray::from(vec![Some(positions.bytes())])) as ArrayRef, - ); - - let mut offsets_builder = ListBuilder::new(UInt32Builder::new()); - for &offset in positions.block_offsets() { - offsets_builder.values().append_value(offset); - } - offsets_builder.append(true); - columns.push(Arc::new(offsets_builder.finish()) as ArrayRef); - Ok(columns) - } - } - } - - fn build_batch( - self, - compressed: LargeBinaryArray, - max_score: f32, - schema: SchemaRef, - positions: Option, - ) -> Result { - let length = self.len(); - let offsets = OffsetBuffer::new(ScalarBuffer::from(vec![0, compressed.len() as i32])); - let mut columns = vec![ - Arc::new(ListArray::try_new( - Arc::new(Field::new("item", datatypes::DataType::LargeBinary, true)), - offsets, - Arc::new(compressed), - None, - )?) as ArrayRef, - Arc::new(Float32Array::from_iter_values(std::iter::once(max_score))) as ArrayRef, - Arc::new(UInt32Array::from_iter_values(std::iter::once( - length as u32, - ))) as ArrayRef, - ]; - columns.extend(Self::build_position_columns(positions)?); - - let batch = RecordBatch::try_new(schema, columns)?; - Ok(batch) - } - - fn build_legacy_positions(&self) -> Result { - let mut positions_builder = ListBuilder::new(LargeBinaryBuilder::new()); - self.for_each_entry(|_doc_id, frequency, positions| { - let positions = positions.ok_or_else(|| { - Error::index(format!( - "legacy position writer missing positions for frequency {}", - frequency - )) - })?; - let compressed = super::encoding::compress_positions(positions.as_slice())?; - for block_idx in 0..compressed.len() { - positions_builder - .values() - .append_value(compressed.value(block_idx)); - } - positions_builder.append(true); - Ok::<(), Error>(()) - })?; - Ok(positions_builder.finish()) - } - - pub(super) fn append_to_batch_with_docs( - self, - docs: &DocSet, - batch_builder: &mut PostingListBatchBuilder, - format_version: InvertedListFormatVersion, - ) -> Result<()> { - let legacy_positions = - if self.with_positions && !format_version.uses_shared_position_stream() { - Some(self.build_legacy_positions()?) - } else { - None - }; - let Self { - with_positions, - posting_tail_codec, - encoded_blocks, - encoded_position_blocks, - tail_entries, - tail_positions, - open_doc_id, - open_doc_frequency, - open_doc_last_position, - len, - .. - } = self; - debug_assert!(open_doc_id.is_none()); - debug_assert_eq!(open_doc_frequency, 0); - debug_assert!(open_doc_last_position.is_none()); - let parts = PostingListParts { - with_positions, - posting_tail_codec, - length: len as usize, - encoded_blocks: encoded_blocks - .map(|encoded_blocks| *encoded_blocks) - .unwrap_or_default(), - encoded_position_blocks: encoded_position_blocks - .map(|encoded_positions| *encoded_positions) - .unwrap_or_default(), - tail_entries: tail_entries.as_slice(), - tail_position_block: with_positions.then(|| tail_positions.finish()), - }; - let (compressed, shared_positions, max_score) = - Self::build_compressed_with_scores_from_parts(parts, docs)?; - let positions = match legacy_positions { - Some(positions) => Some(CompressedPositionStorage::LegacyPerDoc(positions)), - None => shared_positions.map(CompressedPositionStorage::SharedStream), - }; - batch_builder.append(compressed, max_score, len, positions.as_ref()) - } - - fn extend_tail_components( - tail_entries: &[RawDocInfo], - doc_ids: &mut Vec, - frequencies: &mut Vec, - ) { - doc_ids.clear(); - frequencies.clear(); - doc_ids.extend(tail_entries.iter().map(|entry| entry.doc_id)); - frequencies.extend(tail_entries.iter().map(|entry| entry.frequency)); - } - - fn build_compressed_with_scores_from_parts( - parts: PostingListParts<'_>, - docs: &DocSet, - ) -> Result<(LargeBinaryArray, Option, f32)> { - let PostingListParts { - with_positions, - posting_tail_codec, - length, - mut encoded_blocks, - mut encoded_position_blocks, - tail_entries, - tail_position_block, - } = parts; - let avgdl = docs.average_length(); - let idf_scale = idf(length, docs.len()) * (K1 + 1.0); - let mut max_score = f32::MIN; - let mut doc_ids = Vec::with_capacity(BLOCK_SIZE); - let mut frequencies = Vec::with_capacity(BLOCK_SIZE); - - for index in 0..encoded_blocks.len() { - let block = encoded_blocks.block(index); - doc_ids.clear(); - frequencies.clear(); - super::encoding::decode_full_posting_block(block, &mut doc_ids, &mut frequencies); - let block_score = compute_block_score( - docs, - avgdl, - idf_scale, - doc_ids.iter().copied(), - frequencies.iter().copied(), - ); - max_score = max_score.max(block_score); - encoded_blocks.set_block_score(index, block_score); - } - - if !tail_entries.is_empty() { - Self::extend_tail_components(tail_entries, &mut doc_ids, &mut frequencies); - let block_score = compute_block_score( - docs, - avgdl, - idf_scale, - doc_ids.iter().copied(), - frequencies.iter().copied(), - ); - max_score = max_score.max(block_score); - encoded_blocks.append_remainder_block_with_codec( - doc_ids.as_slice(), - frequencies.as_slice(), - posting_tail_codec, - )?; - encoded_blocks.set_block_score(encoded_blocks.len() - 1, block_score); - if with_positions { - encoded_position_blocks.push_encoded_block( - tail_position_block - .as_deref() - .expect("tail position block must exist for postings with positions"), - ); - } - } - - Ok(( - encoded_blocks.into_array(), - with_positions.then(|| encoded_position_blocks.into_stream()), - max_score, - )) - } - - fn build_compressed_with_block_scores_from_parts( - with_positions: bool, - posting_tail_codec: PostingTailCodec, - mut encoded_blocks: EncodedBlocks, - mut encoded_position_blocks: EncodedPositionBlocks, - tail_entries: &[RawDocInfo], - tail_position_block: Option>, - mut block_max_scores: impl Iterator, - ) -> Result<(LargeBinaryArray, Option, f32)> { - let mut max_score = f32::MIN; - let mut doc_ids = Vec::with_capacity(BLOCK_SIZE); - let mut frequencies = Vec::with_capacity(BLOCK_SIZE); - - for index in 0..encoded_blocks.len() { - let block_score = block_max_scores - .next() - .ok_or_else(|| Error::index("missing block max score".to_owned()))?; - max_score = max_score.max(block_score); - encoded_blocks.set_block_score(index, block_score); - } - - if !tail_entries.is_empty() { - let block_score = block_max_scores - .next() - .ok_or_else(|| Error::index("missing tail block max score".to_owned()))?; - max_score = max_score.max(block_score); - Self::extend_tail_components(tail_entries, &mut doc_ids, &mut frequencies); - encoded_blocks.append_remainder_block_with_codec( - doc_ids.as_slice(), - frequencies.as_slice(), - posting_tail_codec, - )?; - encoded_blocks.set_block_score(encoded_blocks.len() - 1, block_score); - if with_positions { - encoded_position_blocks.push_encoded_block( - tail_position_block - .as_deref() - .expect("tail position block must exist for postings with positions"), - ); - } - } - - Ok(( - encoded_blocks.into_array(), - with_positions.then(|| encoded_position_blocks.into_stream()), - max_score, - )) - } - - pub fn to_batch(self, block_max_scores: Vec) -> Result { - let format_version = if self.posting_tail_codec == PostingTailCodec::Fixed32 { - InvertedListFormatVersion::V1 - } else { - InvertedListFormatVersion::V2 - }; - let schema = inverted_list_schema_for_version(self.has_positions(), format_version); - let legacy_positions = - if self.with_positions && !format_version.uses_shared_position_stream() { - Some(self.build_legacy_positions()?) - } else { - None - }; - let Self { - with_positions, - posting_tail_codec, - encoded_blocks, - encoded_position_blocks, - tail_entries, - tail_positions, - open_doc_id, - open_doc_frequency, - open_doc_last_position, - len, - .. - } = self; - debug_assert!(open_doc_id.is_none()); - debug_assert_eq!(open_doc_frequency, 0); - debug_assert!(open_doc_last_position.is_none()); - let (compressed, shared_positions, max_score) = - Self::build_compressed_with_block_scores_from_parts( - with_positions, - posting_tail_codec, - encoded_blocks - .map(|encoded_blocks| *encoded_blocks) - .unwrap_or_default(), - encoded_position_blocks - .map(|encoded_positions| *encoded_positions) - .unwrap_or_default(), - tail_entries.as_slice(), - with_positions.then(|| tail_positions.finish()), - block_max_scores.into_iter(), - )?; - let builder = Self { - with_positions, - posting_tail_codec, - encoded_blocks: None, - encoded_position_blocks: None, - tail_entries: Vec::new(), - tail_positions: PositionBlockBuilder::default(), - open_doc_id: None, - open_doc_frequency: 0, - open_doc_last_position: None, - memory_size_bytes: 0, - len, - }; - let positions = match legacy_positions { - Some(positions) => Some(CompressedPositionStorage::LegacyPerDoc(positions)), - None => shared_positions.map(CompressedPositionStorage::SharedStream), - }; - builder.build_batch(compressed, max_score, schema, positions) - } - - pub fn to_batch_with_docs(self, docs: &DocSet, schema: SchemaRef) -> Result { - let format_version = if schema.column_with_name(POSITION_COL).is_some() - && schema.column_with_name(COMPRESSED_POSITION_COL).is_none() - { - InvertedListFormatVersion::V1 - } else { - InvertedListFormatVersion::V2 - }; - let legacy_positions = - if self.with_positions && !format_version.uses_shared_position_stream() { - Some(self.build_legacy_positions()?) - } else { - None - }; - let Self { - with_positions, - posting_tail_codec, - encoded_blocks, - encoded_position_blocks, - tail_entries, - tail_positions, - open_doc_id, - open_doc_frequency, - open_doc_last_position, - len, - .. - } = self; - debug_assert!(open_doc_id.is_none()); - debug_assert_eq!(open_doc_frequency, 0); - debug_assert!(open_doc_last_position.is_none()); - let parts = PostingListParts { - with_positions, - posting_tail_codec, - length: len as usize, - encoded_blocks: encoded_blocks - .map(|encoded_blocks| *encoded_blocks) - .unwrap_or_default(), - encoded_position_blocks: encoded_position_blocks - .map(|encoded_positions| *encoded_positions) - .unwrap_or_default(), - tail_entries: tail_entries.as_slice(), - tail_position_block: with_positions.then(|| tail_positions.finish()), - }; - let (compressed, shared_positions, max_score) = - Self::build_compressed_with_scores_from_parts(parts, docs)?; - let builder = Self { - with_positions, - posting_tail_codec, - encoded_blocks: None, - encoded_position_blocks: None, - tail_entries: Vec::new(), - tail_positions: PositionBlockBuilder::default(), - open_doc_id: None, - open_doc_frequency: 0, - open_doc_last_position: None, - memory_size_bytes: 0, - len, - }; - let positions = match legacy_positions { - Some(positions) => Some(CompressedPositionStorage::LegacyPerDoc(positions)), - None => shared_positions.map(CompressedPositionStorage::SharedStream), - }; - builder.build_batch(compressed, max_score, schema, positions) - } - - pub fn remap(&mut self, removed: &[u32]) { - let mut cursor = 0; - let mut new_builder = - Self::new_with_posting_tail_codec(self.has_positions(), self.posting_tail_codec); - for (doc_id, freq, positions) in self.iter() { - while cursor < removed.len() && removed[cursor] < doc_id { - cursor += 1; - } - if cursor < removed.len() && removed[cursor] == doc_id { - continue; - } - let positions = match positions { - Some(positions) => PositionRecorder::Position(positions.into()), - None => PositionRecorder::Count(freq), - }; - new_builder.add(doc_id - cursor as u32, positions); - } - - *self = new_builder; - } -} - -fn compute_block_score( - docs: &DocSet, - avgdl: f32, - idf_scale: f32, - doc_ids: impl Iterator, - frequencies: impl Iterator, -) -> f32 { - let mut block_max_score = f32::MIN; - for (doc_id, freq) in doc_ids.zip(frequencies) { - let doc_norm = K1 * (1.0 - B + B * docs.num_tokens(doc_id) as f32 / avgdl); - let freq = freq as f32; - let score = freq / (freq + doc_norm); - block_max_score = block_max_score.max(score); - } - block_max_score * idf_scale -} - -#[derive(Debug, Clone, DeepSizeOf, Copy)] -pub enum DocInfo { - Located(LocatedDocInfo), - Raw(RawDocInfo), -} - -impl DocInfo { - pub fn doc_id(&self) -> u64 { - match self { - Self::Raw(info) => info.doc_id as u64, - Self::Located(info) => info.row_id, - } - } - - pub fn frequency(&self) -> u32 { - match self { - Self::Raw(info) => info.frequency, - Self::Located(info) => info.frequency as u32, - } - } -} - -impl Eq for DocInfo {} - -impl PartialEq for DocInfo { - fn eq(&self, other: &Self) -> bool { - self.doc_id() == other.doc_id() - } -} - -impl PartialOrd for DocInfo { - fn partial_cmp(&self, other: &Self) -> Option { - Some(self.cmp(other)) - } -} - -impl Ord for DocInfo { - fn cmp(&self, other: &Self) -> std::cmp::Ordering { - self.doc_id().cmp(&other.doc_id()) - } -} - -#[derive(Debug, Clone, Default, DeepSizeOf, Copy)] -pub struct LocatedDocInfo { - pub row_id: u64, - pub frequency: f32, -} - -impl LocatedDocInfo { - pub fn new(row_id: u64, frequency: f32) -> Self { - Self { row_id, frequency } - } -} - -impl Eq for LocatedDocInfo {} - -impl PartialEq for LocatedDocInfo { - fn eq(&self, other: &Self) -> bool { - self.row_id == other.row_id - } -} - -impl PartialOrd for LocatedDocInfo { - fn partial_cmp(&self, other: &Self) -> Option { - Some(self.cmp(other)) - } -} - -impl Ord for LocatedDocInfo { - fn cmp(&self, other: &Self) -> std::cmp::Ordering { - self.row_id.cmp(&other.row_id) - } -} - -#[derive(Debug, Clone, Default, DeepSizeOf, Copy)] -pub struct RawDocInfo { - pub doc_id: u32, - pub frequency: u32, -} - -impl RawDocInfo { - pub fn new(doc_id: u32, frequency: u32) -> Self { - Self { doc_id, frequency } - } -} - -impl Eq for RawDocInfo {} - -impl PartialEq for RawDocInfo { - fn eq(&self, other: &Self) -> bool { - self.doc_id == other.doc_id - } -} - -impl PartialOrd for RawDocInfo { - fn partial_cmp(&self, other: &Self) -> Option { - Some(self.cmp(other)) - } -} - -impl Ord for RawDocInfo { - fn cmp(&self, other: &Self) -> std::cmp::Ordering { - self.doc_id.cmp(&other.doc_id) - } -} - -// DocSet is a mapping from row ids to the number of tokens in the document -// It's used to sort the documents by the bm25 score -#[derive(Debug, Clone, Default, DeepSizeOf)] -pub struct DocSet { - row_ids: Vec, - num_tokens: Vec, - // (row_id, doc_id) pairs sorted by row_id - inv: Vec<(u64, u32)>, - - total_tokens: u64, -} - -impl DocSet { - #[inline] - pub fn len(&self) -> usize { - // Use num_tokens instead of row_ids so the deferred-row_ids - // scoring path (which constructs a DocSet via - // [`Self::from_num_tokens_only`]) still reports the right doc - // count. - self.num_tokens.len() - } - - pub fn is_empty(&self) -> bool { - self.len() == 0 - } - - /// True iff the per-doc `row_id` array is populated. The - /// deferred-row_id scoring path constructs DocSets with the array - /// left empty so wand can skip the load; callers that need to do - /// row_id lookups in the inner loop must check this and fall back - /// to async resolution otherwise. - #[inline] - pub fn has_row_ids(&self) -> bool { - !self.row_ids.is_empty() - } - - pub fn iter(&self) -> impl Iterator { - self.row_ids.iter().zip(self.num_tokens.iter()) - } - - pub fn row_id(&self, doc_id: u32) -> u64 { - self.row_ids[doc_id as usize] - } - - /// Resolve a `row_id` to every `doc_id` it owns. - /// - /// A scalar column maps each row to a single document, but a - /// `list` column indexes every element as its own document, so a - /// single `row_id` can own several `doc_id`s sharing that key in `inv`. - /// The prefilter path (`flat_search`) walks an allow-list of row_ids and - /// must evaluate *all* of a row's documents; resolving to one `doc_id` - /// silently drops matches at non-last list positions (lancedb#3352). - pub fn doc_ids(&self, row_id: u64) -> impl Iterator + '_ { - if self.inv.is_empty() { - // in legacy format, the row id is doc id (one document per row) - let found = self.row_ids.binary_search(&row_id).is_ok(); - Either::Left(found.then_some(row_id).into_iter()) - } else { - // `inv` is sorted by row_id, so the entries sharing this key form a - // contiguous run; yield the doc_id of each. - let lo = self.inv.partition_point(|entry| entry.0 < row_id); - let hi = self.inv.partition_point(|entry| entry.0 <= row_id); - Either::Right(self.inv[lo..hi].iter().map(|entry| entry.1 as u64)) - } - } - pub fn total_tokens_num(&self) -> u64 { - self.total_tokens - } - - #[inline] - pub fn average_length(&self) -> f32 { - self.total_tokens as f32 / self.len() as f32 - } - - pub fn calculate_block_max_scores<'a>( - &self, - doc_ids: impl Iterator, - freqs: impl Iterator, - ) -> Vec { - let avgdl = self.average_length(); - let length = doc_ids.size_hint().0; - let num_blocks = length.div_ceil(BLOCK_SIZE); - let mut block_max_scores = Vec::with_capacity(num_blocks); - let idf_scale = idf(length, self.len()) * (K1 + 1.0); - let mut max_score = f32::MIN; - for (i, (doc_id, freq)) in doc_ids.zip(freqs).enumerate() { - let doc_norm = K1 * (1.0 - B + B * self.num_tokens(*doc_id) as f32 / avgdl); - let freq = *freq as f32; - let score = freq / (freq + doc_norm); - if score > max_score { - max_score = score; - } - if (i + 1) % BLOCK_SIZE == 0 { - max_score *= idf_scale; - block_max_scores.push(max_score); - max_score = f32::MIN; - } - } - if !length.is_multiple_of(BLOCK_SIZE) { - max_score *= idf_scale; - block_max_scores.push(max_score); - } - block_max_scores - } - - pub fn to_batch(&self) -> Result { - let row_id_col = UInt64Array::from_iter_values(self.row_ids.iter().cloned()); - let num_tokens_col = UInt32Array::from_iter_values(self.num_tokens.iter().cloned()); - - let schema = arrow_schema::Schema::new(vec![ - arrow_schema::Field::new(ROW_ID, DataType::UInt64, false), - arrow_schema::Field::new(NUM_TOKEN_COL, DataType::UInt32, false), - ]); - - let batch = RecordBatch::try_new( - Arc::new(schema), - vec![ - Arc::new(row_id_col) as ArrayRef, - Arc::new(num_tokens_col) as ArrayRef, - ], - )?; - Ok(batch) - } - - pub async fn load( - reader: Arc, - is_legacy: bool, - frag_reuse_index: Option>, - ) -> Result { - let batch = reader.read_range(0..reader.num_rows(), None).await?; - let row_id_col = batch[ROW_ID].as_primitive::(); - let num_tokens_col = batch[NUM_TOKEN_COL].as_primitive::(); - Self::from_columns(row_id_col, num_tokens_col, is_legacy, frag_reuse_index) - } - - /// Build a `DocSet` carrying only the per-doc `num_tokens` array; - /// `row_ids` and `inv` are left empty. Used by the deferred-row_id - /// scoring path: wand checks `has_row_ids()` to skip `row_id` / - /// `num_tokens_by_row_id` calls, and the per-partition caller - /// resolves doc_id → row_id for the surviving top-K post-wand. - pub fn from_num_tokens_only(num_tokens_col: &arrow_array::UInt32Array) -> Self { - let num_tokens = num_tokens_col.values().to_vec(); - let total_tokens = num_tokens.iter().map(|&n| n as u64).sum(); - Self { - row_ids: Vec::new(), - num_tokens, - inv: Vec::new(), - total_tokens, - } - } - - /// Build a `DocSet` from already-loaded `row_id` and `num_tokens` - /// arrow columns. Lets callers that have one column already in hand - /// (e.g. `LazyDocSet` after `total_tokens_num` pre-fetched - /// `num_tokens`) skip re-reading that column. - pub fn from_columns( - row_id_col: &UInt64Array, - num_tokens_col: &arrow_array::UInt32Array, - is_legacy: bool, - frag_reuse_index: Option>, - ) -> Result { - // for legacy format, the row id is doc id; sorting keeps binary search viable - if is_legacy { - let (row_ids, num_tokens): (Vec<_>, Vec<_>) = row_id_col - .values() - .iter() - .filter_map(|id| { - if let Some(frag_reuse_index_ref) = frag_reuse_index.as_ref() { - frag_reuse_index_ref.remap_row_id(*id) - } else { - Some(*id) - } - }) - .zip(num_tokens_col.values().iter()) - .sorted_unstable_by_key(|x| x.0) - .unzip(); - - let total_tokens = num_tokens.iter().map(|&x| x as u64).sum(); - return Ok(Self { - row_ids, - num_tokens, - inv: Vec::new(), - total_tokens, - }); - } - - // If frag reuse happened, remap the row_ids through it. Crucially we - // must NOT drop the rows the reuse index deleted, because the posting - // lists reference doc_ids *positionally* (a doc_id is an index into - // these arrays, fixed at build time). Dropping deleted rows would - // renumber every later doc_id and desync the posting lists, so wand - // would index `num_tokens`/`row_ids` out of bounds or score the wrong - // doc. Instead we tombstone deleted rows in place: their slot survives - // (so doc_ids stay aligned with the posting lists) carrying - // `RowAddress::TOMBSTONE_ROW`, which wand skips, and they are left out - // of `inv` so a row_id lookup never resolves to a deleted doc. The - // heavyweight physical remap (`DocSet::remap`) is what actually - // renumbers and compacts; this load-time path only has to stay - // consistent until then. - if let Some(frag_reuse_index_ref) = frag_reuse_index.as_ref() { - let mut row_ids = Vec::with_capacity(row_id_col.len()); - let num_tokens = num_tokens_col.values().to_vec(); - let mut inv = Vec::with_capacity(row_id_col.len()); - for (doc_id, row_id) in row_id_col.values().iter().enumerate() { - match frag_reuse_index_ref.remap_row_id(*row_id) { - Some(new_row_id) => { - row_ids.push(new_row_id); - inv.push((new_row_id, doc_id as u32)); - } - None => { - // Deleted: keep the slot (doc_ids must not shift) but - // tombstone it and leave it out of `inv`. - row_ids.push(RowAddress::TOMBSTONE_ROW); - } - } - } - inv.sort_unstable_by_key(|entry| entry.0); - - let total_tokens = num_tokens.iter().map(|&x| x as u64).sum(); - return Ok(Self { - row_ids, - num_tokens, - inv, - total_tokens, - }); - } - - let row_ids = row_id_col.values().to_vec(); - let num_tokens = num_tokens_col.values().to_vec(); - let mut inv: Vec<(u64, u32)> = row_ids - .iter() - .enumerate() - .map(|(doc_id, row_id)| (*row_id, doc_id as u32)) - .collect(); - if !row_ids.is_sorted() { - inv.sort_unstable_by_key(|entry| entry.0); - } - let total_tokens = num_tokens.iter().map(|&x| x as u64).sum(); - Ok(Self { - row_ids, - num_tokens, - inv, - total_tokens, - }) - } - - // remap the row ids to the new row ids - // returns the removed doc ids - pub fn remap(&mut self, mapping: &RowAddrRemap) -> Vec { - let mut removed = Vec::new(); - let len = self.len(); - let row_ids = std::mem::replace(&mut self.row_ids, Vec::with_capacity(len)); - let num_tokens = std::mem::replace(&mut self.num_tokens, Vec::with_capacity(len)); - self.total_tokens = 0; - for (doc_id, (row_id, num_token)) in std::iter::zip(row_ids, num_tokens).enumerate() { - match mapping.get(row_id) { - Some(Some(new_row_id)) => { - self.row_ids.push(new_row_id); - self.num_tokens.push(num_token); - self.total_tokens += num_token as u64; - } - Some(None) => { - removed.push(doc_id as u32); - } - None => { - self.row_ids.push(row_id); - self.num_tokens.push(num_token); - self.total_tokens += num_token as u64; - } - } - } - removed - } - - #[inline] - pub fn num_tokens(&self, doc_id: u32) -> u32 { - self.num_tokens[doc_id as usize] - } - - // this can be used only if it's a legacy format, - // which store the sorted row ids so that we can use binary search - #[inline] - pub fn num_tokens_by_row_id(&self, row_id: u64) -> u32 { - self.row_ids - .binary_search(&row_id) - .map(|idx| self.num_tokens[idx]) - .unwrap_or(0) - } - - // append a document to the doc set - // returns the doc_id (the number of documents before appending) - pub fn append(&mut self, row_id: u64, num_tokens: u32) -> u32 { - self.row_ids.push(row_id); - self.num_tokens.push(num_tokens); - self.total_tokens += num_tokens as u64; - self.row_ids.len() as u32 - 1 - } - - pub(crate) fn memory_size(&self) -> usize { - self.row_ids.capacity() * std::mem::size_of::() - + self.num_tokens.capacity() * std::mem::size_of::() - + self.inv.capacity() * std::mem::size_of::<(u64, u32)>() - } -} - -pub fn flat_full_text_search( - batches: &[&RecordBatch], - doc_col: &str, - query: &str, - tokenizer: Option>, -) -> Result> { - if batches.is_empty() { - return Ok(vec![]); - } - - if is_phrase_query(query) { - return Err(Error::invalid_input( - "phrase query is not supported for flat full text search, try using FTS index", - )); - } - - match batches[0][doc_col].data_type() { - DataType::Utf8 => do_flat_full_text_search::(batches, doc_col, query, tokenizer), - DataType::LargeUtf8 => do_flat_full_text_search::(batches, doc_col, query, tokenizer), - data_type => Err(Error::invalid_input(format!( - "unsupported data type {} for inverted index", - data_type - ))), - } -} - -fn do_flat_full_text_search( - batches: &[&RecordBatch], - doc_col: &str, - query: &str, - tokenizer: Option>, -) -> Result> { - let mut results = Vec::new(); - let mut tokenizer = - tokenizer.unwrap_or_else(|| InvertedIndexParams::default().build().unwrap()); - let query_tokens = collect_query_tokens(query, &mut tokenizer); - - for batch in batches { - let row_id_array = batch[ROW_ID].as_primitive::(); - let doc_array = batch[doc_col].as_string::(); - for i in 0..row_id_array.len() { - let doc = doc_array.value(i); - if has_query_token(doc, &mut tokenizer, &query_tokens) { - results.push(row_id_array.value(i)); - // What is this assertion for? Why would doc contain query? Don't we reach - // here only if they share at least one token? Why is it not debug_assert? - assert!(doc.contains(query)); - } - } - } - - Ok(results) -} - -const FLAT_ROW_ID_COL_IDX: usize = 0; -const FLAT_ALL_TOKENS_COL_IDX: usize = 1; -const FLAT_QUERY_TOKEN_COUNTS_COL_IDX: usize = 2; - -/// If we accumulate this many bytes we warn the user they probably want to use an FTS index instead. -const BYTES_ACCUMULATED_WARNING_THRESHOLD: u64 = 1024 * 1024 * 1024; // 1GB - -/// Consumes a stream of record batches and produces token counts -/// -/// The resulting batch will have three columns: -/// - row_id: the row id of the document -/// - all_tokens: the total number of tokens in the document -/// - query_token_counts: a fixed size list of the count of each query token in the document -/// -/// This is an unbounded accumulation, however, for most queries, the per-row -/// growth will be fairly small. As a result we can process millions of tokens -/// with fairly modest memory usage. -/// -/// However, it is unwise to do a flat search across billions of rows. An FTS -/// index should be created instead. -async fn tokenize_and_count( - input: impl Stream> + Send, - tokenizer: Box, - query_tokens: Arc, - doc_col_idx: usize, - elapsed_compute: Option

    ( existing: &[P::ZoneStatistics], trainer: ZoneTrainer

    , stream: SendableRecordBatchStream, -) -> Result> +) -> Result<(Vec, RowAddrTreeMap)> where P: ZoneProcessor, P::ZoneStatistics: Clone, { let mut combined = existing.to_vec(); - let mut new_zones = trainer.train(stream).await?; + let (mut new_zones, null_rows) = trainer.train(stream).await?; combined.append(&mut new_zones); - Ok(combined) + Ok((combined, null_rows)) } #[cfg(test)] @@ -362,7 +373,7 @@ mod tests { let processor = MockProcessor::new(); let trainer = ZoneTrainer::new(processor, 4).unwrap(); - let stats = trainer.train(stream).await.unwrap(); + let (stats, _) = trainer.train(stream).await.unwrap(); // Three zones: offsets [0..=3], [4..=7], [8..=9] assert_eq!(stats.len(), 3); @@ -393,7 +404,7 @@ mod tests { let processor = MockProcessor::new(); let trainer = ZoneTrainer::new(processor, 10).unwrap(); - let stats = trainer.train(stream).await.unwrap(); + let (stats, _) = trainer.train(stream).await.unwrap(); // Two zones, one per fragment (capacity=10 is large enough) assert_eq!(stats.len(), 2); @@ -447,7 +458,7 @@ mod tests { let processor = MockProcessor::new(); let trainer = ZoneTrainer::new(processor, 10).unwrap(); - let stats = trainer.train(stream).await.unwrap(); + let (stats, _) = trainer.train(stream).await.unwrap(); // One zone containing the 3 valid rows (empty batches skipped) assert_eq!(stats.len(), 1); @@ -469,7 +480,7 @@ mod tests { let processor = MockProcessor::new(); let trainer = ZoneTrainer::new(processor, 1).unwrap(); - let stats = trainer.train(stream).await.unwrap(); + let (stats, _) = trainer.train(stream).await.unwrap(); // Three zones, one per row (capacity=1) assert_eq!(stats.len(), 3); @@ -494,7 +505,7 @@ mod tests { let processor = MockProcessor::new(); let trainer = ZoneTrainer::new(processor, 10000).unwrap(); - let stats = trainer.train(stream).await.unwrap(); + let (stats, _) = trainer.train(stream).await.unwrap(); // One zone containing all 100 rows (capacity is large enough) assert_eq!(stats.len(), 1); @@ -530,7 +541,7 @@ mod tests { let processor = MockProcessor::new(); let trainer = ZoneTrainer::new(processor, 4).unwrap(); - let stats = trainer.train(stream).await.unwrap(); + let (stats, _) = trainer.train(stream).await.unwrap(); // Two zones: first 4 rows, then remaining 2 rows assert_eq!(stats.len(), 2); @@ -561,7 +572,7 @@ mod tests { let processor = MockProcessor::new(); let trainer = ZoneTrainer::new(processor, 3).unwrap(); - let stats = trainer.train(stream).await.unwrap(); + let (stats, _) = trainer.train(stream).await.unwrap(); // Three zones: frag 0 full zone, frag 0 partial (flushed at boundary), frag 1 assert_eq!(stats.len(), 3); @@ -602,7 +613,7 @@ mod tests { let processor = MockProcessor::new(); let trainer = ZoneTrainer::new(processor, 4).unwrap(); - let stats = trainer.train(stream).await.unwrap(); + let (stats, _) = trainer.train(stream).await.unwrap(); // Should create 2 zones (capacity=4): // Zone 0: rows at offsets [0, 1, 5, 7] (4 rows) @@ -637,7 +648,7 @@ mod tests { let processor = MockProcessor::new(); let trainer = ZoneTrainer::new(processor, 10).unwrap(); - let stats = trainer.train(stream).await.unwrap(); + let (stats, _) = trainer.train(stream).await.unwrap(); // One zone with 3 rows, but offset span [0..=200] so length=201 due to large gaps assert_eq!(stats.len(), 1); @@ -663,7 +674,7 @@ mod tests { let processor = MockProcessor::new(); let trainer = ZoneTrainer::new(processor, 10).unwrap(); - let stats = trainer.train(stream).await.unwrap(); + let (stats, _) = trainer.train(stream).await.unwrap(); // Should create 3 zones (one per fragment) assert_eq!(stats.len(), 3); @@ -810,7 +821,7 @@ mod tests { )); let trainer = ZoneTrainer::new(MockProcessor::new(), 2).unwrap(); - let rebuilt = rebuild_zones(&existing, trainer, stream).await.unwrap(); + let (rebuilt, _) = rebuild_zones(&existing, trainer, stream).await.unwrap(); // Existing zone should remain unchanged and new stats appended afterwards assert_eq!(rebuilt.len(), 2); assert_eq!(rebuilt[0].sum, 50); @@ -840,7 +851,7 @@ mod tests { )); let trainer = ZoneTrainer::new(MockProcessor::new(), 2).unwrap(); - let rebuilt = rebuild_zones(&existing, trainer, stream).await.unwrap(); + let (rebuilt, _) = rebuild_zones(&existing, trainer, stream).await.unwrap(); // Existing zone plus two new fragments should yield three total zones assert_eq!(rebuilt.len(), 3); assert_eq!(rebuilt[0].bound.fragment_id, 0); diff --git a/rust/lance-index/src/scalar/zonemap.rs b/rust/lance-index/src/scalar/zonemap.rs index 770188f3378..9e4f6bb63e6 100644 --- a/rust/lance-index/src/scalar/zonemap.rs +++ b/rust/lance-index/src/scalar/zonemap.rs @@ -12,12 +12,12 @@ //! false positives that require rechecking. //! //! -use crate::Any; use crate::pbold; use crate::scalar::expression::{SargableQueryParser, ScalarQueryParser}; use crate::scalar::registry::{ BasicTrainer, ScalarIndexPlugin, TrainingCriteria, TrainingOrdering, TrainingRequest, }; +use crate::scalar::seed::IndexSeedWriter; use crate::scalar::{ BuiltinIndexType, CreatedIndex, IndexFile, SargableQuery, ScalarIndexParams, UpdateCriteria, compute_next_prefix, @@ -26,6 +26,7 @@ use lance_arrow_stats::StatisticsAccumulator; use lance_core::cache::{LanceCache, WeakLanceCache}; use lance_core::utils::row_addr_remap::RowAddrRemap; use serde::{Deserialize, Serialize}; +use std::any::Any; use std::sync::LazyLock; use arrow_array::{ @@ -34,7 +35,8 @@ use arrow_array::{ use arrow_schema::{DataType, Field}; use datafusion::execution::SendableRecordBatchStream; use datafusion_common::ScalarValue; -use std::sync::Arc; +use lance_select::RowAddrTreeMap; +use std::{collections::HashMap, sync::Arc}; use super::{AnyQuery, IndexStore, MetricsCollector, ScalarIndex, SearchResult}; use crate::scalar::RowIdRemapper; @@ -50,11 +52,13 @@ const ROWS_PER_ZONE_DEFAULT: u64 = 8192; // 1 zone every two batches const ZONEMAP_FILENAME: &str = "zonemap.lance"; const ZONEMAP_SIZE_META_KEY: &str = "rows_per_zone"; +const NULL_BITMAP_META_KEY: &str = "null_bitmap"; +const SEED_NULL_BITMAP_META_KEY: &str = "seed_null_bitmap"; const ZONEMAP_INDEX_VERSION: u32 = 0; /// Basic stats about zonemap index #[derive(Debug, PartialEq, Clone)] -struct ZoneMapStatistics { +pub(crate) struct ZoneMapStatistics { min: ScalarValue, max: ScalarValue, null_count: u32, @@ -107,9 +111,13 @@ pub struct ZoneMapIndex { data_type: DataType, // The maximum rows per zone provided by user rows_per_zone: u64, + use_seeds: bool, store: Arc, fri: Option>, index_cache: WeakLanceCache, + // Exact set of null row addresses across all zones; None when loaded from an + // older index that did not persist this bitmap. + null_rows: Option, } impl std::fmt::Debug for ZoneMapIndex { @@ -118,6 +126,7 @@ impl std::fmt::Debug for ZoneMapIndex { .field("zones", &self.zones) .field("data_type", &self.data_type) .field("rows_per_zone", &self.rows_per_zone) + .field("use_seeds", &self.use_seeds) .field("store", &self.store) .field("fri", &self.fri) .field("index_cache", &self.index_cache) @@ -127,11 +136,16 @@ impl std::fmt::Debug for ZoneMapIndex { impl DeepSizeOf for ZoneMapIndex { fn deep_size_of_children(&self, context: &mut lance_core::deepsize::Context) -> usize { - self.zones.deep_size_of_children(context) + self.zones.deep_size_of_children(context) + self.null_rows.deep_size_of_children(context) } } impl ZoneMapIndex { + /// Returns the rows-per-zone parameter for this index. + pub fn rows_per_zone(&self) -> u64 { + self.rows_per_zone + } + fn scalar_is_nan(value: &ScalarValue) -> bool { match value { ScalarValue::Float16(Some(value)) => value.is_nan(), @@ -170,19 +184,25 @@ impl ZoneMapIndex { ) -> Option<(ScalarValue, ScalarValue)> { let mut min: Option<&ScalarValue> = None; let mut max: Option<&ScalarValue> = None; - for zone in segments.into_iter().flat_map(|seg| seg.zones.iter()) { - if Self::scalar_is_nan(&zone.max) { + for seg in segments.into_iter() { + // Nested types have no meaningful ordering + if seg.data_type.is_nested() { return None; } - if Self::scalar_is_finite_bound(&zone.min) - && min.is_none_or(|cur| zone.min.partial_cmp(cur).is_some_and(|o| o.is_lt())) - { - min = Some(&zone.min); - } - if Self::scalar_is_finite_bound(&zone.max) - && max.is_none_or(|cur| zone.max.partial_cmp(cur).is_some_and(|o| o.is_gt())) - { - max = Some(&zone.max); + for zone in seg.zones.iter() { + if Self::scalar_is_nan(&zone.max) { + return None; + } + if Self::scalar_is_finite_bound(&zone.min) + && min.is_none_or(|cur| zone.min.partial_cmp(cur).is_some_and(|o| o.is_lt())) + { + min = Some(&zone.min); + } + if Self::scalar_is_finite_bound(&zone.max) + && max.is_none_or(|cur| zone.max.partial_cmp(cur).is_some_and(|o| o.is_gt())) + { + max = Some(&zone.max); + } } } Some((min?.clone(), max?.clone())) @@ -205,6 +225,29 @@ impl ZoneMapIndex { ) -> Result { use std::ops::Bound; + // For nested types we only track null_count; prune only when certain. + if self.data_type.is_nested() { + let all_null = zone.null_count as usize == zone.bound.length; + return match query { + SargableQuery::IsNull() => Ok(zone.null_count > 0), + SargableQuery::Equals(target) => { + if target.is_null() { + Ok(zone.null_count > 0) + } else { + Ok(!all_null) + } + } + SargableQuery::IsIn(values) => { + if values.iter().any(|v| !v.is_null()) { + Ok(!all_null) + } else { + Ok(zone.null_count > 0) + } + } + _ => Ok(!all_null), + }; + } + match query { SargableQuery::IsNull() => { // Zone contains matching values if it has any null values @@ -260,10 +303,8 @@ impl ZoneMapIndex { return Ok(zone.nan_count > 0); } } - ScalarValue::Float64(Some(f)) => { - if f.is_nan() { - return Ok(zone.nan_count > 0); - } + ScalarValue::Float64(Some(f)) if f.is_nan() => { + return Ok(zone.nan_count > 0); } _ => {} } @@ -290,10 +331,8 @@ impl ZoneMapIndex { return Ok(false); // Nothing is greater than NaN } } - ScalarValue::Float64(Some(f)) => { - if f.is_nan() { - return Ok(false); // Nothing is greater than NaN - } + ScalarValue::Float64(Some(f)) if f.is_nan() => { + return Ok(false); // Nothing is greater than NaN } _ => {} } @@ -317,10 +356,8 @@ impl ZoneMapIndex { return Ok(zone.nan_count > 0 || zone_min <= e); } } - ScalarValue::Float64(Some(f)) => { - if f.is_nan() { - return Ok(zone.nan_count > 0 || zone_min <= e); - } + ScalarValue::Float64(Some(f)) if f.is_nan() => { + return Ok(zone.nan_count > 0 || zone_min <= e); } _ => {} } @@ -340,10 +377,8 @@ impl ZoneMapIndex { return Ok(true); } } - ScalarValue::Float64(Some(f)) => { - if f.is_nan() { - return Ok(true); - } + ScalarValue::Float64(Some(f)) if f.is_nan() => { + return Ok(true); } _ => {} } @@ -454,6 +489,7 @@ impl ZoneMapIndex { store: Arc, fri: Option>, index_cache: &LanceCache, + use_seeds: bool, ) -> Result> where Self: Sized, @@ -469,12 +505,25 @@ impl ZoneMapIndex { .get(ZONEMAP_SIZE_META_KEY) .and_then(|bs| bs.parse().ok()) .unwrap_or(ROWS_PER_ZONE_DEFAULT); + + let null_rows = if let Some(idx_str) = file_schema.metadata.get(NULL_BITMAP_META_KEY) { + let idx = idx_str.parse::().map_err(|e| { + Error::invalid_input(format!("invalid null bitmap buffer index: {e}")) + })?; + let bytes = index_file.read_global_buffer(idx).await?; + Some(RowAddrTreeMap::deserialize_from(bytes.as_ref())?) + } else { + None + }; + Ok(Arc::new(Self::try_from_serialized( zone_maps, store, fri, index_cache, rows_per_zone, + null_rows, + use_seeds, )?)) } @@ -484,6 +533,8 @@ impl ZoneMapIndex { fri: Option>, index_cache: &LanceCache, rows_per_zone: u64, + null_rows: Option, + use_seeds: bool, ) -> Result { // The RecordBatch should have columns: min, max, null_count let min_col = data @@ -542,9 +593,11 @@ impl ZoneMapIndex { zones: Vec::new(), data_type, rows_per_zone, + use_seeds, store, fri, index_cache: WeakLanceCache::from(index_cache), + null_rows, }); } @@ -573,9 +626,11 @@ impl ZoneMapIndex { zones, data_type, rows_per_zone, + use_seeds, store, fri, index_cache: WeakLanceCache::from(index_cache), + null_rows, }) } } @@ -626,11 +681,21 @@ impl ScalarIndex for ZoneMapIndex { metrics: &dyn MetricsCollector, ) -> Result { let query = query.as_any().downcast_ref::().unwrap(); + if let SargableQuery::IsNull() = query + && let Some(null_rows) = &self.null_rows + { + return Ok(SearchResult::exact(null_rows.clone())); + } + search_zones(&self.zones, metrics, |zone| { self.evaluate_zone_against_query(zone, query) }) } + fn results_are_row_addresses(&self) -> bool { + true + } + fn can_remap(&self) -> bool { false } @@ -660,19 +725,34 @@ impl ScalarIndex for ZoneMapIndex { let options = ZoneMapIndexBuilderParams::new(self.rows_per_zone); let processor = ZoneMapProcessor::new(value_type.clone())?; let trainer = ZoneTrainer::new(processor, self.rows_per_zone)?; - let updated_zones = rebuild_zones(&self.zones, trainer, new_data).await?; + let (updated_zones, new_null_rows) = rebuild_zones(&self.zones, trainer, new_data).await?; + + // Merge existing and new null rows. If the existing index had no null bitmap + // (legacy format — null positions unknown), preserve that None: updating cannot + // recover the missing information, and claiming the result has zero nulls would + // be a false negative. Only a full retrain produces a fresh, complete bitmap. + let merged_null_rows = self.null_rows.as_ref().map(|existing| { + let mut merged = existing.clone(); + merged |= &new_null_rows; + merged + }); // Serialize the combined zones back into the index file let mut builder = ZoneMapIndexBuilder::try_new(options, self.data_type.clone())?; builder.options.rows_per_zone = self.rows_per_zone; builder.maps = updated_zones; - let file = builder.write_index(dest_store).await?; + builder.null_rows = merged_null_rows; + let has_null_bitmap = builder.null_rows.is_some(); + let files = builder.write_index(dest_store).await?; Ok(CreatedIndex { - index_details: prost_types::Any::from_msg(&pbold::ZoneMapIndexDetails::default()) - .unwrap(), + index_details: make_zone_map_index_details( + self.rows_per_zone, + self.use_seeds, + has_null_bitmap, + ), index_version: ZONEMAP_INDEX_VERSION, - files: vec![file], + files, }) } @@ -683,17 +763,86 @@ impl ScalarIndex for ZoneMapIndex { } fn derive_index_params(&self) -> Result { - let params = serde_json::to_value(ZoneMapIndexBuilderParams::new(self.rows_per_zone))?; + let params = serde_json::to_value(ZoneMapIndexBuilderParams { + rows_per_zone: self.rows_per_zone, + use_seeds: Some(self.use_seeds), + })?; Ok(ScalarIndexParams::for_builtin(BuiltinIndexType::ZoneMap).with_params(¶ms)) } /// Single-segment `[min, max]` folded from this index's zones; see /// [`value_range_over`](Self::value_range_over) for the full contract. fn value_range(&self) -> Option<(ScalarValue, ScalarValue)> { + // We don't record min/max for nested types + if self.data_type.is_nested() { + return None; + } Self::value_range_over([self]) } } +impl ZoneMapIndex { + async fn try_update_with_seeds( + &self, + seeds: &[crate::scalar::seed::FragmentSeed], + dest_store: &dyn IndexStore, + ) -> Result> { + let mut new_zones = self.zones.clone(); + let mut merged_null_rows = self.null_rows.clone(); + let mut any_missing_bitmap = self.null_rows.is_none(); + + for seed in seeds { + let (mut zones, seed_null_bitmap) = ZoneMapSeedWriter::deserialize_seed( + seed.fragment_id, + &seed.bytes, + self.rows_per_zone, + )?; + new_zones.append(&mut zones); + + if !any_missing_bitmap { + match seed_null_bitmap { + Some(bitmap) => { + let frag_id = u32::try_from(seed.fragment_id).map_err(|_| { + Error::invalid_input(format!( + "fragment_id {} exceeds u32::MAX", + seed.fragment_id + )) + })?; + merged_null_rows + .as_mut() + .unwrap() + .insert_bitmap(frag_id, bitmap); + } + None => { + any_missing_bitmap = true; + merged_null_rows = None; + } + } + } + } + new_zones.sort_by_key(|z| (z.bound.fragment_id, z.bound.start)); + + let mut builder = ZoneMapIndexBuilder::try_new( + ZoneMapIndexBuilderParams::new(self.rows_per_zone), + self.data_type.clone(), + )?; + builder.maps = new_zones; + builder.null_rows = merged_null_rows; + let has_null_bitmap = builder.null_rows.is_some(); + let files = builder.write_index(dest_store).await?; + + Ok(Some(CreatedIndex { + index_details: make_zone_map_index_details( + self.rows_per_zone, + self.use_seeds, + has_null_bitmap, + ), + index_version: ZONEMAP_INDEX_VERSION, + files, + })) + } +} + /// Merge caller-selected ZoneMap segments into one self-contained segment. pub async fn merge_zonemap_indices( source_indices: &[&ZoneMapIndex], @@ -704,9 +853,12 @@ pub async fn merge_zonemap_indices( Error::invalid_input("merge_zonemap_indices requires at least one source index") })?; let rows_per_zone = first.rows_per_zone; + let use_seeds = first.use_seeds; let data_type = first.data_type.clone(); let mut zones = Vec::new(); + let mut merged_null_rows = RowAddrTreeMap::new(); + let mut any_missing_bitmap = false; for source in source_indices { if source.rows_per_zone != rows_per_zone { return Err(Error::invalid_input(format!( @@ -730,18 +882,30 @@ pub async fn merge_zonemap_indices( }) .cloned(), ); + match &source.null_rows { + Some(null_rows) => { + let mut filtered = null_rows.clone(); + filtered.retain_fragments(fragment_filter.iter()); + merged_null_rows |= &filtered; + } + None => any_missing_bitmap = true, + } } zones.sort_by_key(|zone| (zone.bound.fragment_id, zone.bound.start)); let mut builder = ZoneMapIndexBuilder::try_new(ZoneMapIndexBuilderParams::new(rows_per_zone), data_type)?; builder.maps = zones; - builder.write_index(dest_store).await?; + if !any_missing_bitmap { + builder.null_rows = Some(merged_null_rows); + } + let has_null_bitmap = builder.null_rows.is_some(); + let files = builder.write_index(dest_store).await?; Ok(CreatedIndex { - index_details: prost_types::Any::from_msg(&pbold::ZoneMapIndexDetails::default()).unwrap(), + index_details: make_zone_map_index_details(rows_per_zone, use_seeds, has_null_bitmap), index_version: ZONEMAP_INDEX_VERSION, - files: dest_store.list_files_with_sizes().await?, + files, }) } @@ -753,6 +917,12 @@ fn default_rows_per_zone() -> u64 { pub struct ZoneMapIndexBuilderParams { #[serde(default = "default_rows_per_zone")] rows_per_zone: u64, + /// Whether to embed per-fragment seed buffers in data files for use during + /// incremental index updates. `None` means auto-detect based on column type + /// (see [`default_use_seeds`]). Resolved to a concrete `bool` during + /// training in [`ZoneMapIndexPlugin::new_training_request`]. + #[serde(default)] + use_seeds: Option, } static DEFAULT_ROWS_PER_ZONE: LazyLock = LazyLock::new(|| { @@ -766,13 +936,17 @@ impl Default for ZoneMapIndexBuilderParams { fn default() -> Self { Self { rows_per_zone: *DEFAULT_ROWS_PER_ZONE, + use_seeds: None, } } } impl ZoneMapIndexBuilderParams { pub fn new(rows_per_zone: u64) -> Self { - Self { rows_per_zone } + Self { + rows_per_zone, + use_seeds: None, + } } pub fn rows_per_zone(&self) -> u64 { @@ -786,6 +960,10 @@ pub struct ZoneMapIndexBuilder { items_type: DataType, maps: Vec, + // None means "legacy index — null positions unknown"; Some means a complete bitmap. + // write_index omits the null-bitmap global buffer when this is None, preserving the + // legacy format so that downstream searches remain conservative. + null_rows: Option, } impl ZoneMapIndexBuilder { @@ -794,6 +972,7 @@ impl ZoneMapIndexBuilder { options, items_type, maps: Vec::new(), + null_rows: None, }) } @@ -803,7 +982,9 @@ impl ZoneMapIndexBuilder { pub async fn train(&mut self, batches_source: SendableRecordBatchStream) -> Result<()> { let processor = ZoneMapProcessor::new(self.items_type.clone())?; let trainer = ZoneTrainer::new(processor, self.options.rows_per_zone)?; - self.maps = trainer.train(batches_source).await?; + let (maps, null_rows) = trainer.train(batches_source).await?; + self.maps = maps; + self.null_rows = Some(null_rows); Ok(()) } @@ -856,7 +1037,7 @@ impl ZoneMapIndexBuilder { Ok(RecordBatch::try_new(schema, columns)?) } - pub async fn write_index(self, index_store: &dyn IndexStore) -> Result { + pub async fn write_index(self, index_store: &dyn IndexStore) -> Result> { let record_batch = self.zonemap_stats_as_batch()?; let mut file_schema = record_batch.schema().as_ref().clone(); @@ -869,12 +1050,34 @@ impl ZoneMapIndexBuilder { .new_index_file(ZONEMAP_FILENAME, Arc::new(file_schema)) .await?; index_file.write_record_batch(record_batch).await?; - index_file.finish().await + + let zonemap_file = if let Some(null_rows) = self.null_rows { + let mut null_bitmap_bytes = Vec::with_capacity(null_rows.serialized_size()); + null_rows.serialize_into(&mut null_bitmap_bytes)?; + let null_bitmap_idx = index_file + .add_global_buffer(bytes::Bytes::from(null_bitmap_bytes)) + .await?; + index_file + .finish_with_metadata(HashMap::from([( + NULL_BITMAP_META_KEY.to_string(), + null_bitmap_idx.to_string(), + )])) + .await? + } else { + index_file.finish_with_metadata(HashMap::new()).await? + }; + + Ok(vec![zonemap_file]) } } -/// Index-specific processor that computes min/max statistics for each zone while the -/// trainer takes care of chunking and fragment boundaries. +/// Index-specific processor that computes zone statistics while the trainer +/// handles chunking and fragment boundaries. +/// +/// For non-nested types, tracks min, max, null_count, and nan_count. +/// For nested types (List, FixedSizeList, Struct, Map, etc.), tracks only +/// null_count; min and max are stored as typed null values. +#[derive(Debug)] struct ZoneMapProcessor { data_type: DataType, statistics: StatisticsAccumulator, @@ -882,9 +1085,10 @@ struct ZoneMapProcessor { impl ZoneMapProcessor { fn new(data_type: DataType) -> Result { + let statistics = StatisticsAccumulator::new(&data_type); Ok(Self { - statistics: StatisticsAccumulator::new(&data_type), data_type, + statistics, }) } @@ -943,6 +1147,19 @@ impl ZoneProcessor for ZoneMapProcessor { fn finish_zone(&mut self, bound: ZoneBound) -> Result { let statistics = self.statistics.statistics(); + let null_count = Self::stat_count_to_u32("null_count", statistics.null_count)?; + + // For nested types, only null_count is meaningful; store null min/max. + if self.data_type.is_nested() { + return Ok(ZoneMapStatistics { + min: ScalarValue::try_new_null(&self.data_type)?, + max: ScalarValue::try_new_null(&self.data_type)?, + null_count, + nan_count: 0, + bound, + }); + } + let nan_count = Self::stat_count_to_u32("nan_count", statistics.nan_count.unwrap_or(0))?; Ok(ZoneMapStatistics { min: Self::scalar_value_from_stat( @@ -954,7 +1171,7 @@ impl ZoneProcessor for ZoneMapProcessor { &self.data_type, nan_count, )?, - null_count: Self::stat_count_to_u32("null_count", statistics.null_count)?, + null_count, nan_count, bound, }) @@ -966,6 +1183,44 @@ impl ZoneProcessor for ZoneMapProcessor { } } +fn make_zone_map_index_details( + rows_per_zone: u64, + use_seeds: bool, + has_null_bitmap: bool, +) -> prost_types::Any { + prost_types::Any::from_msg(&pbold::ZoneMapIndexDetails { + rows_per_zone: Some(rows_per_zone), + use_seeds: Some(use_seeds), + has_null_bitmap: Some(has_null_bitmap), + }) + .unwrap() +} + +/// Returns true when seed-based incremental updates should be enabled by +/// default for the given column type. +/// +/// Seeds pay off for variable-length types (strings, binary) — which can be +/// arbitrarily wide — and fixed-width types wider than 8 bytes (e.g. +/// Decimal128, FixedSizeBinary tensors). Fixed-width types ≤ 8 bytes (Int64, +/// Float64, …) scan fast enough that the seed overhead is not worth it. +fn default_use_seeds(data_type: &DataType) -> bool { + match data_type { + // Variable-length: width is unbounded, skipping scans is always valuable. + DataType::Utf8 + | DataType::LargeUtf8 + | DataType::Utf8View + | DataType::Binary + | DataType::LargeBinary + | DataType::BinaryView => true, + // Fixed-width types wider than 8 bytes. + DataType::Decimal128(_, _) | DataType::Decimal256(_, _) => true, + DataType::FixedSizeBinary(n) => *n > 8, + // Nested types (FSL, List, Struct, Map, …): typically wide. + _ if data_type.is_nested() => true, + _ => false, + } +} + #[derive(Debug, Default)] pub struct ZoneMapIndexPlugin; @@ -974,8 +1229,7 @@ impl ZoneMapIndexPlugin { batches_source: SendableRecordBatchStream, index_store: &dyn IndexStore, options: Option, - ) -> Result { - // train_zonemap_index: calling scan_aligned_chunks + ) -> Result> { let value_type = batches_source.schema().field(0).data_type().clone(); let mut builder = ZoneMapIndexBuilder::try_new(options.unwrap_or_default(), value_type)?; @@ -1016,14 +1270,12 @@ impl BasicTrainer for ZoneMapIndexPlugin { params: &str, field: &Field, ) -> Result> { - if field.data_type().is_nested() { - return Err(Error::invalid_input_source( - "A zone map index can only be created on a non-nested field.".into(), - )); + let mut params = serde_json::from_str::(params)?; + // Resolve None → type-based default so train_index always sees Some(bool). + if params.use_seeds.is_none() { + params.use_seeds = Some(default_use_seeds(field.data_type())); } - let params = serde_json::from_str::(params)?; - Ok(Box::new(ZoneMapIndexTrainingRequest::new(params))) } @@ -1042,12 +1294,15 @@ impl BasicTrainer for ZoneMapIndexPlugin { "must provide training request created by new_training_request".into(), ) })?; - let file = Self::train_zonemap_index(data, index_store, Some(request.params)).await?; + let rows_per_zone = request.params.rows_per_zone; + let use_seeds = request.params.use_seeds.unwrap_or(false); + let files = Self::train_zonemap_index(data, index_store, Some(request.params)).await?; + // Training a new index will always populate the null bitmap + let has_null_bitmap = true; Ok(CreatedIndex { - index_details: prost_types::Any::from_msg(&pbold::ZoneMapIndexDetails::default()) - .unwrap(), + index_details: make_zone_map_index_details(rows_per_zone, use_seeds, has_null_bitmap), index_version: ZONEMAP_INDEX_VERSION, - files: vec![file], + files, }) } } @@ -1073,24 +1328,428 @@ impl ScalarIndexPlugin for ZoneMapIndexPlugin { fn new_query_parser( &self, index_name: String, - _index_details: &prost_types::Any, + index_details: &prost_types::Any, ) -> Option> { - Some(Box::new(SargableQueryParser::new( - index_name, - self.name().to_string(), - true, - ))) + let has_null_bitmap = index_details + .to_msg::() + .ok() + .and_then(|d| d.has_null_bitmap) + .unwrap_or(false); + let parser = SargableQueryParser::new(index_name, self.name().to_string(), true); + let parser = if has_null_bitmap { + parser.with_exact_null_tracking() + } else { + parser + }; + Some(Box::new(parser)) } async fn load_index( &self, index_store: Arc, - _index_details: &prost_types::Any, + index_details: &prost_types::Any, frag_reuse_index: Option>, cache: &LanceCache, ) -> Result> { - Ok(ZoneMapIndex::load(index_store, frag_reuse_index, cache).await? as Arc) + let use_seeds = index_details + .to_msg::() + .ok() + .and_then(|d| d.use_seeds) + .unwrap_or(false); + Ok( + ZoneMapIndex::load(index_store, frag_reuse_index, cache, use_seeds).await? + as Arc, + ) + } + + fn might_use_seeds(&self, index_details: &prost_types::Any) -> bool { + index_details + .to_msg::() + .ok() + .and_then(|d| d.use_seeds) + .unwrap_or(false) + } + + async fn create_seed_writer( + &self, + field_path: &str, + data_type: &DataType, + index_details: &prost_types::Any, + ) -> Result>> { + let details = index_details.to_msg::().ok(); + let Some(rows_per_zone) = details.as_ref().and_then(|d| d.rows_per_zone) else { + return Ok(None); + }; + if !details.as_ref().and_then(|d| d.use_seeds).unwrap_or(false) { + return Ok(None); + } + Ok(Some(Box::new(ZoneMapSeedWriter::new( + field_path, + rows_per_zone, + data_type.clone(), + )?))) + } + + async fn update_from_seeds( + &self, + seeds: Vec, + reference_index: Arc, + index_details: &prost_types::Any, + dest_store: &dyn IndexStore, + ) -> Result> { + let Some(rows_per_zone) = index_details + .to_msg::() + .ok() + .and_then(|d| d.rows_per_zone) + else { + return Ok(None); + }; + + // Validate each seed was written with the same rows_per_zone. + for seed in &seeds { + let rpz_in_seed = seed + .metadata_value + .split_once(':') + .and_then(|(_, rpz)| rpz.parse::().ok()); + if rpz_in_seed != Some(rows_per_zone) { + return Ok(None); + } + } + + let Some(zone_map) = reference_index.as_any().downcast_ref::() else { + return Ok(None); + }; + zone_map.try_update_with_seeds(&seeds, dest_store).await + } +} + +/// A seed writer that observes column values during data file writes and +/// accumulates zone map statistics for later harvest during index updates. +/// +/// Zone statistics are serialized as Arrow IPC bytes and embedded in the data +/// file footer as a global buffer, keyed by `"lance.seed."`. +#[derive(Debug)] +pub struct ZoneMapSeedWriter { + column_name: String, + rows_per_zone: u64, + data_type: DataType, + completed_zones: Vec, + processor: ZoneMapProcessor, + rows_in_current_zone: u64, + next_zone_start: u64, + /// Null row offsets within this fragment (sequential, 0-indexed). + null_offsets: RoaringBitmap, +} + +impl ZoneMapSeedWriter { + /// Create a new `ZoneMapSeedWriter` for the given column. + pub fn new( + column_name: impl Into, + rows_per_zone: u64, + data_type: DataType, + ) -> Result { + if rows_per_zone == 0 { + return Err(lance_core::Error::invalid_input( + "rows_per_zone must be greater than zero", + )); + } + let processor = ZoneMapProcessor::new(data_type.clone())?; + Ok(Self { + column_name: column_name.into(), + rows_per_zone, + data_type, + completed_zones: Vec::new(), + processor, + rows_in_current_zone: 0, + next_zone_start: 0, + null_offsets: RoaringBitmap::new(), + }) + } + + fn seed_batch_from_zones( + zones: &[ZoneMapStatistics], + data_type: &DataType, + ) -> Result { + let mins = if zones.is_empty() { + arrow_array::new_empty_array(data_type) + } else { + datafusion_common::ScalarValue::iter_to_array(zones.iter().map(|s| s.min.clone()))? + }; + let maxs = if zones.is_empty() { + arrow_array::new_empty_array(data_type) + } else { + datafusion_common::ScalarValue::iter_to_array(zones.iter().map(|s| s.max.clone()))? + }; + let null_counts = + arrow_array::UInt32Array::from_iter_values(zones.iter().map(|s| s.null_count)); + let nan_counts = + arrow_array::UInt32Array::from_iter_values(zones.iter().map(|s| s.nan_count)); + let zone_lengths = + arrow_array::UInt64Array::from_iter_values(zones.iter().map(|s| s.bound.length as u64)); + + let schema = Arc::new(arrow_schema::Schema::new(vec![ + Field::new("min", data_type.clone(), true), + Field::new("max", data_type.clone(), true), + Field::new("null_count", DataType::UInt32, false), + Field::new("nan_count", DataType::UInt32, false), + Field::new("zone_length", DataType::UInt64, false), + ])); + + let columns: Vec = vec![ + mins, + maxs, + Arc::new(null_counts) as ArrayRef, + Arc::new(nan_counts) as ArrayRef, + Arc::new(zone_lengths) as ArrayRef, + ]; + Ok(arrow_array::RecordBatch::try_new(schema, columns)?) + } + + /// Deserialize zone map seed bytes (Arrow IPC) into zone statistics and a null bitmap. + /// + /// Returns zone statistics (bounds reconstructed from sequential fragment layout) and an + /// optional `RoaringBitmap` of null row offsets within the fragment. The bitmap is absent + /// for seeds written by older code that did not track null positions. + pub(crate) fn deserialize_seed( + fragment_id: u64, + bytes: &bytes::Bytes, + rows_per_zone: u64, + ) -> Result<(Vec, Option)> { + use arrow_ipc::reader::FileReader; + use std::io::Cursor; + + let cursor = Cursor::new(bytes.as_ref()); + let mut reader = FileReader::try_new(cursor, None).map_err(|e| { + lance_core::Error::invalid_input(format!("failed to read zone map seed IPC: {}", e)) + })?; + + let null_bitmap = if let Some(hex) = reader.schema().metadata.get(SEED_NULL_BITMAP_META_KEY) + { + let bitmap_bytes = hex_decode(hex).map_err(|e| { + lance_core::Error::invalid_input(format!( + "failed to decode seed null bitmap hex: {}", + e + )) + })?; + let bitmap = if bitmap_bytes.is_empty() { + RoaringBitmap::default() + } else { + RoaringBitmap::deserialize_from(bitmap_bytes.as_slice()).map_err(|e| { + lance_core::Error::invalid_input(format!( + "failed to deserialize seed null bitmap: {}", + e + )) + })? + }; + Some(bitmap) + } else { + None + }; + + let batch = match reader.next() { + Some(Ok(batch)) => batch, + Some(Err(e)) => { + return Err(lance_core::Error::invalid_input(format!( + "failed to read zone map seed batch: {}", + e + ))); + } + None => return Ok((Vec::new(), null_bitmap)), + }; + + let min_col = batch + .column_by_name("min") + .ok_or_else(|| lance_core::Error::invalid_input("seed batch missing 'min' column"))?; + let max_col = batch + .column_by_name("max") + .ok_or_else(|| lance_core::Error::invalid_input("seed batch missing 'max' column"))?; + let null_count_col = batch + .column_by_name("null_count") + .ok_or_else(|| { + lance_core::Error::invalid_input("seed batch missing 'null_count' column") + })? + .as_any() + .downcast_ref::() + .ok_or_else(|| lance_core::Error::invalid_input("seed 'null_count' is not UInt32"))?; + let nan_count_col = batch + .column_by_name("nan_count") + .ok_or_else(|| { + lance_core::Error::invalid_input("seed batch missing 'nan_count' column") + })? + .as_any() + .downcast_ref::() + .ok_or_else(|| lance_core::Error::invalid_input("seed 'nan_count' is not UInt32"))?; + let zone_length_col = batch + .column_by_name("zone_length") + .ok_or_else(|| { + lance_core::Error::invalid_input("seed batch missing 'zone_length' column") + })? + .as_any() + .downcast_ref::() + .ok_or_else(|| lance_core::Error::invalid_input("seed 'zone_length' is not UInt64"))?; + + let num_zones = batch.num_rows(); + let mut zones = Vec::with_capacity(num_zones); + for i in 0..num_zones { + let zone_start = i as u64 * rows_per_zone; + let zone_length = zone_length_col.value(i) as usize; + zones.push(ZoneMapStatistics { + min: datafusion_common::ScalarValue::try_from_array(min_col, i)?, + max: datafusion_common::ScalarValue::try_from_array(max_col, i)?, + null_count: null_count_col.value(i), + nan_count: nan_count_col.value(i), + bound: ZoneBound { + fragment_id, + start: zone_start, + length: zone_length, + }, + }); + } + Ok((zones, null_bitmap)) + } +} + +impl IndexSeedWriter for ZoneMapSeedWriter { + fn column_name(&self) -> &str { + &self.column_name + } + + fn observe_batch(&mut self, values: &ArrayRef) -> lance_core::Result<()> { + let mut offset = 0usize; + + while offset < values.len() { + let remaining_in_zone = self.rows_per_zone - self.rows_in_current_zone; + let chunk_len = ((values.len() as u64 - offset as u64).min(remaining_in_zone)) as usize; + let chunk = values.slice(offset, chunk_len); + self.processor.process_chunk(&chunk)?; + + if chunk.null_count() > 0 { + let base_offset = (self.next_zone_start + self.rows_in_current_zone) as u32; + for i in 0..chunk_len { + if chunk.is_null(i) { + self.null_offsets.insert(base_offset + i as u32); + } + } + } + + self.rows_in_current_zone += chunk_len as u64; + offset += chunk_len; + + if self.rows_in_current_zone >= self.rows_per_zone { + let bound = ZoneBound { + fragment_id: 0, + start: self.next_zone_start, + length: self.rows_per_zone as usize, + }; + let stats = self.processor.finish_zone(bound)?; + self.processor.reset()?; + self.completed_zones.push(stats); + self.next_zone_start += self.rows_per_zone; + self.rows_in_current_zone = 0; + } + } + Ok(()) + } + + fn finish(&mut self) -> lance_core::Result> { + use arrow_ipc::writer::FileWriter; + use std::io::Cursor; + + // Flush partial final zone + if self.rows_in_current_zone > 0 { + let bound = ZoneBound { + fragment_id: 0, + start: self.next_zone_start, + length: self.rows_in_current_zone as usize, + }; + let stats = self.processor.finish_zone(bound)?; + self.processor.reset()?; + self.completed_zones.push(stats); + self.next_zone_start += self.rows_in_current_zone; + self.rows_in_current_zone = 0; + } + + if self.completed_zones.is_empty() { + return Ok(None); + } + + let batch = Self::seed_batch_from_zones(&self.completed_zones, &self.data_type)?; + + // Embed null bitmap in schema metadata so deserialize_seed can reconstruct null_rows. + let null_offsets = std::mem::take(&mut self.null_offsets); + let bitmap_bytes = if null_offsets.is_empty() { + Vec::default() + } else { + let mut bitmap_bytes = Vec::with_capacity(null_offsets.serialized_size()); + null_offsets + .serialize_into(&mut bitmap_bytes) + .map_err(|e| { + lance_core::Error::invalid_input(format!( + "failed to serialize seed null bitmap: {}", + e + )) + })?; + bitmap_bytes + }; + let mut schema = batch.schema().as_ref().clone(); + schema.metadata.insert( + SEED_NULL_BITMAP_META_KEY.to_string(), + hex_encode(&bitmap_bytes), + ); + let batch = RecordBatch::try_new(Arc::new(schema), batch.columns().to_vec())?; + + // Serialize to Arrow IPC + let mut buf = Cursor::new(Vec::new()); + { + let mut writer = FileWriter::try_new(&mut buf, batch.schema_ref()).map_err(|e| { + lance_core::Error::invalid_input(format!("failed to create IPC writer: {}", e)) + })?; + writer.write(&batch).map_err(|e| { + lance_core::Error::invalid_input(format!("failed to write IPC batch: {}", e)) + })?; + writer.finish().map_err(|e| { + lance_core::Error::invalid_input(format!("failed to finish IPC writer: {}", e)) + })?; + } + + // Reset state for next fragment + self.completed_zones.clear(); + self.next_zone_start = 0; + self.processor = ZoneMapProcessor::new(self.data_type.clone())?; + + Ok(Some(bytes::Bytes::from(buf.into_inner()))) + } + + fn schema_metadata_key(&self) -> String { + format!( + "{}{}", + crate::scalar::seed::SEED_META_KEY_PREFIX, + self.column_name + ) + } + + fn schema_metadata_value(&self, buf_index: u32) -> String { + format!("{}:{}", buf_index, self.rows_per_zone) + } +} + +fn hex_encode(data: &[u8]) -> String { + data.iter().map(|b| format!("{:02x}", b)).collect() +} + +fn hex_decode(s: &str) -> std::result::Result, String> { + if s.is_empty() { + return Ok(Vec::default()); + } + if !s.len().is_multiple_of(2) { + return Err(format!("odd hex length: {}", s.len())); } + (0..s.len()) + .step_by(2) + .map(|i| { + u8::from_str_radix(&s[i..i + 2], 16).map_err(|e| format!("invalid hex at {}: {}", i, e)) + }) + .collect() } #[cfg(test)] @@ -1117,13 +1776,14 @@ mod tests { use lance_datagen::ArrayGeneratorExt; use lance_datagen::{BatchCount, RowCount, array}; use lance_io::object_store::ObjectStore; - use lance_select::{NullableRowAddrSet, RowAddrTreeMap}; + use lance_select::RowAddrTreeMap; use crate::scalar::{ SargableQuery, ScalarIndex, SearchResult, lance_format::LanceIndexStore, zonemap::{ ZONEMAP_FILENAME, ZONEMAP_SIZE_META_KEY, ZoneMapIndex, ZoneMapIndexBuilderParams, + merge_zonemap_indices, }, }; @@ -1197,7 +1857,7 @@ mod tests { .await .unwrap(); - ZoneMapIndex::load(test_store.clone(), None, &LanceCache::no_cache()) + ZoneMapIndex::load(test_store.clone(), None, &LanceCache::no_cache(), false) .await .expect("Failed to load ZoneMapIndex") } @@ -1302,7 +1962,7 @@ mod tests { log::debug!("Successfully wrote the index file"); // Read the index file back and check its contents - let index = ZoneMapIndex::load(test_store.clone(), None, &LanceCache::no_cache()) + let index = ZoneMapIndex::load(test_store.clone(), None, &LanceCache::no_cache(), false) .await .expect("Failed to load ZoneMapIndex"); assert_eq!(index.zones.len(), 0); @@ -1346,7 +2006,7 @@ mod tests { log::debug!("Successfully wrote the index file"); // Read the index file back and check its contents - let index = ZoneMapIndex::load(test_store.clone(), None, &LanceCache::no_cache()) + let index = ZoneMapIndex::load(test_store.clone(), None, &LanceCache::no_cache(), false) .await .expect("Failed to load ZoneMapIndex"); assert_eq!(index.zones.len(), 10); @@ -1398,9 +2058,10 @@ mod tests { .unwrap(); // Verify the updated index has more zones - let updated_index = ZoneMapIndex::load(test_store.clone(), None, &LanceCache::no_cache()) - .await - .expect("Failed to load updated ZoneMapIndex"); + let updated_index = + ZoneMapIndex::load(test_store.clone(), None, &LanceCache::no_cache(), false) + .await + .expect("Failed to load updated ZoneMapIndex"); // Should have original 10 zones + 1 new zone (5000 rows with zone size 5000) assert_eq!(updated_index.zones.len(), 11); @@ -1469,7 +2130,7 @@ mod tests { .unwrap(); let cache = LanceCache::with_capacity(1024 * 1024); - let index = ZoneMapIndex::load(store.clone(), None, &cache) + let index = ZoneMapIndex::load(store.clone(), None, &cache, false) .await .unwrap(); @@ -1572,7 +2233,7 @@ mod tests { .unwrap(); // Load the index - let index = ZoneMapIndex::load(test_store.clone(), None, &LanceCache::no_cache()) + let index = ZoneMapIndex::load(test_store.clone(), None, &LanceCache::no_cache(), false) .await .expect("Failed to load ZoneMapIndex"); @@ -1676,7 +2337,7 @@ mod tests { // Test IsNull query (should match nothing since there are no null values) let query = SargableQuery::IsNull(); let result = index.search(&query, &NoOpMetricsCollector).await.unwrap(); - assert_eq!(result, SearchResult::AtMost(NullableRowAddrSet::empty())); + assert_eq!(result, SearchResult::exact(RowAddrTreeMap::new())); // Test range queries with NaN bounds // Range with NaN as start bound (included) @@ -1719,7 +2380,7 @@ mod tests { ); let result = index.search(&query, &NoOpMetricsCollector).await.unwrap(); // Should match nothing since nothing is greater than NaN - assert_eq!(result, SearchResult::AtMost(NullableRowAddrSet::empty())); + assert_eq!(result, SearchResult::at_most(RowAddrTreeMap::new())); // Test IsIn query with mixed float types (Float16, Float32, Float64) let query = SargableQuery::IsIn(vec![ @@ -1807,7 +2468,7 @@ mod tests { assert_eq!(metadata.get(ZONEMAP_SIZE_META_KEY).unwrap(), "100"); // Read the index file back and check its contents - let index = ZoneMapIndex::load(test_store.clone(), None, &LanceCache::no_cache()) + let index = ZoneMapIndex::load(test_store.clone(), None, &LanceCache::no_cache(), false) .await .expect("Failed to load ZoneMapIndex"); assert_eq!(index.zones.len(), 2); @@ -1903,7 +2564,7 @@ mod tests { // 8. IsNull query (no nulls in data, should match nothing) let query = SargableQuery::IsNull(); let result = index.search(&query, &NoOpMetricsCollector).await.unwrap(); - assert_eq!(result, SearchResult::at_most(RowAddrTreeMap::new())); + assert_eq!(result, SearchResult::exact(RowAddrTreeMap::new())); // 9. IsIn query: [0, 100, 101, 50] let query = SargableQuery::IsIn(vec![ ScalarValue::Int32(Some(0)), @@ -1970,7 +2631,7 @@ mod tests { log::debug!("Successfully wrote the index file"); // Read the index file back and check its contents - let index = ZoneMapIndex::load(test_store.clone(), None, &LanceCache::no_cache()) + let index = ZoneMapIndex::load(test_store.clone(), None, &LanceCache::no_cache(), false) .await .expect("Failed to load ZoneMapIndex"); assert_eq!(index.zones.len(), 3); @@ -2130,9 +2791,10 @@ mod tests { .unwrap(); // Read the index file back and check its contents - let index = ZoneMapIndex::load(test_store.clone(), None, &LanceCache::no_cache()) - .await - .expect("Failed to load ZoneMapIndex"); + let index = + ZoneMapIndex::load(test_store.clone(), None, &LanceCache::no_cache(), false) + .await + .expect("Failed to load ZoneMapIndex"); assert_eq!(index.zones.len(), 5); assert_eq!( index.zones, @@ -2338,9 +3000,10 @@ mod tests { .unwrap(); // Read the index file back and check its contents - let index = ZoneMapIndex::load(test_store.clone(), None, &LanceCache::no_cache()) - .await - .expect("Failed to load ZoneMapIndex"); + let index = + ZoneMapIndex::load(test_store.clone(), None, &LanceCache::no_cache(), false) + .await + .expect("Failed to load ZoneMapIndex"); assert_eq!(index.zones.len(), 3); assert_eq!( index.zones, @@ -2413,9 +3076,10 @@ mod tests { .unwrap(); // Read the index file back and check its contents - let index = ZoneMapIndex::load(test_store.clone(), None, &LanceCache::no_cache()) - .await - .expect("Failed to load ZoneMapIndex"); + let index = + ZoneMapIndex::load(test_store.clone(), None, &LanceCache::no_cache(), false) + .await + .expect("Failed to load ZoneMapIndex"); assert_eq!(index.zones.len(), 3); assert_eq!( index.zones, @@ -2609,9 +3273,11 @@ mod tests { zones, data_type: DataType::Utf8, rows_per_zone: ROWS_PER_ZONE_DEFAULT, + use_seeds: false, store: test_store, fri: None, index_cache: WeakLanceCache::from(&LanceCache::no_cache()), + null_rows: None, }; // Test LikePrefix query for "foo" @@ -2681,9 +3347,11 @@ mod tests { zones, data_type: DataType::Utf8, rows_per_zone: ROWS_PER_ZONE_DEFAULT, + use_seeds: false, store: test_store, fri: None, index_cache: WeakLanceCache::from(&LanceCache::no_cache()), + null_rows: None, }; // Test LikePrefix "test" @@ -2748,9 +3416,11 @@ mod tests { zones, data_type: DataType::LargeUtf8, rows_per_zone: ROWS_PER_ZONE_DEFAULT, + use_seeds: false, store: test_store, fri: None, index_cache: WeakLanceCache::from(&LanceCache::no_cache()), + null_rows: None, }; // Test LikePrefix with LargeUtf8 @@ -2803,4 +3473,1143 @@ mod tests { // All max characters assert_eq!(compute_next_prefix("\u{10FFFF}\u{10FFFF}"), None); } + + // When merging zone map segments, if ANY source segment has null_rows = None + // (legacy — null positions unknown), the merged result must also be None. + // The bug: any_null_bitmap is set to true as soon as one source has Some(...), + // and the None sources are silently skipped. The merged index then has + // null_rows = Some(partial_bitmap), so an IsNull search returns exact results + // that only cover the modern segment's nulls — a false negative for the legacy + // segment whose null positions were never tracked. + #[tokio::test] + async fn test_merge_with_legacy_none_segment_not_treated_as_no_nulls() { + let tmpdir = TempObjDir::default(); + let store = Arc::new(LanceIndexStore::new( + Arc::new(ObjectStore::local()), + tmpdir.clone(), + Arc::new(LanceCache::no_cache()), + )); + + use arrow_array::{Int32Array, UInt32Array}; + + // Index A: fragment 0, modern — has a complete null bitmap with 2 known null rows. + let schema_a = Arc::new(Schema::new(vec![ + Field::new("min", DataType::Int32, true), + Field::new("max", DataType::Int32, true), + Field::new("null_count", DataType::UInt32, false), + Field::new("nan_count", DataType::UInt32, false), + Field::new("fragment_id", DataType::UInt64, false), + Field::new("zone_start", DataType::UInt64, false), + Field::new("zone_length", DataType::UInt64, false), + ])); + let batch_a = RecordBatch::try_new( + schema_a, + vec![ + Arc::new(Int32Array::from(vec![Some(1i32)])) as _, + Arc::new(Int32Array::from(vec![Some(5i32)])) as _, + Arc::new(UInt32Array::from(vec![2u32])) as _, + Arc::new(UInt32Array::from(vec![0u32])) as _, + Arc::new(UInt64Array::from(vec![0u64])) as _, + Arc::new(UInt64Array::from(vec![0u64])) as _, + Arc::new(UInt64Array::from(vec![10u64])) as _, + ], + ) + .unwrap(); + let mut modern_null_rows = RowAddrTreeMap::new(); + modern_null_rows.insert(3); // frag 0 row 3 + modern_null_rows.insert(7); // frag 0 row 7 + let cache = LanceCache::no_cache(); + let index_a = Arc::new( + ZoneMapIndex::try_from_serialized( + batch_a, + store.clone(), + None, + &cache, + 10, + Some(modern_null_rows), // modern: complete bitmap + false, + ) + .unwrap(), + ); + + // Index B: fragment 1, legacy — null_rows = None despite null_count = 3. + let schema_b = Arc::new(Schema::new(vec![ + Field::new("min", DataType::Int32, true), + Field::new("max", DataType::Int32, true), + Field::new("null_count", DataType::UInt32, false), + Field::new("nan_count", DataType::UInt32, false), + Field::new("fragment_id", DataType::UInt64, false), + Field::new("zone_start", DataType::UInt64, false), + Field::new("zone_length", DataType::UInt64, false), + ])); + let batch_b = RecordBatch::try_new( + schema_b, + vec![ + Arc::new(Int32Array::from(vec![Some(10i32)])) as _, + Arc::new(Int32Array::from(vec![Some(20i32)])) as _, + Arc::new(UInt32Array::from(vec![3u32])) as _, + Arc::new(UInt32Array::from(vec![0u32])) as _, + Arc::new(UInt64Array::from(vec![1u64])) as _, + Arc::new(UInt64Array::from(vec![0u64])) as _, + Arc::new(UInt64Array::from(vec![10u64])) as _, + ], + ) + .unwrap(); + let index_b = Arc::new( + ZoneMapIndex::try_from_serialized( + batch_b, + store.clone(), + None, + &cache, + 10, + None, // legacy: null positions unknown + false, + ) + .unwrap(), + ); + + let dest_tmpdir = TempObjDir::default(); + let dest_store = Arc::new(LanceIndexStore::new( + Arc::new(ObjectStore::local()), + dest_tmpdir.clone(), + Arc::new(LanceCache::no_cache()), + )); + + let all_frags = RoaringBitmap::from_iter([0u32, 1]); + merge_zonemap_indices( + &[index_a.as_ref(), index_b.as_ref()], + dest_store.as_ref(), + &all_frags, + ) + .await + .unwrap(); + + let merged = ZoneMapIndex::load(dest_store.clone(), None, &LanceCache::no_cache(), false) + .await + .unwrap(); + + // Index B had null_rows = None, so the merged index cannot know all null positions. + // IsNull must NOT return exact — that would be a false negative for fragment 1's nulls. + let result = merged + .search(&SargableQuery::IsNull(), &NoOpMetricsCollector) + .await + .unwrap(); + + // With the bug: any_null_bitmap=true (from A) → null_rows=Some(A's bitmap only) + // → IsNull returns exact, missing B's unknown nulls ← FALSE NEGATIVE + // With the fix: any_null_bitmap=false (because B is None) → null_rows=None + // → IsNull falls through to zone scan → AtMost + assert!( + !result.is_exact(), + "IsNull on a merged index where one source had null_rows=None must not return \ + exact; the legacy segment had null_count=3 so its nulls exist at unknown positions" + ); + } + + // Writes a zonemap file in the legacy format (no null bitmap global buffer), + // simulating an index created before the null bitmap feature was added. + async fn write_legacy_zonemap(store: &dyn IndexStore, null_count: u32) { + use arrow_array::{Int32Array, UInt32Array}; + let schema = Arc::new(Schema::new(vec![ + Field::new("min", DataType::Int32, true), + Field::new("max", DataType::Int32, true), + Field::new("null_count", DataType::UInt32, false), + Field::new("nan_count", DataType::UInt32, false), + Field::new("fragment_id", DataType::UInt64, false), + Field::new("zone_start", DataType::UInt64, false), + Field::new("zone_length", DataType::UInt64, false), + ])); + let batch = RecordBatch::try_new( + schema.clone(), + vec![ + Arc::new(Int32Array::from(vec![Some(0)])) as _, + Arc::new(Int32Array::from(vec![Some(99)])) as _, + Arc::new(UInt32Array::from(vec![null_count])) as _, + Arc::new(UInt32Array::from(vec![0u32])) as _, + Arc::new(UInt64Array::from(vec![0u64])) as _, + Arc::new(UInt64Array::from(vec![0u64])) as _, + Arc::new(UInt64Array::from(vec![100u64])) as _, + ], + ) + .unwrap(); + let mut file_schema = schema.as_ref().clone(); + file_schema + .metadata + .insert(ZONEMAP_SIZE_META_KEY.to_string(), "8192".to_string()); + let mut writer = store + .new_index_file(ZONEMAP_FILENAME, Arc::new(file_schema)) + .await + .unwrap(); + writer.write_record_batch(batch).await.unwrap(); + writer.finish().await.unwrap(); + } + + #[tokio::test] + async fn test_legacy_zonemap_no_null_bitmap() { + let tmpdir = TempObjDir::default(); + let store = Arc::new(LanceIndexStore::new( + Arc::new(ObjectStore::local()), + tmpdir.clone(), + Arc::new(LanceCache::no_cache()), + )); + + // Write a legacy index with one zone that has nulls but no null bitmap. + write_legacy_zonemap(store.as_ref(), 10).await; + + let index = ZoneMapIndex::load(store, None, &LanceCache::no_cache(), false) + .await + .expect("failed to load legacy zonemap"); + + assert!( + index.null_rows.is_none(), + "legacy index should have no null bitmap" + ); + + // IS NULL should fall back to the zone-scan path and return AtMost, not Exact. + let result = index + .search(&SargableQuery::IsNull(), &NoOpMetricsCollector) + .await + .unwrap(); + assert!( + !result.is_exact(), + "IS NULL on a legacy index should not be exact" + ); + } + + #[tokio::test] + async fn test_zone_map_seed_writer_round_trip() { + use crate::scalar::seed::IndexSeedWriter; + use crate::scalar::zonemap::ZoneMapSeedWriter; + use arrow_array::{ArrayRef, Int32Array}; + use datafusion_common::ScalarValue; + + let rows_per_zone = 4u64; + let data_type = DataType::Int32; + let mut writer = ZoneMapSeedWriter::new("test_col", rows_per_zone, data_type).unwrap(); + + // Batch 1: values 0..4 (fills exactly one zone) + let batch1: ArrayRef = Arc::new(Int32Array::from_iter_values(0..4)); + writer.observe_batch(&batch1).unwrap(); + + // Batch 2: values 10..14 (fills a second zone exactly) + let batch2: ArrayRef = Arc::new(Int32Array::from_iter_values(10..14)); + writer.observe_batch(&batch2).unwrap(); + + // Batch 3: values 20..22 (partial final zone) + let batch3: ArrayRef = Arc::new(Int32Array::from_iter_values(20..22)); + writer.observe_batch(&batch3).unwrap(); + + let bytes = writer.finish().unwrap().expect("should produce bytes"); + + // Check schema metadata key/value format + assert_eq!(writer.schema_metadata_key(), "lance.seed.test_col"); + let meta_val = writer.schema_metadata_value(3); + assert_eq!(meta_val, "3:4"); + + // Deserialize and verify + let (zones, null_bitmap) = + ZoneMapSeedWriter::deserialize_seed(42, &bytes, rows_per_zone).unwrap(); + assert_eq!(zones.len(), 3, "expected 3 zones"); + + // No nulls in the input, so null bitmap is empty + assert_eq!(null_bitmap, Some(RoaringBitmap::default())); + + // Zone 0: values 0..4 -> min=0, max=3 + assert_eq!(zones[0].bound.fragment_id, 42); + assert_eq!(zones[0].bound.start, 0); + assert_eq!(zones[0].min, ScalarValue::Int32(Some(0))); + assert_eq!(zones[0].max, ScalarValue::Int32(Some(3))); + assert_eq!(zones[0].null_count, 0); + + // Zone 1: values 10..14 -> min=10, max=13 + assert_eq!(zones[1].bound.start, 4); + assert_eq!(zones[1].min, ScalarValue::Int32(Some(10))); + assert_eq!(zones[1].max, ScalarValue::Int32(Some(13))); + + // Zone 2: values 20..22 -> min=20, max=21, partial zone of 2 rows + assert_eq!(zones[2].bound.start, 8); + assert_eq!( + zones[2].bound.length, 2, + "partial zone length must be exact" + ); + assert_eq!(zones[2].min, ScalarValue::Int32(Some(20))); + assert_eq!(zones[2].max, ScalarValue::Int32(Some(21))); + + // Full zones must have the full rows_per_zone length + assert_eq!(zones[0].bound.length, rows_per_zone as usize); + assert_eq!(zones[1].bound.length, rows_per_zone as usize); + } + + #[tokio::test] + async fn test_zone_map_seed_writer_spanning_batches() { + use crate::scalar::seed::IndexSeedWriter; + use crate::scalar::zonemap::ZoneMapSeedWriter; + use arrow_array::{ArrayRef, Int32Array}; + use datafusion_common::ScalarValue; + + let rows_per_zone = 5u64; + let data_type = DataType::Int32; + let mut writer = ZoneMapSeedWriter::new("val", rows_per_zone, data_type).unwrap(); + + // Single batch with 12 values -> should produce 2 complete zones + 1 partial + let batch: ArrayRef = Arc::new(Int32Array::from_iter_values(0..12)); + writer.observe_batch(&batch).unwrap(); + + let bytes = writer.finish().unwrap().expect("should produce bytes"); + let (zones, _null_bitmap) = + ZoneMapSeedWriter::deserialize_seed(1, &bytes, rows_per_zone).unwrap(); + assert_eq!( + zones.len(), + 3, + "expected 3 zones from 12 rows with zone size 5" + ); + + // Zone 0: rows 0..5 + assert_eq!(zones[0].bound.length, 5); + assert_eq!(zones[0].min, ScalarValue::Int32(Some(0))); + assert_eq!(zones[0].max, ScalarValue::Int32(Some(4))); + // Zone 1: rows 5..10 + assert_eq!(zones[1].bound.length, 5); + assert_eq!(zones[1].min, ScalarValue::Int32(Some(5))); + assert_eq!(zones[1].max, ScalarValue::Int32(Some(9))); + // Zone 2: rows 10..12 (partial) + assert_eq!(zones[2].bound.length, 2, "partial zone length must be 2"); + assert_eq!(zones[2].min, ScalarValue::Int32(Some(10))); + assert_eq!(zones[2].max, ScalarValue::Int32(Some(11))); + } + + #[tokio::test] + async fn test_zone_map_seed_writer_empty() { + use crate::scalar::seed::IndexSeedWriter; + use crate::scalar::zonemap::ZoneMapSeedWriter; + + let mut writer = ZoneMapSeedWriter::new("col", 8, DataType::Int32).unwrap(); + let result = writer.finish().unwrap(); + assert!(result.is_none(), "empty fragment should return None"); + } + + /// Seeds produced for fragments that contain null values must carry a null + /// bitmap so that `try_update_with_seeds` can reconstruct `null_rows` without + /// rescanning data. + #[tokio::test] + async fn test_seed_null_bitmap_round_trip() { + use crate::scalar::seed::IndexSeedWriter; + use crate::scalar::zonemap::ZoneMapSeedWriter; + use arrow_array::Int32Array; + use lance_select::RowSetOps; + + // rows_per_zone = 4, 10 rows: nulls at positions 1, 3, 5, 9 + // zone 0: rows 0-3 → nulls at 1, 3 + // zone 1: rows 4-7 → null at 5 + // zone 2: rows 8-9 → null at 9 + let rows_per_zone = 4u64; + let values: ArrayRef = Arc::new(Int32Array::from(vec![ + Some(0), + None, // offset 1 + Some(2), + None, // offset 3 + Some(4), + None, // offset 5 + Some(6), + Some(7), + Some(8), + None, // offset 9 + ])); + + let mut writer = ZoneMapSeedWriter::new("col", rows_per_zone, DataType::Int32).unwrap(); + writer.observe_batch(&values).unwrap(); + let bytes = writer.finish().unwrap().expect("should produce bytes"); + + let (zones, null_bitmap) = + ZoneMapSeedWriter::deserialize_seed(7, &bytes, rows_per_zone).unwrap(); + assert_eq!(zones.len(), 3); + assert_eq!(zones[0].null_count, 2); + assert_eq!(zones[1].null_count, 1); + assert_eq!(zones[2].null_count, 1); + + let bitmap = null_bitmap.expect("null bitmap must be present"); + assert!(bitmap.contains(1), "offset 1 must be in bitmap"); + assert!(bitmap.contains(3), "offset 3 must be in bitmap"); + assert!(bitmap.contains(5), "offset 5 must be in bitmap"); + assert!(bitmap.contains(9), "offset 9 must be in bitmap"); + assert_eq!(bitmap.len(), 4, "exactly 4 null positions"); + + // Reconstruct a RowAddrTreeMap as try_update_with_seeds would do. + let mut null_rows = RowAddrTreeMap::new(); + null_rows.insert_bitmap(7, bitmap); + // Verify the row addresses are (fragment 7 << 32 | offset). + let frag7 = 7u64 << 32; + assert!(null_rows.contains(frag7 | 1)); + assert!(null_rows.contains(frag7 | 3)); + assert!(null_rows.contains(frag7 | 5)); + assert!(null_rows.contains(frag7 | 9)); + assert!(!null_rows.contains(frag7)); + } + + /// After updating a zone map index via seeds from a fragment that contains + /// nulls, IS NULL must return an exact result (not a zone-scan approximation). + #[tokio::test] + async fn test_update_with_seeds_propagates_null_bitmap() { + use crate::scalar::seed::{FragmentSeed, IndexSeedWriter}; + use crate::scalar::zonemap::{ZoneMapIndexBuilder, ZoneMapSeedWriter}; + use arrow_array::Int32Array; + use lance_select::RowSetOps; + + // Build an initial zone map index with one fragment that has nulls at rows 0 and 2. + let tmpdir = TempObjDir::default(); + let store = Arc::new(LanceIndexStore::new( + Arc::new(ObjectStore::local()), + tmpdir.clone(), + Arc::new(LanceCache::no_cache()), + )); + + // Initial training: fragment 0, rows 0-3, nulls at 0 and 2. + let rows_per_zone = 4u64; + let initial_values: ArrayRef = Arc::new(Int32Array::from(vec![ + None, // frag 0, row 0 + Some(1), + None, // frag 0, row 2 + Some(3), + ])); + let mut builder = ZoneMapIndexBuilder::try_new( + ZoneMapIndexBuilderParams::new(rows_per_zone), + DataType::Int32, + ) + .unwrap(); + let row_addrs: ArrayRef = Arc::new(UInt64Array::from(vec![0u64, 1, 2, 3])); + let schema = Arc::new(arrow_schema::Schema::new(vec![ + Field::new("value", DataType::Int32, true), + Field::new(ROW_ADDR, DataType::UInt64, false), + ])); + let batch = RecordBatch::try_new(schema.clone(), vec![initial_values, row_addrs]).unwrap(); + let stream = Box::pin(RecordBatchStreamAdapter::new( + schema.clone(), + futures::stream::once(async { Ok(batch) }), + )); + builder.train(stream).await.unwrap(); + builder.write_index(store.as_ref()).await.unwrap(); + + // Load the trained index. + let index = ZoneMapIndex::load(store.clone(), None, &LanceCache::no_cache(), true) + .await + .unwrap(); + assert!( + index.null_rows.is_some(), + "initial index must have null bitmap" + ); + + // Build a seed for fragment 1: rows 0-3, null at row 1. + let seed_values: ArrayRef = Arc::new(Int32Array::from(vec![ + Some(10), + None, // frag 1, row 1 + Some(12), + Some(13), + ])); + let mut seed_writer = + ZoneMapSeedWriter::new("value", rows_per_zone, DataType::Int32).unwrap(); + seed_writer.observe_batch(&seed_values).unwrap(); + let seed_bytes = seed_writer + .finish() + .unwrap() + .expect("seed must produce bytes"); + + let seed = FragmentSeed { + fragment_id: 1, + bytes: seed_bytes, + metadata_value: format!("0:{}", rows_per_zone), + }; + + // Update the index with the seed. + let dest_tmpdir = TempObjDir::default(); + let dest_store = Arc::new(LanceIndexStore::new( + Arc::new(ObjectStore::local()), + dest_tmpdir.clone(), + Arc::new(LanceCache::no_cache()), + )); + index + .try_update_with_seeds(&[seed], dest_store.as_ref()) + .await + .unwrap() + .expect("update must produce a result"); + + let updated_index = ZoneMapIndex::load(dest_store, None, &LanceCache::no_cache(), true) + .await + .unwrap(); + + // The updated index must have a null bitmap covering both fragments. + let null_rows = updated_index + .null_rows + .as_ref() + .expect("updated index must have null bitmap"); + + // Fragment 0: nulls at offsets 0 and 2 + assert!(null_rows.contains(0u64), "frag 0 row 0 must be null"); + assert!(null_rows.contains(2u64), "frag 0 row 2 must be null"); + assert!(!null_rows.contains(1u64), "frag 0 row 1 must not be null"); + + // Fragment 1: null at offset 1 + let frag1 = 1u64 << 32; + assert!(null_rows.contains(frag1 | 1), "frag 1 row 1 must be null"); + assert!(!null_rows.contains(frag1), "frag 1 row 0 must not be null"); + + // IS NULL must return exact results (not a zone-scan approximation). + let result = updated_index + .search(&SargableQuery::IsNull(), &NoOpMetricsCollector) + .await + .unwrap(); + assert!(result.is_exact(), "IS NULL after seed update must be exact"); + } + + /// A legacy index (null_rows = None) updated with a seed must still have + /// null_rows = None in the result. The seed cannot retroactively supply null + /// positions for the rows already in the old index, so the updated index must + /// stay conservative and fall back to zone-scan for IS NULL. + #[tokio::test] + async fn test_update_with_seeds_legacy_index_keeps_null_rows_none() { + use crate::scalar::seed::{FragmentSeed, IndexSeedWriter}; + use crate::scalar::zonemap::{ZoneMapIndex, ZoneMapSeedWriter}; + use arrow_array::{Int32Array, UInt32Array}; + + let tmpdir = TempObjDir::default(); + let store = Arc::new(LanceIndexStore::new( + Arc::new(ObjectStore::local()), + tmpdir.clone(), + Arc::new(LanceCache::no_cache()), + )); + + // Build a legacy index (fragment 0, null_rows = None). + let schema = Arc::new(Schema::new(vec![ + Field::new("min", DataType::Int32, true), + Field::new("max", DataType::Int32, true), + Field::new("null_count", DataType::UInt32, false), + Field::new("nan_count", DataType::UInt32, false), + Field::new("fragment_id", DataType::UInt64, false), + Field::new("zone_start", DataType::UInt64, false), + Field::new("zone_length", DataType::UInt64, false), + ])); + let batch = RecordBatch::try_new( + schema, + vec![ + Arc::new(Int32Array::from(vec![Some(0i32)])) as _, + Arc::new(Int32Array::from(vec![Some(9i32)])) as _, + Arc::new(UInt32Array::from(vec![2u32])) as _, // 2 nulls, positions unknown + Arc::new(UInt32Array::from(vec![0u32])) as _, + Arc::new(UInt64Array::from(vec![0u64])) as _, + Arc::new(UInt64Array::from(vec![0u64])) as _, + Arc::new(UInt64Array::from(vec![10u64])) as _, + ], + ) + .unwrap(); + let cache = LanceCache::no_cache(); + let legacy_index = Arc::new( + ZoneMapIndex::try_from_serialized( + batch, + store.clone(), + None, + &cache, + 10, + None, // legacy: null positions unknown + true, + ) + .unwrap(), + ); + assert!(legacy_index.null_rows.is_none()); + + // Build a seed for fragment 1 with known null positions. + let rows_per_zone = 10u64; + let seed_values: ArrayRef = Arc::new(Int32Array::from(vec![ + None, // frag 1, row 0 + Some(1), + ])); + let mut seed_writer = + ZoneMapSeedWriter::new("value", rows_per_zone, DataType::Int32).unwrap(); + seed_writer.observe_batch(&seed_values).unwrap(); + let seed_bytes = seed_writer + .finish() + .unwrap() + .expect("seed must produce bytes"); + + let dest_tmpdir = TempObjDir::default(); + let dest_store = Arc::new(LanceIndexStore::new( + Arc::new(ObjectStore::local()), + dest_tmpdir.clone(), + Arc::new(LanceCache::no_cache()), + )); + legacy_index + .try_update_with_seeds( + &[FragmentSeed { + fragment_id: 1, + bytes: seed_bytes, + metadata_value: format!("0:{}", rows_per_zone), + }], + dest_store.as_ref(), + ) + .await + .unwrap() + .expect("update must produce a result"); + + let updated = ZoneMapIndex::load(dest_store, None, &LanceCache::no_cache(), true) + .await + .unwrap(); + + assert!( + updated.null_rows.is_none(), + "updating a legacy index must not fabricate a null bitmap: \ + the old fragment's null positions were never tracked" + ); + + let result = updated + .search(&SargableQuery::IsNull(), &NoOpMetricsCollector) + .await + .unwrap(); + assert!( + !result.is_exact(), + "IS NULL on an index derived from a legacy source must use zone scan, not exact lookup" + ); + } + + /// Merging a modern index (null_rows = Some) with a legacy index (null_rows = None) + /// must produce a merged index with null_rows = None. Returning the modern + /// fragment's bitmap as-is would be a false negative for the legacy fragment. + #[tokio::test] + async fn test_merge_modern_and_legacy_index_drops_null_bitmap() { + use arrow_array::{Int32Array, UInt32Array}; + + let tmpdir = TempObjDir::default(); + let store = Arc::new(LanceIndexStore::new( + Arc::new(ObjectStore::local()), + tmpdir.clone(), + Arc::new(LanceCache::no_cache()), + )); + let cache = LanceCache::no_cache(); + + // Modern index: fragment 0, null_rows = Some(bitmap with 1 known null). + let schema = Arc::new(Schema::new(vec![ + Field::new("min", DataType::Int32, true), + Field::new("max", DataType::Int32, true), + Field::new("null_count", DataType::UInt32, false), + Field::new("nan_count", DataType::UInt32, false), + Field::new("fragment_id", DataType::UInt64, false), + Field::new("zone_start", DataType::UInt64, false), + Field::new("zone_length", DataType::UInt64, false), + ])); + let modern_batch = RecordBatch::try_new( + schema.clone(), + vec![ + Arc::new(Int32Array::from(vec![Some(1i32)])) as _, + Arc::new(Int32Array::from(vec![Some(5i32)])) as _, + Arc::new(UInt32Array::from(vec![1u32])) as _, + Arc::new(UInt32Array::from(vec![0u32])) as _, + Arc::new(UInt64Array::from(vec![0u64])) as _, + Arc::new(UInt64Array::from(vec![0u64])) as _, + Arc::new(UInt64Array::from(vec![10u64])) as _, + ], + ) + .unwrap(); + let mut modern_null_rows = RowAddrTreeMap::new(); + modern_null_rows.insert(2u64); // frag 0, row 2 + let modern_index = Arc::new( + ZoneMapIndex::try_from_serialized( + modern_batch, + store.clone(), + None, + &cache, + 10, + Some(modern_null_rows), + false, + ) + .unwrap(), + ); + + // Legacy index: fragment 1, null_rows = None. + let legacy_batch = RecordBatch::try_new( + schema, + vec![ + Arc::new(Int32Array::from(vec![Some(10i32)])) as _, + Arc::new(Int32Array::from(vec![Some(20i32)])) as _, + Arc::new(UInt32Array::from(vec![3u32])) as _, // 3 nulls, positions unknown + Arc::new(UInt32Array::from(vec![0u32])) as _, + Arc::new(UInt64Array::from(vec![1u64])) as _, + Arc::new(UInt64Array::from(vec![0u64])) as _, + Arc::new(UInt64Array::from(vec![10u64])) as _, + ], + ) + .unwrap(); + let legacy_index = Arc::new( + ZoneMapIndex::try_from_serialized( + legacy_batch, + store.clone(), + None, + &cache, + 10, + None, // legacy: null positions unknown + false, + ) + .unwrap(), + ); + + let dest_tmpdir = TempObjDir::default(); + let dest_store = Arc::new(LanceIndexStore::new( + Arc::new(ObjectStore::local()), + dest_tmpdir.clone(), + Arc::new(LanceCache::no_cache()), + )); + let all_frags = RoaringBitmap::from_iter([0u32, 1]); + merge_zonemap_indices( + &[modern_index.as_ref(), legacy_index.as_ref()], + dest_store.as_ref(), + &all_frags, + ) + .await + .unwrap(); + + let merged = ZoneMapIndex::load(dest_store, None, &LanceCache::no_cache(), false) + .await + .unwrap(); + + assert!( + merged.null_rows.is_none(), + "merging a modern index with a legacy index must drop the null bitmap: \ + the legacy fragment's null positions are unknown" + ); + + let result = merged + .search(&SargableQuery::IsNull(), &NoOpMetricsCollector) + .await + .unwrap(); + assert!( + !result.is_exact(), + "IS NULL on a merged index with a legacy source must use zone scan, not exact lookup" + ); + } + + // ───────────────────────────── Nested type zone map tests ───────────────────────────── + + use arrow_array::{ArrayRef, FixedSizeListArray, Float32Array}; + + /// Build a FixedSizeList zone map from `rows` (each row is a + /// `Vec>` of length `list_size`), then load and return the index. + async fn train_and_load_fsl(rows: Vec>>, list_size: i32) -> Arc { + let item_field = Arc::new(Field::new("item", DataType::Float32, true)); + let fsl_type = DataType::FixedSizeList(item_field.clone(), list_size); + let schema = Arc::new(Schema::new(vec![ + Field::new(VALUE_COLUMN_NAME, fsl_type.clone(), true), + Field::new(ROW_ADDR, DataType::UInt64, false), + ])); + + let mut fsl_builder = + arrow_array::builder::FixedSizeListBuilder::new(Float32Builder::new(), list_size); + for row in &rows { + assert_eq!(row.len(), list_size as usize); + for &v in row { + match v { + Some(f) => fsl_builder.values().append_value(f), + None => fsl_builder.values().append_null(), + } + } + fsl_builder.append(true); + } + let fsl_arr: ArrayRef = Arc::new(fsl_builder.finish()); + let n = rows.len() as u64; + let row_addr: ArrayRef = Arc::new(UInt64Array::from_iter_values(0..n)); + let batch = RecordBatch::try_new(schema.clone(), vec![fsl_arr, row_addr]).unwrap(); + + let tmpdir = TempObjDir::default(); + let test_store = Arc::new(LanceIndexStore::new( + Arc::new(ObjectStore::local()), + tmpdir.clone(), + Arc::new(LanceCache::no_cache()), + )); + let stream: SendableRecordBatchStream = Box::pin(RecordBatchStreamAdapter::new( + schema, + futures::stream::once(std::future::ready(Ok(batch))), + )); + + ZoneMapIndexPlugin::train_zonemap_index( + stream, + test_store.as_ref(), + Some(ZoneMapIndexBuilderParams::new(4)), + ) + .await + .unwrap(); + + ZoneMapIndex::load(test_store.clone(), None, &LanceCache::no_cache(), false) + .await + .expect("failed to load ZoneMapIndex") + } + + use arrow_array::builder::Float32Builder; + use datafusion_common::ScalarValue as SV; + + fn fsl_scalar(values: Vec>) -> SV { + let list_size = values.len() as i32; + let item_field = Arc::new(Field::new("item", DataType::Float32, true)); + let items: ArrayRef = Arc::new(Float32Array::from(values)); + let arr = Arc::new(FixedSizeListArray::new(item_field, list_size, items, None)); + SV::FixedSizeList(arr) + } + + // Zones for nested types have null min/max; all non-null queries are conservative (not pruned) + // unless every row in the zone is null. + #[tokio::test] + async fn test_fsl_zonemap_conservative_equals() { + // Two zones, 4 rows each, no null rows. + let index = train_and_load_fsl( + vec![ + vec![Some(1.0), Some(2.0)], + vec![Some(3.0), Some(4.0)], + vec![Some(5.0), Some(6.0)], + vec![Some(7.0), Some(8.0)], + vec![Some(10.0), Some(20.0)], + vec![Some(30.0), Some(40.0)], + vec![Some(50.0), Some(60.0)], + vec![Some(70.0), Some(80.0)], + ], + 2, + ) + .await; + assert_eq!(index.zones.len(), 2); + // min/max are null for nested types + assert!(index.zones[0].min.is_null()); + assert!(index.zones[0].max.is_null()); + assert_eq!(index.zones[0].null_count, 0); + assert_eq!(index.zones[0].nan_count, 0); + + // Non-null Equals: conservative — neither zone is pruned + let q = SargableQuery::Equals(fsl_scalar(vec![Some(100.0), Some(200.0)])); + assert!( + index + .evaluate_zone_against_query(&index.zones[0], &q) + .unwrap(), + "non-null Equals on nested type is conservative" + ); + assert!( + index + .evaluate_zone_against_query(&index.zones[1], &q) + .unwrap(), + "non-null Equals on nested type is conservative" + ); + + // IsNull: no null rows → both zones pruned + let q_null = SargableQuery::IsNull(); + assert!( + !index + .evaluate_zone_against_query(&index.zones[0], &q_null) + .unwrap(), + "IsNull pruned when null_count=0" + ); + assert!( + !index + .evaluate_zone_against_query(&index.zones[1], &q_null) + .unwrap(), + "IsNull pruned when null_count=0" + ); + } + + #[tokio::test] + async fn test_fsl_zonemap_null_list() { + let item_field = Arc::new(Field::new("item", DataType::Float32, true)); + let fsl_type = DataType::FixedSizeList(item_field.clone(), 2); + let schema = Arc::new(Schema::new(vec![ + Field::new(VALUE_COLUMN_NAME, fsl_type.clone(), true), + Field::new(ROW_ADDR, DataType::UInt64, false), + ])); + + // Build FSL array with some null entries + let mut builder = arrow_array::builder::FixedSizeListBuilder::new(Float32Builder::new(), 2); + // Row 0: null list + builder.values().append_value(0.0); + builder.values().append_value(0.0); + builder.append(false); + // Row 1: [3.0, 4.0] + builder.values().append_value(3.0); + builder.values().append_value(4.0); + builder.append(true); + let fsl_arr: ArrayRef = Arc::new(builder.finish()); + let row_addr: ArrayRef = Arc::new(UInt64Array::from_iter_values(0..2)); + let batch = RecordBatch::try_new(schema.clone(), vec![fsl_arr, row_addr]).unwrap(); + + let tmpdir = TempObjDir::default(); + let test_store = Arc::new(LanceIndexStore::new( + Arc::new(ObjectStore::local()), + tmpdir.clone(), + Arc::new(LanceCache::no_cache()), + )); + let stream: SendableRecordBatchStream = Box::pin(RecordBatchStreamAdapter::new( + schema, + futures::stream::once(std::future::ready(Ok(batch))), + )); + ZoneMapIndexPlugin::train_zonemap_index( + stream, + test_store.as_ref(), + Some(ZoneMapIndexBuilderParams::new(4)), + ) + .await + .unwrap(); + let index = ZoneMapIndex::load(test_store.clone(), None, &LanceCache::no_cache(), false) + .await + .unwrap(); + + assert_eq!(index.zones.len(), 1); + assert_eq!(index.zones[0].null_count, 1, "one null list"); + + // IsNull should match + let q = SargableQuery::IsNull(); + assert!( + index + .evaluate_zone_against_query(&index.zones[0], &q) + .unwrap() + ); + + // Equals null should match (null_count > 0) + let null_scalar = SV::FixedSizeList(Arc::new(FixedSizeListArray::new_null( + item_field.clone(), + 2, + 1, + ))); + let q2 = SargableQuery::Equals(null_scalar); + assert!( + index + .evaluate_zone_against_query(&index.zones[0], &q2) + .unwrap() + ); + } + + #[tokio::test] + async fn test_fsl_zonemap_is_in_conservative() { + // Two zones (4 rows each), no null rows. + let index = train_and_load_fsl( + vec![ + vec![Some(1.0), Some(2.0)], + vec![Some(3.0), Some(4.0)], + vec![Some(5.0), Some(6.0)], + vec![Some(7.0), Some(8.0)], + vec![Some(10.0), Some(20.0)], + vec![Some(30.0), Some(40.0)], + vec![Some(50.0), Some(60.0)], + vec![Some(70.0), Some(80.0)], + ], + 2, + ) + .await; + assert_eq!(index.zones.len(), 2); + + // IsIn with non-null values: conservative — neither zone is pruned + let q = SargableQuery::IsIn(vec![ + fsl_scalar(vec![Some(4.0), Some(5.0)]), + fsl_scalar(vec![Some(100.0), Some(200.0)]), + ]); + assert!( + index + .evaluate_zone_against_query(&index.zones[0], &q) + .unwrap(), + "IsIn with non-null values is conservative for nested types" + ); + assert!( + index + .evaluate_zone_against_query(&index.zones[1], &q) + .unwrap(), + "IsIn with non-null values is conservative for nested types" + ); + } + + #[tokio::test] + async fn test_fsl_zonemap_value_range_is_none() { + let index = train_and_load_fsl( + vec![vec![Some(1.0), Some(2.0)], vec![Some(3.0), Some(4.0)]], + 2, + ) + .await; + assert_eq!(index.value_range(), None, "FSL index has no scalar range"); + } + + #[tokio::test] + async fn test_fsl_zonemap_all_null_zone() { + let item_field = Arc::new(Field::new("item", DataType::Float32, true)); + let fsl_type = DataType::FixedSizeList(item_field.clone(), 2); + let schema = Arc::new(Schema::new(vec![ + Field::new(VALUE_COLUMN_NAME, fsl_type.clone(), true), + Field::new(ROW_ADDR, DataType::UInt64, false), + ])); + let mut builder = arrow_array::builder::FixedSizeListBuilder::new(Float32Builder::new(), 2); + for _ in 0..2 { + builder.values().append_value(0.0); + builder.values().append_value(0.0); + builder.append(false); // null list + } + let fsl_arr: ArrayRef = Arc::new(builder.finish()); + let row_addr: ArrayRef = Arc::new(UInt64Array::from_iter_values(0..2)); + let batch = RecordBatch::try_new(schema.clone(), vec![fsl_arr, row_addr]).unwrap(); + let tmpdir = TempObjDir::default(); + let test_store = Arc::new(LanceIndexStore::new( + Arc::new(ObjectStore::local()), + tmpdir.clone(), + Arc::new(LanceCache::no_cache()), + )); + let stream: SendableRecordBatchStream = Box::pin(RecordBatchStreamAdapter::new( + schema, + futures::stream::once(std::future::ready(Ok(batch))), + )); + ZoneMapIndexPlugin::train_zonemap_index( + stream, + test_store.as_ref(), + Some(ZoneMapIndexBuilderParams::new(4)), + ) + .await + .unwrap(); + let index = ZoneMapIndex::load(test_store.clone(), None, &LanceCache::no_cache(), false) + .await + .unwrap(); + + assert_eq!(index.zones[0].null_count, 2); + assert!(index.zones[0].min.is_null(), "all-null zone: min is null"); + + // When every row is null (null_count == zone length), a non-null query is pruned. + let q = SargableQuery::Equals(fsl_scalar(vec![Some(1.0), Some(2.0)])); + assert!( + !index + .evaluate_zone_against_query(&index.zones[0], &q) + .unwrap(), + "all-null zone (null_count == length) is pruned for non-null target" + ); + + // Range and IsIn are also pruned when all rows are null. + let q_range = SargableQuery::Range(std::ops::Bound::Unbounded, std::ops::Bound::Unbounded); + assert!( + !index + .evaluate_zone_against_query(&index.zones[0], &q_range) + .unwrap(), + "all-null zone is pruned for Range query" + ); + } + + /// Build a zone map on a Struct column and verify training succeeds and stats are correct. + #[tokio::test] + async fn test_struct_zonemap_null_tracking() { + use arrow_array::StructArray; + use arrow_schema::Fields; + + let item_field = Arc::new(Field::new("x", DataType::Int32, true)); + let struct_type = DataType::Struct(Fields::from(vec![item_field.clone()])); + let schema = Arc::new(Schema::new(vec![ + Field::new(VALUE_COLUMN_NAME, struct_type.clone(), true), + Field::new(ROW_ADDR, DataType::UInt64, false), + ])); + + // 3 non-null struct rows + let x_values: ArrayRef = Arc::new(arrow_array::Int32Array::from(vec![10, 20, 30])); + let struct_arr: ArrayRef = + Arc::new(StructArray::from(vec![(item_field.clone(), x_values)])); + let row_addr: ArrayRef = Arc::new(UInt64Array::from_iter_values(0..3)); + let batch = RecordBatch::try_new(schema.clone(), vec![struct_arr, row_addr]).unwrap(); + + let tmpdir = TempObjDir::default(); + let test_store = Arc::new(LanceIndexStore::new( + Arc::new(ObjectStore::local()), + tmpdir.clone(), + Arc::new(LanceCache::no_cache()), + )); + let stream: SendableRecordBatchStream = Box::pin(RecordBatchStreamAdapter::new( + schema, + futures::stream::once(std::future::ready(Ok(batch))), + )); + ZoneMapIndexPlugin::train_zonemap_index( + stream, + test_store.as_ref(), + Some(ZoneMapIndexBuilderParams::new(10)), + ) + .await + .unwrap(); + + let index = ZoneMapIndex::load(test_store.clone(), None, &LanceCache::no_cache(), false) + .await + .unwrap(); + + assert_eq!(index.zones.len(), 1); + assert_eq!(index.zones[0].null_count, 0, "no null struct rows"); + assert_eq!(index.zones[0].nan_count, 0); + // min/max are typed null ScalarValues (exact representation varies by DataFusion version) + // but IsNull correctly returns false and Equals is conservative + + // IsNull: no null rows → zone is pruned + assert!( + !index + .evaluate_zone_against_query(&index.zones[0], &SargableQuery::IsNull()) + .unwrap() + ); + + // Non-null Equals: conservative (not pruned) since not all rows are null + assert!( + index + .evaluate_zone_against_query( + &index.zones[0], + &SargableQuery::Equals(SV::Int32(Some(99))) + ) + .unwrap() + ); + + // value_range: always None for nested types + assert_eq!(index.value_range(), None); + } + + /// Build a zone map on a List column and verify null tracking works. + #[tokio::test] + async fn test_list_zonemap_null_tracking() { + use arrow_array::builder::ListBuilder; + + let item_field = Arc::new(Field::new("item", DataType::Int32, true)); + let list_type = DataType::List(item_field.clone()); + let schema = Arc::new(Schema::new(vec![ + Field::new(VALUE_COLUMN_NAME, list_type.clone(), true), + Field::new(ROW_ADDR, DataType::UInt64, false), + ])); + + // Build: [null, [1,2], [3]] + let mut builder = ListBuilder::new(arrow_array::builder::Int32Builder::new()); + builder.append_null(); + builder.values().append_value(1); + builder.values().append_value(2); + builder.append(true); + builder.values().append_value(3); + builder.append(true); + let list_arr: ArrayRef = Arc::new(builder.finish()); + let row_addr: ArrayRef = Arc::new(UInt64Array::from_iter_values(0..3)); + let batch = RecordBatch::try_new(schema.clone(), vec![list_arr, row_addr]).unwrap(); + + let tmpdir = TempObjDir::default(); + let test_store = Arc::new(LanceIndexStore::new( + Arc::new(ObjectStore::local()), + tmpdir.clone(), + Arc::new(LanceCache::no_cache()), + )); + let stream: SendableRecordBatchStream = Box::pin(RecordBatchStreamAdapter::new( + schema, + futures::stream::once(std::future::ready(Ok(batch))), + )); + ZoneMapIndexPlugin::train_zonemap_index( + stream, + test_store.as_ref(), + Some(ZoneMapIndexBuilderParams::new(10)), + ) + .await + .unwrap(); + + let index = ZoneMapIndex::load(test_store.clone(), None, &LanceCache::no_cache(), false) + .await + .unwrap(); + + assert_eq!(index.zones.len(), 1); + assert_eq!(index.zones[0].null_count, 1, "one null list row"); + // min/max are typed null ScalarValues for nested types + + // IsNull search: the null bitmap is populated during training so + // the result is an exact set containing just the null row (row address 0). + let result = index + .search(&SargableQuery::IsNull(), &NoOpMetricsCollector) + .await + .unwrap(); + let mut exact_nulls = RowAddrTreeMap::new(); + exact_nulls.insert(0); // only row 0 is null + assert_eq!(result, SearchResult::exact(exact_nulls)); + } } diff --git a/rust/lance-index/src/traits.rs b/rust/lance-index/src/traits.rs index 5cb5c830666..f5441bd80f0 100644 --- a/rust/lance-index/src/traits.rs +++ b/rust/lance-index/src/traits.rs @@ -1,10 +1,14 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright The Lance Authors +use std::fmt::Display; + use lance_core::Result; use lance_table::format::IndexMetadata; +use crate::scalar::inverted::DocumentGranularity; + /// A set of criteria used to filter potential indices to use for a query #[derive(Debug, Default)] pub struct IndexCriteria<'a> { @@ -15,6 +19,8 @@ pub struct IndexCriteria<'a> { pub has_name: Option<&'a str>, /// If true, only consider indices that support FTS pub must_support_fts: bool, + /// Logical FTS document boundary. FTS lookups default to row documents. + pub fts_document_granularity: Option, /// If true, only consider indices that support exact equality pub must_support_exact_equality: bool, } @@ -39,6 +45,15 @@ impl<'a> IndexCriteria<'a> { self } + /// Select an FTS index with the requested logical document boundary. + pub fn with_fts_document_granularity( + mut self, + document_granularity: DocumentGranularity, + ) -> Self { + self.fts_document_granularity = Some(document_granularity); + self + } + /// Only consider indices that support exact equality /// /// This will disqualify, for example, the ngram and inverted indices @@ -58,6 +73,9 @@ pub type ScalarIndexCriteria<'a> = IndexCriteria<'a>; pub struct FtsPrewarmOptions { /// If true, prewarm positions along with posting lists. pub with_position: bool, + /// Controls whether prewarm requires the full requested FTS working set to + /// remain resident when the operation completes. + pub mode: FtsPrewarmMode, } impl FtsPrewarmOptions { @@ -69,6 +87,207 @@ impl FtsPrewarmOptions { self.with_position = with_position; self } + + pub fn with_mode(mut self, mode: FtsPrewarmMode) -> Self { + self.mode = mode; + self + } + + pub fn best_effort(mut self) -> Self { + self.mode = FtsPrewarmMode::BestEffort; + self + } +} + +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub enum FtsPrewarmMode { + #[default] + Strict, + BestEffort, +} + +impl FtsPrewarmMode { + pub fn is_best_effort(self) -> bool { + matches!(self, Self::BestEffort) + } +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct FtsPrewarmResult { + pub fully_resident: bool, + pub diagnostics: Option, +} + +impl FtsPrewarmResult { + pub fn fully_resident() -> Self { + Self { + fully_resident: true, + diagnostics: None, + } + } + + pub fn partial(diagnostics: FtsPrewarmDiagnostics) -> Self { + Self { + fully_resident: false, + diagnostics: Some(diagnostics), + } + } +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct FtsPrewarmDiagnostics { + pub partition_count: usize, + pub failing_segments: Vec, + pub failing_partitions: Vec, +} + +impl FtsPrewarmDiagnostics { + pub fn fully_resident(&self) -> bool { + self.failing_segments.is_empty() && self.failing_partitions.is_empty() + } +} + +impl Display for FtsPrewarmDiagnostics { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!( + f, + "FTS prewarm completed without publishing query-ready scalar container, document, and posting state; \ + {} segment(s) and {} of {} partition(s) are not fully resident", + self.failing_segments.len(), + self.failing_partitions.len(), + self.partition_count + )?; + if !self.failing_segments.is_empty() || !self.failing_partitions.is_empty() { + write!(f, ": ")?; + let mut first = true; + for segment in &self.failing_segments { + if !first { + write!(f, "; ")?; + } + first = false; + write!(f, "{segment}")?; + } + for partition in &self.failing_partitions { + if !first { + write!(f, "; ")?; + } + first = false; + write!(f, "{partition}")?; + } + } + write!( + f, + ". Likely cause: index cache pressure or insufficient capacity. \ + Suggested remediation: increase index-cache capacity, reduce the number of FTS \ + segments assigned to each executor, or adjust placement/segment sizing." + ) + } +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct FtsPrewarmSegmentStatus { + pub segment_id: String, + pub scalar_index_container_resident: bool, + pub scalar_index_container_matches_prewarmed: bool, +} + +impl FtsPrewarmSegmentStatus { + pub fn query_ready(&self) -> bool { + self.scalar_index_container_resident && self.scalar_index_container_matches_prewarmed + } +} + +impl Display for FtsPrewarmSegmentStatus { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + let mut missing = Vec::new(); + if !self.scalar_index_container_resident { + missing.push("resident scalar index container"); + } + if !self.scalar_index_container_matches_prewarmed { + missing.push("stable scalar index container identity"); + } + write!( + f, + "segment {} missing {}", + self.segment_id, + missing.join(", ") + ) + } +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct FtsPrewarmPartitionStatus { + pub segment_id: Option, + pub partition_id: u64, + pub documents: FtsPrewarmDocumentStatus, + pub posting_validation_ready: bool, + pub posting_resident: bool, + pub position_resident: Option, +} + +impl FtsPrewarmPartitionStatus { + pub fn query_ready(&self) -> bool { + self.documents.query_ready() + && self.posting_validation_ready + && self.posting_resident + && self.position_resident.unwrap_or(true) + } +} + +impl Display for FtsPrewarmPartitionStatus { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + let mut missing = Vec::new(); + if !self.documents.prewarm_complete { + missing.push("document prewarm completion"); + } + if !self.documents.scoring_ready { + missing.push("scoring lengths/norms"); + } + if !self.documents.reverse_lookup_ready { + missing.push("reverse document lookup"); + } + if !self.documents.projection_resident { + missing.push("resident row-address projection"); + } + if !self.posting_validation_ready { + missing.push("posting validation"); + } + if !self.posting_resident { + missing.push("resident posting lists"); + } + if self.position_resident == Some(false) { + missing.push("resident positions"); + } + let segment = self + .segment_id + .as_deref() + .map(|segment_id| format!("segment {segment_id} ")) + .unwrap_or_default(); + write!( + f, + "{}partition {} missing {}", + segment, + self.partition_id, + missing.join(", ") + ) + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct FtsPrewarmDocumentStatus { + pub prewarm_complete: bool, + pub scoring_ready: bool, + pub reverse_lookup_ready: bool, + pub projection_resident: bool, +} + +impl FtsPrewarmDocumentStatus { + pub fn query_ready(&self) -> bool { + self.prewarm_complete + && self.scoring_ready + && self.reverse_lookup_ready + && self.projection_resident + } } /// Options for prewarming an index. diff --git a/rust/lance-index/src/vector.rs b/rust/lance-index/src/vector.rs index 76b9ccc7a51..a41973f57c8 100644 --- a/rust/lance-index/src/vector.rs +++ b/rust/lance-index/src/vector.rs @@ -85,8 +85,9 @@ pub const DEFAULT_QUERY_PARALLELISM: i32 = 0; /// Controls the speed / accuracy tradeoff for approximate vector search. /// -/// This currently only affects RQ-quantized vector indexes, such as IVF_RQ. -/// Other index types ignore this setting. +/// This currently affects RQ-quantized vector indexes (such as IVF_RQ) and +/// prefiltered search on HNSW sub-indexes, where `Fast` enables the ACORN +/// traversal. Other index types ignore this setting. #[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] pub enum ApproxMode { /// Prefer lower query latency, which can reduce recall. diff --git a/rust/lance-index/src/vector/bq/storage.rs b/rust/lance-index/src/vector/bq/storage.rs index 7bcc2526b43..7fac4a677cd 100644 --- a/rust/lance-index/src/vector/bq/storage.rs +++ b/rust/lance-index/src/vector/bq/storage.rs @@ -22,7 +22,7 @@ use itertools::{Itertools, izip}; use lance_arrow::{ArrowFloatType, FixedSizeListArrayExt, FloatArray, RecordBatchExt}; use lance_core::deepsize::DeepSizeOf; use lance_core::{Error, ROW_ID, Result}; -use lance_file::previous::reader::FileReader as PreviousFileReader; +use lance_file::versions::v1::reader::FileReader as V1FileReader; use lance_linalg::distance::{DistanceType, Dot, dot, l2::l2}; use lance_linalg::simd::{ self, @@ -464,7 +464,7 @@ impl QuantizerMetadata for RabitQuantizationMetadata { } } - async fn load(reader: &PreviousFileReader) -> Result { + async fn load(reader: &V1FileReader) -> Result { let metadata_str = reader .schema() .metadata @@ -519,8 +519,8 @@ impl RabitQuantizationStorage { fn residual_query_factor(&self, dist_q_c: f32) -> f32 { match self.distance_type { - DistanceType::L2 => dist_q_c, - DistanceType::Cosine | DistanceType::Dot => dist_q_c - 1.0, + DistanceType::L2 | DistanceType::Cosine => dist_q_c, + DistanceType::Dot => dist_q_c - 1.0, _ => unimplemented!( "RabitQ does not support distance type: {}", self.distance_type @@ -2545,7 +2545,7 @@ impl QuantizerStorage for RabitQuantizationStorage { } async fn load_partition( - reader: &PreviousFileReader, + reader: &V1FileReader, range: std::ops::Range, distance_type: DistanceType, metadata: &Self::Metadata, @@ -3930,7 +3930,7 @@ mod tests { .into_iter() .map(|(id, dist)| (id as u64, dist)) .collect::>(); - expected.sort_by(|left, right| left.0.cmp(&right.0)); + expected.sort_by_key(|left| left.0); let mut heap = BinaryHeap::with_capacity(k); let mut distances = Vec::new(); @@ -3952,7 +3952,7 @@ mod tests { .into_iter() .map(|node| (node.id, node.dist.0)) .collect::>(); - actual.sort_by(|left, right| left.0.cmp(&right.0)); + actual.sort_by_key(|left| left.0); assert_eq!(actual.len(), expected.len()); for ((actual_id, actual_dist), (expected_id, expected_dist)) in @@ -4298,20 +4298,33 @@ mod tests { } #[test] - fn test_try_from_batch_keeps_cosine_for_legacy_residual_query() { + fn test_residual_query_cosine_uses_l2_query_factor() { let original_codes = make_test_codes(50, 64); let mut metadata = make_test_metadata(original_codes.value_length() as usize * 8); metadata.query_estimator = RabitQueryEstimator::ResidualQuery; + let batch = make_test_batch(original_codes); - let storage = RabitQuantizationStorage::try_from_batch( - make_test_batch(original_codes), + let cosine_storage = RabitQuantizationStorage::try_from_batch( + batch.clone(), &metadata, DistanceType::Cosine, None, ) .unwrap(); + let l2_storage = + RabitQuantizationStorage::try_from_batch(batch, &metadata, DistanceType::L2, None) + .unwrap(); + + assert_eq!(cosine_storage.distance_type(), DistanceType::Cosine); + + let query = Arc::new(Float32Array::from(vec![0.125; 64])) as ArrayRef; + let dist_q_c = 0.25; + let cosine_distances = cosine_storage + .dist_calculator(query.clone(), dist_q_c) + .distance_all(0); + let l2_distances = l2_storage.dist_calculator(query, dist_q_c).distance_all(0); - assert_eq!(storage.distance_type(), DistanceType::Cosine); + assert_eq!(cosine_distances, l2_distances); } #[test] diff --git a/rust/lance-index/src/vector/distributed/index_merger.rs b/rust/lance-index/src/vector/distributed/index_merger.rs index 70371ad4794..938e3e1aeeb 100755 --- a/rust/lance-index/src/vector/distributed/index_merger.rs +++ b/rust/lance-index/src/vector/distributed/index_merger.rs @@ -39,8 +39,9 @@ use crate::{INDEX_AUXILIARY_FILE_NAME, INDEX_METADATA_SCHEMA_KEY}; use arrow_schema::{DataType, Field, Schema as ArrowSchema}; use bytes::Bytes; use lance_core::datatypes::Schema as LanceSchema; -use lance_encoding::version::LanceFileVersion; use lance_file::reader::{FileReader as V2Reader, FileReaderOptions as V2ReaderOptions}; +use lance_file::version::ConcreteFileVersion; +use lance_file::versions; use lance_file::writer::{FileWriter as V2Writer, FileWriter, FileWriterOptions}; use lance_io::scheduler::{ScanScheduler, SchedulerConfig}; use lance_io::utils::CachedFileSize; @@ -91,6 +92,11 @@ fn fixed_size_list_equal(a: &FixedSizeListArray, b: &FixedSizeListArray) -> bool let vb = b.values().as_primitive::(); va.values() == vb.values() } + (DataType::UInt8, DataType::UInt8) => { + let va = a.values().as_primitive::(); + let vb = b.values().as_primitive::(); + va.values() == vb.values() + } _ => false, } } @@ -112,6 +118,9 @@ fn fixed_size_list_almost_equal(a: &FixedSizeListArray, b: &FixedSizeListArray, return false; } for i in 0..av.len() { + if av[i].is_nan() || bv[i].is_nan() { + return false; + } if (av[i] - bv[i]).abs() > tol { return false; } @@ -127,6 +136,9 @@ fn fixed_size_list_almost_equal(a: &FixedSizeListArray, b: &FixedSizeListArray, return false; } for i in 0..av.len() { + if av[i].is_nan() || bv[i].is_nan() { + return false; + } if (av[i] - bv[i]).abs() > tol as f64 { return false; } @@ -144,6 +156,9 @@ fn fixed_size_list_almost_equal(a: &FixedSizeListArray, b: &FixedSizeListArray, for i in 0..av.len() { let da = av[i].to_f32(); let db = bv[i].to_f32(); + if da.is_nan() || db.is_nan() { + return false; + } if (da - db).abs() > tol { return false; } @@ -154,6 +169,63 @@ fn fixed_size_list_almost_equal(a: &FixedSizeListArray, b: &FixedSizeListArray, } } +fn ensure_fixed_size_list_compatible( + what: &str, + reference: &FixedSizeListArray, + candidate: &FixedSizeListArray, +) -> Result<()> { + if !fixed_size_list_equal(reference, candidate) { + const TOL: f32 = 1e-5; + if !fixed_size_list_almost_equal(reference, candidate, TOL) { + return Err(Error::index(format!("{what} mismatch across shards"))); + } + log::warn!("{what} differs within tolerance; proceeding with first shard value"); + } + Ok(()) +} + +async fn try_read_ivf_proto(reader: &V2Reader) -> Result> { + let Some(ivf_idx) = reader.metadata().file_schema.metadata.get(IVF_METADATA_KEY) else { + return Ok(None); + }; + let ivf_idx = ivf_idx + .parse() + .map_err(|_| Error::index("IVF index parse error".to_string()))?; + let bytes = reader.read_global_buffer(ivf_idx).await?; + Ok(Some(pb::Ivf::decode(bytes)?)) +} + +fn ivf_centroids_from_proto(ivf: &pb::Ivf) -> Result> { + ivf.centroids_tensor + .as_ref() + .map(FixedSizeListArray::try_from) + .transpose() +} + +async fn open_sibling_index_reader( + object_store: &lance_io::object_store::ObjectStore, + sched: &Arc, + idx_path: &object_store::path::Path, +) -> Result> { + if !object_store.exists(idx_path).await? { + return Ok(None); + } + + let fh = sched + .open_file(idx_path, &CachedFileSize::unknown()) + .await?; + Ok(Some( + V2Reader::try_open( + fh, + None, + Arc::default(), + &lance_core::cache::LanceCache::no_cache(), + V2ReaderOptions::default(), + ) + .await?, + )) +} + /// Initialize schema-level metadata on a writer for a given storage. /// /// It writes the distance type and the storage metadata (as a vector payload), @@ -183,7 +255,7 @@ pub async fn init_writer_for_flat( d0: usize, item_type: &DataType, dt: DistanceType, - format_version: LanceFileVersion, + format_version: ConcreteFileVersion, ) -> Result { let arrow_schema = ArrowSchema::new(vec![ (*ROW_ID_FIELD).clone(), @@ -197,13 +269,11 @@ pub async fn init_writer_for_flat( ), ]); let writer = object_store.create(aux_out).await?; - let mut w = FileWriter::try_new( + let mut w = versions::create_writer( + format_version, writer, LanceSchema::try_from(&arrow_schema)?, - FileWriterOptions { - format_version: Some(format_version), - ..Default::default() - }, + FileWriterOptions::default(), )?; let meta_json = serde_json::to_string(&FlatMetadata { dim: d0 })?; init_writer_for_storage(&mut w, dt, &meta_json, "")?; @@ -219,7 +289,7 @@ pub async fn init_writer_for_pq( aux_out: &object_store::path::Path, dt: DistanceType, pm: &ProductQuantizationMetadata, - format_version: LanceFileVersion, + format_version: ConcreteFileVersion, ) -> Result { let num_bytes = if pm.nbits == 4 { pm.num_sub_vectors / 2 @@ -238,13 +308,11 @@ pub async fn init_writer_for_pq( ), ]); let writer = object_store.create(aux_out).await?; - let mut w = FileWriter::try_new( + let mut w = versions::create_writer( + format_version, writer, LanceSchema::try_from(&arrow_schema)?, - FileWriterOptions { - format_version: Some(format_version), - ..Default::default() - }, + FileWriterOptions::default(), )?; let mut pm_init = pm.clone(); let cb = pm_init @@ -266,7 +334,7 @@ pub async fn init_writer_for_sq( aux_out: &object_store::path::Path, dt: DistanceType, sq_meta: &ScalarQuantizationMetadata, - format_version: LanceFileVersion, + format_version: ConcreteFileVersion, ) -> Result { let d0 = sq_meta.dim; let arrow_schema = ArrowSchema::new(vec![ @@ -281,13 +349,11 @@ pub async fn init_writer_for_sq( ), ]); let writer = object_store.create(aux_out).await?; - let mut w = FileWriter::try_new( + let mut w = versions::create_writer( + format_version, writer, LanceSchema::try_from(&arrow_schema)?, - FileWriterOptions { - format_version: Some(format_version), - ..Default::default() - }, + FileWriterOptions::default(), )?; let meta_json = serde_json::to_string(sq_meta)?; init_writer_for_storage(&mut w, dt, &meta_json, SQ_METADATA_KEY)?; @@ -300,7 +366,7 @@ pub async fn init_writer_for_rq( aux_out: &object_store::path::Path, dt: DistanceType, rq_meta: &RabitQuantizationMetadata, - format_version: LanceFileVersion, + format_version: ConcreteFileVersion, ) -> Result { let mut fields = vec![ (*ROW_ID_FIELD).clone(), @@ -318,13 +384,11 @@ pub async fn init_writer_for_rq( } let arrow_schema = ArrowSchema::new(fields); let writer = object_store.create(aux_out).await?; - let mut w = FileWriter::try_new( + let mut w = versions::create_writer( + format_version, writer, LanceSchema::try_from(&arrow_schema)?, - FileWriterOptions { - format_version: Some(format_version), - ..Default::default() - }, + FileWriterOptions::default(), )?; let mut rq_meta_init = rq_meta.clone(); @@ -728,7 +792,7 @@ pub async fn merge_partial_vector_auxiliary_files( let mut dim: Option = None; let mut detected_index_type: Option = None; // Inherit file format version from the first shard (set on first iteration) - let mut format_version: Option = None; + let mut format_version: Option = None; // Prepare output path; we'll create writer once when we know schema let aux_out = target_dir.clone().join(INDEX_AUXILIARY_FILE_NAME); @@ -771,6 +835,12 @@ pub async fn merge_partial_vector_auxiliary_files( ) .await?; let meta = reader.metadata(); + let idx_path = aux + .parent() + .unwrap_or_default() + .join(crate::INDEX_FILE_NAME); + let mut idx_reader: Option = None; + let mut idx_reader_checked = false; // Inherit format version from the first shard file if format_version.is_none() { @@ -795,45 +865,33 @@ pub async fn merge_partial_vector_auxiliary_files( // Detect index type (first iteration only) if detected_index_type.is_none() { // Try to derive precise type from sibling partial index.idx metadata if available - let idx_path = aux - .parent() - .unwrap_or_default() - .join(crate::INDEX_FILE_NAME); - if object_store.exists(&idx_path).await.unwrap_or(false) { - let fh2 = sched - .open_file(&idx_path, &CachedFileSize::unknown()) - .await?; - let idx_reader = V2Reader::try_open( - fh2, - None, - Arc::default(), - &lance_core::cache::LanceCache::no_cache(), - V2ReaderOptions::default(), - ) - .await?; - if let Some(idx_meta_json) = idx_reader + if !idx_reader_checked { + idx_reader = open_sibling_index_reader(object_store, &sched, &idx_path).await?; + idx_reader_checked = true; + } + if let Some(idx_reader) = idx_reader.as_ref() + && let Some(idx_meta_json) = idx_reader .metadata() .file_schema .metadata .get(INDEX_METADATA_SCHEMA_KEY) - { - let idx_meta: IndexMetaSchema = serde_json::from_str(idx_meta_json)?; - detected_index_type = Some(match idx_meta.index_type.as_str() { - "IVF_FLAT" => SupportedIvfIndexType::IvfFlat, - "IVF_PQ" => SupportedIvfIndexType::IvfPq, - "IVF_SQ" => SupportedIvfIndexType::IvfSq, - "IVF_RQ" => SupportedIvfIndexType::IvfRq, - "IVF_HNSW_FLAT" => SupportedIvfIndexType::IvfHnswFlat, - "IVF_HNSW_PQ" => SupportedIvfIndexType::IvfHnswPq, - "IVF_HNSW_SQ" => SupportedIvfIndexType::IvfHnswSq, - other => { - return Err(Error::index(format!( - "Unsupported index type in shard index.idx: {}", - other - ))); - } - }); - } + { + let idx_meta: IndexMetaSchema = serde_json::from_str(idx_meta_json)?; + detected_index_type = Some(match idx_meta.index_type.as_str() { + "IVF_FLAT" => SupportedIvfIndexType::IvfFlat, + "IVF_PQ" => SupportedIvfIndexType::IvfPq, + "IVF_SQ" => SupportedIvfIndexType::IvfSq, + "IVF_RQ" => SupportedIvfIndexType::IvfRq, + "IVF_HNSW_FLAT" => SupportedIvfIndexType::IvfHnswFlat, + "IVF_HNSW_PQ" => SupportedIvfIndexType::IvfHnswPq, + "IVF_HNSW_SQ" => SupportedIvfIndexType::IvfHnswSq, + other => { + return Err(Error::index(format!( + "Unsupported index type in shard index.idx: {}", + other + ))); + } + }); } // Fallback: infer from auxiliary schema if detected_index_type.is_none() { @@ -843,43 +901,60 @@ pub async fn merge_partial_vector_auxiliary_files( } // Read IVF lengths from global buffer - let ivf_idx: u32 = reader - .metadata() - .file_schema - .metadata - .get(IVF_METADATA_KEY) - .ok_or_else(|| Error::index("IVF meta missing".to_string()))? - .parse() - .map_err(|_| Error::index("IVF index parse error".to_string()))?; - let bytes = reader.read_global_buffer(ivf_idx).await?; - let pb_ivf: pb::Ivf = prost::Message::decode(bytes)?; + let pb_ivf = try_read_ivf_proto(&reader) + .await? + .ok_or_else(|| Error::index("IVF meta missing".to_string()))?; let lengths = pb_ivf.lengths.clone(); let nlist = lengths.len(); + let mut current_centroids = ivf_centroids_from_proto(&pb_ivf)?; + if current_centroids.is_none() { + if !idx_reader_checked { + idx_reader = open_sibling_index_reader(object_store, &sched, &idx_path).await?; + } + if let Some(idx_reader) = idx_reader.as_ref() + && let Some(index_ivf) = try_read_ivf_proto(idx_reader).await? + { + current_centroids = ivf_centroids_from_proto(&index_ivf)?; + } + } if nlist_opt.is_none() { nlist_opt = Some(nlist); accumulated_lengths = vec![0; nlist]; - // Try load centroids tensor if present - if let Some(tensor) = pb_ivf.centroids_tensor.as_ref() { - let arr = FixedSizeListArray::try_from(tensor)?; - first_centroids = Some(arr.clone()); + if let Some(arr) = current_centroids { let d0 = arr.value_length() as usize; if dim.is_none() { dim = Some(d0); } + first_centroids = Some(arr); } } else if nlist_opt.as_ref().map(|v| *v != nlist).unwrap_or(false) { return Err(Error::index( "IVF partition count mismatch across shards".to_string(), )); + } else { + match (&first_centroids, ¤t_centroids) { + (Some(reference), Some(candidate)) => { + ensure_fixed_size_list_compatible("IVF centroids", reference, candidate)?; + } + (Some(_), None) => { + return Err(Error::index("IVF centroids missing from shard".to_string())); + } + (None, Some(_)) => { + return Err(Error::index( + "IVF centroids missing from first shard".to_string(), + )); + } + (None, None) => {} + } } // Handle logic based on detected index type let idx_type = detected_index_type .ok_or_else(|| Error::index("Unable to detect index type".to_string()))?; - // Compute format version once; defaults to V2_0 if no shards processed yet - let fv = format_version.unwrap_or(LanceFileVersion::V2_0); + // Preserve the historical fallback while keeping the writer boundary exact. + let fv = format_version.unwrap_or(ConcreteFileVersion::V2_0); match idx_type { SupportedIvfIndexType::IvfSq => { @@ -985,6 +1060,11 @@ pub async fn merge_partial_vector_auxiliary_files( rq_meta_parsed.parse_buffer(rotate_mat_bytes)?; } validate_rq_num_bits(rq_meta_parsed.num_bits)?; + if rq_meta_parsed.packed { + return Err(Error::index(format!( + "Distributed RQ merge: source shard {idx} stores packed RQ codes; expected row-major distributed shard" + ))); + } let d0 = rq_meta_parsed.rotated_dim(); if d0 == 0 { @@ -1001,7 +1081,9 @@ pub async fn merge_partial_vector_auxiliary_files( if let Some(existing_rq) = rq_meta.as_ref() && (existing_rq.code_dim != rq_meta_parsed.code_dim || existing_rq.num_bits != rq_meta_parsed.num_bits - || existing_rq.rotation_type != rq_meta_parsed.rotation_type) + || existing_rq.rotation_type != rq_meta_parsed.rotation_type + || existing_rq.query_estimator != rq_meta_parsed.query_estimator + || existing_rq.fast_rotation_signs != rq_meta_parsed.fast_rotation_signs) { return Err(Error::index(format!( "Distributed RQ merge: structural mismatch across shards; first(code_dim={}, num_bits={}, rotation_type={:?}), current(code_dim={}, num_bits={}, rotation_type={:?})", @@ -1013,6 +1095,24 @@ pub async fn merge_partial_vector_auxiliary_files( rq_meta_parsed.rotation_type ))); } + if let Some(existing_rq) = rq_meta.as_ref() { + match (&existing_rq.rotate_mat, &rq_meta_parsed.rotate_mat) { + (Some(reference), Some(candidate)) => { + ensure_fixed_size_list_compatible( + "RQ rotation matrix", + reference, + candidate, + )?; + } + (Some(_), None) | (None, Some(_)) => { + return Err(Error::index( + "Distributed RQ merge: rotation matrix mismatch across shards" + .to_string(), + )); + } + (None, None) => {} + } + } if rq_meta.is_none() { rq_meta = Some(rq_meta_parsed.clone()); } @@ -1061,6 +1161,11 @@ pub async fn merge_partial_vector_auxiliary_files( }; let mut pm: ProductQuantizationMetadata = serde_json::from_str(&pm_json) .map_err(|e| Error::index(format!("PQ metadata parse error: {}", e)))?; + if pm.transposed { + return Err(Error::index(format!( + "Distributed PQ merge: source shard {idx} stores transposed PQ codes; expected row-major distributed shard" + ))); + } // Load codebook from global buffer if not present if pm.codebook.is_none() { let tensor_bytes = reader @@ -1100,18 +1205,11 @@ pub async fn merge_partial_vector_auxiliary_files( .codebook .as_ref() .ok_or_else(|| Error::index("PQ codebook missing in shard".to_string()))?; - if !fixed_size_list_equal(existing_cb, current_cb) { - const TOL: f32 = 1e-5; - if !fixed_size_list_almost_equal(existing_cb, current_cb, TOL) { - return Err(Error::index( - "PQ codebook content mismatch across shards".to_string(), - )); - } else { - log::warn!( - "PQ codebook differs within tolerance; proceeding with first shard codebook" - ); - } - } + ensure_fixed_size_list_compatible( + "PQ codebook content", + existing_cb, + current_cb, + )?; } if pq_meta.is_none() { pq_meta = Some(pm.clone()); @@ -1222,6 +1320,11 @@ pub async fn merge_partial_vector_auxiliary_files( }; let mut pm: ProductQuantizationMetadata = serde_json::from_str(&pm_json) .map_err(|e| Error::index(format!("PQ metadata parse error: {}", e)))?; + if pm.transposed { + return Err(Error::index(format!( + "Distributed PQ merge: source shard {idx} stores transposed PQ codes; expected row-major distributed shard" + ))); + } if pm.codebook.is_none() { let tensor_bytes = reader .read_global_buffer(pm.codebook_position as u32) @@ -1260,18 +1363,11 @@ pub async fn merge_partial_vector_auxiliary_files( .codebook .as_ref() .ok_or_else(|| Error::index("PQ codebook missing in shard".to_string()))?; - if !fixed_size_list_equal(existing_cb, current_cb) { - const TOL: f32 = 1e-5; - if !fixed_size_list_almost_equal(existing_cb, current_cb, TOL) { - return Err(Error::index( - "PQ codebook content mismatch across shards".to_string(), - )); - } else { - log::warn!( - "PQ codebook differs within tolerance; proceeding with first shard codebook" - ); - } - } + ensure_fixed_size_list_compatible( + "PQ codebook content", + existing_cb, + current_cb, + )?; } if pq_meta.is_none() { pq_meta = Some(pm.clone()); @@ -1554,6 +1650,31 @@ mod tests { lance_core::Result<()> ); + #[test] + fn test_uint8_fixed_size_list_compatibility() { + let values = (0_u8..16).collect::>(); + let reference = + FixedSizeListArray::try_new_from_values(UInt8Array::from(values.clone()), 8).unwrap(); + let matching = + FixedSizeListArray::try_new_from_values(UInt8Array::from(values.clone()), 8).unwrap(); + + ensure_fixed_size_list_compatible("IVF centroids", &reference, &matching).unwrap(); + + let mut differing_values = values; + differing_values[15] = 16; + let differing = + FixedSizeListArray::try_new_from_values(UInt8Array::from(differing_values), 8).unwrap(); + let error = + ensure_fixed_size_list_compatible("IVF centroids", &reference, &differing).unwrap_err(); + + assert!(matches!(&error, Error::Index { .. })); + assert!( + error + .to_string() + .contains("IVF centroids mismatch across shards") + ); + } + async fn write_flat_partial_aux( store: &ObjectStore, aux_path: &Path, @@ -1561,6 +1682,7 @@ mod tests { lengths: &[u32], base_row_id: u64, distance_type: DistanceType, + file_version: ConcreteFileVersion, ) -> Result { let arrow_schema = ArrowSchema::new(vec![ (*ROW_ID_FIELD).clone(), @@ -1572,7 +1694,8 @@ mod tests { ]); let writer = store.create(aux_path).await?; - let mut v2w = V2Writer::try_new( + let mut v2w = versions::create_writer( + file_version, writer, lance_core::datatypes::Schema::try_from(&arrow_schema)?, V2WriterOptions::default(), @@ -1642,7 +1765,7 @@ mod tests { ]); let writer = store.create(aux_path).await?; - let mut v2w = V2Writer::try_new( + let mut v2w = versions::v2_1::create_writer( writer, lance_core::datatypes::Schema::try_from(&arrow_schema)?, V2WriterOptions::default(), @@ -1703,12 +1826,28 @@ mod tests { let lengths1 = vec![1_u32, 2_u32]; let dim = 2_i32; - write_flat_partial_aux(&object_store, &aux0, dim, &lengths0, 0, DistanceType::L2) - .await - .unwrap(); - write_flat_partial_aux(&object_store, &aux1, dim, &lengths1, 100, DistanceType::L2) - .await - .unwrap(); + write_flat_partial_aux( + &object_store, + &aux0, + dim, + &lengths0, + 0, + DistanceType::L2, + ConcreteFileVersion::V2_2, + ) + .await + .unwrap(); + write_flat_partial_aux( + &object_store, + &aux1, + dim, + &lengths1, + 100, + DistanceType::L2, + ConcreteFileVersion::V2_1, + ) + .await + .unwrap(); let progress = Arc::new(RecordingProgress::default()); merge_partial_vector_auxiliary_files( @@ -1812,6 +1951,7 @@ mod tests { .await .unwrap(); let meta = reader.metadata(); + assert_eq!(meta.version(), ConcreteFileVersion::V2_2); // Validate IVF lengths aggregation. let ivf_idx: u32 = meta @@ -1871,9 +2011,17 @@ mod tests { let lengths = vec![2_u32, 2_u32]; let dim = 2_i32; - write_flat_partial_aux(&object_store, &aux0, dim, &lengths, 0, DistanceType::L2) - .await - .unwrap(); + write_flat_partial_aux( + &object_store, + &aux0, + dim, + &lengths, + 0, + DistanceType::L2, + ConcreteFileVersion::V2_1, + ) + .await + .unwrap(); write_flat_partial_aux( &object_store, &aux1, @@ -1881,6 +2029,7 @@ mod tests { &lengths, 100, DistanceType::Cosine, + ConcreteFileVersion::V2_1, ) .await .unwrap(); @@ -1976,6 +2125,7 @@ mod tests { base_row_id: u64, distance_type: DistanceType, codebook: &FixedSizeListArray, + transposed: bool, ) -> Result { let num_bytes = if nbits == 4 { // Two 4-bit codes per byte. @@ -1997,7 +2147,7 @@ mod tests { ]); let writer = store.create(aux_path).await?; - let mut v2w = V2Writer::try_new( + let mut v2w = versions::v2_1::create_writer( writer, lance_core::datatypes::Schema::try_from(&arrow_schema)?, V2WriterOptions::default(), @@ -2014,7 +2164,7 @@ mod tests { dimension, codebook: Some(codebook.clone()), codebook_tensor: Vec::new(), - transposed: true, + transposed, }; let codebook_tensor: pb::Tensor = pb::Tensor::try_from(codebook)?; @@ -2109,7 +2259,7 @@ mod tests { let arrow_schema = ArrowSchema::new(fields); let writer = store.create(aux_path).await?; - let mut v2w = V2Writer::try_new( + let mut v2w = versions::v2_1::create_writer( writer, lance_core::datatypes::Schema::try_from(&arrow_schema)?, V2WriterOptions::default(), @@ -2225,6 +2375,7 @@ mod tests { 0, DistanceType::L2, &codebook, + false, ) .await .unwrap(); @@ -2239,6 +2390,7 @@ mod tests { 1_000, DistanceType::L2, &codebook, + false, ) .await .unwrap(); @@ -2319,6 +2471,66 @@ mod tests { assert!(fixed_size_list_equal(&codebook, &merged_codebook)); } + #[tokio::test] + async fn test_merge_ivf_pq_rejects_transposed_source_shard() { + let object_store = ObjectStore::memory(); + let index_dir = Path::from("index/uuid_pq_transposed"); + + let partial0 = index_dir.clone().join("partial_0"); + let aux0 = partial0.clone().join(INDEX_AUXILIARY_FILE_NAME); + let lengths = vec![2_u32, 1_u32]; + + let nbits = 4_u32; + let num_sub_vectors = 2_usize; + let dimension = 8_usize; + let num_centroids = 1_usize << nbits; + let num_codebook_vectors = num_centroids * num_sub_vectors; + let total_values = num_codebook_vectors * dimension; + let values = Float32Array::from_iter((0..total_values).map(|v| v as f32)); + let codebook = FixedSizeListArray::try_new_from_values(values, dimension as i32).unwrap(); + + write_pq_partial_aux( + &object_store, + &aux0, + nbits, + num_sub_vectors, + dimension, + &lengths, + 0, + DistanceType::L2, + &codebook, + true, + ) + .await + .unwrap(); + + let res = merge_partial_vector_auxiliary_files( + &object_store, + std::slice::from_ref(&aux0), + &index_dir, + crate::progress::noop_progress(), + ) + .await; + match res { + Err(Error::Index { message, .. }) => { + assert!( + message.contains("source shard 0"), + "unexpected message: {}", + message + ); + assert!( + message.contains("transposed PQ codes"), + "unexpected message: {}", + message + ); + } + other => panic!( + "expected Error::Index for transposed PQ source shard, got {:?}", + other + ), + } + } + #[tokio::test] async fn test_merge_ivf_rq_success() { let object_store = ObjectStore::memory(); @@ -2455,6 +2667,64 @@ mod tests { assert_eq!(total_rows, expected_total); } + #[tokio::test] + async fn test_merge_ivf_rq_rejects_packed_source_shard() { + let object_store = ObjectStore::memory(); + let index_dir = Path::from("index/uuid_rq_packed"); + + let partial0 = index_dir.clone().join("partial_0"); + let aux0 = partial0.clone().join(INDEX_AUXILIARY_FILE_NAME); + let lengths = vec![2_u32, 1_u32]; + + let rq_meta = RabitQuantizationMetadata { + rotate_mat: None, + rotate_mat_position: None, + fast_rotation_signs: Some(vec![0xAA; 2]), + rotation_type: RQRotationType::Fast, + code_dim: 16, + num_bits: 1, + packed: true, + query_estimator: RabitQueryEstimator::RawQuery, + }; + + write_rq_partial_aux( + &object_store, + &aux0, + &rq_meta, + &lengths, + 0, + DistanceType::L2, + ) + .await + .unwrap(); + + let res = merge_partial_vector_auxiliary_files( + &object_store, + std::slice::from_ref(&aux0), + &index_dir, + crate::progress::noop_progress(), + ) + .await; + match res { + Err(Error::Index { message, .. }) => { + assert!( + message.contains("source shard 0"), + "unexpected message: {}", + message + ); + assert!( + message.contains("packed RQ codes"), + "unexpected message: {}", + message + ); + } + other => panic!( + "expected Error::Index for packed RQ source shard, got {:?}", + other + ), + } + } + #[tokio::test] async fn test_merge_ivf_rq_multi_bit_preserves_split_columns() { let object_store = ObjectStore::memory(); @@ -2612,6 +2882,7 @@ mod tests { 0, DistanceType::L2, &codebook0, + false, ) .await .unwrap(); @@ -2626,6 +2897,7 @@ mod tests { 1_000, DistanceType::L2, &codebook1, + false, ) .await .unwrap(); @@ -2691,6 +2963,7 @@ mod tests { 0, DistanceType::L2, &codebook, + false, ) .await .unwrap(); @@ -2706,6 +2979,7 @@ mod tests { 1_000, DistanceType::L2, &codebook, + false, ) .await .unwrap(); diff --git a/rust/lance-index/src/vector/flat/index.rs b/rust/lance-index/src/vector/flat/index.rs index cc6f6d021eb..3a17208d469 100644 --- a/rust/lance-index/src/vector/flat/index.rs +++ b/rust/lance-index/src/vector/flat/index.rs @@ -13,7 +13,7 @@ use arrow_array::{Array, ArrayRef, Float32Array, RecordBatch, UInt64Array}; use arrow_schema::{DataType, Field, Schema, SchemaRef}; use lance_core::deepsize::DeepSizeOf; use lance_core::{Error, ROW_ID_FIELD, Result}; -use lance_file::previous::reader::FileReader as PreviousFileReader; +use lance_file::versions::v1::reader::FileReader as V1FileReader; use lance_linalg::distance::DistanceType; use serde::{Deserialize, Serialize}; @@ -329,7 +329,7 @@ pub struct FlatMetadata { #[async_trait::async_trait] impl QuantizerMetadata for FlatMetadata { - async fn load(_: &PreviousFileReader) -> Result { + async fn load(_: &V1FileReader) -> Result { unimplemented!("Flat will be used in new index builder which doesn't require this") } } @@ -570,7 +570,7 @@ mod tests { .zip(dists.values().iter()) .map(|(row_id, dist)| (*row_id, *dist)) .collect::>(); - results.sort_by(|left, right| left.0.cmp(&right.0)); + results.sort_by_key(|left| left.0); results } @@ -579,7 +579,7 @@ mod tests { .into_iter() .map(|node| (node.id, node.dist.0)) .collect::>(); - results.sort_by(|left, right| left.0.cmp(&right.0)); + results.sort_by_key(|left| left.0); results } diff --git a/rust/lance-index/src/vector/flat/storage.rs b/rust/lance-index/src/vector/flat/storage.rs index c3ec30d5086..23b3f2cadc9 100644 --- a/rust/lance-index/src/vector/flat/storage.rs +++ b/rust/lance-index/src/vector/flat/storage.rs @@ -19,7 +19,7 @@ use arrow_array::{ use arrow_schema::{DataType, SchemaRef}; use lance_core::deepsize::DeepSizeOf; use lance_core::{Error, ROW_ID, Result}; -use lance_file::previous::reader::FileReader as PreviousFileReader; +use lance_file::versions::v1::reader::FileReader as V1FileReader; use lance_linalg::distance::hamming::hamming; use lance_linalg::distance::{Cosine, DistanceType, Dot, L2}; @@ -87,7 +87,7 @@ impl QuantizerStorage for FlatFloatStorage { } async fn load_partition( - _: &PreviousFileReader, + _: &V1FileReader, _: std::ops::Range, _: DistanceType, _: &Self::Metadata, @@ -134,7 +134,7 @@ impl VectorStore for FlatFloatStorage { fn append_batch(&self, batch: RecordBatch, _vector_column: &str) -> Result { // TODO: use chunked storage - let new_batch = concat_batches(&batch.schema(), vec![&self.batch, &batch].into_iter())?; + let new_batch = concat_batches(&batch.schema(), vec![&self.batch, &batch])?; let mut storage = self.clone(); storage.row_ids = Arc::new( new_batch @@ -249,7 +249,7 @@ impl QuantizerStorage for FlatBinStorage { } async fn load_partition( - _: &PreviousFileReader, + _: &V1FileReader, _: std::ops::Range, _: DistanceType, _: &Self::Metadata, @@ -296,7 +296,7 @@ impl VectorStore for FlatBinStorage { fn append_batch(&self, batch: RecordBatch, _vector_column: &str) -> Result { // TODO: use chunked storage - let new_batch = concat_batches(&batch.schema(), vec![&self.batch, &batch].into_iter())?; + let new_batch = concat_batches(&batch.schema(), vec![&self.batch, &batch])?; let mut storage = self.clone(); storage.row_ids = Arc::new( new_batch diff --git a/rust/lance-index/src/vector/flat/transform.rs b/rust/lance-index/src/vector/flat/transform.rs index 75a465ce262..6e5c5af5f9e 100644 --- a/rust/lance-index/src/vector/flat/transform.rs +++ b/rust/lance-index/src/vector/flat/transform.rs @@ -26,7 +26,7 @@ impl FlatTransformer { impl Transformer for FlatTransformer { #[instrument(name = "FlatTransformer::transform", level = "debug", skip_all)] - fn transform(&self, batch: &RecordBatch) -> crate::Result { + fn transform(&self, batch: &RecordBatch) -> lance_core::Result { let input_arr = batch .column_by_name(&self.input_column) .ok_or(Error::index(format!( @@ -45,3 +45,91 @@ impl Transformer for FlatTransformer { Ok(batch) } } + +#[cfg(test)] +mod tests { + use super::*; + + use std::sync::Arc; + + use arrow_array::{Array, FixedSizeListArray, Float32Array, Int32Array}; + use arrow_schema::{DataType, Schema}; + use lance_arrow::FixedSizeListArrayExt; + + const DIM: i32 = 4; + const ROWS: usize = 8; + + fn batch() -> RecordBatch { + let values = Float32Array::from_iter((0..(DIM as usize * ROWS)).map(|v| v as f32)); + let vectors = Arc::new(FixedSizeListArray::try_new_from_values(values, DIM).unwrap()); + let schema = Schema::new(vec![ + Field::new("vec", vectors.data_type().clone(), true), + Field::new("other", DataType::Int32, false), + ]); + RecordBatch::try_new( + Arc::new(schema), + vec![ + vectors, + Arc::new(Int32Array::from_iter_values(0..ROWS as i32)), + ], + ) + .unwrap() + } + + #[test] + fn test_flat_transform_renames_vector_column() { + let input = batch(); + let output = FlatTransformer::new("vec").transform(&input).unwrap(); + + assert!(output.column_by_name("vec").is_none()); + assert!( + output.column_by_name("other").is_some(), + "unrelated columns should survive" + ); + assert_eq!(output.num_rows(), ROWS); + + let flat = output.column_by_name(FLAT_COLUMN).unwrap(); + assert_eq!( + flat.data_type(), + input.column_by_name("vec").unwrap().data_type() + ); + assert_eq!(flat.as_ref(), input.column_by_name("vec").unwrap().as_ref()); + } + + #[test] + fn test_flat_transform_preserves_nullability() { + // The renamed field copies the source column's nullability rather than + // hardcoding it, so a non-nullable vector column stays non-nullable. + let values = Float32Array::from_iter((0..(DIM as usize * ROWS)).map(|v| v as f32)); + let vectors = Arc::new(FixedSizeListArray::try_new_from_values(values, DIM).unwrap()); + let schema = Schema::new(vec![ + Field::new("vec", vectors.data_type().clone(), false), + Field::new("other", DataType::Int32, false), + ]); + let input = RecordBatch::try_new( + Arc::new(schema), + vec![ + vectors, + Arc::new(Int32Array::from_iter_values(0..ROWS as i32)), + ], + ) + .unwrap(); + + let output = FlatTransformer::new("vec").transform(&input).unwrap(); + let field = output + .schema() + .field_with_name(FLAT_COLUMN) + .unwrap() + .clone(); + assert!(!field.is_nullable()); + } + + #[test] + fn test_flat_transform_reports_missing_column() { + let message = FlatTransformer::new("absent") + .transform(&batch()) + .unwrap_err() + .to_string(); + assert!(message.contains("column absent"), "{message}"); + } +} diff --git a/rust/lance-index/src/vector/graph.rs b/rust/lance-index/src/vector/graph.rs index 097aa064d67..049feefb60e 100644 --- a/rust/lance-index/src/vector/graph.rs +++ b/rust/lance-index/src/vector/graph.rs @@ -5,7 +5,7 @@ //! use std::cmp::Reverse; -use std::collections::BinaryHeap; +use std::collections::{BinaryHeap, VecDeque}; use std::sync::Arc; use arrow_schema::{DataType, Field}; @@ -321,6 +321,10 @@ macro_rules! beam_search_loop { } $visited.insert(neighbor); let dist: OrderedFloat = $dist_calc.distance(neighbor).into(); + // Algorithm 2 refreshes the furthest result after every + // update to W, including updates made by an earlier neighbor + // of this same expanded node. + let furthest = furthest_distance(&$results); if dist <= furthest || $results.len() < $k { if $accepts_result(neighbor, dist) { push_result(&mut $results, (dist, neighbor).into(), $k); @@ -546,6 +550,152 @@ pub fn beam_search_borrowed( results.into_sorted_vec() } +/// Number of mask-passing nodes used to seed [beam_search_acorn]'s frontier. +const ACORN_SEED_COUNT: usize = 16; + +/// Cap on starved-frontier waypoint expansions in [beam_search_acorn], +/// as a multiple of `ef`. +const ACORN_BRIDGE_BUDGET_FACTOR: usize = 4; + +/// Beam search over the mask-passing subgraph (ACORN-1). +/// +/// Only nodes in `bitset` get distances. A filtered-out neighbor contributes +/// its own neighbors instead, expanded once via `expanded`. Deeper masked +/// chains are crossed through unscored waypoints under a budget. The frontier +/// starts from the entry point plus mask-sampled seeds. May return fewer than +/// `min(ef, passing)` results if the budget runs out, so callers needing a +/// guarantee must check the count. +#[allow(clippy::too_many_arguments)] +pub fn beam_search_acorn( + graph: &impl BorrowingGraph, + ep: &OrderedNode, + params: &HnswQueryParams, + dist_calc: &impl DistCalculator, + bitset: &Visited, + prefetch_distance: Option, + visited: &mut Visited, + expanded: &mut Visited, +) -> Vec { + let ef = params.ef; + let lower_bound: OrderedFloat = params.lower_bound.unwrap_or(f32::MIN).into(); + let upper_bound: OrderedFloat = params.upper_bound.unwrap_or(f32::MAX).into(); + let passing_total = bitset.count_ones(); + let mut candidates = BinaryHeap::with_capacity(ef); + let mut results = BinaryHeap::with_capacity(ef); + // collected per node before scoring so prefetch targets are the ids + // that actually get distances + let mut passing: Vec = Vec::with_capacity(64); + // masked nodes seen two hops out, expandable if the frontier starves, + // deduped against `expanded` at pop rather than at push + let mut waypoints: VecDeque = VecDeque::new(); + let mut bridge_budget = ACORN_BRIDGE_BUDGET_FACTOR * ef; + + // the entry point seeds the traversal even if it fails the mask + visited.insert(ep.id); + candidates.push(Reverse(ep.clone())); + if bitset.contains(ep.id) && ep.dist >= lower_bound && ep.dist < upper_bound { + results.push(ep.clone()); + } + + let stride = (passing_total / ACORN_SEED_COUNT).max(1); + for seed in bitset.iter_ones().step_by(stride).take(ACORN_SEED_COUNT) { + let seed = seed as u32; + if visited.contains(seed) { + continue; + } + visited.insert(seed); + let dist: OrderedFloat = dist_calc.distance(seed).into(); + if dist >= lower_bound && dist < upper_bound { + push_result(&mut results, (dist, seed).into(), ef); + } + candidates.push(Reverse((dist, seed).into())); + } + + loop { + let Some(Reverse(current)) = candidates.pop() else { + // frontier starved: burn bridge budget through masked waypoints + // until a new passing node is found + if results.len() >= ef.min(passing_total) { + break; + } + let mut found = false; + while let Some(waypoint) = waypoints.pop_front() { + if bridge_budget == 0 { + break; + } + if expanded.contains(waypoint) { + continue; + } + expanded.insert(waypoint); + bridge_budget -= 1; + for &neighbor in graph.neighbors(waypoint) { + if bitset.contains(neighbor) { + if !visited.contains(neighbor) { + visited.insert(neighbor); + let dist: OrderedFloat = dist_calc.distance(neighbor).into(); + if dist >= lower_bound && dist < upper_bound { + push_result(&mut results, (dist, neighbor).into(), ef); + } + candidates.push(Reverse((dist, neighbor).into())); + found = true; + } + } else if !expanded.contains(neighbor) { + waypoints.push_back(neighbor); + } + } + if found { + break; + } + } + if !found { + break; + } + continue; + }; + if current.dist > furthest_distance(&results) && results.len() == ef { + break; + } + + passing.clear(); + for &neighbor in graph.neighbors(current.id) { + if bitset.contains(neighbor) { + if !visited.contains(neighbor) { + visited.insert(neighbor); + passing.push(neighbor); + } + } else if !expanded.contains(neighbor) { + expanded.insert(neighbor); + for &second_hop in graph.neighbors(neighbor) { + if bitset.contains(second_hop) { + if !visited.contains(second_hop) { + visited.insert(second_hop); + passing.push(second_hop); + } + } else if !expanded.contains(second_hop) { + waypoints.push_back(second_hop); + } + } + } + } + + process_neighbors_with_look_ahead( + &passing, + |node| { + let dist: OrderedFloat = dist_calc.distance(node).into(); + if dist <= furthest_distance(&results) || results.len() < ef { + if dist >= lower_bound && dist < upper_bound { + push_result(&mut results, (dist, node).into(), ef); + } + candidates.push(Reverse((dist, node).into())); + } + }, + prefetch_distance, + dist_calc, + ); + } + results.into_sorted_vec() +} + /// Greedy search over a graph /// /// This searches for only one result, only used for finding the entry point @@ -617,4 +767,155 @@ pub fn greedy_search_borrowed( } #[cfg(test)] -mod tests {} +mod tests { + use std::cell::Cell; + + use super::*; + + struct ChainGraph { + neighbors: Vec>, + } + + impl BorrowingGraph for ChainGraph { + fn len(&self) -> usize { + self.neighbors.len() + } + + fn neighbors(&self, key: u32) -> &[u32] { + &self.neighbors[key as usize] + } + } + + struct ZeroDistance; + + impl DistCalculator for ZeroDistance { + fn distance(&self, _id: u32) -> f32 { + 0.0 + } + + fn distance_all(&self, _k_hint: usize) -> Vec { + Vec::new() + } + } + + struct CountingDistCalculator<'a> { + distances: &'a [f32], + computations: Cell, + } + + impl DistCalculator for CountingDistCalculator<'_> { + fn distance(&self, id: u32) -> f32 { + self.computations.set(self.computations.get() + 1); + self.distances[id as usize] + } + + fn distance_all(&self, _k_hint: usize) -> Vec { + self.distances.to_vec() + } + } + + #[test] + fn test_beam_search_refreshes_furthest_neighbor_bound() { + // The entry point is closer than the first candidates. Once those + // candidates fill W, its furthest distance grows from 1 to 10. The + // next neighbor must be compared with that new bound so it can enter + // the candidate queue and lead the search to node 4. + let graph = ChainGraph { + neighbors: vec![vec![1, 2, 3], vec![], vec![], vec![4], vec![]], + }; + let distances = [1.0, 10.0, 9.0, 8.0, 7.0]; + let dist_calc = CountingDistCalculator { + distances: &distances, + computations: Cell::new(0), + }; + let params = HnswQueryParams { + ef: 3, + lower_bound: None, + upper_bound: None, + dist_q_c: 0.0, + use_acorn: false, + }; + let entry = OrderedNode::new(0, distances[0].into()); + let mut visited_generator = VisitedGenerator::new(graph.len()); + + let results = beam_search_borrowed( + &graph, + &entry, + ¶ms, + &dist_calc, + None, + None, + &mut visited_generator.generate(graph.len()), + ); + + assert_eq!( + results.iter().map(|node| node.id).collect::>(), + vec![0, 4, 3] + ); + assert_eq!(dist_calc.computations.get(), 4); + } + + /// Passing components joined only through chains of two masked nodes + /// must still all be found (from review: without waypoint expansion + /// only the seeded nodes return). + #[test] + fn test_acorn_reaches_across_masked_chains() { + const PASSING_COUNT: usize = 20; + const FAILING_COUNT: usize = (PASSING_COUNT - 1) * 2; + let mut neighbors = vec![Vec::new(); PASSING_COUNT + FAILING_COUNT]; + for index in 0..PASSING_COUNT - 1 { + let left = index as u32; + let first_failing = (PASSING_COUNT + index * 2) as u32; + let second_failing = first_failing + 1; + let right = left + 1; + + neighbors[left as usize].push(first_failing); + neighbors[first_failing as usize].extend([left, second_failing]); + neighbors[second_failing as usize].extend([first_failing, right]); + neighbors[right as usize].push(second_failing); + } + let graph = ChainGraph { neighbors }; + let params = HnswQueryParams { + ef: 30, + lower_bound: None, + upper_bound: None, + dist_q_c: 0.0, + use_acorn: false, + }; + let entry = OrderedNode::new(0, 0.0.into()); + + let mut mask_generator = VisitedGenerator::new(graph.len()); + let mut mask = mask_generator.generate(graph.len()); + for id in 0..PASSING_COUNT as u32 { + mask.insert(id); + } + + let mut acorn_visited_generator = VisitedGenerator::new(graph.len()); + let mut acorn_expanded_generator = VisitedGenerator::new(graph.len()); + let acorn_results = beam_search_acorn( + &graph, + &entry, + ¶ms, + &ZeroDistance, + &mask, + None, + &mut acorn_visited_generator.generate(graph.len()), + &mut acorn_expanded_generator.generate(graph.len()), + ); + + let mut basic_visited_generator = VisitedGenerator::new(graph.len()); + let basic_results = beam_search_borrowed( + &graph, + &entry, + ¶ms, + &ZeroDistance, + Some(&mask), + None, + &mut basic_visited_generator.generate(graph.len()), + ); + + assert_eq!(basic_results.len(), PASSING_COUNT); + assert_eq!(acorn_results.len(), PASSING_COUNT); + assert!(acorn_results.iter().all(|node| mask.contains(node.id))); + } +} diff --git a/rust/lance-index/src/vector/graph/builder.rs b/rust/lance-index/src/vector/graph/builder.rs index 36e42b7cf9a..2cbc8855568 100644 --- a/rust/lance-index/src/vector/graph/builder.rs +++ b/rust/lance-index/src/vector/graph/builder.rs @@ -60,25 +60,4 @@ impl GraphBuilderNode { self.bottom_neighbors = self.level_neighbors[0].clone(); } } - - pub(crate) fn cutoff(&self, level: u16, max_size: usize) -> OrderedFloat { - let neighbors = &self.level_neighbors_ranked[level as usize]; - if neighbors.len() < max_size { - OrderedFloat(f32::INFINITY) - } else { - neighbors.last().unwrap().dist - } - } -} - -#[derive(Debug)] -pub struct GraphBuilderStats { - #[allow(dead_code)] - pub num_nodes: usize, - #[allow(dead_code)] - pub max_edges: usize, - #[allow(dead_code)] - pub mean_edges: f32, - #[allow(dead_code)] - pub mean_distance: f32, } diff --git a/rust/lance-index/src/vector/hnsw.rs b/rust/lance-index/src/vector/hnsw.rs index a618f34753f..0b86f4d2bf7 100644 --- a/rust/lance-index/src/vector/hnsw.rs +++ b/rust/lance-index/src/vector/hnsw.rs @@ -48,7 +48,7 @@ pub struct HnswMetadata { impl Default for HnswMetadata { fn default() -> Self { let params = HnswBuildParams::default(); - let level_offsets = vec![0; params.max_level as usize]; + let level_offsets = vec![0; params.max_level as usize + 1]; Self { entry_point: 0, params, @@ -59,6 +59,13 @@ impl Default for HnswMetadata { /// Algorithm 4 in the HNSW paper. /// +/// This uses the paper's `extendCandidates = false` and +/// `keepPrunedConnections = true` configuration. Keeping pruned connections +/// fills the requested degree without exceeding it, reducing sparse directed +/// components. It does not guarantee that every node is reachable: lower +/// construction settings trade graph quality for size and build work. Callers +/// supply the complete candidate set that should participate in this selection. +/// /// # NOTE /// The results are not ordered. pub(crate) fn select_neighbors_heuristic( @@ -85,14 +92,56 @@ pub(crate) fn select_neighbors_heuristic_owned( candidates.sort_unstable(); let mut results: Vec = Vec::with_capacity(k); - for u in candidates.iter() { + let mut pruned = Vec::with_capacity(candidates.len()); + for candidate in candidates { if results.len() >= k { break; } - if results.is_empty() || storage.prefers_candidate(u, &results) { - results.push(u.clone()); + if results.is_empty() || storage.prefers_candidate(&candidate, &results) { + results.push(candidate); + } else { + pruned.push(candidate); } } + results.extend(pruned.into_iter().take(k - results.len())); results } + +#[cfg(test)] +mod tests { + use arrow_array::{FixedSizeListArray, Float32Array}; + use lance_arrow::FixedSizeListArrayExt; + use lance_linalg::distance::DistanceType; + + use super::select_neighbors_heuristic_owned; + use crate::vector::flat::storage::FlatFloatStorage; + use crate::vector::graph::OrderedNode; + use crate::vector::storage::VectorStore; + + /// A reciprocal candidate must join the complete old-plus-new set before + /// Algorithm 4 runs. A farther, directionally diverse connection is + /// selected first, then pruned connections refill the remaining capacity. + #[test] + fn test_selection_retains_farther_diverse_reciprocal_candidate() { + let vectors = Float32Array::from(vec![ + 0.0, 0.0, // node receiving the reciprocal connection + 1.0, 0.0, // close candidate + 2.0, 0.0, // redundant candidate + 3.0, 0.0, // redundant candidate + 4.0, 0.0, // redundant candidate + 0.0, 5.0, // farther but directionally diverse candidate + ]); + let vectors = FixedSizeListArray::try_new_from_values(vectors, 2).unwrap(); + let storage = FlatFloatStorage::new(vectors, DistanceType::L2); + let candidates = (1..=5) + .map(|id| OrderedNode::new(id, storage.dist_between(0, id).into())) + .collect(); + + let selected = select_neighbors_heuristic_owned(&storage, candidates, 4); + assert_eq!( + selected.iter().map(|node| node.id).collect::>(), + vec![1, 5, 2, 3] + ); + } +} diff --git a/rust/lance-index/src/vector/hnsw/builder.rs b/rust/lance-index/src/vector/hnsw/builder.rs index cc1ac1abf81..fbbabcd321d 100644 --- a/rust/lance-index/src/vector/hnsw/builder.rs +++ b/rust/lance-index/src/vector/hnsw/builder.rs @@ -18,7 +18,6 @@ use rayon::prelude::*; use std::cmp::min; use std::collections::{BinaryHeap, HashMap, VecDeque}; use std::fmt::Debug; -use std::iter; use std::sync::Arc; use std::sync::RwLock; use std::sync::atomic::{AtomicUsize, Ordering}; @@ -40,10 +39,12 @@ use crate::vector::graph::{ BorrowingGraph, DISTS_FIELD, Graph, NEIGHBORS_COL, NEIGHBORS_FIELD, OrderedFloat, OrderedNode, VisitedGenerator, }; -use crate::vector::graph::{Visited, beam_search_borrowed, greedy_search, greedy_search_borrowed}; +use crate::vector::graph::{ + Visited, beam_search_acorn, beam_search_borrowed, greedy_search, greedy_search_borrowed, +}; use crate::vector::storage::{DistCalculator, VectorStore}; use crate::vector::v3::subindex::IvfSubIndex; -use crate::vector::{Query, VECTOR_RESULT_SCHEMA}; +use crate::vector::{ApproxMode, Query, VECTOR_RESULT_SCHEMA}; pub const HNSW_METADATA_KEY: &str = "lance:hnsw"; @@ -56,10 +57,23 @@ pub const HNSW_METADATA_KEY: &str = "lance:hnsw"; /// ([`super::online::OnlineHnswBuilder`]) builders so both produce comparable graphs. pub(crate) const HNSW_LEVEL_RNG_SEED: u64 = 42; +/// Minimum edge count that avoids severely fragmented graphs during parallel +/// construction while still allowing deliberately small test and index sizes. +pub(crate) const MIN_HNSW_M: usize = 4; + +/// Draw a node level using the distribution from Algorithm 1. +pub(crate) fn random_level_with(params: &HnswBuildParams, rng: &mut R) -> u16 { + let ml = 1.0 / (params.m as f32).ln(); + min( + (-rng.random::().ln() * ml) as u16, + params.max_level - 1, + ) +} + /// Parameters of building HNSW index #[derive(Debug, Clone, Serialize, Deserialize, DeepSizeOf)] pub struct HnswBuildParams { - /// max level ofm + /// Maximum number of levels in the graph. pub max_level: u16, /// number of connections to establish while inserting new element @@ -95,14 +109,17 @@ impl Default for HnswBuildParams { impl HnswBuildParams { /// The maximum level of the graph. - /// The default value is `8`. + /// The default value is `7`. pub fn max_level(mut self, max_level: u16) -> Self { self.max_level = max_level; self } /// The number of connections to establish while inserting new element - /// The default value is `30`. + /// + /// Must be at least 4. Smaller values produce severely fragmented graphs + /// during parallel construction. + /// The default value is `20`. pub fn num_edges(mut self, m: usize) -> Self { self.m = m; self @@ -111,12 +128,41 @@ impl HnswBuildParams { /// Number of candidates to be considered when searching for the nearest neighbors /// during the construction of the graph. /// - /// The default value is `100`. + /// The default value is `150`. pub fn ef_construction(mut self, ef_construction: usize) -> Self { self.ef_construction = ef_construction; self } + pub(crate) fn validate(&self) -> Result<()> { + if self.max_level == 0 { + return Err(Error::invalid_input(format!( + "HnswBuildParams::max_level must be greater than 0, got {}", + self.max_level + ))); + } + if self.m < MIN_HNSW_M { + return Err(Error::invalid_input(format!( + "HnswBuildParams::m must be at least {MIN_HNSW_M} to avoid severely fragmented graphs, got {}", + self.m, + ))); + } + if self.m > usize::MAX / 2 { + return Err(Error::invalid_input(format!( + "HnswBuildParams::m must be at most {} so the level-0 reciprocal limit can be represented, got {}", + usize::MAX / 2, + self.m + ))); + } + if self.ef_construction < self.m { + return Err(Error::invalid_input(format!( + "HnswBuildParams::ef_construction must be at least m ({}), got {}", + self.m, self.ef_construction + ))); + } + Ok(()) + } + /// Build the HNSW index from the given data. /// /// # Parameters @@ -176,7 +222,10 @@ impl DeepSizeOf for HnswCore { impl HnswCore { fn max_level(&self) -> u16 { - self.params.max_level + self.level_count + .iter() + .rposition(|count| *count != 0) + .map_or(0, |level| level + 1) as u16 } fn num_nodes(&self, level: usize) -> usize { @@ -338,8 +387,13 @@ impl HNSW { L: BorrowingGraph, B: BorrowingGraph, { + // Greedy descent stops at level 1 (HNSW paper, Algorithm 2): level 0 + // is searched only by the ef-bounded beam below. Greedily descending + // into level 0 first collapses the entry point into a deep local + // minimum, which costs extra distance computations and measurably + // hurts recall at low ef (https://github.com/lance-format/lance/issues/5208). let mut ep = ep; - for level in (0..self.max_level()).rev() { + for level in (1..self.max_level()).rev() { let cur_level = make_level(level); ep = greedy_search_borrowed( &cur_level, @@ -397,6 +451,141 @@ impl HNSW { result } + /// Like [Self::search_basic] but the bottom level runs + /// [beam_search_acorn], which only scores mask-passing nodes. + pub fn search_acorn( + &self, + query: ArrayRef, + k: usize, + params: &HnswQueryParams, + bitset: &Visited, + storage: &impl VectorStore, + ) -> Result> { + let mut visited_generator = self + .inner + .visited_generator_queue + .pop() + .unwrap_or_else(|| VisitedGenerator::new(storage.len())); + let mut expanded_generator = self + .inner + .visited_generator_queue + .pop() + .unwrap_or_else(|| VisitedGenerator::new(storage.len())); + + let result = self.search_acorn_inner( + query, + k, + params, + bitset, + &mut visited_generator, + &mut expanded_generator, + storage, + Some(2), + ); + + // if the queue is full, we just don't push it back, so ignore the error here + let _ = self.inner.visited_generator_queue.push(visited_generator); + let _ = self.inner.visited_generator_queue.push(expanded_generator); + result + } + + #[allow(clippy::too_many_arguments)] + fn search_acorn_inner( + &self, + query: ArrayRef, + k: usize, + params: &HnswQueryParams, + bitset: &Visited, + visited_generator: &mut VisitedGenerator, + expanded_generator: &mut VisitedGenerator, + storage: &impl VectorStore, + prefetch_distance: Option, + ) -> Result> { + let dist_calc = storage.dist_calculator(query, params.dist_q_c); + let entry = self.inner.entry_point; + let ep = OrderedNode::new(entry, dist_calc.distance(entry).into()); + + let result = match &self.inner.graph { + HnswGraph::Built(nodes) => { + let nodes = nodes.as_slice(); + self.run_search_acorn( + ep, + params, + bitset, + visited_generator, + expanded_generator, + storage.len(), + prefetch_distance, + &dist_calc, + |level| ImmutableHnswLevelView::new(level, nodes), + ImmutableHnswBottomView::new(nodes), + ) + } + HnswGraph::Loaded(graph) => { + let graph = graph.as_ref(); + self.run_search_acorn( + ep, + params, + bitset, + visited_generator, + expanded_generator, + storage.len(), + prefetch_distance, + &dist_calc, + |level| LoadedHnswLevelView::new(level, graph), + LoadedHnswBottomView::new(graph), + ) + } + }; + Ok(result.into_iter().take(k).collect()) + } + + /// [Self::run_search] for the ACORN traversal: same level descent, but + /// the bottom level runs [beam_search_acorn]. + #[allow(clippy::too_many_arguments)] + fn run_search_acorn( + &self, + ep: OrderedNode, + params: &HnswQueryParams, + bitset: &Visited, + visited_generator: &mut VisitedGenerator, + expanded_generator: &mut VisitedGenerator, + storage_len: usize, + prefetch_distance: Option, + dist_calc: &impl DistCalculator, + make_level: impl Fn(u16) -> L, + bottom: B, + ) -> Vec + where + L: BorrowingGraph, + B: BorrowingGraph, + { + // Same level descent as [Self::run_search]: greedy stops at level 1 + // (see the comment there); level 0 is left to the beam below. + let mut ep = ep; + for level in (1..self.max_level()).rev() { + let cur_level = make_level(level); + ep = greedy_search_borrowed( + &cur_level, + ep, + dist_calc, + self.inner.params.prefetch_distance, + ); + } + let mut visited = visited_generator.generate(storage_len); + let mut expanded = expanded_generator.generate(storage_len); + beam_search_acorn( + &bottom, + &ep, + params, + dist_calc, + bitset, + prefetch_distance, + &mut visited, + &mut expanded, + ) + } + #[instrument(level = "debug", skip(self, storage, query, prefilter_bitset))] fn flat_search( &self, @@ -464,19 +653,18 @@ impl HNSW { /// Returns the metadata of this [`HNSW`]. pub fn metadata(&self) -> HnswMetadata { - // calculate the offsets of each level, - // start from 0 - let level_offsets = self - .inner - .level_count - .iter() - .chain(iter::once(&0)) - .scan(0, |state, x| { - let start = *state; - *state += *x; - Some(start) - }) - .collect(); + // Version-1 readers use params.max_level as the number of level + // batches and index them directly. Preserve that configured shape, + // padding levels above the sampled graph height with empty ranges. + let configured_levels = self.inner.params.max_level as usize; + let mut level_offsets = Vec::with_capacity(configured_levels + 1); + let mut offset = 0; + level_offsets.push(0); + for level in 0..configured_levels { + let level_count = self.inner.level_count.get(level).copied().unwrap_or(0); + offset += level_count; + level_offsets.push(offset); + } HnswMetadata { entry_point: self.inner.entry_point, @@ -508,7 +696,7 @@ impl DeepSizeOf for HnswBuilder { impl HnswBuilder { fn finish(self) -> HNSW { - let nodes = match Arc::try_unwrap(self.nodes) { + let nodes: Vec = match Arc::try_unwrap(self.nodes) { Ok(nodes) => nodes .into_iter() .map(|node| node.into_inner().expect("builder lock poisoned")) @@ -519,9 +707,14 @@ impl HnswBuilder { .collect(), }; + let actual_levels = nodes + .get(self.entry_point as usize) + .map(|node| node.level_neighbors.len()) + .unwrap_or(0); let level_count = self .level_count .into_iter() + .take(actual_levels) .map(|count| count.load(Ordering::Relaxed)) .collect(); @@ -564,34 +757,24 @@ impl HnswBuilder { } let mut nodes = Vec::with_capacity(len); - { - if len > 0 { - nodes.push(RwLock::new(GraphBuilderNode::new(0, max_level as usize))); - } - let mut level_rng = SmallRng::seed_from_u64(HNSW_LEVEL_RNG_SEED); - for i in 1..len { - nodes.push(RwLock::new(GraphBuilderNode::new( - i as u32, - builder.random_level(&mut level_rng) as usize + 1, - ))); + let mut level_rng = SmallRng::seed_from_u64(HNSW_LEVEL_RNG_SEED); + let mut highest_level = 0; + for i in 0..len { + let target_level = random_level_with(&builder.params, &mut level_rng); + if target_level > highest_level { + highest_level = target_level; + builder.entry_point = i as u32; } + nodes.push(RwLock::new(GraphBuilderNode::new( + i as u32, + target_level as usize + 1, + ))); } builder.nodes = Arc::new(nodes); builder } - /// New node's level - /// - /// See paper `Algorithm 1` - fn random_level(&self, rng: &mut R) -> u16 { - let ml = 1.0 / (self.params.m as f32).ln(); - min( - (-rng.random::().ln() * ml) as u16, - self.params.max_level - 1, - ) - } - /// Insert one node. fn insert( &self, @@ -601,6 +784,12 @@ impl HnswBuilder { ) { let nodes = &self.nodes; let target_level = nodes[node as usize].read().unwrap().level_neighbors.len() as u16 - 1; + let entry_level = nodes[self.entry_point as usize] + .read() + .unwrap() + .level_neighbors + .len() as u16 + - 1; let dist_calc = storage.dist_calculator_from_id(node); let mut ep = OrderedNode::new( self.entry_point, @@ -615,7 +804,7 @@ impl HnswBuilder { // ep = Select-Neighbors(W, 1) // } // ``` - for level in (target_level + 1..self.params.max_level).rev() { + for level in (target_level + 1..=entry_level).rev() { let cur_level = HnswLevelView::new(level, nodes); ep = greedy_search(&cur_level, ep, &dist_calc, self.params.prefetch_distance); } @@ -631,7 +820,9 @@ impl HnswBuilder { for neighbor in &neighbors { current_node.add_neighbor(neighbor.id, neighbor.dist, level); } - self.prune(storage, &mut current_node, level); + // Algorithm 1 selects M neighbors for the new node. Mmax0 is + // reserved for reciprocal edges on existing level-0 nodes. + self.prune(storage, &mut current_node, level, self.params.m); pruned_neighbors_per_level[level as usize] .clone_from(¤t_node.level_neighbors_ranked[level as usize]); @@ -639,22 +830,19 @@ impl HnswBuilder { } } for (level, pruned_neighbors) in pruned_neighbors_per_level.iter().enumerate() { - for unpruned_edge in pruned_neighbors { - let level = level as u16; - let m_max = match level { - 0 => self.params.m * 2, - _ => self.params.m, - }; - if unpruned_edge.dist - < nodes[unpruned_edge.id as usize] - .read() - .unwrap() - .cutoff(level, m_max) - { - let mut chosen_node = nodes[unpruned_edge.id as usize].write().unwrap(); - chosen_node.add_neighbor(node, unpruned_edge.dist, level); - self.prune(storage, &mut chosen_node, level); - } + let level = level as u16; + let reciprocal_limit = if level == 0 { + self.params.m * 2 + } else { + self.params.m + }; + for selected_edge in pruned_neighbors { + // Algorithm 1 adds the reciprocal candidate before applying + // SELECT-NEIGHBORS. A distance-only cutoff would incorrectly + // discard farther candidates that improve directional diversity. + let mut chosen_node = nodes[selected_edge.id as usize].write().unwrap(); + chosen_node.add_neighbor(node, selected_edge.dist, level); + self.prune(storage, &mut chosen_node, level, reciprocal_limit); } } } @@ -677,6 +865,7 @@ impl HnswBuilder { lower_bound: None, upper_bound: None, dist_q_c: 0.0, + use_acorn: false, }, dist_calc, None, @@ -685,20 +874,22 @@ impl HnswBuilder { ) } - fn prune(&self, storage: &impl VectorStore, builder_node: &mut GraphBuilderNode, level: u16) { - let m_max = match level { - 0 => self.params.m * 2, - _ => self.params.m, - }; - + fn prune( + &self, + storage: &impl VectorStore, + builder_node: &mut GraphBuilderNode, + level: u16, + max_connections: usize, + ) { let neighbors_ranked = &mut builder_node.level_neighbors_ranked[level as usize]; - if neighbors_ranked.len() <= m_max { + if neighbors_ranked.len() <= max_connections { builder_node.update_from_ranked_neighbors(level); return; } let level_neighbors = std::mem::take(neighbors_ranked); - *neighbors_ranked = select_neighbors_heuristic_owned(storage, level_neighbors, m_max); + *neighbors_ranked = + select_neighbors_heuristic_owned(storage, level_neighbors, max_connections); builder_node.update_from_ranked_neighbors(level); } } @@ -798,12 +989,12 @@ enum LevelLookup { /// `__vector_id` column. /// /// We do *not* assume the column is sorted or that the slice is aligned - /// to a true level boundary: `level_offsets`/`level_count` omit the - /// entry-point node (it is written at every level by `to_batch` but only - /// counted at level 0), so upper-level slices can be off-by-one and - /// non-monotonic. Keying by the `__vector_id` value -- exactly what the - /// old per-node `load` did -- preserves behavior bit-for-bit. Upper - /// levels shrink geometrically, so this map stays tiny. + /// to a true level boundary. Indices written before issue #5156 was fixed + /// omitted the entry point from every upper-level count, so their slices + /// start progressively earlier than the true boundaries. Keying by the + /// `__vector_id` value -- exactly what the old per-node `load` did -- + /// preserves their behavior bit-for-bit. Upper levels shrink + /// geometrically, so this map stays tiny. Sparse(HashMap), } @@ -965,6 +1156,7 @@ pub struct HnswQueryParams { pub lower_bound: Option, pub upper_bound: Option, pub dist_q_c: f32, + pub use_acorn: bool, } impl From<&Query> for HnswQueryParams { @@ -975,6 +1167,7 @@ impl From<&Query> for HnswQueryParams { lower_bound: query.lower_bound, upper_bound: query.upper_bound, dist_q_c: query.dist_q_c, + use_acorn: query.approx_mode == ApproxMode::Fast, } } } @@ -1125,6 +1318,15 @@ impl IvfSubIndex for HNSW { .into() } + // `schema()` governs the on-disk index file, and readers older than v8.0.0 + // index the distance column there unconditionally, panicking when it is + // absent, so it cannot be dropped from what we write. Skipping it on read + // costs nothing in compatibility and keeps most of the graph bytes off the + // wire. + fn read_columns() -> Option<&'static [&'static str]> { + Some(&[VECTOR_ID_COL, NEIGHBORS_COL]) + } + #[instrument(level = "debug", skip(self, query, storage, prefilter, _metrics))] fn search( &self, @@ -1151,27 +1353,40 @@ impl IvfSubIndex for HNSW { .visited_generator_queue .pop() .unwrap_or_else(|| VisitedGenerator::new(storage.len())); - let prefilter_bitset = if prefilter.is_empty() { - None + let results = if prefilter.is_empty() { + self.search_basic(query, k, ¶ms, None, storage)? } else { + // the bitset must be moved into a callee on every path so its + // borrow of `prefilter_generator` ends before the push below let indices = prefilter.filter_row_ids(Box::new(storage.row_ids())); - let mut bitset = prefilter_generator.generate(storage.len()); - for indices in indices { - bitset.insert(indices as u32); + let mut prefilter_bitset = prefilter_generator.generate(storage.len()); + for index in indices { + prefilter_bitset.insert(index as u32); + } + let remained = prefilter_bitset.count_ones(); + if remained == storage.len() { + // mask passes every row: same as unfiltered + drop(prefilter_bitset); + self.search_basic(query, k, ¶ms, None, storage)? + } else if remained < self.len() * 10 / 100 { + // few matching rows: brute force is cheaper and exact + self.flat_search(storage, query, k, prefilter_bitset, ¶ms) + } else if params.use_acorn { + let acorn_results = + self.search_acorn(query.clone(), k, ¶ms, &prefilter_bitset, storage)?; + // under-delivery means the budget ran out on a fragmented + // mask, except range-bounded queries which return short + // legitimately + let bounded = params.lower_bound.is_some() || params.upper_bound.is_some(); + if !bounded && acorn_results.len() < k.min(remained) { + self.search_basic(query, k, ¶ms, Some(prefilter_bitset), storage)? + } else { + drop(prefilter_bitset); + acorn_results + } + } else { + self.search_basic(query, k, ¶ms, Some(prefilter_bitset), storage)? } - Some(bitset) - }; - - let remained = prefilter_bitset - .as_ref() - .map(|b| b.count_ones()) - .unwrap_or(storage.len()); - let results = if remained < self.len() * 10 / 100 { - let prefilter_bitset = - prefilter_bitset.expect("the prefilter bitset must be set for flat search"); - self.flat_search(storage, query, k, prefilter_bitset, ¶ms) - } else { - self.search_basic(query, k, ¶ms, prefilter_bitset, storage)? }; // if the queue is full, we just don't push it back, so ignore the error here let _ = self.inner.visited_generator_queue.push(prefilter_generator); @@ -1193,6 +1408,7 @@ impl IvfSubIndex for HNSW { where Self: Sized, { + params.validate()?; let builder = HnswBuilder::with_params(params, storage); log::debug!( @@ -1209,13 +1425,23 @@ impl IvfSubIndex for HNSW { } let len = storage.len(); - builder.level_count[0].fetch_add(1, Ordering::Relaxed); - (1..len).into_par_iter().for_each_init( - || VisitedGenerator::new(len), - |visited_generator, node| { - builder.insert(node as u32, visited_generator, storage); - }, - ); + let entry_levels = builder.nodes[builder.entry_point as usize] + .read() + .unwrap() + .level_neighbors + .len(); + for count in builder.level_count.iter().take(entry_levels) { + count.fetch_add(1, Ordering::Relaxed); + } + (0..len) + .into_par_iter() + .filter(|node| *node as u32 != builder.entry_point) + .for_each_init( + || VisitedGenerator::new(len), + |visited_generator, node| { + builder.insert(node as u32, visited_generator, storage); + }, + ); assert_eq!(builder.level_count[0].load(Ordering::Relaxed), len); Ok(builder.finish()) @@ -1310,16 +1536,17 @@ impl IvfSubIndex for HNSW { #[cfg(test)] mod tests { use std::sync::Arc; + use std::sync::atomic::{AtomicUsize, Ordering}; - use arrow_array::{ArrayRef, FixedSizeListArray, RecordBatch, UInt8Array, UInt32Array}; + use arrow_array::{ + ArrayRef, FixedSizeListArray, Float32Array, RecordBatch, UInt8Array, UInt32Array, + }; use arrow_schema::Schema; use lance_arrow::FixedSizeListArrayExt; - use lance_core::deepsize::DeepSizeOf; - use lance_file::previous::{ - reader::FileReader as PreviousFileReader, - writer::{ - FileWriter as PreviousFileWriter, FileWriterOptions as PreviousFileWriterOptions, - }, + use lance_core::{Error, deepsize::DeepSizeOf}; + use lance_file::versions::v1::{ + reader::FileReader as V1FileReader, + writer::{FileWriter as V1FileWriter, FileWriterOptions as V1FileWriterOptions}, }; use lance_io::object_store::ObjectStore; use lance_linalg::distance::DistanceType; @@ -1327,21 +1554,35 @@ mod tests { use lance_table::io::manifest::ManifestDescribing; use lance_testing::datagen::generate_random_array; use object_store::path::Path; + use rand::{Rng, SeedableRng, rngs::SmallRng}; use rstest::rstest; - use super::HnswGraph; - use crate::scalar::IndexWriter; + use super::{ + HNSW_LEVEL_RNG_SEED, HNSW_METADATA_KEY, HnswBuilder, HnswGraph, ImmutableHnswBottomView, + ImmutableHnswLevelView, MIN_HNSW_M, random_level_with, + }; + use crate::vector::graph::builder::GraphBuilderNode; use crate::vector::storage::{DistCalculator, VectorStore}; use crate::vector::v3::subindex::IvfSubIndex; use crate::vector::{ flat::storage::{FlatBinStorage, FlatFloatStorage}, - graph::{DISTS_FIELD, NEIGHBORS_FIELD}, + graph::{DISTS_FIELD, NEIGHBORS_FIELD, OrderedNode, VisitedGenerator}, hnsw::{ - HNSW, VECTOR_ID_FIELD, + HNSW, HnswMetadata, VECTOR_ID_FIELD, builder::{HnswBuildParams, HnswQueryParams}, }, }; + fn with_hnsw_metadata(batch: &RecordBatch, hnsw_metadata: HnswMetadata) -> RecordBatch { + let mut metadata = batch.schema_ref().metadata().clone(); + metadata.insert( + HNSW_METADATA_KEY.to_string(), + serde_json::to_string(&hnsw_metadata).unwrap(), + ); + let schema = batch.schema().as_ref().clone().with_metadata(metadata); + RecordBatch::try_new(Arc::new(schema), batch.columns().to_vec()).unwrap() + } + #[tokio::test] async fn test_builder_write_load() { const DIM: usize = 32; @@ -1367,18 +1608,18 @@ mod tests { DISTS_FIELD.clone(), ]); let schema = lance_core::datatypes::Schema::try_from(&schema).unwrap(); - let mut writer = PreviousFileWriter::::with_object_writer( + let mut writer = V1FileWriter::::with_object_writer( writer, schema, - &PreviousFileWriterOptions::default(), + &V1FileWriterOptions::default(), ) .unwrap(); let batch = builder.to_batch().unwrap(); let metadata = batch.schema_ref().metadata().clone(); - writer.write_record_batch(batch).await.unwrap(); + writer.write(&[batch]).await.unwrap(); writer.finish_with_metadata(&metadata).await.unwrap(); - let reader = PreviousFileReader::try_new_self_described(&object_store, &path, None) + let reader = V1FileReader::try_new_self_described(&object_store, &path, None) .await .unwrap(); let batch = reader @@ -1394,6 +1635,7 @@ mod tests { lower_bound: None, upper_bound: None, dist_q_c: 0.0, + use_acorn: false, }; let builder_results = builder .search_basic(query.clone(), k, ¶ms, None, store.as_ref()) @@ -1428,18 +1670,18 @@ mod tests { DISTS_FIELD.clone(), ]); let schema = lance_core::datatypes::Schema::try_from(&schema).unwrap(); - let mut writer = PreviousFileWriter::::with_object_writer( + let mut writer = V1FileWriter::::with_object_writer( writer, schema, - &PreviousFileWriterOptions::default(), + &V1FileWriterOptions::default(), ) .unwrap(); let batch = builder.to_batch().unwrap(); let metadata = batch.schema_ref().metadata().clone(); - writer.write_record_batch(batch).await.unwrap(); + writer.write(&[batch]).await.unwrap(); writer.finish_with_metadata(&metadata).await.unwrap(); - let reader = PreviousFileReader::try_new_self_described(&object_store, &path, None) + let reader = V1FileReader::try_new_self_described(&object_store, &path, None) .await .unwrap(); let batch = reader @@ -1455,6 +1697,7 @@ mod tests { lower_bound: None, upper_bound: None, dist_q_c: 0.0, + use_acorn: false, }; let builder_results = builder .search_basic(query.clone(), k, ¶ms, None, store.as_ref()) @@ -1475,6 +1718,165 @@ mod tests { all.into_iter().take(k).map(|(_, id)| id).collect() } + #[rstest] + #[case::zero_max_level( + HnswBuildParams::default().max_level(0), + "max_level must be greater than 0" + )] + #[case::zero_m( + HnswBuildParams::default().num_edges(0), + "m must be at least 4" + )] + #[case::one_m( + HnswBuildParams::default().num_edges(1), + "m must be at least 4" + )] + #[case::three_m( + HnswBuildParams::default().num_edges(3), + "m must be at least 4" + )] + #[case::small_ef( + HnswBuildParams::default().num_edges(20).ef_construction(19), + "ef_construction must be at least m (20)" + )] + #[case::overflowing_level_zero_limit( + HnswBuildParams::default() + .num_edges(usize::MAX) + .ef_construction(usize::MAX), + "level-0 reciprocal limit can be represented" + )] + fn test_rejects_invalid_build_params( + #[case] params: HnswBuildParams, + #[case] expected_message: &str, + ) { + let fsl = + FixedSizeListArray::try_new_from_values(Float32Array::from(vec![0.0, 0.0]), 2).unwrap(); + let store = FlatFloatStorage::new(fsl, DistanceType::L2); + + let error = HNSW::index_vectors(&store, params).unwrap_err(); + assert!(matches!(error, Error::InvalidInput { .. })); + assert!( + error.to_string().contains(expected_message), + "unexpected error: {error}" + ); + } + + /// The lowest accepted construction settings retain a useful graph, but + /// do not promise full reachability. This seeded floor makes that quality + /// trade-off explicit and guards against catastrophic fragmentation. + #[test] + fn test_minimum_params_reachability() { + const DIM: usize = 32; + const TOTAL: usize = 2048; + let mut rng = SmallRng::seed_from_u64(0); + let values = Float32Array::from( + (0..TOTAL * DIM) + .map(|_| rng.random::()) + .collect::>(), + ); + let vectors = FixedSizeListArray::try_new_from_values(values, DIM as i32).unwrap(); + let store = FlatFloatStorage::new(vectors.clone(), DistanceType::L2); + let hnsw = HNSW::index_vectors( + &store, + HnswBuildParams::default() + .num_edges(MIN_HNSW_M) + .ef_construction(MIN_HNSW_M), + ) + .unwrap(); + let results = hnsw + .search_basic( + vectors.value(0), + TOTAL, + &HnswQueryParams { + ef: TOTAL, + lower_bound: None, + upper_bound: None, + dist_q_c: 0.0, + use_acorn: false, + }, + None, + &store, + ) + .unwrap(); + + let minimum_reachable = TOTAL * 90 / 100; + assert!( + results.len() >= minimum_reachable, + "minimum HNSW construction settings reached only {} of {TOTAL} nodes; expected at least {minimum_reachable}", + results.len(), + ); + } + + /// Algorithm 1 limits a newly inserted node to M connections even on + /// level 0. The larger Mmax0 limit only applies when old nodes receive + /// reciprocal connections. + #[test] + fn test_new_node_uses_m_connections() { + // Four equidistant, mutually diverse points around the final point. + // With the old shared Mmax0 limit, node 4 retained all four. + let values = Float32Array::from(vec![ + 1.0, 0.0, // east + 0.0, 1.0, // north + -1.0, 0.0, // west + 0.0, -1.0, // south + 0.0, 0.0, // final node + ]); + let fsl = FixedSizeListArray::try_new_from_values(values, 2).unwrap(); + let store = FlatFloatStorage::new(fsl, DistanceType::L2); + let params = HnswBuildParams::default() + .max_level(1) + .num_edges(2) + .ef_construction(5); + let builder = HnswBuilder::with_params(params, &store); + let mut visited_generator = VisitedGenerator::new(store.len()); + + for node in 1..store.len() as u32 { + builder.insert(node, &mut visited_generator, &store); + } + + let final_node = builder.nodes[4].read().unwrap(); + assert_eq!(final_node.level_neighbors_ranked[0].len(), 2); + assert!( + builder + .nodes + .iter() + .all(|node| node.read().unwrap().level_neighbors_ranked[0].len() <= 4), + "existing level-0 nodes must remain bounded by Mmax0" + ); + } + + /// Offline construction pre-assigns the same random node heights as the + /// online builder and uses the first globally highest node as the anchor + /// for parallel insertion. This matches the final entry point produced by + /// sequential dynamic promotion without forcing node 0 to full height. + #[test] + fn test_offline_entry_point_uses_random_node_levels() { + const TOTAL: usize = 2048; + let fsl = + FixedSizeListArray::try_new_from_values(generate_random_array(TOTAL * 2), 2).unwrap(); + let store = FlatFloatStorage::new(fsl, DistanceType::L2); + let params = HnswBuildParams::default(); + let builder = HnswBuilder::with_params(params.clone(), &store); + + let mut level_rng = SmallRng::seed_from_u64(HNSW_LEVEL_RNG_SEED); + let expected_levels = (0..TOTAL) + .map(|_| random_level_with(¶ms, &mut level_rng)) + .collect::>(); + let highest_level = *expected_levels.iter().max().unwrap(); + let expected_entry = expected_levels + .iter() + .position(|level| *level == highest_level) + .unwrap() as u32; + + for (node, expected_level) in builder.nodes.iter().zip(expected_levels) { + assert_eq!( + node.read().unwrap().level_neighbors.len(), + expected_level as usize + 1 + ); + } + assert_eq!(builder.entry_point, expected_entry); + } + /// The Arrow-backed loaded graph must search bit-identically to the /// in-memory build, across distance types and graph sizes (single node, /// pair, and a multi-level graph exercising the sparse upper-level @@ -1511,6 +1913,7 @@ mod tests { lower_bound: None, upper_bound: None, dist_q_c: 0.0, + use_acorn: false, }; let query = fsl.value(0); @@ -1534,17 +1937,370 @@ mod tests { assert!(recall >= 0.5, "recall {recall} below 0.5 (k={k})"); } - /// Regression guard for the `level_offsets` misalignment (issue #6746). - /// `to_batch` writes the entry-point node at *every* level, but - /// `level_count` only counts it at level 0, so the serialized batch has - /// strictly more rows than `sum(level_count)` and the upper-level - /// `level_offsets` slices are off-by-one / non-monotonic. The Arrow-backed - /// loaded graph must still search bit-identically to the in-memory build: - /// it keys upper levels by `__vector_id` value via the `Sparse` map - /// (last-write-wins), never `row == id`. A naive `row == id` - /// reimplementation would pass the small cases but break here. + /// A [`DistCalculator`] over fixed per-node distances that counts how + /// many distances it computes, so a test can assert exactly which search + /// phases ran. + struct CountingDistCalculator<'a> { + distances: &'a [f32], + calls: &'a AtomicUsize, + } + + impl DistCalculator for CountingDistCalculator<'_> { + fn distance(&self, id: u32) -> f32 { + self.calls.fetch_add(1, Ordering::Relaxed); + self.distances[id as usize] + } + + fn distance_all(&self, _k_hint: usize) -> Vec { + self.distances.to_vec() + } + } + + /// Regression test for : + /// the query-time greedy descent must stop at level 1 (HNSW paper, + /// Algorithm 2); level 0 must be searched only by the ef-bounded beam. + /// + /// Drives [`HNSW::run_search`] / [`HNSW::run_search_acorn`] directly over + /// a hand-built graph: node 0 is the entry point and the only node at + /// level 1 (with no level-1 neighbors), so the greedy descent over + /// levels >= 1 computes no distances, and with ef == N the bottom beam + /// visits every level-0 node exactly once. A greedy step at level 0 + /// would show up as 10 extra distance computations: the level-0 degrees + /// along the path are 1,2,2,2,2,1 and the strictly decreasing distances + /// make greedy walk it end to end. + #[test] + fn test_greedy_descent_stops_before_level_0() { + const N: usize = 6; + // Strictly decreasing along the path; the true top-3 is nodes 5,4,3. + let distances: Vec = (0..N).map(|id| (N - id) as f32).collect(); + + // Level 0 is the path 0-1-...-5, mirrored into both the bottom view + // and the level-0 view (only the bottom view may be used at query + // time, which is what this test asserts). + let mut nodes: Vec = (0..N as u32) + .map(|id| GraphBuilderNode::new(id, 2)) + .collect(); + for (id, node) in nodes.iter_mut().enumerate() { + let mut adjacency = Vec::new(); + if id > 0 { + adjacency.push(id as u32 - 1); + } + if id + 1 < N { + adjacency.push(id as u32 + 1); + } + let adjacency = Arc::new(adjacency); + node.bottom_neighbors = adjacency.clone(); + node.level_neighbors[0] = adjacency; + } + + let build_params = HnswBuildParams { + max_level: 2, + m: 4, + ef_construction: 10, + prefetch_distance: None, + }; + let hnsw = HNSW::from_parts(build_params, nodes.clone(), vec![N, 1], 0); + let query_params = HnswQueryParams { + ef: N, + lower_bound: None, + upper_bound: None, + dist_q_c: 0.0, + use_acorn: false, + }; + + let calls = AtomicUsize::new(0); + let dist_calc = CountingDistCalculator { + distances: &distances, + calls: &calls, + }; + // The entry-point distance (1 call) mirrors `search_inner`. + let ep = OrderedNode::new(0, dist_calc.distance(0).into()); + let mut visited_generator = VisitedGenerator::new(N); + + let results = hnsw.run_search( + ep.clone(), + 3, + &query_params, + None, + &mut visited_generator, + N, + None, + &dist_calc, + |level| ImmutableHnswLevelView::new(level, &nodes), + ImmutableHnswBottomView::new(&nodes), + ); + assert_eq!( + results.iter().map(|node| node.id).collect::>(), + vec![5, 4, 3] + ); + // Exactly 1 entry-point distance + 5 beam visits (one per remaining + // node); the pre-fix level-0 greedy step brought this to 16. + assert_eq!(calls.load(Ordering::Relaxed), N); + + // The ACORN traversal shares the descent. Its beam additionally seeds + // mask-passing nodes with distance computations, so only bound the + // total: a level-0 greedy step would push it past one call per node. + let mut mask_generator = VisitedGenerator::new(N); + let mut mask = mask_generator.generate(N); + for id in 0..N as u32 { + mask.insert(id); + } + let mut expanded_generator = VisitedGenerator::new(N); + calls.store(0, Ordering::Relaxed); + let results = hnsw.run_search_acorn( + ep, + &query_params, + &mask, + &mut visited_generator, + &mut expanded_generator, + N, + None, + &dist_calc, + |level| ImmutableHnswLevelView::new(level, &nodes), + ImmutableHnswBottomView::new(&nodes), + ); + assert_eq!( + results + .iter() + .take(3) + .map(|node| node.id) + .collect::>(), + vec![5, 4, 3] + ); + assert!( + calls.load(Ordering::Relaxed) <= N, + "level-0 greedy descent adds distance computations beyond the \ + beam's one per node" + ); + } + + /// Brute-force top-`k` restricted to mask-passing ids. + fn brute_force_topk_masked( + store: &FlatFloatStorage, + query: ArrayRef, + k: usize, + passes: impl Fn(u32) -> bool, + ) -> Vec { + let dist_calc = store.dist_calculator(query, 0.0); + let mut matching: Vec<(f32, u32)> = (0..store.len() as u32) + .filter(|id| passes(*id)) + .map(|id| (dist_calc.distance(id), id)) + .collect(); + matching.sort_by(|a, b| a.0.partial_cmp(&b.0).unwrap()); + matching.into_iter().take(k).map(|(_, id)| id).collect() + } + + /// ACORN returns only mask-passing nodes, searches built and loaded + /// graphs identically, and holds recall vs brute force over the mask. #[tokio::test] - async fn test_loaded_level_offsets_misalignment_invariant() { + async fn test_acorn_filtered_search() { + const DIM: usize = 32; + const TOTAL: usize = 2048; + let fsl = + FixedSizeListArray::try_new_from_values(generate_random_array(TOTAL * DIM), DIM as i32) + .unwrap(); + let store = Arc::new(FlatFloatStorage::new(fsl.clone(), DistanceType::L2)); + let builder = HNSW::index_vectors( + store.as_ref(), + HnswBuildParams::default().num_edges(20).ef_construction(50), + ) + .unwrap(); + let loaded = HNSW::load(builder.to_batch().unwrap()).unwrap(); + + let mut mask_generator = VisitedGenerator::new(TOTAL); + let k = 10; + let params = HnswQueryParams { + ef: 50, + lower_bound: None, + upper_bound: None, + dist_q_c: 0.0, + use_acorn: false, + }; + let query = fsl.value(0); + let truth: std::collections::HashSet = + brute_force_topk_masked(store.as_ref(), query.clone(), k, |id| id % 2 == 0) + .into_iter() + .collect(); + + let mut all_results = vec![]; + for hnsw in [&builder, &loaded] { + let mut bitset = mask_generator.generate(TOTAL); + for id in (0..TOTAL as u32).step_by(2) { + bitset.insert(id); + } + let results = hnsw + .search_acorn(query.clone(), k, ¶ms, &bitset, store.as_ref()) + .unwrap(); + assert_eq!(results.len(), k); + assert!(results.iter().all(|node| node.id % 2 == 0)); + assert!(results.windows(2).all(|w| w[0].dist <= w[1].dist)); + let hits = results.iter().filter(|n| truth.contains(&n.id)).count(); + let recall = hits as f32 / k as f32; + assert!(recall >= 0.5, "recall {recall} below 0.5 (k={k})"); + all_results.push(results); + } + assert_eq!(all_results[0], all_results[1]); + + // default ef (k + k/2) and a deletion-style mask (all but a few rows) + let default_ef_params = HnswQueryParams { + ef: k + k / 2, + ..params + }; + for excluded_stride in [2, 400] { + let passes = |id: u32| id % excluded_stride != 1; + let mut bitset = mask_generator.generate(TOTAL); + for id in (0..TOTAL as u32).filter(|id| passes(*id)) { + bitset.insert(id); + } + let truth: std::collections::HashSet = + brute_force_topk_masked(store.as_ref(), query.clone(), k, passes) + .into_iter() + .collect(); + let results = builder + .search_acorn( + query.clone(), + k, + &default_ef_params, + &bitset, + store.as_ref(), + ) + .unwrap(); + assert_eq!(results.len(), k); + assert!(results.iter().all(|node| passes(node.id))); + let hits = results.iter().filter(|n| truth.contains(&n.id)).count(); + let recall = hits as f32 / k as f32; + assert!(recall >= 0.5, "recall {recall} below 0.5 (k={k})"); + } + } + + /// Dispatch: dense prefilters take the graph traversal, sparse ones the + /// exact flat scan, and both return only mask-passing row ids. + #[tokio::test] + async fn test_subindex_prefilter_dispatch() { + use arrow_array::cast::AsArray; + use async_trait::async_trait; + use lance_core::Result; + use lance_select::{RowAddrMask, RowAddrTreeMap}; + + use crate::metrics::NoOpMetricsCollector; + use crate::prefilter::PreFilter; + + struct MaskPreFilter { + mask: Arc, + } + + #[async_trait] + impl PreFilter for MaskPreFilter { + async fn wait_for_ready(&self) -> Result<()> { + Ok(()) + } + fn is_empty(&self) -> bool { + false + } + fn mask(&self) -> Arc { + self.mask.clone() + } + fn filter_row_ids<'a>( + &self, + row_ids: Box + 'a>, + ) -> Vec { + self.mask.selected_indices(row_ids) + } + } + + const DIM: usize = 32; + const TOTAL: usize = 2048; + let fsl = + FixedSizeListArray::try_new_from_values(generate_random_array(TOTAL * DIM), DIM as i32) + .unwrap(); + let store = Arc::new(FlatFloatStorage::new(fsl.clone(), DistanceType::L2)); + let hnsw = HNSW::index_vectors( + store.as_ref(), + HnswBuildParams::default().num_edges(20).ef_construction(50), + ) + .unwrap(); + + let k = 10; + let query_key = fsl.value(0); + + let search_row_ids = |allowed: Vec, use_acorn: bool| { + let params = HnswQueryParams { + ef: 50, + lower_bound: None, + upper_bound: None, + dist_q_c: 0.0, + use_acorn, + }; + let filter = Arc::new(MaskPreFilter { + mask: Arc::new(RowAddrMask::from_allowed(RowAddrTreeMap::from_iter( + allowed, + ))), + }); + let batch = hnsw + .search( + query_key.clone(), + k, + params, + store.as_ref(), + filter, + &NoOpMetricsCollector, + ) + .unwrap(); + batch[lance_core::ROW_ID] + .as_primitive::() + .values() + .to_vec() + }; + + // Dense mask (50% of rows), in both modes. + let dense: Vec = (0..TOTAL as u64).step_by(2).collect(); + for use_acorn in [false, true] { + let row_ids = search_row_ids(dense.clone(), use_acorn); + assert_eq!(row_ids.len(), k); + assert!(row_ids.iter().all(|id| id % 2 == 0)); + } + + // All-pass mask: shortcuts to the unfiltered path. + let all: Vec = (0..TOTAL as u64).collect(); + let unfiltered = hnsw + .search_basic( + query_key.clone(), + k, + &HnswQueryParams { + ef: 50, + lower_bound: None, + upper_bound: None, + dist_q_c: 0.0, + use_acorn: false, + }, + None, + store.as_ref(), + ) + .unwrap(); + let row_ids = search_row_ids(all, true); + assert_eq!( + row_ids, + unfiltered.iter().map(|n| n.id as u64).collect::>() + ); + + // Sparse mask (< 10% of rows): the flat scan, which is exact. + let sparse: Vec = (0..TOTAL as u64).step_by(25).collect(); + let row_ids = search_row_ids(sparse.clone(), true); + assert_eq!(row_ids.len(), k); + let truth = brute_force_topk_masked(store.as_ref(), query_key.clone(), k, |id| { + sparse.contains(&(id as u64)) + }); + let mut got: Vec = row_ids.iter().map(|id| *id as u32).collect(); + got.sort_unstable(); + let mut expected = truth; + expected.sort_unstable(); + assert_eq!(got, expected); + } + + /// Every fresh `level_offsets` range must exactly delimit the rows emitted + /// for that HNSW level (issue #5156). + #[test] + fn test_level_offsets_match_serialized_levels() { use arrow::array::AsArray; use arrow::datatypes::UInt32Type; @@ -1553,7 +2309,7 @@ mod tests { let fsl = FixedSizeListArray::try_new_from_values(generate_random_array(TOTAL * DIM), DIM as i32) .unwrap(); - let store = Arc::new(FlatFloatStorage::new(fsl.clone(), DistanceType::L2)); + let store = Arc::new(FlatFloatStorage::new(fsl, DistanceType::L2)); let builder = HNSW::index_vectors( store.as_ref(), HnswBuildParams::default().num_edges(20).ef_construction(50), @@ -1568,53 +2324,122 @@ mod tests { ); let batch = builder.to_batch().unwrap(); - let md = builder.metadata(); - let total_counted = *md.level_offsets.last().unwrap(); - - // The exact misalignment: more serialized rows than `level_count` sums - // to, because the entry-point node is written at every level yet - // counted only at level 0. - assert!( - batch.num_rows() > total_counted, - "expected serialized rows ({}) to exceed sum(level_count) ({}) -- \ - entry point should be written at every level", + let metadata = builder.metadata(); + assert_eq!( + *metadata.level_offsets.last().unwrap(), batch.num_rows(), - total_counted, + "level offsets must cover every serialized row", ); - // Level-0 slice must still be exactly `[0, N)` with - // `__vector_id == row` -- the precondition for `LevelLookup::Dense`. - let n = md.level_offsets[1]; - assert_eq!(n, TOTAL); - let level0 = batch.slice(0, n); - let ids = level0.column(0).as_primitive::(); - assert!( - ids.values() + let nodes = builder.nodes().unwrap(); + for level in 0..builder.max_level() as usize { + let start = metadata.level_offsets[level]; + let end = metadata.level_offsets[level + 1]; + let level_batch = batch.slice(start, end - start); + let ids = level_batch.column(0).as_primitive::(); + let expected_ids = nodes .iter() .enumerate() - .all(|(row, id)| *id == row as u32), - "level-0 __vector_id must equal the row index", + .filter_map(|(id, node)| (level < node.level_neighbors.len()).then_some(id as u32)) + .collect::>(); + + assert_eq!( + ids.values().as_ref(), + expected_ids.as_slice(), + "serialized ids do not match level {level}", + ); + assert_eq!(builder.num_nodes(level), expected_ids.len()); + } + } + + /// Version-1 readers use the configured max level to index serialized + /// level batches directly. Empty trailing ranges keep that persisted + /// shape while current readers use only the sampled, non-empty height. + #[test] + fn test_metadata_preserves_configured_empty_levels() { + const CONFIGURED_LEVELS: usize = 7; + let params = HnswBuildParams::default().max_level(CONFIGURED_LEVELS as u16); + let hnsw = HNSW::from_parts(params, vec![GraphBuilderNode::new(0, 1)], vec![1], 0); + + assert_eq!(hnsw.max_level(), 1); + let metadata = hnsw.metadata(); + assert_eq!(metadata.level_offsets.len(), CONFIGURED_LEVELS + 1); + assert_eq!(metadata.level_offsets[0], 0); + assert!( + metadata.level_offsets[1..] + .iter() + .all(|offset| *offset == 1) ); - // Despite the surplus rows and off-by-one upper slices, the loaded - // graph searches bit-identically to the in-memory build (old `load` - // semantics preserved via the `Sparse` last-write-wins map). - let loaded = HNSW::load(batch).unwrap(); + let loaded = HNSW::load(hnsw.to_batch().unwrap()).unwrap(); + assert_eq!(loaded.max_level(), 1); + match &loaded.inner.graph { + HnswGraph::Loaded(graph) => { + assert_eq!(graph.level_neighbors.len(), CONFIGURED_LEVELS); + assert_eq!(graph.level_count[0], 1); + assert!(graph.level_count[1..].iter().all(|count| *count == 0)); + } + HnswGraph::Built(_) => panic!("expected an Arrow-backed loaded graph"), + } + } + + /// Indices written before issue #5156 was fixed omitted the entry point + /// from every upper-level count. Loading those misaligned slices must keep + /// the previous id-keyed, last-write-wins behavior. + #[test] + fn test_load_legacy_misaligned_level_offsets() { + const DIM: usize = 32; + const TOTAL: usize = 2048; + let fsl = + FixedSizeListArray::try_new_from_values(generate_random_array(TOTAL * DIM), DIM as i32) + .unwrap(); + let store = Arc::new(FlatFloatStorage::new(fsl.clone(), DistanceType::L2)); + let builder = HNSW::index_vectors( + store.as_ref(), + HnswBuildParams::default().num_edges(20).ef_construction(50), + ) + .unwrap(); + assert!(builder.max_level() >= 2); + + let batch = builder.to_batch().unwrap(); + let mut metadata = builder.metadata(); + let mut legacy_offsets = Vec::with_capacity(metadata.level_offsets.len()); + legacy_offsets.push(0); + for level in 0..builder.max_level() as usize { + let level_count = metadata.level_offsets[level + 1] - metadata.level_offsets[level]; + let legacy_level_count = level_count - usize::from(level > 0); + legacy_offsets.push(legacy_offsets.last().unwrap() + legacy_level_count); + } + metadata.level_offsets = legacy_offsets; + assert!(*metadata.level_offsets.last().unwrap() < batch.num_rows()); + + let legacy_batch = with_hnsw_metadata(&batch, metadata); + let loaded = HNSW::load(legacy_batch).unwrap(); assert!(matches!(loaded.inner.graph, HnswGraph::Loaded(_))); let params = HnswQueryParams { ef: 50, lower_bound: None, upper_bound: None, dist_q_c: 0.0, + use_acorn: false, }; - let query = fsl.value(0); - let builder_results = builder - .search_basic(query.clone(), 10, ¶ms, None, store.as_ref()) - .unwrap(); - let loaded_results = loaded - .search_basic(query, 10, ¶ms, None, store.as_ref()) - .unwrap(); - assert_eq!(builder_results, loaded_results); + let entry_point = builder.inner.entry_point as usize; + let query_indices = [0, 1, TOTAL / 3, TOTAL - 1] + .into_iter() + .filter(|query_index| *query_index != entry_point) + .take(3) + .collect::>(); + assert_eq!(query_indices.len(), 3); + for query_index in query_indices { + let query = fsl.value(query_index); + let builder_results = builder + .search_basic(query.clone(), 10, ¶ms, None, store.as_ref()) + .unwrap(); + let loaded_results = loaded + .search_basic(query, 10, ¶ms, None, store.as_ref()) + .unwrap(); + assert_eq!(builder_results, loaded_results); + } } /// `load()` must reject a batch whose level-0 `__vector_id` no longer @@ -1664,8 +2489,6 @@ mod tests { /// time. #[tokio::test] async fn test_load_rejects_out_of_range_entry_point() { - use super::{HNSW_METADATA_KEY, HnswMetadata}; - const DIM: usize = 16; const TOTAL: usize = 256; let fsl = @@ -1679,21 +2502,18 @@ mod tests { .unwrap(); let batch = builder.to_batch().unwrap(); - let mut metadata = batch.schema_ref().metadata().clone(); - let mut md: HnswMetadata = - serde_json::from_str(metadata.get(HNSW_METADATA_KEY).unwrap()).unwrap(); + let mut md: HnswMetadata = serde_json::from_str( + batch + .schema_ref() + .metadata() + .get(HNSW_METADATA_KEY) + .unwrap(), + ) + .unwrap(); // Valid entry points are `[0, N)`; `level_offsets[1]` == N is one past. let n = md.level_offsets[1]; md.entry_point = n as u32; - metadata.insert( - HNSW_METADATA_KEY.to_string(), - serde_json::to_string(&md).unwrap(), - ); - // Rebuild the batch under the rewritten metadata. `with_schema` would - // reject this: it requires the new metadata to be a superset, but we - // are changing an existing key's value, not adding one. - let schema = batch.schema().as_ref().clone().with_metadata(metadata); - let corrupted = RecordBatch::try_new(Arc::new(schema), batch.columns().to_vec()).unwrap(); + let corrupted = with_hnsw_metadata(&batch, md); assert!( HNSW::load(corrupted).is_err(), @@ -1752,6 +2572,7 @@ mod tests { lower_bound: None, upper_bound: None, dist_q_c: 0.0, + use_acorn: false, }; let query = fsl.value(7); let a = builder diff --git a/rust/lance-index/src/vector/hnsw/index.rs b/rust/lance-index/src/vector/hnsw/index.rs index dca9ea78955..4a5d23e63f2 100644 --- a/rust/lance-index/src/vector/hnsw/index.rs +++ b/rust/lance-index/src/vector/hnsw/index.rs @@ -16,7 +16,7 @@ use lance_arrow::RecordBatchExt; use lance_core::ROW_ID; use lance_core::deepsize::DeepSizeOf; use lance_core::{Error, Result, datatypes::Schema}; -use lance_file::previous::reader::FileReader as PreviousFileReader; +use lance_file::versions::v1::reader::FileReader as V1FileReader; use lance_io::traits::Reader; use lance_linalg::distance::DistanceType; use lance_table::format::SelfDescribingFileReader; @@ -70,8 +70,7 @@ impl HNSWIndex { aux_reader: Arc, options: HNSWIndexOptions, ) -> Result { - let reader = - PreviousFileReader::try_new_self_described_from_reader(reader.clone(), None).await?; + let reader = V1FileReader::try_new_self_described_from_reader(reader.clone(), None).await?; let partition_metadata = match reader.schema().metadata.get(IVF_PARTITION_KEY) { Some(json) => { @@ -215,7 +214,7 @@ impl VectorIndex for HNSWIndex { VECTOR_ID_FIELD.clone(), ]))?; - let reader = PreviousFileReader::try_new_from_reader( + let reader = V1FileReader::try_new_from_reader( reader.path(), reader.clone(), None, @@ -247,7 +246,7 @@ impl VectorIndex for HNSWIndex { length: usize, partition_id: usize, ) -> Result> { - let reader = PreviousFileReader::try_new_self_described_from_reader(reader, None).await?; + let reader = V1FileReader::try_new_self_described_from_reader(reader, None).await?; let metadata = self.get_partition_metadata(partition_id)?; let storage = Arc::new(self.partition_storage.load_partition(partition_id).await?); diff --git a/rust/lance-index/src/vector/hnsw/online.rs b/rust/lance-index/src/vector/hnsw/online.rs index 8fbdcbcb1c5..566e63cbd1a 100644 --- a/rust/lance-index/src/vector/hnsw/online.rs +++ b/rust/lance-index/src/vector/hnsw/online.rs @@ -18,7 +18,7 @@ //! //! # Lifecycle //! -//! 1. `OnlineHnswBuilder::with_capacity(...)` pre-allocates fixed-size node +//! 1. `OnlineHnswBuilder::try_with_capacity(...)` pre-allocates fixed-size node //! arrays. Each slot has its target level pre-assigned so concurrent //! inserts don't need to allocate. //! 2. Writer calls `insert(id, storage)` for `id` in `0..capacity`. The vector @@ -29,22 +29,24 @@ //! 4. `finalize()` consumes the builder and returns an immutable //! [`super::HNSW`] that can be serialized via `HNSW::to_batch()`. -use std::cmp::min; use std::sync::Arc; use std::sync::Mutex; use std::sync::atomic::{AtomicU32, AtomicUsize, Ordering}; use arc_swap::ArcSwap; use crossbeam_queue::ArrayQueue; -use rand::{Rng, SeedableRng, rngs::SmallRng}; +use rand::{SeedableRng, rngs::SmallRng}; -use super::builder::{HNSW, HNSW_LEVEL_RNG_SEED, HnswBuildParams, HnswQueryParams}; +use super::builder::{ + HNSW, HNSW_LEVEL_RNG_SEED, HnswBuildParams, HnswQueryParams, random_level_with, +}; use super::select_neighbors_heuristic; use crate::vector::graph::builder::GraphBuilderNode; use crate::vector::graph::{ Graph, OrderedFloat, OrderedNode, VisitedGenerator, beam_search, greedy_search, }; use crate::vector::storage::{DistCalculator, VectorStore}; +use lance_core::Result; use lance_core::utils::tokio::get_num_compute_intensive_cpus; /// A node in the online HNSW graph. @@ -97,22 +99,6 @@ impl OnlineGraphBuilderNode { ranked[level as usize].push(OrderedNode { dist, id: v }); } - fn cutoff(&self, level: u16, max_size: usize) -> OrderedFloat { - if !self.has_level(level) { - return OrderedFloat(f32::NEG_INFINITY); - } - let ranked = self - .level_neighbors_ranked - .lock() - .expect("level_neighbors_ranked mutex poisoned"); - let neighbors = &ranked[level as usize]; - if neighbors.len() < max_size { - OrderedFloat(f32::INFINITY) - } else { - neighbors.last().unwrap().dist - } - } - /// Rebuild `level_neighbors[level]` from the current ranked list and /// publish it via `ArcSwap`. Also updates `bottom_neighbors` for level 0. fn publish_from_ranked(&self, level: u16) { @@ -156,28 +142,48 @@ pub struct OnlineHnswBuilder { } impl OnlineHnswBuilder { - /// Create a new builder with the given capacity. Each node's target level - /// is pre-assigned at random. + /// Create a new builder with validated parameters. + /// + /// Each node's target level is pre-assigned at random. + /// + /// # Examples + /// + /// ``` + /// use lance_index::vector::hnsw::{OnlineHnswBuilder, builder::HnswBuildParams}; + /// + /// let builder = OnlineHnswBuilder::try_with_capacity( + /// 1_024, + /// HnswBuildParams::default(), + /// )?; + /// assert_eq!(builder.capacity(), 1_024); + /// # Ok::<(), lance_core::Error>(()) + /// ``` + pub fn try_with_capacity(capacity: usize, params: HnswBuildParams) -> Result { + params.validate()?; + Ok(Self::new(capacity, params)) + } + + /// Create a new builder with the given capacity. + /// + /// Use [`Self::try_with_capacity`] to handle invalid parameters without + /// panicking. + /// + /// # Panics + /// + /// Panics when `params` violates an HNSW construction precondition. + #[deprecated(note = "use OnlineHnswBuilder::try_with_capacity")] pub fn with_capacity(capacity: usize, params: HnswBuildParams) -> Self { - assert!( - params.max_level > 0, - "HnswBuildParams::max_level must be > 0" - ); + Self::try_with_capacity(capacity, params) + .unwrap_or_else(|error| panic!("invalid HNSW build parameters: {error}")) + } + + fn new(capacity: usize, params: HnswBuildParams) -> Self { let max_level = params.max_level; let level_count = (0..max_level).map(|_| AtomicUsize::new(0)).collect(); let mut level_rng = SmallRng::seed_from_u64(HNSW_LEVEL_RNG_SEED); let nodes: Vec<_> = (0..capacity) - .map(|i| { - let target_level = if i == 0 { - // First inserted node anchors the graph; matches offline - // builder which always starts at level 0. - 0 - } else { - Self::random_level_with(¶ms, &mut level_rng) - }; - OnlineGraphBuilderNode::new(target_level) - }) + .map(|_| OnlineGraphBuilderNode::new(random_level_with(¶ms, &mut level_rng))) .collect(); let queue_size = get_num_compute_intensive_cpus().max(1); @@ -196,14 +202,6 @@ impl OnlineHnswBuilder { } } - fn random_level_with(params: &HnswBuildParams, rng: &mut R) -> u16 { - let ml = 1.0 / (params.m as f32).ln(); - min( - (-rng.random::().ln() * ml) as u16, - params.max_level - 1, - ) - } - pub fn capacity(&self) -> usize { self.nodes.len() } @@ -258,10 +256,11 @@ impl OnlineHnswBuilder { } let entry = self.entry_point.load(Ordering::Acquire); + let entry_target_level = nodes[entry as usize].target_level(); let mut ep = OrderedNode::new(entry, dist_calc.distance(entry).into()); // Walk down upper levels with greedy_search to refine the entry point. - for level in (target_level + 1..self.params.max_level).rev() { + for level in (target_level + 1..=entry_target_level).rev() { let cur_level = OnlineHnswLevelView::new(level, nodes); ep = greedy_search(&cur_level, ep, &dist_calc, self.params.prefetch_distance); } @@ -284,7 +283,9 @@ impl OnlineHnswBuilder { } current_node.add_neighbor(neighbor.id, neighbor.dist, level); } - self.prune(storage, current_node, level); + // New nodes select M connections at every level. The larger + // level-0 limit applies only to reciprocal edges on old nodes. + self.prune(storage, current_node, level, self.params.m); // Snapshot the pruned ranked list before publishing. let snapshot = { let ranked = current_node @@ -310,18 +311,16 @@ impl OnlineHnswBuilder { // Add reverse edges to chosen neighbors, prune them too. for (level, pruned_neighbors) in pruned_neighbors_per_level.iter().enumerate() { let level = level as u16; - let m_max = if level == 0 { + let reciprocal_limit = if level == 0 { self.params.m * 2 } else { self.params.m }; - for unpruned_edge in pruned_neighbors { - let chosen = &nodes[unpruned_edge.id as usize]; - if unpruned_edge.dist < chosen.cutoff(level, m_max) { - chosen.add_neighbor(id, unpruned_edge.dist, level); - self.prune(storage, chosen, level); - chosen.publish_from_ranked(level); - } + for selected_edge in pruned_neighbors { + let chosen = &nodes[selected_edge.id as usize]; + chosen.add_neighbor(id, selected_edge.dist, level); + self.prune(storage, chosen, level, reciprocal_limit); + chosen.publish_from_ranked(level); } } @@ -331,7 +330,6 @@ impl OnlineHnswBuilder { // plain store to keep `entry_point` updates atomic against concurrent // searches and to leave the door open if the writer ever becomes // multi-threaded. - let entry_target_level = nodes[entry as usize].target_level(); if target_level > entry_target_level { let _ = self.entry_point @@ -359,6 +357,7 @@ impl OnlineHnswBuilder { lower_bound: None, upper_bound: None, dist_q_c: 0.0, + use_acorn: false, }, dist_calc, None, @@ -367,22 +366,23 @@ impl OnlineHnswBuilder { ) } - fn prune(&self, storage: &impl VectorStore, node: &OnlineGraphBuilderNode, level: u16) { - let m_max = if level == 0 { - self.params.m * 2 - } else { - self.params.m - }; - + fn prune( + &self, + storage: &impl VectorStore, + node: &OnlineGraphBuilderNode, + level: u16, + max_connections: usize, + ) { let mut ranked = node .level_neighbors_ranked .lock() .expect("level_neighbors_ranked mutex poisoned"); let level_neighbors = ranked[level as usize].clone(); - if level_neighbors.len() <= m_max { + if level_neighbors.len() <= max_connections { return; } - ranked[level as usize] = select_neighbors_heuristic(storage, &level_neighbors, m_max); + ranked[level as usize] = + select_neighbors_heuristic(storage, &level_neighbors, max_connections); } /// Search the graph for the k nearest neighbors of `query`. @@ -418,7 +418,7 @@ impl OnlineHnswBuilder { let mut ep = OrderedNode::new(entry, dist_calc.distance(entry).into()); let nodes = self.nodes.as_slice(); - for level in (1..self.params.max_level).rev() { + for level in (1..=nodes[entry as usize].target_level()).rev() { let cur_level = OnlineHnswLevelView::new(level, nodes); ep = greedy_search(&cur_level, ep, &dist_calc, self.params.prefetch_distance); } @@ -430,6 +430,7 @@ impl OnlineHnswBuilder { lower_bound: None, upper_bound: None, dist_q_c: 0.0, + use_acorn: false, }; let result = beam_search( &bottom, @@ -455,35 +456,30 @@ impl OnlineHnswBuilder { /// Only nodes whose insert has fully completed are included. Caller must /// ensure no concurrent inserts while this runs. /// - /// The entry point node is padded to full `max_level` height (with empty - /// neighbor lists at unused levels) so that search at upper levels can - /// safely traverse from it. `level_count` is recomputed from the actual - /// per-level emissions so the serialized batch and metadata stay in sync. + /// `level_count` is recomputed from the actual per-level emissions so the + /// serialized batch and metadata stay in sync. pub fn to_hnsw(&self) -> HNSW { let inserted = self.inserted_len.load(Ordering::Acquire); let entry_point = self.entry_point.load(Ordering::Acquire); - let max_level = self.params.max_level as usize; + let actual_levels = if inserted == 0 { + 0 + } else { + self.nodes[entry_point as usize].level_neighbors.len() + }; let mut frozen_nodes: Vec = Vec::with_capacity(inserted); - for (idx, node) in self.nodes.iter().enumerate().take(inserted) { - let mut level_neighbors: Vec>> = node + for node in self.nodes.iter().take(inserted) { + let level_neighbors: Vec>> = node .level_neighbors .iter() .map(|sl| sl.load_full()) .collect(); - let mut level_neighbors_ranked = node + let level_neighbors_ranked = node .level_neighbors_ranked .lock() .expect("level_neighbors_ranked mutex poisoned") .clone(); - if idx as u32 == entry_point { - while level_neighbors.len() < max_level { - level_neighbors.push(Arc::new(Vec::new())); - level_neighbors_ranked.push(Vec::new()); - } - } - let bottom_neighbors = level_neighbors .first() .cloned() @@ -495,9 +491,9 @@ impl OnlineHnswBuilder { )); } - let mut level_count: Vec = vec![0; max_level]; + let mut level_count: Vec = vec![0; actual_levels]; for node in &frozen_nodes { - let levels = node.level_neighbors.len().min(max_level); + let levels = node.level_neighbors.len().min(actual_levels); for count in level_count.iter_mut().take(levels) { *count += 1; } @@ -566,10 +562,11 @@ impl Graph for OnlineHnswBottomView<'_> { mod tests { use super::*; use crate::vector::flat::storage::FlatFloatStorage; - use arrow_array::FixedSizeListArray; + use arrow_array::{FixedSizeListArray, Float32Array}; use lance_arrow::FixedSizeListArrayExt; use lance_linalg::distance::DistanceType; use lance_testing::datagen::generate_random_array; + use rstest::rstest; use std::sync::Arc; fn build_storage(n: usize, dim: usize) -> (Arc, FixedSizeListArray) { @@ -579,6 +576,47 @@ mod tests { (storage, fsl) } + #[rstest] + #[case::zero_max_level( + HnswBuildParams::default().max_level(0), + "max_level must be greater than 0" + )] + #[case::zero_m( + HnswBuildParams::default().num_edges(0), + "m must be at least 4" + )] + #[case::one_m( + HnswBuildParams::default().num_edges(1), + "m must be at least 4" + )] + #[case::three_m( + HnswBuildParams::default().num_edges(3), + "m must be at least 4" + )] + #[case::small_ef( + HnswBuildParams::default().num_edges(20).ef_construction(19), + "ef_construction must be at least m (20)" + )] + #[case::overflowing_level_zero_limit( + HnswBuildParams::default() + .num_edges(usize::MAX) + .ef_construction(usize::MAX), + "level-0 reciprocal limit can be represented" + )] + fn test_try_with_capacity_rejects_invalid_params( + #[case] params: HnswBuildParams, + #[case] expected_message: &str, + ) { + let Err(error) = OnlineHnswBuilder::try_with_capacity(2, params) else { + panic!("expected invalid HNSW parameters to be rejected"); + }; + assert!(matches!(&error, lance_core::Error::InvalidInput { .. })); + assert!( + error.to_string().contains(expected_message), + "unexpected error: {error}" + ); + } + #[test] fn test_online_hnsw_recall() { const N: usize = 1000; @@ -588,7 +626,7 @@ mod tests { let params = HnswBuildParams::default() .num_edges(16) .ef_construction(100); - let builder = OnlineHnswBuilder::with_capacity(N, params); + let builder = OnlineHnswBuilder::try_with_capacity(N, params).unwrap(); for i in 0..N { builder.insert(i as u32, storage.as_ref()); @@ -644,7 +682,7 @@ mod tests { let params = HnswBuildParams::default() .num_edges(16) .ef_construction(100); - let builder = OnlineHnswBuilder::with_capacity(N, params); + let builder = OnlineHnswBuilder::try_with_capacity(N, params).unwrap(); for i in 0..N { builder.insert(i as u32, storage.as_ref()); } @@ -662,6 +700,7 @@ mod tests { lower_bound: None, upper_bound: None, dist_q_c: 0.0, + use_acorn: false, }, None, &mut visited, @@ -688,9 +727,71 @@ mod tests { #[test] fn test_online_hnsw_empty_search() { let params = HnswBuildParams::default(); - let builder = OnlineHnswBuilder::with_capacity(16, params); + let builder = OnlineHnswBuilder::try_with_capacity(16, params).unwrap(); let (storage, fsl) = build_storage(1, 8); let results = builder.search(fsl.value(0), 10, 32, storage.as_ref()); assert!(results.is_empty()); } + + /// Online construction uses the same Algorithm 1 distinction as the + /// offline builder: M for a new node and Mmax0 for reciprocal edges. + #[test] + fn test_online_new_node_uses_m_connections() { + let values = Float32Array::from(vec![ + 1.0, 0.0, // east + 0.0, 1.0, // north + -1.0, 0.0, // west + 0.0, -1.0, // south + 0.0, 0.0, // final node + ]); + let fsl = FixedSizeListArray::try_new_from_values(values, 2).unwrap(); + let storage = FlatFloatStorage::new(fsl, DistanceType::L2); + let params = HnswBuildParams::default() + .max_level(1) + .num_edges(2) + .ef_construction(5); + // Bypass public validation to isolate the Algorithm 1 degree rule. + let builder = OnlineHnswBuilder::new(5, params); + + for id in 0..5 { + builder.insert(id, &storage); + } + + let final_node = builder.nodes[4].level_neighbors_ranked.lock().unwrap(); + assert_eq!(final_node[0].len(), 2); + drop(final_node); + assert!( + builder + .nodes + .iter() + .all(|node| { node.level_neighbors_ranked.lock().unwrap()[0].len() <= 4 }), + "existing level-0 nodes must remain bounded by Mmax0" + ); + } + + #[test] + fn test_online_promotes_first_highest_random_level() { + const TOTAL: usize = 128; + let (storage, _) = build_storage(TOTAL, 4); + let params = HnswBuildParams::default(); + let builder = OnlineHnswBuilder::try_with_capacity(TOTAL, params.clone()).unwrap(); + + let mut level_rng = SmallRng::seed_from_u64(HNSW_LEVEL_RNG_SEED); + let expected_levels = (0..TOTAL) + .map(|_| random_level_with(¶ms, &mut level_rng)) + .collect::>(); + for (node, expected_level) in builder.nodes.iter().zip(&expected_levels) { + assert_eq!(node.target_level(), *expected_level); + } + + for id in 0..TOTAL as u32 { + builder.insert(id, storage.as_ref()); + } + let highest_level = *expected_levels.iter().max().unwrap(); + let expected_entry = expected_levels + .iter() + .position(|level| *level == highest_level) + .unwrap() as u32; + assert_eq!(builder.entry_point.load(Ordering::Acquire), expected_entry); + } } diff --git a/rust/lance-index/src/vector/ivf/shuffler.rs b/rust/lance-index/src/vector/ivf/shuffler.rs index 175d5ecb9fb..57254a42f8c 100644 --- a/rust/lance-index/src/vector/ivf/shuffler.rs +++ b/rust/lance-index/src/vector/ivf/shuffler.rs @@ -32,10 +32,11 @@ use lance_core::utils::futures::StreamOnDropExt; use lance_core::utils::tokio::get_num_compute_intensive_cpus; use lance_core::{Error, ROW_ID, Result, datatypes::Schema}; use lance_encoding::decoder::{DecoderPlugins, FilterExpression}; -use lance_encoding::version::LanceFileVersion; -use lance_file::previous::reader::FileReader as PreviousFileReader; -use lance_file::previous::writer::FileWriter as PreviousFileWriter; use lance_file::reader::{FileReader as Lancev2FileReader, FileReaderOptions}; +use lance_file::version::ConcreteFileVersion; +use lance_file::versions; +use lance_file::versions::v1::reader::FileReader as V1FileReader; +use lance_file::versions::v1::writer::FileWriter as V1FileWriter; use lance_file::writer::FileWriterOptions; use lance_io::ReadBatchParams; use lance_io::object_store::ObjectStore; @@ -406,7 +407,7 @@ pub struct IvfShuffler { shuffle_output_root_filename: String, - format_version: LanceFileVersion, + format_version: ConcreteFileVersion, } /// Represents a range of batches in a file that should be shuffled @@ -446,11 +447,11 @@ impl IvfShuffler { unsorted_buffers: vec![], is_legacy, shuffle_output_root_filename, - format_version: LanceFileVersion::V2_0, + format_version: ConcreteFileVersion::V2_0, }) } - pub fn with_format_version(mut self, format_version: LanceFileVersion) -> Self { + pub fn with_format_version(mut self, format_version: ConcreteFileVersion) -> Self { self.format_version = format_version; self } @@ -496,7 +497,7 @@ impl IvfShuffler { info!("Writing unsorted data to disk at {}", path); info!("with schema: {:?}", schema); - let mut file_writer = PreviousFileWriter::::with_object_writer( + let mut file_writer = V1FileWriter::::with_object_writer( writer, Schema::try_from(schema.as_ref())?, &Default::default(), @@ -528,7 +529,7 @@ impl IvfShuffler { if self.is_legacy { let reader = - PreviousFileReader::try_new_self_described(&object_store, &path, None).await?; + V1FileReader::try_new_self_described(&object_store, &path, None).await?; total_batches.push(reader.num_batches()); } else { let scheduler_config = SchedulerConfig::max_bandwidth(&object_store); @@ -572,7 +573,7 @@ impl IvfShuffler { if self.is_legacy { let reader = - PreviousFileReader::try_new_self_described(&object_store, &path, None).await?; + V1FileReader::try_new_self_described(&object_store, &path, None).await?; let lance_schema = reader .schema() .project(&[PART_ID_COLUMN]) @@ -655,9 +656,8 @@ impl IvfShuffler { let mut _reader_handle = None; let mut stream = if self.is_legacy { - _reader_handle = Some( - PreviousFileReader::try_new_self_described(&object_store, &path, None).await?, - ); + _reader_handle = + Some(V1FileReader::try_new_self_described(&object_store, &path, None).await?); stream::iter(start..end) .map(|i| { @@ -806,13 +806,11 @@ impl IvfShuffler { true, )])); let lance_schema = Schema::try_from(sorted_file_schema.as_ref())?; - let mut file_writer = lance_file::writer::FileWriter::try_new( + let mut file_writer = versions::create_writer( + this.format_version, writer, lance_schema, - FileWriterOptions { - format_version: Some(this.format_version), - ..Default::default() - }, + FileWriterOptions::default(), )?; for partition_and_idx in shuffled.into_iter().enumerate() { diff --git a/rust/lance-index/src/vector/ivf/storage.rs b/rust/lance-index/src/vector/ivf/storage.rs index 5d58401bb12..0781ebc1ea5 100644 --- a/rust/lance-index/src/vector/ivf/storage.rs +++ b/rust/lance-index/src/vector/ivf/storage.rs @@ -8,8 +8,8 @@ use itertools::Itertools; use lance_arrow::FixedSizeListArrayExt; use lance_core::deepsize::DeepSizeOf; use lance_core::{Error, Result}; -use lance_file::previous::{ - reader::FileReader as PreviousFileReader, writer::FileWriter as PreviousFileWriter, +use lance_file::versions::v1::{ + reader::FileReader as V1FileReader, writer::FileWriter as V1FileWriter, }; use lance_io::{traits::WriteExt, utils::read_message}; use lance_linalg::distance::DistanceType; @@ -146,7 +146,7 @@ impl IvfModel { start..end } - pub async fn load(reader: &PreviousFileReader) -> Result { + pub async fn load(reader: &V1FileReader) -> Result { let schema = reader.schema(); let meta_str = schema .metadata @@ -167,7 +167,7 @@ impl IvfModel { } /// Write the IVF metadata to the lance file. - pub async fn write(&self, writer: &mut PreviousFileWriter) -> Result<()> { + pub async fn write(&self, writer: &mut V1FileWriter) -> Result<()> { let pb = PbIvf::try_from(self)?; let pos = writer.object_writer.write_protobuf(&pb).await?; let ivf_metadata = IvfMetadata { pb_position: pos }; @@ -286,14 +286,10 @@ mod tests { let schema = Schema::try_from(&arrow_schema).unwrap(); { - let mut writer = PreviousFileWriter::try_new( - &object_store, - &path, - schema.clone(), - &Default::default(), - ) - .await - .unwrap(); + let mut writer = + V1FileWriter::try_new(&object_store, &path, schema.clone(), &Default::default()) + .await + .unwrap(); // Write some dummy data let batch = RecordBatch::try_new( Arc::new(arrow_schema), @@ -305,7 +301,7 @@ mod tests { writer.finish().await.unwrap(); } - let reader = PreviousFileReader::try_new_self_described(&object_store, &path, None) + let reader = V1FileReader::try_new_self_described(&object_store, &path, None) .await .unwrap(); assert!(reader.schema().metadata.contains_key(IVF_METADATA_KEY)); diff --git a/rust/lance-index/src/vector/kmeans.rs b/rust/lance-index/src/vector/kmeans.rs index b11fb70bed0..ceff08c2d45 100644 --- a/rust/lance-index/src/vector/kmeans.rs +++ b/rust/lance-index/src/vector/kmeans.rs @@ -45,7 +45,7 @@ use { }; use crate::vector::utils::SimpleIndex; -use crate::{Error, Result}; +use lance_core::{Error, Result}; /// KMean initialization method. #[derive(Debug, PartialEq)] @@ -1156,7 +1156,16 @@ impl KMeans { target_k ); } - debug_assert_eq!(heap.len(), target_k); + if heap.len() < target_k { + return Err(ArrowError::InvalidArgumentError(format!( + "Cannot create {target_k} IVF partitions: k-means could only form {} non-empty \ + clusters from {n} training vectors. The dataset is likely too small or has too \ + many (near-)duplicate vectors for this many partitions. Reduce num_partitions to \ + <= {} or provide more diverse data.", + heap.len(), + heap.len() + ))); + } // Construct final KMeans model with all centroids let mut all_clusters: Vec> = heap.into_vec(); @@ -1722,6 +1731,47 @@ mod tests { } } + #[tokio::test] + async fn test_hierarchical_kmeans_too_few_distinct_vectors_errors() { + // Regression test for https://github.com/lance-format/lance/issues/7867 + // + // With a small number of distinct vectors repeated many times (heavy + // near-duplication) and dot distance, hierarchical k-means cannot form + // `target_k` non-empty clusters no matter how it splits: every split of a + // cluster of identical vectors is either ineffective (`all_same`) or + // immediately hits the "<= 1 point" floor. This used to trip + // `debug_assert_eq!(heap.len(), target_k)` (panic in debug builds) or + // silently return a half-empty centroid set (release builds). It should + // now return a clear error instead. + const DIM: usize = 8; + const NUM_DISTINCT: usize = 5; + const REPEATS: usize = 200; + const TARGET_K: usize = 300; // > 256 to trigger hierarchical clustering + + let base_vectors = generate_random_array(NUM_DISTINCT * DIM); + let mut values = Vec::with_capacity(NUM_DISTINCT * REPEATS * DIM); + for _ in 0..REPEATS { + values.extend_from_slice(base_vectors.values()); + } + let values = Float32Array::from(values); + let fsl = FixedSizeListArray::try_new_from_values(values, DIM as i32).unwrap(); + + let params = KMeansParams { + max_iters: 10, + hierarchical_k: 16, + distance_type: DistanceType::Dot, + ..Default::default() + }; + + let err = KMeans::new_with_params(&fsl, TARGET_K, ¶ms) + .expect_err("training should fail rather than panic or silently under-produce"); + let msg = err.to_string(); + assert!( + msg.contains("Cannot create") && msg.contains(&TARGET_K.to_string()), + "unexpected error message: {msg}" + ); + } + #[tokio::test] async fn test_float16_underflow_fix() { // This test verifies the fix for float16 division underflow diff --git a/rust/lance-index/src/vector/pq/storage.rs b/rust/lance-index/src/vector/pq/storage.rs index 4e98df9906d..cda5fc9c2ff 100644 --- a/rust/lance-index/src/vector/pq/storage.rs +++ b/rust/lance-index/src/vector/pq/storage.rs @@ -24,8 +24,8 @@ use bytes::{Bytes, BytesMut}; use lance_arrow::{FixedSizeListArrayExt, RecordBatchExt}; use lance_core::deepsize::DeepSizeOf; use lance_core::{Error, ROW_ID, Result}; -use lance_file::previous::{ - reader::FileReader as PreviousFileReader, writer::FileWriter as PreviousFileWriter, +use lance_file::versions::v1::{ + reader::FileReader as V1FileReader, writer::FileWriter as V1FileWriter, }; use lance_io::{object_store::ObjectStore, utils::read_message}; use lance_linalg::distance::{Cosine, DistanceType, Dot, L2}; @@ -126,7 +126,7 @@ impl QuantizerMetadata for ProductQuantizationMetadata { } } - async fn load(reader: &PreviousFileReader) -> Result { + async fn load(reader: &V1FileReader) -> Result { let metadata = reader .schema() .metadata @@ -386,7 +386,7 @@ impl ProductQuantizationStorage { path: &Path, frag_reuse_index: Option>, ) -> Result { - let reader = PreviousFileReader::try_new_self_described(object_store, path, None).await?; + let reader = V1FileReader::try_new_self_described(object_store, path, None).await?; let schema = reader.schema(); let metadata_str = schema @@ -469,7 +469,7 @@ impl ProductQuantizationStorage { /// pub async fn write_partition( &self, - writer: &mut PreviousFileWriter, + writer: &mut V1FileWriter, ) -> Result { let batch_size: usize = 10240; // TODO: make it configurable for offset in (0..self.batch.num_rows()).step_by(batch_size) { @@ -613,9 +613,9 @@ impl QuantizerStorage for ProductQuantizationStorage { /// /// Parameters /// ---------- - /// - *reader: &PreviousFileReader + /// - *reader: &V1FileReader async fn load_partition( - reader: &PreviousFileReader, + reader: &V1FileReader, range: std::ops::Range, distance_type: DistanceType, metadata: &Self::Metadata, diff --git a/rust/lance-index/src/vector/quantizer.rs b/rust/lance-index/src/vector/quantizer.rs index 6f4b191098b..59cab5643bc 100644 --- a/rust/lance-index/src/vector/quantizer.rs +++ b/rust/lance-index/src/vector/quantizer.rs @@ -15,7 +15,7 @@ use bytes::Bytes; use lance_arrow::RecordBatchExt; use lance_core::deepsize::DeepSizeOf; use lance_core::{Error, ROW_ID, Result}; -use lance_file::previous::reader::FileReader as PreviousFileReader; +use lance_file::versions::v1::reader::FileReader as V1FileReader; use lance_io::traits::Reader; use lance_linalg::distance::DistanceType; use lance_table::format::SelfDescribingFileReader; @@ -81,7 +81,9 @@ impl FromStr for QuantizationType { "FLATBIN" => Ok(Self::FlatBin), "PQ" => Ok(Self::Product), "SQ" => Ok(Self::Scalar), - "RABIT" => Ok(Self::Rabit), + // `Display` writes "RQ"; "RABIT" is accepted for headers written + // before this variant round-tripped. + "RQ" | "RABIT" => Ok(Self::Rabit), _ => Err(Error::index(format!("Unknown quantization type: {}", s))), } } @@ -223,7 +225,7 @@ pub trait QuantizerMetadata: Ok(None) } - async fn load(reader: &PreviousFileReader) -> Result; + async fn load(reader: &V1FileReader) -> Result; } #[async_trait::async_trait] @@ -277,7 +279,7 @@ pub trait QuantizerStorage: Clone + Sized + DeepSizeOf + VectorStore { } async fn load_partition( - reader: &PreviousFileReader, + reader: &V1FileReader, range: std::ops::Range, distance_type: DistanceType, metadata: &Self::Metadata, @@ -287,7 +289,7 @@ pub trait QuantizerStorage: Clone + Sized + DeepSizeOf + VectorStore { /// Loader to load partitioned [VectorStore] from disk. pub struct IvfQuantizationStorage { - reader: PreviousFileReader, + reader: V1FileReader, distance_type: DistanceType, quantizer: Quantizer, @@ -322,7 +324,7 @@ impl IvfQuantizationStorage { /// /// pub async fn open(reader: Arc) -> Result { - let reader = PreviousFileReader::try_new_self_described_from_reader(reader, None).await?; + let reader = V1FileReader::try_new_self_described_from_reader(reader, None).await?; let schema = reader.schema(); let metadata_str = schema @@ -385,3 +387,34 @@ impl IvfQuantizationStorage { .await } } + +#[cfg(test)] +mod tests { + use super::*; + use rstest::rstest; + + /// `IvfIndexState` persists the quantization type with `Display` and reads + /// it back with `FromStr`, so the two must agree for every variant. + #[rstest] + #[case::flat(QuantizationType::Flat)] + #[case::flat_bin(QuantizationType::FlatBin)] + #[case::product(QuantizationType::Product)] + #[case::scalar(QuantizationType::Scalar)] + #[case::rabit(QuantizationType::Rabit)] + fn test_display_from_str_round_trip(#[case] quantization_type: QuantizationType) { + let encoded = quantization_type.to_string(); + assert_eq!( + encoded.parse::().unwrap(), + quantization_type, + "{encoded} did not round-trip" + ); + } + + #[test] + fn test_from_str_accepts_legacy_rabit_spelling() { + assert_eq!( + "RABIT".parse::().unwrap(), + QuantizationType::Rabit + ); + } +} diff --git a/rust/lance-index/src/vector/sq/storage.rs b/rust/lance-index/src/vector/sq/storage.rs index 785aadc289d..5011fbada46 100644 --- a/rust/lance-index/src/vector/sq/storage.rs +++ b/rust/lance-index/src/vector/sq/storage.rs @@ -15,7 +15,7 @@ use async_trait::async_trait; use lance_arrow::ArrowFloatType; use lance_core::deepsize::DeepSizeOf; use lance_core::{Error, ROW_ID, Result}; -use lance_file::previous::reader::FileReader as PreviousFileReader; +use lance_file::versions::v1::reader::FileReader as V1FileReader; use lance_io::object_store::ObjectStore; use lance_linalg::distance::{DistanceType, dot_u8::dot_u8, l2_u8::l2_u8}; use lance_table::format::SelfDescribingFileReader; @@ -53,7 +53,7 @@ impl DeepSizeOf for ScalarQuantizationMetadata { #[async_trait] impl QuantizerMetadata for ScalarQuantizationMetadata { - async fn load(reader: &PreviousFileReader) -> Result { + async fn load(reader: &V1FileReader) -> Result { let metadata_str = reader .schema() .metadata @@ -215,7 +215,7 @@ impl ScalarQuantizationStorage { path: &Path, frag_reuse_index: Option>, ) -> Result { - let reader = PreviousFileReader::try_new_self_described(object_store, path, None).await?; + let reader = V1FileReader::try_new_self_described(object_store, path, None).await?; let schema = reader.schema(); let metadata_str = schema @@ -292,7 +292,7 @@ impl QuantizerStorage for ScalarQuantizationStorage { /// - *metric_type: metric type of the vectors /// - *metadata: scalar quantization metadata async fn load_partition( - reader: &PreviousFileReader, + reader: &V1FileReader, range: std::ops::Range, distance_type: DistanceType, metadata: &Self::Metadata, diff --git a/rust/lance-index/src/vector/sq/transform.rs b/rust/lance-index/src/vector/sq/transform.rs index 3a81734347b..78ca2942bf6 100644 --- a/rust/lance-index/src/vector/sq/transform.rs +++ b/rust/lance-index/src/vector/sq/transform.rs @@ -56,16 +56,21 @@ impl Transformer for SQTransformer { "SQ Transform: column {} not found in batch", self.input_column )))?; - let fsl = input - .as_fixed_size_list_opt() - .ok_or(Error::index("input column is not vector type".to_string()))?; + let fsl = input.as_fixed_size_list_opt().ok_or_else(|| { + Error::index(format!( + "SQ Transform: column {} is not a fixed size list vector: {}", + self.input_column, + input.data_type() + )) + })?; let sq_code = match fsl.value_type() { DataType::Float16 => self.quantizer.transform::(input)?, DataType::Float32 => self.quantizer.transform::(input)?, DataType::Float64 => self.quantizer.transform::(input)?, _ => { return Err(Error::index(format!( - "unsupported data type: {}", + "SQ Transform: column {} has unsupported value type: {}", + self.input_column, fsl.value_type() ))); } @@ -78,3 +83,87 @@ impl Transformer for SQTransformer { Ok(batch) } } + +#[cfg(test)] +mod tests { + use super::*; + + use arrow_array::{Array, FixedSizeListArray, Float32Array, Int32Array}; + use arrow_schema::Schema; + use lance_arrow::FixedSizeListArrayExt; + + const SQ_COLUMN: &str = "sq"; + const VECTOR_COLUMN: &str = "v"; + + fn transformer(dim: usize) -> SQTransformer { + let mut quantizer = ScalarQuantizer::new(8, dim); + quantizer.metadata.bounds = 0.0..1.0; + SQTransformer::new(quantizer, VECTOR_COLUMN.into(), SQ_COLUMN.into()) + } + + fn vector_batch(values: Float32Array, dim: i32) -> RecordBatch { + let fsl = FixedSizeListArray::try_new_from_values(values, dim).unwrap(); + let schema = Schema::new(vec![Field::new( + VECTOR_COLUMN, + fsl.data_type().clone(), + true, + )]); + RecordBatch::try_new(schema.into(), vec![Arc::new(fsl)]).unwrap() + } + + #[test] + fn test_sq_transform_replaces_vector_with_code_column() { + let batch = vector_batch(Float32Array::from(vec![0.0, 0.5, 1.0, 0.25]), 2); + let output = transformer(2).transform(&batch).unwrap(); + + assert!( + output.column_by_name(VECTOR_COLUMN).is_none(), + "input column should be dropped" + ); + let codes = output + .column_by_name(SQ_COLUMN) + .unwrap() + .as_fixed_size_list(); + assert_eq!(codes.len(), 2); + assert_eq!(codes.value_length(), 2); + } + + #[test] + fn test_sq_transform_reports_missing_column() { + let batch = vector_batch(Float32Array::from(vec![0.0, 1.0]), 2); + let transformer = SQTransformer::new( + ScalarQuantizer::new(8, 2), + "absent".into(), + SQ_COLUMN.into(), + ); + let message = transformer.transform(&batch).unwrap_err().to_string(); + assert!(message.contains("column absent"), "{message}"); + } + + #[test] + fn test_sq_transform_reports_non_vector_column() { + let batch = RecordBatch::try_new( + Schema::new(vec![Field::new(VECTOR_COLUMN, DataType::Int32, false)]).into(), + vec![Arc::new(Int32Array::from(vec![1, 2]))], + ) + .unwrap(); + let message = transformer(2).transform(&batch).unwrap_err().to_string(); + assert!(message.contains("column v"), "{message}"); + assert!(message.contains("Int32"), "{message}"); + } + + #[test] + fn test_sq_transform_reports_unsupported_value_type() { + let values = Int32Array::from(vec![1, 2, 3, 4]); + let fsl = FixedSizeListArray::try_new_from_values(values, 2).unwrap(); + let schema = Schema::new(vec![Field::new( + VECTOR_COLUMN, + fsl.data_type().clone(), + true, + )]); + let batch = RecordBatch::try_new(schema.into(), vec![Arc::new(fsl)]).unwrap(); + let message = transformer(2).transform(&batch).unwrap_err().to_string(); + assert!(message.contains("column v"), "{message}"); + assert!(message.contains("Int32"), "{message}"); + } +} diff --git a/rust/lance-index/src/vector/transform.rs b/rust/lance-index/src/vector/transform.rs index 01372a14048..ac2ead8da17 100644 --- a/rust/lance-index/src/vector/transform.rs +++ b/rust/lance-index/src/vector/transform.rs @@ -229,6 +229,7 @@ mod tests { use super::*; use approx::assert_relative_eq; + use arrow::buffer::NullBuffer; use arrow_array::{FixedSizeListArray, Float16Array, Float32Array, Int32Array}; use arrow_schema::Schema; use half::f16; @@ -345,6 +346,91 @@ mod tests { assert!(dup_drop_result.is_ok()); } + /// Builds a 2-dim FSL column with one row per `rows` entry. `None` means a + /// null vector; otherwise the two f32 values are used as-is. + fn fsl_batch(rows: &[Option<[f32; 2]>]) -> RecordBatch { + let values = Float32Array::from( + rows.iter() + .flat_map(|row| row.unwrap_or([0.0, 0.0])) + .collect::>(), + ); + let nulls = NullBuffer::from(rows.iter().map(Option::is_some).collect::>()); + let item = Arc::new(Field::new("item", DataType::Float32, true)); + let fsl = + FixedSizeListArray::try_new(item.clone(), 2, Arc::new(values), Some(nulls)).unwrap(); + let schema = Schema::new(vec![Field::new( + "v", + DataType::FixedSizeList(item, 2), + true, + )]); + RecordBatch::try_new(schema.into(), vec![Arc::new(fsl)]).unwrap() + } + + #[test] + fn test_keep_finite_vectors_drops_null_and_non_finite_rows() { + let batch = fsl_batch(&[ + Some([1.0, 2.0]), + None, + Some([f32::NAN, 1.0]), + Some([f32::INFINITY, 1.0]), + Some([f32::NEG_INFINITY, 1.0]), + Some([3.0, 4.0]), + ]); + let output = KeepFiniteVectors::new("v").transform(&batch).unwrap(); + + let kept = output.column_by_name("v").unwrap().as_fixed_size_list(); + assert_eq!(kept.len(), 2, "only the two finite rows survive"); + assert_eq!(kept.null_count(), 0); + assert_eq!( + kept.values().as_primitive::().values(), + &[1.0, 2.0, 3.0, 4.0] + ); + } + + #[test] + fn test_keep_finite_vectors_on_all_null_and_empty_batches() { + let all_null = fsl_batch(&[None, None]); + assert_eq!( + KeepFiniteVectors::new("v") + .transform(&all_null) + .unwrap() + .num_rows(), + 0 + ); + + let empty = fsl_batch(&[]); + assert_eq!( + KeepFiniteVectors::new("v") + .transform(&empty) + .unwrap() + .num_rows(), + 0 + ); + } + + #[test] + fn test_keep_finite_vectors_passes_through_missing_column() { + // A batch without the configured column is returned untouched rather + // than erroring, so the transform is a no-op on unrelated batches. + let batch = fsl_batch(&[Some([1.0, 2.0])]); + let output = KeepFiniteVectors::new("other").transform(&batch).unwrap(); + assert_eq!(output.num_rows(), 1); + } + + #[test] + fn test_keep_finite_vectors_rejects_non_list_column() { + let batch = RecordBatch::try_new( + Schema::new(vec![Field::new("v", DataType::Int32, false)]).into(), + vec![Arc::new(Int32Array::from(vec![1, 2]))], + ) + .unwrap(); + let error = KeepFiniteVectors::new("v").transform(&batch).unwrap_err(); + assert!(matches!(error, Error::Index { .. }), "{error:?}"); + let message = error.to_string(); + assert!(message.contains("column v"), "{message}"); + assert!(message.contains("Int32"), "{message}"); + } + #[test] fn test_is_all_finite() { let array = Float32Array::from(vec![1.0, 2.0]); diff --git a/rust/lance-index/src/vector/utils.rs b/rust/lance-index/src/vector/utils.rs index fb4f9004c57..74215040bc6 100644 --- a/rust/lance-index/src/vector/utils.rs +++ b/rust/lance-index/src/vector/utils.rs @@ -1,13 +1,16 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright The Lance Authors -use arrow::compute::cast; +use arrow::{ + array::{ArrayData, make_array}, + buffer::Buffer, + compute::cast, +}; use arrow_array::types::{Float16Type, Float32Type, Float64Type}; use arrow_array::{Array, ArrayRef, BooleanArray, FixedSizeListArray, cast::AsArray}; use arrow_schema::{DataType, Field}; -use lance_arrow::FixedSizeListArrayExt; +use lance_arrow::{BufferExt, DataTypeExt, FixedSizeListArrayExt}; use lance_core::{Error, Result}; -use lance_io::encodings::plain::bytes_to_array; use lance_linalg::distance::DistanceType; use prost::bytes; use std::sync::LazyLock; @@ -109,6 +112,7 @@ impl SimpleIndex { lower_bound: None, upper_bound: None, dist_q_c: 0.0, + use_acorn: false, }; let res = match &self.store { SimpleStore::Float(store) => self.index.search_basic(query, 1, ¶ms, None, store)?, @@ -212,23 +216,40 @@ impl TryFrom<&pb::Tensor> for FixedSizeListArray { } let dim = tensor.shape[1] as usize; let num_rows = tensor.shape[0] as usize; - - let data = bytes::Bytes::from(tensor.data.clone()); - let flat_array = bytes_to_array( - &DataType::from(pb::tensor::DataType::try_from(tensor.data_type).unwrap()), - data, - dim * num_rows, - 0, - )?; - - if flat_array.len() != dim * num_rows { + let num_values = dim.checked_mul(num_rows).ok_or_else(|| { + Error::index(format!( + "Tensor shape {:?} exceeds the supported size", + tensor.shape + )) + })?; + let data_type = DataType::from(pb::tensor::DataType::try_from(tensor.data_type).unwrap()); + let expected_data_len = + num_values + .checked_mul(data_type.byte_width()) + .ok_or_else(|| { + Error::index(format!( + "Tensor shape {:?} exceeds the supported byte length", + tensor.shape + )) + })?; + if tensor.data.len() != expected_data_len { return Err(Error::index(format!( - "Tensor shape {:?} does not match to data len: {}", + "Tensor shape {:?} with data type {data_type} requires {expected_data_len} bytes, got {}", tensor.shape, - flat_array.len() + tensor.data.len() ))); } + let buffer = Buffer::from_bytes_bytes( + bytes::Bytes::from(tensor.data.clone()), + data_type.byte_width() as u64, + ); + let data = ArrayData::builder(data_type) + .len(num_values) + .null_count(0) + .add_buffer(buffer) + .build()?; + let flat_array = make_array(data); let field = Field::new("item", flat_array.data_type().clone(), true); Ok(Self::try_new( Arc::new(field), @@ -277,6 +298,7 @@ mod tests { use half::f16; use lance_arrow::FixedSizeListArrayExt; use num_traits::identities::Zero; + use rayon::ThreadPoolBuilder; use arrow::compute::cast; use rstest::rstest; @@ -307,7 +329,8 @@ mod tests { (0..100).flat_map(|i| std::iter::repeat_n(i as f32, 16)).collect::>(), )) as ArrayRef, 42.0f32)] fn test_simple_index_nearest_centroid(#[case] centroids: ArrayRef, #[case] query_val: f32) { - let index = build_index(centroids, 16); + let thread_pool = ThreadPoolBuilder::new().num_threads(1).build().unwrap(); + let index = thread_pool.install(|| build_index(centroids, 16)); let query: ArrayRef = Arc::new(Float32Array::from(vec![query_val; 16])); let (id, dist) = index.search(query).unwrap(); assert_eq!(id, 42); @@ -351,6 +374,8 @@ mod tests { assert_eq!(tensor.data_type, pb::tensor::DataType::Float16 as i32); assert_eq!(tensor.shape, vec![4, 5]); assert_eq!(tensor.data.len(), 20 * 2); + let decoded = FixedSizeListArray::try_from(&tensor).unwrap(); + assert_eq!(decoded.values().to_data(), fsl.values().to_data()); let fsl = FixedSizeListArray::try_new_from_values(Float32Array::from(vec![0.0; 20]), 5).unwrap(); @@ -358,6 +383,8 @@ mod tests { assert_eq!(tensor.data_type, pb::tensor::DataType::Float32 as i32); assert_eq!(tensor.shape, vec![4, 5]); assert_eq!(tensor.data.len(), 20 * 4); + let decoded = FixedSizeListArray::try_from(&tensor).unwrap(); + assert_eq!(decoded.values().to_data(), fsl.values().to_data()); let fsl = FixedSizeListArray::try_new_from_values(Float64Array::from(vec![0.0; 20]), 5).unwrap(); @@ -365,5 +392,21 @@ mod tests { assert_eq!(tensor.data_type, pb::tensor::DataType::Float64 as i32); assert_eq!(tensor.shape, vec![4, 5]); assert_eq!(tensor.data.len(), 20 * 8); + let decoded = FixedSizeListArray::try_from(&tensor).unwrap(); + assert_eq!(decoded.values().to_data(), fsl.values().to_data()); + } + + #[rstest] + #[case::too_short(vec![0; 7])] + #[case::too_long(vec![0; 9])] + fn test_tensor_to_fsl_rejects_invalid_data_length(#[case] data: Vec) { + let tensor = pb::Tensor { + data_type: pb::tensor::DataType::Uint32 as i32, + shape: vec![1, 2], + data, + }; + + let error = FixedSizeListArray::try_from(&tensor).unwrap_err(); + assert!(error.to_string().contains("requires 8 bytes")); } } diff --git a/rust/lance-index/src/vector/v3/shuffler.rs b/rust/lance-index/src/vector/v3/shuffler.rs index 4203d099d0b..46bfe9fbf9e 100644 --- a/rust/lance-index/src/vector/v3/shuffler.rs +++ b/rust/lance-index/src/vector/v3/shuffler.rs @@ -20,9 +20,10 @@ use lance_core::{ utils::tokio::{get_num_compute_intensive_cpus, spawn_cpu}, }; use lance_encoding::decoder::{DecoderPlugins, FilterExpression}; -use lance_encoding::version::LanceFileVersion; use lance_file::reader::{FileReader, FileReaderOptions}; -use lance_file::writer::{FileWriter, FileWriterOptions}; +use lance_file::version::ConcreteFileVersion; +use lance_file::versions; +use lance_file::writer::FileWriterOptions; use lance_io::{ ReadBatchParams, object_store::ObjectStore, @@ -71,7 +72,7 @@ pub struct IvfShuffler { object_store: Arc, output_dir: Path, num_partitions: usize, - format_version: LanceFileVersion, + format_version: ConcreteFileVersion, progress: Arc, } @@ -82,12 +83,12 @@ impl IvfShuffler { object_store: Arc::new(ObjectStore::local()), output_dir, num_partitions, - format_version: LanceFileVersion::V2_0, + format_version: ConcreteFileVersion::V2_0, progress: crate::progress::noop_progress(), } } - pub fn with_format_version(mut self, format_version: LanceFileVersion) -> Self { + pub fn with_format_version(mut self, format_version: ConcreteFileVersion) -> Self { self.format_version = format_version; self } @@ -122,13 +123,11 @@ impl Shuffler for IvfShuffler { let format_version = self.format_version; async move { let writer = object_store.create(&part_path).await?; - let file_writer = FileWriter::try_new( + let file_writer = versions::create_writer( + format_version, writer, lance_core::datatypes::Schema::try_from(&schema)?, - FileWriterOptions { - format_version: Some(format_version), - ..Default::default() - }, + FileWriterOptions::default(), )? .with_page_metadata_spill(object_store.clone(), spill_path); Result::Ok(file_writer) @@ -315,7 +314,7 @@ impl ShuffleReader for EmptyReader { pub fn create_ivf_shuffler( output_dir: Path, num_partitions: usize, - format_version: LanceFileVersion, + format_version: ConcreteFileVersion, progress: Option>, ) -> Box { let use_legacy = std::env::var("LANCE_LEGACY_SHUFFLER") @@ -475,7 +474,7 @@ impl Shuffler for TwoFileShuffler { let data_path = self.output_dir.clone().join("shuffle_data.lance"); let spill_path = self.output_dir.clone().join("shuffle_data.spill"); let writer = self.object_store.create(&data_path).await?; - let mut file_writer = FileWriter::try_new( + let mut file_writer = versions::v2_1::create_writer( writer, lance_core::datatypes::Schema::try_from(&schema)?, Default::default(), @@ -486,7 +485,7 @@ impl Shuffler for TwoFileShuffler { let offsets_path = self.output_dir.clone().join("shuffle_offsets.lance"); let spill_path = self.output_dir.clone().join("shuffle_offsets.spill"); let writer = self.object_store.create(&offsets_path).await?; - let mut offsets_writer = FileWriter::try_new( + let mut offsets_writer = versions::v2_1::create_writer( writer, lance_core::datatypes::Schema::try_from(offsets_schema.as_ref())?, Default::default(), @@ -577,8 +576,8 @@ impl Shuffler for TwoFileShuffler { /// Returns `(total_rows_written, per_partition_row_counts)`. async fn flush_shuffle_batch( accumulated: Vec, - file_writer: &mut FileWriter, - offsets_writer: &mut FileWriter, + file_writer: &mut versions::v2_1::Writer, + offsets_writer: &mut versions::v2_1::Writer, offsets_schema: Arc, num_partitions: usize, global_row_count: u64, @@ -853,10 +852,34 @@ mod tests { // Partition 2: rows with values 30 let batch = make_batch(&[0, 1, 2, 0, 1], &[10, 20, 30, 40, 50], None); - let shuffler = TwoFileShuffler::new(output_dir, num_partitions); + let shuffler = TwoFileShuffler::new(output_dir.clone(), num_partitions); let stream = batches_to_stream(vec![batch]); let reader = shuffler.shuffle(stream).await.unwrap(); + let object_store = Arc::new(ObjectStore::local()); + let scheduler = ScanScheduler::new( + object_store.clone(), + SchedulerConfig::max_bandwidth(&object_store), + ); + for filename in ["shuffle_data.lance", "shuffle_offsets.lance"] { + let file_reader = FileReader::try_open( + scheduler + .open_file( + &output_dir.clone().join(filename), + &CachedFileSize::unknown(), + ) + .await + .unwrap(), + None, + Arc::::default(), + &LanceCache::no_cache(), + FileReaderOptions::default(), + ) + .await + .unwrap(); + assert_eq!(file_reader.version(), ConcreteFileVersion::V2_1); + } + // Verify partition sizes assert_eq!(reader.partition_size(0).unwrap(), 2); assert_eq!(reader.partition_size(1).unwrap(), 2); diff --git a/rust/lance-index/src/vector/v3/subindex.rs b/rust/lance-index/src/vector/v3/subindex.rs index 9e82921347f..cf712c1b246 100644 --- a/rust/lance-index/src/vector/v3/subindex.rs +++ b/rust/lance-index/src/vector/v3/subindex.rs @@ -32,6 +32,15 @@ pub trait IvfSubIndex: Send + Sync + Debug + DeepSizeOf { /// Return the schema of the sub index fn schema() -> arrow_schema::SchemaRef; + /// The subset of [`Self::schema`] that [`Self::load`] actually reads. + /// + /// Index files always carry the full `schema()`, so narrowing the read is + /// purely a storage optimization: it keeps write-only columns from being + /// fetched, without changing what is written. `None` reads every column. + fn read_columns() -> Option<&'static [&'static str]> { + None + } + /// Search the sub index for nearest neighbors. /// # Arguments: /// * `query` - The query vector diff --git a/rust/lance-io/Cargo.toml b/rust/lance-io/Cargo.toml index 6cee04d5fd9..03c6780263a 100644 --- a/rust/lance-io/Cargo.toml +++ b/rust/lance-io/Cargo.toml @@ -20,14 +20,8 @@ lance-arrow.workspace = true lance-core.workspace = true lance-namespace.workspace = true arrow = { workspace = true, features = ["ffi"] } -arrow-arith.workspace = true arrow-array.workspace = true -arrow-buffer.workspace = true -arrow-cast.workspace = true -arrow-data.workspace = true arrow-schema.workspace = true -arrow-select.workspace = true -async-recursion.workspace = true async-trait.workspace = true aws-config = { workspace = true, optional = true } aws-credential-types = { workspace = true, optional = true } @@ -37,10 +31,11 @@ chrono.workspace = true futures.workspace = true http.workspace = true log.workspace = true +metrics = { workspace = true, optional = true } moka.workspace = true pin-project.workspace = true prost.workspace = true -serde.workspace = true +serde = { workspace = true, features = ["derive"] } tokio.workspace = true tracing.workspace = true url.workspace = true @@ -60,6 +55,7 @@ rstest.workspace = true mock_instant.workspace = true tokio = { workspace = true, features = ["test-util"] } tracing-mock = { workspace = true } +metrics-util = { workspace = true } [[bench]] name = "scheduler" @@ -67,6 +63,7 @@ harness = false [features] default = ["aws", "azure", "gcp"] +metrics = ["dep:metrics"] gcs-test = [] goosefs-test = [] gcp = ["object_store/gcp", "dep:opendal", "opendal/services-gcs", "dep:object_store_opendal"] diff --git a/rust/lance-io/src/encodings.rs b/rust/lance-io/src/encodings.rs deleted file mode 100644 index de2737f2366..00000000000 --- a/rust/lance-io/src/encodings.rs +++ /dev/null @@ -1,38 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// SPDX-FileCopyrightText: Copyright The Lance Authors - -//! Data encodings -//! - -use arrow_array::{Array, ArrayRef, UInt32Array}; -use async_trait::async_trait; - -pub mod binary; -pub mod dictionary; -pub mod plain; - -use crate::ReadBatchParams; -use lance_core::Result; - -/// Encoder - Write an arrow array to the file. -#[async_trait] -pub trait Encoder { - /// Write an slice of Arrays, and returns the file offset of the beginning of the batch. - async fn encode(&mut self, array: &[&dyn Array]) -> Result; -} - -/// Decoder - Read Arrow Data. -#[async_trait] -pub trait Decoder: Send + AsyncIndex { - async fn decode(&self) -> Result; - - /// Take by indices. - async fn take(&self, indices: &UInt32Array) -> Result; -} - -#[async_trait] -pub trait AsyncIndex { - type Output: Send + Sync; - - async fn get(&self, index: IndexType) -> Self::Output; -} diff --git a/rust/lance-io/src/lib.rs b/rust/lance-io/src/lib.rs index 2ef686fd551..b6bdc404e39 100644 --- a/rust/lance-io/src/lib.rs +++ b/rust/lance-io/src/lib.rs @@ -11,7 +11,6 @@ use arrow_array::{PrimitiveArray, UInt32Array}; use lance_core::{Error, Result}; -pub mod encodings; pub mod ffi; pub mod local; pub mod object_reader; diff --git a/rust/lance-io/src/local.rs b/rust/lance-io/src/local.rs index 2b8a339331a..0f28dbea0b8 100644 --- a/rust/lance-io/src/local.rs +++ b/rust/lance-io/src/local.rs @@ -3,9 +3,11 @@ //! Optimized local I/Os +use std::collections::HashSet; use std::fs::File; use std::io::{ErrorKind, Read, SeekFrom}; use std::ops::Range; +use std::path::PathBuf; use std::sync::Arc; // TODO: Clean up windows/unix stuff @@ -16,6 +18,7 @@ use std::os::windows::fs::FileExt; use async_trait::async_trait; use bytes::{Bytes, BytesMut}; +use chrono::{DateTime, Utc}; use futures::future::BoxFuture; use lance_core::deepsize::DeepSizeOf; use lance_core::{Error, Result}; @@ -49,6 +52,127 @@ pub fn remove_dir_all(path: &Path) -> Result<()> { Ok(()) } +/// Remove eligible empty directories below `root` without following symbolic links. +pub(crate) fn remove_empty_dirs( + root: &Path, + retained_dirs: &HashSet, + verified_dirs: &HashSet, + unmodified_since: Option>, +) -> Result<()> { + let root_path = PathBuf::from(to_local_path(root)); + let root_metadata = match std::fs::symlink_metadata(&root_path) { + Ok(metadata) => metadata, + Err(err) if err.kind() == ErrorKind::NotFound => return Ok(()), + Err(err) => return Err(Error::from(err)), + }; + if !root_metadata.file_type().is_dir() { + return Ok(()); + } + let canonical_root = std::fs::canonicalize(&root_path)?; + + let mut pending_dirs = vec![(root_path, root.clone(), None::)]; + let mut discovered_dirs = Vec::new(); + let mut file_bearing_roots = HashSet::new(); + let mut first_error = None; + while let Some((local_dir, object_store_dir, index_root)) = pending_dirs.pop() { + let entries = match std::fs::read_dir(&local_dir) { + Ok(entries) => entries, + Err(err) if err.kind() == ErrorKind::NotFound => continue, + Err(err) => return Err(Error::from(err)), + }; + for entry in entries { + let entry = entry?; + // DirEntry::file_type does not follow symbolic links. Non-directories, including + // symlinks, remain in place and prevent their parent from being removed as empty. + if !entry.file_type()?.is_dir() { + if let Some(index_root) = &index_root { + file_bearing_roots.insert(index_root.clone()); + } + continue; + } + let Some(name) = entry.file_name().to_str().map(str::to_owned) else { + continue; + }; + let child = object_store_dir.clone().join(name); + if retained_dirs.contains(&child) { + continue; + } + let local_child = entry.path(); + let index_root = index_root.clone().unwrap_or_else(|| child.clone()); + // Removing a child changes its parent's mtime, so capture age eligibility before + // deepest-first deletion starts. + let is_old_enough = match std::fs::symlink_metadata(&local_child) { + Ok(metadata) => unmodified_since.is_none_or(|threshold| { + metadata + .modified() + .ok() + .map(DateTime::::from) + .is_some_and(|modified| modified < threshold) + }), + Err(err) if err.kind() == ErrorKind::NotFound => continue, + Err(err) => { + first_error.get_or_insert_with(|| Error::from(err)); + false + } + }; + discovered_dirs.push(( + local_child.clone(), + child.clone(), + index_root.clone(), + is_old_enough, + )); + pending_dirs.push((local_child, child, Some(index_root))); + } + } + + discovered_dirs.sort_unstable_by_key(|(_, path, _, _)| std::cmp::Reverse(path.parts_count())); + for (local_dir, object_store_dir, index_root, is_old_enough) in discovered_dirs { + if file_bearing_roots.contains(&index_root) { + continue; + } + match std::fs::symlink_metadata(&local_dir) { + Ok(metadata) if metadata.file_type().is_dir() => {} + Ok(_) => continue, + Err(err) if err.kind() == ErrorKind::NotFound => continue, + Err(err) => { + first_error.get_or_insert_with(|| Error::from(err)); + continue; + } + } + let is_verified = verified_dirs.contains(&object_store_dir); + if !is_verified && !is_old_enough { + continue; + } + + // Canonicalization rejects any directory reached through a symlink outside the supplied + // root. Removing the canonical path also avoids trusting prefixes returned by an object + // store listing. + let canonical_dir = match std::fs::canonicalize(&local_dir) { + Ok(path) if path != canonical_root && path.starts_with(&canonical_root) => path, + Ok(_) => continue, + Err(err) if err.kind() == ErrorKind::NotFound => continue, + Err(err) => { + first_error.get_or_insert_with(|| Error::from(err)); + continue; + } + }; + if let Err(err) = std::fs::remove_dir(canonical_dir) + && !matches!( + err.kind(), + ErrorKind::NotFound | ErrorKind::DirectoryNotEmpty + ) + { + first_error.get_or_insert_with(|| Error::from(err)); + } + } + + if let Some(error) = first_error { + Err(error) + } else { + Ok(()) + } +} + /// Copy a file from one location to another, supporting cross-filesystem copies. /// /// Unlike hard links, this function works across filesystem boundaries. @@ -68,6 +192,24 @@ pub fn copy_file(from: &Path, to: &Path) -> Result<()> { Ok(()) } +/// Await a filesystem operation running on a blocking thread, flattening the +/// join and IO errors into a single `object_store` error. +/// +/// Deliberately not written as `handle.await?` at the call sites: a `JoinError` +/// means the operation panicked, and short-circuiting on it would skip the +/// caller's metrics recording for exactly the failure worth counting. +pub(crate) async fn join_local_io( + handle: tokio::task::JoinHandle>, +) -> object_store::Result { + match handle.await { + Ok(result) => result.map_err(|err| object_store::Error::Generic { + store: "LocalFileSystem", + source: err.into(), + }), + Err(err) => Err(err.into()), + } +} + /// Object reader for local file system. #[derive(Debug)] pub struct LocalObjectReader { @@ -165,14 +307,13 @@ impl Reader for LocalObjectReader { let file = self.file.clone(); self.size .get_or_try_init(|| async move { - let metadata = tokio::task::spawn_blocking(move || { - file.metadata().map_err(|err| object_store::Error::Generic { - store: "LocalFileSystem", - source: err.into(), - }) - }) - .await??; - Ok(metadata.len() as usize) + // The metadata lookup is this reader's equivalent of the HEAD + // request a cloud reader makes to learn the object size. + let metrics = self.io_tracker.begin_io("head"); + let result = + join_local_io(tokio::task::spawn_blocking(move || file.metadata())).await; + metrics.record(&result, 0); + Ok(result?.len() as usize) }) .await .cloned() @@ -189,7 +330,8 @@ impl Reader for LocalObjectReader { let range_u64 = (range.start as u64)..(range.end as u64); Box::pin(async move { - let result = tokio::task::spawn_blocking(move || { + let metrics = io_tracker.begin_io("get"); + let result = join_local_io(tokio::task::spawn_blocking(move || { let mut buf = BytesMut::with_capacity(range.len()); // Safety: `buf` is set with appropriate capacity above. It is // written to below and we check all data is initialized at that point. @@ -200,13 +342,10 @@ impl Reader for LocalObjectReader { read_exact_at(file, buf.as_mut(), range.start as u64)?; Ok(buf.freeze()) - }) - .await? - .map_err(|err: std::io::Error| object_store::Error::Generic { - store: "LocalFileSystem", - source: err.into(), - }); + })) + .await; + metrics.record(&result, num_bytes); if result.is_ok() { io_tracker.record_read("get_range", path, num_bytes, Some(range_u64)); } @@ -223,17 +362,16 @@ impl Reader for LocalObjectReader { let io_tracker = self.io_tracker.clone(); let path = self.path.clone(); - let result = tokio::task::spawn_blocking(move || { + let metrics = io_tracker.begin_io("get"); + let result = join_local_io(tokio::task::spawn_blocking(move || { let mut buf = Vec::new(); file.read_to_end(buf.as_mut())?; Ok(Bytes::from(buf)) - }) - .await? - .map_err(|err: std::io::Error| object_store::Error::Generic { - store: "LocalFileSystem", - source: err.into(), - }); + })) + .await; + let num_bytes = result.as_ref().map_or(0, |bytes| bytes.len() as u64); + metrics.record(&result, num_bytes); if let Ok(bytes) = &result { io_tracker.record_read("get_all", path, bytes.len() as u64, None); } diff --git a/rust/lance-io/src/object_reader.rs b/rust/lance-io/src/object_reader.rs index 1c27800c90f..000686aaf56 100644 --- a/rust/lance-io/src/object_reader.rs +++ b/rust/lance-io/src/object_reader.rs @@ -5,6 +5,7 @@ use std::fs::File; use std::ops::Range; use std::sync::Arc; +use crate::local::join_local_io; #[cfg(windows)] use crate::local::read_exact_at; #[cfg(unix)] @@ -419,7 +420,9 @@ pub(crate) fn stream_local_range( let next = (start + chunk_size).min(end); let file_clone = file.clone(); let path_clone = path.clone(); - let bytes = tokio::task::spawn_blocking(move || { + let num_bytes = (next - start) as u64; + let metrics = io_tracker.begin_io("get"); + let result = join_local_io(tokio::task::spawn_blocking(move || { let mut buf = bytes::BytesMut::with_capacity(next - start); // Safety: buffer capacity matches the exact number of bytes we read below. unsafe { buf.set_len(next - start) }; @@ -428,17 +431,15 @@ pub(crate) fn stream_local_range( #[cfg(windows)] read_exact_at(file_clone, buf.as_mut(), start as u64)?; Ok::<_, std::io::Error>(buf.freeze()) - }) - .await? - .map_err(|err: std::io::Error| object_store::Error::Generic { - store: "LocalFileSystem", - source: err.into(), - })?; + })) + .await; + metrics.record(&result, num_bytes); + let bytes = result?; io_tracker.record_read( "get_range_stream", path_clone, - (next - start) as u64, + num_bytes, Some(start as u64..next as u64), ); diff --git a/rust/lance-io/src/object_store.rs b/rust/lance-io/src/object_store.rs index dafb5e5342f..a8230a578ca 100644 --- a/rust/lance-io/src/object_store.rs +++ b/rust/lance-io/src/object_store.rs @@ -4,7 +4,7 @@ //! Extend [object_store::ObjectStore] functionalities use std::borrow::Cow; -use std::collections::HashMap; +use std::collections::{HashMap, HashSet}; use std::ops::Range; use std::pin::Pin; use std::str::FromStr; @@ -40,6 +40,8 @@ pub(crate) mod dynamic_credentials; #[cfg(any(feature = "oss", feature = "huggingface", feature = "tos"))] pub(crate) mod dynamic_opendal; mod list_retry; +#[cfg(feature = "metrics")] +pub mod metrics; pub mod providers; pub mod storage_options; #[cfg(test)] @@ -284,6 +286,12 @@ impl ObjectStoreParams { } } +fn wrapper_allocation_ptr(wrapper: &Arc) -> *const () { + // Trait object pointers include vtable metadata, which is not stable across codegen units. + // Cache identity must follow the Arc allocation instead. + Arc::as_ptr(wrapper) as *const () +} + // We implement hash for caching impl std::hash::Hash for ObjectStoreParams { #[allow(deprecated)] @@ -300,7 +308,7 @@ impl std::hash::Hash for ObjectStoreParams { Arc::as_ptr(aws_credentials).hash(state); } if let Some(wrapper) = &self.object_store_wrapper { - Arc::as_ptr(wrapper).hash(state); + wrapper_allocation_ptr(wrapper).hash(state); } if let Some(accessor) = &self.storage_options_accessor { accessor.accessor_id().hash(state); @@ -332,8 +340,14 @@ impl PartialEq for ObjectStoreParams { .as_ref() .map(|(store, url)| (Arc::as_ptr(store), url)) && self.s3_credentials_refresh_offset == other.s3_credentials_refresh_offset - && self.object_store_wrapper.as_ref().map(Arc::as_ptr) - == other.object_store_wrapper.as_ref().map(Arc::as_ptr) + && self + .object_store_wrapper + .as_ref() + .map(wrapper_allocation_ptr) + == other + .object_store_wrapper + .as_ref() + .map(wrapper_allocation_ptr) && self .storage_options_accessor .as_ref() @@ -486,12 +500,15 @@ impl ObjectStore { let mut inner = store.clone(); let store_prefix = registry.calculate_object_store_prefix(uri, params.storage_options())?; + + let mut io_tracker = IOTracker::default(); + meter_store(&mut inner, &mut io_tracker, &store_prefix); + if let Some(wrapper) = params.object_store_wrapper.as_ref() { inner = wrapper.wrap(&store_prefix, inner); } // Always wrap with IO tracking - let io_tracker = IOTracker::default(); let tracked_store = io_tracker.wrap("", inner); let store = Self { @@ -845,7 +862,10 @@ impl ObjectStore { ) -> Result<()> { if self.is_local() { // Use std::fs::copy for local filesystem to support cross-filesystem copies - return super::local::copy_file(from, to); + let metrics = self.io_tracker.begin_io("copy"); + let result = super::local::copy_file(from, to); + metrics.record(&result, 0); + return result; } if multipart_copy_fallback { // Reuse the reader for both the size lookup (a single cached HEAD) @@ -909,7 +929,12 @@ impl ObjectStore { if self.is_local() { // The local file system provider needs to delete both files and directories. - return super::local::remove_dir_all(&path); + // Counted as a single delete request, matching how `delete_stream` + // counts one batched request regardless of how many paths it removes. + let metrics = self.io_tracker.begin_io("delete"); + let result = super::local::remove_dir_all(&path); + metrics.record(&result, 0); + return result; } let sub_entries = self .inner @@ -928,6 +953,53 @@ impl ObjectStore { Ok(()) } + /// Remove eligible materialized empty directories below a local root. + /// + /// This is a no-op for object stores, which do not materialize directories. + /// Traversal does not follow symbolic links. Directories in `retained_dirs` and their + /// descendants are preserved. Other directories are removed only if they are empty and + /// either appear in `verified_dirs` or predate `unmodified_since`. Passing `None` for + /// `unmodified_since` disables the age check. + /// + /// ``` + /// # use std::collections::HashSet; + /// # use chrono::Utc; + /// # use lance_core::Result; + /// # use lance_io::object_store::ObjectStore; + /// # async fn remove_stale_index_dirs(store: &ObjectStore) -> Result<()> { + /// store + /// .remove_empty_dirs( + /// "dataset/_indices", + /// HashSet::new(), + /// HashSet::new(), + /// Some(Utc::now()), + /// ) + /// .await?; + /// # Ok(()) + /// # } + /// ``` + pub async fn remove_empty_dirs( + &self, + root_path: impl Into, + retained_dirs: HashSet, + verified_dirs: HashSet, + unmodified_since: Option>, + ) -> Result<()> { + if !self.is_local() && self.scheme != "file-object-store" { + return Ok(()); + } + + let path = Path::parse(root_path.into())?; + let metrics = self.io_tracker.begin_io("delete"); + let result = tokio::task::spawn_blocking(move || { + super::local::remove_empty_dirs(&path, &retained_dirs, &verified_dirs, unmodified_since) + }) + .await + .map_err(|error| Error::io(format!("empty-directory cleanup task failed: {error}")))?; + metrics.record(&result, 0); + result + } + pub fn remove_stream<'a>( &'a self, locations: BoxStream<'a, Result>, @@ -1107,7 +1179,7 @@ static DEFAULT_OBJECT_STORE_REGISTRY: std::sync::LazyLock = impl ObjectStore { #[allow(clippy::too_many_arguments)] pub fn new( - store: Arc, + mut store: Arc, location: Url, block_size: Option, wrapper: Option>, @@ -1132,13 +1204,15 @@ impl ObjectStore { store_prefix } }; + let mut io_tracker = IOTracker::default(); + meter_store(&mut store, &mut io_tracker, &store_prefix); + let store = match wrapper { Some(wrapper) => wrapper.wrap(&store_prefix, store), None => store, }; // Always wrap with IO tracking - let io_tracker = IOTracker::default(); let tracked_store = io_tracker.wrap("", store); Self { @@ -1156,6 +1230,29 @@ impl ObjectStore { } } +/// Wrap `inner` so its operations publish metrics labelled by `store_prefix`, +/// and label `io_tracker` with the same prefix so the local reads and writes +/// that bypass `inner` publish under it too. +/// +/// The two go together on purpose: a store metered on one path but not the other +/// would report a partial picture that reads like a complete one. Every +/// constructor that hands an [`ObjectStore`] to a caller must route its `inner` +/// through here, or through nothing at all. +#[cfg(feature = "metrics")] +fn meter_store(inner: &mut Arc, io_tracker: &mut IOTracker, store_prefix: &str) { + use crate::object_store::metrics::ObjectStoreMetricsExt; + io_tracker.set_metrics_base(store_prefix); + *inner = inner.clone().metered(store_prefix.to_owned()); +} + +#[cfg(not(feature = "metrics"))] +fn meter_store( + _inner: &mut Arc, + _io_tracker: &mut IOTracker, + _store_prefix: &str, +) { +} + fn infer_block_size(scheme: &str) -> usize { // Block size: On local file systems, we use 4KB block size. On cloud // object stores, we use 64KB block size. This is generally the largest @@ -1433,6 +1530,84 @@ mod tests { assert!(!path.join("foo").exists()); } + #[rstest] + #[case("file")] + #[case("file-object-store")] + #[tokio::test] + async fn test_remove_empty_directories(#[case] scheme: &str) { + let path = TempStdDir::default(); + let stale_dir = path.join("stale"); + let nested_stale_dir = path.join("nested_stale"); + let nested_stale_child = nested_stale_dir.join("child"); + create_dir_all(&stale_dir).unwrap(); + create_dir_all(&nested_stale_child).unwrap(); + create_dir_all(path.join("retained").join("child")).unwrap(); + write_to_file( + path.join("file_bearing") + .join("test_file") + .to_str() + .unwrap(), + "keep", + ) + .unwrap(); + create_dir_all(path.join("file_bearing").join("empty_child")).unwrap(); + + let file_url = Url::from_directory_path(&path).unwrap(); + let mut url = Url::parse(&format!("{scheme}:///")).unwrap(); + url.set_path(file_url.path()); + let (store, base) = ObjectStore::from_uri(url.as_ref()).await.unwrap(); + + #[cfg(unix)] + let unmodified_since = { + let old_modified_time = + std::time::SystemTime::now() - std::time::Duration::from_secs(10 * 24 * 60 * 60); + for directory in [&stale_dir, &nested_stale_dir, &nested_stale_child] { + std::fs::File::open(directory) + .unwrap() + .set_times(std::fs::FileTimes::new().set_modified(old_modified_time)) + .unwrap(); + } + DateTime::::from(std::time::SystemTime::now()) + - chrono::TimeDelta::try_days(7).unwrap() + }; + #[cfg(not(unix))] + let unmodified_since = DateTime::::from(std::time::SystemTime::now()) + + chrono::TimeDelta::try_days(1).unwrap(); + + store + .remove_empty_dirs( + base.clone(), + HashSet::from([base.clone().join("retained")]), + HashSet::new(), + Some(unmodified_since), + ) + .await + .unwrap(); + + assert!(!path.join("stale").exists()); + assert!(!path.join("nested_stale").exists()); + assert!(path.join("retained").join("child").exists()); + assert!(path.join("file_bearing").join("empty_child").exists()); + + create_dir_all(path.join("fresh")).unwrap(); + create_dir_all(path.join("verified")).unwrap(); + store + .remove_empty_dirs( + base.clone(), + HashSet::from([base.clone().join("retained")]), + HashSet::from([base.clone().join("verified")]), + Some( + DateTime::::from(std::time::SystemTime::now()) + - chrono::TimeDelta::try_days(7).unwrap(), + ), + ) + .await + .unwrap(); + + assert!(path.join("fresh").exists()); + assert!(!path.join("verified").exists()); + } + #[derive(Debug)] struct TestWrapper { called: AtomicBool, @@ -1459,6 +1634,37 @@ mod tests { } } + #[tokio::test] + async fn test_wrapper_identity_is_stable_across_tasks() { + let wrapper = Arc::new(TestWrapper { + called: AtomicBool::new(false), + return_value: Arc::new(InMemory::new()), + }); + let initial_params = ObjectStoreParams { + object_store_wrapper: Some(wrapper.clone()), + ..ObjectStoreParams::default() + }; + let task_params = tokio::spawn(async move { + ObjectStoreParams { + object_store_wrapper: Some(wrapper), + ..ObjectStoreParams::default() + } + }) + .await + .unwrap(); + + assert_eq!(initial_params, task_params); + + let mut initial_hasher = std::hash::DefaultHasher::new(); + std::hash::Hash::hash(&initial_params, &mut initial_hasher); + let mut task_hasher = std::hash::DefaultHasher::new(); + std::hash::Hash::hash(&task_params, &mut task_hasher); + assert_eq!( + std::hash::Hasher::finish(&initial_hasher), + std::hash::Hasher::finish(&task_hasher) + ); + } + #[tokio::test] async fn test_wrapping_object_store_option_is_used() { // Make a store for the inner store first diff --git a/rust/lance-io/src/object_store/dynamic_opendal.rs b/rust/lance-io/src/object_store/dynamic_opendal.rs index 3367eacd507..ce9809be1d7 100644 --- a/rust/lance-io/src/object_store/dynamic_opendal.rs +++ b/rust/lance-io/src/object_store/dynamic_opendal.rs @@ -278,7 +278,7 @@ mod tests { "Failed to create memory operator: {e:?}" )) })?; - Ok(OpendalStore::new(operator.finish())) + Ok(OpendalStore::new(operator)) }, ); @@ -316,7 +316,7 @@ mod tests { "Failed to create memory operator: {e:?}" )) })?; - Ok(OpendalStore::new(operator.finish())) + Ok(OpendalStore::new(operator)) }, ) .with_protected_keys(["bucket", "root"]); diff --git a/rust/lance-io/src/object_store/metrics.rs b/rust/lance-io/src/object_store/metrics.rs new file mode 100644 index 00000000000..de00c82ffaa --- /dev/null +++ b/rust/lance-io/src/object_store/metrics.rs @@ -0,0 +1,1821 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright The Lance Authors + +//! Publishes object store metrics via the [`metrics`] crate. +//! +//! Two layers cooperate: +//! +//! * [`MeteredObjectStore`] wraps any [`object_store::ObjectStore`] and records +//! per-operation request counts, transferred bytes, latency, errors, and the +//! number of requests currently in flight. It works for every store +//! regardless of backend. +//! * [`MeteringHttpConnector`] wraps the HTTP client used by the native cloud +//! stores (S3 / GCS / Azure) and records throttle / retryable responses per +//! attempt. Because `object_store`'s retry loop re-issues each request +//! through the [`HttpService`](object_store::client::HttpService), this sees +//! every retried response, which a store-level wrapper cannot observe. +//! +//! The two layers have different coverage: every store gets the request-level +//! metrics from [`MeteredObjectStore`], but only the native cloud stores get +//! the HTTP-level throttle metrics. Opendal-backed stores (tos, oss, etc.) +//! bypass `object_store`'s HTTP client, so there is no place to install the +//! connector for them. +//! +//! Neither layer sees the optimized local reads and writes ([`LocalObjectReader`], +//! [`LocalWriter`], the io_uring readers, and the local `copy` / recursive delete +//! shortcuts), which go straight to the filesystem. Those publish the same +//! request-level metrics themselves through +//! [`IOTracker::begin_io`](crate::utils::tracking_store::IOTracker::begin_io). +//! The two are installed together, so a store either publishes for all of its +//! IO or for none of it. A store built by calling a provider's `new_store` +//! directly, bypassing both `ObjectStore` constructors — as +//! [`ObjectStore::local`](crate::object_store::ObjectStore::local) and +//! [`ObjectStore::memory`](crate::object_store::ObjectStore::memory) do — is in +//! the "none of it" case. +//! +//! [`LocalObjectReader`]: crate::local::LocalObjectReader +//! [`LocalWriter`]: crate::object_writer::LocalWriter +//! +//! Metrics carry a `base` label identifying the store. Its cardinality is +//! controlled by the `LANCE_OBJECT_STORE_METRICS_LABEL` environment variable +//! ([`BASE_LABEL_ENV_VAR`]): +//! +//! * `scheme` (default) — scheme only, e.g. `s3`; low, bounded cardinality. +//! * `full` — the full store prefix, e.g. `s3$bucket` or `az$container@account`, +//! so multiple buckets on the same cloud can be told apart. +//! * `off` — omit the `base` label entirely. +//! +//! The metric name constants ([`METRIC_REQUESTS`] etc.) and the recording +//! helpers ([`record_request`], [`record_count`], [`record_error`], +//! [`InFlightGuard`]) are public so custom object stores can emit the same +//! metrics. + +use std::ops::Range; +use std::pin::Pin; +use std::sync::{Arc, OnceLock}; +use std::task::{Context, Poll}; +use std::time::Instant; + +use bytes::Bytes; +use futures::stream::BoxStream; +use futures::{FutureExt, Stream, StreamExt}; +use object_store::path::Path; +use object_store::{ + CopyOptions, GetOptions, GetResult, GetResultPayload, ListResult, MultipartUpload, ObjectMeta, + PutMultipartOptions, PutOptions, PutPayload, PutResult, RenameOptions, Result as OSResult, + UploadPart, +}; + +/// Total number of object store requests, labelled by `operation` and `base`. +pub const METRIC_REQUESTS: &str = "lance_object_store_requests_total"; +/// Total bytes transferred by object store requests, labelled by `operation` and `base`. +pub const METRIC_BYTES: &str = "lance_object_store_request_bytes_total"; +/// Object store request latency in seconds, labelled by `operation` and `base`. +pub const METRIC_DURATION: &str = "lance_object_store_request_duration_seconds"; +/// Total number of failed object store requests, labelled by `operation` and `base`. +pub const METRIC_ERRORS: &str = "lance_object_store_errors_total"; +/// Total number of throttle responses (HTTP 429 / 503) seen at the HTTP layer, +/// labelled by `status` and `base`. Counts every attempt, including retries. +pub const METRIC_THROTTLE: &str = "lance_object_store_throttle_total"; +/// Total number of retryable responses (HTTP 5xx / 429 / 408) seen at the HTTP +/// layer, labelled by `status` and `base`. Counts every attempt, including +/// retries. This is a superset of [`METRIC_THROTTLE`]; 409 (conflict) is +/// deliberately excluded so commit conflicts are not counted as retries. +pub const METRIC_RETRYABLE: &str = "lance_object_store_retryable_responses_total"; +/// Number of object store requests currently in flight, labelled by `operation` +/// and `base`. +pub const METRIC_IN_FLIGHT: &str = "lance_object_store_in_flight_requests"; + +/// Environment variable controlling the cardinality of the `base` label. +pub const BASE_LABEL_ENV_VAR: &str = "LANCE_OBJECT_STORE_METRICS_LABEL"; + +/// Controls how much of a store's identity the `base` label carries, traded off +/// against metric cardinality. Selected via [`BASE_LABEL_ENV_VAR`]. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum BaseLabelMode { + /// Full store prefix, e.g. `s3$bucket` or `az$container@account`. Highest + /// cardinality: one series family per bucket/container. + Full, + /// Scheme only, e.g. `s3`. The default: low, bounded cardinality. + Scheme, + /// Omit the `base` label entirely. + Off, +} + +fn parse_base_label_mode(value: Option<&str>) -> BaseLabelMode { + match value { + Some("full") => BaseLabelMode::Full, + Some("off") | Some("none") => BaseLabelMode::Off, + Some("scheme") | None => BaseLabelMode::Scheme, + Some(other) => { + tracing::warn!( + "Unrecognized {BASE_LABEL_ENV_VAR}={other:?}; \ + expected one of full, scheme, off. Defaulting to scheme." + ); + BaseLabelMode::Scheme + } + } +} + +/// The label mode is read once from the environment and cached for the process. +fn base_label_mode() -> BaseLabelMode { + static MODE: OnceLock = OnceLock::new(); + *MODE.get_or_init(|| parse_base_label_mode(std::env::var(BASE_LABEL_ENV_VAR).ok().as_deref())) +} + +/// Reduce a full store prefix (`scheme$authority`, or just `scheme` for stores +/// without buckets) to the configured `base` label value, or `None` when the +/// label should be omitted. +fn scoped_base(mode: BaseLabelMode, base: &str) -> Option { + match mode { + BaseLabelMode::Full => Some(base.to_owned()), + BaseLabelMode::Scheme => Some(base.split('$').next().unwrap_or(base).to_owned()), + BaseLabelMode::Off => None, + } +} + +/// Build the `operation` (+ optional `base`) label set shared by all +/// store-level metrics, honoring the configured label mode. +fn operation_labels(base: &str, operation: &'static str) -> Vec { + let mut labels = vec![metrics::Label::new("operation", operation)]; + if let Some(base) = scoped_base(base_label_mode(), base) { + labels.push(metrics::Label::new("base", base)); + } + labels +} + +/// Recommended histogram bucket boundaries for [`METRIC_DURATION`], in seconds. +/// +/// Object store requests can take anywhere from a few milliseconds to the +/// client timeout (commonly ~120s), so the boundaries are dense below 10s and +/// keep useful resolution through the timeout band out to 5 minutes. Exporters +/// that aggregate into fixed buckets (e.g. the OpenTelemetry bridge in the +/// Python bindings) use these. +pub const REQUEST_DURATION_BOUNDS: &[f64] = &[ + 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1.0, 2.5, 5.0, // sub-10s + 10.0, 20.0, 30.0, 45.0, 60.0, 90.0, 120.0, 150.0, 180.0, 240.0, 300.0, // 10s–5min +]; + +/// Register descriptions (units and help text) for the object store metrics. +/// +/// This routes through whatever [`metrics::Recorder`] is currently installed, +/// so it must be called *after* the recorder is set. Exporters that build a +/// catalog of available metrics (such as the OpenTelemetry bridge) rely on +/// these descriptions to discover metric names, kinds, and units up front. +pub fn describe_metrics() { + metrics::describe_counter!( + METRIC_REQUESTS, + metrics::Unit::Count, + "Total number of object store requests, by operation and scheme." + ); + metrics::describe_counter!( + METRIC_BYTES, + metrics::Unit::Bytes, + "Total bytes transferred by object store requests, by operation and scheme." + ); + metrics::describe_histogram!( + METRIC_DURATION, + metrics::Unit::Seconds, + "Object store request latency in seconds, by operation and scheme." + ); + metrics::describe_counter!( + METRIC_ERRORS, + metrics::Unit::Count, + "Total number of failed object store requests, by operation and scheme." + ); + metrics::describe_counter!( + METRIC_THROTTLE, + metrics::Unit::Count, + "Total number of throttle responses (HTTP 429 / 503) seen at the HTTP layer, by status and scheme." + ); + metrics::describe_counter!( + METRIC_RETRYABLE, + metrics::Unit::Count, + "Total number of retryable responses (HTTP 5xx / 429 / 408) seen at the HTTP layer, by status and scheme." + ); + metrics::describe_gauge!( + METRIC_IN_FLIGHT, + metrics::Unit::Count, + "Number of object store requests currently in flight, by operation and scheme." + ); +} + +/// Recommended fixed bucket boundaries for the histogram metrics defined here, +/// as `(metric_name, boundaries)` pairs. Exporters that aggregate histograms +/// into fixed buckets read this to configure each histogram. +pub fn histogram_bounds() -> &'static [(&'static str, &'static [f64])] { + &[(METRIC_DURATION, REQUEST_DURATION_BOUNDS)] +} + +/// Record the outcome of a unary request: count, latency, bytes (on success), and errors. +pub fn record_request( + base: &str, + operation: &'static str, + start: Instant, + bytes: u64, + result: &OSResult, +) { + record_outcome(base, operation, start, bytes, result.is_err()); +} + +/// Record count, latency, and either transferred bytes or an error for a +/// completed request. Used both for unary requests and for streamed GETs whose +/// bytes are only known once the body finishes. +pub fn record_outcome( + base: &str, + operation: &'static str, + start: Instant, + bytes: u64, + is_error: bool, +) { + let elapsed = start.elapsed().as_secs_f64(); + let labels = operation_labels(base, operation); + metrics::counter!(METRIC_REQUESTS, labels.clone()).increment(1); + metrics::histogram!(METRIC_DURATION, labels.clone()).record(elapsed); + if is_error { + metrics::counter!(METRIC_ERRORS, labels).increment(1); + } else if bytes > 0 { + metrics::counter!(METRIC_BYTES, labels).increment(bytes); + } +} + +/// Record a single request count without latency, used for streaming operations +/// (list / delete) whose work happens lazily as the stream is polled. +pub fn record_count(base: &str, operation: &'static str) { + metrics::counter!(METRIC_REQUESTS, operation_labels(base, operation)).increment(1); +} + +/// Record a single error for an operation. +pub fn record_error(base: &str, operation: &'static str) { + metrics::counter!(METRIC_ERRORS, operation_labels(base, operation)).increment(1); +} + +/// Raises the in-flight gauge for an operation on creation and lowers it on +/// drop, so the count stays balanced even if the request future or stream is +/// cancelled or dropped before completing. +pub struct InFlightGuard { + labels: Vec, +} + +impl InFlightGuard { + pub fn new(base: &str, operation: &'static str) -> Self { + let labels = operation_labels(base, operation); + metrics::gauge!(METRIC_IN_FLIGHT, labels.clone()).increment(1.0); + Self { labels } + } +} + +impl Drop for InFlightGuard { + fn drop(&mut self) { + metrics::gauge!(METRIC_IN_FLIGHT, self.labels.clone()).decrement(1.0); + } +} + +#[derive(Debug)] +pub struct MeteredObjectStore { + target: Arc, + base: String, +} + +impl std::fmt::Display for MeteredObjectStore { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "MeteredObjectStore({})", self.target) + } +} + +#[async_trait::async_trait] +#[deny(clippy::missing_trait_methods)] +impl object_store::ObjectStore for MeteredObjectStore { + async fn put_opts( + &self, + location: &Path, + bytes: PutPayload, + opts: PutOptions, + ) -> OSResult { + let size = bytes.content_length() as u64; + let _in_flight = InFlightGuard::new(&self.base, "put"); + let start = Instant::now(); + let result = self.target.put_opts(location, bytes, opts).await; + record_request(&self.base, "put", start, size, &result); + result + } + + async fn put_multipart_opts( + &self, + location: &Path, + opts: PutMultipartOptions, + ) -> OSResult> { + let upload = self.target.put_multipart_opts(location, opts).await?; + Ok(Box::new(MeteredMultipartUpload { + target: upload, + base: self.base.clone(), + })) + } + + async fn get_opts(&self, location: &Path, options: GetOptions) -> OSResult { + // `head()` is implemented as a `get_opts` call with `head = true`, so we + // distinguish it here to keep HEAD and GET as separate operations. + let is_head = options.head; + let operation = if is_head { "head" } else { "get" }; + let in_flight = InFlightGuard::new(&self.base, operation); + let start = Instant::now(); + let result = self.target.get_opts(location, options).await; + + // A HEAD transfers only metadata, and errors carry no payload, so both + // are recorded immediately. `get_opts` only resolves once the response + // headers arrive; the body is streamed afterwards, so for a successful + // GET we defer recording until the body has been drained (see below). + if is_head || result.is_err() { + record_request(&self.base, operation, start, 0, &result); + return result; + } + + let result = result.expect("checked to be Ok above"); + Ok(meter_get_result( + result, + self.base.clone(), + start, + in_flight, + )) + } + + async fn get_ranges(&self, location: &Path, ranges: &[Range]) -> OSResult> { + let _in_flight = InFlightGuard::new(&self.base, "get"); + let start = Instant::now(); + let result = self.target.get_ranges(location, ranges).await; + let bytes = match &result { + Ok(parts) => parts.iter().map(|b| b.len() as u64).sum(), + Err(_) => 0, + }; + record_request(&self.base, "get", start, bytes, &result); + result + } + + fn delete_stream( + &self, + locations: BoxStream<'static, OSResult>, + ) -> BoxStream<'static, OSResult> { + let base = self.base.clone(); + // Count one logical delete request per call, matching `list`: a single + // `delete_stream` maps to one batched request on stores that support it + // (e.g. S3's `DeleteObjects`), so counting per yielded path would + // over-count. Errors are still recorded per failing path. + record_count(&self.base, "delete"); + let in_flight = InFlightGuard::new(&self.base, "delete"); + self.target + .delete_stream(locations) + .map(move |result| { + // Reference `in_flight` so this `move` closure captures (owns) + // the guard, keeping the gauge raised until the stream is + // dropped (a move closure only captures the variables it uses). + let _in_flight = &in_flight; + if result.is_err() { + record_error(&base, "delete"); + } + result + }) + .boxed() + } + + fn list(&self, prefix: Option<&Path>) -> BoxStream<'static, OSResult> { + record_count(&self.base, "list"); + meter_list_stream( + self.target.list(prefix), + self.base.clone(), + InFlightGuard::new(&self.base, "list"), + ) + } + + fn list_with_offset( + &self, + prefix: Option<&Path>, + offset: &Path, + ) -> BoxStream<'static, OSResult> { + record_count(&self.base, "list"); + meter_list_stream( + self.target.list_with_offset(prefix, offset), + self.base.clone(), + InFlightGuard::new(&self.base, "list"), + ) + } + + async fn list_with_delimiter(&self, prefix: Option<&Path>) -> OSResult { + let _in_flight = InFlightGuard::new(&self.base, "list"); + let start = Instant::now(); + let result = self.target.list_with_delimiter(prefix).await; + record_request(&self.base, "list", start, 0, &result); + result + } + + async fn copy_opts(&self, from: &Path, to: &Path, opts: CopyOptions) -> OSResult<()> { + let _in_flight = InFlightGuard::new(&self.base, "copy"); + let start = Instant::now(); + let result = self.target.copy_opts(from, to, opts).await; + record_request(&self.base, "copy", start, 0, &result); + result + } + + async fn rename_opts(&self, from: &Path, to: &Path, opts: RenameOptions) -> OSResult<()> { + let _in_flight = InFlightGuard::new(&self.base, "rename"); + let start = Instant::now(); + let result = self.target.rename_opts(from, to, opts).await; + record_request(&self.base, "rename", start, 0, &result); + result + } +} + +/// Count errors yielded while draining a list stream. The request itself is +/// counted once when the stream is created (a single LIST may return many items). +fn meter_list_stream( + stream: BoxStream<'static, OSResult>, + base: String, + in_flight: InFlightGuard, +) -> BoxStream<'static, OSResult> { + stream + .map(move |result| { + // Reference `in_flight` so this `move` closure captures (owns) the + // guard: a move closure only captures the variables it uses, and + // holding it here keeps the gauge raised until the stream is dropped. + let _in_flight = &in_flight; + if result.is_err() { + record_error(&base, "list"); + } + result + }) + .boxed() +} + +/// Wrap a successful GET so the request is recorded once its body has been +/// fully read, capturing the true transfer duration and byte count rather than +/// the time-to-first-byte and declared range. For payloads without a body +/// stream (e.g. a local file handle) the request is recorded immediately. +fn meter_get_result( + mut result: GetResult, + base: String, + start: Instant, + in_flight: InFlightGuard, +) -> GetResult { + match result.payload { + GetResultPayload::Stream(stream) => { + result.payload = GetResultPayload::Stream( + MeteredGetStream { + inner: stream, + base, + start, + bytes: 0, + errored: false, + recorded: false, + _in_flight: in_flight, + } + .boxed(), + ); + result + } + // No body stream to observe (e.g. a local file), so record now. + other => { + let bytes = result.range.end - result.range.start; + record_outcome(&base, "get", start, bytes, false); + result.payload = other; + result + } + } +} + +/// Stream wrapper over a GET body that records the request (count, duration, +/// bytes, errors) once the body is fully drained or the stream is dropped. +struct MeteredGetStream { + inner: BoxStream<'static, OSResult>, + base: String, + start: Instant, + bytes: u64, + errored: bool, + recorded: bool, + _in_flight: InFlightGuard, +} + +impl MeteredGetStream { + fn record(&mut self) { + if self.recorded { + return; + } + self.recorded = true; + record_outcome(&self.base, "get", self.start, self.bytes, self.errored); + } +} + +impl Stream for MeteredGetStream { + type Item = OSResult; + + fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { + match self.inner.poll_next_unpin(cx) { + Poll::Ready(Some(Ok(chunk))) => { + self.bytes += chunk.len() as u64; + Poll::Ready(Some(Ok(chunk))) + } + Poll::Ready(Some(Err(e))) => { + self.errored = true; + Poll::Ready(Some(Err(e))) + } + Poll::Ready(None) => { + self.record(); + Poll::Ready(None) + } + Poll::Pending => Poll::Pending, + } + } +} + +impl Drop for MeteredGetStream { + fn drop(&mut self) { + // Records the partial transfer if the body was dropped before it drained. + self.record(); + } +} + +#[derive(Debug)] +struct MeteredMultipartUpload { + target: Box, + base: String, +} + +#[async_trait::async_trait] +impl MultipartUpload for MeteredMultipartUpload { + fn put_part(&mut self, data: PutPayload) -> UploadPart { + // Each part upload is a distinct request, recorded under the `put_part` + // operation with the same count / bytes / latency / error set as a + // unary put. + let base = self.base.clone(); + let size = data.content_length() as u64; + let inner = self.target.put_part(data); + async move { + let _in_flight = InFlightGuard::new(&base, "put_part"); + let start = Instant::now(); + let result = inner.await; + record_request(&base, "put_part", start, size, &result); + result + } + .boxed() + } + + async fn complete(&mut self) -> OSResult { + // Completing a multipart upload issues its own request that can throttle + // or fail, so it is metered like any other operation. + let _in_flight = InFlightGuard::new(&self.base, "complete_multipart"); + let start = Instant::now(); + let result = self.target.complete().await; + record_request(&self.base, "complete_multipart", start, 0, &result); + result + } + + async fn abort(&mut self) -> OSResult<()> { + let _in_flight = InFlightGuard::new(&self.base, "abort_multipart"); + let start = Instant::now(); + let result = self.target.abort().await; + record_request(&self.base, "abort_multipart", start, 0, &result); + result + } +} + +pub trait ObjectStoreMetricsExt { + /// Wrap this store so its operations publish metrics under the given `base` label. + fn metered(self, base: String) -> Arc; +} + +impl ObjectStoreMetricsExt for Arc { + fn metered(self, base: String) -> Arc { + Arc::new(MeteredObjectStore { target: self, base }) + } +} + +// --- Layer 2: HTTP-level throttle metrics for native cloud stores --- + +#[cfg(any(feature = "aws", feature = "azure", feature = "gcp"))] +mod http { + use super::*; + use object_store::client::{ + ClientOptions, HttpClient, HttpConnector, HttpError, HttpRequest, HttpResponse, + HttpService, ReqwestConnector, + }; + + /// An [`HttpConnector`] that records throttle and retryable responses + /// observed by the underlying HTTP client. Install it on the S3 / GCS / + /// Azure builders via `with_http_connector`. + #[derive(Debug)] + pub struct MeteringHttpConnector { + base: String, + inner: ReqwestConnector, + } + + impl MeteringHttpConnector { + pub fn new(base: String) -> Self { + Self { + base, + inner: ReqwestConnector::default(), + } + } + } + + impl HttpConnector for MeteringHttpConnector { + fn connect(&self, options: &ClientOptions) -> object_store::Result { + let client = self.inner.connect(options)?; + Ok(HttpClient::new(MeteringHttpService { + base: self.base.clone(), + inner: client, + })) + } + } + + #[derive(Debug)] + struct MeteringHttpService { + base: String, + inner: HttpClient, + } + + #[async_trait::async_trait] + impl HttpService for MeteringHttpService { + async fn call(&self, req: HttpRequest) -> Result { + let response = self.inner.execute(req).await?; + let status = response.status().as_u16(); + // Each attempt that object_store may retry is recorded with its + // numeric status. Throttles (429 / 503) are a distinct, narrower + // signal than the broader set of retryable responses, so they get + // their own counter. 409 (conflict) is intentionally excluded from + // the retryable set so commit conflicts are not counted as retries. + let is_throttle = status == 429 || status == 503; + let is_retryable = status == 429 || status == 408 || (500..600).contains(&status); + if is_throttle { + metrics::counter!(METRIC_THROTTLE, status_labels(&self.base, status)).increment(1); + } + if is_retryable { + metrics::counter!(METRIC_RETRYABLE, status_labels(&self.base, status)).increment(1); + } + Ok(response) + } + } + + /// Build the `status` (+ optional `base`) label set for HTTP-layer metrics, + /// honoring the configured label mode. + fn status_labels(base: &str, status: u16) -> Vec { + let mut labels = vec![metrics::Label::new("status", status.to_string())]; + if let Some(base) = scoped_base(base_label_mode(), base) { + labels.push(metrics::Label::new("base", base)); + } + labels + } + + #[cfg(test)] + mod tests { + use super::*; + use metrics_util::debugging::{DebugValue, DebuggingRecorder}; + use object_store::client::{HttpRequestBody, HttpResponseBody}; + + /// A mock [`HttpService`] that always responds with a fixed status code. + #[derive(Debug)] + struct StaticStatusService { + status: u16, + } + + #[async_trait::async_trait] + impl HttpService for StaticStatusService { + async fn call(&self, _req: HttpRequest) -> Result { + Ok(::http::Response::builder() + .status(self.status) + .body(HttpResponseBody::from(Bytes::new())) + .unwrap()) + } + } + + fn request() -> HttpRequest { + ::http::Request::builder() + .method("GET") + .uri("http://example.com/obj") + .body(HttpRequestBody::empty()) + .unwrap() + } + + fn metric_count( + metrics: &[(metrics::Key, DebugValue)], + name: &str, + base: &str, + status: &str, + ) -> u64 { + for (key, value) in metrics { + if key.name() != name { + continue; + } + let labels: std::collections::HashMap<&str, &str> = + key.labels().map(|l| (l.key(), l.value())).collect(); + if labels.get("base") == Some(&base) + && labels.get("status") == Some(&status) + && let DebugValue::Counter(v) = value + { + return *v; + } + } + 0 + } + + #[test] + fn test_throttle_and_retryable_responses_counted_by_status() { + let recorder = DebuggingRecorder::new(); + let snapshotter = recorder.snapshotter(); + metrics::with_local_recorder(&recorder, || { + let rt = tokio::runtime::Builder::new_current_thread() + .build() + .unwrap(); + rt.block_on(async { + // Each attempt that object_store retries flows through `call` + // again; here we simulate that by issuing several responses. + // The base is baked into the connector, so it labels the + // metric. Bases here have no `$`, so they are unaffected by + // the label mode and this test isolates status handling. + for (base, status) in [ + ("s3", 429u16), + ("s3", 503), + ("s3", 503), + ("s3", 500), + ("s3", 408), + ("s3", 409), + ("s3", 200), + ("s3", 404), + ("gs", 429), + ] { + let service = MeteringHttpService { + base: base.into(), + inner: HttpClient::new(StaticStatusService { status }), + }; + service.call(request()).await.unwrap(); + } + }); + }); + + let recorded: Vec<_> = snapshotter + .snapshot() + .into_vec() + .into_iter() + .map(|(ck, _unit, _desc, value)| (ck.key().clone(), value)) + .collect(); + + let throttle = |base, status| metric_count(&recorded, METRIC_THROTTLE, base, status); + let retryable = |base, status| metric_count(&recorded, METRIC_RETRYABLE, base, status); + + // Throttles are only 429 and 503. + assert_eq!(throttle("s3", "429"), 1); + assert_eq!(throttle("s3", "503"), 2); + assert_eq!(throttle("s3", "500"), 0); + assert_eq!(throttle("s3", "408"), 0); + + // Retryable is the broader set: 5xx, 429, 408 (but not 409). + assert_eq!(retryable("s3", "429"), 1); + assert_eq!(retryable("s3", "503"), 2); + assert_eq!(retryable("s3", "500"), 1); + assert_eq!(retryable("s3", "408"), 1); + // 409 conflict is excluded so commit conflicts are not counted as retries. + assert_eq!(retryable("s3", "409"), 0); + + // Success and non-retryable client errors count as neither. + assert_eq!(throttle("s3", "200"), 0); + assert_eq!(retryable("s3", "404"), 0); + + // The base label is taken from the connector, not shared across stores. + assert_eq!(throttle("gs", "429"), 1); + assert_eq!(retryable("gs", "429"), 1); + assert_eq!(throttle("gs", "503"), 0); + } + } +} + +#[cfg(any(feature = "aws", feature = "azure", feature = "gcp"))] +pub use http::MeteringHttpConnector; + +#[cfg(test)] +mod tests { + use super::*; + + use lance_core::utils::tempfile::TempStdDir; + use metrics_util::debugging::{DebugValue, DebuggingRecorder, Snapshotter}; + use object_store::memory::InMemory; + use object_store::{ObjectStoreExt, PutPayload}; + use tokio::io::AsyncWriteExt; + use url::Url; + + use crate::object_store::ObjectStore as LanceObjectStore; + use crate::traits::Writer; + + fn payload(data: &[u8]) -> PutPayload { + PutPayload::from_bytes(Bytes::copy_from_slice(data)) + } + + fn metered_store() -> Arc { + (Arc::new(InMemory::new()) as Arc).metered("memory".into()) + } + + /// A single materialized snapshot of recorded metrics. It must be taken + /// only once: the snapshotter *drains* histogram samples on every + /// `snapshot()` call, so a second snapshot would see empty histograms. + type Metrics = Vec<(metrics::Key, DebugValue)>; + + /// Materialize the current recorder state. Histogram samples are *drained* + /// on each call, so a metric must be read from a single snapshot. + fn snapshot(snapshotter: &Snapshotter) -> Metrics { + snapshotter + .snapshot() + .into_vec() + .into_iter() + .map(|(ck, _unit, _desc, value)| (ck.key().clone(), value)) + .collect() + } + + /// Run an async closure with a thread-local metrics recorder installed and + /// return the resulting metrics. Uses a current-thread runtime so all polls + /// happen on the thread that holds the recorder guard. + fn capture_metrics(f: F) -> Metrics + where + F: FnOnce() -> Fut, + Fut: std::future::Future, + { + let recorder = DebuggingRecorder::new(); + let snapshotter = recorder.snapshotter(); + metrics::with_local_recorder(&recorder, || { + let rt = tokio::runtime::Builder::new_current_thread() + .build() + .unwrap(); + rt.block_on(f()); + }); + snapshot(&snapshotter) + } + + fn key_matches(key: &metrics::Key, name: &str, labels: &[(&str, &str)]) -> bool { + if key.name() != name { + return false; + } + let actual: std::collections::HashSet<(&str, &str)> = + key.labels().map(|l| (l.key(), l.value())).collect(); + labels.len() == actual.len() && labels.iter().all(|l| actual.contains(l)) + } + + fn counter_value(metrics: &Metrics, name: &str, labels: &[(&str, &str)]) -> u64 { + for (key, value) in metrics { + if key_matches(key, name, labels) + && let DebugValue::Counter(v) = value + { + return *v; + } + } + 0 + } + + fn histogram_count(metrics: &Metrics, name: &str, labels: &[(&str, &str)]) -> usize { + for (key, value) in metrics { + if key_matches(key, name, labels) + && let DebugValue::Histogram(samples) = value + { + return samples.len(); + } + } + 0 + } + + fn gauge_value(metrics: &Metrics, name: &str, labels: &[(&str, &str)]) -> f64 { + for (key, value) in metrics { + if key_matches(key, name, labels) + && let DebugValue::Gauge(v) = value + { + return v.0; + } + } + 0.0 + } + + fn has_metric(metrics: &Metrics, name: &str, labels: &[(&str, &str)]) -> bool { + metrics + .iter() + .any(|(key, _)| key_matches(key, name, labels)) + } + + #[test] + fn test_parse_base_label_mode() { + assert_eq!(parse_base_label_mode(None), BaseLabelMode::Scheme); + assert_eq!(parse_base_label_mode(Some("scheme")), BaseLabelMode::Scheme); + assert_eq!(parse_base_label_mode(Some("full")), BaseLabelMode::Full); + assert_eq!(parse_base_label_mode(Some("off")), BaseLabelMode::Off); + assert_eq!(parse_base_label_mode(Some("none")), BaseLabelMode::Off); + // Unrecognized values fall back to the conservative default. + assert_eq!(parse_base_label_mode(Some("bogus")), BaseLabelMode::Scheme); + } + + #[test] + fn test_scoped_base() { + assert_eq!( + scoped_base(BaseLabelMode::Full, "s3$bucket").as_deref(), + Some("s3$bucket") + ); + assert_eq!( + scoped_base(BaseLabelMode::Scheme, "s3$bucket").as_deref(), + Some("s3") + ); + // Azure keeps only the scheme even though its prefix carries the account. + assert_eq!( + scoped_base(BaseLabelMode::Scheme, "az$container@account").as_deref(), + Some("az") + ); + // A prefix without `$` (e.g. memory/file) is unchanged by scheme mode. + assert_eq!( + scoped_base(BaseLabelMode::Scheme, "memory").as_deref(), + Some("memory") + ); + assert_eq!(scoped_base(BaseLabelMode::Off, "s3$bucket"), None); + } + + #[test] + fn test_base_label_defaults_to_scheme() { + // No env var is set in the test process, so the default `scheme` mode + // applies: the full prefix collapses to just the scheme. + let recorded = capture_metrics(|| async { + let store = (Arc::new(InMemory::new()) as Arc) + .metered("s3$my-bucket".into()); + store.put(&Path::from("a"), payload(b"x")).await.unwrap(); + }); + + assert_eq!( + counter_value( + &recorded, + METRIC_REQUESTS, + &[("operation", "put"), ("base", "s3")] + ), + 1 + ); + // The full prefix is not emitted as the label under the default mode. + assert_eq!( + counter_value( + &recorded, + METRIC_REQUESTS, + &[("operation", "put"), ("base", "s3$my-bucket")] + ), + 0 + ); + } + + #[test] + fn test_put_records_count_bytes_and_latency() { + let data = b"hello world"; + let recorded = capture_metrics(|| async { + let store = metered_store(); + store + .put(&Path::from("a/b.bin"), payload(data)) + .await + .unwrap(); + }); + + let labels = [("operation", "put"), ("base", "memory")]; + assert_eq!(counter_value(&recorded, METRIC_REQUESTS, &labels), 1); + assert_eq!( + counter_value(&recorded, METRIC_BYTES, &labels), + data.len() as u64 + ); + assert_eq!(histogram_count(&recorded, METRIC_DURATION, &labels), 1); + } + + #[test] + fn test_get_records_count_and_bytes() { + let data = b"hello world"; + let recorded = capture_metrics(|| async { + let store = metered_store(); + let path = Path::from("a/b.bin"); + store.put(&path, payload(data)).await.unwrap(); + // The GET is only recorded once its body has been fully drained. + store.get(&path).await.unwrap().bytes().await.unwrap(); + }); + + let labels = [("operation", "get"), ("base", "memory")]; + assert_eq!(counter_value(&recorded, METRIC_REQUESTS, &labels), 1); + assert_eq!( + counter_value(&recorded, METRIC_BYTES, &labels), + data.len() as u64 + ); + assert_eq!(histogram_count(&recorded, METRIC_DURATION, &labels), 1); + } + + #[test] + fn test_get_not_recorded_until_body_drained() { + let data = b"hello world"; + let recorder = DebuggingRecorder::new(); + let snapshotter = recorder.snapshotter(); + let labels = [("operation", "get"), ("base", "memory")]; + metrics::with_local_recorder(&recorder, || { + let rt = tokio::runtime::Builder::new_current_thread() + .build() + .unwrap(); + rt.block_on(async { + let store = metered_store(); + let path = Path::from("a/b.bin"); + store.put(&path, payload(data)).await.unwrap(); + + // Holding the result without reading the body records nothing yet. + let result = store.get(&path).await.unwrap(); + assert_eq!( + counter_value(&snapshot(&snapshotter), METRIC_REQUESTS, &labels), + 0 + ); + + // Draining the body records the request with the true byte count. + let bytes = result.bytes().await.unwrap(); + assert_eq!(bytes.len(), data.len()); + let recorded = snapshot(&snapshotter); + assert_eq!(counter_value(&recorded, METRIC_REQUESTS, &labels), 1); + assert_eq!( + counter_value(&recorded, METRIC_BYTES, &labels), + data.len() as u64 + ); + }); + }); + } + + #[test] + fn test_head_is_a_separate_operation() { + let recorded = capture_metrics(|| async { + let store = metered_store(); + let path = Path::from("a/b.bin"); + store.put(&path, payload(b"hello world")).await.unwrap(); + store.head(&path).await.unwrap(); + }); + + assert_eq!( + counter_value( + &recorded, + METRIC_REQUESTS, + &[("operation", "head"), ("base", "memory")] + ), + 1 + ); + // The head call must not be counted as a get. + assert_eq!( + counter_value( + &recorded, + METRIC_REQUESTS, + &[("operation", "get"), ("base", "memory")] + ), + 0 + ); + // A HEAD transfers only metadata, so it records no payload bytes. + assert_eq!( + counter_value( + &recorded, + METRIC_BYTES, + &[("operation", "head"), ("base", "memory")] + ), + 0 + ); + } + + #[test] + fn test_delete_records_one_request_per_call() { + let recorded = capture_metrics(|| async { + let store = metered_store(); + for i in 0..3 { + store + .put(&Path::from(format!("a/{i}.bin")), payload(b"x")) + .await + .unwrap(); + } + // `delete` drives `delete_stream`; deleting three paths is still one + // logical delete request (a single batched request on real stores). + let paths = + futures::stream::iter((0..3).map(|i| Ok(Path::from(format!("a/{i}.bin"))))).boxed(); + let _: Vec<_> = store.delete_stream(paths).collect().await; + }); + + assert_eq!( + counter_value( + &recorded, + METRIC_REQUESTS, + &[("operation", "delete"), ("base", "memory")] + ), + 1 + ); + } + + #[test] + fn test_list_counts_one_request_not_per_item() { + let recorded = capture_metrics(|| async { + let store = metered_store(); + for i in 0..3 { + store + .put(&Path::from(format!("a/{i}.bin")), payload(b"x")) + .await + .unwrap(); + } + let _: Vec<_> = store.list(Some(&Path::from("a"))).collect().await; + }); + + assert_eq!( + counter_value( + &recorded, + METRIC_REQUESTS, + &[("operation", "list"), ("base", "memory")] + ), + 1 + ); + } + + #[test] + fn test_error_is_counted() { + let recorded = capture_metrics(|| async { + let store = metered_store(); + // Getting a missing object errors. + let _ = store.get(&Path::from("does/not/exist")).await; + }); + + let labels = [("operation", "get"), ("base", "memory")]; + assert_eq!(counter_value(&recorded, METRIC_ERRORS, &labels), 1); + // A failed request is still counted as a request, with latency recorded. + assert_eq!(counter_value(&recorded, METRIC_REQUESTS, &labels), 1); + assert_eq!(histogram_count(&recorded, METRIC_DURATION, &labels), 1); + // No bytes are transferred on a failed get. + assert_eq!(counter_value(&recorded, METRIC_BYTES, &labels), 0); + } + + #[test] + fn test_get_ranges_sums_part_bytes_and_labels_get() { + let recorded = capture_metrics(|| async { + let store = metered_store(); + let path = Path::from("a/b.bin"); + store.put(&path, payload(b"hello world")).await.unwrap(); + // Two disjoint ranges of 3 bytes each. + store.get_ranges(&path, &[2..5, 6..9]).await.unwrap(); + }); + + let labels = [("operation", "get"), ("base", "memory")]; + assert_eq!(counter_value(&recorded, METRIC_REQUESTS, &labels), 1); + assert_eq!(counter_value(&recorded, METRIC_BYTES, &labels), 6); + assert_eq!(histogram_count(&recorded, METRIC_DURATION, &labels), 1); + } + + #[test] + fn test_copy_and_rename_record_zero_bytes() { + let recorded = capture_metrics(|| async { + let store = metered_store(); + store + .put(&Path::from("a/src"), payload(b"x")) + .await + .unwrap(); + store + .copy(&Path::from("a/src"), &Path::from("a/copy")) + .await + .unwrap(); + store + .rename(&Path::from("a/copy"), &Path::from("a/moved")) + .await + .unwrap(); + }); + + for operation in ["copy", "rename"] { + let labels = [("operation", operation), ("base", "memory")]; + assert_eq!(counter_value(&recorded, METRIC_REQUESTS, &labels), 1); + assert_eq!(counter_value(&recorded, METRIC_BYTES, &labels), 0); + assert_eq!(histogram_count(&recorded, METRIC_DURATION, &labels), 1); + } + } + + #[test] + fn test_list_with_delimiter_records_latency() { + let recorded = capture_metrics(|| async { + let store = metered_store(); + store.put(&Path::from("a/b"), payload(b"x")).await.unwrap(); + store + .list_with_delimiter(Some(&Path::from("a"))) + .await + .unwrap(); + }); + + let labels = [("operation", "list"), ("base", "memory")]; + assert_eq!(counter_value(&recorded, METRIC_REQUESTS, &labels), 1); + assert_eq!(histogram_count(&recorded, METRIC_DURATION, &labels), 1); + assert_eq!(counter_value(&recorded, METRIC_BYTES, &labels), 0); + } + + #[test] + fn test_list_with_offset_counts_one_request() { + let recorded = capture_metrics(|| async { + let store = metered_store(); + for i in 0..3 { + store + .put(&Path::from(format!("a/{i}")), payload(b"x")) + .await + .unwrap(); + } + let _: Vec<_> = store + .list_with_offset(Some(&Path::from("a")), &Path::from("a/0")) + .collect() + .await; + }); + + assert_eq!( + counter_value( + &recorded, + METRIC_REQUESTS, + &[("operation", "list"), ("base", "memory")] + ), + 1 + ); + } + + #[test] + fn test_multipart_records_each_part_and_complete() { + let recorded = capture_metrics(|| async { + let store = metered_store(); + let mut upload = store.put_multipart(&Path::from("a/big")).await.unwrap(); + upload.put_part(payload(b"hello")).await.unwrap(); // 5 bytes + upload.put_part(payload(b"world!!")).await.unwrap(); // 7 bytes + upload.complete().await.unwrap(); + }); + + let part_labels = [("operation", "put_part"), ("base", "memory")]; + assert_eq!(counter_value(&recorded, METRIC_REQUESTS, &part_labels), 2); + assert_eq!(counter_value(&recorded, METRIC_BYTES, &part_labels), 12); + // Each part records its own latency sample, like a unary put. + assert_eq!(histogram_count(&recorded, METRIC_DURATION, &part_labels), 2); + // A successful part upload records no error. + assert_eq!(counter_value(&recorded, METRIC_ERRORS, &part_labels), 0); + + // Completing the upload is its own metered request. + let complete_labels = [("operation", "complete_multipart"), ("base", "memory")]; + assert_eq!( + counter_value(&recorded, METRIC_REQUESTS, &complete_labels), + 1 + ); + assert_eq!( + histogram_count(&recorded, METRIC_DURATION, &complete_labels), + 1 + ); + } + + #[test] + fn test_multipart_abort_is_recorded() { + let recorded = capture_metrics(|| async { + let store = metered_store(); + let mut upload = store.put_multipart(&Path::from("a/big")).await.unwrap(); + upload.put_part(payload(b"hello")).await.unwrap(); + upload.abort().await.unwrap(); + }); + + let labels = [("operation", "abort_multipart"), ("base", "memory")]; + assert_eq!(counter_value(&recorded, METRIC_REQUESTS, &labels), 1); + assert_eq!(histogram_count(&recorded, METRIC_DURATION, &labels), 1); + } + + #[test] + fn test_multipart_part_error_is_counted() { + let recorded = capture_metrics(|| async { + let store = (Arc::new(FailingStreamStore) as Arc) + .metered("memory".into()); + let mut upload = store.put_multipart(&Path::from("a/big")).await.unwrap(); + let _ = upload.put_part(payload(b"data")).await; + }); + + let labels = [("operation", "put_part"), ("base", "memory")]; + assert_eq!(counter_value(&recorded, METRIC_REQUESTS, &labels), 1); + assert_eq!(counter_value(&recorded, METRIC_ERRORS, &labels), 1); + assert_eq!(histogram_count(&recorded, METRIC_DURATION, &labels), 1); + // A failed part transfers no counted bytes. + assert_eq!(counter_value(&recorded, METRIC_BYTES, &labels), 0); + } + + #[test] + fn test_in_flight_guard_tracks_and_releases() { + let recorder = DebuggingRecorder::new(); + let snapshotter = recorder.snapshotter(); + let labels = [("operation", "get"), ("base", "memory")]; + metrics::with_local_recorder(&recorder, || { + let g1 = InFlightGuard::new("memory", "get"); + let g2 = InFlightGuard::new("memory", "get"); + assert_eq!( + gauge_value(&snapshot(&snapshotter), METRIC_IN_FLIGHT, &labels), + 2.0 + ); + drop(g1); + assert_eq!( + gauge_value(&snapshot(&snapshotter), METRIC_IN_FLIGHT, &labels), + 1.0 + ); + drop(g2); + assert_eq!( + gauge_value(&snapshot(&snapshotter), METRIC_IN_FLIGHT, &labels), + 0.0 + ); + }); + } + + #[test] + fn test_in_flight_gauge_is_wired_and_balances() { + let recorded = capture_metrics(|| async { + let store = metered_store(); + let path = Path::from("a/b.bin"); + store.put(&path, payload(b"hello")).await.unwrap(); + store.get(&path).await.unwrap(); + }); + + // The gauge is emitted for each operation (guard is wired in) and, once + // the operation completes, balances back to zero. + for operation in ["put", "get"] { + let labels = [("operation", operation), ("base", "memory")]; + assert!(has_metric(&recorded, METRIC_IN_FLIGHT, &labels)); + assert_eq!(gauge_value(&recorded, METRIC_IN_FLIGHT, &labels), 0.0); + } + } + + #[test] + fn test_list_stream_holds_in_flight_until_dropped() { + let recorder = DebuggingRecorder::new(); + let snapshotter = recorder.snapshotter(); + let labels = [("operation", "list"), ("base", "memory")]; + metrics::with_local_recorder(&recorder, || { + let rt = tokio::runtime::Builder::new_current_thread() + .build() + .unwrap(); + rt.block_on(async { + let store = metered_store(); + store.put(&Path::from("a/x"), payload(b"x")).await.unwrap(); + + // Creating the stream raises the gauge; it stays raised until the + // stream is dropped, even before any items are drained. + let stream = store.list(Some(&Path::from("a"))); + assert_eq!( + gauge_value(&snapshot(&snapshotter), METRIC_IN_FLIGHT, &labels), + 1.0 + ); + drop(stream); + assert_eq!( + gauge_value(&snapshot(&snapshotter), METRIC_IN_FLIGHT, &labels), + 0.0 + ); + }); + }); + } + + #[test] + fn test_delete_stream_holds_in_flight_until_dropped() { + let recorder = DebuggingRecorder::new(); + let snapshotter = recorder.snapshotter(); + let labels = [("operation", "delete"), ("base", "memory")]; + metrics::with_local_recorder(&recorder, || { + let rt = tokio::runtime::Builder::new_current_thread() + .build() + .unwrap(); + rt.block_on(async { + let store = metered_store(); + let locations = futures::stream::iter(vec![Ok(Path::from("a/b"))]).boxed(); + + // Like list, creating the delete stream raises the gauge and holds + // it until the stream is dropped, before any items are drained. + let stream = store.delete_stream(locations); + assert_eq!( + gauge_value(&snapshot(&snapshotter), METRIC_IN_FLIGHT, &labels), + 1.0 + ); + drop(stream); + assert_eq!( + gauge_value(&snapshot(&snapshotter), METRIC_IN_FLIGHT, &labels), + 0.0 + ); + }); + }); + } + + #[test] + fn test_in_flight_released_when_operation_future_dropped() { + let recorder = DebuggingRecorder::new(); + let snapshotter = recorder.snapshotter(); + let labels = [("operation", "get"), ("base", "memory")]; + metrics::with_local_recorder(&recorder, || { + let rt = tokio::runtime::Builder::new_current_thread() + .build() + .unwrap(); + rt.block_on(async { + let started = Arc::new(tokio::sync::Notify::new()); + // Never signalled: the request stays blocked mid-flight. + let release = Arc::new(tokio::sync::Notify::new()); + let store = (Arc::new(BlockingStore { + started: started.clone(), + release, + }) as Arc) + .metered("memory".into()); + + let path = Path::from("a/b"); + let mut fut = Box::pin(store.get(&path)); + // Drive the request until it is blocked inside the inner store. + tokio::select! { + _ = &mut fut => unreachable!("the blocking store never returns"), + _ = started.notified() => {} + } + + // The gauge is raised while the request is outstanding, and + // dropping the future before it completes releases it. + assert_eq!( + gauge_value(&snapshot(&snapshotter), METRIC_IN_FLIGHT, &labels), + 1.0 + ); + drop(fut); + assert_eq!( + gauge_value(&snapshot(&snapshotter), METRIC_IN_FLIGHT, &labels), + 0.0 + ); + }); + }); + } + + #[test] + fn test_streaming_errors_are_counted() { + let recorded = capture_metrics(|| async { + let delete_store = (Arc::new(FailingStreamStore) as Arc) + .metered("memory".into()); + let _ = delete_store.delete(&Path::from("a/b")).await; + + let list_store = (Arc::new(FailingStreamStore) as Arc) + .metered("memory".into()); + let _: Vec<_> = list_store.list(None).collect().await; + }); + + // delete_stream counts the item and records an error when it fails. + let delete_labels = [("operation", "delete"), ("base", "memory")]; + assert_eq!(counter_value(&recorded, METRIC_REQUESTS, &delete_labels), 1); + assert_eq!(counter_value(&recorded, METRIC_ERRORS, &delete_labels), 1); + + // A list request is counted once; a failure while draining records an error. + let list_labels = [("operation", "list"), ("base", "memory")]; + assert_eq!(counter_value(&recorded, METRIC_REQUESTS, &list_labels), 1); + assert_eq!(counter_value(&recorded, METRIC_ERRORS, &list_labels), 1); + } + + /// The optimized local reads and writes talk to the filesystem directly, so + /// they never reach [`MeteredObjectStore`] and publish these metrics + /// themselves. They must land under the same `base` label as the store's + /// metered operations, which for a local store is its scheme. + #[test] + fn test_local_filesystem_io_is_metered() { + let tmp = TempStdDir::default(); + let dir = tmp.join("sub"); + let data = b"hello world"; + let recorded = capture_metrics(|| async { + // Built through the registry, like any store opened from a URI. + let (store, path) = LanceObjectStore::from_uri(dir.join("a.bin").to_str().unwrap()) + .await + .unwrap(); + // Writes go through LocalWriter. + store.put(&path, data).await.unwrap(); + + // Reads go through LocalObjectReader. + let reader = store.open(&path).await.unwrap(); + assert_eq!(reader.size().await.unwrap(), data.len()); + assert_eq!(reader.get_range(0..5).await.unwrap().len(), 5); + assert_eq!(reader.get_all().await.unwrap().len(), data.len()); + // The file is smaller than the block size, so it streams as one chunk. + let chunks: Vec<_> = reader.get_stream().await.unwrap().collect().await; + assert_eq!(chunks.len(), 1); + + // Copy and recursive delete both shortcut to the filesystem too. + store + .copy(&path, &Path::from_absolute_path(dir.join("b.bin")).unwrap()) + .await + .unwrap(); + store + .remove_dir_all(Path::from_absolute_path(&dir).unwrap()) + .await + .unwrap(); + }); + + let put_labels = [("operation", "put"), ("base", "file")]; + assert_eq!(counter_value(&recorded, METRIC_REQUESTS, &put_labels), 1); + assert_eq!( + counter_value(&recorded, METRIC_BYTES, &put_labels), + data.len() as u64 + ); + assert_eq!(histogram_count(&recorded, METRIC_DURATION, &put_labels), 1); + assert_eq!(gauge_value(&recorded, METRIC_IN_FLIGHT, &put_labels), 0.0); + + // One request each for the range read, the full read and the single + // streamed chunk. + let get_labels = [("operation", "get"), ("base", "file")]; + assert_eq!(counter_value(&recorded, METRIC_REQUESTS, &get_labels), 3); + assert_eq!( + counter_value(&recorded, METRIC_BYTES, &get_labels), + (5 + 2 * data.len()) as u64 + ); + assert_eq!(histogram_count(&recorded, METRIC_DURATION, &get_labels), 3); + assert_eq!(gauge_value(&recorded, METRIC_IN_FLIGHT, &get_labels), 0.0); + + // The size lookup is the local equivalent of a HEAD, and transfers no + // payload bytes. + let head_labels = [("operation", "head"), ("base", "file")]; + assert_eq!(counter_value(&recorded, METRIC_REQUESTS, &head_labels), 1); + assert_eq!(counter_value(&recorded, METRIC_BYTES, &head_labels), 0); + + for operation in ["copy", "delete"] { + let labels = [("operation", operation), ("base", "file")]; + assert_eq!(counter_value(&recorded, METRIC_REQUESTS, &labels), 1); + assert_eq!(counter_value(&recorded, METRIC_BYTES, &labels), 0); + } + + assert_eq!(counter_value(&recorded, METRIC_ERRORS, &get_labels), 0); + assert_eq!(counter_value(&recorded, METRIC_ERRORS, &put_labels), 0); + } + + #[test] + fn test_local_read_error_is_counted() { + let tmp = TempStdDir::default(); + let recorded = capture_metrics(|| async { + let (store, path) = LanceObjectStore::from_uri(tmp.join("a.bin").to_str().unwrap()) + .await + .unwrap(); + store.put(&path, b"hello").await.unwrap(); + + let reader = store.open(&path).await.unwrap(); + // Reading past the end of the file fails. + assert!(reader.get_range(0..100).await.is_err()); + }); + + let labels = [("operation", "get"), ("base", "file")]; + assert_eq!(counter_value(&recorded, METRIC_ERRORS, &labels), 1); + // A failed read is still counted as a request, with latency recorded. + assert_eq!(counter_value(&recorded, METRIC_REQUESTS, &labels), 1); + assert_eq!(histogram_count(&recorded, METRIC_DURATION, &labels), 1); + assert_eq!(counter_value(&recorded, METRIC_BYTES, &labels), 0); + } + + /// A local write is reported as a single `put` covering the whole file, so it + /// stays in flight until the file is persisted under its final path. + #[test] + fn test_local_write_is_in_flight_until_persisted() { + let tmp = TempStdDir::default(); + let recorder = DebuggingRecorder::new(); + let snapshotter = recorder.snapshotter(); + let labels = [("operation", "put"), ("base", "file")]; + metrics::with_local_recorder(&recorder, || { + let rt = tokio::runtime::Builder::new_current_thread() + .build() + .unwrap(); + rt.block_on(async { + let (store, path) = LanceObjectStore::from_uri(tmp.join("a.bin").to_str().unwrap()) + .await + .unwrap(); + let mut writer = store.create(&path).await.unwrap(); + writer.write_all(b"hello").await.unwrap(); + + let recorded = snapshot(&snapshotter); + assert_eq!(gauge_value(&recorded, METRIC_IN_FLIGHT, &labels), 1.0); + assert_eq!(counter_value(&recorded, METRIC_REQUESTS, &labels), 0); + + Writer::shutdown(writer.as_mut()).await.unwrap(); + let recorded = snapshot(&snapshotter); + assert_eq!(gauge_value(&recorded, METRIC_IN_FLIGHT, &labels), 0.0); + assert_eq!(counter_value(&recorded, METRIC_REQUESTS, &labels), 1); + assert_eq!(counter_value(&recorded, METRIC_BYTES, &labels), 5); + }); + }); + } + + /// A store handed in by the caller is metered like one built by the registry. + #[test] + fn test_caller_supplied_store_is_metered() { + let recorded = capture_metrics(|| async { + #[allow(deprecated)] + let params = crate::object_store::ObjectStoreParams { + object_store: Some(( + Arc::new(InMemory::new()) as Arc, + Url::parse("memory:///").unwrap(), + )), + ..Default::default() + }; + let (store, _) = LanceObjectStore::from_uri_and_params( + Arc::new(crate::object_store::ObjectStoreRegistry::default()), + "memory:///", + ¶ms, + ) + .await + .unwrap(); + store.put(&Path::from("a"), b"hello").await.unwrap(); + }); + + assert_eq!( + counter_value( + &recorded, + METRIC_REQUESTS, + &[("operation", "put"), ("base", "memory")] + ), + 1 + ); + } + + /// `ObjectStore::new` is how `DatasetBuilder` wraps a caller-supplied store, + /// so it must meter both halves of the store: the operations that go through + /// `inner`, and the local ones that bypass it. Metering only one half would + /// report a partial picture that reads like a complete one. + #[test] + fn test_store_built_from_new_is_metered() { + let tmp = TempStdDir::default(); + let recorded = capture_metrics(|| async { + let store = LanceObjectStore::new( + Arc::new(object_store::local::LocalFileSystem::new()), + Url::parse("file:///").unwrap(), + None, + None, + false, + false, + 1, + 3, + None, + ); + let path = Path::from_absolute_path(tmp.join("a.bin")).unwrap(); + // put and open bypass `inner` and publish for themselves. + store.put(&path, b"hello").await.unwrap(); + let reader = store.open(&path).await.unwrap(); + assert_eq!(reader.get_all().await.unwrap().len(), 5); + // delete goes through `inner`, so only MeteredObjectStore can count it. + store.delete(&path).await.unwrap(); + }); + + for (operation, bytes) in [("put", 5), ("get", 5), ("delete", 0)] { + let labels = [("operation", operation), ("base", "file")]; + assert_eq!( + counter_value(&recorded, METRIC_REQUESTS, &labels), + 1, + "expected one {operation} request" + ); + assert_eq!(counter_value(&recorded, METRIC_BYTES, &labels), bytes); + } + } + + /// A store whose stream-producing operations always yield an error, used to + /// exercise the error branches of the streaming wrappers. + #[derive(Debug)] + struct FailingStreamStore; + + impl std::fmt::Display for FailingStreamStore { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "FailingStreamStore") + } + } + + fn test_error() -> object_store::Error { + object_store::Error::Generic { + store: "FailingStreamStore", + source: "injected failure".into(), + } + } + + #[async_trait::async_trait] + impl object_store::ObjectStore for FailingStreamStore { + async fn put_opts( + &self, + _location: &Path, + _bytes: PutPayload, + _opts: PutOptions, + ) -> OSResult { + unimplemented!() + } + + async fn put_multipart_opts( + &self, + _location: &Path, + _opts: PutMultipartOptions, + ) -> OSResult> { + Ok(Box::new(FailingUpload)) + } + + async fn get_opts(&self, _location: &Path, _options: GetOptions) -> OSResult { + unimplemented!() + } + + fn delete_stream( + &self, + _locations: BoxStream<'static, OSResult>, + ) -> BoxStream<'static, OSResult> { + futures::stream::once(async { Err(test_error()) }).boxed() + } + + fn list(&self, _prefix: Option<&Path>) -> BoxStream<'static, OSResult> { + futures::stream::once(async { Err(test_error()) }).boxed() + } + + fn list_with_offset( + &self, + _prefix: Option<&Path>, + _offset: &Path, + ) -> BoxStream<'static, OSResult> { + unimplemented!() + } + + async fn list_with_delimiter(&self, _prefix: Option<&Path>) -> OSResult { + unimplemented!() + } + + async fn copy_opts(&self, _from: &Path, _to: &Path, _opts: CopyOptions) -> OSResult<()> { + unimplemented!() + } + + async fn rename_opts( + &self, + _from: &Path, + _to: &Path, + _opts: RenameOptions, + ) -> OSResult<()> { + unimplemented!() + } + } + + /// A [`MultipartUpload`] whose part uploads always fail, used to exercise the + /// error branch of the metered `put_part`. + #[derive(Debug)] + struct FailingUpload; + + #[async_trait::async_trait] + impl MultipartUpload for FailingUpload { + fn put_part(&mut self, _data: PutPayload) -> UploadPart { + async { Err(test_error()) }.boxed() + } + + async fn complete(&mut self) -> OSResult { + unimplemented!() + } + + async fn abort(&mut self) -> OSResult<()> { + unimplemented!() + } + } + + /// A store whose `get_opts` blocks after signalling `started`, so a request + /// can be observed mid-flight and then dropped before it completes. + #[derive(Debug)] + struct BlockingStore { + started: Arc, + release: Arc, + } + + impl std::fmt::Display for BlockingStore { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "BlockingStore") + } + } + + #[async_trait::async_trait] + impl object_store::ObjectStore for BlockingStore { + async fn put_opts( + &self, + _location: &Path, + _bytes: PutPayload, + _opts: PutOptions, + ) -> OSResult { + unimplemented!() + } + + async fn put_multipart_opts( + &self, + _location: &Path, + _opts: PutMultipartOptions, + ) -> OSResult> { + unimplemented!() + } + + async fn get_opts(&self, _location: &Path, _options: GetOptions) -> OSResult { + self.started.notify_one(); + self.release.notified().await; + unreachable!("release is never signalled in the test") + } + + fn delete_stream( + &self, + _locations: BoxStream<'static, OSResult>, + ) -> BoxStream<'static, OSResult> { + unimplemented!() + } + + fn list(&self, _prefix: Option<&Path>) -> BoxStream<'static, OSResult> { + unimplemented!() + } + + fn list_with_offset( + &self, + _prefix: Option<&Path>, + _offset: &Path, + ) -> BoxStream<'static, OSResult> { + unimplemented!() + } + + async fn list_with_delimiter(&self, _prefix: Option<&Path>) -> OSResult { + unimplemented!() + } + + async fn copy_opts(&self, _from: &Path, _to: &Path, _opts: CopyOptions) -> OSResult<()> { + unimplemented!() + } + + async fn rename_opts( + &self, + _from: &Path, + _to: &Path, + _opts: RenameOptions, + ) -> OSResult<()> { + unimplemented!() + } + } +} diff --git a/rust/lance-io/src/object_store/providers.rs b/rust/lance-io/src/object_store/providers.rs index 775c98552a8..b84dc7362b8 100644 --- a/rust/lance-io/src/object_store/providers.rs +++ b/rust/lance-io/src/object_store/providers.rs @@ -265,6 +265,10 @@ impl ObjectStoreRegistry { store.inner = store.inner.traced(); + // Label metrics by the store's unique prefix (e.g. `s3$bucket`, + // `az$container@account`) so multiple stores on one cloud differ. + crate::object_store::meter_store(&mut store.inner, &mut store.io_tracker, &cache_path); + if let Some(wrapper) = ¶ms.object_store_wrapper { store.inner = wrapper.wrap(&cache_path, store.inner); } diff --git a/rust/lance-io/src/object_store/providers/aws.rs b/rust/lance-io/src/object_store/providers/aws.rs index 9aad637bce2..8464f228933 100644 --- a/rust/lance-io/src/object_store/providers/aws.rs +++ b/rust/lance-io/src/object_store/providers/aws.rs @@ -13,7 +13,11 @@ use object_store::ObjectStore as OSObjectStore; use object_store_opendal::OpendalStore; use opendal::{Operator, services::S3}; +use aws_config::Region; use aws_config::default_provider::credentials::DefaultCredentialsChain; +use aws_config::ecs::EcsCredentialsProvider; +use aws_config::provider_config::ProviderConfig; +use aws_config::web_identity_token::WebIdentityTokenCredentialsProvider; use aws_credential_types::provider::ProvideCredentials; use object_store::{ ClientOptions, CredentialProvider, Result as ObjectStoreResult, RetryConfig, @@ -30,7 +34,7 @@ use crate::object_store::{ DEFAULT_CLOUD_BLOCK_SIZE, DEFAULT_CLOUD_IO_PARALLELISM, DEFAULT_MAX_IOP_SIZE, ObjectStore, ObjectStoreParams, ObjectStoreProvider, StorageOptions, StorageOptionsAccessor, dynamic_credentials::{NamespaceCredentialsProvider, build_dynamic_credential_provider}, - throttle::{AimdThrottleConfig, AimdThrottledStore}, + throttle::{AimdThrottleConfig, AimdThrottleState, AimdThrottledStore, cloud_http_connector}, }; use lance_core::error::{Error, Result}; @@ -44,6 +48,7 @@ impl AwsStoreProvider { params: &ObjectStoreParams, storage_options: &StorageOptions, is_s3_express: bool, + throttle_state: Option<&AimdThrottleState>, ) -> Result> { // Use a low retry count since the AIMD throttle layer handles // throttle recovery with its own retry loop. @@ -59,12 +64,15 @@ impl AwsStoreProvider { // Get accessor from params let accessor = params.get_accessor(); + let provider_scheme = storage_options.aws_provider_scheme()?; + let (aws_creds, region) = build_aws_credential( params.s3_credentials_refresh_offset, params.aws_credentials.clone(), Some(&s3_storage_options), region, accessor, + provider_scheme, ) .await?; @@ -73,6 +81,11 @@ impl AwsStoreProvider { s3_storage_options.insert(AmazonS3ConfigKey::S3Express, true.to_string()); } + // Compute the metrics label before rewriting the URL below so it + // matches the prefix the registry uses to key this store. + let store_prefix = + self.calculate_object_store_prefix(base_path, Some(&storage_options.0))?; + // before creating the OSObjectStore we need to rewrite the url to drop ddb related parts base_path.set_scheme("s3").unwrap(); base_path.set_query(None); @@ -89,6 +102,8 @@ impl AwsStoreProvider { .with_retry(retry_config) .with_region(region); + builder = builder.with_http_connector(cloud_http_connector(throttle_state, store_prefix)); + Ok(Arc::new(builder.build()?) as Arc) } @@ -108,6 +123,13 @@ impl AwsStoreProvider { // OpenDAL will handle environment variables through its default credentials chain let mut config_map: HashMap = storage_options.0.clone(); + if let Some(provider_scheme) = storage_options.aws_provider_scheme()? { + return Result::Err(Error::not_supported(format!( + "OpendalStore does not currently support an explicit provider_scheme (currently set to {:?})", + provider_scheme + ))); + } + // Set required OpenDAL configuration config_map.insert("bucket".to_string(), bucket); @@ -116,8 +138,7 @@ impl AwsStoreProvider { } let operator = Operator::from_iter::(config_map) - .map_err(|e| Error::invalid_input(format!("Failed to create S3 operator: {:?}", e)))? - .finish(); + .map_err(|e| Error::invalid_input(format!("Failed to create S3 operator: {:?}", e)))?; Ok(Arc::new(OpendalStore::new(operator)) as Arc) } @@ -151,20 +172,36 @@ impl ObjectStoreProvider for AwsStoreProvider { .map(|endpoint| endpoint.contains("r2.cloudflarestorage.com")) .unwrap_or(false); + let throttle_config = AimdThrottleConfig::from_storage_options(params.storage_options())?; + let throttle_state = if throttle_config.is_disabled() { + None + } else { + Some(AimdThrottleState::new(throttle_config)?) + }; + let inner = if use_opendal { // Use OpenDAL implementation self.build_opendal_s3_store(&base_path, &storage_options) .await? } else { // Use default Amazon S3 implementation - self.build_amazon_s3_store(&mut base_path, params, &storage_options, is_s3_express) - .await? + self.build_amazon_s3_store( + &mut base_path, + params, + &storage_options, + is_s3_express, + throttle_state.as_ref(), + ) + .await? }; - let throttle_config = AimdThrottleConfig::from_storage_options(params.storage_options())?; - let inner = if throttle_config.is_disabled() { - inner + let inner = if let Some(throttle_state) = throttle_state { + Arc::new(AimdThrottledStore::new_with_state( + inner, + throttle_state, + !use_opendal, + )) as Arc } else { - Arc::new(AimdThrottledStore::new(inner, throttle_config)?) as Arc + inner }; Ok(ObjectStore { @@ -228,14 +265,36 @@ async fn resolve_s3_region( } } +/// Selects which AWS credential provider to use for a dataset. +/// +/// When set, overrides automatic credential resolution for everything except an +/// explicitly-supplied `credentials` provider or `storage_options_accessor`. +#[derive(Debug, Clone, PartialEq)] +pub enum AwsProviderScheme { + /// Require static access-key credentials (`aws_access_key_id` + + /// `aws_secret_access_key`). Returns an error if they are absent. + Token, + /// Use the ECS/Pod Identity container credential endpoint. + /// The endpoint URI is read from the `AWS_CONTAINER_CREDENTIALS_FULL_URI` + /// or `AWS_CONTAINER_CREDENTIALS_RELATIVE_URI` environment variables. + Ecs, + /// Use IRSA (IAM Roles for Service Accounts) web identity token credentials. + /// The token file and role ARN are read from the `AWS_WEB_IDENTITY_TOKEN_FILE` + /// and `AWS_ROLE_ARN` environment variables. + Irsa, +} + /// Build AWS credentials /// /// This resolves credentials from the following sources in order: -/// 1. An explicit `storage_options_accessor` with a provider -/// 2. An explicit `credentials` provider -/// 3. Explicit credentials in storage_options (as in `aws_access_key_id`, -/// `aws_secret_access_key`, `aws_session_token`) -/// 4. The default credential provider chain from AWS SDK. +/// 1. An explicit `credentials` provider +/// 2. An explicit `storage_options_accessor` with a provider +/// 3. If `provider_scheme` is set: +/// - [`AwsProviderScheme::Token`]: static access-key credentials (error if absent) +/// - [`AwsProviderScheme::Ecs`]: ECS container credential provider +/// - [`AwsProviderScheme::Irsa`]: web identity token (IRSA) provider +/// 4. Static access-key credentials from `storage_options`, if present +/// 5. The default AWS credential provider chain /// /// # Storage Options Accessor /// @@ -250,6 +309,7 @@ pub async fn build_aws_credential( storage_options: Option<&HashMap>, region: Option, storage_options_accessor: Option>, + provider_scheme: Option, ) -> Result<(AwsCredentialProvider, String)> { use aws_config::meta::region::RegionProviderChain; const DEFAULT_REGION: &str = "us-west-2"; @@ -265,18 +325,22 @@ pub async fn build_aws_credential( .unwrap_or(DEFAULT_REGION.to_string()) }; - let storage_options_credentials = storage_options.and_then(extract_static_s3_credentials); + // If the user supplied their own credential provider that takes top priority + if let Some(creds) = credentials { + return Ok((creds, region)); + } - // Explicit aws_credentials takes precedence over dynamic credentials. - if credentials.is_none() - && let Some(dynamic_creds) = build_dynamic_credential_provider::( - storage_options_accessor.clone(), - ) - .await? + // Otherwise, if the user provided a storage_options_accessor, try and use that + if let Some(dynamic_creds) = build_dynamic_credential_provider::( + storage_options_accessor.clone(), + ) + .await? { return Ok((dynamic_creds, region)); } + // If the user provided a storage_options_accessor, then it must not have matched AWS. + // Log a message and ignore it. if storage_options_accessor .as_ref() .is_some_and(|a| a.has_provider()) @@ -287,22 +351,61 @@ pub async fn build_aws_credential( ); } - // Fall back to existing logic for static credentials - if let Some(creds) = credentials { - Ok((creds, region)) - } else if let Some(creds) = storage_options_credentials { - Ok((Arc::new(creds), region)) - } else { - let credentials_provider = DefaultCredentialsChain::builder().build().await; + // If the caller specified an explicit provider scheme, use only that provider. + if let Some(scheme) = provider_scheme { + return match scheme { + AwsProviderScheme::Token => { + let creds = storage_options + .and_then(extract_static_s3_credentials) + .ok_or_else(|| { + Error::invalid_input( + "aws_provider_scheme=token requires aws_access_key_id \ + and aws_secret_access_key to be set", + ) + })?; + Ok((Arc::new(creds), region)) + } + AwsProviderScheme::Ecs => { + let provider = EcsCredentialsProvider::builder().build(); + Ok(( + Arc::new(AwsCredentialAdapter::new( + Arc::new(provider), + credentials_refresh_offset, + )), + region, + )) + } + AwsProviderScheme::Irsa => { + let conf = ProviderConfig::default().with_region(Some(Region::new(region.clone()))); + let provider = WebIdentityTokenCredentialsProvider::builder() + .configure(&conf) + .build(); + Ok(( + Arc::new(AwsCredentialAdapter::new( + Arc::new(provider), + credentials_refresh_offset, + )), + region, + )) + } + }; + } - Ok(( - Arc::new(AwsCredentialAdapter::new( - Arc::new(credentials_provider), - credentials_refresh_offset, - )), - region, - )) + if let Some(opts) = storage_options { + // Check for static credentials (access key & secret) + if let Some(creds) = extract_static_s3_credentials(opts) { + return Ok((Arc::new(creds), region)); + } } + + let credentials_provider = DefaultCredentialsChain::builder().build().await; + Ok(( + Arc::new(AwsCredentialAdapter::new( + Arc::new(credentials_provider), + credentials_refresh_offset, + )), + region, + )) } fn extract_static_s3_credentials( @@ -389,10 +492,12 @@ impl CredentialProvider for AwsCredentialAdapter { token: creds.session_token().map(|s| s.to_string()), })) } else { - let refreshed_creds = - Arc::new(self.inner.provide_credentials().await.map_err(|e| { - Error::internal(format!("Failed to get AWS credentials: {:?}", e)) - })?); + let refreshed_creds = Arc::new( + self.inner + .provide_credentials() + .await + .map_err(|e| Error::io(format!("Failed to get AWS credentials: {:?}", e)))?, + ); self.cache .write() @@ -409,7 +514,10 @@ impl CredentialProvider for AwsCredentialAdapter { } impl StorageOptions { - /// Add values from the environment to storage options + /// Add values from the environment to storage options. + /// + /// Only adds keys that are not already present, so explicitly-set options + /// (including empty-string sentinels) always take precedence over env vars. pub fn with_env_s3(&mut self) { for (os_key, os_value) in std::env::vars_os() { if let (Some(key), Some(value)) = (os_key.to_str(), os_value.to_str()) @@ -432,6 +540,20 @@ impl StorageOptions { }) .collect() } + + /// Parse the `aws_provider_scheme` storage option, if set. + pub fn aws_provider_scheme(&self) -> Result> { + match self.0.get("aws_provider_scheme").map(|s| s.as_str()) { + None | Some("") => Ok(None), + Some("token") => Ok(Some(AwsProviderScheme::Token)), + Some("ecs") => Ok(Some(AwsProviderScheme::Ecs)), + Some("irsa") => Ok(Some(AwsProviderScheme::Irsa)), + Some(other) => Err(Error::invalid_input(format!( + "Invalid aws_provider_scheme '{}'. Valid values are: token, ecs, irsa", + other + ))), + } + } } impl ObjectStoreParams { @@ -460,6 +582,7 @@ pub type DynamicStorageOptionsCredentialProvider = mod tests { use crate::object_store::ObjectStoreRegistry; use crate::object_store::StorageOptionsProvider; + use aws_credential_types::provider::error::CredentialsError; use mock_instant::thread_local::MockClock; use object_store::path::Path; use std::sync::atomic::{AtomicBool, Ordering}; @@ -485,6 +608,46 @@ mod tests { } } + #[derive(Debug)] + struct FailingAwsCredentialsProvider; + + impl ProvideCredentials for FailingAwsCredentialsProvider { + fn provide_credentials<'a>( + &'a self, + ) -> aws_credential_types::provider::future::ProvideCredentials<'a> + where + Self: 'a, + { + aws_credential_types::provider::future::ProvideCredentials::new(async { + Err(CredentialsError::provider_error(Box::new( + std::io::Error::other("Glue credential endpoint unavailable"), + ))) + }) + } + } + + #[tokio::test] + async fn test_aws_credential_failure_is_io_error() { + let provider = AwsCredentialAdapter::new( + Arc::new(FailingAwsCredentialsProvider), + Duration::from_secs(60), + ); + + let error = provider.get_credential().await.unwrap_err(); + let object_store::Error::Generic { source, .. } = &error else { + panic!("expected a generic object store error, got {error}"); + }; + assert!(matches!( + source.downcast_ref::(), + Some(Error::IO { .. }) + )); + + let message = error.to_string(); + assert!(message.contains("Failed to get AWS credentials")); + assert!(message.contains("Glue credential endpoint unavailable")); + assert!(!message.contains("Encountered internal error")); + } + #[tokio::test] async fn test_injected_aws_creds_option_is_used() { let mock_provider = Arc::new(MockAwsCredentialsProvider::default()); @@ -1031,6 +1194,7 @@ mod tests { None, // no storage_options Some("us-west-2".to_string()), Some(accessor), + None, ) .await .unwrap(); @@ -1094,6 +1258,7 @@ mod tests { None, // no storage_options Some("us-west-2".to_string()), Some(accessor), + None, ) .await .unwrap(); @@ -1117,4 +1282,126 @@ mod tests { // Storage options provider should have been called once assert_eq!(mock_storage_provider.get_call_count().await, 1); } + + // Test that aws_provider_scheme=token selects static credentials. + #[tokio::test] + async fn test_provider_scheme_token() { + let opts = HashMap::from([ + (AmazonS3ConfigKey::AccessKeyId, "AKID".to_string()), + (AmazonS3ConfigKey::SecretAccessKey, "SECRET".to_string()), + ]); + + let (provider, _) = build_aws_credential( + Duration::from_secs(300), + None, + Some(&opts), + Some("us-east-1".to_string()), + None, + Some(AwsProviderScheme::Token), + ) + .await + .unwrap(); + + let cred = provider.get_credential().await.unwrap(); + assert_eq!(cred.key_id, "AKID"); + assert_eq!(cred.secret_key, "SECRET"); + } + + // Test that aws_provider_scheme=token errors when no static credentials are present. + #[tokio::test] + async fn test_provider_scheme_token_errors_without_credentials() { + let opts: HashMap = HashMap::new(); + + let result = build_aws_credential( + Duration::from_secs(300), + None, + Some(&opts), + Some("us-east-1".to_string()), + None, + Some(AwsProviderScheme::Token), + ) + .await; + assert!(result.is_err()); + assert!( + result + .unwrap_err() + .to_string() + .contains("aws_provider_scheme=token"), + "error should mention aws_provider_scheme=token" + ); + } + + // Test that aws_provider_scheme=ecs builds a provider without error. + // The ECS provider itself reads from env vars lazily; construction always succeeds. + #[tokio::test] + async fn test_provider_scheme_ecs() { + let opts: HashMap = HashMap::new(); + + let result = build_aws_credential( + Duration::from_secs(300), + None, + Some(&opts), + Some("us-east-1".to_string()), + None, + Some(AwsProviderScheme::Ecs), + ) + .await; + assert!(result.is_ok(), "ECS provider should build without error"); + } + + // Test that aws_provider_scheme=irsa builds a provider and attempts credential + // retrieval (which fails with a provider error, not a config error like + // "Missing Region" — confirming the region is wired through to the STS client). + #[tokio::test] + async fn test_provider_scheme_irsa() { + let opts: HashMap = HashMap::new(); + + let (provider, _) = build_aws_credential( + Duration::from_secs(300), + None, + Some(&opts), + Some("us-east-1".to_string()), + None, + Some(AwsProviderScheme::Irsa), + ) + .await + .unwrap(); + + // Credential retrieval must fail with a provider error (missing env vars or + // network), NOT a configuration error like "Invalid Configuration: Missing Region". + let err = provider.get_credential().await.unwrap_err(); + assert!( + !err.to_string().contains("Missing Region"), + "should not fail with Missing Region; region was provided. got: {err}" + ); + } + + // Test that an invalid aws_provider_scheme value produces a clear error. + #[test] + fn test_provider_scheme_invalid_value() { + let opts = StorageOptions::new(HashMap::from([( + "aws_provider_scheme".to_string(), + "magic".to_string(), + )])); + let result = opts.aws_provider_scheme(); + assert!(result.is_err()); + assert!(result.unwrap_err().to_string().contains("magic")); + } + + // Test that no aws_provider_scheme falls through to DefaultCredentialsChain without error. + #[tokio::test] + async fn test_no_provider_scheme_uses_default_chain() { + let opts: HashMap = HashMap::new(); + + let result = build_aws_credential( + Duration::from_secs(300), + None, + Some(&opts), + Some("us-east-1".to_string()), + None, + None, + ) + .await; + assert!(result.is_ok()); + } } diff --git a/rust/lance-io/src/object_store/providers/azure.rs b/rust/lance-io/src/object_store/providers/azure.rs index e61f3f3b364..2d407cd6df2 100644 --- a/rust/lance-io/src/object_store/providers/azure.rs +++ b/rust/lance-io/src/object_store/providers/azure.rs @@ -22,7 +22,7 @@ use crate::object_store::{ DEFAULT_CLOUD_BLOCK_SIZE, DEFAULT_CLOUD_IO_PARALLELISM, DEFAULT_MAX_IOP_SIZE, ObjectStore, ObjectStoreParams, ObjectStoreProvider, StorageOptions, StorageOptionsAccessor, dynamic_credentials::build_dynamic_credential_provider, - throttle::{AimdThrottleConfig, AimdThrottledStore}, + throttle::{AimdThrottleConfig, AimdThrottleState, AimdThrottledStore, cloud_http_connector}, }; use lance_core::error::{Error, Result}; @@ -106,14 +106,9 @@ impl AzureBlobStoreProvider { config_map.insert("root".to_string(), format!("/{}", prefix)); } - Operator::from_iter::(config_map) - .map_err(|e| { - Error::invalid_input(format!( - "Failed to create Azure Blob operator: {:?}", - e - )) - }) - .map(|b| b.finish()) + Operator::from_iter::(config_map).map_err(|e| { + Error::invalid_input(format!("Failed to create Azure Blob operator: {:?}", e)) + }) } "abfss" => { let filesystem = base_path.username(); @@ -139,14 +134,12 @@ impl AzureBlobStoreProvider { config_map.insert("root".to_string(), format!("/{}", root_path)); } - Operator::from_iter::(config_map) - .map_err(|e| { - Error::invalid_input(format!( - "Failed to create Azure DFS (ADLS Gen2) operator: {:?}", - e - )) - }) - .map(|b| b.finish()) + Operator::from_iter::(config_map).map_err(|e| { + Error::invalid_input(format!( + "Failed to create Azure DFS (ADLS Gen2) operator: {:?}", + e + )) + }) } _ => Err(Error::invalid_input(format!( "Unsupported Azure scheme: {}", @@ -169,6 +162,7 @@ impl AzureBlobStoreProvider { base_path: &Url, storage_options: &StorageOptions, accessor: Option>, + throttle_state: Option<&AimdThrottleState>, ) -> Result> { // Use a low retry count since the AIMD throttle layer handles // throttle recovery with its own retry loop. @@ -192,6 +186,10 @@ impl AzureBlobStoreProvider { builder = builder.with_credentials(credentials); } + let store_prefix = + self.calculate_object_store_prefix(base_path, Some(&storage_options.0))?; + builder = builder.with_http_connector(cloud_http_connector(throttle_state, store_prefix)); + Ok(Arc::new(builder.build()?) as Arc) } @@ -257,20 +255,35 @@ impl ObjectStoreProvider for AzureBlobStoreProvider { let accessor = params.get_accessor(); + let throttle_config = AimdThrottleConfig::from_storage_options(params.storage_options())?; + let throttle_state = if throttle_config.is_disabled() { + None + } else { + Some(AimdThrottleState::new(throttle_config)?) + }; + let inner: Arc = if use_opendal { // OpenDAL Azure intentionally uses static/environment-backed configuration only. // Namespace-vended dynamic credentials are supported on the native object_store path. self.build_opendal_azure_store(&base_path, &storage_options) .await? } else { - self.build_microsoft_azure_store(&base_path, &storage_options, accessor) - .await? + self.build_microsoft_azure_store( + &base_path, + &storage_options, + accessor, + throttle_state.as_ref(), + ) + .await? }; - let throttle_config = AimdThrottleConfig::from_storage_options(params.storage_options())?; - let inner = if throttle_config.is_disabled() { - inner + let inner = if let Some(throttle_state) = throttle_state { + Arc::new(AimdThrottledStore::new_with_state( + inner, + throttle_state, + !use_opendal, + )) as Arc } else { - Arc::new(AimdThrottledStore::new(inner, throttle_config)?) as Arc + inner }; Ok(ObjectStore { @@ -642,8 +655,8 @@ mod tests { let abfss_operator = AzureBlobStoreProvider::build_opendal_operator(&abfss_url, &common_opts).unwrap(); - let azblob_cap = az_operator.info().native_capability(); - let azdls_cap = abfss_operator.info().native_capability(); + let azblob_cap = az_operator.info().capability(); + let azdls_cap = abfss_operator.info().capability(); // Both support basic operations assert!(azblob_cap.read); diff --git a/rust/lance-io/src/object_store/providers/gcp.rs b/rust/lance-io/src/object_store/providers/gcp.rs index f7f0a7672ff..1a93c3ac9f0 100644 --- a/rust/lance-io/src/object_store/providers/gcp.rs +++ b/rust/lance-io/src/object_store/providers/gcp.rs @@ -17,7 +17,7 @@ use crate::object_store::{ DEFAULT_CLOUD_BLOCK_SIZE, DEFAULT_CLOUD_IO_PARALLELISM, DEFAULT_MAX_IOP_SIZE, ObjectStore, ObjectStoreParams, ObjectStoreProvider, StorageOptions, StorageOptionsAccessor, dynamic_credentials::build_dynamic_credential_provider, - throttle::{AimdThrottleConfig, AimdThrottledStore}, + throttle::{AimdThrottleConfig, AimdThrottleState, AimdThrottledStore, cloud_http_connector}, }; use lance_core::error::{Error, Result}; @@ -49,8 +49,7 @@ impl GcsStoreProvider { } let operator = Operator::from_iter::(config_map) - .map_err(|e| Error::invalid_input(format!("Failed to create GCS operator: {:?}", e)))? - .finish(); + .map_err(|e| Error::invalid_input(format!("Failed to create GCS operator: {:?}", e)))?; Ok(Arc::new(OpendalStore::new(operator)) as Arc) } @@ -60,6 +59,7 @@ impl GcsStoreProvider { base_path: &Url, storage_options: &StorageOptions, accessor: Option>, + throttle_state: Option<&AimdThrottleState>, ) -> Result> { // Use a low retry count since the AIMD throttle layer handles // throttle recovery with its own retry loop. @@ -89,6 +89,10 @@ impl GcsStoreProvider { builder = builder.with_credentials(credential_provider); } + let store_prefix = + self.calculate_object_store_prefix(base_path, Some(&storage_options.0))?; + builder = builder.with_http_connector(cloud_http_connector(throttle_state, store_prefix)); + Ok(Arc::new(builder.build()?) as Arc) } } @@ -110,20 +114,35 @@ impl ObjectStoreProvider for GcsStoreProvider { let accessor = params.get_accessor(); + let throttle_config = AimdThrottleConfig::from_storage_options(params.storage_options())?; + let throttle_state = if throttle_config.is_disabled() { + None + } else { + Some(AimdThrottleState::new(throttle_config)?) + }; + let inner = if use_opendal { // OpenDAL GCS intentionally uses static/environment-backed configuration only. // Namespace-vended dynamic credentials are supported on the native object_store path. self.build_opendal_gcs_store(&base_path, &storage_options) .await? } else { - self.build_google_cloud_store(&base_path, &storage_options, accessor) - .await? + self.build_google_cloud_store( + &base_path, + &storage_options, + accessor, + throttle_state.as_ref(), + ) + .await? }; - let throttle_config = AimdThrottleConfig::from_storage_options(params.storage_options())?; - let inner = if throttle_config.is_disabled() { - inner + let inner = if let Some(throttle_state) = throttle_state { + Arc::new(AimdThrottledStore::new_with_state( + inner, + throttle_state, + !use_opendal, + )) as Arc } else { - Arc::new(AimdThrottledStore::new(inner, throttle_config)?) as Arc + inner }; Ok(ObjectStore { diff --git a/rust/lance-io/src/object_store/providers/goosefs.rs b/rust/lance-io/src/object_store/providers/goosefs.rs index 5d2a648522b..fe3002bccc0 100644 --- a/rust/lance-io/src/object_store/providers/goosefs.rs +++ b/rust/lance-io/src/object_store/providers/goosefs.rs @@ -171,11 +171,9 @@ impl ObjectStoreProvider for GooseFsStoreProvider { } // Create OpenDAL Operator with GooseFS service - let operator = Operator::from_iter::(config_map) - .map_err(|e| { - Error::invalid_input(format!("Failed to create GooseFS operator: {:?}", e)) - })? - .finish(); + let operator = Operator::from_iter::(config_map).map_err(|e| { + Error::invalid_input(format!("Failed to create GooseFS operator: {:?}", e)) + })?; // Wrap as object_store::ObjectStore via OpendalStore bridge let opendal_store = Arc::new(OpendalStore::new(operator)); diff --git a/rust/lance-io/src/object_store/providers/huggingface.rs b/rust/lance-io/src/object_store/providers/huggingface.rs index cfef0440068..cda56e36fbe 100644 --- a/rust/lance-io/src/object_store/providers/huggingface.rs +++ b/rust/lance-io/src/object_store/providers/huggingface.rs @@ -160,11 +160,9 @@ fn build_hf_store(config_map: HashMap) -> Result { builder = builder.download_mode(download_mode); } - let operator = Operator::new(builder) - .map_err(|e| { - Error::invalid_input(format!("Failed to create Huggingface operator: {:?}", e)) - })? - .finish(); + let operator = Operator::new(builder).map_err(|e| { + Error::invalid_input(format!("Failed to create Huggingface operator: {:?}", e)) + })?; Ok(OpendalStore::new(operator)) } diff --git a/rust/lance-io/src/object_store/providers/oss.rs b/rust/lance-io/src/object_store/providers/oss.rs index b84afb8ed1f..3d116e2e3cc 100644 --- a/rust/lance-io/src/object_store/providers/oss.rs +++ b/rust/lance-io/src/object_store/providers/oss.rs @@ -95,8 +95,7 @@ impl OssStoreProvider { fn build_oss_store(config_map: HashMap) -> Result { let operator = Operator::from_iter::(config_map) - .map_err(|e| Error::invalid_input(format!("Failed to create OSS operator: {:?}", e)))? - .finish(); + .map_err(|e| Error::invalid_input(format!("Failed to create OSS operator: {:?}", e)))?; Ok(OpendalStore::new(operator)) } diff --git a/rust/lance-io/src/object_store/providers/tencent.rs b/rust/lance-io/src/object_store/providers/tencent.rs index 5fa885ea5a9..d29d5a6ad62 100644 --- a/rust/lance-io/src/object_store/providers/tencent.rs +++ b/rust/lance-io/src/object_store/providers/tencent.rs @@ -80,8 +80,7 @@ impl ObjectStoreProvider for TencentStoreProvider { } let operator = Operator::from_iter::(config_map) - .map_err(|e| Error::invalid_input(format!("Failed to create COS operator: {:?}", e)))? - .finish(); + .map_err(|e| Error::invalid_input(format!("Failed to create COS operator: {:?}", e)))?; let opendal_store = Arc::new(OpendalStore::new(operator)); diff --git a/rust/lance-io/src/object_store/providers/tos.rs b/rust/lance-io/src/object_store/providers/tos.rs index 923186484c6..7dee659f5f9 100644 --- a/rust/lance-io/src/object_store/providers/tos.rs +++ b/rust/lance-io/src/object_store/providers/tos.rs @@ -101,8 +101,7 @@ impl TosStoreProvider { fn build_tos_store(config_map: HashMap) -> Result { let operator = Operator::from_iter::(config_map) - .map_err(|e| Error::invalid_input(format!("Failed to create TOS operator: {:?}", e)))? - .finish(); + .map_err(|e| Error::invalid_input(format!("Failed to create TOS operator: {:?}", e)))?; Ok(OpendalStore::new(operator)) } diff --git a/rust/lance-io/src/object_store/storage_options.rs b/rust/lance-io/src/object_store/storage_options.rs index fa5f4995afd..355845fd9b6 100644 --- a/rust/lance-io/src/object_store/storage_options.rs +++ b/rust/lance-io/src/object_store/storage_options.rs @@ -138,6 +138,8 @@ impl StorageOptionsProvider for LanceNamespaceStorageOptionsProvider { async fn fetch_storage_options(&self) -> Result>> { let request = DescribeTableRequest { id: Some(self.table_id.clone()), + // Some server implementations may not return credentials unless explicitly requested + vend_credentials: Some(true), ..Default::default() }; diff --git a/rust/lance-io/src/object_store/throttle.rs b/rust/lance-io/src/object_store/throttle.rs index bac1d3a538e..4876de01b64 100644 --- a/rust/lance-io/src/object_store/throttle.rs +++ b/rust/lance-io/src/object_store/throttle.rs @@ -33,6 +33,11 @@ use lance_core::utils::aimd::{AimdConfig, AimdController, RequestOutcome}; use lance_core::utils::tracing::TRACE_OBJECT_STORE_THROTTLE; #[cfg(test)] use object_store::ObjectStoreExt; +#[cfg(any(feature = "aws", feature = "azure", feature = "gcp"))] +use object_store::client::{ + ClientOptions, HttpClient, HttpConnector, HttpError, HttpErrorKind, HttpRequest, HttpResponse, + HttpResponseBody, HttpService, +}; use object_store::path::Path; use object_store::{ CopyOptions, GetOptions, GetResult, ListResult, MultipartUpload, ObjectMeta, ObjectStore, @@ -407,24 +412,44 @@ impl OperationThrottle { } Err(_) => RequestOutcome::Success, }; - let prev_rate = self.controller.current_rate(); - let new_rate = self.controller.record_outcome(outcome); - if new_rate < prev_rate - && let Err(err) = result.as_ref() - { - warn!( - target: TRACE_OBJECT_STORE_THROTTLE, - previous_rate = format!("{prev_rate:.1}"), - new_rate = format!("{new_rate:.1}"), - error = %err, - "AIMD throttle: rate reduced due to throttle errors" - ); - } + let error = result + .as_ref() + .err() + .map(|error| error as &dyn std::fmt::Display); + let new_rate = self.record_outcome(outcome, error); if let Ok(mut bucket) = self.bucket.try_lock() { bucket.rate = new_rate; } } + fn record_outcome( + &self, + outcome: RequestOutcome, + error: Option<&dyn std::fmt::Display>, + ) -> f64 { + let prev_rate = self.controller.current_rate(); + let new_rate = self.controller.record_outcome(outcome); + if new_rate < prev_rate { + if let Some(error) = error { + warn!( + target: TRACE_OBJECT_STORE_THROTTLE, + previous_rate = format!("{prev_rate:.1}"), + new_rate = format!("{new_rate:.1}"), + error = %error, + "AIMD throttle: rate reduced due to throttle errors" + ); + } else { + warn!( + target: TRACE_OBJECT_STORE_THROTTLE, + previous_rate = format!("{prev_rate:.1}"), + new_rate = format!("{new_rate:.1}"), + "AIMD throttle: rate reduced due to throttle errors" + ); + } + } + new_rate + } + /// Execute an operation with throttling: acquire token, run, classify result. /// On throttle errors, retries up to `max_retries` times with a random /// backoff between `min_backoff_ms` and `max_backoff_ms` between attempts. @@ -448,19 +473,11 @@ impl OperationThrottle { } Err(_) => RequestOutcome::Success, // Non-throttle errors don't indicate capacity problems }; - let prev_rate = self.controller.current_rate(); - let new_rate = self.controller.record_outcome(outcome); - if new_rate < prev_rate - && let Err(err) = result.as_ref() - { - warn!( - target: TRACE_OBJECT_STORE_THROTTLE, - previous_rate = format!("{prev_rate:.1}"), - new_rate = format!("{new_rate:.1}"), - error = %err, - "AIMD throttle: rate reduced due to throttle errors" - ); - } + let error = result + .as_ref() + .err() + .map(|error| error as &dyn std::fmt::Display); + let new_rate = self.record_outcome(outcome, error); self.update_bucket_rate(new_rate).await; match &result { @@ -494,12 +511,229 @@ impl Debug for OperationThrottle { } } -/// A [`MultipartUpload`] wrapper that throttles and retries `put_part`, -/// `complete`, and `abort`, feeding outcomes back to the write AIMD -/// controller. +#[derive(Clone)] +pub(crate) struct AimdThrottleState { + read: Arc, + write: Arc, + delete: Arc, + list: Arc, +} + +impl AimdThrottleState { + pub(crate) fn new(config: AimdThrottleConfig) -> lance_core::Result { + let burst_capacity = config.burst_capacity as f64; + let max_retries = config.max_retries; + let min_backoff_ms = config.min_backoff_ms; + let max_backoff_ms = config.max_backoff_ms; + Ok(Self { + read: Arc::new(OperationThrottle::new( + config.read, + burst_capacity, + max_retries, + min_backoff_ms, + max_backoff_ms, + )?), + write: Arc::new(OperationThrottle::new( + config.write, + burst_capacity, + max_retries, + min_backoff_ms, + max_backoff_ms, + )?), + delete: Arc::new(OperationThrottle::new( + config.delete, + burst_capacity, + max_retries, + min_backoff_ms, + max_backoff_ms, + )?), + list: Arc::new(OperationThrottle::new( + config.list, + burst_capacity, + max_retries, + min_backoff_ms, + max_backoff_ms, + )?), + }) + } +} + +#[cfg(any(feature = "aws", feature = "azure", feature = "gcp"))] +#[derive(Debug)] +pub(crate) struct AimdMultipartUploadConnector { + inner: C, + write: Option>, +} + +#[cfg(any(feature = "aws", feature = "azure", feature = "gcp"))] +impl AimdMultipartUploadConnector { + fn new(inner: C, state: Option<&AimdThrottleState>) -> Self { + Self { + inner, + write: state.map(|state| Arc::clone(&state.write)), + } + } +} + +#[cfg(all( + any(feature = "aws", feature = "azure", feature = "gcp"), + feature = "metrics" +))] +pub(crate) fn cloud_http_connector( + state: Option<&AimdThrottleState>, + metrics_base: String, +) -> AimdMultipartUploadConnector { + AimdMultipartUploadConnector::new( + crate::object_store::metrics::MeteringHttpConnector::new(metrics_base), + state, + ) +} + +#[cfg(all( + any(feature = "aws", feature = "azure", feature = "gcp"), + not(feature = "metrics") +))] +pub(crate) fn cloud_http_connector( + state: Option<&AimdThrottleState>, + _metrics_base: String, +) -> AimdMultipartUploadConnector { + AimdMultipartUploadConnector::new(object_store::client::ReqwestConnector::default(), state) +} + +#[cfg(any(feature = "aws", feature = "azure", feature = "gcp"))] +impl HttpConnector for AimdMultipartUploadConnector { + fn connect(&self, options: &ClientOptions) -> object_store::Result { + Ok(HttpClient::new(AimdMultipartUploadService { + inner: self.inner.connect(options)?, + write: self.write.clone(), + })) + } +} + +#[cfg(any(feature = "aws", feature = "azure", feature = "gcp"))] +#[derive(Debug)] +struct AimdMultipartUploadService { + inner: HttpClient, + write: Option>, +} + +#[cfg(any(feature = "aws", feature = "azure", feature = "gcp"))] +fn is_multipart_part_request(request: &HttpRequest) -> bool { + if request.method() != ::http::Method::PUT { + return false; + } + request.uri().query().is_some_and(|query| { + url::form_urlencoded::parse(query.as_bytes()).any(|(key, value)| { + key.eq_ignore_ascii_case("partNumber") + || (key.eq_ignore_ascii_case("comp") && value.eq_ignore_ascii_case("block")) + }) + }) +} + +#[cfg(any(feature = "aws", feature = "azure", feature = "gcp"))] +fn is_retryable_http_error(error: &HttpError) -> bool { + matches!( + error.kind(), + HttpErrorKind::Connect + | HttpErrorKind::Request + | HttpErrorKind::Timeout + | HttpErrorKind::Interrupted + ) +} + +#[cfg(any(feature = "aws", feature = "azure", feature = "gcp"))] +#[async_trait] +impl HttpService for AimdMultipartUploadService { + async fn call(&self, request: HttpRequest) -> Result { + let Some(write) = self.write.as_ref() else { + return self.inner.execute(request).await; + }; + if !is_multipart_part_request(&request) { + return self.inner.execute(request).await; + } + + for attempt in 0..=write.max_retries { + write.acquire_token().await; + let mut result = self.inner.execute(request.clone()).await; + let mut is_retryable = result.as_ref().err().is_some_and(is_retryable_http_error); + let mut is_throttle = false; + let mut response_status = None; + + if let Ok(response) = result { + let status = response.status(); + response_status = Some(status); + is_retryable = status == ::http::StatusCode::REQUEST_TIMEOUT + || status == ::http::StatusCode::TOO_MANY_REQUESTS + || status.is_server_error(); + is_throttle = status == ::http::StatusCode::TOO_MANY_REQUESTS + || status == ::http::StatusCode::SERVICE_UNAVAILABLE; + + let (parts, body) = response.into_parts(); + result = match body.bytes().await { + Ok(bytes) => { + let body = String::from_utf8_lossy(&bytes).to_ascii_lowercase(); + let is_throttle_body = body.contains("requesttimeout") + || body.contains("slowdown") + || body.contains("serverbusy") + || body.contains("throttl"); + is_retryable |= is_throttle_body; + is_throttle |= is_throttle_body; + Ok(HttpResponse::from_parts( + parts, + HttpResponseBody::from(bytes), + )) + } + Err(error) => { + is_retryable = is_retryable_http_error(&error); + Err(error) + } + }; + } + + let detail = response_status + .filter(|status| !status.is_success()) + .map(|status| format!("HTTP status {status}")); + let error = result + .as_ref() + .err() + .map(|error| error as &dyn std::fmt::Display) + .or_else(|| { + detail + .as_ref() + .map(|detail| detail as &dyn std::fmt::Display) + }); + let outcome = if is_throttle { + RequestOutcome::Throttled + } else { + RequestOutcome::Success + }; + let new_rate = write.record_outcome(outcome, error); + write.update_bucket_rate(new_rate).await; + + if is_retryable && attempt < write.max_retries { + let backoff_ms = + rand::rng().random_range(write.min_backoff_ms..=write.max_backoff_ms); + debug!( + target: TRACE_OBJECT_STORE_THROTTLE, + attempt = attempt + 1, + max_retries = write.max_retries, + backoff_ms, + "Retrying multipart upload part after retryable HTTP response" + ); + tokio::time::sleep(std::time::Duration::from_millis(backoff_ms)).await; + continue; + } + return result; + } + unreachable!() + } +} + +/// A [`MultipartUpload`] wrapper that applies the write AIMD controller. struct ThrottledMultipartUpload { target: Box, write: Arc, + parts_throttled_at_http: bool, } impl Debug for ThrottledMultipartUpload { @@ -511,10 +745,13 @@ impl Debug for ThrottledMultipartUpload { #[async_trait] impl MultipartUpload for ThrottledMultipartUpload { fn put_part(&mut self, data: PutPayload) -> UploadPart { - let write = Arc::clone(&self.write); // Call put_part synchronously to preserve part ordering regardless // of which futures are awaited first. let fut = self.target.put_part(data); + if self.parts_throttled_at_http { + return fut; + } + let write = Arc::clone(&self.write); Box::pin(async move { write.acquire_token().await; let result = fut.await; @@ -585,6 +822,7 @@ pub struct AimdThrottledStore { write: Arc, delete: Arc, list: Arc, + multipart_parts_throttled_at_http: bool, } impl Debug for AimdThrottledStore { @@ -595,6 +833,10 @@ impl Debug for AimdThrottledStore { .field("write", &self.write) .field("delete", &self.delete) .field("list", &self.list) + .field( + "multipart_parts_throttled_at_http", + &self.multipart_parts_throttled_at_http, + ) .finish() } } @@ -610,41 +852,26 @@ impl AimdThrottledStore { target: Arc, config: AimdThrottleConfig, ) -> lance_core::Result { - let burst = config.burst_capacity as f64; - let max_retries = config.max_retries; - let min_backoff_ms = config.min_backoff_ms; - let max_backoff_ms = config.max_backoff_ms; - Ok(Self { + Ok(Self::new_with_state( target, - read: Arc::new(OperationThrottle::new( - config.read, - burst, - max_retries, - min_backoff_ms, - max_backoff_ms, - )?), - write: Arc::new(OperationThrottle::new( - config.write, - burst, - max_retries, - min_backoff_ms, - max_backoff_ms, - )?), - delete: Arc::new(OperationThrottle::new( - config.delete, - burst, - max_retries, - min_backoff_ms, - max_backoff_ms, - )?), - list: Arc::new(OperationThrottle::new( - config.list, - burst, - max_retries, - min_backoff_ms, - max_backoff_ms, - )?), - }) + AimdThrottleState::new(config)?, + false, + )) + } + + pub(crate) fn new_with_state( + target: Arc, + state: AimdThrottleState, + multipart_parts_throttled_at_http: bool, + ) -> Self { + Self { + target, + read: state.read, + write: state.write, + delete: state.delete, + list: state.list, + multipart_parts_throttled_at_http, + } } } @@ -674,6 +901,7 @@ impl ObjectStore for AimdThrottledStore { Ok(Box::new(ThrottledMultipartUpload { target, write: Arc::clone(&self.write), + parts_throttled_at_http: self.multipart_parts_throttled_at_http, })) } @@ -811,6 +1039,21 @@ mod tests { assert!(!is_throttle_error(&err)); } + #[cfg(any(feature = "aws", feature = "azure", feature = "gcp"))] + #[rstest] + #[case::s3("https://bucket/object?partNumber=1&uploadId=id", true)] + #[case::azure_block("https://account/object?comp=block&blockid=id", true)] + #[case::azure_block_list("https://account/object?comp=blocklist", false)] + #[case::ordinary_put("https://bucket/object", false)] + fn test_is_multipart_part_request(#[case] uri: &str, #[case] expected: bool) { + let request = ::http::Request::builder() + .method(::http::Method::PUT) + .uri(uri) + .body(object_store::client::HttpRequestBody::empty()) + .unwrap(); + assert_eq!(is_multipart_part_request(&request), expected); + } + #[tokio::test] async fn test_basic_put_get_through_wrapper() { let store = Arc::new(InMemory::new()); @@ -1682,6 +1925,145 @@ mod tests { assert_eq!(mock.get_call_count.load(Ordering::Relaxed), 4); } + #[cfg(feature = "aws")] + #[derive(Debug)] + struct MultipartRetryState { + failures_remaining: AtomicUsize, + part_uris: std::sync::Mutex>, + } + + #[cfg(feature = "aws")] + #[derive(Debug)] + struct MultipartRetryConnector { + state: Arc, + } + + #[cfg(feature = "aws")] + impl HttpConnector for MultipartRetryConnector { + fn connect(&self, _options: &ClientOptions) -> object_store::Result { + Ok(HttpClient::new(MultipartRetryService { + state: Arc::clone(&self.state), + })) + } + } + + #[cfg(feature = "aws")] + #[derive(Debug)] + struct MultipartRetryService { + state: Arc, + } + + #[cfg(feature = "aws")] + #[async_trait] + impl HttpService for MultipartRetryService { + async fn call(&self, request: HttpRequest) -> Result { + let method = request.method().clone(); + let query = request.uri().query().unwrap_or_default(); + let (status, body, e_tag) = if method == ::http::Method::POST + && query + .split('&') + .any(|part| part == "uploads" || part == "uploads=") + { + ( + ::http::StatusCode::OK, + "bucketobjectupload-id", + None, + ) + } else if method == ::http::Method::PUT && query.contains("partNumber=") { + self.state + .part_uris + .lock() + .unwrap() + .push(request.uri().to_string()); + let mut remaining = self.state.failures_remaining.load(Ordering::SeqCst); + let should_fail = loop { + let Some(next) = remaining.checked_sub(1) else { + break false; + }; + match self.state.failures_remaining.compare_exchange_weak( + remaining, + next, + Ordering::SeqCst, + Ordering::SeqCst, + ) { + Ok(_) => break true, + Err(actual) => remaining = actual, + } + }; + if should_fail { + ( + ::http::StatusCode::SERVICE_UNAVAILABLE, + "SlowDownPlease reduce your request rate.", + None, + ) + } else { + (::http::StatusCode::OK, "", Some("\"part-etag\"")) + } + } else if method == ::http::Method::POST && query.contains("uploadId=") { + ( + ::http::StatusCode::OK, + "https://bucket/objectbucketobject\"object-etag\"", + None, + ) + } else { + (::http::StatusCode::BAD_REQUEST, "unexpected request", None) + }; + + let mut response = ::http::Response::builder().status(status); + if let Some(e_tag) = e_tag { + response = response.header(::http::header::ETAG, e_tag); + } + Ok(response + .body(HttpResponseBody::from(body.to_string())) + .unwrap()) + } + } + + /// Retries must remain inside the original S3 `put_part` call. Re-entering + /// `MultipartUpload::put_part` would allocate a new part number and leave a + /// gap that makes `complete` fail with "Missing part". + #[cfg(feature = "aws")] + #[tokio::test(start_paused = true)] + async fn test_multipart_http_retry_reuses_part_number() { + use object_store::RetryConfig; + use object_store::aws::AmazonS3Builder; + + let retry_state = Arc::new(MultipartRetryState { + failures_remaining: AtomicUsize::new(3), + part_uris: std::sync::Mutex::new(Vec::new()), + }); + let throttle_state = AimdThrottleState::new(AimdThrottleConfig::default()).unwrap(); + let connector = AimdMultipartUploadConnector::new( + MultipartRetryConnector { + state: Arc::clone(&retry_state), + }, + Some(&throttle_state), + ); + let store = AmazonS3Builder::new() + .with_bucket_name("bucket") + .with_region("us-east-1") + .with_skip_signature(true) + .with_retry(RetryConfig { + max_retries: 0, + ..Default::default() + }) + .with_http_connector(connector) + .build() + .unwrap(); + + let mut upload = store.put_multipart(&Path::from("object")).await.unwrap(); + upload + .put_part(PutPayload::from_static(b"payload")) + .await + .unwrap(); + upload.complete().await.unwrap(); + + let part_uris = retry_state.part_uris.lock().unwrap(); + assert_eq!(part_uris.len(), 4); + assert!(part_uris.iter().all(|uri| uri == &part_uris[0])); + assert!(part_uris[0].contains("partNumber=1")); + } + #[tokio::test] async fn test_throttled_multipart_reorders_parts() { let store = Arc::new(InMemory::new()) as Arc; diff --git a/rust/lance-io/src/object_writer.rs b/rust/lance-io/src/object_writer.rs index 0fd0a30f9e7..73167112eee 100644 --- a/rust/lance-io/src/object_writer.rs +++ b/rust/lance-io/src/object_writer.rs @@ -11,9 +11,8 @@ use async_trait::async_trait; use bytes::Bytes; use futures::FutureExt; use futures::future::BoxFuture; -use object_store::{Error as OSError, ObjectStore, Result as OSResult, path::Path}; use object_store::{MultipartUpload, ObjectStoreExt}; -use rand::Rng; +use object_store::{ObjectStore, Result as OSResult, path::Path}; use tokio::io::{AsyncWrite, AsyncWriteExt}; use tokio::task::JoinSet; @@ -21,7 +20,7 @@ use lance_core::{Error, Result}; use tracing::Instrument; use crate::traits::Writer; -use crate::utils::tracking_store::IOTracker; +use crate::utils::tracking_store::{IOTracker, IoMetricsGuard}; use tokio::runtime::Handle; /// Start at 5MB. @@ -37,16 +36,6 @@ fn max_upload_parallelism() -> usize { }) } -fn max_conn_reset_retries() -> u16 { - static MAX_CONN_RESET_RETRIES: OnceLock = OnceLock::new(); - *MAX_CONN_RESET_RETRIES.get_or_init(|| { - std::env::var("LANCE_CONN_RESET_RETRIES") - .ok() - .and_then(|s| s.parse::().ok()) - .unwrap_or(20) - }) -} - /// Maximum body size for a single S3 PUT: strictly less than 5 GiB. /// AWS rejects single-PUT bodies of exactly 5 GiB (= 5 * 1024^3) with /// `EntityTooLarge`, so we clamp `LANCE_INITIAL_UPLOAD_SIZE` one byte @@ -94,7 +83,6 @@ pub struct ObjectWriter { state: UploadState, path: Arc, cursor: usize, - connection_resets: u16, buffer: Vec, // TODO: use constant size to support R2 use_constant_size_upload_parts: bool, @@ -116,7 +104,7 @@ enum UploadState { InProgress { part_idx: u16, upload: Box, - futures: JoinSet>, + futures: JoinSet>, }, /// The writer is in the process of uploading data in a single PUT request. /// This happens when shutdown is called before the buffer is full. @@ -178,7 +166,6 @@ impl ObjectWriter { state: UploadState::Started(object_store.inner.clone()), cursor: 0, path: Arc::new(path.clone()), - connection_resets: 0, buffer: Vec::with_capacity(initial_upload_size()), use_constant_size_upload_parts: object_store.use_constant_size_upload_parts, }) @@ -202,25 +189,12 @@ impl ObjectWriter { fn put_part( upload: &mut dyn MultipartUpload, buffer: Bytes, - part_idx: u16, - sleep: Option, - ) -> BoxFuture<'static, std::result::Result<(), UploadPutError>> { + ) -> BoxFuture<'static, OSResult<()>> { log::debug!( "MultipartUpload submitting part with {} bytes", buffer.len() ); - let fut = upload.put_part(buffer.clone().into()); - Box::pin(async move { - if let Some(sleep) = sleep { - tokio::time::sleep(sleep).await; - } - fut.await.map_err(|source| UploadPutError { - part_idx, - buffer, - source, - })?; - Ok(()) - }) + upload.put_part(buffer.into()) } fn poll_tasks( @@ -240,7 +214,7 @@ impl ObjectWriter { 0, mut_self.use_constant_size_upload_parts, ); - futures.spawn(Self::put_part(upload.as_mut(), data, 0, None)); + futures.spawn(Self::put_part(upload.as_mut(), data)); mut_self.state = UploadState::InProgress { part_idx: 1, // We just used 0 @@ -251,43 +225,12 @@ impl ObjectWriter { Poll::Ready(Err(e)) => return Err(std::io::Error::other(e)), Poll::Pending => break, }, - UploadState::InProgress { - upload, futures, .. - } => { + UploadState::InProgress { futures, .. } => { while let Poll::Ready(Some(res)) = futures.poll_join_next(cx) { match res { Ok(Ok(())) => {} Err(err) => return Err(std::io::Error::other(err)), - Ok(Err(err)) if should_retry_upload_put(&err.source) => { - if mut_self.connection_resets < max_conn_reset_retries() { - // Retry, but only up to max_conn_reset_retries of them. - mut_self.connection_resets += 1; - - // Resubmit with random jitter - let sleep_time_ms = rand::rng().random_range(2_000..8_000); - let sleep_time = - std::time::Duration::from_millis(sleep_time_ms); - - futures.spawn(Self::put_part( - upload.as_mut(), - err.buffer, - err.part_idx, - Some(sleep_time), - )); - } else { - return Err(io::Error::new( - io::ErrorKind::ConnectionReset, - Box::new(ConnectionResetError { - message: format!( - "Hit max retries ({}) for retryable upload error", - max_conn_reset_retries() - ), - source: Box::new(err.source), - }), - )); - } - } - Ok(Err(err)) => return Err(err.source.into()), + Ok(Err(err)) => return Err(err.into()), } } break; @@ -333,38 +276,6 @@ impl Drop for ObjectWriter { } } -/// Returned error from trying to upload a part. -/// Has the part_idx and buffer so we can pass -/// them to the retry logic. -struct UploadPutError { - part_idx: u16, - buffer: Bytes, - source: OSError, -} - -fn should_retry_upload_put(source: &OSError) -> bool { - let OSError::Generic { source, .. } = source else { - return false; - }; - - let message = source.to_string().to_ascii_lowercase(); - message.contains("connection reset by peer") || message.contains("requesttimeout") -} - -#[derive(Debug)] -struct ConnectionResetError { - message: String, - source: Box, -} - -impl std::error::Error for ConnectionResetError {} - -impl std::fmt::Display for ConnectionResetError { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - write!(f, "{}: {}", self.message, self.source) - } -} - impl AsyncWrite for ObjectWriter { fn poll_write( mut self: std::pin::Pin<&mut Self>, @@ -392,25 +303,22 @@ impl AsyncWrite for ObjectWriter { let fut = Box::pin(async move { store.put_multipart(path.as_ref()).await }); self.state = UploadState::CreatingUpload(fut); } + // TODO: Make max concurrency configurable from storage options. UploadState::InProgress { upload, part_idx, futures, .. - } => { - // TODO: Make max concurrency configurable from storage options. - if futures.len() < max_upload_parallelism() { - let data = Self::next_part_buffer( - &mut mut_self.buffer, - *part_idx, - mut_self.use_constant_size_upload_parts, - ); - futures.spawn( - Self::put_part(upload.as_mut(), data, *part_idx, None) - .instrument(tracing::Span::current()), - ); - *part_idx += 1; - } + } if futures.len() < max_upload_parallelism() => { + let data = Self::next_part_buffer( + &mut mut_self.buffer, + *part_idx, + mut_self.use_constant_size_upload_parts, + ); + futures.spawn( + Self::put_part(upload.as_mut(), data).instrument(tracing::Span::current()), + ); + *part_idx += 1; } _ => {} } @@ -467,16 +375,14 @@ impl AsyncWrite for ObjectWriter { self.state.started_to_putting_single(path, part); } UploadState::InProgress { - upload, - futures, - part_idx, + upload, futures, .. } => { // Flush final batch if !mut_self.buffer.is_empty() && futures.len() < max_upload_parallelism() { // We can just use `take` since we don't need the buffer anymore. let data = Bytes::from(std::mem::take(&mut mut_self.buffer)); futures.spawn( - Self::put_part(upload.as_mut(), data, *part_idx, None) + Self::put_part(upload.as_mut(), data) .instrument(tracing::Span::current()), ); // We need to go back to beginning of loop to poll the @@ -524,7 +430,7 @@ pub struct LocalWriter { #[derive(Default)] enum LocalWriteState { - Writing(WritingState), + Writing(Box), Finishing { size: usize, future: BoxFuture<'static, Result>, @@ -540,6 +446,10 @@ struct WritingState { /// Temp path that auto-deletes on drop. Set to `None` after `persist()`. temp_path: tempfile::TempPath, io_tracker: Arc, + /// The whole file is reported as a single `put`, so this covers everything + /// from opening the file to it being durable under its final path. A writer + /// dropped before `persist()` records nothing, like an aborted upload. + metrics: IoMetricsGuard, } impl LocalWriter { @@ -551,12 +461,13 @@ impl LocalWriter { ) -> Self { Self { path, - state: LocalWriteState::Writing(WritingState { + state: LocalWriteState::Writing(Box::new(WritingState { writer: tokio::io::BufWriter::new(file), cursor: 0, temp_path, + metrics: io_tracker.begin_io("put"), io_tracker, - }), + })), } } @@ -576,9 +487,10 @@ impl LocalWriter { final_path: Path, size: usize, io_tracker: Arc, + metrics: IoMetricsGuard, ) -> Result { let local_path = crate::local::to_local_path(&final_path); - let e_tag = tokio::task::spawn_blocking(move || -> Result { + let persisted = tokio::task::spawn_blocking(move || -> Result { temp_path.persist(&local_path).map_err(|e| { Error::io(format!( "failed to persist temp file to {}: {}", @@ -592,7 +504,11 @@ impl LocalWriter { Ok(get_etag(&metadata)) }) .await - .map_err(|e| Error::io(format!("spawn_blocking failed: {}", e)))??; + .map_err(|e| Error::io(format!("spawn_blocking failed: {}", e))) + .and_then(|e_tag| e_tag); + + metrics.record(&persisted, size as u64); + let e_tag = persisted?; io_tracker.record_write("put", final_path, size as u64); @@ -657,6 +573,7 @@ impl AsyncWrite for LocalWriter { mut_self.path.clone(), size, state.io_tracker, + state.metrics, )), }; } @@ -862,35 +779,6 @@ mod tests { assert_eq!(clamp_initial_upload_size(mid), (mid, false)); } - #[test] - fn should_retry_upload_put_detects_transient_errors() { - let request_timeout = OSError::Generic { - store: "S3", - source: Box::new(io::Error::other( - "Server returned non-2xx status code: 400 Bad Request: \ - RequestTimeoutYour socket connection to the server \ - was not read from or written to within the timeout period. Idle connections will \ - be closed.", - )), - }; - assert!(should_retry_upload_put(&request_timeout)); - - let connection_reset = OSError::Generic { - store: "S3", - source: Box::new(io::Error::new( - io::ErrorKind::ConnectionReset, - "connection reset by peer", - )), - }; - assert!(should_retry_upload_put(&connection_reset)); - - let not_retryable = OSError::Generic { - store: "S3", - source: Box::new(io::Error::other("access denied")), - }; - assert!(!should_retry_upload_put(¬_retryable)); - } - #[test] fn clamp_initial_upload_size_above_max_is_clamped_down() { assert_eq!( diff --git a/rust/lance-io/src/scheduler.rs b/rust/lance-io/src/scheduler.rs index 0c2d4dc44bd..60f9910e5fd 100644 --- a/rust/lance-io/src/scheduler.rs +++ b/rust/lance-io/src/scheduler.rs @@ -29,6 +29,7 @@ mod lite; const BACKPRESSURE_MIN: u64 = 5; // Don't log backpressure warnings more than once / minute const BACKPRESSURE_DEBOUNCE: u64 = 60; +const SCHEDULER_STATE_EVENT_TARGET: &str = "lance_io::scheduler::state"; // Global counter of how many IOPS we have issued static IOPS_COUNTER: AtomicU64 = AtomicU64::new(0); @@ -83,11 +84,23 @@ impl PrioritiesInFlight { self.in_flight.remove(pos); } } + + fn len(&self) -> usize { + self.in_flight.len() + } + + fn is_empty(&self) -> bool { + self.in_flight.is_empty() + } } struct IoQueueState { + // The configured number of IOPS that can be issued concurrently. + io_capacity: u32, // Number of IOPS we can issue concurrently before pausing I/O iops_avail: u32, + // The configured byte budget for unread I/O. + io_buffer_size: u64, // Number of bytes we are allowed to buffer in memory before pausing I/O // // This can dip below 0 due to I/O prioritization @@ -110,7 +123,9 @@ struct IoQueueState { impl IoQueueState { fn new(io_capacity: u32, io_buffer_size: u64) -> Self { Self { + io_capacity, iops_avail: io_capacity, + io_buffer_size, bytes_avail: io_buffer_size as i64, pending_requests: BinaryHeap::new(), priorities_in_flight: PrioritiesInFlight::new(io_capacity), @@ -121,6 +136,65 @@ impl IoQueueState { } } + fn scheduler_state_event(&self) -> Option { + if !tracing::enabled!(target: SCHEDULER_STATE_EVENT_TARGET, tracing::Level::TRACE) { + return None; + } + + let pending_bytes = self + .pending_requests + .iter() + .map(IoTask::num_bytes) + .sum::(); + let head_task = self.pending_requests.peek(); + let min_in_flight_priority = if self.priorities_in_flight.is_empty() { + None + } else { + Some(self.priorities_in_flight.min_in_flight()) + }; + let head_task_priority_bypass = head_task.map(|task| { + self.no_backpressure + || task.bypass_backpressure + || task.priority <= self.priorities_in_flight.min_in_flight() + }); + let head_task_blocked_by_iops = head_task.map(|_| self.iops_avail == 0); + let head_task_blocked_by_bytes = head_task.map(|task| { + let bypasses_bytes = self.no_backpressure + || task.bypass_backpressure + || task.priority <= self.priorities_in_flight.min_in_flight(); + !bypasses_bytes && task.num_bytes() as i64 > self.bytes_avail + }); + let head_task_can_deliver = head_task.map(|task| self.can_deliver_without_warning(task)); + let head_task_bytes = head_task.map(IoTask::num_bytes); + let (head_task_priority_high, head_task_priority_low) = + split_priority(head_task.map(|task| task.priority)); + let (min_in_flight_priority_high, min_in_flight_priority_low) = + split_priority(min_in_flight_priority); + + Some(SchedulerStateEvent { + queue_kind: "standard", + io_capacity: u64::from(self.io_capacity), + iops_available: u64::from(self.iops_avail), + active_iops: u64::from(self.io_capacity.saturating_sub(self.iops_avail)), + pending_iops: self.pending_requests.len() as u64, + pending_bytes, + bytes_available: self.bytes_avail, + bytes_reserved: self.io_buffer_size as i64 - self.bytes_avail, + io_buffer_size_bytes: self.io_buffer_size, + priorities_in_flight: self.priorities_in_flight.len() as u64, + no_backpressure: self.no_backpressure, + head_task_bytes, + head_task_priority_high, + head_task_priority_low, + min_in_flight_priority_high, + min_in_flight_priority_low, + head_task_can_deliver, + head_task_priority_bypass, + head_task_blocked_by_iops, + head_task_blocked_by_bytes, + }) + } + fn warn_if_needed(&self) { let seconds_elapsed = self.start.elapsed().as_secs(); let last_warn = self.last_warn.load(Ordering::Acquire); @@ -140,6 +214,20 @@ impl IoQueueState { } fn can_deliver(&self, task: &IoTask) -> bool { + let can_deliver = self.can_deliver_without_warning(task); + if !can_deliver + && self.iops_avail > 0 + && !(self.no_backpressure + || task.bypass_backpressure + || task.priority <= self.priorities_in_flight.min_in_flight()) + && task.num_bytes() as i64 > self.bytes_avail + { + self.warn_if_needed(); + } + can_deliver + } + + fn can_deliver_without_warning(&self, task: &IoTask) -> bool { if self.iops_avail == 0 { false } else if self.no_backpressure @@ -151,11 +239,8 @@ impl IoQueueState { || self.priorities_in_flight.contains(task.priority) { true - } else if task.num_bytes() as i64 > self.bytes_avail { - self.warn_if_needed(); - false } else { - true + task.num_bytes() as i64 <= self.bytes_avail } } @@ -191,13 +276,15 @@ struct IoQueue { state: Mutex, // Used to signal new I/O requests have arrived that might potentially be runnable notify: Notify, + stats: IoStats, } impl IoQueue { - fn new(io_capacity: u32, io_buffer_size: u64) -> Self { + fn new(io_capacity: u32, io_buffer_size: u64, stats: IoStats) -> Self { Self { state: Mutex::new(IoQueueState::new(io_capacity, io_buffer_size)), notify: Notify::new(), + stats, } } @@ -208,9 +295,12 @@ impl IoQueue { task.priority >> 64, task.priority & 0xFFFFFFFFFFFFFFFF ); - let mut state = self.state.lock().unwrap(); - state.pending_requests.push(task); - drop(state); + let event = { + let mut state = self.state.lock().unwrap(); + state.pending_requests.push(task); + state.scheduler_state_event() + }; + emit_scheduler_state_event(event, &self.stats); self.notify.notify_one(); } @@ -220,6 +310,9 @@ impl IoQueue { { let mut state = self.state.lock().unwrap(); if let Some(task) = state.next_task() { + let event = state.scheduler_state_event(); + drop(state); + emit_scheduler_state_event(event, &self.stats); return Some(task); } @@ -233,29 +326,39 @@ impl IoQueue { } fn on_iop_complete(&self) { - let mut state = self.state.lock().unwrap(); - state.iops_avail += 1; - drop(state); + let event = { + let mut state = self.state.lock().unwrap(); + state.iops_avail += 1; + state.scheduler_state_event() + }; + emit_scheduler_state_event(event, &self.stats); self.notify.notify_one(); } fn on_bytes_consumed(&self, bytes: u64, priority: u128, num_reqs: usize) { - let mut state = self.state.lock().unwrap(); - state.bytes_avail += bytes as i64; - for _ in 0..num_reqs { - state.priorities_in_flight.remove(priority); - } - drop(state); + let event = { + let mut state = self.state.lock().unwrap(); + state.bytes_avail += bytes as i64; + for _ in 0..num_reqs { + state.priorities_in_flight.remove(priority); + } + state.scheduler_state_event() + }; + emit_scheduler_state_event(event, &self.stats); self.notify.notify_one(); } fn close(&self) { - let mut state = self.state.lock().unwrap(); - state.done_scheduling = true; - let pending_requests = std::mem::take(&mut state.pending_requests); - drop(state); + let (pending_requests, event) = { + let mut state = self.state.lock().unwrap(); + state.done_scheduling = true; + let pending_requests = std::mem::take(&mut state.pending_requests); + let event = state.scheduler_state_event(); + (pending_requests, event) + }; + emit_scheduler_state_event(event, &self.stats); for request in pending_requests { request.cancel(); } @@ -274,9 +377,13 @@ struct MutableBatch { num_bytes: u64, priority: u128, num_reqs: usize, - err: Option>, + num_delivered: usize, + err: Option, // When true, report 0 bytes consumed so the backpressure budget is unaffected bypass_backpressure: bool, + // Queue the batch's backpressure reservation is refunded to once its response + // is delivered or discarded (see `Response`'s `Drop`). + io_queue: Arc, } impl MutableBatch { @@ -286,6 +393,7 @@ impl MutableBatch { priority: u128, num_reqs: usize, bypass_backpressure: bool, + io_queue: Arc, ) -> Self { Self { when_done: Some(when_done), @@ -293,8 +401,10 @@ impl MutableBatch { num_bytes: 0, priority, num_reqs, + num_delivered: 0, err: None, bypass_backpressure, + io_queue, } } } @@ -305,9 +415,16 @@ impl MutableBatch { // data. impl Drop for MutableBatch { fn drop(&mut self) { - // If we have an error, return that. Otherwise return the data - let result = if self.err.is_some() { - Err(Error::wrapped(self.err.take().unwrap())) + // If we have an error, return that. Otherwise return the data, as long as the I/O requests have been processed. + let result = if let Some(err) = self.err.take() { + Err(err) + } else if self.num_delivered < self.data_buffers.len() { + // This usually happens on tokio runtime shutdown + Err(Error::io(format!( + "I/O request was dropped before completion ({} of {} reads delivered)", + self.num_delivered, + self.data_buffers.len() + ))) } else { let mut data = Vec::new(); std::mem::swap(&mut data, &mut self.data_buffers); @@ -316,7 +433,8 @@ impl Drop for MutableBatch { // We don't really care if no one is around to receive it, just let // the result go out of scope and get cleaned up let response = Response { - data: result, + data: Some(result), + io_queue: self.io_queue.clone(), // Report 0 bytes for bypass tasks so the backpressure budget is unaffected num_bytes: if self.bypass_backpressure { 0 @@ -344,13 +462,14 @@ impl DataSink for MutableBatch { // Called by worker tasks to add data to the MutableBatch fn deliver_data(&mut self, data: DataChunk) { self.num_bytes += data.num_bytes; + self.num_delivered += 1; match data.data { Ok(data_bytes) => { self.data_buffers[data.task_idx] = data_bytes; } Err(err) => { // This keeps the original error, if present - self.err.get_or_insert(Box::new(err)); + self.err.get_or_insert(err); } } } @@ -364,6 +483,23 @@ struct IoTask { bypass_backpressure: bool, } +fn validate_read_length( + file_path: &Path, + requested_range: &Range, + bytes: Bytes, +) -> Result { + let expected_len = requested_range.end - requested_range.start; + if bytes.len() as u64 != expected_len { + return Err(Error::io(format!( + "I/O request for file {file_path} and range {}..{} returned {} bytes, expected {expected_len} bytes", + requested_range.start, + requested_range.end, + bytes.len() + ))); + } + Ok(bytes) +} + impl Eq for IoTask {} impl PartialEq for IoTask { @@ -415,6 +551,7 @@ impl IoTask { }) .await .map_err(Error::from) + .and_then(|bytes| validate_read_length(self.reader.path(), &self.to_read, bytes)) }; // Emit per-file I/O trace event only when tracing is enabled tracing::trace!( @@ -519,6 +656,84 @@ impl ScanStats { } } +fn split_priority(priority: Option) -> (Option, Option) { + priority + .map(|priority| ((priority >> 64) as u64, priority as u64)) + .unzip() +} + +#[derive(Debug, Clone, Copy)] +pub(super) struct SchedulerStateEvent { + pub(super) queue_kind: &'static str, + pub(super) io_capacity: u64, + pub(super) iops_available: u64, + pub(super) active_iops: u64, + pub(super) pending_iops: u64, + pub(super) pending_bytes: u64, + pub(super) bytes_available: i64, + pub(super) bytes_reserved: i64, + pub(super) io_buffer_size_bytes: u64, + pub(super) priorities_in_flight: u64, + pub(super) no_backpressure: bool, + pub(super) head_task_bytes: Option, + pub(super) head_task_priority_high: Option, + pub(super) head_task_priority_low: Option, + pub(super) min_in_flight_priority_high: Option, + pub(super) min_in_flight_priority_low: Option, + pub(super) head_task_can_deliver: Option, + pub(super) head_task_priority_bypass: Option, + pub(super) head_task_blocked_by_iops: Option, + pub(super) head_task_blocked_by_bytes: Option, +} + +impl SchedulerStateEvent { + fn trace(self, stats: ScanStats) { + tracing::event!( + target: SCHEDULER_STATE_EVENT_TARGET, + tracing::Level::TRACE, + queue_kind = self.queue_kind, + scheduler_iops = stats.iops, + scheduler_requests = stats.requests, + scheduler_bytes_read = stats.bytes_read, + io_capacity = self.io_capacity, + iops_available = self.iops_available, + active_iops = self.active_iops, + pending_iops = self.pending_iops, + pending_bytes = self.pending_bytes, + bytes_available = self.bytes_available, + bytes_reserved = self.bytes_reserved, + io_buffer_size_bytes = self.io_buffer_size_bytes, + priorities_in_flight = self.priorities_in_flight, + no_backpressure = self.no_backpressure, + head_task_bytes_present = self.head_task_bytes.is_some(), + head_task_bytes = self.head_task_bytes.unwrap_or_default(), + head_task_priority_high_present = self.head_task_priority_high.is_some(), + head_task_priority_high = self.head_task_priority_high.unwrap_or_default(), + head_task_priority_low_present = self.head_task_priority_low.is_some(), + head_task_priority_low = self.head_task_priority_low.unwrap_or_default(), + min_in_flight_priority_high_present = self.min_in_flight_priority_high.is_some(), + min_in_flight_priority_high = self.min_in_flight_priority_high.unwrap_or_default(), + min_in_flight_priority_low_present = self.min_in_flight_priority_low.is_some(), + min_in_flight_priority_low = self.min_in_flight_priority_low.unwrap_or_default(), + head_task_can_deliver_present = self.head_task_can_deliver.is_some(), + head_task_can_deliver = self.head_task_can_deliver.unwrap_or(false), + head_task_priority_bypass_present = self.head_task_priority_bypass.is_some(), + head_task_priority_bypass = self.head_task_priority_bypass.unwrap_or(false), + head_task_blocked_by_iops_present = self.head_task_blocked_by_iops.is_some(), + head_task_blocked_by_iops = self.head_task_blocked_by_iops.unwrap_or(false), + head_task_blocked_by_bytes_present = self.head_task_blocked_by_bytes.is_some(), + head_task_blocked_by_bytes = self.head_task_blocked_by_bytes.unwrap_or(false), + "Scheduler state" + ); + } +} + +pub(super) fn emit_scheduler_state_event(event: Option, stats: &IoStats) { + if let Some(event) = event { + event.trace(stats.snapshot()); + } +} + /// A shareable, cloneable handle to a set of cumulative I/O counters. /// /// All clones share the same underlying counters. This serves two purposes: @@ -598,12 +813,25 @@ impl Debug for ScanScheduler { } struct Response { - data: Result>, + // `Option` so the caller can take the data out while the response (and its + // backpressure refund on drop) stays intact. + data: Option>>, + io_queue: Arc, priority: u128, num_reqs: usize, num_bytes: u64, } +// Refund the batch's backpressure reservation when the response is dropped, be +// that on delivery or when a cancelled request's undelivered response is +// discarded. This releases the budget even if the caller drops the future early. +impl Drop for Response { + fn drop(&mut self) { + self.io_queue + .on_bytes_consumed(self.num_bytes, self.priority, self.num_reqs); + } +} + #[derive(Debug, Clone, Copy)] pub struct SchedulerConfig { /// the # of bytes that can be buffered but not yet requested. @@ -659,6 +887,7 @@ impl ScanScheduler { /// * config - configuration settings for the scheduler pub fn new(object_store: Arc, config: SchedulerConfig) -> Arc { let io_capacity = object_store.io_parallelism(); + let stats = IoStats::new(); let use_lite = config .use_lite_scheduler .unwrap_or_else(|| object_store.prefers_lite_scheduler()); @@ -666,12 +895,14 @@ impl ScanScheduler { let io_queue = Arc::new(lite::IoQueue::new( io_capacity as u64, config.io_buffer_size_bytes, + stats.clone(), )); IoQueueType::Lite(io_queue) } else { let io_queue = Arc::new(IoQueue::new( io_capacity as u32, config.io_buffer_size_bytes, + stats.clone(), )); let io_queue_clone = io_queue.clone(); // Best we can do here is fire and forget. If the I/O loop is still running when the scheduler is @@ -683,7 +914,7 @@ impl ScanScheduler { Arc::new(Self { object_store, io_queue, - stats: IoStats::new(), + stats, }) } @@ -782,6 +1013,7 @@ impl ScanScheduler { priority, request.len(), bypass_backpressure, + io_queue.clone(), )))); for (task_idx, iop) in request.into_iter().enumerate() { @@ -820,14 +1052,11 @@ impl ScanScheduler { self.do_submit_request(reader, request, tx, priority, io_queue, bypass_backpressure); - let io_queue_clone = io_queue.clone(); - - rx.map(move |wrapped_rsp| { - // Right now, it isn't possible for I/O to be cancelled so a cancel error should - // not occur - let rsp = wrapped_rsp.unwrap(); - io_queue_clone.on_bytes_consumed(rsp.num_bytes, rsp.priority, rsp.num_reqs); - rsp.data + rx.map(|wrapped_rsp| { + // A cancel error can't occur: the sender always sends before dropping. + // The reservation is refunded on `Response` drop, so just take the data. + let mut rsp = wrapped_rsp.unwrap(); + rsp.data.take().unwrap() }) } @@ -845,11 +1074,15 @@ impl ScanScheduler { .map(|task| { let reader = reader.clone(); let queue = io_queue.clone(); + let requested_range = task.clone(); let run_fn = Box::new(move || { - reader - .get_range(task.start as usize..task.end as usize) - .map_err(Error::from) - .boxed() + let bytes_fut = reader + .get_range(requested_range.start as usize..requested_range.end as usize); + async move { + let bytes = bytes_fut.await.map_err(Error::from)?; + validate_read_length(reader.path(), &requested_range, bytes) + } + .boxed() }); queue.submit(task, priority, run_fn, bypass_backpressure) }) @@ -1134,6 +1367,7 @@ mod tests { use futures::poll; use lance_core::utils::tempfile::TempObjFile; use rand::RngCore; + use rstest::rstest; use object_store::{GetRange, ObjectStore as OSObjectStore, ObjectStoreExt, memory::InMemory}; use tokio::{runtime::Handle, time::timeout}; @@ -1159,6 +1393,47 @@ mod tests { } } + #[test] + fn test_scheduler_state_event_fields() { + use tracing_mock::{expect, subscriber}; + + let event = expect::event() + .with_target(SCHEDULER_STATE_EVENT_TARGET) + .at_level(tracing::Level::TRACE) + .with_fields( + expect::field("queue_kind") + .with_value(&"standard") + .and(expect::field("scheduler_iops").with_value(&7u64)) + .and(expect::field("scheduler_requests").with_value(&3u64)) + .and(expect::field("scheduler_bytes_read").with_value(&4096u64)) + .and(expect::field("io_capacity").with_value(&4u64)) + .and(expect::field("pending_iops").with_value(&1u64)) + .and(expect::field("bytes_available").with_value(&128i64)) + .and(expect::field("head_task_bytes_present").with_value(&true)) + .and(expect::field("head_task_bytes").with_value(&1u64)) + .and(expect::field("head_task_can_deliver_present").with_value(&true)) + .and(expect::field("head_task_can_deliver").with_value(&true)), + ); + let (subscriber, handle) = subscriber::mock().event(event).run_with_handle(); + + let stats = IoStats::new(); + stats.add_scan_stats(&ScanStats { + iops: 7, + requests: 3, + bytes_read: 4096, + }); + let mut state = IoQueueState::new(4, 192); + state.iops_avail = 2; + state.bytes_avail = 128; + state.pending_requests.push(make_task(1, false)); + + tracing::subscriber::with_default(subscriber, || { + emit_scheduler_state_event(state.scheduler_state_event(), &stats); + }); + + handle.assert_finished(); + } + #[test] fn test_iotask_ordering() { // Bypass tasks must come out of the heap before non-bypass tasks. @@ -1176,6 +1451,29 @@ mod tests { assert_eq!(order, vec![(5, true), (20, true), (1, false), (10, false)]); } + #[test] + fn test_batch_with_undelivered_slot_is_error() { + let response = Arc::new(Mutex::new(None)); + let response_clone = response.clone(); + let io_queue = Arc::new(IoQueue::new(1, 1024, IoStats::new())); + let batch = MutableBatch::new( + move |rsp| *response_clone.lock().unwrap() = Some(rsp), + 2, // num_data_buffers + 0, // priority + 2, // num_reqs + false, + io_queue, + ); + drop(batch); + + let mut rsp = response.lock().unwrap().take().unwrap(); + let data = rsp.data.take().unwrap(); + assert!( + data.is_err(), + "undelivered slot must yield an error, got {data:?}", + ); + } + #[tokio::test] async fn test_full_seq_read() { let tmp_file = TempObjFile::default(); @@ -1248,6 +1546,75 @@ mod tests { assert_eq!(bytes[0], some_data); } + #[derive(Debug)] + struct ShortReader { + path: Path, + } + + impl lance_core::deepsize::DeepSizeOf for ShortReader { + fn deep_size_of_children(&self, _context: &mut lance_core::deepsize::Context) -> usize { + 0 + } + } + + impl Reader for ShortReader { + fn path(&self) -> &Path { + &self.path + } + + fn block_size(&self) -> usize { + 4096 + } + + fn io_parallelism(&self) -> usize { + 1 + } + + fn size(&self) -> futures::future::BoxFuture<'_, object_store::Result> { + Box::pin(async { Ok(0) }) + } + + fn get_range( + &self, + _range: Range, + ) -> futures::future::BoxFuture<'static, object_store::Result> { + Box::pin(async { Ok(Bytes::new()) }) + } + + fn get_all(&self) -> futures::future::BoxFuture<'_, object_store::Result> { + Box::pin(async { Ok(Bytes::new()) }) + } + } + + #[rstest] + #[case::standard(false)] + #[case::lite(true)] + #[tokio::test] + async fn test_short_read_returns_io_error(#[case] use_lite_scheduler: bool) { + let config = SchedulerConfig { + use_lite_scheduler: Some(use_lite_scheduler), + ..SchedulerConfig::default_for_testing() + }; + let scheduler = ScanScheduler::new(Arc::new(ObjectStore::memory()), config); + let reader = Arc::new(ShortReader { + path: Path::parse("short-file").unwrap(), + }); + let file_scheduler = scheduler.open_reader(reader); + + let error = file_scheduler + .submit_request(vec![0..8], 0) + .await + .unwrap_err(); + + assert!(matches!(error, Error::IO { .. }), "{error:?}"); + assert!( + error.to_string().contains( + "I/O request for file short-file and range 0..8 returned 0 bytes, expected 8 bytes" + ), + "{error}" + ); + } + #[tokio::test] async fn test_split_coalesce() { let tmp_file = TempObjFile::default(); @@ -1474,6 +1841,136 @@ mod tests { assert!(second_fut.await.unwrap().unwrap().len() == 20); } + #[tokio::test] + async fn test_standard_scheduler_state_tracks_queue_state() { + let some_path = Path::parse("foo").unwrap(); + let base_store = Arc::new(InMemory::new()); + base_store + .put(&some_path, vec![0; 1000].into()) + .await + .unwrap(); + + let semaphore = Arc::new(tokio::sync::Semaphore::new(0)); + let mut obj_store = MockObjectStore::default(); + let semaphore_copy = semaphore.clone(); + obj_store + .expect_get_opts() + .returning(move |location, options| { + let semaphore = semaphore.clone(); + let base_store = base_store.clone(); + let location = location.clone(); + async move { + semaphore.acquire().await.unwrap().forget(); + base_store.get_opts(&location, options).await + } + .boxed() + }); + let obj_store = Arc::new(ObjectStore::new( + Arc::new(obj_store), + Url::parse("mem://").unwrap(), + Some(500), + None, + false, + false, + 1, + DEFAULT_DOWNLOAD_RETRY_COUNT, + None, + )); + + let scheduler = ScanScheduler::new( + obj_store, + SchedulerConfig { + io_buffer_size_bytes: 1024 * 1024, + use_lite_scheduler: Some(false), + }, + ); + let file_scheduler = scheduler + .open_file(&Path::parse("foo").unwrap(), &CachedFileSize::new(1000)) + .await + .unwrap(); + + let first_fut = timeout( + Duration::from_secs(10), + file_scheduler.submit_single(0..10, 0), + ) + .boxed(); + let second_fut = timeout( + Duration::from_secs(10), + file_scheduler.submit_single(0..20, 100), + ) + .boxed(); + let third_fut = timeout( + Duration::from_secs(10), + file_scheduler.submit_single(0..30, 0), + ) + .boxed(); + + let io_queue = match &scheduler.io_queue { + IoQueueType::Standard(io_queue) => io_queue.clone(), + IoQueueType::Lite(_) => unreachable!("test forces the standard scheduler"), + }; + let ( + io_capacity, + iops_available, + pending_bytes, + bytes_reserved, + priorities_in_flight, + head_task_bytes, + head_task_blocked_by_iops, + head_task_blocked_by_bytes, + ) = timeout(Duration::from_secs(5), async { + loop { + let observed = { + let state = io_queue.state.lock().unwrap(); + let active_iops = state.io_capacity.saturating_sub(state.iops_avail); + if active_iops == 1 && state.pending_requests.len() == 2 { + let pending_bytes = state + .pending_requests + .iter() + .map(IoTask::num_bytes) + .sum::(); + let head_task = state.pending_requests.peek().unwrap(); + let bypasses_bytes = state.no_backpressure + || head_task.bypass_backpressure + || head_task.priority <= state.priorities_in_flight.min_in_flight(); + Some(( + state.io_capacity, + state.iops_avail, + pending_bytes, + state.io_buffer_size as i64 - state.bytes_avail, + state.priorities_in_flight.len(), + head_task.num_bytes(), + state.iops_avail == 0, + !bypasses_bytes && head_task.num_bytes() as i64 > state.bytes_avail, + )) + } else { + None + } + }; + if let Some(observed) = observed { + break observed; + } + tokio::task::yield_now().await; + } + }) + .await + .unwrap(); + + assert_eq!(io_capacity, 1); + assert_eq!(iops_available, 0); + assert_eq!(pending_bytes, 50); + assert_eq!(bytes_reserved, 10); + assert_eq!(priorities_in_flight, 1); + assert_eq!(head_task_bytes, 30); + assert!(head_task_blocked_by_iops); + assert!(!head_task_blocked_by_bytes); + + semaphore_copy.add_permits(3); + assert_eq!(first_fut.await.unwrap().unwrap().len(), 10); + assert_eq!(third_fut.await.unwrap().unwrap().len(), 30); + assert_eq!(second_fut.await.unwrap().unwrap().len(), 20); + } + #[tokio::test(flavor = "multi_thread")] async fn test_backpressure() { let some_path = Path::parse("foo").unwrap(); @@ -1609,6 +2106,7 @@ mod tests { #[derive(Debug)] struct BlockingReader { semaphore: Arc, + get_range_count: Arc, path: Path, } @@ -1639,6 +2137,7 @@ mod tests { &self, range: Range, ) -> futures::future::BoxFuture<'static, object_store::Result> { + self.get_range_count.fetch_add(1, Ordering::Release); let semaphore = self.semaphore.clone(); let num_bytes = range.end - range.start; Box::pin(async move { @@ -1675,6 +2174,7 @@ mod tests { let semaphore = Arc::new(tokio::sync::Semaphore::new(0)); let reader: Arc = Arc::new(BlockingReader { semaphore: semaphore.clone(), + get_range_count: Arc::new(AtomicU64::new(0)), path: Path::parse("test").unwrap(), }); @@ -1981,4 +2481,105 @@ mod tests { .unwrap(); assert_eq!(bytes_dispatched.load(Ordering::Acquire), 30); } + + // Against a 100-byte budget: submit fut1 (50 bytes, priority 0), drop it while + // its read is still blocked in get_range, then submit fut2 (60 bytes, priority 1). + // fut2's priority can't win the priority-bypass, so it needs 60 of the budget -- + // available only if fut1's dropped reservation was refunded. Returns whether fut2 + // completed within 2s (false = the reservation leaked and fut2 deadlocked). + async fn run_caller_drop_scenario(use_lite_scheduler: bool) -> (bool, Duration) { + let obj_store = Arc::new(ObjectStore::new( + Arc::new(InMemory::new()), + Url::parse("mem://").unwrap(), + Some(4096), + None, + false, + false, + 1, + DEFAULT_DOWNLOAD_RETRY_COUNT, + None, + )); + let scheduler = ScanScheduler::new( + obj_store, + SchedulerConfig { + io_buffer_size_bytes: 100, + use_lite_scheduler: Some(use_lite_scheduler), + }, + ); + + let semaphore = Arc::new(tokio::sync::Semaphore::new(0)); + let get_range_count = Arc::new(AtomicU64::new(0)); + let reader: Arc = Arc::new(BlockingReader { + semaphore: semaphore.clone(), + get_range_count: get_range_count.clone(), + path: Path::parse("test").unwrap(), + }); + + // Step 1: reserve 50 of the 100 budget bytes with a read we never consume. + // Spawn it so we can cancel the caller-side future while it is still parked + // waiting for the (blocked) read to finish. + let fut1 = scheduler.submit_request(reader.clone(), vec![0..50], 0, false); + let handle = tokio::spawn(async move { + let _ = fut1.await; + }); + + // Wait until the read is genuinely in flight (blocked on the semaphore). + // This guarantees the 50-byte reservation has been taken before we drop + // the caller, closing the race between the I/O loop and the abort. + while get_range_count.load(Ordering::Acquire) == 0 { + tokio::time::sleep(Duration::from_millis(1)).await; + } + + // Step 2: drop the caller-side future while its `rx` is still pending. + handle.abort(); + let _ = handle.await; + + // Step 3: let the in-flight read finish. The reservation should be refunded + // now that the request is done, whether or not the caller is still around. + semaphore.add_permits(1); + // Give the read time to run to completion so the refund would already have + // happened. + tokio::time::sleep(Duration::from_millis(50)).await; + + // Step 4: submit the follow-up. Add a permit up front so that, if it *is* + // admitted, its own read can complete rather than block on the semaphore. + semaphore.add_permits(1); + let fut2 = scheduler.submit_request(reader, vec![100..160], 1, false); + + let start = std::time::Instant::now(); + let outcome = timeout(Duration::from_secs(2), fut2).await; + let elapsed = start.elapsed(); + match outcome { + Ok(res) => { + assert_eq!(res.unwrap().iter().map(|b| b.len()).sum::(), 60); + (true, elapsed) + } + Err(_) => (false, elapsed), + } + } + + /// Dropping a standard-scheduler request future while its read is in flight must + /// still refund the backpressure reservation, so a later request that needs the + /// budget does not deadlock. + #[tokio::test(flavor = "multi_thread")] + async fn standard_scheduler_refunds_reservation_on_caller_drop() { + let (completed, elapsed) = run_caller_drop_scenario(false).await; + assert!( + completed, + "standard scheduler deadlocked the follow-up request (elapsed {elapsed:?}); \ + the dropped request's reservation was not refunded" + ); + } + + /// Same guarantee for the lite scheduler: dropping a request future mid-read + /// releases its reservation via the `TaskHandle` drop path. + #[tokio::test(flavor = "multi_thread")] + async fn lite_scheduler_refunds_reservation_on_caller_drop() { + let (completed, elapsed) = run_caller_drop_scenario(true).await; + assert!( + completed, + "lite scheduler deadlocked the follow-up request (elapsed {elapsed:?}); \ + the dropped request's reservation was not refunded" + ); + } } diff --git a/rust/lance-io/src/scheduler/lite.rs b/rust/lance-io/src/scheduler/lite.rs index fc666139a05..b7a06e11bb7 100644 --- a/rust/lance-io/src/scheduler/lite.rs +++ b/rust/lance-io/src/scheduler/lite.rs @@ -33,7 +33,10 @@ use std::{ use bytes::Bytes; use lance_core::{Error, Result}; -use super::{BACKPRESSURE_DEBOUNCE, BACKPRESSURE_MIN}; +use super::{ + BACKPRESSURE_DEBOUNCE, BACKPRESSURE_MIN, IoStats, SCHEDULER_STATE_EVENT_TARGET, + SchedulerStateEvent, emit_scheduler_state_event, +}; type RunFn = Box Pin> + Send>> + Send>; @@ -67,6 +70,26 @@ enum TaskState { }, } +impl TaskState { + fn backpressure_reservation(&self) -> Option { + match self { + Self::Reserved { + backpressure_reservation, + .. + } + | Self::Running { + backpressure_reservation, + .. + } + | Self::Finished { + backpressure_reservation, + .. + } => Some(*backpressure_reservation), + Self::Initial { .. } | Self::Broken => None, + } + } +} + /// A custom error type that might have a backpressure reservation /// /// This is used instead of Lance's standard error type so we can ensure @@ -85,25 +108,14 @@ impl BrokenTaskError { // This will capture any backpressure reservation the task has and put it into the // error so we make sure to release it when returning the error. fn new(task_state: TaskState, message: String) -> Self { - match task_state { - TaskState::Reserved { - backpressure_reservation, - .. - } - | TaskState::Running { - backpressure_reservation, - .. - } - | TaskState::Finished { - backpressure_reservation, - .. - } => Self { + match task_state.backpressure_reservation() { + None => Self { message, - backpressure_reservation: Some(backpressure_reservation), + backpressure_reservation: None, }, - TaskState::Broken | TaskState::Initial { .. } => Self { + Some(reservation) => Self { message, - backpressure_reservation: None, + backpressure_reservation: Some(reservation), }, } } @@ -237,6 +249,7 @@ trait BackpressureThrottle: Send { /// Unconditionally acquire a zero-cost reservation, tracking only the priority. /// Used for bypass tasks that must never be blocked by backpressure. fn force_acquire(&mut self, priority: u128) -> BackpressureReservation; + fn state(&self) -> BackpressureState; } // We want to allow requests that have a lower priority than any @@ -279,9 +292,22 @@ impl PrioritiesInFlight { self.in_flight.remove(pos); } } + + fn len(&self) -> usize { + self.in_flight.len() + } +} + +#[derive(Debug, Clone, Copy)] +struct BackpressureState { + max_bytes: u64, + bytes_available: i64, + priorities_in_flight: u64, + no_backpressure: bool, } struct SimpleBackpressureThrottle { + max_bytes: u64, start: Instant, last_warn: AtomicU64, bytes_available: i64, @@ -297,6 +323,7 @@ impl SimpleBackpressureThrottle { panic!("Max bytes must be less than {}", i64::MAX); } Self { + max_bytes, start: Instant::now(), last_warn: AtomicU64::new(0), bytes_available: max_bytes as i64, @@ -358,6 +385,15 @@ impl BackpressureThrottle for SimpleBackpressureThrottle { priority, } } + + fn state(&self) -> BackpressureState { + BackpressureState { + max_bytes: self.max_bytes, + bytes_available: self.bytes_available, + priorities_in_flight: self.priorities_in_flight.len() as u64, + no_backpressure: self.no_backpressure, + } + } } struct TaskEntry { @@ -427,6 +463,48 @@ impl IoQueueState { Ok(()) } } + + fn scheduler_state_event(&self) -> Option { + if !tracing::enabled!(target: SCHEDULER_STATE_EVENT_TARGET, tracing::Level::TRACE) { + return None; + } + + let backpressure = self.backpressure_throttle.state(); + let pending_bytes = self + .pending_tasks + .iter() + .filter_map(|entry| self.tasks.get(&entry.task_id)) + .map(|task| task.num_bytes) + .sum::(); + let active_iops = self + .tasks + .values() + .filter(|task| matches!(task.state, TaskState::Running { .. })) + .count() as u64; + + Some(SchedulerStateEvent { + queue_kind: "lite", + io_capacity: 0, + iops_available: 0, + active_iops, + pending_iops: self.pending_tasks.len() as u64, + pending_bytes, + bytes_available: backpressure.bytes_available, + bytes_reserved: backpressure.max_bytes as i64 - backpressure.bytes_available, + io_buffer_size_bytes: backpressure.max_bytes, + priorities_in_flight: backpressure.priorities_in_flight, + no_backpressure: backpressure.no_backpressure, + head_task_bytes: None, + head_task_priority_high: None, + head_task_priority_low: None, + min_in_flight_priority_high: None, + min_in_flight_priority_low: None, + head_task_can_deliver: None, + head_task_priority_bypass: None, + head_task_blocked_by_iops: None, + head_task_blocked_by_bytes: None, + }) + } } /// A queue of I/O tasks to be shared between the I/O scheduler and the I/O decoder. @@ -449,12 +527,14 @@ impl IoQueueState { /// day as well) pub(super) struct IoQueue { state: Arc>, + stats: IoStats, } impl IoQueue { - pub fn new(max_concurrency: u64, max_bytes: u64) -> Self { + pub fn new(max_concurrency: u64, max_bytes: u64, stats: IoStats) -> Self { Self { state: Arc::new(Mutex::new(IoQueueState::new(max_concurrency, max_bytes))), + stats, } } @@ -471,6 +551,9 @@ impl IoQueue { state.handle_result(task.reserve(reservation))?; state.handle_result(task.start())?; state.tasks.insert(task_id, task); + let event = state.scheduler_state_event(); + drop(state); + emit_scheduler_state_event(event, &self.stats); return Ok(()); } @@ -480,6 +563,9 @@ impl IoQueue { reserved: task.is_reserved(), }); state.tasks.insert(task_id, task); + let event = state.scheduler_state_event(); + drop(state); + emit_scheduler_state_event(event, &self.stats); Ok(()) } @@ -518,34 +604,42 @@ impl IoQueue { // When a task completes we should check to see if any other tasks are now runnable fn on_task_complete(&self, mut state: MutexGuard) -> Result<()> { - let state_ref = &mut *state; - let mut task_result = TaskResult::Ok(()); - while !state_ref.pending_tasks.is_empty() { - // Unwrap safe here since we just checked the queue is not empty - let next_task = state_ref.pending_tasks.peek().unwrap(); - let Some(task) = state_ref.tasks.get_mut(&next_task.task_id) else { - log::warn!("Task with id {} was lost", next_task.task_id); - continue; - }; - if !task.is_reserved() { - let Some(reservation) = state_ref - .backpressure_throttle - .try_acquire(task.num_bytes, task.priority) - else { - break; + let result = { + let state_ref = &mut *state; + let mut task_result = TaskResult::Ok(()); + while !state_ref.pending_tasks.is_empty() { + // Unwrap safe here since we just checked the queue is not empty + let task_id = state_ref.pending_tasks.peek().unwrap().task_id; + let Some(task) = state_ref.tasks.get_mut(&task_id) else { + // The caller dropped this task's handle (see `abandon`); discard the + // stale queue entry instead of spinning on it. + state_ref.pending_tasks.pop(); + continue; }; - if let Err(e) = task.reserve(reservation) { + if !task.is_reserved() { + let Some(reservation) = state_ref + .backpressure_throttle + .try_acquire(task.num_bytes, task.priority) + else { + break; + }; + if let Err(e) = task.reserve(reservation) { + task_result = Err(e); + break; + } + } + state_ref.pending_tasks.pop(); + if let Err(e) = task.start() { task_result = Err(e); break; } } - state_ref.pending_tasks.pop(); - if let Err(e) = task.start() { - task_result = Err(e); - break; - } - } - state_ref.handle_result(task_result) + state_ref.handle_result(task_result) + }; + let event = state.scheduler_state_event(); + drop(state); + emit_scheduler_state_event(event, &self.stats); + result } fn poll(&self, task_id: u64, cx: &mut Context<'_>) -> Poll> { @@ -573,10 +667,34 @@ impl IoQueue { } pub(super) fn close(&self) { + let event = { + let mut state = self.state.lock().unwrap(); + for task in std::mem::take(&mut state.tasks).values_mut() { + task.cancel(); + } + state.scheduler_state_event() + }; + emit_scheduler_state_event(event, &self.stats); + } + + // Called when a caller drops a task's handle before the task finishes. Removes + // the task and returns any backpressure reservation it holds to the budget, then + // re-checks the queue so newly-affordable tasks can start. Unlike the standard + // release path (`poll`), this runs without the task being polled to completion, + // so a cancelled read does not leak its reservation. + fn abandon(&self, task_id: u64) { let mut state = self.state.lock().unwrap(); - for task in std::mem::take(&mut state.tasks).values_mut() { - task.cancel(); + let Some(task) = state.tasks.remove(&task_id) else { + // Already consumed by `poll`; nothing to release. + return; + }; + + if let Some(reservation) = task.state.backpressure_reservation() { + state.backpressure_throttle.release(reservation); } + // Freed budget may make queued tasks runnable; there is no caller to surface + // an error to here. + let _ = self.on_task_complete(state); } } @@ -592,6 +710,12 @@ impl Future for TaskHandle { } } +impl Drop for TaskHandle { + fn drop(&mut self) { + self.queue.abandon(self.task_id); + } +} + #[cfg(test)] mod tests { use super::*; @@ -600,7 +724,7 @@ mod tests { #[tokio::test] async fn test_priority_ordering() { // Backpressure budget of 10 bytes: only one 10-byte task runs at a time. - let queue = Arc::new(IoQueue::new(128, 10)); + let queue = Arc::new(IoQueue::new(128, 10, IoStats::default())); // Records the priority of each task when its run_fn is invoked (i.e. when // the task transitions to Running). @@ -708,7 +832,7 @@ mod tests { async fn test_zero_buffer_bypasses_backpressure() { // Budget = 0 sets no_backpressure = true, so all tasks start immediately // regardless of how many bytes are "outstanding". - let queue = Arc::new(IoQueue::new(128, 0)); + let queue = Arc::new(IoQueue::new(128, 0, IoStats::default())); let start_order: Arc>> = Arc::new(Mutex::new(Vec::new())); let make_run_fn = @@ -750,7 +874,7 @@ mod tests { async fn test_bypass_flag_proceeds_past_exhausted_budget() { // Budget of 10 bytes. A blocker task fills it. A task with bypass=true starts // immediately despite the exhausted budget; a normal task stays queued. - let queue = Arc::new(IoQueue::new(128, 10)); + let queue = Arc::new(IoQueue::new(128, 10, IoStats::default())); let start_order: Arc>> = Arc::new(Mutex::new(Vec::new())); let make_run_fn = diff --git a/rust/lance-io/src/uring/current_thread.rs b/rust/lance-io/src/uring/current_thread.rs index abac772218b..ae2e3414ee9 100644 --- a/rust/lance-io/src/uring/current_thread.rs +++ b/rust/lance-io/src/uring/current_thread.rs @@ -14,8 +14,8 @@ use crate::traits::Reader; use crate::uring::DEFAULT_URING_QUEUE_DEPTH; use crate::utils::tracking_store::IOTracker; use bytes::{Bytes, BytesMut}; +use futures::FutureExt; use futures::future::BoxFuture; -use futures::{FutureExt, TryFutureExt}; use io_uring::{IoUring, opcode, types}; use lance_core::deepsize::DeepSizeOf; use lance_core::{Error, Result}; @@ -396,10 +396,14 @@ impl Reader for UringCurrentThreadReader { let num_bytes = range.len() as u64; let range_u64 = (range.start as u64)..(range.end as u64); + let metrics = self.io_tracker.begin_io("get"); self.submit_read(range.start as u64, range.len()) - .map_ok(move |bytes| { - io_tracker.record_read("get_range", path, num_bytes, Some(range_u64)); - bytes + .map(move |result| { + metrics.record(&result, num_bytes); + if result.is_ok() { + io_tracker.record_read("get_range", path, num_bytes, Some(range_u64)); + } + result }) .boxed() } @@ -411,10 +415,15 @@ impl Reader for UringCurrentThreadReader { let io_tracker = self.io_tracker.clone(); let path = self.handle.path.clone(); + let metrics = self.io_tracker.begin_io("get"); self.submit_read(0, size) - .map_ok(move |bytes| { - io_tracker.record_read("get_all", path, bytes.len() as u64, None); - bytes + .map(move |result| { + let num_bytes = result.as_ref().map_or(0, |bytes| bytes.len() as u64); + metrics.record(&result, num_bytes); + if result.is_ok() { + io_tracker.record_read("get_all", path, num_bytes, None); + } + result }) .boxed() } diff --git a/rust/lance-io/src/uring/reader.rs b/rust/lance-io/src/uring/reader.rs index a948e6c63dc..2d6bb11c4a4 100644 --- a/rust/lance-io/src/uring/reader.rs +++ b/rust/lance-io/src/uring/reader.rs @@ -12,8 +12,8 @@ use crate::traits::Reader; use crate::uring::requests::RequestState; use crate::utils::tracking_store::IOTracker; use bytes::{Bytes, BytesMut}; +use futures::FutureExt; use futures::future::BoxFuture; -use futures::{FutureExt, TryFutureExt}; use lance_core::deepsize::DeepSizeOf; use lance_core::{Error, Result}; use object_store::path::Path; @@ -267,10 +267,14 @@ impl Reader for UringReader { let num_bytes = range.len() as u64; let range_u64 = (range.start as u64)..(range.end as u64); + let metrics = self.io_tracker.begin_io("get"); self.submit_read(range.start as u64, range.len()) - .map_ok(move |bytes| { - io_tracker.record_read("get_range", path, num_bytes, Some(range_u64)); - bytes + .map(move |result| { + metrics.record(&result, num_bytes); + if result.is_ok() { + io_tracker.record_read("get_range", path, num_bytes, Some(range_u64)); + } + result }) .boxed() } @@ -282,10 +286,15 @@ impl Reader for UringReader { let io_tracker = self.io_tracker.clone(); let path = self.handle.path.clone(); + let metrics = self.io_tracker.begin_io("get"); self.submit_read(0, size) - .map_ok(move |bytes| { - io_tracker.record_read("get_all", path, bytes.len() as u64, None); - bytes + .map(move |result| { + let num_bytes = result.as_ref().map_or(0, |bytes| bytes.len() as u64); + metrics.record(&result, num_bytes); + if result.is_ok() { + io_tracker.record_read("get_all", path, num_bytes, None); + } + result }) .boxed() } diff --git a/rust/lance-io/src/utils.rs b/rust/lance-io/src/utils.rs index a36aadec9d9..4b471d8b009 100644 --- a/rust/lance-io/src/utils.rs +++ b/rust/lance-io/src/utils.rs @@ -3,81 +3,17 @@ use std::{cmp::min, num::NonZero, sync::atomic::AtomicU64}; -use arrow_array::{ - ArrayRef, - types::{BinaryType, LargeBinaryType, LargeUtf8Type, Utf8Type}, -}; -use arrow_schema::DataType; use byteorder::{ByteOrder, LittleEndian}; use bytes::Bytes; -use lance_arrow::*; use lance_core::deepsize::DeepSizeOf; use prost::Message; use serde::{Deserialize, Serialize}; -use crate::{ReadBatchParams, traits::Reader}; -use crate::{ - encodings::{AsyncIndex, Decoder, binary::BinaryDecoder, plain::PlainDecoder}, - traits::ProtoStruct, -}; +use crate::traits::{ProtoStruct, Reader}; use lance_core::{Error, Result}; pub mod tracking_store; -/// Read a binary array from a [Reader]. -/// -pub async fn read_binary_array( - reader: &dyn Reader, - data_type: &DataType, - nullable: bool, - position: usize, - length: usize, - params: impl Into, -) -> Result { - use arrow_schema::DataType::*; - let decoder: Box> + Send> = match data_type { - Utf8 => Box::new(BinaryDecoder::::new( - reader, position, length, nullable, - )), - Binary => Box::new(BinaryDecoder::::new( - reader, position, length, nullable, - )), - LargeUtf8 => Box::new(BinaryDecoder::::new( - reader, position, length, nullable, - )), - LargeBinary => Box::new(BinaryDecoder::::new( - reader, position, length, nullable, - )), - _ => { - return Err(Error::invalid_input(format!( - "Unsupported binary type: {}", - data_type - ))); - } - }; - let fut = decoder.as_ref().get(params.into()); - fut.await -} - -/// Read a fixed stride array from disk. -/// -pub async fn read_fixed_stride_array( - reader: &dyn Reader, - data_type: &DataType, - position: usize, - length: usize, - params: impl Into, -) -> Result { - if !data_type.is_fixed_stride() { - return Err(Error::schema(format!( - "{data_type} is not a fixed stride type" - ))); - } - // TODO: support more than plain encoding here. - let decoder = PlainDecoder::new(reader, data_type, position, length)?; - decoder.get(params.into()).await -} - /// Read a protobuf message at file position 'pos'. /// /// We write protobuf by first writing the length of the message as a u32, diff --git a/rust/lance-io/src/utils/tracking_store.rs b/rust/lance-io/src/utils/tracking_store.rs index 588ff8f71b3..70306192018 100644 --- a/rust/lance-io/src/utils/tracking_store.rs +++ b/rust/lance-io/src/utils/tracking_store.rs @@ -13,6 +13,8 @@ use std::ops::Range; #[cfg(feature = "test-util")] use std::sync::atomic::AtomicU16; use std::sync::{Arc, Mutex}; +#[cfg(feature = "metrics")] +use std::time::Instant; use bytes::Bytes; use futures::StreamExt; @@ -26,9 +28,19 @@ use object_store::{ }; use crate::object_store::WrappingObjectStore; +#[cfg(feature = "metrics")] +use crate::object_store::metrics::{InFlightGuard, record_outcome}; #[derive(Debug, Default, Clone)] -pub struct IOTracker(Arc>); +pub struct IOTracker { + stats: Arc>, + /// The `base` label for the object store metrics published by IO that + /// bypasses the `object_store` layer (see [`Self::begin_io`]). `None` when + /// the IO cannot be attributed to a store, in which case no metrics are + /// published. + #[cfg(feature = "metrics")] + metrics_base: Option>, +} impl IOTracker { /// Get IO statistics and reset the counters (incremental pattern). @@ -36,7 +48,7 @@ impl IOTracker { /// This returns the accumulated statistics since the last call and resets /// the internal counters to zero. pub fn incremental_stats(&self) -> IoStats { - std::mem::take(&mut *self.0.lock().unwrap()) + std::mem::take(&mut *self.stats.lock().unwrap()) } /// Get a snapshot of current IO statistics without resetting counters. @@ -44,7 +56,7 @@ impl IOTracker { /// This returns a clone of the current statistics without modifying the /// internal state. Use this when you need to check stats without resetting. pub fn stats(&self) -> IoStats { - self.0.lock().unwrap().clone() + self.stats.lock().unwrap().clone() } /// Record a read operation for tracking. @@ -58,7 +70,7 @@ impl IOTracker { num_bytes: u64, #[allow(unused_variables)] range: Option>, ) { - let mut stats = self.0.lock().unwrap(); + let mut stats = self.stats.lock().unwrap(); stats.read_iops += 1; stats.read_bytes += num_bytes; #[cfg(feature = "test-util")] @@ -79,7 +91,7 @@ impl IOTracker { #[allow(unused_variables)] path: Path, num_bytes: u64, ) { - let mut stats = self.0.lock().unwrap(); + let mut stats = self.stats.lock().unwrap(); stats.write_iops += 1; stats.written_bytes += num_bytes; #[cfg(feature = "test-util")] @@ -89,11 +101,87 @@ impl IOTracker { range: None, }); } + + /// Label the metrics published through [`Self::begin_io`] with the prefix of + /// the store this tracker belongs to, so IO that bypasses the `object_store` + /// layer carries the same `base` label as the store's metered operations. + /// + /// Only `meter_store` should call this, so that labelling the tracker and + /// wrapping the store stay inseparable — see the rationale there. + #[cfg(feature = "metrics")] + pub(crate) fn set_metrics_base(&mut self, base: &str) { + self.metrics_base = Some(base.into()); + } + + /// Begin an operation that talks to storage without going through the + /// `object_store` layer, and so is invisible to the `MeteredObjectStore` + /// wrapper: the optimized local reads and writes go straight to the + /// filesystem. `operation` must be one of the labels that wrapper uses + /// (`get`, `put`, `head`, ...) so this IO aggregates with the rest. + /// + /// The returned guard keeps the in-flight gauge raised until it is dropped. + #[cfg(feature = "metrics")] + pub fn begin_io(&self, operation: &'static str) -> IoMetricsGuard { + IoMetricsGuard { + state: self.metrics_base.as_ref().map(|base| IoMetricsState { + _in_flight: InFlightGuard::new(base, operation), + base: base.clone(), + operation, + start: Instant::now(), + }), + } + } + + /// Without the `metrics` feature there is nothing to publish. + #[cfg(not(feature = "metrics"))] + pub fn begin_io(&self, _operation: &'static str) -> IoMetricsGuard { + IoMetricsGuard {} + } +} + +/// Publishes the object store metrics for a single operation that bypassed the +/// `object_store` layer (see [`IOTracker::begin_io`]). +/// +/// The operation is only counted by [`Self::record`]; one dropped before that — +/// a cancelled read, an abandoned write — counts as neither a success nor a +/// failure, and only lowers the in-flight gauge. +#[must_use = "the operation is not recorded until `record` is called"] +pub struct IoMetricsGuard { + #[cfg(feature = "metrics")] + state: Option, +} + +#[cfg(feature = "metrics")] +struct IoMetricsState { + base: Arc, + operation: &'static str, + start: Instant, + /// Lowers the in-flight gauge when the guard is dropped. + _in_flight: InFlightGuard, +} + +impl IoMetricsGuard { + /// Record the operation's count and latency, along with `num_bytes` + /// transferred if `result` is `Ok` or an error if it is not. + pub fn record(self, result: &std::result::Result, num_bytes: u64) { + #[cfg(feature = "metrics")] + if let Some(state) = self.state { + record_outcome( + &state.base, + state.operation, + state.start, + num_bytes, + result.is_err(), + ); + } + #[cfg(not(feature = "metrics"))] + let _ = (result, num_bytes); + } } impl WrappingObjectStore for IOTracker { fn wrap(&self, _store_prefix: &str, target: Arc) -> Arc { - Arc::new(IoTrackingStore::new(target, self.0.clone())) + Arc::new(IoTrackingStore::new(target, self.stats.clone())) } } diff --git a/rust/lance-io/tests/goosefs_integration.rs b/rust/lance-io/tests/goosefs_integration.rs index fd9015bcda0..0cd18d65c3c 100644 --- a/rust/lance-io/tests/goosefs_integration.rs +++ b/rust/lance-io/tests/goosefs_integration.rs @@ -25,7 +25,7 @@ fn get_operator() -> Operator { cfg.insert("master_addr".to_string(), addr); cfg.insert("root".to_string(), "/lance-test/opendal".to_string()); cfg.insert("auth_type".to_string(), auth_type); - Operator::from_iter::(cfg).unwrap().finish() + Operator::from_iter::(cfg).unwrap() } // ============================================================ @@ -192,6 +192,17 @@ async fn test_diag_lance_io_write_modes() { .await .expect("Failed to create ObjectStore"); + // Best-effort cleanup of leftover test files. object_store_opendal maps + // relative paths to the GooseFS root, so prior runs leave files at / + // even when the ObjectStore URI uses a per-run subdirectory. The + // concurrent exactly-one-winner race now lives in its own + // `test_diag_concurrent_put_create_exactly_one_winner` test, which + // uses a per-run filename so it does not share state with this test. + for name in ["test_file.txt", "test_create.txt"].iter() { + let p = object_store::path::Path::parse(name).unwrap(); + let _ = object_store.inner.delete(&p).await; + } + // Test 1: Basic put + get let test_path = object_store::path::Path::parse("test_file.txt").unwrap(); let test_data = bytes::Bytes::from("Hello from lance-io ObjectStore!"); @@ -224,12 +235,14 @@ async fn test_diag_lance_io_write_modes() { Err(e) => eprintln!("[DIAG] Read FAILED: {:?}", e), } - // Test 2: PutMode::Create (if_not_exists) + // PutMode::Create (if_not_exists) — required by + // ConditionalPutCommitHandler for concurrent-safe manifest commits. + let create_path = object_store::path::Path::parse("test_create.txt").unwrap(); eprintln!("[DIAG] Writing with PutMode::Create (if_not_exists)..."); - match object_store + object_store .inner .put_opts( - &object_store::path::Path::parse("test_create.txt").unwrap(), + &create_path, bytes::Bytes::from("conditional write!").into(), object_store::PutOptions { mode: object_store::PutMode::Create, @@ -237,12 +250,30 @@ async fn test_diag_lance_io_write_modes() { }, ) .await - { - Ok(_) => eprintln!("[DIAG] PutMode::Create succeeded! ✅"), - Err(e) => { - eprintln!("[DIAG] PutMode::Create FAILED: {:?}", e); - } - } + .expect("PutMode::Create should succeed for a new path"); + eprintln!("[DIAG] PutMode::Create succeeded! ✅"); + + eprintln!("[DIAG] Second PutMode::Create on same path (expect AlreadyExists)..."); + let conflict = object_store + .inner + .put_opts( + &create_path, + bytes::Bytes::from("should not overwrite").into(), + object_store::PutOptions { + mode: object_store::PutMode::Create, + ..Default::default() + }, + ) + .await; + assert!( + matches!( + conflict, + Err(object_store::Error::AlreadyExists { .. }) + | Err(object_store::Error::Precondition { .. }) + ), + "second PutMode::Create must fail with AlreadyExists/Precondition, got: {conflict:?}" + ); + eprintln!("[DIAG] PutMode::Create conflict correctly rejected! ✅"); // Test 3: rename_if_not_exists eprintln!("[DIAG] Testing rename_if_not_exists..."); @@ -269,3 +300,186 @@ async fn test_diag_lance_io_write_modes() { eprintln!("[DIAG] lance-io direct write test complete ✅"); } + +/// Concurrency regression: two `PutMode::Create` writers racing on the +/// same fresh path must produce exactly one winner, and the stored bytes +/// must match the winner's payload byte-for-byte. This is the property +/// that makes `ConditionalPutCommitHandler` safe under concurrent +/// manifest commits: every writer either commits its manifest or observes +/// the loser's precondition failure and retries against a fresh version. +/// +/// Split out of `test_diag_lance_io_write_modes` so the race assertions +/// cannot be silently skipped — the basic-write `return` on error in the +/// diagnostic test no longer covers the race, and the race is now a +/// first-class ignored test with its own setup/teardown invariants: +/// +/// 1. **Per-run path.** The race filename embeds a nanosecond timestamp so +/// concurrent or back-to-back runs in the same process never share +/// state. `object_store_opendal` maps the relative filename to the +/// GooseFS root regardless of the ObjectStore URI path, so a +/// per-run URI subdirectory would not isolate runs on its own. +/// 2. **Explicit pre-cleanup.** A leftover from a prior crashed run is +/// expected (`NotFound` is the only accepted pre-cleanup error); any +/// other error fails the test rather than being silently ignored. +/// 3. **Strict setup.** Every setup step uses `expect` so a transient +/// `ObjectStore` construction or filename-parse error is surfaced +/// rather than logged and skipped. +/// 4. **Post-cleanup.** The winning path is removed on success; cleanup +/// must report `Ok` or `NotFound`. +/// +/// Requires a live GooseFS cluster. +#[tokio::test] +#[ignore = "Requires GooseFS cluster"] +async fn test_diag_concurrent_put_create_exactly_one_winner() { + let addr = std::env::var("GOOSEFS_MASTER_ADDR").unwrap_or_else(|_| "127.0.0.1:9200".into()); + // Nanosecond timestamp gives effectively unique filenames even for + // back-to-back runs in the same process. The race path is *not* put + // under the per-run URI subdirectory because `object_store_opendal` + // ignores the URL path when resolving relative keys. + let ts = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_nanos(); + let race_filename = format!("test_create_race_{}.txt", ts); + let root = format!("goosefs://{}/lance-test", addr); + eprintln!( + "[DIAG] Creating ObjectStore at: {} (race path: {})", + root, race_filename + ); + + let params = ObjectStoreParams::default(); + let registry = Arc::new(ObjectStoreRegistry::default()); + let (object_store, _path) = ObjectStore::from_uri_and_params(registry.clone(), &root, ¶ms) + .await + .expect("Failed to create ObjectStore"); + + let race_path = object_store::path::Path::parse(&race_filename).unwrap(); + + // Pre-flight cleanup: the race path is per-run, so a leftover can + // only come from a prior run that crashed before its post-cleanup + // completed. `Ok` and `NotFound` are both expected; any other error + // is a setup failure that would otherwise silently skip the race. + match object_store.inner.delete(&race_path).await { + Ok(_) => {} + Err(object_store::Error::NotFound { .. }) => {} + Err(e) => panic!("pre-cleanup of race path {race_filename} failed: {e:?}"), + } + + let store_a = object_store.clone(); + let store_b = object_store.clone(); + let path_a = race_path.clone(); + let path_b = race_path.clone(); + let payload_a = bytes::Bytes::from("writer-A"); + let payload_b = bytes::Bytes::from("writer-B"); + + eprintln!("[DIAG] Launching two concurrent PutMode::Create on fresh path..."); + let fut_a = tokio::spawn(async move { + store_a + .inner + .put_opts( + &path_a, + payload_a.clone().into(), + object_store::PutOptions { + mode: object_store::PutMode::Create, + ..Default::default() + }, + ) + .await + .map(|_| payload_a) + }); + let fut_b = tokio::spawn(async move { + store_b + .inner + .put_opts( + &path_b, + payload_b.clone().into(), + object_store::PutOptions { + mode: object_store::PutMode::Create, + ..Default::default() + }, + ) + .await + .map(|_| payload_b) + }); + let (res_a, res_b) = tokio::join!(fut_a, fut_b); + let res_a = res_a.expect("writer A join"); + let res_b = res_b.expect("writer B join"); + + let mut wins = 0usize; + let mut conflicts = 0usize; + let mut other_err: Option = None; + let mut winner_payload: Option = None; + match (&res_a, &res_b) { + (Ok(p), Err(_)) => { + wins = 1; + conflicts = 1; + winner_payload = Some(p.clone()); + } + (Err(_), Ok(p)) => { + wins = 1; + conflicts = 1; + winner_payload = Some(p.clone()); + } + (Ok(_), Ok(_)) => { + other_err = Some("both writers reported success — if-not-exists violated".into()); + } + (Err(ea), Err(eb)) => { + other_err = Some(format!("both writers failed: a={ea:?} b={eb:?}")); + } + } + for r in [&res_a, &res_b] { + if let Err(e) = r { + let s = format!("{e:?}").to_lowercase(); + assert!( + s.contains("already exists") + || s.contains("precondition") + || s.contains("conditionnotmatch") + || s.contains("if_not_exists"), + "loser must report AlreadyExists/Precondition, got: {e:?}" + ); + } + } + assert!( + other_err.is_none(), + "exactly-one-winner violated: {}", + other_err.unwrap() + ); + assert_eq!(wins, 1, "exactly one writer must win (got {wins})"); + assert_eq!( + conflicts, 1, + "exactly one writer must conflict (got {conflicts})" + ); + + // Read back — stored bytes must match the winner's payload exactly. + let stored = object_store + .inner + .get(&race_path) + .await + .expect("get winning payload") + .bytes() + .await + .expect("read winning payload"); + let expected = winner_payload.expect("winner payload recorded"); + assert_eq!( + stored, expected, + "stored bytes must match the winner's payload" + ); + eprintln!( + "[DIAG] concurrent exactly-one-winner ok ✅ stored={} bytes", + stored.len() + ); + + // Post-flight cleanup: must succeed. `NotFound` after a winning write + // would mean the file never actually persisted, contradicting the + // assertion above, so we surface it as a failure rather than masking + // a real regression. + match object_store.inner.delete(&race_path).await { + Ok(_) => {} + Err(object_store::Error::NotFound { .. }) => { + panic!( + "post-cleanup of race path {race_filename} reported NotFound after a winning write" + ) + } + Err(e) => panic!("post-cleanup of race path {race_filename} failed: {e:?}"), + } +} diff --git a/rust/lance-linalg/Cargo.toml b/rust/lance-linalg/Cargo.toml index 6a188ec3c62..e7d6c7aab5d 100644 --- a/rust/lance-linalg/Cargo.toml +++ b/rust/lance-linalg/Cargo.toml @@ -25,6 +25,7 @@ approx = { workspace = true } criterion = { workspace = true } lance-testing = { path = "../lance-testing" } proptest.workspace = true +rstest.workspace = true [build-dependencies] cc = "1.0.83" @@ -59,5 +60,9 @@ harness = false name = "dist_table" harness = false +[[bench]] +name = "batch_distance" +harness = false + [lints] workspace = true diff --git a/rust/lance-linalg/README.md b/rust/lance-linalg/README.md index 06db3e9684e..c436da15617 100644 --- a/rust/lance-linalg/README.md +++ b/rust/lance-linalg/README.md @@ -1,6 +1,6 @@ # Lance Linear Algebra Library -`lance-linalg` is a internal sub-crate, containing [Apache-Arrow](https://github.com/apache/arrow-rs) -native linear algebra algorithm used the [Lance](https://github.com/lancedb/lance). +`lance-linalg` is an internal sub-crate, containing [Apache-Arrow](https://github.com/apache/arrow-rs) +native linear algebra algorithms used by [Lance](https://github.com/lance-format/lance). diff --git a/rust/lance-linalg/benches/batch_distance.rs b/rust/lance-linalg/benches/batch_distance.rs new file mode 100644 index 00000000000..d4921b1a80c --- /dev/null +++ b/rust/lance-linalg/benches/batch_distance.rs @@ -0,0 +1,61 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright The Lance Authors + +use std::hint::black_box; +use std::time::Duration; + +use criterion::{Criterion, criterion_group, criterion_main}; +use lance_linalg::distance::{Cosine, Dot, L2}; + +const DIMENSION: usize = 8; +const TOTAL_VALUES: usize = 1024 * 1024; + +fn bench_batch_distance(c: &mut Criterion) { + let key = (0..DIMENSION) + .map(|index| index as f32 * 0.125 + 0.25) + .collect::>(); + let batch = (0..TOTAL_VALUES) + .map(|index| (index % 31) as f32 * 0.03125 - 0.5) + .collect::>(); + + c.bench_function("Batch distance dot f32 dim 8", |b| { + b.iter(|| { + black_box( + ::dot_batch(black_box(&key), black_box(&batch), DIMENSION).sum::(), + ) + }) + }); + c.bench_function("Batch distance l2 f32 dim 8", |b| { + b.iter(|| { + black_box( + ::l2_batch(black_box(&key), black_box(&batch), DIMENSION).sum::(), + ) + }) + }); + c.bench_function("Batch distance cosine f32 dim 8", |b| { + b.iter(|| { + black_box( + ::cosine_batch(black_box(&key), black_box(&batch), DIMENSION) + .sum::(), + ) + }) + }); +} + +fn bench_time() -> Duration { + let seconds = option_env!("TARGET_TIME") + .unwrap_or("5") + .parse() + .expect("TARGET_TIME must be an integer number of seconds"); + Duration::from_secs(seconds) +} + +criterion_group!( + name = benches; + config = Criterion::default() + .significance_level(0.1) + .sample_size(10) + .measurement_time(bench_time()); + targets = bench_batch_distance +); +criterion_main!(benches); diff --git a/rust/lance-linalg/proptest-regressions/distance/cosine.txt b/rust/lance-linalg/proptest-regressions/distance/cosine.txt new file mode 100644 index 00000000000..04e6dcb967f --- /dev/null +++ b/rust/lance-linalg/proptest-regressions/distance/cosine.txt @@ -0,0 +1,7 @@ +# Seeds for failure cases proptest has generated in the past. It is +# automatically read and these particular cases re-run before any +# novel cases are generated. +# +# It is recommended to check this file in to source control so that +# everyone who runs the test benefits from these saved cases. +cc 679c764b02e2726ec2d943b3abdc7d7d3565e168d4081b155860c1e69b616fce # shrinks to (x, y) = ([-1.2933521e-15, -1.1194652e-8, 7.447341e-32, -0.0, -14247490.0, -8.55e-43, -4576.872, 0.009181444], [4.926e-42, -9.267065e-15, -0.0, 7e-45, -0.00036999694, -0.0, -5.7810876e-8, 1.9273644e-21]) diff --git a/rust/lance-linalg/src/distance.rs b/rust/lance-linalg/src/distance.rs index 23d1cae2d63..a76157bd112 100644 --- a/rust/lance-linalg/src/distance.rs +++ b/rust/lance-linalg/src/distance.rs @@ -25,12 +25,218 @@ pub mod l2; pub mod l2_u8; pub mod norm_l2; +/// Number of distances computed per call into a runtime-selected batch kernel. +/// +/// Keeping a small output buffer amortizes the `#[target_feature]` call while +/// avoiding the allocation and full-batch materialization that dominate the +/// common dimension-8 case. +#[cfg(all( + target_arch = "x86_64", + not(all(target_feature = "avx2", target_feature = "fma")) +))] +const BATCH_BUFFER_SIZE: usize = 64; + +#[cfg(all( + target_arch = "x86_64", + not(all(target_feature = "avx2", target_feature = "fma")) +))] +pub(crate) type BatchKernel = unsafe fn(&[f32], &[f32], usize, &mut [f32]); + +/// Runtime-selected target-feature tier for a batch kernel. +#[cfg(all( + target_arch = "x86_64", + not(all(target_feature = "avx2", target_feature = "fma")) +))] +#[derive(Clone, Copy)] +pub(crate) enum BatchKind { + Scalar, + Avx, + AvxFma, + Avx512, +} + +/// Per-vector operations used when a batch consumer can be folded directly. +#[cfg(all( + target_arch = "x86_64", + not(all(target_feature = "avx2", target_feature = "fma")) +))] +pub(crate) trait BatchOperation { + fn fold_scalar(key: &[f32], batch: &[f32], dimension: usize, init: B, f: F) -> B + where + F: FnMut(B, f32) -> B; + + unsafe fn fold_avx(key: &[f32], batch: &[f32], dimension: usize, init: B, f: F) -> B + where + F: FnMut(B, f32) -> B; + + unsafe fn fold_avx_fma(key: &[f32], batch: &[f32], dimension: usize, init: B, f: F) -> B + where + F: FnMut(B, f32) -> B; + + unsafe fn fold_avx512(key: &[f32], batch: &[f32], dimension: usize, init: B, f: F) -> B + where + F: FnMut(B, f32) -> B; +} + +/// Allocation-free iterator over a runtime-selected f32 distance kernel. +#[cfg(all( + target_arch = "x86_64", + not(all(target_feature = "avx2", target_feature = "fma")) +))] +pub(crate) struct BatchIter<'a, O> { + key: &'a [f32], + batch: &'a [f32], + dimension: usize, + kernel: BatchKernel, + kind: BatchKind, + buffer: [f32; BATCH_BUFFER_SIZE], + buffer_index: usize, + buffer_len: usize, + operation: std::marker::PhantomData, +} + +#[cfg(all( + target_arch = "x86_64", + not(all(target_feature = "avx2", target_feature = "fma")) +))] +impl<'a, O> BatchIter<'a, O> { + /// Creates an iterator after the caller has verified `kernel`'s CPU feature + /// requirements. + /// + /// # Safety + /// The host must support every target feature required by `kernel`. + #[inline] + pub(crate) unsafe fn new( + key: &'a [f32], + batch: &'a [f32], + dimension: usize, + kernel: BatchKernel, + kind: BatchKind, + ) -> Self { + // Match `chunks_exact` validation before the buffered iterator performs + // division by the dimension. + let _ = batch.chunks_exact(dimension); + Self { + key, + batch, + dimension, + kernel, + kind, + buffer: [0.0; BATCH_BUFFER_SIZE], + buffer_index: 0, + buffer_len: 0, + operation: std::marker::PhantomData, + } + } + + #[inline] + fn refill(&mut self) -> bool { + let num_vectors = (self.batch.len() / self.dimension).min(BATCH_BUFFER_SIZE); + if num_vectors == 0 { + return false; + } + + let num_values = num_vectors * self.dimension; + let (input, remaining) = self.batch.split_at(num_values); + // SAFETY: `new` requires the caller to verify the selected kernel's + // target features before constructing the iterator. + unsafe { + (self.kernel)( + self.key, + input, + self.dimension, + &mut self.buffer[..num_vectors], + ); + } + self.batch = remaining; + self.buffer_index = 0; + self.buffer_len = num_vectors; + true + } +} + +#[cfg(all( + target_arch = "x86_64", + not(all(target_feature = "avx2", target_feature = "fma")) +))] +impl Iterator for BatchIter<'_, O> { + type Item = f32; + + #[inline] + fn next(&mut self) -> Option { + if self.buffer_index == self.buffer_len && !self.refill() { + return None; + } + let value = self.buffer[self.buffer_index]; + self.buffer_index += 1; + Some(value) + } + + #[inline] + fn size_hint(&self) -> (usize, Option) { + let len = self.len(); + (len, Some(len)) + } + + /// Processes each filled buffer directly so consumers such as `sum` do not + /// pay a refill check for every distance. + #[inline] + fn fold(self, init: B, mut f: F) -> B + where + F: FnMut(B, Self::Item) -> B, + { + let accumulator = self.buffer[self.buffer_index..self.buffer_len] + .iter() + .copied() + .fold(init, &mut f); + + // SAFETY: `new` requires the caller to verify the selected tier's + // target features. Each helper runs the complete remaining loop in + // that target-feature context, avoiding intermediate output writes. + match self.kind { + BatchKind::Scalar => { + O::fold_scalar(self.key, self.batch, self.dimension, accumulator, f) + } + BatchKind::Avx => unsafe { + O::fold_avx(self.key, self.batch, self.dimension, accumulator, f) + }, + BatchKind::AvxFma => unsafe { + O::fold_avx_fma(self.key, self.batch, self.dimension, accumulator, f) + }, + BatchKind::Avx512 => unsafe { + O::fold_avx512(self.key, self.batch, self.dimension, accumulator, f) + }, + } + } + + #[inline] + fn for_each(self, mut f: F) + where + F: FnMut(Self::Item), + { + self.fold((), |(), value| f(value)); + } +} + +#[cfg(all( + target_arch = "x86_64", + not(all(target_feature = "avx2", target_feature = "fma")) +))] +impl ExactSizeIterator for BatchIter<'_, O> { + #[inline] + fn len(&self) -> usize { + self.buffer_len - self.buffer_index + self.batch.len() / self.dimension + } +} + pub use cosine::*; pub use dot::*; pub use hamming::{ - Cluster, ClusteringResult, PairwiseResult, UnionFind, cluster_edges, cluster_pairwise_result, - extract_hashes_from_fixed_list, hamming_distance_arrow_batch, hamming_u64, - pairwise_hamming_distance, pairwise_hamming_distance_parallel, + BinaryHashValues, Cluster, ClusteringResult, PairwiseResult, UnionFind, cluster_edges, + cluster_pairwise_result, extract_binary_hashes_from_fixed_list, extract_hashes_from_fixed_list, + hamming_distance_arrow_batch, hamming_u64, pairwise_hamming_distance, + pairwise_hamming_distance_binary, pairwise_hamming_distance_binary_parallel, + pairwise_hamming_distance_parallel, }; pub use l2::*; use lance_core::deepsize::DeepSizeOf; diff --git a/rust/lance-linalg/src/distance/cosine.rs b/rust/lance-linalg/src/distance/cosine.rs index 995191b77eb..432d43921b6 100644 --- a/rust/lance-linalg/src/distance/cosine.rs +++ b/rust/lance-linalg/src/distance/cosine.rs @@ -7,6 +7,10 @@ //! //! `bf16, f16, f32, f64` types are supported. +#[cfg(all(target_arch = "x86_64", not(target_feature = "avx2")))] +use std::arch::x86_64::{ + _mm_add_ps, _mm_add_ss, _mm_cvtss_f32, _mm_loadu_ps, _mm_movehl_ps, _mm_mul_ps, _mm_shuffle_ps, +}; use std::sync::Arc; use arrow_array::{ @@ -17,12 +21,14 @@ use arrow_array::{ use arrow_schema::DataType; use half::{bf16, f16}; use lance_arrow::{ArrowFloatType, FixedSizeListArrayExt, FloatArray}; -use lance_core::utils::cpu::SIMD_SUPPORT; -#[cfg(feature = "fp16kernels")] -use lance_core::utils::cpu::SimdSupport; +#[allow(unused_imports)] +use lance_core::utils::cpu::{SIMD_SUPPORT, SimdSupport}; use super::{Dot, norm_l2::norm_l2}; use super::{Normalize, dot::dot}; +#[cfg(all(target_arch = "x86_64", not(target_feature = "avx2")))] +use crate::distance::BatchKind; +#[allow(unused_imports)] use crate::simd::{ FloatSimd, SIMD, f32::{f32x8, f32x16}, @@ -127,6 +133,10 @@ impl Cosine for bf16 { SimdSupport::Lsx => unsafe { bf16_kernel::cosine_bf16_lsx(x.as_ptr(), x_norm, y.as_ptr(), y.len() as u32) }, + // SimdSupport::AvxFma and SimdSupport::Avx fall through here: + // the bf16 C kernels in `bf16_kernel::*` are compiled with + // `-march=haswell` minimum (which requires AVX2), so they cannot + // run on AVX-only or AVX+FMA hosts. Scalar is the correct route. _ => cosine_scalar(x, x_norm, y), } } @@ -168,7 +178,7 @@ impl Cosine for f16 { kernel::cosine_f16_avx512(x.as_ptr(), x_norm, y.as_ptr(), y.len() as u32) }, #[cfg(all(feature = "fp16kernels", target_arch = "x86_64"))] - SimdSupport::Avx2 => unsafe { + SimdSupport::Avx2 | SimdSupport::Avx512 => unsafe { kernel::cosine_f16_avx2(x.as_ptr(), x_norm, y.as_ptr(), y.len() as u32) }, #[cfg(all(feature = "fp16kernels", target_arch = "loongarch64"))] @@ -179,86 +189,394 @@ impl Cosine for f16 { SimdSupport::Lsx => unsafe { kernel::cosine_f16_lsx(x.as_ptr(), x_norm, y.as_ptr(), y.len() as u32) }, + // SimdSupport::AvxFma and SimdSupport::Avx fall through here: + // the f16 C kernels are compiled with `-march=haswell` minimum + // (AVX2), so they cannot run on AVX-only or AVX+FMA hosts. _ => cosine_scalar(x, x_norm, y), } } } -/// f32 kernels for Cosine +/// f32 single-vector cosine helpers used by `cosine_batch` for fixed +/// dimensions 8 and 16. +/// +/// These were previously a single generic `cosine_once` but the +/// monomorphizations have to dispatch on `SIMD_SUPPORT` for the SIMD path +/// to stay correct under any compile baseline. Splitting them into two +/// concrete entry points keeps the dispatch site flat and lets each width +/// route to a `#[target_feature]` AVX2 inner function. mod f32 { use super::*; - // TODO: how can we explicitly infer N? + #[cfg(any(test, not(target_arch = "x86_64")))] #[inline] - pub(super) fn cosine_once, const N: usize>( - x: &[f32], - x_norm: f32, - y: &[f32], - ) -> f32 { - let x = unsafe { S::load_unaligned(x.as_ptr()) }; - let y = unsafe { S::load_unaligned(y.as_ptr()) }; - let y2 = y * y; - let xy = x * y; + pub(super) fn cosine_once_8(x: &[f32], x_norm: f32, y: &[f32]) -> f32 { + #[cfg(target_arch = "x86_64")] + { + match *SIMD_SUPPORT { + SimdSupport::Avx512 | SimdSupport::Avx512FP16 => unsafe { + cosine_once_x86::cosine_once_8_avx512(x, x_norm, y) + }, + SimdSupport::Avx2 | SimdSupport::AvxFma => unsafe { + cosine_once_x86::cosine_once_8_avx_fma(x, x_norm, y) + }, + SimdSupport::Avx => unsafe { cosine_once_x86::cosine_once_8_avx(x, x_norm, y) }, + _ => cosine_once_8_scalar(x, x_norm, y), + } + } + #[cfg(not(target_arch = "x86_64"))] + { + cosine_once_8_other(x, x_norm, y) + } + } + + #[cfg(any(test, not(target_arch = "x86_64")))] + #[inline] + pub(super) fn cosine_once_16(x: &[f32], x_norm: f32, y: &[f32]) -> f32 { + #[cfg(target_arch = "x86_64")] + { + match *SIMD_SUPPORT { + SimdSupport::Avx512 | SimdSupport::Avx512FP16 => unsafe { + cosine_once_x86::cosine_once_16_avx512(x, x_norm, y) + }, + SimdSupport::Avx2 | SimdSupport::AvxFma => unsafe { + cosine_once_x86::cosine_once_16_avx_fma(x, x_norm, y) + }, + SimdSupport::Avx => unsafe { cosine_once_x86::cosine_once_16_avx(x, x_norm, y) }, + _ => cosine_once_16_scalar(x, x_norm, y), + } + } + #[cfg(not(target_arch = "x86_64"))] + { + cosine_once_16_other(x, x_norm, y) + } + } + + /// Portable scalar `cosine_once` for length-8 vectors. Matches the SIMD + /// path modulo summation order. + #[cfg(all(target_arch = "x86_64", test))] + #[inline] + pub(super) fn cosine_once_8_scalar(x: &[f32], x_norm: f32, y: &[f32]) -> f32 { + let mut xy = 0.0f32; + let mut y2 = 0.0f32; + for i in 0..8 { + xy += x[i] * y[i]; + y2 += y[i] * y[i]; + } + 1.0 - xy / x_norm / y2.sqrt() + } + + #[cfg(all(target_arch = "x86_64", any(test, not(target_feature = "avx2"))))] + #[inline] + pub(super) fn cosine_once_16_scalar(x: &[f32], x_norm: f32, y: &[f32]) -> f32 { + let mut xy = 0.0f32; + let mut y2 = 0.0f32; + for i in 0..16 { + xy += x[i] * y[i]; + y2 += y[i] * y[i]; + } + 1.0 - xy / x_norm / y2.sqrt() + } + + #[cfg(target_arch = "x86_64")] + pub(super) mod cosine_once_x86 { + use std::arch::x86_64::*; + + #[cfg(test)] + use super::f32x8; + #[cfg(any(test, not(target_feature = "avx2")))] + use super::f32x16; + #[cfg(any(test, not(target_feature = "avx2")))] + use crate::simd::SIMD; + + /// AVX + FMA path for 8-lane cosine. Covers both AvxFma and AVX2 dispatch (body uses no AVX2-specific intrinsics). + #[cfg(test)] + #[inline] + #[target_feature(enable = "avx,fma")] + pub unsafe fn cosine_once_8_avx_fma(x: &[f32], x_norm: f32, y: &[f32]) -> f32 { + let xv = f32x8::load_unaligned(x.as_ptr()); + let yv = f32x8::load_unaligned(y.as_ptr()); + let y2 = yv * yv; + let xy = xv * yv; + 1.0 - xy.reduce_sum() / x_norm / y2.reduce_sum().sqrt() + } + + /// AVX + FMA path for 16-lane cosine. Covers both AvxFma and AVX2 dispatch (body uses no AVX2-specific intrinsics). + #[cfg(any(test, not(target_feature = "avx2")))] + #[inline] + #[target_feature(enable = "avx,fma")] + pub unsafe fn cosine_once_16_avx_fma(x: &[f32], x_norm: f32, y: &[f32]) -> f32 { + let xv = f32x16::load_unaligned(x.as_ptr()); + let yv = f32x16::load_unaligned(y.as_ptr()); + let y2 = yv * yv; + let xy = xv * yv; + 1.0 - xy.reduce_sum() / x_norm / y2.reduce_sum().sqrt() + } + + /// AVX-only path for 8-lane cosine (no FMA): body unchanged from AVX2 path; gated on Sandy/Ivy Bridge. + #[cfg(test)] + #[inline] + #[target_feature(enable = "avx")] + pub unsafe fn cosine_once_8_avx(x: &[f32], x_norm: f32, y: &[f32]) -> f32 { + let xv = f32x8::load_unaligned(x.as_ptr()); + let yv = f32x8::load_unaligned(y.as_ptr()); + let y2 = yv * yv; + let xy = xv * yv; + 1.0 - xy.reduce_sum() / x_norm / y2.reduce_sum().sqrt() + } + + /// AVX-only path for 16-lane cosine (no FMA): body unchanged from AVX2 path; gated on Sandy/Ivy Bridge. + #[cfg(any(test, not(target_feature = "avx2")))] + #[inline] + #[target_feature(enable = "avx")] + pub unsafe fn cosine_once_16_avx(x: &[f32], x_norm: f32, y: &[f32]) -> f32 { + let xv = f32x16::load_unaligned(x.as_ptr()); + let yv = f32x16::load_unaligned(y.as_ptr()); + let y2 = yv * yv; + let xy = xv * yv; + 1.0 - xy.reduce_sum() / x_norm / y2.reduce_sum().sqrt() + } + + /// AVX-512 path for 8-lane cosine: masked load into a `__m512` lower half, reduce. + #[cfg(any(test, target_feature = "avx2"))] + #[inline] + #[target_feature(enable = "avx512f")] + pub unsafe fn cosine_once_8_avx512(x: &[f32], x_norm: f32, y: &[f32]) -> f32 { + // mask 0x00FF: load the lower 8 f32 lanes, zero the upper 8. + let mask: __mmask16 = 0x00FF; + let xv = _mm512_maskz_loadu_ps(mask, x.as_ptr()); + let yv = _mm512_maskz_loadu_ps(mask, y.as_ptr()); + let xy = _mm512_mul_ps(xv, yv); + let y2 = _mm512_mul_ps(yv, yv); + let xy_sum = _mm512_reduce_add_ps(xy); + let y2_sum = _mm512_reduce_add_ps(y2); + 1.0 - xy_sum / x_norm / y2_sum.sqrt() + } + + /// AVX-512 path for 16-lane cosine: single full-width `__m512` load (16 f32 fits one `zmm`). + #[inline] + #[target_feature(enable = "avx512f")] + pub unsafe fn cosine_once_16_avx512(x: &[f32], x_norm: f32, y: &[f32]) -> f32 { + let xv = _mm512_loadu_ps(x.as_ptr()); + let yv = _mm512_loadu_ps(y.as_ptr()); + let xy = _mm512_mul_ps(xv, yv); + let y2 = _mm512_mul_ps(yv, yv); + let xy_sum = _mm512_reduce_add_ps(xy); + let y2_sum = _mm512_reduce_add_ps(y2); + 1.0 - xy_sum / x_norm / y2_sum.sqrt() + } + } + + #[cfg(not(target_arch = "x86_64"))] + #[inline] + fn cosine_once_8_other(x: &[f32], x_norm: f32, y: &[f32]) -> f32 { + let xv = unsafe { f32x8::load_unaligned(x.as_ptr()) }; + let yv = unsafe { f32x8::load_unaligned(y.as_ptr()) }; + let y2 = yv * yv; + let xy = xv * yv; 1.0 - xy.reduce_sum() / x_norm / y2.reduce_sum().sqrt() } -} -impl Cosine for f32 { + #[cfg(not(target_arch = "x86_64"))] #[inline] - fn cosine_fast(x: &[Self], x_norm: Self, other: &[Self]) -> f32 { - let dim = x.len(); - let unrolled_len = dim / 16 * 16; - let mut y_norm16 = f32x16::zeros(); - let mut xy16 = f32x16::zeros(); - for i in (0..unrolled_len).step_by(16) { - unsafe { - let x = f32x16::load_unaligned(x.as_ptr().add(i)); - let y = f32x16::load_unaligned(other.as_ptr().add(i)); - xy16.multiply_add(x, y); - y_norm16.multiply_add(y, y); + fn cosine_once_16_other(x: &[f32], x_norm: f32, y: &[f32]) -> f32 { + let xv = unsafe { f32x16::load_unaligned(x.as_ptr()) }; + let yv = unsafe { f32x16::load_unaligned(y.as_ptr()) }; + let y2 = yv * yv; + let xy = xv * yv; + 1.0 - xy.reduce_sum() / x_norm / y2.reduce_sum().sqrt() + } + + /// Batch-level SIMD helper used for direct AVX/FMA parity tests. + #[cfg(all(target_arch = "x86_64", test))] + #[target_feature(enable = "avx,fma")] + pub(super) unsafe fn cosine_batch_avx_fma( + x: &[f32], + x_norm: f32, + batch: &[f32], + dimension: usize, + output: &mut [f32], + ) { + debug_assert_eq!(output.len(), batch.len() / dimension); + match dimension { + 8 => { + let x_values = unsafe { f32x8::load_unaligned(x.as_ptr()) }; + output + .iter_mut() + .zip(batch.chunks_exact(8)) + .for_each(|(distance, y)| { + let y_values = unsafe { f32x8::load_unaligned(y.as_ptr()) }; + let y2 = y_values * y_values; + let xy = x_values * y_values; + *distance = 1.0 - xy.reduce_sum() / x_norm / y2.reduce_sum().sqrt(); + }); } + 16 => output + .iter_mut() + .zip(batch.chunks_exact(16)) + .for_each(|(distance, y)| { + *distance = unsafe { cosine_once_x86::cosine_once_16_avx_fma(x, x_norm, y) }; + }), + _ => output + .iter_mut() + .zip(batch.chunks_exact(dimension)) + .for_each(|(distance, y)| { + *distance = unsafe { super::f32_x86::cosine_fast_avx_fma(x, x_norm, y) }; + }), } - let aligned_len = dim / 8 * 8; - let mut y_norm8 = f32x8::zeros(); - let mut xy8 = f32x8::zeros(); - for i in (unrolled_len..aligned_len).step_by(8) { - unsafe { - let x = f32x8::load_unaligned(x.as_ptr().add(i)); - let y = f32x8::load_unaligned(other.as_ptr().add(i)); - xy8.multiply_add(x, y); - y_norm8.multiply_add(y, y); + } + + #[cfg(all(target_arch = "x86_64", any(test, target_feature = "avx2")))] + #[target_feature(enable = "avx512f")] + pub(super) unsafe fn cosine_batch_avx512( + x: &[f32], + x_norm: f32, + batch: &[f32], + dimension: usize, + output: &mut [f32], + ) { + debug_assert_eq!(output.len(), batch.len() / dimension); + match dimension { + 8 => output + .iter_mut() + .zip(batch.chunks_exact(8)) + .for_each(|(distance, y)| { + *distance = unsafe { cosine_once_x86::cosine_once_8_avx512(x, x_norm, y) }; + }), + 16 => output + .iter_mut() + .zip(batch.chunks_exact(16)) + .for_each(|(distance, y)| { + *distance = unsafe { cosine_once_x86::cosine_once_16_avx512(x, x_norm, y) }; + }), + _ => output + .iter_mut() + .zip(batch.chunks_exact(dimension)) + .for_each(|(distance, y)| { + *distance = unsafe { super::f32_x86::cosine_fast_avx512(x, x_norm, y) }; + }), + } + } + + #[cfg(all(target_arch = "x86_64", test))] + #[target_feature(enable = "avx")] + pub(super) unsafe fn cosine_batch_avx( + x: &[f32], + x_norm: f32, + batch: &[f32], + dimension: usize, + output: &mut [f32], + ) { + debug_assert_eq!(output.len(), batch.len() / dimension); + match dimension { + 8 => { + let x_values = unsafe { f32x8::load_unaligned(x.as_ptr()) }; + output + .iter_mut() + .zip(batch.chunks_exact(8)) + .for_each(|(distance, y)| { + let y_values = unsafe { f32x8::load_unaligned(y.as_ptr()) }; + let y2 = y_values * y_values; + let xy = x_values * y_values; + *distance = 1.0 - xy.reduce_sum() / x_norm / y2.reduce_sum().sqrt(); + }); } + 16 => output + .iter_mut() + .zip(batch.chunks_exact(16)) + .for_each(|(distance, y)| { + *distance = unsafe { cosine_once_x86::cosine_once_16_avx(x, x_norm, y) }; + }), + _ => output + .iter_mut() + .zip(batch.chunks_exact(dimension)) + .for_each(|(distance, y)| { + *distance = unsafe { super::f32_x86::cosine_fast_avx(x, x_norm, y) }; + }), } - let y_norm = - y_norm16.reduce_sum() + y_norm8.reduce_sum() + norm_l2(&other[aligned_len..]).powi(2); - let xy = - xy16.reduce_sum() + xy8.reduce_sum() + dot(&x[aligned_len..], &other[aligned_len..]); - 1.0 - xy / x_norm / y_norm.sqrt() } +} + +/// Inlined f32 cosine kernels for builds whose baseline already guarantees +/// AVX2. No `#[target_feature]`, no runtime dispatch: under +/// `target-feature=+avx2,+fma` these compile to AVX2 and inline into the batch +/// loop, so explicitly tuned builds are not taxed by the runtime-dispatch +/// machinery needed below the AVX2 baseline. +#[cfg(all(target_arch = "x86_64", target_feature = "avx2"))] +mod f32_baseline { + use super::{dot, f32x8, f32x16, norm_l2}; + use crate::simd::{FloatSimd, SIMD}; #[inline] - fn cosine_with_norms(x: &[Self], x_norm: Self, y_norm: Self, y: &[Self]) -> Self { - let dim = x.len(); - let unrolled_len = dim / 16 * 16; - let mut xy16 = f32x16::zeros(); - for i in (0..unrolled_len).step_by(16) { - unsafe { - let x = f32x16::load_unaligned(x.as_ptr().add(i)); - let y = f32x16::load_unaligned(y.as_ptr().add(i)); - xy16.multiply_add(x, y); - } + pub fn cosine_once_8(x: &[f32], x_norm: f32, y: &[f32]) -> f32 { + unsafe { + let xv = f32x8::load_unaligned(x.as_ptr()); + let yv = f32x8::load_unaligned(y.as_ptr()); + let y2 = yv * yv; + let xy = xv * yv; + 1.0 - xy.reduce_sum() / x_norm / y2.reduce_sum().sqrt() } - let aligned_len = dim / 8 * 8; - let mut xy8 = f32x8::zeros(); - for i in (unrolled_len..aligned_len).step_by(8) { - unsafe { - let x = f32x8::load_unaligned(x.as_ptr().add(i)); - let y = f32x8::load_unaligned(y.as_ptr().add(i)); - xy8.multiply_add(x, y); + } + + #[inline] + pub fn cosine_once_16(x: &[f32], x_norm: f32, y: &[f32]) -> f32 { + unsafe { + let xv = f32x16::load_unaligned(x.as_ptr()); + let yv = f32x16::load_unaligned(y.as_ptr()); + let y2 = yv * yv; + let xy = xv * yv; + 1.0 - xy.reduce_sum() / x_norm / y2.reduce_sum().sqrt() + } + } + + #[inline] + pub fn cosine_fast(x: &[f32], x_norm: f32, other: &[f32]) -> f32 { + unsafe { + let dim = x.len(); + let unrolled_len = dim / 16 * 16; + let mut y_norm16 = f32x16::zeros(); + let mut xy16 = f32x16::zeros(); + for i in (0..unrolled_len).step_by(16) { + let xv = f32x16::load_unaligned(x.as_ptr().add(i)); + let yv = f32x16::load_unaligned(other.as_ptr().add(i)); + xy16.multiply_add(xv, yv); + y_norm16.multiply_add(yv, yv); } + let aligned_len = dim / 8 * 8; + let mut y_norm8 = f32x8::zeros(); + let mut xy8 = f32x8::zeros(); + for i in (unrolled_len..aligned_len).step_by(8) { + let xv = f32x8::load_unaligned(x.as_ptr().add(i)); + let yv = f32x8::load_unaligned(other.as_ptr().add(i)); + xy8.multiply_add(xv, yv); + y_norm8.multiply_add(yv, yv); + } + let y_norm = y_norm16.reduce_sum() + + y_norm8.reduce_sum() + + norm_l2(&other[aligned_len..]).powi(2); + let xy = xy16.reduce_sum() + + xy8.reduce_sum() + + dot(&x[aligned_len..], &other[aligned_len..]); + 1.0 - xy / x_norm / y_norm.sqrt() } - let xy = xy16.reduce_sum() + xy8.reduce_sum() + dot(&x[aligned_len..], &y[aligned_len..]); - 1.0 - xy / x_norm / y_norm + } +} + +impl Cosine for f32 { + #[inline] + fn cosine_fast(x: &[Self], x_norm: Self, other: &[Self]) -> f32 { + // Trait methods cannot carry `#[target_feature]` attributes, so the body + // lives in a free function that runtime-dispatches via `*SIMD_SUPPORT` + // to an AVX2 inner kernel on capable hosts, or a portable scalar fallback. + cosine_fast_f32_dispatched(x, x_norm, other) + } + + #[inline] + fn cosine_with_norms(x: &[Self], x_norm: Self, y_norm: Self, y: &[Self]) -> Self { + // Trait methods cannot carry `#[target_feature]` attributes, so the body + // lives in a free function that runtime-dispatches via `*SIMD_SUPPORT` + // to an AVX2 inner kernel on capable hosts, or a portable scalar fallback. + cosine_with_norms_f32_dispatched(x, x_norm, y_norm, y) } fn cosine_batch<'a>( @@ -266,59 +584,362 @@ impl Cosine for f32 { batch: &'a [Self], dimension: usize, ) -> Box + 'a> { + // Preserve the original `chunks_exact` validation before constructing + // a specialized iterator. + let _ = batch.chunks_exact(dimension); let x_norm = norm_l2(x); - match dimension { - 8 => Box::new( - batch - .chunks_exact(dimension) - .map(move |y| f32::cosine_once::(x, x_norm, y)), - ), - 16 => Box::new( - batch - .chunks_exact(dimension) - .map(move |y| f32::cosine_once::(x, x_norm, y)), - ), - _ => Box::new( - batch - .chunks_exact(dimension) - .map(move |y| Self::cosine_fast(x, x_norm, y)), - ), + // On a build whose baseline already guarantees AVX2, avoid the + // per-vector runtime dispatch + `#[target_feature]` wrapping that taxes + // the tuned path. Dispatch ONCE per batch: AVX-512 hosts get the wide + // kernel; everyone else uses the inlined AVX2 baseline path. The + // runtime-dispatch path below is only compiled/reached when the baseline + // is below AVX2. + #[cfg(all(target_arch = "x86_64", target_feature = "avx2"))] + { + // dim 8/16 always use the inlined AVX2 baseline: AVX-512 gives no + // benefit for such tiny vectors (a masked 512-bit load is slower than + // a plain AVX2 load) and only adds dispatch + eager-collect overhead. + // Only the larger-dim path routes to AVX-512 on capable hosts — that's + // where the wider lanes actually pay off. + match dimension { + 8 => Box::new( + batch + .chunks_exact(8) + .map(move |y| f32_baseline::cosine_once_8(x, x_norm, y)), + ), + 16 => Box::new( + batch + .chunks_exact(16) + .map(move |y| f32_baseline::cosine_once_16(x, x_norm, y)), + ), + _ => { + if matches!(*SIMD_SUPPORT, SimdSupport::Avx512 | SimdSupport::Avx512FP16) { + let mut distances = vec![0.0; batch.len() / dimension]; + unsafe { + f32::cosine_batch_avx512(x, x_norm, batch, dimension, &mut distances); + } + Box::new(distances.into_iter()) + } else { + Box::new( + batch + .chunks_exact(dimension) + .map(move |y| f32_baseline::cosine_fast(x, x_norm, y)), + ) + } + } + } + } + + // Sub-AVX2 build: select once. Dimension 8 uses an inlined SSE + // iterator; wider dimensions enter the selected target-feature kernel + // once per vector without materializing the full batch. + #[cfg(all(target_arch = "x86_64", not(target_feature = "avx2")))] + { + if dimension == 8 && x.len() >= 8 { + // SAFETY: both loads read from the verified eight-element key, + // and x86_64 guarantees SSE. + return Box::new(unsafe { CosineBatch8Iter::new(x, x_norm, batch) }); + } + + let kind = match *SIMD_SUPPORT { + SimdSupport::Avx512 | SimdSupport::Avx512FP16 if dimension > 16 => { + BatchKind::Avx512 + } + SimdSupport::Avx512 | SimdSupport::Avx512FP16 + if std::is_x86_feature_detected!("fma") => + { + BatchKind::AvxFma + } + SimdSupport::Avx2 | SimdSupport::AvxFma => BatchKind::AvxFma, + SimdSupport::Avx512 | SimdSupport::Avx512FP16 | SimdSupport::Avx => BatchKind::Avx, + _ => BatchKind::Scalar, + }; + Box::new(CosineBatchIter { + key: x, + key_norm: x_norm, + batch, + dimension, + offset: 0, + kind, + }) + } + + // Scalar / non-x86 fallback. + #[cfg(not(target_arch = "x86_64"))] + { + match dimension { + 8 => Box::new( + batch + .chunks_exact(dimension) + .map(move |y| f32::cosine_once_8(x, x_norm, y)), + ), + 16 => Box::new( + batch + .chunks_exact(dimension) + .map(move |y| f32::cosine_once_16(x, x_norm, y)), + ), + _ => Box::new( + batch + .chunks_exact(dimension) + .map(move |y| Self::cosine_fast(x, x_norm, y)), + ), + } } } } +#[cfg(all(target_arch = "x86_64", not(target_feature = "avx2")))] +struct CosineBatchIter<'a> { + key: &'a [f32], + key_norm: f32, + batch: &'a [f32], + dimension: usize, + offset: usize, + kind: BatchKind, +} + +#[cfg(all(target_arch = "x86_64", not(target_feature = "avx2")))] +struct CosineBatch8Iter<'a> { + key_lo: std::arch::x86_64::__m128, + key_hi: std::arch::x86_64::__m128, + key_norm: f32, + batch: &'a [f32], + offset: usize, +} + +#[cfg(all(target_arch = "x86_64", not(target_feature = "avx2")))] +impl<'a> CosineBatch8Iter<'a> { + /// # Safety + /// `key` must contain at least eight elements. + #[inline] + unsafe fn new(key: &[f32], key_norm: f32, batch: &'a [f32]) -> Self { + Self { + key_lo: unsafe { _mm_loadu_ps(key.as_ptr()) }, + key_hi: unsafe { _mm_loadu_ps(key.as_ptr().add(4)) }, + key_norm, + batch, + offset: 0, + } + } +} + +/// SSE implementation for the dimension-8 boxed iterator. SSE is guaranteed +/// on x86_64, so this inlines into `Iterator::next` without a target-feature +/// call boundary while wider kernels remain runtime-dispatched. +#[cfg(all(target_arch = "x86_64", not(target_feature = "avx2")))] +#[inline] +unsafe fn cosine_once_8_sse( + key_lo: std::arch::x86_64::__m128, + key_hi: std::arch::x86_64::__m128, + key_norm: f32, + vector: &[f32], +) -> f32 { + let vector_lo = unsafe { _mm_loadu_ps(vector.as_ptr()) }; + let vector_hi = unsafe { _mm_loadu_ps(vector.as_ptr().add(4)) }; + let xy = unsafe { _mm_add_ps(_mm_mul_ps(key_lo, vector_lo), _mm_mul_ps(key_hi, vector_hi)) }; + let y2 = unsafe { + _mm_add_ps( + _mm_mul_ps(vector_lo, vector_lo), + _mm_mul_ps(vector_hi, vector_hi), + ) + }; + 1.0 - unsafe { hsum128_ps(xy) } / key_norm / unsafe { hsum128_ps(y2) }.sqrt() +} + +#[cfg(all(target_arch = "x86_64", not(target_feature = "avx2")))] +impl Iterator for CosineBatch8Iter<'_> { + type Item = f32; + + #[inline] + fn next(&mut self) -> Option { + let end = self.offset + 8; + let vector = self.batch.get(self.offset..end)?; + self.offset = end; + // SAFETY: the constructor preloads a valid key and `vector` contains + // exactly eight elements. x86_64 guarantees SSE. + Some(unsafe { cosine_once_8_sse(self.key_lo, self.key_hi, self.key_norm, vector) }) + } + + #[inline] + fn size_hint(&self) -> (usize, Option) { + let len = self.len(); + (len, Some(len)) + } +} + +#[cfg(all(target_arch = "x86_64", not(target_feature = "avx2")))] +impl ExactSizeIterator for CosineBatch8Iter<'_> { + #[inline] + fn len(&self) -> usize { + (self.batch.len() - self.offset) / 8 + } +} + +#[cfg(all(target_arch = "x86_64", not(target_feature = "avx2")))] +#[inline] +unsafe fn hsum128_ps(values: std::arch::x86_64::__m128) -> f32 { + let sum64 = unsafe { _mm_add_ps(values, _mm_movehl_ps(values, values)) }; + let sum32 = unsafe { _mm_add_ss(sum64, _mm_shuffle_ps(sum64, sum64, 0x55)) }; + unsafe { _mm_cvtss_f32(sum32) } +} + +#[cfg(all(target_arch = "x86_64", not(target_feature = "avx2")))] +impl Iterator for CosineBatchIter<'_> { + type Item = f32; + + #[inline] + fn next(&mut self) -> Option { + let remaining = &self.batch[self.offset..]; + if remaining.len() < self.dimension { + return None; + } + let vector = &remaining[..self.dimension]; + self.offset += self.dimension; + + let distance = match self.kind { + BatchKind::Scalar => match self.dimension { + 16 => f32::cosine_once_16_scalar(self.key, self.key_norm, vector), + _ => cosine_scalar(self.key, self.key_norm, vector), + }, + // SAFETY: each kind is selected only after runtime detection of + // the kernel's required target features. + BatchKind::Avx => match self.dimension { + 16 => unsafe { + f32::cosine_once_x86::cosine_once_16_avx(self.key, self.key_norm, vector) + }, + _ => unsafe { f32_x86::cosine_fast_avx(self.key, self.key_norm, vector) }, + }, + BatchKind::AvxFma => match self.dimension { + 16 => unsafe { + f32::cosine_once_x86::cosine_once_16_avx_fma(self.key, self.key_norm, vector) + }, + _ => unsafe { f32_x86::cosine_fast_avx_fma(self.key, self.key_norm, vector) }, + }, + BatchKind::Avx512 => match self.dimension { + 16 => unsafe { + f32::cosine_once_x86::cosine_once_16_avx512(self.key, self.key_norm, vector) + }, + _ => unsafe { f32_x86::cosine_fast_avx512(self.key, self.key_norm, vector) }, + }, + }; + Some(distance) + } + + #[inline] + fn size_hint(&self) -> (usize, Option) { + let len = self.len(); + (len, Some(len)) + } +} + +#[cfg(all(target_arch = "x86_64", not(target_feature = "avx2")))] +impl ExactSizeIterator for CosineBatchIter<'_> { + #[inline] + fn len(&self) -> usize { + (self.batch.len() - self.offset) / self.dimension + } +} + impl Cosine for f64 { #[inline] fn cosine_fast(x: &[Self], x_norm: f32, y: &[Self]) -> f32 { - use crate::simd::f64::{f64x4, f64x8}; - use crate::simd::{FloatSimd, SIMD}; + // Trait methods cannot carry `#[target_feature]` attributes, so the body + // lives in a free function that runtime-dispatches via `*SIMD_SUPPORT` + // to an AVX2 inner kernel on capable hosts, or a portable scalar fallback. + cosine_fast_f64_dispatched(x, x_norm, y) + } +} +/// Fast cosine for f64, runtime-dispatched via `SIMD_SUPPORT` on x86_64 +/// (AVX-512 / AVX2+FMA / AVX+FMA / AVX / scalar). Non-x86 uses the SIMD +/// primitives in `crate::simd::f64`. +#[inline] +fn cosine_fast_f64_dispatched(x: &[f64], x_norm: f32, y: &[f64]) -> f32 { + #[cfg(target_arch = "x86_64")] + { + match *SIMD_SUPPORT { + SimdSupport::Avx512 | SimdSupport::Avx512FP16 => unsafe { + f64_x86::cosine_fast_avx512(x, x_norm, y) + }, + SimdSupport::Avx2 | SimdSupport::AvxFma => unsafe { + f64_x86::cosine_fast_avx_fma(x, x_norm, y) + }, + SimdSupport::Avx => unsafe { f64_x86::cosine_fast_avx(x, x_norm, y) }, + _ => cosine_scalar(x, x_norm, y), + } + } + #[cfg(not(target_arch = "x86_64"))] + { + cosine_fast_f64_simd_other(x, x_norm, y) + } +} + +/// AVX2 + FMA implementation of the f64 cosine_fast kernel. +/// +/// Lives in a `#[target_feature]`-annotated function so the SIMD primitives +/// in `crate::simd::f64` (which use raw AVX intrinsics) inline correctly +/// even when the compile baseline does not have AVX2 enabled. Caller must +/// ensure the host supports AVX2 + FMA. +#[cfg(target_arch = "x86_64")] +mod f64_x86 { + use std::arch::x86_64::*; + + use crate::simd::f64::{f64x4, f64x8}; + use crate::simd::x86::hsum256_pd; + use crate::simd::{FloatSimd, SIMD}; + + /// AVX-512 path for f64 fast cosine: 8-wide `__m512d` xy/yy with `vfmadd231pd` per iteration. + #[target_feature(enable = "avx512f")] + pub unsafe fn cosine_fast_avx512(x: &[f64], x_norm: f32, y: &[f64]) -> f32 { + let dim = x.len(); + let unrolled_len = dim / 8 * 8; + + let mut acc_xy = _mm512_setzero_pd(); + let mut acc_yy = _mm512_setzero_pd(); + for i in (0..unrolled_len).step_by(8) { + let xv = _mm512_loadu_pd(x.as_ptr().add(i)); + let yv = _mm512_loadu_pd(y.as_ptr().add(i)); + acc_xy = _mm512_fmadd_pd(xv, yv, acc_xy); + acc_yy = _mm512_fmadd_pd(yv, yv, acc_yy); + } + + let mut xy = _mm512_reduce_add_pd(acc_xy); + let mut yy = _mm512_reduce_add_pd(acc_yy); + for i in unrolled_len..dim { + xy += x[i] * y[i]; + yy += y[i] * y[i]; + } + + let y_norm_sq = yy as f32; + let xy_f32 = xy as f32; + 1.0 - xy_f32 / x_norm / y_norm_sq.sqrt() + } + + /// AVX + FMA path for f64 fast cosine. Covers both AvxFma and AVX2 dispatch (body uses no AVX2-specific intrinsics). + #[target_feature(enable = "avx,fma")] + pub unsafe fn cosine_fast_avx_fma(x: &[f64], x_norm: f32, y: &[f64]) -> f32 { let dim = x.len(); let unrolled_len = dim / 8 * 8; let mut y_norm8 = f64x8::zeros(); let mut xy8 = f64x8::zeros(); for i in (0..unrolled_len).step_by(8) { - unsafe { - let xv = f64x8::load_unaligned(x.as_ptr().add(i)); - let yv = f64x8::load_unaligned(y.as_ptr().add(i)); - xy8.multiply_add(xv, yv); - y_norm8.multiply_add(yv, yv); - } + let xv = f64x8::load_unaligned(x.as_ptr().add(i)); + let yv = f64x8::load_unaligned(y.as_ptr().add(i)); + xy8.multiply_add(xv, yv); + y_norm8.multiply_add(yv, yv); } let aligned_len = dim / 4 * 4; let mut y_norm4 = f64x4::zeros(); let mut xy4 = f64x4::zeros(); for i in (unrolled_len..aligned_len).step_by(4) { - unsafe { - let xv = f64x4::load_unaligned(x.as_ptr().add(i)); - let yv = f64x4::load_unaligned(y.as_ptr().add(i)); - xy4.multiply_add(xv, yv); - y_norm4.multiply_add(yv, yv); - } + let xv = f64x4::load_unaligned(x.as_ptr().add(i)); + let yv = f64x4::load_unaligned(y.as_ptr().add(i)); + xy4.multiply_add(xv, yv); + y_norm4.multiply_add(yv, yv); } - let tail_y_norm: Self = y[aligned_len..].iter().map(|&v| v * v).sum(); - let tail_xy: Self = x[aligned_len..] + let tail_y_norm: f64 = y[aligned_len..].iter().map(|&v| v * v).sum(); + let tail_xy: f64 = x[aligned_len..] .iter() .zip(y[aligned_len..].iter()) .map(|(&a, &b)| a * b) @@ -328,6 +949,336 @@ impl Cosine for f64 { let xy = (xy8.reduce_sum() + xy4.reduce_sum() + tail_xy) as f32; 1.0 - xy / x_norm / y_norm_sq.sqrt() } + + /// AVX-only path for f64 fast cosine (no FMA): `_mm256_mul_pd` + `_mm256_add_pd` per iteration; tail handled inline. + #[target_feature(enable = "avx")] + pub unsafe fn cosine_fast_avx(x: &[f64], x_norm: f32, y: &[f64]) -> f32 { + let dim = x.len(); + let aligned_len = dim / 4 * 4; + + let mut acc_xy = _mm256_setzero_pd(); + let mut acc_yy = _mm256_setzero_pd(); + for i in (0..aligned_len).step_by(4) { + let xv = _mm256_loadu_pd(x.as_ptr().add(i)); + let yv = _mm256_loadu_pd(y.as_ptr().add(i)); + acc_xy = _mm256_add_pd(acc_xy, _mm256_mul_pd(xv, yv)); + acc_yy = _mm256_add_pd(acc_yy, _mm256_mul_pd(yv, yv)); + } + + let xy_main = hsum256_pd(acc_xy); + let yy_main = hsum256_pd(acc_yy); + + let tail_y_norm: f64 = y[aligned_len..].iter().map(|&v| v * v).sum(); + let tail_xy: f64 = x[aligned_len..] + .iter() + .zip(y[aligned_len..].iter()) + .map(|(&a, &b)| a * b) + .sum(); + + let y_norm_sq = (yy_main + tail_y_norm) as f32; + let xy = (xy_main + tail_xy) as f32; + 1.0 - xy / x_norm / y_norm_sq.sqrt() + } +} + +#[cfg(not(target_arch = "x86_64"))] +#[inline] +fn cosine_fast_f64_simd_other(x: &[f64], x_norm: f32, y: &[f64]) -> f32 { + use crate::simd::f64::{f64x4, f64x8}; + use crate::simd::{FloatSimd, SIMD}; + + let dim = x.len(); + let unrolled_len = dim / 8 * 8; + let mut y_norm8 = f64x8::zeros(); + let mut xy8 = f64x8::zeros(); + for i in (0..unrolled_len).step_by(8) { + unsafe { + let xv = f64x8::load_unaligned(x.as_ptr().add(i)); + let yv = f64x8::load_unaligned(y.as_ptr().add(i)); + xy8.multiply_add(xv, yv); + y_norm8.multiply_add(yv, yv); + } + } + let aligned_len = dim / 4 * 4; + let mut y_norm4 = f64x4::zeros(); + let mut xy4 = f64x4::zeros(); + for i in (unrolled_len..aligned_len).step_by(4) { + unsafe { + let xv = f64x4::load_unaligned(x.as_ptr().add(i)); + let yv = f64x4::load_unaligned(y.as_ptr().add(i)); + xy4.multiply_add(xv, yv); + y_norm4.multiply_add(yv, yv); + } + } + let tail_y_norm: f64 = y[aligned_len..].iter().map(|&v| v * v).sum(); + let tail_xy: f64 = x[aligned_len..] + .iter() + .zip(y[aligned_len..].iter()) + .map(|(&a, &b)| a * b) + .sum(); + + let y_norm_sq = (y_norm8.reduce_sum() + y_norm4.reduce_sum() + tail_y_norm) as f32; + let xy = (xy8.reduce_sum() + xy4.reduce_sum() + tail_xy) as f32; + 1.0 - xy / x_norm / y_norm_sq.sqrt() +} + +/// Cosine for f32 with known norms, runtime-dispatched via `SIMD_SUPPORT` +/// on x86_64 (AVX-512 / AVX2+FMA / AVX+FMA / AVX / scalar). Non-x86 uses +/// the auto-vectorised scalar loop. +#[inline] +fn cosine_with_norms_f32_dispatched(x: &[f32], x_norm: f32, y_norm: f32, y: &[f32]) -> f32 { + #[cfg(target_arch = "x86_64")] + { + match *SIMD_SUPPORT { + SimdSupport::Avx512 | SimdSupport::Avx512FP16 => unsafe { + f32_x86::cosine_with_norms_avx512(x, x_norm, y_norm, y) + }, + SimdSupport::Avx2 | SimdSupport::AvxFma => unsafe { + f32_x86::cosine_with_norms_avx_fma(x, x_norm, y_norm, y) + }, + SimdSupport::Avx => unsafe { f32_x86::cosine_with_norms_avx(x, x_norm, y_norm, y) }, + _ => cosine_scalar_fast(x, x_norm, y, y_norm), + } + } + #[cfg(not(target_arch = "x86_64"))] + { + cosine_with_norms_f32_simd_other(x, x_norm, y_norm, y) + } +} + +#[cfg(not(target_arch = "x86_64"))] +#[inline] +fn cosine_with_norms_f32_simd_other(x: &[f32], x_norm: f32, y_norm: f32, y: &[f32]) -> f32 { + let dim = x.len(); + let unrolled_len = dim / 16 * 16; + let mut xy16 = f32x16::zeros(); + for i in (0..unrolled_len).step_by(16) { + unsafe { + let xv = f32x16::load_unaligned(x.as_ptr().add(i)); + let yv = f32x16::load_unaligned(y.as_ptr().add(i)); + xy16.multiply_add(xv, yv); + } + } + let aligned_len = dim / 8 * 8; + let mut xy8 = f32x8::zeros(); + for i in (unrolled_len..aligned_len).step_by(8) { + unsafe { + let xv = f32x8::load_unaligned(x.as_ptr().add(i)); + let yv = f32x8::load_unaligned(y.as_ptr().add(i)); + xy8.multiply_add(xv, yv); + } + } + let xy = xy16.reduce_sum() + xy8.reduce_sum() + dot(&x[aligned_len..], &y[aligned_len..]); + 1.0 - xy / x_norm / y_norm +} + +/// Fast cosine for f32, runtime-dispatched via `SIMD_SUPPORT` on x86_64 +/// (AVX-512 / AVX2+FMA / AVX+FMA / AVX / scalar). Non-x86 uses the +/// `simd::f32` primitives, unconditionally backed by NEON / LSX. +#[inline] +fn cosine_fast_f32_dispatched(x: &[f32], x_norm: f32, other: &[f32]) -> f32 { + #[cfg(target_arch = "x86_64")] + { + match *SIMD_SUPPORT { + SimdSupport::Avx512 | SimdSupport::Avx512FP16 => unsafe { + f32_x86::cosine_fast_avx512(x, x_norm, other) + }, + SimdSupport::Avx2 | SimdSupport::AvxFma => unsafe { + f32_x86::cosine_fast_avx_fma(x, x_norm, other) + }, + SimdSupport::Avx => unsafe { f32_x86::cosine_fast_avx(x, x_norm, other) }, + _ => cosine_scalar(x, x_norm, other), + } + } + #[cfg(not(target_arch = "x86_64"))] + { + cosine_fast_f32_simd_other(x, x_norm, other) + } +} + +/// AVX2 + FMA implementation of the f32 fast cosine kernel. +/// +/// Lives in a `#[target_feature]`-annotated function so the SIMD primitives +/// in `crate::simd::f32` (which use raw AVX intrinsics) inline correctly +/// even when the compile baseline does not have AVX2 enabled. Caller must +/// ensure the host supports AVX2 + FMA. +#[cfg(target_arch = "x86_64")] +mod f32_x86 { + use std::arch::x86_64::*; + + use super::{dot, f32x8, f32x16, norm_l2}; + use crate::simd::x86::hsum256_ps; + use crate::simd::{FloatSimd, SIMD}; + + /// AVX + FMA path for f32 fast cosine. Covers both AvxFma and AVX2 dispatch (body uses no AVX2-specific intrinsics). + #[target_feature(enable = "avx,fma")] + pub unsafe fn cosine_fast_avx_fma(x: &[f32], x_norm: f32, other: &[f32]) -> f32 { + let dim = x.len(); + let unrolled_len = dim / 16 * 16; + let mut y_norm16 = f32x16::zeros(); + let mut xy16 = f32x16::zeros(); + for i in (0..unrolled_len).step_by(16) { + let xv = f32x16::load_unaligned(x.as_ptr().add(i)); + let yv = f32x16::load_unaligned(other.as_ptr().add(i)); + xy16.multiply_add(xv, yv); + y_norm16.multiply_add(yv, yv); + } + let aligned_len = dim / 8 * 8; + let mut y_norm8 = f32x8::zeros(); + let mut xy8 = f32x8::zeros(); + for i in (unrolled_len..aligned_len).step_by(8) { + let xv = f32x8::load_unaligned(x.as_ptr().add(i)); + let yv = f32x8::load_unaligned(other.as_ptr().add(i)); + xy8.multiply_add(xv, yv); + y_norm8.multiply_add(yv, yv); + } + let y_norm = + y_norm16.reduce_sum() + y_norm8.reduce_sum() + norm_l2(&other[aligned_len..]).powi(2); + let xy = + xy16.reduce_sum() + xy8.reduce_sum() + dot(&x[aligned_len..], &other[aligned_len..]); + 1.0 - xy / x_norm / y_norm.sqrt() + } + + /// AVX-only path for f32 fast cosine (no FMA): `_mm256_mul_ps` + `_mm256_add_ps` per iteration; tail via trait-routed `dot`/`norm_l2`. + #[target_feature(enable = "avx")] + pub unsafe fn cosine_fast_avx(x: &[f32], x_norm: f32, other: &[f32]) -> f32 { + let dim = x.len(); + let aligned_len = dim / 8 * 8; + + let mut acc_xy = _mm256_setzero_ps(); + let mut acc_yy = _mm256_setzero_ps(); + for i in (0..aligned_len).step_by(8) { + let xv = _mm256_loadu_ps(x.as_ptr().add(i)); + let yv = _mm256_loadu_ps(other.as_ptr().add(i)); + acc_xy = _mm256_add_ps(acc_xy, _mm256_mul_ps(xv, yv)); + acc_yy = _mm256_add_ps(acc_yy, _mm256_mul_ps(yv, yv)); + } + + let xy_main = hsum256_ps(acc_xy); + let yy_main = hsum256_ps(acc_yy); + + let y_norm = yy_main + norm_l2(&other[aligned_len..]).powi(2); + let xy = xy_main + dot(&x[aligned_len..], &other[aligned_len..]); + 1.0 - xy / x_norm / y_norm.sqrt() + } + + /// AVX-512 path for f32 fast cosine: 16-wide `__m512` xy/yy with `vfmadd231ps` per iteration. + #[target_feature(enable = "avx512f")] + pub unsafe fn cosine_fast_avx512(x: &[f32], x_norm: f32, other: &[f32]) -> f32 { + let dim = x.len(); + let unrolled_len = dim / 16 * 16; + + let mut acc_xy = _mm512_setzero_ps(); + let mut acc_yy = _mm512_setzero_ps(); + for i in (0..unrolled_len).step_by(16) { + let xv = _mm512_loadu_ps(x.as_ptr().add(i)); + let yv = _mm512_loadu_ps(other.as_ptr().add(i)); + acc_xy = _mm512_fmadd_ps(xv, yv, acc_xy); + acc_yy = _mm512_fmadd_ps(yv, yv, acc_yy); + } + + let mut xy = _mm512_reduce_add_ps(acc_xy); + let mut yy = _mm512_reduce_add_ps(acc_yy); + for i in unrolled_len..dim { + xy += x[i] * other[i]; + yy += other[i] * other[i]; + } + + 1.0 - xy / x_norm / yy.sqrt() + } + + /// AVX-512 path for f32 cosine with known norms: 16-wide `__m512` with `vfmadd231ps` per iteration. + #[target_feature(enable = "avx512f")] + pub unsafe fn cosine_with_norms_avx512(x: &[f32], x_norm: f32, y_norm: f32, y: &[f32]) -> f32 { + let dim = x.len(); + let unrolled_len = dim / 16 * 16; + + let mut acc = _mm512_setzero_ps(); + for i in (0..unrolled_len).step_by(16) { + let xv = _mm512_loadu_ps(x.as_ptr().add(i)); + let yv = _mm512_loadu_ps(y.as_ptr().add(i)); + acc = _mm512_fmadd_ps(xv, yv, acc); + } + + let mut xy = _mm512_reduce_add_ps(acc); + for i in unrolled_len..dim { + xy += x[i] * y[i]; + } + + 1.0 - xy / x_norm / y_norm + } + + /// AVX + FMA path for f32 cosine with known norms. Covers both AvxFma and AVX2 dispatch (body uses no AVX2-specific intrinsics). + #[target_feature(enable = "avx,fma")] + pub unsafe fn cosine_with_norms_avx_fma(x: &[f32], x_norm: f32, y_norm: f32, y: &[f32]) -> f32 { + let dim = x.len(); + let unrolled_len = dim / 16 * 16; + let mut xy16 = f32x16::zeros(); + for i in (0..unrolled_len).step_by(16) { + let xv = f32x16::load_unaligned(x.as_ptr().add(i)); + let yv = f32x16::load_unaligned(y.as_ptr().add(i)); + xy16.multiply_add(xv, yv); + } + let aligned_len = dim / 8 * 8; + let mut xy8 = f32x8::zeros(); + for i in (unrolled_len..aligned_len).step_by(8) { + let xv = f32x8::load_unaligned(x.as_ptr().add(i)); + let yv = f32x8::load_unaligned(y.as_ptr().add(i)); + xy8.multiply_add(xv, yv); + } + let xy = xy16.reduce_sum() + xy8.reduce_sum() + dot(&x[aligned_len..], &y[aligned_len..]); + 1.0 - xy / x_norm / y_norm + } + + /// AVX-only path for f32 cosine with known norms (no FMA): `_mm256_mul_ps` + `_mm256_add_ps` per iteration; tail via trait-routed `dot`. + #[target_feature(enable = "avx")] + pub unsafe fn cosine_with_norms_avx(x: &[f32], x_norm: f32, y_norm: f32, y: &[f32]) -> f32 { + let dim = x.len(); + let aligned_len = dim / 8 * 8; + + let mut acc = _mm256_setzero_ps(); + for i in (0..aligned_len).step_by(8) { + let xv = _mm256_loadu_ps(x.as_ptr().add(i)); + let yv = _mm256_loadu_ps(y.as_ptr().add(i)); + acc = _mm256_add_ps(acc, _mm256_mul_ps(xv, yv)); + } + + let xy_main = hsum256_ps(acc); + let xy = xy_main + dot(&x[aligned_len..], &y[aligned_len..]); + 1.0 - xy / x_norm / y_norm + } +} + +#[cfg(not(target_arch = "x86_64"))] +#[inline] +fn cosine_fast_f32_simd_other(x: &[f32], x_norm: f32, other: &[f32]) -> f32 { + let dim = x.len(); + let unrolled_len = dim / 16 * 16; + let mut y_norm16 = f32x16::zeros(); + let mut xy16 = f32x16::zeros(); + for i in (0..unrolled_len).step_by(16) { + unsafe { + let xv = f32x16::load_unaligned(x.as_ptr().add(i)); + let yv = f32x16::load_unaligned(other.as_ptr().add(i)); + xy16.multiply_add(xv, yv); + y_norm16.multiply_add(yv, yv); + } + } + let aligned_len = dim / 8 * 8; + let mut y_norm8 = f32x8::zeros(); + let mut xy8 = f32x8::zeros(); + for i in (unrolled_len..aligned_len).step_by(8) { + unsafe { + let xv = f32x8::load_unaligned(x.as_ptr().add(i)); + let yv = f32x8::load_unaligned(other.as_ptr().add(i)); + xy8.multiply_add(xv, yv); + y_norm8.multiply_add(yv, yv); + } + } + let y_norm = + y_norm16.reduce_sum() + y_norm8.reduce_sum() + norm_l2(&other[aligned_len..]).powi(2); + let xy = xy16.reduce_sum() + xy8.reduce_sum() + dot(&x[aligned_len..], &other[aligned_len..]); + 1.0 - xy / x_norm / y_norm.sqrt() } /// Fallback non-SIMD implementation @@ -437,6 +1388,25 @@ pub fn cosine_distance_arrow_batch( } } +/// Portable scalar reference cosine over f64 inputs. Used by parity tests +/// to compare against every dispatched per-tier inner kernel. Computes +/// `1 - xy / (x_norm * y_norm_sq.sqrt())` in f64 then casts to f32, matching +/// the reduction order of the dispatched kernels. +#[cfg(test)] +fn cosine_fast_scalar(x: &[f64], x_norm: f32, y: &[f64]) -> f32 { + let xy: f64 = x.iter().zip(y.iter()).map(|(&a, &b)| a * b).sum(); + let y_norm_sq: f64 = y.iter().map(|&v| v * v).sum(); + 1.0 - (xy as f32) / x_norm / (y_norm_sq as f32).sqrt() +} + +/// Portable scalar reference cosine when both norms are known. Mirrors +/// `cosine_with_norms_f32_dispatched` for parity testing. +#[cfg(test)] +fn cosine_with_norms_scalar(x: &[f64], x_norm: f32, y_norm: f32, y: &[f64]) -> f32 { + let xy: f64 = x.iter().zip(y.iter()).map(|(&a, &b)| a * b).sum(); + 1.0 - (xy as f32) / x_norm / y_norm +} + #[cfg(test)] mod tests { use super::*; @@ -559,5 +1529,476 @@ mod tests { prop_assume!(norm_l2(&y) > 1e-20); do_cosine_test(&x, &y)?; } + + /// Cross-backend parity for the f32 cosine_fast kernel. Exercises the + /// scalar fallback (`cosine_scalar`) against the dispatched SIMD path + /// so the runtime fallback is exercised even on AVX2-capable CI hosts. + #[test] + fn test_cosine_fast_f32_scalar_simd_parity( + (x, y) in arbitrary_vector_pair(arbitrary_f32, 4..4048) + ) { + prop_assume!(norm_l2(&x) > 1e-10); + prop_assume!(norm_l2(&y) > 1e-10); + let x_norm = norm_l2(&x); + let x_f64: Vec = x.iter().map(|&v| v as f64).collect(); + let y_f64: Vec = y.iter().map(|&v| v as f64).collect(); + let scalar = cosine_fast_scalar(&x_f64, x_norm, &y_f64); + let simd = ::cosine_fast(&x, x_norm, &y); + prop_assert!(approx::relative_eq!(scalar, simd, max_relative = 1e-3)); + } + + /// AVX-512-direct parity for the f32 cosine_fast kernel. Early-returns + /// on hosts without AVX-512F so the test stays portable. + #[cfg(target_arch = "x86_64")] + #[test] + fn test_cosine_fast_f32_scalar_vs_avx512_parity( + (x, y) in arbitrary_vector_pair(arbitrary_f32, 4..4048) + ) { + if !std::is_x86_feature_detected!("avx512f") { + return Ok(()); + } + prop_assume!(norm_l2(&x) > 1e-10); + prop_assume!(norm_l2(&y) > 1e-10); + let x_norm = norm_l2(&x); + let scalar = cosine_scalar(&x, x_norm, &y); + let avx512 = unsafe { f32_x86::cosine_fast_avx512(&x, x_norm, &y) }; + prop_assert!(approx::relative_eq!(scalar, avx512, max_relative = 1e-5)); + } + + /// AVX + FMA-direct parity for the f32 cosine_fast kernel. Covers + /// the AMD Piledriver / Steamroller / FX-7500 tier. Early-returns + /// on hosts without both AVX and FMA. + #[cfg(target_arch = "x86_64")] + #[test] + fn test_cosine_fast_f32_scalar_vs_avx_fma_parity( + (x, y) in arbitrary_vector_pair(arbitrary_f32, 4..4048) + ) { + if !(std::is_x86_feature_detected!("avx") && std::is_x86_feature_detected!("fma")) { + return Ok(()); + } + prop_assume!(norm_l2(&x) > 1e-10); + prop_assume!(norm_l2(&y) > 1e-10); + let x_norm = norm_l2(&x); + let scalar = cosine_scalar(&x, x_norm, &y); + let avx_fma = unsafe { f32_x86::cosine_fast_avx_fma(&x, x_norm, &y) }; + prop_assert!(approx::relative_eq!(scalar, avx_fma, max_relative = 1e-5)); + } + + /// AVX-only-direct parity for the f32 cosine_fast kernel. Covers + /// the Intel Sandy Bridge / Ivy Bridge tier. Early-returns on + /// hosts without AVX. + #[cfg(target_arch = "x86_64")] + #[test] + fn test_cosine_fast_f32_scalar_vs_avx_parity( + (x, y) in arbitrary_vector_pair(arbitrary_f32, 4..4048) + ) { + if !std::is_x86_feature_detected!("avx") { + return Ok(()); + } + prop_assume!(norm_l2(&x) > 1e-10); + prop_assume!(norm_l2(&y) > 1e-10); + let x_norm = norm_l2(&x); + let scalar = cosine_scalar(&x, x_norm, &y); + let avx = unsafe { f32_x86::cosine_fast_avx(&x, x_norm, &y) }; + prop_assert!(approx::relative_eq!(scalar, avx, max_relative = 1e-5)); + } + + /// Cross-backend parity for the f32 cosine_with_norms kernel. + /// Exercises the scalar fallback (`cosine_scalar_fast`) against the + /// dispatched SIMD path so the runtime fallback is exercised even on + /// AVX2-capable CI hosts. + #[test] + fn test_cosine_with_norms_f32_scalar_simd_parity( + (x, y) in arbitrary_vector_pair(arbitrary_f32, 4..4048) + ) { + prop_assume!(norm_l2(&x) > 1e-10); + prop_assume!(norm_l2(&y) > 1e-10); + let x_norm = norm_l2(&x); + let y_norm = norm_l2(&y); + let x_f64: Vec = x.iter().map(|&v| v as f64).collect(); + let y_f64: Vec = y.iter().map(|&v| v as f64).collect(); + let scalar = cosine_with_norms_scalar(&x_f64, x_norm, y_norm, &y_f64); + let simd = ::cosine_with_norms(&x, x_norm, y_norm, &y); + prop_assert!(approx::relative_eq!(scalar, simd, max_relative = 1e-3)); + } + + /// AVX-512-direct parity for the f32 cosine_with_norms kernel. + /// Early-returns on hosts without AVX-512F. + #[cfg(target_arch = "x86_64")] + #[test] + fn test_cosine_with_norms_f32_scalar_vs_avx512_parity( + (x, y) in arbitrary_vector_pair(arbitrary_f32, 4..4048) + ) { + if !std::is_x86_feature_detected!("avx512f") { + return Ok(()); + } + prop_assume!(norm_l2(&x) > 1e-10); + prop_assume!(norm_l2(&y) > 1e-10); + let x_norm = norm_l2(&x); + let y_norm = norm_l2(&y); + let scalar = cosine_scalar_fast(&x, x_norm, &y, y_norm); + let avx512 = unsafe { f32_x86::cosine_with_norms_avx512(&x, x_norm, y_norm, &y) }; + prop_assert!(approx::relative_eq!(scalar, avx512, max_relative = 1e-5)); + } + + /// AVX + FMA-direct parity for the f32 cosine_with_norms kernel. + /// Covers the AMD Piledriver / Steamroller / FX-7500 tier. + /// Early-returns on hosts without both AVX and FMA. + #[cfg(target_arch = "x86_64")] + #[test] + fn test_cosine_with_norms_f32_scalar_vs_avx_fma_parity( + (x, y) in arbitrary_vector_pair(arbitrary_f32, 4..4048) + ) { + if !(std::is_x86_feature_detected!("avx") && std::is_x86_feature_detected!("fma")) { + return Ok(()); + } + prop_assume!(norm_l2(&x) > 1e-10); + prop_assume!(norm_l2(&y) > 1e-10); + let x_norm = norm_l2(&x); + let y_norm = norm_l2(&y); + let scalar = cosine_scalar_fast(&x, x_norm, &y, y_norm); + let avx_fma = unsafe { f32_x86::cosine_with_norms_avx_fma(&x, x_norm, y_norm, &y) }; + prop_assert!(approx::relative_eq!(scalar, avx_fma, max_relative = 1e-5)); + } + + /// AVX-only-direct parity for the f32 cosine_with_norms kernel. + /// Covers the Intel Sandy Bridge / Ivy Bridge tier. Early-returns + /// on hosts without AVX. + #[cfg(target_arch = "x86_64")] + #[test] + fn test_cosine_with_norms_f32_scalar_vs_avx_parity( + (x, y) in arbitrary_vector_pair(arbitrary_f32, 4..4048) + ) { + if !std::is_x86_feature_detected!("avx") { + return Ok(()); + } + prop_assume!(norm_l2(&x) > 1e-10); + prop_assume!(norm_l2(&y) > 1e-10); + let x_norm = norm_l2(&x); + let y_norm = norm_l2(&y); + let scalar = cosine_scalar_fast(&x, x_norm, &y, y_norm); + let avx = unsafe { f32_x86::cosine_with_norms_avx(&x, x_norm, y_norm, &y) }; + prop_assert!(approx::relative_eq!(scalar, avx, max_relative = 1e-5)); + } + + /// Cross-backend parity for the f64 cosine_fast kernel. Uses the + /// hand-rolled `cosine_fast_scalar` (not the trait-routed + /// `cosine_scalar`, which would itself dispatch through `dot::`) + /// so the reference stays free of any AVX path on AVX2-capable hosts. + #[test] + fn test_cosine_fast_f64_scalar_simd_parity( + (x, y) in arbitrary_vector_pair(arbitrary_f64, 4..4048) + ) { + prop_assume!(norm_l2(&x) > 1e-20); + prop_assume!(norm_l2(&y) > 1e-20); + let x_norm = norm_l2(&x); + let scalar = cosine_fast_scalar(&x, x_norm, &y); + let simd = ::cosine_fast(&x, x_norm, &y); + prop_assert!(approx::relative_eq!(scalar, simd, max_relative = 1e-3)); + } + + /// AVX-512-direct parity for the f64 cosine_fast kernel. Early-returns + /// on hosts without AVX-512F. + #[cfg(target_arch = "x86_64")] + #[test] + fn test_cosine_fast_f64_scalar_vs_avx512_parity( + (x, y) in arbitrary_vector_pair(arbitrary_f64, 4..4048) + ) { + if !std::is_x86_feature_detected!("avx512f") { + return Ok(()); + } + prop_assume!(norm_l2(&x) > 1e-20); + prop_assume!(norm_l2(&y) > 1e-20); + let x_norm = norm_l2(&x); + let scalar = cosine_fast_scalar(&x, x_norm, &y); + let avx512 = unsafe { f64_x86::cosine_fast_avx512(&x, x_norm, &y) }; + prop_assert!(approx::relative_eq!(scalar, avx512, max_relative = 1e-5)); + } + + /// AVX + FMA-direct parity for the f64 cosine_fast kernel. Covers + /// the AMD Piledriver / Steamroller / FX-7500 tier. Early-returns + /// on hosts without both AVX and FMA. + #[cfg(target_arch = "x86_64")] + #[test] + fn test_cosine_fast_f64_scalar_vs_avx_fma_parity( + (x, y) in arbitrary_vector_pair(arbitrary_f64, 4..4048) + ) { + if !(std::is_x86_feature_detected!("avx") && std::is_x86_feature_detected!("fma")) { + return Ok(()); + } + prop_assume!(norm_l2(&x) > 1e-20); + prop_assume!(norm_l2(&y) > 1e-20); + let x_norm = norm_l2(&x); + let scalar = cosine_fast_scalar(&x, x_norm, &y); + let avx_fma = unsafe { f64_x86::cosine_fast_avx_fma(&x, x_norm, &y) }; + prop_assert!(approx::relative_eq!(scalar, avx_fma, max_relative = 1e-5)); + } + + /// AVX-only-direct parity for the f64 cosine_fast kernel. Covers + /// the Intel Sandy Bridge / Ivy Bridge tier. Early-returns on + /// hosts without AVX. + #[cfg(target_arch = "x86_64")] + #[test] + fn test_cosine_fast_f64_scalar_vs_avx_parity( + (x, y) in arbitrary_vector_pair(arbitrary_f64, 4..4048) + ) { + if !std::is_x86_feature_detected!("avx") { + return Ok(()); + } + prop_assume!(norm_l2(&x) > 1e-20); + prop_assume!(norm_l2(&y) > 1e-20); + let x_norm = norm_l2(&x); + let scalar = cosine_fast_scalar(&x, x_norm, &y); + let avx = unsafe { f64_x86::cosine_fast_avx(&x, x_norm, &y) }; + prop_assert!(approx::relative_eq!(scalar, avx, max_relative = 1e-5)); + } + + /// Parity check for `cosine_once_8` (despecialised 8-lane width). + /// + /// The `epsilon = 1e-6` clause handles the case where the proptest + /// generator produces inputs with extreme dynamic range (e.g., mixing + /// `1e-43` with `1e7` in the same vector). When the dot product is + /// dominated by one large term and the cosine result is near zero, + /// the f32-precision SIMD path and the f64-precision scalar reference + /// can legitimately differ by more than `max_relative = 1e-3` of the + /// (near-zero) result. The absolute epsilon catches these without + /// masking real bugs (where the absolute error would be > 1e-6). + #[test] + fn test_cosine_once_8_scalar_simd_parity( + (x, y) in arbitrary_vector_pair(arbitrary_f32, 8..9) + ) { + prop_assume!(norm_l2(&x) > 1e-10); + prop_assume!(norm_l2(&y) > 1e-10); + let x_norm = norm_l2(&x); + let x_f64: Vec = x.iter().map(|&v| v as f64).collect(); + let y_f64: Vec = y.iter().map(|&v| v as f64).collect(); + let scalar = cosine_fast_scalar(&x_f64, x_norm, &y_f64); + let simd = f32::cosine_once_8(&x, x_norm, &y); + prop_assert!(approx::relative_eq!(scalar, simd, max_relative = 1e-3, epsilon = 1e-6)); + } + + /// Parity check for `cosine_once_16` (despecialised 16-lane width). + /// See `test_cosine_once_8_scalar_simd_parity` for `epsilon` rationale. + #[test] + fn test_cosine_once_16_scalar_simd_parity( + (x, y) in arbitrary_vector_pair(arbitrary_f32, 16..17) + ) { + prop_assume!(norm_l2(&x) > 1e-10); + prop_assume!(norm_l2(&y) > 1e-10); + let x_norm = norm_l2(&x); + let x_f64: Vec = x.iter().map(|&v| v as f64).collect(); + let y_f64: Vec = y.iter().map(|&v| v as f64).collect(); + let scalar = cosine_fast_scalar(&x_f64, x_norm, &y_f64); + let simd = f32::cosine_once_16(&x, x_norm, &y); + prop_assert!(approx::relative_eq!(scalar, simd, max_relative = 1e-3, epsilon = 1e-6)); + } + + /// AVX-512-direct parity for the 8-lane cosine_once kernel. Verifies + /// the masked-load (mask 0x00FF) AVX-512 implementation produces + /// the same result as the scalar reference. Early-returns on hosts + /// without AVX-512F. + #[cfg(target_arch = "x86_64")] + #[test] + fn test_cosine_once_8_scalar_vs_avx512_parity( + (x, y) in arbitrary_vector_pair(arbitrary_f32, 8..9) + ) { + if !std::is_x86_feature_detected!("avx512f") { + return Ok(()); + } + prop_assume!(norm_l2(&x) > 1e-10); + prop_assume!(norm_l2(&y) > 1e-10); + let x_norm = norm_l2(&x); + let scalar = super::f32::cosine_once_8_scalar(&x, x_norm, &y); + let avx512 = + unsafe { super::f32::cosine_once_x86::cosine_once_8_avx512(&x, x_norm, &y) }; + prop_assert!(approx::relative_eq!(scalar, avx512, max_relative = 1e-5)); + } + + /// AVX-512-direct parity for the 16-lane cosine_once kernel. Verifies + /// the full-width `__m512` load implementation produces the same + /// result as the scalar reference. Early-returns on hosts without + /// AVX-512F. + #[cfg(target_arch = "x86_64")] + #[test] + fn test_cosine_once_16_scalar_vs_avx512_parity( + (x, y) in arbitrary_vector_pair(arbitrary_f32, 16..17) + ) { + if !std::is_x86_feature_detected!("avx512f") { + return Ok(()); + } + prop_assume!(norm_l2(&x) > 1e-10); + prop_assume!(norm_l2(&y) > 1e-10); + let x_norm = norm_l2(&x); + let scalar = super::f32::cosine_once_16_scalar(&x, x_norm, &y); + let avx512 = + unsafe { super::f32::cosine_once_x86::cosine_once_16_avx512(&x, x_norm, &y) }; + prop_assert!(approx::relative_eq!(scalar, avx512, max_relative = 1e-5)); + } + + /// AVX + FMA-direct parity for the 8-lane cosine_once kernel. + /// Covers the AMD Piledriver / Steamroller / FX-7500 tier. + /// Early-returns on hosts without both AVX and FMA. + #[cfg(target_arch = "x86_64")] + #[test] + fn test_cosine_once_8_scalar_vs_avx_fma_parity( + (x, y) in arbitrary_vector_pair(arbitrary_f32, 8..9) + ) { + if !(std::is_x86_feature_detected!("avx") && std::is_x86_feature_detected!("fma")) { + return Ok(()); + } + prop_assume!(norm_l2(&x) > 1e-10); + prop_assume!(norm_l2(&y) > 1e-10); + let x_norm = norm_l2(&x); + let scalar = super::f32::cosine_once_8_scalar(&x, x_norm, &y); + let avx_fma = + unsafe { super::f32::cosine_once_x86::cosine_once_8_avx_fma(&x, x_norm, &y) }; + prop_assert!(approx::relative_eq!(scalar, avx_fma, max_relative = 1e-5)); + } + + /// AVX + FMA-direct parity for the 16-lane cosine_once kernel. + /// Covers the AMD Piledriver / Steamroller / FX-7500 tier. + /// Early-returns on hosts without both AVX and FMA. + #[cfg(target_arch = "x86_64")] + #[test] + fn test_cosine_once_16_scalar_vs_avx_fma_parity( + (x, y) in arbitrary_vector_pair(arbitrary_f32, 16..17) + ) { + if !(std::is_x86_feature_detected!("avx") && std::is_x86_feature_detected!("fma")) { + return Ok(()); + } + prop_assume!(norm_l2(&x) > 1e-10); + prop_assume!(norm_l2(&y) > 1e-10); + let x_norm = norm_l2(&x); + let scalar = super::f32::cosine_once_16_scalar(&x, x_norm, &y); + let avx_fma = + unsafe { super::f32::cosine_once_x86::cosine_once_16_avx_fma(&x, x_norm, &y) }; + prop_assert!(approx::relative_eq!(scalar, avx_fma, max_relative = 1e-5)); + } + + /// AVX-only-direct parity for the 8-lane cosine_once kernel. + /// Covers the Intel Sandy Bridge / Ivy Bridge tier. Early-returns + /// on hosts without AVX. + #[cfg(target_arch = "x86_64")] + #[test] + fn test_cosine_once_8_scalar_vs_avx_parity( + (x, y) in arbitrary_vector_pair(arbitrary_f32, 8..9) + ) { + if !std::is_x86_feature_detected!("avx") { + return Ok(()); + } + prop_assume!(norm_l2(&x) > 1e-10); + prop_assume!(norm_l2(&y) > 1e-10); + let x_norm = norm_l2(&x); + let scalar = super::f32::cosine_once_8_scalar(&x, x_norm, &y); + let avx = unsafe { super::f32::cosine_once_x86::cosine_once_8_avx(&x, x_norm, &y) }; + prop_assert!(approx::relative_eq!(scalar, avx, max_relative = 1e-5)); + } + + /// AVX-only-direct parity for the 16-lane cosine_once kernel. + /// Covers the Intel Sandy Bridge / Ivy Bridge tier. Early-returns + /// on hosts without AVX. + #[cfg(target_arch = "x86_64")] + #[test] + fn test_cosine_once_16_scalar_vs_avx_parity( + (x, y) in arbitrary_vector_pair(arbitrary_f32, 16..17) + ) { + if !std::is_x86_feature_detected!("avx") { + return Ok(()); + } + prop_assume!(norm_l2(&x) > 1e-10); + prop_assume!(norm_l2(&y) > 1e-10); + let x_norm = norm_l2(&x); + let scalar = super::f32::cosine_once_16_scalar(&x, x_norm, &y); + let avx = unsafe { super::f32::cosine_once_x86::cosine_once_16_avx(&x, x_norm, &y) }; + prop_assert!(approx::relative_eq!(scalar, avx, max_relative = 1e-5)); + } + } + + #[rstest::rstest] + #[case::dim_8(8)] + #[case::dim_16(16)] + #[case::dim_40(40)] + fn test_cosine_batch_matches_per_vector(#[case] dimension: usize) { + let num_vectors = 5; + let x: Vec = (0..dimension) + .map(|index| (index % 13) as f32 * 0.25 + 1.0) + .collect(); + let batch: Vec = (0..dimension * num_vectors) + .map(|index| (index % 11) as f32 * 0.5 - 2.0) + .collect(); + + let boxed: Vec = f32::cosine_batch(&x, &batch, dimension).collect(); + + assert_eq!(boxed.len(), num_vectors); + for (boxed_distance, vector) in boxed.iter().zip(batch.chunks_exact(dimension)) { + let expected = f32::cosine(&x, vector); + assert_relative_eq!( + *boxed_distance, + expected, + max_relative = 1e-5, + epsilon = 1e-6 + ); + } + } + + /// Asserts a batch-level f32 cosine SIMD kernel matches the scalar + /// `cosine_fast` reference for every vector in a multi-vector batch. Runs + /// each of the kernel's three internal dimension arms (8, 16, and the + /// general `chunks_exact` path). The AVX/FMA helpers are test-only, while + /// the AVX-512 helper is selected only by AVX2-baseline builds, so direct + /// calls cover them under the default x86-64-v2 baseline. + #[cfg(target_arch = "x86_64")] + fn check_cosine_batch_kernel(kernel: unsafe fn(&[f32], f32, &[f32], usize, &mut [f32])) { + for dimension in [8_usize, 16, 40] { + let x: Vec = (0..dimension).map(|i| (i as f32) * 0.5 + 1.0).collect(); + let x_norm = norm_l2(&x); + let num_vectors = 3; + let batch: Vec = (0..dimension * num_vectors) + .map(|i| ((i % 7) as f32) + 1.0) + .collect(); + + let mut got = vec![0.0; num_vectors]; + unsafe { kernel(&x, x_norm, &batch, dimension, &mut got) }; + assert_eq!(got.len(), num_vectors); + + let x_f64: Vec = x.iter().map(|&v| v as f64).collect(); + for (chunk, &g) in batch.chunks_exact(dimension).zip(got.iter()) { + let y_f64: Vec = chunk.iter().map(|&v| v as f64).collect(); + let expected = cosine_fast_scalar(&x_f64, x_norm, &y_f64); + assert_relative_eq!(g, expected, max_relative = 1e-3, epsilon = 1e-6); + } + } + } + + /// AVX + FMA batch kernel parity (AVX2 / AVX+FMA tiers). Runs on any + /// Haswell-or-newer host; early-returns without AVX and FMA. + #[cfg(target_arch = "x86_64")] + #[test] + fn test_cosine_batch_avx_fma_matches_scalar() { + if !(std::is_x86_feature_detected!("avx") && std::is_x86_feature_detected!("fma")) { + return; + } + check_cosine_batch_kernel(super::f32::cosine_batch_avx_fma); + } + + /// AVX-only batch kernel parity (Sandy Bridge / Ivy Bridge tier). + /// Early-returns on hosts without AVX. + #[cfg(target_arch = "x86_64")] + #[test] + fn test_cosine_batch_avx_matches_scalar() { + if !std::is_x86_feature_detected!("avx") { + return; + } + check_cosine_batch_kernel(super::f32::cosine_batch_avx); + } + + /// AVX-512 batch kernel parity. Early-returns on hosts without AVX-512F. + #[cfg(target_arch = "x86_64")] + #[test] + fn test_cosine_batch_avx512_matches_scalar() { + if !std::is_x86_feature_detected!("avx512f") { + return; + } + check_cosine_batch_kernel(super::f32::cosine_batch_avx512); } } diff --git a/rust/lance-linalg/src/distance/cosine_u8.rs b/rust/lance-linalg/src/distance/cosine_u8.rs index b2d06d35c31..59833e020a7 100644 --- a/rust/lance-linalg/src/distance/cosine_u8.rs +++ b/rust/lance-linalg/src/distance/cosine_u8.rs @@ -199,6 +199,10 @@ fn select_backend() -> CosineU8AccumFn { if is_x86_feature_detected!("avx2") { return |a, b| unsafe { x86::cosine_u8_accum_avx2(a, b) }; } + // AvxFma and Avx hosts (AMD Piledriver / Steamroller, Intel Sandy + // Bridge / Ivy Bridge) fall through to scalar: the AVX2 inner uses + // `vpmaddubsw` / `vpmaddwd` integer ops which neither AVX nor + // AVX+FMA provides. } cosine_u8_accum_scalar diff --git a/rust/lance-linalg/src/distance/dot.rs b/rust/lance-linalg/src/distance/dot.rs index 5903d24e0e5..7d2c3aace0d 100644 --- a/rust/lance-linalg/src/distance/dot.rs +++ b/rust/lance-linalg/src/distance/dot.rs @@ -3,6 +3,11 @@ //! Dot product. +#[cfg(all( + target_arch = "x86_64", + not(all(target_feature = "avx2", target_feature = "fma")) +))] +use std::arch::x86_64::{_mm256_loadu_ps, _mm256_mul_ps}; use std::iter::Sum; use std::ops::AddAssign; use std::sync::Arc; @@ -14,12 +19,21 @@ use arrow_schema::DataType; use half::{bf16, f16}; use lance_arrow::{ArrowFloatType, FixedSizeListArrayExt, FloatArray}; use lance_core::assume_eq; -use lance_core::utils::cpu::SIMD_SUPPORT; -#[cfg(feature = "fp16kernels")] -use lance_core::utils::cpu::SimdSupport; +#[allow(unused_imports)] +use lance_core::utils::cpu::{SIMD_SUPPORT, SimdSupport}; use num_traits::{AsPrimitive, Num, real::Real}; use crate::Result; +#[cfg(all( + target_arch = "x86_64", + not(all(target_feature = "avx2", target_feature = "fma")) +))] +use crate::distance::{BatchIter, BatchKernel, BatchKind, BatchOperation}; +#[cfg(all( + target_arch = "x86_64", + not(all(target_feature = "avx2", target_feature = "fma")) +))] +use crate::simd::x86::hsum256_ps; /// Default implementation of dot product. /// @@ -111,6 +125,24 @@ pub fn dot_distance(from: &[T], to: &[T]) -> f32 { pub trait Dot: Num { /// Dot product. fn dot(x: &[Self], y: &[Self]) -> f32; + + /// Dot product of `x` against each `dimension`-sized vector in `batch`. + /// + /// The default calls [`Dot::dot`] per vector. `f32` overrides it so the + /// SIMD tier is chosen once for the whole batch instead of once per + /// vector — on a build whose baseline already implies AVX2, per-vector + /// dispatch costs more than the kernel it selects. + /// + /// Returns `impl Iterator` rather than a trait object: hot consumers drive + /// this one element at a time, so a `Box` would cost a + /// virtual call per element and an allocation per batch. + fn dot_batch<'a>( + x: &'a [Self], + batch: &'a [Self], + dimension: usize, + ) -> impl Iterator + 'a { + batch.chunks_exact(dimension).map(move |y| Self::dot(x, y)) + } } #[cfg(feature = "fp16kernels")] @@ -161,6 +193,9 @@ impl Dot for bf16 { SimdSupport::Lsx => unsafe { bf16_kernel::dot_bf16_lsx(x.as_ptr(), y.as_ptr(), x.len() as u32) }, + // SimdSupport::AvxFma and SimdSupport::Avx fall through here: + // the bf16 C kernels are compiled with `-march=haswell` minimum + // (AVX2), so they cannot run on AVX-only or AVX+FMA hosts. _ => dot_scalar::(x, y), } } @@ -203,7 +238,7 @@ impl Dot for f16 { kernel::dot_f16_avx512(x.as_ptr(), y.as_ptr(), x.len() as u32) }, #[cfg(all(feature = "fp16kernels", target_arch = "x86_64"))] - SimdSupport::Avx2 => unsafe { + SimdSupport::Avx2 | SimdSupport::Avx512 => unsafe { kernel::dot_f16_avx2(x.as_ptr(), y.as_ptr(), x.len() as u32) }, #[cfg(all(feature = "fp16kernels", target_arch = "loongarch64"))] @@ -214,6 +249,9 @@ impl Dot for f16 { SimdSupport::Lsx => unsafe { kernel::dot_f16_lsx(x.as_ptr(), y.as_ptr(), x.len() as u32) }, + // SimdSupport::AvxFma and SimdSupport::Avx fall through here: + // the f16 C kernels are compiled with `-march=haswell` minimum + // (AVX2), so they cannot run on AVX-only or AVX+FMA hosts. _ => dot_scalar::(x, y), } } @@ -222,10 +260,221 @@ impl Dot for f16 { impl Dot for f32 { #[inline] fn dot(x: &[Self], y: &[Self]) -> f32 { - dot_scalar::(x, y) + // Trait methods cannot carry `#[target_feature]` attributes, so the body + // lives in a free function that runtime-dispatches via `*SIMD_SUPPORT` + // to an AVX2 or AVX-512 inner kernel on capable hosts, or a portable + // scalar fallback. Same shape as the f64 sibling and the existing + // u8 distance kernels in `dot_u8.rs`. + dot_f32_dispatched(x, y) + } + + fn dot_batch<'a>( + x: &'a [Self], + batch: &'a [Self], + dimension: usize, + ) -> impl Iterator + 'a { + // Exactly one arm compiles. Keeping each a tail expression (rather than + // an early `return` guarded by `cfg`) mirrors `dot_f32_dispatched` and + // avoids an unreachable tail on AVX2-baseline builds. + // On an AVX2-baseline build, hoist the tier choice out of the loop, but + // keep the SIMD kernel: the baseline already guarantees avx2+fma, so + // call the AVX+FMA kernel directly rather than re-checking per vector. + // Falling back to the scalar kernel here would lose ~4x at small + // dimensions, which is where batch calls live (PQ sub-vectors are 8 + // wide). + // + // The iterator is a bare `Map`: `Map` is `TrustedLen`, + // so `.collect()` preallocates, and `Map::fold` drives `ChunksExact` in + // one inlined loop. Any wrapper — trait object or enum — loses both. + #[cfg(all( + target_arch = "x86_64", + target_feature = "avx2", + target_feature = "fma" + ))] + { + // See `L2::l2_batch` for f32: below 16 lanes `dot_scalar`'s chunking + // degenerates to a scalar remainder loop, so the explicit AVX kernel + // wins big; above it the autovectorizer is already good and the + // 8-wide kernel can lose, so keep the pre-dispatch kernel exactly. + // + // SAFETY: avx2+fma are enabled for the whole crate by the build + // baseline, so the kernel's `#[target_feature]` contract holds + // statically. + let narrow = dimension <= 16; + batch.chunks_exact(dimension).map(move |y| { + if narrow { + unsafe { x86::dot_f32_avx_fma(x, y) } + } else { + dot_f32_scalar(x, y) + } + }) + } + #[cfg(all( + target_arch = "x86_64", + not(all(target_feature = "avx2", target_feature = "fma")) + ))] + { + dot_batch_f32_runtime_dispatch(x, batch, dimension) + } + #[cfg(not(target_arch = "x86_64"))] + { + batch.chunks_exact(dimension).map(move |y| Self::dot(x, y)) + } + } +} + +/// Sub-AVX2 builds: the scalar kernel cannot reach the wide registers, so pick +/// a `#[target_feature]` kernel — once for the batch, not once per vector. +#[cfg(all( + target_arch = "x86_64", + not(all(target_feature = "avx2", target_feature = "fma")) +))] +#[inline] +fn dot_batch_f32_runtime_dispatch<'a>( + x: &'a [f32], + batch: &'a [f32], + dimension: usize, +) -> impl Iterator + 'a { + let (kernel, kind): (BatchKernel, BatchKind) = match *SIMD_SUPPORT { + // AVX-512 has no useful work for an eight-element vector. Retain the + // AVX/FMA kernel used by the former Haswell baseline for PQ dimensions. + SimdSupport::Avx512 | SimdSupport::Avx512FP16 if dimension > 16 => { + (x86::dot_batch_f32_avx512, BatchKind::Avx512) + } + SimdSupport::Avx512 | SimdSupport::Avx512FP16 if std::is_x86_feature_detected!("fma") => { + (x86::dot_batch_f32_avx_fma, BatchKind::AvxFma) + } + SimdSupport::Avx2 | SimdSupport::AvxFma => (x86::dot_batch_f32_avx_fma, BatchKind::AvxFma), + SimdSupport::Avx512 | SimdSupport::Avx512FP16 | SimdSupport::Avx => { + (x86::dot_batch_f32_avx, BatchKind::Avx) + } + _ => (dot_batch_f32_scalar, BatchKind::Scalar), + }; + + // SAFETY: the runtime tier and the explicit FMA check above establish the + // selected kernel's target-feature contract. + unsafe { BatchIter::::new(x, batch, dimension, kernel, kind) } +} + +#[cfg(all( + target_arch = "x86_64", + not(all(target_feature = "avx2", target_feature = "fma")) +))] +struct DotBatch; + +#[cfg(all( + target_arch = "x86_64", + not(all(target_feature = "avx2", target_feature = "fma")) +))] +impl BatchOperation for DotBatch { + #[inline] + fn fold_scalar(key: &[f32], batch: &[f32], dimension: usize, init: B, mut f: F) -> B + where + F: FnMut(B, f32) -> B, + { + batch + .chunks_exact(dimension) + .fold(init, |acc, vector| f(acc, dot_f32_scalar(key, vector))) + } + + #[target_feature(enable = "avx")] + unsafe fn fold_avx(key: &[f32], batch: &[f32], dimension: usize, init: B, mut f: F) -> B + where + F: FnMut(B, f32) -> B, + { + if dimension == 8 { + let key_values = unsafe { _mm256_loadu_ps(key.as_ptr()) }; + return batch.chunks_exact(8).fold(init, |acc, vector| { + let vector_values = unsafe { _mm256_loadu_ps(vector.as_ptr()) }; + let product = _mm256_mul_ps(key_values, vector_values); + f(acc, unsafe { hsum256_ps(product) }) + }); + } + batch.chunks_exact(dimension).fold(init, |acc, vector| { + f(acc, unsafe { x86::dot_f32_avx(key, vector) }) + }) + } + + #[target_feature(enable = "avx,fma")] + unsafe fn fold_avx_fma( + key: &[f32], + batch: &[f32], + dimension: usize, + init: B, + mut f: F, + ) -> B + where + F: FnMut(B, f32) -> B, + { + if dimension == 8 { + let key_values = unsafe { _mm256_loadu_ps(key.as_ptr()) }; + return batch.chunks_exact(8).fold(init, |acc, vector| { + let vector_values = unsafe { _mm256_loadu_ps(vector.as_ptr()) }; + let product = _mm256_mul_ps(key_values, vector_values); + f(acc, unsafe { hsum256_ps(product) }) + }); + } + batch.chunks_exact(dimension).fold(init, |acc, vector| { + f(acc, unsafe { x86::dot_f32_avx_fma(key, vector) }) + }) + } + + #[target_feature(enable = "avx512f")] + unsafe fn fold_avx512( + key: &[f32], + batch: &[f32], + dimension: usize, + init: B, + mut f: F, + ) -> B + where + F: FnMut(B, f32) -> B, + { + batch.chunks_exact(dimension).fold(init, |acc, vector| { + f(acc, unsafe { x86::dot_f32_avx512(key, vector) }) + }) + } +} + +#[cfg(all( + target_arch = "x86_64", + not(all(target_feature = "avx2", target_feature = "fma")) +))] +unsafe fn dot_batch_f32_scalar(x: &[f32], batch: &[f32], dimension: usize, output: &mut [f32]) { + debug_assert_eq!(output.len(), batch.len() / dimension); + for (distance, y) in output.iter_mut().zip(batch.chunks_exact(dimension)) { + *distance = dot_f32_scalar(x, y); + } +} + +/// Dot product for f32, runtime-dispatched via `SIMD_SUPPORT` on x86_64 +/// (AVX-512 / AVX2+FMA / AVX+FMA / AVX / scalar). Non-x86 uses the +/// auto-vectorised scalar loop. +#[inline] +fn dot_f32_dispatched(x: &[f32], y: &[f32]) -> f32 { + #[cfg(target_arch = "x86_64")] + { + match *SIMD_SUPPORT { + SimdSupport::Avx512 | SimdSupport::Avx512FP16 => unsafe { x86::dot_f32_avx512(x, y) }, + SimdSupport::Avx2 | SimdSupport::AvxFma => unsafe { x86::dot_f32_avx_fma(x, y) }, + SimdSupport::Avx => unsafe { x86::dot_f32_avx(x, y) }, + _ => dot_f32_scalar(x, y), + } + } + #[cfg(not(target_arch = "x86_64"))] + { + dot_f32_scalar(x, y) } } +/// Portable scalar dot product for f32. Used as the x86_64 fallback when no +/// AVX2 is detected, and as the only path on non-x86 architectures. The +/// `LANES = 16` chunking matches the explicit-SIMD inner kernels above. +#[inline] +fn dot_f32_scalar(x: &[f32], y: &[f32]) -> f32 { + dot_scalar::(x, y) +} + impl Dot for f64 { #[inline] fn dot(x: &[Self], y: &[Self]) -> f32 { @@ -233,9 +482,256 @@ impl Dot for f64 { } } -/// Explicit SIMD dot product for f64. +/// Dot product for f64, runtime-dispatched via `SIMD_SUPPORT` on x86_64 +/// (AVX-512 / AVX2+FMA / AVX+FMA / AVX / scalar). Non-x86 uses the SIMD +/// primitives in `crate::simd::f64`, unconditionally backed by NEON / LSX-LASX. #[inline] fn dot_f64_simd(x: &[f64], y: &[f64]) -> f32 { + #[cfg(target_arch = "x86_64")] + { + match *SIMD_SUPPORT { + SimdSupport::Avx512 | SimdSupport::Avx512FP16 => unsafe { x86::dot_f64_avx512(x, y) }, + SimdSupport::Avx2 | SimdSupport::AvxFma => unsafe { x86::dot_f64_avx_fma(x, y) }, + SimdSupport::Avx => unsafe { x86::dot_f64_avx(x, y) }, + _ => dot_f64_scalar(x, y), + } + } + #[cfg(not(target_arch = "x86_64"))] + { + dot_f64_simd_other(x, y) + } +} + +/// Portable scalar dot product for f64. Used as the x86_64 fallback when no +/// AVX2 is detected, and exposed for cross-backend parity testing. +#[cfg(target_arch = "x86_64")] +#[inline] +fn dot_f64_scalar(x: &[f64], y: &[f64]) -> f32 { + x.iter().zip(y.iter()).map(|(&a, &b)| a * b).sum::() as f32 +} + +#[cfg(target_arch = "x86_64")] +mod x86 { + use std::arch::x86_64::*; + + use crate::simd::f64::{f64x4, f64x8}; + use crate::simd::x86::hsum256_ps; + use crate::simd::{FloatSimd, SIMD}; + + /// Dot product of `x` against every `dimension`-sized vector in `batch`, + /// entering the AVX-512 tier once for the whole batch rather than once per + /// vector. + /// + /// # Safety + /// The host must support AVX-512F. + /// + /// Only compiled for builds whose baseline is below avx2+fma; at or above + /// that baseline `dot_batch` inlines the kernel directly and never runtime- + /// dispatches, so this wrapper would be dead code (see `dot_batch`). + #[cfg(not(all(target_feature = "avx2", target_feature = "fma")))] + #[target_feature(enable = "avx512f")] + pub(super) unsafe fn dot_batch_f32_avx512( + x: &[f32], + batch: &[f32], + dimension: usize, + output: &mut [f32], + ) { + debug_assert_eq!(output.len(), batch.len() / dimension); + for (distance, y) in output.iter_mut().zip(batch.chunks_exact(dimension)) { + *distance = unsafe { dot_f32_avx512(x, y) }; + } + } + + /// As [`dot_batch_f32_avx512`], for the AVX2 and AVX+FMA tiers. + /// + /// # Safety + /// The host must support AVX and FMA. + #[cfg(not(all(target_feature = "avx2", target_feature = "fma")))] + #[target_feature(enable = "avx,fma")] + pub(super) unsafe fn dot_batch_f32_avx_fma( + x: &[f32], + batch: &[f32], + dimension: usize, + output: &mut [f32], + ) { + debug_assert_eq!(output.len(), batch.len() / dimension); + for (distance, y) in output.iter_mut().zip(batch.chunks_exact(dimension)) { + *distance = unsafe { dot_f32_avx_fma(x, y) }; + } + } + + /// As [`dot_batch_f32_avx512`], for the AVX-without-FMA tier. + /// + /// # Safety + /// The host must support AVX. + #[cfg(not(all(target_feature = "avx2", target_feature = "fma")))] + #[target_feature(enable = "avx")] + pub(super) unsafe fn dot_batch_f32_avx( + x: &[f32], + batch: &[f32], + dimension: usize, + output: &mut [f32], + ) { + debug_assert_eq!(output.len(), batch.len() / dimension); + for (distance, y) in output.iter_mut().zip(batch.chunks_exact(dimension)) { + *distance = unsafe { dot_f32_avx(x, y) }; + } + } + + /// AVX-512 path for f64: 8-wide `__m512d` with `vfmadd231pd` per iteration. + #[target_feature(enable = "avx512f")] + pub unsafe fn dot_f64_avx512(x: &[f64], y: &[f64]) -> f32 { + let dim = x.len(); + let unrolled_len = dim / 8 * 8; + + let mut acc = _mm512_setzero_pd(); + for i in (0..unrolled_len).step_by(8) { + let a = _mm512_loadu_pd(x.as_ptr().add(i)); + let b = _mm512_loadu_pd(y.as_ptr().add(i)); + acc = _mm512_fmadd_pd(a, b, acc); + } + + let tail: f64 = x[unrolled_len..] + .iter() + .zip(y[unrolled_len..].iter()) + .map(|(&a, &b)| a * b) + .sum(); + + (_mm512_reduce_add_pd(acc) + tail) as f32 + } + + /// AVX + FMA path for f64. Covers both AvxFma and AVX2 dispatch (body uses no AVX2-specific intrinsics). + #[target_feature(enable = "avx,fma")] + pub unsafe fn dot_f64_avx_fma(x: &[f64], y: &[f64]) -> f32 { + let dim = x.len(); + let unrolled_len = dim / 8 * 8; + + let mut acc8 = f64x8::zeros(); + for i in (0..unrolled_len).step_by(8) { + let a = f64x8::load_unaligned(x.as_ptr().add(i)); + let b = f64x8::load_unaligned(y.as_ptr().add(i)); + acc8.multiply_add(a, b); + } + + let aligned_len = dim / 4 * 4; + let mut acc4 = f64x4::zeros(); + for i in (unrolled_len..aligned_len).step_by(4) { + let a = f64x4::load_unaligned(x.as_ptr().add(i)); + let b = f64x4::load_unaligned(y.as_ptr().add(i)); + acc4.multiply_add(a, b); + } + + let tail: f64 = x[aligned_len..] + .iter() + .zip(y[aligned_len..].iter()) + .map(|(&a, &b)| a * b) + .sum(); + + (acc8.reduce_sum() + acc4.reduce_sum() + tail) as f32 + } + + /// AVX-only path for f64 (no FMA): `_mm256_mul_pd` + `_mm256_add_pd` per iteration for Sandy/Ivy Bridge. + #[target_feature(enable = "avx")] + pub unsafe fn dot_f64_avx(x: &[f64], y: &[f64]) -> f32 { + let dim = x.len(); + let unrolled_len = dim / 4 * 4; + + let mut acc = _mm256_setzero_pd(); + for i in (0..unrolled_len).step_by(4) { + let a = _mm256_loadu_pd(x.as_ptr().add(i)); + let b = _mm256_loadu_pd(y.as_ptr().add(i)); + acc = _mm256_add_pd(acc, _mm256_mul_pd(a, b)); + } + + // Horizontal sum of __m256d -> f64. Two pairwise adds across lanes. + let lo = _mm256_castpd256_pd128(acc); + let hi = _mm256_extractf128_pd(acc, 1); + let sum128 = _mm_add_pd(lo, hi); + let sum64 = _mm_add_pd(sum128, _mm_unpackhi_pd(sum128, sum128)); + let acc_sum = _mm_cvtsd_f64(sum64); + + let tail: f64 = x[unrolled_len..] + .iter() + .zip(y[unrolled_len..].iter()) + .map(|(&a, &b)| a * b) + .sum(); + + (acc_sum + tail) as f32 + } + + /// AVX-512 path for f32: 16-wide `__m512` with `vfmadd231ps` per iteration. + #[inline] + #[target_feature(enable = "avx512f")] + pub unsafe fn dot_f32_avx512(x: &[f32], y: &[f32]) -> f32 { + let dim = x.len(); + let unrolled_len = dim / 16 * 16; + + let mut acc = _mm512_setzero_ps(); + for i in (0..unrolled_len).step_by(16) { + let a = _mm512_loadu_ps(x.as_ptr().add(i)); + let b = _mm512_loadu_ps(y.as_ptr().add(i)); + acc = _mm512_fmadd_ps(a, b, acc); + } + + let tail: f32 = x[unrolled_len..] + .iter() + .zip(y[unrolled_len..].iter()) + .map(|(&a, &b)| a * b) + .sum(); + + _mm512_reduce_add_ps(acc) + tail + } + + /// AVX + FMA path for f32. Covers both AvxFma and AVX2 dispatch (body uses no AVX2-specific intrinsics). + #[inline] + #[target_feature(enable = "avx,fma")] + pub unsafe fn dot_f32_avx_fma(x: &[f32], y: &[f32]) -> f32 { + let dim = x.len(); + let unrolled_len = dim / 8 * 8; + + let mut acc = _mm256_setzero_ps(); + for i in (0..unrolled_len).step_by(8) { + let a = _mm256_loadu_ps(x.as_ptr().add(i)); + let b = _mm256_loadu_ps(y.as_ptr().add(i)); + acc = _mm256_fmadd_ps(a, b, acc); + } + + let tail: f32 = x[unrolled_len..] + .iter() + .zip(y[unrolled_len..].iter()) + .map(|(&a, &b)| a * b) + .sum(); + + hsum256_ps(acc) + tail + } + + /// AVX-only path for f32 (no FMA): `_mm256_mul_ps` + `_mm256_add_ps` per iteration for Sandy/Ivy Bridge. + #[inline] + #[target_feature(enable = "avx")] + pub unsafe fn dot_f32_avx(x: &[f32], y: &[f32]) -> f32 { + let dim = x.len(); + let unrolled_len = dim / 8 * 8; + + let mut acc = _mm256_setzero_ps(); + for i in (0..unrolled_len).step_by(8) { + let a = _mm256_loadu_ps(x.as_ptr().add(i)); + let b = _mm256_loadu_ps(y.as_ptr().add(i)); + acc = _mm256_add_ps(acc, _mm256_mul_ps(a, b)); + } + + let tail: f32 = x[unrolled_len..] + .iter() + .zip(y[unrolled_len..].iter()) + .map(|(&a, &b)| a * b) + .sum(); + + hsum256_ps(acc) + tail + } +} + +#[cfg(not(target_arch = "x86_64"))] +#[inline] +fn dot_f64_simd_other(x: &[f64], y: &[f64]) -> f32 { use crate::simd::f64::{f64x4, f64x8}; use crate::simd::{FloatSimd, SIMD}; @@ -285,7 +781,7 @@ pub fn dot_distance_batch<'a, T: Dot>( ) -> Box + 'a> { assume_eq!(from.len(), dimension); assume_eq!(to.len() % dimension, 0); - Box::new(to.chunks_exact(dimension).map(|v| dot_distance(from, v))) + Box::new(T::dot_batch(from, to, dimension).map(|d| 1.0 - d)) } fn do_dot_distance_arrow_batch( @@ -309,10 +805,9 @@ where to.value_type() )))?; - let dists = to_values - .as_slice() - .chunks_exact(dimension) - .map(|v| dot_distance(from.as_slice(), v)); + // Route through `dot_distance_batch` rather than mapping `dot_distance` per + // vector, so this entry point gets the same hoisted dispatch. + let dists = dot_distance_batch(from.as_slice(), to_values.as_slice(), dimension); Ok(Arc::new(Float32Array::new( dists.collect(), @@ -480,5 +975,265 @@ mod tests { fn test_dot_f64((x, y) in arbitrary_vector_pair(arbitrary_f64, 4..4048)){ do_dot_test(&x, &y)?; } + + /// Cross-backend parity: scalar fallback must match the dispatched + /// SIMD path within numerical tolerance. Exercises `dot_f64_scalar` + /// directly so the runtime fallback is exercised even on AVX2-capable + /// CI hosts. + #[cfg(target_arch = "x86_64")] + #[test] + fn test_dot_f64_scalar_simd_parity( + (x, y) in arbitrary_vector_pair(arbitrary_f64, 4..4048) + ) { + let scalar = dot_f64_scalar(&x, &y); + let simd = dot_f64_simd(&x, &y); + let max_error = max_error::(&x, &y); + prop_assert!(approx::relative_eq!(scalar, simd, epsilon = max_error)); + } + + /// Parity check for `dot_f32_dispatched` (Branch B exclusive: the + /// auto-vectorised scalar dot path). The dispatched kernel must + /// agree with a portable f64-precision scalar reference within + /// numerical tolerance. The reference is hand-rolled here to keep + /// this test architecture-agnostic (the x86_64-only `dot_f64_scalar` + /// helper is gated above). + #[test] + fn test_dot_f32_scalar_simd_parity( + (x, y) in arbitrary_vector_pair(arbitrary_f32, 4..4048) + ) { + let x_f64: Vec = x.iter().map(|&v| v as f64).collect(); + let y_f64: Vec = y.iter().map(|&v| v as f64).collect(); + let scalar = x_f64 + .iter() + .zip(y_f64.iter()) + .map(|(&a, &b)| a * b) + .sum::() as f32; + let simd = ::dot(&x, &y); + let max_error = max_error::(&x_f64, &y_f64); + prop_assert!(approx::relative_eq!(scalar, simd, epsilon = max_error)); + } + + /// AVX-512-direct parity for f32: explicitly compares the scalar + /// fallback against the native f32 AVX-512 inner kernel on + /// AVX-512F-capable hosts. Early-returns on hosts without AVX-512F. + #[cfg(target_arch = "x86_64")] + #[test] + fn test_dot_f32_scalar_vs_avx512_parity( + (x, y) in arbitrary_vector_pair(arbitrary_f32, 4..4048) + ) { + if !std::is_x86_feature_detected!("avx512f") { + return Ok(()); + } + let scalar = dot_f32_scalar(&x, &y); + let avx512 = unsafe { x86::dot_f32_avx512(&x, &y) }; + let x_f64: Vec = x.iter().map(|&v| v as f64).collect(); + let y_f64: Vec = y.iter().map(|&v| v as f64).collect(); + let max_error = max_error::(&x_f64, &y_f64); + prop_assert!(approx::relative_eq!(scalar, avx512, epsilon = max_error)); + } + + /// AVX + FMA-direct parity for the f32 dot kernel. Covers the AMD + /// Piledriver / Steamroller / FX-7500 tier. Early-returns on hosts + /// without both AVX and FMA. + #[cfg(target_arch = "x86_64")] + #[test] + fn test_dot_f32_scalar_vs_avx_fma_parity( + (x, y) in arbitrary_vector_pair(arbitrary_f32, 4..4048) + ) { + if !(std::is_x86_feature_detected!("avx") && std::is_x86_feature_detected!("fma")) { + return Ok(()); + } + let scalar = dot_f32_scalar(&x, &y); + let avx_fma = unsafe { x86::dot_f32_avx_fma(&x, &y) }; + let x_f64: Vec = x.iter().map(|&v| v as f64).collect(); + let y_f64: Vec = y.iter().map(|&v| v as f64).collect(); + let max_error = max_error::(&x_f64, &y_f64); + prop_assert!(approx::relative_eq!(scalar, avx_fma, epsilon = max_error)); + } + + /// AVX-only-direct parity for the f32 dot kernel. Covers the Intel + /// Sandy Bridge / Ivy Bridge tier. Early-returns on hosts without + /// AVX. + #[cfg(target_arch = "x86_64")] + #[test] + fn test_dot_f32_scalar_vs_avx_parity( + (x, y) in arbitrary_vector_pair(arbitrary_f32, 4..4048) + ) { + if !std::is_x86_feature_detected!("avx") { + return Ok(()); + } + let scalar = dot_f32_scalar(&x, &y); + let avx = unsafe { x86::dot_f32_avx(&x, &y) }; + let x_f64: Vec = x.iter().map(|&v| v as f64).collect(); + let y_f64: Vec = y.iter().map(|&v| v as f64).collect(); + let max_error = max_error::(&x_f64, &y_f64); + prop_assert!(approx::relative_eq!(scalar, avx, epsilon = max_error)); + } + + /// AVX-512-direct parity: explicitly compares the scalar fallback + /// against the native AVX-512 inner kernel on AVX-512F-capable hosts + /// (Skylake-X+, Ice Lake, Sapphire Rapids, Zen 4). Early-returns on + /// hosts without AVX-512F. + #[cfg(target_arch = "x86_64")] + #[test] + fn test_dot_f64_scalar_vs_avx512_parity( + (x, y) in arbitrary_vector_pair(arbitrary_f64, 4..4048) + ) { + if !std::is_x86_feature_detected!("avx512f") { + return Ok(()); + } + let scalar = dot_f64_scalar(&x, &y); + let avx512 = unsafe { x86::dot_f64_avx512(&x, &y) }; + let max_error = max_error::(&x, &y); + prop_assert!(approx::relative_eq!(scalar, avx512, epsilon = max_error)); + } + + /// AVX + FMA-direct parity for the f64 dot kernel. Covers the AMD + /// Piledriver / Steamroller / FX-7500 tier. Early-returns on hosts + /// without both AVX and FMA. + #[cfg(target_arch = "x86_64")] + #[test] + fn test_dot_f64_scalar_vs_avx_fma_parity( + (x, y) in arbitrary_vector_pair(arbitrary_f64, 4..4048) + ) { + if !(std::is_x86_feature_detected!("avx") && std::is_x86_feature_detected!("fma")) { + return Ok(()); + } + let scalar = dot_f64_scalar(&x, &y); + let avx_fma = unsafe { x86::dot_f64_avx_fma(&x, &y) }; + let max_error = max_error::(&x, &y); + prop_assert!(approx::relative_eq!(scalar, avx_fma, epsilon = max_error)); + } + + /// AVX-only-direct parity for the f64 dot kernel. Covers the Intel + /// Sandy Bridge / Ivy Bridge tier (AVX without FMA). Early-returns + /// on hosts without AVX. + #[cfg(target_arch = "x86_64")] + #[test] + fn test_dot_f64_scalar_vs_avx_parity( + (x, y) in arbitrary_vector_pair(arbitrary_f64, 4..4048) + ) { + if !std::is_x86_feature_detected!("avx") { + return Ok(()); + } + let scalar = dot_f64_scalar(&x, &y); + let avx = unsafe { x86::dot_f64_avx(&x, &y) }; + let max_error = max_error::(&x, &y); + prop_assert!(approx::relative_eq!(scalar, avx, epsilon = max_error)); + } + } + + /// `dot_batch` must agree with the per-vector `dot` it replaced, on every + /// build: AVX2-baseline, hoisted-dispatch, and portable fallback all + /// funnel through here. + #[rstest::rstest] + #[case::dim_8(8)] + #[case::dim_16(16)] + #[case::dim_32(32)] + #[case::dim_1024(1024)] + fn test_dot_batch_f32_matches_per_vector_dot(#[case] dimension: usize) { + let num_vectors = 5; + let x: Vec = (0..dimension) + .map(|i| ((i % 13) as f32) * 0.25 + 1.0) + .collect(); + let batch: Vec = (0..dimension * num_vectors) + .map(|i| ((i % 11) as f32) * 0.5 - 2.0) + .collect(); + + let got: Vec = f32::dot_batch(&x, &batch, dimension).collect(); + let want: Vec = batch + .chunks_exact(dimension) + .map(|y| f32::dot(&x, y)) + .collect(); + + assert_eq!(got.len(), num_vectors); + for (g, w) in got.iter().zip(want.iter()) { + assert!( + approx::relative_eq!(g, w, epsilon = 1e-4), + "dim {dimension}: batch {g} != per-vector {w}" + ); + } + } + + /// `dot_distance_batch` still yields `1.0 - dot`, unchanged by the hoist. + #[test] + fn test_dot_distance_batch_preserves_distance_semantics() { + let dimension = 32; + let x: Vec = (0..dimension).map(|i| (i as f32) * 0.1).collect(); + let batch: Vec = (0..dimension * 3).map(|i| (i as f32) * 0.05).collect(); + + let got: Vec = dot_distance_batch(&x, &batch, dimension).collect(); + for (chunk, &g) in batch.chunks_exact(dimension).zip(got.iter()) { + assert!(approx::relative_eq!( + g, + 1.0 - f32::dot(&x, chunk), + epsilon = 1e-5 + )); + } + } + + /// The per-batch `#[target_feature]` kernels are only reached on sub-AVX2 + /// builds or AVX-512 hosts, so call them directly to cover them. + #[cfg(all( + target_arch = "x86_64", + not(all(target_feature = "avx2", target_feature = "fma")) + ))] + fn check_dot_batch_kernel(kernel: BatchKernel) { + for dimension in [8_usize, 16, 40] { + let num_vectors = 3; + let x: Vec = (0..dimension).map(|i| (i as f32) * 0.5 + 1.0).collect(); + let batch: Vec = (0..dimension * num_vectors) + .map(|i| ((i % 7) as f32) + 1.0) + .collect(); + + let mut got = vec![0.0; num_vectors]; + unsafe { kernel(&x, &batch, dimension, &mut got) }; + assert_eq!(got.len(), num_vectors); + for (chunk, &g) in batch.chunks_exact(dimension).zip(got.iter()) { + let want = dot_scalar::(&x, chunk); + assert!( + approx::relative_eq!(g, want, epsilon = 1e-4), + "dim {dimension}: kernel {g} != scalar {want}" + ); + } + } + } + + // The runtime-dispatch batch kernels only exist in sub-avx2+fma builds + // (see `x86::dot_batch_f32_avx512`), so gate their tests the same way. + #[cfg(all( + target_arch = "x86_64", + not(all(target_feature = "avx2", target_feature = "fma")) + ))] + #[test] + fn test_dot_batch_avx_fma_matches_scalar() { + if !std::is_x86_feature_detected!("avx") || !std::is_x86_feature_detected!("fma") { + return; + } + check_dot_batch_kernel(x86::dot_batch_f32_avx_fma); + } + + #[cfg(all( + target_arch = "x86_64", + not(all(target_feature = "avx2", target_feature = "fma")) + ))] + #[test] + fn test_dot_batch_avx_matches_scalar() { + if !std::is_x86_feature_detected!("avx") { + return; + } + check_dot_batch_kernel(x86::dot_batch_f32_avx); + } + + #[cfg(all( + target_arch = "x86_64", + not(all(target_feature = "avx2", target_feature = "fma")) + ))] + #[test] + fn test_dot_batch_avx512_matches_scalar() { + if !std::is_x86_feature_detected!("avx512f") { + return; + } + check_dot_batch_kernel(x86::dot_batch_f32_avx512); } } diff --git a/rust/lance-linalg/src/distance/dot_u8.rs b/rust/lance-linalg/src/distance/dot_u8.rs index de5522cddfe..7b2e335f094 100644 --- a/rust/lance-linalg/src/distance/dot_u8.rs +++ b/rust/lance-linalg/src/distance/dot_u8.rs @@ -134,6 +134,10 @@ fn select_backend() -> DotU8Fn { if is_x86_feature_detected!("avx2") { return |a, b| unsafe { x86::dot_u8_avx2(a, b) }; } + // AvxFma and Avx hosts (AMD Piledriver / Steamroller, Intel Sandy + // Bridge / Ivy Bridge) fall through to scalar: the AVX2 inner uses + // `vpmaddubsw` / `vpmaddwd` integer ops which neither AVX nor + // AVX+FMA provides. } dot_u8_scalar diff --git a/rust/lance-linalg/src/distance/hamming.rs b/rust/lance-linalg/src/distance/hamming.rs index a6f4b038195..1eeb8e42c38 100644 --- a/rust/lance-linalg/src/distance/hamming.rs +++ b/rust/lance-linalg/src/distance/hamming.rs @@ -4,7 +4,7 @@ //! Hamming distance. //! //! This module provides hamming distance computation for binary vectors, -//! including SIMD-accelerated pairwise hamming distance for 64-bit hashes. +//! including SIMD-accelerated pairwise hamming distance for binary hashes. use std::collections::HashMap; use std::sync::Arc; @@ -102,6 +102,196 @@ pub fn hamming_u64(a: u64, b: u64) -> u32 { (a ^ b).count_ones() } +/// Binary hash values stored as 64-bit lanes in lane-major order. +/// +/// For a hash width of `N` bytes, `N` must be a positive multiple of 8 and the +/// number of lanes is `N / 8`. Lane-major layout keeps each lane contiguous for +/// all rows, which allows the pairwise path to reuse the SIMD `u64` batch +/// implementation for wider hashes. +#[derive(Debug, Clone)] +pub struct BinaryHashValues { + lane_values: Vec, + num_rows: usize, + byte_width: usize, +} + +impl BinaryHashValues { + /// Create hash values from lane-major `u64` values. + pub fn try_new(lane_values: Vec, num_rows: usize, byte_width: usize) -> Result { + let num_lanes = validate_hash_byte_width(byte_width)?; + let expected_values = checked_lane_value_count(num_rows, num_lanes)?; + if lane_values.len() != expected_values { + return Err(Error::InvalidArgumentError(format!( + "Expected {} lane values for {} rows and {} byte hashes, got {}", + expected_values, + num_rows, + byte_width, + lane_values.len() + ))); + } + Ok(Self { + lane_values, + num_rows, + byte_width, + }) + } + + /// Extract binary hash values from a `FixedSizeList` Arrow array. + pub fn from_fixed_size_list(array: &FixedSizeListArray) -> Result { + let byte_width = usize::try_from(array.value_length()).map_err(|_| { + Error::InvalidArgumentError(format!( + "Expected FixedSizeList with a positive size that is a multiple of 8 bytes, got size {}", + array.value_length() + )) + })?; + let num_lanes = validate_hash_byte_width(byte_width)?; + + let values = array + .values() + .as_any() + .downcast_ref::() + .ok_or_else(|| { + Error::InvalidArgumentError( + "Expected UInt8Array values in FixedSizeList".to_string(), + ) + })?; + + let num_rows = array.len(); + let value_count = checked_lane_value_count(num_rows, num_lanes)?; + let expected_bytes = checked_hash_byte_count(num_rows, byte_width)?; + let bytes = values.values(); + if bytes.len() != expected_bytes { + return Err(Error::InvalidArgumentError(format!( + "Expected {} bytes for {} rows and {} byte hashes, got {}", + expected_bytes, + num_rows, + byte_width, + bytes.len() + ))); + } + + let mut lane_values = vec![0u64; value_count]; + for row in 0..num_rows { + let row_start = row * byte_width; + for lane in 0..num_lanes { + let start = row_start + lane * 8; + let mut arr = [0u8; 8]; + arr.copy_from_slice(&bytes[start..start + 8]); + lane_values[lane * num_rows + row] = u64::from_le_bytes(arr); + } + } + + Ok(Self { + lane_values, + num_rows, + byte_width, + }) + } + + /// Concatenate chunks with the same hash width into a single lane-major set. + pub fn concat(chunks: &[Self]) -> Result { + let Some(first) = chunks.first() else { + return Err(Error::InvalidArgumentError( + "Cannot concatenate zero binary hash chunks".to_string(), + )); + }; + + let byte_width = first.byte_width; + let num_lanes = first.num_lanes(); + let mut num_rows = 0usize; + for chunk in chunks { + if chunk.byte_width != byte_width { + return Err(Error::InvalidArgumentError(format!( + "Cannot concatenate binary hash chunks with different widths: {} and {} bytes", + byte_width, chunk.byte_width + ))); + } + num_rows = num_rows.checked_add(chunk.num_rows).ok_or_else(|| { + Error::InvalidArgumentError( + "Binary hash row count overflowed while concatenating chunks".to_string(), + ) + })?; + } + + let value_count = checked_lane_value_count(num_rows, num_lanes)?; + let mut lane_values = vec![0u64; value_count]; + let mut row_offset = 0; + for chunk in chunks { + for lane in 0..num_lanes { + let dest_start = lane * num_rows + row_offset; + let dest_end = dest_start + chunk.num_rows; + lane_values[dest_start..dest_end].copy_from_slice(chunk.lane(lane)); + } + row_offset += chunk.num_rows; + } + + Ok(Self { + lane_values, + num_rows, + byte_width, + }) + } + + pub fn len(&self) -> usize { + self.num_rows + } + + pub fn is_empty(&self) -> bool { + self.num_rows == 0 + } + + pub fn byte_width(&self) -> usize { + self.byte_width + } + + pub fn num_lanes(&self) -> usize { + self.byte_width / 8 + } + + pub fn lane(&self, lane: usize) -> &[u64] { + let start = lane * self.num_rows; + &self.lane_values[start..start + self.num_rows] + } + + pub fn into_u64_values(self) -> Result> { + if self.num_lanes() != 1 { + return Err(Error::InvalidArgumentError(format!( + "Expected 8-byte binary hashes, got {} byte hashes", + self.byte_width + ))); + } + Ok(self.lane_values) + } +} + +fn checked_lane_value_count(num_rows: usize, num_lanes: usize) -> Result { + num_rows.checked_mul(num_lanes).ok_or_else(|| { + Error::InvalidArgumentError(format!( + "Binary hash lane value count overflowed for {} rows and {} lanes", + num_rows, num_lanes + )) + }) +} + +fn checked_hash_byte_count(num_rows: usize, byte_width: usize) -> Result { + num_rows.checked_mul(byte_width).ok_or_else(|| { + Error::InvalidArgumentError(format!( + "Binary hash byte count overflowed for {} rows and {} byte hashes", + num_rows, byte_width + )) + }) +} + +fn validate_hash_byte_width(byte_width: usize) -> Result { + if byte_width == 0 || !byte_width.is_multiple_of(8) { + return Err(Error::InvalidArgumentError(format!( + "Expected FixedSizeList with a positive size that is a multiple of 8 bytes, got size {}", + byte_width + ))); + } + Ok(byte_width / 8) +} + /// Result of pairwise hamming distance computation. #[derive(Debug, Clone)] pub struct PairwiseResult { @@ -386,6 +576,88 @@ pub fn pairwise_hamming_distance_parallel( combined } +/// Compute pairwise hamming distances for all pairs of fixed-width binary hashes. +/// +/// This supports any hash width that is a positive multiple of 8 bytes. For +/// 8-byte hashes this delegates to the existing `u64` implementation. +pub fn pairwise_hamming_distance_binary( + hashes: &BinaryHashValues, + row_ids: Option<&[u64]>, + threshold: Option, +) -> PairwiseResult { + let n = hashes.len(); + if n < 2 { + return PairwiseResult::new(); + } + + if hashes.num_lanes() == 1 { + return pairwise_hamming_distance(hashes.lane(0), row_ids, threshold); + } + + let threshold = threshold.unwrap_or(u32::MAX); + let num_pairs = n * (n - 1) / 2; + let mut result = PairwiseResult::with_capacity(num_pairs.min(1_000_000)); + + for i in 0..n { + for j in (i + 1)..n { + let mut dist = 0; + for lane in 0..hashes.num_lanes() { + let lane_values = hashes.lane(lane); + dist += hamming_u64(lane_values[i], lane_values[j]); + } + if dist <= threshold { + let id_a = row_ids.map_or(i as u64, |ids| ids[i]); + let id_b = row_ids.map_or(j as u64, |ids| ids[j]); + result.push(id_a, id_b, dist); + } + } + } + + result +} + +/// Compute pairwise hamming distances in parallel for fixed-width binary hashes. +/// +/// Wider hashes reuse the SIMD `u64` batch implementation lane-by-lane. +pub fn pairwise_hamming_distance_binary_parallel( + hashes: &BinaryHashValues, + row_ids: Option<&[u64]>, + threshold: Option, +) -> PairwiseResult { + let n = hashes.len(); + if n < 2 { + return PairwiseResult::new(); + } + + if hashes.num_lanes() == 1 { + return pairwise_hamming_distance_parallel(hashes.lane(0), row_ids, threshold); + } + + let threshold = threshold.unwrap_or(u32::MAX); + let total_pairs = n * (n - 1) / 2; + + if total_pairs < 10_000 { + return pairwise_hamming_distance_binary(hashes, row_ids, Some(threshold)); + } + + let threads = rayon::current_num_threads(); + let pairs_per_chunk = total_pairs.div_ceil(threads); + let chunks = compute_balanced_chunks(n, pairs_per_chunk); + + let results: Vec = chunks + .into_par_iter() + .map(|(start_row, end_row)| { + process_row_range_binary(hashes, row_ids, threshold, start_row, end_row) + }) + .collect(); + + let mut combined = PairwiseResult::new(); + for r in results { + combined.extend(r); + } + combined +} + /// Compute balanced chunks for parallel processing. fn compute_balanced_chunks(n: usize, target_pairs_per_chunk: usize) -> Vec<(usize, usize)> { let mut chunks = Vec::new(); @@ -439,36 +711,80 @@ fn process_row_range( result } +/// Process a range of rows for pairwise comparison of wider binary hashes. +fn process_row_range_binary( + hashes: &BinaryHashValues, + row_ids: Option<&[u64]>, + threshold: u32, + start_row: usize, + end_row: usize, +) -> PairwiseResult { + let n = hashes.len(); + let mut result = PairwiseResult::new(); + let mut distances = Vec::new(); + let mut lane_distances = Vec::new(); + + for i in start_row..end_row { + let remaining = n - i - 1; + if remaining == 0 { + continue; + } + + distances.clear(); + distances.resize(remaining, 0); + + let first_lane = hashes.lane(0); + hamming_batch_u64( + first_lane[i], + &first_lane[i + 1..], + distances.as_mut_slice(), + ); + + for lane in 1..hashes.num_lanes() { + lane_distances.clear(); + lane_distances.resize(remaining, 0); + let lane_values = hashes.lane(lane); + hamming_batch_u64( + lane_values[i], + &lane_values[i + 1..], + lane_distances.as_mut_slice(), + ); + for (distance, lane_distance) in distances.iter_mut().zip(&lane_distances) { + *distance += *lane_distance; + } + } + + let id_a = row_ids.map_or(i as u64, |ids| ids[i]); + for (j_offset, &dist) in distances.iter().enumerate() { + if dist <= threshold { + let j = i + 1 + j_offset; + let id_b = row_ids.map_or(j as u64, |ids| ids[j]); + result.push(id_a, id_b, dist); + } + } + } + + result +} + /// Extract u64 hashes from a FixedSizeList Arrow array. pub fn extract_hashes_from_fixed_list(array: &FixedSizeListArray) -> Result> { - let list_size = array.value_length(); - if list_size != 8 { + let hashes = extract_binary_hashes_from_fixed_list(array)?; + if hashes.byte_width() != 8 { return Err(Error::InvalidArgumentError(format!( "Expected FixedSizeList with size 8, got size {}", - list_size + hashes.byte_width() ))); } + hashes.into_u64_values() +} - let values = array - .values() - .as_any() - .downcast_ref::() - .ok_or_else(|| { - Error::InvalidArgumentError("Expected UInt8Array values in FixedSizeList".to_string()) - })?; - - let n = array.len(); - let mut hashes = Vec::with_capacity(n); - - for i in 0..n { - let start = i * 8; - let bytes = &values.values()[start..start + 8]; - let mut arr = [0u8; 8]; - arr.copy_from_slice(bytes); - hashes.push(u64::from_le_bytes(arr)); - } - - Ok(hashes) +/// Extract binary hashes from a `FixedSizeList` Arrow array where +/// `N` is a positive multiple of 8 bytes. +pub fn extract_binary_hashes_from_fixed_list( + array: &FixedSizeListArray, +) -> Result { + BinaryHashValues::from_fixed_size_list(array) } /// Union-Find data structure with path compression for clustering. @@ -733,6 +1049,7 @@ pub fn cluster_pairwise_result(result: &PairwiseResult) -> ClusteringResult { #[cfg(test)] mod tests { use super::*; + use lance_arrow::FixedSizeListArrayExt; #[test] fn test_hamming() { @@ -912,6 +1229,121 @@ mod tests { v } + #[test] + fn test_extract_binary_hashes_from_fixed_list_128() { + use arrow_array::UInt8Array; + + let rows = [ + (0x0102_0304_0506_0708u64, 0x1112_1314_1516_1718u64), + (0x2122_2324_2526_2728u64, 0x3132_3334_3536_3738u64), + ]; + let bytes: Vec = rows + .iter() + .flat_map(|(lo, hi)| lo.to_le_bytes().into_iter().chain(hi.to_le_bytes())) + .collect(); + let array = FixedSizeListArray::try_new_from_values(UInt8Array::from(bytes), 16).unwrap(); + + let hashes = extract_binary_hashes_from_fixed_list(&array).unwrap(); + assert_eq!(hashes.len(), 2); + assert_eq!(hashes.byte_width(), 16); + assert_eq!(hashes.num_lanes(), 2); + assert_eq!(hashes.lane(0), &[rows[0].0, rows[1].0]); + assert_eq!(hashes.lane(1), &[rows[0].1, rows[1].1]); + } + + #[test] + fn test_extract_binary_hashes_rejects_non_u64_multiple() { + use arrow_array::UInt8Array; + + let array = + FixedSizeListArray::try_new_from_values(UInt8Array::from(vec![0u8; 24]), 12).unwrap(); + let err = extract_binary_hashes_from_fixed_list(&array).unwrap_err(); + assert!(err.to_string().contains("multiple of 8 bytes"), "{}", err); + } + + #[test] + fn test_binary_hash_values_rejects_width_not_divisible_by_8() { + let err = BinaryHashValues::try_new(Vec::new(), 0, 12).unwrap_err(); + assert!(err.to_string().contains("multiple of 8 bytes"), "{}", err); + } + + #[test] + fn test_pairwise_binary_hashes_128() { + let hashes = BinaryHashValues::try_new( + vec![ + 0, + 0, + 1, + u64::MAX, // lane 0 + 0, + 1, + 1, + u64::MAX, // lane 1 + ], + 4, + 16, + ) + .unwrap(); + + let seq = pairwise_hamming_distance_binary(&hashes, None, Some(1)); + let par = pairwise_hamming_distance_binary_parallel(&hashes, None, Some(1)); + let expected = vec![(0, 1, 1), (1, 2, 1)]; + assert_eq!(result_to_sorted_vec(&seq), expected); + assert_eq!(result_to_sorted_vec(&par), expected); + } + + #[test] + fn test_pairwise_binary_hashes_parallel_128_matches_sequential() { + let rows: Vec<(u64, u64)> = (0..80) + .flat_map(|i| { + let lo = (i as u64).wrapping_mul(0x9E37_79B9_7F4A_7C15); + let hi = !lo.rotate_left(17); + [(lo, hi), (lo ^ 1, hi)] + }) + .collect(); + let mut lane_values = Vec::with_capacity(rows.len() * 2); + lane_values.extend(rows.iter().map(|(lo, _)| *lo)); + lane_values.extend(rows.iter().map(|(_, hi)| *hi)); + let hashes = BinaryHashValues::try_new(lane_values, rows.len(), 16).unwrap(); + let row_ids: Vec = (0..rows.len()).map(|i| 10_000 + i as u64).collect(); + + let seq = pairwise_hamming_distance_binary(&hashes, Some(&row_ids), Some(1)); + let par = pairwise_hamming_distance_binary_parallel(&hashes, Some(&row_ids), Some(1)); + + assert_eq!(result_to_sorted_vec(&par), result_to_sorted_vec(&seq)); + assert!(par.len() >= 80); + } + + #[test] + fn test_pairwise_binary_hashes_32_with_row_ids() { + let hashes = BinaryHashValues::try_new( + vec![ + 0, + 0, + 1, // lane 0 + 0, + 1, + 1, // lane 1 + 7, + 7, + 7, // lane 2 + u64::MAX, + u64::MAX, + u64::MAX - 1, // lane 3 + ], + 3, + 32, + ) + .unwrap(); + let row_ids = [10, 20, 30]; + + let result = pairwise_hamming_distance_binary_parallel(&hashes, Some(&row_ids), Some(2)); + assert_eq!( + result_to_sorted_vec(&result), + vec![(10, 20, 1), (20, 30, 2)] + ); + } + #[test] fn test_pairwise_correctness_small() { // Deterministic hashes with known distances diff --git a/rust/lance-linalg/src/distance/l2.rs b/rust/lance-linalg/src/distance/l2.rs index c47aedd749f..ba95fd42567 100644 --- a/rust/lance-linalg/src/distance/l2.rs +++ b/rust/lance-linalg/src/distance/l2.rs @@ -4,6 +4,11 @@ //! L2 (Euclidean) distance. //! +#[cfg(all( + target_arch = "x86_64", + not(all(target_feature = "avx2", target_feature = "fma")) +))] +use std::arch::x86_64::{_mm256_loadu_ps, _mm256_mul_ps, _mm256_sub_ps}; use std::iter::Sum; use std::ops::AddAssign; use std::sync::Arc; @@ -20,18 +25,46 @@ use lance_arrow::{ArrowFloatType, FixedSizeListArrayExt, FloatArray}; use lance_core::assume_eq; use lance_core::deepsize::DeepSizeOf; use lance_core::utils::cpu::SIMD_SUPPORT; -#[cfg(feature = "fp16kernels")] +// Named tiers are only matched on x86_64, or by the fp16 kernels on the other +// architectures; without either, nothing below names a `SimdSupport` variant. +#[cfg(any(feature = "fp16kernels", target_arch = "x86_64"))] use lance_core::utils::cpu::SimdSupport; use num_traits::{AsPrimitive, Num}; +#[cfg(all( + target_arch = "x86_64", + not(all(target_feature = "avx2", target_feature = "fma")) +))] +use crate::distance::{BatchIter, BatchKernel, BatchKind, BatchOperation}; +#[cfg(all( + target_arch = "x86_64", + not(all(target_feature = "avx2", target_feature = "fma")) +))] +use crate::simd::x86::hsum256_ps; + /// Calculate the L2 distance between two vectors. /// pub trait L2: Num { /// Calculate the L2 distance between two vectors. fn l2(x: &[Self], y: &[Self]) -> f32; - fn l2_batch(x: &[Self], y: &[Self], dimension: usize) -> impl Iterator { - y.chunks_exact(dimension).map(|v| Self::l2(x, v)) + /// L2 distance from `x` to each `dimension`-sized vector in `y`. + /// + /// The default calls [`L2::l2`] per vector. `f32` overrides it so the SIMD + /// tier is chosen once for the whole batch instead of once per vector — + /// on a build whose baseline already implies AVX2, per-vector dispatch + /// costs more than the kernel it selects. + /// + /// Returns `impl Iterator` rather than a trait object: the k-means + /// assignment loop drives this one element at a time, so a + /// `Box` would cost a virtual call per element and an + /// allocation per batch. + fn l2_batch<'a>( + x: &'a [Self], + y: &'a [Self], + dimension: usize, + ) -> impl Iterator + 'a { + y.chunks_exact(dimension).map(move |v| Self::l2(x, v)) } } @@ -44,15 +77,13 @@ pub fn l2(from: &[T], to: &[T]) -> f32 { /// available at runtime. /// /// On x86_64 with AVX-512 this uses 16-wide f32 lanes; otherwise it falls back -/// to [`l2`], which auto-vectorizes to the compiled target (AVX2 on the default -/// `haswell` build). Lance ships an AVX2-baseline binary, so the generic -/// [`l2`] never emits AVX-512 even on capable CPUs — this dispatcher recovers -/// that throughput for callers in the hot path (e.g. the in-memory HNSW index). +/// to [`l2`], whose x86_64 implementation selects the best runtime-supported +/// kernel. This entry point gives hot-path callers such as the in-memory HNSW +/// index an explicit f32 API. #[inline] pub fn l2_f32(x: &[f32], y: &[f32]) -> f32 { #[cfg(target_arch = "x86_64")] { - use lance_core::utils::cpu::SimdSupport; if matches!(*SIMD_SUPPORT, SimdSupport::Avx512 | SimdSupport::Avx512FP16) { // SAFETY: guarded by the runtime AVX-512 detection above. return unsafe { l2_f32_avx512(x, y) }; @@ -191,6 +222,9 @@ impl L2 for bf16 { SimdSupport::Lsx => unsafe { bf16_kernel::l2_bf16_lsx(x.as_ptr(), y.as_ptr(), x.len() as u32) }, + // SimdSupport::AvxFma and SimdSupport::Avx fall through here: + // the bf16 C kernels are compiled with `-march=haswell` minimum + // (AVX2), so they cannot run on AVX-only or AVX+FMA hosts. _ => l2_scalar::(x, y), } } @@ -233,7 +267,7 @@ impl L2 for f16 { kernel::l2_f16_avx512(x.as_ptr(), y.as_ptr(), x.len() as u32) }, #[cfg(all(feature = "fp16kernels", target_arch = "x86_64"))] - SimdSupport::Avx2 => unsafe { + SimdSupport::Avx2 | SimdSupport::Avx512 => unsafe { kernel::l2_f16_avx2(x.as_ptr(), y.as_ptr(), x.len() as u32) }, #[cfg(all(feature = "fp16kernels", target_arch = "loongarch64"))] @@ -244,6 +278,9 @@ impl L2 for f16 { SimdSupport::Lsx => unsafe { kernel::l2_f16_lsx(x.as_ptr(), y.as_ptr(), x.len() as u32) }, + // SimdSupport::AvxFma and SimdSupport::Avx fall through here: + // the f16 C kernels are compiled with `-march=haswell` minimum + // (AVX2), so they cannot run on AVX-only or AVX+FMA hosts. _ => l2_scalar::(x, y), } } @@ -252,12 +289,212 @@ impl L2 for f16 { impl L2 for f32 { #[inline] fn l2(x: &[Self], y: &[Self]) -> f32 { - // 16 = 512 (avx512) / 8 bits / 4 (sizeof(f32)) - // See https://github.com/lance-format/lance/pull/2450. - l2_scalar::(x, y) + // Trait methods cannot carry `#[target_feature]` attributes, so the body + // lives in a free function that runtime-dispatches via `*SIMD_SUPPORT` + // to an AVX2 or AVX-512 inner kernel on capable hosts, or a portable + // scalar fallback. + l2_f32_dispatched(x, y) + } + + fn l2_batch<'a>( + x: &'a [Self], + y: &'a [Self], + dimension: usize, + ) -> impl Iterator + 'a { + // Exactly one arm compiles; see `Dot::dot_batch` for f32. + #[cfg(all( + target_arch = "x86_64", + target_feature = "avx2", + target_feature = "fma" + ))] + { + // `l2_scalar::<_, _, 16>` chunks the vector by 16 lanes. At or below + // that width the chunking degenerates to its scalar remainder loop + // and vectorizes nothing, so the explicit AVX kernel is worth ~40%. + // Above it the autovectorizer already does well and the 8-wide + // kernel can lose, so keep the exact kernel the pre-dispatch code + // used and stay non-regressing by construction. + // + // SAFETY: the build baseline enables avx2+fma, which imply avx+fma, + // so the kernel's `#[target_feature]` contract is met statically. + let narrow = dimension <= 16; + y.chunks_exact(dimension).map(move |v| { + if narrow { + unsafe { x86::l2_f32_avx_fma(x, v) } + } else { + l2_f32_scalar(x, v) + } + }) + } + #[cfg(all( + target_arch = "x86_64", + not(all(target_feature = "avx2", target_feature = "fma")) + ))] + { + l2_batch_f32_runtime_dispatch(x, y, dimension) + } + #[cfg(not(target_arch = "x86_64"))] + { + y.chunks_exact(dimension).map(move |v| Self::l2(x, v)) + } } } +/// Sub-AVX2 builds: pick a `#[target_feature]` kernel once for the batch. +#[cfg(all( + target_arch = "x86_64", + not(all(target_feature = "avx2", target_feature = "fma")) +))] +#[inline] +fn l2_batch_f32_runtime_dispatch<'a>( + x: &'a [f32], + y: &'a [f32], + dimension: usize, +) -> impl Iterator + 'a { + let (kernel, kind): (BatchKernel, BatchKind) = match *SIMD_SUPPORT { + // AVX-512 has no useful work for an eight-element vector. Retain the + // AVX/FMA kernel used by the former Haswell baseline for PQ dimensions. + SimdSupport::Avx512 | SimdSupport::Avx512FP16 if dimension > 16 => { + (x86::l2_batch_f32_avx512, BatchKind::Avx512) + } + SimdSupport::Avx512 | SimdSupport::Avx512FP16 if std::is_x86_feature_detected!("fma") => { + (x86::l2_batch_f32_avx_fma, BatchKind::AvxFma) + } + SimdSupport::Avx2 | SimdSupport::AvxFma => (x86::l2_batch_f32_avx_fma, BatchKind::AvxFma), + SimdSupport::Avx512 | SimdSupport::Avx512FP16 | SimdSupport::Avx => { + (x86::l2_batch_f32_avx, BatchKind::Avx) + } + _ => (l2_batch_f32_scalar, BatchKind::Scalar), + }; + + // SAFETY: the runtime tier and the explicit FMA check above establish the + // selected kernel's target-feature contract. + unsafe { BatchIter::::new(x, y, dimension, kernel, kind) } +} + +#[cfg(all( + target_arch = "x86_64", + not(all(target_feature = "avx2", target_feature = "fma")) +))] +struct L2Batch; + +#[cfg(all( + target_arch = "x86_64", + not(all(target_feature = "avx2", target_feature = "fma")) +))] +impl BatchOperation for L2Batch { + #[inline] + fn fold_scalar(key: &[f32], batch: &[f32], dimension: usize, init: B, mut f: F) -> B + where + F: FnMut(B, f32) -> B, + { + batch + .chunks_exact(dimension) + .fold(init, |acc, vector| f(acc, l2_f32_scalar(key, vector))) + } + + #[target_feature(enable = "avx")] + unsafe fn fold_avx(key: &[f32], batch: &[f32], dimension: usize, init: B, mut f: F) -> B + where + F: FnMut(B, f32) -> B, + { + if dimension == 8 { + let key_values = unsafe { _mm256_loadu_ps(key.as_ptr()) }; + return batch.chunks_exact(8).fold(init, |acc, vector| { + let vector_values = unsafe { _mm256_loadu_ps(vector.as_ptr()) }; + let difference = _mm256_sub_ps(key_values, vector_values); + let squared = _mm256_mul_ps(difference, difference); + f(acc, unsafe { hsum256_ps(squared) }) + }); + } + batch.chunks_exact(dimension).fold(init, |acc, vector| { + f(acc, unsafe { x86::l2_f32_avx(key, vector) }) + }) + } + + #[target_feature(enable = "avx,fma")] + unsafe fn fold_avx_fma( + key: &[f32], + batch: &[f32], + dimension: usize, + init: B, + mut f: F, + ) -> B + where + F: FnMut(B, f32) -> B, + { + if dimension == 8 { + let key_values = unsafe { _mm256_loadu_ps(key.as_ptr()) }; + return batch.chunks_exact(8).fold(init, |acc, vector| { + let vector_values = unsafe { _mm256_loadu_ps(vector.as_ptr()) }; + let difference = _mm256_sub_ps(key_values, vector_values); + let squared = _mm256_mul_ps(difference, difference); + f(acc, unsafe { hsum256_ps(squared) }) + }); + } + batch.chunks_exact(dimension).fold(init, |acc, vector| { + f(acc, unsafe { x86::l2_f32_avx_fma(key, vector) }) + }) + } + + #[target_feature(enable = "avx512f")] + unsafe fn fold_avx512( + key: &[f32], + batch: &[f32], + dimension: usize, + init: B, + mut f: F, + ) -> B + where + F: FnMut(B, f32) -> B, + { + batch.chunks_exact(dimension).fold(init, |acc, vector| { + f(acc, unsafe { x86::l2_f32_avx512(key, vector) }) + }) + } +} + +#[cfg(all( + target_arch = "x86_64", + not(all(target_feature = "avx2", target_feature = "fma")) +))] +unsafe fn l2_batch_f32_scalar(x: &[f32], batch: &[f32], dimension: usize, output: &mut [f32]) { + debug_assert_eq!(output.len(), batch.len() / dimension); + for (distance, y) in output.iter_mut().zip(batch.chunks_exact(dimension)) { + *distance = l2_f32_scalar(x, y); + } +} + +/// L2 distance for f32, runtime-dispatched via `SIMD_SUPPORT` on x86_64 +/// (AVX-512 / AVX2+FMA / AVX+FMA / AVX / scalar). Non-x86 uses the +/// auto-vectorised scalar loop. +#[inline] +fn l2_f32_dispatched(x: &[f32], y: &[f32]) -> f32 { + #[cfg(target_arch = "x86_64")] + { + match *SIMD_SUPPORT { + SimdSupport::Avx512 | SimdSupport::Avx512FP16 => unsafe { x86::l2_f32_avx512(x, y) }, + SimdSupport::Avx2 | SimdSupport::AvxFma => unsafe { x86::l2_f32_avx_fma(x, y) }, + SimdSupport::Avx => unsafe { x86::l2_f32_avx(x, y) }, + _ => l2_f32_scalar(x, y), + } + } + #[cfg(not(target_arch = "x86_64"))] + { + l2_f32_scalar(x, y) + } +} + +/// Portable scalar L2 distance for f32. Used as the x86_64 fallback when no +/// AVX2 is detected, and as the only path on non-x86 architectures. The +/// `LANES = 16` chunking matches the explicit-SIMD inner kernels above. +#[inline] +fn l2_f32_scalar(x: &[f32], y: &[f32]) -> f32 { + // 16 = 512 (avx512) / 8 bits / 4 (sizeof(f32)) + // See https://github.com/lance-format/lance/pull/2450. + l2_scalar::(x, y) +} + impl L2 for f64 { #[inline] fn l2(x: &[Self], y: &[Self]) -> f32 { @@ -265,9 +502,287 @@ impl L2 for f64 { } } -/// Explicit SIMD L2 distance for f64. +/// L2 distance for f64, runtime-dispatched via `SIMD_SUPPORT` on x86_64 +/// (AVX-512 / AVX2+FMA / AVX+FMA / AVX / scalar). Non-x86 uses the SIMD +/// primitives in `crate::simd::f64`, unconditionally backed by NEON / LSX-LASX. #[inline] fn l2_f64_simd(x: &[f64], y: &[f64]) -> f32 { + #[cfg(target_arch = "x86_64")] + { + match *SIMD_SUPPORT { + SimdSupport::Avx512 | SimdSupport::Avx512FP16 => unsafe { x86::l2_f64_avx512(x, y) }, + SimdSupport::Avx2 | SimdSupport::AvxFma => unsafe { x86::l2_f64_avx_fma(x, y) }, + SimdSupport::Avx => unsafe { x86::l2_f64_avx(x, y) }, + _ => l2_f64_scalar(x, y), + } + } + #[cfg(not(target_arch = "x86_64"))] + { + l2_f64_simd_other(x, y) + } +} + +/// Portable scalar L2 distance for f64. Used as the x86_64 fallback when no +/// AVX2 is detected, and exposed for cross-backend parity testing. +#[cfg(target_arch = "x86_64")] +#[inline] +fn l2_f64_scalar(x: &[f64], y: &[f64]) -> f32 { + x.iter() + .zip(y.iter()) + .map(|(&a, &b)| { + let diff = a - b; + diff * diff + }) + .sum::() as f32 +} + +#[cfg(target_arch = "x86_64")] +mod x86 { + use std::arch::x86_64::*; + + use crate::simd::f64::{f64x4, f64x8}; + use crate::simd::x86::hsum256_ps; + use crate::simd::{FloatSimd, SIMD}; + + /// L2 distance from `x` to every `dimension`-sized vector in `batch`, with + /// the AVX-512 tier entered once for the whole batch rather than once per + /// vector. + /// + /// # Safety + /// The host must support AVX-512F. + /// + /// Only compiled for builds whose baseline is below avx2+fma; at or above + /// that baseline `l2_batch` inlines the kernel directly and never runtime- + /// dispatches, so this wrapper would be dead code (see `l2_batch`). + #[cfg(not(all(target_feature = "avx2", target_feature = "fma")))] + #[target_feature(enable = "avx512f")] + pub(super) unsafe fn l2_batch_f32_avx512( + x: &[f32], + batch: &[f32], + dimension: usize, + output: &mut [f32], + ) { + debug_assert_eq!(output.len(), batch.len() / dimension); + for (distance, y) in output.iter_mut().zip(batch.chunks_exact(dimension)) { + *distance = unsafe { l2_f32_avx512(x, y) }; + } + } + + /// As [`l2_batch_f32_avx512`], for the AVX+FMA and AVX2 tiers. + /// + /// # Safety + /// The host must support AVX and FMA. + #[cfg(not(all(target_feature = "avx2", target_feature = "fma")))] + #[target_feature(enable = "avx,fma")] + pub(super) unsafe fn l2_batch_f32_avx_fma( + x: &[f32], + batch: &[f32], + dimension: usize, + output: &mut [f32], + ) { + debug_assert_eq!(output.len(), batch.len() / dimension); + for (distance, y) in output.iter_mut().zip(batch.chunks_exact(dimension)) { + *distance = unsafe { l2_f32_avx_fma(x, y) }; + } + } + + /// As [`l2_batch_f32_avx512`], for the AVX-without-FMA tier. + /// + /// # Safety + /// The host must support AVX. + #[cfg(not(all(target_feature = "avx2", target_feature = "fma")))] + #[target_feature(enable = "avx")] + pub(super) unsafe fn l2_batch_f32_avx( + x: &[f32], + batch: &[f32], + dimension: usize, + output: &mut [f32], + ) { + debug_assert_eq!(output.len(), batch.len() / dimension); + for (distance, y) in output.iter_mut().zip(batch.chunks_exact(dimension)) { + *distance = unsafe { l2_f32_avx(x, y) }; + } + } + + /// AVX-512 path for f64: 8-wide `__m512d` with `vsubpd` + `vfmadd231pd` per iteration. + #[target_feature(enable = "avx512f")] + pub unsafe fn l2_f64_avx512(x: &[f64], y: &[f64]) -> f32 { + let dim = x.len(); + let unrolled_len = dim / 8 * 8; + + let mut acc = _mm512_setzero_pd(); + for i in (0..unrolled_len).step_by(8) { + let a = _mm512_loadu_pd(x.as_ptr().add(i)); + let b = _mm512_loadu_pd(y.as_ptr().add(i)); + let diff = _mm512_sub_pd(a, b); + acc = _mm512_fmadd_pd(diff, diff, acc); + } + + let tail: f64 = x[unrolled_len..] + .iter() + .zip(y[unrolled_len..].iter()) + .map(|(&a, &b)| { + let diff = a - b; + diff * diff + }) + .sum(); + + (_mm512_reduce_add_pd(acc) + tail) as f32 + } + + /// AVX + FMA path for f64. Covers both AvxFma and AVX2 dispatch (body uses no AVX2-specific intrinsics). + #[target_feature(enable = "avx,fma")] + pub unsafe fn l2_f64_avx_fma(x: &[f64], y: &[f64]) -> f32 { + let dim = x.len(); + let unrolled_len = dim / 8 * 8; + + let mut acc8 = f64x8::zeros(); + for i in (0..unrolled_len).step_by(8) { + let a = f64x8::load_unaligned(x.as_ptr().add(i)); + let b = f64x8::load_unaligned(y.as_ptr().add(i)); + let diff = a - b; + acc8.multiply_add(diff, diff); + } + + let aligned_len = dim / 4 * 4; + let mut acc4 = f64x4::zeros(); + for i in (unrolled_len..aligned_len).step_by(4) { + let a = f64x4::load_unaligned(x.as_ptr().add(i)); + let b = f64x4::load_unaligned(y.as_ptr().add(i)); + let diff = a - b; + acc4.multiply_add(diff, diff); + } + + let tail: f64 = x[aligned_len..] + .iter() + .zip(y[aligned_len..].iter()) + .map(|(&a, &b)| { + let diff = a - b; + diff * diff + }) + .sum(); + + (acc8.reduce_sum() + acc4.reduce_sum() + tail) as f32 + } + + /// AVX-only path for f64 (no FMA): squared diff via `_mm256_mul_pd` + `_mm256_add_pd` for Sandy/Ivy Bridge. + #[target_feature(enable = "avx")] + pub unsafe fn l2_f64_avx(x: &[f64], y: &[f64]) -> f32 { + let dim = x.len(); + let unrolled_len = dim / 4 * 4; + + let mut acc = _mm256_setzero_pd(); + for i in (0..unrolled_len).step_by(4) { + let a = _mm256_loadu_pd(x.as_ptr().add(i)); + let b = _mm256_loadu_pd(y.as_ptr().add(i)); + let diff = _mm256_sub_pd(a, b); + acc = _mm256_add_pd(acc, _mm256_mul_pd(diff, diff)); + } + + // Horizontal sum of __m256d -> f64. + let lo = _mm256_castpd256_pd128(acc); + let hi = _mm256_extractf128_pd(acc, 1); + let sum128 = _mm_add_pd(lo, hi); + let sum64 = _mm_add_pd(sum128, _mm_unpackhi_pd(sum128, sum128)); + let acc_sum = _mm_cvtsd_f64(sum64); + + let tail: f64 = x[unrolled_len..] + .iter() + .zip(y[unrolled_len..].iter()) + .map(|(&a, &b)| { + let diff = a - b; + diff * diff + }) + .sum(); + + (acc_sum + tail) as f32 + } + + /// AVX-512 path for f32: 16-wide `__m512` with `vsubps` + `vfmadd231ps` per iteration. + #[inline] + #[target_feature(enable = "avx512f")] + pub unsafe fn l2_f32_avx512(x: &[f32], y: &[f32]) -> f32 { + let dim = x.len(); + let unrolled_len = dim / 16 * 16; + + let mut acc = _mm512_setzero_ps(); + for i in (0..unrolled_len).step_by(16) { + let a = _mm512_loadu_ps(x.as_ptr().add(i)); + let b = _mm512_loadu_ps(y.as_ptr().add(i)); + let diff = _mm512_sub_ps(a, b); + acc = _mm512_fmadd_ps(diff, diff, acc); + } + + let tail: f32 = x[unrolled_len..] + .iter() + .zip(y[unrolled_len..].iter()) + .map(|(&a, &b)| { + let diff = a - b; + diff * diff + }) + .sum(); + + _mm512_reduce_add_ps(acc) + tail + } + + /// AVX + FMA path for f32. Covers both AvxFma and AVX2 dispatch (body uses no AVX2-specific intrinsics). + #[inline] + #[target_feature(enable = "avx,fma")] + pub unsafe fn l2_f32_avx_fma(x: &[f32], y: &[f32]) -> f32 { + let dim = x.len(); + let unrolled_len = dim / 8 * 8; + + let mut acc = _mm256_setzero_ps(); + for i in (0..unrolled_len).step_by(8) { + let a = _mm256_loadu_ps(x.as_ptr().add(i)); + let b = _mm256_loadu_ps(y.as_ptr().add(i)); + let diff = _mm256_sub_ps(a, b); + acc = _mm256_fmadd_ps(diff, diff, acc); + } + + let tail: f32 = x[unrolled_len..] + .iter() + .zip(y[unrolled_len..].iter()) + .map(|(&a, &b)| { + let diff = a - b; + diff * diff + }) + .sum(); + + hsum256_ps(acc) + tail + } + + /// AVX-only path for f32 (no FMA): squared diff via `_mm256_mul_ps` + `_mm256_add_ps` for Sandy/Ivy Bridge. + #[inline] + #[target_feature(enable = "avx")] + pub unsafe fn l2_f32_avx(x: &[f32], y: &[f32]) -> f32 { + let dim = x.len(); + let unrolled_len = dim / 8 * 8; + + let mut acc = _mm256_setzero_ps(); + for i in (0..unrolled_len).step_by(8) { + let a = _mm256_loadu_ps(x.as_ptr().add(i)); + let b = _mm256_loadu_ps(y.as_ptr().add(i)); + let diff = _mm256_sub_ps(a, b); + acc = _mm256_add_ps(acc, _mm256_mul_ps(diff, diff)); + } + + let tail: f32 = x[unrolled_len..] + .iter() + .zip(y[unrolled_len..].iter()) + .map(|(&a, &b)| { + let diff = a - b; + diff * diff + }) + .sum(); + + hsum256_ps(acc) + tail + } +} + +#[cfg(not(target_arch = "x86_64"))] +#[inline] +fn l2_f64_simd_other(x: &[f64], y: &[f64]) -> f32 { use crate::simd::f64::{f64x4, f64x8}; use crate::simd::{FloatSimd, SIMD}; @@ -671,6 +1186,136 @@ mod tests { fn test_l2_distance_f64((x, y) in arbitrary_vector_pair(arbitrary_f64, 4..4048)){ do_l2_test(&x, &y)?; } + + /// Cross-backend parity: scalar fallback must match the dispatched + /// SIMD path within numerical tolerance. Exercises `l2_f64_scalar` + /// directly so the runtime fallback is exercised even on AVX2-capable + /// CI hosts. + #[cfg(target_arch = "x86_64")] + #[test] + fn test_l2_f64_scalar_simd_parity( + (x, y) in arbitrary_vector_pair(arbitrary_f64, 4..4048) + ) { + let scalar = l2_f64_scalar(&x, &y); + let simd = l2_f64_simd(&x, &y); + prop_assert!(approx::relative_eq!(scalar, simd, max_relative = 1e-6)); + } + + /// Parity check for `l2_f32_dispatched` (Branch B exclusive: the + /// auto-vectorised scalar L2 path). The dispatched kernel must + /// agree with a portable f64-precision scalar reference within + /// numerical tolerance. The reference is hand-rolled here to keep + /// this test architecture-agnostic (the x86_64-only `l2_f64_scalar` + /// helper is gated above). + #[test] + fn test_l2_f32_scalar_simd_parity( + (x, y) in arbitrary_vector_pair(arbitrary_f32, 4..4048) + ) { + let scalar = x + .iter() + .zip(y.iter()) + .map(|(&a, &b)| ((a as f64) - (b as f64)).powi(2)) + .sum::() as f32; + let simd = ::l2(&x, &y); + prop_assert!(approx::relative_eq!(scalar, simd, max_relative = 1e-3)); + } + + /// AVX-512-direct parity: explicitly compares the scalar fallback + /// against the native AVX-512 inner kernel on AVX-512F-capable hosts + /// (Skylake-X+, Ice Lake, Sapphire Rapids, Zen 4). Early-returns on + /// hosts without AVX-512F. + #[cfg(target_arch = "x86_64")] + #[test] + fn test_l2_f64_scalar_vs_avx512_parity( + (x, y) in arbitrary_vector_pair(arbitrary_f64, 4..4048) + ) { + if !std::is_x86_feature_detected!("avx512f") { + return Ok(()); + } + let scalar = l2_f64_scalar(&x, &y); + let avx512 = unsafe { x86::l2_f64_avx512(&x, &y) }; + prop_assert!(approx::relative_eq!(scalar, avx512, max_relative = 1e-6)); + } + + /// AVX + FMA-direct parity for the f64 L2 kernel. Covers the AMD + /// Piledriver / Steamroller / FX-7500 tier. Early-returns on hosts + /// without both AVX and FMA. + #[cfg(target_arch = "x86_64")] + #[test] + fn test_l2_f64_scalar_vs_avx_fma_parity( + (x, y) in arbitrary_vector_pair(arbitrary_f64, 4..4048) + ) { + if !(std::is_x86_feature_detected!("avx") && std::is_x86_feature_detected!("fma")) { + return Ok(()); + } + let scalar = l2_f64_scalar(&x, &y); + let avx_fma = unsafe { x86::l2_f64_avx_fma(&x, &y) }; + prop_assert!(approx::relative_eq!(scalar, avx_fma, max_relative = 1e-6)); + } + + /// AVX-only-direct parity for the f64 L2 kernel. Covers the Intel + /// Sandy Bridge / Ivy Bridge tier. Early-returns on hosts without + /// AVX. + #[cfg(target_arch = "x86_64")] + #[test] + fn test_l2_f64_scalar_vs_avx_parity( + (x, y) in arbitrary_vector_pair(arbitrary_f64, 4..4048) + ) { + if !std::is_x86_feature_detected!("avx") { + return Ok(()); + } + let scalar = l2_f64_scalar(&x, &y); + let avx = unsafe { x86::l2_f64_avx(&x, &y) }; + prop_assert!(approx::relative_eq!(scalar, avx, max_relative = 1e-6)); + } + + /// AVX-512-direct parity for f32: explicitly compares the scalar + /// fallback against the native f32 AVX-512 inner kernel on + /// AVX-512F-capable hosts. + #[cfg(target_arch = "x86_64")] + #[test] + fn test_l2_f32_scalar_vs_avx512_parity( + (x, y) in arbitrary_vector_pair(arbitrary_f32, 4..4048) + ) { + if !std::is_x86_feature_detected!("avx512f") { + return Ok(()); + } + let scalar = l2_f32_scalar(&x, &y); + let avx512 = unsafe { x86::l2_f32_avx512(&x, &y) }; + prop_assert!(approx::relative_eq!(scalar, avx512, max_relative = 1e-3)); + } + + /// AVX + FMA-direct parity for the f32 L2 kernel. Covers the AMD + /// Piledriver / Steamroller / FX-7500 tier. Early-returns on hosts + /// without both AVX and FMA. + #[cfg(target_arch = "x86_64")] + #[test] + fn test_l2_f32_scalar_vs_avx_fma_parity( + (x, y) in arbitrary_vector_pair(arbitrary_f32, 4..4048) + ) { + if !(std::is_x86_feature_detected!("avx") && std::is_x86_feature_detected!("fma")) { + return Ok(()); + } + let scalar = l2_f32_scalar(&x, &y); + let avx_fma = unsafe { x86::l2_f32_avx_fma(&x, &y) }; + prop_assert!(approx::relative_eq!(scalar, avx_fma, max_relative = 1e-3)); + } + + /// AVX-only-direct parity for the f32 L2 kernel. Covers the Intel + /// Sandy Bridge / Ivy Bridge tier. Early-returns on hosts without + /// AVX. + #[cfg(target_arch = "x86_64")] + #[test] + fn test_l2_f32_scalar_vs_avx_parity( + (x, y) in arbitrary_vector_pair(arbitrary_f32, 4..4048) + ) { + if !std::is_x86_feature_detected!("avx") { + return Ok(()); + } + let scalar = l2_f32_scalar(&x, &y); + let avx = unsafe { x86::l2_f32_avx(&x, &y) }; + prop_assert!(approx::relative_eq!(scalar, avx, max_relative = 1e-3)); + } } #[test] @@ -787,4 +1432,101 @@ mod tests { assert_relative_eq!(d1[0], 0.0); // q1 == target[0] assert_relative_eq!(d2[1], 0.0); // q2 == target[1] } + + /// `l2_batch` must agree with the per-vector `l2` it replaced, on every + /// build: the AVX2-baseline path, the hoisted-dispatch path, and the + /// portable fallback all funnel through here. + #[rstest::rstest] + #[case::dim_8(8)] + #[case::dim_16(16)] + #[case::dim_32(32)] + #[case::dim_1024(1024)] + fn test_l2_batch_f32_matches_per_vector_l2(#[case] dimension: usize) { + let num_vectors = 5; + let x: Vec = (0..dimension) + .map(|i| ((i % 13) as f32) * 0.25 + 1.0) + .collect(); + let batch: Vec = (0..dimension * num_vectors) + .map(|i| ((i % 11) as f32) * 0.5 - 2.0) + .collect(); + + let got: Vec = f32::l2_batch(&x, &batch, dimension).collect(); + let want: Vec = batch + .chunks_exact(dimension) + .map(|y| f32::l2(&x, y)) + .collect(); + + assert_eq!(got.len(), num_vectors); + for (g, w) in got.iter().zip(want.iter()) { + assert!( + approx::relative_eq!(g, w, epsilon = 1e-4), + "dim {dimension}: batch {g} != per-vector {w}" + ); + } + } + + /// The per-batch `#[target_feature]` kernels are only reached on sub-AVX2 + /// builds or AVX-512 hosts, so call them directly to cover them. + #[cfg(all( + target_arch = "x86_64", + not(all(target_feature = "avx2", target_feature = "fma")) + ))] + fn check_l2_batch_kernel(kernel: BatchKernel) { + for dimension in [8_usize, 16, 40] { + let num_vectors = 3; + let x: Vec = (0..dimension).map(|i| (i as f32) * 0.5 + 1.0).collect(); + let batch: Vec = (0..dimension * num_vectors) + .map(|i| ((i % 7) as f32) + 1.0) + .collect(); + + let mut got = vec![0.0; num_vectors]; + unsafe { kernel(&x, &batch, dimension, &mut got) }; + assert_eq!(got.len(), num_vectors); + for (chunk, &g) in batch.chunks_exact(dimension).zip(got.iter()) { + let want = l2_scalar::(&x, chunk); + assert!( + approx::relative_eq!(g, want, epsilon = 1e-4), + "dim {dimension}: kernel {g} != scalar {want}" + ); + } + } + } + + // The runtime-dispatch batch kernels only exist in sub-avx2+fma builds + // (see `x86::l2_batch_f32_avx512`), so gate their tests the same way. + #[cfg(all( + target_arch = "x86_64", + not(all(target_feature = "avx2", target_feature = "fma")) + ))] + #[test] + fn test_l2_batch_avx_fma_matches_scalar() { + if !std::is_x86_feature_detected!("avx") || !std::is_x86_feature_detected!("fma") { + return; + } + check_l2_batch_kernel(x86::l2_batch_f32_avx_fma); + } + + #[cfg(all( + target_arch = "x86_64", + not(all(target_feature = "avx2", target_feature = "fma")) + ))] + #[test] + fn test_l2_batch_avx_matches_scalar() { + if !std::is_x86_feature_detected!("avx") { + return; + } + check_l2_batch_kernel(x86::l2_batch_f32_avx); + } + + #[cfg(all( + target_arch = "x86_64", + not(all(target_feature = "avx2", target_feature = "fma")) + ))] + #[test] + fn test_l2_batch_avx512_matches_scalar() { + if !std::is_x86_feature_detected!("avx512f") { + return; + } + check_l2_batch_kernel(x86::l2_batch_f32_avx512); + } } diff --git a/rust/lance-linalg/src/distance/l2_u8.rs b/rust/lance-linalg/src/distance/l2_u8.rs index 1b111f91338..5fbf8a3af55 100644 --- a/rust/lance-linalg/src/distance/l2_u8.rs +++ b/rust/lance-linalg/src/distance/l2_u8.rs @@ -143,6 +143,10 @@ fn select_backend() -> L2U8Fn { if is_x86_feature_detected!("avx2") { return |a, b| unsafe { x86::l2_u8_avx2(a, b) }; } + // AvxFma and Avx hosts (AMD Piledriver / Steamroller, Intel Sandy + // Bridge / Ivy Bridge) fall through to scalar: the AVX2 inner uses + // AVX2 integer ops (`vpsubusb` / `vpmaddwd`) which neither AVX nor + // AVX+FMA provides. } l2_u8_scalar diff --git a/rust/lance-linalg/src/distance/norm_l2.rs b/rust/lance-linalg/src/distance/norm_l2.rs index b1daf85ab3b..06218830112 100644 --- a/rust/lance-linalg/src/distance/norm_l2.rs +++ b/rust/lance-linalg/src/distance/norm_l2.rs @@ -9,9 +9,7 @@ use arrow_array::types::{Float16Type, Float32Type, Float64Type}; use arrow_schema::DataType; use half::{bf16, f16}; #[allow(unused_imports)] -use lance_core::utils::cpu::SIMD_SUPPORT; -#[cfg(feature = "fp16kernels")] -use lance_core::utils::cpu::SimdSupport; +use lance_core::utils::cpu::{SIMD_SUPPORT, SimdSupport}; use num_traits::{AsPrimitive, Float, Num}; /// L2 normalization @@ -64,7 +62,7 @@ impl Normalize for f16 { kernel::norm_l2_f16_avx512(vector.as_ptr(), vector.len() as u32) }, #[cfg(all(feature = "fp16kernels", target_arch = "x86_64"))] - SimdSupport::Avx2 => unsafe { + SimdSupport::Avx2 | SimdSupport::Avx512 => unsafe { kernel::norm_l2_f16_avx2(vector.as_ptr(), vector.len() as u32) }, #[cfg(all(feature = "fp16kernels", target_arch = "loongarch64"))] @@ -75,6 +73,9 @@ impl Normalize for f16 { SimdSupport::Lsx => unsafe { kernel::norm_l2_f16_lsx(vector.as_ptr(), vector.len() as u32) }, + // SimdSupport::AvxFma and SimdSupport::Avx fall through here: + // the f16 C kernels are compiled with `-march=haswell` minimum + // (AVX2), so they cannot run on AVX-only or AVX+FMA hosts. _ => norm_l2_impl::(vector), } } @@ -126,6 +127,9 @@ impl Normalize for bf16 { SimdSupport::Lsx => unsafe { bf16_kernel::norm_l2_bf16_lsx(vector.as_ptr(), vector.len() as u32) }, + // SimdSupport::AvxFma and SimdSupport::Avx fall through here: + // the bf16 C kernels are compiled with `-march=haswell` minimum + // (AVX2), so they cannot run on AVX-only or AVX+FMA hosts. _ => norm_l2_impl::(vector), } } @@ -134,10 +138,40 @@ impl Normalize for bf16 { impl Normalize for f32 { #[inline] fn norm_l2(vector: &[Self]) -> f32 { - norm_l2_impl::(vector) + norm_l2_f32_dispatched(vector) } } +/// L2 norm for f32, runtime-dispatched via `SIMD_SUPPORT` on x86_64 +/// (AVX-512 / AVX2+FMA / AVX+FMA / AVX / scalar). Non-x86 uses the +/// auto-vectorised scalar loop. +#[inline] +fn norm_l2_f32_dispatched(vector: &[f32]) -> f32 { + #[cfg(target_arch = "x86_64")] + { + match *SIMD_SUPPORT { + SimdSupport::Avx512 | SimdSupport::Avx512FP16 => unsafe { + x86::norm_l2_f32_avx512(vector) + }, + SimdSupport::Avx2 | SimdSupport::AvxFma => unsafe { x86::norm_l2_f32_avx_fma(vector) }, + SimdSupport::Avx => unsafe { x86::norm_l2_f32_avx(vector) }, + _ => norm_l2_f32_scalar(vector), + } + } + #[cfg(not(target_arch = "x86_64"))] + { + norm_l2_f32_scalar(vector) + } +} + +/// Portable scalar L2 norm for f32. Used as the x86_64 fallback when no +/// AVX2 is detected, and as the only path on non-x86 architectures. The +/// `LANES = 16` chunking matches the explicit-SIMD inner kernels above. +#[inline] +fn norm_l2_f32_scalar(vector: &[f32]) -> f32 { + norm_l2_impl::(vector) +} + impl Normalize for f64 { #[inline] fn norm_l2(vector: &[Self]) -> f32 { @@ -145,11 +179,166 @@ impl Normalize for f64 { } } -/// Explicit SIMD implementation of L2 norm for f64. +/// L2 norm for f64. Runtime-dispatched to the best available backend. /// -/// Two-level unrolling: f64x8 main loop, f64x4 remainder, scalar tail. +/// On x86_64, dispatches via `SIMD_SUPPORT` to a native AVX-512 inner kernel +/// (Skylake-X+, Ice Lake, Sapphire Rapids, Zen 4), an AVX2 + FMA kernel +/// (Haswell+), an AVX + FMA kernel (AMD Piledriver / Steamroller), an +/// AVX-only kernel (Intel Sandy Bridge / Ivy Bridge), or a portable scalar +/// fallback. The per-tier inner functions each carry their own +/// `#[target_feature]` so they stay correct under any compile baseline. +/// On aarch64 and loongarch64, the SIMD primitives in `crate::simd::f64` +/// are unconditionally backed by NEON / LSX-LASX respectively, so no +/// runtime gate is required. #[inline] pub fn norm_l2_f64_simd(vector: &[f64]) -> f32 { + #[cfg(target_arch = "x86_64")] + { + match *SIMD_SUPPORT { + SimdSupport::Avx512 | SimdSupport::Avx512FP16 => unsafe { + x86::norm_l2_f64_avx512(vector) + }, + SimdSupport::Avx2 | SimdSupport::AvxFma => unsafe { x86::norm_l2_f64_avx_fma(vector) }, + SimdSupport::Avx => unsafe { x86::norm_l2_f64_avx(vector) }, + _ => norm_l2_f64_scalar(vector), + } + } + #[cfg(not(target_arch = "x86_64"))] + { + norm_l2_f64_simd_other(vector) + } +} + +/// Portable scalar L2 norm. Used as the x86_64 fallback when no AVX2 is +/// detected, and exposed for cross-backend parity testing. +#[cfg(target_arch = "x86_64")] +#[inline] +fn norm_l2_f64_scalar(vector: &[f64]) -> f32 { + vector.iter().map(|v| v * v).sum::().sqrt() as f32 +} + +#[cfg(target_arch = "x86_64")] +mod x86 { + use std::arch::x86_64::*; + + use crate::simd::f64::{f64x4, f64x8}; + use crate::simd::x86::hsum256_ps; + use crate::simd::{FloatSimd, SIMD}; + + /// AVX-512 path for f64: 8-wide `__m512d` with `vfmadd231pd` per iteration. + #[target_feature(enable = "avx512f")] + pub unsafe fn norm_l2_f64_avx512(vector: &[f64]) -> f32 { + let dim = vector.len(); + let unrolled_len = dim / 8 * 8; + + let mut acc = _mm512_setzero_pd(); + for i in (0..unrolled_len).step_by(8) { + let v = _mm512_loadu_pd(vector.as_ptr().add(i)); + acc = _mm512_fmadd_pd(v, v, acc); + } + + let tail: f64 = vector[unrolled_len..].iter().map(|&v| v * v).sum(); + (_mm512_reduce_add_pd(acc) + tail).sqrt() as f32 + } + + /// AVX + FMA path for f64. Covers both AvxFma and AVX2 dispatch (body uses no AVX2-specific intrinsics). + #[target_feature(enable = "avx,fma")] + pub unsafe fn norm_l2_f64_avx_fma(vector: &[f64]) -> f32 { + let dim = vector.len(); + let unrolled_len = dim / 8 * 8; + + let mut acc8 = f64x8::zeros(); + for i in (0..unrolled_len).step_by(8) { + let v = f64x8::load_unaligned(vector.as_ptr().add(i)); + acc8.multiply_add(v, v); + } + + let aligned_len = dim / 4 * 4; + let mut acc4 = f64x4::zeros(); + for i in (unrolled_len..aligned_len).step_by(4) { + let v = f64x4::load_unaligned(vector.as_ptr().add(i)); + acc4.multiply_add(v, v); + } + + let tail: f64 = vector[aligned_len..].iter().map(|&v| v * v).sum(); + (acc8.reduce_sum() + acc4.reduce_sum() + tail).sqrt() as f32 + } + + /// AVX-only path for f64 (no FMA): `_mm256_mul_pd` + `_mm256_add_pd` per iteration for Sandy/Ivy Bridge. + #[target_feature(enable = "avx")] + pub unsafe fn norm_l2_f64_avx(vector: &[f64]) -> f32 { + let dim = vector.len(); + let unrolled_len = dim / 4 * 4; + + let mut acc = _mm256_setzero_pd(); + for i in (0..unrolled_len).step_by(4) { + let v = _mm256_loadu_pd(vector.as_ptr().add(i)); + acc = _mm256_add_pd(acc, _mm256_mul_pd(v, v)); + } + + // Horizontal sum of __m256d -> f64. Two pairwise adds across lanes. + let lo = _mm256_castpd256_pd128(acc); + let hi = _mm256_extractf128_pd(acc, 1); + let sum128 = _mm_add_pd(lo, hi); + let sum64 = _mm_add_pd(sum128, _mm_unpackhi_pd(sum128, sum128)); + let acc_sum = _mm_cvtsd_f64(sum64); + + let tail: f64 = vector[unrolled_len..].iter().map(|&v| v * v).sum(); + (acc_sum + tail).sqrt() as f32 + } + + /// AVX-512 path for f32: 16-wide `__m512` with `vfmadd231ps` per iteration. + #[target_feature(enable = "avx512f")] + pub unsafe fn norm_l2_f32_avx512(vector: &[f32]) -> f32 { + let dim = vector.len(); + let unrolled_len = dim / 16 * 16; + + let mut acc = _mm512_setzero_ps(); + for i in (0..unrolled_len).step_by(16) { + let v = _mm512_loadu_ps(vector.as_ptr().add(i)); + acc = _mm512_fmadd_ps(v, v, acc); + } + + let tail: f32 = vector[unrolled_len..].iter().map(|&v| v * v).sum(); + (_mm512_reduce_add_ps(acc) + tail).sqrt() + } + + /// AVX + FMA path for f32. Covers both AvxFma and AVX2 dispatch (body uses no AVX2-specific intrinsics). + #[target_feature(enable = "avx,fma")] + pub unsafe fn norm_l2_f32_avx_fma(vector: &[f32]) -> f32 { + let dim = vector.len(); + let unrolled_len = dim / 8 * 8; + + let mut acc = _mm256_setzero_ps(); + for i in (0..unrolled_len).step_by(8) { + let v = _mm256_loadu_ps(vector.as_ptr().add(i)); + acc = _mm256_fmadd_ps(v, v, acc); + } + + let tail: f32 = vector[unrolled_len..].iter().map(|&v| v * v).sum(); + (hsum256_ps(acc) + tail).sqrt() + } + + /// AVX-only path for f32 (no FMA): `_mm256_mul_ps` + `_mm256_add_ps` per iteration for Sandy/Ivy Bridge. + #[target_feature(enable = "avx")] + pub unsafe fn norm_l2_f32_avx(vector: &[f32]) -> f32 { + let dim = vector.len(); + let unrolled_len = dim / 8 * 8; + + let mut acc = _mm256_setzero_ps(); + for i in (0..unrolled_len).step_by(8) { + let v = _mm256_loadu_ps(vector.as_ptr().add(i)); + acc = _mm256_add_ps(acc, _mm256_mul_ps(v, v)); + } + + let tail: f32 = vector[unrolled_len..].iter().map(|&v| v * v).sum(); + (hsum256_ps(acc) + tail).sqrt() + } +} + +#[cfg(not(target_arch = "x86_64"))] +#[inline] +fn norm_l2_f64_simd_other(vector: &[f64]) -> f32 { use crate::simd::f64::{f64x4, f64x8}; use crate::simd::{FloatSimd, SIMD}; @@ -291,5 +480,133 @@ mod tests { fn test_l2_norm_f64(data in prop::collection::vec(arbitrary_f64(), 4..4048)){ do_norm_l2_test(&data)?; } + + /// Cross-backend parity: scalar fallback must match the dispatched + /// SIMD path within numerical tolerance. Exercises `norm_l2_f64_scalar` + /// directly so the runtime fallback is exercised even on AVX2-capable + /// CI hosts. + #[cfg(target_arch = "x86_64")] + #[test] + fn test_l2_norm_f64_scalar_simd_parity( + data in prop::collection::vec(arbitrary_f64(), 4..4048) + ) { + let scalar = norm_l2_f64_scalar(&data); + let simd = norm_l2_f64_simd(&data); + prop_assert!(approx::relative_eq!(scalar, simd, max_relative = 1e-6)); + } + + /// Parity check for `norm_l2_f32_dispatched` (Branch B exclusive: the + /// auto-vectorised scalar L2-norm path). The dispatched kernel must + /// agree with a portable f64-precision scalar reference within + /// numerical tolerance. The reference is hand-rolled here to keep this + /// test architecture-agnostic (the x86_64-only `norm_l2_f64_scalar` + /// helper is gated above). + #[test] + fn test_l2_norm_f32_scalar_simd_parity( + data in prop::collection::vec(arbitrary_f32(), 4..4048) + ) { + let scalar = data.iter().map(|&v| (v as f64).powi(2)).sum::().sqrt() as f32; + let simd = ::norm_l2(&data); + prop_assert!(approx::relative_eq!(scalar, simd, max_relative = 1e-3)); + } + + /// AVX-512-direct parity: explicitly compares the scalar fallback + /// against the native AVX-512 inner kernel on AVX-512F-capable hosts + /// (Skylake-X+, Ice Lake, Sapphire Rapids, Zen 4). Early-returns on + /// hosts without AVX-512F so the test stays portable; CI runners with + /// AVX-512F exercise the `_mm512_*` path. + #[cfg(target_arch = "x86_64")] + #[test] + fn test_l2_norm_f64_scalar_vs_avx512_parity( + data in prop::collection::vec(arbitrary_f64(), 4..4048) + ) { + if !std::is_x86_feature_detected!("avx512f") { + return Ok(()); + } + let scalar = norm_l2_f64_scalar(&data); + let avx512 = unsafe { x86::norm_l2_f64_avx512(&data) }; + prop_assert!(approx::relative_eq!(scalar, avx512, max_relative = 1e-6)); + } + + /// AVX + FMA-direct parity for the f64 L2-norm kernel. Covers the + /// AMD Piledriver / Steamroller / FX-7500 tier. Early-returns on + /// hosts without both AVX and FMA. + #[cfg(target_arch = "x86_64")] + #[test] + fn test_l2_norm_f64_scalar_vs_avx_fma_parity( + data in prop::collection::vec(arbitrary_f64(), 4..4048) + ) { + if !(std::is_x86_feature_detected!("avx") && std::is_x86_feature_detected!("fma")) { + return Ok(()); + } + let scalar = norm_l2_f64_scalar(&data); + let avx_fma = unsafe { x86::norm_l2_f64_avx_fma(&data) }; + prop_assert!(approx::relative_eq!(scalar, avx_fma, max_relative = 1e-6)); + } + + /// AVX-only-direct parity for the f64 L2-norm kernel. Covers the + /// Intel Sandy Bridge / Ivy Bridge tier (AVX without FMA). + /// Early-returns on hosts without AVX. + #[cfg(target_arch = "x86_64")] + #[test] + fn test_l2_norm_f64_scalar_vs_avx_parity( + data in prop::collection::vec(arbitrary_f64(), 4..4048) + ) { + if !std::is_x86_feature_detected!("avx") { + return Ok(()); + } + let scalar = norm_l2_f64_scalar(&data); + let avx = unsafe { x86::norm_l2_f64_avx(&data) }; + prop_assert!(approx::relative_eq!(scalar, avx, max_relative = 1e-6)); + } + + /// AVX-512-direct parity for the f32 L2-norm kernel. Explicitly + /// compares the scalar fallback against the native AVX-512 inner + /// kernel on AVX-512F-capable hosts. Early-returns on hosts without + /// AVX-512F; CI runners with AVX-512F exercise the `_mm512_*` path. + #[cfg(target_arch = "x86_64")] + #[test] + fn test_norm_l2_f32_scalar_vs_avx512_parity( + data in prop::collection::vec(arbitrary_f32(), 4..4048) + ) { + if !std::is_x86_feature_detected!("avx512f") { + return Ok(()); + } + let scalar = norm_l2_f32_scalar(&data); + let avx512 = unsafe { x86::norm_l2_f32_avx512(&data) }; + prop_assert!(approx::relative_eq!(scalar, avx512, max_relative = 1e-3)); + } + + /// AVX + FMA-direct parity for the f32 L2-norm kernel. Covers the + /// AMD Piledriver / Steamroller / FX-7500 tier. Early-returns on + /// hosts without both AVX and FMA. + #[cfg(target_arch = "x86_64")] + #[test] + fn test_norm_l2_f32_scalar_vs_avx_fma_parity( + data in prop::collection::vec(arbitrary_f32(), 4..4048) + ) { + if !(std::is_x86_feature_detected!("avx") && std::is_x86_feature_detected!("fma")) { + return Ok(()); + } + let scalar = norm_l2_f32_scalar(&data); + let avx_fma = unsafe { x86::norm_l2_f32_avx_fma(&data) }; + prop_assert!(approx::relative_eq!(scalar, avx_fma, max_relative = 1e-3)); + } + + /// AVX-only-direct parity for the f32 L2-norm kernel. Covers the + /// Intel Sandy Bridge / Ivy Bridge tier (AVX without FMA). + /// Early-returns on hosts without AVX. + #[cfg(target_arch = "x86_64")] + #[test] + fn test_norm_l2_f32_scalar_vs_avx_parity( + data in prop::collection::vec(arbitrary_f32(), 4..4048) + ) { + if !std::is_x86_feature_detected!("avx") { + return Ok(()); + } + let scalar = norm_l2_f32_scalar(&data); + let avx = unsafe { x86::norm_l2_f32_avx(&data) }; + prop_assert!(approx::relative_eq!(scalar, avx, max_relative = 1e-3)); + } } } diff --git a/rust/lance-linalg/src/simd.rs b/rust/lance-linalg/src/simd.rs index 91dc1c6959d..6722eafd768 100644 --- a/rust/lance-linalg/src/simd.rs +++ b/rust/lance-linalg/src/simd.rs @@ -19,6 +19,8 @@ pub mod f32; pub mod f64; pub mod i32; pub mod u8; +#[cfg(target_arch = "x86_64")] +pub(crate) mod x86; use num_traits::{Float, Num}; use u8::u8x16; diff --git a/rust/lance-linalg/src/simd/dist_table.rs b/rust/lance-linalg/src/simd/dist_table.rs index 626c1581b15..e337953ec10 100644 --- a/rust/lance-linalg/src/simd/dist_table.rs +++ b/rust/lance-linalg/src/simd/dist_table.rs @@ -73,6 +73,10 @@ pub fn sum_4bit_dist_table( ) } }, + // SimdSupport::AvxFma and SimdSupport::Avx fall through here: + // the AVX2 inner uses `_mm256_shuffle_epi8` / `_mm256_and_si256` / + // `_mm256_srli_epi16` / `_mm256_add_epi16` integer ops which + // neither AVX nor AVX+FMA provides. Scalar is the correct route. _ => sum_4bit_dist_table_scalar(code_len, codes, dist_table, dists), } } diff --git a/rust/lance-linalg/src/simd/f32.rs b/rust/lance-linalg/src/simd/f32.rs index 78042997121..4ce7f64706d 100644 --- a/rust/lance-linalg/src/simd/f32.rs +++ b/rust/lance-linalg/src/simd/f32.rs @@ -46,14 +46,42 @@ impl std::fmt::Debug for f32x8 { } impl f32x8 { + /// Gather 8 f32 values from `slice` at the offsets in `indices`. + /// + /// On x86_64 this uses the AVX2 `vgatherdps` instruction when the host + /// supports it (gated at runtime via `is_x86_feature_detected!`). On + /// other architectures (and on x86_64 hosts without AVX2) the function + /// falls back to a per-index scalar load followed by `Self::from(&out)`, + /// which goes through `load_unaligned` (NEON / LASX / `_mm256_loadu_ps` + /// depending on platform). Per-tier macro stamping (e.g., the + /// `multiversion` crate) was considered but doesn't fit here: the function + /// returns `Self` and `_mm256_i32gather_ps::<4>` requires the const-generic + /// stride to be a compile-time literal — neither composes with the macro. + /// + /// # Panics + /// + /// If any index is negative or lands outside `slice`. #[inline] pub fn gather(slice: &[f32], indices: &[i32; 8]) -> Self { - #[cfg(target_arch = "x86_64")] - unsafe { - use super::i32::i32x8; + // Every backend below reads without bounds checking: `vgatherdps` does + // none, and the NEON / LASX arms offset a raw pointer. Check once here + // so an out-of-range index panics on every host rather than reading out + // of bounds on some and panicking on others. + for &i in indices { + assert!( + (i as usize) < slice.len(), + "gather index {i} is out of bounds for a slice of length {}", + slice.len() + ); + } - let idx = i32x8::from(indices); - Self(_mm256_i32gather_ps::<4>(slice.as_ptr(), idx.0)) + #[cfg(target_arch = "x86_64")] + { + if is_x86_feature_detected!("avx2") { + unsafe { gather_avx2(slice, indices) } + } else { + gather_scalar_x86(slice, indices) + } } #[cfg(target_arch = "aarch64")] @@ -94,6 +122,30 @@ impl f32x8 { } } +/// AVX2 gather. Caller must ensure the host supports AVX2 (gated by +/// the `is_x86_feature_detected!("avx2")` check in `f32x8::gather`). +#[cfg(target_arch = "x86_64")] +#[target_feature(enable = "avx2")] +unsafe fn gather_avx2(slice: &[f32], indices: &[i32; 8]) -> f32x8 { + use super::i32::i32x8; + + let idx = i32x8::from(indices); + f32x8(_mm256_i32gather_ps::<4>(slice.as_ptr(), idx.0)) +} + +/// Portable scalar gather for x86_64 hosts without AVX2. +/// +/// Indexes the slice rather than offsetting a raw pointer: this is the slow +/// path already, so an out-of-range index should panic instead of reading out +/// of bounds. +#[cfg(target_arch = "x86_64")] +#[inline] +fn gather_scalar_x86(slice: &[f32], indices: &[i32; 8]) -> f32x8 { + let values = indices.map(|i| slice[i as usize]); + // SAFETY: `values` is eight contiguous, initialized `f32`. + unsafe { f32x8::load_unaligned(values.as_ptr()) } +} + impl From<&[f32]> for f32x8 { fn from(value: &[f32]) -> Self { unsafe { Self::load_unaligned(value.as_ptr()) } @@ -439,15 +491,18 @@ impl Mul for f32x8 { } } -/// 16 of 32-bit `f32` values. Use 512-bit SIMD if possible. +/// 16 of 32-bit `f32` values. Stored as a pair of 256-bit AVX vectors on +/// x86_64. Originally there was a sibling AVX-512 variant gated on +/// `target_feature = "avx512f"`, but no project CI configuration enables +/// `+avx512f` globally (and one of the avx512 arms still contained `todo!()`), +/// so the variant was dead code. Removed in the runtime-SIMD-dispatch +/// retrofit; per-tier dispatch happens in the kernel functions in +/// `crate::distance::*` via `match *SIMD_SUPPORT` + per-tier +/// `#[target_feature(enable = "...")]` inner functions. #[allow(non_camel_case_types)] -#[cfg(all(target_arch = "x86_64", not(target_feature = "avx512f")))] +#[cfg(target_arch = "x86_64")] #[derive(Clone, Copy)] pub struct f32x16(__m256, __m256); -#[allow(non_camel_case_types)] -#[cfg(all(target_arch = "x86_64", target_feature = "avx512f"))] -#[derive(Clone, Copy)] -pub struct f32x16(__m512); /// 16 of 32-bit `f32` values. Use 512-bit SIMD if possible. #[allow(non_camel_case_types)] @@ -486,11 +541,7 @@ impl<'a> From<&'a [f32; 16]> for f32x16 { impl SIMD for f32x16 { #[inline] fn splat(val: f32) -> Self { - #[cfg(all(target_arch = "x86_64", target_feature = "avx512f"))] - unsafe { - Self(_mm512_set1_ps(val)) - } - #[cfg(all(target_arch = "x86_64", not(target_feature = "avx512f")))] + #[cfg(target_arch = "x86_64")] unsafe { Self(_mm256_set1_ps(val), _mm256_set1_ps(val)) } @@ -514,11 +565,7 @@ impl SIMD for f32x16 { #[inline] fn zeros() -> Self { - #[cfg(all(target_arch = "x86_64", target_feature = "avx512f"))] - unsafe { - Self(_mm512_setzero_ps()) - } - #[cfg(all(target_arch = "x86_64", not(target_feature = "avx512f")))] + #[cfg(target_arch = "x86_64")] unsafe { Self(_mm256_setzero_ps(), _mm256_setzero_ps()) } @@ -534,14 +581,10 @@ impl SIMD for f32x16 { #[inline] unsafe fn load(ptr: *const f32) -> Self { - #[cfg(all(target_arch = "x86_64", not(target_feature = "avx512f")))] + #[cfg(target_arch = "x86_64")] unsafe { Self(_mm256_load_ps(ptr), _mm256_load_ps(ptr.add(8))) } - #[cfg(all(target_arch = "x86_64", target_feature = "avx512f"))] - unsafe { - Self(_mm512_load_ps(ptr)) - } #[cfg(target_arch = "aarch64")] { Self::load_unaligned(ptr) @@ -557,14 +600,10 @@ impl SIMD for f32x16 { #[inline] unsafe fn load_unaligned(ptr: *const f32) -> Self { - #[cfg(all(target_arch = "x86_64", not(target_feature = "avx512f")))] + #[cfg(target_arch = "x86_64")] unsafe { Self(_mm256_loadu_ps(ptr), _mm256_loadu_ps(ptr.add(8))) } - #[cfg(all(target_arch = "x86_64", target_feature = "avx512f"))] - unsafe { - Self(_mm512_loadu_ps(ptr)) - } #[cfg(target_arch = "aarch64")] { Self(vld1q_f32_x4(ptr)) @@ -580,11 +619,7 @@ impl SIMD for f32x16 { #[inline] unsafe fn store(&self, ptr: *mut f32) { - #[cfg(all(target_arch = "x86_64", target_feature = "avx512f"))] - unsafe { - _mm512_store_ps(ptr, self.0) - } - #[cfg(all(target_arch = "x86_64", not(target_feature = "avx512f")))] + #[cfg(target_arch = "x86_64")] unsafe { _mm256_store_ps(ptr, self.0); _mm256_store_ps(ptr.add(8), self.1); @@ -602,11 +637,7 @@ impl SIMD for f32x16 { #[inline] unsafe fn store_unaligned(&self, ptr: *mut f32) { - #[cfg(all(target_arch = "x86_64", target_feature = "avx512f"))] - unsafe { - _mm512_storeu_ps(ptr, self.0) - } - #[cfg(all(target_arch = "x86_64", not(target_feature = "avx512f")))] + #[cfg(target_arch = "x86_64")] unsafe { _mm256_storeu_ps(ptr, self.0); _mm256_storeu_ps(ptr.add(8), self.1); @@ -622,12 +653,9 @@ impl SIMD for f32x16 { } } + #[inline] fn reduce_sum(&self) -> f32 { - #[cfg(all(target_arch = "x86_64", target_feature = "avx512f"))] - unsafe { - _mm512_mask_reduce_add_ps(0xFFFF, self.0) - } - #[cfg(all(target_arch = "x86_64", not(target_feature = "avx512f")))] + #[cfg(target_arch = "x86_64")] unsafe { let mut sum = _mm256_add_ps(self.0, self.1); // Shift and add vector, until only 1 value left. @@ -657,11 +685,7 @@ impl SIMD for f32x16 { #[inline] fn reduce_min(&self) -> f32 { - #[cfg(all(target_arch = "x86_64", target_feature = "avx512f"))] - unsafe { - _mm512_mask_reduce_min_ps(0xFFFF, self.0) - } - #[cfg(all(target_arch = "x86_64", not(target_feature = "avx512f")))] + #[cfg(target_arch = "x86_64")] unsafe { let mut m1 = _mm256_min_ps(self.0, self.1); let mut m2 = _mm256_permute2f128_ps(m1, m1, 1); @@ -695,11 +719,7 @@ impl SIMD for f32x16 { #[inline] fn min(&self, rhs: &Self) -> Self { - #[cfg(all(target_arch = "x86_64", target_feature = "avx512f"))] - unsafe { - Self(_mm512_min_ps(self.0, rhs.0)) - } - #[cfg(all(target_arch = "x86_64", not(target_feature = "avx512f")))] + #[cfg(target_arch = "x86_64")] unsafe { Self(_mm256_min_ps(self.0, rhs.0), _mm256_min_ps(self.1, rhs.1)) } @@ -718,21 +738,15 @@ impl SIMD for f32x16 { } } + #[inline] fn find(&self, val: f32) -> Option { - #[cfg(all(target_arch = "x86_64", target_feature = "avx512f"))] - unsafe { - // let tgt = _mm512_set1_ps(val); - // let mask = _mm512_cmpeq_ps_mask(self.0, tgt); - // if mask != 0 { - // return Some(mask.trailing_zeros() as i32); - // } - todo!() - } - #[cfg(all(target_arch = "x86_64", not(target_feature = "avx512f")))] + #[cfg(target_arch = "x86_64")] unsafe { - // _mm256_cmpeq_ps_mask requires "avx512l". + // _mm256_cmpeq_ps_mask requires AVX-512 (avx512f); use a scalar scan here + // since we only require AVX2. + let arr = self.as_array(); for i in 0..16 { - if self.as_array().get_unchecked(i) == &val { + if arr.get_unchecked(i) == &val { return Some(i as i32); } } @@ -774,11 +788,7 @@ impl SIMD for f32x16 { impl FloatSimd for f32x16 { #[inline] fn multiply_add(&mut self, a: Self, b: Self) { - #[cfg(all(target_arch = "x86_64", target_feature = "avx512f"))] - unsafe { - self.0 = _mm512_fmadd_ps(a.0, b.0, self.0) - } - #[cfg(all(target_arch = "x86_64", not(target_feature = "avx512f")))] + #[cfg(target_arch = "x86_64")] unsafe { self.0 = _mm256_fmadd_ps(a.0, b.0, self.0); self.1 = _mm256_fmadd_ps(a.1, b.1, self.1); @@ -803,11 +813,7 @@ impl Add for f32x16 { #[inline] fn add(self, rhs: Self) -> Self::Output { - #[cfg(all(target_arch = "x86_64", target_feature = "avx512f"))] - unsafe { - Self(_mm512_add_ps(self.0, rhs.0)) - } - #[cfg(all(target_arch = "x86_64", not(target_feature = "avx512f")))] + #[cfg(target_arch = "x86_64")] unsafe { Self(_mm256_add_ps(self.0, rhs.0), _mm256_add_ps(self.1, rhs.1)) } @@ -830,11 +836,7 @@ impl Add for f32x16 { impl AddAssign for f32x16 { #[inline] fn add_assign(&mut self, rhs: Self) { - #[cfg(all(target_arch = "x86_64", target_feature = "avx512f"))] - unsafe { - self.0 = _mm512_add_ps(self.0, rhs.0) - } - #[cfg(all(target_arch = "x86_64", not(target_feature = "avx512f")))] + #[cfg(target_arch = "x86_64")] unsafe { self.0 = _mm256_add_ps(self.0, rhs.0); self.1 = _mm256_add_ps(self.1, rhs.1); @@ -859,11 +861,7 @@ impl Mul for f32x16 { #[inline] fn mul(self, rhs: Self) -> Self::Output { - #[cfg(all(target_arch = "x86_64", target_feature = "avx512f"))] - unsafe { - Self(_mm512_mul_ps(self.0, rhs.0)) - } - #[cfg(all(target_arch = "x86_64", not(target_feature = "avx512f")))] + #[cfg(target_arch = "x86_64")] unsafe { Self(_mm256_mul_ps(self.0, rhs.0), _mm256_mul_ps(self.1, rhs.1)) } @@ -888,11 +886,7 @@ impl Sub for f32x16 { #[inline] fn sub(self, rhs: Self) -> Self::Output { - #[cfg(all(target_arch = "x86_64", target_feature = "avx512f"))] - unsafe { - Self(_mm512_sub_ps(self.0, rhs.0)) - } - #[cfg(all(target_arch = "x86_64", not(target_feature = "avx512f")))] + #[cfg(target_arch = "x86_64")] unsafe { Self(_mm256_sub_ps(self.0, rhs.0), _mm256_sub_ps(self.1, rhs.1)) } @@ -915,11 +909,7 @@ impl Sub for f32x16 { impl SubAssign for f32x16 { #[inline] fn sub_assign(&mut self, rhs: Self) { - #[cfg(all(target_arch = "x86_64", target_feature = "avx512f"))] - unsafe { - self.0 = _mm512_sub_ps(self.0, rhs.0) - } - #[cfg(all(target_arch = "x86_64", not(target_feature = "avx512f")))] + #[cfg(target_arch = "x86_64")] unsafe { self.0 = _mm256_sub_ps(self.0, rhs.0); self.1 = _mm256_sub_ps(self.1, rhs.1); @@ -943,9 +933,17 @@ impl SubAssign for f32x16 { mod tests { use super::*; + use rstest::rstest; #[test] fn test_basic_ops() { + // Load / store / arithmetic on `f32x8` lower to AVX intrinsics, and + // `multiply_add` lowers to `_mm256_fmadd_ps`, which needs FMA. Both + // are present from the AvxFma tier up. + #[cfg(target_arch = "x86_64")] + if !std::is_x86_feature_detected!("avx") || !std::is_x86_feature_detected!("fma") { + return; + } let a = (0..8).map(|f| f as f32).collect::>(); let b = (10..18).map(|f| f as f32).collect::>(); @@ -983,6 +981,11 @@ mod tests { #[test] fn test_f32x8_cmp_ops() { + // `min` / `reduce_min` are AVX intrinsics; `find` is a scalar scan. + #[cfg(target_arch = "x86_64")] + if !std::is_x86_feature_detected!("avx") { + return; + } let a = [1.0_f32, 2.0, 5.0, 6.0, 7.0, 3.0, 2.0, 1.0]; let b = [2.0_f32, 1.0, 4.0, 5.0, 9.0, 5.0, 6.0, 2.0]; let c = [2.0_f32, 1.0, 4.0, 5.0, 7.0, 3.0, 2.0, 1.0]; @@ -1007,6 +1010,11 @@ mod tests { #[test] fn test_basic_f32x16_ops() { + // `f32x16` is a pair of `__m256`; `multiply_add` needs FMA. + #[cfg(target_arch = "x86_64")] + if !std::is_x86_feature_detected!("avx") || !std::is_x86_feature_detected!("fma") { + return; + } let a = (0..16).map(|f| f as f32).collect::>(); let b = (10..26).map(|f| f as f32).collect::>(); @@ -1041,6 +1049,11 @@ mod tests { #[test] fn test_f32x16_cmp_ops() { + // `min` / `reduce_min` are AVX intrinsics; `find` is a scalar scan. + #[cfg(target_arch = "x86_64")] + if !std::is_x86_feature_detected!("avx") { + return; + } let a = [ 1.0_f32, 2.0, 5.0, 6.0, 7.0, 3.0, 2.0, 1.0, -0.5, 5.0, 6.0, 7.0, 8.0, 9.0, 1.0, 2.0, ]; @@ -1074,9 +1087,59 @@ mod tests { #[test] fn test_f32x8_gather() { + // `f32x8::gather` does its own runtime AVX2 detection and falls back + // to a scalar gather, so this test only needs whatever reading the + // `__m256`-backed result costs: AVX, for `reduce_sum`. + #[cfg(target_arch = "x86_64")] + if !std::is_x86_feature_detected!("avx") { + return; + } let a = (0..256).map(|f| f as f32).collect::>(); let idx = [0_i32, 4, 8, 12, 16, 20, 24, 29]; let v = f32x8::gather(&a, &idx); assert_eq!(v.reduce_sum(), 113.0); } + + /// Directly exercises `gather_scalar_x86`, the per-index scalar fallback + /// `f32x8::gather` takes on x86_64 hosts without AVX2. Runtime AVX2 hosts + /// route through `gather_avx2` instead, so the fallback is otherwise never + /// hit under coverage. Reading the `__m256`-backed result needs AVX, so + /// skip on hosts without it. + #[cfg(target_arch = "x86_64")] + #[test] + fn test_gather_scalar_x86() { + if !std::is_x86_feature_detected!("avx") { + return; + } + let a = (0..256).map(|f| f as f32).collect::>(); + let idx = [0_i32, 4, 8, 12, 16, 20, 24, 29]; + let v = gather_scalar_x86(&a, &idx); + let expected = idx.map(|i| a[i as usize]); + assert_eq!(v.as_array(), expected); + } + + /// An index past the end of the slice panics rather than reading out of + /// bounds. The bounds check fires before any AVX instruction, so this + /// case runs on every x86_64 host. + #[cfg(target_arch = "x86_64")] + #[test] + #[should_panic(expected = "index out of bounds")] + fn test_gather_scalar_x86_rejects_out_of_range_index() { + let a = (0..8).map(|f| f as f32).collect::>(); + let idx = [0_i32, 1, 2, 3, 4, 5, 6, 99]; + let _ = gather_scalar_x86(&a, &idx); + } + + /// `gather` validates before dispatching, so every backend — `vgatherdps`, + /// the x86 scalar fallback, and the NEON / LASX raw-pointer arms — rejects + /// a bad index identically instead of reading out of bounds. + #[rstest] + #[case::past_end(99)] + #[case::negative(-1)] + #[should_panic(expected = "out of bounds")] + fn test_gather_rejects_invalid_index(#[case] bad_index: i32) { + let a = (0..8).map(|f| f as f32).collect::>(); + let idx = [0_i32, 1, 2, 3, 4, 5, 6, bad_index]; + let _ = f32x8::gather(&a, &idx); + } } diff --git a/rust/lance-linalg/src/simd/f64.rs b/rust/lance-linalg/src/simd/f64.rs index 32c0d389e5b..1276f54da56 100644 --- a/rust/lance-linalg/src/simd/f64.rs +++ b/rust/lance-linalg/src/simd/f64.rs @@ -355,14 +355,15 @@ impl Mul for f64x4 { // f64x8: 8 × f64 values (512-bit SIMD or 2 × 256-bit) // --------------------------------------------------------------------------- -/// 8 of 64-bit `f64` values. Uses 512-bit SIMD if possible. +/// 8 of 64-bit `f64` values. Stored as a pair of 256-bit AVX vectors on +/// x86_64. Originally there was a sibling AVX-512 variant gated on +/// `target_feature = "avx512f"`, but no project CI configuration enables +/// `+avx512f` globally, so the variant was dead code. Removed in the +/// runtime-SIMD-dispatch retrofit; per-tier dispatch happens in the kernel +/// functions in `crate::distance::*` via `match *SIMD_SUPPORT` + per-tier +/// `#[target_feature(enable = "...")]` inner functions. #[allow(non_camel_case_types)] -#[cfg(all(target_arch = "x86_64", target_feature = "avx512f"))] -#[derive(Clone, Copy)] -pub struct f64x8(__m512d); - -#[allow(non_camel_case_types)] -#[cfg(all(target_arch = "x86_64", not(target_feature = "avx512f")))] +#[cfg(target_arch = "x86_64")] #[derive(Clone, Copy)] pub struct f64x8(__m256d, __m256d); @@ -401,11 +402,7 @@ impl<'a> From<&'a [f64; 8]> for f64x8 { impl SIMD for f64x8 { #[inline] fn splat(val: f64) -> Self { - #[cfg(all(target_arch = "x86_64", target_feature = "avx512f"))] - unsafe { - Self(_mm512_set1_pd(val)) - } - #[cfg(all(target_arch = "x86_64", not(target_feature = "avx512f")))] + #[cfg(target_arch = "x86_64")] unsafe { Self(_mm256_set1_pd(val), _mm256_set1_pd(val)) } @@ -423,11 +420,7 @@ impl SIMD for f64x8 { #[inline] fn zeros() -> Self { - #[cfg(all(target_arch = "x86_64", target_feature = "avx512f"))] - unsafe { - Self(_mm512_setzero_pd()) - } - #[cfg(all(target_arch = "x86_64", not(target_feature = "avx512f")))] + #[cfg(target_arch = "x86_64")] unsafe { Self(_mm256_setzero_pd(), _mm256_setzero_pd()) } @@ -443,11 +436,7 @@ impl SIMD for f64x8 { #[inline] unsafe fn load(ptr: *const f64) -> Self { - #[cfg(all(target_arch = "x86_64", target_feature = "avx512f"))] - unsafe { - Self(_mm512_load_pd(ptr)) - } - #[cfg(all(target_arch = "x86_64", not(target_feature = "avx512f")))] + #[cfg(target_arch = "x86_64")] unsafe { Self(_mm256_load_pd(ptr), _mm256_load_pd(ptr.add(4))) } @@ -466,11 +455,7 @@ impl SIMD for f64x8 { #[inline] unsafe fn load_unaligned(ptr: *const f64) -> Self { - #[cfg(all(target_arch = "x86_64", target_feature = "avx512f"))] - unsafe { - Self(_mm512_loadu_pd(ptr)) - } - #[cfg(all(target_arch = "x86_64", not(target_feature = "avx512f")))] + #[cfg(target_arch = "x86_64")] unsafe { Self(_mm256_loadu_pd(ptr), _mm256_loadu_pd(ptr.add(4))) } @@ -489,11 +474,7 @@ impl SIMD for f64x8 { #[inline] unsafe fn store(&self, ptr: *mut f64) { - #[cfg(all(target_arch = "x86_64", target_feature = "avx512f"))] - unsafe { - _mm512_store_pd(ptr, self.0) - } - #[cfg(all(target_arch = "x86_64", not(target_feature = "avx512f")))] + #[cfg(target_arch = "x86_64")] unsafe { _mm256_store_pd(ptr, self.0); _mm256_store_pd(ptr.add(4), self.1); @@ -512,11 +493,7 @@ impl SIMD for f64x8 { #[inline] unsafe fn store_unaligned(&self, ptr: *mut f64) { - #[cfg(all(target_arch = "x86_64", target_feature = "avx512f"))] - unsafe { - _mm512_storeu_pd(ptr, self.0) - } - #[cfg(all(target_arch = "x86_64", not(target_feature = "avx512f")))] + #[cfg(target_arch = "x86_64")] unsafe { _mm256_storeu_pd(ptr, self.0); _mm256_storeu_pd(ptr.add(4), self.1); @@ -533,12 +510,9 @@ impl SIMD for f64x8 { } } + #[inline] fn reduce_sum(&self) -> f64 { - #[cfg(all(target_arch = "x86_64", target_feature = "avx512f"))] - unsafe { - _mm512_mask_reduce_add_pd(0xFF, self.0) - } - #[cfg(all(target_arch = "x86_64", not(target_feature = "avx512f")))] + #[cfg(target_arch = "x86_64")] unsafe { let sum = _mm256_add_pd(self.0, self.1); let hi = _mm256_permute2f128_pd(sum, sum, 1); @@ -561,11 +535,7 @@ impl SIMD for f64x8 { #[inline] fn reduce_min(&self) -> f64 { - #[cfg(all(target_arch = "x86_64", target_feature = "avx512f"))] - unsafe { - _mm512_mask_reduce_min_pd(0xFF, self.0) - } - #[cfg(all(target_arch = "x86_64", not(target_feature = "avx512f")))] + #[cfg(target_arch = "x86_64")] unsafe { let m = _mm256_min_pd(self.0, self.1); let hi = _mm256_permute2f128_pd(m, m, 1); @@ -592,11 +562,7 @@ impl SIMD for f64x8 { #[inline] fn min(&self, rhs: &Self) -> Self { - #[cfg(all(target_arch = "x86_64", target_feature = "avx512f"))] - unsafe { - Self(_mm512_min_pd(self.0, rhs.0)) - } - #[cfg(all(target_arch = "x86_64", not(target_feature = "avx512f")))] + #[cfg(target_arch = "x86_64")] unsafe { Self(_mm256_min_pd(self.0, rhs.0), _mm256_min_pd(self.1, rhs.1)) } @@ -613,6 +579,7 @@ impl SIMD for f64x8 { } } + #[inline] fn find(&self, val: f64) -> Option { unsafe { for i in 0..8 { @@ -628,11 +595,7 @@ impl SIMD for f64x8 { impl FloatSimd for f64x8 { #[inline] fn multiply_add(&mut self, a: Self, b: Self) { - #[cfg(all(target_arch = "x86_64", target_feature = "avx512f"))] - unsafe { - self.0 = _mm512_fmadd_pd(a.0, b.0, self.0) - } - #[cfg(all(target_arch = "x86_64", not(target_feature = "avx512f")))] + #[cfg(target_arch = "x86_64")] unsafe { self.0 = _mm256_fmadd_pd(a.0, b.0, self.0); self.1 = _mm256_fmadd_pd(a.1, b.1, self.1); @@ -657,11 +620,7 @@ impl Add for f64x8 { #[inline] fn add(self, rhs: Self) -> Self::Output { - #[cfg(all(target_arch = "x86_64", target_feature = "avx512f"))] - unsafe { - Self(_mm512_add_pd(self.0, rhs.0)) - } - #[cfg(all(target_arch = "x86_64", not(target_feature = "avx512f")))] + #[cfg(target_arch = "x86_64")] unsafe { Self(_mm256_add_pd(self.0, rhs.0), _mm256_add_pd(self.1, rhs.1)) } @@ -682,11 +641,7 @@ impl Add for f64x8 { impl AddAssign for f64x8 { #[inline] fn add_assign(&mut self, rhs: Self) { - #[cfg(all(target_arch = "x86_64", target_feature = "avx512f"))] - unsafe { - self.0 = _mm512_add_pd(self.0, rhs.0) - } - #[cfg(all(target_arch = "x86_64", not(target_feature = "avx512f")))] + #[cfg(target_arch = "x86_64")] unsafe { self.0 = _mm256_add_pd(self.0, rhs.0); self.1 = _mm256_add_pd(self.1, rhs.1); @@ -711,11 +666,7 @@ impl Mul for f64x8 { #[inline] fn mul(self, rhs: Self) -> Self::Output { - #[cfg(all(target_arch = "x86_64", target_feature = "avx512f"))] - unsafe { - Self(_mm512_mul_pd(self.0, rhs.0)) - } - #[cfg(all(target_arch = "x86_64", not(target_feature = "avx512f")))] + #[cfg(target_arch = "x86_64")] unsafe { Self(_mm256_mul_pd(self.0, rhs.0), _mm256_mul_pd(self.1, rhs.1)) } @@ -738,11 +689,7 @@ impl Sub for f64x8 { #[inline] fn sub(self, rhs: Self) -> Self::Output { - #[cfg(all(target_arch = "x86_64", target_feature = "avx512f"))] - unsafe { - Self(_mm512_sub_pd(self.0, rhs.0)) - } - #[cfg(all(target_arch = "x86_64", not(target_feature = "avx512f")))] + #[cfg(target_arch = "x86_64")] unsafe { Self(_mm256_sub_pd(self.0, rhs.0), _mm256_sub_pd(self.1, rhs.1)) } @@ -763,11 +710,7 @@ impl Sub for f64x8 { impl SubAssign for f64x8 { #[inline] fn sub_assign(&mut self, rhs: Self) { - #[cfg(all(target_arch = "x86_64", target_feature = "avx512f"))] - unsafe { - self.0 = _mm512_sub_pd(self.0, rhs.0) - } - #[cfg(all(target_arch = "x86_64", not(target_feature = "avx512f")))] + #[cfg(target_arch = "x86_64")] unsafe { self.0 = _mm256_sub_pd(self.0, rhs.0); self.1 = _mm256_sub_pd(self.1, rhs.1); @@ -793,6 +736,12 @@ mod tests { #[test] fn test_f64x4_basic_ops() { + // The `f64x4` constructor / load / store / arithmetic paths all lower + // to AVX intrinsics on x86_64; none of them need AVX2. + #[cfg(target_arch = "x86_64")] + if !std::is_x86_feature_detected!("avx") { + return; + } let a = [1.0_f64, 2.0, 3.0, 4.0]; let b = [5.0_f64, 6.0, 7.0, 8.0]; @@ -814,6 +763,11 @@ mod tests { #[test] fn test_f64x4_fma() { + // `multiply_add` lowers to `_mm256_fmadd_pd`, which needs FMA. + #[cfg(target_arch = "x86_64")] + if !std::is_x86_feature_detected!("avx") || !std::is_x86_feature_detected!("fma") { + return; + } let a = [1.0_f64, 2.0, 3.0, 4.0]; let b = [2.0_f64, 3.0, 4.0, 5.0]; @@ -826,6 +780,11 @@ mod tests { #[test] fn test_f64x4_min() { + // `min` / `reduce_min` are AVX intrinsics. + #[cfg(target_arch = "x86_64")] + if !std::is_x86_feature_detected!("avx") { + return; + } let a = [1.0_f64, 5.0, 2.0, 8.0]; let b = [3.0_f64, 2.0, 4.0, 1.0]; let simd_a: f64x4 = (&a).into(); @@ -838,6 +797,11 @@ mod tests { #[test] fn test_f64x8_basic_ops() { + // `f64x8` is a pair of `__m256d`; add / reduce are AVX intrinsics. + #[cfg(target_arch = "x86_64")] + if !std::is_x86_feature_detected!("avx") { + return; + } let a: [f64; 8] = [1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0]; let b: [f64; 8] = [10.0, 20.0, 30.0, 40.0, 50.0, 60.0, 70.0, 80.0]; @@ -856,6 +820,11 @@ mod tests { #[test] fn test_f64x8_fma() { + // `multiply_add` lowers to `_mm256_fmadd_pd`, which needs FMA. + #[cfg(target_arch = "x86_64")] + if !std::is_x86_feature_detected!("avx") || !std::is_x86_feature_detected!("fma") { + return; + } let a: [f64; 8] = [1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0]; let b: [f64; 8] = [1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0]; @@ -869,6 +838,11 @@ mod tests { #[test] fn test_f64x8_min() { + // `min` / `reduce_min` are AVX intrinsics. + #[cfg(target_arch = "x86_64")] + if !std::is_x86_feature_detected!("avx") { + return; + } let a: [f64; 8] = [5.0, 1.0, 8.0, 3.0, 9.0, 2.0, 7.0, 4.0]; let b: [f64; 8] = [2.0, 6.0, 3.0, 7.0, 1.0, 8.0, 4.0, 9.0]; let simd_a: f64x8 = (&a).into(); diff --git a/rust/lance-linalg/src/simd/x86.rs b/rust/lance-linalg/src/simd/x86.rs new file mode 100644 index 00000000000..437018669e2 --- /dev/null +++ b/rust/lance-linalg/src/simd/x86.rs @@ -0,0 +1,89 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright The Lance Authors + +//! Reduction helpers shared by the x86_64 AVX kernels in [`crate::distance`]. +//! +//! Each distance kernel accumulates into a 256-bit register and folds it down +//! to a scalar once, after the main loop. The fold is identical across +//! `cosine`, `dot`, `l2` and `norm_l2`, so it lives here instead of being +//! copied into each kernel's private `mod x86`. +//! +//! The module itself is `pub(crate)`, which is what keeps these helpers off the +//! public API; the items are `pub` rather than `pub(crate)` only because +//! `clippy::redundant_pub_crate` fires on the narrower visibility. + +use std::arch::x86_64::*; + +/// Horizontal sum of the eight `f32` lanes of an `__m256`. +/// +/// Folds the upper 128-bit lane into the lower one, then reduces the +/// remaining four lanes pairwise. Uses `movehl`/`shuffle` plus scalar adds +/// rather than two `vhaddps`, which is one fewer uop on most cores. +/// +/// # Safety +/// +/// The host must support AVX. Callers are `#[target_feature]`-annotated +/// kernels that the runtime dispatcher only selects after checking. +#[inline] +#[target_feature(enable = "avx")] +pub unsafe fn hsum256_ps(v: __m256) -> f32 { + let lo = _mm256_castps256_ps128(v); + let hi = _mm256_extractf128_ps(v, 1); + let sum128 = _mm_add_ps(lo, hi); + let sum64 = _mm_add_ps(sum128, _mm_movehl_ps(sum128, sum128)); + // 0x55 broadcasts lane 1 into lane 0, so the scalar add below lands the + // last of the four partial sums. + let sum32 = _mm_add_ss(sum64, _mm_shuffle_ps(sum64, sum64, 0x55)); + _mm_cvtss_f32(sum32) +} + +/// Horizontal sum of the four `f64` lanes of an `__m256d`. +/// +/// Folds the upper 128-bit lane into the lower one, then adds the remaining +/// pair. +/// +/// # Safety +/// +/// The host must support AVX. Callers are `#[target_feature]`-annotated +/// kernels that the runtime dispatcher only selects after checking. +#[inline] +#[target_feature(enable = "avx")] +pub unsafe fn hsum256_pd(v: __m256d) -> f64 { + let lo = _mm256_castpd256_pd128(v); + let hi = _mm256_extractf128_pd(v, 1); + let sum128 = _mm_add_pd(lo, hi); + let sum64 = _mm_add_pd(sum128, _mm_unpackhi_pd(sum128, sum128)); + _mm_cvtsd_f64(sum64) +} + +#[cfg(test)] +mod tests { + use super::*; + use rstest::rstest; + + #[rstest] + #[case::ascending([1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0], 36.0)] + #[case::negative_lanes([-1.5, 2.0, -3.0, 4.5, 0.0, -0.5, 1.0, 2.5], 5.0)] + #[case::zeros([0.0; 8], 0.0)] + #[case::cancelling([1.0, -1.0, 2.0, -2.0, 3.0, -3.0, 4.0, -4.0], 0.0)] + fn hsum256_ps_sums_every_lane(#[case] lanes: [f32; 8], #[case] expected: f32) { + if !std::is_x86_feature_detected!("avx") { + return; + } + let sum = unsafe { hsum256_ps(_mm256_loadu_ps(lanes.as_ptr())) }; + assert_eq!(sum, expected); + } + + #[rstest] + #[case::ascending([1.0, 2.0, 3.0, 4.0], 10.0)] + #[case::negative_lanes([-1.5, 2.0, -3.0, 4.5], 2.0)] + #[case::zeros([0.0; 4], 0.0)] + #[case::cancelling([1.0, -1.0, 2.0, -2.0], 0.0)] + fn hsum256_pd_sums_every_lane(#[case] lanes: [f64; 4], #[case] expected: f64) { + if !std::is_x86_feature_detected!("avx") { + return; + } + let sum = unsafe { hsum256_pd(_mm256_loadu_pd(lanes.as_ptr())) }; + assert_eq!(sum, expected); + } +} diff --git a/rust/lance-namespace-datafusion/src/catalog.rs b/rust/lance-namespace-datafusion/src/catalog.rs index 4fe57f63c9b..ce699037ba0 100755 --- a/rust/lance-namespace-datafusion/src/catalog.rs +++ b/rust/lance-namespace-datafusion/src/catalog.rs @@ -1,7 +1,6 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright The Lance Authors -use std::any::Any; use std::collections::HashSet; use std::sync::Arc; @@ -53,10 +52,6 @@ impl LanceCatalogProviderList { } impl CatalogProviderList for LanceCatalogProviderList { - fn as_any(&self) -> &dyn Any { - self - } - /// Adds a new catalog to this catalog list. /// If a catalog of the same name existed before, it is replaced in the list and returned. fn register_catalog( @@ -116,10 +111,6 @@ impl LanceCatalogProvider { } impl CatalogProvider for LanceCatalogProvider { - fn as_any(&self) -> &dyn Any { - self - } - fn schema_names(&self) -> Vec { self.schemas .iter() diff --git a/rust/lance-namespace-datafusion/src/schema.rs b/rust/lance-namespace-datafusion/src/schema.rs index 9acf30a97bf..194346001c3 100755 --- a/rust/lance-namespace-datafusion/src/schema.rs +++ b/rust/lance-namespace-datafusion/src/schema.rs @@ -1,7 +1,6 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright The Lance Authors -use std::any::Any; use std::sync::Arc; use async_trait::async_trait; @@ -51,10 +50,6 @@ impl LanceSchemaProvider { #[async_trait] impl SchemaProvider for LanceSchemaProvider { - fn as_any(&self) -> &dyn Any { - self - } - fn table_names(&self) -> Vec { self.tables .iter() diff --git a/rust/lance-namespace-impls/Cargo.toml b/rust/lance-namespace-impls/Cargo.toml index 27b9a4bc0e2..d682d39f02b 100644 --- a/rust/lance-namespace-impls/Cargo.toml +++ b/rust/lance-namespace-impls/Cargo.toml @@ -79,11 +79,6 @@ base64 = { version = "0.22", optional = true } aws-sdk-sts = { version = "1.38.0", optional = true, default-features = false, features = ["default-https-client", "rt-tokio"] } aws-config = { workspace = true, optional = true } -# Pin: time 0.3.48 conflicts with aws-smithy-types (E0119: conflicting `From` impls), which this -# crate pulls in via the AWS credential vendor. Capping time here forces the workspace resolver to -# 0.3.47 even for no-lock builds. Not used directly; remove once the upstream conflict is resolved. -time = "=0.3.47" - # GCP credential vending dependencies (optional, enabled by "credential-vendor-gcp" feature) ring = { version = "0.17", optional = true } rustls-pki-types = { version = "1", optional = true } @@ -91,7 +86,7 @@ rustls-pki-types = { version = "1", optional = true } # Azure credential vending dependencies (optional, enabled by "credential-vendor-azure" feature) chrono = { workspace = true, optional = true } hmac = { version = "0.12", optional = true } -quick-xml = { version = "0.38", optional = true } +quick-xml = { version = "0.40", optional = true } [dev-dependencies] opendal = { workspace = true, features = ["services-goosefs"] } diff --git a/rust/lance-namespace-impls/src/credentials.rs b/rust/lance-namespace-impls/src/credentials.rs index e841ac620f6..23f9346cfb3 100644 --- a/rust/lance-namespace-impls/src/credentials.rs +++ b/rust/lance-namespace-impls/src/credentials.rs @@ -223,6 +223,14 @@ pub mod aws_props { /// AWS credential duration in milliseconds. /// Default: 3600000 (1 hour). Range: 900000 (15 min) to 43200000 (12 hours). pub const DURATION_MILLIS: &str = "aws_duration_millis"; + + /// When "true", the scoped assume is performed via `AssumeRoleWithWebIdentity` + /// using the pod's projected service-account OIDC token + pub const ASSUME_VIA_POD_WEB_IDENTITY: &str = "aws_assume_via_pod_web_identity"; + + /// Explicit path to the pod's projected SA OIDC token file. Overrides + /// `AWS_WEB_IDENTITY_TOKEN_FILE` when set. + pub const POD_WEB_IDENTITY_TOKEN_FILE: &str = "aws_pod_web_identity_token_file"; } /// GCP-specific property keys (short form, without prefix) @@ -503,6 +511,43 @@ async fn create_aws_vendor( config = config.with_role_session_name(session_name); } + // Direct (non-chained) web-identity assume for the pod, when enabled. An + // explicit token-file path wins; otherwise, if opted in, resolve the + // EKS-injected `AWS_WEB_IDENTITY_TOKEN_FILE`. Falling back to the chained + // AssumeRole path when neither is present keeps existing behavior. + let assume_via_pod = properties + .get(aws_props::ASSUME_VIA_POD_WEB_IDENTITY) + .map(|v| v.eq_ignore_ascii_case("true")) + .unwrap_or(false); + let pod_token_file = properties + .get(aws_props::POD_WEB_IDENTITY_TOKEN_FILE) + .cloned() + .or_else(|| { + assume_via_pod + .then(|| std::env::var("AWS_WEB_IDENTITY_TOKEN_FILE").ok()) + .flatten() + }); + // Log the resolved assume path once at vendor init so a deployment can + // confirm at runtime which branch `assume_scoped` will take. + match &pod_token_file { + Some(path) => log::info!( + "AWS credential vendor (role {role_arn}): direct AssumeRoleWithWebIdentity \ + via pod token file '{path}'" + ), + None if assume_via_pod => log::warn!( + "AWS credential vendor (role {role_arn}): aws_assume_via_pod_web_identity=true \ + but no token file resolved (aws_pod_web_identity_token_file unset and \ + AWS_WEB_IDENTITY_TOKEN_FILE not in env); falling back to chained AssumeRole" + ), + None => log::info!( + "AWS credential vendor (role {role_arn}): chained AssumeRole \ + (pod web-identity not enabled)" + ), + } + if let Some(path) = pod_token_file { + config = config.with_pod_web_identity_token_file(path); + } + let vendor = AwsCredentialVendor::new(config).await?; Ok(Some(Box::new(vendor))) } diff --git a/rust/lance-namespace-impls/src/credentials/aws.rs b/rust/lance-namespace-impls/src/credentials/aws.rs index 7dedaa6e108..56dc9a54c2c 100644 --- a/rust/lance-namespace-impls/src/credentials/aws.rs +++ b/rust/lance-namespace-impls/src/credentials/aws.rs @@ -24,6 +24,18 @@ use super::{ redact_credential, }; +/// Render an error together with its full `source()` chain. +fn full_error_chain(err: &dyn std::error::Error) -> String { + let mut out = err.to_string(); + let mut source = err.source(); + while let Some(cause) = source { + out.push_str(": "); + out.push_str(&cause.to_string()); + source = cause.source(); + } + out +} + /// Configuration for AWS credential vending. #[derive(Debug, Clone)] pub struct AwsCredentialVendorConfig { @@ -60,6 +72,17 @@ pub struct AwsCredentialVendorConfig { /// When an API key is provided, its hash is looked up in this map. /// If found, the mapped permission is used instead of the default permission. pub api_key_hash_permissions: HashMap, + + /// Optional path to the pod's projected service-account OIDC token file (typically the + /// EKS-injected `AWS_WEB_IDENTITY_TOKEN_FILE`). + /// This is the recommended method when running in Kubernetes. + /// When set, the scoped assume is performed with `AssumeRoleWithWebIdentity` + /// using this token instead of a role-chained `AssumeRole` from the process's + /// ambient credentials. A web-identity assumption is *not* role chaining, so + /// STS honors the role's `MaxSessionDuration` (up to 12h) and the returned + /// expiration is accurate. When `None`, the `AssumeRole` path + /// is used and `duration_millis` is subject to the source role duration and may be invalid + pub pod_web_identity_token_file: Option, } impl AwsCredentialVendorConfig { @@ -74,6 +97,7 @@ impl AwsCredentialVendorConfig { permission: VendedPermission::default(), api_key_salt: None, api_key_hash_permissions: HashMap::new(), + pod_web_identity_token_file: None, } } @@ -101,6 +125,14 @@ impl AwsCredentialVendorConfig { self } + /// Set the pod web-identity token file, enabling direct (non-chained) + /// `AssumeRoleWithWebIdentity` for the scoped assume. See + /// [`AwsCredentialVendorConfig::pod_web_identity_token_file`]. + pub fn with_pod_web_identity_token_file(mut self, path: impl Into) -> Self { + self.pod_web_identity_token_file = Some(path.into()); + self + } + /// Set the permission level for vended credentials. pub fn with_permission(mut self, permission: VendedPermission) -> Self { self.permission = permission; @@ -407,7 +439,8 @@ impl AwsCredentialVendor { lance_core::Error::from(NamespaceError::Internal { message: format!( "AssumeRoleWithWebIdentity failed for role '{}': {}", - self.config.role_arn, e + self.config.role_arn, + full_error_chain(&e) ), }) })?; @@ -421,6 +454,95 @@ impl AwsCredentialVendor { } /// Vend credentials using AssumeRole with API key validation. + /// Perform the scoped assume for `(bucket, prefix, permission)`, attaching the + /// per-table session policy. + /// + /// When [`AwsCredentialVendorConfig::pod_web_identity_token_file`] is set this + /// uses `AssumeRoleWithWebIdentity` with the pod's projected SA OIDC token -- a + /// *direct*, non-chained assumption that honors the role's `MaxSessionDuration` + /// (so `expires_at_millis` is accurate). Otherwise it falls back to the legacy + /// role-chained `AssumeRole` from ambient credentials (STS-capped at 1h). + /// + /// `external_id` is only applied on the chained `AssumeRole` path; + /// `AssumeRoleWithWebIdentity` has no external-id parameter (the OIDC + /// `sub`/`aud` trust condition is the binding instead). + async fn assume_scoped( + &self, + bucket: &str, + prefix: &str, + permission: VendedPermission, + session_name: &str, + external_id: Option<&str>, + ) -> Result { + let policy = Self::build_policy(bucket, prefix, permission); + let duration_secs = self.config.duration_millis.div_ceil(1000).clamp(900, 43200) as i32; + + let credentials = if let Some(token_file) = &self.config.pod_web_identity_token_file { + // DIRECT (non-chained): AssumeRoleWithWebIdentity with the pod's SA + // OIDC token. Re-read the file every vend -- kubelet rotates it. + let token = tokio::fs::read_to_string(token_file).await.map_err(|e| { + lance_core::Error::from(NamespaceError::Internal { + message: format!( + "failed to read pod web identity token '{}': {}", + token_file, e + ), + }) + })?; + debug!( + "AWS AssumeRoleWithWebIdentity (pod): role={}, session={}, permission={}", + self.config.role_arn, session_name, permission + ); + let response = self + .sts_client + .assume_role_with_web_identity() + .role_arn(&self.config.role_arn) + .web_identity_token(token.trim()) + .role_session_name(session_name) + .policy(&policy) + .duration_seconds(duration_secs) + .send() + .await + .map_err(|e| { + lance_core::Error::from(NamespaceError::Internal { + message: format!( + "AssumeRoleWithWebIdentity (pod) failed for role '{}': {}", + self.config.role_arn, + full_error_chain(&e) + ), + }) + })?; + response.credentials().cloned() + } else { + // LEGACY chained path: AssumeRole from ambient credentials (1h cap). + debug!( + "AWS AssumeRole (chained): role={}, session={}, permission={}", + self.config.role_arn, session_name, permission + ); + let mut request = self + .sts_client + .assume_role() + .role_arn(&self.config.role_arn) + .role_session_name(session_name) + .policy(&policy) + .duration_seconds(duration_secs); + if let Some(external_id) = external_id { + request = request.external_id(external_id); + } + let response = request.send().await.map_err(|e| { + lance_core::Error::from(NamespaceError::Internal { + message: format!( + "AssumeRole failed for role '{}': {}", + self.config.role_arn, + full_error_chain(&e) + ), + }) + })?; + response.credentials().cloned() + }; + + self.extract_credentials(credentials.as_ref(), bucket, prefix, permission) + } + async fn vend_with_api_key( &self, bucket: &str, @@ -452,42 +574,19 @@ impl AwsCredentialVendor { }) })?; - let policy = Self::build_policy(bucket, prefix, permission); + // The api_key authorizes the client and picks the permission; the AWS + // assume itself goes through `assume_scoped` (pod web-identity when + // configured, else chained AssumeRole with the key hash as external_id). let session_name = Self::cap_session_name(&format!("lance-api-{}", &key_hash[..16])); - let duration_secs = self.config.duration_millis.div_ceil(1000).clamp(900, 43200) as i32; - - debug!( - "AWS AssumeRole with API key: role={}, session={}, permission={}", - self.config.role_arn, session_name, permission - ); - - let request = self - .sts_client - .assume_role() - .role_arn(&self.config.role_arn) - .role_session_name(&session_name) - .policy(&policy) - .duration_seconds(duration_secs) - .external_id(&key_hash); // Use hash as external_id - - let response = request.send().await.map_err(|e| { - lance_core::Error::from(NamespaceError::Internal { - message: format!( - "AssumeRole with API key failed for role '{}': {}", - self.config.role_arn, e - ), - }) - })?; - - self.extract_credentials(response.credentials(), bucket, prefix, permission) + self.assume_scoped(bucket, prefix, permission, &session_name, Some(&key_hash)) + .await } - /// Vend credentials using AssumeRole with static configuration. + /// Vend credentials using the vendor's static (default) permission. async fn vend_with_static_config( &self, bucket: &str, prefix: &str, - policy: &str, ) -> Result { let role_session_name = self .config @@ -496,40 +595,14 @@ impl AwsCredentialVendor { .unwrap_or_else(|| "lance-credential-vending".to_string()); let role_session_name = Self::cap_session_name(&role_session_name); - let duration_secs = self.config.duration_millis.div_ceil(1000).clamp(900, 43200) as i32; - - debug!( - "AWS AssumeRole (static): role={}, session={}, permission={}", - self.config.role_arn, role_session_name, self.config.permission - ); - - let mut request = self - .sts_client - .assume_role() - .role_arn(&self.config.role_arn) - .role_session_name(&role_session_name) - .policy(policy) - .duration_seconds(duration_secs); - - if let Some(ref external_id) = self.config.external_id { - request = request.external_id(external_id); - } - - let response = request.send().await.map_err(|e| { - lance_core::Error::from(NamespaceError::Internal { - message: format!( - "AssumeRole failed for role '{}': {}", - self.config.role_arn, e - ), - }) - })?; - - self.extract_credentials( - response.credentials(), + self.assume_scoped( bucket, prefix, self.config.permission, + &role_session_name, + self.config.external_id.as_deref(), ) + .await } } @@ -575,10 +648,8 @@ impl CredentialVendor for AwsCredentialVendor { .into()) } None => { - // Use AssumeRole with static configuration - let policy = Self::build_policy(&bucket, &prefix, self.config.permission); - self.vend_with_static_config(&bucket, &prefix, &policy) - .await + // Use the vendor's static (default) permission + self.vend_with_static_config(&bucket, &prefix).await } } } @@ -743,6 +814,41 @@ mod tests { assert_eq!(config.duration_millis, 7200000); assert_eq!(config.role_session_name, Some("my-session".to_string())); assert_eq!(config.region, Some("us-west-2".to_string())); + // Defaults to the legacy chained AssumeRole path. + assert_eq!(config.pod_web_identity_token_file, None); + + let pod_config = AwsCredentialVendorConfig::new("arn:aws:iam::123456789012:role/MyRole") + .with_pod_web_identity_token_file("/var/run/secrets/.../token"); + assert_eq!( + pod_config.pod_web_identity_token_file, + Some("/var/run/secrets/.../token".to_string()) + ); + } + + #[tokio::test] + async fn test_pod_web_identity_path_reads_token_file() { + // When pod_web_identity_token_file is set, the scoped assume takes the + // AssumeRoleWithWebIdentity branch and reads the token file first. Point + // it at a missing file so we deterministically hit the read error without + // needing a live STS -- this proves the branch selection + file read. + let sdk_config = aws_config::SdkConfig::builder() + .behavior_version(aws_config::BehaviorVersion::latest()) + .region(aws_config::Region::new("us-east-2")) + .build(); + let sts_client = StsClient::new(&sdk_config); + let config = AwsCredentialVendorConfig::new("arn:aws:iam::123456789012:role/MyRole") + .with_pod_web_identity_token_file("/nonexistent/pod/web-identity-token"); + let vendor = AwsCredentialVendor::with_sts_client(config, sts_client); + + let err = vendor + .vend_credentials("s3://bucket/prefix", None) + .await + .expect_err("missing token file must fail before any STS call"); + assert!( + err.to_string() + .contains("failed to read pod web identity token"), + "unexpected error: {err}" + ); } // ============================================================================ diff --git a/rust/lance-namespace-impls/src/dir.rs b/rust/lance-namespace-impls/src/dir.rs index dc2d83cf278..132f36460b7 100644 --- a/rust/lance-namespace-impls/src/dir.rs +++ b/rust/lance-namespace-impls/src/dir.rs @@ -34,15 +34,19 @@ use lance_index::vector::{ sq::builder::SQBuildParams, }; use lance_index::{IndexType, is_system_index}; +use lance_io::object_store::throttle::is_throttle_error; use lance_io::object_store::{ObjectStore, ObjectStoreParams, ObjectStoreRegistry}; use lance_linalg::distance::MetricType; use lance_table::io::commit::{ManifestNamingScheme, VERSIONS_DIR}; use object_store::ObjectStoreExt; use object_store::path::Path; -use object_store::{Error as ObjectStoreError, ObjectStore as OSObjectStore, PutMode, PutOptions}; +use object_store::{ + Error as ObjectStoreError, ObjectMeta, ObjectStore as OSObjectStore, PutMode, PutOptions, +}; use std::collections::HashMap; use std::io::Cursor; use std::sync::{Arc, Mutex}; +use tokio::sync::OnceCell; use crate::context::DynamicContextProvider; use lance_namespace::models::{ @@ -77,6 +81,9 @@ use lance_namespace::models::{ }; use lance_core::{Error, Result, box_error}; +use lance_index::scalar::inverted::query::{ + BooleanQuery, BoostQuery, FtsQuery, MatchQuery, MultiMatchQuery, Occur, Operator, PhraseQuery, +}; use lance_namespace::LanceNamespace; use lance_namespace::error::NamespaceError; use lance_namespace::schema::arrow_schema_to_json; @@ -738,7 +745,7 @@ impl DirectoryNamespaceBuilder { Self::initialize_object_store(&self.root, &self.storage_options, &self.session).await?; let manifest_ns = if self.manifest_enabled { - match manifest::ManifestNamespace::from_directory( + match manifest::ManifestNamespace::open_from_directory( self.root.clone(), self.storage_options.clone(), self.session.clone(), @@ -757,18 +764,19 @@ impl DirectoryNamespaceBuilder { // degrading to a directory-listing view that ignores it. return Err(e); } - Err(e) => { - // Failed to initialize manifest namespace, fall back to directory listing only - log::warn!( - "Failed to initialize manifest namespace, falling back to directory listing only: {}", - e - ); + Err(e) if manifest::ManifestNamespace::is_not_found_load_error(&e) => { + log::debug!("Manifest namespace does not exist yet: {}", e); None } + Err(e) => return Err(e), } } else { None }; + let manifest_cell = OnceCell::new(); + if let Some(manifest_ns) = manifest_ns { + let _ = manifest_cell.set(manifest_ns); + } // Create credential vendor once during initialization if enabled let credential_vendor = if has_credential_vendor_config(&self.credential_vendor_properties) @@ -792,8 +800,12 @@ impl DirectoryNamespaceBuilder { session: self.session, object_store, base_path, - manifest_ns, + manifest_ns: manifest_cell, + write_manifest_ns: OnceCell::new(), + manifest_enabled: self.manifest_enabled, dir_listing_enabled: self.dir_listing_enabled, + inline_optimization_enabled: self.inline_optimization_enabled, + commit_retries: self.commit_retries, dir_listing_to_manifest_migration_enabled: self .dir_listing_to_manifest_migration_enabled, table_version_tracking_enabled: self.table_version_tracking_enabled, @@ -870,8 +882,12 @@ pub struct DirectoryNamespace { session: Option>, object_store: Arc, base_path: Path, - manifest_ns: Option>, + manifest_ns: OnceCell>, + write_manifest_ns: OnceCell>, + manifest_enabled: bool, dir_listing_enabled: bool, + inline_optimization_enabled: bool, + commit_retries: Option, /// When true, root-level table operations check the manifest first before /// falling back to directory listing. When false, root-level tables skip /// the manifest check and use directory listing directly. @@ -907,6 +923,16 @@ impl std::fmt::Display for DirectoryNamespace { } } +/// Inputs for resolving an already-published `create_table_version` target. +struct ExistingTableVersionResolve<'a> { + staging_path: &'a Path, + final_path: &'a Path, + version: u64, + table_uri: &'a str, + final_meta: &'a ObjectMeta, + request_manifest_size: Option, +} + /// Describes the version ranges to delete for a single table. /// Used by `batch_delete_table_versions` and `delete_physical_version_files`. struct TableDeleteEntry { @@ -983,6 +1009,98 @@ impl TransactionAlteration { } impl DirectoryNamespace { + fn manifest_ns_for_read(&self) -> Option<&Arc> { + self.write_manifest_ns + .get() + .or_else(|| self.manifest_ns.get()) + } + + async fn manifest_ns_for_write(&self) -> Result>> { + if !self.manifest_enabled { + return Ok(None); + } + + let manifest_ns = self + .write_manifest_ns + .get_or_try_init(|| async { + manifest::ManifestNamespace::from_directory( + self.root.clone(), + self.storage_options.clone(), + self.session.clone(), + self.object_store.clone(), + self.base_path.clone(), + self.dir_listing_enabled, + self.inline_optimization_enabled, + self.commit_retries, + ) + .await + .map(Arc::new) + }) + .await?; + Ok(Some(manifest_ns.clone())) + } + + /// Lazily open the `__manifest` dataset (read-only) into the read cell. + /// + /// `manifest_ns` is populated at construction only if `__manifest` already + /// existed then. A manifest created afterwards -- e.g. by another + /// connection's write, since Phalanx caches a connection per db and the + /// first op on a fresh db is usually a read -- would otherwise stay + /// invisible to this connection's reads forever, so describe/list/exists + /// report "not found" for a table that is in fact registered. Re-open on + /// demand so reads self-heal once the manifest exists. Idempotent and cheap + /// once the cell is populated; unlike `manifest_ns_for_write` it never + /// creates the manifest. + async fn ensure_read_manifest(&self) -> Result<()> { + if !self.manifest_enabled + || self.manifest_ns.get().is_some() + || self.write_manifest_ns.get().is_some() + { + return Ok(()); + } + match self + .manifest_ns + .get_or_try_init(|| async { + manifest::ManifestNamespace::open_from_directory( + self.root.clone(), + self.storage_options.clone(), + self.session.clone(), + self.object_store.clone(), + self.base_path.clone(), + self.dir_listing_enabled, + self.inline_optimization_enabled, + self.commit_retries, + ) + .await + .map(Arc::new) + }) + .await + { + Ok(_) => Ok(()), + // Manifest still doesn't exist: leave the cell empty so a later read + // retries once it does. A genuinely absent table is still reported + // not-found by the callers' existing `manifest_ns_for_read() == None` + // branch, exactly as before. + Err(e) if manifest::ManifestNamespace::is_not_found_load_error(&e) => Ok(()), + Err(e) => Err(e), + } + } + + fn child_namespace_requires_manifest_error(&self) -> Error { + if self.manifest_enabled { + NamespaceError::NamespaceNotFound { + message: "Child namespace reads require an existing __manifest dataset".to_string(), + } + .into() + } else { + NamespaceError::Unsupported { + message: "Child namespaces are only supported when manifest mode is enabled" + .to_string(), + } + .into() + } + } + /// Apply pagination to a list of table names /// /// Sorts the list alphabetically and applies pagination using page_token (start_after) and limit. @@ -1056,7 +1174,7 @@ impl DirectoryNamespace { let table_name = &path[..path.len() - 6]; // Use atomic check to skip deregistered tables. - let status = self.check_table_status(table_name).await; + let status = self.check_table_status(table_name).await?; if status.is_deregistered { continue; } @@ -1407,17 +1525,19 @@ impl DirectoryNamespace { .load() .await .map_err(|e| { - lance_core::Error::from(NamespaceError::TableNotFound { - message: format!( - "branch '{}' not found for table at '{}': {}", - branch, table_uri, e - ), - }) + let message = format!( + "branch '{}' not found for table at '{}': {}", + branch, table_uri, e + ); + Self::map_open_error(e, NamespaceError::TableNotFound { message }) })?; - dataset.branches().get(branch).await.map_err(|_| { - lance_core::Error::from(NamespaceError::TableNotFound { - message: format!("branch '{}' not found for table at '{}'", branch, table_uri), - }) + dataset.branches().get(branch).await.map_err(|e| { + Self::map_open_error( + e, + NamespaceError::TableNotFound { + message: format!("branch '{}' not found for table at '{}'", branch, table_uri), + }, + ) })?; Ok(dataset) } @@ -1430,29 +1550,35 @@ impl DirectoryNamespace { .uri) } - /// Resolves a branch to its `(uri, object-store path)` for `create_table_version`. + /// Resolves a branch to its `(uri, object-store path, parent_version)` for + /// `create_table_version`. /// /// `BranchContents` is the source of truth, so check the ref first: a - /// registered branch commits directly. With no ref, accept the commit only on - /// an empty chain (the `create_branch` bootstrap, whose first commit precedes - /// its ref); reject a chain that already holds committed versions as a zombie. + /// registered branch commits directly and returns its `parent_version` for + /// empty-chain CAS. With no ref, accept the commit only on an empty chain + /// (the `create_branch` bootstrap, whose first commit precedes its ref) and + /// return `parent_version = None`; reject a chain that already holds + /// committed versions as a zombie. async fn resolve_branch_for_commit( &self, table_uri: &str, branch: &str, - ) -> Result<(String, Path)> { + ) -> Result<(String, Path, Option)> { let main = self .configured_builder(table_uri) .load() .await .map_err(|e| { - lance_core::Error::from(NamespaceError::TableNotFound { - message: format!("table at '{}' not found: {}", table_uri, e), - }) + let message = format!("table at '{}' not found: {}", table_uri, e); + Self::map_open_error(e, NamespaceError::TableNotFound { message }) })?; let branch_location = main.branch_location().find_branch(Some(branch))?; match main.branches().get(branch).await { - Ok(_) => Ok((branch_location.uri, branch_location.path)), + Ok(contents) => Ok(( + branch_location.uri, + branch_location.path, + Some(contents.parent_version), + )), Err(lance_core::Error::RefNotFound { .. }) => { if self .branch_has_committed_versions(&branch_location.path) @@ -1466,7 +1592,7 @@ impl DirectoryNamespace { } .into()); } - Ok((branch_location.uri, branch_location.path)) + Ok((branch_location.uri, branch_location.path, None)) } Err(e) => Err(e), } @@ -1543,6 +1669,235 @@ impl DirectoryNamespace { ManifestNamingScheme::detect_scheme(filename)?.parse_version(filename) } + /// Build a successful `CreateTableVersionResponse` from an existing final manifest. + fn create_table_version_response( + version: u64, + final_path: &Path, + final_meta: &ObjectMeta, + ) -> CreateTableVersionResponse { + CreateTableVersionResponse { + transaction_id: None, + version: Some(Box::new(TableVersion { + version: version as i64, + manifest_path: final_path.to_string(), + manifest_size: Some(final_meta.size as i64), + e_tag: final_meta.e_tag.clone(), + timestamp_millis: None, + metadata: None, + })), + } + } + + /// Whether the staging blob matches the already-published version blob. + /// + /// Used for idempotent retries of `create_table_version`. Object-store + /// `e_tag` is opaque metadata (not a validated content hash) and may also + /// change across Create/rename materialize, so it is never used for + /// identity. Size mismatch is a cheap negative check; byte equality is the + /// durable success condition. + async fn staging_matches_final_manifest( + &self, + staging_path: &Path, + final_path: &Path, + final_meta: &ObjectMeta, + request_manifest_size: Option, + ) -> Result { + if let Some(size) = request_manifest_size + && size != final_meta.size as i64 + { + return Ok(false); + } + + let staging_bytes = match self.object_store.inner.get(staging_path).await { + Ok(r) => r.bytes().await.map_err(|e| { + lance_core::Error::from(NamespaceError::Internal { + message: format!( + "Failed to read staging manifest at '{}': {}", + staging_path, e + ), + }) + })?, + Err(ObjectStoreError::NotFound { .. }) => return Ok(false), + Err(e) => { + return Err(lance_core::Error::from(NamespaceError::Internal { + message: format!( + "Failed to read staging manifest at '{}': {}", + staging_path, e + ), + })); + } + }; + + let final_bytes = self + .object_store + .inner + .get(final_path) + .await + .map_err(|e| { + lance_core::Error::from(NamespaceError::Internal { + message: format!( + "Failed to read existing version manifest at '{}': {}", + final_path, e + ), + }) + })? + .bytes() + .await + .map_err(|e| { + lance_core::Error::from(NamespaceError::Internal { + message: format!( + "Failed to read existing version manifest bytes at '{}': {}", + final_path, e + ), + }) + })?; + + Ok(staging_bytes.as_ref() == final_bytes.as_ref()) + } + + /// Idempotent success or conflict when the target version path already exists. + async fn resolve_existing_table_version( + &self, + args: ExistingTableVersionResolve<'_>, + ) -> Result { + if self + .staging_matches_final_manifest( + args.staging_path, + args.final_path, + args.final_meta, + args.request_manifest_size, + ) + .await? + { + // Best-effort cleanup of a retry's staging blob. + if let Err(e) = self.object_store.inner.delete(args.staging_path).await { + log::warn!( + "Failed to delete staging manifest at '{}': {:?}", + args.staging_path, + e + ); + } + return Ok(Self::create_table_version_response( + args.version, + args.final_path, + args.final_meta, + )); + } + + Err(lance_core::Error::from( + NamespaceError::ConcurrentModification { + message: format!( + "Version {} already exists for table at '{}' with different content", + args.version, args.table_uri + ), + }, + )) + } + + /// Enforce version CAS: requested version must be `latest + 1` (or bootstrap). + /// + /// Empty-chain bootstrap: + /// - main must start at v1 + /// - a registered branch must start at `BranchContents.parent_version` (the + /// shallow-clone fork version, which may be > 1) + /// - an unregistered branch (create_branch phase-1, ref not written yet) + /// accepts the requested version because `parent_version` is not known yet + async fn enforce_create_table_version_cas( + &self, + table_path: &Path, + version: u64, + table_uri: &str, + is_branch: bool, + branch_parent_version: Option, + ) -> Result<()> { + let latest = self.list_versions_under(table_path, true, Some(1)).await?; + let expected = match latest.first() { + Some(v) => (v.version as u64).checked_add(1).ok_or_else(|| { + lance_core::Error::from(NamespaceError::ConcurrentModification { + message: format!( + "Version overflow computing next version for table at '{}': \ + latest version {} cannot advance", + table_uri, v.version + ), + }) + })?, + None => { + if is_branch { + // Prefer BranchContents.parent_version when the ref exists so a + // branch forked at v5 cannot bootstrap at an arbitrary version. + match branch_parent_version { + Some(parent_version) => parent_version, + None => version, + } + } else { + 1 + } + } + }; + if version != expected { + let latest_display = latest + .first() + .map(|v| v.version.to_string()) + .unwrap_or_else(|| "none".to_string()); + return Err(lance_core::Error::from( + NamespaceError::ConcurrentModification { + message: format!( + "Version CAS failed for table at '{}': requested {}, expected {} (latest {})", + table_uri, version, expected, latest_display + ), + }, + )); + } + Ok(()) + } + + /// Materialize staging → final with Create semantics only (never overwrite). + async fn materialize_version_manifest_create( + &self, + staging_path: &Path, + final_path: &Path, + staging_manifest_path: &str, + ) -> std::result::Result<(), ObjectStoreError> { + match self + .object_store + .inner + .copy_if_not_exists(staging_path, final_path) + .await + { + Ok(()) => Ok(()), + Err(ObjectStoreError::NotImplemented { .. }) + | Err(ObjectStoreError::NotSupported { .. }) => { + let manifest_data = self + .object_store + .inner + .get(staging_path) + .await? + .bytes() + .await + .map_err(|e| ObjectStoreError::Generic { + store: "DirectoryNamespace", + source: Box::new(std::io::Error::other(format!( + "Failed to read staging manifest bytes at '{}': {}", + staging_manifest_path, e + ))), + })?; + self.object_store + .inner + .put_opts( + final_path, + manifest_data.into(), + PutOptions { + mode: PutMode::Create, + ..Default::default() + }, + ) + .await + .map(|_| ()) + } + Err(e) => Err(e), + } + } + async fn list_table_versions_from_storage( &self, table_uri: &str, @@ -1613,9 +1968,9 @@ impl DirectoryNamespace { if needs_sort { if descending { - table_versions.sort_by(|a, b| b.version.cmp(&a.version)); + table_versions.sort_by_key(|v| std::cmp::Reverse(v.version)); } else { - table_versions.sort_by(|a, b| a.version.cmp(&b.version)); + table_versions.sort_by_key(|v| v.version); } } @@ -1634,10 +1989,24 @@ impl DirectoryNamespace { request: DescribeTableRequest, ) -> Result { let is_root_level = request.id.as_ref().is_some_and(|id| id.len() == 1); + let is_child_table = request.id.as_ref().is_some_and(|id| id.len() > 1); let skip_manifest_for_root = self.dir_listing_enabled && is_root_level && !self.dir_listing_to_manifest_migration_enabled; - if let Some(ref manifest_ns) = self.manifest_ns + // Self-heal the manifest wherever it can be authoritative: a child table + // (no dir-listing fallback), a manifest-only namespace, or migration mode + // (manifest-first -- it can hold registered_table -> external .lance + // aliases that dir-listing cannot resolve, so a reader built before + // __manifest must re-probe to see them). The bypass -- skipping the probe + // -- applies ONLY to migration-disabled directory-backed root reads, + // which are served entirely from the directory listing. + if is_child_table + || !self.dir_listing_enabled + || self.dir_listing_to_manifest_migration_enabled + { + self.ensure_read_manifest().await?; + } + if let Some(manifest_ns) = self.manifest_ns_for_read() && !skip_manifest_for_root { match manifest_ns.describe_table(request.clone()).await { @@ -1661,19 +2030,31 @@ impl DirectoryNamespace { // rather than degrading to a directory-listing view. return Err(e); } - Err(_) if self.dir_listing_enabled && is_root_level => { - // Fall through to directory check only for single-level IDs + Err(e) if self.dir_listing_enabled && is_root_level => { + // Only a genuinely-absent table (e.g. an unmigrated on-disk + // table) may fall through to the directory check; any other + // manifest error must propagate rather than be read as missing. + if !Self::is_manifest_table_absent_error(&e) { + return Err(Self::classify_storage_error(e)); + } } Err(e) => return Err(e), } } + if is_child_table { + return Err(self.child_namespace_requires_manifest_error()); + } let table_name = Self::table_name_from_id(&request.id)?; let table_id = Self::format_table_id_from_request(&request.id); + if !self.dir_listing_enabled { + return Err(NamespaceError::TableNotFound { message: table_id }.into()); + } + let table_uri = self.table_full_uri(&table_name); // Atomically check table existence and deregistration status - let status = self.check_table_status(&table_name).await; + let status = self.check_table_status(&table_name).await?; if !status.exists { return Err(NamespaceError::TableNotFound { @@ -1774,12 +2155,14 @@ impl DirectoryNamespace { .checkout_version(requested_version as u64) .await .map_err(|e| { - lance_core::Error::from(NamespaceError::TableVersionNotFound { - message: format!( - "Version {} not found for table '{}': {}", - requested_version, table_name, e - ), - }) + let message = format!( + "Version {} not found for table '{}': {}", + requested_version, table_name, e + ); + Self::map_open_error( + e, + NamespaceError::TableVersionNotFound { message }, + ) })?; } @@ -1893,22 +2276,20 @@ impl DirectoryNamespace { let builder = self.configured_builder(table_uri); let dataset = builder.load().await.map_err(|e| { - lance_core::Error::from(NamespaceError::TableNotFound { - message: format!( - "Failed to open table at '{}' for {}: {}", - table_uri, operation, e - ), - }) + let message = format!( + "Failed to open table at '{}' for {}: {}", + table_uri, operation, e + ); + Self::map_open_error(e, NamespaceError::TableNotFound { message }) })?; if let Some(version) = version { return dataset.checkout_version(version as u64).await.map_err(|e| { - lance_core::Error::from(NamespaceError::TableVersionNotFound { - message: format!( - "Failed to checkout version {} for table at '{}' during {}: {}", - version, table_uri, operation, e - ), - }) + let message = format!( + "Failed to checkout version {} for table at '{}' during {}: {}", + version, table_uri, operation, e + ); + Self::map_open_error(e, NamespaceError::TableVersionNotFound { message }) }); } @@ -2125,6 +2506,7 @@ impl DirectoryNamespace { Operation::CreateIndex { new_indices, removed_indices, + .. } if new_indices.is_empty() && !removed_indices.is_empty() => "DropIndex".to_string(), _ => transaction.operation.to_string(), } @@ -2339,7 +2721,7 @@ impl DirectoryNamespace { } fn table_full_uri(&self, table_name: &str) -> String { - format!("{}/{}.lance", &self.root, table_name) + format!("{}/{}.lance", self.root, table_name) } /// Get the object store path for a table (relative to base_path) @@ -2370,50 +2752,105 @@ impl DirectoryNamespace { /// This performs a single directory listing to get a consistent snapshot of the /// table's state, avoiding race conditions between checking existence and /// checking deregistration status. - pub(crate) async fn check_table_status(&self, table_name: &str) -> TableStatus { + pub(crate) async fn check_table_status(&self, table_name: &str) -> Result { let table_path = self.table_path(table_name); match self.object_store.read_dir(table_path).await { Ok(entries) => { let exists = !entries.is_empty(); let is_deregistered = entries.iter().any(|e| e.ends_with(".lance-deregistered")); let has_reserved_file = entries.iter().any(|e| e.ends_with(".lance-reserved")); - TableStatus { + Ok(TableStatus { exists, is_deregistered, has_reserved_file, - } + }) } - Err(_) => TableStatus { + // Local filesystems error on a missing directory where object stores + // return an empty listing; both mean the table does not exist. + Err(e) if manifest::ManifestNamespace::is_not_found_load_error(&e) => Ok(TableStatus { exists: false, is_deregistered: false, has_reserved_file: false, - }, + }), + // Any other failure must propagate: collapsing it to "does not exist" + // lets a transient error overwrite a live table via create/exist-ok + // callers and destroys the retry evidence classifiers depend on. + Err(e) => Err(Self::classify_storage_error(e)), } } - async fn put_marker_file_atomic( - &self, - path: &Path, - file_description: &str, - ) -> std::result::Result<(), String> { - let put_opts = PutOptions { - mode: PutMode::Create, - ..Default::default() - }; - - match self - .object_store - .inner - .put_opts(path, bytes::Bytes::new().into(), put_opts) - .await + /// Classify a storage error into a typed [`NamespaceError`]. The full source + /// text is embedded in the message because the pyo3 layer flattens namespace + /// errors to message-only (no `__cause__`), so that is the only place the + /// 429/503 evidence survives to Python. + fn classify_storage_error(err: Error) -> Error { + if matches!(&err, Error::Namespace { .. }) { + return err; + } + let detail = err.to_string(); + if let Error::IO { source, .. } = &err + && let Some(os_err) = source.downcast_ref::() { - Ok(_) => Ok(()), - Err(ObjectStoreError::AlreadyExists { .. }) - | Err(ObjectStoreError::Precondition { .. }) => { - Err(format!("{} already exists", file_description)) + if is_throttle_error(os_err) { + return NamespaceError::Throttling { + message: format!( + "Storage request was throttled while resolving table: {detail}" + ), + } + .into(); + } + if Self::is_service_unavailable_error(os_err) { + return NamespaceError::ServiceUnavailable { + message: format!("Storage service unavailable while resolving table: {detail}"), + } + .into(); } - Err(e) => Err(format!("Failed to create {}: {:?}", file_description, e)), } + NamespaceError::Internal { + message: format!("Storage error while resolving table: {detail}"), + } + .into() + } + + /// Detect a clearly-transient 5xx not already caught by [`is_throttle_error`]. + /// `object_store` does not expose HTTP status codes, so match the (deliberately + /// narrow) canonical status phrases in the message. + fn is_service_unavailable_error(err: &ObjectStoreError) -> bool { + if let ObjectStoreError::Generic { source, .. } = err { + let message = source.to_string().to_ascii_lowercase(); + message.contains("503 service unavailable") + || message.contains("502 bad gateway") + || message.contains("504 gateway timeout") + } else { + false + } + } + + /// Whether a manifest error means the table is genuinely absent (rather than a + /// storage failure while consulting the manifest). Only such errors may fall + /// through to the directory listing; anything else must propagate. + fn is_manifest_table_absent_error(err: &Error) -> bool { + if manifest::ManifestNamespace::is_not_found_load_error(err) { + return true; + } + if let Error::Namespace { source, .. } = err + && let Some(ns_err) = source.downcast_ref::() + { + return matches!(ns_err, NamespaceError::TableNotFound { .. }); + } + false + } + + /// Map a dataset/version/branch open error: a transient IO error propagates + /// typed via [`classify_storage_error`], while a genuine not-found (missing + /// dataset, version, or ref) keeps the caller's `not_found` variant. + fn map_open_error(err: Error, not_found: NamespaceError) -> Error { + if matches!(&err, Error::IO { .. }) + && !manifest::ManifestNamespace::is_not_found_load_error(&err) + { + return Self::classify_storage_error(err); + } + not_found.into() } /// Get storage options for a table, using credential vending if configured. @@ -2525,7 +2962,7 @@ impl DirectoryNamespace { /// - Manifest registration fails pub async fn migrate(&self) -> Result { // We only care about tables in the root namespace - let Some(ref manifest_ns) = self.manifest_ns else { + let Some(manifest_ns) = self.manifest_ns_for_write().await? else { return Ok(0); // No manifest, nothing to migrate }; @@ -2797,10 +3234,14 @@ impl LanceNamespace for DirectoryNamespace { request: ListNamespacesRequest, ) -> Result { self.record_op("list_namespaces"); - if let Some(ref manifest_ns) = self.manifest_ns { + self.ensure_read_manifest().await?; + if let Some(manifest_ns) = self.manifest_ns_for_read() { return manifest_ns.list_namespaces(request).await; } + if request.id.as_ref().is_some_and(|id| !id.is_empty()) { + return Err(self.child_namespace_requires_manifest_error()); + } Self::validate_root_namespace_id(&request.id)?; Ok(ListNamespacesResponse::new(vec![])) } @@ -2810,10 +3251,14 @@ impl LanceNamespace for DirectoryNamespace { request: DescribeNamespaceRequest, ) -> Result { self.record_op("describe_namespace"); - if let Some(ref manifest_ns) = self.manifest_ns { + self.ensure_read_manifest().await?; + if let Some(manifest_ns) = self.manifest_ns_for_read() { return manifest_ns.describe_namespace(request).await; } + if request.id.as_ref().is_some_and(|id| !id.is_empty()) { + return Err(self.child_namespace_requires_manifest_error()); + } Self::validate_root_namespace_id(&request.id)?; #[allow(clippy::needless_update)] Ok(DescribeNamespaceResponse { @@ -2827,7 +3272,7 @@ impl LanceNamespace for DirectoryNamespace { request: CreateNamespaceRequest, ) -> Result { self.record_op("create_namespace"); - if let Some(ref manifest_ns) = self.manifest_ns { + if let Some(manifest_ns) = self.manifest_ns_for_write().await? { return manifest_ns.create_namespace(request).await; } @@ -2847,7 +3292,7 @@ impl LanceNamespace for DirectoryNamespace { async fn drop_namespace(&self, request: DropNamespaceRequest) -> Result { self.record_op("drop_namespace"); - if let Some(ref manifest_ns) = self.manifest_ns { + if let Some(manifest_ns) = self.manifest_ns_for_write().await? { return manifest_ns.drop_namespace(request).await; } @@ -2867,7 +3312,8 @@ impl LanceNamespace for DirectoryNamespace { async fn namespace_exists(&self, request: NamespaceExistsRequest) -> Result<()> { self.record_op("namespace_exists"); - if let Some(ref manifest_ns) = self.manifest_ns { + self.ensure_read_manifest().await?; + if let Some(manifest_ns) = self.manifest_ns_for_read() { return manifest_ns.namespace_exists(request).await; } @@ -2875,11 +3321,7 @@ impl LanceNamespace for DirectoryNamespace { return Ok(()); } - Err(NamespaceError::NamespaceNotFound { - message: "Child namespaces are only supported when manifest mode is enabled" - .to_string(), - } - .into()) + Err(self.child_namespace_requires_manifest_error()) } async fn list_tables(&self, request: ListTablesRequest) -> Result { @@ -2891,33 +3333,44 @@ impl LanceNamespace for DirectoryNamespace { }) })?; + // Self-heal the manifest wherever it can be authoritative: a child + // namespace, a manifest-only namespace, or migration mode (which merges + // manifest entries -- including registered aliases -- into the root + // listing). The bypass applies only to migration-disabled directory-backed + // root lists. + if !namespace_id.is_empty() + || !self.dir_listing_enabled + || self.dir_listing_to_manifest_migration_enabled + { + self.ensure_read_manifest().await?; + } + // For child namespaces, always delegate to manifest (if enabled) if !namespace_id.is_empty() { - if let Some(ref manifest_ns) = self.manifest_ns { + if let Some(manifest_ns) = self.manifest_ns_for_read() { return manifest_ns.list_tables(request).await; } - return Err(NamespaceError::Unsupported { - message: "Child namespaces are only supported when manifest mode is enabled" - .to_string(), - } - .into()); + return Err(self.child_namespace_requires_manifest_error()); } // When only manifest is enabled (no directory listing), delegate directly to manifest - if let Some(ref manifest_ns) = self.manifest_ns + if let Some(manifest_ns) = self.manifest_ns_for_read() && !self.dir_listing_enabled { return manifest_ns.list_tables(request).await; } + if !self.dir_listing_enabled { + return Ok(ListTablesResponse::new(vec![])); + } // When both manifest and directory listing are enabled with migration mode, // we need to merge and deduplicate - let mut tables = if self.manifest_ns.is_some() + let mut tables = if self.manifest_ns_for_read().is_some() && self.dir_listing_enabled && self.dir_listing_to_manifest_migration_enabled { // Get all manifest table locations (for deduplication) - let manifest_locations = if let Some(ref manifest_ns) = self.manifest_ns { + let manifest_locations = if let Some(manifest_ns) = self.manifest_ns_for_read() { manifest_ns.list_manifest_table_locations().await? } else { std::collections::HashSet::new() @@ -2927,7 +3380,7 @@ impl LanceNamespace for DirectoryNamespace { let mut manifest_request = request.clone(); manifest_request.limit = None; manifest_request.page_token = None; - let manifest_tables = if let Some(ref manifest_ns) = self.manifest_ns { + let manifest_tables = if let Some(manifest_ns) = self.manifest_ns_for_read() { let manifest_response = manifest_ns.list_tables(manifest_request).await?; manifest_response.tables } else { @@ -2975,10 +3428,19 @@ impl LanceNamespace for DirectoryNamespace { async fn table_exists(&self, request: TableExistsRequest) -> Result<()> { self.record_op("table_exists"); let is_root_level = request.id.as_ref().is_some_and(|id| id.len() == 1); + let is_child_table = request.id.as_ref().is_some_and(|id| id.len() > 1); let skip_manifest_for_root = self.dir_listing_enabled && is_root_level && !self.dir_listing_to_manifest_migration_enabled; - if let Some(ref manifest_ns) = self.manifest_ns + // Child table, manifest-only, or migration mode (see describe_table_impl). + // Only a migration-disabled directory-backed root read bypasses the probe. + if is_child_table + || !self.dir_listing_enabled + || self.dir_listing_to_manifest_migration_enabled + { + self.ensure_read_manifest().await?; + } + if let Some(manifest_ns) = self.manifest_ns_for_read() && !skip_manifest_for_root { match manifest_ns.table_exists(request.clone()).await { @@ -2988,18 +3450,29 @@ impl LanceNamespace for DirectoryNamespace { // rather than degrading to a directory-listing view. return Err(e); } - Err(_) if self.dir_listing_enabled && is_root_level => { - // Fall through to directory check only for single-level IDs + Err(e) if self.dir_listing_enabled && is_root_level => { + // Only a genuinely-absent table (e.g. an unmigrated on-disk + // table) may fall through to the directory check; any other + // manifest error must propagate rather than be read as missing. + if !Self::is_manifest_table_absent_error(&e) { + return Err(Self::classify_storage_error(e)); + } } Err(e) => return Err(e), } } + if is_child_table { + return Err(self.child_namespace_requires_manifest_error()); + } let table_name = Self::table_name_from_id(&request.id)?; let table_id = Self::format_table_id_from_request(&request.id); + if !self.dir_listing_enabled { + return Err(NamespaceError::TableNotFound { message: table_id }.into()); + } // Atomically check table existence and deregistration status - let status = self.check_table_status(&table_name).await; + let status = self.check_table_status(&table_name).await?; if !status.exists { return Err(NamespaceError::TableNotFound { @@ -3020,7 +3493,7 @@ impl LanceNamespace for DirectoryNamespace { async fn drop_table(&self, request: DropTableRequest) -> Result { self.record_op("drop_table"); - if let Some(ref manifest_ns) = self.manifest_ns { + if let Some(manifest_ns) = self.manifest_ns_for_write().await? { return manifest_ns.drop_table(request).await; } @@ -3050,7 +3523,7 @@ impl LanceNamespace for DirectoryNamespace { request_data: Bytes, ) -> Result { self.record_op("create_table"); - if let Some(ref manifest_ns) = self.manifest_ns { + if let Some(manifest_ns) = self.manifest_ns_for_write().await? { return manifest_ns.create_table(request, request_data).await; } @@ -3058,7 +3531,7 @@ impl LanceNamespace for DirectoryNamespace { let table_name = Self::table_name_from_id(&request.id)?; let table_uri = self.table_full_uri(&table_name); - let status = self.check_table_status(&table_name).await; + let status = self.check_table_status(&table_name).await?; let (reader, _num_rows) = Self::ipc_reader_from_request_data(&request_data, "create_table")?; @@ -3097,7 +3570,7 @@ impl LanceNamespace for DirectoryNamespace { async fn declare_table(&self, request: DeclareTableRequest) -> Result { self.record_op("declare_table"); - if let Some(ref manifest_ns) = self.manifest_ns { + if let Some(manifest_ns) = self.manifest_ns_for_write().await? { let mut response = manifest_ns.declare_table(request.clone()).await?; if let Some(ref location) = response.location { // For backwards compatibility, only skip vending credentials when explicitly set to false @@ -3136,7 +3609,7 @@ impl LanceNamespace for DirectoryNamespace { // Check if table already has data (created via create_table). // The atomic put only prevents races between concurrent declare_table calls, // not between declare_table and existing data. - let status = self.check_table_status(&table_name).await; + let status = self.check_table_status(&table_name).await?; if status.exists && !status.has_reserved_file { // Table has data but no reserved file - it was created with data return Err(NamespaceError::TableAlreadyExists { @@ -3150,22 +3623,27 @@ impl LanceNamespace for DirectoryNamespace { // concurrent declare_table calls. let reserved_file_path = self.table_reserved_file_path(&table_name); - self.put_marker_file_atomic(&reserved_file_path, &format!("table {}", table_name)) - .await - .map_err(|e| { - if e.contains("already exists") { - lance_core::Error::from(NamespaceError::TableAlreadyExists { - message: table_name.to_string(), - }) - } else { - lance_core::Error::from(NamespaceError::Internal { message: e }) - } - })?; - - // For backwards compatibility, only skip vending credentials when explicitly set to false - let vend_credentials = request.vend_credentials.unwrap_or(true); - let identity = request.identity.as_deref(); - let storage_options = self + put_marker_file_atomic( + &self.object_store, + &reserved_file_path, + &format!("table {}", table_name), + ) + .await + .map_err(|e| match e { + MarkerFileError::AlreadyExists { .. } => { + lance_core::Error::from(NamespaceError::TableAlreadyExists { + message: table_name.to_string(), + }) + } + MarkerFileError::Other { message } => { + lance_core::Error::from(NamespaceError::Internal { message }) + } + })?; + + // For backwards compatibility, only skip vending credentials when explicitly set to false + let vend_credentials = request.vend_credentials.unwrap_or(true); + let identity = request.identity.as_deref(); + let storage_options = self .get_storage_options_for_table(&table_uri, vend_credentials, identity) .await?; @@ -3188,7 +3666,7 @@ impl LanceNamespace for DirectoryNamespace { ) -> Result { self.record_op("register_table"); // If manifest is enabled, delegate to manifest namespace - if let Some(ref manifest_ns) = self.manifest_ns { + if let Some(manifest_ns) = self.manifest_ns_for_write().await? { return LanceNamespace::register_table(manifest_ns.as_ref(), request).await; } @@ -3205,7 +3683,7 @@ impl LanceNamespace for DirectoryNamespace { ) -> Result { self.record_op("deregister_table"); // If manifest is enabled, delegate to manifest namespace - if let Some(ref manifest_ns) = self.manifest_ns { + if let Some(manifest_ns) = self.manifest_ns_for_write().await? { return LanceNamespace::deregister_table(manifest_ns.as_ref(), request).await; } @@ -3215,7 +3693,7 @@ impl LanceNamespace for DirectoryNamespace { // Check table existence and deregistration status. // This provides better error messages for common cases. - let status = self.check_table_status(&table_name).await; + let status = self.check_table_status(&table_name).await?; if !status.exists { return Err(NamespaceError::TableNotFound { @@ -3237,18 +3715,20 @@ impl LanceNamespace for DirectoryNamespace { // If a race occurs and another process already created the file, // we'll get an AlreadyExists error which we convert to a proper message. let deregistered_path = self.table_deregistered_file_path(&table_name); - self.put_marker_file_atomic( + put_marker_file_atomic( + &self.object_store, &deregistered_path, &format!("deregistration marker for table {}", table_name), ) .await - .map_err(|e| { - if e.contains("already exists") { + .map_err(|e| match e { + MarkerFileError::AlreadyExists { .. } => { lance_core::Error::from(NamespaceError::InvalidTableState { message: format!("Table is already deregistered: {}", table_name), }) - } else { - lance_core::Error::from(NamespaceError::Internal { message: e }) + } + MarkerFileError::Other { message } => { + lance_core::Error::from(NamespaceError::Internal { message }) } })?; @@ -3263,7 +3743,7 @@ impl LanceNamespace for DirectoryNamespace { &self, request: AlterTableAddColumnsRequest, ) -> Result { - if let Some(ref manifest_ns) = self.manifest_ns { + if let Some(manifest_ns) = self.manifest_ns_for_write().await? { return manifest_ns.alter_table_add_columns(request).await; } @@ -3272,7 +3752,7 @@ impl LanceNamespace for DirectoryNamespace { let table_uri = self.table_full_uri(&table_name); // Check table existence and deregistration status before opening the dataset - let status = self.check_table_status(&table_name).await; + let status = self.check_table_status(&table_name).await?; if !status.exists { return Err(NamespaceError::TableNotFound { message: table_name, @@ -3321,7 +3801,7 @@ impl LanceNamespace for DirectoryNamespace { &self, request: AlterTableAlterColumnsRequest, ) -> Result { - if let Some(ref manifest_ns) = self.manifest_ns { + if let Some(manifest_ns) = self.manifest_ns_for_write().await? { return manifest_ns.alter_table_alter_columns(request).await; } @@ -3329,7 +3809,7 @@ impl LanceNamespace for DirectoryNamespace { let table_uri = self.table_full_uri(&table_name); // Check table existence and deregistration status before opening the dataset - let status = self.check_table_status(&table_name).await; + let status = self.check_table_status(&table_name).await?; if !status.exists { return Err(NamespaceError::TableNotFound { message: table_name, @@ -3371,7 +3851,7 @@ impl LanceNamespace for DirectoryNamespace { &self, request: AlterTableDropColumnsRequest, ) -> Result { - if let Some(ref manifest_ns) = self.manifest_ns { + if let Some(manifest_ns) = self.manifest_ns_for_write().await? { return manifest_ns.alter_table_drop_columns(request).await; } @@ -3379,7 +3859,7 @@ impl LanceNamespace for DirectoryNamespace { let table_uri = self.table_full_uri(&table_name); // Check table existence and deregistration status before opening the dataset - let status = self.check_table_status(&table_name).await; + let status = self.check_table_status(&table_name).await?; if !status.exists { return Err(NamespaceError::TableNotFound { message: table_name, @@ -3445,11 +3925,11 @@ impl LanceNamespace for DirectoryNamespace { self.record_op("create_table_version"); let branch = Self::normalized_branch(request.branch.as_deref())?; let table_uri = self.resolve_table_location(&request.id).await?; - let (table_uri, table_path) = match branch { + let (table_uri, table_path, branch_parent_version) = match branch { Some(b) => self.resolve_branch_for_commit(&table_uri, b).await?, None => { let table_path = self.object_store_path_from_uri(&table_uri)?; - (table_uri, table_path) + (table_uri, table_path, None) } }; @@ -3474,66 +3954,79 @@ impl LanceNamespace for DirectoryNamespace { }) })?; - let copy_result = match self - .object_store - .inner - .copy_if_not_exists(&staging_path, &final_path) - .await - { - Ok(()) => Ok(()), - Err(ObjectStoreError::NotImplemented { .. }) - | Err(ObjectStoreError::NotSupported { .. }) => { - let manifest_data = self - .object_store - .inner - .get(&staging_path) - .await - .map_err(|e| { - lance_core::Error::from(NamespaceError::Internal { - message: format!( - "Failed to read staging manifest at '{}': {}", - staging_manifest_path, e - ), - }) - })? - .bytes() - .await - .map_err(|e| { - lance_core::Error::from(NamespaceError::Internal { - message: format!( - "Failed to read staging manifest bytes at '{}': {}", - staging_manifest_path, e - ), - }) - })?; - self.object_store - .inner - .put_opts( - &final_path, - manifest_data.into(), - PutOptions { - mode: PutMode::Create, - ..Default::default() - }, - ) - .await - .map(|_| ()) + // Idempotent retry: version path already published with the same content. + match self.object_store.inner.head(&final_path).await { + Ok(existing_meta) => { + return self + .resolve_existing_table_version(ExistingTableVersionResolve { + staging_path: &staging_path, + final_path: &final_path, + version, + table_uri: &table_uri, + final_meta: &existing_meta, + request_manifest_size: request.manifest_size, + }) + .await; } - Err(e) => Err(e), - }; + Err(ObjectStoreError::NotFound { .. }) => {} + Err(e) => { + return Err(lance_core::Error::from(NamespaceError::Internal { + message: format!( + "Failed to stat version {} for table at '{}': {}", + version, table_uri, e + ), + })); + } + } + + // Strict CAS: only allow appending latest+1 (or the empty-chain bootstrap + // version: v1 on main, BranchContents.parent_version on a registered branch). + let is_branch = branch.is_some(); + self.enforce_create_table_version_cas( + &table_path, + version, + &table_uri, + is_branch, + branch_parent_version, + ) + .await?; + + // Materialize with Create / copy_if_not_exists only — never overwrite. + let copy_result = self + .materialize_version_manifest_create(&staging_path, &final_path, staging_manifest_path) + .await; match copy_result { Ok(()) => {} Err(ObjectStoreError::AlreadyExists { .. }) | Err(ObjectStoreError::Precondition { .. }) => { - return Err(lance_core::Error::from( - NamespaceError::ConcurrentModification { + // Lost a Create race: succeed only if the winner published identical bytes. + let existing_meta = self.object_store.inner.head(&final_path).await.map_err(|e| { + lance_core::Error::from(NamespaceError::Internal { message: format!( - "Version {} already exists for table at '{}'", - version, table_uri + "Version {} conflict for table at '{}' but failed to stat winner: {}", + version, table_uri, e ), - }, - )); + }) + })?; + return self + .resolve_existing_table_version(ExistingTableVersionResolve { + staging_path: &staging_path, + final_path: &final_path, + version, + table_uri: &table_uri, + final_meta: &existing_meta, + request_manifest_size: request.manifest_size, + }) + .await; + } + Err(ObjectStoreError::NotFound { .. }) => { + return Err(lance_core::Error::from(NamespaceError::InvalidInput { + message: format!( + "Staging manifest not found at '{}' for version {} of table at '{}'", + staging_manifest_path, version, table_uri + ), + })); } Err(e) => { return Err(lance_core::Error::from(NamespaceError::Internal { @@ -3558,7 +4051,6 @@ impl LanceNamespace for DirectoryNamespace { ), }) })?; - let manifest_size = final_meta.size as i64; // Delete the staging manifest after successful copy if let Err(e) = self.object_store.inner.delete(&staging_path).await { @@ -3569,17 +4061,11 @@ impl LanceNamespace for DirectoryNamespace { ); } - Ok(CreateTableVersionResponse { - transaction_id: None, - version: Some(Box::new(TableVersion { - version: version as i64, - manifest_path: final_path.to_string(), - manifest_size: Some(manifest_size), - e_tag: final_meta.e_tag, - timestamp_millis: None, - metadata: None, - })), - }) + Ok(Self::create_table_version_response( + version, + &final_path, + &final_meta, + )) } async fn describe_table_version( @@ -3784,7 +4270,7 @@ impl LanceNamespace for DirectoryNamespace { .map(|field_id| { dataset .schema() - .field_path(i32::try_from(*field_id).map_err(|e| { + .field_path_minimal(i32::try_from(*field_id).map_err(|e| { lance_core::Error::from(NamespaceError::Internal { message: format!( "Field id {} does not fit in i32 for table '{}': {}", @@ -4896,14 +5382,21 @@ impl LanceNamespace for DirectoryNamespace { })?; } + scanner + .full_text_search(fts) + .map_err(|e| NamespaceError::InvalidInput { + message: format!("Invalid full text search: {:?}", e), + })?; + } else if let Some(ref structured_query) = fts_query.structured_query { + // Structured FTS: map the namespace query model into the engine FtsQuery. + let engine_query = build_engine_fts_query(&structured_query.query)?; + let fts = FullTextSearchQuery::new_query(engine_query); scanner .full_text_search(fts) .map_err(|e| NamespaceError::InvalidInput { message: format!("Invalid full text search: {:?}", e), })?; } - // Note: structured_query would require more complex parsing - // For now, we only support string_query } // Apply column projection if specified @@ -5319,10 +5812,372 @@ impl LanceNamespace for DirectoryNamespace { } } +/// Error from [`put_marker_file_atomic`]. +#[derive(Debug)] +pub(crate) enum MarkerFileError { + /// The final marker path is already present (Create / rename race). + AlreadyExists { description: String }, + /// Staging or publish failed for a non-conflict reason. + Other { message: String }, +} + +impl std::fmt::Display for MarkerFileError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::AlreadyExists { description } => { + write!(f, "{} already exists", description) + } + Self::Other { message } => write!(f, "{}", message), + } + } +} + +/// Atomically create a marker file (e.g. `.lance-reserved`) with Create semantics. +/// +/// Some object stores implement `PutMode::Create` via temp+rename that reuses the +/// final basename. Dotfile targets such as `.lance-reserved` therefore produce +/// temp names containing `..`, which these stores reject. Stage under a non-dot +/// sibling, then claim the final path with `rename_if_not_exists`. +/// +/// When `rename_if_not_exists` is unavailable, fall back to +/// `copy_if_not_exists(staging → target)`, then `PutMode::Create` on the target. +/// That Create path is only for stores whose Create is a true conditional PUT +/// (not basename-derived temp+rename); such stores are exactly the ones that +/// typically omit rename/copy conditionals. +/// +/// Some object stores also fail to flush empty objects, so the conditional rename +/// can fail with NotFound. Use a tiny non-empty payload. +/// +/// Staging cleanup is best-effort with a few short retries. A delete that still +/// fails after retries leaves a tiny `lance-marker.staging.*` orphan. Async Drop +/// cannot await object-store I/O, so RAII is not used here. Each call uses a +/// unique staging UUID, so concurrent callers never contend on the same cleanup. +pub(crate) async fn put_marker_file_atomic( + object_store: &ObjectStore, + path: &Path, + file_description: &str, +) -> std::result::Result<(), MarkerFileError> { + let staging_name = format!("lance-marker.staging.{}", uuid::Uuid::new_v4().simple()); + let path_str = path.as_ref(); + let staging_path = match path_str.rfind('/') { + Some(idx) => Path::from(format!("{}/{}", &path_str[..idx], staging_name)), + None => Path::from(staging_name.as_str()), + }; + + object_store + .inner + .put(&staging_path, bytes::Bytes::from_static(b"reserved").into()) + .await + .map_err(|e| MarkerFileError::Other { + message: format!("Failed to stage {}: {:?}", file_description, e), + })?; + + // Successful rename consumes the staging object; every other path must + // delete it (best-effort) so conflict/fallback races do not accumulate. + let mut staging_consumed = false; + let publish_result = match object_store + .inner + .rename_if_not_exists(&staging_path, path) + .await + { + Ok(()) => { + staging_consumed = true; + Ok(()) + } + Err(ObjectStoreError::NotImplemented { .. }) + | Err(ObjectStoreError::NotSupported { .. }) => { + match object_store + .inner + .copy_if_not_exists(&staging_path, path) + .await + { + Ok(()) => Ok(()), + Err(ObjectStoreError::NotImplemented { .. }) + | Err(ObjectStoreError::NotSupported { .. }) => object_store + .inner + .put_opts( + path, + bytes::Bytes::from_static(b"reserved").into(), + PutOptions { + mode: PutMode::Create, + ..Default::default() + }, + ) + .await + .map(|_| ()), + Err(e) => Err(e), + } + } + Err(e) => Err(e), + }; + + if !staging_consumed { + delete_staging_marker_best_effort(object_store, &staging_path).await; + } + + match publish_result { + Ok(()) => Ok(()), + Err(ObjectStoreError::AlreadyExists { .. }) + | Err(ObjectStoreError::Precondition { .. }) => Err(MarkerFileError::AlreadyExists { + description: file_description.to_string(), + }), + Err(e) => Err(MarkerFileError::Other { + message: format!("Failed to create {}: {:?}", file_description, e), + }), + } +} + +/// Best-effort delete of a per-call staging marker, with short retries for +/// transient store errors. `NotFound` is treated as success (delete may have +/// succeeded despite an earlier ambiguous failure). +async fn delete_staging_marker_best_effort(object_store: &ObjectStore, staging_path: &Path) { + const MAX_ATTEMPTS: u32 = 3; + const BACKOFF_MS: [u64; 2] = [20, 50]; + + let mut last_err: Option = None; + for attempt in 0..MAX_ATTEMPTS { + match object_store.inner.delete(staging_path).await { + Ok(()) => return, + Err(ObjectStoreError::NotFound { .. }) => return, + Err(e) => { + last_err = Some(e); + if let Some(&delay_ms) = BACKOFF_MS.get(attempt as usize) { + tokio::time::sleep(std::time::Duration::from_millis(delay_ms)).await; + } + } + } + } + if let Some(del_err) = last_err { + log::warn!( + "Failed to delete staging marker at '{}' after {} attempts: {:?}", + staging_path, + MAX_ATTEMPTS, + del_err + ); + } +} + +/// Maps a namespace structured `FtsQuery` model into the engine `FtsQuery`. Mirrors the mapping the +/// JNI scanner performs, so the local `queryTable` path honors `structured_query` the same way a +/// `fragment.newScan(fullTextQuery)` does. +fn build_engine_fts_query( + query: &lance_namespace::models::FtsQuery, +) -> std::result::Result { + if let Some(ref m) = query.r#match { + Ok(FtsQuery::Match(build_engine_match_query(m)?)) + } else if let Some(ref p) = query.phrase { + let mut phrase = PhraseQuery::new(p.terms.clone()); + if let Some(ref column) = p.column { + phrase = phrase.with_column(Some(column.clone())); + } + if let Some(slop) = p.slop { + phrase = phrase.with_slop(slop as u32); + } + Ok(FtsQuery::Phrase(phrase)) + } else if let Some(ref mm) = query.multi_match { + let match_queries = mm + .match_queries + .iter() + .map(build_engine_match_query) + .collect::, _>>()?; + Ok(FtsQuery::MultiMatch(MultiMatchQuery { match_queries })) + } else if let Some(ref b) = query.boolean { + let mut clauses: Vec<(Occur, FtsQuery)> = Vec::new(); + for clause in &b.must { + clauses.push((Occur::Must, build_engine_fts_query(clause)?)); + } + for clause in &b.should { + clauses.push((Occur::Should, build_engine_fts_query(clause)?)); + } + for clause in &b.must_not { + clauses.push((Occur::MustNot, build_engine_fts_query(clause)?)); + } + Ok(FtsQuery::Boolean(BooleanQuery::new(clauses))) + } else if let Some(ref boost) = query.boost { + let positive = build_engine_fts_query(&boost.positive)?; + let negative = build_engine_fts_query(&boost.negative)?; + Ok(FtsQuery::Boost(BoostQuery::new( + positive, + negative, + boost.negative_boost, + ))) + } else { + Err(NamespaceError::InvalidInput { + message: "structured_query.query must set exactly one of match, phrase, multi_match, \ + boolean, or boost" + .to_string(), + }) + } +} + +fn build_engine_match_query( + m: &lance_namespace::models::MatchQuery, +) -> std::result::Result { + let mut match_query = MatchQuery::new(m.terms.clone()); + if let Some(ref column) = m.column { + match_query = match_query.with_column(Some(column.clone())); + } + if let Some(boost) = m.boost { + match_query = match_query.with_boost(boost); + } + if let Some(fuzziness) = m.fuzziness { + match_query = match_query.with_fuzziness(Some(fuzziness as u32)); + } + if let Some(max_expansions) = m.max_expansions { + match_query = match_query.with_max_expansions(max_expansions as usize); + } + if let Some(ref operator) = m.operator { + let op = + Operator::try_from(operator.as_str()).map_err(|e| NamespaceError::InvalidInput { + message: format!("Invalid FTS operator: {:?}", e), + })?; + match_query = match_query.with_operator(op); + } + if let Some(prefix_length) = m.prefix_length { + match_query = match_query.with_prefix_length(prefix_length as u32); + } + Ok(match_query) +} + #[cfg(test)] mod tests { use super::*; use arrow_ipc::reader::{FileReader, StreamReader}; + + #[test] + fn test_build_engine_fts_query_match() { + let mut ns_match = lance_namespace::models::MatchQuery::new("hello world".to_string()); + ns_match.column = Some("body".to_string()); + ns_match.operator = Some("AND".to_string()); + ns_match.fuzziness = Some(1); + ns_match.max_expansions = Some(30); + ns_match.boost = Some(2.0); + ns_match.prefix_length = Some(2); + + let mut ns_query = lance_namespace::models::FtsQuery::new(); + ns_query.r#match = Some(Box::new(ns_match)); + + match build_engine_fts_query(&ns_query).unwrap() { + FtsQuery::Match(m) => { + assert_eq!(m.terms, "hello world"); + assert_eq!(m.column, Some("body".to_string())); + assert_eq!(m.operator, Operator::And); + assert_eq!(m.fuzziness, Some(1)); + assert_eq!(m.max_expansions, 30); + assert_eq!(m.boost, 2.0); + assert_eq!(m.prefix_length, 2); + } + other => panic!("expected Match, got {:?}", other), + } + } + + /// Wraps a namespace `MatchQuery` (with a column) as an `FtsQuery` for use as a clause in + /// compound queries (boolean / boost). + fn ns_match_query(terms: &str, column: &str) -> lance_namespace::models::FtsQuery { + let mut m = lance_namespace::models::MatchQuery::new(terms.to_string()); + m.column = Some(column.to_string()); + let mut q = lance_namespace::models::FtsQuery::new(); + q.r#match = Some(Box::new(m)); + q + } + + #[test] + fn test_build_engine_fts_query_phrase() { + let mut ns_phrase = lance_namespace::models::PhraseQuery::new("hello world".to_string()); + ns_phrase.column = Some("body".to_string()); + ns_phrase.slop = Some(2); + + let mut ns_query = lance_namespace::models::FtsQuery::new(); + ns_query.phrase = Some(Box::new(ns_phrase)); + + match build_engine_fts_query(&ns_query).unwrap() { + FtsQuery::Phrase(p) => { + assert_eq!(p.terms, "hello world"); + assert_eq!(p.column, Some("body".to_string())); + assert_eq!(p.slop, 2); + } + other => panic!("expected Phrase, got {:?}", other), + } + } + + #[test] + fn test_build_engine_fts_query_multi_match() { + let mut m1 = lance_namespace::models::MatchQuery::new("hello".to_string()); + m1.column = Some("title".to_string()); + let mut m2 = lance_namespace::models::MatchQuery::new("hello".to_string()); + m2.column = Some("body".to_string()); + m2.boost = Some(2.0); + + let ns_multi = lance_namespace::models::MultiMatchQuery::new(vec![m1, m2]); + let mut ns_query = lance_namespace::models::FtsQuery::new(); + ns_query.multi_match = Some(Box::new(ns_multi)); + + match build_engine_fts_query(&ns_query).unwrap() { + FtsQuery::MultiMatch(mm) => { + assert_eq!(mm.match_queries.len(), 2); + assert_eq!(mm.match_queries[0].terms, "hello"); + assert_eq!(mm.match_queries[0].column, Some("title".to_string())); + assert_eq!(mm.match_queries[1].column, Some("body".to_string())); + assert_eq!(mm.match_queries[1].boost, 2.0); + } + other => panic!("expected MultiMatch, got {:?}", other), + } + } + + #[test] + fn test_build_engine_fts_query_boolean() { + // BooleanQuery::new(must, must_not, should) + let ns_boolean = lance_namespace::models::BooleanQuery::new( + vec![ns_match_query("must-term", "body")], + vec![ns_match_query("must-not-term", "body")], + vec![ns_match_query("should-term", "body")], + ); + let mut ns_query = lance_namespace::models::FtsQuery::new(); + ns_query.boolean = Some(Box::new(ns_boolean)); + + match build_engine_fts_query(&ns_query).unwrap() { + FtsQuery::Boolean(b) => { + assert!(matches!(&b.must[..], [FtsQuery::Match(m)] if m.terms == "must-term")); + assert!( + matches!(&b.must_not[..], [FtsQuery::Match(m)] if m.terms == "must-not-term") + ); + assert!(matches!(&b.should[..], [FtsQuery::Match(m)] if m.terms == "should-term")); + } + other => panic!("expected Boolean, got {:?}", other), + } + } + + #[test] + fn test_build_engine_fts_query_boost() { + let mut ns_boost = lance_namespace::models::BoostQuery::new( + ns_match_query("positive-term", "body"), + ns_match_query("negative-term", "body"), + ); + ns_boost.negative_boost = Some(0.25); + + let mut ns_query = lance_namespace::models::FtsQuery::new(); + ns_query.boost = Some(Box::new(ns_boost)); + + match build_engine_fts_query(&ns_query).unwrap() { + FtsQuery::Boost(b) => { + assert!( + matches!(b.positive.as_ref(), FtsQuery::Match(m) if m.terms == "positive-term") + ); + assert!( + matches!(b.negative.as_ref(), FtsQuery::Match(m) if m.terms == "negative-term") + ); + assert_eq!(b.negative_boost, 0.25); + } + other => panic!("expected Boost, got {:?}", other), + } + } + + #[test] + fn test_build_engine_fts_query_requires_a_variant() { + // An FtsQuery with no variant set is rejected rather than silently ignored. + let empty = lance_namespace::models::FtsQuery::new(); + assert!(build_engine_fts_query(&empty).is_err()); + } use lance::dataset::Dataset; use lance::index::DatasetIndexExt; use lance_core::utils::tempfile::{TempStdDir, TempStrDir}; @@ -5465,77 +6320,585 @@ mod tests { Arc::new(Session::new(0, 0, registry)) } - /// Helper to create test IPC data from a schema - fn create_test_ipc_data(schema: &JsonArrowSchema) -> Vec { - use arrow::ipc::writer::StreamWriter; + // Fault-injection store: returns a runtime-toggleable result from + // `list_with_delimiter` (the call `check_table_status` makes) and delegates + // everything else, so a table can be created before failures are injected. + use futures::stream::BoxStream; + use object_store::{ + CopyOptions, GetOptions, GetResult, ListResult, MultipartUpload, ObjectMeta, + PutMultipartOptions, PutPayload, PutResult, Result as OSResult, + }; + use std::ops::Range; - let arrow_schema = convert_json_arrow_schema(schema).unwrap(); - let arrow_schema = Arc::new(arrow_schema); - let batch = arrow::record_batch::RecordBatch::new_empty(arrow_schema.clone()); - let mut buffer = Vec::new(); - { - let mut writer = StreamWriter::try_new(&mut buffer, &arrow_schema).unwrap(); - writer.write(&batch).unwrap(); - writer.finish().unwrap(); - } - buffer + #[derive(Debug, Clone, Copy)] + enum ListBehavior { + Throttle, + ServiceUnavailable, + Internal, + NotFound, + EmptyListing, } - fn create_ipc_data_from_batches( - schema: Arc, - batches: Vec, - ) -> Vec { - use arrow::ipc::writer::StreamWriter; + #[derive(Debug)] + struct FailingListStore { + target: Arc, + behavior: Arc>>, + } - let mut buffer = Vec::new(); - { - let mut writer = StreamWriter::try_new(&mut buffer, &schema).unwrap(); - for batch in &batches { - writer.write(batch).unwrap(); - } - writer.finish().unwrap(); + impl std::fmt::Display for FailingListStore { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "FailingListStore({})", self.target) } - buffer } - fn create_non_empty_test_ipc_data() -> Vec { - use arrow::array::{Int32Array, StringArray}; - use arrow::record_batch::RecordBatch; + #[async_trait] + impl OSObjectStore for FailingListStore { + async fn put_opts( + &self, + location: &Path, + bytes: PutPayload, + opts: PutOptions, + ) -> OSResult { + self.target.put_opts(location, bytes, opts).await + } - let schema = Arc::new(convert_json_arrow_schema(&create_test_schema()).unwrap()); - let batch = RecordBatch::try_new( - schema.clone(), - vec![ - Arc::new(Int32Array::from(vec![1, 2])), - Arc::new(StringArray::from(vec![Some("alice"), Some("bob")])), - ], - ) - .unwrap(); - create_ipc_data_from_batches(schema, vec![batch]) - } + async fn put_multipart_opts( + &self, + location: &Path, + opts: PutMultipartOptions, + ) -> OSResult> { + self.target.put_multipart_opts(location, opts).await + } - fn create_single_row_test_ipc_data() -> Vec { - use arrow::array::{Int32Array, StringArray}; - use arrow::record_batch::RecordBatch; + async fn get_opts(&self, location: &Path, options: GetOptions) -> OSResult { + self.target.get_opts(location, options).await + } - let schema = Arc::new(convert_json_arrow_schema(&create_test_schema()).unwrap()); - let batch = RecordBatch::try_new( - schema.clone(), - vec![ - Arc::new(Int32Array::from(vec![10])), - Arc::new(StringArray::from(vec![Some("carol")])), - ], - ) - .unwrap(); - create_ipc_data_from_batches(schema, vec![batch]) - } + async fn get_ranges(&self, location: &Path, ranges: &[Range]) -> OSResult> { + self.target.get_ranges(location, ranges).await + } - /// Helper to create a simple test schema - fn create_test_schema() -> JsonArrowSchema { - let int_type = JsonArrowDataType::new("int32".to_string()); - let string_type = JsonArrowDataType::new("utf8".to_string()); + fn delete_stream( + &self, + locations: BoxStream<'static, OSResult>, + ) -> BoxStream<'static, OSResult> { + self.target.delete_stream(locations) + } - let id_field = JsonArrowField { + fn list(&self, prefix: Option<&Path>) -> BoxStream<'static, OSResult> { + self.target.list(prefix) + } + + async fn list_with_delimiter(&self, prefix: Option<&Path>) -> OSResult { + let behavior = *self.behavior.lock().unwrap(); + match behavior { + None => self.target.list_with_delimiter(prefix).await, + Some(ListBehavior::EmptyListing) => Ok(ListResult { + common_prefixes: Vec::new(), + objects: Vec::new(), + }), + // Mirrors the object_store retry-exhaustion message shape for an + // Azure ServerBusy response, which is what the incident produced. + Some(ListBehavior::Throttle) => Err(ObjectStoreError::Generic { + store: "test", + source: "Error performing list request: response error, after 3 retries, \ + max_retries: 3, retry_timeout: 180s - HTTP status server error \ + (503 Service Unavailable): ServerBusy: The server is busy" + .into(), + }), + Some(ListBehavior::ServiceUnavailable) => Err(ObjectStoreError::Generic { + store: "test", + source: "Error performing list request: 503 Service Unavailable".into(), + }), + Some(ListBehavior::Internal) => Err(ObjectStoreError::Generic { + store: "test", + source: "Error performing list request: catastrophic unclassified failure" + .into(), + }), + Some(ListBehavior::NotFound) => Err(ObjectStoreError::NotFound { + path: "test_table.lance".to_string(), + source: "entity not found".into(), + }), + } + } + + async fn copy_opts(&self, from: &Path, to: &Path, opts: CopyOptions) -> OSResult<()> { + self.target.copy_opts(from, to, opts).await + } + } + + #[derive(Debug)] + struct FailingListStoreProvider { + behavior: Arc>>, + } + + #[async_trait] + impl lance_io::object_store::ObjectStoreProvider for FailingListStoreProvider { + async fn new_store( + &self, + base_path: Url, + params: &ObjectStoreParams, + ) -> Result { + let mut store = FileStoreProvider.new_store(base_path, params).await?; + store.inner = Arc::new(FailingListStore { + target: store.inner.clone(), + behavior: self.behavior.clone(), + }); + Ok(store) + } + + fn extract_path(&self, url: &Url) -> Result { + FileStoreProvider.extract_path(url) + } + + fn calculate_object_store_prefix( + &self, + url: &Url, + storage_options: Option<&HashMap>, + ) -> Result { + FileStoreProvider.calculate_object_store_prefix(url, storage_options) + } + } + + fn build_failing_list_session(behavior: Arc>>) -> Arc { + let registry = Arc::new(ObjectStoreRegistry::default()); + registry.insert( + "file-object-store", + Arc::new(FailingListStoreProvider { behavior }), + ); + Arc::new(Session::new(0, 0, registry)) + } + + /// Build a dir-listing namespace whose object store's listing calls follow a + /// shared, runtime-toggleable behavior. Returns the namespace, the temp dir + /// (kept alive for the store), and the behavior toggle. + async fn failing_list_namespace() -> ( + DirectoryNamespace, + TempStdDir, + Arc>>, + ) { + let temp_dir = TempStdDir::default(); + let root_uri = file_object_store_uri(temp_dir.to_str().unwrap()); + let behavior = Arc::new(Mutex::new(None)); + let session = build_failing_list_session(behavior.clone()); + let namespace = DirectoryNamespaceBuilder::new(root_uri) + .session(session) + .manifest_enabled(false) + .dir_listing_enabled(true) + .build() + .await + .unwrap(); + (namespace, temp_dir, behavior) + } + + async fn create_named_dir_table(namespace: &DirectoryNamespace, name: &str) { + let schema = create_test_schema(); + let ipc_data = create_test_ipc_data(&schema); + let mut create_req = CreateTableRequest::new(); + create_req.id = Some(vec![name.to_string()]); + namespace + .create_table(create_req, Bytes::from(ipc_data)) + .await + .unwrap(); + } + + /// Regression test for the throttling-induced TableNotFound bug: a storage + /// error while resolving a table must surface as a typed storage error + /// (Throttling / ServiceUnavailable / Internal) carrying the underlying + /// evidence in its message — never as TableNotFound. + #[tokio::test] + async fn test_table_resolution_propagates_storage_errors_not_table_not_found() { + for (behavior, expected_code, evidence) in [ + (ListBehavior::Throttle, ErrorCode::Throttling, "serverbusy"), + ( + ListBehavior::ServiceUnavailable, + ErrorCode::ServiceUnavailable, + "503 service unavailable", + ), + (ListBehavior::Internal, ErrorCode::Internal, "catastrophic"), + ] { + let (namespace, _temp_dir, toggle) = failing_list_namespace().await; + create_named_dir_table(&namespace, "checkpoint").await; + *toggle.lock().unwrap() = Some(behavior); + + let mut describe_req = DescribeTableRequest::new(); + describe_req.id = Some(vec!["checkpoint".to_string()]); + let err = namespace.describe_table(describe_req).await.unwrap_err(); + let msg = err.to_string(); + assert_eq!( + mutation_error_code(err), + expected_code, + "describe_table under {behavior:?}; msg: {msg}" + ); + assert!( + msg.to_ascii_lowercase().contains(evidence), + "describe_table message must carry storage evidence '{evidence}', got: {msg}" + ); + + let mut exists_req = TableExistsRequest::new(); + exists_req.id = Some(vec!["checkpoint".to_string()]); + let err = namespace.table_exists(exists_req).await.unwrap_err(); + let msg = err.to_string(); + assert_eq!( + mutation_error_code(err), + expected_code, + "table_exists under {behavior:?}; msg: {msg}" + ); + assert!( + msg.to_ascii_lowercase().contains(evidence), + "table_exists message must carry storage evidence '{evidence}', got: {msg}" + ); + } + } + + /// A genuine not-found error and an empty listing must both still resolve to + /// TableNotFound (the local-FS and object-store representations of "missing"). + #[tokio::test] + async fn test_table_resolution_missing_table_yields_table_not_found() { + for behavior in [ListBehavior::NotFound, ListBehavior::EmptyListing] { + let (namespace, _temp_dir, toggle) = failing_list_namespace().await; + *toggle.lock().unwrap() = Some(behavior); + + let mut describe_req = DescribeTableRequest::new(); + describe_req.id = Some(vec!["missing".to_string()]); + let err = namespace.describe_table(describe_req).await.unwrap_err(); + assert_eq!( + mutation_error_code(err), + ErrorCode::TableNotFound, + "describe_table under {behavior:?} should be TableNotFound" + ); + + let mut exists_req = TableExistsRequest::new(); + exists_req.id = Some(vec!["missing".to_string()]); + let err = namespace.table_exists(exists_req).await.unwrap_err(); + assert_eq!( + mutation_error_code(err), + ErrorCode::TableNotFound, + "table_exists under {behavior:?} should be TableNotFound" + ); + } + } + + /// Hybrid (manifest + directory) resolution must exercise the + /// manifest→directory fall-through for a table that exists on disk but is not + /// registered in the manifest: the fall-through must succeed normally, and + /// must not degrade a storage error into TableNotFound. + /// + /// A `__manifest` table must actually exist for the manifest branch to run; + /// otherwise `manifest_ns_for_read()` is None and the manifest branch (and its + /// fall-through arm) is skipped entirely. We therefore create a *separate* + /// table through a manifest-enabled namespace first so `__manifest` exists. + #[tokio::test] + async fn test_hybrid_resolution_falls_through_and_does_not_mask_throttle() { + let temp_dir = TempStdDir::default(); + let root_uri = file_object_store_uri(temp_dir.to_str().unwrap()); + let behavior = Arc::new(Mutex::new(None)); + let session = build_failing_list_session(behavior.clone()); + + // Seed table via a manifest-enabled namespace so `__manifest` exists. + let manifest_ns = DirectoryNamespaceBuilder::new(root_uri.clone()) + .session(session.clone()) + .manifest_enabled(true) + .dir_listing_enabled(true) + .build() + .await + .unwrap(); + create_named_dir_table(&manifest_ns, "seed").await; + + // The table under test: on disk but never registered in the manifest. + let dir_ns = DirectoryNamespaceBuilder::new(root_uri.clone()) + .session(session.clone()) + .manifest_enabled(false) + .dir_listing_enabled(true) + .build() + .await + .unwrap(); + create_named_dir_table(&dir_ns, "checkpoint").await; + + // Migration enabled so root-level reads consult the manifest and fall + // through to the directory check on a manifest miss. + let hybrid_ns = DirectoryNamespaceBuilder::new(root_uri) + .session(session) + .manifest_enabled(true) + .dir_listing_enabled(true) + .dir_listing_to_manifest_migration_enabled(true) + .build() + .await + .unwrap(); + + // (a) Healthy fall-through: the manifest reports "checkpoint" absent and it + // resolves via the directory listing (guards the migration lookup). + let mut describe_req = DescribeTableRequest::new(); + describe_req.id = Some(vec!["checkpoint".to_string()]); + hybrid_ns + .describe_table(describe_req) + .await + .expect("unregistered on-disk table should resolve via the manifest fall-through"); + + // (b) The throttle here surfaces from the directory check after the manifest + // reports absent; the fall-through arm's own storage-error guard is covered + // by the classify_storage_error / is_manifest_table_absent_error unit tests. + *behavior.lock().unwrap() = Some(ListBehavior::Throttle); + let mut describe_req = DescribeTableRequest::new(); + describe_req.id = Some(vec!["checkpoint".to_string()]); + let err = hybrid_ns.describe_table(describe_req).await.unwrap_err(); + let code = mutation_error_code(err); + assert_ne!( + code, + ErrorCode::TableNotFound, + "hybrid resolution masked a throttle as TableNotFound" + ); + assert!( + matches!( + code, + ErrorCode::Throttling | ErrorCode::ServiceUnavailable | ErrorCode::Internal + ), + "hybrid resolution should surface a storage error, got {code:?}" + ); + } + + #[test] + fn test_classify_storage_error_maps_variants_and_preserves_evidence() { + let throttle: Error = ObjectStoreError::Generic { + store: "test", + source: "list request failed, after 3 retries, max_retries: 3 - 503 ServerBusy".into(), + } + .into(); + assert!(matches!(throttle, Error::IO { .. })); + let classified = DirectoryNamespace::classify_storage_error(throttle); + let msg = classified.to_string(); + assert_eq!(mutation_error_code(classified), ErrorCode::Throttling); + assert!( + msg.to_ascii_lowercase().contains("serverbusy"), + "throttle evidence lost: {msg}" + ); + + let service: Error = ObjectStoreError::Generic { + store: "test", + source: "504 Gateway Timeout".into(), + } + .into(); + assert_eq!( + mutation_error_code(DirectoryNamespace::classify_storage_error(service)), + ErrorCode::ServiceUnavailable + ); + + let internal: Error = ObjectStoreError::Generic { + store: "test", + source: "disk caught fire".into(), + } + .into(); + assert_eq!( + mutation_error_code(DirectoryNamespace::classify_storage_error(internal)), + ErrorCode::Internal + ); + + // A pre-existing namespace error keeps its own code rather than being reclassified. + let preexisting: Error = NamespaceError::TableAlreadyExists { + message: "t".to_string(), + } + .into(); + assert_eq!( + mutation_error_code(DirectoryNamespace::classify_storage_error(preexisting)), + ErrorCode::TableAlreadyExists + ); + } + + #[test] + fn test_is_manifest_table_absent_error() { + let table_not_found: Error = NamespaceError::TableNotFound { + message: "t".to_string(), + } + .into(); + assert!(DirectoryNamespace::is_manifest_table_absent_error( + &table_not_found + )); + let raw_not_found: Error = ObjectStoreError::NotFound { + path: "t".to_string(), + source: "x".into(), + } + .into(); + assert!(DirectoryNamespace::is_manifest_table_absent_error( + &raw_not_found + )); + + let throttle: Error = ObjectStoreError::Generic { + store: "test", + source: "after 3 retries, max_retries: 3 ServerBusy".into(), + } + .into(); + assert!(!DirectoryNamespace::is_manifest_table_absent_error( + &throttle + )); + let internal: Error = NamespaceError::Internal { + message: "boom".to_string(), + } + .into(); + assert!(!DirectoryNamespace::is_manifest_table_absent_error( + &internal + )); + } + + #[test] + fn test_map_open_error() { + let not_found = || NamespaceError::TableNotFound { + message: "table at 'x' not found: ...".to_string(), + }; + + let throttle: Error = ObjectStoreError::Generic { + store: "test", + source: "after 3 retries, max_retries: 3 - 503 ServerBusy".into(), + } + .into(); + assert_eq!( + mutation_error_code(DirectoryNamespace::map_open_error(throttle, not_found())), + ErrorCode::Throttling + ); + + let generic_io: Error = ObjectStoreError::Generic { + store: "test", + source: "connection reset".into(), + } + .into(); + assert_eq!( + mutation_error_code(DirectoryNamespace::map_open_error(generic_io, not_found())), + ErrorCode::Internal + ); + + let io_not_found: Error = ObjectStoreError::NotFound { + path: "x".to_string(), + source: "missing".into(), + } + .into(); + assert_eq!( + mutation_error_code(DirectoryNamespace::map_open_error( + io_not_found, + not_found() + )), + ErrorCode::TableNotFound + ); + + let dataset_not_found = Error::dataset_not_found("x".to_string(), "missing".into()); + assert_eq!( + mutation_error_code(DirectoryNamespace::map_open_error( + dataset_not_found, + not_found() + )), + ErrorCode::TableNotFound + ); + + // RefNotFound is not an IO error, so it is not reclassified as a storage error. + let ref_not_found = Error::RefNotFound { + message: "branch 'b' does not exist".to_string(), + }; + assert_eq!( + mutation_error_code(DirectoryNamespace::map_open_error( + ref_not_found, + not_found() + )), + ErrorCode::TableNotFound + ); + + // The caller's not-found variant is honored, but a throttle still propagates. + let version_miss = Error::RefNotFound { + message: "version 5 does not exist".to_string(), + }; + assert_eq!( + mutation_error_code(DirectoryNamespace::map_open_error( + version_miss, + NamespaceError::TableVersionNotFound { + message: "version 5 not found".to_string(), + }, + )), + ErrorCode::TableVersionNotFound + ); + let version_throttle: Error = ObjectStoreError::Generic { + store: "test", + source: "after 3 retries, max_retries: 3 - 503 ServerBusy".into(), + } + .into(); + assert_eq!( + mutation_error_code(DirectoryNamespace::map_open_error( + version_throttle, + NamespaceError::TableVersionNotFound { + message: "version 5 not found".to_string(), + }, + )), + ErrorCode::Throttling + ); + } + + /// Helper to create test IPC data from a schema + fn create_test_ipc_data(schema: &JsonArrowSchema) -> Vec { + use arrow::ipc::writer::StreamWriter; + + let arrow_schema = convert_json_arrow_schema(schema).unwrap(); + let arrow_schema = Arc::new(arrow_schema); + let batch = arrow::record_batch::RecordBatch::new_empty(arrow_schema.clone()); + let mut buffer = Vec::new(); + { + let mut writer = StreamWriter::try_new(&mut buffer, &arrow_schema).unwrap(); + writer.write(&batch).unwrap(); + writer.finish().unwrap(); + } + buffer + } + + fn create_ipc_data_from_batches( + schema: Arc, + batches: Vec, + ) -> Vec { + use arrow::ipc::writer::StreamWriter; + + let mut buffer = Vec::new(); + { + let mut writer = StreamWriter::try_new(&mut buffer, &schema).unwrap(); + for batch in &batches { + writer.write(batch).unwrap(); + } + writer.finish().unwrap(); + } + buffer + } + + fn create_non_empty_test_ipc_data() -> Vec { + use arrow::array::{Int32Array, StringArray}; + use arrow::record_batch::RecordBatch; + + let schema = Arc::new(convert_json_arrow_schema(&create_test_schema()).unwrap()); + let batch = RecordBatch::try_new( + schema.clone(), + vec![ + Arc::new(Int32Array::from(vec![1, 2])), + Arc::new(StringArray::from(vec![Some("alice"), Some("bob")])), + ], + ) + .unwrap(); + create_ipc_data_from_batches(schema, vec![batch]) + } + + fn create_single_row_test_ipc_data() -> Vec { + use arrow::array::{Int32Array, StringArray}; + use arrow::record_batch::RecordBatch; + + let schema = Arc::new(convert_json_arrow_schema(&create_test_schema()).unwrap()); + let batch = RecordBatch::try_new( + schema.clone(), + vec![ + Arc::new(Int32Array::from(vec![10])), + Arc::new(StringArray::from(vec![Some("carol")])), + ], + ) + .unwrap(); + create_ipc_data_from_batches(schema, vec![batch]) + } + + /// Helper to create a simple test schema + fn create_test_schema() -> JsonArrowSchema { + let int_type = JsonArrowDataType::new("int32".to_string()); + let string_type = JsonArrowDataType::new("utf8".to_string()); + + let id_field = JsonArrowField { name: "id".to_string(), r#type: Box::new(int_type), nullable: false, @@ -5574,6 +6937,43 @@ mod tests { create_ipc_data_from_batches(schema, vec![batch]) } + async fn create_legacy_manifest_without_primary_key_metadata(root: &str) { + use arrow::datatypes::{DataType, Field, Schema as ArrowSchema}; + use arrow::record_batch::{RecordBatch, RecordBatchIterator}; + + let schema = Arc::new(ArrowSchema::new(vec![ + Field::new("object_id", DataType::Utf8, false), + Field::new("object_type", DataType::Utf8, false), + Field::new("location", DataType::Utf8, true), + Field::new("metadata", DataType::Utf8, true), + Field::new( + "base_objects", + DataType::List(Arc::new(Field::new("object_id", DataType::Utf8, true))), + true, + ), + ])); + let batch = RecordBatch::new_empty(schema.clone()); + let reader = RecordBatchIterator::new(vec![Ok(batch)], schema); + Dataset::write(Box::new(reader), &format!("{}/__manifest", root), None) + .await + .unwrap(); + } + + async fn manifest_has_primary_key_metadata(root: &str) -> bool { + let dataset = Dataset::open(&format!("{}/__manifest", root)) + .await + .unwrap(); + dataset + .schema() + .field("object_id") + .map(|field| { + field + .metadata + .contains_key(lance_core::datatypes::LANCE_UNENFORCED_PRIMARY_KEY_POSITION) + }) + .unwrap_or(false) + } + fn create_vector_table_ipc_data() -> Vec { use arrow::array::{FixedSizeListArray, Float32Array, Int32Array}; use arrow::datatypes::{DataType, Field, Schema as ArrowSchema}; @@ -8218,19 +9618,152 @@ mod tests { assert_eq!(tables.len(), 1); assert_eq!(tables[0], "table1"); - // Verify table exists + // Verify table exists + let mut exists_req = TableExistsRequest::new(); + exists_req.id = Some(vec!["test_ns".to_string(), "table1".to_string()]); + let result = namespace.table_exists(exists_req).await; + assert!(result.is_ok()); + + // Describe table in child namespace + let mut describe_req = DescribeTableRequest::new(); + describe_req.id = Some(vec!["test_ns".to_string(), "table1".to_string()]); + let result = namespace.describe_table(describe_req).await; + assert!(result.is_ok()); + let response = result.unwrap(); + assert!(response.location.is_some()); + } + + /// Regression: a connection built before `__manifest` exists must still + /// resolve a child-namespaced table that a *different* connection registers + /// afterwards. Phalanx caches one DirectoryNamespace per db and the first op + /// on a fresh db is usually a read, so without a self-healing read path the + /// cached reader pins an empty manifest cell and every describe/exists/list + /// on the table reports "not found" forever -- even though `create_table` + /// reports it already exists. This is the geneva `__system$geneva_jobs` + /// open->create->open livelock. + #[tokio::test] + async fn test_read_self_heals_after_manifest_created_by_other_connection() { + let temp_dir = TempStdDir::default(); + let root = temp_dir.to_str().unwrap(); + + // Reader is built while no `__manifest` exists yet -> its read cell is + // empty and, before the fix, stays empty forever. + let reader = DirectoryNamespaceBuilder::new(root).build().await.unwrap(); + + // A *separate* connection creates the child namespace + table, which + // lazily creates `__manifest` and registers the entry. + let writer = DirectoryNamespaceBuilder::new(root).build().await.unwrap(); + let mut create_ns_req = CreateNamespaceRequest::new(); + create_ns_req.id = Some(vec!["test_ns".to_string()]); + writer.create_namespace(create_ns_req).await.unwrap(); + let ipc_data = create_test_ipc_data(&create_test_schema()); + let mut create_table_req = CreateTableRequest::new(); + create_table_req.id = Some(vec!["test_ns".to_string(), "table1".to_string()]); + writer + .create_table(create_table_req, bytes::Bytes::from(ipc_data)) + .await + .unwrap(); + + // The reader, though built before the manifest existed, must now resolve + // the table on every read path (was TableNotFound before the fix). + let mut describe_req = DescribeTableRequest::new(); + describe_req.id = Some(vec!["test_ns".to_string(), "table1".to_string()]); + let resp = reader + .describe_table(describe_req) + .await + .expect("describe_table must resolve a table registered after build"); + assert!(resp.location.is_some()); + + let mut exists_req = TableExistsRequest::new(); + exists_req.id = Some(vec!["test_ns".to_string(), "table1".to_string()]); + reader + .table_exists(exists_req) + .await + .expect("table_exists must resolve a table registered after build"); + + let list_req = ListTablesRequest { + id: Some(vec!["test_ns".to_string()]), + ..Default::default() + }; + let tables = reader.list_tables(list_req).await.unwrap().tables; + assert_eq!(tables, vec!["table1".to_string()]); + } + + /// Migration mode promises manifest-first lookup even at the root, so a + /// reader built before `__manifest` existed must still resolve a + /// manifest-only alias (`registered_table` -> `external_table.lance`) that + /// dir-listing cannot produce. Before the read path self-healed in migration + /// mode, the root gate bypassed the manifest probe and fell back to + /// dir-listing, which sees `external_table` but never `registered_table` -> + /// permanent TableNotFound for every registration made after build. + #[tokio::test] + async fn test_migration_root_read_self_heals_registered_alias() { + use lance_namespace::models::RegisterTableRequest; + + let temp_dir = TempStdDir::default(); + let temp_path = temp_dir.to_str().unwrap(); + + // Reader built while the root is empty -> no `__manifest`, read cell + // empty (and, before the fix, frozen empty forever). + let reader = DirectoryNamespaceBuilder::new(temp_path) + .dir_listing_enabled(true) + .dir_listing_to_manifest_migration_enabled(true) + .build() + .await + .unwrap(); + + // A separate connection writes an external dataset and registers it in + // the manifest under a *different* logical name -- an alias dir-listing + // cannot resolve. This is what lazily creates `__manifest`. + let writer = DirectoryNamespaceBuilder::new(temp_path) + .dir_listing_enabled(true) + .dir_listing_to_manifest_migration_enabled(true) + .build() + .await + .unwrap(); + let ipc_data = create_test_ipc_data(&create_test_schema()); + let table_uri = format!("{}/external_table.lance", temp_path); + let cursor = Cursor::new(ipc_data); + let stream_reader = StreamReader::try_new(cursor, None).unwrap(); + let batches: Vec<_> = stream_reader + .collect::, _>>() + .unwrap(); + let schema = batches[0].schema(); + let batch_results: Vec<_> = batches.into_iter().map(Ok).collect(); + let batch_reader = RecordBatchIterator::new(batch_results, schema); + Dataset::write(Box::new(batch_reader), &table_uri, None) + .await + .unwrap(); + let mut register_req = RegisterTableRequest::new("external_table.lance".to_string()); + register_req.id = Some(vec!["registered_table".to_string()]); + writer.register_table(register_req).await.unwrap(); + + // The reader, built before `__manifest` existed, must now resolve the + // manifest-only alias on every read path. + let mut describe_req = DescribeTableRequest::new(); + describe_req.id = Some(vec!["registered_table".to_string()]); + reader + .describe_table(describe_req) + .await + .expect("describe_table must resolve a manifest alias registered after build"); + let mut exists_req = TableExistsRequest::new(); - exists_req.id = Some(vec!["test_ns".to_string(), "table1".to_string()]); - let result = namespace.table_exists(exists_req).await; - assert!(result.is_ok()); + exists_req.id = Some(vec!["registered_table".to_string()]); + reader + .table_exists(exists_req) + .await + .expect("table_exists must resolve a manifest alias registered after build"); - // Describe table in child namespace - let mut describe_req = DescribeTableRequest::new(); - describe_req.id = Some(vec!["test_ns".to_string(), "table1".to_string()]); - let result = namespace.describe_table(describe_req).await; - assert!(result.is_ok()); - let response = result.unwrap(); - assert!(response.location.is_some()); + let list_req = ListTablesRequest { + id: Some(vec![]), + ..Default::default() + }; + let tables = reader.list_tables(list_req).await.unwrap().tables; + assert!( + tables.contains(&"registered_table".to_string()), + "list_tables must include the manifest alias registered after build, got {:?}", + tables + ); } #[tokio::test] @@ -9633,6 +11166,45 @@ mod tests { ); } + #[tokio::test] + async fn test_declare_table_with_manifest_marker_already_exists() { + // Pre-existing .lance-reserved (concurrent/incomplete declare) must map to + // TableAlreadyExists, not Internal. + use lance_namespace::error::ErrorCode; + use lance_namespace::models::DeclareTableRequest; + + let temp_dir = TempStdDir::default(); + let temp_path = temp_dir.to_str().unwrap(); + + let namespace = DirectoryNamespaceBuilder::new(temp_path) + .manifest_enabled(true) + .dir_listing_enabled(true) + .build() + .await + .unwrap(); + + let table_dir = temp_dir.join("test_table.lance"); + std::fs::create_dir_all(&table_dir).unwrap(); + std::fs::write(table_dir.join(".lance-reserved"), b"reserved").unwrap(); + + let mut declare_req = DeclareTableRequest::new(); + declare_req.id = Some(vec!["test_table".to_string()]); + let err = namespace + .declare_table(declare_req) + .await + .expect_err("declare with existing marker must fail"); + let msg = err.to_string(); + assert!( + msg.contains("already exists") || msg.contains("TableAlreadyExists"), + "expected TableAlreadyExists, got: {msg}" + ); + assert_eq!( + mutation_error_code(err), + ErrorCode::TableAlreadyExists, + "expected TableAlreadyExists error code" + ); + } + #[tokio::test] async fn test_declare_table_when_table_exists() { use lance_namespace::models::DeclareTableRequest; @@ -9932,7 +11504,7 @@ mod tests { .unwrap(); // Table status should show exists=true, is_deregistered=false - let status = namespace.check_table_status("test_table").await; + let status = namespace.check_table_status("test_table").await.unwrap(); assert!(status.exists); assert!(!status.is_deregistered); assert!(!status.has_reserved_file); @@ -10319,12 +11891,257 @@ mod tests { .await .unwrap(); - // Should return version 3 as it's the latest - assert_eq!(describe_resp.version.version, 3); + // Should return version 3 as it's the latest + assert_eq!(describe_resp.version.version, 3); + } + + #[tokio::test] + async fn test_create_table_version() { + use futures::TryStreamExt; + use lance::dataset::builder::DatasetBuilder; + use lance_namespace::models::CreateTableVersionRequest; + + let temp_dir = TempStrDir::default(); + let temp_path: &str = &temp_dir; + + let namespace: Arc = Arc::new( + DirectoryNamespaceBuilder::new(temp_path) + .table_version_tracking_enabled(true) + .build() + .await + .unwrap(), + ); + + // Create a table + let schema = create_test_schema(); + let ipc_data = create_test_ipc_data(&schema); + let mut create_req = CreateTableRequest::new(); + create_req.id = Some(vec!["test_table".to_string()]); + namespace + .create_table(create_req, bytes::Bytes::from(ipc_data)) + .await + .unwrap(); + + // Open the dataset using from_namespace to get proper object_store and paths + let table_id = vec!["test_table".to_string()]; + let dataset = DatasetBuilder::from_namespace(namespace.clone(), table_id.clone()) + .await + .unwrap() + .load() + .await + .unwrap(); + + // Use dataset's object_store to find and copy the manifest + let versions_path = dataset.versions_dir(); + let manifest_metas: Vec<_> = dataset + .object_store(None) + .await + .unwrap() + .inner + .list(Some(&versions_path)) + .try_collect() + .await + .unwrap(); + + let manifest_meta = manifest_metas + .iter() + .find(|m| { + m.location + .filename() + .map(|f| f.ends_with(".manifest")) + .unwrap_or(false) + }) + .expect("No manifest file found"); + + // Read the existing manifest data + let manifest_data = dataset + .object_store(None) + .await + .unwrap() + .inner + .get(&manifest_meta.location) + .await + .unwrap() + .bytes() + .await + .unwrap(); + + // Write to a staging location using the dataset's object_store + let staging_path = dataset.versions_dir().join("staging_manifest"); + dataset + .object_store(None) + .await + .unwrap() + .inner + .put(&staging_path, manifest_data.into()) + .await + .unwrap(); + + // Create version 2 from staging manifest + // Use the same naming scheme as the existing dataset (V2) + let mut create_version_req = CreateTableVersionRequest::new(2, staging_path.to_string()); + create_version_req.id = Some(table_id.clone()); + create_version_req.naming_scheme = Some("V2".to_string()); + + let result = namespace.create_table_version(create_version_req).await; + assert!( + result.is_ok(), + "create_table_version should succeed: {:?}", + result + ); + + // Verify version 2 was created at the path returned in the response + let response = result.unwrap(); + let version_info = response + .version + .expect("response should contain version info"); + let version_2_path = Path::parse(&version_info.manifest_path).unwrap(); + let head_result = dataset + .object_store(None) + .await + .unwrap() + .inner + .head(&version_2_path) + .await; + assert!( + head_result.is_ok(), + "Version 2 manifest should exist at {}", + version_2_path + ); + + // Verify the staging file has been deleted + let staging_head_result = dataset + .object_store(None) + .await + .unwrap() + .inner + .head(&staging_path) + .await; + assert!( + staging_head_result.is_err(), + "Staging manifest should have been deleted after create_table_version" + ); + } + + #[tokio::test] + async fn test_create_table_version_idempotent() { + // A network retry of create_table_version with the same staging content + // must succeed (not ConcurrentModification) once the version is published. + use futures::TryStreamExt; + use lance::dataset::builder::DatasetBuilder; + use lance_namespace::models::CreateTableVersionRequest; + + let temp_dir = TempStrDir::default(); + let temp_path: &str = &temp_dir; + + let namespace: Arc = Arc::new( + DirectoryNamespaceBuilder::new(temp_path) + .table_version_tracking_enabled(true) + .build() + .await + .unwrap(), + ); + + let schema = create_test_schema(); + let ipc_data = create_test_ipc_data(&schema); + let mut create_req = CreateTableRequest::new(); + create_req.id = Some(vec!["test_table".to_string()]); + namespace + .create_table(create_req, bytes::Bytes::from(ipc_data)) + .await + .unwrap(); + + let table_id = vec!["test_table".to_string()]; + let dataset = DatasetBuilder::from_namespace(namespace.clone(), table_id.clone()) + .await + .unwrap() + .load() + .await + .unwrap(); + + let versions_path = dataset.versions_dir(); + let manifest_metas: Vec<_> = dataset + .object_store(None) + .await + .unwrap() + .inner + .list(Some(&versions_path)) + .try_collect() + .await + .unwrap(); + + let manifest_meta = manifest_metas + .iter() + .find(|m| { + m.location + .filename() + .map(|f| f.ends_with(".manifest")) + .unwrap_or(false) + }) + .expect("No manifest file found"); + + let manifest_data = dataset + .object_store(None) + .await + .unwrap() + .inner + .get(&manifest_meta.location) + .await + .unwrap() + .bytes() + .await + .unwrap(); + + let staging_path = dataset.versions_dir().join("staging_manifest"); + dataset + .object_store(None) + .await + .unwrap() + .inner + .put(&staging_path, manifest_data.clone().into()) + .await + .unwrap(); + + let mut create_version_req = CreateTableVersionRequest::new(2, staging_path.to_string()); + create_version_req.id = Some(table_id.clone()); + create_version_req.naming_scheme = Some("V2".to_string()); + let first = namespace + .create_table_version(create_version_req) + .await + .expect("first create_table_version should succeed"); + + // Re-stage identical bytes (simulates Lance commit retry rewriting staging). + let retry_staging = dataset.versions_dir().join("staging_manifest_retry"); + dataset + .object_store(None) + .await + .unwrap() + .inner + .put(&retry_staging, manifest_data.into()) + .await + .unwrap(); + + let mut retry_req = CreateTableVersionRequest::new(2, retry_staging.to_string()); + retry_req.id = Some(table_id.clone()); + retry_req.naming_scheme = Some("V2".to_string()); + let second = namespace + .create_table_version(retry_req) + .await + .expect("idempotent retry must succeed"); + + assert_eq!( + first.version.as_ref().map(|v| v.version), + second.version.as_ref().map(|v| v.version) + ); + assert_eq!( + first.version.as_ref().map(|v| &v.manifest_path), + second.version.as_ref().map(|v| &v.manifest_path) + ); } #[tokio::test] - async fn test_create_table_version() { + async fn test_create_table_version_conflict() { + // Same version with different content must fail ConcurrentModification. use futures::TryStreamExt; use lance::dataset::builder::DatasetBuilder; use lance_namespace::models::CreateTableVersionRequest; @@ -10405,56 +12222,75 @@ mod tests { .await .unwrap(); - // Create version 2 from staging manifest - // Use the same naming scheme as the existing dataset (V2) + // First create version 2 (should succeed) let mut create_version_req = CreateTableVersionRequest::new(2, staging_path.to_string()); create_version_req.id = Some(table_id.clone()); create_version_req.naming_scheme = Some("V2".to_string()); - - let result = namespace.create_table_version(create_version_req).await; + let first_result = namespace.create_table_version(create_version_req).await; assert!( - result.is_ok(), - "create_table_version should succeed: {:?}", - result + first_result.is_ok(), + "First create_table_version for version 2 should succeed: {:?}", + first_result ); - // Verify version 2 was created at the path returned in the response - let response = result.unwrap(); - let version_info = response - .version - .expect("response should contain version info"); - let version_2_path = Path::parse(&version_info.manifest_path).unwrap(); - let head_result = dataset + // Get the path from the response for verification + let version_2_path = Path::parse( + &first_result + .unwrap() + .version + .expect("response should contain version info") + .manifest_path, + ) + .unwrap(); + + // Different content for the same version number must conflict. + let conflict_staging = dataset.versions_dir().join("staging_manifest_conflict"); + dataset .object_store(None) .await .unwrap() .inner - .head(&version_2_path) - .await; + .put( + &conflict_staging, + bytes::Bytes::from_static(b"not-a-real-manifest").into(), + ) + .await + .unwrap(); + + let mut create_version_req = + CreateTableVersionRequest::new(2, conflict_staging.to_string()); + create_version_req.id = Some(table_id.clone()); + create_version_req.naming_scheme = Some("V2".to_string()); + + let result = namespace.create_table_version(create_version_req).await; assert!( - head_result.is_ok(), - "Version 2 manifest should exist at {}", - version_2_path + result.is_err(), + "create_table_version should fail for existing version with different content" + ); + let err = result.unwrap_err().to_string(); + assert!( + err.contains("already exists") || err.contains("ConcurrentModification"), + "expected ConcurrentModification, got: {err}" ); - // Verify the staging file has been deleted - let staging_head_result = dataset + // Verify version 2 still exists using the dataset's object_store + let head_result = dataset .object_store(None) .await .unwrap() .inner - .head(&staging_path) + .head(&version_2_path) .await; assert!( - staging_head_result.is_err(), - "Staging manifest should have been deleted after create_table_version" + head_result.is_ok(), + "Version 2 manifest should still exist at {}", + version_2_path ); } #[tokio::test] - async fn test_create_table_version_conflict() { - // create_table_version should fail if the version already exists. - // Each version always writes to a new file location. + async fn test_create_table_version_cas_rejects_gap() { + // Strict CAS: version must be latest+1; skipping ahead is ConcurrentModification. use futures::TryStreamExt; use lance::dataset::builder::DatasetBuilder; use lance_namespace::models::CreateTableVersionRequest; @@ -10470,7 +12306,6 @@ mod tests { .unwrap(), ); - // Create a table let schema = create_test_schema(); let ipc_data = create_test_ipc_data(&schema); let mut create_req = CreateTableRequest::new(); @@ -10480,7 +12315,6 @@ mod tests { .await .unwrap(); - // Open the dataset using from_namespace to get proper object_store and paths let table_id = vec!["test_table".to_string()]; let dataset = DatasetBuilder::from_namespace(namespace.clone(), table_id.clone()) .await @@ -10489,7 +12323,6 @@ mod tests { .await .unwrap(); - // Use dataset's object_store to find and copy the manifest let versions_path = dataset.versions_dir(); let manifest_metas: Vec<_> = dataset .object_store(None) @@ -10500,7 +12333,6 @@ mod tests { .try_collect() .await .unwrap(); - let manifest_meta = manifest_metas .iter() .find(|m| { @@ -10510,8 +12342,6 @@ mod tests { .unwrap_or(false) }) .expect("No manifest file found"); - - // Read the existing manifest data let manifest_data = dataset .object_store(None) .await @@ -10524,8 +12354,7 @@ mod tests { .await .unwrap(); - // Write to a staging location using the dataset's object_store - let staging_path = dataset.versions_dir().join("staging_manifest"); + let staging_path = dataset.versions_dir().join("staging_gap"); dataset .object_store(None) .await @@ -10535,51 +12364,148 @@ mod tests { .await .unwrap(); - // First create version 2 (should succeed) - let mut create_version_req = CreateTableVersionRequest::new(2, staging_path.to_string()); - create_version_req.id = Some(table_id.clone()); - create_version_req.naming_scheme = Some("V2".to_string()); - let first_result = namespace.create_table_version(create_version_req).await; + // After create_table, latest is 1; requesting 5 must fail CAS. + let mut req = CreateTableVersionRequest::new(5, staging_path.to_string()); + req.id = Some(table_id); + req.naming_scheme = Some("V2".to_string()); + let err = namespace + .create_table_version(req) + .await + .expect_err("gap create must fail CAS"); + let msg = err.to_string(); assert!( - first_result.is_ok(), - "First create_table_version for version 2 should succeed: {:?}", - first_result + msg.contains("CAS") || msg.contains("ConcurrentModification"), + "expected CAS ConcurrentModification, got: {msg}" ); + } - // Get the path from the response for verification - let version_2_path = Path::parse( - &first_result - .unwrap() - .version - .expect("response should contain version info") - .manifest_path, - ) - .unwrap(); + #[tokio::test] + async fn test_create_table_version_branch_cas_requires_parent_version() { + // Empty branch chain with BranchContents must bootstrap at parent_version, + // not an arbitrary version (e.g. 1 when forked from v2). + use futures::TryStreamExt; + use lance_namespace::models::CreateTableVersionRequest; - // Create version 2 again (should fail - conflict) - let mut create_version_req = CreateTableVersionRequest::new(2, staging_path.to_string()); - create_version_req.id = Some(table_id.clone()); - create_version_req.naming_scheme = Some("V2".to_string()); + let (namespace, _temp_dir) = create_test_namespace().await; + create_scalar_table(&namespace, "users").await; + let main_uri = open_dataset(&namespace, "users").await.uri().to_string(); + append_scalar_version(&main_uri, 10).await; // main -> v2 - let result = namespace.create_table_version(create_version_req).await; - assert!( - result.is_err(), - "create_table_version should fail for existing version" + let mut main = open_dataset(&namespace, "users").await; + let fork_version = main.version().version; + assert_eq!(fork_version, 2); + let branch_uri = main + .create_branch("exp", fork_version, None) + .await + .unwrap() + .uri() + .to_string(); + + let branch_ds = Dataset::open(&branch_uri).await.unwrap(); + let versions_dir = branch_ds.versions_dir(); + let store = branch_ds.object_store(None).await.unwrap(); + let manifests: Vec<_> = store + .inner + .list(Some(&versions_dir)) + .try_collect() + .await + .unwrap(); + for meta in &manifests { + if meta + .location + .filename() + .is_some_and(|f| f.ends_with(".manifest")) + { + store.inner.delete(&meta.location).await.unwrap(); + } + } + // Confirm the branch object-store chain is empty (do not open the dataset: + // with no manifests, Dataset::open would fail). + let remaining_manifests = store + .inner + .list(Some(&versions_dir)) + .try_collect::>() + .await + .unwrap() + .into_iter() + .filter(|m| { + m.location + .filename() + .is_some_and(|f| f.ends_with(".manifest")) + }) + .count(); + assert_eq!( + remaining_manifests, 0, + "branch version chain should be empty after deleting manifests" ); - // Verify version 2 still exists using the dataset's object_store - let head_result = dataset - .object_store(None) + // Stage bytes from a main manifest. + let main_ds = open_dataset(&namespace, "users").await; + let main_versions = main_ds.versions_dir(); + let main_store = main_ds.object_store(None).await.unwrap(); + let source_meta = main_store + .inner + .list(Some(&main_versions)) + .try_collect::>() .await .unwrap() + .into_iter() + .find(|m| { + m.location + .filename() + .is_some_and(|f| f.ends_with(".manifest")) + }) + .expect("main should have a manifest"); + let source_bytes = main_store .inner - .head(&version_2_path) - .await; + .get(&source_meta.location) + .await + .unwrap() + .bytes() + .await + .unwrap(); + + let staging_wrong = versions_dir.clone().join("staging_wrong"); + store + .inner + .put(&staging_wrong, source_bytes.clone().into()) + .await + .unwrap(); + let err = namespace + .create_table_version(CreateTableVersionRequest { + id: Some(vec!["users".to_string()]), + version: 1, + manifest_path: staging_wrong.to_string(), + naming_scheme: Some("V2".to_string()), + branch: Some("exp".to_string()), + ..Default::default() + }) + .await + .expect_err("bootstrap at v1 must fail when parent_version is 2"); + let msg = err.to_string(); assert!( - head_result.is_ok(), - "Version 2 manifest should still exist at {}", - version_2_path + msg.contains("CAS") || msg.contains("ConcurrentModification"), + "expected CAS ConcurrentModification, got: {msg}" ); + + let staging_ok = versions_dir.join("staging_ok"); + store + .inner + .put(&staging_ok, source_bytes.into()) + .await + .unwrap(); + let resp = namespace + .create_table_version(CreateTableVersionRequest { + id: Some(vec!["users".to_string()]), + version: 2, + manifest_path: staging_ok.to_string(), + naming_scheme: Some("V2".to_string()), + branch: Some("exp".to_string()), + ..Default::default() + }) + .await + .expect("bootstrap at parent_version must succeed"); + assert_eq!(resp.version.as_ref().map(|v| v.version), Some(2)); } #[tokio::test] @@ -12256,7 +14182,7 @@ mod tests { &plan_str, &[ "ProjectionExec: expr=[id@0 as id, name@2 as name", - "Take: columns=\"id, _rowid, (name)\"", + "projection=[name], source=stream(_rowid)", "LanceRead: uri=", "projection=[id]", "row_id=true, row_addr=false", @@ -12353,9 +14279,7 @@ mod tests { "AnalyzeExec verbose=true", "ProjectionExec: elapsed=", "expr=[id@0 as id, name@2 as name", - "Take: elapsed=", - "columns=\"id, _rowid, (name)\"", - "CoalesceBatchesExec: elapsed=", + "projection=[name], source=stream(_rowid)", "LanceRead: elapsed=", "projection=[id]", "row_id=true, row_addr=false", @@ -12435,6 +14359,109 @@ mod tests { ); } + #[tokio::test] + async fn test_build_and_root_reads_do_not_create_manifest() { + let temp_dir = TempStdDir::default(); + let temp_path = temp_dir.to_str().unwrap(); + let manifest_path = std::path::Path::new(temp_path).join("__manifest"); + + let dir_only_ns = DirectoryNamespaceBuilder::new(temp_path) + .manifest_enabled(false) + .dir_listing_enabled(true) + .build() + .await + .unwrap(); + create_scalar_table(&dir_only_ns, "catalog").await; + assert!(!manifest_path.exists()); + + let namespace = DirectoryNamespaceBuilder::new(temp_path) + .manifest_enabled(true) + .dir_listing_enabled(true) + .build() + .await + .unwrap(); + assert!(!manifest_path.exists()); + + let mut exists_req = TableExistsRequest::new(); + exists_req.id = Some(vec!["catalog".to_string()]); + namespace.table_exists(exists_req).await.unwrap(); + assert!(!manifest_path.exists()); + + let mut describe_req = DescribeTableRequest::new(); + describe_req.id = Some(vec!["catalog".to_string()]); + namespace.describe_table(describe_req).await.unwrap(); + assert!(!manifest_path.exists()); + + let list_response = namespace + .list_tables(ListTablesRequest { + id: Some(vec![]), + ..Default::default() + }) + .await + .unwrap(); + assert_eq!(list_response.tables, vec!["catalog".to_string()]); + assert!(!manifest_path.exists()); + + let mut list_namespaces_req = ListNamespacesRequest::new(); + list_namespaces_req.id = Some(vec!["workspace".to_string()]); + let err = namespace + .list_namespaces(list_namespaces_req) + .await + .unwrap_err(); + assert!(err.to_string().contains("__manifest")); + assert!(!manifest_path.exists()); + + let err = namespace + .list_tables(ListTablesRequest { + id: Some(vec!["workspace".to_string()]), + ..Default::default() + }) + .await + .unwrap_err(); + assert!(err.to_string().contains("__manifest")); + assert!(!manifest_path.exists()); + + let mut child_describe_req = DescribeTableRequest::new(); + child_describe_req.id = Some(vec!["workspace".to_string(), "catalog".to_string()]); + let err = namespace + .describe_table(child_describe_req) + .await + .unwrap_err(); + assert!(err.to_string().contains("__manifest")); + assert!(!manifest_path.exists()); + + let mut child_exists_req = TableExistsRequest::new(); + child_exists_req.id = Some(vec!["workspace".to_string(), "catalog".to_string()]); + let err = namespace.table_exists(child_exists_req).await.unwrap_err(); + assert!(err.to_string().contains("__manifest")); + assert!(!manifest_path.exists()); + + let mut create_ns_req = CreateNamespaceRequest::new(); + create_ns_req.id = Some(vec!["workspace".to_string()]); + namespace.create_namespace(create_ns_req).await.unwrap(); + assert!(manifest_path.exists()); + } + + #[tokio::test] + async fn test_migrate_updates_read_opened_legacy_manifest() { + let temp_dir = TempStdDir::default(); + let temp_path = temp_dir.to_str().unwrap(); + create_legacy_manifest_without_primary_key_metadata(temp_path).await; + assert!(!manifest_has_primary_key_metadata(temp_path).await); + + let namespace = DirectoryNamespaceBuilder::new(temp_path) + .manifest_enabled(true) + .dir_listing_enabled(true) + .build() + .await + .unwrap(); + assert!(!manifest_has_primary_key_metadata(temp_path).await); + + let migrated = namespace.migrate().await.unwrap(); + assert_eq!(migrated, 0); + assert!(manifest_has_primary_key_metadata(temp_path).await); + } + #[tokio::test] async fn test_describe_declared_table_checks_versions_only_when_requested() { let temp_dir = TempStdDir::default(); @@ -12518,9 +14545,13 @@ mod tests { .await .unwrap(); - // table_exists first checks __manifest (which on local FS uses the - // version hint and does no list call), then falls back to the table - // directory (one list_with_delimiter on test_table.lance). + // In migration mode the manifest is authoritative, so table_exists first + // probes __manifest via ensure_read_manifest() to self-heal any + // manifest-registered aliases (why the gate now admits this mode). Here + // the table is dir-only and __manifest does not exist yet, so that probe + // costs one list to confirm absence; the table-directory fallback is the + // second. (When __manifest exists the probe uses the version hint and + // adds no list, so a real self-heal is free.) listing_count.store(0, Ordering::SeqCst); let mut exists_req = TableExistsRequest::new(); @@ -12529,13 +14560,14 @@ mod tests { let count = listing_count.load(Ordering::SeqCst); assert_eq!( - count, 1, - "Expected exactly 1 listing call for table_exists with migration mode \ - (table directory fallback; manifest reload uses the version hint), but got {}", + count, 2, + "Expected 2 listing calls for table_exists with migration mode \ + (absent-__manifest probe + table directory fallback), but got {}", count ); - // describe_table follows the same path when the table is not yet registered in __manifest. + // describe_table follows the same path: an ensure_read_manifest() probe + // of the (absent) __manifest, then the table-directory fallback. listing_count.store(0, Ordering::SeqCst); let mut describe_req = DescribeTableRequest::new(); @@ -12544,9 +14576,9 @@ mod tests { let count = listing_count.load(Ordering::SeqCst); assert_eq!( - count, 1, - "Expected exactly 1 listing call for describe_table with migration mode \ - (table directory fallback; manifest reload uses the version hint), but got {}", + count, 2, + "Expected 2 listing calls for describe_table with migration mode \ + (absent-__manifest probe + table directory fallback), but got {}", count ); } diff --git a/rust/lance-namespace-impls/src/dir/manifest.rs b/rust/lance-namespace-impls/src/dir/manifest.rs index bca7408369e..93cd7429e6f 100644 --- a/rust/lance-namespace-impls/src/dir/manifest.rs +++ b/rust/lance-namespace-impls/src/dir/manifest.rs @@ -36,7 +36,9 @@ use lance_index::progress::noop_progress; use lance_index::registry::IndexPluginRegistry; use lance_index::scalar::lance_format::LanceIndexStore; use lance_index::scalar::registry::VALUE_COLUMN_NAME; -use lance_index::scalar::{BuiltinIndexType, CreatedIndex, ScalarIndexParams}; +use lance_index::scalar::{ + BuiltinIndexType, CreatedIndex, ScalarIndexParams, index_files_to_table, +}; use lance_io::object_store::{ObjectStore, ObjectStoreParams}; use lance_io::stream::RecordBatchStream as LanceRecordBatchStream; use lance_namespace::LanceNamespace; @@ -852,7 +854,60 @@ impl ManifestNamespace { Self::ensure_manifest_table_up_to_date(&root, &storage_options, session.clone()) .await?; - Ok(Self { + Ok(Self::new( + root, + storage_options, + session, + object_store, + base_path, + manifest_dataset, + dir_listing_enabled, + inline_optimization_enabled, + commit_retries, + )) + } + + /// Open an existing manifest dataset without creating or migrating it. + #[allow(clippy::too_many_arguments)] + pub async fn open_from_directory( + root: String, + storage_options: Option>, + session: Option>, + object_store: Arc, + base_path: Path, + dir_listing_enabled: bool, + inline_optimization_enabled: bool, + commit_retries: Option, + ) -> Result { + let manifest_dataset = + Self::open_manifest_table(&root, &storage_options, session.clone()).await?; + + Ok(Self::new( + root, + storage_options, + session, + object_store, + base_path, + manifest_dataset, + dir_listing_enabled, + inline_optimization_enabled, + commit_retries, + )) + } + + #[allow(clippy::too_many_arguments)] + fn new( + root: String, + storage_options: Option>, + session: Option>, + object_store: Arc, + base_path: Path, + manifest_dataset: DatasetConsistencyWrapper, + dir_listing_enabled: bool, + inline_optimization_enabled: bool, + commit_retries: Option, + ) -> Self { + Self { root, storage_options, session, @@ -863,7 +918,7 @@ impl ManifestNamespace { inline_optimization_enabled, commit_retries, manifest_mutation_lock: Arc::new(Mutex::new(())), - }) + } } /// Build object ID from namespace path and name @@ -1226,7 +1281,7 @@ impl ManifestNamespace { index_version: trained_index.created_index.index_version as i32, created_at: None, base_id: None, - files: Some(trained_index.created_index.files), + files: Some(index_files_to_table(trained_index.created_index.files)), }) } @@ -2411,6 +2466,37 @@ impl ManifestNamespace { Ok(found_result) } + /// Load an existing manifest dataset without creating or migrating it. + async fn open_manifest_table( + root: &str, + storage_options: &Option>, + session: Option>, + ) -> Result { + let manifest_path = format!("{}/{}", root, MANIFEST_TABLE_NAME); + log::debug!("Attempting to load manifest from {}", manifest_path); + let store_options = ObjectStoreParams { + storage_options_accessor: storage_options.as_ref().map(|opts| { + Arc::new( + lance_io::object_store::StorageOptionsAccessor::with_static_options( + opts.clone(), + ), + ) + }), + ..Default::default() + }; + let read_params = ReadParams { + session, + store_options: Some(store_options), + ..Default::default() + }; + let dataset = DatasetBuilder::from_uri(&manifest_path) + .with_read_params(read_params) + .load() + .await?; + ensure_readable(dataset.metadata())?; + Ok(DatasetConsistencyWrapper::new(dataset)) + } + /// Create or load the manifest dataset, ensuring it has the latest schema setup. /// /// This function will: @@ -2443,129 +2529,135 @@ impl ManifestNamespace { .with_read_params(read_params) .load() .await; - if let Ok(mut dataset) = dataset_result { - // Reject a manifest written with a reader feature flag this build - // does not understand before touching it. - ensure_readable(dataset.metadata())?; - - // Check if the object_id field has primary key metadata, migrate if not - let needs_pk_migration = dataset - .schema() - .field("object_id") - .map(|f| { - !f.metadata - .contains_key(LANCE_UNENFORCED_PRIMARY_KEY_POSITION) - }) - .unwrap_or(false); - - if needs_pk_migration { - // This legacy migration writes to the manifest, so confirm this - // build is allowed to write the current format first. - ensure_writable(dataset.metadata())?; - log::info!("Migrating __manifest table to add primary key metadata on object_id"); - dataset - .update_field_metadata() - .update("object_id", [(LANCE_UNENFORCED_PRIMARY_KEY_POSITION, "0")]) - .map_err(|e| { - lance_core::Error::from(NamespaceError::Internal { - message: format!( - "Failed to find object_id field for migration: {:?}", - e - ), - }) - })? - .await - .map_err(|e| { - lance_core::Error::from(NamespaceError::Internal { - message: format!("Failed to migrate primary key metadata: {:?}", e), - }) - })?; - } - - Ok(DatasetConsistencyWrapper::new(dataset)) - } else { - log::info!("Creating new manifest table at {}", manifest_path); - let schema = Self::manifest_schema(); - let empty_batch = RecordBatch::new_empty(schema.clone()); - let reader = RecordBatchIterator::new(vec![Ok(empty_batch)], schema.clone()); - - let store_params = ObjectStoreParams { - storage_options_accessor: storage_options.as_ref().map(|opts| { - Arc::new( - lance_io::object_store::StorageOptionsAccessor::with_static_options( - opts.clone(), - ), - ) - }), - ..Default::default() - }; - let write_params = WriteParams { - session: session.clone(), - store_params: Some(store_params), - ..Default::default() - }; - - let dataset = - Dataset::write(Box::new(reader), &manifest_path, Some(write_params)).await; + match dataset_result { + Ok(mut dataset) => { + // Reject a manifest written with a reader feature flag this build + // does not understand before touching it. + ensure_readable(dataset.metadata())?; + + // Check if the object_id field has primary key metadata, migrate if not + let needs_pk_migration = dataset + .schema() + .field("object_id") + .map(|f| { + !f.metadata + .contains_key(LANCE_UNENFORCED_PRIMARY_KEY_POSITION) + }) + .unwrap_or(false); - // Handle race condition where another process created the manifest concurrently - match dataset { - Ok(dataset) => { - log::info!( - "Successfully created manifest table at {}, version={}, uri={}", - manifest_path, - dataset.version().version, - dataset.uri() - ); - Ok(DatasetConsistencyWrapper::new(dataset)) - } - Err(ref e) - if matches!( - e, - LanceError::DatasetAlreadyExists { .. } - | LanceError::CommitConflict { .. } - | LanceError::IncompatibleTransaction { .. } - | LanceError::RetryableCommitConflict { .. } - ) => - { - // Another process created the manifest concurrently, try to load it + if needs_pk_migration { + // This legacy migration writes to the manifest, so confirm this + // build is allowed to write the current format first. + ensure_writable(dataset.metadata())?; log::info!( - "Manifest table was created by another process, loading it: {}", - manifest_path + "Migrating __manifest table to add primary key metadata on object_id" ); - let recovery_store_options = ObjectStoreParams { - storage_options_accessor: storage_options.as_ref().map(|opts| { - Arc::new( - lance_io::object_store::StorageOptionsAccessor::with_static_options( - opts.clone(), - ), - ) - }), - ..Default::default() - }; - let recovery_read_params = ReadParams { - session, - store_options: Some(recovery_store_options), - ..Default::default() - }; - let dataset = DatasetBuilder::from_uri(&manifest_path) - .with_read_params(recovery_read_params) - .load() - .await + dataset + .update_field_metadata() + .update("object_id", [(LANCE_UNENFORCED_PRIMARY_KEY_POSITION, "0")]) .map_err(|e| { lance_core::Error::from(NamespaceError::Internal { message: format!( - "Failed to load manifest dataset after creation conflict: {}", + "Failed to find object_id field for migration: {:?}", e ), }) + })? + .await + .map_err(|e| { + lance_core::Error::from(NamespaceError::Internal { + message: format!("Failed to migrate primary key metadata: {:?}", e), + }) })?; - Ok(DatasetConsistencyWrapper::new(dataset)) } - Err(e) => Err(lance_core::Error::from(NamespaceError::Internal { - message: format!("Failed to create manifest dataset: {:?}", e), - })), + + Ok(DatasetConsistencyWrapper::new(dataset)) + } + Err(err) if Self::is_not_found_load_error(&err) => { + log::info!("Creating new manifest table at {}", manifest_path); + let schema = Self::manifest_schema(); + let empty_batch = RecordBatch::new_empty(schema.clone()); + let reader = RecordBatchIterator::new(vec![Ok(empty_batch)], schema.clone()); + + let store_params = ObjectStoreParams { + storage_options_accessor: storage_options.as_ref().map(|opts| { + Arc::new( + lance_io::object_store::StorageOptionsAccessor::with_static_options( + opts.clone(), + ), + ) + }), + ..Default::default() + }; + let write_params = WriteParams { + session: session.clone(), + store_params: Some(store_params), + ..Default::default() + }; + + let dataset = + Dataset::write(Box::new(reader), &manifest_path, Some(write_params)).await; + + // Handle race condition where another process created the manifest concurrently + match dataset { + Ok(dataset) => { + log::info!( + "Successfully created manifest table at {}, version={}, uri={}", + manifest_path, + dataset.version().version, + dataset.uri() + ); + Ok(DatasetConsistencyWrapper::new(dataset)) + } + Err(ref e) + if matches!( + e, + LanceError::DatasetAlreadyExists { .. } + | LanceError::CommitConflict { .. } + | LanceError::IncompatibleTransaction { .. } + | LanceError::RetryableCommitConflict { .. } + ) => + { + // Another process created the manifest concurrently, try to load it + log::info!( + "Manifest table was created by another process, loading it: {}", + manifest_path + ); + let recovery_store_options = ObjectStoreParams { + storage_options_accessor: storage_options.as_ref().map(|opts| { + Arc::new( + lance_io::object_store::StorageOptionsAccessor::with_static_options( + opts.clone(), + ), + ) + }), + ..Default::default() + }; + let recovery_read_params = ReadParams { + session, + store_options: Some(recovery_store_options), + ..Default::default() + }; + let dataset = DatasetBuilder::from_uri(&manifest_path) + .with_read_params(recovery_read_params) + .load() + .await + .map_err(|e| { + lance_core::Error::from(NamespaceError::Internal { + message: format!( + "Failed to load manifest dataset after creation conflict: {}", + e + ), + }) + })?; + Ok(DatasetConsistencyWrapper::new(dataset)) + } + Err(e) => Err(lance_core::Error::from(NamespaceError::Internal { + message: format!("Failed to create manifest dataset: {:?}", e), + })), + } } + Err(err) => Err(err), } } @@ -3398,30 +3490,26 @@ impl LanceNamespace for ManifestNamespace { } } - // Create the .lance-reserved file to mark the table as existing + // Atomically create the .lance-reserved file to mark the table as declared. + // Shared with DirectoryNamespace via put_marker_file_atomic (dotfile-safe + // staging + MarkerFileError::AlreadyExists → TableAlreadyExists). let reserved_file_path = table_path.clone().join(".lance-reserved"); - - self.object_store - .create(&reserved_file_path) - .await - .map_err(|e| { - lance_core::Error::from(NamespaceError::Internal { - message: format!( - "Failed to create .lance-reserved file for table {}: {}", - table_name, e - ), - }) - })? - .shutdown() - .await - .map_err(|e| { - lance_core::Error::from(NamespaceError::Internal { - message: format!( - "Failed to finalize .lance-reserved file for table {}: {}", - table_name, e - ), + super::put_marker_file_atomic( + &self.object_store, + &reserved_file_path, + &format!("table {}", table_name), + ) + .await + .map_err(|e| match e { + super::MarkerFileError::AlreadyExists { .. } => { + lance_core::Error::from(NamespaceError::TableAlreadyExists { + message: table_name.to_string(), }) - })?; + } + super::MarkerFileError::Other { message } => { + lance_core::Error::from(NamespaceError::Internal { message }) + } + })?; let metadata = Self::serialize_metadata(request.properties.as_ref(), "table", &object_id)?; diff --git a/rust/lance-table/Cargo.toml b/rust/lance-table/Cargo.toml index 042ae92c618..583873507a9 100644 --- a/rust/lance-table/Cargo.toml +++ b/rust/lance-table/Cargo.toml @@ -24,6 +24,7 @@ arrow-ipc.workspace = true arrow-schema.workspace = true async-trait.workspace = true aws-credential-types = { workspace = true, optional = true } +blake3.workspace = true aws-sdk-dynamodb = { workspace = true, optional = true, default-features = false, features = ["default-https-client", "rt-tokio"] } byteorder.workspace = true bytes.workspace = true diff --git a/rust/lance-table/benches/manifest_intern.rs b/rust/lance-table/benches/manifest_intern.rs index 78b7e352207..81bd57c1a22 100644 --- a/rust/lance-table/benches/manifest_intern.rs +++ b/rust/lance-table/benches/manifest_intern.rs @@ -59,6 +59,7 @@ fn make_uniform_pb_fragments(n: u64, num_fields: usize) -> Vec file_size_bytes: 0, base_id: None, }], + overlays: vec![], deletion_file: None, row_id_sequence: None, physical_rows: 1000, @@ -135,6 +136,7 @@ fn make_diverse_pb_fragments( file_size_bytes: 0, base_id: None, }], + overlays: vec![], deletion_file: None, row_id_sequence: None, physical_rows: 1000, diff --git a/rust/lance-table/src/feature_flags.rs b/rust/lance-table/src/feature_flags.rs index 096f0da79e5..9c4d626829d 100644 --- a/rust/lance-table/src/feature_flags.rs +++ b/rust/lance-table/src/feature_flags.rs @@ -20,8 +20,35 @@ pub const FLAG_TABLE_CONFIG: u64 = 8; pub const FLAG_BASE_PATHS: u64 = 16; /// Disable writing transaction file under _transaction/, this flag is set when we only want to write inline transaction in manifest pub const FLAG_DISABLE_TRANSACTION_FILE: u64 = 32; +/// Fragments contain data overlay files, which supply new values for a subset of +/// cells without rewriting base data files. A reader that does not understand +/// overlays must refuse the dataset, since ignoring an overlay would silently +/// return stale base values. +/// +/// Data overlay files are not yet a released feature: in release builds this flag +/// is treated as unknown (so a release reader/writer refuses an overlay dataset) +/// unless [`ENABLE_UNSTABLE_DATA_OVERLAY_FILES_ENV`] is set, which lets benchmarks opt in. +/// Debug builds always understand it so tests exercise the path. +pub const FLAG_UNSTABLE_DATA_OVERLAY_FILES: u64 = 64; +/// `index_catchup` is maintained on this table, so a missing entry means the +/// index is *not* caught up rather than fully caught up. +/// +/// A reader without this bit would read a missing `index_catchup` entry as +/// "fully caught up" and could answer an index-only query without the SSTables +/// holding the newest rows. A writer without it would change an index without +/// invalidating the catch-up position recorded for that index, leaving a stale +/// position behind. Both must refuse the table. +pub const FLAG_MEM_WAL_INDEX_CATCHUP: u64 = 128; /// The first bit that is unknown as a feature flag -pub const FLAG_UNKNOWN: u64 = 64; +pub const FLAG_UNKNOWN: u64 = 256; + +// This build only understands flags below the unknown boundary, so a bit +// allocated at or above it would be refused by the very readers meant to use it. +const _: () = assert!(FLAG_MEM_WAL_INDEX_CATCHUP < FLAG_UNKNOWN); + +/// Environment variable that opts a release build into reading and writing data +/// overlay files before the feature is generally released. +pub const ENABLE_UNSTABLE_DATA_OVERLAY_FILES_ENV: &str = "LANCE_ENABLE_UNSTABLE_DATA_OVERLAY_FILES"; /// Set the reader and writer feature flags in the manifest based on the contents of the manifest. pub fn apply_feature_flags( @@ -29,6 +56,26 @@ pub fn apply_feature_flags( enable_stable_row_id: bool, disable_transaction_file: bool, ) -> Result<()> { + // Carried across the reset. This bit is not derivable from the manifest -- + // it depends on the `__lance_mem_wal` index details, which a manifest only + // points at -- and this function runs twice per commit: once in + // `build_manifest` and again in `write_manifest_file`. Dropping it here + // would clear it immediately before the write, so an activated table would + // report success and stay legacy. + // + // Only a consistent state carries: one bit set is neither mode, and + // `inherit_mem_wal_index_catchup` refuses it at the boundary where a + // manifest is derived from another. Reaching here half-set means the + // manifest was already written that way, so leave it for the reader check + // rather than silently completing it. + let mem_wal_index_catchup = if manifest.reader_feature_flags & FLAG_MEM_WAL_INDEX_CATCHUP != 0 + && manifest.writer_feature_flags & FLAG_MEM_WAL_INDEX_CATCHUP != 0 + { + FLAG_MEM_WAL_INDEX_CATCHUP + } else { + 0 + }; + // Reset flags manifest.reader_feature_flags = 0; manifest.writer_feature_flags = 0; @@ -71,24 +118,118 @@ pub fn apply_feature_flags( manifest.writer_feature_flags |= FLAG_BASE_PATHS; } + // Overlay files change cell values on read, so a reader that ignores them + // would return stale base values. Both readers and writers must understand + // them. + let has_overlays = manifest + .fragments + .iter() + .any(|frag| !frag.overlays.is_empty()); + if has_overlays { + manifest.reader_feature_flags |= FLAG_UNSTABLE_DATA_OVERLAY_FILES; + manifest.writer_feature_flags |= FLAG_UNSTABLE_DATA_OVERLAY_FILES; + } + if disable_transaction_file { manifest.writer_feature_flags |= FLAG_DISABLE_TRANSACTION_FILE; } + + manifest.reader_feature_flags |= mem_wal_index_catchup; + manifest.writer_feature_flags |= mem_wal_index_catchup; + Ok(()) } +/// Carry [`FLAG_MEM_WAL_INDEX_CATCHUP`] from the manifest a new one is derived +/// from. +/// +/// [`apply_feature_flags`] carries this bit across its own reset, but it only +/// ever sees one manifest. It cannot help where a *new* manifest is derived from +/// an existing one -- `Manifest::new_from_previous` and `shallow_clone` both +/// zero the feature words -- because the destination starts with nothing to +/// carry. That transition is this function's job. +/// +/// A half-set state is refused rather than normalized: one bit set means a +/// legacy reader or a legacy writer is still permitted, which is neither mode. +pub fn inherit_mem_wal_index_catchup(destination: &mut Manifest, source: &Manifest) -> Result<()> { + let reader = source.reader_feature_flags & FLAG_MEM_WAL_INDEX_CATCHUP != 0; + let writer = source.writer_feature_flags & FLAG_MEM_WAL_INDEX_CATCHUP != 0; + match (reader, writer) { + (false, false) => Ok(()), + (true, true) => { + destination.reader_feature_flags |= FLAG_MEM_WAL_INDEX_CATCHUP; + destination.writer_feature_flags |= FLAG_MEM_WAL_INDEX_CATCHUP; + Ok(()) + } + _ => Err(Error::invalid_input( + "Manifest has only one of the MemWAL index-catchup reader and writer \ + feature bits set, so its catch-up semantics are undefined", + )), + } +} + +/// Whether this build understands data overlay files: always in debug builds, +/// and in release builds only when [`ENABLE_UNSTABLE_DATA_OVERLAY_FILES_ENV`] is set. +fn data_overlay_files_enabled() -> bool { + cfg!(debug_assertions) || std::env::var_os(ENABLE_UNSTABLE_DATA_OVERLAY_FILES_ENV).is_some() +} + +/// Clear `flag` from `flags` when its gating feature is not enabled in this +/// build; leave it set otherwise. One call per unstable flag, so support for +/// several unstable features chains cleanly. +fn mark_supported(flags: &mut u64, flag: u64, feature_enabled: bool) { + if !feature_enabled { + *flags &= !flag; + } +} + +/// The feature-flag bits this build understands, given whether overlay support +/// is enabled. Split out from [`supported_flags`] so the policy is testable +/// without toggling the build profile or environment. +fn supported_flags_when(overlay_enabled: bool) -> u64 { + let mut supported = FLAG_UNKNOWN - 1; + mark_supported( + &mut supported, + FLAG_UNSTABLE_DATA_OVERLAY_FILES, + overlay_enabled, + ); + supported +} + +fn supported_flags() -> u64 { + supported_flags_when(data_overlay_files_enabled()) +} + pub fn can_read_dataset(reader_flags: u64) -> bool { - reader_flags < FLAG_UNKNOWN + reader_flags & !supported_flags() == 0 } pub fn can_write_dataset(writer_flags: u64) -> bool { - writer_flags < FLAG_UNKNOWN + writer_flags & !supported_flags() == 0 } pub fn has_deprecated_v2_feature_flag(writer_flags: u64) -> bool { writer_flags & FLAG_USE_V2_FORMAT_DEPRECATED != 0 } +/// Refuse a manifest whose MemWAL index-catchup bits disagree. +/// +/// One word set and the other not is neither mode: it would let a legacy reader +/// or a legacy writer through on a table where the other half is enforcing. The +/// commit path refuses to *produce* this, so seeing it on read means the +/// manifest was written by something that did not. +pub fn validate_mem_wal_index_catchup_flags(manifest: &Manifest) -> Result<()> { + let reader = manifest.reader_feature_flags & FLAG_MEM_WAL_INDEX_CATCHUP != 0; + let writer = manifest.writer_feature_flags & FLAG_MEM_WAL_INDEX_CATCHUP != 0; + if reader != writer { + return Err(Error::invalid_input( + "Manifest has only one of the MemWAL index-catchup reader and writer \ + feature bits set, so its catch-up semantics are undefined", + )); + } + Ok(()) +} + #[cfg(test)] mod tests { use super::*; @@ -103,6 +244,13 @@ mod tests { assert!(can_read_dataset(super::FLAG_TABLE_CONFIG)); assert!(can_read_dataset(super::FLAG_BASE_PATHS)); assert!(can_read_dataset(super::FLAG_DISABLE_TRANSACTION_FILE)); + // Overlay support is gated on the build profile / env opt-in, so the + // flag is readable exactly when overlays are enabled (see + // test_data_overlay_flag_release_gating for the full policy). + assert_eq!( + can_read_dataset(super::FLAG_UNSTABLE_DATA_OVERLAY_FILES), + data_overlay_files_enabled() + ); assert!(can_read_dataset( super::FLAG_DELETION_FILES | super::FLAG_STABLE_ROW_IDS @@ -111,6 +259,58 @@ mod tests { assert!(!can_read_dataset(super::FLAG_UNKNOWN)); } + #[test] + fn test_data_overlay_flag_release_gating() { + // Release default (overlays disabled): the overlay flag is treated as + // unknown so the dataset is refused, while other known flags still pass. + let supported = supported_flags_when(false); + assert_eq!(supported & FLAG_UNSTABLE_DATA_OVERLAY_FILES, 0); + assert_eq!(FLAG_DELETION_FILES & !supported, 0); + assert_ne!(FLAG_UNSTABLE_DATA_OVERLAY_FILES & !supported, 0); + // Enabled (debug or env opt-in): the overlay flag is understood. + let supported = supported_flags_when(true); + assert_eq!(FLAG_UNSTABLE_DATA_OVERLAY_FILES & !supported, 0); + } + + #[test] + fn test_apply_feature_flags_sets_overlay_flag() { + use crate::format::overlay::{DataOverlayFile, OverlayCoverage}; + use crate::format::{DataFile, DataStorageFormat, Fragment}; + use arrow_schema::{Field as ArrowField, Schema as ArrowSchema}; + use lance_core::datatypes::Schema; + use roaring::RoaringBitmap; + use std::collections::HashMap; + use std::sync::Arc; + + let arrow_schema = ArrowSchema::new(vec![ArrowField::new( + "id", + arrow_schema::DataType::Int64, + false, + )]); + let schema = Schema::try_from(&arrow_schema).unwrap(); + let mut fragment = Fragment::new(0); + fragment.overlays = vec![DataOverlayFile { + data_file: DataFile::new_legacy_from_fields("o.lance", vec![0], None), + coverage: OverlayCoverage::dense(RoaringBitmap::from_iter([0u32])), + committed_version: 1, + }]; + let mut manifest = Manifest::new( + schema, + Arc::new(vec![fragment]), + DataStorageFormat::default(), + HashMap::new(), + ); + apply_feature_flags(&mut manifest, false, false).unwrap(); + assert_ne!( + manifest.reader_feature_flags & FLAG_UNSTABLE_DATA_OVERLAY_FILES, + 0 + ); + assert_ne!( + manifest.writer_feature_flags & FLAG_UNSTABLE_DATA_OVERLAY_FILES, + 0 + ); + } + #[test] fn test_write_check() { assert!(can_write_dataset(0)); @@ -120,6 +320,13 @@ mod tests { assert!(can_write_dataset(super::FLAG_TABLE_CONFIG)); assert!(can_write_dataset(super::FLAG_BASE_PATHS)); assert!(can_write_dataset(super::FLAG_DISABLE_TRANSACTION_FILE)); + // Overlay support is gated on the build profile / env opt-in, so the + // flag is writable exactly when overlays are enabled (see + // test_data_overlay_flag_release_gating for the full policy). + assert_eq!( + can_write_dataset(super::FLAG_UNSTABLE_DATA_OVERLAY_FILES), + data_overlay_files_enabled() + ); assert!(can_write_dataset( super::FLAG_DELETION_FILES | super::FLAG_STABLE_ROW_IDS @@ -181,4 +388,112 @@ mod tests { 0 ); } + + /// The MemWAL bit depends on the `__lance_mem_wal` index details, which a + /// manifest cannot see — it holds only a byte offset to its index section. + /// So an unrelated recomputation must preserve the bit rather than derive + /// it, or a later transaction would silently downgrade the table to legacy + /// semantics and a reader would treat missing coverage as complete. + #[test] + fn inheriting_carries_the_mem_wal_bit_from_the_source() { + let mut source = empty_manifest(); + source.reader_feature_flags = FLAG_MEM_WAL_INDEX_CATCHUP; + source.writer_feature_flags = FLAG_MEM_WAL_INDEX_CATCHUP; + // What `Manifest::new_from_previous` hands us: both words zeroed. + let mut destination = empty_manifest(); + + inherit_mem_wal_index_catchup(&mut destination, &source).unwrap(); + + assert_ne!( + destination.reader_feature_flags & FLAG_MEM_WAL_INDEX_CATCHUP, + 0 + ); + assert_ne!( + destination.writer_feature_flags & FLAG_MEM_WAL_INDEX_CATCHUP, + 0 + ); + } + + #[test] + fn inheriting_refuses_a_half_set_source() { + for (reader, writer) in [ + (FLAG_MEM_WAL_INDEX_CATCHUP, 0), + (0, FLAG_MEM_WAL_INDEX_CATCHUP), + ] { + let mut source = empty_manifest(); + source.reader_feature_flags = reader; + source.writer_feature_flags = writer; + let mut destination = empty_manifest(); + + let err = inherit_mem_wal_index_catchup(&mut destination, &source).unwrap_err(); + + assert!(err.to_string().contains("only one of"), "{err}"); + } + } + + #[test] + fn apply_feature_flags_carries_the_mem_wal_bit_across_its_reset() { + // It runs twice per commit -- `build_manifest` and `write_manifest_file` + // -- so dropping the bit here would clear it immediately before the + // write, and an activated table would report success and stay legacy. + let mut manifest = empty_manifest(); + manifest.reader_feature_flags = FLAG_MEM_WAL_INDEX_CATCHUP; + manifest.writer_feature_flags = FLAG_MEM_WAL_INDEX_CATCHUP; + + apply_feature_flags(&mut manifest, false, false).unwrap(); + + assert_ne!( + manifest.reader_feature_flags & FLAG_MEM_WAL_INDEX_CATCHUP, + 0 + ); + assert_ne!( + manifest.writer_feature_flags & FLAG_MEM_WAL_INDEX_CATCHUP, + 0 + ); + } + + #[test] + fn apply_feature_flags_drops_a_half_set_mem_wal_bit() { + // Neither mode, so leave it for the reader check rather than completing it. + let mut manifest = empty_manifest(); + manifest.reader_feature_flags = FLAG_MEM_WAL_INDEX_CATCHUP; + + apply_feature_flags(&mut manifest, false, false).unwrap(); + + assert_eq!( + manifest.reader_feature_flags & FLAG_MEM_WAL_INDEX_CATCHUP, + 0 + ); + assert_eq!( + manifest.writer_feature_flags & FLAG_MEM_WAL_INDEX_CATCHUP, + 0 + ); + } + + fn empty_manifest() -> Manifest { + use crate::format::DataStorageFormat; + use arrow_schema::{DataType, Field as ArrowField, Schema as ArrowSchema}; + use lance_core::datatypes::Schema; + use std::collections::HashMap; + use std::sync::Arc; + + let arrow_schema = ArrowSchema::new(vec![ArrowField::new("i", DataType::Int32, false)]); + Manifest::new( + Schema::try_from(&arrow_schema).unwrap(), + Arc::new(vec![]), + DataStorageFormat::default(), + HashMap::new(), + ) + } + + /// A build that does not know the bit must refuse the table rather than + /// continue with legacy semantics. + #[test] + fn the_mem_wal_bit_is_below_the_unknown_boundary() { + assert!(can_read_dataset(FLAG_MEM_WAL_INDEX_CATCHUP)); + assert!(can_write_dataset(FLAG_MEM_WAL_INDEX_CATCHUP)); + // The next bit up is still unknown, so allocating this one did not + // silently widen what this build claims to understand. + assert!(!can_read_dataset(FLAG_UNKNOWN)); + } } diff --git a/rust/lance-table/src/format.rs b/rust/lance-table/src/format.rs index 842c76f1e58..99763c2dab4 100644 --- a/rust/lance-table/src/format.rs +++ b/rust/lance-table/src/format.rs @@ -7,6 +7,8 @@ use uuid::Uuid; mod fragment; mod index; mod manifest; +pub mod overlay; +mod row_ids; mod transaction; pub use crate::rowids::version::{ @@ -17,8 +19,9 @@ pub use index::{IndexFile, IndexMetadata, index_metadata_codec, list_index_files pub use manifest::{ BasePath, DETACHED_VERSION_MASK, DataStorageFormat, Manifest, SelfDescribingFileReader, - WriterVersion, is_detached_version, + WriterVersion, is_detached_version, populate_manifest_schema_dictionaries, }; +pub use row_ids::{ExternalFile, InlineRowIds, RowIdMeta}; pub use transaction::Transaction; use lance_core::{Error, Result}; diff --git a/rust/lance-table/src/format/fragment.rs b/rust/lance-table/src/format/fragment.rs index 431e466dbd4..149a9b44fc6 100644 --- a/rust/lance-table/src/format/fragment.rs +++ b/rust/lance-table/src/format/fragment.rs @@ -7,12 +7,13 @@ use std::sync::Arc; use lance_core::Error; use lance_core::deepsize::DeepSizeOf; -use lance_file::format::{MAJOR_VERSION, MINOR_VERSION}; -use lance_file::version::LanceFileVersion; +use lance_file::version::ConcreteFileVersion; use lance_io::utils::CachedFileSize; use object_store::path::Path; use serde::{Deserialize, Deserializer, Serialize, Serializer}; +use super::overlay::{DataOverlayFile, TOMBSTONE_FIELD_ID, sort_overlays_newest_last}; +use super::row_ids::{ExternalFile, RowIdMeta}; use crate::format::pb; use crate::rowids::version::{ @@ -36,7 +37,7 @@ pub struct DataFile { pub fields: Arc<[i32]>, /// The offsets of the fields listed in `fields`, empty in v1 files /// - /// Note that -1 is a possibility and it indices that the field has + /// Note that -1 is a possibility and it indicates that the field has /// no top-level column in the file. /// /// Columns that lack a field id may still exist as extra entries in @@ -103,15 +104,16 @@ impl<'de> Deserialize<'de> for DataFile { } impl DataFile { + /// Create a `DataFile` and encode its exact format version for manifest storage. pub fn new( path: impl Into, fields: Vec, column_indices: Vec, - file_major_version: u32, - file_minor_version: u32, + file_version: ConcreteFileVersion, file_size_bytes: Option>, base_id: Option, ) -> Self { + let (file_major_version, file_minor_version) = file_version.to_data_file_numbers(); Self { path: path.into(), fields: Arc::from(fields), @@ -123,12 +125,9 @@ impl DataFile { } } - /// Create a new `DataFile` with the expectation that fields and column_indices will be set later - pub fn new_unstarted( - path: impl Into, - file_major_version: u32, - file_minor_version: u32, - ) -> Self { + /// Create a new `DataFile` whose fields and column indices will be set later. + pub fn new_unstarted(path: impl Into, file_version: ConcreteFileVersion) -> Self { + let (file_major_version, file_minor_version) = file_version.to_data_file_numbers(); Self { path: path.into(), fields: Arc::from([]), @@ -145,15 +144,7 @@ impl DataFile { fields: Vec, base_id: Option, ) -> Self { - Self::new( - path, - fields, - vec![], - MAJOR_VERSION as u32, - MINOR_VERSION as u32, - None, - base_id, - ) + Self::new(path, fields, vec![], ConcreteFileVersion::V1, None, base_id) } pub fn new_legacy( @@ -168,8 +159,7 @@ impl DataFile { path, field_ids, vec![], - MAJOR_VERSION as u32, - MINOR_VERSION as u32, + ConcreteFileVersion::V1, file_size_bytes, base_id, ) @@ -179,13 +169,30 @@ impl DataFile { full_schema.project_by_ids(&self.fields, false) } - pub fn is_legacy_file(&self) -> bool { + fn uses_v1_data_file_encoding(&self) -> bool { self.file_major_version == 0 && self.file_minor_version < 3 } + /// Decode the exact file version stored in this `DataFile` metadata. + pub fn file_version(&self) -> Result { + ConcreteFileVersion::from_data_file_numbers( + self.file_major_version, + self.file_minor_version, + ) + } + pub fn validate(&self, base_path: &Path) -> Result<()> { - if self.is_legacy_file() { - if !self.fields.windows(2).all(|w| w[0] < w[1]) { + if self.uses_v1_data_file_encoding() { + // A tombstone marks a field superseded by a later data file. It is + // not a field id, so it carries no ordering; the live ids around it + // must still be sorted and distinct. + let live: Vec = self + .fields + .iter() + .copied() + .filter(|field| *field != TOMBSTONE_FIELD_ID) + .collect(); + if !live.windows(2).all(|w| w[0] < w[1]) { return Err(Error::corrupt_file( base_path.clone().join(self.path.clone()), "contained unsorted or duplicate field ids", @@ -375,6 +382,15 @@ impl DataFileFieldInterner { .into_iter() .map(|f| self.intern_data_file(f)) .collect::>()?, + overlays: { + let mut overlays = p + .overlays + .into_iter() + .map(DataOverlayFile::try_from) + .collect::>>()?; + sort_overlays_newest_last(&mut overlays); + overlays + }, deletion_file: p.deletion_file.map(DeletionFile::try_from).transpose()?, row_id_meta: p.row_id_sequence.map(RowIdMeta::try_from).transpose()?, physical_rows, @@ -439,38 +455,6 @@ impl TryFrom for DeletionFile { } } -/// A reference to a part of a file. -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, DeepSizeOf)] -pub struct ExternalFile { - pub path: String, - pub offset: u64, - pub size: u64, -} - -/// Metadata about location of the row id sequence. -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, DeepSizeOf)] -pub enum RowIdMeta { - Inline(Vec), - External(ExternalFile), -} - -impl TryFrom for RowIdMeta { - type Error = Error; - - fn try_from(value: pb::data_fragment::RowIdSequence) -> Result { - match value { - pb::data_fragment::RowIdSequence::InlineRowIds(data) => Ok(Self::Inline(data)), - pb::data_fragment::RowIdSequence::ExternalRowIds(file) => { - Ok(Self::External(ExternalFile { - path: file.path.clone(), - offset: file.offset, - size: file.size, - })) - } - } - } -} - /// Data fragment. /// /// A fragment is a set of files which represent the different columns of the same rows. @@ -483,6 +467,12 @@ pub struct Fragment { /// Files within the fragment. pub files: Vec, + /// Overlay files supplying new values for a subset of cells without + /// rewriting the base data files. Order is significant: a later entry is + /// newer than an earlier one. See [`DataOverlayFile`] for resolution rules. + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub overlays: Vec, + /// Optional file with deleted local row offsets. #[serde(skip_serializing_if = "Option::is_none")] pub deletion_file: Option, @@ -510,6 +500,7 @@ impl Fragment { Self { id, files: vec![], + overlays: vec![], deletion_file: None, row_id_meta: None, physical_rows: None, @@ -534,6 +525,51 @@ impl Fragment { } } + /// Every Lance-format file this fragment references: the base data files + /// plus the data file of each overlay. The fragment's other referenced + /// files (deletion files, external row-id files) are not in this format. + /// + /// Prefer this over `files`, which is the base data files only and so omits + /// overlays. + pub fn referenced_lance_files(&self) -> impl Iterator + '_ { + // Destructured on purpose: a new field here fails to compile until + // someone decides whether it references files. + let Self { + id: _, + files, + overlays, + deletion_file: _, + row_id_meta: _, + physical_rows: _, + last_updated_at_version_meta: _, + created_at_version_meta: _, + } = self; + files + .iter() + .chain(overlays.iter().map(|overlay| &overlay.data_file)) + } + + /// Mutable counterpart of [`Self::referenced_lance_files`], for rewriting + /// the fields a clone has to normalize (`base_id`) across base and overlay + /// files alike. + pub fn referenced_lance_files_mut(&mut self) -> impl Iterator + '_ { + // Destructured for the same reason as `referenced_lance_files`, and so + // the two disjoint field borrows are visible to the borrow checker. + let Self { + id: _, + files, + overlays, + deletion_file: _, + row_id_meta: _, + physical_rows: _, + last_updated_at_version_meta: _, + created_at_version_meta: _, + } = self; + files + .iter_mut() + .chain(overlays.iter_mut().map(|overlay| &mut overlay.data_file)) + } + pub fn from_json(json: &str) -> Result { let fragment: Self = serde_json::from_str(json)?; Ok(fragment) @@ -549,6 +585,7 @@ impl Fragment { Self { id, files: vec![DataFile::new_legacy(path, schema, None, None)], + overlays: vec![], deletion_file: None, physical_rows, row_id_meta: None, @@ -562,16 +599,14 @@ impl Fragment { path: impl Into, field_ids: Vec, column_indices: Vec, - version: &LanceFileVersion, + version: ConcreteFileVersion, file_size_bytes: Option>, ) -> Self { - let (major, minor) = version.to_numbers(); let data_file = DataFile::new( path, field_ids, column_indices, - major, - minor, + version, file_size_bytes, None, ); @@ -589,16 +624,14 @@ impl Fragment { path: impl Into, field_ids: Vec, column_indices: Vec, - version: &LanceFileVersion, + version: ConcreteFileVersion, file_size_bytes: Option>, ) { - let (major, minor) = version.to_numbers(); self.files.push(DataFile::new( path, field_ids, column_indices, - major, - minor, + version, file_size_bytes, None, )); @@ -610,17 +643,11 @@ impl Fragment { .push(DataFile::new_legacy(path, schema, None, None)); } - // True if this fragment is made up of legacy v1 files, false otherwise - pub fn has_legacy_files(&self) -> bool { - // If any file in a fragment is legacy then all files in the fragment must be - self.files[0].is_legacy_file() - } - // Helper method to infer the Lance version from a set of fragments // // Returns None if there are no data files // Returns an error if the data files have different versions - pub fn try_infer_version(fragments: &[Self]) -> Result> { + pub fn try_infer_version(fragments: &[Self]) -> Result> { // Otherwise we need to check the actual file versions // Determine version from first file let Some(sample_file) = fragments @@ -630,17 +657,11 @@ impl Fragment { else { return Ok(None); }; - let file_version = LanceFileVersion::try_from_major_minor( - sample_file.file_major_version, - sample_file.file_minor_version, - )?; + let file_version = sample_file.file_version()?; // Ensure all files match for frag in fragments { for file in &frag.files { - let this_file_version = LanceFileVersion::try_from_major_minor( - file.file_major_version, - file.file_minor_version, - )?; + let this_file_version = file.file_version()?; if file_version != this_file_version { return Err(Error::invalid_input(format!( "All data files must have the same version. Detected both {} and {}", @@ -669,6 +690,15 @@ impl TryFrom for Fragment { .into_iter() .map(DataFile::try_from) .collect::>()?, + overlays: { + let mut overlays = p + .overlays + .into_iter() + .map(DataOverlayFile::try_from) + .collect::>>()?; + sort_overlays_newest_last(&mut overlays); + overlays + }, deletion_file: p.deletion_file.map(DeletionFile::try_from).transpose()?, row_id_meta: p.row_id_sequence.map(RowIdMeta::try_from).transpose()?, physical_rows, @@ -701,7 +731,9 @@ impl From<&Fragment> for pb::DataFragment { }); let row_id_sequence = f.row_id_meta.as_ref().map(|m| match m { - RowIdMeta::Inline(data) => pb::data_fragment::RowIdSequence::InlineRowIds(data.clone()), + RowIdMeta::Inline(data) => { + pb::data_fragment::RowIdSequence::InlineRowIds(data.to_vec()) + } RowIdMeta::External(file) => { pb::data_fragment::RowIdSequence::ExternalRowIds(pb::ExternalFile { path: file.path.clone(), @@ -716,6 +748,7 @@ impl From<&Fragment> for pb::DataFragment { Self { id: f.id, files: f.files.iter().map(pb::DataFile::from).collect(), + overlays: f.overlays.iter().map(pb::DataOverlayFile::from).collect(), deletion_file, row_id_sequence, physical_rows: f.physical_rows.unwrap_or_default() as u64, @@ -728,12 +761,109 @@ impl From<&Fragment> for pb::DataFragment { #[cfg(test)] mod tests { use super::*; + use crate::format::overlay::OverlayCoverage; use arrow_schema::{ DataType, Field as ArrowField, Fields as ArrowFields, Schema as ArrowSchema, }; + use lance_file::format::{MAJOR_VERSION, MINOR_VERSION}; use object_store::path::Path; + use roaring::RoaringBitmap; use serde_json::{Value, json}; + #[test] + fn test_data_overlay_roundtrip() { + // A fragment carrying a dense overlay round-trips through protobuf and + // back, and the parsed coverage bitmap is recovered per field. + let mut bitmap = RoaringBitmap::new(); + bitmap.insert(1); + bitmap.insert(3); + + let overlay = DataOverlayFile { + data_file: DataFile::new_legacy_from_fields("overlay-0.lance", vec![3], None), + coverage: OverlayCoverage::dense(bitmap.clone()), + committed_version: 7, + }; + let mut fragment = Fragment::new(0); + fragment.files = vec![DataFile::new_legacy_from_fields( + "base.lance", + vec![1, 3], + None, + )]; + fragment.overlays = vec![overlay]; + + let proto = pb::DataFragment::from(&fragment); + assert_eq!(proto.overlays.len(), 1); + let round_tripped = Fragment::try_from(proto).unwrap(); + assert_eq!(round_tripped, fragment); + + // Dense coverage applies to every field. + let recovered = round_tripped.overlays[0].coverage_for_field(0).unwrap(); + assert_eq!(*recovered, bitmap); + assert_eq!( + *round_tripped.overlays[0].coverage_for_field(5).unwrap(), + bitmap + ); + } + + #[test] + fn test_data_overlay_sparse_per_field_coverage() { + // A sparse overlay carries one bitmap per field, recovered by position. + let name_coverage = RoaringBitmap::from_iter([2u32, 3]); + let embedding_coverage = RoaringBitmap::from_iter([1u32]); + let overlay = DataOverlayFile { + data_file: DataFile::new_legacy_from_fields("overlay-1.lance", vec![2, 4], None), + coverage: OverlayCoverage::sparse(vec![ + name_coverage.clone(), + embedding_coverage.clone(), + ]), + committed_version: 3, + }; + let mut fragment = Fragment::new(1); + fragment.overlays = vec![overlay]; + + let round_tripped = Fragment::try_from(pb::DataFragment::from(&fragment)).unwrap(); + assert_eq!( + *round_tripped.overlays[0].coverage_for_field(0).unwrap(), + name_coverage + ); + assert_eq!( + *round_tripped.overlays[0].coverage_for_field(1).unwrap(), + embedding_coverage + ); + } + + #[test] + fn test_overlays_sorted_newest_last_on_load() { + // Overlays load stable-sorted by committed_version (newest last), with + // list position preserved as the tiebreak for equal versions. + let mk = |version: u64, field: i32| DataOverlayFile { + data_file: DataFile::new_legacy_from_fields("o.lance", vec![field], None), + coverage: OverlayCoverage::dense(RoaringBitmap::from_iter([0u32])), + committed_version: version, + }; + let mut fragment = Fragment::new(0); + // Written out of order: v5, v2, v2 (second), v3. + fragment.overlays = vec![mk(5, 1), mk(2, 2), mk(2, 3), mk(3, 4)]; + + let loaded = Fragment::try_from(pb::DataFragment::from(&fragment)).unwrap(); + let versions: Vec = loaded + .overlays + .iter() + .map(|o| o.committed_version) + .collect(); + assert_eq!(versions, vec![2, 2, 3, 5]); + // Stable: the two v2 overlays keep their original relative order (field 2 + // before field 3). + assert_eq!( + loaded.overlays[0].data_file.fields.as_ref(), + [2i32].as_slice() + ); + assert_eq!( + loaded.overlays[1].data_file.fields.as_ref(), + [3i32].as_slice() + ); + } + #[test] fn test_new_fragment() { let path = "foobar.lance"; @@ -786,6 +916,56 @@ mod tests { assert_eq!(fragment, fragment2); } + #[test] + fn infer_exact_file_version_and_reject_mixed_fragments() { + assert_eq!(Fragment::try_infer_version(&[]).unwrap(), None); + + let v2_0 = Fragment::new(0).with_file( + "v2_0.lance", + vec![0], + vec![0], + ConcreteFileVersion::V2_0, + None, + ); + assert_eq!( + ( + v2_0.files[0].file_major_version, + v2_0.files[0].file_minor_version + ), + (2, 0) + ); + let unstarted = DataFile::new_unstarted("unstarted.lance", ConcreteFileVersion::V2_0); + assert_eq!( + (unstarted.file_major_version, unstarted.file_minor_version), + (2, 0) + ); + let v2_0_second = Fragment::new(1).with_file( + "v2_0_second.lance", + vec![0], + vec![0], + ConcreteFileVersion::V2_0, + None, + ); + assert_eq!( + Fragment::try_infer_version(&[v2_0.clone(), v2_0_second]).unwrap(), + Some(ConcreteFileVersion::V2_0) + ); + + let v2_1 = Fragment::new(2).with_file( + "v2_1.lance", + vec![0], + vec![0], + ConcreteFileVersion::V2_1, + None, + ); + let error = Fragment::try_infer_version(&[v2_0, v2_1]).unwrap_err(); + assert!(matches!(error, Error::InvalidInput { .. })); + let message = error.to_string(); + assert!(message.contains("All data files must have the same version")); + assert!(message.contains("2.0")); + assert!(message.contains("2.1")); + } + #[test] fn test_to_json() { let mut fragment = Fragment::new(123); diff --git a/rust/lance-table/src/format/manifest.rs b/rust/lance-table/src/format/manifest.rs index 9845061b7e4..61d401fee96 100644 --- a/rust/lance-table/src/format/manifest.rs +++ b/rust/lance-table/src/format/manifest.rs @@ -4,9 +4,11 @@ use async_trait::async_trait; use chrono::prelude::*; use lance_core::deepsize::DeepSizeOf; -use lance_file::datatypes::{Fields, FieldsWithMeta, populate_schema_dictionary}; -use lance_file::previous::reader::FileReader as PreviousFileReader; -use lance_file::version::{LEGACY_FORMAT_VERSION, LanceFileVersion}; +use lance_file::datatypes::{Fields, FieldsWithMeta}; +use lance_file::version::{ConcreteFileVersion, stable_file_version}; +use lance_file::versions::v1::{ + encoding::populate_schema_dictionaries, reader::FileReader as V1FileReader, +}; use lance_io::traits::{ProtoStruct, Reader}; use object_store::path::Path; use prost::Message; @@ -16,6 +18,7 @@ use std::ops::Range; use std::sync::Arc; use super::Fragment; +use crate::feature_flags::FLAG_MEM_WAL_INDEX_CATCHUP; use crate::feature_flags::{FLAG_STABLE_ROW_IDS, has_deprecated_v2_feature_flag}; use crate::format::fragment::DataFileFieldInterner; use crate::format::pb; @@ -248,7 +251,7 @@ impl Manifest { .iter() .map(|fragment| { let mut cloned_fragment = fragment.clone(); - for file in &mut cloned_fragment.files { + for file in cloned_fragment.referenced_lance_files_mut() { if file.base_id.is_none() { file.base_id = Some(ref_base_id); } @@ -273,8 +276,11 @@ impl Manifest { index_section: None, // These will be set on commit timestamp_nanos: self.timestamp_nanos, tag: None, - reader_feature_flags: 0, // These will be set on commit - writer_feature_flags: 0, // These will be set on commit + // Not derivable from the manifest, so it would be lost like any + // other zeroed word -- and a clone of a table that requires index + // catch-up would silently come back as legacy. + reader_feature_flags: self.reader_feature_flags & FLAG_MEM_WAL_INDEX_CATCHUP, + writer_feature_flags: self.writer_feature_flags & FLAG_MEM_WAL_INDEX_CATCHUP, max_fragment_id: self.max_fragment_id, transaction_file: Some(transaction_file), transaction_section: None, @@ -501,10 +507,6 @@ impl Manifest { pb_manifest.encode_to_vec() } - pub fn should_use_legacy_format(&self) -> bool { - self.data_storage_format.version == LEGACY_FORMAT_VERSION - } - /// Get the summary information of a manifest. /// /// This function calculates various statistics about the manifest, including: @@ -552,6 +554,40 @@ impl Manifest { } } +/// Populate dictionary values stored outside a V1 manifest. +/// +/// Other exact file versions store their schema dictionaries inline, so this +/// is a no-op for those manifests. +/// +/// # Examples +/// +/// ``` +/// # use lance_core::Result; +/// # use lance_io::traits::Reader; +/// # use lance_table::format::{Manifest, populate_manifest_schema_dictionaries}; +/// # async fn hydrate_v1_manifest( +/// # manifest: &mut Manifest, +/// # reader: &dyn Reader, +/// # ) -> Result<()> { +/// populate_manifest_schema_dictionaries(manifest, reader).await +/// # } +/// ``` +pub async fn populate_manifest_schema_dictionaries( + manifest: &mut Manifest, + reader: &dyn Reader, +) -> Result<()> { + match manifest.data_storage_format.version { + ConcreteFileVersion::V1 => { + populate_schema_dictionaries(&mut manifest.schema, reader).await?; + } + ConcreteFileVersion::V2_0 + | ConcreteFileVersion::V2_1 + | ConcreteFileVersion::V2_2 + | ConcreteFileVersion::V2_3 => {} + } + Ok(()) +} + #[derive(Debug, Clone, PartialEq)] pub struct BasePath { pub id: u32, @@ -606,36 +642,39 @@ pub struct WriterVersion { #[derive(Debug, Clone, PartialEq, DeepSizeOf)] pub struct DataStorageFormat { pub file_format: String, - pub version: String, + pub version: ConcreteFileVersion, } const LANCE_FORMAT_NAME: &str = "lance"; impl DataStorageFormat { - pub fn new(version: LanceFileVersion) -> Self { + pub fn new(version: ConcreteFileVersion) -> Self { Self { file_format: LANCE_FORMAT_NAME.to_string(), - version: version.resolve().to_string(), + version, } } - pub fn lance_file_version(&self) -> Result { - self.version.parse::() + /// Return the exact file format version persisted by this manifest. + pub fn lance_file_format(&self) -> ConcreteFileVersion { + self.version } } impl Default for DataStorageFormat { fn default() -> Self { - Self::new(LanceFileVersion::default()) + Self::new(stable_file_version()) } } -impl From for DataStorageFormat { - fn from(pb: pb::manifest::DataStorageFormat) -> Self { - Self { +impl TryFrom for DataStorageFormat { + type Error = Error; + + fn try_from(pb: pb::manifest::DataStorageFormat) -> Result { + Ok(Self { file_format: pb.file_format, - version: pb.version, - } + version: ConcreteFileVersion::from_manifest_string(&pb.version)?, + }) } } @@ -891,16 +930,16 @@ impl TryFrom for Manifest { } else { // No fragments to inspect, best we can do is look at writer flags if has_deprecated_v2_feature_flag(p.writer_feature_flags) { - DataStorageFormat::new(LanceFileVersion::Stable) + DataStorageFormat::new(stable_file_version()) } else { - DataStorageFormat::new(LanceFileVersion::Legacy) + DataStorageFormat::new(ConcreteFileVersion::V1) } } } - Some(format) => DataStorageFormat::from(format), + Some(format) => DataStorageFormat::try_from(format)?, }; - let schema = Schema::from(fields_with_meta); + let schema = Schema::try_from(fields_with_meta)?; Ok(Self { schema, @@ -980,7 +1019,11 @@ impl From<&Manifest> for pb::Manifest { next_row_id: m.next_row_id, data_format: Some(pb::manifest::DataStorageFormat { file_format: m.data_storage_format.file_format.clone(), - version: m.data_storage_format.version.clone(), + version: m + .data_storage_format + .version + .to_manifest_string() + .to_string(), }), config: m.config.clone(), base_paths: m @@ -1028,7 +1071,7 @@ pub trait SelfDescribingFileReader { } #[async_trait] -impl SelfDescribingFileReader for PreviousFileReader { +impl SelfDescribingFileReader for V1FileReader { async fn try_new_self_described_from_reader( reader: Arc, cache: Option<&LanceCache>, @@ -1039,9 +1082,7 @@ impl SelfDescribingFileReader for PreviousFileReader { reader.path(), )))?; let mut manifest: Manifest = read_struct(reader.as_ref(), manifest_position).await?; - if manifest.should_use_legacy_format() { - populate_schema_dictionary(&mut manifest.schema, reader.as_ref()).await?; - } + populate_manifest_schema_dictionaries(&mut manifest, reader.as_ref()).await?; let schema = manifest.schema; let max_field_id = schema.max_field_id().unwrap_or_default(); Self::try_new_from_reader( @@ -1060,6 +1101,7 @@ impl SelfDescribingFileReader for PreviousFileReader { #[cfg(test)] mod tests { + use crate::feature_flags::FLAG_USE_V2_FORMAT_DEPRECATED; use crate::format::{DataFile, DeletionFile, DeletionFileType}; use std::num::NonZero; @@ -1068,6 +1110,146 @@ mod tests { use arrow_schema::{Field as ArrowField, Schema as ArrowSchema}; use lance_core::datatypes::Field; + /// A shallow clone points every local file at the parent through `base_id`. + /// An overlay's data file lives in the parent too, so it needs the same + /// stamp; without it the clone looks for the overlay under its own root. + #[test] + fn shallow_clone_stamps_base_id_on_overlay_files() { + use crate::format::overlay::{DataOverlayFile, OverlayCoverage}; + use roaring::RoaringBitmap; + + let arrow_schema = ArrowSchema::new(vec![ArrowField::new( + "a", + arrow_schema::DataType::Int64, + false, + )]); + let schema = Schema::try_from(&arrow_schema).unwrap(); + + let mut fragment = Fragment::with_file_legacy(0, "base.lance", &schema, Some(10)); + fragment.overlays = vec![DataOverlayFile { + data_file: DataFile::new_legacy_from_fields("overlay.lance", vec![0], None), + coverage: OverlayCoverage::Shared(Arc::new(RoaringBitmap::from_iter([0_u32]))), + committed_version: 1, + }]; + let manifest = Manifest::new( + schema, + Arc::new(vec![fragment]), + DataStorageFormat::default(), + HashMap::new(), + ); + + let cloned = manifest.shallow_clone( + Some("parent".to_string()), + "memory://parent".to_string(), + 7, + None, + String::new(), + ); + + let fragment = &cloned.fragments[0]; + assert_eq!(fragment.files[0].base_id, Some(7)); + assert_eq!( + fragment.overlays[0].data_file.base_id, + Some(7), + "the overlay's data file resolves against the parent as well" + ); + } + + #[test] + fn old_empty_manifest_recovers_v1_or_current_stable() { + let old_manifest = pb::Manifest { + data_format: None, + ..Default::default() + }; + let recovered_v1 = Manifest::try_from(old_manifest.clone()).unwrap(); + assert_eq!( + recovered_v1.data_storage_format.lance_file_format(), + ConcreteFileVersion::V1 + ); + + let recovered_stable = Manifest::try_from(pb::Manifest { + writer_feature_flags: FLAG_USE_V2_FORMAT_DEPRECATED, + ..old_manifest + }) + .unwrap(); + assert_eq!( + recovered_stable.data_storage_format.lance_file_format(), + stable_file_version() + ); + } + + #[test] + fn manifest_persistence_rejects_selectors_and_public_aliases() { + for version in ["stable", "next", "legacy", "0.3"] { + let manifest = pb::Manifest { + data_format: Some(pb::manifest::DataStorageFormat { + file_format: LANCE_FORMAT_NAME.to_string(), + version: version.to_string(), + }), + ..Default::default() + }; + assert!(Manifest::try_from(manifest).is_err(), "accepted {version}"); + } + } + + #[test] + fn manifest_codec_writes_canonical_exact_string() { + let manifest = Manifest::new( + Schema::default(), + Arc::new(Vec::new()), + DataStorageFormat::new(ConcreteFileVersion::V2_0), + HashMap::new(), + ); + let encoded = pb::Manifest::from(&manifest); + assert_eq!(encoded.data_format.unwrap().version, "2.0"); + } + + #[test] + fn missing_format_infers_exact_version_and_rejects_mixed_files() { + let v2_0 = Fragment::new(0).with_file( + "v2_0.lance", + vec![0], + vec![0], + ConcreteFileVersion::V2_0, + None, + ); + let manifest = Manifest::new( + Schema::default(), + Arc::new(vec![v2_0.clone()]), + DataStorageFormat::new(ConcreteFileVersion::V1), + HashMap::new(), + ); + let mut encoded = pb::Manifest::from(&manifest); + encoded.data_format = None; + let recovered = Manifest::try_from(encoded).unwrap(); + assert_eq!( + recovered.data_storage_format.lance_file_format(), + ConcreteFileVersion::V2_0 + ); + + let v2_1 = Fragment::new(1).with_file( + "v2_1.lance", + vec![0], + vec![0], + ConcreteFileVersion::V2_1, + None, + ); + let mixed_manifest = Manifest::new( + Schema::default(), + Arc::new(vec![v2_0, v2_1]), + DataStorageFormat::new(ConcreteFileVersion::V2_0), + HashMap::new(), + ); + let mut encoded = pb::Manifest::from(&mixed_manifest); + encoded.data_format = None; + let error = Manifest::try_from(encoded).unwrap_err(); + assert!( + error + .to_string() + .contains("All data files must have the same version") + ); + } + #[test] fn test_writer_version() { let wv = WriterVersion::default(); @@ -1316,6 +1498,7 @@ mod tests { vec![0, 1, 2], None, )], + overlays: vec![], deletion_file: None, row_id_meta: None, physical_rows: None, @@ -1328,6 +1511,7 @@ mod tests { DataFile::new_legacy_from_fields("path2", vec![0, 1, 43], None), DataFile::new_legacy_from_fields("path3", vec![2], None), ], + overlays: vec![], deletion_file: None, row_id_meta: None, physical_rows: None, @@ -1448,14 +1632,13 @@ mod tests { assert_eq!(real_data_summary.total_data_file_rows, 425); assert_eq!(real_data_summary.total_deletion_files, 0); - let file_version = LanceFileVersion::default(); // Step 4: write deletion files and verify summary let mut fragment_with_deletion = Fragment::new(0) .with_file( "data_with_deletion.lance", vec![0, 1], vec![0, 1], - &file_version, + stable_file_version(), NonZero::new(1000), ) .with_physical_rows(50); diff --git a/rust/lance-table/src/format/overlay.rs b/rust/lance-table/src/format/overlay.rs new file mode 100644 index 00000000000..5e729411502 --- /dev/null +++ b/rust/lance-table/src/format/overlay.rs @@ -0,0 +1,443 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright The Lance Authors + +//! Data overlay files. +//! +//! An overlay file supplies new values for a subset of `(physical offset, field)` +//! cells within a fragment, without rewriting the fragment's base data files. See +//! the Data Overlay Files specification for the full rules; the invariants this +//! module relies on are: +//! +//! - **Physical-offset coverage.** Coverage bitmaps index *physical* row offsets +//! (positions in the base data files, counting deleted rows), so they are stable +//! across deletions, like deletion vectors. +//! - **Rank-based values.** The overlay's `data_file` stores one value column per +//! field, with no row-offset key column. Within a value column, a covered +//! offset's value sits at its **rank** — the 0-based count of set bits below it +//! in that field's coverage bitmap. +//! - **Dense vs. sparse coverage.** A dense overlay shares one bitmap across every +//! field ([`OverlayCoverage::Shared`]); a sparse overlay carries one bitmap per +//! field ([`OverlayCoverage::PerField`]). +//! - **Parse once.** Bitmaps are parsed from their 32-bit Roaring encoding a single +//! time when the fragment loads and held behind an `Arc`, so cloning a fragment +//! is cheap. +//! - **Newest-last ordering.** A fragment's overlays are stored newest-last and +//! stable-sorted by `committed_version` on load (see [`sort_overlays_newest_last`]), +//! with list position breaking ties for equal versions. When two overlays cover +//! the same `(offset, field)`, the higher `committed_version` wins. +//! - **Field tombstones.** When new base values are written for a field (a +//! DataReplacement, or an in-place column rewrite), any overlay value for that +//! field is stale and must stop shadowing the fresh base. The field is marked +//! obsolete in the overlay's `data_file.fields` with [`TOMBSTONE_FIELD_ID`] +//! (the same sentinel used for obsolete base columns) rather than physically +//! removed, so the overlay's other fields — and its coverage positions — stay +//! intact (see [`tombstone_overlay_fields`]). + +use std::sync::Arc; + +use lance_core::Error; +use lance_core::deepsize::DeepSizeOf; +use lance_core::error::Result; +use roaring::RoaringBitmap; +use serde::{Deserialize, Serialize}; + +use object_store::path::Path; + +use super::DataFile; +use crate::format::pb; + +/// Field-id sentinel marking a tombstoned (obsolete) field within an overlay's +/// `data_file.fields`. Matches the tombstone convention for obsolete columns in +/// base data files; a tombstoned field's values are ignored on read. +pub const TOMBSTONE_FIELD_ID: i32 = -2; + +/// Which `(physical offset, field)` cells a [`DataOverlayFile`] provides values +/// for. +/// +/// Bitmaps are parsed from their 32-bit Roaring encoding once when the fragment +/// is loaded and held behind an `Arc` so cloning a fragment is cheap; use +/// [`DataOverlayFile::coverage_for_field`] to obtain the one that applies to a +/// given field. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(into = "OverlayCoverageBytes", try_from = "OverlayCoverageBytes")] +pub enum OverlayCoverage { + /// A single bitmap that applies to every field in the overlay's + /// `data_file.fields` (a dense / rectangular overlay): every covered offset + /// has a value for every field. + Shared(Arc), + /// One bitmap per field, in the same order as the overlay's + /// `data_file.fields` (a sparse overlay): different fields may cover + /// different offset sets. + PerField(Vec>), +} + +/// Serialized form of [`OverlayCoverage`] — each bitmap as its 32-bit Roaring +/// byte encoding. The in-memory form parses these once at load. +#[derive(Debug, Clone, Serialize, Deserialize)] +enum OverlayCoverageBytes { + Shared(Vec), + PerField(Vec>), +} + +// The bytes come from a persisted overlay (the protobuf manifest or a +// serialized fragment), so a decode failure is on-disk corruption, not caller +// input. `path` locates the overlay's data file when known (empty on the serde +// path, which deserializes coverage in isolation). +fn deserialize_roaring(bytes: &[u8], path: &Path) -> Result { + RoaringBitmap::deserialize_from(bytes).map_err(|e| { + Error::corrupt_file( + path.clone(), + format!("failed to deserialize overlay coverage bitmap: {e}"), + ) + }) +} + +fn serialize_roaring(bitmap: &RoaringBitmap) -> Vec { + let mut bytes = Vec::with_capacity(bitmap.serialized_size()); + // Writing to a Vec is infallible. + bitmap.serialize_into(&mut bytes).unwrap(); + bytes +} + +impl From for OverlayCoverageBytes { + fn from(coverage: OverlayCoverage) -> Self { + match coverage { + OverlayCoverage::Shared(bitmap) => Self::Shared(serialize_roaring(&bitmap)), + OverlayCoverage::PerField(bitmaps) => { + Self::PerField(bitmaps.iter().map(|b| serialize_roaring(b)).collect()) + } + } + } +} + +impl TryFrom for OverlayCoverage { + type Error = Error; + + fn try_from(bytes: OverlayCoverageBytes) -> Result { + // Serde deserializes the coverage in isolation, so the owning data + // file's path is not available here. + let path = Path::default(); + Ok(match bytes { + OverlayCoverageBytes::Shared(b) => { + Self::Shared(Arc::new(deserialize_roaring(&b, &path)?)) + } + OverlayCoverageBytes::PerField(bs) => Self::PerField( + bs.iter() + .map(|b| deserialize_roaring(b, &path).map(Arc::new)) + .collect::>()?, + ), + }) + } +} + +impl DeepSizeOf for OverlayCoverage { + fn deep_size_of_children(&self, context: &mut lance_core::deepsize::Context) -> usize { + // The same `Arc` is shared across every clone of a + // fragment, so mark each Arc's pointer and count its heap only the first + // time it is seen — otherwise walking many fragments double-counts the + // shared bitmaps. RoaringBitmap does not expose its allocation size; its + // serialized size is a cheap, close proxy for the heap it holds. + let bitmap_heap = |bitmap: &Arc, + context: &mut lance_core::deepsize::Context| { + if context.mark_seen(Arc::as_ptr(bitmap) as usize) { + std::mem::size_of::() + bitmap.serialized_size() + } else { + 0 + } + }; + match self { + Self::Shared(bitmap) => bitmap_heap(bitmap, context), + Self::PerField(bitmaps) => { + bitmaps.capacity() * std::mem::size_of::>() + + bitmaps + .iter() + .map(|b| bitmap_heap(b, context)) + .sum::() + } + } + } +} + +impl OverlayCoverage { + /// Build a dense coverage from a single bitmap shared across every field. + pub fn dense(bitmap: RoaringBitmap) -> Self { + Self::Shared(Arc::new(bitmap)) + } + + /// Build a sparse coverage from one bitmap per field. + pub fn sparse(bitmaps: Vec) -> Self { + Self::PerField(bitmaps.into_iter().map(Arc::new).collect()) + } +} + +/// An overlay file supplies new values for a subset of `(physical offset, field)` +/// cells within a fragment, without rewriting the fragment's base data files. See +/// the [module documentation](self) for the coverage, rank, and versioning rules. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, DeepSizeOf)] +pub struct DataOverlayFile { + /// The data file storing the overlay's new cell values. + pub data_file: DataFile, + /// Which cells this overlay provides values for. + pub coverage: OverlayCoverage, + /// The dataset version at which this overlay became effective (the version of + /// the commit that introduced it, stamped at commit time and re-stamped on + /// retry). Higher wins when two overlays cover the same `(offset, field)`. + pub committed_version: u64, +} + +impl DataOverlayFile { + /// The parsed coverage bitmap that applies to the field stored at + /// `field_pos` within `data_file.fields`. + /// + /// For a dense overlay the same shared bitmap is returned for every field; + /// for a sparse overlay the per-field bitmap at `field_pos` is returned. The + /// bitmap is already parsed, so this is a cheap `Arc` clone. + pub fn coverage_for_field(&self, field_pos: usize) -> Result> { + match &self.coverage { + OverlayCoverage::Shared(bitmap) => Ok(bitmap.clone()), + OverlayCoverage::PerField(bitmaps) => { + bitmaps.get(field_pos).cloned().ok_or_else(|| { + Error::invalid_input(format!( + "overlay per-field coverage has {} bitmaps but field position {} was requested", + bitmaps.len(), + field_pos + )) + }) + } + } + } +} + +/// Stable-sort a fragment's overlays newest-last by `committed_version`. The +/// stable sort preserves list position as the tiebreak for equal versions, so +/// resolution can rely on the ordering without re-checking. See the [module +/// documentation](self) for the ordering invariant. +pub fn sort_overlays_newest_last(overlays: &mut [DataOverlayFile]) { + overlays.sort_by_key(|overlay| overlay.committed_version); +} + +/// Verify a fragment's overlays are stored newest-last (non-decreasing +/// `committed_version`), the ordering invariant readers rely on for +/// resolution. Returns an error identifying the first out-of-order pair. +/// +/// [`sort_overlays_newest_last`] normalizes on load; this is the write-side +/// guard that rejects any commit path that assembled overlays out of order. See +/// the [module documentation](self) for the ordering invariant. +pub fn verify_overlays_newest_last(overlays: &[DataOverlayFile]) -> Result<()> { + for pair in overlays.windows(2) { + if pair[0].committed_version > pair[1].committed_version { + return Err(Error::invalid_input(format!( + "overlay files must be stored newest-last, but committed_version {} precedes {}", + pair[0].committed_version, pair[1].committed_version + ))); + } + } + Ok(()) +} + +/// Tombstone `fields` across a fragment's `overlays`, dropping any overlay left +/// with no live fields. +/// +/// Called when new base values are written for those fields (a DataReplacement, +/// or an in-place column rewrite): the stale overlay values must stop shadowing +/// the fresh base. Each matching field id is replaced with [`TOMBSTONE_FIELD_ID`] +/// in place, preserving the overlay's remaining fields and its coverage positions +/// (a per-field coverage bitmap stays aligned with `data_file.fields`). An overlay +/// whose fields are now all tombstoned is removed entirely. See the [module +/// documentation](self) for the tombstone invariant. +pub fn tombstone_overlay_fields(overlays: &mut Vec, fields: &[u32]) { + for overlay in overlays.iter_mut() { + let tombstoned: Vec = overlay + .data_file + .fields + .iter() + .map(|&field| { + if field >= 0 && fields.contains(&(field as u32)) { + TOMBSTONE_FIELD_ID + } else { + field + } + }) + .collect(); + overlay.data_file.fields = tombstoned.into(); + } + overlays.retain(|overlay| { + overlay + .data_file + .fields + .iter() + .any(|&field| field != TOMBSTONE_FIELD_ID) + }); +} + +impl From<&DataOverlayFile> for pb::DataOverlayFile { + fn from(overlay: &DataOverlayFile) -> Self { + let coverage = match &overlay.coverage { + OverlayCoverage::Shared(bitmap) => { + pb::data_overlay_file::Coverage::SharedOffsetBitmap(serialize_roaring(bitmap)) + } + OverlayCoverage::PerField(bitmaps) => { + pb::data_overlay_file::Coverage::FieldCoverage(pb::FieldCoverage { + offset_bitmaps: bitmaps.iter().map(|b| serialize_roaring(b)).collect(), + }) + } + }; + Self { + data_file: Some(pb::DataFile::from(&overlay.data_file)), + coverage: Some(coverage), + committed_version: overlay.committed_version, + } + } +} + +impl TryFrom for DataOverlayFile { + type Error = Error; + + fn try_from(proto: pb::DataOverlayFile) -> Result { + let data_file = proto + .data_file + .ok_or_else(|| Error::invalid_input("DataOverlayFile is missing its data_file"))?; + let path = Path::from(data_file.path.as_str()); + let coverage = match proto.coverage { + Some(pb::data_overlay_file::Coverage::SharedOffsetBitmap(bytes)) => { + OverlayCoverage::Shared(Arc::new(deserialize_roaring(&bytes, &path)?)) + } + Some(pb::data_overlay_file::Coverage::FieldCoverage(fc)) => OverlayCoverage::PerField( + fc.offset_bitmaps + .iter() + .map(|b| deserialize_roaring(b, &path).map(Arc::new)) + .collect::>()?, + ), + None => { + return Err(Error::invalid_input( + "DataOverlayFile is missing its coverage", + )); + } + }; + Ok(Self { + data_file: DataFile::try_from(data_file)?, + coverage, + committed_version: proto.committed_version, + }) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_data_overlay_missing_fields_error() { + // A DataOverlayFile proto missing its coverage or data_file is rejected. + let no_coverage = pb::DataOverlayFile { + data_file: Some(pb::DataFile::from(&DataFile::new_legacy_from_fields( + "overlay.lance", + vec![3], + None, + ))), + coverage: None, + committed_version: 1, + }; + let err = DataOverlayFile::try_from(no_coverage).unwrap_err(); + assert!(err.to_string().contains("missing its coverage"), "{err}"); + + let no_data_file = pb::DataOverlayFile { + data_file: None, + coverage: Some(pb::data_overlay_file::Coverage::SharedOffsetBitmap( + serialize_roaring(&RoaringBitmap::from_iter([0u32])), + )), + committed_version: 1, + }; + let err = DataOverlayFile::try_from(no_data_file).unwrap_err(); + assert!(err.to_string().contains("missing its data_file"), "{err}"); + } + + #[test] + fn test_overlay_coverage_serde_json_roundtrip() { + // The custom serde impl round-trips through JSON for dense/sparse, + // including empty bitmaps and a zero-bitmap sparse coverage. + for coverage in [ + OverlayCoverage::dense(RoaringBitmap::from_iter([1u32, 5, 100])), + OverlayCoverage::dense(RoaringBitmap::new()), + OverlayCoverage::sparse(vec![ + RoaringBitmap::from_iter([2u32, 3]), + RoaringBitmap::new(), + ]), + OverlayCoverage::sparse(vec![]), + ] { + let json = serde_json::to_string(&coverage).unwrap(); + let back: OverlayCoverage = serde_json::from_str(&json).unwrap(); + assert_eq!(back, coverage); + } + } + + #[test] + fn test_tombstone_overlay_fields() { + // An overlay covering fields [3, 5]: replacing field 5 tombstones just + // field 5's slot and keeps field 3. An overlay covering only field 5 is + // dropped entirely. An overlay touching no replaced field is untouched. + let mut overlays = vec![ + DataOverlayFile { + data_file: DataFile::new_legacy_from_fields("a.lance", vec![3, 5], None), + coverage: OverlayCoverage::sparse(vec![ + RoaringBitmap::from_iter([0u32]), + RoaringBitmap::from_iter([1u32]), + ]), + committed_version: 1, + }, + DataOverlayFile { + data_file: DataFile::new_legacy_from_fields("b.lance", vec![5], None), + coverage: OverlayCoverage::dense(RoaringBitmap::from_iter([0u32])), + committed_version: 1, + }, + DataOverlayFile { + data_file: DataFile::new_legacy_from_fields("c.lance", vec![7], None), + coverage: OverlayCoverage::dense(RoaringBitmap::from_iter([0u32])), + committed_version: 1, + }, + ]; + + tombstone_overlay_fields(&mut overlays, &[5]); + + // The single-field overlay on field 5 is gone; the others remain. + assert_eq!(overlays.len(), 2); + // Field 3 preserved, field 5 tombstoned in place (coverage stays aligned). + assert_eq!( + overlays[0].data_file.fields.as_ref(), + &[3, TOMBSTONE_FIELD_ID] + ); + // The untouched overlay keeps its field. + assert_eq!(overlays[1].data_file.fields.as_ref(), &[7]); + } + + #[test] + fn test_verify_overlays_newest_last() { + let mk = |version: u64| DataOverlayFile { + data_file: DataFile::new_legacy_from_fields("o.lance", vec![3], None), + coverage: OverlayCoverage::dense(RoaringBitmap::from_iter([0u32])), + committed_version: version, + }; + // Non-decreasing (including equal versions) is accepted. + assert!(verify_overlays_newest_last(&[]).is_ok()); + assert!(verify_overlays_newest_last(&[mk(1), mk(2), mk(2), mk(5)]).is_ok()); + // A newer version before an older one is rejected. + let err = verify_overlays_newest_last(&[mk(2), mk(1)]).unwrap_err(); + assert!(err.to_string().contains("newest-last"), "{err}"); + } + + #[test] + fn test_coverage_for_field_out_of_bounds() { + let overlay = DataOverlayFile { + data_file: DataFile::new_legacy_from_fields("o.lance", vec![2, 4], None), + coverage: OverlayCoverage::sparse(vec![ + RoaringBitmap::from_iter([1u32]), + RoaringBitmap::from_iter([2u32]), + ]), + committed_version: 1, + }; + assert!(overlay.coverage_for_field(0).is_ok()); + assert!(overlay.coverage_for_field(1).is_ok()); + let err = overlay.coverage_for_field(5).unwrap_err(); + assert!(err.to_string().contains("field position"), "{err}"); + } +} diff --git a/rust/lance-table/src/format/row_ids.rs b/rust/lance-table/src/format/row_ids.rs new file mode 100644 index 00000000000..4cac0f8c7a0 --- /dev/null +++ b/rust/lance-table/src/format/row_ids.rs @@ -0,0 +1,210 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright The Lance Authors + +use std::ops::Deref; +use std::sync::{Arc, OnceLock}; + +use lance_core::deepsize::{Context, DeepSizeOf}; +use lance_core::{Error, Result}; +use serde::{Deserialize, Deserializer, Serialize, Serializer}; + +use super::pb; + +/// A reference to a part of a file. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, DeepSizeOf)] +pub struct ExternalFile { + pub path: String, + pub offset: u64, + pub size: u64, +} + +/// A fragment's row id sequence, encoded inline in the manifest. +/// +/// Carries a memoized [`digest`](Self::digest) of the encoded bytes. The digest +/// identifies *which* sequence these bytes are, which is what the row id +/// sequence cache keys on: a fragment id alone does not identify a sequence, +/// because fragment ids are reused across dataset generations. +/// +/// The digest is memoized because computing it is proportional to the encoded +/// size. A run-encoded sequence is a handful of bytes per run, but a heavily +/// fragmented one is array-encoded at 8 bytes per row, and the cache is +/// consulted on every scan, count, prefilter and index load. +/// +/// Bytes and memo share one immutable allocation, so cloning shares both. That +/// is what makes the memo worth having: callers clone `Fragment` before loading +/// its sequence (see `count_from_mask`), and a memo held per clone would be +/// filled and dropped by each scan, rehashing the whole sequence every time. +/// Sharing also keeps a cloned fragment from duplicating the encoded bytes. +/// +/// The digest lives *with* the bytes rather than beside them so the two cannot +/// drift: several write paths replace a fragment's `row_id_meta` after the +/// fragment is built, and a digest that outlived its bytes would silently +/// resolve to another generation's sequence. +#[derive(Clone)] +pub struct InlineRowIds { + inner: Arc, +} + +struct InlineRowIdsInner { + data: Vec, + digest: OnceLock<[u8; 32]>, +} + +impl InlineRowIds { + /// Digest of the encoded bytes, computed on first use and shared by clones. + pub fn digest(&self) -> &[u8; 32] { + self.inner + .digest + .get_or_init(|| blake3::hash(&self.inner.data).into()) + } +} + +impl From> for InlineRowIds { + fn from(data: Vec) -> Self { + Self { + inner: Arc::new(InlineRowIdsInner { + data, + digest: OnceLock::new(), + }), + } + } +} + +impl Deref for InlineRowIds { + type Target = [u8]; + + fn deref(&self) -> &Self::Target { + &self.inner.data + } +} + +// Debug, equality and serialization all present the bytes alone: the memo is a +// derived value and must not show up in output, comparisons or the manifest. +impl std::fmt::Debug for InlineRowIds { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + self.inner.data.fmt(f) + } +} + +impl PartialEq for InlineRowIds { + fn eq(&self, other: &Self) -> bool { + Arc::ptr_eq(&self.inner, &other.inner) || self.inner.data == other.inner.data + } +} + +impl Eq for InlineRowIds {} + +impl Serialize for InlineRowIds { + fn serialize(&self, serializer: S) -> std::result::Result { + self.inner.data.serialize(serializer) + } +} + +impl<'de> Deserialize<'de> for InlineRowIds { + fn deserialize>(deserializer: D) -> std::result::Result { + Vec::::deserialize(deserializer).map(Self::from) + } +} + +impl DeepSizeOf for InlineRowIds { + fn deep_size_of_children(&self, context: &mut Context) -> usize { + // Delegate to the `Arc` so clones sharing one allocation are counted once. + self.inner.deep_size_of_children(context) + } +} + +impl DeepSizeOf for InlineRowIdsInner { + fn deep_size_of_children(&self, context: &mut Context) -> usize { + self.data.deep_size_of_children(context) + } +} + +/// Metadata about location of the row id sequence. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, DeepSizeOf)] +pub enum RowIdMeta { + Inline(InlineRowIds), + External(ExternalFile), +} + +impl TryFrom for RowIdMeta { + type Error = Error; + + fn try_from(value: pb::data_fragment::RowIdSequence) -> Result { + match value { + pb::data_fragment::RowIdSequence::InlineRowIds(data) => Ok(Self::Inline(data.into())), + pb::data_fragment::RowIdSequence::ExternalRowIds(file) => { + Ok(Self::External(ExternalFile { + path: file.path.clone(), + offset: file.offset, + size: file.size, + })) + } + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::format::Fragment; + + #[test] + fn inline_row_ids_digest_identifies_contents() { + let first = InlineRowIds::from(vec![1, 2, 3]); + let same = InlineRowIds::from(vec![1, 2, 3]); + let other = InlineRowIds::from(vec![1, 2, 4]); + + // The digest is what the row id sequence cache keys on, so it must + // follow the bytes exactly: same bytes, same key; any change, new key. + assert_eq!(first.digest(), same.digest()); + assert_ne!(first.digest(), other.digest()); + assert_eq!(first, same); + assert_ne!(first, other); + + // Memoized on first use, so repeat use must return the same digest. + let memoized = *first.digest(); + assert_eq!(first.digest(), &memoized); + } + + #[test] + fn inline_row_ids_clones_share_bytes_and_memo() { + let first = InlineRowIds::from(vec![1, 2, 3]); + let cloned = first.clone(); + + // Callers clone `Fragment` before loading its row id sequence, so a memo + // held per clone would be filled and dropped by each scan and rehash the + // whole sequence every time. Sharing one allocation is what makes the + // memo pay off, and it keeps clones from duplicating the bytes. + assert!(std::ptr::eq(first.as_ptr(), cloned.as_ptr())); + assert!(std::ptr::eq(first.digest(), cloned.digest())); + + // Guard against the assertions above passing vacuously: only clones + // share storage, equal bytes built separately do not. + let separate = InlineRowIds::from(vec![1, 2, 3]); + assert_eq!(first, separate); + assert!(!std::ptr::eq(first.as_ptr(), separate.as_ptr())); + + // Computing through one clone must be visible from the other. + let fresh = InlineRowIds::from(vec![4, 5, 6]); + let fresh_clone = fresh.clone(); + let via_clone = *fresh_clone.digest(); + assert_eq!(fresh.digest(), &via_clone); + } + + #[test] + fn inline_row_ids_serializes_as_bare_bytes() { + // The manifest is a stable format: the memo must not reach the wire. + let meta = RowIdMeta::Inline(InlineRowIds::from(vec![7, 8, 9])); + let json = serde_json::to_string(&meta).unwrap(); + assert_eq!(json, r#"{"Inline":[7,8,9]}"#); + assert_eq!(serde_json::from_str::(&json).unwrap(), meta); + + // ...and round-trips through protobuf unchanged. + let fragment = Fragment { + row_id_meta: Some(meta.clone()), + ..Fragment::new(0) + }; + let restored = Fragment::try_from(pb::DataFragment::from(&fragment)).unwrap(); + assert_eq!(restored.row_id_meta, Some(meta)); + } +} diff --git a/rust/lance-table/src/io/commit.rs b/rust/lance-table/src/io/commit.rs index e1a4086730b..ea75fb7db4a 100644 --- a/rust/lance-table/src/io/commit.rs +++ b/rust/lance-table/src/io/commit.rs @@ -781,6 +781,27 @@ const DDB_URL_QUERY_KEY: &str = "ddbTableName"; #[async_trait::async_trait] #[allow(clippy::too_many_arguments)] pub trait CommitHandler: Debug + Send + Sync { + /// Whether a not-found result from [`Self::resolve_version_location`] is + /// definitive immediately after a commit attempt. + /// + /// Handlers backed by an eventually consistent or external source of + /// truth should keep the conservative default. This prevents callers from + /// deleting files that a newly committed manifest may reference while the + /// manifest is not yet visible through the resolver. + fn is_version_not_found_definitive(&self) -> bool { + false + } + + /// Whether an error should still be returned after readback proves that + /// the manifest from the current commit attempt landed. + /// + /// The conservative default preserves errors from custom handlers. Built-in + /// object-store handlers override this because their commit errors may be + /// ambiguous transport failures whose successful outcome is authoritative. + fn propagate_commit_error_after_success(&self) -> bool { + true + } + async fn resolve_latest_location( &self, base_path: &Path, @@ -1091,9 +1112,10 @@ pub async fn commit_handler_from_url( match url.scheme() { "file" | "file-object-store" => Ok(local_handler), - "s3" | "gs" | "az" | "abfss" | "memory" | "oss" | "cos" | "shared-memory" => { + "s3" | "gs" | "az" | "abfss" | "memory" | "oss" | "tos" | "shared-memory" | "goosefs" => { Ok(Arc::new(ConditionalPutCommitHandler)) } + "cos" => Ok(Arc::new(TencentCosCommitHandler)), #[cfg(not(feature = "dynamodb"))] "s3+ddb" => Err(Error::invalid_input_source( "`s3+ddb://` scheme requires `dynamodb` feature to be enabled".into(), @@ -1133,12 +1155,15 @@ pub async fn commit_handler_from_url( // Get accessor from the options let accessor = options.get_accessor(); + let provider_scheme = storage_options_raw.aws_provider_scheme()?; + let (aws_creds, region) = build_aws_credential( options.s3_credentials_refresh_offset, options.aws_credentials.clone(), Some(&storage_options), region, accessor, + provider_scheme, ) .await?; @@ -1201,6 +1226,14 @@ pub struct UnsafeCommitHandler; #[async_trait::async_trait] #[allow(clippy::too_many_arguments)] impl CommitHandler for UnsafeCommitHandler { + fn is_version_not_found_definitive(&self) -> bool { + true + } + + fn propagate_commit_error_after_success(&self) -> bool { + false + } + async fn commit( &self, manifest: &mut Manifest, @@ -1328,6 +1361,10 @@ impl CommitHandler for T where T::Lease: 'static, { + fn is_version_not_found_definitive(&self) -> bool { + true + } + async fn commit( &self, manifest: &mut Manifest, @@ -1387,6 +1424,14 @@ impl CommitHandler for Arc where T::Lease: 'static, { + fn is_version_not_found_definitive(&self) -> bool { + self.as_ref().is_version_not_found_definitive() + } + + fn propagate_commit_error_after_success(&self) -> bool { + self.as_ref().propagate_commit_error_after_success() + } + async fn commit( &self, manifest: &mut Manifest, @@ -1418,6 +1463,14 @@ pub struct RenameCommitHandler; #[async_trait::async_trait] impl CommitHandler for RenameCommitHandler { + fn is_version_not_found_definitive(&self) -> bool { + true + } + + fn propagate_commit_error_after_success(&self) -> bool { + false + } + async fn commit( &self, manifest: &mut Manifest, @@ -1477,6 +1530,14 @@ pub struct ConditionalPutCommitHandler; #[async_trait::async_trait] impl CommitHandler for ConditionalPutCommitHandler { + fn is_version_not_found_definitive(&self) -> bool { + true + } + + fn propagate_commit_error_after_success(&self) -> bool { + false + } + async fn commit( &self, manifest: &mut Manifest, @@ -1537,6 +1598,45 @@ impl Debug for ConditionalPutCommitHandler { } } +/// A read-capable handler that prevents unsafe default commits to Tencent COS. +/// +/// COS silently ignores its put-if-not-exists header on buckets that have ever +/// had versioning enabled. Since that bucket history cannot be inferred from +/// the URI or storage options, using [`ConditionalPutCommitHandler`] here can +/// let concurrent writers overwrite the same manifest without reporting a +/// conflict. +struct TencentCosCommitHandler; + +#[async_trait::async_trait] +impl CommitHandler for TencentCosCommitHandler { + fn is_version_not_found_definitive(&self) -> bool { + true + } + + async fn commit( + &self, + _manifest: &mut Manifest, + _indices: Option>, + _base_path: &Path, + _object_store: &ObjectStore, + _manifest_writer: ManifestWriter, + _naming_scheme: ManifestNamingScheme, + _transaction: Option, + ) -> std::result::Result { + Err(CommitError::OtherError(Error::not_supported( + "Default writes to Tencent COS are disabled because COS does not reliably enforce \ + put-if-not-exists after bucket versioning has ever been enabled. Provide a \ + distributed commit_lock in Python or a custom CommitHandler in Rust.", + ))) + } +} + +impl Debug for TencentCosCommitHandler { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("TencentCosCommitHandler").finish() + } +} + #[derive(Debug, Clone)] pub struct CommitConfig { pub num_retries: u32, @@ -1966,19 +2066,27 @@ mod tests { } #[tokio::test] - async fn test_commit_handler_from_url_memory_schemes() { - // Both `memory://` and `shared-memory://` must route to - // ConditionalPutCommitHandler — otherwise concurrent writers fall - // through to UnsafeCommitHandler and silently clobber each other's - // manifests. - for url in ["memory://bucket-a/ds", "shared-memory://bucket-a/ds"] { - let handler = commit_handler_from_url(url, &None).await.unwrap(); - assert_eq!( - format!("{:?}", handler), - "ConditionalPutCommitHandler", - "{url} should route to ConditionalPutCommitHandler", - ); - } + #[rstest::rstest] + #[case::memory("memory://bucket-a/ds")] + #[case::shared_memory("shared-memory://bucket-a/ds")] + #[case::s3("s3://bucket-a/ds")] + #[case::gs("gs://bucket-a/ds")] + #[case::az("az://bucket-a/ds")] + #[case::abfss("abfss://bucket-a/ds")] + #[case::oss("oss://bucket-a/ds")] + #[case::tos("tos://bucket-a/ds")] + #[case::goosefs("goosefs://bucket-a/ds")] + async fn test_commit_handler_from_url_conditional_put_schemes(#[case] url: &str) { + // Every scheme whose store supports atomic put-if-not-exists must + // route to ConditionalPutCommitHandler — otherwise concurrent writers + // fall through to UnsafeCommitHandler and silently clobber each + // other's manifests. + let handler = commit_handler_from_url(url, &None).await.unwrap(); + assert_eq!( + format!("{:?}", handler), + "ConditionalPutCommitHandler", + "{url} should route to ConditionalPutCommitHandler", + ); } /// A [CommitLock] whose lease records whether it was released, so we can @@ -2069,6 +2177,51 @@ mod tests { Box::pin(async move { Ok(WriteResult::default()) }) } + fn test_manifest() -> Manifest { + use std::collections::HashMap; + + use arrow_schema::{DataType, Field as ArrowField, Schema as ArrowSchema}; + use lance_core::datatypes::Schema; + use lance_file::version::LanceFileVersion; + + use crate::format::DataStorageFormat; + + let arrow_schema = ArrowSchema::new(vec![ArrowField::new("i", DataType::Int32, false)]); + Manifest::new( + Schema::try_from(&arrow_schema).unwrap(), + Arc::new(vec![]), + DataStorageFormat::new(LanceFileVersion::Stable.resolve()), + HashMap::new(), + ) + } + + #[tokio::test] + async fn test_cos_commit_requires_custom_handler() { + let handler = commit_handler_from_url("cos://bucket-a/ds", &None) + .await + .unwrap(); + assert_eq!(format!("{:?}", handler), "TencentCosCommitHandler"); + + let mut manifest = test_manifest(); + let error = handler + .commit( + &mut manifest, + None, + &Path::from("test"), + &ObjectStore::memory(), + succeeding_manifest_writer, + ManifestNamingScheme::V2, + None, + ) + .await + .unwrap_err(); + let CommitError::OtherError(error) = error else { + panic!("expected a not-supported commit error"); + }; + assert!(matches!(error, Error::NotSupported { .. })); + assert!(error.to_string().contains("distributed commit_lock")); + } + /// A manifest writer that never completes, simulating a hung object store. fn hanging_manifest_writer<'a>( _object_store: &'a ObjectStore, @@ -2087,16 +2240,9 @@ mod tests { /// still release the lock; otherwise it leaks until the lease's TTL expires. #[tokio::test] async fn test_commit_lock_released_on_cancellation() { - use std::collections::HashMap; use std::sync::atomic::Ordering; use std::time::Duration; - use arrow_schema::{DataType, Field as ArrowField, Schema as ArrowSchema}; - use lance_core::datatypes::Schema; - use lance_file::version::LanceFileVersion; - - use crate::format::DataStorageFormat; - let released = Arc::new(AtomicBool::new(false)); let lock = TrackingLock { released: released.clone(), @@ -2104,13 +2250,7 @@ mod tests { let object_store = ObjectStore::memory(); let base_path = Path::from("test"); - let arrow_schema = ArrowSchema::new(vec![ArrowField::new("i", DataType::Int32, false)]); - let mut manifest = Manifest::new( - Schema::try_from(&arrow_schema).unwrap(), - Arc::new(vec![]), - DataStorageFormat::new(LanceFileVersion::Stable), - HashMap::new(), - ); + let mut manifest = test_manifest(); // The commit will hang on the manifest writer while holding the lock. // Cancel it the same way a commit timeout would: drop the future. @@ -2144,16 +2284,9 @@ mod tests { /// lock via the drop-path best-effort release. #[tokio::test] async fn test_commit_lock_released_on_cancellation_during_release() { - use std::collections::HashMap; use std::sync::atomic::Ordering; use std::time::Duration; - use arrow_schema::{DataType, Field as ArrowField, Schema as ArrowSchema}; - use lance_core::datatypes::Schema; - use lance_file::version::LanceFileVersion; - - use crate::format::DataStorageFormat; - let release_calls = Arc::new(AtomicUsize::new(0)); let released = Arc::new(AtomicBool::new(false)); let lock = HangingReleaseLock { @@ -2163,13 +2296,7 @@ mod tests { let object_store = ObjectStore::memory(); let base_path = Path::from("test"); - let arrow_schema = ArrowSchema::new(vec![ArrowField::new("i", DataType::Int32, false)]); - let mut manifest = Manifest::new( - Schema::try_from(&arrow_schema).unwrap(), - Arc::new(vec![]), - DataStorageFormat::new(LanceFileVersion::Stable), - HashMap::new(), - ); + let mut manifest = test_manifest(); // The manifest writer succeeds, so the commit reaches the explicit // release, which hangs. Cancel it the same way a commit timeout would. diff --git a/rust/lance-table/src/io/commit/external_manifest.rs b/rust/lance-table/src/io/commit/external_manifest.rs index 22ebaa10b4a..5d1c8f65dfc 100644 --- a/rust/lance-table/src/io/commit/external_manifest.rs +++ b/rust/lance-table/src/io/commit/external_manifest.rs @@ -134,20 +134,18 @@ pub trait ExternalManifestStore: std::fmt::Debug + Send + Sync { info!(target: TRACE_FILE_AUDIT, mode=AUDIT_MODE_CREATE, r#type=AUDIT_TYPE_MANIFEST, path = final_path.as_ref()); } - // Get final e_tag (may change after copy for large files) - let e_tag = if copied && size < 5 * 1024 * 1024 { - e_tag - } else { - let meta = object_store.head(&final_path).await?; - meta.e_tag - }; + // A copy creates a new object whose metadata may differ from the source. + // Read the destination metadata before publishing the final path. + let final_meta = object_store.head(&final_path).await?; + let final_size = final_meta.size; + let final_e_tag = final_meta.e_tag; let location = ManifestLocation { version, path: final_path.clone(), - size: Some(size), + size: Some(final_size), naming_scheme, - e_tag: e_tag.clone(), + e_tag: final_e_tag.clone(), }; if !copied { @@ -159,8 +157,8 @@ pub trait ExternalManifestStore: std::fmt::Debug + Send + Sync { base_path.as_ref(), version, final_path.as_ref(), - size, - e_tag, + final_size, + final_e_tag, ) .await?; @@ -347,6 +345,69 @@ pub struct ExternalManifestCommitHandler { } impl ExternalManifestCommitHandler { + async fn verify_finalized_manifest_location( + &self, + base_path: &Path, + location: ManifestLocation, + object_store: &dyn OSObjectStore, + ) -> std::result::Result { + match object_store.head(&location.path).await { + Ok(ObjectMeta { size, e_tag, .. }) => { + let ManifestLocation { + version, + path, + size: expected_size, + naming_scheme, + e_tag: expected_e_tag, + } = location; + + let size = match expected_size { + Some(expected_size) if expected_size != size => { + return Err(Error::corrupt_file( + path, + format!( + "Manifest size mismatch for version {}: external store expected {}, object store returned {}", + version, expected_size, size + ), + )); + } + Some(expected_size) => Some(expected_size), + None => Some(size), + }; + + let e_tag = match expected_e_tag { + Some(expected_e_tag) => { + if e_tag.as_ref() != Some(&expected_e_tag) { + return Err(Error::corrupt_file( + path, + format!( + "Manifest e_tag mismatch for version {}: external store expected {:?}, object store returned {:?}", + version, expected_e_tag, e_tag + ), + )); + } + Some(expected_e_tag) + } + None => e_tag, + }; + + Ok(ManifestLocation { + version, + path, + size, + naming_scheme, + e_tag, + }) + } + Err(ObjectStoreError::NotFound { .. }) => { + // The external store may hold a stale finalized V2 path while + // the object store still has the manifest at the V1 location. + default_resolve_version(base_path, location.version, object_store).await + } + Err(e) => Err(e.into()), + } + } + /// The manifest is considered committed once the staging manifest is written /// to object store and that path is committed to the external store. /// @@ -363,7 +424,6 @@ impl ExternalManifestCommitHandler { staging_manifest_path: &Path, version: u64, size: u64, - e_tag: Option, store: &dyn OSObjectStore, naming_scheme: ManifestNamingScheme, ) -> std::result::Result { @@ -380,25 +440,18 @@ impl ExternalManifestCommitHandler { info!(target: TRACE_FILE_AUDIT, mode=AUDIT_MODE_CREATE, r#type=AUDIT_TYPE_MANIFEST, path = final_manifest_path.as_ref()); } - // On S3, the etag can change if originally was MultipartUpload and later was Copy - // https://docs.aws.amazon.com/AmazonS3/latest/API/API_Object.html#AmazonS3-Type-Object-ETag - // We only do MultipartUpload for > 5MB files, so we can skip this check - // if size < 5MB. However, we need to double check the final_manifest_path - // exists before we change the external store, otherwise we may point to a - // non-existing manifest. - let e_tag = if copied && size < 5 * 1024 * 1024 { - e_tag - } else { - let meta = store.head(&final_manifest_path).await?; - meta.e_tag - }; + // A copy creates a new object whose metadata may differ from the source. + // Read the destination metadata before publishing the final path. + let final_meta = store.head(&final_manifest_path).await?; + let final_size = final_meta.size; + let final_e_tag = final_meta.e_tag; let location = ManifestLocation { version, path: final_manifest_path, - size: Some(size), + size: Some(final_size), naming_scheme, - e_tag, + e_tag: final_e_tag, }; if !copied { @@ -411,7 +464,7 @@ impl ExternalManifestCommitHandler { base_path.as_ref(), version, location.path.as_ref(), - size, + final_size, location.e_tag.clone(), ) .await?; @@ -441,29 +494,30 @@ impl CommitHandler for ExternalManifestCommitHandler { .await?; match location { - Some(ManifestLocation { - version, - path, - size, - naming_scheme, - e_tag, - }) => { - // The path is finalized, no need to check object store - if path.extension() == Some(MANIFEST_EXTENSION) { - return Ok(ManifestLocation { - version, - path, - size, - naming_scheme, - e_tag, - }); + Some(location) => { + if location.path.extension() == Some(MANIFEST_EXTENSION) { + return self + .verify_finalized_manifest_location( + base_path, + location, + object_store.inner.as_ref(), + ) + .await; } - let (size, e_tag) = if let Some(size) = size { - (size, e_tag) + let ManifestLocation { + version, + path, + size, + naming_scheme, + e_tag: _, + } = location; + + let size = if let Some(size) = size { + size } else { match object_store.inner.head(&path).await { - Ok(meta) => (meta.size, meta.e_tag), + Ok(meta) => meta.size, Err(ObjectStoreError::NotFound { .. }) => { // there may be other threads that have finished executing finalize_manifest. let new_location = self @@ -482,7 +536,6 @@ impl CommitHandler for ExternalManifestCommitHandler { &path, version, size, - e_tag.clone(), &object_store.inner, naming_scheme, ) @@ -552,19 +605,20 @@ impl CommitHandler for ExternalManifestCommitHandler { Err(e) => return Err(e), }; - // finalized path, just return if location.path.extension() == Some(MANIFEST_EXTENSION) { - return Ok(location); + return self + .verify_finalized_manifest_location(base_path, location, object_store) + .await; } let naming_scheme = ManifestNamingScheme::detect_scheme_staging(location.path.filename().unwrap()); - let (size, e_tag) = if let Some(size) = location.size { - (size, location.e_tag.clone()) + let size = if let Some(size) = location.size { + size } else { let meta = object_store.head(&location.path).await?; - (meta.size as u64, meta.e_tag) + meta.size }; self.finalize_manifest( @@ -572,7 +626,6 @@ impl CommitHandler for ExternalManifestCommitHandler { &location.path, version, size, - e_tag, object_store, naming_scheme, ) @@ -642,15 +695,39 @@ impl CommitHandler for ExternalManifestCommitHandler { write_version_hint(object_store, base_path, manifest.version).await; Ok(location) } - Err(_) => { - // delete the staging manifest - match object_store.inner.delete(&staging_path).await { - Ok(_) => {} - Err(ObjectStoreError::NotFound { .. }) => {} - Err(e) => return Err(CommitError::OtherError(e.into())), + Err(error) => { + // A different recorded path proves this staging manifest lost + // the version and is safe to remove. Otherwise, the external + // store may have recorded our staging path before its response + // was lost, so retain it for outcome verification/finalization. + let recorded_location = self + .external_manifest_store + .get_manifest_location(base_path.as_ref(), manifest.version) + .await; + if matches!( + &recorded_location, + Ok(location) if location.path != staging_path + ) { + match object_store.inner.delete(&staging_path).await { + Ok(()) => { + info!(target: TRACE_FILE_AUDIT, mode=AUDIT_MODE_DELETE, r#type=AUDIT_TYPE_MANIFEST, path = staging_path.as_ref()); + } + Err(ObjectStoreError::NotFound { .. }) => {} + Err(delete_error) => { + warn!( + "Failed to delete losing staging manifest '{}': {}", + staging_path, delete_error + ); + } + } + return Err(CommitError::CommitConflict); } - info!(target: TRACE_FILE_AUDIT, mode=AUDIT_MODE_DELETE, r#type=AUDIT_TYPE_MANIFEST, path = staging_path.as_ref()); - Err(CommitError::CommitConflict {}) + warn!( + "External manifest commit for version {} failed; retaining staging manifest \ + '{}' until the commit outcome is resolved: {}", + manifest.version, staging_path, error + ); + Err(CommitError::CommitConflict) } } } @@ -661,3 +738,294 @@ impl CommitHandler for ExternalManifestCommitHandler { .await } } + +#[cfg(test)] +mod tests { + use std::collections::HashMap; + use std::sync::Mutex; + use std::sync::atomic::{AtomicBool, Ordering}; + + use arrow_schema::{DataType, Field as ArrowField, Schema as ArrowSchema}; + use lance_core::datatypes::Schema; + use lance_core::utils::testing::{ProxyObjectStore, ProxyObjectStorePolicy}; + use lance_file::version::LanceFileVersion; + + use super::*; + use crate::format::DataStorageFormat; + use crate::io::commit::write_manifest_file_to_path; + + #[derive(Debug, Clone)] + struct StoredManifest { + path: String, + size: u64, + e_tag: Option, + } + + #[derive(Debug)] + struct TestExternalManifestStore { + manifests: Mutex>, + fail_next_put_response: AtomicBool, + } + + impl TestExternalManifestStore { + fn new(fail_next_put_response: bool) -> Self { + Self { + manifests: Mutex::new(HashMap::new()), + fail_next_put_response: AtomicBool::new(fail_next_put_response), + } + } + } + + #[async_trait] + impl ExternalManifestStore for TestExternalManifestStore { + async fn get(&self, base_uri: &str, version: u64) -> Result { + self.manifests + .lock() + .unwrap() + .get(&(base_uri.to_string(), version)) + .map(|manifest| manifest.path.clone()) + .ok_or_else(|| Error::not_found(format!("{base_uri}@{version}"))) + } + + async fn get_manifest_location( + &self, + base_uri: &str, + version: u64, + ) -> Result { + let stored = self + .manifests + .lock() + .unwrap() + .get(&(base_uri.to_string(), version)) + .cloned() + .ok_or_else(|| Error::not_found(format!("{base_uri}@{version}")))?; + let path = Path::from(stored.path); + Ok(ManifestLocation { + version, + naming_scheme: detect_naming_scheme_from_path(&path)?, + path, + size: Some(stored.size), + e_tag: stored.e_tag, + }) + } + + async fn get_latest_version(&self, base_uri: &str) -> Result> { + Ok(self + .manifests + .lock() + .unwrap() + .iter() + .filter(|((stored_base, _), _)| stored_base == base_uri) + .max_by_key(|((_, version), _)| *version) + .map(|((_, version), manifest)| (*version, manifest.path.clone()))) + } + + async fn put_if_not_exists( + &self, + base_uri: &str, + version: u64, + path: &str, + size: u64, + e_tag: Option, + ) -> Result<()> { + let key = (base_uri.to_string(), version); + let mut manifests = self.manifests.lock().unwrap(); + if manifests.contains_key(&key) { + return Err(Error::commit_conflict_source( + version, + "manifest already exists".to_string().into(), + )); + } + manifests.insert( + key, + StoredManifest { + path: path.to_string(), + size, + e_tag, + }, + ); + drop(manifests); + if self.fail_next_put_response.swap(false, Ordering::SeqCst) { + Err(Error::io("simulated lost external-store response")) + } else { + Ok(()) + } + } + + async fn put_if_exists( + &self, + base_uri: &str, + version: u64, + path: &str, + size: u64, + e_tag: Option, + ) -> Result<()> { + let key = (base_uri.to_string(), version); + let mut manifests = self.manifests.lock().unwrap(); + let manifest = manifests + .get_mut(&key) + .ok_or_else(|| Error::not_found(format!("{base_uri}@{version}")))?; + *manifest = StoredManifest { + path: path.to_string(), + size, + e_tag, + }; + Ok(()) + } + } + + fn test_manifest() -> Manifest { + let arrow_schema = ArrowSchema::new(vec![ArrowField::new("id", DataType::Int32, false)]); + Manifest::new( + Schema::try_from(&arrow_schema).unwrap(), + Arc::new(vec![]), + DataStorageFormat::new(LanceFileVersion::Stable.resolve()), + HashMap::new(), + ) + } + + #[tokio::test] + async fn test_lost_external_store_response_retains_staging_manifest() { + let external_store = Arc::new(TestExternalManifestStore::new(true)); + let handler = ExternalManifestCommitHandler { + external_manifest_store: external_store.clone(), + }; + let object_store = ObjectStore::memory(); + let base_path = Path::from("dataset"); + let mut manifest = test_manifest(); + + let commit_error = handler + .commit( + &mut manifest, + None, + &base_path, + &object_store, + write_manifest_file_to_path, + ManifestNamingScheme::V2, + None, + ) + .await + .expect_err("the simulated response loss must be surfaced"); + assert!(matches!(commit_error, CommitError::CommitConflict)); + + let staging_path = Path::from(external_store.get("dataset", 1).await.unwrap()); + object_store.inner.head(&staging_path).await.unwrap(); + + let resolved = handler + .resolve_version_location(&base_path, 1, object_store.inner.as_ref()) + .await + .expect("the retained staging manifest must allow finalization"); + assert_eq!( + resolved.path, + ManifestNamingScheme::V2.manifest_path(&base_path, 1) + ); + object_store.inner.head(&resolved.path).await.unwrap(); + } + + #[tokio::test] + async fn test_copy_failure_after_external_store_commit_retains_staging_manifest() { + let external_store = Arc::new(TestExternalManifestStore::new(false)); + let handler = ExternalManifestCommitHandler { + external_manifest_store: external_store.clone(), + }; + + let mut object_store = ObjectStore::memory(); + let fail_next_copy = Arc::new(AtomicBool::new(true)); + let failed_copy_source = Arc::new(Mutex::new(None)); + let mut policy = ProxyObjectStorePolicy::new(); + let policy_fail_next_copy = fail_next_copy.clone(); + let policy_failed_copy_source = failed_copy_source.clone(); + policy.set_before_policy( + "fail-copy-once", + Arc::new(move |method, location| { + if method == "copy" && policy_fail_next_copy.swap(false, Ordering::SeqCst) { + *policy_failed_copy_source.lock().unwrap() = Some(location.clone()); + return Err(Error::io("simulated copy failure")); + } + Ok(()) + }), + ); + let policy = Arc::new(Mutex::new(policy)); + object_store.inner = Arc::new(ProxyObjectStore::new( + object_store.inner.clone(), + policy.clone(), + )); + + let base_path = Path::from("dataset"); + let mut manifest = test_manifest(); + let version = manifest.version; + let canonical_path = ManifestNamingScheme::V2.manifest_path(&base_path, version); + + let commit_error = handler + .commit( + &mut manifest, + None, + &base_path, + &object_store, + write_manifest_file_to_path, + ManifestNamingScheme::V2, + None, + ) + .await + .expect_err("the simulated copy failure must be surfaced"); + assert!(matches!(commit_error, CommitError::CommitConflict)); + assert!( + !fail_next_copy.load(Ordering::SeqCst), + "the one-shot copy failure must be consumed" + ); + + let recorded_location = external_store + .get_manifest_location(base_path.as_ref(), version) + .await + .expect("the external store must retain the committed staging location"); + let staging_path = failed_copy_source + .lock() + .unwrap() + .clone() + .expect("the failure must be injected at copy(staging, canonical)"); + assert_eq!(recorded_location.path, staging_path); + object_store + .inner + .head(&staging_path) + .await + .expect("the winning staging manifest must be retained"); + + let canonical_error = object_store + .inner + .head(&canonical_path) + .await + .expect_err("copy failed before creating the canonical manifest"); + assert!( + matches!(canonical_error, ObjectStoreError::NotFound { .. }), + "unexpected canonical manifest error: {canonical_error}" + ); + + policy.lock().unwrap().clear_before_policy("fail-copy-once"); + let resolved = handler + .resolve_version_location(&base_path, version, object_store.inner.as_ref()) + .await + .expect("the retained staging manifest must allow finalization"); + assert_eq!(resolved.path, canonical_path); + + let finalized_location = external_store + .get_manifest_location(base_path.as_ref(), version) + .await + .expect("the external store must publish the canonical location"); + assert_eq!(finalized_location.path, canonical_path); + object_store + .inner + .head(&canonical_path) + .await + .expect("the canonical manifest must exist after finalization"); + + let staging_error = object_store + .inner + .head(&staging_path) + .await + .expect_err("successful finalization must clean up the staging manifest"); + assert!( + matches!(staging_error, ObjectStoreError::NotFound { .. }), + "unexpected staging manifest error: {staging_error}" + ); + } +} diff --git a/rust/lance-table/src/io/manifest.rs b/rust/lance-table/src/io/manifest.rs index 6a6bfd2724c..625fcf961cf 100644 --- a/rust/lance-table/src/io/manifest.rs +++ b/rust/lance-table/src/io/manifest.rs @@ -4,9 +4,11 @@ use async_trait::async_trait; use byteorder::{ByteOrder, LittleEndian}; use bytes::{Bytes, BytesMut}; -use lance_arrow::DataTypeExt; use lance_file::{ - previous::writer::ManifestProvider as PreviousManifestProvider, version::LanceFileVersion, + version::ConcreteFileVersion, + versions::v1::{ + encoding::write_schema_dictionaries, writer::ManifestProvider as V1ManifestProvider, + }, }; use object_store::ObjectStoreExt; use object_store::path::Path; @@ -17,7 +19,6 @@ use tracing::instrument; use lance_core::{Error, Result, datatypes::Schema}; use lance_io::{ - encodings::{Encoder, binary::BinaryEncoder, plain::PlainEncoder}, object_store::ObjectStore, traits::{WriteExt, Writer}, utils::read_message, @@ -168,6 +169,10 @@ async fn do_write_manifest( }; let pos = writer.write_protobuf(§ion).await?; manifest.index_section = Some(pos); + } else { + // No index section is written to this file, so an inherited offset + // would point at unrelated bytes in the new manifest file. + manifest.index_section = None; } // Write inline transaction if presented. @@ -176,6 +181,11 @@ async fn do_write_manifest( let pb_tx: pb::Transaction = tx.into(); let pos = writer.write_protobuf(&pb_tx).await?; manifest.transaction_section = Some(pos); + } else { + // No inline copy is written to this file. Clear any offset inherited + // from a previous manifest (e.g. via restore or clone), which would + // otherwise point at arbitrary bytes of the file being written. + manifest.transaction_section = None; } writer.write_struct(manifest).await @@ -188,45 +198,14 @@ pub async fn write_manifest( indices: Option>, transaction: Option, ) -> Result { - // Write dictionary values. - let max_field_id = manifest.schema.max_field_id().unwrap_or(-1); - let is_legacy_storage = manifest.should_use_legacy_format(); - for field_id in 0..max_field_id + 1 { - if let Some(field) = manifest.schema.mut_field_by_id(field_id) - && field.data_type().is_dictionary() - && is_legacy_storage - { - let dict_info = field.dictionary.as_mut().ok_or_else(|| { - Error::io(format!("Lance field {} misses dictionary info", field.name)) - })?; - - let value_arr = dict_info.values.as_ref().ok_or_else(|| { - Error::io(format!( - "Lance field {} is dictionary type, but misses the dictionary value array", - field.name - )) - })?; - - let data_type = value_arr.data_type(); - let pos = match data_type { - dt if dt.is_numeric() => { - let mut encoder = PlainEncoder::new(writer, dt); - encoder.encode(&[value_arr]).await? - } - dt if dt.is_binary_like() => { - let mut encoder = BinaryEncoder::new(writer); - encoder.encode(&[value_arr]).await? - } - _ => { - return Err(Error::schema(format!( - "Does not support {} as dictionary value type", - value_arr.data_type() - ))); - } - }; - dict_info.offset = pos; - dict_info.length = value_arr.len(); + match manifest.data_storage_format.version { + ConcreteFileVersion::V1 => { + write_schema_dictionaries(writer, &mut manifest.schema).await?; } + ConcreteFileVersion::V2_0 + | ConcreteFileVersion::V2_1 + | ConcreteFileVersion::V2_2 + | ConcreteFileVersion::V2_3 => {} } do_write_manifest(writer, manifest, indices, transaction).await @@ -237,7 +216,7 @@ pub async fn write_manifest( pub struct ManifestDescribing {} #[async_trait] -impl PreviousManifestProvider for ManifestDescribing { +impl V1ManifestProvider for ManifestDescribing { async fn store_schema( object_writer: &mut dyn Writer, schema: &Schema, @@ -245,7 +224,7 @@ impl PreviousManifestProvider for ManifestDescribing { let mut manifest = Manifest::new( schema.clone(), Arc::new(vec![]), - DataStorageFormat::new(LanceFileVersion::Legacy), + DataStorageFormat::new(ConcreteFileVersion::V1), HashMap::new(), ); let pos = do_write_manifest(object_writer, &mut manifest, None, None).await?; @@ -261,8 +240,8 @@ mod test { use crate::format::SelfDescribingFileReader; use arrow_schema::{DataType, Field as ArrowField, Schema as ArrowSchema}; use lance_file::format::{MAGIC, MAJOR_VERSION, MINOR_VERSION}; - use lance_file::previous::{ - reader::FileReader as PreviousFileReader, writer::FileWriter as PreviousFileWriter, + use lance_file::versions::v1::{ + reader::FileReader as V1FileReader, writer::FileWriter as V1FileWriter, }; use rand::{Rng, distr::Alphanumeric}; use tokio::io::AsyncWriteExt; @@ -324,6 +303,33 @@ mod test { test_roundtrip_manifest(1000, 1000).await; } + #[tokio::test] + async fn test_write_manifest_clears_unwritten_index_section() { + let store = ObjectStore::memory(); + let path = Path::from("/clear_unwritten_index_section"); + let mut writer = store.create(&path).await.unwrap(); + let mut manifest = Manifest::new( + Schema::default(), + Arc::new(vec![]), + DataStorageFormat::default(), + HashMap::new(), + ); + manifest.index_section = Some(42); + + let pos = write_manifest(writer.as_mut(), &mut manifest, None, None) + .await + .unwrap(); + writer + .write_magics(pos, MAJOR_VERSION, MINOR_VERSION, MAGIC) + .await + .unwrap(); + Writer::shutdown(writer.as_mut()).await.unwrap(); + + assert!(manifest.index_section.is_none()); + let roundtripped_manifest = read_manifest(&store, &path, None).await.unwrap(); + assert!(roundtripped_manifest.index_section.is_none()); + } + #[tokio::test] async fn test_update_schema_metadata() { let store = ObjectStore::memory(); @@ -335,7 +341,7 @@ mod test { false, )])); let schema = Schema::try_from(arrow_schema.as_ref()).unwrap(); - let mut file_writer = PreviousFileWriter::::try_new( + let mut file_writer = V1FileWriter::::try_new( &store, &path, schema.clone(), @@ -355,7 +361,7 @@ mod test { file_writer.finish_with_metadata(&metadata).await.unwrap(); let reader = store.open(&path).await.unwrap(); - let reader = PreviousFileReader::try_new_self_described_from_reader(reader.into(), None) + let reader = V1FileReader::try_new_self_described_from_reader(reader.into(), None) .await .unwrap(); let schema = ArrowSchema::from(reader.schema()); diff --git a/rust/lance-table/src/rowids.rs b/rust/lance-table/src/rowids.rs index ab5dac72b48..fc6bf08c063 100644 --- a/rust/lance-table/src/rowids.rs +++ b/rust/lance-table/src/rowids.rs @@ -98,11 +98,52 @@ impl From<&[u64]> for RowIdSequence { } } +/// Return some value that appears more than once in `row_ids`, if any. +/// +/// The already-sorted case is the common one for row id sequences, and is +/// checked in a single pass without allocating. +fn find_duplicate(row_ids: &[u64]) -> Option { + if row_ids.windows(2).all(|pair| pair[0] < pair[1]) { + return None; + } + let mut sorted = row_ids.to_vec(); + sorted.sort_unstable(); + sorted + .windows(2) + .find(|pair| pair[0] == pair[1]) + .map(|pair| pair[0]) +} + impl RowIdSequence { pub fn new() -> Self { Self::default() } + /// Build a sequence from row ids, rejecting duplicates within the sequence. + /// + /// The segment encodings represent a sorted run as a range plus its holes, + /// so a repeated value would be silently encoded as a shorter sequence with + /// a spurious hole. Callers assembling a sequence from untrusted input + /// should use this instead of the infallible `From` conversions, which + /// assume uniqueness. + /// + /// Row ids must also be unique across the dataset. That is not checked + /// here, and commit does not re-check it either. + pub fn try_from_iter(row_ids: impl IntoIterator) -> Result { + let row_ids: Vec = row_ids.into_iter().collect(); + if row_ids.is_empty() { + return Ok(Self::new()); + } + if let Some(duplicate) = find_duplicate(&row_ids) { + return Err(Error::invalid_input(format!( + "Row ids must be unique, but row id {} appears more than once in the sequence of {} row ids", + duplicate, + row_ids.len() + ))); + } + Ok(Self(vec![U64Segment::from_iter(row_ids)])) + } + pub fn iter(&self) -> impl DoubleEndedIterator + '_ { self.0.iter().flat_map(|segment| segment.iter()) } @@ -355,11 +396,7 @@ impl RowIdSequence { while (index - rows_passed) >= cur_seg_len { rows_passed += cur_seg_len; cur_seg = seg_iter.next(); - if let Some(cur_seg) = cur_seg { - cur_seg_len = cur_seg.len(); - } else { - return None; - } + cur_seg_len = cur_seg?.len(); } Some(cur_seg.unwrap().get(index - rows_passed).unwrap()) @@ -788,6 +825,50 @@ mod test { assert_eq!(iter.collect::>(), (0..10).collect::>()); } + #[rstest::rstest] + #[case::sorted_contiguous(vec![0, 1, 2, 3])] + #[case::sorted_with_gaps(vec![0, 2, 4])] + #[case::sparse(vec![0, 1_000_000])] + #[case::unsorted(vec![12, 11, 10])] + fn test_row_id_sequence_try_from_iter(#[case] row_ids: Vec) { + let sequence = RowIdSequence::try_from_iter(row_ids.clone()).unwrap(); + assert_eq!(sequence.len(), row_ids.len() as u64); + assert_eq!(sequence.iter().collect::>(), row_ids); + } + + #[test] + fn test_row_id_sequence_try_from_iter_contiguous_is_a_range() { + let sequence = RowIdSequence::try_from_iter(0..10).unwrap(); + assert_eq!(sequence.0, vec![U64Segment::Range(0..10)]); + } + + #[test] + fn test_row_id_sequence_try_from_iter_empty() { + let sequence = RowIdSequence::try_from_iter(std::iter::empty()).unwrap(); + assert_eq!(sequence.len(), 0); + assert!(sequence.is_empty()); + } + + #[rstest::rstest] + #[case::adjacent(vec![1, 1, 2])] + #[case::separated(vec![1, 2, 3, 1])] + #[case::unsorted(vec![5, 3, 5])] + fn test_row_id_sequence_try_from_iter_rejects_duplicates(#[case] row_ids: Vec) { + // Without validation these encode to a shorter sequence with a spurious + // hole rather than failing, so assert the error rather than the output. + let error = RowIdSequence::try_from_iter(row_ids).unwrap_err(); + assert!( + matches!(error, Error::InvalidInput { .. }), + "expected InvalidInput, got {:?}", + error + ); + assert!( + error.to_string().contains("must be unique"), + "unexpected message: {}", + error + ); + } + #[test] fn test_row_id_sequence_extend() { let mut sequence = RowIdSequence::from(0..10); diff --git a/rust/lance-table/src/rowids/index.rs b/rust/lance-table/src/rowids/index.rs index 4fed0e651c5..b8ba5f725f4 100644 --- a/rust/lance-table/src/rowids/index.rs +++ b/rust/lance-table/src/rowids/index.rs @@ -368,7 +368,10 @@ fn merge_overlapping_chunks(overlapping_chunks: Vec) -> Result, + frag_size_range: std::ops::Range, + ) -> impl Strategy, Arc)>> { + arbitrary_row_ids(num_fragments_range, frag_size_range) + .prop_flat_map(|row_ids| { + let num_rows = row_ids + .iter() + .map(|(_, sequence)| sequence.len() as usize) + .sum::(); + ( + Just(row_ids), + proptest::collection::vec(any::(), num_rows), + ) + }) + .prop_map(|(row_ids, deleted_rows)| { + let mut deleted_rows = deleted_rows.into_iter(); + row_ids + .into_iter() + .map(|(fragment_id, sequence)| { + let mut deletion_bitmap = roaring::RoaringBitmap::new(); + for offset in 0..sequence.len() as u32 { + if deleted_rows.next().unwrap() { + deletion_bitmap.insert(offset); + } + } + ( + fragment_id, + sequence, + Arc::new(DeletionVector::Bitmap(deletion_bitmap)), + ) + }) + .collect() + }) + } + #[test] fn test_large_range_segments_no_deletions() { // Simulates a real-world scenario: many fragments with large Range segments @@ -794,22 +833,29 @@ mod tests { proptest::proptest! { #[test] - fn test_new_index_robustness(row_ids in arbitrary_row_ids(0..5, 0..32)) { + fn test_new_index_robustness( + row_ids in arbitrary_row_ids_with_deletions(0..5, 0..32) + ) { let fragment_indices: Vec = row_ids .iter() - .map(|(frag_id, sequence)| FragmentRowIdIndex { + .map(|(frag_id, sequence, deletion_vector)| FragmentRowIdIndex { fragment_id: *frag_id, row_id_sequence: sequence.clone(), - deletion_vector: Arc::new(DeletionVector::default()), + deletion_vector: deletion_vector.clone(), }) .collect(); let index = RowIdIndex::new(&fragment_indices).unwrap(); - for (frag_id, sequence) in row_ids.iter() { + for (frag_id, sequence, deletion_vector) in row_ids.iter() { for (local_offset, row_id) in sequence.iter().enumerate() { + let expected = if deletion_vector.contains(local_offset as u32) { + None + } else { + Some(RowAddress::new_from_parts(*frag_id, local_offset as u32)) + }; prop_assert_eq!( index.get(row_id), - Some(RowAddress::new_from_parts(*frag_id, local_offset as u32)), + expected, "Row id {} in sequence {:?} not found in index {:?}", row_id, sequence, @@ -818,5 +864,63 @@ mod tests { } } } + + #[test] + fn test_new_index_moved_row_id( + row_id in any::(), + source_fragment in 0u32..1024, + fragment_delta in 1u32..1024, + ) { + let target_fragment = source_fragment + fragment_delta; + let fragment_indices = [ + FragmentRowIdIndex { + fragment_id: source_fragment, + row_id_sequence: Arc::new(RowIdSequence::from(&[row_id][..])), + deletion_vector: Arc::new(DeletionVector::Bitmap( + roaring::RoaringBitmap::from_iter([0]), + )), + }, + FragmentRowIdIndex { + fragment_id: target_fragment, + row_id_sequence: Arc::new(RowIdSequence::from(&[row_id][..])), + deletion_vector: Arc::new(DeletionVector::default()), + }, + ]; + + let index = RowIdIndex::new(&fragment_indices).unwrap(); + prop_assert_eq!( + index.get(row_id), + Some(RowAddress::new_from_parts(target_fragment, 0)) + ); + } + + #[test] + fn test_new_index_rejects_duplicate_live_row_id( + row_id in any::(), + first_fragment in 0u32..1024, + fragment_delta in 1u32..1024, + ) { + let second_fragment = first_fragment + fragment_delta; + let fragment_indices = [ + FragmentRowIdIndex { + fragment_id: first_fragment, + row_id_sequence: Arc::new(RowIdSequence::from(&[row_id][..])), + deletion_vector: Arc::new(DeletionVector::default()), + }, + FragmentRowIdIndex { + fragment_id: second_fragment, + row_id_sequence: Arc::new(RowIdSequence::from(&[row_id][..])), + deletion_vector: Arc::new(DeletionVector::default()), + }, + ]; + + let error = RowIdIndex::new(&fragment_indices).unwrap_err(); + let is_internal = matches!(&error, Error::Internal { .. }); + let expected_message = + format!("stable row id {row_id} is live in multiple fragments"); + let error_message = error.to_string(); + prop_assert!(is_internal); + prop_assert!(error_message.contains(&expected_message)); + } } } diff --git a/rust/lance-table/src/rowids/segment.rs b/rust/lance-table/src/rowids/segment.rs index 6fba8599016..c3b8d24b8a4 100644 --- a/rust/lance-table/src/rowids/segment.rs +++ b/rust/lance-table/src/rowids/segment.rs @@ -282,23 +282,19 @@ impl U64Segment { } /// Get the min and max value of the segment, excluding tombstones. + /// + /// Returns `None` for an empty segment, which has no extrema. Decoding accepts an + /// empty encoding of every variant, so no arm here may assume it holds a value. pub fn range(&self) -> Option> { match self { - Self::Range(range) if range.is_empty() => None, Self::Range(range) | Self::RangeWithBitmap { range, .. } - | Self::RangeWithHoles { range, .. } => Some(range.start..=(range.end - 1)), - Self::SortedArray(array) => { - // We can assume that the array is sorted. - let min_value = array.first().unwrap(); - let max_value = array.last().unwrap(); - Some(min_value..=max_value) - } - Self::Array(array) => { - let min_value = array.min().unwrap(); - let max_value = array.max().unwrap(); - Some(min_value..=max_value) + | Self::RangeWithHoles { range, .. } => { + (!range.is_empty()).then(|| range.start..=(range.end - 1)) } + // We can assume that the array is sorted. + Self::SortedArray(array) => Some(array.first()?..=array.last()?), + Self::Array(array) => Some(array.min()?..=array.max()?), } } @@ -765,6 +761,30 @@ mod test { ); } + /// Decoding accepts an empty encoding of every variant, so `range()` must report the + /// absence of extrema for all of them rather than unwrapping a value or computing + /// `end - 1` on a zero-length range. + #[test] + fn test_empty_segments_have_no_range() { + let empty: Vec = Vec::new(); + let segments = [ + U64Segment::Range(5..5), + U64Segment::RangeWithHoles { + range: 0..0, + holes: empty.clone().into(), + }, + U64Segment::RangeWithBitmap { + range: 0..0, + bitmap: Bitmap::new_empty(0), + }, + U64Segment::SortedArray(empty.clone().into()), + U64Segment::Array(empty.into()), + ]; + for segment in segments { + assert_eq!(segment.range(), None, "{segment:?} should have no range"); + } + } + #[test] fn test_segment_overflow_boundary() { // Sparse range spanning i64::MAX — the original overflow reproducer. diff --git a/rust/lance-table/src/rowids/serde.rs b/rust/lance-table/src/rowids/serde.rs index c087fa603dc..bad9e35a8e8 100644 --- a/rust/lance-table/src/rowids/serde.rs +++ b/rust/lance-table/src/rowids/serde.rs @@ -7,16 +7,87 @@ use lance_core::{Error, Result}; use super::{RowIdSequence, U64Segment, encoded_array::EncodedU64Array}; use prost::Message; +const ROW_ID_METADATA: &str = "row ID metadata"; + +fn corrupt_row_id_metadata(message: impl Into) -> Error { + Error::corrupt_file_named(ROW_ID_METADATA, message) +} + +fn validate_range(segment_type: &str, start: u64, end: u64) -> Result { + let len = end.checked_sub(start).ok_or_else(|| { + corrupt_row_id_metadata(format!( + "{segment_type} range start {start} exceeds end {end}" + )) + })?; + usize::try_from(len).map_err(|_| { + corrupt_row_id_metadata(format!( + "{segment_type} range length {len} for start {start} and end {end} exceeds usize::MAX" + )) + }) +} + +fn validate_packed_array_length(array_type: &str, byte_len: usize, width: usize) -> Result<()> { + if !byte_len.is_multiple_of(width) { + return Err(corrupt_row_id_metadata(format!( + "encoded {array_type} array byte length {byte_len} is not a multiple of element width {width}" + ))); + } + Ok(()) +} + +fn first_descending_pair(array: &EncodedU64Array) -> Option<(usize, u64, u64)> { + match array { + EncodedU64Array::U16 { offsets, .. } => offsets + .windows(2) + .position(|pair| pair[0] > pair[1]) + .map(|index| (index, offsets[index] as u64, offsets[index + 1] as u64)), + EncodedU64Array::U32 { offsets, .. } => offsets + .windows(2) + .position(|pair| pair[0] > pair[1]) + .map(|index| (index, offsets[index] as u64, offsets[index + 1] as u64)), + EncodedU64Array::U64(values) => values + .windows(2) + .position(|pair| pair[0] > pair[1]) + .map(|index| (index, values[index], values[index + 1])), + } +} + +fn first_non_increasing_pair(array: &EncodedU64Array) -> Option<(usize, u64, u64)> { + let mut values = array.iter(); + let previous = values.next()?; + values + .scan(previous, |previous, value| { + let pair = (*previous, value); + *previous = value; + Some(pair) + }) + .enumerate() + .find_map(|(index, (previous, next))| (previous >= next).then_some((index, previous, next))) +} + impl TryFrom for RowIdSequence { type Error = Error; fn try_from(pb: pb::RowIdSequence) -> Result { - Ok(Self( - pb.segments - .into_iter() - .map(U64Segment::try_from) - .collect::>>()?, - )) + let segments = pb + .segments + .into_iter() + .map(U64Segment::try_from) + .collect::>>()?; + // Each segment length fits a usize on its own, but the total need not fit a u64. + // Reject that here so `RowIdSequence::len()` stays total for anything decoded. + segments + .iter() + .try_fold(0_u64, |total, segment| { + total.checked_add(segment.len() as u64) + }) + .ok_or_else(|| { + corrupt_row_id_metadata(format!( + "row ID sequence of {} segments has a total length exceeding u64::MAX", + segments.len() + )) + })?; + Ok(Self(segments)) } } @@ -27,30 +98,74 @@ impl TryFrom for U64Segment { use pb::u64_segment as pb_seg; use pb::u64_segment::Segment::*; match pb.segment { - Some(Range(pb_seg::Range { start, end })) => Ok(Self::Range(start..end)), + Some(Range(pb_seg::Range { start, end })) => { + validate_range("Range", start, end)?; + Ok(Self::Range(start..end)) + } Some(RangeWithHoles(pb_seg::RangeWithHoles { start, end, holes })) => { + validate_range("RangeWithHoles", start, end)?; let holes = holes - .ok_or_else(|| Error::invalid_input("missing hole"))? + .ok_or_else(|| { + corrupt_row_id_metadata("RangeWithHoles is missing its holes array") + })? .try_into()?; + if let Some((index, previous, next)) = first_non_increasing_pair(&holes) { + return Err(corrupt_row_id_metadata(format!( + "RangeWithHoles values are not strictly increasing at indices {index} and {}: {previous} is not less than {next}", + index + 1 + ))); + } + if let Some(hole) = holes.iter().find(|hole| *hole < start || *hole >= end) { + return Err(corrupt_row_id_metadata(format!( + "RangeWithHoles hole {hole} is outside the range {start}..{end}" + ))); + } Ok(Self::RangeWithHoles { range: start..end, holes, }) } Some(RangeWithBitmap(pb_seg::RangeWithBitmap { start, end, bitmap })) => { + let range_len = validate_range("RangeWithBitmap", start, end)?; + let expected_bitmap_len = range_len.div_ceil(8); + if bitmap.len() != expected_bitmap_len { + return Err(corrupt_row_id_metadata(format!( + "RangeWithBitmap byte length {} does not match expected {expected_bitmap_len} for range start {start}, end {end}, and length {range_len}", + bitmap.len() + ))); + } + let remainder = range_len % 8; + if remainder != 0 { + let padding_mask = !((1_u8 << remainder) - 1); + let last_byte = bitmap[expected_bitmap_len - 1]; + if last_byte & padding_mask != 0 { + return Err(corrupt_row_id_metadata(format!( + "RangeWithBitmap padding bits must be zero for range start {start}, end {end}, and length {range_len}: last byte {last_byte:#04x} has padding mask {padding_mask:#04x} set" + ))); + } + } Ok(Self::RangeWithBitmap { range: start..end, bitmap: Bitmap { data: bitmap, - len: (end - start) as usize, + len: range_len, }, }) } - Some(SortedArray(array)) => Ok(Self::SortedArray(EncodedU64Array::try_from(array)?)), + Some(SortedArray(array)) => { + let array = EncodedU64Array::try_from(array)?; + if let Some((index, previous, next)) = first_descending_pair(&array) { + return Err(corrupt_row_id_metadata(format!( + "SortedArray values are not sorted at indices {index} and {}: {previous} exceeds {next}", + index + 1 + ))); + } + Ok(Self::SortedArray(array)) + } Some(Array(array)) => Ok(Self::Array(EncodedU64Array::try_from(array)?)), // TODO: why non-exhaustive? // Some(_) => Err(Error::invalid_input("unknown segment type")), - None => Err(Error::invalid_input("missing segment type")), + None => Err(corrupt_row_id_metadata("missing row ID segment type")), } } } @@ -63,32 +178,37 @@ impl TryFrom for EncodedU64Array { use pb::encoded_u64_array::Array::*; match pb.array { Some(U16Array(pb_arr::U16Array { base, offsets })) => { - assert!( - offsets.len() % 2 == 0, - "Must have even number of bytes to store u16 array" - ); + validate_packed_array_length("u16", offsets.len(), 2)?; let offsets = offsets .chunks_exact(2) .map(|chunk| u16::from_le_bytes([chunk[0], chunk[1]])) - .collect(); + .collect::>(); + if let Some(max_offset) = offsets.iter().copied().max() + && base.checked_add(u64::from(max_offset)).is_none() + { + return Err(corrupt_row_id_metadata(format!( + "U16Array base {base} plus maximum offset {max_offset} overflows u64" + ))); + } Ok(Self::U16 { base, offsets }) } Some(U32Array(pb_arr::U32Array { base, offsets })) => { - assert!( - offsets.len() % 4 == 0, - "Must have even number of bytes to store u32 array" - ); + validate_packed_array_length("u32", offsets.len(), 4)?; let offsets = offsets .chunks_exact(4) .map(|chunk| u32::from_le_bytes([chunk[0], chunk[1], chunk[2], chunk[3]])) - .collect(); + .collect::>(); + if let Some(max_offset) = offsets.iter().copied().max() + && base.checked_add(u64::from(max_offset)).is_none() + { + return Err(corrupt_row_id_metadata(format!( + "U32Array base {base} plus maximum offset {max_offset} overflows u64" + ))); + } Ok(Self::U32 { base, offsets }) } Some(U64Array(pb_arr::U64Array { values })) => { - assert!( - values.len() % 8 == 0, - "Must have even number of bytes to store u64 array" - ); + validate_packed_array_length("u64", values.len(), 8)?; let values = values .chunks_exact(8) .map(|chunk| { @@ -102,7 +222,7 @@ impl TryFrom for EncodedU64Array { } // TODO: shouldn't this enum be non-exhaustive? // Some(_) => Err(Error::invalid_input("unknown array type")), - None => Err(Error::invalid_input("missing array type")), + None => Err(corrupt_row_id_metadata("missing encoded row ID array type")), } } } @@ -199,7 +319,9 @@ pub fn write_row_ids(sequence: &RowIdSequence) -> Vec { /// Deserialize a rowid sequence from some bytes. pub fn read_row_ids(reader: &[u8]) -> Result { - let pb_sequence = pb::RowIdSequence::decode(reader)?; + let pb_sequence = pb::RowIdSequence::decode(reader).map_err(|error| { + corrupt_row_id_metadata(format!("failed to decode row ID sequence: {error}")) + })?; RowIdSequence::try_from(pb_sequence) } @@ -207,6 +329,51 @@ pub fn read_row_ids(reader: &[u8]) -> Result { mod test { use super::*; use pretty_assertions::assert_eq; + use proptest::prelude::*; + use rstest::rstest; + + fn read_segment(segment: pb::u64_segment::Segment) -> Result { + let sequence = pb::RowIdSequence { + segments: vec![pb::U64Segment { + segment: Some(segment), + }], + }; + read_row_ids(&sequence.encode_to_vec()) + } + + fn assert_corrupt_segment(segment: pb::u64_segment::Segment, expected_message: &str) { + let error = read_segment(segment).unwrap_err(); + assert!(matches!(&error, Error::CorruptFile { .. })); + assert!( + error.to_string().contains(expected_message), + "expected error containing {expected_message:?}, got {error}" + ); + } + + /// Each segment length fits a usize, but the aggregate does not fit a u64. Accepting + /// this would leave `RowIdSequence::len()` overflowing for callers such as + /// `Dataset::validate()`. + #[test] + fn test_reject_sequence_length_overflow() { + let segment = || pb::U64Segment { + segment: Some(pb::u64_segment::Segment::Range(pb::u64_segment::Range { + start: 0, + end: u64::MAX, + })), + }; + let sequence = pb::RowIdSequence { + segments: vec![segment(), segment()], + }; + + let error = read_row_ids(&sequence.encode_to_vec()).unwrap_err(); + assert!(matches!(&error, Error::CorruptFile { .. })); + assert!( + error + .to_string() + .contains("total length exceeding u64::MAX"), + "got {error}" + ); + } #[test] fn test_write_read_row_ids() { @@ -216,9 +383,11 @@ mod test { range: 100..200, holes: EncodedU64Array::U64(vec![104, 108, 150]), }); + let mut bitmap = Bitmap::new_empty(100); + bitmap.set(99); sequence.0.push(U64Segment::RangeWithBitmap { range: 200..300, - bitmap: Bitmap::new_empty(100), + bitmap, }); sequence .0 @@ -228,7 +397,7 @@ mod test { })); sequence .0 - .push(U64Segment::Array(EncodedU64Array::U64(vec![1, 2, 3]))); + .push(U64Segment::Array(EncodedU64Array::U64(vec![3, 1, 2]))); let serialized = write_row_ids(&sequence); @@ -236,4 +405,275 @@ mod test { assert_eq!(sequence.0, sequence2.0); } + + proptest! { + #[test] + fn test_row_id_sequence_len_round_trips( + values in proptest::collection::btree_set(any::(), 0..128) + ) { + let values = values.into_iter().collect::>(); + let sequence = RowIdSequence::from(values.as_slice()); + let deserialized = read_row_ids(&write_row_ids(&sequence)).unwrap(); + + prop_assert_eq!(deserialized.len(), sequence.len()); + prop_assert_eq!(deserialized.iter().collect::>(), values); + } + + #[test] + fn test_rejects_wrong_range_with_bitmap_length( + range_len in 1usize..512, + ) { + let expected_len = range_len.div_ceil(8); + for actual_len in [expected_len - 1, expected_len + 1] { + let segment = pb::u64_segment::Segment::RangeWithBitmap( + pb::u64_segment::RangeWithBitmap { + start: 0, + end: range_len as u64, + bitmap: vec![0; actual_len], + }, + ); + + let error = read_segment(segment).unwrap_err(); + let is_corrupt_file = matches!(&error, Error::CorruptFile { .. }); + prop_assert!(is_corrupt_file); + prop_assert!(error.to_string().contains("byte length")); + } + } + + #[test] + fn test_rejects_range_with_bitmap_padding_bits( + full_bytes in 0usize..64, + valid_bits in 1usize..8, + ) { + let range_len = full_bytes * 8 + valid_bits; + let mut bitmap = vec![0; full_bytes + 1]; + bitmap[full_bytes] = 1 << valid_bits; + let segment = pb::u64_segment::Segment::RangeWithBitmap( + pb::u64_segment::RangeWithBitmap { + start: 0, + end: range_len as u64, + bitmap, + }, + ); + + let error = read_segment(segment).unwrap_err(); + let is_corrupt_file = matches!(&error, Error::CorruptFile { .. }); + prop_assert!(is_corrupt_file); + prop_assert!(error.to_string().contains("padding bits must be zero")); + } + + #[test] + fn test_rejects_reversed_range_with_bitmap( + start in 1u64..u64::MAX, + ) { + let segment = pb::u64_segment::Segment::RangeWithBitmap( + pb::u64_segment::RangeWithBitmap { + start, + end: start - 1, + bitmap: Vec::new(), + }, + ); + + let error = read_segment(segment).unwrap_err(); + let is_corrupt_file = matches!(&error, Error::CorruptFile { .. }); + prop_assert!(is_corrupt_file); + prop_assert!(error.to_string().contains("range start")); + } + + #[test] + fn test_rejects_misaligned_encoded_array_bytes( + encoding in 0u8..3, + element_count in 0usize..16, + ) { + let width = match encoding { + 0 => 2, + 1 => 4, + _ => 8, + }; + let bytes = vec![0; element_count * width + 1]; + let array = match encoding { + 0 => pb::encoded_u64_array::Array::U16Array( + pb::encoded_u64_array::U16Array { base: 0, offsets: bytes }, + ), + 1 => pb::encoded_u64_array::Array::U32Array( + pb::encoded_u64_array::U32Array { base: 0, offsets: bytes }, + ), + _ => pb::encoded_u64_array::Array::U64Array( + pb::encoded_u64_array::U64Array { values: bytes }, + ), + }; + let segment = pb::u64_segment::Segment::Array(pb::EncodedU64Array { + array: Some(array), + }); + + let error = read_segment(segment).unwrap_err(); + let is_corrupt_file = matches!(&error, Error::CorruptFile { .. }); + prop_assert!(is_corrupt_file); + prop_assert!(error.to_string().contains("byte length")); + } + } + + #[test] + fn test_rejects_encoded_offset_overflow() { + use pb::encoded_u64_array as pb_array; + + let arrays = [ + pb_array::Array::U16Array(pb_array::U16Array { + base: u64::MAX, + offsets: 1u16.to_le_bytes().to_vec(), + }), + pb_array::Array::U32Array(pb_array::U32Array { + base: u64::MAX, + offsets: 1u32.to_le_bytes().to_vec(), + }), + ]; + for array in arrays { + assert_corrupt_segment( + pb::u64_segment::Segment::Array(pb::EncodedU64Array { array: Some(array) }), + "overflows u64", + ); + } + } + + #[rstest] + #[case::descending(vec![6, 5], "not strictly increasing")] + #[case::duplicate(vec![5, 5], "not strictly increasing")] + #[case::below_range(vec![4], "outside the range")] + #[case::at_end(vec![7], "outside the range")] + fn test_rejects_invalid_range_with_holes(#[case] values: Vec, #[case] message: &str) { + let values = values.into_iter().flat_map(u64::to_le_bytes).collect(); + assert_corrupt_segment( + pb::u64_segment::Segment::RangeWithHoles(pb::u64_segment::RangeWithHoles { + start: 5, + end: 7, + holes: Some(pb::EncodedU64Array { + array: Some(pb::encoded_u64_array::Array::U64Array( + pb::encoded_u64_array::U64Array { values }, + )), + }), + }), + message, + ); + } + + #[test] + fn test_rejects_missing_range_with_holes_array() { + assert_corrupt_segment( + pb::u64_segment::Segment::RangeWithHoles(pb::u64_segment::RangeWithHoles { + start: 5, + end: 7, + holes: None, + }), + "missing its holes array", + ); + } + + #[rstest] + #[case::u16( + pb::encoded_u64_array::Array::U16Array(pb::encoded_u64_array::U16Array { + base: 0, + offsets: vec![1], + }), + "encoded u16 array byte length 1 is not a multiple of element width 2" + )] + #[case::u32( + pb::encoded_u64_array::Array::U32Array(pb::encoded_u64_array::U32Array { + base: 0, + offsets: vec![1, 2, 3], + }), + "encoded u32 array byte length 3 is not a multiple of element width 4" + )] + #[case::u64( + pb::encoded_u64_array::Array::U64Array(pb::encoded_u64_array::U64Array { + values: vec![1, 2, 3, 4, 5, 6, 7], + }), + "encoded u64 array byte length 7 is not a multiple of element width 8" + )] + fn test_rejects_misaligned_encoded_array( + #[case] array: pb::encoded_u64_array::Array, + #[case] message: &str, + ) { + assert_corrupt_segment( + pb::u64_segment::Segment::Array(pb::EncodedU64Array { array: Some(array) }), + message, + ); + } + + #[rstest] + #[case::range(pb::u64_segment::Segment::Range(pb::u64_segment::Range { + start: 10, + end: 9, + }))] + #[case::range_with_holes(pb::u64_segment::Segment::RangeWithHoles( + pb::u64_segment::RangeWithHoles { + start: 10, + end: 9, + holes: Some(pb::EncodedU64Array { + array: Some(pb::encoded_u64_array::Array::U64Array( + pb::encoded_u64_array::U64Array { values: Vec::new() }, + )), + }), + } + ))] + #[case::range_with_bitmap(pb::u64_segment::Segment::RangeWithBitmap( + pb::u64_segment::RangeWithBitmap { + start: 10, + end: 9, + bitmap: Vec::new(), + } + ))] + fn test_rejects_reversed_range(#[case] segment: pb::u64_segment::Segment) { + assert_corrupt_segment(segment, "range start 10 exceeds end 9"); + } + + #[rstest] + #[case::short(vec![0])] + #[case::long(vec![0, 0, 0])] + fn test_rejects_incorrect_bitmap_length(#[case] bitmap: Vec) { + assert_corrupt_segment( + pb::u64_segment::Segment::RangeWithBitmap(pb::u64_segment::RangeWithBitmap { + start: 5, + end: 14, + bitmap, + }), + "does not match expected 2 for range start 5, end 14, and length 9", + ); + } + + #[test] + fn test_rejects_set_bitmap_padding_bits() { + assert_corrupt_segment( + pb::u64_segment::Segment::RangeWithBitmap(pb::u64_segment::RangeWithBitmap { + start: 5, + end: 14, + bitmap: vec![0xff, 0x03], + }), + "padding bits must be zero", + ); + } + + #[rstest] + #[case::u16(pb::encoded_u64_array::Array::U16Array( + pb::encoded_u64_array::U16Array { + base: 100, + offsets: vec![2, 0, 1, 0], + } + ))] + #[case::u32(pb::encoded_u64_array::Array::U32Array( + pb::encoded_u64_array::U32Array { + base: 100, + offsets: vec![2, 0, 0, 0, 1, 0, 0, 0], + } + ))] + #[case::u64(pb::encoded_u64_array::Array::U64Array( + pb::encoded_u64_array::U64Array { + values: vec![2, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0], + } + ))] + fn test_rejects_unsorted_sorted_array(#[case] array: pb::encoded_u64_array::Array) { + assert_corrupt_segment( + pb::u64_segment::Segment::SortedArray(pb::EncodedU64Array { array: Some(array) }), + "SortedArray values are not sorted at indices 0 and 1: 2 exceeds 1", + ); + } } diff --git a/rust/lance-table/src/system_index/mem_wal.rs b/rust/lance-table/src/system_index/mem_wal.rs index 1d82fd9e44f..7cf4eb76958 100644 --- a/rust/lance-table/src/system_index/mem_wal.rs +++ b/rust/lance-table/src/system_index/mem_wal.rs @@ -15,45 +15,45 @@ pub const MEM_WAL_INDEX_NAME: &str = "__lance_mem_wal"; /// Type alias for shard identifier (UUID v4). pub type ShardId = Uuid; -/// A flushed MemTable generation and its storage location. +/// An SSTable: the immutable result of flushing a MemTable, stored as a Lance dataset. #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, DeepSizeOf)] -pub struct FlushedGeneration { +pub struct SsTable { pub generation: u64, pub path: String, } -impl From<&FlushedGeneration> for pb::FlushedGeneration { - fn from(fg: &FlushedGeneration) -> Self { +impl From<&SsTable> for pb::SsTable { + fn from(sstable: &SsTable) -> Self { Self { - generation: fg.generation, - path: fg.path.clone(), + generation: sstable.generation, + path: sstable.path.clone(), } } } -impl From for FlushedGeneration { - fn from(fg: pb::FlushedGeneration) -> Self { +impl From for SsTable { + fn from(sstable: pb::SsTable) -> Self { Self { - generation: fg.generation, - path: fg.path, + generation: sstable.generation, + path: sstable.path, } } } -/// A shard's merged generation, used in MemWalIndexDetails. +/// A pointer to the latest SSTable compacted for a shard. #[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Hash, Serialize, Deserialize)] -pub struct MergedGeneration { +pub struct CompactedSsTable { pub shard_id: Uuid, pub generation: u64, } -impl DeepSizeOf for MergedGeneration { +impl DeepSizeOf for CompactedSsTable { fn deep_size_of_children(&self, _context: &mut lance_core::deepsize::Context) -> usize { 0 // UUID is 16 bytes fixed size, no heap allocations } } -impl MergedGeneration { +impl CompactedSsTable { pub fn new(shard_id: Uuid, generation: u64) -> Self { Self { shard_id, @@ -62,41 +62,41 @@ impl MergedGeneration { } } -impl From<&MergedGeneration> for pb::MergedGeneration { - fn from(mg: &MergedGeneration) -> Self { +impl From<&CompactedSsTable> for pb::CompactedSsTable { + fn from(sstable: &CompactedSsTable) -> Self { Self { - shard_id: Some((&mg.shard_id).into()), - generation: mg.generation, + shard_id: Some((&sstable.shard_id).into()), + generation: sstable.generation, } } } -impl TryFrom for MergedGeneration { +impl TryFrom for CompactedSsTable { type Error = Error; - fn try_from(mg: pb::MergedGeneration) -> lance_core::Result { - let shard_id = mg + fn try_from(sstable: pb::CompactedSsTable) -> lance_core::Result { + let shard_id = sstable .shard_id .as_ref() .map(Uuid::try_from) - .ok_or_else(|| Error::invalid_input("Missing shard_id in MergedGeneration"))??; + .ok_or_else(|| Error::invalid_input("Missing shard_id in CompactedSsTable"))??; Ok(Self { shard_id, - generation: mg.generation, + generation: sstable.generation, }) } } -/// Tracks which merged generation a base table index has been rebuilt to cover. -/// Used to determine whether to read from flushed MemTable indexes or base table. +/// Tracks which compacted SSTable generation a base table index covers. +/// Used to determine whether to read from SSTable indexes or base table. #[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize, DeepSizeOf)] pub struct IndexCatchupProgress { pub index_name: String, - pub caught_up_generations: Vec, + pub caught_up_generations: Vec, } impl IndexCatchupProgress { - pub fn new(index_name: String, caught_up_generations: Vec) -> Self { + pub fn new(index_name: String, caught_up_generations: Vec) -> Self { Self { index_name, caught_up_generations, @@ -108,8 +108,8 @@ impl IndexCatchupProgress { pub fn caught_up_generation_for_shard(&self, shard_id: &Uuid) -> Option { self.caught_up_generations .iter() - .find(|mg| &mg.shard_id == shard_id) - .map(|mg| mg.generation) + .find(|sstable| &sstable.shard_id == shard_id) + .map(|sstable| sstable.generation) } } @@ -120,7 +120,7 @@ impl From<&IndexCatchupProgress> for pb::IndexCatchupProgress { caught_up_generations: icp .caught_up_generations .iter() - .map(|mg| mg.into()) + .map(|sstable| sstable.into()) .collect(), } } @@ -135,7 +135,7 @@ impl TryFrom for IndexCatchupProgress { caught_up_generations: icp .caught_up_generations .into_iter() - .map(MergedGeneration::try_from) + .map(CompactedSsTable::try_from) .collect::>()?, }) } @@ -198,7 +198,7 @@ pub struct ShardManifest { /// 1-based. pub wal_entry_position_last_seen: u64, pub current_generation: u64, - pub flushed_generations: Vec, + pub sstables: Vec, /// Lifecycle status (drop-table 2PC). Defaults to `Active`; preserved /// across claims via `..base` so only fresh constructions set it. pub status: ShardStatus, @@ -207,7 +207,7 @@ pub struct ShardManifest { impl DeepSizeOf for ShardManifest { fn deep_size_of_children(&self, context: &mut lance_core::deepsize::Context) -> usize { self.shard_field_values.deep_size_of_children(context) - + self.flushed_generations.deep_size_of_children(context) + + self.sstables.deep_size_of_children(context) } } @@ -229,7 +229,7 @@ impl From<&ShardManifest> for pb::ShardManifest { replay_after_wal_entry_position: rm.replay_after_wal_entry_position, wal_entry_position_last_seen: rm.wal_entry_position_last_seen, current_generation: rm.current_generation, - flushed_generations: rm.flushed_generations.iter().map(|fg| fg.into()).collect(), + sstables: rm.sstables.iter().map(|sstable| sstable.into()).collect(), status: rm.status.to_i32(), } } @@ -258,11 +258,7 @@ impl TryFrom for ShardManifest { replay_after_wal_entry_position: rm.replay_after_wal_entry_position, wal_entry_position_last_seen: rm.wal_entry_position_last_seen, current_generation: rm.current_generation, - flushed_generations: rm - .flushed_generations - .into_iter() - .map(FlushedGeneration::from) - .collect(), + sstables: rm.sstables.into_iter().map(SsTable::from).collect(), status: ShardStatus::from_i32(rm.status), }) } @@ -338,7 +334,7 @@ pub struct MemWalIndexDetails { pub inline_snapshots: Option>, pub sharding_specs: Vec, pub maintained_indexes: Vec, - pub merged_generations: Vec, + pub compacted_sstables: Vec, pub index_catchup: Vec, /// Default `ShardWriter` configuration values for this MemWAL index. /// @@ -357,10 +353,10 @@ impl From<&MemWalIndexDetails> for pb::MemWalIndexDetails { inline_snapshots: details.inline_snapshots.clone(), sharding_specs: details.sharding_specs.iter().map(|rs| rs.into()).collect(), maintained_indexes: details.maintained_indexes.clone(), - merged_generations: details - .merged_generations + compacted_sstables: details + .compacted_sstables .iter() - .map(|mg| mg.into()) + .map(|sstable| sstable.into()) .collect(), index_catchup: details.index_catchup.iter().map(|icp| icp.into()).collect(), writer_config_defaults: details.writer_config_defaults.clone(), @@ -382,10 +378,10 @@ impl TryFrom for MemWalIndexDetails { .map(ShardingSpec::from) .collect(), maintained_indexes: details.maintained_indexes, - merged_generations: details - .merged_generations + compacted_sstables: details + .compacted_sstables .into_iter() - .map(MergedGeneration::try_from) + .map(CompactedSsTable::try_from) .collect::>()?, index_catchup: details .index_catchup @@ -408,12 +404,12 @@ impl MemWalIndex { Self { details } } - pub fn merged_generation_for_shard(&self, shard_id: &Uuid) -> Option { + pub fn compacted_generation_for_shard(&self, shard_id: &Uuid) -> Option { self.details - .merged_generations + .compacted_sstables .iter() - .find(|mg| &mg.shard_id == shard_id) - .map(|mg| mg.generation) + .find(|sstable| &sstable.shard_id == shard_id) + .map(|sstable| sstable.generation) } /// Get the caught up generation for a specific index and shard. @@ -426,13 +422,21 @@ impl MemWalIndex { .and_then(|icp| icp.caught_up_generation_for_shard(shard_id)) } - /// Check if an index is fully caught up for a shard. - /// Returns true if the index covers all merged data for the shard. - pub fn is_index_caught_up(&self, index_name: &str, shard_id: &Uuid) -> bool { - let merged_gen = self.merged_generation_for_shard(shard_id).unwrap_or(0); + /// Whether an index covers all compacted data for a shard, under **legacy** + /// semantics. + /// + /// A missing `index_catchup` entry is read here as "fully caught up". That + /// is only correct for a table without the index-catchup feature bit. On + /// an activated table a missing entry means *unknown*: the index has not + /// recorded anything, so its SSTables must be retained and a repair + /// scheduled. This accessor cannot see the manifest, so it cannot make that + /// distinction -- callers must select the semantics from the feature bit, + /// and the name says which one they get here. + pub fn is_index_caught_up_legacy(&self, index_name: &str, shard_id: &Uuid) -> bool { + let compacted_gen = self.compacted_generation_for_shard(shard_id).unwrap_or(0); let caught_up_gen = self.index_caught_up_generation(index_name, shard_id); - // If not tracked in index_catchup, assumed fully caught up - caught_up_gen.is_none_or(|generation| generation >= merged_gen) + // Missing means "caught up" only because this is the legacy reading. + caught_up_gen.is_none_or(|generation| generation >= compacted_gen) } } diff --git a/rust/lance-table/src/utils/stream.rs b/rust/lance-table/src/utils/stream.rs index f6fbbd45a61..849f5a0975f 100644 --- a/rust/lance-table/src/utils/stream.rs +++ b/rust/lance-table/src/utils/stream.rs @@ -1,18 +1,18 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright The Lance Authors -use std::sync::Arc; +use std::{fmt, sync::Arc}; use arrow_array::{BooleanArray, RecordBatch, RecordBatchOptions, UInt64Array, make_array}; use arrow_buffer::NullBuffer; use futures::{ FutureExt, Stream, StreamExt, - future::BoxFuture, + future::{BoxFuture, Shared}, stream::{BoxStream, FuturesOrdered}, }; use lance_arrow::RecordBatchExt; use lance_core::{ - ROW_ADDR, ROW_ADDR_FIELD, ROW_CREATED_AT_VERSION_FIELD, ROW_ID, ROW_ID_FIELD, + Error, ROW_ADDR, ROW_ADDR_FIELD, ROW_CREATED_AT_VERSION_FIELD, ROW_ID, ROW_ID_FIELD, ROW_LAST_UPDATED_AT_VERSION_FIELD, Result, utils::{address::RowAddress, deletion::DeletionVector}, }; @@ -31,27 +31,124 @@ pub struct ReadBatchTask { pub type ReadBatchTaskStream = BoxStream<'static, ReadBatchTask>; pub type ReadBatchFutStream = BoxStream<'static, ReadBatchFut>; +type SharedReadBatchFut = Shared>>>; + +#[derive(Debug)] +struct SharedReadError(Arc); + +impl fmt::Display for SharedReadError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + self.0.fmt(f) + } +} + +impl std::error::Error for SharedReadError { + fn source(&self) -> Option<&(dyn std::error::Error + 'static)> { + Some(self.0.as_ref()) + } +} + +struct PendingReadBatch { + task: Option, + shared_task: Option, + offset: u32, + num_rows: u32, +} + +impl PendingReadBatch { + fn new(task: ReadBatchTask) -> Self { + Self { + task: Some(task.task), + shared_task: None, + offset: 0, + num_rows: task.num_rows, + } + } + + fn take(&mut self, num_rows: u32) -> ReadBatchFut { + debug_assert!(num_rows <= self.num_rows); + + if self.offset == 0 && num_rows == self.num_rows && self.shared_task.is_none() { + self.num_rows = 0; + let Some(task) = self.task.take() else { + return async { + Err(Error::internal( + "missing read task while merging aligned streams".to_string(), + )) + } + .boxed(); + }; + return task; + } + + let shared_task = self + .shared_task + .get_or_insert_with(|| { + let task = self.task.take(); + async move { + let Some(task) = task else { + return Err(Arc::new(Error::internal( + "missing read task while splitting a merged stream".to_string(), + ))); + }; + task.await.map_err(Arc::new) + } + .boxed() + .shared() + }) + .clone(); + let offset = self.offset; + self.offset += num_rows; + self.num_rows -= num_rows; + + async move { + match shared_task.await { + Ok(batch) => Ok(batch.slice(offset as usize, num_rows as usize)), + Err(error) => Err(Error::wrapped(Box::new(SharedReadError(error)))), + } + } + .boxed() + } +} + struct MergeStream { streams: Vec, - next_batch: FuturesOrdered, - next_num_rows: u32, + pending: Vec>, index: usize, } impl MergeStream { fn emit(&mut self) -> ReadBatchTask { - let mut iter = std::mem::take(&mut self.next_batch); + let num_rows = self + .pending + .iter() + .filter_map(|pending| pending.as_ref().map(|pending| pending.num_rows)) + .min() + .unwrap_or_default(); + let mut batches = FuturesOrdered::new(); + for pending in &mut self.pending { + let Some(pending_batch) = pending.as_mut() else { + continue; + }; + batches.push_back(pending_batch.take(num_rows)); + if pending_batch.num_rows == 0 { + *pending = None; + } + } let task = async move { - let mut batch = iter.next().await.unwrap()?; - while let Some(next) = iter.next().await { + let Some(first) = batches.next().await else { + return Err(Error::internal( + "cannot merge an empty set of read batches".to_string(), + )); + }; + let mut batch = first?; + while let Some(next) = batches.next().await { let next = next?; batch = batch.merge(&next)?; } Ok(batch) } .boxed(); - let num_rows = self.next_num_rows; - self.next_num_rows = 0; ReadBatchTask { task, num_rows } } } @@ -64,21 +161,19 @@ impl Stream for MergeStream { cx: &mut std::task::Context<'_>, ) -> std::task::Poll> { loop { + if self.pending.iter().all(Option::is_some) { + return std::task::Poll::Ready(Some(self.emit())); + } + let index = self.index; + if self.pending[index].is_some() { + self.index = (index + 1) % self.streams.len(); + continue; + } match self.streams[index].poll_next_unpin(cx) { std::task::Poll::Ready(Some(batch_task)) => { - if self.index == 0 { - self.next_num_rows = batch_task.num_rows; - } else { - debug_assert_eq!(self.next_num_rows, batch_task.num_rows); - } - self.next_batch.push_back(batch_task.task); - self.index += 1; - if self.index == self.streams.len() { - self.index = 0; - let next_batch = self.emit(); - return std::task::Poll::Ready(Some(next_batch)); - } + self.pending[index] = Some(PendingReadBatch::new(batch_task)); + self.index = (index + 1) % self.streams.len(); } std::task::Poll::Ready(None) => { return std::task::Poll::Ready(None); @@ -95,19 +190,20 @@ impl Stream for MergeStream { /// /// This pulls one batch from each stream and then combines the columns from /// all of the batches into a single batch. The order of the batches in the -/// streams is maintained and the merged batch columns will be in order from -/// first to last stream. +/// streams is maintained and the merged batch columns will be in order from first +/// to last stream. If the streams use different batch boundaries then batches are +/// sliced so each merged output remains row-aligned. /// /// This stream ends as soon as any of the input streams ends (we do not /// verify that the other input streams are finished as well) -/// -/// This will panic if any of the input streams return a batch with a different -/// number of rows than the first stream. pub fn merge_streams(streams: Vec) -> ReadBatchTaskStream { + if streams.is_empty() { + return futures::stream::empty().boxed(); + } + let pending = (0..streams.len()).map(|_| None).collect(); MergeStream { streams, - next_batch: FuturesOrdered::new(), - next_num_rows: 0, + pending, index: 0, } .boxed() @@ -422,7 +518,10 @@ mod tests { use arrow::{array::AsArray, datatypes::UInt64Type}; use arrow_array::{RecordBatch, UInt32Array, types::Int32Type}; use arrow_schema::ArrowError; - use futures::{FutureExt, StreamExt, TryStreamExt, stream::BoxStream}; + use futures::{ + FutureExt, StreamExt, TryStreamExt, + stream::{self, BoxStream}, + }; use lance_core::{ ROW_ID, utils::{address::RowAddress, deletion::DeletionVector}, @@ -477,6 +576,46 @@ mod tests { assert_eq!(merged, expected); } + #[tokio::test] + async fn test_zip_with_different_batch_boundaries() { + let left_batch = + arrow_array::record_batch!(("x", Int32, (0..10).collect::>())).unwrap(); + let right_batch = + arrow_array::record_batch!(("y", Int32, (10..20).collect::>())).unwrap(); + let left = batch_task_stream( + stream::iter([Ok(left_batch.slice(0, 6)), Ok(left_batch.slice(6, 4))]).boxed(), + ); + let right = batch_task_stream( + stream::iter([Ok(right_batch.slice(0, 4)), Ok(right_batch.slice(4, 6))]).boxed(), + ); + + let merged = super::merge_streams(vec![left, right]) + .map(|batch_task| batch_task.task) + .buffered(3) + .try_collect::>() + .await + .unwrap(); + + let expected = vec![ + arrow_array::record_batch!( + ("x", Int32, (0..4).collect::>()), + ("y", Int32, (10..14).collect::>()) + ) + .unwrap(), + arrow_array::record_batch!( + ("x", Int32, (4..6).collect::>()), + ("y", Int32, (14..16).collect::>()) + ) + .unwrap(), + arrow_array::record_batch!( + ("x", Int32, (6..10).collect::>()), + ("y", Int32, (16..20).collect::>()) + ) + .unwrap(), + ]; + assert_eq!(merged, expected); + } + async fn check_row_id(params: ReadBatchParams, expected: impl IntoIterator) { let expected = Vec::from_iter(expected); diff --git a/rust/lance-tokenizer/Cargo.toml b/rust/lance-tokenizer/Cargo.toml index e1006cd93c7..ebb7b0060f8 100644 --- a/rust/lance-tokenizer/Cargo.toml +++ b/rust/lance-tokenizer/Cargo.toml @@ -15,7 +15,26 @@ rust-version.workspace = true icu_segmenter = { workspace = true } jieba-rs = { workspace = true, optional = true } lindera = { workspace = true, optional = true } -rust-stemmers = "1.2.0" +frostem = { version = "1.20260804.0", default-features = false, features = [ + "arabic", + "danish", + "dutch", + "english", + "finnish", + "french", + "german", + "greek", + "hungarian", + "italian", + "norwegian", + "portuguese", + "romanian", + "russian", + "spanish", + "swedish", + "tamil", + "turkish", +] } serde = { workspace = true, features = ["derive"] } stop-words = { version = "0.10.0", default-features = false, features = ["iso", "nltk"] } unicode-normalization = "0.1.25" diff --git a/rust/lance-tokenizer/src/code_tokenizer.rs b/rust/lance-tokenizer/src/code_tokenizer.rs new file mode 100644 index 00000000000..0d00aa2b94c --- /dev/null +++ b/rust/lance-tokenizer/src/code_tokenizer.rs @@ -0,0 +1,191 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright The Lance Authors + +use std::{iter::Peekable, str::CharIndices}; + +use crate::{Token, TokenStream, Tokenizer}; + +/// Tokenizer for code-like text. +/// +/// Identifiers are Unicode alphanumeric characters plus `_`. Other characters +/// are lexical boundaries. When operator indexing is enabled, recognized +/// multi-character operators use longest-match tokenization and remaining +/// operator characters are emitted individually. +/// +/// # Examples +/// +/// ``` +/// use lance_tokenizer::{CodeLexTokenizer, TextAnalyzer, TokenStream}; +/// +/// let mut analyzer = TextAnalyzer::builder(CodeLexTokenizer::new(true)).build(); +/// let mut stream = analyzer.token_stream("a::b"); +/// +/// assert!(stream.advance()); +/// assert_eq!(stream.token().text, "a"); +/// assert!(stream.advance()); +/// assert_eq!(stream.token().text, "::"); +/// ``` +#[derive(Clone, Default)] +pub struct CodeLexTokenizer { + index_operators: bool, + token: Token, +} + +impl CodeLexTokenizer { + pub fn new(index_operators: bool) -> Self { + Self { + index_operators, + token: Token::default(), + } + } +} + +/// Token stream produced by [`CodeLexTokenizer`]. +pub struct CodeLexTokenStream<'a> { + text: &'a str, + chars: Peekable>, + token: &'a mut Token, + index_operators: bool, +} + +impl Tokenizer for CodeLexTokenizer { + type TokenStream<'a> = CodeLexTokenStream<'a>; + + fn token_stream<'a>(&'a mut self, text: &'a str) -> Self::TokenStream<'a> { + self.token.reset(); + CodeLexTokenStream { + text, + chars: text.char_indices().peekable(), + token: &mut self.token, + index_operators: self.index_operators, + } + } +} + +fn is_identifier_char(ch: char) -> bool { + ch == '_' || ch.is_alphanumeric() +} + +fn is_operator_char(ch: char) -> bool { + matches!( + ch, + '!' | '%' | '&' | '*' | '+' | '-' | '/' | ':' | '<' | '=' | '>' | '?' | '^' | '|' | '~' + ) +} + +const MULTI_CHAR_OPERATORS: &[&str] = &[ + ">>>=", "<<=", ">>=", "&&=", "||=", "??=", "**=", "//=", "===", "!==", ">>>", "<=>", "::", + "->", "=>", "==", "!=", "<=", ">=", "&&", "||", "++", "--", "+=", "-=", "*=", "/=", "%=", "&=", + "|=", "^=", "<<", ">>", "**", "//", "??", ":=", "<-", "|>", "~=", +]; + +impl CodeLexTokenStream<'_> { + fn search_token_end(&mut self, predicate: impl Fn(char) -> bool) -> usize { + while let Some((_, ch)) = self.chars.peek() { + if !predicate(*ch) { + break; + } + self.chars.next(); + } + self.chars + .peek() + .map(|(offset, _)| *offset) + .unwrap_or(self.text.len()) + } + + fn operator_token_end(&mut self, offset_from: usize) -> usize { + let remaining = &self.text[offset_from..]; + let operator_len = MULTI_CHAR_OPERATORS + .iter() + .filter(|operator| remaining.starts_with(**operator)) + .map(|operator| operator.len()) + .max() + .unwrap_or(1); + let token_end = offset_from + operator_len; + while self + .chars + .peek() + .is_some_and(|(offset, _)| *offset < token_end) + { + self.chars.next(); + } + token_end + } +} + +impl TokenStream for CodeLexTokenStream<'_> { + fn advance(&mut self) -> bool { + self.token.text.clear(); + while let Some((offset_from, ch)) = self.chars.next() { + let token_end = if is_identifier_char(ch) { + self.search_token_end(is_identifier_char) + } else if self.index_operators && is_operator_char(ch) { + self.operator_token_end(offset_from) + } else { + continue; + }; + + self.token.position = self.token.position.wrapping_add(1); + self.token.position_length = 1; + self.token.offset_from = offset_from; + self.token.offset_to = token_end; + self.token.text.push_str(&self.text[offset_from..token_end]); + return true; + } + false + } + + fn token(&self) -> &Token { + self.token + } + + fn token_mut(&mut self) -> &mut Token { + self.token + } +} + +#[cfg(test)] +mod tests { + use crate::{CodeLexTokenizer, TextAnalyzer, Token}; + + fn collect_tokens(text: &str, index_operators: bool) -> Vec { + let mut analyzer = TextAnalyzer::builder(CodeLexTokenizer::new(index_operators)).build(); + let mut stream = analyzer.token_stream(text); + let mut tokens = Vec::new(); + stream.process(&mut |token| tokens.push(token.clone())); + tokens + } + + #[test] + fn test_code_lex_tokenizer_identifiers() { + let tokens = collect_tokens("std::vector user-name parse.HTML2JSON", false); + let texts = tokens + .iter() + .map(|token| token.text.as_str()) + .collect::>(); + assert_eq!( + texts, + vec!["std", "vector", "user", "name", "parse", "HTML2JSON"] + ); + } + + #[test] + fn test_code_lex_tokenizer_operators() { + let tokens = collect_tokens("a::b != c->d", true); + let texts = tokens + .iter() + .map(|token| token.text.as_str()) + .collect::>(); + assert_eq!(texts, vec!["a", "::", "b", "!=", "c", "->", "d"]); + } + + #[test] + fn test_code_lex_tokenizer_splits_adjacent_operators() { + let tokens = collect_tokens("value.parse::()", true); + let texts = tokens + .iter() + .map(|token| token.text.as_str()) + .collect::>(); + assert_eq!(texts, vec!["value", "parse", "::", "<", "usize", ">"]); + } +} diff --git a/rust/lance-tokenizer/src/lib.rs b/rust/lance-tokenizer/src/lib.rs index 0708022b384..fc1d9512cfd 100644 --- a/rust/lance-tokenizer/src/lib.rs +++ b/rust/lance-tokenizer/src/lib.rs @@ -4,6 +4,7 @@ mod alphanum_only; mod analyzer; mod ascii_folding_filter; +mod code_tokenizer; mod icu; #[cfg(feature = "tokenizer-jieba")] mod jieba; @@ -16,6 +17,7 @@ mod stemmer; mod stop_word_filter; mod tokenizer_api; mod whitespace_tokenizer; +mod word_delimiter_filter; #[cfg(feature = "tokenizer-lindera")] mod lindera; @@ -23,6 +25,7 @@ mod lindera; pub use alphanum_only::AlphaNumOnlyFilter; pub use analyzer::{TextAnalyzer, TextAnalyzerBuilder}; pub use ascii_folding_filter::AsciiFoldingFilter; +pub use code_tokenizer::CodeLexTokenizer; pub use icu::IcuTokenizer; #[cfg(feature = "tokenizer-jieba")] pub use jieba::JiebaTokenizer; @@ -37,3 +40,4 @@ pub use stemmer::{Language, Stemmer}; pub use stop_word_filter::StopWordFilter; pub use tokenizer_api::{BoxTokenStream, Token, TokenFilter, TokenStream, Tokenizer}; pub use whitespace_tokenizer::WhitespaceTokenizer; +pub use word_delimiter_filter::WordDelimiterFilter; diff --git a/rust/lance-tokenizer/src/stemmer.rs b/rust/lance-tokenizer/src/stemmer.rs index 03fcf118019..a79dd20d5f5 100644 --- a/rust/lance-tokenizer/src/stemmer.rs +++ b/rust/lance-tokenizer/src/stemmer.rs @@ -7,7 +7,7 @@ use std::borrow::Cow; use std::mem; -use rust_stemmers::Algorithm; +use frostem::Algorithm; use serde::{Deserialize, Serialize}; use crate::{Token, TokenFilter, TokenStream, Tokenizer}; @@ -101,7 +101,7 @@ impl Tokenizer for StemmerFilter { fn token_stream<'a>(&'a mut self, text: &'a str) -> Self::TokenStream<'a> { StemmerTokenStream { tail: self.inner.token_stream(text), - stemmer: rust_stemmers::Stemmer::create(self.stemmer_algorithm), + stemmer: frostem::Stemmer::new(self.stemmer_algorithm), buffer: String::new(), } } @@ -109,7 +109,7 @@ impl Tokenizer for StemmerFilter { pub struct StemmerTokenStream { tail: T, - stemmer: rust_stemmers::Stemmer, + stemmer: frostem::Stemmer, buffer: String, } @@ -139,3 +139,19 @@ impl TokenStream for StemmerTokenStream { self.tail.token_mut() } } + +#[cfg(test)] +mod tests { + use crate::{Language, RawTokenizer, Stemmer, TextAnalyzer, TokenStream}; + + #[test] + fn test_greek_stemmer_handles_multibyte_suffixes() { + let mut analyzer = TextAnalyzer::builder(RawTokenizer::default()) + .filter(Stemmer::new(Language::Greek)) + .build(); + let mut stream = analyzer.token_stream("αντιθετε"); + + assert!(stream.advance()); + assert_eq!(stream.token().text, "ανετ"); + } +} diff --git a/rust/lance-tokenizer/src/stop_word_filter.rs b/rust/lance-tokenizer/src/stop_word_filter.rs index 2acf0b3dbd5..9a690b0ec06 100644 --- a/rust/lance-tokenizer/src/stop_word_filter.rs +++ b/rust/lance-tokenizer/src/stop_word_filter.rs @@ -17,7 +17,12 @@ fn all_stop_words() -> impl Iterator { stop_words::get("ar"), stopwords::DANISH, stopwords::DUTCH, - stopwords::ENGLISH, + // Use the fuller `stop-words` crate English list (~198 words) rather + // than the local Tantivy-style list (~33 words), which omits extremely + // common pronouns/function words (you, my, your, we, she, what, ...). + // Those omissions let the highest-frequency English tokens through the + // ICU stop-word path and build pathologically large posting lists. + stop_words::get("en"), stopwords::FINNISH, stopwords::FRENCH, stopwords::GERMAN, @@ -51,7 +56,11 @@ impl StopWordFilter { Language::Arabic => stop_words::get("ar"), Language::Danish => stopwords::DANISH, Language::Dutch => stopwords::DUTCH, - Language::English => stopwords::ENGLISH, + // Use the fuller `stop-words` crate English list (~198 words); the + // local Tantivy-style list (~33 words) omits common pronouns/function + // words (you, my, your, we, ...) that would otherwise leak through + // stop-word removal and build pathologically large posting lists. + Language::English => stop_words::get("en"), Language::Finnish => stopwords::FINNISH, Language::French => stopwords::FRENCH, Language::German => stopwords::GERMAN, diff --git a/rust/lance-tokenizer/src/stop_word_filter/stopwords.rs b/rust/lance-tokenizer/src/stop_word_filter/stopwords.rs index 227556ba527..2ac3f4a28aa 100644 --- a/rust/lance-tokenizer/src/stop_word_filter/stopwords.rs +++ b/rust/lance-tokenizer/src/stop_word_filter/stopwords.rs @@ -37,12 +37,6 @@ ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ -pub const ENGLISH: &[&str] = &[ - "a", "an", "and", "are", "as", "at", "be", "but", "by", "for", "if", "in", "into", "is", "it", - "no", "not", "of", "on", "or", "such", "that", "the", "their", "then", "there", "these", - "they", "this", "to", "was", "will", "with", -]; - pub const DANISH: &[&str] = &[ "og", "i", "jeg", "det", "at", "en", "den", "til", "er", "som", "på", "de", "med", "han", "af", "for", "ikke", "der", "var", "mig", "sig", "men", "et", "har", "om", "vi", "min", "havde", diff --git a/rust/lance-tokenizer/src/word_delimiter_filter.rs b/rust/lance-tokenizer/src/word_delimiter_filter.rs new file mode 100644 index 00000000000..70138529efe --- /dev/null +++ b/rust/lance-tokenizer/src/word_delimiter_filter.rs @@ -0,0 +1,297 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright The Lance Authors + +use std::collections::VecDeque; + +use crate::{Token, TokenFilter, TokenStream, Tokenizer}; + +/// Splits code identifiers into subword tokens. +/// +/// The original identifier can be preserved at the first subword position. If +/// a token is not a compound identifier then it is passed through unchanged. +/// +/// # Examples +/// +/// ``` +/// use lance_tokenizer::{CodeLexTokenizer, TextAnalyzer, TokenStream, WordDelimiterFilter}; +/// +/// let mut analyzer = TextAnalyzer::builder(CodeLexTokenizer::new(false)) +/// .filter(WordDelimiterFilter::new(true, true)) +/// .build(); +/// let mut stream = analyzer.token_stream("parseHTML2JSON"); +/// +/// let mut tokens = Vec::new(); +/// stream.process(&mut |token| tokens.push(token.text.clone())); +/// assert_eq!(tokens, vec!["parseHTML2JSON", "parse", "HTML", "2", "JSON"]); +/// ``` +#[derive(Clone)] +pub struct WordDelimiterFilter { + preserve_original: bool, + split_on_numerics: bool, +} + +impl WordDelimiterFilter { + pub fn new(preserve_original: bool, split_on_numerics: bool) -> Self { + Self { + preserve_original, + split_on_numerics, + } + } +} + +impl TokenFilter for WordDelimiterFilter { + type Tokenizer = WordDelimiterFilterWrapper; + + fn transform(self, tokenizer: T) -> Self::Tokenizer { + WordDelimiterFilterWrapper { + tokenizer, + preserve_original: self.preserve_original, + split_on_numerics: self.split_on_numerics, + } + } +} + +#[derive(Clone)] +/// Tokenizer wrapper produced by [`WordDelimiterFilter`]. +pub struct WordDelimiterFilterWrapper { + tokenizer: T, + preserve_original: bool, + split_on_numerics: bool, +} + +impl Tokenizer for WordDelimiterFilterWrapper { + type TokenStream<'a> = WordDelimiterTokenStream>; + + fn token_stream<'a>(&'a mut self, text: &'a str) -> Self::TokenStream<'a> { + WordDelimiterTokenStream { + tail: self.tokenizer.token_stream(text), + current: Token::default(), + pending: VecDeque::new(), + preserve_original: self.preserve_original, + split_on_numerics: self.split_on_numerics, + position_offset: 0, + } + } +} + +/// Token stream produced by [`WordDelimiterFilterWrapper`]. +pub struct WordDelimiterTokenStream { + tail: T, + current: Token, + pending: VecDeque, + preserve_original: bool, + split_on_numerics: bool, + position_offset: usize, +} + +#[derive(Clone, Copy)] +struct CharInfo { + offset: usize, + ch: char, +} + +fn is_identifier_text(text: &str) -> bool { + text.chars().any(|ch| ch == '_' || ch.is_alphanumeric()) +} + +fn split_identifier(text: &str, split_on_numerics: bool) -> Vec<(usize, usize)> { + let mut pieces = Vec::new(); + let mut segment_start = None; + for (offset, ch) in text.char_indices() { + if ch == '_' { + if let Some(start) = segment_start.take() { + split_segment(text, start, offset, split_on_numerics, &mut pieces); + } + } else if segment_start.is_none() { + segment_start = Some(offset); + } + } + if let Some(start) = segment_start { + split_segment(text, start, text.len(), split_on_numerics, &mut pieces); + } + pieces +} + +fn split_segment( + text: &str, + start: usize, + end: usize, + split_on_numerics: bool, + pieces: &mut Vec<(usize, usize)>, +) { + if start == end { + return; + } + let chars = text[start..end] + .char_indices() + .map(|(offset, ch)| CharInfo { + offset: start + offset, + ch, + }) + .collect::>(); + let mut piece_start = start; + for idx in 1..chars.len() { + if is_boundary(&chars, idx, split_on_numerics) { + pieces.push((piece_start, chars[idx].offset)); + piece_start = chars[idx].offset; + } + } + pieces.push((piece_start, end)); +} + +fn is_boundary(chars: &[CharInfo], idx: usize, split_on_numerics: bool) -> bool { + let prev = chars[idx - 1].ch; + let cur = chars[idx].ch; + if split_on_numerics && prev.is_ascii_digit() != cur.is_ascii_digit() { + return true; + } + if prev.is_lowercase() && cur.is_uppercase() { + return true; + } + if prev.is_uppercase() + && cur.is_uppercase() + && chars + .get(idx + 1) + .is_some_and(|next| next.ch.is_lowercase()) + { + return true; + } + false +} + +impl WordDelimiterTokenStream { + fn refill(&mut self) -> bool { + while self.pending.is_empty() { + if !self.tail.advance() { + return false; + } + self.enqueue_current_tail_token(); + } + true + } + + fn enqueue_current_tail_token(&mut self) { + let token = self.tail.token(); + let adjusted_position = token.position.saturating_add(self.position_offset); + if !is_identifier_text(&token.text) { + let mut token = token.clone(); + token.position = adjusted_position; + self.pending.push_back(token); + return; + } + + let parts = split_identifier(&token.text, self.split_on_numerics); + if parts.len() <= 1 { + let mut token = token.clone(); + token.position = adjusted_position; + self.pending.push_back(token); + return; + } + + if self.preserve_original { + let mut original = token.clone(); + original.position = adjusted_position; + original.position_length = parts.len(); + self.pending.push_back(original); + } + + for (part_idx, (start, end)) in parts.iter().copied().enumerate() { + let mut part = token.clone(); + part.offset_from = token.offset_from + start; + part.offset_to = token.offset_from + end; + part.position = adjusted_position + part_idx; + part.position_length = 1; + part.text.clear(); + part.text.push_str(&token.text[start..end]); + self.pending.push_back(part); + } + self.position_offset += parts.len() - 1; + } +} + +impl TokenStream for WordDelimiterTokenStream { + fn advance(&mut self) -> bool { + if !self.refill() { + return false; + } + let Some(token) = self.pending.pop_front() else { + debug_assert!(false, "pending token should be available after refill"); + return false; + }; + self.current = token; + true + } + + fn token(&self) -> &Token { + &self.current + } + + fn token_mut(&mut self) -> &mut Token { + &mut self.current + } +} + +#[cfg(test)] +mod tests { + use crate::{CodeLexTokenizer, TextAnalyzer, Token, WordDelimiterFilter}; + + fn collect_tokens(text: &str) -> Vec { + let mut analyzer = TextAnalyzer::builder(CodeLexTokenizer::new(false)) + .filter(WordDelimiterFilter::new(true, true)) + .build(); + let mut stream = analyzer.token_stream(text); + let mut tokens = Vec::new(); + stream.process(&mut |token| tokens.push(token.clone())); + tokens + } + + #[test] + fn test_word_delimiter_code_identifiers() { + let tokens = collect_tokens("getUserName XMLHttpRequest parseHTML2JSON utf8_reader"); + let texts = tokens + .iter() + .map(|token| token.text.as_str()) + .collect::>(); + assert_eq!( + texts, + vec![ + "getUserName", + "get", + "User", + "Name", + "XMLHttpRequest", + "XML", + "Http", + "Request", + "parseHTML2JSON", + "parse", + "HTML", + "2", + "JSON", + "utf8_reader", + "utf", + "8", + "reader", + ] + ); + } + + #[test] + fn test_word_delimiter_positions_span_compounds() { + let tokens = collect_tokens("getUserName next"); + let positions = tokens + .iter() + .map(|token| (token.text.as_str(), token.position, token.position_length)) + .collect::>(); + assert_eq!( + positions, + vec![ + ("getUserName", 0, 3), + ("get", 0, 1), + ("User", 1, 1), + ("Name", 2, 1), + ("next", 3, 1), + ] + ); + } +} diff --git a/rust/lance/Cargo.toml b/rust/lance/Cargo.toml index 36ea5facc29..64253478000 100644 --- a/rust/lance/Cargo.toml +++ b/rust/lance/Cargo.toml @@ -12,7 +12,8 @@ keywords.workspace = true categories.workspace = true [package.metadata.docs.rs] -features = [] +# Publish the feature-gated metrics catalogue; keep cloud backends off. +features = ["metrics"] no-default-features = true [package.metadata.cargo-machete] @@ -139,6 +140,7 @@ reqwest = { version = "0.12", default-features = false, features = ["rustls-tls" [features] default = ["aws", "azure", "gcp", "oss", "huggingface", "tencent", "tos", "goosefs", "geo"] +backtrace = ["lance-core/backtrace"] fp16kernels = ["lance-linalg/fp16kernels"] # Prevent dynamic linking of lzma, which comes from datafusion cli = ["dep:clap", "lzma-sys/static"] @@ -160,6 +162,8 @@ tencent = ["lance-io/tencent"] goosefs = ["lance-io/goosefs"] tos = ["lance-io/tos"] huggingface = ["lance-io/huggingface"] +# Publish object store metrics via the `metrics` crate. +metrics = ["lance-io/metrics"] geo = ["lance-datafusion/geo", "lance-index/geo"] # Enable slow integration tests (disabled by default in CI) slow_tests = [] @@ -195,6 +199,10 @@ harness = false name = "scan" harness = false +[[bench]] +name = "s3_file_reader_diagnostics" +harness = false + [[bench]] name = "count_pushdown" harness = false diff --git a/rust/lance/benches/mem_wal/fts/mem_wal_fineweb_fts.rs b/rust/lance/benches/mem_wal/fts/mem_wal_fineweb_fts.rs index 63f9368c3c5..188e2c73ce2 100644 --- a/rust/lance/benches/mem_wal/fts/mem_wal_fineweb_fts.rs +++ b/rust/lance/benches/mem_wal/fts/mem_wal_fineweb_fts.rs @@ -111,11 +111,6 @@ impl Mode { fn durable_write(self) -> bool { matches!(self, Self::SyncNoIndex | Self::SyncIndexed) } - - /// Index update happens inline in `put` (only meaningful when indexed). - fn sync_indexed_write(self) -> bool { - matches!(self, Self::SyncIndexed) - } } #[derive(Debug, Clone, Copy, PartialEq, Eq)] @@ -428,7 +423,6 @@ fn shard_writer_config(args: &Args, shard_id: Uuid, disable_auto_flush: bool) -> }; let mut config = ShardWriterConfig::new(shard_id) .with_durable_write(args.mode.durable_write()) - .with_sync_indexed_write(args.mode.sync_indexed_write()) .with_max_memtable_size(args.max_memtable_size) .with_max_unflushed_memtable_bytes(args.max_unflushed_memtable_bytes) .with_max_memtable_rows(max_rows) @@ -638,7 +632,7 @@ async fn run_read(args: &Args, uri: &str, corpus: &[String]) -> Result Result Result { } "--base-rows" => args.base_rows = parse_val(&flag, &value)?, "--max-memtable-rows" => args.max_memtable_rows = parse_val(&flag, &value)?, - "--flushed-generations" => args.flushed_generations = parse_val(&flag, &value)?, + "--sstables" => args.sstables = parse_val(&flag, &value)?, "--batch-rows" => args.batch_rows = parse_val(&flag, &value)?, "--queries" => args.queries = parse_val(&flag, &value)?, "--k" => args.k = parse_val(&flag, &value)?, @@ -664,7 +664,7 @@ async fn run_search(args: &Args) -> Result { // covering both the memtable payload and the query-term sample, instead // of re-reading the whole base corpus from parquet. let active_rows = args.max_memtable_rows / 2; - let total_memtable_rows = args.flushed_generations * args.max_memtable_rows + active_rows; + let total_memtable_rows = args.sstables * args.max_memtable_rows + active_rows; let sample_rows = args.base_rows.min(50_000); let load_rows = total_memtable_rows.max(sample_rows); println!("loading {load_rows} FineWeb rows for memtable payload + query sample ..."); @@ -688,7 +688,6 @@ async fn run_search(args: &Args) -> Result { wal_persist_retry_base_delay: std::time::Duration::from_millis(50), shard_spec_id: 0, durable_write: false, - sync_indexed_write: false, max_memtable_size: args.max_memtable_rows * row_bytes * 100, max_memtable_rows: args.max_memtable_rows, max_memtable_batches: batches_per_gen, @@ -704,10 +703,8 @@ async fn run_search(args: &Args) -> Result { Duration::from_millis(500) }; - // Ingest flushed generations + 1 active (50% full). - let mut gen_sizes: Vec = (0..args.flushed_generations) - .map(|_| args.max_memtable_rows) - .collect(); + // Ingest SSTables + 1 active (50% full). + let mut gen_sizes: Vec = (0..args.sstables).map(|_| args.max_memtable_rows).collect(); gen_sizes.push(active_rows); let id_base = args.base_rows as i64; @@ -724,19 +721,19 @@ async fn run_search(args: &Args) -> Result { cursor += chunk; written += chunk; } - let is_flushed = gen_idx < args.flushed_generations; + let is_sstable = gen_idx < args.sstables; println!( " gen {}: wrote {} rows ({})", gen_idx + 1, gen_rows, - if is_flushed { "flushed" } else { "active" } + if is_sstable { "sstable" } else { "active" } ); - if is_flushed { + if is_sstable { tokio::time::sleep(flush_wait).await; } } // Wait for any triggered (sealed) memtable flushes to commit to the - // manifest before we snapshot it — otherwise the flushed generations + // manifest before we snapshot it — otherwise the SSTables // race the read and may not all be visible yet. writer.wait_for_flush_drain().await?; println!( @@ -750,17 +747,14 @@ async fn run_search(args: &Args) -> Result { let mut shard_snapshot = ShardSnapshot::new(shard_id); if let Some(ref m) = manifest { shard_snapshot = shard_snapshot.with_current_generation(m.current_generation); - for fg in &m.flushed_generations { - shard_snapshot = shard_snapshot.with_flushed_generation(fg.generation, fg.path.clone()); + for sstable in &m.sstables { + shard_snapshot = shard_snapshot.with_sstable(sstable.generation, sstable.path.clone()); } } - let num_flushed = manifest - .as_ref() - .map(|m| m.flushed_generations.len()) - .unwrap_or(0); - println!("manifest: {num_flushed} flushed generations"); + let num_sstables = manifest.as_ref().map(|m| m.sstables.len()).unwrap_or(0); + println!("manifest: {num_sstables} SSTables"); - // Flushed generations carry the same maintained secondary indexes as + // SSTables carry the same maintained secondary indexes as // the active memtable: the flush handler builds them during flush // (lance #6901), so each generation already has the FTS index and // both scoring modes use the fast indexed path. No manual indexing @@ -827,7 +821,7 @@ async fn run_search(args: &Args) -> Result { "uri_kind": if is_cloud_uri(&args.uri) { "cloud" } else { "local" }, "base_rows": args.base_rows, "max_memtable_rows": args.max_memtable_rows, - "flushed_generations": num_flushed, + "sstables": num_sstables, "active_rows": active_rows, "k": args.k, "queries": queries.len(), @@ -848,12 +842,12 @@ async fn run_search(args: &Args) -> Result { async fn run(args: Args) -> Result<()> { println!( - "bench=mem_wal_fts_read phase={} uri={} base_rows={} max_memtable_rows={} flushed_generations={} queries={} k={} with_baseline={}", + "bench=mem_wal_fts_read phase={} uri={} base_rows={} max_memtable_rows={} sstables={} queries={} k={} with_baseline={}", args.phase.as_str(), args.uri, args.base_rows, args.max_memtable_rows, - args.flushed_generations, + args.sstables, args.queries, args.k, args.with_baseline, diff --git a/rust/lance/benches/mem_wal/fts/run_fts_read_sweep.sh b/rust/lance/benches/mem_wal/fts/run_fts_read_sweep.sh index 7ea692ee0b1..31ededa04e2 100755 --- a/rust/lance/benches/mem_wal/fts/run_fts_read_sweep.sh +++ b/rust/lance/benches/mem_wal/fts/run_fts_read_sweep.sh @@ -8,7 +8,7 @@ # # For each (backend, base_rows) the bench's `prepare` phase runs once to # write the base dataset + FTS index + MemWAL; then for each k the `search` -# phase ingests flushed generations + an active memtable through ShardWriter +# phase ingests SSTables + an active memtable through ShardWriter # and times the FTS query panel under both Local and LocalWithGlobalRescore # scoring modes. # @@ -23,8 +23,8 @@ # CACHE_DIR FineWeb shard download cache (default /lance-fineweb-cache) # BASE_ROWS_LIST space-separated base sizes (default "100000 1000000") # K_LIST space-separated top-k values (default "10 100") -# MAX_MEMTABLE_ROWS active/flushed memtable cap (default 100000) -# GENS_LIST space-separated flushed-generation counts (default "1 2 5") +# MAX_MEMTABLE_ROWS active/SSTable cap (default 100000) +# GENS_LIST space-separated SSTable counts (default "1 2 5") # QUERIES queries per config (default 200) # WITH_BASELINE "1" to also build the merged-index accuracy baseline # and report local-vs-merged Jaccard (default off) @@ -117,7 +117,7 @@ for backend in $BACKENDS; do --base-rows "$base_rows" --batch-rows 1000 \ --cache-dir "$CACHE_DIR" || continue - # search for each (flushed-generations, k) + # search for each (SSTables, k) for gens in $GENS_LIST; do for k in $K_LIST; do name="search_${backend}_${btag}_g${gens}_k${k}" @@ -136,7 +136,7 @@ for backend in $BACKENDS; do --phase search --uri "$uri" \ --base-rows "$base_rows" \ --max-memtable-rows "$MAX_MEMTABLE_ROWS" \ - --flushed-generations "$gens" \ + --sstables "$gens" \ --batch-rows 1000 \ --queries "$QUERIES" --k "$k" \ "${baseline_flag[@]}" \ diff --git a/rust/lance/benches/mem_wal/kv/mem_wal_kv_point_lookup.rs b/rust/lance/benches/mem_wal/kv/mem_wal_kv_point_lookup.rs index 757c2539642..335a41d3bbd 100644 --- a/rust/lance/benches/mem_wal/kv/mem_wal_kv_point_lookup.rs +++ b/rust/lance/benches/mem_wal/kv/mem_wal_kv_point_lookup.rs @@ -43,8 +43,7 @@ use datafusion::common::ScalarValue; use datafusion::prelude::SessionContext; use futures::TryStreamExt; use lance::dataset::mem_wal::scanner::{ - FlushedMemTableCache, InMemoryMemTableRef, LsmDataSourceCollector, LsmPointLookupPlanner, - ShardSnapshot, + InMemoryMemTableRef, LsmDataSourceCollector, LsmPointLookupPlanner, ShardSnapshot, SsTableCache, }; use lance::dataset::mem_wal::{DatasetMemWalExt, ShardWriterConfig}; use lance::dataset::{Dataset, WriteParams}; @@ -422,7 +421,7 @@ enum LanceReadMode { Plan, /// Probe the active MemTable's BTree index directly and materialize the /// row from the BatchStore, bypassing DataFusion. Single-active-memtable - /// fast path (no flushed generations); misses fall through as "not found". + /// fast path (no SSTables); misses fall through as "not found". Fast, /// Call the production `LsmPointLookupPlanner::lookup` API, which uses the /// direct BTree fast path internally and falls back to the plan path for @@ -492,27 +491,27 @@ impl KeyType { } /// Where the data under test lives. `Active` = the in-memory active MemTable -/// (never flushed). `Flushed` = an on-disk flushed generation (a Lance data +/// (never flushed). `SsTable` = an on-disk SSTable (a Lance data /// file + on-disk BTree index, read via the indexed-scan path) vs a single /// RocksDB SST on disk. #[derive(Debug, Clone, Copy, PartialEq)] enum Storage { Active, - Flushed, + SsTable, } impl Storage { fn parse(v: &str) -> std::result::Result { match v { "active" => Ok(Self::Active), - "flushed" => Ok(Self::Flushed), - _ => Err(format!("unknown storage '{v}', expected active|flushed")), + "sstable" => Ok(Self::SsTable), + _ => Err(format!("unknown storage '{v}', expected active|sstable")), } } fn as_str(self) -> &'static str { match self { Self::Active => "active", - Self::Flushed => "flushed", + Self::SsTable => "sstable", } } } @@ -553,7 +552,7 @@ struct Args { engine: Engine, key_type: KeyType, storage: Storage, - /// Number of flushed generations below the single active MemTable (Lance) / + /// Number of SSTables below the single active MemTable (Lance) / /// immutable SSTs below the active memtable (RocksDB). 0 = the existing /// single-tier behavior. >0 builds a full LSM: rows are split into /// `generations+1` parts, the first `generations` are flushed to on-disk @@ -574,12 +573,12 @@ struct Args { /// the caches and hit NVMe. Caps the RocksDB write buffer + uses a small /// block cache + compacts to one SST, and drops the OS page cache before /// the read phase (both engines). Use with a `--rows`×`--value-size` larger - /// than RAM. Only affects the `--storage flushed` path. + /// than RAM. Only affects the `--storage sstable` path. cold: bool, - /// Prewarm all flushed generations (open + warm indexes) into the dataset + /// Prewarm all SSTables (open + warm indexes) into the dataset /// session before the read phase, via `DatasetMemWalExt::prewarm_mem_wal`. /// Default on. `--prewarm false` disables it to measure the lazy-warm - /// baseline (the flushed cache is still set, so each generation is opened + /// baseline (the SSTable cache is still set, so each generation is opened /// on its first gen-key lookup instead of up front). Only affects the Lance /// `--storage active` LSM path. prewarm: bool, @@ -792,7 +791,6 @@ async fn run_lance( wal_persist_retry_base_delay: std::time::Duration::from_millis(50), shard_spec_id: 0, durable_write: true, - sync_indexed_write: true, max_memtable_size: big, max_memtable_rows: args.rows * 4 + 1_000_000, max_memtable_batches: args.rows / args.batch_rows + 1_000_000, @@ -834,7 +832,7 @@ async fn run_lance( let n = writer .manifest() .await? - .map(|m| m.flushed_generations.len()) + .map(|m| m.sstables.len()) .unwrap_or(0); if n > g { break; @@ -850,10 +848,10 @@ async fn run_lance( let n_gens = writer .manifest() .await? - .map(|m| m.flushed_generations.len()) + .map(|m| m.sstables.len()) .unwrap_or(0); println!( - "[lance] wrote {} rows in {:.2}s = {:.0} rows/s (cpu {:.2}s, flushed_gens={n_gens}+active)", + "[lance] wrote {} rows in {:.2}s = {:.0} rows/s (cpu {:.2}s, sstables={n_gens}+active)", args.rows, write_s, write_rows_per_s, write_cpu_s ); @@ -863,8 +861,8 @@ async fn run_lance( let mut shard_snapshot = ShardSnapshot::new(shard_id); if let Some(ref m) = manifest { shard_snapshot = shard_snapshot.with_current_generation(m.current_generation); - for fg in &m.flushed_generations { - shard_snapshot = shard_snapshot.with_flushed_generation(fg.generation, fg.path.clone()); + for sstable in &m.sstables { + shard_snapshot = shard_snapshot.with_sstable(sstable.generation, sstable.path.clone()); } } // Keep a handle to the active MemTable for the direct fast path before @@ -872,22 +870,22 @@ async fn run_lance( let active = Arc::new(in_memory_refs.active.clone()); let collector = LsmDataSourceCollector::new(dataset.clone(), vec![shard_snapshot.clone()]) .with_in_memory_memtables(shard_id, in_memory_refs); - // Thread the dataset session + a flushed-dataset cache into the planner, and - // prewarm every flushed generation (open + warm its indexes) up front via + // Thread the dataset session + an SSTable cache into the planner, and + // prewarm every SSTable (open + warm its indexes) up front via // the general MemWAL API, so gen-key lookups never re-open a generation per // query (the equivalent of RocksDB keeping its DB + SSTs resident). Without // this, each plan-path lookup pays a fresh manifest read + Dataset open — a // fixed per-lookup cost independent of generation count. - let flushed_cache = Arc::new(FlushedMemTableCache::new((gens as u64).max(1))); + let sstable_cache = Arc::new(SsTableCache::new((gens as u64).max(1))); if args.prewarm { dataset - .prewarm_mem_wal(std::slice::from_ref(&shard_snapshot), Some(&flushed_cache)) + .prewarm_mem_wal(std::slice::from_ref(&shard_snapshot), Some(&sstable_cache)) .await?; } let planner = Arc::new( LsmPointLookupPlanner::new(collector, vec![KEY_COL.to_string()], arrow_schema) .with_session(dataset.session()) - .with_flushed_cache(flushed_cache), + .with_sstable_cache(sstable_cache), ); // Warmup + correctness: a hit key must resolve to exactly one row under @@ -1206,7 +1204,7 @@ async fn run_lance( } // ---------------------------------------------------------------------- -// Lance flushed (on-disk) engine +// Lance SSTable (on-disk) engine // ---------------------------------------------------------------------- /// Drop the OS page cache so subsequent reads hit storage (cold). Best-effort: @@ -1221,7 +1219,7 @@ fn drop_page_cache() { /// One indexed point lookup via the **DataFusion** path: `scan().filter("id = /// key")` parses + plans + executes a query per lookup (uses the on-disk BTree /// index). Returns the matched row count. -async fn flushed_probe(dataset: &Dataset, key: i64) -> Result { +async fn sstable_probe(dataset: &Dataset, key: i64) -> Result { use futures::StreamExt; let mut scanner = dataset.scan(); scanner.filter(&format!("{KEY_COL} = {key}"))?; @@ -1235,8 +1233,8 @@ async fn flushed_probe(dataset: &Dataset, key: i64) -> Result { /// One point lookup via the **direct** path: search the on-disk BTree scalar /// index for the row id, then `take` that row — bypassing DataFusion plan -/// construction. Diagnostic for how much of the flushed read cost is the plan. -async fn flushed_probe_direct( +/// construction. Diagnostic for how much of the SSTable read cost is the plan. +async fn sstable_probe_direct( dataset: &Dataset, scalar_index: &Arc, key: i64, @@ -1257,25 +1255,25 @@ async fn flushed_probe_direct( Ok(batch.num_rows()) } -/// Dispatch a flushed point lookup: `direct` = on-disk BTree index search + take +/// Dispatch an SSTable point lookup: `direct` = on-disk BTree index search + take /// (no DataFusion); otherwise the DataFusion `scan().filter()` path. -async fn flushed_lookup( +async fn sstable_lookup( dataset: &Dataset, scalar_index: &Arc, key: i64, direct: bool, ) -> Result { if direct { - flushed_probe_direct(dataset, scalar_index, key).await + sstable_probe_direct(dataset, scalar_index, key).await } else { - flushed_probe(dataset, key).await + sstable_probe(dataset, key).await } } -/// Batched flushed lookup over a chunk of keys: `direct` searches the index for +/// Batched SSTable lookup over a chunk of keys: `direct` searches the index for /// each key then issues one `take_rows` for all; otherwise one DataFusion scan /// with `id IN (...)`. Returns the total matched row count. -async fn flushed_batch( +async fn sstable_batch( dataset: &Dataset, scalar_index: &Arc, keys: &[i64], @@ -1321,11 +1319,11 @@ async fn flushed_batch( } } -/// Flushed Lance: write all rows as one on-disk Lance dataset with a BTree +/// SSTable Lance: write all rows as one on-disk Lance dataset with a BTree /// scalar index — the exact artifact a MemTable flush emits (forward-written /// data file + on-disk BTree index) — then point-lookup through the indexed /// scan path. Int keys only (the SQL filter literal is the integer). -async fn run_lance_flushed( +async fn run_lance_sstable( args: &Args, insert_order: &[i64], queries: &[(i64, bool)], @@ -1333,12 +1331,12 @@ async fn run_lance_flushed( assert_eq!( args.key_type, KeyType::Int, - "flushed mode currently supports --key-type int only" + "sstable mode currently supports --key-type int only" ); let sampler = RssSampler::start(); let key_type = args.key_type; let schema = make_schema(key_type); - let uri = format!("{}/lance_flushed", args.uri.trim_end_matches('/')); + let uri = format!("{}/lance_sstable", args.uri.trim_end_matches('/')); let _ = std::fs::remove_dir_all(&uri); // --- write + flush: build the on-disk data file + BTree index --- @@ -1379,13 +1377,13 @@ async fn run_lance_flushed( .iter() .find(|i| i.name == BTREE_INDEX_NAME) .map(|i| i.uuid.to_string()) - .ok_or_else(|| lance_core::Error::internal("flushed: btree index not found"))?; + .ok_or_else(|| lance_core::Error::internal("sstable: btree index not found"))?; dataset .open_scalar_index(KEY_COL, &uuid, &NoOpMetricsCollector) .await? }; println!( - "[lance] flushed read path = {}", + "[lance] sstable read path = {}", if direct { "direct btree-index search + take" } else { @@ -1395,8 +1393,8 @@ async fn run_lance_flushed( // warmup + correctness: a hit resolves to exactly one row via the index. if let Some((probe, _)) = queries.iter().find(|(_, h)| *h) { - let n = flushed_lookup(&dataset, &scalar_index, *probe, direct).await?; - assert_eq!(n, 1, "flushed warmup lookup for key {probe} returned {n}"); + let n = sstable_lookup(&dataset, &scalar_index, *probe, direct).await?; + assert_eq!(n, 1, "sstable warmup lookup for key {probe} returned {n}"); } // Cold mode: drop the OS page cache so reads hit NVMe (data > RAM assumed). @@ -1405,7 +1403,7 @@ async fn run_lance_flushed( println!("[lance] dropped page cache (cold reads from NVMe)"); } - // --- batch-get path: gather `batch_get` keys per call from the flushed gen --- + // --- batch-get path: gather `batch_get` keys per call from the SSTable --- if args.batch_get > 0 { let bg = args.batch_get; let hit_keys: Vec = queries @@ -1419,7 +1417,7 @@ async fn run_lance_flushed( let t = Instant::now(); for chunk in hit_keys.chunks(bg) { let t0 = Instant::now(); - found_total += flushed_batch(&dataset, &scalar_index, chunk, direct).await?; + found_total += sstable_batch(&dataset, &scalar_index, chunk, direct).await?; latencies_us.push(t0.elapsed().as_nanos() as f64 / 1000.0); } let read_qps_1t = hit_keys.len() as f64 / t.elapsed().as_secs_f64().max(1e-9); @@ -1439,7 +1437,7 @@ async fn run_lance_flushed( let chunks: Vec<&[i64]> = keys.chunks(bg).collect(); let mut i = shard; while i < chunks.len() { - let _ = flushed_batch(&dataset, &si, chunks[i], direct).await; + let _ = sstable_batch(&dataset, &si, chunks[i], direct).await; i += threads; } })); @@ -1456,7 +1454,7 @@ async fn run_lance_flushed( args.threads, stats.p50_us, stats.p99_us ); return Ok(EngineResult { - engine: "lance-flushed-batch", + engine: "lance-sstable-batch", write_rows_per_s, write_cpu_s, read_p50_us: stats.p50_us, @@ -1481,7 +1479,7 @@ async fn run_lance_flushed( let t_read = Instant::now(); for &(key, expect_hit) in queries { let t0 = Instant::now(); - let n = flushed_lookup(&dataset, &scalar_index, key, direct).await?; + let n = sstable_lookup(&dataset, &scalar_index, key, direct).await?; latencies_us.push(t0.elapsed().as_nanos() as f64 / 1000.0); if expect_hit { assert_eq!(n, 1, "expected hit for key {key}, got {n}"); @@ -1511,7 +1509,7 @@ async fn run_lance_flushed( handles.push(tokio::spawn(async move { let mut i = shard; while i < keys.len() { - let _ = flushed_lookup(&dataset, &scalar_index, keys[i], direct).await; + let _ = sstable_lookup(&dataset, &scalar_index, keys[i], direct).await; i += threads; } })); @@ -1536,7 +1534,7 @@ async fn run_lance_flushed( ); Ok(EngineResult { - engine: "lance-flushed", + engine: "lance-sstable", write_rows_per_s, write_cpu_s, read_p50_us: stats.p50_us, @@ -1585,7 +1583,7 @@ fn run_rocksdb(args: &Args, insert_order: &[i64], queries: &[(i64, bool)]) -> Re opts.set_min_write_buffer_number_to_merge(2); opts.set_disable_auto_compactions(true); opts.set_db_write_buffer_size(write_buf); - // Block cache for the `--storage flushed` SST reads. Warm: large enough to + // Block cache for the `--storage sstable` SST reads. Warm: large enough to // hold the SST index/filter + hot data blocks. Cold: small (128MB) so data // blocks miss the cache and reads go to NVMe (index/filter stay in memory). { @@ -1654,17 +1652,17 @@ fn run_rocksdb(args: &Args, insert_order: &[i64], queries: &[(i64, bool)]) -> Re write_buf >> 20 ); - // Single-tier `--storage flushed` (no extra generations): flush the active + // Single-tier `--storage sstable` (no extra generations): flush the active // to one SST (compact to one if cold). With generations>0 the per-chunk // flushes already produced N separate L0 SSTs + the active memtable. - if gens == 0 && args.storage == Storage::Flushed { + if gens == 0 && args.storage == Storage::SsTable { db.flush() .map_err(|e| lance_core::Error::io(format!("rocksdb flush: {e}")))?; if args.cold { db.compact_range::<&[u8], &[u8]>(None, None); } } - if args.storage == Storage::Flushed || gens > 0 { + if args.storage == Storage::SsTable || gens > 0 { let n_sst = db .property_int_value("rocksdb.num-files-at-level0") .ok() @@ -1939,7 +1937,7 @@ async fn run(args: Args) -> Result<()> { if matches!(args.engine, Engine::Lance | Engine::Both) { let res = match args.storage { Storage::Active => run_lance(&args, &insert_order, &queries).await?, - Storage::Flushed => run_lance_flushed(&args, &insert_order, &queries).await?, + Storage::SsTable => run_lance_sstable(&args, &insert_order, &queries).await?, }; results.push(res); } diff --git a/rust/lance/benches/mem_wal/point_lookup/mem_wal_point_lookup_bench.rs b/rust/lance/benches/mem_wal/point_lookup/mem_wal_point_lookup_bench.rs index cb9e6413ac1..0b92d1d9c73 100644 --- a/rust/lance/benches/mem_wal/point_lookup/mem_wal_point_lookup_bench.rs +++ b/rust/lance/benches/mem_wal/point_lookup/mem_wal_point_lookup_bench.rs @@ -4,8 +4,8 @@ //! Standalone CLI benchmark for PK-based point lookups across LSM levels. //! //! Measures lookup latency against three tiers of the LSM tree: -//! - Base table (on-disk, merged data) -//! - Flushed MemTable generations (on-disk L0) +//! - Base table (on-disk, compacted data) +//! - SSTables (on-disk L0) //! - Active MemTable (in-memory write buffer) //! //! Two phases, selected with `--phase`: @@ -151,7 +151,7 @@ struct Args { uri: String, base_rows: usize, max_memtable_rows: usize, - flushed_generations: usize, + sstables: usize, batch_rows: usize, queries: usize, output: Option, @@ -164,7 +164,7 @@ impl Default for Args { uri: String::new(), base_rows: 1_000_000, max_memtable_rows: 100_000, - flushed_generations: 2, + sstables: 2, batch_rows: 1_000, queries: 500, output: None, @@ -205,7 +205,7 @@ fn parse_args() -> Result { } "--base-rows" => args.base_rows = parse_val(&flag, &value)?, "--max-memtable-rows" => args.max_memtable_rows = parse_val(&flag, &value)?, - "--flushed-generations" => args.flushed_generations = parse_val(&flag, &value)?, + "--sstables" => args.sstables = parse_val(&flag, &value)?, "--batch-rows" => args.batch_rows = parse_val(&flag, &value)?, "--queries" => args.queries = parse_val(&flag, &value)?, "--output" => args.output = Some(PathBuf::from(value)), @@ -276,11 +276,11 @@ fn is_cloud_uri(uri: &str) -> bool { fn generate_lookup_ids( base_rows: usize, max_memtable_rows: usize, - flushed_generations: usize, + sstables: usize, queries: usize, ) -> (Vec>, Vec<&'static str>) { - let flushed_total = flushed_generations * max_memtable_rows; - let active_start = base_rows + flushed_total; + let sstable_total = sstables * max_memtable_rows; + let active_start = base_rows + sstable_total; let active_end = active_start + max_memtable_rows / 2; let mut groups = Vec::new(); @@ -296,19 +296,19 @@ fn generate_lookup_ids( groups.push(base_ids); names.push("base"); - // Flushed IDs (only if there are flushed generations) - if flushed_generations > 0 { - let flushed_start = base_rows; - let flushed_end = base_rows + flushed_total; - let flushed_ids: Vec = (0..queries) + // SSTable IDs (only if there are SSTables) + if sstables > 0 { + let sstable_start = base_rows; + let sstable_end = base_rows + sstable_total; + let sstable_ids: Vec = (0..queries) .map(|i| { - let range = flushed_end - flushed_start; + let range = sstable_end - sstable_start; let step = range.max(1) / queries.max(1); - (flushed_start + (i * step) % range) as i64 + (sstable_start + (i * step) % range) as i64 }) .collect(); - groups.push(flushed_ids); - names.push("flushed"); + groups.push(sstable_ids); + names.push("sstable"); } // Active memtable IDs @@ -338,7 +338,6 @@ async fn run_lookup(args: &Args) -> Result { wal_persist_retry_base_delay: std::time::Duration::from_millis(50), shard_spec_id: 0, durable_write: false, - sync_indexed_write: false, max_memtable_size: max_memtable_rows * 200, max_memtable_rows, max_wal_flush_interval: Some(Duration::from_secs(60)), @@ -354,11 +353,11 @@ async fn run_lookup(args: &Args) -> Result { }; let id_base = args.base_rows as i64; - let num_flushed = args.flushed_generations; + let num_sstables = args.sstables; let active_rows = max_memtable_rows / 2; - // Ingest flushed generations (each triggers a flush) + 1 active (50% full) - let mut gen_sizes: Vec = (0..num_flushed).map(|_| max_memtable_rows).collect(); + // Ingest SSTables (each triggers a flush) + 1 active (50% full) + let mut gen_sizes: Vec = (0..num_sstables).map(|_| max_memtable_rows).collect(); gen_sizes.push(active_rows); let mut cursor = 0usize; @@ -371,22 +370,22 @@ async fn run_lookup(args: &Args) -> Result { writer.put(vec![batch]).await?; cursor += rows; } - let is_flushed = gen_idx < num_flushed; + let is_sstable = gen_idx < num_sstables; println!( " gen {}: wrote {} rows ({}) cursor={}", gen_idx + 1, gen_rows, - if is_flushed { "flushed" } else { "active" }, + if is_sstable { "sstable" } else { "active" }, cursor, ); - if is_flushed { + if is_sstable { tokio::time::sleep(flush_wait).await; } } println!( - "ingested {} rows total ({} flushed gens + active)", - cursor, num_flushed + "ingested {} rows total ({} SSTables + active)", + cursor, num_sstables ); let manifest = writer.manifest().await.unwrap(); @@ -396,18 +395,15 @@ async fn run_lookup(args: &Args) -> Result { let mut shard_snapshot = ShardSnapshot::new(shard_id); if let Some(ref m) = manifest { shard_snapshot = shard_snapshot.with_current_generation(m.current_generation); - for fg in &m.flushed_generations { - shard_snapshot = shard_snapshot.with_flushed_generation(fg.generation, fg.path.clone()); + for sstable in &m.sstables { + shard_snapshot = shard_snapshot.with_sstable(sstable.generation, sstable.path.clone()); } } - let num_flushed = manifest - .as_ref() - .map(|m| m.flushed_generations.len()) - .unwrap_or(0); + let num_sstables = manifest.as_ref().map(|m| m.sstables.len()).unwrap_or(0); println!( - "manifest: {} flushed generations, current_generation={}", - num_flushed, + "manifest: {} SSTables, current_generation={}", + num_sstables, manifest.as_ref().map(|m| m.current_generation).unwrap_or(0) ); @@ -418,8 +414,12 @@ async fn run_lookup(args: &Args) -> Result { let planner = LsmPointLookupPlanner::new(collector, pk_columns, arrow_schema); // Generate lookup IDs for each category - let (id_groups, category_names) = - generate_lookup_ids(args.base_rows, max_memtable_rows, num_flushed, args.queries); + let (id_groups, category_names) = generate_lookup_ids( + args.base_rows, + max_memtable_rows, + num_sstables, + args.queries, + ); let session_ctx = SessionContext::new(); let task_ctx = session_ctx.task_ctx(); @@ -476,7 +476,7 @@ async fn run_lookup(args: &Args) -> Result { output.insert("phase".into(), json!("lookup")); output.insert("base_rows".into(), json!(args.base_rows)); output.insert("max_memtable_rows".into(), json!(max_memtable_rows)); - output.insert("flushed_generations".into(), json!(num_flushed)); + output.insert("sstables".into(), json!(num_sstables)); output.insert("active_rows".into(), json!(active_rows)); output.insert("queries_per_category".into(), json!(args.queries)); for (key, val) in &results { @@ -491,12 +491,12 @@ async fn run_lookup(args: &Args) -> Result { async fn run(args: Args) -> Result<()> { println!( - "bench=mem_wal_point_lookup phase={} uri={} base_rows={} max_memtable_rows={} flushed_generations={} batch_rows={} queries={}", + "bench=mem_wal_point_lookup phase={} uri={} base_rows={} max_memtable_rows={} sstables={} batch_rows={} queries={}", args.phase.as_str(), args.uri, args.base_rows, args.max_memtable_rows, - args.flushed_generations, + args.sstables, args.batch_rows, args.queries, ); diff --git a/rust/lance/benches/mem_wal/vector/hnsw/disk_ann_compare.py b/rust/lance/benches/mem_wal/vector/hnsw/disk_ann_compare.py index e3b81123ece..f93a06e5188 100644 --- a/rust/lance/benches/mem_wal/vector/hnsw/disk_ann_compare.py +++ b/rust/lance/benches/mem_wal/vector/hnsw/disk_ann_compare.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright The Lance Authors -"""Disk-ANN comparison: Lance on-disk IVF_HNSW_SQ (the flushed-memtable index) +"""Disk-ANN comparison: Lance on-disk IVF_HNSW_SQ (the SSTable index) vs DiskANN vs FAISS, all backed by local NVMe. faiss/diskannpy/lance bundle conflicting tcmalloc/MKL/OpenMP and crash if @@ -18,7 +18,12 @@ recall@10 vs p50/p99 latency and QPS. The Lance index is served fully cached (large index_cache_size_bytes). """ -import argparse, json, os, time + +import argparse +import json +import os +import time + import numpy as np K = 10 @@ -27,7 +32,9 @@ DIM = 1536 EF_SWEEP = [16, 32, 64, 128, 256] HF_TREE = "https://huggingface.co/api/datasets/KShivendu/dbpedia-entities-openai-1M/tree/main/data" -HF_BASE = "https://huggingface.co/datasets/KShivendu/dbpedia-entities-openai-1M/resolve/main/" +HF_BASE = ( + "https://huggingface.co/datasets/KShivendu/dbpedia-entities-openai-1M/resolve/main/" +) def data_dir(base, rows): @@ -42,10 +49,13 @@ def normalize(x): # ---------------- prepare ---------------- def load_corpus(cache_dir, needed): - import requests, pyarrow.parquet as pq + import pyarrow.parquet as pq + import requests + os.makedirs(cache_dir, exist_ok=True) shards = sorted( - e["path"] for e in requests.get(HF_TREE, timeout=60).json() + e["path"] + for e in requests.get(HF_TREE, timeout=60).json() if e["type"] == "file" and e["path"].endswith(".parquet") ) out = np.empty((needed, DIM), dtype=np.float32) @@ -62,7 +72,7 @@ def load_corpus(cache_dir, needed): col = pq.read_table(local, columns=["openai"]).column("openai") arr = np.stack(col.to_pylist()).astype(np.float32) take = min(len(arr), needed - n) - out[n:n + take] = arr[:take] + out[n : n + take] = arr[:take] n += take print(f" shard {os.path.basename(rel)} -> {take} (cum {n})", flush=True) assert n == needed, f"only got {n}/{needed}" @@ -73,7 +83,6 @@ def numpy_ground_truth(corpus, queries): gt = np.empty((len(queries), K), dtype=np.int64) # corpus is normalized -> cosine == inner product; chunk over corpus. chunk = 200_000 - sims_top = None # Compute full similarity in query-major chunks to bound memory. sim = np.zeros((len(queries), len(corpus)), dtype=np.float32) for s in range(0, len(corpus), chunk): @@ -98,10 +107,13 @@ def cmd_prepare(args): rng = np.random.default_rng(SEED) qidx = rng.choice(args.rows, size=NUM_QUERIES, replace=False) queries = corpus[qidx].copy() - print(f"corpus={len(corpus)} queries={len(queries)} dim={DIM}; computing GT...", flush=True) + print( + f"corpus={len(corpus)} queries={len(queries)} dim={DIM}; computing GT...", + flush=True, + ) t = time.perf_counter() gt = numpy_ground_truth(corpus, queries) - print(f" GT in {time.perf_counter()-t:.1f}s", flush=True) + print(f" GT in {time.perf_counter() - t:.1f}s", flush=True) np.save(os.path.join(d, "corpus.npy"), corpus) np.save(os.path.join(d, "queries.npy"), queries) np.save(os.path.join(d, "gt.npy"), gt) @@ -110,7 +122,9 @@ def cmd_prepare(args): # ---------------- shared run helpers ---------------- def recall_at_k(gt, got): - return sum(len(set(g.tolist()) & set(r.tolist())) for g, r in zip(gt, got)) / (len(gt) * K) + return sum(len(set(g.tolist()) & set(r.tolist())) for g, r in zip(gt, got)) / ( + len(gt) * K + ) def latency_qps(query_fn, queries, repeats=3): @@ -133,56 +147,97 @@ def sweep(name, make_q, params, queries, gt): got = np.stack([qf(v) for v in queries]) rec = recall_at_k(gt, got) p50, p99, qps = latency_qps(qf, queries) - rows.append({"param": p, "recall": rec, "p50_us": p50, "p99_us": p99, "qps": qps}) - print(f" {name} param={p} recall={rec:.4f} p50={p50:.0f}us p99={p99:.0f}us qps={qps:.0f}", flush=True) + rows.append( + {"param": p, "recall": rec, "p50_us": p50, "p99_us": p99, "qps": qps} + ) + print( + f" {name} param={p} recall={rec:.4f} " + f"p50={p50:.0f}us p99={p99:.0f}us qps={qps:.0f}", + flush=True, + ) return rows # ---------------- systems ---------------- def run_lance(base, rows, corpus, queries, gt): - import lance, pyarrow as pa, shutil + import shutil + + import lance + import pyarrow as pa + uri = os.path.join(base, f"lance_{rows}") shutil.rmtree(uri, ignore_errors=True) - vecs = pa.FixedSizeListArray.from_arrays(pa.array(corpus.reshape(-1), type=pa.float32()), DIM) + vecs = pa.FixedSizeListArray.from_arrays( + pa.array(corpus.reshape(-1), type=pa.float32()), DIM + ) tbl = pa.table({"id": pa.array(np.arange(rows, dtype=np.int64)), "vec": vecs}) ds = lance.write_dataset(tbl, uri, mode="overwrite") - # The flushed memtable index is a SINGLE-partition HNSW+SQ, so model it with + # The SSTable index is a SINGLE-partition HNSW+SQ, so model it with # num_partitions=1 (nprobes=1); ef is the search knob, like DiskANN/FAISS. t = time.perf_counter() - ds.create_index("vec", "IVF_HNSW_SQ", metric="cosine", num_partitions=1, - m=20, ef_construction=150) + ds.create_index( + "vec", + "IVF_HNSW_SQ", + metric="cosine", + num_partitions=1, + m=20, + ef_construction=150, + ) build_s = time.perf_counter() - t ds = lance.dataset(uri, index_cache_size_bytes=48 * 1024**3) def make_q(ef): def q(v): - return ds.to_table(nearest={"column": "vec", "q": v, "k": K, - "nprobes": 1, "ef": ef}, - columns=["id"]).column("id").to_numpy() + return ( + ds.to_table( + nearest={"column": "vec", "q": v, "k": K, "nprobes": 1, "ef": ef}, + columns=["id"], + ) + .column("id") + .to_numpy() + ) + return q - return {"build_s": build_s, "nlist": 1, "sweep": sweep("lance", make_q, None, queries, gt)} + + return { + "build_s": build_s, + "nlist": 1, + "sweep": sweep("lance", make_q, None, queries, gt), + } -def run_lance_flushed(base, rows, corpus, queries, gt, lance_path, id_offset, column): - # Open a flushed MemTable generation directly from its dataset path and +def run_lance_sstable(base, rows, corpus, queries, gt, lance_path, id_offset, column): + # Open an SSTable generation directly from its dataset path and # benchmark its on-disk IVF_HNSW_SQ index (single partition), fully cached. import lance + ds = lance.dataset(lance_path, index_cache_size_bytes=48 * 1024**3) def make_q(ef): def q(v): - ids = ds.to_table(nearest={"column": column, "q": v, "k": K, - "nprobes": 1, "ef": ef}, - columns=["id"]).column("id").to_numpy() - return ids - id_offset # map flushed-gen id -> corpus index + ids = ( + ds.to_table( + nearest={"column": column, "q": v, "k": K, "nprobes": 1, "ef": ef}, + columns=["id"], + ) + .column("id") + .to_numpy() + ) + return ids - id_offset # map SSTable id -> corpus index + return q - return {"lance_path": lance_path, "id_offset": id_offset, - "sweep": sweep("lance_flushed", make_q, None, queries, gt)} + + return { + "lance_path": lance_path, + "id_offset": id_offset, + "sweep": sweep("lance_sstable", make_q, None, queries, gt), + } def run_faiss(base, rows, corpus, queries, gt): # Full-precision HNSW reference (shows what no quantization buys). import faiss + index = faiss.IndexHNSWFlat(DIM, 32, faiss.METRIC_INNER_PRODUCT) index.hnsw.efConstruction = 200 t = time.perf_counter() @@ -194,16 +249,20 @@ def make_q(ef): def q(v): index.hnsw.efSearch = ef return index.search(v.reshape(1, -1), K)[1][0] + return q + return {"build_s": build_s, "sweep": sweep("faiss", make_q, None, queries, gt)} def run_faiss_sq(base, rows, corpus, queries, gt): # HNSW + 8-bit scalar quantization — apples-to-apples with Lance IVF_HNSW_SQ. import faiss + try: - index = faiss.IndexHNSWSQ(DIM, faiss.ScalarQuantizer.QT_8bit, 32, - faiss.METRIC_INNER_PRODUCT) + index = faiss.IndexHNSWSQ( + DIM, faiss.ScalarQuantizer.QT_8bit, 32, faiss.METRIC_INNER_PRODUCT + ) except Exception: # Fall back to L2; on unit-normalized vectors L2 ranking == cosine. index = faiss.IndexHNSWSQ(DIM, faiss.ScalarQuantizer.QT_8bit, 32) @@ -218,28 +277,44 @@ def make_q(ef): def q(v): index.hnsw.efSearch = ef return index.search(v.reshape(1, -1), K)[1][0] + return q + return {"build_s": build_s, "sweep": sweep("faiss_sq", make_q, None, queries, gt)} def run_diskann(base, rows, corpus, queries, gt): import diskannpy as dap + idx_dir = os.path.join(base, f"diskann_{rows}") os.makedirs(idx_dir, exist_ok=True) t = time.perf_counter() dap.build_memory_index( - data=corpus, distance_metric="cosine", index_directory=idx_dir, - index_prefix="ann", complexity=150, graph_degree=64, - num_threads=0, alpha=1.2, use_pq_build=False, num_pq_bytes=0, + data=corpus, + distance_metric="cosine", + index_directory=idx_dir, + index_prefix="ann", + complexity=150, + graph_degree=64, + num_threads=0, + alpha=1.2, + use_pq_build=False, + num_pq_bytes=0, ) build_s = time.perf_counter() - t - idx = dap.StaticMemoryIndex(index_directory=idx_dir, index_prefix="ann", - num_threads=0, initial_search_complexity=256) + idx = dap.StaticMemoryIndex( + index_directory=idx_dir, + index_prefix="ann", + num_threads=0, + initial_search_complexity=256, + ) def make_q(L): def q(v): return idx.search(v, k_neighbors=K, complexity=max(L, K)).identifiers + return q + return {"build_s": build_s, "sweep": sweep("diskann", make_q, None, queries, gt)} @@ -249,12 +324,24 @@ def cmd_run(args): queries = np.load(os.path.join(d, "queries.npy")) gt = np.load(os.path.join(d, "gt.npy")) print(f"=== {args.system} rows={args.rows} corpus={len(corpus)} ===", flush=True) - if args.system == "lance_flushed": - res = run_lance_flushed(args.base, args.rows, corpus, queries, gt, - args.lance_path, args.id_offset, args.column) + if args.system == "lance_sstable": + res = run_lance_sstable( + args.base, + args.rows, + corpus, + queries, + gt, + args.lance_path, + args.id_offset, + args.column, + ) else: - fn = {"lance": run_lance, "faiss": run_faiss, "faiss_sq": run_faiss_sq, - "diskann": run_diskann}[args.system] + fn = { + "lance": run_lance, + "faiss": run_faiss, + "faiss_sq": run_faiss_sq, + "diskann": run_diskann, + }[args.system] res = fn(args.base, args.rows, corpus, queries, gt) res["rows"] = args.rows res["system"] = args.system @@ -267,9 +354,16 @@ def cmd_run(args): def main(): ap = argparse.ArgumentParser() sub = ap.add_subparsers(dest="cmd", required=True) - p = sub.add_parser("prepare"); p.add_argument("--rows", type=int, required=True); p.add_argument("--base", required=True) - r = sub.add_parser("run"); r.add_argument("--rows", type=int, required=True); r.add_argument("--base", required=True); r.add_argument("--system", required=True) - r.add_argument("--lance-path", default=None); r.add_argument("--id-offset", type=int, default=0); r.add_argument("--column", default="vector") + p = sub.add_parser("prepare") + p.add_argument("--rows", type=int, required=True) + p.add_argument("--base", required=True) + r = sub.add_parser("run") + r.add_argument("--rows", type=int, required=True) + r.add_argument("--base", required=True) + r.add_argument("--system", required=True) + r.add_argument("--lance-path", default=None) + r.add_argument("--id-offset", type=int, default=0) + r.add_argument("--column", default="vector") args = ap.parse_args() (cmd_prepare if args.cmd == "prepare" else cmd_run)(args) diff --git a/rust/lance/benches/mem_wal/vector/hnsw/mem_wal_recall_hnsw.rs b/rust/lance/benches/mem_wal/vector/hnsw/mem_wal_recall_hnsw.rs index 7b9f35b73c9..625900cf834 100644 --- a/rust/lance/benches/mem_wal/vector/hnsw/mem_wal_recall_hnsw.rs +++ b/rust/lance/benches/mem_wal/vector/hnsw/mem_wal_recall_hnsw.rs @@ -355,7 +355,7 @@ async fn run_checkpoint( let temp = tempfile::tempdir().map_err(|e| lance_core::Error::io(format!("tempdir: {}", e)))?; // When BENCH_URI_BASE is set, write to a persistent path and flush the // MemTable to an on-disk generation (instead of querying the active - // MemTable), printing the flushed generation's dataset path for a + // MemTable), printing the SSTable's dataset path for a // downstream direct-read benchmark. let flush_base = std::env::var("BENCH_URI_BASE").ok(); let local_dir = flush_base.as_ref().map(|b| format!("{}/cp_{}", b, cp)); @@ -389,7 +389,6 @@ async fn run_checkpoint( wal_persist_retry_base_delay: std::time::Duration::from_millis(50), shard_spec_id: 0, durable_write: false, - sync_indexed_write: true, max_memtable_size: cp.saturating_mul(row_size_estimate).saturating_mul(4), max_memtable_rows: cp.saturating_mul(2), max_memtable_batches: total_batches_max.saturating_mul(2).max(8_000), @@ -416,11 +415,11 @@ async fn run_checkpoint( writer.put(vec![batch]).await?; } - let target_batch_pos = total_batches.saturating_sub(1); + let target_indexed_count = total_batches; let mut spins = 0u64; loop { let active = writer.active_memtable_ref().await?; - if active.index_store.max_visible_batch_position() >= target_batch_pos { + if active.index_store.indexed_count() >= target_indexed_count { break; } drop(active); @@ -450,9 +449,14 @@ async fn run_checkpoint( let mut waited = 0u64; let gen_path = loop { if let Some(m) = writer.manifest().await? - && let Some(fg) = m.flushed_generations.last() + && let Some(sstable) = m.sstables.last() { - break format!("{}/_mem_wal/{}/{}", dir, shard_id.as_hyphenated(), fg.path); + break format!( + "{}/_mem_wal/{}/{}", + dir, + shard_id.as_hyphenated(), + sstable.path + ); } tokio::time::sleep(Duration::from_millis(100)).await; waited += 1; @@ -467,7 +471,7 @@ async fn run_checkpoint( } }; println!( - "FLUSHED_OK cp={} id_offset={} flush_s={:.2} path={}", + "SSTABLE_OK cp={} id_offset={} flush_s={:.2} path={}", cp, id_offset, seal_start.elapsed().as_secs_f64(), @@ -476,7 +480,7 @@ async fn run_checkpoint( std::io::stdout().flush().ok(); writer.close().await?; - // Item 4: open the flushed generation directly from its dataset path and + // Item 4: open the SSTable directly from its dataset path and // benchmark its on-disk IVF_HNSW_SQ read in Rust (single-thread per-query // latency + recall vs brute force), sweeping ef. nprobes=1 (single part). let gen_uri = format!("file://{}", gen_path); @@ -526,7 +530,7 @@ async fn run_checkpoint( let p99_q = lat[lat.len() * 99 / 100]; let mean_recall = recall_sum / num_queries as f64; println!( - "FLUSHED_READ cp={} ef={} mean_recall={:.4} median_us={} p99_us={}", + "SSTABLE_READ cp={} ef={} mean_recall={:.4} median_us={} p99_us={}", cp, ef, mean_recall, median_q, p99_q ); std::io::stdout().flush().ok(); @@ -537,13 +541,13 @@ async fn run_checkpoint( // Raw-index path: call VectorIndex::search() directly (partition-find + // HNSW+SQ search, returns _rowid/_distance) bypassing the DataFusion - // scanner/take/projection. The RAW_READ vs FLUSHED_READ latency delta at + // scanner/take/projection. The RAW_READ vs SSTABLE_READ latency delta at // matched ef isolates the DataFusion per-query overhead from the actual // index search cost. let idx_metas = gen_ds.load_indices_by_name(VECTOR_INDEX_NAME).await?; let uuid = idx_metas .first() - .ok_or_else(|| lance_core::Error::io("flushed gen has no vector index".to_string()))? + .ok_or_else(|| lance_core::Error::io("SSTable has no vector index".to_string()))? .uuid; let vidx = gen_ds .open_vector_index(VECTOR_COL, &uuid, &NoOpMetricsCollector) @@ -595,7 +599,7 @@ async fn run_checkpoint( }; // IVFIndex::search is intentionally unimplemented (top-level does // partition-aware search); replicate the ANN exec node: pick the - // closest partition then search it. Single-partition flushed gen. + // closest partition then search it. Single-partition SSTable. let t = Instant::now(); let (parts, _) = vidx.find_partitions(&query)?; let pid = parts.value(0) as usize; diff --git a/rust/lance/benches/mem_wal/vector/mem_wal_index_micro.rs b/rust/lance/benches/mem_wal/vector/mem_wal_index_micro.rs index 8bff64dfcca..c6fb42834a3 100644 --- a/rust/lance/benches/mem_wal/vector/mem_wal_index_micro.rs +++ b/rust/lance/benches/mem_wal/vector/mem_wal_index_micro.rs @@ -10,7 +10,9 @@ //! index maintained, opens `mem_wal_writer` with a `ShardWriterConfig` //! sized to hold the largest checkpoint without flushing, and times //! `writer.put(batch)` calls. Index updates therefore go through the -//! parallel `IndexStore::insert_batches_parallel` path. At each +//! `IndexStore::insert_batches` path, which indexes inline at or below +//! `PARALLEL_INDEX_MIN_ROWS` rows and spawns a thread per index above it — +//! so the checkpoint sizes here straddle that crossover. At each //! checkpoint, queries are issued against `active_memtable_ref()` via //! `MemTableScanner::nearest`. //! @@ -212,7 +214,6 @@ async fn main() -> lance_core::Result<()> { wal_persist_retry_base_delay: std::time::Duration::from_millis(50), shard_spec_id: 0, durable_write, - sync_indexed_write: true, max_memtable_size: max_rows.saturating_mul(row_size_estimate).saturating_mul(4), max_memtable_rows: max_rows.saturating_mul(2), max_memtable_batches: total_batches_max.saturating_mul(2).max(8_000), @@ -253,7 +254,7 @@ async fn main() -> lance_core::Result<()> { while next_cp_idx < checkpoints.len() && total_inserted >= checkpoints[next_cp_idx] { let cp = checkpoints[next_cp_idx]; - let target_batch_pos = (cp / batch_size).saturating_sub(1); + let target_indexed_count = cp / batch_size; // The WAL flush handler only updates the index watermark when a // flush is triggered, and the time-based trigger inside the // writer runs only when `put()` is called. After the final put @@ -265,7 +266,7 @@ async fn main() -> lance_core::Result<()> { let mut spins = 0u64; loop { let active = writer.active_memtable_ref().await?; - if active.index_store.max_visible_batch_position() >= target_batch_pos { + if active.index_store.indexed_count() >= target_indexed_count { break; } drop(active); @@ -375,12 +376,11 @@ async fn measure_flush( memtable.set_indexes(registry); let total_batches = cp.div_ceil(batch_size); - for (wal_pos, i) in (0_u64..).zip(0..total_batches) { + for i in 0..total_batches { let start = (i * batch_size) as i64; let rows = batch_size.min(cp - i * batch_size); let batch = make_batch(start, rows, dim); - let frag_id = memtable.insert(batch).await?; - memtable.mark_wal_flushed(&[frag_id], wal_pos + 1, &[i]); + let _frag_id = memtable.insert(batch).await?; } let temp_dir = @@ -406,11 +406,19 @@ async fn measure_flush( let flusher = MemTableFlusher::new(store, base_path, uri, shard_id, manifest_store); // total_batches WAL entries were stamped at positions 1..=total_batches - // by the mark_wal_flushed loop above (1-based positions). + // by the loop above (1-based positions). let covered_wal_entry_position = total_batches as u64; + // Every batch was appended above, so the writer-global durable count is all of them. + let durable = total_batches; let t = Instant::now(); let _result = flusher - .flush_with_indexes(&memtable, epoch, index_configs, covered_wal_entry_position) + .flush_with_indexes( + &memtable, + epoch, + index_configs, + covered_wal_entry_position, + durable, + ) .await?; let elapsed = t.elapsed(); diff --git a/rust/lance/benches/mem_wal/vector/mem_wal_vector_bench.rs b/rust/lance/benches/mem_wal/vector/mem_wal_vector_bench.rs index 632bd37626c..e5bd050fd65 100644 --- a/rust/lance/benches/mem_wal/vector/mem_wal_vector_bench.rs +++ b/rust/lance/benches/mem_wal/vector/mem_wal_vector_bench.rs @@ -6,7 +6,7 @@ //! //! Uses real embeddings from the `lance-format/fineweb-edu` HuggingFace //! dataset (384-dim) with an IVF-RQ index on the base table, then ingests -//! additional rows through ShardWriter to populate flushed generations and +//! additional rows through ShardWriter to populate SSTables and //! an active memtable. //! //! Three phases, selected with `--phase`: @@ -102,7 +102,7 @@ struct Args { uri: String, base_rows: usize, max_memtable_rows: usize, - flushed_generations: usize, + sstables: usize, batch_rows: usize, queries: usize, k: usize, @@ -120,7 +120,7 @@ impl Default for Args { uri: String::new(), base_rows: 1_000_000, max_memtable_rows: 100_000, - flushed_generations: 2, + sstables: 2, batch_rows: 1_000, queries: 100, k: 10, @@ -166,7 +166,7 @@ fn parse_args() -> Result { } "--base-rows" => args.base_rows = parse_val(&flag, &value)?, "--max-memtable-rows" => args.max_memtable_rows = parse_val(&flag, &value)?, - "--flushed-generations" => args.flushed_generations = parse_val(&flag, &value)?, + "--sstables" => args.sstables = parse_val(&flag, &value)?, "--batch-rows" => args.batch_rows = parse_val(&flag, &value)?, "--queries" => args.queries = parse_val(&flag, &value)?, "--k" => args.k = parse_val(&flag, &value)?, @@ -520,7 +520,6 @@ async fn run_search(args: &Args) -> Result { wal_persist_retry_base_delay: std::time::Duration::from_millis(50), shard_spec_id: 0, durable_write: false, - sync_indexed_write: false, max_memtable_size: args.max_memtable_rows * row_bytes * 2, max_memtable_rows: args.max_memtable_rows, max_unflushed_memtable_bytes: args.max_memtable_rows * row_bytes * 6, @@ -535,11 +534,11 @@ async fn run_search(args: &Args) -> Result { Duration::from_millis(500) }; - // Ingest N flushed generations + 1 active (50% full) - let num_flushed_target = args.flushed_generations; + // Ingest N SSTables + 1 active (50% full) + let num_sstable_target = args.sstables; let active_rows = args.max_memtable_rows / 2; - let total_memtable_rows = num_flushed_target * args.max_memtable_rows + active_rows; - let mut gen_sizes: Vec = (0..num_flushed_target) + let total_memtable_rows = num_sstable_target * args.max_memtable_rows + active_rows; + let mut gen_sizes: Vec = (0..num_sstable_target) .map(|_| args.max_memtable_rows) .collect(); gen_sizes.push(active_rows); @@ -616,14 +615,14 @@ async fn run_search(args: &Args) -> Result { gen_rows, ingest_start.elapsed().as_secs_f64(), ); - if gen_idx < num_flushed_target { + if gen_idx < num_sstable_target { tokio::time::sleep(flush_wait).await; } } println!( - "ingested {} total memtable rows ({} flushed + active) in {:.1}s", + "ingested {} total memtable rows ({} SSTables + active) in {:.1}s", total_memtable_rows, - num_flushed_target, + num_sstable_target, ingest_start.elapsed().as_secs_f64(), ); @@ -633,17 +632,14 @@ async fn run_search(args: &Args) -> Result { let mut shard_snapshot = ShardSnapshot::new(shard_id); if let Some(ref m) = manifest { shard_snapshot = shard_snapshot.with_current_generation(m.current_generation); - for fg in &m.flushed_generations { - shard_snapshot = shard_snapshot.with_flushed_generation(fg.generation, fg.path.clone()); + for sstable in &m.sstables { + shard_snapshot = shard_snapshot.with_sstable(sstable.generation, sstable.path.clone()); } } - let num_flushed = manifest - .as_ref() - .map(|m| m.flushed_generations.len()) - .unwrap_or(0); + let num_sstables = manifest.as_ref().map(|m| m.sstables.len()).unwrap_or(0); println!( - "manifest: {} flushed generations, current_generation={}", - num_flushed, + "manifest: {} SSTables, current_generation={}", + num_sstables, manifest.as_ref().map(|m| m.current_generation).unwrap_or(0) ); @@ -761,7 +757,7 @@ async fn run_search(args: &Args) -> Result { "phase": "search", "base_rows": args.base_rows, "max_memtable_rows": args.max_memtable_rows, - "flushed_generations": num_flushed, + "sstables": num_sstables, "active_rows": active_rows, "vector_dim": VECTOR_DIM, "k": args.k, diff --git a/rust/lance/benches/mem_wal/write/mem_wal_replay.rs b/rust/lance/benches/mem_wal/write/mem_wal_replay.rs index 14ec406e5e5..6fda0748006 100644 --- a/rust/lance/benches/mem_wal/write/mem_wal_replay.rs +++ b/rust/lance/benches/mem_wal/write/mem_wal_replay.rs @@ -179,7 +179,6 @@ async fn populate_shard_wal( let dataset = Dataset::open(dataset_uri).await.unwrap(); let mut config = ShardWriterConfig::new(shard_id); config.durable_write = true; - config.sync_indexed_write = false; let writer = dataset.mem_wal_writer(shard_id, config).await.unwrap(); for i in 0..num_entries { diff --git a/rust/lance/benches/mem_wal/write/mem_wal_shard_writer_backpressure.rs b/rust/lance/benches/mem_wal/write/mem_wal_shard_writer_backpressure.rs index f4e4fb47395..79298daa872 100644 --- a/rust/lance/benches/mem_wal/write/mem_wal_shard_writer_backpressure.rs +++ b/rust/lance/benches/mem_wal/write/mem_wal_shard_writer_backpressure.rs @@ -99,10 +99,6 @@ impl Mode { fn durable_write(self) -> bool { matches!(self, Self::SyncNoIndex | Self::SyncIndexed) } - - fn sync_indexed_write(self) -> bool { - matches!(self, Self::SyncIndexed) - } } /// Which index the MemTable maintains in the indexed (`*_idx`) modes. @@ -192,7 +188,6 @@ struct Args { max_memtable_batches: Option, max_wal_buffer_size: usize, max_wal_flush_interval_ms: u64, - async_index_buffer_rows: usize, sample_interval_ms: u64, target_rows_per_sec: Option, num_partitions: usize, @@ -223,7 +218,6 @@ impl Default for Args { max_memtable_batches: None, max_wal_buffer_size: 10 * 1024 * 1024, max_wal_flush_interval_ms: 100, - async_index_buffer_rows: 10_000, sample_interval_ms: 500, target_rows_per_sec: None, num_partitions: 1, @@ -343,11 +337,9 @@ async fn run(args: Args) -> Result<()> { let shard_id = Uuid::new_v4(); let mut config = ShardWriterConfig::new(shard_id) .with_durable_write(args.mode.durable_write()) - .with_sync_indexed_write(args.mode.sync_indexed_write()) .with_max_memtable_size(args.max_memtable_size) .with_max_unflushed_memtable_bytes(args.max_unflushed_memtable_bytes) .with_max_wal_buffer_size(args.max_wal_buffer_size) - .with_async_index_buffer_rows(args.async_index_buffer_rows) .with_max_memtable_rows(memtable_limits.rows) .with_max_memtable_batches(memtable_limits.batches); if args.max_wal_flush_interval_ms == 0 { @@ -544,7 +536,6 @@ async fn run(args: Args) -> Result<()> { "max_memtable_batches": memtable_limits.batches, "max_wal_buffer_size": args.max_wal_buffer_size, "max_wal_flush_interval_ms": args.max_wal_flush_interval_ms, - "async_index_buffer_rows": args.async_index_buffer_rows, "sample_interval_ms": args.sample_interval_ms, "skip_close": args.skip_close, "setup_seconds": setup_s, @@ -650,7 +641,7 @@ fn push_sample( "active_memtable_bytes": memtable.as_ref().map(|stats| stats.estimated_size), "active_memtable_generation": memtable.as_ref().map(|stats| stats.generation), "active_memtable_max_buffered_batch_position": memtable.as_ref().and_then(|stats| stats.max_buffered_batch_position), - "active_memtable_max_flushed_batch_position": memtable.as_ref().and_then(|stats| stats.max_flushed_batch_position), + "active_memtable_durable_batch_count": memtable.as_ref().map(|stats| stats.durable_batch_count), "wal_queue_pending_batches": memtable.as_ref().map(|stats| stats.pending_wal_batch_count), "wal_queue_pending_rows": memtable.as_ref().map(|stats| stats.pending_wal_row_count), "wal_queue_pending_bytes": memtable.as_ref().map(|stats| stats.pending_wal_estimated_bytes), @@ -667,7 +658,7 @@ fn memtable_stats_json(memtable: Option<&MemTableStats>) -> serde_json::Value { "estimated_size": stats.estimated_size, "generation": stats.generation, "max_buffered_batch_position": stats.max_buffered_batch_position, - "max_flushed_batch_position": stats.max_flushed_batch_position, + "durable_batch_count": stats.durable_batch_count, "wal_queue_pending_start_batch_position": stats.pending_wal_start_batch_position, "wal_queue_pending_end_batch_position": stats.pending_wal_end_batch_position, "wal_queue_pending_batches": stats.pending_wal_batch_count, @@ -980,7 +971,6 @@ fn parse_args() -> Result { "--max-wal-flush-interval-ms" => { args.max_wal_flush_interval_ms = parse(&flag, &value)?; } - "--async-index-buffer-rows" => args.async_index_buffer_rows = parse(&flag, &value)?, "--sample-interval-ms" => args.sample_interval_ms = parse(&flag, &value)?, "--target-rows-per-sec" => args.target_rows_per_sec = Some(parse(&flag, &value)?), "--num-partitions" => args.num_partitions = parse(&flag, &value)?, diff --git a/rust/lance/benches/mem_wal/write/mem_wal_write.rs b/rust/lance/benches/mem_wal/write/mem_wal_write.rs index 6d3a31c4011..b5018fe09f1 100644 --- a/rust/lance/benches/mem_wal/write/mem_wal_write.rs +++ b/rust/lance/benches/mem_wal/write/mem_wal_write.rs @@ -30,7 +30,6 @@ //! - `BATCH_SIZE`: Number of rows per write batch (default: 20) //! - `NUM_BATCHES`: Total number of batches to write (default: 1000) //! - `DURABLE_WRITE`: yes/no/both (default: no) - whether writes wait for WAL flush -//! - `INDEXED_WRITE`: yes/no/both (default: no) - whether writes update indexes synchronously //! - `MAX_WAL_BUFFER_SIZE`: WAL buffer size in bytes (default: 1MB from ShardWriterConfig) //! - `MAX_FLUSH_INTERVAL_MS`: WAL flush interval in milliseconds, 0 to disable (default: 1000ms) //! - `MAX_MEMTABLE_SIZE`: MemTable size threshold in bytes (default: 64MB from ShardWriterConfig) @@ -44,8 +43,7 @@ //! `no` uses WAL-only mode (no MemTable, no indexes, no Lance flushes; pure WAL throughput). //! `both` runs each combination twice, once per mode, side-by-side. //! When `no` or `both`, the WAL-only branch always runs with -//! `MEMWAL_MAINTAINED_INDEXES=none` and skips `INDEXED_WRITE=yes` -//! (sync-indexed writes require a MemTable). +//! `MEMWAL_MAINTAINED_INDEXES=none`. //! - `SAMPLE_SIZE`: Number of benchmark iterations (default: 10, minimum: 10) #![allow(clippy::print_stdout, clippy::print_stderr)] @@ -119,11 +117,6 @@ fn get_durable_write_options() -> Vec { parse_yes_no_both("DURABLE_WRITE", "no") } -/// Get indexed write settings from environment. -fn get_indexed_write_options() -> Vec { - parse_yes_no_both("INDEXED_WRITE", "no") -} - /// Get enable_memtable settings from environment. Default `yes` keeps /// existing benchmark behavior; `no` runs WAL-only mode; `both` runs both /// modes side-by-side for comparison. @@ -442,31 +435,26 @@ fn build_label( num_batches: usize, batch_size: usize, durable: bool, - indexed: bool, enable_memtable: bool, storage: &str, ) -> String { let durable_str = if durable { "durable" } else { "nondurable" }; - // sync_indexed_write controls sync vs async index updates - let indexed_str = if indexed { "sync_idx" } else { "async_idx" }; let mode_str = if enable_memtable { "memtable" } else { "wal_only" }; format!( - "{}x{} {} {} {} ({})", - num_batches, batch_size, mode_str, durable_str, indexed_str, storage + "{}x{} {} {} ({})", + num_batches, batch_size, mode_str, durable_str, storage ) } /// Build dataset name prefix from config options. -fn build_name_prefix(durable: bool, indexed: bool, enable_memtable: bool) -> String { +fn build_name_prefix(durable: bool, enable_memtable: bool) -> String { let d = if durable { "d" } else { "nd" }; - // sync_indexed_write: sync (si) vs async (ai) - let i = if indexed { "si" } else { "ai" }; let m = if enable_memtable { "mt" } else { "wo" }; - format!("{}_{}_{}", m, d, i) + format!("{}_{}", m, d) } /// Benchmark Lance MemWAL write throughput. @@ -490,7 +478,6 @@ fn bench_lance_memwal_write(c: &mut Criterion) { let maintained_indexes = get_maintained_indexes(); let durable_options = get_durable_write_options(); - let indexed_options = get_indexed_write_options(); let enable_memtable_options = get_enable_memtable_options(); let max_wal_buffer_size = get_max_wal_buffer_size(); let max_flush_interval = get_max_flush_interval(); @@ -550,70 +537,56 @@ fn bench_lance_memwal_write(c: &mut Criterion) { // Generate benchmarks for all combinations for &enable_memtable in &enable_memtable_options { for &durable in &durable_options { - for &indexed in &indexed_options { - if !enable_memtable && indexed { - eprintln!( - "Skipping wal_only + sync_idx (sync_indexed_write requires a MemTable)" - ); - continue; - } - - let label = build_label( - num_batches, - batch_size, - durable, - indexed, - enable_memtable, - storage_label, - ); - let name_prefix = build_name_prefix(durable, indexed, enable_memtable); - - // WAL-only mode never uses indexes; force the dataset - // setup to skip the MemWAL index list. - let effective_indexes: Vec = if enable_memtable { - maintained_indexes.clone() - } else { - Vec::new() - }; - - // Create dataset ONCE before benchmark iterations - // Each iteration will use a different shard on the same dataset - let dataset = rt.block_on(create_dataset( - &schema, - &name_prefix, - vector_dim, - &effective_indexes, - &dataset_prefix, - )); - let dataset_uri = dataset.uri().to_string(); - - // Pre-generate all batches before timing (outside iter_custom) - let batches: Arc> = Arc::new( - (0..num_batches) - .map(|i| { - create_test_batch( - &schema, - (i * batch_size) as i64, - batch_size, - vector_dim, - ) - }) - .collect(), - ); - - println!("Running: {}", label); - - // Track if we've printed stats (only print once across all samples) - let stats_printed = Arc::new(AtomicBool::new(false)); - - group.bench_with_input( - BenchmarkId::new("Lance MemWAL", &label), - &(batch_size, num_batches, durable, indexed, row_size_bytes), - |b, &(_batch_size, _num_batches, durable, indexed, row_size_bytes)| { - let dataset_uri = dataset_uri.clone(); - let batches = batches.clone(); - let stats_printed = stats_printed.clone(); - b.to_async(&rt).iter_custom(|iters| { + let label = build_label( + num_batches, + batch_size, + durable, + enable_memtable, + storage_label, + ); + let name_prefix = build_name_prefix(durable, enable_memtable); + + // WAL-only mode never uses indexes; force the dataset + // setup to skip the MemWAL index list. + let effective_indexes: Vec = if enable_memtable { + maintained_indexes.clone() + } else { + Vec::new() + }; + + // Create dataset ONCE before benchmark iterations + // Each iteration will use a different shard on the same dataset + let dataset = rt.block_on(create_dataset( + &schema, + &name_prefix, + vector_dim, + &effective_indexes, + &dataset_prefix, + )); + let dataset_uri = dataset.uri().to_string(); + + // Pre-generate all batches before timing (outside iter_custom) + let batches: Arc> = Arc::new( + (0..num_batches) + .map(|i| { + create_test_batch(&schema, (i * batch_size) as i64, batch_size, vector_dim) + }) + .collect(), + ); + + println!("Running: {}", label); + + // Track if we've printed stats (only print once across all samples) + let stats_printed = Arc::new(AtomicBool::new(false)); + + group.bench_with_input( + BenchmarkId::new("Lance MemWAL", &label), + &(batch_size, num_batches, durable, row_size_bytes), + |b, &(_batch_size, _num_batches, durable, row_size_bytes)| { + let dataset_uri = dataset_uri.clone(); + let batches = batches.clone(); + let stats_printed = stats_printed.clone(); + b.to_async(&rt).iter_custom(|iters| { let dataset_uri = dataset_uri.clone(); let batches = batches.clone(); let stats_printed = stats_printed.clone(); @@ -631,10 +604,10 @@ fn bench_lance_memwal_write(c: &mut Criterion) { shard_id, shard_spec_id: 0, max_wal_persist_retries: 3, - wal_persist_retry_base_delay: - std::time::Duration::from_millis(50), + wal_persist_retry_base_delay: std::time::Duration::from_millis( + 50, + ), durable_write: durable, - sync_indexed_write: indexed, max_wal_buffer_size: max_wal_buffer_size .unwrap_or(default_config.max_wal_buffer_size), max_wal_flush_interval: max_flush_interval @@ -643,8 +616,6 @@ fn bench_lance_memwal_write(c: &mut Criterion) { .unwrap_or(default_config.max_memtable_size), max_memtable_rows: default_config.max_memtable_rows, max_memtable_batches: default_config.max_memtable_batches, - async_index_buffer_rows: default_config.async_index_buffer_rows, - async_index_interval: default_config.async_index_interval, manifest_scan_batch_size: default_config .manifest_scan_batch_size, max_unflushed_memtable_bytes: default_config @@ -656,6 +627,8 @@ fn bench_lance_memwal_write(c: &mut Criterion) { enable_memtable, hnsw_params: default_config.hnsw_params, warmer: None, + store_params: default_config.store_params, + session: default_config.session, }; // Get writer through Dataset API (index configs loaded automatically) @@ -704,9 +677,8 @@ fn bench_lance_memwal_write(c: &mut Criterion) { total_duration } }) - }, - ); - } + }, + ); } } diff --git a/rust/lance/benches/random_access.rs b/rust/lance/benches/random_access.rs index ef86f812ea4..6fb76f0783b 100644 --- a/rust/lance/benches/random_access.rs +++ b/rust/lance/benches/random_access.rs @@ -7,7 +7,7 @@ use arrow_array::{Float64Array, Int64Array, RecordBatch, RecordBatchIterator, St use arrow_schema::{DataType, Field, Schema as ArrowSchema}; use criterion::{Criterion, criterion_group, criterion_main}; use lance::dataset::{Dataset, ProjectionRequest, WriteParams}; -use lance_file::version::LanceFileVersion; +use lance_file::version::{ConcreteFileVersion, LanceFileVersion}; use std::collections::HashMap; use tokio::runtime::Runtime; use uuid::Uuid; @@ -85,7 +85,12 @@ fn utf8_field_without_fsst(name: &str) -> Field { } fn utf8_field_for(version: LanceFileVersion, enable_fsst: bool, name: &str) -> Field { - if enable_fsst && version >= LanceFileVersion::V2_1 { + if enable_fsst + && matches!( + version.resolve(), + ConcreteFileVersion::V2_1 | ConcreteFileVersion::V2_2 | ConcreteFileVersion::V2_3 + ) + { Field::new(name, DataType::Utf8, false) } else { utf8_field_without_fsst(name) diff --git a/rust/lance/benches/s3_file_reader_diagnostics.rs b/rust/lance/benches/s3_file_reader_diagnostics.rs new file mode 100644 index 00000000000..1762048d495 --- /dev/null +++ b/rust/lance/benches/s3_file_reader_diagnostics.rs @@ -0,0 +1,2357 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright The Lance Authors + +#![allow(clippy::print_stdout)] +#![recursion_limit = "256"] + +use std::collections::BTreeMap; +use std::env; +use std::fs; +use std::hint::black_box; +use std::ops::Range; +use std::sync::{ + Arc, Mutex, + atomic::{AtomicU64, Ordering}, +}; +use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH}; + +use arrow_array::RecordBatch; +use futures::future::BoxFuture; +use futures::stream::{FuturesOrdered, FuturesUnordered}; +use futures::{FutureExt, StreamExt, TryStreamExt}; +use lance::dataset::ProjectionRequest; +use lance::dataset::builder::DatasetBuilder; +use lance::dataset::fragment::{FileFragment, FragReadConfig}; +use lance::dataset::scanner::{ExecutionStatsCallback, ExecutionSummaryCounts}; +use lance_core::datatypes::Schema; +use lance_encoding::decoder::PageEncoding; +use lance_encoding::format::pb21; +use lance_file::reader::{ + DEFAULT_READ_CHUNK_SIZE, FileReader as LanceFileReader, FileReaderOptions, +}; +use lance_io::object_store::ObjectStore as LanceObjectStore; +use lance_io::scheduler::{FileScheduler, ScanScheduler, ScanStats, SchedulerConfig}; +use lance_io::utils::CachedFileSize; +use serde_json::{Value, json}; +use tracing::field::{Field, Visit}; +use tracing::subscriber::Interest; +use tracing::{Event, Metadata, Subscriber}; +use tracing_subscriber::layer::{Context, Layer}; +use tracing_subscriber::prelude::*; + +type Error = Box; +type Result = std::result::Result; + +const GIB: u64 = 1024 * 1024 * 1024; +const SCHEDULER_STATE_EVENT_TARGET: &str = "lance_io::scheduler::state"; + +#[derive(Debug, Clone, Copy, Eq, PartialEq)] +enum SchedulerQueueKind { + Standard, + Lite, +} + +#[derive(Debug, Clone, Copy)] +struct SchedulerDiagnostics { + kind: SchedulerQueueKind, + stats: ScanStats, + io_capacity: u64, + iops_available: u64, + active_iops: u64, + pending_iops: u64, + pending_bytes: u64, + bytes_available: i64, + bytes_reserved: i64, + io_buffer_size_bytes: u64, + priorities_in_flight: u64, + no_backpressure: bool, + head_task_bytes: Option, + head_task_priority_high: Option, + head_task_priority_low: Option, + min_in_flight_priority_high: Option, + min_in_flight_priority_low: Option, + head_task_can_deliver: Option, + head_task_priority_bypass: Option, + head_task_blocked_by_iops: Option, + head_task_blocked_by_bytes: Option, +} + +#[derive(Debug, Clone)] +struct Config { + backend: Backend, + uri: String, + dataset_version: u64, + columns: Option>, + limit_rows: u64, + target_bytes: Option, + raw_range_size_bytes: u64, + raw_range_mode: RawRangeMode, + raw_column_indices: Option>, + raw_submit_mode: RawSubmitMode, + raw_completion_mode: RawCompletionMode, + take_repetitions: u64, + io_buffer_gib: Vec>, + batch_size: u32, + batch_size_bytes: Option, + skip_batch_byte_accounting: bool, + read_chunk_size: Option, + fragment_concurrency: usize, + batch_concurrency: usize, + sample_ms: u64, + out_dir: String, + case_name: String, + describe_layout: bool, + detach_fragment_streams: bool, + drop_read_tasks: bool, +} + +#[derive(Debug, Clone, Copy, Eq, PartialEq)] +enum Backend { + FileReader, + Scanner, + SchedulerRaw, + DatasetTake, +} + +#[derive(Debug, Clone, Copy, Eq, PartialEq)] +enum RawSubmitMode { + Single, + SplitNoConcat, + SplitConcat, +} + +#[derive(Debug, Clone, Copy, Eq, PartialEq)] +enum RawRangeMode { + FileSequential, + MetadataPages, + MetadataPagesRoundRobin, +} + +#[derive(Debug, Clone, Copy, Eq, PartialEq)] +enum RawCompletionMode { + Unordered, + Ordered, +} + +impl RawRangeMode { + fn name(self) -> &'static str { + match self { + Self::FileSequential => "file-sequential", + Self::MetadataPages => "metadata-pages", + Self::MetadataPagesRoundRobin => "metadata-pages-round-robin", + } + } +} + +impl RawSubmitMode { + fn name(self) -> &'static str { + match self { + Self::Single => "single", + Self::SplitNoConcat => "split-no-concat", + Self::SplitConcat => "split-concat", + } + } +} + +impl RawCompletionMode { + fn name(self) -> &'static str { + match self { + Self::Unordered => "unordered", + Self::Ordered => "ordered", + } + } +} + +impl Backend { + fn name(self) -> &'static str { + match self { + Self::FileReader => "lance-file-reader", + Self::Scanner => "lance-scanner", + Self::SchedulerRaw => "lance-scheduler-raw", + Self::DatasetTake => "lance-dataset-take", + } + } + + fn layer(self) -> &'static str { + match self { + Self::FileReader => "file-reader", + Self::Scanner => "scanner", + Self::SchedulerRaw => "scheduler", + Self::DatasetTake => "dataset-take", + } + } +} + +#[derive(Debug, Clone, Copy)] +struct CpuSample { + idle: u64, + total: u64, +} + +#[derive(Debug, Default)] +struct SharedCounters { + fragments_started: AtomicU64, + fragments_completed: AtomicU64, + batch_futures_emitted: AtomicU64, + batch_futures_received: AtomicU64, + batches_completed: AtomicU64, + rows_completed: AtomicU64, + arrow_bytes: AtomicU64, + open_reader_ns: AtomicU64, + read_stream_create_ns: AtomicU64, + next_batch_poll_ns: AtomicU64, + channel_send_wait_ns: AtomicU64, + decode_ns: AtomicU64, + raw_reassemble_ns: AtomicU64, +} + +#[derive(Debug)] +struct CaseStats { + rows: u64, + batches: u64, + arrow_bytes: u64, + planned_fragments: usize, + planned_rows: u64, + elapsed: Duration, + producer_finished_at: Option, + peak_decode_in_flight: usize, + cpu_avg: Option, + scheduler_diagnostics: SchedulerDiagnostics, + counters: Arc, + samples: Vec, +} + +#[derive(Debug)] +struct LastSample { + elapsed: Duration, + scheduler_stats: ScanStats, + rows: u64, + batches: u64, + arrow_bytes: u64, +} + +fn usage() -> &'static str { + "usage: s3_file_reader_diagnostics --uri \ + [--backend ] \ + [--dataset-version ] [--columns ] \ + [--limit-rows ] [--target-bytes ] [--raw-range-size-bytes ] \ + [--raw-range-mode ] \ + [--raw-column-indices ] \ + [--raw-submit-mode ] \ + [--raw-completion-mode ] \ + [--take-repetitions ] \ + [--io-buffer-gib ] \ + [--batch-size ] [--batch-size-bytes ] [--read-chunk-size ] \ + [--skip-batch-byte-accounting] \ + [--fragment-concurrency ] [--batch-concurrency ] \ + [--sample-ms ] [--out-dir ] [--case ] \ + [--detach-fragment-streams] [--drop-read-tasks] [--describe-layout]" +} + +fn parse_args() -> Result { + let mut backend = Backend::FileReader; + let mut uri = None; + let mut dataset_version = 1u64; + let mut columns = Some(vec!["vector".to_string()]); + let mut limit_rows = 67_108_864u64; + let mut target_bytes = None; + let mut raw_range_size_bytes = 16 * 1024 * 1024; + let mut raw_range_mode = RawRangeMode::FileSequential; + let mut raw_column_indices = None; + let mut raw_submit_mode = RawSubmitMode::Single; + let mut raw_completion_mode = RawCompletionMode::Unordered; + let mut take_repetitions = 100u64; + let mut io_buffer_gib = vec![Some(8)]; + let mut batch_size = 8192u32; + let mut batch_size_bytes = None; + let mut skip_batch_byte_accounting = false; + let mut read_chunk_size = None; + let mut fragment_concurrency = 256usize; + let mut batch_concurrency = 256usize; + let mut sample_ms = 1000u64; + let mut out_dir = "/tmp/lance-s3-bottleneck-results".to_string(); + let mut case_name = "lance-file-reader-diagnostics".to_string(); + let mut describe_layout = false; + let mut detach_fragment_streams = false; + let mut drop_read_tasks = false; + + let mut args = env::args().skip(1); + while let Some(arg) = args.next() { + match arg.as_str() { + "--backend" => { + let value = args + .next() + .ok_or_else(|| format!("missing value for --backend. {}", usage()))?; + backend = parse_backend(&value)?; + } + "--uri" => uri = args.next(), + "--dataset-version" => { + dataset_version = parse_required_value(&mut args, "--dataset-version")?; + } + "--columns" => { + let value = args + .next() + .ok_or_else(|| format!("missing value for --columns. {}", usage()))?; + columns = parse_columns(&value)?; + } + "--limit-rows" => { + limit_rows = parse_required_value(&mut args, "--limit-rows")?; + } + "--target-bytes" => { + target_bytes = Some(parse_required_value(&mut args, "--target-bytes")?); + } + "--raw-range-size-bytes" => { + raw_range_size_bytes = parse_required_value(&mut args, "--raw-range-size-bytes")?; + } + "--raw-range-mode" => { + let value = args + .next() + .ok_or_else(|| format!("missing value for --raw-range-mode. {}", usage()))?; + raw_range_mode = parse_raw_range_mode(&value)?; + } + "--raw-column-indices" => { + let value = args.next().ok_or_else(|| { + format!("missing value for --raw-column-indices. {}", usage()) + })?; + raw_column_indices = parse_raw_column_indices(&value)?; + } + "--raw-submit-mode" => { + let value = args + .next() + .ok_or_else(|| format!("missing value for --raw-submit-mode. {}", usage()))?; + raw_submit_mode = parse_raw_submit_mode(&value)?; + } + "--raw-completion-mode" => { + let value = args.next().ok_or_else(|| { + format!("missing value for --raw-completion-mode. {}", usage()) + })?; + raw_completion_mode = parse_raw_completion_mode(&value)?; + } + "--take-repetitions" => { + take_repetitions = parse_required_value(&mut args, "--take-repetitions")?; + } + "--io-buffer-gib" => { + let value = args + .next() + .ok_or_else(|| format!("missing value for --io-buffer-gib. {}", usage()))?; + io_buffer_gib = parse_io_buffer_gib(&value)?; + } + "--batch-size" => { + batch_size = parse_required_value(&mut args, "--batch-size")?; + } + "--batch-size-bytes" => { + batch_size_bytes = Some(parse_required_value(&mut args, "--batch-size-bytes")?); + } + "--skip-batch-byte-accounting" => { + skip_batch_byte_accounting = true; + } + "--read-chunk-size" => { + read_chunk_size = Some(parse_required_value(&mut args, "--read-chunk-size")?); + } + "--fragment-concurrency" => { + fragment_concurrency = parse_required_value(&mut args, "--fragment-concurrency")?; + } + "--batch-concurrency" => { + batch_concurrency = parse_required_value(&mut args, "--batch-concurrency")?; + } + "--sample-ms" => { + sample_ms = parse_required_value(&mut args, "--sample-ms")?; + } + "--out-dir" => { + out_dir = args + .next() + .ok_or_else(|| format!("missing value for --out-dir. {}", usage()))?; + } + "--case" => { + case_name = args + .next() + .ok_or_else(|| format!("missing value for --case. {}", usage()))?; + } + "--describe-layout" => { + describe_layout = true; + } + "--detach-fragment-streams" => { + detach_fragment_streams = true; + } + "--drop-read-tasks" => { + drop_read_tasks = true; + } + "--help" | "-h" => { + println!("{}", usage()); + std::process::exit(0); + } + "--bench" => { + // Cargo appends this flag when running harness-free benches. + } + other => { + return Err(format!("unknown argument {other}. {}", usage()).into()); + } + } + } + + let uri = uri.ok_or_else(|| format!("missing required --uri. {}", usage()))?; + if limit_rows == 0 { + return Err("--limit-rows must be greater than zero".into()); + } + if matches!(target_bytes, Some(0)) { + return Err("--target-bytes must be greater than zero".into()); + } + if raw_range_size_bytes == 0 { + return Err("--raw-range-size-bytes must be greater than zero".into()); + } + if take_repetitions == 0 { + return Err("--take-repetitions must be greater than zero".into()); + } + if io_buffer_gib.is_empty() { + return Err("--io-buffer-gib must not be empty".into()); + } + if batch_size == 0 { + return Err("--batch-size must be greater than zero".into()); + } + if matches!(batch_size_bytes, Some(0)) { + return Err("--batch-size-bytes must be greater than zero".into()); + } + if matches!(read_chunk_size, Some(0)) { + return Err("--read-chunk-size must be greater than zero".into()); + } + if fragment_concurrency == 0 && !matches!(backend, Backend::Scanner) { + return Err("--fragment-concurrency must be greater than zero".into()); + } + if batch_concurrency == 0 && !matches!(backend, Backend::Scanner) { + return Err("--batch-concurrency must be greater than zero".into()); + } + if sample_ms == 0 { + return Err("--sample-ms must be greater than zero".into()); + } + + Ok(Config { + backend, + uri, + dataset_version, + columns, + limit_rows, + target_bytes, + raw_range_size_bytes, + raw_range_mode, + raw_column_indices, + raw_submit_mode, + raw_completion_mode, + take_repetitions, + io_buffer_gib, + batch_size, + batch_size_bytes, + skip_batch_byte_accounting, + read_chunk_size, + fragment_concurrency, + batch_concurrency, + sample_ms, + out_dir, + case_name, + describe_layout, + detach_fragment_streams, + drop_read_tasks, + }) +} + +fn parse_backend(value: &str) -> Result { + match value { + "file-reader" | "lance-file-reader" => Ok(Backend::FileReader), + "scanner" | "lance-scanner" => Ok(Backend::Scanner), + "scheduler-raw" | "lance-scheduler-raw" => Ok(Backend::SchedulerRaw), + "dataset-take" | "take" | "lance-dataset-take" => Ok(Backend::DatasetTake), + other => Err(format!( + "invalid --backend value {other}; expected file-reader, scanner, scheduler-raw, or dataset-take" + ) + .into()), + } +} + +fn parse_raw_submit_mode(value: &str) -> Result { + match value { + "single" => Ok(RawSubmitMode::Single), + "split-no-concat" => Ok(RawSubmitMode::SplitNoConcat), + "split-concat" => Ok(RawSubmitMode::SplitConcat), + other => Err(format!( + "invalid --raw-submit-mode value {other}; expected single, split-no-concat, or split-concat" + ) + .into()), + } +} + +fn parse_raw_range_mode(value: &str) -> Result { + match value { + "file-sequential" => Ok(RawRangeMode::FileSequential), + "metadata-pages" => Ok(RawRangeMode::MetadataPages), + "metadata-pages-round-robin" => Ok(RawRangeMode::MetadataPagesRoundRobin), + other => Err(format!( + "invalid --raw-range-mode value {other}; expected file-sequential, metadata-pages, or metadata-pages-round-robin" + ) + .into()), + } +} + +fn parse_raw_column_indices(value: &str) -> Result>> { + if value == "all" { + return Ok(None); + } + + let indices = value + .split(',') + .map(str::trim) + .filter(|part| !part.is_empty()) + .map(|part| { + part.parse::() + .map_err(|err| format!("invalid raw column index {part}: {err}").into()) + }) + .collect::>>()?; + if indices.is_empty() { + return Err("--raw-column-indices must specify at least one column index or all".into()); + } + Ok(Some(indices)) +} + +fn parse_raw_completion_mode(value: &str) -> Result { + match value { + "unordered" => Ok(RawCompletionMode::Unordered), + "ordered" => Ok(RawCompletionMode::Ordered), + other => Err(format!( + "invalid --raw-completion-mode value {other}; expected unordered or ordered" + ) + .into()), + } +} + +fn parse_required_value(args: &mut impl Iterator, name: &str) -> Result +where + T: std::str::FromStr, + T::Err: std::fmt::Display + Send + Sync + 'static, +{ + let value = args + .next() + .ok_or_else(|| format!("missing value for {name}. {}", usage()))?; + value + .parse() + .map_err(|err| format!("invalid {name} value {value}: {err}").into()) +} + +fn parse_columns(value: &str) -> Result>> { + match value { + "all" => Ok(None), + "empty" => Err("FileReader benchmark requires at least one data column".into()), + _ => Ok(Some( + value + .split(',') + .map(str::trim) + .filter(|column| !column.is_empty()) + .map(ToString::to_string) + .collect(), + )), + } +} + +fn parse_io_buffer_gib(value: &str) -> Result>> { + value + .split(',') + .map(|part| { + let part = part.trim(); + if part == "auto" { + Ok(None) + } else { + part.parse::() + .map(Some) + .map_err(|err| format!("invalid --io-buffer-gib value {part}: {err}").into()) + } + }) + .collect() +} + +fn projection_name(columns: &Option>) -> String { + match columns { + None => "all".to_string(), + Some(columns) => columns.join(","), + } +} + +fn page_layout_kind(encoding: &PageEncoding) -> &'static str { + match encoding { + PageEncoding::Legacy(_) => "legacy", + PageEncoding::Structural(layout) => match layout.layout.as_ref() { + Some(pb21::page_layout::Layout::MiniBlockLayout(_)) => "miniblock", + Some(pb21::page_layout::Layout::ConstantLayout(_)) => "constant", + Some(pb21::page_layout::Layout::FullZipLayout(_)) => "fullzip", + Some(pb21::page_layout::Layout::BlobLayout(_)) => "blob", + Some(pb21::page_layout::Layout::SparseLayout(_)) => "sparse", + None => "missing", + }, + } +} + +fn summarize_u64(values: &[u64]) -> Value { + if values.is_empty() { + return json!({ + "count": 0, + "min": null, + "p50": null, + "p90": null, + "max": null, + "sum": 0, + }); + } + let mut sorted = values.to_vec(); + sorted.sort_unstable(); + let percentile = |p: f64| { + let idx = ((sorted.len() - 1) as f64 * p).round() as usize; + sorted[idx] + }; + json!({ + "count": sorted.len(), + "min": sorted[0], + "p50": percentile(0.5), + "p90": percentile(0.9), + "max": *sorted.last().unwrap(), + "sum": values.iter().sum::(), + }) +} + +async fn describe_layout(config: &Config) -> Result<()> { + let dataset = Arc::new( + DatasetBuilder::from_uri(&config.uri) + .with_version(config.dataset_version) + .load() + .await?, + ); + let fragment = dataset + .fragments() + .first() + .ok_or("dataset has no fragments")?; + let data_file = fragment + .files + .first() + .ok_or("first fragment has no data files")?; + if data_file.base_id.is_some() { + return Err("layout diagnostics do not support external base data files yet".into()); + } + + let data_path = dataset.data_dir().join(data_file.path.as_str()); + let (object_store, _) = LanceObjectStore::from_uri(&config.uri).await?; + let scheduler = ScanScheduler::new(object_store, SchedulerConfig::new(8 * GIB)); + let file_scheduler = scheduler + .open_file(&data_path, &CachedFileSize::unknown()) + .await?; + let metadata = LanceFileReader::read_all_metadata(&file_scheduler).await?; + + let columns = metadata + .column_infos + .iter() + .map(|column| { + let mut layout_counts = BTreeMap::new(); + let mut page_rows = Vec::with_capacity(column.page_infos.len()); + let mut page_bytes = Vec::with_capacity(column.page_infos.len()); + for page in column.page_infos.iter() { + *layout_counts + .entry(page_layout_kind(&page.encoding)) + .or_insert(0usize) += 1; + page_rows.push(page.num_rows); + page_bytes.push( + page.buffer_offsets_and_sizes + .iter() + .map(|(_, size)| *size) + .sum::(), + ); + } + json!({ + "column_index": column.index, + "num_pages": column.page_infos.len(), + "layout_counts": layout_counts, + "page_rows": summarize_u64(&page_rows), + "page_bytes": summarize_u64(&page_bytes), + }) + }) + .collect::>(); + + println!( + "{}", + serde_json::to_string_pretty(&json!({ + "dataset_uri": config.uri, + "dataset_version": config.dataset_version, + "fragment_id": fragment.id, + "data_file_path": data_file.path, + "resolved_data_path": data_path.to_string(), + "file_version": metadata.version().to_string(), + "num_rows": metadata.num_rows, + "num_data_bytes": metadata.num_data_bytes, + "columns": columns, + }))? + ); + Ok(()) +} + +fn projected_schema(dataset_schema: &Schema, columns: &Option>) -> Result { + Ok(match columns { + None => dataset_schema.clone(), + Some(columns) => dataset_schema.project(columns)?, + }) +} + +fn file_reader_options(config: &Config) -> Option { + if config.batch_size_bytes.is_none() && config.read_chunk_size.is_none() { + return None; + } + Some(FileReaderOptions { + batch_size_bytes: config.batch_size_bytes, + read_chunk_size: config.read_chunk_size.unwrap_or(DEFAULT_READ_CHUNK_SIZE), + ..Default::default() + }) +} + +fn add_duration(counter: &AtomicU64, duration: Duration) { + let nanos = duration.as_nanos().min(u128::from(u64::MAX)) as u64; + counter.fetch_add(nanos, Ordering::Relaxed); +} + +fn ns_to_seconds(ns: u64) -> f64 { + ns as f64 / 1_000_000_000.0 +} + +fn diff_u64(current: u64, previous: u64) -> u64 { + current.saturating_sub(previous) +} + +fn scheduler_kind_name(kind: SchedulerQueueKind) -> &'static str { + match kind { + SchedulerQueueKind::Standard => "standard", + SchedulerQueueKind::Lite => "lite", + } +} + +fn diagnostics_json(diagnostics: SchedulerDiagnostics) -> Value { + json!({ + "queue_kind": scheduler_kind_name(diagnostics.kind), + "scheduler_iops": diagnostics.stats.iops, + "scheduler_requests": diagnostics.stats.requests, + "scheduler_bytes_read": diagnostics.stats.bytes_read, + "io_capacity": diagnostics.io_capacity, + "iops_available": diagnostics.iops_available, + "active_iops": diagnostics.active_iops, + "pending_iops": diagnostics.pending_iops, + "pending_bytes": diagnostics.pending_bytes, + "bytes_available": diagnostics.bytes_available, + "bytes_reserved": diagnostics.bytes_reserved, + "io_buffer_size_bytes": diagnostics.io_buffer_size_bytes, + "priorities_in_flight": diagnostics.priorities_in_flight, + "no_backpressure": diagnostics.no_backpressure, + "head_task_bytes": diagnostics.head_task_bytes, + "head_task_priority_high": diagnostics.head_task_priority_high, + "head_task_priority_low": diagnostics.head_task_priority_low, + "min_in_flight_priority_high": diagnostics.min_in_flight_priority_high, + "min_in_flight_priority_low": diagnostics.min_in_flight_priority_low, + "head_task_can_deliver": diagnostics.head_task_can_deliver, + "head_task_priority_bypass": diagnostics.head_task_priority_bypass, + "head_task_blocked_by_iops": diagnostics.head_task_blocked_by_iops, + "head_task_blocked_by_bytes": diagnostics.head_task_blocked_by_bytes, + }) +} + +#[derive(Debug, Default)] +struct ExecutionStatsHolder { + collected_stats: Arc>>, +} + +impl ExecutionStatsHolder { + fn get_setter(&self) -> ExecutionStatsCallback { + let collected_stats = self.collected_stats.clone(); + Arc::new(move |stats| { + *collected_stats.lock().unwrap() = Some(stats.clone()); + }) + } + + fn consume(self) -> Option { + self.collected_stats.lock().unwrap().take() + } +} + +#[derive(Debug, Clone, Default)] +struct SchedulerDiagnosticsCollector { + latest: Arc>>, +} + +impl SchedulerDiagnosticsCollector { + fn clear(&self) { + *self.latest.lock().unwrap() = None; + } + + fn observe(&self, diagnostics: SchedulerDiagnostics) { + *self.latest.lock().unwrap() = Some(diagnostics); + } + + fn snapshot(&self, io_buffer_gib: Option) -> SchedulerDiagnostics { + self.latest + .lock() + .unwrap() + .as_ref() + .copied() + .unwrap_or_else(|| diagnostics_from_scan_stats(ScanStats::default(), io_buffer_gib)) + } +} + +#[derive(Debug, Clone)] +struct SchedulerDiagnosticsLayer { + collector: SchedulerDiagnosticsCollector, +} + +impl SchedulerDiagnosticsLayer { + fn new(collector: SchedulerDiagnosticsCollector) -> Self { + Self { collector } + } +} + +fn is_scheduler_state_metadata(metadata: &Metadata<'_>) -> bool { + // The scheduler uses `tracing::enabled!` before constructing the event; + // that guard registers a HINT callsite, not an EVENT callsite. + metadata.target() == SCHEDULER_STATE_EVENT_TARGET && *metadata.level() == tracing::Level::TRACE +} + +impl Layer for SchedulerDiagnosticsLayer +where + S: Subscriber, +{ + fn register_callsite(&self, metadata: &'static Metadata<'static>) -> Interest { + if is_scheduler_state_metadata(metadata) { + Interest::always() + } else { + Interest::never() + } + } + + fn enabled(&self, metadata: &Metadata<'_>, _ctx: Context<'_, S>) -> bool { + is_scheduler_state_metadata(metadata) + } + + fn on_event(&self, event: &Event<'_>, _ctx: Context<'_, S>) { + if !is_scheduler_state_metadata(event.metadata()) { + return; + } + let mut visitor = SchedulerDiagnosticsVisitor::default(); + event.record(&mut visitor); + if let Some(diagnostics) = visitor.into_diagnostics() { + self.collector.observe(diagnostics); + } + } +} + +#[derive(Debug, Default)] +struct SchedulerDiagnosticsVisitor { + kind: Option, + scheduler_iops: Option, + scheduler_requests: Option, + scheduler_bytes_read: Option, + io_capacity: Option, + iops_available: Option, + active_iops: Option, + pending_iops: Option, + pending_bytes: Option, + bytes_available: Option, + bytes_reserved: Option, + io_buffer_size_bytes: Option, + priorities_in_flight: Option, + no_backpressure: Option, + head_task_bytes_present: bool, + head_task_bytes: Option, + head_task_priority_high_present: bool, + head_task_priority_high: Option, + head_task_priority_low_present: bool, + head_task_priority_low: Option, + min_in_flight_priority_high_present: bool, + min_in_flight_priority_high: Option, + min_in_flight_priority_low_present: bool, + min_in_flight_priority_low: Option, + head_task_can_deliver_present: bool, + head_task_can_deliver: Option, + head_task_priority_bypass_present: bool, + head_task_priority_bypass: Option, + head_task_blocked_by_iops_present: bool, + head_task_blocked_by_iops: Option, + head_task_blocked_by_bytes_present: bool, + head_task_blocked_by_bytes: Option, +} + +impl SchedulerDiagnosticsVisitor { + fn into_diagnostics(self) -> Option { + Some(SchedulerDiagnostics { + kind: self.kind?, + stats: ScanStats { + iops: self.scheduler_iops.unwrap_or_default(), + requests: self.scheduler_requests.unwrap_or_default(), + bytes_read: self.scheduler_bytes_read.unwrap_or_default(), + }, + io_capacity: self.io_capacity.unwrap_or_default(), + iops_available: self.iops_available.unwrap_or_default(), + active_iops: self.active_iops.unwrap_or_default(), + pending_iops: self.pending_iops.unwrap_or_default(), + pending_bytes: self.pending_bytes.unwrap_or_default(), + bytes_available: self.bytes_available.unwrap_or_default(), + bytes_reserved: self.bytes_reserved.unwrap_or_default(), + io_buffer_size_bytes: self.io_buffer_size_bytes.unwrap_or_default(), + priorities_in_flight: self.priorities_in_flight.unwrap_or_default(), + no_backpressure: self.no_backpressure.unwrap_or(false), + head_task_bytes: optional_u64(self.head_task_bytes_present, self.head_task_bytes), + head_task_priority_high: optional_u64( + self.head_task_priority_high_present, + self.head_task_priority_high, + ), + head_task_priority_low: optional_u64( + self.head_task_priority_low_present, + self.head_task_priority_low, + ), + min_in_flight_priority_high: optional_u64( + self.min_in_flight_priority_high_present, + self.min_in_flight_priority_high, + ), + min_in_flight_priority_low: optional_u64( + self.min_in_flight_priority_low_present, + self.min_in_flight_priority_low, + ), + head_task_can_deliver: optional_bool( + self.head_task_can_deliver_present, + self.head_task_can_deliver, + ), + head_task_priority_bypass: optional_bool( + self.head_task_priority_bypass_present, + self.head_task_priority_bypass, + ), + head_task_blocked_by_iops: optional_bool( + self.head_task_blocked_by_iops_present, + self.head_task_blocked_by_iops, + ), + head_task_blocked_by_bytes: optional_bool( + self.head_task_blocked_by_bytes_present, + self.head_task_blocked_by_bytes, + ), + }) + } +} + +impl Visit for SchedulerDiagnosticsVisitor { + fn record_bool(&mut self, field: &Field, value: bool) { + match field.name() { + "no_backpressure" => self.no_backpressure = Some(value), + "head_task_bytes_present" => self.head_task_bytes_present = value, + "head_task_priority_high_present" => self.head_task_priority_high_present = value, + "head_task_priority_low_present" => self.head_task_priority_low_present = value, + "min_in_flight_priority_high_present" => { + self.min_in_flight_priority_high_present = value; + } + "min_in_flight_priority_low_present" => { + self.min_in_flight_priority_low_present = value; + } + "head_task_can_deliver_present" => self.head_task_can_deliver_present = value, + "head_task_can_deliver" => self.head_task_can_deliver = Some(value), + "head_task_priority_bypass_present" => { + self.head_task_priority_bypass_present = value; + } + "head_task_priority_bypass" => self.head_task_priority_bypass = Some(value), + "head_task_blocked_by_iops_present" => { + self.head_task_blocked_by_iops_present = value; + } + "head_task_blocked_by_iops" => self.head_task_blocked_by_iops = Some(value), + "head_task_blocked_by_bytes_present" => { + self.head_task_blocked_by_bytes_present = value; + } + "head_task_blocked_by_bytes" => self.head_task_blocked_by_bytes = Some(value), + _ => {} + } + } + + fn record_i64(&mut self, field: &Field, value: i64) { + match field.name() { + "bytes_available" => self.bytes_available = Some(value), + "bytes_reserved" => self.bytes_reserved = Some(value), + _ => {} + } + } + + fn record_u64(&mut self, field: &Field, value: u64) { + match field.name() { + "scheduler_iops" => self.scheduler_iops = Some(value), + "scheduler_requests" => self.scheduler_requests = Some(value), + "scheduler_bytes_read" => self.scheduler_bytes_read = Some(value), + "io_capacity" => self.io_capacity = Some(value), + "iops_available" => self.iops_available = Some(value), + "active_iops" => self.active_iops = Some(value), + "pending_iops" => self.pending_iops = Some(value), + "pending_bytes" => self.pending_bytes = Some(value), + "io_buffer_size_bytes" => self.io_buffer_size_bytes = Some(value), + "priorities_in_flight" => self.priorities_in_flight = Some(value), + "head_task_bytes" => self.head_task_bytes = Some(value), + "head_task_priority_high" => self.head_task_priority_high = Some(value), + "head_task_priority_low" => self.head_task_priority_low = Some(value), + "min_in_flight_priority_high" => self.min_in_flight_priority_high = Some(value), + "min_in_flight_priority_low" => self.min_in_flight_priority_low = Some(value), + _ => {} + } + } + + fn record_str(&mut self, field: &Field, value: &str) { + if field.name() == "queue_kind" { + self.kind = match value { + "standard" => Some(SchedulerQueueKind::Standard), + "lite" => Some(SchedulerQueueKind::Lite), + _ => None, + }; + } + } + + fn record_debug(&mut self, _field: &Field, _value: &dyn std::fmt::Debug) {} +} + +fn optional_u64(present: bool, value: Option) -> Option { + present.then(|| value.unwrap_or_default()) +} + +fn optional_bool(present: bool, value: Option) -> Option { + present.then(|| value.unwrap_or(false)) +} + +fn diagnostics_from_scan_stats( + stats: ScanStats, + io_buffer_gib: Option, +) -> SchedulerDiagnostics { + SchedulerDiagnostics { + kind: SchedulerQueueKind::Standard, + stats, + io_capacity: 0, + iops_available: 0, + active_iops: 0, + pending_iops: 0, + pending_bytes: 0, + bytes_available: 0, + bytes_reserved: 0, + io_buffer_size_bytes: io_buffer_gib.map(|value| value * GIB).unwrap_or_default(), + priorities_in_flight: 0, + no_backpressure: false, + head_task_bytes: None, + head_task_priority_high: None, + head_task_priority_low: None, + min_in_flight_priority_high: None, + min_in_flight_priority_low: None, + head_task_can_deliver: None, + head_task_priority_bypass: None, + head_task_blocked_by_iops: None, + head_task_blocked_by_bytes: None, + } +} + +fn scan_stats_from_execution_summary(summary: &ExecutionSummaryCounts) -> ScanStats { + ScanStats { + iops: summary.iops as u64, + requests: summary.requests as u64, + bytes_read: summary.bytes_read as u64, + } +} + +fn sample_json( + started: Instant, + counters: &SharedCounters, + diagnostics: SchedulerDiagnostics, + decode_in_flight: usize, + channel_buffered: usize, + last: &mut LastSample, +) -> Value { + let elapsed = started.elapsed(); + let interval = elapsed.saturating_sub(last.elapsed); + let interval_secs = interval.as_secs_f64(); + let rows = counters.rows_completed.load(Ordering::Relaxed); + let batches = counters.batches_completed.load(Ordering::Relaxed); + let arrow_bytes = counters.arrow_bytes.load(Ordering::Relaxed); + let scheduler_stats = diagnostics.stats; + let delta_scheduler_bytes = + diff_u64(scheduler_stats.bytes_read, last.scheduler_stats.bytes_read); + let delta_rows = diff_u64(rows, last.rows); + let delta_arrow_bytes = diff_u64(arrow_bytes, last.arrow_bytes); + let physical_gbps = if interval_secs > 0.0 { + delta_scheduler_bytes as f64 * 8.0 / interval_secs / 1_000_000_000.0 + } else { + 0.0 + }; + let logical_gbps = if interval_secs > 0.0 { + delta_arrow_bytes as f64 * 8.0 / interval_secs / 1_000_000_000.0 + } else { + 0.0 + }; + let rows_per_second = if interval_secs > 0.0 { + delta_rows as f64 / interval_secs + } else { + 0.0 + }; + + last.elapsed = elapsed; + last.scheduler_stats = scheduler_stats; + last.rows = rows; + last.batches = batches; + last.arrow_bytes = arrow_bytes; + + json!({ + "elapsed_seconds": elapsed.as_secs_f64(), + "interval_seconds": interval_secs, + "physical_gbps": physical_gbps, + "logical_gbps": logical_gbps, + "rows_per_second": rows_per_second, + "rows": rows, + "batches": batches, + "arrow_bytes": arrow_bytes, + "delta_rows": delta_rows, + "delta_arrow_bytes": delta_arrow_bytes, + "delta_scheduler_bytes": delta_scheduler_bytes, + "fragments_started": counters.fragments_started.load(Ordering::Relaxed), + "fragments_completed": counters.fragments_completed.load(Ordering::Relaxed), + "batch_futures_emitted": counters.batch_futures_emitted.load(Ordering::Relaxed), + "batch_futures_received": counters.batch_futures_received.load(Ordering::Relaxed), + "batches_completed": counters.batches_completed.load(Ordering::Relaxed), + "decode_in_flight": decode_in_flight, + "channel_buffered": channel_buffered, + "open_reader_seconds_total": ns_to_seconds(counters.open_reader_ns.load(Ordering::Relaxed)), + "read_stream_create_seconds_total": ns_to_seconds(counters.read_stream_create_ns.load(Ordering::Relaxed)), + "next_batch_poll_seconds_total": ns_to_seconds(counters.next_batch_poll_ns.load(Ordering::Relaxed)), + "channel_send_wait_seconds_total": ns_to_seconds(counters.channel_send_wait_ns.load(Ordering::Relaxed)), + "decode_seconds_total": ns_to_seconds(counters.decode_ns.load(Ordering::Relaxed)), + "raw_reassemble_seconds_total": ns_to_seconds(counters.raw_reassemble_ns.load(Ordering::Relaxed)), + "scheduler": diagnostics_json(diagnostics), + }) +} + +async fn run_scanner_case( + config: &Config, + io_buffer_gib: Option, + scheduler_diagnostics: &SchedulerDiagnosticsCollector, +) -> Result { + scheduler_diagnostics.clear(); + let dataset = Arc::new( + DatasetBuilder::from_uri(&config.uri) + .with_version(config.dataset_version) + .load() + .await?, + ); + + let mut remaining_rows = config.limit_rows; + let mut planned_fragments = 0usize; + let mut planned_rows = 0u64; + for fragment in dataset.fragments().iter() { + if remaining_rows == 0 { + break; + } + let fragment_rows = fragment + .num_rows() + .ok_or_else(|| format!("fragment {} is missing num_rows", fragment.id))? + as u64; + let rows = fragment_rows.min(remaining_rows); + planned_fragments += 1; + planned_rows += rows; + remaining_rows -= rows; + } + if planned_fragments == 0 { + return Err("no fragments selected".into()); + } + + let counters = Arc::new(SharedCounters::default()); + let stats_holder = ExecutionStatsHolder::default(); + let cpu_before = read_cpu_sample(); + let started = Instant::now(); + + let mut scanner = dataset.scan(); + if let Some(columns) = config.columns.as_ref() { + scanner.project(columns)?; + } + scanner + .batch_size(config.batch_size as usize) + .scan_in_order(false) + .scan_stats_callback(stats_holder.get_setter()); + if config.batch_concurrency > 0 { + scanner + .batch_readahead(config.batch_concurrency) + .target_parallelism(config.batch_concurrency); + } + if config.fragment_concurrency > 0 { + scanner.fragment_readahead(config.fragment_concurrency); + } + if let Some(file_reader_options) = file_reader_options(config) { + scanner.with_file_reader_options(file_reader_options); + } + if let Some(batch_size_bytes) = config.batch_size_bytes { + scanner.batch_size_bytes(batch_size_bytes); + } + if let Some(io_buffer_gib) = io_buffer_gib { + scanner.io_buffer_size(io_buffer_gib * GIB); + } + let limit_rows = i64::try_from(config.limit_rows) + .map_err(|_| "--limit-rows is too large for scanner limit")?; + scanner.limit(Some(limit_rows), None)?; + + let mut stream = scanner.try_into_stream().await?; + let mut rows = 0u64; + let mut batches = 0u64; + let mut arrow_bytes = 0u64; + let mut samples = Vec::new(); + let mut sample_interval = tokio::time::interval(Duration::from_millis(config.sample_ms)); + sample_interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip); + let mut last_sample = LastSample { + elapsed: Duration::default(), + scheduler_stats: ScanStats::default(), + rows: 0, + batches: 0, + arrow_bytes: 0, + }; + loop { + tokio::select! { + maybe_batch = stream.next() => { + let Some(batch) = maybe_batch else { + break; + }; + let batch = batch?; + let batch_bytes = if config.skip_batch_byte_accounting { + 0 + } else { + batch.get_array_memory_size() as u64 + }; + rows += batch.num_rows() as u64; + batches += 1; + arrow_bytes += batch_bytes; + counters.batches_completed.fetch_add(1, Ordering::Relaxed); + counters + .rows_completed + .fetch_add(batch.num_rows() as u64, Ordering::Relaxed); + counters.arrow_bytes.fetch_add(batch_bytes, Ordering::Relaxed); + } + _ = sample_interval.tick() => { + samples.push(sample_json( + started, + counters.as_ref(), + scheduler_diagnostics.snapshot(io_buffer_gib), + 0, + 0, + &mut last_sample, + )); + } + } + } + drop(stream); + + let elapsed = started.elapsed(); + let summary = stats_holder + .consume() + .ok_or("scanner execution stats callback did not run")?; + let scheduler_stats = scan_stats_from_execution_summary(&summary); + let mut final_diagnostics = scheduler_diagnostics.snapshot(io_buffer_gib); + final_diagnostics.stats = scheduler_stats; + let cpu_after = read_cpu_sample(); + samples.push(sample_json( + started, + counters.as_ref(), + final_diagnostics, + 0, + 0, + &mut last_sample, + )); + + Ok(CaseStats { + rows, + batches, + arrow_bytes, + planned_fragments, + planned_rows, + elapsed, + producer_finished_at: Some(elapsed), + peak_decode_in_flight: 0, + cpu_avg: cpu_before.zip(cpu_after).and_then(|(before, after)| { + let total = after.total.checked_sub(before.total)?; + let idle = after.idle.checked_sub(before.idle)?; + if total == 0 { + return None; + } + Some((total - idle) as f64 / total as f64 * 100.0) + }), + scheduler_diagnostics: final_diagnostics, + counters, + samples, + }) +} + +async fn run_dataset_take_case( + config: &Config, + scheduler_diagnostics: &SchedulerDiagnosticsCollector, +) -> Result { + scheduler_diagnostics.clear(); + let dataset = DatasetBuilder::from_uri(&config.uri) + .with_version(config.dataset_version) + .load() + .await?; + let projection = Arc::new(projected_schema(dataset.schema(), &config.columns)?); + let total_rows = dataset + .fragments() + .iter() + .map(|fragment| { + fragment + .num_rows() + .map(|rows| rows as u64) + .ok_or_else(|| format!("fragment {} is missing num_rows", fragment.id)) + }) + .collect::, _>>()? + .into_iter() + .sum::(); + if total_rows == 0 { + return Err("dataset has no rows".into()); + } + + let counters = Arc::new(SharedCounters::default()); + let cpu_before = read_cpu_sample(); + let started = Instant::now(); + let mut rows = 0u64; + let mut batches = 0u64; + let mut arrow_bytes = 0u64; + const STRIDE: u64 = 104_729; + + for repetition in 0..config.take_repetitions { + let row_ids = (0..config.limit_rows) + .map(|offset| { + repetition + .wrapping_mul(STRIDE) + .wrapping_add(offset.wrapping_mul(STRIDE)) + % total_rows + }) + .collect::>(); + let batch = dataset + .take(&row_ids, ProjectionRequest::Schema(projection.clone())) + .await?; + if batch.num_rows() as u64 != config.limit_rows { + return Err(format!( + "take_rows returned {} rows, expected {}", + batch.num_rows(), + config.limit_rows + ) + .into()); + } + black_box(&batch); + rows += batch.num_rows() as u64; + batches += 1; + let batch_bytes = if config.skip_batch_byte_accounting { + 0 + } else { + batch.get_array_memory_size() as u64 + }; + arrow_bytes += batch_bytes; + counters.batches_completed.fetch_add(1, Ordering::Relaxed); + counters + .rows_completed + .fetch_add(batch.num_rows() as u64, Ordering::Relaxed); + counters + .arrow_bytes + .fetch_add(batch_bytes, Ordering::Relaxed); + } + + let elapsed = started.elapsed(); + let cpu_after = read_cpu_sample(); + Ok(CaseStats { + rows, + batches, + arrow_bytes, + planned_fragments: dataset.fragments().len(), + planned_rows: total_rows, + elapsed, + producer_finished_at: Some(elapsed), + peak_decode_in_flight: 0, + cpu_avg: cpu_before.zip(cpu_after).and_then(|(before, after)| { + let total = after.total.checked_sub(before.total)?; + let idle = after.idle.checked_sub(before.idle)?; + if total == 0 { + return None; + } + Some((total - idle) as f64 / total as f64 * 100.0) + }), + scheduler_diagnostics: scheduler_diagnostics.snapshot(None), + counters, + samples: Vec::new(), + }) +} + +enum RawInFlight { + Unordered(FuturesUnordered>>), + Ordered(FuturesOrdered>>), +} + +impl RawInFlight { + fn new(mode: RawCompletionMode) -> Self { + match mode { + RawCompletionMode::Unordered => Self::Unordered(FuturesUnordered::new()), + RawCompletionMode::Ordered => Self::Ordered(FuturesOrdered::new()), + } + } + + fn len(&self) -> usize { + match self { + Self::Unordered(in_flight) => in_flight.len(), + Self::Ordered(in_flight) => in_flight.len(), + } + } + + fn is_empty(&self) -> bool { + match self { + Self::Unordered(in_flight) => in_flight.is_empty(), + Self::Ordered(in_flight) => in_flight.is_empty(), + } + } + + fn push(&mut self, future: BoxFuture<'static, lance_core::Result>) { + match self { + Self::Unordered(in_flight) => in_flight.push(future), + Self::Ordered(in_flight) => in_flight.push_back(future), + } + } + + async fn next(&mut self) -> Option> { + match self { + Self::Unordered(in_flight) => in_flight.next().await, + Self::Ordered(in_flight) => in_flight.next().await, + } + } +} + +fn raw_read_future( + file_scheduler: FileScheduler, + range: Range, + priority: u64, + raw_submit_mode: RawSubmitMode, + read_chunk_size: u64, + counters: Arc, +) -> BoxFuture<'static, lance_core::Result> { + async move { + match raw_submit_mode { + RawSubmitMode::Single => { + let bytes = file_scheduler.submit_single(range, priority).await?; + Ok(bytes.len()) + } + RawSubmitMode::SplitNoConcat => { + let ranges = split_range_by_size(range, read_chunk_size); + let bytes = file_scheduler.submit_request(ranges, priority).await?; + Ok(bytes.iter().map(bytes::Bytes::len).sum()) + } + RawSubmitMode::SplitConcat => { + let ranges = split_range_by_size(range, read_chunk_size); + let bytes = file_scheduler.submit_request(ranges, priority).await?; + let reassemble_started = Instant::now(); + let total_size = bytes.iter().map(bytes::Bytes::len).sum(); + let mut combined = Vec::with_capacity(total_size); + for chunk in bytes { + combined.extend_from_slice(&chunk); + } + add_duration(&counters.raw_reassemble_ns, reassemble_started.elapsed()); + let len = combined.len(); + black_box(&combined); + Ok(len) + } + } + } + .boxed() +} + +fn split_range_by_size(range: Range, chunk_size: u64) -> Vec> { + let range_size = range.end - range.start; + if range_size <= chunk_size { + return vec![range]; + } + + let num_chunks = range_size.div_ceil(chunk_size); + let per_chunk = range_size / num_chunks; + let mut ranges = Vec::with_capacity(num_chunks as usize); + for idx in 0..num_chunks { + let start = range.start + idx * per_chunk; + let end = if idx == num_chunks - 1 { + range.end + } else { + start + per_chunk + }; + ranges.push(start..end); + } + ranges +} + +fn push_split_planned_ranges( + planned: &mut Vec<(usize, Range)>, + file_idx: usize, + range: Range, + chunk_size: u64, + remaining: &mut u64, +) { + let mut start = range.start; + while start < range.end && *remaining > 0 { + let bytes_to_read = chunk_size.min(range.end - start).min(*remaining); + if bytes_to_read == 0 { + break; + } + let end = start + bytes_to_read; + planned.push((file_idx, start..end)); + *remaining -= bytes_to_read; + start = end; + } +} + +fn push_split_ranges(ranges: &mut Vec>, range: Range, chunk_size: u64) { + let mut start = range.start; + while start < range.end { + let bytes_to_read = chunk_size.min(range.end - start); + if bytes_to_read == 0 { + break; + } + let end = start + bytes_to_read; + ranges.push(start..end); + start = end; + } +} + +async fn run_scheduler_raw_case( + config: &Config, + io_buffer_gib: Option, + scheduler_diagnostics: &SchedulerDiagnosticsCollector, +) -> Result { + scheduler_diagnostics.clear(); + let target_bytes = config + .target_bytes + .ok_or("--target-bytes is required for --backend scheduler-raw")?; + let dataset = Arc::new( + DatasetBuilder::from_uri(&config.uri) + .with_version(config.dataset_version) + .load() + .await?, + ); + let (object_store, _) = LanceObjectStore::from_uri(&config.uri).await?; + let scheduler_config = io_buffer_gib + .map(|gib| SchedulerConfig::new(gib * GIB)) + .unwrap_or_else(|| SchedulerConfig::max_bandwidth(object_store.as_ref())); + let scheduler = ScanScheduler::new(object_store, scheduler_config); + + let (selected_files, planned) = match config.raw_range_mode { + RawRangeMode::FileSequential => { + let mut selected_files = Vec::new(); + let mut selected_file_bytes = 0u64; + for fragment in dataset.fragments().iter() { + for data_file in &fragment.files { + if data_file.base_id.is_some() { + continue; + } + let Some(file_size) = data_file.file_size_bytes.get() else { + continue; + }; + let path = dataset.data_dir().join(data_file.path.as_str()); + let file_scheduler = scheduler + .open_file_with_priority(&path, 0, &data_file.file_size_bytes) + .await?; + selected_file_bytes += file_size.get(); + selected_files.push((file_scheduler, file_size.get())); + if selected_file_bytes >= target_bytes { + break; + } + } + if selected_file_bytes >= target_bytes { + break; + } + } + if selected_files.is_empty() { + return Err("scheduler-raw found no data files with known sizes".into()); + } + + let mut offsets = vec![0u64; selected_files.len()]; + let mut planned = Vec::new(); + let mut remaining = target_bytes; + let mut file_idx = 0usize; + while remaining > 0 { + let idx = file_idx % selected_files.len(); + let file_size = selected_files[idx].1; + if offsets[idx] >= file_size { + offsets[idx] = 0; + } + let available = file_size - offsets[idx]; + let bytes_to_read = config.raw_range_size_bytes.min(available).min(remaining); + let start = offsets[idx]; + let end = start + bytes_to_read; + planned.push((idx, start..end)); + offsets[idx] = end; + remaining -= bytes_to_read; + file_idx += 1; + } + (selected_files, planned) + } + RawRangeMode::MetadataPages | RawRangeMode::MetadataPagesRoundRobin => { + let mut selected_files = Vec::new(); + let mut per_file_ranges = Vec::>>::new(); + let mut candidate_bytes = 0u64; + + 'fragments: for fragment in dataset.fragments().iter() { + for data_file in &fragment.files { + if data_file.base_id.is_some() { + continue; + } + if data_file.file_size_bytes.get().is_none() { + continue; + } + let path = dataset.data_dir().join(data_file.path.as_str()); + let file_scheduler = scheduler + .open_file_with_priority(&path, 0, &data_file.file_size_bytes) + .await?; + let metadata = LanceFileReader::read_all_metadata(&file_scheduler).await?; + let mut file_ranges = Vec::new(); + + let raw_column_indices = config + .raw_column_indices + .clone() + .unwrap_or_else(|| (0..metadata.column_infos.len() as u32).collect()); + for column_index in raw_column_indices { + let column_info = metadata + .column_infos + .get(column_index as usize) + .ok_or_else(|| { + format!( + "raw metadata-pages requested column index {column_index} but file has {} columns", + metadata.column_infos.len() + ) + })?; + for page in column_info.page_infos.iter() { + for (offset, size) in page.buffer_offsets_and_sizes.iter() { + if *size == 0 { + continue; + } + push_split_ranges( + &mut file_ranges, + *offset..(*offset + *size), + config.raw_range_size_bytes, + ); + } + } + } + + if !file_ranges.is_empty() { + candidate_bytes += file_ranges + .iter() + .map(|range| range.end - range.start) + .sum::(); + selected_files.push(( + file_scheduler, + data_file.file_size_bytes.get().unwrap().get(), + )); + per_file_ranges.push(file_ranges); + if candidate_bytes >= target_bytes { + break 'fragments; + } + } + } + } + if selected_files.is_empty() || per_file_ranges.is_empty() { + return Err("scheduler-raw metadata-pages found no readable page buffers".into()); + } + + let mut planned = Vec::new(); + let mut remaining = target_bytes; + match config.raw_range_mode { + RawRangeMode::MetadataPages => { + 'ranges: for (file_idx, ranges) in per_file_ranges.iter().enumerate() { + for range in ranges { + push_split_planned_ranges( + &mut planned, + file_idx, + range.clone(), + config.raw_range_size_bytes, + &mut remaining, + ); + if remaining == 0 { + break 'ranges; + } + } + } + } + RawRangeMode::MetadataPagesRoundRobin => { + let mut positions = vec![0usize; per_file_ranges.len()]; + while remaining > 0 { + let mut made_progress = false; + for (file_idx, ranges) in per_file_ranges.iter().enumerate() { + if positions[file_idx] >= ranges.len() { + continue; + } + let range = ranges[positions[file_idx]].clone(); + positions[file_idx] += 1; + made_progress = true; + push_split_planned_ranges( + &mut planned, + file_idx, + range, + config.raw_range_size_bytes, + &mut remaining, + ); + if remaining == 0 { + break; + } + } + if !made_progress { + break; + } + } + } + RawRangeMode::FileSequential => unreachable!(), + } + + if remaining > 0 { + return Err(format!( + "scheduler-raw metadata-pages planned {} bytes but target is {target_bytes}", + target_bytes - remaining + ) + .into()); + } + (selected_files, planned) + } + }; + let planned_bytes = planned + .iter() + .map(|(_, range)| range.end - range.start) + .sum::(); + + let counters = Arc::new(SharedCounters::default()); + let cpu_before = read_cpu_sample(); + let started = Instant::now(); + let mut samples = Vec::new(); + let mut sample_interval = tokio::time::interval(Duration::from_millis(config.sample_ms)); + sample_interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip); + let mut last_sample = LastSample { + elapsed: Duration::default(), + scheduler_stats: ScanStats::default(), + rows: 0, + batches: 0, + arrow_bytes: 0, + }; + let mut in_flight = RawInFlight::new(config.raw_completion_mode); + let mut next_range = 0usize; + let read_chunk_size = config.read_chunk_size.unwrap_or(DEFAULT_READ_CHUNK_SIZE); + while next_range < planned.len() && in_flight.len() < config.batch_concurrency { + let (idx, range) = planned[next_range].clone(); + in_flight.push(raw_read_future( + selected_files[idx].0.clone(), + range, + next_range as u64, + config.raw_submit_mode, + read_chunk_size, + counters.clone(), + )); + next_range += 1; + } + + let mut bytes_read = 0u64; + let mut requests_completed = 0u64; + while !in_flight.is_empty() { + tokio::select! { + maybe_bytes = in_flight.next() => { + let bytes = maybe_bytes.expect("raw read future disappeared")?; + let bytes = bytes as u64; + bytes_read += bytes; + requests_completed += 1; + counters.batches_completed.fetch_add(1, Ordering::Relaxed); + counters.arrow_bytes.fetch_add(bytes, Ordering::Relaxed); + if next_range < planned.len() { + let (idx, range) = planned[next_range].clone(); + in_flight.push(raw_read_future( + selected_files[idx].0.clone(), + range, + next_range as u64, + config.raw_submit_mode, + read_chunk_size, + counters.clone(), + )); + next_range += 1; + } + } + _ = sample_interval.tick() => { + samples.push(sample_json( + started, + counters.as_ref(), + scheduler_diagnostics.snapshot(io_buffer_gib), + in_flight.len(), + planned.len().saturating_sub(next_range), + &mut last_sample, + )); + } + } + } + counters.arrow_bytes.store(bytes_read, Ordering::Relaxed); + let mut final_diagnostics = scheduler_diagnostics.snapshot(io_buffer_gib); + final_diagnostics.stats = scheduler.stats(); + samples.push(sample_json( + started, + counters.as_ref(), + final_diagnostics, + in_flight.len(), + 0, + &mut last_sample, + )); + let elapsed = started.elapsed(); + let cpu_after = read_cpu_sample(); + + Ok(CaseStats { + rows: 0, + batches: requests_completed, + arrow_bytes: bytes_read, + planned_fragments: selected_files.len(), + planned_rows: planned_bytes, + elapsed, + producer_finished_at: Some(elapsed), + peak_decode_in_flight: config.batch_concurrency, + cpu_avg: cpu_before.zip(cpu_after).and_then(|(before, after)| { + let total = after.total.checked_sub(before.total)?; + let idle = after.idle.checked_sub(before.idle)?; + if total == 0 { + return None; + } + Some((total - idle) as f64 / total as f64 * 100.0) + }), + scheduler_diagnostics: final_diagnostics, + counters, + samples, + }) +} + +async fn run_case( + config: &Config, + io_buffer_gib: Option, + scheduler_diagnostics: &SchedulerDiagnosticsCollector, +) -> Result { + scheduler_diagnostics.clear(); + let dataset = Arc::new( + DatasetBuilder::from_uri(&config.uri) + .with_version(config.dataset_version) + .load() + .await?, + ); + let projection = Arc::new(projected_schema(dataset.schema(), &config.columns)?); + let (object_store, _) = LanceObjectStore::from_uri(&config.uri).await?; + let scheduler_config = io_buffer_gib + .map(|gib| SchedulerConfig::new(gib * GIB)) + .unwrap_or_else(|| SchedulerConfig::max_bandwidth(object_store.as_ref())); + let scheduler = ScanScheduler::new(object_store, scheduler_config); + + let mut planned = Vec::new(); + let mut remaining_rows = config.limit_rows; + for fragment in dataset.fragments().iter() { + if remaining_rows == 0 { + break; + } + let fragment_rows = fragment + .num_rows() + .ok_or_else(|| format!("fragment {} is missing num_rows", fragment.id))? + as u64; + let rows = fragment_rows.min(remaining_rows); + planned.push((fragment.clone(), rows)); + remaining_rows -= rows; + } + if planned.is_empty() { + return Err("no fragments selected".into()); + } + let planned_rows: u64 = planned.iter().map(|(_, rows)| *rows).sum(); + let planned_fragments = planned.len(); + + let counters = Arc::new(SharedCounters::default()); + let cpu_before = read_cpu_sample(); + let started = Instant::now(); + + let (tx, mut rx) = tokio::sync::mpsc::channel::< + BoxFuture<'static, lance_core::Result>, + >(config.batch_concurrency * 2); + let producer = if config.detach_fragment_streams { + tokio::spawn({ + let dataset = dataset.clone(); + let projection = projection.clone(); + let scheduler = scheduler.clone(); + let counters = counters.clone(); + let batch_size = config.batch_size; + let file_reader_options = file_reader_options(config); + let fragment_concurrency = config.fragment_concurrency; + async move { + let drainers = futures::stream::iter(planned.into_iter().enumerate()) + .map({ + move |(priority, (fragment, rows))| { + let dataset = dataset.clone(); + let projection = projection.clone(); + let scheduler = scheduler.clone(); + let tx = tx.clone(); + let counters = counters.clone(); + let file_reader_options = file_reader_options.clone(); + async move { + counters.fragments_started.fetch_add(1, Ordering::Relaxed); + let file_fragment = FileFragment::new(dataset, fragment); + let read_config = FragReadConfig::default() + .with_scan_scheduler(scheduler) + .with_reader_priority(priority as u32); + let read_config = if let Some(file_reader_options) = + file_reader_options.clone() + { + read_config.with_file_reader_options(file_reader_options) + } else { + read_config + }; + + let open_started = Instant::now(); + let reader = + file_fragment.open(projection.as_ref(), read_config).await?; + add_duration(&counters.open_reader_ns, open_started.elapsed()); + + let create_stream_started = Instant::now(); + let mut read_stream = + reader.read_ranges(vec![0..rows].into(), batch_size).await?; + add_duration( + &counters.read_stream_create_ns, + create_stream_started.elapsed(), + ); + + let drainer = tokio::spawn(async move { + loop { + let next_started = Instant::now(); + let maybe_batch_fut = read_stream.next().await; + add_duration( + &counters.next_batch_poll_ns, + next_started.elapsed(), + ); + let Some(batch_fut) = maybe_batch_fut else { + break; + }; + counters + .batch_futures_emitted + .fetch_add(1, Ordering::Relaxed); + let send_started = Instant::now(); + tx.send(batch_fut) + .await + .map_err(|_| "batch consumer dropped")?; + add_duration( + &counters.channel_send_wait_ns, + send_started.elapsed(), + ); + } + counters.fragments_completed.fetch_add(1, Ordering::Relaxed); + Ok::<_, Error>(()) + }); + Ok::<_, Error>(drainer) + } + } + }) + .buffer_unordered(fragment_concurrency) + .try_collect::>() + .await?; + + for drainer in drainers { + drainer.await.map_err(Error::from)??; + } + Ok::<_, Error>(()) + } + }) + } else { + tokio::spawn({ + let dataset = dataset.clone(); + let projection = projection.clone(); + let scheduler = scheduler.clone(); + let counters = counters.clone(); + let batch_size = config.batch_size; + let file_reader_options = file_reader_options(config); + let fragment_concurrency = config.fragment_concurrency; + async move { + futures::stream::iter(planned.into_iter().enumerate()) + .map({ + move |(priority, (fragment, rows))| { + let dataset = dataset.clone(); + let projection = projection.clone(); + let scheduler = scheduler.clone(); + let tx = tx.clone(); + let counters = counters.clone(); + let file_reader_options = file_reader_options.clone(); + async move { + counters.fragments_started.fetch_add(1, Ordering::Relaxed); + let file_fragment = FileFragment::new(dataset, fragment); + let read_config = FragReadConfig::default() + .with_scan_scheduler(scheduler) + .with_reader_priority(priority as u32); + let read_config = if let Some(file_reader_options) = + file_reader_options.clone() + { + read_config.with_file_reader_options(file_reader_options) + } else { + read_config + }; + + let open_started = Instant::now(); + let reader = + file_fragment.open(projection.as_ref(), read_config).await?; + add_duration(&counters.open_reader_ns, open_started.elapsed()); + + let create_stream_started = Instant::now(); + let mut read_stream = + reader.read_ranges(vec![0..rows].into(), batch_size).await?; + add_duration( + &counters.read_stream_create_ns, + create_stream_started.elapsed(), + ); + + loop { + let next_started = Instant::now(); + let maybe_batch_fut = read_stream.next().await; + add_duration( + &counters.next_batch_poll_ns, + next_started.elapsed(), + ); + let Some(batch_fut) = maybe_batch_fut else { + break; + }; + counters + .batch_futures_emitted + .fetch_add(1, Ordering::Relaxed); + let send_started = Instant::now(); + tx.send(batch_fut) + .await + .map_err(|_| "batch consumer dropped")?; + add_duration( + &counters.channel_send_wait_ns, + send_started.elapsed(), + ); + } + counters.fragments_completed.fetch_add(1, Ordering::Relaxed); + Ok::<_, Error>(()) + } + } + }) + .buffer_unordered(fragment_concurrency) + .try_collect::>() + .await?; + Ok::<_, Error>(()) + } + }) + }; + + let mut in_flight = FuturesUnordered::new(); + let skip_batch_byte_accounting = config.skip_batch_byte_accounting; + let drop_read_tasks = config.drop_read_tasks; + let mut producer_done = false; + let mut producer_finished_at = None; + let mut rows = 0u64; + let mut batches = 0u64; + let mut arrow_bytes = 0u64; + let mut peak_decode_in_flight = 0usize; + let mut samples = Vec::new(); + let mut sample_interval = tokio::time::interval(Duration::from_millis(config.sample_ms)); + sample_interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip); + let mut last_sample = LastSample { + elapsed: Duration::default(), + scheduler_stats: ScanStats::default(), + rows: 0, + batches: 0, + arrow_bytes: 0, + }; + + loop { + if producer_done && in_flight.is_empty() { + break; + } + tokio::select! { + maybe_batch_fut = rx.recv(), if !producer_done && in_flight.len() < config.batch_concurrency => { + if let Some(batch_fut) = maybe_batch_fut { + counters.batch_futures_received.fetch_add(1, Ordering::Relaxed); + if drop_read_tasks { + drop(batch_fut); + counters.batches_completed.fetch_add(1, Ordering::Relaxed); + batches += 1; + continue; + } + let counters_for_task = counters.clone(); + in_flight.push(async move { + let decode_started = Instant::now(); + let batch = batch_fut.await?; + let batch_bytes = if skip_batch_byte_accounting { + 0 + } else { + batch.get_array_memory_size() as u64 + }; + add_duration(&counters_for_task.decode_ns, decode_started.elapsed()); + counters_for_task.batches_completed.fetch_add(1, Ordering::Relaxed); + counters_for_task.rows_completed.fetch_add(batch.num_rows() as u64, Ordering::Relaxed); + counters_for_task.arrow_bytes.fetch_add(batch_bytes, Ordering::Relaxed); + Ok::<_, lance_core::Error>((batch, batch_bytes)) + }); + peak_decode_in_flight = peak_decode_in_flight.max(in_flight.len()); + } else { + producer_done = true; + producer_finished_at = Some(started.elapsed()); + } + } + maybe_batch = in_flight.next(), if !in_flight.is_empty() => { + let (batch, batch_bytes) = maybe_batch.expect("in-flight batch future disappeared")?; + rows += batch.num_rows() as u64; + batches += 1; + arrow_bytes += batch_bytes; + } + _ = sample_interval.tick() => { + samples.push(sample_json( + started, + counters.as_ref(), + scheduler_diagnostics.snapshot(io_buffer_gib), + in_flight.len(), + rx.len(), + &mut last_sample, + )); + } + } + } + producer.await??; + + let mut final_diagnostics = scheduler_diagnostics.snapshot(io_buffer_gib); + final_diagnostics.stats = scheduler.stats(); + samples.push(sample_json( + started, + counters.as_ref(), + final_diagnostics, + in_flight.len(), + rx.len(), + &mut last_sample, + )); + + let elapsed = started.elapsed(); + let cpu_after = read_cpu_sample(); + + Ok(CaseStats { + rows, + batches, + arrow_bytes, + planned_fragments, + planned_rows, + elapsed, + producer_finished_at, + peak_decode_in_flight, + cpu_avg: cpu_before.zip(cpu_after).and_then(|(before, after)| { + let total = after.total.checked_sub(before.total)?; + let idle = after.idle.checked_sub(before.idle)?; + if total == 0 { + return None; + } + Some((total - idle) as f64 / total as f64 * 100.0) + }), + scheduler_diagnostics: final_diagnostics, + counters, + samples, + }) +} + +fn read_cpu_sample() -> Option { + let contents = fs::read_to_string("/proc/stat").ok()?; + let line = contents.lines().next()?; + let values = line + .split_whitespace() + .skip(1) + .map(|value| value.parse::()) + .collect::, _>>() + .ok()?; + if values.len() < 5 { + return None; + } + + let idle = values[3] + values[4]; + let total = values.iter().sum(); + Some(CpuSample { idle, total }) +} + +fn now_unix_secs() -> u64 { + SystemTime::now() + .duration_since(UNIX_EPOCH) + .map(|duration| duration.as_secs()) + .unwrap_or_default() +} + +fn current_commit() -> String { + option_env!("LANCE_BENCH_COMMIT") + .or_else(|| option_env!("GIT_COMMIT")) + .unwrap_or("unknown") + .to_string() +} + +fn env_var(name: &str) -> Option { + env::var(name).ok() +} + +#[tokio::main] +async fn main() -> Result<()> { + let config = parse_args()?; + if config.describe_layout { + return describe_layout(&config).await; + } + let scheduler_diagnostics = SchedulerDiagnosticsCollector::default(); + let subscriber = tracing_subscriber::registry().with(SchedulerDiagnosticsLayer::new( + scheduler_diagnostics.clone(), + )); + tracing::subscriber::set_global_default(subscriber).map_err(|error| { + std::io::Error::other(format!( + "failed to install scheduler diagnostics subscriber: {error}" + )) + })?; + + fs::create_dir_all(&config.out_dir)?; + let output_path = format!( + "{}/s3_file_reader_diagnostics_{}.jsonl", + config.out_dir, + now_unix_secs() + ); + let mut jsonl = String::new(); + let commit = current_commit(); + let instance = env_var("EC2_INSTANCE_TYPE").unwrap_or_else(|| "unknown".to_string()); + let region = env_var("AWS_REGION") + .or_else(|| env_var("AWS_DEFAULT_REGION")) + .unwrap_or_else(|| "unknown".to_string()); + let projection = projection_name(&config.columns); + + for io_buffer_gib in &config.io_buffer_gib { + println!( + "running case={} backend={} projection={} limit_rows={} io_buffer_gib={} batch_size={} fragment_concurrency={} batch_concurrency={} sample_ms={}", + config.case_name, + config.backend.name(), + projection, + config.limit_rows, + io_buffer_gib + .map(|value| value.to_string()) + .unwrap_or_else(|| "auto".to_string()), + config.batch_size, + config.fragment_concurrency, + config.batch_concurrency, + config.sample_ms + ); + let stats = match config.backend { + Backend::FileReader => { + run_case(&config, *io_buffer_gib, &scheduler_diagnostics).await? + } + Backend::Scanner => { + run_scanner_case(&config, *io_buffer_gib, &scheduler_diagnostics).await? + } + Backend::SchedulerRaw => { + run_scheduler_raw_case(&config, *io_buffer_gib, &scheduler_diagnostics).await? + } + Backend::DatasetTake => run_dataset_take_case(&config, &scheduler_diagnostics).await?, + }; + let elapsed_secs = stats.elapsed.as_secs_f64(); + let scheduler_stats = stats.scheduler_diagnostics.stats; + let logical_gbps = if elapsed_secs > 0.0 { + stats.arrow_bytes as f64 * 8.0 / elapsed_secs / 1_000_000_000.0 + } else { + 0.0 + }; + let physical_gbps = if elapsed_secs > 0.0 { + scheduler_stats.bytes_read as f64 * 8.0 / elapsed_secs / 1_000_000_000.0 + } else { + 0.0 + }; + let rows_per_second = if elapsed_secs > 0.0 { + stats.rows as f64 / elapsed_secs + } else { + 0.0 + }; + let bytes_per_row = stats.arrow_bytes.checked_div(stats.rows).unwrap_or(0); + let avg_bytes_per_scheduler_request = scheduler_stats + .bytes_read + .checked_div(scheduler_stats.requests) + .unwrap_or(0); + let avg_bytes_per_scheduler_iop = scheduler_stats + .bytes_read + .checked_div(scheduler_stats.iops) + .unwrap_or(0); + let counters = stats.counters.as_ref(); + let record = json!({ + "case": config.case_name, + "instance": instance, + "region": region, + "layer": config.backend.layer(), + "backend": config.backend.name(), + "dataset_uri": config.uri, + "dataset_version": config.dataset_version, + "lance_commit": commit, + "projection": projection, + "limit_rows": config.limit_rows, + "target_bytes": config.target_bytes, + "raw_range_size_bytes": config.raw_range_size_bytes, + "raw_range_mode": config.raw_range_mode.name(), + "raw_column_indices": config.raw_column_indices.clone(), + "raw_submit_mode": config.raw_submit_mode.name(), + "raw_completion_mode": config.raw_completion_mode.name(), + "take_repetitions": config.take_repetitions, + "raw_read_chunk_size_bytes": config.read_chunk_size.unwrap_or(DEFAULT_READ_CHUNK_SIZE), + "planned_rows": stats.planned_rows, + "planned_fragments": stats.planned_fragments, + "rows": stats.rows, + "batches": stats.batches, + "batch_size": config.batch_size, + "batch_size_bytes": config.batch_size_bytes, + "skip_batch_byte_accounting": config.skip_batch_byte_accounting, + "read_chunk_size": config.read_chunk_size, + "fragment_concurrency": config.fragment_concurrency, + "batch_concurrency": config.batch_concurrency, + "detach_fragment_streams": config.detach_fragment_streams, + "drop_read_tasks": config.drop_read_tasks, + "sample_ms": config.sample_ms, + "io_buffer_bytes": io_buffer_gib.map(|value| value * GIB), + "io_buffer_mode": if io_buffer_gib.is_some() { "explicit" } else { "auto" }, + "lance_io_threads": env_var("LANCE_IO_THREADS").or_else(|| env_var("IO_THREADS")), + "lance_default_io_buffer_size": env_var("LANCE_DEFAULT_IO_BUFFER_SIZE"), + "lance_max_iop_size": env_var("LANCE_MAX_IOP_SIZE"), + "lance_use_lite_scheduler": env_var("LANCE_USE_LITE_SCHEDULER"), + "lance_inline_scheduling_threshold": env_var("LANCE_INLINE_SCHEDULING_THRESHOLD"), + "elapsed_seconds": elapsed_secs, + "producer_finished_seconds": stats.producer_finished_at.map(|duration| duration.as_secs_f64()), + "logical_gbps": logical_gbps, + "physical_gbps": physical_gbps, + "rows_per_second": rows_per_second, + "arrow_bytes": stats.arrow_bytes, + "bytes_per_row": bytes_per_row, + "avg_bytes_per_scheduler_request": avg_bytes_per_scheduler_request, + "avg_bytes_per_scheduler_iop": avg_bytes_per_scheduler_iop, + "scheduler_iops": scheduler_stats.iops, + "scheduler_requests": scheduler_stats.requests, + "scheduler_bytes_read": scheduler_stats.bytes_read, + "scheduler_diagnostics": diagnostics_json(stats.scheduler_diagnostics), + "fragments_started": counters.fragments_started.load(Ordering::Relaxed), + "fragments_completed": counters.fragments_completed.load(Ordering::Relaxed), + "batch_futures_emitted": counters.batch_futures_emitted.load(Ordering::Relaxed), + "batch_futures_received": counters.batch_futures_received.load(Ordering::Relaxed), + "batches_completed": counters.batches_completed.load(Ordering::Relaxed), + "peak_decode_in_flight": stats.peak_decode_in_flight, + "open_reader_seconds_total": ns_to_seconds(counters.open_reader_ns.load(Ordering::Relaxed)), + "read_stream_create_seconds_total": ns_to_seconds(counters.read_stream_create_ns.load(Ordering::Relaxed)), + "next_batch_poll_seconds_total": ns_to_seconds(counters.next_batch_poll_ns.load(Ordering::Relaxed)), + "channel_send_wait_seconds_total": ns_to_seconds(counters.channel_send_wait_ns.load(Ordering::Relaxed)), + "decode_seconds_total": ns_to_seconds(counters.decode_ns.load(Ordering::Relaxed)), + "raw_reassemble_seconds_total": ns_to_seconds(counters.raw_reassemble_ns.load(Ordering::Relaxed)), + "cpu_avg": stats.cpu_avg, + "samples": stats.samples, + }); + println!( + "case={} backend={} projection={} io_buffer_gib={} elapsed={:.3}s logical_gbps={:.2} physical_gbps={:.2} rows={} batches={} scheduler_bytes_read={} scheduler_iops={} scheduler_requests={} active_iops={} pending_iops={} cpu_avg={}", + config.case_name, + config.backend.name(), + projection, + io_buffer_gib + .map(|value| value.to_string()) + .unwrap_or_else(|| "auto".to_string()), + elapsed_secs, + logical_gbps, + physical_gbps, + stats.rows, + stats.batches, + scheduler_stats.bytes_read, + scheduler_stats.iops, + scheduler_stats.requests, + stats.scheduler_diagnostics.active_iops, + stats.scheduler_diagnostics.pending_iops, + stats + .cpu_avg + .map(|value| format!("{value:.1}%")) + .unwrap_or_else(|| "unknown".to_string()) + ); + jsonl.push_str(&serde_json::to_string(&record)?); + jsonl.push('\n'); + fs::write(&output_path, &jsonl)?; + } + + println!("wrote {output_path}"); + Ok(()) +} diff --git a/rust/lance/benches/take_blob.rs b/rust/lance/benches/take_blob.rs index 7f3483bffda..0da30d9f277 100644 --- a/rust/lance/benches/take_blob.rs +++ b/rust/lance/benches/take_blob.rs @@ -15,7 +15,7 @@ use lance::dataset::{Dataset, ProjectionRequest, ReadParams, WriteParams}; use lance_arrow::BLOB_META_KEY; use lance_encoding::decoder::DecoderConfig; use lance_file::reader::FileReaderOptions; -use lance_file::version::LanceFileVersion; +use lance_file::version::{ConcreteFileVersion, LanceFileVersion}; #[cfg(target_os = "linux")] use lance_testing::pprof::{Output, PProfProfiler}; use tokio::runtime::Runtime; @@ -209,7 +209,10 @@ async fn write_blob_dataset( version: LanceFileVersion, cache_repetition_index: bool, ) -> Dataset { - let batches = if version >= LanceFileVersion::V2_2 { + let batches = if matches!( + version.resolve(), + ConcreteFileVersion::V2_2 | ConcreteFileVersion::V2_3 + ) { make_blob_v2_batches() } else { make_legacy_blob_batches() diff --git a/rust/lance/src/blob.rs b/rust/lance/src/blob.rs index b112f011419..6ad2a2ad26c 100644 --- a/rust/lance/src/blob.rs +++ b/rust/lance/src/blob.rs @@ -4,7 +4,7 @@ //! Builders and file-level writer helpers for Lance blob v2 columns. //! //! Logical blob input uses `Struct`. File-level blob -//! descriptors use a physical writer-side struct with `kind`, `blob_id`, and range fields. +//! preparation produces a kind-aware writer intermediate with `blob_id` and range fields. use std::collections::{HashMap, HashSet}; use std::num::NonZeroUsize; @@ -21,14 +21,17 @@ use arrow_array::{ types::{UInt8Type, UInt32Type, UInt64Type}, }; use arrow_buffer::NullBufferBuilder; -use arrow_schema::{DataType, Field, Fields}; +use arrow_schema::{DataType, Field}; use bytes::Bytes; use lance_arrow::{ ARROW_EXT_NAME_KEY, BLOB_DEDICATED_SIZE_THRESHOLD_META_KEY, BLOB_INLINE_SIZE_THRESHOLD_META_KEY, BLOB_V2_EXT_NAME, FieldExt, }; use lance_core::{ - datatypes::{BlobKind, Field as LanceField, Schema as LanceSchema}, + datatypes::{ + BLOB_V2_LOGICAL_MINIMAL_FIELDS, BLOB_V2_PREPARED_FIELDS, BLOB_V2_PREPARED_TYPE, BlobKind, + BlobV2Layout, Field as LanceField, Schema as LanceSchema, + }, utils::blob::blob_path, }; use lance_io::{ @@ -112,125 +115,79 @@ pub fn blob_field_with_options(name: &str, nullable: bool, options: BlobFieldOpt } Field::new( name, - DataType::Struct( - vec![ - Field::new("data", DataType::LargeBinary, true), - Field::new("uri", DataType::Utf8, true), - ] - .into(), - ), + DataType::Struct(BLOB_V2_LOGICAL_MINIMAL_FIELDS.clone()), nullable, ) .with_metadata(metadata) } -fn prepared_blob_child_fields() -> Fields { - Fields::from(vec![ - Field::new("kind", DataType::UInt8, true), - Field::new("data", DataType::LargeBinary, true), - Field::new("uri", DataType::Utf8, true), - Field::new("blob_id", DataType::UInt32, true), - Field::new("blob_size", DataType::UInt64, true), - Field::new("position", DataType::UInt64, true), - ]) -} - fn prepared_blob_field_with_metadata( name: &str, nullable: bool, metadata: HashMap, ) -> Field { - Field::new( - name, - DataType::Struct(prepared_blob_child_fields()), - nullable, - ) - .with_metadata(metadata) + Field::new(name, BLOB_V2_PREPARED_TYPE.clone(), nullable).with_metadata(metadata) } fn logical_blob_lance_children() -> Result> { - [ - Field::new("data", DataType::LargeBinary, true), - Field::new("uri", DataType::Utf8, true), - ] - .iter() - .map(LanceField::try_from) - .collect() -} - -fn field_matches(field: &Field, name: &str, data_type: &DataType, nullable: bool) -> bool { - field.name() == name && field.data_type() == data_type && field.is_nullable() == nullable -} - -fn blob_v2_shape_error(field: &Field) -> Error { - Error::invalid_input(format!( - "Blob v2 field '{}' must use either logical struct \ - with optional position/size UInt64 fields or prepared struct", - field.name() - )) + BLOB_V2_LOGICAL_MINIMAL_FIELDS + .iter() + .map(|field| LanceField::try_from(field.as_ref())) + .collect() } -/// Returns true when `field` is the writer-side prepared blob v2 struct. -pub(crate) fn is_prepared_blob_v2_field(field: &Field) -> bool { +pub(crate) fn blob_v2_layout(field: &Field) -> Option { if !field.is_blob_v2() { - return false; + return None; } let DataType::Struct(fields) = field.data_type() else { - return false; + return None; }; - let expected = prepared_blob_child_fields(); - fields.len() == expected.len() - && fields - .iter() - .zip(expected.iter()) - .all(|(actual, expected)| actual.as_ref() == expected.as_ref()) + BlobV2Layout::classify(fields) } -/// Returns true when `field` is the logical blob v2 input struct. -pub(crate) fn is_logical_blob_v2_field(field: &Field) -> bool { - if !field.is_blob_v2() { - return false; - } - let DataType::Struct(fields) = field.data_type() else { - return false; +pub(crate) fn blob_v2_shape_error(field: &Field, expected: &[BlobV2Layout]) -> Error { + let expected = expected + .iter() + .map(ToString::to_string) + .collect::>() + .join(" or "); + let actual = match field.data_type() { + DataType::Struct(fields) => BlobV2Layout::classify(fields) + .map(|layout| format!("{layout} layout")) + .unwrap_or_else(|| format!("unrecognized layout {fields:?}")), + data_type => format!("non-struct type {data_type}"), }; - match fields.len() { - 2 => { - field_matches(fields[0].as_ref(), "data", &DataType::LargeBinary, true) - && field_matches(fields[1].as_ref(), "uri", &DataType::Utf8, true) - } - 4 => { - field_matches(fields[0].as_ref(), "data", &DataType::LargeBinary, true) - && field_matches(fields[1].as_ref(), "uri", &DataType::Utf8, true) - && fields[2].name() == "position" - && fields[2].data_type() == &DataType::UInt64 - && fields[3].name() == "size" - && fields[3].data_type() == &DataType::UInt64 - } - _ => false, - } + Error::invalid_input(format!( + "Blob v2 field '{}' has {actual}; expected {expected} layout", + field.name(), + )) } -fn normalize_prepared_blob_lance_field(field: &LanceField) -> Result { +fn prepared_to_logical_blob_lance_field(field: &LanceField) -> Result { if field.is_blob_v2() { let arrow_field = Field::from(field); - if is_prepared_blob_v2_field(&arrow_field) { - let mut normalized = field.clone(); - let mut logical_children = logical_blob_lance_children()?; - for (logical_child, prepared_child) in - logical_children.iter_mut().zip(field.children.iter()) - { - logical_child.id = prepared_child.id; - logical_child.parent_id = field.id; + match blob_v2_layout(&arrow_field) { + Some(BlobV2Layout::Prepared) => { + let mut normalized = field.clone(); + let mut logical_children = logical_blob_lance_children()?; + for (logical_child, prepared_child) in + logical_children.iter_mut().zip(field.children.iter()) + { + logical_child.id = prepared_child.id; + logical_child.parent_id = field.id; + } + normalized.children = logical_children; + return Ok(normalized); + } + Some(BlobV2Layout::Logical) => return Ok(field.clone()), + _ => { + return Err(blob_v2_shape_error( + &arrow_field, + &[BlobV2Layout::Logical, BlobV2Layout::Prepared], + )); } - normalized.children = logical_children; - return Ok(normalized); - } - if is_logical_blob_v2_field(&arrow_field) { - return Ok(field.clone()); } - return Err(blob_v2_shape_error(&arrow_field)); } if field.children.is_empty() { @@ -240,7 +197,7 @@ fn normalize_prepared_blob_lance_field(field: &LanceField) -> Result let normalized_children = field .children .iter() - .map(normalize_prepared_blob_lance_field) + .map(prepared_to_logical_blob_lance_field) .collect::>>()?; Ok(LanceField { @@ -249,11 +206,11 @@ fn normalize_prepared_blob_lance_field(field: &LanceField) -> Result }) } -pub(crate) fn normalize_prepared_blob_schema(schema: &LanceSchema) -> Result { +pub(crate) fn prepared_to_logical_blob_schema(schema: &LanceSchema) -> Result { let fields = schema .fields .iter() - .map(normalize_prepared_blob_lance_field) + .map(prepared_to_logical_blob_lance_field) .collect::>>()?; Ok(LanceSchema { fields, @@ -331,14 +288,23 @@ fn validate_range(offset: u64, size: u64, object_size: u64, label: &str) -> Resu } fn validate_prepared_blob_value_array(field: &Field, array: &ArrayRef) -> Result<()> { - if !is_prepared_blob_v2_field(field) { - return Err(blob_v2_shape_error(field)); + if blob_v2_layout(field) != Some(BlobV2Layout::Prepared) { + return Err(blob_v2_shape_error(field, &[BlobV2Layout::Prepared])); } let struct_arr = array .as_any() .downcast_ref::() .ok_or_else(|| Error::invalid_input("Prepared blob column was not a struct array"))?; + if BlobV2Layout::classify(struct_arr.fields()) != Some(BlobV2Layout::Prepared) { + let actual = BlobV2Layout::classify(struct_arr.fields()) + .map(|layout| layout.to_string()) + .unwrap_or_else(|| format!("unrecognized ({:?})", struct_arr.fields())); + return Err(Error::invalid_input(format!( + "Prepared blob column '{}' has {actual} array layout; expected prepared layout", + field.name() + ))); + } let kind_col = struct_arr .column_by_name("kind") .ok_or_else(|| Error::invalid_input("Prepared blob struct missing `kind` field"))? @@ -465,7 +431,7 @@ pub struct BlobRange { pub size: u64, } -/// A physical blob descriptor row. +/// A kind-aware row used to build the writer-prepared blob representation. #[derive(Clone, Debug, PartialEq, Eq)] pub enum BlobDescriptor { /// A null blob row. @@ -489,19 +455,22 @@ pub enum BlobDescriptor { }, } -/// A physical blob descriptor column ready to be included in a [`RecordBatch`](arrow_array::RecordBatch). +/// A writer-prepared blob column ready to be included in a [`RecordBatch`](arrow_array::RecordBatch). +/// +/// Despite the legacy type name, its Arrow representation is +/// [`BlobV2Layout::Prepared`], not the descriptor stored in a Lance file. pub struct BlobDescriptorColumn { field: Field, array: ArrayRef, } impl BlobDescriptorColumn { - /// Return the Arrow field for the descriptor column. + /// Return the Arrow field for the prepared column. pub fn field(&self) -> &Field { &self.field } - /// Return the Arrow array for the descriptor column. + /// Return the Arrow array for the prepared column. pub fn array(&self) -> &ArrayRef { &self.array } @@ -512,9 +481,9 @@ impl BlobDescriptorColumn { } } -/// Builds physical blob descriptors for one blob v2 column. +/// Builds the writer-prepared representation for one blob v2 column. /// -/// This builder only produces the writer-side descriptor struct array. It does not allocate blob ids, +/// This builder only produces the writer-prepared struct array. It does not allocate blob ids, /// choose sidecar paths, write blob objects, or commit data files. pub struct BlobDescriptorArrayBuilder { field: Field, @@ -606,12 +575,12 @@ impl BlobDescriptorArrayBuilder { self.push(BlobDescriptor::Null) } - /// Return the descriptor Arrow field for this blob column. + /// Return the prepared Arrow field for this blob column. pub fn field(&self) -> &Field { &self.field } - /// Finish this column and return the writer-side descriptor struct array. + /// Finish this column and return the writer-prepared struct array. pub fn finish(self) -> Result { let mut kind_builder = PrimitiveBuilder::::with_capacity(self.values.len()); let mut data_builder = LargeBinaryBuilder::with_capacity(self.values.len(), 0); @@ -682,7 +651,7 @@ impl BlobDescriptorArrayBuilder { } let array = Arc::new(StructArray::try_new( - prepared_blob_child_fields(), + BLOB_V2_PREPARED_FIELDS.clone(), vec![ Arc::new(kind_builder.finish()), Arc::new(data_builder.finish()), @@ -736,12 +705,28 @@ fn validate_blob_descriptor(value: &BlobDescriptor) -> Result<()> { } } +fn packed_descriptor(blob_id: u32, offset: u64, size: u64) -> Result<(BlobDescriptor, u64)> { + let next_offset = offset.checked_add(size).ok_or_else(|| { + Error::invalid_input(format!( + "Packed blob writer offset overflowed: offset={offset}, size={size}" + )) + })?; + Ok(( + BlobDescriptor::Packed { + blob_id, + offset, + size, + }, + next_offset, + )) +} + /// Writes a Lance-owned packed sidecar blob for one data file and returns descriptors. pub struct PackedBlobWriter { object_store: ObjectStore, path: Path, blob_id: u32, - writer: Box, + writer: Option>, offset: u64, values: Vec, } @@ -759,7 +744,7 @@ impl PackedBlobWriter { object_store, path, blob_id, - writer, + writer: Some(writer), offset: 0, values: Vec::new(), }) @@ -782,10 +767,60 @@ impl PackedBlobWriter { Ok(()) } + /// Append multiple logical blobs, one per iterator item. + /// + /// Each `Some(bytes)` is appended to the sidecar and records a packed + /// descriptor; an empty slice records a valid zero-length blob. Each `None` + /// records a [`BlobDescriptor::Null`] without writing any bytes, so the + /// descriptors returned by [`Self::finish`] stay row-aligned with the input. + /// + /// If writing fails or the future is cancelled after a partial write, no + /// descriptors from this call are recorded, the active writer is dropped, + /// and this instance cannot be reused. + /// + /// ``` + /// # use lance::{PackedBlobWriter, Result}; + /// # async fn write(mut writer: PackedBlobWriter) -> Result<()> { + /// writer + /// .write_packed_blobs([Some(b"first".as_slice()), None, Some(b"second".as_slice())]) + /// .await?; + /// let descriptors = writer.finish().await?; + /// assert_eq!(descriptors.len(), 3); + /// # Ok(()) + /// # } + /// ``` + pub async fn write_packed_blobs<'a>( + &mut self, + blobs: impl IntoIterator>, + ) -> Result<()> { + let mut writer = self.take_writer()?; + let mut descriptors = Vec::new(); + let mut next_offset = self.offset; + for blob in blobs { + let Some(blob) = blob else { + descriptors.push(BlobDescriptor::Null); + continue; + }; + let (descriptor, following_offset) = + packed_descriptor(self.blob_id, next_offset, blob.len() as u64)?; + if !blob.is_empty() { + writer.write_all(blob).await?; + } + descriptors.push(descriptor); + next_offset = following_offset; + } + self.writer = Some(writer); + self.offset = next_offset; + self.values.extend(descriptors); + Ok(()) + } + pub(crate) async fn write_blob_bytes(&mut self, bytes: &[u8]) -> Result { let size = bytes.len() as u64; let offset = self.offset; - self.writer.write_all(bytes).await?; + let mut writer = self.take_writer()?; + writer.write_all(bytes).await?; + self.writer = Some(writer); self.record_written_blob(offset, size) } @@ -796,28 +831,32 @@ impl PackedBlobWriter { ) -> Result { let size = range.len() as u64; let offset = self.offset; - self.writer.copy_range_from_reader(reader, range).await?; + let mut writer = self.take_writer()?; + writer.copy_range_from_reader(reader, range).await?; + self.writer = Some(writer); self.record_written_blob(offset, size) } fn record_written_blob(&mut self, offset: u64, size: u64) -> Result { - self.offset = self.offset.checked_add(size).ok_or_else(|| { - Error::invalid_input(format!( - "Packed blob writer offset overflowed: offset={offset}, size={size}" - )) - })?; - let value = BlobDescriptor::Packed { - blob_id: self.blob_id, - offset, - size, - }; + let (value, next_offset) = packed_descriptor(self.blob_id, offset, size)?; + self.offset = next_offset; self.values.push(value.clone()); Ok(value) } + fn take_writer(&mut self) -> Result> { + self.writer.take().ok_or_else(|| { + Error::io(format!( + "Packed blob writer for '{}' has no active upload", + self.path + )) + }) + } + /// Finish the packed sidecar and return descriptors in write order. pub async fn finish(mut self) -> Result> { - Writer::shutdown(self.writer.as_mut()).await?; + let mut writer = self.take_writer()?; + Writer::shutdown(writer.as_mut()).await?; let object_size = self.object_store.size(&self.path).await?; validate_range(0, self.offset, object_size, "Packed blob")?; Ok(self.values) @@ -987,11 +1026,7 @@ impl BlobArrayBuilder { let validity = self.validity.finish(); let struct_array = StructArray::try_new( - vec![ - Field::new("data", DataType::LargeBinary, true), - Field::new("uri", DataType::Utf8, true), - ] - .into(), + BLOB_V2_LOGICAL_MINIMAL_FIELDS.clone(), vec![data as ArrayRef, uri as ArrayRef], validity, )?; @@ -1010,13 +1045,105 @@ impl BlobArrayBuilder { #[cfg(test)] mod tests { + use std::future::Future; + use std::io; use std::num::NonZeroUsize; + use std::pin::Pin; + use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; + use std::task::{Context, Poll}; use super::*; use arrow_array::cast::AsArray; use arrow_array::{Array, StringArray}; use arrow_schema::Schema as ArrowSchema; + use async_trait::async_trait; + use futures::task::noop_waker; use lance_core::utils::tempfile::TempDir; + use lance_io::object_writer::WriteResult; + use tokio::io::AsyncWrite; + + #[derive(Clone, Copy)] + enum WriteTerminal { + Error, + Pending, + } + + struct PartialWriter { + bytes_before_terminal: usize, + terminal: WriteTerminal, + bytes_written: Arc, + dropped: Arc, + } + + impl AsyncWrite for PartialWriter { + fn poll_write( + mut self: Pin<&mut Self>, + _cx: &mut Context<'_>, + bytes: &[u8], + ) -> Poll> { + if self.bytes_before_terminal > 0 { + let written = self.bytes_before_terminal.min(bytes.len()); + self.bytes_before_terminal -= written; + self.bytes_written.fetch_add(written, Ordering::SeqCst); + return Poll::Ready(Ok(written)); + } + match self.terminal { + WriteTerminal::Error => { + Poll::Ready(Err(io::Error::other("injected write failure"))) + } + WriteTerminal::Pending => Poll::Pending, + } + } + + fn poll_flush(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll> { + Poll::Ready(Ok(())) + } + + fn poll_shutdown(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll> { + Poll::Ready(Ok(())) + } + } + + #[async_trait] + impl Writer for PartialWriter { + async fn tell(&mut self) -> Result { + Ok(self.bytes_written.load(Ordering::SeqCst)) + } + + async fn shutdown(&mut self) -> Result { + Ok(WriteResult::default()) + } + } + + impl Drop for PartialWriter { + fn drop(&mut self) { + self.dropped.store(true, Ordering::SeqCst); + } + } + + fn partial_writer( + terminal: WriteTerminal, + ) -> (Box, Arc, Arc) { + let bytes_written = Arc::new(AtomicUsize::new(0)); + let dropped = Arc::new(AtomicBool::new(false)); + ( + Box::new(PartialWriter { + bytes_before_terminal: 2, + terminal, + bytes_written: bytes_written.clone(), + dropped: dropped.clone(), + }), + bytes_written, + dropped, + ) + } + + fn cancel_pending(future: F) { + let mut future = Box::pin(future); + let waker = noop_waker(); + let mut context = Context::from_waker(&waker); + assert!(future.as_mut().poll(&mut context).is_pending()); + } #[test] fn test_field_metadata() { @@ -1103,7 +1230,7 @@ mod tests { writer.push_null().unwrap(); let column = writer.finish().unwrap(); - assert!(is_prepared_blob_v2_field(column.field())); + assert_eq!(blob_v2_layout(column.field()), Some(BlobV2Layout::Prepared)); let struct_arr = column.array().as_struct(); let kinds = struct_arr .column_by_name("kind") @@ -1145,7 +1272,7 @@ mod tests { ] { let array = Arc::new( StructArray::try_new( - prepared_blob_child_fields(), + BLOB_V2_PREPARED_FIELDS.clone(), vec![ Arc::new(arrow_array::UInt8Array::from(vec![kind as u8])) as ArrayRef, Arc::new(arrow_array::LargeBinaryArray::from_iter([None::<&[u8]>])), @@ -1165,7 +1292,7 @@ mod tests { } #[test] - fn test_normalize_prepared_blob_schema_preserves_non_blob_fields() { + fn test_prepared_to_logical_blob_schema_preserves_non_blob_fields() { let mut metadata = HashMap::new(); metadata.insert(ARROW_EXT_NAME_KEY.to_string(), BLOB_V2_EXT_NAME.to_string()); let prepared_field = prepared_blob_field_with_metadata("blob", true, metadata); @@ -1182,7 +1309,7 @@ mod tests { let dictionary_values = Arc::new(StringArray::from(vec!["a", "b"])) as ArrayRef; schema.fields[0].set_dictionary_values(&dictionary_values); - let normalized = normalize_prepared_blob_schema(&schema).unwrap(); + let normalized = prepared_to_logical_blob_schema(&schema).unwrap(); assert_eq!(normalized.fields[0].id, 42); assert_eq!( @@ -1265,4 +1392,106 @@ mod tests { let column = builder.finish().unwrap(); assert_eq!(column.array().len(), 3); } + + #[tokio::test] + async fn test_packed_blob_writer_bulk_bytes() { + let temp_dir = TempDir::default(); + let data_dir = Path::from_absolute_path(temp_dir.std_path().join("data")).unwrap(); + let data_file_path = data_dir.join("data-file.lance"); + let mut writer = PackedBlobWriter::try_new(ObjectStore::local(), data_file_path, 7) + .await + .unwrap(); + + writer + .write_packed_blobs([ + Some(b"a".as_slice()), + Some(b"".as_slice()), + None, + Some(b"bc".as_slice()), + ]) + .await + .unwrap(); + + assert_eq!( + writer.finish().await.unwrap(), + vec![ + BlobDescriptor::Packed { + blob_id: 7, + offset: 0, + size: 1, + }, + BlobDescriptor::Packed { + blob_id: 7, + offset: 1, + size: 0, + }, + BlobDescriptor::Null, + BlobDescriptor::Packed { + blob_id: 7, + offset: 1, + size: 2, + }, + ] + ); + } + + #[tokio::test] + async fn test_packed_blob_writer_bulk_drops_after_partial_write_error() { + let (partial_writer, bytes_written, dropped) = partial_writer(WriteTerminal::Error); + let previous_descriptor = BlobDescriptor::Packed { + blob_id: 7, + offset: 0, + size: 3, + }; + let mut writer = PackedBlobWriter { + object_store: ObjectStore::local(), + path: Path::from("packed.blob"), + blob_id: 7, + writer: Some(partial_writer), + offset: 3, + values: vec![previous_descriptor.clone()], + }; + + let error = writer + .write_packed_blobs([Some(b"abcdef".as_slice())]) + .await + .unwrap_err(); + + assert!(matches!(error, Error::IO { .. })); + assert!(error.to_string().contains("injected write failure")); + assert_eq!(bytes_written.load(Ordering::SeqCst), 2); + assert!(dropped.load(Ordering::SeqCst)); + assert!(writer.writer.is_none()); + assert_eq!(writer.offset, 3); + assert_eq!(writer.values, vec![previous_descriptor]); + let retry_error = writer.write_blob(b"retry").await.unwrap_err(); + assert!(matches!(retry_error, Error::IO { .. })); + assert!(retry_error.to_string().contains("no active upload")); + } + + #[test] + fn test_packed_blob_writer_bulk_drops_if_cancelled() { + let (partial_writer, bytes_written, dropped) = partial_writer(WriteTerminal::Pending); + let previous_descriptor = BlobDescriptor::Packed { + blob_id: 7, + offset: 0, + size: 3, + }; + let mut writer = PackedBlobWriter { + object_store: ObjectStore::local(), + path: Path::from("packed.blob"), + blob_id: 7, + writer: Some(partial_writer), + offset: 3, + values: vec![previous_descriptor.clone()], + }; + + cancel_pending(writer.write_packed_blobs([Some(b"abcdef".as_slice())])); + + assert_eq!(bytes_written.load(Ordering::SeqCst), 2); + assert!(dropped.load(Ordering::SeqCst)); + assert!(writer.writer.is_none()); + assert_eq!(writer.offset, 3); + assert_eq!(writer.values, vec![previous_descriptor]); + } } diff --git a/rust/lance/src/datafusion/dataframe.rs b/rust/lance/src/datafusion/dataframe.rs index 00db9920bf9..7ebc35edbaa 100644 --- a/rust/lance/src/datafusion/dataframe.rs +++ b/rust/lance/src/datafusion/dataframe.rs @@ -1,10 +1,7 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright The Lance Authors -use std::{ - any::Any, - sync::{Arc, Mutex}, -}; +use std::sync::{Arc, Mutex}; use arrow_schema::{Schema, SchemaRef}; use async_trait::async_trait; @@ -42,6 +39,7 @@ pub struct LanceTableProvider { row_id_idx: Option, row_addr_idx: Option, ordered: bool, + blob_handling: Option, } impl LanceTableProvider { @@ -72,7 +70,29 @@ impl LanceTableProvider { row_id_idx, row_addr_idx, ordered, + blob_handling: None, + } + } + + /// Overrides how blob columns are read during [`TableProvider::scan`]. + /// When unset, the underlying dataset scan uses its default + /// [`BlobHandling`](lance_core::datatypes::BlobHandling) policy. + pub fn with_blob_handling(mut self, handling: lance_core::datatypes::BlobHandling) -> Self { + let converted = self + .dataset + .full_projection() + .with_blob_handling(handling.clone()) + .to_bare_schema(); + let mut full_schema = Schema::from(&converted); + if self.row_id_idx.is_some() { + full_schema = full_schema.try_with_column(ROW_ID_FIELD.clone()).unwrap(); + } + if self.row_addr_idx.is_some() { + full_schema = full_schema.try_with_column(ROW_ADDR_FIELD.clone()).unwrap(); } + self.full_schema = Arc::new(full_schema); + self.blob_handling = Some(handling); + self } pub fn dataset(&self) -> Arc { @@ -82,10 +102,6 @@ impl LanceTableProvider { #[async_trait] impl TableProvider for LanceTableProvider { - fn as_any(&self) -> &dyn Any { - self - } - fn schema(&self) -> SchemaRef { self.full_schema.clone() } @@ -102,6 +118,10 @@ impl TableProvider for LanceTableProvider { limit: Option, ) -> datafusion::common::Result> { let mut scan = self.dataset.scan(); + if let Some(handling) = self.blob_handling.clone() { + scan.blob_handling(handling); + } + match projection { Some(projection) if projection.is_empty() => { scan.empty_project()?; @@ -166,13 +186,7 @@ pub trait SessionContextExt { with_row_id: bool, with_row_addr: bool, ) -> datafusion::common::Result; - /// Creates a DataFrame for reading a Lance dataset without ordering - fn read_lance_unordered( - &self, - dataset: Arc, - with_row_id: bool, - with_row_addr: bool, - ) -> datafusion::common::Result; + /// Creates a DataFrame for reading a stream of data /// /// This dataframe may only be queried once, future queries will fail @@ -232,20 +246,6 @@ impl SessionContextExt for SessionContext { ))) } - fn read_lance_unordered( - &self, - dataset: Arc, - with_row_id: bool, - with_row_addr: bool, - ) -> datafusion::common::Result { - self.read_table(Arc::new(LanceTableProvider::new_with_ordering( - dataset, - with_row_id, - with_row_addr, - false, - ))) - } - fn read_one_shot( &self, data: SendableRecordBatchStream, diff --git a/rust/lance/src/datafusion/logical_plan.rs b/rust/lance/src/datafusion/logical_plan.rs index a9fe0ed7750..039aa75864f 100644 --- a/rust/lance/src/datafusion/logical_plan.rs +++ b/rust/lance/src/datafusion/logical_plan.rs @@ -1,7 +1,7 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright The Lance Authors -use std::{any::Any, borrow::Cow, sync::Arc}; +use std::{borrow::Cow, sync::Arc}; use arrow_schema::Schema as ArrowSchema; use async_trait::async_trait; @@ -19,10 +19,6 @@ use crate::Dataset; #[async_trait] impl TableProvider for Dataset { - fn as_any(&self) -> &dyn Any { - self - } - fn schema(&self) -> Arc { Arc::new(self.schema().into()) } @@ -167,17 +163,11 @@ mod tests { // DataFusion will create a cooperative execution plan, so we need to get its inner plan let physical_plan = physical_plan - .as_any() .downcast_ref::() .unwrap() .children()[0]; - assert!( - physical_plan - .as_any() - .downcast_ref::() - .is_some() - ); + assert!(physical_plan.downcast_ref::().is_some()); let expected_fields = schema .fields() diff --git a/rust/lance/src/dataset.rs b/rust/lance/src/dataset.rs index 470c6873dc7..630e769afd3 100644 --- a/rust/lance/src/dataset.rs +++ b/rust/lance/src/dataset.rs @@ -5,7 +5,6 @@ //! use arrow_array::{RecordBatch, RecordBatchReader}; -use arrow_schema::DataType; use byteorder::{ByteOrder, LittleEndian}; use chrono::{Duration, prelude::*}; use futures::future::BoxFuture; @@ -27,9 +26,8 @@ use lance_core::utils::tracing::{ DATASET_DELETING_EVENT, DATASET_DROPPING_COLUMN_EVENT, TRACE_DATASET_EVENTS, }; use lance_datafusion::projection::ProjectionPlan; -use lance_file::datatypes::populate_schema_dictionary; use lance_file::reader::{FileReader, FileReaderOptions}; -use lance_file::version::LanceFileVersion; +use lance_file::versions as file_versions; use lance_index::{IndexType, progress::IndexBuildProgress}; use lance_io::object_store::{ ChainedWrappingObjectStore, LanceNamespaceStorageOptionsProvider, ObjectStore, @@ -37,13 +35,14 @@ use lance_io::object_store::{ WrappingObjectStore, }; use lance_io::scheduler::{ScanScheduler, SchedulerConfig}; +use lance_io::traits::{WriteExt, Writer}; use lance_io::utils::{ CachedFileSize, read_last_block, read_message, read_metadata_offset, read_struct, }; use lance_namespace::LanceNamespace; use lance_table::format::{ DataFile, DataStorageFormat, DeletionFile, Fragment, IndexMetadata, MAGIC, Manifest, RowIdMeta, - pb, + pb, populate_manifest_schema_dictionaries, }; use lance_table::io::commit::{ CommitConfig, CommitError, CommitHandler, CommitLock, ManifestLocation, ManifestNamingScheme, @@ -60,7 +59,7 @@ use roaring::RoaringBitmap; use rowids::get_row_id_index; use serde::{Deserialize, Serialize}; use std::borrow::Cow; -use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet}; +use std::collections::{BTreeMap, HashMap, HashSet}; use std::fmt::Debug; use std::num::NonZero; use std::ops::Range; @@ -80,9 +79,10 @@ pub mod index; pub mod mem_wal; mod metadata; pub mod optimize; +pub(crate) mod overlay; pub mod progress; pub mod refs; -pub(crate) mod rowids; +pub mod rowids; pub mod scanner; mod schema_evolution; pub mod sql; @@ -92,6 +92,7 @@ pub mod transaction; pub mod udtf; pub mod updater; mod utils; +pub(crate) mod versions; pub mod write; pub(crate) use take::row_offsets_to_row_addresses; @@ -111,19 +112,24 @@ use crate::dataset::sql::SqlQueryBuilder; use crate::datatypes::Schema; use crate::index::retain_supported_indices; use crate::io::commit::{ - commit_detached_transaction, commit_new_dataset, commit_transaction, - detect_overlapping_fragments, + DEFAULT_COMMIT_RETRY_TIMEOUT, commit_detached_transaction, commit_new_dataset, + commit_transaction, detect_overlapping_fragments, }; use crate::session::Session; use crate::utils::temporal::{SystemTime, timestamp_to_nanos, utc_now}; use crate::{Error, Result}; -pub use blob::{BlobFile, ReadBlob, ReadBlobsBuilder, ReadBlobsStream}; +pub use blob::{ + BlobFile, BlobRangeRequest, BlobReadRange, ReadBlob, ReadBlobRange, ReadBlobRangesBuilder, + ReadBlobRangesStream, ReadBlobsBuilder, ReadBlobsStream, +}; use hash_joiner::HashJoiner; pub use lance_core::ROW_ID; use lance_core::box_error; use lance_index::scalar::lance_format::LanceIndexStore; use lance_namespace::models::{DeclareTableRequest, DescribeTableRequest}; -use lance_table::feature_flags::{apply_feature_flags, can_read_dataset}; +use lance_table::feature_flags::{ + apply_feature_flags, can_read_dataset, validate_mem_wal_index_catchup_flags, +}; use lance_table::io::deletion::{DELETIONS_DIR, relative_deletion_file_path}; pub use schema_evolution::{ BatchInfo, BatchUDF, ColumnAlteration, NewColumnTransform, UDFCheckpointStore, @@ -131,8 +137,8 @@ pub use schema_evolution::{ pub use take::TakeBuilder; use uuid::Uuid; pub use write::merge_insert::{ - MergeInsertBuilder, MergeInsertJob, MergeStats, UncommittedMergeInsert, WhenMatched, - WhenNotMatched, WhenNotMatchedBySource, + MergeInsertBuilder, MergeInsertJob, MergeStats, SourceDedupeBehavior, UncommittedMergeInsert, + WhenMatched, WhenNotMatched, WhenNotMatchedBySource, }; use crate::dataset::index::LanceIndexStoreExt; @@ -229,6 +235,23 @@ impl From<&Manifest> for Version { } } +/// The transaction that produced a version of the dataset, along with the +/// version's commit timestamp. +/// +/// Returned by [`Dataset::read_version_transaction`], which reads this +/// information directly from storage without checking out the version. +#[derive(Debug, Clone)] +pub struct VersionTransaction { + /// Version number. + pub version: u64, + + /// Timestamp the version was committed, in UTC. + pub timestamp: DateTime, + + /// The transaction that produced this version, if one was recorded. + pub transaction: Option, +} + /// Customize read behavior of a dataset. #[derive(Clone, Debug)] pub struct ReadParams { @@ -527,7 +550,7 @@ impl Dataset { ) .with_object_store(Arc::new(self.object_store.as_ref().clone())) .with_commit_handler(self.commit_handler.clone()) - .with_storage_format(self.manifest.data_storage_format.lance_file_version()?); + .with_exact_storage_format(self.manifest.data_storage_format.lance_file_format()); let dataset = builder.execute(transaction).await?; // Create BranchContents after shallow_clone @@ -599,7 +622,7 @@ impl Dataset { return Ok(self.clone()); } - let manifest = Self::load_manifest( + let manifest = Self::get_manifest( self.object_store.as_ref(), &manifest_location, &new_location.uri, @@ -625,7 +648,7 @@ impl Dataset { self.object_store.clone(), new_location.path, new_location.uri, - Arc::new(manifest), + manifest, manifest_location, self.session.clone(), self.commit_handler.clone(), @@ -695,6 +718,8 @@ impl Dataset { read_struct(object_reader.as_ref(), offset).await }?; + validate_mem_wal_index_catchup_flags(&manifest)?; + if !can_read_dataset(manifest.reader_feature_flags) { let message = format!( "This dataset cannot be read by this version of Lance. \ @@ -723,6 +748,7 @@ impl Dataset { let ds_index_cache = session.index_cache.for_dataset(uri); let metadata_key = crate::session::index_caches::IndexMetadataKey { version: manifest_location.version, + store_identity: &object_store.store_prefix, }; ds_index_cache .insert_with_key(&metadata_key, Arc::new(indices)) @@ -750,9 +776,7 @@ impl Dataset { .await; } - if manifest.should_use_legacy_format() { - populate_schema_dictionary(&mut manifest.schema, object_reader.as_ref()).await?; - } + populate_manifest_schema_dictionaries(&mut manifest, object_reader.as_ref()).await?; Ok(manifest) } @@ -765,6 +789,11 @@ impl Dataset { uri: &str, session: &Session, ) -> Result> { + if manifest_location.size.is_none() { + return Ok(Arc::new( + Self::load_manifest(object_store, manifest_location, uri, session).await?, + )); + } let metadata_cache = session.metadata_cache.for_dataset(uri); let manifest_key = ManifestKey { version: manifest_location.version, @@ -1123,15 +1152,6 @@ impl Dataset { delta::DatasetDeltaBuilder::new(self.clone()) } - // TODO: Cache this - pub(crate) fn is_legacy_storage(&self) -> bool { - self.manifest - .data_storage_format - .lance_file_version() - .unwrap() - == LanceFileVersion::Legacy - } - pub async fn latest_manifest(&self) -> Result<(Arc, ManifestLocation)> { let location = self .commit_handler @@ -1152,7 +1172,7 @@ impl Dataset { return Ok((self.manifest.clone(), self.manifest_location.clone())); } let mut manifest = read_manifest(&self.object_store, &location.path, location.size).await?; - if manifest.schema.has_dictionary_types() && manifest.should_use_legacy_format() { + if manifest.schema.has_dictionary_types() { let reader = if let Some(size) = location.size { self.object_store .open_with_size(&location.path, size as usize) @@ -1160,7 +1180,7 @@ impl Dataset { } else { self.object_store.open(&location.path).await? }; - populate_schema_dictionary(&mut manifest.schema, reader.as_ref()).await?; + populate_manifest_schema_dictionaries(&mut manifest, reader.as_ref()).await?; } let manifest_arc = Arc::new(manifest); self.metadata_cache @@ -1181,43 +1201,146 @@ impl Dataset { return Ok(Some((*transaction).clone())); } + let transaction = self + .read_transaction_from_storage(&self.manifest, &self.manifest_location) + .await?; + + if let Some(tx) = transaction.as_ref() { + self.metadata_cache + .insert_with_key(&transaction_key, Arc::new(tx.clone())) + .await; + } + Ok(transaction) + } + + /// Read the transaction recorded by `manifest` directly from storage, + /// without consulting or populating any session cache. + async fn read_transaction_from_storage( + &self, + manifest: &Manifest, + manifest_location: &ManifestLocation, + ) -> Result> { // Prefer inline transaction from manifest when available - let transaction = if let Some(pos) = self.manifest.transaction_section { - let reader = if let Some(size) = self.manifest_location.size { - self.object_store - .open_with_size(&self.manifest_location.path, size as usize) - .await? - } else { - self.object_store.open(&self.manifest_location.path).await? + if let Some(pos) = manifest.transaction_section { + let reader = match manifest_location.size { + Some(size) => { + self.object_store + .open_with_size(&manifest_location.path, size as usize) + .await? + } + None => self.object_store.open(&manifest_location.path).await?, }; - let tx: pb::Transaction = read_message(reader.as_ref(), pos).await?; - Transaction::try_from(tx).map(Some)? - } else if let Some(path) = &self.manifest.transaction_file { + // A concurrent overwrite can leave the listed size too small; retry + // once with the true size. + let tx: pb::Transaction = match read_message(reader.as_ref(), pos).await { + Err(e) + if manifest_location.size.is_some() + && e.to_string().contains("file size is too small") => + { + let reader = self.object_store.open(&manifest_location.path).await?; + read_message(reader.as_ref(), pos).await? + } + other => other?, + }; + Transaction::try_from(tx).map(Some) + } else if let Some(path) = &manifest.transaction_file { // Fallback: read external transaction file if present let path = self.transactions_dir().join(path.as_str()); let data = self.object_store.inner.get(&path).await?.bytes().await?; let transaction = lance_table::format::pb::Transaction::decode(data)?; - Transaction::try_from(transaction).map(Some)? + Transaction::try_from(transaction).map(Some) } else { - None - }; + Ok(None) + } + } - if let Some(tx) = transaction.as_ref() { - self.metadata_cache - .insert_with_key(&transaction_key, Arc::new(tx.clone())) - .await; + /// Read the transaction (if any) and commit timestamp of a version of the + /// dataset. `version` is a version number on this dataset's current branch. + /// + /// Reads the version's manifest transiently: no historical `Dataset` is + /// constructed, no `IndexSection` is decoded, and no session cache is read + /// or written, so scanning many historical versions does not fill the + /// shared caches. + /// + /// Returns an error if the version does not exist (for example, if it has + /// been cleaned up). + /// + /// # Example + /// + /// ``` + /// # use lance::{Dataset, Result}; + /// # async fn example(dataset: &Dataset) -> Result<()> { + /// let record = dataset.read_version_transaction(5).await?; + /// let committed_at = record.timestamp; + /// let operation = record.transaction.as_ref().map(|t| t.operation.name()); + /// # Ok(()) + /// # } + /// ``` + pub async fn read_version_transaction(&self, version: u64) -> Result { + // Resolve against this dataset's current branch. + let manifest_location = self + .commit_handler + .resolve_version_location(&self.base, version, &self.object_store.inner) + .await?; + + // Keep the DatasetNotFound variant callers expect for a missing version. + let manifest = read_manifest( + &self.object_store, + &manifest_location.path, + manifest_location.size, + ) + .await + .map_err(|e| match &e { + Error::NotFound { uri, .. } => Error::dataset_not_found(uri.clone(), box_error(e)), + _ => e, + })?; + + // The resolved manifest must belong to this dataset's branch. A + // mismatch means the commit handler resolved against a different chain + // (for example an external manifest store that ignores + // branch-qualified paths); error loudly rather than hand back another + // branch's transaction. + if manifest.branch != self.manifest.branch { + return Err(Error::internal(format!( + "reading version {} on branch '{}' resolved a manifest belonging to branch '{}'", + version, + refs::normalize_branch(self.manifest.branch.as_deref()), + refs::normalize_branch(manifest.branch.as_deref()), + ))); } - Ok(transaction) + + let transaction = self + .read_transaction_from_storage(&manifest, &manifest_location) + .await?; + + Ok(VersionTransaction { + version: manifest.version, + timestamp: manifest.timestamp(), + transaction, + }) } /// Read the transaction file for this version of the dataset. /// /// If there was no transaction file written for this version of the dataset /// then this will return None. + /// + /// Does not populate the session caches; see + /// [`Self::read_version_transaction`]. + /// + /// # Example + /// + /// ``` + /// # use lance::{Dataset, Result}; + /// # async fn example(dataset: &Dataset) -> Result<()> { + /// let transaction = dataset.read_transaction_by_version(5).await?; + /// let operation = transaction.as_ref().map(|t| t.operation.name()); + /// # Ok(()) + /// # } + /// ``` pub async fn read_transaction_by_version(&self, version: u64) -> Result> { - let dataset_version = self.checkout_version(version).await?; - dataset_version.read_transaction().await + Ok(self.read_version_transaction(version).await?.transaction) } /// List transactions for the dataset, up to a maximum number. @@ -1489,6 +1612,7 @@ impl Dataset { &transaction, write_config, commit_config, + DEFAULT_COMMIT_RETRY_TIMEOUT, self.manifest_location.naming_scheme, None, ) @@ -1604,11 +1728,30 @@ impl Dataset { } /// Take [BlobFile] by row IDs. + /// + /// The returned vector has one element per row ID. Null blob values are + /// represented as `None`; valid empty blobs return a `BlobFile` with size + /// zero. + /// + /// ``` + /// # use std::sync::Arc; + /// # use lance::dataset::Dataset; + /// # use lance::Result; + /// # async fn example(dataset: Arc) -> Result<()> { + /// let blobs = dataset.take_blobs(&[42], "images").await?; + /// match &blobs[0] { + /// None => { /* The selected blob is null. */ } + /// Some(blob) if blob.size() == 0 => { /* The selected blob is valid but empty. */ } + /// Some(blob) => { let _size = blob.size(); } + /// } + /// # Ok(()) + /// # } + /// ``` pub async fn take_blobs( self: &Arc, row_ids: &[u64], column: impl AsRef, - ) -> Result> { + ) -> Result>> { blob::take_blobs(self, row_ids, column.as_ref()).await } @@ -1618,21 +1761,57 @@ impl Dataset { /// Use this method when you already have row addresses, for example from /// a scan with `with_row_address()`. For row IDs (stable identifiers), use /// [`Self::take_blobs`]. For row indices (offsets), use - /// [`Self::take_blobs_by_indices`]. + /// [`Self::take_blobs_by_indices`]. The result has the same null and empty + /// blob representation as [`Self::take_blobs`]. + /// + /// ``` + /// # use std::sync::Arc; + /// # use lance::dataset::Dataset; + /// # use lance::Result; + /// # async fn example(dataset: Arc, row_address: u64) -> Result<()> { + /// let blobs = dataset + /// .take_blobs_by_addresses(&[row_address], "images") + /// .await?; + /// match &blobs[0] { + /// None => { /* The selected blob is null. */ } + /// Some(blob) if blob.size() == 0 => { /* The selected blob is valid but empty. */ } + /// Some(blob) => { let _size = blob.size(); } + /// } + /// # Ok(()) + /// # } + /// ``` pub async fn take_blobs_by_addresses( self: &Arc, row_addrs: &[u64], column: impl AsRef, - ) -> Result> { + ) -> Result>> { blob::take_blobs_by_addresses(self, row_addrs, column.as_ref()).await } /// Take [BlobFile] by row indices (offsets in the dataset). + /// + /// The result has the same null and empty blob representation as + /// [`Self::take_blobs`]. + /// + /// ``` + /// # use std::sync::Arc; + /// # use lance::dataset::Dataset; + /// # use lance::Result; + /// # async fn example(dataset: Arc) -> Result<()> { + /// let blobs = dataset.take_blobs_by_indices(&[0], "images").await?; + /// match &blobs[0] { + /// None => { /* The selected blob is null. */ } + /// Some(blob) if blob.size() == 0 => { /* The selected blob is valid but empty. */ } + /// Some(blob) => { let _size = blob.size(); } + /// } + /// # Ok(()) + /// # } + /// ``` pub async fn take_blobs_by_indices( self: &Arc, row_indices: &[u64], column: impl AsRef, - ) -> Result> { + ) -> Result>> { let fragments = self.get_fragments(); let row_addrs = row_offsets_to_row_addresses(&fragments, row_indices).await?; blob::take_blobs_by_addresses(self, &row_addrs, column.as_ref()).await @@ -1643,7 +1822,9 @@ impl Dataset { /// This API complements [`Self::take_blobs`]. `take_blobs` returns /// [`BlobFile`] handles for caller-driven random access, while /// `read_blobs` builds a streaming read plan for sequential or batched blob - /// retrieval. + /// retrieval. Every selected row produces one result: null blob values have + /// `ReadBlob::data` set to `None`, while valid empty blobs contain an empty + /// buffer. /// /// ```rust /// # use std::sync::Arc; @@ -1670,6 +1851,38 @@ impl Dataset { )) } + /// Create a planned reader for row-specific blob-local byte ranges. + /// + /// Each [`BlobRangeRequest`] contains both its row selector and byte range, + /// so requests can be repeated or reordered without coordinating parallel + /// selector and range lists. Every request produces one result. A null blob + /// has `ReadBlobRange::data` set to `None`; an empty range on a non-null blob + /// contains an empty buffer. + /// + /// ```rust + /// # use std::sync::Arc; + /// # use lance::dataset::{BlobRangeRequest, Dataset}; + /// # use lance::Result; + /// # async fn example(dataset: Arc) -> Result<()> { + /// let ranges = dataset + /// .read_blob_ranges("images")? + /// .with_row_indices([ + /// BlobRangeRequest::new(7, 0, 1024), + /// BlobRangeRequest::new(7, 4096, 1024), + /// ]) + /// .execute() + /// .await?; + /// # let _ = ranges; + /// # Ok(()) + /// # } + /// ``` + pub fn read_blob_ranges( + self: &Arc, + column: impl AsRef, + ) -> Result { + Ok(ReadBlobRangesBuilder::new(self.read_blobs(column)?)) + } + /// Get a stream of batches based on iterator of ranges of row numbers. /// /// This is an experimental API. It may change at any time. @@ -1711,26 +1924,7 @@ impl Dataset { )); } - let selected_fragment_ids = fragment_ids.iter().copied().collect::>(); - let selected_fragments = self - .get_fragments() - .into_iter() - .filter(|fragment| selected_fragment_ids.contains(&(fragment.id() as u32))) - .collect::>(); - - if selected_fragments.len() != selected_fragment_ids.len() { - let present_fragment_ids = selected_fragments - .iter() - .map(|fragment| fragment.id() as u32) - .collect::>(); - let missing_fragment_ids = selected_fragment_ids - .into_iter() - .filter(|fragment_id| !present_fragment_ids.contains(fragment_id)) - .collect::>(); - return Err(Error::invalid_input(format!( - "Dataset::sample received fragment ids that are not part of the current dataset version: {missing_fragment_ids:?}", - ))); - } + let selected_fragments = self.get_fragments_from_ids(fragment_ids)?; let num_rows = stream::iter(selected_fragments.iter().cloned()) .map(|fragment| async move { fragment.count_rows(None).await }) @@ -2029,12 +2223,7 @@ impl Dataset { .await?; let file_metadata = FileReader::read_all_metadata(&file).await?; - let file_version = LanceFileVersion::try_from_major_minor( - file_metadata.major_version as u32, - file_metadata.minor_version as u32, - )?; - - let is_structural = file_version >= LanceFileVersion::V2_1; + let lance_file_format = file_metadata.version; let physical_columns = file_metadata.column_metadatas.len(); let has_footer_orphans = file_metadata.file_schema.fields.len() > physical_columns; let dataset_schema = self.schema(); @@ -2042,29 +2231,6 @@ impl Dataset { let mut column_names = Vec::new(); let mut consumed_top_level_fields = 0usize; - fn physical_column_count( - field: &lance_core::datatypes::Field, - is_structural: bool, - ) -> usize { - if !is_structural { - return 1 + field - .children - .iter() - .map(|child| physical_column_count(child, is_structural)) - .sum::(); - } - - if field.children.is_empty() || field.is_blob() || field.is_packed_struct() { - 1 - } else { - field - .children - .iter() - .map(|child| physical_column_count(child, is_structural)) - .sum() - } - } - fn field_contains_blob(field: &lance_core::datatypes::Field) -> bool { field.is_blob() || field.children.iter().any(field_contains_blob) } @@ -2072,15 +2238,15 @@ impl Dataset { fn field_names_match( fields: &[lance_core::datatypes::Field], start: usize, - names: &[&str], + expected: &arrow_schema::Fields, ) -> bool { fields - .get(start..start + names.len()) + .get(start..start + expected.len()) .is_some_and(|candidate| { candidate .iter() - .zip(names) - .all(|(field, name)| field.name == *name) + .zip(expected.iter()) + .all(|(field, expected)| field.name == expected.name().as_str()) }) } @@ -2088,51 +2254,15 @@ impl Dataset { fields: &[lance_core::datatypes::Field], start: usize, ) -> usize { - const BLOB_V2_DESCRIPTOR_FIELDS: &[&str] = - &["kind", "position", "size", "blob_id", "blob_uri"]; - const BLOB_V1_DESCRIPTOR_FIELDS: &[&str] = &["position", "size"]; - - if field_names_match(fields, start, BLOB_V2_DESCRIPTOR_FIELDS) { - BLOB_V2_DESCRIPTOR_FIELDS.len() - } else if field_names_match(fields, start, BLOB_V1_DESCRIPTOR_FIELDS) { - BLOB_V1_DESCRIPTOR_FIELDS.len() + if field_names_match(fields, start, &lance_core::datatypes::BLOB_V2_DESC_FIELDS) { + lance_core::datatypes::BLOB_V2_DESC_FIELDS.len() + } else if field_names_match(fields, start, &lance_core::datatypes::BLOB_DESC_FIELDS) { + lance_core::datatypes::BLOB_DESC_FIELDS.len() } else { 0 } } - fn collect_columns( - field: &lance_core::datatypes::Field, - is_structural: bool, - fields: &mut Vec, - column_indices: &mut Vec, - curr_column_idx: &mut i32, - ) { - let contributes = !is_structural - || field.children.is_empty() - || field.is_blob() - || field.is_packed_struct(); - let recurse = !is_structural || (!field.is_blob() && !field.is_packed_struct()); - - if contributes { - fields.push(field.id); - column_indices.push(*curr_column_idx); - *curr_column_idx += 1; - } - - if recurse { - for child in &field.children { - collect_columns( - child, - is_structural, - fields, - column_indices, - curr_column_idx, - ); - } - } - } - fn validate_file_field_matches_dataset( dataset_field: &lance_core::datatypes::Field, file_field: &lance_core::datatypes::Field, @@ -2186,7 +2316,7 @@ impl Dataset { }; validate_file_field_matches_dataset(dataset_field, field, &field.name)?; - represented_columns += physical_column_count(field, is_structural); + represented_columns += file_versions::physical_column_count(lance_file_format, field); column_names.push(field.name.as_str()); consumed_top_level_fields = idx + 1; idx += 1; @@ -2219,23 +2349,17 @@ impl Dataset { let projected_ds_schema = self.schema().project(&column_names)?; - let mut fields = Vec::new(); - let mut column_indices = Vec::new(); - let mut curr_column_idx: i32 = 0; - for field in &projected_ds_schema.fields { - collect_columns( - field, - is_structural, - &mut fields, - &mut column_indices, - &mut curr_column_idx, - ); - } + let (fields, column_indices) = + file_versions::data_file_columns(lance_file_format, &projected_ds_schema); + let represented_dataset_columns = column_indices + .iter() + .filter(|column_index| **column_index >= 0) + .count(); - if curr_column_idx as usize != physical_columns { + if represented_dataset_columns != physical_columns { return Err(Error::invalid_input(format!( "Schema mismatch: dataset projection maps to {} physical columns but file has {} columns", - curr_column_idx, physical_columns + represented_dataset_columns, physical_columns ))); } @@ -2250,8 +2374,7 @@ impl Dataset { path, fields, column_indices, - file_metadata.major_version as u32, - file_metadata.minor_version as u32, + lance_file_format, file_size_nz, base_id, )) @@ -2304,6 +2427,13 @@ impl Dataset { } } + /// The `ObjectStoreParams` this dataset was opened with, or `None` when + /// opened without explicit params. Lets a caller re-open a derived path + /// (e.g. a MemWAL SSTable) with the same store this dataset used. + pub fn store_params(&self) -> Option<&ObjectStoreParams> { + self.store_params.as_deref() + } + pub(crate) async fn object_store_for_data_file( &self, data_file: &DataFile, @@ -2426,6 +2556,17 @@ impl Dataset { Ok(versions) } + /// Get the number of versions in the current version history. + /// + /// Unlike [`Self::versions`], this only enumerates manifest locations and does not read or + /// deserialize every manifest. + pub async fn count_versions(&self) -> Result { + self.commit_handler + .list_manifest_locations(&self.base, &self.object_store, false) + .try_fold(0_u64, |count, _| async move { Ok(count + 1) }) + .await + } + /// List all detached manifest locations. /// /// Detached manifests are versions that are not part of the main version history. @@ -2540,41 +2681,121 @@ impl Dataset { &self.manifest.fragments } - // Gets a filtered list of fragments from ids in O(N) time instead of using - // `get_fragment` which would require O(N^2) time. - pub fn get_frags_from_ordered_ids(&self, ordered_ids: &[u32]) -> Vec> { - let mut fragments = Vec::with_capacity(ordered_ids.len()); - let mut id_iter = ordered_ids.iter(); - let mut id = id_iter.next(); - // This field is just used to assert the ids are in order - let mut last_id: i64 = -1; - for frag in self.manifest.fragments.iter() { - let mut the_id = if let Some(id) = id { *id } else { break }; - // Assert the given ids are, in fact, in order - assert!(the_id as i64 > last_id); - // For any IDs we've passed we can assume that no fragment exists any longer - // with that ID. - while the_id < frag.id as u32 { - fragments.push(None); - last_id = the_id as i64; - id = id_iter.next(); - the_id = if let Some(id) = id { *id } else { break }; - } + pub(crate) fn normalize_fragment_ids(fragment_ids: &[u32]) -> Vec { + let mut ids = fragment_ids.to_vec(); + ids.sort_unstable(); + ids.dedup(); + ids + } - if the_id == frag.id as u32 { - fragments.push(Some(FileFragment::new( - Arc::new(self.clone()), - frag.clone(), - ))); - last_id = the_id as i64; - id = id_iter.next(); - } + pub(crate) fn get_fragments_from_ids(&self, fragment_ids: &[u32]) -> Result> { + let ordered_ids = Self::normalize_fragment_ids(fragment_ids); + let fragments = self.get_frags_from_ordered_ids(&ordered_ids); + if let Some(missing_id) = fragments + .iter() + .zip(ordered_ids.iter()) + .find_map(|(fragment, fragment_id)| fragment.is_none().then_some(*fragment_id)) + { + return Err(Error::invalid_input(format!( + "Unknown fragment id {missing_id} in fragment filter; not part of the current dataset version" + ))); } - fragments + + Ok(fragments.into_iter().flatten().collect()) + } + + pub(crate) fn get_existing_fragments_from_ids( + &self, + fragment_ids: &[u32], + ) -> Vec { + let ordered_ids = Self::normalize_fragment_ids(fragment_ids); + self.get_frags_from_ordered_ids(&ordered_ids) + .into_iter() + .flatten() + .collect() + } + + pub(crate) fn get_fragment_metadata_from_ids( + &self, + fragment_ids: &[u32], + ) -> Result> { + Ok(self + .get_fragments_from_ids(fragment_ids)? + .into_iter() + .map(|fragment| fragment.metadata().clone()) + .collect()) + } + + pub(crate) fn get_existing_fragment_metadata_from_ids( + &self, + fragment_ids: &[u32], + ) -> Vec { + self.get_existing_fragments_from_ids(fragment_ids) + .into_iter() + .map(|fragment| fragment.metadata().clone()) + .collect() + } + + pub(crate) async fn count_rows_in_fragments(&self, fragment_ids: &[u32]) -> Result { + let fragments = self.get_fragments_from_ids(fragment_ids)?; + self.count_rows_in_resolved_fragments(fragments).await + } + + pub(crate) async fn count_rows_in_existing_fragments( + &self, + fragment_ids: &[u32], + ) -> Result { + let fragments = self.get_existing_fragments_from_ids(fragment_ids); + self.count_rows_in_resolved_fragments(fragments).await + } + + async fn count_rows_in_resolved_fragments( + &self, + fragments: Vec, + ) -> Result { + let counts = stream::iter(fragments) + .map(|fragment| async move { fragment.count_rows(None).await }) + .buffer_unordered(16) + .try_collect::>() + .await?; + Ok(counts.iter().sum()) + } + + /// Resolves fragments for the given ids without scanning the manifest. + /// + /// The ids do not need to be sorted or deduplicated. Each id is resolved + /// independently via the fragment bitmap. + pub fn get_frags_from_ordered_ids(&self, ordered_ids: &[u32]) -> Vec> { + let dataset = Arc::new(self.clone()); + ordered_ids + .iter() + .map(|id| { + if !self.fragment_bitmap.contains(*id) { + return None; + } + let fragment_index = self.fragment_bitmap.rank(*id) as usize - 1; + let fragment = self.manifest.fragments.get(fragment_index)?; + debug_assert_eq!( + fragment.id, *id as u64, + "fragment_bitmap rank({id}) resolved to fragment {}, but fragment_bitmap and manifest.fragments are expected to stay in sync", + fragment.id + ); + Some(FileFragment::new(dataset.clone(), fragment.clone())) + }) + .collect() } // This method filters deleted items from `addr_or_ids` using `addrs` as a reference async fn filter_addr_or_ids(&self, addr_or_ids: &[u64], addrs: &[u64]) -> Result> { + // The final zip pairs these positionally; misalignment must fail + // loud rather than truncate. + if addr_or_ids.len() != addrs.len() { + return Err(Error::internal(format!( + "filter_addr_or_ids: addr_or_ids has {} entries but addrs has {}", + addr_or_ids.len(), + addrs.len() + ))); + } if addrs.is_empty() { return Ok(Vec::new()); } @@ -2686,17 +2907,24 @@ impl Dataset { } pub(crate) async fn filter_deleted_ids(&self, ids: &[u64]) -> Result> { - let addresses = if let Some(row_id_index) = get_row_id_index(self).await? { - let addresses = ids - .iter() - .filter_map(|id| row_id_index.get(*id).map(|address| address.into())) - .collect::>(); - Cow::Owned(addresses) + let (ids, addresses) = if let Some(row_id_index) = get_row_id_index(self).await? { + // Ids absent from the deletion-aware index are deleted; drop + // them from both lists to keep the zip aligned. ids.len() is an + // upper bound on the output size, so allocate once up front. + let mut live_ids = Vec::with_capacity(ids.len()); + let mut addresses = Vec::with_capacity(ids.len()); + for id in ids { + if let Some(address) = row_id_index.get(*id) { + live_ids.push(*id); + addresses.push(u64::from(address)); + } + } + (Cow::Owned(live_ids), Cow::Owned(addresses)) } else { - Cow::Borrowed(ids) + (Cow::Borrowed(ids), Cow::Borrowed(ids)) }; - self.filter_addr_or_ids(ids, &addresses).await + self.filter_addr_or_ids(&ids, &addresses).await } /// Gets the number of files that are so small they don't even have a full @@ -2758,6 +2986,8 @@ impl Dataset { .try_collect::>() .await?; + rowids::validate_stable_row_ids(self).await?; + // Validate indices let indices = self.load_indices().await?; self.validate_indices(&indices)?; @@ -2774,7 +3004,7 @@ impl Dataset { self.manifest_location.path.clone(), format!( "Duplicate index id {} found in dataset {:?}", - &index.uuid, self.base + index.uuid, self.base ), )); } @@ -2867,19 +3097,29 @@ impl Dataset { ) .with_object_store(Arc::new(self.object_store.as_ref().clone())) .with_commit_handler(self.commit_handler.clone()) - .with_storage_format(self.manifest.data_storage_format.lance_file_version()?); + .with_exact_storage_format(self.manifest.data_storage_format.lance_file_format()); builder.execute(transaction).await } /// Deep clone the target version into a new dataset at target_path. - /// This performs a server-side copy of all relevant dataset files (data files, - /// deletion files, and any external row-id files) into the target dataset - /// without loading data into memory. + /// This copies all relevant dataset files (data files, deletion files, and + /// index files) into the target dataset without loading data into memory. + /// + /// The source files are read through this dataset's own object store while the + /// copies are written through the target object store built from `store_params`. + /// This makes the clone work across accounts/stores (e.g. between two abfss + /// accounts): when the source and target stores are the same the copy stays + /// server-side, otherwise the data is streamed through this process. /// /// Parameters: /// - `target_path`: the URI string to clone the dataset into. /// - `version`: the version cloned from, could be a version number, branch head, or tag. - /// - `store_params`: the object store params to use for the new dataset. + /// - `store_params`: the object store params for the target dataset (e.g. the + /// credentials of the target account). + /// + /// Note: external `base_paths` referenced by the source manifest are read through + /// this dataset's object store; per-base distinct source credentials are not yet + /// supported (see ). pub async fn deep_clone( &mut self, target_path: &str, @@ -2920,7 +3160,12 @@ impl Dataset { path }; - // TODO: Leverage object store bulk copy for efficient deep_clone + // When the source and target live in the same store we can keep the copy + // server-side. Otherwise (e.g. cloning across accounts) we stream each file + // from the source store to the target store. + let same_store = src_ds.object_store.store_prefix == target_store.store_prefix; + + // TODO: Leverage object store bulk copy for efficient same-store deep_clone. // // All cloud storage providers support batch copy APIs that would provide significant // performance improvements. We use single file copy before we have upstream support. @@ -2930,10 +3175,21 @@ impl Dataset { let copy_futures = src_paths .iter() .map(|(relative_path, base)| { - let store = Arc::clone(&target_store); + let source_store = Arc::clone(&src_ds.object_store); + let target_store = Arc::clone(&target_store); let src_path = build_absolute_path(relative_path, base); let target_path = build_absolute_path(relative_path, &target_base); - async move { store.copy(&src_path, &target_path).await.map(|_| ()) } + async move { + if same_store { + target_store.copy(&src_path, &target_path).await?; + } else { + let reader = source_store.open(&src_path).await?; + let mut writer = target_store.create(&target_path).await?; + writer.copy_from_reader(reader.as_ref()).await?; + writer.shutdown().await?; + } + Result::Ok(()) + } }) .collect::>(); @@ -2958,8 +3214,9 @@ impl Dataset { let builder = CommitBuilder::new(WriteDestination::Uri(target_path)) .with_store_params(store_params.clone().unwrap_or_default()) .with_object_store(target_store.clone()) + .with_source_store(src_ds.object_store.clone()) .with_commit_handler(self.commit_handler.clone()) - .with_storage_format(self.manifest.data_storage_format.lance_file_version()?); + .with_exact_storage_format(self.manifest.data_storage_format.lance_file_format()); let new_ds = builder.execute(txn).await?; Ok(new_ds) } @@ -2999,7 +3256,7 @@ impl Dataset { external_file.path ))); } - for data_file in fragment.files.iter() { + for data_file in fragment.referenced_lance_files() { let base_root = if let Some(base_id) = data_file.base_id { let base_path = self.manifest.base_paths.get(&base_id).ok_or_else(|| { @@ -3072,29 +3329,6 @@ impl Dataset { pub fn sql(&self, sql: &str) -> SqlQueryBuilder { SqlQueryBuilder::new(self.clone(), sql) } - - /// Returns true if Lance supports writing this datatype with nulls. - pub(crate) fn lance_supports_nulls(&self, datatype: &DataType) -> bool { - match self - .manifest() - .data_storage_format - .lance_file_version() - .unwrap_or(LanceFileVersion::Legacy) - .resolve() - { - LanceFileVersion::Legacy => matches!( - datatype, - DataType::Utf8 - | DataType::LargeUtf8 - | DataType::Binary - | DataType::List(_) - | DataType::FixedSizeBinary(_) - | DataType::FixedSizeList(_, _) - ), - LanceFileVersion::V2_0 => !matches!(datatype, DataType::Struct(..)), - _ => true, - } - } } pub(crate) struct NewTransactionResult<'a> { @@ -3319,11 +3553,14 @@ impl Dataset { .try_collect::>() .await?; + let preserves_nullability = + !schema_evolution::merge_introduces_required_field(self.schema(), &new_schema); let transaction = Transaction::new( self.manifest.version, Operation::Merge { fragments: updated_fragments, schema: new_schema, + preserves_nullability, }, None, ); @@ -3655,6 +3892,12 @@ impl ManifestWriteConfig { pub fn disable_transaction_file(&self) -> bool { self.disable_transaction_file } + + #[cfg(test)] + pub(crate) fn with_transaction_file_disabled(mut self) -> Self { + self.disable_transaction_file = true; + self + } } /// Commit a manifest file and create a copy at the latest manifest path. @@ -3667,7 +3910,7 @@ pub(crate) async fn write_manifest_file( indices: Option>, config: &ManifestWriteConfig, naming_scheme: ManifestNamingScheme, - mut transaction: Option<&Transaction>, + transaction: Option, ) -> std::result::Result { if config.auto_set_feature_flags { // build_manifest may have already set FLAG_STABLE_ROW_IDS on the manifest. @@ -3693,7 +3936,7 @@ pub(crate) async fn write_manifest_file( object_store, write_manifest_file_to_path, naming_scheme, - transaction.take().map(|tx| tx.into()), + transaction, ) .await } diff --git a/rust/lance/src/dataset/blob.rs b/rust/lance/src/dataset/blob.rs index 84c1b8bdcad..381c993747c 100644 --- a/rust/lance/src/dataset/blob.rs +++ b/rust/lance/src/dataset/blob.rs @@ -2,26 +2,28 @@ // SPDX-FileCopyrightText: Copyright The Lance Authors use std::{ - collections::{BTreeMap, HashMap, VecDeque}, + collections::{BTreeMap, HashMap, HashSet}, future::Future, ops::{DerefMut, Range}, panic::AssertUnwindSafe, sync::Arc, - task::Poll, }; use arrow::array::AsArray; use arrow::datatypes::{UInt8Type, UInt32Type, UInt64Type}; -use arrow_array::RecordBatch; -use arrow_array::{Array, ArrayRef}; -use arrow_schema::{DataType as ArrowDataType, Field as ArrowField}; +use arrow_array::{ + Array, ArrayRef, GenericListArray, OffsetSizeTrait, RecordBatch, builder::LargeBinaryBuilder, +}; +use arrow_buffer::{ArrowNativeType, OffsetBuffer, ScalarBuffer}; +use arrow_schema::{DataType as ArrowDataType, Field as ArrowField, Schema as ArrowSchema}; use bytes::Bytes; use futures::future::BoxFuture; use futures::stream::BoxStream; use futures::{FutureExt, StreamExt, TryStreamExt, stream}; use lance_arrow::{ - BLOB_DEDICATED_SIZE_THRESHOLD_META_KEY, BLOB_INLINE_SIZE_THRESHOLD_META_KEY, - BLOB_PACK_FILE_SIZE_THRESHOLD_META_KEY, FieldExt, r#struct::StructArrayExt, + ARROW_EXT_NAME_KEY, BLOB_DEDICATED_SIZE_THRESHOLD_META_KEY, + BLOB_INLINE_SIZE_THRESHOLD_META_KEY, BLOB_PACK_FILE_SIZE_THRESHOLD_META_KEY, FieldExt, + list::ListArrayExt, r#struct::StructArrayExt, }; use lance_io::object_store::{ObjectStore, ObjectStoreParams, ObjectStoreRegistry}; use lance_io::scheduler::{FileScheduler, ScanScheduler, SchedulerConfig}; @@ -29,17 +31,20 @@ use object_store::path::Path; use tokio::sync::{Mutex, OnceCell, oneshot}; use url::Url; -use super::take::TakeBuilder; +use super::take::{MissingRowPolicy, TakeBuilder}; use super::write::ExternalBlobMode; use super::{Dataset, ProjectionRequest}; use crate::blob::{ BlobDescriptor, BlobDescriptorArrayBuilder, BlobIdAllocator, BlobRange, PackedBlobWriter, - is_logical_blob_v2_field, is_prepared_blob_v2_field, validate_prepared_blob_array, + blob_v2_layout, blob_v2_shape_error, validate_prepared_blob_array, }; use arrow_array::StructArray; -use lance_core::datatypes::{BlobKind, BlobVersion, parse_field_path}; +use lance_core::datatypes::{ + BLOB_DESC_FIELDS, BlobKind, BlobV2Layout, BlobVersion, Field as LanceField, Schema, + parse_field_path, +}; use lance_core::utils::blob::blob_path; -use lance_core::{Error, Result, utils::address::RowAddress}; +use lance_core::{Error, ROW_ADDR, Result, utils::address::RowAddress}; use lance_io::traits::Reader; use lance_io::utils::CachedFileSize; @@ -183,6 +188,158 @@ impl ExternalBaseResolver { } } +fn arrow_field_contains_blob_v2(field: &ArrowField) -> bool { + if field.is_blob_v2() { + return true; + } + match field.data_type() { + ArrowDataType::Struct(children) => children + .iter() + .any(|child| arrow_field_contains_blob_v2(child)), + ArrowDataType::List(child) | ArrowDataType::LargeList(child) => { + arrow_field_contains_blob_v2(child) + } + _ => false, + } +} + +fn collect_external_blob_uris( + field: &ArrowField, + array: &ArrayRef, + selected_rows: &[bool], + field_path: &str, + external_uris: &mut Vec<(String, String)>, +) -> Result<()> { + if !arrow_field_contains_blob_v2(field) { + return Ok(()); + } + if array.len() != selected_rows.len() { + return Err(Error::internal(format!( + "Blob field '{}' row count {} did not match selection length {}", + field_path, + array.len(), + selected_rows.len() + ))); + } + + if field.is_blob_v2() { + let struct_array = array.as_struct(); + if BlobV2Layout::classify(struct_array.fields()) != Some(BlobV2Layout::Logical) { + return Err(blob_v2_shape_error(field, &[BlobV2Layout::Logical])); + } + let uri_column = struct_array + .column_by_name("uri") + .ok_or_else(|| Error::invalid_input("Blob struct missing `uri` field"))? + .as_string::(); + for (row_idx, is_selected) in selected_rows.iter().copied().enumerate() { + if is_selected && struct_array.is_valid(row_idx) && uri_column.is_valid(row_idx) { + external_uris.push(( + field_path.to_string(), + uri_column.value(row_idx).to_string(), + )); + } + } + return Ok(()); + } + + match field.data_type() { + ArrowDataType::Struct(children) => { + let struct_array = array.as_struct(); + let child_selection = selected_rows + .iter() + .copied() + .enumerate() + .map(|(row_idx, is_selected)| is_selected && struct_array.is_valid(row_idx)) + .collect::>(); + for (child_field, child_array) in children.iter().zip(struct_array.columns()) { + let child_path = format!("{}.{}", field_path, child_field.name()); + collect_external_blob_uris( + child_field, + child_array, + &child_selection, + &child_path, + external_uris, + )?; + } + } + ArrowDataType::List(child) => { + let list_array = array.as_list::(); + let mut child_selection = vec![false; list_array.values().len()]; + for (row_idx, is_selected) in selected_rows.iter().copied().enumerate() { + if is_selected && list_array.is_valid(row_idx) { + let start = list_array.value_offsets()[row_idx].as_usize(); + let end = list_array.value_offsets()[row_idx + 1].as_usize(); + child_selection[start..end].fill(true); + } + } + let child_path = format!("{}.{}", field_path, child.name()); + collect_external_blob_uris( + child, + list_array.values(), + &child_selection, + &child_path, + external_uris, + )?; + } + ArrowDataType::LargeList(child) => { + let list_array = array.as_list::(); + let mut child_selection = vec![false; list_array.values().len()]; + for (row_idx, is_selected) in selected_rows.iter().copied().enumerate() { + if is_selected && list_array.is_valid(row_idx) { + let start = list_array.value_offsets()[row_idx].as_usize(); + let end = list_array.value_offsets()[row_idx + 1].as_usize(); + child_selection[start..end].fill(true); + } + } + let child_path = format!("{}.{}", field_path, child.name()); + collect_external_blob_uris( + child, + list_array.values(), + &child_selection, + &child_path, + external_uris, + )?; + } + _ => {} + } + Ok(()) +} + +/// Validate external blob references supplied by selected input rows. +/// +/// Existing rows can contain trusted absolute references that were accepted by an earlier write. +/// Update paths use this check before allowing those fallback values through the writer, so newly +/// matched values must still resolve beneath a registered external base. +pub(super) async fn validate_external_blob_references( + resolver: &ExternalBaseResolver, + batch: &RecordBatch, + selected_rows: &[bool], +) -> Result<()> { + let mut external_uris = Vec::new(); + for (field, array) in batch.schema().fields().iter().zip(batch.columns()) { + collect_external_blob_uris( + field, + array, + selected_rows, + field.name(), + &mut external_uris, + )?; + } + + let mut validated_uris = HashSet::new(); + for (field_path, uri) in external_uris { + if validated_uris.insert(uri.clone()) + && resolver.resolve_external_uri(&uri).await?.is_none() + { + return Err(Error::invalid_input(format!( + "External blob URI '{}' in field '{}' is outside registered external bases (dataset root is not allowed)", + uri, field_path + ))); + } + } + Ok(()) +} + struct RollingPackedBlobWriter { current: Option, current_size: usize, @@ -323,43 +480,41 @@ enum BlobPreprocessFieldKind { Struct { children: Vec, }, + List { + child: Box, + }, Passthrough, } impl BlobPreprocessField { fn new(field: &ArrowField) -> Result { if field.is_blob_v2() { - if is_prepared_blob_v2_field(field) { - return Ok(Self { + return match blob_v2_layout(field) { + Some(BlobV2Layout::Prepared) => Ok(Self { kind: BlobPreprocessFieldKind::Passthrough, - }); - } - if !is_logical_blob_v2_field(field) { - return Err(Error::invalid_input(format!( - "Blob v2 field '{}' must use either logical struct with optional position/size UInt64 fields or prepared \ - struct", - field.name() - ))); - } - return Ok(Self { - kind: BlobPreprocessFieldKind::BlobV2 { - inline_threshold: blob_inline_threshold_from_metadata( - field.metadata(), - field.name(), - )?, - dedicated_threshold: blob_dedicated_threshold_from_metadata( - field.metadata(), - field.name(), - )?, - pack_file_threshold: blob_pack_file_threshold_from_metadata( - field.metadata(), - field.name(), - )?, - writer_metadata: field.metadata().clone(), - }, - }); + }), + Some(BlobV2Layout::Logical) => Ok(Self { + kind: BlobPreprocessFieldKind::BlobV2 { + inline_threshold: blob_inline_threshold_from_metadata( + field.metadata(), + field.name(), + )?, + dedicated_threshold: blob_dedicated_threshold_from_metadata( + field.metadata(), + field.name(), + )?, + pack_file_threshold: blob_pack_file_threshold_from_metadata( + field.metadata(), + field.name(), + )?, + writer_metadata: field.metadata().clone(), + }, + }), + _ => Err(blob_v2_shape_error( + field, + &[BlobV2Layout::Logical, BlobV2Layout::Prepared], + )), + }; } if let ArrowDataType::Struct(children) = field.data_type() { @@ -374,6 +529,17 @@ impl BlobPreprocessField { } } + if let ArrowDataType::List(child) | ArrowDataType::LargeList(child) = field.data_type() { + let child = Self::new(child.as_ref())?; + if child.requires_preprocessing() { + return Ok(Self { + kind: BlobPreprocessFieldKind::List { + child: Box::new(child), + }, + }); + } + } + Ok(Self { kind: BlobPreprocessFieldKind::Passthrough, }) @@ -632,7 +798,7 @@ impl BlobPreprocessor { field: &'a Arc, ) -> BoxFuture<'a, Result<(ArrayRef, Arc)>> { async move { - if is_prepared_blob_v2_field(field.as_ref()) { + if blob_v2_layout(field.as_ref()) == Some(BlobV2Layout::Prepared) { validate_prepared_blob_array(field.as_ref(), &array)?; return Ok((array, field.clone())); } @@ -645,7 +811,7 @@ impl BlobPreprocessor { pack_file_threshold, writer_metadata, } => { - self.preprocess_blob_array( + self.logical_to_prepared_blob_array( array, field.as_ref(), *inline_threshold, @@ -659,6 +825,20 @@ impl BlobPreprocessor { self.preprocess_struct_array(array, field.as_ref(), children) .await } + BlobPreprocessFieldKind::List { child } => match field.data_type() { + ArrowDataType::List(_) => { + self.preprocess_list_array::(array, field.as_ref(), child) + .await + } + ArrowDataType::LargeList(_) => { + self.preprocess_list_array::(array, field.as_ref(), child) + .await + } + _ => Err(Error::internal(format!( + "Blob list preprocessor received non-list field '{}'", + field.name() + ))), + }, } } .boxed() @@ -691,10 +871,8 @@ impl BlobPreprocessor { let mut new_columns = Vec::with_capacity(children.len()); let mut new_fields = Vec::with_capacity(children.len()); - for ((child_processor, child_array), child_field) in children - .iter() - .zip(child_columns.into_iter()) - .zip(child_fields.iter()) + for ((child_processor, child_array), child_field) in + children.iter().zip(child_columns).zip(child_fields.iter()) { let (new_column, new_field) = self .preprocess_field(child_processor, child_array, child_field) @@ -716,7 +894,80 @@ impl BlobPreprocessor { Ok((Arc::new(struct_array), field)) } - async fn preprocess_blob_array( + async fn preprocess_list_array( + &mut self, + array: ArrayRef, + field: &ArrowField, + child: &BlobPreprocessField, + ) -> Result<(ArrayRef, Arc)> { + let list_arr = array.as_list::(); + let list_arr = if list_arr.null_count() > 0 { + list_arr.filter_garbage_nulls() + } else { + list_arr.clone() + }; + + let first_offset = *list_arr + .offsets() + .first() + .ok_or_else(|| Error::invalid_input("List offsets cannot be empty"))?; + let last_offset = *list_arr + .offsets() + .last() + .ok_or_else(|| Error::invalid_input("List offsets cannot be empty"))?; + let values_len = list_arr.values().len(); + let needs_trim = first_offset != O::zero() + || last_offset.to_usize().ok_or_else(|| { + Error::invalid_input(format!( + "List field '{}' offset does not fit into usize", + field.name() + )) + })? != values_len; + + let (offsets, values) = if needs_trim { + let values = list_arr.trimmed_values(); + let offsets = list_arr + .offsets() + .iter() + .map(|offset| *offset - first_offset) + .collect::>(); + (OffsetBuffer::new(ScalarBuffer::from(offsets)), values) + } else { + (list_arr.offsets().clone(), list_arr.values().clone()) + }; + + let child_field = match field.data_type() { + ArrowDataType::List(child_field) | ArrowDataType::LargeList(child_field) => { + child_field.clone() + } + other => { + return Err(Error::invalid_input(format!( + "Blob list preprocessor expected list field '{}', got {other}", + field.name() + ))); + } + }; + let (new_values, new_child_field) = + self.preprocess_field(child, values, &child_field).await?; + + let list_array = GenericListArray::::try_new( + new_child_field, + offsets, + new_values, + list_arr.nulls().cloned(), + )?; + let field = Arc::new( + ArrowField::new( + field.name(), + list_array.data_type().clone(), + field.is_nullable(), + ) + .with_metadata(field.metadata().clone()), + ); + Ok((Arc::new(list_array), field)) + } + + async fn logical_to_prepared_blob_array( &mut self, array: ArrayRef, field: &ArrowField, @@ -729,6 +980,15 @@ impl BlobPreprocessor { .as_any() .downcast_ref::() .ok_or_else(|| Error::invalid_input("Blob column was not a struct array"))?; + if BlobV2Layout::classify(struct_arr.fields()) != Some(BlobV2Layout::Logical) { + let actual = BlobV2Layout::classify(struct_arr.fields()) + .map(|layout| layout.to_string()) + .unwrap_or_else(|| format!("unrecognized ({:?})", struct_arr.fields())); + return Err(Error::invalid_input(format!( + "Blob v2 field '{}' has {actual} array layout; expected logical layout before preparation", + field.name() + ))); + } let data_col = struct_arr .column_by_name("data") @@ -987,7 +1247,9 @@ impl BlobSource { /// Drain currently queued requests and submit them as scheduler batches. /// /// Each loop iteration grabs the queued requests with a short mutex hold and - /// immediately releases the lock before any I/O is awaited. + /// dispatches them without waiting for earlier batches to finish. Awaiting a + /// batch here would hold later, naturally staggered callers behind its I/O. + /// [`FileScheduler`] owns the concurrency and backpressure for dispatched I/O. async fn drain_pending_reads(self: Arc, scheduler: FileScheduler) { loop { let batch = { @@ -998,7 +1260,10 @@ impl BlobSource { } std::mem::take(&mut pending_reads.requests) }; - fulfill_pending_blob_reads(&scheduler, batch).await; + let scheduler = scheduler.clone(); + tokio::spawn(async move { + fulfill_pending_blob_reads(&scheduler, batch).await; + }); } } } @@ -1468,13 +1733,80 @@ impl BlobFile { pub struct ReadBlob { /// Row address of the blob that was read. pub row_address: u64, - /// Blob payload bytes. - pub data: Bytes, + /// Blob payload bytes, or `None` when the selected blob value is null. + /// + /// A valid empty blob is represented as `Some(Bytes::new())`. + pub data: Option, +} + +/// A byte range relative to the beginning of one logical blob value. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct BlobReadRange { + /// Byte offset from the beginning of the blob value. + pub offset: u64, + /// Number of bytes to read. + pub length: u64, +} + +impl BlobReadRange { + /// Create a blob-local byte range from an offset and length. + pub const fn new(offset: u64, length: u64) -> Self { + Self { offset, length } + } + + fn checked_range(self, request_index: usize) -> Result> { + let end = self.offset.checked_add(self.length).ok_or_else(|| { + Error::invalid_input(format!( + "Blob range request {request_index} offset + length overflowed u64: offset={}, length={}", + self.offset, self.length + )) + })?; + Ok(self.offset..end) + } +} + +/// One row-specific blob range read request. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct BlobRangeRequest { + /// Row selector interpreted by the selection method on + /// [`ReadBlobRangesBuilder`]. + pub row: u64, + /// Blob-local byte range to read from the selected row. + pub range: BlobReadRange, +} + +impl BlobRangeRequest { + /// Create a request for `length` bytes at `offset` in the selected row. + pub const fn new(row: u64, offset: u64, length: u64) -> Self { + Self { + row, + range: BlobReadRange::new(offset, length), + } + } +} + +/// Bytes materialized for one request submitted through [`ReadBlobRangesBuilder`]. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ReadBlobRange { + /// Zero-based position of this request in the caller's request list. + pub request_index: usize, + /// Physical row address of the selected blob value. + pub row_address: u64, + /// Blob-local range supplied for this request. + pub range: BlobReadRange, + /// Bytes in `range`, or `None` when the selected blob value is null. + /// + /// An empty range on a non-null blob is represented as + /// `Some(Bytes::new())`. + pub data: Option, } /// Stream returned by [`ReadBlobsBuilder::try_into_stream`]. pub type ReadBlobsStream = BoxStream<'static, Result>; +/// Stream returned by [`ReadBlobRangesBuilder::try_into_stream`]. +pub type ReadBlobRangesStream = BoxStream<'static, Result>; + /// Row selector configured on [`ReadBlobsBuilder`]. #[derive(Debug, Clone)] enum ReadBlobsSelection { @@ -1484,6 +1816,43 @@ enum ReadBlobsSelection { RowAddresses(Vec), } +#[derive(Debug, Clone)] +enum ReadBlobRangesSelection { + None, + RowIds(Vec), + RowIndices(Vec), + RowAddresses(Vec), +} + +impl ReadBlobRangesSelection { + fn requests(&self) -> Option<&[BlobRangeRequest]> { + match self { + Self::None => None, + Self::RowIds(requests) | Self::RowIndices(requests) | Self::RowAddresses(requests) => { + Some(requests) + } + } + } + + fn into_row_selection(self) -> Option<(ReadBlobsSelection, Vec)> { + match self { + Self::None => None, + Self::RowIds(requests) => { + let rows = requests.iter().map(|request| request.row).collect(); + Some((ReadBlobsSelection::RowIds(rows), requests)) + } + Self::RowIndices(requests) => { + let rows = requests.iter().map(|request| request.row).collect(); + Some((ReadBlobsSelection::RowIndices(rows), requests)) + } + Self::RowAddresses(requests) => { + let rows = requests.iter().map(|request| request.row).collect(); + Some((ReadBlobsSelection::RowAddresses(rows), requests)) + } + } + } +} + /// Planner knobs for [`ReadBlobsBuilder`]. /// /// Options that shape how `read_blobs` uses Lance's existing schedulers. @@ -1559,77 +1928,51 @@ impl ReadBlobsBuilder { /// Execute the planned blob read and return a stream of blob payloads. /// - /// The stream yields one [`ReadBlob`] per selected non-null blob row. + /// The stream yields one [`ReadBlob`] per selected row. Null blob values + /// have `data` set to `None`; valid empty blobs contain an empty buffer. pub async fn try_into_stream(self) -> Result { self.validate()?; - let entries = collect_blob_entries_for_selection( + let collected = collect_blob_selection_for_selection( &self.dataset, self.blob_field_id, &self.column, &self.selection, ) .await?; - let expected_selection_indices = entries - .iter() - .map(|entry| entry.selection_index) - .collect::>(); - let plans = plan_blob_read_plans(entries); - let execution = Arc::new(ReadBlobsExecution::new(self.options.io_buffer_size_bytes)); - if plans.is_empty() { - return Ok(stream::empty().boxed()); - } - - let plan_stream = stream::iter(plans.into_iter().map(move |plan| { - let execution = execution.clone(); - execute_blob_read_plan(plan, execution) - })) - .buffer_unordered(self.dataset.object_store.io_parallelism().max(1)); - - if !self.options.preserve_order { - return Ok(plan_stream - .map_ok(|blobs| { - stream::iter(blobs.into_iter().map(|blob| Ok(into_read_blob(blob)))) - }) - .try_flatten() - .boxed()); - } - - let mut plan_stream = plan_stream.boxed(); - let mut expected_selection_indices = expected_selection_indices; - let mut ready = BTreeMap::::new(); - - Ok(stream::poll_fn(move |cx| { - loop { - let Some(next_selection_index) = expected_selection_indices.front().copied() else { - return Poll::Ready(None); - }; - - if let Some(blob) = ready.remove(&next_selection_index) { - expected_selection_indices.pop_front(); - return Poll::Ready(Some(Ok(blob))); - } - - match plan_stream.poll_next_unpin(cx) { - Poll::Ready(Some(Ok(blobs))) => { - for blob in blobs { - ready.insert(blob.selection_index, into_read_blob(blob)); - } - } - Poll::Ready(Some(Err(err))) => { - return Poll::Ready(Some(Err(err))); - } - Poll::Ready(None) => { - let err = Error::internal(format!( - "planned blob read stream completed before selection index {} was produced", - next_selection_index - )); - return Poll::Ready(Some(Err(err))); - } - Poll::Pending => return Poll::Pending, + let partitioned = + partition_blob_selections(collected, |selection_index, row_address, range| { + if range.is_some() { + return Err(Error::internal(format!( + "Whole-blob selection {} unexpectedly carried range metadata", + selection_index + ))); } - } + Ok(ReadBlob { + row_address, + data: None, + }) + })?; + let physical_buffer_size = self.options.io_buffer_size_bytes.unwrap_or_else(|| { + SchedulerConfig::max_bandwidth(self.dataset.object_store.as_ref()).io_buffer_size_bytes + }); + let batches = plan_blob_read_batches(partitioned.reads, physical_buffer_size)?; + let execution = Arc::new(ReadBlobsExecution::new(self.options.io_buffer_size_bytes)); + let non_null_blobs = execute_blob_read_batches_stream( + batches, + execution, + self.dataset.object_store.io_parallelism(), + ) + .map_ok(|blob| { + let selection_index = blob.selection_index; + (selection_index, into_read_blob(blob)) }) - .boxed()) + .boxed(); + Ok(totalize_blob_selection_stream( + non_null_blobs, + partitioned.ready_values, + partitioned.selection_count, + self.options.preserve_order, + )) } /// Execute the planned blob read and collect the full result in memory. @@ -1650,12 +1993,177 @@ impl ReadBlobsBuilder { } } +/// Builder for planned blob-local range reads. +/// +/// Planning retains `O(request_count)` metadata. [`Self::try_into_stream`] +/// bounds each scheduler-visible physical batch by the configured I/O buffer +/// size after accounting for range coalescing, except that one request larger +/// than the buffer must be materialized by itself. The existing scheduler +/// separately applies byte backpressure to physical I/O. [`Self::execute`] +/// additionally retains all returned payload bytes. +#[derive(Debug, Clone)] +pub struct ReadBlobRangesBuilder { + inner: ReadBlobsBuilder, + selection: ReadBlobRangesSelection, +} + +impl ReadBlobRangesBuilder { + pub(crate) fn new(inner: ReadBlobsBuilder) -> Self { + Self { + inner, + selection: ReadBlobRangesSelection::None, + } + } + + /// Read requests whose row values are stable row ids. + pub fn with_row_ids(mut self, requests: impl IntoIterator) -> Self { + self.selection = ReadBlobRangesSelection::RowIds(requests.into_iter().collect()); + self + } + + /// Read requests whose row values are offsets in dataset order. + pub fn with_row_indices( + mut self, + requests: impl IntoIterator, + ) -> Self { + self.selection = ReadBlobRangesSelection::RowIndices(requests.into_iter().collect()); + self + } + + /// Read requests whose row values are physical row addresses. + pub fn with_row_addresses( + mut self, + requests: impl IntoIterator, + ) -> Self { + self.selection = ReadBlobRangesSelection::RowAddresses(requests.into_iter().collect()); + self + } + + /// Set the scheduler I/O buffer size used while materializing ranges. + pub fn with_io_buffer_size_bytes(mut self, bytes: u64) -> Self { + self.inner.options.io_buffer_size_bytes = Some(bytes); + self + } + + /// Whether results must follow the caller's request order. + pub fn preserve_order(mut self, preserve: bool) -> Self { + self.inner.options.preserve_order = preserve; + self + } + + /// Execute the planned range read and return one result per request. + /// + /// Null blob requests have `data` set to `None`. Empty ranges on non-null + /// blobs contain an empty buffer without issuing payload I/O. By default, + /// results follow input order; `request_index` identifies the original + /// request even when ordering is disabled. Blob-local bounds are not + /// evaluated for null values because they have no logical payload length. + pub async fn try_into_stream(self) -> Result { + self.validate()?; + let (row_selection, requests) = self.selection.into_row_selection().ok_or_else(|| { + Error::internal("Validated blob range selection was missing".to_string()) + })?; + let mut selections = collect_blob_selection_for_selection( + &self.inner.dataset, + self.inner.blob_field_id, + &self.inner.column, + &row_selection, + ) + .await?; + if selections.len() != requests.len() { + return Err(Error::internal(format!( + "Resolved blob selection count {} did not match range request count {}", + selections.len(), + requests.len() + ))); + } + for (selection, request) in selections.iter_mut().zip(requests) { + selection.requested_range = Some(request.range); + } + let partitioned = partition_blob_selections( + selections, + |request_index, row_address, requested_range| { + let range = requested_range.ok_or_else(|| { + Error::internal(format!( + "Blob range request {} was missing range metadata", + request_index + )) + })?; + Ok(ReadBlobRange { + request_index, + row_address, + range, + data: None, + }) + }, + )?; + let physical_buffer_size = self.inner.options.io_buffer_size_bytes.unwrap_or_else(|| { + SchedulerConfig::max_bandwidth(self.inner.dataset.object_store.as_ref()) + .io_buffer_size_bytes + }); + let batches = plan_blob_read_batches(partitioned.reads, physical_buffer_size)?; + let execution = Arc::new(ReadBlobsExecution::new( + self.inner.options.io_buffer_size_bytes, + )); + let non_null_ranges = execute_blob_read_batches_stream( + batches, + execution, + self.inner.dataset.object_store.io_parallelism(), + ) + .map(|result| { + result.and_then(|blob| { + let selection_index = blob.selection_index; + into_read_blob_range(blob).map(|range| (selection_index, range)) + }) + }) + .boxed(); + Ok(totalize_blob_selection_stream( + non_null_ranges, + partitioned.ready_values, + partitioned.selection_count, + self.inner.options.preserve_order, + )) + } + + /// Execute the planned range read and collect all returned bytes in memory. + pub async fn execute(self) -> Result> { + self.try_into_stream().await?.try_collect().await + } + + fn validate(&self) -> Result<()> { + let requests = self.selection.requests().ok_or_else(|| { + Error::invalid_input( + "ReadBlobRangesBuilder requires requests; call one of with_row_ids, with_row_indices, or with_row_addresses".to_string(), + ) + })?; + if self.inner.options.io_buffer_size_bytes == Some(0) { + return Err(Error::invalid_input( + "ReadBlobRangesBuilder io_buffer_size must be greater than 0".to_string(), + )); + } + for (request_index, request) in requests.iter().enumerate() { + request.range.checked_range(request_index)?; + } + Ok(()) + } +} + /// One logical blob selected for planned reading. #[derive(Debug)] struct BlobEntry { selection_index: usize, row_address: u64, file: BlobFile, + requested_range: Option, +} + +/// One resolved logical selection. Its position in the containing vector is +/// the caller's selection index. +#[derive(Debug)] +struct ResolvedBlobSelection { + row_address: u64, + file: Option, + requested_range: Option, } /// Physical read input derived from one [`BlobEntry`]. @@ -1663,9 +2171,26 @@ struct BlobEntry { struct PlannedBlobRead { selection_index: usize, row_address: u64, + requested_range: Option, physical_range: Range, } +/// A slice of one disjoint physical range submitted to the file scheduler. +#[derive(Debug)] +struct PlannedBlobReadSlice { + read_index: usize, + physical_range_index: usize, + relative_range: Range, +} + +/// One physical read paired with the backing source used to batch it. +#[derive(Debug)] +struct SourcePlannedBlobRead { + source_key: BlobSourceKey, + source: Arc, + read: PlannedBlobRead, +} + /// One per-source read plan emitted by `read_blobs`. #[derive(Debug)] struct BlobReadPlan { @@ -1674,19 +2199,139 @@ struct BlobReadPlan { reads: Vec, } -/// Operation-scoped scheduler cache for one [`ReadBlobsBuilder`] execution. -/// -/// We reuse one [`ScanScheduler`] per object store during a single `read_blobs` -/// operation and still submit exactly one request per physical file. +/// A payload-bounded group of per-source plans. #[derive(Debug)] -struct ReadBlobsExecution { - io_buffer_size_bytes: Option, - schedulers: std::sync::Mutex>>, +struct BlobReadBatch { + plans: Vec, } -impl ReadBlobsExecution { - fn new(io_buffer_size_bytes: Option) -> Self { - Self { +#[derive(Debug)] +struct CoalescedRangeInsertion { + merged: Range, + replaced_starts: Vec, + replaced_bytes: u64, +} + +impl CoalescedRangeInsertion { + fn additional_bytes(&self) -> Result { + let merged_bytes = self.merged.end - self.merged.start; + merged_bytes.checked_sub(self.replaced_bytes).ok_or_else(|| { + Error::internal(format!( + "Coalesced blob range {:?} was smaller than its replaced ranges totaling {} bytes", + self.merged, self.replaced_bytes + )) + }) + } +} + +/// Physical ranges already charged to one source in the current batch. +#[derive(Debug, Default)] +struct CoalescedPhysicalRanges { + ranges: BTreeMap, +} + +impl CoalescedPhysicalRanges { + fn plan_insertion( + &self, + range: &Range, + block_size: u64, + ) -> Result> { + if range.is_empty() { + return Ok(None); + } + + let scan_start = self + .ranges + .range(..=range.start) + .next_back() + .map(|(start, _)| *start) + .unwrap_or(range.start); + let mut merged = range.clone(); + let mut replaced_starts = Vec::new(); + let mut replaced_bytes = 0_u64; + for (&start, &end) in self.ranges.range(scan_start..) { + if end.saturating_add(block_size) < merged.start { + continue; + } + if start > merged.end.saturating_add(block_size) { + break; + } + merged.start = merged.start.min(start); + merged.end = merged.end.max(end); + replaced_starts.push(start); + replaced_bytes = replaced_bytes + .checked_add(end - start) + .ok_or_else(|| Error::internal("Coalesced blob range size overflow".to_string()))?; + } + + Ok(Some(CoalescedRangeInsertion { + merged, + replaced_starts, + replaced_bytes, + })) + } + + fn insert(&mut self, insertion: CoalescedRangeInsertion) { + for start in insertion.replaced_starts { + self.ranges.remove(&start); + } + self.ranges + .insert(insertion.merged.start, insertion.merged.end); + } +} + +/// Exact scheduler-visible physical footprint of the current batch. +#[derive(Debug, Default)] +struct BlobReadBatchFootprint { + sources: HashMap, + physical_bytes: u64, +} + +impl BlobReadBatchFootprint { + fn try_insert( + &mut self, + planned: &SourcePlannedBlobRead, + physical_buffer_size: u64, + allow_oversized: bool, + ) -> Result { + let remaining = physical_buffer_size.saturating_sub(self.physical_bytes); + let additional_bytes = { + let source_ranges = self.sources.entry(planned.source_key.clone()).or_default(); + let insertion = source_ranges.plan_insertion( + &planned.read.physical_range, + planned.source.object_store.block_size() as u64, + )?; + let Some(insertion) = insertion else { + return Ok(true); + }; + let additional_bytes = insertion.additional_bytes()?; + if !allow_oversized && additional_bytes > remaining { + return Ok(false); + } + source_ranges.insert(insertion); + additional_bytes + }; + self.physical_bytes = self + .physical_bytes + .checked_add(additional_bytes) + .ok_or_else(|| Error::internal("Blob read batch size overflow".to_string()))?; + Ok(true) + } +} + +/// Operation-scoped scheduler cache for one [`ReadBlobsBuilder`] execution. +/// +/// We reuse one [`ScanScheduler`] per object store during a single `read_blobs` +/// operation and still submit exactly one request per physical file. +#[derive(Debug)] +struct ReadBlobsExecution { + io_buffer_size_bytes: Option, + schedulers: std::sync::Mutex>>, +} + +impl ReadBlobsExecution { + fn new(io_buffer_size_bytes: Option) -> Self { + Self { io_buffer_size_bytes, schedulers: std::sync::Mutex::new(HashMap::new()), } @@ -1715,42 +2360,301 @@ impl ReadBlobsExecution { struct IndexedReadBlob { selection_index: usize, row_address: u64, + requested_range: Option, data: Bytes, } fn into_read_blob(blob: IndexedReadBlob) -> ReadBlob { + debug_assert!( + blob.requested_range.is_none(), + "whole-blob reads must not carry a requested range" + ); ReadBlob { row_address: blob.row_address, - data: blob.data, + data: Some(blob.data), + } +} + +struct PartitionedBlobSelection { + selection_count: usize, + reads: Vec, + ready_values: Vec<(usize, T)>, +} + +fn partition_blob_selections( + selections: Vec, + mut make_ready_value: impl FnMut(usize, u64, Option) -> Result, +) -> Result> { + let selection_count = selections.len(); + let read_count = selections + .iter() + .filter(|selection| selection.file.is_some()) + .count(); + let mut reads = Vec::with_capacity(read_count); + let mut ready_values = Vec::with_capacity(selection_count - read_count); + for (selection_index, selection) in selections.into_iter().enumerate() { + match selection.file { + Some(file) => reads.push(BlobEntry { + selection_index, + row_address: selection.row_address, + file, + requested_range: selection.requested_range, + }), + None => { + let value = make_ready_value( + selection_index, + selection.row_address, + selection.requested_range, + )?; + ready_values.push((selection_index, value)); + } + } + } + Ok(PartitionedBlobSelection { + selection_count, + reads, + ready_values, + }) +} + +fn totalize_blob_selection_stream( + read_values: BoxStream<'static, Result<(usize, T)>>, + ready_values: Vec<(usize, T)>, + selection_count: usize, + preserve_order: bool, +) -> BoxStream<'static, Result> { + if preserve_order { + return totalize_ordered_blob_selection_stream(read_values, ready_values, selection_count); + } + + let ready_values = stream::iter(ready_values.into_iter().map(Ok::<(usize, T), Error>)); + let values = stream::select(ready_values, read_values).boxed(); + totalize_unordered_blob_selection_stream(values, selection_count) +} + +struct UnorderedBlobSelectionState { + values: BoxStream<'static, Result<(usize, T)>>, + seen_selection_indices: Vec, + remaining: usize, +} + +fn totalize_unordered_blob_selection_stream( + values: BoxStream<'static, Result<(usize, T)>>, + selection_count: usize, +) -> BoxStream<'static, Result> { + let state = UnorderedBlobSelectionState { + values, + seen_selection_indices: vec![false; selection_count], + remaining: selection_count, + }; + stream::try_unfold(state, move |mut state| async move { + let Some((selection_index, value)) = state.values.try_next().await? else { + return if state.remaining == 0 { + Ok(None) + } else { + Err(Error::internal(format!( + "planned blob read stream completed after producing {} of {} selections", + selection_count - state.remaining, + selection_count + ))) + }; + }; + + if selection_index >= selection_count { + return Err(Error::internal(format!( + "Blob selection index {} exceeded selected row count {}", + selection_index, selection_count + ))); + } + if state.seen_selection_indices[selection_index] { + return Err(Error::internal(format!( + "Blob selection index {} was produced more than once", + selection_index + ))); + } + state.seen_selection_indices[selection_index] = true; + state.remaining -= 1; + Ok(Some((value, state))) + }) + .boxed() +} + +struct OrderedBlobSelectionState { + read_values: BoxStream<'static, Result<(usize, T)>>, + ready: BTreeMap, + next_selection_index: usize, +} + +fn totalize_ordered_blob_selection_stream( + read_values: BoxStream<'static, Result<(usize, T)>>, + ready_values: Vec<(usize, T)>, + selection_count: usize, +) -> BoxStream<'static, Result> { + // Seed the reorder buffer so ordered consumers do not start later physical + // I/O while the next logical selection is already available. + let mut ready = BTreeMap::new(); + for (selection_index, value) in ready_values { + if selection_index >= selection_count { + let error = Error::internal(format!( + "Blob selection index {} exceeded selected row count {}", + selection_index, selection_count + )); + return stream::once(async move { Err(error) }).boxed(); + } + if ready.insert(selection_index, value).is_some() { + let error = Error::internal(format!( + "Blob selection index {} was produced more than once", + selection_index + )); + return stream::once(async move { Err(error) }).boxed(); + } + } + let state = OrderedBlobSelectionState { + read_values, + ready, + next_selection_index: 0, + }; + stream::try_unfold(state, move |mut state| async move { + loop { + if let Some(value) = state.ready.remove(&state.next_selection_index) { + state.next_selection_index += 1; + return Ok(Some((value, state))); + } + + match state.read_values.try_next().await? { + Some((selection_index, value)) => { + if selection_index >= selection_count { + return Err(Error::internal(format!( + "Blob selection index {} exceeded selected row count {}", + selection_index, selection_count + ))); + } + if selection_index < state.next_selection_index { + return Err(Error::internal(format!( + "Blob selection index {} was produced more than once", + selection_index + ))); + } + if state.ready.insert(selection_index, value).is_some() { + return Err(Error::internal(format!( + "Blob selection index {} was produced more than once", + selection_index + ))); + } + } + None if state.next_selection_index == selection_count => return Ok(None), + None => { + return Err(Error::internal(format!( + "planned blob read stream completed before selection index {} was produced", + state.next_selection_index + ))); + } + } + } + }) + .boxed() +} + +fn into_read_blob_range(blob: IndexedReadBlob) -> Result { + let range = blob.requested_range.ok_or_else(|| { + Error::internal(format!( + "Blob range request {} completed without range metadata", + blob.selection_index + )) + })?; + Ok(ReadBlobRange { + request_index: blob.selection_index, + row_address: blob.row_address, + range, + data: Some(blob.data), + }) +} + +/// Split selected reads into request-order batches before grouping each batch +/// by physical source. Each batch accounts for the physical spans that the file +/// scheduler will read after coalescing nearby ranges. +fn plan_blob_read_batches( + mut entries: Vec, + physical_buffer_size: u64, +) -> Result> { + debug_assert!(physical_buffer_size > 0); + entries.sort_by_key(|entry| entry.selection_index); + let planned_reads = plan_blob_reads(entries)?; + + let mut batches = Vec::new(); + let mut current_reads = Vec::new(); + let mut footprint = BlobReadBatchFootprint::default(); + for planned in planned_reads { + if !footprint.try_insert(&planned, physical_buffer_size, current_reads.is_empty())? { + batches.push(into_blob_read_batch(std::mem::take(&mut current_reads))); + footprint = BlobReadBatchFootprint::default(); + let inserted = footprint.try_insert(&planned, physical_buffer_size, true)?; + debug_assert!(inserted, "an empty batch must accept one physical read"); + } + current_reads.push(planned); + } + if !current_reads.is_empty() { + batches.push(into_blob_read_batch(current_reads)); + } + + Ok(batches) +} + +fn plan_blob_reads(entries: Vec) -> Result> { + let mut planned_reads = Vec::with_capacity(entries.len()); + for entry in entries { + let logical_range = match entry.requested_range { + Some(range) => range.checked_range(entry.selection_index)?, + None => 0..entry.file.size, + }; + let physical_range = entry.file.read_phys_range(logical_range).map_err(|err| { + Error::invalid_input(format!( + "Blob range request {} for row address {} is invalid: {}", + entry.selection_index, entry.row_address, err + )) + })?; + let source_key = BlobSourceKey::new(&entry.file.source); + planned_reads.push(SourcePlannedBlobRead { + source_key, + source: entry.file.source.clone(), + read: PlannedBlobRead { + selection_index: entry.selection_index, + row_address: entry.row_address, + requested_range: entry.requested_range, + physical_range, + }, + }); + } + + Ok(planned_reads) +} + +fn into_blob_read_batch(planned_reads: Vec) -> BlobReadBatch { + BlobReadBatch { + plans: group_blob_read_plans(planned_reads), } } /// Group selected blobs by physical source and sort each group's ranges by /// physical offset before handing them to the file scheduler. -fn plan_blob_read_plans(entries: Vec) -> Vec { +fn group_blob_read_plans(planned_reads: Vec) -> Vec { let mut plan_indices = HashMap::::new(); let mut plans = Vec::::new(); - for entry in entries { - let source_key = BlobSourceKey::new(&entry.file.source); - let plan_index = if let Some(plan_index) = plan_indices.get(&source_key) { + for planned in planned_reads { + let plan_index = if let Some(plan_index) = plan_indices.get(&planned.source_key) { *plan_index } else { let plan_index = plans.len(); plans.push(BlobReadPlan { - source_key: source_key.clone(), - source: entry.file.source.clone(), + source_key: planned.source_key.clone(), + source: planned.source, reads: Vec::new(), }); - plan_indices.insert(source_key.clone(), plan_index); + plan_indices.insert(planned.source_key, plan_index); plan_index }; - - plans[plan_index].reads.push(PlannedBlobRead { - selection_index: entry.selection_index, - row_address: entry.row_address, - physical_range: entry.file.position..(entry.file.position + entry.file.size), - }); + plans[plan_index].reads.push(planned.read); } plans.sort_by(|left, right| { @@ -1773,22 +2677,102 @@ fn plan_blob_read_plans(entries: Vec) -> Vec { plans } +fn plan_blob_read_plans(entries: Vec) -> Result> { + Ok(group_blob_read_plans(plan_blob_reads(entries)?)) +} + +/// Merge overlapping physical reads before submitting them to [`FileScheduler`]. +/// +/// `FileScheduler` can safely coalesce and split disjoint ranges. Original +/// overlapping ranges need to be mapped onto their union first so a split does +/// not advance past the start of a later nested range while reconstructing the +/// caller's buffers. +fn plan_disjoint_blob_reads( + reads: &[PlannedBlobRead], +) -> (Vec>, Vec) { + let mut non_empty_ranges = reads + .iter() + .enumerate() + .filter(|(_, read)| !read.physical_range.is_empty()) + .map(|(read_index, read)| (read_index, read.physical_range.clone())) + .collect::>(); + non_empty_ranges.sort_by_key(|(read_index, range)| (range.start, range.end, *read_index)); + + let mut physical_ranges = Vec::>::with_capacity(non_empty_ranges.len()); + let mut slices = Vec::with_capacity(non_empty_ranges.len()); + for (read_index, range) in non_empty_ranges { + let physical_range_index = match physical_ranges.last_mut() { + Some(physical_range) if range.start <= physical_range.end => { + physical_range.end = physical_range.end.max(range.end); + physical_ranges.len() - 1 + } + _ => { + physical_ranges.push(range.clone()); + physical_ranges.len() - 1 + } + }; + let physical_start = physical_ranges[physical_range_index].start; + slices.push(PlannedBlobReadSlice { + read_index, + physical_range_index, + relative_range: range.start - physical_start..range.end - physical_start, + }); + } + + (physical_ranges, slices) +} + /// Execute one per-source blob read plan with a single scheduler submission. async fn execute_blob_read_plan( task: BlobReadPlan, execution: Arc, ) -> Result> { - let ranges = task - .reads - .iter() - .map(|read| read.physical_range.clone()) - .collect::>(); - let scheduler = execution.scheduler_for(&task.source); - let file_scheduler = scheduler - .open_file(&task.source.path, &task.source.file_size) - .await?; - let priority = ranges[0].start; - let bytes = file_scheduler.submit_request(ranges, priority).await?; + let (physical_ranges, slices) = plan_disjoint_blob_reads(&task.reads); + let mut bytes = vec![Bytes::new(); task.reads.len()]; + if let Some(first_range) = physical_ranges.first() { + let scheduler = execution.scheduler_for(&task.source); + let file_scheduler = scheduler + .open_file(&task.source.path, &task.source.file_size) + .await?; + let priority = first_range.start; + let physical_range_count = physical_ranges.len(); + let returned = file_scheduler + .submit_request(physical_ranges, priority) + .await?; + if returned.len() != physical_range_count { + return Err(Error::internal(format!( + "Blob read scheduler returned {} ranges for {} disjoint physical ranges from {}", + returned.len(), + physical_range_count, + task.source.path + ))); + } + for slice in slices { + let start = usize::try_from(slice.relative_range.start).map_err(|_| { + Error::internal(format!( + "Blob read slice start {} does not fit into usize for {}", + slice.relative_range.start, task.source.path + )) + })?; + let end = usize::try_from(slice.relative_range.end).map_err(|_| { + Error::internal(format!( + "Blob read slice end {} does not fit into usize for {}", + slice.relative_range.end, task.source.path + )) + })?; + let data = &returned[slice.physical_range_index]; + if end > data.len() { + return Err(Error::internal(format!( + "Blob read slice {:?} exceeds the {} bytes returned for physical range {} from {}", + slice.relative_range, + data.len(), + slice.physical_range_index, + task.source.path + ))); + } + bytes[slice.read_index] = data.slice(start..end); + } + } Ok(task .reads @@ -1797,27 +2781,82 @@ async fn execute_blob_read_plan( .map(|(read, data)| IndexedReadBlob { selection_index: read.selection_index, row_address: read.row_address, + requested_range: read.requested_range, data, }) .collect()) } +fn execute_blob_read_batches_stream( + batches: Vec, + execution: Arc, + io_parallelism: usize, +) -> BoxStream<'static, Result> { + let streams = batches.into_iter().map(move |batch| { + execute_blob_read_plans_stream(batch.plans, execution.clone(), io_parallelism) + }); + stream::iter(streams).flatten().boxed() +} + +fn execute_blob_read_plans_stream( + plans: Vec, + execution: Arc, + io_parallelism: usize, +) -> BoxStream<'static, Result> { + if plans.is_empty() { + return stream::empty().boxed(); + } + + let plan_stream = stream::iter(plans.into_iter().map(move |plan| { + let execution = execution.clone(); + execute_blob_read_plan(plan, execution) + })) + .buffer_unordered(io_parallelism.max(1)); + + plan_stream + .map_ok(|blobs| stream::iter(blobs.into_iter().map(Ok))) + .try_flatten() + .boxed() +} + +async fn execute_blob_entries( + entries: Vec, + io_parallelism: usize, + io_buffer_size_bytes: Option, +) -> Result> { + let plans = plan_blob_read_plans(entries)?; + if plans.is_empty() { + return Ok(Vec::new()); + } + + let execution = Arc::new(ReadBlobsExecution::new(io_buffer_size_bytes)); + let batches = stream::iter(plans.into_iter().map(move |plan| { + let execution = execution.clone(); + execute_blob_read_plan(plan, execution) + })) + .buffer_unordered(io_parallelism.max(1)) + .try_collect::>() + .await?; + Ok(batches.into_iter().flatten().collect()) +} + pub(super) async fn take_blobs( dataset: &Arc, row_ids: &[u64], column: &str, -) -> Result> { +) -> Result>> { let blob_field_id = validate_blob_column(dataset, column)?; - Ok(collect_blob_entries_for_selection( + let collected = collect_blob_selection_for_selection( dataset, blob_field_id, column, &ReadBlobsSelection::RowIds(row_ids.to_vec()), ) - .await? - .into_iter() - .map(|entry| entry.file) - .collect()) + .await?; + Ok(collected + .into_iter() + .map(|selection| selection.file) + .collect()) } /// Take [BlobFile] by row addresses. @@ -1831,18 +2870,19 @@ pub async fn take_blobs_by_addresses( dataset: &Arc, row_addrs: &[u64], column: &str, -) -> Result> { +) -> Result>> { let blob_field_id = validate_blob_column(dataset, column)?; - Ok(collect_blob_entries_for_selection( + let collected = collect_blob_selection_for_selection( dataset, blob_field_id, column, &ReadBlobsSelection::RowAddresses(row_addrs.to_vec()), ) - .await? - .into_iter() - .map(|entry| entry.file) - .collect()) + .await?; + Ok(collected + .into_iter() + .map(|selection| selection.file) + .collect()) } /// Validate that `column` exists and is a blob column, returning its field id. @@ -1868,6 +2908,7 @@ async fn take_blob_descriptions_by_row_ids( let projection = dataset.schema().project(&[column])?; dataset .take_builder(row_ids, projection)? + .with_missing_row_policy(MissingRowPolicy::Error) .with_row_address(true) .execute() .await @@ -1888,14 +2929,13 @@ async fn take_blob_descriptions_by_row_addresses( .await } -/// Resolve a caller selection into [`BlobEntry`] values that share `BlobSource` -/// instances by physical backing object. -async fn collect_blob_entries_for_selection( +/// Resolve every selected row address and the non-null blob entries among them. +async fn collect_blob_selection_for_selection( dataset: &Arc, blob_field_id: u32, column: &str, selection: &ReadBlobsSelection, -) -> Result> { +) -> Result> { let description_and_addr = match selection { ReadBlobsSelection::None => { return Err(Error::invalid_input( @@ -1923,256 +2963,779 @@ async fn collect_blob_entries_for_selection( let descriptions = leaf_descriptor_struct(&description_and_addr, column)?; let row_addrs = description_and_addr.column(1).as_primitive::(); - match blob_version_from_descriptions(descriptions)? { - BlobVersion::V1 => collect_blob_entries_v1(dataset, blob_field_id, descriptions, row_addrs), + let files = match blob_version_from_descriptions(descriptions)? { + BlobVersion::V1 => collect_blob_files_v1(dataset, blob_field_id, descriptions, row_addrs), BlobVersion::V2 => { - collect_blob_entries_v2(dataset, blob_field_id, descriptions, row_addrs).await + collect_blob_files_v2(dataset, blob_field_id, descriptions, row_addrs).await } + }?; + if files.len() != row_addrs.len() { + return Err(Error::internal(format!( + "Resolved blob file count {} did not match row address count {}", + files.len(), + row_addrs.len() + ))); } + Ok(row_addrs + .values() + .iter() + .copied() + .zip(files) + .map(|(row_address, file)| ResolvedBlobSelection { + row_address, + file, + requested_range: None, + }) + .collect()) } /// Walk into the descriptor `RecordBatch` at `column` and return the leaf /// descriptor `StructArray`, descending through nested struct children for /// dotted paths. fn leaf_descriptor_struct<'a>(batch: &'a RecordBatch, column: &str) -> Result<&'a StructArray> { + let current = leaf_descriptor_array(batch, column)?; + current + .as_any() + .downcast_ref::() + .ok_or_else(|| { + Error::invalid_input_source( + format!( + "Blob column '{}' expected descriptor struct but got {}", + column, + current.data_type() + ) + .into(), + ) + }) +} + +fn leaf_descriptor_array<'a>(batch: &'a RecordBatch, column: &str) -> Result<&'a dyn Array> { let path = parse_field_path(column)?; - let mut current = batch + let mut current: &dyn Array = batch .column_by_name(&path[0]) - .expect("validate_blob_column ensured column exists") - .as_struct(); + .ok_or_else(|| { + Error::invalid_input(format!( + "Blob column '{}' was not found in descriptor batch", + column + )) + })? + .as_ref(); for segment in &path[1..] { - current = current + let struct_array = current + .as_any() + .downcast_ref::() + .ok_or_else(|| { + Error::invalid_input_source( + format!( + "Blob column path '{}' expected struct before segment '{}' but got {}", + column, + segment, + current.data_type() + ) + .into(), + ) + })?; + current = struct_array .column_by_name(segment) - .expect("validate_blob_column ensured all path segments exist") - .as_struct(); + .ok_or_else(|| { + Error::invalid_input(format!( + "Blob column path '{}' missing segment '{}'", + column, segment + )) + })? + .as_ref(); } Ok(current) } +fn blob_descriptor_fields_match( + actual: &arrow_schema::Fields, + expected: &arrow_schema::Fields, +) -> bool { + actual.len() == expected.len() + && actual + .iter() + .zip(expected.iter()) + .all(|(actual, expected)| { + actual.name() == expected.name() && actual.data_type() == expected.data_type() + }) +} + fn blob_version_from_descriptions(descriptions: &StructArray) -> Result { let fields = descriptions.fields(); - if fields.len() == 2 && fields[0].name() == "position" && fields[1].name() == "size" { + if blob_descriptor_fields_match(fields, &BLOB_DESC_FIELDS) { return Ok(BlobVersion::V1); } - if fields.len() == 5 - && fields[0].name() == "kind" - && fields[1].name() == "position" - && fields[2].name() == "size" - && fields[3].name() == "blob_id" - && fields[4].name() == "blob_uri" - { + if BlobV2Layout::classify(fields) == Some(BlobV2Layout::Descriptor) { return Ok(BlobVersion::V2); } Err(Error::invalid_input_source(format!( - "Unrecognized blob descriptions schema: expected v1 (position,size) or v2 (kind,position,size,blob_id,blob_uri) but got {:?}", - fields.iter().map(|f| f.name().as_str()).collect::>(), + "Unrecognized blob descriptions schema: expected v1 descriptor or v2 descriptor layout but got {fields:?}", ) .into())) } -/// Convert blob v1 descriptors into logical blob entries. -fn collect_blob_entries_v1( +struct BlobV2DescriptorColumns<'a> { + descriptions: &'a StructArray, + kinds: &'a arrow::array::PrimitiveArray, + positions: &'a arrow::array::PrimitiveArray, + sizes: &'a arrow::array::PrimitiveArray, + blob_ids: &'a arrow::array::PrimitiveArray, + blob_uris: &'a arrow::array::GenericStringArray, +} + +impl<'a> BlobV2DescriptorColumns<'a> { + fn new(descriptions: &'a StructArray) -> Self { + Self { + descriptions, + kinds: descriptions.column(0).as_primitive::(), + positions: descriptions.column(1).as_primitive::(), + sizes: descriptions.column(2).as_primitive::(), + blob_ids: descriptions.column(3).as_primitive::(), + blob_uris: descriptions.column(4).as_string::(), + } + } + + fn is_null_blob(&self, idx: usize) -> bool { + self.descriptions.is_null(idx) || self.kinds.is_null(idx) + } +} + +/// Resolve blob v1 descriptors without dropping null selection slots. +fn collect_blob_files_v1( dataset: &Arc, blob_field_id: u32, descriptions: &StructArray, row_addrs: &arrow::array::PrimitiveArray, -) -> Result> { +) -> Result>> { + if descriptions.len() != row_addrs.len() { + return Err(Error::internal(format!( + "Blob descriptor count {} did not match row address count {}", + descriptions.len(), + row_addrs.len() + ))); + } let positions = descriptions.column(0).as_primitive::(); let sizes = descriptions.column(1).as_primitive::(); let mut source_cache = HashMap::>::new(); - row_addrs - .values() - .iter() - .zip(positions.iter()) - .zip(sizes.iter()) - .enumerate() - .filter_map(|(selection_index, ((row_addr, position), size))| { - let position = position?; - let size = size?; - Some((selection_index, *row_addr, position, size)) - }) - .map(|(selection_index, row_addr, position, size)| { - let frag_id = RowAddress::from(row_addr).fragment_id(); - let frag = dataset.get_fragment(frag_id as usize).ok_or_else(|| { - Error::invalid_input(format!( - "Blob row address {} references missing fragment {}", - row_addr, frag_id - )) - })?; - let data_file = frag.data_file_for_field(blob_field_id).ok_or_else(|| { - Error::invalid_input(format!( - "Blob field {} has no data file in fragment {} for row address {}", - blob_field_id, frag_id, row_addr - )) - })?; - let data_file_path = dataset.data_dir().join(data_file.path.as_str()); - Ok(BlobEntry { - selection_index, - row_address: row_addr, - file: BlobFile::with_source( - shared_blob_source( - &mut source_cache, - dataset.object_store.clone(), - &data_file_path, - ), - position, - size, - BlobKind::Inline, - None, - ), - }) - }) - .collect() -} - -/// Convert blob v2 descriptors into logical blob entries. -async fn collect_blob_entries_v2( - dataset: &Arc, - blob_field_id: u32, - descriptions: &StructArray, - row_addrs: &arrow::array::PrimitiveArray, -) -> Result> { - let kinds = descriptions.column(0).as_primitive::(); - let positions = descriptions.column(1).as_primitive::(); - let sizes = descriptions.column(2).as_primitive::(); - let blob_ids = descriptions.column(3).as_primitive::(); - let blob_uris = descriptions.column(4).as_string::(); - let mut files = Vec::with_capacity(row_addrs.len()); - let mut fragment_cache = HashMap::::new(); - let mut store_cache = HashMap::>::new(); - let mut external_base_path_cache = HashMap::::new(); - let mut source_cache = HashMap::>::new(); - for (selection_index, row_addr) in row_addrs.values().iter().enumerate() { - let idx = selection_index; - let kind = BlobKind::try_from(kinds.value(idx))?; + for selection_index in 0..row_addrs.len() { + if descriptions.is_null(selection_index) + || positions.is_null(selection_index) + || sizes.is_null(selection_index) + { + files.push(None); + continue; + } - // Struct is non-nullable; null rows are encoded as inline with zero position/size and empty uri - if matches!(kind, BlobKind::Inline) && positions.value(idx) == 0 && sizes.value(idx) == 0 { + let position = positions.value(selection_index); + let size = sizes.value(selection_index); + // V1 encodes valid empty blobs as (0, 0) and smuggles null + // repetition/definition levels through a non-zero zero-sized position. + if size == 0 && position != 0 { + files.push(None); continue; } - match kind { - BlobKind::Inline => { - let position = positions.value(idx); - let size = sizes.value(idx); - let location = resolve_blob_read_location( - dataset, - blob_field_id, - *row_addr, - &mut fragment_cache, - &mut store_cache, - ) - .await?; - let source = shared_blob_source( - &mut source_cache, - location.object_store, - &location.data_file_path, - ); - files.push(BlobEntry { - selection_index, - row_address: *row_addr, - file: BlobFile::with_source(source, position, size, BlobKind::Inline, None), - }); - } - BlobKind::Dedicated => { - let blob_id = blob_ids.value(idx); - let size = sizes.value(idx); - let location = resolve_blob_read_location( - dataset, - blob_field_id, - *row_addr, - &mut fragment_cache, - &mut store_cache, - ) - .await?; - let path = blob_path(&location.data_file_dir, &location.data_file_key, blob_id); - let source = shared_blob_source(&mut source_cache, location.object_store, &path); - files.push(BlobEntry { - selection_index, - row_address: *row_addr, - file: BlobFile::with_source(source, 0, size, BlobKind::Dedicated, None), - }); + let row_addr = row_addrs.value(selection_index); + let frag_id = RowAddress::from(row_addr).fragment_id(); + let frag = dataset.get_fragment(frag_id as usize).ok_or_else(|| { + Error::invalid_input(format!( + "Blob row address {} references missing fragment {}", + row_addr, frag_id + )) + })?; + let data_file = frag.data_file_for_field(blob_field_id).ok_or_else(|| { + Error::invalid_input(format!( + "Blob field {} has no data file in fragment {} for row address {}", + blob_field_id, frag_id, row_addr + )) + })?; + let data_file_path = dataset.data_dir().join(data_file.path.as_str()); + files.push(Some(BlobFile::with_source( + shared_blob_source( + &mut source_cache, + dataset.object_store.clone(), + &data_file_path, + ), + position, + size, + BlobKind::Inline, + None, + ))); + } + Ok(files) +} + +/// Resolve blob v2 descriptors without dropping null selection slots. +async fn collect_blob_files_v2( + dataset: &Arc, + blob_field_id: u32, + descriptions: &StructArray, + row_addrs: &arrow::array::PrimitiveArray, +) -> Result>> { + collect_blob_v2_descriptor_files(dataset, blob_field_id, descriptions, row_addrs.values()).await +} + +/// Resolve blob v2 descriptors to lazy handles without materializing their payloads. +pub(super) async fn collect_blob_v2_descriptor_files( + dataset: &Arc, + blob_field_id: u32, + descriptions: &StructArray, + row_addrs: &[u64], +) -> Result>> { + if descriptions.len() != row_addrs.len() { + return Err(Error::internal(format!( + "Blob descriptor count {} did not match row address count {}", + descriptions.len(), + row_addrs.len() + ))); + } + let columns = BlobV2DescriptorColumns::new(descriptions); + let mut files = Vec::with_capacity(row_addrs.len()); + let mut read_context = BlobV2ReadContext::new(dataset, blob_field_id); + for (selection_index, row_addr) in row_addrs.iter().enumerate() { + files.push( + read_context + .collect_file(&columns, selection_index, *row_addr) + .await?, + ); + } + + Ok(files) +} + +fn is_blob_v2_binary_view(field: &LanceField) -> bool { + field.is_blob_v2() && matches!(field.data_type(), ArrowDataType::LargeBinary) +} + +fn public_blob_v2_binary_output_field(mut field: LanceField) -> LanceField { + if is_blob_v2_binary_view(&field) { + field.metadata.remove(ARROW_EXT_NAME_KEY); + } + field.children = field + .children + .into_iter() + .map(public_blob_v2_binary_output_field) + .collect(); + field +} + +/// Return the public Arrow-facing schema for a blob v2 binary scan. +/// +/// Scan planning uses a blob v2 extension marker on `LargeBinary` leaves to +/// identify payloads that need descriptor-based materialization. This helper +/// removes that internal marker before the schema is exposed to callers. +pub fn public_blob_v2_binary_output_schema(schema: &Schema) -> Schema { + Schema { + fields: schema + .fields + .iter() + .cloned() + .map(public_blob_v2_binary_output_field) + .collect(), + metadata: schema.metadata.clone(), + } +} + +fn field_has_blob_v2_binary_view(field: &LanceField) -> bool { + is_blob_v2_binary_view(field) || field.children.iter().any(field_has_blob_v2_binary_view) +} + +/// Return true if the schema contains a blob v2 leaf in binary payload view. +/// +/// This detects the internal `LargeBinary` view created by +/// [`BlobHandling::AllBinary`](lance_core::datatypes::BlobHandling::AllBinary) +/// or selective binary blob handling. +pub fn schema_has_blob_v2_binary_view(schema: &Schema) -> bool { + schema.fields.iter().any(field_has_blob_v2_binary_view) +} + +fn blob_v2_descriptor_field(mut field: LanceField) -> LanceField { + if is_blob_v2_binary_view(&field) { + field.unloaded_mut(); + return field; + } + + field.children = field + .children + .into_iter() + .map(blob_v2_descriptor_field) + .collect(); + field +} + +/// Convert blob v2 binary-view leaves back to descriptor-view leaves. +/// +/// Readers use this schema to fetch stored blob descriptors first. The scan +/// layer then materializes those descriptors into the caller's binary payload +/// view after row addresses are available. +pub fn blob_v2_descriptor_schema(schema: &Schema) -> Schema { + Schema { + fields: schema + .fields + .iter() + .cloned() + .map(blob_v2_descriptor_field) + .collect(), + metadata: schema.metadata.clone(), + } +} + +/// Materialize blob v2 descriptor arrays in a decoded batch into binary arrays. +/// +/// The input batch must include `_rowaddr`, which is used to resolve packed, +/// dedicated, inline, and external blob payload locations. `output_schema` +/// defines the exact returned columns, including requested system columns, with +/// blob v2 binary leaves exposed as plain `LargeBinary` fields. +pub async fn materialize_blob_v2_binary_batch( + dataset: &Arc, + output_schema: &Schema, + batch: RecordBatch, +) -> Result { + let row_addr_idx = batch + .schema() + .column_with_name(ROW_ADDR) + .ok_or_else(|| { + Error::internal(format!( + "_rowaddr column missing from blob v2 binary scan batch, columns: {:?}", + batch + .schema() + .fields() + .iter() + .map(|field| field.name()) + .collect::>() + )) + })? + .0; + let row_addrs = batch + .column(row_addr_idx) + .as_primitive::() + .values() + .iter() + .copied() + .collect::>(); + let row_addrs: Arc<[u64]> = row_addrs.into(); + + let mut columns = Vec::with_capacity(output_schema.fields.len()); + let mut fields = Vec::with_capacity(output_schema.fields.len()); + + for field in &output_schema.fields { + let input = batch + .column_by_name(&field.name) + .ok_or_else(|| { + Error::internal(format!( + "blob v2 binary scan batch missing projected column '{}'", + field.name + )) + })? + .clone(); + let materialized = + materialize_blob_v2_binary_array(dataset, field, input, row_addrs.clone()).await?; + columns.push(materialized); + let output_field = public_blob_v2_binary_output_field(field.clone()); + fields.push(ArrowField::from(&output_field)); + } + + Ok(RecordBatch::try_new( + Arc::new(ArrowSchema::new_with_metadata( + fields, + batch.schema().metadata().clone(), + )), + columns, + )?) +} + +fn materialize_blob_v2_binary_array<'a>( + dataset: &'a Arc, + field: &'a LanceField, + array: ArrayRef, + row_addrs: Arc<[u64]>, +) -> BoxFuture<'a, Result> { + async move { + if is_blob_v2_binary_view(field) { + let descriptions = array.as_struct(); + return materialize_blob_v2_descriptors( + dataset, + field.id as u32, + descriptions, + row_addrs.as_ref(), + ) + .await; + } + + match field.data_type() { + ArrowDataType::Struct(_) => { + let struct_array = array.as_struct(); + let mut children = Vec::with_capacity(field.children.len()); + for (child_field, child_array) in + field.children.iter().zip(struct_array.columns().iter()) + { + children.push( + materialize_blob_v2_binary_array( + dataset, + child_field, + child_array.clone(), + row_addrs.clone(), + ) + .await?, + ); + } + let public_field = public_blob_v2_binary_output_field(field.clone()); + let ArrowDataType::Struct(fields) = public_field.data_type() else { + unreachable!("public output field preserved struct type") + }; + Ok(Arc::new(StructArray::try_new( + fields, + children, + struct_array.nulls().cloned(), + )?) as ArrayRef) } - BlobKind::Packed => { - let blob_id = blob_ids.value(idx); - let size = sizes.value(idx); - let position = positions.value(idx); - let location = resolve_blob_read_location( - dataset, - blob_field_id, - *row_addr, - &mut fragment_cache, - &mut store_cache, - ) - .await?; - let path = blob_path(&location.data_file_dir, &location.data_file_key, blob_id); - let source = shared_blob_source(&mut source_cache, location.object_store, &path); - files.push(BlobEntry { - selection_index, - row_address: *row_addr, - file: BlobFile::with_source(source, position, size, BlobKind::Packed, None), - }); + ArrowDataType::List(_) => { + let list_array = array.as_list::(); + materialize_blob_v2_list_array::(dataset, field, list_array, row_addrs).await } - BlobKind::External => { - let uri_or_path = blob_uris.value(idx).to_string(); - let position = positions.value(idx); - let size = sizes.value(idx); - let base_id = blob_ids.value(idx); - let (object_store, path) = if base_id == 0 { - let registry = dataset.session.store_registry(); - let params = dataset - .store_params - .as_ref() - .map(|p| Arc::new((**p).clone())) - .unwrap_or_else(|| Arc::new(ObjectStoreParams::default())); - ObjectStore::from_uri_and_params(registry, &uri_or_path, ¶ms).await? - } else { - let object_store = if let Some(store) = store_cache.get(&base_id) { - store.clone() - } else { - let store = dataset.object_store(Some(base_id)).await?; - store_cache.insert(base_id, store.clone()); - store - }; - let base_root = if let Some(path) = external_base_path_cache.get(&base_id) { - path.clone() - } else { - let base = dataset.manifest.base_paths.get(&base_id).ok_or_else(|| { - Error::invalid_input(format!( - "External blob references unknown base_id {}", - base_id - )) - })?; - let path = base.extract_path(dataset.session.store_registry())?; - external_base_path_cache.insert(base_id, path.clone()); - path - }; - let path = join_base_and_relative_path(&base_root, &uri_or_path)?; - (object_store, path) - }; - let size = if size > 0 { - size - } else { - object_store.size(&path).await? - }; - let source = shared_blob_source(&mut source_cache, object_store, &path); - files.push(BlobEntry { - selection_index, - row_address: *row_addr, - file: BlobFile::with_source( - source, - position, - size, - BlobKind::External, - Some(uri_or_path), - ), - }); + ArrowDataType::LargeList(_) => { + let list_array = array.as_list::(); + materialize_blob_v2_list_array::(dataset, field, list_array, row_addrs).await } + _ => Ok(array), } } + .boxed() +} - Ok(files) +async fn materialize_blob_v2_list_array( + dataset: &Arc, + field: &LanceField, + list_array: &GenericListArray, + row_addrs: Arc<[u64]>, +) -> Result { + let offsets = list_array.value_offsets(); + let values_start = offsets[0].as_usize(); + let values_end = offsets[list_array.len()].as_usize(); + if values_end < values_start { + return Err(Error::internal(format!( + "List field '{}' has invalid offsets while materializing blob v2 binary scan", + field.name + ))); + } + + let values_len = values_end - values_start; + let mut normalized_offsets = Vec::with_capacity(list_array.len() + 1); + normalized_offsets.push(O::usize_as(0)); + let mut child_row_addrs = Vec::with_capacity(values_len); + for row_idx in 0..list_array.len() { + let start = offsets[row_idx].as_usize(); + let end = offsets[row_idx + 1].as_usize(); + if end < start { + return Err(Error::internal(format!( + "List field '{}' has decreasing offsets while materializing blob v2 binary scan", + field.name + ))); + } + let row_addr = row_addrs.get(row_idx).copied().ok_or_else(|| { + Error::internal(format!( + "List field '{}' row address count {} did not match row count {}", + field.name, + row_addrs.len(), + list_array.len() + )) + })?; + for _ in start..end { + child_row_addrs.push(row_addr); + } + normalized_offsets.push(O::usize_as(end - values_start)); + } + let child_row_addrs: Arc<[u64]> = child_row_addrs.into(); + let child = field.children.first().ok_or_else(|| { + Error::internal(format!( + "List field '{}' missing child while materializing blob v2 binary scan", + field.name + )) + })?; + let values = list_array.values().slice(values_start, values_len); + let values = materialize_blob_v2_binary_array(dataset, child, values, child_row_addrs).await?; + let child_field = public_blob_v2_binary_output_field(child.clone()); + let list_array = GenericListArray::::try_new( + Arc::new(ArrowField::from(&child_field)), + OffsetBuffer::new(ScalarBuffer::from(normalized_offsets)), + values, + list_array.nulls().cloned(), + )?; + Ok(Arc::new(list_array)) +} + +async fn materialize_blob_v2_descriptors( + dataset: &Arc, + blob_field_id: u32, + descriptions: &StructArray, + row_addrs: &[u64], +) -> Result { + if descriptions.len() != row_addrs.len() { + return Err(Error::internal(format!( + "blob v2 descriptor count {} did not match row address count {}", + descriptions.len(), + row_addrs.len() + ))); + } + match blob_version_from_descriptions(descriptions)? { + BlobVersion::V1 => { + return Err(Error::not_supported( + "Blob v2 binary materialization received a legacy blob descriptor".to_string(), + )); + } + BlobVersion::V2 => {} + } + + let columns = BlobV2DescriptorColumns::new(descriptions); + let mut read_context = BlobV2ReadContext::new(dataset, blob_field_id); + let mut entries = Vec::with_capacity(descriptions.len()); + let mut payloads = vec![None; descriptions.len()]; + + for (idx, row_addr) in row_addrs.iter().copied().enumerate() { + if descriptions.is_null(idx) || columns.kinds.is_null(idx) { + continue; + } + + let kind = BlobKind::try_from(columns.kinds.value(idx))?; + if matches!(kind, BlobKind::Inline) + && columns.positions.value(idx) == 0 + && columns.sizes.value(idx) == 0 + { + payloads[idx] = Some(Bytes::new()); + continue; + } + + let file = read_context + .collect_file(&columns, idx, row_addr) + .await? + .ok_or_else(|| { + Error::internal(format!( + "blob v2 descriptor at index {idx} unexpectedly resolved to null" + )) + })?; + entries.push(BlobEntry { + selection_index: idx, + row_address: row_addr, + file, + requested_range: None, + }); + } + + let blobs = execute_blob_entries(entries, dataset.object_store.io_parallelism(), None).await?; + for blob in blobs { + let payload = payloads.get_mut(blob.selection_index).ok_or_else(|| { + Error::internal(format!( + "blob result selection index {} exceeded descriptor count {}", + blob.selection_index, + descriptions.len() + )) + })?; + if payload.replace(blob.data).is_some() { + return Err(Error::internal(format!( + "blob result selection index {} was produced more than once", + blob.selection_index + ))); + } + } + + let payload_capacity = payloads.iter().flatten().map(Bytes::len).sum::(); + let mut builder = LargeBinaryBuilder::with_capacity(descriptions.len(), payload_capacity); + for (idx, payload) in payloads.into_iter().enumerate() { + if descriptions.is_null(idx) || columns.kinds.is_null(idx) { + builder.append_null(); + } else { + let payload = payload.ok_or_else(|| { + Error::internal(format!( + "blob v2 descriptor at index {idx} did not produce a payload" + )) + })?; + builder.append_value(payload); + } + } + + Ok(Arc::new(builder.finish())) +} + +struct BlobV2ReadContext<'a> { + dataset: &'a Arc, + blob_field_id: u32, + fragment_cache: HashMap, + store_cache: HashMap>, + external_base_path_cache: HashMap, + source_cache: HashMap>, +} + +impl<'a> BlobV2ReadContext<'a> { + fn new(dataset: &'a Arc, blob_field_id: u32) -> Self { + Self { + dataset, + blob_field_id, + fragment_cache: HashMap::new(), + store_cache: HashMap::new(), + external_base_path_cache: HashMap::new(), + source_cache: HashMap::new(), + } + } + + async fn collect_file( + &mut self, + columns: &BlobV2DescriptorColumns<'_>, + idx: usize, + row_addr: u64, + ) -> Result> { + if columns.is_null_blob(idx) { + return Ok(None); + } + + let kind = BlobKind::try_from(columns.kinds.value(idx))?; + let file = match kind { + BlobKind::Inline => self.collect_inline(columns, idx, row_addr).await?, + BlobKind::Dedicated => self.collect_dedicated(columns, idx, row_addr).await?, + BlobKind::Packed => self.collect_packed(columns, idx, row_addr).await?, + BlobKind::External => self.collect_external(columns, idx).await?, + }; + + Ok(Some(file)) + } + + async fn blob_read_location(&mut self, row_addr: u64) -> Result { + resolve_blob_read_location( + self.dataset, + self.blob_field_id, + row_addr, + &mut self.fragment_cache, + &mut self.store_cache, + ) + .await + } + + async fn collect_inline( + &mut self, + columns: &BlobV2DescriptorColumns<'_>, + idx: usize, + row_addr: u64, + ) -> Result { + let position = columns.positions.value(idx); + let size = columns.sizes.value(idx); + let location = self.blob_read_location(row_addr).await?; + let source = shared_blob_source( + &mut self.source_cache, + location.object_store, + &location.data_file_path, + ); + Ok(BlobFile::with_source( + source, + position, + size, + BlobKind::Inline, + None, + )) + } + + async fn collect_dedicated( + &mut self, + columns: &BlobV2DescriptorColumns<'_>, + idx: usize, + row_addr: u64, + ) -> Result { + let blob_id = columns.blob_ids.value(idx); + let size = columns.sizes.value(idx); + let location = self.blob_read_location(row_addr).await?; + let path = blob_path(&location.data_file_dir, &location.data_file_key, blob_id); + let source = shared_blob_source(&mut self.source_cache, location.object_store, &path); + Ok(BlobFile::with_source( + source, + 0, + size, + BlobKind::Dedicated, + None, + )) + } + + async fn collect_packed( + &mut self, + columns: &BlobV2DescriptorColumns<'_>, + idx: usize, + row_addr: u64, + ) -> Result { + let blob_id = columns.blob_ids.value(idx); + let size = columns.sizes.value(idx); + let position = columns.positions.value(idx); + let location = self.blob_read_location(row_addr).await?; + let path = blob_path(&location.data_file_dir, &location.data_file_key, blob_id); + let source = shared_blob_source(&mut self.source_cache, location.object_store, &path); + Ok(BlobFile::with_source( + source, + position, + size, + BlobKind::Packed, + None, + )) + } + + async fn collect_external( + &mut self, + columns: &BlobV2DescriptorColumns<'_>, + idx: usize, + ) -> Result { + let uri_or_path = columns.blob_uris.value(idx).to_string(); + let position = columns.positions.value(idx); + let size = columns.sizes.value(idx); + let base_id = columns.blob_ids.value(idx); + let (object_store, path) = if base_id == 0 { + let registry = self.dataset.session.store_registry(); + let params = self + .dataset + .store_params + .as_ref() + .map(|p| Arc::new((**p).clone())) + .unwrap_or_else(|| Arc::new(ObjectStoreParams::default())); + ObjectStore::from_uri_and_params(registry, &uri_or_path, ¶ms).await? + } else { + let object_store = if let Some(store) = self.store_cache.get(&base_id) { + store.clone() + } else { + let store = self.dataset.object_store(Some(base_id)).await?; + self.store_cache.insert(base_id, store.clone()); + store + }; + let base_root = if let Some(path) = self.external_base_path_cache.get(&base_id) { + path.clone() + } else { + let base = self + .dataset + .manifest + .base_paths + .get(&base_id) + .ok_or_else(|| { + Error::invalid_input(format!( + "External blob references unknown base_id {}", + base_id + )) + })?; + let path = base.extract_path(self.dataset.session.store_registry())?; + self.external_base_path_cache.insert(base_id, path.clone()); + path + }; + let path = join_base_and_relative_path(&base_root, &uri_or_path)?; + (object_store, path) + }; + let size = if size > 0 { + size + } else { + object_store.size(&path).await? + }; + let source = shared_blob_source(&mut self.source_cache, object_store, &path); + Ok(BlobFile::with_source( + source, + position, + size, + BlobKind::External, + Some(uri_or_path), + )) + } } fn normalize_external_absolute_uri(uri: &str) -> Result { @@ -2259,7 +3822,10 @@ fn data_file_key_from_path(path: &str) -> &str { mod tests { use std::collections::HashMap; use std::ops::Range; - use std::sync::Arc; + use std::sync::{ + Arc, + atomic::{AtomicUsize, Ordering}, + }; use std::time::Duration; use arrow::{ @@ -2271,11 +3837,12 @@ mod tests { Array, ArrayRef, Int32Array, LargeBinaryArray, RecordBatchIterator, StringArray, StructArray, UInt8Array, UInt32Array, UInt64Array, }; + use arrow_buffer::NullBuffer; use arrow_schema::{DataType, Field, Schema}; use async_trait::async_trait; use bytes::Bytes; use chrono::Utc; - use futures::{StreamExt, TryStreamExt, future::try_join_all}; + use futures::{StreamExt, TryStreamExt}; use lance_arrow::{ ARROW_EXT_NAME_KEY, BLOB_DEDICATED_SIZE_THRESHOLD_META_KEY, BLOB_INLINE_SIZE_THRESHOLD_META_KEY, BLOB_META_KEY, BLOB_PACK_FILE_SIZE_THRESHOLD_META_KEY, @@ -2295,30 +3862,31 @@ mod tests { MultipartUpload, ObjectMeta, PutMultipartOptions, PutOptions, PutPayload, PutResult, path::Path, }; - use tokio::sync::Notify; + use rstest::rstest; + use tokio::sync::{Notify, Semaphore}; use url::Url; use lance_core::{ - Error, Result, + Error, ROW_ADDR, ROW_CREATED_AT_VERSION, ROW_ID, ROW_LAST_UPDATED_AT_VERSION, Result, utils::tempfile::{TempDir, TempStrDir}, }; use lance_datagen::{BatchCount, RowCount, array}; - use lance_file::{ - version::LanceFileVersion, - writer::{FileWriter, FileWriterOptions}, - }; + use lance_file::{version::LanceFileVersion, writer::FileWriterOptions}; use uuid::Uuid; use super::{ - BlobEntry, BlobFile, BlobSource, ExternalBaseCandidate, ExternalBaseResolver, - ReadBlobsExecution, collect_blob_entries_v1, data_file_key_from_path, - execute_blob_read_plan, plan_blob_read_plans, + BlobEntry, BlobFile, BlobRangeRequest, BlobReadRange, BlobSource, ExternalBaseCandidate, + ExternalBaseResolver, ReadBlobsExecution, blob_version_from_descriptions, + collect_blob_files_v1, data_file_key_from_path, execute_blob_entries, + execute_blob_read_batches_stream, execute_blob_read_plan, plan_blob_read_batches, + plan_blob_read_plans, }; use crate::{ Dataset, blob::{BlobArrayBuilder, BlobDescriptorArrayBuilder, PackedBlobWriter, blob_field}, dataset::{ CommitBuilder, ExternalBlobMode, WriteMode, WriteParams, + scanner::MaterializationStyle, transaction::{DataReplacementGroup, Operation, Transaction}, }, utils::test::TestDatasetGenerator, @@ -2336,6 +3904,37 @@ mod tests { expected: Vec, } + #[test] + fn test_blob_version_rejects_malformed_v2_descriptor_layout() { + let descriptions = StructArray::try_new( + vec![ + Field::new("kind", DataType::UInt8, false), + Field::new("position", DataType::UInt64, false), + Field::new("size", DataType::UInt32, false), + Field::new("blob_id", DataType::UInt32, false), + Field::new("blob_uri", DataType::Utf8, false), + ] + .into(), + vec![ + Arc::new(UInt8Array::from(vec![BlobKind::Inline as u8])), + Arc::new(UInt64Array::from(vec![0])), + Arc::new(UInt32Array::from(vec![0])), + Arc::new(UInt32Array::from(vec![0])), + Arc::new(StringArray::from(vec![""])), + ], + None, + ) + .unwrap(); + + let error = blob_version_from_descriptions(&descriptions).unwrap_err(); + assert!(matches!(error, Error::InvalidInput { .. })); + assert!( + error + .to_string() + .contains("expected v1 descriptor or v2 descriptor layout") + ); + } + fn nested_blob_v2_batch(blob_array: ArrayRef) -> (Arc, RecordBatch) { let blob_field = blob_field("blob", true); let info_fields = vec![Field::new("name", DataType::Utf8, false), blob_field]; @@ -2538,8 +4137,12 @@ mod tests { #[derive(Debug)] struct RecordingRangeObjectStore { data: Bytes, - gate: Option>, + gate: Option>, requested_ranges: std::sync::Mutex>>, + started_blob_requests: AtomicUsize, + active_blob_requests: AtomicUsize, + peak_active_blob_requests: AtomicUsize, + request_started: Notify, } impl RecordingRangeObjectStore { @@ -2548,17 +4151,39 @@ mod tests { data, gate: None, requested_ranges: std::sync::Mutex::new(Vec::new()), + started_blob_requests: AtomicUsize::new(0), + active_blob_requests: AtomicUsize::new(0), + peak_active_blob_requests: AtomicUsize::new(0), + request_started: Notify::new(), } } - fn with_gate(data: Bytes, gate: Arc) -> Self { + fn with_gate(data: Bytes, gate: Arc) -> Self { Self { data, gate: Some(gate), requested_ranges: std::sync::Mutex::new(Vec::new()), + started_blob_requests: AtomicUsize::new(0), + active_blob_requests: AtomicUsize::new(0), + peak_active_blob_requests: AtomicUsize::new(0), + request_started: Notify::new(), } } + async fn wait_for_blob_requests(&self, expected: usize) { + loop { + let request_started = self.request_started.notified(); + if self.started_blob_requests.load(Ordering::Acquire) >= expected { + return; + } + request_started.await; + } + } + + fn peak_active_blob_requests(&self) -> usize { + self.peak_active_blob_requests.load(Ordering::Acquire) + } + fn requested_ranges(&self) -> Vec> { self.requested_ranges.lock().unwrap().clone() } @@ -2622,10 +4247,21 @@ mod tests { } }; let is_full_object_probe = range.start == 0 && range.end == self.data.len() as u64; - if !is_full_object_probe && let Some(gate) = &self.gate { - gate.notified().await; - } self.requested_ranges.lock().unwrap().push(range.clone()); + if !is_full_object_probe { + let active = self.active_blob_requests.fetch_add(1, Ordering::AcqRel) + 1; + self.peak_active_blob_requests + .fetch_max(active, Ordering::AcqRel); + self.started_blob_requests.fetch_add(1, Ordering::AcqRel); + self.request_started.notify_waiters(); + if let Some(gate) = &self.gate { + gate.acquire() + .await + .expect("test gate should remain open") + .forget(); + } + self.active_blob_requests.fetch_sub(1, Ordering::AcqRel); + } let bytes = self.data.slice(range.start as usize..range.end as usize); Ok(GetResult { payload: GetResultPayload::Stream( @@ -2668,9 +4304,10 @@ mod tests { } } - fn recording_range_store_with_url( + fn recording_range_store_with_url_and_block_size( data: Bytes, url: &str, + block_size: Option, ) -> (Arc, Arc) { const TEST_RANGE_STORE_SIZE: usize = 128 * 1024; let mut padded = vec![0; TEST_RANGE_STORE_SIZE.max(data.len())]; @@ -2679,7 +4316,7 @@ mod tests { let store = Arc::new(ObjectStore::new( inner.clone() as Arc, Url::parse(url).unwrap(), - None, + block_size, None, false, true, @@ -2690,22 +4327,40 @@ mod tests { (store, inner) } + fn recording_range_store_with_url( + data: Bytes, + url: &str, + ) -> (Arc, Arc) { + recording_range_store_with_url_and_block_size(data, url, None) + } + fn recording_range_store(data: Bytes) -> (Arc, Arc) { recording_range_store_with_url(data, "mock://recording/blob-range-tests") } - fn gated_range_store( + fn recording_range_store_with_block_size( data: Bytes, - url: &str, - ) -> ( - Arc, + block_size: usize, + ) -> (Arc, Arc) { + recording_range_store_with_url_and_block_size( + data, + "mock://recording/blob-range-tests", + Some(block_size), + ) + } + + fn gated_range_store( + data: Bytes, + url: &str, + ) -> ( + Arc, Arc, - Arc, + Arc, ) { const TEST_RANGE_STORE_SIZE: usize = 128 * 1024; let mut padded = vec![0; TEST_RANGE_STORE_SIZE.max(data.len())]; padded[..data.len()].copy_from_slice(data.as_ref()); - let gate = Arc::new(Notify::new()); + let gate = Arc::new(Semaphore::new(0)); let inner = Arc::new(RecordingRangeObjectStore::with_gate( Bytes::from(padded), gate.clone(), @@ -2750,6 +4405,55 @@ mod tests { } } + async fn stable_row_id_blob_dataset_with_deleted_row() -> (TempStrDir, Arc, Vec) { + let test_dir = TempStrDir::default(); + let mut blob_builder = BlobArrayBuilder::new(3); + blob_builder.push_bytes(b"AAaa").unwrap(); + blob_builder.push_bytes(b"BBbb").unwrap(); + blob_builder.push_bytes(b"CCcc").unwrap(); + let schema = Arc::new(Schema::new(vec![ + blob_field("blob", true), + Field::new("idx", DataType::UInt64, false), + ])); + let batch = RecordBatch::try_new( + schema.clone(), + vec![ + blob_builder.finish().unwrap(), + Arc::new(UInt64Array::from(vec![0, 1, 2])), + ], + ) + .unwrap(); + let mut dataset = Dataset::write( + RecordBatchIterator::new(vec![Ok(batch)], schema), + &test_dir, + Some(WriteParams { + data_storage_version: Some(LanceFileVersion::V2_2), + enable_stable_row_ids: true, + ..Default::default() + }), + ) + .await + .unwrap(); + + let with_row_ids = dataset + .scan() + .project(&["idx"]) + .unwrap() + .with_row_id() + .try_into_batch() + .await + .unwrap(); + let indices = with_row_ids["idx"].as_primitive::(); + let ids = with_row_ids[ROW_ID].as_primitive::(); + let mut row_ids = vec![0; with_row_ids.num_rows()]; + for (index, row_id) in indices.values().iter().zip(ids.values()) { + row_ids[*index as usize] = *row_id; + } + + dataset.delete("idx = 1").await.unwrap(); + (test_dir, Arc::new(dataset), row_ids) + } + async fn create_multi_base_blob_v2_fixture( payload: Vec, dedicated_threshold: Option, @@ -2843,7 +4547,7 @@ mod tests { for (actual_idx, (expected_batch_idx, expected_row_idx)) in [(5, 5), (6, 7), (8, 3)].iter().enumerate() { - let val = blobs[actual_idx].read().await.unwrap(); + let val = blobs[actual_idx].as_ref().unwrap().read().await.unwrap(); let expected = fixture.data[*expected_batch_idx] .column(1) .as_binary::() @@ -2889,7 +4593,7 @@ mod tests { .column(1) .as_binary::() .value(*expected_row_idx); - assert_eq!(blobs[actual_idx].data.as_ref(), expected); + assert_eq!(blobs[actual_idx].data.as_deref(), Some(expected)); } } @@ -2924,10 +4628,84 @@ mod tests { let blobs2 = fixture.dataset.take_blobs(&row_ids, "blobs").await.unwrap(); for (blob1, blob2) in blobs.iter().zip(blobs2.iter()) { + let blob1 = blob1.as_ref().unwrap(); + let blob2 = blob2.as_ref().unwrap(); assert_eq!(blob1.position(), blob2.position()); assert_eq!(blob1.size(), blob2.size()); assert_eq!(blob1.data_path(), blob2.data_path()); } + + // Unsorted indices spanning fragments use the take remapping path, which + // carries _rowaddr internally and must still preserve the requested order. + let indices = [33_u64, 17, 5, 28, 12, 39]; + let blobs = fixture + .dataset + .take_blobs_by_indices(&indices, "blobs") + .await + .unwrap(); + for (blob, index) in blobs.iter().zip(indices) { + let actual = blob.as_ref().unwrap().read().await.unwrap(); + let index = index as usize; + let expected = fixture.data[index / 10] + .column(1) + .as_binary::() + .value(index % 10); + assert_eq!(actual.as_ref(), expected); + } + } + + #[rstest] + #[case::all_valid_first(false)] + #[case::nullable_first(true)] + #[tokio::test] + async fn test_write_blob_batches_with_mixed_nullability(#[case] nulls_first: bool) { + let test_dir = TempStrDir::default(); + let schema = Arc::new(Schema::new(vec![ + Field::new("blob", DataType::LargeBinary, true).with_metadata(HashMap::from([( + BLOB_META_KEY.to_string(), + "true".to_string(), + )])), + ])); + let batch = |values: Vec>| { + RecordBatch::try_new( + schema.clone(), + vec![Arc::new(LargeBinaryArray::from(values))], + ) + .unwrap() + }; + + // Definition-level semantics come from each batch's validity bitmap. Both + // transitions must start a new descriptor page so nulls remain distinct + // from valid empty blobs. + let all_valid = batch(vec![Some(b"a".as_slice()), Some(b"".as_slice())]); + let with_null = batch(vec![Some(b"c".as_slice()), None]); + let (batches, expected) = if nulls_first { + ( + vec![with_null, all_valid], + vec![Some(b"c".as_slice()), None, Some(b"a"), Some(b"")], + ) + } else { + ( + vec![all_valid, with_null], + vec![Some(b"a".as_slice()), Some(b""), Some(b"c"), None], + ) + }; + + let reader = RecordBatchIterator::new(batches.into_iter().map(Ok), schema); + let dataset = Arc::new(Dataset::write(reader, &test_dir, None).await.unwrap()); + + let blobs = dataset + .take_blobs_by_indices(&[0, 1, 2, 3], "blob") + .await + .unwrap(); + assert_eq!(blobs.len(), expected.len()); + for (row_idx, (blob, expected)) in blobs.into_iter().zip(expected).enumerate() { + let actual = match blob { + Some(blob) => Some(blob.read().await.unwrap()), + None => None, + }; + assert_eq!(actual.as_deref(), expected, "row {row_idx}"); + } } #[tokio::test] @@ -2940,7 +4718,63 @@ mod tests { } #[tokio::test] - async fn test_collect_blob_entries_v1_rejects_missing_fragment() { + async fn test_take_blobs_by_ids_rejects_deleted_stable_row_id() { + let (_test_dir, dataset, row_ids) = stable_row_id_blob_dataset_with_deleted_row().await; + + let err = dataset.take_blobs(&row_ids, "blob").await.unwrap_err(); + + assert!(matches!(err, Error::InvalidInput { .. })); + assert!( + err.to_string() + .contains("Could not resolve all requested row IDs") + ); + } + + #[tokio::test] + async fn test_read_blobs_by_ids_rejects_deleted_stable_row_id() { + let (_test_dir, dataset, row_ids) = stable_row_id_blob_dataset_with_deleted_row().await; + + let err = dataset + .read_blobs("blob") + .unwrap() + .with_row_ids(row_ids) + .execute() + .await + .unwrap_err(); + + assert!(matches!(err, Error::InvalidInput { .. })); + assert!( + err.to_string() + .contains("Could not resolve all requested row IDs") + ); + } + + #[tokio::test] + async fn test_read_blob_ranges_by_ids_rejects_deleted_stable_row_id() { + let (_test_dir, dataset, row_ids) = stable_row_id_blob_dataset_with_deleted_row().await; + let requests = [ + BlobRangeRequest::new(row_ids[0], 0, 2), + BlobRangeRequest::new(row_ids[1], 0, 2), + BlobRangeRequest::new(row_ids[2], 2, 2), + ]; + + let err = dataset + .read_blob_ranges("blob") + .unwrap() + .with_row_ids(requests) + .execute() + .await + .unwrap_err(); + + assert!(matches!(err, Error::InvalidInput { .. })); + assert!( + err.to_string() + .contains("Could not resolve all requested row IDs") + ); + } + + #[tokio::test] + async fn test_collect_blob_files_v1_rejects_missing_fragment() { let fixture = BlobTestFixture::new().await; let blob_field_id = fixture.dataset.schema().project(&["blobs"]).unwrap().fields[0].id as u32; @@ -2956,9 +4790,8 @@ mod tests { ]); let row_addrs = UInt64Array::from(vec![(999_u64 << 32) | 7]); - let err = - collect_blob_entries_v1(&fixture.dataset, blob_field_id, &descriptions, &row_addrs) - .unwrap_err(); + let err = collect_blob_files_v1(&fixture.dataset, blob_field_id, &descriptions, &row_addrs) + .unwrap_err(); assert!(err.to_string().contains("references missing fragment")); } @@ -3079,7 +4912,7 @@ mod tests { // Verify we can read the blob content for blob in &blobs { - let content = blob.read().await.unwrap(); + let content = blob.as_ref().unwrap().read().await.unwrap(); assert!(!content.is_empty(), "Blob content should not be empty"); } @@ -3092,7 +4925,7 @@ mod tests { // Verify we can read the blob content from second fragment for blob in &blobs { - let content = blob.read().await.unwrap(); + let content = blob.as_ref().unwrap().read().await.unwrap(); assert!(!content.is_empty(), "Blob content should not be empty"); } @@ -3280,7 +5113,10 @@ mod tests { .unwrap(); assert_eq!(blob_files.len(), payloads.len()); for (blob_file, expected) in blob_files.iter().zip(payloads) { - assert_eq!(blob_file.read().await.unwrap().as_ref(), expected); + assert_eq!( + blob_file.as_ref().unwrap().read().await.unwrap().as_ref(), + expected + ); } let read_blobs = dataset @@ -3292,7 +5128,7 @@ mod tests { .unwrap(); assert_eq!(read_blobs.len(), payloads.len()); for (read_blob, expected) in read_blobs.iter().zip(payloads) { - assert_eq!(read_blob.data.as_ref(), expected); + assert_eq!(read_blob.data.as_deref(), Some(expected)); } } @@ -3338,8 +5174,8 @@ mod tests { .unwrap(); assert_eq!(blobs.len(), 2); - let first = blobs[0].read().await.unwrap(); - let second = blobs[1].read().await.unwrap(); + let first = blobs[0].as_ref().unwrap().read().await.unwrap(); + let second = blobs[1].as_ref().unwrap().read().await.unwrap(); assert_eq!(first.as_ref(), b"hello"); assert_eq!(second.as_ref(), b"world"); } @@ -3389,7 +5225,10 @@ mod tests { let blobs = dataset.take_blobs_by_indices(&[0], "blob").await.unwrap(); assert_eq!(blobs.len(), 1); - assert_eq!(blobs[0].read().await.unwrap().as_ref(), b"prepared-inline"); + assert_eq!( + blobs[0].as_ref().unwrap().read().await.unwrap().as_ref(), + b"prepared-inline" + ); } #[tokio::test] @@ -3455,13 +5294,10 @@ mod tests { .unwrap(); let object_writer = dataset.object_store.create(&data_file_path).await.unwrap(); - let mut file_writer = FileWriter::try_new( + let mut file_writer = lance_file::versions::v2_2::create_writer( object_writer, crate::datatypes::Schema::try_from(append_schema.as_ref()).unwrap(), - FileWriterOptions { - format_version: Some(LanceFileVersion::V2_2), - ..Default::default() - }, + FileWriterOptions::default(), ) .unwrap(); file_writer.write_batch(&replacement_batch).await.unwrap(); @@ -3489,8 +5325,9 @@ mod tests { let blobs = dataset.take_blobs_by_indices(&[0], "blob").await.unwrap(); assert_eq!(blobs.len(), 1); - assert_eq!(blobs[0].read().await.unwrap().as_ref(), b"prepared-packed"); - assert_eq!(blobs[0].kind(), BlobKind::Packed); + let blob = blobs[0].as_ref().unwrap(); + assert_eq!(blob.read().await.unwrap().as_ref(), b"prepared-packed"); + assert_eq!(blob.kind(), BlobKind::Packed); } #[tokio::test] @@ -3571,8 +5408,14 @@ mod tests { .await .unwrap(); assert_eq!(blobs.len(), 2); - assert_eq!(blobs[0].read().await.unwrap().as_ref(), b"initial"); - assert_eq!(blobs[1].read().await.unwrap().as_ref(), b"append"); + assert_eq!( + blobs[0].as_ref().unwrap().read().await.unwrap().as_ref(), + b"initial" + ); + assert_eq!( + blobs[1].as_ref().unwrap().read().await.unwrap().as_ref(), + b"append" + ); } #[tokio::test] @@ -3634,13 +5477,10 @@ mod tests { RecordBatch::try_new(replacement_schema.clone(), vec![info_array]).unwrap(); let object_writer = dataset.object_store.create(&data_file_path).await.unwrap(); - let mut file_writer = FileWriter::try_new( + let mut file_writer = lance_file::versions::v2_2::create_writer( object_writer, crate::datatypes::Schema::try_from(replacement_schema.as_ref()).unwrap(), - FileWriterOptions { - format_version: Some(LanceFileVersion::V2_2), - ..Default::default() - }, + FileWriterOptions::default(), ) .unwrap(); file_writer.write_batch(&replacement_batch).await.unwrap(); @@ -3672,10 +5512,10 @@ mod tests { .unwrap(); assert_eq!(blobs.len(), 1); assert_eq!( - blobs[0].read().await.unwrap().as_ref(), + blobs[0].as_ref().unwrap().read().await.unwrap().as_ref(), b"nested-replacement" ); - assert_eq!(blobs[0].kind(), BlobKind::Packed); + assert_eq!(blobs[0].as_ref().unwrap().kind(), BlobKind::Packed); } #[tokio::test] @@ -3741,13 +5581,10 @@ mod tests { RecordBatch::try_new(replacement_schema.clone(), vec![info_array]).unwrap(); let object_writer = dataset.object_store.create(&data_file_path).await.unwrap(); - let mut file_writer = FileWriter::try_new( + let mut file_writer = lance_file::versions::v2_2::create_writer( object_writer, crate::datatypes::Schema::try_from(replacement_schema.as_ref()).unwrap(), - FileWriterOptions { - format_version: Some(LanceFileVersion::V2_2), - ..Default::default() - }, + FileWriterOptions::default(), ) .unwrap(); file_writer.write_batch(&replacement_batch).await.unwrap(); @@ -3778,7 +5615,807 @@ mod tests { let dataset = Arc::new( Dataset::write( - reader, + reader, + &test_dir, + Some(WriteParams { + data_storage_version: Some(LanceFileVersion::V2_2), + ..Default::default() + }), + ) + .await + .unwrap(), + ); + + let info_batch = dataset + .scan() + .project(&["info"]) + .unwrap() + .try_into_batch() + .await + .unwrap(); + let blob_desc = info_batch + .column(0) + .as_struct() + .column_by_name("blob") + .unwrap() + .as_struct(); + assert_eq!( + blob_desc + .column_by_name("kind") + .unwrap() + .as_primitive::() + .value(0), + BlobKind::Inline as u8 + ); + assert_eq!( + blob_desc + .column_by_name("kind") + .unwrap() + .as_primitive::() + .value(1), + BlobKind::Packed as u8 + ); + + let blobs = dataset + .take_blobs_by_indices(&[0, 1], "info.blob") + .await + .unwrap(); + assert_eq!(blobs.len(), 2); + assert_eq!( + blobs[0].as_ref().unwrap().read().await.unwrap().as_ref(), + b"hello" + ); + assert_eq!( + blobs[1].as_ref().unwrap().read().await.unwrap().as_ref(), + packed_payload.as_slice() + ); + + let null_blobs = dataset + .take_blobs_by_indices(&[2], "info.blob") + .await + .unwrap(); + assert_eq!(null_blobs.len(), 1); + assert!(null_blobs[0].is_none()); + + let filtered = dataset + .scan() + .project(&["info"]) + .unwrap() + .filter("info.blob IS NOT NULL") + .unwrap() + .try_into_batch() + .await + .unwrap(); + assert_eq!(filtered.num_rows(), 2); + } + + #[tokio::test] + async fn test_write_and_scan_list_blob_v2_descriptions() { + let test_dir = TempStrDir::default(); + let packed_payload = vec![0x4B; super::INLINE_MAX + 1024]; + + let mut blob_builder = BlobArrayBuilder::new(4); + blob_builder.push_bytes(b"hello").unwrap(); + blob_builder.push_null().unwrap(); + blob_builder.push_bytes(&packed_payload).unwrap(); + blob_builder.push_bytes(b"tail").unwrap(); + let blob_values = blob_builder.finish().unwrap(); + + let item_field = Arc::new(blob_field("item", true)); + let list_array: ArrayRef = Arc::new( + arrow_array::ListArray::try_new( + item_field.clone(), + arrow_buffer::OffsetBuffer::new(arrow_buffer::ScalarBuffer::from(vec![ + 0i32, 3, 3, 3, 4, + ])), + blob_values, + Some(arrow_buffer::NullBuffer::from(vec![ + true, true, false, true, + ])), + ) + .unwrap(), + ); + + let schema = Arc::new(Schema::new(vec![ + Field::new("blobs", DataType::List(item_field), true), + Field::new("id", DataType::Int32, false), + ])); + let batch = RecordBatch::try_new( + schema.clone(), + vec![list_array, Arc::new(Int32Array::from(vec![0, 1, 2, 3]))], + ) + .unwrap(); + let reader = RecordBatchIterator::new(vec![batch].into_iter().map(Ok), schema); + + let dataset = Arc::new( + Dataset::write( + reader, + &test_dir, + Some(WriteParams { + data_storage_version: Some(LanceFileVersion::V2_2), + enable_stable_row_ids: true, + ..Default::default() + }), + ) + .await + .unwrap(), + ); + + let descriptions = dataset + .scan() + .project(&["blobs"]) + .unwrap() + .try_into_batch() + .await + .unwrap(); + let lists = descriptions.column(0).as_list::(); + assert_eq!(lists.offsets().inner().as_ref(), &[0, 3, 3, 3, 4]); + assert!(lists.is_valid(0)); + assert!(lists.is_valid(1)); + assert!(lists.is_null(2)); + assert!(lists.is_valid(3)); + + let DataType::List(descriptor_field) = lists.data_type() else { + panic!("unexpected list type: {}", lists.data_type()); + }; + assert!(matches!(descriptor_field.data_type(), DataType::Struct(_))); + assert!(!descriptor_field.metadata().contains_key(ARROW_EXT_NAME_KEY)); + let descriptors = lists.values().as_struct(); + assert_eq!(descriptors.fields().len(), 5); + assert_eq!(descriptors.fields()[0].name(), "kind"); + assert!(descriptors.is_valid(0)); + assert!(descriptors.is_null(1)); + assert!(descriptors.is_valid(2)); + assert!(descriptors.is_valid(3)); + let kinds = descriptors + .column_by_name("kind") + .unwrap() + .as_primitive::(); + assert_eq!(kinds.value(0), BlobKind::Inline as u8); + assert_eq!(kinds.value(2), BlobKind::Packed as u8); + assert_eq!(kinds.value(3), BlobKind::Inline as u8); + + let filtered = dataset + .scan() + .project(&["blobs"]) + .unwrap() + .filter("blobs IS NOT NULL") + .unwrap() + .try_into_batch() + .await + .unwrap(); + assert_eq!(filtered.num_rows(), 3); + + let mut scanner = dataset.scan(); + scanner.blob_handling(BlobHandling::AllBinary); + let bytes = scanner + .project(&["blobs"]) + .unwrap() + .try_into_batch() + .await + .unwrap(); + let lists = bytes.column(0).as_list::(); + assert_eq!(lists.offsets().inner().as_ref(), &[0, 3, 3, 3, 4]); + assert!(lists.is_valid(0)); + assert!(lists.is_valid(1)); + assert!(lists.is_null(2)); + assert!(lists.is_valid(3)); + let DataType::List(value_field) = lists.data_type() else { + panic!("unexpected list type: {}", lists.data_type()); + }; + assert_eq!(value_field.data_type(), &DataType::LargeBinary); + assert!(!value_field.metadata().contains_key(ARROW_EXT_NAME_KEY)); + let values = lists.values().as_binary::(); + assert_eq!(values.value(0), b"hello"); + assert!(values.is_null(1)); + assert_eq!(values.value(2), packed_payload.as_slice()); + assert_eq!(values.value(3), b"tail"); + + for (filter, materialization_style) in [ + (None, MaterializationStyle::Heuristic), + (Some("id >= 2"), MaterializationStyle::Heuristic), + (Some("id >= 2"), MaterializationStyle::AllEarly), + ] { + let mut scanner = dataset.scan(); + scanner.blob_handling(BlobHandling::AllBinary); + scanner.materialization_style(materialization_style); + scanner + .project(&["blobs", ROW_LAST_UPDATED_AT_VERSION, ROW_CREATED_AT_VERSION]) + .unwrap() + .with_row_id() + .with_row_address(); + if let Some(filter) = filter { + scanner.filter(filter).unwrap(); + } + + let expected_schema = scanner.schema().await.unwrap(); + let batch = scanner.try_into_batch().await.unwrap(); + assert_eq!(batch.schema().as_ref(), expected_schema.as_ref()); + assert_eq!(batch.num_rows(), if filter.is_some() { 2 } else { 4 }); + for column in [ + ROW_ID, + ROW_ADDR, + ROW_LAST_UPDATED_AT_VERSION, + ROW_CREATED_AT_VERSION, + ] { + assert!( + batch.column_by_name(column).is_some(), + "requested system column {column} was missing" + ); + } + } + } + + #[tokio::test] + async fn test_write_and_scan_struct_nested_list_blob_v2() { + let test_dir = TempStrDir::default(); + + let mut blob_builder = BlobArrayBuilder::new(2); + blob_builder.push_bytes(b"nested").unwrap(); + blob_builder.push_null().unwrap(); + let blob_values = blob_builder.finish().unwrap(); + + let item_field = Arc::new(blob_field("item", true)); + let list_field = Field::new("blobs", DataType::List(item_field.clone()), true); + let list_array: ArrayRef = Arc::new( + arrow_array::ListArray::try_new( + item_field, + arrow_buffer::OffsetBuffer::new(arrow_buffer::ScalarBuffer::from(vec![0i32, 2, 2])), + blob_values, + None, + ) + .unwrap(), + ); + let info_fields = vec![Field::new("name", DataType::Utf8, false), list_field]; + let info_array: ArrayRef = Arc::new( + StructArray::try_new( + info_fields.clone().into(), + vec![ + Arc::new(StringArray::from(vec!["row-0", "row-1"])) as ArrayRef, + list_array, + ], + None, + ) + .unwrap(), + ); + + let schema = Arc::new(Schema::new(vec![Field::new( + "info", + DataType::Struct(info_fields.into()), + true, + )])); + let batch = RecordBatch::try_new(schema.clone(), vec![info_array]).unwrap(); + let reader = RecordBatchIterator::new(vec![batch].into_iter().map(Ok), schema); + + let dataset = Arc::new( + Dataset::write( + reader, + &test_dir, + Some(WriteParams { + data_storage_version: Some(LanceFileVersion::V2_2), + ..Default::default() + }), + ) + .await + .unwrap(), + ); + + let descriptions = dataset + .scan() + .project(&["info"]) + .unwrap() + .try_into_batch() + .await + .unwrap(); + let info = descriptions.column(0).as_struct(); + assert_eq!( + info.column_by_name("name") + .unwrap() + .as_string::() + .value(0), + "row-0" + ); + let lists = info.column_by_name("blobs").unwrap().as_list::(); + assert_eq!(lists.offsets().inner().as_ref(), &[0, 2, 2]); + let DataType::List(descriptor_field) = lists.data_type() else { + panic!("unexpected nested list type: {}", lists.data_type()); + }; + assert!(matches!(descriptor_field.data_type(), DataType::Struct(_))); + assert!(!descriptor_field.metadata().contains_key(ARROW_EXT_NAME_KEY)); + let descriptors = lists.values().as_struct(); + assert_eq!(descriptors.fields().len(), 5); + assert!(descriptors.is_valid(0)); + assert!(descriptors.is_null(1)); + + let mut scanner = dataset.scan(); + scanner.blob_handling(BlobHandling::AllBinary); + let bytes = scanner + .project(&["info"]) + .unwrap() + .try_into_batch() + .await + .unwrap(); + let info = bytes.column(0).as_struct(); + let lists = info.column_by_name("blobs").unwrap().as_list::(); + assert_eq!(lists.offsets().inner().as_ref(), &[0, 2, 2]); + let DataType::List(value_field) = lists.data_type() else { + panic!("unexpected nested list type: {}", lists.data_type()); + }; + assert_eq!(value_field.data_type(), &DataType::LargeBinary); + assert!(!value_field.metadata().contains_key(ARROW_EXT_NAME_KEY)); + let values = lists.values().as_binary::(); + assert_eq!(values.value(0), b"nested"); + assert!(values.is_null(1)); + } + + #[tokio::test] + async fn test_nested_blob_v2_requires_v2_2() { + let test_dir = TempStrDir::default(); + + let mut blob_builder = BlobArrayBuilder::new(1); + blob_builder.push_bytes(b"hello").unwrap(); + let blob_array: ArrayRef = blob_builder.finish().unwrap(); + + let (schema, batch) = nested_blob_v2_batch(blob_array); + let reader = RecordBatchIterator::new(vec![batch].into_iter().map(Ok), schema); + + let result = Dataset::write( + reader, + &test_dir, + Some(WriteParams { + data_storage_version: Some(LanceFileVersion::V2_1), + ..Default::default() + }), + ) + .await; + + assert!( + result.is_err(), + "Nested blob v2 should be rejected for file version 2.1" + ); + assert!( + result + .unwrap_err() + .to_string() + .contains("Blob v2 requires file version >= 2.2") + ); + } + + #[tokio::test] + async fn test_blob_file_read_empty_range_returns_empty_bytes() { + let store = reject_empty_range_store(); + let path = Path::from("blobs/test.bin"); + + let empty_blob = BlobFile::new_packed(store.clone(), path.clone(), 1, 0); + assert!(empty_blob.read().await.unwrap().is_empty()); + assert!(empty_blob.read_up_to(16).await.unwrap().is_empty()); + } + + #[tokio::test] + async fn test_blob_file_read_tracks_relative_cursor() { + let test_dir = TempDir::default(); + let file_path = test_dir.std_path().join("blob.bin"); + std::fs::write(&file_path, b"abcd").unwrap(); + + let path = Path::from_absolute_path(file_path).unwrap(); + let blob = BlobFile::new_packed(Arc::new(ObjectStore::local()), path, 1, 2); + + assert_eq!(blob.read().await.unwrap().as_ref(), b"bc"); + assert_eq!(blob.tell().await.unwrap(), 2); + assert!(blob.read().await.unwrap().is_empty()); + assert!(blob.read_up_to(1).await.unwrap().is_empty()); + assert_eq!(blob.tell().await.unwrap(), 2); + } + + #[tokio::test] + async fn test_blob_file_read_range_does_not_change_cursor() { + let (store, _) = recording_range_store(Bytes::from_static(b"abcdefgh")); + let path = Path::from("blobs/test.bin"); + let blob = BlobFile::new_packed(store, path, 1, 6); + + let bytes = blob.read_range(2..5).await.unwrap(); + assert_eq!(bytes.as_ref(), b"def"); + assert_eq!(blob.tell().await.unwrap(), 0); + } + + #[tokio::test] + async fn test_blob_file_read_ranges_preserves_input_order() { + let (store, inner) = recording_range_store(Bytes::from_static(b"abcdefghij")); + let path = Path::from("blobs/test.bin"); + let blob = BlobFile::new_packed(store, path, 1, 6); + + let chunks = blob.read_ranges(&[4..6, 0..2, 2..4, 2..2]).await.unwrap(); + assert_eq!(chunks[0].as_ref(), b"fg"); + assert_eq!(chunks[1].as_ref(), b"bc"); + assert_eq!(chunks[2].as_ref(), b"de"); + assert!(chunks[3].is_empty()); + assert_eq!(inner.requested_blob_ranges(), vec![1..7]); + } + + #[tokio::test] + async fn test_blob_file_read_range_rejects_out_of_bounds() { + let (store, _) = recording_range_store(Bytes::from_static(b"abcdef")); + let path = Path::from("blobs/test.bin"); + let blob = BlobFile::new_packed(store, path, 0, 4); + + let err = blob.read_range(1..5).await.unwrap_err(); + assert!(err.to_string().contains("exceeds blob size")); + } + + #[tokio::test] + async fn test_blob_file_staggered_same_source_ranges_run_concurrently() { + let (store, inner, gate) = gated_range_store( + Bytes::from_static(b"abcdefgh"), + "mock://same-source/blob-range-tests", + ); + let blob = Arc::new(BlobFile::new_dedicated( + store, + Path::from("blobs/test.bin"), + 8, + )); + + let first_blob = blob.clone(); + let first = tokio::spawn(async move { first_blob.read_range(0..3).await }); + inner.wait_for_blob_requests(1).await; + + let second_blob = blob.clone(); + let second = tokio::spawn(async move { second_blob.read_range(4..7).await }); + tokio::time::timeout(Duration::from_secs(1), inner.wait_for_blob_requests(2)) + .await + .expect("the second same-source range should start while the first is in flight"); + + assert_eq!(inner.peak_active_blob_requests(), 2); + gate.add_permits(2); + assert_eq!(first.await.unwrap().unwrap().as_ref(), b"abc"); + assert_eq!(second.await.unwrap().unwrap().as_ref(), b"efg"); + } + + #[tokio::test] + async fn test_blob_file_staggered_multiple_source_ranges_run_concurrently() { + let (first_store, first_inner, first_gate) = gated_range_store( + Bytes::from_static(b"abcdefgh"), + "mock://first-source/blob-range-tests", + ); + let (second_store, second_inner, second_gate) = gated_range_store( + Bytes::from_static(b"ijklmnop"), + "mock://second-source/blob-range-tests", + ); + let first_blob = Arc::new(BlobFile::new_dedicated( + first_store, + Path::from("blobs/first.bin"), + 8, + )); + let second_blob = Arc::new(BlobFile::new_dedicated( + second_store, + Path::from("blobs/second.bin"), + 8, + )); + + let first = tokio::spawn(async move { first_blob.read_range(0..3).await }); + first_inner.wait_for_blob_requests(1).await; + + let second = tokio::spawn(async move { second_blob.read_range(4..7).await }); + tokio::time::timeout( + Duration::from_secs(1), + second_inner.wait_for_blob_requests(1), + ) + .await + .expect("a read from another source should start while the first is in flight"); + + first_gate.add_permits(1); + second_gate.add_permits(1); + assert_eq!(first.await.unwrap().unwrap().as_ref(), b"abc"); + assert_eq!(second.await.unwrap().unwrap().as_ref(), b"mno"); + } + + #[tokio::test] + async fn test_blob_files_share_source_and_coalesce() { + let (store, inner) = recording_range_store(Bytes::from_static(b"abcdefghij")); + let source = Arc::new(BlobSource::new(store, Path::from("blobs/test.bin"))); + let blob1 = BlobFile::with_source(source.clone(), 1, 3, BlobKind::Packed, None); + let blob2 = BlobFile::with_source(source, 4, 3, BlobKind::Packed, None); + + let (data1, data2) = tokio::join!(blob1.read(), blob2.read()); + assert_eq!(data1.unwrap().as_ref(), b"bcd"); + assert_eq!(data2.unwrap().as_ref(), b"efg"); + assert_eq!(inner.requested_blob_ranges(), vec![1..7]); + } + + #[tokio::test] + async fn test_execute_blob_entries_preserves_order_and_coalesces() { + let (store, inner) = recording_range_store(Bytes::from_static(b"abcdefghij")); + let source = Arc::new(BlobSource::new(store, Path::from("blobs/test.bin"))); + let entries = vec![ + BlobEntry { + selection_index: 0, + row_address: 10, + file: BlobFile::with_source(source.clone(), 4, 3, BlobKind::Packed, None), + requested_range: None, + }, + BlobEntry { + selection_index: 1, + row_address: 11, + file: BlobFile::with_source(source, 1, 3, BlobKind::Packed, None), + requested_range: None, + }, + ]; + let mut blobs = execute_blob_entries(entries, 2, None).await.unwrap(); + blobs.sort_by_key(|blob| blob.selection_index); + + assert_eq!(blobs.len(), 2); + assert_eq!(blobs[0].row_address, 10); + assert_eq!(blobs[0].data.as_ref(), b"efg"); + assert_eq!(blobs[1].row_address, 11); + assert_eq!(blobs[1].data.as_ref(), b"bcd"); + assert_eq!(inner.requested_blob_ranges(), vec![1..7]); + } + + #[test] + fn test_blob_read_batches_bound_physical_bytes() { + let (store, _) = recording_range_store(Bytes::from_static(b"abcdefghij")); + let source = Arc::new(BlobSource::new(store, Path::from("blobs/test.bin"))); + let entries = vec![ + BlobEntry { + selection_index: 1, + row_address: 11, + file: BlobFile::with_source(source.clone(), 4, 6, BlobKind::Packed, None), + requested_range: Some(BlobReadRange::new(0, 5)), + }, + BlobEntry { + selection_index: 0, + row_address: 10, + file: BlobFile::with_source(source, 1, 3, BlobKind::Packed, None), + requested_range: Some(BlobReadRange::new(0, 3)), + }, + ]; + + let batches = plan_blob_read_batches(entries, 4).unwrap(); + + assert_eq!(batches.len(), 2); + let selection_indices = batches + .iter() + .map(|batch| { + batch + .plans + .iter() + .flat_map(|plan| plan.reads.iter()) + .map(|read| read.selection_index) + .collect::>() + }) + .collect::>(); + assert_eq!(selection_indices, vec![vec![0], vec![1]]); + } + + #[tokio::test] + async fn test_blob_read_batches_bound_coalesced_physical_bytes() { + const BLOCK_SIZE: usize = 64 * 1024; + const RANGE_GAP: u64 = 32 * 1024; + const REQUEST_COUNT: usize = 4; + + let mut data = vec![0_u8; 128 * 1024]; + for request_index in 0..REQUEST_COUNT { + data[request_index * RANGE_GAP as usize] = request_index as u8 + 1; + } + let (store, inner) = recording_range_store_with_block_size(Bytes::from(data), BLOCK_SIZE); + let source = Arc::new(BlobSource::new(store, Path::from("blobs/dense.pack"))); + let entries = (0..REQUEST_COUNT) + .map(|request_index| BlobEntry { + selection_index: request_index, + row_address: request_index as u64, + file: BlobFile::with_source( + source.clone(), + request_index as u64 * RANGE_GAP, + 1, + BlobKind::Packed, + None, + ), + requested_range: Some(BlobReadRange::new(0, 1)), + }) + .collect(); + + let batches = plan_blob_read_batches(entries, REQUEST_COUNT as u64).unwrap(); + assert_eq!(batches.len(), REQUEST_COUNT); + + let results = execute_blob_read_batches_stream( + batches, + Arc::new(ReadBlobsExecution::new(Some(REQUEST_COUNT as u64))), + REQUEST_COUNT, + ) + .try_collect::>() + .await + .unwrap(); + + assert_eq!(results.len(), REQUEST_COUNT); + for (request_index, result) in results.iter().enumerate() { + assert_eq!(result.selection_index, request_index); + assert_eq!(result.data.as_ref(), &[request_index as u8 + 1]); + } + assert_eq!( + inner.requested_blob_ranges(), + (0..REQUEST_COUNT) + .map(|request_index| { + let start = request_index as u64 * RANGE_GAP; + start..start + 1 + }) + .collect::>() + ); + } + + #[tokio::test] + async fn test_read_blob_ranges_preserves_request_order_across_fragments() { + let fixture = BlobTestFixture::new().await; + let selections = [ + (22_u64, 2_usize, 2_usize), + (2, 0, 2), + (12, 1, 2), + (12, 1, 2), + ]; + let ranges = vec![ + BlobReadRange::new(2, 3), + BlobReadRange::new(0, 2), + BlobReadRange::new(3, 2), + BlobReadRange::new(0, 3), + ]; + + let results = fixture + .dataset + .read_blob_ranges("blobs") + .unwrap() + .with_row_indices( + selections + .iter() + .zip(&ranges) + .map(|((row_index, _, _), range)| { + BlobRangeRequest::new(*row_index, range.offset, range.length) + }) + .collect::>(), + ) + .execute() + .await + .unwrap(); + + assert_eq!(results.len(), selections.len()); + for (request_index, (((_, batch_index, row_index), range), result)) in + selections.iter().zip(&ranges).zip(&results).enumerate() + { + let payload = fixture.data[*batch_index] + .column(1) + .as_binary::() + .value(*row_index); + let start = range.offset as usize; + let end = start + range.length as usize; + assert_eq!(result.request_index, request_index); + assert_eq!(result.range, *range); + assert_eq!(result.data.as_deref(), Some(&payload[start..end])); + } + } + + #[tokio::test] + async fn test_blob_selection_apis_preserve_v1_nulls_and_empty_values() { + let test_dir = TempStrDir::default(); + let blob_metadata = HashMap::from([(BLOB_META_KEY.to_string(), "true".to_string())]); + let schema = Arc::new(Schema::new(vec![ + Field::new("blob", DataType::LargeBinary, true).with_metadata(blob_metadata), + ])); + let blobs = + LargeBinaryArray::from(vec![None, Some(b"".as_slice()), Some(b"abc".as_slice())]); + let batch = RecordBatch::try_new(schema.clone(), vec![Arc::new(blobs)]).unwrap(); + let dataset = Arc::new( + Dataset::write( + RecordBatchIterator::new(vec![Ok(batch)], schema), + &test_dir, + Some(WriteParams { + data_storage_version: Some(LanceFileVersion::V2_0), + ..Default::default() + }), + ) + .await + .unwrap(), + ); + + let blob_files = dataset + .take_blobs_by_indices(&[2, 1, 0], "blob") + .await + .unwrap(); + assert_eq!(blob_files.len(), 3); + assert_eq!( + blob_files[0] + .as_ref() + .unwrap() + .read() + .await + .unwrap() + .as_ref(), + b"abc" + ); + assert!( + blob_files[1] + .as_ref() + .unwrap() + .read() + .await + .unwrap() + .is_empty() + ); + assert!(blob_files[2].is_none()); + + let blobs = dataset + .read_blobs("blob") + .unwrap() + .with_row_indices([2, 1, 0]) + .execute() + .await + .unwrap(); + assert_eq!( + blobs + .iter() + .map(|blob| (blob.row_address, blob.data.as_deref())) + .collect::>(), + vec![ + (2, Some(b"abc".as_slice())), + (1, Some(b"".as_slice())), + (0, None), + ] + ); + + let results = dataset + .read_blob_ranges("blob") + .unwrap() + .with_row_indices([ + BlobRangeRequest::new(0, 0, 0), + BlobRangeRequest::new(0, 0, 1), + BlobRangeRequest::new(1, 0, 0), + BlobRangeRequest::new(2, 1, 1), + ]) + .execute() + .await + .unwrap(); + + assert_eq!( + results + .iter() + .map(|result| (result.request_index, result.data.as_deref())) + .collect::>(), + vec![ + (0, None), + (1, None), + (2, Some(b"".as_slice())), + (3, Some(b"b".as_slice())), + ] + ); + + let descriptions = StructArray::try_new( + vec![ + Arc::new(Field::new("position", DataType::UInt64, false)), + Arc::new(Field::new("size", DataType::UInt64, false)), + ] + .into(), + vec![ + Arc::new(UInt64Array::from(vec![0])) as ArrayRef, + Arc::new(UInt64Array::from(vec![0])) as ArrayRef, + ], + Some(NullBuffer::from(vec![false])), + ) + .unwrap(); + let row_addrs = UInt64Array::from(vec![u64::MAX]); + let files = collect_blob_files_v1(&dataset, u32::MAX, &descriptions, &row_addrs).unwrap(); + assert_eq!(files.len(), 1); + assert!(files[0].is_none()); + } + + #[tokio::test] + async fn test_blob_selection_apis_preserve_v2_nulls_and_empty_values() { + let test_dir = TempStrDir::default(); + let mut blob_builder = BlobArrayBuilder::new(3); + blob_builder.push_empty().unwrap(); + blob_builder.push_null().unwrap(); + blob_builder.push_bytes(b"abc").unwrap(); + let schema = Arc::new(Schema::new(vec![blob_field("blob", true)])); + let batch = + RecordBatch::try_new(schema.clone(), vec![blob_builder.finish().unwrap()]).unwrap(); + let dataset = Arc::new( + Dataset::write( + RecordBatchIterator::new(vec![Ok(batch)], schema), &test_dir, Some(WriteParams { data_storage_version: Some(LanceFileVersion::V2_2), @@ -3789,194 +6426,406 @@ mod tests { .unwrap(), ); - let info_batch = dataset - .scan() - .project(&["info"]) - .unwrap() - .try_into_batch() + let blob_files = dataset + .take_blobs_by_indices(&[2, 1, 0], "blob") .await .unwrap(); - let blob_desc = info_batch - .column(0) - .as_struct() - .column_by_name("blob") - .unwrap() - .as_struct(); + assert_eq!(blob_files.len(), 3); assert_eq!( - blob_desc - .column_by_name("kind") + blob_files[0] + .as_ref() .unwrap() - .as_primitive::() - .value(0), - BlobKind::Inline as u8 + .read() + .await + .unwrap() + .as_ref(), + b"abc" ); - assert_eq!( - blob_desc - .column_by_name("kind") + assert!(blob_files[1].is_none()); + assert!( + blob_files[2] + .as_ref() .unwrap() - .as_primitive::() - .value(1), - BlobKind::Packed as u8 + .read() + .await + .unwrap() + .is_empty() ); let blobs = dataset - .take_blobs_by_indices(&[0, 1], "info.blob") + .read_blobs("blob") + .unwrap() + .with_row_indices([2, 1, 0]) + .execute() .await .unwrap(); - assert_eq!(blobs.len(), 2); - assert_eq!(blobs[0].read().await.unwrap().as_ref(), b"hello"); assert_eq!( - blobs[1].read().await.unwrap().as_ref(), - packed_payload.as_slice() + blobs + .iter() + .map(|blob| (blob.row_address, blob.data.as_deref())) + .collect::>(), + vec![ + (2, Some(b"abc".as_slice())), + (1, None), + (0, Some(b"".as_slice())), + ] ); - let null_blobs = dataset - .take_blobs_by_indices(&[2], "info.blob") + let results = dataset + .read_blob_ranges("blob") + .unwrap() + .with_row_indices([ + BlobRangeRequest::new(0, 0, 0), + BlobRangeRequest::new(1, 0, 0), + BlobRangeRequest::new(1, 0, 1), + BlobRangeRequest::new(2, 1, 1), + ]) + .execute() .await .unwrap(); - assert!(null_blobs.is_empty()); + + assert_eq!(results.len(), 4); + assert_eq!(results[0].request_index, 0); + assert_eq!(results[0].data.as_deref(), Some(b"".as_slice())); + assert_eq!(results[1].request_index, 1); + assert!(results[1].data.is_none()); + assert_eq!(results[2].request_index, 2); + assert!(results[2].data.is_none()); + assert_eq!(results[3].request_index, 3); + assert_eq!(results[3].data.as_deref(), Some(b"b".as_slice())); } #[tokio::test] - async fn test_nested_blob_v2_requires_v2_2() { - let test_dir = TempStrDir::default(); - - let mut blob_builder = BlobArrayBuilder::new(1); - blob_builder.push_bytes(b"hello").unwrap(); - let blob_array: ArrayRef = blob_builder.finish().unwrap(); - - let (schema, batch) = nested_blob_v2_batch(blob_array); - let reader = RecordBatchIterator::new(vec![batch].into_iter().map(Ok), schema); + async fn test_read_blob_ranges_handles_overlaps_across_scheduler_splits() { + let max_iop_size = *lance_io::object_store::DEFAULT_MAX_IOP_SIZE; + let value_size = max_iop_size + 12; + let nested_start = value_size / 2 - 1; + let nested_end = value_size; + let (store, inner) = recording_range_store(Bytes::from(vec![7; value_size as usize])); + let source = Arc::new(BlobSource::new(store, Path::from("blobs/overlapping.bin"))); + let entries = vec![ + BlobEntry { + selection_index: 0, + row_address: 10, + file: BlobFile::with_source( + source.clone(), + 0, + value_size, + BlobKind::Dedicated, + None, + ), + requested_range: Some(BlobReadRange::new(0, value_size)), + }, + BlobEntry { + selection_index: 1, + row_address: 10, + file: BlobFile::with_source(source, 0, value_size, BlobKind::Dedicated, None), + requested_range: Some(BlobReadRange::new(nested_start, nested_end - nested_start)), + }, + ]; - let result = Dataset::write( - reader, - &test_dir, - Some(WriteParams { - data_storage_version: Some(LanceFileVersion::V2_1), - ..Default::default() - }), + let mut plans = plan_blob_read_plans(entries).unwrap(); + let results = execute_blob_read_plan( + plans.pop().unwrap(), + Arc::new(ReadBlobsExecution::new(None)), ) - .await; + .await + .unwrap(); - assert!( - result.is_err(), - "Nested blob v2 should be rejected for file version 2.1" + assert_eq!(results.len(), 2); + assert_eq!(results[0].selection_index, 0); + assert_eq!(results[0].data.len(), value_size as usize); + assert!(results[0].data.iter().all(|byte| *byte == 7)); + assert_eq!(results[1].selection_index, 1); + assert_eq!(results[1].data.len(), (nested_end - nested_start) as usize); + assert!(results[1].data.iter().all(|byte| *byte == 7)); + + let mut physical_ranges = inner.requested_blob_ranges(); + physical_ranges.sort_by_key(|range| range.start); + assert_eq!( + physical_ranges.len(), + value_size.div_ceil(max_iop_size) as usize ); + assert_eq!(physical_ranges[0].start, 0); + assert_eq!(physical_ranges.last().unwrap().end, value_size); assert!( - result - .unwrap_err() - .to_string() - .contains("Blob v2 requires file version >= 2.2") + physical_ranges + .windows(2) + .all(|ranges| ranges[0].end == ranges[1].start) ); } #[tokio::test] - async fn test_blob_file_read_empty_range_returns_empty_bytes() { - let store = reject_empty_range_store(); - let path = Path::from("blobs/test.bin"); - - let empty_blob = BlobFile::new_packed(store.clone(), path.clone(), 1, 0); - assert!(empty_blob.read().await.unwrap().is_empty()); - assert!(empty_blob.read_up_to(16).await.unwrap().is_empty()); - } - - #[tokio::test] - async fn test_blob_file_read_tracks_relative_cursor() { - let test_dir = TempDir::default(); - let file_path = test_dir.std_path().join("blob.bin"); - std::fs::write(&file_path, b"abcd").unwrap(); + async fn test_read_blob_ranges_rejects_overflow_and_out_of_bounds() { + let fixture = BlobTestFixture::new().await; - let path = Path::from_absolute_path(file_path).unwrap(); - let blob = BlobFile::new_packed(Arc::new(ObjectStore::local()), path, 1, 2); + let err = fixture + .dataset + .read_blob_ranges("blobs") + .unwrap() + .execute() + .await + .unwrap_err(); + assert!(matches!(err, Error::InvalidInput { .. })); + assert!(err.to_string().contains("requires requests")); - assert_eq!(blob.read().await.unwrap().as_ref(), b"bc"); - assert_eq!(blob.tell().await.unwrap(), 2); - assert!(blob.read().await.unwrap().is_empty()); - assert!(blob.read_up_to(1).await.unwrap().is_empty()); - assert_eq!(blob.tell().await.unwrap(), 2); - } + let err = fixture + .dataset + .read_blob_ranges("blobs") + .unwrap() + .with_row_indices([BlobRangeRequest::new(0, u64::MAX, 1)]) + .execute() + .await + .unwrap_err(); + assert!(matches!(err, Error::InvalidInput { .. })); + assert!(err.to_string().contains("offset + length overflowed")); + let err = fixture + .dataset + .read_blob_ranges("blobs") + .unwrap() + .with_row_indices([BlobRangeRequest::new(0, 0, u64::MAX)]) + .execute() + .await + .unwrap_err(); + assert!(matches!(err, Error::InvalidInput { .. })); + assert!(err.to_string().contains("exceeds blob size")); + assert!(err.to_string().contains("request 0")); + } + + #[derive(Clone, Copy, Debug)] + enum BlobSelectionStreamCase { + Normal, + Empty, + Missing, + Duplicate, + DuplicateReplacesMissing, + OutOfRange, + UpstreamError, + } + + #[rstest] + #[case::normal(BlobSelectionStreamCase::Normal)] + #[case::empty(BlobSelectionStreamCase::Empty)] + #[case::missing(BlobSelectionStreamCase::Missing)] + #[case::duplicate(BlobSelectionStreamCase::Duplicate)] + #[case::duplicate_replaces_missing(BlobSelectionStreamCase::DuplicateReplacesMissing)] + #[case::out_of_range(BlobSelectionStreamCase::OutOfRange)] + #[case::upstream_error(BlobSelectionStreamCase::UpstreamError)] #[tokio::test] - async fn test_blob_file_read_range_does_not_change_cursor() { - let (store, _) = recording_range_store(Bytes::from_static(b"abcdefgh")); - let path = Path::from("blobs/test.bin"); - let blob = BlobFile::new_packed(store, path, 1, 6); - - let bytes = blob.read_range(2..5).await.unwrap(); - assert_eq!(bytes.as_ref(), b"def"); - assert_eq!(blob.tell().await.unwrap(), 0); - } + async fn test_blob_selection_stream( + #[case] case: BlobSelectionStreamCase, + #[values(true, false)] preserve_order: bool, + ) { + let (non_null, null_values, selection_count, mut expected_values, expected_error) = + match case { + BlobSelectionStreamCase::Normal => ( + vec![Ok((2, "row2")), Ok((0, "row0"))], + vec![(1, "null1")], + 3, + vec!["row0", "null1", "row2"], + None, + ), + BlobSelectionStreamCase::Empty => (Vec::new(), Vec::new(), 0, Vec::new(), None), + BlobSelectionStreamCase::Missing => ( + vec![Ok((0, "row0"))], + vec![(1, "null1")], + 3, + vec!["row0", "null1"], + Some(if preserve_order { + "completed before selection index 2 was produced" + } else { + "completed after producing 2 of 3 selections" + }), + ), + BlobSelectionStreamCase::Duplicate => ( + vec![Ok((0, "row0")), Ok((0, "duplicate"))], + Vec::new(), + 1, + vec!["row0"], + Some("selection index 0 was produced more than once"), + ), + BlobSelectionStreamCase::DuplicateReplacesMissing => ( + vec![Ok((0, "row0")), Ok((0, "duplicate"))], + Vec::new(), + 2, + vec!["row0"], + Some("selection index 0 was produced more than once"), + ), + BlobSelectionStreamCase::OutOfRange => ( + vec![Ok((1, "out-of-range"))], + Vec::new(), + 1, + Vec::new(), + Some("selection index 1 exceeded selected row count 1"), + ), + BlobSelectionStreamCase::UpstreamError => ( + vec![ + Ok((0, "row0")), + Err(Error::internal("planned blob read failed".to_string())), + ], + Vec::new(), + 1, + vec!["row0"], + Some("planned blob read failed"), + ), + }; + let non_null = futures::stream::iter(non_null).boxed(); + let results = super::totalize_blob_selection_stream( + non_null, + null_values, + selection_count, + preserve_order, + ) + .take(8) + .collect::>() + .await; - #[tokio::test] - async fn test_blob_file_read_ranges_preserves_input_order() { - let (store, inner) = recording_range_store(Bytes::from_static(b"abcdefghij")); - let path = Path::from("blobs/test.bin"); - let blob = BlobFile::new_packed(store, path, 1, 6); + let mut actual_values = results + .iter() + .filter_map(|result| result.as_ref().ok().copied()) + .collect::>(); + if !preserve_order { + actual_values.sort_unstable(); + expected_values.sort_unstable(); + } + assert_eq!(actual_values, expected_values); - let chunks = blob.read_ranges(&[4..6, 0..2, 2..4, 2..2]).await.unwrap(); - assert_eq!(chunks[0].as_ref(), b"fg"); - assert_eq!(chunks[1].as_ref(), b"bc"); - assert_eq!(chunks[2].as_ref(), b"de"); - assert!(chunks[3].is_empty()); - assert_eq!(inner.requested_blob_ranges(), vec![1..7]); + let errors = results + .iter() + .filter_map(|result| result.as_ref().err()) + .collect::>(); + assert_eq!( + errors.len(), + if expected_error.is_some() { 1 } else { 0 }, + "stream must terminate after its first error: {errors:?}" + ); + if let Some(expected_error) = expected_error { + assert!(errors[0].to_string().contains(expected_error)); + } } #[tokio::test] - async fn test_blob_file_read_range_rejects_out_of_bounds() { - let (store, _) = recording_range_store(Bytes::from_static(b"abcdef")); - let path = Path::from("blobs/test.bin"); - let blob = BlobFile::new_packed(store, path, 0, 4); + async fn test_ordered_blob_selection_stream_drains_ready_prefix_before_reads() { + let read_poll_count = Arc::new(AtomicUsize::new(0)); + let read_poll_count_for_stream = read_poll_count.clone(); + let read_values = futures::stream::once(async move { + read_poll_count_for_stream.fetch_add(1, Ordering::SeqCst); + Ok::<_, Error>((2, "row2")) + }) + .boxed(); + let mut values = super::totalize_blob_selection_stream( + read_values, + vec![(0, "null0"), (1, "null1")], + 3, + true, + ); - let err = blob.read_range(1..5).await.unwrap_err(); - assert!(err.to_string().contains("exceeds blob size")); + assert_eq!(values.try_next().await.unwrap(), Some("null0")); + assert_eq!(read_poll_count.load(Ordering::SeqCst), 0); + assert_eq!(values.try_next().await.unwrap(), Some("null1")); + assert_eq!(read_poll_count.load(Ordering::SeqCst), 0); + assert_eq!(values.try_next().await.unwrap(), Some("row2")); + assert_eq!(read_poll_count.load(Ordering::SeqCst), 1); + assert_eq!(values.try_next().await.unwrap(), None); } #[tokio::test] - async fn test_blob_files_share_source_and_coalesce() { - let (store, inner) = recording_range_store(Bytes::from_static(b"abcdefghij")); - let source = Arc::new(BlobSource::new(store, Path::from("blobs/test.bin"))); - let blob1 = BlobFile::with_source(source.clone(), 1, 3, BlobKind::Packed, None); - let blob2 = BlobFile::with_source(source, 4, 3, BlobKind::Packed, None); + async fn test_planned_blob_range_avoids_whole_value_read_amplification() { + const VALUE_SIZE: u64 = 500 * 1024 * 1024; + const WINDOW_SIZE: u64 = 100 * 1024; + + let (store, inner) = recording_range_store(Bytes::from(vec![7; WINDOW_SIZE as usize])); + let source = Arc::new(BlobSource::new(store, Path::from("blobs/large.bin"))); + source + .file_size + .set(std::num::NonZeroU64::new(VALUE_SIZE).unwrap()); + let entries = vec![BlobEntry { + selection_index: 0, + row_address: 10, + file: BlobFile::with_source(source, 0, VALUE_SIZE, BlobKind::Dedicated, None), + requested_range: Some(BlobReadRange::new(0, WINDOW_SIZE)), + }]; + let mut plans = plan_blob_read_plans(entries).unwrap(); + let results = execute_blob_read_plan( + plans.pop().unwrap(), + Arc::new(ReadBlobsExecution::new(None)), + ) + .await + .unwrap(); - let (data1, data2) = tokio::join!(blob1.read(), blob2.read()); - assert_eq!(data1.unwrap().as_ref(), b"bcd"); - assert_eq!(data2.unwrap().as_ref(), b"efg"); - assert_eq!(inner.requested_blob_ranges(), vec![1..7]); + assert_eq!(results.len(), 1); + assert_eq!(results[0].data.len(), WINDOW_SIZE as usize); + let physical_ranges = inner.requested_blob_ranges(); + assert_eq!(physical_ranges, vec![0..WINDOW_SIZE]); + assert_eq!( + physical_ranges + .iter() + .map(|range| range.end - range.start) + .sum::(), + WINDOW_SIZE + ); + assert!(!physical_ranges.contains(&(0..VALUE_SIZE))); } #[tokio::test] - async fn test_read_blobs_plan_preserves_order_and_coalesces() { - let (store, inner) = recording_range_store(Bytes::from_static(b"abcdefghij")); - let source = Arc::new(BlobSource::new(store, Path::from("blobs/test.bin"))); - let entries = vec![ - BlobEntry { - selection_index: 0, - row_address: 10, - file: BlobFile::with_source(source.clone(), 4, 3, BlobKind::Packed, None), - }, - BlobEntry { - selection_index: 1, - row_address: 11, - file: BlobFile::with_source(source, 1, 3, BlobKind::Packed, None), - }, - ]; - let execution = Arc::new(ReadBlobsExecution::new(None)); - let blobs = try_join_all( - plan_blob_read_plans(entries) - .into_iter() - .map(|plan| execute_blob_read_plan(plan, execution.clone())), + async fn test_read_blob_ranges_avoids_whole_value_read_amplification_end_to_end() { + const VALUE_SIZE: usize = 8 * 1024 * 1024; + const WINDOW_SIZE: u64 = 100 * 1024; + + let test_dir = TempStrDir::default(); + let mut blob_builder = BlobArrayBuilder::new(1); + blob_builder.push_bytes(vec![0xA5; VALUE_SIZE]).unwrap(); + let mut blob_field = blob_field("blob", false); + let mut metadata = blob_field.metadata().clone(); + metadata.insert( + BLOB_DEDICATED_SIZE_THRESHOLD_META_KEY.to_string(), + "1".to_string(), + ); + blob_field = blob_field.with_metadata(metadata); + let schema = Arc::new(Schema::new(vec![blob_field])); + let batch = RecordBatch::try_new( + schema.clone(), + vec![Arc::new(blob_builder.finish().unwrap())], ) - .await .unwrap(); - let mut blobs = blobs.into_iter().flatten().collect::>(); - blobs.sort_by_key(|blob| blob.selection_index); + let dataset = Arc::new( + Dataset::write( + RecordBatchIterator::new(vec![Ok(batch)], schema), + &test_dir, + Some(WriteParams { + data_storage_version: Some(LanceFileVersion::V2_2), + ..Default::default() + }), + ) + .await + .unwrap(), + ); - assert_eq!(blobs.len(), 2); - assert_eq!(blobs[0].row_address, 10); - assert_eq!(blobs[0].data.as_ref(), b"efg"); - assert_eq!(blobs[1].row_address, 11); - assert_eq!(blobs[1].data.as_ref(), b"bcd"); - assert_eq!(inner.requested_blob_ranges(), vec![1..7]); + let _ = dataset.object_store.io_stats_incremental(); + let results = dataset + .read_blob_ranges("blob") + .unwrap() + .with_row_indices([BlobRangeRequest::new(0, 4 * 1024 * 1024, WINDOW_SIZE)]) + .execute() + .await + .unwrap(); + let stats = dataset.object_store.io_stats_incremental(); + + assert_eq!(results.len(), 1); + let data = results[0].data.as_ref().unwrap(); + assert_eq!(data.len(), WINDOW_SIZE as usize); + assert!(data.iter().all(|byte| *byte == 0xA5)); + let blob_payload_ranges = stats + .requests + .iter() + .filter(|request| request.path.as_ref().ends_with(".blob")) + .filter_map(|request| request.range.clone()) + .collect::>(); + assert_eq!( + blob_payload_ranges, + vec![4 * 1024 * 1024..4 * 1024 * 1024 + WINDOW_SIZE] + ); } #[tokio::test] @@ -4000,6 +6849,7 @@ mod tests { BlobKind::Packed, None, ), + requested_range: None, }, BlobEntry { selection_index: 1, @@ -4011,11 +6861,13 @@ mod tests { BlobKind::Packed, None, ), + requested_range: None, }, ]; let execution = Arc::new(ReadBlobsExecution::new(None)); let mut stream: super::ReadBlobsStream = futures::stream::iter( plan_blob_read_plans(entries) + .unwrap() .into_iter() .map(move |plan| execute_blob_read_plan(plan, execution.clone())), ) @@ -4036,16 +6888,16 @@ mod tests { .unwrap() .unwrap(); assert_eq!(first.row_address, 11); - assert_eq!(first.data.as_ref(), b"uvw"); + assert_eq!(first.data.as_deref(), Some(b"uvw".as_slice())); - slow_gate.notify_one(); + slow_gate.add_permits(1); let second = tokio::time::timeout(Duration::from_secs(1), stream.next()) .await .unwrap() .unwrap() .unwrap(); assert_eq!(second.row_address, 10); - assert_eq!(second.data.as_ref(), b"abc"); + assert_eq!(second.data.as_deref(), Some(b"abc".as_slice())); } #[tokio::test] @@ -4059,9 +6911,10 @@ mod tests { .unwrap(); assert_eq!(blobs.len(), 1); - assert_eq!(blobs[0].kind(), BlobKind::Inline); + let blob = blobs[0].as_ref().unwrap(); + assert_eq!(blob.kind(), BlobKind::Inline); assert_eq!( - blobs[0].read().await.unwrap().as_ref(), + blob.read().await.unwrap().as_ref(), fixture.expected.as_slice() ); } @@ -4079,9 +6932,10 @@ mod tests { .unwrap(); assert_eq!(blobs.len(), 1); - assert_eq!(blobs[0].kind(), BlobKind::Packed); + let blob = blobs[0].as_ref().unwrap(); + assert_eq!(blob.kind(), BlobKind::Packed); assert_eq!( - blobs[0].read().await.unwrap().as_ref(), + blob.read().await.unwrap().as_ref(), fixture.expected.as_slice() ); } @@ -4097,9 +6951,10 @@ mod tests { .unwrap(); assert_eq!(blobs.len(), 1); - assert_eq!(blobs[0].kind(), BlobKind::Dedicated); + let blob = blobs[0].as_ref().unwrap(); + assert_eq!(blob.kind(), BlobKind::Dedicated); assert_eq!( - blobs[0].read().await.unwrap().as_ref(), + blob.read().await.unwrap().as_ref(), fixture.expected.as_slice() ); } @@ -4117,9 +6972,10 @@ mod tests { .unwrap(); assert_eq!(blobs.len(), 1); - assert_eq!(blobs[0].kind(), BlobKind::Packed); + let blob = blobs[0].as_ref().unwrap(); + assert_eq!(blob.kind(), BlobKind::Packed); assert_eq!( - blobs[0].read().await.unwrap().as_ref(), + blob.read().await.unwrap().as_ref(), fixture.expected.as_slice() ); } @@ -4240,7 +7096,10 @@ mod tests { let blobs = dataset.take_blobs_by_indices(&[0], "blob").await.unwrap(); assert_eq!(blobs.len(), 1); - assert_eq!(blobs[0].read().await.unwrap().as_ref(), b"outside"); + assert_eq!( + blobs[0].as_ref().unwrap().read().await.unwrap().as_ref(), + b"outside" + ); } #[tokio::test] @@ -4303,7 +7162,10 @@ mod tests { let blobs = dataset.take_blobs_by_indices(&[0], "blob").await.unwrap(); assert_eq!(blobs.len(), 1); - assert_eq!(blobs[0].read().await.unwrap().as_ref(), b"mapped"); + assert_eq!( + blobs[0].as_ref().unwrap().read().await.unwrap().as_ref(), + b"mapped" + ); } #[tokio::test] @@ -4388,8 +7250,9 @@ mod tests { let blobs = dataset.take_blobs_by_indices(&[0], "blob").await.unwrap(); assert_eq!(blobs.len(), 1); - assert_eq!(blobs[0].kind(), BlobKind::Inline); - assert_eq!(blobs[0].read().await.unwrap().as_ref(), b"inline"); + let blob = blobs[0].as_ref().unwrap(); + assert_eq!(blob.kind(), BlobKind::Inline); + assert_eq!(blob.read().await.unwrap().as_ref(), b"inline"); } #[tokio::test] @@ -4442,8 +7305,9 @@ mod tests { let blobs = dataset.take_blobs_by_indices(&[0], "blob").await.unwrap(); assert_eq!(blobs.len(), 1); - assert_eq!(blobs[0].kind(), BlobKind::Packed); - assert_eq!(blobs[0].read().await.unwrap().as_ref(), payload.as_slice()); + let blob = blobs[0].as_ref().unwrap(); + assert_eq!(blob.kind(), BlobKind::Packed); + assert_eq!(blob.read().await.unwrap().as_ref(), payload.as_slice()); } #[tokio::test] @@ -4486,8 +7350,9 @@ mod tests { let blobs = dataset.take_blobs_by_indices(&[0], "blob").await.unwrap(); assert_eq!(blobs.len(), 1); - assert_eq!(blobs[0].kind(), BlobKind::Packed); - assert_eq!(blobs[0].read().await.unwrap().as_ref(), payload.as_slice()); + let blob = blobs[0].as_ref().unwrap(); + assert_eq!(blob.kind(), BlobKind::Packed); + assert_eq!(blob.read().await.unwrap().as_ref(), payload.as_slice()); } #[tokio::test] @@ -4540,8 +7405,9 @@ mod tests { let blobs = dataset.take_blobs_by_indices(&[0], "blob").await.unwrap(); assert_eq!(blobs.len(), 1); - assert_eq!(blobs[0].kind(), BlobKind::Dedicated); - assert_eq!(blobs[0].read().await.unwrap().as_ref(), payload.as_slice()); + let blob = blobs[0].as_ref().unwrap(); + assert_eq!(blob.kind(), BlobKind::Dedicated); + assert_eq!(blob.read().await.unwrap().as_ref(), payload.as_slice()); } #[tokio::test] diff --git a/rust/lance/src/dataset/builder.rs b/rust/lance/src/dataset/builder.rs index aecbf92f0d8..3c04a88c336 100644 --- a/rust/lance/src/dataset/builder.rs +++ b/rust/lance/src/dataset/builder.rs @@ -11,7 +11,6 @@ use crate::io::commit::namespace_manifest::LanceNamespaceExternalManifestStore; use crate::{Dataset, Error, Result, session::Session}; use futures::FutureExt; use lance_core::utils::tracing::{DATASET_LOADING_EVENT, TRACE_DATASET_EVENTS}; -use lance_file::datatypes::populate_schema_dictionary; use lance_file::reader::FileReaderOptions; use lance_io::object_store::{ DEFAULT_CLOUD_IO_PARALLELISM, LanceNamespaceStorageOptionsProvider, ObjectStore, @@ -20,7 +19,7 @@ use lance_io::object_store::{ use lance_namespace::LanceNamespace; use lance_namespace::models::DescribeTableRequest; use lance_table::{ - format::Manifest, + format::{Manifest, populate_manifest_schema_dictionaries}, io::commit::external_manifest::ExternalManifestCommitHandler, io::commit::{CommitHandler, commit_handler_from_url}, }; @@ -838,9 +837,9 @@ impl DatasetBuilder { let location = commit_handler .resolve_version_location(&base_path, manifest.version, &object_store.inner) .await?; - if manifest.schema.has_dictionary_types() && manifest.should_use_legacy_format() { + if manifest.schema.has_dictionary_types() { let reader = object_store.open(&location.path).await?; - populate_schema_dictionary(&mut manifest.schema, reader.as_ref()).await?; + populate_manifest_schema_dictionaries(&mut manifest, reader.as_ref()).await?; } (Arc::new(manifest), location) } else { diff --git a/rust/lance/src/dataset/cleanup.rs b/rust/lance/src/dataset/cleanup.rs index 65928038cea..0fabc4e6d94 100644 --- a/rust/lance/src/dataset/cleanup.rs +++ b/rust/lance/src/dataset/cleanup.rs @@ -58,8 +58,8 @@ use lance_table::{ manifest::{read_manifest, read_manifest_indexes}, }, }; +use object_store::ObjectMeta; use object_store::path::Path; -use object_store::{Error as ObjectStoreError, ObjectMeta}; use std::fmt::Debug; use std::{ collections::{HashMap, HashSet}, @@ -69,7 +69,7 @@ use std::{ }; use tokio::time::{MissedTickBehavior, interval}; use tokio_stream::wrappers::IntervalStream; -use tracing::{Span, debug, info, instrument}; +use tracing::{Span, debug, info, instrument, warn}; #[derive(Clone, Debug, Default)] struct ReferencedFiles { @@ -523,17 +523,35 @@ impl<'a> CleanupTask<'a> { // ignore it then we might delete valid data files thinking they are not // referenced. - let manifest = - read_manifest(&self.dataset.object_store, &location.path, location.size).await?; + let manifest_and_indexes = async { + let manifest = + read_manifest(&self.dataset.object_store, &location.path, location.size).await?; + let indexes = + read_manifest_indexes(&self.dataset.object_store, &location, &manifest).await?; + Ok::<_, Error>((manifest, indexes)) + } + .await; + let (manifest, indexes) = match manifest_and_indexes { + Ok(manifest_and_indexes) => manifest_and_indexes, + Err(error) if location.version < self.read_version && error.is_not_found() => { + // Another cleanup may remove an old manifest after this cleanup lists it. + // The current manifest is never safe to skip because it anchors our snapshot. + debug!( + manifest_version = location.version, + read_version = self.read_version, + manifest_path = %location.path, + "Skipping old manifest removed by concurrent cleanup" + ); + return Ok(()); + } + Err(error) => return Err(error), + }; // Don't delete the latest version, even if it is old. Don't delete tagged versions, // regardless of age. Don't delete manifests if their version is newer than the dataset // version. These are either in-progress or newly added since we started. let is_latest = self.read_version <= manifest.version; let is_tagged = tagged_versions.contains(&manifest.version); let in_working_set = is_latest || !self.policy.should_clean(&manifest) || is_tagged; - let indexes = - read_manifest_indexes(&self.dataset.object_store, &location, &manifest).await?; - let mut inspection = inspection.lock().unwrap(); // Track tagged old versions in case we want to return a `CleanupError` later. @@ -576,7 +594,7 @@ impl<'a> CleanupTask<'a> { }; for fragment in manifest.fragments.iter() { - for file in fragment.files.iter() { + for file in fragment.referenced_lance_files() { let full_data_path = self.dataset.data_dir().clone().join(file.path.as_str()); let relative_data_path = remove_prefix(&full_data_path, &self.dataset.base); referenced_files.data_paths.insert(relative_data_path); @@ -621,26 +639,29 @@ impl<'a> CleanupTask<'a> { ) -> Result { let cleanup_result = Mutex::new(CleanupRunResult::default()); let deletes_files = self.action.deletes_files(); + let removes_empty_dirs = matches!( + self.dataset.object_store.scheme(), + "file" | "file+uring" | "file-object-store" + ); + let indices_dir = self.dataset.indices_dir(); + let retained_index_dirs = inspection + .referenced_files + .index_uuids + .iter() + .map(|uuid| indices_dir.clone().join(uuid.as_str())) + .collect::>(); + let index_dirs_to_remove = Mutex::new(HashSet::new()); let candidate_file_limit = self.action.candidate_file_limit(); let verification_threshold = utc_now() - TimeDelta::try_days(UNVERIFIED_THRESHOLD_DAYS).expect("TimeDelta::try_days"); - let is_not_found_err = |e: &Error| { - matches!( - e, - Error::IO { source,.. } - if source - .downcast_ref::() - .map(|os_err| matches!(os_err, ObjectStoreError::NotFound {.. })) - .unwrap_or(false) - ) - }; + let is_not_found_err = |e: &Error| matches!(e, Error::NotFound { .. }); // Build stream for a managed subtree - let build_listing_stream = |dir: Path| { + let build_listing_stream = |dir: Path, unmodified_since| { let inspection_ref = &inspection; self.dataset .object_store - .read_dir_all(&dir, inspection.earliest_retained_manifest_time) + .read_dir_all(&dir, unmodified_since) .map_ok(|obj| stream::once(future::ready(Ok(obj))).boxed()) .or_else(|e| { // If the directory doesn't exist then we can just return an empty stream. @@ -667,12 +688,17 @@ impl<'a> CleanupTask<'a> { }; // Restrict scanning to Lance-managed subtrees for safety and performance. + let unmodified_since = inspection.earliest_retained_manifest_time; let streams = vec![ - build_listing_stream(self.dataset.versions_dir()), - build_listing_stream(self.dataset.transactions_dir()), - build_listing_stream(self.dataset.data_dir()), - build_listing_stream(self.dataset.indices_dir()), - build_listing_stream(self.dataset.deletions_dir()), + build_listing_stream(self.dataset.versions_dir(), unmodified_since), + build_listing_stream(self.dataset.transactions_dir(), unmodified_since), + build_listing_stream(self.dataset.data_dir(), unmodified_since), + // Index UUIDs from manifests being removed are proof that their files are + // safe to delete. Scan every index artifact while that proof is available; + // a retained-manifest cutoff can otherwise skip newer artifacts and lose + // the proof when the old manifests are removed by this cleanup pass. + build_listing_stream(self.dataset.indices_dir(), None), + build_listing_stream(self.dataset.deletions_dir(), unmodified_since), ]; let unreferenced_files = stream::iter(streams).flatten().boxed(); @@ -720,6 +746,17 @@ impl<'a> CleanupTask<'a> { .lock() .unwrap() .record_file(&file, candidate_file_limit, self.track_removed_manifests); + if deletes_files && removes_empty_dirs && matches!(file.kind, CleanupFileKind::Index) { + let mut parent = file.path.parent(); + let mut index_dirs = index_dirs_to_remove.lock().unwrap(); + while let Some(dir_path) = parent { + if dir_path == indices_dir || !dir_path.prefix_matches(&indices_dir) { + break; + } + index_dirs.insert(dir_path.clone()); + parent = dir_path.parent(); + } + } Ok(file.path) }); @@ -743,6 +780,25 @@ impl<'a> CleanupTask<'a> { .remove_stream(paths_to_delete) .try_for_each(|_| future::ready(Ok(()))) .await?; + + if removes_empty_dirs + && let Err(error) = self + .dataset + .object_store + .remove_empty_dirs( + indices_dir.clone(), + retained_index_dirs, + index_dirs_to_remove.into_inner().unwrap(), + (!self.policy.delete_unverified).then_some(verification_threshold), + ) + .await + { + warn!( + path = indices_dir.as_ref(), + error = %error, + "Failed to remove empty index directories" + ); + } } else { // Drain the stream to populate stats, but do not call remove_stream. all_paths_to_remove @@ -1138,7 +1194,7 @@ impl<'a> CleanupTask<'a> { let mut is_referenced = false; for fragment in manifest.fragments.iter() { - for file in fragment.files.iter() { + for file in fragment.referenced_lance_files() { if let Some(base_id) = file.base_id { let base_path = manifest.base_paths.get(&base_id); if let Some(base_path) = base_path @@ -1292,7 +1348,16 @@ impl CleanupPolicyBuilder { } /// Cleanup all versions except the last `n` versions of the dataset. + /// + /// # Errors + /// + /// Returns an error if `n` is zero. pub async fn retain_n_versions(mut self, dataset: &Dataset, n: usize) -> Result { + if n == 0 { + return Err(Error::invalid_input(format!( + "retain_versions must be greater than 0, got {n}" + ))); + } let versions = dataset.versions().await?; self.policy.before_version = if versions.len() <= n { Some(versions[0].version) @@ -1567,6 +1632,7 @@ mod tests { use lance_table::io::commit::RenameCommitHandler; use lance_testing::datagen::{BatchGenerator, IncrementingInt32, RandomVector, some_batch}; use mock_instant::thread_local::MockClock; + use rstest::rstest; use uuid::Uuid; #[derive(Debug)] @@ -1633,7 +1699,7 @@ mod tests { struct MockDatasetFixture { // This is a temporary directory that will be deleted when the fixture // is dropped - _tmpdir: TempStrDir, + tmpdir: TempStrDir, dataset_path: String, mock_store: Arc, } @@ -1653,12 +1719,19 @@ mod tests { }; let dataset_path = format!("file-object-store://{path_prefix}{tmpdir_path}/my_db"); Ok(Self { - _tmpdir: tmpdir, + tmpdir, dataset_path, mock_store: Arc::new(MockObjectStore::new()), }) } + fn local_index_dir(&self, uuid: Uuid) -> std::path::PathBuf { + std::path::Path::new(self.tmpdir.as_str()) + .join("my_db") + .join(crate::dataset::INDICES_DIR) + .join(uuid.to_string()) + } + fn os_params(&self) -> ObjectStoreParams { ObjectStoreParams { object_store_wrapper: Some(self.mock_store.clone()), @@ -2041,6 +2114,42 @@ mod tests { assert_gt!(after_count.num_tx_files, 0); } + #[tokio::test] + async fn cleanup_ignores_old_manifest_removed_after_listing() { + let fixture = MockDatasetFixture::try_new().unwrap(); + fixture.create_some_data().await.unwrap(); + fixture.overwrite_some_data().await.unwrap(); + let dataset = fixture.open().await.unwrap(); + + let old_manifest = dataset + .commit_handler + .list_manifest_locations(&dataset.base, &dataset.object_store, false) + .try_filter(|location| future::ready(location.version == 1)) + .try_next() + .await + .unwrap() + .unwrap(); + dataset + .object_store + .delete(&old_manifest.path) + .await + .unwrap(); + + let cleanup = CleanupTask::new( + &dataset, + CleanupPolicyBuilder::default().build(), + CleanupAction::Execute, + ); + cleanup + .process_manifest_file( + old_manifest, + &Mutex::new(CleanupInspection::default()), + &HashSet::new(), + ) + .await + .unwrap(); + } + #[tokio::test] async fn explain_cleanup_does_not_delete_files() { let fixture = MockDatasetFixture::try_new().unwrap(); @@ -2655,6 +2764,147 @@ mod tests { assert_gt!(removed.deletion_files_removed, 0); } + /// A branch reaches its parent's files through `base_id`, and + /// `retain_branch_lineage_files` promotes those into the parent's keep set. An + /// overlay inherited that way must be promoted too, or the parent deletes a + /// file the branch still reads. + #[tokio::test] + async fn lineage_retention_covers_inherited_overlay_files() { + use lance_table::format::overlay::{DataOverlayFile, OverlayCoverage}; + use roaring::RoaringBitmap; + + let fixture = MockDatasetFixture::try_new().unwrap(); + fixture.create_some_data().await.unwrap(); + + // Give the parent an overlay, then branch from it: `shallow_clone` stamps + // the overlay's `base_id` so the branch resolves it against the parent. + let mut dataset = fixture.open().await.unwrap(); + let mut fragments: Vec<_> = dataset + .get_fragments() + .iter() + .map(|f| f.metadata().clone()) + .collect(); + let mut overlay_file = fragments[0].files[0].clone(); + overlay_file.path = "overlay.lance".to_string(); + fragments[0].overlays = vec![DataOverlayFile { + data_file: overlay_file, + coverage: OverlayCoverage::Shared(Arc::new(RoaringBitmap::from_iter([0_u32]))), + committed_version: dataset.manifest.version, + }]; + let transaction = Transaction::new( + dataset.manifest.version, + Operation::Overwrite { + fragments, + schema: dataset.schema().clone(), + config_upsert_values: None, + initial_bases: None, + }, + None, + ); + dataset + .apply_commit(transaction, &Default::default(), &Default::default()) + .await + .unwrap(); + + let branch = fixture + .create_branch_and_load(&mut dataset, "child", (None, None)) + .await + .unwrap(); + let branch_fragments = branch.get_fragments(); + let inherited = &branch_fragments[0].metadata().overlays[0].data_file; + assert!( + inherited.base_id.is_some(), + "the branch must reach the parent's overlay through base_id" + ); + + // The parent's cleanup walks the branch's manifest and must promote that + // overlay out of `verified_files`. + let task = CleanupTask::new( + &dataset, + CleanupPolicyBuilder::default() + .before_timestamp(utc_now()) + .build(), + CleanupAction::Execute, + ); + let inspection = task.process_manifests(&HashSet::new()).await.unwrap(); + let referenced_branches = task.find_referenced_branches().await.unwrap(); + let inspection = task + .retain_branch_lineage_files(inspection, &referenced_branches, &HashSet::new()) + .await + .unwrap(); + + let overlay_path = Path::from("data/overlay.lance"); + assert!( + inspection + .referenced_files + .data_paths + .contains(&overlay_path), + "the inherited overlay must be promoted into the parent's keep set" + ); + } + + /// A keep set built from `fragment.files` alone omits overlay data files, so + /// cleanup would delete live data. + #[tokio::test] + async fn keep_set_covers_referenced_overlay_files() { + use lance_table::format::overlay::{DataOverlayFile, OverlayCoverage}; + use roaring::RoaringBitmap; + + let fixture = MockDatasetFixture::try_new().unwrap(); + fixture.create_some_data().await.unwrap(); + + // The overlay file need not exist: the keep set comes from manifest + // metadata alone. + let mut dataset = fixture.open().await.unwrap(); + let mut fragments: Vec<_> = dataset + .get_fragments() + .iter() + .map(|f| f.metadata().clone()) + .collect(); + let mut overlay_file = fragments[0].files[0].clone(); + overlay_file.path = "overlay.lance".to_string(); + fragments[0].overlays = vec![DataOverlayFile { + data_file: overlay_file, + coverage: OverlayCoverage::Shared(Arc::new(RoaringBitmap::from_iter([0_u32]))), + committed_version: dataset.manifest.version, + }]; + let transaction = Transaction::new( + dataset.manifest.version, + Operation::Overwrite { + fragments, + schema: dataset.schema().clone(), + config_upsert_values: None, + initial_bases: None, + }, + None, + ); + dataset + .apply_commit(transaction, &Default::default(), &Default::default()) + .await + .unwrap(); + + let task = CleanupTask::new( + &dataset, + CleanupPolicyBuilder::default() + .before_timestamp(utc_now()) + .build(), + CleanupAction::Execute, + ); + let inspection = task.process_manifests(&HashSet::new()).await.unwrap(); + let kept: HashSet<&Path> = inspection + .referenced_files + .data_paths + .iter() + .chain(inspection.verified_files.data_paths.iter()) + .collect(); + + let overlay_path = Path::from("data/overlay.lance"); + assert!( + kept.contains(&overlay_path), + "the overlay's data file must be in the keep set, got {kept:?}" + ); + } + #[tokio::test] async fn dont_clean_index_data_files() { // Indexes have .lance files in them that are not referenced @@ -2676,6 +2926,119 @@ mod tests { assert_eq!(before_count, after_count); } + #[tokio::test] + async fn cleanup_removes_preexisting_empty_index_directories() { + let fixture = MockDatasetFixture::try_new().unwrap(); + fixture.create_some_data().await.unwrap(); + + let mut dataset = fixture.open().await.unwrap(); + let field_id = dataset.schema().field("indexable").unwrap().id; + let stale_uuid = Uuid::new_v4(); + let nested_stale_uuid = Uuid::new_v4(); + let referenced_uuid = Uuid::new_v4(); + + std::fs::create_dir_all(fixture.local_index_dir(stale_uuid)).unwrap(); + std::fs::create_dir_all( + fixture + .local_index_dir(nested_stale_uuid) + .join("empty_nested_dir"), + ) + .unwrap(); + std::fs::create_dir_all(fixture.local_index_dir(referenced_uuid)).unwrap(); + + let referenced_index = dummy_index_metadata(&dataset, field_id, referenced_uuid, [0_u32]); + let create_index_tx = Transaction::new( + dataset.manifest.version, + Operation::CreateIndex { + new_indices: vec![referenced_index], + removed_indices: vec![], + }, + None, + ); + dataset + .apply_commit(create_index_tx, &Default::default(), &Default::default()) + .await + .unwrap(); + + let real_now = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap(); + MockClock::set_system_time(real_now + TimeDelta::try_days(10).unwrap().to_std().unwrap()); + let in_progress_uuid = Uuid::new_v4(); + write_dummy_index_artifact(&dataset, in_progress_uuid) + .await + .unwrap(); + let in_progress_empty_dir = fixture + .local_index_dir(in_progress_uuid) + .join("empty_in_progress_dir"); + std::fs::create_dir_all(&in_progress_empty_dir).unwrap(); + + let removed = fixture + .run_cleanup(utc_now() - TimeDelta::try_days(7).unwrap()) + .await + .unwrap(); + + assert_eq!(removed.index_files_removed, 0); + assert!(!fixture.local_index_dir(stale_uuid).exists()); + assert!(!fixture.local_index_dir(nested_stale_uuid).exists()); + assert!(fixture.local_index_dir(referenced_uuid).exists()); + assert!(fixture.local_index_dir(in_progress_uuid).exists()); + assert!(in_progress_empty_dir.exists()); + } + + #[rstest] + #[case::default_policy(false, true)] + #[case::delete_unverified(true, false)] + #[tokio::test] + async fn cleanup_applies_unverified_policy_to_fresh_empty_index_directory( + #[case] delete_unverified: bool, + #[case] should_preserve: bool, + ) { + let real_now = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap(); + MockClock::set_system_time(real_now); + + let fixture = MockDatasetFixture::try_new().unwrap(); + fixture.create_some_data().await.unwrap(); + let in_progress_dir = fixture.local_index_dir(Uuid::new_v4()); + std::fs::create_dir_all(&in_progress_dir).unwrap(); + + fixture + .run_cleanup_with_override( + utc_now() - TimeDelta::try_days(7).unwrap(), + Some(delete_unverified), + None, + ) + .await + .unwrap(); + + assert_eq!(in_progress_dir.exists(), should_preserve); + } + + #[cfg(unix)] + #[tokio::test] + async fn cleanup_does_not_remove_empty_directory_through_index_symlink() { + let fixture = MockDatasetFixture::try_new().unwrap(); + fixture.create_some_data().await.unwrap(); + + let outside = tempfile::tempdir().unwrap(); + let outside_empty_dir = outside.path().join("must_remain"); + std::fs::create_dir_all(&outside_empty_dir).unwrap(); + + let link_path = fixture.local_index_dir(Uuid::new_v4()); + std::fs::create_dir_all(link_path.parent().unwrap()).unwrap(); + std::os::unix::fs::symlink(outside.path(), &link_path).unwrap(); + + fixture + .run_cleanup_with_override(utc_now(), Some(true), None) + .await + .unwrap(); + + assert!(outside_empty_dir.exists()); + assert!(link_path.is_symlink()); + } + #[tokio::test] async fn cleanup_old_replaced_segment_keeps_still_referenced_segments() { let fixture = MockDatasetFixture::try_new().unwrap(); @@ -2728,6 +3091,7 @@ mod tests { .unwrap(); assert_eq!(removed.index_files_removed, 2); + assert!(!fixture.local_index_dir(seg_a).exists()); assert!( !dataset .object_store @@ -2772,6 +3136,96 @@ mod tests { ); } + #[tokio::test] + async fn cleanup_recent_replaced_index_with_short_retention() { + let fixture = MockDatasetFixture::try_new().unwrap(); + fixture.create_some_data().await.unwrap(); + fixture.append_some_data().await.unwrap(); + + let mut dataset = fixture.open().await.unwrap(); + let field_id = dataset.schema().field("indexable").unwrap().id; + let old_uuid = Uuid::new_v4(); + let current_uuid = Uuid::new_v4(); + + let old_index = dummy_index_metadata(&dataset, field_id, old_uuid, [0_u32, 1]); + dataset + .apply_commit( + Transaction::new( + dataset.manifest.version, + Operation::CreateIndex { + new_indices: vec![old_index.clone()], + removed_indices: vec![], + }, + None, + ), + &Default::default(), + &Default::default(), + ) + .await + .unwrap(); + + MockClock::set_system_time(TimeDelta::try_minutes(1).unwrap().to_std().unwrap()); + let current_index = dummy_index_metadata(&dataset, field_id, current_uuid, [0_u32, 1]); + dataset + .apply_commit( + Transaction::new( + dataset.manifest.version, + Operation::CreateIndex { + new_indices: vec![current_index], + removed_indices: vec![old_index], + }, + None, + ), + &Default::default(), + &Default::default(), + ) + .await + .unwrap(); + + // Model index artifacts whose storage timestamp is newer than the retained + // manifest. UUID verification must not be hidden by the manifest cutoff. + MockClock::set_system_time(TimeDelta::try_minutes(2).unwrap().to_std().unwrap()); + write_dummy_index_artifact(&dataset, old_uuid) + .await + .unwrap(); + write_dummy_index_artifact(&dataset, current_uuid) + .await + .unwrap(); + + let short_retention = TimeDelta::try_seconds(30).unwrap(); + let removed = fixture + .run_cleanup(utc_now() - short_retention) + .await + .unwrap(); + assert_eq!(removed.old_versions, 3); + assert_eq!(removed.index_files_removed, 2); + + let old_index_file = dataset + .indices_dir() + .join(old_uuid.to_string()) + .join("index.idx"); + let current_index_file = dataset + .indices_dir() + .join(current_uuid.to_string()) + .join("index.idx"); + assert!( + !dataset + .object_store + .as_ref() + .exists(&old_index_file) + .await + .unwrap() + ); + assert!( + dataset + .object_store + .as_ref() + .exists(¤t_index_file) + .await + .unwrap() + ); + } + #[tokio::test] async fn cleanup_old_uncommitted_index_artifacts() { let fixture = MockDatasetFixture::try_new().unwrap(); @@ -2798,6 +3252,8 @@ mod tests { assert_eq!(removed.old_versions, 0); assert_eq!(removed.index_files_removed, 4); + assert!(!fixture.local_index_dir(staging_uuid).exists()); + assert!(!fixture.local_index_dir(built_segment_uuid).exists()); assert!( !dataset .object_store @@ -2949,6 +3405,26 @@ mod tests { assert_eq!(after_count.num_manifest_files, 1); } + #[tokio::test] + async fn cleanup_rejects_retain_zero_versions() { + let fixture = MockDatasetFixture::try_new().unwrap(); + fixture.create_some_data().await.unwrap(); + + let error = CleanupPolicyBuilder::default() + .retain_n_versions(&fixture.open().await.unwrap(), 0) + .await + .err() + .expect("retaining zero versions should return an error"); + + assert!(matches!(&error, Error::InvalidInput { .. })); + assert!( + error + .to_string() + .contains("retain_versions must be greater than 0, got 0"), + "unexpected error: {error}" + ); + } + #[tokio::test] async fn cleanup_and_retain_3_recent_versions() { let fixture = MockDatasetFixture::try_new().unwrap(); @@ -2987,10 +3463,8 @@ mod tests { async fn cleanup_before_ts_and_retain_n_recent_versions() { let fixture = MockDatasetFixture::try_new().unwrap(); fixture.create_some_data().await.unwrap(); - let mut time = 1i64; - for _ in 0..4 { + for time in (1i64..).take(4) { MockClock::set_system_time(TimeDelta::try_days(time).unwrap().to_std().unwrap()); - time += 1i64; fixture.overwrite_some_data().await.unwrap(); } diff --git a/rust/lance/src/dataset/files.rs b/rust/lance/src/dataset/files.rs index 848add7e4a8..e97545dfdf8 100644 --- a/rust/lance/src/dataset/files.rs +++ b/rust/lance/src/dataset/files.rs @@ -57,17 +57,29 @@ struct FileRow<'a> { file_type: FileType, } -/// Resolve the base URI a file lives under. Files referenced from a shallow -/// clone carry a `base_id` pointing into `manifest.base_paths`; otherwise they -/// live under this dataset's own `base_uri`. -fn resolve_base_uri<'a>( +struct ResolvedFileBase<'a> { + uri: &'a str, + is_dataset_root: bool, +} + +/// Resolve the base a file lives under. Files referenced from a shallow clone +/// carry a `base_id` pointing into `manifest.base_paths`; otherwise they live +/// under this dataset's own root. +fn resolve_file_base<'a>( manifest: &'a lance_table::format::Manifest, base_id: Option, base_uri: &'a str, -) -> &'a str { +) -> ResolvedFileBase<'a> { base_id - .and_then(|id| manifest.base_paths.get(&id).map(|bp| bp.path.as_str())) - .unwrap_or(base_uri) + .and_then(|id| manifest.base_paths.get(&id)) + .map(|base_path| ResolvedFileBase { + uri: base_path.path.as_str(), + is_dataset_root: base_path.is_dataset_root, + }) + .unwrap_or(ResolvedFileBase { + uri: base_uri, + is_dataset_root: true, + }) } fn manifest_file_rows<'a>( @@ -98,7 +110,9 @@ fn manifest_file_rows<'a>( }; for fragment in manifest.fragments.iter() { - files += fragment.files.len(); + // Precount with the same accessor as the iterator below, or `exact_size` + // drifts. + files += fragment.referenced_lance_files().count(); if fragment.deletion_file.is_some() { files += 1; @@ -106,12 +120,17 @@ fn manifest_file_rows<'a>( } let data_files = manifest.fragments.iter().flat_map(move |fragment| { - fragment.files.iter().map(move |data_file| { - let effective_base_uri = resolve_base_uri(manifest, data_file.base_id, base_uri); + fragment.referenced_lance_files().map(move |data_file| { + let resolved_base = resolve_file_base(manifest, data_file.base_id, base_uri); + let path = if resolved_base.is_dataset_root { + Cow::Owned(format!("{}/{}", DATA_DIR, data_file.path)) + } else { + Cow::Borrowed(data_file.path.as_str()) + }; FileRow { version: manifest.version, - base_uri: Cow::Borrowed(effective_base_uri), - path: Cow::Owned(format!("{}/{}", DATA_DIR, data_file.path)), + base_uri: Cow::Borrowed(resolved_base.uri), + path, file_type: FileType::DataFile, } }) @@ -120,7 +139,7 @@ fn manifest_file_rows<'a>( let deletion_files = manifest.fragments.iter().filter_map(|fragment| { fragment.deletion_file.as_ref().map(|del_file| FileRow { version: manifest.version, - base_uri: Cow::Borrowed(resolve_base_uri(manifest, del_file.base_id, base_uri)), + base_uri: Cow::Borrowed(resolve_file_base(manifest, del_file.base_id, base_uri).uri), path: Cow::Owned(relative_deletion_file_path(fragment.id, del_file)), file_type: FileType::DeletionFile, }) @@ -1002,12 +1021,13 @@ mod tests { ); } - /// Each `DataFile` inside a fragment carries its own `base_id`; the - /// emitted `base_uri` must be looked up per file, not per fragment. + /// Each `DataFile` inside a fragment carries its own `base_id`; both the + /// emitted `base_uri` and relative path must honor that base's layout. #[test] fn test_manifest_file_rows_per_file_base_id() { use lance_core::datatypes::{Field as LanceField, Schema as LanceSchema}; use lance_io::utils::CachedFileSize; + use lance_table::format::overlay::{DataOverlayFile, OverlayCoverage}; use lance_table::format::{ BasePath, DataFile, DataStorageFormat, DeletionFile, DeletionFileType, Fragment, Manifest, @@ -1036,6 +1056,13 @@ mod tests { // No base_id -> falls back to the dataset base_uri. mk_file("c.lance", None), ], + // An overlay's data file is reported like any other, and resolves + // its own base_id. + overlays: vec![DataOverlayFile { + data_file: mk_file("d.lance", Some(1)), + coverage: OverlayCoverage::Shared(Arc::new(Default::default())), + committed_version: 1, + }], // Deletion files also carry a base_id when they originate from a // shallow clone, and must resolve against base_paths too. deletion_file: Some(DeletionFile { @@ -1058,7 +1085,7 @@ mod tests { ); base_paths.insert( 2, - BasePath::new(2, "s3://bucket-b/root".to_string(), None, false), + BasePath::new(2, "s3://bucket-b/root".to_string(), None, true), ); let manifest = Manifest::new( @@ -1076,9 +1103,10 @@ mod tests { .map(|r| (r.path.as_ref(), r.base_uri.as_ref())) .collect(); - assert_eq!(by_path.get("data/a.lance"), Some(&"s3://bucket-a/root")); + assert_eq!(by_path.get("a.lance"), Some(&"s3://bucket-a/root")); assert_eq!(by_path.get("data/b.lance"), Some(&"s3://bucket-b/root")); assert_eq!(by_path.get("data/c.lance"), Some(&"memory://main")); + assert_eq!(by_path.get("d.lance"), Some(&"s3://bucket-a/root")); let deletion = rows .iter() diff --git a/rust/lance/src/dataset/files/file_types.rs b/rust/lance/src/dataset/files/file_types.rs index 7c20a81eae5..6734c5169aa 100644 --- a/rust/lance/src/dataset/files/file_types.rs +++ b/rust/lance/src/dataset/files/file_types.rs @@ -31,3 +31,47 @@ impl From for i8 { file_type as Self } } + +#[cfg(test)] +mod tests { + use super::*; + + use arrow_array::{Array, StringArray, cast::AsArray}; + + use crate::dataset::files::arrow::FILE_TYPE_DICT_ARRAY; + + const ALL: [FileType; 5] = [ + FileType::Manifest, + FileType::DataFile, + FileType::DeletionFile, + FileType::TransactionFile, + FileType::IndexFile, + ]; + + /// The discriminants double as dictionary keys for the `tracked_files` + /// output, so reordering either list would silently mislabel every row. + #[test] + fn test_discriminants_index_into_the_dictionary() { + let dict: &StringArray = FILE_TYPE_DICT_ARRAY.as_string(); + assert_eq!( + dict.len(), + ALL.len(), + "every variant needs a dictionary slot" + ); + + for file_type in ALL { + let key = i8::from(file_type); + assert_eq!( + dict.value(key as usize), + file_type.to_string(), + "{file_type:?} has key {key}" + ); + } + } + + #[test] + fn test_discriminants_are_contiguous_from_zero() { + let keys: Vec = ALL.iter().copied().map(i8::from).collect(); + assert_eq!(keys, (0..ALL.len() as i8).collect::>()); + } +} diff --git a/rust/lance/src/dataset/fragment.rs b/rust/lance/src/dataset/fragment.rs index 175b1d7d2de..f7cea042995 100644 --- a/rust/lance/src/dataset/fragment.rs +++ b/rust/lance/src/dataset/fragment.rs @@ -22,29 +22,32 @@ use datafusion::logical_expr::Expr; use datafusion::scalar::ScalarValue; use futures::future::{BoxFuture, try_join_all}; use futures::{FutureExt, StreamExt, TryFutureExt, TryStreamExt, join, stream}; +use lance_arrow::json::{convert_json_columns, has_json_fields, is_arrow_json_field}; use lance_arrow::{RecordBatchExt, SchemaExt}; -use lance_core::datatypes::{OnMissing, OnTypeMismatch, SchemaCompareOptions}; +use lance_core::datatypes::{BlobHandling, OnMissing, OnTypeMismatch, SchemaCompareOptions}; use lance_core::utils::address::RowAddress; use lance_core::utils::deletion::DeletionVector; use lance_core::utils::tokio::get_num_compute_intensive_cpus; -use lance_core::{Error, Result, cache::CacheKey, datatypes::Schema}; +use lance_core::{ + Error, Result, + cache::{CacheKey, CacheKeySchema, KeyBuilder}, + datatypes::Schema, +}; use lance_core::{ ROW_ADDR, ROW_ADDR_FIELD, ROW_CREATED_AT_VERSION_FIELD, ROW_ID, ROW_ID_FIELD, ROW_LAST_UPDATED_AT_VERSION_FIELD, }; use lance_datafusion::utils::StreamingWriteSource; use lance_encoding::decoder::DecoderPlugins; -use lance_file::previous::reader::{ - FileReader as PreviousFileReader, read_batch as previous_read_batch, -}; use lance_file::reader::{ - CachedFileMetadata, FileMetadataIndex, FileReaderOptions, ProjectedFileReader, ReaderProjection, + CachedFileMetadata, FileMetadataIndex, FileReaderOptions, ProjectedFileReader, }; -use lance_file::version::LanceFileVersion; -use lance_file::{LanceEncodingsIo, determine_file_version}; +use lance_file::versions::v1::reader::{FileReader as V1FileReader, read_batch as v1_read_batch}; +use lance_file::{LanceEncodingsIo, determine_file_version, versions as file_versions}; use lance_io::ReadBatchParams; use lance_io::scheduler::{FileScheduler, ScanScheduler, SchedulerConfig}; use lance_io::utils::CachedFileSize; +use lance_table::format::overlay::TOMBSTONE_FIELD_ID; use lance_table::format::{DataFile, DeletionFile, Fragment}; use lance_table::io::deletion::{deletion_file_path, write_deletion_file}; use lance_table::rowids::RowIdSequence; @@ -64,6 +67,9 @@ use super::updater::Updater; use super::{NewColumnTransform, WriteParams, schema_evolution}; use crate::dataset::Dataset; use crate::dataset::fragment::session::FragmentSession; +use crate::dataset::overlay::{ + OverlayReadPlanner, merge_overlay_batch, plan_overlays, resolve_overlays, +}; use crate::io::deletion::read_dataset_deletion_file; /// Result of [`FileFragment::update_columns_with_offsets`]: updated fragment metadata, modified field ids, @@ -137,30 +143,13 @@ pub trait GenericFileReader: std::fmt::Debug + Send + Sync { /// Get storage statistics for this file (ignored by v1 reader) fn storage_stats(&self) -> Result>; - // Helper functions to fallback to the legacy implementation while we - // slowly migrate functionality over to the generic reader - // Clone the reader, this is needed because Box doesn't // implement Clone fn clone_box(&self) -> Box; - // Return true if the reader is a v1 reader - fn is_legacy(&self) -> bool; - // Return a reference to the legacy reader, panics if called on a v2 - // file. - fn as_legacy(&self) -> &PreviousFileReader { - self.as_legacy_opt() - .expect("legacy function called on v2 file") - } - // Return a reference to the legacy reader if this is a v1 reader and - // return None otherwise - fn as_legacy_opt(&self) -> Option<&PreviousFileReader>; - // Return a mutable reference to the legacy reader if this is a v1 reader - // and return None otherwise - fn as_legacy_opt_mut(&mut self) -> Option<&mut PreviousFileReader>; } fn ranges_to_tasks( - reader: &PreviousFileReader, + reader: &V1FileReader, ranges: Vec<(i32, Range)>, projection: Arc, ) -> ReadBatchTaskStream { @@ -171,7 +160,7 @@ fn ranges_to_tasks( let reader = reader.clone(); let projection = projection.clone(); let task = tokio::task::spawn(async move { - previous_read_batch( + v1_read_batch( &reader, &ReadBatchParams::Range(range.clone()), &projection, @@ -191,12 +180,12 @@ fn ranges_to_tasks( #[derive(Clone, Debug)] struct V1Reader { - reader: PreviousFileReader, + reader: V1FileReader, projection: Arc, } impl V1Reader { - fn new(reader: PreviousFileReader, projection: Arc) -> Self { + fn new(reader: V1FileReader, projection: Arc) -> Self { Self { reader, projection } } } @@ -309,18 +298,6 @@ impl GenericFileReader for V1Reader { fn clone_box(&self) -> Box { Box::new(self.clone()) } - - fn is_legacy(&self) -> bool { - true - } - - fn as_legacy_opt(&self) -> Option<&PreviousFileReader> { - Some(&self.reader) - } - - fn as_legacy_opt_mut(&mut self) -> Option<&mut PreviousFileReader> { - Some(&mut self.reader) - } } mod v2_adapter { @@ -364,7 +341,7 @@ mod v2_adapter { projection: Arc, ) -> BoxFuture<'_, Result> { async move { - let projection = ReaderProjection::from_field_ids( + let projection = file_versions::reader_projection_from_field_ids( self.reader.version(), projection.as_ref(), self.field_id_to_column_idx.as_ref(), @@ -394,7 +371,7 @@ mod v2_adapter { projection: Arc, ) -> BoxFuture<'_, Result> { async move { - let projection = ReaderProjection::from_field_ids( + let projection = file_versions::reader_projection_from_field_ids( self.reader.version(), projection.as_ref(), self.field_id_to_column_idx.as_ref(), @@ -423,7 +400,7 @@ mod v2_adapter { projection: Arc, ) -> BoxFuture<'_, Result> { async move { - let projection = ReaderProjection::from_field_ids( + let projection = file_versions::reader_projection_from_field_ids( self.reader.version(), projection.as_ref(), self.field_id_to_column_idx.as_ref(), @@ -455,7 +432,7 @@ mod v2_adapter { ) -> BoxFuture<'_, Result> { let indices = UInt32Array::from(indices.to_vec()); async move { - let projection = ReaderProjection::from_field_ids( + let projection = file_versions::reader_projection_from_field_ids( self.reader.version(), projection.as_ref(), self.field_id_to_column_idx.as_ref(), @@ -524,18 +501,6 @@ mod v2_adapter { fn clone_box(&self) -> Box { Box::new(self.clone()) } - - fn is_legacy(&self) -> bool { - false - } - - fn as_legacy_opt(&self) -> Option<&PreviousFileReader> { - None - } - - fn as_legacy_opt_mut(&mut self) -> Option<&mut PreviousFileReader> { - None - } } } @@ -634,21 +599,9 @@ impl GenericFileReader for NullReader { fn clone_box(&self) -> Box { Box::new(self.clone()) } - - fn is_legacy(&self) -> bool { - false - } - - fn as_legacy_opt(&self) -> Option<&PreviousFileReader> { - None - } - - fn as_legacy_opt_mut(&mut self) -> Option<&mut PreviousFileReader> { - None - } } -#[derive(Debug, Default)] +#[derive(Debug, Default, Clone)] pub struct FragReadConfig { // Add the row id column pub with_row_id: bool, @@ -720,7 +673,7 @@ impl FragReadConfig { } #[derive(Debug, Clone, Copy, PartialEq, Eq)] -enum MetadataMode { +pub(crate) enum MetadataMode { LazyAllowed, Full, } @@ -777,68 +730,78 @@ impl FileFragment { let file_version = determine_file_version(dataset.object_store.as_ref(), &filepath, None).await?; - if file_version != dataset.manifest.data_storage_format.lance_file_version()? { - return Err(Error::invalid_input(format!( - "File version mismatch. Dataset version: {:?} Fragment version: {:?}", - dataset.manifest.data_storage_format.lance_file_version()?, - file_version - ))); - } + super::versions::create_fragment_from_file( + file_version, + dataset.manifest.data_storage_format.lance_file_format(), + filename, + dataset, + fragment_id, + physical_rows, + ) + .await + } - if file_version == LanceFileVersion::Legacy { - let fragment = Fragment::with_file_legacy( - fragment_id as u64, - filename, - dataset.schema(), - physical_rows, - ); - Ok(fragment) - } else { - // Load the file metadata, confirm the schema is compatible, and - // determine the column offsets - let mut frag = Fragment::new(fragment_id as u64); - let scheduler = ScanScheduler::new( - dataset.object_store.clone(), - SchedulerConfig::max_bandwidth(&dataset.object_store), - ); - let file_scheduler = scheduler - .open_file(&filepath, &CachedFileSize::unknown()) - .await?; - let reader = lance_file::reader::FileReader::try_open( - file_scheduler, - None, - Arc::::default(), - &dataset.metadata_cache.file_metadata_cache(&filepath), - dataset.file_reader_options.clone().unwrap_or_default(), - ) - .await?; - // If the schemas are not compatible we can't calculate field id offsets - reader - .schema() - .check_compatible(dataset.schema(), &SchemaCompareOptions::default())?; - let projection = lance_file::reader::ReaderProjection::from_whole_schema( - dataset.schema(), - reader.metadata().version(), - ); - let physical_rows = reader.metadata().num_rows as usize; - frag.physical_rows = Some(physical_rows); - frag.id = fragment_id as u64; + pub(crate) async fn create_from_v1_file( + filename: &str, + dataset: &Dataset, + fragment_id: usize, + physical_rows: Option, + ) -> Result { + Ok(Fragment::with_file_legacy( + fragment_id as u64, + filename, + dataset.schema(), + physical_rows, + )) + } - let column_indices = projection - .column_indices - .into_iter() - .map(|c| c as i32) - .collect(); + pub(crate) async fn create_from_current_file( + filename: &str, + dataset: &Dataset, + fragment_id: usize, + ) -> Result { + let filepath = dataset.data_dir().join(filename); + // Load the file metadata, confirm the schema is compatible, and + // determine the column offsets + let mut frag = Fragment::new(fragment_id as u64); + let scheduler = ScanScheduler::new( + dataset.object_store.clone(), + SchedulerConfig::max_bandwidth(&dataset.object_store), + ); + let file_scheduler = scheduler + .open_file(&filepath, &CachedFileSize::unknown()) + .await?; + let reader = lance_file::reader::FileReader::try_open( + file_scheduler, + None, + Arc::::default(), + &dataset.metadata_cache.file_metadata_cache(&filepath), + dataset.file_reader_options.clone().unwrap_or_default(), + ) + .await?; + // If the schemas are not compatible we can't calculate field id offsets + reader + .schema() + .check_compatible(dataset.schema(), &SchemaCompareOptions::default())?; + let projection = file_versions::reader_projection_from_whole_schema( + dataset.schema(), + reader.metadata().version(), + ); + frag.physical_rows = Some(reader.metadata().num_rows as usize); - frag.add_file( - filename, - dataset.schema().field_ids(), - column_indices, - &file_version, - None, - ); - Ok(frag) - } + let column_indices = projection + .column_indices + .into_iter() + .map(|column| column as i32) + .collect(); + frag.add_file( + filename, + dataset.schema().field_ids(), + column_indices, + reader.metadata().version(), + None, + ); + Ok(frag) } /// Returns storage stats as `(field_id, bytes_on_disk)` pairs for this fragment. @@ -948,6 +911,19 @@ impl FileFragment { Arc::new(self.metadata.clone()), )?; + // Plan overlay resolution from coverage metadata (no files opened here); the + // readers are opened lazily on read, pruned to the rows each read touches. + if !self.metadata.overlays.is_empty() { + let planner = plan_overlays(self, projection)?; + if !planner.is_empty() { + reader.overlay = Some(OverlayReadState { + planner: Arc::new(planner), + fragment: Arc::new(self.clone()), + read_config: Arc::new(read_config.clone()), + }); + } + } + if read_config.with_row_id { reader.with_row_id(); } @@ -964,28 +940,46 @@ impl FileFragment { Ok(reader) } - fn get_field_id_offset(data_file: &DataFile) -> u32 { - data_file.fields.first().copied().unwrap_or(0) as u32 + pub(crate) async fn open_v1_fragment_reader( + &self, + projection: &Schema, + read_config: &FragReadConfig, + ) -> Result { + let open_readers = async { + let mut readers = Vec::new(); + for data_file in &self.metadata.files { + if let Some(reader) = self.open_v1_reader(data_file, Some(projection)).await? { + readers.push(reader); + } + } + Result::Ok(readers) + }; + let deletion_vec_load = self.get_deletion_vector(); + let row_id_load = if self.dataset.manifest.uses_stable_row_ids() { + futures::future::Either::Left( + load_row_id_sequence(&self.dataset, &self.metadata).map_ok(Some), + ) + } else { + futures::future::Either::Right(futures::future::ready(Ok(None))) + }; + let (readers, deletion_vec, row_id_sequence) = + join!(open_readers, deletion_vec_load, row_id_load); + let mut reader = + V1FragmentReader::try_new(readers?, deletion_vec?, row_id_sequence?, self.id())?; + if read_config.with_row_id { + reader.with_row_id(); + } + if read_config.with_row_address { + reader.with_row_address(); + } + Ok(reader) } - fn should_try_indexed_metadata( - data_file: &DataFile, - projection: &ReaderProjection, - file_version: LanceFileVersion, - ) -> bool { - if !ProjectedFileReader::supports_projection(projection, file_version) { - return false; - } - let total_columns = data_file - .column_indices - .iter() - .filter(|column_index| **column_index >= 0) - .count(); - let selected_columns = projection.column_indices.len(); - selected_columns.saturating_mul(4) < total_columns + fn get_field_id_offset(data_file: &DataFile) -> u32 { + data_file.fields.first().copied().unwrap_or(0) as u32 } - async fn open_reader( + pub(super) async fn open_reader( &self, data_file: &DataFile, projection: Option<&Schema>, @@ -1018,140 +1012,173 @@ impl FileFragment { metadata_mode: MetadataMode, ) -> BoxFuture<'a, Result>>> { async move { - let full_schema = self.dataset.schema(); - // The data file may contain fields that are not part of the dataset any longer, remove those - let data_file_schema = Arc::new(data_file.schema(full_schema)); - let projection = projection.unwrap_or(full_schema); - // Also remove any fields that are not part of the user's provided projection - let schema_per_file = - Arc::new(projection.intersection_ignore_types(data_file_schema.as_ref())?); - - if data_file.is_legacy_file() { - let max_field_id = data_file.fields.iter().max().unwrap(); - if !schema_per_file.fields.is_empty() { - let path = self - .dataset - .data_file_dir(data_file)? - .join(data_file.path.as_str()); - let object_store = self.dataset.object_store_for_data_file(data_file).await?; - let field_id_offset = Self::get_field_id_offset(data_file); - let reader = PreviousFileReader::try_new_with_fragment_id( - &object_store, - &path, - self.schema().clone(), - self.id() as u32, - field_id_offset as i32, - *max_field_id, - Some(&self.dataset.metadata_cache.file_metadata_cache(&path)), - ) + super::versions::open_file_reader( + data_file.file_version()?, + self, + data_file, + projection, + read_config, + metadata_mode, + ) + .await + } + .boxed() + } + + pub(crate) async fn open_v1_file_reader( + &self, + data_file: &DataFile, + projection: Option<&Schema>, + ) -> Result>> { + Ok(self + .open_v1_reader(data_file, projection) + .await? + .map(|reader| Box::new(reader) as Box)) + } + + async fn open_v1_reader( + &self, + data_file: &DataFile, + projection: Option<&Schema>, + ) -> Result> { + let full_schema = self.dataset.schema(); + let data_file_schema = Arc::new(data_file.schema(full_schema)); + let projection = projection.unwrap_or(full_schema); + let schema_per_file = + Arc::new(projection.intersection_ignore_types(data_file_schema.as_ref())?); + if schema_per_file.fields.is_empty() { + return Ok(None); + } + + let max_field_id = data_file.fields.iter().max().ok_or_else(|| { + Error::invalid_input(format!( + "Legacy data file {} does not contain any fields", + data_file.path + )) + })?; + let path = self + .dataset + .data_file_dir(data_file)? + .join(data_file.path.as_str()); + let object_store = self.dataset.object_store_for_data_file(data_file).await?; + let field_id_offset = Self::get_field_id_offset(data_file); + let reader = V1FileReader::try_new_with_fragment_id( + &object_store, + &path, + self.schema().clone(), + self.id() as u32, + field_id_offset as i32, + *max_field_id, + Some(&self.dataset.metadata_cache.file_metadata_cache(&path)), + ) + .await?; + let initialized_schema = reader.schema().project_by_schema( + schema_per_file.as_ref(), + OnMissing::Error, + OnTypeMismatch::Error, + )?; + Ok(Some(V1Reader::new(reader, Arc::new(initialized_schema)))) + } + + pub(crate) async fn open_current_file_reader( + &self, + data_file: &DataFile, + projection: Option<&Schema>, + read_config: &FragReadConfig, + metadata_mode: MetadataMode, + ) -> Result>> { + let full_schema = self.dataset.schema(); + let data_file_schema = Arc::new(data_file.schema(full_schema)); + let projection = projection.unwrap_or(full_schema); + let schema_per_file = + Arc::new(projection.intersection_ignore_types(data_file_schema.as_ref())?); + if schema_per_file.fields.is_empty() { + return Ok(None); + } + + let path = self + .dataset + .data_file_dir(data_file)? + .join(data_file.path.as_str()); + let (store_scheduler, reader_priority) = if let Some(base_id) = data_file.base_id { + // TODO: make object stores for non-default bases reuse the same scan scheduler + // currently we always create a new one + let object_store = self.dataset.object_store(Some(base_id)).await?; + let config = SchedulerConfig::max_bandwidth(&object_store); + ( + ScanScheduler::new(object_store, config), + read_config.reader_priority.unwrap_or(0), + ) + } else if let Some(scan_scheduler) = read_config.scan_scheduler.as_ref() { + ( + scan_scheduler.clone(), + read_config.reader_priority.unwrap_or(0), + ) + } else { + ( + ScanScheduler::new( + self.dataset.object_store.clone(), + SchedulerConfig::max_bandwidth(&self.dataset.object_store), + ), + 0, + ) + }; + let file_scheduler = store_scheduler + .open_file_with_priority(&path, reader_priority as u64, &data_file.file_size_bytes) + .await?; + let path = file_scheduler.reader().path().clone(); + let metadata_cache = self.dataset.metadata_cache.file_metadata_cache(&path); + let field_id_to_column_idx = Arc::new(BTreeMap::from_iter( + data_file + .fields + .iter() + .copied() + .zip(data_file.column_indices.iter().copied()) + .filter_map(|(field_id, column_index)| { + (column_index >= 0).then_some((field_id as u32, column_index as u32)) + }), + )); + let file_version = data_file.file_version()?; + let reader_projection = file_versions::reader_projection_from_field_ids( + file_version, + schema_per_file.as_ref(), + field_id_to_column_idx.as_ref(), + )?; + let file_reader_options = read_config + .file_reader_options + .clone() + .or_else(|| self.dataset.file_reader_options.clone()) + .unwrap_or_default(); + let prefer_indexed = metadata_mode == MetadataMode::LazyAllowed + && reader_projection.column_indices.len().saturating_mul(4) + < data_file + .column_indices + .iter() + .filter(|column_index| **column_index >= 0) + .count(); + let known_schema = self + .metadata + .physical_rows + .map(|num_rows| (data_file_schema.clone(), num_rows as u64)); + + let encodings_io = Arc::new( + LanceEncodingsIo::new(file_scheduler.clone()) + .with_read_chunk_size(file_reader_options.read_chunk_size), + ); + let reader = file_versions::open_projected_reader( + file_version, + &reader_projection, + prefer_indexed, + || async { + let metadata_index = self + .get_file_metadata_index(&file_scheduler, known_schema.clone()) .await?; - let initialized_schema = reader.schema().project_by_schema( - schema_per_file.as_ref(), - OnMissing::Error, - OnTypeMismatch::Error, - )?; - let reader = V1Reader::new(reader, Arc::new(initialized_schema)); - let reader: Box = Box::new(reader); - Ok(Some(reader)) - } else { - Ok(None) + if (reader_projection.column_indices.len() as u32).saturating_mul(4) + >= metadata_index.num_columns() + { + return Ok(None); } - } else if schema_per_file.fields.is_empty() { - Ok(None) - } else { - let path = self - .dataset - .data_file_dir(data_file)? - .join(data_file.path.as_str()); - let (store_scheduler, reader_priority) = if let Some(base_id) = data_file.base_id { - // TODO: make object stores for non-default bases reuse the same scan scheduler - // currently we always create a new one - let object_store = self.dataset.object_store(Some(base_id)).await?; - let config = SchedulerConfig::max_bandwidth(&object_store); - ( - ScanScheduler::new(object_store, config), - read_config.reader_priority.unwrap_or(0), - ) - } else if let Some(scan_scheduler) = read_config.scan_scheduler.as_ref() { - ( - scan_scheduler.clone(), - read_config.reader_priority.unwrap_or(0), - ) - } else { - ( - ScanScheduler::new( - self.dataset.object_store.clone(), - SchedulerConfig::max_bandwidth(&self.dataset.object_store), - ), - 0, - ) - }; - let file_scheduler = store_scheduler - .open_file_with_priority( - &path, - reader_priority as u64, - &data_file.file_size_bytes, - ) - .await?; - let path = file_scheduler.reader().path().clone(); - let metadata_cache = self.dataset.metadata_cache.file_metadata_cache(&path); - let field_id_to_column_idx = Arc::new(BTreeMap::from_iter( - data_file - .fields - .iter() - .copied() - .zip(data_file.column_indices.iter().copied()) - .filter_map(|(field_id, column_index)| { - if column_index < 0 { - None - } else { - Some((field_id as u32, column_index as u32)) - } - }), - )); - let file_version = LanceFileVersion::try_from_major_minor( - data_file.file_major_version, - data_file.file_minor_version, - )?; - let reader_projection = ReaderProjection::from_field_ids( - file_version, - schema_per_file.as_ref(), - field_id_to_column_idx.as_ref(), - )?; - let file_reader_options = read_config - .file_reader_options - .clone() - .or_else(|| self.dataset.file_reader_options.clone()) - .unwrap_or_default(); - let metadata_index = if metadata_mode == MetadataMode::LazyAllowed - && Self::should_try_indexed_metadata( - data_file, - &reader_projection, - file_version, - ) { - let known_schema = self - .metadata - .physical_rows - .map(|num_rows| (data_file_schema.clone(), num_rows as u64)); - let metadata_index = self - .get_file_metadata_index(&file_scheduler, known_schema) - .await?; - if (reader_projection.column_indices.len() as u32).saturating_mul(4) - < metadata_index.num_columns() - { - Some(metadata_index) - } else { - None - } - } else { - None - }; - - let encodings_io = Arc::new( - LanceEncodingsIo::new(file_scheduler.clone()) - .with_read_chunk_size(file_reader_options.read_chunk_size), - ); - let reader = if let Some(metadata_index) = metadata_index { + Ok(Some( ProjectedFileReader::try_open_with_metadata_index( encodings_io.clone(), path.clone(), @@ -1161,32 +1188,31 @@ impl FileFragment { &metadata_cache, file_reader_options.clone(), ) - .await? - } else { - let file_metadata = self.get_file_metadata(&file_scheduler).await?; - ProjectedFileReader::try_open_with_file_metadata( - encodings_io, - path.clone(), - None, - Arc::::default(), - file_metadata, - &metadata_cache, - file_reader_options, - ) - .await? - }; - let reader = v2_adapter::Reader::new( - Arc::new(reader), - schema_per_file, - field_id_to_column_idx, - reader_priority, - file_scheduler, - ); - let reader: Box = Box::new(reader); - Ok(Some(reader)) - } - } - .boxed() + .await?, + )) + }, + || async { + let file_metadata = self.get_file_metadata(&file_scheduler).await?; + ProjectedFileReader::try_open_with_file_metadata( + encodings_io.clone(), + path.clone(), + None, + Arc::::default(), + file_metadata, + &metadata_cache, + file_reader_options.clone(), + ) + .await + }, + ) + .await?; + Ok(Some(Box::new(v2_adapter::Reader::new( + Arc::new(reader), + schema_per_file, + field_id_to_column_idx, + reader_priority, + file_scheduler, + )))) } async fn open_readers( @@ -1402,6 +1428,11 @@ impl FileFragment { for data_file in &self.metadata.files { let last = -1; for field_id in data_file.fields.iter() { + // A tombstone marks a field superseded by a later data file. + // It is not a field id: it has no ordering and can repeat. + if *field_id == TOMBSTONE_FIELD_ID { + continue; + } if *field_id <= last { return Err(Error::corrupt_file( self.dataset @@ -1428,14 +1459,15 @@ impl FileFragment { } } - if self.metadata.files.iter().any(|f| f.is_legacy_file()) - != self.metadata.files.iter().all(|f| f.is_legacy_file()) - { + if let Err(error) = Fragment::try_infer_version(std::slice::from_ref(&self.metadata)) { + let first_file = self.metadata.files.first().ok_or_else(|| { + Error::internal("mixed file versions reported for an empty fragment") + })?; return Err(Error::corrupt_file( self.dataset - .data_file_dir(&self.metadata.files[0])? - .join(self.metadata.files[0].path.as_str()), - "Fragment contains a mix of v1 and v2 data files".to_string(), + .data_file_dir(first_file)? + .join(first_file.path.as_str()), + format!("Fragment contains mixed file versions: {error}"), )); } @@ -1603,7 +1635,7 @@ impl FileFragment { } /// Get the file metadata for this fragment, using the cache if available. - async fn get_file_metadata( + pub async fn get_file_metadata( &self, file_scheduler: &FileScheduler, ) -> Result> { @@ -1620,7 +1652,7 @@ impl FileFragment { Ok(file_metadata) } - async fn get_file_metadata_index( + pub async fn get_file_metadata_index( &self, file_scheduler: &FileScheduler, known_schema: Option<(Arc, u64)>, @@ -1678,7 +1710,7 @@ impl FileFragment { if row_offsets.len() > 1 && Self::row_ids_contiguous(row_offsets) { let range = (row_offsets[0] as usize)..(row_offsets[row_offsets.len() - 1] as usize + 1); - reader.legacy_read_range_as_batch(range).await + reader.read_range_as_batch(range).await } else { // FIXME, change this method to streams reader.take_as_batch(row_offsets, None).await @@ -1726,11 +1758,15 @@ impl FileFragment { /// at a time. This can be useful to control memory usage when processing very large /// fields. The batch_size will only be used if the dataset is a v2 dataset. It will /// be ignored for v1 datasets. + /// + /// The `blob_handling` parameter controls the in-memory representation of blob + /// columns read by the updater. If unset, the dataset schema is used unchanged. pub(crate) async fn updater>( &self, columns: Option<&[T]>, schemas: Option<(Schema, Schema)>, batch_size: Option, + blob_handling: Option, ) -> Result { let mut schema = self.dataset.schema().clone(); @@ -1750,6 +1786,14 @@ impl FileFragment { schema = schema.project(&projection)?; } + if let Some(blob_handling) = blob_handling { + schema.fields = schema + .fields + .into_iter() + .map(|field| blob_handling.unload_if_needed(field)) + .collect(); + } + // If there is no projection, we at least need to read the row addresses with_row_addr |= !with_row_id && schema.fields.is_empty(); @@ -1819,7 +1863,7 @@ impl FileFragment { } pub(crate) async fn merge(mut self, join_column: &str, joiner: &HashJoiner) -> Result { - let mut updater = self.updater(Some(&[join_column]), None, None).await?; + let mut updater = self.updater(Some(&[join_column]), None, None, None).await?; while let Some(batch) = updater.next().await? { let batch = joiner @@ -1898,14 +1942,62 @@ impl FileFragment { if !read_columns.iter().any(|n| n.as_str() == ROW_ADDR) { read_columns.push(ROW_ADDR.to_string()); } + let selected_field_ids = read_columns + .iter() + .filter_map(|column| self.schema().field(column)) + .map(|field| field.id) + .collect::>(); + let descriptor_blob_ids = self + .schema() + .project_by_ids(&selected_field_ids, true) + .fields_pre_order() + .filter(|field| field.is_blob_v2()) + .filter_map(|field| u32::try_from(field.id).ok()) + .collect::>(); + let has_blob_v2 = !descriptor_blob_ids.is_empty(); + let blob_handling = has_blob_v2.then(|| { + let materialized_blob_ids = self + .schema() + .fields_pre_order() + .filter(|field| field.is_blob()) + .filter_map(|field| u32::try_from(field.id).ok()) + .filter(|field_id| !descriptor_blob_ids.contains(field_id)) + .collect(); + BlobHandling::SomeBlobsBinary(materialized_blob_ids) + }); let mut updater = self .updater( Some(&read_columns), Some((write_schema.clone(), self.schema().clone())), None, + blob_handling, ) .await?; + if has_blob_v2 { + updater.allow_external_blob_outside_bases(); + } + let external_base_resolver = if has_blob_v2 { + super::write::blob_v2_external_base_resolver( + Some(self.dataset()), + &WriteParams::default(), + &write_schema, + ) + .await? + } else { + None + }; // Hash join: rows matched on the right-hand stream rewrite columns; track physical offsets via `_rowaddr`. + // Convert Arrow JSON columns (Utf8) to Lance JSON (LargeBinary) in the right stream + // so they match the physical storage format read from the fragment's left batch. + let right_stream: Box = if right_schema + .fields() + .iter() + .any(|f| is_arrow_json_field(f) || has_json_fields(f)) + { + Box::new(JsonConvertingReader::new(right_stream)) + } else { + right_stream + }; let joiner = Arc::new(HashJoiner::try_new(right_stream, right_on).await?); let mut matched_offsets = RoaringBitmap::new(); let frag_id_u32 = u32::try_from(self.metadata.id).map_err(|_| { @@ -1915,6 +2007,17 @@ impl FileFragment { )) })?; while let Some(batch) = updater.next().await? { + let batch = if has_blob_v2 { + crate::dataset::optimize::transform_blob_v2_batch( + &self.dataset, + self.schema(), + batch.clone(), + true, + ) + .await? + } else { + batch.clone() + }; let index_column = batch[left_on].clone(); let matched = joiner.matched_join_rows(index_column.clone())?; if let Some(addr_col) = batch.column_by_name(ROW_ADDR) { @@ -1930,8 +2033,12 @@ impl FileFragment { } } let updated_batch = joiner - .collect_with_fallback(batch, index_column, self.dataset()) + .collect_with_fallback(&batch, index_column, self.dataset()) .await?; + if let Some(resolver) = external_base_resolver.as_deref() { + super::blob::validate_external_blob_references(resolver, &updated_batch, &matched) + .await?; + } updater.update(updated_batch).await?; } @@ -1977,7 +2084,7 @@ impl FileFragment { read_columns: Option>, batch_size: Option, ) -> Result<(Fragment, Schema)> { - let (fragments, schema, _) = schema_evolution::add_columns_to_fragments( + let (fragments, schema, _, _) = schema_evolution::add_columns_to_fragments( self.dataset.as_ref(), transforms, read_columns, @@ -2166,6 +2273,12 @@ impl CacheKey for FileMetadataCacheKey { fn type_name() -> &'static str { "FileMetadata" } + + fn schema() -> CacheKeySchema { + CacheKeySchema::new("lance.dataset.fragment-file-metadata-key", 1) + } + + fn write_key(&self, _builder: &mut KeyBuilder) {} } #[derive(Debug, Clone)] @@ -2181,6 +2294,12 @@ impl CacheKey for FileMetadataIndexCacheKey { fn type_name() -> &'static str { "FileMetadataIndex" } + + fn schema() -> CacheKeySchema { + CacheKeySchema::new("lance.dataset.fragment-file-metadata-index-key", 1) + } + + fn write_key(&self, _builder: &mut KeyBuilder) {} } impl From for Fragment { @@ -2189,16 +2308,217 @@ impl From for Fragment { } } -/// [`FragmentReader`] is an abstract reader for a [`FileFragment`]. +/// Typed v1-only read operations used by the legacy pushdown path. /// -/// It opens the data files that contains the columns of the projection schema, and -/// reconstruct the RecordBatch from columns read from each data file. -#[derive(Debug)] -pub struct FragmentReader { - /// Readers and schema of each opened data file. - readers: Vec>, - - /// The output schema. The defines the order in which the columns are returned. +/// Keeping the previous readers here avoids exposing legacy downcasts through +/// [`GenericFileReader`]. Modern readers never implement or simulate these +/// row-group and page-statistics operations. +#[derive(Clone, Debug)] +pub(crate) struct V1FragmentReader { + readers: Vec, + deletion_vec: Option>, + row_id_sequence: Option>, + fragment_id: usize, + with_row_id: bool, + with_row_addr: bool, + make_deletions_null: bool, +} + +impl V1FragmentReader { + fn try_new( + readers: Vec, + deletion_vec: Option>, + row_id_sequence: Option>, + fragment_id: usize, + ) -> Result { + let first_reader = readers.first().ok_or_else(|| { + Error::invalid_input("Cannot create a v1 fragment reader without data files") + })?; + let num_batches = first_reader.reader.num_batches(); + if readers + .iter() + .any(|reader| reader.reader.num_batches() != num_batches) + { + return Err(Error::invalid_input( + "Cannot create a v1 fragment reader from data files with different numbers of batches" + .to_string(), + )); + } + Ok(Self { + readers, + deletion_vec, + row_id_sequence, + fragment_id, + with_row_id: false, + with_row_addr: false, + make_deletions_null: false, + }) + } + + pub(crate) fn with_row_id(&mut self) -> &mut Self { + self.with_row_id = true; + self + } + + pub(crate) fn with_row_address(&mut self) -> &mut Self { + self.with_row_addr = true; + self + } + + pub(crate) fn with_make_deletions_null(&mut self) -> &mut Self { + self.make_deletions_null = true; + self + } + + pub(crate) fn num_batches(&self) -> usize { + self.readers[0].reader.num_batches() + } + + pub(crate) fn num_rows_in_batch(&self, batch_id: u32) -> Option { + let reader = &self.readers[0].reader; + (batch_id < reader.num_batches() as u32) + .then(|| reader.num_rows_in_batch(batch_id as i32) as u32) + } + + pub(crate) async fn read_page_stats( + &self, + projection: Option<&Schema>, + ) -> Result> { + let mut stats_batches = Vec::new(); + for reader in &self.readers { + let schema = match projection { + Some(projection) => Arc::new(reader.projection.intersection(projection)?), + None => reader.projection.clone(), + }; + if let Some(stats_batch) = reader.reader.read_page_stats(&schema.field_ids()).await? { + stats_batches.push(stats_batch); + } + } + + if stats_batches.is_empty() { + Ok(None) + } else { + Ok(Some(merge_batches(&stats_batches)?)) + } + } + + pub(crate) async fn read_batch_projected( + &self, + batch_id: usize, + params: impl Into + Clone, + projection: &Schema, + ) -> Result { + let first_reader = &self.readers[0].reader; + // All batches have the same size in v1, except for the last one. + let batch_offset = batch_id * first_reader.num_rows_in_batch(0); + let rows_in_batch = first_reader.num_rows_in_batch(batch_id as i32); + + let batches = if !projection.fields.is_empty() { + let read_tasks = self.readers.iter().map(|reader| { + let projection = reader.projection.intersection(projection); + let params = params.clone(); + let reader = &reader.reader; + + async move { + let projection = projection?; + if projection.fields.is_empty() { + Result::Ok(None) + } else { + Ok(Some( + reader + .read_batch(batch_id as i32, params, &projection) + .await?, + )) + } + } + }); + try_join_all(read_tasks) + .await? + .into_iter() + .flatten() + .collect::>() + } else { + let expected_rows = params + .clone() + .into() + .slice(0, rows_in_batch) + .unwrap() + .to_offsets()? + .len(); + vec![RecordBatch::from(StructArray::new_empty_fields( + expected_rows, + None, + ))] + }; + + let params = params.into(); + let result = merge_batches(&batches)?; + let file_params = match params { + ReadBatchParams::Indices(indices) => ReadBatchParams::Indices( + indices + .values() + .iter() + .map(|i| *i + batch_offset as u32) + .collect(), + ), + ReadBatchParams::Ranges(_) => { + return Err(Error::internal( + "ReadBatchParams::Ranges should not be used in v1 files".to_string(), + )); + } + ReadBatchParams::RangeFull => { + ReadBatchParams::Range(batch_offset..(batch_offset + rows_in_batch)) + } + ReadBatchParams::RangeFrom(start) => { + ReadBatchParams::Range((start.start + batch_offset)..(batch_offset + rows_in_batch)) + } + ReadBatchParams::RangeTo(end) => { + ReadBatchParams::Range(batch_offset..(end.end + batch_offset)) + } + ReadBatchParams::Range(range) => { + ReadBatchParams::Range((range.start + batch_offset)..(range.end + batch_offset)) + } + }; + let result = lance_table::utils::stream::apply_row_id_and_deletes( + result, + 0, + self.fragment_id as u32, + &RowIdAndDeletesConfig { + params: file_params, + deletion_vector: self.deletion_vec.clone(), + row_id_sequence: self.row_id_sequence.clone(), + with_row_id: self.with_row_id, + with_row_addr: self.with_row_addr, + with_row_last_updated_at_version: false, + with_row_created_at_version: false, + last_updated_at_sequence: None, + created_at_sequence: None, + make_deletions_null: self.make_deletions_null, + total_num_rows: first_reader.len() as u32, + }, + )?; + + let mut output_schema = ArrowSchema::from(projection); + if self.with_row_id { + output_schema = output_schema.try_with_column(ROW_ID_FIELD.clone())?; + } + if self.with_row_addr { + output_schema = output_schema.try_with_column(ROW_ADDR_FIELD.clone())?; + } + Ok(result.project_by_schema(&output_schema)?) + } +} + +/// [`FragmentReader`] is an abstract reader for a [`FileFragment`]. +/// +/// It opens the data files that contains the columns of the projection schema, and +/// reconstruct the RecordBatch from columns read from each data file. +#[derive(Debug)] +pub struct FragmentReader { + /// Readers and schema of each opened data file. + readers: Vec>, + + /// The output schema. The defines the order in which the columns are returned. output_schema: ArrowSchema, /// The deleted row IDs @@ -2242,6 +2562,23 @@ pub struct FragmentReader { // total number of physical rows in the fragment (all rows, ignoring deletions) num_physical_rows: usize, + + /// Read-time state for resolving data overlay files: the coverage plan plus + /// what is needed to open overlay readers. `None` when the fragment has no + /// overlays. Overlays are merged into base batches (by `offset_in_frag`) before + /// deletion filtering, opening only the files each read's rows touch. + overlay: Option, +} + +/// What [`FragmentReader`] needs to resolve overlays at read time: the coverage +/// plan (from metadata, cheap to build), and the fragment + config needed to open +/// overlay readers once the read's rows — and therefore which files it touches — +/// are known. All `Arc` so cloning a reader stays cheap. +#[derive(Clone, Debug)] +struct OverlayReadState { + planner: Arc, + fragment: Arc, + read_config: Arc, } // Custom clone impl needed because it is not easy to clone Box @@ -2270,6 +2607,7 @@ impl Clone for FragmentReader { created_at_sequence: self.created_at_sequence.clone(), num_rows: self.num_rows, num_physical_rows: self.num_physical_rows, + overlay: self.overlay.clone(), } } } @@ -2306,21 +2644,6 @@ impl FragmentReader { num_physical_rows: usize, fragment: Arc, ) -> Result { - if let Some(legacy_reader) = readers.first().and_then(|reader| reader.as_legacy_opt()) { - let num_batches = legacy_reader.num_batches(); - for reader in readers.iter().skip(1) { - if let Some(other_legacy) = reader.as_legacy_opt() { - if other_legacy.num_batches() != num_batches { - return Err(Error::invalid_input("Cannot create FragmentReader from data files with different number of batches" - .to_string())); - } - } else { - return Err(Error::invalid_input( - "Cannot mix legacy and non-legacy readers".to_string(), - )); - } - } - } Ok(Self { readers, output_schema, @@ -2337,6 +2660,7 @@ impl FragmentReader { created_at_sequence: None, num_rows, num_physical_rows, + overlay: None, }) } @@ -2405,194 +2729,76 @@ impl FragmentReader { self } - /// TODO: This method is relied upon by the v1 pushdown mechanism and will need to stay - /// in place until v1 is removed. v2 uses a different mechanism for pushdown and so there - /// is little benefit in updating the v1 pushdown node. - pub(crate) fn legacy_num_batches(&self) -> usize { - let legacy_reader = self.readers[0].as_legacy(); - let num_batches = legacy_reader.num_batches(); - assert!( - self.readers - .iter() - .all(|r| r.as_legacy().num_batches() == num_batches), - "Data files have varying number of batches, which is not yet supported." - ); - num_batches - } - - /// TODO: This method is relied upon by the v1 pushdown mechanism and will need to stay - /// in place until v1 is removed. v2 uses a different mechanism for pushdown and so there - /// is little benefit in updating the v1 pushdown node. + /// Merge data overlay values onto a stream of base batches. /// - /// This method is also used by the updater. Even though the updater has been updated to - /// use streams, the updater still needs to know the batch size in v1 so that it can create - /// files with the same batch size. - pub(crate) fn legacy_num_rows_in_batch(&self, batch_id: u32) -> Option { - if let Some(legacy_reader) = self.readers.first().and_then(|r| r.as_legacy_opt()) { - if batch_id < legacy_reader.num_batches() as u32 { - Some(legacy_reader.num_rows_in_batch(batch_id as i32) as u32) - } else { - None - } - } else { - None - } - } - - /// Read the page statistics of the fragment for the specified fields. + /// Runs on physical rows in read order, *before* deletion filtering, so each + /// row can be addressed by its position in the fragment (its `offset_in_frag`, + /// derived from `params`) and deletions take precedence naturally: an overlay + /// value for a deleted row is dropped along with the row downstream. A no-op + /// when the fragment has no overlays. /// - /// TODO: This method is relied upon by the v1 pushdown mechanism and will need to stay - /// in place until v1 is removed. v2 uses a different mechanism for pushdown and so there - /// is little benefit in updating the v1 pushdown node. - pub(crate) async fn legacy_read_page_stats( + /// The read's `offset_in_frag` values are known from `params` up front, so + /// overlays are resolved here to just the files this read's rows touch — an + /// overlay whose cells fall outside the read is not opened at all. Within each + /// batch, the overlay reads (only the values that batch needs) are then issued + /// concurrently with the base read rather than after it. + async fn merge_overlays( &self, - projection: Option<&Schema>, - ) -> Result> { - let mut stats_batches = vec![]; - for reader in self.readers.iter() { - let schema = match projection { - Some(projection) => Arc::new(reader.projection().intersection(projection)?), - None => reader.projection().clone(), - }; - let reader = reader.as_legacy(); - if let Some(stats_batch) = reader.read_page_stats(&schema.field_ids()).await? { - stats_batches.push(stats_batch); - } - } - - if stats_batches.is_empty() { - Ok(None) - } else { - Ok(Some(merge_batches(&stats_batches)?)) + merged: ReadBatchTaskStream, + params: &ReadBatchParams, + total_num_rows: u32, + ) -> Result { + let Some(overlay) = &self.overlay else { + return Ok(merged); + }; + // The offset_in_frag of every row this read will return, materialized once. + // Cost is one u32 per output row (a whole-fragment scan is 4 bytes/row), and + // it lets us both prune overlays to the read and slice each batch's offsets + // below without reading any data. Only paid when the fragment has overlays. + // + // TODO(overlay perf): this could be avoided by teaching `ReadBatchParams` to + // yield a coverage bitmap directly (for pruning) and to slice per batch (for + // the routing below), or by moving `ReadBatchParams` to a roaring bitmap + // wholesale — a larger refactor tracked separately. + let offsets_in_frag: Arc> = + Arc::new(params.to_offsets_total(total_num_rows).values().to_vec()); + + // Open only the overlay readers this read touches (pruned by row selection). + let plans = resolve_overlays( + &overlay.planner, + &offsets_in_frag, + &overlay.fragment, + &overlay.read_config, + ) + .await?; + if plans.is_empty() { + return Ok(merged); } - } - - /// Read a batch of rows from the fragment, with a subset of columns. - /// - /// Note: the projection must be a subset of the schema the reader was created with. - /// Otherwise incorrect data will be returned. - /// - /// TODO: This method is relied upon by the v1 pushdown mechanism and will need to stay - /// in place until v1 is removed. v2 uses a different mechanism for pushdown and so there - /// is little benefit in updating the v1 pushdown node. - pub(crate) async fn legacy_read_batch_projected( - &self, - batch_id: usize, - params: impl Into + Clone, - projection: &Schema, - ) -> Result { - let first_reader = self.readers[0].as_legacy(); - // All batches have the same size in v1, except for the last one. - let batch_offset = batch_id * first_reader.num_rows_in_batch(0); - let rows_in_batch = first_reader.num_rows_in_batch(batch_id as i32); - - let batches = if !projection.fields.is_empty() { - let read_tasks = self.readers.iter().map(|reader| { - let projection = reader.projection().intersection(projection); - let params = params.clone(); - - let reader = reader.as_legacy(); - - async move { - // Apply ? inside the task to keep read_tasks a simple iter of futures - // for try_join_all - let projection = projection?; - if projection.fields.is_empty() { - // The projection caused one of the data files to become - // irrelevant and so we can skip it - Result::Ok(None) - } else { - Ok(Some( - reader - .read_batch(batch_id as i32, params, &projection) - .await?, - )) + let plans = Arc::new(plans); + + // Batches arrive in physical read order, so a running total of the rows seen + // so far gives each batch its starting offset_in_batch into `offsets_in_frag`. + let mut rows_seen = 0usize; + let stream = merged + .map(move |task| { + let num_rows = task.num_rows; + let start = rows_seen; + rows_seen += num_rows as usize; + let offsets_in_frag = offsets_in_frag.clone(); + let plans = plans.clone(); + let inner = task.task; + ReadBatchTask { + num_rows, + task: async move { + let batch_offsets = &offsets_in_frag[start..start + num_rows as usize]; + merge_overlay_batch(inner, batch_offsets, &plans).await } + .boxed(), } - }); - let results = try_join_all(read_tasks).await?; - results.into_iter().flatten().collect::>() - } else { - // If we are selecting no columns, we can assume we are just getting - // the row ids. If this is the case, we need to generate an empty - // batch with the correct number of rows. - let expected_rows = params - .clone() - .into() - .slice(0, rows_in_batch) - .unwrap() - .to_offsets()? - .len(); - vec![RecordBatch::from(StructArray::new_empty_fields( - expected_rows, - None, - ))] - }; - - let params = params.into(); - let result = merge_batches(&batches)?; - - // Need to apply deletions and row ids. - // In order to apply deletions we need to change the parameters to be - // relative to the file, not the batch. - let file_params = match params { - ReadBatchParams::Indices(indices) => ReadBatchParams::Indices( - indices - .values() - .iter() - .map(|i| *i + batch_offset as u32) - .collect(), - ), - ReadBatchParams::Ranges(_) => { - return Err(Error::internal( - "ReadBatchParams::Ranges should not be used in v1 files".to_string(), - )); - } - ReadBatchParams::RangeFull => { - ReadBatchParams::Range(batch_offset..(batch_offset + rows_in_batch)) - } - ReadBatchParams::RangeFrom(start) => { - ReadBatchParams::Range((start.start + batch_offset)..(batch_offset + rows_in_batch)) - } - ReadBatchParams::RangeTo(end) => { - ReadBatchParams::Range(batch_offset..(end.end + batch_offset)) - } - ReadBatchParams::Range(range) => { - ReadBatchParams::Range((range.start + batch_offset)..(range.end + batch_offset)) - } - }; - let result = lance_table::utils::stream::apply_row_id_and_deletes( - result, - 0, - self.fragment_id as u32, - &RowIdAndDeletesConfig { - params: file_params, - deletion_vector: self.deletion_vec.clone(), - row_id_sequence: self.row_id_sequence.clone(), - with_row_id: self.with_row_id, - with_row_addr: self.with_row_addr, - with_row_last_updated_at_version: self.with_row_last_updated_at_version, - with_row_created_at_version: self.with_row_created_at_version, - last_updated_at_sequence: self.last_updated_at_sequence.clone(), - created_at_sequence: self.created_at_sequence.clone(), - make_deletions_null: self.make_deletions_null, - total_num_rows: first_reader.len() as u32, - }, - )?; - - let output_schema = { - let mut output_schema = ArrowSchema::from(projection); - if self.with_row_id { - output_schema = output_schema.try_with_column(ROW_ID_FIELD.clone())?; - } - if self.with_row_addr { - output_schema = output_schema.try_with_column(ROW_ADDR_FIELD.clone())?; - } - output_schema - }; - - Ok(result.project_by_schema(&output_schema)?) - } + }) + .boxed(); + Ok(stream) + } async fn new_read_impl<'a, F>( &'a self, @@ -2661,6 +2867,8 @@ impl FragmentReader { lance_table::utils::stream::merge_streams(read_streams) }; + let merged = self.merge_overlays(merged, ¶ms, total_num_rows).await?; + // Add the row id column (if needed) and delete rows (if a deletion // vector is present). let config = RowIdAndDeletesConfig { @@ -2831,6 +3039,11 @@ impl FragmentReader { lance_table::utils::stream::merge_streams(read_streams) }; + let params = ReadBatchParams::Ranges(ranges); + let merged_stream = self + .merge_overlays(merged_stream, ¶ms, total_num_rows) + .await?; + // Add the row id column (if needed) and delete rows (if a deletion // vector is present). let config = RowIdAndDeletesConfig { @@ -2843,7 +3056,7 @@ impl FragmentReader { with_row_created_at_version: self.with_row_created_at_version, last_updated_at_sequence: self.last_updated_at_sequence.clone(), created_at_sequence: self.created_at_sequence.clone(), - params: ReadBatchParams::Ranges(ranges), + params, total_num_rows, }; let output_schema = Arc::new(self.output_schema.clone()); @@ -2864,11 +3077,8 @@ impl FragmentReader { ) } - // Legacy function that reads a range of data and concatenates the results - // into a single batch - // - // TODO: Move away from this by changing callers to support consuming a stream - pub async fn legacy_read_range_as_batch(&self, range: Range) -> Result { + /// Reads a range and concatenates the result into one batch. + pub async fn read_range_as_batch(&self, range: Range) -> Result { let batches = self .take_range( range.start as u32..range.end as u32, @@ -2930,113 +3140,1831 @@ impl FragmentReader { (Cow::Borrowed(indices), None) }; - let batches = self - .take(&unique_indices, u32::MAX, take_priority) - .await? - .buffered(get_num_compute_intensive_cpus()) - .try_collect::>() - .await?; - let mut batch = concat_batches(&Arc::new(self.output_schema.clone()), batches.iter())?; + let batches = self + .take(&unique_indices, u32::MAX, take_priority) + .await? + .buffered(get_num_compute_intensive_cpus()) + .try_collect::>() + .await?; + let mut batch = concat_batches(&Arc::new(self.output_schema.clone()), batches.iter())?; + + if let Some(expand_map) = expand_map { + batch = arrow_select::take::take_record_batch(&batch, &expand_map)?; + } + + Ok(batch) + } +} + +/// A wrapper around a `RecordBatchReader` that converts Arrow JSON columns +/// (Utf8/LargeUtf8 with `arrow.json` extension) to Lance JSON columns +/// (LargeBinary with `lance.json` extension / JSONB format). +/// +/// This is needed when user-provided data contains Arrow JSON fields but the +/// dataset stores them in Lance's JSONB binary format. +struct JsonConvertingReader { + inner: Box, + schema: arrow_schema::SchemaRef, +} + +impl JsonConvertingReader { + fn new(inner: Box) -> Self { + use lance_arrow::json::arrow_json_to_lance_json; + + // Build the converted schema (Arrow JSON fields → Lance JSON fields) + let orig_schema = inner.schema(); + let new_fields: Vec = orig_schema + .fields() + .iter() + .map(|f| { + if is_arrow_json_field(f) || has_json_fields(f) { + Arc::new(arrow_json_to_lance_json(f)) + } else { + Arc::clone(f) + } + }) + .collect(); + let schema = Arc::new(arrow_schema::Schema::new_with_metadata( + new_fields, + orig_schema.metadata().clone(), + )); + + Self { inner, schema } + } +} + +impl Iterator for JsonConvertingReader { + type Item = std::result::Result; + + fn next(&mut self) -> Option { + self.inner + .next() + .map(|result| result.and_then(|batch| convert_json_columns(&batch))) + } +} + +impl RecordBatchReader for JsonConvertingReader { + fn schema(&self) -> arrow_schema::SchemaRef { + self.schema.clone() + } +} + +#[cfg(test)] +mod tests { + use arrow_arith::numeric::mul; + use arrow_array::{ + ArrayRef, BooleanArray, Int32Array, Int64Array, RecordBatchIterator, StringArray, + }; + use arrow_schema::{DataType, Field as ArrowField, Schema as ArrowSchema}; + use lance_core::ROW_ID; + use lance_core::utils::tempfile::TempStrDir; + use lance_datagen::{RowCount, array, gen_batch}; + use lance_file::version::{ConcreteFileVersion, LanceFileVersion}; + use lance_file::writer::FileWriterOptions; + use lance_io::{assert_io_eq, assert_io_lt, object_store::ObjectStore}; + use pretty_assertions::assert_eq; + use rstest::rstest; + use std::collections::HashMap; + + use super::*; + use crate::{ + dataset::{ + InsertBuilder, + transaction::{Operation, UpdateMode, UpdatedFragmentOffsets}, + }, + session::Session, + utils::test::TestDatasetGenerator, + }; + + async fn create_dataset(test_uri: &str, data_storage_version: LanceFileVersion) -> Dataset { + let schema = Arc::new(ArrowSchema::new(vec![ + ArrowField::new("i", DataType::Int32, true), + ArrowField::new("s", DataType::Utf8, true), + ])); + + let batches: Vec = (0..10) + .map(|i| { + RecordBatch::try_new( + schema.clone(), + vec![ + Arc::new(Int32Array::from_iter_values(i * 20..(i + 1) * 20)), + Arc::new(StringArray::from_iter_values( + (i * 20..(i + 1) * 20).map(|v| format!("s-{}", v)), + )), + ], + ) + .unwrap() + }) + .collect(); + + let write_params = WriteParams { + max_rows_per_file: 40, + max_rows_per_group: 10, + data_storage_version: Some(data_storage_version), + ..Default::default() + }; + let batches = RecordBatchIterator::new(batches.into_iter().map(Ok), schema.clone()); + Dataset::write(batches, test_uri, Some(write_params)) + .await + .unwrap(); + + Dataset::open(test_uri).await.unwrap() + } + + async fn create_dataset_v2(test_uri: &str) -> Dataset { + let schema = Arc::new(ArrowSchema::new(vec![ArrowField::new( + "i", + DataType::Int32, + true, + )])); + + let batches: Vec = (0..10) + .map(|i| { + RecordBatch::try_new( + schema.clone(), + vec![Arc::new(Int32Array::from_iter_values(i * 20..(i + 1) * 20))], + ) + .unwrap() + }) + .collect(); + + let write_params = WriteParams { + max_rows_per_file: 40, + max_rows_per_group: 10, + data_storage_version: Some(LanceFileVersion::Stable), + ..Default::default() + }; + let batches = RecordBatchIterator::new(batches.into_iter().map(Ok), schema.clone()); + Dataset::write(batches, test_uri, Some(write_params)) + .await + .unwrap(); + + Dataset::open(test_uri).await.unwrap() + } + + /// End-to-end tests for reading data overlay files (OSS-1324): overlays are + /// written, committed via the `DataOverlay` transaction, and then resolved on + /// the `take` and scan read paths. + mod overlay_read { + use std::sync::Arc; + + use arrow_array::{ + Array, ArrayRef, Int32Array, RecordBatch, RecordBatchIterator, StructArray, UInt64Array, + }; + use arrow_schema::{DataType, Field as ArrowField, Fields, Schema as ArrowSchema}; + use lance_core::datatypes::Schema; + use lance_file::version::LanceFileVersion; + use lance_file::writer::FileWriterOptions; + use lance_io::utils::CachedFileSize; + use lance_table::format::DataFile; + use lance_table::format::overlay::{DataOverlayFile, OverlayCoverage}; + use object_store::path::Path; + use roaring::RoaringBitmap; + use rstest::rstest; + + use crate::dataset::transaction::{DataOverlayGroup, Operation}; + use crate::dataset::{Dataset, WriteDestination, WriteParams}; + + fn bitmap(offsets: impl IntoIterator) -> RoaringBitmap { + RoaringBitmap::from_iter(offsets) + } + + fn i32_array(values: impl IntoIterator>) -> ArrayRef { + Arc::new(Int32Array::from_iter(values)) + } + + /// Two-fragment Int32 dataset: `id` (field 0) = 0..12 and `val` (field 1) + /// = id * 10, written 6 rows per file (fragments 0 and 1). + /// + async fn create_base_dataset(version: LanceFileVersion) -> Dataset { + let schema = Arc::new(ArrowSchema::new(vec![ + ArrowField::new("id", DataType::Int32, true), + ArrowField::new("val", DataType::Int32, true), + ])); + let batch = RecordBatch::try_new( + schema.clone(), + vec![ + Arc::new(Int32Array::from_iter_values(0..12)), + Arc::new(Int32Array::from_iter_values((0..12).map(|v| v * 10))), + ], + ) + .unwrap(); + let write_params = WriteParams { + max_rows_per_file: 6, + max_rows_per_group: 6, + data_storage_version: Some(version), + ..Default::default() + }; + let reader = RecordBatchIterator::new(vec![Ok(batch)], schema.clone()); + Dataset::write(reader, "memory://", Some(write_params)) + .await + .unwrap() + } + + /// Write an overlay file covering `fields` (dataset field ids) of + /// `fragment_id` with the given coverage and per-field value columns, then + /// commit it as a `DataOverlay` transaction. `name` makes the file unique. + #[allow(clippy::too_many_arguments)] + async fn commit_overlay( + dataset: Dataset, + name: &str, + fragment_id: u64, + fields: &[i32], + coverage: OverlayCoverage, + columns: Vec, + version: LanceFileVersion, + ) -> Dataset { + let read_version = dataset.version().version; + let overlay_schema = dataset.schema().project_by_ids(fields, true); + + let filename = format!("{name}.lance"); + // The manifest records the bare filename; only the physical write is + // data-dir qualified. + let path = dataset.data_dir().join(filename.as_str()); + let obj_writer = dataset.object_store.create(&path).await.unwrap(); + let file_version = version.resolve(); + let mut writer = lance_file::versions::create_writer( + file_version, + obj_writer, + overlay_schema, + FileWriterOptions::default(), + ) + .unwrap(); + for (column_index, array) in columns.into_iter().enumerate() { + writer.write_column(column_index, array).await.unwrap(); + } + let summary = writer.finish().await.unwrap(); + + let mut data_file = DataFile::new_unstarted(filename, file_version); + data_file.fields = writer + .field_id_to_column_indices() + .iter() + .map(|(field_id, _)| *field_id as i32) + .collect::>() + .into(); + data_file.column_indices = writer + .field_id_to_column_indices() + .iter() + .map(|(_, column_index)| *column_index as i32) + .collect::>() + .into(); + data_file.file_size_bytes = CachedFileSize::new(summary.size_bytes); + + let overlay = DataOverlayFile { + data_file, + coverage, + committed_version: 0, + }; + Dataset::commit( + WriteDestination::Dataset(Arc::new(dataset)), + Operation::DataOverlay { + groups: vec![DataOverlayGroup { + fragment_id, + overlays: vec![overlay], + }], + }, + Some(read_version), + None, + None, + Arc::new(Default::default()), + false, + ) + .await + .unwrap() + } + + /// `collect_paths` feeds `deep_clone`'s copy loop, so an overlay data file + /// it omits is referenced by the clone's manifest but never copied. The + /// clone then fails to read the overlaid values. + #[tokio::test] + async fn deep_clone_copies_overlay_files() { + use lance_core::utils::tempfile::TempStdDir; + + let version = LanceFileVersion::Stable; + let test_dir = TempStdDir::default(); + let source_uri = test_dir.join("source").to_str().unwrap().to_string(); + let clone_uri = test_dir.join("clone").to_str().unwrap().to_string(); + + let schema = Arc::new(ArrowSchema::new(vec![ + ArrowField::new("id", DataType::Int32, true), + ArrowField::new("val", DataType::Int32, true), + ])); + let batch = RecordBatch::try_new( + schema.clone(), + vec![ + Arc::new(Int32Array::from_iter_values(0..6)), + Arc::new(Int32Array::from_iter_values((0..6).map(|v| v * 10))), + ], + ) + .unwrap(); + let reader = RecordBatchIterator::new(vec![Ok(batch)], schema.clone()); + let dataset = Dataset::write( + reader, + &source_uri, + Some(WriteParams { + max_rows_per_file: 6, + max_rows_per_group: 6, + data_storage_version: Some(version), + ..Default::default() + }), + ) + .await + .unwrap(); + + // Overlay `val` on offsets 0 and 1 of fragment 0. + let mut dataset = commit_overlay( + dataset, + "overlay0", + 0, + &[1], + OverlayCoverage::Shared(Arc::new(bitmap([0, 1]))), + vec![i32_array([Some(700), Some(701)])], + version, + ) + .await; + + let source_version = dataset.version().version; + dataset + .tags() + .create("clone-me", source_version) + .await + .unwrap(); + let cloned = dataset + .deep_clone(&clone_uri, "clone-me", None) + .await + .unwrap(); + + let batch = cloned + .scan() + .try_into_batch() + .await + .expect("the clone must be readable, including its overlay files"); + let val = col(&batch, "val"); + assert_eq!( + val.values()[..2], + [700, 701], + "the clone must return the overlaid values, not the base ones" + ); + } + + /// Deep-cloning a shallow clone has to drop every `base_id`, since the + /// result owns its files. An overlay whose `base_id` survives points at a + /// base the new manifest no longer lists. + #[tokio::test] + async fn deep_clone_of_shallow_clone_clears_overlay_base_id() { + use lance_core::utils::tempfile::TempStdDir; + + let version = LanceFileVersion::Stable; + let test_dir = TempStdDir::default(); + let source_uri = test_dir.join("source").to_str().unwrap().to_string(); + let shallow_uri = test_dir.join("shallow").to_str().unwrap().to_string(); + let deep_uri = test_dir.join("deep").to_str().unwrap().to_string(); + + let schema = Arc::new(ArrowSchema::new(vec![ + ArrowField::new("id", DataType::Int32, true), + ArrowField::new("val", DataType::Int32, true), + ])); + let batch = RecordBatch::try_new( + schema.clone(), + vec![ + Arc::new(Int32Array::from_iter_values(0..6)), + Arc::new(Int32Array::from_iter_values((0..6).map(|v| v * 10))), + ], + ) + .unwrap(); + let reader = RecordBatchIterator::new(vec![Ok(batch)], schema.clone()); + let dataset = Dataset::write( + reader, + &source_uri, + Some(WriteParams { + max_rows_per_file: 6, + data_storage_version: Some(version), + ..Default::default() + }), + ) + .await + .unwrap(); + + let mut dataset = commit_overlay( + dataset, + "overlay0", + 0, + &[1], + OverlayCoverage::Shared(Arc::new(bitmap([0, 1]))), + vec![i32_array([Some(700), Some(701)])], + version, + ) + .await; + + let source_version = dataset.version().version; + dataset.tags().create("v", source_version).await.unwrap(); + let mut shallow = dataset + .shallow_clone(&shallow_uri, "v", None) + .await + .unwrap(); + // The shallow clone reaches the overlay through the parent. + assert!( + shallow.get_fragments()[0].metadata().overlays[0] + .data_file + .base_id + .is_some(), + "the shallow clone must reference the parent's overlay by base_id" + ); + + let shallow_version = shallow.version().version; + shallow.tags().create("v", shallow_version).await.unwrap(); + let deep = shallow.deep_clone(&deep_uri, "v", None).await.unwrap(); + + assert_eq!( + deep.get_fragments()[0].metadata().overlays[0] + .data_file + .base_id, + None, + "a deep clone owns its files, so the overlay must carry no base_id" + ); + assert!( + deep.manifest.base_paths.is_empty(), + "a deep clone lists no external bases" + ); + } + + fn full_schema(dataset: &Dataset) -> Schema { + dataset.schema().clone() + } + + fn col(batch: &RecordBatch, name: &str) -> Int32Array { + let idx = batch.schema().index_of(name).unwrap(); + batch + .column(idx) + .as_any() + .downcast_ref::() + .unwrap() + .clone() + } + + #[rstest] + #[tokio::test] + async fn test_take_covered_and_uncovered( + #[values(LanceFileVersion::V2_0, LanceFileVersion::V2_1)] version: LanceFileVersion, + ) { + let dataset = create_base_dataset(version).await; + // Overlay fragment 0's `val` at physical offsets {1, 4}. + let dataset = commit_overlay( + dataset, + "ov", + 0, + &[1], + OverlayCoverage::dense(bitmap([1, 4])), + vec![i32_array([Some(111), Some(444)])], + version, + ) + .await; + + let frag = dataset.get_fragment(0).unwrap(); + let batch = frag + .take(&[0, 1, 2, 4], &full_schema(&dataset)) + .await + .unwrap(); + // Offsets 1 and 4 take overlay values; 0 and 2 fall through to base. + assert_eq!(col(&batch, "val").values(), &[0, 111, 20, 444]); + // The unrelated `id` column is untouched. + assert_eq!(col(&batch, "id").values(), &[0, 1, 2, 4]); + } + + #[rstest] + #[tokio::test] + async fn test_take_newest_overlay_wins( + #[values(LanceFileVersion::V2_0, LanceFileVersion::V2_1)] version: LanceFileVersion, + ) { + let dataset = create_base_dataset(version).await; + let dataset = commit_overlay( + dataset, + "older", + 0, + &[1], + OverlayCoverage::dense(bitmap([1, 4])), + vec![i32_array([Some(111), Some(444)])], + version, + ) + .await; + // A newer overlay (later commit -> higher committed_version) re-covers + // offset 1. + let dataset = commit_overlay( + dataset, + "newer", + 0, + &[1], + OverlayCoverage::dense(bitmap([1])), + vec![i32_array([Some(999)])], + version, + ) + .await; + + let frag = dataset.get_fragment(0).unwrap(); + let batch = frag.take(&[1, 4], &full_schema(&dataset)).await.unwrap(); + // Offset 1 -> newest overlay (999); offset 4 -> only older covers it. + assert_eq!(col(&batch, "val").values(), &[999, 444]); + } + + #[rstest] + #[tokio::test] + async fn test_take_per_field_coverage( + #[values(LanceFileVersion::V2_0, LanceFileVersion::V2_1)] version: LanceFileVersion, + ) { + let dataset = create_base_dataset(version).await; + // Sparse overlay: `id` covers {2}, `val` covers {2, 3} — different + // offset sets and therefore unequal-length value columns. + let dataset = commit_overlay( + dataset, + "sparse", + 0, + &[0, 1], + OverlayCoverage::sparse(vec![bitmap([2]), bitmap([2, 3])]), + vec![i32_array([Some(777)]), i32_array([Some(220), Some(330)])], + version, + ) + .await; + + let frag = dataset.get_fragment(0).unwrap(); + let batch = frag.take(&[2, 3], &full_schema(&dataset)).await.unwrap(); + // id: offset 2 covered (777), offset 3 falls through (3). + assert_eq!(col(&batch, "id").values(), &[777, 3]); + // val: both offsets covered (220, 330). + assert_eq!(col(&batch, "val").values(), &[220, 330]); + } + + #[rstest] + #[tokio::test] + async fn test_take_null_override( + #[values(LanceFileVersion::V2_0, LanceFileVersion::V2_1)] version: LanceFileVersion, + ) { + let dataset = create_base_dataset(version).await; + let dataset = commit_overlay( + dataset, + "nullov", + 0, + &[1], + OverlayCoverage::dense(bitmap([0])), + vec![i32_array([None])], + version, + ) + .await; + + let frag = dataset.get_fragment(0).unwrap(); + let batch = frag.take(&[0, 1], &full_schema(&dataset)).await.unwrap(); + let val = col(&batch, "val"); + // Offset 0 is covered with a NULL value -> resolves to NULL; offset 1 + // falls through to the base value. + assert!(val.is_null(0)); + assert_eq!(val.value(1), 10); + } + + /// Overlays interact correctly with NULL *base* cells (distinct from a NULL + /// overlay value): a covered row whose base value is NULL is overridden to the + /// overlay's non-null value, while an uncovered NULL base cell falls through + /// and stays NULL. + #[rstest] + #[tokio::test] + async fn test_take_null_base_cell( + #[values(LanceFileVersion::V2_0, LanceFileVersion::V2_1)] version: LanceFileVersion, + ) { + let schema = Arc::new(ArrowSchema::new(vec![ + ArrowField::new("id", DataType::Int32, true), + ArrowField::new("val", DataType::Int32, true), + ])); + let batch = RecordBatch::try_new( + schema.clone(), + vec![ + Arc::new(Int32Array::from_iter_values(0..6)), + // `val` is NULL at offsets 1 and 3. + Arc::new(Int32Array::from_iter([ + Some(0), + None, + Some(20), + None, + Some(40), + Some(50), + ])), + ], + ) + .unwrap(); + let write_params = WriteParams { + max_rows_per_file: 6, + max_rows_per_group: 6, + data_storage_version: Some(version), + ..Default::default() + }; + let reader = RecordBatchIterator::new(vec![Ok(batch)], schema.clone()); + let dataset = Dataset::write(reader, "memory://", Some(write_params)) + .await + .unwrap(); + + // Cover offset 1 (NULL base) and offset 4 (non-null base); leave offset + // 3's NULL base uncovered. + let dataset = commit_overlay( + dataset, + "nullbase", + 0, + &[1], + OverlayCoverage::dense(bitmap([1, 4])), + vec![i32_array([Some(111), Some(444)])], + version, + ) + .await; + + let frag = dataset.get_fragment(0).unwrap(); + let batch = frag.take(&[1, 3, 4], &full_schema(&dataset)).await.unwrap(); + let val = col(&batch, "val"); + // Offset 1: NULL base overridden to 111. Offset 3: uncovered NULL base + // stays NULL. Offset 4: non-null base overridden to 444. + assert_eq!(val.value(0), 111); + assert!(val.is_null(1)); + assert_eq!(val.value(2), 444); + } + + #[rstest] + #[tokio::test] + async fn test_overlay_on_deleted_row_is_inert( + #[values(LanceFileVersion::V2_0, LanceFileVersion::V2_1)] version: LanceFileVersion, + ) { + let mut dataset = create_base_dataset(version).await; + // Delete global row 1 (fragment 0, physical offset 1). + dataset.delete("id = 1").await.unwrap(); + // Overlay covers the deleted offset 1 and the live offset 4. + let dataset = commit_overlay( + dataset, + "delov", + 0, + &[1], + OverlayCoverage::dense(bitmap([1, 4])), + vec![i32_array([Some(111), Some(444)])], + version, + ) + .await; + + // Scan fragment 0: row 1 is gone, and offset 4's overlay value survives + // even though the deletion shifts logical positions — coverage is keyed + // by physical offset. + let frag = dataset.get_fragment(0).unwrap(); + let mut scanner = frag.scan(); + let batch = scanner + .project(&["id", "val"]) + .unwrap() + .try_into_batch() + .await + .unwrap(); + assert_eq!(col(&batch, "id").values(), &[0, 2, 3, 4, 5]); + assert_eq!(col(&batch, "val").values(), &[0, 20, 30, 444, 50]); + } + + #[rstest] + #[tokio::test] + async fn test_scan_multi_fragment_overlays( + #[values(LanceFileVersion::V2_0, LanceFileVersion::V2_1)] version: LanceFileVersion, + ) { + let dataset = create_base_dataset(version).await; + // Overlay fragment 0 at offset 0 and fragment 1 at offset 0 (global + // row 6). Each fragment's coverage is independent. + let dataset = commit_overlay( + dataset, + "frag0", + 0, + &[1], + OverlayCoverage::dense(bitmap([0])), + vec![i32_array([Some(1000)])], + version, + ) + .await; + let dataset = commit_overlay( + dataset, + "frag1", + 1, + &[1], + OverlayCoverage::dense(bitmap([0])), + vec![i32_array([Some(6000)])], + version, + ) + .await; + + let batch = dataset + .scan() + .project(&["id", "val"]) + .unwrap() + .try_into_batch() + .await + .unwrap(); + assert_eq!(batch.num_rows(), 12); + let expected: Vec = (0..12) + .map(|i| match i { + 0 => 1000, + 6 => 6000, + other => other * 10, + }) + .collect(); + assert_eq!(col(&batch, "val").values(), &expected); + } + + /// A `take` of a few rows must read only the overlay values those rows + /// touch — not the whole column. Uses v2.1 (which slices pages on read) and + /// an incompressible, all-covering overlay, so reading the full column would + /// be far more bytes than reading a couple of values. This is the regression + /// guard for the lazy, value-pushdown overlay read. + #[tokio::test] + async fn test_take_reads_only_needed_overlay_values() { + let version = LanceFileVersion::V2_1; + const N: usize = 100_000; + + let schema = Arc::new(ArrowSchema::new(vec![ + ArrowField::new("id", DataType::Int32, true), + ArrowField::new("val", DataType::Int32, true), + ])); + let base = RecordBatch::try_new( + schema.clone(), + vec![ + Arc::new(Int32Array::from_iter_values(0..N as i32)), + Arc::new(Int32Array::from_iter_values((0..N as i32).map(|v| v * 10))), + ], + ) + .unwrap(); + let write_params = WriteParams { + max_rows_per_file: N, + max_rows_per_group: N, + data_storage_version: Some(version), + ..Default::default() + }; + let reader = RecordBatchIterator::new(vec![Ok(base)], schema.clone()); + let dataset = Dataset::write(reader, "memory://", Some(write_params)) + .await + .unwrap(); + + // Overlay `val` over ALL N offsets with incompressible values, so the + // value column is ~N*4 bytes on disk. + let values: Vec = (0..N as u64) + .map(|i| { + let mut x = i; + x ^= x >> 33; + x = x.wrapping_mul(0xff51_afd7_ed55_8ccd); + x ^= x >> 33; + x as i32 + }) + .collect(); + let dataset = commit_overlay( + dataset, + "big", + 0, + &[1], + OverlayCoverage::dense(bitmap(0..N as u32)), + vec![Arc::new(Int32Array::from(values.clone())) as ArrayRef], + version, + ) + .await; + + let frag = dataset.get_fragment(0).unwrap(); + let val_only = dataset.schema().project_by_ids(&[1], true); + + // Measure only the reads that resolve the take. + dataset.object_store.io_stats_incremental(); + let batch = frag.take(&[0, 1], &val_only).await.unwrap(); + let io = dataset.object_store.io_stats_incremental(); + + // The overlay's `val` column alone is N*4 bytes; resolving two adjacent + // offsets must read only a small fraction of it. + let full_column_bytes = (N * std::mem::size_of::()) as u64; + assert!( + io.read_bytes > 0 && io.read_bytes < full_column_bytes / 4, + "take read {} bytes; expected far less than the {}-byte overlay \ + column (a take must not read the whole value column)", + io.read_bytes, + full_column_bytes, + ); + + // ...and it still resolves correctly. + let val = col(&batch, "val"); + assert_eq!(val.value(0), values[0]); + assert_eq!(val.value(1), values[1]); + } + + /// Row-selection pruning: an overlay whose coverage is disjoint from the + /// requested rows must not be opened at all. Proven by deleting the overlay's + /// data file — a `take` that misses its coverage still succeeds (the file is + /// never touched), while a `take` that hits it then fails because the file is + /// genuinely needed. + #[rstest] + #[tokio::test] + async fn test_take_prunes_overlays_outside_row_selection( + #[values(LanceFileVersion::V2_0, LanceFileVersion::V2_1)] version: LanceFileVersion, + ) { + let dataset = create_base_dataset(version).await; + // Overlay on fragment 0 (offsets 0..6) covering only offset_in_frag 5. + let dataset = commit_overlay( + dataset, + "miss", + 0, + &[1], + OverlayCoverage::dense(bitmap([5])), + vec![i32_array([Some(5000)])], + version, + ) + .await; + + // Delete the overlay's data file: opening it now fails. + dataset + .object_store + .delete(&Path::from("data/miss.lance")) + .await + .unwrap(); + + let frag = dataset.get_fragment(0).unwrap(); + let val_only = dataset.schema().project_by_ids(&[1], true); + + // A take that misses the overlay's coverage must not open it, so it + // succeeds and returns base values (val = offset * 10). + let batch = frag.take(&[0, 1], &val_only).await.unwrap(); + assert_eq!(col(&batch, "val").values(), &[0, 10]); + + // A take that hits the coverage does need the file, so it now fails with + // a not-found error naming the missing overlay file. + let err = frag.take(&[5], &val_only).await.unwrap_err(); + let message = format!("{err:?}"); + assert!( + err.is_not_found() && message.contains("miss.lance"), + "take hitting the overlay's coverage should fail with a not-found error \ + for its missing file, got: {message}", + ); + } + + /// The overlay merge runs before `wrap_with_row_id_and_delete`, so the + /// `_rowid` system column must coexist with overlay-resolved data columns: + /// the row ids are unaffected by the merge and the overlay value still wins. + #[rstest] + #[tokio::test] + async fn test_scan_with_row_id_alongside_overlay( + #[values(LanceFileVersion::V2_0, LanceFileVersion::V2_1)] version: LanceFileVersion, + ) { + let dataset = create_base_dataset(version).await; + let dataset = commit_overlay( + dataset, + "rowidov", + 0, + &[1], + OverlayCoverage::dense(bitmap([0])), + vec![i32_array([Some(1000)])], + version, + ) + .await; + + let frag = dataset.get_fragment(0).unwrap(); + let batch = frag + .scan() + .with_row_id() + .project(&["id", "val"]) + .unwrap() + .try_into_batch() + .await + .unwrap(); + // Overlay value resolves... + assert_eq!(col(&batch, "val").values()[0], 1000); + assert_eq!(&col(&batch, "val").values()[1..], &[10, 20, 30, 40, 50]); + // ...and the row ids for fragment 0 are the untouched physical offsets. + let row_ids = batch + .column(batch.schema().index_of("_rowid").unwrap()) + .as_any() + .downcast_ref::() + .unwrap(); + assert_eq!(row_ids.values(), &[0, 1, 2, 3, 4, 5]); + } + + /// When the newest overlay covers every requested offset, an older overlay + /// in the same plan needs zero values and its value column must not be read + /// (the empty-input branch of `fetch_overlay_values`). The result still + /// resolves to the newest overlay. + #[rstest] + #[tokio::test] + async fn test_take_older_overlay_contributes_no_values( + #[values(LanceFileVersion::V2_0, LanceFileVersion::V2_1)] version: LanceFileVersion, + ) { + let dataset = create_base_dataset(version).await; + // Older covers {1, 4}; newer re-covers {1}. A take of only offset 1 + // routes entirely to the newer overlay, leaving the older one with no + // values to fetch even though it is part of the field's plan. + let dataset = commit_overlay( + dataset, + "older", + 0, + &[1], + OverlayCoverage::dense(bitmap([1, 4])), + vec![i32_array([Some(111), Some(444)])], + version, + ) + .await; + let dataset = commit_overlay( + dataset, + "newer", + 0, + &[1], + OverlayCoverage::dense(bitmap([1])), + vec![i32_array([Some(999)])], + version, + ) + .await; + + let frag = dataset.get_fragment(0).unwrap(); + let batch = frag.take(&[1], &full_schema(&dataset)).await.unwrap(); + assert_eq!(col(&batch, "val").values(), &[999]); + } + + /// A newest overlay whose value is NULL must shadow an older overlay's + /// non-null value at the same offset — the merge resolves to NULL, it does + /// not fall back to the older overlay. + #[rstest] + #[tokio::test] + async fn test_take_newest_null_shadows_older( + #[values(LanceFileVersion::V2_0, LanceFileVersion::V2_1)] version: LanceFileVersion, + ) { + let dataset = create_base_dataset(version).await; + let dataset = commit_overlay( + dataset, + "older", + 0, + &[1], + OverlayCoverage::dense(bitmap([1])), + vec![i32_array([Some(111)])], + version, + ) + .await; + let dataset = commit_overlay( + dataset, + "newer_null", + 0, + &[1], + OverlayCoverage::dense(bitmap([1])), + vec![i32_array([None])], + version, + ) + .await; + + let frag = dataset.get_fragment(0).unwrap(); + let batch = frag.take(&[1], &full_schema(&dataset)).await.unwrap(); + let val = col(&batch, "val"); + assert!(val.is_null(0), "newest NULL must win over older 111"); + } + + /// Newest-wins is resolved independently per field across multiple sparse + /// overlays: for the same offset, `id` can resolve to one overlay while + /// `val` resolves to the other, depending on which overlay newly covers + /// that field at that offset. + #[rstest] + #[tokio::test] + async fn test_take_multi_sparse_per_field_newest_wins( + #[values(LanceFileVersion::V2_0, LanceFileVersion::V2_1)] version: LanceFileVersion, + ) { + let dataset = create_base_dataset(version).await; + // Older: id covers {3}, val covers {2}. + let dataset = commit_overlay( + dataset, + "older", + 0, + &[0, 1], + OverlayCoverage::sparse(vec![bitmap([3]), bitmap([2])]), + vec![i32_array([Some(7773)]), i32_array([Some(2772)])], + version, + ) + .await; + // Newer: id covers {2}, val covers {3} — the mirror image. + let dataset = commit_overlay( + dataset, + "newer", + 0, + &[0, 1], + OverlayCoverage::sparse(vec![bitmap([2]), bitmap([3])]), + vec![i32_array([Some(9992)]), i32_array([Some(9993)])], + version, + ) + .await; + + let frag = dataset.get_fragment(0).unwrap(); + let batch = frag.take(&[2, 3], &full_schema(&dataset)).await.unwrap(); + // id: offset 2 -> newer (9992), offset 3 -> older (7773). + assert_eq!(col(&batch, "id").values(), &[9992, 7773]); + // val: offset 2 -> older (2772), offset 3 -> newer (9993). + assert_eq!(col(&batch, "val").values(), &[2772, 9993]); + } + + /// A fragment with an overlay plan, but a take that touches only uncovered + /// offsets, must fall entirely through to the base values (the + /// `!routing.any_overlay` early-return with a plan present). + #[rstest] + #[tokio::test] + async fn test_take_plan_present_all_offsets_uncovered( + #[values(LanceFileVersion::V2_0, LanceFileVersion::V2_1)] version: LanceFileVersion, + ) { + let dataset = create_base_dataset(version).await; + let dataset = commit_overlay( + dataset, + "ov", + 0, + &[1], + OverlayCoverage::dense(bitmap([1, 4])), + vec![i32_array([Some(111), Some(444)])], + version, + ) + .await; + + let frag = dataset.get_fragment(0).unwrap(); + // None of {0, 2, 5} are covered: the plan exists but contributes nothing. + let batch = frag.take(&[0, 2, 5], &full_schema(&dataset)).await.unwrap(); + assert_eq!(col(&batch, "val").values(), &[0, 20, 50]); + assert_eq!(col(&batch, "id").values(), &[0, 2, 5]); + } + + /// A dataset-level `take` spanning multiple fragments, each with its own + /// overlay, routes every global row index to the right fragment's overlay. + #[rstest] + #[tokio::test] + async fn test_dataset_take_multi_fragment_overlays( + #[values(LanceFileVersion::V2_0, LanceFileVersion::V2_1)] version: LanceFileVersion, + ) { + let dataset = create_base_dataset(version).await; + let dataset = commit_overlay( + dataset, + "frag0", + 0, + &[1], + OverlayCoverage::dense(bitmap([0])), + vec![i32_array([Some(1000)])], + version, + ) + .await; + let dataset = commit_overlay( + dataset, + "frag1", + 1, + &[1], + OverlayCoverage::dense(bitmap([0])), + vec![i32_array([Some(6000)])], + version, + ) + .await; + + // Global rows 0 and 6 are the overlaid offset-0 rows of fragments 0 and + // 1; rows 1 and 7 fall through to base. + let batch = dataset + .take(&[0, 1, 6, 7], full_schema(&dataset)) + .await + .unwrap(); + assert_eq!(col(&batch, "id").values(), &[0, 1, 6, 7]); + assert_eq!(col(&batch, "val").values(), &[1000, 10, 6000, 70]); + } + + /// A scan whose read splits into multiple batches must slice + /// `offsets_in_frag` per batch correctly — the running `rows_seen` + /// accumulator in `merge_overlays` gives each batch its start. Every other + /// scan test uses single-batch fragments, so this is the only guard for the + /// cross-batch (`start > 0`) path. + #[rstest] + #[tokio::test] + async fn test_scan_multi_batch_overlay_slicing( + #[values(LanceFileVersion::V2_0, LanceFileVersion::V2_1)] version: LanceFileVersion, + ) { + use futures::TryStreamExt; + + // One fragment of 10 rows so the read can be chunked below. + let schema = Arc::new(ArrowSchema::new(vec![ + ArrowField::new("id", DataType::Int32, true), + ArrowField::new("val", DataType::Int32, true), + ])); + let batch = RecordBatch::try_new( + schema.clone(), + vec![ + Arc::new(Int32Array::from_iter_values(0..10)), + Arc::new(Int32Array::from_iter_values((0..10).map(|v| v * 10))), + ], + ) + .unwrap(); + let write_params = WriteParams { + max_rows_per_file: 100, + max_rows_per_group: 100, + data_storage_version: Some(version), + ..Default::default() + }; + let reader = RecordBatchIterator::new(vec![Ok(batch)], schema.clone()); + let dataset = Dataset::write(reader, "memory://", Some(write_params)) + .await + .unwrap(); + + // Overlay one offset in each batch that batch_size 4 produces (batches + // [0,4), [4,8), [8,10)): offsets 1, 5, 9 with distinct values. A wrong + // per-batch slice would misalign these. + let dataset = commit_overlay( + dataset, + "multibatch", + 0, + &[1], + OverlayCoverage::dense(bitmap([1, 5, 9])), + vec![i32_array([Some(111), Some(555), Some(999)])], + version, + ) + .await; + + let frag = dataset.get_fragment(0).unwrap(); + let mut scanner = frag.scan(); + scanner.batch_size(4).project(&["val"]).unwrap(); + let batches: Vec = scanner + .try_into_stream() + .await + .unwrap() + .try_collect() + .await + .unwrap(); + // Guard the guard: the read must actually span multiple batches, else + // this would not exercise the cross-batch slice at all. + assert!( + batches.len() > 1, + "expected a multi-batch scan, got {} batch(es)", + batches.len() + ); + + let merged = + arrow_select::concat::concat_batches(&batches[0].schema(), &batches).unwrap(); + let expected: Vec = (0..10) + .map(|i| match i { + 1 => 111, + 5 => 555, + 9 => 999, + other => other * 10, + }) + .collect(); + assert_eq!(col(&merged, "val").values(), &expected); + } + + /// An empty selection must not trip over the overlay path: the plan exists + /// but there are no offsets to route, so the result is an empty batch. + #[rstest] + #[tokio::test] + async fn test_take_empty_selection( + #[values(LanceFileVersion::V2_0, LanceFileVersion::V2_1)] version: LanceFileVersion, + ) { + let dataset = create_base_dataset(version).await; + let dataset = commit_overlay( + dataset, + "ov", + 0, + &[1], + OverlayCoverage::dense(bitmap([1, 4])), + vec![i32_array([Some(111), Some(444)])], + version, + ) + .await; + + let frag = dataset.get_fragment(0).unwrap(); + let batch = frag.take(&[], &full_schema(&dataset)).await.unwrap(); + assert_eq!(batch.num_rows(), 0); + } + + /// Overlays resolve variable-width columns end-to-end, not just fixed-width + /// ones: the value column is fetched through the real file reader (a + /// different value-pushdown path than the fixed-width case) and assembled. + #[rstest] + #[tokio::test] + async fn test_string_overlay_end_to_end( + #[values(LanceFileVersion::V2_0, LanceFileVersion::V2_1)] version: LanceFileVersion, + ) { + use arrow_array::StringArray; + + let schema = Arc::new(ArrowSchema::new(vec![ + ArrowField::new("id", DataType::Int32, true), + ArrowField::new("name", DataType::Utf8, true), + ])); + let batch = RecordBatch::try_new( + schema.clone(), + vec![ + Arc::new(Int32Array::from_iter_values(0..6)), + Arc::new(StringArray::from(vec!["a", "b", "c", "d", "e", "f"])), + ], + ) + .unwrap(); + let write_params = WriteParams { + max_rows_per_file: 6, + max_rows_per_group: 6, + data_storage_version: Some(version), + ..Default::default() + }; + let reader = RecordBatchIterator::new(vec![Ok(batch)], schema.clone()); + let dataset = Dataset::write(reader, "memory://", Some(write_params)) + .await + .unwrap(); + + // Overlay `name` at offsets {1, 4}, one of the values NULL. + let dataset = commit_overlay( + dataset, + "strov", + 0, + &[1], + OverlayCoverage::dense(bitmap([1, 4])), + vec![Arc::new(StringArray::from(vec![Some("B"), None])) as ArrayRef], + version, + ) + .await; + + let frag = dataset.get_fragment(0).unwrap(); + let batch = frag.take(&[0, 1, 4], &full_schema(&dataset)).await.unwrap(); + let name = batch + .column(batch.schema().index_of("name").unwrap()) + .as_any() + .downcast_ref::() + .unwrap(); + assert_eq!(name.value(0), "a"); // falls through to base + assert_eq!(name.value(1), "B"); // overlay value + assert!(name.is_null(2)); // overlay NULL wins + } + + /// Projection pruning must do NO IO to overlay files whose fields are not + /// projected. Proven the same way as row-selection pruning: delete the + /// overlay's data file, then read projecting only the *unrelated* `id` + /// column — it must succeed (the `val` overlay file is never opened), while + /// projecting the overlaid `val` column then fails because its file is gone. + #[rstest] + #[tokio::test] + async fn test_projection_prunes_overlay_files_no_io( + #[values(LanceFileVersion::V2_0, LanceFileVersion::V2_1)] version: LanceFileVersion, + ) { + let dataset = create_base_dataset(version).await; + // Overlay covers `val` (field 1) only. + let dataset = commit_overlay( + dataset, + "valov", + 0, + &[1], + OverlayCoverage::dense(bitmap([0, 1])), + vec![i32_array([Some(1000), Some(1010)])], + version, + ) + .await; + + // Delete the overlay's data file: opening it now fails. + dataset + .object_store + .delete(&Path::from("data/valov.lance")) + .await + .unwrap(); + + let frag = dataset.get_fragment(0).unwrap(); + let id_only = dataset.schema().project_by_ids(&[0], true); + let val_only = dataset.schema().project_by_ids(&[1], true); + + // Projecting only `id` must not open the `val` overlay file, so it + // succeeds and returns untouched base values. + let batch = frag.take(&[0, 1], &id_only).await.unwrap(); + assert_eq!(col(&batch, "id").values(), &[0, 1]); + // A scan projecting only `id` must likewise never touch the file. + let batch = frag + .scan() + .project(&["id"]) + .unwrap() + .try_into_batch() + .await + .unwrap(); + assert_eq!(col(&batch, "id").values(), &[0, 1, 2, 3, 4, 5]); + + // Projecting the overlaid `val` column does need the file, so it fails + // with a not-found error naming the missing overlay file. + let err = frag.take(&[0], &val_only).await.unwrap_err(); + let message = format!("{err:?}"); + assert!( + err.is_not_found() && message.contains("valov.lance"), + "projecting the overlaid column should fail with a not-found error \ + for its missing file, got: {message}", + ); + } + + /// A top-level struct column resolves through overlays: the overlay stores + /// the struct's leaf columns (under V2_1 those are the only ids in + /// `data_file.fields`), and `plan_overlays` maps them back to the top-level + /// struct so the whole value is fetched and replaced as a unit. + #[rstest] + #[tokio::test] + async fn test_struct_overlay_end_to_end( + #[values(LanceFileVersion::V2_0, LanceFileVersion::V2_1)] version: LanceFileVersion, + ) { + let struct_fields = Fields::from(vec![ + ArrowField::new("x", DataType::Int32, true), + ArrowField::new("y", DataType::Int32, true), + ]); + let schema = Arc::new(ArrowSchema::new(vec![ + ArrowField::new("id", DataType::Int32, true), + ArrowField::new("info", DataType::Struct(struct_fields.clone()), true), + ])); + let info = Arc::new(StructArray::new( + struct_fields.clone(), + vec![ + Arc::new(Int32Array::from_iter_values(0..6)), + Arc::new(Int32Array::from_iter_values((0..6).map(|v| v * 100))), + ], + None, + )); + let batch = RecordBatch::try_new( + schema.clone(), + vec![Arc::new(Int32Array::from_iter_values(0..6)), info], + ) + .unwrap(); + let write_params = WriteParams { + max_rows_per_file: 6, + max_rows_per_group: 6, + data_storage_version: Some(version), + ..Default::default() + }; + let reader = RecordBatchIterator::new(vec![Ok(batch)], schema.clone()); + let dataset = Dataset::write(reader, "memory://", Some(write_params)) + .await + .unwrap(); + + // Overlay the whole `info` struct (top-level field id 1) at offset 2. + let overlay_info = Arc::new(StructArray::new( + struct_fields, + vec![ + Arc::new(Int32Array::from(vec![777])), + Arc::new(Int32Array::from(vec![888])), + ], + None, + )) as ArrayRef; + let dataset = commit_overlay( + dataset, + "structov", + 0, + &[1], + OverlayCoverage::dense(bitmap([2])), + vec![overlay_info], + version, + ) + .await; + + let frag = dataset.get_fragment(0).unwrap(); + let batch = frag.take(&[1, 2], &full_schema(&dataset)).await.unwrap(); + let info = batch + .column(batch.schema().index_of("info").unwrap()) + .as_any() + .downcast_ref::() + .unwrap(); + let x = info + .column(0) + .as_any() + .downcast_ref::() + .unwrap(); + let y = info + .column(1) + .as_any() + .downcast_ref::() + .unwrap(); + // Offset 1 falls through to base {1, 100}; offset 2 takes the overlay. + assert_eq!(x.values(), &[1, 777]); + assert_eq!(y.values(), &[100, 888]); + } + + /// A top-level list column resolves through overlays the same way — the + /// overlay's leaf (item) id maps back to the top-level list, and the whole + /// list value at a covered offset is replaced. + #[rstest] + #[tokio::test] + async fn test_list_overlay_end_to_end( + #[values(LanceFileVersion::V2_0, LanceFileVersion::V2_1)] version: LanceFileVersion, + ) { + use arrow_array::ListArray; + use arrow_array::types::Int32Type; + + let item = Arc::new(ArrowField::new("item", DataType::Int32, true)); + let schema = Arc::new(ArrowSchema::new(vec![ + ArrowField::new("id", DataType::Int32, true), + ArrowField::new("tags", DataType::List(item.clone()), true), + ])); + let base_tags = ListArray::from_iter_primitive::( + (0..6i32).map(|i| Some(vec![Some(i), Some(i * 10)])), + ); + let batch = RecordBatch::try_new( + schema.clone(), + vec![ + Arc::new(Int32Array::from_iter_values(0..6)), + Arc::new(base_tags), + ], + ) + .unwrap(); + let write_params = WriteParams { + max_rows_per_file: 6, + max_rows_per_group: 6, + data_storage_version: Some(version), + ..Default::default() + }; + let reader = RecordBatchIterator::new(vec![Ok(batch)], schema.clone()); + let dataset = Dataset::write(reader, "memory://", Some(write_params)) + .await + .unwrap(); - if let Some(expand_map) = expand_map { - batch = arrow_select::take::take_record_batch(&batch, &expand_map)?; + // Overlay `tags` (top-level field id 1) at offset 2 with a new list. + let overlay_tags = + ListArray::from_iter_primitive::(std::iter::once(Some(vec![ + Some(77), + Some(88), + Some(99), + ]))); + let dataset = commit_overlay( + dataset, + "listov", + 0, + &[1], + OverlayCoverage::dense(bitmap([2])), + vec![Arc::new(overlay_tags) as ArrayRef], + version, + ) + .await; + + let frag = dataset.get_fragment(0).unwrap(); + let batch = frag.take(&[1, 2], &full_schema(&dataset)).await.unwrap(); + let tags = batch + .column(batch.schema().index_of("tags").unwrap()) + .as_any() + .downcast_ref::() + .unwrap(); + let row1 = tags.value(0); + let row1 = row1.as_any().downcast_ref::().unwrap(); + let row2 = tags.value(1); + let row2 = row2.as_any().downcast_ref::().unwrap(); + // Offset 1 falls through to base [1, 10]; offset 2 takes the overlay. + assert_eq!(row1.values(), &[1, 10]); + assert_eq!(row2.values(), &[77, 88, 99]); } - Ok(batch) - } -} + /// A top-level Map column resolves as a single atomic field even though its + /// value spans two leaves (key and value): both leaf ids map back to the one + /// Map atomic field, and the whole map value at a covered offset is replaced. + /// Maps require + /// the 2.2+ file format, so this runs only at V2_2 (unlike the V2_0/V2_1 + /// parametrized tests). + #[tokio::test] + async fn test_map_overlay_end_to_end() { + use arrow_array::MapArray; + use arrow_array::builder::{Int32Builder, MapBuilder}; + + let version = LanceFileVersion::V2_2; + + // Base row i holds the single entry {i: i * 10}. + let mut builder = MapBuilder::new(None, Int32Builder::new(), Int32Builder::new()); + for i in 0..6i32 { + builder.keys().append_value(i); + builder.values().append_value(i * 10); + builder.append(true).unwrap(); + } + let base_attrs = builder.finish(); + let schema = Arc::new(ArrowSchema::new(vec![ + ArrowField::new("id", DataType::Int32, true), + ArrowField::new("attrs", base_attrs.data_type().clone(), true), + ])); + let batch = RecordBatch::try_new( + schema.clone(), + vec![ + Arc::new(Int32Array::from_iter_values(0..6)), + Arc::new(base_attrs), + ], + ) + .unwrap(); + let write_params = WriteParams { + max_rows_per_file: 6, + max_rows_per_group: 6, + data_storage_version: Some(version), + ..Default::default() + }; + let reader = RecordBatchIterator::new(vec![Ok(batch)], schema.clone()); + let dataset = Dataset::write(reader, "memory://", Some(write_params)) + .await + .unwrap(); -#[cfg(test)] -mod tests { - use arrow_arith::numeric::mul; - use arrow_array::{ - ArrayRef, BooleanArray, Int32Array, Int64Array, RecordBatchIterator, StringArray, - }; - use arrow_schema::{DataType, Field as ArrowField, Schema as ArrowSchema}; - use lance_core::ROW_ID; - use lance_core::utils::tempfile::TempStrDir; - use lance_datagen::{RowCount, array, gen_batch}; - use lance_file::version::LanceFileVersion; - use lance_file::writer::FileWriterOptions; - use lance_io::{assert_io_eq, assert_io_lt, object_store::ObjectStore}; - use pretty_assertions::assert_eq; - use rstest::rstest; - use std::collections::HashMap; + // Overlay `attrs` (top-level field id 1) at offset 2 with a two-entry map. + let mut ov = MapBuilder::new(None, Int32Builder::new(), Int32Builder::new()); + ov.keys().append_value(7); + ov.values().append_value(77); + ov.keys().append_value(8); + ov.values().append_value(88); + ov.append(true).unwrap(); + let overlay_attrs = ov.finish(); + let dataset = commit_overlay( + dataset, + "mapov", + 0, + &[1], + OverlayCoverage::dense(bitmap([2])), + vec![Arc::new(overlay_attrs) as ArrayRef], + version, + ) + .await; - use super::*; - use crate::{ - dataset::{ - InsertBuilder, - transaction::{Operation, UpdateMode, UpdatedFragmentOffsets}, - }, - session::Session, - utils::test::TestDatasetGenerator, - }; + let frag = dataset.get_fragment(0).unwrap(); + let batch = frag.take(&[1, 2], &full_schema(&dataset)).await.unwrap(); + let attrs = batch + .column(batch.schema().index_of("attrs").unwrap()) + .as_any() + .downcast_ref::() + .unwrap(); - async fn create_dataset(test_uri: &str, data_storage_version: LanceFileVersion) -> Dataset { - let schema = Arc::new(ArrowSchema::new(vec![ - ArrowField::new("i", DataType::Int32, true), - ArrowField::new("s", DataType::Utf8, true), - ])); + let entries = |i: usize| -> (Vec, Vec) { + let row = attrs.value(i); + let keys = row.column(0).as_any().downcast_ref::().unwrap(); + let vals = row.column(1).as_any().downcast_ref::().unwrap(); + (keys.values().to_vec(), vals.values().to_vec()) + }; + // Offset 1 falls through to the base entry {1: 10}; offset 2 takes the + // overlay map {7: 77, 8: 88}. + assert_eq!(entries(0), (vec![1], vec![10])); + assert_eq!(entries(1), (vec![7, 8], vec![77, 88])); + } - let batches: Vec = (0..10) - .map(|i| { - RecordBatch::try_new( - schema.clone(), - vec![ - Arc::new(Int32Array::from_iter_values(i * 20..(i + 1) * 20)), - Arc::new(StringArray::from_iter_values( - (i * 20..(i + 1) * 20).map(|v| format!("s-{}", v)), - )), - ], - ) + /// Base `id` + a struct `s { a, b }` (6 rows). Field ids: s=1, a=2, b=3. + async fn create_struct_dataset(version: LanceFileVersion) -> (Dataset, Fields) { + let s_fields = Fields::from(vec![ + ArrowField::new("a", DataType::Int32, true), + ArrowField::new("b", DataType::Int32, true), + ]); + let schema = Arc::new(ArrowSchema::new(vec![ + ArrowField::new("id", DataType::Int32, true), + ArrowField::new("s", DataType::Struct(s_fields.clone()), true), + ])); + let s = Arc::new(StructArray::new( + s_fields.clone(), + vec![ + Arc::new(Int32Array::from_iter_values(0..6)), + Arc::new(Int32Array::from_iter_values((0..6).map(|v| v * 100))), + ], + None, + )); + let batch = RecordBatch::try_new( + schema.clone(), + vec![Arc::new(Int32Array::from_iter_values(0..6)), s], + ) + .unwrap(); + let write_params = WriteParams { + max_rows_per_file: 6, + max_rows_per_group: 6, + data_storage_version: Some(version), + ..Default::default() + }; + let reader = RecordBatchIterator::new(vec![Ok(batch)], schema.clone()); + let dataset = Dataset::write(reader, "memory://", Some(write_params)) + .await + .unwrap(); + (dataset, s_fields) + } + + fn struct_col<'a>(batch: &'a RecordBatch, name: &str) -> &'a StructArray { + batch + .column(batch.schema().index_of(name).unwrap()) + .as_any() + .downcast_ref::() .unwrap() - }) - .collect(); + } - let write_params = WriteParams { - max_rows_per_file: 40, - max_rows_per_group: 10, - data_storage_version: Some(data_storage_version), - ..Default::default() - }; - let batches = RecordBatchIterator::new(batches.into_iter().map(Ok), schema.clone()); - Dataset::write(batches, test_uri, Some(write_params)) - .await - .unwrap(); + fn i32_child(s: &StructArray, i: usize) -> Int32Array { + s.column(i) + .as_any() + .downcast_ref::() + .unwrap() + .clone() + } - Dataset::open(test_uri).await.unwrap() - } + /// The reviewer's core case (r3553495147): an overlay stores only sub-field + /// `s.a`, but the read projects the whole struct `s`. The overlay must splice + /// into `a` and leave `b` untouched (previously this panicked because the merge + /// fetched the whole `s` from an overlay file holding only `a`). + #[rstest] + #[tokio::test] + async fn test_overlay_subfield_projecting_parent_struct( + #[values(LanceFileVersion::V2_0, LanceFileVersion::V2_1)] version: LanceFileVersion, + ) { + let (dataset, _) = create_struct_dataset(version).await; + // Overlay ONLY `s.a` (field id 2) at offset 2. + let a_only = Fields::from(vec![ArrowField::new("a", DataType::Int32, true)]); + let overlay = Arc::new(StructArray::new( + a_only, + vec![Arc::new(Int32Array::from(vec![777]))], + None, + )) as ArrayRef; + let dataset = commit_overlay( + dataset, + "aov", + 0, + &[2], + OverlayCoverage::dense(bitmap([2])), + vec![overlay], + version, + ) + .await; - async fn create_dataset_v2(test_uri: &str) -> Dataset { - let schema = Arc::new(ArrowSchema::new(vec![ArrowField::new( - "i", - DataType::Int32, - true, - )])); + let frag = dataset.get_fragment(0).unwrap(); + let batch = frag.take(&[1, 2], &full_schema(&dataset)).await.unwrap(); + let s = struct_col(&batch, "s"); + // a: offset 1 base (1), offset 2 overlaid (777). + assert_eq!(i32_child(s, 0).values(), &[1, 777]); + // b: untouched base (100, 200). + assert_eq!(i32_child(s, 1).values(), &[100, 200]); + } - let batches: Vec = (0..10) - .map(|i| { - RecordBatch::try_new( - schema.clone(), - vec![Arc::new(Int32Array::from_iter_values(i * 20..(i + 1) * 20))], - ) - .unwrap() - }) - .collect(); + /// An overlay on a non-projected sibling leaf must be skipped and its file + /// never opened: overlay covers `s.b`, but the read projects only `s.a`. + #[rstest] + #[tokio::test] + async fn test_overlay_nonprojected_sibling_skipped( + #[values(LanceFileVersion::V2_0, LanceFileVersion::V2_1)] version: LanceFileVersion, + ) { + let (dataset, _) = create_struct_dataset(version).await; + let b_only = Fields::from(vec![ArrowField::new("b", DataType::Int32, true)]); + let overlay = Arc::new(StructArray::new( + b_only, + vec![Arc::new(Int32Array::from(vec![888]))], + None, + )) as ArrayRef; + let dataset = commit_overlay( + dataset, + "bov", + 0, + &[3], + OverlayCoverage::dense(bitmap([2])), + vec![overlay], + version, + ) + .await; + // Delete the overlay file: if projecting only `s.a` opened it, this fails. + dataset + .object_store + .delete(&Path::from("data/bov.lance")) + .await + .unwrap(); - let write_params = WriteParams { - max_rows_per_file: 40, - max_rows_per_group: 10, - data_storage_version: Some(LanceFileVersion::Stable), - ..Default::default() - }; - let batches = RecordBatchIterator::new(batches.into_iter().map(Ok), schema.clone()); - Dataset::write(batches, test_uri, Some(write_params)) - .await + let frag = dataset.get_fragment(0).unwrap(); + let a_only = dataset.schema().project_by_ids(&[2], true); + let batch = frag.take(&[1, 2], &a_only).await.unwrap(); + let s = struct_col(&batch, "s"); + // Only `a` is projected, unchanged base values. + assert_eq!(i32_child(s, 0).values(), &[1, 2]); + } + + /// Two overlays target different sub-fields of the same struct, and a third + /// re-overlays `s.a`. Each leaf resolves independently and newest wins on `a`. + #[rstest] + #[tokio::test] + async fn test_overlay_multiple_subfields_newest_wins( + #[values(LanceFileVersion::V2_0, LanceFileVersion::V2_1)] version: LanceFileVersion, + ) { + let (dataset, _) = create_struct_dataset(version).await; + let a_field = Fields::from(vec![ArrowField::new("a", DataType::Int32, true)]); + let b_field = Fields::from(vec![ArrowField::new("b", DataType::Int32, true)]); + // Older: a := 700 at offset 2. + let dataset = commit_overlay( + dataset, + "a_old", + 0, + &[2], + OverlayCoverage::dense(bitmap([2])), + vec![Arc::new(StructArray::new( + a_field.clone(), + vec![Arc::new(Int32Array::from(vec![700]))], + None, + )) as ArrayRef], + version, + ) + .await; + // b := 800 at offset 2. + let dataset = commit_overlay( + dataset, + "b_ov", + 0, + &[3], + OverlayCoverage::dense(bitmap([2])), + vec![Arc::new(StructArray::new( + b_field, + vec![Arc::new(Int32Array::from(vec![800]))], + None, + )) as ArrayRef], + version, + ) + .await; + // Newest: a := 999 at offset 2 (shadows the older `a` overlay). + let dataset = commit_overlay( + dataset, + "a_new", + 0, + &[2], + OverlayCoverage::dense(bitmap([2])), + vec![Arc::new(StructArray::new( + a_field, + vec![Arc::new(Int32Array::from(vec![999]))], + None, + )) as ArrayRef], + version, + ) + .await; + + let frag = dataset.get_fragment(0).unwrap(); + let batch = frag.take(&[2], &full_schema(&dataset)).await.unwrap(); + let s = struct_col(&batch, "s"); + assert_eq!(i32_child(s, 0).values(), &[999]); // newest `a` wins + assert_eq!(i32_child(s, 1).values(), &[800]); // `b` from its own overlay + } + + /// Three levels of nesting: `outer { middle { a, b } }`. An overlay on the + /// deep leaf `outer.middle.a` splices correctly when the whole `outer` is read. + #[rstest] + #[tokio::test] + async fn test_overlay_deeply_nested_subfield( + #[values(LanceFileVersion::V2_0, LanceFileVersion::V2_1)] version: LanceFileVersion, + ) { + let mid_fields = Fields::from(vec![ + ArrowField::new("a", DataType::Int32, true), + ArrowField::new("b", DataType::Int32, true), + ]); + let outer_fields = Fields::from(vec![ArrowField::new( + "middle", + DataType::Struct(mid_fields.clone()), + true, + )]); + let schema = Arc::new(ArrowSchema::new(vec![ + ArrowField::new("id", DataType::Int32, true), + ArrowField::new("outer", DataType::Struct(outer_fields.clone()), true), + ])); + // Field ids: outer=1, middle=2, a=3, b=4. + let middle = Arc::new(StructArray::new( + mid_fields.clone(), + vec![ + Arc::new(Int32Array::from_iter_values(0..6)), + Arc::new(Int32Array::from_iter_values((0..6).map(|v| v * 100))), + ], + None, + )); + let outer = Arc::new(StructArray::new(outer_fields, vec![middle], None)); + let batch = RecordBatch::try_new( + schema.clone(), + vec![Arc::new(Int32Array::from_iter_values(0..6)), outer], + ) .unwrap(); + let write_params = WriteParams { + max_rows_per_file: 6, + max_rows_per_group: 6, + data_storage_version: Some(version), + ..Default::default() + }; + let reader = RecordBatchIterator::new(vec![Ok(batch)], schema.clone()); + let dataset = Dataset::write(reader, "memory://", Some(write_params)) + .await + .unwrap(); - Dataset::open(test_uri).await.unwrap() + // Overlay the deep leaf `outer.middle.a` (field id 3) at offset 2. + let a_leaf = Fields::from(vec![ArrowField::new("a", DataType::Int32, true)]); + let mid_a = Fields::from(vec![ArrowField::new( + "middle", + DataType::Struct(a_leaf.clone()), + true, + )]); + let overlay = Arc::new(StructArray::new( + mid_a, + vec![Arc::new(StructArray::new( + a_leaf, + vec![Arc::new(Int32Array::from(vec![777]))], + None, + ))], + None, + )) as ArrayRef; + let dataset = commit_overlay( + dataset, + "deepov", + 0, + &[3], + OverlayCoverage::dense(bitmap([2])), + vec![overlay], + version, + ) + .await; + + let frag = dataset.get_fragment(0).unwrap(); + let batch = frag.take(&[1, 2], &full_schema(&dataset)).await.unwrap(); + let outer = struct_col(&batch, "outer"); + let middle = outer + .column(0) + .as_any() + .downcast_ref::() + .unwrap(); + // a: offset 1 base (1), offset 2 overlaid (777); b untouched. + assert_eq!(i32_child(middle, 0).values(), &[1, 777]); + assert_eq!(i32_child(middle, 1).values(), &[100, 200]); + + // Projecting the *intermediate* struct `outer.middle` (field id 2) while + // the overlay targets a deeper field (id 3) must still apply: the + // overlay's leaf id falls inside the projected subtree, so it maps to a + // projected atomic field. (This is the case wjones127/westonpace flagged where a + // top-level-only mapping would miss the overlay.) + let middle_only = dataset.schema().project_by_ids(&[2], true); + let batch = frag.take(&[2], &middle_only).await.unwrap(); + let middle = struct_col(&batch, "outer") + .column(0) + .as_any() + .downcast_ref::() + .unwrap(); + assert_eq!(i32_child(middle, 0).values(), &[777]); + } } #[rstest] @@ -3182,7 +5110,7 @@ mod tests { updated_fragments: vec![u1.fragment], new_fragments: vec![], fields_modified: u1.fields_modified, - merged_generations: Vec::new(), + compacted_sstables: Vec::new(), fields_for_preserving_frag_bitmap: vec![], update_mode: Some(UpdateMode::RewriteColumns), inserted_rows_filter: None, @@ -3263,7 +5191,7 @@ mod tests { updated_fragments: vec![u2.fragment], new_fragments: vec![], fields_modified: u2.fields_modified, - merged_generations: Vec::new(), + compacted_sstables: Vec::new(), fields_for_preserving_frag_bitmap: vec![], update_mode: Some(UpdateMode::RewriteColumns), inserted_rows_filter: None, @@ -3447,19 +5375,20 @@ mod tests { dataset.delete("i >= 0 and i < 15").await.unwrap(); let fragment = &dataset.get_fragments()[0]; - let mut reader = fragment - .open( + let read_config = FragReadConfig::default().with_row_id(true); + + if data_storage_version == LanceFileVersion::Legacy { + let mut reader = crate::dataset::versions::open_v1_fragment_reader( + fragment, dataset.schema(), - FragReadConfig::default().with_row_id(true), + &read_config, ) .await .unwrap(); - reader.with_make_deletions_null(); - - if data_storage_version == LanceFileVersion::Legacy { + reader.with_make_deletions_null(); // The first batch is entirely deleted, deleted rows will be marked null with null row ids. let batch1 = reader - .legacy_read_batch_projected(0, .., dataset.schema()) + .read_batch_projected(0, .., dataset.schema()) .await .unwrap(); assert_eq!( @@ -3470,7 +5399,7 @@ mod tests { // The second batch is partially deleted, so the deleted rows will be // marked null with null row ids. let batch2 = reader - .legacy_read_batch_projected(1, .., dataset.schema()) + .read_batch_projected(1, .., dataset.schema()) .await .unwrap(); assert_eq!( @@ -3480,7 +5409,7 @@ mod tests { // The final batch is not deleted, so it will be returned as-is. let batch3 = reader - .legacy_read_batch_projected(2, .., dataset.schema()) + .read_batch_projected(2, .., dataset.schema()) .await .unwrap(); assert_eq!( @@ -3488,6 +5417,8 @@ mod tests { &UInt64Array::from_iter_values(20..30) ); } else { + let mut reader = fragment.open(dataset.schema(), read_config).await.unwrap(); + reader.with_make_deletions_null(); let to_batches = |range: Range| { let batch_size = range.len() as u32; let fut = reader.take_range(range, batch_size); @@ -3837,48 +5768,68 @@ mod tests { assert_eq!(dataset.count_rows(None).await.unwrap(), 195); } - let fragment = &mut dataset.get_fragment(0).unwrap(); - let mut updater = fragment.updater(Some(&["i"]), None, None).await.unwrap(); let new_schema = Arc::new(ArrowSchema::new(vec![ArrowField::new( "double_i", DataType::Int32, true, )])); - while let Some(batch) = updater.next().await.unwrap() { - let input_col = batch.column_by_name("i").unwrap(); - let result_col = mul(input_col, &Int32Array::new_scalar(2)).unwrap(); - let batch = RecordBatch::try_new( - new_schema.clone(), - vec![Arc::new(result_col) as ArrayRef], - ) - .unwrap(); - updater.update(batch).await.unwrap(); - } - let new_fragment = updater.finish().await.unwrap(); + // Merge keeps the fragment list intact, so every fragment gets the new + // column. Fragment 0 is the one carrying the deletions. + let fragment_ids = dataset + .manifest + .fragments + .iter() + .map(|f| f.id as usize) + .collect::>(); + let mut merged_fragments = Vec::new(); + for fragment_id in fragment_ids { + let fragment = &mut dataset.get_fragment(fragment_id).unwrap(); + let mut updater = fragment + .updater(Some(&["i"]), None, None, None) + .await + .unwrap(); + while let Some(batch) = updater.next().await.unwrap() { + let input_col = batch.column_by_name("i").unwrap(); + let result_col = mul(input_col, &Int32Array::new_scalar(2)).unwrap(); + let batch = RecordBatch::try_new( + new_schema.clone(), + vec![Arc::new(result_col) as ArrayRef], + ) + .unwrap(); + updater.update(batch).await.unwrap(); + } + let new_fragment = updater.finish().await.unwrap(); - assert_eq!(new_fragment.files.len(), 2); + assert_eq!(new_fragment.files.len(), 2); + merged_fragments.push(new_fragment); + } // Scan again let mut full_schema = dataset.schema().merge(new_schema.as_ref()).unwrap(); full_schema.set_field_id(None); let before_version = dataset.version().version; - let op = Operation::Overwrite { - fragments: vec![new_fragment], + let op = Operation::Merge { + fragments: merged_fragments, schema: full_schema.clone(), - config_upsert_values: None, - initial_bases: None, + preserves_nullability: true, }; - let dataset = - Dataset::commit(test_uri, op, None, None, None, Default::default(), false) - .await - .unwrap(); + let dataset = Dataset::commit( + test_uri, + op, + Some(before_version), + None, + None, + Default::default(), + false, + ) + .await + .unwrap(); - // We only kept the first fragment of 40 rows assert_eq!( dataset.count_rows(None).await.unwrap(), - if with_delete { 35 } else { 40 } + if with_delete { 195 } else { 200 } ); assert_eq!(dataset.version().version, before_version + 1); dataset.validate().await.unwrap(); @@ -4029,7 +5980,7 @@ mod tests { .unwrap(); let (object_store, base_path) = ObjectStore::from_uri(test_uri).await.unwrap(); - let file_reader = PreviousFileReader::try_new_with_fragment_id( + let file_reader = V1FileReader::try_new_with_fragment_id( &object_store, &base_path .clone() @@ -4092,7 +6043,7 @@ mod tests { let fragment = dataset.get_fragments().pop().unwrap(); // Write batch_s using add_columns - let mut updater = fragment.updater(Some(&["i"]), None, None).await?; + let mut updater = fragment.updater(Some(&["i"]), None, None, None).await?; updater.next().await?; updater.update(batch_s.clone()).await?; let frag = updater.finish().await?; @@ -4105,6 +6056,7 @@ mod tests { Operation::Merge { schema, fragments: vec![frag], + preserves_nullability: true, }, Some(dataset.manifest.version), None, @@ -4180,7 +6132,7 @@ mod tests { FragReadConfig::default().with_row_id(true), ) .await?; - let batch = reader.legacy_read_range_as_batch(0..20).await?; + let batch = reader.read_range_as_batch(0..20).await?; let expected_data = RecordBatch::try_new( Arc::new(ArrowSchema::new(vec![ROW_ID_FIELD.clone()])), @@ -4220,8 +6172,12 @@ mod tests { let store = ObjectStore::local(); let file_path = dataset.data_dir().join("some_file.lance"); let object_writer = store.create(&file_path).await.unwrap(); - let mut file_writer = - lance_file::writer::FileWriter::new_lazy(object_writer, FileWriterOptions::default()); + let mut file_writer = lance_file::versions::create_lazy_writer( + LanceFileVersion::Stable.resolve(), + object_writer, + FileWriterOptions::default(), + ) + .unwrap(); file_writer.write_batch(&new_data).await.unwrap(); file_writer.finish().await.unwrap(); @@ -4236,6 +6192,23 @@ mod tests { LanceFileVersion::Stable.resolve() ); + let mismatched_path = dataset.data_dir().join("mismatched_file.lance"); + let object_writer = store.create(&mismatched_path).await.unwrap(); + let mut mismatched_writer = lance_file::versions::create_lazy_writer( + lance_file::version::ConcreteFileVersion::V2_0, + object_writer, + FileWriterOptions::default(), + ) + .unwrap(); + mismatched_writer.write_batch(&new_data).await.unwrap(); + mismatched_writer.finish().await.unwrap(); + + let err = FileFragment::create_from_file("mismatched_file.lance", &dataset, 1, Some(128)) + .await + .unwrap_err(); + assert!(matches!(err, Error::InvalidInput { .. })); + assert!(err.to_string().contains("File version mismatch")); + let op = Operation::Append { fragments: vec![frag], }; @@ -4359,6 +6332,59 @@ mod tests { ); } + #[test] + fn test_indexed_metadata_heuristic_counts_selected_physical_columns() { + let schema = Schema::try_from(&ArrowSchema::new(vec![ + ArrowField::new( + "s", + DataType::Struct( + vec![ + ArrowField::new("x", DataType::Int32, true), + ArrowField::new("y", DataType::Int32, true), + ] + .into(), + ), + true, + ), + ArrowField::new("a", DataType::Int32, true), + ArrowField::new("b", DataType::Int32, true), + ArrowField::new("c", DataType::Int32, true), + ])) + .unwrap(); + let data_file = DataFile { + path: "wide.lance".to_string(), + fields: Arc::from([0, 1, 2, 3, 4, 5]), + column_indices: Arc::from([-1, 0, 1, 2, 3, 4]), + file_major_version: 2, + file_minor_version: 1, + file_size_bytes: CachedFileSize::unknown(), + base_id: None, + }; + + let full_struct = file_versions::reader_projection_from_column_names( + ConcreteFileVersion::V2_1, + &schema, + &["s"], + ) + .unwrap(); + assert_eq!(full_struct.column_indices.len(), 2); + let valid_column_count = data_file + .column_indices + .iter() + .filter(|column_index| **column_index >= 0) + .count(); + assert!(full_struct.column_indices.len().saturating_mul(4) >= valid_column_count); + + let partial_struct = file_versions::reader_projection_from_column_names( + ConcreteFileVersion::V2_1, + &schema, + &["s.x"], + ) + .unwrap(); + assert_eq!(partial_struct.column_indices.len(), 1); + assert!(partial_struct.column_indices.len().saturating_mul(4) < valid_column_count); + } + #[tokio::test] async fn test_iops_read_small() { // Create a file that has 8 columns. @@ -4420,4 +6446,80 @@ mod tests { assert_io_eq!(stats, read_iops, 1); assert_io_lt!(stats, read_bytes, 4096); } + + #[tokio::test] + async fn test_update_columns_with_json_extension_type() { + use arrow_array::UInt64Array; + use lance_arrow::ARROW_EXT_NAME_KEY; + use lance_arrow::json::ARROW_JSON_EXT_NAME; + use lance_core::ROW_ID; + use std::collections::HashMap; + + // Create a dataset with an Arrow JSON extension column + let test_dir = TempStrDir::default(); + let mut json_metadata = HashMap::new(); + json_metadata.insert( + ARROW_EXT_NAME_KEY.to_string(), + ARROW_JSON_EXT_NAME.to_string(), + ); + let schema = Arc::new(ArrowSchema::new(vec![ + ArrowField::new("id", DataType::Int64, false), + ArrowField::new("name", DataType::Utf8, true), + ArrowField::new("meta", DataType::Utf8, true).with_metadata(json_metadata.clone()), + ])); + let batch = RecordBatch::try_new( + schema.clone(), + vec![ + Arc::new(Int64Array::from(vec![1, 2, 3, 4, 5])), + Arc::new(StringArray::from(vec!["a", "b", "c", "d", "e"])), + Arc::new(StringArray::from(vec![ + r#"{"x":1}"#, + r#"{"x":2}"#, + r#"{"x":3}"#, + r#"{"x":4}"#, + r#"{"x":5}"#, + ])), + ], + ) + .unwrap(); + let reader = RecordBatchIterator::new(vec![Ok(batch)], schema.clone()); + let dataset = Dataset::write(reader, test_dir.as_ref(), None) + .await + .unwrap(); + + // Build the right stream with Arrow JSON column (Utf8 + arrow.json extension) + // Only update rows with row_id 1 and 3 + let update_schema = Arc::new(ArrowSchema::new(vec![ + ArrowField::new(ROW_ID, DataType::UInt64, false), + ArrowField::new("meta", DataType::Utf8, true).with_metadata(json_metadata), + ])); + let update_batch = RecordBatch::try_new( + update_schema.clone(), + vec![ + Arc::new(UInt64Array::from(vec![1, 3])), + Arc::new(StringArray::from(vec![ + r#"{"updated":true,"id":2}"#, + r#"{"updated":true,"id":4}"#, + ])), + ], + ) + .unwrap(); + let right_stream: Box = Box::new(RecordBatchIterator::new( + vec![Ok(update_batch)], + update_schema, + )); + + // Perform update_columns - this should NOT fail with type mismatch + // Previously this would error with: + // "It is not possible to interleave arrays of different data types (Utf8 and LargeBinary)" + let mut fragment = dataset.get_fragment(0).unwrap(); + let (updated_fragment, fields_modified) = fragment + .update_columns(right_stream, ROW_ID, ROW_ID) + .await + .unwrap(); + + // Verify the operation produced valid results + assert!(!fields_modified.is_empty()); + assert!(!updated_fragment.files.is_empty()); + } } diff --git a/rust/lance/src/dataset/fragment/session.rs b/rust/lance/src/dataset/fragment/session.rs index de50255bb72..77e4125bbc4 100644 --- a/rust/lance/src/dataset/fragment/session.rs +++ b/rust/lance/src/dataset/fragment/session.rs @@ -74,7 +74,7 @@ impl FragmentSession { if row_offsets.len() > 1 && FileFragment::row_ids_contiguous(row_offsets) { let range = (row_offsets[0] as usize)..(row_offsets[row_offsets.len() - 1] as usize + 1); - self.reader.legacy_read_range_as_batch(range).await + self.reader.read_range_as_batch(range).await } else { self.reader.take_as_batch(row_offsets, None).await } @@ -89,7 +89,7 @@ mod tests { use arrow_schema::{DataType, Field as ArrowField, Schema as ArrowSchema}; use lance_core::ROW_ADDR; use lance_core::utils::tempfile::TempStrDir; - use lance_encoding::version::LanceFileVersion; + use lance_file::version::LanceFileVersion; use rstest::rstest; use std::sync::Arc; diff --git a/rust/lance/src/dataset/fragment/write.rs b/rust/lance/src/dataset/fragment/write.rs index a61f0e0c46a..eabade7afdf 100644 --- a/rust/lance/src/dataset/fragment/write.rs +++ b/rust/lance/src/dataset/fragment/write.rs @@ -8,10 +8,13 @@ use lance_core::Error; use lance_core::datatypes::Schema; use lance_datafusion::chunker::{break_stream, chunk_stream}; use lance_datafusion::utils::StreamingWriteSource; -use lance_file::previous::writer::FileWriter as PreviousFileWriter; +#[cfg(test)] use lance_file::version::LanceFileVersion; -use lance_file::writer::FileWriterOptions; +use lance_file::version::stable_file_version; +use lance_file::versions::v1::writer::FileWriter as V1FileWriter; +use lance_file::writer::FileWriter; use lance_io::object_store::ObjectStore; +use lance_io::traits::Writer; use lance_io::utils::CachedFileSize; use lance_table::format::{DataFile, Fragment}; use lance_table::io::manifest::ManifestDescribing; @@ -21,7 +24,8 @@ use uuid::Uuid; use crate::Result; use crate::dataset::builder::DatasetBuilder; -use crate::dataset::write::{do_write_fragments, validate_and_resolve_target_bases_with_primary}; +use crate::dataset::utils::SchemaAdapter; +use crate::dataset::write::validate_and_resolve_target_bases_with_primary; use crate::dataset::{DATA_DIR, Dataset, ReadParams, WriteMode, WriteParams}; /// Generates a filename optimized for S3 throughput using a UUID-based approach. @@ -106,7 +110,25 @@ impl<'a> FragmentCreateBuilder<'a> { id: Option, ) -> Result { let (stream, schema) = self.get_stream_and_schema(Box::new(source)).await?; - self.write_impl(stream, schema, id).await + // Convert Arrow JSON columns (`arrow.json`, stored as Utf8) into Lance JSON + // (`lance.json`, stored as JSONB-encoded LargeBinary) before writing. The + // multi-fragment and dataset write paths perform this through + // `versions::write_fragments_direct`; + // the single-fragment create path must do the same or the raw UTF-8 string bytes + // would be written into a column whose schema declares JSONB, corrupting reads. + let stream = SchemaAdapter::new(stream.schema()).to_physical_stream(stream); + let version = self + .write_params + .map(|params| params.storage_version_or_default()) + .unwrap_or_else(stable_file_version); + crate::dataset::versions::write_fragment( + version, + self, + stream, + schema, + id.unwrap_or_default(), + ) + .await } /// Write multi fragment which separated by max_rows_per_file. @@ -118,12 +140,16 @@ impl<'a> FragmentCreateBuilder<'a> { self.write_fragments_v2_impl(stream, schema).await } - async fn write_v2_impl( + pub(crate) async fn write_current_impl( &self, + create_writer: F, stream: SendableRecordBatchStream, schema: Schema, id: u64, - ) -> Result { + ) -> Result + where + F: FnOnce(Box, Schema, String) -> Result<(FileWriter, DataFile)>, + { let params = self.write_params.map(Cow::Borrowed).unwrap_or_default(); let progress = params.progress.as_ref(); @@ -140,18 +166,7 @@ impl<'a> FragmentCreateBuilder<'a> { let mut fragment = Fragment::new(id); let full_path = base_path.clone().join(DATA_DIR).join(filename.clone()); let obj_writer = object_store.create(&full_path).await?; - let mut writer = lance_file::writer::FileWriter::try_new( - obj_writer, - schema, - FileWriterOptions { - format_version: params.data_storage_version, - ..Default::default() - }, - )?; - - let (major, minor) = writer.version().to_numbers(); - - let data_file = DataFile::new_unstarted(filename, major, minor); + let (mut writer, data_file) = create_writer(obj_writer, schema, filename)?; fragment.files.push(data_file); progress.begin(&fragment).await?; @@ -203,7 +218,7 @@ impl<'a> FragmentCreateBuilder<'a> { Self::validate_schema(&schema, stream.schema().as_ref())?; - let version = params.data_storage_version.unwrap_or_default(); + let version = params.storage_version_or_default(); let needs_existing_dataset = params.target_base_names_or_paths.is_some() || params.target_bases.is_some() || params.target_all_bases.is_some() @@ -234,34 +249,27 @@ impl<'a> FragmentCreateBuilder<'a> { } else { None }; - do_write_fragments( + crate::dataset::versions::write_fragments_direct( + version, existing_dataset.as_ref(), object_store, &base_path, &schema, stream, params, - version, target_bases_info, + Vec::new(), ) .await } - async fn write_impl( + pub(crate) async fn write_v1_impl( &self, stream: SendableRecordBatchStream, schema: Schema, - id: Option, + id: u64, ) -> Result { - let id = id.unwrap_or_default(); - let params = self.write_params.map(Cow::Borrowed).unwrap_or_default(); - - let storage_version = params.storage_version_or_default(); - - if storage_version != LanceFileVersion::Legacy { - return self.write_v2_impl(stream, schema, id).await; - } let progress = params.progress.as_ref(); Self::validate_schema(&schema, stream.schema().as_ref())?; @@ -275,7 +283,7 @@ impl<'a> FragmentCreateBuilder<'a> { let filename = format!("{}.lance", generate_random_filename()); let mut fragment = Fragment::with_file_legacy(id, &filename, &schema, None); let full_path = base_path.clone().join(DATA_DIR).join(filename.clone()); - let mut writer = PreviousFileWriter::::try_new( + let mut writer = V1FileWriter::::try_new( &object_store, &full_path, schema, @@ -317,30 +325,27 @@ impl<'a> FragmentCreateBuilder<'a> { } async fn existing_dataset_schema(&self) -> Result> { - let mut builder = DatasetBuilder::from_uri(self.dataset_uri); - let accessor = self - .write_params - .and_then(|p| p.store_params.as_ref()) - .and_then(|p| p.storage_options_accessor.clone()); - if let Some(accessor) = accessor { - builder = builder.with_storage_options_accessor(accessor); - } - match builder.load().await { - Ok(dataset) => { - // Use the schema from the dataset, because it has the correct - // field ids. - Ok(Some(dataset.schema().clone())) - } - Err(Error::DatasetNotFound { .. }) => { - // If the dataset does not exist, we can use the schema from - // the reader. - Ok(None) - } + let params = self.write_params.map(Cow::Borrowed).unwrap_or_default(); + match self.load_existing_dataset(¶ms).await { + // Use the schema from the dataset, because it has the correct + // field ids. + Ok(dataset) => Ok(Some(dataset.schema().clone())), + // If the dataset does not exist, we can use the schema from + // the reader. + Err(Error::DatasetNotFound { .. }) => Ok(None), Err(e) => Err(e), } } async fn existing_dataset(&self, params: &WriteParams) -> Result> { + match self.load_existing_dataset(params).await { + Ok(dataset) => Ok(Some(dataset)), + Err(Error::DatasetNotFound { .. } | Error::NotFound { .. }) => Ok(None), + Err(e) => Err(e), + } + } + + async fn load_existing_dataset(&self, params: &WriteParams) -> Result { let mut builder = DatasetBuilder::from_uri(self.dataset_uri).with_read_params(ReadParams { store_options: params.store_params.clone(), commit_handler: params.commit_handler.clone(), @@ -352,11 +357,7 @@ impl<'a> FragmentCreateBuilder<'a> { builder = builder.with_base_store_params(base_path, store_params.clone()); } } - match builder.load().await { - Ok(dataset) => Ok(Some(dataset)), - Err(Error::DatasetNotFound { .. } | Error::NotFound { .. }) => Ok(None), - Err(e) => Err(e), - } + builder.load().await } fn validate_schema(expected: &Schema, actual: &ArrowSchema) -> Result<()> { @@ -375,7 +376,7 @@ mod tests { use std::sync::Arc; use arrow_array::{ - Int64Array, RecordBatch, RecordBatchIterator, RecordBatchReader, StringArray, + Int64Array, RecordBatch, RecordBatchIterator, RecordBatchReader, StringArray, record_batch, }; use arrow_schema::{DataType, Field as ArrowField}; use lance_arrow::SchemaExt; @@ -415,7 +416,7 @@ mod tests { matches!(result.as_ref().unwrap_err(), Error::InvalidInput { source, .. } if source.to_string().contains("Cannot write with an empty schema.")), "{:?}", - &result + result ); // Writing empty reader produces an error @@ -429,7 +430,7 @@ mod tests { matches!(result.as_ref().unwrap_err(), Error::InvalidInput { source, .. } if source.to_string().contains("Input data was empty.")), "{:?}", - &result + result ); // Writing with incorrect schema produces an error. @@ -447,7 +448,7 @@ mod tests { matches!(result.as_ref().unwrap_err(), Error::SchemaMismatch { difference, .. } if difference.contains("fields did not match")), "{:?}", - &result + result ); } @@ -492,6 +493,52 @@ mod tests { assert_eq!(fragment.files[0].column_indices.as_ref(), &[0, 1]); } + #[tokio::test] + async fn test_fragment_create_with_session() { + let session = Arc::new(crate::session::Session::new( + 0, + 1024 * 1024, + Default::default(), + )); + let write_params = WriteParams { + session: Some(session.clone()), + ..Default::default() + }; + // Keep the dataset alive: the registry only holds weak references, so + // the in-memory store survives through the dataset's strong reference. + let initial_batch = + record_batch!(("a", Int64, [1, 2, 3]), ("b", Utf8, ["a", "b", "c"])).unwrap(); + let mut dataset = InsertBuilder::new("memory://") + .with_params(&write_params) + .execute(vec![initial_batch]) + .await + .unwrap(); + // Drop a column so the surviving field id is non-trivial (!= 0). + dataset.drop_columns(&["a"]).await.unwrap(); + let field_id = dataset.schema().field("b").unwrap().id; + assert_ne!(field_id, 0); + + let append_batch = record_batch!(("b", Utf8, ["d", "e"])).unwrap(); + let append_data = + RecordBatchIterator::new([Ok(append_batch.clone())], append_batch.schema()); + + let append_params = WriteParams { + session: Some(session.clone()), + mode: WriteMode::Append, + ..Default::default() + }; + let fragment = FragmentCreateBuilder::new(dataset.uri()) + .write_params(&append_params) + .write(append_data, None) + .await + .unwrap(); + + assert_eq!(fragment.files[0].fields.as_ref(), &[field_id]); + // The manifest load for schema inference went through the shared + // session's metadata cache. + assert!(session.metadata_cache_stats().await.num_entries > 0); + } + #[tokio::test] async fn test_write_fragments_validation() { // Writing with empty schema produces an error @@ -506,7 +553,7 @@ mod tests { matches!(result.as_ref().unwrap_err(), Error::InvalidInput { source, .. } if source.to_string().contains("Cannot write with an empty schema.")), "{:?}", - &result + result ); // Writing empty reader produces an error @@ -533,7 +580,7 @@ mod tests { matches!(result.as_ref().unwrap_err(), Error::SchemaMismatch { difference, .. } if difference.contains("fields did not match")), "{:?}", - &result + result ); } @@ -685,7 +732,7 @@ mod tests { assert!(!fragment.files.is_empty()); fragment.files.iter().for_each(|f| { - let (major_version, minor_version) = file_version.to_numbers(); + let (major_version, minor_version) = file_version.resolve().to_data_file_numbers(); assert_eq!(f.file_major_version, major_version); assert_eq!(f.file_minor_version, minor_version); }) @@ -717,7 +764,7 @@ mod tests { assert!(!fragment.is_empty()); fragment[0].files.iter().for_each(|f| { - let (major_version, minor_version) = file_version.to_numbers(); + let (major_version, minor_version) = file_version.resolve().to_data_file_numbers(); assert_eq!(f.file_major_version, major_version); assert_eq!(f.file_minor_version, minor_version); }) diff --git a/rust/lance/src/dataset/hash_joiner.rs b/rust/lance/src/dataset/hash_joiner.rs index 92b67599134..d5410d9ae3a 100644 --- a/rust/lance/src/dataset/hash_joiner.rs +++ b/rust/lance/src/dataset/hash_joiner.rs @@ -194,21 +194,11 @@ impl HashJoiner { } pub fn check_lance_support_null(array: &ArrayRef, dataset: &Dataset) -> Result<()> { - if array.null_count() > 0 && !dataset.lance_supports_nulls(array.data_type()) { - return Err(Error::invalid_input(format!( - "Join produced null values for type: {:?}, but storing \ - nulls for this data type is not supported by the \ - dataset's current Lance file format version: {:?}. This \ - can be caused by an explicit null in the new data.", - array.data_type(), - dataset - .manifest() - .data_storage_format - .lance_file_version() - .unwrap() - ))); - } - Ok(()) + super::versions::validate_nulls( + dataset.manifest().data_storage_format.lance_file_format(), + array.data_type(), + array.null_count() > 0, + ) } /// Collecting the data using the index column from left table, @@ -307,6 +297,48 @@ mod tests { Dataset::open(&uri).await.unwrap() } + #[test] + fn test_null_validation_is_selected_by_exact_version() { + use lance_file::version::ConcreteFileVersion; + + assert!( + super::super::versions::validate_nulls( + ConcreteFileVersion::V1, + &DataType::Int32, + true, + ) + .is_err() + ); + assert!( + super::super::versions::validate_nulls(ConcreteFileVersion::V1, &DataType::Utf8, true,) + .is_ok() + ); + assert!( + super::super::versions::validate_nulls( + ConcreteFileVersion::V2_0, + &DataType::Struct(arrow_schema::Fields::empty()), + true, + ) + .is_err() + ); + assert!( + super::super::versions::validate_nulls( + ConcreteFileVersion::V2_1, + &DataType::Struct(arrow_schema::Fields::empty()), + true, + ) + .is_ok() + ); + assert!( + super::super::versions::validate_nulls( + ConcreteFileVersion::V1, + &DataType::Int32, + false, + ) + .is_ok() + ); + } + #[tokio::test] async fn test_joiner_collect() { let schema = Arc::new(Schema::new(vec![ diff --git a/rust/lance/src/dataset/index.rs b/rust/lance/src/dataset/index.rs index 7cac7815125..9ce3407e9cc 100644 --- a/rust/lance/src/dataset/index.rs +++ b/rust/lance/src/dataset/index.rs @@ -16,8 +16,8 @@ use crate::index::scalar::infer_scalar_index_details; use arrow_schema::DataType; use async_trait::async_trait; use lance_core::{Error, Result}; -use lance_encoding::version::LanceFileVersion; -use lance_index::frag_reuse::FRAG_REUSE_INDEX_NAME; +use lance_file::version::ConcreteFileVersion; +use lance_index::is_system_index; use lance_index::pb::VectorIndexDetails; use lance_index::scalar::lance_format::LanceIndexStore; use lance_table::format::IndexMetadata; @@ -25,23 +25,44 @@ use serde::{Deserialize, Serialize}; use uuid::Uuid; use super::optimize::{IndexRemapper, IndexRemapperOptions}; +use super::versions; #[derive(Debug, Default, Clone, Serialize, Deserialize, PartialEq, Eq)] pub struct DatasetIndexRemapperOptions {} +/// Loads index metadata when compaction has at least one index to remap. +/// +/// Returns all index metadata, including system indices, so the remapper uses a +/// consistent snapshot. Returns `None` when there are no non-system indices. +pub(crate) async fn load_indices_for_remapping( + dataset: &Dataset, +) -> Result>>> { + if dataset.manifest.index_section.is_none() { + return Ok(None); + } + + let indices = dataset.load_indices().await?; + let has_remappable_index = indices.iter().any(|index| !is_system_index(index)); + Ok(has_remappable_index.then_some(indices)) +} + +#[async_trait] impl IndexRemapperOptions for DatasetIndexRemapperOptions { - fn create_remapper( - &self, - dataset: &Dataset, - ) -> crate::Result> { - Ok(Box::new(DatasetIndexRemapper { + async fn create_remapper(&self, dataset: &Dataset) -> Result>> { + let Some(indices) = load_indices_for_remapping(dataset).await? else { + return Ok(None); + }; + + Ok(Some(Box::new(DatasetIndexRemapper { dataset: Arc::new(dataset.clone()), - })) + indices, + }))) } } struct DatasetIndexRemapper { dataset: Arc, + indices: Arc>, } impl DatasetIndexRemapper { @@ -62,10 +83,9 @@ impl IndexRemapper for DatasetIndexRemapper { affected_fragment_ids: &[u64], ) -> Result> { let affected_frag_ids = HashSet::::from_iter(affected_fragment_ids.iter().copied()); - let indices = self.dataset.load_indices().await?; - let mut remapped = Vec::with_capacity(indices.len()); - for index in indices.iter() { - let needs_remapped = index.name != FRAG_REUSE_INDEX_NAME + let mut remapped = Vec::with_capacity(self.indices.len()); + for index in self.indices.iter() { + let needs_remapped = !is_system_index(index) && match &index.fragment_bitmap { None => true, Some(fragment_bitmap) => fragment_bitmap @@ -73,7 +93,12 @@ impl IndexRemapper for DatasetIndexRemapper { .any(|frag_idx| affected_frag_ids.contains(&(frag_idx as u64))), }; if needs_remapped { - let remap_result = self.remap_index(index, &mapping).await?; + // Box the remap future at the call site: inlining `remap_index` into this + // loop's async layout otherwise exceeds rustc's depth limit. It has to be + // boxed here, not inside `remap_index` — boxing internally turns the + // future's `Send` check into a `Box: Send` trait obligation that + // overflows the solver through the cache types (E0275 downstream). + let remap_result = Box::pin(self.remap_index(index, &mapping)).await?; match remap_result { RemapResult::Drop => continue, RemapResult::Keep(id) => { @@ -133,18 +158,12 @@ pub trait LanceIndexStoreExt { Self: Sized; } -/// Extract the lance file version from a dataset, floored at V2_0. +/// Select the exact file version used for index files in this dataset version. /// /// Index files should never use the legacy format. If the dataset uses legacy -/// format or doesn't have a version set, V2_0 is used as the minimum. -pub(crate) fn dataset_format_version(dataset: &Dataset) -> LanceFileVersion { - dataset - .manifest - .data_storage_format - .lance_file_version() - .ok() - .map(|v| v.resolve().max(LanceFileVersion::V2_0)) - .unwrap_or(LanceFileVersion::V2_0) +/// format, V2_0 is selected explicitly by the dataset composition table. +pub(crate) fn dataset_format_version(dataset: &Dataset) -> ConcreteFileVersion { + versions::index_file_version(dataset.manifest.data_storage_format.lance_file_format()) } #[async_trait] @@ -178,13 +197,55 @@ impl LanceIndexStoreExt for LanceIndexStore { mod tests { use super::*; use crate::dataset::WriteParams; - use crate::index::DatasetIndexExt; + use crate::dataset::transaction::{Operation, Transaction}; + use crate::index::frag_reuse::build_frag_reuse_index_metadata; use crate::index::vector::VectorIndexParams; + use crate::index::{DatasetIndexExt, IntoIndexSegment}; use lance_datagen::{BatchCount, RowCount, array}; use lance_index::IndexType; + use lance_index::frag_reuse::{FRAG_REUSE_INDEX_NAME, FragReuseIndexDetails}; use lance_linalg::distance::MetricType; use uuid::Uuid; + #[tokio::test] + async fn test_remapper_not_created_without_remappable_indices() { + let reader = lance_datagen::gen_batch() + .col("id", array::step::()) + .into_reader_rows(RowCount::from(1), BatchCount::from(1)); + let mut dataset = Dataset::write(reader, "memory://", None).await.unwrap(); + let options = DatasetIndexRemapperOptions::default(); + + assert!(options.create_remapper(&dataset).await.unwrap().is_none()); + + let frag_reuse_index = build_frag_reuse_index_metadata( + &dataset, + None, + FragReuseIndexDetails { + versions: Vec::new(), + }, + Default::default(), + ) + .await + .unwrap(); + let transaction = Transaction::new( + dataset.manifest.version, + Operation::CreateIndex { + new_indices: vec![frag_reuse_index], + removed_indices: Vec::new(), + }, + None, + ); + dataset + .apply_commit(transaction, &Default::default(), &Default::default()) + .await + .unwrap(); + + let indices = dataset.load_indices().await.unwrap(); + assert_eq!(indices.len(), 1); + assert_eq!(indices[0].name, FRAG_REUSE_INDEX_NAME); + assert!(options.create_remapper(&dataset).await.unwrap().is_none()); + } + #[tokio::test] async fn test_remapper_only_touches_segments_with_affected_fragments() { let test_dir = tempfile::tempdir().unwrap(); @@ -257,20 +318,10 @@ mod tests { let segments = segments .iter() .map(|segment| { - crate::index::IndexSegment::new( - segment.uuid, - segment - .fragment_bitmap - .as_ref() - .expect("test segment metadata should have fragment coverage") - .iter(), - segment - .index_details - .as_ref() - .expect("test segment metadata should have index details") - .clone(), - segment.index_version, - ) + segment + .clone() + .into_index_segment() + .expect("test segment metadata should convert to an index segment") }) .collect::>(); @@ -296,7 +347,9 @@ mod tests { let remapper = DatasetIndexRemapperOptions::default() .create_remapper(&dataset) - .unwrap(); + .await + .unwrap() + .expect("vector index should require a remapper"); let remapped = remapper .remap_indices(RowAddrRemap::empty(), &[target_fragments[0].id() as u64]) .await diff --git a/rust/lance/src/dataset/index/frag_reuse.rs b/rust/lance/src/dataset/index/frag_reuse.rs index ceebe456bbf..08a3e8f42fd 100644 --- a/rust/lance/src/dataset/index/frag_reuse.rs +++ b/rust/lance/src/dataset/index/frag_reuse.rs @@ -16,15 +16,34 @@ use roaring::RoaringBitmap; /// If all the indices currently available are already caught up to as a specific reuse version, /// all older reuse versions (inclusive) can be cleaned up. /// -/// An index is considered caught up against a specific reuse version if -/// 1. the index is created after or at the same dataset version as the reuse version -/// 2. there is no old fragment in the version that is covered by the index and can be remapped. -/// If an index's fragment bitmap is missing, we will consider it as caught up. -/// Otherwise, we will never be able to clean up the reuse version. +/// An index is considered caught up against a specific reuse version if either: +/// 1. its coverage is disjoint from the fragments the reuse chain touches, so it +/// holds nothing the FRI would remap (the common multi-index case: a +/// compaction rewrote a sibling index's fragments, not this one); or +/// 2. it is at or past the reuse version's dataset version and no old fragment +/// in the version is still in its bitmap. A missing bitmap counts as caught +/// up, else the version could never be cleaned up. /// /// Note that there could be a race condition that an index is being added during the cleanup, /// This will make that specific index not efficient until the next reindex, /// but it will not cause any correctness problem. +/// +/// Typically run after [`compact_files`] with deferred remap and per-index +/// [`remap_column_index`] have caught the indexes up. +/// +/// # Example +/// +/// ```no_run +/// # use lance::dataset::index::frag_reuse::cleanup_frag_reuse_index; +/// # async fn example(dataset: &mut lance::Dataset) -> lance::Result<()> { +/// // Trim the fragment-reuse index to the versions still needed by some index. +/// cleanup_frag_reuse_index(dataset).await?; +/// # Ok(()) +/// # } +/// ``` +/// +/// [`compact_files`]: crate::dataset::optimize::compact_files +/// [`remap_column_index`]: crate::dataset::optimize::remapping::remap_column_index pub async fn cleanup_frag_reuse_index(dataset: &mut Dataset) -> lance_core::Result<()> { // check against index metadata before auto-remap let indices = read_manifest_indexes( @@ -42,12 +61,14 @@ pub async fn cleanup_frag_reuse_index(dataset: &mut Dataset) -> lance_core::Resu .await .unwrap(); + let chain_frag_bitmap = reuse_chain_frag_bitmap(&frag_reuse_details.versions); + let mut retained_versions = Vec::new(); let mut fragment_bitmaps = RoaringBitmap::new(); for version in frag_reuse_details.versions.iter() { let check_results = indices .iter() - .map(|idx| is_index_remap_caught_up(version, idx)) + .map(|idx| is_index_remap_caught_up(version, idx, &chain_frag_bitmap)) .collect::>(); if check_results @@ -97,14 +118,38 @@ pub async fn cleanup_frag_reuse_index(dataset: &mut Dataset) -> lance_core::Resu Ok(()) } +/// Every fragment the reuse chain touches (old + new) across all versions. An +/// index disjoint from this set holds no row address the FRI remaps, so trimming +/// can never strand it (fragment ids are never reused). +fn reuse_chain_frag_bitmap(versions: &[FragReuseVersion]) -> RoaringBitmap { + let mut bitmap = RoaringBitmap::new(); + for version in versions { + bitmap.extend(version.old_frag_ids().iter().map(|&id| id as u32)); + bitmap.extend(version.new_frag_ids().iter().map(|&id| id as u32)); + } + bitmap +} + fn is_index_remap_caught_up( frag_reuse_version: &FragReuseVersion, index_meta: &IndexMetadata, + chain_frag_bitmap: &RoaringBitmap, ) -> lance_core::Result { if is_system_index(index_meta) { return Ok(true); } + // Disjoint coverage => caught up regardless of dataset_version, bypassing the + // stale-version gate below (see fn docs). The chain includes NEW fragments + // deliberately: a deferred-remap commit advances a covering index's bitmap + // onto them before its data is remapped, so an old-frag-only check would + // clear a still-stale index and trim a version it needs. + if let Some(index_frag_bitmap) = &index_meta.fragment_bitmap + && index_frag_bitmap.is_disjoint(chain_frag_bitmap) + { + return Ok(true); + } + if index_meta.dataset_version < frag_reuse_version.dataset_version { return Ok(false); } @@ -157,6 +202,105 @@ mod tests { use lance_index::IndexType; use lance_index::scalar::ScalarIndexParams; + fn frag_digest(id: u64) -> lance_index::frag_reuse::FragDigest { + lance_index::frag_reuse::FragDigest { + id, + physical_rows: 100, + num_deleted_rows: 0, + } + } + + fn reuse_version(dataset_version: u64, old: &[u64], new: &[u64]) -> FragReuseVersion { + FragReuseVersion { + dataset_version, + groups: vec![lance_index::frag_reuse::FragReuseGroup { + changed_row_addrs: Vec::new(), + old_frags: old.iter().copied().map(frag_digest).collect(), + new_frags: new.iter().copied().map(frag_digest).collect(), + }], + } + } + + fn index_covering(dataset_version: u64, covered: &[u32]) -> IndexMetadata { + IndexMetadata { + uuid: uuid::Uuid::new_v4(), + fields: vec![0], + name: "test_idx".into(), + dataset_version, + fragment_bitmap: Some(RoaringBitmap::from_iter(covered.iter().copied())), + index_details: None, + index_version: 0, + created_at: None, + base_id: None, + files: None, + } + } + + /// The catch-up determination must not pin the FRI on an index that is + /// simply unrelated to the compaction, while still retaining versions that a + /// covering-but-not-yet-remapped index needs. + #[test] + fn test_caught_up_uses_fragment_coverage_not_only_version() { + // A reuse version at dataset_version 10 rewrote fragments [4, 5] -> [6]. + let version = reuse_version(10, &[4, 5], &[6]); + let chain = reuse_chain_frag_bitmap(std::slice::from_ref(&version)); + + // Non-covering, stale version: touches none of the rewritten frags, so + // caught up despite version 5 < 10 (the case the old gate got wrong). + assert_true!( + is_index_remap_caught_up(&version, &index_covering(5, &[1, 2, 3]), &chain).unwrap() + ); + + // Still holds an old fragment: not caught up. + assert_false!( + is_index_remap_caught_up(&version, &index_covering(5, &[1, 4, 5]), &chain).unwrap() + ); + + // Bitmap advanced onto the new fragment but data not yet remapped: not + // caught up (why the chain must include new frags). + assert_false!( + is_index_remap_caught_up(&version, &index_covering(5, &[1, 6]), &chain).unwrap() + ); + + // Once remapped (version advanced): caught up. + assert_true!( + is_index_remap_caught_up(&version, &index_covering(11, &[1, 6]), &chain).unwrap() + ); + } + + /// The chain spans every reuse version, not just the one being checked: a + /// stale index touching only a *later* version's fragment must still fall to + /// the version gate (a per-version chain would wrongly clear it). + #[test] + fn test_caught_up_uses_whole_reuse_chain() { + let v1 = reuse_version(10, &[4, 5], &[6]); // 4,5 -> 6 + let v2 = reuse_version(11, &[6], &[7]); // 6 -> 7 + let chain = reuse_chain_frag_bitmap(&[v1.clone(), v2]); + + // Stale index (version 5) covering only v2's new fragment [7]: not + // disjoint from the chain, so not caught up on v1. + assert_false!(is_index_remap_caught_up(&v1, &index_covering(5, &[1, 7]), &chain).unwrap()); + } + + /// Whole-fragment removal (every row deleted, no replacement): an index + /// emptied by the deletion has an empty bitmap and must count as caught up -- + /// it holds only dead rows -- else its stale version pins the removed-fragment + /// version forever (remap hits the drop-everything path, never advancing it). + #[test] + fn test_caught_up_handles_fragment_removal() { + // Reuse version 20 removed fragment [7] outright (no replacement). + let version = reuse_version(20, &[7], &[]); + let chain = reuse_chain_frag_bitmap(std::slice::from_ref(&version)); + + // Index emptied by the deletion (empty bitmap): caught up. + assert_true!(is_index_remap_caught_up(&version, &index_covering(5, &[]), &chain).unwrap()); + + // Bitmap still lists the removed fragment (not yet updated): retained. + assert_false!( + is_index_remap_caught_up(&version, &index_covering(5, &[7]), &chain).unwrap() + ); + } + #[tokio::test] async fn test_cleanup_frag_reuse_index() { let mut dataset = lance_datagen::gen_batch() @@ -209,7 +353,12 @@ mod tests { let scalar_index = indices.iter().find(|idx| idx.name == "scalar").unwrap(); // Should not be considered caught up because index was created at an old dataset version assert_false!( - is_index_remap_caught_up(&frag_reuse_details.versions[0], scalar_index).unwrap() + is_index_remap_caught_up( + &frag_reuse_details.versions[0], + scalar_index, + &reuse_chain_frag_bitmap(&frag_reuse_details.versions), + ) + .unwrap() ); // Remap and check index is caught up @@ -219,7 +368,12 @@ mod tests { let indices = dataset.load_indices().await.unwrap(); let scalar_index = indices.iter().find(|idx| idx.name == "scalar").unwrap(); assert_true!( - is_index_remap_caught_up(&frag_reuse_details.versions[0], scalar_index).unwrap() + is_index_remap_caught_up( + &frag_reuse_details.versions[0], + scalar_index, + &reuse_chain_frag_bitmap(&frag_reuse_details.versions), + ) + .unwrap() ); // Cleanup frag reuse index and check there is no reuse version @@ -313,7 +467,12 @@ mod tests { .find(|idx| idx.name == format!("{col}_idx")) .unwrap(); assert!( - is_index_remap_caught_up(&frag_reuse_details.versions[0], index).unwrap(), + is_index_remap_caught_up( + &frag_reuse_details.versions[0], + index, + &reuse_chain_frag_bitmap(&frag_reuse_details.versions), + ) + .unwrap(), "index {col}_idx was not caught up after remap" ); } diff --git a/rust/lance/src/dataset/mem_wal.rs b/rust/lance/src/dataset/mem_wal.rs index f5b89d06ff4..2ef95432e4e 100644 --- a/rust/lance/src/dataset/mem_wal.rs +++ b/rust/lance/src/dataset/mem_wal.rs @@ -52,19 +52,19 @@ use arrow_schema::{DataType, Field as ArrowField, Schema as ArrowSchema}; /// Column name for the mem_wal tombstone (delete sentinel) marker. /// /// `_tombstone` is a *physical* column present only in mem_wal memtables and -/// flushed generations — it is deliberately kept out of the base table (hard +/// SSTables — it is deliberately kept out of the base table (hard /// delete), so it is **not** a virtual [`is_system_column`](lance_core::is_system_column). /// A row with `_tombstone = true` is a delete sentinel: the newest value for /// its primary key, carrying null in every non-PK column, that wins /// newest-per-PK resolution and is then silently dropped from query results. /// -/// The column is owned end-to-end by lance: callers pass the base schema and +/// The column is owned end-to-end by lance: callers pass the logical schema and /// lance injects the column on the write path ([`write::ShardWriter::put`] / /// [`write::ShardWriter::delete`]), so no caller ever constructs or names it. pub const TOMBSTONE: &str = "_tombstone"; -/// The mem_wal tombstone field appended to the base schema to form the -/// memtable/generation schema. +/// The mem_wal tombstone field appended to the logical schema on the way to the +/// storage schema. /// /// Non-nullable: the write path always populates it (`false` for normal rows, /// `true` for tombstones). Non-nullability also lets the point-lookup base arm @@ -74,8 +74,46 @@ pub fn tombstone_field() -> ArrowField { ArrowField::new(TOMBSTONE, DataType::Boolean, false) } -/// Extend a base schema with the trailing `_tombstone` column to form the -/// mem_wal memtable/generation schema. +/// Derive a shard's *storage* schema from its *logical* (base table) schema by +/// widening every top-level field to nullable except the primary key and +/// `_tombstone`. +/// +/// A tombstone carries the primary key and null everywhere else, so storage +/// must permit a null wherever the base table does not. The logical schema +/// stays the caller's contract — validated at [`write::ShardWriter::put`], +/// restored at the scan's egress. +/// +/// Top-level only: Arrow validates nullability only there, so a vector column's +/// item field is untouched. The primary key is excluded because +/// [`lance_core::datatypes::Schema`] requires it non-nullable, `_tombstone` +/// because the write path always populates it. Idempotent. +pub fn relax_non_pk_nullability( + logical_schema: &ArrowSchema, + pk_columns: &[String], +) -> Arc { + let fields: Vec = logical_schema + .fields() + .iter() + .map(|field| { + let keep = field.is_nullable() + || field.name() == TOMBSTONE + || pk_columns.iter().any(|c| c == field.name()); + let field = field.as_ref().clone(); + if keep { + field + } else { + field.with_nullable(true) + } + }) + .collect(); + Arc::new(ArrowSchema::new_with_metadata( + fields, + logical_schema.metadata().clone(), + )) +} + +/// Extend the logical schema with the trailing `_tombstone` column — the +/// intermediate [`relax_non_pk_nullability`] widens into the storage schema. /// /// Idempotent: a schema that already carries `_tombstone` (a reopen/replay /// path) is returned unchanged. Schema-level metadata and per-field metadata @@ -92,7 +130,8 @@ pub fn schema_with_tombstone(base: &ArrowSchema) -> Arc { )) } -pub use api::{DatasetMemWalExt, InitializeMemWalBuilder}; +pub use api::{DatasetMemWalExt, InitializeMemWalBuilder, validate_maintained_indexes}; +pub use index::MemIndexKind; pub use manifest::ShardManifestStore; pub use memtable::scanner::MemTableScanner; pub use scanner::{LsmDataSource, LsmGeneration, LsmScanner, ShardSnapshot}; @@ -101,6 +140,98 @@ pub use sharding::{ evaluate_sharding_spec_with_source_columns, }; pub use wal::{BatchDurableWatcher, WalAppendResult, WalAppender, WalReadEntry, WalTailer}; +pub use write::SealFence; pub use write::ShardWriter; pub use write::ShardWriterConfig; pub use write::WriteResult; + +#[cfg(test)] +mod tests { + use super::*; + use arrow_schema::Fields; + + fn logical() -> ArrowSchema { + ArrowSchema::new(vec![ + ArrowField::new("id", DataType::Int32, false), + ArrowField::new("count", DataType::Int64, false), + ArrowField::new("note", DataType::Utf8, true), + ]) + } + + #[test] + fn relax_widens_every_non_pk_field_and_leaves_the_key_alone() { + let relaxed = relax_non_pk_nullability(&logical(), &["id".to_string()]); + + assert!( + !relaxed.field(0).is_nullable(), + "the primary key stays strict" + ); + assert!( + relaxed.field(1).is_nullable(), + "`count` must accept a tombstone null" + ); + assert!( + relaxed.field(2).is_nullable(), + "already-nullable is untouched" + ); + } + + #[test] + fn relax_leaves_nested_fields_exactly_as_declared() { + // Arrow validates nullability only at the top level, and a vector + // column's item field must not gain a validity layer. + let item = Arc::new(ArrowField::new("item", DataType::Float32, false)); + let child = ArrowField::new("a", DataType::Int32, false); + let schema = ArrowSchema::new(vec![ + ArrowField::new("id", DataType::Int32, false), + ArrowField::new("vector", DataType::FixedSizeList(item, 4), false), + ArrowField::new("s", DataType::Struct(Fields::from(vec![child])), false), + ]); + + let relaxed = relax_non_pk_nullability(&schema, &["id".to_string()]); + + assert!(relaxed.field(1).is_nullable()); + match relaxed.field(1).data_type() { + DataType::FixedSizeList(f, _) => assert!(!f.is_nullable(), "item field untouched"), + other => panic!("expected FixedSizeList, got {other:?}"), + } + match relaxed.field(2).data_type() { + DataType::Struct(fields) => assert!(!fields[0].is_nullable(), "child field untouched"), + other => panic!("expected Struct, got {other:?}"), + } + } + + #[test] + fn relax_keeps_tombstone_non_nullable_and_is_idempotent() { + let pk = ["id".to_string()]; + let once = relax_non_pk_nullability(&schema_with_tombstone(&logical()), &pk); + let twice = relax_non_pk_nullability(&once, &pk); + + let tombstone = once.field_with_name(TOMBSTONE).unwrap(); + assert!( + !tombstone.is_nullable(), + "the write path always populates _tombstone" + ); + assert_eq!(once, twice); + } + + #[test] + fn relax_preserves_schema_and_field_metadata() { + // The `lance-schema:unenforced-primary-key` marker rides on field + // metadata, so losing it here would silently drop the shard's PK. + let marked = ArrowField::new("count", DataType::Int64, false) + .with_metadata([("k".to_string(), "v".to_string())].into()); + let schema = ArrowSchema::new_with_metadata( + vec![ArrowField::new("id", DataType::Int32, false), marked], + [("s".to_string(), "m".to_string())].into(), + ); + + let relaxed = relax_non_pk_nullability(&schema, &["id".to_string()]); + + assert_eq!(relaxed.metadata().get("s").map(String::as_str), Some("m")); + assert_eq!( + relaxed.field(1).metadata().get("k").map(String::as_str), + Some("v") + ); + } +} diff --git a/rust/lance/src/dataset/mem_wal/api.rs b/rust/lance/src/dataset/mem_wal/api.rs index 597c65dce83..b5d0185c122 100644 --- a/rust/lance/src/dataset/mem_wal/api.rs +++ b/rust/lance/src/dataset/mem_wal/api.rs @@ -5,16 +5,24 @@ //! //! This module provides the user-facing API for initializing and using MemWAL //! on a Dataset. +//! +//! # Limitations +//! +//! MemWAL does not track dataset changes made after it is initialized: dropping +//! or replacing a maintained index, or projecting away its column, leaves +//! `maintained_indexes` naming something the writer cannot build. A change that +//! races the initialization commit lands the same way. Both surface as a failing +//! `mem_wal_writer`; handling them is follow-up work. use std::collections::HashMap; use std::sync::Arc; -use arrow_schema::DataType; +use arrow_schema::{DataType, Schema as ArrowSchema}; use async_trait::async_trait; +use lance_core::datatypes::Schema as LanceSchema; use lance_core::{Error, Result}; use lance_index::mem_wal::{MEM_WAL_INDEX_NAME, MemWalIndexDetails, ShardingField, ShardingSpec}; use lance_index::vector::hnsw::builder::HnswBuildParams; -use lance_io::object_store::ObjectStore; use uuid::Uuid; use crate::Dataset; @@ -25,8 +33,11 @@ use crate::index::DatasetIndexInternalExt; use crate::index::mem_wal::{load_mem_wal_index_details, new_mem_wal_index_meta}; use super::ShardWriterConfig; -use super::scanner::flushed_cache::open_flushed_dataset; +use super::index::{MemIndexKind, unsupported_index_type, validate_index_configs}; +use super::scanner::sstable_cache::open_sstable; use super::scanner::{DatasetCache, ShardSnapshot}; +use super::schema_with_tombstone; +use super::util::derived_store_params; use super::write::MemIndexConfig; use super::write::ShardWriter; @@ -143,9 +154,9 @@ impl<'a> InitializeMemWalBuilder<'a> { /// Set the base-table indexes to maintain in MemTables, replacing any /// previously set list. /// - /// Each name must reference an index that already exists on the dataset. - /// The primary key btree, when present, is maintained implicitly and must - /// not be listed. + /// Each name must reference an existing index the MemWAL can maintain; + /// [`execute`](Self::execute) enforces both. The primary key btree, when + /// present, is maintained implicitly and must not be listed. pub fn maintained_indexes(mut self, indexes: I) -> Self where I: IntoIterator, @@ -187,8 +198,12 @@ impl<'a> InitializeMemWalBuilder<'a> { /// Initialize MemWAL on the dataset, committing the MemWAL system index. /// - /// Fails if any maintained index does not exist, if the selected sharding - /// configuration is invalid, or if MemWAL is already initialized. + /// Fails if any maintained index does not exist or cannot be maintained by + /// the MemWAL, if the selected sharding configuration is invalid, or if + /// MemWAL is already initialized. + /// + /// Validated against the dataset as it stands here; see the module-level + /// limitations for changes made afterwards. pub async fn execute(self) -> Result<()> { let Self { dataset, @@ -201,20 +216,16 @@ impl<'a> InitializeMemWalBuilder<'a> { let (sharding_specs, num_shards) = resolve_sharding(dataset, sharding)?; let indices = dataset.load_indices().await?; - for index_name in &maintained_indexes { - if !indices.iter().any(|idx| &idx.name == index_name) { - return Err(Error::invalid_input(format!( - "Index '{}' not found on dataset. maintained_indexes must reference existing indexes.", - index_name - ))); - } - } if indices.iter().any(|idx| idx.name == MEM_WAL_INDEX_NAME) { return Err(Error::invalid_input( "MemWAL is already initialized on this dataset.", )); } + // Gate the commit, not just a preflight a caller may skip: a set the + // writer cannot open leaves the table unwritable. + validate_maintained_indexes(dataset, &maintained_indexes).await?; + let details = MemWalIndexDetails { num_shards, sharding_specs, @@ -392,10 +403,6 @@ fn writer_config_to_defaults(config: &ShardWriterConfig) -> HashMap HashMap Result<()> { + Ok(()) + } + /// List current MemWAL shard IDs from object storage directory listing. async fn list_mem_wal_latest_shard_ids(&self) -> Result> { Ok(Vec::new()) } - /// Prewarm the flushed generations of the given MemWAL shards into this + /// Prewarm the SSTables of the given MemWAL shards into this /// dataset's session caches. /// - /// For every flushed generation in `snapshots`, opens the generation's + /// For every SSTable in `snapshots`, opens the generation's /// on-disk dataset (populating the session's metadata/index caches, and the /// optional `cache` of opened `Arc`s) and prewarms each of its /// indexes. Opens run concurrently. @@ -566,6 +579,30 @@ impl DatasetMemWalExt for Dataset { load_mem_wal_index_details(index_meta).map(Some) } + async fn require_mem_wal_index_catchup(&mut self) -> Result<()> { + if self.load_index_by_name(MEM_WAL_INDEX_NAME).await?.is_none() { + return Err(Error::invalid_input( + "Cannot require MemWAL index catch-up: MemWAL is not initialized on \ + this dataset.", + )); + } + + let transaction = Transaction::new( + self.manifest.version, + Operation::UpdateMemWalState { + compacted_sstables: Vec::new(), + require_index_catchup: true, + }, + None, + ); + // Assigned back: leaving the receiver on the pre-activation manifest + // would report success while `self` still reads as legacy. + *self = CommitBuilder::new(Arc::new(self.clone())) + .execute(transaction) + .await?; + Ok(()) + } + async fn list_mem_wal_latest_shard_ids(&self) -> Result> { let prefix = super::util::mem_wal_path(&self.branch_location().path); let object_store = self.object_store(None).await?; @@ -597,7 +634,9 @@ impl DatasetMemWalExt for Dataset { cache: Option<&Arc>, ) -> Result<()> { let session = self.session(); - // Resolve flushed paths exactly as the LSM collector does, so the + // Every open below targets a generation URI, never the base's own. + let store_params = self.store_params().map(derived_store_params); + // Resolve SSTable paths exactly as the LSM collector does, so the // session/cache entries we warm key-match the paths later lookups open. let base_path = self.uri().trim_end_matches('/').to_string(); let opens = snapshots @@ -606,11 +645,13 @@ impl DatasetMemWalExt for Dataset { let shard_id = snapshot.shard_id; let base_path = &base_path; let session = &session; - snapshot.flushed_generations.iter().map(move |flushed| { - let path = format!("{}/_mem_wal/{}/{}", base_path, shard_id, flushed.path); + let store_params = &store_params; + snapshot.sstables.iter().map(move |sstable| { + let path = format!("{}/_mem_wal/{}/{}", base_path, shard_id, sstable.path); async move { let dataset = - open_flushed_dataset(&path, Some(session), cache, None).await?; + open_sstable(&path, Some(session), store_params.as_ref(), cache, None) + .await?; prewarm_all_indexes(&dataset).await } }) @@ -640,69 +681,23 @@ impl DatasetMemWalExt for Dataset { // Get maintained_indexes from the MemWalIndex details let maintained_indexes = &mem_wal_index.details.maintained_indexes; - // Load index configs for each maintained index - let mut index_configs = Vec::new(); - for index_name in maintained_indexes { - // A maintained index can split into multiple physical segments - // (e.g. `optimize_indices(append)` deltas), which the singular - // `load_index_by_name` rejects. Every segment carries the same - // type and params, so take the first match. - let index_meta = self - .load_indices_by_name(index_name) - .await? - .into_iter() - .next() - .ok_or_else(|| { - Error::invalid_input(format!( - "Index '{}' from maintained_indexes not found on dataset", - index_name - )) - })?; - - // Detect index type and create appropriate config - let type_url = index_meta - .index_details - .as_ref() - .map(|d| d.type_url.as_str()) - .unwrap_or(""); - - let index_type = MemIndexConfig::detect_index_type(type_url)?; - - match index_type { - "btree" => { - index_configs.push(MemIndexConfig::btree_from_metadata( - &index_meta, - self.schema(), - )?); - } - "fts" => { - index_configs.push(MemIndexConfig::fts_from_metadata( - &index_meta, - self.schema(), - )?); - } - "vector" => { - let hnsw_params = config.hnsw_params.get(index_name).cloned(); - let vector_config = - load_vector_index_config(self, index_name, &index_meta, hnsw_params) - .await?; - index_configs.push(vector_config); - } - _ => { - return Err(Error::invalid_input(format!( - "Unknown index type: {}", - index_type - ))); - } - }; - } + let index_configs = + build_index_configs(self, maintained_indexes, &config.hnsw_params).await?; // Set shard_id in config config.shard_id = shard_id; - // Get object store and base path + // Inject the dataset's store params + session so the flusher opens the + // base + generations with the same store the base was resolved with. + config.store_params = self.store_params().cloned(); + config.session = Some(self.session()); + + // Reuse the dataset's own object store + base path; `ObjectStore::from_uri` + // would discard the store params the dataset was opened with, signing WAL + // writes with the ambient identity. Mirrors `list_mem_wal_latest_shard_ids`. let base_uri = self.uri(); - let (store, base_path) = ObjectStore::from_uri(base_uri).await?; + let store = self.object_store(None).await?; + let base_path = self.branch_location().path; // Create ShardWriter ShardWriter::open( @@ -717,6 +712,95 @@ impl DatasetMemWalExt for Dataset { } } +/// Build the in-memory index configurations for `index_names`. +/// +/// Shared by [`DatasetMemWalExt::mem_wal_writer`] and +/// [`validate_maintained_indexes`], so a set that validates is one the writer +/// can build. +async fn build_index_configs( + dataset: &Dataset, + index_names: &[String], + hnsw_params: &HashMap, +) -> Result> { + let mut index_configs = Vec::with_capacity(index_names.len()); + for index_name in index_names { + // A maintained index can split into multiple physical segments + // (e.g. `optimize_indices(append)` deltas), which the singular + // `load_index_by_name` rejects. Every segment carries the same + // type and params, so take the first match. + let index_meta = dataset + .load_indices_by_name(index_name) + .await? + .into_iter() + .next() + .ok_or_else(|| { + Error::invalid_input(format!( + "Index '{}' from maintained_indexes not found on dataset", + index_name + )) + })?; + + // Detect index kind and create appropriate config + let type_url = index_meta + .index_details + .as_ref() + .map(|d| d.type_url.as_str()) + .unwrap_or(""); + + let kind = MemIndexKind::from_type_url(type_url) + .ok_or_else(|| unsupported_index_type(index_name, type_url))?; + + // Exhaustive: a new kind must be built here, or a maintained set could + // name an index this writer cannot open, failing every memtable claim. + index_configs.push(match kind { + MemIndexKind::BTree => { + MemIndexConfig::btree_from_metadata(&index_meta, dataset.schema())? + } + MemIndexKind::Fts => MemIndexConfig::fts_from_metadata(&index_meta, dataset.schema())?, + MemIndexKind::Hnsw => { + let hnsw_params = hnsw_params.get(index_name).cloned(); + load_vector_index_config(dataset, index_name, &index_meta, hnsw_params).await? + } + }); + } + Ok(index_configs) +} + +/// Whether the MemWAL can maintain `index_names` on `dataset`. +/// +/// Applies the same rules [`ShardWriter::open`] does, so a set that passes here +/// is a set the writer can open. [`InitializeMemWalBuilder::execute`] runs it +/// before committing; it is public so a caller inferring a set can ask the same +/// question first. A type url alone cannot decide this — every vector sub-type +/// maps to [`MemIndexKind::Hnsw`], but the memtable's HNSW needs a +/// `FixedSizeList` column. +/// +/// All-or-nothing: it reports the first index it cannot maintain rather than +/// returning a usable subset, so a caller inferring a set surfaces the error +/// instead of dropping an index it believes is maintained. +/// +/// Judges `dataset` as given; see the module-level limitations. +/// +/// Opens each vector index to inherit its distance type. +pub async fn validate_maintained_indexes(dataset: &Dataset, index_names: &[String]) -> Result<()> { + // Validation reads an index's name, column, and field id, never its HNSW + // tuning, so the writer's build params are not needed here. + let index_configs = build_index_configs(dataset, index_names, &HashMap::new()).await?; + + // The shard schema is base + `_tombstone`, as `ShardWriter::open` extends + // it; field ids and the primary key resolve against that, not the base. + let base_schema: ArrowSchema = dataset.schema().into(); + let schema = schema_with_tombstone(&base_schema); + let lance_schema = LanceSchema::try_from(schema.as_ref())?; + let pk_columns: Vec = lance_schema + .unenforced_primary_key() + .iter() + .map(|field| field.name.clone()) + .collect(); + + validate_index_configs(&index_configs, schema.as_ref(), &lance_schema, &pk_columns) +} + /// Build an in-memory HNSW vector index configuration from a base-table /// vector index entry. /// @@ -771,7 +855,7 @@ async fn load_vector_index_config( #[cfg(test)] mod tests { - use super::super::scanner::FlushedMemTableCache; + use super::super::scanner::SsTableCache; use super::*; use arrow_array::{Int32Array, RecordBatch, RecordBatchIterator}; @@ -788,6 +872,76 @@ mod tests { ])) } + /// A dataset of 256 rows with an IVF vector index `vector_idx` over a + /// `FixedSizeList` column. + async fn dataset_with_vector_index(uri: &str, item_type: DataType) -> Dataset { + use crate::index::vector::VectorIndexParams; + use arrow_array::ArrayRef; + use arrow_array::builder::{FixedSizeListBuilder, Float32Builder, Float64Builder}; + use lance_linalg::distance::DistanceType; + + const ROWS: i32 = 256; + const DIM: i32 = 4; + + let schema = Arc::new(ArrowSchema::new(vec![ + Field::new("id", DataType::Int32, false), + Field::new( + "vector", + DataType::FixedSizeList(Arc::new(Field::new("item", item_type.clone(), true)), DIM), + true, + ), + ])); + + let vectors: ArrayRef = match item_type { + DataType::Float32 => { + let mut builder = FixedSizeListBuilder::new(Float32Builder::new(), DIM); + for row in 0..ROWS { + for d in 0..DIM { + builder.values().append_value((row * DIM + d) as f32); + } + builder.append(true); + } + Arc::new(builder.finish()) + } + DataType::Float64 => { + let mut builder = FixedSizeListBuilder::new(Float64Builder::new(), DIM); + for row in 0..ROWS { + for d in 0..DIM { + builder.values().append_value((row * DIM + d) as f64); + } + builder.append(true); + } + Arc::new(builder.finish()) + } + other => panic!("unhandled vector item type {other:?}"), + }; + + let batch = RecordBatch::try_new( + schema.clone(), + vec![ + Arc::new(Int32Array::from((0..ROWS).collect::>())), + vectors, + ], + ) + .unwrap(); + + let reader = RecordBatchIterator::new([Ok(batch)], schema.clone()); + let mut dataset = Dataset::write(reader, uri, Some(WriteParams::default())) + .await + .unwrap(); + dataset + .create_index( + &["vector"], + IndexType::Vector, + Some("vector_idx".to_string()), + &VectorIndexParams::ivf_flat(1, DistanceType::L2), + true, + ) + .await + .unwrap(); + dataset + } + fn id_v_batch(schema: &Arc, ids: &[i32]) -> RecordBatch { let vs: Vec = ids.iter().map(|i| i * 10).collect(); RecordBatch::try_new( @@ -800,11 +954,183 @@ mod tests { .unwrap() } + #[tokio::test] + async fn test_validate_maintained_indexes_rejects_non_f32_vector_column() { + // A `FixedSizeList` vector index is a valid durable index whose + // type url resolves to `Hnsw` like any other, but the memtable HNSW needs + // Float32 — so committing it would leave the table unwritable. + let tmp = tempfile::tempdir().unwrap(); + let uri = format!("{}/base", tmp.path().to_str().unwrap()); + let dataset = dataset_with_vector_index(&uri, DataType::Float64).await; + + let index_meta = dataset + .load_indices_by_name("vector_idx") + .await + .unwrap() + .into_iter() + .next() + .unwrap(); + assert_eq!( + MemIndexKind::from_type_url( + index_meta.index_details.as_ref().unwrap().type_url.as_str() + ), + Some(MemIndexKind::Hnsw), + "the type url cannot see the column type" + ); + + let error = validate_maintained_indexes(&dataset, &["vector_idx".to_string()]) + .await + .expect_err("a Float64 vector column is not maintainable"); + assert!( + error.to_string().contains("FixedSizeList"), + "unexpected error: {error}" + ); + } + + #[tokio::test] + async fn test_validate_maintained_indexes_accepts_f32_vector_column() { + let tmp = tempfile::tempdir().unwrap(); + let uri = format!("{}/base", tmp.path().to_str().unwrap()); + let dataset = dataset_with_vector_index(&uri, DataType::Float32).await; + + validate_maintained_indexes(&dataset, &["vector_idx".to_string()]) + .await + .expect("a Float32 vector column is maintainable"); + } + + #[tokio::test] + async fn test_validate_maintained_indexes_accepts_btree() { + // Guards the shard-schema plumbing: validation resolves field ids against + // base + `_tombstone`, so a scalar index on an ordinary column must pass. + let tmp = tempfile::tempdir().unwrap(); + let uri = format!("{}/base", tmp.path().to_str().unwrap()); + let schema = id_v_schema(); + let reader = + RecordBatchIterator::new([Ok(id_v_batch(&schema, &[1, 2, 3]))], schema.clone()); + let mut dataset = Dataset::write(reader, &uri, Some(WriteParams::default())) + .await + .unwrap(); + dataset + .create_index( + &["id"], + IndexType::BTree, + Some("id_idx".to_string()), + &ScalarIndexParams::default(), + true, + ) + .await + .unwrap(); + + validate_maintained_indexes(&dataset, &["id_idx".to_string()]) + .await + .expect("a BTree index on an Int32 column is maintainable"); + } + + #[tokio::test] + async fn test_validate_maintained_indexes_rejects_unmaintainable_kind() { + // A bitmap index is a valid durable index the memtable cannot build. + // The error names it, so a caller validating a set knows which to drop. + let tmp = tempfile::tempdir().unwrap(); + let uri = format!("{}/base", tmp.path().to_str().unwrap()); + let schema = id_v_schema(); + let reader = + RecordBatchIterator::new([Ok(id_v_batch(&schema, &[1, 2, 3]))], schema.clone()); + let mut dataset = Dataset::write(reader, &uri, Some(WriteParams::default())) + .await + .unwrap(); + dataset + .create_index( + &["v"], + IndexType::Bitmap, + Some("v_bitmap".to_string()), + &ScalarIndexParams::for_builtin(lance_index::scalar::BuiltinIndexType::Bitmap), + true, + ) + .await + .unwrap(); + + let error = validate_maintained_indexes(&dataset, &["v_bitmap".to_string()]) + .await + .expect_err("the memtable cannot build a bitmap index"); + assert!( + error.to_string().contains("v_bitmap"), + "the error must name the index: {error}" + ); + } + + #[tokio::test] + async fn test_validate_maintained_indexes_rejects_unknown_name() { + let tmp = tempfile::tempdir().unwrap(); + let uri = format!("{}/base", tmp.path().to_str().unwrap()); + let schema = id_v_schema(); + let reader = RecordBatchIterator::new([Ok(id_v_batch(&schema, &[1]))], schema.clone()); + let dataset = Dataset::write(reader, &uri, Some(WriteParams::default())) + .await + .unwrap(); + + let error = validate_maintained_indexes(&dataset, &["nope".to_string()]) + .await + .expect_err("an index that does not exist cannot be maintained"); + assert!( + error.to_string().contains("not found"), + "unexpected: {error}" + ); + } + + #[tokio::test] + async fn test_initialize_mem_wal_rejects_unmaintainable_index() { + // Initialization persists the set, so it must apply the writer's rules + // itself: a Float64 vector index committed here leaves the table unwritable. + let tmp = tempfile::tempdir().unwrap(); + let uri = format!("{}/base", tmp.path().to_str().unwrap()); + let mut dataset = dataset_with_vector_index(&uri, DataType::Float64).await; + + let error = dataset + .initialize_mem_wal() + .unsharded() + .maintained_indexes(["vector_idx"]) + .execute() + .await + .expect_err("a Float64 vector column is not maintainable"); + assert!( + error.to_string().contains("FixedSizeList"), + "unexpected error: {error}" + ); + assert!( + dataset.mem_wal_index_details().await.unwrap().is_none(), + "a rejected maintained set must not be committed" + ); + } + + #[tokio::test] + async fn test_initialize_mem_wal_rejects_unknown_index_name() { + let tmp = tempfile::tempdir().unwrap(); + let uri = format!("{}/base", tmp.path().to_str().unwrap()); + let schema = id_v_schema(); + let reader = RecordBatchIterator::new([Ok(id_v_batch(&schema, &[1]))], schema.clone()); + let mut dataset = Dataset::write(reader, &uri, Some(WriteParams::default())) + .await + .unwrap(); + + let error = dataset + .initialize_mem_wal() + .unsharded() + .maintained_indexes(["nope"]) + .execute() + .await + .expect_err("maintained_indexes must reference existing indexes"); + assert!( + error.to_string().contains("nope") && error.to_string().contains("not found"), + "unexpected error: {error}" + ); + assert!(dataset.mem_wal_index_details().await.unwrap().is_none()); + } + #[tokio::test] async fn test_prewarm_mem_wal_opens_and_warms_indexes() { - // `prewarm_mem_wal` opens each flushed generation (into the base + // `prewarm_mem_wal` opens each SSTable (into the base // dataset's session + the supplied cache) and warms its indexes. We - // place a flushed-generation dataset with a BTree index at the + // place an SSTable dataset with a BTree index at the // canonical `{base}/_mem_wal/{shard}/{folder}` path, prewarm it via a // snapshot, and assert the generation is cached and its index loadable. let tmp = tempfile::tempdir().unwrap(); @@ -817,7 +1143,7 @@ mod tests { .await .unwrap(); - // Flushed generation with a BTree index on `id`. + // SSTable with a BTree index on `id`. let shard_id = Uuid::new_v4(); let folder = "deadbeef_gen_1"; let gen_uri = format!("{}/_mem_wal/{}/{}", base_uri, shard_id, folder); @@ -839,9 +1165,9 @@ mod tests { let snapshot = ShardSnapshot::new(shard_id) .with_current_generation(2) - .with_flushed_generation(1, folder.to_string()); + .with_sstable(1, folder.to_string()); - let cache: Arc = Arc::new(FlushedMemTableCache::new(4)); + let cache: Arc = Arc::new(SsTableCache::new(4)); base.prewarm_mem_wal(std::slice::from_ref(&snapshot), Some(&cache)) .await .expect("prewarm must open the generation and warm its index"); @@ -849,7 +1175,7 @@ mod tests { // The generation is resident in the cache (same session), with its // index loadable — a later lookup that opens this path is a pure hit. let warmed = cache - .get_or_open(&gen_uri, Some(base.session())) + .get_or_open(&gen_uri, Some(base.session()), base.store_params().cloned()) .await .unwrap(); assert_eq!(warmed.load_indices().await.unwrap().len(), 1); @@ -857,7 +1183,7 @@ mod tests { #[tokio::test] async fn test_prewarm_mem_wal_empty_is_noop() { - // No snapshots / no flushed generations: prewarm is a clean no-op. + // No snapshots / no SSTables: prewarm is a clean no-op. let tmp = tempfile::tempdir().unwrap(); let base_uri = format!("{}/base", tmp.path().to_str().unwrap()); let schema = id_v_schema(); diff --git a/rust/lance/src/dataset/mem_wal/hnsw/graph.rs b/rust/lance/src/dataset/mem_wal/hnsw/graph.rs index 4009557702c..570122581b7 100644 --- a/rust/lance/src/dataset/mem_wal/hnsw/graph.rs +++ b/rust/lance/src/dataset/mem_wal/hnsw/graph.rs @@ -594,9 +594,12 @@ impl HnswGraph { } fn validate_source(&self, vectors: &impl VectorSource, needed_len: usize) -> Result<()> { + // Not caller input: the graph was sized below what the memtable holds. + // See the matching note in `storage.rs::append_batch`. if needed_len > self.nodes.len() { - return Err(Error::invalid_input(format!( - "graph capacity {} exhausted: need {needed_len}", + return Err(Error::internal(format!( + "HNSW graph capacity {} exhausted: need {needed_len}; \ + the graph is sized below the memtable's row capacity", self.nodes.len() ))); } diff --git a/rust/lance/src/dataset/mem_wal/hnsw/storage.rs b/rust/lance/src/dataset/mem_wal/hnsw/storage.rs index bbeb57a5fe2..c0ee9d57ff6 100644 --- a/rust/lance/src/dataset/mem_wal/hnsw/storage.rs +++ b/rust/lance/src/dataset/mem_wal/hnsw/storage.rs @@ -269,25 +269,30 @@ impl ArrowFixedSizeListVectorStore { ))); }; + // Exhaustion is a shard-construction bug (store sized below the memtable), + // not caller input — hence `internal`, not `invalid_input`. let start = self.committed_len.load(Ordering::Relaxed); let end = start.checked_add(num_rows).ok_or_else(|| { - Error::invalid_input(format!( + Error::internal(format!( "vector count overflow: start={}, batch_len={}", start, num_rows )) })?; if end > self.capacity { - return Err(Error::invalid_input(format!( - "capacity {} exhausted: inserting rows [{}..{})", + return Err(Error::internal(format!( + "HNSW vector store capacity {} exhausted: inserting rows [{}..{}); \ + the store is sized below the memtable's row capacity", self.capacity, start, end ))); } let batch_idx = self.committed_batches.load(Ordering::Relaxed); if batch_idx >= self.max_batches { - return Err(Error::invalid_input(format!( - "max_batches {} exhausted", - self.max_batches + return Err(Error::internal(format!( + "HNSW vector store max_batches {} exhausted at batch_idx {} \ + (inserting rows [{}..{})); the store is sized below the \ + memtable's batch capacity", + self.max_batches, batch_idx, start, end ))); } diff --git a/rust/lance/src/dataset/mem_wal/index.rs b/rust/lance/src/dataset/mem_wal/index.rs index d16d3105551..b9c1af446c1 100644 --- a/rust/lance/src/dataset/mem_wal/index.rs +++ b/rust/lance/src/dataset/mem_wal/index.rs @@ -23,16 +23,18 @@ mod pk_key; use std::collections::HashMap; use std::sync::Arc; use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; +use std::time::Instant; use datafusion::common::ScalarValue; use super::memtable::batch_store::StoredBatch; +use super::wal::WriterCursors; use arrow_array::RecordBatch; +use arrow_schema::{DataType, Schema as ArrowSchema}; use lance_core::datatypes::Schema as LanceSchema; use lance_core::{Error, Result}; use lance_index::pbold; use lance_index::scalar::InvertedIndexParams; -use lance_index::scalar::inverted::InvertedListFormatVersion; use lance_index::vector::hnsw::builder::HnswBuildParams; use lance_linalg::distance::DistanceType; use lance_table::format::IndexMetadata; @@ -58,6 +60,19 @@ use pk_key::encode_pk_batch; /// [`BTreeMemIndex`]'s byte backend indexes it directly. const PK_KEY_COLUMN: &str = "__pk_key__"; +/// Row count at or below which [`IndexStore::insert_batches`] indexes inline +/// rather than spawning a thread per index. +/// +/// The spawn is one OS thread *per index* — tens of microseconds each, and a table can +/// carry several BTrees alongside its HNSW and FTS — so for a small batch it costs more +/// than the indexing it parallelizes. Small batches are not the exceptional case: a +/// durable put triggers a WAL flush covering only the batch it just inserted, so this +/// path is routinely called with a single short batch. +/// +/// The crossover depends on per-row HNSW cost, which varies with dimension and +/// `ef_construction`; tune against `benches/mem_wal/vector/mem_wal_index_micro.rs`. +const PARALLEL_INDEX_MIN_ROWS: usize = 64; + /// The memtable's primary-key index, used to answer "newest visible version of /// this key" for dedup. Single-column PKs reuse the column's compact typed /// [`BTreeMemIndex`] (no second copy); composite PKs key a `BTreeMemIndex` on @@ -80,9 +95,232 @@ enum PkIndex { // Index Store // ============================================================================ -/// Configuration for an index in MemWAL. +/// Validate every configured in-memory index, and the composite primary key, +/// against the shard schema. Call once at shard open, before any write can land. +/// +/// This is what makes poison-and-replay *terminating*. An index insert that +/// fails deterministically on a row that is already WAL-durable cannot be +/// recovered from: the writer poisons, the operator reopens, replay re-reads the +/// same WAL rows, the same insert fails again, and `open()` propagates it — a +/// shard that never comes back. Every such failure is an index *config* +/// disagreeing with the schema, never a property of the data, so one pass here +/// closes the whole class before a single row is accepted. +/// +/// The data-dependent errors inside the index layer are already unreachable +/// through `put`: `MemTable::insert_batches_only` does a full `Arc` +/// equality check, so a batch that would trip one is rejected before it reaches +/// the batch store, let alone the WAL. +/// +/// It also rejects a config whose `field_id` names a different column than its +/// `column`. Index *selection* keys off `field_id` — a single-column PK reuses +/// the BTree whose `field_id` matches its key — so a config resolved only by name +/// could be bound under the wrong identity, serving stale reads and flushing the +/// wrong column into the durable PK sidecar. `lance_schema` supplies the +/// authoritative name→id mapping. +pub fn validate_index_configs( + configs: &[MemIndexConfig], + schema: &ArrowSchema, + lance_schema: &LanceSchema, + pk_columns: &[String], +) -> Result<()> { + for config in configs { + let column = config.column(); + if let MemIndexConfig::Fts(config) = config { + let resolved = crate::index::scalar::inverted::resolve_fts_field( + lance_schema, + column, + config.params.get_document_granularity(), + ) + .map_err(|error| { + Error::invalid_input(format!( + "FTS index '{}' is invalid for field path '{}': {error}", + config.name, column + )) + })?; + if resolved.final_field_id != config.field_id { + return Err(Error::invalid_input(format!( + "index '{}' is configured with field_id {} but its field path '{}' has \ + final field_id {} in the shard schema", + config.name, config.field_id, column, resolved.final_field_id, + ))); + } + continue; + } + + let field = schema.field_with_name(column).map_err(|_| { + Error::invalid_input(format!( + "index '{}' is configured on column '{}', which is not in the shard schema; \ + available columns: [{}]", + config.name(), + column, + schema + .fields() + .iter() + .map(|f| f.name().as_str()) + .collect::>() + .join(", ") + )) + })?; + + match config { + // BTree falls back to per-row `ScalarValue` extraction, so it + // accepts any column type the schema can hold. Existence is the + // only precondition. + MemIndexConfig::BTree(_) => {} + MemIndexConfig::Fts(_) => unreachable!("FTS configs are validated by schema path"), + MemIndexConfig::Hnsw(_) => match field.data_type() { + DataType::FixedSizeList(item, dim) => { + if item.data_type() != &DataType::Float32 { + return Err(Error::invalid_input(format!( + "HNSW index '{}' requires a FixedSizeList column; \ + column '{}' has item type {:?}", + config.name(), + column, + item.data_type() + ))); + } + // `HnswMemIndex.dim` is a placeholder until the first batch + // pins it (`hnsw.rs`), so a zero-width vector would only + // surface at insert time — i.e. on already-durable data. + if *dim <= 0 { + return Err(Error::invalid_input(format!( + "HNSW index '{}' requires a vector dimension > 0; column '{}' has \ + dimension {dim}", + config.name(), + column, + ))); + } + } + other => { + return Err(Error::invalid_input(format!( + "HNSW index '{}' requires a FixedSizeList column; \ + column '{}' is {:?}", + config.name(), + column, + other + ))); + } + }, + } + + // The column resolves, but index selection keys off `field_id`, not name. + // A config whose `field_id` identifies a *different* column would be bound + // under the wrong identity (e.g. reused as the single-column PK index), so + // reject any `field_id` that does not name the resolved column. + let resolved_field_id = lance_schema + .field(column) + .ok_or_else(|| { + Error::invalid_input(format!( + "index '{}' is configured on column '{}', which is present in the Arrow \ + schema but absent from the Lance schema", + config.name(), + column, + )) + })? + .id; + if resolved_field_id != config.field_id() { + return Err(Error::invalid_input(format!( + "index '{}' is configured with field_id {} but its column '{}' has field_id {} \ + in the shard schema", + config.name(), + config.field_id(), + column, + resolved_field_id, + ))); + } + } + + // Every PK column must exist in the schema. A single-column PK aliases a + // BTree entry (any type); only a *composite* PK builds an order-preserving + // encoded key, and only some types encode. + for column in pk_columns { + let field = schema.field_with_name(column).map_err(|_| { + Error::invalid_input(format!( + "primary-key column '{column}' is not in the shard schema" + )) + })?; + if pk_columns.len() > 1 && !is_encodable_pk_type(field.data_type()) { + return Err(Error::invalid_input(format!( + "composite primary-key column '{column}' has type {:?}, which has no \ + order-preserving key encoding", + field.data_type() + ))); + } + } + + Ok(()) +} + +/// Types `pk_key::encode_value` can encode into an order-preserving composite key. +fn is_encodable_pk_type(data_type: &DataType) -> bool { + matches!( + data_type, + DataType::Int8 + | DataType::Int16 + | DataType::Int32 + | DataType::Int64 + | DataType::UInt8 + | DataType::UInt16 + | DataType::UInt32 + | DataType::UInt64 + | DataType::Date32 + | DataType::Date64 + | DataType::Boolean + | DataType::Utf8 + | DataType::LargeUtf8 + | DataType::Binary + | DataType::LargeBinary + | DataType::FixedSizeBinary(_) + ) +} + +/// The index kinds a MemTable can maintain — the registry of MemWAL index +/// support. Data-free because indexes are identified by type url before any +/// [`MemIndexConfig`] exists. +/// +/// Adding a variant is a compile error in [`details_suffix`](Self::details_suffix), +/// `MemIndexConfig::kind`, and `Dataset::mem_wal_writer` until each handles it. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub enum MemIndexKind { + /// BTree index for scalar fields (point lookups, range queries). + BTree, + /// HNSW vector index built incrementally, queryable while building. + Hnsw, + /// Full-text search index. + Fts, +} + +impl MemIndexKind { + /// Every maintainable kind. A kind missing here is never detected, so it + /// goes unmaintained rather than reaching a memtable that cannot build it. + pub const ALL: &'static [Self] = &[Self::BTree, Self::Hnsw, Self::Fts]; + + /// Suffix of the protobuf details message identifying this kind. + /// + /// Only the suffix: the prefix varies by dataset version + /// (`/lance.table.`, `/lance.index.pb.`, and the `type.googleapis.com/` + /// form MemWAL flush once wrote), and all must resolve. + pub const fn details_suffix(self) -> &'static str { + match self { + Self::BTree => "BTreeIndexDetails", + Self::Hnsw => "VectorIndexDetails", + Self::Fts => "InvertedIndexDetails", + } + } + + /// The kind a base-table index of this protobuf type maps to, or `None` + /// when a memtable cannot maintain it. + pub fn from_type_url(type_url: &str) -> Option { + Self::ALL + .iter() + .copied() + .find(|kind| type_url.ends_with(kind.details_suffix())) + } +} + +/// Configuration for an index in MemWAL. Pairs 1:1 with [`MemIndexKind`] via +/// [`kind`](Self::kind). /// -/// Each variant contains all the configuration needed for that index type. /// `Hnsw` is boxed because `HnswBuildParams` is small but the variant may /// grow with future config (e.g. shard-specific tuning). #[derive(Debug, Clone)] @@ -96,6 +334,16 @@ pub enum MemIndexConfig { } impl MemIndexConfig { + /// The kind this config builds. Links the config enum to the registry, so + /// a new variant must declare its kind. + pub const fn kind(&self) -> MemIndexKind { + match self { + Self::BTree(_) => MemIndexKind::BTree, + Self::Hnsw(_) => MemIndexKind::Hnsw, + Self::Fts(_) => MemIndexKind::Fts, + } + } + /// Get the index name. pub fn name(&self) -> &str { match self { @@ -135,26 +383,37 @@ impl MemIndexConfig { /// Create an FTS index config from base table IndexMetadata. pub fn fts_from_metadata(index_meta: &IndexMetadata, schema: &LanceSchema) -> Result { - let (field_id, column) = Self::extract_field_info(index_meta, schema)?; + let (field_id, _) = Self::extract_field_info(index_meta, schema)?; // Extract InvertedIndexParams from index_details if available - let params = if let Some(details_any) = &index_meta.index_details { - if let Ok(details) = pbold::InvertedIndexDetails::decode(details_any.value.as_slice()) { - InvertedIndexParams::try_from(&details)? - } else { - InvertedIndexParams::default() - } + let details = if let Some(details_any) = &index_meta.index_details { + pbold::InvertedIndexDetails::decode(details_any.value.as_slice()).map_err(|err| { + Error::io(format!( + "failed to decode InvertedIndexDetails for MemWAL FTS index '{}': {}", + index_meta.name, err + )) + })? } else { - InvertedIndexParams::default() + pbold::InvertedIndexDetails::default() }; - let params = params.format_version(Self::fts_format_version_from_metadata(index_meta)?); - - Ok(Self::Fts(FtsIndexConfig::with_params( - index_meta.name.clone(), + let details = + crate::index::scalar::inverted::normalize_inverted_details(index_meta, details)?; + let params = InvertedIndexParams::try_from(&details)?; + let resolved = crate::index::scalar::inverted::resolve_fts_field_by_id( + schema, field_id, - column, - params, - ))) + params.get_document_granularity(), + )?; + + Ok(Self::Fts( + FtsIndexConfig::try_with_params( + index_meta.name.clone(), + field_id, + resolved.canonical_path.clone(), + params, + )? + .with_resolved_field(resolved), + )) } /// Create an HNSW vector index config. @@ -181,37 +440,6 @@ impl MemIndexConfig { )) } - /// Detect index type from protobuf type_url. - pub fn detect_index_type(type_url: &str) -> Result<&'static str> { - if type_url.ends_with("BTreeIndexDetails") { - Ok("btree") - } else if type_url.ends_with("InvertedIndexDetails") { - Ok("fts") - } else if type_url.ends_with("VectorIndexDetails") { - Ok("vector") - } else { - Err(Error::invalid_input(format!( - "Unsupported index type for MemWAL: {}. Supported: BTree, Inverted, Vector", - type_url - ))) - } - } - - fn fts_format_version_from_metadata( - index_meta: &IndexMetadata, - ) -> Result { - match index_meta.index_version { - // Legacy Arrow FTS indexes did not use the v1/v2 metadata values, but - // the maintained-index path can only write the modern format. - 0 | 1 => Ok(InvertedListFormatVersion::V1), - 2 => Ok(InvertedListFormatVersion::V2), - version => Err(Error::invalid_input(format!( - "FTS index '{}' has unsupported index_version {}; expected 0, 1, or 2", - index_meta.name, version - ))), - } - } - /// Extract field ID and column name from index metadata. fn extract_field_info( index_meta: &IndexMetadata, @@ -232,14 +460,23 @@ impl MemIndexConfig { } } +/// Names the index, not just its type: a caller validating a maintained set +/// needs to know which one to drop. +pub(crate) fn unsupported_index_type(index_name: &str, type_url: &str) -> Error { + Error::invalid_input(format!( + "index '{}' has type {}, which the MemWAL cannot maintain. Supported: BTree, Inverted, Vector", + index_name, type_url + )) +} + /// Registry managing all in-memory indexes for a MemTable. /// /// Indexes are keyed by index name. Each index stores its field_id for /// stable column-to-index resolution (column name → field_id → index). /// -/// The store also carries the MemTable's `max_visible_batch_position` -/// watermark — the highest batch position that is durable in the WAL and -/// therefore safe for scanners to read. Scanners snapshot this at plan +/// The store also carries the MemTable's two cursors: `indexed_count` (what the +/// index layer has ingested) and `visible_count` (what is indexed *and* durable, +/// and therefore safe for scanners to read). Scanners snapshot the latter at plan /// construction time so every plan keys on a stable MVCC cursor. pub struct IndexStore { /// BTree indexes keyed by index name. `Arc` so the primary-key BTrees can be @@ -253,10 +490,23 @@ pub struct IndexStore { /// primary key. Queried via [`Self::pk_newest_visible`] (see /// [`Self::enable_pk_index`]). pk_index: Option, - /// Maximum batch position that is durable in the WAL and therefore - /// visible to scanners. Advanced unconditionally after a WAL append - /// succeeds; not gated on whether any indexes are configured. - max_visible_batch_position: AtomicUsize, + /// How many batches of this memtable have been fully indexed. An exclusive + /// count: 0 means none. + /// + /// This has only ever been an *indexed* cursor — it is advanced at the end of + /// `insert_batches`, once every index insert for the batch has completed, and + /// never before. It was named `max_visible_batch_position` and treated as a + /// visibility cursor by five read sites, which is how rows became readable + /// before they were durable. Publishing is a separate step, and it is the + /// writer's to make — see `visible_count`. + indexed_count: AtomicUsize, + + /// The writer's cursors, and this memtable's coordinate within them. `None` + /// for a bare `IndexStore` (tests, benches), where visibility is just the + /// indexed prefix. + /// + /// Visibility is **derived, never stored**: see `visible_count`. + durability: Option<(Arc, usize)>, /// Conservative flag set once this memtable has observed any primary-key /// rewrite while maintaining a search index. Search planners can push top-k /// into HNSW/FTS for append-only PK data, but must switch to @@ -271,7 +521,8 @@ impl Default for IndexStore { hnsw_indexes: HashMap::new(), fts_indexes: HashMap::new(), pk_index: None, - max_visible_batch_position: AtomicUsize::new(0), + indexed_count: AtomicUsize::new(0), + durability: None, pk_has_overrides: AtomicBool::new(false), } } @@ -299,10 +550,7 @@ impl std::fmt::Debug for IndexStore { } }, ) - .field( - "max_visible_batch_position", - &self.max_visible_batch_position.load(Ordering::Acquire), - ) + .field("indexed_count", &self.indexed_count.load(Ordering::Acquire)) .field( "pk_has_overrides", &self.pk_has_overrides.load(Ordering::Acquire), @@ -352,8 +600,19 @@ impl IndexStore { registry.hnsw_indexes.insert(c.name.clone(), index); } MemIndexConfig::Fts(c) => { - let index = - FtsMemIndex::with_params(c.field_id, c.column.clone(), c.params.clone()); + let index = match c.resolved_field.as_deref() { + Some(resolved) => FtsMemIndex::try_with_resolved_field( + c.field_id, + c.column.clone(), + c.params.clone(), + resolved.clone(), + )?, + None => FtsMemIndex::try_with_params( + c.field_id, + c.column.clone(), + c.params.clone(), + )?, + }; registry.fts_indexes.insert(c.name.clone(), index); } } @@ -452,13 +711,16 @@ impl IndexStore { field_id: i32, column: String, params: InvertedIndexParams, - ) { + ) -> Result<()> { assert!( self.pk_index.is_none() || self.pk_is_empty(), "FTS indexes must be configured before inserting rows into a PK memtable" ); - self.fts_indexes - .insert(name, FtsMemIndex::with_params(field_id, column, params)); + self.fts_indexes.insert( + name, + FtsMemIndex::try_with_params(field_id, column, params)?, + ); + Ok(()) } /// Maintain a primary-key index so the memtable can answer "newest visible @@ -516,7 +778,7 @@ impl IndexStore { /// BTree (the sidecar dedup index). Single-column emits the typed PK value; /// composite emits the order-preserving `Binary` encoded tuple. Empty when /// there is no primary key. Row positions line up 1:1 with the forward- - /// written data file, so they are the flushed row ids directly. + /// written data file, so they are the SSTable row ids directly. pub fn pk_training_batches(&self, batch_size: usize) -> Result> { match &self.pk_index { None => Ok(Vec::new()), @@ -688,26 +950,25 @@ impl IndexStore { let had_existing = self.insert_composite_pk(batch, row_offset, track_pk_overrides)?; self.mark_pk_overrides_if_needed(had_existing); - // Update global watermark after all indexes have been updated + // Update the indexed prefix after every index has been updated. if let Some(bp) = batch_position { - self.advance_max_visible_batch_position(bp); + self.advance_indexed_count(bp + 1); } Ok(()) } - /// Advance the visibility watermark to at least `batch_pos`. + /// Advance the indexed prefix to at least `count` batches. /// - /// The watermark only ever moves forward (idempotent max). The vector - /// planner relies on the insert paths setting `pk_has_overrides` before - /// calling this method, so any snapshot that can see a PK rewrite also - /// observes `pk_has_overrides == true`. - pub(crate) fn advance_max_visible_batch_position(&self, batch_pos: usize) { - let mut current = self.max_visible_batch_position.load(Ordering::Acquire); - while batch_pos > current { - match self.max_visible_batch_position.compare_exchange_weak( + /// Only ever moves forward (idempotent max). The vector planner relies on the + /// insert paths setting `pk_has_overrides` before this is called, so any + /// snapshot that can see a PK rewrite also observes `pk_has_overrides == true`. + pub(crate) fn advance_indexed_count(&self, count: usize) { + let mut current = self.indexed_count.load(Ordering::Acquire); + while count > current { + match self.indexed_count.compare_exchange_weak( current, - batch_pos, + count, Ordering::Release, Ordering::Acquire, ) { @@ -717,43 +978,134 @@ impl IndexStore { } } - /// Insert multiple batches into all indexes with cross-batch optimization. + /// Insert multiple batches into every index. + /// + /// Above `PARALLEL_INDEX_MIN_ROWS` rows each index runs on its own thread, which + /// maximizes parallelism when several indexes are maintained. At or below it they run + /// inline on the calling thread: the spawn is one OS thread *per index*, and for a + /// handful of rows that costs more than the indexing itself. + /// + /// Returns a map of index names to their update durations for performance tracking. #[instrument(name = "idx_insert_batches", level = "debug", skip_all, fields(batch_count = batches.len()))] - pub fn insert_batches(&self, batches: &[StoredBatch]) -> Result<()> { + pub fn insert_batches( + &self, + batches: &[StoredBatch], + ) -> Result> { if batches.is_empty() { - return Ok(()); + return Ok(std::collections::HashMap::new()); } let track_pk_overrides = self.should_track_pk_overrides(); - // BTree indexes: iterate batches (no cross-batch optimization benefit) - for index in self.btree_indexes.values() { + + // One task per index, boxed so the inline and the threaded path drive the very + // same closures. Each reports whether it saw an already-present PK. + type IndexTask<'a> = Box Result + Send + Sync + 'a>; + let mut tasks: Vec<(&str, IndexTask<'_>)> = Vec::new(); + + for (name, index) in &self.btree_indexes { let track_this_index = track_pk_overrides && self.is_single_pk_btree(index); - let mut had_existing = false; - for stored in batches { - if track_this_index { - had_existing |= - index.insert_and_report_existing(&stored.data, stored.row_offset)?; - } else { - index.insert(&stored.data, stored.row_offset)?; - } - } - self.mark_pk_overrides_if_needed(had_existing); + tasks.push(( + name.as_str(), + Box::new(move || { + let mut had_existing = false; + for stored in batches { + if track_this_index { + had_existing |= index + .insert_and_report_existing(&stored.data, stored.row_offset)?; + } else { + index.insert(&stored.data, stored.row_offset)?; + } + } + Ok(had_existing) + }), + )); } - // HNSW indexes: use batched insert - for index in self.hnsw_indexes.values() { - index.insert_batches(batches)?; + for (name, index) in &self.hnsw_indexes { + tasks.push(( + name.as_str(), + Box::new(move || index.insert_batches(batches).map(|_| false)), + )); } - // FTS indexes: iterate batches (potential future optimization) - for index in self.fts_indexes.values() { - for stored in batches { - index.insert(&stored.data, stored.row_offset)?; + for (name, index) in &self.fts_indexes { + tasks.push(( + name.as_str(), + Box::new(move || { + for stored in batches { + index.insert(&stored.data, stored.row_offset)?; + } + Ok(false) + }), + )); + } + + // Keep the raw `Duration` so sub-millisecond timings (the steady state for BTree + // updates) survive instead of truncating to 0. + let total_rows: usize = batches.iter().map(|b| b.num_rows).sum(); + let results: Vec<(&str, std::time::Duration, Result)> = + if tasks.len() < 2 || total_rows <= PARALLEL_INDEX_MIN_ROWS { + tasks + .iter() + .map(|(name, task)| { + let start = Instant::now(); + let result = task(); + (*name, start.elapsed(), result) + }) + .collect() + } else { + std::thread::scope(|scope| { + let handles: Vec<_> = tasks + .iter() + .map(|(name, task)| { + let handle = scope.spawn(move || { + let start = Instant::now(); + let result = task(); + (start.elapsed(), result) + }); + (*name, handle) + }) + .collect(); + + handles + .into_iter() + .map(|(name, handle)| match handle.join() { + Ok((duration, result)) => (name, duration, result), + Err(_) => ( + name, + std::time::Duration::ZERO, + Err(Error::internal(format!("Index '{}' thread panicked", name))), + ), + }) + .collect() + }) + }; + + // Every task ran to completion whether or not a peer failed (the threaded path + // joins all handles unconditionally). Keep the first error; there is no rollback, + // so a failure here is terminal for the writer. + let mut first_error: Option = None; + let mut had_existing_pk = false; + let mut duration_map = + std::collections::HashMap::::with_capacity(results.len()); + + for (name, duration, result) in results { + duration_map.insert(name.to_string(), duration); + match result { + Ok(had_existing) => had_existing_pk |= had_existing, + Err(e) if first_error.is_none() => first_error = Some(e), + Err(_) => {} } } - // Single-column PK aliases a `btree_indexes` entry (maintained above); - // a composite PK has its own index, maintained here. + if let Some(e) = first_error { + return Err(e); + } + self.mark_pk_overrides_if_needed(had_existing_pk); + + // Single-column PK aliases a `btree_indexes` entry — its task above already + // maintained it. A composite PK has its own index; maintain it here before the + // watermark advances so the visible prefix is fully indexed. let mut had_existing = false; for stored in batches { had_existing |= @@ -761,146 +1113,12 @@ impl IndexStore { } self.mark_pk_overrides_if_needed(had_existing); - // Update global watermark to the max batch position + // The indexed prefix now covers every batch up to and including the + // highest position in this call, so the count is that position plus one. let max_bp = batches.iter().map(|b| b.batch_position).max().unwrap(); - self.advance_max_visible_batch_position(max_bp); - - Ok(()) - } - - /// Insert multiple batches into all indexes in parallel. - /// - /// Each individual index runs in its own thread, regardless of type. - /// This maximizes parallelism when multiple indexes are maintained. - /// - /// This is used during WAL flush to parallelize index updates with WAL I/O. - /// Insert batches into all indexes in parallel. - /// - /// Returns a map of index names to their update durations for performance tracking. - #[allow(clippy::print_stderr)] - #[instrument(name = "idx_insert_batches_parallel", level = "debug", skip_all, fields(batch_count = batches.len()))] - pub fn insert_batches_parallel( - &self, - batches: &[StoredBatch], - ) -> Result> { - use std::time::Instant; - - if batches.is_empty() { - return Ok(std::collections::HashMap::new()); - } - - let track_pk_overrides = self.should_track_pk_overrides(); - // Use std::thread::scope for parallel CPU-bound work - std::thread::scope(|scope| { - // Each handle returns (index_name, index_type, duration, Result) - let mut handles: Vec<( - &str, - &str, - std::thread::ScopedJoinHandle<'_, (std::time::Duration, Result)>, - )> = Vec::new(); - - // Spawn a thread for each BTree index - for (name, index) in &self.btree_indexes { - let track_this_index = track_pk_overrides && self.is_single_pk_btree(index); - let handle = scope.spawn(move || -> (std::time::Duration, Result) { - let start = Instant::now(); - let result = (|| { - let mut had_existing = false; - for stored in batches { - if track_this_index { - had_existing |= index - .insert_and_report_existing(&stored.data, stored.row_offset)?; - } else { - index.insert(&stored.data, stored.row_offset)?; - } - } - Ok(had_existing) - })(); - (start.elapsed(), result) - }); - handles.push((name.as_str(), "btree", handle)); - } - - // Spawn a thread for each HNSW index - for (name, index) in &self.hnsw_indexes { - let handle = scope.spawn(move || -> (std::time::Duration, Result) { - let start = Instant::now(); - let result = index.insert_batches(batches).map(|_| false); - (start.elapsed(), result) - }); - handles.push((name.as_str(), "hnsw", handle)); - } - - // Spawn a thread for each FTS index - for (name, index) in &self.fts_indexes { - let handle = scope.spawn(move || -> (std::time::Duration, Result) { - let start = Instant::now(); - let result = (|| { - for stored in batches { - index.insert(&stored.data, stored.row_offset)?; - } - Ok(false) - })(); - (start.elapsed(), result) - }); - handles.push((name.as_str(), "fts", handle)); - } - - // Collect results, log timing, and check for errors. Keep the raw - // `Duration` so sub-millisecond timings (the steady-state case for - // BTree updates) are preserved instead of getting truncated to 0. - let mut first_error: Option = None; - let mut timings: Vec<(&str, &str, std::time::Duration)> = Vec::new(); - let mut had_existing_pk = false; - - for (name, idx_type, handle) in handles { - match handle.join() { - Ok((duration, Ok(had_existing))) => { - timings.push((name, idx_type, duration)); - had_existing_pk |= had_existing; - } - Ok((duration, Err(e))) => { - timings.push((name, idx_type, duration)); - if first_error.is_none() { - first_error = Some(e); - } - } - Err(_) => { - if first_error.is_none() { - first_error = - Some(Error::internal(format!("Index '{}' thread panicked", name))); - } - } - } - } + self.advance_indexed_count(max_bp + 1); - if let Some(e) = first_error { - return Err(e); - } - self.mark_pk_overrides_if_needed(had_existing_pk); - - let duration_map: std::collections::HashMap = timings - .into_iter() - .map(|(name, _idx_type, duration)| (name.to_string(), duration)) - .collect(); - - // Single-column PK aliases a `btree_indexes` entry — its thread above - // already maintained it (and joined). A composite PK has its own - // index; maintain it here before the watermark advances so the - // visible prefix is fully indexed. - let mut had_existing = false; - for stored in batches { - had_existing |= - self.insert_composite_pk(&stored.data, stored.row_offset, track_pk_overrides)?; - } - self.mark_pk_overrides_if_needed(had_existing); - - // Update global watermark to the max batch position - let max_bp = batches.iter().map(|b| b.batch_position).max().unwrap(); - self.advance_max_visible_batch_position(max_bp); - - Ok(duration_map) - }) + Ok(duration_map) } /// Get a BTree index by name. @@ -941,9 +1159,20 @@ impl IndexStore { /// Searches through all FTS indexes to find one matching the field_id. /// Use this for column-to-index resolution (column → field_id → index). pub fn get_fts_by_field_id(&self, field_id: i32) -> Option<&FtsMemIndex> { - self.fts_indexes - .values() - .find(|idx| idx.field_id() == field_id) + self.get_fts_by_field_id_and_granularity( + field_id, + lance_index::scalar::inverted::DocumentGranularity::Row, + ) + } + + pub fn get_fts_by_field_id_and_granularity( + &self, + field_id: i32, + document_granularity: lance_index::scalar::inverted::DocumentGranularity, + ) -> Option<&FtsMemIndex> { + self.fts_indexes.values().find(|idx| { + idx.field_id() == field_id && idx.document_granularity() == document_granularity + }) } /// Get a BTree index by column name. @@ -963,9 +1192,40 @@ impl IndexStore { /// Get an FTS index by column name. pub fn get_fts_by_column(&self, column: &str) -> Option<&FtsMemIndex> { - self.fts_indexes + self.get_fts_by_column_and_granularity( + column, + lance_index::scalar::inverted::DocumentGranularity::Row, + ) + } + + pub fn get_fts_by_column_and_granularity( + &self, + column: &str, + document_granularity: lance_index::scalar::inverted::DocumentGranularity, + ) -> Option<&FtsMemIndex> { + self.fts_indexes.values().find(|idx| { + idx.column_name() == column && idx.document_granularity() == document_granularity + }) + } + + /// Return the distinct persisted document granularities for FTS indexes on + /// `column`, ordered from row to list-element. + pub fn fts_document_granularities_by_column( + &self, + column: &str, + ) -> Vec { + let mut granularities = self + .fts_indexes .values() - .find(|idx| idx.column_name() == column) + .filter(|index| index.column_name() == column) + .map(|index| index.document_granularity()) + .collect::>(); + granularities.sort_by_key(|document_granularity| match document_granularity { + lance_index::scalar::inverted::DocumentGranularity::Row => 0, + lance_index::scalar::inverted::DocumentGranularity::ListElement => 1, + }); + granularities.dedup(); + granularities } /// Check if the registry has any indexes. @@ -973,20 +1233,59 @@ impl IndexStore { self.btree_indexes.is_empty() && self.hnsw_indexes.is_empty() && self.fts_indexes.is_empty() } + /// Name every index this memtable carries, for diagnostics. + /// + /// Answers "is my fresh-tier vector search brute-force" — an absent + /// name is the whole explanation, and there is no other way to see it + /// from outside. Sorted so repeated calls compare cleanly; `HashMap` + /// iteration order alone would not. + pub fn index_names(&self) -> Vec { + let mut out: Vec = self + .btree_indexes + .keys() + .chain(self.hnsw_indexes.keys()) + .chain(self.fts_indexes.keys()) + .cloned() + .collect(); + out.sort(); + out + } + /// Get the total number of indexes. pub fn len(&self) -> usize { self.btree_indexes.len() + self.hnsw_indexes.len() + self.fts_indexes.len() } - /// Get the visibility watermark (max batch position safe to read). + /// How many batches of this memtable have been fully indexed (exclusive + /// count; 0 before any batch is indexed). /// - /// Returns the highest batch position whose data is durable in the WAL - /// and therefore visible to scanners. Scanners snapshot this at plan - /// construction time so every plan runs against a stable cursor. + /// This is the *indexed* cursor, not the visibility watermark: it advances + /// once every index insert for a batch completes, regardless of WAL + /// durability. Readers must snapshot [`Self::visible_count`], which derives + /// what is safe to read from this cursor and the writer's durability cursor. + pub fn indexed_count(&self) -> usize { + self.indexed_count.load(Ordering::Acquire) + } + + /// The prefix of this memtable that readers may see. Snapshot this, never + /// `indexed_count`. /// - /// Returns 0 before any WAL flush has advanced the watermark. - pub fn max_visible_batch_position(&self) -> usize { - self.max_visible_batch_position.load(Ordering::Acquire) + /// Derived on every call from the two cursors rather than cached, so there is + /// no published value that can be left stale by a race between the two tasks + /// that advance them. A bare `IndexStore` has no writer, so its visible + /// prefix is simply what has been indexed. + pub fn visible_count(&self) -> usize { + let indexed = self.indexed_count(); + match &self.durability { + Some((cursors, global_offset)) => cursors.visible_count(indexed, *global_offset), + None => indexed, + } + } + + /// Bind this memtable's indexes to the writer's cursors. Called once at + /// construction, before the memtable is published. + pub(crate) fn set_durability(&mut self, cursors: Arc, global_offset: usize) { + self.durability = Some((cursors, global_offset)); } } @@ -994,26 +1293,60 @@ impl IndexStore { mod tests { use super::*; use arrow_array::{Int32Array, StringArray}; - use arrow_schema::{DataType, Field, Schema as ArrowSchema}; - use log::warn; + use arrow_schema::{DataType, Field, Fields, Schema as ArrowSchema}; + use lance_index::scalar::inverted::InvertedListFormatVersion; + use rstest::rstest; use std::sync::Arc; use uuid::Uuid; - /// Check if an index type is supported and log warning if not. - fn check_index_type_supported(index_type: &str) -> bool { - match index_type.to_lowercase().as_str() { - "btree" | "scalar" => true, - "hnsw" | "vector" => true, - "fts" | "inverted" | "fulltext" => true, - _ => { - warn!( - "Index type '{}' is not supported for MemWAL. \ - Supported types: btree, hnsw, fts. Skipping.", - index_type - ); - false - } + /// Matching is on the message-name suffix, not the whole url: `Any::from_msg` + /// emits the package (`/lance.table.`, `/lance.index.pb.`), while MemWAL flush + /// used to hand-write a `type.googleapis.com/` url that existing datasets + /// still carry. + #[rstest] + #[case::btree("/lance.table.BTreeIndexDetails", Some(MemIndexKind::BTree))] + #[case::fts("/lance.table.InvertedIndexDetails", Some(MemIndexKind::Fts))] + #[case::fts_legacy("/lance.index.pb.InvertedIndexDetails", Some(MemIndexKind::Fts))] + #[case::vector("/lance.index.pb.VectorIndexDetails", Some(MemIndexKind::Hnsw))] + // What MemWAL flush wrote before it switched to `Any::from_msg`. + #[case::vector_legacy_flush( + "type.googleapis.com/lance.index.VectorIndexDetails", + Some(MemIndexKind::Hnsw) + )] + #[case::bitmap("/lance.table.BitmapIndexDetails", None)] + #[case::label_list("/lance.table.LabelListIndexDetails", None)] + #[case::ngram("/lance.table.NGramIndexDetails", None)] + #[case::zone_map("/lance.table.ZoneMapIndexDetails", None)] + #[case::bloom_filter("/lance.index.pb.BloomFilterIndexDetails", None)] + #[case::json("/lance.index.pb.JsonIndexDetails", None)] + #[case::fm("/lance.index.pb.FMIndexDetails", None)] + #[case::absent("", None)] + fn type_urls_resolve_to_the_kind_the_writer_builds( + #[case] type_url: &str, + #[case] expected: Option, + ) { + assert_eq!(MemIndexKind::from_type_url(type_url), expected); + } + + /// `ALL` is hand-maintained, so a kind left out of it stops resolving. + #[test] + fn every_kind_is_registered_and_uniquely_identified() { + for kind in MemIndexKind::ALL { + assert_eq!( + MemIndexKind::from_type_url(&format!("/lance.table.{}", kind.details_suffix())), + Some(*kind), + "{kind:?} does not resolve from its own suffix", + ); } + let suffixes: std::collections::HashSet<_> = MemIndexKind::ALL + .iter() + .map(|k| k.details_suffix()) + .collect(); + assert_eq!( + suffixes.len(), + MemIndexKind::ALL.len(), + "two kinds share a details suffix, so one can never be resolved", + ); } fn create_test_schema() -> Arc { @@ -1040,11 +1373,37 @@ mod tests { .unwrap() } + fn create_sized_batch(schema: &ArrowSchema, start_id: i32, num_rows: usize) -> RecordBatch { + let ids: Vec = (0..num_rows as i32).map(|i| start_id + i).collect(); + let names: Vec = ids.iter().map(|id| format!("name-{id}")).collect(); + let descriptions: Vec = ids.iter().map(|id| format!("hello world {id}")).collect(); + RecordBatch::try_new( + Arc::new(schema.clone()), + vec![ + Arc::new(Int32Array::from(ids)), + Arc::new(StringArray::from(names)), + Arc::new(StringArray::from(descriptions)), + ], + ) + .unwrap() + } + fn fts_index_metadata(index_version: i32) -> IndexMetadata { - let details = - pbold::InvertedIndexDetails::try_from(&InvertedIndexParams::default()).unwrap(); - let mut value = Vec::new(); - details.encode(&mut value).unwrap(); + fts_index_metadata_with_details(index_version, None) + } + + fn fts_index_metadata_with_details( + index_version: i32, + details: Option, + ) -> IndexMetadata { + let index_details = details.map(|details| { + let mut value = Vec::new(); + details.encode(&mut value).unwrap(); + Arc::new(prost_types::Any { + type_url: "type.googleapis.com/lance.index.InvertedIndexDetails".to_string(), + value, + }) + }); IndexMetadata { uuid: Uuid::new_v4(), @@ -1052,10 +1411,7 @@ mod tests { name: "desc_idx".to_string(), dataset_version: 1, fragment_bitmap: None, - index_details: Some(Arc::new(prost_types::Any { - type_url: "type.googleapis.com/lance.index.InvertedIndexDetails".to_string(), - value, - })), + index_details, index_version, created_at: None, base_id: None, @@ -1344,15 +1700,41 @@ mod tests { } #[test] - fn test_check_index_type_supported() { - assert!(check_index_type_supported("btree")); - assert!(check_index_type_supported("BTree")); - assert!(check_index_type_supported("hnsw")); - assert!(check_index_type_supported("vector")); - assert!(check_index_type_supported("fts")); - assert!(check_index_type_supported("inverted")); + fn fts_registry_routes_row_and_element_targets_independently() { + let mut registry = IndexStore::new(); + registry + .add_fts_with_params( + "tags_idx".to_string(), + 1, + "tags".to_string(), + InvertedIndexParams::default(), + ) + .unwrap(); + registry + .add_fts_with_params( + "tags_element_idx".to_string(), + 1, + "tags".to_string(), + InvertedIndexParams::default().document_granularity( + lance_index::scalar::inverted::DocumentGranularity::ListElement, + ), + ) + .unwrap(); - assert!(!check_index_type_supported("unknown")); + assert_eq!( + registry.get_fts_by_column("tags").unwrap().column_name(), + "tags" + ); + assert_eq!( + registry + .get_fts_by_column_and_granularity( + "tags", + lance_index::scalar::inverted::DocumentGranularity::ListElement, + ) + .unwrap() + .column_name(), + "tags" + ); } #[test] @@ -1364,6 +1746,7 @@ mod tests { (0, InvertedListFormatVersion::V1), (1, InvertedListFormatVersion::V1), (2, InvertedListFormatVersion::V2), + (3, InvertedListFormatVersion::V3), ] { let config = MemIndexConfig::fts_from_metadata(&fts_index_metadata(index_version), &schema) @@ -1386,13 +1769,106 @@ mod tests { let arrow_schema = create_test_schema(); let schema = LanceSchema::try_from(arrow_schema.as_ref()).unwrap(); - let err = MemIndexConfig::fts_from_metadata(&fts_index_metadata(3), &schema).unwrap_err(); + let err = MemIndexConfig::fts_from_metadata(&fts_index_metadata(4), &schema).unwrap_err(); assert!( - err.to_string().contains("unsupported index_version 3"), + err.to_string().contains("unsupported index_version 4"), "{err}" ); } + #[test] + fn fts_from_metadata_accepts_element_document_v3_capability() { + let arrow_schema = Arc::new(ArrowSchema::new(vec![ + Field::new("id", DataType::Int32, false), + Field::new("name", DataType::Utf8, true), + Field::new( + "tags", + DataType::List(Arc::new(Field::new("item", DataType::Utf8, true))), + true, + ), + ])); + let schema = LanceSchema::try_from(arrow_schema.as_ref()).unwrap(); + let tags = schema.field("tags").unwrap(); + for (block_size, expected_format_version) in [ + (128, InvertedListFormatVersion::V2), + (256, InvertedListFormatVersion::V3), + ] { + let params = InvertedIndexParams::default() + .block_size(block_size) + .unwrap() + .document_granularity( + lance_index::scalar::inverted::DocumentGranularity::ListElement, + ); + let details = pbold::InvertedIndexDetails::try_from(¶ms).unwrap(); + let mut metadata = fts_index_metadata_with_details(3, Some(details)); + metadata.fields = vec![tags.id]; + let config = MemIndexConfig::fts_from_metadata(&metadata, &schema).unwrap(); + + let MemIndexConfig::Fts(config) = config else { + unreachable!("fts metadata should create an FTS config") + }; + assert_eq!(config.field_id, tags.id); + assert_eq!(config.column, "tags"); + assert_eq!( + config.params.get_document_granularity(), + lance_index::scalar::inverted::DocumentGranularity::ListElement + ); + assert_eq!( + config.params.resolved_format_version(), + expected_format_version + ); + } + } + + #[test] + fn fts_from_metadata_accepts_v3_with_legacy_block_size() { + let arrow_schema = create_test_schema(); + let schema = LanceSchema::try_from(arrow_schema.as_ref()).unwrap(); + let mut legacy_details = + pbold::InvertedIndexDetails::try_from(&InvertedIndexParams::default()).unwrap(); + legacy_details.posting_format_version = None; + + for metadata in [ + fts_index_metadata(3), + fts_index_metadata_with_details(3, Some(legacy_details)), + ] { + let config = MemIndexConfig::fts_from_metadata(&metadata, &schema).unwrap(); + let MemIndexConfig::Fts(config) = config else { + unreachable!("FTS metadata should create an FTS config"); + }; + assert_eq!( + config.params.resolved_format_version(), + InvertedListFormatVersion::V3 + ); + assert_eq!(config.params.posting_block_size(), 128); + } + } + + #[test] + fn fts_from_metadata_accepts_v3_with_256_block_size() { + let arrow_schema = create_test_schema(); + let schema = LanceSchema::try_from(arrow_schema.as_ref()).unwrap(); + let params = InvertedIndexParams::default().block_size(256).unwrap(); + let details = pbold::InvertedIndexDetails::try_from(¶ms).unwrap(); + + let config = MemIndexConfig::fts_from_metadata( + &fts_index_metadata_with_details(3, Some(details)), + &schema, + ) + .unwrap(); + + match config { + MemIndexConfig::Fts(config) => { + assert_eq!( + config.params.resolved_format_version(), + InvertedListFormatVersion::V3 + ); + assert_eq!(config.params.posting_block_size(), 256); + } + _ => unreachable!("fts metadata should create an FTS config"), + } + } + #[test] fn test_from_configs() { let configs = vec![ @@ -1417,8 +1893,200 @@ mod tests { assert!(registry.get_fts_by_field_id(2).is_some()); } + fn vector_schema() -> Arc { + Arc::new(ArrowSchema::new(vec![ + Field::new("id", DataType::Int32, false), + Field::new("description", DataType::Utf8, true), + Field::new( + "vector", + DataType::FixedSizeList(Arc::new(Field::new("item", DataType::Float32, true)), 4), + true, + ), + Field::new( + "f64_vector", + DataType::FixedSizeList(Arc::new(Field::new("item", DataType::Float64, true)), 4), + true, + ), + ])) + } + + /// Every index config that would fail *deterministically* on insert must be + /// rejected at open instead. Such a config also fails on WAL replay, so once + /// a row is durable the shard could never reopen — poison-and-replay would + /// not terminate. + #[rstest] + #[case::btree_ok(MemIndexConfig::BTree(BTreeIndexConfig { + name: "idx".into(), field_id: 0, column: "id".into(), + }), None)] + #[case::btree_missing_column(MemIndexConfig::BTree(BTreeIndexConfig { + name: "idx".into(), field_id: 9, column: "nope".into(), + }), Some("not in the shard schema"))] + // Column exists, but its field_id names a *different* column ("id" is 0, not 1). + #[case::btree_field_id_column_mismatch(MemIndexConfig::BTree(BTreeIndexConfig { + name: "idx".into(), field_id: 1, column: "id".into(), + }), Some("has field_id 0"))] + #[case::fts_ok(MemIndexConfig::Fts(FtsIndexConfig::new( + "idx".into(), 1, "description".into(), + )), None)] + #[case::fts_non_utf8(MemIndexConfig::Fts(FtsIndexConfig::new( + "idx".into(), 0, "id".into(), + )), Some("must resolve to Utf8, LargeUtf8, Utf8View, or JSON"))] + #[case::fts_missing_column(MemIndexConfig::Fts(FtsIndexConfig::new( + "idx".into(), 9, "nope".into(), + )), Some("does not exist in the dataset schema"))] + #[case::hnsw_ok(MemIndexConfig::Hnsw(Box::new(HnswIndexConfig::new( + "idx".into(), 2, "vector".into(), DistanceType::L2, + ))), None)] + #[case::hnsw_not_a_vector(MemIndexConfig::Hnsw(Box::new(HnswIndexConfig::new( + "idx".into(), 0, "id".into(), DistanceType::L2, + ))), Some("requires a FixedSizeList column"))] + #[case::hnsw_wrong_item_type(MemIndexConfig::Hnsw(Box::new(HnswIndexConfig::new( + "idx".into(), 3, "f64_vector".into(), DistanceType::L2, + ))), Some("item type Float64"))] + #[case::hnsw_missing_column(MemIndexConfig::Hnsw(Box::new(HnswIndexConfig::new( + "idx".into(), 9, "nope".into(), DistanceType::L2, + ))), Some("not in the shard schema"))] + fn test_validate_index_configs( + #[case] config: MemIndexConfig, + #[case] expected_error: Option<&str>, + ) { + let schema = vector_schema(); + let lance_schema = LanceSchema::try_from(schema.as_ref()).unwrap(); + let result = validate_index_configs(&[config], &schema, &lance_schema, &[]); + match expected_error { + None => result.expect("valid config must pass validation"), + Some(fragment) => { + let message = result + .expect_err("invalid config must be rejected") + .to_string(); + assert!( + message.contains(fragment), + "error must explain the mismatch; wanted {fragment:?}, got {message:?}" + ); + } + } + } + #[test] - fn test_index_store_max_visible_batch_position() { + fn test_validate_nested_fts_index_config() { + let content_fields = Fields::from(vec![Field::new("content", DataType::Utf8, true)]); + let doc_item = Arc::new(Field::new("item", DataType::Struct(content_fields), true)); + let group_fields = Fields::from(vec![Field::new("docs", DataType::List(doc_item), true)]); + let group_item = Arc::new(Field::new("item", DataType::Struct(group_fields), true)); + let schema = Arc::new(ArrowSchema::new(vec![Field::new( + "groups", + DataType::List(group_item), + true, + )])); + let lance_schema = LanceSchema::try_from(schema.as_ref()).unwrap(); + let resolved = crate::index::scalar::inverted::resolve_fts_field( + &lance_schema, + "groups.docs.content", + lance_index::scalar::inverted::DocumentGranularity::ListElement, + ) + .unwrap(); + + let params = InvertedIndexParams::default() + .document_granularity(lance_index::scalar::inverted::DocumentGranularity::ListElement); + let config = MemIndexConfig::Fts(FtsIndexConfig::with_params( + "idx".into(), + resolved.final_field_id, + "groups.docs.content".into(), + params.clone(), + )); + validate_index_configs(&[config], &schema, &lance_schema, &[]).unwrap(); + + let wrong_field_id = MemIndexConfig::Fts(FtsIndexConfig::with_params( + "idx".into(), + resolved.final_field_id + 1, + "groups.docs.content".into(), + params, + )); + let error = + validate_index_configs(&[wrong_field_id], &schema, &lance_schema, &[]).unwrap_err(); + assert!(error.to_string().contains("final field_id"), "{error}"); + } + + #[test] + fn test_validate_index_configs_rejects_diverged_lance_schema() { + let arrow_schema = ArrowSchema::new(vec![Field::new("id", DataType::Int32, false)]); + let lance_schema = LanceSchema::try_from(&ArrowSchema::new(vec![Field::new( + "other", + DataType::Int32, + false, + )])) + .expect("test Lance schema must be valid"); + let config = MemIndexConfig::BTree(BTreeIndexConfig { + name: "idx".into(), + field_id: 0, + column: "id".into(), + }); + + let error = validate_index_configs(&[config], &arrow_schema, &lance_schema, &[]) + .expect_err("diverged Arrow and Lance schemas must be rejected"); + assert!( + matches!(error, Error::InvalidInput { .. }), + "expected InvalidInput, got {error:?}" + ); + let message = error.to_string(); + assert!( + message.contains("index 'idx'"), + "error must name the index: {message}" + ); + assert!( + message.contains("column 'id'"), + "error must name the column: {message}" + ); + assert!( + message.contains("absent from the Lance schema"), + "error must explain the schema divergence: {message}" + ); + } + + /// A composite PK builds an order-preserving encoded key, so its columns must + /// be encodable. A single-column PK aliases a BTree entry, which accepts any + /// type — so it must *not* be rejected here. + #[test] + fn test_validate_composite_pk_column_types() { + let schema = Arc::new(ArrowSchema::new(vec![ + Field::new("id", DataType::Int32, false), + Field::new("name", DataType::Utf8, false), + Field::new( + "coords", + DataType::FixedSizeList(Arc::new(Field::new("item", DataType::Float32, true)), 2), + true, + ), + ])); + let lance_schema = LanceSchema::try_from(schema.as_ref()).unwrap(); + + validate_index_configs(&[], &schema, &lance_schema, &["id".into(), "name".into()]) + .expect("Int32 + Utf8 composite PK must be encodable"); + + let err = + validate_index_configs(&[], &schema, &lance_schema, &["id".into(), "coords".into()]) + .expect_err("a FixedSizeList PK column has no order-preserving encoding"); + assert!( + err.to_string().contains("order-preserving key encoding"), + "error must name the reason, got {err}" + ); + + // A single-column PK of the same type is fine: it aliases a BTree. + validate_index_configs(&[], &schema, &lance_schema, &["coords".into()]) + .expect("single-column PK aliases a BTree and accepts any type"); + + // But every PK column must exist. A single-column PK naming an absent + // column is rejected here, not left to fail deterministically on every + // later index build and WAL replay. + let err = validate_index_configs(&[], &schema, &lance_schema, &["missing".into()]) + .expect_err("a single-column PK on an absent column must be rejected"); + assert!( + err.to_string().contains("not in the shard schema"), + "error must name the missing column, got {err}" + ); + } + + #[test] + fn test_index_store_indexed_count() { let schema = create_test_schema(); let mut registry = IndexStore::new(); @@ -1427,7 +2095,7 @@ mod tests { registry.add_fts("desc_idx".to_string(), 2, "description".to_string()); // Initial watermark should be 0 (no data indexed yet) - assert_eq!(registry.max_visible_batch_position(), 0); + assert_eq!(registry.indexed_count(), 0); // Insert with batch position tracking let batch = create_test_batch(&schema, 0); @@ -1435,20 +2103,54 @@ mod tests { .insert_with_batch_position(&batch, 0, Some(5)) .unwrap(); - // Now watermark should be 5 - assert_eq!(registry.max_visible_batch_position(), 5); + // Indexing batch position 5 means the prefix [0, 6) is indexed. + assert_eq!(registry.indexed_count(), 6); // Insert with higher batch position registry .insert_with_batch_position(&batch, 3, Some(10)) .unwrap(); - // Watermark should advance to 10 - assert_eq!(registry.max_visible_batch_position(), 10); + // Advances to cover batch position 10. + assert_eq!(registry.indexed_count(), 11); - // Insert without batch position shouldn't change watermark + // Insert without batch position shouldn't change the cursor registry.insert(&batch, 6).unwrap(); - assert_eq!(registry.max_visible_batch_position(), 10); + assert_eq!(registry.indexed_count(), 11); + } + + /// `insert_batches` picks the inline or the threaded path by row count, so + /// exercise both and assert they leave the same index state: every row indexed + /// exactly once, in every index, with a timing reported for each. + #[rstest] + #[case::inline(8)] + #[case::threaded(PARALLEL_INDEX_MIN_ROWS + 64)] + fn test_insert_batches_indexes_every_row_once(#[case] num_rows: usize) { + let schema = create_test_schema(); + let mut registry = IndexStore::new(); + registry.add_btree("id_idx".to_string(), 0, "id".to_string()); + registry.add_fts("desc_idx".to_string(), 2, "description".to_string()); + + let batch = create_sized_batch(&schema, 0, num_rows); + let durations = registry + .insert_batches(&[StoredBatch::new(batch, 0, 2)]) + .unwrap(); + + assert_eq!(durations.len(), 2, "expected one timing per index"); + assert!(durations.contains_key("id_idx")); + assert!(durations.contains_key("desc_idx")); + + let btree = registry.get_btree("id_idx").unwrap(); + for id in 0..num_rows as i32 { + let positions = btree.get(&ScalarValue::Int32(Some(id))); + assert_eq!( + positions.len(), + 1, + "id={id} should be indexed exactly once, got {positions:?}" + ); + } + assert_eq!(registry.get_fts("desc_idx").unwrap().doc_count(), num_rows); + assert_eq!(registry.indexed_count(), 3); } #[test] diff --git a/rust/lance/src/dataset/mem_wal/index/fts.rs b/rust/lance/src/dataset/mem_wal/index/fts.rs index 61c797db041..e0b2cb90ef8 100644 --- a/rust/lance/src/dataset/mem_wal/index/fts.rs +++ b/rust/lance/src/dataset/mem_wal/index/fts.rs @@ -47,26 +47,26 @@ use std::cmp::Reverse; use std::collections::{BinaryHeap, HashMap, HashSet}; -use std::sync::Arc; -use std::sync::Mutex; use std::sync::atomic::{AtomicUsize, Ordering}; +use std::sync::{Arc, Mutex, OnceLock}; use arc_swap::ArcSwap; -use arrow_array::{Array, LargeStringArray, RecordBatch, StringArray, StringViewArray}; -use arrow_schema::DataType; +use arrow_array::RecordBatch; use crossbeam_skiplist::SkipMap; use fst::{Map, Streamer}; use lance_bitpacking::{BitPacker, BitPacker4x}; +use lance_core::datatypes::Schema as LanceSchema; use lance_core::{Error, Result}; use lance_index::scalar::InvertedIndexParams; -use lance_index::scalar::inverted::query::Operator; -use lance_index::scalar::inverted::tokenizer::document_tokenizer::LanceTokenizer; +use lance_index::scalar::inverted::query::{Operator, Tokens}; +use lance_index::scalar::inverted::tokenizer::document_tokenizer::{DocType, LanceTokenizer}; use lance_index::scalar::inverted::{DocSet, MemBM25Scorer, Scorer, TokenSet}; use lance_tokenizer::TokenStream; use rayon::prelude::*; use rustc_hash::FxHashMap; use super::RowPosition; +use crate::index::scalar::inverted::{ResolvedFtsField, resolve_fts_field}; // ============================================================================ // Public types preserved from previous API @@ -77,10 +77,46 @@ use super::RowPosition; pub struct FtsEntry { /// Row position in MemTable. pub row_position: RowPosition, + /// Root-to-leaf physical list ordinals for a ListElement document. + pub doc_index: Option>, /// BM25 score for this document. pub score: f32, } +#[derive(Debug, Clone, PartialEq, Eq, Hash, PartialOrd, Ord)] +struct DocumentKey { + row_position: RowPosition, + doc_index: Vec, +} + +impl FtsEntry { + fn key(&self) -> DocumentKey { + DocumentKey { + row_position: self.row_position, + doc_index: self.doc_index.clone().unwrap_or_default(), + } + } +} + +#[derive(Debug, Clone)] +struct DocumentMetadata { + key: DocumentKey, + num_tokens: u32, +} + +fn doc_set_key(docs: &DocSet, doc_id: u32) -> DocumentKey { + DocumentKey { + row_position: docs.row_id(doc_id), + doc_index: (0..docs.coordinate_rank()) + .map(|rank| docs.coordinate(doc_id, rank)) + .collect(), + } +} + +fn public_doc_index(coordinates: &[u32]) -> Option> { + (!coordinates.is_empty()).then(|| coordinates.to_vec()) +} + /// Full-text search query expression for composable queries. #[derive(Debug, Clone)] pub enum FtsQueryExpr { @@ -118,7 +154,7 @@ pub enum FtsQueryExpr { }, /// Boolean combination of queries. Boolean { - /// All MUST clauses must match for a document to be included. + /// All MUST clauses must match and contribute to the score. must: Vec, /// At least one SHOULD clause should match (adds to score). should: Vec, @@ -371,6 +407,41 @@ fn char_prefix(term: &str, prefix_length: u32) -> &str { .unwrap_or(term) } +fn query_tokens_to_vec(tokens: &Tokens) -> Vec { + (0..tokens.len()) + .map(|idx| tokens.get_token(idx).to_string()) + .collect() +} + +fn has_grouped_positions(tokens: &Tokens) -> bool { + let mut seen = HashSet::new(); + (0..tokens.len()).any(|idx| !seen.insert(tokens.position(idx))) +} + +fn query_position_groups(tokens: &Tokens) -> Vec> { + let mut groups = Vec::new(); + let mut current_position = None; + for idx in 0..tokens.len() { + let position = tokens.position(idx); + if current_position != Some(position) { + current_position = Some(position); + groups.push(Vec::new()); + } + let group = groups + .last_mut() + .expect("a group should exist after pushing for position"); + let token = tokens.get_token(idx).to_string(); + if !group.contains(&token) { + group.push(token); + } + } + groups +} + +fn position_groups_to_tokens(groups: &[Vec]) -> Vec { + groups.iter().flatten().cloned().collect() +} + /// Builder for constructing Boolean queries. #[derive(Debug, Clone, Default)] pub struct BooleanQueryBuilder { @@ -537,27 +608,31 @@ impl<'a> Iterator for TermChunkIter<'a> { } } -/// Per-batch row metadata. +/// Per-batch document metadata. #[derive(Debug)] struct BatchMeta { batch_position: usize, - row_offset: u64, - /// `doc_lengths[i]` is the token count of the row at `row_offset + i`. - doc_lengths: Vec, - rows: u32, + /// Dense tail-local document position of `documents[0]`. + document_position_start: u64, + documents: Vec, } impl BatchMeta { - fn dl(&self, row_position: u64) -> Option { - if row_position < self.row_offset { + fn document(&self, document_position: u64) -> Option<&DocumentMetadata> { + if document_position < self.document_position_start { return None; } - let idx = (row_position - self.row_offset) as usize; - self.doc_lengths.get(idx).copied() + let idx = (document_position - self.document_position_start) as usize; + self.documents.get(idx) + } + + fn dl(&self, document_position: u64) -> Option { + self.document(document_position).map(|doc| doc.num_tokens) } fn memory_size(&self) -> usize { - std::mem::size_of::() + self.doc_lengths.capacity() * std::mem::size_of::() + std::mem::size_of::() + + self.documents.capacity() * std::mem::size_of::() } } @@ -639,9 +714,9 @@ struct Snapshot { /// visible_count` for any snapshot the writer has stored (each publish /// appends one entry and bumps `visible_count`). batches: BatchLog, - /// `Σ batches[i].rows` for `i < visible_count`. + /// `Σ batches[i].documents.len()` for `i < visible_count`. cumulative_doc_count: u64, - /// `Σ batches[i].doc_lengths.iter().sum()` for `i < visible_count`. + /// `Σ batches[i].documents[*].num_tokens` for `i < visible_count`. cumulative_total_tokens: u64, } @@ -804,9 +879,8 @@ impl TailIndex { fn append_batch( &self, batch_position: usize, - row_offset: u64, - rows: u32, - doc_lengths: Vec, + document_position_start: u64, + documents: Vec, total_tokens: u64, term_builders: FxHashMap, BatchTermBuilder>, with_position: bool, @@ -830,17 +904,18 @@ impl TailIndex { slot.store(TermSlice::push(cur, chunk)); } drop(cache); + let document_count = documents.len() as u64; let new_meta = Arc::new(BatchMeta { batch_position, - row_offset, - doc_lengths, - rows, + document_position_start, + documents, }); let cur = self.snapshot.load(); + debug_assert_eq!(document_position_start, cur.cumulative_doc_count); self.snapshot.store(Arc::new(Snapshot { visible_count: cur.visible_count + 1, batches: cur.batches.pushed(new_meta), - cumulative_doc_count: cur.cumulative_doc_count + rows as u64, + cumulative_doc_count: cur.cumulative_doc_count + document_count, cumulative_total_tokens: cur.cumulative_total_tokens + total_tokens, })); } @@ -884,8 +959,9 @@ impl IndexState { /// model and visibility contract. pub struct FtsMemIndex { field_id: i32, - column_name: String, + source_column_name: String, params: InvertedIndexParams, + resolved_field: OnceLock, tokenizer_pool: Arc, /// Writer-only tokenizer slot. Held under a Mutex purely so `insert` @@ -916,12 +992,29 @@ struct PendingMerge { result: Option>, } +fn publish_pending_merge(slot: &Mutex>, merged: Result) { + let Ok(mut guard) = slot.lock() else { + return; + }; + match merged { + Ok(merged) => { + if let Some(pending) = guard.as_mut() { + pending.result = Some(Arc::new(merged)); + } + } + Err(error) => { + tracing::error!(?error, "background FTS tiered merge failed"); + *guard = None; + } + } +} + impl std::fmt::Debug for FtsMemIndex { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { let st = self.state.load(); f.debug_struct("FtsMemIndex") .field("field_id", &self.field_id) - .field("column_name", &self.column_name) + .field("source_column_name", &self.source_column_name) .field("doc_count", &self.doc_count()) .field("partitions", &st.partitions.len()) .field("params", &self.params) @@ -960,19 +1053,43 @@ impl FtsMemIndex { /// Create a new FTS index with custom tokenizer parameters. pub fn with_params(field_id: i32, column_name: String, params: InvertedIndexParams) -> Self { - let pool = TokenizerPool::new(¶ms, Self::DEFAULT_TOKENIZER_POOL_CAP) - .expect("Failed to build tokenizer"); + Self::try_with_params(field_id, column_name, params) + .expect("invalid MemWAL FTS index parameters") + } + + /// Try to create a new FTS index with custom tokenizer parameters. + pub fn try_with_params( + field_id: i32, + column_name: String, + params: InvertedIndexParams, + ) -> Result { + params.validate_format_version()?; + let pool = TokenizerPool::new(¶ms, Self::DEFAULT_TOKENIZER_POOL_CAP)?; let writer_tokenizer = pool.template.box_clone(); - Self { + Ok(Self { field_id, - column_name, + source_column_name: column_name, params, + resolved_field: OnceLock::new(), tokenizer_pool: Arc::new(pool), writer_tokenizer: Mutex::new(writer_tokenizer), state: ArcSwap::from(IndexState::empty()), freeze_threshold_rows: Self::DEFAULT_FREEZE_THRESHOLD_ROWS, merge: Arc::new(Mutex::new(None)), - } + }) + } + + pub(crate) fn try_with_resolved_field( + field_id: i32, + column_name: String, + params: InvertedIndexParams, + resolved_field: ResolvedFtsField, + ) -> Result { + let index = Self::try_with_params(field_id, column_name, params)?; + index.resolved_field.set(resolved_field).map_err(|_| { + Error::internal("MemWAL FTS traversal was initialized twice".to_string()) + })?; + Ok(index) } /// Override the tail freeze threshold (docs) — the analogue of Lucene's @@ -989,13 +1106,21 @@ impl FtsMemIndex { } pub fn column_name(&self) -> &str { - &self.column_name + &self.source_column_name + } + + pub fn source_column_name(&self) -> &str { + &self.source_column_name } pub fn params(&self) -> &InvertedIndexParams { &self.params } + pub fn document_granularity(&self) -> lance_index::scalar::inverted::DocumentGranularity { + self.params.get_document_granularity() + } + /// Number of visible documents across all partitions and the tail. pub fn doc_count(&self) -> usize { let st = self.state.load(); @@ -1005,7 +1130,7 @@ impl FtsMemIndex { /// Whether there are any visible documents. pub fn is_empty(&self) -> bool { let st = self.state.load(); - st.partitions.is_empty() && st.tail.visible_count() == 0 + st.partitions.is_empty() && st.tail.doc_count() == 0 } /// Total number of visible (term, doc) postings. @@ -1086,30 +1211,20 @@ impl FtsMemIndex { fn insert_batch(&self, batch: &RecordBatch, row_offset: u64) -> Result<()> { let st = self.state.load_full(); - let batch_position = st.tail.next_position(); - - let Some(col_idx) = batch - .schema() - .column_with_name(&self.column_name) - .map(|(idx, _)| idx) - else { - // Column missing: nothing to index, but publish an empty batch so - // the tail's visibility counters keep up with the writer. - st.tail.append_batch( - batch_position, - row_offset, - batch.num_rows() as u32, - vec![0; batch.num_rows()], - 0, - FxHashMap::default(), - self.params.has_positions(), - ); - return Ok(()); - }; - - let column = batch.column(col_idx); - let texts = extract_texts(column.as_ref())?; - debug_assert_eq!(texts.len(), batch.num_rows()); + let document_position_start = st.tail.doc_count(); + if self.resolved_field.get().is_none() { + let schema = LanceSchema::try_from(batch.schema().as_ref())?; + let resolved = resolve_fts_field( + &schema, + &self.source_column_name, + self.params.get_document_granularity(), + )?; + let _ = self.resolved_field.set(resolved); + } + let resolved = self.resolved_field.get().ok_or_else(|| { + Error::internal("MemWAL FTS traversal was not initialized".to_string()) + })?; + let extracted_documents = resolved.documents_from_batch(batch)?; let mut tok_guard = self .writer_tokenizer @@ -1122,55 +1237,50 @@ impl FtsMemIndex { // per-document map and per-`(term, doc)` `Vec` allocation that // dominated insert cost. `FxHashMap` skips SipHash on the hot lookup. let mut term_builders: FxHashMap, BatchTermBuilder> = FxHashMap::default(); - let mut doc_lengths: Vec = Vec::with_capacity(batch.num_rows()); + let mut documents: Vec = Vec::with_capacity(batch.num_rows()); let mut total_tokens: u64 = 0; - - for (local_doc_idx, text_opt) in texts.iter().enumerate() { - // Track each doc's token count even for null/missing rows so the - // dense `doc_lengths` array stays aligned with `row_offset + i`. - let mut doc_token_count: u32 = 0; - let row_position = row_offset + local_doc_idx as u64; - - if let Some(text) = text_opt { - let mut stream = tokenizer.token_stream_for_doc(text); - let mut position: u32 = 0; - while let Some(tok) = stream.next() { - let term = tok.text.as_str(); - // One hash lookup per token: extend the term's builder, or - // intern its `Arc` once on first sight this batch. - if let Some(builder) = term_builders.get_mut(term) { - builder.observe(row_position, position); - } else { - term_builders.insert( - Arc::::from(term), - BatchTermBuilder::with_first(row_position, position), - ); - } - position += 1; - doc_token_count += 1; - } + let preserve_zero_token_documents = + self.params.get_document_granularity().is_list_element(); + let mut index_document = |key: DocumentKey, text: &str| -> Result<()> { + let document_position = document_position_start + documents.len() as u64; + let num_tokens = index_text(text, document_position, tokenizer, &mut term_builders)?; + if preserve_zero_token_documents || num_tokens > 0 { + documents.push(DocumentMetadata { key, num_tokens }); + total_tokens += num_tokens as u64; } + Ok(()) + }; - doc_lengths.push(doc_token_count); - total_tokens += doc_token_count as u64; + for document in extracted_documents { + index_document( + DocumentKey { + row_position: row_offset + document.row_index as u64, + doc_index: document.doc_index, + }, + &document.text, + )?; + } + + if documents.is_empty() { + return Ok(()); } // Drop the tokenizer guard before publishing so we don't hold it // across the snapshot install. drop(tok_guard); + let batch_position = st.tail.next_position(); st.tail.append_batch( batch_position, - row_offset, - batch.num_rows() as u32, - doc_lengths, + document_position_start, + documents, total_tokens, term_builders, self.params.has_positions(), ); if st.tail.doc_count() >= self.freeze_threshold_rows as u64 { - self.freeze(&st); + self.freeze(&st)?; } Ok(()) } @@ -1178,37 +1288,42 @@ impl FtsMemIndex { /// Freeze the current tail into a new immutable partition and publish a /// fresh empty tail. Only the writer calls this; readers snapshotting the /// old `IndexState` keep a consistent view across the freeze. - fn freeze(&self, st: &IndexState) { - let Some(partition) = Partition::from_tail(&st.tail) else { - return; + fn freeze(&self, st: &IndexState) -> Result<()> { + let Some(partition) = Partition::from_tail(&st.tail)? else { + return Ok(()); }; let mut partitions: Vec> = st.partitions.iter().cloned().collect(); partitions.push(Arc::new(partition)); // Fold in any completed background merge before re-evaluating tiers. - self.install_pending_merge(&mut partitions); + self.install_pending_merge(&mut partitions)?; if partitions.len() > Self::MAX_PARTITIONS { - partitions = vec![Arc::new(Partition::merge(&partitions))]; + partitions = vec![Arc::new(Partition::merge(&partitions)?)]; } self.state.store(Arc::new(IndexState { partitions: Arc::from(partitions.into_boxed_slice()), tail: TailIndex::new(), })); // Kick off a background merge if a size tier is now over-full. - self.maybe_start_merge(); + self.maybe_start_merge()?; + Ok(()) } /// Install a completed background merge into `partitions` (in place): /// drop the merged-away source partitions and append the merged one. /// No-op while a merge is still running or none is pending. - fn install_pending_merge(&self, partitions: &mut Vec>) { - let mut guard = self.merge.lock().expect("merge slot poisoned"); - let Some(pending) = guard.as_ref() else { - return; + fn install_pending_merge(&self, partitions: &mut Vec>) -> Result<()> { + let mut guard = self + .merge + .lock() + .map_err(|_| Error::internal("FTS merge slot mutex poisoned"))?; + let Some(pending) = guard.as_mut() else { + return Ok(()); }; - let Some(merged) = pending.result.clone() else { - return; // still running + let Some(merged) = pending.result.take() else { + return Ok(()); // still running }; let sources: HashSet = pending.sources.iter().copied().collect(); + *guard = None; // Install only if every source is still live. If a synchronous // `MAX_PARTITIONS` collapse merged the sources away while this merge // ran, the merged docs are already present — appending it would @@ -1221,19 +1336,22 @@ impl FtsMemIndex { partitions.retain(|p| !sources.contains(&(Arc::as_ptr(p) as usize))); partitions.push(merged); } - *guard = None; + Ok(()) } /// If no merge is in flight and some size tier holds at least /// `MERGE_FACTOR` partitions, dispatch their merge to a background thread. - fn maybe_start_merge(&self) { - let mut guard = self.merge.lock().expect("merge slot poisoned"); + fn maybe_start_merge(&self) -> Result<()> { + let mut guard = self + .merge + .lock() + .map_err(|_| Error::internal("FTS merge slot mutex poisoned"))?; if guard.is_some() { - return; // one merge at a time + return Ok(()); // one merge at a time } let partitions = self.state.load(); let Some(group) = select_merge_group(&partitions.partitions, Self::MERGE_FACTOR) else { - return; + return Ok(()); }; let sources: Vec = group.iter().map(|p| Arc::as_ptr(p) as usize).collect(); *guard = Some(PendingMerge { @@ -1247,13 +1365,9 @@ impl FtsMemIndex { // `group`, the source partitions), so it is safe even if the index is // dropped mid-merge. std::thread::spawn(move || { - let merged = Arc::new(Partition::merge(&group)); - if let Ok(mut g) = slot.lock() - && let Some(p) = g.as_mut() - { - p.result = Some(merged); - } + publish_pending_merge(slot.as_ref(), Partition::merge(&group)); }); + Ok(()) } // ------------------------------------------------------------------ @@ -1267,7 +1381,7 @@ impl FtsMemIndex { /// use `search_with_options` for sorted/limited output. pub fn search(&self, term: &str) -> Vec { let st = self.state.load_full(); - let tokens = self.tokenize_for_search(term); + let tokens = self.analyze_for_search(term); self.search_match(&st, &tokens, Operator::Or, None, true, true) } @@ -1275,7 +1389,7 @@ impl FtsMemIndex { /// `slop` intervening tokens between consecutive query tokens. pub fn search_phrase(&self, phrase: &str, slop: u32) -> Vec { let st = self.state.load_full(); - let tokens = self.tokenize_for_search(phrase); + let tokens = self.analyze_for_search(phrase); self.search_phrase_tokens(&st, &tokens, slop, true) } @@ -1285,8 +1399,10 @@ impl FtsMemIndex { /// when the tail is empty. Writer-side: callers hold the single-writer role. pub fn flush(&self) { let st = self.state.load_full(); - if st.tail.visible_count() > 0 { - self.freeze(&st); + if st.tail.visible_count() > 0 + && let Err(error) = self.freeze(&st) + { + tracing::error!(?error, "failed to freeze the FTS MemWAL tail"); } } @@ -1322,6 +1438,22 @@ impl FtsMemIndex { /// shared rising threshold (instead of every partition cold-starting). /// Without a limit, an exact O(matches) scan across partitions + tail. fn search_match( + &self, + st: &IndexState, + query_tokens: &Tokens, + operator: Operator, + limit: Option, + include_tail: bool, + tail_skip: bool, + ) -> Vec { + if operator == Operator::And && has_grouped_positions(query_tokens) { + return self.search_grouped_and(st, query_tokens, limit, include_tail, tail_skip); + } + let tokens = query_tokens_to_vec(query_tokens); + self.search_match_strings(st, &tokens, operator, limit, include_tail, tail_skip) + } + + fn search_match_strings( &self, st: &IndexState, tokens: &[String], @@ -1365,7 +1497,7 @@ impl FtsMemIndex { &scorer, theta, ) { - topk.offer(e.score, e.row_position); + topk.offer(e.score, e.key()); } } topk.into_entries() @@ -1390,19 +1522,83 @@ impl FtsMemIndex { } } + fn search_grouped_and( + &self, + st: &IndexState, + query_tokens: &Tokens, + limit: Option, + include_tail: bool, + tail_skip: bool, + ) -> Vec { + let mut result_map: Option> = None; + for group in query_position_groups(query_tokens) { + let group_results = + self.search_match_strings(st, &group, Operator::Or, None, include_tail, tail_skip); + let group_map = group_results + .into_iter() + .map(|entry| (entry.key(), entry.score)) + .collect::>(); + let Some(current) = result_map.as_mut() else { + result_map = Some(group_map); + continue; + }; + current.retain(|key, score| { + if let Some(group_score) = group_map.get(key) { + *score += group_score; + true + } else { + false + } + }); + if current.is_empty() { + return Vec::new(); + } + } + + let mut results = result_map + .unwrap_or_default() + .into_iter() + .map(|(key, score)| FtsEntry { + row_position: key.row_position, + doc_index: public_doc_index(&key.doc_index), + score, + }) + .collect::>(); + if let Some(limit) = limit { + results.sort_by(|a, b| { + b.score + .partial_cmp(&a.score) + .unwrap_or(std::cmp::Ordering::Equal) + }); + results.truncate(limit); + } + results + } + fn search_phrase_tokens( &self, st: &IndexState, - tokens: &[String], + query_tokens: &Tokens, slop: u32, include_tail: bool, ) -> Vec { - if tokens.is_empty() { + if query_tokens.is_empty() { return Vec::new(); } - if tokens.len() == 1 { + let groups = query_position_groups(query_tokens); + if groups.is_empty() { + return Vec::new(); + } + if groups.len() == 1 { // A single-token phrase reduces to a regular term search. - return self.search_match(st, tokens, Operator::Or, None, include_tail, true); + return self.search_match_strings( + st, + &groups[0], + Operator::Or, + None, + include_tail, + true, + ); } // A multi-token phrase needs token positions; without them (the index // was built `with_position = false`) phrase search is unsupported, as @@ -1410,24 +1606,40 @@ impl FtsMemIndex { if !self.params.has_positions() { return Vec::new(); } + let has_grouped_terms = groups.iter().any(|group| group.len() > 1); + let tokens = position_groups_to_tokens(&groups); let tail_snap = st.tail.snapshot(); let scan_tail = include_tail && tail_snap.visible_count > 0; - let scorer = build_scorer(st, &tail_snap, tokens, include_tail); + let scorer = build_scorer(st, &tail_snap, &tokens, include_tail); if scorer.num_docs() == 0 { return Vec::new(); } let mut results = Vec::new(); for p in st.partitions.iter() { - results.extend(p.search_phrase(tokens, slop, &scorer)); + if has_grouped_terms { + results.extend(p.search_phrase_groups(&groups, slop, &scorer)); + } else { + results.extend(p.search_phrase(&tokens, slop, &scorer)); + } } if scan_tail { - results.extend(phrase_search_tail( - &tail_snap, - &st.tail.terms, - tokens, - slop, - &scorer, - )); + if has_grouped_terms { + results.extend(phrase_search_tail_groups( + &tail_snap, + &st.tail.terms, + &groups, + slop, + &scorer, + )); + } else { + results.extend(phrase_search_tail( + &tail_snap, + &st.tail.terms, + &tokens, + slop, + &scorer, + )); + } } results } @@ -1464,7 +1676,7 @@ impl FtsMemIndex { if expanded.is_empty() { return Vec::new(); } - self.search_match(st, &expanded, Operator::Or, None, include_tail, true) + self.search_match_strings(st, &expanded, Operator::Or, None, include_tail, true) } /// Expand `term` against the term dictionaries of every partition (and the @@ -1556,14 +1768,14 @@ impl FtsMemIndex { operator, boost, } => { - let tokens = self.tokenize_for_search(query); + let tokens = self.analyze_for_search(query); let mut results = self.search_match(st, &tokens, *operator, limit, include_tail, tail_skip); apply_boost(&mut results, *boost); results } FtsQueryExpr::Phrase { query, slop, boost } => { - let tokens = self.tokenize_for_search(query); + let tokens = self.analyze_for_search(query); let mut results = self.search_phrase_tokens(st, &tokens, *slop, include_tail); apply_boost(&mut results, *boost); results @@ -1660,12 +1872,10 @@ impl FtsMemIndex { return results; }; let negative_results = self.search_query_with_state(neg, st, None, include_tail, true); - let negative_set: HashSet = negative_results - .into_iter() - .map(|e| e.row_position) - .collect(); + let negative_set: HashSet = + negative_results.iter().map(FtsEntry::key).collect(); for entry in &mut results { - if negative_set.contains(&entry.row_position) { + if negative_set.contains(&entry.key()) { entry.score *= negative_boost; } } @@ -1680,32 +1890,32 @@ impl FtsMemIndex { st: &IndexState, include_tail: bool, ) -> Vec { - let excluded: HashSet = must_not + let excluded: HashSet = must_not .iter() .flat_map(|q| self.search_query_with_state(q, st, None, include_tail, true)) - .map(|e| e.row_position) + .map(|entry| entry.key()) .collect(); - let mut result_map: HashMap = if must.is_empty() { - let mut map: HashMap = HashMap::new(); + let mut result_map: HashMap = if must.is_empty() { + let mut map: HashMap = HashMap::new(); for q in should { for entry in self.search_query_with_state(q, st, None, include_tail, true) { - *map.entry(entry.row_position).or_default() += entry.score; + *map.entry(entry.key()).or_default() += entry.score; } } map } else { let first_results = self.search_query_with_state(&must[0], st, None, include_tail, true); - let mut map: HashMap = first_results + let mut map: HashMap = first_results .into_iter() - .map(|e| (e.row_position, e.score)) + .map(|entry| (entry.key(), entry.score)) .collect(); for q in must.iter().skip(1) { let results = self.search_query_with_state(q, st, None, include_tail, true); - let result_set: HashMap = results + let result_set: HashMap = results .into_iter() - .map(|e| (e.row_position, e.score)) + .map(|entry| (entry.key(), entry.score)) .collect(); map = map .into_iter() @@ -1714,7 +1924,7 @@ impl FtsMemIndex { } for q in should { for entry in self.search_query_with_state(q, st, None, include_tail, true) { - if let Some(score) = map.get_mut(&entry.row_position) { + if let Some(score) = map.get_mut(&entry.key()) { *score += entry.score; } } @@ -1728,21 +1938,28 @@ impl FtsMemIndex { result_map .into_iter() - .map(|(row_position, score)| FtsEntry { - row_position, + .map(|(key, score)| FtsEntry { + row_position: key.row_position, + doc_index: public_doc_index(&key.doc_index), score, }) .collect() } fn tokenize_for_search(&self, text: &str) -> Vec { + query_tokens_to_vec(&self.analyze_for_search(text)) + } + + fn analyze_for_search(&self, text: &str) -> Tokens { let mut tok = PooledTokenizer::new(&self.tokenizer_pool); let mut stream = tok.get_mut().token_stream_for_search(text); - let mut out = Vec::new(); + let mut tokens = Vec::new(); + let mut positions = Vec::new(); while let Some(t) = stream.next() { - out.push(t.text.clone()); + tokens.push(t.text.clone()); + positions.push(t.position as u32); } - out + Tokens::with_positions(tokens, positions, DocType::Text) } // ------------------------------------------------------------------ @@ -1752,8 +1969,9 @@ impl FtsMemIndex { /// Export the in-memory FTS index to an `InnerBuilder` ready to be /// written to disk. /// - /// Doc row positions are kept in insert order to match the forward-written - /// flush data file 1:1. `total_rows` is used only to validate positions. + /// Stored documents are kept in insert order and retain their positions in + /// the forward-written flush data file. `total_rows` is used only to + /// validate positions. pub fn to_index_builder( &self, partition_id: u64, @@ -1764,51 +1982,58 @@ impl FtsMemIndex { let st = self.state.load_full(); let with_position = self.params.has_positions(); + let block_size = self.params.posting_block_size(); let format_version = self.params.resolved_format_version(); let posting_tail_codec = format_version.posting_tail_codec(); let total_rows_u64 = total_rows as u64; - // Step 1: collect (original_pos, num_tokens) for every doc across all + // Step 1: collect (document key, num_tokens) for every doc across all // immutable partitions and the visible tail. - let mut all_docs: Vec<(u64, u32)> = Vec::new(); + let mut all_docs: Vec<(DocumentKey, u32)> = Vec::new(); for p in st.partitions.iter() { - for (row_pos, num_tokens) in p.docs.iter() { - all_docs.push((*row_pos, *num_tokens)); + for (doc_id, (_, num_tokens)) in p.docs.iter().enumerate() { + all_docs.push((doc_set_key(&p.docs, doc_id as u32), *num_tokens)); } } let tail_snap = st.tail.snapshot(); for batch in tail_snap.batches.iter().take(tail_snap.visible_count) { - for i in 0..batch.rows as usize { - all_docs.push((batch.row_offset + i as u64, batch.doc_lengths[i])); + for document in &batch.documents { + all_docs.push((document.key.clone(), document.num_tokens)); } } if all_docs.is_empty() { - return Ok(InnerBuilder::new_with_format_version( + return Ok(InnerBuilder::new_with_format_version_and_block_size( partition_id, with_position, Default::default(), format_version, + block_size, )); } - // Step 2: assign doc_ids in ascending insert-position order, so the - // stored row positions line up 1:1 with the forward-written data file. - let mut entries: Vec<(u64, u32)> = Vec::with_capacity(all_docs.len()); - for (original, num_tokens) in &all_docs { - if *original >= total_rows_u64 { + // Step 2: assign doc_ids in ascending insert-position order while + // preserving each document's position in the forward-written data file. + let mut entries: Vec<(DocumentKey, u32)> = Vec::with_capacity(all_docs.len()); + for (key, num_tokens) in &all_docs { + if key.row_position >= total_rows_u64 { return Err(Error::io(format!( "FTS flush: row position {} >= total_rows {}", - original, total_rows + key.row_position, total_rows ))); } - entries.push((*original, *num_tokens)); + entries.push((key.clone(), *num_tokens)); } - entries.sort_by_key(|(original, _)| *original); + entries.sort_by(|left, right| left.0.cmp(&right.0)); let mut docs = DocSet::default(); - let mut original_to_doc_id: HashMap = HashMap::with_capacity(entries.len()); - for (original, num_tokens) in &entries { - let doc_id = docs.append(*original, *num_tokens); - original_to_doc_id.insert(*original, doc_id); + let mut original_to_doc_id: HashMap = + HashMap::with_capacity(entries.len()); + for (key, num_tokens) in &entries { + let doc_id = if !key.doc_index.is_empty() { + docs.append_with_doc_index(key.row_position, *num_tokens, &key.doc_index)? + } else { + docs.append(key.row_position, *num_tokens) + }; + original_to_doc_id.insert(key.clone(), doc_id); } // Step 3: merge per-term postings across every partition and the tail. @@ -1819,8 +2044,8 @@ impl FtsMemIndex { let bucket = term_postings.entry(term.to_string()).or_default(); let mut cursor = PostingCursor::new(p, term_id); while let Some(local_doc) = cursor.doc() { - let row_pos = p.docs.row_id(local_doc); - if let Some(&doc_id) = original_to_doc_id.get(&row_pos) { + let key = doc_set_key(&p.docs, local_doc); + if let Some(&doc_id) = original_to_doc_id.get(&key) { let pos = if with_position { Some(cursor.positions().to_vec()) } else { @@ -1840,8 +2065,11 @@ impl FtsMemIndex { if chunk.batch_position >= tail_snap.visible_count { continue; } - for (i, row_position) in chunk.row_positions.iter().enumerate() { - let Some(&doc_id) = original_to_doc_id.get(row_position) else { + for (i, document_position) in chunk.row_positions.iter().enumerate() { + let Some(document) = lookup_document(&tail_snap, *document_position) else { + continue; + }; + let Some(&doc_id) = original_to_doc_id.get(&document.key) else { continue; }; let pos = if with_position { @@ -1873,10 +2101,13 @@ impl FtsMemIndex { docs_for_term.sort_by_key(|(doc_id, _, _)| *doc_id); let token_id = tokens.add(token) as usize; debug_assert_eq!(token_id, posting_lists.len()); - posting_lists.push(PostingListBuilder::new_with_posting_tail_codec( - with_position, - posting_tail_codec, - )); + posting_lists.push( + PostingListBuilder::new_with_posting_tail_codec_and_block_size( + with_position, + posting_tail_codec, + block_size, + ), + ); let plb = &mut posting_lists[token_id]; for (doc_id, freq, pos) in docs_for_term { let recorder = if with_position { @@ -1888,11 +2119,12 @@ impl FtsMemIndex { } } - let mut builder = InnerBuilder::new_with_format_version( + let mut builder = InnerBuilder::new_with_format_version_and_block_size( partition_id, with_position, Default::default(), format_version, + block_size, ); builder.set_tokens(tokens); builder.set_docs(docs); @@ -1979,58 +2211,59 @@ impl BatchTermBuilder { } } -/// Borrowed text for a row, or `None` for null/missing. -type TextOpt<'a> = Option<&'a str>; - -fn extract_texts(column: &dyn Array) -> Result>> { - match column.data_type() { - DataType::Utf8 => { - let array = column - .as_any() - .downcast_ref::() - .expect("Utf8 array"); - Ok((0..array.len()) - .map(|i| (!array.is_null(i)).then(|| array.value(i))) - .collect()) - } - DataType::LargeUtf8 => { - let array = column - .as_any() - .downcast_ref::() - .expect("LargeUtf8 array"); - Ok((0..array.len()) - .map(|i| (!array.is_null(i)).then(|| array.value(i))) - .collect()) - } - DataType::Utf8View => { - let array = column - .as_any() - .downcast_ref::() - .expect("Utf8View array"); - Ok((0..array.len()) - .map(|i| (!array.is_null(i)).then(|| array.value(i))) - .collect()) - } - other => Err(Error::invalid_input(format!( - "FTS index only supports Utf8, LargeUtf8, and Utf8View columns; got {other:?}" - ))), +fn index_text( + text: &str, + document_position: u64, + tokenizer: &mut dyn LanceTokenizer, + term_builders: &mut FxHashMap, BatchTermBuilder>, +) -> Result { + let mut stream = tokenizer.token_stream_for_doc(text); + let mut num_tokens = 0u32; + while let Some(token) = stream.next() { + let position = u32::try_from(token.position).map_err(|_| { + Error::invalid_input(format!( + "token position overflow for document_position={document_position}: token_position={}", + token.position + )) + })?; + let term = token.text.as_str(); + if let Some(builder) = term_builders.get_mut(term) { + builder.observe(document_position, position); + } else { + term_builders.insert( + Arc::::from(term), + BatchTermBuilder::with_first(document_position, position), + ); + } + num_tokens = num_tokens.checked_add(1).ok_or_else(|| { + Error::invalid_input(format!( + "token count overflow for document_position={document_position}" + )) + })?; } + Ok(num_tokens) } fn has_visible_chunk(slice: &TermSlice, visible_count: usize) -> bool { slice.chunks().any(|c| c.batch_position < visible_count) } -fn lookup_dl(snap: &Snapshot, row_position: u64) -> Option { - snap.batches.iter().find_map(|b| b.dl(row_position)) +fn lookup_document(snap: &Snapshot, document_position: u64) -> Option { + snap.batches + .iter() + .find_map(|batch| batch.document(document_position).cloned()) +} + +fn lookup_dl(snap: &Snapshot, document_position: u64) -> Option { + lookup_document(snap, document_position).map(|document| document.num_tokens) } fn find_doc_in_chunks( chunks: &[Arc], - row_position: u64, + document_position: u64, ) -> Option<(&Arc, usize)> { for chunk in chunks { - if let Ok(idx) = chunk.row_positions.binary_search(&row_position) { + if let Ok(idx) = chunk.row_positions.binary_search(&document_position) { return Some((chunk, idx)); } } @@ -2137,9 +2370,8 @@ fn score_terms( if tail_ub <= theta { return Vec::new(); } - let mut doc_scores: HashMap = HashMap::new(); - let mut doc_hits: Option> = - (operator == Operator::And).then(HashMap::new); + let mut doc_scores: HashMap = HashMap::new(); + let mut doc_hits: Option> = (operator == Operator::And).then(HashMap::new); for (qw, slice) in tail_terms { for chunk in slice.chunks() { if chunk.batch_position >= snap.visible_count { @@ -2148,28 +2380,31 @@ fn score_terms( let Some(meta) = snap.batch_for(chunk.batch_position) else { continue; }; - for (i, &row_position) in chunk.row_positions.iter().enumerate() { - let dl = meta.dl(row_position).unwrap_or(1); + for (i, &document_position) in chunk.row_positions.iter().enumerate() { + let dl = meta.dl(document_position).unwrap_or(1); let score = qw * scorer.doc_weight(chunk.frequencies[i], dl); - *doc_scores.entry(row_position).or_default() += score; + *doc_scores.entry(document_position).or_default() += score; if let Some(doc_hits) = &mut doc_hits { - *doc_hits.entry(row_position).or_default() += 1; + *doc_hits.entry(document_position).or_default() += 1; } } } } doc_scores .into_iter() - .filter(|(row_position, _)| { + .filter(|(document_position, _)| { operator == Operator::Or || doc_hits .as_ref() - .and_then(|doc_hits| doc_hits.get(row_position)) + .and_then(|doc_hits| doc_hits.get(document_position)) .is_some_and(|hits| *hits >= tokens.len() as u32) }) - .map(|(row_position, score)| FtsEntry { - row_position, - score, + .filter_map(|(document_position, score)| { + lookup_document(snap, document_position).map(|document| FtsEntry { + row_position: document.key.row_position, + doc_index: public_doc_index(&document.key.doc_index), + score, + }) }) .collect() } @@ -2214,7 +2449,7 @@ fn phrase_search_tail( let mut results = Vec::new(); for chunk in &per_token_chunks[smallest_idx] { - for (doc_idx, &row_position) in chunk.row_positions.iter().enumerate() { + for (doc_idx, &document_position) in chunk.row_positions.iter().enumerate() { let Some(pos) = chunk .positions .as_ref() @@ -2231,7 +2466,7 @@ fn phrase_search_tail( if ti == smallest_idx { continue; } - match find_doc_in_chunks(chunks, row_position) { + match find_doc_in_chunks(chunks, document_position) { Some((c, other_idx)) => { frequencies[ti] = c.frequencies[other_idx]; all_positions[ti] = c @@ -2249,21 +2484,148 @@ fn phrase_search_tail( if !all_present || !phrase_matches(&all_positions, slop) { continue; } - let dl = lookup_dl(snap, row_position).unwrap_or(1); + let dl = lookup_dl(snap, document_position).unwrap_or(1); let score: f32 = tokens .iter() .enumerate() .map(|(ti, tok)| scorer.query_weight(tok) * scorer.doc_weight(frequencies[ti], dl)) .sum(); - results.push(FtsEntry { - row_position, - score, - }); + if let Some(document) = lookup_document(snap, document_position) { + results.push(FtsEntry { + row_position: document.key.row_position, + doc_index: public_doc_index(&document.key.doc_index), + score, + }); + } } } results } +#[derive(Default, Clone)] +struct PhraseGroupDoc { + positions: Vec, + score: f32, +} + +struct PhraseCandidate { + positions_by_group: Vec>, + score: f32, +} + +fn merge_phrase_group( + candidates: &mut Option>, + group_idx: usize, + group_count: usize, + group_docs: HashMap, +) -> bool +where + K: Eq + std::hash::Hash, +{ + let Some(current) = candidates.as_mut() else { + *candidates = Some( + group_docs + .into_iter() + .map(|(doc, group_doc)| { + let mut positions_by_group = vec![Vec::new(); group_count]; + positions_by_group[group_idx] = group_doc.positions; + ( + doc, + PhraseCandidate { + positions_by_group, + score: group_doc.score, + }, + ) + }) + .collect(), + ); + return true; + }; + + current.retain(|doc, candidate| { + if let Some(group_doc) = group_docs.get(doc) { + candidate.positions_by_group[group_idx] = group_doc.positions.clone(); + candidate.score += group_doc.score; + true + } else { + false + } + }); + !current.is_empty() +} + +fn phrase_search_tail_groups( + snap: &Snapshot, + terms: &SkipMap, Arc>>, + groups: &[Vec], + slop: u32, + scorer: &MemBM25Scorer, +) -> Vec { + let mut candidates: Option> = None; + for (group_idx, group) in groups.iter().enumerate() { + let group_docs = tail_phrase_group_docs(snap, terms, group, scorer); + if group_docs.is_empty() + || !merge_phrase_group(&mut candidates, group_idx, groups.len(), group_docs) + { + return Vec::new(); + } + } + + candidates + .unwrap_or_default() + .into_iter() + .filter(|(_, candidate)| phrase_matches(&candidate.positions_by_group, slop)) + .map(|(key, candidate)| FtsEntry { + row_position: key.row_position, + doc_index: public_doc_index(&key.doc_index), + score: candidate.score, + }) + .collect() +} + +fn tail_phrase_group_docs( + snap: &Snapshot, + terms: &SkipMap, Arc>>, + group: &[String], + scorer: &MemBM25Scorer, +) -> HashMap { + let mut docs: HashMap = HashMap::new(); + for token in group { + let Some(entry) = terms.get(token.as_str()) else { + continue; + }; + let qw = scorer.query_weight(token); + let slice = entry.value().load_full(); + for chunk in slice.chunks() { + if chunk.batch_position >= snap.visible_count { + continue; + } + let Some(meta) = snap.batch_for(chunk.batch_position) else { + continue; + }; + let Some(positions) = &chunk.positions else { + continue; + }; + for (i, &document_position) in chunk.row_positions.iter().enumerate() { + let Some(document) = meta.document(document_position) else { + continue; + }; + let entry = docs.entry(document.key.clone()).or_default(); + entry + .positions + .extend_from_slice(positions.doc_positions(i)); + let dl = document.num_tokens; + entry.score += qw * scorer.doc_weight(chunk.frequencies[i], dl); + } + } + } + for doc in docs.values_mut() { + doc.positions.sort_unstable(); + doc.positions.dedup(); + } + docs +} + fn phrase_matches>(positions: &[T], slop: u32) -> bool { if positions.is_empty() { return false; @@ -2314,6 +2676,7 @@ pub struct FtsIndexConfig { pub field_id: i32, pub column: String, pub params: InvertedIndexParams, + pub(crate) resolved_field: Option>, } impl FtsIndexConfig { @@ -2323,6 +2686,7 @@ impl FtsIndexConfig { field_id, column, params: InvertedIndexParams::default(), + resolved_field: None, } } @@ -2332,12 +2696,29 @@ impl FtsIndexConfig { column: String, params: InvertedIndexParams, ) -> Self { - Self { + Self::try_with_params(name, field_id, column, params) + .expect("invalid MemWAL FTS index config parameters") + } + + pub fn try_with_params( + name: String, + field_id: i32, + column: String, + params: InvertedIndexParams, + ) -> Result { + params.validate_format_version()?; + Ok(Self { name, field_id, column, params, - } + resolved_field: None, + }) + } + + pub(crate) fn with_resolved_field(mut self, resolved_field: ResolvedFtsField) -> Self { + self.resolved_field = Some(Arc::new(resolved_field)); + self } } @@ -2959,21 +3340,31 @@ impl Partition { /// Freeze the visible contents of `tail` into a new partition. Returns /// `None` if the tail has no visible docs. - fn from_tail(tail: &TailIndex) -> Option { + fn from_tail(tail: &TailIndex) -> Result> { let snap = tail.snapshot(); if snap.visible_count == 0 { - return None; + return Ok(None); } // Assign dense local doc ids in row-position order. let mut docs = DocSet::default(); let mut pos_to_doc: HashMap = HashMap::new(); for batch in snap.batches.iter().take(snap.visible_count) { - for i in 0..batch.rows as usize { - let rp = batch.row_offset + i as u64; - let doc_id = docs.append(rp, batch.doc_lengths[i]); - pos_to_doc.insert(rp, doc_id); + for (offset, document) in batch.documents.iter().enumerate() { + let doc_id = if !document.key.doc_index.is_empty() { + docs.append_with_doc_index( + document.key.row_position, + document.num_tokens, + &document.key.doc_index, + )? + } else { + docs.append(document.key.row_position, document.num_tokens) + }; + pos_to_doc.insert(batch.document_position_start + offset as u64, doc_id); } } + if docs.is_empty() { + return Ok(None); + } // Snapshot the term slices (a cheap sequential skip-list walk of Arc // clones), then build each term's sorted posting list in parallel — the // per-term work (chunk traversal + sort) dominates the freeze and is @@ -3010,18 +3401,35 @@ impl Partition { Some((key, docs_for_term)) }) .collect(); - Some(build_partition(entries, docs)) + Ok(Some(build_partition(entries, docs))) } /// Merge several partitions into one. Local doc ids are reassigned by /// concatenation, which keeps each merged per-term posting list sorted. - fn merge(parts: &[Arc]) -> Self { + fn merge(parts: &[Arc]) -> Result { let mut merged: HashMap, Vec<(u32, u32, Vec)>> = HashMap::new(); let mut docs = DocSet::default(); let mut doc_offset: u32 = 0; + let coordinate_rank = parts + .first() + .map_or(0, |partition| partition.docs.coordinate_rank()); for p in parts { - for (rp, nt) in p.docs.iter() { - docs.append(*rp, *nt); + if p.docs.coordinate_rank() != coordinate_rank { + return Err(Error::index(format!( + "cannot merge MemWAL FTS partitions with coordinate ranks {coordinate_rank} and {}", + p.docs.coordinate_rank() + ))); + } + for (doc_id, (row_position, num_tokens)) in p.docs.iter().enumerate() { + if coordinate_rank > 0 { + docs.append_with_doc_index( + *row_position, + *num_tokens, + &p.docs.doc_index(doc_id as u32), + )?; + } else { + docs.append(*row_position, *num_tokens); + } } let terms = p.collect_terms(); for (term_id, term) in terms.into_iter().enumerate() { @@ -3036,7 +3444,7 @@ impl Partition { } doc_offset += p.docs.len() as u32; } - build_partition(merged.into_iter().collect(), docs) + Ok(build_partition(merged.into_iter().collect(), docs)) } /// Exact O(matches) BM25 OR/AND-search of the partition by direct posting @@ -3091,8 +3499,10 @@ impl Partition { if hits[doc] == 0 || (operator == Operator::And && hits[doc] < need) { continue; } + let key = doc_set_key(&self.docs, doc as u32); results.push(FtsEntry { - row_position: self.docs.row_id(doc as u32), + row_position: key.row_position, + doc_index: public_doc_index(&key.doc_index), score: scores[doc], }); } @@ -3141,7 +3551,7 @@ impl Partition { let doc = docs[i]; let dl = self.docs.num_tokens(doc); let score = qw * scorer.doc_weight(freqs[i], dl); - topk.offer(score, self.docs.row_id(doc)); + topk.offer(score, doc_set_key(&self.docs, doc)); } } } @@ -3242,7 +3652,7 @@ impl Partition { } } if alive { - topk.offer(score, self.docs.row_id(cand)); + topk.offer(score, doc_set_key(&self.docs, cand)); } // Advance the essential lanes that were positioned at the candidate. for l in lanes[ne..].iter_mut() { @@ -3304,8 +3714,10 @@ impl Partition { .zip(&freqs) .map(|(t, &f)| scorer.query_weight(t) * scorer.doc_weight(f, dl)) .sum(); + let key = doc_set_key(&self.docs, doc); results.push(FtsEntry { - row_position: self.docs.row_id(doc), + row_position: key.row_position, + doc_index: public_doc_index(&key.doc_index), score, }); } @@ -3313,13 +3725,70 @@ impl Partition { } results } + + fn search_phrase_groups( + &self, + groups: &[Vec], + slop: u32, + scorer: &MemBM25Scorer, + ) -> Vec { + let mut candidates: Option> = None; + for (group_idx, group) in groups.iter().enumerate() { + let group_docs = self.phrase_group_docs(group, scorer); + if group_docs.is_empty() + || !merge_phrase_group(&mut candidates, group_idx, groups.len(), group_docs) + { + return Vec::new(); + } + } + + candidates + .unwrap_or_default() + .into_iter() + .filter(|(_, candidate)| phrase_matches(&candidate.positions_by_group, slop)) + .map(|(doc, candidate)| FtsEntry { + row_position: self.docs.row_id(doc), + doc_index: public_doc_index(&self.docs.doc_index(doc)), + score: candidate.score, + }) + .collect() + } + + fn phrase_group_docs( + &self, + group: &[String], + scorer: &MemBM25Scorer, + ) -> HashMap { + let mut docs: HashMap = HashMap::new(); + for token in group { + let Some(term_id) = self.term_id(token) else { + continue; + }; + let qw = scorer.query_weight(token); + let mut cursor = PostingCursor::new(self, term_id); + while let Some(doc) = cursor.cursor_doc() { + let positions = cursor.positions().to_vec(); + let freq = cursor.freq(); + let dl = self.docs.num_tokens(doc); + let entry = docs.entry(doc).or_default(); + entry.positions.extend_from_slice(&positions); + entry.score += qw * scorer.doc_weight(freq, dl); + cursor.advance(); + } + } + for doc in docs.values_mut() { + doc.positions.sort_unstable(); + doc.positions.dedup(); + } + docs + } } /// A scored MemTable row, ordered by score then row position (`total_cmp`, /// so a stray non-finite score cannot panic the heap). struct ScoredEntry { score: f32, - row_position: u64, + key: DocumentKey, } impl PartialEq for ScoredEntry { @@ -3337,7 +3806,7 @@ impl Ord for ScoredEntry { fn cmp(&self, other: &Self) -> std::cmp::Ordering { self.score .total_cmp(&other.score) - .then(self.row_position.cmp(&other.row_position)) + .then(self.key.cmp(&other.key)) } } @@ -3366,18 +3835,12 @@ impl TopK { } } - fn offer(&mut self, score: f32, row_position: u64) { + fn offer(&mut self, score: f32, key: DocumentKey) { if self.heap.len() < self.k { - self.heap.push(Reverse(ScoredEntry { - score, - row_position, - })); + self.heap.push(Reverse(ScoredEntry { score, key })); } else if score > self.heap.peek().unwrap().0.score { self.heap.pop(); - self.heap.push(Reverse(ScoredEntry { - score, - row_position, - })); + self.heap.push(Reverse(ScoredEntry { score, key })); } } @@ -3385,7 +3848,8 @@ impl TopK { self.heap .into_iter() .map(|Reverse(e)| FtsEntry { - row_position: e.row_position, + row_position: e.key.row_position, + doc_index: public_doc_index(&e.key.doc_index), score: e.score, }) .collect() @@ -3590,8 +4054,11 @@ impl<'a> PostingCursor<'a> { #[cfg(test)] mod tests { use super::*; - use arrow_array::{Int32Array, StringArray}; - use arrow_schema::{DataType, Field, Schema as ArrowSchema}; + use arrow_array::builder::{ListBuilder, StringBuilder}; + use arrow_array::{Array, Int32Array, ListArray, StringArray, StructArray}; + use arrow_buffer::{OffsetBuffer, ScalarBuffer}; + use arrow_schema::{DataType, Field, Fields, Schema as ArrowSchema}; + use lance_index::scalar::inverted::DocumentGranularity; use std::sync::Arc; use std::sync::atomic::AtomicBool; @@ -3628,6 +4095,211 @@ mod tests { .unwrap() } + fn create_element_test_batch() -> RecordBatch { + let mut tags = ListBuilder::new(StringBuilder::new()); + tags.values().append_value("alpha beta"); + tags.values().append_null(); + tags.values().append_value(""); + tags.values().append_value("beta gamma"); + tags.append(true); + tags.values().append_value("alpha"); + tags.values().append_value("beta"); + tags.append(true); + let tags = tags.finish(); + let schema = Arc::new(ArrowSchema::new(vec![ + Field::new("id", DataType::Int32, false), + Field::new("tags", tags.data_type().clone(), true), + ])); + RecordBatch::try_new( + schema, + vec![Arc::new(Int32Array::from(vec![0, 1])), Arc::new(tags)], + ) + .unwrap() + } + + #[test] + fn test_element_document_search_and_flush_identity() { + let params = InvertedIndexParams::default() + .with_position(true) + .document_granularity(DocumentGranularity::ListElement); + let index = FtsMemIndex::try_with_params(1, "tags".to_string(), params) + .unwrap() + .with_freeze_threshold_rows(1); + assert_eq!(index.column_name(), "tags"); + assert_eq!(index.source_column_name(), "tags"); + + index.insert(&create_element_test_batch(), 0).unwrap(); + assert_eq!(index.doc_count(), 6); + + let mut beta = index.search("beta"); + beta.sort_by_key(|entry| (entry.row_position, entry.doc_index.clone())); + assert_eq!( + beta.iter() + .map(|entry| (entry.row_position, entry.doc_index.clone())) + .collect::>(), + vec![(0, Some(vec![0])), (0, Some(vec![3])), (1, Some(vec![1]))] + ); + + let phrase = index.search_phrase("beta gamma", 0); + assert_eq!(phrase.len(), 1); + assert_eq!( + (phrase[0].row_position, phrase[0].doc_index.clone()), + (0, Some(vec![3])) + ); + + let conjunctive = FtsQueryExpr::boolean() + .must(FtsQueryExpr::match_query("alpha")) + .must(FtsQueryExpr::match_query("beta")) + .build(); + let results = index.search_query(&conjunctive); + assert_eq!(results.len(), 1); + assert_eq!( + (results[0].row_position, results[0].doc_index.clone()), + (0, Some(vec![0])) + ); + + index.to_index_builder(7, 2).unwrap(); + } + + #[test] + fn test_nested_element_documents_keep_all_list_ordinals() { + let doc_fields = Fields::from(vec![Field::new("content", DataType::Utf8, true)]); + let doc_values = StructArray::new( + doc_fields.clone(), + vec![Arc::new(StringArray::from(vec!["alpha", "beta", "alpha"]))], + None, + ); + let doc_item = Arc::new(Field::new("item", DataType::Struct(doc_fields), true)); + let docs_type = DataType::List(doc_item.clone()); + let docs = ListArray::new( + doc_item, + OffsetBuffer::new(ScalarBuffer::from(vec![0i32, 1, 3])), + Arc::new(doc_values), + None, + ); + let group_fields = Fields::from(vec![Field::new("docs", docs_type, true)]); + let group_values = StructArray::new(group_fields.clone(), vec![Arc::new(docs)], None); + let group_item = Arc::new(Field::new("item", DataType::Struct(group_fields), true)); + let groups = ListArray::new( + group_item, + OffsetBuffer::new(ScalarBuffer::from(vec![0i32, 2])), + Arc::new(group_values), + None, + ); + let batch = RecordBatch::try_new( + Arc::new(ArrowSchema::new(vec![Field::new( + "groups", + groups.data_type().clone(), + true, + )])), + vec![Arc::new(groups)], + ) + .unwrap(); + let index = FtsMemIndex::try_with_params( + 1, + "groups.docs.content".to_string(), + InvertedIndexParams::default().document_granularity(DocumentGranularity::ListElement), + ) + .unwrap() + .with_freeze_threshold_rows(1); + + index.insert(&batch, 7).unwrap(); + let mut entries = index.search("alpha"); + entries.sort_by_key(|entry| entry.doc_index.clone()); + assert_eq!( + entries + .iter() + .map(|entry| (entry.row_position, entry.doc_index.clone())) + .collect::>(), + vec![(7, Some(vec![0, 0])), (7, Some(vec![1, 1]))] + ); + + index.insert(&batch, 8).unwrap(); + let state = index.state.load_full(); + assert_eq!(state.partitions.len(), 2); + let merged = Partition::merge(&state.partitions).unwrap(); + assert_eq!(merged.docs.coordinate_rank(), 2); + assert_eq!( + (0..merged.docs.len() as u32) + .map(|doc_id| doc_set_key(&merged.docs, doc_id)) + .collect::>(), + vec![ + DocumentKey { + row_position: 7, + doc_index: vec![0, 0], + }, + DocumentKey { + row_position: 7, + doc_index: vec![1, 0], + }, + DocumentKey { + row_position: 7, + doc_index: vec![1, 1], + }, + DocumentKey { + row_position: 8, + doc_index: vec![0, 0], + }, + DocumentKey { + row_position: 8, + doc_index: vec![1, 0], + }, + DocumentKey { + row_position: 8, + doc_index: vec![1, 1], + }, + ] + ); + index.to_index_builder(8, 9).unwrap(); + } + + #[test] + fn test_failed_background_merge_releases_pending_slot() { + let slot = Mutex::new(Some(PendingMerge { + sources: vec![1], + result: None, + })); + + publish_pending_merge( + &slot, + Err(Error::index("inconsistent test partitions".to_string())), + ); + + assert!(slot.lock().unwrap().is_none()); + } + + #[test] + fn test_zero_token_element_documents_remain_addressable_after_flush() { + let mut tags = ListBuilder::new(StringBuilder::new()); + tags.values().append_null(); + tags.values().append_value(""); + tags.values().append_value("!!!"); + tags.append(true); + let tags = tags.finish(); + let batch = RecordBatch::try_new( + Arc::new(ArrowSchema::new(vec![Field::new( + "tags", + tags.data_type().clone(), + true, + )])), + vec![Arc::new(tags)], + ) + .unwrap(); + let index = FtsMemIndex::try_with_params( + 1, + "tags".to_string(), + InvertedIndexParams::default().document_granularity(DocumentGranularity::ListElement), + ) + .unwrap(); + + index.insert(&batch, 0).unwrap(); + index.flush(); + + assert!(!index.is_empty()); + assert_eq!(index.doc_count(), 3); + assert!(index.search("missing").is_empty()); + } + #[test] fn test_fts_index_insert_and_search() { let schema = create_test_schema(); @@ -3652,6 +4324,234 @@ mod tests { assert!(entries.is_empty()); } + #[test] + fn test_code_analyzer_and_query_uses_position_alternatives() { + let schema = create_test_schema(); + let index = FtsMemIndex::with_params( + 1, + "description".to_string(), + InvertedIndexParams::code().split_identifiers(true), + ); + let batch = RecordBatch::try_new( + schema, + vec![ + Arc::new(Int32Array::from(vec![0, 1])), + Arc::new(StringArray::from(vec!["get user name", "getUserName"])), + ], + ) + .unwrap(); + index.insert(&batch, 0).unwrap(); + + let query = FtsQueryExpr::match_query_with_operator("getUserName", Operator::And); + let mut rows = index + .search_query(&query) + .into_iter() + .map(|entry| entry.row_position) + .collect::>(); + rows.sort_unstable(); + assert_eq!(rows, vec![0, 1]); + } + + #[test] + fn test_code_analyzer_queries_keep_element_document_identity() { + let mut tags = ListBuilder::new(StringBuilder::new()); + tags.values().append_value("get"); + tags.values().append_value("x user name"); + tags.append(true); + tags.values().append_value("getUserName"); + tags.append(true); + let tags = tags.finish(); + let batch = RecordBatch::try_new( + Arc::new(ArrowSchema::new(vec![Field::new( + "tags", + tags.data_type().clone(), + true, + )])), + vec![Arc::new(tags)], + ) + .unwrap(); + let index = FtsMemIndex::try_with_params( + 1, + "tags".to_string(), + InvertedIndexParams::code() + .with_position(true) + .split_identifiers(true) + .document_granularity(DocumentGranularity::ListElement), + ) + .unwrap(); + index.insert(&batch, 0).unwrap(); + + let and_query = FtsQueryExpr::match_query_with_operator("getUserName", Operator::And); + let phrase_query = FtsQueryExpr::phrase("getUserName"); + for entries in [ + index.search_query(&and_query), + index.search_query(&phrase_query), + ] { + assert_eq!(entries.len(), 1); + assert_eq!(entries[0].row_position, 1); + assert_eq!(entries[0].doc_index, Some(vec![0])); + } + + index.flush(); + let partition_only = SearchOptions::new().with_include_tail(false); + for entries in [ + index.search_with_options(&and_query, partition_only.clone()), + index.search_with_options(&phrase_query, partition_only), + ] { + assert_eq!(entries.len(), 1); + assert_eq!(entries[0].row_position, 1); + assert_eq!(entries[0].doc_index, Some(vec![0])); + } + } + + fn tail_positions_for(index: &FtsMemIndex, term: &str, row_position: RowPosition) -> Vec { + let st = index.state.load_full(); + let snap = st.tail.snapshot(); + let entry = st.tail.terms.get(term).expect("term should be indexed"); + let slice = entry.value().load(); + for chunk in slice.chunks() { + if chunk.batch_position >= snap.visible_count { + continue; + } + if let Ok(doc_idx) = chunk.row_positions.binary_search(&row_position) { + return chunk + .positions + .as_ref() + .expect("test index stores positions") + .doc_positions(doc_idx) + .to_vec(); + } + } + panic!("term {term} should be present in row {row_position}"); + } + + #[test] + fn test_code_analyzer_phrase_uses_token_positions_and_alternatives() { + let schema = create_test_schema(); + let index = FtsMemIndex::with_params( + 1, + "description".to_string(), + InvertedIndexParams::code() + .with_position(true) + .split_identifiers(true), + ); + let batch = RecordBatch::try_new( + schema, + vec![ + Arc::new(Int32Array::from(vec![0, 1, 2, 3])), + Arc::new(StringArray::from(vec![ + "get user name", + "getUserName", + "get user", + "get fast user name", + ])), + ], + ) + .unwrap(); + index.insert(&batch, 0).unwrap(); + + assert_eq!(tail_positions_for(&index, "getusername", 1), vec![0]); + assert_eq!(tail_positions_for(&index, "get", 1), vec![0]); + assert_eq!(tail_positions_for(&index, "user", 1), vec![1]); + assert_eq!(tail_positions_for(&index, "name", 1), vec![2]); + + let query = FtsQueryExpr::phrase("getUserName"); + assert_eq!(rows(index.search_phrase("getUserName", 0)), vec![0, 1]); + assert_eq!(rows(index.search_query(&query)), vec![0, 1]); + + index.flush(); + let partition_only = SearchOptions::new().with_include_tail(false); + assert_eq!( + rows(index.search_with_options(&query, partition_only)), + vec![0, 1] + ); + } + + #[test] + fn test_zero_token_documents_are_skipped_across_memwal_paths() { + let params = + InvertedIndexParams::new("whitespace".to_string(), lance_tokenizer::Language::English) + .remove_stop_words(true) + .stem(false) + .max_token_length(Some(6)); + let schema = create_test_schema(); + let batch = RecordBatch::try_new( + schema, + vec![ + Arc::new(Int32Array::from(vec![0, 1, 2, 3, 4, 5])), + Arc::new(StringArray::from(vec![ + Some(""), + Some(" "), + Some("the"), + Some("overlength"), + None, + Some("hello"), + ])), + ], + ) + .unwrap(); + let index = FtsMemIndex::with_params(1, "description".to_string(), params.clone()); + index.insert(&batch, 0).unwrap(); + + assert_eq!(index.doc_count(), 1); + let st = index.state.load_full(); + let tail_snap = st.tail.snapshot(); + let tokens = vec!["hello".to_string()]; + let tail_scorer = build_scorer(&st, &tail_snap, &tokens, true); + let expected_scorer = MemBM25Scorer::new(1, 1, HashMap::from([("hello".to_string(), 1)])); + assert_eq!(tail_scorer.total_tokens, 1); + assert_eq!(tail_scorer.num_docs(), 1); + assert_eq!(tail_scorer.num_docs_containing_token("hello"), 1); + assert_eq!( + tail_scorer.avg_doc_length(), + expected_scorer.avg_doc_length() + ); + assert_eq!( + tail_scorer.query_weight("hello"), + expected_scorer.query_weight("hello") + ); + let tail_results = index.search("hello"); + assert_eq!(rows(tail_results.clone()), vec![5]); + let tail_score = tail_results[0].score; + assert!(!index.to_index_builder(0, 6).unwrap().is_empty()); + + index.flush(); + let st = index.state.load_full(); + assert_eq!(st.partitions.len(), 1); + assert_eq!( + st.partitions[0] + .docs + .iter() + .map(|(row_id, num_tokens)| (*row_id, *num_tokens)) + .collect::>(), + vec![(5, 1)] + ); + let frozen_scorer = build_scorer(&st, &st.tail.snapshot(), &tokens, true); + assert_eq!(frozen_scorer.total_tokens, 1); + assert_eq!(frozen_scorer.num_docs(), 1); + assert_eq!(frozen_scorer.num_docs_containing_token("hello"), 1); + assert_eq!( + frozen_scorer.avg_doc_length(), + expected_scorer.avg_doc_length() + ); + assert_eq!( + frozen_scorer.query_weight("hello"), + expected_scorer.query_weight("hello") + ); + let frozen_results = index.search("hello"); + assert_eq!(rows(frozen_results.clone()), vec![5]); + assert!((frozen_results[0].score - tail_score).abs() < f32::EPSILON); + + let all_zero_batch = batch.slice(0, 5); + let all_zero_index = FtsMemIndex::with_params(1, "description".to_string(), params); + all_zero_index.insert(&all_zero_batch, 0).unwrap(); + assert!(all_zero_index.is_empty()); + assert_eq!(all_zero_index.doc_count(), 0); + assert!(all_zero_index.to_index_builder(0, 5).unwrap().is_empty()); + all_zero_index.flush(); + assert!(all_zero_index.state.load().partitions.is_empty()); + } + fn create_phrase_test_batch(schema: &ArrowSchema) -> RecordBatch { RecordBatch::try_new( Arc::new(schema.clone()), @@ -3820,14 +4720,35 @@ mod tests { let batch = create_boolean_test_batch(&schema); index.insert(&batch, 0).unwrap(); + let rust = FtsQueryExpr::match_query("rust").with_boost(2.0); + let programming = FtsQueryExpr::match_query("programming").with_boost(3.0); + let rust_score = index + .search_query(&rust) + .into_iter() + .find(|entry| entry.row_position == 0) + .unwrap() + .score; + let programming_score = index + .search_query(&programming) + .into_iter() + .find(|entry| entry.row_position == 0) + .unwrap() + .score; let query = FtsQueryExpr::boolean() - .must(FtsQueryExpr::match_query("rust")) - .must(FtsQueryExpr::match_query("programming")) + .must(rust.clone()) + .must(programming.clone()) .build(); let entries = index.search_query(&query); assert_eq!(entries.len(), 1); assert_eq!(entries[0].row_position, 0); + let expected_score = rust_score + programming_score; + assert!((entries[0].score - expected_score).abs() < 1e-6); + + let reversed = FtsQueryExpr::boolean().must(programming).must(rust).build(); + let reversed_entries = index.search_query(&reversed); + assert_eq!(reversed_entries.len(), 1); + assert!((reversed_entries[0].score - expected_score).abs() < 1e-6); } #[test] @@ -4666,6 +5587,18 @@ mod tests { assert!(builder.id() > 0 || builder.id() == 42); } + #[test] + fn test_to_index_builder_supports_block_size_256() { + let schema = create_test_schema(); + let params = InvertedIndexParams::default().block_size(256).unwrap(); + let index = FtsMemIndex::try_with_params(1, "description".to_string(), params).unwrap(); + let batch = create_test_batch(&schema); + index.insert(&batch, 0).unwrap(); + + let builder = index.to_index_builder(42, 3).unwrap(); + assert_eq!(builder.id(), 42); + } + #[test] fn test_unsupported_column_type_errors() { let schema = Arc::new(ArrowSchema::new(vec![ @@ -4684,7 +5617,12 @@ mod tests { .unwrap(); let err = index.insert(&batch, 0).unwrap_err(); - assert!(err.to_string().contains("only supports"), "{err}"); + let message = err.to_string(); + assert!( + message.contains("must resolve to Utf8, LargeUtf8, Utf8View, or JSON") + && message.contains("got Int32"), + "{err}" + ); } // ===== Partition-structured redesign ===== diff --git a/rust/lance/src/dataset/mem_wal/manifest.rs b/rust/lance/src/dataset/mem_wal/manifest.rs index 9c2a3aa2163..acfcbcc3a7c 100644 --- a/rust/lance/src/dataset/mem_wal/manifest.rs +++ b/rust/lance/src/dataset/mem_wal/manifest.rs @@ -146,7 +146,7 @@ impl ShardManifestStore { replay_after_wal_entry_position: 0, wal_entry_position_last_seen: 0, current_generation: 1, - flushed_generations: vec![], + sstables: vec![], status: ShardStatus::Active, }; @@ -462,7 +462,7 @@ impl ShardManifestStore { replay_after_wal_entry_position: 0, wal_entry_position_last_seen: 0, current_generation: 1, - flushed_generations: vec![], + sstables: vec![], status: ShardStatus::Active, } }; @@ -619,7 +619,7 @@ mod tests { replay_after_wal_entry_position: 0, wal_entry_position_last_seen: 0, current_generation: 1, - flushed_generations: vec![], + sstables: vec![], status: ShardStatus::Active, } } diff --git a/rust/lance/src/dataset/mem_wal/memtable.rs b/rust/lance/src/dataset/mem_wal/memtable.rs index 77611fd7c47..102271bfd23 100644 --- a/rust/lance/src/dataset/mem_wal/memtable.rs +++ b/rust/lance/src/dataset/mem_wal/memtable.rs @@ -7,7 +7,6 @@ pub mod batch_store; pub mod flush; pub mod scanner; -use std::collections::{HashMap, HashSet}; use std::sync::Arc; use std::time::{Duration, Instant}; @@ -79,13 +78,6 @@ pub struct MemTable { /// Generation number (incremented on flush). generation: u64, - /// WAL batch mapping: batch_position -> (wal_entry_position, position within WAL entry). - wal_batch_mapping: HashMap, - /// Last WAL entry position that has been flushed. - last_flushed_wal_entry_position: u64, - /// Set of batch IDs that have been flushed to WAL. - flushed_batch_positions: HashSet, - /// Primary key bloom filter for staleness detection. pk_bloom_filter: Sbbf, /// Primary key field IDs (for bloom filter updates). @@ -187,6 +179,29 @@ impl MemTable { pk_field_ids: Vec, cache_config: CacheConfig, batch_capacity: usize, + ) -> Result { + Self::with_capacity_at( + schema, + generation, + pk_field_ids, + cache_config, + batch_capacity, + 0, + ) + } + + /// Create a memtable whose batch 0 sits at `global_offset` in the writer's + /// batch sequence. Every memtable after the writer's first is rotated in by + /// `freeze_memtable`, which stamps the outgoing memtable's `global_end()` + /// here so writer-global cursors stay mappable onto local batch positions. + #[allow(clippy::too_many_arguments)] + pub fn with_capacity_at( + schema: Arc, + generation: u64, + pk_field_ids: Vec, + cache_config: CacheConfig, + batch_capacity: usize, + global_offset: usize, ) -> Result { let lance_schema = Schema::try_from(schema.as_ref())?; @@ -205,7 +220,7 @@ impl MemTable { let dataset_uri = format!("memory://{}", Uuid::new_v4()); // Create lock-free batch store - let batch_store = Arc::new(BatchStore::with_capacity(batch_capacity)); + let batch_store = Arc::new(BatchStore::with_capacity_at(batch_capacity, global_offset)); // Create memtable_flush_completion cell immediately so backpressure can // wait on it even before the memtable is frozen. Every memtable will @@ -220,9 +235,6 @@ impl MemTable { cache_config, cached_dataset: RwLock::new(None), generation, - wal_batch_mapping: HashMap::new(), - last_flushed_wal_entry_position: 0, - flushed_batch_positions: HashSet::new(), pk_bloom_filter, pk_field_ids, // Initialize with an empty IndexStore so the visibility cursor has @@ -481,53 +493,41 @@ impl MemTable { self.batch_store.is_full() || self.batch_store.estimated_bytes() >= max_bytes } - /// Get batches visible up to a specific batch position (inclusive). + /// Get the batches in the visible prefix. /// - /// A batch at position `i` is visible if `i <= max_visible_batch_position`. + /// A batch at position `i` is visible if `i < visible_count`. /// /// # Arguments /// - /// * `max_visible_batch_position` - The maximum batch position to include (inclusive) + /// * `visible_count` - Exclusive count of batch positions to include /// /// # Returns /// /// Vector of visible batches. - pub async fn get_visible_batches(&self, max_visible_batch_position: usize) -> Vec { - self.batch_store - .visible_record_batches(max_visible_batch_position) + pub async fn get_visible_batches(&self, visible_count: usize) -> Vec { + self.batch_store.visible_record_batches(visible_count) } - /// Get batch positions visible up to a specific batch position (inclusive). + /// Get the batch positions in the visible prefix. /// /// This is useful for filtering index results by visibility. - pub async fn get_max_visible_batch_positions( - &self, - max_visible_batch_position: usize, - ) -> Vec { - self.batch_store - .max_visible_batch_positions(max_visible_batch_position) + pub async fn get_visible_batch_positions(&self, visible_count: usize) -> Vec { + self.batch_store.visible_batch_positions(visible_count) } /// Check if a specific batch is visible at a given visibility position. /// /// Returns true if the batch is visible, false if not visible or doesn't exist. - pub async fn is_batch_visible( - &self, - batch_position: usize, - max_visible_batch_position: usize, - ) -> bool { + pub async fn is_batch_visible(&self, batch_position: usize, visible_count: usize) -> bool { self.batch_store - .is_batch_visible(batch_position, max_visible_batch_position) + .is_batch_visible(batch_position, visible_count) } /// Scan batches visible up to a specific batch position. /// /// This combines `get_visible_batches` with the scan interface. - pub async fn scan_batches_at_position( - &self, - max_visible_batch_position: usize, - ) -> Result> { - Ok(self.get_visible_batches(max_visible_batch_position).await) + pub async fn scan_batches_at_position(&self, visible_count: usize) -> Result> { + Ok(self.get_visible_batches(visible_count).await) } /// Update the bloom filter with primary keys from a batch. @@ -561,30 +561,6 @@ impl MemTable { Ok(()) } - /// Mark batches as flushed to WAL. - /// - /// Updates the WAL batch mapping for use during MemTable flush. - /// Also updates the batch_store's watermark to the highest flushed batch_position. - pub fn mark_wal_flushed( - &mut self, - batch_positions: &[usize], - wal_entry_position: u64, - positions: &[usize], - ) { - for (idx, &batch_position) in batch_positions.iter().enumerate() { - self.wal_batch_mapping - .insert(batch_position, (wal_entry_position, positions[idx])); - self.flushed_batch_positions.insert(batch_position); - } - self.last_flushed_wal_entry_position = wal_entry_position; - - // Update batch_store watermark to the highest batch_position flushed (inclusive) - if let Some(&max_batch_position) = batch_positions.iter().max() { - self.batch_store - .set_max_flushed_batch_position(max_batch_position); - } - } - /// Get or create a Dataset for reading. /// /// Uses caching based on the configured eventual consistency strategy: @@ -654,7 +630,7 @@ impl MemTable { /// /// This is used when flushing MemTable to persistent storage to ensure /// the flushed data is ordered from newest to oldest. This enables more - /// efficient K-way merge during LSM scan because flushed generations + /// efficient K-way merge during LSM scan because SSTables /// will be pre-sorted in the order needed for deduplication. /// /// The total number of rows in the MemTable is also returned to allow @@ -726,16 +702,6 @@ impl MemTable { self.batch_store.estimated_bytes() + self.pk_bloom_filter.estimated_memory_size() } - /// Get the WAL batch mapping. - pub fn wal_batch_mapping(&self) -> &HashMap { - &self.wal_batch_mapping - } - - /// Get the last flushed WAL entry position. - pub fn last_flushed_wal_entry_position(&self) -> u64 { - self.last_flushed_wal_entry_position - } - /// Get the bloom filter for serialization. pub fn bloom_filter(&self) -> &Sbbf { &self.pk_bloom_filter @@ -757,17 +723,15 @@ impl MemTable { self.indexes.take() } - /// Check if all batches have been flushed to WAL. - pub fn all_flushed_to_wal(&self) -> bool { - self.batch_store.pending_wal_flush_count() == 0 + /// Whether every committed batch in this memtable is WAL-durable, given the + /// writer-global durability cursor. The L0 flush's precondition. + pub fn all_flushed_to_wal(&self, durable: usize) -> bool { + self.batch_store.pending_wal_flush_count(durable) == 0 } - /// Get unflushed batch IDs. - pub fn unflushed_batch_positions(&self) -> Vec { - let batch_count = self.batch_count(); - (0..batch_count) - .filter(|id| !self.flushed_batch_positions.contains(id)) - .collect() + /// Writer-global coordinate one past this memtable's last committed batch. + pub fn global_end(&self) -> usize { + self.batch_store.global_end() } /// Get cache configuration. @@ -785,18 +749,13 @@ impl MemTable { self.batch_store.remaining_capacity() } - /// Check if batch store is full. - pub fn is_batch_store_full(&self) -> bool { - self.batch_store.is_full() - } - /// Create a scanner for querying this MemTable. /// /// # Arguments /// - /// * `max_visible_batch_position` - Maximum batch position visible (inclusive) + /// * `visible_count` - Maximum batch position visible (inclusive) /// - /// The scanner captures the current `max_visible_batch_position` from the + /// The scanner captures the current `visible_count` from the /// `IndexStore` at construction time to ensure consistent visibility. /// /// # Panics @@ -931,29 +890,11 @@ mod tests { assert_eq!(total_rows, 15); } + /// `all_flushed_to_wal(durable)` is the L0 flush's precondition (`flush.rs:171`): + /// false while any committed batch is still un-appended, true once the + /// durability watermark covers every one of them. #[tokio::test] - async fn test_memtable_wal_mapping() { - let schema = create_test_schema(); - let mut memtable = MemTable::new(schema.clone(), 1, vec![]).unwrap(); - - let batch_position = memtable - .insert(create_test_batch(&schema, 10)) - .await - .unwrap(); - assert!(!memtable.all_flushed_to_wal()); - - memtable.mark_wal_flushed(&[batch_position], 5, &[0]); - - assert!(memtable.all_flushed_to_wal()); - assert_eq!( - memtable.wal_batch_mapping().get(&batch_position), - Some(&(5, 0)) - ); - assert_eq!(memtable.last_flushed_wal_entry_position(), 5); - } - - #[tokio::test] - async fn test_memtable_unflushed_batches() { + async fn test_all_flushed_to_wal_tracks_the_durability_watermark() { let schema = create_test_schema(); let mut memtable = MemTable::new(schema.clone(), 1, vec![]).unwrap(); @@ -965,12 +906,16 @@ mod tests { .insert(create_test_batch(&schema, 5)) .await .unwrap(); + assert!(!memtable.all_flushed_to_wal(0), "nothing is durable yet"); - assert_eq!(memtable.unflushed_batch_positions(), vec![batch1, batch2]); - - memtable.mark_wal_flushed(&[batch1], 1, &[0]); + let durable = batch1 + 1; + assert!( + !memtable.all_flushed_to_wal(durable), + "batch2 is still waiting on its WAL append" + ); - assert_eq!(memtable.unflushed_batch_positions(), vec![batch2]); + let durable = batch2 + 1; + assert!(memtable.all_flushed_to_wal(durable)); } #[tokio::test] @@ -992,23 +937,21 @@ mod tests { .await .unwrap(); - // max_visible_batch_position=1 means positions 0 and 1 are visible - let visible = memtable.get_visible_batches(1).await; + // A count of N exposes the prefix [0, N). + let visible = memtable.get_visible_batches(2).await; assert_eq!(visible.len(), 2); let total_rows: usize = visible.iter().map(|b| b.num_rows()).sum(); assert_eq!(total_rows, 15); // 10 + 5 - // max_visible_batch_position=2 means all batches are visible - let visible = memtable.get_visible_batches(2).await; + let visible = memtable.get_visible_batches(3).await; assert_eq!(visible.len(), 3); - // max_visible_batch_position=0 means only position 0 is visible - let visible = memtable.get_visible_batches(0).await; - assert_eq!(visible.len(), 1); + // A count of 0 exposes nothing — not "batch 0". + assert!(memtable.get_visible_batches(0).await.is_empty()); } #[tokio::test] - async fn test_memtable_get_max_visible_batch_positions() { + async fn test_memtable_get_visible_batch_positions() { let schema = create_test_schema(); let mut memtable = MemTable::new(schema.clone(), 1, vec![]).unwrap(); @@ -1026,17 +969,15 @@ mod tests { .await .unwrap(); - // max_visible_batch_position=1 means positions 0 and 1 visible - let visible_ids = memtable.get_max_visible_batch_positions(1).await; + // A count of N exposes the prefix [0, N). + let visible_ids = memtable.get_visible_batch_positions(2).await; assert_eq!(visible_ids, vec![0, 1]); - // max_visible_batch_position=2 means all positions visible - let visible_ids = memtable.get_max_visible_batch_positions(2).await; + let visible_ids = memtable.get_visible_batch_positions(3).await; assert_eq!(visible_ids, vec![0, 1, 2]); - // max_visible_batch_position=0 means only position 0 visible - let visible_ids = memtable.get_max_visible_batch_positions(0).await; - assert_eq!(visible_ids, vec![0]); + // A count of 0 exposes nothing. + assert!(memtable.get_visible_batch_positions(0).await.is_empty()); } #[tokio::test] @@ -1057,14 +998,14 @@ mod tests { .await .unwrap(); // position 2 - // batch_position 0 is visible when max_visible_batch_position >= 0 - assert!(memtable.is_batch_visible(0, 0).await); + // A count of 0 means nothing is visible, batch 0 included. + assert!(!memtable.is_batch_visible(0, 0).await); + + // Batch i is visible once the count exceeds i. assert!(memtable.is_batch_visible(0, 1).await); assert!(memtable.is_batch_visible(0, 2).await); - - // batch_position 2 is only visible when max_visible_batch_position >= 2 assert!(!memtable.is_batch_visible(2, 1).await); - assert!(memtable.is_batch_visible(2, 2).await); + assert!(!memtable.is_batch_visible(2, 2).await); assert!(memtable.is_batch_visible(2, 3).await); // Non-existent batch @@ -1085,12 +1026,21 @@ mod tests { .await .unwrap(); // position 1 - let batches = memtable.scan_batches_at_position(0).await.unwrap(); + let batches = memtable.scan_batches_at_position(1).await.unwrap(); assert_eq!(batches.len(), 1); assert_eq!(batches[0].num_rows(), 10); - let batches = memtable.scan_batches_at_position(1).await.unwrap(); + let batches = memtable.scan_batches_at_position(2).await.unwrap(); assert_eq!(batches.len(), 2); + + // Nothing indexed yet => nothing scannable. + assert!( + memtable + .scan_batches_at_position(0) + .await + .unwrap() + .is_empty() + ); } #[tokio::test] @@ -1101,7 +1051,7 @@ mod tests { assert_eq!(memtable.batch_capacity(), 3); assert_eq!(memtable.remaining_batch_capacity(), 3); - assert!(!memtable.is_batch_store_full()); + assert!(!memtable.batch_store().is_full()); // Fill up the store memtable @@ -1117,7 +1067,7 @@ mod tests { .await .unwrap(); - assert!(memtable.is_batch_store_full()); + assert!(memtable.batch_store().is_full()); assert_eq!(memtable.remaining_batch_capacity(), 0); // Next insert should fail diff --git a/rust/lance/src/dataset/mem_wal/memtable/batch_store.rs b/rust/lance/src/dataset/mem_wal/memtable/batch_store.rs index 054d9b1630e..f16f9bea1fd 100644 --- a/rust/lance/src/dataset/mem_wal/memtable/batch_store.rs +++ b/rust/lance/src/dataset/mem_wal/memtable/batch_store.rs @@ -43,7 +43,9 @@ use std::cell::UnsafeCell; use std::mem::MaybeUninit; use std::sync::atomic::{AtomicUsize, Ordering}; +use arrow::array::ArrayData; use arrow_array::RecordBatch; +use arrow_schema::DataType; /// A batch stored in the lock-free store. #[derive(Clone)] @@ -75,14 +77,56 @@ impl StoredBatch { } /// Estimate the memory size of a RecordBatch. + /// + /// Sums each column's slice-aware buffer size (see + /// [`Self::estimate_array_size`]) plus the struct overhead, so a column that + /// is a zero-copy slice of a larger parent contributes only its own window + /// rather than the whole shared buffer. fn estimate_batch_size(batch: &RecordBatch) -> usize { batch .columns() .iter() - .map(|col| col.get_array_memory_size()) + .map(|col| Self::estimate_array_size(&col.to_data())) .sum::() + std::mem::size_of::() } + + /// Slice-aware buffer size of a single array. + /// + /// [`ArrayData::get_slice_memory_size`] reports each buffer's own window + /// (not the whole shared buffer), but omits the variadic data buffers of + /// `Utf8View`/`BinaryView` (values > 12 bytes) while still returning `Ok`, so + /// [`Self::view_data_buffers_size`] adds them. Those buffers are shared across + /// zero-copy slices and are counted at full capacity for each slice — an + /// over-count in the safe direction. + fn estimate_array_size(data: &ArrayData) -> usize { + match data.get_slice_memory_size() { + Ok(size) => size + Self::view_data_buffers_size(data), + // Fall back to the full-buffer sum for layouts the slice-aware call + // cannot handle. + Err(_) => data.get_array_memory_size(), + } + } + + /// Capacity of the variadic `Utf8View`/`BinaryView` data buffers that + /// [`ArrayData::get_slice_memory_size`] omits, summed recursively over children. + fn view_data_buffers_size(data: &ArrayData) -> usize { + let mut size = 0; + if matches!(data.data_type(), DataType::Utf8View | DataType::BinaryView) { + // buffers()[0] is the 16-byte view array that get_slice_memory_size + // already counts; [1..] are the data buffers it skips. + size += data + .buffers() + .iter() + .skip(1) + .map(|b| b.capacity()) + .sum::(); + } + for child in data.child_data() { + size += Self::view_data_buffers_size(child); + } + size + } } /// Snapshot of the active batches that have not yet been flushed to WAL. @@ -151,10 +195,14 @@ pub struct BatchStore { /// Estimated size in bytes (for flush threshold). estimated_bytes: AtomicUsize, - /// WAL flush watermark: the last batch ID that has been flushed to WAL (inclusive). - /// Uses usize::MAX as sentinel for "nothing flushed yet". - /// This is per-memtable tracking, not global. - max_flushed_batch_position: AtomicUsize, + /// Writer-global coordinate of this store's batch 0. + /// + /// A *coordinate*, not a cursor: stamped once at construction and never + /// moved. `global_position = global_offset + local_position`. Batch + /// positions restart at 0 in every memtable, so this is the only thing that + /// lets a writer-global cursor (the WAL durability count) be mapped onto a + /// particular store. + global_offset: usize, } // SAFETY: Safe to share across threads because: @@ -177,6 +225,13 @@ impl BatchStore { /// /// Panics if capacity is 0. pub fn with_capacity(capacity: usize) -> Self { + Self::with_capacity_at(capacity, 0) + } + + /// Create a store whose batch 0 sits at `global_offset` in the writer's + /// batch sequence. Used by `freeze_memtable` for every memtable after the + /// first; the first starts at 0. + pub fn with_capacity_at(capacity: usize, global_offset: usize) -> Self { assert!(capacity > 0, "capacity must be > 0"); // Allocate uninitialized storage @@ -191,7 +246,7 @@ impl BatchStore { capacity, total_rows: AtomicUsize::new(0), estimated_bytes: AtomicUsize::new(0), - max_flushed_batch_position: AtomicUsize::new(usize::MAX), // Nothing flushed yet + global_offset, } } @@ -393,74 +448,54 @@ impl BatchStore { // WAL Flush Tracking API // ========================================================================= - /// Get the WAL flush watermark (the last batch ID that was flushed, inclusive). - /// Returns None if nothing has been flushed yet. + /// Writer-global coordinate one past this store's last committed batch. #[inline] - pub fn max_flushed_batch_position(&self) -> Option { - let watermark = self.max_flushed_batch_position.load(Ordering::Acquire); - if watermark == usize::MAX { - None - } else { - Some(watermark) - } + pub fn global_end(&self) -> usize { + self.global_offset + self.committed_len.load(Ordering::Acquire) } - /// Update the WAL flush watermark after successful WAL flush. - /// - /// # Arguments - /// - /// * `batch_position` - The last batch ID that was flushed (inclusive) + /// This store's writer-global coordinate for batch 0. #[inline] - pub fn set_max_flushed_batch_position(&self, batch_position: usize) { - debug_assert!( - batch_position != usize::MAX, - "batch_position cannot be usize::MAX (reserved as sentinel)" - ); - self.max_flushed_batch_position - .store(batch_position, Ordering::Release); + pub fn global_offset(&self) -> usize { + self.global_offset } - /// Get the number of batches pending WAL flush. + /// The local exclusive end of this store covered by a writer-global cursor. + /// + /// Saturating in both directions, and both directions are reachable in + /// normal operation: a cursor *below* this store's offset means "nothing + /// here yet" (the store was rotated in after the cursor last advanced — + /// the ordinary state of a fresh memtable), and a cursor beyond its end + /// clamps to what is committed. + /// + /// This is the **only** place the global-to-local subtraction is written. + /// Open-coding it underflows on every memtable rotation, which in release + /// wraps to a huge end and makes the whole new memtable instantly visible. #[inline] - pub fn pending_wal_flush_count(&self) -> usize { - let committed = self.committed_len.load(Ordering::Acquire); - let watermark = self.max_flushed_batch_position.load(Ordering::Acquire); - if watermark == usize::MAX { - // Nothing flushed yet, all committed batches are pending - committed - } else { - // Batches [0, watermark] are flushed, so pending = committed - (watermark + 1) - committed.saturating_sub(watermark + 1) - } + pub fn local_end(&self, global_cursor: usize) -> usize { + global_cursor + .saturating_sub(self.global_offset) + .min(self.committed_len.load(Ordering::Acquire)) } - /// Check if all committed batches have been WAL-flushed. + /// Batches in this store still waiting on their WAL append. #[inline] - pub fn is_wal_flush_complete(&self) -> bool { - self.pending_wal_flush_count() == 0 + pub fn pending_wal_flush_count(&self, durable: usize) -> usize { + self.committed_len.load(Ordering::Acquire) - self.local_end(durable) } - /// Get the range of batch IDs pending WAL flush: [start, end). - /// Returns None if nothing pending. + /// Local range `[start, end)` of batches still waiting on their WAL append, + /// or `None` when the store is fully durable. #[inline] - pub fn pending_wal_flush_range(&self) -> Option<(usize, usize)> { - let committed = self.committed_len.load(Ordering::Acquire); - let watermark = self.max_flushed_batch_position.load(Ordering::Acquire); - let start = if watermark == usize::MAX { - 0 - } else { - watermark + 1 - }; - if committed > start { - Some((start, committed)) - } else { - None - } + pub fn pending_wal_flush_range(&self, durable: usize) -> Option<(usize, usize)> { + let start = self.local_end(durable); + let end = self.committed_len.load(Ordering::Acquire); + (end > start).then_some((start, end)) } /// Get a point-in-time summary of batches pending WAL flush. - pub fn pending_wal_flush_stats(&self) -> PendingWalFlushStats { - let Some((start, end)) = self.pending_wal_flush_range() else { + pub fn pending_wal_flush_stats(&self, durable: usize) -> PendingWalFlushStats { + let Some((start, end)) = self.pending_wal_flush_range(durable) else { return PendingWalFlushStats::default(); }; @@ -599,66 +634,53 @@ impl BatchStore { // Visibility API // ========================================================================= - /// Get batches visible up to a specific batch position (inclusive). + /// Batches in the visible prefix `[0, visible_count)`. /// - /// A batch at position `i` is visible if `i <= max_visible_batch_position`. - pub fn visible_batches(&self, max_visible_batch_position: usize) -> Vec<&StoredBatch> { - let len = self.committed_len.load(Ordering::Acquire); - let end = (max_visible_batch_position + 1).min(len); + /// `visible_count` is an **exclusive count**, not an inclusive position: 0 + /// means nothing is visible. As an inclusive position, 0 meant *both* + /// "nothing visible" and "batch 0 is visible", so a batch that was committed + /// to the store but not yet indexed or WAL-durable was readable for a full + /// PUT round-trip. The count makes that off-by-one inexpressible. + pub fn visible_batches(&self, visible_count: usize) -> Vec<&StoredBatch> { + let end = visible_count.min(self.committed_len.load(Ordering::Acquire)); (0..end).filter_map(|i| self.get(i)).collect() } - /// Get batch positions visible up to a specific batch position (inclusive). - pub fn max_visible_batch_positions(&self, max_visible_batch_position: usize) -> Vec { - let len = self.committed_len.load(Ordering::Acquire); - let end = (max_visible_batch_position + 1).min(len); + /// Positions of the batches in the visible prefix. + pub fn visible_batch_positions(&self, visible_count: usize) -> Vec { + let end = visible_count.min(self.committed_len.load(Ordering::Acquire)); (0..end).collect() } - /// The inclusive maximum visible *row* position at `max_visible_batch_position`, - /// or `None` when no rows are visible. The visible batches are the committed - /// prefix `[0, last_visible_idx]`; each batch carries its cumulative - /// `row_offset`, so this is the end of the last visible batch minus one. - /// Used to bound MVCC seeks against the maintained PK-position index. - pub fn max_visible_row(&self, max_visible_batch_position: usize) -> Option { - let len = self.committed_len.load(Ordering::Acquire); - if len == 0 { - return None; - } - let last_visible_idx = max_visible_batch_position.min(len - 1); - let last = self.get(last_visible_idx)?; + /// The inclusive maximum visible *row* position, or `None` when no rows are + /// visible. Each batch carries its cumulative `row_offset`, so this is the + /// end of the last visible batch minus one. Bounds MVCC seeks against the + /// maintained PK-position index. + pub fn max_visible_row(&self, visible_count: usize) -> Option { + let end = visible_count.min(self.committed_len.load(Ordering::Acquire)); + let last = self.get(end.checked_sub(1)?)?; let visible_end = last.row_offset + last.num_rows as u64; // exclusive visible_end.checked_sub(1) } - /// Check if a specific batch is visible at a given visibility position. + /// Whether a batch falls inside the visible prefix. #[inline] - pub fn is_batch_visible( - &self, - batch_position: usize, - max_visible_batch_position: usize, - ) -> bool { + pub fn is_batch_visible(&self, batch_position: usize, visible_count: usize) -> bool { let len = self.committed_len.load(Ordering::Acquire); - batch_position < len && batch_position <= max_visible_batch_position + batch_position < len && batch_position < visible_count } - /// Get visible RecordBatches (clones the data). - pub fn visible_record_batches(&self, max_visible_batch_position: usize) -> Vec { - self.visible_batches(max_visible_batch_position) + /// Visible RecordBatches (clones the data). + pub fn visible_record_batches(&self, visible_count: usize) -> Vec { + self.visible_batches(visible_count) .into_iter() .map(|b| b.data.clone()) .collect() } - /// Get visible RecordBatches with their row offsets. - /// - /// Returns tuples of (batch, row_offset) for each visible batch. - /// The row_offset is the starting row position for that batch. - pub fn visible_batches_with_offsets( - &self, - max_visible_batch_position: usize, - ) -> Vec<(RecordBatch, u64)> { - self.visible_batches(max_visible_batch_position) + /// Visible RecordBatches paired with the row position each one starts at. + pub fn visible_batches_with_offsets(&self, visible_count: usize) -> Vec<(RecordBatch, u64)> { + self.visible_batches(visible_count) .into_iter() .map(|b| (b.data.clone(), b.row_offset)) .collect() @@ -891,17 +913,52 @@ mod tests { store.append(create_test_batch(10)).unwrap(); // position 3 store.append(create_test_batch(10)).unwrap(); // position 4 - // max_visible_batch_position=2 means positions 0, 1, 2 are visible - let visible = store.max_visible_batch_positions(2); - assert_eq!(visible, vec![0, 1, 2]); + // A count of N exposes the prefix [0, N). + assert_eq!(store.visible_batch_positions(3), vec![0, 1, 2]); + assert_eq!(store.visible_batch_positions(5), vec![0, 1, 2, 3, 4]); + + // A count of 0 exposes nothing. Under the old inclusive cursor this + // case was indistinguishable from "batch 0 is visible", so every + // memtable leaked its first batch before it was indexed or durable. + assert!(store.visible_batch_positions(0).is_empty()); + + // Beyond the committed range, clamp. + assert_eq!(store.visible_batch_positions(99), vec![0, 1, 2, 3, 4]); + } + + /// The zero of the visibility cursor must be unambiguous. + /// + /// `BatchStore::append` publishes `committed_len` on the put path, under the + /// state lock, *before* the WAL flush that indexes the batch is even + /// triggered — and that flush is a ~100ms S3 PUT on another task. So batch 0 + /// sits committed and readable for a full round-trip before it is indexed or + /// durable. As an inclusive position, a cursor of 0 meant both "nothing is + /// visible" and "batch 0 is visible", so every read arm backed by the batch + /// store served that batch while the index-backed arms did not — the tiers + /// actively disagreed. An exclusive count makes the state inexpressible. + #[test] + fn test_zero_cursor_hides_the_committed_but_unindexed_prefix() { + let store = BatchStore::with_capacity(4); + store.append(create_test_batch(10)).unwrap(); + store.append(create_test_batch(10)).unwrap(); - // max_visible_batch_position=4 means all visible - let visible = store.max_visible_batch_positions(4); - assert_eq!(visible, vec![0, 1, 2, 3, 4]); + // Committed, but nothing indexed yet: every visibility query must agree + // that there is nothing to read. + assert!(store.visible_batches(0).is_empty()); + assert!(store.visible_batch_positions(0).is_empty()); + assert!(store.visible_record_batches(0).is_empty()); + assert!(store.visible_batches_with_offsets(0).is_empty()); + assert!(!store.is_batch_visible(0, 0)); + assert_eq!(store.max_visible_row(0), None); + + // The batches are there — they are simply not yet published. + assert_eq!(store.len(), 2); - // max_visible_batch_position=0 means only position 0 visible - let visible = store.max_visible_batch_positions(0); - assert_eq!(visible, vec![0]); + // Indexing batch 0 publishes exactly batch 0. + assert_eq!(store.visible_batches(1).len(), 1); + assert!(store.is_batch_visible(0, 1)); + assert!(!store.is_batch_visible(1, 1)); + assert_eq!(store.max_visible_row(1), Some(9)); } #[test] @@ -912,14 +969,17 @@ mod tests { store.append(create_test_batch(10)).unwrap(); // position 1 store.append(create_test_batch(10)).unwrap(); // position 2 - // Batch at position 0 is visible when max_visible_batch_position >= 0 - assert!(store.is_batch_visible(0, 0)); + // A count of 0 means *nothing* is visible — including batch 0. As an + // inclusive position this case was indistinguishable from "batch 0 is + // visible", so a batch that was committed to the store but not yet + // indexed or WAL-durable was readable for a full PUT round-trip. + assert!(!store.is_batch_visible(0, 0)); + + // Batch i is visible once the count exceeds i. assert!(store.is_batch_visible(0, 1)); assert!(store.is_batch_visible(0, 2)); - - // Batch at position 2 is only visible when max_visible_batch_position >= 2 assert!(!store.is_batch_visible(2, 1)); - assert!(store.is_batch_visible(2, 2)); + assert!(!store.is_batch_visible(2, 2)); assert!(store.is_batch_visible(2, 3)); // Batch 3 doesn't exist @@ -928,7 +988,7 @@ mod tests { #[test] fn test_max_visible_row() { - // (1) Empty store: no rows are visible at any position. + // (1) Empty store: no rows are visible at any count. let store = BatchStore::with_capacity(10); assert_eq!(store.max_visible_row(0), None); assert_eq!(store.max_visible_row(100), None); @@ -938,23 +998,24 @@ mod tests { store.append(create_test_batch(20)).unwrap(); // position 1 store.append(create_test_batch(30)).unwrap(); // position 2 - // (2) A position within range yields the inclusive end of that prefix. - assert_eq!(store.max_visible_row(0), Some(9)); // batch 0: 0..10 - assert_eq!(store.max_visible_row(1), Some(29)); // batch 1: 10..30 - assert_eq!(store.max_visible_row(2), Some(59)); // batch 2: 30..60 + // (2) A count of 0 means nothing is visible — not "batch 0 is visible". + assert_eq!(store.max_visible_row(0), None); + + // (3) A count of N yields the inclusive last row of the prefix [0, N). + assert_eq!(store.max_visible_row(1), Some(9)); // batch 0: 0..10 + assert_eq!(store.max_visible_row(2), Some(29)); // + batch 1: 10..30 + assert_eq!(store.max_visible_row(3), Some(59)); // + batch 2: 30..60 - // (3) A position beyond the committed range clamps to the last batch, - // i.e. the inclusive max over all rows. + // (4) A count beyond the committed range clamps to the last batch. assert_eq!(store.max_visible_row(100), Some(59)); - // (4) An empty leading batch contributes no rows: at its own position - // the inclusive end underflows to None, while a later non-empty batch - // is reported correctly. + // (5) An empty leading batch contributes no rows, so a prefix covering + // only it still yields None, while a later non-empty batch is reported. let store = BatchStore::with_capacity(10); store.append(create_test_batch(0)).unwrap(); // position 0: rows [0,0) store.append(create_test_batch(5)).unwrap(); // position 1: rows [0,5) - assert_eq!(store.max_visible_row(0), None); // empty prefix → no rows - assert_eq!(store.max_visible_row(1), Some(4)); // through batch 1 + assert_eq!(store.max_visible_row(1), None); // empty prefix → no rows + assert_eq!(store.max_visible_row(2), Some(4)); // through batch 1 } #[test] @@ -972,6 +1033,122 @@ mod tests { assert_eq!(cap, 16); // minimum } + #[test] + fn test_estimated_size_is_slice_aware() { + // A batch that is a zero-copy slice of a larger parent must contribute + // only its own window to the estimate, not the whole shared buffer. + // `get_array_memory_size` counts every buffer's full capacity regardless + // of offset/length, so N slices tiling one parent each report the + // parent's size and inflate the memtable estimate ~N×, tripping the + // flush threshold far below the configured size. + let chunk = 1_000; + let num_slices = 100; + let parent = create_test_batch(chunk * num_slices); + + // One window vs an equivalently-sized owned batch should track each + // other; the buggy per-slice estimate would be ~num_slices× larger. + let slice_est = StoredBatch::estimate_batch_size(&parent.slice(0, chunk)); + let owned_est = StoredBatch::estimate_batch_size(&create_test_batch(chunk)); + assert!( + slice_est <= owned_est * 2, + "slice estimate {slice_est} should track its own window (~{owned_est}), not the parent" + ); + + // End-to-end: tiling the parent with zero-copy slices must not multiply + // the store's running estimate. Track what the old full-buffer behavior + // would have summed to for contrast. + let store = BatchStore::with_capacity(num_slices); + let mut over_counting_sum = 0usize; + for k in 0..num_slices { + let s = parent.slice(k * chunk, chunk); + over_counting_sum += s + .columns() + .iter() + .map(|col| col.get_array_memory_size()) + .sum::() + + std::mem::size_of::(); + store.append(s).unwrap(); + } + + // Two non-nullable Int32 columns → exactly 4 bytes/row/col of payload. + let payload_bytes = num_slices * chunk * 2 * std::mem::size_of::(); + let estimated = store.estimated_bytes(); + assert!( + estimated >= payload_bytes, + "estimate {estimated} should cover the actual payload {payload_bytes}" + ); + // The old behavior over-counts by ~num_slices×; the fix must be far + // below it (generous 10× margin against struct/alignment overhead). + assert!( + estimated * 10 < over_counting_sum, + "estimate {estimated} should be far below the over-counting sum {over_counting_sum}" + ); + } + + #[test] + fn test_estimated_size_counts_view_data_buffers() { + // Long Utf8View/BinaryView values live in variadic data buffers that + // `get_slice_memory_size` ignores (returning ~16 * rows). The estimate + // must include them, both for a top-level view column and for a view + // array nested in a container, which is only reached via child_data + // recursion. + use arrow_array::{Array, ArrayRef, StringViewArray, StructArray}; + + let num_rows = 1_000; + // Each value exceeds the 12-byte inline limit, so it spills to a data buffer. + let long_value = "x".repeat(64); + let payload_bytes = num_rows * long_value.len(); + // What the slice-aware call alone reports: just the 16-byte view entries. + let view_entries_only = num_rows * 16; + + let make_views = || { + StringViewArray::from( + (0..num_rows) + .map(|_| Some(long_value.as_str())) + .collect::>(), + ) + }; + let assert_covers = |batch: &RecordBatch| { + let estimated = StoredBatch::estimate_batch_size(batch); + assert!( + estimated >= payload_bytes, + "estimate {estimated} should cover the view data-buffer payload {payload_bytes}" + ); + assert!( + estimated > view_entries_only * 2, + "estimate {estimated} must exceed the ~{view_entries_only}-byte view-entry-only undercount" + ); + }; + + // Top-level view column. + let flat = RecordBatch::try_new( + Arc::new(ArrowSchema::new(vec![Field::new( + "s", + DataType::Utf8View, + false, + )])), + vec![Arc::new(make_views())], + ) + .unwrap(); + assert_covers(&flat); + + // View nested inside a struct — reachable only through child_data recursion. + let nested = StructArray::from(vec![( + Arc::new(Field::new("s", DataType::Utf8View, false)), + Arc::new(make_views()) as ArrayRef, + )]); + let nested = RecordBatch::try_new( + Arc::new(ArrowSchema::new(vec![Field::new( + "st", + nested.data_type().clone(), + false, + )])), + vec![Arc::new(nested)], + ) + .unwrap(); + assert_covers(&nested); + } + #[test] fn test_to_vec() { let store = BatchStore::with_capacity(10); diff --git a/rust/lance/src/dataset/mem_wal/memtable/flush.rs b/rust/lance/src/dataset/mem_wal/memtable/flush.rs index 410823c31db..57718e69e5c 100644 --- a/rust/lance/src/dataset/mem_wal/memtable/flush.rs +++ b/rust/lance/src/dataset/mem_wal/memtable/flush.rs @@ -12,9 +12,9 @@ use lance_core::cache::LanceCache; use lance_core::utils::deletion::DeletionVector; use lance_core::{Error, Result}; use lance_index::IndexType; -use lance_index::mem_wal::{FlushedGeneration, ShardManifest}; +use lance_index::mem_wal::{ShardManifest, SsTable}; use lance_index::scalar::{IndexStore, ScalarIndexParams}; -use lance_io::object_store::ObjectStore; +use lance_io::object_store::{ObjectStore, ObjectStoreParams}; use lance_table::format::IndexMetadata; use lance_table::io::commit::write_manifest_file_to_path; use lance_table::io::deletion::write_deletion_file; @@ -28,14 +28,17 @@ use uuid::Uuid; use super::super::index::MemIndexConfig; use super::super::memtable::MemTable; use crate::Dataset; +use crate::dataset::builder::DatasetBuilder; use crate::dataset::mem_wal::manifest::ShardManifestStore; -use crate::dataset::mem_wal::scanner::GenerationWarmer; +use crate::dataset::mem_wal::scanner::SsTableWarmer; use crate::dataset::mem_wal::scanner::exec::{compute_pk_hash, validate_pk_types}; -use crate::dataset::mem_wal::util::{flushed_memtable_path, generate_random_hash}; +use crate::dataset::mem_wal::util::{derived_store_params, generate_random_hash, sstable_path}; +use crate::index::vector::details::vector_index_details_default; +use crate::session::Session; #[derive(Debug, Clone)] pub struct FlushResult { - pub generation: FlushedGeneration, + pub sstable: SsTable, pub rows_flushed: usize, pub covered_wal_entry_position: u64, } @@ -71,7 +74,14 @@ pub struct MemTableFlusher { manifest_store: Arc, /// When present, each new generation is warmed before it is committed, so /// the first query sees zero cold reads. `None` => no warming. - warmer: Option>, + warmer: Option>, + /// Store params the base dataset was opened with, reused for the flusher's + /// own opens + writes. Used verbatim only for the base's own URI; generation + /// URIs go through [`derived_store_params`]. `None` opens by URI alone. + store_params: Option, + /// Session for those opens, sharing the base's store registry. `None` opens + /// with a fresh session. + session: Option>, } impl MemTableFlusher { @@ -89,15 +99,62 @@ impl MemTableFlusher { shard_id, manifest_store, warmer: None, + store_params: None, + session: None, } } /// Attach the warmer fired pre-commit for each new generation. - pub fn with_warmer(mut self, warmer: Option>) -> Self { + pub fn with_warmer(mut self, warmer: Option>) -> Self { self.warmer = warmer; self } + /// Set the store params + session used for derived-URI opens. Injected by + /// `mem_wal_writer` from the base `Dataset`. + pub fn with_storage_context( + mut self, + store_params: Option, + session: Option>, + ) -> Self { + self.store_params = store_params; + self.session = session; + self + } + + /// Open the base table, reusing the injected store params verbatim — they + /// were resolved for exactly this URI, so a path-bound `object_store` + /// binding still points where it should. + async fn open_base(&self) -> Result { + self.open_uri(&self.base_uri, self.store_params.clone()) + .await + } + + /// Open an SSTable under `_mem_wal/`. The params must be adapted + /// first: a path-bound store binding would redirect the open at the base + /// table (see [`derived_store_params`]). + async fn open_generation(&self, uri: &str) -> Result { + self.open_uri(uri, self.store_params.as_ref().map(derived_store_params)) + .await + } + + /// Open `uri` with the injected session, or by URI alone when nothing was + /// injected. + async fn open_uri( + &self, + uri: &str, + store_params: Option, + ) -> Result { + let mut builder = DatasetBuilder::from_uri(uri); + if let Some(params) = store_params { + builder = builder.with_store_params(params); + } + if let Some(session) = &self.session { + builder = builder.with_session(session.clone()); + } + builder.load().await + } + /// Warm a just-written generation before it is committed. Best-effort: a /// failure is logged and the flush proceeds — warming is never a commit /// gate. No-op without a warmer. `uri` must be the resolved reader path @@ -130,20 +187,18 @@ impl MemTableFlusher { } } - /// Storage file version of the shard's base dataset. Flushed generations + /// Storage file version of the shard's base dataset. SSTables /// (data fragments and index files) are written at this same version so the - /// whole shard stays on one format (e.g. a 2.2 base => 2.2 flushed gens). + /// whole shard stays on one format (e.g. a 2.2 base => 2.2 SSTables). /// - /// Falls back to [`LanceFileVersion::default`] when no base dataset exists at + /// Falls back to the default selector's exact version when no base dataset exists at /// `base_uri` (e.g. flusher unit tests that run without a committed base). /// In production MemWAL is always initialized on a real dataset, so the base /// version is inherited; other open errors are propagated. - async fn base_storage_version(&self) -> Result { - match Dataset::open(&self.base_uri).await { - Ok(dataset) => dataset.manifest().data_storage_format.lance_file_version(), - Err(Error::DatasetNotFound { .. }) => { - Ok(lance_file::version::LanceFileVersion::default()) - } + async fn base_storage_version(&self) -> Result { + match self.open_base().await { + Ok(dataset) => Ok(dataset.manifest().data_storage_format.lance_file_format()), + Err(Error::DatasetNotFound { .. }) => Ok(lance_file::version::stable_file_version()), Err(e) => Err(e), } } @@ -161,6 +216,7 @@ impl MemTableFlusher { memtable: &MemTable, epoch: u64, covered_wal_entry_position: u64, + durable: usize, ) -> Result { self.manifest_store.check_fenced(epoch).await?; @@ -168,7 +224,7 @@ impl MemTableFlusher { return Err(Error::invalid_input("Cannot flush empty MemTable")); } - if !memtable.all_flushed_to_wal() { + if !memtable.all_flushed_to_wal(durable) { return Err(Error::invalid_input( "MemTable has unflushed fragments - WAL flush required first", )); @@ -177,8 +233,7 @@ impl MemTableFlusher { let random_hash = generate_random_hash(); let generation = memtable.generation(); let gen_folder_name = format!("{}_gen_{}", random_hash, generation); - let gen_path = - flushed_memtable_path(&self.base_path, &self.shard_id, &random_hash, generation); + let gen_path = sstable_path(&self.base_path, &self.shard_id, &random_hash, generation); info!( "Flushing MemTable generation {} to {} ({} rows, {} batches)", @@ -190,11 +245,11 @@ impl MemTableFlusher { let (rows_flushed, deleted) = self.write_data_file(&gen_path, memtable).await?; - // Persist the within-generation deletion vector so the flushed - // generation exposes newest-per-PK on every read path. + // Persist the within-generation deletion vector so the + // SSTable exposes newest-per-PK on every read path. if !deleted.is_empty() { let uri = self.path_to_uri(&gen_path); - let dataset = Dataset::open(&uri).await?; + let dataset = self.open_generation(&uri).await?; self.finalize_generation(&dataset, &deleted, None).await?; } @@ -222,12 +277,12 @@ impl MemTableFlusher { .await?; info!( - "Flushed generation {} for shard {} (manifest version {})", + "Flushed SSTable {} for shard {} (manifest version {})", generation, self.shard_id, new_manifest.version ); Ok(FlushResult { - generation: FlushedGeneration { + sstable: SsTable { generation, path: gen_folder_name, }, @@ -296,13 +351,19 @@ impl MemTableFlusher { let reader = RecordBatchIterator::new(batches.into_iter().map(Ok), memtable.schema().clone()); - // Use very large max_rows_per_file to ensure 1 fragment per flushed memtable. - // Inherit the base dataset's storage version so the flushed generation + // Use very large max_rows_per_file to ensure 1 fragment per SSTable. + // Inherit the base dataset's storage version so the SSTable // matches it (a 2.2 base also fixes the v2.1 miniblock 32 KiB chunk cap // that the dense HNSW graph List columns overflow at scale). let write_params = WriteParams { max_rows_per_file: usize::MAX, - data_storage_version: Some(self.base_storage_version().await?), + data_storage_version: Some(self.base_storage_version().await?.to_selector()), + // Write the generation through the base's store params + session so it + // uses the same store the base was opened with. Adapted for the + // generation URI: a path-bound store binding would send this write at + // the base table's own path (see [`derived_store_params`]). + store_params: self.store_params.as_ref().map(derived_store_params), + session: self.session.clone(), ..Default::default() }; Dataset::write(reader, &uri, Some(write_params)).await?; @@ -334,7 +395,7 @@ impl MemTableFlusher { let dv = DeletionVector::from(deleted.clone()); let deletion_file = write_deletion_file( &dataset.base, - 0, // 1 fragment per flushed generation + 0, // 1 fragment per SSTable dataset.version().version, &dv, dataset.object_store.as_ref(), @@ -388,6 +449,7 @@ impl MemTableFlusher { epoch: u64, index_configs: &[MemIndexConfig], covered_wal_entry_position: u64, + durable: usize, ) -> Result { self.manifest_store.check_fenced(epoch).await?; @@ -395,7 +457,7 @@ impl MemTableFlusher { return Err(Error::invalid_input("Cannot flush empty MemTable")); } - if !memtable.all_flushed_to_wal() { + if !memtable.all_flushed_to_wal(durable) { return Err(Error::invalid_input( "MemTable has unflushed fragments - WAL flush required first", )); @@ -404,8 +466,7 @@ impl MemTableFlusher { let random_hash = generate_random_hash(); let generation = memtable.generation(); let gen_folder_name = format!("{}_gen_{}", random_hash, generation); - let gen_path = - flushed_memtable_path(&self.base_path, &self.shard_id, &random_hash, generation); + let gen_path = sstable_path(&self.base_path, &self.shard_id, &random_hash, generation); info!( "Flushing MemTable generation {} with indexes to {} ({} rows, {} batches)", @@ -420,7 +481,7 @@ impl MemTableFlusher { // Open the dataset once for all index building. Dataset::write already // created a v1 manifest with the fragment data. let uri = self.path_to_uri(&gen_path); - let mut dataset = Dataset::open(&uri).await?; + let mut dataset = self.open_generation(&uri).await?; // Collect all index metadata without committing individually. // We write a single manifest containing both data and all indexes. @@ -431,7 +492,7 @@ impl MemTableFlusher { .await?; if !btree_indexes.is_empty() { info!( - "Created {} BTree indexes on flushed generation {}", + "Created {} BTree indexes on SSTable {}", btree_indexes.len(), generation ); @@ -450,7 +511,7 @@ impl MemTableFlusher { .await? else { info!( - "Skipped empty HNSW index '{}' on flushed generation {} (no vectors)", + "Skipped empty HNSW index '{}' on SSTable {} (no vectors)", hnsw_config.name, generation ); continue; @@ -474,7 +535,7 @@ impl MemTableFlusher { all_indexes.push(index_meta); info!( - "Created HNSW index '{}' on flushed generation {}", + "Created HNSW index '{}' on SSTable {}", hnsw_config.name, generation ); } @@ -520,12 +581,12 @@ impl MemTableFlusher { .await?; info!( - "Flushed generation {} for shard {} (manifest version {})", + "Flushed SSTable {} for shard {} (manifest version {})", generation, self.shard_id, new_manifest.version ); Ok(FlushResult { - generation: FlushedGeneration { + sstable: SsTable { generation, path: gen_folder_name, }, @@ -534,7 +595,7 @@ impl MemTableFlusher { }) } - /// Create BTree indexes on the flushed dataset (uncommitted). + /// Create BTree indexes on the SSTable dataset (uncommitted). /// /// Returns index metadata without committing to the dataset manifest. /// The caller is responsible for writing a single manifest with all indexes. @@ -602,7 +663,7 @@ impl MemTableFlusher { /// keys index the typed value; composite keys index the order-preserving /// `Binary` encoded tuple (see [`super::super::index::encode_pk_tuple`]). /// Row positions line up 1:1 with the forward-written data file, so they are - /// the flushed row ids directly. No-op without a primary-key index. + /// the SSTable row ids directly. No-op without a primary-key index. async fn create_pk_index( &self, gen_path: &Path, @@ -703,16 +764,15 @@ impl MemTableFlusher { let index_details = prost_types::Any::from_msg(&details) .map_err(|e| Error::io(format!("Failed to serialize index details: {}", e)))?; - let schema = dataset.schema(); - let field_idx = schema.field(&fts_cfg.column).map(|f| f.id).ok_or_else(|| { - Error::invalid_input(format!( - "FTS index '{}' references column '{}' which is not in the dataset schema", - fts_cfg.name, fts_cfg.column - )) - })?; + let field_idx = fts_cfg.field_id; let fragment_ids: roaring::RoaringBitmap = dataset.fragment_bitmap.as_ref().clone(); let format_version = fts_cfg.params.resolved_format_version(); + let index_version = if fts_cfg.params.get_document_granularity().is_list_element() { + lance_index::scalar::inverted::INVERTED_INDEX_VERSION_V3 + } else { + format_version.index_version() + }; let index_meta = IndexMetadata { uuid: index_uuid, @@ -721,7 +781,7 @@ impl MemTableFlusher { dataset_version: dataset.version().version, fragment_bitmap: Some(fragment_ids), index_details: Some(Arc::new(index_details)), - index_version: format_version.index_version() as i32, + index_version: index_version as i32, created_at: None, base_id: None, files: None, @@ -749,8 +809,9 @@ impl MemTableFlusher { use std::sync::Arc; use lance_index::scalar::inverted::{ - POSITIONS_CODEC_KEY, POSITIONS_CODEC_PACKED_DELTA_V1, POSITIONS_LAYOUT_KEY, - POSITIONS_LAYOUT_SHARED_STREAM_V2, POSTING_TAIL_CODEC_KEY, TokenSetFormat, + FTS_FORMAT_VERSION_KEY, POSITIONS_CODEC_KEY, POSITIONS_CODEC_PACKED_DELTA_V1, + POSITIONS_LAYOUT_KEY, POSITIONS_LAYOUT_SHARED_STREAM_V2, POSTING_BLOCK_SIZE_KEY, + POSTING_TAIL_CODEC_KEY, TokenSetFormat, }; // Create metadata with params and partitions in schema metadata (this is what InvertedIndex expects) @@ -766,6 +827,14 @@ impl MemTableFlusher { POSTING_TAIL_CODEC_KEY.to_string(), format_version.posting_tail_codec().as_str().to_string(), ), + ( + FTS_FORMAT_VERSION_KEY.to_string(), + format_version.index_version().to_string(), + ), + ( + POSTING_BLOCK_SIZE_KEY.to_string(), + config.params.posting_block_size().to_string(), + ), ] .into_iter() .collect::>(); @@ -807,7 +876,7 @@ impl MemTableFlusher { /// the existing Lance `IVF_HNSW_SQ` reader path. /// /// # Arguments - /// * `gen_path` - Path to the flushed generation folder + /// * `gen_path` - Path to the SSTable folder /// * `config` - HNSW index configuration /// * `mem_index` - In-memory HNSW index (snapshotted, not consumed) /// @@ -831,7 +900,8 @@ impl MemTableFlusher { use arrow_schema::Schema as ArrowSchema; use lance_arrow::FixedSizeListArrayExt; use lance_core::ROW_ID; - use lance_file::writer::{FileWriter, FileWriterOptions}; + use lance_file::versions as file_versions; + use lance_file::writer::FileWriterOptions; use lance_index::pb; use lance_index::vector::DISTANCE_TYPE_KEY; use lance_index::vector::SQ_CODE_COLUMN; @@ -850,7 +920,8 @@ impl MemTableFlusher { // Write the index files at the base dataset's storage version (matches // the flushed data fragments; 2.2 avoids the v2.1 miniblock chunk cap). - let storage_version = self.base_storage_version().await?; + let storage_version = + crate::dataset::versions::index_file_version(self.base_storage_version().await?); let index_uuid = uuid::Uuid::new_v4(); let index_dir = gen_path @@ -922,13 +993,11 @@ impl MemTableFlusher { storage_ivf.add_partition(storage_batch.num_rows() as u32); let storage_path = index_dir.clone().join(INDEX_AUXILIARY_FILE_NAME); - let mut storage_writer = FileWriter::try_new( + let mut storage_writer = file_versions::create_writer( + storage_version, self.object_store.create(&storage_path).await?, (&storage_schema).try_into()?, - FileWriterOptions { - format_version: Some(storage_version), - ..Default::default() - }, + FileWriterOptions::default(), )?; storage_writer.write_batch(&storage_batch).await?; @@ -997,13 +1066,11 @@ impl MemTableFlusher { ArrowSchema::new(fields) }; let index_path = index_dir.clone().join(INDEX_FILE_NAME); - let mut index_writer = FileWriter::try_new( + let mut index_writer = file_versions::create_writer( + storage_version, self.object_store.create(&index_path).await?, (&index_schema).try_into()?, - FileWriterOptions { - format_version: Some(storage_version), - ..Default::default() - }, + FileWriterOptions::default(), )?; index_writer.write_batch(&hnsw_batch).await?; @@ -1034,10 +1101,9 @@ impl MemTableFlusher { ); index_writer.finish().await?; - let index_details = Some(Arc::new(prost_types::Any { - type_url: "type.googleapis.com/lance.index.VectorIndexDetails".to_string(), - value: vec![], - })); + // Packed the same way index creation does; hand-building the `Any` here + // produced a `type.googleapis.com/` url no other writer in lance emits. + let index_details = Some(Arc::new(vector_index_details_default())); let index_meta = IndexMetadata { uuid: index_uuid, name: config.name.clone(), @@ -1054,7 +1120,7 @@ impl MemTableFlusher { Ok(Some(index_meta)) } - /// Update the shard manifest with the new flushed generation. + /// Update the shard manifest with the new SSTable. async fn update_manifest( &self, epoch: u64, @@ -1066,8 +1132,8 @@ impl MemTableFlusher { self.manifest_store .commit_update(epoch, |current| { - let mut flushed_generations = current.flushed_generations.clone(); - flushed_generations.push(FlushedGeneration { + let mut sstables = current.sstables.clone(); + sstables.push(SsTable { generation, path: gen_path.clone(), }); @@ -1079,7 +1145,7 @@ impl MemTableFlusher { .wal_entry_position_last_seen .max(covered_wal_entry_position), current_generation: generation + 1, - flushed_generations, + sstables, ..current.clone() } }) @@ -1119,6 +1185,7 @@ mod tests { use super::*; use arrow_array::{Int32Array, RecordBatch, StringArray}; use arrow_schema::{DataType, Field, Schema as ArrowSchema}; + use lance_index::scalar::inverted::INVERTED_INDEX_VERSION_V2; use std::sync::Arc; use tempfile::TempDir; @@ -1185,11 +1252,12 @@ mod tests { .await .unwrap(); - // Not flushed to WAL yet - assert!(!memtable.all_flushed_to_wal()); + // Nothing is durable yet, so the L0 flush must refuse. + let durable = 0; + assert!(!memtable.all_flushed_to_wal(durable)); let flusher = MemTableFlusher::new(store, base_path, base_uri, shard_id, manifest_store); - let result = flusher.flush(&memtable, epoch, 0).await; + let result = flusher.flush(&memtable, epoch, 0, 0).await; assert!(result.is_err()); assert!( @@ -1218,7 +1286,7 @@ mod tests { let memtable = MemTable::new(schema, 1, vec![]).unwrap(); let flusher = MemTableFlusher::new(store, base_path, base_uri, shard_id, manifest_store); - let result = flusher.flush(&memtable, epoch, 0).await; + let result = flusher.flush(&memtable, epoch, 0, 0).await; assert!(result.is_err()); assert!(result.unwrap_err().to_string().contains("empty MemTable")); @@ -1246,8 +1314,8 @@ mod tests { .unwrap(); // Simulate WAL flush - memtable.mark_wal_flushed(&[frag_id], 1, &[0]); - assert!(memtable.all_flushed_to_wal()); + let durable = frag_id + 1; + assert!(memtable.all_flushed_to_wal(durable)); let flusher = MemTableFlusher::new( store.clone(), @@ -1256,9 +1324,9 @@ mod tests { shard_id, manifest_store.clone(), ); - let result = flusher.flush(&memtable, epoch, 1).await.unwrap(); + let result = flusher.flush(&memtable, epoch, 1, durable).await.unwrap(); - assert_eq!(result.generation.generation, 1); + assert_eq!(result.sstable.generation, 1); assert_eq!(result.rows_flushed, 10); assert_eq!(result.covered_wal_entry_position, 1); @@ -1267,10 +1335,10 @@ mod tests { assert_eq!(updated_manifest.version, 2); assert_eq!(updated_manifest.replay_after_wal_entry_position, 1); assert_eq!(updated_manifest.current_generation, 2); - assert_eq!(updated_manifest.flushed_generations.len(), 1); + assert_eq!(updated_manifest.sstables.len(), 1); } - /// A `GenerationWarmer` that counts calls and optionally fails. + /// A `SsTableWarmer` that counts calls and optionally fails. #[derive(Debug)] struct CountingWarmer { calls: Arc, @@ -1278,7 +1346,7 @@ mod tests { } #[async_trait::async_trait] - impl GenerationWarmer for CountingWarmer { + impl SsTableWarmer for CountingWarmer { async fn warm(&self, _path: &str) -> Result<()> { self.calls.fetch_add(1, std::sync::atomic::Ordering::SeqCst); if self.fail { @@ -1310,10 +1378,10 @@ mod tests { .insert(create_test_batch(&schema, 10)) .await .unwrap(); - memtable.mark_wal_flushed(&[frag_id], 1, &[0]); + let durable = frag_id + 1; let calls = Arc::new(std::sync::atomic::AtomicUsize::new(0)); - let warmer: Arc = Arc::new(CountingWarmer { + let warmer: Arc = Arc::new(CountingWarmer { calls: calls.clone(), fail: true, }); @@ -1327,9 +1395,9 @@ mod tests { ) .with_warmer(Some(warmer)); // Flush must succeed despite the warmer erroring. - let result = flusher.flush(&memtable, epoch, 1).await.unwrap(); + let result = flusher.flush(&memtable, epoch, 1, durable).await.unwrap(); - assert_eq!(result.generation.generation, 1); + assert_eq!(result.sstable.generation, 1); assert_eq!( calls.load(std::sync::atomic::Ordering::SeqCst), 1, @@ -1337,14 +1405,14 @@ mod tests { ); let updated = manifest_store.read_latest().await.unwrap().unwrap(); assert_eq!( - updated.flushed_generations.len(), + updated.sstables.len(), 1, "generation still committed after a failed warm" ); } /// Flushing a generation with within-generation duplicate PKs writes a - /// deletion vector so the flushed dataset exposes newest-per-PK on scan. + /// deletion vector so the SSTable dataset exposes newest-per-PK on scan. #[tokio::test] async fn test_flush_writes_dedup_deletion_vector() { use futures::TryStreamExt; @@ -1371,7 +1439,7 @@ mod tests { ) .unwrap(); let frag_id = memtable.insert(batch).await.unwrap(); - memtable.mark_wal_flushed(&[frag_id], 1, &[0]); + let durable = frag_id + 1; let flusher = MemTableFlusher::new( store.clone(), @@ -1380,16 +1448,16 @@ mod tests { shard_id, manifest_store, ); - let result = flusher.flush(&memtable, epoch, 1).await.unwrap(); + let result = flusher.flush(&memtable, epoch, 1, durable).await.unwrap(); assert_eq!(result.rows_flushed, 5, "all physical rows are written"); - // Scanning the flushed generation must honor the deletion vector and + // Scanning the SSTable must honor the deletion vector and // return only the newest version of each PK. let gen_uri = format!( "{}/_mem_wal/{}/{}", base_uri.trim_end_matches('/'), shard_id, - result.generation.path + result.sstable.path ); let dataset = Dataset::open(&gen_uri).await.unwrap(); let batches: Vec = dataset @@ -1436,7 +1504,7 @@ mod tests { /// probe by value — including for a within-gen-superseded PK (existence, /// not visibility). #[tokio::test] - async fn flushed_pk_index_sidecar_is_probeable() { + async fn sstable_pk_index_sidecar_is_probeable() { use lance_core::cache::LanceCache; use lance_index::metrics::NoOpMetricsCollector; use lance_index::registry::IndexPluginRegistry; @@ -1474,7 +1542,7 @@ mod tests { ) .unwrap(); let frag_id = memtable.insert(batch).await.unwrap(); - memtable.mark_wal_flushed(&[frag_id], 1, &[0]); + let durable = frag_id + 1; let flusher = MemTableFlusher::new( store.clone(), @@ -1484,7 +1552,7 @@ mod tests { manifest_store.clone(), ); let result = flusher - .flush_with_indexes(&memtable, epoch, &[], 1) + .flush_with_indexes(&memtable, epoch, &[], 1, durable) .await .unwrap(); @@ -1493,7 +1561,7 @@ mod tests { .clone() .join("_mem_wal") .join(shard_id.to_string()) - .join(result.generation.path.as_str()); + .join(result.sstable.path.as_str()); let index_store = Arc::new(LanceIndexStore::new( store.clone(), pk_index_path(&gen_path), @@ -1574,7 +1642,7 @@ mod tests { ) .unwrap(); let frag_id = memtable.insert(batch).await.unwrap(); - memtable.mark_wal_flushed(&[frag_id], 1, &[0]); + let durable = frag_id + 1; let flusher = MemTableFlusher::new( store.clone(), @@ -1584,13 +1652,13 @@ mod tests { manifest_store.clone(), ); // The plain-flush path — what the writer dispatches to with no indexes. - let result = flusher.flush(&memtable, epoch, 1).await.unwrap(); + let result = flusher.flush(&memtable, epoch, 1, durable).await.unwrap(); let gen_path = base_path .clone() .join("_mem_wal") .join(shard_id.to_string()) - .join(result.generation.path.as_str()); + .join(result.sstable.path.as_str()); let index_store = Arc::new(LanceIndexStore::new( store.clone(), pk_index_path(&gen_path), @@ -1668,7 +1736,7 @@ mod tests { ) .unwrap(); let frag_id = memtable.insert(batch).await.unwrap(); - memtable.mark_wal_flushed(&[frag_id], 1, &[0]); + let durable = frag_id + 1; let flusher = MemTableFlusher::new( store.clone(), @@ -1678,7 +1746,7 @@ mod tests { manifest_store.clone(), ); let result = flusher - .flush_with_indexes(&memtable, epoch, &index_configs, 1) + .flush_with_indexes(&memtable, epoch, &index_configs, 1, durable) .await .unwrap(); assert_eq!(result.rows_flushed, 5, "all physical rows are written"); @@ -1687,13 +1755,13 @@ mod tests { "{}/_mem_wal/{}/{}", base_uri.trim_end_matches('/'), shard_id, - result.generation.path + result.sstable.path ); let dataset = Dataset::open(&gen_uri).await.unwrap(); assert_eq!( dataset.version().version, 1, - "flushed dataset must be a single-version dataset" + "SSTable dataset must be a single-version dataset" ); // Index half of the combined manifest. @@ -1800,7 +1868,7 @@ mod tests { .unwrap(); // Simulate WAL flush - memtable.mark_wal_flushed(&[frag_id], 1, &[0]); + let durable = frag_id + 1; let flusher = MemTableFlusher::new( store.clone(), @@ -1810,23 +1878,20 @@ mod tests { manifest_store.clone(), ); let result = flusher - .flush_with_indexes(&memtable, epoch, &index_configs, 1) + .flush_with_indexes(&memtable, epoch, &index_configs, 1, durable) .await .unwrap(); - assert_eq!(result.generation.generation, 1); + assert_eq!(result.sstable.generation, 1); assert_eq!(result.rows_flushed, 10); - // Verify the flushed dataset is a single-version dataset with the BTree index - let gen_uri = format!( - "{}/_mem_wal/{}/{}", - base_uri, shard_id, result.generation.path - ); + // Verify the SSTable dataset is a single-version dataset with the BTree index + let gen_uri = format!("{}/_mem_wal/{}/{}", base_uri, shard_id, result.sstable.path); let dataset = Dataset::open(&gen_uri).await.unwrap(); assert_eq!( dataset.version().version, 1, - "flushed dataset must be a single-version dataset" + "SSTable dataset must be a single-version dataset" ); let indices = dataset.load_indices().await.unwrap(); @@ -1937,7 +2002,7 @@ mod tests { let frag_id = memtable.insert(batch).await.unwrap(); // Simulate WAL flush - memtable.mark_wal_flushed(&[frag_id], 1, &[0]); + let durable = frag_id + 1; let flusher = MemTableFlusher::new( store.clone(), @@ -1947,30 +2012,27 @@ mod tests { manifest_store.clone(), ); let result = flusher - .flush_with_indexes(&memtable, epoch, &index_configs, 1) + .flush_with_indexes(&memtable, epoch, &index_configs, 1, durable) .await .unwrap(); - assert_eq!(result.generation.generation, 1); + assert_eq!(result.sstable.generation, 1); assert_eq!(result.rows_flushed, num_vectors); - // Verify the flushed dataset is a single-version dataset with the HNSW index - let gen_uri = format!( - "{}/_mem_wal/{}/{}", - base_uri, shard_id, result.generation.path - ); + // Verify the SSTable dataset is a single-version dataset with the HNSW index + let gen_uri = format!("{}/_mem_wal/{}/{}", base_uri, shard_id, result.sstable.path); let dataset = Dataset::open(&gen_uri).await.unwrap(); assert_eq!( dataset.version().version, 1, - "flushed dataset must be a single-version dataset" + "SSTable dataset must be a single-version dataset" ); let indices = dataset.load_indices().await.unwrap(); assert_eq!(indices.len(), 1); assert_eq!(indices[0].name, "vector_hnsw"); - // End-to-end query: pick a row from the flushed dataset, query for + // End-to-end query: pick a row from the SSTable dataset, query for // it, and verify the index path returns it as the nearest neighbor. // This exercises the on-disk HNSW + SQ8 format including the IVF // partition routing and the storage_metadata ScalarQuantizationMetadata @@ -2087,7 +2149,7 @@ mod tests { let frag_id = memtable.insert(batch).await.unwrap(); // Simulate WAL flush - memtable.mark_wal_flushed(&[frag_id], 1, &[0]); + let durable = frag_id + 1; let flusher = MemTableFlusher::new( store.clone(), @@ -2097,28 +2159,26 @@ mod tests { manifest_store.clone(), ); let result = flusher - .flush_with_indexes(&memtable, epoch, &index_configs, 1) + .flush_with_indexes(&memtable, epoch, &index_configs, 1, durable) .await .unwrap(); - assert_eq!(result.generation.generation, 1); + assert_eq!(result.sstable.generation, 1); assert_eq!(result.rows_flushed, 3); - // Verify the flushed dataset is a single-version dataset with the FTS index - let gen_uri = format!( - "{}/_mem_wal/{}/{}", - base_uri, shard_id, result.generation.path - ); + // Verify the SSTable dataset is a single-version dataset with the FTS index + let gen_uri = format!("{}/_mem_wal/{}/{}", base_uri, shard_id, result.sstable.path); let dataset = Dataset::open(&gen_uri).await.unwrap(); assert_eq!( dataset.version().version, 1, - "flushed dataset must be a single-version dataset" + "SSTable dataset must be a single-version dataset" ); let indices = dataset.load_indices().await.unwrap(); assert_eq!(indices.len(), 1); assert_eq!(indices[0].name, "text_fts"); + assert_eq!(indices[0].index_version, INVERTED_INDEX_VERSION_V2 as i32); // Verify FTS query returns correct results // Searching for "hello" should find the first document @@ -2172,9 +2232,8 @@ mod tests { crate::utils::test::assert_plan_node_equals( plan, "ProjectionExec: expr=[id@2 as id, text@3 as text, _score@1 as _score] - Take: ... - CoalesceBatchesExec: ... - MatchQuery: column=text, query=hello", + LanceRead: ..., source=stream(_rowid) + MatchQuery: column=text, query=[hello]", ) .await .unwrap(); diff --git a/rust/lance/src/dataset/mem_wal/memtable/scanner/builder.rs b/rust/lance/src/dataset/mem_wal/memtable/scanner/builder.rs index e1f0d6e689c..cc921791028 100644 --- a/rust/lance/src/dataset/mem_wal/memtable/scanner/builder.rs +++ b/rust/lance/src/dataset/mem_wal/memtable/scanner/builder.rs @@ -18,6 +18,7 @@ use lance_datafusion::expr::safe_coerce_scalar; use lance_datafusion::planner::Planner; use lance_index::scalar::FullTextSearchQuery; use lance_index::scalar::inverted::query::{FtsQuery as IndexFtsQuery, Operator}; +use lance_index::scalar::inverted::{DOC_INDEX_FIELD, DocumentGranularity}; use lance_linalg::distance::DistanceType; use super::exec::{ @@ -75,7 +76,7 @@ pub enum FtsQueryType { }, /// Boolean query with MUST/SHOULD/MUST_NOT. Boolean { - /// Terms that must match. + /// Terms that must match and contribute to the score. must: Vec, /// Terms that should match (adds to score). should: Vec, @@ -105,6 +106,8 @@ pub struct FtsQuery { pub column: String, /// Query type. pub query_type: FtsQueryType, + /// Logical document unit. Defaults to one document per dataset row. + pub document_granularity: DocumentGranularity, /// WAND factor for early termination (0.0 to 1.0). /// 1.0 = full recall (default), <1.0 = faster but may miss low-scoring results. pub wand_factor: f32, @@ -141,6 +144,7 @@ impl FtsQuery { operator, boost: 1.0, }, + document_granularity: DocumentGranularity::Row, wand_factor: DEFAULT_WAND_FACTOR, limit: None, include_tail: true, @@ -155,6 +159,7 @@ impl FtsQuery { query: query.into(), slop, }, + document_granularity: DocumentGranularity::Row, wand_factor: DEFAULT_WAND_FACTOR, limit: None, include_tail: true, @@ -175,6 +180,7 @@ impl FtsQuery { should, must_not, }, + document_granularity: DocumentGranularity::Row, wand_factor: DEFAULT_WAND_FACTOR, limit: None, include_tail: true, @@ -197,6 +203,7 @@ impl FtsQuery { max_expansions: DEFAULT_MAX_EXPANSIONS, boost: 1.0, }, + document_granularity: DocumentGranularity::Row, wand_factor: DEFAULT_WAND_FACTOR, limit: None, include_tail: true, @@ -218,6 +225,7 @@ impl FtsQuery { max_expansions: DEFAULT_MAX_EXPANSIONS, boost: 1.0, }, + document_granularity: DocumentGranularity::Row, wand_factor: DEFAULT_WAND_FACTOR, limit: None, include_tail: true, @@ -241,6 +249,7 @@ impl FtsQuery { max_expansions, boost: 1.0, }, + document_granularity: DocumentGranularity::Row, wand_factor: DEFAULT_WAND_FACTOR, limit: None, include_tail: true, @@ -269,6 +278,11 @@ impl FtsQuery { self } + pub fn with_document_granularity(mut self, document_granularity: DocumentGranularity) -> Self { + self.document_granularity = document_granularity; + self + } + fn with_boost(mut self, boost: f32) -> Self { match &mut self.query_type { FtsQueryType::Match { boost: b, .. } | FtsQueryType::Fuzzy { boost: b, .. } => { @@ -286,7 +300,30 @@ impl FtsQuery { /// phrase leaf queries; the column must be bound on the query. Compound queries /// (boolean / boost / multi-match) cannot be modeled by the MemTable path and /// return a `not_supported` error rather than failing deep in planning. -fn local_fts_query(query: FullTextSearchQuery) -> Result { +fn resolve_memtable_document_granularity( + column: &str, + requested: Option, + indexes: Option<&IndexStore>, +) -> Result { + let available = indexes + .map(|indexes| indexes.fts_document_granularities_by_column(column)) + .unwrap_or_default(); + match requested { + Some(requested) if available.is_empty() || available.contains(&requested) => Ok(requested), + Some(requested) => Err(Error::invalid_input(format!( + "FTS query for field '{column}' requested {requested:?} document granularity, but \ + the active MemTable FTS index uses a different granularity: {available:?}" + ))), + None if available.is_empty() => Ok(DocumentGranularity::Row), + None if available.len() == 1 => Ok(available[0]), + None => Err(Error::invalid_input(format!( + "FTS query for field '{column}' is ambiguous because Row and ListElement active \ + MemTable indexes coexist; specify document_granularity" + ))), + } +} + +fn local_fts_query(query: FullTextSearchQuery, indexes: Option<&IndexStore>) -> Result { let wand_factor = query.wand_factor.unwrap_or(DEFAULT_WAND_FACTOR); let limit = query .limit @@ -312,7 +349,9 @@ fn local_fts_query(query: FullTextSearchQuery) -> Result { let local = match query.query { IndexFtsQuery::Match(m) => { let column = require_column(m.column)?; - match m.fuzziness { + let document_granularity = + resolve_memtable_document_granularity(&column, m.document_granularity, indexes)?; + let local = match m.fuzziness { // Some(0) is an exact match in the index model. Some(0) => FtsQuery::match_query_with_operator(column, m.terms, m.operator) .with_boost(m.boost), @@ -330,9 +369,16 @@ fn local_fts_query(query: FullTextSearchQuery) -> Result { m.max_expansions, ) .with_boost(m.boost), - } + }; + local.with_document_granularity(document_granularity) + } + IndexFtsQuery::Phrase(p) => { + let column = require_column(p.column)?; + let document_granularity = + resolve_memtable_document_granularity(&column, p.document_granularity, indexes)?; + FtsQuery::phrase(column, p.terms, p.slop) + .with_document_granularity(document_granularity) } - IndexFtsQuery::Phrase(p) => FtsQuery::phrase(require_column(p.column)?, p.terms, p.slop), other => { return Err(Error::not_supported(format!( "MemTable full-text search supports match and phrase queries, got: {other}" @@ -378,7 +424,7 @@ impl ScalarPredicate { /// /// # Index Visibility Model /// -/// The scanner captures `max_visible_batch_position` from the `IndexStore` at +/// The scanner captures `visible_count` from the `IndexStore` at /// construction time. This frozen visibility ensures queries only see data /// that has been indexed, providing consistent results. /// @@ -401,8 +447,8 @@ pub struct MemTableScanner { indexes: Arc, schema: SchemaRef, /// Frozen visibility captured at scanner construction time. - /// This is the `max_visible_batch_position` from the IndexStore. - max_visible_batch_position: usize, + /// This is the `visible_count` from the IndexStore. + visible_count: usize, projection: Option>, filter: Option, limit: Option, @@ -427,7 +473,7 @@ pub struct MemTableScanner { impl MemTableScanner { /// Create a new scanner. /// - /// Captures `max_visible_batch_position` from the `IndexStore` at construction + /// Captures `visible_count` from the `IndexStore` at construction /// time to ensure consistent query visibility. /// /// # Arguments @@ -439,13 +485,13 @@ impl MemTableScanner { // Snapshot the visibility cursor at construction time. The cursor is // advanced by `flush_from_batch_store` after the WAL append succeeds, // so this snapshot reflects WAL-durable data. - let max_visible_batch_position = indexes.max_visible_batch_position(); + let visible_count = indexes.visible_count(); Self { batch_store, indexes, schema, - max_visible_batch_position, + visible_count, projection: None, filter: None, limit: None, @@ -504,12 +550,12 @@ impl MemTableScanner { self } - /// The `max_visible_batch_position` snapshot this scanner latched at + /// The `visible_count` snapshot this scanner latched at /// construction. A downstream recency filter must key on this same snapshot /// (not a fresh read of the IndexStore watermark, which a concurrent append /// could have advanced) so it stays consistent with the rows the search saw. - pub fn max_visible_batch_position(&self) -> usize { - self.max_visible_batch_position + pub fn visible_count(&self) -> usize { + self.visible_count } /// Include the _rowaddr column in output. @@ -688,7 +734,7 @@ impl MemTableScanner { /// queries are supported; compound queries (boolean/boost/multi-match) are /// not yet supported by the MemTable path and return an error. pub fn full_text_search(&mut self, query: FullTextSearchQuery) -> Result<&mut Self> { - self.full_text_query = Some(local_fts_query(query)?); + self.full_text_query = Some(local_fts_query(query, Some(self.indexes.as_ref()))?); Ok(self) } @@ -971,7 +1017,7 @@ impl MemTableScanner { let scan = MemTableScanExec::with_filter( self.batch_store.clone(), - self.max_visible_batch_position, + self.visible_count, projection_indices, self.output_schema(), self.schema.clone(), @@ -1033,7 +1079,7 @@ impl MemTableScanner { Ok(Arc::new(MemTableDedupScanExec::new( self.batch_store.clone(), - self.max_visible_batch_position, + self.visible_count, projection_indices, self.output_schema(), pk_indices, @@ -1056,7 +1102,7 @@ impl MemTableScanner { return self.plan_full_scan().await; } - let max_visible = self.max_visible_batch_position; + let max_visible = self.visible_count; let projection_indices = self.compute_projection_indices()?; let index_exec = BTreeIndexExec::new( @@ -1095,7 +1141,7 @@ impl MemTableScanner { } async fn plan_vector_search(&self, query: &VectorQuery) -> Result> { - let max_visible = self.max_visible_batch_position; + let max_visible = self.visible_count; let projection_indices = self.compute_projection_indices()?; let base_schema = self.base_output_schema(); let filter_predicate = self.filter_predicate()?; @@ -1150,11 +1196,11 @@ impl MemTableScanner { /// Uses the effective visibility (min of max_visible and max_indexed) to ensure /// queries only see indexed data. async fn plan_fts_search(&self, query: &FtsQuery) -> Result> { - if !self.has_fts_index(&query.column) { - return self.empty_fts_plan(); + if !self.has_fts_index(&query.column, query.document_granularity) { + return self.empty_fts_plan(query.document_granularity); } - let max_visible = self.max_visible_batch_position; + let max_visible = self.visible_count; let projection_indices = self.compute_projection_indices()?; let filter_predicate = self.filter_predicate()?; if let Some(pk_columns) = &self.pk_columns { @@ -1175,7 +1221,10 @@ impl MemTableScanner { self.apply_post_index_ops(Arc::new(index_exec)).await } - fn empty_fts_plan(&self) -> Result> { + fn empty_fts_plan( + &self, + document_granularity: DocumentGranularity, + ) -> Result> { use datafusion::physical_plan::empty::EmptyExec; let mut fields: Vec = self @@ -1184,6 +1233,9 @@ impl MemTableScanner { .iter() .map(|f| f.as_ref().clone()) .collect(); + if document_granularity.is_list_element() { + fields.push(DOC_INDEX_FIELD.clone()); + } fields.push(Field::new(SCORE_COLUMN, DataType::Float32, true)); if self.with_row_id { fields.push(Field::new(ROW_ID, DataType::UInt64, true)); @@ -1325,8 +1377,10 @@ impl MemTableScanner { } /// Check if an FTS index exists for a column. - fn has_fts_index(&self, column: &str) -> bool { - self.indexes.get_fts_by_column(column).is_some() + fn has_fts_index(&self, column: &str, document_granularity: DocumentGranularity) -> bool { + self.indexes + .get_fts_by_column_and_granularity(column, document_granularity) + .is_some() } } @@ -1426,7 +1480,7 @@ mod tests { let indexes = Arc::new(index_store); let scanner = MemTableScanner::new(batch_store, indexes, schema.clone()); let result = scanner.try_into_batch().await.unwrap(); - // max_visible_batch_position is 1, so we see batches 0 and 1 (20 rows) + // visible_count is 1, so we see batches 0 and 1 (20 rows) assert_eq!(result.num_rows(), 20); } @@ -1549,20 +1603,67 @@ mod tests { let q = FullTextSearchQuery::new("hello".to_string()) .with_column("text".to_string()) .unwrap(); - let local = local_fts_query(q).unwrap(); + let local = local_fts_query(q, None).unwrap(); assert_eq!(local.column, "text"); assert!( matches!(local.query_type, FtsQueryType::Match { query, operator, .. } if query == "hello" && operator == Operator::Or) ); + let mut indexes = IndexStore::new(); + indexes + .add_fts_with_params( + "tags_list_element_idx".to_string(), + 1, + "tags".to_string(), + lance_index::scalar::inverted::InvertedIndexParams::default() + .document_granularity(DocumentGranularity::ListElement), + ) + .unwrap(); + let inferred = FullTextSearchQuery::new("hello".to_string()) + .with_column("tags".to_string()) + .unwrap(); + let inferred = local_fts_query(inferred, Some(&indexes)).unwrap(); + assert_eq!( + inferred.document_granularity, + DocumentGranularity::ListElement + ); + let conflicting = FullTextSearchQuery::new_query(IndexFtsQuery::Match( + MatchQuery::new("hello".to_string()) + .with_column(Some("tags".to_string())) + .with_document_granularity(DocumentGranularity::Row), + )); + assert!( + local_fts_query(conflicting, Some(&indexes)) + .unwrap_err() + .to_string() + .contains("different granularity") + ); + indexes + .add_fts_with_params( + "tags_idx".to_string(), + 1, + "tags".to_string(), + lance_index::scalar::inverted::InvertedIndexParams::default(), + ) + .unwrap(); + let ambiguous = FullTextSearchQuery::new("hello".to_string()) + .with_column("tags".to_string()) + .unwrap(); + assert!( + local_fts_query(ambiguous, Some(&indexes)) + .unwrap_err() + .to_string() + .contains("ambiguous") + ); + let exact_and = FullTextSearchQuery::new_query(IndexFtsQuery::Match( MatchQuery::new("hello world".to_string()) .with_operator(Operator::And) .with_boost(3.0) .with_column(Some("text".to_string())), )); - let local = local_fts_query(exact_and).unwrap(); + let local = local_fts_query(exact_and, None).unwrap(); assert!( matches!(local.query_type, FtsQueryType::Match { query, operator, boost } if query == "hello world" && operator == Operator::And && boost == 3.0) @@ -1576,7 +1677,7 @@ mod tests { .with_boost(2.5) .with_column(Some("text".to_string())), )); - let local = local_fts_query(fuzzy).unwrap(); + let local = local_fts_query(fuzzy, None).unwrap(); assert!( matches!(local.query_type, FtsQueryType::Fuzzy { fuzziness, prefix_length, boost, .. } if fuzziness == Some(2) && prefix_length == 2 && boost == 2.5) @@ -1589,7 +1690,7 @@ mod tests { .with_column(Some("text".to_string())), )); assert!( - local_fts_query(fuzzy_and).is_err(), + local_fts_query(fuzzy_and, None).is_err(), "fuzzy AND cannot be represented by the local memtable query" ); @@ -1597,7 +1698,7 @@ mod tests { let phrase = FullTextSearchQuery::new_query(IndexFtsQuery::Phrase( PhraseQuery::new("quick fox".to_string()).with_column(Some("text".to_string())), )); - let local = local_fts_query(phrase).unwrap(); + let local = local_fts_query(phrase, None).unwrap(); assert!(matches!(local.query_type, FtsQueryType::Phrase { .. })); // Compound (boolean) -> not supported. @@ -1609,14 +1710,14 @@ mod tests { ), )]))); assert!( - local_fts_query(boolean).is_err(), + local_fts_query(boolean, None).is_err(), "boolean must be rejected" ); // Missing column -> error. let no_col = FullTextSearchQuery::new("hi".to_string()); assert!( - local_fts_query(no_col).is_err(), + local_fts_query(no_col, None).is_err(), "missing column must error" ); } diff --git a/rust/lance/src/dataset/mem_wal/memtable/scanner/exec.rs b/rust/lance/src/dataset/mem_wal/memtable/scanner/exec.rs index 2a593a0f215..9a476edb81b 100644 --- a/rust/lance/src/dataset/mem_wal/memtable/scanner/exec.rs +++ b/rust/lance/src/dataset/mem_wal/memtable/scanner/exec.rs @@ -36,7 +36,7 @@ pub use vector::VectorIndexExec; pub(super) fn newest_pk_positions( batch_store: &BatchStore, pk_columns: &[String], - max_visible_batch_position: usize, + visible_count: usize, max_visible_row: u64, ) -> DataFusionResult> { let mut newest: HashMap, u64> = HashMap::new(); @@ -46,7 +46,7 @@ pub(super) fn newest_pk_positions( if n == 0 { continue; } - if batch_position > max_visible_batch_position { + if batch_position >= visible_count { current_row += n as u64; continue; } diff --git a/rust/lance/src/dataset/mem_wal/memtable/scanner/exec/brute_force_vector.rs b/rust/lance/src/dataset/mem_wal/memtable/scanner/exec/brute_force_vector.rs index 8a239605b50..e852d2ad917 100644 --- a/rust/lance/src/dataset/mem_wal/memtable/scanner/exec/brute_force_vector.rs +++ b/rust/lance/src/dataset/mem_wal/memtable/scanner/exec/brute_force_vector.rs @@ -11,7 +11,6 @@ //! or new rows in the window between commit and next memtable rotation), this //! exec keeps KNN correct by computing exact distances row-by-row. -use std::any::Any; use std::fmt::{Debug, Formatter}; use std::sync::Arc; @@ -40,7 +39,7 @@ use crate::dataset::mem_wal::write::BatchStore; /// Distance metric used when [`VectorQuery::distance_type`] is `None`. The /// indexed path defers to the index's own metric, but with no index there is /// no inherent default — L2 matches what most callers configure and what the -/// flushed/base arms use when re-ranking unindexed candidates. +/// SSTable/base arms use when re-ranking unindexed candidates. const DEFAULT_DISTANCE_TYPE: DistanceType = DistanceType::L2; /// Brute-force KNN over an active memtable without an HNSW. Produces the same @@ -48,7 +47,7 @@ const DEFAULT_DISTANCE_TYPE: DistanceType = DistanceType::L2; pub struct MemTableBruteForceVectorExec { batch_store: Arc, query: VectorQuery, - max_visible_batch_position: usize, + visible_count: usize, projection: Option>, output_schema: SchemaRef, properties: Arc, @@ -69,10 +68,7 @@ impl Debug for MemTableBruteForceVectorExec { f.debug_struct("MemTableBruteForceVectorExec") .field("column", &self.query.column) .field("k", &self.query.k) - .field( - "max_visible_batch_position", - &self.max_visible_batch_position, - ) + .field("visible_count", &self.visible_count) .field("with_row_id", &self.with_row_id) .finish() } @@ -85,7 +81,7 @@ impl MemTableBruteForceVectorExec { pub fn new( batch_store: Arc, query: VectorQuery, - max_visible_batch_position: usize, + visible_count: usize, projection: Option>, base_schema: SchemaRef, with_row_id: bool, @@ -111,7 +107,7 @@ impl MemTableBruteForceVectorExec { Ok(Self { batch_store, query, - max_visible_batch_position, + visible_count, projection, output_schema, properties, @@ -160,7 +156,7 @@ impl MemTableBruteForceVectorExec { Ok(Some(mask)) } - /// Last row position visible under `max_visible_batch_position`, or `None` + /// Last row position visible under `visible_count`, or `None` /// if no batches are visible. Identical to `VectorIndexExec`'s helper so /// both arms cut at the same MVCC boundary. fn compute_max_visible_row(&self) -> Option { @@ -169,7 +165,7 @@ impl MemTableBruteForceVectorExec { for (batch_position, stored_batch) in self.batch_store.iter().enumerate() { let batch_end = current_row + stored_batch.num_rows as u64; - if batch_position <= self.max_visible_batch_position { + if batch_position < self.visible_count { max_visible_row_exclusive = batch_end; } current_row = batch_end; @@ -225,7 +221,7 @@ impl MemTableBruteForceVectorExec { newest_pk_positions( &self.batch_store, pk_columns, - self.max_visible_batch_position, + self.visible_count, max_visible_row, ) .map_err(|e| Error::invalid_input(e.to_string()))?, @@ -245,7 +241,7 @@ impl MemTableBruteForceVectorExec { if n == 0 { continue; } - if batch_position > self.max_visible_batch_position { + if batch_position >= self.visible_count { current_row += n as u64; continue; } @@ -426,10 +422,6 @@ impl ExecutionPlan for MemTableBruteForceVectorExec { "MemTableBruteForceVectorExec" } - fn as_any(&self) -> &dyn Any { - self - } - fn schema(&self) -> SchemaRef { self.output_schema.clone() } @@ -466,12 +458,12 @@ impl ExecutionPlan for MemTableBruteForceVectorExec { ))) } - fn partition_statistics(&self, _partition: Option) -> DataFusionResult { - Ok(Statistics { + fn partition_statistics(&self, _partition: Option) -> DataFusionResult> { + Ok(Arc::new(Statistics { num_rows: Precision::Exact(self.query.k), total_byte_size: Precision::Absent, column_statistics: vec![], - }) + })) } fn metrics(&self) -> Option { @@ -615,7 +607,7 @@ mod tests { MemTableBruteForceVectorExec::new( store, query, - /* max_visible_batch_position = */ usize::MAX, + /* visible_count = */ usize::MAX, None, schema, false, @@ -668,7 +660,7 @@ mod tests { } #[tokio::test] - async fn respects_max_visible_batch_position() { + async fn respects_indexed_count() { // Two batches of two rows. Freeze at batch 0 — only ids 0,1 are // visible candidates; the (closer) ids 2,3 in batch 1 are excluded. let schema = make_schema(); @@ -678,7 +670,7 @@ mod tests { let query = query_for([0.0, 0.0], 4); let exec = Arc::new( MemTableBruteForceVectorExec::new( - store, query, /* max_visible_batch_position = */ 0, None, schema, false, + store, query, /* visible_count = */ 1, None, schema, false, ) .expect("ctor"), ); diff --git a/rust/lance/src/dataset/mem_wal/memtable/scanner/exec/btree.rs b/rust/lance/src/dataset/mem_wal/memtable/scanner/exec/btree.rs index fed61698fab..01781d07ad8 100644 --- a/rust/lance/src/dataset/mem_wal/memtable/scanner/exec/btree.rs +++ b/rust/lance/src/dataset/mem_wal/memtable/scanner/exec/btree.rs @@ -3,7 +3,6 @@ //! BTreeIndexExec - BTree index queries with MVCC visibility. -use std::any::Any; use std::fmt::{Debug, Formatter}; use std::sync::Arc; @@ -31,7 +30,7 @@ pub struct BTreeIndexExec { batch_store: Arc, indexes: Arc, predicate: ScalarPredicate, - max_visible_batch_position: usize, + visible_count: usize, projection: Option>, output_schema: SchemaRef, properties: Arc, @@ -48,10 +47,7 @@ impl Debug for BTreeIndexExec { fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { f.debug_struct("BTreeIndexExec") .field("predicate", &self.predicate) - .field( - "max_visible_batch_position", - &self.max_visible_batch_position, - ) + .field("visible_count", &self.visible_count) .field("with_row_id", &self.with_row_id) .field("with_row_address", &self.with_row_address) .field("column", &self.column) @@ -67,7 +63,7 @@ impl BTreeIndexExec { /// * `batch_store` - Lock-free batch store containing data /// * `indexes` - Index registry with BTree indexes /// * `predicate` - Scalar predicate to apply - /// * `max_visible_batch_position` - MVCC visibility sequence number + /// * `visible_count` - MVCC visibility sequence number /// * `projection` - Optional column indices to project /// * `output_schema` - Schema after projection (should include _rowid/_rowaddr if requested) /// * `with_row_id` - Whether to include _rowid column (row position) @@ -77,7 +73,7 @@ impl BTreeIndexExec { batch_store: Arc, indexes: Arc, predicate: ScalarPredicate, - max_visible_batch_position: usize, + visible_count: usize, projection: Option>, output_schema: SchemaRef, with_row_id: bool, @@ -103,7 +99,7 @@ impl BTreeIndexExec { batch_store, indexes, predicate, - max_visible_batch_position, + visible_count, projection, output_schema, properties, @@ -114,7 +110,7 @@ impl BTreeIndexExec { }) } - /// Compute the maximum visible row position based on max_visible_batch_position. + /// Compute the maximum visible row position based on visible_count. /// Returns None if no batches are visible. fn compute_max_visible_row(&self) -> Option { let mut max_visible_row_exclusive: u64 = 0; @@ -122,7 +118,7 @@ impl BTreeIndexExec { for (batch_position, stored_batch) in self.batch_store.iter().enumerate() { let batch_end = current_row + stored_batch.num_rows as u64; - if batch_position <= self.max_visible_batch_position { + if batch_position < self.visible_count { max_visible_row_exclusive = batch_end; } current_row = batch_end; @@ -312,10 +308,6 @@ impl ExecutionPlan for BTreeIndexExec { "BTreeIndexExec" } - fn as_any(&self) -> &dyn Any { - self - } - fn schema(&self) -> SchemaRef { self.output_schema.clone() } @@ -358,13 +350,13 @@ impl ExecutionPlan for BTreeIndexExec { ))) } - fn partition_statistics(&self, _partition: Option) -> DataFusionResult { + fn partition_statistics(&self, _partition: Option) -> DataFusionResult> { // We can't know the exact count without querying the index - Ok(Statistics { + Ok(Arc::new(Statistics { num_rows: Precision::Absent, total_byte_size: Precision::Absent, column_statistics: vec![], - }) + })) } fn metrics(&self) -> Option { @@ -434,7 +426,7 @@ mod tests { batch_store, indexes, predicate, - 0, // max_visible_batch_position (batch at position 0) + 1, // visible_count (batch at position 0) None, schema, false, @@ -478,7 +470,7 @@ mod tests { batch_store, indexes, predicate, - 0, + 1, None, schema, false, @@ -523,7 +515,7 @@ mod tests { batch_store.clone(), indexes.clone(), predicate.clone(), - 0, + 1, None, schema.clone(), false, @@ -543,7 +535,7 @@ mod tests { batch_store, indexes, predicate, - 1, + 2, None, schema, false, @@ -592,7 +584,7 @@ mod tests { batch_store, indexes, predicate, - 0, + 1, None, schema_with_rowid.clone(), true, @@ -656,7 +648,7 @@ mod tests { batch_store.clone(), indexes.clone(), predicate.clone(), - 0, + 1, None, schema.clone(), false, @@ -684,7 +676,7 @@ mod tests { batch_store, indexes, predicate, - 0, + 1, None, schema_with_rowid, true, diff --git a/rust/lance/src/dataset/mem_wal/memtable/scanner/exec/dedup_scan.rs b/rust/lance/src/dataset/mem_wal/memtable/scanner/exec/dedup_scan.rs index ba5947e4b12..66d73becb69 100644 --- a/rust/lance/src/dataset/mem_wal/memtable/scanner/exec/dedup_scan.rs +++ b/rust/lance/src/dataset/mem_wal/memtable/scanner/exec/dedup_scan.rs @@ -16,7 +16,6 @@ //! forward-aligned mask. A single `filter_record_batch` over the original //! batch then emits the survivors with no per-column reverse copy. -use std::any::Any; use std::collections::HashSet; use std::fmt::{Debug, Formatter}; use std::sync::Arc; @@ -44,7 +43,7 @@ use crate::dataset::mem_wal::write::BatchStore; /// that satisfy the (optional) predicate. See the module doc. pub struct MemTableDedupScanExec { batch_store: Arc, - max_visible_batch_position: usize, + visible_count: usize, /// Column indices to project (into the source schema). projection: Option>, output_schema: SchemaRef, @@ -62,10 +61,7 @@ pub struct MemTableDedupScanExec { impl Debug for MemTableDedupScanExec { fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { f.debug_struct("MemTableDedupScanExec") - .field( - "max_visible_batch_position", - &self.max_visible_batch_position, - ) + .field("visible_count", &self.visible_count) .field("projection", &self.projection) .field("pk_indices", &self.pk_indices) .field("with_row_address", &self.with_row_address) @@ -79,7 +75,7 @@ impl MemTableDedupScanExec { #[allow(clippy::too_many_arguments)] pub fn new( batch_store: Arc, - max_visible_batch_position: usize, + visible_count: usize, projection: Option>, output_schema: SchemaRef, pk_indices: Vec, @@ -97,7 +93,7 @@ impl MemTableDedupScanExec { Self { batch_store, - max_visible_batch_position, + visible_count, projection, output_schema, pk_indices, @@ -155,10 +151,6 @@ impl ExecutionPlan for MemTableDedupScanExec { "MemTableDedupScanExec" } - fn as_any(&self) -> &dyn Any { - self - } - fn schema(&self) -> SchemaRef { self.output_schema.clone() } @@ -188,7 +180,7 @@ impl ExecutionPlan for MemTableDedupScanExec { // back-to-front below. let mut batches = self .batch_store - .visible_batches_with_offsets(self.max_visible_batch_position); + .visible_batches_with_offsets(self.visible_count); batches.reverse(); let projection = self.projection.clone(); @@ -280,12 +272,12 @@ impl ExecutionPlan for MemTableDedupScanExec { ))) } - fn partition_statistics(&self, _partition: Option) -> DataFusionResult { - Ok(Statistics { + fn partition_statistics(&self, _partition: Option) -> DataFusionResult> { + Ok(Arc::new(Statistics { num_rows: Precision::Absent, total_byte_size: Precision::Absent, column_statistics: vec![], - }) + })) } fn metrics(&self) -> Option { @@ -347,7 +339,7 @@ mod tests { /// Run the exec and collect (id -> (value, rowaddr)). async fn run( store: Arc, - max_visible: usize, + visible_count: usize, filter: Option, ) -> HashMap, u64)> { let filter_predicate = filter.map(|expr| { @@ -358,7 +350,7 @@ mod tests { let filter_expr = None; let exec = MemTableDedupScanExec::new( store, - max_visible, + visible_count, None, output_schema(), vec![0], @@ -392,11 +384,11 @@ mod tests { // id=10 inserted (100) then updated to NULL, all in one batch. store.append(batch(&[(10, Some(100)), (10, None)])).unwrap(); - let no_filter = run(store.clone(), 0, None).await; + let no_filter = run(store.clone(), 1, None).await; assert_eq!(no_filter.len(), 1); assert_eq!(no_filter[&10].0, None, "newest version of id=10 is NULL"); - let not_null = run(store, 0, Some(col("value").is_not_null())).await; + let not_null = run(store, 1, Some(col("value").is_not_null())).await; assert!( !not_null.contains_key(&10), "id=10 newest is NULL; the stale value=100 must not leak under value IS NOT NULL" @@ -413,7 +405,7 @@ mod tests { store.append(batch(&[(20, Some(999)), (30, None)])).unwrap(); // No filter: newest per PK = {10:NULL@2, 20:999@3, 30:NULL@4}. - let all = run(store.clone(), 1, None).await; + let all = run(store.clone(), 2, None).await; assert_eq!(all.len(), 3); assert_eq!(all[&10], (None, 2)); assert_eq!(all[&20], (Some(999), 3)); @@ -421,7 +413,7 @@ mod tests { // value IS NOT NULL: only id=20 (newest 999) survives; 10 and 30 are // newest-NULL so they must be absent (no stale leak). - let not_null = run(store, 1, Some(col("value").is_not_null())).await; + let not_null = run(store, 2, Some(col("value").is_not_null())).await; assert_eq!(not_null.len(), 1); assert_eq!(not_null[&20], (Some(999), 3)); } @@ -434,7 +426,7 @@ mod tests { // id=40 inserted NULL then updated to 400 (newest non-NULL). store.append(batch(&[(40, None), (40, Some(400))])).unwrap(); - let is_null = run(store, 0, Some(col("value").is_null())).await; + let is_null = run(store, 1, Some(col("value").is_null())).await; assert!( !is_null.contains_key(&40), "id=40 newest is 400; the stale NULL must not leak under value IS NULL" diff --git a/rust/lance/src/dataset/mem_wal/memtable/scanner/exec/fts.rs b/rust/lance/src/dataset/mem_wal/memtable/scanner/exec/fts.rs index 364261c3276..236208d0eea 100644 --- a/rust/lance/src/dataset/mem_wal/memtable/scanner/exec/fts.rs +++ b/rust/lance/src/dataset/mem_wal/memtable/scanner/exec/fts.rs @@ -3,10 +3,10 @@ //! FtsIndexExec - Full-text search with MVCC visibility. -use std::any::Any; use std::fmt::{Debug, Formatter}; use std::sync::Arc; +use arrow_array::builder::{ListBuilder, UInt32Builder}; use arrow_array::{BooleanArray, Float32Array, RecordBatch, UInt32Array, UInt64Array}; use arrow_schema::{DataType, Field, Schema, SchemaRef}; use datafusion::common::ScalarValue; @@ -23,6 +23,7 @@ use datafusion::physical_plan::{ use datafusion_physical_expr::{EquivalenceProperties, PhysicalExprRef}; use futures::stream::{self, StreamExt}; use lance_core::{Error, Result}; +use lance_index::scalar::inverted::DOC_INDEX_FIELD; use super::super::builder::{FtsQuery, FtsQueryType}; use super::newest_pk_positions; @@ -41,24 +42,31 @@ struct BatchRange { batch_id: usize, } -type MaterializedFtsRows = (Vec>, Vec, Vec); +type MaterializedFtsRows = ( + Vec>, + Vec, + Vec, + Vec>>, +); /// ExecutionPlan node that queries FTS index with MVCC visibility. pub struct FtsIndexExec { batch_store: Arc, indexes: Arc, query: FtsQuery, - max_visible_batch_position: usize, + visible_count: usize, projection: Option>, output_schema: SchemaRef, properties: Arc, metrics: ExecutionPlanMetricsSet, /// Pre-computed batch ranges for O(log n) lookup. batch_ranges: Vec, - /// Maximum visible row position based on max_visible_batch_position (None if nothing visible). + /// Maximum visible row position based on visible_count (None if nothing visible). max_visible_row: Option, /// Whether to include _rowid column (row position) in output. with_row_id: bool, + /// Whether results identify element documents with `_doc_index`. + with_doc_index: bool, /// Optional prefilter predicate, compiled against the memtable schema. /// Applied to the materialized full-schema hits before projection so the /// FTS arm only returns rows matching the predicate. @@ -73,10 +81,7 @@ impl Debug for FtsIndexExec { f.debug_struct("FtsIndexExec") .field("column", &self.query.column) .field("query_type", &self.query.query_type) - .field( - "max_visible_batch_position", - &self.max_visible_batch_position, - ) + .field("visible_count", &self.visible_count) .field("with_row_id", &self.with_row_id) .finish() } @@ -90,7 +95,7 @@ impl FtsIndexExec { /// * `batch_store` - Lock-free batch store containing data /// * `indexes` - Index registry with FTS indexes /// * `query` - FTS query parameters - /// * `max_visible_batch_position` - MVCC visibility sequence number + /// * `visible_count` - MVCC visibility sequence number /// * `projection` - Optional column indices to project /// * `base_schema` - Schema before adding score column (and _rowid if with_row_id) /// * `with_row_id` - Whether to include _rowid column (row position) @@ -98,30 +103,36 @@ impl FtsIndexExec { batch_store: Arc, indexes: Arc, query: FtsQuery, - max_visible_batch_position: usize, + visible_count: usize, projection: Option>, base_schema: SchemaRef, with_row_id: bool, ) -> Result { // Verify the index exists for this column let column = &query.column; - if indexes.get_fts_by_column(column).is_none() { + let Some(_index) = + indexes.get_fts_by_column_and_granularity(column, query.document_granularity) + else { return Err(Error::invalid_input(format!( "No FTS index found for column '{}'", column ))); - } + }; + let with_doc_index = query.document_granularity.is_list_element(); - // Build output schema: base fields + _score + optional _rowid + // Build output schema: base fields + optional _doc_index + _score + optional _rowid let mut fields: Vec = base_schema .fields() .iter() .map(|f| f.as_ref().clone()) .collect(); + if with_doc_index { + fields.push(DOC_INDEX_FIELD.clone()); + } // `_score` is nullable here to stay schema-compatible with - // `lance_index::scalar::inverted::FTS_SCHEMA` (the schema base/flushed + // `lance_index::scalar::inverted::FTS_SCHEMA` (the schema base/SSTable // FTS exec nodes emit). The LSM `full_text_search` planner unions the - // active arm with base/flushed arms; UnionExec requires schema equality + // active arm with base/SSTable arms; UnionExec requires schema equality // including nullability. The actual emitted column is always populated. fields.push(Field::new(SCORE_COLUMN, DataType::Float32, true)); if with_row_id { @@ -149,7 +160,7 @@ impl FtsIndexExec { end: batch_end, batch_id, }); - if batch_id <= max_visible_batch_position { + if batch_id < visible_count { max_visible_row_exclusive = batch_end as u64; } current_row = batch_end; @@ -166,7 +177,7 @@ impl FtsIndexExec { batch_store, indexes, query, - max_visible_batch_position, + visible_count, projection, output_schema, properties, @@ -174,6 +185,7 @@ impl FtsIndexExec { batch_ranges, max_visible_row, with_row_id, + with_doc_index, filter: None, pk_columns: None, }) @@ -203,8 +215,11 @@ impl FtsIndexExec { } /// Query the index and return matching rows with BM25 scores. - fn query_index(&self) -> Vec<(u64, f32)> { - let Some(index) = self.indexes.get_fts_by_column(&self.query.column) else { + fn query_index(&self) -> Vec<(u64, Option>, f32)> { + let Some(index) = self + .indexes + .get_fts_by_column_and_granularity(&self.query.column, self.query.document_granularity) + else { return vec![]; }; @@ -265,21 +280,24 @@ impl FtsIndexExec { } let entries = index.search_with_options(&query_expr, options); - // Convert to (row_position, score) pairs + // Convert to (row_position, element ordinal, score) tuples. entries .into_iter() - .map(|entry| (entry.row_position, entry.score)) + .map(|entry| (entry.row_position, entry.doc_index, entry.score)) .collect() } /// Filter results by MVCC visibility using max_row_position. O(n). - fn filter_by_visibility(&self, results: Vec<(u64, f32)>) -> Vec<(u64, f32)> { + fn filter_by_visibility( + &self, + results: Vec<(u64, Option>, f32)>, + ) -> Vec<(u64, Option>, f32)> { let Some(max_visible) = self.max_visible_row else { return vec![]; }; results .into_iter() - .filter(|&(pos, _)| pos <= max_visible) + .filter(|(pos, _, _)| *pos <= max_visible) .collect() } @@ -289,7 +307,7 @@ impl FtsIndexExec { /// then combines them into a single batch. fn materialize_rows_sorted( &self, - results: &[(u64, f32)], + results: &[(u64, Option>, f32)], ) -> DataFusionResult> { if results.is_empty() { return Ok(vec![]); @@ -299,6 +317,7 @@ impl FtsIndexExec { let mut all_rows: Vec = Vec::with_capacity(results.len()); let mut all_scores: Vec = Vec::with_capacity(results.len()); let mut all_row_positions: Vec = Vec::with_capacity(results.len()); + let mut all_doc_indices: Vec>> = Vec::with_capacity(results.len()); let mut all_columns: Vec>> = Vec::new(); // Initialize column vectors based on first batch's schema @@ -309,7 +328,9 @@ impl FtsIndexExec { } } - for &(pos, score) in results { + for (pos, doc_index, score) in results { + let pos = *pos; + let score = *score; if let Some(batch_range) = self.find_batch(pos as usize) && let Some(stored) = self.batch_store.get(batch_range.batch_id) { @@ -328,6 +349,7 @@ impl FtsIndexExec { all_rows.push(row_in_batch); all_scores.push(score); all_row_positions.push(pos); + all_doc_indices.push(doc_index.clone()); } } @@ -352,7 +374,7 @@ impl FtsIndexExec { // NULL result excludes the row, matching SQL). When a predicate exists, // query_index deliberately avoids pushing the limit into the index so // this remains an exact prefilter, not a lossy post-filter. - let (final_columns, all_scores, all_row_positions) = + let (final_columns, all_scores, all_row_positions, all_doc_indices) = if let Some(ref predicate) = self.filter { let Some(first) = self.batch_store.get(0) else { return Ok(vec![]); @@ -384,13 +406,34 @@ impl FtsIndexExec { .zip(mask.iter()) .filter_map(|(p, keep)| keep.unwrap_or(false).then_some(*p)) .collect(); - (filtered_columns, filtered_scores, filtered_positions) + let filtered_doc_indices: Vec>> = all_doc_indices + .iter() + .zip(mask.iter()) + .filter(|(_, keep)| keep.unwrap_or(false)) + .map(|(index, _)| index.clone()) + .collect(); + ( + filtered_columns, + filtered_scores, + filtered_positions, + filtered_doc_indices, + ) } else { - (final_columns, all_scores, all_row_positions) + ( + final_columns, + all_scores, + all_row_positions, + all_doc_indices, + ) }; - let (mut final_columns, mut all_scores, mut all_row_positions) = - self.filter_to_newest_pk(final_columns, all_scores, all_row_positions)?; + let (mut final_columns, mut all_scores, mut all_row_positions, mut all_doc_indices) = self + .filter_to_newest_pk( + final_columns, + all_scores, + all_row_positions, + all_doc_indices, + )?; if all_scores.is_empty() { return Ok(vec![]); @@ -405,6 +448,26 @@ impl FtsIndexExec { .collect(); all_scores.truncate(limit); all_row_positions.truncate(limit); + all_doc_indices.truncate(limit); + } + + if self.with_doc_index { + let mut builder = ListBuilder::new(UInt32Builder::new()).with_field(Field::new( + "item", + DataType::UInt32, + false, + )); + for doc_index in all_doc_indices { + let doc_index = doc_index.ok_or_else(|| { + datafusion::error::DataFusionError::Internal( + "element-document FTS result is missing its document coordinate" + .to_string(), + ) + })?; + builder.values().append_slice(&doc_index); + builder.append(true); + } + final_columns.push(Arc::new(builder.finish())); } // Add score column @@ -416,6 +479,9 @@ impl FtsIndexExec { .iter() .map(|&i| final_columns[i].clone()) .collect(); + if self.with_doc_index { + projected.push(final_columns[final_columns.len() - 2].clone()); + } // Always include score as last column projected.push(final_columns.last().unwrap().clone()); projected @@ -437,21 +503,47 @@ impl FtsIndexExec { final_columns: Vec>, all_scores: Vec, all_row_positions: Vec, + all_doc_indices: Vec>>, ) -> DataFusionResult { let Some(pk_columns) = &self.pk_columns else { - return Ok((final_columns, all_scores, all_row_positions)); + return Ok(( + final_columns, + all_scores, + all_row_positions, + all_doc_indices, + )); }; if pk_columns.is_empty() || all_scores.is_empty() { - return Ok((final_columns, all_scores, all_row_positions)); + return Ok(( + final_columns, + all_scores, + all_row_positions, + all_doc_indices, + )); } let Some(max_visible_row) = self.max_visible_row else { - return Ok((final_columns, all_scores, all_row_positions)); + return Ok(( + final_columns, + all_scores, + all_row_positions, + all_doc_indices, + )); }; if self.indexes.has_pk_index() && !self.indexes.pk_has_overrides() { - return Ok((final_columns, all_scores, all_row_positions)); + return Ok(( + final_columns, + all_scores, + all_row_positions, + all_doc_indices, + )); } let Some(first) = self.batch_store.get(0) else { - return Ok((final_columns, all_scores, all_row_positions)); + return Ok(( + final_columns, + all_scores, + all_row_positions, + all_doc_indices, + )); }; let newest_positions = if self.indexes.has_pk_index() { None @@ -459,7 +551,7 @@ impl FtsIndexExec { Some(newest_pk_positions( &self.batch_store, pk_columns, - self.max_visible_batch_position, + self.visible_count, max_visible_row, )?) }; @@ -498,8 +590,18 @@ impl FtsIndexExec { .zip(keep.iter()) .filter_map(|(p, keep)| keep.then_some(p)) .collect(); + let filtered_doc_indices = all_doc_indices + .into_iter() + .zip(keep.iter()) + .filter_map(|(index, keep)| keep.then_some(index)) + .collect(); - Ok((filtered_columns, filtered_scores, filtered_positions)) + Ok(( + filtered_columns, + filtered_scores, + filtered_positions, + filtered_doc_indices, + )) } } @@ -529,10 +631,6 @@ impl ExecutionPlan for FtsIndexExec { "FtsIndexExec" } - fn as_any(&self) -> &dyn Any { - self - } - fn schema(&self) -> SchemaRef { self.output_schema.clone() } @@ -565,7 +663,7 @@ impl ExecutionPlan for FtsIndexExec { let mut visible_results = self.filter_by_visibility(results); // Sort by score descending (best matches first) - visible_results.sort_by(|a, b| b.1.partial_cmp(&a.1).unwrap_or(std::cmp::Ordering::Equal)); + visible_results.sort_by(|a, b| b.2.partial_cmp(&a.2).unwrap_or(std::cmp::Ordering::Equal)); // Materialize the rows (preserving sort order) let batches = self.materialize_rows_sorted(&visible_results)?; @@ -578,12 +676,12 @@ impl ExecutionPlan for FtsIndexExec { ))) } - fn partition_statistics(&self, _partition: Option) -> DataFusionResult { - Ok(Statistics { + fn partition_statistics(&self, _partition: Option) -> DataFusionResult> { + Ok(Arc::new(Statistics { num_rows: Precision::Absent, total_byte_size: Precision::Absent, column_statistics: vec![], - }) + })) } fn metrics(&self) -> Option { @@ -602,9 +700,13 @@ impl ExecutionPlan for FtsIndexExec { #[cfg(test)] mod tests { use super::*; - use arrow_array::{Int32Array, StringArray}; + use arrow::array::AsArray; + use arrow_array::builder::{ListBuilder, StringBuilder}; + use arrow_array::{Array, Int32Array, StringArray}; use arrow_schema::{DataType, Field, Schema}; use futures::TryStreamExt; + use lance_index::scalar::InvertedIndexParams; + use lance_index::scalar::inverted::{DOC_INDEX_COL, DocumentGranularity}; fn create_test_schema() -> Arc { Arc::new(Schema::new(vec![ @@ -646,7 +748,7 @@ mod tests { let query = FtsQuery::match_query("text", "hello"); - let exec = FtsIndexExec::new(batch_store, indexes, query, 0, None, schema, false).unwrap(); + let exec = FtsIndexExec::new(batch_store, indexes, query, 1, None, schema, false).unwrap(); let ctx = Arc::new(TaskContext::default()); let stream = exec.execute(0, ctx).unwrap(); @@ -661,6 +763,67 @@ mod tests { assert!(result_schema.field_with_name(SCORE_COLUMN).is_ok()); } + #[tokio::test] + async fn test_element_document_fts_index_search() { + let mut tags = ListBuilder::new(StringBuilder::new()); + tags.values().append_value("alpha beta"); + tags.values().append_value("beta gamma"); + tags.append(true); + tags.values().append_value("beta"); + tags.append(true); + let tags = tags.finish(); + let schema = Arc::new(Schema::new(vec![ + Field::new("id", DataType::Int32, false), + Field::new("tags", tags.data_type().clone(), true), + ])); + let batch = RecordBatch::try_new( + schema.clone(), + vec![Arc::new(Int32Array::from(vec![0, 1])), Arc::new(tags)], + ) + .unwrap(); + let batch_store = Arc::new(BatchStore::with_capacity(10)); + let mut registry = IndexStore::new(); + registry + .add_fts_with_params( + "tags_element_idx".to_string(), + 1, + "tags".to_string(), + InvertedIndexParams::default() + .document_granularity(DocumentGranularity::ListElement), + ) + .unwrap(); + registry.insert(&batch, 0).unwrap(); + batch_store.append(batch).unwrap(); + + let exec = FtsIndexExec::new( + batch_store, + Arc::new(registry), + FtsQuery::match_query("tags", "beta") + .with_document_granularity(DocumentGranularity::ListElement), + 1, + None, + schema, + false, + ) + .unwrap(); + let stream = exec.execute(0, Arc::new(TaskContext::default())).unwrap(); + let batches: Vec = stream.try_collect().await.unwrap(); + let mut hits = Vec::new(); + for batch in batches { + let ids = batch["id"].as_primitive::(); + let coordinates = batch[DOC_INDEX_COL].as_list::(); + for row in 0..batch.num_rows() { + let coordinate = coordinates + .value(row) + .as_primitive::() + .value(0); + hits.push((ids.value(row), coordinate)); + } + } + hits.sort_unstable(); + assert_eq!(hits, vec![(0, 0), (0, 1), (1, 0)]); + } + #[tokio::test] async fn test_fts_index_visibility() { let schema = create_test_schema(); @@ -687,7 +850,7 @@ mod tests { batch_store.clone(), indexes.clone(), query.clone(), - 0, + 1, None, schema.clone(), false, @@ -702,7 +865,7 @@ mod tests { assert_eq!(total_rows, 2); // "hello" in batch1 docs 0 and 2 // Query with max_visible=1 should see both batches - let exec = FtsIndexExec::new(batch_store, indexes, query, 1, None, schema, false).unwrap(); + let exec = FtsIndexExec::new(batch_store, indexes, query, 2, None, schema, false).unwrap(); let ctx = Arc::new(TaskContext::default()); let stream = exec.execute(0, ctx).unwrap(); diff --git a/rust/lance/src/dataset/mem_wal/memtable/scanner/exec/scan.rs b/rust/lance/src/dataset/mem_wal/memtable/scanner/exec/scan.rs index c56e960048d..6126f392f1f 100644 --- a/rust/lance/src/dataset/mem_wal/memtable/scanner/exec/scan.rs +++ b/rust/lance/src/dataset/mem_wal/memtable/scanner/exec/scan.rs @@ -3,7 +3,6 @@ //! MemTableScanExec - Full table scan with MVCC visibility filtering. -use std::any::Any; use std::fmt::{Debug, Formatter}; use std::sync::Arc; @@ -31,12 +30,12 @@ pub const ROW_ADDRESS_COLUMN: &str = "_rowaddr"; /// ExecutionPlan node that scans all visible batches from a MemTable. /// /// This node implements visibility filtering, returning only batches -/// where `batch_position <= max_visible_batch_position`. +/// where `batch_position <= visible_count`. /// /// Supports filter pushdown for efficient predicate evaluation during scan. pub struct MemTableScanExec { batch_store: Arc, - max_visible_batch_position: usize, + visible_count: usize, projection: Option>, output_schema: SchemaRef, /// Schema of the source data (before projection), used for filter evaluation. @@ -56,10 +55,7 @@ pub struct MemTableScanExec { impl Debug for MemTableScanExec { fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { f.debug_struct("MemTableScanExec") - .field( - "max_visible_batch_position", - &self.max_visible_batch_position, - ) + .field("visible_count", &self.visible_count) .field("projection", &self.projection) .field("with_row_id", &self.with_row_id) .field("with_row_address", &self.with_row_address) @@ -74,20 +70,20 @@ impl MemTableScanExec { /// # Arguments /// /// * `batch_store` - Lock-free batch store containing data - /// * `max_visible_batch_position` - Maximum batch position visible (inclusive) + /// * `visible_count` - Maximum batch position visible (inclusive) /// * `projection` - Optional column indices to project /// * `output_schema` - Schema after projection (should include _rowid/_rowaddr if requested) /// * `with_row_id` - Whether to include _rowid column (row position) pub fn new( batch_store: Arc, - max_visible_batch_position: usize, + visible_count: usize, projection: Option>, output_schema: SchemaRef, with_row_id: bool, ) -> Self { Self::with_filter( batch_store, - max_visible_batch_position, + visible_count, projection, output_schema.clone(), output_schema, @@ -103,7 +99,7 @@ impl MemTableScanExec { /// # Arguments /// /// * `batch_store` - Lock-free batch store containing data - /// * `max_visible_batch_position` - Maximum batch position visible (inclusive) + /// * `visible_count` - Maximum batch position visible (inclusive) /// * `projection` - Optional column indices to project /// * `output_schema` - Schema after projection (should include _rowid/_rowaddr if requested) /// * `source_schema` - Schema of source data (before projection), used for filter evaluation @@ -114,7 +110,7 @@ impl MemTableScanExec { #[allow(clippy::too_many_arguments)] pub fn with_filter( batch_store: Arc, - max_visible_batch_position: usize, + visible_count: usize, projection: Option>, output_schema: SchemaRef, source_schema: SchemaRef, @@ -132,7 +128,7 @@ impl MemTableScanExec { Self { batch_store, - max_visible_batch_position, + visible_count, projection, output_schema, source_schema, @@ -194,10 +190,6 @@ impl ExecutionPlan for MemTableScanExec { "MemTableScanExec" } - fn as_any(&self) -> &dyn Any { - self - } - fn schema(&self) -> SchemaRef { self.output_schema.clone() } @@ -226,7 +218,7 @@ impl ExecutionPlan for MemTableScanExec { // Get visible batches with their row offsets let batches_with_offsets = self .batch_store - .visible_batches_with_offsets(self.max_visible_batch_position); + .visible_batches_with_offsets(self.visible_count); let projection = self.projection.clone(); let schema = self.output_schema.clone(); @@ -339,14 +331,14 @@ impl ExecutionPlan for MemTableScanExec { ))) } - fn partition_statistics(&self, _partition: Option) -> DataFusionResult { + fn partition_statistics(&self, _partition: Option) -> DataFusionResult> { // Report statistics as Absent to avoid DataFusion analysis bugs // with selectivity calculation on in-memory tables. - Ok(Statistics { + Ok(Arc::new(Statistics { num_rows: Precision::Absent, total_byte_size: Precision::Absent, column_statistics: vec![], - }) + })) } fn metrics(&self) -> Option { @@ -399,7 +391,7 @@ mod tests { batch_store.append(batch).unwrap(); // Batch is at position 0, max_visible=0 means position 0 is visible - let exec = MemTableScanExec::new(batch_store, 0, None, schema, false); + let exec = MemTableScanExec::new(batch_store, 1, None, schema, false); let ctx = Arc::new(TaskContext::default()); let stream = exec.execute(0, ctx).unwrap(); @@ -425,8 +417,8 @@ mod tests { .append(create_test_batch(&schema, 20, 10)) .unwrap(); - // max_visible_batch_position=1 means positions 0 and 1 are visible (2 batches) - let exec = MemTableScanExec::new(batch_store.clone(), 1, None, schema.clone(), false); + // visible_count=1 means positions 0 and 1 are visible (2 batches) + let exec = MemTableScanExec::new(batch_store.clone(), 2, None, schema.clone(), false); let ctx = Arc::new(TaskContext::default()); let stream = exec.execute(0, ctx).unwrap(); let batches: Vec = stream.try_collect().await.unwrap(); @@ -447,7 +439,7 @@ mod tests { // Project only "id" column (index 0) let projected_schema = Arc::new(Schema::new(vec![Field::new("id", DataType::Int32, false)])); - let exec = MemTableScanExec::new(batch_store, 0, Some(vec![0]), projected_schema, false); + let exec = MemTableScanExec::new(batch_store, 1, Some(vec![0]), projected_schema, false); let ctx = Arc::new(TaskContext::default()); let stream = exec.execute(0, ctx).unwrap(); @@ -464,7 +456,7 @@ mod tests { let batch_store = Arc::new(BatchStore::with_capacity(100)); // Empty store with max_visible=0 should return no batches - let exec = MemTableScanExec::new(batch_store, 0, None, schema, false); + let exec = MemTableScanExec::new(batch_store, 1, None, schema, false); let ctx = Arc::new(TaskContext::default()); let stream = exec.execute(0, ctx).unwrap(); @@ -486,7 +478,7 @@ mod tests { .unwrap(); // max_visible=1 means positions 0 and 1 are visible - let exec = MemTableScanExec::new(batch_store, 1, None, schema, false); + let exec = MemTableScanExec::new(batch_store, 2, None, schema, false); let stats = exec.partition_statistics(None).unwrap(); // Statistics are Absent to avoid DataFusion analysis bugs @@ -513,7 +505,7 @@ mod tests { Field::new("_rowid", DataType::UInt64, true), ])); - let exec = MemTableScanExec::new(batch_store, 1, None, schema_with_rowid, true); + let exec = MemTableScanExec::new(batch_store, 2, None, schema_with_rowid, true); let ctx = Arc::new(TaskContext::default()); let stream = exec.execute(0, ctx).unwrap(); diff --git a/rust/lance/src/dataset/mem_wal/memtable/scanner/exec/vector.rs b/rust/lance/src/dataset/mem_wal/memtable/scanner/exec/vector.rs index c3453db68a2..0c8f4c27ff9 100644 --- a/rust/lance/src/dataset/mem_wal/memtable/scanner/exec/vector.rs +++ b/rust/lance/src/dataset/mem_wal/memtable/scanner/exec/vector.rs @@ -3,7 +3,6 @@ //! VectorIndexExec - HNSW vector search with MVCC visibility. -use std::any::Any; use std::fmt::{Debug, Formatter}; use std::sync::Arc; @@ -35,7 +34,7 @@ pub struct VectorIndexExec { batch_store: Arc, indexes: Arc, query: VectorQuery, - max_visible_batch_position: usize, + visible_count: usize, projection: Option>, output_schema: SchemaRef, properties: Arc, @@ -59,10 +58,7 @@ impl Debug for VectorIndexExec { if let Some(metric) = &self.query.distance_type { debug.field("distance_type", metric); } - debug.field( - "max_visible_batch_position", - &self.max_visible_batch_position, - ); + debug.field("visible_count", &self.visible_count); debug.field("with_row_id", &self.with_row_id); debug.finish() } @@ -76,7 +72,7 @@ impl VectorIndexExec { /// * `batch_store` - Lock-free batch store containing data /// * `indexes` - Index registry with HNSW vector indexes /// * `query` - Vector query parameters - /// * `max_visible_batch_position` - MVCC visibility sequence number + /// * `visible_count` - MVCC visibility sequence number /// * `projection` - Optional column indices to project /// * `base_schema` - Schema after projection (will add _distance column, and _rowid if with_row_id) /// * `with_row_id` - Whether to include _rowid column (row position) @@ -84,7 +80,7 @@ impl VectorIndexExec { batch_store: Arc, indexes: Arc, query: VectorQuery, - max_visible_batch_position: usize, + visible_count: usize, projection: Option>, base_schema: SchemaRef, with_row_id: bool, @@ -120,7 +116,7 @@ impl VectorIndexExec { batch_store, indexes, query, - max_visible_batch_position, + visible_count, projection, output_schema, properties, @@ -129,9 +125,9 @@ impl VectorIndexExec { }) } - /// Compute the maximum visible row position based on max_visible_batch_position. + /// Compute the maximum visible row position based on visible_count. /// - /// Returns the last row position that is visible at the given max_visible_batch_position, + /// Returns the last row position that is visible at the given visible_count, /// or None if no batches are visible. fn compute_max_visible_row(&self) -> Option { let mut max_visible_row_exclusive: u64 = 0; @@ -139,7 +135,7 @@ impl VectorIndexExec { for (batch_position, stored_batch) in self.batch_store.iter().enumerate() { let batch_end = current_row + stored_batch.num_rows as u64; - if batch_position <= self.max_visible_batch_position { + if batch_position < self.visible_count { max_visible_row_exclusive = batch_end; } current_row = batch_end; @@ -310,10 +306,6 @@ impl ExecutionPlan for VectorIndexExec { "VectorIndexExec" } - fn as_any(&self) -> &dyn Any { - self - } - fn schema(&self) -> SchemaRef { self.output_schema.clone() } @@ -355,12 +347,12 @@ impl ExecutionPlan for VectorIndexExec { ))) } - fn partition_statistics(&self, _partition: Option) -> DataFusionResult { - Ok(Statistics { + fn partition_statistics(&self, _partition: Option) -> DataFusionResult> { + Ok(Arc::new(Statistics { num_rows: Precision::Exact(self.query.k), total_byte_size: Precision::Absent, column_statistics: vec![], - }) + })) } fn metrics(&self) -> Option { diff --git a/rust/lance/src/dataset/mem_wal/scanner.rs b/rust/lance/src/dataset/mem_wal/scanner.rs index f1d84611e04..907250a1cb1 100644 --- a/rust/lance/src/dataset/mem_wal/scanner.rs +++ b/rust/lance/src/dataset/mem_wal/scanner.rs @@ -5,8 +5,8 @@ //! //! This module provides scanners that read from multiple data sources //! in an LSM tree architecture: -//! - Base table (merged data) -//! - Flushed MemTables (persisted but not yet merged) +//! - Base table (compacted data) +//! - SSTables (persisted but not yet compacted) //! - Active MemTable (in-memory buffer) //! //! The scanner handles deduplication by primary key, keeping the newest @@ -41,11 +41,11 @@ mod builder; mod collector; mod data_source; pub mod exec; -pub(crate) mod flushed_cache; mod fts_search; mod planner; mod point_lookup; mod projection; +pub(crate) mod sstable_cache; mod vector_search; pub use block_list::write_pk_sidecar; @@ -53,13 +53,11 @@ pub use builder::LsmScanner; pub use collector::{ ActiveMemTableRef, InMemoryMemTableRef, InMemoryMemTables, LsmDataSourceCollector, }; -pub use data_source::{ - FlushedGeneration, FreshTierWatermark, LsmDataSource, LsmGeneration, ShardSnapshot, -}; -pub use flushed_cache::{DatasetCache, FlushedMemTableCache, GenerationWarmer}; +pub use data_source::{FreshTierWatermark, LsmDataSource, LsmGeneration, ShardSnapshot, SsTable}; pub use fts_search::{LsmFtsSearchPlanner, SCORE_COLUMN}; pub use point_lookup::LsmPointLookupPlanner; pub use projection::DISTANCE_COLUMN; +pub use sstable_cache::{DatasetCache, SsTableCache, SsTableWarmer}; pub use vector_search::LsmVectorSearchPlanner; /// Parse a SQL filter expression against a MemWAL source schema. diff --git a/rust/lance/src/dataset/mem_wal/scanner/block_list.rs b/rust/lance/src/dataset/mem_wal/scanner/block_list.rs index 69d16930888..91fdda8c65f 100644 --- a/rust/lance/src/dataset/mem_wal/scanner/block_list.rs +++ b/rust/lance/src/dataset/mem_wal/scanner/block_list.rs @@ -5,7 +5,7 @@ //! //! A generation's membership is a [`GenMembership`]: in-memory generations //! (active / frozen) are probed by value against their maintained primary-key -//! index (no per-query set), while flushed generations are probed against their +//! index (no per-query set), while SSTables are probed against their //! standalone on-disk PK BTree (the sidecar written at flush, opened by path). //! Probing is batched — [`GenMembership::contains_keys`] tests a whole batch of //! keys per generation in one pass. Each source gets a `Vec` of @@ -32,11 +32,12 @@ use lance_index::scalar::{ use uuid::Uuid; use super::data_source::{FreshTierWatermark, LsmDataSource, LsmGeneration}; -use super::flushed_cache::{DatasetCache, open_flushed_dataset}; +use super::sstable_cache::{DatasetCache, open_sstable}; use crate::dataset::mem_wal::index::encode_pk_tuple; use crate::dataset::mem_wal::util::PK_INDEX_DIR; use crate::dataset::mem_wal::write::{BatchStore, IndexStore}; use crate::session::Session; +use lance_io::object_store::ObjectStoreParams; /// Default-plugin registry, used only to load the standalone PK BTree by its /// `BTreeIndexDetails` type. Built once. @@ -54,7 +55,7 @@ pub enum GenMembership { /// Inclusive visible row watermark; `None` when no rows are visible. max_visible_row: Option, }, - /// Probe the flushed generation's standalone on-disk PK BTree. + /// Probe the SSTable's standalone on-disk PK BTree. OnDisk(Arc), } @@ -110,7 +111,7 @@ impl GenMembership { } /// Whether this generation has no (visible) membership — used to skip adding - /// an empty blocked set. A flushed generation always has rows (flush rejects + /// an empty blocked set. An SSTable always has rows (flush rejects /// an empty memtable), so it is never empty. fn is_empty(&self) -> bool { match self { @@ -157,7 +158,8 @@ type ShardGenSets = HashMap>; pub async fn compute_source_block_lists( sources: &[LsmDataSource], session: Option<&Arc>, - flushed_cache: Option<&Arc>, + store_params: Option<&ObjectStoreParams>, + sstable_cache: Option<&Arc>, ) -> Result { // Membership per non-base source, grouped by shard (generations are // per-shard, so supersession is within-shard only). @@ -165,7 +167,7 @@ pub async fn compute_source_block_lists( let mut has_base = false; // Flushed PK-BTree opens are cold S3 reads; overlap them with // `try_join_all`. Order is irrelevant — gens are sorted per-shard below. - let mut flushed_loads = Vec::new(); + let mut sstable_loads = Vec::new(); for source in sources { match source { LsmDataSource::BaseTable { .. } => has_base = true, @@ -182,18 +184,18 @@ pub async fn compute_source_block_lists( .or_default() .push((*generation, membership)); } - LsmDataSource::FlushedMemTable { + LsmDataSource::SsTable { path, shard_id, generation, .. - } => flushed_loads.push(async move { - let index = open_pk_index(path, session, flushed_cache).await?; + } => sstable_loads.push(async move { + let index = open_pk_index(path, session, store_params, sstable_cache).await?; Ok::<_, Error>((*shard_id, *generation, GenMembership::OnDisk(index))) }), } } - for (shard_id, generation, membership) in futures::future::try_join_all(flushed_loads).await? { + for (shard_id, generation, membership) in futures::future::try_join_all(sstable_loads).await? { by_shard .entry(shard_id) .or_default() @@ -225,7 +227,7 @@ pub async fn compute_source_block_lists( /// The fresh-tier block-list: one [`GenMembership`] per generation that shadows /// the base table — active + frozen memtables (probed against their index) and -/// flushed generations (probed against their on-disk PK BTree). A base/external +/// SSTables (probed against their on-disk PK BTree). A base/external /// reader can test any PK against these (via [`GenMembership::contains`]) to /// decide whether the fresh tier shadows it. The base source, if present, is /// skipped (it is what gets shadowed). @@ -238,14 +240,15 @@ pub async fn compute_source_block_lists( pub async fn fresh_tier_block_list( sources: &[LsmDataSource], session: Option<&Arc>, - flushed_cache: Option<&Arc>, + store_params: Option<&ObjectStoreParams>, + sstable_cache: Option<&Arc>, watermarks: Option<&HashMap>, ) -> Result> { // Membership per source, in source order (`None` = skipped). Flushed // PK-BTree opens are cold S3 reads, so collect them tagged with their slot // and overlap with `try_join_all` rather than opening one at a time. let mut slots: Vec> = Vec::with_capacity(sources.len()); - let mut flushed_loads = Vec::new(); + let mut sstable_loads = Vec::new(); for source in sources { match source { LsmDataSource::BaseTable { .. } => slots.push(None), @@ -278,7 +281,7 @@ pub async fn fresh_tier_block_list( }; slots.push(membership); } - LsmDataSource::FlushedMemTable { + LsmDataSource::SsTable { path, shard_id, generation, @@ -298,15 +301,16 @@ pub async fn fresh_tier_block_list( } else { let slot = slots.len(); slots.push(None); - flushed_loads.push(async move { - let index = open_pk_index(path, session, flushed_cache).await?; + sstable_loads.push(async move { + let index = + open_pk_index(path, session, store_params, sstable_cache).await?; Ok::<_, Error>((slot, GenMembership::OnDisk(index))) }); } } } } - for (slot, membership) in futures::future::try_join_all(flushed_loads).await? { + for (slot, membership) in futures::future::try_join_all(sstable_loads).await? { slots[slot] = Some(membership); } Ok(slots @@ -324,7 +328,7 @@ fn in_memory_membership( batch_store: &Arc, index_store: &Arc, ) -> GenMembership { - let max_visible_row = batch_store.max_visible_row(index_store.max_visible_batch_position()); + let max_visible_row = batch_store.max_visible_row(index_store.visible_count()); GenMembership::InMemory { index_store: index_store.clone(), max_visible_row, @@ -342,20 +346,21 @@ fn bounded_in_memory_membership( index_store: &Arc, batch_count: u64, ) -> GenMembership { - let max_visible_row = batch_count - .checked_sub(1) - .and_then(|last_batch| batch_store.max_visible_row(last_batch as usize)); + // `batch_count` is already an exclusive count, and so is what + // `max_visible_row` takes, so it passes straight through: no + // count-to-inclusive-position conversion, and `0` yields `None` on its own. + let max_visible_row = batch_store.max_visible_row(batch_count as usize); GenMembership::InMemory { index_store: index_store.clone(), max_visible_row, } } -/// Open the standalone PK BTree at `{flushed gen}/_pk_index` for one flushed -/// generation. Reuses the flushed dataset's (session-configured) object store +/// Open the standalone PK BTree at `{SSTable gen}/_pk_index` for one +/// SSTable. Reuses the SSTable dataset's (session-configured) object store /// and **its index cache**, then loads the sidecar directly by path through the /// BTree plugin — it is not a manifest index. The opened index and its pages -/// are cached in the session's index cache (keyed by the immutable flushed +/// are cached in the session's index cache (keyed by the immutable SSTable /// path), so repeated probes reuse them with no separate cache path and no /// upfront scan; concurrent first-opens may each load before the cache fills. /// A stable cache UUID for a non-manifest index identified only by its path. @@ -379,12 +384,13 @@ fn path_cache_uuid(path: &str) -> Uuid { async fn open_pk_index( path: &str, session: Option<&Arc>, - flushed_cache: Option<&Arc>, + store_params: Option<&ObjectStoreParams>, + sstable_cache: Option<&Arc>, ) -> Result> { - let dataset = open_flushed_dataset(path, session, flushed_cache, None).await?; - // Namespace the session index cache by the (immutable) flushed path so this + let dataset = open_sstable(path, session, store_params, sstable_cache, None).await?; + // Namespace the session index cache by the (immutable) SSTable path so this // sidecar's pages live alongside every other index instead of a bespoke - // cache. `fri_uuid` is None — flushed generations carry no fragment-reuse. + // cache. `fri_uuid` is None — SSTables carry no fragment-reuse. let index_cache = dataset.index_cache.for_index(&path_cache_uuid(path), None); let index_dir = dataset.base.clone().join(PK_INDEX_DIR); let store: Arc = Arc::new(LanceIndexStore::new( @@ -410,13 +416,13 @@ async fn open_pk_index( Ok(index) } -/// Write a flushed generation's standalone PK sidecar at `{uri}/_pk_index` from +/// Write an SSTable's standalone PK sidecar at `{uri}/_pk_index` from /// `batches`, mirroring what flush does in production. `pk_columns` are the /// primary-key column names (field ids are synthesized by position — `insert` /// resolves columns by name). A no-op when no batch carries the PK columns. /// /// Used by Rust scanner tests and by the Python test-support binding to stage -/// faithful flushed generations (a flushed dataset alone, with no sidecar, is +/// faithful SSTables (an SSTable dataset alone, with no sidecar, is /// not a state production ever produces). pub async fn write_pk_sidecar( uri: &str, @@ -534,7 +540,7 @@ mod tests { active_source(shard, 1, &[3]), ]; - let memberships = fresh_tier_block_list(&sources, None, None, None) + let memberships = fresh_tier_block_list(&sources, None, None, None, None) .await .unwrap(); @@ -555,7 +561,7 @@ mod tests { active_source(shard, 2, &[1, 2]), ]; - let blocked = Box::pin(compute_source_block_lists(&sources, None, None)) + let blocked = Box::pin(compute_source_block_lists(&sources, None, None, None)) .await .unwrap(); @@ -591,7 +597,7 @@ mod tests { active_source(Uuid::new_v4(), 1, &[1, 2]), ]; - let blocked = Box::pin(compute_source_block_lists(&sources, None, None)) + let blocked = Box::pin(compute_source_block_lists(&sources, None, None, None)) .await .unwrap(); @@ -619,7 +625,7 @@ mod tests { active_source(b, 2, &[2]), ]; - let blocked = Box::pin(compute_source_block_lists(&sources, None, None)) + let blocked = Box::pin(compute_source_block_lists(&sources, None, None, None)) .await .unwrap(); @@ -670,7 +676,7 @@ mod tests { generation: LsmGeneration::memtable(2), }; - let blocked = Box::pin(compute_source_block_lists(&[g1, g2], None, None)) + let blocked = Box::pin(compute_source_block_lists(&[g1, g2], None, None, None)) .await .unwrap(); @@ -708,7 +714,7 @@ mod tests { )] .into_iter() .collect(); - let sets = fresh_tier_block_list(&sources, None, None, Some(&watermarks)) + let sets = fresh_tier_block_list(&sources, None, None, None, Some(&watermarks)) .await .unwrap(); assert!(blocks(&sets, 1).await); @@ -716,7 +722,7 @@ mod tests { assert!(!blocks(&sets, 3).await); // No watermark → live tier: all three are members. - let sets = fresh_tier_block_list(&sources, None, None, None) + let sets = fresh_tier_block_list(&sources, None, None, None, None) .await .unwrap(); for id in [1, 2, 3] { @@ -750,7 +756,7 @@ mod tests { )] .into_iter() .collect(); - let sets = fresh_tier_block_list(&sources, None, None, Some(&watermarks)) + let sets = fresh_tier_block_list(&sources, None, None, None, Some(&watermarks)) .await .unwrap(); assert!(blocks(&sets, 1).await); // gen 1, whole @@ -760,32 +766,32 @@ mod tests { assert!(!blocks(&sets, 100).await); // gen 3 — after the snapshot } - /// A flushed generation at or above the active generation was produced by a + /// An SSTable at or above the active generation was produced by a /// flush after the snapshot and is excluded; one strictly below it is /// immutable and included. #[tokio::test] - async fn fresh_tier_watermark_excludes_flushed_at_or_above_active() { + async fn fresh_tier_watermark_excludes_sstables_at_or_above_active() { use crate::dataset::mem_wal::scanner::data_source::FreshTierWatermark; use crate::dataset::{Dataset, WriteParams}; use arrow_array::RecordBatchIterator; use std::collections::HashMap; - // A flushed generation 2 holding pk=5, staged as a flushed dataset with + // An SSTable 2 holding pk=5, staged as an SSTable dataset with // its standalone PK sidecar (what the on-disk membership probes). - let flushed_batch = id_batch(&[5]); - let schema = flushed_batch.schema(); + let sstable_batch = id_batch(&[5]); + let schema = sstable_batch.schema(); let tmp = tempfile::tempdir().unwrap(); let path = format!("{}/gen2", tmp.path().to_str().unwrap()); - let reader = RecordBatchIterator::new(vec![Ok(flushed_batch.clone())], schema.clone()); + let reader = RecordBatchIterator::new(vec![Ok(sstable_batch.clone())], schema.clone()); Dataset::write(reader, &path, Some(WriteParams::default())) .await .unwrap(); - write_pk_sidecar(&path, &[flushed_batch], &["id"]) + write_pk_sidecar(&path, &[sstable_batch], &["id"]) .await .unwrap(); let shard = Uuid::new_v4(); - let sources = vec![LsmDataSource::FlushedMemTable { + let sources = vec![LsmDataSource::SsTable { path, shard_id: shard, generation: LsmGeneration::memtable(2), @@ -801,7 +807,7 @@ mod tests { )] .into_iter() .collect(); - let sets = fresh_tier_block_list(&sources, None, None, Some(&at)) + let sets = fresh_tier_block_list(&sources, None, None, None, Some(&at)) .await .unwrap(); assert!(!blocks(&sets, 5).await); @@ -816,7 +822,7 @@ mod tests { )] .into_iter() .collect(); - let sets = fresh_tier_block_list(&sources, None, None, Some(&above)) + let sets = fresh_tier_block_list(&sources, None, None, None, Some(&above)) .await .unwrap(); assert!(blocks(&sets, 5).await); diff --git a/rust/lance/src/dataset/mem_wal/scanner/builder.rs b/rust/lance/src/dataset/mem_wal/scanner/builder.rs index 2947ef1464f..9da723886f1 100644 --- a/rust/lance/src/dataset/mem_wal/scanner/builder.rs +++ b/rust/lance/src/dataset/mem_wal/scanner/builder.rs @@ -3,8 +3,8 @@ //! LSM Scanner builder. -use std::collections::HashMap; use std::collections::hash_map::Entry; +use std::collections::{HashMap, HashSet}; use std::sync::Arc; use arrow_array::builder::{FixedSizeListBuilder, Float32Builder}; @@ -19,18 +19,21 @@ use datafusion::physical_plan::{ExecutionPlan, SendableRecordBatchStream}; use datafusion::prelude::{Expr, SessionContext}; use futures::TryStreamExt; use lance_core::{Error, Result, is_system_column}; +use lance_datafusion::expr::safe_coerce_scalar; use lance_index::scalar::FullTextSearchQuery; use lance_linalg::distance::DistanceType; use uuid::Uuid; use super::collector::{InMemoryMemTableRef, InMemoryMemTables, LsmDataSourceCollector}; use super::data_source::{FreshTierWatermark, ShardSnapshot}; -use super::flushed_cache::{DatasetCache, GenerationWarmer}; use super::planner::LsmScanPlanner; use super::point_lookup::LsmPointLookupPlanner; use super::projection::validate_projection_names; +use super::sstable_cache::{DatasetCache, SsTableWarmer}; use crate::dataset::Dataset; +use crate::dataset::mem_wal::util::derived_store_params; use crate::session::Session; +use lance_io::object_store::ObjectStoreParams; /// Vector (KNN) search state, set by [`LsmScanner::nearest`] and friends. Mirrors /// the subset of `lance::dataset::scanner::Query` the LSM vector planner honors. @@ -54,9 +57,15 @@ struct LsmVectorQuery { /// If `filter` is a point-lookup shape on `pk_col` — `pk = lit` (either /// operand order) or `pk IN (lit, …)` — return the literal key values. Any /// other shape returns `None`, so the scanner falls through to the general -/// scan plan. Type coercion is left to the lookup path (an exact-type literal -/// takes the fast BTree path; a coercible one falls back internally). -fn extract_pk_point_keys(filter: &Expr, pk_col: &str) -> Option> { +/// scan plan. `IN` keys are normalized to `pk_type` only for deduplication; +/// the first original literal is retained so the lookup path still chooses +/// between its exact-type fast path and coercing fallback. A literal that +/// cannot be normalized routes the expression through the general scan. +fn extract_pk_point_keys( + filter: &Expr, + pk_col: &str, + pk_type: &DataType, +) -> Option> { match filter { Expr::BinaryExpr(b) if matches!(b.op, Operator::Eq) => { match (b.left.as_ref(), b.right.as_ref()) { @@ -77,11 +86,15 @@ fn extract_pk_point_keys(filter: &Expr, pk_col: &str) -> Option return None; } let mut vals = Vec::with_capacity(in_list.list.len()); + let mut seen = HashSet::with_capacity(in_list.list.len()); for e in &in_list.list { let Expr::Literal(lit, _) = e else { return None; // a non-literal IN element → not a point lookup }; - vals.push(lit.clone()); + let identity = safe_coerce_scalar(lit, pk_type)?; + if seen.insert(identity) { + vals.push(lit.clone()); + } } (!vals.is_empty()).then_some(vals) } @@ -90,7 +103,7 @@ fn extract_pk_point_keys(filter: &Expr, pk_col: &str) -> Option } /// Either a base Lance table, or an explicit base path used to resolve -/// flushed-generation directories when no base dataset is configured. +/// SSTable directories when no base dataset is configured. enum BaseSource { Table(Arc), PathOnly(String), @@ -150,12 +163,12 @@ fn key_to_fsl(key: &dyn Array, dim: i32) -> Result { Ok(builder.finish()) } -/// Scanner for LSM tree data spanning base table, flushed MemTables, and active MemTable. +/// Scanner for LSM tree data spanning base table, SSTables, and active MemTable. /// /// This scanner provides a unified interface for querying data across multiple /// LSM tree levels: -/// - Base table (merged data, generation = 0) -/// - Flushed MemTables (persisted but not yet merged, generation = 1, 2, ...) +/// - Base table (compacted data, generation = 0) +/// - SSTables (persisted but not yet compacted, generation = 1, 2, ...) /// - Active MemTable (in-memory buffer, highest generation) /// /// The scanner automatically handles deduplication by primary key, keeping @@ -206,15 +219,17 @@ pub struct LsmScanner { // Primary key columns (required for deduplication) pk_columns: Vec, - /// Session threaded into flushed-generation opens so the first open of - /// each generation populates the shared index / file-metadata caches. - /// Defaults to the base table's session when one is present. + /// Session for opening SSTables (shares the base's caches). + /// Defaults to the base table's session. session: Option>, - /// Cache of opened flushed-generation datasets. When set, repeated + /// Store params for opening SSTables, reusing the base dataset's + /// store. Defaults to the base table's params. + store_params: Option, + /// Cache of opened SSTable datasets. When set, repeated /// queries against the same generation skip the manifest read entirely. - flushed_cache: Option>, - /// Optional warmer fired on first open of a flushed generation. - warmer: Option>, + sstable_cache: Option>, + /// Optional warmer fired on first open of an SSTable. + warmer: Option>, /// Over-fetch multiple for block-listed sources in search plans /// (see [`super::LsmFtsSearchPlanner::with_overfetch_factor`]). overfetch_factor: Option, @@ -225,7 +240,7 @@ impl LsmScanner { /// /// # Arguments /// - /// * `base_table` - The base Lance table (merged data) + /// * `base_table` - The base Lance table (compacted data) /// * `shard_snapshots` - Snapshots of shard states from MemWAL index /// * `pk_columns` - Primary key column names for deduplication pub fn new( @@ -239,6 +254,10 @@ impl LsmScanner { // the shared index / metadata caches without extra wiring. An // explicit `with_session` still overrides this. let session = Some(base_table.session()); + // The scanner only ever opens SSTables with these — the base + // table is already open and handed in — so they must not carry a + // path-bound store binding. + let store_params = base_table.store_params().map(derived_store_params); Self { base: BaseSource::Table(base_table), schema: Arc::new(arrow_schema), @@ -254,25 +273,26 @@ impl LsmScanner { with_memtable_gen: false, pk_columns, session, - flushed_cache: None, + store_params, + sstable_cache: None, warmer: None, overfetch_factor: None, } } /// Create a scanner that reads only the fresh tier (active memtable and - /// flushed generations) without including a base Lance table. + /// SSTables) without including a base Lance table. /// /// This is useful when the caller owns the base read path separately and - /// only needs the WAL's contribution: active memtable ∪ L0 flushed - /// generations. Deduplication semantics are unchanged — newer generations + /// only needs the WAL's contribution: active memtable ∪ L0 SSTables. + /// Deduplication semantics are unchanged — newer generations /// still win on PK conflicts. /// /// # Arguments /// /// * `schema` - Schema used for projection, filter parsing, and empty plans. - /// Should match the schema flushed generations were written with. - /// * `base_path` - Table-root URI used to resolve relative flushed paths. + /// Should match the schema SSTables were written with. + /// * `base_path` - Table-root URI used to resolve relative SSTable paths. /// * `shard_snapshots` - Snapshots of shard states from MemWAL index. /// * `pk_columns` - Primary key column names for deduplication. pub fn without_base_table( @@ -296,7 +316,8 @@ impl LsmScanner { with_memtable_gen: false, pk_columns, session: None, - flushed_cache: None, + store_params: None, + sstable_cache: None, warmer: None, overfetch_factor: None, } @@ -331,33 +352,40 @@ impl LsmScanner { self } - /// Thread an existing session into flushed-generation opens. - /// - /// The first open of each flushed generation then populates the shared - /// index / file-metadata caches, so later queries skip re-decoding them. - /// When a base table is configured this defaults to its session; call - /// this to override (e.g. on a fresh-tier-only scanner that owns its own - /// long-lived session). + /// Set the session used to open SSTables. Defaults to the base + /// table's; set explicitly on a fresh-tier-only scanner (no base table). pub fn with_session(mut self, session: Arc) -> Self { self.session = Some(session); self } - /// Inject a cache of opened flushed-generation datasets. + /// Set the store params used to open SSTables. Defaults to the + /// base table's; set explicitly on a fresh-tier-only scanner (no base table). + /// + /// Pass the params the *base* was opened with. As in [`Self::new`], they are + /// adapted for generation URIs: a path-bound `object_store` binding would + /// redirect every generation open at the base table itself, so it is dropped + /// while storage options, wrapper, and credentials carry over. + pub fn with_store_params(mut self, store_params: ObjectStoreParams) -> Self { + self.store_params = Some(derived_store_params(&store_params)); + self + } + + /// Inject a cache of opened SSTable datasets. /// /// With a cache, repeated queries against the same generation become a /// pure `Arc::clone` with no manifest read or object-store I/O. The cache /// is owned and sized by the caller (any [`DatasetCache`] impl, e.g. - /// [`FlushedMemTableCache`](super::FlushedMemTableCache)); not set by + /// [`SsTableCache`](super::SsTableCache)); not set by /// default, so behavior is unchanged unless opted in. - pub fn with_flushed_cache(mut self, cache: Arc) -> Self { - self.flushed_cache = Some(cache); + pub fn with_sstable_cache(mut self, cache: Arc) -> Self { + self.sstable_cache = Some(cache); self } - /// Inject the warmer fired on first open of a flushed generation. Not set by + /// Inject the warmer fired on first open of an SSTable. Not set by /// default, so behavior is unchanged unless opted in. - pub fn with_warmer(mut self, warmer: Arc) -> Self { + pub fn with_warmer(mut self, warmer: Arc) -> Self { self.warmer = Some(warmer); self } @@ -419,7 +447,7 @@ impl LsmScanner { } /// Find the `k` nearest neighbors of `key` in `column`. Routes `create_plan` - /// through the LSM vector planner (base ∪ flushed ∪ in-memory). Mirrors + /// through the LSM vector planner (base ∪ SSTables ∪ in-memory). Mirrors /// [`crate::dataset::scanner::Scanner::nearest`]; the LSM path supports a /// single Float32 query vector. When combined with an offset, the LSM path /// fetches `k + offset` per source before applying the final page. Tune with @@ -537,7 +565,7 @@ impl LsmScanner { Arc::new(GlobalLimitExec::new(plan, skip, self.limit)) } - /// Vector (KNN) search across base ∪ flushed ∪ in-memory, via the LSM vector + /// Vector (KNN) search across base ∪ SSTables ∪ in-memory, via the LSM vector /// planner. Honors the builder filter as a prefilter. async fn plan_vector(&self) -> Result> { let nearest = self @@ -564,8 +592,11 @@ impl LsmScanner { if let Some(session) = &self.session { planner = planner.with_session(session.clone()); } - if let Some(cache) = &self.flushed_cache { - planner = planner.with_flushed_cache(cache.clone()); + if let Some(store_params) = &self.store_params { + planner = planner.with_store_params(store_params.clone()); + } + if let Some(cache) = &self.sstable_cache { + planner = planner.with_sstable_cache(cache.clone()); } if let Some(warmer) = &self.warmer { planner = planner.with_warmer(warmer.clone()); @@ -588,7 +619,7 @@ impl LsmScanner { Ok(self.apply_limit_offset(plan)) } - /// Full-text search across base ∪ flushed ∪ in-memory, via the LSM FTS + /// Full-text search across base ∪ SSTables ∪ in-memory, via the LSM FTS /// planner. Query/scanner limits bound per-source fetches when present; /// otherwise the search remains unbounded and any offset is applied above. async fn plan_fts(&self) -> Result> { @@ -609,9 +640,6 @@ impl LsmScanner { ) })?; let base_schema = self.schema(); - base_schema.field_with_name(&column).map_err(|_| { - Error::invalid_input(format!("Column '{}' not found in schema", column)) - })?; let query_limit = query .limit .map(|limit| { @@ -640,8 +668,11 @@ impl LsmScanner { if let Some(session) = &self.session { planner = planner.with_session(session.clone()); } - if let Some(cache) = &self.flushed_cache { - planner = planner.with_flushed_cache(cache.clone()); + if let Some(store_params) = &self.store_params { + planner = planner.with_store_params(store_params.clone()); + } + if let Some(cache) = &self.sstable_cache { + planner = planner.with_sstable_cache(cache.clone()); } if let Some(warmer) = &self.warmer { planner = planner.with_warmer(warmer.clone()); @@ -660,7 +691,7 @@ impl LsmScanner { Ok(self.apply_limit_offset(plan)) } - /// Plain (filter / projection / limit) scan over base ∪ flushed ∪ in-memory. + /// Plain (filter / projection / limit) scan over base ∪ SSTables ∪ in-memory. async fn plan_scan(&self) -> Result> { let collector = self.build_collector(); let base_schema = self.schema(); @@ -678,15 +709,20 @@ impl LsmScanner { && !self.with_row_address && !self.projection_has_system_columns() && let Some(filter) = &self.filter - && let Some(keys) = extract_pk_point_keys(filter, &self.pk_columns[0]) + && let Ok(pk_field) = base_schema.field_with_name(&self.pk_columns[0]) + && let Some(keys) = + extract_pk_point_keys(filter, &self.pk_columns[0], pk_field.data_type()) { let mut planner = LsmPointLookupPlanner::new(collector, self.pk_columns.clone(), base_schema); if let Some(session) = &self.session { planner = planner.with_session(session.clone()); } - if let Some(cache) = &self.flushed_cache { - planner = planner.with_flushed_cache(cache.clone()); + if let Some(store_params) = &self.store_params { + planner = planner.with_store_params(store_params.clone()); + } + if let Some(cache) = &self.sstable_cache { + planner = planner.with_sstable_cache(cache.clone()); } if let Some(warmer) = &self.warmer { planner = planner.with_warmer(warmer.clone()); @@ -704,15 +740,15 @@ impl LsmScanner { if let Some(session) = &self.session { planner = planner.with_session(session.clone()); } - if let Some(cache) = &self.flushed_cache { - planner = planner.with_flushed_cache(cache.clone()); + if let Some(store_params) = &self.store_params { + planner = planner.with_store_params(store_params.clone()); + } + if let Some(cache) = &self.sstable_cache { + planner = planner.with_sstable_cache(cache.clone()); } if let Some(warmer) = &self.warmer { planner = planner.with_warmer(warmer.clone()); } - if let Some(factor) = self.overfetch_factor { - planner = planner.with_overfetch_factor(factor); - } planner .plan_scan( @@ -727,7 +763,7 @@ impl LsmScanner { } /// Find rows matching a full-text query. Routes `create_plan` through the - /// LSM FTS planner (base ∪ flushed ∪ in-memory), local-scored by BM25 and + /// LSM FTS planner (base ∪ SSTables ∪ in-memory), local-scored by BM25 and /// merged by `_score` DESC. Mirrors /// [`crate::dataset::scanner::Scanner::full_text_search`]: the searched /// column(s) come from the query (set via `FullTextSearchQuery::with_column`); @@ -778,7 +814,7 @@ impl LsmScanner { } /// Test which `pks` have been (re)written in the WAL fresh tier — the active - /// and frozen memtables and flushed generations this scanner spans — i.e. + /// and frozen memtables and SSTables this scanner spans — i.e. /// are shadowed above the base table. `pks` is a batch whose columns include /// the primary-key columns; the returned `Vec` is aligned with its /// rows. Hashing matches the scanner's internal dedup, so the caller never @@ -802,7 +838,8 @@ impl LsmScanner { let memberships = super::block_list::fresh_tier_block_list( &sources, self.session.as_ref(), - self.flushed_cache.as_ref(), + self.store_params.as_ref(), + self.sstable_cache.as_ref(), watermarks, ) .await?; @@ -892,6 +929,12 @@ impl std::fmt::Debug for LsmScanner { #[cfg(test)] mod tests { use super::*; + use arrow_array::{Int32Array, ListArray, StringArray, StructArray, UInt32Array}; + use arrow_buffer::{OffsetBuffer, ScalarBuffer}; + use arrow_schema::{Field, Fields}; + use lance_index::scalar::inverted::{DOC_INDEX_COL, DocumentGranularity, InvertedIndexParams}; + + use crate::dataset::mem_wal::write::{BatchStore, IndexStore}; #[test] fn test_lsm_scanner_builder() { @@ -907,6 +950,22 @@ mod tests { assert!(shard_snapshots.is_empty()); } + #[test] + fn point_lookup_extraction_requires_normalizable_literals() { + use datafusion::prelude::{col, lit}; + + let filters = [ + col("id").in_list(vec![lit(1i32), lit("not an integer")], false), + col("id").in_list(vec![lit(1i32), col("other")], false), + ]; + for filter in filters { + assert!( + extract_pk_point_keys(&filter, "id", &DataType::Int32).is_none(), + "unsupported point-lookup filter must use the scan path: {filter}" + ); + } + } + #[test] fn test_shard_snapshot_construction() { use super::super::data_source::ShardSnapshot; @@ -915,13 +974,13 @@ mod tests { let snapshot = ShardSnapshot::new(shard_id) .with_spec_id(1) .with_current_generation(5) - .with_flushed_generation(1, "path/gen_1".to_string()) - .with_flushed_generation(2, "path/gen_2".to_string()); + .with_sstable(1, "path/gen_1".to_string()) + .with_sstable(2, "path/gen_2".to_string()); assert_eq!(snapshot.shard_id, shard_id); assert_eq!(snapshot.spec_id, 1); assert_eq!(snapshot.current_generation, 5); - assert_eq!(snapshot.flushed_generations.len(), 2); + assert_eq!(snapshot.sstables.len(), 2); } #[test] @@ -973,44 +1032,69 @@ mod tests { } #[tokio::test] - async fn invalid_overfetch_factor_is_rejected() { - let shard = Uuid::new_v4(); - let scanner = LsmScanner::without_base_table( + async fn overfetch_factor_only_applies_to_searches() { + // Plain scans refill exactly via LocalLimitExec, so overfetch_factor is + // search-only and must not affect scan planning. + let scan = LsmScanner::without_base_table( pk_schema(), - "memory://t", + "memory://scan", vec![], vec!["id".to_string()], ) - .with_in_memory_memtables( - shard, - InMemoryMemTables { - active: mk_pk_memtable(&[1, 2], 2), - frozen: vec![], - }, + .with_overfetch_factor(0.5) + .try_into_batch() + .await + .unwrap(); + assert_eq!(scan.num_rows(), 0); + + let vector_schema = pk_schema_with(arrow_schema::Field::new( + "vector", + DataType::FixedSizeList( + Arc::new(arrow_schema::Field::new("item", DataType::Float32, true)), + 4, + ), + false, + )); + let vector_search = LsmScanner::without_base_table( + vector_schema, + "memory://vector", + vec![], + vec!["id".to_string()], + ) + .nearest( + "vector", + &arrow_array::Float32Array::from(vec![0.0f32, 0.0, 0.0, 0.0]), + 1, ) + .unwrap() .with_overfetch_factor(0.5); - - let Err(err) = scanner.try_into_stream().await else { - panic!("invalid overfetch factor should fail planning"); + let Err(err) = vector_search.try_into_stream().await else { + panic!("invalid overfetch factor should fail vector search planning"); }; assert!( err.to_string().contains("overfetch_factor"), "unexpected error for invalid overfetch factor: {err}" ); - let empty_scanner = LsmScanner::without_base_table( - pk_schema(), - "memory://empty", + let fts_search = LsmScanner::without_base_table( + pk_schema_with(arrow_schema::Field::new("text", DataType::Utf8, true)), + "memory://fts", vec![], vec!["id".to_string()], ) + .full_text_search( + FullTextSearchQuery::new("lance".to_string()) + .with_column("text".to_string()) + .unwrap(), + ) + .unwrap() .with_overfetch_factor(0.5); - let Err(err) = empty_scanner.try_into_stream().await else { - panic!("invalid overfetch factor should fail even when there are no sources"); + let Err(err) = fts_search.try_into_stream().await else { + panic!("invalid overfetch factor should fail full-text search planning"); }; assert!( err.to_string().contains("overfetch_factor"), - "unexpected error for invalid empty-source overfetch factor: {err}" + "unexpected error for invalid overfetch factor: {err}" ); } @@ -1778,6 +1862,103 @@ mod tests { ); } + #[tokio::test] + async fn full_text_search_supports_nested_list_element_path() { + let doc_fields = Fields::from(vec![Field::new("content", DataType::Utf8, true)]); + let doc_values = StructArray::new( + doc_fields.clone(), + vec![Arc::new(StringArray::from(vec!["alpha", "beta", "alpha"]))], + None, + ); + let doc_item = Arc::new(Field::new("item", DataType::Struct(doc_fields), true)); + let docs = ListArray::new( + doc_item.clone(), + OffsetBuffer::new(ScalarBuffer::from(vec![0_i32, 1, 3])), + Arc::new(doc_values), + None, + ); + let group_fields = Fields::from(vec![Field::new("docs", DataType::List(doc_item), true)]); + let group_values = StructArray::new(group_fields.clone(), vec![Arc::new(docs)], None); + let group_item = Arc::new(Field::new("item", DataType::Struct(group_fields), true)); + let groups = ListArray::new( + group_item, + OffsetBuffer::new(ScalarBuffer::from(vec![0_i32, 2])), + Arc::new(group_values), + None, + ); + let schema = pk_schema_with(Field::new("groups", groups.data_type().clone(), true)); + let batch = RecordBatch::try_new( + schema.clone(), + vec![Arc::new(Int32Array::from(vec![7])), Arc::new(groups)], + ) + .unwrap(); + + let batch_store = Arc::new(BatchStore::with_capacity(16)); + let mut indexes = IndexStore::new(); + indexes.enable_pk_index(&[("id".to_string(), 0)]); + indexes + .add_fts_with_params( + "content_list_element_fts".to_string(), + 1, + "groups.docs.content".to_string(), + InvertedIndexParams::default() + .document_granularity(DocumentGranularity::ListElement), + ) + .unwrap(); + let (batch_position, row_offset, _) = batch_store.append(batch.clone()).unwrap(); + indexes + .insert_with_batch_position(&batch, row_offset, Some(batch_position)) + .unwrap(); + + let scanner = LsmScanner::without_base_table( + schema.clone(), + "memory://nested_fts", + vec![], + vec!["id".to_string()], + ) + .with_in_memory_memtables( + Uuid::new_v4(), + InMemoryMemTables { + active: InMemoryMemTableRef { + batch_store, + index_store: Arc::new(indexes), + schema, + generation: 1, + }, + frozen: vec![], + }, + ) + .project(&["id"]) + .unwrap() + .full_text_search( + FullTextSearchQuery::new("alpha".to_string()) + .with_column("groups.docs.content".to_string()) + .unwrap(), + ) + .unwrap(); + + let batch = scanner.try_into_batch().await.unwrap(); + let ids = batch["id"].as_any().downcast_ref::().unwrap(); + let coordinates = batch[DOC_INDEX_COL] + .as_any() + .downcast_ref::() + .unwrap(); + let mut hits = (0..batch.num_rows()) + .map(|row| { + let coordinate = coordinates + .value(row) + .as_any() + .downcast_ref::() + .unwrap() + .values() + .to_vec(); + (ids.value(row), coordinate) + }) + .collect::>(); + hits.sort_unstable(); + assert_eq!(hits, vec![(7, vec![0, 0]), (7, vec![1, 1])]); + } + /// Empty search results must still preserve the execution plan schema. #[tokio::test] async fn try_into_batch_empty_fts_keeps_score_schema() { @@ -1924,6 +2105,48 @@ mod tests { ); assert_eq!(count(plan).await, 2); + // Duplicate literals retain predicate semantics: each matching row is + // emitted once, including when a limit would otherwise hide a distinct + // match behind the duplicate. + let plan = scanner() + .filter_expr(col("id").in_list(vec![lit(1i32), lit(1i32), lit(2i32)], false)) + .limit(Some(2), None) + .unwrap() + .create_plan() + .await + .unwrap(); + assert_eq!(collect_ids(plan).await, vec![1, 2]); + + // Distinct literal types can coerce to the same primary-key value and + // must share one deduplication identity before the limit is applied. + let plan = scanner() + .filter_expr(col("id").in_list(vec![lit(1i32), lit(1i64), lit(2i32)], false)) + .limit(Some(2), None) + .unwrap() + .create_plan() + .await + .unwrap(); + assert_eq!(collect_ids(plan).await, vec![1, 2]); + + // Coercible literals use the per-key fallback and must have the same + // set semantics as exact-type keys. + let plan = scanner() + .filter_expr(col("id").in_list(vec![lit(1i64), lit(1i64), lit(3i64)], false)) + .create_plan() + .await + .unwrap(); + assert_eq!(collect_ids(plan).await, vec![1, 3]); + + // NULL literals cannot match, and duplicate NULLs must not affect the + // non-NULL matches. + let null = lit(ScalarValue::Int32(None)); + let plan = scanner() + .filter_expr(col("id").in_list(vec![null.clone(), lit(2i32), null], false)) + .create_plan() + .await + .unwrap(); + assert_eq!(collect_ids(plan).await, vec![2]); + // A range filter is NOT a point lookup → falls through to the scan path. let plan = scanner() .filter_expr(col("id").gt(lit(2i32))) diff --git a/rust/lance/src/dataset/mem_wal/scanner/collector.rs b/rust/lance/src/dataset/mem_wal/scanner/collector.rs index 6645f159b12..b8df6edcb2e 100644 --- a/rust/lance/src/dataset/mem_wal/scanner/collector.rs +++ b/rust/lance/src/dataset/mem_wal/scanner/collector.rs @@ -49,18 +49,18 @@ pub struct InMemoryMemTables { /// /// This collector gathers all data sources that need to be scanned /// for a query, including: -/// - The base table (merged data) — optional; omit for fresh-tier-only scans -/// - Flushed MemTables from each shard +/// - The base table (compacted data) — optional; omit for fresh-tier-only scans +/// - SSTables from each shard /// - In-memory memtables per shard (active + frozen-awaiting-flush) /// /// When the base table is omitted (see [`Self::without_base_table`]), `collect` -/// returns only flushed-generation and active-memtable sources. This is used +/// returns only SSTable and active-memtable sources. This is used /// by callers that own the base read path elsewhere and only need the WAL's -/// fresh tier (active memtable ∪ L0 flushed generations). +/// fresh tier (active memtable ∪ L0 SSTables). pub struct LsmDataSourceCollector { /// Base Lance table (None when scanning only the fresh tier). base_table: Option>, - /// Base path for resolving relative flushed-generation paths. + /// Base path for resolving relative SSTable paths. base_path: String, /// Shard snapshots from MemWAL index. shard_snapshots: Vec, @@ -73,7 +73,7 @@ impl LsmDataSourceCollector { /// /// # Arguments /// - /// * `base_table` - The base Lance table (merged data) + /// * `base_table` - The base Lance table (compacted data) /// * `shard_snapshots` - Snapshots of shard states from MemWAL index pub fn new(base_table: Arc, shard_snapshots: Vec) -> Self { // Use the dataset's URI as base path for resolving relative paths. @@ -89,8 +89,8 @@ impl LsmDataSourceCollector { /// Create a collector without a base table (fresh-tier scan only). /// - /// The collector emits only flushed-generation and active-memtable sources. - /// `base_path` is the table-root URI used to resolve relative flushed paths + /// The collector emits only SSTable and active-memtable sources. + /// `base_path` is the table-root URI used to resolve relative SSTable paths /// (typically the same URI that would have been the base dataset's URI). pub fn without_base_table( base_path: impl Into, @@ -147,16 +147,12 @@ impl LsmDataSourceCollector { &self.in_memory_memtables } - /// Whether the collector has any on-disk source (base table or a flushed - /// generation). The point-lookup fast path uses this to decide, after + /// Whether the collector has any on-disk source (base table or an + /// SSTable). The point-lookup fast path uses this to decide, after /// missing every in-memory memtable, between "definitely absent" (`false`) /// and "must consult disk via the plan path" (`true`). Cheap: no allocation. pub fn has_on_disk_sources(&self) -> bool { - self.base_table.is_some() - || self - .shard_snapshots - .iter() - .any(|s| !s.flushed_generations.is_empty()) + self.base_table.is_some() || self.shard_snapshots.iter().any(|s| !s.sstables.is_empty()) } /// The in-memory memtables (active + frozen across all shards) as @@ -233,10 +229,10 @@ impl LsmDataSourceCollector { /// frozen memtable. During the post-flush grace window a generation is both /// committed to the manifest (a flushed source) and held in memory (an /// in-memory source); it must be served only from memory — which preserves - /// the per-batch boundaries the flushed dataset has lost, so as-of reads + /// the per-batch boundaries the SSTable dataset has lost, so as-of reads /// stay snapshot-bounded — and its on-disk copy skipped to avoid scanning /// the generation twice. See `ShardWriterConfig::frozen_memtable_grace`. - fn flushed_gen_pinned_in_memory(&self, shard_id: &Uuid, generation: u64) -> bool { + fn sstable_pinned_in_memory(&self, shard_id: &Uuid, generation: u64) -> bool { self.in_memory_memtables .get(shard_id) .is_some_and(|mems| mems.frozen.iter().any(|f| f.generation == generation)) @@ -246,7 +242,7 @@ impl LsmDataSourceCollector { /// /// Returns sources in a consistent order: /// 1. Base table (gen=0), if configured - /// 2. Flushed MemTables per shard, ordered by generation + /// 2. SSTables per shard, ordered by generation /// 3. In-memory memtables per shard (active + frozen-awaiting-flush) pub fn collect(&self) -> Result> { let mut sources = Vec::new(); @@ -258,15 +254,15 @@ impl LsmDataSourceCollector { } for snapshot in &self.shard_snapshots { - for flushed in &snapshot.flushed_generations { - if self.flushed_gen_pinned_in_memory(&snapshot.shard_id, flushed.generation) { + for sstable in &snapshot.sstables { + if self.sstable_pinned_in_memory(&snapshot.shard_id, sstable.generation) { continue; } - let path = self.resolve_flushed_path(&snapshot.shard_id, &flushed.path); - sources.push(LsmDataSource::FlushedMemTable { + let path = self.resolve_sstable_path(&snapshot.shard_id, &sstable.path); + sources.push(LsmDataSource::SsTable { path, shard_id: snapshot.shard_id, - generation: LsmGeneration::memtable(flushed.generation), + generation: LsmGeneration::memtable(sstable.generation), }); } } @@ -299,15 +295,15 @@ impl LsmDataSourceCollector { continue; } - for flushed in &snapshot.flushed_generations { - if self.flushed_gen_pinned_in_memory(&snapshot.shard_id, flushed.generation) { + for sstable in &snapshot.sstables { + if self.sstable_pinned_in_memory(&snapshot.shard_id, sstable.generation) { continue; } - let path = self.resolve_flushed_path(&snapshot.shard_id, &flushed.path); - sources.push(LsmDataSource::FlushedMemTable { + let path = self.resolve_sstable_path(&snapshot.shard_id, &sstable.path); + sources.push(LsmDataSource::SsTable { path, shard_id: snapshot.shard_id, - generation: LsmGeneration::memtable(flushed.generation), + generation: LsmGeneration::memtable(sstable.generation), }); } } @@ -325,25 +321,21 @@ impl LsmDataSourceCollector { /// Get the total number of data sources. pub fn num_sources(&self) -> usize { - let flushed_count: usize = self - .shard_snapshots - .iter() - .map(|s| s.flushed_generations.len()) - .sum(); + let sstable_count: usize = self.shard_snapshots.iter().map(|s| s.sstables.len()).sum(); let base_count = if self.base_table.is_some() { 1 } else { 0 }; let in_memory_count: usize = self .in_memory_memtables .values() .map(|m| 1 + m.frozen.len()) .sum(); - base_count + flushed_count + in_memory_count + base_count + sstable_count + in_memory_count } - /// Resolve a flushed MemTable path to an absolute path. + /// Resolve an SSTable path to an absolute path. /// - /// Flushed MemTables are stored at: `{base_path}/_mem_wal/{shard_id}/{folder_name}` - /// The `folder_name` is what's stored in `FlushedGeneration.path`. - fn resolve_flushed_path(&self, shard_id: &Uuid, folder_name: &str) -> String { + /// SSTables are stored at: `{base_path}/_mem_wal/{shard_id}/{folder_name}` + /// The `folder_name` is what's stored in `SsTable.path`. + fn resolve_sstable_path(&self, shard_id: &Uuid, folder_name: &str) -> String { format!("{}/_mem_wal/{}/{}", self.base_path, shard_id, folder_name) } } @@ -351,7 +343,7 @@ impl LsmDataSourceCollector { #[cfg(test)] mod tests { use super::*; - use crate::dataset::mem_wal::scanner::data_source::FlushedGeneration; + use crate::dataset::mem_wal::scanner::data_source::SsTable; fn create_test_snapshots() -> Vec { let shard_a = Uuid::new_v4(); @@ -362,12 +354,12 @@ mod tests { shard_id: shard_a, spec_id: 1, current_generation: 3, - flushed_generations: vec![ - FlushedGeneration { + sstables: vec![ + SsTable { generation: 1, path: "abc_gen_1".to_string(), }, - FlushedGeneration { + SsTable { generation: 2, path: "def_gen_2".to_string(), }, @@ -377,7 +369,7 @@ mod tests { shard_id: shard_b, spec_id: 1, current_generation: 2, - flushed_generations: vec![FlushedGeneration { + sstables: vec![SsTable { generation: 1, path: "xyz_gen_1".to_string(), }], @@ -390,8 +382,8 @@ mod tests { let snapshots = create_test_snapshots(); // 1 base table + 2 flushed from shard_a + 1 flushed from shard_b = 4 // Using a mock dataset is complex, so we just test the counting logic - assert_eq!(snapshots[0].flushed_generations.len(), 2); - assert_eq!(snapshots[1].flushed_generations.len(), 1); + assert_eq!(snapshots[0].sstables.len(), 2); + assert_eq!(snapshots[1].sstables.len(), 1); } #[test] @@ -466,10 +458,10 @@ mod tests { /// During the post-flush grace window a generation is both committed to the /// manifest (a flushed source) and still pinned in memory (a frozen /// source). The collector must emit it once, from memory — so as-of reads - /// keep batch-resolved membership — and skip the on-disk copy. Flushed - /// generations NOT pinned in memory are still emitted from disk. + /// keep batch-resolved membership — and skip the on-disk copy. SSTables + /// NOT pinned in memory are still emitted from disk. #[test] - fn test_collect_suppresses_flushed_gen_pinned_in_memory() { + fn test_collect_suppresses_sstable_pinned_in_memory() { let shard = Uuid::new_v4(); // Manifest lists gens 1 and 2 as flushed; gen 2 is still pinned in // memory (just flushed, within grace), gen 1 has been swept. @@ -477,12 +469,12 @@ mod tests { shard_id: shard, spec_id: 0, current_generation: 3, - flushed_generations: vec![ - FlushedGeneration { + sstables: vec![ + SsTable { generation: 1, path: "gen_1".to_string(), }, - FlushedGeneration { + SsTable { generation: 2, path: "gen_2".to_string(), }, @@ -498,7 +490,7 @@ mod tests { let sources = collector.collect().unwrap(); // gen 1: on-disk (not pinned). gen 2: in-memory only (pinned, disk // copy suppressed). gen 3: active. No duplicate gen 2. - let flushed: Vec = sources + let sstable_gens: Vec = sources .iter() .filter(|s| !s.is_active_memtable()) .map(|s| s.generation().as_u64()) @@ -508,7 +500,11 @@ mod tests { .filter(|s| s.is_active_memtable()) .map(|s| s.generation().as_u64()) .collect(); - assert_eq!(flushed, vec![1], "only the unpinned flushed gen from disk"); + assert_eq!( + sstable_gens, + vec![1], + "only the unpinned SSTable gen from disk" + ); assert_eq!(in_memory, vec![2, 3], "pinned gen 2 served from memory"); } } diff --git a/rust/lance/src/dataset/mem_wal/scanner/data_source.rs b/rust/lance/src/dataset/mem_wal/scanner/data_source.rs index 0d5f3fdc925..86e6041142b 100644 --- a/rust/lance/src/dataset/mem_wal/scanner/data_source.rs +++ b/rust/lance/src/dataset/mem_wal/scanner/data_source.rs @@ -44,7 +44,7 @@ pub struct FreshTierWatermark { pub struct LsmGeneration(u64); impl LsmGeneration { - /// Generation for the base table (merged data). + /// Generation for the base table (compacted data). pub const BASE_TABLE: Self = Self(0); /// Create a generation for a MemTable. @@ -93,12 +93,12 @@ impl Default for LsmGeneration { } } -/// A flushed generation with its storage path. +/// An SSTable with its storage path. #[derive(Debug, Clone)] -pub struct FlushedGeneration { +pub struct SsTable { /// Generation number. pub generation: u64, - /// Path to the flushed MemTable directory (relative to table root). + /// Path to the SSTable directory (relative to table root). pub path: String, } @@ -114,8 +114,8 @@ pub struct ShardSnapshot { pub spec_id: u32, /// Current generation being written (next flush will be this generation). pub current_generation: u64, - /// List of flushed generations and their paths. - pub flushed_generations: Vec, + /// List of SSTables and their paths. + pub sstables: Vec, } impl ShardSnapshot { @@ -125,7 +125,7 @@ impl ShardSnapshot { shard_id, spec_id: 0, current_generation: 1, - flushed_generations: Vec::new(), + sstables: Vec::new(), } } @@ -141,10 +141,9 @@ impl ShardSnapshot { self } - /// Add a flushed generation. - pub fn with_flushed_generation(mut self, generation: u64, path: String) -> Self { - self.flushed_generations - .push(FlushedGeneration { generation, path }); + /// Add an SSTable. + pub fn with_sstable(mut self, generation: u64, path: String) -> Self { + self.sstables.push(SsTable { generation, path }); self } } @@ -156,9 +155,9 @@ pub enum LsmDataSource { /// The base dataset. dataset: Arc, }, - /// Flushed MemTable stored as Lance table on disk. - FlushedMemTable { - /// Absolute path to the flushed MemTable directory. + /// SSTable stored as Lance table on disk. + SsTable { + /// Absolute path to the SSTable directory. path: String, /// Shard this MemTable belongs to. shard_id: Uuid, @@ -185,7 +184,7 @@ impl LsmDataSource { pub fn generation(&self) -> LsmGeneration { match self { Self::BaseTable { .. } => LsmGeneration::BASE_TABLE, - Self::FlushedMemTable { generation, .. } => *generation, + Self::SsTable { generation, .. } => *generation, Self::ActiveMemTable { generation, .. } => *generation, } } @@ -194,7 +193,7 @@ impl LsmDataSource { pub fn shard_id(&self) -> Option { match self { Self::BaseTable { .. } => None, - Self::FlushedMemTable { shard_id, .. } => Some(*shard_id), + Self::SsTable { shard_id, .. } => Some(*shard_id), Self::ActiveMemTable { shard_id, .. } => Some(*shard_id), } } @@ -213,7 +212,7 @@ impl LsmDataSource { pub fn display_name(&self) -> String { match self { Self::BaseTable { .. } => "base_table".to_string(), - Self::FlushedMemTable { + Self::SsTable { shard_id, generation, .. @@ -279,14 +278,14 @@ mod tests { let snapshot = ShardSnapshot::new(shard_id) .with_spec_id(1) .with_current_generation(5) - .with_flushed_generation(1, "abc123_gen_1".to_string()) - .with_flushed_generation(2, "def456_gen_2".to_string()); + .with_sstable(1, "abc123_gen_1".to_string()) + .with_sstable(2, "def456_gen_2".to_string()); assert_eq!(snapshot.shard_id, shard_id); assert_eq!(snapshot.spec_id, 1); assert_eq!(snapshot.current_generation, 5); - assert_eq!(snapshot.flushed_generations.len(), 2); - assert_eq!(snapshot.flushed_generations[0].generation, 1); - assert_eq!(snapshot.flushed_generations[1].generation, 2); + assert_eq!(snapshot.sstables.len(), 2); + assert_eq!(snapshot.sstables[0].generation, 1); + assert_eq!(snapshot.sstables[1].generation, 2); } } diff --git a/rust/lance/src/dataset/mem_wal/scanner/exec.rs b/rust/lance/src/dataset/mem_wal/scanner/exec.rs index 1498c5f60ea..9c47c893d8d 100644 --- a/rust/lance/src/dataset/mem_wal/scanner/exec.rs +++ b/rust/lance/src/dataset/mem_wal/scanner/exec.rs @@ -10,12 +10,14 @@ //! - [`BloomFilterGuardExec`]: Guards child execution with bloom filter check //! - [`CoalesceFirstExec`]: Returns first non-empty result with short-circuit //! - [`PkBlockFilterExec`]: Drops rows whose PK was superseded by a newer generation (the cross-generation block-list) +//! - [`SchemaRelabelExec`]: Re-labels batches to an exact schema (the logical/storage nullability boundary) mod bloom_guard; mod coalesce_first; mod generation_tag; mod pk; mod pk_block_filter; +mod schema_relabel; pub use bloom_guard::{BloomFilterGuardExec, compute_pk_hash_from_scalars}; pub use coalesce_first::CoalesceFirstExec; @@ -25,3 +27,4 @@ pub use pk::{ validate_pk_types, }; pub use pk_block_filter::PkBlockFilterExec; +pub use schema_relabel::SchemaRelabelExec; diff --git a/rust/lance/src/dataset/mem_wal/scanner/exec/bloom_guard.rs b/rust/lance/src/dataset/mem_wal/scanner/exec/bloom_guard.rs index 632b08a753f..e3a710bd367 100644 --- a/rust/lance/src/dataset/mem_wal/scanner/exec/bloom_guard.rs +++ b/rust/lance/src/dataset/mem_wal/scanner/exec/bloom_guard.rs @@ -5,7 +5,6 @@ //! //! Used in point lookup queries to skip generations that definitely don't contain the key. -use std::any::Any; use std::fmt; use std::pin::Pin; use std::sync::Arc; @@ -134,10 +133,6 @@ impl ExecutionPlan for BloomFilterGuardExec { "BloomFilterGuardExec" } - fn as_any(&self) -> &dyn Any { - self - } - fn schema(&self) -> SchemaRef { self.schema.clone() } diff --git a/rust/lance/src/dataset/mem_wal/scanner/exec/coalesce_first.rs b/rust/lance/src/dataset/mem_wal/scanner/exec/coalesce_first.rs index 9e158c86b4a..c212b47c013 100644 --- a/rust/lance/src/dataset/mem_wal/scanner/exec/coalesce_first.rs +++ b/rust/lance/src/dataset/mem_wal/scanner/exec/coalesce_first.rs @@ -5,7 +5,6 @@ //! //! Used in point lookup queries to stop searching after finding the first match. -use std::any::Any; use std::fmt; use std::pin::Pin; use std::sync::Arc; @@ -111,10 +110,6 @@ impl ExecutionPlan for CoalesceFirstExec { "CoalesceFirstExec" } - fn as_any(&self) -> &dyn Any { - self - } - fn schema(&self) -> SchemaRef { self.schema.clone() } diff --git a/rust/lance/src/dataset/mem_wal/scanner/exec/generation_tag.rs b/rust/lance/src/dataset/mem_wal/scanner/exec/generation_tag.rs index ba9d565316f..9c1d0060a78 100644 --- a/rust/lance/src/dataset/mem_wal/scanner/exec/generation_tag.rs +++ b/rust/lance/src/dataset/mem_wal/scanner/exec/generation_tag.rs @@ -3,7 +3,6 @@ //! MemTable generation tagging execution node. -use std::any::Any; use std::fmt; use std::pin::Pin; use std::sync::Arc; @@ -100,10 +99,6 @@ impl ExecutionPlan for MemtableGenTagExec { "MemtableGenTagExec" } - fn as_any(&self) -> &dyn Any { - self - } - fn schema(&self) -> SchemaRef { self.schema.clone() } diff --git a/rust/lance/src/dataset/mem_wal/scanner/exec/pk_block_filter.rs b/rust/lance/src/dataset/mem_wal/scanner/exec/pk_block_filter.rs index 89dbd7adc61..ef2dc695dfa 100644 --- a/rust/lance/src/dataset/mem_wal/scanner/exec/pk_block_filter.rs +++ b/rust/lance/src/dataset/mem_wal/scanner/exec/pk_block_filter.rs @@ -5,7 +5,7 @@ //! //! Drops a row when any newer generation's membership ([`GenMembership`]) //! contains its primary key — in-memory generations probe their PK index by -//! value, flushed generations probe their on-disk PK BTree. Each generation is +//! value, SSTables probe their on-disk PK BTree. Each generation is //! probed once per batch (see the perf note below). Used both as the KNN //! post-filter (vector search, with over-fetch) and the cross-generation scan //! filter (`k = 0`). @@ -22,12 +22,11 @@ //! `BTreeIndex::contains_keys` (one page pass, no per-key `SearchResult` //! allocation); the in-memory arm maps a sync PK lookup over the keys. Probes //! are not disk-bound in steady state: the opened index and its (small, -//! memtable-sized) pages are held by the injected `FlushedMemTableCache` / +//! memtable-sized) pages are held by the injected `SsTableCache` / //! `LanceCache`, so after the first touch every probe is memory-resident. //! Already-blocked rows are dropped from the key set before probing older //! generations, preserving the per-row short-circuit. -use std::any::Any; use std::fmt; use std::pin::Pin; use std::sync::Arc; @@ -109,10 +108,6 @@ impl ExecutionPlan for PkBlockFilterExec { "PkBlockFilterExec" } - fn as_any(&self) -> &dyn Any { - self - } - fn schema(&self) -> SchemaRef { self.input.schema() } @@ -185,8 +180,8 @@ struct PkBlockFilterStream { warned: bool, } -/// Keep only the rows no newer-gen membership contains. Async because flushed -/// generations are probed against their on-disk PK BTree. +/// Keep only the rows no newer-gen membership contains. Async because SSTables +/// are probed against their on-disk PK BTree. async fn filter_batch(batch: RecordBatch, config: Arc) -> DFResult { let FilterConfig { pk_columns, @@ -322,7 +317,7 @@ mod tests { let (bp, off, _) = store.append(b.clone()).unwrap(); index.insert_with_batch_position(&b, off, Some(bp)).unwrap(); } - let max_visible_row = store.max_visible_row(index.max_visible_batch_position()); + let max_visible_row = store.max_visible_row(index.visible_count()); GenMembership::InMemory { index_store: Arc::new(index), max_visible_row, diff --git a/rust/lance/src/dataset/mem_wal/scanner/exec/schema_relabel.rs b/rust/lance/src/dataset/mem_wal/scanner/exec/schema_relabel.rs new file mode 100644 index 00000000000..88691f99d63 --- /dev/null +++ b/rust/lance/src/dataset/mem_wal/scanner/exec/schema_relabel.rs @@ -0,0 +1,245 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright The Lance Authors + +//! Schema re-labeling execution node. + +use std::fmt; +use std::pin::Pin; +use std::sync::Arc; +use std::task::{Context, Poll}; + +use arrow_array::{RecordBatch, RecordBatchOptions}; +use arrow_schema::SchemaRef; +use datafusion::error::{DataFusionError, Result as DFResult}; +use datafusion::execution::TaskContext; +use datafusion::physical_expr::EquivalenceProperties; +use datafusion::physical_plan::{ + DisplayAs, DisplayFormatType, ExecutionPlan, ExecutionPlanProperties, PlanProperties, + SendableRecordBatchStream, +}; +use futures::{Stream, StreamExt}; + +/// Re-labels every batch to an exact target schema, leaving the arrays +/// untouched. `ProjectionExec` cannot: DataFusion derives output nullability +/// from the expressions, not from the schema the planner intended. +/// +/// **Widening** makes a shard's storage schema (see `relax_non_pk_nullability`) +/// agree with the base-table arm before `UnionExec` / `CoalesceFirstExec`. +/// **Narrowing** restores the logical schema at the scan's output boundary and +/// doubles as the tombstone-leak check, since `RecordBatch` validation rejects +/// a null in a non-nullable column. +#[derive(Debug)] +pub struct SchemaRelabelExec { + input: Arc, + schema: SchemaRef, + properties: Arc, +} + +impl SchemaRelabelExec { + /// Wrap `input` so its batches are re-labeled to `schema`: same column + /// count, order, and data types; only names, nullability, and metadata may + /// differ. A mismatch surfaces per batch at execution time, not plan time. + pub fn new(input: Arc, schema: SchemaRef) -> Self { + let properties = Arc::new(PlanProperties::new( + EquivalenceProperties::new(schema.clone()), + input.output_partitioning().clone(), + input.pipeline_behavior(), + input.boundedness(), + )); + Self { + input, + schema, + properties, + } + } +} + +impl DisplayAs for SchemaRelabelExec { + fn fmt_as(&self, t: DisplayFormatType, f: &mut fmt::Formatter) -> fmt::Result { + match t { + DisplayFormatType::Default + | DisplayFormatType::Verbose + | DisplayFormatType::TreeRender => { + write!(f, "SchemaRelabelExec") + } + } + } +} + +impl ExecutionPlan for SchemaRelabelExec { + fn name(&self) -> &str { + "SchemaRelabelExec" + } + + fn schema(&self) -> SchemaRef { + self.schema.clone() + } + + fn properties(&self) -> &Arc { + &self.properties + } + + fn children(&self) -> Vec<&Arc> { + vec![&self.input] + } + + fn with_new_children( + self: Arc, + children: Vec>, + ) -> DFResult> { + if children.len() != 1 { + return Err(DataFusionError::Internal( + "SchemaRelabelExec requires exactly one child".to_string(), + )); + } + Ok(Arc::new(Self::new( + children[0].clone(), + self.schema.clone(), + ))) + } + + fn execute( + &self, + partition: usize, + context: Arc, + ) -> DFResult { + Ok(Box::pin(SchemaRelabelStream { + input: self.input.execute(partition, context)?, + schema: self.schema.clone(), + })) + } +} + +struct SchemaRelabelStream { + input: SendableRecordBatchStream, + schema: SchemaRef, +} + +impl Stream for SchemaRelabelStream { + type Item = DFResult; + + fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { + match self.input.poll_next_unpin(cx) { + Poll::Ready(Some(Ok(batch))) => { + // Carry the row count explicitly: `try_new` infers it from the + // first column, which a column-less batch does not have. + let relabeled = RecordBatch::try_new_with_options( + self.schema.clone(), + batch.columns().to_vec(), + &RecordBatchOptions::new().with_row_count(Some(batch.num_rows())), + ) + .map_err(|e| DataFusionError::ArrowError(Box::new(e), None)); + Poll::Ready(Some(relabeled)) + } + other => other, + } + } +} + +impl datafusion::physical_plan::RecordBatchStream for SchemaRelabelStream { + fn schema(&self) -> SchemaRef { + self.schema.clone() + } +} + +#[cfg(test)] +mod tests { + use super::*; + use arrow_array::{Int32Array, StringArray}; + use arrow_schema::{DataType, Field, Schema}; + use datafusion::prelude::SessionContext; + use datafusion_physical_plan::test::TestMemoryExec; + use futures::TryStreamExt; + + fn schema_with(nullable: bool) -> SchemaRef { + Arc::new(Schema::new(vec![ + Field::new("id", DataType::Int32, false), + Field::new("name", DataType::Utf8, nullable), + ])) + } + + fn source(batch: RecordBatch) -> Arc { + TestMemoryExec::try_new_exec(&[vec![batch.clone()]], batch.schema(), None).unwrap() + } + + fn batch(schema: SchemaRef, names: Vec>) -> RecordBatch { + let ids: Vec = (0..names.len() as i32).collect(); + RecordBatch::try_new( + schema, + vec![ + Arc::new(Int32Array::from(ids)), + Arc::new(StringArray::from(names)), + ], + ) + .unwrap() + } + + async fn run(plan: Arc) -> DFResult> { + let ctx = SessionContext::new(); + plan.execute(0, ctx.task_ctx())?.try_collect().await + } + + #[tokio::test] + async fn widening_preserves_rows_and_reports_target_schema() { + let input = source(batch(schema_with(false), vec![Some("a"), Some("b")])); + let relabeled = Arc::new(SchemaRelabelExec::new(input, schema_with(true))); + + assert_eq!(relabeled.schema(), schema_with(true)); + let out = run(relabeled).await.unwrap(); + assert_eq!(out.len(), 1); + assert_eq!(out[0].schema(), schema_with(true)); + assert_eq!(out[0].num_rows(), 2); + } + + #[tokio::test] + async fn narrowing_succeeds_when_no_nulls_remain() { + // Post-tombstone-filter: `name` is nullable in storage, but every + // surviving row has a value. + let input = source(batch(schema_with(true), vec![Some("a"), Some("b")])); + let relabeled = Arc::new(SchemaRelabelExec::new(input, schema_with(false))); + + let out = run(relabeled).await.unwrap(); + assert_eq!(out[0].schema(), schema_with(false)); + assert_eq!(out[0].num_rows(), 2); + } + + #[tokio::test] + async fn narrowing_rejects_a_surviving_null() { + // A tombstone that escaped its filter must error here, not reach the + // caller as a row of nulls. + let input = source(batch(schema_with(true), vec![Some("a"), None])); + let relabeled = Arc::new(SchemaRelabelExec::new(input, schema_with(false))); + + let error = run(relabeled).await.unwrap_err().to_string(); + assert!( + error.contains("non-nullable") && error.contains("name"), + "expected a nullability error naming the column, got: {error}" + ); + } + + #[tokio::test] + async fn empty_batch_is_relabeled() { + let input = source(batch(schema_with(true), vec![])); + let relabeled = Arc::new(SchemaRelabelExec::new(input, schema_with(false))); + + let out = run(relabeled).await.unwrap(); + assert!(out.iter().all(|b| b.num_rows() == 0)); + assert!(out.iter().all(|b| b.schema() == schema_with(false))); + } + + #[tokio::test] + async fn empty_batch_is_still_checked_against_the_target_schema() { + let input = source(batch(schema_with(true), vec![])); + let mistyped = Arc::new(Schema::new(vec![ + Field::new("id", DataType::Int32, false), + Field::new("name", DataType::Int32, false), + ])); + let relabeled = Arc::new(SchemaRelabelExec::new(input, mistyped)); + + let error = run(relabeled).await.unwrap_err().to_string(); + assert!( + error.contains("column types must match"), + "expected a data type error, got: {error}" + ); + } +} diff --git a/rust/lance/src/dataset/mem_wal/scanner/fts_search.rs b/rust/lance/src/dataset/mem_wal/scanner/fts_search.rs index d783540bf6b..12f58db6354 100644 --- a/rust/lance/src/dataset/mem_wal/scanner/fts_search.rs +++ b/rust/lance/src/dataset/mem_wal/scanner/fts_search.rs @@ -4,7 +4,7 @@ //! Full-text search planner for LSM scanner (local scoring). //! //! Builds an execution plan that scores an FTS query across the base -//! table, flushed memtable generations, and the active/frozen-undrained +//! table, SSTable generations, and the active/frozen-undrained //! in-memory memtables, returning rows ordered by BM25 `_score` DESC. //! //! # Scoring @@ -22,17 +22,17 @@ //! benchmark in this PR shows it carries a real latency penalty, so the //! local path lands first and the global option is optimized separately. //! -//! Staleness: within a flushed generation, the deletion vector written +//! Staleness: within an SSTable, the deletion vector written //! at flush time (see #6929) already masks rows superseded by a newer //! generation, so per-source results are clean within each tier. The -//! same primary key can still appear across tiers (active vs flushed) +//! same primary key can still appear across tiers (active vs SSTable) //! when an updated row sits in the active memtable while the older -//! copy lives in a flushed generation; cross-tier deduplication is +//! copy lives in an SSTable; cross-tier deduplication is //! left to the caller in local mode. //! //! Everything here is contained in the `mem_wal` module — it reuses the //! existing per-source FTS read paths (`scanner.full_text_search` for -//! base/flushed Lance datasets, `MemTableScanner` for the active +//! base/SSTable Lance datasets, `MemTableScanner` for the active //! memtable) and requires no changes to `lance-index`. use std::sync::Arc; @@ -48,16 +48,19 @@ use datafusion::prelude::Expr; use lance_core::{Error, Result, is_system_column}; use lance_index::scalar::FullTextSearchQuery; use lance_index::scalar::inverted::query::{FtsQuery as IndexFtsQuery, Operator}; +use lance_index::scalar::inverted::{DOC_INDEX_COL, DOC_INDEX_FIELD, DocumentGranularity}; use tracing::instrument; use super::block_list::compute_source_block_lists; use super::collector::LsmDataSourceCollector; use super::data_source::LsmDataSource; use super::exec::PkBlockFilterExec; -use super::flushed_cache::{DatasetCache, GenerationWarmer, open_flushed_dataset}; use super::projection::{project_to_canonical, validate_projection_names}; +use super::sstable_cache::{DatasetCache, SsTableWarmer, open_sstable}; use crate::dataset::mem_wal::memtable::scanner::MemTableScanner; +use crate::index::scalar::inverted::{indexed_fts_document_granularities, resolve_fts_field}; use crate::session::Session; +use lance_io::object_store::ObjectStoreParams; /// `_score` column name in FTS results — kept aligned with /// `lance_index::scalar::inverted::SCORE_COL` so this module doesn't @@ -69,6 +72,140 @@ pub const SCORE_COLUMN: &str = "_score"; /// so a blocked source still yields `k` live rows after the block-list filter. const DEFAULT_OVERFETCH_FACTOR: f64 = 1.0; +fn requested_query_document_granularity( + query: &IndexFtsQuery, +) -> Result> { + fn merge( + current: &mut Option, + requested: Option, + ) -> Result<()> { + let Some(requested) = requested else { + return Ok(()); + }; + if let Some(current) = current + && *current != requested + { + return Err(Error::invalid_input( + "FTS queries cannot mix Row and ListElement document granularities".to_string(), + )); + } + *current = Some(requested); + Ok(()) + } + + fn visit(query: &IndexFtsQuery, current: &mut Option) -> Result<()> { + match query { + IndexFtsQuery::Match(query) => merge(current, query.document_granularity), + IndexFtsQuery::Phrase(query) => merge(current, query.document_granularity), + IndexFtsQuery::Boost(query) => { + visit(&query.positive, current)?; + visit(&query.negative, current) + } + IndexFtsQuery::Boolean(query) => { + for child in query + .must + .iter() + .chain(&query.should) + .chain(&query.must_not) + { + visit(child, current)?; + } + Ok(()) + } + IndexFtsQuery::MultiMatch(query) => { + for child in &query.match_queries { + merge(current, child.document_granularity)?; + } + Ok(()) + } + } + } + + let mut requested = None; + visit(query, &mut requested)?; + Ok(requested) +} + +fn set_query_document_granularity( + query: &mut IndexFtsQuery, + document_granularity: DocumentGranularity, +) { + match query { + IndexFtsQuery::Match(query) => { + query.document_granularity = Some(document_granularity); + } + IndexFtsQuery::Phrase(query) => { + query.document_granularity = Some(document_granularity); + } + IndexFtsQuery::Boost(query) => { + set_query_document_granularity(&mut query.positive, document_granularity); + set_query_document_granularity(&mut query.negative, document_granularity); + } + IndexFtsQuery::Boolean(query) => { + for child in query + .must + .iter_mut() + .chain(&mut query.should) + .chain(&mut query.must_not) + { + set_query_document_granularity(child, document_granularity); + } + } + IndexFtsQuery::MultiMatch(query) => { + for child in &mut query.match_queries { + child.document_granularity = Some(document_granularity); + } + } + } +} + +fn query_document_granularity(query: &FullTextSearchQuery) -> Result { + requested_query_document_granularity(&query.query)?.ok_or_else(|| { + Error::internal("LSM FTS query document granularity was not resolved".to_string()) + }) +} + +fn resolve_document_granularity_from_candidates( + column: &str, + requested: Option, + mut available: Vec, +) -> Result { + available.sort_by_key(|document_granularity| match document_granularity { + DocumentGranularity::Row => 0, + DocumentGranularity::ListElement => 1, + }); + available.dedup(); + match requested { + Some(requested) if available.is_empty() || available.contains(&requested) => Ok(requested), + Some(requested) => Err(Error::invalid_input(format!( + "FTS query for field '{column}' requested {requested:?} document granularity, but \ + the MemWAL sources use a different indexed granularity: {available:?}" + ))), + None if available.is_empty() => Ok(DocumentGranularity::Row), + None if available.len() == 1 => Ok(available[0]), + None => Err(Error::invalid_input(format!( + "FTS query for field '{column}' is ambiguous because Row and ListElement indexes \ + coexist across MemWAL sources; specify document_granularity" + ))), + } +} + +fn validate_source_document_granularities( + column: &str, + resolved: DocumentGranularity, + source_granularities: &[Vec], +) -> Result<()> { + for granularities in source_granularities { + if !granularities.is_empty() && !granularities.contains(&resolved) { + return Err(Error::invalid_input(format!( + "FTS query for field '{column}' resolved to {resolved:?}, but a MemWAL source has \ + only incompatible indexed granularities: {granularities:?}" + ))); + } + } + Ok(()) +} + fn validate_lsm_fts_query(query: &FullTextSearchQuery) -> Result<()> { match &query.query { IndexFtsQuery::Match(m) => { @@ -86,7 +223,11 @@ fn validate_lsm_fts_query(query: &FullTextSearchQuery) -> Result<()> { } } -fn active_source_can_execute_fts(source: &LsmDataSource, column: &str) -> bool { +fn active_source_can_execute_fts( + source: &LsmDataSource, + column: &str, + document_granularity: DocumentGranularity, +) -> bool { match source { LsmDataSource::ActiveMemTable { batch_store, @@ -94,10 +235,10 @@ fn active_source_can_execute_fts(source: &LsmDataSource, column: &str) -> bool { .. } => { index_store - .get_fts_by_column(column) + .get_fts_by_column_and_granularity(column, document_granularity) .is_some_and(|index| !index.is_empty()) && batch_store - .max_visible_row(index_store.max_visible_batch_position()) + .max_visible_row(index_store.visible_count()) .is_some() } _ => false, @@ -109,16 +250,18 @@ pub struct LsmFtsSearchPlanner { collector: LsmDataSourceCollector, pk_columns: Vec, base_schema: SchemaRef, - /// Session threaded into flushed-generation opens (shared caches). + /// Session threaded into SSTable opens (shared caches). session: Option>, - /// Cache of opened flushed-generation datasets. - flushed_cache: Option>, - /// Optional warmer fired on first open of a flushed generation. - warmer: Option>, + /// Store params for opening SSTables, reusing the base dataset's store. + store_params: Option, + /// Cache of opened SSTable datasets. + sstable_cache: Option>, + /// Optional warmer fired on first open of an SSTable. + warmer: Option>, /// Over-fetch multiple for blocked sources. overfetch_factor: f64, /// Optional prefilter predicate applied to every source arm so FTS hits - /// failing the predicate are dropped. Base/flushed arms use the dataset + /// failing the predicate are dropped. Base/SSTable arms use the dataset /// scanner's native filter; memtable arms filter the materialized hits. filter: Option, } @@ -135,7 +278,8 @@ impl LsmFtsSearchPlanner { pk_columns, base_schema, session: None, - flushed_cache: None, + store_params: None, + sstable_cache: None, warmer: None, overfetch_factor: DEFAULT_OVERFETCH_FACTOR, filter: None, @@ -144,7 +288,7 @@ impl LsmFtsSearchPlanner { /// Attach an optional prefilter predicate. Every source arm restricts its /// FTS hits to rows matching the predicate, matching a normal filtered - /// full-text scan over base ∪ flushed ∪ in-memory data. + /// full-text scan over base ∪ SSTables ∪ in-memory data. pub fn with_filter(mut self, filter: Option) -> Self { self.filter = filter; self @@ -158,22 +302,27 @@ impl LsmFtsSearchPlanner { self } - /// Thread a session into flushed-generation opens so the first open - /// populates the shared index / file-metadata caches. + /// Set the session used to open SSTables. pub fn with_session(mut self, session: Arc) -> Self { self.session = Some(session); self } - /// Inject a cache of opened flushed-generation datasets, making repeated + /// Set the store params used to open SSTables. + pub fn with_store_params(mut self, store_params: ObjectStoreParams) -> Self { + self.store_params = Some(store_params); + self + } + + /// Inject a cache of opened SSTable datasets, making repeated /// searches against the same generation a pure `Arc::clone`. - pub fn with_flushed_cache(mut self, cache: Arc) -> Self { - self.flushed_cache = Some(cache); + pub fn with_sstable_cache(mut self, cache: Arc) -> Self { + self.sstable_cache = Some(cache); self } - /// Inject the warmer fired on first open of a flushed generation. - pub fn with_warmer(mut self, warmer: Arc) -> Self { + /// Inject the warmer fired on first open of an SSTable. + pub fn with_warmer(mut self, warmer: Arc) -> Self { self.warmer = Some(warmer); self } @@ -184,7 +333,7 @@ impl LsmFtsSearchPlanner { /// /// * `column` — text column to search. /// * `query` — the FTS query (match / phrase / boolean / fuzzy for - /// base/flushed Lance sources; the active memtable currently + /// base/SSTable Lance sources; the active memtable currently /// supports `MatchQuery`). /// * `limit` — optional global top-k to return. /// * `projection` — user columns to project. PK columns are @@ -202,19 +351,68 @@ impl LsmFtsSearchPlanner { pub async fn plan_search( &self, column: &str, - query: FullTextSearchQuery, + mut query: FullTextSearchQuery, limit: Option, projection: Option<&[String]>, ) -> Result> { let sources = self.collector.collect()?; + let requested = requested_query_document_granularity(&query.query)?; + let mut available = Vec::new(); + let mut source_granularities = Vec::with_capacity(sources.len()); + for source in &sources { + let granularities = match source { + LsmDataSource::BaseTable { dataset } => { + indexed_fts_document_granularities(dataset, column) + .await? + .into_iter() + .map(|(_, document_granularity)| document_granularity) + .collect::>() + } + LsmDataSource::SsTable { path, .. } => { + let dataset = open_sstable( + path, + self.session.as_ref(), + self.store_params.as_ref(), + self.sstable_cache.as_ref(), + self.warmer.as_ref(), + ) + .await?; + indexed_fts_document_granularities(&dataset, column) + .await? + .into_iter() + .map(|(_, document_granularity)| document_granularity) + .collect::>() + } + LsmDataSource::ActiveMemTable { index_store, .. } => { + index_store.fts_document_granularities_by_column(column) + } + }; + available.extend(granularities.iter().copied()); + source_granularities.push(granularities); + } + let document_granularity = + resolve_document_granularity_from_candidates(column, requested, available)?; + validate_source_document_granularities( + column, + document_granularity, + &source_granularities, + )?; + let schema = lance_core::datatypes::Schema::try_from(self.base_schema.as_ref())?; + resolve_fts_field(&schema, column, document_granularity)?; + set_query_document_granularity(&mut query.query, document_granularity); if sources .iter() - .any(|source| active_source_can_execute_fts(source, column)) + .any(|source| active_source_can_execute_fts(source, column, document_granularity)) { validate_lsm_fts_query(&query)?; } - validate_projection_names(projection, &self.base_schema, &[SCORE_COLUMN])?; - let target_schema = self.canonical_fts_schema(projection); + let allowed_system_columns: &[&str] = if document_granularity.is_list_element() { + &[SCORE_COLUMN, DOC_INDEX_COL] + } else { + &[SCORE_COLUMN] + }; + validate_projection_names(projection, &self.base_schema, allowed_system_columns)?; + let target_schema = self.canonical_fts_schema(projection, document_granularity); let overfetch = super::validate_overfetch_factor(self.overfetch_factor)?; if sources.is_empty() { @@ -228,7 +426,8 @@ impl LsmFtsSearchPlanner { let block_lists = Box::pin(compute_source_block_lists( &sources, self.session.as_ref(), - self.flushed_cache.as_ref(), + self.store_params.as_ref(), + self.sstable_cache.as_ref(), )) .await?; @@ -370,11 +569,12 @@ impl LsmFtsSearchPlanner { scanner.full_text_search(bound_query)?; scanner.create_plan().await } - LsmDataSource::FlushedMemTable { path, .. } => { - let dataset = open_flushed_dataset( + LsmDataSource::SsTable { path, .. } => { + let dataset = open_sstable( path, self.session.as_ref(), - self.flushed_cache.as_ref(), + self.store_params.as_ref(), + self.sstable_cache.as_ref(), self.warmer.as_ref(), ) .await?; @@ -402,8 +602,10 @@ impl LsmFtsSearchPlanner { schema, .. } => { - if !active_source_can_execute_fts(source, column) { - return self.empty_plan(&self.canonical_fts_schema(projection)); + let document_granularity = query_document_granularity(query)?; + if !active_source_can_execute_fts(source, column, document_granularity) { + return self + .empty_plan(&self.canonical_fts_schema(projection, document_granularity)); } validate_lsm_fts_query(query)?; let mut scanner = @@ -460,7 +662,11 @@ impl LsmFtsSearchPlanner { } /// Canonical FTS output: user-projected cols + PK + `_score`. - fn canonical_fts_schema(&self, user_projection: Option<&[String]>) -> SchemaRef { + fn canonical_fts_schema( + &self, + user_projection: Option<&[String]>, + document_granularity: DocumentGranularity, + ) -> SchemaRef { let mut ordered: Vec = if let Some(p) = user_projection { p.to_vec() } else { @@ -475,6 +681,9 @@ impl LsmFtsSearchPlanner { ordered.push(pk.clone()); } } + if document_granularity.is_list_element() && !ordered.iter().any(|c| c == DOC_INDEX_COL) { + ordered.push(DOC_INDEX_COL.to_string()); + } if !ordered.iter().any(|c| c == SCORE_COLUMN) { ordered.push(SCORE_COLUMN.to_string()); } @@ -483,6 +692,8 @@ impl LsmFtsSearchPlanner { .filter_map(|name| { if name == SCORE_COLUMN { Some(Arc::new(Field::new(SCORE_COLUMN, DataType::Float32, true))) + } else if name == DOC_INDEX_COL { + Some(Arc::new(DOC_INDEX_FIELD.clone())) } else if is_system_column(name) { Some(Arc::new(Field::new(name.clone(), DataType::UInt64, true))) } else { @@ -508,9 +719,14 @@ mod tests { use crate::dataset::mem_wal::scanner::collector::{InMemoryMemTableRef, InMemoryMemTables}; use crate::dataset::mem_wal::write::{BatchStore, IndexStore}; use crate::dataset::{Dataset, WriteParams}; - use arrow_array::{BooleanArray, Int32Array, RecordBatch, RecordBatchIterator, StringArray}; + use arrow_array::builder::{ListBuilder, StringBuilder}; + use arrow_array::{ + Array, BooleanArray, Int32Array, ListArray, RecordBatch, RecordBatchIterator, StringArray, + UInt32Array, + }; use arrow_schema::{DataType, Field, Schema as ArrowSchema}; use futures::TryStreamExt; + use lance_index::scalar::inverted::query::MatchQuery; use std::collections::HashMap; fn fts_schema() -> Arc { @@ -608,10 +824,321 @@ mod tests { ); } + #[test] + fn resolves_document_granularity_from_memwal_indexes() { + assert_eq!( + resolve_document_granularity_from_candidates("tags", None, vec![]).unwrap(), + DocumentGranularity::Row + ); + assert_eq!( + resolve_document_granularity_from_candidates( + "tags", + None, + vec![DocumentGranularity::ListElement], + ) + .unwrap(), + DocumentGranularity::ListElement + ); + assert!( + resolve_document_granularity_from_candidates( + "tags", + Some(DocumentGranularity::Row), + vec![DocumentGranularity::ListElement], + ) + .unwrap_err() + .to_string() + .contains("different indexed granularity") + ); + let both = vec![DocumentGranularity::Row, DocumentGranularity::ListElement]; + assert!( + resolve_document_granularity_from_candidates("tags", None, both.clone()) + .unwrap_err() + .to_string() + .contains("ambiguous") + ); + assert_eq!( + resolve_document_granularity_from_candidates( + "tags", + Some(DocumentGranularity::ListElement), + both, + ) + .unwrap(), + DocumentGranularity::ListElement + ); + assert!( + validate_source_document_granularities( + "tags", + DocumentGranularity::ListElement, + &[ + vec![DocumentGranularity::ListElement], + vec![DocumentGranularity::Row], + ], + ) + .unwrap_err() + .to_string() + .contains("incompatible indexed granularities") + ); + } + + #[tokio::test] + async fn memwal_rejects_list_element_without_a_list_path() { + let schema = fts_schema(); + let tmp = tempfile::tempdir().unwrap(); + let base_uri = format!("{}/base", tmp.path().to_str().unwrap()); + let collector = LsmDataSourceCollector::without_base_table(base_uri, vec![]); + let planner = LsmFtsSearchPlanner::new(collector, vec!["id".to_string()], schema); + let query = FullTextSearchQuery::new_query(IndexFtsQuery::Match( + MatchQuery::new("lance".to_string()) + .with_document_granularity(DocumentGranularity::ListElement), + )); + + let error = planner + .plan_search("text", query, Some(1), None) + .await + .unwrap_err(); + assert!(error.to_string().contains("has no List layer"), "{error}"); + } + + #[tokio::test] + async fn active_element_document_search_returns_physical_ordinals() { + use lance_index::scalar::inverted::InvertedIndexParams; + + let mut id_meta = HashMap::new(); + id_meta.insert( + "lance-schema:unenforced-primary-key".to_string(), + "true".to_string(), + ); + let id_field = Field::new("id", DataType::Int32, false).with_metadata(id_meta); + + let mut tags = ListBuilder::new(StringBuilder::new()); + tags.values().append_value("alpha"); + tags.values().append_value("beta gamma"); + tags.values().append_null(); + tags.values().append_value("beta"); + tags.append(true); + tags.append(true); + tags.values().append_value("beta"); + tags.append(true); + let tags = tags.finish(); + let schema = Arc::new(ArrowSchema::new(vec![ + id_field, + Field::new("tags", tags.data_type().clone(), true), + ])); + let active_batch = RecordBatch::try_new( + schema.clone(), + vec![Arc::new(Int32Array::from(vec![1, 2, 3])), Arc::new(tags)], + ) + .unwrap(); + + let batch_store = Arc::new(BatchStore::with_capacity(16)); + let mut indexes = IndexStore::new(); + indexes.enable_pk_index(&[("id".to_string(), 0)]); + indexes + .add_fts_with_params( + "tags_list_element_fts".to_string(), + 1, + "tags".to_string(), + InvertedIndexParams::default() + .document_granularity(DocumentGranularity::ListElement), + ) + .unwrap(); + let (_, row_offset, batch_position) = batch_store.append(active_batch.clone()).unwrap(); + indexes + .insert_with_batch_position(&active_batch, row_offset, Some(batch_position)) + .unwrap(); + + let tmp = tempfile::tempdir().unwrap(); + let base_uri = format!("{}/base", tmp.path().to_str().unwrap()); + let collector = LsmDataSourceCollector::without_base_table(base_uri, vec![]) + .with_in_memory_memtables( + uuid::Uuid::new_v4(), + InMemoryMemTables { + active: InMemoryMemTableRef { + batch_store, + index_store: Arc::new(indexes), + schema: schema.clone(), + generation: 1, + }, + frozen: vec![], + }, + ); + let planner = LsmFtsSearchPlanner::new(collector, vec!["id".to_string()], schema.clone()); + let projection = vec!["id".to_string()]; + let plan = planner + .plan_search( + "tags", + FullTextSearchQuery::new_query(IndexFtsQuery::Match(MatchQuery::new( + "beta".to_string(), + ))), + Some(10), + Some(&projection), + ) + .await + .unwrap(); + + let ctx = datafusion::prelude::SessionContext::new(); + let stream = plan.execute(0, ctx.task_ctx()).unwrap(); + let batches: Vec = stream.try_collect().await.unwrap(); + let mut hits = Vec::new(); + for batch in batches { + let ids = batch["id"].as_any().downcast_ref::().unwrap(); + let doc_indices = batch[DOC_INDEX_COL] + .as_any() + .downcast_ref::() + .unwrap(); + for row in 0..batch.num_rows() { + let coordinate = doc_indices + .value(row) + .as_any() + .downcast_ref::() + .unwrap() + .value(0); + hits.push((ids.value(row), coordinate)); + } + } + hits.sort_unstable(); + assert_eq!(hits, vec![(1, 1), (1, 3), (3, 0)]); + } + + #[tokio::test] + async fn element_document_search_unions_base_and_active_sources() { + use crate::index::DatasetIndexExt; + use lance_index::IndexType; + use lance_index::scalar::inverted::InvertedIndexParams; + + let mut id_meta = HashMap::new(); + id_meta.insert( + "lance-schema:unenforced-primary-key".to_string(), + "true".to_string(), + ); + let id_field = Field::new("id", DataType::Int32, false).with_metadata(id_meta); + let list_type = DataType::List(Arc::new(Field::new("item", DataType::Utf8, true))); + let schema = Arc::new(ArrowSchema::new(vec![ + id_field, + Field::new("tags", list_type, true), + ])); + + let mut base_tags = ListBuilder::new(StringBuilder::new()); + base_tags.values().append_value("beta"); + base_tags.values().append_value("other"); + base_tags.append(true); + let base_batch = RecordBatch::try_new( + schema.clone(), + vec![ + Arc::new(Int32Array::from(vec![1])), + Arc::new(base_tags.finish()), + ], + ) + .unwrap(); + let tmp = tempfile::tempdir().unwrap(); + let base_uri = format!("{}/base", tmp.path().to_str().unwrap()); + let mut base_ds = Dataset::write( + RecordBatchIterator::new(vec![Ok(base_batch)], schema.clone()), + &base_uri, + None, + ) + .await + .unwrap(); + base_ds + .create_index( + &["tags"], + IndexType::Inverted, + Some("tags_list_element_fts".to_string()), + &InvertedIndexParams::default() + .document_granularity(DocumentGranularity::ListElement), + false, + ) + .await + .unwrap(); + let base_ds = Arc::new(Dataset::open(&base_uri).await.unwrap()); + + let mut active_tags = ListBuilder::new(StringBuilder::new()); + active_tags.values().append_value("other"); + active_tags.values().append_value("beta"); + active_tags.append(true); + let active_batch = RecordBatch::try_new( + schema.clone(), + vec![ + Arc::new(Int32Array::from(vec![2])), + Arc::new(active_tags.finish()), + ], + ) + .unwrap(); + let batch_store = Arc::new(BatchStore::with_capacity(16)); + let mut indexes = IndexStore::new(); + indexes.enable_pk_index(&[("id".to_string(), 0)]); + indexes + .add_fts_with_params( + "tags_list_element_fts".to_string(), + 1, + "tags".to_string(), + InvertedIndexParams::default() + .document_granularity(DocumentGranularity::ListElement), + ) + .unwrap(); + let (_, row_offset, batch_position) = batch_store.append(active_batch.clone()).unwrap(); + indexes + .insert_with_batch_position(&active_batch, row_offset, Some(batch_position)) + .unwrap(); + + let collector = LsmDataSourceCollector::new(base_ds, vec![]).with_in_memory_memtables( + uuid::Uuid::new_v4(), + InMemoryMemTables { + active: InMemoryMemTableRef { + batch_store, + index_store: Arc::new(indexes), + schema: schema.clone(), + generation: 1, + }, + frozen: vec![], + }, + ); + let planner = LsmFtsSearchPlanner::new(collector, vec!["id".to_string()], schema); + let projection = vec!["id".to_string()]; + let plan = planner + .plan_search( + "tags", + FullTextSearchQuery::new_query(IndexFtsQuery::Match(MatchQuery::new( + "beta".to_string(), + ))), + Some(10), + Some(&projection), + ) + .await + .unwrap(); + + let ctx = datafusion::prelude::SessionContext::new(); + let batches: Vec = plan + .execute(0, ctx.task_ctx()) + .unwrap() + .try_collect() + .await + .unwrap(); + let mut hits = Vec::new(); + for batch in batches { + let ids = batch["id"].as_any().downcast_ref::().unwrap(); + let coordinates = batch[DOC_INDEX_COL] + .as_any() + .downcast_ref::() + .unwrap(); + for row in 0..batch.num_rows() { + let coordinate = coordinates + .value(row) + .as_any() + .downcast_ref::() + .unwrap() + .value(0); + hits.push((ids.value(row), coordinate)); + } + } + hits.sort_unstable(); + assert_eq!(hits, vec![(1, 0), (2, 1)]); + } + #[tokio::test] async fn local_mode_unions_base_and_active_with_consistent_score_schema() { // Regression for the `_score` nullability mismatch between - // FtsIndexExec (active arm) and FTS_SCHEMA (base/flushed). The + // FtsIndexExec (active arm) and FTS_SCHEMA (base/SSTable). The // active-only test below would not catch this — UnionExec rejects // schema-inequality, so we need at least one base + one active // source to exercise that code path. @@ -818,12 +1345,12 @@ mod tests { ); } - /// The flushed arm must apply the filter as a true FTS prefilter, and that + /// The SSTable arm must apply the filter as a true FTS prefilter, and that /// prefiltered candidate set must compose with cross-generation block-list /// filtering plus over-fetch. Gen 1's best predicate-matching hit (id=3) is /// superseded by gen 2; with over-fetch, gen 1 should still contribute id=4. #[tokio::test] - async fn prefilter_on_flushed_composes_with_block_list() { + async fn prefilter_on_sstable_composes_with_block_list() { use crate::dataset::mem_wal::scanner::data_source::ShardSnapshot; use crate::index::DatasetIndexExt; use datafusion::prelude::{col, lit}; @@ -837,7 +1364,7 @@ mod tests { // Gen 1: id=1 matches strongly but fails the predicate. id=3 matches // strongly but is stale (blocked by gen 2). id=4 is the next live - // predicate match that only survives if the flushed arm prefilters and + // predicate match that only survives if the SSTable arm prefilters and // over-fetches before the block-list drops id=3. let gen1_uri = format!("{}/_mem_wal/{}/gen_1", base_uri, shard_id); let mut gen1 = write_dataset( @@ -875,8 +1402,8 @@ mod tests { let snapshot = ShardSnapshot::new(shard_id) .with_current_generation(3) - .with_flushed_generation(1, "gen_1".to_string()) - .with_flushed_generation(2, "gen_2".to_string()); + .with_sstable(1, "gen_1".to_string()) + .with_sstable(2, "gen_2".to_string()); let collector = LsmDataSourceCollector::without_base_table(base_uri, vec![snapshot]); let planner = LsmFtsSearchPlanner::new(collector, vec!["id".to_string()], schema) @@ -890,7 +1417,7 @@ mod tests { None, ) .await - .expect("planner should produce a filtered flushed plan"); + .expect("planner should produce a filtered SSTable plan"); let ctx = datafusion::prelude::SessionContext::new(); let stream = plan.execute(0, ctx.task_ctx()).unwrap(); @@ -911,7 +1438,7 @@ mod tests { assert_eq!( ids, vec![4], - "flushed FTS prefilter should return live id=4 after stale id=3 is blocked; got {ids:?}" + "SSTable FTS prefilter should return live id=4 after stale id=3 is blocked; got {ids:?}" ); } diff --git a/rust/lance/src/dataset/mem_wal/scanner/planner.rs b/rust/lance/src/dataset/mem_wal/scanner/planner.rs index ec13da1df66..2a83447fb17 100644 --- a/rust/lance/src/dataset/mem_wal/scanner/planner.rs +++ b/rust/lance/src/dataset/mem_wal/scanner/planner.rs @@ -18,16 +18,17 @@ use crate::dataset::mem_wal::TOMBSTONE; use super::collector::LsmDataSourceCollector; use super::data_source::LsmDataSource; use super::exec::{MEMTABLE_GEN_COLUMN, MemtableGenTagExec, PkBlockFilterExec, ROW_ADDRESS_COLUMN}; -use super::flushed_cache::{DatasetCache, GenerationWarmer, open_flushed_dataset}; use super::projection::{ build_scanner_projection, canonical_output_schema, null_columns, project_to_canonical, validate_projection_names, }; +use super::sstable_cache::{DatasetCache, SsTableWarmer, open_sstable}; use crate::session::Session; +use lance_io::object_store::ObjectStoreParams; /// Combine the user filter (if any) with `NOT _tombstone` so tombstone rows are /// dropped from a WAL-arm scan. Used only for sources whose schema carries the -/// column (active / flushed generations written since deletes existed). +/// column (active / SSTables written since deletes existed). fn fold_not_tombstone(filter: Option<&Expr>) -> Expr { let live = !col(TOMBSTONE); match filter { @@ -44,25 +45,14 @@ pub struct LsmScanPlanner { pk_columns: Vec, /// Schema of the base table. base_schema: SchemaRef, - /// Session threaded into flushed-generation opens (shared caches). + /// Session threaded into SSTable opens (shared caches). session: Option>, - /// Cache of opened flushed-generation datasets. - flushed_cache: Option>, - /// Optional warmer fired on first open of a flushed generation. - warmer: Option>, - /// Over-fetch multiple for the per-source limit pushdown: block-listed - /// sources scan `(offset + limit) * factor` rows so cross-gen dedup drops - /// still leave enough live rows. Clamped to `>= 1.0`. - /// - /// This headroom must also absorb deletes: a tombstone shadows the older - /// real row without emitting a replacement (shadow-without-replace), so it - /// is pure subtraction from a block-listed source. If delete density inside - /// a source's fetch window exceeds `factor - 1`, that source can deliver - /// `< k` live rows (a recall shortfall, not wrong content — see the - /// under-fetch `warn!` in `PkBlockFilterExec`). Steady-state this is - /// self-limiting because L0→base compaction drains tombstones from the - /// fresh tier; the exposure is a delete-heavy burst between compactions. - overfetch_factor: f64, + /// Store params for opening SSTables, reusing the base dataset's store. + store_params: Option, + /// Cache of opened SSTable datasets. + sstable_cache: Option>, + /// Optional warmer fired on first open of an SSTable. + warmer: Option>, } impl LsmScanPlanner { @@ -77,37 +67,34 @@ impl LsmScanPlanner { pk_columns, base_schema, session: None, - flushed_cache: None, + store_params: None, + sstable_cache: None, warmer: None, - overfetch_factor: 1.0, } } - /// Thread a session into flushed-generation opens so the first open - /// populates the shared index / file-metadata caches. + /// Set the session used to open SSTables. pub fn with_session(mut self, session: Arc) -> Self { self.session = Some(session); self } - /// Inject a cache of opened flushed-generation datasets, making repeated - /// queries against the same generation a pure `Arc::clone`. - pub fn with_flushed_cache(mut self, cache: Arc) -> Self { - self.flushed_cache = Some(cache); + /// Set the store params used to open SSTables. + pub fn with_store_params(mut self, store_params: ObjectStoreParams) -> Self { + self.store_params = Some(store_params); self } - /// Inject the warmer fired on first open of a flushed generation. - pub fn with_warmer(mut self, warmer: Arc) -> Self { - self.warmer = Some(warmer); + /// Inject a cache of opened SSTable datasets, making repeated + /// queries against the same generation a pure `Arc::clone`. + pub fn with_sstable_cache(mut self, cache: Arc) -> Self { + self.sstable_cache = Some(cache); self } - /// Set the over-fetch multiple for the per-source limit pushdown - /// (see the field docs). Values below `1.0` are rejected by - /// [`Self::plan_scan`]. - pub fn with_overfetch_factor(mut self, factor: f64) -> Self { - self.overfetch_factor = factor; + /// Inject the warmer fired on first open of an SSTable. + pub fn with_warmer(mut self, warmer: Arc) -> Self { + self.warmer = Some(warmer); self } @@ -154,7 +141,6 @@ impl LsmScanPlanner { // 1. Collect all data sources let sources = self.collector.collect()?; - let overfetch = super::validate_overfetch_factor(self.overfetch_factor)?; if sources.is_empty() { // Return empty plan @@ -167,7 +153,8 @@ impl LsmScanPlanner { let block_lists = Box::pin(super::block_list::compute_source_block_lists( &sources, self.session.as_ref(), - self.flushed_cache.as_ref(), + self.store_params.as_ref(), + self.sstable_cache.as_ref(), )) .await?; @@ -177,14 +164,13 @@ impl LsmScanPlanner { let sources: Vec<_> = sources.into_iter().rev().collect(); // Per-source limit pushdown: an unordered LIMIT needs only - // `offset + limit` live rows from EACH source to fill the global - // limit after dedup (any-N semantics), so cap every on-disk source - // instead of scanning whole generations and trimming above the - // union. Block-listed sources over-fetch by `overfetch_factor` so - // cross-gen dedup drops still leave `n_needed` live rows; the - // PkBlockFilter warns when that was not enough. The active memtable - // is in-memory and within-gen append duplicates are resolved by its - // own dedup, so it is never capped here. + // `offset + limit` live rows from each source to fill the global limit + // (any-N semantics). However, a source with a cross-generation block + // list can lose any number of rows after its scan, so a finite fetch + // before that filter is not safe. Leave those scans unbounded and let + // the LocalLimitExec below pull until it sees `n_needed` live rows or + // reaches EOF. Sources without a block filter can still push the limit + // down safely. The active memtable is in-memory and is never capped. let n_needed = limit.map(|l| l.saturating_add(offset.unwrap_or(0))); let mut source_plans = Vec::new(); @@ -194,12 +180,9 @@ impl LsmScanPlanner { let blocked = block_lists .get(&(source.shard_id(), source.generation())) .cloned(); - let fetch = match (n_needed, is_active) { - (Some(n), false) => Some(if blocked.is_some() && !self.pk_columns.is_empty() { - ((n as f64) * overfetch).ceil() as usize - } else { - n - }), + let has_block_filter = blocked.is_some() && !self.pk_columns.is_empty(); + let fetch = match (n_needed, is_active, has_block_filter) { + (Some(n), false, false) => Some(n), _ => None, }; let scan = self @@ -207,14 +190,14 @@ impl LsmScanPlanner { .await?; // Drop cross-generation stale rows (PKs superseded by a newer gen). - // With a limit, `k = n_needed` arms the under-fetch warning; with - // no limit `k = 0` keeps it silent. + // Plain scans refill exactly, so keep the approximate-search + // under-fetch warning disabled with k = 0. let scan = match blocked { Some(set) if !self.pk_columns.is_empty() => Arc::new(PkBlockFilterExec::new( scan, self.pk_columns.clone(), set, - n_needed.unwrap_or(0), + 0, )) as Arc, _ => scan, @@ -338,11 +321,12 @@ impl LsmScanPlanner { scanner.create_plan().await } - LsmDataSource::FlushedMemTable { path, .. } => { - let dataset = open_flushed_dataset( + LsmDataSource::SsTable { path, .. } => { + let dataset = open_sstable( path, self.session.as_ref(), - self.flushed_cache.as_ref(), + self.store_params.as_ref(), + self.sstable_cache.as_ref(), self.warmer.as_ref(), ) .await?; @@ -368,7 +352,7 @@ impl LsmScanPlanner { if let Some(expr) = effective { scanner.filter_expr(expr.clone()); } - // Per-source limit pushdown: flushed generations are + // Per-source limit pushdown: SSTables are // within-gen live (dedup-on-flush deletion vectors), so any // `fetch` post-filter rows are valid contributions. if let Some(fetch) = fetch { @@ -472,10 +456,10 @@ mod tests { let shard_id = uuid::Uuid::new_v4(); let snapshot = ShardSnapshot::new(shard_id) .with_current_generation(5) - .with_flushed_generation(1, "gen_1".to_string()) - .with_flushed_generation(2, "gen_2".to_string()); + .with_sstable(1, "gen_1".to_string()) + .with_sstable(2, "gen_2".to_string()); - assert_eq!(snapshot.flushed_generations.len(), 2); + assert_eq!(snapshot.sstables.len(), 2); assert_eq!(snapshot.current_generation, 5); } } @@ -535,7 +519,7 @@ mod integration_tests { } /// Create a dataset at the given URI with the provided batches. Also writes - /// the standalone PK sidecar (on `id`) so a flushed-generation source can be + /// the standalone PK sidecar (on `id`) so an SSTable source can be /// probed by the block-list; harmless for a base table (never probed). async fn create_dataset(uri: &str, batches: Vec) -> Dataset { let schema = batches[0].schema(); @@ -568,8 +552,8 @@ mod integration_tests { /// Setup a multi-level LSM structure with: /// - Base table: ids 1-5 with "base" prefix - /// - Flushed gen1: ids 3,4 (updates) with "gen1" prefix - /// - Flushed gen2: ids 4,5 (updates) + id 6 (new) with "gen2" prefix + /// - SSTable gen1: ids 3,4 (updates) with "gen1" prefix + /// - SSTable gen2: ids 4,5 (updates) + id 6 (new) with "gen2" prefix /// - Active memtable: ids 5,6 (updates) + id 7 (new) with "active" prefix /// /// Expected deduplication results: @@ -596,13 +580,13 @@ mod integration_tests { let base_batch = create_test_batch(&schema, &[1, 2, 3, 4, 5], "base"); let base_dataset = Arc::new(create_dataset(&base_uri, vec![base_batch]).await); - // Create flushed gen1 as a separate dataset + // Create SSTable gen1 as a separate dataset let shard_id = Uuid::new_v4(); let gen1_uri = format!("{}/_mem_wal/{}/gen_1", base_uri, shard_id); let gen1_batch = create_test_batch(&schema, &[3, 4], "gen1"); create_dataset(&gen1_uri, vec![gen1_batch]).await; - // Create flushed gen2 as a separate dataset + // Create SSTable gen2 as a separate dataset let gen2_uri = format!("{}/_mem_wal/{}/gen_2", base_uri, shard_id); let gen2_batch = create_test_batch(&schema, &[4, 5, 6], "gen2"); create_dataset(&gen2_uri, vec![gen2_batch]).await; @@ -610,8 +594,8 @@ mod integration_tests { // Build shard snapshot let shard_snapshot = ShardSnapshot::new(shard_id) .with_current_generation(3) - .with_flushed_generation(1, "gen_1".to_string()) - .with_flushed_generation(2, "gen_2".to_string()); + .with_sstable(1, "gen_1".to_string()) + .with_sstable(2, "gen_2".to_string()); // Create active memtable let (batch_store, index_store) = @@ -835,11 +819,11 @@ mod integration_tests { } /// Regression for the concurrent-read-vs-flush hole: a sealed - /// (frozen-awaiting-flush) memtable is not yet recorded as a flushed - /// generation, but its rows must still be in the scan's read union and + /// (frozen-awaiting-flush) memtable is not yet recorded as an + /// SSTable, but its rows must still be in the scan's read union and /// dedup correctly by generation across the active/frozen seam. /// - /// Layout: base(0) ids 1-5, flushed gen1 ids 3,4, flushed gen2 ids + /// Layout: base(0) ids 1-5, SSTable gen1 ids 3,4, SSTable gen2 ids /// 4,5,6, frozen memtable gen3 ids 6,7, active memtable gen4 ids 7,8. #[tokio::test] async fn test_lsm_scan_frozen_memtable_in_read_union() { @@ -868,8 +852,8 @@ mod integration_tests { let shard_snapshot = ShardSnapshot::new(shard_id) .with_current_generation(4) - .with_flushed_generation(1, "gen_1".to_string()) - .with_flushed_generation(2, "gen_2".to_string()); + .with_sstable(1, "gen_1".to_string()) + .with_sstable(2, "gen_2".to_string()); // Frozen gen3 (sealed, NOT in the manifest) and active gen4. let (frozen_store, frozen_index) = @@ -928,7 +912,7 @@ mod integration_tests { assert_eq!(results.get(&3), Some(&"gen1_3".to_string())); assert_eq!(results.get(&4), Some(&"gen2_4".to_string())); assert_eq!(results.get(&5), Some(&"gen2_5".to_string())); - // id=6: in flushed gen2 AND frozen gen3 -> frozen wins. This is the + // id=6: in SSTable gen2 AND frozen gen3 -> frozen wins. This is the // bug: pre-fix the frozen memtable fell out of the read union and // id=6 resolved to "gen2_6". assert_eq!(results.get(&6), Some(&"frozen_6".to_string())); @@ -996,6 +980,68 @@ mod integration_tests { assert_eq!(total_rows, 3, "Should have 3 rows due to limit"); } + #[tokio::test] + async fn test_lsm_scan_limit_offset_refills_after_update_shadow_across_fragments() { + let schema = create_pk_schema(); + let temp = tempfile::tempdir().unwrap(); + let base_uri = format!("{}/base", temp.path().to_str().unwrap()); + let base_batch = create_test_batch(&schema, &[1, 2, 3, 4], "base"); + let reader = RecordBatchIterator::new([Ok(base_batch)], schema.clone()); + let base = Arc::new( + Dataset::write( + reader, + &base_uri, + Some(WriteParams { + max_rows_per_file: 2, + ..Default::default() + }), + ) + .await + .unwrap(), + ); + assert_eq!( + base.get_fragments().len(), + 2, + "the stale prefix and live rows must occupy separate fragments" + ); + + // The newest values for ids 1 and 2 no longer match the predicate, so + // their matching base values are shadowed. The second base fragment + // still contains the live matches that must fill offset + limit. + let (batch_store, index_store) = + pk_indexed(&[create_test_batch(&schema, &[1, 2], "active")]); + let scanner = LsmScanner::new(base, vec![], vec!["id".to_string()]) + .with_in_memory_memtables( + Uuid::new_v4(), + InMemoryMemTables { + active: InMemoryMemTableRef { + batch_store, + index_store, + schema, + generation: 1, + }, + frozen: vec![], + }, + ) + .filter("name LIKE 'base%'") + .unwrap() + .limit(Some(1), Some(1)) + .unwrap(); + + let batch = scanner.try_into_batch().await.unwrap(); + let ids = batch + .column_by_name("id") + .unwrap() + .as_any() + .downcast_ref::() + .unwrap(); + assert_eq!( + ids.iter().collect::>(), + vec![Some(4)], + "offset=1, limit=1 must skip id=3 after shadow filtering" + ); + } + #[tokio::test] async fn test_lsm_scan_with_offset_without_limit() { let (base_dataset, shard_snapshots, active_memtable, pk_columns, _temp_path) = @@ -1030,7 +1076,7 @@ mod integration_tests { let (base_dataset, _, _, pk_columns, _temp_path) = setup_multi_level_lsm().await; // Create scanner with only base table (no shard snapshots or active memtable) - let scanner = LsmScanner::new(base_dataset, vec![], pk_columns); + let scanner = LsmScanner::new(base_dataset.clone(), vec![], pk_columns.clone()); let plan = scanner.create_plan().await.unwrap(); @@ -1044,6 +1090,22 @@ mod integration_tests { .await .unwrap(); + // A base-only source has no cross-generation block filter, so its + // finite limit remains safe to push into the physical Lance read. + let scanner = LsmScanner::new(base_dataset, vec![], pk_columns) + .limit(Some(3), None) + .unwrap(); + let plan = scanner.create_plan().await.unwrap(); + assert_plan_node_equals( + plan, + "GlobalLimitExec: skip=0, fetch=3 + ProjectionExec:... + LocalLimitExec: fetch=3 + LanceRead:...base/data...range_before=Some(0..3)...refine_filter=--", + ) + .await + .unwrap(); + // Execute and verify all 5 base rows are returned let scanner = LsmScanner::new( Arc::new( @@ -1067,7 +1129,7 @@ mod integration_tests { } #[tokio::test] - async fn test_lsm_scan_flushed_only_no_active() { + async fn test_lsm_scan_sstable_only_no_active() { let (base_dataset, shard_snapshots, _, pk_columns, _temp_path) = setup_multi_level_lsm().await; @@ -1229,7 +1291,7 @@ mod integration_tests { /// /// Similar to setup_multi_level_lsm but: /// - Active memtable has a BTree index on the `id` column - /// - Flushed datasets have BTree index created (enabling ScalarIndexQuery) + /// - SSTables have BTree index created (enabling ScalarIndexQuery) async fn setup_multi_level_lsm_with_btree_index() -> ( Arc, Vec, @@ -1259,7 +1321,7 @@ mod integration_tests { // Reload dataset to pick up the index let base_dataset = Arc::new(Dataset::open(&base_uri).await.unwrap()); - // Create flushed gen1 with BTree index + // Create SSTable gen1 with BTree index let shard_id = Uuid::new_v4(); let gen1_uri = format!("{}/_mem_wal/{}/gen_1", base_uri, shard_id); let gen1_batch = create_test_batch(&schema, &[3, 4], "gen1"); @@ -1268,7 +1330,7 @@ mod integration_tests { .await .unwrap(); - // Create flushed gen2 with BTree index + // Create SSTable gen2 with BTree index let gen2_uri = format!("{}/_mem_wal/{}/gen_2", base_uri, shard_id); let gen2_batch = create_test_batch(&schema, &[4, 5, 6], "gen2"); let mut gen2_dataset = create_dataset(&gen2_uri, vec![gen2_batch]).await; @@ -1279,8 +1341,8 @@ mod integration_tests { // Build shard snapshot let shard_snapshot = ShardSnapshot::new(shard_id) .with_current_generation(3) - .with_flushed_generation(1, "gen_1".to_string()) - .with_flushed_generation(2, "gen_2".to_string()); + .with_sstable(1, "gen_1".to_string()) + .with_sstable(2, "gen_2".to_string()); // Create active memtable with BTree index let batch_store = Arc::new(BatchStore::with_capacity(100)); @@ -1615,7 +1677,7 @@ mod integration_tests { let (base_dataset, shard_snapshots, active_memtable, pk_columns, _temp_path) = setup_multi_level_lsm().await; - // Use the same base URI the flushed generations were created under, so + // Use the same base URI the SSTables were created under, so // relative `gen_N` folders resolve to real datasets on disk. let base_uri = base_dataset.uri().to_string(); let arrow_schema: arrow_schema::Schema = base_dataset.schema().into(); @@ -1640,7 +1702,7 @@ mod integration_tests { ); assert!( plan_str.contains("gen_1") && plan_str.contains("gen_2"), - "Plan must scan flushed generations, got: {}", + "Plan must scan SSTables, got: {}", plan_str ); assert!( @@ -1747,8 +1809,8 @@ mod integration_tests { } #[tokio::test] - async fn test_lsm_scan_without_base_table_no_flushed_no_active() { - // No base, no flushed, no active → empty result, valid plan. + async fn test_lsm_scan_without_base_table_no_sstable_no_active() { + // No base, no SSTable, no active → empty result, valid plan. let schema = create_pk_schema(); let scanner = LsmScanner::without_base_table( schema, @@ -2082,8 +2144,48 @@ mod integration_tests { } #[tokio::test] - async fn test_lsm_scan_flushed_tombstone_masks_base() { - // A tombstone living in a flushed generation masks the older base row by + async fn test_lsm_scan_limit_refills_after_active_tombstone_shadows_base() { + let base_schema = create_pk_schema(); + let mem_schema = ts_pk_schema(); + let temp = tempfile::tempdir().unwrap(); + let base_uri = format!("{}/base", temp.path().to_str().unwrap()); + let base = Arc::new( + create_dataset( + &base_uri, + vec![create_test_batch(&base_schema, &[1, 2, 3], "base")], + ) + .await, + ); + + let active_batch = ts_batch(&mem_schema, &[(1, None, true)]); + let (batch_store, index_store) = pk_indexed(&[active_batch]); + let scanner = LsmScanner::new(base, vec![], vec!["id".to_string()]) + .with_in_memory_memtables( + Uuid::new_v4(), + InMemoryMemTables { + active: InMemoryMemTableRef { + batch_store, + index_store, + schema: mem_schema, + generation: 1, + }, + frozen: vec![], + }, + ) + .limit(Some(2), None) + .unwrap(); + + let batch = scanner.try_into_batch().await.unwrap(); + assert_eq!( + collect_sorted_ids(&[batch]), + vec![2, 3], + "the base scan must continue past the shadowed row to fill the limit" + ); + } + + #[tokio::test] + async fn test_lsm_scan_sstable_tombstone_masks_base() { + // A tombstone living in an SSTable masks the older base row by // PK presence (block-list) and is itself dropped by the folded predicate. let base_schema = create_pk_schema(); let mem_schema = ts_pk_schema(); @@ -2098,14 +2200,14 @@ mod integration_tests { .await, ); - // Flushed gen 1 holds only a tombstone for id=2 (written with the - // `_tombstone` schema, so the flushed arm folds `NOT _tombstone`). + // SSTable gen 1 holds only a tombstone for id=2 (written with the + // `_tombstone` schema, so the SSTable arm folds `NOT _tombstone`). let shard_id = Uuid::new_v4(); let gen1_uri = format!("{}/_mem_wal/{}/gen_1", base_uri, shard_id); create_dataset(&gen1_uri, vec![ts_batch(&mem_schema, &[(2, None, true)])]).await; let shard_snapshot = ShardSnapshot::new(shard_id) .with_current_generation(2) - .with_flushed_generation(1, "gen_1".to_string()); + .with_sstable(1, "gen_1".to_string()); let scanner = LsmScanner::new(base, vec![shard_snapshot], vec!["id".to_string()]); let batches: Vec = scanner @@ -2118,13 +2220,13 @@ mod integration_tests { assert_eq!( collect_sorted_ids(&batches), vec![1, 3], - "id=2 deleted via flushed-generation tombstone" + "id=2 deleted via an SSTable tombstone" ); } #[tokio::test] async fn test_lsm_scan_tombstone_does_not_consume_limit() { - // A single (newest) flushed generation holds both tombstones and live + // A single (newest) SSTable holds both tombstones and live // rows. With LIMIT 2 the folded `NOT _tombstone` runs *before* the // per-source pushdown limit, so the limit counts only live rows — we get // 2 live rows, not 0 (which is what a post-limit tombstone filter, or a @@ -2152,7 +2254,7 @@ mod integration_tests { .await; let shard_snapshot = ShardSnapshot::new(shard_id) .with_current_generation(2) - .with_flushed_generation(1, "gen_1".to_string()); + .with_sstable(1, "gen_1".to_string()); let scanner = LsmScanner::without_base_table( base_schema, diff --git a/rust/lance/src/dataset/mem_wal/scanner/point_lookup.rs b/rust/lance/src/dataset/mem_wal/scanner/point_lookup.rs index 4f7c5f093f6..9639c0d4f9f 100644 --- a/rust/lance/src/dataset/mem_wal/scanner/point_lookup.rs +++ b/rust/lance/src/dataset/mem_wal/scanner/point_lookup.rs @@ -27,19 +27,20 @@ use lance_core::{Result, is_system_column}; use lance_datafusion::exec::OneShotExec; use tracing::instrument; -use crate::dataset::mem_wal::TOMBSTONE; use crate::dataset::mem_wal::index::IndexStore; use crate::dataset::mem_wal::memtable::batch_store::BatchStore; +use crate::dataset::mem_wal::{TOMBSTONE, relax_non_pk_nullability}; use super::collector::LsmDataSourceCollector; use super::data_source::LsmDataSource; use super::exec::{BloomFilterGuardExec, CoalesceFirstExec, compute_pk_hash_from_scalars}; -use super::flushed_cache::{DatasetCache, GenerationWarmer, open_flushed_dataset}; use super::projection::{ - DISTANCE_COLUMN, build_scanner_projection, canonical_output_schema, null_columns, + DISTANCE_COLUMN, build_scanner_projection, canonical_output_schema, force_schema, null_columns, project_to_canonical, validate_projection_names, wants_row_address, wants_row_id, }; +use super::sstable_cache::{DatasetCache, SsTableWarmer, open_sstable}; use crate::session::Session; +use lance_io::object_store::ObjectStoreParams; /// Plans point lookup queries over LSM data. /// @@ -87,12 +88,14 @@ pub struct LsmPointLookupPlanner { /// Bloom filters for each memtable generation. /// Map: generation -> bloom filter bloom_filters: std::collections::HashMap>, - /// Session threaded into flushed-generation opens (shared caches). + /// Session threaded into SSTable opens (shared caches). session: Option>, - /// Cache of opened flushed-generation datasets. - flushed_cache: Option>, - /// Optional warmer fired on first open of a flushed generation. - warmer: Option>, + /// Store params for opening SSTables, reusing the base dataset's store. + store_params: Option, + /// Cache of opened SSTable datasets. + sstable_cache: Option>, + /// Optional warmer fired on first open of an SSTable. + warmer: Option>, /// Precomputed canonical output schema for the no-projection case, so the /// hot `lookup(.., None)` path clones an `Arc` instead of rebuilding the /// schema on every call. @@ -124,32 +127,38 @@ impl LsmPointLookupPlanner { base_schema, bloom_filters: std::collections::HashMap::new(), session: None, - flushed_cache: None, + store_params: None, + sstable_cache: None, warmer: None, none_target, task_ctx: SessionContext::new().task_ctx(), } } - /// Thread a session into flushed-generation opens so the first open - /// populates the shared index / file-metadata caches. + /// Set the session used to open SSTables. pub fn with_session(mut self, session: Arc) -> Self { self.session = Some(session); self } - /// Inject a cache of opened flushed-generation datasets, making repeated + /// Set the store params used to open SSTables. + pub fn with_store_params(mut self, store_params: ObjectStoreParams) -> Self { + self.store_params = Some(store_params); + self + } + + /// Inject a cache of opened SSTable datasets, making repeated /// lookups against the same generation a pure `Arc::clone`. Populate it up /// front during scan setup via /// [`DatasetMemWalExt::prewarm_mem_wal`](crate::dataset::mem_wal::DatasetMemWalExt::prewarm_mem_wal) /// so the first gen-key lookup does not pay the dataset open. - pub fn with_flushed_cache(mut self, cache: Arc) -> Self { - self.flushed_cache = Some(cache); + pub fn with_sstable_cache(mut self, cache: Arc) -> Self { + self.sstable_cache = Some(cache); self } - /// Inject the warmer fired on first open of a flushed generation. - pub fn with_warmer(mut self, warmer: Arc) -> Self { + /// Inject the warmer fired on first open of an SSTable. + pub fn with_warmer(mut self, warmer: Arc) -> Self { self.warmer = Some(warmer); self } @@ -312,7 +321,7 @@ impl LsmPointLookupPlanner { ) -> Result { let canonical = canonical_output_schema(projection, &self.base_schema, &self.pk_columns, false); - let target = carry_schema(&canonical); + let target = carry_schema(&canonical, &self.pk_columns); let mut out: Vec = Vec::with_capacity(keys.len()); for key in keys { if let Some(b) = self.lookup_keep_tombstone(key, projection).await? { @@ -332,7 +341,7 @@ impl LsmPointLookupPlanner { /// For a single-column primary key this probes the in-memory memtables' /// BTree index directly — no DataFusion plan — newest generation first, and /// returns on the first hit. Only when the lookup must consult an on-disk - /// source (a flushed generation or the base table), a memtable lacks a + /// source (an SSTable or the base table), a memtable lacks a /// BTree on the key, the key is multi-column, or the projection requests /// system columns does it fall back to [`Self::plan_lookup`]. The result is /// identical to executing `plan_lookup` and taking the first row; the fast @@ -407,7 +416,7 @@ impl LsmPointLookupPlanner { None => { // Every in-memory memtable missed. If there is no // on-disk source, the key does not exist; otherwise the - // plan path consults the base table / flushed gens. + // plan path consults the base table / SSTables. if !self.collector.has_on_disk_sources() { return Ok(None); } @@ -645,13 +654,18 @@ impl LsmPointLookupPlanner { scanner.with_row_address(); } scanner.filter_expr(filter.clone()); - scanner.create_plan().await? + // Box at the call site: `create_plan`'s inlined async layout exceeds + // rustc's depth limit up this point-lookup chain, and boxing inside + // `create_plan` instead triggers a `Box: Send` solver overflow + // (E0275 downstream). Same for the other arms below. + Box::pin(scanner.create_plan()).await? } - LsmDataSource::FlushedMemTable { path, .. } => { - let dataset = open_flushed_dataset( + LsmDataSource::SsTable { path, .. } => { + let dataset = open_sstable( path, self.session.as_ref(), - self.flushed_cache.as_ref(), + self.store_params.as_ref(), + self.sstable_cache.as_ref(), self.warmer.as_ref(), ) .await?; @@ -662,7 +676,7 @@ impl LsmPointLookupPlanner { let cols = cols_with_tombstone(&cols, dataset.schema().field(TOMBSTONE).is_some()); scanner.project(&cols.iter().map(|s| s.as_str()).collect::>())?; scanner.filter_expr(filter.clone()); - scanner.create_plan().await? + Box::pin(scanner.create_plan()).await? } LsmDataSource::ActiveMemTable { batch_store, @@ -685,7 +699,7 @@ impl LsmPointLookupPlanner { // over insert-ordered scan would return the *oldest* of // multiple rows sharing the target primary key. scanner.with_row_id(); - let raw = scanner.create_plan().await?; + let raw = Box::pin(scanner.create_plan()).await?; // The filter already restricts to the exact PK value, so the // scan yields that key's insert history. Within the active // memtable larger `_rowid` = newer insert, so sorting `_rowid` @@ -714,7 +728,7 @@ impl LsmPointLookupPlanner { // Output carries `_tombstone` (canonical + the marker) so it survives // the union/coalesce to the post-coalesce filter; base / legacy sources // that lack the column get a synthesized `false`. - project_to_carry(scan, &target) + project_to_carry(scan, &target, &self.pk_columns) } /// Create an empty execution plan with the canonical output schema. @@ -742,11 +756,18 @@ fn cols_with_tombstone(cols: &[String], present: bool) -> Vec { out } -/// Carry schema = canonical output + a trailing non-nullable `_tombstone` -/// Boolean. Non-nullable so the base arm's synthesized `Literal(false)` matches -/// the WAL arms' real column under `CoalesceFirstExec`'s exact-schema check. -fn carry_schema(canonical: &SchemaRef) -> SchemaRef { - let mut fields: Vec> = canonical.fields().iter().cloned().collect(); +/// Carry schema = canonical output widened to the storage schema's +/// nullability, plus a trailing non-nullable `_tombstone` Boolean. +/// +/// Widened because tombstone rows — null in every non-PK column — are still in +/// flight; [`filter_tombstones_after_coalesce`] drops them past +/// `CoalesceFirstExec`, and only then does the plan narrow back to the logical +/// schema. `_tombstone` stays non-nullable so the base arm's synthesized +/// `Literal(false)` matches the WAL arms' real column under +/// `CoalesceFirstExec`'s exact-schema check. +fn carry_schema(canonical: &SchemaRef, pk_columns: &[String]) -> SchemaRef { + let widened = relax_non_pk_nullability(canonical, pk_columns); + let mut fields: Vec> = widened.fields().iter().cloned().collect(); fields.push(Arc::new(Field::new(TOMBSTONE, DataType::Boolean, false))); Arc::new(Schema::new(fields)) } @@ -758,9 +779,10 @@ fn carry_schema(canonical: &SchemaRef) -> SchemaRef { fn project_to_carry( plan: Arc, canonical: &SchemaRef, + pk_columns: &[String], ) -> Result> { let input = plan.schema(); - let carry = carry_schema(canonical); + let carry = carry_schema(canonical, pk_columns); let mut project_exprs: Vec<(Arc, String)> = Vec::with_capacity(carry.fields().len()); for field in carry.fields() { @@ -784,11 +806,11 @@ fn project_to_carry( }; project_exprs.push((expr, name.clone())); } - Ok(Arc::new( - ProjectionExec::try_new(project_exprs, plan).map_err(|e| { - lance_core::Error::internal(format!("Failed to build carry ProjectionExec: {}", e)) - })?, - )) + let projected = Arc::new(ProjectionExec::try_new(project_exprs, plan).map_err(|e| { + lance_core::Error::internal(format!("Failed to build carry ProjectionExec: {}", e)) + })?); + // `CoalesceFirstExec` panics unless every arm lands on exactly `carry`. + Ok(force_schema(projected, &carry)) } /// Drop tombstone rows after `CoalesceFirstExec` has already picked the newest @@ -881,7 +903,12 @@ fn probe_position( if len == 0 { return Ok(ProbePos::Miss); } - let last_visible_idx = index_store.max_visible_batch_position().min(len - 1); + // The cursor is an exclusive count, so the last visible batch sits at + // `count - 1`. A count of 0 means nothing is visible yet — not "batch 0". + let visible_count = index_store.visible_count().min(len); + let Some(last_visible_idx) = visible_count.checked_sub(1) else { + return Ok(ProbePos::Miss); + }; let last = batch_store.get(last_visible_idx).ok_or_else(|| { lance_core::Error::internal("point-lookup: visible batch index out of range") })?; @@ -1100,7 +1127,7 @@ mod tests { let shard_snapshot = ShardSnapshot::new(shard_id) .with_current_generation(2) - .with_flushed_generation(1, "gen_1".to_string()); + .with_sstable(1, "gen_1".to_string()); // Create collector let collector = LsmDataSourceCollector::new(base_dataset, vec![shard_snapshot]); @@ -1181,10 +1208,10 @@ mod tests { let base_path = temp_dir.path().to_str().unwrap(); // No base dataset is created. We still need a base URI so the collector - // can resolve flushed-generation paths. + // can resolve SSTable paths. let base_uri = format!("{}/base", base_path); - // Create a flushed generation under {base_uri}/_mem_wal/{shard}/gen_1 + // Create an SSTable under {base_uri}/_mem_wal/{shard}/gen_1 let shard_id = Uuid::new_v4(); let gen1_uri = format!("{}/_mem_wal/{}/gen_1", base_uri, shard_id); let gen1_batch = create_test_batch(&schema, &[2, 3], "gen1"); @@ -1192,12 +1219,12 @@ mod tests { let shard_snapshot = ShardSnapshot::new(shard_id) .with_current_generation(2) - .with_flushed_generation(1, "gen_1".to_string()); + .with_sstable(1, "gen_1".to_string()); let collector = LsmDataSourceCollector::without_base_table(base_uri, vec![shard_snapshot]); let planner = LsmPointLookupPlanner::new(collector, vec!["id".to_string()], schema); - // id=3 lives in the flushed generation + // id=3 lives in the SSTable let pk_values = vec![ScalarValue::Int32(Some(3))]; let plan = planner.plan_lookup(&pk_values, None).await.unwrap(); @@ -1378,8 +1405,9 @@ mod tests { let batch_store = Arc::new(BatchStore::with_capacity(16)); let mut index_store = IndexStore::new(); - // BTree on the PK so that `max_visible_batch_position` advances as - // we insert, otherwise the scanner sees no batches at all. + // BTree on the PK: the point lookup resolves keys through the indexed PK + // path, which this exercises. (`indexed_count`/`visible_count` advance + // from the batch position regardless of whether any index is configured.) index_store.add_btree("id_idx".to_string(), 0, "id".to_string()); // Two writes to pk=1, then an unrelated pk=2. The "new" row goes @@ -1520,10 +1548,10 @@ mod tests { } #[tokio::test] - async fn test_point_lookup_flushed_memtable_returns_newest_duplicate() { - // Regression / invariant pin: when a flushed memtable contains two + async fn test_point_lookup_sstable_returns_newest_duplicate() { + // Regression / invariant pin: when an SSTable contains two // rows for the same PK, the lookup must return the newer one. The - // flushed dataset is reverse-written (newest at the smallest + // SSTable dataset is reverse-written (newest at the smallest // physical position), so we simulate that here by writing the // dataset with the new row first. The point-lookup plan today // returns the first match (smallest `_rowid`) under reverse-write, @@ -1544,7 +1572,7 @@ mod tests { let shard_snapshot = ShardSnapshot::new(shard_id) .with_current_generation(2) - .with_flushed_generation(1, "gen_1".to_string()); + .with_sstable(1, "gen_1".to_string()); let collector = LsmDataSourceCollector::without_base_table(base_uri, vec![shard_snapshot]); let planner = LsmPointLookupPlanner::new(collector, vec!["id".to_string()], schema); @@ -1564,7 +1592,7 @@ mod tests { assert_eq!( name_arr.value(0), "new_1", - "flushed-arm lookup must return the row at the smallest _rowid (newest under reverse-write)" + "SSTable-arm lookup must return the row at the smallest _rowid (newest under reverse-write)" ); } diff --git a/rust/lance/src/dataset/mem_wal/scanner/projection.rs b/rust/lance/src/dataset/mem_wal/scanner/projection.rs index 0ec482aebf8..fd52ac90c0b 100644 --- a/rust/lance/src/dataset/mem_wal/scanner/projection.rs +++ b/rust/lance/src/dataset/mem_wal/scanner/projection.rs @@ -6,8 +6,8 @@ //! //! `MemTableScanner::project()` only special-cases `_rowid`; passing other //! system columns through it errors. And cross-LSM values for system -//! columns aren't comparable (a `_rowid` of 5 in the base and in a flushed -//! memtable refer to different rows). +//! columns aren't comparable (a `_rowid` of 5 in the base and in an +//! SSTable refer to different rows). //! //! - [`build_scanner_projection`] — strips system / `_distance` cols, appends PKs. //! - [`canonical_output_schema`] — final schema honoring user order; system @@ -25,6 +25,8 @@ use datafusion::physical_plan::projection::ProjectionExec; use datafusion::scalar::ScalarValue; use lance_core::{ROW_ADDR, ROW_ID, Result, is_system_column}; +use super::exec::SchemaRelabelExec; + /// Column name for distance in vector search results. pub const DISTANCE_COLUMN: &str = "_distance"; @@ -189,9 +191,26 @@ pub fn null_columns( Ok(Arc::new(projection_exec)) } +/// Force `plan` to report exactly `target_schema`; a no-op when they agree. +/// +/// `ProjectionExec` derives its nullability from the expressions, so the +/// storage schema's widened columns leave the WAL arms disagreeing with the +/// base arm — which `CoalesceFirstExec` and `concat_batches` both reject. +pub(super) fn force_schema( + plan: Arc, + target_schema: &SchemaRef, +) -> Arc { + if plan.schema() == *target_schema { + return plan; + } + Arc::new(SchemaRelabelExec::new(plan, target_schema.clone())) +} + /// Wrap `plan` to emit exactly `target_schema`. Source columns are /// forwarded by name; system / `_distance` cols missing from the source /// are NULL-filled. Other missing columns are an internal error. +/// +/// Reports `target_schema` exactly, nullability included — see [`force_schema`]. pub fn project_to_canonical( plan: Arc, target_schema: &SchemaRef, @@ -222,13 +241,15 @@ pub fn project_to_canonical( let projection_exec = ProjectionExec::try_new(project_exprs, plan).map_err(|e| { lance_core::Error::internal(format!("Failed to build canonical ProjectionExec: {}", e)) })?; - Ok(Arc::new(projection_exec)) + Ok(force_schema(Arc::new(projection_exec), target_schema)) } #[cfg(test)] mod tests { use super::*; + use arrow_array::RecordBatch; use arrow_schema::Schema as ArrowSchema; + use datafusion_physical_plan::test::TestMemoryExec; fn schema() -> SchemaRef { Arc::new(ArrowSchema::new(vec![ @@ -238,6 +259,20 @@ mod tests { ])) } + /// [`schema`] as `relax_non_pk_nullability` would leave it: `id` widened. + fn widened_schema() -> SchemaRef { + Arc::new(ArrowSchema::new(vec![ + Field::new("id", DataType::Int32, true), + Field::new("name", DataType::Utf8, true), + Field::new("vector", DataType::Float32, true), + ])) + } + + fn plan_emitting(schema: SchemaRef) -> Arc { + let batch = RecordBatch::new_empty(schema.clone()); + TestMemoryExec::try_new_exec(&[vec![batch]], schema, None).unwrap() + } + #[test] fn scanner_projection_strips_system_and_distance() { let s = schema(); @@ -320,4 +355,30 @@ mod tests { // _distance dropped because include_distance=false (e.g. point lookup / scan). assert_eq!(names, vec!["vector", "id"]); } + + #[test] + fn force_schema_leaves_a_matching_plan_alone() { + let plan = plan_emitting(schema()); + let forced = force_schema(plan.clone(), &schema()); + assert!( + Arc::ptr_eq(&plan, &forced), + "a plan already reporting the target schema must not be wrapped" + ); + } + + #[test] + fn force_schema_relabels_a_nullability_mismatch() { + let forced = force_schema(plan_emitting(widened_schema()), &schema()); + assert_eq!(forced.name(), "SchemaRelabelExec"); + assert_eq!(forced.schema(), schema()); + } + + #[test] + fn project_to_canonical_reports_the_target_schema() { + // The ProjectionExec alone would follow its input and report `id` as + // nullable; the relabel is what pins the output to the target. + let target = schema(); + let plan = project_to_canonical(plan_emitting(widened_schema()), &target).unwrap(); + assert_eq!(plan.schema(), target); + } } diff --git a/rust/lance/src/dataset/mem_wal/scanner/flushed_cache.rs b/rust/lance/src/dataset/mem_wal/scanner/sstable_cache.rs similarity index 75% rename from rust/lance/src/dataset/mem_wal/scanner/flushed_cache.rs rename to rust/lance/src/dataset/mem_wal/scanner/sstable_cache.rs index 7a5280bedb8..9bb8dadcf02 100644 --- a/rust/lance/src/dataset/mem_wal/scanner/flushed_cache.rs +++ b/rust/lance/src/dataset/mem_wal/scanner/sstable_cache.rs @@ -1,9 +1,9 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright The Lance Authors -//! Cache of opened flushed-generation datasets for the LSM scanner. +//! Cache of opened SSTable datasets for the LSM scanner. //! -//! Flushed generations are written exactly once to a globally-unique, +//! SSTables are written exactly once to a globally-unique, //! content-addressed path (see `memtable/flush.rs`): a fresh random hash per //! flush invocation means the same path always maps to the same immutable //! bytes. A cached `Arc` therefore can never go stale and needs no @@ -11,11 +11,11 @@ //! optimization driven by the consumer at compaction time. //! //! ```text -//! query ──> open_flushed_dataset(path, session, cache) +//! query ──> open_sstable(path, session, cache) //! │ //! cache.is_some() ──────┤────── cache.is_none() //! │ │ -//! FlushedMemTableCache::get_or_open DatasetBuilder::from_uri +//! SsTableCache::get_or_open DatasetBuilder::from_uri //! (single-flight, shared Arc) (cold open every call) //! ``` @@ -24,31 +24,41 @@ use std::sync::Arc; use async_trait::async_trait; use lance_core::{Error, Result}; +use lance_io::object_store::ObjectStoreParams; use crate::dataset::{Dataset, DatasetBuilder}; use crate::session::Session; -/// Cache of opened flushed-generation datasets, keyed by resolved path. +/// Cache of opened SSTable datasets, keyed by resolved path. /// -/// Flushed generations live at a globally-unique, immutable path, so cached +/// SSTables live at a globally-unique, immutable path, so cached /// entries are never stale and require no TTL. Intended to be held by a /// long-lived owner (one per process or per table) and injected into /// per-request scanners via [`crate::dataset::mem_wal::scanner::LsmScanner`] /// (and the point-lookup / vector-search planners). /// -/// The key is the resolved absolute flushed path +/// The key is the resolved absolute SSTable path /// (`{base}/_mem_wal/{shard}/{folder}`), which is globally unique, so a single /// cache can safely span multiple tables. -pub struct FlushedMemTableCache { +/// +/// `store_params` is deliberately *not* part of the key: the first caller to +/// open a path binds the store that every later hit reuses. Credential rotation +/// still works — a vended-credential store holds the live +/// `StorageOptionsAccessor` and re-resolves per request, so a cached handle +/// never carries expired credentials. What this does assume is that a given +/// path is only ever served under one store configuration. Serving one table +/// through a single cache under two different `ObjectStoreParams` would hand +/// every caller the store the first one opened with. +pub struct SsTableCache { // `moka`'s async cache gives a bounded size plus single-flight // `try_get_with`, so concurrent first-queries on a just-flushed - // generation open the dataset exactly once. The opened dataset carries the + // SSTable open the dataset exactly once. The opened dataset carries the // session index cache, which also backs each generation's standalone PK // dedup index (see `block_list::open_pk_index`) — no separate cache path. inner: moka::future::Cache>, } -impl FlushedMemTableCache { +impl SsTableCache { /// Create a cache holding at most `max_entries` opened datasets. /// /// Eviction is size-only (no TTL): an evicted-then-re-requested generation @@ -66,16 +76,14 @@ impl FlushedMemTableCache { } /// Get the dataset for `path`, opening it (exactly once) on a miss. - /// - /// `session` is threaded into the open so the first open populates the - /// shared index / file-metadata caches; subsequent hits are a pure - /// `Arc::clone` with zero object-store I/O. Concurrent callers for the - /// same path share a single open via `moka`'s single-flight - /// `try_get_with`. + /// Concurrent callers share a single open via `moka`'s single-flight + /// `try_get_with`; hits are a pure `Arc::clone`. `session` / `store_params` + /// configure the open. pub async fn get_or_open( &self, path: &str, session: Option>, + store_params: Option, ) -> Result> { self.inner .try_get_with(path.to_string(), async move { @@ -83,6 +91,9 @@ impl FlushedMemTableCache { if let Some(session) = session { builder = builder.with_session(session); } + if let Some(store_params) = store_params { + builder = builder.with_store_params(store_params); + } builder.load().await.map(Arc::new) }) .await @@ -108,36 +119,46 @@ impl FlushedMemTableCache { } } -impl std::fmt::Debug for FlushedMemTableCache { +impl std::fmt::Debug for SsTableCache { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - f.debug_struct("FlushedMemTableCache") + f.debug_struct("SsTableCache") .field("entry_count", &self.inner.entry_count()) .finish() } } -/// Caching of opened flushed-generation datasets, keyed by immutable path. The +/// Caching of opened SSTable datasets, keyed by immutable path. The /// opened dataset carries the session index cache, which also backs each /// generation's secondary indexes and its PK dedup sidecar (see /// `block_list::open_pk_index`) — so a single `get_or_open` is the -/// whole caching surface. Implemented by [`FlushedMemTableCache`]; a -/// [`GenerationWarmer`] composes one to warm through it, and a consumer may +/// whole caching surface. Implemented by [`SsTableCache`]; a +/// [`SsTableWarmer`] composes one to warm through it, and a consumer may /// supply its own implementation. #[async_trait] pub trait DatasetCache: Send + Sync + std::fmt::Debug { - async fn get_or_open(&self, path: &str, session: Option>) -> Result>; + async fn get_or_open( + &self, + path: &str, + session: Option>, + store_params: Option, + ) -> Result>; /// Drop cached entries whose path is not in `live_paths`. Async so an /// implementation can evict retired generations' index objects (e.g. /// `Session::invalidate_index_prefix`) without a later breaking signature - /// change; [`FlushedMemTableCache`]'s own eviction is synchronous. + /// change; [`SsTableCache`]'s own eviction is synchronous. async fn retain_paths(&self, live_paths: &HashSet); } #[async_trait] -impl DatasetCache for FlushedMemTableCache { - async fn get_or_open(&self, path: &str, session: Option>) -> Result> { - Self::get_or_open(self, path, session).await +impl DatasetCache for SsTableCache { + async fn get_or_open( + &self, + path: &str, + session: Option>, + store_params: Option, + ) -> Result> { + Self::get_or_open(self, path, session, store_params).await } async fn retain_paths(&self, live_paths: &HashSet) { @@ -145,7 +166,7 @@ impl DatasetCache for FlushedMemTableCache { } } -/// Proactively warms a flushed generation into the shared caches: open the +/// Proactively warms an SSTable into the shared caches: open the /// dataset and pre-load its secondary indexes and PK dedup sidecar so the first /// query sees no cold reads. This is the **seam** the flush and read paths fire /// — lance defines it; the consumer (e.g. the WAL pod) implements it. `None` => @@ -161,34 +182,42 @@ impl DatasetCache for FlushedMemTableCache { /// and cheap when the path is already warm** (e.g. dedup in-flight and /// completed paths) — a redundant call must not re-do work or fail. #[async_trait] -pub trait GenerationWarmer: Send + Sync + std::fmt::Debug { +pub trait SsTableWarmer: Send + Sync + std::fmt::Debug { async fn warm(&self, path: &str) -> Result<()>; } -/// Open a flushed-generation dataset, shared by all three LSM open sites +/// Open an SSTable dataset, shared by all three LSM open sites /// (scan, point lookup, vector search). /// /// - `cache` present: route through a [`DatasetCache`] (e.g. -/// [`FlushedMemTableCache`]: single-flight, shared `Arc`, manifest read +/// [`SsTableCache`]: single-flight, shared `Arc`, manifest read /// amortized across queries). /// - `cache` absent: cold open via [`DatasetBuilder`]. Passing `session` /// still reuses the shared index / metadata caches; `None`/`None` /// reproduces the original per-query cold-open behavior exactly. /// - `warmer` present: fire a fire-and-forget warm-on-open backstop behind the /// returned handle (the warmer dedups already-warm paths). `None` => no warming. -pub async fn open_flushed_dataset( +pub async fn open_sstable( path: &str, session: Option<&Arc>, + store_params: Option<&ObjectStoreParams>, cache: Option<&Arc>, - warmer: Option<&Arc>, + warmer: Option<&Arc>, ) -> Result> { let dataset = match cache { - Some(cache) => cache.get_or_open(path, session.cloned()).await?, + Some(cache) => { + cache + .get_or_open(path, session.cloned(), store_params.cloned()) + .await? + } None => { let mut builder = DatasetBuilder::from_uri(path); if let Some(session) = session { builder = builder.with_session(session.clone()); } + if let Some(store_params) = store_params { + builder = builder.with_store_params(store_params.clone()); + } Arc::new(builder.load().await?) } }; @@ -238,9 +267,9 @@ mod tests { let uri = format!("{}/gen_1", temp_dir.path().to_str().unwrap()); write_dataset(&uri, &[1, 2, 3]).await; - let cache = FlushedMemTableCache::new(8); - let first = cache.get_or_open(&uri, None).await.unwrap(); - let second = cache.get_or_open(&uri, None).await.unwrap(); + let cache = SsTableCache::new(8); + let first = cache.get_or_open(&uri, None, None).await.unwrap(); + let second = cache.get_or_open(&uri, None, None).await.unwrap(); assert!( Arc::ptr_eq(&first, &second), @@ -261,7 +290,7 @@ mod tests { let uri = format!("{}/gen_1", temp_dir.path().to_str().unwrap()); write_dataset(&uri, &[1, 2, 3]).await; - let cache = Arc::new(FlushedMemTableCache::new(8)); + let cache = Arc::new(SsTableCache::new(8)); let calls = Arc::new(AtomicUsize::new(0)); let mut handles = Vec::new(); @@ -271,7 +300,7 @@ mod tests { let calls = calls.clone(); handles.push(tokio::spawn(async move { calls.fetch_add(1, Ordering::SeqCst); - cache.get_or_open(&uri, None).await.unwrap() + cache.get_or_open(&uri, None, None).await.unwrap() })); } @@ -298,9 +327,9 @@ mod tests { write_dataset(&keep_uri, &[1]).await; write_dataset(&drop_uri, &[2]).await; - let cache = FlushedMemTableCache::new(8); - cache.get_or_open(&keep_uri, None).await.unwrap(); - cache.get_or_open(&drop_uri, None).await.unwrap(); + let cache = SsTableCache::new(8); + cache.get_or_open(&keep_uri, None, None).await.unwrap(); + cache.get_or_open(&drop_uri, None, None).await.unwrap(); cache.inner.run_pending_tasks().await; assert_eq!(cache.inner.entry_count(), 2); @@ -314,15 +343,15 @@ mod tests { } #[tokio::test] - async fn test_open_flushed_dataset_no_cache_matches_direct_open() { + async fn test_open_sstable_no_cache_matches_direct_open() { // The `None`/`None` path must reproduce a plain cold open: same data, // independent Arc per call (no caching). let temp_dir = tempfile::tempdir().unwrap(); let uri = format!("{}/gen_1", temp_dir.path().to_str().unwrap()); write_dataset(&uri, &[7, 8, 9]).await; - let a = open_flushed_dataset(&uri, None, None, None).await.unwrap(); - let b = open_flushed_dataset(&uri, None, None, None).await.unwrap(); + let a = open_sstable(&uri, None, None, None, None).await.unwrap(); + let b = open_sstable(&uri, None, None, None, None).await.unwrap(); assert!( !Arc::ptr_eq(&a, &b), "no-cache path must cold-open each call" @@ -330,11 +359,11 @@ mod tests { assert_eq!(a.count_rows(None).await.unwrap(), 3); // With a cache, the second call is a shared clone. - let cache: Arc = Arc::new(FlushedMemTableCache::new(8)); - let c = open_flushed_dataset(&uri, None, Some(&cache), None) + let cache: Arc = Arc::new(SsTableCache::new(8)); + let c = open_sstable(&uri, None, None, Some(&cache), None) .await .unwrap(); - let d = open_flushed_dataset(&uri, None, Some(&cache), None) + let d = open_sstable(&uri, None, None, Some(&cache), None) .await .unwrap(); assert!(Arc::ptr_eq(&c, &d), "cached path must reuse the Arc"); @@ -348,7 +377,7 @@ mod tests { } #[async_trait] - impl GenerationWarmer for NotifyingWarmer { + impl SsTableWarmer for NotifyingWarmer { async fn warm(&self, _path: &str) -> Result<()> { self.calls.fetch_add(1, Ordering::SeqCst); self.notify.notify_one(); @@ -357,7 +386,7 @@ mod tests { } #[tokio::test] - async fn test_open_flushed_dataset_fires_warm_on_open() { + async fn test_open_sstable_fires_warm_on_open() { // The warm-on-open backstop fires the warmer (fire-and-forget) when a // generation is opened, so generations the flusher never warmed still // get warmed lazily on first read. @@ -367,12 +396,12 @@ mod tests { let calls = Arc::new(AtomicUsize::new(0)); let notify = Arc::new(tokio::sync::Notify::new()); - let warmer: Arc = Arc::new(NotifyingWarmer { + let warmer: Arc = Arc::new(NotifyingWarmer { calls: calls.clone(), notify: notify.clone(), }); - let ds = open_flushed_dataset(&uri, None, None, Some(&warmer)) + let ds = open_sstable(&uri, None, None, None, Some(&warmer)) .await .unwrap(); assert_eq!(ds.count_rows(None).await.unwrap(), 3); diff --git a/rust/lance/src/dataset/mem_wal/scanner/vector_search.rs b/rust/lance/src/dataset/mem_wal/scanner/vector_search.rs index 59f721aa08c..3c314b36a75 100644 --- a/rust/lance/src/dataset/mem_wal/scanner/vector_search.rs +++ b/rust/lance/src/dataset/mem_wal/scanner/vector_search.rs @@ -28,19 +28,20 @@ use crate::io::exec::TakeExec; use super::collector::LsmDataSourceCollector; use super::data_source::LsmDataSource; -use super::flushed_cache::{DatasetCache, GenerationWarmer, open_flushed_dataset}; use super::projection::{ DISTANCE_COLUMN, build_scanner_projection, canonical_output_schema, null_columns, project_to_canonical, validate_projection_names, wants_row_id, }; +use super::sstable_cache::{DatasetCache, SsTableWarmer, open_sstable}; use crate::session::Session; +use lance_io::object_store::ObjectStoreParams; /// Plans vector search queries over LSM data. /// /// Each source is independently newest-per-PK before the union — the active /// memtable via exact brute-force KNN when PK rewrites or a filter require it /// (append-only active data can still use HNSW), -/// flushed generations via their within-generation deletion vector — and the +/// SSTables via their within-generation deletion vector — and the /// cross-generation block-list ([`super::exec::PkBlockFilterExec`]) drops any /// PK superseded by a newer generation. So each PK reaches the union from /// exactly one source and a distance-ordered merge yields the global top-k; no @@ -58,9 +59,9 @@ use crate::session::Session; /// MemTableBruteForceVectorExec or VectorIndexExec: active memtable KNN /// ProjectionExec (canonical output schema) /// ProjectionExec (null_columns _rowid) -/// PkBlockFilterExec: block-list (flushed) -/// KNNExec: flushed gen N, fetch=ceil(k*overfetch) (fast_search) -/// … one per flushed gen … +/// PkBlockFilterExec: block-list (SSTable) +/// KNNExec: SSTable gen N, fetch=ceil(k*overfetch) (fast_search) +/// … one per SSTable gen … /// ProjectionExec (canonical output schema) /// PkBlockFilterExec: block-list (base) /// KNNExec: base table, k (fast_search)[.refine()?] @@ -68,11 +69,11 @@ use crate::session::Session; /// /// # Index-Only Search (fast_search) /// -/// For base table and flushed memtables we use `fast_search()` to only +/// For base table and SSTables we use `fast_search()` to only /// search indexed data. This is correct because: -/// - Each flushed memtable has its own vector index built during flush. +/// - Each SSTable has its own vector index built during flush. /// - The active memtable covers any unindexed data. -/// - Searching unindexed data in base/flushed would be redundant. +/// - Searching unindexed data in base/SSTable would be redundant. pub struct LsmVectorSearchPlanner { /// Data source collector. collector: LsmDataSourceCollector, @@ -89,15 +90,17 @@ pub struct LsmVectorSearchPlanner { /// the per-source KNN output. Memtable rows already carry all columns; /// the take only fetches additional data for base rows (real `_rowid`). dataset: Option>, - /// Session threaded into flushed-generation opens (shared caches). + /// Session threaded into SSTable opens (shared caches). session: Option>, - /// Cache of opened flushed-generation datasets. - flushed_cache: Option>, - /// Optional warmer fired on first open of a flushed generation. - warmer: Option>, + /// Store params for opening SSTables, reusing the base dataset's store. + store_params: Option, + /// Cache of opened SSTable datasets. + sstable_cache: Option>, + /// Optional warmer fired on first open of an SSTable. + warmer: Option>, /// Optional prefilter predicate applied to every source arm before its KNN /// search, so rows failing the predicate never enter the top-k. Base and - /// flushed arms use the dataset scanner's native prefilter; memtable arms + /// SSTable arms use the dataset scanner's native prefilter; memtable arms /// route to a filtered brute-force scan. filter: Option, } @@ -127,7 +130,8 @@ impl LsmVectorSearchPlanner { distance_type, dataset: None, session: None, - flushed_cache: None, + store_params: None, + sstable_cache: None, warmer: None, filter: None, } @@ -135,28 +139,33 @@ impl LsmVectorSearchPlanner { /// Attach an optional prefilter predicate. Every source arm restricts its /// KNN to rows matching the predicate (true prefilter), so results match a - /// normal filtered vector scan over base ∪ flushed ∪ in-memory data. + /// normal filtered vector scan over base ∪ SSTables ∪ in-memory data. pub fn with_filter(mut self, filter: Option) -> Self { self.filter = filter; self } - /// Thread a session into flushed-generation opens so the first open - /// populates the shared index / file-metadata caches. + /// Set the session used to open SSTables. pub fn with_session(mut self, session: Arc) -> Self { self.session = Some(session); self } - /// Inject a cache of opened flushed-generation datasets, making repeated + /// Set the store params used to open SSTables. + pub fn with_store_params(mut self, store_params: ObjectStoreParams) -> Self { + self.store_params = Some(store_params); + self + } + + /// Inject a cache of opened SSTable datasets, making repeated /// searches against the same generation a pure `Arc::clone`. - pub fn with_flushed_cache(mut self, cache: Arc) -> Self { - self.flushed_cache = Some(cache); + pub fn with_sstable_cache(mut self, cache: Arc) -> Self { + self.sstable_cache = Some(cache); self } - /// Inject the warmer fired on first open of a flushed generation. - pub fn with_warmer(mut self, warmer: Arc) -> Self { + /// Inject the warmer fired on first open of an SSTable. + pub fn with_warmer(mut self, warmer: Arc) -> Self { self.warmer = Some(warmer); self } @@ -235,7 +244,8 @@ impl LsmVectorSearchPlanner { let block_lists = Box::pin(super::block_list::compute_source_block_lists( &sources, self.session.as_ref(), - self.flushed_cache.as_ref(), + self.store_params.as_ref(), + self.sstable_cache.as_ref(), )) .await?; @@ -296,8 +306,8 @@ impl LsmVectorSearchPlanner { // * active: append-only memtables can use HNSW directly; once a // PK rewrite is observed, `MemTableBruteForceVectorExec` drops // superseded versions before the top-k cut. - // * flushed/base: drop cross-gen superseded rows via the - // block-list (within-gen is handled by the flushed DV). + // * SSTable/base: drop cross-gen superseded rows via the + // block-list (within-gen is handled by the SSTable DV). let knn = match blocked { Some(_) if self.pk_columns.is_empty() => knn, Some(set) => Arc::new(super::exec::PkBlockFilterExec::new( @@ -442,11 +452,12 @@ impl LsmVectorSearchPlanner { } scanner.create_plan().await } - LsmDataSource::FlushedMemTable { path, .. } => { - let dataset = open_flushed_dataset( + LsmDataSource::SsTable { path, .. } => { + let dataset = open_sstable( path, self.session.as_ref(), - self.flushed_cache.as_ref(), + self.store_params.as_ref(), + self.sstable_cache.as_ref(), self.warmer.as_ref(), ) .await?; @@ -669,7 +680,7 @@ mod tests { let dataset = Dataset::write(reader, uri, Some(WriteParams::default())) .await .unwrap(); - // Also write the standalone PK sidecar (on `id`) so a flushed-generation + // Also write the standalone PK sidecar (on `id`) so an SSTable // source can be probed by the block-list (harmless for a base table). if has_id { crate::dataset::mem_wal::scanner::block_list::write_pk_sidecar(uri, &batches, &["id"]) @@ -847,7 +858,7 @@ mod tests { out_cols ); // Internal columns must not leak: `_rowid` (added by Lance's fast_search - // in the base/flushed arms) and `_memtable_gen` (added by the LSM merge + // in the base/SSTable arms) and `_memtable_gen` (added by the LSM merge // when bloom filters are present) are bookkeeping, not API. assert!( out_schema.field_with_name("_rowid").is_err(), @@ -1215,13 +1226,13 @@ mod tests { ); } - /// The flushed arm must also apply the filter as a true prefilter, and that + /// The SSTable arm must also apply the filter as a true prefilter, and that /// prefiltered candidate set must compose with cross-generation block-list /// filtering plus over-fetch. Gen 1's closest predicate-matching row (id=3) /// is superseded by gen 2; with over-fetch, gen 1 should still contribute /// the next live predicate match (id=4). #[tokio::test] - async fn test_vector_search_flushed_prefilter_composes_with_block_list() { + async fn test_vector_search_sstable_prefilter_composes_with_block_list() { use crate::dataset::mem_wal::scanner::data_source::ShardSnapshot; use crate::index::DatasetIndexExt; use crate::index::vector::VectorIndexParams; @@ -1260,8 +1271,8 @@ mod tests { let snapshot = ShardSnapshot::new(shard_id) .with_current_generation(3) - .with_flushed_generation(1, "gen_1".to_string()) - .with_flushed_generation(2, "gen_2".to_string()); + .with_sstable(1, "gen_1".to_string()) + .with_sstable(2, "gen_2".to_string()); let collector = LsmDataSourceCollector::without_base_table(base_uri, vec![snapshot]); let planner = LsmVectorSearchPlanner::new( @@ -1286,7 +1297,7 @@ mod tests { assert_eq!(rows.len(), 1, "expected one result, got {:?}", rows); assert_eq!( rows[0].0, 4, - "flushed prefilter should return live id=4 after stale id=3 is blocked; got {:?}", + "SSTable prefilter should return live id=4 after stale id=3 is blocked; got {:?}", rows ); } @@ -1996,14 +2007,14 @@ mod tests { #[tokio::test] async fn test_vector_search_dedup_across_generations() { // Regression: same primary key inserted into two sources (older - // flushed gen and newer active memtable) with different vectors. - // Without the cross-source PK dedup the older flushed row would + // SSTable gen and newer active memtable) with different vectors. + // Without the cross-source PK dedup the older SSTable row would // still appear in top-k. The newer-generation row must win. // - // We simulate a "flushed gen 1" by writing a tiny Lance dataset + // We simulate a "SSTable gen 1" by writing a tiny Lance dataset // under {base_uri}/_mem_wal/{shard}/gen_1 and pointing the // collector at it. Real flush would reverse-write, but for this - // test we only have one row in the flushed gen so order is moot. + // test we only have one row in the SSTable gen so order is moot. use crate::dataset::mem_wal::scanner::collector::{InMemoryMemTableRef, InMemoryMemTables}; use crate::dataset::mem_wal::scanner::data_source::ShardSnapshot; use crate::dataset::mem_wal::write::{BatchStore, IndexStore}; @@ -2015,7 +2026,7 @@ mod tests { let base_path = temp_dir.path().to_str().unwrap(); let base_uri = format!("{}/base", base_path); - // Flushed gen 1 holds an older version of pk=1 with a "wrong" vector. + // SSTable gen 1 holds an older version of pk=1 with a "wrong" vector. let shard_id = uuid::Uuid::new_v4(); let gen1_uri = format!("{}/_mem_wal/{}/gen_1", base_uri, shard_id); let old_pk1 = create_test_batch_with_vector(&schema, 1, [9.0, 9.0, 9.0, 9.0]); @@ -2048,7 +2059,7 @@ mod tests { let shard_snapshot = ShardSnapshot::new(shard_id) .with_current_generation(2) - .with_flushed_generation(1, "gen_1".to_string()); + .with_sstable(1, "gen_1".to_string()); let collector = LsmDataSourceCollector::without_base_table(base_uri, vec![shard_snapshot]) .with_in_memory_memtables( shard_id, @@ -2104,7 +2115,7 @@ mod tests { async fn test_vector_search_system_columns_real_only_for_base() { // Covers three properties of the per-source system columns: // 1. base-hit `_rowid`/`_rowaddr` carry real values - // 2. flushed-memtable arm runs without erroring + // 2. SSTable arm runs without erroring // 3. `_rowaddr` symmetry with `_rowid` (same code path, both are // surfaced when requested and NULL'd outside the base arm) use crate::dataset::mem_wal::scanner::collector::{InMemoryMemTableRef, InMemoryMemTables}; @@ -2131,7 +2142,7 @@ mod tests { .unwrap(); let base_dataset = Arc::new(base_dataset); - // Flushed memtable: id=2 (a separate Lance dataset under + // SSTable: id=2 (a separate Lance dataset under // {base_uri}/_mem_wal/{shard}/gen_1) with its own vector index. let shard_id = uuid::Uuid::new_v4(); let gen1_uri = format!("{}/_mem_wal/{}/gen_1", base_uri, shard_id); @@ -2163,7 +2174,7 @@ mod tests { let shard_snapshot = ShardSnapshot::new(shard_id) .with_current_generation(2) - .with_flushed_generation(1, "gen_1".to_string()); + .with_sstable(1, "gen_1".to_string()); let collector = LsmDataSourceCollector::new(base_dataset, vec![shard_snapshot]) .with_in_memory_memtables( @@ -2237,10 +2248,10 @@ mod tests { "`_rowaddr` is incompatible with vector_search's fast_search; must be NULL" ); - // id=2 (flushed): both NULL — per-source values would collide with base. - let (rid_null, raddr_null) = seen.get(&2).expect("flushed row id=2 missing"); - assert!(rid_null, "flushed row `_rowid` must be NULL"); - assert!(raddr_null, "flushed row `_rowaddr` must be NULL"); + // id=2 (SSTable): both NULL — per-source values would collide with base. + let (rid_null, raddr_null) = seen.get(&2).expect("SSTable row id=2 missing"); + assert!(rid_null, "SSTable row `_rowid` must be NULL"); + assert!(raddr_null, "SSTable row `_rowaddr` must be NULL"); // id=3 (active): both NULL — BatchStore position is not a Lance row id. let (rid_null, raddr_null) = seen.get(&3).expect("active row id=3 missing"); @@ -2909,9 +2920,9 @@ mod tests { } #[tokio::test] - async fn test_vector_search_flushed_superseded_by_newer_flushed() { - // An older flushed generation's stale row must be suppressed by a newer - // flushed generation (cross-flushed blocking, no base/active involved). + async fn test_vector_search_sstable_superseded_by_newer_sstable() { + // An older SSTable's stale row must be suppressed by a newer + // SSTable (cross-SSTable blocking, no base/active involved). use crate::dataset::mem_wal::scanner::data_source::ShardSnapshot; use crate::index::DatasetIndexExt; use crate::index::vector::VectorIndexParams; @@ -2946,8 +2957,8 @@ mod tests { let snapshot = ShardSnapshot::new(shard_id) .with_current_generation(3) - .with_flushed_generation(1, "gen_1".to_string()) - .with_flushed_generation(2, "gen_2".to_string()); + .with_sstable(1, "gen_1".to_string()) + .with_sstable(2, "gen_2".to_string()); let collector = LsmDataSourceCollector::without_base_table(base_uri, vec![snapshot]); let planner = LsmVectorSearchPlanner::new( diff --git a/rust/lance/src/dataset/mem_wal/test_util.rs b/rust/lance/src/dataset/mem_wal/test_util.rs index 43a3861e686..48e68340d3c 100644 --- a/rust/lance/src/dataset/mem_wal/test_util.rs +++ b/rust/lance/src/dataset/mem_wal/test_util.rs @@ -1,12 +1,14 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright The Lance Authors -//! Test-only object store that injects WAL-write failures, for exercising the -//! WAL persistence-failure fencing path. +//! Test-only object store that injects WAL-write failures (for the WAL +//! persistence-failure fencing path) and records the paths it serves (for +//! asserting which opens actually resolved through a given `ObjectStoreParams`). use std::fmt::{Debug, Display, Formatter}; use std::ops::Range; use std::sync::Arc; +use std::sync::Mutex as StdMutex; use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; use bytes::Bytes; @@ -33,6 +35,10 @@ pub struct FailControls { simulate_lost_ack: AtomicBool, /// WAL-entry `put_opts` attempts observed, for assertions. wal_put_attempts: AtomicUsize, + /// Every location written through this store. + put_paths: StdMutex>, + /// Every location read through this store. + get_paths: StdMutex>, } impl FailControls { @@ -48,6 +54,26 @@ impl FailControls { pub fn attempts(&self) -> usize { self.wal_put_attempts.load(Ordering::SeqCst) } + + /// Did any write land on a path containing `needle`? An open that resolved + /// its store from other params never reaches this store, so a `false` here + /// means the params under test did not reach that open. + pub fn wrote_under(&self, needle: &str) -> bool { + self.put_paths + .lock() + .unwrap() + .iter() + .any(|p| p.contains(needle)) + } + + /// Did any read land on a path containing `needle`? See [`Self::wrote_under`]. + pub fn read_under(&self, needle: &str) -> bool { + self.get_paths + .lock() + .unwrap() + .iter() + .any(|p| p.contains(needle)) + } } /// Wraps the inner store with [`FailingObjectStore`] at construction. @@ -101,6 +127,11 @@ impl OSObjectStore for FailingObjectStore { payload: PutPayload, opts: PutOptions, ) -> OSResult { + self.controls + .put_paths + .lock() + .unwrap() + .push(location.to_string()); if Self::is_wal_entry(location) { self.controls .wal_put_attempts @@ -124,14 +155,30 @@ impl OSObjectStore for FailingObjectStore { location: &Path, opts: PutMultipartOptions, ) -> OSResult> { + // Data files (`*.lance`) are written multipart, not via `put_opts`. + self.controls + .put_paths + .lock() + .unwrap() + .push(location.to_string()); self.inner.put_multipart_opts(location, opts).await } async fn get_opts(&self, location: &Path, options: GetOptions) -> OSResult { + self.controls + .get_paths + .lock() + .unwrap() + .push(location.to_string()); self.inner.get_opts(location, options).await } async fn get_ranges(&self, location: &Path, ranges: &[Range]) -> OSResult> { + self.controls + .get_paths + .lock() + .unwrap() + .push(location.to_string()); self.inner.get_ranges(location, ranges).await } @@ -167,9 +214,11 @@ impl OSObjectStore for FailingObjectStore { } } -/// Build an in-memory `ObjectStore` whose WAL-entry writes can be failed on -/// demand. Returns the store, its base path, and the shared controls. -pub async fn failing_memory_store() -> (Arc, Path, Arc) { +/// `ObjectStoreParams` carrying the observable store wrapper, plus the controls +/// to drive and inspect it. Open a dataset with these and every store resolved +/// *from these params* — the base and any derived URI they are threaded to — +/// reports its traffic back through the controls. +pub fn observable_store_params() -> (ObjectStoreParams, Arc) { let controls = Arc::new(FailControls::default()); let params = ObjectStoreParams { object_store_wrapper: Some(Arc::new(FailingWrapper { @@ -177,6 +226,13 @@ pub async fn failing_memory_store() -> (Arc, Path, Arc (Arc, Path, Arc) { + let (params, controls) = observable_store_params(); let (store, base) = ObjectStore::from_uri_and_params( Arc::new(ObjectStoreRegistry::default()), "memory:///", diff --git a/rust/lance/src/dataset/mem_wal/util.rs b/rust/lance/src/dataset/mem_wal/util.rs index 3f5090f6b40..a51f269cb67 100644 --- a/rust/lance/src/dataset/mem_wal/util.rs +++ b/rust/lance/src/dataset/mem_wal/util.rs @@ -3,6 +3,7 @@ //! Utility functions for MemWAL operations. +use lance_io::object_store::ObjectStoreParams; use object_store::path::Path; use uuid::Uuid; @@ -91,7 +92,8 @@ impl WatchableOnceCellReader { /// optimizing S3 throughput by spreading sequential writes across internal partitions. /// /// # Example -/// ```ignore +/// ``` +/// # use lance::dataset::mem_wal::util::bit_reverse_u64; /// // 5 in binary: 000...101 /// // Reversed: 101...000 /// assert_eq!(bit_reverse_u64(5), 0xa000000000000000); @@ -129,6 +131,26 @@ pub fn parse_bit_reversed_filename(filename: &str) -> Option { Some(bit_reverse_u64(reversed)) } +/// Adapt the store params a base dataset was opened with for use on a URI +/// *derived* from it (an SSTable under `_mem_wal/`). +/// +/// The deprecated `object_store` binding pins a store to one location: given +/// `Some((store, url))`, both `ObjectStore::from_uri_and_params` and +/// `DatasetBuilder::build_object_store` take the path from `url` and ignore the +/// URI they were asked to open. Carried onto a generation URI it would silently +/// redirect the open — and, on the flush path, the write — at the base table +/// itself. Drop it so the generation URI resolves its own store; everything +/// else (storage options, wrapper, credentials, block size) still carries over. +/// +/// Only the base's *own* URI may reuse the params verbatim. +pub(crate) fn derived_store_params(params: &ObjectStoreParams) -> ObjectStoreParams { + #[allow(deprecated)] + ObjectStoreParams { + object_store: None, + ..params.clone() + } +} + /// Path to the MemWAL root directory. /// /// Returns: `{base_path}/_mem_wal/` @@ -157,29 +179,24 @@ pub fn shard_manifest_path(base_path: &Path, shard_id: &Uuid) -> Path { shard_base_path(base_path, shard_id).join("manifest") } -/// Path to a flushed MemTable directory. +/// Path to an SSTable directory. /// /// Returns: `{base_path}/_mem_wal/{shard_id}/{random_hash}_gen_{generation}/` -pub fn flushed_memtable_path( - base_path: &Path, - shard_id: &Uuid, - random_hash: &str, - generation: u64, -) -> Path { +pub fn sstable_path(base_path: &Path, shard_id: &Uuid, random_hash: &str, generation: u64) -> Path { shard_base_path(base_path, shard_id).join(format!("{}_gen_{}", random_hash, generation)) } -/// Subdirectory of a flushed generation holding its standalone primary-key +/// Subdirectory of an SSTable holding its standalone primary-key /// dedup index (a sidecar BTree, not registered in the manifest). Both the /// flush writer and the block-list probe join this onto the generation path. pub const PK_INDEX_DIR: &str = "_pk_index"; -/// Path to a flushed generation's standalone primary-key dedup index. +/// Path to an SSTable's standalone primary-key dedup index. pub fn pk_index_path(gen_path: &Path) -> Path { gen_path.clone().join(PK_INDEX_DIR) } -/// Generate an 8-character random hex string for flushed MemTable directories. +/// Generate an 8-character random hex string for SSTable directories. pub fn generate_random_hash() -> String { let bytes: [u8; 4] = rand::random(); format!( @@ -288,7 +305,7 @@ mod tests { ); assert_eq!( - flushed_memtable_path(&base_path, &shard_id, "a1b2c3d4", 5).as_ref(), + sstable_path(&base_path, &shard_id, "a1b2c3d4", 5).as_ref(), "my/dataset/_mem_wal/550e8400-e29b-41d4-a716-446655440000/a1b2c3d4_gen_5" ); @@ -372,4 +389,40 @@ mod tests { drop(cell); assert_eq!(handle.await.unwrap(), None); } + + /// The path-bound store binding is the only thing dropped — credentials and + /// storage options must still reach the generation's store. + #[test] + fn test_derived_store_params_drops_only_the_path_bound_store() { + let accessor = lance_io::object_store::StorageOptionsAccessor::with_static_options( + std::collections::HashMap::from([("access_key_id".to_string(), "key".to_string())]), + ); + #[allow(deprecated)] + let params = ObjectStoreParams { + object_store: Some(( + std::sync::Arc::new(object_store::memory::InMemory::new()), + url::Url::parse("memory:///base").unwrap(), + )), + block_size: Some(1234), + storage_options_accessor: Some(std::sync::Arc::new(accessor)), + ..Default::default() + }; + + let derived = derived_store_params(¶ms); + + #[allow(deprecated)] + { + assert!( + derived.object_store.is_none(), + "a store pinned to the base path must not be reused for a generation URI" + ); + } + assert_eq!(derived.block_size, Some(1234)); + assert_eq!( + derived + .storage_options() + .and_then(|o| o.get("access_key_id")), + Some(&"key".to_string()), + ); + } } diff --git a/rust/lance/src/dataset/mem_wal/wal.rs b/rust/lance/src/dataset/mem_wal/wal.rs index b4caff180e1..68eb41a03c2 100644 --- a/rust/lance/src/dataset/mem_wal/wal.rs +++ b/rust/lance/src/dataset/mem_wal/wal.rs @@ -9,7 +9,8 @@ use std::io::Cursor; use std::sync::Arc; use std::sync::Mutex as StdMutex; -use std::sync::atomic::{AtomicU64, Ordering}; +use std::sync::MutexGuard as StdMutexGuard; +use std::sync::atomic::{AtomicU64, AtomicUsize, Ordering}; use std::time::Instant; use std::time::Duration; @@ -87,68 +88,200 @@ impl WalFlushFailure { } } -/// Watcher for batch durability using watermark-based tracking. +/// The writer's cursors, shared by the WAL-append task, the index-apply task, +/// every memtable's `IndexStore`, and every `put` waiting to become visible. /// -/// Uses a shared watch channel that broadcasts the durable watermark. -/// The watcher waits until the watermark reaches or exceeds its target batch ID. -#[derive(Clone)] -pub struct BatchDurableWatcher { - /// Watch receiver for the durable watermark. - rx: watch::Receiver, - /// Target batch ID to wait for. - target_batch_position: usize, - /// Terminal flush failure shared with the flusher. When set, the watermark - /// can never reach the target, so `wait` returns this typed error instead of - /// blocking forever. +/// Two cursors are stored, one view is derived: +/// +/// - `durable` — writer-global count of WAL-durable batches, advanced by the +/// WAL-append task. Exclusive; 0 means none. +/// - `indexed` — per-memtable count of indexed batches, advanced by the +/// index-apply task. It lives on the memtable's own `IndexStore`, not here, +/// because each memtable has its own indexes. +/// - `visible` — **derived, never stored**: a batch is visible once it is +/// indexed and, under `durable_write`, also durable. +/// +/// Deriving `visible` rather than caching it is deliberate. A cached +/// `min(indexed, durable)` recomputed by two independent tasks is the classic +/// store-buffer race: with `Release`/`Acquire` each task can read the other's +/// *pre-store* value, so both compute a minimum below the true one, and a +/// max-clamped publish then leaves the cached value permanently short. A `put` +/// blocked on it would hang until some unrelated write happened to move a cursor +/// again. With nothing cached there is nothing to leave stale — `notify` is a +/// bare wake-up and every waiter recomputes from the cursors themselves. +pub struct WriterCursors { + durable: AtomicUsize, + /// Bumped whenever a cursor advances or the writer poisons. Carries no + /// value; it exists only to wake waiters, which then recompute. + notify_tx: watch::Sender, + notify_rx: watch::Receiver, + /// First terminal failure. Shared so a poisoned writer wakes every waiter + /// with the typed error, instead of leaving it blocked on a cursor that can + /// never advance again. terminal_error: Arc>>, + /// Whether durability is part of visibility. Per-writer, not per-write: + /// `visible` is a writer-wide definition, so a mix would need two visibility + /// views over one memtable. + durable_write: bool, +} + +impl WriterCursors { + pub fn new(durable_write: bool) -> Self { + let (notify_tx, notify_rx) = watch::channel(0); + Self { + durable: AtomicUsize::new(0), + notify_tx, + notify_rx, + terminal_error: Arc::new(StdMutex::new(None)), + durable_write, + } + } + + /// Writer-global count of WAL-durable batches. + pub fn durable(&self) -> usize { + self.durable.load(Ordering::Acquire) + } + + pub fn durable_write(&self) -> bool { + self.durable_write + } + + /// Advance the durability cursor. Monotonic, so an out-of-order completion + /// can never walk it backwards. + pub(crate) fn advance_durable(&self, global_count: usize) { + self.durable.fetch_max(global_count, Ordering::AcqRel); + self.wake(); + } + + /// Wake every waiter so it recomputes. Called after any cursor advances, and + /// after the writer poisons. + pub(crate) fn wake(&self) { + self.notify_tx.send_modify(|version| *version += 1); + } + + /// The visible prefix of one memtable, given its indexed prefix and its + /// writer-global coordinate. + pub fn visible_count(&self, indexed_count: usize, global_offset: usize) -> usize { + if !self.durable_write { + return indexed_count; + } + indexed_count.min(self.durable().saturating_sub(global_offset)) + } + + /// Lock `terminal_error`, ignoring mutex poisoning. + /// + /// A panic under this lock cannot tear the `Option` it guards: the sole + /// writer builds the value first and assigns it whole, so a panic mid-section + /// leaves the slot exactly as it was. Surfacing poison as an error instead + /// would mask the latched `FenceReason` behind an unrelated "mutex poisoned" + /// precisely when a caller needs the real reason — and would strand + /// `mark_terminal_failure`, which has no error to return. + fn lock_terminal_error(&self) -> StdMutexGuard<'_, Option> { + self.terminal_error + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()) + } + + pub(crate) fn check_poisoned(&self) -> Result<()> { + if let Some(failure) = self.lock_terminal_error().clone() { + return Err(failure.into_error()); + } + Ok(()) + } + + pub(crate) fn mark_terminal_failure(&self, error: &Error) { + { + let mut slot = self.lock_terminal_error(); + if slot.is_none() { + *slot = Some(WalFlushFailure::from_error(error)); + } + } + // Wake waiters without advancing anything: each re-checks `terminal_error` + // and returns the error rather than blocking on a cursor that can no + // longer move. + self.wake(); + } +} + +/// Blocks a `put` until its batches are visible: indexed, and — in durable mode +/// — WAL-durable too. +/// +/// Recomputes the condition on every wake instead of comparing against a cached +/// watermark. `notify` only says "something moved"; this decides what that means. +pub struct BatchDurableWatcher { + cursors: Arc, + rx: watch::Receiver, + /// The memtable the batches landed in; its `indexed_count` is the apply + /// cursor being waited on. `None` in WAL-only mode, which has no indexes. + indexes: Option>, + /// Local exclusive count this write needs indexed. + target_indexed: usize, + /// Writer-global exclusive count this write needs durable. Batch positions + /// restart at 0 in every memtable while the durability cursor spans the + /// writer's whole life, so the caller must globalize this. + target_durable: usize, } impl BatchDurableWatcher { - /// Create a new watcher for a specific batch ID. pub fn new( - rx: watch::Receiver, - target_batch_position: usize, - terminal_error: Arc>>, + cursors: Arc, + indexes: Option>, + target_indexed: usize, + target_durable: usize, ) -> Self { + let rx = cursors.notify_rx.clone(); Self { + cursors, rx, - target_batch_position, - terminal_error, + indexes, + target_indexed, + target_durable, } } - /// Wait until the batch is durable. - /// - /// Returns Ok(()) when `durable_watermark >= target_batch_position`, or - /// Err if a terminal flush failure (e.g. a fence) means the watermark can - /// never reach the target. + /// Whether the write is readable yet. + fn is_visible(&self) -> bool { + // WAL-only mode has no indexes, so there is nothing to index-wait on. + let indexed = match &self.indexes { + Some(indexes) => indexes.indexed_count(), + None => self.target_indexed, + }; + if indexed < self.target_indexed { + return false; + } + !self.cursors.durable_write() || self.cursors.durable() >= self.target_durable + } + + /// Wait until the write is visible, or until the writer poisons — in which + /// case no cursor will ever reach the target, so surface the typed error + /// rather than blocking forever. pub async fn wait(&mut self) -> Result<()> { loop { - if let Some(failure) = self.terminal_error.lock().unwrap().clone() { - return Err(failure.into_error()); - } - let current = *self.rx.borrow(); - if current >= self.target_batch_position { + // Mark the current version seen *before* testing, so a wake-up landing + // between the test and `changed()` below is not lost. + self.rx.borrow_and_update(); + self.cursors.check_poisoned()?; + if self.is_visible() { return Ok(()); } self.rx .changed() .await - .map_err(|_| Error::io("Durable watermark channel closed"))?; + .map_err(|_| Error::io("Writer cursor channel closed"))?; } } - /// Check if the batch is already durable (non-blocking). + /// Non-blocking check. pub fn is_durable(&self) -> bool { - *self.rx.borrow() >= self.target_batch_position + self.is_visible() } } impl std::fmt::Debug for BatchDurableWatcher { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { f.debug_struct("BatchDurableWatcher") - .field("target_batch_position", &self.target_batch_position) - .field("current_watermark", &*self.rx.borrow()) + .field("target_indexed", &self.target_indexed) + .field("target_durable", &self.target_durable) .finish() } } @@ -164,20 +297,15 @@ pub struct WalEntry { pub num_batches: usize, } -/// Result of a parallel WAL flush with index update. +/// Result of a WAL flush. Append-only: index application runs on its own task +/// (see `apply_index_range`), which records its own stats, so this no longer +/// carries index-update timing or row counts. #[derive(Debug, Clone)] pub struct WalFlushResult { /// WAL entry that was written (if any). pub entry: Option, /// Duration of WAL I/O operation. pub wal_io_duration: std::time::Duration, - /// Overall wall-clock duration of the index update operation. - /// This includes any overhead from thread scheduling and context switching. - pub index_update_duration: std::time::Duration, - /// Per-index update durations. Key is index name, value is duration. - pub index_update_duration_breakdown: std::collections::HashMap, - /// Number of rows indexed. - pub rows_indexed: usize, /// Size of WAL data written in bytes. pub wal_bytes: usize, } @@ -185,32 +313,142 @@ pub struct WalFlushResult { /// Source for a WAL flush — either a `BatchStore` range (MemTable mode) or /// a drainable in-memory pending queue (WAL-only mode). pub enum WalFlushSource { - /// MemTable mode: read a `[max_flushed+1, end_batch_position)` range - /// from a `BatchStore`. Indexes are updated in parallel with the WAL - /// append. - BatchStore { - batch_store: Arc, - indexes: Option>, - }, + /// MemTable mode: append the `[durable, end_batch_position)` range of a + /// `BatchStore` to the WAL. Append-only — the index apply runs on its own + /// task, so a failed append can no longer publish rows through it. + BatchStore { batch_store: Arc }, + /// Timer-driven: append whichever store still owes the WAL an append, + /// resolved when the message is *handled*, not when it is enqueued. + /// + /// Carries no store on purpose. `MessageFactory` is synchronous and cannot + /// take the async state lock, and capturing an `Arc` once at + /// handler construction would pin the first memtable forever. + /// + /// Resolution walks the live stores **oldest first** and takes the first with + /// `global_end() > durable`. It must not simply take "the active memtable": + /// a tick enqueued before a freeze is handled *after* it, would resolve to + /// the new memtable, and would append its batches ahead of the outgoing + /// memtable's tail. WAL entry positions are assigned in append-call order, + /// replay walks them ascending, row positions follow, and primary-key recency + /// is "newest visible row position wins" — so an out-of-order append silently + /// inverts dedup after a crash: the stale row wins. A full scan looks fine. + NextPending, /// WAL-only mode: drain all pending batches from the shared /// `WalOnlyState`. There are no in-memory indexes to update. WalOnly { state: Arc }, } impl WalFlushSource { - fn pending_count(&self) -> usize { + fn kind(&self) -> &'static str { match self { - Self::BatchStore { batch_store, .. } => batch_store.pending_wal_flush_count(), - Self::WalOnly { state } => state - .pending - .lock() - .ok() - .map(|p| p.batches.len()) - .unwrap_or(0), + Self::BatchStore { .. } => "BatchStore", + Self::WalOnly { .. } => "WalOnly", + Self::NextPending => "NextPending", } } } +/// Message to trigger an index apply. +/// +/// Carries the store the batches actually landed in, captured on the put path +/// *before* any freeze can rotate the memtable — pairing a new store with an old +/// store's end position would index the wrong range. +#[derive(Clone)] +pub struct TriggerIndexApply { + pub batch_store: Arc, + pub indexes: Arc, + /// Local exclusive end of the range to cover. + pub end_batch_position: usize, +} + +impl std::fmt::Debug for TriggerIndexApply { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("TriggerIndexApply") + .field("end_batch_position", &self.end_batch_position) + .finish() + } +} + +/// What an `apply_index_range` call actually indexed. `rows_indexed == 0` +/// marks a routine coalesced no-op (the range was already covered) that must +/// not be recorded as an index update. +#[derive(Debug, Default, Clone, Copy)] +pub struct IndexApplyStats { + pub rows_indexed: usize, + /// Wall-clock time spent applying the range, including the thread-scheduling + /// overhead of the blocking hand-off. + pub duration: std::time::Duration, +} + +/// Apply a contiguous range of batches to a memtable's in-memory indexes. +/// +/// Runs on its own task, as the single sequential consumer of its own channel. +/// Being a single consumer is what makes it safe: it guarantees in-order, +/// contiguous ranges, which is exactly what `HnswGraph::insert_batch` requires — +/// it hard-rejects any range whose start is not `indexed_len`. Ordering comes +/// from the task, not from the flush interval, so triggering per-put is exactly +/// as safe as triggering on a timer. +/// +/// It has its own channel rather than sharing the WAL flusher's, because +/// `TaskDispatcher::run` awaits `handle()` inline: a shared channel would put a +/// ~100ms S3 PUT in front of every latency-sensitive index apply. +pub async fn apply_index_range( + cursors: &Arc, + message: TriggerIndexApply, +) -> Result { + let TriggerIndexApply { + batch_store, + indexes, + end_batch_position, + } = message; + + // Self-batching: a message handled while more puts queue behind it covers + // everything committed so far, so the ones behind it find their range already + // applied. Redundant messages are therefore *routine*, not exceptional, and + // must be a clean no-op — never a call into `insert_batches`, whose HNSW arm + // hard-rejects a non-contiguous start, and which is now terminal for the + // writer. + let start = indexes.indexed_count(); + if end_batch_position <= start { + return Ok(IndexApplyStats::default()); + } + + // Every position in the range must exist. `get` returns `None` only for a + // position past `committed_len`, so a hole means the caller asked to index a + // batch the store never committed. Silently skipping it would let + // `insert_batches` advance `indexed_count` past a never-indexed batch — + // rows counted visible but absent from every index. Fail loudly; the handler + // poisons and reopen rebuilds the indexes from the WAL. + let stored: Vec = (start..end_batch_position) + .map(|position| { + batch_store.get(position).cloned().ok_or_else(|| { + Error::internal(format!( + "index apply range [{start}, {end_batch_position}) is missing batch \ + position {position}; batch_store committed_len is {}", + batch_store.len() + )) + }) + }) + .collect::>()?; + + // `insert_batches` advances `indexed_count` itself, once every index has + // taken the batch. Time the whole hand-off so the recorded latency reflects + // the blocking-pool scheduling too, matching the old inline-flush measurement. + let rows_indexed: usize = stored.iter().map(|b| b.num_rows).sum(); + let apply_start = Instant::now(); + tokio::task::spawn_blocking(move || indexes.insert_batches(&stored)) + .await + .map_err(|e| Error::internal(format!("Index apply task panicked: {e}")))??; + let duration = apply_start.elapsed(); + + // Wake anything waiting to become visible. + cursors.wake(); + Ok(IndexApplyStats { + rows_indexed, + duration, + }) +} + /// Message to trigger a WAL flush. /// /// Carries a `source` describing where to read batches from (BatchStore range @@ -218,7 +456,7 @@ impl WalFlushSource { pub struct TriggerWalFlush { pub source: WalFlushSource, /// End batch position (exclusive). For `BatchStore`, flush batches after - /// `max_flushed_batch_position` up to this. For `WalOnly`, indicates the + /// the writer-global durable cursor up to this. For `WalOnly`, indicates the /// position the durability watermark must reach for callers waiting on /// this flush. Use `usize::MAX` to flush all pending batches. pub end_batch_position: usize, @@ -232,7 +470,7 @@ pub struct TriggerWalFlush { impl std::fmt::Debug for TriggerWalFlush { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { f.debug_struct("TriggerWalFlush") - .field("pending_batches", &self.source.pending_count()) + .field("source", &self.source.kind()) .field("end_batch_position", &self.end_batch_position) .finish() } @@ -361,11 +599,9 @@ impl WalOnlyState { /// shared `WalAppender`. The flusher delegates the actual WAL write to the /// appender, optionally running a parallel index update in MemTable mode. pub struct WalFlusher { - /// Watch channel sender for durable watermark. - /// Broadcasts the highest batch_position that is now durable. - durable_watermark_tx: watch::Sender, - /// Watch channel receiver for creating new watchers. - durable_watermark_rx: watch::Receiver, + /// The writer's cursors. Shared with the index-apply task and every + /// memtable's `IndexStore`, so all three agree on what is visible. + cursors: Arc, /// Underlying WAL append primitive — owns object store, epoch, and /// position discovery. wal_appender: Arc, @@ -377,35 +613,37 @@ pub struct WalFlusher { /// Created at construction and recreated after each flush. /// Used by backpressure to wait for WAL flushes. wal_flush_cell: std::sync::Mutex>>, - /// First terminal flush failure, shared with every `BatchDurableWatcher`. It - /// wakes durability waiters (the watermark never advances) and is read by - /// `check_poisoned` so the write path fails fast. - terminal_error: Arc>>, } impl WalFlusher { /// Create a new WAL flusher backed by an existing `WalAppender`. /// /// The appender owns object store, epoch, and position state. The - /// flusher adds the durability watermark, trigger channel, and - /// completion cell on top. + /// flusher adds the trigger channel and completion cell on top, and shares + /// the writer's cursors. pub fn new(wal_appender: Arc) -> Self { + // Defaults to durable visibility; the writer replaces this with cursors + // built from its own config. + Self::with_cursors(wal_appender, Arc::new(WriterCursors::new(true))) + } + + pub fn with_cursors(wal_appender: Arc, cursors: Arc) -> Self { let shard_id = wal_appender.shard_id(); - // Initialize durable watermark at 0 (no batches durable yet) - let (durable_watermark_tx, durable_watermark_rx) = watch::channel(0); - // Create initial WAL flush cell for backpressure let wal_flush_cell = WatchableOnceCell::new(); Self { - durable_watermark_tx, - durable_watermark_rx, + cursors, wal_appender, shard_id, flush_tx: None, wal_flush_cell: std::sync::Mutex::new(Some(wal_flush_cell)), - terminal_error: Arc::new(StdMutex::new(None)), } } + /// The writer's cursors, for the index-apply task and the memtables. + pub fn cursors(&self) -> &Arc { + &self.cursors + } + /// Set the flush channel for background flush handler. pub fn set_flush_channel(&mut self, tx: mpsc::UnboundedSender) { self.flush_tx = Some(tx); @@ -420,45 +658,65 @@ impl WalFlusher { /// /// Returns a `BatchDurableWatcher` that can be awaited for durability. /// The actual batch data is stored in the BatchStore. - pub fn track_batch(&self, batch_position: usize) -> BatchDurableWatcher { - // Return a watcher that waits for this batch to become durable - // batch_position is 0-indexed, so we wait for watermark > batch_position (i.e., >= batch_position + 1) + /// Watch a write until it becomes visible. + /// + /// `target_indexed` is a **memtable-local** exclusive count; `target_durable` + /// is a **writer-global** one. They are different coordinate spaces on + /// purpose: batch positions restart at 0 in every memtable, while the + /// durability cursor spans the writer's whole life. A local durable target + /// would already be satisfied by a *previous* memtable's appends, and would + /// ack a write that never reached the WAL. Callers globalize via + /// `BatchStore::global_offset`. + pub fn track_batch( + &self, + indexes: Option>, + target_indexed: usize, + target_durable: usize, + ) -> BatchDurableWatcher { BatchDurableWatcher::new( - self.durable_watermark_rx.clone(), - batch_position + 1, - Arc::clone(&self.terminal_error), + Arc::clone(&self.cursors), + indexes, + target_indexed, + target_durable, ) } - /// Latch a terminal flush failure and wake every durability waiter (the - /// watermark never advances, so they must observe the error, not block). + /// The writer-global WAL durability cursor: how many batches of this + /// writer's batch sequence are durable. Exclusive count; 0 means none. + pub fn durable(&self) -> usize { + self.cursors.durable() + } + + /// Advance the durability cursor and wake waiters. + pub(crate) fn advance_durable(&self, global_count: usize) { + self.cursors.advance_durable(global_count); + } + + /// Latch a terminal flush failure and wake every waiter (no cursor will + /// advance again, so they must observe the error rather than block). /// Idempotent: only the first failure is retained. fn mark_terminal_failure(&self, error: &Error) { - { - let mut slot = self.terminal_error.lock().unwrap(); - if slot.is_none() { - *slot = Some(WalFlushFailure::from_error(error)); - } - } - // Wake `wait`ers without advancing the watermark; each re-checks - // `terminal_error` and returns the error. - self.durable_watermark_tx.send_modify(|_| {}); + self.cursors.mark_terminal_failure(error); + } + + /// Latch a terminal failure from outside the flush path (the index-apply + /// task). Same effect: reads and writes fail fast, waiters wake with the + /// typed error, and recovery is reopen -> replay. + pub(crate) fn poison(&self, error: &Error) { + self.cursors.mark_terminal_failure(error); } /// Fail fast with the typed error if this writer has been fenced (by a peer - /// or its own persistence failure). The write path calls this before touching - /// the memtable so a poisoned writer can't diverge further. Recovery is to - /// reopen the shard (replay the WAL). + /// or its own persistence failure). Both the read and write paths call this + /// so a poisoned writer can neither diverge further nor serve a snapshot + /// that replay will not reproduce. Recovery is to reopen and replay. pub fn check_poisoned(&self) -> Result<()> { - if let Some(failure) = self.terminal_error.lock().unwrap().clone() { - return Err(failure.into_error()); - } - Ok(()) + self.cursors.check_poisoned() } /// Get the current durable watermark. pub fn durable_watermark(&self) -> usize { - *self.durable_watermark_rx.borrow() + self.cursors.durable() } /// Get a watcher for WAL flush completion. @@ -527,14 +785,16 @@ impl WalFlusher { end_batch_position: usize, ) -> Result { let result = match source { - WalFlushSource::BatchStore { - batch_store, - indexes, - } => { - self.flush_from_batch_store(batch_store, indexes.clone(), end_batch_position) + WalFlushSource::BatchStore { batch_store } => { + self.flush_from_batch_store(batch_store, end_batch_position) .await } WalFlushSource::WalOnly { state } => self.flush_from_wal_only(state).await, + // The handler resolves a tick to a concrete store before it gets + // here, because only it can take the async state lock. + WalFlushSource::NextPending => Err(Error::internal( + "WalFlushSource::NextPending must be resolved by the flush handler", + )), }; // A terminal failure means the watermark can never advance; latch the // poison so waiters wake with the typed error and later writes fail fast. @@ -546,80 +806,64 @@ impl WalFlusher { result } + /// Append this store's un-appended suffix to the WAL. **Append-only.** + /// + /// The index apply used to run here, concurrently, under a `tokio::join!`. + /// That was the source of the dirty read: `join!` runs both arms to + /// completion and does not cancel the index arm when the append fails, so a + /// failed append still advanced the cursor readers keyed off. The two + /// operations have nothing in common — one is an in-memory microsecond write, + /// the other a ~100ms S3 PUT billed per call — so they now run on separate + /// tasks with separate cursors, and this one only ever touches the WAL. async fn flush_from_batch_store( &self, batch_store: &BatchStore, - indexes: Option>, end_batch_position: usize, ) -> Result { - // Get current flush position from per-memtable watermark (inclusive) - // start_batch_position is the first batch to flush - let start_batch_position = batch_store - .max_flushed_batch_position() - .map(|w| w + 1) - .unwrap_or(0); - - // If we've already flushed past this end, nothing to do + // Where this store's un-appended suffix begins, derived from the + // writer-global durability cursor. `local_end` clamps a cursor that + // predates this memtable to 0 and one past its end to `committed_len`. + let start_batch_position = batch_store.local_end(self.durable()); + + // Already appended past this end: nothing to do. Redundant triggers are + // routine (a put and the freeze can both target the same range), so this + // is the common case, not an error. if start_batch_position >= end_batch_position { return Ok(empty_flush_result()); } - // Collect batches in range [start_batch_position, end_batch_position) - let mut stored_batches: Vec = - Vec::with_capacity(end_batch_position - start_batch_position); - - for batch_position in start_batch_position..end_batch_position { - if let Some(stored) = batch_store.get(batch_position) { - stored_batches.push(stored.clone()); - } - } - - if stored_batches.is_empty() { - return Ok(empty_flush_result()); - } + // Every position in the range must exist. `get` returns `None` only for a + // position past `committed_len`, so a hole means we were asked to append a + // batch the store never committed. Silently skipping it while still + // advancing durability to `end_batch_position` (below) would mark an + // un-appended batch durable and lose it on replay — a divergence that + // survives the crash and hides from a full scan. Return a terminal error + // so `flush` poisons the writer; reopen replays the WAL. + let stored_batches: Vec = (start_batch_position..end_batch_position) + .map(|batch_position| { + batch_store.get(batch_position).cloned().ok_or_else(|| { + Error::writer_poisoned(format!( + "WAL flush range [{start_batch_position}, {end_batch_position}) is \ + missing batch position {batch_position}; batch_store committed_len is {}", + batch_store.len() + )) + }) + }) + .collect::>()?; - let rows_to_index: usize = stored_batches.iter().map(|b| b.num_rows).sum(); let record_batches: Vec = stored_batches.iter().map(|s| s.data.clone()).collect(); - let appender = self.wal_appender.clone(); - let (append_result, index_result) = if let Some(idx_registry) = indexes { - let wal_future = async move { - let start = Instant::now(); - let r = appender.append(record_batches).await?; - Ok::<_, Error>((r, start.elapsed())) - }; - let index_future = async { - let start = Instant::now(); - let per_index = tokio::task::spawn_blocking(move || { - idx_registry.insert_batches_parallel(&stored_batches) - }) - .await - .map_err(|e| Error::internal(format!("Index update task panicked: {}", e)))??; - Ok::<_, Error>((start.elapsed(), per_index)) - }; - tokio::join!(wal_future, index_future) - } else { - let wal_future = async move { - let start = Instant::now(); - let r = appender.append(record_batches).await?; - Ok::<_, Error>((r, start.elapsed())) - }; - ( - wal_future.await, - Ok((std::time::Duration::ZERO, std::collections::HashMap::new())), - ) - }; - - let (append_result, wal_io_duration) = append_result?; - let (index_update_duration, index_update_duration_breakdown) = index_result?; - - // Update per-memtable watermark (inclusive: last batch ID that was flushed) - batch_store.set_max_flushed_batch_position(end_batch_position - 1); + let start = Instant::now(); + let append_result = self.wal_appender.append(record_batches).await?; + let wal_io_duration = start.elapsed(); - // Notify durability waiters (global channel) - let _ = self.durable_watermark_tx.send(end_batch_position); - // Signal WAL flush completion for backpressure waiters + // Advance the writer-global durability cursor and wake waiters. The range + // just appended is `[start, end)` *local to this store*, so it must be + // lifted into the writer's coordinate space before it is published — + // otherwise a fresh memtable's small local end would be compared against a + // cursor carrying a previous memtable's larger one. + self.advance_durable(batch_store.global_offset() + end_batch_position); self.signal_wal_flush_complete(); Ok(WalFlushResult { @@ -629,9 +873,6 @@ impl WalFlusher { num_batches: append_result.num_batches, }), wal_io_duration, - index_update_duration, - index_update_duration_breakdown, - rows_indexed: rows_to_index, wal_bytes: append_result.wal_bytes, }) } @@ -649,13 +890,15 @@ impl WalFlusher { let append_result = self.wal_appender.append(snapshot.batches).await?; let wal_io_duration = start.elapsed(); - // Append succeeded — remove the flushed batches from the front of - // the queue. Note: WAL-only mode does not use the global durability - // watermark (`durable_watermark_tx`) — `put_wal_only` waits on the - // per-trigger `done` cell instead — so we don't advance it here. - // Same for the wal-flush-completion cell, which is only consulted - // by MemTable-mode backpressure waiters. + // Append succeeded — remove the flushed batches from the front of the + // queue, then advance the writer-global durability cursor so durable + // `put_wal_only` callers waiting on their `BatchDurableWatcher` wake. + // The queue is strict FIFO with contiguous positions and its front + // always sits at the current watermark, so the newly-durable suffix + // ends at `durable + count`. Flushes are serialized on the single + // handler task, so this read-then-advance cannot race another flush. state.commit_flushed(snapshot.count); + self.advance_durable(self.durable() + snapshot.count); Ok(WalFlushResult { entry: Some(WalEntry { @@ -664,9 +907,6 @@ impl WalFlusher { num_batches: append_result.num_batches, }), wal_io_duration, - index_update_duration: std::time::Duration::ZERO, - index_update_duration_breakdown: std::collections::HashMap::new(), - rows_indexed: 0, wal_bytes: append_result.wal_bytes, }) } @@ -699,9 +939,6 @@ pub fn empty_flush_result() -> WalFlushResult { WalFlushResult { entry: None, wal_io_duration: std::time::Duration::ZERO, - index_update_duration: std::time::Duration::ZERO, - index_update_duration_breakdown: std::collections::HashMap::new(), - rows_indexed: 0, wal_bytes: 0, } } @@ -774,7 +1011,7 @@ impl WalEntryData { /// First valid WAL entry position. Positions are 1-based so that a /// `ShardManifest::replay_after_wal_entry_position` of 0 unambiguously means /// "no flush has ever stamped the cursor" — replay then starts at position 1 -/// without needing to consult `flushed_generations`, which an external +/// without needing to consult `sstables`, which an external /// compactor may legitimately drain back to empty. const FIRST_WAL_ENTRY_POSITION: u64 = 1; const MAX_APPEND_CREATE_CONFLICTS: usize = 1024; @@ -1555,18 +1792,22 @@ mod tests { fn batch_store_source(batch_store: &Arc) -> WalFlushSource { WalFlushSource::BatchStore { batch_store: batch_store.clone(), - indexes: None, } } - fn batch_store_source_with_indexes( - batch_store: &Arc, - indexes: &Arc, - ) -> WalFlushSource { - WalFlushSource::BatchStore { - batch_store: batch_store.clone(), - indexes: Some(indexes.clone()), - } + /// Run the index-apply task's body, as the writer's index task would. + async fn apply_all(batch_store: &Arc, indexes: &Arc) -> Result<()> { + let cursors = Arc::new(WriterCursors::new(true)); + apply_index_range( + &cursors, + TriggerIndexApply { + batch_store: batch_store.clone(), + indexes: indexes.clone(), + end_batch_position: batch_store.len(), + }, + ) + .await + .map(|_| ()) } #[tokio::test] @@ -1576,7 +1817,7 @@ mod tests { let buffer = build_test_flusher(store, &base_path, shard_id, 1); // Track a batch - let watcher = buffer.track_batch(0); + let watcher = buffer.track_batch(None, 0, 1); // Watcher should not be durable yet assert!(!watcher.is_durable()); @@ -1595,7 +1836,7 @@ mod tests { let batch_store = Arc::new(BatchStore::with_capacity(10)); batch_store.append(create_test_batch(&schema, 10)).unwrap(); - let mut watcher = flusher.track_batch(0); + let mut watcher = flusher.track_batch(None, 0, 1); // wait() must NOT resolve before the flush happens let result = @@ -1628,13 +1869,13 @@ mod tests { batch_store.append(batch2).unwrap(); // Track batch IDs in WAL flusher - let mut watcher1 = buffer.track_batch(0); - let mut watcher2 = buffer.track_batch(1); + let mut watcher1 = buffer.track_batch(None, 0, 1); + let mut watcher2 = buffer.track_batch(None, 0, 2); // Verify initial state assert!(!watcher1.is_durable()); assert!(!watcher2.is_durable()); - assert!(batch_store.max_flushed_batch_position().is_none()); + assert_eq!(buffer.durable(), 0); // Flush all pending batches let source = batch_store_source(&batch_store); @@ -1646,8 +1887,8 @@ mod tests { assert_eq!(entry.position, FIRST_WAL_ENTRY_POSITION); assert_eq!(entry.writer_epoch, 1); assert_eq!(entry.num_batches, 2); - // After flushing 2 batches (positions 0 and 1), max flushed position is 1 (inclusive) - assert_eq!(batch_store.max_flushed_batch_position(), Some(1)); + // Two batches appended => the writer-global durable count is 2 (exclusive). + assert_eq!(buffer.durable(), 2); // Watchers should be notified watcher1.wait().await.unwrap(); @@ -1658,12 +1899,19 @@ mod tests { // Regression test for the visibility-cursor bug: with an empty IndexStore // (the common case for WAL-managed tables that mirror an index-less base - // dataset), a WAL flush must still advance `max_visible_batch_position` so - // scanners can see every batch up to the durable position — not just - // batch 0. Before the fix, the cursor stayed at 0 for the lifetime of the - // memtable and scanners returned only the first row. + /// The index apply and the WAL append are separate tasks with separate + /// cursors. Appending makes a range durable; it does not index it. Indexing + /// makes it indexed; it does not make it durable. Only both together make it + /// visible. + /// + /// This also covers the empty-registry case (a memtable with no configured + /// indexes), which used to be skipped entirely by the flush's index arm and + /// so left the cursor stuck at 0 for the memtable's whole life. + #[rstest::rstest] + #[case::no_indexes(false)] + #[case::btree_index(true)] #[tokio::test] - async fn test_wal_flush_advances_visibility_with_empty_indexes() { + async fn test_append_and_index_advance_separate_cursors(#[case] with_btree: bool) { let (store, base_path, _temp_dir) = create_local_store().await; let shard_id = Uuid::new_v4(); let flusher = build_test_flusher(store, &base_path, shard_id, 1); @@ -1674,28 +1922,149 @@ mod tests { batch_store.append(create_test_batch(&schema, 5)).unwrap(); } - // Empty registry, mimicking a memtable with `index_configs = []`. - let indexes = Arc::new(IndexStore::new()); - assert_eq!(indexes.max_visible_batch_position(), 0); + let mut idx = IndexStore::new(); + if with_btree { + idx.add_btree("id_idx".to_string(), 0, "id".to_string()); + } + let indexes = Arc::new(idx); - let source = batch_store_source_with_indexes(&batch_store, &indexes); - flusher.flush(&source, batch_store.len()).await.unwrap(); + // The append alone makes the range durable and indexes nothing. + flusher + .flush(&batch_store_source(&batch_store), batch_store.len()) + .await + .unwrap(); + assert_eq!(flusher.durable(), 3); + assert_eq!(indexes.indexed_count(), 0); - // Cursor must advance to the highest flushed batch position (2), - // making all three batches visible to scanners. - assert_eq!(indexes.max_visible_batch_position(), 2); - assert_eq!(batch_store.max_flushed_batch_position(), Some(2)); + // The index apply alone advances the index cursor. + apply_all(&batch_store, &indexes).await.unwrap(); + assert_eq!(indexes.indexed_count(), 3); } - // Regression guard for the indexed path: with at least one BTree index - // configured, the cursor advance still fires (this was already working - // before the fix — keeping the test to lock in the behavior). + /// A WAL flush asked to cover a range past the store's committed length must + /// fail terminally, not silently short-append. The store is append-only, so + /// `get` returns `None` only past `committed_len`; skipping that position + /// while still advancing durability to `end_batch_position` would mark an + /// un-appended batch durable and lose it on replay. The flush must poison + /// instead, and durability must not move. #[tokio::test] - async fn test_wal_flush_advances_visibility_with_btree_index() { + async fn test_flush_rejects_range_past_committed_len() { let (store, base_path, _temp_dir) = create_local_store().await; let shard_id = Uuid::new_v4(); let flusher = build_test_flusher(store, &base_path, shard_id, 1); + let schema = create_test_schema(); + let batch_store = Arc::new(BatchStore::with_capacity(10)); + batch_store.append(create_test_batch(&schema, 5)).unwrap(); + batch_store.append(create_test_batch(&schema, 5)).unwrap(); + + // Two batches committed (positions 0, 1); ask to flush through position 2. + let err = flusher + .flush(&batch_store_source(&batch_store), batch_store.len() + 1) + .await + .unwrap_err(); + assert_eq!(err.fence_reason(), Some(FenceReason::PersistenceFailure)); + assert!( + err.to_string().contains("missing batch position 2"), + "unexpected error: {err}" + ); + + // Terminal: the flusher is poisoned and durability never advanced past + // what was actually appended. + assert!(flusher.check_poisoned().is_err()); + assert_eq!(flusher.durable(), 0); + } + + /// The index-apply path has the same invariant: a range past the committed + /// length must error rather than silently under-index and advance the cursor + /// past a batch that was never inserted. + #[tokio::test] + async fn test_index_apply_rejects_range_past_committed_len() { + let schema = create_test_schema(); + let batch_store = Arc::new(BatchStore::with_capacity(10)); + batch_store.append(create_test_batch(&schema, 5)).unwrap(); + + let indexes = Arc::new(IndexStore::new()); + let cursors = Arc::new(WriterCursors::new(true)); + + // One batch committed (position 0); ask to index through position 1. + let err = apply_index_range( + &cursors, + TriggerIndexApply { + batch_store: batch_store.clone(), + indexes: indexes.clone(), + end_batch_position: batch_store.len() + 1, + }, + ) + .await + .unwrap_err(); + assert!( + err.to_string().contains("missing batch position 1"), + "unexpected error: {err}" + ); + // The cursor did not advance past the hole. + assert_eq!(indexes.indexed_count(), 0); + } + + #[tokio::test] + async fn test_writer_cursors_advance_and_visibility() { + // The durable cursor starts at zero and advances to the value passed. + let cursors = WriterCursors::new(true); + assert_eq!(cursors.durable(), 0); + cursors.advance_durable(5); + assert_eq!(cursors.durable(), 5); + + // fetch_max: a lower value never walks the cursor backwards. + cursors.advance_durable(3); + assert_eq!(cursors.durable(), 5); + + // durable_write = true: visibility is clamped by the writer-global durable + // cursor, offset by this memtable's global coordinate. + assert!(cursors.durable_write()); + // global_offset 0: min(indexed = 10, durable = 5) = 5. + assert_eq!(cursors.visible_count(10, 0), 5); + // global_offset 2: min(indexed = 10, durable 5 - 2 = 3) = 3. + assert_eq!(cursors.visible_count(10, 2), 3); + // A memtable that starts past the durable cursor sees nothing. + assert_eq!(cursors.visible_count(10, 8), 0); + // Indexing, not durability, is the tighter bound here. + assert_eq!(cursors.visible_count(2, 0), 2); + + // durable_write = false: durability is not part of visibility, so the + // indexed count passes through unchanged regardless of the cursor. + let non_durable = WriterCursors::new(false); + assert!(!non_durable.durable_write()); + assert_eq!(non_durable.visible_count(7, 0), 7); + non_durable.advance_durable(1); + assert_eq!(non_durable.visible_count(7, 0), 7); + } + + #[tokio::test] + async fn test_writer_cursors_advance_is_monotonic() { + let cursors = Arc::new(WriterCursors::new(true)); + let mut handles = Vec::new(); + for target in [4usize, 1, 9, 3, 7, 2] { + let cursors = cursors.clone(); + handles.push(tokio::spawn(async move { + let before = cursors.durable(); + cursors.advance_durable(target); + // An advance can only move the cursor forward, never back — even + // when a smaller target races a larger one. + assert!(cursors.durable() >= before); + })); + } + for handle in handles { + handle.await.unwrap(); + } + // Whatever order the concurrent advances landed in, the cursor ends at the + // largest target. + assert_eq!(cursors.durable(), 9); + } + + /// The happy path of the index-apply task: a valid range advances the index + /// cursor to its end and reports exactly the rows it covered. + #[tokio::test] + async fn test_index_apply_advances_cursor_and_counts_rows() { let schema = create_test_schema(); let batch_store = Arc::new(BatchStore::with_capacity(10)); for _ in 0..3 { @@ -1705,12 +2074,38 @@ mod tests { let mut idx = IndexStore::new(); idx.add_btree("id_idx".to_string(), 0, "id".to_string()); let indexes = Arc::new(idx); - - let source = batch_store_source_with_indexes(&batch_store, &indexes); - flusher.flush(&source, batch_store.len()).await.unwrap(); - - assert_eq!(indexes.max_visible_batch_position(), 2); - assert_eq!(batch_store.max_flushed_batch_position(), Some(2)); + let cursors = Arc::new(WriterCursors::new(true)); + + let stats = apply_index_range( + &cursors, + TriggerIndexApply { + batch_store: batch_store.clone(), + indexes: indexes.clone(), + end_batch_position: batch_store.len(), + }, + ) + .await + .unwrap(); + + // Three batches of five rows each were indexed, and the cursor advanced to + // cover them. + assert_eq!(indexes.indexed_count(), 3); + assert_eq!(stats.rows_indexed, 15); + + // Re-applying the same range is a coalesced no-op: the cursor holds and no + // rows are recounted. + let repeat = apply_index_range( + &cursors, + TriggerIndexApply { + batch_store: batch_store.clone(), + indexes: indexes.clone(), + end_batch_position: batch_store.len(), + }, + ) + .await + .unwrap(); + assert_eq!(indexes.indexed_count(), 3); + assert_eq!(repeat.rows_indexed, 0); } #[tokio::test] @@ -1726,8 +2121,8 @@ mod tests { batch_store.append(create_test_batch(&schema, 5)).unwrap(); // Track batch IDs and flush all pending batches - let _watcher1 = buffer.track_batch(0); - let _watcher2 = buffer.track_batch(1); + let _watcher1 = buffer.track_batch(None, 0, 1); + let _watcher2 = buffer.track_batch(None, 0, 2); let source = batch_store_source(&batch_store); let result = buffer.flush(&source, batch_store.len()).await.unwrap(); let entry = result.entry.unwrap(); @@ -1905,7 +2300,7 @@ mod tests { // A durable put on the predecessor: stage a batch and track it. let batch_store = Arc::new(BatchStore::with_capacity(10)); batch_store.append(create_test_batch(&schema, 1)).unwrap(); - let mut watcher = flusher.track_batch(0); + let mut watcher = flusher.track_batch(None, 0, 1); // Flushing collides with the sentinel and fences. Both the flush result // and the watcher must report the fence — and the watcher must resolve @@ -2067,6 +2462,132 @@ mod tests { ); } + /// A failed WAL append must not make rows visible, even when the index apply + /// has already taken them. + /// + /// The two now run on separate tasks, so an index apply that lands while the + /// append is failing advances `indexed_count` — which is fine, and + /// unavoidable: indexes are derived state and replay rebuilds them. What must + /// not happen is for that to make the rows *readable*, because they are not in + /// the WAL and replay will not reproduce them. + /// + /// Visibility is derived, not published, so this holds by construction: with + /// `durable_write`, `visible = min(indexed, durable)`, and a failed append + /// leaves `durable` at 0. + #[tokio::test] + async fn test_failed_append_indexes_but_stays_invisible() { + let (store, base, controls) = failing_memory_store().await; + let shard_id = Uuid::new_v4(); + controls.fail_wal_puts(usize::MAX); + let manifest_store = Arc::new(ShardManifestStore::new(store.clone(), &base, shard_id, 2)); + let (epoch, _) = manifest_store.claim_epoch(0).await.unwrap(); + let appender = Arc::new(WalAppender::with_claimed_epoch( + store, + base, + shard_id, + manifest_store, + epoch, + 0, + WalRetryConfig { + max_retries: 1, + base_delay: Duration::from_millis(1), + }, + )); + let cursors = Arc::new(WriterCursors::new(true)); + let flusher = WalFlusher::with_cursors(appender, Arc::clone(&cursors)); + + let schema = create_test_schema(); + let batch_store = Arc::new(BatchStore::with_capacity(10)); + batch_store.append(create_test_batch(&schema, 1)).unwrap(); + + let mut idx = IndexStore::new(); + idx.add_btree("id_idx".to_string(), 0, "id".to_string()); + idx.set_durability(Arc::clone(&cursors), 0); + let indexes = Arc::new(idx); + + // The index apply succeeds. + apply_index_range( + &cursors, + TriggerIndexApply { + batch_store: batch_store.clone(), + indexes: indexes.clone(), + end_batch_position: 1, + }, + ) + .await + .unwrap(); + assert_eq!(indexes.indexed_count(), 1); + + // The append does not. + let err = flusher + .flush(&batch_store_source(&batch_store), batch_store.len()) + .await + .expect_err("the WAL PUT is failing, so the append must fail"); + assert_eq!(err.fence_reason(), Some(FenceReason::PersistenceFailure)); + + // Indexed, but not durable — so not visible. + assert_eq!(flusher.durable(), 0); + assert_eq!( + indexes.visible_count(), + 0, + "a row whose WAL append failed must never become readable" + ); + } + + /// An index-apply failure poisons the writer. + /// + /// A partial apply cannot be rolled back — `insert_batches` joins every index + /// thread unconditionally, so a failure leaves the others fully applied, and + /// none of HNSW, FTS or BTree has a delete. Continuing would re-cover the + /// range on the next attempt and corrupt the indexes that *did* succeed. So + /// the failure is terminal: reads and writes fail fast, and recovery is + /// reopen -> replay, which rebuilds the indexes from the WAL. + #[tokio::test] + async fn test_index_failure_poisons_the_writer() { + let (store, base_path, _temp_dir) = create_local_store().await; + let shard_id = Uuid::new_v4(); + let flusher = build_test_flusher(store, &base_path, shard_id, 1); + let cursors = Arc::clone(flusher.cursors()); + + let schema = create_test_schema(); + let batch_store = Arc::new(BatchStore::with_capacity(10)); + batch_store.append(create_test_batch(&schema, 1)).unwrap(); + + // An HNSW index on `id`, which is an Int32 and not a vector, so every + // insert of this batch fails deterministically. `validate_index_configs` + // rejects this at shard open — that is what makes poison-and-replay + // terminating — so the store has to be built by hand to reach it at all. + let mut idx = IndexStore::new(); + idx.add_hnsw( + "bad_hnsw".to_string(), + 0, + "id".to_string(), + lance_linalg::distance::DistanceType::L2, + 128, + 8, + ); + let indexes = Arc::new(idx); + + let err = apply_index_range( + &cursors, + TriggerIndexApply { + batch_store: batch_store.clone(), + indexes: indexes.clone(), + end_batch_position: 1, + }, + ) + .await + .expect_err("indexing an Int32 column as a vector must fail"); + + // The index task latches it, exactly as `IndexApplyHandler` does. + flusher.poison(&err); + assert!( + flusher.check_poisoned().is_err(), + "the writer must be poisoned rather than limp on with a corrupt index" + ); + assert_eq!(indexes.visible_count(), 0); + } + // A persistence failure during flush latches the poison: the flush result, // `check_poisoned`, and the durability watcher all report the typed error // (rather than the watcher hanging on a watermark that never advances). @@ -2094,7 +2615,7 @@ mod tests { let schema = create_test_schema(); let batch_store = Arc::new(BatchStore::with_capacity(10)); batch_store.append(create_test_batch(&schema, 1)).unwrap(); - let mut watcher = flusher.track_batch(0); + let mut watcher = flusher.track_batch(None, 0, 1); let source = batch_store_source(&batch_store); let flush_err = flusher.flush(&source, batch_store.len()).await.unwrap_err(); @@ -2114,4 +2635,33 @@ mod tests { .expect_err("watcher must surface the poison"); assert_eq!(waited.fence_reason(), Some(FenceReason::PersistenceFailure)); } + + // A panic under the `terminal_error` lock must not cost the writer its + // latched failure. Recovery is reopen -> replay, which is driven by the real + // `FenceReason`; reporting the mutex poisoning instead would bury it. + #[tokio::test] + async fn test_poisoned_terminal_error_mutex_still_reports_typed_failure() { + let cursors = Arc::new(WriterCursors::new(true)); + cursors.mark_terminal_failure(&Error::writer_poisoned("injected persistence failure")); + + let terminal_error = Arc::clone(&cursors.terminal_error); + let panicked = std::thread::spawn(move || { + let _guard = terminal_error.lock().unwrap(); + panic!("poison the terminal error mutex"); + }) + .join(); + assert!(panicked.is_err()); + assert!(cursors.terminal_error.is_poisoned()); + + let error = cursors.check_poisoned().unwrap_err(); + assert_eq!(error.fence_reason(), Some(FenceReason::PersistenceFailure)); + assert!(error.to_string().contains("injected persistence failure")); + + // The latch still takes writes, and still keeps the first failure. + cursors.mark_terminal_failure(&Error::fenced_by_peer("later peer fence")); + assert_eq!( + cursors.check_poisoned().unwrap_err().fence_reason(), + Some(FenceReason::PersistenceFailure) + ); + } } diff --git a/rust/lance/src/dataset/mem_wal/write.rs b/rust/lance/src/dataset/mem_wal/write.rs index 1b505354813..cd107612d8c 100644 --- a/rust/lance/src/dataset/mem_wal/write.rs +++ b/rust/lance/src/dataset/mem_wal/write.rs @@ -25,7 +25,7 @@ use lance_core::datatypes::Schema; use lance_core::{Error, Result}; use lance_index::mem_wal::ShardManifest; use lance_index::vector::hnsw::builder::HnswBuildParams; -use lance_io::object_store::ObjectStore; +use lance_io::object_store::{ObjectStore, ObjectStoreParams}; use log::{debug, error, info, warn}; use object_store::path::Path; use tokio::sync::{RwLock, mpsc}; @@ -37,6 +37,7 @@ use uuid::Uuid; pub use super::index::{ BTreeIndexConfig, BTreeMemIndex, FtsIndexConfig, HnswIndexConfig, IndexStore, MemIndexConfig, + MemIndexKind, validate_index_configs, }; pub use super::memtable::CacheConfig; pub use super::memtable::MemTable; @@ -47,12 +48,13 @@ pub use super::util::{WatchableOnceCell, WatchableOnceCellReader}; pub use super::wal::{WalEntry, WalEntryData, WalFlushFailure, WalFlushResult, WalFlusher}; use super::memtable::flush::TriggerMemTableFlush; -use super::scanner::GenerationWarmer; +use super::scanner::SsTableWarmer; use super::wal::{ - BatchDurableWatcher, TriggerWalFlush, WalAppender, WalFlushSource, WalOnlyState, - WalRetryConfig, WalTailer, empty_flush_result, + BatchDurableWatcher, TriggerIndexApply, TriggerWalFlush, WalAppender, WalFlushSource, + WalOnlyState, WalRetryConfig, WalTailer, WriterCursors, apply_index_range, empty_flush_result, }; -use super::{TOMBSTONE, schema_with_tombstone}; +use super::{TOMBSTONE, relax_non_pk_nullability, schema_with_tombstone}; +use crate::session::Session; use super::manifest::ShardManifestStore; @@ -83,17 +85,6 @@ pub struct ShardWriterConfig { /// - Lower latency, batched S3 operations pub durable_write: bool, - /// Whether to update indexes synchronously on each write. - /// - /// When true: - /// - Newly written data is immediately searchable via indexes - /// - Higher latency due to index update overhead - /// - /// When false: - /// - Index updates are deferred - /// - New data may not appear in index-accelerated queries immediately - pub sync_indexed_write: bool, - /// Maximum WAL buffer size in bytes before triggering a flush. /// /// This is a soft threshold - write batches are atomic and won't be split. @@ -165,23 +156,6 @@ pub struct ShardWriterConfig { /// Default: 30 seconds pub backpressure_log_interval: Duration, - /// Maximum rows to buffer before flushing to async indexes. - /// - /// Only applies when `sync_indexed_write` is false. Larger values enable - /// better vectorization but increase memory usage and latency before data - /// becomes searchable. - /// - /// Default: 10,000 rows - pub async_index_buffer_rows: usize, - - /// Maximum time to buffer before flushing to async indexes. - /// - /// Only applies when `sync_indexed_write` is false. Ensures bounded latency - /// for data to become searchable even during low write throughput. - /// - /// Default: 1 second - pub async_index_interval: Duration, - /// Interval for periodic stats logging. /// /// Stats (write throughput, backpressure events, memtable size) are logged @@ -191,14 +165,14 @@ pub struct ShardWriterConfig { pub stats_log_interval: Option, /// How long a frozen memtable lingers in memory after its flush commits, - /// before it is evicted and served only from the on-disk flushed dataset. + /// before it is evicted and served only from the on-disk SSTable dataset. /// /// `Duration::ZERO` (the default) disables retention: evict on commit, no /// sweep ticker. Correct for single-shot queries, which can't observe a /// generation evicted mid-read. /// /// A non-zero value is required only for queries split across reads (e.g. - /// fresh tier and base table read separately, then deduped): the flushed + /// fresh tier and base table read separately, then deduped): the SSTable /// dataset loses the per-batch boundaries that bound as-of membership /// (see [`crate::dataset::mem_wal::scanner::FreshTierWatermark`]), so a /// generation evicted between a query's reads can serve a stale row. Set it @@ -223,8 +197,7 @@ pub struct ShardWriterConfig { /// `durable_write` settings as MemTable mode. /// /// MemTable-tied tunables (`max_memtable_size`, `max_memtable_rows`, - /// `max_memtable_batches`, `sync_indexed_write`, `async_index_buffer_rows`, - /// `async_index_interval`) are ignored when `enable_memtable == false`. + /// `max_memtable_batches`) are ignored when `enable_memtable == false`. /// /// For raw single-entry synchronous atomic appends with no buffering and /// no background tasks, use `WalAppender` directly — it is a strictly @@ -237,7 +210,7 @@ pub struct ShardWriterConfig { /// These control the in-memory HNSW graph this writer builds for its /// MemTable (and, on flush, the on-disk graph serialized from it). They are /// a property of the writer that builds the MemTable, not of the index - /// definition: each flushed generation is independent, so different writers + /// definition: each SSTable is independent, so different writers /// may use different parameters. An index without an entry uses the default /// build parameters. `num_edges` is the HNSW graph degree (level 0 retains /// `2 * num_edges`), equivalent to FAISS's `M`. @@ -248,7 +221,18 @@ pub struct ShardWriterConfig { /// Optional warmer fired pre-commit for each new generation (zero cold reads /// on first query). Wired to the flusher; supplied by the consumer (e.g. the /// WAL pod). Default: `None`. - pub warmer: Option>, + pub warmer: Option>, + + /// Store params the base dataset was opened with, reused for the flusher's + /// opens + writes (base + generations). Injected by `mem_wal_writer`; set + /// these to the params of the dataset at `base_uri`, not to params bound to + /// some other path — generation URIs are derived from them. + /// Default: `None` (open by URI alone). + pub store_params: Option, + + /// Session for those opens, injected alongside `store_params`. + /// Default: `None`. + pub session: Option>, } impl Default for ShardWriterConfig { @@ -257,7 +241,6 @@ impl Default for ShardWriterConfig { shard_id: Uuid::new_v4(), shard_spec_id: 0, durable_write: true, - sync_indexed_write: true, max_wal_buffer_size: 10 * 1024 * 1024, // 10MB max_wal_flush_interval: Some(Duration::from_millis(100)), // 100ms max_wal_persist_retries: 3, @@ -268,13 +251,13 @@ impl Default for ShardWriterConfig { manifest_scan_batch_size: 2, max_unflushed_memtable_bytes: 1024 * 1024 * 1024, // 1GB backpressure_log_interval: Duration::from_secs(30), - async_index_buffer_rows: 10_000, - async_index_interval: Duration::from_secs(1), stats_log_interval: Some(Duration::from_secs(60)), // 1 minute frozen_memtable_grace: Duration::ZERO, enable_memtable: true, hnsw_params: HashMap::new(), warmer: None, + store_params: None, + session: None, } } } @@ -300,12 +283,6 @@ impl ShardWriterConfig { self } - /// Set indexed writes mode. - pub fn with_sync_indexed_write(mut self, indexed: bool) -> Self { - self.sync_indexed_write = indexed; - self - } - /// Set maximum WAL buffer size. pub fn with_max_wal_buffer_size(mut self, size: usize) -> Self { self.max_wal_buffer_size = size; @@ -368,25 +345,13 @@ impl ShardWriterConfig { self } - /// Set async index buffer rows. - pub fn with_async_index_buffer_rows(mut self, rows: usize) -> Self { - self.async_index_buffer_rows = rows; - self - } - - /// Set async index interval. - pub fn with_async_index_interval(mut self, interval: Duration) -> Self { - self.async_index_interval = interval; - self - } - /// Set stats logging interval. Use None to disable periodic stats logging. pub fn with_stats_log_interval(mut self, interval: Option) -> Self { self.stats_log_interval = interval; self } - /// Set how long a flushed memtable lingers in memory before eviction. MUST + /// Set how long an SSTable lingers in memory before eviction. MUST /// exceed the maximum query elapsed time — see `frozen_memtable_grace`. pub fn with_frozen_memtable_grace(mut self, grace: Duration) -> Self { self.frozen_memtable_grace = grace; @@ -453,7 +418,13 @@ impl TaskDispatcher { let mut ticker_intervals: Vec<(Interval, MessageFactory)> = tickers .into_iter() .map(|(duration, factory)| { - let interval = interval_at(tokio::time::Instant::now() + duration, duration); + let mut interval = interval_at(tokio::time::Instant::now() + duration, duration); + // `Burst` (the default) replays every tick missed while `handle()` + // was running. A WAL append can easily outlast its own interval, so + // the missed ticks pile up, the ticker arm below is always ready, + // and — being `biased` — it starves `rx` indefinitely: freeze + // completion cells and `close()`'s final append never get handled. + interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay); (interval, factory) }) .collect(); @@ -496,12 +467,12 @@ impl TaskDispatcher { debug!("Task '{}' received cancellation", self.name); break Ok(()); } - _ = first_interval.tick() => { - let message = (ticker_intervals[0].1)(); - if let Err(e) = self.handler.handle(message).await { - error!("Task '{}' error handling ticker message: {}", self.name, e); - } - } + // Explicit messages outrank the ticker. A tick is a backstop + // — it only ever *adds* an append that a real trigger would + // have made anyway — whereas a message may be a freeze's + // completion cell or `close()`'s final append, which nothing + // else will deliver. Polling the ticker first (as this did) + // lets a ready tick starve them. msg = self.rx.recv() => { match msg { Some(message) => { @@ -515,6 +486,12 @@ impl TaskDispatcher { } } } + _ = first_interval.tick() => { + let message = (ticker_intervals[0].1)(); + if let Err(e) = self.handler.handle(message).await { + error!("Task '{}' error handling ticker message: {}", self.name, e); + } + } } } }; @@ -559,20 +536,39 @@ impl TaskExecutor { Ok(()) } + /// Cancel and join every handler registered by [`Self::add_handler`]. + /// + /// Cancellation causes each handler's dispatcher to stop accepting messages and call + /// [`MessageHandler::cleanup`]. This method waits for every dispatcher to finish, even if + /// cleanup fails or a dispatcher panics, and then returns the first such failure. It returns + /// `Ok(())` only after every registered handler has been cleaned up successfully. pub async fn shutdown_all(&self) -> Result<()> { info!("Shutting down all tasks"); self.cancellation_token.cancel(); let tasks = std::mem::take(&mut *self.tasks.write().unwrap()); + let mut first_error = None; for (name, handle) in tasks { match handle.await { Ok(Ok(())) => debug!("Task '{}' completed successfully", name), - Ok(Err(e)) => warn!("Task '{}' completed with error: {}", name, e), - Err(e) => error!("Task '{}' panicked: {}", name, e), + Ok(Err(e)) => { + warn!("Task '{}' completed with error: {}", name, e); + if first_error.is_none() { + first_error = Some(e); + } + } + Err(e) => { + error!("Task '{}' panicked: {}", name, e); + if first_error.is_none() { + first_error = Some(Error::internal(format!( + "Task '{name}' panicked during shutdown: {e}" + ))); + } + } } } - Ok(()) + first_error.map_or(Ok(()), Err) } } @@ -631,10 +627,12 @@ pub type DurabilityCell = WatchableOnceCell; /// Statistics for backpressure monitoring. #[derive(Debug, Default)] pub struct BackpressureStats { - /// Total number of times backpressure was applied. + /// Total number of *completed* waits. total_count: AtomicU64, - /// Total time spent waiting on backpressure (in milliseconds). + /// Total time completed waits spent parked (in milliseconds). total_wait_ms: AtomicU64, + /// Writers parked in `maybe_apply_backpressure` right now. + active_count: AtomicU64, } impl BackpressureStats { @@ -643,18 +641,26 @@ impl BackpressureStats { Self::default() } - /// Record a backpressure event. + /// Record a completed backpressure wait. pub fn record(&self, wait_ms: u64) { self.total_count.fetch_add(1, Ordering::Relaxed); self.total_wait_ms.fetch_add(wait_ms, Ordering::Relaxed); } - /// Get the total backpressure count. + /// Count a writer as parked until the returned guard drops. Drop-based + /// because the caller's future can be cancelled mid-wait, which would + /// otherwise strand a waiter that never returns. + pub fn begin_wait(&self) -> BackpressureWaitGuard<'_> { + self.active_count.fetch_add(1, Ordering::Relaxed); + BackpressureWaitGuard(self) + } + + /// Get the completed-wait count. pub fn count(&self) -> u64 { self.total_count.load(Ordering::Relaxed) } - /// Get the total time spent waiting on backpressure. + /// Get the total time completed waits spent parked. pub fn total_wait_ms(&self) -> u64 { self.total_wait_ms.load(Ordering::Relaxed) } @@ -664,17 +670,34 @@ impl BackpressureStats { BackpressureStatsSnapshot { total_count: self.total_count.load(Ordering::Relaxed), total_wait_ms: self.total_wait_ms.load(Ordering::Relaxed), + active_count: self.active_count.load(Ordering::Relaxed), } } } +/// Keeps its writer counted in `active_count` for as long as it is held. +#[derive(Debug)] +pub struct BackpressureWaitGuard<'a>(&'a BackpressureStats); + +impl Drop for BackpressureWaitGuard<'_> { + fn drop(&mut self) { + self.0.active_count.fetch_sub(1, Ordering::Relaxed); + } +} + /// Snapshot of backpressure statistics. #[derive(Debug, Clone, Default)] pub struct BackpressureStatsSnapshot { - /// Total number of times backpressure was applied. + /// Number of waits that have *finished*. A wait in progress is not counted + /// here, and a cancelled one never is. pub total_count: u64, - /// Total time spent waiting on backpressure (in milliseconds). + /// Total time finished waits spent parked (in milliseconds), on the same + /// denominator as `total_count`. pub total_wait_ms: u64, + /// Writers parked right now. This is the field that answers "am I being + /// throttled at this instant" — the totals only move once a wait ends, so + /// they read zero throughout a first, still-ongoing stall. + pub active_count: u64, } /// Backpressure controller for managing write flow. @@ -717,6 +740,9 @@ impl BackpressureController { { let start = std::time::Instant::now(); let mut iteration = 0u32; + // Held for the whole stall so an operator polling mid-wait sees it; the + // totals below cannot, since they only move once the wait ends. + let mut active_wait = None; loop { let (unflushed_memtable_bytes, oldest_watcher) = get_state(); @@ -731,6 +757,9 @@ impl BackpressureController { } iteration += 1; + if active_wait.is_none() { + active_wait = Some(self.stats.begin_wait()); + } debug!( "Backpressure triggered: unflushed_bytes={}, max={}, iteration={}", @@ -822,7 +851,7 @@ fn now_millis() -> u64 { start_time().elapsed().as_millis() as u64 } -/// Replay WAL entries written after the last successfully-flushed generation +/// Replay WAL entries written after the last successfully-flushed SSTable /// into the freshly-built MemTable. Updates any in-memory indexes attached to /// the MemTable so replayed rows are immediately searchable. /// @@ -836,32 +865,61 @@ fn now_millis() -> u64 { /// Aborts with an error if any replayed entry's `writer_epoch` is strictly /// greater than `our_epoch` — that indicates a successor writer claimed the /// shard between our `claim_epoch` and this replay, fencing us. +/// Outcome of replaying a shard's WAL into memory. +struct ReplayResult { + /// The active memtable — the final, partial one replay left unsealed. A fresh + /// shard yields an empty one; every sealed memtable was flushed to a Lance + /// generation during replay and is not returned. + active: MemTable, + /// One past the highest WAL entry position observed — the next write position. + next_wal_position: u64, +} + +/// Replay a shard's WAL into memory, flushing sealed memtables as the batch store +/// fills. +/// +/// A single memtable holds at most `max_memtable_batches` batches, but a WAL is +/// unbounded — so replay must rotate exactly as the live write path does. It +/// seals a full memtable and, because the data is already durable, flushes it to +/// a Lance generation right here (the same `MemTableFlusher::flush` the live path +/// uses), rather than holding every sealed memtable in memory until open +/// finishes. That bounds resident memory to ~two memtables and truncates the WAL +/// as it goes, so a later reopen replays only the unflushed tail. Only the final +/// partial memtable is returned, as the active one. +/// +/// `make_memtable(generation, global_offset)` builds a fresh, cursor-bound +/// memtable. Rotation happens at WAL-entry boundaries, never mid-entry, so each +/// sealed memtable covers a clean range of complete entries and stamps the last +/// one as its SSTable's `replay_after_wal_entry_position`. +#[allow(clippy::too_many_arguments)] async fn replay_memtable_from_wal( object_store: Arc, base_path: Path, shard_id: Uuid, our_epoch: u64, manifest: &ShardManifest, - memtable: &mut MemTable, -) -> Result { + base_generation: u64, + mut make_memtable: impl FnMut(u64, usize) -> Result, + flusher: &MemTableFlusher, + wal_flusher: &WalFlusher, + index_configs: &[MemIndexConfig], + max_memtable_size: usize, +) -> Result { // WAL positions are 1-based (see `FIRST_WAL_ENTRY_POSITION`), so a // cursor of 0 means "no flush has ever stamped this shard" and replay // starts at position 1. After flushing position N the cursor holds N // and replay starts at N+1. The arithmetic collapses to a single // saturating_add(1) in both cases — we deliberately do not consult - // `flushed_generations` here, since an external compactor may + // `sstables` here, since an external compactor may // legitimately drain that vector back to empty after merging its // contents into the base table. let start_position = manifest.replay_after_wal_entry_position.saturating_add(1); - // The MemTable is always freshly built before this function runs, so - // any existing BatchStore entries can only have come from this replay - // pass. We index everything in `[0, batch_count)` at the end. - debug_assert_eq!(memtable.batch_count(), 0); - let tailer = WalTailer::new(object_store, base_path, shard_id); let mut position = start_position; + let mut active = make_memtable(base_generation, 0)?; + loop { match tailer.read_entry(position).await? { // The first NotFound proves the WAL tip is at `position`, which @@ -877,16 +935,62 @@ async fn replay_memtable_from_wal( // Fence sentinels deserialize to zero batches and are skipped // here — they carry only a position, no rows. if !entry.batches.is_empty() { - // Entries written before deletes existed lack `_tombstone`; - // inject `false` so they match the extended memtable schema. - // Normal entries already carry it and pass through unchanged. - let target_schema = memtable.schema().clone(); + // Re-label to the current storage schema; entries written + // before deletes existed also need `_tombstone = false`. + let storage_schema = active.schema().clone(); let batches = entry .batches .into_iter() - .map(|b| ensure_tombstone_column(b, &target_schema)) + .map(|b| ensure_tombstone_column(b, &storage_schema)) .collect::>>()?; - memtable.insert_batches_only(batches).await?; + + // Seal + flush at the entry boundary on the *same* criteria the + // live path uses (`maybe_trigger_memtable_flush`): the memtable + // is at or over `max_memtable_size` bytes, or this whole entry + // won't fit the batch store. The byte trigger is the one that + // matters beyond avoiding overflow — it is what keeps a memtable + // under `max_memtable_rows`, and therefore keeps the in-memory + // HNSW index (sized to `max_memtable_rows`) from exhausting its + // capacity when the final active memtable is indexed. + // + // Rotate at the entry boundary so no entry is split across two + // memtables and each sealed one covers a clean range of complete + // entries. Never rotate an empty memtable — if a single entry + // has more batches than a memtable can hold, a fresh one would + // overflow too, the same hard limit the live put path has, left + // to the insert below to surface. + if !active.batch_store().is_empty() + && memtable_reached_flush_threshold( + &active, + max_memtable_size, + batches.len(), + ) + { + let store = active.batch_store(); + // The last entry this memtable fully absorbed is the one + // before the entry about to be inserted. + let covered = position.saturating_sub(1); + let generation = active.generation() + 1; + let global_end = store.global_end(); + + // The sealed data is already durable in the WAL — mark it + // so the flush's `all_flushed_to_wal` precondition holds and + // no WAL re-append is attempted. + wal_flusher.advance_durable(global_end); + flush_replayed_memtable( + flusher, + &active, + our_epoch, + covered, + global_end, + index_configs, + ) + .await?; + + active = make_memtable(generation, global_end)?; + } + + active.insert_batches_only(batches).await?; } position = position.checked_add(1).ok_or_else(|| { Error::io(format!( @@ -898,21 +1002,18 @@ async fn replay_memtable_from_wal( } } - // Update in-memory indexes with the replayed batches so readers see them - // through the index path (matching what would have happened on the - // pre-crash writer's WAL flush). Indexes from the previous writer don't - // persist; this rebuilds them from the WAL. - if let Some(indexes) = memtable.indexes_arc() { - let batches_after = memtable.batch_count(); - if batches_after > 0 { - let store = memtable.batch_store(); - let mut stored: Vec = Vec::with_capacity(batches_after); - for pos in 0..batches_after { - if let Some(s) = store.get(pos) { - stored.push(s.clone()); - } - } - tokio::task::spawn_blocking(move || indexes.insert_batches_parallel(&stored)) + // Rebuild the active memtable's in-memory indexes from the batches just + // replayed, so readers see them through the index path — matching what the + // pre-crash writer's flush would have done. Sealed memtables needed no + // in-memory index build: they were flushed straight to disk and are gone. + if let Some(indexes) = active.indexes_arc() { + let batch_count = active.batch_count(); + if batch_count > 0 { + let store = active.batch_store(); + let stored: Vec = (0..batch_count) + .filter_map(|pos| store.get(pos).cloned()) + .collect(); + tokio::task::spawn_blocking(move || indexes.insert_batches(&stored)) .await .map_err(|e| { Error::internal(format!("WAL replay index update task panicked: {}", e)) @@ -920,7 +1021,50 @@ async fn replay_memtable_from_wal( } } - Ok(position) + Ok(ReplayResult { + active, + next_wal_position: position, + }) +} + +/// Whether a memtable has reached the threshold at which it should be sealed and +/// flushed: at or over `max_memtable_size` bytes, or without room in its batch +/// store for `incoming_batches` more. +/// +/// The single source of truth for the flush trigger, shared by the live put path +/// (`maybe_trigger_memtable_flush`, checking post-insert with `incoming_batches = +/// 1` — "is there room for the next batch") and by replay (checking pre-insert +/// with the next WAL entry's batch count). Keeping one predicate is what stops the +/// two from drifting — e.g. someone adding a third criterion to one and not the +/// other, which is the exact class of bug this whole change set is about. +fn memtable_reached_flush_threshold( + memtable: &MemTable, + max_memtable_size: usize, + incoming_batches: usize, +) -> bool { + memtable.estimated_size() >= max_memtable_size + || memtable.batch_store().remaining_capacity() < incoming_batches +} + +/// Flush a sealed replay memtable to a Lance generation, choosing the indexed +/// path when secondary indexes are configured (mirroring the live memtable-flush +/// handler). Commits the manifest, stamping `covered` as the generation's +/// `replay_after_wal_entry_position` so a later reopen skips these entries. +async fn flush_replayed_memtable( + flusher: &MemTableFlusher, + memtable: &MemTable, + epoch: u64, + covered: u64, + durable: usize, + index_configs: &[MemIndexConfig], +) -> Result<()> { + if index_configs.is_empty() { + flusher.flush(memtable, epoch, covered, durable).await?; + } else { + Box::pin(flusher.flush_with_indexes(memtable, epoch, index_configs, covered, durable)) + .await?; + } + Ok(()) } /// Pair each primary-key column name with its field id (both derived from the @@ -933,46 +1077,44 @@ fn pk_index_columns(pk_columns: &[String], pk_field_ids: &[i32]) -> Vec<(String, .collect() } -/// Ensure `batch` carries the `_tombstone` column required by the extended -/// memtable schema, injecting `false` for every row when it is absent. +/// Re-label `batch` to the storage schema, injecting `_tombstone = false` when +/// absent — callers pass logical-shaped batches, and WAL entries written before +/// deletes existed lack the column. /// -/// Used on the normal write path ([`ShardWriter::put`]) where callers pass -/// base-shaped batches, and on WAL replay of entries written before deletes -/// existed (legacy entries lack the column). A batch that already carries -/// `_tombstone` (a normal replayed entry) is returned unchanged. +/// A batch that already carries `_tombstone` is re-labeled too, so an entry +/// written under an older storage schema replays into the current one. fn ensure_tombstone_column( batch: RecordBatch, - target_schema: &Arc, + storage_schema: &Arc, ) -> Result { - if batch.schema().column_with_name(TOMBSTONE).is_some() { - return Ok(batch); - } let n = batch.num_rows(); let mut columns: Vec = batch.columns().to_vec(); - columns.push(Arc::new(BooleanArray::from(vec![false; n]))); - RecordBatch::try_new(target_schema.clone(), columns).map_err(|e| { + if batch.schema().column_with_name(TOMBSTONE).is_none() { + columns.push(Arc::new(BooleanArray::from(vec![false; n]))); + } + RecordBatch::try_new(storage_schema.clone(), columns).map_err(|e| { Error::invalid_input(format!( - "failed to inject _tombstone column (does the batch match the base schema?): {}", + "failed to inject _tombstone column (does the batch match the base table schema?): {}", e )) }) } -/// Build a tombstone batch from a key-only `keys` batch: the primary key -/// columns are carried through, `_tombstone` is set to `true`, and every other -/// column in the memtable schema is null. +/// Build a tombstone batch from a key-only `keys` batch: primary keys carried +/// through, `_tombstone` true, every other column null. /// -/// Errors if `keys` is missing a primary key column, or if a non-PK column is -/// non-nullable (a tombstone must null it) — surfaced via the `RecordBatch` -/// validation. +/// Non-PK columns are nullable in the storage schema however the base table +/// declares them — that is what lets a strict table have tombstones at all. +/// Primary keys are not, so the validation below still rejects a null, +/// mistyped, or missing key. fn build_tombstone_batch( keys: &RecordBatch, - target_schema: &Arc, + storage_schema: &Arc, pk_columns: &[String], ) -> Result { let n = keys.num_rows(); - let mut columns: Vec = Vec::with_capacity(target_schema.fields().len()); - for field in target_schema.fields() { + let mut columns: Vec = Vec::with_capacity(storage_schema.fields().len()); + for field in storage_schema.fields() { let name = field.name(); if name == TOMBSTONE { columns.push(Arc::new(BooleanArray::from(vec![true; n]))); @@ -988,9 +1130,9 @@ fn build_tombstone_batch( columns.push(new_null_array(field.data_type(), n)); } } - RecordBatch::try_new(target_schema.clone(), columns).map_err(|e| { + RecordBatch::try_new(storage_schema.clone(), columns).map_err(|e| { Error::invalid_input(format!( - "failed to build tombstone batch (is every non-primary-key column nullable?): {}", + "failed to build tombstone batch (do the delete keys match the primary key?): {}", e )) }) @@ -1001,6 +1143,10 @@ struct SharedWriterState { state: Arc>, wal_flusher: Arc, wal_flush_tx: mpsc::UnboundedSender, + /// The index-apply task's channel. Separate from the WAL flusher's on + /// purpose: `TaskDispatcher::run` awaits `handle()` inline, so sharing one + /// would put a ~100ms S3 PUT in front of every latency-sensitive index apply. + index_apply_tx: mpsc::UnboundedSender, memtable_flush_tx: mpsc::UnboundedSender, config: ShardWriterConfig, schema: Arc, @@ -1019,6 +1165,7 @@ impl SharedWriterState { state: Arc>, wal_flusher: Arc, wal_flush_tx: mpsc::UnboundedSender, + index_apply_tx: mpsc::UnboundedSender, memtable_flush_tx: mpsc::UnboundedSender, config: ShardWriterConfig, schema: Arc, @@ -1032,6 +1179,7 @@ impl SharedWriterState { state, wal_flusher, wal_flush_tx, + index_apply_tx, memtable_flush_tx, config, schema, @@ -1043,59 +1191,100 @@ impl SharedWriterState { } } + /// Ask the index-apply task to cover `[indexed, end_batch_position)` of this + /// store. Cheap and idempotent: a range already covered is a no-op, which is + /// the common case under load, since one apply coalesces the puts queued + /// behind it. + fn trigger_index_apply( + &self, + batch_store: Arc, + indexes: Arc, + end_batch_position: usize, + ) -> Result<()> { + self.index_apply_tx + .send(TriggerIndexApply { + batch_store, + indexes, + end_batch_position, + }) + .map_err(|_| Error::io("index apply channel closed")) + } + /// Freeze the current memtable and send it to the flush handler. /// /// Takes `&mut WriterState` directly since caller already holds the lock. fn freeze_memtable(&self, state: &mut WriterState) -> Result { - let pending_wal_range = state.memtable.batch_store().pending_wal_flush_range(); + let durable = self.wal_flusher.durable(); + let pending_wal_range = state + .memtable + .batch_store() + .pending_wal_flush_range(durable); let last_wal_entry_position = state.last_flushed_wal_entry_position; let old_batch_store = state.memtable.batch_store(); - let old_indexes = state.memtable.indexes_arc(); let next_generation = state.memtable.generation() + 1; - let mut new_memtable = MemTable::with_capacity( + // The incoming memtable's batch 0 continues the writer's batch sequence + // where the outgoing one ends. Without this coordinate, local positions + // (which restart at 0 every rotation) cannot be mapped onto the + // writer-global durability cursor. + let next_global_offset = old_batch_store.global_end(); + let mut new_memtable = MemTable::with_capacity_at( self.schema.clone(), next_generation, self.pk_field_ids.clone(), CacheConfig::default(), self.max_memtable_batches, + next_global_offset, )?; - // Build an IndexStore when there are user indexes *or* a primary key: - // the PK dedup index (and its flushed on-disk sidecar) is required for - // cross-generation dedup even when no secondary index is configured. - if !self.index_configs.is_empty() || !self.pk_columns.is_empty() { - let mut indexes = IndexStore::from_configs( - &self.index_configs, - self.max_memtable_rows, - self.max_memtable_batches, - )?; + // Always build and bind an IndexStore, even with no user indexes and no + // primary key. It is what carries the memtable's `indexed_count`, and + // binding it to the writer's cursors is what lets a reader derive the + // visible prefix — so an index-less memtable that skipped this would fall + // back to `visible == indexed` and publish rows before they were durable. + // (A PK memtable also needs the PK dedup index and its flushed sidecar.) + let mut indexes = IndexStore::from_configs( + &self.index_configs, + self.max_memtable_rows, + self.max_memtable_batches, + )?; + if !self.pk_columns.is_empty() { indexes.enable_pk_index(&pk_index_columns(&self.pk_columns, &self.pk_field_ids)); - new_memtable.set_indexes_arc(Arc::new(indexes)); } + indexes.set_durability(Arc::clone(self.wal_flusher.cursors()), next_global_offset); + new_memtable.set_indexes_arc(Arc::new(indexes)); let mut old_memtable = std::mem::replace(&mut state.memtable, new_memtable); old_memtable.freeze(last_wal_entry_position); + + // Set up completion tracking on the outgoing table before it is retained + // and before any fallible dispatch, so the retained table already carries + // its cells and a failed send below can poison-and-return without leaving + // partial state to unwind. let _memtable_flush_watcher = old_memtable.create_memtable_flush_completion(); - if pending_wal_range.is_some() { + // The outgoing memtable may still owe an index apply — the puts that + // filled it triggered one, but the task need not have drained yet, and + // this is the last chance to name that store. Its L0 flush is gated on + // the WAL append (below), not on indexing, so without this its tail could + // stay unindexed and invisible for the rest of its life. + let pending_index_apply = match old_memtable.indexes_arc() { + Some(old_indexes) if old_indexes.indexed_count() < old_batch_store.len() => { + Some((old_batch_store.clone(), old_indexes, old_batch_store.len())) + } + _ => None, + }; + + let pending_wal_flush = if pending_wal_range.is_some() { let completion_cell: WatchableOnceCell< std::result::Result, > = WatchableOnceCell::new(); - let completion_reader = completion_cell.reader(); - old_memtable.set_wal_flush_completion(completion_reader); - - let end_batch_position = old_batch_store.len(); - self.wal_flusher.trigger_flush( - WalFlushSource::BatchStore { - batch_store: old_batch_store, - indexes: old_indexes, - }, - end_batch_position, - Some(completion_cell), - )?; - } + old_memtable.set_wal_flush_completion(completion_cell.reader()); + Some((old_batch_store.len(), completion_cell)) + } else { + None + }; let frozen_size = old_memtable.estimated_size(); state.frozen_memtable_bytes += frozen_size; @@ -1109,14 +1298,38 @@ impl SharedWriterState { let frozen_memtable = Arc::new(old_memtable); - // Keep this generation queryable past its manifest commit (swept after - // the grace by `SweepExpired`). Arc refcount, not a copy — the flush - // task holds it alive for the whole drain anyway. + // Retain the outgoing table in the read view *before* the fallible + // dispatches below. `state.memtable` was already replaced, so a failed + // send that returned here without this push would drop the table and its + // accepted rows would silently vanish from every scan. Keep it queryable + // past its manifest commit too (swept after the grace by `SweepExpired`); + // Arc refcount, not a copy — the flush task holds it alive anyway. state.frozen_memtables.push_back(FrozenMemTable { memtable: frozen_memtable.clone(), flushed_at_ms: None, }); + // Dispatch can only fail if a background task's channel is already closed, + // i.e. the writer is being torn down. Poison so the read path fails fast + // with the typed error instead of serving the retained-but-never-durable + // tail, then return — the table stays in the read view. + if let Some((batch_store, indexes, end_batch_position)) = pending_index_apply { + self.trigger_index_apply(batch_store, indexes, end_batch_position) + .inspect_err(|e| self.wal_flusher.poison(e))?; + } + + if let Some((end_batch_position, completion_cell)) = pending_wal_flush { + self.wal_flusher + .trigger_flush( + WalFlushSource::BatchStore { + batch_store: old_batch_store, + }, + end_batch_position, + Some(completion_cell), + ) + .inspect_err(|e| self.wal_flusher.poison(e))?; + } + debug!( "Frozen memtable generation {}, pending_count = {}", next_generation - 1, @@ -1131,9 +1344,19 @@ impl SharedWriterState { Ok(next_generation) } - /// Track batch for WAL durability. - fn track_batch_for_wal(&self, batch_position: usize) -> super::wal::BatchDurableWatcher { - self.wal_flusher.track_batch(batch_position) + /// Watch for a write to become visible: indexed, and — in durable mode — + /// WAL-durable too. + /// + /// `target_indexed` is memtable-local; `target_durable` is writer-global. + /// Different coordinate spaces on purpose — see `WalFlusher::track_batch`. + fn track_batch_for_wal( + &self, + indexes: Option>, + target_indexed: usize, + target_durable: usize, + ) -> super::wal::BatchDurableWatcher { + self.wal_flusher + .track_batch(indexes, target_indexed, target_durable) } /// Check if memtable flush is needed and trigger if so. @@ -1144,8 +1367,10 @@ impl SharedWriterState { return Ok(()); } - let should_flush = state.memtable.estimated_size() >= self.config.max_memtable_size - || state.memtable.is_batch_store_full(); + // Checked post-insert: flush if there is no longer room for even one more + // batch (or the byte threshold is crossed). Same predicate replay uses. + let should_flush = + memtable_reached_flush_threshold(&state.memtable, self.config.max_memtable_size, 1); if should_flush { state.flush_requested = true; @@ -1164,10 +1389,9 @@ impl SharedWriterState { let batch_count = state.memtable.batch_count(); let total_bytes = state.memtable.estimated_size(); let batch_store = state.memtable.batch_store(); - let indexes = state.memtable.indexes_arc(); // Check if there are any unflushed batches - let has_pending = batch_store.pending_wal_flush_count() > 0; + let has_pending = batch_store.pending_wal_flush_count(self.wal_flusher.durable()) > 0; // Check time-based trigger first let time_trigger = if let Some(interval) = self.config.max_wal_flush_interval { @@ -1196,10 +1420,7 @@ impl SharedWriterState { // If time trigger fired, send a flush message if time_trigger.is_some() { let _ = self.wal_flush_tx.send(TriggerWalFlush { - source: WalFlushSource::BatchStore { - batch_store, - indexes, - }, + source: WalFlushSource::BatchStore { batch_store }, end_batch_position: batch_count, done: None, }); @@ -1224,7 +1445,6 @@ impl SharedWriterState { let _ = self.wal_flush_tx.send(TriggerWalFlush { source: WalFlushSource::BatchStore { batch_store: batch_store.clone(), - indexes: indexes.clone(), }, end_batch_position: batch_count, done: None, @@ -1311,6 +1531,12 @@ pub struct ShardWriter { manifest_store: Arc, stats: SharedWriteStats, mode: WriterMode, + /// The base table's schema as the caller passed it — no `_tombstone`, + /// nullability untouched. Caller input is held to it (see + /// [`Self::validate_against_logical_schema`]) and the scan narrows back to + /// it; the memtable, WAL, and SSTables carry the widened storage schema + /// ([`relax_non_pk_nullability`]) instead. + logical_schema: Arc, } impl ShardWriter { @@ -1334,10 +1560,26 @@ impl ShardWriter { )); } - // Callers pass the base schema; lance owns the `_tombstone` column and - // appends it here so the memtable/generation schema = base + tombstone. - // Idempotent, so a reopen that already extended the schema is a no-op. - let schema = schema_with_tombstone(&schema); + // A durable writer needs a flush ticker to make progress, in either + // mode. With `durable_write` on, a put becomes durable only once its WAL + // append lands, and neither mode self-triggers that append per put — the + // background ticker drives it. Without an interval (or with a zero one, + // which tokio cannot schedule), a small put that never fills the + // size-triggered buffer would block until close. Reject the config here + // rather than let a put hang. + if config.durable_write && config.max_wal_flush_interval.is_none_or(|d| d.is_zero()) { + return Err(Error::invalid_input( + "durable_write requires a positive max_wal_flush_interval: with no \ + flush ticker a durable put has nothing to drive its WAL append and \ + would block until close", + )); + } + + // The caller's schema is the shard's logical schema; the storage schema + // is derived below, once the primary key is known. lance owns + // `_tombstone` and appends it here — idempotent across reopens. + let logical_schema = schema; + let tombstoned = schema_with_tombstone(&logical_schema); let base_uri = base_uri.into(); let shard_id = config.shard_id; @@ -1348,6 +1590,37 @@ impl ShardWriter { config.manifest_scan_batch_size, )); + // Derive PK metadata and run every side-effect-free validation *before* + // claiming the epoch. `claim_epoch` durably bumps the stored epoch and, + // for a successor, `write_fence_sentinel` fences the predecessor — so an + // open doomed by purely local input (an index config that disagrees with + // the schema) must fail here, before it can knock the healthy incumbent off + // the shard. Memtable-only: WAL-only mode has no indexes to validate. + let memtable_validation = if config.enable_memtable { + let lance_schema = Schema::try_from(tombstoned.as_ref())?; + let pk_fields = lance_schema.unenforced_primary_key(); + let pk_field_ids: Vec = pk_fields.iter().map(|f| f.id).collect(); + let pk_columns: Vec = pk_fields.iter().map(|f| f.name.clone()).collect(); + + // Reject an index config that disagrees with the schema *before* a + // single row is accepted. Such a config fails deterministically on + // every insert, including inserts replayed from the WAL — so once a row + // is durable the shard can never reopen. Fail the open instead. + validate_index_configs( + &index_configs, + tombstoned.as_ref(), + &lance_schema, + &pk_columns, + )?; + + // Widen only now that the primary key is known — a tombstone nulls + // every non-PK column, and PK detection needs the strict schema. + let storage_schema = relax_non_pk_nullability(&tombstoned, &pk_columns); + Some((pk_field_ids, pk_columns, storage_schema)) + } else { + None + }; + // Claim the shard (epoch-based fencing) — done once, then shared // with the WalAppender via `with_claimed_epoch`. let (epoch, manifest) = manifest_store.claim_epoch(config.shard_spec_id).await?; @@ -1389,7 +1662,12 @@ impl ShardWriter { } // Create WAL flusher backed by the shared appender. - let mut wal_flusher = WalFlusher::new(wal_appender); + // Build the cursors from *this writer's* config. `durable_write` is what + // decides whether durability is part of visibility, so a flusher that + // defaulted it would leave a non-durable put waiting on a durability + // cursor nothing ever advances. + let cursors = Arc::new(WriterCursors::new(config.durable_write)); + let mut wal_flusher = WalFlusher::with_cursors(wal_appender, cursors); let (wal_flush_tx, wal_flush_rx) = mpsc::unbounded_channel(); wal_flusher.set_flush_channel(wal_flush_tx.clone()); @@ -1399,11 +1677,15 @@ impl ShardWriter { let task_executor = Arc::new(TaskExecutor::new()); let mode = if config.enable_memtable { + let (pk_field_ids, pk_columns, storage_schema) = memtable_validation + .expect("memtable_validation is Some when enable_memtable is true"); Self::open_memtable_mode( &config, - &schema, + &storage_schema, &manifest, &index_configs, + pk_field_ids, + pk_columns, wal_flusher.clone(), wal_flush_tx, wal_flush_rx, @@ -1436,6 +1718,7 @@ impl ShardWriter { manifest_store, stats, mode, + logical_schema, }) } @@ -1445,6 +1728,8 @@ impl ShardWriter { schema: &Arc, manifest: &ShardManifest, index_configs: &[MemIndexConfig], + pk_field_ids: Vec, + pk_columns: Vec, wal_flusher: Arc, wal_flush_tx: mpsc::UnboundedSender, wal_flush_rx: mpsc::UnboundedReceiver, @@ -1457,50 +1742,93 @@ impl ShardWriter { stats: SharedWriteStats, task_executor: &Arc, ) -> Result { - // Create MemTable with primary key field IDs from schema - let lance_schema = Schema::try_from(schema.as_ref())?; - let pk_fields = lance_schema.unenforced_primary_key(); - let pk_field_ids: Vec = pk_fields.iter().map(|f| f.id).collect(); - let pk_columns: Vec = pk_fields.iter().map(|f| f.name.clone()).collect(); - let mut memtable = MemTable::with_capacity( - schema.clone(), - manifest.current_generation, - pk_field_ids.clone(), - CacheConfig::default(), - config.max_memtable_batches, - )?; - - // Create indexes if configured and set them on the MemTable. The - // PK-position index is enabled before any WAL replay below so replayed - // rows are recorded in it. A primary key alone (no secondary index) - // still needs the PK index so flush writes its on-disk dedup sidecar. - if !index_configs.is_empty() || !pk_columns.is_empty() { + // PK metadata and index/interval validation were resolved in `open` + // before the epoch was claimed (a doomed open must not fence the + // incumbent first). + + // Build a fresh, cursor-bound memtable at a given generation and + // writer-global coordinate. Replay calls this for the first memtable and + // after every rotation. Always builds and binds an `IndexStore`, even + // with no user indexes and no primary key — see the note in + // `freeze_memtable` for why an index-less memtable still needs one. + let make_bound_memtable = |generation: u64, global_offset: usize| -> Result { + let mut memtable = MemTable::with_capacity_at( + schema.clone(), + generation, + pk_field_ids.clone(), + CacheConfig::default(), + config.max_memtable_batches, + global_offset, + )?; let mut indexes = IndexStore::from_configs( index_configs, config.max_memtable_rows, config.max_memtable_batches, )?; - indexes.enable_pk_index(&pk_index_columns(&pk_columns, &pk_field_ids)); + if !pk_columns.is_empty() { + indexes.enable_pk_index(&pk_index_columns(&pk_columns, &pk_field_ids)); + } + indexes.set_durability(Arc::clone(wal_flusher.cursors()), global_offset); memtable.set_indexes_arc(Arc::new(indexes)); - } + Ok(memtable) + }; + + // The flusher writes sealed memtables to Lance generations — both the + // ones replay seals below and the ones the live path freezes later. + let flusher = Arc::new( + MemTableFlusher::new( + object_store.clone(), + base_path.clone(), + base_uri.clone(), + shard_id, + manifest_store.clone(), + ) + .with_warmer(config.warmer.clone()) + .with_storage_context(config.store_params.clone(), config.session.clone()), + ); // Replay any WAL entries written after the last successfully-flushed - // generation. Each entry's writer_epoch is checked against ours; an - // entry with a strictly greater epoch indicates a successor writer - // claimed the shard between our `claim_epoch` and replay, so we - // abort the open with a fence error. The replay walked the tailer - // up to the WAL tip, so we hand the discovered next-write position - // straight to the appender — its first append skips the - // discover_next_position probe entirely. - let next_wal_position = replay_memtable_from_wal( + // SSTable, flushing sealed memtables to Lance SSTables as the batch + // store fills. Each entry's writer_epoch is checked against ours; an entry + // with a strictly greater epoch means a successor claimed the shard + // between our `claim_epoch` and replay, so we abort with a fence error. + // Replay walks the tailer to the WAL tip and returns the discovered + // next-write position, so the appender's first append skips the + // discover_next_position probe. + let ReplayResult { + active: memtable, + next_wal_position, + } = replay_memtable_from_wal( object_store.clone(), base_path.clone(), shard_id, epoch, manifest, - &mut memtable, + manifest.current_generation, + make_bound_memtable, + &flusher, + &wal_flusher, + index_configs, + config.max_memtable_size, ) .await?; + + // Mark the active memtable's replayed batches durable. They came *from* + // the WAL, and replay has already re-derived its indexes over them. + // + // Without this the durability cursor stays at the last sealed generation, + // so the next WAL flush re-covers the active tail: it re-appends the + // already-durable rows *and* re-inserts every replayed row into the + // indexes. None of the three in-memory indexes is idempotent (HNSW mints + // fresh node ids for the same row, FTS increments doc_count/df rather than + // recomputing them, BTree is a multiset), so a full scan keeps looking + // healthy while every index-accelerated query silently returns duplicates + // — and it compounds, because the WAL now holds those rows twice. + // + // `global_end()` is the writer-global batch count through this memtable, + // since its coordinate continues where the last sealed generation ended. + wal_flusher.advance_durable(memtable.batch_store().global_end()); + wal_flusher .wal_appender() .seed_next_position(next_wal_position) @@ -1512,8 +1840,8 @@ impl ShardWriter { // it is durably reflected in this writer's memtable. We can't // seed from `manifest.wal_entry_position_last_seen` — that field // is bumped on every successful tailer read by other readers, so - // it may sit above what's actually covered by any flushed - // generation. Subtracting 1 from a fresh shard's `next_wal_position` + // it may sit above what's actually covered by any + // SSTable. Subtracting 1 from a fresh shard's `next_wal_position` // of `FIRST_WAL_ENTRY_POSITION` (= 1) yields 0, which correctly // means "no entry covered yet." let initial_covered_wal_entry_position = next_wal_position.saturating_sub(1); @@ -1531,16 +1859,16 @@ impl ShardWriter { let (memtable_flush_tx, memtable_flush_rx) = mpsc::unbounded_channel(); - let flusher = Arc::new( - MemTableFlusher::new(object_store, base_path, base_uri, shard_id, manifest_store) - .with_warmer(config.warmer.clone()), - ); - let backpressure = BackpressureController::new(config.clone()); // Background WAL flush handler — parallel WAL I/O + index updates. - let wal_handler = - WalFlushHandler::new(wal_flusher.clone(), Some(state.clone()), stats.clone()); + let wal_handler = WalFlushHandler::new( + wal_flusher.clone(), + Some(state.clone()), + None, + config.max_wal_flush_interval, + stats.clone(), + ); task_executor.add_handler( "wal_flusher".to_string(), Box::new(wal_handler), @@ -1548,13 +1876,14 @@ impl ShardWriter { )?; // Background MemTable flush handler — frozen memtable to Lance file. - // It rebuilds the same secondary indexes on each flushed generation. + // It rebuilds the same secondary indexes on each SSTable. let memtable_handler = MemTableFlushHandler::new( state.clone(), flusher, + wal_flusher.clone(), epoch, index_configs.to_vec(), - stats, + stats.clone(), config.frozen_memtable_grace, ); task_executor.add_handler( @@ -1563,11 +1892,27 @@ impl ShardWriter { memtable_flush_rx, )?; + // The index-apply task. Its own channel and its own dispatcher: the + // dispatcher awaits `handle()` inline, so sharing the WAL flusher's + // channel would queue every index apply behind a ~100ms S3 PUT. + let (index_apply_tx, index_apply_rx) = mpsc::unbounded_channel(); + let index_handler = IndexApplyHandler { + cursors: Arc::clone(wal_flusher.cursors()), + wal_flusher: wal_flusher.clone(), + stats, + }; + task_executor.add_handler( + "index_applier".to_string(), + Box::new(index_handler), + index_apply_rx, + )?; + // Shared state used by `put()` to dispatch trigger checks. let writer_state = Arc::new(SharedWriterState::new( state.clone(), wal_flusher, wal_flush_tx, + index_apply_tx, memtable_flush_tx, config.clone(), schema.clone(), @@ -1593,9 +1938,25 @@ impl ShardWriter { stats: SharedWriteStats, task_executor: &Arc, ) -> Result { + // The pending queue is shared with the flush handler so a background + // tick can resolve to the batches still owed an append — the WAL-only + // analog of resolving a tick against the durability cursor in MemTable + // mode. A durable put waits on the durability cursor the handler's + // append advances, so the ticker must run (`open` rejects durable + + // no-interval); a non-durable writer may pass no interval and rely on + // the size/close triggers alone. + let state = Arc::new(WalOnlyState::default()); + // Background WAL flush handler — no MemTable state to consult, so - // pass `None` for the frozen-vs-active detection. - let wal_handler = WalFlushHandler::new(wal_flusher, None, stats); + // pass `None` for the frozen-vs-active detection; the pending queue and + // the flush interval drive the background append instead. + let wal_handler = WalFlushHandler::new( + wal_flusher, + None, + Some(state.clone()), + config.max_wal_flush_interval, + stats, + ); task_executor.add_handler( "wal_flusher".to_string(), Box::new(wal_handler), @@ -1610,7 +1971,7 @@ impl ShardWriter { let backpressure = BackpressureController::new(config.clone()); Ok(WriterMode::WalOnly { - state: Arc::new(WalOnlyState::default()), + state, wal_flush_tx, trigger: StdRwLock::new(WalOnlyTriggerState::default()), backpressure, @@ -1639,6 +2000,7 @@ impl ShardWriter { #[instrument(name = "sw_put", level = "info", skip_all, fields(batch_count = batches.len(), shard_id = %self.config.shard_id))] pub async fn put(&self, batches: Vec) -> Result { Self::validate_non_empty(&batches)?; + self.validate_against_logical_schema(&batches)?; match &self.mode { WriterMode::MemTable { @@ -1646,9 +2008,8 @@ impl ShardWriter { writer_state, backpressure, } => { - // Inject `_tombstone = false` so the batch matches the - // extended memtable schema; callers only ever pass base-shaped - // batches and never name the column. + // Callers pass logical-shaped batches and never name + // `_tombstone`. let batches = batches .into_iter() .map(|b| ensure_tombstone_column(b, &writer_state.schema)) @@ -1677,9 +2038,8 @@ impl ShardWriter { /// its key: it wins newest-per-PK resolution (suppressing the older real /// row) and is then dropped from query results. /// - /// Only supported in memtable mode. Because a tombstone nulls every non-PK - /// column, those columns must be nullable in the base schema; a delete - /// against a schema with a non-nullable non-PK column errors. + /// Only supported in memtable mode. Works against non-nullable base columns: + /// tombstones live in the storage schema, which widens them to nullable. /// /// ``` /// # use lance::Result; @@ -1766,6 +2126,7 @@ impl ShardWriter { batches: Vec, ) -> Result<(WriteResult, Option)> { Self::validate_non_empty(&batches)?; + self.validate_against_logical_schema(&batches)?; match &self.mode { WriterMode::MemTable { @@ -1773,8 +2134,7 @@ impl ShardWriter { writer_state, backpressure, } => { - // Inject `_tombstone = false` to match the extended memtable - // schema, mirroring `put`. + // Mirrors `put`. let batches = batches .into_iter() .map(|b| ensure_tombstone_column(b, &writer_state.schema)) @@ -1788,6 +2148,49 @@ impl ShardWriter { } } + /// Reject caller input that violates the logical schema: wrong column names, + /// order, count, or types, or a null where the base table declares + /// non-nullable. + /// + /// The *only* gate on that contract — the storage schema accepts the null, + /// append and `merge_insert` compare with `NullabilityComparison::Ignore`, + /// and the encoder takes validity from the array, not the field — so a null + /// that gets past here reaches the base table silently. + /// + /// Runs before the WAL append: a batch rejected only afterwards would fail + /// identically on every replay, leaving the shard unable to reopen. + fn validate_against_logical_schema(&self, batches: &[RecordBatch]) -> Result<()> { + for (i, batch) in batches.iter().enumerate() { + // Everything downstream matches columns by position, so a swapped + // pair of same-typed columns would be stored under each other's + // names unless caught here. + for (col, (expected, actual)) in self + .logical_schema + .fields() + .iter() + .zip(batch.schema().fields()) + .enumerate() + { + if expected.name() != actual.name() { + return Err(Error::invalid_input(format!( + "batch {i} column {col} is named '{}', but the base table schema \ + declares '{}' at that position", + actual.name(), + expected.name() + ))); + } + } + RecordBatch::try_new(self.logical_schema.clone(), batch.columns().to_vec()).map_err( + |e| { + Error::invalid_input(format!( + "batch {i} does not match the base table schema: {e}" + )) + }, + )?; + } + Ok(()) + } + fn validate_non_empty(batches: &[RecordBatch]) -> Result<()> { if batches.is_empty() { return Err(Error::invalid_input("Cannot write empty batch list")); @@ -1850,46 +2253,74 @@ impl ShardWriter { // 1. Insert all batches into memtable atomically let results = state.memtable.insert_batches_only(batches).await?; - // Get batch position range + // 2. Capture the store the batches actually landed in, *before* step + // 4 below can freeze and swap the active memtable. Reading it + // afterwards hands the flush trigger the **new** store paired with + // the **old** store's end position, so the new store's watermark + // jumps past batches that were never appended. + let batch_store = state.memtable.batch_store(); + let indexes = state.memtable.indexes_arc(); + let start_pos = results.first().map(|(pos, _, _)| *pos).unwrap_or(0); let end_pos = results.last().map(|(pos, _, _)| pos + 1).unwrap_or(0); let batch_positions = start_pos..end_pos; - // 2. Track last batch for WAL durability - let durable_watcher = writer_state.track_batch_for_wal(end_pos.saturating_sub(1)); + // 3. Watch for this write to become *visible*: indexed, and — under + // `durable_write` — WAL-durable too. + // + // The two targets live in different coordinate spaces. `end_pos` + // is memtable-local, which is what the index apply works in. The + // durability cursor is writer-global, because batch positions + // restart at 0 in every memtable while that cursor spans the + // writer's whole life — a local durable target would already be + // satisfied by a *previous* memtable's appends, so the first N + // puts into every post-rotation memtable would ack as durable with + // no WAL append ever happening. + let durable_watcher = writer_state.track_batch_for_wal( + indexes.clone(), + end_pos, + batch_store.global_offset() + end_pos, + ); - // 3. Check if WAL flush should be triggered + // 4. Check if WAL flush should be triggered writer_state.maybe_trigger_wal_flush(&mut state); - // 4. Check if memtable flush is needed + // 5. Check if memtable flush is needed (may freeze and rotate) if let Err(e) = writer_state.maybe_trigger_memtable_flush(&mut state) { warn!("Failed to trigger memtable flush: {}", e); } - // Get batch_store and indexes while we have the lock (for durable_write case) - let batch_store = state.memtable.batch_store(); - let indexes = state.memtable.indexes_arc(); - (batch_positions, durable_watcher, batch_store, indexes) }; // Lock released here self.stats.record_put(start.elapsed()); - // Trigger the flush here (outside the lock) so the watcher can resolve; - // only the `wait()` is the caller's to schedule. - let watcher = if self.config.durable_write { - self.wal_flusher.trigger_flush( - WalFlushSource::BatchStore { - batch_store, - indexes, - }, - batch_positions.end, - None, - )?; - Some(durable_watcher) - } else { - None - }; + // Trigger the index apply, in **both** modes. This is what makes reads + // read-your-writes regardless of `durable_write`: skipping durability now + // costs the caller durability only, not visibility. It is cheap + // (in-memory, ~ms), so there is no reason to batch it onto the WAL's + // schedule — that schedule exists to bound S3 API cost, which an + // in-memory index apply does not incur. + if let Some(indexes) = indexes { + writer_state.trigger_index_apply(batch_store, indexes, batch_positions.end)?; + } + + // The WAL append is *not* triggered here. It happens on the background + // ticker (and on the size trigger, and at freeze/close), which is the only + // way the flush interval can mean anything: while every durable put + // triggered its own append, the interval could add a redundant trigger but + // never delay or batch one. + // + // The cost is real and accepted: a single client's sequential *durable* + // throughput drops from ~10 writes/sec (one PUT round-trip) to roughly one + // per tick. That is a policy choice — the interval should mean what it + // says, and S3 API cost should be bounded. Latency-sensitive callers want + // `durable_write: false`, which now costs them durability only, not + // visibility. + + // The watcher is returned in both modes now. A non-durable put still + // waits — for its index apply (~ms), not for an S3 PUT (~100ms). + let watcher = Some(durable_watcher); Ok((WriteResult { batch_positions }, watcher)) } @@ -1920,54 +2351,48 @@ impl ShardWriter { // Push batches into the pending queue and capture the assigned // [start, end) range. `next_batch_position` is monotonic across the // writer's lifetime; positions are not BatchStore indices but they - // are used the same way for durability tracking. + // are used the same way for durability tracking. Because the queue is + // strict FIFO with contiguous positions and its front always sits at + // the durability cursor, `end` is exactly the writer-global durable + // count this put reaches once its append lands — no globalizing offset + // as in MemTable mode, which restarts positions per generation. let batch_positions = state.push(batches); - // Time- and size-based triggers, mirroring MemTable mode but reading - // pending bytes from `WalOnlyState` instead of an active MemTable. - // Only fires for non-durable writes; durable writes go through the - // explicit done-cell path below so flush errors (e.g., fence) reach - // the caller. - if !self.config.durable_write { - let target_position = batch_positions.end; - let pending_bytes = state.estimated_size(); - self.maybe_trigger_wal_flush_wal_only( - state, - wal_flush_tx, - trigger, - target_position, - pending_bytes, - ); - } + // Under `durable_write` the put becomes durable only once its WAL + // append lands. Track it on the writer-global durability cursor *before* + // triggering, so a flush that completes between here and the wait is not + // missed: the watcher recomputes visibility from the cursor rather than + // latching a one-shot wake. WAL-only mode has no indexes, so the + // index-visibility half of the watcher is a no-op (`None`, target 0). + let durable_watcher = self + .config + .durable_write + .then(|| self.wal_flusher.track_batch(None, 0, batch_positions.end)); + + // Time- and size-based triggers on the write path, for durable and + // non-durable puts alike — mirroring MemTable mode's + // `maybe_trigger_wal_flush`. The background ticker drives the append + // too; whichever fires first wins, and a redundant trigger is a cheap + // no-op because the flush snapshot/commit is idempotent. + self.maybe_trigger_wal_flush_wal_only( + state, + wal_flush_tx, + trigger, + batch_positions.end, + state.estimated_size(), + ); self.stats.record_put(start.elapsed()); - // For durable writes, trigger an immediate flush and wait for the - // done cell. Using the done cell instead of the durability watermark - // watcher ensures flush errors (e.g., the WalAppender returning a - // fence error) propagate back to `put` instead of hanging. - if self.config.durable_write { - let done = WatchableOnceCell::new(); - let reader = done.reader(); - self.wal_flusher.trigger_flush( - WalFlushSource::WalOnly { - state: state.clone(), - }, - batch_positions.end, - Some(done), - )?; - let mut reader = reader; - match reader.await_value().await { - Some(Ok(_)) => {} - // Rebuild the typed error (peer fence vs. persistence-failure - // self-fence) so a WAL-only durable caller can tell them apart. - Some(Err(failure)) => return Err(failure.into_error()), - None => { - return Err(Error::io( - "WAL flush handler exited before reporting durability", - )); - } - } + // Durable writes wait on the durability cursor, advanced by the append + // the ticker (or the trigger above) drives — the same watermark path + // MemTable mode uses. A terminal flush failure (a peer fence or an + // exhausted-retry persistence self-fence) poisons the writer and wakes + // the waiter with that typed error instead of leaving it to hang. This + // is why `open` rejects `durable_write` with no flush ticker: nothing + // else would advance the cursor a small put is parked on. + if let Some(mut watcher) = durable_watcher { + watcher.wait().await?; } Ok(WriteResult { batch_positions }) @@ -2067,36 +2492,62 @@ impl ShardWriter { /// Get current MemTable statistics. Returns an error in WAL-only mode /// (no MemTable exists). + /// + /// Deliberately does *not* `check_poisoned`, unlike the read and write + /// paths: a poisoned writer is exactly when an operator most needs to see + /// its state, and the caller deciding whether to evict reads these stats. pub async fn memtable_stats(&self) -> Result { let state_lock = self.memtable_state_lock()?; let state = state_lock.read().await; let batch_store = state.memtable.batch_store(); - let pending_wal = batch_store.pending_wal_flush_stats(); + let durable = self.wal_flusher.durable(); + let pending_wal = batch_store.pending_wal_flush_stats(durable); Ok(MemTableStats { row_count: state.memtable.row_count(), batch_count: state.memtable.batch_count(), estimated_size: state.memtable.estimated_size(), generation: state.memtable.generation(), max_buffered_batch_position: batch_store.max_buffered_batch_position(), - max_flushed_batch_position: batch_store.max_flushed_batch_position(), + durable_batch_count: durable, + global_offset: batch_store.global_offset(), pending_wal_start_batch_position: pending_wal.start_batch_position, pending_wal_end_batch_position: pending_wal.end_batch_position, pending_wal_batch_count: pending_wal.batch_count, pending_wal_row_count: pending_wal.row_count, pending_wal_estimated_bytes: pending_wal.estimated_bytes, + frozen_count: state.frozen_memtables.len(), + // Summed from the read view rather than read off `frozen_memtable_bytes`: + // that counter drains on flush *completion* so a failure cannot wedge + // writes, which would report zero for a table still sitting in memory. + frozen_bytes: state + .frozen_memtables + .iter() + .filter(|frozen| frozen.flushed_at_ms.is_none()) + .map(|frozen| frozen.memtable.estimated_size()) + .sum(), }) } + /// Snapshot of the backpressure counters. Both writer modes answer, so a + /// caller asking "am I throttled" need not know which mode it is in. + pub fn backpressure_stats(&self) -> BackpressureStatsSnapshot { + match &self.mode { + WriterMode::MemTable { backpressure, .. } + | WriterMode::WalOnly { backpressure, .. } => backpressure.stats().snapshot(), + } + } + /// Create a scanner for querying the current MemTable data. /// /// The scanner provides read access to all data currently in the MemTable, /// with optional filtering, projection, and index support. /// - /// The scanner captures the current `max_visible_batch_position` from the + /// The scanner captures the current `visible_count` from the /// `IndexStore` at construction time to ensure consistent visibility. /// - /// Returns an error in WAL-only mode. + /// Returns an error in WAL-only mode, or if the writer is poisoned. pub async fn scan(&self) -> Result { + self.wal_flusher.check_poisoned()?; let state_lock = self.memtable_state_lock()?; let state = state_lock.read().await; Ok(state.memtable.scan()) @@ -2106,10 +2557,11 @@ impl ShardWriter { /// Prefer [`Self::in_memory_memtable_refs`] on the read path — it also /// carries frozen-awaiting-flush generations. /// - /// Returns an error in WAL-only mode. + /// Returns an error in WAL-only mode, or if the writer is poisoned. pub async fn active_memtable_ref( &self, ) -> Result { + self.wal_flusher.check_poisoned()?; let state_lock = self.memtable_state_lock()?; let state = state_lock.read().await; Ok(in_memory_ref(&state.memtable)) @@ -2121,10 +2573,11 @@ impl ShardWriter { /// path uses this instead of [`Self::active_memtable_ref`] so a /// concurrent reader sees no hole while a flush drains. /// - /// Returns an error in WAL-only mode. + /// Returns an error in WAL-only mode, or if the writer is poisoned. pub async fn in_memory_memtable_refs( &self, ) -> Result { + self.wal_flusher.check_poisoned()?; let state_lock = self.memtable_state_lock()?; let state = state_lock.read().await; Ok(crate::dataset::mem_wal::scanner::InMemoryMemTables { @@ -2155,10 +2608,16 @@ impl ShardWriter { } } - /// Seal the active memtable so it's queued for L0 flush. No-op when - /// the active memtable is empty. Errors in WAL-only mode or if this - /// writer has been fenced by a successor. Pair with - /// [`Self::wait_for_flush_drain`] to wait for the queued flush. + /// Seal the active memtable so it's queued for L0 flush. Errors in + /// WAL-only mode or if this writer has been fenced by a successor. + /// + /// The returned [`SealFence`] is what makes a *bounded* wait possible: + /// a caller that needs "everything written before my seal is in L0" + /// awaits [`SealFence::wait`], which covers the flushes outstanding at + /// seal time and nothing else. [`Self::wait_for_flush_drain`] instead + /// loops until the frozen set is *empty*, re-collecting it each round, + /// so it also waits on every memtable frozen *while it waits*. Under + /// sustained writes that set may never empty. /// /// Beyond test setup where deterministic flush points are required, /// this is the primary lever for callers that need to drive flushes @@ -2168,7 +2627,7 @@ impl ShardWriter { /// the next epoch starts with no replayable entries from the old /// layout. #[instrument(name = "sw_force_seal_active", level = "info", skip_all, fields(shard_id = %self.config.shard_id, epoch = self.epoch))] - pub async fn force_seal_active(&self) -> Result<()> { + pub async fn force_seal_active(&self) -> Result { match &self.mode { WriterMode::MemTable { state, @@ -2178,11 +2637,28 @@ impl ShardWriter { self.check_fenced().await?; self.wal_flusher.check_poisoned()?; let mut state = state.write().await; - if state.memtable.batch_count() == 0 { - return Ok(()); - } - writer_state.freeze_memtable(&mut state)?; - Ok(()) + let sealed_generation = if state.memtable.batch_count() == 0 { + None + } else { + let generation = state.memtable.generation(); + writer_state.freeze_memtable(&mut state)?; + Some(generation) + }; + // Capture the outstanding set under the same lock that froze, + // so no freeze can slip between the two and widen the fence. + // It covers more than the generation just sealed: a + // size/interval trigger freezes generations asynchronously, so + // an empty active memtable does *not* mean every pre-call write + // already reached L0. Watchers are popped as flushes settle, so + // whatever remains here is exactly what is still owed. + Ok(SealFence { + sealed_generation, + watchers: state + .frozen_flush_watchers + .iter() + .map(|(_, w)| w.clone()) + .collect(), + }) } WriterMode::WalOnly { .. } => Err(Error::invalid_input( "force_seal_active not available in WAL-only mode (no MemTable)", @@ -2196,12 +2672,10 @@ impl ShardWriter { /// want everything-on-disk. Errors in WAL-only mode, or if any /// awaited flush reports `DurabilityResult::Failed`. /// - /// Useful in tests for deterministic post-flush assertions, and in - /// production wherever a caller needs a synchronous fence after - /// [`Self::force_seal_active`] — e.g. trimming memtable residency - /// across shards in a multi-table WAL writer, or ensuring the WAL - /// is fully drained to Lance storage before rolling to a new - /// format/epoch. + /// Useful in tests for deterministic post-flush assertions. In + /// production prefer [`SealFence::wait`] from the seal itself: this + /// drains the queue to empty, including memtables frozen after the + /// call, so under sustained writes it may not return. #[instrument(name = "sw_wait_for_flush_drain", level = "info", skip_all, fields(shard_id = %self.config.shard_id, epoch = self.epoch))] pub async fn wait_for_flush_drain(&self) -> Result<()> { let state_lock = match &self.mode { @@ -2268,12 +2742,65 @@ impl ShardWriter { Ok(()) } + /// Send the close-time final WAL flush and await its completion. + /// + /// Sends directly on the flush channel rather than via + /// [`WalFlusher::trigger_flush`]: the latter silently returns `Ok` when the + /// flusher's `flush_tx` is unset, which would let close report success + /// without ever persisting the final WAL entry. A closed send channel must + /// surface as an error here so close never acknowledges durability it did + /// not achieve. + async fn flush_final_wal( + wal_flush_tx: &mpsc::UnboundedSender, + source: WalFlushSource, + end_batch_position: usize, + ) -> Result<()> { + let done = WatchableOnceCell::new(); + let mut reader = done.reader(); + if wal_flush_tx + .send(TriggerWalFlush { + source, + end_batch_position, + done: Some(done), + }) + .is_err() + { + return Err(Error::io("WAL flush channel closed during close")); + } + + match reader.await_value().await { + Some(Ok(_)) => Ok(()), + Some(Err(failure)) => Err(failure.into_error()), + None => Err(Error::io( + "WAL flush handler exited before reporting durability during close", + )), + } + } + + fn merge_close_stage( + close_result: Result<()>, + stage: &str, + stage_result: Result<()>, + ) -> Result<()> { + if let (Err(_), Err(stage_error)) = (&close_result, &stage_result) { + warn!("Close stage '{stage}' also failed: {stage_error}"); + } + close_result.and(stage_result) + } + /// Close the writer gracefully. /// /// Flushes pending data and shuts down background tasks. + /// + /// # Errors + /// + /// Returns an error if pending WAL data cannot be persisted, an active or + /// frozen MemTable cannot be flushed, a flush handler exits before reporting + /// completion, or background tasks cannot be shut down. #[instrument(name = "sw_close", level = "info", skip_all, fields(shard_id = %self.config.shard_id, epoch = self.epoch))] pub async fn close(self) -> Result<()> { info!("Closing ShardWriter for shard {}", self.config.shard_id); + let mut close_result: Result<()> = Ok(()); match &self.mode { WriterMode::MemTable { @@ -2281,31 +2808,46 @@ impl ShardWriter { writer_state, .. } => { - // Send final WAL flush message and wait for completion + // Drain *both* tasks against the active memtable. The index apply + // and the WAL append are independent now, so closing has to + // settle both: the L0 flush below turns this memtable into a + // Lance generation, and a generation whose indexes never saw the + // tail is a generation with a hole in it. let st = state.read().await; let batch_store = st.memtable.batch_store(); let indexes = st.memtable.indexes_arc(); let batch_count = st.memtable.batch_count(); drop(st); + if batch_count > 0 + && let Some(indexes) = indexes + && indexes.indexed_count() < batch_count + { + let mut watcher = self.wal_flusher.track_batch( + Some(Arc::clone(&indexes)), + batch_count, + 0, // durability is settled by the WAL flush below + ); + writer_state.trigger_index_apply( + Arc::clone(&batch_store), + indexes, + batch_count, + )?; + watcher.wait().await?; + } + if batch_count > 0 { - let done = WatchableOnceCell::new(); - let reader = done.reader(); - if writer_state - .wal_flush_tx - .send(TriggerWalFlush { - source: WalFlushSource::BatchStore { - batch_store, - indexes, - }, - end_batch_position: batch_count, - done: Some(done), - }) - .is_ok() - { - let mut reader = reader; - let _ = reader.await_value().await; - } + // Append-only source: on this branch the index apply is a + // separate task (drained above), so the final WAL flush carries + // no indexes. #7769's failure propagation still applies. + let stage_result = Self::flush_final_wal( + &writer_state.wal_flush_tx, + WalFlushSource::BatchStore { batch_store }, + batch_count, + ) + .await; + close_result = + Self::merge_close_stage(close_result, "final WAL flush", stage_result); } // Freeze the active memtable (if any rows) so it joins the @@ -2321,22 +2863,37 @@ impl ShardWriter { // Propagate any freeze error: at close time the caller // has explicitly asked for full durability, so silently // dropping a freeze failure would lose data without any - // signal. If freeze fails, surface the error rather than - // continuing on to drain only the pre-existing frozen - // memtables (whose flushes can still be waited on, but - // the caller now knows the close was incomplete). + // signal. If freeze fails, its error is recorded as the + // first causal failure, but close still drains any + // pre-existing frozen MemTable watchers so a successor + // failure is logged without replacing the first error. let watchers: Vec<_> = { let mut st = state.write().await; if st.memtable.row_count() > 0 { - writer_state.freeze_memtable(&mut st)?; + let freeze_result = writer_state.freeze_memtable(&mut st).map(|_| ()); + close_result = Self::merge_close_stage( + close_result, + "active MemTable freeze", + freeze_result, + ); } st.frozen_flush_watchers .iter() .map(|(_, w)| w.clone()) .collect() }; - for mut w in watchers { - let _ = w.await_value().await; + for mut watcher in watchers { + let stage_result = match watcher.await_value().await { + Some(durability) => durability.into_result(), + None => Err(Error::io( + "MemTable flush handler exited before reporting completion during close", + )), + }; + close_result = Self::merge_close_stage( + close_result, + "frozen MemTable flush watcher", + stage_result, + ); } } WriterMode::WalOnly { @@ -2349,30 +2906,32 @@ impl ShardWriter { let pending = state.batch_count(); let end_position = state.next_batch_position(); if pending > 0 { - let done = WatchableOnceCell::new(); - let reader = done.reader(); - if wal_flush_tx - .send(TriggerWalFlush { - source: WalFlushSource::WalOnly { - state: state.clone(), - }, - end_batch_position: end_position, - done: Some(done), - }) - .is_ok() - { - let mut reader = reader; - let _ = reader.await_value().await; - } + let stage_result = Self::flush_final_wal( + wal_flush_tx, + WalFlushSource::WalOnly { + state: state.clone(), + }, + end_position, + ) + .await; + close_result = + Self::merge_close_stage(close_result, "final WAL flush", stage_result); } } } // Shutdown background tasks - self.task_executor.shutdown_all().await?; - - info!("ShardWriter closed for shard {}", self.config.shard_id); - Ok(()) + let shutdown_result = self.task_executor.shutdown_all().await; + let close_result = Self::merge_close_stage(close_result, "task shutdown", shutdown_result); + + match &close_result { + Ok(()) => info!("ShardWriter closed for shard {}", self.config.shard_id), + Err(error) => warn!( + "ShardWriter close for shard {} failed: {error}", + self.config.shard_id + ), + } + close_result } } @@ -2384,12 +2943,30 @@ pub struct MemTableStats { pub estimated_size: usize, pub generation: u64, pub max_buffered_batch_position: Option, - pub max_flushed_batch_position: Option, + /// Writer-global count of WAL-durable batches. Exclusive: 0 means none. + /// Compare against `global_offset + batch_count` to see what this memtable + /// still owes the WAL. + pub durable_batch_count: usize, + /// Writer-global coordinate of this memtable's batch 0. + pub global_offset: usize, pub pending_wal_start_batch_position: Option, pub pending_wal_end_batch_position: Option, pub pending_wal_batch_count: usize, pub pending_wal_row_count: usize, pub pending_wal_estimated_bytes: usize, + /// Frozen memtables in the read view: sealed-awaiting-flush, plus flushed + /// ones still inside `frozen_memtable_grace`. + pub frozen_count: usize, + /// Heap bytes still owed to flush: frozen memtables awaiting a first flush, + /// plus any left resident by a failed one. Drains on flush commit, so unlike + /// `frozen_count` it excludes in-grace tables. + /// + /// Plus the active memtable's `estimated_size`, this is roughly what + /// backpressure meters against `max_unflushed_memtable_bytes` — but only + /// roughly: backpressure's own counter drains whenever a flush *completes*, + /// so bytes stranded by a failed flush stay visible here while no longer + /// throttling writes. + pub frozen_bytes: usize, } /// WAL statistics. @@ -2399,16 +2976,126 @@ pub struct WalStats { pub next_wal_entry_position: u64, } -/// Background handler for WAL flush operations. +/// The L0 flushes outstanding when [`ShardWriter::force_seal_active`] +/// returned — the exact predicate for "everything written before that call +/// is in L0". +/// +/// The set is fixed at seal time, so [`Self::wait`] is bounded no matter how +/// many memtables freeze while it waits, and each entry reports the outcome of +/// its own flush. That is why the fence is a captured watcher set and not a +/// generation number compared against `ShardManifest::current_generation`: the +/// manifest advances to `generation + 1` on every committed flush without +/// checking for a gap, so a later generation's success moves it past a +/// generation that failed and never reached L0 — the watermark would report +/// durability that does not exist. +#[derive(Debug)] +pub struct SealFence { + sealed_generation: Option, + watchers: Vec, +} + +impl SealFence { + /// The generation the seal froze, or `None` when the active memtable + /// was empty (a no-op seal). Independent of what the fence covers: an + /// empty active memtable says nothing about generations frozen earlier + /// and still awaiting flush, which the fence still waits on. + pub fn sealed_generation(&self) -> Option { + self.sealed_generation + } + + /// Block until every flush this fence covers has landed in L0. + /// + /// Errors if any of them reports `DurabilityResult::Failed`, or if the + /// flush handler exited without reporting. + pub async fn wait(self) -> Result<()> { + for mut watcher in self.watchers { + match watcher.await_value().await { + Some(durability) => durability.into_result()?, + None => { + return Err(Error::io( + "MemTable flush handler exited before reporting completion", + )); + } + } + } + Ok(()) + } +} + +/// The oldest store that still owes the WAL an append, or `None` when everything +/// is durable. +/// +/// Ordering is the whole point. WAL entry positions are assigned in append-call +/// order; replay walks them ascending; row positions follow; and primary-key +/// recency is "newest visible row position wins". So appending a newer memtable +/// ahead of an older one's tail silently inverts dedup after a crash. +/// +/// Taking "the active memtable" would do exactly that, because a timer tick +/// enqueued before a freeze is handled after it and would resolve to the incoming +/// memtable. Selecting by cursor instead makes the target a function of what is +/// actually durable, not of when the timer happened to fire. +fn next_pending_store( + frozen: impl Iterator>, + active: Arc, + durable: usize, +) -> Option> { + frozen + .chain(std::iter::once(active)) + .find(|store| store.global_end() > durable) +} + +/// The index-apply task: one sequential consumer of the index-apply channel. /// -/// This handler does parallel WAL I/O + index updates during flush. -/// Indexes are passed through the TriggerWalFlush message. +/// Sequential consumption is the safety property. `HnswGraph::insert_batch` hard- +/// rejects any range whose start is not its `indexed_len`, so the apply must see +/// contiguous, in-order ranges — and a single consumer guarantees that +/// regardless of how many putters race behind it. Ordering comes from the task, +/// not from a flush interval, which is why triggering per-put is exactly as safe +/// as triggering on a timer, and lets a put become visible in milliseconds +/// instead of waiting on an S3 round-trip. +struct IndexApplyHandler { + cursors: Arc, + wal_flusher: Arc, + stats: SharedWriteStats, +} + +#[async_trait] +impl MessageHandler for IndexApplyHandler { + async fn handle(&mut self, message: TriggerIndexApply) -> Result<()> { + match apply_index_range(&self.cursors, message).await { + Ok(applied) => { + // A coalesced no-op indexes nothing (`rows_indexed == 0`); + // recording it would inflate the count and skew avg latency. + if applied.rows_indexed > 0 { + self.stats + .record_index_update(applied.duration, applied.rows_indexed); + } + Ok(()) + } + // An index apply cannot be partially rolled back, so a failure is + // terminal: poison, and let reopen rebuild the indexes from the WAL. + // See the note in `WalFlusher::flush_from_batch_store`. + Err(e) => { + self.wal_flusher.poison(&e); + Err(e) + } + } + } +} + struct WalFlushHandler { wal_flusher: Arc, /// MemTable-mode writer state, used to detect "frozen vs active" flushes /// via Arc::ptr_eq on the active batch_store. `None` when running in /// WAL-only mode (no MemTable, no frozen-vs-active distinction). memtable_state: Option>>, + /// WAL-only-mode pending queue, so a background tick can resolve to the + /// batches still owed an append. `None` in MemTable mode. Exactly one of + /// `memtable_state` / `wal_only_state` is `Some`. + wal_only_state: Option>, + /// How often to append in the background. `None` disables the ticker, leaving + /// the append size-triggered (and freeze/close-triggered) only. + flush_interval: Option, stats: SharedWriteStats, } @@ -2416,11 +3103,15 @@ impl WalFlushHandler { fn new( wal_flusher: Arc, memtable_state: Option>>, + wal_only_state: Option>, + flush_interval: Option, stats: SharedWriteStats, ) -> Self { Self { wal_flusher, memtable_state, + wal_only_state, + flush_interval, stats, } } @@ -2428,6 +3119,33 @@ impl WalFlushHandler { #[async_trait] impl MessageHandler for WalFlushHandler { + /// Append periodically in the background. + /// + /// This is what the flush interval was always supposed to mean. It routed to + /// a timer that was only ever *evaluated on the write path*, so it could add + /// a redundant trigger but never delay or batch one — with every durable put + /// triggering its own append, tuning the knob did nothing at all. + /// + /// The ticker exists to bound S3 API cost: an append is a PUT, billed per + /// call, and it is the only thing on this schedule. The index apply is not — + /// it is in-memory and free to batch, so it runs per-put on its own task. + fn tickers(&mut self) -> Vec<(Duration, MessageFactory)> { + // No interval => no ticker. A zero interval would panic in tokio. + let Some(interval) = self.flush_interval.filter(|d| !d.is_zero()) else { + return vec![]; + }; + // The tick names no store: `MessageFactory` is synchronous and cannot take + // the async state lock. `handle()` resolves it against the cursor. + vec![( + interval, + Box::new(|| TriggerWalFlush { + source: WalFlushSource::NextPending, + end_batch_position: 0, + done: None, + }), + )] + } + async fn handle(&mut self, message: TriggerWalFlush) -> Result<()> { let TriggerWalFlush { source, @@ -2435,6 +3153,16 @@ impl MessageHandler for WalFlushHandler { done, } = message; + // A timer tick names no store — resolve it now, at handle time. + let (source, end_batch_position) = match source { + WalFlushSource::NextPending => match self.resolve_next_pending().await { + Some(resolved) => resolved, + // Everything is already durable; the tick has nothing to do. + None => return Ok(()), + }, + other => (other, end_batch_position), + }; + let result = self.do_flush(source, end_batch_position).await; // Propagate the just-appended WAL entry position back into the @@ -2465,6 +3193,64 @@ impl MessageHandler for WalFlushHandler { } impl WalFlushHandler { + /// Pick the store the WAL still owes an append, oldest first. + /// + /// **WAL entries must be appended in global batch-position order for the + /// writer's lifetime.** `WalAppender::append` assigns each entry's position + /// from its own counter, in call order; replay walks those positions + /// ascending and assigns row positions in that order; and primary-key recency + /// is "newest visible row position wins". So append order fixes dedup order. + /// Append two memtables out of order and replay silently hands the dedup to + /// the *stale* row — corruption that survives the crash that caused it, and + /// that a full scan cannot see. + /// + /// Resolving to "the active memtable" would break exactly that: a tick + /// enqueued before a freeze is handled after it, resolves to the incoming + /// memtable, and appends its batches ahead of the outgoing memtable's tail. + /// So the target is a function of `durable`, not of when the timer fired. + /// + /// Safe to walk the frozen list because a store that still owes an append + /// cannot be swept: its L0 flush is blocked on the completion cell that only + /// that append fires. + /// + /// In WAL-only mode there is a single FIFO pending queue and no memtable + /// rotation, so the ordering hazard above cannot arise: the tick resolves to + /// the queue whenever it holds un-appended batches. + async fn resolve_next_pending(&self) -> Option<(WalFlushSource, usize)> { + if let Some(state_lock) = self.memtable_state.as_ref() { + let state = state_lock.read().await; + let durable = self.wal_flusher.durable(); + + return next_pending_store( + state + .frozen_memtables + .iter() + .map(|frozen| frozen.memtable.batch_store()), + state.memtable.batch_store(), + durable, + ) + .map(|store| { + let end = store.len(); + (WalFlushSource::BatchStore { batch_store: store }, end) + }); + } + + let state = self.wal_only_state.as_ref()?; + if state.batch_count() == 0 { + // Everything already appended; the tick has nothing to do. + return None; + } + // `flush_from_wal_only` snapshots the whole queue, so the end position + // is informational here; carry the next position for symmetry. + let end = state.next_batch_position(); + Some(( + WalFlushSource::WalOnly { + state: Arc::clone(state), + }, + end, + )) + } + /// Unified flush method for both active and frozen memtables and for /// WAL-only mode. /// @@ -2484,18 +3270,6 @@ impl WalFlushHandler { ) -> Result { let start = Instant::now(); - // Whether this flush actually updates any in-memory indexes — only - // a BatchStore source carrying a non-empty `IndexStore` does. Used - // to gate the `record_index_update` stat so WAL-only flushes don't - // pollute the index-update counters. - let has_indexes = matches!( - &source, - WalFlushSource::BatchStore { - indexes: Some(_), - .. - } - ); - // Early-out for BatchStore sources where the watermark already // covers the requested end position. Detection of "frozen flush" // requires the active memtable's batch_store; WAL-only handlers @@ -2503,8 +3277,7 @@ impl WalFlushHandler { // BatchStore source, so the early-out simplifies to the watermark // comparison. if let WalFlushSource::BatchStore { batch_store, .. } = &source { - let max_flushed = batch_store.max_flushed_batch_position(); - let flushed_up_to = max_flushed.map(|p| p + 1).unwrap_or(0); + let flushed_up_to = batch_store.local_end(self.wal_flusher.durable()); let is_frozen_flush = if let Some(state_lock) = &self.memtable_state { let state = state_lock.read().await; !Arc::ptr_eq(batch_store, &state.memtable.batch_store()) @@ -2529,12 +3302,6 @@ impl WalFlushHandler { self.stats .record_wal_flush(start.elapsed(), flush_result.wal_bytes); self.stats.record_wal_io(flush_result.wal_io_duration); - if has_indexes { - self.stats.record_index_update( - flush_result.index_update_duration, - flush_result.rows_indexed, - ); - } } Ok(flush_result) @@ -2550,10 +3317,13 @@ impl WalFlushHandler { struct MemTableFlushHandler { state: Arc>, flusher: Arc, + /// Source of the writer-global durability cursor, which the L0 flush asserts + /// covers the whole frozen memtable before it writes a generation. + wal_flusher: Arc, epoch: u64, - /// Secondary index configs to rebuild on each flushed generation. When + /// Secondary index configs to rebuild on each SSTable. When /// non-empty the handler flushes via [`MemTableFlusher::flush_with_indexes`] - /// so queries over flushed generations use index lookups instead of full + /// so queries over SSTables use index lookups instead of full /// scans — and so vector search's index-only `fast_search` can see the data /// at all. index_configs: Vec, @@ -2564,9 +3334,11 @@ struct MemTableFlushHandler { } impl MemTableFlushHandler { + #[allow(clippy::too_many_arguments)] fn new( state: Arc>, flusher: Arc, + wal_flusher: Arc, epoch: u64, index_configs: Vec, stats: SharedWriteStats, @@ -2575,6 +3347,7 @@ impl MemTableFlushHandler { Self { state, flusher, + wal_flusher, epoch, index_configs, stats, @@ -2681,14 +3454,20 @@ impl MemTableFlushHandler { let covered_wal_entry_position = wal_flushed_position .or_else(|| memtable.frozen_at_wal_entry_position()) .unwrap_or(0); - // Rebuild secondary indexes on the flushed generation so later + // Rebuild secondary indexes on the SSTable so later // queries hit an index instead of scanning. Skip the extra // dataset open when there are no indexes to build. The indexed // path's future is boxed to keep this async block's nesting // under the type-layout recursion limit. + // Read the durability cursor *after* the WAL-append completion above, + // not before: the append that makes this memtable durable is the very + // thing we just waited on, so a cursor sampled earlier would still be + // short of it and trip the flush precondition. + let durable = self.wal_flusher.durable(); + if self.index_configs.is_empty() { self.flusher - .flush(&memtable, self.epoch, covered_wal_entry_position) + .flush(&memtable, self.epoch, covered_wal_entry_position, durable) .await } else { Box::pin(self.flusher.flush_with_indexes( @@ -2696,6 +3475,7 @@ impl MemTableFlushHandler { self.epoch, &self.index_configs, covered_wal_entry_position, + durable, )) .await } @@ -2725,15 +3505,15 @@ impl MemTableFlushHandler { // the read union until a later flush or WAL replay, else a transient // error reopens the hole. if flush_result.is_ok() { - let flushed_generation = memtable.generation(); + let sstable = memtable.generation(); if self.grace.is_zero() { state .frozen_memtables - .retain(|frozen| frozen.memtable.generation() != flushed_generation); + .retain(|frozen| frozen.memtable.generation() != sstable); } else { let now = now_millis(); for frozen in state.frozen_memtables.iter_mut() { - if frozen.memtable.generation() == flushed_generation { + if frozen.memtable.generation() == sstable { frozen.flushed_at_ms = Some(now); } } @@ -2748,7 +3528,7 @@ impl MemTableFlushHandler { info!( "Flushed frozen memtable generation {} ({} rows in {:?})", - result.generation.generation, + result.sstable.generation, result.rows_flushed, start.elapsed() ); @@ -2935,11 +3715,7 @@ impl WriteStatsSnapshot { /// Get average WAL flush size in bytes. pub fn avg_wal_flush_bytes(&self) -> Option { - if self.wal_flush_count > 0 { - Some(self.wal_flush_bytes / self.wal_flush_count) - } else { - None - } + self.wal_flush_bytes.checked_div(self.wal_flush_count) } /// Get WAL write throughput (bytes per second based on WAL flush time). @@ -2971,11 +3747,7 @@ impl WriteStatsSnapshot { /// Get average rows per index update. pub fn avg_index_update_rows(&self) -> Option { - if self.index_update_count > 0 { - Some(self.index_update_rows / self.index_update_count) - } else { - None - } + self.index_update_rows.checked_div(self.index_update_count) } /// Get average MemTable flush latency. @@ -2989,11 +3761,8 @@ impl WriteStatsSnapshot { /// Get average MemTable flush size in rows. pub fn avg_memtable_flush_rows(&self) -> Option { - if self.memtable_flush_count > 0 { - Some(self.memtable_flush_rows / self.memtable_flush_count) - } else { - None - } + self.memtable_flush_rows + .checked_div(self.memtable_flush_count) } /// Log stats summary using tracing (for structured telemetry). @@ -3052,6 +3821,7 @@ mod tests { use arrow_array::{Int32Array, StringArray}; use arrow_schema::{DataType, Field}; use lance_core::FenceReason; + use rstest::rstest; use tempfile::TempDir; async fn create_local_store() -> (Arc, Path, String, TempDir) { @@ -3061,6 +3831,26 @@ mod tests { (store, path, uri, temp_dir) } + #[test] + fn test_merge_close_stage_preserves_first_error() { + let result = ShardWriter::merge_close_stage( + Err(Error::io("primary close error")), + "secondary close stage", + Err(Error::io("secondary close error")), + ); + + let error = result.expect_err("close must preserve the first error"); + assert!(matches!(&error, Error::IO { .. })); + assert!( + error.to_string().contains("primary close error"), + "unexpected error: {error}" + ); + assert!( + !error.to_string().contains("secondary close error"), + "secondary error replaced the primary error: {error}" + ); + } + /// Base schema with `id` marked as the unenforced primary key (delete needs /// a PK). `name` is nullable so a tombstone can null it. fn create_pk_test_schema() -> Arc { @@ -3076,6 +3866,17 @@ mod tests { ])) } + /// [`create_pk_test_schema`] with a non-nullable `name` — the shape that + /// used to make `delete` fail. + fn create_strict_pk_test_schema() -> Arc { + let fields: Vec = create_pk_test_schema() + .fields() + .iter() + .map(|f| f.as_ref().clone().with_nullable(false)) + .collect(); + Arc::new(ArrowSchema::new(fields)) + } + fn id_only_keys(ids: &[i32]) -> RecordBatch { RecordBatch::try_new( Arc::new(ArrowSchema::new(vec![Field::new( @@ -3088,12 +3889,61 @@ mod tests { .unwrap() } + /// Two same-typed columns handed over swapped pass every positional check + /// downstream, so the logical-schema gate has to catch them by name. + #[tokio::test] + async fn test_put_rejects_swapped_same_typed_columns() { + let (store, base_path, base_uri, _temp) = create_local_store().await; + let schema = Arc::new(ArrowSchema::new(vec![ + Field::new("name", DataType::Utf8, true), + Field::new("email", DataType::Utf8, true), + ])); + let writer = ShardWriter::open( + store, + base_path, + base_uri, + ShardWriterConfig { + shard_id: Uuid::new_v4(), + ..Default::default() + }, + schema, + vec![], + ) + .await + .unwrap(); + + let swapped = RecordBatch::try_new( + Arc::new(ArrowSchema::new(vec![ + Field::new("email", DataType::Utf8, true), + Field::new("name", DataType::Utf8, true), + ])), + vec![ + Arc::new(StringArray::from(vec!["a@example.com"])), + Arc::new(StringArray::from(vec!["a"])), + ], + ) + .unwrap(); + + let error = writer.put(vec![swapped]).await.unwrap_err(); + assert!( + matches!(error, Error::InvalidInput { .. }), + "expected InvalidInput, got {error:?}" + ); + let message = error.to_string(); + assert!( + message.contains("column 0") && message.contains("email") && message.contains("name"), + "error should name the position and both columns: {message}" + ); + + writer.close().await.unwrap(); + } + #[test] fn test_ensure_tombstone_column_injects_false() { let base = create_test_schema(); - let target = schema_with_tombstone(&base); - let out = ensure_tombstone_column(create_test_batch(&base, 0, 3), &target).unwrap(); - assert_eq!(out.schema(), target); + let storage = schema_with_tombstone(&base); + let out = ensure_tombstone_column(create_test_batch(&base, 0, 3), &storage).unwrap(); + assert_eq!(out.schema(), storage); let ts = out .column_by_name(TOMBSTONE) .unwrap() @@ -3105,16 +3955,16 @@ mod tests { "put injects _tombstone = false" ); // Idempotent: a batch already carrying the column passes through. - let again = ensure_tombstone_column(out.clone(), &target).unwrap(); + let again = ensure_tombstone_column(out.clone(), &storage).unwrap(); assert_eq!(again.schema(), out.schema()); } #[test] fn test_build_tombstone_batch_shape() { - let target = schema_with_tombstone(&create_test_schema()); + let storage = schema_with_tombstone(&create_test_schema()); let tomb = - build_tombstone_batch(&id_only_keys(&[5, 7]), &target, &["id".to_string()]).unwrap(); - assert_eq!(tomb.schema(), target); + build_tombstone_batch(&id_only_keys(&[5, 7]), &storage, &["id".to_string()]).unwrap(); + assert_eq!(tomb.schema(), storage); assert_eq!(tomb.num_rows(), 2); let ids = tomb .column_by_name("id") @@ -3139,7 +3989,7 @@ mod tests { #[test] fn test_build_tombstone_batch_missing_pk_errors() { - let target = schema_with_tombstone(&create_test_schema()); + let storage = schema_with_tombstone(&create_test_schema()); let keys = RecordBatch::try_new( Arc::new(ArrowSchema::new(vec![Field::new( "other", @@ -3149,18 +3999,62 @@ mod tests { vec![Arc::new(Int32Array::from(vec![1]))], ) .unwrap(); - assert!(build_tombstone_batch(&keys, &target, &["id".to_string()]).is_err()); + assert!(build_tombstone_batch(&keys, &storage, &["id".to_string()]).is_err()); + } + + #[test] + fn test_build_tombstone_batch_nulls_non_nullable_base_column() { + // The point of the storage schema: a tombstone nulls `v` even though the + // base table declares it non-nullable. + let pk = ["id".to_string()]; + let base = Arc::new(ArrowSchema::new(vec![ + Field::new("id", DataType::Int32, false), + Field::new("v", DataType::Int32, false), + ])); + let storage = relax_non_pk_nullability(&schema_with_tombstone(&base), &pk); + + let batch = build_tombstone_batch(&id_only_keys(&[1]), &storage, &pk).unwrap(); + + assert!(batch["v"].is_null(0), "the tombstone must null `v`"); + assert!(!batch["id"].is_null(0), "the primary key survives"); + assert!( + batch[TOMBSTONE] + .as_any() + .downcast_ref::() + .unwrap() + .value(0) + ); } #[test] - fn test_build_tombstone_batch_non_nullable_nonpk_errors() { - // A tombstone must null every non-PK column; a non-nullable one fails. + fn test_build_tombstone_batch_rejects_null_primary_key() { + // Primary keys are never relaxed, so the storage schema still rejects a + // null key — the delete path needs no separate check for it. + let pk = ["id".to_string()]; let base = Arc::new(ArrowSchema::new(vec![ Field::new("id", DataType::Int32, false), Field::new("v", DataType::Int32, false), ])); - let target = schema_with_tombstone(&base); - assert!(build_tombstone_batch(&id_only_keys(&[1]), &target, &["id".to_string()]).is_err()); + let storage = relax_non_pk_nullability(&schema_with_tombstone(&base), &pk); + let keys = RecordBatch::try_new( + Arc::new(ArrowSchema::new(vec![Field::new( + "id", + DataType::Int32, + true, + )])), + vec![Arc::new(Int32Array::from(vec![None::]))], + ) + .unwrap(); + + let error = build_tombstone_batch(&keys, &storage, &pk).unwrap_err(); + assert!( + matches!(error, Error::InvalidInput { .. }), + "expected InvalidInput, got {error:?}" + ); + assert!( + error.to_string().contains("non-nullable"), + "error should name the nullability violation: {error}" + ); } #[tokio::test] @@ -3229,20 +4123,19 @@ mod tests { writer.close().await.unwrap(); } - /// `delete_no_wait` lands the tombstone in the in-memory tier (visible at - /// the batch-store level the instant it returns) and hands back the - /// durability watcher *without* awaiting it. Index-driven LSM read - /// visibility — folding the tombstone out — follows once the watcher - /// resolves the flush that advances the visibility watermark and updates - /// the PK index. The delete analog of - /// `test_put_no_wait_durable_visible_then_durable`. + /// Delete works against a base table with non-nullable non-PK columns, and + /// survivors come back through the narrowing egress relabel intact. #[tokio::test] - async fn test_shard_writer_delete_no_wait_durable_visible_after_watcher() { + async fn test_delete_against_non_nullable_base_column_round_trip() { use crate::dataset::mem_wal::scanner::LsmScanner; use futures::TryStreamExt; let (store, base_path, base_uri, _temp) = create_local_store().await; - let schema = create_pk_test_schema(); + let schema = create_strict_pk_test_schema(); + assert!( + !schema.field_with_name("name").unwrap().is_nullable(), + "the point of this test is a non-nullable non-PK column" + ); let config = ShardWriterConfig { shard_id: Uuid::new_v4(), durable_write: true, @@ -3264,24 +4157,7 @@ mod tests { .put(vec![create_test_batch(&schema, 0, 5)]) .await .unwrap(); - - // Tombstone id=2 without blocking on durability. - let (_result, watcher) = writer - .delete_no_wait(vec![id_only_keys(&[2])]) - .await - .unwrap(); - - // The tombstone is in the in-memory tier immediately (5 rows + 1 - // tombstone), even though the index-driven read can't fold it until the - // flush behind the watcher lands. `durable_write` is on, so a watcher is - // returned to await. - assert_eq!(writer.memtable_stats().await.unwrap().row_count, 6); - let mut watcher = watcher.expect("durable_write returns a watcher"); - - // Awaiting the watcher waits for the flush, which advances the - // visibility watermark and updates the PK index — only then does the LSM - // read fold the delete. - watcher.wait().await.unwrap(); + writer.delete(vec![id_only_keys(&[2])]).await.unwrap(); let refs = writer.in_memory_memtable_refs().await.unwrap(); let scanner = LsmScanner::without_base_table( @@ -3298,69 +4174,243 @@ mod tests { .try_collect() .await .unwrap(); - let mut ids: Vec = Vec::new(); + + let mut rows: Vec<(i32, String)> = Vec::new(); for b in &batches { - let arr = b - .column_by_name("id") - .unwrap() - .as_any() - .downcast_ref::() - .unwrap(); - ids.extend((0..arr.len()).map(|i| arr.value(i))); + assert!( + !b.schema().field_with_name("name").unwrap().is_nullable(), + "egress must narrow back to the logical schema" + ); + let ids = b["id"].as_any().downcast_ref::().unwrap(); + let names = b["name"].as_any().downcast_ref::().unwrap(); + rows.extend((0..ids.len()).map(|i| (ids.value(i), names.value(i).to_string()))); } - ids.sort_unstable(); + rows.sort_unstable(); + assert_eq!( - ids, - vec![0, 1, 3, 4], - "delete folded by the LSM read once the watcher resolves" + rows, + vec![ + (0, "name_0".to_string()), + (1, "name_1".to_string()), + (3, "name_3".to_string()), + (4, "name_4".to_string()), + ], + "id=2 deleted; every survivor keeps its non-nullable value" ); writer.close().await.unwrap(); } - /// With `durable_write` off, `delete_no_wait` returns no watcher (nothing to - /// await), but the tombstone still lands in the in-memory tier. The delete - /// analog of `test_put_no_wait_non_durable_returns_no_watcher`. + /// The storage schema no longer rejects a caller's null, so `put` is the + /// only thing standing between a null and a non-nullable base column. #[tokio::test] - async fn test_shard_writer_delete_no_wait_non_durable_returns_no_watcher() { + async fn test_put_rejects_null_in_non_nullable_base_column() { let (store, base_path, base_uri, _temp) = create_local_store().await; - let schema = create_pk_test_schema(); - let config = ShardWriterConfig { - shard_id: Uuid::new_v4(), - durable_write: false, - ..Default::default() - }; - let writer = ShardWriter::open(store, base_path, base_uri, config, schema.clone(), vec![]) - .await - .unwrap(); - - writer - .put(vec![create_test_batch(&schema, 0, 5)]) - .await - .unwrap(); - - let (_result, watcher) = writer - .delete_no_wait(vec![id_only_keys(&[2])]) - .await - .unwrap(); - assert!(watcher.is_none(), "non-durable delete has nothing to await"); + let schema = create_strict_pk_test_schema(); + let writer = ShardWriter::open( + store, + base_path, + base_uri, + ShardWriterConfig { + shard_id: Uuid::new_v4(), + ..Default::default() + }, + schema.clone(), + vec![], + ) + .await + .unwrap(); - // Tombstone landed in the in-memory tier (5 rows + 1 tombstone). - assert_eq!(writer.memtable_stats().await.unwrap().row_count, 6); + let error = writer.put(vec![null_name_batch()]).await.unwrap_err(); + assert!( + matches!(error, Error::InvalidInput { .. }), + "expected InvalidInput, got {error:?}" + ); + assert!( + error.to_string().contains("base table schema"), + "error should point at the schema contract: {error}" + ); writer.close().await.unwrap(); } - /// Read every surviving `id` through the full LSM read path (which folds - /// `NOT _tombstone` per source) over the writer's *flushed* generations, - /// with an optional filter. Mirrors how a query reads a WAL table after a - /// flush — the path the wallop fuzz exercised when it caught a deleted row - /// resurfacing. - async fn read_flushed_ids_via_lsm( - writer: &ShardWriter, - schema: Arc, - base_uri: &str, - shard_id: Uuid, + /// WAL-only mode validates too — it has no memtable, so before this gate + /// nothing checked its input at all. + #[tokio::test] + async fn test_wal_only_put_rejects_null_in_non_nullable_base_column() { + let (store, base_path, base_uri, _temp) = create_local_store().await; + let schema = create_strict_pk_test_schema(); + let writer = ShardWriter::open( + store, + base_path, + base_uri, + wal_only_config(Uuid::new_v4()), + schema.clone(), + vec![], + ) + .await + .unwrap(); + + let error = writer.put(vec![null_name_batch()]).await.unwrap_err(); + assert!( + matches!(error, Error::InvalidInput { .. }), + "expected InvalidInput, got {error:?}" + ); + + writer.close().await.unwrap(); + } + + /// A caller-shaped batch that declares `name` nullable and carries a null — + /// legal Arrow, illegal against a base table that declares it non-nullable. + fn null_name_batch() -> RecordBatch { + RecordBatch::try_new( + Arc::new(ArrowSchema::new(vec![ + Field::new("id", DataType::Int32, false), + Field::new("name", DataType::Utf8, true), + ])), + vec![ + Arc::new(Int32Array::from(vec![0, 1])), + Arc::new(StringArray::from(vec![Some("a"), None])), + ], + ) + .unwrap() + } + + /// `delete_no_wait` lands the tombstone in the in-memory tier (visible at + /// the batch-store level the instant it returns) and hands back the + /// durability watcher *without* awaiting it. Index-driven LSM read + /// visibility — folding the tombstone out — follows once the watcher + /// resolves the flush that advances the visibility watermark and updates + /// the PK index. The delete analog of + /// `test_put_no_wait_durable_visible_then_durable`. + #[tokio::test] + async fn test_shard_writer_delete_no_wait_durable_visible_after_watcher() { + use crate::dataset::mem_wal::scanner::LsmScanner; + use futures::TryStreamExt; + + let (store, base_path, base_uri, _temp) = create_local_store().await; + let schema = create_pk_test_schema(); + let config = ShardWriterConfig { + shard_id: Uuid::new_v4(), + durable_write: true, + ..Default::default() + }; + let shard_id = config.shard_id; + let writer = ShardWriter::open( + store, + base_path, + base_uri.clone(), + config, + schema.clone(), + vec![], + ) + .await + .unwrap(); + + writer + .put(vec![create_test_batch(&schema, 0, 5)]) + .await + .unwrap(); + + // Tombstone id=2 without blocking on durability. + let (_result, watcher) = writer + .delete_no_wait(vec![id_only_keys(&[2])]) + .await + .unwrap(); + + // The tombstone is in the in-memory tier immediately (5 rows + 1 + // tombstone), even though the index-driven read can't fold it until the + // flush behind the watcher lands. `durable_write` is on, so a watcher is + // returned to await. + assert_eq!(writer.memtable_stats().await.unwrap().row_count, 6); + let mut watcher = watcher.expect("durable_write returns a watcher"); + + // Awaiting the watcher waits for the flush, which advances the + // visibility watermark and updates the PK index — only then does the LSM + // read fold the delete. + watcher.wait().await.unwrap(); + + let refs = writer.in_memory_memtable_refs().await.unwrap(); + let scanner = LsmScanner::without_base_table( + schema.clone(), + base_uri, + vec![], + vec!["id".to_string()], + ) + .with_in_memory_memtables(shard_id, refs); + let batches: Vec = scanner + .try_into_stream() + .await + .unwrap() + .try_collect() + .await + .unwrap(); + let mut ids: Vec = Vec::new(); + for b in &batches { + let arr = b + .column_by_name("id") + .unwrap() + .as_any() + .downcast_ref::() + .unwrap(); + ids.extend((0..arr.len()).map(|i| arr.value(i))); + } + ids.sort_unstable(); + assert_eq!( + ids, + vec![0, 1, 3, 4], + "delete folded by the LSM read once the watcher resolves" + ); + + writer.close().await.unwrap(); + } + + /// With `durable_write` off, `delete_no_wait` returns no watcher (nothing to + /// await), but the tombstone still lands in the in-memory tier. The delete + /// analog of `test_put_no_wait_non_durable_returns_no_watcher`. + #[tokio::test] + async fn test_non_durable_delete_is_read_your_writes() { + let (store, base_path, base_uri, _temp) = create_local_store().await; + let schema = create_pk_test_schema(); + let config = ShardWriterConfig { + shard_id: Uuid::new_v4(), + durable_write: false, + ..Default::default() + }; + let writer = ShardWriter::open(store, base_path, base_uri, config, schema.clone(), vec![]) + .await + .unwrap(); + + writer + .put(vec![create_test_batch(&schema, 0, 5)]) + .await + .unwrap(); + + let (_result, watcher) = writer + .delete_no_wait(vec![id_only_keys(&[2])]) + .await + .unwrap(); + // As with a put: a non-durable delete awaits its index apply, not an S3 + // round-trip. It is read-your-writes, just not durable. + let mut watcher = watcher.expect("a non-durable delete awaits its index apply"); + watcher.wait().await.unwrap(); + + // Tombstone landed in the in-memory tier (5 rows + 1 tombstone). + assert_eq!(writer.memtable_stats().await.unwrap().row_count, 6); + + writer.close().await.unwrap(); + } + + /// Read every surviving `id` through the full LSM read path (which folds + /// `NOT _tombstone` per source) over the writer's *flushed* generations, + /// with an optional filter. Mirrors how a query reads a WAL table after a + /// flush — the path the wallop fuzz exercised when it caught a deleted row + /// resurfacing. + async fn read_sstable_ids_via_lsm( + writer: &ShardWriter, + schema: Arc, + base_uri: &str, + shard_id: Uuid, filter: Option<&str>, ) -> Vec { use crate::dataset::mem_wal::scanner::{LsmScanner, ShardSnapshot}; @@ -3369,8 +4419,8 @@ mod tests { let manifest = writer.manifest().await.unwrap().unwrap(); let mut snapshot = ShardSnapshot::new(shard_id).with_current_generation(manifest.current_generation); - for fg in &manifest.flushed_generations { - snapshot = snapshot.with_flushed_generation(fg.generation, fg.path.clone()); + for sstable in &manifest.sstables { + snapshot = snapshot.with_sstable(sstable.generation, sstable.path.clone()); } let mut scanner = LsmScanner::without_base_table( schema, @@ -3406,14 +4456,13 @@ mod tests { ShardWriterConfig { shard_id, durable_write: false, - sync_indexed_write: true, manifest_scan_batch_size: 2, ..Default::default() } } /// Delete a key, then flush: the tombstone and the live row land in the - /// *same* flushed generation, so flush-time dedup must keep the tombstone + /// *same* SSTable, so flush-time dedup must keep the tombstone /// (newest) and the read must fold it away. Regression for the wallop /// phantom (deleted row resurfacing in a filtered read after flush). #[tokio::test] @@ -3442,14 +4491,14 @@ mod tests { writer.wait_for_flush_drain().await.unwrap(); assert_eq!( - read_flushed_ids_via_lsm(&writer, schema.clone(), &base_uri, shard_id, None).await, + read_sstable_ids_via_lsm(&writer, schema.clone(), &base_uri, shard_id, None).await, vec![0, 1, 3, 4], - "id=2 deleted before flush; tombstone must not surface in a flushed-gen scan" + "id=2 deleted before flush; tombstone must not surface in an SSTable scan" ); // The filtered read path (folds NOT _tombstone into the predicate) must // also drop it — this is the exact wallop failure shape (`id < 3`). assert_eq!( - read_flushed_ids_via_lsm(&writer, schema.clone(), &base_uri, shard_id, Some("id < 3")) + read_sstable_ids_via_lsm(&writer, schema.clone(), &base_uri, shard_id, Some("id < 3")) .await, vec![0, 1], "filtered read after flush must not resurface deleted id=2" @@ -3463,7 +4512,7 @@ mod tests { /// mask the older row by PK. This is the wallop scenario (seed flushed, /// then delete, then flush). #[tokio::test] - async fn test_shard_writer_delete_across_flushed_generations() { + async fn test_shard_writer_delete_across_sstables() { let (store, base_path, base_uri, _temp) = create_local_store().await; let schema = create_pk_test_schema(); let shard_id = Uuid::new_v4(); @@ -3492,12 +4541,12 @@ mod tests { writer.wait_for_flush_drain().await.unwrap(); assert_eq!( - read_flushed_ids_via_lsm(&writer, schema.clone(), &base_uri, shard_id, None).await, + read_sstable_ids_via_lsm(&writer, schema.clone(), &base_uri, shard_id, None).await, vec![1, 2, 3, 4], "id=0 tombstoned in a newer gen must mask the older gen's live row" ); assert_eq!( - read_flushed_ids_via_lsm(&writer, schema.clone(), &base_uri, shard_id, Some("id < 1")) + read_sstable_ids_via_lsm(&writer, schema.clone(), &base_uri, shard_id, Some("id < 1")) .await, Vec::::new(), "filtered read 'id < 1' must not resurface cross-gen deleted id=0" @@ -3506,13 +4555,13 @@ mod tests { writer.close().await.unwrap(); } - /// Same as the cross-generation case, but the flushed generations carry a + /// Same as the cross-generation case, but the SSTables carry a /// BTree index on `id` (as every wallop table does). A filtered read /// `id < 1` resolves through the scalar index; the `NOT _tombstone` residual /// must still be applied or the deleted row leaks. This is the exact wallop /// failure (BTree id + `FilteredRead 'id < 1'` resurfacing deleted id=0). #[tokio::test] - async fn test_shard_writer_delete_across_flushed_generations_indexed() { + async fn test_shard_writer_delete_across_sstables_indexed() { let (store, base_path, base_uri, _temp) = create_local_store().await; let schema = create_pk_test_schema(); let shard_id = Uuid::new_v4(); @@ -3544,12 +4593,12 @@ mod tests { writer.wait_for_flush_drain().await.unwrap(); assert_eq!( - read_flushed_ids_via_lsm(&writer, schema.clone(), &base_uri, shard_id, None).await, + read_sstable_ids_via_lsm(&writer, schema.clone(), &base_uri, shard_id, None).await, vec![1, 2, 3, 4], "indexed cross-gen: full scan must mask deleted id=0" ); assert_eq!( - read_flushed_ids_via_lsm(&writer, schema.clone(), &base_uri, shard_id, Some("id < 1")) + read_sstable_ids_via_lsm(&writer, schema.clone(), &base_uri, shard_id, Some("id < 1")) .await, Vec::::new(), "indexed filtered read 'id < 1' must not resurface deleted id=0 (wallop repro)" @@ -3731,9 +4780,8 @@ mod tests { shard_id: Uuid::new_v4(), shard_spec_id: 0, durable_write: false, - sync_indexed_write: false, max_wal_buffer_size: 1024 * 1024, - max_wal_flush_interval: None, + max_wal_flush_interval: Some(Duration::from_millis(10)), max_memtable_size: 64 * 1024 * 1024, manifest_scan_batch_size: 2, ..Default::default() @@ -3774,9 +4822,8 @@ mod tests { shard_id: Uuid::new_v4(), shard_spec_id: 0, durable_write: true, - sync_indexed_write: false, max_wal_buffer_size: 1024 * 1024, - max_wal_flush_interval: None, + max_wal_flush_interval: Some(Duration::from_millis(10)), max_memtable_size: 64 * 1024 * 1024, manifest_scan_batch_size: 2, ..Default::default() @@ -3803,7 +4850,7 @@ mod tests { } #[tokio::test] - async fn test_put_no_wait_non_durable_returns_no_watcher() { + async fn test_non_durable_put_is_read_your_writes() { let (store, base_path, base_uri, _temp_dir) = create_local_store().await; let schema = create_test_schema(); @@ -3811,9 +4858,8 @@ mod tests { shard_id: Uuid::new_v4(), shard_spec_id: 0, durable_write: false, - sync_indexed_write: false, max_wal_buffer_size: 1024 * 1024, - max_wal_flush_interval: None, + max_wal_flush_interval: Some(Duration::from_millis(10)), max_memtable_size: 64 * 1024 * 1024, manifest_scan_batch_size: 2, ..Default::default() @@ -3826,7 +4872,21 @@ mod tests { let batch = create_test_batch(&schema, 0, 10); let (result, watcher) = writer.put_no_wait(vec![batch]).await.unwrap(); assert_eq!(result.batch_positions, 0..1); - assert!(watcher.is_none(), "non-durable put has nothing to await"); + + // A non-durable put still has something to await: its *index apply*. + // Skipping durability now costs the caller durability only — not + // visibility. Before the index apply was split off the WAL flush, a + // non-durable write was not read-your-writes at all: the row stayed + // invisible until some later flush happened to index it. + let mut watcher = watcher.expect("a non-durable put awaits its index apply"); + watcher.wait().await.unwrap(); + + let scanned = writer.scan().await.unwrap().try_into_batch().await.unwrap(); + assert_eq!( + scanned.num_rows(), + 10, + "a non-durable put must be readable as soon as it returns" + ); let stats = writer.memtable_stats().await.unwrap(); assert_eq!(stats.row_count, 10); @@ -3843,9 +4903,8 @@ mod tests { shard_id: Uuid::new_v4(), shard_spec_id: 0, durable_write: false, - sync_indexed_write: false, max_wal_buffer_size: 1024 * 1024, - max_wal_flush_interval: None, + max_wal_flush_interval: Some(Duration::from_millis(10)), max_memtable_size: 64 * 1024 * 1024, manifest_scan_batch_size: 2, ..Default::default() @@ -3878,9 +4937,8 @@ mod tests { shard_id: Uuid::new_v4(), shard_spec_id: 0, durable_write: false, - sync_indexed_write: true, max_wal_buffer_size: 1024 * 1024, - max_wal_flush_interval: None, + max_wal_flush_interval: Some(Duration::from_millis(10)), max_memtable_size: 64 * 1024 * 1024, manifest_scan_batch_size: 2, ..Default::default() @@ -3914,12 +4972,12 @@ mod tests { } /// End-to-end check that the background flush handler rebuilds secondary - /// indexes on every flushed generation. Before this, the handler flushed - /// via plain `flush`, leaving flushed generations unindexed — point + /// indexes on every SSTable. Before this, the handler flushed + /// via plain `flush`, leaving SSTables unindexed — point /// lookups had to full-scan and vector search's index-only `fast_search` /// couldn't see the data at all. #[tokio::test] - async fn test_flushed_generation_is_indexed() { + async fn test_sstable_is_indexed() { use crate::index::DatasetIndexExt; let (store, base_path, base_uri, _temp_dir) = create_local_store().await; @@ -3930,9 +4988,8 @@ mod tests { shard_id, shard_spec_id: 0, durable_write: false, - sync_indexed_write: true, max_wal_buffer_size: 1024 * 1024, - max_wal_flush_interval: None, + max_wal_flush_interval: Some(Duration::from_millis(10)), max_memtable_size: 64 * 1024 * 1024, manifest_scan_batch_size: 2, ..Default::default() @@ -3964,22 +5021,18 @@ mod tests { writer.force_seal_active().await.unwrap(); writer.wait_for_flush_drain().await.unwrap(); - // Resolve the flushed generation recorded in the manifest. + // Resolve the SSTable recorded in the manifest. let manifest = writer.manifest().await.unwrap().unwrap(); - assert_eq!( - manifest.flushed_generations.len(), - 1, - "expected exactly one flushed generation" - ); + assert_eq!(manifest.sstables.len(), 1, "expected exactly one SSTable"); let gen_uri = format!( "{}/_mem_wal/{}/{}", - base_uri, shard_id, manifest.flushed_generations[0].path + base_uri, shard_id, manifest.sstables[0].path ); - // The flushed generation must carry the BTree index built during flush. + // The SSTable must carry the BTree index built during flush. let dataset = crate::Dataset::open(&gen_uri).await.unwrap(); let indices = dataset.load_indices().await.unwrap(); - assert_eq!(indices.len(), 1, "flushed generation should have one index"); + assert_eq!(indices.len(), 1, "SSTable should have one index"); assert_eq!(indices[0].name, "id_idx"); // A PK filter over it must resolve through the index, not a full scan. @@ -4019,9 +5072,8 @@ mod tests { shard_id: Uuid::new_v4(), shard_spec_id: 0, durable_write: false, - sync_indexed_write: false, max_wal_buffer_size: 1024 * 1024, - max_wal_flush_interval: None, + max_wal_flush_interval: Some(Duration::from_millis(10)), max_memtable_size: 1024, // Very small - will trigger flush quickly manifest_scan_batch_size: 2, ..Default::default() @@ -4052,6 +5104,88 @@ mod tests { writer.close().await.unwrap(); } + /// The two fields count different things on purpose: bytes drain on flush + /// commit, the handle lingers for `frozen_memtable_grace`. A long grace + /// therefore leaves count non-zero with bytes back at zero. + #[tokio::test] + async fn test_memtable_stats_frozen_count_outlives_frozen_bytes() { + let (store, base_path, base_uri, _temp_dir) = create_local_store().await; + let schema = create_test_schema(); + + let config = ShardWriterConfig { + shard_id: Uuid::new_v4(), + shard_spec_id: 0, + durable_write: false, + max_wal_buffer_size: 1024 * 1024, + max_wal_flush_interval: Some(Duration::from_millis(10)), + max_memtable_size: 1024, // small enough to seal within the loop below + frozen_memtable_grace: Duration::from_secs(600), + manifest_scan_batch_size: 2, + ..Default::default() + }; + + let writer = ShardWriter::open(store, base_path, base_uri, config, schema.clone(), vec![]) + .await + .unwrap(); + + let fresh = writer.memtable_stats().await.unwrap(); + assert_eq!(fresh.frozen_count, 0); + assert_eq!(fresh.frozen_bytes, 0); + + for i in 0..20 { + let batch = create_test_batch(&schema, i * 10, 10); + writer.put(vec![batch]).await.unwrap(); + } + writer.wait_for_flush_drain().await.unwrap(); + + let stats = writer.memtable_stats().await.unwrap(); + assert!( + stats.frozen_count > 0, + "flushed memtables must stay in the read view for the grace window" + ); + assert_eq!( + stats.frozen_bytes, 0, + "every seal flushed, so nothing is owed to flush" + ); + + writer.close().await.unwrap(); + } + + /// The controller sits in a private `WriterMode` variant, reachable only + /// through the writer. Both modes must answer. + #[rstest::rstest] + #[case::memtable(true)] + #[case::wal_only(false)] + #[tokio::test] + async fn test_backpressure_stats_reachable_in_both_modes(#[case] enable_memtable: bool) { + let (store, base_path, base_uri, _temp_dir) = create_local_store().await; + let schema = create_test_schema(); + + let config = ShardWriterConfig { + shard_id: Uuid::new_v4(), + shard_spec_id: 0, + durable_write: false, + max_wal_buffer_size: 1024 * 1024, + max_wal_flush_interval: Some(Duration::from_millis(10)), + max_memtable_size: 64 * 1024 * 1024, + manifest_scan_batch_size: 2, + enable_memtable, + ..Default::default() + }; + + let writer = ShardWriter::open(store, base_path, base_uri, config, schema.clone(), vec![]) + .await + .unwrap(); + + // A writer well under the threshold has never throttled. + let stats = writer.backpressure_stats(); + assert_eq!(stats.total_count, 0); + assert_eq!(stats.total_wait_ms, 0); + assert_eq!(stats.active_count, 0); + + writer.close().await.unwrap(); + } + /// Regression for #6713: a single failing `handle()` must not kill /// the dispatcher. Earlier the loop would `break Err(e)` on the /// first message error, dropping the rx side and stranding @@ -4110,35 +5244,140 @@ mod tests { call_count.load(Ordering::SeqCst) ); - executor.shutdown_all().await.ok(); + executor + .shutdown_all() + .await + .expect("dispatcher should shut down successfully"); } - /// Same as the local-fs test but against memory:// — closer to S3 - /// semantics (conditional PUT, list-prefix consistency). #[tokio::test] - async fn test_shard_writer_auto_flush_repeatedly_memory_store() { - let base_uri = "memory:///bench_test_flush"; - let (store, base_path) = ObjectStore::from_uri(base_uri).await.unwrap(); - let base_uri = base_uri.to_string(); - let schema = create_test_schema(); + async fn test_task_executor_shutdown_propagates_cleanup_error_and_joins_all_tasks() { + use std::sync::atomic::{AtomicUsize, Ordering}; - let config = ShardWriterConfig { - shard_id: Uuid::new_v4(), - shard_spec_id: 0, - durable_write: true, - sync_indexed_write: false, - max_wal_buffer_size: 1024 * 1024, - max_wal_flush_interval: None, - max_memtable_size: 64, - manifest_scan_batch_size: 2, - ..Default::default() - }; + struct CleanupHandler { + cleanup_count: Arc, + error_message: Option<&'static str>, + } - let writer = ShardWriter::open(store, base_path, base_uri, config, schema.clone(), vec![]) - .await - .unwrap(); + #[async_trait] + impl MessageHandler for CleanupHandler { + async fn handle(&mut self, _message: u32) -> Result<()> { + Ok(()) + } - let initial_gen = writer.memtable_stats().await.unwrap().generation; + async fn cleanup(&mut self, _shutdown_ok: bool) -> Result<()> { + self.cleanup_count.fetch_add(1, Ordering::SeqCst); + match self.error_message { + Some(message) => Err(Error::io(message)), + None => Ok(()), + } + } + } + + let executor = TaskExecutor::new(); + let cleanup_count = Arc::new(AtomicUsize::new(0)); + let (_failing_tx, failing_rx) = mpsc::unbounded_channel::(); + executor + .add_handler( + "failing-cleanup".to_string(), + Box::new(CleanupHandler { + cleanup_count: cleanup_count.clone(), + error_message: Some("intentional cleanup failure"), + }), + failing_rx, + ) + .unwrap(); + let (_successful_tx, successful_rx) = mpsc::unbounded_channel::(); + executor + .add_handler( + "successful-cleanup".to_string(), + Box::new(CleanupHandler { + cleanup_count: cleanup_count.clone(), + error_message: None, + }), + successful_rx, + ) + .unwrap(); + + let error = executor + .shutdown_all() + .await + .expect_err("shutdown must propagate the handler cleanup failure"); + assert!(matches!(&error, Error::IO { .. })); + assert!( + error.to_string().contains("intentional cleanup failure"), + "unexpected error: {error}" + ); + assert_eq!( + cleanup_count.load(Ordering::SeqCst), + 2, + "shutdown must join and clean up every task after the first failure" + ); + assert!(executor.tasks.read().unwrap().is_empty()); + } + + #[tokio::test] + async fn test_task_executor_shutdown_propagates_task_panic() { + struct PanickingCleanupHandler; + + #[async_trait] + impl MessageHandler for PanickingCleanupHandler { + async fn handle(&mut self, _message: u32) -> Result<()> { + Ok(()) + } + + async fn cleanup(&mut self, _shutdown_ok: bool) -> Result<()> { + panic!("intentional cleanup panic"); + } + } + + let executor = TaskExecutor::new(); + let (_tx, rx) = mpsc::unbounded_channel::(); + executor + .add_handler( + "panicking-cleanup".to_string(), + Box::new(PanickingCleanupHandler), + rx, + ) + .unwrap(); + + let error = executor + .shutdown_all() + .await + .expect_err("shutdown must propagate the task panic"); + assert!(matches!(&error, Error::Internal { .. })); + assert!( + error.to_string().contains("panicking-cleanup") + && error.to_string().contains("panicked during shutdown"), + "unexpected error: {error}" + ); + } + + /// Same as the local-fs test but against memory:// — closer to S3 + /// semantics (conditional PUT, list-prefix consistency). + #[tokio::test] + async fn test_shard_writer_auto_flush_repeatedly_memory_store() { + let base_uri = "memory:///bench_test_flush"; + let (store, base_path) = ObjectStore::from_uri(base_uri).await.unwrap(); + let base_uri = base_uri.to_string(); + let schema = create_test_schema(); + + let config = ShardWriterConfig { + shard_id: Uuid::new_v4(), + shard_spec_id: 0, + durable_write: true, + max_wal_buffer_size: 1024 * 1024, + max_wal_flush_interval: Some(Duration::from_millis(10)), + max_memtable_size: 64, + manifest_scan_batch_size: 2, + ..Default::default() + }; + + let writer = ShardWriter::open(store, base_path, base_uri, config, schema.clone(), vec![]) + .await + .unwrap(); + + let initial_gen = writer.memtable_stats().await.unwrap().generation; for i in 0..1000 { let batch = create_test_batch(&schema, i * 10, 10); @@ -4171,9 +5410,8 @@ mod tests { shard_id: Uuid::new_v4(), shard_spec_id: 0, durable_write: true, - sync_indexed_write: false, max_wal_buffer_size: 1024 * 1024, - max_wal_flush_interval: None, + max_wal_flush_interval: Some(Duration::from_millis(10)), // Tiny size threshold — every batch crosses it. max_memtable_size: 64, manifest_scan_batch_size: 2, @@ -4238,9 +5476,8 @@ mod tests { shard_id, shard_spec_id: 0, durable_write: true, - sync_indexed_write: false, max_wal_buffer_size: 1024 * 1024, - max_wal_flush_interval: None, + max_wal_flush_interval: Some(Duration::from_millis(10)), max_memtable_size: usize::MAX, max_unflushed_memtable_bytes: usize::MAX, manifest_scan_batch_size: 2, @@ -4305,6 +5542,55 @@ mod tests { reopened.close().await.unwrap(); } + #[rstest] + #[case::memtable(true)] + #[case::wal_only(false)] + #[tokio::test] + async fn test_close_propagates_final_wal_persistence_failure(#[case] enable_memtable: bool) { + let (store, base_path, controls) = failing_memory_store().await; + let base_uri = "memory:///"; + let schema = create_test_schema(); + let config = ShardWriterConfig { + shard_id: Uuid::new_v4(), + shard_spec_id: 0, + durable_write: false, + enable_memtable, + max_wal_buffer_size: usize::MAX, + max_wal_flush_interval: None, + max_wal_persist_retries: 0, + max_memtable_size: usize::MAX, + max_unflushed_memtable_bytes: usize::MAX, + manifest_scan_batch_size: 2, + ..Default::default() + }; + let writer = ShardWriter::open(store, base_path, base_uri, config, schema.clone(), vec![]) + .await + .unwrap(); + let task_executor = writer.task_executor.clone(); + + writer + .put(vec![create_test_batch(&schema, 0, 10)]) + .await + .unwrap(); + controls.fail_wal_puts(usize::MAX); + + let error = writer + .close() + .await + .expect_err("close must propagate the final WAL persistence failure"); + assert_eq!(error.fence_reason(), Some(FenceReason::PersistenceFailure)); + assert!( + error + .to_string() + .contains("injected transient WAL put failure"), + "unexpected error: {error}" + ); + assert!( + task_executor.tasks.read().unwrap().is_empty(), + "close must join background tasks before returning an error" + ); + } + /// Regression: the memtable flush should successfully fire many /// times in a row. A bug where every flush wrote the same path was /// caught by lance-format/lance#6713. @@ -4319,9 +5605,8 @@ mod tests { shard_id: Uuid::new_v4(), shard_spec_id: 0, durable_write: true, - sync_indexed_write: false, max_wal_buffer_size: 1024 * 1024, - max_wal_flush_interval: None, + max_wal_flush_interval: Some(Duration::from_millis(10)), // Tiny size threshold so a few batches cross it. max_memtable_size: 1024, manifest_scan_batch_size: 2, @@ -4402,6 +5687,92 @@ mod tests { assert_eq!(call_count.load(std::sync::atomic::Ordering::Relaxed), 4); // Should have recorded backpressure wait time (waited 3 times) assert_eq!(controller.stats().count(), 1); + assert_eq!( + controller.stats().snapshot().active_count, + 0, + "the wait is over, so nobody is parked" + ); + } + + /// The totals only move when a wait ends, so a first stall would be + /// invisible to a poller if `active_count` did not report it live. + #[tokio::test] + async fn test_backpressure_in_progress_wait_is_observable() { + use std::sync::atomic::AtomicUsize; + use std::sync::atomic::Ordering as AtomicOrdering; + use std::time::Duration; + + let config = ShardWriterConfig::default() + .with_max_unflushed_memtable_bytes(100) + .with_backpressure_log_interval(Duration::from_millis(50)); + + let controller = BackpressureController::new(config); + let stats = controller.stats().clone(); + + let unflushed = Arc::new(AtomicUsize::new(1000)); + let release = unflushed.clone(); + + let parked = + controller.maybe_apply_backpressure(|| (unflushed.load(AtomicOrdering::Relaxed), None)); + + let observer = async { + // Bounded so a regression that never publishes the park fails here + // instead of hanging the suite. + let deadline = Instant::now() + Duration::from_secs(5); + while stats.snapshot().active_count == 0 { + assert!( + Instant::now() < deadline, + "an ongoing wait was never published" + ); + tokio::time::sleep(Duration::from_millis(5)).await; + } + let mid = stats.snapshot(); + assert_eq!(mid.active_count, 1); + assert_eq!( + mid.total_count, 0, + "a wait still in progress is not a completed one" + ); + assert_eq!(mid.total_wait_ms, 0); + + release.store(0, AtomicOrdering::Relaxed); + }; + + let (result, ()) = tokio::join!(parked, observer); + result.unwrap(); + + let after = stats.snapshot(); + assert_eq!(after.active_count, 0, "the guard drops when the wait ends"); + assert_eq!(after.total_count, 1); + } + + /// A `put` whose caller times out drops the wait future mid-park. The gauge + /// must come back down, or a cancelled writer is throttled forever on paper. + #[tokio::test] + async fn test_backpressure_cancelled_wait_does_not_leak_active_count() { + use std::time::Duration; + + let config = ShardWriterConfig::default() + .with_max_unflushed_memtable_bytes(100) + .with_backpressure_log_interval(Duration::from_millis(50)); + + let controller = BackpressureController::new(config); + + // Never drops below the threshold, so the timeout is what ends the wait. + assert!( + tokio::time::timeout( + Duration::from_millis(50), + controller.maybe_apply_backpressure(|| (1000, None)), + ) + .await + .is_err() + ); + + let after = controller.stats().snapshot(); + assert_eq!(after.active_count, 0, "cancellation must release the guard"); + assert_eq!( + after.total_count, 0, + "a cancelled wait never completed, so it is not a completed wait" + ); } #[test] @@ -4462,7 +5833,7 @@ mod tests { durable_write: true, enable_memtable: false, max_wal_buffer_size: 1024 * 1024, - max_wal_flush_interval: None, + max_wal_flush_interval: Some(Duration::from_millis(10)), manifest_scan_batch_size: 2, ..Default::default() } @@ -4487,8 +5858,9 @@ mod tests { .await .unwrap(); - // Two durable puts → two WAL entries (durable_write triggers an - // explicit flush per put). + // Two durable puts → two WAL entries. Each `put` awaits its own append + // (driven by the background ticker) before returning, so the second + // batch is only pushed after the first is durable — they never coalesce. let r1 = writer .put(vec![create_test_batch(&schema, 0, 4)]) .await @@ -4519,6 +5891,51 @@ mod tests { assert!(e0.writer_epoch >= 1); } + /// A durable WAL-only put is driven by the background ticker, not an inline + /// per-put flush: `put().await` must not return until the ticker's append + /// advances the durability watermark it waits on. With a long interval and a + /// buffer the write cannot cross, the ticker is the *only* thing that can + /// make the put durable — so if the WAL entry is present the instant `put` + /// returns (before any `close()`), the watermark-wait is doing its job. + #[tokio::test] + async fn test_wal_only_durable_put_waits_for_ticker_append() { + use crate::dataset::mem_wal::wal::WalTailer; + + let (store, base_path, base_uri, _temp_dir) = create_local_store().await; + let schema = create_test_schema(); + let shard_id = Uuid::new_v4(); + + let mut config = wal_only_config(shard_id); + config.max_wal_flush_interval = Some(Duration::from_millis(100)); + config.max_wal_buffer_size = 100 * 1024 * 1024; // never crossed + + let writer = ShardWriter::open( + store.clone(), + base_path.clone(), + base_uri, + config, + schema.clone(), + vec![], + ) + .await + .unwrap(); + + writer + .put(vec![create_test_batch(&schema, 0, 4)]) + .await + .unwrap(); + + // `put` returned, so the batch must already be durable — read the WAL + // directly, without closing the writer. + let tailer = WalTailer::new(store, base_path, shard_id); + assert_eq!(tailer.next_position().await.unwrap(), 2); + let entry = tailer.read_entry(1).await.unwrap().unwrap(); + assert_eq!(entry.batches.len(), 1); + assert_eq!(entry.batches[0].num_rows(), 4); + + writer.close().await.unwrap(); + } + #[tokio::test] async fn test_wal_only_rejects_index_configs() { let (store, base_path, base_uri, _temp_dir) = create_local_store().await; @@ -4753,9 +6170,8 @@ mod tests { shard_id, shard_spec_id: 0, durable_write: true, - sync_indexed_write: false, max_wal_buffer_size: 1024 * 1024, - max_wal_flush_interval: None, + max_wal_flush_interval: Some(Duration::from_millis(10)), max_memtable_size: 64 * 1024 * 1024, manifest_scan_batch_size: 2, ..Default::default() @@ -4779,8 +6195,9 @@ mod tests { } // A durable write whose WAL PUT keeps failing poisons the writer with a - // typed persistence failure; the next write fails fast with the same reason; - // and once storage heals, reopening replays the WAL and writes resume. + // typed persistence failure; the next write *and every read* fail fast with + // the same reason; and once storage heals, reopening replays the WAL and + // writes resume. #[tokio::test] async fn test_writer_poisons_on_persistence_failure_and_recovers_on_reopen() { let (store, base_path, controls) = failing_memory_store().await; @@ -4819,6 +6236,31 @@ mod tests { .await .unwrap_err(); assert_eq!(err.fence_reason(), Some(FenceReason::PersistenceFailure)); + + // ...and rejects *reads* too. Batch 0 was committed to the BatchStore + // before its WAL PUT failed, so a poisoned writer that still served + // reads would hand out a row that is not durable and that replay will + // not reproduce — a divergent snapshot. Mirrors SlateDB's + // `check_closed()` at the top of every read. + for reason in [ + writer.scan().await.err().and_then(|e| e.fence_reason()), + writer + .active_memtable_ref() + .await + .err() + .and_then(|e| e.fence_reason()), + writer + .in_memory_memtable_refs() + .await + .err() + .and_then(|e| e.fence_reason()), + ] { + assert_eq!(reason, Some(FenceReason::PersistenceFailure)); + } + + // Stats stay readable: this is what an operator (and the eviction path) + // inspects to decide what to do about the poisoned shard. + writer.memtable_stats().await.unwrap(); drop(writer); // Storage heals: reopening replays the WAL and accepts writes again. @@ -4832,69 +6274,712 @@ mod tests { .unwrap(); } - /// Replay-on-open recovers durable WAL entries that were never flushed - /// to a Lance generation. Setup: writer A durably writes batches, drops - /// without close (so MemTable freeze never runs); writer B reopens and - /// must see A's rows in its MemTable scan. + /// A doomed open must fail on local validation *before* it claims the epoch, + /// so it cannot fence the healthy writer already serving the shard. The + /// index-config check used to run *after* `claim_epoch` (and, for a successor, + /// after `write_fence_sentinel`), so a rejected open still bumped the stored + /// epoch and fenced the incumbent. #[tokio::test] - async fn test_memtable_replay_recovers_unflushed_writes() { + async fn test_doomed_open_does_not_fence_incumbent() { let (store, base_path, base_uri, _temp_dir) = create_local_store().await; let schema = schema_with_pk(); let shard_id = Uuid::new_v4(); - // Writer A: write two durable batches, then drop without close. - // The WAL files persist; the in-memory MemTable does not. - { - let writer_a = ShardWriter::open( - store.clone(), - base_path.clone(), - base_uri.clone(), - memtable_config_with_pk(shard_id), - schema.clone(), - vec![], - ) - .await - .unwrap(); - writer_a - .put(vec![create_test_batch(&schema, 0, 5)]) - .await - .unwrap(); - writer_a - .put(vec![create_test_batch(&schema, 100, 3)]) - .await - .unwrap(); - // intentionally drop without close() - } - - // Writer B reopens. Replay must rehydrate A's two batches into the - // active MemTable. - let writer_b = ShardWriter::open( - store, - base_path, - base_uri, + let writer_a = ShardWriter::open( + store.clone(), + base_path.clone(), + base_uri.clone(), memtable_config_with_pk(shard_id), - schema, + schema.clone(), vec![], ) .await .unwrap(); + writer_a + .put(vec![create_test_batch(&schema, 0, 1)]) + .await + .unwrap(); - let stats = writer_b.memtable_stats().await.unwrap(); - assert_eq!( - stats.row_count, 8, - "expected replay to insert 5 + 3 = 8 rows, got {}", - stats.row_count - ); - assert_eq!( - stats.batch_count, 2, - "expected replay to insert 2 batches, got {}", - stats.batch_count + // An index config that disagrees with the schema (FTS on the Int32 `id` + // column) is rejected on local validation. On the old path this rejection + // landed only after the epoch had already been claimed. + let bad_fts = MemIndexConfig::Fts(FtsIndexConfig::new( + "bad_fts".to_string(), + 0, + "id".to_string(), + )); + let err = ShardWriter::open( + store.clone(), + base_path.clone(), + base_uri.clone(), + memtable_config_with_pk(shard_id), + schema.clone(), + vec![bad_fts], + ) + .await + .map(|_| ()) + .expect_err("an FTS index on a non-Utf8 column must be rejected"); + assert!( + err.to_string().contains("bad_fts") && err.to_string().contains("Utf8"), + "unexpected error: {err}" ); - writer_b.close().await.unwrap(); + // The incumbent is untouched: not fenced, still accepting writes. + writer_a.check_fenced().await.unwrap(); + writer_a + .put(vec![create_test_batch(&schema, 1, 1)]) + .await + .unwrap(); + writer_a.close().await.unwrap(); } - /// Replay is a no-op on a fresh shard: the MemTable starts empty. + /// A failed dispatch during `freeze_memtable` must not drop the outgoing + /// table's rows from the read view. The active memtable is replaced before + /// the WAL-flush and index-apply sends; a send that failed (background tasks + /// gone) used to return before the outgoing table was retained in + /// `frozen_memtables`, so its accepted rows silently vanished — a scan + /// returned 0 rows with no error. The writer must instead retain the table + /// and poison, so reads fail fast rather than serve a divergent snapshot. + #[tokio::test] + async fn test_freeze_dispatch_failure_retains_rows_and_poisons() { + let (store, base_path, base_uri, _temp_dir) = create_local_store().await; + let schema = schema_with_pk(); + let shard_id = Uuid::new_v4(); + + // Non-durable + no ticker: the put is read-your-writes (waits for its + // index apply) but nothing is WAL-flushed, so the freeze below still owes + // a WAL append. + let config = ShardWriterConfig { + durable_write: false, + max_wal_flush_interval: None, + ..memtable_config_with_pk(shard_id) + }; + let writer = ShardWriter::open(store, base_path, base_uri, config, schema.clone(), vec![]) + .await + .unwrap(); + + writer + .put(vec![create_test_batch(&schema, 0, 10)]) + .await + .unwrap(); + assert_eq!(writer.memtable_stats().await.unwrap().row_count, 10); + + // Tear the background tasks down out from under the writer, so the + // freeze's dispatch sends hit closed channels. + writer.abort().await.unwrap(); + + let err = writer + .force_seal_active() + .await + .expect_err("force_seal_active must surface the failed dispatch"); + assert!( + err.to_string().contains("channel closed"), + "unexpected error: {err}" + ); + + // The failure poisoned the writer: reads fail fast instead of returning a + // silent zero-row snapshot of a shard whose rows were dropped. + assert!( + writer.scan().await.is_err(), + "a poisoned writer must reject reads, not serve a divergent snapshot" + ); + assert!(writer.in_memory_memtable_refs().await.is_err()); + } + + /// A WAL holding more batches than one memtable's capacity must reopen. + /// + /// One memtable holds at most `max_memtable_batches` batches, but a WAL is + /// unbounded, so replay has to rotate — seal the full memtable, start a fresh + /// one — exactly as the live write path does. Before, replay stuffed + /// everything into a single memtable and `open()` failed outright with + /// "MemTable batch store is full", leaving the shard permanently unopenable. + #[tokio::test] + async fn test_replay_rotates_when_wal_exceeds_one_memtable() { + let (store, base_path, base_uri, _temp_dir) = create_local_store().await; + let schema = schema_with_pk(); + let shard_id = Uuid::new_v4(); + + const N: i32 = 8; + + // Writer A has a *large* capacity, so its eight one-batch puts all land in + // a single memtable and it never freezes or flushes a generation of its + // own. Dropping it without close leaves an eight-entry WAL and no + // generations — a WAL that no single small memtable could hold. + let writer_a_config = ShardWriterConfig { + max_memtable_batches: 1000, + ..memtable_config_with_pk(shard_id) + }; + // Writer B has a *two-batch* capacity, so replaying that eight-entry WAL is + // exactly what must rotate. Keeping the configs distinct isolates replay + // rotation from the live rotation writer A would otherwise do concurrently. + let config = ShardWriterConfig { + max_memtable_batches: 2, + ..memtable_config_with_pk(shard_id) + }; + + { + let writer_a = ShardWriter::open( + store.clone(), + base_path.clone(), + base_uri.clone(), + writer_a_config, + schema.clone(), + vec![], + ) + .await + .unwrap(); + for id in 0..N { + writer_a + .put(vec![create_test_batch(&schema, id, 1)]) + .await + .unwrap(); + } + // Drop without close: only the WAL survives, and it holds more batches + // than writer B's memtable can. + } + + // Total rows across the active memtable plus every SSTable. + // Distinct ids, so no cross-generation dedup — a plain sum is exact. + async fn total_rows(writer: &ShardWriter, base_uri: &str, shard_id: Uuid) -> usize { + let mut rows = writer.memtable_stats().await.unwrap().row_count; + let manifest = writer.manifest().await.unwrap().unwrap(); + for sstable in &manifest.sstables { + let gen_uri = format!("{}/_mem_wal/{}/{}", base_uri, shard_id, sstable.path); + let dataset = crate::Dataset::open(&gen_uri).await.unwrap(); + rows += dataset.count_rows(None).await.unwrap(); + } + rows + } + + // Reopen. This used to fail with a full-batch-store error. + let writer_b = ShardWriter::open( + store.clone(), + base_path.clone(), + base_uri.clone(), + config.clone(), + schema.clone(), + vec![], + ) + .await + .expect("a WAL larger than one memtable must still reopen"); + + // Rotation produced sealed memtables, and replay flushed each to a Lance + // generation rather than holding it in memory or leaving it in the WAL. + let manifest = writer_b.manifest().await.unwrap().unwrap(); + assert!( + !manifest.sstables.is_empty(), + "replay must have sealed and flushed at least one full memtable" + ); + + // Every row survived, split between the SSTables and the + // active (partial) memtable. + assert_eq!( + total_rows(&writer_b, &base_uri, shard_id).await as i32, + N, + "every replayed row must be durable, across generations and the active memtable" + ); + writer_b.close().await.unwrap(); + + // Because the sealed memtables were flushed, the manifest's replay cursor + // advanced past their WAL entries — so a second reopen replays only the + // tail and still accounts for every row. The WAL truncates across reopens + // rather than growing without bound. + let writer_c = + ShardWriter::open(store, base_path, base_uri.clone(), config, schema, vec![]) + .await + .unwrap(); + assert_eq!(total_rows(&writer_c, &base_uri, shard_id).await as i32, N); + writer_c.close().await.unwrap(); + } + + /// Replay-on-open recovers durable WAL entries that were never flushed + /// to a Lance generation. Setup: writer A durably writes batches, drops + /// without close (so MemTable freeze never runs); writer B reopens and + /// must see A's rows in its MemTable scan. + #[tokio::test] + async fn test_memtable_replay_recovers_unflushed_writes() { + let (store, base_path, base_uri, _temp_dir) = create_local_store().await; + let schema = schema_with_pk(); + let shard_id = Uuid::new_v4(); + + // Writer A: write two durable batches, then drop without close. + // The WAL files persist; the in-memory MemTable does not. + { + let writer_a = ShardWriter::open( + store.clone(), + base_path.clone(), + base_uri.clone(), + memtable_config_with_pk(shard_id), + schema.clone(), + vec![], + ) + .await + .unwrap(); + writer_a + .put(vec![create_test_batch(&schema, 0, 5)]) + .await + .unwrap(); + writer_a + .put(vec![create_test_batch(&schema, 100, 3)]) + .await + .unwrap(); + // intentionally drop without close() + } + + // Writer B reopens. Replay must rehydrate A's two batches into the + // active MemTable. + let writer_b = ShardWriter::open( + store, + base_path, + base_uri, + memtable_config_with_pk(shard_id), + schema, + vec![], + ) + .await + .unwrap(); + + let stats = writer_b.memtable_stats().await.unwrap(); + assert_eq!( + stats.row_count, 8, + "expected replay to insert 5 + 3 = 8 rows, got {}", + stats.row_count + ); + assert_eq!( + stats.batch_count, 2, + "expected replay to insert 2 batches, got {}", + stats.batch_count + ); + + writer_b.close().await.unwrap(); + } + + /// Replayed batches are already WAL-durable, so the first flush after a + /// reopen must not re-append them to the WAL or re-insert them into the + /// indexes. Before replay stamped the durability cursor it stayed at + /// "nothing flushed", so the next flush re-covered `[0, end)`: it appended + /// the already-durable rows a second time *and* re-indexed them. None of + /// the in-memory indexes is idempotent, so an indexed PK lookup returned + /// the row twice while a full scan still looked healthy. + #[tokio::test] + async fn test_replay_does_not_reappend_or_reindex() { + use crate::dataset::mem_wal::wal::WalTailer; + + let (store, base_path, base_uri, _temp_dir) = create_local_store().await; + let schema = schema_with_pk(); + let shard_id = Uuid::new_v4(); + + // Writer A: two durable batches (5 + 3 = 8 rows), dropped without close. + { + let writer_a = ShardWriter::open( + store.clone(), + base_path.clone(), + base_uri.clone(), + memtable_config_with_pk(shard_id), + schema.clone(), + vec![], + ) + .await + .unwrap(); + writer_a + .put(vec![create_test_batch(&schema, 0, 5)]) + .await + .unwrap(); + writer_a + .put(vec![create_test_batch(&schema, 100, 3)]) + .await + .unwrap(); + } + + // Writer B reopens and replays A's two batches. + let writer_b = ShardWriter::open( + store.clone(), + base_path.clone(), + base_uri, + memtable_config_with_pk(shard_id), + schema.clone(), + vec![], + ) + .await + .unwrap(); + + let stats = writer_b.memtable_stats().await.unwrap(); + assert_eq!(stats.batch_count, 2); + assert_eq!( + stats.durable_batch_count, 2, + "replayed batches came from the WAL, so the durability cursor must already cover them" + ); + + // One more durable put. Its flush must cover only the new batch. + writer_b + .put(vec![create_test_batch(&schema, 200, 2)]) + .await + .unwrap(); + + let tailer = WalTailer::new(store, base_path, shard_id); + let first = tailer.first_position().await.unwrap(); + let next = tailer.next_position().await.unwrap(); + let mut wal_rows = 0; + for position in first..next { + if let Some(entry) = tailer.read_entry(position).await.unwrap() { + wal_rows += entry.batches.iter().map(|b| b.num_rows()).sum::(); + } + } + assert_eq!( + wal_rows, 10, + "WAL must hold 8 replayed + 2 new rows; a re-covering flush re-appends the replayed 8" + ); + + // The indexed arm must not see a replayed row twice. + let mut scanner = writer_b.scan().await.unwrap(); + scanner.filter("id = 0").unwrap(); + let hit = scanner.try_into_batch().await.unwrap(); + assert_eq!( + hit.num_rows(), + 1, + "indexed PK lookup returned the replayed row more than once" + ); + + let all = writer_b + .scan() + .await + .unwrap() + .try_into_batch() + .await + .unwrap(); + assert_eq!(all.num_rows(), 10); + + writer_b.close().await.unwrap(); + } + + /// A non-durable put is readable through the **index-backed** arms the moment + /// it returns, not just through a full scan. + /// + /// This is what splitting the index apply off the WAL flush buys. Before, the + /// index apply only ran as one arm of the flush, so with `durable_write: + /// false` nothing triggered it on the put path at all: the row sat in the + /// batch store, unindexed, until some later flush happened along. A full scan + /// (which reads the batch store directly) could still find it, while every + /// index-accelerated query could not — the tiers disagreed. Now the apply is + /// triggered per-put in both modes, so `put` returning means "indexed". + #[tokio::test] + async fn test_non_durable_put_is_visible_through_the_index() { + let (store, base_path, base_uri, _temp_dir) = create_local_store().await; + let schema = schema_with_pk(); + let shard_id = Uuid::new_v4(); + + let config = ShardWriterConfig { + durable_write: false, + ..memtable_config_with_pk(shard_id) + }; + let writer = ShardWriter::open(store, base_path, base_uri, config, schema.clone(), vec![]) + .await + .unwrap(); + + writer + .put(vec![create_test_batch(&schema, 0, 5)]) + .await + .unwrap(); + + // The indexed PK lookup must find it — this is the arm that saw nothing + // before, because the index apply had never run. + let mut scanner = writer.scan().await.unwrap(); + scanner.filter("id = 3").unwrap(); + let hit = scanner.try_into_batch().await.unwrap(); + assert_eq!( + hit.num_rows(), + 1, + "an index-backed lookup must see a non-durable put as soon as it returns" + ); + + // ...and so must the unindexed full scan, i.e. the tiers agree. + let all = writer.scan().await.unwrap().try_into_batch().await.unwrap(); + assert_eq!(all.num_rows(), 5); + + writer.close().await.unwrap(); + } + + /// The durability cursor is writer-global, so a put into a *post-rotation* + /// memtable must still wait for its own WAL append. + /// + /// Batch positions restart at 0 in every memtable, but the durability watch + /// channel spans the writer's whole life and is never reset. When the put + /// path targeted a memtable-local position, the first N puts into every + /// memtable after the first were already "satisfied" by the *previous* + /// memtable's N appends: they acked instantly, with no WAL append, and + /// `durable_write: true` silently degraded to non-durable. Worse, the next + /// append then sent a *smaller* value, walking the watermark backwards and + /// hanging any watcher still waiting on the old, higher one. + /// + /// The cursor is now a writer-global exclusive count and every target is + /// lifted through the store's `global_offset`, so it only ever moves forward + /// and a post-rotation put can only be acked by its own append. + #[tokio::test] + async fn test_durable_ack_after_rotation_requires_its_own_wal_append() { + use crate::dataset::mem_wal::wal::WalTailer; + + let (store, base_path, base_uri, _temp_dir) = create_local_store().await; + let schema = schema_with_pk(); + let shard_id = Uuid::new_v4(); + + // A two-batch memtable, so the third put forces a freeze + rotation. + let config = ShardWriterConfig { + max_memtable_batches: 2, + ..memtable_config_with_pk(shard_id) + }; + + let writer = ShardWriter::open( + store.clone(), + base_path.clone(), + base_uri, + config, + schema.clone(), + vec![], + ) + .await + .unwrap(); + + // Fill and rotate the first memtable. + for i in 0..3 { + writer + .put(vec![create_test_batch(&schema, i * 5, 5)]) + .await + .unwrap(); + } + let stats = writer.memtable_stats().await.unwrap(); + assert!( + stats.global_offset > 0, + "expected a rotation; the active memtable is still the writer's first" + ); + + // Every batch acked so far must be genuinely durable, and the cursor must + // cover the active memtable's entire prefix rather than lagging inside it. + let stats = writer.memtable_stats().await.unwrap(); + assert!( + stats.durable_batch_count >= stats.global_offset + stats.batch_count, + "durable_write acked a put the WAL never received: durable={} but the active \ + memtable spans [{}, {})", + stats.durable_batch_count, + stats.global_offset, + stats.global_offset + stats.batch_count + ); + + // And the WAL really holds every row we acked (3 puts x 5 rows). + writer.close().await.unwrap(); + let tailer = WalTailer::new(store, base_path, shard_id); + let first = tailer.first_position().await.unwrap(); + let next = tailer.next_position().await.unwrap(); + let mut wal_rows = 0; + for position in first..next { + if let Some(entry) = tailer.read_entry(position).await.unwrap() { + wal_rows += entry.batches.iter().map(|b| b.num_rows()).sum::(); + } + } + assert_eq!( + wal_rows, 15, + "every acked row must be in the WAL; a post-rotation put that acked without an \ + append would leave rows missing" + ); + } + + /// A background tick must append the **oldest** store that still owes the WAL, + /// never "whatever memtable is active". + /// + /// A tick carries no store: it is enqueued by a timer and resolved when it is + /// handled. So a tick enqueued before a freeze is handled *after* it, and + /// resolving to the active memtable would append the incoming memtable's + /// batches ahead of the outgoing memtable's tail. WAL entry positions are + /// assigned in append-call order, replay walks them ascending, row positions + /// follow, and primary-key recency is "newest visible row position wins" — so + /// that inverts dedup after a crash, handing the key to the stale row. It + /// survives the crash that caused it, and a full scan cannot see it. + #[test] + fn test_next_pending_store_picks_the_oldest_owing_an_append() { + let schema = create_test_schema(); + + // A frozen store of 2 batches at coordinate 0, and the active store that + // rotated in behind it at coordinate 2. + let frozen = Arc::new(BatchStore::with_capacity(4)); + frozen.append(create_test_batch(&schema, 0, 1)).unwrap(); + frozen.append(create_test_batch(&schema, 1, 1)).unwrap(); + let active = Arc::new(BatchStore::with_capacity_at(4, 2)); + active.append(create_test_batch(&schema, 2, 1)).unwrap(); + + let frozen_list = || std::iter::once(Arc::clone(&frozen)); + + // Nothing durable: the frozen store owes the oldest append, so it wins — + // even though the active memtable also has un-appended batches. + let picked = next_pending_store(frozen_list(), Arc::clone(&active), 0).unwrap(); + assert!( + Arc::ptr_eq(&picked, &frozen), + "the outgoing memtable's tail must be appended before the incoming one's head" + ); + + // Still true partway through the frozen store. + let picked = next_pending_store(frozen_list(), Arc::clone(&active), 1).unwrap(); + assert!(Arc::ptr_eq(&picked, &frozen)); + + // Once the frozen store is fully durable, the active one is next. + let picked = next_pending_store(frozen_list(), Arc::clone(&active), 2).unwrap(); + assert!(Arc::ptr_eq(&picked, &active)); + + // Everything durable: nothing to do. + assert!(next_pending_store(frozen_list(), Arc::clone(&active), 3).is_none()); + } + + /// A durable writer with no flush ticker cannot make progress in either + /// mode — the ticker is the only thing that drives the WAL append the put + /// waits on — so `open()` rejects it rather than letting a put block forever. + #[rstest] + #[case::memtable(true)] + #[case::wal_only(false)] + #[tokio::test] + async fn test_open_rejects_durable_write_without_a_ticker(#[case] enable_memtable: bool) { + let (store, base_path, base_uri, _temp_dir) = create_local_store().await; + let schema = schema_with_pk(); + let shard_id = Uuid::new_v4(); + + let config = ShardWriterConfig { + durable_write: true, + enable_memtable, + max_wal_flush_interval: None, + ..memtable_config_with_pk(shard_id) + }; + + let Err(err) = ShardWriter::open(store, base_path, base_uri, config, schema, vec![]).await + else { + panic!("durable_write with no ticker must be rejected"); + }; + assert!( + err.to_string().contains("max_wal_flush_interval"), + "the error must name the knob, got: {err}" + ); + } + + /// WAL entries must be appended in global batch-position order across a + /// memtable rotation, because append order *is* primary-key recency order. + /// + /// `WalAppender::append` assigns each entry's position from its own counter, + /// in call order. Replay walks those positions ascending and assigns row + /// positions in that order. Primary-key recency is "newest visible row + /// position wins". So an out-of-order append silently inverts dedup after a + /// crash — the stale row wins — and a full scan cannot see it. + /// + /// The hazard is the background ticker. If it resolved to "whatever memtable + /// is active" rather than to the oldest store still owing an append, a tick + /// enqueued before a freeze but handled after it would append the *incoming* + /// memtable's batches ahead of the outgoing memtable's tail. So the target is + /// resolved from the durability cursor, not from wall-clock timing. + #[tokio::test] + async fn test_wal_append_order_preserves_pk_recency_across_rotation() { + use crate::dataset::mem_wal::wal::WalTailer; + + let (store, base_path, base_uri, _temp_dir) = create_local_store().await; + let schema = schema_with_pk(); + let shard_id = Uuid::new_v4(); + + // Two batches per memtable, so the second put fills it and rotates. + let config = ShardWriterConfig { + max_memtable_batches: 2, + ..memtable_config_with_pk(shard_id) + }; + + let writer = ShardWriter::open( + store.clone(), + base_path.clone(), + base_uri, + config, + schema.clone(), + vec![], + ) + .await + .unwrap(); + + // Memtable 1: ids 7 then 20 (the second fills it and triggers the freeze). + writer + .put(vec![create_test_batch(&schema, 7, 1)]) + .await + .unwrap(); + writer + .put(vec![create_test_batch(&schema, 20, 1)]) + .await + .unwrap(); + // Memtable 2 overwrites id=7 with a newer row. Its append must land in the + // WAL *after* memtable 1's, or replay would resolve id=7 to the stale copy. + writer + .put(vec![create_test_batch(&schema, 30, 1)]) + .await + .unwrap(); + writer.close().await.unwrap(); + + // Walk the WAL in entry order and collect the ids as replay would see them. + let tailer = WalTailer::new(store, base_path, shard_id); + let first = tailer.first_position().await.unwrap(); + let next = tailer.next_position().await.unwrap(); + let mut ids: Vec = Vec::new(); + for position in first..next { + let Some(entry) = tailer.read_entry(position).await.unwrap() else { + continue; + }; + for batch in &entry.batches { + let column = batch + .column_by_name("id") + .unwrap() + .as_any() + .downcast_ref::() + .unwrap(); + ids.extend((0..column.len()).map(|i| column.value(i))); + } + } + + assert_eq!( + ids, + vec![7, 20, 30], + "WAL entries must follow global batch-position order; memtable 1's rows must \ + precede memtable 2's, or replay inverts primary-key recency" + ); + } + + /// An index config that disagrees with the schema fails `open()` outright. + /// It must not be allowed to accept writes: the insert would fail + /// deterministically on every batch, including batches replayed from the + /// WAL, so once a row was durable the shard could never reopen. Before this + /// check, an FTS index on a non-Utf8 column silently indexed nothing and the + /// shard reported healthy. + #[tokio::test] + async fn test_open_rejects_index_config_that_disagrees_with_schema() { + let (store, base_path, base_uri, _temp_dir) = create_local_store().await; + let schema = schema_with_pk(); + let shard_id = Uuid::new_v4(); + + // `id` is Int32, not a string column. + let bad_fts = MemIndexConfig::Fts(FtsIndexConfig::new( + "bad_fts".to_string(), + 0, + "id".to_string(), + )); + + let Err(err) = ShardWriter::open( + store, + base_path, + base_uri, + memtable_config_with_pk(shard_id), + schema, + vec![bad_fts], + ) + .await + else { + panic!("open must reject an FTS index on a non-Utf8 column"); + }; + + let message = err.to_string(); + assert!( + message.contains("bad_fts") && message.contains("Utf8"), + "error must name the index and the constraint, got: {message}" + ); + } + + /// Replay is a no-op on a fresh shard: the MemTable starts empty. #[tokio::test] async fn test_memtable_replay_no_op_on_fresh_shard() { let (store, base_path, base_uri, _temp_dir) = create_local_store().await; @@ -4919,13 +7004,13 @@ mod tests { /// Regression for the OSS-WAL compactor-drain bug: after a flush /// records its generation in the manifest and an external compactor - /// later drains `flushed_generations` back to empty (the legitimate - /// outcome of merging the generation into the base table), reopening + /// later drains `sstables` back to empty (the legitimate + /// outcome after compacting the SSTable into the base table), reopening /// the writer must not re-replay the already-flushed WAL entry into /// the active memtable. /// /// Under the pre-fix logic, replay disambiguated "fresh shard" from - /// "flushed-then-compacted" with `flushed_generations.is_empty()`, + /// "flushed-then-compacted" with `sstables.is_empty()`, /// which collapsed both cases into start-at-0. With 1-based WAL /// positions and a default cursor of 0 meaning "no flush stamped", /// the flush-then-drain sequence leaves `replay_after_wal_entry_position` @@ -4939,7 +7024,7 @@ mod tests { let shard_id = Uuid::new_v4(); // Writer A: write 5 rows, close (forces a flush of the active - // memtable). The manifest now records a flushed generation and + // memtable). The manifest now records an SSTable and // pins `replay_after_wal_entry_position` to the covered WAL entry. { let writer_a = ShardWriter::open( @@ -4959,14 +7044,14 @@ mod tests { writer_a.close().await.unwrap(); } - // Simulate an external compactor merging the flushed generation - // into the base table: drain `flushed_generations` to empty via a + // Simulate an external compactor compacting the SSTable. + // into the base table: drain `sstables` to empty via a // direct manifest commit. The cursor stays where the flush put it. let manifest_store = ShardManifestStore::new(store.clone(), &base_path, shard_id, 2); let pre = manifest_store.read_latest().await.unwrap().unwrap(); assert!( - !pre.flushed_generations.is_empty(), - "writer A's close() should have stamped a flushed generation" + !pre.sstables.is_empty(), + "writer A's close() should have stamped an SSTable" ); let cursor_at_flush = pre.replay_after_wal_entry_position; assert!( @@ -4980,22 +7065,22 @@ mod tests { manifest_store .commit_update(compactor_epoch, |current| ShardManifest { version: current.version + 1, - flushed_generations: vec![], + sstables: vec![], ..current.clone() }) .await .unwrap(); let post = manifest_store.read_latest().await.unwrap().unwrap(); assert!( - post.flushed_generations.is_empty(), - "compactor drain should have left flushed_generations empty" + post.sstables.is_empty(), + "compactor drain should have left sstables empty" ); assert_eq!( post.replay_after_wal_entry_position, cursor_at_flush, "compactor must not touch the replay cursor" ); - // Writer B reopens. Pre-fix: replay saw flushed_generations empty, + // Writer B reopens. Pre-fix: replay saw sstables empty, // restarted at WAL position 0, and re-inserted writer A's rows. // Post-fix: replay starts at cursor + 1, finds no entry, and the // memtable stays empty. @@ -5230,8 +7315,9 @@ mod tests { /// fence), the drained batches were dropped — the next concurrent put /// would then see an empty pending queue and spuriously return Ok, /// hiding the data loss. With the snapshot/commit fix, the failed flush - /// leaves the batches in the queue, and the concurrent put gets a clean - /// fence error too (when its own flush attempts the same WAL position). + /// leaves the batches in the queue for retry, and — because the flush is + /// terminal — it poisons the writer, waking *both* parked durability + /// waiters with the typed fence error instead of either one hanging. #[tokio::test] async fn test_wal_only_fenced_concurrent_puts_do_not_silently_succeed() { use std::sync::Arc; @@ -5282,9 +7368,10 @@ mod tests { // the destructive-drain bug, the first flush would consume both // pending batches into a failing append; the second flush would // see an empty queue and return spurious success, silently losing - // the second put's data. With the snapshot/commit fix, the failed - // append leaves both batches in the queue and the second flush - // also fails with the fence error. + // the second put's data. Now both puts park on the durability + // watermark; the ticker's append fails with the fence, poisons the + // writer, and both waiters wake with the fence error — batches intact + // in the queue. let a1 = writer_a.clone(); let a2 = writer_a.clone(); let schema1 = schema.clone(); @@ -5343,55 +7430,225 @@ mod tests { } #[tokio::test] - async fn test_force_seal_active_and_wait_for_flush_drain() { + async fn test_memtable_stats_record_index_update() { + // MemTable mode with a BTree index: index application runs on its own + // task and must record an index-update stat. Regression for the stat + // silently reading zero after index apply moved off the WAL-flush path. + let (store, base_path, base_uri, _temp_dir) = create_local_store().await; + let schema = create_pk_test_schema(); + let index_configs = vec![MemIndexConfig::BTree(BTreeIndexConfig { + name: "id_idx".to_string(), + field_id: 0, + column: "id".to_string(), + })]; + + let writer = ShardWriter::open( + store, + base_path, + base_uri, + flush_test_config(Uuid::new_v4()), + schema.clone(), + index_configs, + ) + .await + .unwrap(); + writer + .put(vec![create_test_batch(&schema, 0, 3)]) + .await + .unwrap(); + + let stats_handle = writer.stats_handle(); + // `close()` drains the index-apply task, so the apply is settled here. + writer.close().await.unwrap(); + + let snapshot = stats_handle.snapshot(); + assert!( + snapshot.index_update_count >= 1, + "the index apply must record an index-update stat, got {}", + snapshot.index_update_count + ); + assert_eq!( + snapshot.index_update_rows, 3, + "every indexed row must be counted exactly once, got {}", + snapshot.index_update_rows + ); + assert!( + snapshot.avg_index_update_latency().is_some(), + "a recorded index update must expose an average latency" + ); + } + + #[tokio::test] + async fn test_force_seal_active_and_wait_for_flush_drain() { + let (store, base_path, base_uri, _temp_dir) = create_local_store().await; + let schema = create_test_schema(); + + // Thresholds high enough that auto-flush won't fire; the seal is + // the only thing that should rotate the memtable. + let config = ShardWriterConfig { + shard_id: Uuid::new_v4(), + shard_spec_id: 0, + durable_write: false, + max_wal_buffer_size: 64 * 1024 * 1024, + max_wal_flush_interval: Some(Duration::from_millis(10)), + max_memtable_size: 64 * 1024 * 1024, + manifest_scan_batch_size: 2, + ..Default::default() + }; + + let writer = ShardWriter::open(store, base_path, base_uri, config, schema.clone(), vec![]) + .await + .unwrap(); + + let initial_gen = writer.memtable_stats().await.unwrap().generation; + let flushed_before = writer + .manifest() + .await + .unwrap() + .map(|m| m.sstables.len()) + .unwrap_or(0); + + writer + .put(vec![create_test_batch(&schema, 0, 10)]) + .await + .unwrap(); + let fence = writer.force_seal_active().await.unwrap(); + assert_eq!(fence.sealed_generation(), Some(initial_gen)); + fence.wait().await.unwrap(); + + let stats = writer.memtable_stats().await.unwrap(); + assert_eq!(stats.generation, initial_gen + 1); + assert_eq!(stats.batch_count, 0); + + let manifest = writer + .manifest() + .await + .unwrap() + .expect("manifest should exist after flush"); + assert_eq!(manifest.sstables.len(), flushed_before + 1); + + writer.close().await.unwrap(); + } + + /// Durable writes so `put` returns only once the row is indexed and + /// WAL-durable. Both fence tests tear the background tasks down before + /// freezing, and a freeze still owing an index apply or a WAL append + /// would dispatch onto a closed channel and poison the writer. + fn seal_fence_test_config(shard_id: Uuid) -> ShardWriterConfig { + ShardWriterConfig { + shard_id, + shard_spec_id: 0, + durable_write: true, + max_wal_flush_interval: Some(Duration::from_millis(10)), + max_memtable_size: 64 * 1024 * 1024, + manifest_scan_batch_size: 2, + ..Default::default() + } + } + + /// An empty active memtable does not mean every pre-call write is in + /// L0: a size/interval trigger swaps generation N for an empty N+1 + /// while N's flush is still in flight. The seal must fence that + /// generation anyway — reporting "nothing sealed, nothing to wait for" + /// lets a caller acknowledge a flush that has not happened. + #[tokio::test] + async fn test_force_seal_active_fences_pending_generation_when_active_is_empty() { + let (store, base_path, base_uri, _temp_dir) = create_local_store().await; + let schema = create_test_schema(); + let writer = ShardWriter::open( + store, + base_path, + base_uri, + seal_fence_test_config(Uuid::new_v4()), + schema.clone(), + vec![], + ) + .await + .unwrap(); + writer + .put(vec![create_test_batch(&schema, 0, 10)]) + .await + .unwrap(); + + // Stop the flush tasks, then freeze by hand: this is the post-rotation + // state held still — generation N frozen and unflushed, N+1 active and + // empty. + writer.task_executor.shutdown_all().await.unwrap(); + let pending_generation = match &writer.mode { + WriterMode::MemTable { + state, + writer_state, + .. + } => { + let mut state = state.write().await; + writer_state.freeze_memtable(&mut state).unwrap() - 1 + } + WriterMode::WalOnly { .. } => unreachable!("opened in memtable mode"), + }; + + let fence = writer.force_seal_active().await.unwrap(); + assert_eq!( + fence.sealed_generation(), + None, + "the active memtable was empty, so this seal froze nothing" + ); + assert!( + tokio::time::timeout(Duration::from_millis(200), fence.wait()) + .await + .is_err(), + "generation {pending_generation} is still awaiting flush; the fence must not be satisfied" + ); + } + + /// The fence is tied to each flush's own outcome, never to + /// `ShardManifest::current_generation`. The manifest advances to + /// `generation + 1` on every committed flush without checking for a gap, + /// so a later generation's success moves it past one that failed — a + /// watermark comparison would report durability that does not exist. + #[tokio::test] + async fn test_force_seal_active_fence_ignores_manifest_generation_advance() { let (store, base_path, base_uri, _temp_dir) = create_local_store().await; let schema = create_test_schema(); - - // Thresholds high enough that auto-flush won't fire; the seal is - // the only thing that should rotate the memtable. - let config = ShardWriterConfig { - shard_id: Uuid::new_v4(), - shard_spec_id: 0, - durable_write: false, - sync_indexed_write: false, - max_wal_buffer_size: 64 * 1024 * 1024, - max_wal_flush_interval: None, - max_memtable_size: 64 * 1024 * 1024, - manifest_scan_batch_size: 2, - ..Default::default() - }; - - let writer = ShardWriter::open(store, base_path, base_uri, config, schema.clone(), vec![]) - .await - .unwrap(); - - let initial_gen = writer.memtable_stats().await.unwrap().generation; - let flushed_before = writer - .manifest() - .await - .unwrap() - .map(|m| m.flushed_generations.len()) - .unwrap_or(0); - + let writer = ShardWriter::open( + store, + base_path, + base_uri, + seal_fence_test_config(Uuid::new_v4()), + schema.clone(), + vec![], + ) + .await + .unwrap(); writer .put(vec![create_test_batch(&schema, 0, 10)]) .await .unwrap(); - writer.force_seal_active().await.unwrap(); - writer.wait_for_flush_drain().await.unwrap(); - let stats = writer.memtable_stats().await.unwrap(); - assert_eq!(stats.generation, initial_gen + 1); - assert_eq!(stats.batch_count, 0); + // No flush task, so the sealed generation never reaches L0. + writer.task_executor.shutdown_all().await.unwrap(); + let fence = writer.force_seal_active().await.unwrap(); + let sealed = fence + .sealed_generation() + .expect("the active memtable held rows"); - let manifest = writer - .manifest() + // Advance the manifest past the sealed generation, as a later + // generation's successful flush would. + writer + .manifest_store + .commit_update(writer.epoch(), |current| ShardManifest { + version: current.version + 1, + current_generation: sealed + 2, + ..current.clone() + }) .await - .unwrap() - .expect("manifest should exist after flush"); - assert_eq!(manifest.flushed_generations.len(), flushed_before + 1); + .unwrap(); - writer.close().await.unwrap(); + assert!( + tokio::time::timeout(Duration::from_millis(200), fence.wait()) + .await + .is_err(), + "generation {sealed} never reached L0; a manifest advance past it must not satisfy its fence" + ); } /// `abort` tears down the background flush tasks WITHOUT flushing — @@ -5408,9 +7665,8 @@ mod tests { shard_id: Uuid::new_v4(), shard_spec_id: 0, durable_write: false, - sync_indexed_write: false, max_wal_buffer_size: 64 * 1024 * 1024, - max_wal_flush_interval: None, + max_wal_flush_interval: Some(Duration::from_millis(10)), max_memtable_size: 64 * 1024 * 1024, manifest_scan_batch_size: 2, ..Default::default() @@ -5428,7 +7684,7 @@ mod tests { .manifest() .await .unwrap() - .map(|m| m.flushed_generations.len()) + .map(|m| m.sstables.len()) .unwrap_or(0); writer.abort().await.unwrap(); @@ -5439,7 +7695,7 @@ mod tests { .manifest() .await .unwrap() - .map(|m| m.flushed_generations.len()) + .map(|m| m.sstables.len()) .unwrap_or(0); assert_eq!( flushed_after, flushed_before, @@ -5463,9 +7719,8 @@ mod tests { shard_id: Uuid::new_v4(), shard_spec_id: 0, durable_write: false, - sync_indexed_write: false, max_wal_buffer_size: 64 * 1024 * 1024, - max_wal_flush_interval: None, + max_wal_flush_interval: Some(Duration::from_millis(10)), max_memtable_size: 64 * 1024 * 1024, manifest_scan_batch_size: 2, // Short grace so the sweep is observable without a slow test. @@ -5488,10 +7743,10 @@ mod tests { let manifest = writer.manifest().await.unwrap().expect("manifest exists"); assert!( manifest - .flushed_generations + .sstables .iter() .any(|g| g.generation == initial_gen), - "flushed generation must be recorded in the manifest" + "SSTable must be recorded in the manifest" ); // Still queryable in memory immediately after commit (within grace). @@ -5499,7 +7754,7 @@ mod tests { assert_eq!(refs.active.generation, initial_gen + 1); assert!( refs.frozen.iter().any(|f| f.generation == initial_gen), - "flushed generation must stay queryable during the grace window" + "SSTable must stay queryable during the grace window" ); // After the grace elapses (plus a sweep tick) the handle is evicted. @@ -5523,9 +7778,8 @@ mod tests { shard_id: Uuid::new_v4(), shard_spec_id: 0, durable_write: false, - sync_indexed_write: false, max_wal_buffer_size: 64 * 1024 * 1024, - max_wal_flush_interval: None, + max_wal_flush_interval: Some(Duration::from_millis(10)), max_memtable_size: 64 * 1024 * 1024, manifest_scan_batch_size: 2, frozen_memtable_grace: Duration::ZERO, @@ -5547,10 +7801,10 @@ mod tests { let manifest = writer.manifest().await.unwrap().expect("manifest exists"); assert!( manifest - .flushed_generations + .sstables .iter() .any(|g| g.generation == initial_gen), - "flushed generation must be recorded in the manifest" + "SSTable must be recorded in the manifest" ); // ...and the in-memory handle is already gone, no sweep tick needed. @@ -5563,6 +7817,54 @@ mod tests { writer.close().await.unwrap(); } + #[tokio::test] + async fn test_close_propagates_frozen_memtable_flush_failure() { + let (store, base_path, base_uri, _temp_dir) = create_local_store().await; + let schema = schema_with_pk(); + let shard_id = Uuid::new_v4(); + let writer_a = ShardWriter::open( + store.clone(), + base_path.clone(), + base_uri.clone(), + memtable_config_with_pk(shard_id), + schema.clone(), + vec![], + ) + .await + .unwrap(); + writer_a + .put(vec![create_test_batch(&schema, 0, 10)]) + .await + .unwrap(); + + let writer_b = ShardWriter::open( + store, + base_path, + base_uri, + memtable_config_with_pk(shard_id), + schema, + vec![], + ) + .await + .unwrap(); + assert!(writer_b.epoch() > writer_a.epoch()); + + let error = writer_a + .close() + .await + .expect_err("close must propagate the fenced MemTable flush"); + assert!( + matches!(error, Error::IO { .. }), + "unexpected error: {error}" + ); + assert!( + error.to_string().contains("Writer fenced"), + "unexpected error: {error}" + ); + + writer_b.close().await.unwrap(); + } + /// Regression: a transient flush failure must NOT reopen the /// concurrent-read-vs-flush hole. The sealed generation stays in the /// queryable set (rows intact) until a later flush or WAL replay. @@ -5636,6 +7938,18 @@ mod tests { ); assert_eq!(refs.active.generation, initial_gen + 1); + // Nor did it vanish from the stats. Backpressure released these bytes on + // flush completion so the failure cannot wedge writes, but the memtable + // is still resident, and this snapshot is what an operator reads to + // decide whether to evict the shard. + let stats = writer_a.memtable_stats().await.unwrap(); + assert_eq!(stats.frozen_count, 1); + assert!( + stats.frozen_bytes >= refs.frozen[0].batch_store.estimated_bytes(), + "a failed flush must keep owing its resident bytes, got {}", + stats.frozen_bytes + ); + writer_b.close().await.unwrap(); } } @@ -5786,9 +8100,7 @@ mod shard_writer_tests { Some("100") ); // Every tunable field is present. - assert!(defaults.contains_key("sync_indexed_write")); assert!(defaults.contains_key("enable_memtable")); - assert!(defaults.contains_key("async_index_interval_ms")); // add_writer_config_default records arbitrary keys. assert_eq!( defaults.get("custom_knob").map(String::as_str), @@ -5811,7 +8123,12 @@ mod shard_writer_tests { let vector_dim = 32; let schema = create_test_schema(vector_dim); - let uri = format!("memory://test_multi_segment_index_{}", Uuid::new_v4()); + // The generation flusher reopens by URI, so this independent open must + // resolve to the same in-memory backend. The unique authority isolates the test. + let uri = format!( + "shared-memory://multi-segment-index-{}/", + Uuid::new_v4().simple() + ); // Initial fragment + an IVF vector index covering it. let initial = create_test_batch(&schema, 0, 256, vector_dim); @@ -5949,7 +8266,7 @@ mod shard_writer_tests { // The tombstone-only generation still flushed (data without an HNSW index). let manifest = writer.manifest().await.unwrap().expect("manifest exists"); assert_eq!( - manifest.flushed_generations.len(), + manifest.sstables.len(), 1, "the all-tombstone generation must still flush" ); @@ -5995,9 +8312,7 @@ mod shard_writer_tests { .expect("Failed to initialize MemWAL"); let shard_id = Uuid::new_v4(); - let config = ShardWriterConfig::new(shard_id) - .with_durable_write(true) - .with_sync_indexed_write(true); + let config = ShardWriterConfig::new(shard_id).with_durable_write(true); let writer = dataset .mem_wal_writer(shard_id, config) .await @@ -6018,22 +8333,22 @@ mod shard_writer_tests { .await .expect("Failed to read manifest") .expect("Manifest should exist"); - assert_eq!(manifest.flushed_generations.len(), 1); + assert_eq!(manifest.sstables.len(), 1); - let flushed = &manifest.flushed_generations[0]; - let gen_uri = format!("{}/_mem_wal/{}/{}", uri, shard_id, flushed.path); - let flushed_dataset = Dataset::open(&gen_uri) + let sstable = &manifest.sstables[0]; + let gen_uri = format!("{}/_mem_wal/{}/{}", uri, shard_id, sstable.path); + let sstable = Dataset::open(&gen_uri) .await - .expect("Failed to open flushed generation"); - let flushed_indices = flushed_dataset.load_indices().await.unwrap(); - assert_eq!(flushed_indices.len(), 1); - assert_eq!(flushed_indices[0].name, "text_fts"); + .expect("Failed to open SSTable"); + let sstable_indices = sstable.load_indices().await.unwrap(); + assert_eq!(sstable_indices.len(), 1); + assert_eq!(sstable_indices[0].name, "text_fts"); assert_eq!( - flushed_indices[0].index_version, 1, + sstable_indices[0].index_version, 1, "maintained v1 FTS index must flush as v1" ); - let results = flushed_dataset + let results = sstable .scan() .full_text_search(FullTextSearchQuery::new("Sample".to_owned())) .unwrap() @@ -6049,7 +8364,12 @@ mod shard_writer_tests { let vector_dim = 32; let schema = create_test_schema(vector_dim); - let uri = format!("memory://test_writer_hnsw_params_{}", Uuid::new_v4()); + // The generation flusher reopens by URI, so this independent open must + // resolve to the same in-memory backend. The unique authority isolates the test. + let uri = format!( + "shared-memory://writer-hnsw-params-{}/", + Uuid::new_v4().simple() + ); let initial = create_test_batch(&schema, 0, 256, vector_dim); let batches = RecordBatchIterator::new([Ok(initial)], schema.clone()); @@ -6330,9 +8650,7 @@ mod shard_writer_tests { // Create shard writer let shard_id = Uuid::new_v4(); - let config = ShardWriterConfig::new(shard_id) - .with_durable_write(false) - .with_sync_indexed_write(false); + let config = ShardWriterConfig::new(shard_id).with_durable_write(false); let writer = dataset .mem_wal_writer(shard_id, config) @@ -6357,7 +8675,12 @@ mod shard_writer_tests { let target_id = 1_000i64 + 37; let schema = create_test_schema(vector_dim); - let uri = format!("memory://test_shard_writer_hnsw_{}", Uuid::new_v4()); + // The generation flusher reopens by URI, so this independent open must + // resolve to the same in-memory backend. The unique authority isolates the test. + let uri = format!( + "shared-memory://shard-writer-hnsw-{}/", + Uuid::new_v4().simple() + ); let initial_batch = create_test_batch(&schema, 0, 256, vector_dim); let batches = RecordBatchIterator::new([Ok(initial_batch)], schema.clone()); @@ -6385,9 +8708,7 @@ mod shard_writer_tests { .expect("Failed to initialize MemWAL"); let shard_id = Uuid::new_v4(); - let config = ShardWriterConfig::new(shard_id) - .with_durable_write(true) - .with_sync_indexed_write(true); + let config = ShardWriterConfig::new(shard_id).with_durable_write(true); let writer = dataset .mem_wal_writer(shard_id, config) @@ -6526,9 +8847,7 @@ mod shard_writer_tests { // Create shard writer with default config let shard_id = Uuid::new_v4(); - let config = ShardWriterConfig::new(shard_id) - .with_durable_write(false) - .with_sync_indexed_write(false); + let config = ShardWriterConfig::new(shard_id).with_durable_write(false); let writer = dataset .mem_wal_writer(shard_id, config) @@ -6553,7 +8872,7 @@ mod shard_writer_tests { /// 2. File system layout is correct (WAL files, manifest, generation directories) /// 3. WAL entries contain expected data /// 4. Data can be read after each flush cycle - /// 5. Manifest tracks flushed generations correctly + /// 5. Manifest tracks SSTables correctly /// /// Run with: cargo test -p lance shard_writer_tests::test_shard_writer_e2e_correctness -- --nocapture #[tokio::test] @@ -6605,7 +8924,6 @@ mod shard_writer_tests { let shard_id = Uuid::new_v4(); let config = ShardWriterConfig::new(shard_id) .with_durable_write(true) // Ensure WAL files are written - .with_sync_indexed_write(true) .with_max_memtable_size(50 * 1024) // 50KB - triggers flush after ~8 batches .with_max_wal_buffer_size(10 * 1024) // 10KB WAL buffer .with_max_wal_flush_interval(Duration::from_millis(50)); // Fast flush @@ -6682,24 +9000,24 @@ mod shard_writer_tests { .expect("Failed to read manifest") .expect("Manifest should exist"); - // Verify flushed generations exist on disk + // Verify SSTables exist on disk assert!( - !manifest.flushed_generations.is_empty(), - "Should have at least one flushed generation" + !manifest.sstables.is_empty(), + "Should have at least one SSTable" ); - for flushed_gen in &manifest.flushed_generations { + for sstable in &manifest.sstables { // The path stored in manifest is relative to the shard directory // Construct full path: temp_dir/_mem_wal/shard_id/generation_folder let gen_path = temp_dir .path() .join("_mem_wal") .join(shard_id.to_string()) - .join(&flushed_gen.path); + .join(&sstable.path); // The generation directory should exist assert!( gen_path.exists(), - "Flushed generation directory should exist at {:?}", + "SSTable directory should exist at {:?}", gen_path ); @@ -6729,9 +9047,13 @@ mod shard_writer_tests { // Re-open dataset and create new writer to verify recovery let dataset = Dataset::open(&uri).await.expect("Failed to reopen dataset"); let new_shard_id = Uuid::new_v4(); - let new_config = ShardWriterConfig::new(new_shard_id) - .with_durable_write(false) - .with_sync_indexed_write(true); + // `durable_write(true)` so the put waits for its flush, which is what + // currently publishes the rows. A non-durable put is *not* yet + // read-your-writes: the index apply is welded to the WAL flush, so the + // rows stay invisible until the next flush. This test used to pass with + // `durable_write(false)` only because an un-advanced cursor of 0 was + // misread as "batch 0 is visible" — it was asserting the dirty read. + let new_config = ShardWriterConfig::new(new_shard_id).with_durable_write(true); let new_writer = dataset .mem_wal_writer(new_shard_id, new_config) @@ -6754,4 +9076,309 @@ mod shard_writer_tests { .await .expect("Failed to close new writer"); } + + /// Regression: a base opened with a *path-bound* store binding (the + /// deprecated `ObjectStoreParams::object_store`) must still flush and read + /// generations at their own paths. + /// + /// The binding pins a store to one location, and both + /// `ObjectStore::from_uri_and_params` and `DatasetBuilder::build_object_store` + /// take the path from it while ignoring the URI they were handed. Reusing the + /// base's params verbatim therefore aimed every generation write and open at + /// the base table itself: the flush failed ("dataset already exists") and any + /// derived open returned base rows as generation rows. + #[tokio::test] + async fn test_flush_and_read_with_path_bound_object_store() { + use crate::dataset::mem_wal::scanner::{LsmScanner, ShardSnapshot}; + use futures::TryStreamExt; + use lance_io::object_store::ObjectStoreParams; + use tempfile::TempDir; + + let vector_dim = 8; + let schema = create_test_schema(vector_dim); + let temp_dir = TempDir::new().unwrap(); + let uri = format!("file://{}", temp_dir.path().display()); + + let initial = create_test_batch(&schema, 0, 16, vector_dim); + let batches = RecordBatchIterator::new([Ok(initial)], schema.clone()); + let mut dataset = Dataset::write(batches, &uri, Some(WriteParams::default())) + .await + .expect("Failed to create dataset"); + dataset + .initialize_mem_wal() + .execute() + .await + .expect("Failed to initialize MemWAL"); + + // Re-bind the base to a store pinned at the base's own path — what + // `DatasetBuilder::with_object_store` leaves on an opened dataset. + #[allow(deprecated)] + let store_params = ObjectStoreParams { + object_store: Some(( + Arc::new(object_store::local::LocalFileSystem::new()), + url::Url::parse(&uri).unwrap(), + )), + ..Default::default() + }; + let dataset = dataset.with_object_store(dataset.object_store.clone(), Some(store_params)); + + let shard_id = Uuid::new_v4(); + let writer = dataset + .mem_wal_writer(shard_id, ShardWriterConfig::new(shard_id)) + .await + .expect("Failed to create writer"); + writer + .put(vec![create_test_batch(&schema, 1_000, 8, vector_dim)]) + .await + .expect("Failed to write"); + writer.force_seal_active().await.unwrap(); + writer + .wait_for_flush_drain() + .await + .expect("flush must not be redirected at the base table"); + + let manifest = writer.manifest().await.unwrap().expect("manifest exists"); + assert_eq!(manifest.sstables.len(), 1); + let sstable = manifest.sstables[0].clone(); + + // The generation landed under `_mem_wal/`, and the base table is untouched. + let gen_uri = format!("{}/_mem_wal/{}/{}", uri, shard_id, sstable.path); + let generation = Dataset::open(&gen_uri) + .await + .expect("generation must exist at its own path"); + assert_eq!(generation.count_rows(None).await.unwrap(), 8); + let base = Dataset::open(&uri).await.unwrap(); + assert_eq!( + base.count_rows(None).await.unwrap(), + 16, + "the generation write must not land in the base table" + ); + + // The read path resolves the generation, not the base: 16 base + 8 flushed. + // Opening the base instead would dedup back down to 16 rows. + let snapshot = ShardSnapshot::new(shard_id) + .with_current_generation(manifest.current_generation) + .with_sstable(sstable.generation, sstable.path.clone()); + let scanner = LsmScanner::new(Arc::new(dataset), vec![snapshot], vec!["id".to_string()]); + let rows: usize = scanner + .try_into_stream() + .await + .unwrap() + .try_collect::>() + .await + .expect("scan must open the generation, not the base") + .iter() + .map(|batch| batch.num_rows()) + .sum(); + assert_eq!(rows, 24); + + writer.close().await.unwrap(); + } + + /// The store params a base was opened with must reach every *derived* open: + /// the flush that writes a generation and the scan that reads it back. + /// + /// This is the point of threading them at all. A namespace-vended store + /// exists only on the params (credentials, endpoint, wrapper), so a + /// generation resolved by URI alone would silently sign with the ambient + /// identity instead — succeeding against a local store and failing against + /// the vended one. Asserting on the *generation folder* rather than + /// `_mem_wal/` is what makes this bite: WAL entries are written through the + /// base dataset's own store, so they would show up here either way. + #[tokio::test] + async fn test_store_params_reach_generation_write_and_read() { + use crate::dataset::builder::DatasetBuilder; + use crate::dataset::mem_wal::scanner::{LsmScanner, ShardSnapshot}; + use crate::dataset::mem_wal::test_util::observable_store_params; + use futures::TryStreamExt; + use tempfile::TempDir; + + let vector_dim = 8; + let schema = create_test_schema(vector_dim); + let temp_dir = TempDir::new().unwrap(); + let uri = format!("file://{}", temp_dir.path().display()); + + let initial = create_test_batch(&schema, 0, 16, vector_dim); + let batches = RecordBatchIterator::new([Ok(initial)], schema.clone()); + Dataset::write(batches, &uri, Some(WriteParams::default())) + .await + .expect("Failed to create dataset"); + + // Open the base through an observable store, exactly as a namespace + // client would hand in a vended-credential store. + let (store_params, controls) = observable_store_params(); + let mut dataset = DatasetBuilder::from_uri(&uri) + .with_store_params(store_params) + .load() + .await + .expect("Failed to open dataset"); + dataset + .initialize_mem_wal() + .execute() + .await + .expect("Failed to initialize MemWAL"); + + let shard_id = Uuid::new_v4(); + let writer = dataset + .mem_wal_writer(shard_id, ShardWriterConfig::new(shard_id)) + .await + .expect("Failed to create writer"); + writer + .put(vec![create_test_batch(&schema, 1_000, 8, vector_dim)]) + .await + .expect("Failed to write"); + writer.force_seal_active().await.unwrap(); + writer.wait_for_flush_drain().await.expect("flush failed"); + + let manifest = writer.manifest().await.unwrap().expect("manifest exists"); + assert_eq!(manifest.sstables.len(), 1); + let sstable = manifest.sstables[0].clone(); + + // The generation's own Lance manifest is the signal to key on. Keying on + // the generation folder alone would pass vacuously: sidecars like + // `{gen}/bloom_filter.bin` are written through the *base* dataset's + // store, which is observable no matter what the params do. And the + // fragments can't be used either — `ObjectStore::create` writes local + // files through `tokio::fs`, bypassing the object store entirely, so + // `{gen}/data/` never reaches a wrapper under `file://`. The manifest + // goes through `put_opts`, and only the flusher's `Dataset::write` / + // `open_generation` writes it — both of which must carry the params. + let gen_manifest = format!("{}/_versions", sstable.path); + + assert!( + controls.wrote_under(&gen_manifest), + "the flush must write the generation through the base's store params, \ + not a store resolved from the generation URI alone" + ); + + // And the read path must resolve the generation through them too. + let snapshot = ShardSnapshot::new(shard_id) + .with_current_generation(manifest.current_generation) + .with_sstable(sstable.generation, sstable.path.clone()); + let scanner = LsmScanner::new(Arc::new(dataset), vec![snapshot], vec!["id".to_string()]); + let rows: usize = scanner + .try_into_stream() + .await + .unwrap() + .try_collect::>() + .await + .expect("scan failed") + .iter() + .map(|batch| batch.num_rows()) + .sum(); + assert_eq!(rows, 24); + + // Reads key on the data files, not the manifest: the flusher already + // pulled the generation's manifest into the shared session cache, so the + // scan's open serves it from memory and never touches the store. The + // fragments are read through it (reads have no local bypass), as is the + // generation's standalone PK index. + assert!( + controls.read_under(&format!("{}/data/", sstable.path)), + "the scan must read the generation through the base's store params" + ); + + writer.close().await.unwrap(); + } + + /// A fresh-tier-only scanner reaches its store params through + /// `with_store_params`, not `new()`, so the setter must strip the path-bound + /// store binding too. Left raw, it redirects the generation open at the base + /// table and the scan silently returns base rows as WAL rows. + #[tokio::test] + async fn test_fresh_tier_scan_with_path_bound_object_store() { + use crate::dataset::mem_wal::scanner::{LsmScanner, ShardSnapshot}; + use futures::TryStreamExt; + use lance_io::object_store::ObjectStoreParams; + use tempfile::TempDir; + + let vector_dim = 8; + let schema = create_test_schema(vector_dim); + let temp_dir = TempDir::new().unwrap(); + let uri = format!("file://{}", temp_dir.path().display()); + + // 16 base rows with ids 0..16; the WAL gets 8 rows with ids 1000..1008, + // so a redirected generation open is unambiguous in the output. + let initial = create_test_batch(&schema, 0, 16, vector_dim); + let batches = RecordBatchIterator::new([Ok(initial)], schema.clone()); + let mut dataset = Dataset::write(batches, &uri, Some(WriteParams::default())) + .await + .expect("Failed to create dataset"); + dataset + .initialize_mem_wal() + .execute() + .await + .expect("Failed to initialize MemWAL"); + + let shard_id = Uuid::new_v4(); + let writer = dataset + .mem_wal_writer(shard_id, ShardWriterConfig::new(shard_id)) + .await + .expect("Failed to create writer"); + writer + .put(vec![create_test_batch(&schema, 1_000, 8, vector_dim)]) + .await + .expect("Failed to write"); + writer.force_seal_active().await.unwrap(); + writer.wait_for_flush_drain().await.expect("flush failed"); + + let manifest = writer.manifest().await.unwrap().expect("manifest exists"); + let sstable = manifest.sstables[0].clone(); + let snapshot = ShardSnapshot::new(shard_id) + .with_current_generation(manifest.current_generation) + .with_sstable(sstable.generation, sstable.path.clone()); + + // What `DatasetBuilder::with_object_store` leaves on an opened dataset: + // a store pinned at the base's own path. + #[allow(deprecated)] + let store_params = ObjectStoreParams { + object_store: Some(( + Arc::new(object_store::local::LocalFileSystem::new()), + url::Url::parse(&uri).unwrap(), + )), + ..Default::default() + }; + + let arrow_schema: Arc = schema.clone(); + let batches = LsmScanner::without_base_table( + arrow_schema, + uri.clone(), + vec![snapshot], + vec!["id".to_string()], + ) + .with_session(dataset.session()) + .with_store_params(store_params) + .try_into_stream() + .await + .unwrap() + .try_collect::>() + .await + .expect("scan must open the generation, not the base"); + + let rows: usize = batches.iter().map(|batch| batch.num_rows()).sum(); + assert_eq!( + rows, 8, + "fresh tier holds only the 8 WAL rows; 16 means the generation open \ + was redirected at the base table" + ); + let ids: Vec = batches + .iter() + .flat_map(|batch| { + batch + .column_by_name("id") + .unwrap() + .as_any() + .downcast_ref::() + .unwrap() + .values() + .to_vec() + }) + .collect(); + assert!( + ids.iter().all(|id| (1_000..1_008).contains(id)), + "expected the WAL's own rows, got {ids:?}" + ); + + writer.close().await.unwrap(); + } } diff --git a/rust/lance/src/dataset/optimize.rs b/rust/lance/src/dataset/optimize.rs index 656f0163ce6..a0060ca6b13 100644 --- a/rust/lance/src/dataset/optimize.rs +++ b/rust/lance/src/dataset/optimize.rs @@ -88,45 +88,53 @@ use std::ops::{AddAssign, Range}; use std::sync::Arc; use super::fragment::FileFragment; -use super::index::DatasetIndexRemapperOptions; +use super::index::{DatasetIndexRemapperOptions, load_indices_for_remapping}; use super::rowids::load_row_id_sequences; use super::transaction::{ Operation, RewriteGroup, RewrittenIndex, Transaction, TransactionBuilder, }; use super::utils::make_rowid_capture_stream; +use super::versions; use super::{WriteMode, WriteParams, cleanup_data_fragments, write_fragments_internal}; use crate::Dataset; use crate::Result; use crate::dataset::utils::CapturedRowIds; use crate::index::DatasetIndexExt; -use crate::io::commit::{commit_transaction, migrate_fragments}; +use crate::io::commit::{DEFAULT_COMMIT_RETRY_TIMEOUT, commit_transaction, migrate_fragments}; use arrow::array::AsArray; use arrow::datatypes::{UInt8Type, UInt32Type, UInt64Type}; -use arrow_array::Array; -use arrow_array::RecordBatch; -use arrow_array::StructArray; use arrow_array::builder::{LargeBinaryBuilder, PrimitiveBuilder, StringBuilder}; -use arrow_buffer::NullBuffer; +use arrow_array::{ + Array, ArrayRef, GenericListArray, OffsetSizeTrait, RecordBatch, StructArray, UInt32Array, +}; +use arrow_buffer::{OffsetBuffer, ScalarBuffer}; +use arrow_schema::{ + DataType as ArrowDataType, Field as ArrowField, Schema as ArrowSchema, SchemaRef, +}; use datafusion::physical_plan::SendableRecordBatchStream; use datafusion::physical_plan::stream::RecordBatchStreamAdapter; -use futures::{StreamExt, TryStreamExt}; +use futures::future::BoxFuture; +use futures::{FutureExt, StreamExt, TryStreamExt}; +use lance_arrow::{list::ListArrayExt, r#struct::StructArrayExt}; use lance_core::Error; -use lance_core::datatypes::{BlobHandling, BlobKind}; +use lance_core::datatypes::{ + BLOB_V2_LOGICAL_FIELDS, BLOB_V2_LOGICAL_TYPE, BlobHandling, BlobKind, BlobV2Layout, + Field as LanceField, +}; use lance_core::utils::tokio::get_num_compute_intensive_cpus; use lance_core::utils::tracing::{DATASET_COMPACTING_EVENT, TRACE_DATASET_EVENTS}; -use lance_index::frag_reuse::FragReuseGroup; +use lance_index::frag_reuse::{FRAG_REUSE_INDEX_NAME, FragReuseGroup}; use lance_index::is_system_index; use lance_table::format::{Fragment, RowIdMeta}; use roaring::{RoaringBitmap, RoaringTreemap}; use serde::{Deserialize, Serialize}; use tracing::{info, warn}; -mod binary_copy; +pub(super) mod binary_copy; pub mod remapping; use crate::index::frag_reuse::build_new_frag_reuse_index; use crate::io::deletion::read_dataset_deletion_file; -use binary_copy::rewrite_files_binary_copy; pub use remapping::{IgnoreRemap, IndexRemapper, IndexRemapperOptions, RemappedIndex}; /// Controls how data is rewritten during compaction. @@ -262,6 +270,15 @@ pub struct CompactionOptions { /// fragments at a time). /// Defaults to `None` (no limit, all eligible fragments are compacted). pub max_source_fragments: Option, + /// Maximum number of data overlay files a fragment may carry before it is + /// fully compacted. When set, any fragment with more than this many overlays + /// is rewritten into a fresh fragment with its overlays (and deletions) + /// materialized into the base data, dropping the fragment from any index + /// left stale by those overlays. + /// Defaults to `Some(10)`. Set to `Some(0)` to compact every fragment that + /// carries any overlay, or `None` to disable the overlay-count trigger + /// entirely. + pub max_overlays_per_fragment: Option, /// Transaction properties to store with this commit. /// /// These key-value pairs are stored in the transaction file @@ -291,6 +308,7 @@ impl Default for CompactionOptions { enable_binary_copy_force: false, binary_copy_read_batch_bytes: Some(16 * 1024 * 1024), max_source_fragments: None, + max_overlays_per_fragment: Some(10), transaction_properties: None, } } @@ -317,6 +335,7 @@ impl CompactionOptions { /// - `lance.compaction.compaction_mode` /// - `lance.compaction.binary_copy_read_batch_bytes` /// - `lance.compaction.max_source_fragments` + /// - `lance.compaction.max_overlays_per_fragment` pub fn from_dataset_config(config: &HashMap) -> Result { let mut opts = Self::default(); opts.apply_dataset_config(config)?; @@ -427,6 +446,19 @@ impl CompactionOptions { )) })?); } + "max_overlays_per_fragment" => { + // The default is `Some(10)`, so an explicit "none" is the only + // way to disable the trigger through the manifest config. + self.max_overlays_per_fragment = match value.to_ascii_lowercase().as_str() { + "none" => None, + _ => Some(value.parse().map_err(|_| { + Error::invalid_input(format!( + "Invalid value for {}: '{}' (expected a non-negative integer or 'none')", + key, value + )) + })?), + }; + } _ => { warn!("Ignoring unknown compaction config key: {}", key); } @@ -480,7 +512,8 @@ async fn can_use_binary_copy( options: &CompactionOptions, fragments: &[Fragment], ) -> bool { - can_use_binary_copy_impl(dataset, options, fragments) + let version = dataset.manifest.data_storage_format.lance_file_format(); + versions::can_use_binary_copy(version, dataset, options, fragments) .await .unwrap_or_else(|err| { log::warn!("Binary copy disabled due to error: {}", err); @@ -488,13 +521,12 @@ async fn can_use_binary_copy( }) } -async fn can_use_binary_copy_impl( +pub(super) async fn can_use_binary_copy_current( dataset: &Dataset, options: &CompactionOptions, fragments: &[Fragment], ) -> Result { use lance_file::reader::FileReader as LFReader; - use lance_file::version::LanceFileVersion; use lance_io::scheduler::{ScanScheduler, SchedulerConfig}; if matches!(options.compaction_mode(), CompactionMode::Reencode) { @@ -511,28 +543,11 @@ async fn can_use_binary_copy_impl( return Ok(false); } - let storage_ok = dataset - .manifest - .data_storage_format - .lance_file_version() - .map(|v| !matches!(v.resolve(), LanceFileVersion::Legacy)) - .unwrap_or(false); - if !storage_ok { - log::debug!("Binary copy disabled: dataset uses legacy storage format"); - return Ok(false); - } - if fragments.is_empty() { log::debug!("Binary copy disabled: no fragments to compact"); return Ok(false); } - let storage_file_version = dataset - .manifest - .data_storage_format - .lance_file_version()? - .resolve(); - if fragments[0].files.is_empty() { log::debug!( "Binary copy disabled: fragment {} has no data files", @@ -542,8 +557,6 @@ async fn can_use_binary_copy_impl( } let ref_fields = &fragments[0].files[0].fields; let ref_cols = &fragments[0].files[0].column_indices; - let mut is_same_version = true; - for fragment in fragments { if fragment.deletion_file.is_some() { log::debug!( @@ -554,16 +567,6 @@ async fn can_use_binary_copy_impl( } for data_file in &fragment.files { - let version_ok = LanceFileVersion::try_from_major_minor( - data_file.file_major_version, - data_file.file_minor_version, - ) - .map(|v| v.resolve()) - .is_ok_and(|v| v == storage_file_version); - - if !version_ok { - is_same_version = false; - } if data_file.fields != *ref_fields || data_file.column_indices != *ref_cols { return Ok(false); } @@ -600,11 +603,6 @@ async fn can_use_binary_copy_impl( } } - if !is_same_version { - log::debug!("Binary copy disabled: data files use different file versions"); - return Ok(false); - } - Ok(true) } @@ -714,7 +712,16 @@ impl CompactionPlanner for DefaultCompactionPlanner { while let Some(res) = fragment_metrics.next().await { let (fragment, metrics) = res?; - let candidacy = if self.options.materialize_deletions + let over_overlay_limit = self + .options + .max_overlays_per_fragment + .is_some_and(|max| fragment.overlays.len() > max); + + let candidacy = if over_overlay_limit { + // Too many overlays: fully compact this fragment on its own, + // regardless of its size or deletion count. + Some(CompactionCandidacy::CompactItself) + } else if self.options.materialize_deletions && metrics.deletion_percentage() > self.options.materialize_deletions_threshold { Some(CompactionCandidacy::CompactItself) @@ -840,7 +847,7 @@ pub async fn compact_files_with_planner( let dataset_ref = &dataset.clone(); - let result_stream = futures::stream::iter(compaction_plan.tasks.into_iter()) + let result_stream = futures::stream::iter(compaction_plan.tasks) .map(|task| rewrite_files(Cow::Borrowed(dataset_ref), task, &compaction_plan.options)) .buffer_unordered( compaction_plan @@ -938,7 +945,7 @@ impl CompactionPlan { /// Classification for one blob v2 row during compaction. /// -/// - `Null`: NULL row or Inline blob with position=0 and size=0. +/// - `Null`: NULL row. /// - `External`: External blob referenced by URI. /// - `DataBlob`: Inline/Packed/Dedicated blob stored in Lance files. enum RowClass { @@ -947,25 +954,6 @@ enum RowClass { DataBlob, } -/// Check if a row is a null Inline blob. -/// -/// This matches `BlobV2StructuralEncoder`'s behavior of encoding null rows as -/// Inline with position=0 and size=0, and `collect_blob_entries_v2`'s behavior -/// of skipping them. -fn is_inline_null_blob( - kind: BlobKind, - position_col: &arrow::array::UInt64Array, - size_col: &arrow::array::UInt64Array, - index: usize, -) -> bool { - if kind != BlobKind::Inline { - return false; - } - let position_is_empty = position_col.is_null(index) || position_col.value(index) == 0; - let size_is_empty = size_col.is_null(index) || size_col.value(index) == 0; - position_is_empty && size_is_empty -} - /// Column views for the 5 fields in a blob v2 descriptor struct. struct BlobV2Descriptor<'a> { kind_col: &'a arrow::array::UInt8Array, @@ -978,6 +966,14 @@ struct BlobV2Descriptor<'a> { impl<'a> BlobV2Descriptor<'a> { /// Extract the 5 descriptor arrays from a blob v2 descriptor struct array. fn try_from_struct(struct_arr: &'a StructArray, column_name: &str) -> Result { + if BlobV2Layout::classify(struct_arr.fields()) != Some(BlobV2Layout::Descriptor) { + let actual = BlobV2Layout::classify(struct_arr.fields()) + .map(|layout| layout.to_string()) + .unwrap_or_else(|| format!("unrecognized ({:?})", struct_arr.fields())); + return Err(Error::invalid_input(format!( + "Blob v2 column '{column_name}' has {actual} layout; expected descriptor layout before conversion to logical" + ))); + } let kind_col = struct_arr .column_by_name("kind") .ok_or_else(|| { @@ -1033,22 +1029,21 @@ impl<'a> BlobV2Descriptor<'a> { } } -/// Result of row classification for blob v2 compaction. +/// Result of row classification for a blob v2 rewrite. struct RowClassification { row_classes: Vec, - blob_read_addrs: Vec, + data_blob_indices: Vec, } /// Classify each row of a blob v2 column as Null, External, or DataBlob. fn classify_rows( struct_arr: &StructArray, descriptor: &BlobV2Descriptor<'_>, - row_addrs: &arrow::array::UInt64Array, column_name: &str, ) -> Result { let num_rows = struct_arr.len(); let mut row_classes = Vec::with_capacity(num_rows); - let mut blob_read_addrs = Vec::with_capacity(num_rows); + let mut data_blob_indices = Vec::with_capacity(num_rows); for i in 0..num_rows { if struct_arr.is_null(i) || descriptor.kind_col.is_null(i) { @@ -1062,41 +1057,101 @@ fn classify_rows( })?; if kind == BlobKind::External { row_classes.push(RowClass::External); - } else if is_inline_null_blob(kind, descriptor.position_col, descriptor.size_col, i) { - row_classes.push(RowClass::Null); } else { row_classes.push(RowClass::DataBlob); - blob_read_addrs.push(row_addrs.value(i)); + data_blob_indices.push(i); } } } Ok(RowClassification { row_classes, - blob_read_addrs, + data_blob_indices, }) } -/// Build a blob v2 user-view struct array from classification and descriptor. +/// Convert a blob v2 descriptor into the logical writer representation. /// /// Reads blob data lazily using row addresses to avoid materializing all blob /// payloads in memory at once. -async fn build_user_view_struct( +async fn descriptor_to_logical_blob_array( dataset: &Arc, + blob_field_id: u32, + struct_arr: &StructArray, descriptor: &BlobV2Descriptor<'_>, classification: &RowClassification, - column_name: &str, - num_rows: usize, - null_buffer: Option, + row_addrs: &[u64], + field_name: &str, ) -> Result { - let blob_files = if classification.blob_read_addrs.is_empty() { + if struct_arr.len() != row_addrs.len() { + return Err(Error::internal(format!( + "Blob v2 field '{}' row count {} did not match row address count {}", + field_name, + struct_arr.len(), + row_addrs.len() + ))); + } + let blob_files = if classification.data_blob_indices.is_empty() { Vec::new() } else { - super::blob::take_blobs_by_addresses(dataset, &classification.blob_read_addrs, column_name) - .await? + let indices = classification + .data_blob_indices + .iter() + .map(|index| { + u32::try_from(*index).map_err(|_| { + Error::internal(format!( + "Blob v2 row index {} in field '{}' does not fit in u32", + index, field_name + )) + }) + }) + .collect::>>()?; + let indices = UInt32Array::from(indices); + let descriptions = arrow_select::take::take(struct_arr, &indices, None)?; + let descriptions = descriptions.as_struct(); + let data_row_addrs = classification + .data_blob_indices + .iter() + .map(|index| row_addrs[*index]) + .collect::>(); + super::blob::collect_blob_v2_descriptor_files( + dataset, + blob_field_id, + descriptions, + &data_row_addrs, + ) + .await? }; - let mut data_builder = LargeBinaryBuilder::with_capacity(num_rows, 0); + let data_capacity = + classification + .data_blob_indices + .iter() + .try_fold(0usize, |data_capacity, row_idx| { + if descriptor.size_col.is_null(*row_idx) { + return Err(Error::internal(format!( + "Non-null blob row {} in field '{}' is missing its size", + row_idx, field_name + ))); + } + let size = usize::try_from(descriptor.size_col.value(*row_idx)).map_err(|_| { + Error::internal(format!( + "Blob size {} at row {} in field '{}' does not fit in usize", + descriptor.size_col.value(*row_idx), + row_idx, + field_name + )) + })?; + data_capacity.checked_add(size).ok_or_else(|| { + Error::internal(format!( + "Total blob size in field '{}' exceeds usize", + field_name + )) + }) + })?; + + let num_rows = struct_arr.len(); + let mut data_builder = LargeBinaryBuilder::with_capacity(num_rows, data_capacity); let mut uri_builder = StringBuilder::with_capacity(num_rows, 0); let mut out_position_builder = PrimitiveBuilder::::with_capacity(num_rows); let mut out_size_builder = PrimitiveBuilder::::with_capacity(num_rows); @@ -1121,7 +1176,7 @@ async fn build_user_view_struct( let base = dataset.manifest().base_paths.get(&base_id).ok_or_else(|| { Error::internal(format!( "External blob in column '{}' references unknown base_id {}", - column_name, base_id + field_name, base_id )) })?; let absolute_uri = format!("{}/{}", base.path.trim_end_matches('/'), uri_val); @@ -1139,7 +1194,16 @@ async fn build_user_view_struct( } } RowClass::DataBlob => { - let data = blob_files[blob_file_idx].read().await?; + let blob_file = blob_files + .get(blob_file_idx) + .and_then(Option::as_ref) + .ok_or_else(|| { + Error::internal(format!( + "Non-null blob row {} in field '{}' resolved to null", + i, field_name + )) + })?; + let data = blob_file.read().await?; blob_file_idx += 1; data_builder.append_value(data.as_ref()); uri_builder.append_null(); @@ -1150,110 +1214,456 @@ async fn build_user_view_struct( } Ok(StructArray::try_new( - lance_core::datatypes::BLOB_V2_USER_FIELDS.clone(), + BLOB_V2_LOGICAL_FIELDS.clone(), vec![ Arc::new(data_builder.finish()), Arc::new(uri_builder.finish()), Arc::new(out_position_builder.finish()), Arc::new(out_size_builder.finish()), ], - null_buffer, + struct_arr.nulls().cloned(), )?) } -async fn transform_blob_v2_batch( - dataset: &Arc, - schema: &lance_core::datatypes::Schema, - batch: RecordBatch, -) -> Result { - let row_addr_idx = batch - .schema() - .column_with_name(lance_core::ROW_ADDR) - .ok_or_else(|| { - Error::internal(format!( - "_rowaddr column missing from batch for blob v2 compaction, columns: {:?}", - batch - .schema() - .fields() - .iter() - .map(|f| f.name()) - .collect::>() - )) - })? - .0; - let row_addrs = batch.column(row_addr_idx).as_primitive::(); +fn transformed_arrow_field(field: &LanceField, data_type: ArrowDataType) -> Arc { + let arrow_field = ArrowField::from(field); + arrow_field_with_data_type(&arrow_field, data_type) +} - let mut new_columns: Vec> = Vec::new(); - let mut new_fields: Vec> = Vec::new(); +fn arrow_field_with_data_type(field: &ArrowField, data_type: ArrowDataType) -> Arc { + Arc::new( + ArrowField::new(field.name(), data_type, field.is_nullable()) + .with_metadata(field.metadata().clone()), + ) +} - let batch_schema = batch.schema(); - for (col_idx, field) in batch_schema.fields().iter().enumerate() { - if field.name() == lance_core::ROW_ADDR { - continue; - } +pub(crate) fn field_contains_blob_v2(field: &LanceField) -> bool { + field.is_blob_v2() || field.children.iter().any(field_contains_blob_v2) +} - let lance_field = schema.field(field.name()); - let is_blob_v2 = lance_field.is_some_and(|f| f.is_blob_v2()); +enum BlobV2FieldRewritePlan { + Passthrough { + output_field: Arc, + }, + Blob { + field_id: u32, + field_name: String, + output_field: Arc, + }, + Struct { + field_name: String, + output_field: Arc, + children: Vec, + }, + List { + field_name: String, + output_field: Arc, + child: Box, + }, + LargeList { + field_name: String, + output_field: Arc, + child: Box, + }, +} - if !is_blob_v2 { - new_columns.push(batch.column(col_idx).clone()); - new_fields.push(field.clone()); - continue; +impl BlobV2FieldRewritePlan { + fn passthrough(field: &ArrowField) -> Self { + Self::Passthrough { + output_field: Arc::new(field.clone()), } + } - let struct_arr = batch - .column(col_idx) - .as_any() - .downcast_ref::() - .ok_or_else(|| { + fn try_new(field: &LanceField, input_field: &ArrowField) -> Result { + if !field_contains_blob_v2(field) { + return Ok(Self::passthrough(input_field)); + } + + if field.is_blob_v2() { + let field_id = u32::try_from(field.id).map_err(|_| { Error::internal(format!( - "Blob v2 column '{}' expected StructArray, got {:?}", - field.name(), - batch.column(col_idx).data_type() + "Blob v2 field id {} for '{}' does not fit in u32", + field.id, field.name )) })?; + let ArrowDataType::Struct(input_children) = input_field.data_type() else { + return Err(Error::invalid_input(format!( + "Blob v2 field '{}' has non-struct input type {:?}", + field.name, + input_field.data_type() + ))); + }; + let output_field = match BlobV2Layout::classify(input_children) { + Some(BlobV2Layout::Logical) => Arc::new(input_field.clone()), + Some(BlobV2Layout::Descriptor) => { + transformed_arrow_field(field, BLOB_V2_LOGICAL_TYPE.clone()) + } + Some(actual) => { + return Err(Error::invalid_input(format!( + "Blob v2 field '{}' has {actual} input layout; expected logical or descriptor layout during rewrite", + field.name + ))); + } + None => { + return Err(Error::invalid_input(format!( + "Blob v2 field '{}' has unrecognized input layout {:?}; expected logical or descriptor layout during rewrite", + field.name, input_children + ))); + } + }; + return Ok(Self::Blob { + field_id, + field_name: field.name.clone(), + output_field, + }); + } + + match (field.data_type(), input_field.data_type()) { + (ArrowDataType::Struct(_), ArrowDataType::Struct(input_children)) => { + if field.children.len() != input_children.len() { + return Err(Error::internal(format!( + "Struct field '{}' expected {} children in blob rewrite plan, got {}", + field.name, + field.children.len(), + input_children.len() + ))); + } + let children = field + .children + .iter() + .zip(input_children.iter()) + .map(|(child, input_child)| Self::try_new(child, input_child)) + .collect::>>()?; + let output_children = children + .iter() + .map(|child| child.output_field().clone()) + .collect::>(); + Ok(Self::Struct { + field_name: field.name.clone(), + output_field: arrow_field_with_data_type( + input_field, + ArrowDataType::Struct(output_children.into()), + ), + children, + }) + } + (ArrowDataType::List(_), ArrowDataType::List(input_child)) => { + let child = field.children.first().ok_or_else(|| { + Error::internal(format!( + "List field '{}' is missing its child in blob rewrite plan", + field.name + )) + })?; + let child = Box::new(Self::try_new(child, input_child)?); + Ok(Self::List { + field_name: field.name.clone(), + output_field: arrow_field_with_data_type( + input_field, + ArrowDataType::List(child.output_field().clone()), + ), + child, + }) + } + (ArrowDataType::LargeList(_), ArrowDataType::LargeList(input_child)) => { + let child = field.children.first().ok_or_else(|| { + Error::internal(format!( + "Large list field '{}' is missing its child in blob rewrite plan", + field.name + )) + })?; + let child = Box::new(Self::try_new(child, input_child)?); + Ok(Self::LargeList { + field_name: field.name.clone(), + output_field: arrow_field_with_data_type( + input_field, + ArrowDataType::LargeList(child.output_field().clone()), + ), + child, + }) + } + (logical_type, input_type) => Err(Error::invalid_input(format!( + "Field '{}' contains blob v2 descendants but logical type {:?} and input type {:?} do not form a supported rewrite container", + field.name, logical_type, input_type + ))), + } + } - let column_name = field.name(); - let descriptor = BlobV2Descriptor::try_from_struct(struct_arr, column_name)?; - let classification = classify_rows(struct_arr, &descriptor, row_addrs, column_name)?; - let num_rows = struct_arr.len(); + fn output_field(&self) -> &Arc { + match self { + Self::Passthrough { output_field } + | Self::Blob { output_field, .. } + | Self::Struct { output_field, .. } + | Self::List { output_field, .. } + | Self::LargeList { output_field, .. } => output_field, + } + } - let new_struct = build_user_view_struct( - dataset, - &descriptor, - &classification, - column_name, - num_rows, - struct_arr.nulls().cloned(), - ) - .await?; + fn transform<'a>( + &'a self, + dataset: &'a Arc, + array: ArrayRef, + row_addrs: Arc<[u64]>, + ) -> BoxFuture<'a, Result> { + async move { + match self { + Self::Passthrough { .. } => Ok(array), + Self::Blob { + field_id, + field_name, + .. + } => { + let struct_arr = array.as_struct(); + match BlobV2Layout::classify(struct_arr.fields()) { + Some(BlobV2Layout::Logical) => Ok(array), + Some(BlobV2Layout::Descriptor) => { + let descriptor = + BlobV2Descriptor::try_from_struct(struct_arr, field_name)?; + let classification = + classify_rows(struct_arr, &descriptor, field_name)?; + let logical = descriptor_to_logical_blob_array( + dataset, + *field_id, + struct_arr, + &descriptor, + &classification, + row_addrs.as_ref(), + field_name, + ) + .await?; + Ok(Arc::new(logical) as ArrayRef) + } + Some(actual) => Err(Error::invalid_input(format!( + "Blob v2 field '{}' has {actual} layout; expected logical or descriptor layout during rewrite", + field_name + ))), + None => Err(Error::invalid_input(format!( + "Blob v2 field '{}' has unrecognized layout {:?}; expected logical or descriptor layout during rewrite", + field_name, + struct_arr.fields() + ))), + } + } + Self::Struct { + field_name, + output_field, + children, + } => { + let struct_arr = array.as_struct().normalize_slicing()?; + let parent_nulls = struct_arr.nulls().cloned(); + let struct_arr = struct_arr.pushdown_nulls()?; + if children.len() != struct_arr.num_columns() { + return Err(Error::internal(format!( + "Struct field '{}' expected {} children during blob rewrite, got {}", + field_name, + children.len(), + struct_arr.num_columns() + ))); + } + let mut child_arrays = Vec::with_capacity(children.len()); + for (child, child_array) in children.iter().zip(struct_arr.columns()) { + child_arrays.push( + child + .transform(dataset, child_array.clone(), row_addrs.clone()) + .await?, + ); + } + let ArrowDataType::Struct(output_fields) = output_field.data_type() else { + return Err(Error::internal(format!( + "Struct field '{}' rewrite plan has non-struct output type {:?}", + field_name, + output_field.data_type() + ))); + }; + Ok(Arc::new(StructArray::try_new( + output_fields.clone(), + child_arrays, + parent_nulls, + )?) as ArrayRef) + } + Self::List { + field_name, child, .. + } => { + transform_blob_v2_list_array::( + dataset, + field_name, + child, + array.as_list::(), + row_addrs, + ) + .await + } + Self::LargeList { + field_name, child, .. + } => { + transform_blob_v2_list_array::( + dataset, + field_name, + child, + array.as_list::(), + row_addrs, + ) + .await + } + } + } + .boxed() + } +} - new_columns.push(Arc::new(new_struct)); - let logical_field = arrow_schema::Field::from(lance_field.ok_or_else(|| { +async fn transform_blob_v2_list_array( + dataset: &Arc, + field_name: &str, + child: &BlobV2FieldRewritePlan, + list_array: &GenericListArray, + row_addrs: Arc<[u64]>, +) -> Result { + let list_array = if list_array.null_count() > 0 { + list_array.filter_garbage_nulls() + } else { + list_array.clone() + }; + let offsets = list_array.value_offsets(); + let values_start = offsets[0].as_usize(); + let values_end = offsets[list_array.len()].as_usize(); + if values_end < values_start { + return Err(Error::internal(format!( + "List field '{}' has invalid offsets during blob rewrite", + field_name + ))); + } + + let values_len = values_end - values_start; + let mut normalized_offsets = Vec::with_capacity(list_array.len() + 1); + normalized_offsets.push(O::usize_as(0)); + let mut child_row_addrs = Vec::with_capacity(values_len); + for row_idx in 0..list_array.len() { + let start = offsets[row_idx].as_usize(); + let end = offsets[row_idx + 1].as_usize(); + if end < start { + return Err(Error::internal(format!( + "List field '{}' has decreasing offsets during blob rewrite", + field_name + ))); + } + let row_addr = row_addrs.get(row_idx).copied().ok_or_else(|| { Error::internal(format!( - "Blob v2 column '{}' missing from dataset schema during compaction", - field.name() + "List field '{}' row address count {} did not match row count {}", + field_name, + row_addrs.len(), + list_array.len() )) - })?); - new_fields.push(Arc::new( - arrow_schema::Field::new( - field.name(), - lance_core::datatypes::BLOB_V2_USER_TYPE.clone(), - field.is_nullable(), - ) - .with_metadata(logical_field.metadata().clone()), - )); + })?; + child_row_addrs.extend(std::iter::repeat_n(row_addr, end - start)); + normalized_offsets.push(O::usize_as(end - values_start)); + } + + let values = list_array.values().slice(values_start, values_len); + let values = child + .transform(dataset, values, Arc::<[u64]>::from(child_row_addrs)) + .await?; + let list_array = GenericListArray::::try_new( + child.output_field().clone(), + OffsetBuffer::new(ScalarBuffer::from(normalized_offsets)), + values, + list_array.nulls().cloned(), + )?; + Ok(Arc::new(list_array)) +} + +pub(crate) struct BlobV2BatchRewritePlan { + row_addr_idx: usize, + columns: Vec<(usize, BlobV2FieldRewritePlan)>, + output_schema: SchemaRef, +} + +impl BlobV2BatchRewritePlan { + pub(crate) fn try_new( + schema: &lance_core::datatypes::Schema, + input_schema: &ArrowSchema, + keep_row_addr: bool, + ) -> Result { + let row_addr_idx = input_schema + .column_with_name(lance_core::ROW_ADDR) + .ok_or_else(|| { + Error::internal(format!( + "_rowaddr column missing from batch for blob v2 rewrite, columns: {:?}", + input_schema + .fields() + .iter() + .map(|f| f.name()) + .collect::>() + )) + })? + .0; + let mut columns = Vec::with_capacity(input_schema.fields().len()); + let mut output_fields = Vec::with_capacity(input_schema.fields().len()); + for (column_idx, input_field) in input_schema.fields().iter().enumerate() { + if input_field.name() == lance_core::ROW_ADDR && !keep_row_addr { + continue; + } + let field_plan = if let Some(field) = schema.field(input_field.name()) { + BlobV2FieldRewritePlan::try_new(field, input_field)? + } else { + BlobV2FieldRewritePlan::passthrough(input_field) + }; + output_fields.push(field_plan.output_field().as_ref().clone()); + columns.push((column_idx, field_plan)); + } + + Ok(Self { + row_addr_idx, + columns, + output_schema: Arc::new(ArrowSchema::new_with_metadata( + output_fields, + input_schema.metadata().clone(), + )), + }) } - let new_schema = Arc::new(arrow_schema::Schema::new_with_metadata( - new_fields + pub(crate) fn output_schema(&self) -> &SchemaRef { + &self.output_schema + } + + pub(crate) async fn transform_batch( + &self, + dataset: &Arc, + batch: RecordBatch, + ) -> Result { + let row_addrs: Arc<[u64]> = batch + .column(self.row_addr_idx) + .as_primitive::() + .values() .iter() - .map(|f| f.as_ref().clone()) - .collect::>(), - batch_schema.metadata().clone(), - )); + .copied() + .collect::>() + .into(); + let mut output_columns = Vec::with_capacity(self.columns.len()); + for (column_idx, field_plan) in &self.columns { + output_columns.push( + field_plan + .transform( + dataset, + batch.column(*column_idx).clone(), + row_addrs.clone(), + ) + .await?, + ); + } + Ok(RecordBatch::try_new( + self.output_schema.clone(), + output_columns, + )?) + } +} - Ok(RecordBatch::try_new(new_schema, new_columns)?) +pub(crate) async fn transform_blob_v2_batch( + dataset: &Arc, + schema: &lance_core::datatypes::Schema, + batch: RecordBatch, + keep_row_addr: bool, +) -> Result { + let plan = BlobV2BatchRewritePlan::try_new(schema, batch.schema().as_ref(), keep_row_addr)?; + plan.transform_batch(dataset, batch).await } /// Build a scan reader for rewrite and optionally capture row IDs. @@ -1417,36 +1827,58 @@ impl CandidateBin { } /// Split into one or more bins with at least `min_num_rows` in them. - fn split_for_size(mut self, min_num_rows: usize) -> Vec { - let mut bins = Vec::new(); - - loop { - let mut bin_len = 0; - let mut bin_row_count = 0; - while bin_row_count < min_num_rows && bin_len < self.row_counts.len() { - bin_row_count += self.row_counts[bin_len]; - bin_len += 1; + fn split_for_size(self, min_num_rows: usize) -> Vec { + let total_rows = self.row_counts.iter().sum::(); + let mut remaining_rows = total_rows; + let mut current_rows = 0; + let mut current_len = 0; + let mut split_lengths = Vec::new(); + + for row_count in &self.row_counts { + current_rows += *row_count; + current_len += 1; + remaining_rows -= *row_count; + + // Only split once the current bin is large enough and there is + // enough left over to form another worthwhile non-empty bin. + if current_rows >= min_num_rows && remaining_rows > 0 && remaining_rows >= min_num_rows + { + split_lengths.push(current_len); + current_rows = 0; + current_len = 0; } + } - // If there's enough remaining to make another worthwhile bin, then - // push what we have as a bin. - if self.row_counts[bin_len..].iter().sum::() >= min_num_rows { - bins.push(Self { - fragments: self.fragments.drain(0..bin_len).collect(), - pos_range: self.pos_range.start..(self.pos_range.start + bin_len), - candidacy: self.candidacy.drain(0..bin_len).collect(), - row_counts: self.row_counts.drain(0..bin_len).collect(), - // By the time we are splitting for size we are done considering indices - indices: Vec::new(), - }); - self.pos_range.start += bin_len; - } else { - // Otherwise, just push the remaining fragments into the last bin - bins.push(self); - break; - } + if split_lengths.is_empty() { + return vec![self]; + } + + let mut bins = Vec::with_capacity(split_lengths.len() + 1); + let mut fragments = self.fragments.into_iter(); + let mut candidacy = self.candidacy.into_iter(); + let mut row_counts = self.row_counts.into_iter(); + let mut pos_start = self.pos_range.start; + + for bin_len in split_lengths { + bins.push(Self { + fragments: fragments.by_ref().take(bin_len).collect(), + pos_range: pos_start..(pos_start + bin_len), + candidacy: candidacy.by_ref().take(bin_len).collect(), + row_counts: row_counts.by_ref().take(bin_len).collect(), + // By the time we are splitting for size we are done considering indices + indices: Vec::new(), + }); + pos_start += bin_len; } + bins.push(Self { + fragments: fragments.collect(), + pos_range: pos_start..self.pos_range.end, + candidacy: candidacy.collect(), + row_counts: row_counts.collect(), + indices: self.indices, + }); + bins } } @@ -1463,7 +1895,12 @@ async fn load_index_fragmaps(dataset: &Dataset) -> Result> { index_fragmaps.push(fragment_bitmap.clone()); } else { let dataset_at_index = dataset.checkout_version(index.dataset_version).await?; - let frags = 0..dataset_at_index.manifest.max_fragment_id.unwrap_or(0); + // max_fragment_id is inclusive (the highest id); +1 for an exclusive + // upper bound so the last fragment is covered (None => empty range). + let frags = 0..dataset_at_index + .manifest + .max_fragment_id + .map_or(0, |m| m + 1); index_fragmaps.push(RoaringBitmap::from_sorted_iter(frags).unwrap()); } } @@ -1519,6 +1956,7 @@ async fn reserve_fragment_ids( &transaction, &Default::default(), &Default::default(), + DEFAULT_COMMIT_RETRY_TIMEOUT, dataset.manifest_location.naming_scheme, None, ) @@ -1570,8 +2008,13 @@ async fn rewrite_files( .iter() .map(|f| f.physical_rows.unwrap() as u64) .sum::(); - // If we aren't using stable row ids, then we need to remap indices. - let needs_remapping = !dataset.manifest.uses_stable_row_ids(); + // Capturing row addresses is only useful if something will consume them: + // an index to remap now, or a deferred remap through the FRI. + let capture_row_addrs = !dataset.manifest.uses_stable_row_ids() + && (options.defer_index_remap + || load_indices_for_remapping(dataset.as_ref()) + .await? + .is_some()); let mut new_fragments: Vec; let task_id = uuid::Uuid::new_v4(); log::info!( @@ -1597,7 +2040,7 @@ async fn rewrite_files( options.batch_size, options.io_buffer_size, true, - needs_remapping, + capture_row_addrs, ) .await?; row_ids_rx = rx_initial; @@ -1616,46 +2059,23 @@ async fn rewrite_files( if has_blob_v2_columns { let dataset_arc = Arc::new(dataset.as_ref().clone()); - let dataset_schema = dataset.schema().clone(); + let rewrite_plan = Arc::new(BlobV2BatchRewritePlan::try_new( + dataset.schema(), + schema.as_ref(), + false, + )?); + let transformed_schema = rewrite_plan.output_schema.clone(); let transformed = reader_with_progress.then(move |batch_result| { let dataset = dataset_arc.clone(); - let schema = dataset_schema.clone(); + let rewrite_plan = rewrite_plan.clone(); async move { let batch = batch_result?; - transform_blob_v2_batch(&dataset, &schema, batch) + rewrite_plan + .transform_batch(&dataset, batch) .await .map_err(|e| datafusion::error::DataFusionError::External(Box::new(e))) } }); - let transformed_schema = { - let mut fields: Vec> = Vec::new(); - for field in schema.fields().iter() { - if field.name() == lance_core::ROW_ADDR { - continue; - } - let lance_field = dataset.schema().field(field.name()); - if let Some(lance_field) = lance_field.filter(|f| f.is_blob_v2()) { - let logical_field = arrow_schema::Field::from(lance_field); - fields.push(Arc::new( - arrow_schema::Field::new( - field.name(), - lance_core::datatypes::BLOB_V2_USER_TYPE.clone(), - field.is_nullable(), - ) - .with_metadata(logical_field.metadata().clone()), - )); - } else { - fields.push(field.clone()); - } - } - Arc::new(arrow_schema::Schema::new_with_metadata( - fields - .iter() - .map(|f| f.as_ref().clone()) - .collect::>(), - schema.metadata().clone(), - )) - }; reader = Some(Box::pin(RecordBatchStreamAdapter::new( transformed_schema, transformed, @@ -1687,7 +2107,9 @@ async fn rewrite_files( } if can_binary_copy { - new_fragments = rewrite_files_binary_copy( + let version = dataset.manifest.data_storage_format.lance_file_format(); + new_fragments = versions::rewrite_files_binary_copy( + version, dataset.as_ref(), &fragments, ¶ms, @@ -1701,7 +2123,7 @@ async fn rewrite_files( )); } - if needs_remapping { + if capture_row_addrs { let (tx, rx) = std::sync::mpsc::channel(); let mut addrs = RoaringTreemap::new(); for frag in &fragments { @@ -1721,6 +2143,7 @@ async fn rewrite_files( } } else { let (frags, _) = write_fragments_internal( + dataset.manifest.data_storage_format.lance_file_format(), Some(dataset.as_ref()), dataset.object_store.clone(), &dataset.base, @@ -1850,7 +2273,7 @@ async fn rechunk_stable_row_ids( for (fragment, sequence) in new_fragments.iter_mut().zip(new_sequences) { // TODO: if large enough, serialize to separate file let serialized = lance_table::rowids::write_row_ids(&sequence); - fragment.row_id_meta = Some(RowIdMeta::Inline(serialized)); + fragment.row_id_meta = Some(RowIdMeta::Inline(serialized.into())); } Ok(()) @@ -1941,8 +2364,8 @@ async fn recalc_versions_for_rewritten_fragments( // Set both version metadata on new fragments for ((fragment, last_updated_seq), created_at_seq) in new_fragments .iter_mut() - .zip(new_last_updated_sequences.into_iter()) - .zip(new_created_at_sequences.into_iter()) + .zip(new_last_updated_sequences) + .zip(new_created_at_sequences) { fragment.last_updated_at_version_meta = Some( lance_table::format::RowDatasetVersionMeta::from_sequence(&last_updated_seq).unwrap(), @@ -1971,8 +2394,17 @@ pub async fn commit_compaction( return Ok(CompactionMetrics::default()); } - // If we aren't using stable row ids, then we need to remap indices. - let needs_remapping = !dataset.manifest.uses_stable_row_ids() && !options.defer_index_remap; + let has_address_style = completed_tasks.iter().any(|t| t.row_addrs.is_some()); + // Address-style results require immediate index remapping unless it is deferred. + let needs_remapping = + !dataset.manifest.uses_stable_row_ids() && !options.defer_index_remap && has_address_style; + + // Confirm there is a remapper before materializing the potentially very large row address map. + let index_remapper = if needs_remapping { + remap_options.create_remapper(dataset).await? + } else { + None + }; // Determine the earliest version at which compaction tasks were planned/executed. // @@ -1995,15 +2427,26 @@ pub async fn commit_compaction( let mut completed_tasks = completed_tasks; + // Collect the rewritten fragments' file paths up front so every failure + // path below can clean them up (or deliberately keep them). Fragment ids + // may still be reassigned by reserve_fragment_ids; cleanup only needs the + // file paths, which never change. + let all_new_fragments: Vec = completed_tasks + .iter() + .flat_map(|t| t.new_fragments.iter().cloned()) + .collect(); + // Single reserve_fragment_ids for all address-style tasks - let has_address_style = completed_tasks.iter().any(|t| t.row_addrs.is_some()); if has_address_style { let frags: Vec<&mut Fragment> = completed_tasks .iter_mut() .filter(|t| t.row_addrs.is_some()) .flat_map(|t| t.new_fragments.iter_mut()) .collect(); - reserve_fragment_ids(dataset, frags.into_iter()).await?; + if let Err(e) = reserve_fragment_ids(dataset, frags.into_iter()).await { + cleanup_compaction_files_after_reservation_failure(dataset, &all_new_fragments).await; + return Err(e); + } } let mut rewrite_groups = Vec::with_capacity(completed_tasks.len()); @@ -2014,6 +2457,31 @@ pub async fn commit_compaction( let mut frag_reuse_groups: Vec = Vec::new(); let mut new_fragment_bitmap: RoaringBitmap = RoaringBitmap::new(); + // Write an FRI only when the compaction touches data an index must later + // remap: a rewrite group covered by a data index, or by the existing FRI's new + // fragments (the composed remap chain). Compacting only not-yet-indexed data + // needs no FRI (one written for it is un-drainable). Decide all-or-nothing per + // compaction, never per group -- a partial FRI is unsound: a concurrent reindex + // can make a skipped fragment indexed and the conflict resolver's FRI-present + // path won't re-check it. + let indexed_frags: RoaringBitmap = if options.defer_index_remap { + let mut covered = RoaringBitmap::new(); + for bm in load_index_fragmaps(dataset).await? { + covered |= bm; + } + if let Some(bm) = dataset + .load_index_by_name(FRAG_REUSE_INDEX_NAME) + .await? + .and_then(|fri| fri.fragment_bitmap) + { + covered |= bm; + } + covered + } else { + RoaringBitmap::new() + }; + let mut any_group_indexed = false; + for task in completed_tasks { metrics += task.metrics; let rewrite_group = RewriteGroup { @@ -2021,7 +2489,7 @@ pub async fn commit_compaction( new_fragments: task.new_fragments.clone(), }; - if needs_remapping { + if index_remapper.is_some() { if let Some(row_addrs_bytes) = task.row_addrs { let row_addrs = RoaringTreemap::deserialize_from(&mut Cursor::new(&row_addrs_bytes))?; @@ -2062,6 +2530,14 @@ pub async fn commit_compaction( } } } else if options.defer_index_remap { + // Record every group; track whether any touches indexed/chain data. + if task + .original_fragments + .iter() + .any(|f| indexed_frags.contains(f.id as u32)) + { + any_group_indexed = true; + } let changed_row_addrs = task.row_addrs.ok_or_else(|| { Error::internal( "defer_index_remap requires row_addrs but none were provided".to_string(), @@ -2080,8 +2556,7 @@ pub async fn commit_compaction( rewrite_groups.push(rewrite_group); } - let rewritten_indices = if needs_remapping { - let index_remapper = remap_options.create_remapper(dataset)?; + let rewritten_indices = if let Some(index_remapper) = index_remapper { let affected_ids = rewrite_groups .iter() .flat_map(|group| group.old_fragments.iter().map(|frag| frag.id)) @@ -2110,25 +2585,27 @@ pub async fn commit_compaction( .iter_mut() .flat_map(|group| group.new_fragments.iter_mut()) .collect::>(); - reserve_fragment_ids(dataset, new_fragments.into_iter()).await?; + if let Err(e) = reserve_fragment_ids(dataset, new_fragments.into_iter()).await { + cleanup_compaction_files_after_reservation_failure(dataset, &all_new_fragments).await; + return Err(e); + } Vec::new() } else { Vec::new() }; - let frag_reuse_index = if options.defer_index_remap { + // No indexed/chain data touched -> no FRI (all-or-nothing, see above). + let frag_reuse_index = if options.defer_index_remap && any_group_indexed { Some(build_new_frag_reuse_index(dataset, frag_reuse_groups, new_fragment_bitmap).await?) } else { + if options.defer_index_remap { + log::debug!( + "skipping fragment-reuse index: no rewritten fragments were covered by an index" + ); + } None }; - // Collect new fragment paths before moving rewrite_groups into the transaction, - // so we can clean them up if the commit fails. - let all_new_fragments: Vec = rewrite_groups - .iter() - .flat_map(|g| g.new_fragments.iter().cloned()) - .collect(); - let transaction = TransactionBuilder::new( // Use the version at which the compaction tasks were *planned*, not the // version of the dataset handle passed to this function. In distributed @@ -2151,19 +2628,36 @@ pub async fn commit_compaction( .apply_commit(transaction, &Default::default(), &Default::default()) .await { - cleanup_data_fragments( - &dataset.object_store, - &dataset.base, - None, - &all_new_fragments, - ) - .await; + // RewriteResult is serializable and may be retried after an earlier + // ambiguous success. A conflict on this call therefore does not prove + // that the rewritten files are unreferenced. Leave them for dataset GC. + log::warn!( + "Compaction commit failed; leaving {} rewritten fragment(s) in place for GC: {}", + all_new_fragments.len(), + e + ); return Err(e); } Ok(metrics) } +/// Remove rewritten files after fragment-id reservation fails. Reservation +/// commits do not reference the rewritten files, so they are still owned by +/// this uncommitted compaction attempt and are safe to delete. +async fn cleanup_compaction_files_after_reservation_failure( + dataset: &Dataset, + all_new_fragments: &[Fragment], +) { + cleanup_data_fragments( + &dataset.object_store, + &dataset.base, + None, + all_new_fragments, + ) + .await; +} + #[cfg(test)] mod tests { @@ -2192,6 +2686,7 @@ mod tests { use lance_datagen::Dimension; use lance_file::version::LanceFileVersion; use lance_index::frag_reuse::FRAG_REUSE_INDEX_NAME; + use lance_index::frag_reuse::FragReuseIndexHandle; use lance_index::scalar::{ BuiltinIndexType, FullTextSearchQuery, InvertedIndexParams, ScalarIndexParams, }; @@ -2221,6 +2716,7 @@ mod tests { let fragment = Fragment { id: 0, files: vec![], + overlays: vec![], deletion_file: None, row_id_meta: None, physical_rows: Some(0), @@ -2261,6 +2757,33 @@ mod tests { assert_eq!(split[0].pos_range, 0..2); assert_eq!(split[1].pos_range, 2..5); assert_eq!(split[2].pos_range, 5..8); + + let zero_min_split_bin = CandidateBin { + fragments: std::iter::repeat_n( + Fragment { + id: 0, + files: vec![], + overlays: vec![], + deletion_file: None, + row_id_meta: None, + physical_rows: Some(0), + last_updated_at_version_meta: None, + created_at_version_meta: None, + }, + 3, + ) + .collect(), + pos_range: 0..3, + candidacy: std::iter::repeat_n(CompactionCandidacy::CompactItself, 3).collect(), + row_counts: vec![100, 200, 300], + indices: vec![], + }; + let split = zero_min_split_bin.split_for_size(0); + assert_eq!(split.len(), 3); + assert!(split.iter().all(|bin| !bin.fragments.is_empty())); + assert_eq!(split[0].pos_range, 0..1); + assert_eq!(split[1].pos_range, 1..2); + assert_eq!(split[2].pos_range, 2..3); } fn sample_data() -> RecordBatch { @@ -2273,6 +2796,20 @@ mod tests { .unwrap() } + /// Build (or, with `replace`, rebuild) a scalar index named "scalar" on `col`. + async fn create_scalar_index(dataset: &mut Dataset, col: &str, replace: bool) { + dataset + .create_index( + &[col], + IndexType::Scalar, + Some("scalar".into()), + &ScalarIndexParams::default(), + replace, + ) + .await + .unwrap(); + } + #[derive(Debug, Default, Clone, PartialEq)] struct MockIndexRemapperExpectation { expected: HashMap>, @@ -2355,9 +2892,10 @@ mod tests { } } + #[async_trait] impl IndexRemapperOptions for MockIndexRemapper { - fn create_remapper(&self, _: &Dataset) -> Result> { - Ok(Box::new(self.clone())) + async fn create_remapper(&self, _: &Dataset) -> Result>> { + Ok(Some(Box::new(self.clone()))) } } @@ -2447,8 +2985,171 @@ mod tests { assert_eq!(plan.tasks().len(), 0); } + fn list_data_files(uri: &str) -> std::collections::BTreeSet { + std::fs::read_dir(std::path::Path::new(uri).join("data")) + .map(|rd| { + rd.filter_map(|e| e.ok()) + .map(|e| e.file_name().to_string_lossy().into_owned()) + .collect() + }) + .unwrap_or_default() + } + + async fn execute_compaction_plan( + dataset: &Dataset, + options: &CompactionOptions, + ) -> Vec { + let plan = plan_compaction(dataset, options).await.unwrap(); + assert!(!plan.tasks.is_empty()); + let snapshot = dataset.clone(); + futures::stream::iter(plan.tasks) + .map(|task| rewrite_files(Cow::Borrowed(&snapshot), task, options)) + .buffer_unordered(1) + .try_collect() + .await + .unwrap() + } + + /// When the compaction commit's status is unknown (the commit errored and + /// verification was unavailable), the rewritten files must NOT be deleted: + /// if the commit landed they are referenced by the new version, and + /// deleting them would corrupt the table. #[tokio::test] - async fn test_compact_blob_columns() { + async fn test_compaction_retry_after_ambiguous_success_preserves_live_files() { + use crate::utils::test::{AmbiguousCommitHandler, AmbiguousFailure}; + + let test_dir = TempStrDir::default(); + let test_uri = test_dir.as_str(); + let data = sample_data(); + let num_rows = data.num_rows(); + let handler = Arc::new(AmbiguousCommitHandler::default()); + + let reader = RecordBatchIterator::new(vec![Ok(data.clone())], data.schema()); + let write_params = WriteParams { + max_rows_per_file: 3_000, + enable_stable_row_ids: true, + commit_handler: Some(handler.clone()), + ..Default::default() + }; + let mut dataset = Dataset::write(reader, test_uri, Some(write_params)) + .await + .unwrap(); + let files_before = list_data_files(test_uri); + let options = CompactionOptions::default(); + let completed = execute_compaction_plan(&dataset, &options).await; + let serialized = serde_json::to_vec(&completed).unwrap(); + let retry_results: Vec = serde_json::from_slice(&serialized).unwrap(); + let rewritten_files = list_data_files(test_uri) + .difference(&files_before) + .cloned() + .collect::>(); + assert!(!rewritten_files.is_empty()); + + handler.fail_next_rewrite(AmbiguousFailure::LandAndError); + handler + .fail_resolve + .store(true, std::sync::atomic::Ordering::SeqCst); + let err = commit_compaction( + &mut dataset, + completed, + Arc::new(DatasetIndexRemapperOptions::default()), + &options, + ) + .await + .expect_err("unknown commit status must surface as an error"); + assert!( + err.is_commit_status_unknown(), + "expected CommitStatusUnknown, got: {:?}", + err + ); + + handler + .fail_resolve + .store(false, std::sync::atomic::Ordering::SeqCst); + + // Retrying the same serialized RewriteResult now conflicts with the + // Rewrite that already landed. The retry must not delete those files. + let retry_error = commit_compaction( + &mut dataset, + retry_results, + Arc::new(DatasetIndexRemapperOptions::default()), + &options, + ) + .await + .expect_err("the replayed rewrite must conflict with its landed predecessor"); + assert!( + matches!(retry_error, Error::RetryableCommitConflict { .. }), + "expected RetryableCommitConflict, got: {retry_error:?}" + ); + let files_after_retry = list_data_files(test_uri); + assert!( + rewritten_files + .iter() + .all(|file| files_after_retry.contains(file)), + "a replayed RewriteResult must not delete files referenced by the landed rewrite" + ); + + let ds = Dataset::open(test_uri).await.unwrap(); + assert_eq!(ds.count_rows(None).await.unwrap(), num_rows); + let scanned = ds.scan().try_into_batch().await.unwrap(); + assert_eq!(scanned.num_rows(), num_rows); + } + + /// A failed ReserveFragments commit cannot reference the rewritten files, + /// so both stable-row-id reservation paths must still clean them up. + #[tokio::test] + async fn test_compaction_cleans_up_files_when_fragment_reservation_fails() { + use crate::utils::test::{AmbiguousCommitHandler, AmbiguousFailure}; + + let test_dir = TempStrDir::default(); + let test_uri = test_dir.as_str(); + let data = sample_data(); + let num_rows = data.num_rows(); + let handler = Arc::new(AmbiguousCommitHandler::default()); + + let reader = RecordBatchIterator::new(vec![Ok(data.clone())], data.schema()); + let write_params = WriteParams { + max_rows_per_file: 3_000, + enable_stable_row_ids: true, + commit_handler: Some(handler.clone()), + ..Default::default() + }; + let mut dataset = Dataset::write(reader, test_uri, Some(write_params)) + .await + .unwrap(); + let files_before = list_data_files(test_uri); + let options = CompactionOptions::default(); + let completed = execute_compaction_plan(&dataset, &options).await; + assert!(list_data_files(test_uri).len() > files_before.len()); + + handler.fail_next_reserve(AmbiguousFailure::FailOutright); + let err = commit_compaction( + &mut dataset, + completed, + Arc::new(DatasetIndexRemapperOptions::default()), + &options, + ) + .await + .expect_err("fragment reservation that did not land must fail"); + assert!( + !err.is_commit_status_unknown(), + "a verified-absent reservation is a definite failure, got: {:?}", + err + ); + + let files_after = list_data_files(test_uri); + let leftover: Vec<_> = files_after.difference(&files_before).collect(); + assert!( + leftover.is_empty(), + "rewritten files must be cleaned up after reservation fails; leftover: {:?}", + leftover + ); + let ds = Dataset::open(test_uri).await.unwrap(); + assert_eq!(ds.count_rows(None).await.unwrap(), num_rows); + } + + #[tokio::test] + async fn test_compact_blob_columns() { let test_dir = TempStrDir::default(); let schema = Arc::new(Schema::new(vec![ Field::new("id", DataType::Int32, false), @@ -2493,7 +3194,7 @@ mod tests { .unwrap(); assert_eq!(blobs.len(), expected_payload.len()); for (blob, expected) in blobs.iter().zip(expected_payload.iter()) { - let bytes = blob.read().await.unwrap(); + let bytes = blob.as_ref().unwrap().read().await.unwrap(); assert_eq!(bytes.as_ref(), expected.as_slice()); } } @@ -2886,9 +3587,70 @@ mod tests { } } + #[async_trait] impl IndexRemapperOptions for IgnoreRemap { - fn create_remapper(&self, _: &Dataset) -> Result> { - Ok(Box::new(Self {})) + async fn create_remapper(&self, _: &Dataset) -> Result>> { + Ok(None) + } + } + + #[rstest] + #[case::without_index(false)] + #[case::with_index(true)] + #[tokio::test] + async fn test_row_addrs_only_used_with_remappable_index(#[case] has_index: bool) { + let data = sample_data(); + let reader = RecordBatchIterator::new(vec![Ok(data.slice(0, 9_000))], data.schema()); + let mut dataset = Dataset::write( + reader, + "memory://", + Some(WriteParams { + max_rows_per_file: 3_000, + data_storage_version: Some(LanceFileVersion::Legacy), + ..Default::default() + }), + ) + .await + .unwrap(); + + if has_index { + create_scalar_index(&mut dataset, "a", false).await; + } + + let options = CompactionOptions { + target_rows_per_fragment: 9_000, + ..Default::default() + }; + let plan = plan_compaction(&dataset, &options).await.unwrap(); + assert_eq!(plan.tasks().len(), 1); + + let mut result = rewrite_files(Cow::Borrowed(&dataset), plan.tasks()[0].clone(), &options) + .await + .unwrap(); + assert_eq!(result.row_addrs.is_some(), has_index); + + if has_index { + let row_addrs_bytes = result + .row_addrs + .as_ref() + .expect("indexed compaction should capture row addresses"); + let row_addrs = + RoaringTreemap::deserialize_from(&mut Cursor::new(row_addrs_bytes)).unwrap(); + assert_eq!(row_addrs.len(), 9_000); + } else { + // Simulate a stale worker result that captured row addresses before the + // dataset no longer needed a remapper. Invalid bytes ensure the commit + // does not attempt to deserialize or materialize the unused map. + result.row_addrs = Some(b"not a roaring treemap".to_vec()); + commit_compaction( + &mut dataset, + vec![result], + Arc::new(DatasetIndexRemapperOptions::default()), + &options, + ) + .await + .unwrap(); + assert_eq!(dataset.get_fragments().len(), 1); } } @@ -3076,6 +3838,77 @@ mod tests { assert_eq!(before_scalar_result, after_scalar_result); } + /// Regression test for https://github.com/lance-format/lance/issues/8076 + /// + /// A zone map or bloom filter index reports matches as physical row addresses, so + /// compaction invalidates it even under stable row ids. Reusing it for the rewritten + /// fragments made a filtered scan fail with an internal error (a fragment referenced + /// by the index no longer existed) or, once translation tolerated that, silently drop + /// every match. + #[rstest] + #[case::zone_map(BuiltinIndexType::ZoneMap, IndexType::ZoneMap)] + #[case::bloom_filter(BuiltinIndexType::BloomFilter, IndexType::BloomFilter)] + #[tokio::test] + async fn test_addr_domain_index_after_compaction_with_stable_row_ids( + #[case] builtin: BuiltinIndexType, + #[case] index_type: IndexType, + ) { + let mut data_gen = + BatchGenerator::new().col(Box::new(IncrementingInt32::new().named("i".to_owned()))); + let mut dataset = Dataset::write( + data_gen.batch(200), + "memory://test/table", + Some(WriteParams { + enable_stable_row_ids: true, + max_rows_per_file: 100, // 2 fragments, so compaction has something to merge + ..Default::default() + }), + ) + .await + .unwrap(); + + dataset + .create_index( + &["i"], + index_type, + None, + &ScalarIndexParams::for_builtin(builtin), + false, + ) + .await + .unwrap(); + + compact_files(&mut dataset, CompactionOptions::default(), None) + .await + .unwrap(); + + // The index only knows the pre-compaction fragments, so it must not claim to + // cover the fragment they were rewritten into. + let live_fragments: RoaringBitmap = + dataset.fragments().iter().map(|f| f.id as u32).collect(); + let index = dataset + .load_indices() + .await + .unwrap() + .iter() + .find(|index| index.fields == vec![0]) + .expect("index must survive compaction") + .clone(); + assert!( + index + .effective_fragment_bitmap(&live_fragments) + .is_none_or(|covered| covered.is_empty()), + "compaction must not point an address-domain index at the fragments it wrote" + ); + + // Every fragment therefore falls back to a full scan, and the filter is answered + // in full. + let mut scanner = dataset.scan(); + scanner.filter("i > 0").unwrap(); + let matched = scanner.try_into_batch().await.unwrap(); + assert_eq!(matched.num_rows(), 199); + } + // Regression test for https://github.com/lancedb/lance/issues/6161 // When FragReuseIndexDetails exceeds 204800 bytes it is written to an external // file. Previously the file was silently dropped (temp file deleted) because @@ -3114,6 +3947,10 @@ mod tests { assert_eq!(dataset.get_fragments().len(), num_fragments); + // An FRI is only written for compactions that touch indexed data, so + // index the column being compacted. + create_scalar_index(&mut dataset, "i", false).await; + // Delete a few rows from each fragment so compaction has something to do. dataset.delete("i % 1000 = 0").await.unwrap(); @@ -3225,17 +4062,10 @@ mod tests { dataset.delete("i < 500").await.unwrap(); dataset2.delete("i < 500").await.unwrap(); - // Create a scalar index to check this is not touched - dataset - .create_index( - &["i"], - IndexType::Scalar, - Some("scalar".into()), - &ScalarIndexParams::default(), - false, - ) - .await - .unwrap(); + // Create the same scalar index on both datasets so deferred and immediate + // remapping are compared under the same conditions. + create_scalar_index(&mut dataset, "i", false).await; + create_scalar_index(&mut dataset2, "i", false).await; // Verify the initial state - no fragment reuse index should exist let initial_indices = dataset.load_indices().await.unwrap(); @@ -3369,7 +4199,9 @@ mod tests { open_frag_reuse_index(frag_reuse_index_meta.uuid, frag_reuse_details.as_ref()) .await .unwrap(); - let stats = frag_reuse_index.statistics().unwrap(); + let stats = FragReuseIndexHandle(Arc::new(frag_reuse_index.clone())) + .statistics() + .unwrap(); assert_eq!( serde_json::to_string(&stats).unwrap(), dataset @@ -3427,6 +4259,52 @@ mod tests { assert_eq!(current_scalar_index.uuid, original_scalar_uuid); } + #[tokio::test] + async fn test_defer_index_remap_skips_fri_when_no_indexed_data() { + // A deferred compaction touching no indexed data must write no FRI -- + // such a version is un-drainable (remap no-ops, trim retains it forever). + let mut data_gen = + BatchGenerator::new().col(Box::new(IncrementingInt32::new().named("i".to_owned()))); + + let mut dataset = Dataset::write( + data_gen.batch(600), + "memory://test/noindex", + Some(WriteParams { + max_rows_per_file: 100, // 6 small files -> compaction has work + ..Default::default() + }), + ) + .await + .unwrap(); + + // No index at all: nothing covers any fragment. + assert!(dataset.load_indices().await.unwrap().is_empty()); + let fragments_before = dataset.get_fragments().len(); + assert!(fragments_before > 1, "need multiple fragments to compact"); + + let options = CompactionOptions { + target_rows_per_fragment: 100_000, + defer_index_remap: true, + ..Default::default() + }; + compact_files(&mut dataset, options, None).await.unwrap(); + + // Compaction actually ran... + assert!( + dataset.get_fragments().len() < fragments_before, + "compaction should have merged fragments" + ); + // ...but no fragment-reuse index was created. + assert!( + dataset + .load_index_by_name(FRAG_REUSE_INDEX_NAME) + .await + .unwrap() + .is_none(), + "deferred compaction with no indexed data must not create an FRI" + ); + } + #[tokio::test] async fn test_defer_index_remap_multiple_compactions() { let mut data_gen = BatchGenerator::new() @@ -3446,6 +4324,10 @@ mod tests { .await .unwrap(); + // FRI is written only for compactions touching indexed data; index "i" so + // the successive deferred compactions build a chained fragment-reuse index. + create_scalar_index(&mut dataset, "i", false).await; + let options = CompactionOptions { target_rows_per_fragment: 2_000, defer_index_remap: true, @@ -3498,13 +4380,100 @@ mod tests { } } + #[tokio::test] + async fn test_defer_index_remap_mixed_records_all_groups() { + // All-or-nothing: a compaction touching any indexed data records the full + // FRI, including the unindexed group (a per-group filter would drop it). + let mut data_gen = + BatchGenerator::new().col(Box::new(IncrementingInt32::new().named("i".to_owned()))); + let mut dataset = Dataset::write( + data_gen.batch(300), + "memory://test/mixed", + Some(WriteParams { + max_rows_per_file: 100, // 3 fragments + ..Default::default() + }), + ) + .await + .unwrap(); + + // Index the initial fragments, then append more that stay unindexed. + create_scalar_index(&mut dataset, "i", false).await; + Dataset::write( + data_gen.batch(300), + WriteDestination::Dataset(Arc::new(dataset.clone())), + Some(WriteParams { + max_rows_per_file: 100, // 3 more, unindexed + mode: WriteMode::Append, + ..Default::default() + }), + ) + .await + .unwrap(); + dataset.checkout_latest().await.unwrap(); + + // Fragments not covered by the scalar index are the "unindexed" ones. + let indexed: HashSet = dataset + .load_index_by_name("scalar") + .await + .unwrap() + .unwrap() + .fragment_bitmap + .unwrap() + .iter() + .collect(); + let unindexed_frags: Vec = dataset + .fragments() + .iter() + .map(|f| f.id) + .filter(|id| !indexed.contains(&(*id as u32))) + .collect(); + assert!( + !unindexed_frags.is_empty(), + "expected some unindexed fragments" + ); + + compact_files( + &mut dataset, + CompactionOptions { + target_rows_per_fragment: 100_000, + defer_index_remap: true, + ..Default::default() + }, + None, + ) + .await + .unwrap(); + + // All-or-nothing: because indexed fragments were compacted, the FRI is + // written AND records the unindexed group too (a per-group filter would + // have dropped it). + let fri_meta = dataset + .load_index_by_name(FRAG_REUSE_INDEX_NAME) + .await + .unwrap() + .expect("mixed compaction must write an FRI"); + let details = load_frag_reuse_index_details(&dataset, &fri_meta) + .await + .unwrap(); + let recorded_old: HashSet = details + .versions + .iter() + .flat_map(|v| v.old_frag_ids()) + .collect(); + for f in &unindexed_frags { + assert!( + recorded_old.contains(f), + "unindexed fragment {f} must be recorded in the FRI (all-or-nothing)" + ); + } + } + #[tokio::test] async fn test_deferred_compaction_not_split_by_frag_reuse_index() { - // A deferred compaction creates a fragment-reuse index covering its - // output. Later small fragments must still compact together with that - // (FRI-covered) output: the FRI is a system index and must not split the - // compaction bin. Without the fix the FRI-covered fragment is isolated, - // so only the new fragments merge and the count never returns to one. + // The fragment-reuse index is a system index and must be excluded from + // compaction bin planning; otherwise its covered fragment is isolated and + // the small fragments never coalesce back to one. let data = sample_data(); let test_dir = TempStrDir::default(); let test_uri = &test_dir; @@ -3526,6 +4495,11 @@ mod tests { ) .await .unwrap(); + + // Index "a" so the deferred compaction records an FRI (only written for + // compactions touching indexed data). The FRI is a system index and must + // still not split later compaction bins -- the property this test guards. + create_scalar_index(&mut dataset, "a", false).await; compact_files(&mut dataset, options.clone(), None) .await .unwrap(); @@ -3553,11 +4527,16 @@ mod tests { .unwrap(); assert_eq!(dataset.get_fragments().len(), 3); + // Reindex so every fragment is data-indexed -- then the FRI (a system + // index, correctly excluded from bin planning) is the only thing that + // could split the bin. + create_scalar_index(&mut dataset, "a", true).await; + compact_files(&mut dataset, options, None).await.unwrap(); assert_eq!( dataset.get_fragments().len(), 1, - "FRI-covered fragment must compact together with the new fragments" + "FRI (a system index) must not split the compaction bin; all fragments coalesce" ); } @@ -3881,6 +4860,9 @@ mod tests { .await .unwrap(); + // Index "i" so the deferred compaction touches indexed data and writes an FRI. + create_scalar_index(&mut dataset, "i", false).await; + let options = CompactionOptions { target_rows_per_fragment: 2_000, defer_index_remap: true, @@ -3977,9 +4959,13 @@ mod tests { .unwrap(); let new_frags3 = frag_reuse_details3.versions.last().unwrap().new_frag_ids(); - // Concurrently commit a frag_reuse_index cleanup operation. - // Because there is no index, it should remove the first version. - // but after rebase it should contain the new compaction versions. + // Concurrently commit a frag_reuse_index cleanup operation. dataset_clone + // only knows the first reuse version; catch its index up so the cleanup + // removes that version. After rebase onto the other compactions it should + // contain the new compaction versions. + remapping::remap_column_index(&mut dataset_clone, &["i"], Some("scalar".into())) + .await + .unwrap(); cleanup_frag_reuse_index(&mut dataset_clone).await.unwrap(); // Load and verify the fragment reuse index content @@ -4019,6 +5005,9 @@ mod tests { .await .unwrap(); + // Index "i" so the deferred compaction touches indexed data and writes an FRI. + create_scalar_index(&mut dataset, "i", false).await; + let options = CompactionOptions { target_rows_per_fragment: 2_000, defer_index_remap: true, @@ -4057,8 +5046,12 @@ mod tests { .unwrap(); assert_eq!(frag_reuse_details.versions.len(), 1); - // First commit the frag_reuse_index cleanup - // Because there is no index, it should remove the first version. + // Catch the index up to the compaction (on `dataset` only; `dataset_clone` + // keeps the un-caught-up index for the concurrent rewrite below), then + // clean up: with the index caught up the trim removes the first version. + remapping::remap_column_index(&mut dataset, &["i"], Some("scalar".into())) + .await + .unwrap(); cleanup_frag_reuse_index(&mut dataset).await.unwrap(); // Load and verify the fragment reuse index content @@ -4129,6 +5122,9 @@ mod tests { .await .unwrap(); + // Index "i" so the deferred compaction touches indexed data and writes an FRI. + create_scalar_index(&mut dataset, "i", false).await; + let options = CompactionOptions { target_rows_per_fragment: 2_000, defer_index_remap: true, @@ -4263,22 +5259,69 @@ mod tests { count2 ); assert_eq!( - dataset - .count_rows(Some("category = 3".to_owned())) - .await - .unwrap(), - count3 + dataset + .count_rows(Some("category = 3".to_owned())) + .await + .unwrap(), + count3 + ); + + // Verify that after index creation and compaction, scan uses bitmap index scan + let mut scanner = dataset.scan(); + scanner.filter("category = 1").unwrap(); + scanner.project::(&[]).unwrap().with_row_id(); + let plan = scanner.explain_plan(false).await.unwrap(); + assert!( + plan.contains("ScalarIndexQuery: query=[category = 1]@category_idx(Bitmap)"), + "Expected index query in plan: {}", + plan + ); + } + + #[tokio::test] + async fn test_read_bloom_filter_index_with_defer_index_remap() { + let mut dataset = lance_datagen::gen_batch() + .col("id", lance_datagen::array::step::()) + .into_ram_dataset(FragmentCount::from(3), FragmentRowCount::from(3)) + .await + .unwrap(); + + dataset + .create_index( + &["id"], + IndexType::BloomFilter, + Some("id_idx".into()), + &ScalarIndexParams::for_builtin(BuiltinIndexType::BloomFilter), + false, + ) + .await + .unwrap(); + + let metrics = compact_files( + &mut dataset, + CompactionOptions { + target_rows_per_fragment: 512, + defer_index_remap: true, + ..Default::default() + }, + None, + ) + .await + .unwrap(); + assert!(metrics.fragments_removed > 0); + assert!(metrics.fragments_added > 0); + + assert_eq!( + dataset.count_rows(Some("id = 2".to_owned())).await.unwrap(), + 1 ); - // Verify that after index creation and compaction, scan uses bitmap index scan let mut scanner = dataset.scan(); - scanner.filter("category = 1").unwrap(); - scanner.project::(&[]).unwrap().with_row_id(); + scanner.filter("id = 2").unwrap(); let plan = scanner.explain_plan(false).await.unwrap(); assert!( - plan.contains("ScalarIndexQuery: query=[category = 1]@category_idx(Bitmap)"), - "Expected index query in plan: {}", - plan + plan.contains("ScalarIndexQuery: query=[id = 2]@id_idx(BloomFilter)"), + "Expected BloomFilter index query in plan: {plan}" ); } @@ -6264,6 +7307,29 @@ mod tests { assert!(err_msg.contains("invalid_mode")); } + #[test] + fn test_from_dataset_config_max_overlays_per_fragment() { + let key = "lance.compaction.max_overlays_per_fragment".to_string(); + + // An integer sets the threshold. + let config = HashMap::from([(key.clone(), "3".to_string())]); + let opts = CompactionOptions::from_dataset_config(&config).unwrap(); + assert_eq!(opts.max_overlays_per_fragment, Some(3)); + + // "none" (case-insensitive) disables the trigger, overriding the Some(10) default. + let config = HashMap::from([(key.clone(), "None".to_string())]); + let opts = CompactionOptions::from_dataset_config(&config).unwrap(); + assert_eq!(opts.max_overlays_per_fragment, None); + + // Anything else is rejected. + let config = HashMap::from([(key, "not_a_number".to_string())]); + let err_msg = CompactionOptions::from_dataset_config(&config) + .unwrap_err() + .to_string(); + assert!(err_msg.contains("max_overlays_per_fragment")); + assert!(err_msg.contains("not_a_number")); + } + #[test] fn test_apply_dataset_config_overrides() { let config = HashMap::from([( @@ -6698,12 +7764,12 @@ mod tests { count_all_files_in(&data_dir).unwrap_or(0) } - /// Site 2 in PR #6320: when `commit_compaction` fails to apply the commit - /// after `rewrite_files` has already written new data files, those files - /// must be cleaned up. We force the commit failure by injecting an error on - /// writes to the `_transactions/` directory. + /// Once `commit_compaction` reaches the Rewrite commit, its input may be a + /// replay of a result whose earlier commit landed ambiguously. A failure on + /// this call must therefore leave data files for GC instead of deleting + /// files that an existing version may reference. #[tokio::test] - async fn test_commit_compaction_cleans_up_data_on_commit_failure() { + async fn test_commit_compaction_leaves_data_for_gc_on_commit_failure() { use crate::dataset::builder::DatasetBuilder; use crate::utils::test::FailingProxyStore; use lance_io::object_store::ObjectStoreParams; @@ -6722,10 +7788,6 @@ mod tests { &routed_uri, Some(WriteParams { max_rows_per_file: 100, - // Stable row IDs lets `commit_compaction` skip the - // `reserve_fragment_ids` pre-commit (which would otherwise fail - // *before* the new data files exist), isolating the failure to - // the `apply_commit` call we want to test. enable_stable_row_ids: true, ..Default::default() }), @@ -6771,15 +7833,14 @@ mod tests { "Compaction should fail when transaction commit fails" ); - assert_eq!( - count_data_files_in(test_uri), - baseline_files, - "Compaction data files should be cleaned up when commit fails" + assert!( + count_data_files_in(test_uri) > baseline_files, + "Compaction data files should be retained for GC after the Rewrite commit fails" ); } #[tokio::test] - async fn test_commit_compaction_cleans_up_blob_v2_sidecars_on_commit_failure() { + async fn test_commit_compaction_leaves_blob_v2_sidecars_for_gc_on_commit_failure() { use crate::BlobArrayBuilder; use crate::dataset::builder::DatasetBuilder; use crate::utils::test::FailingProxyStore; @@ -6852,10 +7913,9 @@ mod tests { "Compaction should fail when transaction commit fails" ); - assert_eq!( - count_data_files_in(test_uri), - baseline_files, - "Blob v2 sidecars should be cleaned up when commit fails" + assert!( + count_data_files_in(test_uri) > baseline_files, + "Blob v2 sidecars should be retained for GC after the Rewrite commit fails" ); } @@ -6885,16 +7945,182 @@ mod tests { let row_id = row_ids.value(i); let id = ids.value(i); let blobs = dataset.take_blobs(&[row_id], column).await.unwrap(); - if blobs.is_empty() { - result.push((id, None)); - } else { - let data = blobs[0].read().await.unwrap(); - result.push((id, Some(data.to_vec()))); + match blobs.into_iter().next().flatten() { + Some(blob) => { + let data = blob.read().await.unwrap(); + result.push((id, Some(data.to_vec()))); + } + None => result.push((id, None)), } } result } + fn mixed_blob_values() -> Vec<(i32, Option>)> { + vec![ + (0, Some(vec![b'0'; 80])), + (1, None), + (2, Some(Vec::new())), + (3, Some(vec![b'3'; 80])), + (4, Some(vec![b'4'; 80])), + (5, Some(vec![b'5'; 80])), + ] + } + + async fn assert_compaction_preserves_blob_values( + mut dataset: Dataset, + expected: &[(i32, Option>)], + ) { + assert_eq!(dataset.get_fragments().len(), 3); + + let mut before = read_blob_bytes_by_index(&Arc::new(dataset.clone()), "blob").await; + before.sort_by_key(|(id, _)| *id); + assert_eq!(before, expected); + + compact_files( + &mut dataset, + CompactionOptions { + target_rows_per_fragment: 1024 * 1024, + ..Default::default() + }, + None, + ) + .await + .unwrap(); + + assert_eq!(dataset.get_fragments().len(), 1); + + let mut after = read_blob_bytes_by_index(&Arc::new(dataset), "blob").await; + after.sort_by_key(|(id, _)| *id); + assert_eq!(after, expected); + } + + #[test] + fn test_blob_v2_rewrite_plan_skips_non_blob_list_subtree() { + let items_field = Field::new( + "items", + DataType::List(Arc::new(Field::new("item", DataType::Int32, true))), + true, + ); + let info_field = Field::new( + "info", + DataType::Struct( + vec![ + Arc::new(items_field), + Arc::new(crate::blob_field("blob", true)), + ] + .into(), + ), + true, + ); + let logical_arrow_schema = Schema::new(vec![info_field]); + let logical_schema = + lance_core::datatypes::Schema::try_from(&logical_arrow_schema).unwrap(); + let mut input_schema = logical_schema.clone(); + input_schema.fields[0].unload_blobs_recursive(); + let input_field = Field::from(&input_schema.fields[0]); + + let plan = + BlobV2FieldRewritePlan::try_new(&logical_schema.fields[0], &input_field).unwrap(); + let BlobV2FieldRewritePlan::Struct { children, .. } = &plan else { + panic!("nested blob field should produce a struct rewrite plan"); + }; + assert!(matches!( + children[0], + BlobV2FieldRewritePlan::Passthrough { .. } + )); + assert!(matches!(children[1], BlobV2FieldRewritePlan::Blob { .. })); + + let DataType::Struct(output_children) = plan.output_field().data_type() else { + panic!("nested blob rewrite plan should declare a struct output"); + }; + let DataType::Struct(input_children) = input_field.data_type() else { + panic!("nested blob input should be a struct"); + }; + assert_eq!(output_children[0], input_children[0]); + let DataType::Struct(blob_children) = output_children[1].data_type() else { + panic!("blob rewrite plan should declare a struct blob output"); + }; + assert_eq!( + BlobV2Layout::classify(blob_children), + Some(BlobV2Layout::Logical) + ); + } + + #[tokio::test] + async fn test_compact_blob_v1_preserves_null_empty_and_payload_order() { + let test_dir = TempStrDir::default(); + let expected = mixed_blob_values(); + let schema = Arc::new(Schema::new(vec![ + Field::new("id", DataType::Int32, false), + Field::new("blob", DataType::LargeBinary, true) + .with_metadata([(BLOB_META_KEY.to_string(), "true".to_string())].into()), + ])); + let batch = RecordBatch::try_new( + schema.clone(), + vec![ + Arc::new(Int32Array::from_iter_values(0..expected.len() as i32)), + Arc::new(LargeBinaryArray::from_iter( + expected.iter().map(|(_, value)| value.as_deref()), + )), + ], + ) + .unwrap(); + let dataset = Dataset::write( + RecordBatchIterator::new(vec![Ok(batch)], schema), + &test_dir, + Some(WriteParams { + data_storage_version: Some(LanceFileVersion::V2_0), + max_rows_per_file: 2, + ..Default::default() + }), + ) + .await + .unwrap(); + + assert_compaction_preserves_blob_values(dataset, &expected).await; + } + + #[tokio::test] + async fn test_compact_blob_v2_preserves_null_empty_and_payload_order() { + use crate::BlobArrayBuilder; + + let test_dir = TempStrDir::default(); + let expected = mixed_blob_values(); + let mut blob_builder = BlobArrayBuilder::new(expected.len()); + for (_, value) in &expected { + match value { + Some(value) => blob_builder.push_bytes(value).unwrap(), + None => blob_builder.push_null().unwrap(), + } + } + let schema = Arc::new(Schema::new(vec![ + Field::new("id", DataType::Int32, false), + crate::blob_field("blob", true), + ])); + let batch = RecordBatch::try_new( + schema.clone(), + vec![ + Arc::new(Int32Array::from_iter_values(0..expected.len() as i32)), + blob_builder.finish().unwrap(), + ], + ) + .unwrap(); + let dataset = Dataset::write( + RecordBatchIterator::new(vec![Ok(batch)], schema), + &test_dir, + Some(WriteParams { + data_storage_version: Some(LanceFileVersion::V2_2), + max_rows_per_file: 2, + ..Default::default() + }), + ) + .await + .unwrap(); + + assert_compaction_preserves_blob_values(dataset, &expected).await; + } + #[tokio::test] async fn test_compact_blob_v2_preserves_external_references() { use crate::BlobArrayBuilder; @@ -7885,4 +9111,337 @@ mod tests { ] ); } + // ---- `max_overlays_per_fragment` compaction trigger ---- + // + // Tests for the trigger that fully compacts a fragment carrying too many data + // overlay files into a fresh fragment with the overlays (and deletions) + // materialized into the base data. + use arrow_array::record_batch; + use lance_file::writer::FileWriterOptions; + use lance_io::utils::CachedFileSize; + use lance_table::format::DataFile; + use lance_table::format::overlay::{DataOverlayFile, OverlayCoverage}; + use std::collections::BTreeMap; + + use crate::dataset::DATA_DIR; + use crate::dataset::transaction::DataOverlayGroup; + + /// Two-fragment Int32 dataset: `id` (field 0) = 0..12 and `val` (field 1) = + /// id * 10, six rows per fragment (fragments 0 and 1). + async fn create_base_dataset(uri: &str) -> Dataset { + let batch = record_batch!( + ("id", Int32, (0..12).collect::>()), + ("val", Int32, (0..12).map(|v| v * 10).collect::>()) + ) + .unwrap(); + let schema = batch.schema(); + let write_params = WriteParams { + max_rows_per_file: 6, + max_rows_per_group: 6, + data_storage_version: Some(LanceFileVersion::Stable), + ..Default::default() + }; + let reader = RecordBatchIterator::new(vec![Ok(batch)], schema.clone()); + Dataset::write(reader, uri, Some(write_params)) + .await + .unwrap() + } + + fn i32_array(values: impl IntoIterator>) -> ArrayRef { + Arc::new(Int32Array::from_iter(values)) + } + + fn bitmap(offsets: impl IntoIterator) -> RoaringBitmap { + RoaringBitmap::from_iter(offsets) + } + + /// Write a dense overlay covering `fields` of `fragment_id` with `columns` + /// as the per-field value columns, then commit it as a `DataOverlay`. + async fn commit_overlay( + dataset: Dataset, + fragment_id: u64, + fields: &[i32], + coverage: OverlayCoverage, + columns: Vec, + ) -> Dataset { + let read_version = dataset.version().version; + let overlay_schema = dataset.schema().project_by_ids(fields, true); + let filename = format!("{}.lance", Uuid::new_v4()); + let path = dataset.base.clone().join(DATA_DIR).join(filename.as_str()); + let obj_writer = dataset.object_store.create(&path).await.unwrap(); + let file_version = LanceFileVersion::Stable.resolve(); + let mut writer = lance_file::versions::create_writer( + file_version, + obj_writer, + overlay_schema, + FileWriterOptions::default(), + ) + .unwrap(); + for (column_index, array) in columns.into_iter().enumerate() { + writer.write_column(column_index, array).await.unwrap(); + } + let summary = writer.finish().await.unwrap(); + + let mut data_file = DataFile::new_unstarted(filename, file_version); + data_file.fields = writer + .field_id_to_column_indices() + .iter() + .map(|(f, _)| *f as i32) + .collect::>() + .into(); + data_file.column_indices = writer + .field_id_to_column_indices() + .iter() + .map(|(_, c)| *c as i32) + .collect::>() + .into(); + data_file.file_size_bytes = CachedFileSize::new(summary.size_bytes); + + Dataset::commit( + WriteDestination::Dataset(Arc::new(dataset)), + Operation::DataOverlay { + groups: vec![DataOverlayGroup { + fragment_id, + overlays: vec![DataOverlayFile { + data_file, + coverage, + committed_version: 0, + }], + }], + }, + Some(read_version), + None, + None, + Arc::new(Default::default()), + false, + ) + .await + .unwrap() + } + + /// Commit `n` distinct single-cell overlays to fragment 0 (offset `i`, val + /// column set to `1000 + i`), so the fragment ends up with `n` overlays. The + /// `1000 +` offset keeps overlaid values clear of the base `id * 10` values. + async fn commit_n_overlays(mut dataset: Dataset, n: u32) -> Dataset { + for i in 0..n { + dataset = commit_overlay( + dataset, + 0, + &[1], + OverlayCoverage::dense(bitmap([i])), + vec![i32_array([Some(1000 + i as i32)])], + ) + .await; + } + dataset + } + + /// Options whose only compaction trigger is the overlay limit: base + /// fragments here are far below the default 1M-row target, which would + /// otherwise make them size-based compaction candidates on their own. + fn overlay_only_options(max_overlays_per_fragment: usize) -> CompactionOptions { + CompactionOptions { + max_overlays_per_fragment: Some(max_overlays_per_fragment), + target_rows_per_fragment: 6, + ..Default::default() + } + } + + /// Scan `id` and `val` and return an `id -> val` map (order-independent). + async fn id_val_map(dataset: &Dataset) -> BTreeMap> { + let mut scanner = dataset.scan(); + scanner.project(&["id", "val"]).unwrap(); + let batch = scanner.try_into_batch().await.unwrap(); + let mut out = BTreeMap::new(); + let ids = batch + .column(0) + .as_any() + .downcast_ref::() + .unwrap(); + let vals = batch + .column(1) + .as_any() + .downcast_ref::() + .unwrap(); + for i in 0..batch.num_rows() { + let v = if vals.is_null(i) { + None + } else { + Some(vals.value(i)) + }; + out.insert(ids.value(i), v); + } + out + } + + #[tokio::test] + async fn test_max_overlays_triggers_full_compaction() { + // Fragment 0 gets 3 overlays; fragment 1 stays clean. + let dataset = create_base_dataset("memory://").await; + let mut dataset = commit_n_overlays(dataset, 3).await; + assert_eq!( + dataset.get_fragment(0).unwrap().metadata().overlays.len(), + 3 + ); + + // Threshold 2: only fragment 0 (3 > 2) is compacted. + let metrics = compact_files(&mut dataset, overlay_only_options(2), None) + .await + .unwrap(); + assert_eq!(metrics.fragments_removed, 1); + assert_eq!(metrics.fragments_added, 1); + + let fragments = dataset.get_fragments(); + assert_eq!(fragments.len(), 2); + // The compacted fragment is a fresh single-data-file fragment with no + // overlays; fragment 1 is untouched. + let compacted = fragments + .iter() + .find(|f| f.id() != 1) + .expect("a new fragment id was assigned"); + assert!(compacted.metadata().overlays.is_empty()); + assert_eq!(compacted.metadata().files.len(), 1); + + // The overlaid values were materialized: id i in 0..3 -> 1000 + i. + let values = id_val_map(&dataset).await; + let expected: BTreeMap> = (0..12) + .map(|id| { + let v = if id < 3 { 1000 + id } else { id * 10 }; + (id, Some(v)) + }) + .collect(); + assert_eq!(values, expected); + } + + #[tokio::test] + async fn test_below_threshold_is_a_noop() { + let dataset = create_base_dataset("memory://").await; + let mut dataset = commit_n_overlays(dataset, 2).await; + + // 2 overlays, threshold 2: `overlays > max` is false, so no compaction. + let metrics = compact_files(&mut dataset, overlay_only_options(2), None) + .await + .unwrap(); + assert_eq!(metrics.fragments_removed, 0); + assert_eq!(metrics.fragments_added, 0); + assert_eq!( + dataset.get_fragment(0).unwrap().metadata().overlays.len(), + 2 + ); + } + + #[tokio::test] + async fn test_overlay_compaction_materializes_deletions() { + let dataset = create_base_dataset("memory://").await; + let mut dataset = commit_n_overlays(dataset, 3).await; + // Delete a row from the overlaid fragment (id 2 is at offset 2). + dataset.delete("id = 2").await.unwrap(); + assert!( + dataset + .get_fragment(0) + .unwrap() + .metadata() + .deletion_file + .is_some() + ); + + compact_files(&mut dataset, overlay_only_options(2), None) + .await + .unwrap(); + + // The deletion was materialized: no deletion file remains and id 2 is gone. + for fragment in dataset.get_fragments() { + assert!(fragment.metadata().deletion_file.is_none()); + assert!(fragment.metadata().overlays.is_empty()); + } + let values = id_val_map(&dataset).await; + assert!(!values.contains_key(&2)); + // Surviving overlaid cells still carry their materialized values. + assert_eq!(values.get(&0), Some(&Some(1000))); + assert_eq!(values.get(&1), Some(&Some(1001))); + } + + #[tokio::test] + async fn test_overlay_compaction_reconciles_stale_index() { + let mut dataset = create_base_dataset("memory://").await; + // Index `val` before any overlay -> the index is stale once val is overlaid. + dataset + .create_index( + &["val"], + IndexType::Scalar, + None, + &ScalarIndexParams::default(), + true, + ) + .await + .unwrap(); + + // Overlay val[0] 0 -> 100 (committed after the index) and push fragment 0 + // over the overlay limit. + let mut dataset = commit_n_overlays(dataset, 3).await; + + let val_index_before = dataset + .load_indices() + .await + .unwrap() + .iter() + .find(|i| i.fields == vec![1]) + .expect("val index present") + .clone(); + assert!( + val_index_before + .fragment_bitmap + .as_ref() + .unwrap() + .contains(0) + ); + + compact_files(&mut dataset, overlay_only_options(2), None) + .await + .unwrap(); + + // The stale val index no longer covers the compacted fragment, so its + // rows fall back to a flat scan instead of serving stale values. + let indices = dataset.load_indices().await.unwrap(); + let val_index = indices + .iter() + .find(|i| i.fields == vec![1]) + .expect("val index present"); + let compacted_id = dataset + .get_fragments() + .iter() + .map(|f| f.id() as u32) + .find(|id| *id != 1) + .unwrap(); + assert!( + !val_index + .fragment_bitmap + .as_ref() + .unwrap() + .contains(compacted_id), + "stale index must drop the compacted fragment from its coverage" + ); + + // The indexed query is correct: the materialized value is found and the + // stale pre-overlay value is gone. + let mut scanner = dataset.scan(); + scanner + .filter("val = 1000") + .unwrap() + .project(&["id"]) + .unwrap(); + let batch = scanner.try_into_batch().await.unwrap(); + let ids = batch + .column(0) + .as_any() + .downcast_ref::() + .unwrap(); + assert_eq!(ids.len(), 1); + assert_eq!(ids.value(0), 0); + + let mut scanner = dataset.scan(); + scanner.filter("val = 0").unwrap().project(&["id"]).unwrap(); + let batch = scanner.try_into_batch().await.unwrap(); + assert_eq!(batch.num_rows(), 0, "stale value 0 must no longer match"); + } } diff --git a/rust/lance/src/dataset/optimize/binary_copy.rs b/rust/lance/src/dataset/optimize/binary_copy.rs index f22737fff89..c76e0ea300f 100644 --- a/rust/lance/src/dataset/optimize/binary_copy.rs +++ b/rust/lance/src/dataset/optimize/binary_copy.rs @@ -7,90 +7,44 @@ use crate::dataset::DATA_DIR; use crate::dataset::WriteParams; use crate::dataset::fragment::write::generate_random_filename; use crate::datatypes::Schema; -use lance_arrow::DataTypeExt; use lance_core::Error; -use lance_encoding::decoder::{ColumnInfo, PageEncoding, PageInfo as DecPageInfo}; -use lance_encoding::version::LanceFileVersion; -use lance_file::format::pbfile; +use lance_encoding::decoder::{ColumnInfo, PageInfo as DecPageInfo}; use lance_file::reader::FileReader as LFReader; +use lance_file::version::ConcreteFileVersion; +use lance_file::versions as file_versions; use lance_file::writer::{FileWriter, FileWriterOptions}; use lance_io::scheduler::{ScanScheduler, SchedulerConfig}; -use lance_io::traits::Writer; use lance_table::format::{DataFile, Fragment}; use prost::Message; use prost_types::Any; use std::ops::Range; use std::sync::Arc; -use tokio::io::AsyncWriteExt; - -const ALIGN: usize = 64; - -/// Apply 64-byte alignment padding for V2.1+ files. -/// -/// For V2.1+, writes padding bytes to align the current position to a 64-byte boundary. -/// For V2.0 and earlier, no padding is applied as alignment is not required. -/// -/// Returns the new position after padding (if any). -async fn apply_alignment_padding( - writer: &mut dyn Writer, - current_pos: u64, - version: LanceFileVersion, -) -> Result { - if version >= LanceFileVersion::V2_1 { - static ZERO_BUFFER: std::sync::OnceLock> = std::sync::OnceLock::new(); - let zero_buf = ZERO_BUFFER.get_or_init(|| vec![0u8; ALIGN]); - - let pad = (ALIGN - (current_pos as usize % ALIGN)) % ALIGN; - if pad != 0 { - writer.write_all(&zero_buf[..pad]).await?; - return Ok(current_pos + pad as u64); - } - } - Ok(current_pos) -} async fn init_writer_if_necessary( dataset: &Dataset, - current_writer: &mut Option>, + version: ConcreteFileVersion, + current_writer: &mut Option, current_filename: &mut Option, ) -> Result { if current_writer.is_none() { let filename = format!("{}.lance", generate_random_filename()); let path = dataset.base.clone().join(DATA_DIR).join(filename.as_str()); - let writer = dataset.object_store.create(&path).await?; - *current_writer = Some(writer); + let object_writer = dataset.object_store.create(&path).await?; + *current_writer = Some(file_versions::create_lazy_writer( + version, + object_writer, + FileWriterOptions::default(), + )?); *current_filename = Some(filename); return Ok(true); } Ok(false) } -/// v2_0 vs v2_1+ field-to-column index mapping -/// - v2_1+ stores only leaf columns; non-leaf fields get `-1` in the mapping -/// - v2_0 includes structural headers as columns; non-leaf fields map to a concrete index -fn compute_field_column_indices( - schema: &Schema, - full_field_ids_len: usize, - version: LanceFileVersion, -) -> Vec { - let is_structural = version >= LanceFileVersion::V2_1; - let mut field_column_indices: Vec = Vec::with_capacity(full_field_ids_len); - let mut curr_col_idx: i32 = 0; - for field in schema.fields_pre_order() { - if field.is_packed_struct() || field.is_leaf() || !is_structural { - field_column_indices.push(curr_col_idx); - curr_col_idx += 1; - } else { - field_column_indices.push(-1); - } - } - field_column_indices -} - /// Finalize the current output file and return it as a single [Fragment]. /// - Ensures an output writer / filename is present (creates a new file if needed). /// - Converts the in-memory `col_pages` / `col_buffers` into `ColumnInfo` metadata, draining them. -/// - Applies v2_0 structural header rules (single page, normalized `num_rows` and `priority`). +/// - Lets the exact file version normalize copied column metadata. /// - Writes the Lance footer via [flush_footer] and registers the resulting [DataFile] in a [Fragment]. /// /// PAY ATTENTION current function will: @@ -99,28 +53,24 @@ fn compute_field_column_indices( #[allow(clippy::too_many_arguments)] async fn finalize_current_output_file( schema: &Schema, - full_field_ids: &[i32], - current_writer: &mut Option>, + version: ConcreteFileVersion, + current_writer: &mut Option, current_filename: &mut Option, current_page_table: &[ColumnInfo], col_pages: &mut [Vec], col_buffers: &mut [Vec<(u64, u64)>], - is_non_leaf_column: &[bool], total_rows_in_current: u64, - version: LanceFileVersion, ) -> Result { let mut final_cols: Vec> = Vec::with_capacity(current_page_table.len()); for (i, column_info) in current_page_table.iter().enumerate() { let mut pages_vec = std::mem::take(&mut col_pages[i]); - // For v2_0 struct headers, force a single page and set num_rows to total - if version == LanceFileVersion::V2_0 - && is_non_leaf_column.get(i).copied().unwrap_or(false) - && !pages_vec.is_empty() - { - pages_vec[0].num_rows = total_rows_in_current; - pages_vec[0].priority = 0; - pages_vec.truncate(1); - } + file_versions::finalize_external_metadata_column( + version, + schema, + i, + &mut pages_vec, + total_rows_in_current, + )?; let pages_arc = Arc::from(pages_vec.into_boxed_slice()); let buffers_vec = std::mem::take(&mut col_buffers[i]); final_cols.push(Arc::new(ColumnInfo::new( @@ -130,15 +80,19 @@ async fn finalize_current_output_file( column_info.encoding.clone(), ))); } - let writer = current_writer.take().unwrap(); - flush_footer(writer, schema, &final_cols, total_rows_in_current, version).await?; + let mut writer = current_writer + .take() + .ok_or_else(|| Error::internal("binary copy output writer was not initialized"))?; + flush_footer(&mut writer, schema, &final_cols, total_rows_in_current).await?; // Register the newly closed output file as a fragment data file - let (maj, min) = version.to_numbers(); let mut fragment = Fragment::new(0); - let field_column_indices = compute_field_column_indices(schema, full_field_ids.len(), version); - let mut data_file = DataFile::new_unstarted(current_filename.take().unwrap(), maj, min); - data_file.fields = full_field_ids.to_vec().into(); + let (field_ids, field_column_indices) = file_versions::data_file_columns(version, schema); + let filename = current_filename + .take() + .ok_or_else(|| Error::internal("binary copy output filename was not initialized"))?; + let mut data_file = DataFile::new_unstarted(filename, version); + data_file.fields = field_ids.into(); data_file.column_indices = field_column_indices.into(); fragment.files.push(data_file); fragment.physical_rows = Some(total_rows_in_current as usize); @@ -157,11 +111,9 @@ async fn finalize_current_output_file( /// └── final flush for remaining rows /// /// Behavior highlights: -/// - Assumes all input files share the same Lance file version; version drives column-count -/// calculation (v2.0 includes structural headers, v2.1+ only leaf columns). +/// - Assumes all input files share the same Lance file version. /// - Preserves stable row ids by concatenating row-id sequences when enabled. -/// - Enforces 64-byte alignment for page and buffer writes in V2.1+ files (V2.0 does not require alignment). -/// - For v2.0, preserves single-page structural headers and normalizes their row counts/priority. +/// - Delegates physical-column mapping and copied metadata normalization to the exact file version. /// - Flushes an output file once `max_rows_per_file` rows are accumulated, then repeats. /// /// Parameters: @@ -170,6 +122,7 @@ async fn finalize_current_output_file( /// - `params`: write parameters (uses `max_rows_per_file`). /// - `read_batch_bytes_opt`: optional I/O batch size when coalescing page reads. pub async fn rewrite_files_binary_copy( + version: ConcreteFileVersion, dataset: &Dataset, fragments: &[Fragment], params: &WriteParams, @@ -185,54 +138,19 @@ pub async fn rewrite_files_binary_copy( // - Reads page and buffer regions directly from source files in bounded batches // - Appends them to a new output file with alignment, updating offsets // - Recomputes page priorities by adding the cumulative row count to preserve order - // - For v2_0, enforces single-page structural header columns when closing a file // - Writes a new footer (schema descriptor, column metadata, offset tables, version) // - Optionally carries forward stable row ids and persists them inline in fragment metadata // Merge small Lance files into larger ones by page-level binary copy. let schema = dataset.schema().clone(); - let full_field_ids = schema.field_ids(); - - // The previous checks have ensured that the file versions of all files are consistent. - let version = LanceFileVersion::try_from_major_minor( - fragments[0].files[0].file_major_version, - fragments[0].files[0].file_minor_version, - ) - .unwrap() - .resolve(); - // v2.0 and v2.1+ handle structural headers differently during file writing: - // - v2_0 materializes ALL fields in pre-order traversal (leaf fields + non-leaf struct headers), - // which means the ColumnInfo set includes all fields in pre-order traversal. - // - v2_1+ materializes fields that are either leaf columns OR packed structs. Non-leaf structural - // headers (unpacked structs with children) are not stored as columns. - // As a result, the ColumnInfo set contains leaf fields and packed structs. - // To correctly align copy layout, we derive `column_count` by version: - // - v2_0: use total number of fields in pre-order (leaf + non-leaf headers) - // - v2_1+: use only the number of leaf fields plus packed structs - let column_count = if version == LanceFileVersion::V2_0 { - schema.fields_pre_order().count() - } else { - schema - .fields_pre_order() - .filter(|f| f.is_packed_struct() || f.is_leaf()) - .count() - }; - - // v2_0 compatibility: build a map to identify non-leaf structural header columns - // - In v2_0 these headers exist as columns and must have a single page - // - In v2_1+ these headers are not stored as columns and this map is unused - let mut is_non_leaf_column: Vec = vec![false; column_count]; - if version == LanceFileVersion::V2_0 { - for (col_idx, field) in schema.fields_pre_order().enumerate() { - // Only mark non-packed Struct fields (lists remain as leaf data carriers) - let is_non_leaf = field.data_type().is_struct() && !field.is_packed_struct(); - is_non_leaf_column[col_idx] = is_non_leaf; - } - } + let column_count = schema + .fields + .iter() + .map(|field| file_versions::physical_column_count(version, field)) + .sum(); let mut out: Vec = Vec::new(); - let mut current_writer: Option> = None; + let mut current_writer: Option = None; let mut current_filename: Option = None; - let mut current_pos: u64 = 0; let mut current_page_table: Vec = Vec::new(); // Baseline column encodings captured from the first source file; all subsequent // files must match per-column to safely concatenate column-level buffers. @@ -279,38 +197,34 @@ pub async fn rewrite_files_binary_copy( .collect(); baseline_col_encoding_bytes = src_column_infos .iter() - .map(|ci| Any::from_msg(&ci.encoding).unwrap().encode_to_vec()) - .collect(); + .map(|ci| Ok(Any::from_msg(&ci.encoding)?.encode_to_vec())) + .collect::>>()?; } // Iterate through each column of the current data file of the current fragment for (col_idx, src_column_info) in src_column_infos.iter().enumerate() { - // v2_0 compatibility: special handling for non-leaf structural header columns - // - v2_0 expects structural header columns to have a SINGLE page; they carry layout - // metadata only and are not true data carriers. - // - When merging multiple input files via binary copy, naively appending pages would - // yield multiple pages for the same structural header column, violating v2_0 rules. - // - To preserve v2_0 invariants, we skip pages beyond the first one for these columns. - // - During finalization we also normalize the single remaining page’s `num_rows` to the - // total number of rows in the output file and reset `priority` to 0. - // - For v2_1+ this logic does not apply because non-leaf headers are not stored as columns. - let is_non_leaf = col_idx < is_non_leaf_column.len() && is_non_leaf_column[col_idx]; - if is_non_leaf && !col_pages[col_idx].is_empty() { - continue; - } - - if init_writer_if_necessary(dataset, &mut current_writer, &mut current_filename) - .await? - { - current_pos = 0; - } + let has_existing_pages = !col_pages[col_idx].is_empty(); + file_versions::copy_external_metadata_column( + version, + &schema, + col_idx, + has_existing_pages, + || async { + init_writer_if_necessary( + dataset, + version, + &mut current_writer, + &mut current_filename, + ) + .await?; - let read_batch_bytes: u64 = read_batch_bytes_opt.unwrap_or(16 * 1024 * 1024) as u64; + let read_batch_bytes: u64 = + read_batch_bytes_opt.unwrap_or(16 * 1024 * 1024) as u64; - let mut page_index = 0; + let mut page_index = 0; - // Iterate through each page of the current column in the current data file of the current fragment - while page_index < src_column_info.page_infos.len() { + // Iterate through each page of the current column in the current data file of the current fragment + while page_index < src_column_info.page_infos.len() { let mut batch_ranges: Vec> = Vec::new(); let mut batch_counts: Vec = Vec::new(); let mut batch_bytes: u64 = 0; @@ -368,49 +282,42 @@ pub async fn rewrite_files_binary_copy( for (buffer_idx, (_, size)) in page.buffer_offsets_and_sizes.iter().enumerate() { - let writer = current_writer.as_mut().unwrap().as_mut(); - current_pos = - apply_alignment_padding(writer, current_pos, version).await?; - let start = current_pos; - if *size == 0 { - new_offsets.push((start, 0)); + let writer = current_writer.as_mut().ok_or_else(|| { + Error::internal("binary copy output writer was not initialized") + })?; + let bytes = if *size == 0 { + None } else { - let bytes = bytes_iter.next().ok_or_else(|| { + Some(bytes_iter.next().ok_or_else(|| { Error::execution(format!( "binary copy: missing page buffer bytes while rewriting data file \ (column {col_idx}, page {page_idx}, buffer {buffer_idx}, expected size {size})", )) - })?; - writer.write_all(&bytes).await?; - current_pos += bytes.len() as u64; - new_offsets.push((start, bytes.len() as u64)); - } + })?) + }; + let (start, written) = writer + .write_external_buffer(bytes.as_deref().unwrap_or_default()) + .await?; + new_offsets.push((start, written)); } - // manual clone encoding - let encoding = if page.encoding.is_structural() { - PageEncoding::Structural(page.encoding.as_structural().clone()) - } else { - PageEncoding::Legacy(page.encoding.as_legacy().clone()) - }; // `priority` acts as the global row offset for this page, ensuring // downstream iterators maintain the correct logical order across // merged inputs. let new_page_info = DecPageInfo { num_rows: page.num_rows, priority: page.priority + total_rows_in_current, - encoding, + encoding: page.encoding.clone(), buffer_offsets_and_sizes: Arc::from(new_offsets.into_boxed_slice()), }; col_pages[col_idx].push(new_page_info); } - } // finished scheduling & copying pages for this column in the current source file + } // finished scheduling & copying pages for this column in the current source file - if !src_column_info.buffer_offsets_and_sizes.is_empty() { + if !src_column_info.buffer_offsets_and_sizes.is_empty() { // Validate column-level encoding compatibility before copying buffers - let src_col_encoding_bytes = Any::from_msg(&src_column_info.encoding) - .unwrap() - .encode_to_vec(); + let src_col_encoding_bytes = + Any::from_msg(&src_column_info.encoding)?.encode_to_vec(); let baseline_bytes = &baseline_col_encoding_bytes[col_idx]; if src_col_encoding_bytes != *baseline_bytes { return Err(Error::execution(format!( @@ -434,24 +341,29 @@ pub async fn rewrite_files_binary_copy( for (buffer_idx, (_, size)) in src_column_info.buffer_offsets_and_sizes.iter().enumerate() { - let writer = current_writer.as_mut().unwrap().as_mut(); - current_pos = apply_alignment_padding(writer, current_pos, version).await?; - let start = current_pos; - if *size == 0 { - col_buffers[col_idx].push((start, 0)); + let writer = current_writer.as_mut().ok_or_else(|| { + Error::internal("binary copy output writer was not initialized") + })?; + let bytes = if *size == 0 { + None } else { - let bytes = bytes_iter.next().ok_or_else(|| { + Some(bytes_iter.next().ok_or_else(|| { Error::execution(format!( "binary copy: missing column buffer bytes while rewriting data file \ (column {col_idx}, buffer {buffer_idx}, expected size {size})", )) - })?; - writer.write_all(&bytes).await?; - current_pos += bytes.len() as u64; - col_buffers[col_idx].push((start, bytes.len() as u64)); - } + })?) + }; + let (start, written) = writer + .write_external_buffer(bytes.as_deref().unwrap_or_default()) + .await?; + col_buffers[col_idx].push((start, written)); } - } + } + Ok(()) + }, + ) + .await?; } // finished all columns in the current source file // Accumulate rows for the current output file and flush when reaching the threshold @@ -459,21 +371,18 @@ pub async fn rewrite_files_binary_copy( if total_rows_in_current >= max_rows_per_file { let fragment_out = finalize_current_output_file( &schema, - &full_field_ids, + version, &mut current_writer, &mut current_filename, ¤t_page_table, &mut col_pages, &mut col_buffers, - &is_non_leaf_column, total_rows_in_current, - version, ) .await?; // Reset state for next output file current_writer = None; - current_pos = 0; current_page_table.clear(); for v in col_pages.iter_mut() { v.clear(); @@ -489,18 +398,17 @@ pub async fn rewrite_files_binary_copy( if total_rows_in_current > 0 { // Flush remaining rows as a final output file - init_writer_if_necessary(dataset, &mut current_writer, &mut current_filename).await?; + init_writer_if_necessary(dataset, version, &mut current_writer, &mut current_filename) + .await?; let frag = finalize_current_output_file( &schema, - &full_field_ids, + version, &mut current_writer, &mut current_filename, ¤t_page_table, &mut col_pages, &mut col_buffers, - &is_non_leaf_column, total_rows_in_current, - version, ) .await?; out.push(frag); @@ -512,9 +420,7 @@ pub async fn rewrite_files_binary_copy( /// /// This function does not manually craft the footer. Instead it: /// - Pads the current `ObjectWriter` position to a 64‑byte boundary (required for v2_1+ readers). -/// - Converts the collected per‑column info (`final_cols`) into `ColumnMetadata`. -/// - Constructs a `lance_file::writer::FileWriter` with the active `schema`, column metadata, -/// and `total_rows_in_current`. +/// - Initializes the active `FileWriter` from the collected column metadata. /// - Calls `FileWriter::finish()` to emit column metadata, offset tables, global buffers /// (schema descriptor), version, and to close the writer. /// @@ -522,81 +428,14 @@ pub async fn rewrite_files_binary_copy( /// - All page data and column‑level buffers referenced by `final_cols` have already been written /// to `writer`; otherwise offsets in the footer will be invalid. /// -/// Version notes: -/// - v2_0 structural single‑page enforcement is handled when building `final_cols`; this function -/// only performs consistent finalization. async fn flush_footer( - mut writer: Box, + writer: &mut FileWriter, schema: &Schema, final_cols: &[Arc], total_rows_in_current: u64, - version: LanceFileVersion, ) -> Result<()> { - let pos = writer.tell().await? as u64; - let _new_pos = apply_alignment_padding(writer.as_mut(), pos, version).await?; - - let mut col_metadatas = Vec::with_capacity(final_cols.len()); - for col in final_cols { - let pages = col - .page_infos - .iter() - .map(|page_info| { - let encoded_encoding = match &page_info.encoding { - PageEncoding::Legacy(array_encoding) => { - Any::from_msg(array_encoding)?.encode_to_vec() - } - PageEncoding::Structural(page_layout) => { - Any::from_msg(page_layout)?.encode_to_vec() - } - }; - let (buffer_offsets, buffer_sizes): (Vec<_>, Vec<_>) = page_info - .buffer_offsets_and_sizes - .as_ref() - .iter() - .cloned() - .unzip(); - Ok(pbfile::column_metadata::Page { - buffer_offsets, - buffer_sizes, - encoding: Some(pbfile::Encoding { - location: Some(pbfile::encoding::Location::Direct( - pbfile::DirectEncoding { - encoding: encoded_encoding, - }, - )), - }), - length: page_info.num_rows, - priority: page_info.priority, - }) - }) - .collect::>>()?; - let (buffer_offsets, buffer_sizes): (Vec<_>, Vec<_>) = - col.buffer_offsets_and_sizes.iter().cloned().unzip(); - let encoded_col_encoding = Any::from_msg(&col.encoding)?.encode_to_vec(); - let column = pbfile::ColumnMetadata { - pages, - buffer_offsets, - buffer_sizes, - encoding: Some(pbfile::Encoding { - location: Some(pbfile::encoding::Location::Direct(pbfile::DirectEncoding { - encoding: encoded_col_encoding, - })), - }), - }; - col_metadatas.push(column); - } - let mut file_writer = FileWriter::new_lazy( - writer, - FileWriterOptions { - format_version: Some(version), - ..Default::default() - }, - ); - file_writer.initialize_with_external_metadata( - schema.clone(), - col_metadatas, - total_rows_in_current, - ); - file_writer.finish().await?; + writer.write_external_buffer(&[]).await?; + writer.initialize_with_external_columns(schema.clone(), final_cols, total_rows_in_current)?; + writer.finish().await?; Ok(()) } diff --git a/rust/lance/src/dataset/optimize/remapping.rs b/rust/lance/src/dataset/optimize/remapping.rs index aef2cd231fc..9a021a44bb0 100644 --- a/rust/lance/src/dataset/optimize/remapping.rs +++ b/rust/lance/src/dataset/optimize/remapping.rs @@ -61,8 +61,13 @@ pub trait IndexRemapper: Send + Sync { /// /// Currently we don't have any options but we may need options in the future and so we /// want to keep a placeholder +#[async_trait] pub trait IndexRemapperOptions: Send + Sync { - fn create_remapper(&self, dataset: &Dataset) -> Result>; + /// Creates a remapper when the dataset has indices that need row address remapping. + /// + /// Returns `None` when no remappable indices exist, allowing compaction to avoid + /// materializing an unused row address map. + async fn create_remapper(&self, dataset: &Dataset) -> Result>>; } #[derive(Debug, Default, Clone, PartialEq, Serialize, Deserialize)] @@ -75,9 +80,10 @@ impl IndexRemapper for IgnoreRemap { } } +#[async_trait] impl IndexRemapperOptions for IgnoreRemap { - fn create_remapper(&self, _: &Dataset) -> Result> { - Ok(Box::new(Self {})) + async fn create_remapper(&self, _: &Dataset) -> Result>> { + Ok(None) } } @@ -243,7 +249,8 @@ async fn remap_index(dataset: &mut Dataset, index_id: &Uuid) -> Result<()> { // coverage-remapped + persisted before the data was remapped (e.g. while // remapping a *sibling* index). let baseline_version = curr_index_meta.dataset_version; - let (should_remap, bitmap_after_remap) = match curr_index_meta.fragment_bitmap.clone() { + let has_unknown_coverage = curr_index_meta.fragment_bitmap.is_none(); + let (should_remap, mut bitmap_after_remap) = match curr_index_meta.fragment_bitmap.clone() { Some(mut index_frag_bitmap) => { let mut should_remap = false; for version in frag_reuse_index.details.versions.iter() { @@ -285,8 +292,6 @@ async fn remap_index(dataset: &mut Dataset, index_id: &Uuid) -> Result<()> { } (should_remap, Some(index_frag_bitmap)) } - // if there is no fragment bitmap for the index, - // we attempt remapping but will not update the fragment bitmap. None => (true, None), }; @@ -316,6 +321,30 @@ async fn remap_index(dataset: &mut Dataset, index_id: &Uuid) -> Result<()> { let remapper = RowAddrRemap::direct(composed_row_id_map); let remap_result = index::remap_index(dataset, index_id, &remapper).await?; + // Remapping advances the index watermark for fragment-reuse cleanup, but it + // does not incorporate overlays committed after the source index was built. + // Exclude those fragments so queries scan their current values instead. + if let Some(fragment_bitmap) = &mut bitmap_after_remap { + for fragment in dataset.manifest.fragments.iter() { + let has_newer_indexed_overlay = fragment.overlays.iter().any(|overlay| { + overlay.committed_version > curr_index_meta.dataset_version + && overlay + .data_file + .fields + .iter() + .any(|field_id| curr_index_meta.fields.contains(field_id)) + }); + if has_newer_indexed_overlay { + fragment_bitmap.remove(fragment.id as u32); + } + } + } + let new_dataset_version = if has_unknown_coverage { + curr_index_meta.dataset_version + } else { + dataset.manifest.version + }; + let new_index_meta = match remap_result { // The composed remap emptied the index (every row deleted). Matching the // prior per-version behavior, leave the existing index untouched and @@ -325,7 +354,7 @@ async fn remap_index(dataset: &mut Dataset, index_id: &Uuid) -> Result<()> { uuid: new_id, name: curr_index_meta.name.clone(), fields: curr_index_meta.fields.clone(), - dataset_version: dataset.manifest.version, + dataset_version: new_dataset_version, fragment_bitmap: bitmap_after_remap, index_details: curr_index_meta.index_details.clone(), index_version: curr_index_meta.index_version, @@ -337,7 +366,7 @@ async fn remap_index(dataset: &mut Dataset, index_id: &Uuid) -> Result<()> { uuid: remapped_index.new_id, name: curr_index_meta.name.clone(), fields: curr_index_meta.fields.clone(), - dataset_version: dataset.manifest.version, + dataset_version: new_dataset_version, fragment_bitmap: bitmap_after_remap, index_details: Some(Arc::new(remapped_index.index_details)), index_version: remapped_index.index_version as i32, diff --git a/rust/lance/src/dataset/optimize/tests/binary_copy.rs b/rust/lance/src/dataset/optimize/tests/binary_copy.rs index 08a4fe024ce..94e28bbd123 100644 --- a/rust/lance/src/dataset/optimize/tests/binary_copy.rs +++ b/rust/lance/src/dataset/optimize/tests/binary_copy.rs @@ -3,9 +3,16 @@ use super::*; +const NON_LEGACY_VERSIONS: [LanceFileVersion; 4] = [ + LanceFileVersion::V2_0, + LanceFileVersion::V2_1, + LanceFileVersion::V2_2, + LanceFileVersion::V2_3, +]; + #[tokio::test] async fn test_binary_copy_merge_small_files() { - for version in LanceFileVersion::iter_non_legacy() { + for version in NON_LEGACY_VERSIONS { do_test_binary_copy_merge_small_files(version).await; } } @@ -45,9 +52,70 @@ async fn do_test_binary_copy_merge_small_files(version: LanceFileVersion) { assert_eq!(before, after); } +#[tokio::test] +async fn test_binary_copy_packed_struct_column_mapping() { + for version in NON_LEGACY_VERSIONS { + do_test_binary_copy_packed_struct_column_mapping(version).await; + } +} + +async fn do_test_binary_copy_packed_struct_column_mapping(version: LanceFileVersion) { + use arrow_array::StructArray; + use arrow_schema::Fields; + use std::collections::HashMap; + + let packed_fields = Fields::from(vec![Field::new("child", DataType::Int32, true)]); + let packed_field = Field::new("packed", DataType::Struct(packed_fields.clone()), true) + .with_metadata(HashMap::from([("packed".to_string(), "true".to_string())])); + let schema = Arc::new(Schema::new(vec![ + packed_field, + Field::new("tail", DataType::Int32, true), + ])); + let packed: ArrayRef = Arc::new(StructArray::new( + packed_fields, + vec![Arc::new(Int32Array::from(vec![1, 2, 3, 4]))], + None, + )); + let tail: ArrayRef = Arc::new(Int32Array::from(vec![10, 20, 30, 40])); + let batch = RecordBatch::try_new(schema.clone(), vec![packed, tail]).unwrap(); + + let test_dir = TempStrDir::default(); + let mut dataset = Dataset::write( + RecordBatchIterator::new(vec![Ok(batch)], schema), + &test_dir, + Some(WriteParams { + data_storage_version: Some(version), + max_rows_per_file: 2, + ..Default::default() + }), + ) + .await + .unwrap(); + assert_eq!(dataset.get_fragments().len(), 2); + + let before = dataset.scan().try_into_batch().await.unwrap(); + compact_files( + &mut dataset, + CompactionOptions { + target_rows_per_fragment: 100_000, + compaction_mode: Some(CompactionMode::ForceBinaryCopy), + ..Default::default() + }, + None, + ) + .await + .unwrap(); + let after = dataset.scan().try_into_batch().await.unwrap(); + assert_eq!(before, after); + + let data_file = &dataset.manifest.fragments[0].files[0]; + assert_eq!(data_file.fields.len(), 2); + assert_eq!(data_file.column_indices.as_ref(), &[0, 1]); +} + #[tokio::test] async fn test_binary_copy_empty_string_scalar_index() { - for version in LanceFileVersion::iter_non_legacy() { + for version in NON_LEGACY_VERSIONS { do_test_binary_copy_empty_string_scalar_index(version).await; } } @@ -115,7 +183,7 @@ async fn do_test_binary_copy_empty_string_scalar_index(version: LanceFileVersion #[tokio::test] async fn test_binary_copy_with_defer_remap() { - for version in LanceFileVersion::iter_non_legacy() { + for version in NON_LEGACY_VERSIONS { do_test_binary_copy_with_defer_remap(version).await; } } @@ -187,7 +255,7 @@ async fn do_test_binary_copy_with_defer_remap(version: LanceFileVersion) { #[tokio::test] async fn test_binary_copy_preserves_stable_row_ids() { - for version in LanceFileVersion::iter_non_legacy() { + for version in NON_LEGACY_VERSIONS { do_binary_copy_preserves_stable_row_ids(version).await; } } @@ -320,7 +388,7 @@ async fn do_binary_copy_preserves_stable_row_ids(version: LanceFileVersion) { #[tokio::test] async fn test_binary_copy_remaps_unstable_row_ids() { - for version in LanceFileVersion::iter_non_legacy() { + for version in NON_LEGACY_VERSIONS { do_binary_copy_remaps_unstable_row_ids(version).await; } } @@ -685,7 +753,8 @@ async fn test_can_use_binary_copy_version_mismatch() { ); // Simulate mixed file versions by marking the second fragment as v2.1. - let (v21_major, v21_minor) = LanceFileVersion::V2_1.to_numbers(); + let (v21_major, v21_minor) = + lance_file::version::ConcreteFileVersion::V2_1.to_data_file_numbers(); for file in &mut frags[1].files { file.file_major_version = v21_major; file.file_minor_version = v21_minor; @@ -721,11 +790,27 @@ async fn test_can_use_binary_copy_reject_deletions() { assert!(!can_use_binary_copy(&dataset, &options, &frags).await); } +#[rstest::rstest] +#[case(LanceFileVersion::V2_0)] +#[case(LanceFileVersion::V2_1)] +#[case(LanceFileVersion::V2_2)] +#[case(LanceFileVersion::V2_3)] #[tokio::test] -async fn test_binary_copy_compaction_with_complex_schema() { - for version in LanceFileVersion::iter_non_legacy() { - do_test_binary_copy_compaction_with_complex_schema(version).await; - } +async fn test_binary_copy_compaction_with_complex_schema(#[case] version: LanceFileVersion) { + do_test_binary_copy_compaction_with_complex_schema(version).await; +} + +#[test] +fn test_binary_copy_complex_schema_covers_every_non_legacy_version() { + assert_eq!( + [ + LanceFileVersion::V2_0, + LanceFileVersion::V2_1, + LanceFileVersion::V2_2, + LanceFileVersion::V2_3 + ], + NON_LEGACY_VERSIONS + ); } async fn do_test_binary_copy_compaction_with_complex_schema(version: LanceFileVersion) { @@ -733,7 +818,10 @@ async fn do_test_binary_copy_compaction_with_complex_schema(version: LanceFileVe use lance_core::utils::tempfile::TempStrDir; use lance_datagen::{BatchCount, Dimension, RowCount, array, gen_batch}; - let row_num = 1_000; + let row_num: u64 = 1_000; + let batches: u32 = 10; + const NUM_FRAGMENTS: usize = 100; + let rows_per_file = (row_num as usize * batches as usize) / NUM_FRAGMENTS; let inner_fields = Fields::from(vec![ Field::new("x", DataType::UInt32, true), @@ -794,7 +882,7 @@ async fn do_test_binary_copy_compaction_with_complex_schema(version: LanceFileVe "events", array::rand_list_any(array::rand_struct(event_fields.clone()), true), ) - .into_reader_rows(RowCount::from(row_num), BatchCount::from(10)); + .into_reader_rows(RowCount::from(row_num), BatchCount::from(batches)); let full_dir = TempStrDir::default(); let mut dataset = Dataset::write( @@ -803,13 +891,19 @@ async fn do_test_binary_copy_compaction_with_complex_schema(version: LanceFileVe Some(WriteParams { enable_stable_row_ids: true, data_storage_version: Some(version), - max_rows_per_file: (row_num / 100) as usize, + max_rows_per_file: rows_per_file, ..Default::default() }), ) .await .unwrap(); + assert_eq!( + dataset.get_fragments().len(), + NUM_FRAGMENTS, + "compaction must have many input fragments to merge" + ); + let opt_full = CompactionOptions { compaction_mode: Some(CompactionMode::Reencode), ..Default::default() diff --git a/rust/lance/src/dataset/overlay.rs b/rust/lance/src/dataset/overlay.rs new file mode 100644 index 00000000000..1beba0ad755 --- /dev/null +++ b/rust/lance/src/dataset/overlay.rs @@ -0,0 +1,1433 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright The Lance Authors + +//! Resolution of data overlay files on read. +//! +//! An overlay supplies replacement values for some `(row, field)` cells without +//! rewriting the base data. Resolving a read means, for each row we return, +//! deciding whether its value comes from the base column or from an overlay. +//! +//! Three coordinate spaces show up throughout this module; keeping them straight +//! is most of the work: +//! +//! - `offset_in_frag`: a row's physical position in the fragment (0-based over all +//! physical rows, ignoring deletions). This is how a cell is addressed on disk +//! and in an overlay's coverage bitmap. +//! - `offset_in_batch`: a row's position within the batch we are currently +//! assembling (0-based). The output column is indexed by this. +//! - `offset_in_overlay`: the position of a value in an overlay's value column. +//! An overlay stores its values densely — one per covered cell, in ascending +//! `offset_in_frag` order — so a covered cell's value is found by counting how +//! many covered cells come before it. (That count is what a roaring bitmap calls +//! the cell's "rank".) +//! +//! For a given field, the overlays covering it are consulted newest to oldest: the +//! first overlay that covers a row wins, and its value is read at that row's +//! `offset_in_overlay`. A row that no overlay covers keeps its base value. +//! +//! The rows to resolve are passed in as a list of `offset_in_frag` (one per output +//! row), so a single code path serves both scans (a contiguous range of offsets) +//! and `take` (arbitrary offsets). +//! +//! Deletions win over overlays, but nothing here handles that: the merge runs on +//! physical rows *before* deletions are applied, so an overlay value computed for a +//! deleted row is simply dropped along with the row. This matches the spec with no +//! special casing. + +use std::collections::{BTreeSet, HashMap}; +use std::sync::Arc; + +use arrow_array::{Array, ArrayRef, RecordBatch, StructArray}; +use arrow_select::interleave::interleave; +use futures::StreamExt; +use lance_core::datatypes::{Field, Schema}; +use lance_core::{Error, Result}; +use roaring::RoaringBitmap; + +use lance_table::format::overlay::DataOverlayFile; +use lance_table::format::{DataFile, Fragment, IndexMetadata}; +use lance_table::utils::stream::ReadBatchFut; + +use crate::dataset::fragment::{FileFragment, FragReadConfig, GenericFileReader}; + +/// The physical offsets within a fragment whose value for an indexed field may be +/// stale relative to an index built at `index_version`, and so must be excluded +/// from that index's results and re-evaluated against current values on the flat +/// path. +/// +/// The set is the union, over every overlay whose `committed_version` is newer +/// than `index_version`, of that overlay's coverage **restricted to the indexed +/// fields**. The restriction makes exclusion field-aware: an overlay that touches +/// only non-indexed fields contributes nothing. An overlay whose +/// `committed_version <= index_version` is already incorporated by the index and +/// is ignored. +pub fn overlay_exclusion_offsets( + overlays: &[DataOverlayFile], + indexed_field_ids: &[i32], + index_version: u64, + schema: &Schema, +) -> Result { + let mut excluded = RoaringBitmap::new(); + for overlay in overlays { + if overlay.committed_version <= index_version { + continue; + } + for (field_pos, field_id) in overlay.data_file.fields.iter().enumerate() { + let overlay_ancestry = schema.field_ancestry_by_id(*field_id); + let affects_index = indexed_field_ids.iter().any(|indexed_field_id| { + indexed_field_id == field_id + || overlay_ancestry.as_ref().is_some_and(|ancestry| { + ancestry + .iter() + .any(|ancestor| ancestor.id == *indexed_field_id) + }) + || schema + .field_ancestry_by_id(*indexed_field_id) + .is_some_and(|ancestry| { + ancestry.iter().any(|ancestor| ancestor.id == *field_id) + }) + }); + if affects_index { + excluded |= &*overlay.coverage_for_field(field_pos)?; + } + } + } + Ok(excluded) +} + +// Stale row offsets contributed by one fragment's overlays for a given index version. +// Applies a cheap version gate first: if every overlay predates the segment it is already +// incorporated by the index, so there is nothing stale and the field/bitmap work is skipped. +fn stale_offsets_for_fragment( + fragment: &Fragment, + fields: &[i32], + index_version: u64, + schema: &Schema, +) -> Result { + if fragment + .overlays + .iter() + .all(|o| o.committed_version <= index_version) + { + return Ok(RoaringBitmap::new()); + } + overlay_exclusion_offsets(&fragment.overlays, fields, index_version, schema) +} + +// A missing `fragment_bitmap` means the index predates fragment-bitmap tracking; treat it as +// covering every fragment (matching `DatasetPreFilter::new`) so overlay-stale rows can't slip +// through unmasked. Only skip fragments explicitly absent from a present bitmap. +fn covers_fragment(coverage: Option<&RoaringBitmap>, frag_id: u32) -> bool { + coverage.is_none_or(|c| c.contains(frag_id)) +} + +/// Index by fragment id the fragments that carry at least one overlay. Overlays are rare, so +/// this is empty on the common path, letting callers skip index loading entirely; when non-empty +/// it bounds the stale-collection loops to `O(overlaid fragments)`. +pub fn overlaid_fragments(fragments: &[Fragment]) -> HashMap { + fragments + .iter() + .filter(|f| !f.overlays.is_empty()) + .map(|f| (f.id as u32, f)) + .collect() +} + +/// Insert into `stale` the ids of fragments covered by `segment` whose index entries may be +/// stale because an overlay committed after the segment was built touches a field the segment +/// indexes. Field-aware and version-gated via [`overlay_exclusion_offsets`]. +/// +/// `overlaid_frags` holds only the fragments that actually carry overlays (rare), so the loop is +/// `O(overlaid_frags)` rather than `O(fragments the segment covers)`. +pub fn collect_overlay_stale_frags( + segment: &IndexMetadata, + overlaid_frags: &HashMap, + stale: &mut RoaringBitmap, + schema: &Schema, +) -> Result<()> { + let coverage = segment.fragment_bitmap.as_ref(); + for (&frag_id, fragment) in overlaid_frags { + if stale.contains(frag_id) || !covers_fragment(coverage, frag_id) { + continue; + } + if !stale_offsets_for_fragment(fragment, &segment.fields, segment.dataset_version, schema)? + .is_empty() + { + stale.insert(frag_id); + } + } + Ok(()) +} + +/// Compute exactly which row offsets within each covered fragment are stale and accumulate them +/// into `stale` (fragment_id → stale row offsets). +/// +/// Used by the scalar and vector paths to block only the affected rows from index results and +/// re-evaluate only those rows on the flat path, keeping overhead proportional to the number of +/// overlaid rows rather than the whole fragment size. +pub fn collect_overlay_stale_rows_for_segment( + segment: &IndexMetadata, + overlaid_frags: &HashMap, + stale: &mut HashMap, + schema: &Schema, +) -> Result<()> { + let coverage = segment.fragment_bitmap.as_ref(); + for (&frag_id, fragment) in overlaid_frags { + if !covers_fragment(coverage, frag_id) { + continue; + } + let excluded = + stale_offsets_for_fragment(fragment, &segment.fields, segment.dataset_version, schema)?; + if !excluded.is_empty() { + *stale.entry(frag_id).or_default() |= &excluded; + } + } + Ok(()) +} + +/// The plan for merging one field's overlays into one batch: which source (base or +/// a particular overlay) supplies each output row, and which overlay values must be +/// fetched to do it. +/// +/// Built by [`route_overlays`] from the coverage bitmaps alone — before any value +/// column is read — so the caller can fetch only the overlay values it will +/// actually use (its `offsets_in_overlay`) rather than whole columns, then build the +/// merged column with [`assemble_overlay_column`]. +struct OverlayRouting { + /// One `(source, position)` pair per output row, ready to hand to `interleave`. + /// Source `0` is the base column, with `position` = the row's `offset_in_batch`; + /// source `k + 1` is overlay `k`'s fetched values, with `position` = the row's + /// index into those fetched values. + indices: Vec<(usize, usize)>, + /// Per overlay (newest-first): the sorted, deduplicated `offset_in_overlay` + /// values this batch needs from that overlay — i.e. exactly which entries of its + /// value column to fetch. + offsets_in_overlay: Vec>, + /// Whether any row is covered by an overlay at all (false ⇒ every row falls + /// through to the base column, so the base is already the answer and no overlay + /// values need to be read). + any_overlay: bool, +} + +/// For each row in `offsets_in_frag`, decide whether its value comes from the base +/// column or from an overlay — and if from an overlay, at which `offset_in_overlay`. +/// +/// Only the coverage bitmaps are consulted (newest-first), so this runs before any +/// value column is read and reports exactly which overlay values the caller must +/// fetch. +/// +/// A scan asks for a contiguous, ascending range of offsets, which enables a faster +/// bitmap-driven path ([`route_contiguous`]); `take` asks for arbitrary offsets and +/// uses the general path ([`route_arbitrary`]). Both produce identical routing. +fn route_overlays( + offsets_in_frag: &[u32], + coverages_newest_first: &[&RoaringBitmap], +) -> OverlayRouting { + match contiguous_frag_start(offsets_in_frag) { + Some(frag_start) => { + route_contiguous(frag_start, offsets_in_frag.len(), coverages_newest_first) + } + None => route_arbitrary(offsets_in_frag, coverages_newest_first), + } +} + +/// If `offsets_in_frag` is a contiguous ascending run `[start, start + 1, ...]`, +/// return `start`; otherwise `None` (including when empty). +fn contiguous_frag_start(offsets_in_frag: &[u32]) -> Option { + let start = *offsets_in_frag.first()?; + offsets_in_frag + .iter() + .enumerate() + .all(|(i, &offset)| offset as u64 == start as u64 + i as u64) + .then_some(start) +} + +/// Fast path for a scan, where the batch is a contiguous run of offsets starting at +/// `frag_start`. Because the offsets are contiguous, a row's `offset_in_batch` is +/// just `offset_in_frag - frag_start`, so a coverage's set bits map straight to +/// output rows — no need to test each row against each coverage. +/// +/// For each coverage we intersect it with the batch's offset range. Roaring does +/// this a block at a time, so a coverage that does not overlap the batch (e.g. a +/// scan batch past the last cell this overlay touches) is skipped cheaply without +/// inspecting individual bits. +/// +/// Within the batch a coverage's cells appear in ascending order, so their +/// `offset_in_overlay` values are consecutive: the first in-batch cell sits at +/// `offset_in_overlay = ` (a single +/// `rank` lookup), and each following cell is one more. Coverages are applied +/// newest-first, and the first overlay to claim a row wins. +fn route_contiguous( + frag_start: u32, + len: usize, + coverages_newest_first: &[&RoaringBitmap], +) -> OverlayRouting { + let mut offsets_in_overlay: Vec> = vec![Vec::new(); coverages_newest_first.len()]; + // Indexed by offset_in_batch: which (overlay, fetch position) supplies the row. + let mut routed: Vec> = vec![None; len]; + let range_end = (frag_start as u64 + len as u64).min(u32::MAX as u64) as u32; + let mut batch_range = RoaringBitmap::new(); + batch_range.insert_range(frag_start..range_end); + + for (k, coverage) in coverages_newest_first.iter().enumerate() { + let covered_in_batch = *coverage & &batch_range; + if covered_in_batch.is_empty() { + continue; + } + // offset_in_overlay of this coverage's first in-batch cell = the number of + // its cells that lie before the batch. + let first_offset_in_overlay = if frag_start == 0 { + 0 + } else { + coverage.rank(frag_start - 1) as u32 + }; + for (nth_in_batch, offset_in_frag) in covered_in_batch.iter().enumerate() { + let offset_in_batch = (offset_in_frag - frag_start) as usize; + if routed[offset_in_batch].is_none() { + routed[offset_in_batch] = Some((k, offsets_in_overlay[k].len())); + offsets_in_overlay[k].push(first_offset_in_overlay + nth_in_batch as u32); + } + } + } + + let mut any_overlay = false; + let indices = routed + .into_iter() + .enumerate() + .map(|(offset_in_batch, routed)| match routed { + None => (0, offset_in_batch), + Some((k, fetch_pos)) => { + any_overlay = true; + (k + 1, fetch_pos) + } + }) + .collect(); + + OverlayRouting { + indices, + offsets_in_overlay, + any_overlay, + } +} + +/// General path for arbitrary offsets (e.g. `take`): test each row's +/// `offset_in_frag` against the coverages newest-first. `take` batches are small, +/// so this `O(rows * overlays)` probing is not a concern. +fn route_arbitrary( + offsets_in_frag: &[u32], + coverages_newest_first: &[&RoaringBitmap], +) -> OverlayRouting { + // Per overlay: the distinct offset_in_overlay values this batch needs, sorted. + let mut offset_sets: Vec> = vec![BTreeSet::new(); coverages_newest_first.len()]; + // Per output row: the (overlay, offset_in_overlay) that supplies it, if any. + let mut routed_per_row: Vec> = Vec::with_capacity(offsets_in_frag.len()); + for &offset_in_frag in offsets_in_frag { + let mut routed = None; + for (k, coverage) in coverages_newest_first.iter().enumerate() { + if coverage.contains(offset_in_frag) { + // offset_in_overlay = number of covered cells before this one. + let offset_in_overlay = coverage.rank(offset_in_frag) as u32 - 1; + offset_sets[k].insert(offset_in_overlay); + routed = Some((k, offset_in_overlay)); + break; + } + } + routed_per_row.push(routed); + } + + let offsets_in_overlay: Vec> = offset_sets + .iter() + .map(|offsets| offsets.iter().copied().collect()) + .collect(); + // For each overlay, map an offset_in_overlay to its position in the fetched + // (sorted, deduplicated) value list. + let fetch_positions: Vec> = offsets_in_overlay + .iter() + .map(|offsets| { + offsets + .iter() + .enumerate() + .map(|(pos, &o)| (o, pos)) + .collect() + }) + .collect(); + + let mut any_overlay = false; + let indices = routed_per_row + .into_iter() + .enumerate() + .map(|(offset_in_batch, routed)| match routed { + None => (0, offset_in_batch), + Some((k, offset_in_overlay)) => { + any_overlay = true; + (k + 1, fetch_positions[k][&offset_in_overlay]) + } + }) + .collect(); + + OverlayRouting { + indices, + offsets_in_overlay, + any_overlay, + } +} + +/// Build the merged column from `base` and the overlay values fetched for the +/// `offset_in_overlay` values [`route_overlays`] asked for. +/// +/// `fetched_newest_first[k]` holds overlay `k`'s values for `routing`'s +/// `offsets_in_overlay[k]`, in that order. The result has the same length and +/// type as `base`. A covered row whose overlay value is NULL resolves **to** NULL +/// (distinct from a fall-through, which keeps the base value). +fn assemble_overlay_column( + base: &ArrayRef, + routing: &OverlayRouting, + fetched_newest_first: &[ArrayRef], +) -> Result { + if !routing.any_overlay { + return Ok(base.clone()); + } + if fetched_newest_first.len() != routing.offsets_in_overlay.len() { + return Err(Error::invalid_input(format!( + "overlay assembly got {} value columns but routing expects {}", + fetched_newest_first.len(), + routing.offsets_in_overlay.len() + ))); + } + for (k, values) in fetched_newest_first.iter().enumerate() { + if values.len() != routing.offsets_in_overlay[k].len() { + return Err(Error::invalid_input(format!( + "overlay value column {} has {} values but {} were requested", + k, + values.len(), + routing.offsets_in_overlay[k].len() + ))); + } + } + + let mut sources: Vec<&dyn Array> = Vec::with_capacity(fetched_newest_first.len() + 1); + sources.push(base.as_ref()); + for values in fetched_newest_first { + sources.push(values.as_ref()); + } + interleave(&sources, &routing.indices).map_err(Error::from) +} + +/// One overlay's contribution to one projected atomic field, with its file reader opened. +#[derive(Debug, Clone)] +struct LoadedAtomicFieldOverlay { + /// The `offset_in_frag` cells this overlay covers for the atomic field. + coverage: Arc, + /// Reader over the overlay data file, projected to the covered atomic fields; shared + /// across the atomic fields that the same file covers. + reader: Arc, +} + +/// The overlays that apply to a single projected atomic field — a per-row field an overlay +/// can replace as a unit (a primitive leaf, or a whole list/map field; structs are +/// recursed through, not treated as atomic fields). Ordered newest-first, with readers opened +/// and pruned to a specific read. Produced by [`resolve_overlays`] and consumed by +/// [`merge_overlay_batch`]. +#[derive(Debug, Clone)] +pub struct LoadedAtomicField { + /// The top-level output column the atomic field lives in (its name locates the batch + /// column; its field tree drives the descend/splice into that column). + top_field: Arc, + /// Child field ids from `top_field` down to the atomic field (empty when the atomic + /// field *is* the top-level column). Drives descending to, and splicing back, the + /// atomic field. + ancestor_ids: Vec, + /// Projection of exactly the atomic field (its ancestor path pruned to the atomic + /// field subtree), used to fetch the atomic field's values from the overlay file. + fetch_projection: Arc, + overlays_newest_first: Vec, +} + +/// One overlay file that may contribute to a read, before it is opened. Opened +/// lazily by [`resolve_overlays`], and only if the read actually touches it. +#[derive(Debug, Clone)] +struct PlannedOverlayFile { + data_file: DataFile, + /// The covered ∩ projected atomic fields to project when the file is opened, so a single + /// reader serves every atomic field the file contributes to. + open_projection: Arc, +} + +/// One overlay's contribution to one projected atomic field, before the file is opened. +#[derive(Debug, Clone)] +struct PlannedAtomicFieldOverlay { + /// Index into [`OverlayReadPlanner::files`] of the file that supplies the value. + file: usize, + coverage: Arc, +} + +/// The overlays that apply to a single projected atomic field, ordered newest-first, before +/// any file is opened. +#[derive(Debug, Clone)] +struct PlannedAtomicField { + top_field: Arc, + ancestor_ids: Vec, + fetch_projection: Arc, + overlays_newest_first: Vec, +} + +/// A fragment's overlay-resolution plan for a projection, derived from coverage +/// metadata alone — no file opened, no IO. [`resolve_overlays`] turns it into opened +/// [`LoadedAtomicField`]s for one specific read, opening only the files whose cells +/// the read's rows actually touch. +#[derive(Debug, Clone)] +pub struct OverlayReadPlanner { + files: Vec, + atomic_fields: Vec, +} + +impl OverlayReadPlanner { + /// True when no projected atomic field has any overlay, so there is nothing to resolve. + pub fn is_empty(&self) -> bool { + self.atomic_fields.is_empty() + } +} + +/// Plan `fragment`'s overlay resolution for a projection from coverage metadata +/// alone. No files are opened here (see [`resolve_overlays`]) — this only reads the +/// already-parsed coverage bitmaps, so it is cheap enough to run on every open. +/// +/// Overlays are stored oldest-first (sorted newest-last on load, see +/// `sort_overlays_newest_last`), so walking them in reverse gives newest-first +/// precedence. +/// +/// Resolution is per *atomic field* — a per-row field that an overlay replaces as a unit: a +/// primitive leaf, or a whole list/map field. Structs are internal nodes, so each +/// leaf of a struct is its own atomic field and can be overlaid independently of its +/// siblings. An overlay is written against the leaf ids it stores (the V2_1 +/// structural encoding records only leaves), so an overlay contributes to a projected +/// atomic field when any id in its `data_file.fields` falls in that atomic field's leaf +/// set. At merge time the atomic field's value is fetched and spliced into its output +/// column, so an overlay on a sub-field never disturbs the column's other leaves. Each +/// contributing overlay *file* appears once in `files`, shared by every atomic field it +/// covers. +pub fn plan_overlays(fragment: &FileFragment, projection: &Schema) -> Result { + let overlays = &fragment.metadata.overlays; + debug_assert!( + overlays + .windows(2) + .all(|w| w[0].committed_version <= w[1].committed_version), + "overlays must be sorted newest-last (see sort_overlays_newest_last)" + ); + + // The projection's atomic fields, and a leaf-id -> atomic-field-index map so an + // overlay's stored leaf ids resolve to the atomic field they belong to in O(1). + struct AtomicFieldInfo<'a> { + top_field: &'a Field, + ancestor_ids: Vec, + atomic_field_id: i32, + } + let mut atomic_field_infos: Vec = Vec::new(); + let mut leaf_to_atomic_field: HashMap = HashMap::new(); + for top in &projection.fields { + for (atomic_field, ancestor_ids) in enumerate_atomic_fields(top) { + let idx = atomic_field_infos.len(); + let mut value_leaf_ids = Vec::new(); + collect_leaf_ids(atomic_field, &mut value_leaf_ids); + for leaf in value_leaf_ids { + leaf_to_atomic_field.insert(leaf, idx); + } + atomic_field_infos.push(AtomicFieldInfo { + top_field: top, + ancestor_ids, + atomic_field_id: atomic_field.id, + }); + } + } + + // Walk overlays newest-first. For each overlay, find the atomic fields it covers and push + // (newest-first, for free) into their per-atomic field overlay lists. + let mut files = Vec::new(); + let mut atomic_field_overlays: Vec> = + vec![Vec::new(); atomic_field_infos.len()]; + for overlay in overlays.iter().rev() { + // atomic field index -> the `data_file.fields` position whose coverage to read. An + // overlay writes one value per row per atomic field, so its leaves share a coverage; + // the first leaf of each atomic field to appear wins. + let mut covered: HashMap = HashMap::new(); + for (field_pos, &field_id) in overlay.data_file.fields.iter().enumerate() { + if let Some(&atomic_field_idx) = leaf_to_atomic_field.get(&field_id) { + covered.entry(atomic_field_idx).or_insert(field_pos); + } + } + if covered.is_empty() { + continue; + } + let file = files.len(); + let covered_ids: Vec = covered + .keys() + .map(|&i| atomic_field_infos[i].atomic_field_id) + .collect(); + files.push(PlannedOverlayFile { + data_file: overlay.data_file.clone(), + open_projection: Arc::new(projection.project_by_ids(&covered_ids, true)), + }); + for (atomic_field_idx, field_pos) in covered { + atomic_field_overlays[atomic_field_idx].push(PlannedAtomicFieldOverlay { + file, + coverage: overlay.coverage_for_field(field_pos)?, + }); + } + } + + // Emit one PlannedAtomicField per projected atomic field that has overlays, in + // atomic field order. + let mut atomic_fields = Vec::new(); + for (idx, info) in atomic_field_infos.iter().enumerate() { + let overlays_newest_first = std::mem::take(&mut atomic_field_overlays[idx]); + if overlays_newest_first.is_empty() { + continue; + } + atomic_fields.push(PlannedAtomicField { + top_field: Arc::new(info.top_field.clone()), + ancestor_ids: info.ancestor_ids.clone(), + fetch_projection: Arc::new(projection.project_by_ids(&[info.atomic_field_id], true)), + overlays_newest_first, + }); + } + Ok(OverlayReadPlanner { + files, + atomic_fields, + }) +} + +/// The per-row atomic fields of a projected top-level field, each with the child-id path from +/// the top-level field down to it. Structs are recursed through; a primitive leaf or a +/// whole list/map field is an atomic field (values are one-per-row). A top-level primitive or +/// list yields a single atomic field with an empty path. +fn enumerate_atomic_fields(top: &Field) -> Vec<(&Field, Vec)> { + fn recurse<'a>(field: &'a Field, path: &mut Vec, out: &mut Vec<(&'a Field, Vec)>) { + if field.logical_type.is_struct() { + for child in &field.children { + path.push(child.id); + recurse(child, path, out); + path.pop(); + } + } else { + out.push((field, path.clone())); + } + } + let mut out = Vec::new(); + let mut path = Vec::new(); + recurse(top, &mut path, &mut out); + out +} + +/// Collect the leaf field ids in `field`'s subtree — the ids an overlay stores for +/// this atomic field (its own id if primitive; its item leaves if a list/map). +fn collect_leaf_ids(field: &Field, out: &mut Vec) { + if field.children.is_empty() { + out.push(field.id); + } else { + for child in &field.children { + collect_leaf_ids(child, out); + } + } +} + +/// Follow a path of child field ids from `field` down through nested structs, taking +/// the corresponding child array at each step. Returns the array at the end of the +/// path (the whole `array` when `ancestor_ids` is empty). +fn descend_by_ids(array: &ArrayRef, field: &Field, ancestor_ids: &[i32]) -> Result { + let mut arr = array.clone(); + let mut fld = field; + for &id in ancestor_ids { + let child_pos = fld + .children + .iter() + .position(|c| c.id == id) + .ok_or_else(|| { + Error::invalid_input(format!( + "overlay descend: field id {id} not found under '{}'", + fld.name + )) + })?; + let structs = arr.as_any().downcast_ref::().ok_or_else(|| { + Error::invalid_input(format!( + "overlay descend: expected a struct at '{}'", + fld.name + )) + })?; + arr = structs.column(child_pos).clone(); + fld = &fld.children[child_pos]; + } + Ok(arr) +} + +/// Rebuild `array` with the array at `ancestor_ids` replaced by `new_atomic_field`, cloning +/// the struct spine along the path and preserving each struct's null buffer and other +/// children. With an empty path this is just `new_atomic_field` (whole-column replacement). +fn splice_by_ids( + array: &ArrayRef, + field: &Field, + ancestor_ids: &[i32], + new_atomic_field: ArrayRef, +) -> Result { + let Some((&id, rest)) = ancestor_ids.split_first() else { + return Ok(new_atomic_field); + }; + let child_pos = field + .children + .iter() + .position(|c| c.id == id) + .ok_or_else(|| { + Error::invalid_input(format!( + "overlay splice: field id {id} not found under '{}'", + field.name + )) + })?; + let structs = array + .as_any() + .downcast_ref::() + .ok_or_else(|| { + Error::invalid_input(format!( + "overlay splice: expected a struct at '{}'", + field.name + )) + })?; + let len = structs.len(); + let (fields, mut children, nulls) = structs.clone().into_parts(); + children[child_pos] = splice_by_ids( + &children[child_pos], + &field.children[child_pos], + rest, + new_atomic_field, + )?; + Ok(Arc::new(StructArray::try_new_with_length( + fields, children, nulls, len, + )?)) +} + +/// Open the overlay readers a specific read needs and return the per-field plans to +/// merge, pruned to that read. +/// +/// `offsets_in_frag` are the rows the read will return. An overlay whose coverage is +/// disjoint from those rows contributes nothing, so it is dropped and its file is +/// never opened — a `take` that misses an overlay's cells pays no IO for it. Each +/// surviving file is opened once, concurrently, projected to the covered fields; the +/// value bytes are still not read here (the per-batch [`merge_overlay_batch`] fetches +/// only the values it needs). +pub async fn resolve_overlays( + planner: &OverlayReadPlanner, + offsets_in_frag: &[u32], + fragment: &FileFragment, + read_config: &FragReadConfig, +) -> Result> { + let read_offsets = read_offsets_bitmap(offsets_in_frag); + + // A file is opened only if some atomic field it covers has cells among the requested rows. + // This is the row-selection pruning: overlays outside the read are skipped. + let mut file_needed = vec![false; planner.files.len()]; + for atomic_field in &planner.atomic_fields { + for overlay in &atomic_field.overlays_newest_first { + if !overlay.coverage.is_disjoint(&read_offsets) { + file_needed[overlay.file] = true; + } + } + } + + // Open each needed file once, concurrently. The reader is shared (via `Arc`) by + // every atomic field that file covers. + // + // These reads use priority 0 (highest): they are issued only when a ready + // consumer polls the batch task (see `merge_overlay_batch`), so we have already + // committed to reading this batch and the overlay reads cannot clog the + // backpressure queue ahead of work we are not ready for. (A future optimization + // could start the overlay fetches earlier to fill compute bubbles, which would + // want a priority tied to the base read.) + let opened: Vec>> = + futures::future::try_join_all(planner.files.iter().enumerate().map(|(i, file)| { + let needed = file_needed[i]; + async move { + if !needed { + return Ok::<_, Error>(None); + } + Ok(fragment + .open_reader(&file.data_file, Some(&file.open_projection), read_config) + .await? + .map(Arc::from)) + } + })) + .await?; + + let mut plans = Vec::new(); + for atomic_field in &planner.atomic_fields { + let mut overlays_newest_first = Vec::new(); + for overlay in &atomic_field.overlays_newest_first { + let Some(reader) = &opened[overlay.file] else { + continue; // pruned: coverage disjoint from the read + }; + overlays_newest_first.push(LoadedAtomicFieldOverlay { + coverage: overlay.coverage.clone(), + reader: reader.clone(), + }); + } + if !overlays_newest_first.is_empty() { + plans.push(LoadedAtomicField { + top_field: atomic_field.top_field.clone(), + ancestor_ids: atomic_field.ancestor_ids.clone(), + fetch_projection: atomic_field.fetch_projection.clone(), + overlays_newest_first, + }); + } + } + Ok(plans) +} + +/// The set of `offset_in_frag` a read will return, as a bitmap for cheap +/// intersection against overlay coverages. Contiguous scans build a single range; +/// arbitrary `take` offsets (small batches) are inserted individually. +fn read_offsets_bitmap(offsets_in_frag: &[u32]) -> RoaringBitmap { + let mut bitmap = RoaringBitmap::new(); + match contiguous_frag_start(offsets_in_frag) { + Some(start) => { + let end = (start as u64 + offsets_in_frag.len() as u64).min(u32::MAX as u64) as u32; + bitmap.insert_range(start..end); + } + None => bitmap.extend(offsets_in_frag.iter().copied()), + } + bitmap +} + +/// Resolve overlays for one base batch: route each projected atomic field against the batch's +/// `offsets_in_frag`, fetch only the overlay values the batch needs (concurrently with +/// the base read), assemble the merged atomic field, and splice it into its output column. +/// AtomicFields with no covered rows, and columns with no plan, pass through. +pub async fn merge_overlay_batch( + base: ReadBatchFut, + offsets_in_frag: &[u32], + plans: &[LoadedAtomicField], +) -> Result { + let atomic_field_work = futures::future::try_join_all(plans.iter().map(|plan| async move { + let coverages: Vec<&RoaringBitmap> = plan + .overlays_newest_first + .iter() + .map(|overlay| overlay.coverage.as_ref()) + .collect(); + let routing = route_overlays(offsets_in_frag, &coverages); + if !routing.any_overlay { + return Ok::<_, Error>((plan, None)); + } + // Fetch each overlay's values and descend to the atomic field array. The fetch is + // projected to the atomic field's ancestor path, so the fetched column is the pruned + // top-level column; `descend_by_ids` walks it down to the atomic field. + let atomic_field = &plan.fetch_projection.fields[0]; + let fetched = futures::future::try_join_all( + plan.overlays_newest_first + .iter() + .zip(&routing.offsets_in_overlay) + .map(|(overlay, offsets_in_overlay)| async move { + let column = fetch_overlay_values( + overlay.reader.as_ref(), + plan.fetch_projection.clone(), + offsets_in_overlay, + ) + .await?; + descend_by_ids(&column, atomic_field, &plan.ancestor_ids) + }), + ) + .await?; + Ok((plan, Some((routing, fetched)))) + })); + + // The base read and every overlay value read proceed concurrently. + let (batch, resolved) = futures::future::try_join(base, atomic_field_work).await?; + + let schema = batch.schema(); + let mut columns = batch.columns().to_vec(); + for (plan, work) in resolved { + let Some((routing, fetched)) = work else { + continue; + }; + let Some(idx) = schema.index_of(&plan.top_field.name).ok() else { + // The plan's column is not in this batch's projection; skip it. + continue; + }; + let base_atomic_field = descend_by_ids(&columns[idx], &plan.top_field, &plan.ancestor_ids)?; + let merged_atomic_field = assemble_overlay_column(&base_atomic_field, &routing, &fetched)?; + columns[idx] = splice_by_ids( + &columns[idx], + &plan.top_field, + &plan.ancestor_ids, + merged_atomic_field, + )?; + } + Ok(RecordBatch::try_new(schema, columns)?) +} + +/// Fetch one overlay's values at the given `offsets_in_overlay` (sorted, unique): +/// the corresponding entries of its value column, as the top-level column pruned to +/// `projection`. Returns `offsets_in_overlay.len()` rows in the same order; empty +/// input reads nothing and returns an empty column. +async fn fetch_overlay_values( + reader: &dyn GenericFileReader, + projection: Arc, + offsets_in_overlay: &[u32], +) -> Result { + if offsets_in_overlay.is_empty() { + return Ok(arrow_array::new_empty_array( + &projection.fields[0].data_type(), + )); + } + let mut tasks = reader + .take_all_tasks( + offsets_in_overlay, + offsets_in_overlay.len() as u32, + projection, + None, + ) + .await?; + let mut chunks: Vec = Vec::new(); + while let Some(task) = tasks.next().await { + let batch = task.task.await?; + chunks.push(batch.column(0).clone()); + } + let chunk_refs: Vec<&dyn arrow_array::Array> = chunks.iter().map(|a| a.as_ref()).collect(); + Ok(arrow_select::concat::concat(&chunk_refs)?) +} + +#[cfg(test)] +mod tests { + use super::*; + use arrow_array::{Int32Array, StringArray, UInt32Array}; + use lance_table::format::overlay::OverlayCoverage; + use std::sync::Arc; + + fn i32_array(values: impl IntoIterator>) -> ArrayRef { + Arc::new(Int32Array::from_iter(values)) + } + + fn bitmap(offsets: impl IntoIterator) -> RoaringBitmap { + RoaringBitmap::from_iter(offsets) + } + + fn flat_test_schema() -> Schema { + use arrow_schema::{DataType, Field as ArrowField, Schema as ArrowSchema}; + + let mut schema = Schema::try_from(&ArrowSchema::new( + (0..5) + .map(|id| ArrowField::new(format!("field_{id}"), DataType::Int32, true)) + .collect::>(), + )) + .unwrap(); + schema.set_field_id(None); + schema + } + + /// Physical offsets for a contiguous range `[start, start + len)`. + fn offsets(start: u32, len: usize) -> Vec { + (start..start + len as u32).collect() + } + + /// Drive the production flow purely in memory: route against the coverage + /// bitmaps, then fetch just the requested `offset_in_overlay` entries from each + /// overlay's *full* value column (exactly what the value-pushdown `take` does on + /// disk), then assemble. `overlays_newest_first` holds each overlay's + /// `(coverage, full value column indexed by offset_in_overlay)`. + fn resolve( + base: &ArrayRef, + offsets: &[u32], + overlays_newest_first: &[(RoaringBitmap, ArrayRef)], + ) -> ArrayRef { + let coverages: Vec<&RoaringBitmap> = overlays_newest_first.iter().map(|(c, _)| c).collect(); + let routing = route_overlays(offsets, &coverages); + let fetched: Vec = overlays_newest_first + .iter() + .zip(&routing.offsets_in_overlay) + .map(|((_, full), offsets_in_overlay)| { + let indices = UInt32Array::from(offsets_in_overlay.clone()); + arrow_select::take::take(full.as_ref(), &indices, None).unwrap() + }) + .collect(); + assemble_overlay_column(base, &routing, &fetched).unwrap() + } + + fn assert_i32_eq(actual: &ArrayRef, expected: impl IntoIterator>) { + let actual = actual.as_any().downcast_ref::().unwrap(); + assert_eq!(actual, &Int32Array::from_iter(expected)); + } + + #[test] + fn test_no_overlays_returns_base() { + let base = i32_array([Some(1), Some(2), Some(3)]); + let resolved = resolve(&base, &offsets(0, 3), &[]); + assert_i32_eq(&resolved, [Some(1), Some(2), Some(3)]); + } + + #[test] + fn test_single_overlay_value_offset() { + // Base ages [30, 25, 40, 22]; overlay sets offset_in_frag 1 -> 26, whose + // value sits at offset_in_overlay 0. + let base = i32_array([Some(30), Some(25), Some(40), Some(22)]); + let overlay = (bitmap([1]), i32_array([Some(26)])); + let resolved = resolve(&base, &offsets(0, 4), &[overlay]); + assert_i32_eq(&resolved, [Some(30), Some(26), Some(40), Some(22)]); + } + + #[test] + fn test_value_offsets_multiple_cells() { + // Coverage {0, 2, 3} -> values at offset_in_overlay 0, 1, 2. + let base = i32_array([Some(10), Some(11), Some(12), Some(13)]); + let overlay = ( + bitmap([0, 2, 3]), + i32_array([Some(100), Some(120), Some(130)]), + ); + let resolved = resolve(&base, &offsets(0, 4), &[overlay]); + assert_i32_eq(&resolved, [Some(100), Some(11), Some(120), Some(130)]); + } + + #[test] + fn test_newest_overlay_wins() { + // Two overlays both cover offset_in_frag 1; the newest (first in the slice) + // wins. + let base = i32_array([Some(0), Some(1), Some(2)]); + let newest = (bitmap([1]), i32_array([Some(999)])); + let older = (bitmap([1, 2]), i32_array([Some(111), Some(222)])); + let resolved = resolve(&base, &offsets(0, 3), &[newest, older]); + // offset 1 -> newest (999); offset 2 -> only older covers it (222). + assert_i32_eq(&resolved, [Some(0), Some(999), Some(222)]); + } + + #[test] + fn test_null_override_vs_fall_through() { + // A covered offset with a NULL value overrides the cell to NULL; an + // absent offset falls through to the base. + let base = i32_array([Some(1), Some(2), Some(3)]); + let overlay = (bitmap([0]), i32_array([None])); + let resolved = resolve(&base, &offsets(0, 3), &[overlay]); + assert_i32_eq(&resolved, [None, Some(2), Some(3)]); + } + + #[test] + fn test_physical_start_offset() { + // The batch covers physical rows [10, 13); the overlay covers offset 11. + let base = i32_array([Some(0), Some(0), Some(0)]); + let overlay = (bitmap([11]), i32_array([Some(7)])); + let resolved = resolve(&base, &offsets(10, 3), &[overlay]); + assert_i32_eq(&resolved, [Some(0), Some(7), Some(0)]); + } + + #[test] + fn test_string_column_merge() { + let base: ArrayRef = Arc::new(StringArray::from(vec!["a", "b", "c"])); + let overlay = ( + bitmap([0, 2]), + Arc::new(StringArray::from(vec!["A", "C"])) as ArrayRef, + ); + let resolved = resolve(&base, &offsets(0, 3), &[overlay]); + let expected: ArrayRef = Arc::new(StringArray::from(vec!["A", "b", "C"])); + assert_eq!(&resolved, &expected); + } + + #[test] + fn test_non_contiguous_offsets() { + // `take` supplies arbitrary, non-contiguous offsets_in_frag. The base rows + // correspond to offsets 5, 1, 8 (in that order); the overlay covers offsets + // {1, 8}, whose values sit at offset_in_overlay 0, 1. + let base = i32_array([Some(50), Some(10), Some(80)]); + let overlay = (bitmap([1, 8]), i32_array([Some(11), Some(88)])); + let resolved = resolve(&base, &[5, 1, 8], &[overlay]); + // offset 5 uncovered -> base 50; offset 1 -> offset_in_overlay 0 (11); + // offset 8 -> offset_in_overlay 1 (88). + assert_i32_eq(&resolved, [Some(50), Some(11), Some(88)]); + } + + #[test] + fn test_routing_dedups_repeated_offsets() { + // A `take` may request the same offset twice; both rows must route to the + // same overlay value, and that value is fetched only once. + let coverage = bitmap([2, 5]); + let routing = route_overlays(&[5, 2, 5], &[&coverage]); + // offset_in_frag 5 is offset_in_overlay 1, offset_in_frag 2 is + // offset_in_overlay 0: distinct values {0, 1}, sorted. + assert_eq!(routing.offsets_in_overlay, vec![vec![0, 1]]); + let full = i32_array([Some(20), Some(50)]); // values at offset_in_overlay 0, 1 + let fetched = vec![ + arrow_select::take::take( + full.as_ref(), + &UInt32Array::from(routing.offsets_in_overlay[0].clone()), + None, + ) + .unwrap(), + ]; + let base = i32_array([Some(0), Some(0), Some(0)]); + let resolved = assemble_overlay_column(&base, &routing, &fetched).unwrap(); + assert_i32_eq(&resolved, [Some(50), Some(20), Some(50)]); + } + + #[test] + fn test_assemble_value_count_mismatch_errors() { + let coverage = bitmap([0, 1]); + let routing = route_overlays(&[0, 1], &[&coverage]); + let base = i32_array([Some(1), Some(2)]); + // One value supplied for two requested offsets is a caller bug. + let fetched = vec![i32_array([Some(9)])]; + assert!(assemble_overlay_column(&base, &routing, &fetched).is_err()); + } + + #[test] + fn test_contiguous_fast_path_matches_general() { + // The contiguous fast path must produce byte-for-byte identical routing to + // the general offset-major path for any contiguous batch. Fuzz a range of + // fragment starts, lengths, overlay counts, and coverage densities — + // including bits outside the batch range — and compare both paths. + let mut state = 0x9e3779b97f4a7c15u64; + let mut next = || { + state = state + .wrapping_mul(6364136223846793005) + .wrapping_add(1442695040888963407); + (state >> 33) as u32 + }; + for _ in 0..500 { + let frag_start = next() % 64; + let len = (next() % 48 + 1) as usize; + let num_overlays = (next() % 5) as usize; + let coverages: Vec = (0..num_overlays) + .map(|_| { + let density = next() % 101; + let mut b = RoaringBitmap::new(); + for off in frag_start.saturating_sub(3)..frag_start + len as u32 + 3 { + if next() % 100 < density { + b.insert(off); + } + } + b + }) + .collect(); + let refs: Vec<&RoaringBitmap> = coverages.iter().collect(); + let contiguous_offsets: Vec = (frag_start..frag_start + len as u32).collect(); + + let fast = route_contiguous(frag_start, len, &refs); + let general = route_arbitrary(&contiguous_offsets, &refs); + assert_eq!(fast.indices, general.indices, "indices differ"); + assert_eq!( + fast.offsets_in_overlay, general.offsets_in_overlay, + "offsets_in_overlay differ" + ); + assert_eq!(fast.any_overlay, general.any_overlay, "any_overlay differs"); + } + } + + /// `outer { middle { a, b } }` for exercising the descend/splice helpers. + fn nested_struct() -> (Schema, ArrayRef) { + use arrow_schema::{DataType, Field as ArrowField, Fields, Schema as ArrowSchema}; + let mid = Fields::from(vec![ + ArrowField::new("a", DataType::Int32, true), + ArrowField::new("b", DataType::Int32, true), + ]); + let outer_fields = + Fields::from(vec![ArrowField::new("middle", DataType::Struct(mid), true)]); + let arrow_schema = ArrowSchema::new(vec![ArrowField::new( + "outer", + DataType::Struct(outer_fields), + true, + )]); + let mut schema = Schema::try_from(&arrow_schema).unwrap(); + schema.set_field_id(None); + let middle = StructArray::from(vec![ + ( + Arc::new(ArrowField::new("a", DataType::Int32, true)), + i32_array([Some(1), Some(2), Some(3)]), + ), + ( + Arc::new(ArrowField::new("b", DataType::Int32, true)), + i32_array([Some(10), Some(20), Some(30)]), + ), + ]); + let outer: ArrayRef = Arc::new(StructArray::from(vec![( + Arc::new(ArrowField::new("middle", middle.data_type().clone(), true)), + Arc::new(middle) as ArrayRef, + )])); + (schema, outer) + } + + #[test] + fn test_descend_and_splice_roundtrip() { + let (schema, outer_arr) = nested_struct(); + let outer_field = &schema.fields[0]; + let middle_id = outer_field.children[0].id; + let a_id = outer_field.children[0].children[0].id; + let path = [middle_id, a_id]; + + // Descend to the deep leaf `outer.middle.a`. + let a = descend_by_ids(&outer_arr, outer_field, &path).unwrap(); + assert_i32_eq(&a, [Some(1), Some(2), Some(3)]); + + // Splice a replacement in; only `a` changes, `b` is preserved. + let spliced = splice_by_ids( + &outer_arr, + outer_field, + &path, + i32_array([Some(7), Some(8), Some(9)]), + ) + .unwrap(); + let middle = spliced + .as_any() + .downcast_ref::() + .unwrap() + .column(0) + .as_any() + .downcast_ref::() + .unwrap() + .clone(); + assert_i32_eq(&middle.column(0).clone(), [Some(7), Some(8), Some(9)]); + assert_i32_eq(&middle.column(1).clone(), [Some(10), Some(20), Some(30)]); + } + + #[test] + fn test_splice_preserves_struct_nulls() { + use arrow_buffer::NullBuffer; + let (schema, base) = nested_struct(); + let outer_field = &schema.fields[0]; + // Rebuild `outer` with a null at row 1 (a null struct value). + let base = base.as_any().downcast_ref::().unwrap(); + let (fields, children, _) = base.clone().into_parts(); + let outer_arr: ArrayRef = Arc::new( + StructArray::try_new( + fields, + children, + Some(NullBuffer::from(vec![true, false, true])), + ) + .unwrap(), + ); + let path = [ + outer_field.children[0].id, + outer_field.children[0].children[0].id, + ]; + let spliced = splice_by_ids( + &outer_arr, + outer_field, + &path, + i32_array([Some(7), Some(8), Some(9)]), + ) + .unwrap(); + let spliced = spliced.as_any().downcast_ref::().unwrap(); + // The outer struct's null buffer survives the splice. + assert!(!spliced.is_null(0)); + assert!(spliced.is_null(1)); + assert!(!spliced.is_null(2)); + } + + /// A dense overlay covering `offsets` for `field_ids`, committed at `version`. + fn dense_overlay( + field_ids: Vec, + offsets: impl IntoIterator, + version: u64, + ) -> lance_table::format::overlay::DataOverlayFile { + DataOverlayFile { + data_file: DataFile::new_legacy_from_fields("o.lance", field_ids, None), + coverage: OverlayCoverage::dense(bitmap(offsets)), + committed_version: version, + } + } + + #[test] + fn test_exclusion_offsets_version_gate() { + let schema = flat_test_schema(); + // index built at version 5; only overlays committed > 5 are excluded. + let overlays = vec![ + dense_overlay(vec![3], [0, 1], 4), + dense_overlay(vec![3], [2, 7], 6), + ]; + let excluded = overlay_exclusion_offsets(&overlays, &[3], 5, &schema).unwrap(); + assert_eq!(excluded, bitmap([2, 7])); + // An overlay exactly at the index version is already incorporated. + let overlays = vec![dense_overlay(vec![3], [9], 5)]; + assert!( + overlay_exclusion_offsets(&overlays, &[3], 5, &schema) + .unwrap() + .is_empty() + ); + } + + #[test] + fn test_exclusion_offsets_is_field_aware() { + let schema = flat_test_schema(); + // An overlay touching only an unrelated field excludes nothing. + let overlays = vec![dense_overlay(vec![2], [0, 1, 2], 9)]; + assert!( + overlay_exclusion_offsets(&overlays, &[3], 1, &schema) + .unwrap() + .is_empty() + ); + // The union spans only the indexed fields the overlay actually carries. + let overlays = vec![dense_overlay(vec![2, 3], [4], 9)]; + assert_eq!( + overlay_exclusion_offsets(&overlays, &[3], 1, &schema).unwrap(), + bitmap([4]) + ); + } + + #[test] + fn test_exclusion_offsets_matches_nested_fields() { + let (schema, _) = nested_struct(); + let outer = &schema.fields[0]; + let middle = &outer.children[0]; + let a = &middle.children[0]; + let b = &middle.children[1]; + + let overlays = vec![dense_overlay(vec![a.id], [1], 9)]; + assert_eq!( + overlay_exclusion_offsets(&overlays, &[outer.id], 1, &schema).unwrap(), + bitmap([1]) + ); + assert!( + overlay_exclusion_offsets(&overlays, &[b.id], 1, &schema) + .unwrap() + .is_empty() + ); + + let overlays = vec![dense_overlay(vec![middle.id], [2], 9)]; + assert_eq!( + overlay_exclusion_offsets(&overlays, &[a.id], 1, &schema).unwrap(), + bitmap([2]) + ); + } + + #[test] + fn test_exclusion_offsets_sparse_per_field() { + let schema = flat_test_schema(); + // Sparse overlay: field 2 covers {2,3}, field 4 covers {1}. + let overlay = DataOverlayFile { + data_file: DataFile::new_legacy_from_fields("o.lance", vec![2, 4], None), + coverage: OverlayCoverage::sparse(vec![bitmap([2, 3]), bitmap([1])]), + committed_version: 9, + }; + let overlays = vec![overlay]; + // Only the bitmap for the indexed field (4) contributes. + assert_eq!( + overlay_exclusion_offsets(&overlays, &[4], 1, &schema).unwrap(), + bitmap([1]) + ); + assert_eq!( + overlay_exclusion_offsets(&overlays, &[2], 1, &schema).unwrap(), + bitmap([2, 3]) + ); + } + + #[test] + fn test_exclusion_offsets_unions_multiple_overlays() { + let schema = flat_test_schema(); + let overlays = vec![ + dense_overlay(vec![3], [1], 6), + dense_overlay(vec![3], [4, 5], 7), + ]; + assert_eq!( + overlay_exclusion_offsets(&overlays, &[3], 1, &schema).unwrap(), + bitmap([1, 4, 5]) + ); + } + + /// An index segment covering `fields`, built at `dataset_version`, with the given + /// fragment coverage (`None` = legacy index predating fragment-bitmap tracking). + fn segment( + fields: Vec, + dataset_version: u64, + fragment_bitmap: Option, + ) -> IndexMetadata { + IndexMetadata { + uuid: uuid::Uuid::new_v4(), + name: "idx".into(), + fields, + dataset_version, + fragment_bitmap, + index_details: None, + index_version: 0, + created_at: None, + base_id: None, + files: None, + } + } + + fn fragment_with_overlay(id: u64, overlay: DataOverlayFile) -> Fragment { + let mut fragment = Fragment::new(id); + fragment.overlays.push(overlay); + fragment + } + + #[test] + fn test_collect_frags_missing_bitmap_covers_all() { + let schema = flat_test_schema(); + let fragment = fragment_with_overlay(3, dense_overlay(vec![3], [1, 2], 9)); + let overlaid: HashMap = HashMap::from([(3u32, &fragment)]); + + let mut stale = RoaringBitmap::new(); + collect_overlay_stale_frags(&segment(vec![3], 1, None), &overlaid, &mut stale, &schema) + .unwrap(); + assert_eq!(stale, bitmap([3]), "missing bitmap must cover fragment 3"); + + let mut stale = RoaringBitmap::new(); + collect_overlay_stale_frags( + &segment(vec![3], 1, Some(bitmap([0]))), + &overlaid, + &mut stale, + &schema, + ) + .unwrap(); + assert!( + stale.is_empty(), + "fragment absent from bitmap is not covered" + ); + + let mut stale = RoaringBitmap::new(); + collect_overlay_stale_frags( + &segment(vec![3], 1, Some(bitmap([3]))), + &overlaid, + &mut stale, + &schema, + ) + .unwrap(); + assert_eq!(stale, bitmap([3])); + } + + #[test] + fn test_collect_rows_missing_bitmap_covers_all() { + let schema = flat_test_schema(); + // Same covers-all guarantee at row-level granularity. + let fragment = fragment_with_overlay(3, dense_overlay(vec![3], [1, 2], 9)); + let overlaid: HashMap = HashMap::from([(3u32, &fragment)]); + + let mut stale = HashMap::new(); + collect_overlay_stale_rows_for_segment( + &segment(vec![3], 1, None), + &overlaid, + &mut stale, + &schema, + ) + .unwrap(); + assert_eq!( + stale.get(&3), + Some(&bitmap([1, 2])), + "missing bitmap must cover fragment 3" + ); + + // A present bitmap that excludes fragment 3 yields no stale rows. + let mut stale = HashMap::new(); + collect_overlay_stale_rows_for_segment( + &segment(vec![3], 1, Some(bitmap([0]))), + &overlaid, + &mut stale, + &schema, + ) + .unwrap(); + assert!( + stale.is_empty(), + "fragment absent from bitmap contributes no rows" + ); + + // A present bitmap that includes fragment 3 contributes its stale rows. + let mut stale = HashMap::new(); + collect_overlay_stale_rows_for_segment( + &segment(vec![3], 1, Some(bitmap([3]))), + &overlaid, + &mut stale, + &schema, + ) + .unwrap(); + assert_eq!(stale.get(&3), Some(&bitmap([1, 2]))); + } +} diff --git a/rust/lance/src/dataset/refs.rs b/rust/lance/src/dataset/refs.rs index 98b4f0cbc0a..cff86638e6a 100644 --- a/rust/lance/src/dataset/refs.rs +++ b/rust/lance/src/dataset/refs.rs @@ -21,7 +21,6 @@ use std::cmp::Ordering; use std::collections::HashMap; use std::fmt; use std::fmt::Formatter; -use std::io::ErrorKind; use uuid::Uuid; pub const MAIN_BRANCH: &str = "main"; @@ -476,7 +475,7 @@ impl Branches<'_> { if !self.object_store().exists(&manifest_file.path).await? { return Err(Error::VersionNotFound { - message: format!("Manifest file {} does not exist", &manifest_file.path), + message: format!("Manifest file {} does not exist", manifest_file.path), }); }; @@ -574,6 +573,28 @@ impl Branches<'_> { log::warn!("BranchContents of {} does not exist", branch); } + // Tags identify snapshots by (branch, version). Deleting a branch removes its entire version chain, + // so any tag whose branch matches the deletion target blocks the operation, regardless of the tagged + // version. + let referenced_tags = self + .refs + .tags() + .fetch_tags() + .await? + .into_iter() + .filter_map(|(name, contents)| { + (contents.branch.as_deref() == Some(branch)).then_some((name, contents.version)) + }) + .collect_vec(); + if !referenced_tags.is_empty() { + return Err(Error::RefConflict { + message: format!( + "Branch {} is referenced by tags {:?}, can not delete", + branch, referenced_tags + ), + }); + } + let root_location = self.refs.root()?; let branch_file = branch_contents_path(&root_location.path, branch); if self.object_store().exists(&branch_file).await? { @@ -611,16 +632,8 @@ impl Branches<'_> { && let Err(e) = self.refs.object_store.remove_dir_all(delete_path).await { match &e { - Error::IO { source, .. } => { - if let Some(io_err) = source.downcast_ref::() { - if io_err.kind() == ErrorKind::NotFound { - log::debug!("Branch directory already deleted: {}", io_err); - } else { - return Err(e); - } - } else { - return Err(e); - } + Error::NotFound { .. } => { + log::debug!("Branch directory already deleted"); } _ => return Err(e), } diff --git a/rust/lance/src/dataset/rowids.rs b/rust/lance/src/dataset/rowids.rs index c57d75386db..74459fc96e7 100644 --- a/rust/lance/src/dataset/rowids.rs +++ b/rust/lance/src/dataset/rowids.rs @@ -1,40 +1,46 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright The Lance Authors +mod validate; + use super::Dataset; use crate::session::caches::{RowIdIndexKey, RowIdSequenceKey}; use crate::{Error, Result}; use futures::{Stream, StreamExt, TryFutureExt, TryStreamExt}; -use lance_core::utils::deletion::DeletionVector; +use lance_core::utils::{address::RowAddress, deletion::DeletionVector}; +use lance_select::{RowAddrSelection, RowAddrTreeMap}; use lance_table::{ format::{Fragment, RowIdMeta}, rowids::{FragmentRowIdIndex, RowIdIndex, RowIdSequence, read_row_ids}, }; use std::sync::Arc; +pub(super) use validate::validate_stable_row_ids; + /// Load a row id sequence from the given dataset and fragment. pub async fn load_row_id_sequence( dataset: &Dataset, fragment: &Fragment, ) -> Result> { - // Virtual path to prevent collisions in the cache. match &fragment.row_id_meta { None => Err(Error::internal("Missing row id meta")), - Some(RowIdMeta::Inline(data)) => { + Some(row_id_meta @ RowIdMeta::Inline(data)) => { let data = data.clone(); let key = RowIdSequenceKey { fragment_id: fragment.id, + row_id_meta, }; dataset .metadata_cache .get_or_insert_with_key(key, || async move { read_row_ids(&data) }) .await } - Some(RowIdMeta::External(file_slice)) => { + Some(row_id_meta @ RowIdMeta::External(file_slice)) => { let file_slice = file_slice.clone(); let dataset_clone = dataset.clone(); let key = RowIdSequenceKey { fragment_id: fragment.id, + row_id_meta, }; dataset .metadata_cache @@ -87,6 +93,146 @@ pub async fn get_row_id_index( } } +/// Map a set of physical row addresses to their stable row ids +/// +/// A fragment's [`RowIdSequence`] holds one id per physical row in offset order; +/// deletions are tracked by the deletion vector and do not compact the sequence, +/// so a physical offset indexes the sequence directly (see [`RowIdIndex`], which +/// maps ids to `start_address + position` and filters deletions separately). +/// Addresses that no longer have a live counterpart are dropped, which is correct: +/// those rows are not part of the answer. That covers both a deleted row and a +/// whole fragment that a maintenance operation replaced — an address-domain index +/// is not remapped by compaction or `update` (neither the zone map nor the bloom +/// filter index supports remap), so its results outlive the fragments they address. +/// The replacement fragments are absent from the index's fragment bitmap, so the +/// scanner routes them to a full scan and no match is lost. +pub(crate) async fn translate_addr_treemap_to_row_ids( + dataset: &Dataset, + addrs: &RowAddrTreeMap, +) -> Result { + let mut row_ids = RowAddrTreeMap::new(); + for (fragment_id, selection) in addrs.iter() { + let Some(file_fragment) = dataset.get_fragment(*fragment_id as usize) else { + continue; + }; + let sequence = load_row_id_sequence(dataset, file_fragment.metadata()).await?; + + match selection { + RowAddrSelection::Full => { + // The whole fragment is selected: every live row's id qualifies. + row_ids |= RowAddrTreeMap::from(sequence.as_ref()); + } + RowAddrSelection::Partial(offsets) => { + let deletion_vector = file_fragment.get_deletion_vector().await?; + for physical_offset in offsets.iter() { + // A deletion does not compact the row id sequence; the deleted row keeps + // its slot (the deletion vector tracks it separately). So a physical offset + // is a direct index into the sequence, regardless of any deletions below it. + // A stale offset that points at a deleted row has no live counterpart and is + // not part of any answer, so it contributes no id to the block/take set. + let deleted = deletion_vector + .as_ref() + .is_some_and(|dv| dv.contains(physical_offset)); + if deleted { + continue; + } + // A selected offset with no sequence entry points past the fragment's rows. + // Silently dropping it would let a stale index result escape masking, so + // treat the mismatch as corruption. + let id = sequence.get(physical_offset as usize).ok_or_else(|| { + Error::internal(format!( + "fragment_id={fragment_id} row-id sequence has no entry at \ + physical_offset={physical_offset} (sequence len={})", + sequence.len() + )) + })?; + row_ids.insert(id); + } + } + } + } + Ok(row_ids) +} + +/// Resolve row addresses to row ids, positionally: `addr = (fragment << 32) | offset` +/// looks up `offset` in the fragment's [`RowIdSequence`]. The inverse companion of +/// [`get_row_id_index`]. +/// +/// Returns one entry per input address, in order. Addresses that no longer +/// resolve — a missing fragment, an out-of-range offset, or a `None` input — +/// yield `None`. On datasets without stable row ids, addresses are the row +/// ids, so the input is returned unchanged. +pub async fn row_addrs_to_row_ids( + dataset: &Dataset, + addrs: impl IntoIterator>, +) -> Result>> { + row_addrs_to_row_ids_impl(dataset, addrs, DeletedRowBehavior::Include).await +} + +/// Resolve physical row addresses to stable row ids only for live physical slots. +/// +/// When stable row ids are enabled, missing fragments, out-of-range offsets, +/// deleted physical slots, and `None` inputs yield `None`. This prevents a +/// deleted slot's stable id from selecting a replacement row written by an +/// update. Without stable row ids, the input is returned unchanged. +pub(crate) async fn live_row_addrs_to_row_ids( + dataset: &Dataset, + addrs: impl IntoIterator>, +) -> Result>> { + row_addrs_to_row_ids_impl(dataset, addrs, DeletedRowBehavior::Exclude).await +} + +#[derive(Clone, Copy)] +enum DeletedRowBehavior { + Include, + Exclude, +} + +async fn row_addrs_to_row_ids_impl( + dataset: &Dataset, + addrs: impl IntoIterator>, + deleted_row_behavior: DeletedRowBehavior, +) -> Result>> { + let addrs: Vec> = addrs.into_iter().collect(); + if !dataset.manifest.uses_stable_row_ids() { + return Ok(addrs); + } + + let mut positions_by_fragment: std::collections::HashMap> = + std::collections::HashMap::new(); + for (position, addr) in addrs.iter().enumerate() { + if let Some(addr) = addr { + positions_by_fragment + .entry(RowAddress::from(*addr).fragment_id()) + .or_default() + .push(position); + } + } + + let mut ids: Vec> = vec![None; addrs.len()]; + for (fragment_id, positions) in positions_by_fragment { + let Some(fragment) = dataset.get_fragment(fragment_id as usize) else { + continue; + }; + let sequence = load_row_id_sequence(dataset, fragment.metadata()).await?; + let deletion_vector = match deleted_row_behavior { + DeletedRowBehavior::Include => None, + DeletedRowBehavior::Exclude => fragment.get_deletion_vector().await?, + }; + for position in positions { + let offset = RowAddress::from(addrs[position].expect("grouped from Some")).row_offset(); + if deletion_vector + .as_ref() + .is_some_and(|deletions| deletions.contains(offset)) + { + continue; + } + ids[position] = sequence.get(offset as usize); + } + } + Ok(ids) +} + async fn load_row_id_index(dataset: &Dataset) -> Result { let sequences = load_row_id_sequences(dataset, &dataset.manifest.fragments) .try_collect::>() @@ -105,10 +251,14 @@ async fn load_row_id_index(dataset: &Dataset) -> Result dv, - Ok(None) | Err(_) => Arc::new(DeletionVector::default()), - } + fragment_clone + .get_deletion_vector() + .await? + .ok_or_else(|| { + Error::internal(format!( + "fragment_id={fragment_id} has deletion-file metadata but no deletion vector" + )) + })? } else { Arc::new(DeletionVector::default()) }; @@ -133,13 +283,16 @@ async fn load_row_id_index(dataset: &Dataset) -> Result| { + let batch = sequence_batch(rows); + let reader = RecordBatchIterator::new(vec![Ok(batch.clone())], batch.schema()); + let params = WriteParams { + enable_stable_row_ids: true, + session: Some(session.clone()), + ..Default::default() + }; + async move { + Dataset::write(reader, tmp_path, Some(params)) + .await + .unwrap() + } + }; + + let dataset = write(0..100).await; + let sequence = load_row_id_sequence(&dataset, &dataset.manifest.fragments[0]) + .await + .unwrap(); + assert_eq!(sequence.len(), 100); + + // Reloading the unchanged fragment must still hit: keying on contents has + // to leave the sequence cacheable, not just make it distinguishable. + let hits_before = session.metadata_cache_stats().await.hits; + load_row_id_sequence(&dataset, &dataset.manifest.fragments[0]) + .await + .unwrap(); + assert_eq!(session.metadata_cache_stats().await.hits, hits_before + 1); + + drop(dataset); + std::fs::remove_dir_all(tmp_path.as_str()).unwrap(); + + // Shorter than the dataset it replaces, so a stale hit is observable: an + // equal-length sequence would be byte-identical and harmless. + let dataset = write(0..60).await; + assert_eq!(dataset.manifest.fragments[0].id, 0); + + let sequence = load_row_id_sequence(&dataset, &dataset.manifest.fragments[0]) + .await + .unwrap(); + assert_eq!( + sequence.iter().collect::>(), + (0..60).collect::>() + ); + } + + #[tokio::test] + async fn test_compaction_after_recreate() { + // Compaction rechunks the row id sequences of the fragments it merges, so + // a sequence cached for a dropped dataset does not just misreport ids, it + // writes ids the fragments do not hold back into the manifest. + let temp_dir = lance_core::utils::tempfile::TempStrDir::default(); + let tmp_path = &temp_dir; + let session = Arc::new(Session::default()); + let params = WriteParams { + enable_stable_row_ids: true, + session: Some(session.clone()), + ..Default::default() + }; + + let write = |rows: Range, mode: WriteMode| { + let batch = sequence_batch(rows); + let reader = RecordBatchIterator::new(vec![Ok(batch.clone())], batch.schema()); + let params = WriteParams { + mode, + ..params.clone() + }; + async move { + Dataset::write(reader, tmp_path, Some(params)) + .await + .unwrap() + } + }; + + let dataset = write(0..100, WriteMode::Create).await; + load_row_id_sequence(&dataset, &dataset.manifest.fragments[0]) + .await + .unwrap(); + + drop(dataset); + std::fs::remove_dir_all(tmp_path.as_str()).unwrap(); + + // The recreated dataset holds 60 rows in fragment 0, not the 100 cached + // above, and a second fragment so compaction has something to merge. + write(0..60, WriteMode::Create).await; + let mut dataset = write(0..100, WriteMode::Append).await; + + compact(&mut dataset, 1024).await; + + assert_eq!(dataset.count_rows(None).await.unwrap(), 160); + let mut scan = dataset.scan(); + scan.with_row_id(); + let batch = scan.try_into_batch().await.unwrap(); + let row_ids = batch[ROW_ID] + .as_primitive::() + .values() + .iter() + .copied() + .collect::>(); + // A stale 100-long sequence would rechunk 0..100 onto the 60 rows of + // fragment 0 and shift everything after it. + assert_eq!(row_ids, (0..160).collect::>()); + } + #[tokio::test] async fn test_row_ids_append() { // Validate we handle row ids well when appending concurrently. @@ -467,6 +837,53 @@ mod test { assert_eq!(index.get(5), None); } + /// 100 sequential rows across 4 fragments with every third row deleted. + async fn deleted_thirds_dataset(enable_stable_row_ids: bool) -> Dataset { + let batch = sequence_batch(0..100); + let reader = RecordBatchIterator::new(vec![Ok(batch.clone())], batch.schema()); + let write_params = WriteParams { + enable_stable_row_ids, + max_rows_per_file: 25, + ..Default::default() + }; + let mut dataset = Dataset::write(reader, "memory://", Some(write_params)) + .await + .unwrap(); + dataset.delete("id % 3 = 0").await.unwrap(); + dataset + } + + #[rstest::rstest] + #[case::interleaved((0..100).collect(), (0..100).filter(|id| id % 3 != 0).collect())] + #[case::all_deleted(vec![0, 3, 9], vec![])] + #[case::all_live(vec![1, 2, 50, 98], vec![1, 2, 50, 98])] + #[tokio::test] + async fn test_filter_deleted_ids_with_stable_row_ids( + #[case] ids: Vec, + #[case] expected: Vec, + ) { + // Regression test for https://github.com/lance-format/lance/issues/7701: + // filter_deleted_ids must return exactly the live ids, in order. + // Sequential inserts, so stable row id == id column value. + let dataset = deleted_thirds_dataset(true).await; + assert_eq!(dataset.filter_deleted_ids(&ids).await.unwrap(), expected); + } + + #[tokio::test] + async fn test_filter_deleted_ids_without_stable_row_ids() { + // Non-stable: ids are row addresses; only deleted rows drop out. + let dataset = deleted_thirds_dataset(false).await; + + // Row addresses: fragment (id / 25) << 32 | offset (id % 25). + let addr = |id: u64| (id / 25) << 32 | (id % 25); + let ids = (0..100).map(addr).collect::>(); + let expected = (0..100) + .filter(|id| id % 3 != 0) + .map(addr) + .collect::>(); + assert_eq!(dataset.filter_deleted_ids(&ids).await.unwrap(), expected); + } + fn build_rowid_to_i_map(row_ids: &UInt64Array, i_array: &Int32Array) -> HashMap { row_ids .values() @@ -489,7 +906,7 @@ mod test { build_rowid_to_i_map(row_ids, i) } - async fn compact(dataset: &mut Dataset, target_rows: usize) { + pub(super) async fn compact(dataset: &mut Dataset, target_rows: usize) { let options = CompactionOptions { target_rows_per_fragment: target_rows, ..Default::default() @@ -497,7 +914,7 @@ mod test { let _ = compact_files(dataset, options, None).await.unwrap(); } - async fn delete(dataset: &mut Dataset, expr: &str) { + pub(super) async fn delete(dataset: &mut Dataset, expr: &str) { dataset.delete(expr).await.unwrap(); } diff --git a/rust/lance/src/dataset/rowids/validate.rs b/rust/lance/src/dataset/rowids/validate.rs new file mode 100644 index 00000000000..27bed6491ac --- /dev/null +++ b/rust/lance/src/dataset/rowids/validate.rs @@ -0,0 +1,460 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright The Lance Authors + +//! Integrity checks for the stable row id invariants that the row id index and the write +//! paths rely on. Reached through [`Dataset::validate`]. + +use super::load_row_id_sequence; +use crate::dataset::Dataset; +use crate::dataset::fragment::FileFragment; +use crate::{Error, Result}; +use futures::{StreamExt, TryStreamExt}; +use lance_core::utils::deletion::DeletionVector; +use lance_table::format::RowDatasetVersionMeta; +use lance_table::rowids::RowIdSequence; +use roaring::RoaringTreemap; +use std::sync::Arc; + +/// The per-fragment state a stable row id invariant is checked against. +type FragmentRowIds = ( + FileFragment, + Arc, + Option>, +); + +/// Check the invariants that the stable row id machinery relies on. +/// +/// A no-op on datasets that don't use stable row ids. See [`Dataset::validate`]. +pub async fn validate_stable_row_ids(dataset: &Dataset) -> Result<()> { + if !dataset.manifest.uses_stable_row_ids() { + return Ok(()); + } + + let corrupt = |message: String| Error::corrupt_file(dataset.base.clone(), message); + + for fragment in dataset.manifest.fragments.iter() { + if fragment.row_id_meta.is_none() { + return Err(corrupt(format!( + "Fragment {} has no row id metadata, but dataset {:?} uses stable row ids", + fragment.id, dataset.base + ))); + } + } + + // `buffered` preserves manifest order, which the uniqueness check below relies on to + // report the earlier of the two fragments holding a duplicate id. + let fragments: Vec = futures::stream::iter(dataset.get_fragments()) + .map(|fragment| async move { + let sequence = load_row_id_sequence(dataset, fragment.metadata()).await?; + let deletion_vector = fragment.get_deletion_vector().await?; + Result::Ok((fragment, sequence, deletion_vector)) + }) + .buffered(dataset.object_store.io_parallelism()) + .try_collect() + .await?; + + let mut live_row_ids = RoaringTreemap::new(); + let mut max_row_id: Option = None; + + for (index, (fragment, sequence, deletion_vector)) in fragments.iter().enumerate() { + let metadata = fragment.metadata(); + let physical_rows = metadata.physical_rows.ok_or_else(|| { + corrupt(format!( + "Fragment {} has an unknown physical row count, but dataset {:?} uses stable row ids", + metadata.id, dataset.base + )) + })? as u64; + + if sequence.len() != physical_rows { + return Err(corrupt(format!( + "Fragment {} has {} row ids, but {} physical rows, in dataset {:?}", + metadata.id, + sequence.len(), + physical_rows, + dataset.base + ))); + } + + for (name, meta) in [ + ("created_at", &metadata.created_at_version_meta), + ("last_updated_at", &metadata.last_updated_at_version_meta), + ] { + // Only inline version metadata can be read back; nothing writes the + // external form yet. + if let Some(meta @ RowDatasetVersionMeta::Inline(_)) = meta { + let versions = meta.load_sequence()?.len(); + if versions != physical_rows { + return Err(corrupt(format!( + "Fragment {} has {} {} versions, but {} physical rows, in dataset {:?}", + metadata.id, versions, name, physical_rows, dataset.base + ))); + } + } + } + + // The bounding range covers tombstoned slots too: their ids are retired and must + // not be handed out again either. + if let Some(range) = sequence.row_id_range() { + max_row_id = max_row_id.max(Some(*range.end())); + } + + for (offset, row_id) in sequence.iter().enumerate() { + // An update rewrites a row into a new fragment under the same id and tombstones + // the original slot, so only live ids are required to be unique. + if deletion_vector + .as_ref() + .is_some_and(|deletions| deletions.contains(offset as u32)) + { + continue; + } + if !live_row_ids.insert(row_id) { + return Err(corrupt(format!( + "Row id {} is live in both {} and fragment {} at offset {} in dataset {:?}", + row_id, + describe_first_live_slot(&fragments[..=index], row_id), + metadata.id, + offset, + dataset.base + ))); + } + } + } + + if let Some(max_row_id) = max_row_id + && dataset.manifest.next_row_id <= max_row_id + { + return Err(corrupt(format!( + "Dataset {:?} will hand out row id {} next, but row id {} is already in use", + dataset.base, dataset.manifest.next_row_id, max_row_id + ))); + } + + Ok(()) +} + +/// Describe where `row_id` is first live, so a uniqueness failure can name both of the +/// slots involved. Only runs on the failure path, hence the linear rescan. +fn describe_first_live_slot(fragments: &[FragmentRowIds], row_id: u64) -> String { + for (fragment, sequence, deletion_vector) in fragments { + // A bounding range that excludes the id rules the whole fragment out. + if sequence + .row_id_range() + .is_none_or(|range| !range.contains(&row_id)) + { + continue; + } + for (offset, candidate) in sequence.iter().enumerate() { + if candidate == row_id + && !deletion_vector + .as_ref() + .is_some_and(|deletions| deletions.contains(offset as u32)) + { + return format!("fragment {} at offset {}", fragment.id(), offset); + } + } + } + "an earlier fragment".to_string() +} + +#[cfg(test)] +mod tests { + use super::*; + + // Shared with the row id tests next door, which cover the same operations. + use super::super::test::{compact, delete}; + + use crate::dataset::builder::DatasetBuilder; + use crate::dataset::{ + MergeInsertBuilder, UpdateBuilder, WhenMatched, WhenNotMatched, WriteMode, WriteParams, + }; + use crate::utils::test::{DatagenExt, FragmentCount, FragmentRowCount}; + use arrow_array::RecordBatchIterator; + use arrow_array::types::Int32Type; + use arrow_schema::Schema as ArrowSchema; + use lance_table::format::{Fragment, RowIdMeta, pb}; + use lance_table::rowids::version::RowDatasetVersionSequence; + use lance_table::rowids::write_row_ids; + use prost::Message; + use rstest::rstest; + + fn encode_segments(segments: Vec) -> Vec { + pb::RowIdSequence { + segments: segments + .into_iter() + .map(|segment| pb::U64Segment { + segment: Some(segment), + }) + .collect(), + } + .encode_to_vec() + } + + fn range_segment(start: u64, end: u64) -> pb::u64_segment::Segment { + pb::u64_segment::Segment::Range(pb::u64_segment::Range { start, end }) + } + + fn empty_encoded_array() -> pb::EncodedU64Array { + pb::EncodedU64Array { + array: Some(pb::encoded_u64_array::Array::U64Array( + pb::encoded_u64_array::U64Array { values: Vec::new() }, + )), + } + } + + fn empty_array_segment() -> pb::u64_segment::Segment { + pb::u64_segment::Segment::Array(empty_encoded_array()) + } + + fn empty_sorted_array_segment() -> pb::u64_segment::Segment { + pb::u64_segment::Segment::SortedArray(empty_encoded_array()) + } + + fn empty_range_with_holes_segment() -> pb::u64_segment::Segment { + pb::u64_segment::Segment::RangeWithHoles(pb::u64_segment::RangeWithHoles { + start: 0, + end: 0, + holes: Some(empty_encoded_array()), + }) + } + + fn empty_range_with_bitmap_segment() -> pb::u64_segment::Segment { + pb::u64_segment::Segment::RangeWithBitmap(pb::u64_segment::RangeWithBitmap { + start: 0, + end: 0, + bitmap: Vec::new(), + }) + } + + /// Two `Range` segments that each fit a `usize` but whose lengths sum past `u64`. + /// Decoding accepts them, so `validate()` must report corruption rather than + /// overflowing while measuring the sequence. + #[tokio::test] + async fn test_validate_rejects_row_id_count_overflow() { + let temp_dir = lance_core::utils::tempfile::TempStrDir::default(); + let mut dataset = validation_fixture(&temp_dir).await; + let encoded = encode_segments(vec![range_segment(0, u64::MAX), range_segment(0, u64::MAX)]); + edit_fragments(&mut dataset, |fragments| { + fragments[1].row_id_meta = Some(RowIdMeta::Inline(encoded.into())); + }); + + assert_invalid(&dataset, "total length exceeding u64::MAX").await; + } + + /// An empty segment carries no minimum or maximum, and decoding accepts an empty + /// encoding of every variant. Neither the cardinality check nor the `next_row_id` + /// bound may unwind on one. + #[rstest] + #[case::array(empty_array_segment())] + #[case::sorted_array(empty_sorted_array_segment())] + #[case::range_with_holes(empty_range_with_holes_segment())] + #[case::range_with_bitmap(empty_range_with_bitmap_segment())] + #[tokio::test] + async fn test_validate_tolerates_empty_segment(#[case] empty: pb::u64_segment::Segment) { + let temp_dir = lance_core::utils::tempfile::TempStrDir::default(); + let mut dataset = validation_fixture(&temp_dir).await; + // Ten ids across two segments matches the fragment's ten physical rows, and the + // ids stay clear of fragment 0's, so this dataset is well-formed. + let encoded = encode_segments(vec![range_segment(10, 20), empty]); + edit_fragments(&mut dataset, |fragments| { + fragments[1].row_id_meta = Some(RowIdMeta::Inline(encoded.into())); + }); + + dataset.validate().await.unwrap(); + } + + /// Write a two-fragment dataset with stable row ids to `uri`, then reopen it with a + /// fresh session so the row id sequence cache is empty. Tests that doctor a + /// fragment's `row_id_meta` need that: the cache is keyed by fragment id alone, so a + /// sequence loaded before the mutation would shadow the doctored one. + async fn validation_fixture(uri: &str) -> Dataset { + lance_datagen::gen_batch() + .col("i", lance_datagen::array::step::()) + .into_dataset_with_params( + uri, + FragmentCount::from(2), + FragmentRowCount::from(10), + Some(WriteParams { + max_rows_per_file: 10, + enable_stable_row_ids: true, + ..Default::default() + }), + ) + .await + .unwrap(); + DatasetBuilder::from_uri(uri).load().await.unwrap() + } + + fn edit_fragments(dataset: &mut Dataset, edit: impl FnOnce(&mut Vec)) { + let mut manifest = dataset.manifest.as_ref().clone(); + let mut fragments = manifest.fragments.as_ref().clone(); + edit(&mut fragments); + manifest.fragments = Arc::new(fragments); + dataset.manifest = Arc::new(manifest); + } + + async fn assert_invalid(dataset: &Dataset, expected_message: &str) { + let err = dataset.validate().await.unwrap_err(); + assert!( + matches!(err, Error::CorruptFile { .. }), + "expected a corrupt file error, got {err:?}" + ); + let message = err.to_string(); + assert!( + message.contains(expected_message), + "expected {expected_message:?} in {message:?}" + ); + } + + #[tokio::test] + async fn test_validate_rejects_missing_row_id_meta() { + let temp_dir = lance_core::utils::tempfile::TempStrDir::default(); + let mut dataset = validation_fixture(&temp_dir).await; + edit_fragments(&mut dataset, |fragments| fragments[1].row_id_meta = None); + + assert_invalid(&dataset, "Fragment 1 has no row id metadata").await; + } + + #[tokio::test] + async fn test_validate_rejects_row_id_sequence_length_mismatch() { + let temp_dir = lance_core::utils::tempfile::TempStrDir::default(); + let mut dataset = validation_fixture(&temp_dir).await; + edit_fragments(&mut dataset, |fragments| { + let short = RowIdSequence::from(&[100u64, 101, 102][..]); + fragments[1].row_id_meta = Some(RowIdMeta::Inline(write_row_ids(&short).into())); + }); + + assert_invalid(&dataset, "Fragment 1 has 3 row ids, but 10 physical rows").await; + } + + #[tokio::test] + async fn test_validate_rejects_duplicate_live_row_ids() { + let temp_dir = lance_core::utils::tempfile::TempStrDir::default(); + let mut dataset = validation_fixture(&temp_dir).await; + edit_fragments(&mut dataset, |fragments| { + fragments[1].row_id_meta = fragments[0].row_id_meta.clone(); + }); + + assert_invalid( + &dataset, + "Row id 0 is live in both fragment 0 at offset 0 and fragment 1 at offset 0", + ) + .await; + } + + #[tokio::test] + async fn test_validate_rejects_reused_next_row_id() { + let temp_dir = lance_core::utils::tempfile::TempStrDir::default(); + let mut dataset = validation_fixture(&temp_dir).await; + let mut manifest = dataset.manifest.as_ref().clone(); + manifest.next_row_id = 19; + dataset.manifest = Arc::new(manifest); + + assert_invalid( + &dataset, + "will hand out row id 19 next, but row id 19 is already in use", + ) + .await; + } + + #[tokio::test] + async fn test_validate_rejects_misaligned_version_sequence() { + let temp_dir = lance_core::utils::tempfile::TempStrDir::default(); + let mut dataset = validation_fixture(&temp_dir).await; + edit_fragments(&mut dataset, |fragments| { + let too_long = RowDatasetVersionSequence::from_uniform_row_count(11, 1); + fragments[1].created_at_version_meta = + Some(RowDatasetVersionMeta::from_sequence(&too_long).unwrap()); + }); + + assert_invalid( + &dataset, + "Fragment 1 has 11 created_at versions, but 10 physical rows", + ) + .await; + } + + /// Number of sequence entries whose row id already appeared in an earlier fragment, + /// tombstoned slots included. + async fn count_repeated_row_ids(dataset: &Dataset) -> usize { + let mut seen = RoaringTreemap::new(); + let mut repeated = 0; + for fragment in dataset.get_fragments() { + let sequence = load_row_id_sequence(dataset, fragment.metadata()) + .await + .unwrap(); + repeated += sequence.iter().filter(|id| !seen.insert(*id)).count(); + } + repeated + } + + /// A tombstoned slot keeps its row id, so after an update the same id lives in the + /// rewritten fragment and lingers, deleted, in the original. Only live ids must be + /// unique — this asserts the checks agree with that. + #[tokio::test] + async fn test_validate_across_write_operations() { + let temp_dir = lance_core::utils::tempfile::TempStrDir::default(); + let dataset = validation_fixture(&temp_dir).await; + dataset.validate().await.unwrap(); + + let batch = lance_datagen::gen_batch() + .col("i", lance_datagen::array::step_custom::(20, 1)) + .into_batch_rows(lance_datagen::RowCount::from(10)) + .unwrap(); + let arrow_schema = Arc::new(ArrowSchema::from(dataset.schema())); + let reader = RecordBatchIterator::new(vec![Ok(batch)], arrow_schema); + let mut dataset = Dataset::write( + reader, + &temp_dir, + Some(WriteParams { + mode: WriteMode::Append, + enable_stable_row_ids: true, + ..Default::default() + }), + ) + .await + .unwrap(); + dataset.validate().await.unwrap(); + + delete(&mut dataset, "i = 4 or i = 12").await; + dataset.validate().await.unwrap(); + + let dataset = UpdateBuilder::new(Arc::new(dataset)) + .update_where("i >= 15") + .unwrap() + .set("i", "i + 1000") + .unwrap() + .build() + .unwrap() + .execute() + .await + .unwrap() + .new_dataset; + dataset.validate().await.unwrap(); + assert!( + count_repeated_row_ids(&dataset).await > 0, + "update should leave rewritten row ids behind in tombstoned slots" + ); + + let merge_source = lance_datagen::gen_batch() + .col("i", lance_datagen::array::step_custom::(8, 1)) + .into_batch_rows(lance_datagen::RowCount::from(6)) + .unwrap(); + let schema = merge_source.schema(); + let merge_job = MergeInsertBuilder::try_new(dataset.clone(), vec!["i".to_string()]) + .unwrap() + .when_matched(WhenMatched::UpdateAll) + .when_not_matched(WhenNotMatched::InsertAll) + .try_build() + .unwrap(); + let source = lance_datafusion::utils::reader_to_stream(Box::new(RecordBatchIterator::new( + [Ok(merge_source)], + schema, + ))); + let (dataset, _stats) = merge_job.execute(source).await.unwrap(); + dataset.validate().await.unwrap(); + + let mut dataset = dataset.as_ref().clone(); + compact(&mut dataset, 20).await; + dataset.validate().await.unwrap(); + } +} diff --git a/rust/lance/src/dataset/scanner.rs b/rust/lance/src/dataset/scanner.rs index 671a2c24333..cf969a2f978 100644 --- a/rust/lance/src/dataset/scanner.rs +++ b/rust/lance/src/dataset/scanner.rs @@ -1,7 +1,7 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright The Lance Authors -use std::collections::HashSet; +use std::collections::{HashMap, HashSet}; use datafusion::config::ConfigOptions; use lance_select::result::IndexExprResultWireFormat; @@ -17,7 +17,7 @@ use arrow_schema::{DataType, Field as ArrowField, Schema as ArrowSchema, SchemaR use arrow_select::concat::concat_batches; use async_recursion::async_recursion; use chrono::Utc; -use datafusion::common::{DFSchema, JoinType, NullEquality, SchemaExt, exec_datafusion_err}; +use datafusion::common::{DFSchema, JoinType, NullEquality, exec_datafusion_err}; use datafusion::functions_aggregate; use datafusion::logical_expr::{Expr, ScalarUDF, col, lit}; use datafusion::physical_expr::PhysicalSortExpr; @@ -51,7 +51,7 @@ use futures::{FutureExt, TryStreamExt}; use lance_arrow::floats::{FloatType, coerce_float_vector}; use lance_arrow::{DataTypeExt, SchemaExt as ArrowSchemaExt}; use lance_core::datatypes::{ - BlobHandling, Field, OnMissing, Projection, escape_field_path_for_project, format_field_path, + BlobHandling, Field, OnMissing, Projection, escape_field_path_for_project, }; use lance_core::error::LanceOptionExt; use lance_core::utils::address::RowAddress; @@ -65,35 +65,49 @@ use lance_datafusion::expr::safe_coerce_scalar; use lance_datafusion::projection::ProjectionPlan; use lance_file::reader::FileReaderOptions; use lance_index::IndexCriteria; +use lance_index::metrics::NoOpMetricsCollector; use lance_index::scalar::FullTextSearchQuery; use lance_index::scalar::expression::PlannerIndexExt; use lance_index::scalar::expression::ScalarIndexExpr; use lance_index::scalar::inverted::query::{ - FtsQuery, FtsQueryNode, FtsSearchParams, MatchQuery, PhraseQuery, fill_fts_query_column, + FtsQuery, FtsQueryNode, FtsSearchParams, MatchQuery, Operator, PhraseQuery, + fill_fts_query_column, }; -use lance_index::scalar::inverted::{SCORE_COL, SCORE_FIELD}; +use lance_index::scalar::inverted::{ + DOC_INDEX_COL, DOC_INDEX_FIELD, DocumentGranularity, SCORE_COL, SCORE_FIELD, fts_schema, +}; +use lance_index::scalar::registry::VALUE_COLUMN_NAME; use lance_index::vector::{ApproxMode, DEFAULT_QUERY_PARALLELISM, DIST_COL, Query}; -use lance_index::{metrics::NoOpMetricsCollector, scalar::inverted::FTS_SCHEMA}; use lance_io::stream::RecordBatchStream; use lance_linalg::distance::MetricType; use lance_select::{IndexExprResult, RowAddrMask, RowAddrTreeMap}; use lance_table::format::{Fragment, IndexMetadata}; +use prost::Message; use roaring::RoaringBitmap; use tracing::{Span, info_span, instrument}; use uuid::Uuid; use super::Dataset; +use super::versions; +use crate::dataset::overlay::{collect_overlay_stale_rows_for_segment, overlaid_fragments}; use crate::dataset::row_offsets_to_row_addresses; +use crate::dataset::rowids::{live_row_addrs_to_row_ids, translate_addr_treemap_to_row_ids}; use crate::dataset::utils::SchemaAdapter; use crate::index::DatasetIndexInternalExt; -use crate::index::scalar::inverted::{load_segment_details, load_segments}; -use crate::index::scalar_logical::scalar_index_fragment_bitmap; +use crate::index::scalar::inverted::{ + fts_index_fragment_bitmap, load_segment_details, load_segments, resolve_fts_field, + resolve_query_document_granularity, +}; +use crate::index::scalar_logical::{load_named_scalar_segments, scalar_index_fragment_bitmap}; use crate::index::vector::utils::{ default_distance_type_for, get_vector_dim, get_vector_type, validate_distance_type_for, }; -use crate::io::exec::filtered_read::{FilteredReadExec, FilteredReadOptions}; +use crate::io::exec::filtered_read::{ + FilteredReadExec, FilteredReadOptions, FilteredReadThreadingMode, +}; use crate::io::exec::fts::{ - BoostQueryExec, FlatMatchFilterExec, FlatMatchQueryExec, MatchQueryExec, PhraseQueryExec, + BoostQueryExec, CompoundQueryExec, FlatMatchFilterExec, FlatMatchQueryExec, FtsDocumentExec, + MatchQueryExec, PhraseQueryExec, SharedFtsScorer, }; use crate::io::exec::knn::MultivectorScoringExec; use crate::io::exec::scalar_index::{MaterializeIndexExec, ScalarIndexExec}; @@ -112,14 +126,148 @@ use crate::io::exec::{ use crate::{Error, Result}; use crate::{ datatypes::Schema, - io::exec::fts::{BoolSlot, BooleanQueryExec, build_boolean_query_children}, + io::exec::fts::{BoolSlot, BooleanQueryExec, build_boolean_query_children_with_schema}, }; pub use lance_datafusion::exec::{ExecutionStatsCallback, ExecutionSummaryCounts}; #[cfg(feature = "substrait")] use lance_datafusion::substrait::parse_substrait; -pub(crate) const BATCH_SIZE_FALLBACK: usize = 8192; +/// Rows per output batch when neither the scan options nor +/// `LANCE_DEFAULT_BATCH_SIZE` specify one. +pub const BATCH_SIZE_FALLBACK: usize = 8192; + +enum FtsOverlayPlan { + Unchanged(Option>), + RowLevel { + stale_rows: HashMap, + segments: Vec, + }, + FullScan, +} + +fn collect_all_fts_columns(query: &FtsQuery, columns: &mut HashSet) { + match query { + FtsQuery::Match(query) => { + if let Some(column) = &query.column { + columns.insert(column.clone()); + } + } + FtsQuery::Phrase(query) => { + if let Some(column) = &query.column { + columns.insert(column.clone()); + } + } + FtsQuery::Boost(query) => { + collect_all_fts_columns(&query.positive, columns); + collect_all_fts_columns(&query.negative, columns); + } + FtsQuery::MultiMatch(query) => { + for match_query in &query.match_queries { + if let Some(column) = &match_query.column { + columns.insert(column.clone()); + } + } + } + FtsQuery::Boolean(query) => { + for child in query + .should + .iter() + .chain(&query.must) + .chain(&query.must_not) + { + collect_all_fts_columns(child, columns); + } + } + } +} + +fn supports_compound_scorer(query: &FtsQuery) -> bool { + fn supports_shape(query: &FtsQuery) -> bool { + match query { + FtsQuery::Match(_) | FtsQuery::Phrase(_) | FtsQuery::MultiMatch(_) => true, + FtsQuery::Boolean(query) => { + (!query.should.is_empty() || !query.must.is_empty()) + && query + .should + .iter() + .chain(&query.must) + .chain(&query.must_not) + .all(supports_shape) + } + FtsQuery::Boost(query) => { + supports_shape(&query.positive) && supports_shape(&query.negative) + } + } + } + + if matches!(query, FtsQuery::Match(_) | FtsQuery::Phrase(_)) || !supports_shape(query) { + return false; + } + let mut columns = HashSet::new(); + collect_all_fts_columns(query, &mut columns); + columns.len() == 1 +} + +fn contains_phrase_query(query: &FtsQuery) -> bool { + match query { + FtsQuery::Phrase(_) => true, + FtsQuery::Match(_) | FtsQuery::MultiMatch(_) => false, + FtsQuery::Boost(query) => { + contains_phrase_query(&query.positive) || contains_phrase_query(&query.negative) + } + FtsQuery::Boolean(query) => query + .should + .iter() + .chain(&query.must) + .chain(&query.must_not) + .any(contains_phrase_query), + } +} + +fn validate_fts_query_contract(query: &FtsQuery) -> Result<()> { + fn validate_multiplier(name: &str, value: f32) -> Result<()> { + if value.is_finite() && value >= 0.0 { + Ok(()) + } else { + Err(Error::invalid_input(format!( + "{name} must be finite and non-negative, got {value}" + ))) + } + } + + match query { + FtsQuery::Match(query) => validate_multiplier("MatchQuery boost", query.boost), + FtsQuery::Phrase(_) => Ok(()), + FtsQuery::Boost(query) => { + validate_multiplier("BoostQuery negative_boost", query.negative_boost)?; + validate_fts_query_contract(&query.positive)?; + validate_fts_query_contract(&query.negative) + } + FtsQuery::MultiMatch(query) => { + for match_query in &query.match_queries { + validate_multiplier("MultiMatchQuery boost", match_query.boost)?; + } + Ok(()) + } + FtsQuery::Boolean(query) => { + if query.should.is_empty() && query.must.is_empty() { + return Err(Error::invalid_input( + "boolean query must have at least one should/must query", + )); + } + for child in query + .should + .iter() + .chain(&query.must) + .chain(&query.must_not) + { + validate_fts_query_contract(child)?; + } + Ok(()) + } + } +} /// Parse an environment variable as a specific type, logging a warning on parse failure. fn parse_env_var(env_var_name: &str, default_val: &str) -> Option @@ -286,10 +434,10 @@ impl MaterializationStyle { } #[derive(Debug)] -struct PlannedFilteredScan { - plan: Arc, - limit_pushed_down: bool, - filter_pushed_down: bool, +pub(super) struct PlannedFilteredScan { + pub(super) plan: Arc, + pub(super) limit_pushed_down: bool, + pub(super) filter_pushed_down: bool, } pub struct FilterPlan { @@ -747,10 +895,11 @@ pub struct Scanner { batch_size: Option, /// If set, the scanner will produce batches whose total size in bytes - /// is approximately this value, overriding the row-based `batch_size`. + /// is approximately this value. When both limits are set, the scanner uses + /// the smaller row count selected by either limit. batch_size_bytes: Option, - /// Number of batches to prefetch + /// Number of batches to decode concurrently batch_readahead: usize, /// Number of fragments to read concurrently @@ -1275,15 +1424,6 @@ impl Scanner { /// .into_stream(); /// ``` pub fn full_text_search(&mut self, query: FullTextSearchQuery) -> Result<&mut Self> { - let fields = query.columns(); - if !fields.is_empty() { - for field in fields.iter() { - if self.dataset.schema().field(field).is_none() { - return Err(Error::invalid_input(format!("Column {} not found", field))); - } - } - } - self.full_text_query = Some(query); Ok(self) } @@ -1315,9 +1455,9 @@ impl Scanner { /// Set the maximum number of rows per batch. /// - /// Note: this can be overridden by [`Self::batch_size_bytes`] or by a dataset-level - /// `batch_size_bytes` set via [`ReadParams::file_reader_options`](crate::dataset::ReadParams::file_reader_options). When a byte-based - /// batch size is active, the row-based batch size is used only as an initial estimate. + /// When a byte limit is also configured through [`Self::batch_size_bytes`] or + /// [`ReadParams::file_reader_options`](crate::dataset::ReadParams::file_reader_options), + /// both limits apply and the one reached first determines the batch size. pub fn batch_size(&mut self, batch_size: usize) -> &mut Self { self.batch_size = Some(batch_size); self @@ -1326,7 +1466,10 @@ impl Scanner { /// Set the target batch size in bytes. /// /// When set, the scanner will produce batches whose total size in bytes - /// is approximately this value, overriding the row-based `batch_size`. + /// is approximately this value. When a row-based `batch_size` is also set, + /// both limits apply and the one reached first determines the batch size. + /// This cannot be combined with [`Self::strict_batch_size`] because strict + /// row batching can merge batches beyond the byte limit. /// /// This can also be configured at the dataset level via /// [`ReadParams::file_reader_options`](crate::dataset::ReadParams::file_reader_options). A scanner-level setting takes @@ -1371,16 +1514,22 @@ impl Scanner { self } - /// Set the prefetch size. - /// Ignored in v2 and newer format + /// Set the number of batches to decode concurrently. + /// + /// This bounds the decode fan-out of the scan: at most this many batch-decode + /// tasks run in flight at once. Defaults to `get_num_compute_intensive_cpus()`. + /// + /// `nbatches` must be greater than zero. pub fn batch_readahead(&mut self, nbatches: usize) -> &mut Self { self.batch_readahead = nbatches; self } - /// Set the fragment readahead. + /// Set the number of fragments whose reads may be scheduled concurrently. /// - /// This is only used if ``scan_in_order`` is set to false. + /// This applies to both ordered and unordered scans. [`Self::scan_in_order`] + /// controls result ordering, not whether fragment I/O overlaps. Set this to + /// `1` to read one fragment at a time. pub fn fragment_readahead(&mut self, nfragments: usize) -> &mut Self { self.fragment_readahead = Some(nfragments); self @@ -1430,6 +1579,10 @@ impl Scanner { /// By default, this is False and output batches are allowed to have fewer than `batch_size` rows /// Setting this to True will require us to merge batches, incurring a data copy, for a minor performance /// penalty. + /// + /// This cannot be enabled when a byte limit is configured through + /// [`Self::batch_size_bytes`] or + /// [`ReadParams::file_reader_options`](crate::dataset::ReadParams::file_reader_options). pub fn strict_batch_size(&mut self, strict_batch_size: bool) -> &mut Self { self.strict_batch_size = strict_batch_size; self @@ -1742,7 +1895,7 @@ impl Scanner { .field(&column.column_name) .ok_or(Error::invalid_input(format!( "Column {} not found", - &column.column_name + column.column_name )))?; } } @@ -1760,8 +1913,9 @@ impl Scanner { /// Configure the speed / accuracy tradeoff for approximate vector search. /// - /// This setting is currently only used by RQ-quantized indexes, such as - /// IVF_RQ. Other index types ignore this setting. + /// This setting is currently used by RQ-quantized indexes (such as + /// IVF_RQ) and by prefiltered search on HNSW indexes, where `Fast` + /// enables the ACORN traversal. Other index types ignore this setting. pub fn approx_mode(&mut self, approx_mode: ApproxMode) -> &mut Self { if let Some(q) = self.nearest.as_mut() { q.approx_mode = approx_mode; @@ -1899,6 +2053,38 @@ impl Scanner { } } + /// Ensure `input` exposes `column_name` as a top-level column. + /// + /// Nested FTS flat-search paths read the projected struct column from storage + /// but the FTS executor consumes a single document column by name. + fn ensure_column_alias( + &self, + input: Arc, + column_name: &str, + ) -> Result> { + let input_schema = input.schema(); + if input_schema.column_with_name(column_name).is_some() { + return Ok(input); + } + + let mut projection_exprs = Vec::with_capacity(input_schema.fields().len() + 1); + for field in input_schema.fields() { + projection_exprs.push(( + Arc::new(Column::new_with_schema( + field.name(), + input_schema.as_ref(), + )?) as Arc, + field.name().clone(), + )); + } + projection_exprs.push(( + Self::create_column_expr(column_name, self.dataset.as_ref(), input_schema.as_ref())?, + column_name.to_string(), + )); + + Ok(Arc::new(ProjectionExec::try_new(projection_exprs, input)?)) + } + /// Set whether to use statistics to optimize the scan (default: true) /// /// This is used for debugging or benchmarking purposes. @@ -1931,7 +2117,11 @@ impl Scanner { } } - fn add_extra_columns(&self, schema: Schema) -> Result { + fn add_extra_columns( + &self, + schema: Schema, + fts_document_granularity: Option, + ) -> Result { let mut extra_columns = vec![ArrowField::new(ROW_OFFSET, DataType::UInt64, true)]; if self.nearest.as_ref().is_some() { @@ -1943,6 +2133,15 @@ impl Scanner { if self.full_text_query.is_some() { extra_columns.push(ArrowField::new(SCORE_COL, DataType::Float32, true)); + if fts_document_granularity + .map(DocumentGranularity::is_list_element) + // `get_expr_filter` is synchronous, so an omitted granularity + // cannot be resolved from index metadata here. Include the + // column conservatively; `create_plan` uses the resolved value. + .unwrap_or(true) + { + extra_columns.push(DOC_INDEX_FIELD.clone()); + } } schema.merge(&ArrowSchema::new(extra_columns)) @@ -1953,6 +2152,17 @@ impl Scanner { /// This is the schema of the dataset, any metadata columns like _rowid or _rowaddr /// and any extra columns like _distance or _score fn filterable_schema(&self) -> Result> { + let fts_document_granularity = self + .full_text_query + .as_ref() + .and_then(|query| self.fts_document_granularity(&query.query).ok()); + self.filterable_schema_with_fts_granularity(fts_document_granularity) + } + + fn filterable_schema_with_fts_granularity( + &self, + fts_document_granularity: Option, + ) -> Result> { let base_schema = Projection::full(self.dataset.clone()) .with_row_id() .with_row_addr() @@ -1960,7 +2170,10 @@ impl Scanner { .with_row_created_at_version() .to_schema(); - Ok(Arc::new(self.add_extra_columns(base_schema)?)) + Ok(Arc::new(self.add_extra_columns( + base_schema, + fts_document_granularity, + )?)) } /// This takes the current output, and the user's requested projection, and calculates the @@ -1975,6 +2188,16 @@ impl Scanner { // of all available columns if the user did not specify a projection) let mut output_expr = self.projection_plan.to_physical_exprs(current_schema)?; + if self.full_text_query.is_some() + && current_schema.field_with_name(DOC_INDEX_COL).is_ok() + && output_expr.iter().all(|(_, name)| name != DOC_INDEX_COL) + { + output_expr.push(( + expressions::col(DOC_INDEX_COL, current_schema)?, + DOC_INDEX_COL.to_string(), + )); + } + // Make sure _distance and _score are _always_ in the output unless user has opted out of the legacy // projection behavior if self.autoproject_scoring_columns { @@ -2186,6 +2409,9 @@ impl Scanner { } #[allow(clippy::type_complexity)] + // TODO(datafusion-54): migrate off the deprecated + // create_aggregate_expr_and_maybe_filter to LoweredAggregateBuilder. + #[allow(deprecated)] fn build_physical_aggregate_expr( &self, expr: &Expr, @@ -2310,11 +2536,12 @@ impl Scanner { MaterializationStyle::AllLate => false, MaterializationStyle::AllEarlyExcept(ref cols) => !cols.contains(&(field.id as u32)), MaterializationStyle::Heuristic => { - if field.is_blob() { - // By default, blobs are loaded as descriptions, and so should be early - // - // TODO: Once we make blob handling configurable, we should use the blob - // handling setting here. + if field.is_blob() && self.blob_handling.returns_description(field) { + // A blob returned as a description (offset + size) is tiny, so it is + // cheaper to read eagerly. When blob_handling materializes the full + // binary value instead (e.g. `all_binary`), fall through to the + // width-based heuristic so a selective filter can late-materialize it + // rather than reading the whole column. return true; } @@ -2362,6 +2589,25 @@ impl Scanner { } fn validate_options(&self) -> Result<()> { + if self.batch_readahead == 0 { + return Err(Error::invalid_input_source( + "batch_readahead must be greater than 0, got 0".into(), + )); + } + + if self.strict_batch_size + && let Some(batch_size_bytes) = self + .resolved_file_reader_options() + .and_then(|options| options.batch_size_bytes) + { + return Err(Error::invalid_input_source( + format!( + "strict_batch_size=true cannot be combined with batch_size_bytes={batch_size_bytes}; strict row batching can merge batches beyond the byte limit" + ) + .into(), + )); + } + if self.include_deleted_rows && !self.projection_plan.physical_projection.with_row_id { return Err(Error::invalid_input_source( "include_deleted_rows is set but with_row_id is false".into(), @@ -2392,8 +2638,14 @@ impl Scanner { Ok(()) } - async fn create_filter_plan(&self, use_scalar_index: bool) -> Result { - let filter_schema = self.filterable_schema()?; + async fn create_filter_plan( + &self, + use_scalar_index: bool, + query_filter: Option, + fts_document_granularity: Option, + ) -> Result { + let filter_schema = + self.filterable_schema_with_fts_granularity(fts_document_granularity)?; let planner = Planner::new(Arc::new(filter_schema.as_ref().into())); // Check expr filter @@ -2423,15 +2675,15 @@ impl Scanner { // fallback to a non-indexed filter let filter_plan = planner.create_filter_plan(expr.clone(), &index_info, false)?; - FilterPlan::new(self.filter.query_filter.clone(), filter_plan) + FilterPlan::new(query_filter.clone(), filter_plan) } else { - FilterPlan::new(self.filter.query_filter.clone(), filter_plan) + FilterPlan::new(query_filter.clone(), filter_plan) } } else { - FilterPlan::new(self.filter.query_filter.clone(), filter_plan) + FilterPlan::new(query_filter.clone(), filter_plan) } } else { - FilterPlan::new(self.filter.query_filter.clone(), ExprFilterPlan::default()) + FilterPlan::new(query_filter, ExprFilterPlan::default()) }; // Check query filter @@ -2544,13 +2796,31 @@ impl Scanner { log::trace!("creating scanner plan"); self.validate_options()?; + let full_text_query = match &self.full_text_query { + Some(query) => Some(self.resolve_full_text_search_query(query).await?), + None => None, + }; + let query_filter = match &self.filter.query_filter { + Some(QueryFilter::Fts(query)) => Some(QueryFilter::Fts( + self.resolve_full_text_search_query(query).await?, + )), + Some(QueryFilter::Vector(query)) => Some(QueryFilter::Vector(query.clone())), + None => None, + }; + let fts_document_granularity = full_text_query + .as_ref() + .map(|query| self.fts_document_granularity(&query.query)) + .transpose()?; + // Scalar indices are only used when prefiltering let use_scalar_index = self.use_scalar_index && (self.prefilter || self.nearest.is_none()); - let mut filter_plan = self.create_filter_plan(use_scalar_index).await?; + let mut filter_plan = self + .create_filter_plan(use_scalar_index, query_filter, fts_document_granularity) + .await?; let mut use_limit_node = true; // Source: either a (K|A)NN search, full text search, or a (full|indexed) scan - let mut plan: Arc = match (&self.nearest, &self.full_text_query) { + let mut plan: Arc = match (&self.nearest, &full_text_query) { (Some(_), None) => self.vector_search_source(&mut filter_plan).await?, (None, Some(query)) => self.fts_search_source(&mut filter_plan, query).await?, (None, None) => { @@ -2746,7 +3016,7 @@ impl Scanner { // Do not call this directly, use filtered_read instead // // First return value is the plan, second is whether the limit was pushed down - async fn legacy_filtered_read( + pub(super) async fn legacy_filtered_read( &self, filter_plan: &ExprFilterPlan, projection: Projection, @@ -2850,7 +3120,7 @@ impl Scanner { // Helper function for filtered_read // // Do not call this directly, use filtered_read instead - async fn new_filtered_read( + pub(super) async fn new_filtered_read( &self, filter_plan: &ExprFilterPlan, projection: Projection, @@ -2858,6 +3128,8 @@ impl Scanner { fragments: Option>>, scan_range: Option>, ) -> Result> { + // Kept for the overlay stale-Take path below, which re-evaluates blocked stale rows. + let user_projection = projection.clone(); let mut read_options = FilteredReadOptions::basic_full_read(&self.dataset) .with_filter_plan(filter_plan.clone()) .with_projection(projection); @@ -2874,6 +3146,11 @@ impl Scanner { read_options = read_options.with_batch_size(batch_size as u32); } + // Bound the decode fan-out by `batch_readahead`. + read_options = read_options.with_threading_mode( + FilteredReadThreadingMode::OnePartitionMultipleThreads(self.batch_readahead), + ); + if let Some(file_reader_options) = self.resolved_file_reader_options() { read_options = read_options.with_file_reader_options(file_reader_options); } @@ -2894,6 +3171,24 @@ impl Scanner { read_options = read_options.with_only_indexed_fragments(); } + // Mask data overlay files: a row with an overlay committed after an index it relies on + // touched an indexed field can no longer be trusted to that index. Block just those rows + // from the index result (their fragments stay indexed, so non-stale rows keep the index) + // and re-evaluate them on a targeted take path below — O(stale_rows), not O(fragment). + let mut overlay_stale_rows: HashMap = HashMap::new(); + if let Some(index_query) = filter_plan.index_query.as_ref() { + let candidate_frags = read_options + .fragments + .clone() + .unwrap_or_else(|| self.dataset.fragments().clone()); + overlay_stale_rows = self + .overlay_stale_index_rows(index_query, &candidate_frags) + .await?; + if let Some(block) = self.stale_rows_block_mask(&overlay_stale_rows).await? { + read_options = read_options.with_overlay_block(block); + } + } + let result_format = self.index_expr_result_format(); let index_input = filter_plan.index_query.clone().map(|index_query| { Arc::new(ScalarIndexExec::new( @@ -2903,59 +3198,70 @@ impl Scanner { )) as Arc }); - Ok(Arc::new(FilteredReadExec::try_new( + let plan: Arc = Arc::new(FilteredReadExec::try_new( self.dataset.clone(), read_options, index_input, + )?); + + if overlay_stale_rows.is_empty() { + return Ok(plan); + } + + // Stale-Take path: take the stale rows' current (overlay-merged) values and re-apply the + // full filter, then union with the indexed read. These rows were blocked from the index + // result above, so this is the only path that can surface them. + let filter = filter_plan.full_expr.as_ref().expect_ok()?; + let filter_cols = Planner::column_names_in_expr(filter); + let take_projection = user_projection.union_columns(filter_cols, OnMissing::Error)?; + + let stale_node = self + .stale_rows_take(&overlay_stale_rows, take_projection) + .await?; + let planner = Planner::new(stale_node.schema()); + let optimized_filter = planner.optimize_expr(filter.clone())?; + let filtered = Arc::new(LanceFilterExec::try_new(optimized_filter, stale_node)?); + let stale_path: Arc = + Arc::new(project(filtered, plan.schema().as_ref())?); + + let unioned = UnionExec::try_new(vec![plan, stale_path])?; + Ok(Arc::new(RepartitionExec::try_new( + unioned, + datafusion::physical_plan::Partitioning::RoundRobinBatch(1), )?)) } // Helper function for filtered read // // Delegates to legacy or new filtered read based on dataset storage version - async fn filtered_read( - &self, - filter_plan: &ExprFilterPlan, + fn filtered_read<'a>( + &'a self, + filter_plan: &'a ExprFilterPlan, projection: Projection, make_deletions_null: bool, fragments: Option>>, scan_range: Option>, is_prefilter: bool, - ) -> Result { - // Use legacy path if dataset uses legacy storage format - if self.dataset.is_legacy_storage() { - self.legacy_filtered_read( - filter_plan, - projection, - make_deletions_null, - fragments, - scan_range, - is_prefilter, - ) - .await - } else { - let limit_pushed_down = scan_range.is_some(); - let plan = self - .new_filtered_read( - filter_plan, - projection, - make_deletions_null, - fragments, - scan_range, - ) - .await?; - Ok(PlannedFilteredScan { - filter_pushed_down: true, - limit_pushed_down, - plan, - }) - } + ) -> BoxFuture<'a, Result> { + versions::filtered_read( + self.dataset + .manifest() + .data_storage_format + .lance_file_format(), + self, + filter_plan, + projection, + make_deletions_null, + fragments, + scan_range, + is_prefilter, + ) + .boxed() } - fn u64s_as_take_input(&self, u64s: Vec) -> Result> { - let row_addrs = RowAddrTreeMap::from_iter(u64s); - let row_addr_mask = RowAddrMask::from_allowed(row_addrs); - let index_result = IndexExprResult::exact(row_addr_mask); + fn row_ids_as_take_input(&self, row_ids: RowAddrTreeMap) -> Result> { + let row_id_mask = RowAddrMask::from_allowed(row_ids); + let index_result = IndexExprResult::exact(row_id_mask); let fragments_covered = self.dataset.fragment_bitmap.as_ref().clone(); let format = self.index_expr_result_format(); let batch = index_result.serialize(&fragments_covered, format)?; @@ -2965,19 +3271,27 @@ impl Scanner { Ok(Arc::new(OneShotExec::new(stream))) } + async fn row_addrs_as_take_input(&self, row_addrs: Vec) -> Result> { + let row_ids = + live_row_addrs_to_row_ids(&self.dataset, row_addrs.into_iter().map(Some)).await?; + self.row_ids_as_take_input(RowAddrTreeMap::from_iter(row_ids.into_iter().flatten())) + } + async fn take_source(&self, take_op: TakeOperation) -> Result> { // We generally assume that late materialization does not make sense for take operations // so we can just use the physical projection let projection = self.projection_plan.physical_projection.clone(); let input = match take_op { - TakeOperation::RowIds(ids) => self.u64s_as_take_input(ids), - TakeOperation::RowAddrs(addrs) => self.u64s_as_take_input(addrs), + TakeOperation::RowIds(ids) => { + self.row_ids_as_take_input(RowAddrTreeMap::from_iter(ids)) + } + TakeOperation::RowAddrs(addrs) => self.row_addrs_as_take_input(addrs).await, TakeOperation::RowOffsets(offsets) => { let mut addrs = row_offsets_to_row_addresses(&self.dataset.get_fragments(), &offsets).await?; addrs.retain(|addr| *addr != RowAddress::TOMBSTONE_ROW); - self.u64s_as_take_input(addrs) + self.row_addrs_as_take_input(addrs).await } }?; @@ -3107,7 +3421,10 @@ impl Scanner { // If we are prefiltering then the ann / knn node will take care of the filter let source: Arc = match &filter_plan.fts_filter() { Some(fts_query) => { - let fts_plan = self.fts(&filter_plan.expr_filter_plan, fts_query).await?; + let mut fts_plan = self.fts(&filter_plan.expr_filter_plan, fts_query).await?; + if fts_plan.schema().field_with_name(DOC_INDEX_COL).is_ok() { + fts_plan = self.deduplicate_fts_filter_rows(fts_plan)?; + } let projection = self .dataset .empty_projection() @@ -3133,25 +3450,45 @@ impl Scanner { } } + /// Convert element-document hits into the row-selection semantics required + /// when FTS is used as a filter for another query. + fn deduplicate_fts_filter_rows( + &self, + input: Arc, + ) -> Result> { + let schema = input.schema(); + let group_expr = vec![( + expressions::col(ROW_ID, schema.as_ref())?, + ROW_ID.to_string(), + )]; + let input = Arc::new(RepartitionExec::try_new( + input, + Partitioning::RoundRobinBatch(1), + )?); + Ok(Arc::new(AggregateExec::try_new( + AggregateMode::Single, + PhysicalGroupBy::new_single(group_expr), + Vec::new(), + Vec::new(), + input, + schema, + )?)) + } + async fn fragments_covered_by_fts_leaf( &self, column: &str, + document_granularity: DocumentGranularity, accum: &mut RoaringBitmap, ) -> Result { - let index = self - .dataset - .load_scalar_index(IndexCriteria::default().for_column(column).supports_fts()) - .await?; - match index { - Some(index) => match &index.fragment_bitmap { - Some(fragmap) => { - *accum |= fragmap; - Ok(true) - } - None => Ok(false), - }, - None => Ok(false), - } + let Some(fragment_bitmap) = + fts_index_fragment_bitmap(&self.dataset, column, document_granularity).await? + else { + return Ok(false); + }; + *accum |= fragment_bitmap; + + Ok(true) } #[async_recursion] @@ -3162,10 +3499,14 @@ impl Scanner { ) -> Result { match query { FtsQuery::Match(match_query) => { + let document_granularity = match_query.document_granularity.ok_or_else(|| { + Error::internal("FTS Match query granularity was not resolved".to_string()) + })?; self.fragments_covered_by_fts_leaf( match_query.column.as_ref().ok_or(Error::invalid_input( "the column must be specified in the query".to_string(), ))?, + document_granularity, accum, ) .await @@ -3178,11 +3519,17 @@ impl Scanner { .await?), FtsQuery::MultiMatch(multi_match) => { for mq in &multi_match.match_queries { + let document_granularity = mq.document_granularity.ok_or_else(|| { + Error::internal( + "FTS MultiMatch query granularity was not resolved".to_string(), + ) + })?; if !self .fragments_covered_by_fts_leaf( mq.column.as_ref().ok_or(Error::invalid_input( "the column must be specified in the query".to_string(), ))?, + document_granularity, accum, ) .await? @@ -3193,10 +3540,14 @@ impl Scanner { Ok(true) } FtsQuery::Phrase(phrase_query) => { + let document_granularity = phrase_query.document_granularity.ok_or_else(|| { + Error::internal("FTS Phrase query granularity was not resolved".to_string()) + })?; self.fragments_covered_by_fts_leaf( phrase_query.column.as_ref().ok_or(Error::invalid_input( "the column must be specified in the query".to_string(), ))?, + document_granularity, accum, ) .await @@ -3239,66 +3590,426 @@ impl Scanner { } } - // Create an execution plan to do full text search - async fn fts( - &self, - filter_plan: &ExprFilterPlan, - query: &FullTextSearchQuery, - ) -> Result> { - let columns = query.columns(); - let mut params = query.params(); - if params.limit.is_none() { - let search_limit = match (self.limit, self.offset) { - (Some(limit), Some(offset)) => Some((limit + offset) as usize), - (Some(limit), None) => Some(limit as usize), - (None, Some(_)) => None, // No limit but has offset - fetch all and let limit_node handle - (None, None) => None, - }; - params = params.with_limit(search_limit); + fn fts_document_granularity(&self, query: &FtsQuery) -> Result { + #[derive(Default)] + struct TargetState { + granularity: Option, + list_element_field_id: Option, } - let query = if columns.is_empty() { - // the field is not specified, - // try to search over all indexed fields including nested ones - let indexed_columns = fts_indexed_columns(self.dataset.clone()).await?; - fill_fts_query_column(&query.query, &indexed_columns, false)? - } else { - query.query.clone() - }; - // TODO: Could maybe walk the query here to find all the indices that will be - // involved in the query to calculate a more accuarate required_fragments than - // get_fragments_as_bitmap but this is safe for now. - let prefilter_source = self - .prefilter_source( - filter_plan, - self.fragments_covered_by_fts_query(&query).await?, - ) - .await?; - let fts_exec = self - .plan_fts(&query, ¶ms, filter_plan, &prefilter_source) - .await?; - Ok(fts_exec) - } + fn add_leaf( + schema: &lance_core::datatypes::Schema, + column: Option<&str>, + document_granularity: Option, + state: &mut TargetState, + ) -> Result<()> { + let document_granularity = document_granularity.ok_or_else(|| { + Error::internal("FTS query document granularity was not resolved".to_string()) + })?; + if let Some(existing) = state.granularity + && existing != document_granularity + { + return Err(Error::invalid_input( + "FTS queries cannot mix Row and ListElement document granularities".to_string(), + )); + } + state.granularity = Some(document_granularity); - async fn plan_fts( - &self, - query: &FtsQuery, - params: &FtsSearchParams, - filter_plan: &ExprFilterPlan, - prefilter_source: &PreFilterSource, - ) -> Result> { - let plan: Arc = match query { - FtsQuery::Match(query) => { - self.plan_match_query(query, params, filter_plan, prefilter_source) - .await? + let Some(column) = column else { + if document_granularity.is_list_element() { + return Err(Error::invalid_input( + "ListElement FTS queries must explicitly specify a field path".to_string(), + )); + } + return Ok(()); + }; + let resolved = resolve_fts_field(schema, column, document_granularity)?; + if document_granularity.is_list_element() { + if let Some(existing) = state.list_element_field_id + && existing != resolved.final_field_id + { + return Err(Error::invalid_input( + "all leaves in a ListElement FTS query must use the same final field" + .to_string(), + )); + } + state.list_element_field_id = Some(resolved.final_field_id); } - FtsQuery::Phrase(query) => { - self.plan_phrase_query(query, params, prefilter_source) - .await? + Ok(()) + } + + fn visit( + schema: &lance_core::datatypes::Schema, + query: &FtsQuery, + state: &mut TargetState, + ) -> Result<()> { + match query { + FtsQuery::Match(query) => add_leaf( + schema, + query.column.as_deref(), + query.document_granularity, + state, + ), + FtsQuery::Phrase(query) => add_leaf( + schema, + query.column.as_deref(), + query.document_granularity, + state, + ), + FtsQuery::Boost(query) => { + visit(schema, &query.positive, state)?; + visit(schema, &query.negative, state) + } + FtsQuery::Boolean(query) => { + for child in query + .must + .iter() + .chain(&query.should) + .chain(&query.must_not) + { + visit(schema, child, state)?; + } + Ok(()) + } + FtsQuery::MultiMatch(query) => { + for child in &query.match_queries { + if child + .document_granularity + .is_some_and(|value| value.is_list_element()) + { + return Err(Error::not_supported( + "MultiMatch does not support ListElement document granularity" + .to_string(), + )); + } + add_leaf( + schema, + child.column.as_deref(), + child.document_granularity, + state, + )?; + } + Ok(()) + } } + } - FtsQuery::Boost(query) => { - // for boost query, we need to erase the limit so that we can find + let mut state = TargetState::default(); + visit(self.dataset.schema(), query, &mut state)?; + state.granularity.ok_or_else(|| { + Error::invalid_input("FTS query must contain at least one leaf query".to_string()) + }) + } + + #[async_recursion] + async fn resolve_fts_query_document_granularity(&self, query: FtsQuery) -> Result { + match query { + FtsQuery::Match(mut query) => { + let column = query.column.as_deref().ok_or_else(|| { + Error::invalid_input("the column must be specified in the query".to_string()) + })?; + query.document_granularity = Some( + resolve_query_document_granularity( + self.dataset.as_ref(), + column, + query.document_granularity, + ) + .await?, + ); + Ok(FtsQuery::Match(query)) + } + FtsQuery::Phrase(mut query) => { + let column = query.column.as_deref().ok_or_else(|| { + Error::invalid_input("the column must be specified in the query".to_string()) + })?; + query.document_granularity = Some( + resolve_query_document_granularity( + self.dataset.as_ref(), + column, + query.document_granularity, + ) + .await?, + ); + Ok(FtsQuery::Phrase(query)) + } + FtsQuery::Boost(mut query) => { + query.positive = Box::new( + self.resolve_fts_query_document_granularity(*query.positive) + .await?, + ); + query.negative = Box::new( + self.resolve_fts_query_document_granularity(*query.negative) + .await?, + ); + Ok(FtsQuery::Boost(query)) + } + FtsQuery::Boolean(mut query) => { + let mut must = Vec::with_capacity(query.must.len()); + for child in query.must { + must.push(self.resolve_fts_query_document_granularity(child).await?); + } + let mut should = Vec::with_capacity(query.should.len()); + for child in query.should { + should.push(self.resolve_fts_query_document_granularity(child).await?); + } + let mut must_not = Vec::with_capacity(query.must_not.len()); + for child in query.must_not { + must_not.push(self.resolve_fts_query_document_granularity(child).await?); + } + query.must = must; + query.should = should; + query.must_not = must_not; + Ok(FtsQuery::Boolean(query)) + } + FtsQuery::MultiMatch(mut query) => { + for child in &mut query.match_queries { + let column = child.column.as_deref().ok_or_else(|| { + Error::invalid_input( + "the column must be specified in the query".to_string(), + ) + })?; + child.document_granularity = Some( + resolve_query_document_granularity( + self.dataset.as_ref(), + column, + child.document_granularity, + ) + .await?, + ); + } + Ok(FtsQuery::MultiMatch(query)) + } + } + } + + fn set_missing_query_granularity( + query: &mut FtsQuery, + document_granularity: DocumentGranularity, + ) { + match query { + FtsQuery::Match(query) => { + query + .document_granularity + .get_or_insert(document_granularity); + } + FtsQuery::Phrase(query) => { + query + .document_granularity + .get_or_insert(document_granularity); + } + FtsQuery::Boost(query) => { + Self::set_missing_query_granularity(&mut query.positive, document_granularity); + Self::set_missing_query_granularity(&mut query.negative, document_granularity); + } + FtsQuery::Boolean(query) => { + for child in query + .must + .iter_mut() + .chain(&mut query.should) + .chain(&mut query.must_not) + { + Self::set_missing_query_granularity(child, document_granularity); + } + } + FtsQuery::MultiMatch(query) => { + for child in &mut query.match_queries { + child + .document_granularity + .get_or_insert(document_granularity); + } + } + } + } + + fn query_requests_list_element(query: &FtsQuery) -> bool { + match query { + FtsQuery::Match(query) => query + .document_granularity + .is_some_and(DocumentGranularity::is_list_element), + FtsQuery::Phrase(query) => query + .document_granularity + .is_some_and(DocumentGranularity::is_list_element), + FtsQuery::Boost(query) => { + Self::query_requests_list_element(&query.positive) + || Self::query_requests_list_element(&query.negative) + } + FtsQuery::Boolean(query) => query + .must + .iter() + .chain(&query.should) + .chain(&query.must_not) + .any(Self::query_requests_list_element), + FtsQuery::MultiMatch(query) => query.match_queries.iter().any(|query| { + query + .document_granularity + .is_some_and(DocumentGranularity::is_list_element) + }), + } + } + + async fn resolve_full_text_search_query( + &self, + query: &FullTextSearchQuery, + ) -> Result { + let mut resolved = query.clone(); + if resolved.columns().is_empty() { + if Self::query_requests_list_element(&resolved.query) { + return Err(Error::invalid_input( + "ListElement FTS queries must explicitly specify a field path".to_string(), + )); + } + let indexed_columns = fts_indexed_columns(self.dataset.clone()).await?; + resolved.query = fill_fts_query_column(&resolved.query, &indexed_columns, false)?; + Self::set_missing_query_granularity(&mut resolved.query, DocumentGranularity::Row); + } + resolved.query = self + .resolve_fts_query_document_granularity(resolved.query) + .await?; + self.fts_document_granularity(&resolved.query)?; + Ok(resolved) + } + + // Create an execution plan to do full text search + async fn fts( + &self, + filter_plan: &ExprFilterPlan, + query: &FullTextSearchQuery, + ) -> Result> { + let mut params = query.params(); + if params.limit.is_none() { + let search_limit = match (self.limit, self.offset) { + (Some(limit), Some(offset)) => Some((limit + offset) as usize), + (Some(limit), None) => Some(limit as usize), + (None, Some(_)) => None, // No limit but has offset - fetch all and let limit_node handle + (None, None) => None, + }; + params = params.with_limit(search_limit); + } + let query = &query.query; + validate_fts_query_contract(query)?; + + // TODO: Could maybe walk the query here to find all the indices that will be + // involved in the query to calculate a more accuarate required_fragments than + // get_fragments_as_bitmap but this is safe for now. + let prefilter_source = self + .prefilter_source( + filter_plan, + self.fragments_covered_by_fts_query(query).await?, + ) + .await?; + // Data overlay masking blocks stale rows from indexed leaves and re-evaluates only those + // rows from their current values on the flat-text path. + let fts_exec = self + .plan_fts(query, ¶ms, filter_plan, &prefilter_source) + .await?; + Ok(fts_exec) + } + + async fn plan_compound_scorer( + &self, + query: &FtsQuery, + params: &FtsSearchParams, + prefilter_source: &PreFilterSource, + document_granularity: DocumentGranularity, + ) -> Result>> { + let mut columns = HashSet::new(); + collect_all_fts_columns(query, &mut columns); + let Some(column) = columns.into_iter().next() else { + return Ok(None); + }; + + let index = self + .dataset + .load_scalar_index( + IndexCriteria::default() + .for_column(&column) + .supports_fts() + .with_fts_document_granularity(document_granularity), + ) + .await?; + let Some(index) = index else { + return Ok(None); + }; + let target_fragments: &[Fragment] = self + .fragments + .as_deref() + .unwrap_or_else(|| self.dataset.fragments()); + if target_fragments.is_empty() { + return Ok(None); + } + if !self + .retain_target_fragments(self.dataset.unindexed_fragments(&index.name).await?) + .is_empty() + { + // Flat and posting-backed leaves do not share a document domain. + // Preserve the exact DataFusion fallback until flat leaves expose + // the same candidate protocol. + return Ok(None); + } + let segments = match self + .fts_overlay_plan(&column, document_granularity, target_fragments) + .await? + { + FtsOverlayPlan::Unchanged(Some(segments)) => segments, + FtsOverlayPlan::Unchanged(None) => { + load_segments(&self.dataset, &column, document_granularity) + .await? + .ok_or_else(|| { + Error::invalid_input(format!("No Inverted index found for column {column}")) + })? + } + FtsOverlayPlan::RowLevel { .. } | FtsOverlayPlan::FullScan => return Ok(None), + }; + if contains_phrase_query(query) { + let details = load_segment_details(&self.dataset, &column, &segments).await?; + if !details.with_position { + return Err(Error::invalid_input( + "position is not found but required for phrase queries, try recreating the index with position" + .to_string(), + )); + } + } + Ok(Some(Arc::new(CompoundQueryExec::new_with_segments( + self.dataset.clone(), + query.clone(), + params.clone(), + prefilter_source.clone(), + segments, + )))) + } + + async fn plan_fts( + &self, + query: &FtsQuery, + params: &FtsSearchParams, + filter_plan: &ExprFilterPlan, + prefilter_source: &PreFilterSource, + ) -> Result> { + let document_granularity = self.fts_document_granularity(query)?; + if !document_granularity.is_list_element() + && supports_compound_scorer(query) + && let Some(plan) = self + .plan_compound_scorer(query, params, prefilter_source, document_granularity) + .await? + { + return Ok(plan); + } + + // Cross-column, flat, and overlay-backed compound queries retain the + // exact DataFusion fallback because their leaves do not share one + // posting document domain. + let plan: Arc = match query { + FtsQuery::Match(query) => { + self.plan_match_query(query, params, filter_plan, prefilter_source) + .await? + } + FtsQuery::Phrase(query) => { + self.plan_phrase_query(query, params, filter_plan, prefilter_source) + .await? + } + + FtsQuery::Boost(query) => { + // for boost query, we need to erase the limit so that we can find // the documents that are not in the top-k results of the positive query, // but in the final top-k results. let unlimited_params = params.clone().with_limit(None); @@ -3361,18 +4072,26 @@ impl Scanner { fts_node, schema, )?); - let sort_expr = PhysicalSortExpr { - expr: expressions::col(SCORE_COL, fts_node.schema().as_ref())?, - options: SortOptions { - descending: true, - nulls_first: false, + let sort_exprs = [ + PhysicalSortExpr { + expr: expressions::col(SCORE_COL, fts_node.schema().as_ref())?, + options: SortOptions { + descending: true, + nulls_first: false, + }, }, - }; + PhysicalSortExpr { + expr: expressions::col(ROW_ID, fts_node.schema().as_ref())?, + options: SortOptions { + descending: false, + nulls_first: false, + }, + }, + ]; - Arc::new( - SortExec::new([sort_expr].into(), fts_node) - .with_fetch(self.limit.map(|l| l as usize)), - ) + // `params.limit` is the recursive planning contract. Compound + // parents pass `None` when they require every candidate. + Arc::new(SortExec::new(sort_exprs.into(), fts_node).with_fetch(params.limit)) } FtsQuery::Boolean(query) => { // TODO: rewrite the query for better performance @@ -3418,11 +4137,32 @@ impl Scanner { ); } - let should = build_boolean_query_children(BoolSlot::Should, should)? - .expect("Should slot always returns Some"); - let must = build_boolean_query_children(BoolSlot::Must, must)?; - let must_not = build_boolean_query_children(BoolSlot::MustNot, must_not)? - .expect("MustNot slot always returns Some"); + let boolean_schema = fts_schema(document_granularity); + let should = build_boolean_query_children_with_schema( + BoolSlot::Should, + should, + boolean_schema.clone(), + )? + .ok_or_else(|| { + Error::internal( + "boolean should planning returned no execution plan".to_string(), + ) + })?; + let must = build_boolean_query_children_with_schema( + BoolSlot::Must, + must, + boolean_schema.clone(), + )?; + let must_not = build_boolean_query_children_with_schema( + BoolSlot::MustNot, + must_not, + boolean_schema, + )? + .ok_or_else(|| { + Error::internal( + "boolean must-not planning returned no execution plan".to_string(), + ) + })?; if query.should.is_empty() && must.is_none() { return Err(Error::invalid_input( @@ -3447,31 +4187,163 @@ impl Scanner { &self, query: &PhraseQuery, params: &FtsSearchParams, + filter_plan: &ExprFilterPlan, prefilter_source: &PreFilterSource, ) -> Result> { let column = query.column.clone().ok_or(Error::invalid_input( "the column must be specified in the query".to_string(), ))?; + let document_granularity = query.document_granularity.ok_or_else(|| { + Error::internal("FTS Phrase query granularity was not resolved".to_string()) + })?; + resolve_fts_field(self.dataset.schema(), &column, document_granularity)?; + let output_schema = fts_schema(document_granularity); + let index = self + .dataset + .load_scalar_index( + IndexCriteria::default() + .for_column(&column) + .supports_fts() + .with_fts_document_granularity(document_granularity), + ) + .await?; + let target_fragments: &[Fragment] = self + .fragments + .as_deref() + .unwrap_or_else(|| self.dataset.fragments()); + if self.fragments.as_ref().is_some_and(Vec::is_empty) { + return Ok(Arc::new(EmptyExec::new(output_schema))); + } + let flat_query = MatchQuery::new(query.terms.clone()) + .with_column(Some(column.clone())) + .with_operator(Operator::And) + .with_document_granularity(document_granularity); + let flat_params = params.clone().with_phrase_slop(Some(query.slop)); + + let (phrase_plan, flat_phrase_plan) = match &index { + Some(index) => { + let unindexed_fragments = self + .retain_target_fragments(self.dataset.unindexed_fragments(&index.name).await?); + if !target_fragments.is_empty() + && unindexed_fragments.len() == target_fragments.len() + { + if self.fast_search { + return Ok(Arc::new(EmptyExec::new(output_schema))); + } + let flat_phrase_plan = self + .plan_flat_match_query( + unindexed_fragments, + HashMap::new(), + &flat_query, + &flat_params, + filter_plan, + None, + ) + .await?; + return Self::combine_fts_leaf_plans(None, Some(flat_phrase_plan), params); + } - let segments = load_segments(&self.dataset, &column) - .await? - .ok_or(Error::invalid_input(format!( - "No Inverted index found for column {}", - column - )))?; - let details = load_segment_details(&self.dataset, &column, &segments).await?; + let (stale_rows, preset_segments) = match self + .fts_overlay_plan(&column, document_granularity, target_fragments) + .await? + { + FtsOverlayPlan::Unchanged(segments) => (HashMap::new(), segments), + FtsOverlayPlan::RowLevel { + stale_rows, + segments, + } => (stale_rows, Some(segments)), + FtsOverlayPlan::FullScan => { + if self.fast_search { + return Ok(Arc::new(EmptyExec::new(output_schema))); + } + let flat_phrase_plan = self + .plan_flat_match_query( + target_fragments.to_vec(), + HashMap::new(), + &flat_query, + &flat_params, + filter_plan, + None, + ) + .await?; + return Self::combine_fts_leaf_plans(None, Some(flat_phrase_plan), params); + } + }; + let overlay_block = self.stale_rows_block_mask(&stale_rows).await?; + let segments = match preset_segments { + Some(segments) => segments, + None => load_segments(&self.dataset, &column, document_granularity) + .await? + .ok_or_else(|| { + Error::internal(format!( + "FTS metadata routed column {column} without loadable segments" + )) + })?, + }; + let details = load_segment_details(&self.dataset, &column, &segments).await?; + if !details.with_position { + return Err(Error::invalid_input("position is not found but required for phrase queries, try recreating the index with position" + .to_string())); + } - if !details.with_position { - return Err(Error::invalid_input("position is not found but required for phrase queries, try recreating the index with position" - .to_string())); - } + let has_flat_path = !self.fast_search + && (!unindexed_fragments.is_empty() || !stale_rows.is_empty()); + let shared_scorer = (has_flat_path && document_granularity.is_list_element()) + .then(|| Arc::new(SharedFtsScorer::new())); + let mut phrase_exec = PhraseQueryExec::new_with_segments_and_document_granularity( + self.dataset.clone(), + query.clone(), + params.clone(), + prefilter_source.clone(), + segments, + document_granularity, + ); + if let Some(overlay_block) = overlay_block { + phrase_exec = phrase_exec.with_overlay_block(overlay_block); + } + if let Some(shared_scorer) = &shared_scorer { + phrase_exec = phrase_exec.with_shared_scorer(shared_scorer.clone()); + } + let phrase_plan = Some(Arc::new(phrase_exec) as Arc); + let flat_phrase_plan = if has_flat_path { + Some( + self.plan_flat_match_query( + unindexed_fragments, + stale_rows, + &flat_query, + &flat_params, + filter_plan, + shared_scorer, + ) + .await?, + ) + } else { + None + }; + (phrase_plan, flat_phrase_plan) + } + None => { + if target_fragments.is_empty() { + return Ok(Arc::new(EmptyExec::new(output_schema))); + } + if self.fast_search { + return Ok(Arc::new(EmptyExec::new(output_schema))); + } + let flat_phrase_plan = self + .plan_flat_match_query( + target_fragments.to_vec(), + HashMap::new(), + &flat_query, + &flat_params, + filter_plan, + None, + ) + .await?; + (None, Some(flat_phrase_plan)) + } + }; - Ok(Arc::new(PhraseQueryExec::new( - self.dataset.clone(), - query.clone(), - params.clone(), - prefilter_source.clone(), - ))) + Self::combine_fts_leaf_plans(phrase_plan, flat_phrase_plan, params) } async fn plan_match_query( @@ -3488,96 +4360,194 @@ impl Scanner { "the column must be specified in the query".to_string(), ))? .clone(); + let document_granularity = query.document_granularity.ok_or_else(|| { + Error::internal("FTS Match query granularity was not resolved".to_string()) + })?; + resolve_fts_field(self.dataset.schema(), &column, document_granularity)?; + let output_schema = fts_schema(document_granularity); let index = self .dataset - .load_scalar_index(IndexCriteria::default().for_column(&column).supports_fts()) + .load_scalar_index( + IndexCriteria::default() + .for_column(&column) + .supports_fts() + .with_fts_document_granularity(document_granularity), + ) .await?; // Get target fragments - let target_fragments = self + let target_fragments: &[Fragment] = self .fragments - .clone() - .unwrap_or_else(|| self.dataset.fragments().to_vec()); + .as_deref() + .unwrap_or_else(|| self.dataset.fragments()); + if self.fragments.as_ref().is_some_and(Vec::is_empty) { + return Ok(Arc::new(EmptyExec::new(output_schema))); + } let (match_plan, flat_match_plan) = match &index { Some(index) => { - // Get unindexed fragments and filter to target fragments let unindexed_fragments = self .retain_target_fragments(self.dataset.unindexed_fragments(&index.name).await?); - - // If all target fragments are unindexed, skip index entirely - if unindexed_fragments.len() == target_fragments.len() { + if !target_fragments.is_empty() + && unindexed_fragments.len() == target_fragments.len() + { if self.fast_search { - return Ok(Arc::new(EmptyExec::new(FTS_SCHEMA.clone()))); + return Ok(Arc::new(EmptyExec::new(output_schema))); } let flat_match_plan = self - .plan_flat_match_query(unindexed_fragments, query, params, filter_plan) + .plan_flat_match_query( + unindexed_fragments, + HashMap::new(), + query, + params, + filter_plan, + None, + ) .await?; - return Ok(flat_match_plan); + return Self::combine_fts_leaf_plans(None, Some(flat_match_plan), params); } - // Mixed case: use index + flat search for unindexed - let match_plan: Arc = Arc::new(MatchQueryExec::new( - self.dataset.clone(), - query.clone(), - params.clone(), - prefilter_source.clone(), - )); - - if self.fast_search || unindexed_fragments.is_empty() { - (Some(match_plan), None) - } else { - let flat_match_plan = self - .plan_flat_match_query(unindexed_fragments, query, params, filter_plan) - .await?; - (Some(match_plan), Some(flat_match_plan)) + let (stale_rows, preset_segments) = match self + .fts_overlay_plan(&column, document_granularity, target_fragments) + .await? + { + FtsOverlayPlan::Unchanged(segments) => (HashMap::new(), segments), + FtsOverlayPlan::RowLevel { + stale_rows, + segments, + } => (stale_rows, Some(segments)), + FtsOverlayPlan::FullScan => { + if self.fast_search { + return Ok(Arc::new(EmptyExec::new(output_schema))); + } + let flat_match_plan = self + .plan_flat_match_query( + target_fragments.to_vec(), + HashMap::new(), + query, + params, + filter_plan, + None, + ) + .await?; + return Self::combine_fts_leaf_plans(None, Some(flat_match_plan), params); + } + }; + let overlay_block = self.stale_rows_block_mask(&stale_rows).await?; + let has_flat_path = !self.fast_search + && (!unindexed_fragments.is_empty() || !stale_rows.is_empty()); + let shared_scorer = (has_flat_path && document_granularity.is_list_element()) + .then(|| Arc::new(SharedFtsScorer::new())); + let mut match_exec = match preset_segments { + Some(segments) => MatchQueryExec::new_with_segments_and_document_granularity( + self.dataset.clone(), + query.clone(), + params.clone(), + prefilter_source.clone(), + segments, + document_granularity, + ), + None => MatchQueryExec::new_with_document_granularity( + self.dataset.clone(), + query.clone(), + params.clone(), + prefilter_source.clone(), + document_granularity, + ), + }; + if let Some(overlay_block) = overlay_block { + match_exec = match_exec.with_overlay_block(overlay_block); + } + if let Some(shared_scorer) = &shared_scorer { + match_exec = match_exec.with_shared_scorer(shared_scorer.clone()); } + let match_plan = Some(Arc::new(match_exec) as Arc); + let flat_match_plan = if has_flat_path { + Some( + self.plan_flat_match_query( + unindexed_fragments, + stale_rows, + query, + params, + filter_plan, + shared_scorer, + ) + .await?, + ) + } else { + None + }; + (match_plan, flat_match_plan) } None => { + if target_fragments.is_empty() { + return Ok(Arc::new(EmptyExec::new(output_schema))); + } if self.fast_search { - return Ok(Arc::new(EmptyExec::new(FTS_SCHEMA.clone()))); + return Ok(Arc::new(EmptyExec::new(output_schema))); } // No index: flat search all target fragments let flat_match_plan = self - .plan_flat_match_query(target_fragments.clone(), query, params, filter_plan) + .plan_flat_match_query( + target_fragments.to_vec(), + HashMap::new(), + query, + params, + filter_plan, + None, + ) .await?; (None, Some(flat_match_plan)) } }; - // Combine plans - let plan = match (match_plan, flat_match_plan) { - (Some(match_plan), Some(flat_match_plan)) => { - let match_plan = UnionExec::try_new(vec![match_plan, flat_match_plan])?; - let match_plan = Arc::new(RepartitionExec::try_new( - match_plan, - Partitioning::RoundRobinBatch(1), - )?); - let sort_expr = PhysicalSortExpr { - expr: expressions::col(SCORE_COL, match_plan.schema().as_ref())?, - options: SortOptions { - descending: true, - nulls_first: false, - }, - }; - Arc::new(SortExec::new([sort_expr].into(), match_plan).with_fetch(params.limit)) + Self::combine_fts_leaf_plans(match_plan, flat_match_plan, params) + } + + fn combine_fts_leaf_plans( + indexed_plan: Option>, + flat_plan: Option>, + params: &FtsSearchParams, + ) -> Result> { + let plan = match (indexed_plan, flat_plan) { + (Some(indexed_plan), Some(flat_plan)) => { + UnionExec::try_new(vec![indexed_plan, flat_plan])? + } + (Some(indexed_plan), None) => return Ok(indexed_plan), + (None, Some(flat_plan)) if params.limit.is_none() => return Ok(flat_plan), + (None, Some(flat_plan)) => flat_plan, + (None, None) => { + return Err(Error::internal( + "FTS leaf planning produced neither an indexed nor a flat plan".to_string(), + )); } - (Some(match_plan), None) => match_plan, - (None, Some(flat_match_plan)) => flat_match_plan, - (None, None) => unreachable!(), }; - - Ok(plan) + let plan = Arc::new(RepartitionExec::try_new( + plan, + Partitioning::RoundRobinBatch(1), + )?); + let sort_expr = PhysicalSortExpr { + expr: expressions::col(SCORE_COL, plan.schema().as_ref())?, + options: SortOptions { + descending: true, + nulls_first: false, + }, + }; + Ok(Arc::new( + SortExec::new([sort_expr].into(), plan).with_fetch(params.limit), + )) } /// Plan match query on unindexed fragments async fn plan_flat_match_query( &self, fragments: Vec, + stale_rows: HashMap, query: &MatchQuery, params: &FtsSearchParams, filter_plan: &ExprFilterPlan, + shared_scorer: Option>, ) -> Result> { let column = query .column @@ -3586,10 +4556,28 @@ impl Scanner { "the column must be specified in the query".to_string(), ))? .clone(); - - let mut columns = vec![column]; - if let Some(refine_expr) = filter_plan.refine_expr.as_ref() { - columns.extend(Planner::column_names_in_expr(refine_expr)); + let document_granularity = query.document_granularity.ok_or_else(|| { + Error::internal("FTS Match query granularity was not resolved".to_string()) + })?; + let resolved = resolve_fts_field(self.dataset.schema(), &column, document_granularity)?; + let scan_column = if resolved.has_lists() { + resolved.root_column.clone() + } else { + resolved.canonical_path.clone() + }; + let document_column = if resolved.has_lists() { + VALUE_COLUMN_NAME.to_string() + } else { + resolved.canonical_path.clone() + }; + let mut columns = vec![scan_column.clone()]; + let filter_expr = if stale_rows.is_empty() { + filter_plan.refine_expr.as_ref() + } else { + filter_plan.full_expr.as_ref() + }; + if let Some(filter_expr) = filter_expr { + columns.extend(Planner::column_names_in_expr(filter_expr)); } let scan_projection = self .dataset @@ -3597,28 +4585,60 @@ impl Scanner { .with_row_id() .union_columns(&columns, OnMissing::Error)?; - let PlannedFilteredScan { mut plan, .. } = self - .filtered_read( - filter_plan, - scan_projection, - /*make_deletions_null=*/ false, - Some(Arc::new(fragments)), - None, - /*is_prefilter=*/ true, - ) - .await?; + let mut inputs = Vec::with_capacity(2); + if !fragments.is_empty() { + let PlannedFilteredScan { mut plan, .. } = self + .filtered_read( + filter_plan, + scan_projection.clone(), + /*make_deletions_null=*/ false, + Some(Arc::new(fragments)), + None, + /*is_prefilter=*/ true, + ) + .await?; + if let Some(refine_expr) = filter_plan.refine_expr.as_ref() { + plan = Arc::new(LanceFilterExec::try_new(refine_expr.clone(), plan)?); + } + inputs.push(plan); + } - if let Some(refine_expr) = filter_plan.refine_expr.as_ref() { - plan = Arc::new(LanceFilterExec::try_new(refine_expr.clone(), plan)?); + if !stale_rows.is_empty() { + let mut plan = self.stale_rows_take(&stale_rows, scan_projection).await?; + if let Some(filter) = filter_plan.full_expr.as_ref() { + let planner = Planner::new(plan.schema()); + let filter = planner.optimize_expr(filter.clone())?; + plan = Arc::new(LanceFilterExec::try_new(filter, plan)?); + } + inputs.push(plan); } - let flat_match_plan = Arc::new(FlatMatchQueryExec::new( + let mut plan: Arc = match inputs.len() { + 0 => { + return Err(Error::internal( + "flat FTS input requires unindexed fragments or stale rows", + )); + } + 1 => inputs.pop().unwrap(), + _ => UnionExec::try_new(inputs)?, + }; + if resolved.has_lists() { + plan = Arc::new(FtsDocumentExec::new(plan, resolved.clone())); + } else { + plan = self.ensure_column_alias(plan, &document_column)?; + } + let mut flat_match_plan = FlatMatchQueryExec::new_with_document_granularity( self.dataset.clone(), query.clone(), params.clone(), plan, - )); - Ok(flat_match_plan) + document_granularity, + document_column, + ); + if let Some(shared_scorer) = shared_scorer { + flat_match_plan = flat_match_plan.with_shared_scorer(shared_scorer); + } + Ok(Arc::new(flat_match_plan)) } // ANN/KNN search execution node with optional prefilter @@ -3784,9 +4804,24 @@ impl Scanner { "Refine factor cannot be zero".to_string(), )); } + // Mask data overlay files: compute which row addresses within each segment have + // been updated by a newer overlay so their ANN entries may be stale. + // These stale rows are blocked from ANN results via the prefilter and re-scored + // on the targeted flat path below — only the specific stale rows, not the whole + // fragment, so sparse overlays incur near-zero overhead. + let stale_rows = self.overlay_stale_vector_rows(&index_segments)?; + // Build a prefilter block mask for stale rows (empty = no-op fast path). + let overlay_block = self.stale_rows_block_mask(&stale_rows).await?; + let ann_node = match vector_type { - DataType::FixedSizeList(_, _) => self.ann(&q, &index_segments, filter_plan).await?, - DataType::List(_) => self.multivec_ann(&q, &index_segments, filter_plan).await?, + DataType::FixedSizeList(_, _) => { + self.ann(&q, &index_segments, filter_plan, overlay_block.clone()) + .await? + } + DataType::List(_) => { + self.multivec_ann(&q, &index_segments, filter_plan, overlay_block.clone()) + .await? + } _ => unreachable!(), }; @@ -3804,7 +4839,14 @@ impl Scanner { if !self.fast_search { knn_node = self - .knn_combined(&q, &index_name, &index_segments, knn_node, filter_plan) + .knn_combined( + &q, + &index_name, + &index_segments, + &stale_rows, + knn_node, + filter_plan, + ) .await?; } @@ -3939,6 +4981,7 @@ impl Scanner { q: &Query, index_name: &str, indexed_segments: &[IndexMetadata], + stale_rows: &HashMap, mut knn_node: Arc, filter_plan: &ExprFilterPlan, ) -> Result> { @@ -3955,30 +4998,41 @@ impl Scanner { self.dataset.unindexed_fragments(index_name).await? }; - if !fallback_fragments.is_empty() { - let q = q.clone(); - debug_assert!(q.metric_type.is_some()); + let has_fallback = !fallback_fragments.is_empty(); + let has_stale = !stale_rows.is_empty(); - // If the vector column is not present, we need to take the vector column, so - // that the distance value is comparable with the flat search ones. - if knn_node.schema().column_with_name(&q.column).is_none() { - let vector_projection = self - .dataset - .empty_projection() - .union_column(&q.column, OnMissing::Error) - .unwrap(); - knn_node = self.take(knn_node, vector_projection)?; - } + if !has_fallback && !has_stale { + return Ok(knn_node); + } - let mut columns = vec![q.column.clone()]; - if let Some(expr) = filter_plan.full_expr.as_ref() { - let filter_columns = Planner::column_names_in_expr(expr); - columns.extend(filter_columns); - } - let vector_scan_projection = Arc::new(self.dataset.schema().project(&columns).unwrap()); - // Note: we could try and use the scalar indices here to reduce the scope of this scan but the - // most common case is that fragments that are newer than the vector index are going to be newer - // than the scalar indices anyways + let q = q.clone(); + debug_assert!(q.metric_type.is_some()); + + // Ensure the vector column is present for distance computation. + if knn_node.schema().column_with_name(&q.column).is_none() { + let vector_projection = self + .dataset + .empty_projection() + .union_column(&q.column, OnMissing::Error)?; + knn_node = self.take(knn_node, vector_projection)?; + } + + let mut columns = vec![q.column.clone()]; + if let Some(expr) = filter_plan.full_expr.as_ref() { + let filter_columns = Planner::column_names_in_expr(expr); + columns.extend(filter_columns); + } + + // Collect flat-path plans; union order matches original (flat before ANN) so test snapshots + // and downstream plan analyses remain stable. + let mut flat_inputs: Vec> = Vec::new(); + + // Flat KNN for unindexed (new-data) fragments. + if has_fallback { + let vector_scan_projection = Arc::new(self.dataset.schema().project(&columns)?); + // Note: we could try and use the scalar indices here to reduce the scope of this scan + // but the most common case is that fragments newer than the vector index are also + // newer than the scalar indices. let mut scan_node = self.scan_fragments( true, false, @@ -3989,39 +5043,51 @@ impl Scanner { Arc::new(fallback_fragments), // Can't pushdown limit/offset in an ANN search None, - // We are re-ordering anyways, so no need to get data in data - // in a deterministic order. + // We are re-ordering anyways, so no need to get data in a deterministic order. false, ); - if let Some(expr) = filter_plan.full_expr.as_ref() { - // If there is a prefilter we need to manually apply it to the new data scan_node = Arc::new(LanceFilterExec::try_new(expr.clone(), scan_node)?); } - // first we do flat search on just the new data - let topk_appended = self.flat_knn(scan_node, &q)?; - - // To do a union, we need to make the schemas match. Right now - // knn_node: _distance, _rowid, vector - // topk_appended: vector, , _rowid, _distance - let topk_appended = project(topk_appended, knn_node.schema().as_ref())?; - assert!( - topk_appended - .schema() - .equivalent_names_and_types(&knn_node.schema()) - ); - // union - let unioned = UnionExec::try_new(vec![Arc::new(topk_appended), knn_node])?; - // Enforce only 1 partition. - let unioned = RepartitionExec::try_new( - unioned, - datafusion::physical_plan::Partitioning::RoundRobinBatch(1), - )?; - // then we do a flat search on KNN(new data) + ANN(indexed data) - return self.flat_knn(Arc::new(unioned), &q); + let topk_fallback = self.flat_knn(scan_node, &q)?; + let topk_fallback: Arc = + Arc::new(project(topk_fallback, knn_node.schema().as_ref())?); + flat_inputs.push(topk_fallback); + } + + // Flat KNN for stale rows only (row-level precision). + // Only specific row addresses need re-scoring, not the whole fragment, so sparse overlays + // incur near-zero overhead. + if has_stale { + // Fetch vector + filter columns for the stale rows. `flat_knn` sorts by row id, so the + // take must carry it (the fallback scan above gets it via `scan_fragments`). + let mut take_proj = self + .dataset + .empty_projection() + .with_row_id() + .union_column(&q.column, OnMissing::Error)?; + if let Some(expr) = filter_plan.full_expr.as_ref() { + let filter_columns = Planner::column_names_in_expr(expr); + take_proj = take_proj.union_columns(filter_columns, OnMissing::Error)?; + } + let mut stale_node = self.stale_rows_take(stale_rows, take_proj).await?; + if let Some(expr) = filter_plan.full_expr.as_ref() { + stale_node = Arc::new(LanceFilterExec::try_new(expr.clone(), stale_node)?); + } + let topk_stale = self.flat_knn(stale_node, &q)?; + let topk_stale: Arc = + Arc::new(project(topk_stale, knn_node.schema().as_ref())?); + flat_inputs.push(topk_stale); } - Ok(knn_node) + // Union: flat paths first (matching original order), then ANN results. + flat_inputs.push(knn_node); + let unioned = UnionExec::try_new(flat_inputs)?; + let unioned = RepartitionExec::try_new( + unioned, + datafusion::physical_plan::Partitioning::RoundRobinBatch(1), + )?; + self.flat_knn(Arc::new(unioned), &q) } #[async_recursion] @@ -4052,29 +5118,238 @@ impl Scanner { } } - /// Given an index query, split the fragments into two sets + /// Given an index query, split the fragments into two groups and collect per-row stale data. + /// + /// - `relevant_frags`: covered by ALL indices. Stale rows within them are returned separately + /// so callers can block them from `MaterializeIndexExec` and re-score via a targeted take. + /// - `missing_frags`: not covered by at least one index; fall back to full scan + filter. + /// - `stale_rows`: per-fragment row offsets whose indexed values are stale due to a data + /// overlay committed after the index was built (field-aware, version-gated). Empty when no + /// overlays are present. + /// + /// There is no point in partially indexing a fragment (some indices cover it, others do not). + /// If we have to do a full scan of a fragment for any reason, we do it entirely. + async fn partition_frags_by_coverage( + &self, + index_expr: &ScalarIndexExpr, + fragments: Arc>, + ) -> Result<(Vec, Vec, HashMap)> { + let covered_frags = self.fragments_covered_by_index_query(index_expr).await?; + let stale_rows = self + .overlay_stale_index_rows(index_expr, &fragments) + .await?; + let mut relevant_frags = Vec::with_capacity(fragments.len()); + let mut missing_frags = Vec::with_capacity(fragments.len()); + for fragment in fragments.iter() { + if covered_frags.contains(fragment.id as u32) { + // Indexed fragments stay on the indexed path. Stale rows within them are blocked + // from the index result and re-evaluated separately via a targeted take. + relevant_frags.push(fragment.clone()); + } else { + missing_frags.push(fragment.clone()); + } + } + Ok((relevant_frags, missing_frags, stale_rows)) + } + + /// Per-row stale offsets for each fragment whose indexed values may be stale because an + /// overlay committed *after* an index was built touches a field that index covers. + /// + /// The check is field-aware (an overlay touching only unindexed fields excludes nothing) and + /// version-gated (an overlay with `committed_version <= index.dataset_version` is already + /// incorporated by the index), via [`overlay_exclusion_offsets`]. + async fn overlay_stale_index_rows( + &self, + index_expr: &ScalarIndexExpr, + fragments: &[Fragment], + ) -> Result> { + // Overlays are rare; skip all index loading when none of the candidate fragments has one. + let overlaid_frags = overlaid_fragments(fragments); + if overlaid_frags.is_empty() { + return Ok(HashMap::new()); + } + + // Walk the (boolean) index expression tree to collect leaf searches. + let mut searches = Vec::new(); + let mut stack = vec![index_expr]; + while let Some(expr) = stack.pop() { + match expr { + ScalarIndexExpr::Not(inner) => stack.push(inner), + ScalarIndexExpr::And(lhs, rhs) | ScalarIndexExpr::Or(lhs, rhs) => { + stack.push(lhs); + stack.push(rhs); + } + ScalarIndexExpr::Query(search) => searches.push(search), + } + } + + // `load_named_scalar_segments` returns cached index metadata — no disk I/O on the hot + // path. Even without the cache, this code is only reached when at least one fragment has + // overlays (rare), so the per-leaf cost is acceptable. + let mut stale: HashMap = HashMap::new(); + for search in searches { + let segments = load_named_scalar_segments( + self.dataset.as_ref(), + &search.column, + &search.index_name, + ) + .await?; + for segment in &segments { + collect_overlay_stale_rows_for_segment( + segment, + &overlaid_frags, + &mut stale, + self.dataset.schema(), + )?; + } + } + Ok(stale) + } + + /// Compute per-row stale data for a vector index's segments. + /// + /// Returns a map from fragment_id to the set of row offsets within that fragment that are stale + /// (their vector values have been updated by a newer overlay since the index was built). An + /// empty map means no stale rows — the fast path where no masking is needed. + fn overlay_stale_vector_rows( + &self, + segments: &[IndexMetadata], + ) -> Result> { + // Scope to the query's target fragments (all dataset fragments if unscoped). + let dataset_frags = self.dataset.fragments(); + let fragments: &[Fragment] = match self.fragments.as_ref() { + Some(f) => f.as_slice(), + None => dataset_frags.as_slice(), + }; + let overlaid_frags = overlaid_fragments(fragments); + if overlaid_frags.is_empty() { + return Ok(HashMap::new()); + } + let mut stale: HashMap = HashMap::new(); + for segment in segments { + collect_overlay_stale_rows_for_segment( + segment, + &overlaid_frags, + &mut stale, + self.dataset.schema(), + )?; + } + Ok(stale) + } + + /// Plan FTS overlay handling at row granularity. + /// + /// Modern segments remain searchable while their overlay-stale rows are blocked and + /// re-evaluated from current values. A legacy segment without fragment coverage falls back + /// to a full target scan when a relevant overlay exists because its indexed row set is + /// unknown. + async fn fts_overlay_plan( + &self, + column: &str, + document_granularity: DocumentGranularity, + target_fragments: &[Fragment], + ) -> Result { + if target_fragments.iter().all(|f| f.overlays.is_empty()) { + return Ok(FtsOverlayPlan::Unchanged(None)); + } + + let Some(segments) = load_segments(&self.dataset, column, document_granularity).await? + else { + return Ok(FtsOverlayPlan::Unchanged(None)); + }; + + let overlaid_frags = overlaid_fragments(target_fragments); + let mut stale_rows = HashMap::new(); + for segment in &segments { + if segment.fragment_bitmap.is_none() { + let mut legacy_stale_rows = HashMap::new(); + collect_overlay_stale_rows_for_segment( + segment, + &overlaid_frags, + &mut legacy_stale_rows, + self.dataset.schema(), + )?; + if !legacy_stale_rows.is_empty() { + return Ok(FtsOverlayPlan::FullScan); + } + } else { + collect_overlay_stale_rows_for_segment( + segment, + &overlaid_frags, + &mut stale_rows, + self.dataset.schema(), + )?; + } + } + + if stale_rows.is_empty() { + Ok(FtsOverlayPlan::Unchanged(Some(segments))) + } else { + Ok(FtsOverlayPlan::RowLevel { + stale_rows, + segments, + }) + } + } + + /// Collect the stale rows into a [`RowAddrTreeMap`] in the domain the index results use. + /// + /// Index results are in the row-id domain (see `ScalarQuery::evaluate_nullable`), and a + /// physical row address equals its row id only when the dataset does not use stable row ids. + /// Under stable row ids the addresses are translated to their row ids so the result lines up + /// with the index output it is combined with (block mask) or taken against. + async fn stale_rows_in_id_domain( + &self, + stale_rows: &HashMap, + ) -> Result { + let mut tree_map = RowAddrTreeMap::new(); + for (&frag_id, offsets) in stale_rows { + tree_map.insert_bitmap(frag_id, offsets.clone()); + } + if self.dataset.manifest.uses_stable_row_ids() { + tree_map = translate_addr_treemap_to_row_ids(&self.dataset, &tree_map).await?; + } + Ok(tree_map) + } + + /// Build a block-list mask over stale rows, or `None` when there are none. + /// + /// The mask removes these rows from an index result so the index never emits them; they + /// are re-evaluated against their current (overlay-merged) values on a targeted take path + /// (see [`Self::stale_rows_take`]). + async fn stale_rows_block_mask( + &self, + stale_rows: &HashMap, + ) -> Result> { + if stale_rows.is_empty() { + return Ok(None); + } + let tree_map = self.stale_rows_in_id_domain(stale_rows).await?; + Ok(Some(RowAddrMask::from_block(tree_map))) + } + + /// Take the stale rows by physical address, projecting `projection`, to re-evaluate only + /// those rows (rather than their whole fragments) against their current overlay-merged values. /// - /// The first set is the relevant fragments, which are covered by ALL indices in the query - /// The second set is the missing fragments, which are missed by at least one index - /// - /// There is no point in handling the case where a fragment is covered by some (but not all) - /// of the indices. If we have to do a full scan of the fragment then we do it - async fn partition_frags_by_coverage( + /// The rows are identified by an address allow list routed through `FilteredReadExec`, not a + /// `_rowid` column (`_rowid` and row address diverge under stable row ids — see + /// [`Self::stale_rows_in_id_domain`]). + async fn stale_rows_take( &self, - index_expr: &ScalarIndexExpr, - fragments: Arc>, - ) -> Result<(Vec, Vec)> { - let covered_frags = self.fragments_covered_by_index_query(index_expr).await?; - let mut relevant_frags = Vec::with_capacity(fragments.len()); - let mut missing_frags = Vec::with_capacity(fragments.len()); - for fragment in fragments.iter() { - if covered_frags.contains(fragment.id as u32) { - relevant_frags.push(fragment.clone()); - } else { - missing_frags.push(fragment.clone()); - } + stale_rows: &HashMap, + projection: Projection, + ) -> Result> { + let take_id_map = self.stale_rows_in_id_domain(stale_rows).await?; + let index_input = self.row_ids_as_take_input(take_id_map)?; + let mut read_options = FilteredReadOptions::new(projection); + if let Some(fragments) = self.fragments.as_ref() { + read_options = read_options.with_fragments(Arc::new(fragments.clone())); } - Ok((relevant_frags, missing_frags)) + Ok(Arc::new(FilteredReadExec::try_new( + self.dataset.clone(), + read_options, + Some(index_input), + )?)) } // First perform a lookup in a scalar index for ids and then perform a take on the @@ -4096,16 +5371,24 @@ impl Scanner { let needs_recheck = index_expr.needs_recheck(); - // Figure out which fragments are covered by ALL indices - let (relevant_frags, missing_frags) = self + // Figure out which fragments are covered by ALL indices, and which rows within + // covered fragments are stale due to data overlay files. + let (relevant_frags, missing_frags, stale_rows) = self .partition_frags_by_coverage(index_expr, fragments) .await?; - let mut plan: Arc = Arc::new(MaterializeIndexExec::new( + // Build the MaterializeIndexExec, blocking stale row addresses so the index never + // emits them. Stale rows are re-scored separately via a targeted take below. + let mat_exec = MaterializeIndexExec::new( self.dataset.clone(), index_expr.clone(), Arc::new(relevant_frags), - )); + ); + let mat_exec = match self.stale_rows_block_mask(&stale_rows).await? { + Some(block) => mat_exec.with_overlay_block(block), + None => mat_exec, + }; + let mut plan: Arc = Arc::new(mat_exec); let refine_expr = filter_plan.refine_expr.as_ref(); @@ -4151,6 +5434,21 @@ impl Scanner { plan = Arc::new(AddRowAddrExec::try_new(plan, self.dataset.clone(), 0)?); } + // Both the missing-fragments path (full scan) and the stale-rows path (targeted take) + // need the user's projection extended with any filter columns. Compute it once. + let fallback_projection: Option = + if !missing_frags.is_empty() || !stale_rows.is_empty() { + let filter = filter_plan.full_expr.as_ref().expect_ok()?; + let filter_cols = Planner::column_names_in_expr(filter); + Some( + projection + .clone() + .union_columns(filter_cols, OnMissing::Error)?, + ) + } else { + None + }; + let new_data_path: Option> = if !missing_frags.is_empty() { log::trace!( "scalar_indexed_scan will need full scan of {} missing fragments", @@ -4169,10 +5467,8 @@ impl Scanner { // If there were no extra columns then we still need the project // because Materialize -> Take puts the row id at the left and // Scan puts the row id at the right - let filter = filter_plan.full_expr.as_ref().unwrap(); - let filter_cols = Planner::column_names_in_expr(filter); - let scan_projection = projection.union_columns(filter_cols, OnMissing::Error)?; - + let scan_projection = fallback_projection.clone().expect_ok()?; + let filter = filter_plan.full_expr.as_ref().expect_ok()?; let scan_schema = Arc::new(scan_projection.to_bare_schema()); let scan_arrow_schema = Arc::new(scan_schema.as_ref().into()); let planner = Planner::new(scan_arrow_schema); @@ -4201,16 +5497,37 @@ impl Scanner { None }; - if let Some(new_data_path) = new_data_path { - let unioned = UnionExec::try_new(vec![plan, new_data_path])?; - // Enforce only 1 partition. - let unioned = Arc::new(RepartitionExec::try_new( - unioned, - datafusion::physical_plan::Partitioning::RoundRobinBatch(1), - )?); - Ok(unioned) + // Stale-Take path: re-evaluate only the stale row addresses against the full filter + // (row-level optimization). These rows were blocked from the index result above; + // here we take their current (overlay-merged) values and re-apply the predicate. + // The schema matches `plan` via `project(…, plan.schema())`. + let stale_take_path: Option> = if stale_rows.is_empty() { + None } else { + let filter = filter_plan.full_expr.as_ref().expect_ok()?; + let take_projection = fallback_projection.expect_ok()?; + + let stale_node = self.stale_rows_take(&stale_rows, take_projection).await?; + + let planner = Planner::new(stale_node.schema()); + let optimized_filter = planner.optimize_expr(filter.clone())?; + let filtered = Arc::new(LanceFilterExec::try_new(optimized_filter, stale_node)?); + Some(Arc::new(project(filtered, plan.schema().as_ref())?)) + }; + + let extra_paths: Vec> = [new_data_path, stale_take_path] + .into_iter() + .flatten() + .collect(); + if extra_paths.is_empty() { Ok(plan) + } else { + let all_paths = std::iter::once(plan).chain(extra_paths).collect(); + let unioned = UnionExec::try_new(all_paths)?; + Ok(Arc::new(RepartitionExec::try_new( + unioned, + datafusion::physical_plan::Partitioning::RoundRobinBatch(1), + )?)) } } @@ -4328,8 +5645,8 @@ impl Scanner { )?)) } - /// Here we use a full text search as a post-filter. Any rows that - /// do not contain at least one query token are removed. + /// Here we use a full text search as a post-filter. Rows are retained + /// according to the match query's token operator. /// /// Only valid (currently) for match queries. async fn flat_fts_filter( @@ -4337,14 +5654,9 @@ impl Scanner { input: Arc, q: &FullTextSearchQuery, ) -> Result> { - let fts_query = if q.columns().is_empty() { - let indexed_columns = fts_indexed_columns(self.dataset.clone()).await?; - fill_fts_query_column(&q.query, &indexed_columns, false)? - } else { - q.query.clone() - }; + let fts_query = &q.query; - match &fts_query { + match fts_query { FtsQuery::Match(match_query) => { let schema = Arc::new((input.schema()).try_with_column(SCORE_FIELD.clone())?); @@ -4355,21 +5667,37 @@ impl Scanner { "the column must be specified in the query".to_string(), ))? .clone(); - let input = if schema.column_with_name(&column).is_none() { + let document_granularity = match_query.document_granularity.ok_or_else(|| { + Error::internal("FTS Match query granularity was not resolved".to_string()) + })?; + let resolved = + resolve_fts_field(self.dataset.schema(), &column, document_granularity)?; + let scan_column = if resolved.has_lists() { + resolved.root_column.clone() + } else { + resolved.canonical_path.clone() + }; + let input = if schema.column_with_name(&scan_column).is_none() { let projection = self .dataset .empty_projection() - .union_column(&column, OnMissing::Error)?; - self.take(input, projection)? + .union_column(&scan_column, OnMissing::Error)?; + let input = self.take(input, projection)?; + if resolved.has_lists() { + input + } else { + self.ensure_column_alias(input, &scan_column)? + } } else { input }; - Ok(Arc::new(FlatMatchFilterExec::new( + Ok(Arc::new(FlatMatchFilterExec::new_with_resolved_field( input, self.dataset.clone(), match_query.clone(), q.params(), + resolved, ))) } _ => Err(Error::not_supported( @@ -4387,14 +5715,9 @@ impl Scanner { input: Arc, q: &FullTextSearchQuery, ) -> Result> { - let fts_query = if q.columns().is_empty() { - let indexed_columns = fts_indexed_columns(self.dataset.clone()).await?; - fill_fts_query_column(&q.query, &indexed_columns, false)? - } else { - q.query.clone() - }; + let fts_query = &q.query; - match &fts_query { + match fts_query { FtsQuery::Match(match_query) => { let schema = Arc::new((input.schema()).try_with_column(SCORE_FIELD.clone())?); @@ -4405,21 +5728,48 @@ impl Scanner { "the column must be specified in the query".to_string(), ))? .clone(); - let input = if schema.column_with_name(&column).is_none() { + let document_granularity = match_query.document_granularity.ok_or_else(|| { + Error::internal("FTS Match query granularity was not resolved".to_string()) + })?; + let resolved = + resolve_fts_field(self.dataset.schema(), &column, document_granularity)?; + let scan_column = if resolved.has_lists() { + resolved.root_column.clone() + } else { + resolved.canonical_path.clone() + }; + let document_column = if resolved.has_lists() { + VALUE_COLUMN_NAME.to_string() + } else { + resolved.canonical_path.clone() + }; + let input = if schema.column_with_name(&scan_column).is_none() { let projection = self .dataset .empty_projection() - .union_column(&column, OnMissing::Error)?; - self.take(input, projection)? + .union_column(&scan_column, OnMissing::Error)?; + let input = self.take(input, projection)?; + if resolved.has_lists() { + input + } else { + self.ensure_column_alias(input, &document_column)? + } + } else { + input + }; + let input = if resolved.has_lists() { + Arc::new(FtsDocumentExec::new(input, resolved)) as Arc } else { input }; - Ok(Arc::new(FlatMatchQueryExec::new( + Ok(Arc::new(FlatMatchQueryExec::new_with_document_granularity( self.dataset.clone(), match_query.clone(), q.params(), input, + document_granularity, + document_column, ))) } _ => { @@ -4642,11 +5992,18 @@ impl Scanner { q: &Query, index: &[IndexMetadata], filter_plan: &ExprFilterPlan, + overlay_block: Option, ) -> Result> { let prefilter_source = self .prefilter_source(filter_plan, self.get_indexed_frags(index)) .await?; - let inner_fanout_search = new_knn_exec(self.dataset.clone(), index, q, prefilter_source)?; + let inner_fanout_search = new_knn_exec( + self.dataset.clone(), + index, + q, + prefilter_source, + overlay_block, + )?; let sort_expr = PhysicalSortExpr { expr: expressions::col(DIST_COL, inner_fanout_search.schema().as_ref())?, options: SortOptions { @@ -4673,6 +6030,7 @@ impl Scanner { q: &Query, index: &[IndexMetadata], filter_plan: &ExprFilterPlan, + overlay_block: Option, ) -> Result> { // we split the query procedure into two steps: // 1. collect the candidates by vector searching on each query vector @@ -4705,6 +6063,7 @@ impl Scanner { index, &query, prefilter_source.clone(), + overlay_block.clone(), )?; let sort_expr = PhysicalSortExpr { expr: expressions::col(DIST_COL, ann_node.schema().as_ref())?, @@ -4785,11 +6144,14 @@ impl Scanner { // are not in the fragments we are scanning. if filter_plan.is_exact_index_search() && self.fragments.is_none() { let index_query = filter_plan.index_query.as_ref().expect_ok()?; - let (_, missing_frags) = self + let (_, missing_frags, stale_rows) = self .partition_frags_by_coverage(index_query, fragments.clone()) .await?; - if missing_frags.is_empty() || self.fast_search { + // Overlay-stale rows must never reach the direct ScalarIndexExec path: it would hand + // ANN/FTS a selection vector containing rows whose indexed values are now stale. When + // any exist, fall through to the filtered-read prefilter, which masks them. + if stale_rows.is_empty() && (missing_frags.is_empty() || self.fast_search) { log::trace!("prefilter entirely satisfied by exact index search"); let result_format = self.index_expr_result_format(); // We can only avoid materializing the index for a prefilter if: @@ -4824,11 +6186,80 @@ impl Scanner { } /// Take row indices produced by input plan from the dataset (with projection) + /// + /// Planned as a [`FilteredReadExec`] row-stream read; legacy (v1) storage + /// keeps using [`TakeExec`]. #[allow(deprecated)] fn take( &self, input: Arc, output_projection: Projection, + ) -> Result> { + let fields_to_take = output_projection + .clone() + .subtract_arrow_schema(input.schema().as_ref(), OnMissing::Ignore)?; + if !fields_to_take.has_data_fields() + && !fields_to_take.with_row_id + && !fields_to_take.with_row_addr + { + // No new columns needed + return Ok(input); + } + + versions::take( + self.dataset + .manifest() + .data_storage_format + .lance_file_format(), + self, + input, + output_projection, + ) + } + + pub(super) fn take_current( + &self, + input: Arc, + output_projection: Projection, + ) -> Result> { + let input_schema = input.schema(); + let has_row_id = input_schema.column_with_name(ROW_ID).is_some(); + let has_row_addr = input_schema.column_with_name(ROW_ADDR).is_some(); + if has_row_id || has_row_addr { + // Pass the full (un-subtracted) target so a rebuild against a + // different child re-derives what to fetch, and preserve carried + // identity columns (downstream nodes may key off them; the final + // ProjectionExec trims for free) + let mut projection = output_projection; + projection.with_row_id |= has_row_id; + projection.with_row_addr |= has_row_addr; + let mut read_options = FilteredReadOptions::new(projection); + if self.include_deleted_rows { + // Forwarded so the row-stream read rejects it: deleted rows + // carry a null row id, which the take would silently drop + read_options = read_options.with_deleted_rows()?; + } + if let Some(batch_size) = self.batch_size { + read_options = read_options.with_batch_size(batch_size as u32); + } + if let Some(fragments) = &self.fragments { + read_options = read_options.with_fragments(Arc::new(fragments.clone())); + } + return Ok(Arc::new(FilteredReadExec::try_new( + self.dataset.clone(), + read_options, + Some(input), + )?)); + } + + self.take_legacy(input, output_projection) + } + + #[allow(deprecated)] + pub(super) fn take_legacy( + &self, + input: Arc, + output_projection: Projection, ) -> Result> { let coalesced = Arc::new(CoalesceBatchesExec::new( input.clone(), @@ -4868,7 +6299,12 @@ impl Scanner { #[instrument(level = "info", skip(self))] pub async fn explain_plan(&self, verbose: bool) -> Result { - let plan = self.create_plan().await?; + // Box the plan-building future at the call site: `create_plan`'s inlined async + // layout otherwise exceeds rustc's depth limit here. It has to be boxed at the + // call site rather than inside `create_plan` — boxing internally turns the + // future's `Send` check into a `Box: Send` trait obligation that + // overflows the solver through the cache types (E0275 in downstream crates). + let plan = Box::pin(self.create_plan()).await?; let display = DisplayableExecutionPlan::new(plan.as_ref()); Ok(format!("{}", display.indent(verbose))) @@ -4896,47 +6332,42 @@ impl Scanner { } // Search over all indexed fields including nested ones, collecting columns that have an -// inverted index +// inverted index. Automatic discovery is intentionally restricted to Row +// granularity because ListElement queries require an explicit field path. async fn fts_indexed_columns(dataset: Arc) -> Result> { let mut indexed_columns = Vec::new(); - for field in dataset.schema().fields_pre_order() { - // Check if this field is a string type that could have an inverted index - let is_string_field = match field.data_type() { - DataType::Utf8 | DataType::LargeUtf8 => true, - DataType::List(inner_field) | DataType::LargeList(inner_field) => { - matches!( - inner_field.data_type(), - DataType::Utf8 | DataType::LargeUtf8 - ) - } - _ => false, + for index in dataset.load_indices().await?.iter() { + let Some(field_id) = index.fields.first().copied() else { + continue; }; - - if is_string_field { - // Build the full field path for nested fields - let column_path = - if let Some(ancestors) = dataset.schema().field_ancestry_by_id(field.id) { - let field_refs: Vec<&str> = ancestors.iter().map(|f| f.name.as_str()).collect(); - format_field_path(&field_refs) - } else { - continue; // Skip if we can't find the field ancestry - }; - - // Check if this field has an inverted index - let has_fts_index = dataset - .load_scalar_index( - IndexCriteria::default() - .for_column(&column_path) - .supports_fts(), - ) - .await? - .is_some(); - - if has_fts_index { - indexed_columns.push(column_path); - } + let Ok(preliminary_path) = dataset.schema().field_path(field_id) else { + continue; + }; + let details = + crate::index::scalar::fetch_index_details(dataset.as_ref(), &preliminary_path, index) + .await?; + if !details.type_url.ends_with("InvertedIndexDetails") { + continue; } + let details = lance_index::pbold::InvertedIndexDetails::decode(details.value.as_slice()) + .map_err(|error| { + Error::io(format!( + "failed to decode InvertedIndexDetails payload: {error}" + )) + })?; + if DocumentGranularity::try_from(details.document_granularity)? != DocumentGranularity::Row + { + continue; + } + let resolved = crate::index::scalar::inverted::resolve_fts_field_by_id( + dataset.schema(), + field_id, + DocumentGranularity::Row, + )?; + indexed_columns.push(resolved.canonical_path); } + indexed_columns.sort(); + indexed_columns.dedup(); Ok(indexed_columns) } @@ -5195,14 +6626,43 @@ pub mod test_dataset { Ok(()) } + fn fts_index_params() -> InvertedIndexParams { + // These scanner tests search for the token "s" (from the `s-{N}` + // column values) to exercise fragment/append coverage, and "s" is + // in the full English stop-word list. Keep the token searchable; + // stop-word behavior itself is covered by the tokenizer tests. + InvertedIndexParams::default() + .with_position(true) + .remove_stop_words(false) + } + pub async fn make_fts_index(&mut self) -> Result<()> { - let params = InvertedIndexParams::default().with_position(true); + let params = Self::fts_index_params(); self.dataset .create_index(&["s"], IndexType::Inverted, None, ¶ms, true) .await?; Ok(()) } + pub async fn make_segmented_fts_index(&mut self) -> Result<()> { + let params = Self::fts_index_params(); + let fragments = self.dataset.get_fragments(); + let mut segments = Vec::with_capacity(fragments.len()); + for fragment in fragments { + let segment = self + .dataset + .create_index_builder(&["s"], IndexType::Inverted, ¶ms) + .name("s_idx".to_string()) + .fragments(vec![fragment.id() as u32]) + .execute_uncommitted() + .await?; + segments.push(segment); + } + self.dataset + .commit_existing_index_segments("s_idx", "s", segments) + .await + } + pub async fn append_new_data(&mut self) -> Result<()> { self.append_data_with_range(400, 410).await } @@ -5263,7 +6723,9 @@ mod test { }; use lance_file::version::LanceFileVersion; use lance_index::optimize::OptimizeOptions; - use lance_index::scalar::inverted::query::{MatchQuery, PhraseQuery}; + use lance_index::scalar::inverted::query::{ + BooleanQuery, BoostQuery, FtsQuery, MatchQuery, Occur, PhraseQuery, + }; use lance_index::vector::hnsw::builder::HnswBuildParams; use lance_index::vector::ivf::IvfBuildParams; use lance_index::vector::pq::PQBuildParams; @@ -5279,14 +6741,36 @@ mod test { use super::*; use crate::dataset::WriteMode; - use crate::dataset::WriteParams; use crate::dataset::optimize::{CompactionOptions, compact_files}; use crate::dataset::scanner::test_dataset::TestVectorDataset; + use crate::dataset::{NewColumnTransform, WriteParams}; use crate::index::vector::{StageParams, VectorIndexParams}; use crate::utils::test::{ DatagenExt, FragmentCount, FragmentRowCount, ThrottledStoreWrapper, assert_plan_node_equals, }; + #[test] + fn test_fts_query_contract_rejects_invalid_values() { + let negative_match = FtsQuery::Match(MatchQuery::new("hello".to_string()).with_boost(-1.0)); + let error = validate_fts_query_contract(&negative_match).unwrap_err(); + assert!(matches!(error, Error::InvalidInput { .. })); + assert!(error.to_string().contains("finite and non-negative")); + + let empty_boolean = FtsQuery::Boolean(BooleanQuery::new(Vec::<(Occur, FtsQuery)>::new())); + let error = validate_fts_query_contract(&empty_boolean).unwrap_err(); + assert!(matches!(error, Error::InvalidInput { .. })); + assert!(error.to_string().contains("at least one should/must query")); + + let infinite_boost = FtsQuery::Boost(BoostQuery::new( + MatchQuery::new("hello".to_string()).into(), + MatchQuery::new("world".to_string()).into(), + Some(f32::INFINITY), + )); + let error = validate_fts_query_contract(&infinite_boost).unwrap_err(); + assert!(matches!(error, Error::InvalidInput { .. })); + assert!(error.to_string().contains("BoostQuery negative_boost")); + } + #[test] fn test_env_var_parsing() { // Test that invalid environment variable values don't panic @@ -5472,6 +6956,120 @@ mod test { } } + #[tokio::test] + async fn test_batch_size_bytes_across_data_files() { + let num_rows = 300; + let schema = Arc::new(ArrowSchema::new(vec![ArrowField::new( + "id", + DataType::Int64, + false, + )])); + let batch = RecordBatch::try_new( + schema.clone(), + vec![Arc::new(Int64Array::from_iter_values(0..num_rows))], + ) + .unwrap(); + let test_dir = TempStrDir::default(); + let mut dataset = Dataset::write( + RecordBatchIterator::new([Ok(batch)], schema), + &test_dir, + Some(WriteParams { + data_storage_version: Some(LanceFileVersion::V2_1), + max_rows_per_file: num_rows as usize + 1, + ..Default::default() + }), + ) + .await + .unwrap(); + + let wide_value = "abcdefghij".repeat(6); + let wide_schema = Arc::new(ArrowSchema::new(vec![ArrowField::new( + "wide", + DataType::Utf8, + false, + )])); + let wide_batch = RecordBatch::try_new( + wide_schema.clone(), + vec![Arc::new(StringArray::from_iter_values( + (0..num_rows).map(|_| wide_value.as_str()), + ))], + ) + .unwrap(); + dataset + .add_columns( + NewColumnTransform::Reader(Box::new(RecordBatchIterator::new( + [Ok(wide_batch)], + wide_schema, + ))), + None, + None, + ) + .await + .unwrap(); + assert_eq!(dataset.get_fragment(0).unwrap().num_data_files(), 2); + + let target_bytes = 8 * 1024; + let mut scan = dataset.scan(); + scan.project(&["id", "wide"]) + .unwrap() + .batch_size_bytes(target_bytes); + let batches = scan + .try_into_stream() + .await + .unwrap() + .try_collect::>() + .await + .unwrap(); + assert_eq!( + batches.iter().map(RecordBatch::num_rows).sum::(), + num_rows as usize + ); + for batch in &batches { + let wide = batch["wide"].as_string::(); + let values_bytes = (*wide.value_offsets().last().unwrap() + - *wide.value_offsets().first().unwrap()) as usize; + let logical_bytes = batch.num_rows() * std::mem::size_of::() + + std::mem::size_of_val(wide.value_offsets()) + + values_bytes; + assert!( + logical_bytes <= target_bytes as usize, + "batch has {logical_bytes} logical bytes, target is {target_bytes}" + ); + } + + let mut scan = dataset.scan(); + scan.project(&["id", "wide"]) + .unwrap() + .batch_size(10) + .batch_size_bytes(target_bytes); + let batches = scan + .try_into_stream() + .await + .unwrap() + .try_collect::>() + .await + .unwrap(); + assert!(batches.iter().all(|batch| batch.num_rows() <= 10)); + + let mut scan = dataset.scan(); + scan.project(&["id", "wide"]) + .unwrap() + .batch_size(200) + .batch_size_bytes(target_bytes) + .strict_batch_size(true); + let error = scan + .try_into_stream() + .await + .err() + .expect("strict row and byte batch limits should be rejected"); + assert!( + error + .to_string() + .contains("strict_batch_size=true cannot be combined with batch_size_bytes=8192"), + "unexpected error: {error}" + ); + } + #[tokio::test] async fn test_strict_batch_size() { let dataset = lance_datagen::gen_batch() @@ -5616,16 +7214,159 @@ mod test { .await?; let batch = concat_batches(&batches[0].schema(), &batches)?; - let expected_batch = RecordBatch::try_new( - // Projected just "s" - Arc::new(test_ds.schema.project(&[1])?), - vec![Arc::new(StringArray::from_iter_values( - (51..400).map(|v| format!("s-{}", v)), - ))], - )?; - assert_eq!(batch, expected_batch); - } - Ok(()) + let expected_batch = RecordBatch::try_new( + // Projected just "s" + Arc::new(test_ds.schema.project(&[1])?), + vec![Arc::new(StringArray::from_iter_values( + (51..400).map(|v| format!("s-{}", v)), + ))], + )?; + assert_eq!(batch, expected_batch); + } + Ok(()) + } + + // Regression for #6580: a scan with `filter` + `project` of a + // `(Large)List` column used to panic in `merge_with_schema` + // (called from `TakeStream::map_batch`) because the filtered batch arrived + // as a sliced view of a larger batch and the cloned list offsets did not + // start at zero. The trigger requires (a) a `(Large)List` + // projection where the struct is split across `filtered_read` and + // `TakeExec` and (b) a sparse-tail selectivity pattern so the trailing + // filter result lands deep inside the values buffer of its source batch. + // Parametrized over `List`/`LargeList` since the fix touches both offset + // widths in `merge_with_schema`. + #[rstest] + #[tokio::test] + async fn test_filter_project_list_struct_sparse_tail( + // The panic is specific to v2.x storage; the legacy reader takes a + // different code path. V2_0 and V2_2 are the versions called out in + // the original report. + #[values( + LanceFileVersion::V2_0, + LanceFileVersion::Stable, + LanceFileVersion::V2_2 + )] + data_storage_version: LanceFileVersion, + #[values(false, true)] large_list: bool, + ) { + use arrow_array::{LargeListArray, ListArray, UInt16Array}; + use arrow_buffer::{OffsetBuffer, ScalarBuffer}; + + let struct_fields = Fields::from(vec![ + Arc::new(ArrowField::new("a", DataType::Int32, true)), + Arc::new(ArrowField::new("b", DataType::Int32, true)), + ]); + let item_field = Arc::new(ArrowField::new( + "item", + DataType::Struct(struct_fields.clone()), + true, + )); + let items_dtype = if large_list { + DataType::LargeList(item_field.clone()) + } else { + DataType::List(item_field.clone()) + }; + let schema = Arc::new(ArrowSchema::new(vec![ + ArrowField::new("id", DataType::Int32, false), + ArrowField::new("grp", DataType::UInt16, false), + ArrowField::new("items", items_dtype, false), + ])); + + let make_batch = |start: i32, n: usize, group: u16| -> RecordBatch { + let ids = Int32Array::from_iter_values(start..start + n as i32); + let groups = UInt16Array::from(vec![group; n]); + + let mut offsets = Vec::with_capacity(n + 1); + let mut a_vals: Vec = Vec::new(); + let mut b_vals: Vec = Vec::new(); + offsets.push(0i64); + for i in 0..n { + // Variable-length lists (1..=18) so offsets don't land on + // batch-row boundaries. + let len = 1 + (i % 18); + for j in 0..len { + a_vals.push(j as i32); + b_vals.push(-(j as i32)); + } + offsets.push(a_vals.len() as i64); + } + let struct_arr = Arc::new(StructArray::new( + struct_fields.clone(), + vec![ + Arc::new(Int32Array::from(a_vals)) as ArrayRef, + Arc::new(Int32Array::from(b_vals)) as ArrayRef, + ], + None, + )); + let items: ArrayRef = if large_list { + Arc::new(LargeListArray::new( + item_field.clone(), + OffsetBuffer::new(ScalarBuffer::from(offsets)), + struct_arr, + None, + )) + } else { + let offsets_i32: Vec = offsets.iter().map(|&o| o as i32).collect(); + Arc::new(ListArray::new( + item_field.clone(), + OffsetBuffer::new(ScalarBuffer::from(offsets_i32)), + struct_arr, + None, + )) + }; + RecordBatch::try_new( + schema.clone(), + vec![ + Arc::new(ids) as ArrayRef, + Arc::new(groups) as ArrayRef, + items, + ], + ) + .unwrap() + }; + + // Sparse-tail selectivity (matching the original report's shape at a + // smaller scale): a large leading block of matches, a large gap of + // non-matches, then a small trailing match. Single fragment. + let batches = vec![ + make_batch(0, 100_000, 7), + make_batch(100_000, 400_000, 1), + make_batch(500_000, 7_300, 7), + ]; + + let reader = RecordBatchIterator::new(batches.into_iter().map(Ok), schema.clone()); + let dataset = Dataset::write( + reader, + "memory://", + Some(WriteParams { + max_rows_per_file: 1_000_000, + data_storage_version: Some(data_storage_version), + ..Default::default() + }), + ) + .await + .unwrap(); + + // Force a column split inside the `items` struct by marking `items.b` + // as a late-materialized field: `filtered_read` returns the batch with + // `items.a`, and `TakeExec` adds `items.b`. `merge_with_schema` then + // takes its `List` branch, which is where the panic was. + let items_b_field_id = dataset + .schema() + .field("items") + .unwrap() + .child("item") + .unwrap() + .child("b") + .unwrap() + .id as u32; + let mut scan = dataset.scan(); + scan.filter("grp = 7").unwrap(); + scan.project(&["id", "items"]).unwrap(); + scan.materialization_style(MaterializationStyle::AllEarlyExcept(vec![items_b_field_id])); + let result = scan.try_into_batch().await.unwrap(); + assert_eq!(result.num_rows(), 107_300); } #[tokio::test] @@ -6878,6 +8619,57 @@ mod test { assert_eq!(expected_i, actual_i); } + #[tokio::test(flavor = "multi_thread")] + async fn test_flat_knn_large_limit_preserves_global_order() { + // Regression test for https://github.com/lance-format/lance/issues/7865. + // + // An exact (flat, no vector index) KNN search with a limit larger than one + // output batch (BATCH_SIZE_FALLBACK = 8192 rows) used to be able to return + // results in the wrong global order: `execute_plan` coalesced the + // partitions the physical optimizer parallelizes above the top-k `SortExec` + // with a plain `CoalescePartitionsExec`, which does not preserve order. + // This only reproduces at real (> 1) parallelism, which is why the + // plan-shape tests elsewhere (pinned to `target_parallelism(1)`) never + // caught it. + let dim = 16u32; + let frag_count = 4u32; + let rows_per_fragment = 5_000u32; + let k = 12_000usize; // > BATCH_SIZE_FALLBACK, so results span multiple batches + + let dataset = gen_batch() + .col("vec", array::rand_vec::(Dimension::from(dim))) + .into_ram_dataset( + FragmentCount::from(frag_count), + FragmentRowCount::from(rows_per_fragment), + ) + .await + .unwrap(); + + let query = Float32Array::from(vec![0.0_f32; dim as usize]); + + // The bug is a scheduling race between parallel partitions, so run + // several iterations to reliably catch it if the ordering guarantee + // regresses. + for _ in 0..10 { + let mut scan = dataset.scan(); + scan.nearest("vec", &query, k).unwrap(); + scan.target_parallelism(8); + + let batch = scan.try_into_batch().await.unwrap(); + assert_eq!(batch.num_rows(), k); + + let distances = batch[DIST_COL].as_primitive::(); + for pair in distances.values().windows(2) { + assert!( + pair[0] <= pair[1], + "flat KNN results must be globally sorted by distance, found {} before {}", + pair[0], + pair[1] + ); + } + } + } + #[rstest] #[tokio::test] async fn test_refine_factor( @@ -7107,6 +8899,30 @@ mod test { assert_eq!(expected_row_ids, actual_row_ids); } + #[tokio::test] + async fn test_filter_legacy_dataset_with_stable_row_ids() { + let test_ds = TestVectorDataset::new(LanceFileVersion::Legacy, true) + .await + .unwrap(); + + let batch = test_ds + .dataset + .scan() + .batch_readahead(get_num_compute_intensive_cpus()) + .project(&["vec"]) + .unwrap() + .with_row_id() + .filter_expr(col("vec").is_not_null()) + .try_into_batch() + .await + .unwrap(); + + let row_ids = batch[ROW_ID].as_primitive::(); + let unique_row_ids = row_ids.values().iter().copied().collect::>(); + assert_eq!(unique_row_ids.len(), 400); + assert_eq!(row_ids.len(), unique_row_ids.len()); + } + #[tokio::test] async fn test_scan_unordered_with_row_id() { // This test doesn't make sense for v2 files, there is no way to get an out-of-order scan @@ -7439,8 +9255,20 @@ mod test { #[values(LanceFileVersion::Legacy, LanceFileVersion::Stable)] data_storage_version: LanceFileVersion, #[values(false, true)] stable_row_ids: bool, + #[values(ApproxMode::Normal, ApproxMode::Fast)] approx_mode: ApproxMode, #[values( VectorIndexParams::ivf_pq(2, 8, 2, MetricType::L2, 2), + VectorIndexParams::ivf_hnsw( + MetricType::L2, + IvfBuildParams::new(2), + HnswBuildParams::default() + ), + VectorIndexParams::with_ivf_hnsw_pq_params( + MetricType::L2, + IvfBuildParams::new(2), + HnswBuildParams::default(), + PQBuildParams::new(2, 8) + ), VectorIndexParams::with_ivf_hnsw_sq_params( MetricType::L2, IvfBuildParams::new(2), @@ -7495,6 +9323,7 @@ mod test { scan.nearest("vector", query_key.as_ref(), 1).unwrap(); scan.minimum_nprobes(100); scan.ef(100); + scan.approx_mode(approx_mode); scan.with_row_id(); let batches = scan @@ -8387,6 +10216,14 @@ mod test { } } + fn uses_legacy_scan(&self) -> bool { + self.dataset + .manifest() + .data_storage_format + .lance_file_format() + == lance_file::version::ConcreteFileVersion::V1 + } + async fn check_vector_scalar_indexed_and_refine(&self, params: &ScalarTestParams) { let (query_plan, batch) = self .run_query( @@ -8396,7 +10233,7 @@ mod test { ) .await; // Materialization is always required if there is a refine - if self.dataset.is_legacy_storage() { + if self.uses_legacy_scan() { assert!(query_plan.contains("MaterializeIndex")); } // The result should not include the sample query @@ -8428,7 +10265,7 @@ mod test { let (query_plan, batch) = self .run_query("indexed != 50", Some(self.sample_query()), params) .await; - if self.dataset.is_legacy_storage() { + if self.uses_legacy_scan() { if params.use_index { // An ANN search whose prefilter is fully satisfied by the index should be // able to use a ScalarIndexQuery @@ -8477,7 +10314,7 @@ mod test { async fn check_simple_indexed_only(&self, params: &ScalarTestParams) { let (query_plan, batch) = self.run_query("indexed != 50", None, params).await; // Materialization is always required for non-vector search - if self.dataset.is_legacy_storage() { + if self.uses_legacy_scan() { assert!(query_plan.contains("MaterializeIndex")); } else { assert!(query_plan.contains("LanceRead")); @@ -8518,7 +10355,7 @@ mod test { params ).await; // Materialization is always required for non-vector search - if self.dataset.is_legacy_storage() { + if self.uses_legacy_scan() { assert!(query_plan.contains("MaterializeIndex")); } else { assert!(query_plan.contains("LanceRead")); @@ -8871,6 +10708,56 @@ mod test { "expected ngram index usage for `{filter}`, got plan:\n{plan_str}" ); } + + // contains with >= 3 characters (uses index) + assert_eq!( + matched(&dataset, "contains(s, 'rhino')").await, + ["rhino", "rhino horn", "rhinos nose"] + ); + assert_eq!( + matched(&dataset, "contains(s, 'cat')").await, + ["cat", "cat dog", "catalog", "category", "scatter"] + ); + + // contains with < 3 characters (must NOT use index, i.e., falls back to full scan, and returns correct results) + assert_eq!( + matched(&dataset, "contains(s, 'ca')").await, + ["cat", "cat dog", "catalog", "category", "scatter"] + ); + assert_eq!( + matched(&dataset, "contains(s, 'a')").await, + [ + "cat", "cat dog", "catalog", "category", "dogma", "elephant", "scatter" + ] + ); + + // Verify index is used for contains >= 3 characters, but NOT used for < 3 characters + for filter in ["contains(s, 'rhino')", "contains(s, 'cat')"] { + let mut scan = dataset.scan(); + scan.filter(filter).unwrap(); + let plan = scan.create_plan().await.unwrap(); + let plan_str = format!( + "{}", + datafusion::physical_plan::displayable(plan.as_ref()).indent(true) + ); + assert!( + plan_str.contains("ScalarIndexQuery") && plan_str.contains("NGram"), + "expected ngram index usage for `{filter}`, got plan:\n{plan_str}" + ); + } + for filter in ["contains(s, 'ca')", "contains(s, 'a')"] { + let mut scan = dataset.scan(); + scan.filter(filter).unwrap(); + let plan = scan.create_plan().await.unwrap(); + let plan_str = format!( + "{}", + datafusion::physical_plan::displayable(plan.as_ref()).indent(true) + ); + assert!( + !plan_str.contains("ScalarIndexQuery"), + "expected NO ngram index usage for `{filter}`, got plan:\n{plan_str}" + ); + } } #[tokio::test] @@ -8926,6 +10813,85 @@ mod test { assert_eq!(count(&dataset, "regexp_like(text, 'a.b')").await, 20); } + #[tokio::test] + async fn test_ngram_contains_untokenizable_needle() { + // A needle the tokenizer cannot turn into a trigram must fall back to a + // full recheck rather than silently matching nothing. Byte length is + // not a usable proxy for this: "éé" is two characters but four bytes. + let unit = ["ééb", "aéé", "abc", "dog", "éab"]; + let values: Vec<&str> = unit.iter().copied().cycle().take(60).collect(); + let array = StringArray::from_iter_values(values); + let schema = Arc::new(ArrowSchema::new(vec![ArrowField::new( + "text", + DataType::Utf8, + false, + )])); + let batch = RecordBatch::try_new(schema.clone(), vec![Arc::new(array)]).unwrap(); + let reader = RecordBatchIterator::new(vec![Ok(batch)], schema); + let write_params = WriteParams { + max_rows_per_file: 20, // 60 rows -> 3 fragments + ..Default::default() + }; + let mut dataset = Dataset::write( + reader, + "memory://test_ngram_contains_utf8", + Some(write_params), + ) + .await + .unwrap(); + dataset + .create_index( + &["text"], + IndexType::NGram, + None, + &ScalarIndexParams::default(), + true, + ) + .await + .unwrap(); + + async fn count(dataset: &Dataset, filter: &str) -> usize { + let mut scan = dataset.scan(); + scan.filter(filter).unwrap(); + let batches = scan + .try_into_stream() + .await + .unwrap() + .try_collect::>() + .await + .unwrap(); + batches.iter().map(|b| b.num_rows()).sum() + } + + async fn plan_of(dataset: &Dataset, filter: &str) -> String { + let mut scan = dataset.scan(); + scan.filter(filter).unwrap(); + let plan = scan.create_plan().await.unwrap(); + format!( + "{}", + datafusion::physical_plan::displayable(plan.as_ref()).indent(true) + ) + } + + // Each unit value appears 12 times in the 60 rows. + // Two characters (four bytes): shorter than a trigram, so the index is + // bypassed and "ééb" / "aéé" are still found. + assert_eq!(count(&dataset, "contains(text, 'éé')").await, 24); + let plan_str = plan_of(&dataset, "contains(text, 'éé')").await; + assert!( + !plan_str.contains("ScalarIndexQuery"), + "expected NO ngram index usage for a two character needle, got plan:\n{plan_str}" + ); + + // Three characters: long enough to tokenize, so the index is used. + assert_eq!(count(&dataset, "contains(text, 'ééb')").await, 12); + let plan_str = plan_of(&dataset, "contains(text, 'ééb')").await; + assert!( + plan_str.contains("ScalarIndexQuery") && plan_str.contains("NGram"), + "expected ngram index usage for a three character needle, got plan:\n{plan_str}" + ); + } + #[tokio::test] async fn test_like_prefix_with_btree_index() { // Create dataset with string data that has various prefixes @@ -9978,6 +11944,30 @@ full_filter=name LIKE Utf8(\"test%2\"), refine_filter=name LIKE Utf8(\"test%2\") ); } + /// A physical deleted-row scan cannot late-materialize: the take would + /// silently drop the tombstone rows (null row id), so the row-stream + /// read must reject the forwarded flag at plan time + #[tokio::test] + async fn test_include_deleted_rows_rejects_late_materialization() { + let data = gen_batch() + .col("i", array::step::()) + .col("payload", array::step::()) + .into_reader_rows(RowCount::from(100), BatchCount::from(1)); + let mut dataset = Dataset::write(data, "memory://test", None).await.unwrap(); + dataset.delete("i = 5").await.unwrap(); + + let mut scan = dataset.scan(); + scan.project(&["payload"]) + .unwrap() + .filter("i > 2") + .unwrap() + .with_row_id() + .include_deleted_rows() + .materialization_style(MaterializationStyle::AllLate); + let err = scan.create_plan().await.unwrap_err(); + assert!(err.to_string().contains("with_deleted_rows"), "{err}"); + } + #[rstest] #[tokio::test] async fn test_late_materialization( @@ -10079,6 +12069,154 @@ full_filter=name LIKE Utf8(\"test%2\"), refine_filter=name LIKE Utf8(\"test%2\") assert_io_lt!(io_stats, read_bytes, index_scan_bytes); } + #[tokio::test] + async fn test_blob_all_binary_late_materialization() { + // A selective filter that projects a blob column with `blob_handling=all_binary` + // must late-materialize the blob (take only the matched rows) rather than eagerly + // reading the whole column. Blobs returned as descriptions stay eager (they are + // tiny), but full binary values should follow the width-based heuristic like any + // other wide column. + use lance_io::assert_io_lt; + use lance_table::io::commit::RenameCommitHandler; + + // 8KB stays under the 64KB inline threshold, so the blob is a normal column in + // the data file rather than a dedicated blob file. + let blob_meta = HashMap::from([("lance-encoding:blob".to_string(), "true".to_string())]); + let blobs = array::rand_fixedbin(ByteCount::from(8 * 1024), true).with_metadata(blob_meta); + let data = gen_batch() + .col("filterme", array::step::()) + .col("blobs", blobs) + .into_reader_rows(RowCount::from(500), BatchCount::from(8)); + + let dataset = Dataset::write( + data, + "memory://test", + Some(WriteParams { + commit_handler: Some(Arc::new(RenameCommitHandler)), + data_storage_version: Some(LanceFileVersion::Stable), + ..Default::default() + }), + ) + .await + .unwrap(); + + // Baseline: read the blob column as binary for the whole table. + let _ = dataset.object_store.as_ref().io_stats_incremental(); // reset + dataset + .scan() + .project(&["blobs"]) + .unwrap() + .blob_handling(BlobHandling::AllBinary) + .try_into_batch() + .await + .unwrap(); + let full_scan_bytes = dataset + .object_store + .as_ref() + .io_stats_incremental() + .read_bytes; + + // A filter matching a single row out of 4000 should read far less than the whole + // column: only the filter leaf plus the one materialized blob. + dataset + .scan() + .project(&["blobs"]) + .unwrap() + .blob_handling(BlobHandling::AllBinary) + .filter("filterme = 100") + .unwrap() + .try_into_batch() + .await + .unwrap(); + let io_stats = dataset.object_store.as_ref().io_stats_incremental(); + assert_io_lt!(io_stats, read_bytes, full_scan_bytes); + } + + #[tokio::test] + async fn test_nested_blob_all_binary_late_materialization() { + // Same as above, but the blob is a leaf *inside* a struct and the filter is on a + // sibling leaf. Materialization is decided per leaf (fields_pre_order), so the + // nested blob must late-materialize under `all_binary` just like a top-level one. + use lance_io::assert_io_lt; + use lance_table::io::commit::RenameCommitHandler; + + let blob_meta = HashMap::from([("lance-encoding:blob".to_string(), "true".to_string())]); + let a_field = ArrowField::new("a", DataType::Int32, false); + let blob_field = + ArrowField::new("blob", DataType::LargeBinary, false).with_metadata(blob_meta); + let struct_fields: Fields = vec![a_field, blob_field].into(); + let schema = Arc::new(ArrowSchema::new(vec![ArrowField::new( + "s", + DataType::Struct(struct_fields.clone()), + false, + )])); + + let rows_per_batch = 500usize; + let batches: Vec = (0..8) + .map(|b| { + let base = (b * rows_per_batch) as i32; + let a = Arc::new(Int32Array::from_iter_values( + base..base + rows_per_batch as i32, + )); + // Vary the payload per row so it does not collapse under compression. + let blobs: Vec> = (0..rows_per_batch) + .map(|r| { + let seed = (base as usize + r).wrapping_mul(2654435761); + (0usize..8 * 1024) + .map(|i| (i.wrapping_mul(31).wrapping_add(seed) & 0xff) as u8) + .collect() + }) + .collect(); + let blob = Arc::new(arrow_array::LargeBinaryArray::from_iter_values( + blobs.iter().map(|v| v.as_slice()), + )); + let s = StructArray::new(struct_fields.clone(), vec![a, blob as ArrayRef], None); + RecordBatch::try_new(schema.clone(), vec![Arc::new(s)]).unwrap() + }) + .collect(); + + let reader = RecordBatchIterator::new(batches.into_iter().map(Ok), schema.clone()); + let dataset = Dataset::write( + reader, + "memory://test", + Some(WriteParams { + commit_handler: Some(Arc::new(RenameCommitHandler)), + data_storage_version: Some(LanceFileVersion::Stable), + ..Default::default() + }), + ) + .await + .unwrap(); + + let _ = dataset.object_store.as_ref().io_stats_incremental(); // reset + dataset + .scan() + .project(&["s"]) + .unwrap() + .blob_handling(BlobHandling::AllBinary) + .try_into_batch() + .await + .unwrap(); + let full_scan_bytes = dataset + .object_store + .as_ref() + .io_stats_incremental() + .read_bytes; + + dataset + .scan() + .project(&["s"]) + .unwrap() + .blob_handling(BlobHandling::AllBinary) + .filter("s.a = 100") + .unwrap() + .try_into_batch() + .await + .unwrap(); + let io_stats = dataset.object_store.as_ref().io_stats_incremental(); + assert_io_lt!(io_stats, read_bytes, full_scan_bytes); + } + #[rstest] #[tokio::test] async fn test_project_nested( @@ -10184,10 +12322,9 @@ full_filter=name LIKE Utf8(\"test%2\"), refine_filter=name LIKE Utf8(\"test%2\") FilterExec: i@0 > 10 AND i@0 < 20 LanceScan: uri..., projection=[i], row_id=true, row_addr=false, ordered=true, range=None" } else { - "ProjectionExec: expr=[s@2 as s] - Take: columns=\"i, _rowid, (s)\" - CoalesceBatchesExec: target_batch_size=8192 - LanceRead: ..., projection=[i], num_fragments=2, range_before=None, range_after=None, row_id=true, row_addr=false, full_filter=i > Int32(10) AND i < Int32(20), refine_filter=i > Int32(10) AND i < Int32(20)" + "ProjectionExec: expr=[s@2 as s] + LanceRead: uri=..., projection=[s], source=stream(_rowid) + LanceRead: ..., projection=[i], num_fragments=2, range_before=None, range_after=None, row_id=true, row_addr=false, full_filter=i > Int32(10) AND i < Int32(20), refine_filter=i > Int32(10) AND i < Int32(20)" }; assert_plan_equals( &dataset.dataset, @@ -10211,10 +12348,9 @@ full_filter=name LIKE Utf8(\"test%2\"), refine_filter=name LIKE Utf8(\"test%2\") LanceScan: uri..., projection=[i, s], row_id=true, row_addr=false, ordered=true, range=None" } else { "ProjectionExec: expr=[i@0 as i, s@1 as s, vec@3 as vec] - Take: columns=\"i, s, _rowid, (vec)\" - CoalesceBatchesExec: target_batch_size=8192 - LanceRead: uri=..., projection=[i, s], num_fragments=2, range_before=None, range_after=None, \ - row_id=true, row_addr=false, full_filter=s IS NOT NULL, refine_filter=s IS NOT NULL" + LanceRead: uri=..., projection=[vec], source=stream(_rowid) + LanceRead: uri=..., projection=[i, s], num_fragments=2, range_before=None, range_after=None, \ + row_id=true, row_addr=false, full_filter=s IS NOT NULL, refine_filter=s IS NOT NULL" }; assert_plan_equals( &dataset.dataset, @@ -10254,10 +12390,9 @@ full_filter=name LIKE Utf8(\"test%2\"), refine_filter=name LIKE Utf8(\"test%2\") LanceScan: uri..., projection=[s], row_id=true, row_addr=false, ordered=true, range=None" } else { "ProjectionExec: expr=[i@2 as i, s@0 as s, vec@3 as vec] - Take: columns=\"s, _rowid, (i), (vec)\" - CoalesceBatchesExec: target_batch_size=8192 - LanceRead: uri=..., projection=[s], num_fragments=2, range_before=None, \ - range_after=None, row_id=true, row_addr=false, full_filter=s IS NOT NULL, refine_filter=s IS NOT NULL" + LanceRead: uri=..., projection=[i, vec], source=stream(_rowid) + LanceRead: uri=..., projection=[s], num_fragments=2, range_before=None, \ + range_after=None, row_id=true, row_addr=false, full_filter=s IS NOT NULL, refine_filter=s IS NOT NULL" }; assert_plan_equals( &dataset.dataset, @@ -10279,10 +12414,9 @@ full_filter=name LIKE Utf8(\"test%2\"), refine_filter=name LIKE Utf8(\"test%2\") LanceScan: uri..., projection=[s, vec], row_id=true, row_addr=false, ordered=true, range=None" } else { "ProjectionExec: expr=[i@3 as i, s@0 as s, vec@1 as vec] - Take: columns=\"s, vec, _rowid, (i)\" - CoalesceBatchesExec: target_batch_size=8192 - LanceRead: uri=..., projection=[s, vec], num_fragments=2, range_before=None, range_after=None, \ - row_id=true, row_addr=false, full_filter=s IS NOT NULL, refine_filter=s IS NOT NULL" + LanceRead: uri=..., projection=[i], source=stream(_rowid) + LanceRead: uri=..., projection=[s, vec], num_fragments=2, range_before=None, range_after=None, \ + row_id=true, row_addr=false, full_filter=s IS NOT NULL, refine_filter=s IS NOT NULL" }; assert_plan_equals( &dataset.dataset, @@ -10325,13 +12459,12 @@ full_filter=name LIKE Utf8(\"test%2\"), refine_filter=name LIKE Utf8(\"test%2\") LanceScan: uri=..., projection=[vec], row_id=true, row_addr=false, ordered=false, range=None" } else { "ProjectionExec: expr=[i@3 as i, s@4 as s, vec@0 as vec, _distance@2 as _distance] - Take: columns=\"vec, _rowid, _distance, (i), (s)\" - CoalesceBatchesExec: target_batch_size=8192 - FilterExec: _distance@2 IS NOT NULL - SortExec: TopK(fetch=5), expr=... - KNNVectorDistance: metric=l2 - LanceRead: uri=..., projection=[vec], num_fragments=2, range_before=None, range_after=None, \ - row_id=true, row_addr=false, full_filter=--, refine_filter=--" + LanceRead: uri=..., projection=[i, s], source=stream(_rowid) + FilterExec: _distance@2 IS NOT NULL + SortExec: TopK(fetch=5), expr=... + KNNVectorDistance: metric=l2 + LanceRead: uri=..., projection=[vec], num_fragments=2, range_before=None, range_after=None, \ + row_id=true, row_addr=false, full_filter=--, refine_filter=--" }; assert_plan_equals( &dataset.dataset, @@ -10355,14 +12488,13 @@ full_filter=name LIKE Utf8(\"test%2\"), refine_filter=name LIKE Utf8(\"test%2\") LanceScan: uri=..., projection=[vec], row_id=true, row_addr=false, ordered=false, range=None" } else { "ProjectionExec: expr=[i@3 as i, s@4 as s, vec@0 as vec, _distance@2 as _distance] - Take: columns=\"vec, _rowid, _distance, (i), (s)\" - CoalesceBatchesExec: target_batch_size=8192 - GlobalLimitExec: skip=0, fetch=1 - FilterExec: _distance@2 IS NOT NULL - SortExec: TopK(fetch=5), expr=... - KNNVectorDistance: metric=l2 - LanceRead: uri=..., projection=[vec], num_fragments=2, range_before=None, range_after=None, \ - row_id=true, row_addr=false, full_filter=--, refine_filter=--" + LanceRead: uri=..., projection=[i, s], source=stream(_rowid) + GlobalLimitExec: skip=0, fetch=1 + FilterExec: _distance@2 IS NOT NULL + SortExec: TopK(fetch=5), expr=... + KNNVectorDistance: metric=l2 + LanceRead: uri=..., projection=[vec], num_fragments=2, range_before=None, range_after=None, \ + row_id=true, row_addr=false, full_filter=--, refine_filter=--" }; assert_plan_equals( &dataset.dataset, @@ -10375,13 +12507,20 @@ full_filter=name LIKE Utf8(\"test%2\"), refine_filter=name LIKE Utf8(\"test%2\") // --------------------------------------------------------------------- dataset.make_vector_index().await?; log::info!("Test case: Basic ANN"); - let expected = + let expected = if data_storage_version == LanceFileVersion::Legacy { "ProjectionExec: expr=[i@2 as i, s@3 as s, vec@4 as vec, _distance@0 as _distance] Take: columns=\"_distance, _rowid, (i), (s), (vec)\" CoalesceBatchesExec: target_batch_size=8192 SortExec: TopK(fetch=42), expr=... ANNSubIndex: name=..., k=42, deltas=1, metric=L2 - ANNIvfPartition: uuid=..., minimum_nprobes=1, maximum_nprobes=None, deltas=1"; + ANNIvfPartition: uuid=..., minimum_nprobes=1, maximum_nprobes=None, deltas=1" + } else { + "ProjectionExec: expr=[i@2 as i, s@3 as s, vec@4 as vec, _distance@0 as _distance] + LanceRead: uri=..., projection=[i, s, vec], source=stream(_rowid) + SortExec: TopK(fetch=42), expr=... + ANNSubIndex: name=..., k=42, deltas=1, metric=L2 + ANNIvfPartition: uuid=..., minimum_nprobes=1, maximum_nprobes=None, deltas=1" + }; assert_plan_equals( &dataset.dataset, |scan| scan.nearest("vec", &q, 42), @@ -10390,7 +12529,7 @@ full_filter=name LIKE Utf8(\"test%2\"), refine_filter=name LIKE Utf8(\"test%2\") .await?; log::info!("Test case: ANN with refine"); - let expected = + let expected = if data_storage_version == LanceFileVersion::Legacy { "ProjectionExec: expr=[i@3 as i, s@4 as s, vec@1 as vec, _distance@2 as _distance] Take: columns=\"_rowid, vec, _distance, (i), (s)\" CoalesceBatchesExec: target_batch_size=8192 @@ -10401,7 +12540,18 @@ full_filter=name LIKE Utf8(\"test%2\"), refine_filter=name LIKE Utf8(\"test%2\") CoalesceBatchesExec: target_batch_size=8192 SortExec: TopK(fetch=40), expr=... ANNSubIndex: name=..., k=40, deltas=1, metric=L2 - ANNIvfPartition: uuid=..., minimum_nprobes=1, maximum_nprobes=None, deltas=1"; + ANNIvfPartition: uuid=..., minimum_nprobes=1, maximum_nprobes=None, deltas=1" + } else { + "ProjectionExec: expr=[i@3 as i, s@4 as s, vec@1 as vec, _distance@2 as _distance] + LanceRead: uri=..., projection=[i, s], source=stream(_rowid) + FilterExec: _distance@... IS NOT NULL + SortExec: TopK(fetch=10), expr=... + KNNVectorDistance: metric=l2 + LanceRead: uri=..., projection=[vec], source=stream(_rowid) + SortExec: TopK(fetch=40), expr=... + ANNSubIndex: name=..., k=40, deltas=1, metric=L2 + ANNIvfPartition: uuid=..., minimum_nprobes=1, maximum_nprobes=None, deltas=1" + }; assert_plan_equals( &dataset.dataset, |scan| Ok(scan.nearest("vec", &q, 10)?.refine(4)), @@ -10421,13 +12571,12 @@ full_filter=name LIKE Utf8(\"test%2\"), refine_filter=name LIKE Utf8(\"test%2\") LanceScan: uri=..., projection=[vec], row_id=true, row_addr=false, ordered=false, range=None" } else { "ProjectionExec: expr=[i@3 as i, s@4 as s, vec@0 as vec, _distance@2 as _distance] - Take: columns=\"vec, _rowid, _distance, (i), (s)\" - CoalesceBatchesExec: target_batch_size=8192 - FilterExec: _distance@... IS NOT NULL - SortExec: TopK(fetch=13), expr=... - KNNVectorDistance: metric=l2 - LanceRead: uri=..., projection=[vec], num_fragments=2, range_before=None, range_after=None, \ - row_id=true, row_addr=false, full_filter=--, refine_filter=--" + LanceRead: uri=..., projection=[i, s], source=stream(_rowid) + FilterExec: _distance@... IS NOT NULL + SortExec: TopK(fetch=13), expr=... + KNNVectorDistance: metric=l2 + LanceRead: uri=..., projection=[vec], num_fragments=2, range_before=None, range_after=None, \ + row_id=true, row_addr=false, full_filter=--, refine_filter=--" }; assert_plan_equals( &dataset.dataset, @@ -10437,7 +12586,8 @@ full_filter=name LIKE Utf8(\"test%2\"), refine_filter=name LIKE Utf8(\"test%2\") .await?; log::info!("Test case: ANN with postfilter"); - let expected = "ProjectionExec: expr=[s@3 as s, vec@4 as vec, _distance@0 as _distance, _rowid@1 as _rowid] + let expected = if data_storage_version == LanceFileVersion::Legacy { + "ProjectionExec: expr=[s@3 as s, vec@4 as vec, _distance@0 as _distance, _rowid@1 as _rowid] Take: columns=\"_distance, _rowid, i, (s), (vec)\" CoalesceBatchesExec: target_batch_size=8192 FilterExec: i@2 > 10 @@ -10445,7 +12595,16 @@ full_filter=name LIKE Utf8(\"test%2\"), refine_filter=name LIKE Utf8(\"test%2\") CoalesceBatchesExec: target_batch_size=8192 SortExec: TopK(fetch=17), expr=... ANNSubIndex: name=..., k=17, deltas=1, metric=L2 - ANNIvfPartition: uuid=..., minimum_nprobes=1, maximum_nprobes=None, deltas=1"; + ANNIvfPartition: uuid=..., minimum_nprobes=1, maximum_nprobes=None, deltas=1" + } else { + "ProjectionExec: expr=[s@3 as s, vec@4 as vec, _distance@0 as _distance, _rowid@1 as _rowid] + LanceRead: uri=..., projection=[s, vec], source=stream(_rowid) + FilterExec: i@2 > 10 + LanceRead: uri=..., projection=[i], source=stream(_rowid) + SortExec: TopK(fetch=17), expr=... + ANNSubIndex: name=..., k=17, deltas=1, metric=L2 + ANNIvfPartition: uuid=..., minimum_nprobes=1, maximum_nprobes=None, deltas=1" + }; assert_plan_equals( &dataset.dataset, |scan| { @@ -10471,13 +12630,12 @@ full_filter=name LIKE Utf8(\"test%2\"), refine_filter=name LIKE Utf8(\"test%2\") LanceScan: uri=..., projection=[i], row_id=true, row_addr=false, ordered=false, range=None" } else { "ProjectionExec: expr=[i@2 as i, s@3 as s, vec@4 as vec, _distance@0 as _distance] - Take: columns=\"_distance, _rowid, (i), (s), (vec)\" - CoalesceBatchesExec: target_batch_size=8192 - SortExec: TopK(fetch=17), expr=... - ANNSubIndex: name=..., k=17, deltas=1, metric=L2 - ANNIvfPartition: uuid=..., minimum_nprobes=1, maximum_nprobes=None, deltas=1 - LanceRead: uri=..., projection=[], num_fragments=2, range_before=None, range_after=None, \ - row_id=true, row_addr=false, full_filter=i > Int32(10), refine_filter=i > Int32(10) + LanceRead: uri=..., projection=[i, s, vec], source=stream(_rowid) + SortExec: TopK(fetch=17), expr=... + ANNSubIndex: name=..., k=17, deltas=1, metric=L2 + ANNIvfPartition: uuid=..., minimum_nprobes=1, maximum_nprobes=None, deltas=1 + LanceRead: uri=..., projection=[], num_fragments=2, range_before=None, range_after=None, \ + row_id=true, row_addr=false, full_filter=i > Int32(10), refine_filter=i > Int32(10) " }; assert_plan_equals( @@ -10494,7 +12652,8 @@ full_filter=name LIKE Utf8(\"test%2\"), refine_filter=name LIKE Utf8(\"test%2\") dataset.append_new_data().await?; log::info!("Test case: Combined KNN/ANN"); - let expected = "ProjectionExec: expr=[i@3 as i, s@4 as s, vec@1 as vec, _distance@2 as _distance] + let expected = if data_storage_version == LanceFileVersion::Legacy { + "ProjectionExec: expr=[i@3 as i, s@4 as s, vec@1 as vec, _distance@2 as _distance] Take: columns=\"_rowid, vec, _distance, (i), (s)\" CoalesceBatchesExec: target_batch_size=8192 FilterExec: _distance@... IS NOT NULL @@ -10511,7 +12670,25 @@ full_filter=name LIKE Utf8(\"test%2\"), refine_filter=name LIKE Utf8(\"test%2\") CoalesceBatchesExec: target_batch_size=8192 SortExec: TopK(fetch=6), expr=... ANNSubIndex: name=..., k=6, deltas=1, metric=L2 - ANNIvfPartition: uuid=..., minimum_nprobes=1, maximum_nprobes=None, deltas=1"; + ANNIvfPartition: uuid=..., minimum_nprobes=1, maximum_nprobes=None, deltas=1" + } else { + "ProjectionExec: expr=[i@3 as i, s@4 as s, vec@1 as vec, _distance@2 as _distance] + LanceRead: uri=..., projection=[i, s], source=stream(_rowid) + FilterExec: _distance@... IS NOT NULL + SortExec: TopK(fetch=6), expr=... + KNNVectorDistance: metric=l2 + CoalescePartitionsExec + UnionExec + ProjectionExec: expr=[_distance@2 as _distance, _rowid@1 as _rowid, vec@0 as vec] + FilterExec: _distance@... IS NOT NULL + SortExec: TopK(fetch=6), expr=... + KNNVectorDistance: metric=l2 + LanceScan: uri=..., projection=[vec], row_id=true, row_addr=false, ordered=false, range=None + LanceRead: uri=..., projection=[vec], source=stream(_rowid) + SortExec: TopK(fetch=6), expr=... + ANNSubIndex: name=..., k=6, deltas=1, metric=L2 + ANNIvfPartition: uuid=..., minimum_nprobes=1, maximum_nprobes=None, deltas=1" + }; assert_plan_equals( &dataset.dataset, |scan| scan.nearest("vec", &q, 6), @@ -10523,7 +12700,8 @@ full_filter=name LIKE Utf8(\"test%2\"), refine_filter=name LIKE Utf8(\"test%2\") // new data and with filter log::info!("Test case: Combined KNN/ANN with postfilter"); - let expected = "ProjectionExec: expr=[i@3 as i, s@4 as s, vec@1 as vec, _distance@2 as _distance] + let expected = if data_storage_version == LanceFileVersion::Legacy { + "ProjectionExec: expr=[i@3 as i, s@4 as s, vec@1 as vec, _distance@2 as _distance] Take: columns=\"_rowid, vec, _distance, i, (s)\" CoalesceBatchesExec: target_batch_size=8192 FilterExec: i@3 > 10 @@ -10543,7 +12721,27 @@ full_filter=name LIKE Utf8(\"test%2\"), refine_filter=name LIKE Utf8(\"test%2\") CoalesceBatchesExec: target_batch_size=8192 SortExec: TopK(fetch=15), expr=... ANNSubIndex: name=..., k=15, deltas=1, metric=L2 - ANNIvfPartition: uuid=..., minimum_nprobes=1, maximum_nprobes=None, deltas=1"; + ANNIvfPartition: uuid=..., minimum_nprobes=1, maximum_nprobes=None, deltas=1" + } else { + "ProjectionExec: expr=[i@3 as i, s@4 as s, vec@1 as vec, _distance@2 as _distance] + LanceRead: uri=..., projection=[s], source=stream(_rowid) + FilterExec: i@3 > 10 + LanceRead: uri=..., projection=[i], source=stream(_rowid) + FilterExec: _distance@... IS NOT NULL + SortExec: TopK(fetch=15), expr=... + KNNVectorDistance: metric=l2 + CoalescePartitionsExec + UnionExec + ProjectionExec: expr=[_distance@2 as _distance, _rowid@1 as _rowid, vec@0 as vec] + FilterExec: _distance@... IS NOT NULL + SortExec: TopK(fetch=15), expr=... + KNNVectorDistance: metric=l2 + LanceScan: uri=..., projection=[vec], row_id=true, row_addr=false, ordered=false, range=None + LanceRead: uri=..., projection=[vec], source=stream(_rowid) + SortExec: TopK(fetch=15), expr=... + ANNSubIndex: name=..., k=15, deltas=1, metric=L2 + ANNIvfPartition: uuid=..., minimum_nprobes=1, maximum_nprobes=None, deltas=1" + }; assert_plan_equals( &dataset.dataset, |scan| scan.nearest("vec", &q, 15)?.filter("i > 10"), @@ -10577,26 +12775,24 @@ full_filter=name LIKE Utf8(\"test%2\"), refine_filter=name LIKE Utf8(\"test%2\") LanceScan: uri=..., projection=[i], row_id=true, row_addr=false, ordered=false, range=None" } else { "ProjectionExec: expr=[i@3 as i, s@4 as s, vec@1 as vec, _distance@2 as _distance] - Take: columns=\"_rowid, vec, _distance, (i), (s)\" - CoalesceBatchesExec: target_batch_size=8192 - FilterExec: _distance@... IS NOT NULL - SortExec: TopK(fetch=5), expr=... - KNNVectorDistance: metric=l2 - CoalescePartitionsExec - UnionExec - ProjectionExec: expr=[_distance@3 as _distance, _rowid@2 as _rowid, vec@0 as vec] - FilterExec: _distance@... IS NOT NULL - SortExec: TopK(fetch=5), expr=... - KNNVectorDistance: metric=l2 - FilterExec: i@1 > 10 - LanceScan: uri=..., projection=[vec, i], row_id=true, row_addr=false, ordered=false, range=None - Take: columns=\"_distance, _rowid, (vec)\" - CoalesceBatchesExec: target_batch_size=8192 - SortExec: TopK(fetch=5), expr=... - ANNSubIndex: name=..., k=5, deltas=1, metric=L2 - ANNIvfPartition: uuid=..., minimum_nprobes=1, maximum_nprobes=None, deltas=1 - LanceRead: uri=..., projection=[], num_fragments=2, range_before=None, range_after=None, \ - row_id=true, row_addr=false, full_filter=i > Int32(10), refine_filter=i > Int32(10)" + LanceRead: uri=..., projection=[i, s], source=stream(_rowid) + FilterExec: _distance@... IS NOT NULL + SortExec: TopK(fetch=5), expr=... + KNNVectorDistance: metric=l2 + CoalescePartitionsExec + UnionExec + ProjectionExec: expr=[_distance@3 as _distance, _rowid@2 as _rowid, vec@0 as vec] + FilterExec: _distance@... IS NOT NULL + SortExec: TopK(fetch=5), expr=... + KNNVectorDistance: metric=l2 + FilterExec: i@1 > 10 + LanceScan: uri=..., projection=[vec, i], row_id=true, row_addr=false, ordered=false, range=None + LanceRead: uri=..., projection=[vec], source=stream(_rowid) + SortExec: TopK(fetch=5), expr=... + ANNSubIndex: name=..., k=5, deltas=1, metric=L2 + ANNIvfPartition: uuid=..., minimum_nprobes=1, maximum_nprobes=None, deltas=1 + LanceRead: uri=..., projection=[], num_fragments=2, range_before=None, range_after=None, \ + row_id=true, row_addr=false, full_filter=i > Int32(10), refine_filter=i > Int32(10)" }; assert_plan_equals( &dataset.dataset, @@ -10619,14 +12815,22 @@ full_filter=name LIKE Utf8(\"test%2\"), refine_filter=name LIKE Utf8(\"test%2\") dataset.make_scalar_index().await?; log::info!("Test case: ANN with scalar index"); - let expected = + let expected = if data_storage_version == LanceFileVersion::Legacy { "ProjectionExec: expr=[i@2 as i, s@3 as s, vec@4 as vec, _distance@0 as _distance] Take: columns=\"_distance, _rowid, (i), (s), (vec)\" CoalesceBatchesExec: target_batch_size=8192 SortExec: TopK(fetch=5), expr=... ANNSubIndex: name=..., k=5, deltas=1, metric=L2 ANNIvfPartition: uuid=..., minimum_nprobes=1, maximum_nprobes=None, deltas=1 - ScalarIndexQuery: query=[i > 10]@i_idx(BTree)"; + ScalarIndexQuery: query=[i > 10]@i_idx(BTree)" + } else { + "ProjectionExec: expr=[i@2 as i, s@3 as s, vec@4 as vec, _distance@0 as _distance] + LanceRead: uri=..., projection=[i, s, vec], source=stream(_rowid) + SortExec: TopK(fetch=5), expr=... + ANNSubIndex: name=..., k=5, deltas=1, metric=L2 + ANNIvfPartition: uuid=..., minimum_nprobes=1, maximum_nprobes=None, deltas=1 + ScalarIndexQuery: query=[i > 10]@i_idx(BTree)" + }; assert_plan_equals( &dataset.dataset, |scan| { @@ -10651,13 +12855,12 @@ full_filter=name LIKE Utf8(\"test%2\"), refine_filter=name LIKE Utf8(\"test%2\") LanceScan: uri=..., projection=[i], row_id=true, row_addr=false, ordered=false, range=None" } else { "ProjectionExec: expr=[i@2 as i, s@3 as s, vec@4 as vec, _distance@0 as _distance] - Take: columns=\"_distance, _rowid, (i), (s), (vec)\" - CoalesceBatchesExec: target_batch_size=8192 - SortExec: TopK(fetch=5), expr=... - ANNSubIndex: name=..., k=5, deltas=1, metric=L2 - ANNIvfPartition: uuid=..., minimum_nprobes=1, maximum_nprobes=None, deltas=1 - LanceRead: uri=..., projection=[], num_fragments=3, range_before=None, \ - range_after=None, row_id=true, row_addr=false, full_filter=i > Int32(10), refine_filter=i > Int32(10)" + LanceRead: uri=..., projection=[i, s, vec], source=stream(_rowid) + SortExec: TopK(fetch=5), expr=... + ANNSubIndex: name=..., k=5, deltas=1, metric=L2 + ANNIvfPartition: uuid=..., minimum_nprobes=1, maximum_nprobes=None, deltas=1 + LanceRead: uri=..., projection=[], num_fragments=3, range_before=None, \ + range_after=None, row_id=true, row_addr=false, full_filter=i > Int32(10), refine_filter=i > Int32(10)" }; assert_plan_equals( &dataset.dataset, @@ -10675,7 +12878,8 @@ full_filter=name LIKE Utf8(\"test%2\"), refine_filter=name LIKE Utf8(\"test%2\") dataset.append_new_data().await?; log::info!("Test case: Combined KNN/ANN with scalar index"); - let expected = "ProjectionExec: expr=[i@3 as i, s@4 as s, vec@1 as vec, _distance@2 as _distance] + let expected = if data_storage_version == LanceFileVersion::Legacy { + "ProjectionExec: expr=[i@3 as i, s@4 as s, vec@1 as vec, _distance@2 as _distance] Take: columns=\"_rowid, vec, _distance, (i), (s)\" CoalesceBatchesExec: target_batch_size=8192 FilterExec: _distance@... IS NOT NULL @@ -10694,7 +12898,27 @@ full_filter=name LIKE Utf8(\"test%2\"), refine_filter=name LIKE Utf8(\"test%2\") SortExec: TopK(fetch=8), expr=... ANNSubIndex: name=..., k=8, deltas=1, metric=L2 ANNIvfPartition: uuid=..., minimum_nprobes=1, maximum_nprobes=None, deltas=1 - ScalarIndexQuery: query=[i > 10]@i_idx(BTree)"; + ScalarIndexQuery: query=[i > 10]@i_idx(BTree)" + } else { + "ProjectionExec: expr=[i@3 as i, s@4 as s, vec@1 as vec, _distance@2 as _distance] + LanceRead: uri=..., projection=[i, s], source=stream(_rowid) + FilterExec: _distance@... IS NOT NULL + SortExec: TopK(fetch=8), expr=... + KNNVectorDistance: metric=l2 + CoalescePartitionsExec + UnionExec + ProjectionExec: expr=[_distance@3 as _distance, _rowid@2 as _rowid, vec@0 as vec] + FilterExec: _distance@... IS NOT NULL + SortExec: TopK(fetch=8), expr=... + KNNVectorDistance: metric=l2 + FilterExec: i@1 > 10 + LanceScan: uri=..., projection=[vec, i], row_id=true, row_addr=false, ordered=false, range=None + LanceRead: uri=..., projection=[vec], source=stream(_rowid) + SortExec: TopK(fetch=8), expr=... + ANNSubIndex: name=..., k=8, deltas=1, metric=L2 + ANNIvfPartition: uuid=..., minimum_nprobes=1, maximum_nprobes=None, deltas=1 + ScalarIndexQuery: query=[i > 10]@i_idx(BTree)" + }; assert_plan_equals( &dataset.dataset, |scan| { @@ -10711,7 +12935,8 @@ full_filter=name LIKE Utf8(\"test%2\"), refine_filter=name LIKE Utf8(\"test%2\") log::info!( "Test case: Combined KNN/ANN with updated scalar index and outdated vector index" ); - let expected = "ProjectionExec: expr=[i@3 as i, s@4 as s, vec@1 as vec, _distance@2 as _distance] + let expected = if data_storage_version == LanceFileVersion::Legacy { + "ProjectionExec: expr=[i@3 as i, s@4 as s, vec@1 as vec, _distance@2 as _distance] Take: columns=\"_rowid, vec, _distance, (i), (s)\" CoalesceBatchesExec: target_batch_size=8192 FilterExec: _distance@... IS NOT NULL @@ -10730,7 +12955,27 @@ full_filter=name LIKE Utf8(\"test%2\"), refine_filter=name LIKE Utf8(\"test%2\") SortExec: TopK(fetch=11), expr=... ANNSubIndex: name=..., k=11, deltas=1, metric=L2 ANNIvfPartition: uuid=..., minimum_nprobes=1, maximum_nprobes=None, deltas=1 - ScalarIndexQuery: query=[i > 10]@i_idx(BTree)"; + ScalarIndexQuery: query=[i > 10]@i_idx(BTree)" + } else { + "ProjectionExec: expr=[i@3 as i, s@4 as s, vec@1 as vec, _distance@2 as _distance] + LanceRead: uri=..., projection=[i, s], source=stream(_rowid) + FilterExec: _distance@... IS NOT NULL + SortExec: TopK(fetch=11), expr=... + KNNVectorDistance: metric=l2 + CoalescePartitionsExec + UnionExec + ProjectionExec: expr=[_distance@3 as _distance, _rowid@2 as _rowid, vec@0 as vec] + FilterExec: _distance@... IS NOT NULL + SortExec: TopK(fetch=11), expr=... + KNNVectorDistance: metric=l2 + FilterExec: i@1 > 10 + LanceScan: uri=..., projection=[vec, i], row_id=true, row_addr=false, ordered=false, range=None + LanceRead: uri=..., projection=[vec], source=stream(_rowid) + SortExec: TopK(fetch=11), expr=... + ANNSubIndex: name=..., k=11, deltas=1, metric=L2 + ANNIvfPartition: uuid=..., minimum_nprobes=1, maximum_nprobes=None, deltas=1 + ScalarIndexQuery: query=[i > 10]@i_idx(BTree)" + }; dataset.make_scalar_index().await?; assert_plan_equals( &dataset.dataset, @@ -10776,10 +13021,9 @@ full_filter=name LIKE Utf8(\"test%2\"), refine_filter=name LIKE Utf8(\"test%2\") .filter("i > 10") }, "ProjectionExec: expr=[s@2 as s] - Take: columns=\"i, _rowid, (s)\" - CoalesceBatchesExec: target_batch_size=8192 - LanceRead: uri=..., projection=[i], num_fragments=4, range_before=None, \ - range_after=None, row_id=true, row_addr=false, full_filter=i > Int32(10), refine_filter=i > Int32(10)", + LanceRead: uri=..., projection=[s], source=stream(_rowid) + LanceRead: uri=..., projection=[i], num_fragments=4, range_before=None, \ + range_after=None, row_id=true, row_addr=false, full_filter=i > Int32(10), refine_filter=i > Int32(10)", ) .await?; } @@ -10888,10 +13132,16 @@ full_filter=name LIKE Utf8(\"test%2\"), refine_filter=name LIKE Utf8(\"test%2\") // All rows are indexed dataset.make_fts_index().await?; log::info!("Test case: Full text search (match query)"); - let expected = r#"ProjectionExec: expr=[s@2 as s, _score@1 as _score, _rowid@0 as _rowid] + let expected = if data_storage_version == LanceFileVersion::Legacy { + r#"ProjectionExec: expr=[s@2 as s, _score@1 as _score, _rowid@0 as _rowid] Take: columns="_rowid, _score, (s)" CoalesceBatchesExec: target_batch_size=8192 - MatchQuery: column=s, query=hello"#; + MatchQuery: column=s, query=[hello]"# + } else { + r#"ProjectionExec: expr=[s@2 as s, _score@1 as _score, _rowid@0 as _rowid] + LanceRead: uri=..., projection=[s], source=stream(_rowid) + MatchQuery: column=s, query=[hello]"# + }; assert_plan_equals( &dataset.dataset, |scan| { @@ -10904,10 +13154,16 @@ full_filter=name LIKE Utf8(\"test%2\"), refine_filter=name LIKE Utf8(\"test%2\") .await?; log::info!("Test case: Full text search (phrase query)"); - let expected = r#"ProjectionExec: expr=[s@2 as s, _score@1 as _score, _rowid@0 as _rowid] + let expected = if data_storage_version == LanceFileVersion::Legacy { + r#"ProjectionExec: expr=[s@2 as s, _score@1 as _score, _rowid@0 as _rowid] Take: columns="_rowid, _score, (s)" CoalesceBatchesExec: target_batch_size=8192 - PhraseQuery: column=s, query=hello world"#; + PhraseQuery: column=s, query=hello world"# + } else { + r#"ProjectionExec: expr=[s@2 as s, _score@1 as _score, _rowid@0 as _rowid] + LanceRead: uri=..., projection=[s], source=stream(_rowid) + PhraseQuery: column=s, query=hello world"# + }; assert_plan_equals( &dataset.dataset, |scan| { @@ -10921,12 +13177,16 @@ full_filter=name LIKE Utf8(\"test%2\"), refine_filter=name LIKE Utf8(\"test%2\") .await?; log::info!("Test case: Full text search (boost query)"); - let expected = r#"ProjectionExec: expr=[s@2 as s, _score@1 as _score, _rowid@0 as _rowid] + let expected = if data_storage_version == LanceFileVersion::Legacy { + r#"ProjectionExec: expr=[s@2 as s, _score@1 as _score, _rowid@0 as _rowid] Take: columns="_rowid, _score, (s)" CoalesceBatchesExec: target_batch_size=8192 - BoostQuery: negative_boost=1 - MatchQuery: column=s, query=hello - MatchQuery: column=s, query=world"#; + CompoundFtsScorer: query=Boosting(positive=Match(MatchQuery { column: Some("s"), terms: "hello", ... }), negative=Match(MatchQuery { column: Some("s"), terms: "world", ... }), negative_boost=1)"# + } else { + r#"ProjectionExec: expr=[s@2 as s, _score@1 as _score, _rowid@0 as _rowid] + LanceRead: uri=..., projection=[s], source=stream(_rowid) + CompoundFtsScorer: query=Boosting(positive=Match(MatchQuery { column: Some("s"), terms: "hello", ... }), negative=Match(MatchQuery { column: Some("s"), terms: "world", ... }), negative_boost=1)"# + }; assert_plan_equals( &dataset.dataset, |scan| { @@ -10948,7 +13208,7 @@ full_filter=name LIKE Utf8(\"test%2\"), refine_filter=name LIKE Utf8(\"test%2\") r#"ProjectionExec: expr=[s@2 as s, _score@1 as _score, _rowid@0 as _rowid] Take: columns="_rowid, _score, (s)" CoalesceBatchesExec: target_batch_size=8192 - MatchQuery: column=s, query=hello + MatchQuery: column=s, query=[hello] CoalescePartitionsExec UnionExec MaterializeIndex: query=[i > 10]@i_idx(BTree) @@ -10957,11 +13217,10 @@ full_filter=name LIKE Utf8(\"test%2\"), refine_filter=name LIKE Utf8(\"test%2\") LanceScan: uri=..., projection=[i], row_id=true, row_addr=false, ordered=false, range=None"# } else { r#"ProjectionExec: expr=[s@2 as s, _score@1 as _score, _rowid@0 as _rowid] - Take: columns="_rowid, _score, (s)" - CoalesceBatchesExec: target_batch_size=8192 - MatchQuery: column=s, query=hello - LanceRead: uri=..., projection=[], num_fragments=5, range_before=None, range_after=None, row_id=true, row_addr=false, full_filter=i > Int32(10), refine_filter=-- - ScalarIndexQuery: query=[i > 10]@i_idx(BTree)"# + LanceRead: uri=..., projection=[s], source=stream(_rowid) + MatchQuery: column=s, query=[hello] + LanceRead: uri=..., projection=[], num_fragments=5, range_before=None, range_after=None, row_id=true, row_addr=false, full_filter=i > Int32(10), refine_filter=-- + ScalarIndexQuery: query=[i > 10]@i_idx(BTree)"# }; assert_plan_equals( &dataset.dataset, @@ -10988,19 +13247,18 @@ full_filter=name LIKE Utf8(\"test%2\"), refine_filter=name LIKE Utf8(\"test%2\") SortExec: expr=[_score@1 DESC NULLS LAST], preserve_partitioning=[false] CoalescePartitionsExec UnionExec - MatchQuery: column=s, query=hello + MatchQuery: column=s, query=[hello] FlatMatchQuery: column=s, query=hello LanceScan: uri=..., projection=[s], row_id=true, row_addr=false, ordered=true, range=None"# } else { r#"ProjectionExec: expr=[s@2 as s, _score@1 as _score, _rowid@0 as _rowid] - Take: columns="_rowid, _score, (s)" - CoalesceBatchesExec: target_batch_size=8192 - SortExec: expr=[_score@1 DESC NULLS LAST], preserve_partitioning=[false] - CoalescePartitionsExec - UnionExec - MatchQuery: column=s, query=hello - FlatMatchQuery: column=s, query=hello - LanceRead: uri=..., projection=[s], num_fragments=1, range_before=None, range_after=None, row_id=true, row_addr=false, full_filter=--, refine_filter=--"# + LanceRead: uri=..., projection=[s], source=stream(_rowid) + SortExec: expr=[_score@1 DESC NULLS LAST], preserve_partitioning=[false] + CoalescePartitionsExec + UnionExec + MatchQuery: column=s, query=[hello] + FlatMatchQuery: column=s, query=hello + LanceRead: uri=..., projection=[s], num_fragments=1, range_before=None, range_after=None, row_id=true, row_addr=false, full_filter=--, refine_filter=--"# }; dataset.append_new_data().await?; assert_plan_equals( @@ -11015,10 +13273,16 @@ full_filter=name LIKE Utf8(\"test%2\"), refine_filter=name LIKE Utf8(\"test%2\") .await?; log::info!("Test case: Full text search with unindexed rows and fast_search"); - let expected = r#"ProjectionExec: expr=[s@2 as s, _score@1 as _score, _rowid@0 as _rowid] + let expected = if data_storage_version == LanceFileVersion::Legacy { + r#"ProjectionExec: expr=[s@2 as s, _score@1 as _score, _rowid@0 as _rowid] Take: columns="_rowid, _score, (s)" CoalesceBatchesExec: target_batch_size=8192 - MatchQuery: column=s, query=hello"#; + MatchQuery: column=s, query=[hello]"# + } else { + r#"ProjectionExec: expr=[s@2 as s, _score@1 as _score, _rowid@0 as _rowid] + LanceRead: uri=..., projection=[s], source=stream(_rowid) + MatchQuery: column=s, query=[hello]"# + }; assert_plan_equals( &dataset.dataset, |scan| { @@ -11045,7 +13309,7 @@ full_filter=name LIKE Utf8(\"test%2\"), refine_filter=name LIKE Utf8(\"test%2\") SortExec: expr=[_score@1 DESC NULLS LAST], preserve_partitioning=[false] CoalescePartitionsExec UnionExec - MatchQuery: column=s, query=hello + MatchQuery: column=s, query=[hello] CoalescePartitionsExec UnionExec MaterializeIndex: query=[i > 10]@i_idx(BTree) @@ -11063,17 +13327,16 @@ full_filter=name LIKE Utf8(\"test%2\"), refine_filter=name LIKE Utf8(\"test%2\") LanceScan: uri=..., projection=[i, s], row_id=true, row_addr=false, ordered=false, range=None"# } else { r#"ProjectionExec: expr=[s@2 as s, _score@1 as _score, _rowid@0 as _rowid] - Take: columns="_rowid, _score, (s)" - CoalesceBatchesExec: target_batch_size=8192 - SortExec: expr=[_score@1 DESC NULLS LAST], preserve_partitioning=[false] - CoalescePartitionsExec - UnionExec - MatchQuery: column=s, query=hello - LanceRead: uri=..., projection=[], num_fragments=5, range_before=None, range_after=None, row_id=true, row_addr=false, full_filter=i > Int32(10), refine_filter=-- - ScalarIndexQuery: query=[i > 10]@i_idx(BTree) - FlatMatchQuery: column=s, query=hello - LanceRead: uri=..., projection=[s], num_fragments=1, range_before=None, range_after=None, row_id=true, row_addr=false, full_filter=i > Int32(10), refine_filter=-- - ScalarIndexQuery: query=[i > 10]@i_idx(BTree)"# + LanceRead: uri=..., projection=[s], source=stream(_rowid) + SortExec: expr=[_score@1 DESC NULLS LAST], preserve_partitioning=[false] + CoalescePartitionsExec + UnionExec + MatchQuery: column=s, query=[hello] + LanceRead: uri=..., projection=[], num_fragments=5, range_before=None, range_after=None, row_id=true, row_addr=false, full_filter=i > Int32(10), refine_filter=-- + ScalarIndexQuery: query=[i > 10]@i_idx(BTree) + FlatMatchQuery: column=s, query=hello + LanceRead: uri=..., projection=[s], num_fragments=1, range_before=None, range_after=None, row_id=true, row_addr=false, full_filter=i > Int32(10), refine_filter=-- + ScalarIndexQuery: query=[i > 10]@i_idx(BTree)"# }; assert_plan_equals( &dataset.dataset, @@ -11150,11 +13413,10 @@ full_filter=name LIKE Utf8(\"test%2\"), refine_filter=name LIKE Utf8(\"test%2\") SortExec: TopK(fetch=34), expr=[_distance@2 ASC NULLS LAST, _rowid@1 ASC NULLS LAST]... KNNVectorDistance: metric=l2 LanceScan: uri=..., projection=[vec], row_id=true, row_addr=false, ordered=false, range=None - Take: columns=\"_distance, _rowid, (vec)\" - CoalesceBatchesExec: target_batch_size=8192 - SortExec: TopK(fetch=34), expr=[_distance@0 ASC NULLS LAST, _rowid@1 ASC NULLS LAST]... - ANNSubIndex: name=idx, k=34, deltas=1, metric=L2 - ANNIvfPartition: uuid=..., minimum_nprobes=1, maximum_nprobes=None, deltas=1", + LanceRead: uri=..., projection=[vec], source=stream(_rowid) + SortExec: TopK(fetch=34), expr=[_distance@0 ASC NULLS LAST, _rowid@1 ASC NULLS LAST]... + ANNSubIndex: name=idx, k=34, deltas=1, metric=L2 + ANNIvfPartition: uuid=..., minimum_nprobes=1, maximum_nprobes=None, deltas=1", ) .await .unwrap(); @@ -11890,6 +14152,111 @@ full_filter=name LIKE Utf8(\"test%2\"), refine_filter=name LIKE Utf8(\"test%2\") .await; } + #[tokio::test] + async fn test_filter_to_take_with_stable_row_ids() { + let ds = lance_datagen::gen_batch() + .col("idx", array::step::()) + .into_ram_dataset_with_params( + FragmentCount::from(3), + FragmentRowCount::from(4), + Some(WriteParams { + max_rows_per_file: 4, + enable_stable_row_ids: true, + ..Default::default() + }), + ) + .await + .unwrap(); + + let row_addrs = [ + (u64::from(RowAddress::new_from_parts(0, 1)), 1), + (u64::from(RowAddress::new_from_parts(1, 1)), 5), + (u64::from(RowAddress::new_from_parts(2, 1)), 9), + ]; + for (row_addr, expected_idx) in row_addrs { + let batch = ds + .scan() + .filter(&format!("{ROW_ADDR} = {row_addr}")) + .unwrap() + .try_into_batch() + .await + .unwrap(); + assert_eq!( + batch["idx"].as_primitive::().values(), + &[expected_idx] + ); + } + + let batch = ds + .scan() + .filter(&format!( + "{ROW_ADDR} IN ({}, {})", + row_addrs[1].0, row_addrs[2].0 + )) + .unwrap() + .try_into_batch() + .await + .unwrap(); + assert_eq!(batch["idx"].as_primitive::().values(), &[5, 9]); + + let batch = ds + .scan() + .filter(&format!("{ROW_ADDR} = 5")) + .unwrap() + .try_into_batch() + .await + .unwrap(); + assert_eq!(batch.num_rows(), 0); + + let batch = ds + .scan() + .filter(&format!("{ROW_OFFSET} IN (5, 9)")) + .unwrap() + .try_into_batch() + .await + .unwrap(); + assert_eq!(batch["idx"].as_primitive::().values(), &[5, 9]); + } + + #[tokio::test] + async fn test_stale_row_address_does_not_follow_stable_id_after_update() { + let ds = lance_datagen::gen_batch() + .col("idx", array::step::()) + .into_ram_dataset_with_params( + FragmentCount::from(2), + FragmentRowCount::from(3), + Some(WriteParams { + max_rows_per_file: 3, + enable_stable_row_ids: true, + ..Default::default() + }), + ) + .await + .unwrap(); + + let old_row_addr = u64::from(RowAddress::new_from_parts(0, 1)); + let ds = crate::dataset::UpdateBuilder::new(Arc::new(ds)) + .update_where("idx = 1") + .unwrap() + .set("idx", "101") + .unwrap() + .build() + .unwrap() + .execute() + .await + .unwrap() + .new_dataset; + + let batch = ds + .scan() + .filter(&format!("{ROW_ADDR} = {old_row_addr}")) + .unwrap() + .try_into_batch() + .await + .unwrap(); + assert_eq!(batch.num_rows(), 0); + } + #[tokio::test] async fn test_filter_to_take() { let mut ds = lance_datagen::gen_batch() @@ -12334,7 +14701,7 @@ full_filter=name LIKE Utf8(\"test%2\"), refine_filter=name LIKE Utf8(\"test%2\") } fn find_filtered_read(plan: &dyn ExecutionPlan) -> Option<&FilteredReadExec> { - if let Some(f) = plan.as_any().downcast_ref::() { + if let Some(f) = plan.downcast_ref::() { return Some(f); } for child in plan.children() { @@ -12371,6 +14738,47 @@ full_filter=name LIKE Utf8(\"test%2\"), refine_filter=name LIKE Utf8(\"test%2\") assert_eq!(filtered.options().io_buffer_size_bytes, Some(7777)); } + #[tokio::test] + async fn test_batch_readahead_bounds_decode_concurrency() { + let data = lance_datagen::gen_batch() + .col("x", lance_datagen::array::step::()) + .into_reader_rows(RowCount::from(8), BatchCount::from(1)); + let dataset = Dataset::write(data, "memory://test_batch_readahead_concurrency", None) + .await + .unwrap(); + + // Default: threading mode falls back to get_num_compute_intensive_cpus(). + let plan = dataset.scan().create_plan().await.unwrap(); + let filtered = find_filtered_read(plan.as_ref()) + .expect("expected a FilteredReadExec in the scan plan"); + assert_eq!( + filtered.options().threading_mode, + FilteredReadThreadingMode::OnePartitionMultipleThreads(get_num_compute_intensive_cpus()), + ); + + // Explicit batch_readahead(N) bounds the decode fan-out to N. + let mut scanner = dataset.scan(); + scanner.batch_readahead(3); + let plan = scanner.create_plan().await.unwrap(); + let filtered = find_filtered_read(plan.as_ref()) + .expect("expected a FilteredReadExec in the scan plan"); + assert_eq!( + filtered.options().threading_mode, + FilteredReadThreadingMode::OnePartitionMultipleThreads(3), + ); + + let mut scanner = dataset.scan(); + scanner.batch_readahead(0); + let Err(Error::InvalidInput { source, .. }) = scanner.create_plan().await else { + panic!("expected batch_readahead=0 to be rejected"); + }; + assert!( + source + .to_string() + .contains("batch_readahead must be greater than 0") + ); + } + // The env var key scopes serial_test's lock so this test only blocks others // that touch LANCE_DEFAULT_IO_BUFFER_SIZE — unrelated tests still run in // parallel. @@ -12753,8 +15161,20 @@ full_filter=name LIKE Utf8(\"test%2\"), refine_filter=name LIKE Utf8(\"test%2\") .await .unwrap(); - // Create FTS index on first 2 fragments - test_ds.make_fts_index().await.unwrap(); + // Create one FTS physical segment per indexed fragment. + test_ds.make_segmented_fts_index().await.unwrap(); + let expected_index_coverage = test_ds + .dataset + .get_fragments() + .iter() + .map(|fragment| fragment.id() as u32) + .collect::(); + let fragment_bitmap = + fts_index_fragment_bitmap(&test_ds.dataset, "s", DocumentGranularity::Row) + .await + .unwrap() + .expect("segmented FTS index"); + assert_eq!(fragment_bitmap, expected_index_coverage); // Append two more unindexed fragments test_ds.append_data_with_range(400, 410).await.unwrap(); @@ -12766,6 +15186,41 @@ full_filter=name LIKE Utf8(\"test%2\"), refine_filter=name LIKE Utf8(\"test%2\") assert_eq!(fragments.len(), 4); // "s-5" matches: s-5, s-50..s-59, s-150..s-159 (frag 0), s-250..s-259, s-350..s-359 (frag 1), s-405 (frag 2), s-415 (frag 3) + async fn fts_ids(dataset: &Dataset, fragments: Option>) -> Vec { + let mut scanner = dataset.scan(); + scanner + .full_text_search(FullTextSearchQuery::new("s-5".into())) + .unwrap(); + if let Some(fragments) = fragments { + scanner.with_fragments(fragments); + } + let batch = scanner.try_into_batch().await.unwrap(); + let mut ids = batch + .column_by_name("i") + .unwrap() + .as_primitive::() + .values() + .to_vec(); + ids.sort_unstable(); + ids + } + + let global_ids = fts_ids(&test_ds.dataset, None).await; + let mut fragmented_ids = Vec::with_capacity(global_ids.len()); + for (fragment, expected_range) in + fragments.iter().zip([0..200, 200..400, 400..410, 410..420]) + { + let ids = fts_ids(&test_ds.dataset, Some(vec![fragment.clone()])).await; + assert!( + !ids.is_empty() && ids.iter().all(|id| expected_range.contains(id)), + "fragment {} should return only matching rows in {expected_range:?}", + fragment.id, + ); + fragmented_ids.extend(ids); + } + fragmented_ids.sort_unstable(); + assert_eq!(fragmented_ids, global_ids); + test_fragment_list_filtering(&test_ds, fragments, |dataset| { let mut scanner = dataset.scan(); scanner @@ -12774,5 +15229,24 @@ full_filter=name LIKE Utf8(\"test%2\"), refine_filter=name LIKE Utf8(\"test%2\") scanner }) .await; + for (fragment, (phrase, expected_i)) in + fragments[..2].iter().zip([("s 5", 5), ("s 205", 205)]) + { + let mut scanner = test_ds.dataset.scan(); + scanner.with_fragments(vec![fragment.clone()]); + scanner + .full_text_search(FullTextSearchQuery::new_query( + PhraseQuery::new(phrase.to_string()) + .with_column(Some("s".to_string())) + .into(), + )) + .unwrap(); + let batch = scanner.try_into_batch().await.unwrap(); + let i_array = batch + .column_by_name("i") + .unwrap() + .as_primitive::(); + assert_eq!(i_array.values(), &[expected_i]); + } } } diff --git a/rust/lance/src/dataset/schema_evolution.rs b/rust/lance/src/dataset/schema_evolution.rs index ce32362f324..3643e8043ec 100644 --- a/rust/lance/src/dataset/schema_evolution.rs +++ b/rust/lance/src/dataset/schema_evolution.rs @@ -25,14 +25,13 @@ use lance_arrow::SchemaExt; use lance_core::datatypes::{Field, Schema}; use lance_datafusion::utils::StreamingWriteSource; use lance_encoding::constants::{PACKED_STRUCT_LEGACY_META_KEY, PACKED_STRUCT_META_KEY}; -use lance_encoding::version::LanceFileVersion; +#[cfg(test)] +use lance_file::version::ConcreteFileVersion; use lance_table::format::Fragment; -mod optimize; +pub mod optimize; -use optimize::{ - ChainedNewColumnTransformOptimizer, NewColumnTransformOptimizer, SqlToAllNullsOptimizer, -}; +use optimize::{ChainedNewColumnTransformOptimizer, NewColumnTransformOptimizer}; async fn validate_no_nulls_before_making_non_nullable(dataset: &Dataset, path: &str) -> Result<()> { let field = dataset.schema().field(path).ok_or_else(|| { @@ -148,44 +147,19 @@ impl ColumnAlteration { } } -/// Limit casts to same type. This is mostly to filter out weird casts like -/// casting a string to a boolean or float to string. -fn is_upcast_downcast(from_type: &DataType, to_type: &DataType, version: LanceFileVersion) -> bool { - use DataType::*; - match (from_type, to_type) { - // Legacy storage cannot materialize a fresh Dictionary column via - // alter because the writer expects `field.dictionary` metadata to be - // pre-populated, which the alter pipeline does not compute. - (_, Dictionary(_, _)) if matches!(version, LanceFileVersion::Legacy) => false, - // These need to be in front - (Dictionary(_, from_value_type), _) => { - is_upcast_downcast(from_value_type, to_type, version) - } - (_, Dictionary(_, to_value_type)) => is_upcast_downcast(from_type, to_value_type, version), - (from, to) if from.is_integer() => to.is_integer(), - (from, to) if from.is_floating() => to.is_floating(), - (from, to) if from.is_temporal() => to.is_temporal(), - (Boolean, to) => matches!(to, Boolean), - (Utf8 | LargeUtf8, to) => matches!(to, Utf8 | LargeUtf8), - (Binary | LargeBinary, to) => matches!(to, Binary | LargeBinary), - (Decimal128(_, _) | Decimal256(_, _), to) => { - matches!(to, Decimal128(_, _) | Decimal256(_, _)) - } - (List(from_field) | LargeList(from_field) | FixedSizeList(from_field, _), to) => match to { - List(to_field) | LargeList(to_field) | FixedSizeList(to_field, _) => { - is_upcast_downcast(from_field.data_type(), to_field.data_type(), version) - } - _ => false, - }, - - _ => false, - } -} - trait ArrowFieldExt { fn is_packed(&self) -> bool; } +#[cfg(test)] +fn is_upcast_downcast( + from_type: &DataType, + to_type: &DataType, + version: ConcreteFileVersion, +) -> bool { + super::versions::is_upcast_downcast(version, from_type, to_type) +} + impl ArrowFieldExt for ArrowField { fn is_packed(&self) -> bool { let metadata = self.metadata(); @@ -196,10 +170,10 @@ impl ArrowFieldExt for ArrowField { } } -fn check_field_conflict( +pub fn check_field_conflict_with( left: &ArrowField, right: &ArrowField, - version: &LanceFileVersion, + validate_nested_column_add: fn(&ArrowField) -> Result<()>, ) -> Result<()> { if left.name() != right.name() { return Ok(()); @@ -207,13 +181,7 @@ fn check_field_conflict( match (left.data_type(), right.data_type()) { (DataType::Struct(fl), DataType::Struct(fr)) => { - if !version.support_add_sub_column() { - return Err(Error::invalid_input(format!( - "Column {} is a struct col, add sub column is not supported in Lance file version {}", - left.name(), - version - ))); - } + validate_nested_column_add(left)?; if left.is_packed() || right.is_packed() { return Err(Error::invalid_input(format!( @@ -224,15 +192,19 @@ fn check_field_conflict( for l_field in fl.iter() { if let Some((_, r_field)) = fr.find(l_field.name()) { - check_field_conflict(l_field, r_field, version)?; + check_field_conflict_with(l_field, r_field, validate_nested_column_add)?; } } Ok(()) } - (DataType::List(fl), DataType::List(fr)) => check_field_conflict(fl, fr, version), - (DataType::LargeList(fl), DataType::LargeList(fr)) => check_field_conflict(fl, fr, version), + (DataType::List(fl), DataType::List(fr)) => { + check_field_conflict_with(fl, fr, validate_nested_column_add) + } + (DataType::LargeList(fl), DataType::LargeList(fr)) => { + check_field_conflict_with(fl, fr, validate_nested_column_add) + } (DataType::FixedSizeList(fl, _), DataType::FixedSizeList(fr, _)) => { - check_field_conflict(fl, fr, version) + check_field_conflict_with(fl, fr, validate_nested_column_add) } (l_type, r_type) if l_type == r_type => Err(Error::invalid_input(format!( "Column {} already exists in the dataset", @@ -248,21 +220,30 @@ fn check_field_conflict( } } +#[cfg(test)] +fn check_field_conflict( + left: &ArrowField, + right: &ArrowField, + version: &ConcreteFileVersion, +) -> Result<()> { + super::versions::check_field_conflict(*version, left, right) +} + pub(super) async fn add_columns_to_fragments( dataset: &Dataset, transforms: NewColumnTransform, read_columns: Option>, fragments: &[FileFragment], batch_size: Option, -) -> Result<(Vec, Schema, Vec)> { +) -> Result<(Vec, Schema, Vec, bool)> { // Check names early (before calling add_columns_impl) to avoid extra work if // the names are wrong. - let version = dataset.manifest.data_storage_format.lance_file_version()?; + let version = dataset.manifest.data_storage_format.lance_file_format(); let check_names = |output_schema: &ArrowSchema| { for field in &dataset.schema().fields { if let Ok(out_field) = output_schema.field_with_name(&field.name) { let ds_field = ArrowField::from(field); - check_field_conflict(&ds_field, out_field, &version)?; + super::versions::check_field_conflict(version, &ds_field, out_field)?; } } Ok::<(), Error>(()) @@ -270,10 +251,7 @@ pub(super) async fn add_columns_to_fragments( // Optimize the transforms let mut optimizer = ChainedNewColumnTransformOptimizer::new(vec![]); - // ALlNull transform can not performed on legacy files - if !dataset.is_legacy_storage() { - optimizer.add_optimizer(Box::new(SqlToAllNullsOptimizer::new())); - } + super::versions::configure_new_column_optimizers(version, &mut optimizer); let transforms = optimizer.optimize(dataset, transforms)?; let (output_schema, new_fragments, fragments_to_cleanup) = match transforms { @@ -392,15 +370,7 @@ pub(super) async fn add_columns_to_fragments( .map(|f| f.metadata.clone()) .collect::>(); - // Check if any of the fragment's files are using the legacy dataset version if so, we - // can't add all-null columns as a metadata-only operation. The reason is because we - // use the NullReader for fragments that have missing columns and we can't mix legacy - // and non-legacy readers when reading the fragment. - if dataset.is_legacy_storage() { - return Err(Error::not_supported_source( - "Cannot add all-null columns to legacy dataset version.".into(), - )); - } + super::versions::validate_metadata_only_null_columns(version)?; Ok((output_schema, fragments, Vec::new())) } @@ -415,7 +385,58 @@ pub(super) async fn add_columns_to_fragments( }; schema.set_field_id(Some(dataset.manifest.max_field_id())); - Ok((new_fragments, schema, fragments_to_cleanup)) + let preserves_nullability = !merge_introduces_required_field(dataset.schema(), &schema); + + Ok(( + new_fragments, + schema, + fragments_to_cleanup, + preserves_nullability, + )) +} + +/// Whether `merged` introduces a field that data staged against `old` cannot +/// safely omit. The first new node on each path decides: a non-nullable new +/// field beneath an existing ancestor reads as unmasked null for stale rows, +/// which do supply the ancestor, while a nullable new field masks its whole +/// subtree whatever the nullability inside, the same rule the AllNulls +/// transform enforces at the top level. +/// +/// A new node under a non-nullable top-level column claims even when the node +/// itself is nullable: the reader synthesizes missing subcolumns against the +/// column's declared nullability, so a stale fragment cannot be read at all +/// under such a column, nullable child or not. +pub(super) fn merge_introduces_required_field(old: &Schema, merged: &Schema) -> bool { + /// (any node in `merged` is new, any first-new node is non-nullable) + fn subtree_new_nodes(old: &[Field], merged: &[Field]) -> (bool, bool) { + let mut any_new = false; + let mut any_required = false; + for field in merged { + match old.iter().find(|o| o.name == field.name) { + Some(old_field) => { + let (new, required) = subtree_new_nodes(&old_field.children, &field.children); + any_new |= new; + any_required |= required; + } + None => { + any_new = true; + any_required |= !field.nullable; + } + } + } + (any_new, any_required) + } + + merged.fields.iter().any( + |field| match old.fields.iter().find(|o| o.name == field.name) { + Some(old_field) => { + let (any_new, any_required) = + subtree_new_nodes(&old_field.children, &field.children); + any_required || (any_new && !field.nullable) + } + None => !field.nullable, + }, + ) } pub(super) async fn add_columns( @@ -424,27 +445,29 @@ pub(super) async fn add_columns( read_columns: Option>, batch_size: Option, ) -> Result<()> { - let (fragments, schema, fragments_to_cleanup) = add_columns_to_fragments( - dataset, - transforms, - read_columns, - &dataset.get_fragments(), - batch_size, - ) - .await?; + let (fragments, schema, _fragments_to_cleanup, preserves_nullability) = + add_columns_to_fragments( + dataset, + transforms, + read_columns, + &dataset.get_fragments(), + batch_size, + ) + .await?; - let operation = Operation::Merge { fragments, schema }; + let operation = Operation::Merge { + fragments, + schema, + preserves_nullability, + }; let transaction = Transaction::new(dataset.manifest.version, operation, None); - match dataset + // Once the manifest commit has been attempted, an error does not prove + // that the new files are unreferenced: the commit may have landed and only + // its response (or a post-commit callback) may have failed. Leave files + // from failed attempts for dataset GC instead of risking live-data loss. + dataset .apply_commit(transaction, &Default::default(), &Default::default()) .await - { - Ok(()) => Ok(()), - Err(e) => { - cleanup_new_column_data_files(&dataset.get_fragments(), &fragments_to_cleanup).await; - Err(e) - } - } } async fn cleanup_new_column_data_files(fragments: &[FileFragment], new_fragments: &[Fragment]) { @@ -539,7 +562,7 @@ async fn add_columns_impl( } let mut updater = match fragment - .updater(read_columns_ref, schemas.clone(), batch_size) + .updater(read_columns_ref, schemas.clone(), batch_size, None) .await { Ok(updater) => updater, @@ -617,7 +640,7 @@ async fn add_columns_from_stream( let mut last_seen_batch: Option = None; for fragment in fragments { let mut updater = match fragment - .updater::(Some(&[]), schemas.clone(), batch_size) + .updater::(Some(&[]), schemas.clone(), batch_size, None) .await { Ok(updater) => updater, @@ -711,9 +734,10 @@ pub(super) async fn alter_columns( // Mapping of old to new fields that need to be casted. let mut cast_fields: Vec<(Field, Field)> = Vec::new(); + let mut tightens_nullability = false; let mut next_field_id = dataset.manifest.max_field_id() + 1; - let version = dataset.manifest.data_storage_format.lance_file_version()?; + let version = dataset.manifest.data_storage_format.lance_file_format(); for alteration in alterations { let field_src = dataset.schema().field(&alteration.path).ok_or_else(|| { @@ -728,6 +752,9 @@ pub(super) async fn alter_columns( && !nullable { validate_no_nulls_before_making_non_nullable(dataset, &alteration.path).await?; + // A write since this version can falsify it, so withhold the + // preserves_nullability assertion from the transaction. + tightens_nullability = true; } let field_dest = new_schema.mut_field_by_id(field_src.id).unwrap(); @@ -740,7 +767,7 @@ pub(super) async fn alter_columns( if let Some(data_type) = &alteration.data_type { if !(can_cast_types(&field_src.data_type(), data_type) - && is_upcast_downcast(&field_src.data_type(), data_type, version)) + && super::versions::is_upcast_downcast(version, &field_src.data_type(), data_type)) { return Err(Error::invalid_input(format!( "Cannot cast column \"{}\" from {:?} to {:?}", @@ -799,11 +826,21 @@ pub(super) async fn alter_columns( } } + if tightens_nullability && !cast_fields.is_empty() { + return Err(Error::invalid_input( + "cannot make a column non-nullable and cast columns in the same call: \ + apply the cast first, then the nullability change", + )); + } + // If we aren't casting a column, we don't need to touch the fragments. let transaction = if cast_fields.is_empty() { Transaction::new( dataset.manifest.version, - Operation::Project { schema: new_schema }, + Operation::Project { + schema: new_schema, + preserves_nullability: !tightens_nullability, + }, // TODO: Make it possible to alter blob columns /*blob_op= */ None, ) @@ -825,6 +862,12 @@ pub(super) async fn alter_columns( // This schema contains the exact field ids we want to write the new fields with. let new_col_schema = new_schema.project_by_ids(&new_ids, true); + // A cast rewrites the column under a new field id, so data staged + // against the pre-cast schema omits that id and its rows read as null. + // Withhold the assertion when any recast field is non-nullable, at any + // depth: a nested field sits under parent values stale rows do supply. + let cast_touches_required = cast_fields.iter().any(|(_old, new)| !new.nullable); + let mapper = move |batch: &RecordBatch| { let mut fields = Vec::with_capacity(cast_fields.len()); let mut columns = Vec::with_capacity(batch.num_columns()); @@ -878,6 +921,7 @@ pub(super) async fn alter_columns( Operation::Merge { schema: new_schema, fragments, + preserves_nullability: !cast_touches_required, }, /*blob_op= */ None, ) @@ -909,9 +953,10 @@ pub(super) async fn drop_columns(dataset: &mut Dataset, columns: &[&str]) -> Res } } - let version = dataset.manifest.data_storage_format.lance_file_version()?; + let version = dataset.manifest.data_storage_format.lance_file_format(); let columns_to_remove = dataset.manifest.schema.project(columns)?; - let new_schema = exclude(&dataset.manifest.schema, &columns_to_remove, &version)?; + let new_schema = + super::versions::exclude_schema(version, &dataset.manifest.schema, &columns_to_remove)?; if new_schema.fields.is_empty() { return Err(Error::invalid_input( @@ -921,7 +966,10 @@ pub(super) async fn drop_columns(dataset: &mut Dataset, columns: &[&str]) -> Res let transaction = Transaction::new( dataset.manifest.version, - Operation::Project { schema: new_schema }, + Operation::Project { + schema: new_schema, + preserves_nullability: true, + }, /*blob_op= */ None, ); @@ -932,17 +980,19 @@ pub(super) async fn drop_columns(dataset: &mut Dataset, columns: &[&str]) -> Res Ok(()) } -/// Exclude the fields from `other` Schema, and returns a new Schema. -pub fn exclude(source: &Schema, other: &Schema, version: &LanceFileVersion) -> Result { +/// Exclude the fields from `other` Schema using the selected nested-field rule. +pub fn exclude_with( + source: &Schema, + other: &Schema, + exclude_nested_field: fn(&Field, &Field) -> Option, +) -> Result { let other: Schema = other.try_into().map_err(|_| { Error::schema("The other schema is not compatible with this schema".to_string()) })?; let mut fields = vec![]; for field in source.fields.iter() { if let Some(other_field) = other.field(&field.name) { - if version.support_remove_sub_column(field) - && let Some(f) = field.exclude(other_field) - { + if let Some(f) = exclude_nested_field(field, other_field) { fields.push(f) } } else { @@ -955,10 +1005,95 @@ pub fn exclude(source: &Schema, other: &Schema, version: &LanceFileVersion) -> R }) } +#[cfg(test)] +fn exclude(source: &Schema, other: &Schema, version: &ConcreteFileVersion) -> Result { + super::versions::exclude_schema(*version, source, other) +} + #[cfg(test)] mod test { use std::{collections::HashMap, fs, num::NonZero, path::Path as StdPath, sync::Mutex}; + #[test] + fn test_merge_introduces_required_field() { + let schema = |fields: Vec| Schema::try_from(&ArrowSchema::new(fields)).unwrap(); + let strukt = |name: &str, nullable: bool, children: Vec| { + ArrowField::new( + name, + DataType::Struct(ArrowFields::from(children)), + nullable, + ) + }; + let int = |name: &str, nullable: bool| ArrowField::new(name, DataType::Int32, nullable); + + let old = schema(vec![ + strukt("s", true, vec![int("a", true)]), + strukt("r", false, vec![int("a", true)]), + ]); + // The first new node on each path decides, at any depth; any new node + // under a non-nullable top-level column claims regardless. + for (merged, expected) in [ + // A nullable new child under a non-nullable top-level column: the + // reader cannot synthesize the missing subcolumn, so claim. + ( + schema(vec![ + strukt("s", true, vec![int("a", true)]), + strukt("r", false, vec![int("a", true), int("b", true)]), + ]), + true, + ), + // Required new child under an existing parent: stale rows supply + // the parent, so the child would read as unmasked null. + ( + schema(vec![strukt( + "s", + true, + vec![int("a", true), int("b", false)], + )]), + true, + ), + ( + schema(vec![strukt( + "s", + true, + vec![int("a", true), int("b", true)], + )]), + false, + ), + // A wholly new nullable container masks its required inside. + ( + schema(vec![ + strukt("s", true, vec![int("a", true)]), + strukt("t", true, vec![int("c", false)]), + ]), + false, + ), + // Same, when the new container hangs under an existing parent. + ( + schema(vec![strukt( + "s", + true, + vec![int("a", true), strukt("t", true, vec![int("c", false)])], + )]), + false, + ), + ( + schema(vec![ + strukt("s", true, vec![int("a", true)]), + int("b", false), + ]), + true, + ), + (schema(vec![strukt("s", true, vec![int("a", true)])]), false), + ] { + assert_eq!( + merge_introduces_required_field(&old, &merged), + expected, + "merged={merged:?}" + ); + } + } + use crate::dataset::WriteParams; use arrow_array::{ ArrayRef, Int32Array, ListArray, RecordBatchIterator, StringArray, StructArray, @@ -967,7 +1102,7 @@ mod test { use super::*; use arrow_schema::Fields as ArrowFields; use lance_core::utils::tempfile::TempStrDir; - use lance_file::version::LanceFileVersion; + use lance_file::version::{ConcreteFileVersion, LanceFileVersion}; use lance_table::format::{BasePath, DataFile}; use rstest::rstest; @@ -1100,6 +1235,69 @@ mod test { Ok(()) } + #[tokio::test] + async fn test_add_columns_preserves_files_when_commit_status_is_unknown() -> Result<()> { + use crate::utils::test::{AmbiguousCommitHandler, AmbiguousFailure}; + + let num_rows = 5; + let schema = Arc::new(ArrowSchema::new(vec![ArrowField::new( + "id", + DataType::Int32, + false, + )])); + let batch = RecordBatch::try_new( + schema.clone(), + vec![Arc::new(Int32Array::from_iter_values(0..num_rows))], + )?; + let reader = RecordBatchIterator::new(vec![Ok(batch)], schema); + let test_dir = TempStrDir::default(); + let test_uri = test_dir.as_str(); + let handler = Arc::new(AmbiguousCommitHandler::default()); + let mut dataset = Dataset::write( + reader, + test_uri, + Some(WriteParams { + commit_handler: Some(handler.clone()), + ..Default::default() + }), + ) + .await?; + let files_before = data_file_paths_in(test_uri); + + handler.fail_next(AmbiguousFailure::LandAndError); + handler + .fail_resolve + .store(true, std::sync::atomic::Ordering::SeqCst); + let error = dataset + .add_columns( + NewColumnTransform::SqlExpressions(vec![("double_id".into(), "2 * id".into())]), + None, + None, + ) + .await + .expect_err("unverifiable commit outcome must be reported"); + assert!( + error.is_commit_status_unknown(), + "expected CommitStatusUnknown, got: {error:?}" + ); + assert!(data_file_paths_in(test_uri).len() > files_before.len()); + + handler + .fail_resolve + .store(false, std::sync::atomic::Ordering::SeqCst); + let reopened = Dataset::open(test_uri).await?; + let data = reopened.scan().try_into_batch().await?; + let double_id = data + .column_by_name("double_id") + .unwrap() + .as_any() + .downcast_ref::() + .unwrap(); + assert_eq!(double_id, &Int32Array::from(vec![0, 2, 4, 6, 8])); + + Ok(()) + } + #[tokio::test] async fn test_add_columns_with_fully_deleted_batch() -> Result<()> { // Regression test: when an entire read batch has been deleted, the @@ -1295,8 +1493,7 @@ mod test { "checkpointed.lance", vec![dataset.manifest.max_field_id() + 1], vec![0], - 2, - 2, + ConcreteFileVersion::V2_2, NonZero::new(17), None, )); @@ -1957,6 +2154,7 @@ mod test { Ok(Some(Fragment { files: vec![], id: 0, + overlays: vec![], deletion_file: None, row_id_meta: None, physical_rows: Some(50), @@ -2458,7 +2656,7 @@ mod test { let schema = Schema::try_from(&arrow_schema).unwrap(); let projection = schema.project(&["a", "b.f2", "b.f3"]).unwrap(); - let excluded = exclude(&schema, &projection, &LanceFileVersion::V2_2).unwrap(); + let excluded = exclude(&schema, &projection, &ConcreteFileVersion::V2_2).unwrap(); let expected_arrow_schema = ArrowSchema::new(vec![ ArrowField::new( @@ -3126,8 +3324,8 @@ mod test { let dict_i16_utf8 = Dictionary(Box::new(Int16), Box::new(Utf8)); let dict_i32_large_utf8 = Dictionary(Box::new(Int32), Box::new(LargeUtf8)); let dict_i32_int64 = Dictionary(Box::new(Int32), Box::new(Int64)); - let stable = LanceFileVersion::Stable; - let legacy = LanceFileVersion::Legacy; + let stable = LanceFileVersion::Stable.resolve(); + let legacy = LanceFileVersion::Legacy.resolve(); // Dict(_, Utf8) -> Utf8 / LargeUtf8 (decode direction): both versions. assert!(is_upcast_downcast(&dict_i32_utf8, &Utf8, stable)); @@ -3623,7 +3821,7 @@ mod test { DataType::Struct(vec![ArrowField::new("a", DataType::Int32, false)].into()), false, ); - assert!(check_field_conflict(&field1, &field2, &LanceFileVersion::V2_2).is_err()); + assert!(check_field_conflict(&field1, &field2, &ConcreteFileVersion::V2_2).is_err()); // different struct let field1 = ArrowField::new( @@ -3636,7 +3834,7 @@ mod test { DataType::Struct(vec![ArrowField::new("b", DataType::Int32, false)].into()), false, ); - assert!(check_field_conflict(&field1, &field2, &LanceFileVersion::V2_2).is_ok()); + assert!(check_field_conflict(&field1, &field2, &ConcreteFileVersion::V2_2).is_ok()); // same nested struct let inner_struct1 = ArrowField::new( @@ -3651,22 +3849,22 @@ mod test { ); let field1 = ArrowField::new("test", DataType::Struct(vec![inner_struct1].into()), false); let field2 = ArrowField::new("test", DataType::Struct(vec![inner_struct2].into()), false); - assert!(check_field_conflict(&field1, &field2, &LanceFileVersion::V2_2).is_err()); + assert!(check_field_conflict(&field1, &field2, &ConcreteFileVersion::V2_2).is_err()); // basic type with different name let field1 = ArrowField::new("test1", DataType::Int32, false); let field2 = ArrowField::new("test2", DataType::Int32, false); - assert!(check_field_conflict(&field1, &field2, &LanceFileVersion::V2_2).is_ok()); + assert!(check_field_conflict(&field1, &field2, &ConcreteFileVersion::V2_2).is_ok()); // basic type with same name let field1 = ArrowField::new("test", DataType::Int32, false); let field2 = ArrowField::new("test", DataType::Int32, false); - assert!(check_field_conflict(&field1, &field2, &LanceFileVersion::V2_2).is_err()); + assert!(check_field_conflict(&field1, &field2, &ConcreteFileVersion::V2_2).is_err()); // different basic type let field1 = ArrowField::new("test", DataType::Int32, false); let field2 = ArrowField::new("test", DataType::Float64, false); - assert!(check_field_conflict(&field1, &field2, &LanceFileVersion::V2_2).is_err()); + assert!(check_field_conflict(&field1, &field2, &ConcreteFileVersion::V2_2).is_err()); // partial conflict let field1 = ArrowField::new( @@ -3691,7 +3889,7 @@ mod test { ), false, ); - assert!(check_field_conflict(&field1, &field2, &LanceFileVersion::V2_2).is_err()); + assert!(check_field_conflict(&field1, &field2, &ConcreteFileVersion::V2_2).is_err()); // same list let field1 = ArrowField::new( @@ -3704,7 +3902,7 @@ mod test { DataType::List(Arc::new(ArrowField::new("item", DataType::Int32, true))), false, ); - assert!(check_field_conflict(&field1, &field2, &LanceFileVersion::V2_2).is_err()); + assert!(check_field_conflict(&field1, &field2, &ConcreteFileVersion::V2_2).is_err()); // list with struct let field1 = ArrowField::new( @@ -3725,7 +3923,7 @@ mod test { ))), false, ); - assert!(check_field_conflict(&field1, &field2, &LanceFileVersion::V2_2).is_err()); + assert!(check_field_conflict(&field1, &field2, &ConcreteFileVersion::V2_2).is_err()); // list with different struct let field1 = ArrowField::new( @@ -3746,7 +3944,7 @@ mod test { ))), false, ); - assert!(check_field_conflict(&field1, &field2, &LanceFileVersion::V2_2).is_ok()); + assert!(check_field_conflict(&field1, &field2, &ConcreteFileVersion::V2_2).is_ok()); // list of struct and basic let field1 = ArrowField::new( @@ -3763,7 +3961,7 @@ mod test { DataType::List(Arc::new(ArrowField::new("item", DataType::Int32, true))), false, ); - assert!(check_field_conflict(&field1, &field2, &LanceFileVersion::V2_2).is_err()); + assert!(check_field_conflict(&field1, &field2, &ConcreteFileVersion::V2_2).is_err()); // FixedSizeList with struct let field1 = ArrowField::new( @@ -3790,7 +3988,7 @@ mod test { ), false, ); - assert!(check_field_conflict(&field1, &field2, &LanceFileVersion::V2_2).is_err()); + assert!(check_field_conflict(&field1, &field2, &ConcreteFileVersion::V2_2).is_err()); // FixedSizeList with different struct let field1 = ArrowField::new( @@ -3817,7 +4015,7 @@ mod test { ), false, ); - assert!(check_field_conflict(&field1, &field2, &LanceFileVersion::V2_2).is_ok()); + assert!(check_field_conflict(&field1, &field2, &ConcreteFileVersion::V2_2).is_ok()); // LargeList with struct let field1 = ArrowField::new( @@ -3838,7 +4036,7 @@ mod test { ))), false, ); - assert!(check_field_conflict(&field1, &field2, &LanceFileVersion::V2_2).is_err()); + assert!(check_field_conflict(&field1, &field2, &ConcreteFileVersion::V2_2).is_err()); // LargeList with different struct let field1 = ArrowField::new( @@ -3859,7 +4057,7 @@ mod test { ))), false, ); - assert!(check_field_conflict(&field1, &field2, &LanceFileVersion::V2_2).is_ok()); + assert!(check_field_conflict(&field1, &field2, &ConcreteFileVersion::V2_2).is_ok()); // packed struct let mut packed_meta = HashMap::new(); @@ -3878,7 +4076,7 @@ mod test { DataType::Struct(vec![ArrowField::new("b", DataType::Int32, false)].into()), false, ); - assert!(check_field_conflict(&field1, &field2, &LanceFileVersion::V2_2).is_ok()); + assert!(check_field_conflict(&field1, &field2, &ConcreteFileVersion::V2_2).is_ok()); let new_packed_field = ArrowField::new( "new_packed", @@ -3891,7 +4089,7 @@ mod test { DataType::Struct(vec![new_packed_field].into()), false, ); - assert!(check_field_conflict(&field1, &field3, &LanceFileVersion::V2_2).is_ok()); + assert!(check_field_conflict(&field1, &field3, &ConcreteFileVersion::V2_2).is_ok()); let conflict_field = ArrowField::new( "packed", @@ -3900,6 +4098,6 @@ mod test { ) .with_metadata(packed_meta); let field4 = ArrowField::new("test", DataType::Struct(vec![conflict_field].into()), false); - assert!(check_field_conflict(&field1, &field4, &LanceFileVersion::V2_2).is_err()); + assert!(check_field_conflict(&field1, &field4, &ConcreteFileVersion::V2_2).is_err()); } } diff --git a/rust/lance/src/dataset/schema_evolution/optimize.rs b/rust/lance/src/dataset/schema_evolution/optimize.rs index cdfdb82b87b..5be44626821 100644 --- a/rust/lance/src/dataset/schema_evolution/optimize.rs +++ b/rust/lance/src/dataset/schema_evolution/optimize.rs @@ -14,7 +14,7 @@ use crate::Result; use super::NewColumnTransform; /// Optimizes a `NewColumnTransform` into -pub(super) trait NewColumnTransformOptimizer: Send + Sync { +pub trait NewColumnTransformOptimizer: Send + Sync { /// Optimize the passed `NewColumnTransform` to a more efficient form. fn optimize( &self, @@ -24,16 +24,16 @@ pub(super) trait NewColumnTransformOptimizer: Send + Sync { } /// A `NewColumnTransformOptimizer` that chains multiple `NewColumnTransformOptimizer`s together. -pub(super) struct ChainedNewColumnTransformOptimizer { +pub struct ChainedNewColumnTransformOptimizer { optimizers: Vec>, } impl ChainedNewColumnTransformOptimizer { - pub(super) fn new(optimizers: Vec>) -> Self { + pub fn new(optimizers: Vec>) -> Self { Self { optimizers } } - pub(super) fn add_optimizer(&mut self, optimizer: Box) { + pub fn add_optimizer(&mut self, optimizer: Box) { self.optimizers.push(optimizer); } } @@ -59,10 +59,10 @@ impl NewColumnTransformOptimizer for ChainedNewColumnTransformOptimizer { /// would be optimized to /// `NewColumnTransform::AllNulls(Schema::new(vec![Field::new("new_col", DataType::Int)]))`. /// -pub(super) struct SqlToAllNullsOptimizer; +pub struct SqlToAllNullsOptimizer; impl SqlToAllNullsOptimizer { - pub(super) fn new() -> Self { + pub fn new() -> Self { Self } @@ -70,7 +70,7 @@ impl SqlToAllNullsOptimizer { match expr { Expr::Cast(cast) => { if matches!(cast.expr.as_ref(), Expr::Literal(ScalarValue::Null, _)) { - let data_type = cast.data_type.clone(); + let data_type = cast.field.data_type().clone(); AllNullsResult::AllNulls(data_type) } else { AllNullsResult::NotAllNulls diff --git a/rust/lance/src/dataset/sql.rs b/rust/lance/src/dataset/sql.rs index 8a1ccda2df6..efd4d65b528 100644 --- a/rust/lance/src/dataset/sql.rs +++ b/rust/lance/src/dataset/sql.rs @@ -9,6 +9,7 @@ use datafusion::dataframe::DataFrame; use datafusion::execution::SendableRecordBatchStream; use datafusion::prelude::SessionContext; use futures::TryStreamExt; +use lance_core::datatypes::BlobHandling; use lance_datafusion::udf::register_functions; use std::sync::Arc; @@ -29,6 +30,9 @@ pub struct SqlQueryBuilder { /// If true, the query result will include the internal row address pub(crate) with_row_addr: bool, + + /// Override how blob columns are materialized for this query. + pub(crate) blob_handling: Option, } impl SqlQueryBuilder { @@ -39,6 +43,7 @@ impl SqlQueryBuilder { table_name: "dataset".to_string(), with_row_id: false, with_row_addr: false, + blob_handling: None, } } @@ -65,18 +70,24 @@ impl SqlQueryBuilder { self } + /// Override how blob columns are materialized for this query. + /// + /// When unset, the underlying dataset scan uses its default + /// [`BlobHandling::BlobsDescriptions`] policy. + pub fn blob_handling(mut self, blob_handling: BlobHandling) -> Self { + self.blob_handling = Some(blob_handling); + self + } + pub async fn build(self) -> lance_core::Result { let ctx = SessionContext::new(); let row_id = self.with_row_id; let row_addr = self.with_row_addr; - ctx.register_table( - self.table_name, - Arc::new(LanceTableProvider::new( - self.dataset.clone(), - row_id, - row_addr, - )), - )?; + let mut provider = LanceTableProvider::new(self.dataset.clone(), row_id, row_addr); + if let Some(blob_handling) = self.blob_handling { + provider = provider.with_blob_handling(blob_handling); + } + ctx.register_table(self.table_name, Arc::new(provider))?; register_functions(&ctx); let df = ctx.sql(&self.sql).await?; Ok(SqlQuery::new(df)) @@ -123,10 +134,12 @@ impl SqlQuery { #[cfg(test)] mod tests { use crate::utils::test::{DatagenExt, FragmentCount, FragmentRowCount, assert_string_matches}; + use crate::{BlobArrayBuilder, blob_field}; use std::collections::HashMap; use std::sync::Arc; use crate::Dataset; + use crate::dataset::write::WriteParams; use all_asserts::assert_true; use arrow_array::cast::AsArray; use arrow_array::types::{Int32Type, Int64Type, UInt64Type}; @@ -135,7 +148,9 @@ mod tests { use arrow_schema::{DataType, Field}; use lance_arrow::ARROW_EXT_NAME_KEY; use lance_arrow::json::ARROW_JSON_EXT_NAME; + use lance_core::datatypes::BlobHandling; use lance_datagen::{array, gen_batch}; + use lance_file::version::LanceFileVersion; #[tokio::test] async fn test_sql_execute() { @@ -186,6 +201,75 @@ mod tests { assert_true!(results.column(3).as_primitive::().value(0) > 100); } + #[tokio::test] + async fn test_sql_blob_all_binary() { + let schema = Arc::new(ArrowSchema::new(vec![blob_field("blob", true)])); + let mut blobs = BlobArrayBuilder::new(2); + blobs.push_bytes(b"foo").unwrap(); + blobs.push_bytes(b"bar").unwrap(); + let batch = RecordBatch::try_new(schema.clone(), vec![blobs.finish().unwrap()]).unwrap(); + let dataset = Dataset::write( + RecordBatchIterator::new([Ok(batch)], schema), + "memory://test_sql_blob_all_binary", + Some(WriteParams { + data_storage_version: Some(LanceFileVersion::V2_3), + ..Default::default() + }), + ) + .await + .unwrap(); + + let batches = dataset + .sql("SELECT blob FROM dataset") + .blob_handling(BlobHandling::AllBinary) + .build() + .await + .unwrap() + .into_batch_records() + .await + .unwrap(); + let blobs = batches[0].column(0).as_binary::(); + assert_eq!(blobs.value(0), b"foo"); + assert_eq!(blobs.value(1), b"bar"); + + // Expressions over the blob column require the planner to see the + // materialized LargeBinary type instead of the blob descriptor struct. + let batches = dataset + .sql("SELECT blob = X'666f6f' FROM dataset") + .blob_handling(BlobHandling::AllBinary) + .build() + .await + .unwrap() + .into_batch_records() + .await + .unwrap(); + let is_foo = batches[0].column(0).as_boolean(); + assert!(is_foo.value(0)); + assert!(!is_foo.value(1)); + + let batches = dataset + .sql("SELECT blob, _rowid, _rowaddr FROM dataset") + .with_row_id(true) + .with_row_addr(true) + .blob_handling(BlobHandling::AllBinary) + .build() + .await + .unwrap() + .into_batch_records() + .await + .unwrap(); + let batch = &batches[0]; + let blobs = batch.column(0).as_binary::(); + assert_eq!(blobs.value(0), b"foo"); + assert_eq!(blobs.value(1), b"bar"); + let row_ids = batch.column(1).as_primitive::(); + assert_eq!(row_ids.value(0), 0); + assert_eq!(row_ids.value(1), 1); + let row_addrs = batch.column(2).as_primitive::(); + assert_eq!(row_addrs.value(0), 0); + assert_eq!(row_addrs.value(1), 1); + } + #[tokio::test] async fn test_sql_count() { let ds = gen_batch() diff --git a/rust/lance/src/dataset/statistics.rs b/rust/lance/src/dataset/statistics.rs index 627ccfc1081..8231a13b30f 100644 --- a/rust/lance/src/dataset/statistics.rs +++ b/rust/lance/src/dataset/statistics.rs @@ -13,7 +13,8 @@ use lance_index::scalar::zonemap::ZoneMapIndex; use lance_io::scheduler::{ScanScheduler, SchedulerConfig}; use roaring::RoaringBitmap; -use super::{Dataset, fragment::FileFragment}; +use super::overlay::{collect_overlay_stale_frags, overlaid_fragments}; +use super::{Dataset, fragment::FileFragment, versions}; use crate::index::{DatasetIndexExt, DatasetIndexInternalExt}; /// Statistics about a single field in the dataset @@ -52,32 +53,12 @@ impl DatasetStatisticsExt for Dataset { }, ) })); - if !self.is_legacy_storage() { - let scan_scheduler = ScanScheduler::new( - self.object_store.clone(), - SchedulerConfig::max_bandwidth(self.object_store.as_ref()), - ); - let schema = self.schema().clone(); - let dataset = self.clone(); - let fragments = self.fragments().as_ref().clone(); - futures::stream::iter(fragments) - .map(|fragment| { - let file_fragment = FileFragment::new(dataset.clone(), fragment); - let schema = schema.clone(); - let scan_scheduler = scan_scheduler.clone(); - async move { file_fragment.storage_stats(&schema, scan_scheduler).await } - }) - .buffer_unordered(self.object_store.io_parallelism()) - .try_for_each(|fragment_stats| { - for (field_id, bytes) in fragment_stats { - if let Some(stats) = field_stats.get_mut(&field_id) { - stats.bytes_on_disk += bytes; - } - } - futures::future::ready(Ok(())) - }) - .await?; - } + versions::collect_data_stats( + self.manifest().data_storage_format.lance_file_format(), + self, + &mut field_stats, + ) + .await?; let field_stats = field_ids .into_iter() .map(|id| field_stats.remove(&(id as u32)).unwrap()) @@ -88,6 +69,35 @@ impl DatasetStatisticsExt for Dataset { } } +pub(super) async fn collect_current_data_stats( + dataset: &Arc, + field_stats: &mut HashMap, +) -> Result<()> { + let scan_scheduler = ScanScheduler::new( + dataset.object_store.clone(), + SchedulerConfig::max_bandwidth(dataset.object_store.as_ref()), + ); + let schema = dataset.schema().clone(); + let fragments = dataset.fragments().as_ref().clone(); + futures::stream::iter(fragments) + .map(|fragment| { + let file_fragment = FileFragment::new(dataset.clone(), fragment); + let schema = schema.clone(); + let scan_scheduler = scan_scheduler.clone(); + async move { file_fragment.storage_stats(&schema, scan_scheduler).await } + }) + .buffer_unordered(dataset.object_store.io_parallelism()) + .try_for_each(|fragment_stats| { + for (field_id, bytes) in fragment_stats { + if let Some(stats) = field_stats.get_mut(&field_id) { + stats.bytes_on_disk += bytes; + } + } + futures::future::ready(Ok(())) + }) + .await +} + /// A read-only handle for cheap, index-derived statistics about a [`Dataset`]. /// /// Obtained via [`Dataset::statistics`]. Groups statistics accessors behind one @@ -108,8 +118,9 @@ impl<'a> DatasetStatistics<'a> { /// /// `None` unless the column's index segments *jointly* cover every live /// fragment and the column can be soundly bounded — fragments appended after - /// the index was built, or a NaN-bearing column, yield `None`. The disjoint - /// segments of a multi-segment index are folded together. + /// the index was built, a data overlay committed after a segment was built, + /// or a NaN-bearing column all yield `None`. The disjoint segments of a + /// multi-segment index are folded together. /// /// When `Some`, the range is a superset of live values, conservative under /// deletion vectors: safe to prune with. See [`ScalarIndex::value_range`]. @@ -159,6 +170,21 @@ impl<'a> DatasetStatistics<'a> { return Ok(None); } + // Soundness: a data overlay committed after a segment was built can move a value + // outside that segment's summaries without the ZoneMap ever seeing it, so the fold + // would no longer bound the live values. There is no way to widen the range without + // reading the overlay, so report "unknown" instead. + let overlaid = overlaid_fragments(&dataset.manifest.fragments); + if !overlaid.is_empty() { + let mut stale = RoaringBitmap::new(); + for idx in &segments { + collect_overlay_stale_frags(idx, &overlaid, &mut stale, dataset.schema())?; + } + if !stale.is_disjoint(dataset.fragment_bitmap.as_ref()) { + return Ok(None); + } + } + // Keep the opened indices alive so the `ZoneMapIndex` refs we fold over // stay borrowed. let mut opened = Vec::with_capacity(segments.len()); diff --git a/rust/lance/src/dataset/take.rs b/rust/lance/src/dataset/take.rs index 5d42e80473b..98bff328ba4 100644 --- a/rust/lance/src/dataset/take.rs +++ b/rust/lance/src/dataset/take.rs @@ -228,7 +228,7 @@ async fn do_take_rows( .with_row_created_at_version(with_row_created_at_version_in_projection) .with_row_last_updated_at_version(with_row_last_updated_at_version_in_projection); let reader = fragment.open(&physical_schema, read_config).await?; - reader.legacy_read_range_as_batch(range).await + reader.read_range_as_batch(range).await } else if row_addr_stats.sorted { // Don't need to re-arrange data, just concatenate let mut batches: Vec<_> = Vec::new(); @@ -299,7 +299,10 @@ async fn do_take_rows( .or_insert_with(|| vec![offset]); }); - let fragments = builder.dataset.get_fragments(); + let addressed_ids: Vec = row_addrs_per_fragment.keys().copied().collect(); + let fragments = builder + .dataset + .get_existing_fragments_from_ids(&addressed_ids); let fragment_and_indices = fragments.into_iter().filter_map(|f| { let row_offset = row_addrs_per_fragment.remove(&(f.id() as u32))?; Some((f, row_offset)) @@ -478,6 +481,14 @@ pub struct TakeBuilder { row_addrs: Option>, projection: Arc, with_row_address: bool, + missing_row_policy: MissingRowPolicy, +} + +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub(super) enum MissingRowPolicy { + #[default] + Ignore, + Error, } impl TakeBuilder { @@ -493,6 +504,7 @@ impl TakeBuilder { projection: Arc::new(projection.into_projection_plan(dataset.clone())?), dataset, with_row_address: false, + missing_row_policy: MissingRowPolicy::default(), }) } @@ -508,6 +520,7 @@ impl TakeBuilder { projection, dataset, with_row_address: false, + missing_row_policy: MissingRowPolicy::default(), }) } @@ -517,6 +530,11 @@ impl TakeBuilder { self } + pub(super) fn with_missing_row_policy(mut self, policy: MissingRowPolicy) -> Self { + self.missing_row_policy = policy; + self + } + /// Execute the take operation and return a single batch pub async fn execute(self) -> Result { take_rows(self).await @@ -537,8 +555,20 @@ impl TakeBuilder { .as_ref() .expect("row_ids must be set if row_addrs is not"); let addrs = if let Some(row_id_index) = get_row_id_index(&self.dataset).await? { - row_id_index - .get_many(row_ids) + let resolved = row_id_index.get_many(row_ids); + if self.missing_row_policy == MissingRowPolicy::Error + && let Some(first_missing_index) = + resolved.iter().position(|address| address.is_none()) + { + let missing_count = resolved.iter().filter(|address| address.is_none()).count(); + return Err(Error::invalid_input(format!( + "Could not resolve all requested row IDs: requested {}, resolved {}; first missing row ID {} was deleted or not found", + row_ids.len(), + row_ids.len() - missing_count, + row_ids[first_missing_index] + ))); + } + resolved .into_iter() .filter_map(|opt| opt.map(|address| address.into())) .collect::>() diff --git a/rust/lance/src/dataset/tests/dataset_aggregate.rs b/rust/lance/src/dataset/tests/dataset_aggregate.rs index 5e55c860f5d..81aa945527d 100644 --- a/rust/lance/src/dataset/tests/dataset_aggregate.rs +++ b/rust/lance/src/dataset/tests/dataset_aggregate.rs @@ -22,7 +22,7 @@ use datafusion_substrait::substrait::proto::{ reference_segment::{self, StructField}, }, extensions::{ - SimpleExtensionDeclaration, SimpleExtensionUri, + SimpleExtensionDeclaration, SimpleExtensionUrn, simple_extension_declaration::{ExtensionFunction, MappingType}, }, function_argument::ArgType, @@ -95,17 +95,6 @@ fn create_aggregate_rel( git_hash: String::new(), producer: "lance-test".to_string(), }), - #[allow(deprecated)] - extension_uris: vec![ - SimpleExtensionUri { - extension_uri_anchor: 1, - uri: "https://github.com/substrait-io/substrait/blob/main/extensions/functions_aggregate_generic.yaml".to_string(), - }, - SimpleExtensionUri { - extension_uri_anchor: 2, - uri: "https://github.com/substrait-io/substrait/blob/main/extensions/functions_arithmetic.yaml".to_string(), - }, - ], extensions, relations: vec![PlanRel { rel_type: Some(datafusion_substrait::substrait::proto::plan_rel::RelType::Root( @@ -117,7 +106,16 @@ fn create_aggregate_rel( }], advanced_extensions: None, expected_type_urls: vec![], - extension_urns: vec![], + extension_urns: vec![ + SimpleExtensionUrn { + extension_urn_anchor: 1, + urn: "https://github.com/substrait-io/substrait/blob/main/extensions/functions_aggregate_generic.yaml".to_string(), + }, + SimpleExtensionUrn { + extension_urn_anchor: 2, + urn: "https://github.com/substrait-io/substrait/blob/main/extensions/functions_arithmetic.yaml".to_string(), + }, + ], parameter_bindings: vec![], type_aliases: vec![], }; @@ -129,9 +127,7 @@ fn create_aggregate_rel( fn agg_extension(anchor: u32, name: &str) -> SimpleExtensionDeclaration { SimpleExtensionDeclaration { mapping_type: Some(MappingType::ExtensionFunction(ExtensionFunction { - #[allow(deprecated)] - extension_uri_reference: 1, - extension_urn_reference: 0, + extension_urn_reference: 1, function_anchor: anchor, name: name.to_string(), })), @@ -1602,7 +1598,7 @@ async fn test_scanner_count_rows_with_fts() { assert_plan_node_equals( plan.clone(), "AggregateExec: mode=Single, gby=[], aggr=[count(Int32(1))] - MatchQuery: column=text, query=document", + MatchQuery: column=text, query=[document]", ) .await .unwrap(); diff --git a/rust/lance/src/dataset/tests/dataset_index.rs b/rust/lance/src/dataset/tests/dataset_index.rs index 961b381e452..d90d23dba68 100644 --- a/rust/lance/src/dataset/tests/dataset_index.rs +++ b/rust/lance/src/dataset/tests/dataset_index.rs @@ -2,7 +2,10 @@ // SPDX-FileCopyrightText: Copyright The Lance Authors use std::collections::{HashMap, HashSet}; -use std::sync::Arc; +use std::future::Future; +use std::pin::Pin; +use std::sync::atomic::{AtomicUsize, Ordering}; +use std::sync::{Arc, Mutex}; use std::vec; use crate::dataset::ROW_ID; @@ -19,28 +22,36 @@ use crate::index::DatasetIndexExt; use arrow::array::{AsArray, GenericListBuilder, GenericStringBuilder}; use arrow::datatypes::UInt64Type; use arrow_array::RecordBatch; -use arrow_array::{Array, GenericStringArray, StructArray, UInt64Array}; +use arrow_array::{Array, GenericStringArray, LargeListArray, ListArray, StructArray, UInt64Array}; use arrow_array::{ ArrayRef, Float32Array, Int32Array, RecordBatchIterator, StringArray, builder::StringDictionaryBuilder, - types::{Float32Type, Int32Type}, + types::{Float32Type, Int32Type, Int64Type}, }; use arrow_schema::{ DataType, Field as ArrowField, Field, Fields as ArrowFields, Schema as ArrowSchema, }; use lance_arrow::ARROW_EXT_NAME_KEY; -use lance_core::cache::LanceCache; +use lance_core::cache::{ + CacheBackend, CacheCodec, CacheEntry, InternalCacheKey, LanceCache, QuickCacheBackend, +}; use lance_core::utils::tempfile::TempStrDir; +use lance_datafusion::exec::ExecutionSummaryCounts; use lance_datagen::{BatchCount, Dimension, RowCount, array, gen_batch}; use lance_file::reader::{FileReader, FileReaderOptions}; -use lance_file::version::LanceFileVersion; +use lance_file::version::{ConcreteFileVersion, LanceFileVersion}; +use lance_index::metrics::{ + COMPOUND_ADDRESS_RESOLUTION_BATCHES_METRIC, COMPOUND_ADDRESSES_RESOLVED_METRIC, + COMPOUND_PEAK_ADDRESS_RESOLUTION_BATCH_SIZE_METRIC, COMPOUND_PEAK_BUFFERED_CANDIDATES_METRIC, + COMPOUND_SCORE_FLOOR_OVERFLOWS_METRIC, +}; use lance_index::optimize::OptimizeOptions; -use lance_index::scalar::FullTextSearchQuery; use lance_index::scalar::inverted::{ - InvertedListFormatVersion, - query::{BooleanQuery, MatchQuery, Occur, Operator, PhraseQuery}, + DocumentGranularity, InvertedListFormatVersion, SCORE_COL, + query::{BooleanQuery, BoostQuery, MatchQuery, Occur, Operator, PhraseQuery}, tokenizer::InvertedIndexParams, }; +use lance_index::scalar::{FullTextSearchQuery, ScalarIndex}; use lance_index::{FtsPrewarmOptions, PrewarmOptions}; use lance_index::{IndexType, scalar::ScalarIndexParams, vector::DIST_COL}; use lance_io::scheduler::{ScanScheduler, SchedulerConfig}; @@ -863,6 +874,722 @@ async fn test_fts_on_multiple_columns() { assert_eq!(results.num_rows(), 1); } +async fn create_fragmented_fts_index(dataset: &mut Dataset, column: &str, with_position: bool) { + create_fragmented_fts_index_with_order(dataset, column, with_position, false).await; +} + +async fn create_fragmented_fts_index_with_order( + dataset: &mut Dataset, + column: &str, + with_position: bool, + reverse_segments: bool, +) { + let index_name = format!("{column}_idx"); + let columns = [column]; + let params = InvertedIndexParams::default().with_position(with_position); + let fragment_ids = dataset + .get_fragments() + .iter() + .map(|fragment| fragment.id() as u32) + .collect::>(); + let mut segments = Vec::with_capacity(fragment_ids.len()); + for fragment_id in &fragment_ids { + let mut builder = dataset + .create_index_builder(&columns, IndexType::Inverted, ¶ms) + .name(index_name.clone()) + .fragments(vec![*fragment_id]); + segments.push(builder.execute_uncommitted().await.unwrap()); + } + if reverse_segments { + segments.reverse(); + } + dataset + .commit_existing_index_segments(&index_name, column, segments) + .await + .unwrap(); + + let segments = + crate::index::scalar::inverted::load_segments(dataset, column, DocumentGranularity::Row) + .await + .unwrap() + .unwrap(); + assert_eq!(segments.len(), fragment_ids.len()); +} + +fn compound_multimatch_query() -> FtsQuery { + MultiMatchQuery::try_new( + "common".to_owned(), + vec!["title".to_owned(), "body".to_owned()], + ) + .unwrap() + .try_with_boosts(vec![10.0, 1.0]) + .unwrap() + .into() +} + +fn compound_match_query(term: &str, column: &str, boost: f32) -> FtsQuery { + MatchQuery::new(term.to_owned()) + .with_column(Some(column.to_owned())) + .with_boost(boost) + .into() +} + +async fn compound_fts_results( + dataset: &Dataset, + query: FtsQuery, + limit: Option, +) -> Vec<(u64, f32)> { + let mut scan = dataset.scan(); + scan.with_row_id() + .full_text_search(FullTextSearchQuery::new_query(query)) + .unwrap(); + if let Some(limit) = limit { + scan.limit(Some(limit), None).unwrap(); + } + let batch = scan.try_into_batch().await.unwrap(); + let row_ids = batch[ROW_ID].as_primitive::().values(); + let scores = batch[SCORE_COL].as_primitive::().values(); + row_ids + .iter() + .copied() + .zip(scores.iter().copied()) + .collect() +} + +async fn assert_compound_fts_top_k(dataset: &Dataset, query: FtsQuery, limit: usize) { + let exhaustive = compound_fts_results(dataset, query.clone(), None).await; + assert!( + exhaustive.len() > limit, + "the exhaustive result must contain candidates beyond k" + ); + let limited = compound_fts_results(dataset, query, Some(limit as i64)).await; + assert_eq!(limited, exhaustive[..limit]); +} + +fn expected_must_score_sum(left: Vec<(u64, f32)>, right: Vec<(u64, f32)>) -> Vec<(u64, f32)> { + let right = right.into_iter().collect::>(); + let mut expected = left + .into_iter() + .filter_map(|(row_id, left_score)| { + right + .get(&row_id) + .map(|right_score| (row_id, left_score + right_score)) + }) + .collect::>(); + expected.sort_unstable_by(|(left_row_id, left_score), (right_row_id, right_score)| { + right_score + .total_cmp(left_score) + .then_with(|| left_row_id.cmp(right_row_id)) + }); + expected +} + +#[tokio::test] +async fn test_boolean_must_scores_sum_across_execution_paths() { + let batch = arrow_array::record_batch!( + ( + "title", + Utf8, + [ + "alpha beta delta", + "alpha alpha beta delta delta", + "alpha delta", + "beta delta", + "alpha beta beta delta delta delta" + ] + ), + ( + "body", + Utf8, + ["gamma", "gamma gamma", "gamma", "gamma", "other"] + ) + ) + .unwrap(); + let schema = batch.schema(); + let mut dataset = Dataset::write( + RecordBatchIterator::new(vec![batch].into_iter().map(Ok), schema), + "memory://", + Some(WriteParams { + max_rows_per_file: 3, + ..Default::default() + }), + ) + .await + .unwrap(); + assert_eq!(dataset.get_fragments().len(), 2); + create_fragmented_fts_index(&mut dataset, "title", false).await; + create_fragmented_fts_index(&mut dataset, "body", false).await; + const LIMIT: usize = 2; + + let match_query = |term: &str, column: &str, boost: f32| -> FtsQuery { + MatchQuery::new(term.to_owned()) + .with_column(Some(column.to_owned())) + .with_boost(boost) + .into() + }; + + let same_column_left = match_query("alpha", "title", 2.0); + let same_column_right = match_query("beta", "title", 3.0); + let expected = expected_must_score_sum( + compound_fts_results(&dataset, same_column_left.clone(), None).await, + compound_fts_results(&dataset, same_column_right.clone(), None).await, + ); + assert!(expected.len() > LIMIT); + let same_column_query: FtsQuery = BooleanQuery::new([ + (Occur::Must, same_column_left.clone()), + (Occur::Must, same_column_right.clone()), + ]) + .into(); + let actual = + compound_fts_results(&dataset, same_column_query.clone(), Some(LIMIT as i64)).await; + assert_eq!(actual, expected[..LIMIT]); + let reversed_same_column_query: FtsQuery = BooleanQuery::new([ + (Occur::Must, same_column_right), + (Occur::Must, same_column_left), + ]) + .into(); + assert_eq!( + compound_fts_results(&dataset, reversed_same_column_query, Some(LIMIT as i64)).await, + expected[..LIMIT] + ); + + let nested_left = match_query("alpha", "title", 2.0); + let nested_middle = match_query("beta", "title", 3.0); + let nested_right = match_query("delta", "title", 5.0); + let expected = expected_must_score_sum( + expected_must_score_sum( + compound_fts_results(&dataset, nested_left.clone(), None).await, + compound_fts_results(&dataset, nested_middle.clone(), None).await, + ), + compound_fts_results(&dataset, nested_right.clone(), None).await, + ); + assert!(expected.len() > LIMIT); + let nested_pair: FtsQuery = + BooleanQuery::new([(Occur::Must, nested_left), (Occur::Must, nested_middle)]).into(); + let nested_query: FtsQuery = + BooleanQuery::new([(Occur::Must, nested_pair), (Occur::Must, nested_right)]).into(); + assert_eq!( + compound_fts_results(&dataset, nested_query, Some(LIMIT as i64)).await, + expected[..LIMIT] + ); + let reversed_nested_pair: FtsQuery = BooleanQuery::new([ + (Occur::Must, match_query("beta", "title", 3.0)), + (Occur::Must, match_query("alpha", "title", 2.0)), + ]) + .into(); + let reversed_nested_query: FtsQuery = BooleanQuery::new([ + (Occur::Must, match_query("delta", "title", 5.0)), + (Occur::Must, reversed_nested_pair), + ]) + .into(); + assert_eq!( + compound_fts_results(&dataset, reversed_nested_query, Some(LIMIT as i64)).await, + expected[..LIMIT] + ); + + let mut scanner = dataset.scan(); + scanner + .full_text_search(FullTextSearchQuery::new_query(same_column_query)) + .unwrap(); + scanner.limit(Some(LIMIT as i64), None).unwrap(); + let plan = scanner.explain_plan(false).await.unwrap(); + assert!( + plan.contains("CompoundFtsScorer"), + "same-column MUST should exercise the composable scorer:\n{plan}" + ); + + let cross_column_left = match_query("alpha", "title", 2.0); + let cross_column_right = match_query("gamma", "body", 3.0); + let expected = expected_must_score_sum( + compound_fts_results(&dataset, cross_column_left.clone(), None).await, + compound_fts_results(&dataset, cross_column_right.clone(), None).await, + ); + assert!(expected.len() > LIMIT); + let cross_column_query: FtsQuery = BooleanQuery::new([ + (Occur::Must, cross_column_left.clone()), + (Occur::Must, cross_column_right.clone()), + ]) + .into(); + let actual = + compound_fts_results(&dataset, cross_column_query.clone(), Some(LIMIT as i64)).await; + assert_eq!(actual, expected[..LIMIT]); + let reversed_cross_column_query: FtsQuery = BooleanQuery::new([ + (Occur::Must, cross_column_right), + (Occur::Must, cross_column_left), + ]) + .into(); + assert_eq!( + compound_fts_results(&dataset, reversed_cross_column_query, Some(LIMIT as i64)).await, + expected[..LIMIT] + ); + + let mut scanner = dataset.scan(); + scanner + .full_text_search(FullTextSearchQuery::new_query(cross_column_query)) + .unwrap(); + scanner.limit(Some(LIMIT as i64), None).unwrap(); + let plan = scanner.explain_plan(false).await.unwrap(); + assert!( + plan.contains("HashJoinExec"), + "cross-column MUST should exercise the exact fallback:\n{plan}" + ); +} + +#[tokio::test] +async fn test_nested_multimatch_limit_propagation() { + let batch = arrow_array::record_batch!( + ( + "title", + Utf8, + [ + "common", + "common filler filler filler filler filler filler filler", + "irrelevant", + "common tie", + "common tie", + "irrelevant" + ] + ), + ( + "body", + Utf8, + [ + "penalty", + "special", + "common", + "neutral", + "neutral", + "common filler filler filler penalty" + ] + ) + ) + .unwrap(); + let schema = batch.schema(); + let mut dataset = Dataset::write( + RecordBatchIterator::new(vec![batch].into_iter().map(Ok), schema), + "memory://", + Some(WriteParams { + max_rows_per_file: 2, + ..Default::default() + }), + ) + .await + .unwrap(); + assert_eq!(dataset.get_fragments().len(), 3); + create_fragmented_fts_index(&mut dataset, "title", false).await; + create_fragmented_fts_index(&mut dataset, "body", false).await; + + let must_query: FtsQuery = BooleanQuery::new([ + (Occur::Must, compound_multimatch_query()), + ( + Occur::Should, + compound_match_query("special", "body", 100.0), + ), + ]) + .into(); + let must_results = compound_fts_results(&dataset, must_query.clone(), None).await; + assert!( + must_results + .windows(2) + .any(|rows| rows[0].1 == rows[1].1 && rows[0].0 < rows[1].0), + "the exhaustive result should include a deterministic score tie" + ); + assert_compound_fts_top_k(&dataset, must_query, 2).await; + + let should_query: FtsQuery = BooleanQuery::new([ + (Occur::Should, compound_multimatch_query()), + ( + Occur::Should, + compound_match_query("special", "body", 100.0), + ), + ]) + .into(); + assert_compound_fts_top_k(&dataset, should_query.clone(), 2).await; + let mut fallback_scanner = dataset.scan(); + fallback_scanner + .with_row_id() + .full_text_search(FullTextSearchQuery::new_query(should_query)) + .unwrap(); + fallback_scanner.limit(Some(2), None).unwrap(); + let fallback_plan = fallback_scanner.explain_plan(false).await.unwrap(); + assert!( + fallback_plan.contains("BooleanQuery"), + "cross-column compound FTS should retain its exact fallback:\n{fallback_plan}" + ); + assert!( + !fallback_plan.contains("CompoundFtsScorer"), + "cross-column compound FTS is not yet supported by the scorer tree:\n{fallback_plan}" + ); + + let boost_query: FtsQuery = BoostQuery::new( + compound_multimatch_query(), + compound_match_query("penalty", "body", 100.0), + Some(1.0), + ) + .into(); + assert_compound_fts_top_k(&dataset, boost_query, 2).await; + + assert_compound_fts_top_k(&dataset, compound_multimatch_query(), 1).await; +} + +#[tokio::test] +async fn test_same_column_compound_scorer_is_exact_and_bounded() { + let batch = arrow_array::record_batch!(( + "text", + Utf8, + [ + "common", + "common filler filler filler", + "irrelevant", + "common tie", + "common tie", + "common filler" + ] + )) + .unwrap(); + let schema = batch.schema(); + let mut dataset = Dataset::write( + RecordBatchIterator::new(vec![batch].into_iter().map(Ok), schema), + "memory://", + Some(WriteParams { + max_rows_per_file: 2, + ..Default::default() + }), + ) + .await + .unwrap(); + create_fragmented_fts_index(&mut dataset, "text", true).await; + + let match_query = |term: &str| { + MatchQuery::new(term.to_owned()) + .with_column(Some("text".to_owned())) + .into() + }; + let query: FtsQuery = BooleanQuery::new([ + (Occur::Must, match_query("common")), + ( + Occur::Should, + BoostQuery::new(match_query("tie"), match_query("filler"), Some(0.5)).into(), + ), + ( + Occur::Should, + PhraseQuery::new("common tie".to_owned()) + .with_column(Some("text".to_owned())) + .into(), + ), + (Occur::MustNot, match_query("irrelevant")), + ]) + .into(); + + assert_compound_fts_top_k(&dataset, query.clone(), 2).await; + + let mut scanner = dataset.scan(); + scanner + .with_row_id() + .full_text_search(FullTextSearchQuery::new_query(query)) + .unwrap(); + scanner.limit(Some(2), None).unwrap(); + let plan = scanner.explain_plan(false).await.unwrap(); + assert!( + plan.contains("CompoundFtsScorer"), + "same-column compound FTS should use the scorer tree:\n{plan}" + ); + assert!( + !plan.contains("HashJoinExec"), + "same-column compound FTS should not materialize intermediate joins:\n{plan}" + ); + + let same_column_multimatch: FtsQuery = MultiMatchQuery::try_new( + "common".to_owned(), + vec!["text".to_owned(), "text".to_owned()], + ) + .unwrap() + .try_with_boosts(vec![1.0, 0.5]) + .unwrap() + .into(); + assert_compound_fts_top_k(&dataset, same_column_multimatch.clone(), 2).await; + let mut scanner = dataset.scan(); + scanner + .with_row_id() + .full_text_search(FullTextSearchQuery::new_query(same_column_multimatch)) + .unwrap(); + scanner.limit(Some(2), None).unwrap(); + let plan = scanner.explain_plan(false).await.unwrap(); + assert!( + plan.contains("CompoundFtsScorer"), + "bounded same-column MultiMatch should use posting-backed scorers:\n{plan}" + ); +} + +#[tokio::test] +async fn test_compound_phrase_confirmation_short_circuit_is_exact() { + let texts = (0..100) + .map(|row| { + if row % 10 == 0 { + "high cost phrase check cheap reject bonus" + } else if row % 5 == 0 { + "high cost phrase check cheap reject" + } else { + "high cost phrase check cheap filler reject" + } + }) + .collect::>(); + let batch = arrow_array::record_batch!(("text", Utf8, texts)).unwrap(); + let schema = batch.schema(); + let mut dataset = Dataset::write( + RecordBatchIterator::new(vec![batch].into_iter().map(Ok), schema), + "memory://", + Some(WriteParams { + max_rows_per_file: 25, + ..Default::default() + }), + ) + .await + .unwrap(); + assert_eq!(dataset.get_fragments().len(), 4); + create_fragmented_fts_index(&mut dataset, "text", true).await; + + let phrase_query = |terms: &str| -> FtsQuery { + PhraseQuery::new(terms.to_owned()) + .with_column(Some("text".to_owned())) + .into() + }; + let query: FtsQuery = BooleanQuery::new([ + (Occur::Must, phrase_query("high cost phrase check")), + (Occur::Must, phrase_query("cheap reject")), + ]) + .into(); + assert_compound_fts_top_k(&dataset, query.clone(), 10).await; + + let nested: FtsQuery = BooleanQuery::new([ + (Occur::Must, query.clone()), + (Occur::Should, compound_match_query("bonus", "text", 1.0)), + ]) + .into(); + assert_compound_fts_top_k(&dataset, nested, 10).await; +} + +#[tokio::test] +async fn test_compound_tie_uses_resolved_row_id() { + let batch = arrow_array::record_batch!(("text", Utf8, vec!["common"; 384])).unwrap(); + let schema = batch.schema(); + let mut dataset = Dataset::write( + RecordBatchIterator::new(vec![batch].into_iter().map(Ok), schema), + "memory://", + Some(WriteParams { + max_rows_per_file: 256, + ..Default::default() + }), + ) + .await + .unwrap(); + create_fragmented_fts_index_with_order(&mut dataset, "text", false, true).await; + + let query: FtsQuery = MultiMatchQuery::try_new( + "common".to_owned(), + vec!["text".to_owned(), "text".to_owned()], + ) + .unwrap() + .into(); + let collected_stats = Arc::new(Mutex::new(None::)); + let stats_setter = collected_stats.clone(); + let mut scanner = dataset.scan(); + scanner + .scan_stats_callback(Arc::new(move |stats| { + *stats_setter.lock().unwrap() = Some(stats.clone()); + })) + .with_row_id() + .full_text_search(FullTextSearchQuery::new_query(query.clone())) + .unwrap(); + scanner.limit(Some(1), None).unwrap(); + let limited = scanner.try_into_batch().await.unwrap(); + let limited_row_id = limited[ROW_ID].as_primitive::().value(0); + + let exhaustive = compound_fts_results(&dataset, query.clone(), None).await; + assert_eq!(limited_row_id, exhaustive[0].0); + assert_eq!(exhaustive.len(), 384); + + let stats = collected_stats.lock().unwrap().take().unwrap(); + assert_eq!( + stats.all_counts.get(COMPOUND_SCORE_FLOOR_OVERFLOWS_METRIC), + Some(&1) + ); + assert_eq!( + stats.all_counts.get(COMPOUND_ADDRESSES_RESOLVED_METRIC), + Some(&384) + ); + assert_eq!( + stats + .all_counts + .get(COMPOUND_ADDRESS_RESOLUTION_BATCHES_METRIC), + Some(&1) + ); + + let mut analyze_scanner = dataset.scan(); + analyze_scanner + .with_row_id() + .full_text_search(FullTextSearchQuery::new_query(query)) + .unwrap(); + analyze_scanner.limit(Some(1), None).unwrap(); + let analysis = analyze_scanner.analyze_plan().await.unwrap(); + let compound_line = analysis + .lines() + .find(|line| line.contains("CompoundFtsScorer")) + .unwrap(); + assert!( + compound_line.contains(&format!("{COMPOUND_PEAK_BUFFERED_CANDIDATES_METRIC}=128")), + "compound FTS metrics missing the bounded candidate peak: {compound_line}" + ); + assert!( + compound_line.contains(&format!( + "{COMPOUND_PEAK_ADDRESS_RESOLUTION_BATCH_SIZE_METRIC}=128" + )), + "compound FTS metrics missing the bounded resolution batch: {compound_line}" + ); +} + +fn nested_fts_batch( + ids: Vec, + a_values: Vec>, + b_values: Vec>, +) -> RecordBatch { + let a_values = Arc::new(StringArray::from(a_values)) as ArrayRef; + let b_values = Arc::new(StringArray::from(b_values)) as ArrayRef; + let struct_array = StructArray::from(vec![ + ( + Arc::new(Field::new("a", DataType::Utf8, true)), + a_values.clone(), + ), + ( + Arc::new(Field::new("b", DataType::Utf8, true)), + b_values.clone(), + ), + ]); + let struct_type = struct_array.data_type().clone(); + RecordBatch::try_new( + Arc::new(ArrowSchema::new(vec![ + Field::new("id", DataType::UInt64, false), + Field::new("s", struct_type, true), + ])), + vec![ + Arc::new(UInt64Array::from(ids)) as ArrayRef, + Arc::new(struct_array) as ArrayRef, + ], + ) + .unwrap() +} + +async fn nested_fts_result_ids(dataset: &Dataset, query: FullTextSearchQuery) -> Vec { + let batch = dataset + .scan() + .full_text_search(query) + .unwrap() + .try_into_batch() + .await + .unwrap(); + let mut ids = batch["id"].as_primitive::().values().to_vec(); + ids.sort_unstable(); + ids +} + +#[tokio::test] +async fn test_fts_on_nested_fields() { + let batch = nested_fts_batch( + vec![0, 1, 2, 3], + vec![ + Some("lance nested alpha"), + Some("plain text"), + None, + Some("phrase target here"), + ], + vec![ + Some("metadata only"), + Some("database nested beta"), + Some("lance beta"), + Some("other"), + ], + ); + let schema = batch.schema(); + let batches = RecordBatchIterator::new(vec![batch].into_iter().map(Ok), schema); + let test_uri = TempStrDir::default(); + let mut dataset = Dataset::write(batches, &test_uri, None).await.unwrap(); + + dataset + .create_index( + &["s.a"], + IndexType::Inverted, + None, + &InvertedIndexParams::default().with_position(true), + true, + ) + .await + .unwrap(); + dataset + .create_index( + &["s.b"], + IndexType::Inverted, + None, + &InvertedIndexParams::default(), + true, + ) + .await + .unwrap(); + + let indices = dataset.load_indices().await.unwrap(); + let indexed_fields = indices + .iter() + .map(|index| dataset.schema().field_path(index.fields[0]).unwrap()) + .collect::>(); + assert_eq!( + indexed_fields, + HashSet::from(["s.a".to_string(), "s.b".to_string()]) + ); + + let query = FullTextSearchQuery::new_query(FtsQuery::Match( + MatchQuery::new("alpha".to_owned()).with_column(Some("s.a".to_owned())), + )); + assert_eq!(nested_fts_result_ids(&dataset, query).await, vec![0]); + + let query = FullTextSearchQuery::new_query(FtsQuery::Match( + MatchQuery::new("beta".to_owned()).with_column(Some("s.b".to_owned())), + )); + assert_eq!(nested_fts_result_ids(&dataset, query).await, vec![1, 2]); + + assert_eq!( + nested_fts_result_ids(&dataset, FullTextSearchQuery::new("lance".to_owned())).await, + vec![0, 2] + ); + + let query = FullTextSearchQuery::new_query(FtsQuery::MultiMatch(MultiMatchQuery { + match_queries: vec![ + MatchQuery::new("nested".to_owned()).with_column(Some("s.a".to_owned())), + MatchQuery::new("nested".to_owned()).with_column(Some("s.b".to_owned())), + ], + })); + assert_eq!(nested_fts_result_ids(&dataset, query).await, vec![0, 1]); + + let query = FullTextSearchQuery::new_query( + PhraseQuery::new("phrase target".to_owned()) + .with_column(Some("s.a".to_owned())) + .into(), + ); + assert_eq!(nested_fts_result_ids(&dataset, query).await, vec![3]); + + let append_batch = nested_fts_batch( + vec![4, 5], + vec![Some("fresh lance append"), Some("plain append")], + vec![Some("other"), Some("fresh beta append")], + ); + let schema = append_batch.schema(); + let batches = RecordBatchIterator::new(vec![append_batch].into_iter().map(Ok), schema); + dataset.append(batches, None).await.unwrap(); + + assert_eq!( + nested_fts_result_ids(&dataset, FullTextSearchQuery::new("fresh".to_owned())).await, + vec![4, 5] + ); +} + #[tokio::test] async fn test_fts_unindexed_data() { let params = InvertedIndexParams::default(); @@ -1803,6 +2530,182 @@ async fn test_fts_index_with_large_string() { test_fts_index::(true).await; } +#[tokio::test] +async fn test_fts_list_index_uses_row_level_documents() { + let tempdir = TempStrDir::default(); + let uri = tempdir.to_owned(); + drop(tempdir); + + let mut list_col = GenericListBuilder::::new(GenericStringBuilder::::new()); + list_col.values().append_value("lance"); + list_col.values().append_value("lance database"); + list_col.append(true); + list_col.values().append_value("database"); + list_col.append(true); + list_col.append(true); + list_col.values().append_null(); + list_col.append(true); + list_col.append(false); + + let docs = Arc::new(list_col.finish()) as ArrayRef; + let ids = Arc::new(UInt64Array::from_iter_values(0..docs.len() as u64)) as ArrayRef; + let batch = RecordBatch::try_new( + Arc::new(ArrowSchema::new(vec![ + ArrowField::new("doc", docs.data_type().clone(), true), + ArrowField::new("id", DataType::UInt64, false), + ])), + vec![docs, ids], + ) + .unwrap(); + let batches = RecordBatchIterator::new(vec![Ok(batch.clone())], batch.schema()); + let mut dataset = Dataset::write(batches, &uri, None).await.unwrap(); + + dataset + .create_index( + &["doc"], + IndexType::Inverted, + None, + &InvertedIndexParams::default(), + true, + ) + .await + .unwrap(); + + let result = dataset + .scan() + .project(&["id"]) + .unwrap() + .full_text_search(FullTextSearchQuery::new("lance".to_owned()).limit(Some(10))) + .unwrap() + .try_into_batch() + .await + .unwrap(); + assert_eq!(result["id"].as_primitive::().values(), &[0]); + + let result = dataset + .scan() + .project(&["id"]) + .unwrap() + .full_text_search(FullTextSearchQuery::new("database".to_owned()).limit(Some(10))) + .unwrap() + .try_into_batch() + .await + .unwrap(); + let mut ids = result["id"] + .as_primitive::() + .values() + .iter() + .copied() + .collect::>(); + ids.sort_unstable(); + assert_eq!(ids, vec![0, 1], "{:?}", result); +} + +#[tokio::test] +async fn test_fts_list_phrase_query_can_cross_elements() { + assert_fts_list_phrase_query_can_cross_elements::().await; +} + +#[tokio::test] +async fn test_fts_large_list_phrase_query_can_cross_elements() { + assert_fts_list_phrase_query_can_cross_elements::().await; +} + +async fn assert_fts_list_phrase_query_can_cross_elements() { + let tempdir = TempStrDir::default(); + let uri = tempdir.to_owned(); + drop(tempdir); + + let mut list_col = GenericListBuilder::::new(GenericStringBuilder::::new()); + let rows: &[&[&str]] = &[ + &["alpha", "beta"], + &["want the", "apple"], + &["want", "apple"], + ]; + for values in rows.iter().copied() { + for value in values { + list_col.values().append_value(value); + } + list_col.append(true); + } + + let docs = Arc::new(list_col.finish()) as ArrayRef; + let ids = Arc::new(UInt64Array::from(vec![0u64, 1, 2])) as ArrayRef; + let batch = RecordBatch::try_new( + Arc::new(ArrowSchema::new(vec![ + ArrowField::new("doc", docs.data_type().clone(), true), + ArrowField::new("id", DataType::UInt64, false), + ])), + vec![docs, ids], + ) + .unwrap(); + let batches = RecordBatchIterator::new(vec![Ok(batch.clone())], batch.schema()); + let mut dataset = Dataset::write(batches, &uri, None).await.unwrap(); + + let cases: [(&str, &[u64]); 3] = [ + ("alpha beta", &[0]), + ("want the apple", &[1]), + ("want apple", &[2]), + ]; + let mut flat_results = Vec::with_capacity(cases.len()); + for (terms, expected) in cases { + let result = dataset + .scan() + .project(&["id"]) + .unwrap() + .full_text_search( + FullTextSearchQuery::new_query( + PhraseQuery::new(terms.to_owned()) + .with_column(Some("doc".to_owned())) + .into(), + ) + .limit(Some(10)), + ) + .unwrap() + .try_into_batch() + .await + .unwrap(); + assert_eq!(result["id"].as_primitive::().values(), expected); + flat_results.push(result); + } + + let params = InvertedIndexParams::default() + .with_position(true) + .remove_stop_words(true); + dataset + .create_index(&["doc"], IndexType::Inverted, None, ¶ms, true) + .await + .unwrap(); + + for ((terms, expected), flat_result) in cases.into_iter().zip(flat_results) { + let indexed_result = dataset + .scan() + .project(&["id"]) + .unwrap() + .full_text_search( + FullTextSearchQuery::new_query( + PhraseQuery::new(terms.to_owned()) + .with_column(Some("doc".to_owned())) + .into(), + ) + .limit(Some(10)), + ) + .unwrap() + .try_into_batch() + .await + .unwrap(); + assert_eq!( + indexed_result["id"].as_primitive::().values(), + expected + ); + assert_eq!( + indexed_result["id"].as_primitive::().values(), + flat_result["id"].as_primitive::().values(), + "query={terms}" + ); + } +} + #[tokio::test] async fn test_fts_accented_chars() { let ds = create_fts_dataset::(false, false, InvertedIndexParams::default()).await; @@ -2155,134 +3058,300 @@ async fn test_prewarm_index_with_position_validation() { .await .unwrap(); - let dataset = open_dataset_with_fresh_session(&uri).await; - let err = dataset - .prewarm_index_with_options("id_idx", &PrewarmOptions::Fts(FtsPrewarmOptions::default())) - .await - .unwrap_err() - .to_string(); - assert_contains!( - err, - "FTS prewarm options are only supported for inverted indices" + let dataset = open_dataset_with_fresh_session(&uri).await; + let err = dataset + .prewarm_index_with_options("id_idx", &PrewarmOptions::Fts(FtsPrewarmOptions::default())) + .await + .unwrap_err() + .to_string(); + assert_contains!( + err, + "FTS prewarm options are only supported for inverted indices" + ); +} + +#[tokio::test] +async fn test_fts_best_effort_prewarm_result_reports_dataset_partial_residency() { + let tmpdir = TempStrDir::default(); + let uri = tmpdir.to_owned(); + drop(tmpdir); + + let doc_col: Arc = Arc::new(GenericStringArray::::from_iter_values( + (0..4096).map(|row| format!("cache pressure token {row}")), + )); + let ids = UInt64Array::from_iter_values(0..doc_col.len() as u64); + let batch = RecordBatch::try_new( + arrow_schema::Schema::new(vec![ + arrow_schema::Field::new("doc", doc_col.data_type().to_owned(), true), + arrow_schema::Field::new("id", DataType::UInt64, false), + ]) + .into(), + vec![Arc::new(doc_col) as ArrayRef, Arc::new(ids) as ArrayRef], + ) + .unwrap(); + let schema = batch.schema(); + let batches = RecordBatchIterator::new(vec![batch].into_iter().map(Ok), schema); + let mut dataset = Dataset::write(batches, &uri, None).await.unwrap(); + dataset + .create_index( + &["doc"], + IndexType::Inverted, + Some("fts_idx".to_owned()), + &InvertedIndexParams::default(), + true, + ) + .await + .unwrap(); + + let session = Arc::new(Session::with_index_cache_backend( + Arc::new(QuickCacheBackend::with_capacity(8 * 1024)), + 8 * 1024, + Arc::new(lance_io::object_store::ObjectStoreRegistry::default()), + )); + let dataset = DatasetBuilder::from_uri(&uri) + .with_session(session) + .load() + .await + .unwrap(); + let options = PrewarmOptions::Fts(FtsPrewarmOptions::default().best_effort()); + + dataset + .prewarm_index_with_options("fts_idx", &options) + .await + .unwrap(); + let result = dataset + .prewarm_index_with_options_result("fts_idx", &options) + .await + .unwrap(); + + assert!( + !result.fully_resident, + "tiny cache should make best-effort dataset prewarm report partial residency" + ); + let diagnostics = result + .diagnostics + .expect("partial dataset prewarm should return aggregate diagnostics"); + assert!(diagnostics.partition_count > 0); + assert!(!diagnostics.failing_segments.is_empty() || !diagnostics.failing_partitions.is_empty()); + assert!( + diagnostics + .failing_partitions + .iter() + .all(|partition| partition.segment_id.is_some()), + "dataset aggregation should attach segment ids to partition diagnostics" + ); +} + +#[derive(Debug)] +struct SingleScalarContainerCacheBackend { + inner: QuickCacheBackend, + scalar_container_inserts: AtomicUsize, +} + +impl SingleScalarContainerCacheBackend { + fn new(capacity: usize) -> Self { + Self { + inner: QuickCacheBackend::with_capacity(capacity), + scalar_container_inserts: AtomicUsize::new(0), + } + } + + fn rejects_scalar_container(entry: &CacheEntry, codec: Option<&CacheCodec>) -> bool { + codec.is_none() && entry.as_ref().is::>() + } +} + +#[async_trait::async_trait] +impl CacheBackend for SingleScalarContainerCacheBackend { + async fn get(&self, key: &InternalCacheKey, codec: Option) -> Option { + self.inner.get(key, codec).await + } + + async fn insert( + &self, + key: &InternalCacheKey, + entry: CacheEntry, + size_bytes: usize, + codec: Option, + ) { + if Self::rejects_scalar_container(&entry, codec.as_ref()) + && self + .scalar_container_inserts + .fetch_add(1, Ordering::Relaxed) + > 0 + { + return; + } + self.inner.insert(key, entry, size_bytes, codec).await; + } + + async fn get_or_insert<'a>( + &self, + key: &InternalCacheKey, + loader: Pin> + Send + 'a>>, + codec: Option, + ) -> Result<(CacheEntry, bool)> { + if codec.is_none() { + if let Some(entry) = self.inner.get(key, None).await { + return Ok((entry, true)); + } + let (entry, size_bytes) = loader.await?; + if Self::rejects_scalar_container(&entry, None) + && self + .scalar_container_inserts + .fetch_add(1, Ordering::Relaxed) + > 0 + { + return Ok((entry, false)); + } + self.inner + .insert(key, entry.clone(), size_bytes, codec) + .await; + return Ok((entry, false)); + } + self.inner.get_or_insert(key, loader, codec).await + } + + async fn clear(&self) { + self.inner.clear().await; + } + + async fn num_entries(&self) -> usize { + self.inner.num_entries().await + } + + async fn size_bytes(&self) -> usize { + self.inner.size_bytes().await + } + + fn approx_num_entries(&self) -> usize { + self.inner.approx_num_entries() + } + + fn approx_size_bytes(&self) -> usize { + self.inner.approx_size_bytes() + } +} + +async fn two_segment_fts_dataset(uri: &str) -> Dataset { + let schema = Arc::new(arrow_schema::Schema::new(vec![ + arrow_schema::Field::new("doc", DataType::Utf8, true), + arrow_schema::Field::new("id", DataType::UInt64, false), + ])); + let make_batch = |fragment: u64| { + RecordBatch::try_new( + schema.clone(), + vec![ + Arc::new(GenericStringArray::::from_iter_values( + (0..32u64).map(|row| format!("segment {fragment} token {row}")), + )) as ArrayRef, + Arc::new(UInt64Array::from_iter_values( + (0..32u64).map(|row| fragment * 32 + row), + )) as ArrayRef, + ], + ) + .unwrap() + }; + + let mut dataset = Dataset::write( + RecordBatchIterator::new(vec![make_batch(0)].into_iter().map(Ok), schema.clone()), + uri, + None, + ) + .await + .unwrap(); + dataset + .create_index( + &["doc"], + IndexType::Inverted, + Some("fts_idx".to_owned()), + &InvertedIndexParams::default(), + true, + ) + .await + .unwrap(); + dataset + .append( + RecordBatchIterator::new(vec![make_batch(1)].into_iter().map(Ok), schema), + None, + ) + .await + .unwrap(); + dataset + .optimize_indices(&OptimizeOptions::append()) + .await + .unwrap(); + assert_eq!( + dataset.load_indices_by_name("fts_idx").await.unwrap().len(), + 2 + ); + dataset +} + +async fn open_with_single_scalar_container_cache(uri: &str) -> Dataset { + let session = Arc::new(Session::with_index_cache_backend( + Arc::new(SingleScalarContainerCacheBackend::new(128 * 1024 * 1024)), + 128 * 1024 * 1024, + Arc::new(lance_io::object_store::ObjectStoreRegistry::default()), + )); + DatasetBuilder::from_uri(uri) + .with_session(session) + .load() + .await + .unwrap() +} + +#[tokio::test] +async fn test_fts_best_effort_prewarm_reports_missing_scalar_container() { + let tmpdir = TempStrDir::default(); + let uri = tmpdir.to_owned(); + two_segment_fts_dataset(&uri).await; + let dataset = open_with_single_scalar_container_cache(&uri).await; + let options = PrewarmOptions::Fts(FtsPrewarmOptions::default().best_effort()); + + let result = dataset + .prewarm_index_with_options_result("fts_idx", &options) + .await + .unwrap(); + + assert!( + !result.fully_resident, + "dataset prewarm must be partial when a selected segment's scalar index \ + container is not cache-resident" + ); + let diagnostics = result + .diagnostics + .expect("missing scalar container should produce aggregate diagnostics"); + assert_eq!( + diagnostics.failing_segments.len(), + 1, + "only the rejected scalar container should be reported as missing" + ); + assert!( + diagnostics.failing_partitions.is_empty(), + "a missing scalar container should not be represented as a partition failure" ); + let failure = &diagnostics.failing_segments[0]; + assert!(!failure.scalar_index_container_resident); + assert!(!failure.scalar_index_container_matches_prewarmed); } -/// Cache backend that exercises the serialization codec on every insert and -/// returns deserialized entries on every get. Items without a codec fall -/// through to an in-memory passthrough so that non-FTS cache traffic still -/// works during the test. -/// -/// Mirrors the helper in `rust/lance/src/index/vector/ivf/v2.rs` tests; if a -/// third user appears, lift this into a shared test utility. -mod fts_serializing_backend { - use std::collections::HashMap; - use std::pin::Pin; - - use futures::Future; - use lance_core::Result; - use lance_core::cache::{ - CacheBackend, CacheCodec, CacheEntry, InternalCacheKey, MokaCacheBackend, - }; - - type SerializedEntry = (bytes::Bytes, CacheCodec, usize); - - #[derive(Debug)] - pub struct SerializingBackend { - serialized: tokio::sync::Mutex>, - passthrough: MokaCacheBackend, - } - - impl SerializingBackend { - pub fn new() -> Self { - Self { - serialized: tokio::sync::Mutex::new(HashMap::new()), - passthrough: MokaCacheBackend::with_capacity(256 * 1024 * 1024), - } - } - - pub async fn serialized_entry_count(&self) -> usize { - self.serialized.lock().await.len() - } - } - - #[async_trait::async_trait] - impl CacheBackend for SerializingBackend { - async fn get( - &self, - key: &InternalCacheKey, - codec: Option, - ) -> Option { - let guard = self.serialized.lock().await; - if let Some((bytes, stored_codec, _)) = guard.get(key) { - return stored_codec.deserialize(&bytes.clone()).hit(); - } - drop(guard); - self.passthrough.get(key, codec).await - } - - async fn insert( - &self, - key: &InternalCacheKey, - entry: CacheEntry, - size_bytes: usize, - codec: Option, - ) { - if let Some(codec) = codec { - let mut bytes = Vec::new(); - codec - .serialize(&entry, &mut bytes) - .expect("serialization should succeed"); - self.serialized - .lock() - .await - .insert(key.clone(), (bytes::Bytes::from(bytes), codec, size_bytes)); - } else { - self.passthrough.insert(key, entry, size_bytes, None).await; - } - } - - async fn get_or_insert<'a>( - &self, - key: &InternalCacheKey, - loader: Pin> + Send + 'a>>, - codec: Option, - ) -> Result<(CacheEntry, bool)> { - if let Some(entry) = self.get(key, codec).await { - return Ok((entry, true)); - } - let (entry, size) = loader.await?; - self.insert(key, entry.clone(), size, codec).await; - Ok((entry, false)) - } - - async fn invalidate_prefix(&self, prefix: &str) { - self.serialized - .lock() - .await - .retain(|k, _| !k.starts_with(prefix)); - self.passthrough.invalidate_prefix(prefix).await; - } - - async fn clear(&self) { - self.serialized.lock().await.clear(); - self.passthrough.clear().await; - } - - async fn num_entries(&self) -> usize { - self.serialized.lock().await.len() + self.passthrough.num_entries().await - } +#[tokio::test] +async fn test_fts_strict_prewarm_fails_missing_scalar_container() { + let tmpdir = TempStrDir::default(); + let uri = tmpdir.to_owned(); + two_segment_fts_dataset(&uri).await; + let dataset = open_with_single_scalar_container_cache(&uri).await; + let options = PrewarmOptions::Fts(FtsPrewarmOptions::default()); - async fn size_bytes(&self) -> usize { - let serialized: usize = self - .serialized - .lock() - .await - .values() - .map(|(_, _, s)| *s) - .sum(); - serialized + self.passthrough.size_bytes().await - } - } + let err = dataset + .prewarm_index_with_options_result("fts_idx", &options) + .await + .expect_err("strict prewarm should fail after final scalar-container audit"); + assert!( + err.to_string().contains("resident scalar index container"), + "strict error should describe the missing scalar container: {err}" + ); } /// Validates the OSS-741 contract: after FTS prewarm through a serializing @@ -2295,7 +3364,7 @@ mod fts_serializing_backend { async fn test_fts_prewarm_with_serializing_backend_serves_query_with_no_io() { use lance_io::assert_io_eq; - use fts_serializing_backend::SerializingBackend; + use crate::utils::test::serializing_cache::SerializingCacheBackend; let tmpdir = TempStrDir::default(); let uri = tmpdir.to_owned(); @@ -2334,7 +3403,7 @@ async fn test_fts_prewarm_with_serializing_backend_serves_query_with_no_io() { // Re-open the dataset on a session whose cache backend serializes every // entry through its codec. Set a generous capacity so nothing is evicted // before we query. - let backend = Arc::new(SerializingBackend::new()); + let backend = Arc::new(SerializingCacheBackend::new()); let session = Arc::new(Session::with_index_cache_backend( backend.clone(), 128 * 1024 * 1024, @@ -2413,7 +3482,7 @@ async fn test_fts_prewarm_with_serializing_backend_serves_query_with_no_io() { async fn test_btree_prewarm_with_serializing_backend_serves_query_with_no_io() { use lance_io::assert_io_eq; - use fts_serializing_backend::SerializingBackend; + use crate::utils::test::serializing_cache::SerializingCacheBackend; let tmpdir = TempStrDir::default(); let uri = tmpdir.to_owned(); @@ -2449,7 +3518,7 @@ async fn test_btree_prewarm_with_serializing_backend_serves_query_with_no_io() { // Re-open on a session whose cache backend serializes every entry through // its codec, with a generous capacity so nothing is evicted before we query. - let backend = Arc::new(SerializingBackend::new()); + let backend = Arc::new(SerializingCacheBackend::new()); let session = Arc::new(Session::with_index_cache_backend( backend.clone(), 128 * 1024 * 1024, @@ -2476,9 +3545,32 @@ async fn test_btree_prewarm_with_serializing_backend_serves_query_with_no_io() { but the serializing store was empty" ); - // After prewarm, an indexed-filter query must reconstruct the index and - // every page it touches from the cache, deserializing via the codec, with - // no disk IO. Project only `_rowid` so the scan does not read a data column. + drop(dataset); + let backend = Arc::new(backend.restart()); + assert_eq!( + backend.l1_entry_count().await, + 0, + "restarting must discard the in-memory L1" + ); + assert_eq!( + backend.serialized_entry_count().await, + serialized_after_prewarm, + "restarting must retain only the serialized entries" + ); + let session = Arc::new(Session::with_index_cache_backend( + backend, + 128 * 1024 * 1024, + Arc::new(lance_io::object_store::ObjectStoreRegistry::default()), + )); + let dataset = DatasetBuilder::from_uri(&uri) + .with_session(session) + .load() + .await + .unwrap(); + + // After recreating the backend, an indexed-filter query must reconstruct + // the index and every page it touches from serialized bytes, with no disk + // IO. Project only `_rowid` so the scan does not read a data column. dataset.object_store.as_ref().io_stats_incremental(); let result = dataset @@ -2516,7 +3608,7 @@ async fn test_btree_prewarm_with_serializing_backend_serves_query_with_no_io() { async fn test_bitmap_prewarm_with_serializing_backend_serves_query_with_no_io() { use lance_io::assert_io_eq; - use fts_serializing_backend::SerializingBackend; + use crate::utils::test::serializing_cache::SerializingCacheBackend; let tmpdir = TempStrDir::default(); let uri = tmpdir.to_owned(); @@ -2550,7 +3642,7 @@ async fn test_bitmap_prewarm_with_serializing_backend_serves_query_with_no_io() .await .unwrap(); - let backend = Arc::new(SerializingBackend::new()); + let backend = Arc::new(SerializingCacheBackend::new()); let session = Arc::new(Session::with_index_cache_backend( backend.clone(), 128 * 1024 * 1024, @@ -2599,6 +3691,95 @@ async fn test_bitmap_prewarm_with_serializing_backend_serves_query_with_no_io() ); } +#[rstest] +#[case::list(false)] +#[case::large_list(true)] +#[tokio::test] +async fn test_label_list_index_types(#[case] large_list: bool) { + let test_uri = TempStrDir::default(); + let label_values = vec![ + Some(vec![Some(1), Some(2)]), + Some(vec![Some(2)]), + Some(vec![Some(1)]), + Some(vec![Some(3)]), + ]; + let labels: ArrayRef = if large_list { + Arc::new(LargeListArray::from_iter_primitive::( + label_values, + )) + } else { + Arc::new(ListArray::from_iter_primitive::( + label_values, + )) + }; + let schema = Arc::new(ArrowSchema::new(vec![ + ArrowField::new("id", DataType::Int32, false), + ArrowField::new("labels", labels.data_type().clone(), true), + ])); + let batch = RecordBatch::try_new( + schema.clone(), + vec![Arc::new(Int32Array::from(vec![0, 1, 2, 3])), labels], + ) + .unwrap(); + let reader = RecordBatchIterator::new(vec![Ok(batch)], schema); + let mut dataset = Dataset::write(reader, &test_uri, None).await.unwrap(); + + let expected = dataset + .scan() + .project(&["id"]) + .unwrap() + .filter("array_has_any(labels, [1])") + .unwrap() + .try_into_batch() + .await + .unwrap(); + let expected_ids = expected + .column(0) + .as_primitive::() + .values() + .to_vec(); + assert_eq!(expected_ids, vec![0, 2]); + + dataset + .create_index( + &["labels"], + IndexType::LabelList, + Some("labels_idx".to_owned()), + &ScalarIndexParams::default(), + true, + ) + .await + .unwrap(); + + let result = dataset + .scan() + .project(&["id"]) + .unwrap() + .filter("array_has_any(labels, [1])") + .unwrap() + .try_into_batch() + .await + .unwrap(); + let result_ids = result + .column(0) + .as_primitive::() + .values() + .to_vec(); + assert_eq!(result_ids, expected_ids); + + let plan = dataset + .scan() + .filter("array_has_any(labels, [1])") + .unwrap() + .explain_plan(false) + .await + .unwrap(); + assert!( + plan.contains("ScalarIndexQuery") && plan.contains("LabelList"), + "Expected LabelList scalar index query in plan: {plan}" + ); +} + /// LabelList analogue: after prewarming, an `array_has_any` query against a /// `LabelList` index serves results without any further IO. Exercises the /// `LabelListIndexState` codec (which embeds the inner bitmap state and the @@ -2607,7 +3788,7 @@ async fn test_bitmap_prewarm_with_serializing_backend_serves_query_with_no_io() async fn test_label_list_prewarm_with_serializing_backend_serves_query_with_no_io() { use lance_io::assert_io_eq; - use fts_serializing_backend::SerializingBackend; + use crate::utils::test::serializing_cache::SerializingCacheBackend; let tmpdir = TempStrDir::default(); let uri = tmpdir.to_owned(); @@ -2651,7 +3832,7 @@ async fn test_label_list_prewarm_with_serializing_backend_serves_query_with_no_i "test dataset must contain at least one row whose labels include 3" ); - let backend = Arc::new(SerializingBackend::new()); + let backend = Arc::new(SerializingCacheBackend::new()); let session = Arc::new(Session::with_index_cache_backend( backend.clone(), 128 * 1024 * 1024, @@ -2759,6 +3940,130 @@ async fn test_fts_phrase_query_with_removed_stop_words() { } } +#[tokio::test] +async fn test_fts_without_index_on_zero_fragment_dataset_is_empty() { + let schema = Arc::new(ArrowSchema::new(vec![ + ArrowField::new("id", DataType::UInt64, false), + ArrowField::new("doc", DataType::Utf8, true), + ])); + let empty_reader = RecordBatchIterator::new(vec![], schema); + let dataset = Dataset::write(empty_reader, "memory://", None) + .await + .unwrap(); + assert!(dataset.fragments().is_empty()); + + let mut scan = dataset.scan(); + scan.project(&["id"]).unwrap(); + scan.full_text_search(FullTextSearchQuery::new_query( + MatchQuery::new("alpha".to_owned()) + .with_column(Some("doc".to_owned())) + .into(), + )) + .unwrap(); + let plan = scan.explain_plan(false).await.unwrap(); + assert!(plan.contains("EmptyExec"), "unexpected plan: {plan}"); + assert_eq!(scan.try_into_batch().await.unwrap().num_rows(), 0); + + let mut phrase_scan = dataset.scan(); + phrase_scan.project(&["id"]).unwrap(); + phrase_scan + .full_text_search(FullTextSearchQuery::new_query( + PhraseQuery::new("alpha beta".to_owned()) + .with_column(Some("doc".to_owned())) + .into(), + )) + .unwrap(); + let phrase_plan = phrase_scan.explain_plan(false).await.unwrap(); + assert!( + phrase_plan.contains("EmptyExec"), + "unexpected phrase plan: {phrase_plan}" + ); + assert_eq!(phrase_scan.try_into_batch().await.unwrap().num_rows(), 0); +} + +#[tokio::test] +async fn test_fts_phrase_query_normalizes_leading_stop_word_position() { + let tmpdir = TempStrDir::default(); + let uri = tmpdir.to_owned(); + drop(tmpdir); + + let initial = RecordBatch::try_from_iter(vec![ + ("id", Arc::new(UInt64Array::from(vec![99])) as ArrayRef), + ( + "doc", + Arc::new(StringArray::from(vec!["placeholder"])) as ArrayRef, + ), + ]) + .unwrap(); + let initial_reader = RecordBatchIterator::new(vec![Ok(initial.clone())], initial.schema()); + let mut dataset = Dataset::write(initial_reader, &uri, None).await.unwrap(); + let index_params = InvertedIndexParams::default() + .with_position(true) + .remove_stop_words(true); + dataset + .create_index(&["doc"], IndexType::Inverted, None, &index_params, true) + .await + .unwrap(); + + let appended = RecordBatch::try_from_iter(vec![ + ("id", Arc::new(UInt64Array::from(vec![0, 1, 2])) as ArrayRef), + ( + "doc", + Arc::new(StringArray::from(vec![ + "alpha beta", + "the alpha beta", + "alpha gap beta", + ])) as ArrayRef, + ), + ]) + .unwrap(); + let appended_reader = RecordBatchIterator::new(vec![Ok(appended.clone())], appended.schema()); + dataset = Dataset::write( + appended_reader, + Arc::new(dataset), + Some(WriteParams { + mode: WriteMode::Append, + ..Default::default() + }), + ) + .await + .unwrap(); + let appended_fragment = dataset.fragments().last().unwrap().clone(); + let query = FullTextSearchQuery::new_query( + PhraseQuery::new("the alpha beta".to_owned()) + .with_column(Some("doc".to_owned())) + .into(), + ); + + let mut flat_scan = dataset.scan(); + flat_scan.with_fragments(vec![appended_fragment.clone()]); + flat_scan.project(&["id"]).unwrap(); + flat_scan.full_text_search(query.clone()).unwrap(); + let flat_result = flat_scan.try_into_batch().await.unwrap(); + let mut flat_ids = flat_result["id"] + .as_primitive::() + .values() + .to_vec(); + flat_ids.sort_unstable(); + assert_eq!(flat_ids, vec![0, 1]); + + dataset + .create_index(&["doc"], IndexType::Inverted, None, &index_params, true) + .await + .unwrap(); + let mut indexed_scan = dataset.scan(); + indexed_scan.with_fragments(vec![appended_fragment]); + indexed_scan.project(&["id"]).unwrap(); + indexed_scan.full_text_search(query).unwrap(); + let indexed_result = indexed_scan.try_into_batch().await.unwrap(); + let mut indexed_ids = indexed_result["id"] + .as_primitive::() + .values() + .to_vec(); + indexed_ids.sort_unstable(); + assert_eq!(indexed_ids, flat_ids); +} + #[tokio::test] async fn test_fts_phrase_query_preserves_stop_word_gaps() { let tmpdir = TempStrDir::default(); @@ -2818,6 +4123,140 @@ async fn test_fts_phrase_query_preserves_stop_word_gaps() { assert!(!ids.contains(&3), "ids={ids:?}"); } +fn json_batch(values: Vec<&str>) -> RecordBatch { + let mut metadata = HashMap::new(); + metadata.insert( + ARROW_EXT_NAME_KEY.to_string(), + ARROW_JSON_EXT_NAME.to_string(), + ); + let schema = Arc::new(ArrowSchema::new(vec![ + ArrowField::new("json", DataType::Utf8, false).with_metadata(metadata), + ])); + RecordBatch::try_new(schema, vec![Arc::new(StringArray::from(values))]).unwrap() +} + +async fn json_btree_dataset(initial_values: Vec<&str>) -> Dataset { + let initial = json_batch(initial_values); + let initial_schema = initial.schema(); + let reader = RecordBatchIterator::new([Ok(initial)], initial_schema); + let mut dataset = Dataset::write(reader, "memory://", None).await.unwrap(); + let params = ScalarIndexParams::new("json".to_string()).with_params(&serde_json::json!({ + "target_index_type": "btree", + "path": "val", + })); + dataset + .create_index( + &["json"], + IndexType::Scalar, + Some("json_idx".to_string()), + ¶ms, + false, + ) + .await + .unwrap(); + dataset +} + +#[tokio::test] +async fn test_json_btree_index_statistics() { + let dataset = json_btree_dataset(vec![ + r#"{"val": 1000}"#, + r#"{"val": 2000}"#, + r#"{"val": 3000}"#, + ]) + .await; + + let stats: serde_json::Value = + serde_json::from_str(&dataset.index_statistics("json_idx").await.unwrap()).unwrap(); + + assert_eq!(stats["name"], "json_idx"); + assert_eq!(stats["num_indices"], 1); + assert_eq!(stats["num_indexed_rows"], 3); + assert_eq!(stats["num_unindexed_rows"], 0); + assert_eq!(stats["indices"][0]["min"], "1000"); + assert_eq!(stats["indices"][0]["max"], "3000"); +} + +#[rstest] +#[case::merge(false)] +#[case::append_rebuild(true)] +#[tokio::test] +async fn test_optimize_json_btree_index(#[case] append_rebuild: bool) { + let mut dataset = json_btree_dataset(vec![r#"{"val": 1000}"#]).await; + + for values in [ + vec![r#"{"val": null}"#, r#"{"val": 2000}"#], + vec![r#"{"other": 1}"#, r#"{"val": 3000}"#], + ] { + let batch = json_batch(values); + let schema = batch.schema(); + dataset + .append(RecordBatchIterator::new([Ok(batch)], schema), None) + .await + .unwrap(); + } + + let options = if append_rebuild { + OptimizeOptions::append() + } else { + OptimizeOptions::default() + }; + dataset.optimize_indices(&options).await.unwrap(); + + let indexed_fragments = dataset + .load_indices_by_name("json_idx") + .await + .unwrap() + .iter() + .flat_map(|index| index.fragment_bitmap.as_ref().unwrap().iter()) + .collect::>(); + assert_eq!(indexed_fragments, HashSet::from([0, 1, 2])); + + let result = dataset + .scan() + .filter("json_get_int(json, 'val') >= 2000") + .unwrap() + .try_into_batch() + .await + .unwrap(); + assert_eq!(result.num_rows(), 2); +} + +#[tokio::test] +async fn test_optimize_append_json_btree_preserves_float_type() { + let mut dataset = json_btree_dataset(vec![r#"{"val": 1.5}"#]).await; + let appended = json_batch(vec![r#"{"val": 2}"#]); + let schema = appended.schema(); + dataset + .append(RecordBatchIterator::new([Ok(appended)], schema), None) + .await + .unwrap(); + dataset + .optimize_indices(&OptimizeOptions::append()) + .await + .unwrap(); + + let predicate = "json_get_float(json, 'val') = 2.0"; + let indexed = dataset + .scan() + .filter(predicate) + .unwrap() + .try_into_batch() + .await + .unwrap(); + let mut baseline_scan = dataset.scan(); + baseline_scan.use_scalar_index(false); + let baseline = baseline_scan + .filter(predicate) + .unwrap() + .try_into_batch() + .await + .unwrap(); + + assert_eq!(baseline.num_rows(), 1); + assert_eq!(indexed.num_rows(), baseline.num_rows()); +} + async fn prepare_json_dataset() -> (Dataset, String) { let text_col = Arc::new(StringArray::from(vec![ r#"{ @@ -3631,7 +5070,7 @@ async fn test_index_inherits_dataset_file_version() { // Verify that the index file uses the same version as the dataset assert_eq!( index_reader.metadata().version(), - dataset_version, + dataset_version.resolve(), "Index file should use the same format version as the dataset" ); @@ -3660,7 +5099,7 @@ async fn test_index_inherits_dataset_file_version() { assert_eq!( aux_reader.metadata().version(), - dataset_version, + dataset_version.resolve(), "Auxiliary index file should use the same format version as the dataset" ); } @@ -3739,7 +5178,7 @@ async fn test_legacy_dataset_uses_v2_0_for_indexes() { // Verify that the index file uses V2_0 (not legacy) assert_eq!( index_reader.metadata().version(), - LanceFileVersion::V2_0, + ConcreteFileVersion::V2_0, "Index files should never use legacy format, even for legacy datasets" ); } @@ -3808,3 +5247,50 @@ async fn test_manifest_read_recovers_from_stale_size() { assert_eq!(indices.len(), 1); assert_eq!(indices[0].name, "id_idx"); } + +/// `load_segment_params` must match the fully opened segment's params, +/// including `custom_stop_words` — the field `InvertedIndexDetails` loses. +#[tokio::test] +async fn test_load_segment_params_full_fidelity() { + use crate::index::DatasetIndexInternalExt; + use lance_index::metrics::NoOpMetricsCollector; + use lance_index::scalar::inverted::{DocumentGranularity, InvertedIndex}; + + let batch = RecordBatch::try_new( + arrow_schema::Schema::new(vec![Field::new("text", DataType::Utf8, false)]).into(), + vec![Arc::new(StringArray::from(vec![ + "the quick brown fox", + "lazy dogs sleep", + ]))], + ) + .unwrap(); + let schema = batch.schema(); + let stream = RecordBatchIterator::new(vec![batch].into_iter().map(Ok), schema); + let mut dataset = Dataset::write(stream, "memory://test/segment_params", None) + .await + .unwrap(); + + let params = InvertedIndexParams::default().custom_stop_words(Some(vec!["quick".to_string()])); + dataset + .create_index(&["text"], IndexType::Inverted, None, ¶ms, true) + .await + .unwrap(); + + let segments = crate::index::scalar::load_segments(&dataset, "text", DocumentGranularity::Row) + .await + .unwrap() + .expect("FTS index segments"); + let read = crate::index::scalar::load_segment_params(&dataset, &segments[0]) + .await + .unwrap(); + + let generic = dataset + .open_generic_index("text", &segments[0].uuid, &NoOpMetricsCollector) + .await + .unwrap(); + let opened = generic + .as_any() + .downcast_ref::() + .expect("inverted index"); + assert_eq!(&read, opened.params()); +} diff --git a/rust/lance/src/dataset/tests/dataset_io.rs b/rust/lance/src/dataset/tests/dataset_io.rs index 1f8c7226bf2..b79fc11d11d 100644 --- a/rust/lance/src/dataset/tests/dataset_io.rs +++ b/rust/lance/src/dataset/tests/dataset_io.rs @@ -12,6 +12,7 @@ use crate::dataset::WriteMode::Overwrite; use crate::dataset::builder::DatasetBuilder; use crate::dataset::{ManifestWriteConfig, write_manifest_file}; use crate::session::Session; +use crate::session::caches::ManifestKey; use crate::{Dataset, Error, Result}; use lance_table::format::DataStorageFormat; @@ -34,8 +35,8 @@ use lance_arrow::{ARROW_EXT_META_KEY, ARROW_EXT_NAME_KEY}; use lance_core::utils::tempfile::{TempStdDir, TempStrDir}; use lance_datagen::{BatchCount, RowCount, array, gen_batch}; use lance_file::{ - version::LanceFileVersion, - writer::{FileWriter, FileWriterOptions}, + version::{ConcreteFileVersion, LanceFileVersion}, + writer::FileWriterOptions, }; use lance_io::assert_io_eq; use lance_table::feature_flags; @@ -448,13 +449,10 @@ async fn test_create_data_file_rejects_nested_schema_mismatch() { .create(&dataset.data_dir().join(file_name)) .await .unwrap(); - let mut writer = FileWriter::try_new( + let mut writer = lance_file::versions::v2_2::create_writer( object_writer, crate::datatypes::Schema::try_from(schema.as_ref()).unwrap(), - FileWriterOptions { - format_version: Some(LanceFileVersion::V2_2), - ..Default::default() - }, + FileWriterOptions::default(), ) .unwrap(); writer.write_batch(&batch).await.unwrap(); @@ -871,6 +869,76 @@ async fn test_load_manifest_iops() { assert_io_eq!(io_stats, read_iops, 1); } +#[tokio::test] +async fn test_checkout_removed_version_not_served_from_cache() { + let test_uri = TempStrDir::default(); + let session = Arc::new(Session::default()); + let schema = Arc::new(ArrowSchema::new(vec![ArrowField::new( + "i", + DataType::Int32, + false, + )])); + let batch = RecordBatch::try_new( + schema.clone(), + vec![Arc::new(Int32Array::from_iter_values(0..10_i32))], + ) + .unwrap(); + let dataset = Dataset::write( + RecordBatchIterator::new(vec![Ok(batch)], schema.clone()), + &test_uri, + Some(WriteParams { + session: Some(session.clone()), + ..Default::default() + }), + ) + .await + .unwrap(); + + let version = dataset.manifest().version; + let location = dataset.manifest_location().clone(); + let cache = session.metadata_cache.for_dataset(&dataset.uri); + + assert!( + cache + .get_with_key(&ManifestKey { + version, + e_tag: location.e_tag.as_deref(), + }) + .await + .is_some(), + "manifest should be cached after the write" + ); + dataset.checkout_version(version).await.unwrap(); + + // Remove the version from storage, as cleanup (or a manual delete) would. + dataset.object_store.delete(&location.path).await.unwrap(); + + let resolved = dataset + .commit_handler + .resolve_version_location(&dataset.base, version, &dataset.object_store.inner) + .await + .unwrap(); + assert!( + resolved.size.is_none(), + "resolving a removed version must fall back to a size-less location, got {:?}", + resolved.size + ); + + cache + .insert_with_key( + &ManifestKey { + version, + e_tag: None, + }, + Arc::new(dataset.manifest().clone()), + ) + .await; + assert!( + dataset.checkout_version(version).await.is_err(), + "checkout of a version removed from storage must not be served from cache" + ); +} + #[rstest] #[tokio::test] async fn test_write_params( @@ -914,15 +982,18 @@ async fn test_write_params( assert_eq!(dataset.count_fragments(), 10); for fragment in &fragments { assert_eq!(fragment.count_rows(None).await.unwrap(), 100); - let reader = fragment - .open(dataset.schema(), FragReadConfig::default()) - .await - .unwrap(); // No group / batch concept in v2 if data_storage_version == LanceFileVersion::Legacy { - assert_eq!(reader.legacy_num_batches(), 10); - for i in 0..reader.legacy_num_batches() as u32 { - assert_eq!(reader.legacy_num_rows_in_batch(i).unwrap(), 10); + let reader = crate::dataset::versions::open_v1_fragment_reader( + fragment, + dataset.schema(), + &FragReadConfig::default(), + ) + .await + .unwrap(); + assert_eq!(reader.num_batches(), 10); + for i in 0..reader.num_batches() as u32 { + assert_eq!(reader.num_rows_in_batch(i).unwrap(), 10); } } } @@ -931,7 +1002,11 @@ async fn test_write_params( #[rstest] #[tokio::test] async fn test_write_manifest( - #[values(LanceFileVersion::Legacy, LanceFileVersion::Stable)] + #[values( + LanceFileVersion::Legacy, + LanceFileVersion::Stable, + LanceFileVersion::Next + )] data_storage_version: LanceFileVersion, ) { use lance_table::feature_flags::FLAG_UNKNOWN; @@ -980,8 +1055,12 @@ async fn test_write_manifest( assert_eq!( manifest.data_storage_format, - DataStorageFormat::new(data_storage_version) + DataStorageFormat::new(data_storage_version.resolve()) ); + assert!(!matches!( + manifest.data_storage_format.version.to_manifest_string(), + "stable" | "next" + )); assert_eq!(manifest.reader_feature_flags, 0); // Create one with deletions @@ -1100,8 +1179,8 @@ async fn test_rle_v2_v23_write_and_append() { .await .unwrap(); assert_eq!( - manifest.data_storage_format.lance_file_version().unwrap(), - LanceFileVersion::V2_3 + manifest.data_storage_format.lance_file_format(), + ConcreteFileVersion::V2_3 ); let append_batch = RecordBatch::try_new( @@ -1123,12 +1202,8 @@ async fn test_rle_v2_v23_write_and_append() { .unwrap(); assert_eq!( - dataset - .manifest - .data_storage_format - .lance_file_version() - .unwrap(), - LanceFileVersion::V2_3 + dataset.manifest.data_storage_format.lance_file_format(), + ConcreteFileVersion::V2_3 ); let actual = dataset.scan().try_into_batch().await.unwrap(); @@ -1170,12 +1245,8 @@ async fn test_rle_v2_uncommitted_create_commits_v23_storage() { .await .unwrap(); assert_eq!( - dataset - .manifest - .data_storage_format - .lance_file_version() - .unwrap(), - LanceFileVersion::V2_3 + dataset.manifest.data_storage_format.lance_file_format(), + ConcreteFileVersion::V2_3 ); } @@ -1210,12 +1281,8 @@ async fn test_rle_v2_shallow_clone_preserves_v23_storage() { .await .unwrap(); assert_eq!( - clone - .manifest - .data_storage_format - .lance_file_version() - .unwrap(), - LanceFileVersion::V2_3 + clone.manifest.data_storage_format.lance_file_format(), + ConcreteFileVersion::V2_3 ); } @@ -1453,6 +1520,132 @@ async fn test_deep_clone( assert_eq!(count_files(store, &dst_root, "_deletions").await, 0); } +#[tokio::test] +async fn test_deep_clone_recognizes_ambiguous_commit_as_own() { + use crate::utils::test::{AmbiguousCommitHandler, AmbiguousFailure}; + + let test_dir = TempStdDir::default(); + let source_dir = test_dir.join("source"); + let source_uri = source_dir.to_str().unwrap(); + let target_dir = test_dir.join("target"); + let target_uri = target_dir.to_str().unwrap(); + let handler = Arc::new(AmbiguousCommitHandler::default()); + let data_reader = gen_batch() + .col("id", array::step::()) + .into_reader_rows(RowCount::from(32), BatchCount::from(1)); + let mut source = Dataset::write( + data_reader, + source_uri, + Some(WriteParams { + commit_handler: Some(handler.clone()), + ..Default::default() + }), + ) + .await + .unwrap(); + let source_transaction_file = source.manifest().transaction_file.clone(); + + handler.fail_next(AmbiguousFailure::LandAndConflict); + let cloned = source + .deep_clone(target_uri, source.version().version, None) + .await + .expect("readback must identify the deep-clone transaction that landed"); + + assert_eq!(cloned.count_rows(None).await.unwrap(), 32); + assert_ne!(cloned.manifest().transaction_file, source_transaction_file); + assert!(cloned.manifest().transaction_section.is_some()); +} + +// Uses an in-memory source store to force a cross-store copy. The in-memory store has +// known platform-specific quirks on Windows (it reads back empty there; see the note in +// tests/resource_tests.rs), so this test is gated to non-Windows. The local write side is +// covered on Windows by `test_deep_clone` (same-store), and the cross-store streaming path +// against real cloud stores is platform-agnostic std/tokio I/O. +#[cfg(not(windows))] +#[rstest] +#[tokio::test] +async fn test_deep_clone_cross_store( + #[values(LanceFileVersion::Legacy, LanceFileVersion::Stable)] + data_storage_version: LanceFileVersion, +) { + // Source lives in an in-memory store while the target is a local directory, so the + // two stores have different `store_prefix`es and `deep_clone` must stream files from + // the source store to the target store (the cross-account code path). + let session = Arc::new(Session::default()); + let test_dir = TempStdDir::default(); + let clone_dir = test_dir.join("clone_ds"); + let cloned_uri = clone_dir.to_str().unwrap(); + + // 64 rows across 4 files exercises the multi-fragment copy path. + let data_reader = gen_batch() + .col("id", array::step::()) + .col("val", array::fill_utf8("deep".to_string())) + .into_reader_rows(RowCount::from(64), BatchCount::from(1)); + + let mut dataset = Dataset::write( + data_reader, + "memory://cross_store_src", + Some(WriteParams { + max_rows_per_file: 16, + max_rows_per_group: 16, + data_storage_version: Some(data_storage_version), + session: Some(session.clone()), + ..Default::default() + }), + ) + .await + .unwrap(); + assert_ne!(dataset.object_store.store_prefix, ""); + + // Create a scalar index so the index files and the manifest index section are also + // copied across stores (the index section is read through the source store at commit). + dataset + .create_index( + &["id"], + IndexType::Scalar, + Some("id_idx".to_string()), + &ScalarIndexParams::default(), + false, + ) + .await + .unwrap(); + + // Delete some rows so a deletion file is also streamed across stores. + dataset.delete("id < 10").await.unwrap(); + let cloned_dataset = dataset + .deep_clone(cloned_uri, dataset.version().version, None) + .await + .unwrap(); + + // The clone targets a local store, distinct from the in-memory source. + assert_ne!( + cloned_dataset.object_store.store_prefix, + dataset.object_store.store_prefix + ); + assert!(cloned_dataset.manifest().base_paths.is_empty()); + + // Re-open the clone from a fresh session to prove the files were physically copied + // into the target store and the clone is fully independent of the source store. + let reopened = DatasetBuilder::from_uri(cloned_uri).load().await.unwrap(); + let batches = reopened + .scan() + .try_into_stream() + .await + .unwrap() + .try_collect::>() + .await + .unwrap(); + let total_rows: usize = batches.iter().map(|b| b.num_rows()).sum(); + assert_eq!(total_rows, 54); // 64 rows - 10 deletions + + // The scalar index must have been copied and resolve against the target store, with its + // base reference normalized to local (no external base_paths). + let cloned_indices = reopened.load_indices().await.unwrap(); + assert_eq!(cloned_indices.len(), 1); + assert_eq!(cloned_indices.first().unwrap().name, "id_idx"); + assert!(cloned_indices.iter().all(|idx| idx.base_id.is_none())); +} + // Helper: count files under a dataset directory (data/_indices/_deletions) async fn count_files(store: &ObjectStore, root: &Path, prefix: &str) -> usize { use futures::StreamExt; @@ -1926,6 +2119,53 @@ async fn append_dictionary( } } +#[rstest] +#[case::appended_null_value( + (0..=i8::MAX) + .map(|value| Some(format!("value-{value}"))) + .collect(), + (0..=i8::MAX).map(Some).chain([None]).collect() +)] +#[case::existing_unaddressable_null_value( + (0..130) + .map(|value| (value != 129).then(|| format!("value-{value}"))) + .collect(), + vec![Some(0_i8), None] +)] +#[tokio::test] +async fn write_rejects_dictionary_null_index_outside_declared_key_range( + #[case] values: Vec>, + #[case] indices: Vec>, +) { + let dictionary = Arc::new(StringArray::from(values)); + let indices = Int8Array::from(indices); + let dictionary = Int8DictionaryArray::try_new(indices, dictionary).unwrap(); + let schema = Arc::new(ArrowSchema::new(vec![ArrowField::new( + "d", + dictionary.data_type().clone(), + true, + )])); + let batch = RecordBatch::try_new(schema.clone(), vec![Arc::new(dictionary)]).unwrap(); + + let error = Dataset::write( + RecordBatchIterator::new([Ok(batch)], schema), + "memory://", + Some(WriteParams { + data_storage_version: Some(LanceFileVersion::V2_0), + ..Default::default() + }), + ) + .await + .expect_err("the widened indices cannot be represented by Int8"); + + assert!(matches!(error, Error::InvalidInput { .. })); + assert!( + error + .to_string() + .contains("dictionary indices use 32 bits but the declared Int8 key type uses 8 bits") + ); +} + #[rstest] #[tokio::test] async fn overwrite_dataset( @@ -1985,9 +2225,10 @@ async fn overwrite_dataset( let fragments = dataset.get_fragments(); assert_eq!(fragments.len(), 1); - // Fragment ids reset after overwrite. - assert_eq!(fragments[0].id(), 0); - assert_eq!(dataset.manifest.max_fragment_id(), Some(0)); + // Fragment ids continue from the dataset's high water mark after an + // overwrite; they are never reused. + assert_eq!(fragments[0].id(), 1); + assert_eq!(dataset.manifest.max_fragment_id(), Some(1)); let actual_ds = Dataset::open(&test_uri).await.unwrap(); assert_eq!(actual_ds.version().version, 2); @@ -2254,12 +2495,8 @@ async fn test_overwrite_mixed_version() { .unwrap(); assert_eq!( - dataset - .manifest - .data_storage_format - .lance_file_version() - .unwrap(), - LanceFileVersion::Legacy + dataset.manifest.data_storage_format.lance_file_format(), + ConcreteFileVersion::V1 ); let reader = RecordBatchIterator::new(vec![data].into_iter().map(Ok), schema); @@ -2275,12 +2512,8 @@ async fn test_overwrite_mixed_version() { .unwrap(); assert_eq!( - dataset - .manifest - .data_storage_format - .lance_file_version() - .unwrap(), - LanceFileVersion::Legacy + dataset.manifest.data_storage_format.lance_file_format(), + ConcreteFileVersion::V1 ); } diff --git a/rust/lance/src/dataset/tests/dataset_merge_update.rs b/rust/lance/src/dataset/tests/dataset_merge_update.rs index 4b81f62dc6c..3d03af9d91f 100644 --- a/rust/lance/src/dataset/tests/dataset_merge_update.rs +++ b/rust/lance/src/dataset/tests/dataset_merge_update.rs @@ -6,14 +6,17 @@ use std::sync::Arc; use std::time::Duration; use std::vec; +use crate::dataset::CommitBuilder; use crate::dataset::ROW_ID; use crate::dataset::WriteDestination; +use crate::dataset::builder::DatasetBuilder; use crate::dataset::optimize::{CompactionOptions, compact_files}; +use crate::dataset::schema_evolution::ColumnAlteration; use crate::dataset::transaction::{DataReplacementGroup, Operation}; use crate::dataset::{AutoCleanupParams, MergeInsertBuilder, ProjectionRequest, UpdateBuilder}; use crate::index::DatasetIndexExt; use crate::{Dataset, Error}; -use lance_core::ROW_ADDR; +use lance_core::{ROW_ADDR, ROW_LAST_UPDATED_AT_VERSION}; use lance_index::IndexType; use lance_index::optimize::OptimizeOptions; use lance_index::scalar::FullTextSearchQuery; @@ -28,7 +31,7 @@ use arrow_array::RecordBatch; use arrow_array::{Array, LargeBinaryArray, StructArray}; use arrow_array::{ ArrayRef, Float32Array, Int32Array, ListArray, RecordBatchIterator, StringArray, - types::Int32Type, + types::{Int32Type, UInt64Type}, }; use arrow_schema::{DataType, Field as ArrowField, Fields, Schema as ArrowSchema}; use lance_arrow::BLOB_META_KEY; @@ -36,7 +39,7 @@ use lance_core::utils::tempfile::{TempDir, TempStrDir}; use lance_datafusion::utils::reader_to_stream; use lance_datagen::{BatchCount, RowCount, array, gen_batch}; use lance_file::version::LanceFileVersion; -use lance_file::writer::FileWriter; + use lance_io::utils::CachedFileSize; use lance_table::format::{BasePath, DataFile, Fragment}; @@ -745,7 +748,7 @@ async fn test_datafile_replacement() { .create(&Path::from("data/test.lance")) .await .unwrap(); - let mut writer = FileWriter::try_new( + let mut writer = lance_file::versions::v2_1::create_writer( object_writer, schema.as_ref().try_into().unwrap(), Default::default(), @@ -830,6 +833,7 @@ async fn test_datafile_partial_replacement() { Operation::Merge { fragments: vec![fragment], schema: extended_schema.as_ref().try_into().unwrap(), + preserves_nullability: true, }, Some(2), None, @@ -852,7 +856,7 @@ async fn test_datafile_partial_replacement() { .create(&Path::from("data/test.lance")) .await .unwrap(); - let mut writer = FileWriter::try_new( + let mut writer = lance_file::versions::v2_1::create_writer( object_writer, partial_schema.as_ref().try_into().unwrap(), Default::default(), @@ -864,7 +868,7 @@ async fn test_datafile_partial_replacement() { writer.write_batch(&batch).await.unwrap(); writer.finish().await.unwrap(); - let (major, minor) = lance_file::version::LanceFileVersion::Stable.to_numbers(); + let (major, minor) = LanceFileVersion::Stable.resolve().to_data_file_numbers(); // find the datafile we want to replace let new_data_file = DataFile { @@ -1016,6 +1020,7 @@ async fn test_datafile_replacement_error() { Operation::Merge { fragments: vec![fragment], schema: extended_schema.as_ref().try_into().unwrap(), + preserves_nullability: true, }, Some(2), None, @@ -2143,6 +2148,87 @@ async fn test_merge_insert_flat_index_stable_row_id_multiple_indexes() { ); } +#[tokio::test] +async fn test_data_replacement_advances_row_lineage() { + let schema = Arc::new(ArrowSchema::new(vec![ArrowField::new( + "value", + DataType::Int32, + true, + )])); + let batch = + RecordBatch::try_new(schema.clone(), vec![Arc::new(Int32Array::from(vec![1, 2]))]).unwrap(); + let reader = RecordBatchIterator::new(vec![Ok(batch)], schema.clone()); + let dataset = Dataset::write( + reader, + "memory://", + Some(WriteParams { + enable_stable_row_ids: true, + ..Default::default() + }), + ) + .await + .unwrap(); + + let replacement = RecordBatch::try_new( + schema.clone(), + vec![Arc::new(Int32Array::from(vec![10, 20]))], + ) + .unwrap(); + let object_writer = dataset + .object_store + .create(&Path::from("data/lineage_replacement.lance")) + .await + .unwrap(); + let mut writer = lance_file::versions::v2_1::create_writer( + object_writer, + schema.as_ref().try_into().unwrap(), + Default::default(), + ) + .unwrap(); + writer.write_batch(&replacement).await.unwrap(); + writer.finish().await.unwrap(); + + let frag = dataset.get_fragment(0).unwrap(); + let mut new_data_file = frag.data_file_for_field(0).unwrap().clone(); + new_data_file.path = "lineage_replacement.lance".to_string(); + + let read_version = dataset.version().version; + let dataset = Dataset::commit( + WriteDestination::Dataset(Arc::new(dataset)), + Operation::DataReplacement { + replacements: vec![DataReplacementGroup(0, new_data_file)], + }, + Some(read_version), + None, + None, + Arc::new(Default::default()), + false, + ) + .await + .unwrap(); + assert_eq!(dataset.version().version, 2); + + // The rows read differently now, so their last-updated stamp has to name + // the version that changed them or get_updated_rows will never see them. + let batch = dataset + .scan() + .project(&["value", ROW_LAST_UPDATED_AT_VERSION]) + .unwrap() + .try_into_batch() + .await + .unwrap(); + assert_eq!( + batch["value"].as_primitive::().values(), + &[10, 20] + ); + assert_eq!( + batch[ROW_LAST_UPDATED_AT_VERSION] + .as_primitive::() + .values(), + &[2, 2] + ); +} + /// DataReplacement should invalidate index fragment bitmaps for replaced fields. #[tokio::test] async fn test_data_replacement_invalidates_index_bitmap() { @@ -2199,7 +2285,7 @@ async fn test_data_replacement_invalidates_index_bitmap() { .create(&Path::from("data/replacement.lance")) .await .unwrap(); - let mut writer = FileWriter::try_new( + let mut writer = lance_file::versions::v2_1::create_writer( object_writer, single_col_schema.as_ref().try_into().unwrap(), Default::default(), @@ -2312,7 +2398,7 @@ fn build_overlay_frag(prev: &Fragment, field_id: i32, new_file: &str) -> Fragmen new_file, vec![field_id], vec![0], - &LanceFileVersion::default(), + LanceFileVersion::default().resolve(), None, ); overlay @@ -2377,7 +2463,7 @@ async fn test_merge_rewriting_indexed_column_keeps_index_consistent() { .unwrap(); let new_a_path = dataset.data_dir().join("merge_new_a.lance"); let object_writer = dataset.object_store.create(&new_a_path).await.unwrap(); - let mut writer = FileWriter::try_new( + let mut writer = lance_file::versions::v2_1::create_writer( object_writer, a_only.as_ref().try_into().unwrap(), Default::default(), @@ -2396,6 +2482,7 @@ async fn test_merge_rewriting_indexed_column_keeps_index_consistent() { Operation::Merge { fragments: vec![overlay], schema: schema.as_ref().try_into().unwrap(), + preserves_nullability: true, }, Some(read_version), None, @@ -2425,7 +2512,6 @@ async fn test_merge_rewriting_indexed_column_keeps_index_consistent() { /// stale index entries are blocked at query time. #[tokio::test] async fn test_data_replacement_populates_invalidated_bitmap() { - use lance_file::writer::FileWriter; use object_store::path::Path; let schema = Arc::new(ArrowSchema::new(vec![ @@ -2481,7 +2567,7 @@ async fn test_data_replacement_populates_invalidated_bitmap() { .create(&Path::from("data/replacement_inv.lance")) .await .unwrap(); - let mut writer = FileWriter::try_new( + let mut writer = lance_file::versions::v2_1::create_writer( object_writer, value_schema.as_ref().try_into().unwrap(), Default::default(), @@ -2606,7 +2692,7 @@ async fn test_fts_stale_entries_after_data_replacement() { .create(&replacement_path) .await .unwrap(); - let mut writer = FileWriter::try_new( + let mut writer = lance_file::versions::v2_1::create_writer( object_writer, schema.as_ref().try_into().unwrap(), Default::default(), @@ -2783,7 +2869,7 @@ async fn test_vector_index_after_data_replacement() { .create(&replacement_path) .await .unwrap(); - let mut writer = FileWriter::try_new( + let mut writer = lance_file::versions::v2_1::create_writer( object_writer, schema.as_ref().try_into().unwrap(), Default::default(), @@ -4294,3 +4380,506 @@ async fn test_merge_insert_target_all_bases() { assert!(all_rows.contains(&row), "missing row {:?}", row); } } + +/// A write landing between the tightening scan and its commit falsifies the +/// claim, leaving a table that validates but cannot be scanned. +#[rstest] +#[case::tightening_conflicts(true, true)] +#[case::rename_does_not(false, false)] +#[tokio::test] +async fn test_alter_columns_conflicts_only_when_asserting( + #[case] tighten: bool, + #[case] expect_conflict: bool, +) { + let batch = arrow_array::record_batch!(("value", Int32, [1, 2])).unwrap(); + let reader = RecordBatchIterator::new(vec![Ok(batch.clone())], batch.schema()); + let dataset = Dataset::write(reader, "memory://", None).await.unwrap(); + + // Leave the first handle a version behind, so the alteration commits stale. + let appended = InsertBuilder::new(WriteDestination::Dataset(Arc::new(dataset.clone()))) + .with_params(&WriteParams { + mode: WriteMode::Append, + ..Default::default() + }) + .execute(vec![ + arrow_array::record_batch!(("value", Int32, [3])).unwrap(), + ]) + .await + .unwrap(); + assert_eq!(appended.version().version, 2); + + let mut stale = dataset; + let alteration = if tighten { + ColumnAlteration::new("value".into()).set_nullable(false) + } else { + ColumnAlteration::new("value".into()).rename("renamed".into()) + }; + let result = stale.alter_columns(&[alteration]).await; + assert_eq!( + result.is_err(), + expect_conflict, + "tighten={tighten}: got {result:?}" + ); +} + +/// read_version is the version the data was validated against. Declaring it +/// honestly puts a later tightening inside the conflict window; declaring a +/// later version skips the checks, which is a caller bug, not a guarantee. +#[rstest] +#[case::honest_read_version_conflicts(1, true)] +#[case::misdeclared_read_version_commits(2, false)] +#[tokio::test] +async fn test_stale_append_protection_follows_read_version( + #[case] declared: u64, + #[case] expect_conflict: bool, +) { + let batch = arrow_array::record_batch!(("value", Int32, [1, 2])).unwrap(); + let reader = RecordBatchIterator::new(vec![Ok(batch.clone())], batch.schema()); + let dataset = Arc::new(Dataset::write(reader, "memory://", None).await.unwrap()); + + let mut tightened = dataset.schema().clone(); + tightened.fields[0].nullable = false; + let tightened = Dataset::commit( + WriteDestination::Dataset(dataset.clone()), + Operation::Project { + schema: tightened, + preserves_nullability: false, + }, + Some(dataset.version().version), + None, + None, + Arc::new(Default::default()), + false, + ) + .await + .unwrap(); + assert_eq!(tightened.version().version, 2); + + let mut append = InsertBuilder::new(WriteDestination::Dataset(dataset.clone())) + .with_params(&WriteParams { + mode: WriteMode::Append, + ..Default::default() + }) + .execute_uncommitted(vec![batch]) + .await + .unwrap(); + append.read_version = declared; + + let result = CommitBuilder::new(Arc::new(tightened)) + .execute(append) + .await; + assert_eq!( + result.is_err(), + expect_conflict, + "declared={declared}: got {result:?}" + ); +} + +/// A field added and tightened after the write snapshot: the tightening's +/// claim is in the honest conflict window, so the operation-wide barrier +/// rejects the stale append. Added-but-nullable commits, since synthesized +/// nulls are legal there. +#[rstest] +#[case::added_then_tightened(true, true)] +#[case::added_still_nullable(false, false)] +#[tokio::test] +async fn test_stale_append_vs_field_added_since( + #[case] tighten: bool, + #[case] expect_conflict: bool, +) { + let batch = arrow_array::record_batch!(("value", Int32, [1, 2])).unwrap(); + let reader = RecordBatchIterator::new(vec![Ok(batch.clone())], batch.schema()); + let dataset = Arc::new(Dataset::write(reader, "memory://", None).await.unwrap()); + + let append = InsertBuilder::new(WriteDestination::Dataset(dataset.clone())) + .with_params(&WriteParams { + mode: WriteMode::Append, + ..Default::default() + }) + .execute_uncommitted(vec![batch]) + .await + .unwrap(); + + let mut latest = dataset.as_ref().clone(); + latest + .add_columns( + crate::dataset::NewColumnTransform::SqlExpressions(vec![( + "new_value".to_string(), + "value".to_string(), + )]), + None, + None, + ) + .await + .unwrap(); + if tighten { + latest + .alter_columns(&[ColumnAlteration::new("new_value".into()).set_nullable(false)]) + .await + .unwrap(); + } + + let result = CommitBuilder::new(Arc::new(latest)).execute(append).await; + assert_eq!( + result.is_err(), + expect_conflict, + "tighten={tighten}: got {result:?}" + ); +} + +#[tokio::test] +async fn test_alter_columns_rejects_cast_with_tightening() { + let batch = arrow_array::record_batch!(("value", Int32, [1, 2])).unwrap(); + let reader = RecordBatchIterator::new(vec![Ok(batch.clone())], batch.schema()); + let mut dataset = Dataset::write(reader, "memory://", None).await.unwrap(); + + let err = dataset + .alter_columns(&[ColumnAlteration::new("value".into()) + .set_nullable(false) + .cast_to(DataType::Int64)]) + .await + .unwrap_err(); + assert!(err.to_string().contains("same call"), "got: {err}"); + + // Separately, both succeed. + dataset + .alter_columns(&[ColumnAlteration::new("value".into()).cast_to(DataType::Int64)]) + .await + .unwrap(); + dataset + .alter_columns(&[ColumnAlteration::new("value".into()).set_nullable(false)]) + .await + .unwrap(); +} + +/// A merge introducing a non-nullable column claims non-null, so a stale +/// append, whose fragments omit the column and would read as null, conflicts. +/// A nullable column keeps the long-standing behavior: the append commits and +/// its rows legally read as null. +#[rstest] +#[case::required_column_conflicts("1", true)] +#[case::nullable_column_commits("value", false)] +#[tokio::test] +async fn test_stale_append_vs_column_added_by_merge( + #[case] expression: &str, + #[case] expect_conflict: bool, +) { + let batch = arrow_array::record_batch!(("value", Int32, [1, 2])).unwrap(); + let reader = RecordBatchIterator::new(vec![Ok(batch.clone())], batch.schema()); + let dataset = Arc::new(Dataset::write(reader, "memory://", None).await.unwrap()); + + let append = InsertBuilder::new(WriteDestination::Dataset(dataset.clone())) + .with_params(&WriteParams { + mode: WriteMode::Append, + ..Default::default() + }) + .execute_uncommitted(vec![batch]) + .await + .unwrap(); + + // A literal is non-nullable; a projection of a nullable column is nullable. + let mut latest = dataset.as_ref().clone(); + latest + .add_columns( + crate::dataset::NewColumnTransform::SqlExpressions(vec![( + "new_value".to_string(), + expression.to_string(), + )]), + None, + None, + ) + .await + .unwrap(); + + let result = CommitBuilder::new(Arc::new(latest)).execute(append).await; + assert_eq!( + result.is_err(), + expect_conflict, + "expression={expression}: got {result:?}" + ); + if let Ok(committed) = result { + committed.scan().try_into_batch().await.unwrap(); + } +} + +/// A cast rewrites the column under a new field id, so a stale append omits it +/// and its rows read as null. Casting a non-nullable column therefore claims +/// non-null and conflicts; casting a nullable one does not. +#[rstest] +#[case::cast_of_required_conflicts(true, true)] +#[case::cast_of_nullable_commits(false, false)] +#[tokio::test] +async fn test_stale_append_vs_cast(#[case] tighten_first: bool, #[case] expect_conflict: bool) { + let batch = arrow_array::record_batch!(("value", Int32, [1, 2])).unwrap(); + let reader = RecordBatchIterator::new(vec![Ok(batch.clone())], batch.schema()); + let mut dataset = Dataset::write(reader, "memory://", None).await.unwrap(); + + if tighten_first { + dataset + .alter_columns(&[ColumnAlteration::new("value".into()).set_nullable(false)]) + .await + .unwrap(); + } + + // Stage against the pre-cast schema, so the cast lands inside the window. + let staged = Arc::new(dataset.clone()); + let append = InsertBuilder::new(WriteDestination::Dataset(staged.clone())) + .with_params(&WriteParams { + mode: WriteMode::Append, + ..Default::default() + }) + .execute_uncommitted(vec![batch]) + .await + .unwrap(); + + dataset + .alter_columns(&[ColumnAlteration::new("value".into()).cast_to(DataType::Int64)]) + .await + .unwrap(); + + let result = CommitBuilder::new(Arc::new(dataset)).execute(append).await; + assert_eq!( + result.is_err(), + expect_conflict, + "tighten_first={tighten_first}: got {result:?}" + ); + if let Ok(committed) = result { + committed.scan().try_into_batch().await.unwrap(); + } +} + +/// A subcolumn addition (V2.2+) merges a new child into an existing struct. +/// Stale rows supply the parent, so a required new child would read as +/// unmasked null: the merge claims and the stale append conflicts. A nullable +/// new child under a nullable parent masks itself and keeps appends flowing. +/// Under a non-nullable parent even a nullable child claims: the reader +/// synthesizes missing subcolumns against the column's declared nullability, +/// so the stale fragment could not be read at all. +#[rstest] +#[case::required_child_conflicts(true, false, true)] +#[case::required_child_nullable_parent_conflicts(true, true, true)] +#[case::nullable_child_nullable_parent_commits(false, true, false)] +#[case::nullable_child_required_parent_conflicts(false, false, true)] +#[tokio::test] +async fn test_stale_append_vs_sub_column_added_by_merge( + #[case] child_required: bool, + #[case] parent_nullable: bool, + #[case] expect_conflict: bool, +) { + let schema = Arc::new(ArrowSchema::new(vec![ArrowField::new( + "b", + DataType::Struct(Fields::from(vec![ArrowField::new( + "c", + DataType::Int32, + true, + )])), + parent_nullable, + )])); + let struct_batch = |values: Vec| { + RecordBatch::try_new( + schema.clone(), + vec![Arc::new(StructArray::from(vec![( + Arc::new(ArrowField::new("c", DataType::Int32, true)), + Arc::new(Int32Array::from(values)) as ArrayRef, + )]))], + ) + .unwrap() + }; + let dataset = Arc::new( + Dataset::write( + RecordBatchIterator::new(vec![Ok(struct_batch(vec![1, 2]))], schema.clone()), + "memory://", + Some(WriteParams { + data_storage_version: Some(LanceFileVersion::V2_2), + ..Default::default() + }), + ) + .await + .unwrap(), + ); + + let append = InsertBuilder::new(WriteDestination::Dataset(dataset.clone())) + .with_params(&WriteParams { + mode: WriteMode::Append, + ..Default::default() + }) + .execute_uncommitted(vec![struct_batch(vec![3])]) + .await + .unwrap(); + + let new_child = Arc::new(ArrowField::new("d", DataType::Int32, !child_required)); + let sub_schema = Arc::new(ArrowSchema::new(vec![ArrowField::new( + "b", + DataType::Struct(Fields::from(vec![new_child.as_ref().clone()])), + parent_nullable, + )])); + let sub_batch = RecordBatch::try_new( + sub_schema.clone(), + vec![Arc::new(StructArray::from(vec![( + new_child, + Arc::new(Int32Array::from(vec![10, 20])) as ArrayRef, + )]))], + ) + .unwrap(); + let mut latest = dataset.as_ref().clone(); + latest + .add_columns( + crate::dataset::NewColumnTransform::Reader(Box::new(RecordBatchIterator::new( + vec![Ok(sub_batch)], + sub_schema, + ))), + None, + None, + ) + .await + .unwrap(); + + let result = CommitBuilder::new(Arc::new(latest)).execute(append).await; + assert_eq!( + result.is_err(), + expect_conflict, + "child_required={child_required} parent_nullable={parent_nullable}: got {result:?}" + ); + if let Ok(committed) = result { + // The stale rows keep their parent values; the new child reads null. + let batch = committed.scan().try_into_batch().await.unwrap(); + assert_eq!(batch.num_rows(), 3); + let parent = batch["b"].as_struct(); + assert_eq!(parent.null_count(), 0); + assert_eq!(parent.column_by_name("d").unwrap().null_count(), 1); + } +} + +/// The barrier is operation-wide, so a tightening of a nested field conflicts +/// with a concurrent write exactly like a top-level one. +#[tokio::test] +async fn test_alter_columns_nested_tightening_conflicts() { + let schema = Arc::new(ArrowSchema::new(vec![ArrowField::new( + "b", + DataType::Struct(Fields::from(vec![ArrowField::new( + "c", + DataType::Int32, + true, + )])), + false, + )])); + let batch = RecordBatch::try_new( + schema.clone(), + vec![Arc::new(StructArray::from(vec![( + Arc::new(ArrowField::new("c", DataType::Int32, true)), + Arc::new(Int32Array::from(vec![1, 2])) as ArrayRef, + )]))], + ) + .unwrap(); + let dataset = Dataset::write( + RecordBatchIterator::new(vec![Ok(batch.clone())], schema.clone()), + "memory://", + None, + ) + .await + .unwrap(); + + // Leave the first handle a version behind, so the tightening commits stale. + InsertBuilder::new(WriteDestination::Dataset(Arc::new(dataset.clone()))) + .with_params(&WriteParams { + mode: WriteMode::Append, + ..Default::default() + }) + .execute(vec![batch]) + .await + .unwrap(); + + let mut stale = dataset; + stale + .alter_columns(&[ColumnAlteration::new("b.c".into()).set_nullable(false)]) + .await + .unwrap_err(); +} + +/// The invariant every piece of the tightening barrier serves: no interleaving +/// of honest writers and schema changes may commit a dataset that validates +/// but cannot be scanned. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn test_concurrent_tightening_stress() { + let dir = TempStrDir::default(); + let uri = dir.as_str().to_string(); + let batch = arrow_array::record_batch!(("value", Int32, [1, 2])).unwrap(); + let reader = RecordBatchIterator::new(vec![Ok(batch.clone())], batch.schema()); + Dataset::write(reader, uri.as_str(), None).await.unwrap(); + + let mut tasks = tokio::task::JoinSet::new(); + + // Appenders: honest read versions, half the batches carry nulls. A null + // append must either land while the column is nullable or be refused -- + // by the writer against a non-null schema, or by the claim barrier when a + // tightening won the race after the write. + for a in 0..4u8 { + let uri = uri.clone(); + tasks.spawn(async move { + let mut outcomes = [0u32; 2]; + for i in 0..12u32 { + let with_null = (a as u32 + i).is_multiple_of(2); + let batch = if with_null { + arrow_array::record_batch!(("value", Int32, [None, Some(3)])).unwrap() + } else { + arrow_array::record_batch!(("value", Int32, [4, 5])).unwrap() + }; + let reader = RecordBatchIterator::new(vec![Ok(batch.clone())], batch.schema()); + let result = Dataset::write( + reader, + uri.as_str(), + Some(WriteParams { + mode: WriteMode::Append, + ..Default::default() + }), + ) + .await; + outcomes[result.is_ok() as usize] += 1; + } + outcomes + }); + } + + // The tightener alternates NOT NULL and back. Either step may lose to + // concurrent writes; losing is an acceptable outcome, corruption is not. + { + let uri = uri.clone(); + tasks.spawn(async move { + let mut outcomes = [0u32; 2]; + for i in 0..10u32 { + let Ok(mut dataset) = DatasetBuilder::from_uri(uri.as_str()).load().await else { + continue; + }; + let result = dataset + .alter_columns( + &[ColumnAlteration::new("value".into()).set_nullable(i % 2 == 1)], + ) + .await; + outcomes[result.is_ok() as usize] += 1; + } + outcomes + }); + } + + let mut totals = [0u32; 2]; + while let Some(res) = tasks.join_next().await { + let [err, ok] = res.unwrap(); + totals[0] += err; + totals[1] += ok; + } + + // The oracle: whatever interleaving happened, the final dataset must be + // internally consistent -- validation and scanning agree. + let dataset = DatasetBuilder::from_uri(uri.as_str()).load().await.unwrap(); + dataset.validate().await.unwrap(); + let scanned = dataset.scan().try_into_batch().await.unwrap(); + assert!(scanned.num_rows() >= 2); + // And every historical version must scan too: a corrupt intermediate + // commit would have been the bug even if later commits papered over it. + for version in 1..=dataset.version().version { + let at = dataset.checkout_version(version).await.unwrap(); + at.validate().await.unwrap(); + at.scan().try_into_batch().await.unwrap(); + } + assert!(totals[1] > 0, "nothing succeeded: {totals:?}"); +} diff --git a/rust/lance/src/dataset/tests/dataset_overlay_index_masking.rs b/rust/lance/src/dataset/tests/dataset_overlay_index_masking.rs new file mode 100644 index 00000000000..a783ca09905 --- /dev/null +++ b/rust/lance/src/dataset/tests/dataset_overlay_index_masking.rs @@ -0,0 +1,2172 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright The Lance Authors + +//! End-to-end tests for data-overlay index masking: a scalar index masks data overlay files so that +//! queries stay correct while overlays remain (stale index hits are dropped and new +//! matches are added by re-evaluating overlay-covered rows on the flat path). + +use std::sync::Arc; + +use futures::TryStreamExt; + +use arrow_array::builder::{ListBuilder, StringBuilder}; +use arrow_array::cast::AsArray; +use arrow_array::types::Int32Type; +use arrow_array::{ArrayRef, Int32Array, RecordBatch, RecordBatchIterator, StringArray}; +use arrow_schema::{DataType, Field as ArrowField, Schema as ArrowSchema}; +use lance_index::IndexType; +use lance_index::optimize::OptimizeOptions; +use lance_index::scalar::BuiltinIndexType; +use lance_index::scalar::FullTextSearchQuery; +use lance_index::scalar::ScalarIndexParams; +use lance_index::scalar::inverted::query::{FtsQuery, MatchQuery, PhraseQuery}; +use lance_index::scalar::inverted::{DocumentGranularity, InvertedIndexParams}; +use lance_io::utils::CachedFileSize; +use lance_linalg::distance::MetricType; +use lance_table::format::DataFile; +use lance_table::format::overlay::{DataOverlayFile, OverlayCoverage}; +use roaring::RoaringBitmap; +use rstest::rstest; + +use lance_file::writer::FileWriterOptions; + +use crate::Dataset; +use crate::dataset::optimize::{CompactionOptions, compact_files, remapping}; +use crate::dataset::transaction::{DataOverlayGroup, Operation}; +use crate::dataset::{WriteDestination, WriteParams}; +use crate::index::vector::VectorIndexParams; +use crate::index::{CreateIndexBuilder, DatasetIndexExt}; +use crate::io::exec::filtered_read::FilteredReadExec; +use crate::io::exec::fts::FlatMatchQueryExec; + +/// Two-fragment Int32 dataset: `id` (field 0) = 0..12 and `age` (field 1) = id * 10, +/// six rows per file (fragments 0 and 1). In-memory store so overlay files can be written +/// with a store-relative `data/.lance` path and committed against the dataset. +async fn create_base_dataset() -> Dataset { + create_base_dataset_with(false).await +} + +async fn create_base_dataset_with(stable_row_ids: bool) -> Dataset { + let schema = Arc::new(ArrowSchema::new(vec![ + ArrowField::new("id", DataType::Int32, true), + ArrowField::new("age", DataType::Int32, true), + ])); + let batch = RecordBatch::try_new( + schema.clone(), + vec![ + Arc::new(Int32Array::from_iter_values(0..12)), + Arc::new(Int32Array::from_iter_values((0..12).map(|v| v * 10))), + ], + ) + .unwrap(); + let write_params = WriteParams { + max_rows_per_file: 6, + enable_stable_row_ids: stable_row_ids, + ..Default::default() + }; + let reader = RecordBatchIterator::new(vec![Ok(batch)], schema.clone()); + Dataset::write(reader, "memory://", Some(write_params)) + .await + .unwrap() +} + +async fn build_age_index(dataset: &mut Dataset) { + dataset + .create_index( + &["age"], + IndexType::BTree, + None, + &ScalarIndexParams::default(), + true, + ) + .await + .unwrap(); +} + +/// Write an overlay file covering `fields` of `fragment_id` with `coverage` and the given +/// per-field value columns, then commit it as a `DataOverlay` transaction. `name` makes +/// the overlay file unique. +async fn commit_overlay( + dataset: Dataset, + name: &str, + fragment_id: u64, + fields: &[i32], + coverage: OverlayCoverage, + columns: Vec, +) -> Dataset { + let read_version = dataset.version().version; + let overlay_schema = dataset.schema().project_by_ids(fields, true); + + let filename = format!("{name}.lance"); + // Use dataset.base so the path is absolute for file:// stores. + // to_local_path() prepends '/' to the object_store path, so a bare + // "data/foo.lance" would resolve to /data/foo.lance (root fs). With + // base we get e.g. tmp/lance-bench/data/foo.lance → /tmp/lance-bench/data/foo.lance. + // For memory:// stores base is empty so the result is the same as before. + let path = dataset.base.clone().join("data").join(filename.as_str()); + let obj_writer = dataset.object_store.create(&path).await.unwrap(); + let file_version = dataset.manifest.data_storage_format.lance_file_format(); + let mut writer = lance_file::versions::create_writer( + file_version, + obj_writer, + overlay_schema, + FileWriterOptions::default(), + ) + .unwrap(); + + for (i, array) in columns.into_iter().enumerate() { + writer.write_column(i, array).await.unwrap(); + } + let summary = writer.finish().await.unwrap(); + + let mut data_file = DataFile::new_unstarted(filename, file_version); + data_file.fields = writer + .field_id_to_column_indices() + .iter() + .map(|(field_id, _)| *field_id as i32) + .collect::>() + .into(); + data_file.column_indices = writer + .field_id_to_column_indices() + .iter() + .map(|(_, column_index)| *column_index as i32) + .collect::>() + .into(); + data_file.file_size_bytes = CachedFileSize::new(summary.size_bytes); + + let overlay = DataOverlayFile { + data_file, + coverage, + committed_version: 0, + }; + Dataset::commit( + WriteDestination::Dataset(Arc::new(dataset)), + Operation::DataOverlay { + groups: vec![DataOverlayGroup { + fragment_id, + overlays: vec![overlay], + }], + }, + Some(read_version), + None, + None, + Arc::new(Default::default()), + false, + ) + .await + .unwrap() +} + +/// Sorted `id` values returned by a filtered scan. +async fn ids_matching(dataset: &Dataset, filter: &str) -> Vec { + ids_matching_opts(dataset, filter, false).await +} + +/// Like [`ids_matching`] but lets a test enable `fast_search()`, which skips unindexed +/// fragments. Overlay masking on indexed fragments must still apply regardless. +async fn ids_matching_opts(dataset: &Dataset, filter: &str, fast_search: bool) -> Vec { + let mut scanner = dataset.scan(); + scanner.filter(filter).unwrap().project(&["id"]).unwrap(); + if fast_search { + scanner.fast_search(); + } + let batch = scanner.try_into_batch().await.unwrap(); + let mut ids = ids_from_batches(std::slice::from_ref(&batch)); + ids.sort_unstable(); + ids +} + +/// Concatenate the `id` (Int32) column from each batch, in batch order. +fn ids_from_batches(batches: &[RecordBatch]) -> Vec { + batches + .iter() + .flat_map(|b| { + b.column_by_name("id") + .unwrap() + .as_primitive::() + .values() + .to_vec() + }) + .collect() +} + +fn i32_array(values: impl IntoIterator>) -> ArrayRef { + Arc::new(Int32Array::from_iter(values)) +} + +fn string_lists(rows: &[&[&str]]) -> ArrayRef { + let mut builder = ListBuilder::new(StringBuilder::new()); + for row in rows { + for value in *row { + builder.values().append_value(value); + } + builder.append(true); + } + Arc::new(builder.finish()) +} + +fn fsl(rows: Vec>, dim: i32) -> ArrayRef { + let flat: Vec = rows.into_iter().flatten().collect(); + let item = Arc::new(ArrowField::new("item", DataType::Float32, true)); + Arc::new( + arrow_array::FixedSizeListArray::try_new( + item, + dim, + Arc::new(arrow_array::Float32Array::from(flat)), + None, + ) + .unwrap(), + ) +} + +/// A newer overlay on the indexed field drops stale index hits (the old value no longer +/// matches) and surfaces new matches (the new value is found even though the index never +/// saw it). Mirrors the spec's Bob 25 -> 26 worked example. +/// +/// Parametrized over `stable_row_ids` to cover the address-based stale-Take path under both +/// row-id schemes. +#[rstest] +#[tokio::test] +async fn test_overlay_stale_drop_and_new_match(#[values(false, true)] stable_row_ids: bool) { + let mut dataset = create_base_dataset_with(stable_row_ids).await; + build_age_index(&mut dataset).await; + + // Fragment 0, offset 1 is id=1, age=10. The overlay (committed after the index) + // changes its age to 999. + let dataset = commit_overlay( + dataset, + "age_overlay", + 0, + &[1], + OverlayCoverage::dense(RoaringBitmap::from_iter([1])), + vec![i32_array([Some(999)])], + ) + .await; + + // Stale-drop: the index still holds age=10 for id=1, but its current value is 999, + // so it must not be returned. + assert_eq!(ids_matching(&dataset, "age = 10").await, Vec::::new()); + // New-match: the index never saw age=999, but re-evaluation finds it. + assert_eq!(ids_matching(&dataset, "age = 999").await, vec![1]); + // An untouched indexed value is unaffected. + assert_eq!(ids_matching(&dataset, "age = 20").await, vec![2]); +} + +/// Row-level BTree precision: when one row in a covered fragment is stale, only that row is +/// blocked from the index result and re-evaluated on the stale-Take path. Non-stale rows in +/// the same fragment (including one that matches the predicate) remain on the indexed path. +/// +/// Setup: fragment 0 has id=5 → age=50 (not stale). Overlay id=1 → age=50 (stale). +/// After the overlay two rows in fragment 0 have age=50. The row-level optimization must +/// return both: id=5 from the index and id=1 from the stale-Take path. +/// +/// Parametrized over `stable_row_ids`: with stable row ids enabled the stale-Take path must +/// identify rows by physical address, not `_rowid`, or it would take the wrong rows. +#[rstest] +#[tokio::test] +async fn test_btree_overlay_row_level_precision(#[values(false, true)] stable_row_ids: bool) { + let mut dataset = create_base_dataset_with(stable_row_ids).await; + build_age_index(&mut dataset).await; + + // Fragment 0: ids 0-5, ages 0,10,20,30,40,50. Overlay offset 1 (id=1): age 10→50. + // After this both id=1 and id=5 have age=50, in the same fragment. + let dataset = commit_overlay( + dataset, + "age_row_level", + 0, + &[1], + OverlayCoverage::dense(RoaringBitmap::from_iter([1])), + vec![i32_array([Some(50)])], + ) + .await; + + // Stale drop: id=1's old age=10 entry must not appear. + assert_eq!(ids_matching(&dataset, "age = 10").await, Vec::::new()); + + // id=5 via index + id=1 via stale-Take path — both in fragment 0. + assert_eq!(ids_matching(&dataset, "age = 50").await, vec![1, 5]); + + // Non-stale rows in the same fragment still return correctly. + assert_eq!(ids_matching(&dataset, "age = 20").await, vec![2]); + assert_eq!(ids_matching(&dataset, "age = 30").await, vec![3]); +} + +/// `fast_search` skips *unindexed fragments*, but overlay masking on indexed fragments must +/// still apply: the drop-stale block and the stale-Take re-eval both run regardless of +/// `fast_search` on the scalar path. A regression that gated overlay masking behind +/// `!fast_search` would leak id=1's stale age=10 hit here. +#[tokio::test] +async fn test_btree_overlay_masked_under_fast_search() { + let mut dataset = create_base_dataset().await; + build_age_index(&mut dataset).await; + + // Fragment 0, offset 1 is id=1, age=10. Overlay (committed after the index) → age=999. + let dataset = commit_overlay( + dataset, + "age_fast_search", + 0, + &[1], + OverlayCoverage::dense(RoaringBitmap::from_iter([1])), + vec![i32_array([Some(999)])], + ) + .await; + + // Stale hit dropped even under fast_search — the block is not gated by fast_search. + assert_eq!( + ids_matching_opts(&dataset, "age = 10", true).await, + Vec::::new() + ); + // The scalar re-eval path is likewise not gated, so the new value is still surfaced. + assert_eq!( + ids_matching_opts(&dataset, "age = 999", true).await, + vec![1] + ); + // An untouched indexed value on the same fragment is unaffected. + assert_eq!(ids_matching_opts(&dataset, "age = 20", true).await, vec![2]); +} + +/// An overlay touching only a non-indexed field excludes nothing from the index on `age`. +#[tokio::test] +async fn test_overlay_on_unrelated_field_excludes_nothing() { + let mut dataset = create_base_dataset().await; + build_age_index(&mut dataset).await; + + // Overlay field 0 (`id`), not the indexed `age`. The age index stays fully trusted. + let dataset = commit_overlay( + dataset, + "id_overlay", + 0, + &[0], + OverlayCoverage::dense(RoaringBitmap::from_iter([1])), + vec![i32_array([Some(777)])], + ) + .await; + + // The age index is still trusted: age=10 finds the offset-1 row, whose id now reads + // through the overlay as 777. The fragment was not routed to the flat path on account + // of an overlay that touches no indexed field. + assert_eq!(ids_matching(&dataset, "age = 10").await, vec![777]); + // An untouched row is unaffected. + assert_eq!(ids_matching(&dataset, "age = 20").await, vec![2]); + // The overlaid id is the new value on read, and the old one is gone. + assert_eq!(ids_matching(&dataset, "id = 777").await, vec![777]); + assert_eq!(ids_matching(&dataset, "id = 1").await, Vec::::new()); +} + +/// An overlay whose `committed_version <= index.dataset_version` is already incorporated by +/// the index (the index was built reading merged values) and is not excluded. +#[tokio::test] +async fn test_overlay_older_than_index_not_excluded() { + let dataset = create_base_dataset().await; + + // Commit the overlay first (age of id=1 becomes 999), then build the index on top. + let mut dataset = commit_overlay( + dataset, + "age_overlay_old", + 0, + &[1], + OverlayCoverage::dense(RoaringBitmap::from_iter([1])), + vec![i32_array([Some(999)])], + ) + .await; + build_age_index(&mut dataset).await; + + // The index incorporates the overlay, so it returns the merged value directly. + assert_eq!(ids_matching(&dataset, "age = 999").await, vec![1]); + assert_eq!(ids_matching(&dataset, "age = 10").await, Vec::::new()); +} + +/// A covered offset whose overlay value is NULL overrides the cell to NULL, so the stale +/// index hit for its old value is dropped. +#[tokio::test] +async fn test_overlay_null_override() { + let mut dataset = create_base_dataset().await; + build_age_index(&mut dataset).await; + + // id=1 (age=10) is overridden to NULL. + let dataset = commit_overlay( + dataset, + "age_overlay_null", + 0, + &[1], + OverlayCoverage::dense(RoaringBitmap::from_iter([1])), + vec![i32_array([None])], + ) + .await; + + assert_eq!(ids_matching(&dataset, "age = 10").await, Vec::::new()); + assert_eq!(ids_matching(&dataset, "age IS NULL").await, vec![1]); +} + +/// Overlays on a non-first fragment are masked correctly, and a query spanning both +/// fragments returns the right rows. +/// +/// Parametrized over `stable_row_ids`, and crucially overlays fragment 1 (ids 6..12), where a +/// physical address diverges from the stable row id — so this exercises the address-vs-row-id +/// distinction that a fragment-0 overlay cannot. +#[rstest] +#[tokio::test] +async fn test_overlay_multi_fragment(#[values(false, true)] stable_row_ids: bool) { + let mut dataset = create_base_dataset_with(stable_row_ids).await; + build_age_index(&mut dataset).await; + + // Fragment 1 holds ids 6..12 (ages 60..110). Offset 2 within fragment 1 is id=8, + // age=80; change it to 60 (a value that also legitimately exists at id=6). + let dataset = commit_overlay( + dataset, + "age_overlay_frag1", + 1, + &[1], + OverlayCoverage::dense(RoaringBitmap::from_iter([2])), + vec![i32_array([Some(60)])], + ) + .await; + + // id=8 no longer has age=80 (stale-drop on fragment 1). + assert_eq!(ids_matching(&dataset, "age = 80").await, Vec::::new()); + // Both id=6 (base) and id=8 (overlay) now have age=60 (new-match added to base hit). + assert_eq!(ids_matching(&dataset, "age = 60").await, vec![6, 8]); + // A value in the untouched fragment 0 is still served correctly. + assert_eq!(ids_matching(&dataset, "age = 30").await, vec![3]); +} + +/// A deletion below an overlaid row must not corrupt the physical-offset → stable-row-id +/// translation used to build the overlay block mask. +/// +/// Under stable row ids the stale-row block/take set is computed by mapping each stale +/// *physical offset* to its stable row id via the fragment's `RowIdSequence`. The sequence +/// keeps one entry per physical row (deleted rows are tracked separately by the deletion +/// vector, not compacted out), so the correct mapping is `sequence.get(offset)`. A regression +/// that instead advanced a `sequence.iter()` cursor only for non-deleted offsets desynced the +/// cursor after any deletion at an offset *below* the stale one, blocking/taking the wrong row +/// id: the stale index hit then leaked and the new value was never surfaced. +/// +/// Setup (stable row ids): fragment 1 holds ids 6..12 at offsets 0..6. Delete id=6 (offset 0), +/// then overlay offset 2 (id=8, age 80 → 999). The deletion at offset 0 sits below the stale +/// offset 2, so a cursor-based translation would map offset 2 to id=7 instead of id=8. +/// +/// Parametrized over `stable_row_ids`: only the stable-row-id path translates offsets to row +/// ids, so the bug is specific to it; the non-stable case (addresses are row ids) is a control. +#[rstest] +#[tokio::test] +async fn test_btree_overlay_stale_row_with_prior_deletion( + #[values(false, true)] stable_row_ids: bool, +) { + let mut dataset = create_base_dataset_with(stable_row_ids).await; + build_age_index(&mut dataset).await; + + // Delete id=6 (fragment 1, offset 0) — a deletion hole below the row the overlay marks stale. + dataset.delete("id = 6").await.unwrap(); + + // Fragment 1, offset 2 is id=8 (age 80). The overlay (committed after the index) → age 999. + let dataset = commit_overlay( + dataset, + "age_overlay_del", + 1, + &[1], + OverlayCoverage::dense(RoaringBitmap::from_iter([2])), + vec![i32_array([Some(999)])], + ) + .await; + + // Stale-drop: id=8's old age=80 index entry must not be returned. + assert_eq!(ids_matching(&dataset, "age = 80").await, Vec::::new()); + // New-match: id=8's current age=999 is found by re-evaluating the stale row. + assert_eq!(ids_matching(&dataset, "age = 999").await, vec![8]); + // A non-stale row in the same deletion-bearing fragment is still served by the index. + assert_eq!(ids_matching(&dataset, "age = 70").await, vec![7]); + // The deleted row is gone. + assert_eq!(ids_matching(&dataset, "age = 60").await, Vec::::new()); +} + +const VEC_DIM: i32 = 8; + +fn vec_query() -> Vec { + vec![1.0_f32, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0] +} + +/// 64-row two-fragment vector dataset with a single-partition IVF_FLAT index, then an overlay +/// on fragment 1 that moves id=35 (offset 3) onto `far` (away from the query) and id=40 +/// (offset 8) onto the query. Built before the overlay, the index still believes id=35 is the +/// query and has never seen id=40 near it. Every other base vector is orthogonal to the query. +/// +/// Overlaying fragment 1 (ids 32..64) is deliberate: a physical address diverges from the +/// stable row id there, so both the ANN prefilter block and the flat re-score take must operate +/// in the row-id domain when `stable_row_ids` is enabled. +async fn create_vector_overlay_dataset(stable_row_ids: bool) -> Dataset { + let query = vec_query(); + let far = vec![0.0_f32, 100.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]; + + let mut vectors: Vec> = Vec::with_capacity(64); + for i in 0..64 { + if i == 35 { + vectors.push(query.clone()); + } else { + let mut v = vec![0.0_f32; VEC_DIM as usize]; + v[1] = (i + 2) as f32; // orthogonal to the query, distinct, far + vectors.push(v); + } + } + + let schema = Arc::new(ArrowSchema::new(vec![ + ArrowField::new("id", DataType::Int32, true), + ArrowField::new( + "vec", + DataType::FixedSizeList( + Arc::new(ArrowField::new("item", DataType::Float32, true)), + VEC_DIM, + ), + true, + ), + ])); + let batch = RecordBatch::try_new( + schema.clone(), + vec![ + Arc::new(Int32Array::from_iter_values(0..64)), + fsl(vectors, VEC_DIM), + ], + ) + .unwrap(); + let write_params = WriteParams { + max_rows_per_file: 32, + enable_stable_row_ids: stable_row_ids, + ..Default::default() + }; + let reader = RecordBatchIterator::new(vec![Ok(batch)], schema.clone()); + let mut dataset = Dataset::write(reader, "memory://", Some(write_params)) + .await + .unwrap(); + + // Single-partition IVF_FLAT: the ANN searches every indexed row with exact distances. + let params = VectorIndexParams::ivf_flat(1, MetricType::L2); + dataset + .create_index(&["vec"], IndexType::Vector, None, ¶ms, true) + .await + .unwrap(); + + commit_overlay( + dataset, + "vec_overlay", + 1, + &[1], + OverlayCoverage::dense(RoaringBitmap::from_iter([3, 8])), + vec![fsl(vec![far, query], VEC_DIM)], + ) + .await +} + +/// Run a top-`k` ANN search for the standard query vector and return the returned `id`s, +/// optionally with `fast_search()` enabled. +async fn vector_query_ids(dataset: &Dataset, k: usize, fast_search: bool) -> Vec { + let mut scanner = dataset.scan(); + scanner + .nearest("vec", &arrow_array::Float32Array::from(vec_query()), k) + .unwrap() + .minimum_nprobes(1) + .project(&["id"]) + .unwrap(); + if fast_search { + scanner.fast_search(); + } + let results = scanner + .try_into_stream() + .await + .unwrap() + .try_collect::>() + .await + .unwrap(); + ids_from_batches(&results) +} + +/// A vector index masks overlays: a row whose vector was moved (by a newer overlay) away +/// from the query is dropped from results, and a row moved *onto* the query is found by +/// re-scoring its current vector on the flat path — even though the index never saw it. +/// +/// Parametrized over `stable_row_ids` to cover the row-id domain for both block and re-score. +#[rstest] +#[tokio::test] +async fn test_vector_index_rescore_on_overlay(#[values(false, true)] stable_row_ids: bool) { + let dataset = create_vector_overlay_dataset(stable_row_ids).await; + let ids = vector_query_ids(&dataset, 3, false).await; + + // id=40 was moved onto the query and is found by re-scoring (new-match recall). + assert!( + ids.contains(&40), + "expected id=40 (re-scored to query) in {ids:?}" + ); + // id=35's stale index entry (the query) must not resurface: its current vector is far. + assert!( + !ids.contains(&35), + "stale vector for id=35 should be dropped, got {ids:?}" + ); +} + +/// The ANN prefilter block that drops stale overlay rows runs regardless of `fast_search`; +/// only the flat re-score is gated by it. So under `fast_search` id=35's stale hit must still +/// be dropped, while id=40 (moved onto the query) is intentionally not re-scored — the same +/// recall tradeoff `fast_search` already makes for unindexed data. A regression that moved the +/// `overlay_block` computation inside the `!fast_search` guard would leak id=35's stale vector. +#[tokio::test] +async fn test_vector_overlay_stale_dropped_under_fast_search() { + let dataset = create_vector_overlay_dataset(false).await; + let ids = vector_query_ids(&dataset, 3, true).await; + + // Correctness: the stale index hit is dropped even though the re-score is skipped. + assert!( + !ids.contains(&35), + "stale vector for id=35 must be dropped under fast_search, got {ids:?}" + ); + // Recall tradeoff: fast_search skips the flat re-score, so the moved-on match is not surfaced. + assert!( + !ids.contains(&40), + "fast_search skips re-score, so id=40 should be absent, got {ids:?}" + ); +} + +/// A compound boolean predicate (age AND id) exercises the ScalarIndexExpr tree-walk in +/// `overlay_stale_index_rows`. An overlay on `age` marks fragment 0 stale from the `age` +/// index's perspective, so the compound query must re-evaluate fragment 0 on the flat path. +#[tokio::test] +async fn test_overlay_stale_with_compound_index_expression() { + let mut dataset = create_base_dataset().await; + // Build BTree indexes on both columns so a compound filter can use both. + build_age_index(&mut dataset).await; + dataset + .create_index( + &["id"], + IndexType::BTree, + None, + &ScalarIndexParams::default(), + true, + ) + .await + .unwrap(); + + // Fragment 0 covers id=0..5, age=0..50. Overlay changes id=1's age from 10 to 999. + let dataset = commit_overlay( + dataset, + "age_compound", + 0, + &[1], + OverlayCoverage::dense(RoaringBitmap::from_iter([1])), + vec![i32_array([Some(999)])], + ) + .await; + + // Compound query: both the `age` and `id` index are involved. The overlay on `age` + // makes fragment 0 stale for the `age` index; it falls to the flat path, which uses + // the merged (overlay) value. Result: the stale age=10 hit is gone, age=999 appears. + assert_eq!(ids_matching(&dataset, "age = 10").await, Vec::::new()); + assert_eq!(ids_matching(&dataset, "age = 999").await, vec![1]); + // A pure `id` query on an unaffected fragment still works correctly. + assert_eq!(ids_matching(&dataset, "id = 2").await, vec![2]); +} + +/// A `RewriteRows` update (under stable row ids) that touches only a *non-indexed* column moves +/// the matched rows to a new fragment and, because the scalar index's field was not modified, +/// extends that index's fragment coverage onto the new fragment +/// (`register_pure_rewrite_rows_update_frags_in_indices`) so its existing entries are reused. +/// +/// That reuse is unsound when a moved row carried a data overlay on the *indexed* field: the +/// update materializes the overlay's current value into the new fragment, but the reused index +/// entry still holds the stale pre-overlay value, and the new fragment (now marked covered) no +/// longer falls to the flat path that previously served the correct value via overlay masking. +/// +/// Here `age` is indexed and overlaid (id=1: age 10 -> 999); the update sets the non-indexed +/// `id` column on that row. After it, `age = 10` must stay dropped and `age = 999` must still +/// find the row — otherwise the stale index entry has resurfaced. +#[tokio::test] +async fn test_update_nonindexed_column_preserves_overlay_masking() { + use crate::dataset::UpdateBuilder; + + let mut dataset = create_base_dataset_with(true).await; + build_age_index(&mut dataset).await; + + // Overlay fragment 0, offset 1 (id=1): age 10 -> 999, committed after the index. + let dataset = commit_overlay( + dataset, + "age_update", + 0, + &[1], + OverlayCoverage::dense(RoaringBitmap::from_iter([1])), + vec![i32_array([Some(999)])], + ) + .await; + + // Masking works before the update. + assert_eq!(ids_matching(&dataset, "age = 10").await, Vec::::new()); + assert_eq!(ids_matching(&dataset, "age = 999").await, vec![1]); + + // Update only the non-indexed `id` column of the overlaid row. This is a rewrite-rows move: + // the row (with age materialized to 999) is written to a new fragment and deleted from + // fragment 0, keeping its stable row id. + let dataset = UpdateBuilder::new(Arc::new(dataset)) + .update_where("id = 1") + .unwrap() + .set("id", "100") + .unwrap() + .build() + .unwrap() + .execute() + .await + .unwrap() + .new_dataset; + + // Still masked: the stale age=10 entry must stay dropped and the overlaid age=999 value must + // still be found (now on the moved row, whose id is 100). + assert_eq!( + ids_matching(&dataset, "age = 10").await, + Vec::::new(), + "stale index entry age=10 resurfaced after updating a non-indexed column" + ); + assert_eq!( + ids_matching(&dataset, "age = 999").await, + vec![100], + "overlaid value age=999 lost after updating a non-indexed column" + ); + // A row untouched by the overlay is unaffected. + assert_eq!(ids_matching(&dataset, "age = 20").await, vec![2]); +} + +/// Text dataset: two fragments, 6 rows each. Schema: id (Int32), text (Utf8). +/// Texts are unique tokens so each row can be identified by its term. +async fn create_text_dataset(stable_row_ids: bool) -> Dataset { + let schema = Arc::new(ArrowSchema::new(vec![ + ArrowField::new("id", DataType::Int32, true), + ArrowField::new("text", DataType::Utf8, true), + ])); + let texts: Vec<&str> = vec![ + "apple pie", + "apple banana", // row 1, fragment 0 — will be overlaid in tests + "cherry cake", + "banana split", + "orange juice", + "grape vine", + "mango sorbet", // fragment 1 starts here + "pear tart", + "lemon curd", + "peach cobbler", + "plum pudding", + "fig newton", + ]; + let batch = RecordBatch::try_new( + schema.clone(), + vec![ + Arc::new(Int32Array::from_iter_values(0..12)), + Arc::new(StringArray::from(texts)), + ], + ) + .unwrap(); + let write_params = WriteParams { + max_rows_per_file: 6, + enable_stable_row_ids: stable_row_ids, + ..Default::default() + }; + let reader = RecordBatchIterator::new(vec![Ok(batch)], schema.clone()); + Dataset::write(reader, "memory://", Some(write_params)) + .await + .unwrap() +} + +async fn build_text_fts_index(dataset: &mut Dataset) { + dataset + .create_index( + &["text"], + IndexType::Inverted, + None, + &InvertedIndexParams::default(), + true, + ) + .await + .unwrap(); +} + +/// FTS index with token positions stored, required for phrase queries. +async fn build_text_fts_index_with_positions(dataset: &mut Dataset) { + dataset + .create_index( + &["text"], + IndexType::Inverted, + None, + &InvertedIndexParams::default().with_position(true), + true, + ) + .await + .unwrap(); +} + +/// Collect sorted IDs of rows returned by an FTS query on `text`. +async fn fts_ids(dataset: &Dataset, query: FullTextSearchQuery) -> Vec { + let results = dataset + .scan() + .full_text_search(query) + .unwrap() + .project(&["id"]) + .unwrap() + .try_into_stream() + .await + .unwrap() + .try_collect::>() + .await + .unwrap(); + let mut ids = ids_from_batches(&results); + ids.sort_unstable(); + ids +} + +async fn fts_ids_matching(dataset: &Dataset, term: &str) -> Vec { + fts_ids(dataset, FullTextSearchQuery::new(term.to_owned())).await +} + +#[tokio::test] +async fn test_ngram_optimize_preserves_overlay_staleness() { + let mut dataset = create_text_dataset(false).await; + let params = ScalarIndexParams::for_builtin(BuiltinIndexType::NGram); + let fragment_ids = dataset + .get_fragments() + .into_iter() + .map(|fragment| fragment.id() as u32) + .collect::>(); + let mut segments = Vec::with_capacity(fragment_ids.len()); + for fragment_id in fragment_ids { + segments.push( + CreateIndexBuilder::new(&mut dataset, &["text"], IndexType::NGram, ¶ms) + .name("text_ngram".to_string()) + .fragments(vec![fragment_id]) + .execute_uncommitted() + .await + .unwrap(), + ); + } + let source_version = segments[0].dataset_version; + dataset + .commit_existing_index_segments("text_ngram", "text", segments) + .await + .unwrap(); + + let mut dataset = commit_overlay( + dataset, + "ngram_text_overlay", + 0, + &[1], + OverlayCoverage::dense(RoaringBitmap::from_iter([1])), + vec![Arc::new(StringArray::from(vec![Some("cherry mango")]))], + ) + .await; + dataset + .optimize_indices(&OptimizeOptions::merge(2)) + .await + .unwrap(); + + let committed = dataset.load_indices_by_name("text_ngram").await.unwrap(); + assert_eq!(committed.len(), 1); + assert_eq!(committed[0].dataset_version, source_version); + assert_eq!( + ids_matching(&dataset, "contains(text, 'apple')").await, + vec![0] + ); + assert_eq!( + ids_matching(&dataset, "contains(text, 'mango')").await, + vec![1, 6] + ); +} + +#[tokio::test] +async fn test_btree_physical_merge_preserves_overlay_staleness() { + let mut dataset = create_base_dataset().await; + let params = ScalarIndexParams::default(); + let mut segments = Vec::new(); + for fragment in dataset.get_fragments() { + segments.push( + CreateIndexBuilder::new(&mut dataset, &["age"], IndexType::BTree, ¶ms) + .name("age_btree".to_string()) + .fragments(vec![fragment.id() as u32]) + .execute_uncommitted() + .await + .unwrap(), + ); + } + let source_version = segments[0].dataset_version; + let mut dataset = commit_overlay( + dataset, + "btree_before_merge", + 0, + &[1], + OverlayCoverage::dense(RoaringBitmap::from_iter([1])), + vec![i32_array([Some(999)])], + ) + .await; + + let merged = dataset + .merge_existing_index_segments(segments) + .await + .unwrap(); + assert_eq!(merged.dataset_version, source_version); + dataset + .commit_existing_index_segments("age_btree", "age", vec![merged]) + .await + .unwrap(); + + assert_eq!(ids_matching(&dataset, "age = 10").await, Vec::::new()); + assert_eq!(ids_matching(&dataset, "age = 999").await, vec![1]); +} + +#[tokio::test] +async fn test_ngram_remap_excludes_newer_overlay_fragments() { + let mut dataset = create_text_dataset(false).await; + let params = ScalarIndexParams::for_builtin(BuiltinIndexType::NGram); + dataset + .create_index( + &["text"], + IndexType::NGram, + Some("text_ngram".to_string()), + ¶ms, + false, + ) + .await + .unwrap(); + let source_version = + dataset.load_indices_by_name("text_ngram").await.unwrap()[0].dataset_version; + + compact_files( + &mut dataset, + CompactionOptions { + target_rows_per_fragment: 12, + defer_index_remap: true, + ..Default::default() + }, + None, + ) + .await + .unwrap(); + let compacted_fragment_id = dataset.get_fragments()[0].id(); + let mut dataset = commit_overlay( + dataset, + "ngram_after_compaction", + compacted_fragment_id as u64, + &[1], + OverlayCoverage::dense(RoaringBitmap::from_iter([1])), + vec![Arc::new(StringArray::from(vec![Some("cherry mango")]))], + ) + .await; + + remapping::remap_column_index(&mut dataset, &["text"], Some("text_ngram".to_string())) + .await + .unwrap(); + + let committed = dataset.load_indices_by_name("text_ngram").await.unwrap(); + assert_eq!(committed.len(), 1); + assert!(committed[0].dataset_version > source_version); + assert!( + !committed[0] + .fragment_bitmap + .as_ref() + .unwrap() + .contains(compacted_fragment_id as u32) + ); + assert_eq!( + ids_matching(&dataset, "contains(text, 'apple')").await, + vec![0] + ); + assert_eq!( + ids_matching(&dataset, "contains(text, 'mango')").await, + vec![1, 6] + ); +} + +async fn fts_phrase_ids_matching(dataset: &Dataset, phrase: &str) -> Vec { + use lance_index::scalar::inverted::query::{FtsQuery, PhraseQuery}; + + let query = FullTextSearchQuery::new_query(FtsQuery::Phrase( + PhraseQuery::new(phrase.to_owned()).with_column(Some("text".to_owned())), + )); + fts_ids(dataset, query).await +} + +/// An overlay committed after the FTS index is built replaces a row's text. Searching for +/// the old term must not return the stale row; searching for the new term must find it. +#[rstest] +#[tokio::test] +async fn test_fts_overlay_stale_drop_and_new_match(#[values(false, true)] stable_row_ids: bool) { + let mut dataset = create_text_dataset(stable_row_ids).await; + build_text_fts_index(&mut dataset).await; + + // fragment 0, row offset 1 (id=1): "apple banana" → "cherry mango" + // field ID 1 is the `text` column. + let dataset = commit_overlay( + dataset, + "text_overlay", + 0, + &[1], + OverlayCoverage::dense(RoaringBitmap::from_iter([1])), + vec![Arc::new(StringArray::from(vec![Some("cherry mango")]))], + ) + .await; + + // "apple" now matches only id=0 ("apple pie"); id=1's stale index entry must be dropped. + assert_eq!(fts_ids_matching(&dataset, "apple").await, vec![0]); + + // "banana" matched id=1 and id=3 before; after overlay id=1's stale entry must be gone. + assert_eq!(fts_ids_matching(&dataset, "banana").await, vec![3]); + + // "cherry" now matches id=1 (via flat path on stale fragment) and id=2 ("cherry cake"). + let cherry_ids = fts_ids_matching(&dataset, "cherry").await; + assert!( + cherry_ids.contains(&1), + "id=1 overlay→cherry mango should be found: {cherry_ids:?}" + ); + assert!( + cherry_ids.contains(&2), + "id=2 cherry cake should still be found: {cherry_ids:?}" + ); + + // "mango" now matches id=1 (overlay) and id=6 ("mango sorbet" in fragment 1). + let mango_ids = fts_ids_matching(&dataset, "mango").await; + assert!( + mango_ids.contains(&1), + "id=1 overlay→cherry mango should be found: {mango_ids:?}" + ); + assert!( + mango_ids.contains(&6), + "id=6 mango sorbet should still be found: {mango_ids:?}" + ); +} + +/// A phrase query must drop stale indexed positions and re-evaluate the current +/// overlay value on the flat phrase path. +#[rstest] +#[tokio::test] +async fn test_fts_phrase_overlay_stale_drop(#[values(false, true)] stable_row_ids: bool) { + let mut dataset = create_text_dataset(stable_row_ids).await; + build_text_fts_index_with_positions(&mut dataset).await; + + // Before any overlay the phrase "apple banana" matches only id=1. + assert_eq!( + fts_phrase_ids_matching(&dataset, "apple banana").await, + vec![1] + ); + + // Overlay id=1's text (field 1) so the phrase no longer applies to its current value. + let dataset = commit_overlay( + dataset, + "phrase_overlay", + 0, + &[1], + OverlayCoverage::dense(RoaringBitmap::from_iter([1])), + vec![Arc::new(StringArray::from(vec![Some("cherry mango")]))], + ) + .await; + + // The stale inverted-index positions for "apple banana" on id=1 must not be returned. + assert_eq!( + fts_phrase_ids_matching(&dataset, "apple banana").await, + Vec::::new() + ); + assert_eq!( + fts_phrase_ids_matching(&dataset, "cherry mango").await, + vec![1] + ); +} + +#[tokio::test] +async fn test_fts_empty_fragment_selection_is_empty() { + let mut dataset = create_text_dataset(false).await; + build_text_fts_index_with_positions(&mut dataset).await; + + let mut match_scan = dataset.scan(); + match_scan.with_fragments(Vec::new()); + match_scan + .full_text_search(FullTextSearchQuery::new("apple".to_owned())) + .unwrap(); + match_scan.project(&["id"]).unwrap(); + let match_plan = match_scan.explain_plan(false).await.unwrap(); + assert!( + match_plan.contains("EmptyExec"), + "explicit empty fragment selection should produce EmptyExec: {match_plan}" + ); + assert_eq!(match_scan.try_into_batch().await.unwrap().num_rows(), 0); + + let mut phrase_scan = dataset.scan(); + phrase_scan.with_fragments(Vec::new()); + phrase_scan + .full_text_search(FullTextSearchQuery::new_query(FtsQuery::Phrase( + PhraseQuery::new("apple pie".to_owned()).with_column(Some("text".to_owned())), + ))) + .unwrap(); + phrase_scan.project(&["id"]).unwrap(); + let phrase_plan = phrase_scan.explain_plan(false).await.unwrap(); + assert!( + phrase_plan.contains("EmptyExec"), + "explicit empty fragment selection should produce EmptyExec: {phrase_plan}" + ); + assert_eq!(phrase_scan.try_into_batch().await.unwrap().num_rows(), 0); +} + +#[rstest] +#[tokio::test] +async fn test_fts_combines_indexed_overlay_stale_and_unindexed_rows( + #[values(false, true)] stable_row_ids: bool, +) { + let mut dataset = create_text_dataset(stable_row_ids).await; + build_text_fts_index_with_positions(&mut dataset).await; + + let batch = + arrow_array::record_batch!(("id", Int32, [12]), ("text", Utf8, ["cherry mango"])).unwrap(); + let schema = batch.schema(); + let reader = RecordBatchIterator::new(vec![Ok(batch)], schema); + let dataset = Dataset::write( + reader, + Arc::new(dataset), + Some(WriteParams { + mode: crate::dataset::write::WriteMode::Append, + ..Default::default() + }), + ) + .await + .unwrap(); + + let dataset = commit_overlay( + dataset, + "fts_combined_flat_paths", + 0, + &[1], + OverlayCoverage::dense(RoaringBitmap::from_iter([1])), + vec![Arc::new(StringArray::from(vec![Some("cherry mango")]))], + ) + .await; + + assert_eq!(fts_ids_matching(&dataset, "mango").await, vec![1, 6, 12]); + assert_eq!( + fts_phrase_ids_matching(&dataset, "cherry mango").await, + vec![1, 12] + ); +} + +#[rstest] +#[tokio::test] +async fn test_fts_overlay_row_level_masking_under_fast_search( + #[values(false, true)] stable_row_ids: bool, +) { + let mut dataset = create_text_dataset(stable_row_ids).await; + build_text_fts_index_with_positions(&mut dataset).await; + + let dataset = commit_overlay( + dataset, + "fts_row_level_fast_search", + 0, + &[1], + OverlayCoverage::dense(RoaringBitmap::from_iter([1])), + vec![Arc::new(StringArray::from(vec![Some("cherry mango")]))], + ) + .await; + + let mut match_scan = dataset.scan(); + match_scan + .full_text_search(FullTextSearchQuery::new("apple".to_owned())) + .unwrap(); + match_scan.project(&["id"]).unwrap(); + match_scan.fast_search(); + let match_result = match_scan.try_into_batch().await.unwrap(); + assert_eq!( + ids_from_batches(std::slice::from_ref(&match_result)), + vec![0] + ); + + let mut indexed_phrase_scan = dataset.scan(); + indexed_phrase_scan + .full_text_search(FullTextSearchQuery::new_query(FtsQuery::Phrase( + PhraseQuery::new("apple pie".to_owned()).with_column(Some("text".to_owned())), + ))) + .unwrap(); + indexed_phrase_scan.project(&["id"]).unwrap(); + indexed_phrase_scan.fast_search(); + let indexed_phrase_result = indexed_phrase_scan.try_into_batch().await.unwrap(); + assert_eq!( + ids_from_batches(std::slice::from_ref(&indexed_phrase_result)), + vec![0] + ); + + let mut new_phrase_scan = dataset.scan(); + new_phrase_scan + .full_text_search(FullTextSearchQuery::new_query(FtsQuery::Phrase( + PhraseQuery::new("cherry mango".to_owned()).with_column(Some("text".to_owned())), + ))) + .unwrap(); + new_phrase_scan.project(&["id"]).unwrap(); + new_phrase_scan.fast_search(); + assert_eq!( + new_phrase_scan.try_into_batch().await.unwrap().num_rows(), + 0 + ); +} + +#[rstest] +#[tokio::test] +async fn test_fts_overlay_flat_path_takes_only_stale_rows( + #[values(false, true)] stable_row_ids: bool, +) { + let mut dataset = create_text_dataset(stable_row_ids).await; + build_text_fts_index(&mut dataset).await; + + let dataset = commit_overlay( + dataset, + "fts_targeted_take", + 0, + &[1], + OverlayCoverage::dense(RoaringBitmap::from_iter([1])), + vec![Arc::new(StringArray::from(vec![Some("cherry mango")]))], + ) + .await; + + let mut scan = dataset.scan(); + scan.full_text_search(FullTextSearchQuery::new("cherry".to_owned())) + .unwrap(); + scan.project(&["id"]).unwrap(); + let plan = scan.create_plan().await.unwrap(); + + let mut nodes = vec![plan]; + let mut flat_path_uses_targeted_take = false; + while let Some(node) = nodes.pop() { + if node.downcast_ref::().is_some() { + let mut flat_nodes = node.children().into_iter().cloned().collect::>(); + while let Some(flat_node) = flat_nodes.pop() { + if flat_node + .downcast_ref::() + .is_some_and(|read| read.index_input().is_some()) + { + flat_path_uses_targeted_take = true; + break; + } + flat_nodes.extend(flat_node.children().into_iter().cloned()); + } + } + nodes.extend(node.children().into_iter().cloned()); + } + + assert!( + flat_path_uses_targeted_take, + "overlay-stale FTS rows must be re-evaluated through a targeted take" + ); +} + +#[tokio::test] +async fn test_fts_phrase_searches_unindexed_fragments_unless_fast_search() { + let mut dataset = create_text_dataset(false).await; + build_text_fts_index_with_positions(&mut dataset).await; + + let batch = arrow_array::record_batch!( + ("id", Int32, [12, 13, 14]), + ( + "text", + Utf8, + [ + "kiwi berry", + "kiwi berry filling", + "kiwi berry filling filling" + ] + ) + ) + .unwrap(); + let schema = batch.schema(); + let reader = RecordBatchIterator::new(vec![Ok(batch)], schema); + let dataset = Dataset::write( + reader, + Arc::new(dataset), + Some(WriteParams { + mode: crate::dataset::write::WriteMode::Append, + ..Default::default() + }), + ) + .await + .unwrap(); + + let query = FullTextSearchQuery::new_query(FtsQuery::Phrase( + PhraseQuery::new("kiwi berry".to_owned()).with_column(Some("text".to_owned())), + )); + assert_eq!(fts_ids(&dataset, query.clone()).await, vec![12, 13, 14]); + + let mut limited_scan = dataset.scan(); + limited_scan.with_fragments(vec![dataset.fragments().last().unwrap().clone()]); + limited_scan + .full_text_search(query.clone().limit(Some(1))) + .unwrap(); + limited_scan.project(&["id"]).unwrap(); + let limited_result = limited_scan.try_into_batch().await.unwrap(); + assert_eq!( + ids_from_batches(std::slice::from_ref(&limited_result)), + vec![12] + ); + + let mut filtered_scan = dataset.scan(); + filtered_scan.full_text_search(query.clone()).unwrap(); + filtered_scan.project(&["id"]).unwrap(); + filtered_scan.filter("id < 12").unwrap(); + assert_eq!(filtered_scan.try_into_batch().await.unwrap().num_rows(), 0); + + let mut fast_scan = dataset.scan(); + fast_scan.full_text_search(query).unwrap(); + fast_scan.project(&["id"]).unwrap(); + fast_scan.fast_search(); + assert_eq!(fast_scan.try_into_batch().await.unwrap().num_rows(), 0); +} + +#[tokio::test] +async fn test_fts_phrase_stale_rows_honor_query_limit() { + let mut dataset = create_text_dataset(false).await; + build_text_fts_index_with_positions(&mut dataset).await; + + let dataset = commit_overlay( + dataset, + "phrase_limit_overlay", + 0, + &[1], + OverlayCoverage::dense(RoaringBitmap::from_iter([1])), + vec![Arc::new(StringArray::from(vec![Some( + "apple banana filling filling", + )]))], + ) + .await; + + let query = FullTextSearchQuery::new_query(FtsQuery::Phrase( + PhraseQuery::new("apple".to_owned()).with_column(Some("text".to_owned())), + )) + .limit(Some(1)); + let mut scan = dataset.scan(); + scan.with_fragments(vec![dataset.fragments()[0].clone()]); + scan.full_text_search(query).unwrap(); + scan.project(&["id"]).unwrap(); + + let result = scan.try_into_batch().await.unwrap(); + assert_eq!(ids_from_batches(std::slice::from_ref(&result)), vec![0]); +} + +/// Overlay routing must select FTS segments by both field and document +/// granularity when Row and ListElement indexes coexist. +#[tokio::test] +async fn test_list_element_fts_overlay_uses_exact_index_and_flat_fallback() { + let ids = Arc::new(Int32Array::from_iter_values(0..4)) as ArrayRef; + let tags = string_lists(&[ + &["old phrase", "keep"], + &["other"], + &["new phrase"], + &["unrelated"], + ]); + let batch = RecordBatch::try_from_iter(vec![("id", ids), ("tags", tags)]).unwrap(); + let schema = batch.schema(); + let mut dataset = Dataset::write( + RecordBatchIterator::new(vec![Ok(batch)], schema), + "memory://", + Some(WriteParams { + max_rows_per_file: 2, + ..Default::default() + }), + ) + .await + .unwrap(); + + dataset + .create_index( + &["tags"], + IndexType::Inverted, + None, + &InvertedIndexParams::default() + .with_position(true) + .document_granularity(DocumentGranularity::ListElement), + true, + ) + .await + .unwrap(); + dataset + .create_index( + &["tags"], + IndexType::Inverted, + None, + &InvertedIndexParams::default().with_position(true), + true, + ) + .await + .unwrap(); + + let dataset = commit_overlay( + dataset, + "list_element_text_overlay", + 0, + &[1], + OverlayCoverage::dense(RoaringBitmap::from_iter([0])), + vec![string_lists(&[&["new phrase", "keep"]])], + ) + .await; + + let list_element_match = |terms: &str| { + FullTextSearchQuery::new_query(FtsQuery::Match( + MatchQuery::new(terms.to_owned()) + .with_column(Some("tags".to_owned())) + .with_document_granularity(DocumentGranularity::ListElement), + )) + }; + let list_element_phrase = |terms: &str| { + FullTextSearchQuery::new_query(FtsQuery::Phrase( + PhraseQuery::new(terms.to_owned()) + .with_column(Some("tags".to_owned())) + .with_document_granularity(DocumentGranularity::ListElement), + )) + }; + + assert_eq!( + fts_ids(&dataset, list_element_match("old")).await, + Vec::::new() + ); + assert_eq!( + fts_ids(&dataset, list_element_match("new")).await, + vec![0, 2] + ); + assert_eq!( + fts_ids(&dataset, list_element_phrase("new phrase")).await, + vec![0, 2] + ); +} + +/// An overlay on a non-FTS field must not exclude the fragment from phrase search. +#[tokio::test] +async fn test_fts_phrase_overlay_unrelated_field_not_excluded() { + let mut dataset = create_text_dataset(false).await; + build_text_fts_index_with_positions(&mut dataset).await; + + // Overlay field 0 (`id`), not the FTS-indexed `text` column: phrase coverage is untouched. + let dataset = commit_overlay( + dataset, + "id_overlay", + 0, + &[0], + OverlayCoverage::dense(RoaringBitmap::from_iter([1])), + vec![i32_array([Some(777)])], + ) + .await; + + assert_eq!( + fts_phrase_ids_matching(&dataset, "apple banana").await, + vec![777] + ); +} + +/// An overlay on a field the FTS index does NOT cover must not exclude anything. +#[tokio::test] +async fn test_fts_overlay_unrelated_field_not_excluded() { + let mut dataset = create_text_dataset(false).await; + build_text_fts_index(&mut dataset).await; + + // Overlay field 0 (id) — not covered by the FTS index on `text`. + let dataset = commit_overlay( + dataset, + "id_overlay_for_fts", + 0, + &[0], + OverlayCoverage::dense(RoaringBitmap::from_iter([1])), + vec![i32_array([Some(999)])], + ) + .await; + + // FTS coverage must be unchanged — both rows containing "apple" are still returned. + // The `id` overlay changes row offset 1's id from 1 to 999, so the projected id column + // reflects the overlay even though the FTS index correctly returned that row. + assert_eq!(fts_ids_matching(&dataset, "apple").await, vec![0, 999]); + assert_eq!(fts_ids_matching(&dataset, "banana").await, vec![3, 999]); +} + +/// Benchmark: measure query latency for BTree, FTS, and vector ANN with 0/4/16 overlay layers. +/// +/// Run with: +/// cargo test -p lance --lib --profile release-with-debug -- overlay_index_masking::bench --ignored --nocapture +#[tokio::test] +#[ignore = "benchmark"] +#[allow(clippy::print_stdout)] +async fn bench_index_query_overlay_overhead() { + use std::time::Instant; + + use arrow_array::Float32Array; + + const DIM: i32 = 32; + const ROWS: i32 = 1_000_000; + const ROWS_PER_FRAG: i32 = 100_000; // 10 fragments + const ITERS: u32 = 10; // large scans — 10 is enough for stable averages + + // Fixed disk path so timings are comparable across runs. Deleted and recreated fresh. + let uri = "/tmp/lance-bench-overlay-oss1325"; + if std::path::Path::new(uri).exists() { + std::fs::remove_dir_all(uri).unwrap(); + } + + // --- Build 1M-row dataset on local disk -------------------------------- + // Schema: id, age, vec, text. Resolve field IDs from the Lance schema instead of + // assuming how nested Arrow child fields are numbered. + + println!("Building {ROWS}-row dataset at {uri} (this takes ~30 s)..."); + + let schema = Arc::new(ArrowSchema::new(vec![ + ArrowField::new("id", DataType::Int32, false), + ArrowField::new("age", DataType::Int32, false), + ArrowField::new( + "vec", + DataType::FixedSizeList( + Arc::new(ArrowField::new("item", DataType::Float32, true)), + DIM, + ), + false, + ), + ArrowField::new("text", DataType::Utf8, false), + ])); + + let row_ids: Vec = (0..ROWS).collect(); + let ages: Vec = row_ids.iter().map(|&i| i * 10).collect(); + // Build the 128 MB flat float array directly (avoids 1M per-row Vec allocations). + let flat_vecs: Vec = (0..(ROWS as usize * DIM as usize)) + .map(|j| (j / DIM as usize) as f32 % 1000.0) + .collect(); + let vec_col = Arc::new( + arrow_array::FixedSizeListArray::try_new( + Arc::new(ArrowField::new("item", DataType::Float32, true)), + DIM, + Arc::new(Float32Array::from(flat_vecs)), + None, + ) + .unwrap(), + ); + let text_col = Arc::new(StringArray::from_iter_values( + (0..ROWS).map(|row| if row == 42 { "needle" } else { "common" }), + )); + + let batch = RecordBatch::try_new( + schema.clone(), + vec![ + Arc::new(Int32Array::from(row_ids)), + Arc::new(Int32Array::from(ages)), + vec_col, + text_col, + ], + ) + .unwrap(); + + let write_params = WriteParams { + max_rows_per_file: ROWS_PER_FRAG as usize, + ..Default::default() + }; + let reader = RecordBatchIterator::new(vec![Ok(batch)], schema.clone()); + let mut dataset = Dataset::write(reader, uri, Some(write_params)) + .await + .unwrap(); + let text_field_id = dataset.schema().field_id("text").unwrap(); + + println!("Building BTree index on age..."); + dataset + .create_index( + &["age"], + IndexType::BTree, + None, + &ScalarIndexParams::default(), + true, + ) + .await + .unwrap(); + + println!("Building IVF_FLAT(1 partition) index on vec..."); + dataset + .create_index( + &["vec"], + IndexType::Vector, + None, + &VectorIndexParams::ivf_flat(1, MetricType::L2), + true, + ) + .await + .unwrap(); + + println!("Building FTS index on text..."); + dataset + .create_index( + &["text"], + IndexType::Inverted, + None, + &InvertedIndexParams::default(), + true, + ) + .await + .unwrap(); + + println!("Indexes built.\n"); + + // --- Timing helper --------------------------------------------------- + + async fn timeit(iters: u32, mut f: F) -> f64 + where + F: FnMut() -> Fut, + Fut: std::future::Future, + { + f().await; // warmup + let t0 = Instant::now(); + for _ in 0..iters { + f().await; + } + t0.elapsed().as_secs_f64() * 1000.0 / iters as f64 + } + + // === Scenario A: BTree query overhead ================================ + // + // Overlay on `age` (field 1), covering only offset 0 of fragment 0. The stale row is + // blocked from the BTree and re-evaluated by targeted take. + // + // btree_same_fragment: `age = 420` → id=42 → in fragment 0 (rows 0..99999). + // The matching row stays indexed even though another row in the fragment is stale. + // + // btree_other_fragment: `age = 1000420` → id=100042 → in fragment 1. + // This isolates the index-lookup baseline outside the overlaid fragment. + println!("=== Scenario A: BTree (one stale row in fragment 0) ==="); + println!( + "{:>10} {:>14} {:>14}", + "overlays", "same_frag_ms", "other_frag_ms" + ); + + let mut committed_a = 0u32; + for num_overlays in [0u32, 1, 4, 16] { + // Commit only the delta since the last iteration. + for layer in committed_a..num_overlays { + dataset = commit_overlay( + dataset, + &format!("age_ol{layer}"), + 0, // fragment 0 + &[1], // field 1 = age + OverlayCoverage::dense(RoaringBitmap::from_iter([0u32])), + vec![i32_array([Some(999)])], + ) + .await; + } + committed_a = num_overlays; + + let ds = Arc::new(dataset.clone()); + + // Same-fragment indexed match plus targeted re-evaluation of the stale row. + let ds2 = ds.clone(); + let same_fragment_ms = timeit(ITERS, || { + let ds = ds2.clone(); + async move { + ds.scan() + .filter("age = 420") + .unwrap() + .project(&["age"]) + .unwrap() + .try_into_batch() + .await + .unwrap(); + } + }) + .await; + + // Fragment 1 never has a stale row and stays entirely index-served. + let ds2 = ds.clone(); + let other_fragment_ms = timeit(ITERS, || { + let ds = ds2.clone(); + async move { + ds.scan() + .filter("age = 1000420") + .unwrap() + .project(&["age"]) + .unwrap() + .try_into_batch() + .await + .unwrap(); + } + }) + .await; + + println!("{num_overlays:>10} {same_fragment_ms:>14.1} {other_fragment_ms:>14.1}"); + } + + // === Scenario B: Vector ANN overhead ================================= + // + // Overlay on `vec` (field 2), covering only offset 0 of fragment 0. + // The field-aware check means the 16 age overlays from Scenario A do NOT affect + // the vector index (they touch field 1, not field 2). Only a vec overlay (field 2) + // marks fragment 0 stale for the vector index. + // + // With a vec overlay, only the stale row is excluded from ANN and re-scored exactly. + println!("\n=== Scenario B: Vector ANN (one stale row re-scored) ==="); + println!("{:>12} {:>10}", "vec_overlays", "ann_ms"); + + let query_vec = Float32Array::from(vec![0.5f32; DIM as usize]); + + for num_vec_overlays in [0u32, 1] { + if num_vec_overlays == 1 { + dataset = commit_overlay( + dataset, + "vec_ol0", + 0, // fragment 0 + &[2], // field 2 = vec (FixedSizeList top-level field) + OverlayCoverage::dense(RoaringBitmap::from_iter([0u32])), + vec![fsl(vec![vec![0.0f32; DIM as usize]], DIM)], + ) + .await; + } + + let ds = Arc::new(dataset.clone()); + let ds2 = ds.clone(); + let qv = query_vec.clone(); + let ann_ms = timeit(ITERS, || { + let ds = ds2.clone(); + let q = qv.clone(); + async move { + ds.scan() + .nearest("vec", &q, 10) + .unwrap() + .minimum_nprobes(1) + .project(&["id"]) + .unwrap() + .try_into_batch() + .await + .unwrap(); + } + }) + .await; + + println!("{num_vec_overlays:>12} {ann_ms:>10.1}"); + } + + // === Scenario C: FTS overhead ======================================== + // + // The FTS index has one segment spanning all 10 fragments. An overlay on one text row must + // keep that segment indexed, block just the stale row, and re-evaluate that row by targeted + // take. `needle` belongs to an unaffected row in the same fragment as the stale row. + println!("\n=== Scenario C: FTS (one stale row in a 1M-row segment) ==="); + println!("{:>13} {:>10}", "text_overlays", "fts_ms"); + + for num_text_overlays in [0u32, 1] { + if num_text_overlays == 1 { + dataset = commit_overlay( + dataset, + "text_ol0", + 0, + &[text_field_id], + OverlayCoverage::dense(RoaringBitmap::from_iter([0u32])), + vec![Arc::new(StringArray::from(vec![Some("updated")]))], + ) + .await; + } + + let ds = Arc::new(dataset.clone()); + let ds2 = ds.clone(); + let fts_ms = timeit(ITERS, || { + let ds = ds2.clone(); + async move { + let result = ds + .scan() + .full_text_search(FullTextSearchQuery::new("needle".to_owned())) + .unwrap() + .project(&["id"]) + .unwrap() + .try_into_batch() + .await + .unwrap(); + assert_eq!(result.num_rows(), 1); + } + }) + .await; + + println!("{num_text_overlays:>13} {fts_ms:>10.1}"); + } +} + +async fn append_age_fragment(dataset: &mut Dataset, ids: std::ops::Range) { + let schema = Arc::new(ArrowSchema::new(vec![ + ArrowField::new("id", DataType::Int32, true), + ArrowField::new("age", DataType::Int32, true), + ])); + let batch = RecordBatch::try_new( + schema.clone(), + vec![ + Arc::new(Int32Array::from_iter_values(ids.clone())), + Arc::new(Int32Array::from_iter_values(ids.map(|v| v * 10))), + ], + ) + .unwrap(); + dataset + .append( + RecordBatchIterator::new(vec![Ok(batch)], schema.clone()), + None, + ) + .await + .unwrap(); +} + +// `OptimizeIndices` merges an index's delta segments without re-reading data overlays, so the +// merged segment carries the old segments' pre-overlay entries. What keeps those entries masked +// is that the merge stamps the new segment with the *oldest* merged segment's `dataset_version` +// rather than the current one, leaving the mask's version gate +// (`overlay.committed_version > segment.dataset_version`) on. +// +// That invariant is easy to break by accident -- stamping the current version un-masks every +// carried-over entry -- and nothing else asserts it end to end. The following tests pin it down +// for each index type. + +/// Scalar (BTree, Bitmap, ZoneMap): a range query over the indexed column after an overlay + +/// optimize must still drop the stale value and surface the overlaid one. +/// +/// ZoneMap is the case worth having: it ignores `OldIndexDataFilter`, so nothing scrubs the +/// pre-overlay zone summaries out of the merged segment. Breaking the version gate shows up here +/// as a false *negative* -- the stale zone prunes the overlaid value -- rather than the resurfaced +/// stale entry the other two produce. +#[rstest] +#[case::btree(IndexType::BTree)] +#[case::bitmap(IndexType::Bitmap)] +#[case::zonemap(IndexType::ZoneMap)] +#[tokio::test] +async fn test_optimize_preserves_scalar_overlay_masking(#[case] index_type: IndexType) { + use crate::index::DatasetIndexExt; + use lance_index::optimize::OptimizeOptions; + use lance_index::scalar::BuiltinIndexType; + + let params = match index_type { + IndexType::Bitmap => ScalarIndexParams::for_builtin(BuiltinIndexType::Bitmap), + IndexType::ZoneMap => ScalarIndexParams::for_builtin(BuiltinIndexType::ZoneMap), + _ => ScalarIndexParams::default(), + }; + let mut dataset = create_base_dataset().await; + dataset + .create_index(&["age"], index_type, None, ¶ms, true) + .await + .unwrap(); + + // Overlay fragment 0, offset 1 (id=1): age 10 -> 999, committed after the index. + let mut dataset = commit_overlay( + dataset, + "age_opt", + 0, + &[1], + OverlayCoverage::dense(RoaringBitmap::from_iter([1])), + vec![i32_array([Some(999)])], + ) + .await; + + // Masking works before optimize. + assert_eq!(ids_matching(&dataset, "age = 10").await, Vec::::new()); + assert_eq!(ids_matching(&dataset, "age = 999").await, vec![1]); + + // Append an unindexed fragment so the merge does real work, then merge all deltas. + append_age_fragment(&mut dataset, 12..18).await; + dataset + .optimize_indices(&OptimizeOptions::merge(10)) + .await + .unwrap(); + + // Still masked: the stale age=10 entry stays dropped and the overlaid age=999 value stays + // visible, because the merged segment kept the old segment's `dataset_version`. + assert_eq!( + ids_matching(&dataset, "age = 10").await, + Vec::::new(), + "stale index entry age=10 for id=1 resurfaced after optimize" + ); + assert_eq!( + ids_matching(&dataset, "age = 999").await, + vec![1], + "overlaid value age=999 dropped after optimize" + ); + // A row untouched by the overlay is unaffected. + assert_eq!(ids_matching(&dataset, "age = 20").await, vec![2]); +} + +/// ZoneMap seed path: an *unindexed* fragment carrying an overlay gets folded into the merged +/// segment from its data file's seed buffer. A seed is a zone summary captured while the base +/// data file was written, so it describes pre-overlay values -- the merged segment ends up +/// holding zones that never saw the overlay. +/// +/// That is only safe because the merge keeps the old `dataset_version`, so the mask still covers +/// those rows. Stamp the current version (or teach the seed path to claim freshness) and the +/// overlaid value becomes unfindable, since its stale zone prunes it. +/// +/// `name` is Utf8, for which seeds are on by default (`default_use_seeds`). +#[tokio::test] +async fn test_optimize_seed_path_respects_overlay() { + use crate::index::DatasetIndexExt; + use lance_index::optimize::OptimizeOptions; + use lance_index::scalar::BuiltinIndexType; + + let schema = Arc::new(ArrowSchema::new(vec![ + ArrowField::new("id", DataType::Int32, true), + ArrowField::new("name", DataType::Utf8, true), + ])); + let names: Vec = (0..12).map(|i| format!("n{i:02}")).collect(); + let batch = RecordBatch::try_new( + schema.clone(), + vec![ + Arc::new(Int32Array::from_iter_values(0..12)), + Arc::new(StringArray::from(names)), + ], + ) + .unwrap(); + let mut dataset = Dataset::write( + RecordBatchIterator::new(vec![Ok(batch)], schema.clone()), + "memory://", + Some(WriteParams { + max_rows_per_file: 6, + ..Default::default() + }), + ) + .await + .unwrap(); + dataset + .create_index( + &["name"], + IndexType::ZoneMap, + None, + &ScalarIndexParams::for_builtin(BuiltinIndexType::ZoneMap), + true, + ) + .await + .unwrap(); + + // Append fragment 2. The index already exists, so the write emits a zone-map seed buffer + // into the new data file. + let appended: Vec = (12..18).map(|i| format!("n{i:02}")).collect(); + let batch = RecordBatch::try_new( + schema.clone(), + vec![ + Arc::new(Int32Array::from_iter_values(12..18)), + Arc::new(StringArray::from(appended)), + ], + ) + .unwrap(); + dataset + .append( + RecordBatchIterator::new(vec![Ok(batch)], schema.clone()), + None, + ) + .await + .unwrap(); + + // Overlay fragment 2, offset 1 (id=13): "n13" -> "zzz", well outside the seed's zone range. + let mut dataset = commit_overlay( + dataset, + "name_seed", + 2, + &[1], + OverlayCoverage::dense(RoaringBitmap::from_iter([1])), + vec![Arc::new(StringArray::from(vec![Some("zzz")]))], + ) + .await; + + // Fragment 2 is unindexed, so the overlaid value is visible via the flat path. + assert_eq!(ids_matching(&dataset, "name = 'zzz'").await, vec![13]); + assert_eq!( + ids_matching(&dataset, "name = 'n13'").await, + Vec::::new() + ); + + dataset + .optimize_indices(&OptimizeOptions::merge(10)) + .await + .unwrap(); + + assert_eq!( + ids_matching(&dataset, "name = 'zzz'").await, + vec![13], + "overlaid value dropped after optimize: the merged segment took pre-overlay zones \ + from fragment 2's seed buffer" + ); + assert_eq!( + ids_matching(&dataset, "name = 'n13'").await, + Vec::::new(), + "stale pre-overlay value resurfaced after optimize" + ); + // Rows the overlay never touched keep working through the index. + assert_eq!(ids_matching(&dataset, "name = 'n14'").await, vec![14]); + assert_eq!(ids_matching(&dataset, "name = 'n03'").await, vec![3]); +} + +/// `DatasetStatistics::column_value_range` folds ZoneMap summaries into a global `[min, max]` +/// that callers may prune with, so it must be a superset of the live values. An overlay can +/// move a value outside the summarised range, and the ZoneMap never saw it. +#[tokio::test] +async fn test_column_value_range_none_under_overlay() { + use crate::index::DatasetIndexExt; + use datafusion::scalar::ScalarValue; + use lance_index::scalar::BuiltinIndexType; + + let mut dataset = create_base_dataset().await; + dataset + .create_index( + &["age"], + IndexType::ZoneMap, + None, + &ScalarIndexParams::for_builtin(BuiltinIndexType::ZoneMap), + true, + ) + .await + .unwrap(); + + assert_eq!( + dataset + .statistics() + .column_value_range("age") + .await + .unwrap(), + Some((ScalarValue::Int32(Some(0)), ScalarValue::Int32(Some(110)))) + ); + + // age 10 -> 999 on fragment 0, committed after the index: 999 is outside [0, 110]. + let dataset = commit_overlay( + dataset, + "age_range", + 0, + &[1], + OverlayCoverage::dense(RoaringBitmap::from_iter([1])), + vec![i32_array([Some(999)])], + ) + .await; + + assert_eq!( + dataset + .statistics() + .column_value_range("age") + .await + .unwrap(), + None, + "ZoneMap range must not be reported once an overlay may have moved a value outside it" + ); +} + +/// FTS: after an overlay replaces a row's text and the index is optimized, searching for the old +/// terms must not return the stale row, and the new terms must find it. +#[tokio::test] +async fn test_optimize_preserves_fts_overlay_masking() { + use crate::index::DatasetIndexExt; + use lance_index::optimize::OptimizeOptions; + + let mut dataset = create_text_dataset(false).await; + build_text_fts_index(&mut dataset).await; + + // fragment 0, offset 1 (id=1): "apple banana" -> "cherry mango". + let mut dataset = commit_overlay( + dataset, + "text_opt", + 0, + &[1], + OverlayCoverage::dense(RoaringBitmap::from_iter([1])), + vec![Arc::new(StringArray::from(vec![Some("cherry mango")]))], + ) + .await; + + // Masking works before optimize: id=1 no longer matches "banana"/"apple". + assert_eq!(fts_ids_matching(&dataset, "banana").await, vec![3]); + assert_eq!(fts_ids_matching(&dataset, "apple").await, vec![0]); + + // Append an unindexed fragment of new text, then merge all deltas. + let schema = Arc::new(ArrowSchema::new(vec![ + ArrowField::new("id", DataType::Int32, true), + ArrowField::new("text", DataType::Utf8, true), + ])); + let batch = RecordBatch::try_new( + schema.clone(), + vec![ + Arc::new(Int32Array::from_iter_values(12..18)), + Arc::new(StringArray::from(vec![ + "kiwi", "melon", "date", "guava", "papaya", "lychee", + ])), + ], + ) + .unwrap(); + dataset + .append( + RecordBatchIterator::new(vec![Ok(batch)], schema.clone()), + None, + ) + .await + .unwrap(); + dataset + .optimize_indices(&OptimizeOptions::merge(10)) + .await + .unwrap(); + + // Still masked: id=1's stale "apple"/"banana" postings stay dropped. + assert_eq!( + fts_ids_matching(&dataset, "banana").await, + vec![3], + "stale FTS posting for id=1 (banana) resurfaced after optimize" + ); + assert_eq!( + fts_ids_matching(&dataset, "apple").await, + vec![0], + "stale FTS posting for id=1 (apple) resurfaced after optimize" + ); + // The overlaid terms are found via the flat path. + assert!(fts_ids_matching(&dataset, "cherry").await.contains(&1)); + assert!(fts_ids_matching(&dataset, "mango").await.contains(&1)); +} + +/// Vector (IVF): after an overlay moves a row's vector and the index is optimized, the ANN must +/// not resurface the stale vector, and the moved-onto-query row is found by flat re-scoring. +#[tokio::test] +async fn test_optimize_preserves_vector_overlay_masking() { + use crate::index::DatasetIndexExt; + use lance_index::optimize::OptimizeOptions; + + // Overlay on fragment 1 moves id=35 away from the query and id=40 onto it. + let mut dataset = create_vector_overlay_dataset(false).await; + + // Masking works before optimize. + let before = vector_query_ids(&dataset, 3, false).await; + assert!( + !before.contains(&35), + "pre-optimize id=35 should be dropped: {before:?}" + ); + assert!( + before.contains(&40), + "pre-optimize id=40 should be found: {before:?}" + ); + + // Append an unindexed fragment of far vectors, then merge all deltas. + let far_vecs: Vec> = (0..32) + .map(|i| { + let mut v = vec![0.0_f32; VEC_DIM as usize]; + v[1] = (i + 200) as f32; + v + }) + .collect(); + let schema = Arc::new(ArrowSchema::new(vec![ + ArrowField::new("id", DataType::Int32, true), + ArrowField::new( + "vec", + DataType::FixedSizeList( + Arc::new(ArrowField::new("item", DataType::Float32, true)), + VEC_DIM, + ), + true, + ), + ])); + let batch = RecordBatch::try_new( + schema.clone(), + vec![ + Arc::new(Int32Array::from_iter_values(64..96)), + fsl(far_vecs, VEC_DIM), + ], + ) + .unwrap(); + dataset + .append( + RecordBatchIterator::new(vec![Ok(batch)], schema.clone()), + None, + ) + .await + .unwrap(); + dataset + .optimize_indices(&OptimizeOptions::merge(10)) + .await + .unwrap(); + + // Still masked: id=35's stale vector stays dropped and id=40 is still found via re-scoring. + let after = vector_query_ids(&dataset, 3, false).await; + assert!( + !after.contains(&35), + "stale index vector for id=35 resurfaced after optimize: {after:?}" + ); + assert!( + after.contains(&40), + "overlaid vector for id=40 dropped after optimize: {after:?}" + ); +} diff --git a/rust/lance/src/dataset/tests/dataset_scanner.rs b/rust/lance/src/dataset/tests/dataset_scanner.rs index 363698d9966..97cecaa6245 100644 --- a/rust/lance/src/dataset/tests/dataset_scanner.rs +++ b/rust/lance/src/dataset/tests/dataset_scanner.rs @@ -11,11 +11,11 @@ use lance_arrow::{ARROW_EXT_NAME_KEY, FixedSizeListArrayExt}; use crate::index::DatasetIndexExt; use arrow::compute::concat_batches; -use arrow_array::UInt64Array; use arrow_array::cast::AsArray; -use arrow_array::{Array, FixedSizeListArray, ListArray, StructArray}; +use arrow_array::{Array, ArrayRef, FixedSizeListArray, LargeListArray, ListArray, StructArray}; use arrow_array::{Float32Array, Int32Array, RecordBatch, RecordBatchIterator, StringArray}; -use arrow_buffer::{OffsetBuffer, ScalarBuffer}; +use arrow_array::{Int64Array, UInt64Array}; +use arrow_buffer::{NullBuffer, OffsetBuffer, ScalarBuffer}; use arrow_schema::{DataType, Field as ArrowField, Fields, Schema as ArrowSchema, SchemaRef}; use futures::TryStreamExt; use lance_arrow::SchemaExt; @@ -25,7 +25,9 @@ use lance_file::reader::{FileReader, FileReaderOptions, describe_encoding}; use lance_file::version::LanceFileVersion; use lance_index::scalar::FullTextSearchQuery; use lance_index::scalar::inverted::{ - SCORE_FIELD, query::PhraseQuery, tokenizer::InvertedIndexParams, + SCORE_FIELD, + query::{FtsQuery, MatchQuery, Operator, PhraseQuery}, + tokenizer::InvertedIndexParams, }; use lance_index::{IndexType, vector::DIST_COL}; use lance_io::scheduler::{ScanScheduler, SchedulerConfig}; @@ -33,14 +35,155 @@ use lance_io::utils::CachedFileSize; use lance_linalg::distance::MetricType; use uuid::Uuid; -use crate::Dataset; +use crate::dataset::NewColumnTransform; use crate::dataset::scanner::{DatasetRecordBatchStream, QueryFilter}; use crate::dataset::write::WriteParams; -use lance_index::scalar::inverted::query::FtsQuery; +use crate::{Dataset, Error}; use lance_index::vector::ivf::IvfBuildParams; use lance_index::vector::pq::PQBuildParams; use lance_index::vector::{DEFAULT_QUERY_PARALLELISM, Query}; use pretty_assertions::assert_eq; +use rstest::rstest; + +/// A null struct must not read back as a valid struct with null children. +/// +/// A scan merges the per-column batches with `lance_arrow::merge`, which used to read an all-null +/// validity buffer as "this side carries no validity" and drop it. A filter that selects only null +/// rows leaves exactly that shape, so the scan reported those rows as valid while `IS NULL` +/// counted them as null. The dataset is created empty and then appended to because that is the +/// path this was found on, and the version is pinned because 2.0 does not encode struct validity. +#[tokio::test] +async fn test_filtered_scan_preserves_nullable_struct_validity() { + let struct_fields = Fields::from(vec![ + ArrowField::new("a", DataType::Int64, true), + ArrowField::new("b", DataType::Utf8, true), + ]); + let item_field = Arc::new(ArrowField::new( + "item", + DataType::Struct(struct_fields.clone()), + true, + )); + let schema = Arc::new(ArrowSchema::new(vec![ + ArrowField::new("id", DataType::UInt64, false), + ArrowField::new("s", DataType::Struct(struct_fields.clone()), true), + ArrowField::new("l", DataType::List(item_field.clone()), true), + ])); + + let empty = RecordBatch::new_empty(schema.clone()); + let reader = RecordBatchIterator::new([Ok(empty)], schema.clone()); + let mut dataset = Dataset::write( + reader, + "memory://", + Some(WriteParams { + data_storage_version: Some(LanceFileVersion::V2_1), + ..WriteParams::default() + }), + ) + .await + .unwrap(); + + // Rows 100 and 177 are null in both nested columns while their children still carry values, + // so losing the top-level validity turns them into valid values instead of null ones. + let validity = NullBuffer::from(vec![false, true, false]); + let structs = StructArray::new( + struct_fields.clone(), + vec![ + Arc::new(Int64Array::from(vec![Some(10), Some(11), Some(12)])), + Arc::new(StringArray::from(vec![Some("x"), Some("y"), Some("z")])), + ], + Some(validity.clone()), + ); + let items = StructArray::new( + struct_fields.clone(), + vec![ + Arc::new(Int64Array::from(vec![Some(20), Some(21), Some(22)])), + Arc::new(StringArray::from(vec![Some("p"), Some("q"), Some("r")])), + ], + None, + ); + let lists = ListArray::new( + item_field, + OffsetBuffer::new(ScalarBuffer::from(vec![0, 1, 2, 3])), + Arc::new(items), + Some(validity), + ); + let batch = RecordBatch::try_new( + schema.clone(), + vec![ + Arc::new(UInt64Array::from(vec![100, 116, 177])), + Arc::new(structs), + Arc::new(lists), + ], + ) + .unwrap(); + dataset + .append( + Box::new(RecordBatchIterator::new([Ok(batch)], schema)), + None, + ) + .await + .unwrap(); + + for (id, expected_is_null) in [(100, true), (116, false), (177, true)] { + let mut scan = dataset.scan(); + scan.filter(&format!("id = {id}")).unwrap(); + let batch = scan.try_into_batch().await.unwrap(); + let structs = batch["s"].as_struct(); + assert_eq!(structs.is_null(0), expected_is_null, "s, row id {id}"); + // A null struct masks its children, so both levels have to agree. + assert_eq!( + structs.column(0).is_null(0), + expected_is_null, + "s.a, row id {id}" + ); + assert_eq!( + batch["l"].as_list::().is_null(0), + expected_is_null, + "l, row id {id}" + ); + } + + assert_eq!( + dataset + .count_rows(Some("s IS NULL".to_owned())) + .await + .unwrap(), + 2 + ); + assert_eq!( + dataset + .count_rows(Some("l IS NULL".to_owned())) + .await + .unwrap(), + 2 + ); + + // A struct column added as all-nulls reaches the same merge through schema evolution, where + // every row is null and there is no other side to recover the validity from. + dataset + .add_columns( + NewColumnTransform::AllNulls(Arc::new(ArrowSchema::new(vec![ArrowField::new( + "t", + DataType::Struct(struct_fields), + true, + )]))), + None, + None, + ) + .await + .unwrap(); + let mut scan = dataset.scan(); + scan.filter("id = 116").unwrap(); + let batch = scan.try_into_batch().await.unwrap(); + assert!(batch["t"].as_struct().is_null(0)); + assert_eq!( + dataset + .count_rows(Some("t IS NULL".to_owned())) + .await + .unwrap(), + 3 + ); +} #[tokio::test] async fn test_scan_wide_fixed_size_list_at_batch_boundary() { @@ -401,6 +544,212 @@ async fn test_fts_filter_vector_search() { assert!(stream.is_err()); } +#[rstest] +#[case::list(false)] +#[case::large_list(true)] +#[tokio::test] +async fn test_fts_list_postfilter_vector_search(#[case] is_large_list: bool) { + async fn indexed_ids(dataset: &Dataset, query: FullTextSearchQuery) -> Vec { + let result = dataset + .scan() + .project(&["id"]) + .unwrap() + .full_text_search(query) + .unwrap() + .try_into_batch() + .await + .unwrap(); + let mut ids = result["id"] + .as_any() + .downcast_ref::() + .unwrap() + .values() + .to_vec(); + ids.sort_unstable(); + ids + } + + async fn postfilter_ids(dataset: &Dataset, query: FullTextSearchQuery) -> Vec { + let query_vector = Float32Array::from(vec![0.0, 0.0]); + let mut scanner = dataset.scan(); + scanner + .nearest("vector", &query_vector, 5) + .unwrap() + .prefilter(false) + .filter_query(QueryFilter::Fts(query)) + .unwrap(); + let plan = scanner.explain_plan(false).await.unwrap(); + let post_filter_position = plan + .find("FlatMatchFilter: column=docs") + .expect("expected FTS to run as a flat match filter"); + let vector_search_position = plan + .find("ANNSubIndex") + .expect("expected the query to use the vector index"); + assert!( + post_filter_position < vector_search_position, + "expected FTS to wrap the vector search as a post-filter, got:\n{plan}" + ); + let result = scanner.try_into_batch().await.unwrap(); + let mut ids = result["id"] + .as_any() + .downcast_ref::() + .unwrap() + .values() + .to_vec(); + ids.sort_unstable(); + ids + } + + fn match_query(terms: &str, operator: Operator) -> FullTextSearchQuery { + FullTextSearchQuery::new_query(FtsQuery::Match( + MatchQuery::new(terms.to_owned()) + .with_column(Some("docs".to_owned())) + .with_operator(operator), + )) + } + + let item_field = Arc::new(ArrowField::new("item", DataType::Utf8, true)); + let values = Arc::new(StringArray::from(vec![ + Some("target"), + Some("alpha"), + Some("beta"), + Some(""), + None, + Some("target"), + ])) as ArrayRef; + let validity = Some(NullBuffer::from(vec![true, true, true, true, false])); + let docs: ArrayRef = if is_large_list { + Arc::new(LargeListArray::new( + item_field.clone(), + OffsetBuffer::new(ScalarBuffer::from(vec![0_i64, 2, 3, 3, 6, 6])), + values, + validity, + )) + } else { + Arc::new(ListArray::new( + item_field, + OffsetBuffer::new(ScalarBuffer::from(vec![0_i32, 2, 3, 3, 6, 6])), + values, + validity, + )) + }; + let vectors = FixedSizeListArray::try_new_from_values( + Float32Array::from(vec![0.0, 0.0, 1.0, 1.0, 2.0, 2.0, 3.0, 3.0, 4.0, 4.0]), + 2, + ) + .unwrap(); + let schema = Arc::new(ArrowSchema::new(vec![ + ArrowField::new("id", DataType::Int32, false), + ArrowField::new("vector", vectors.data_type().clone(), false), + ArrowField::new("docs", docs.data_type().clone(), true), + ])); + let batch = RecordBatch::try_new( + schema.clone(), + vec![ + Arc::new(Int32Array::from_iter_values(0..5)), + Arc::new(vectors), + docs, + ], + ) + .unwrap(); + let mut dataset = Dataset::write( + RecordBatchIterator::new([Ok(batch)], schema), + "memory://", + Some(WriteParams { + max_rows_per_file: 2, + ..Default::default() + }), + ) + .await + .unwrap(); + assert_eq!(dataset.get_fragments().len(), 3); + + dataset + .create_index( + &["docs"], + IndexType::Inverted, + None, + &InvertedIndexParams::default(), + true, + ) + .await + .unwrap(); + dataset + .create_index( + &["vector"], + IndexType::Vector, + None, + &VectorIndexParams::ivf_flat(1, MetricType::L2), + true, + ) + .await + .unwrap(); + + let query = match_query("target", Operator::Or); + assert_eq!(indexed_ids(&dataset, query.clone()).await, [0, 3]); + assert_eq!(postfilter_ids(&dataset, query).await, [0, 3]); + + let query = match_query("target alpha", Operator::And); + assert_eq!(indexed_ids(&dataset, query.clone()).await, [0]); + assert_eq!(postfilter_ids(&dataset, query).await, [0]); + + let query = match_query("target missing", Operator::And); + assert!(indexed_ids(&dataset, query.clone()).await.is_empty()); + assert!(postfilter_ids(&dataset, query).await.is_empty()); + + dataset + .create_index( + &["docs"], + IndexType::Inverted, + None, + &InvertedIndexParams::default().base_tokenizer("raw".to_owned()), + true, + ) + .await + .unwrap(); + let query = match_query("target", Operator::Or); + assert_eq!(indexed_ids(&dataset, query.clone()).await, [3]); + assert_eq!(postfilter_ids(&dataset, query).await, [3]); + + dataset + .create_index( + &["docs"], + IndexType::Inverted, + None, + &InvertedIndexParams::code() + .split_identifiers(true) + .preserve_original(true), + true, + ) + .await + .unwrap(); + let query = match_query("targetAlpha", Operator::And); + assert_eq!(indexed_ids(&dataset, query.clone()).await, [0]); + assert_eq!(postfilter_ids(&dataset, query).await, [0]); + + let fuzzy_query = FullTextSearchQuery::new_query(FtsQuery::Match( + MatchQuery::new("targets".to_owned()) + .with_column(Some("docs".to_owned())) + .with_fuzziness(Some(1)), + )); + let query_vector = Float32Array::from(vec![0.0, 0.0]); + let mut scanner = dataset.scan(); + scanner + .nearest("vector", &query_vector, 5) + .unwrap() + .prefilter(false) + .filter_query(QueryFilter::Fts(fuzzy_query)) + .unwrap(); + let error = scanner.try_into_batch().await.unwrap_err(); + assert!(matches!(&error, Error::NotSupported { .. })); + assert!( + error + .to_string() + .contains("Fuzzy MatchQuery is not supported when FTS is used as a post-filter"), + "unexpected error: {error}" + ); +} + #[tokio::test] async fn test_scan_limit_offset_preserves_json_extension_metadata() { let schema = Arc::new(ArrowSchema::new(vec![ diff --git a/rust/lance/src/dataset/tests/dataset_schema_evolution.rs b/rust/lance/src/dataset/tests/dataset_schema_evolution.rs index d311d62f555..c00f3c4cdaa 100644 --- a/rust/lance/src/dataset/tests/dataset_schema_evolution.rs +++ b/rust/lance/src/dataset/tests/dataset_schema_evolution.rs @@ -10,7 +10,7 @@ use arrow_array::{ use arrow_schema::{ DataType, Field as ArrowField, Field, Fields as ArrowFields, Fields, Schema as ArrowSchema, }; -use lance_encoding::version::LanceFileVersion; +use lance_file::version::LanceFileVersion; use rstest::rstest; use std::collections::HashMap; use std::sync::Arc; diff --git a/rust/lance/src/dataset/tests/dataset_transactions.rs b/rust/lance/src/dataset/tests/dataset_transactions.rs index 3e2a4caa3b3..d8ada2c01fa 100644 --- a/rust/lance/src/dataset/tests/dataset_transactions.rs +++ b/rust/lance/src/dataset/tests/dataset_transactions.rs @@ -272,6 +272,39 @@ pub(super) fn assert_results( ) } +fn gen_rows() -> impl arrow_array::RecordBatchReader + Send + 'static { + lance_datagen::gen_batch() + .col("key", array::step::()) + .into_reader_rows(RowCount::from(10), BatchCount::from(1)) +} + +/// Write a dataset with `versions` versions of 10 rows each. +async fn write_versions(uri: &str, versions: usize, enable_v2_manifest_paths: bool) -> Dataset { + let mut ds = Dataset::write( + gen_rows(), + uri, + Some(WriteParams { + enable_v2_manifest_paths, + ..Default::default() + }), + ) + .await + .unwrap(); + for _ in 1..versions { + ds.append( + gen_rows(), + Some(WriteParams { + mode: WriteMode::Append, + enable_v2_manifest_paths, + ..Default::default() + }), + ) + .await + .unwrap(); + } + ds +} + #[tokio::test] async fn test_inline_transaction() { use arrow_array::{Int32Array, RecordBatch, RecordBatchIterator}; @@ -349,10 +382,13 @@ async fn test_inline_transaction() { // Case 3: manifest does not contain inline transaction, read should fall back to external transaction file let ds = create_dataset(2).await; let tx = make_tx(ds.manifest().version); - let tx_file = - crate::io::commit::write_transaction_file(ds.object_store.as_ref(), &ds.base, &tx) - .await - .unwrap(); + let tx_file = crate::io::commit::write_transaction_file( + ds.object_store.as_ref(), + &ds.base, + &lance_table::format::pb::Transaction::from(&tx), + ) + .await + .unwrap(); let (mut manifest, indices) = tx .build_manifest( Some(ds.manifest.as_ref()), @@ -382,6 +418,197 @@ async fn test_inline_transaction() { assert!(ds_new.manifest.transaction_file.is_some()); let read_tx = ds_new.read_transaction().await.unwrap().unwrap(); assert_eq!(read_tx, tx); + + // The direct read takes the same external-file fallback. + let version_transaction = ds_new + .read_version_transaction(location.version) + .await + .unwrap(); + assert_eq!(version_transaction.transaction, Some(tx)); +} + +#[tokio::test] +async fn test_read_version_transaction_does_not_populate_caches() { + use lance_index::IndexType; + use lance_index::scalar::ScalarIndexParams; + + let test_uri = TempStrDir::default(); + let mut dataset = write_versions(&test_uri, 1, true).await; + // Index the table so historical manifests carry an IndexSection that a + // caching read path would decode. + dataset + .create_index( + &["key"], + IndexType::BTree, + None, + &ScalarIndexParams::default(), + true, + ) + .await + .unwrap(); // version 2 + for _ in 0..18 { + dataset + .append( + gen_rows(), + Some(WriteParams { + mode: WriteMode::Append, + ..Default::default() + }), + ) + .await + .unwrap(); + } + let latest_version = dataset.version().version; + assert_eq!(latest_version, 20); + + // Fresh session so any cache insertion by the API under test shows as growth. + let session = Arc::new(Session::default()); + let dataset = DatasetBuilder::from_uri(&test_uri) + .with_session(session.clone()) + .load() + .await + .unwrap(); + + let metadata_stats_before = session.metadata_cache_stats().await; + let index_stats_before = session.index_cache_stats().await; + + let mut actual = Vec::with_capacity(latest_version as usize); + for version in 1..=latest_version { + let version_transaction = dataset.read_version_transaction(version).await.unwrap(); + assert_eq!(version_transaction.version, version); + actual.push(version_transaction); + } + + let metadata_stats_after = session.metadata_cache_stats().await; + let index_stats_after = session.index_cache_stats().await; + assert_eq!( + metadata_stats_after.num_entries, + metadata_stats_before.num_entries + ); + assert_eq!( + metadata_stats_after.size_bytes, + metadata_stats_before.size_bytes + ); + assert_eq!( + index_stats_after.num_entries, + index_stats_before.num_entries + ); + assert_eq!(index_stats_after.size_bytes, index_stats_before.size_bytes); + + // Results match a full checkout. + for version_transaction in &actual { + let checked_out = dataset + .checkout_version(version_transaction.version) + .await + .unwrap(); + assert_eq!( + version_transaction.transaction, + checked_out.read_transaction().await.unwrap() + ); + assert_eq!( + version_transaction.timestamp, + checked_out.version().timestamp + ); + assert!(version_transaction.transaction.is_some()); + } + + // A missing (e.g. cleaned up) version errors as DatasetNotFound, matching + // the historical checkout_version-based contract of the public API. + let err = dataset.read_version_transaction(9999).await.unwrap_err(); + assert!( + matches!(err, crate::Error::DatasetNotFound { .. }), + "expected DatasetNotFound for a missing version, got {err:?}" + ); +} + +#[tokio::test] +async fn test_read_transaction_recovers_from_stale_manifest_size() { + let test_uri = TempStrDir::default(); + let ds = write_versions(&test_uri, 1, true).await; + let manifest = ds.manifest().clone(); + // Only meaningful for the inline path; a plain write inlines the transaction. + assert!(manifest.transaction_section.is_some()); + + // A size at/under the transaction offset makes the first read_message fail + // "file size is too small"; only the retry at the true size can recover. + let mut stale = ds.manifest_location().clone(); + stale.size = Some(1); + let recovered = ds + .read_transaction_from_storage(&manifest, &stale) + .await + .unwrap(); + assert_eq!(recovered, ds.read_transaction().await.unwrap()); + assert!(recovered.is_some()); +} + +#[tokio::test] +async fn test_read_version_transaction_v1_manifest_naming() { + let test_uri = TempStrDir::default(); + let ds = write_versions(&test_uri, 3, false).await; + assert_eq!( + ds.manifest_location().naming_scheme, + ManifestNamingScheme::V1 + ); + + for version in 1..=3 { + let version_transaction = ds.read_version_transaction(version).await.unwrap(); + let checked_out = ds.checkout_version(version).await.unwrap(); + assert_eq!( + version_transaction.transaction, + checked_out.read_transaction().await.unwrap() + ); + assert_eq!( + version_transaction.timestamp, + checked_out.version().timestamp + ); + } +} + +#[tokio::test] +async fn test_read_version_transaction_on_branch() { + let test_uri = TempStrDir::default(); + let mut main_ds = write_versions(&test_uri, 1, true).await; + let branch_ds = main_ds.create_branch("dev", 1, None).await.unwrap(); + + // Commit on the branch. + let branch_ds = Dataset::write( + gen_rows(), + branch_ds.uri(), + Some(WriteParams { + mode: WriteMode::Append, + ..Default::default() + }), + ) + .await + .unwrap(); + assert_eq!(branch_ds.manifest().branch.as_deref(), Some("dev")); + + // Versions resolve against the branch chain and match a full checkout. + for version in branch_ds.versions().await.unwrap() { + let version_transaction = branch_ds + .read_version_transaction(version.version) + .await + .unwrap(); + assert_eq!(version_transaction.version, version.version); + assert_eq!(version_transaction.timestamp, version.timestamp); + let checked_out = branch_ds.checkout_version(version.version).await.unwrap(); + assert_eq!(checked_out.manifest().branch.as_deref(), Some("dev")); + assert_eq!( + version_transaction.transaction, + checked_out.read_transaction().await.unwrap() + ); + } + + // The append on the branch is the branch's own transaction. + let latest = branch_ds.version().version; + let version_transaction = branch_ds.read_version_transaction(latest).await.unwrap(); + assert!(matches!( + version_transaction.transaction, + Some(Transaction { + operation: Operation::Append { .. }, + .. + }) + )); } #[tokio::test] @@ -502,3 +729,162 @@ async fn test_list_detached_manifests() { assert_eq!(versions.len(), 1); assert_eq!(versions[0].version, 1); } + +/// Transaction properties large enough to push the transaction over the +/// inline threshold. +fn large_props(key: &str) -> Option>> { + use crate::io::commit::MAX_INLINE_TRANSACTION_BYTES; + let mut props = HashMap::new(); + props.insert( + key.to_string(), + "x".repeat(2 * MAX_INLINE_TRANSACTION_BYTES), + ); + Some(Arc::new(props)) +} + +fn spill_test_batch() -> (Arc, RecordBatch) { + let schema = Arc::new(ArrowSchema::new(vec![ArrowField::new( + "i", + DataType::Int32, + false, + )])); + let batch = RecordBatch::try_new( + schema.clone(), + vec![Arc::new(Int32Array::from_iter_values(0..10))], + ) + .unwrap(); + (schema, batch) +} + +/// Load the dataset with a fresh session so assertions hit storage, not caches. +async fn reopen(uri: &str) -> Dataset { + DatasetBuilder::from_uri(uri) + .with_session(Arc::new(Session::default())) + .load() + .await + .unwrap() +} + +#[tokio::test] +async fn test_large_transaction_spills_to_external_file() { + use crate::io::commit::MAX_INLINE_TRANSACTION_BYTES; + + let (schema, batch) = spill_test_batch(); + let test_uri = TempStrDir::default(); + + // New-dataset commit path: a transaction too large to inline is written + // only to the external transaction file. + Dataset::write( + RecordBatchIterator::new([Ok(batch.clone())], schema.clone()), + &test_uri, + Some(WriteParams { + transaction_properties: large_props("payload"), + ..Default::default() + }), + ) + .await + .unwrap(); + let ds = reopen(&test_uri).await; + assert!(ds.manifest.transaction_section.is_none()); + assert!(matches!(ds.manifest.transaction_file.as_deref(), Some(f) if !f.is_empty())); + let tx = ds.read_transaction().await.unwrap().unwrap(); + assert_eq!( + tx.transaction_properties + .unwrap() + .get("payload") + .unwrap() + .len(), + 2 * MAX_INLINE_TRANSACTION_BYTES + ); + + // Normal commit path: a small append is still inlined. + Dataset::write( + RecordBatchIterator::new([Ok(batch)], schema), + &test_uri, + Some(WriteParams { + mode: WriteMode::Append, + ..Default::default() + }), + ) + .await + .unwrap(); + let ds = reopen(&test_uri).await; + assert!(ds.manifest.transaction_section.is_some()); +} + +#[tokio::test] +async fn test_spilled_restore_and_deep_clone_read_own_transaction() { + // Restore and deep clone both rebuild the new manifest from an existing + // manifest file, inheriting its inline transaction offset and external + // transaction file name. When the new transaction is too large to inline, + // readers fall back to exactly those fields, so the stale inherited values + // must not leak into the new manifest. + use crate::dataset::transaction::TransactionBuilder; + + let (schema, batch) = spill_test_batch(); + let source_uri = TempStrDir::default(); + Dataset::write( + RecordBatchIterator::new([Ok(batch.clone())], schema.clone()), + &source_uri, + None, + ) + .await + .unwrap(); + let ds = Dataset::write( + RecordBatchIterator::new([Ok(batch)], schema), + &source_uri, + Some(WriteParams { + mode: WriteMode::Append, + ..Default::default() + }), + ) + .await + .unwrap(); + // The manifests restored from / cloned from below carry an inline + // transaction offset and their own transaction file. + assert!(ds.manifest.transaction_section.is_some()); + let source_version = ds.manifest().version; + let source_ref_path = ds.uri().to_string(); + let source_tx_file = ds.manifest.transaction_file.clone(); + assert!(matches!(source_tx_file.as_deref(), Some(f) if !f.is_empty())); + + // Restore with a spilled transaction: the stale inline offset must be + // cleared and the transaction read back from the external file. + let restore_tx = TransactionBuilder::new(source_version, Operation::Restore { version: 1 }) + .transaction_properties(large_props("payload")) + .build(); + CommitBuilder::new(Arc::new(ds)) + .execute(restore_tx.clone()) + .await + .unwrap(); + let restored = reopen(&source_uri).await; + assert!(restored.manifest.transaction_section.is_none()); + assert_eq!( + restored.read_transaction().await.unwrap().unwrap(), + restore_tx + ); + + // Deep clone with a spilled transaction: the manifest must reference the + // clone's own transaction file, not the source's. + let clone_tx = TransactionBuilder::new( + source_version, + Operation::Clone { + is_shallow: false, + ref_name: None, + ref_version: source_version, + ref_path: source_ref_path, + branch_name: None, + }, + ) + .transaction_properties(large_props("payload")) + .build(); + let clone_uri = TempStrDir::default(); + CommitBuilder::new(&clone_uri) + .execute(clone_tx.clone()) + .await + .unwrap(); + let cloned = reopen(&clone_uri).await; + assert!(cloned.manifest.transaction_section.is_none()); + assert_ne!(cloned.manifest.transaction_file, source_tx_file); + assert_eq!(cloned.read_transaction().await.unwrap().unwrap(), clone_tx); +} diff --git a/rust/lance/src/dataset/tests/dataset_versioning.rs b/rust/lance/src/dataset/tests/dataset_versioning.rs index c04dd0f3183..c95e3a6d250 100644 --- a/rust/lance/src/dataset/tests/dataset_versioning.rs +++ b/rust/lance/src/dataset/tests/dataset_versioning.rs @@ -1,6 +1,7 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright The Lance Authors +use std::collections::HashMap; use std::sync::Arc; use std::vec; @@ -22,6 +23,7 @@ use lance_file::version::LanceFileVersion; use mock_instant::thread_local::MockClock; use crate::dataset::refs::branch_contents_path; +use crate::utils::test::copy_test_data_to_tmp; use futures::TryStreamExt; use lance_core::Error; use object_store::path::Path; @@ -636,6 +638,152 @@ async fn test_fragment_id_never_reset() { assert_eq!(dataset.manifest.max_fragment_id(), Some(4)); } +#[tokio::test] +async fn test_overwrite_does_not_reuse_fragment_ids() { + // An overwrite replaces every fragment, but the ids it hands out must still + // continue from the dataset's high water mark: an id that named one set of + // rows must never name another. + let test_dir = TempStrDir::default(); + let write = |mode: WriteMode, rows: u64| { + let reader = gen_batch() + .col("i", array::step::()) + .into_reader_rows(RowCount::from(rows), BatchCount::from(1)); + let params = WriteParams { + mode, + max_rows_per_file: 10, + ..Default::default() + }; + Dataset::write(reader, test_dir.as_str(), Some(params)) + }; + let fragment_ids = |dataset: &Dataset| { + dataset + .get_fragments() + .iter() + .map(|f| f.id()) + .collect::>() + }; + + let dataset = write(WriteMode::Create, 30).await.unwrap(); + assert_eq!(fragment_ids(&dataset), vec![0, 1, 2]); + + let dataset = write(WriteMode::Overwrite, 20).await.unwrap(); + assert_eq!(fragment_ids(&dataset), vec![3, 4]); + assert_eq!(dataset.manifest.max_fragment_id(), Some(4)); + + let dataset = write(WriteMode::Append, 10).await.unwrap(); + assert_eq!(fragment_ids(&dataset), vec![3, 4, 5]); +} + +#[rstest] +#[tokio::test] +async fn test_overwrite_rejects_fragment_with_deletion_file( + // Every form of the operation must be rejected: upserting config alongside + // the overwrite does not make renumbering the fragment any safer. + #[values(None, Some(HashMap::from([("key".to_string(), "value".to_string())])))] + config_upsert_values: Option>, +) { + // A deletion file's path embeds the fragment id, so it cannot follow its + // fragment to the fresh id an overwrite assigns. Such a fragment belongs to + // the dataset being replaced, so the operation is rejected rather than + // silently losing the deletions. + let test_dir = TempStrDir::default(); + let reader = gen_batch() + .col("i", array::step::()) + .into_reader_rows(RowCount::from(10), BatchCount::from(1)); + let mut dataset = Dataset::write(reader, test_dir.as_str(), None) + .await + .unwrap(); + dataset.delete("i < 3").await.unwrap(); + + let fragment = dataset.manifest.fragments[0].clone(); + assert!(fragment.deletion_file.is_some()); + let schema = dataset.schema().clone(); + let read_version = dataset.manifest.version; + + let err = CommitBuilder::new(Arc::new(dataset)) + .execute(Transaction::new( + read_version, + Operation::Overwrite { + fragments: vec![fragment], + schema, + config_upsert_values, + initial_bases: None, + }, + None, + )) + .await + .unwrap_err(); + assert!( + matches!(err, Error::InvalidInput { .. }), + "expected invalid input, got {err:?}" + ); + assert!( + err.to_string().contains("must be newly written"), + "unexpected message: {err}" + ); +} + +#[tokio::test] +async fn test_commit_rejects_duplicate_fragment_ids() { + // Append honors an id a fragment arrives with, so a caller can still hand in + // one that an existing fragment already uses. Committing that would leave two + // fragments sharing everything keyed by the id. + let test_dir = TempStrDir::default(); + let write = |mode: WriteMode| { + let reader = gen_batch() + .col("i", array::step::()) + .into_reader_rows(RowCount::from(10), BatchCount::from(1)); + let params = WriteParams { + mode, + max_rows_per_file: 5, + ..Default::default() + }; + Dataset::write(reader, test_dir.as_str(), Some(params)) + }; + let dataset = write(WriteMode::Create).await.unwrap(); + let existing = dataset.manifest.fragments[1].clone(); + assert_eq!(existing.id, 1); + + let err = CommitBuilder::new(Arc::new(dataset)) + .execute(Transaction::new( + 1, + Operation::Append { + fragments: vec![existing], + }, + None, + )) + .await + .unwrap_err(); + assert!( + err.to_string().contains("two fragments with id 1"), + "unexpected message: {err}" + ); +} + +#[tokio::test] +async fn test_commit_on_dataset_with_mixed_file_versions() { + // A v0.16 dataset that has both v1 and v2 files also has two fragments with + // id 1, because the id allocation of that era could hand out an id a caller + // had already supplied. The mixture is the more actionable diagnosis, so the + // duplicate check must not preempt it. + let test_dir = copy_test_data_to_tmp("v0.16.0/wrong_data_version_no_fix.lance").unwrap(); + let mut dataset = Dataset::open(&test_dir.path_str()).await.unwrap(); + let ids = dataset + .manifest + .fragments + .iter() + .map(|f| f.id) + .collect::>(); + assert_eq!(ids, vec![0, 1, 1, 2]); + + let err = dataset.delete("false").await.unwrap_err(); + assert!( + err.to_string() + .contains("The dataset contains a mixture of file versions"), + "unexpected message: {err}" + ); +} + /// create_branch and shallow_clone must read the SOURCE ref's chain, not the /// receiver's. Both chains get a version 2 with diverged row counts so a clone /// that wrongly resolves the version under the receiver succeeds silently with @@ -695,6 +843,30 @@ async fn test_create_branch_and_shallow_clone_from_other_branch() { ); } +#[tokio::test] +async fn test_cannot_delete_branch_referenced_by_tag() { + let tempdir = TempDir::default(); + let test_uri = tempdir.path_str(); + let data = gen_batch() + .col("id", array::step::()) + .into_reader_rows(RowCount::from(1), BatchCount::from(1)); + let mut dataset = Dataset::write(data, &test_uri, None).await.unwrap(); + let branch = dataset + .create_branch("dev", /*version=*/ 1, /*store_params=*/ None) + .await + .unwrap(); + dataset + .tags() + .create("keep-dev", ("dev", branch.version().version)) + .await + .unwrap(); + + let error = dataset.delete_branch("dev").await.unwrap_err(); + assert!(matches!(&error, Error::RefConflict { .. })); + assert!(error.to_string().contains("keep-dev")); + dataset.checkout_version("keep-dev").await.unwrap(); +} + #[tokio::test] async fn test_branch() { let tempdir = TempDir::default(); @@ -1026,6 +1198,7 @@ async fn test_branch() { let cleaned_path = Path::parse(format!("{}/tree/feature", test_uri)).unwrap(); assert!(!dataset.object_store.exists(&cleaned_path).await.unwrap()); + dataset.tags().delete("tag1").await.unwrap(); dataset.delete_branch("dev/branch2").await.unwrap(); dataset.delete_branch("branch1").await.unwrap(); diff --git a/rust/lance/src/dataset/tests/fragment_validate_tombstones.rs b/rust/lance/src/dataset/tests/fragment_validate_tombstones.rs new file mode 100644 index 00000000000..41c672ff5e6 --- /dev/null +++ b/rust/lance/src/dataset/tests/fragment_validate_tombstones.rs @@ -0,0 +1,60 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright The Lance Authors + +use std::sync::Arc; + +use arrow_array::{RecordBatchIterator, record_batch}; +use lance_file::version::LanceFileVersion; +use rstest::rstest; + +use crate::Dataset; +use crate::dataset::WriteParams; +use crate::dataset::fragment::FileFragment; + +/// Rewriting a column with `update_columns` tombstones the field in the file +/// that held it, leaving `-2` behind while a new file answers for it. +/// Validation has to accept that: a tombstone marks a superseded field, not a +/// corrupt one. Legacy files are checked by a separate sort rule, so both +/// storage versions are covered. +#[rstest] +#[case::legacy(LanceFileVersion::Legacy)] +#[case::stable(LanceFileVersion::Stable)] +#[tokio::test] +async fn test_validate_accepts_tombstoned_fields(#[case] version: LanceFileVersion) { + let batch = record_batch!(("i", Int32, [1, 2]), ("v", Int64, [10, 20])).unwrap(); + let reader = RecordBatchIterator::new(vec![Ok(batch.clone())], batch.schema()); + let dataset = Arc::new( + Dataset::write( + reader, + "memory://", + Some(WriteParams { + data_storage_version: Some(version), + ..Default::default() + }), + ) + .await + .unwrap(), + ); + + let update = record_batch!(("i1", Int32, [1, 2]), ("v", Int64, [99, 99])).unwrap(); + let right = RecordBatchIterator::new(vec![Ok(update.clone())], update.schema()); + + let mut fragment = dataset.get_fragments().into_iter().next().unwrap(); + let updated = fragment + .update_columns_with_offsets(right, "i", "i1") + .await + .unwrap(); + + let layout: Vec> = updated + .fragment + .files + .iter() + .map(|file| file.fields.as_ref().to_vec()) + .collect(); + assert_eq!(layout, vec![vec![0, -2], vec![1]]); + + FileFragment::new(dataset, updated.fragment) + .validate() + .await + .unwrap(); +} diff --git a/rust/lance/src/dataset/tests/mod.rs b/rust/lance/src/dataset/tests/mod.rs index ecc64587b0c..2c3aa203e15 100644 --- a/rust/lance/src/dataset/tests/mod.rs +++ b/rust/lance/src/dataset/tests/mod.rs @@ -11,7 +11,9 @@ mod dataset_index; mod dataset_io; mod dataset_merge_update; mod dataset_migrations; +mod dataset_overlay_index_masking; mod dataset_scanner; mod dataset_schema_evolution; mod dataset_transactions; mod dataset_versioning; +mod fragment_validate_tombstones; diff --git a/rust/lance/src/dataset/transaction.rs b/rust/lance/src/dataset/transaction.rs index 3261f9300c4..92611347485 100644 --- a/rust/lance/src/dataset/transaction.rs +++ b/rust/lance/src/dataset/transaction.rs @@ -14,8 +14,12 @@ use super::ManifestWriteConfig; use super::write::merge_insert::inserted_rows::KeyExistenceFilter; +use crate::dataset::overlay::collect_overlay_stale_frags; use crate::dataset::transaction::UpdateMode::{RewriteColumns, RewriteRows}; -use crate::index::mem_wal::update_mem_wal_index_merged_generations; +use crate::index::index_results_are_row_addrs; +use crate::index::mem_wal::{ + load_mem_wal_index_details, new_mem_wal_index_meta, update_mem_wal_index_compacted_sstables, +}; use crate::utils::temporal::timestamp_to_nanos; use lance_core::datatypes::{ LANCE_UNENFORCED_CLUSTERING_KEY_POSITION, LANCE_UNENFORCED_PRIMARY_KEY, @@ -23,19 +27,24 @@ use lance_core::datatypes::{ }; use lance_core::deepsize::DeepSizeOf; use lance_core::{Error, Result, datatypes::Schema}; -use lance_file::{datatypes::Fields, version::LanceFileVersion}; -use lance_index::mem_wal::MergedGeneration; +use lance_file::{datatypes::Fields, version::ConcreteFileVersion}; +use lance_index::mem_wal::{CompactedSsTable, IndexCatchupProgress, MEM_WAL_INDEX_NAME}; use lance_index::{frag_reuse::FRAG_REUSE_INDEX_NAME, is_system_index}; use lance_io::object_store::ObjectStore; -use lance_table::feature_flags::{FLAG_STABLE_ROW_IDS, apply_feature_flags}; +use lance_table::feature_flags::{ + FLAG_MEM_WAL_INDEX_CATCHUP, FLAG_STABLE_ROW_IDS, apply_feature_flags, + inherit_mem_wal_index_catchup, validate_mem_wal_index_catchup_flags, +}; use lance_table::rowids::read_row_ids; use lance_table::{ format::{ BasePath, DataFile, DataStorageFormat, Fragment, IndexFile, IndexMetadata, Manifest, - RowDatasetVersionMeta, RowDatasetVersionRun, RowDatasetVersionSequence, RowIdMeta, pb, + RowDatasetVersionMeta, RowDatasetVersionRun, RowDatasetVersionSequence, RowIdMeta, + overlay::DataOverlayFile, pb, }, io::{ commit::CommitHandler, + deletion::relative_deletion_file_path, manifest::{read_manifest, read_manifest_indexes}, }, rowids::{RowIdSequence, segment::U64Segment, version::build_version_meta, write_row_ids}, @@ -44,7 +53,7 @@ use object_store::path::Path; use roaring::RoaringBitmap; use std::cmp::Ordering; use std::{ - collections::{HashMap, HashSet}, + collections::{BTreeMap, HashMap, HashSet}, sync::Arc, }; use uuid::Uuid; @@ -258,6 +267,17 @@ pub struct Transaction { #[derive(Debug, Clone, DeepSizeOf, PartialEq)] pub struct DataReplacementGroup(pub u64, pub DataFile); +/// Overlay files to append to a single fragment, in order (the last entry is +/// newest). The overlays are appended to the fragment's existing `overlays` +/// list rather than replacing it, so overlays written by concurrent commits are +/// preserved. Each overlay's `committed_version` is stamped to the new dataset +/// version at commit time (re-stamped on retry). +#[derive(Debug, Clone, DeepSizeOf, PartialEq)] +pub struct DataOverlayGroup { + pub fragment_id: u64, + pub overlays: Vec, +} + /// An entry for a map update. If value is None, the key will be removed from the map. #[derive(Debug, Clone, DeepSizeOf, PartialEq)] pub struct UpdateMapEntry { @@ -303,6 +323,48 @@ pub struct UpdateMap { pub replace: bool, } +/// Non-system logical index name -> its physical segments, ordered by UUID. +/// +/// Whole segment metadata rather than UUIDs alone: operations such as `Rewrite` +/// prune a segment's fragment bitmap while keeping its UUID, so a UUID-only +/// comparison would keep coverage for an index that no longer spans the same +/// base fragments. +type LogicalIndexSegments = BTreeMap>; + +/// What one physical index segment contributes to coverage. +/// +/// Deliberately not the whole [`IndexMetadata`]. It rests on one contract: +/// changing an index's physical contents mints a new UUID. Of the mutations +/// sanctioned under an existing UUID, only the fragment bitmap changes which +/// rows the index answers for -- an `Update` prunes it in place, and +/// `migrate_indices` recalculates it -- so the UUID alone is not enough and the +/// bitmap has to be compared too. The rest of the metadata, file lists and +/// timestamps and inferred details, is filled in by migrations routinely; +/// comparing it would withdraw coverage for no reason. +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) struct CoverageIdentity { + uuid: Uuid, + fragment_bitmap: Option, +} + +/// The version a transaction read, as the coverage derivation needs it. +/// +/// An index covering every fragment live at this version holds every row +/// compaction had copied into the base table by then, so it is caught up to +/// that version's `compacted_sstables`. That is the only proof available: +/// nothing maps a compaction generation to the fragments its rows landed in. +/// +/// `read_version` is fixed for the life of a transaction and survives rebase, +/// so the credit a commit can prove is stable across attempts. The recorded +/// result may still differ between attempts, because a rebased attempt sees a +/// different head: other commits move the compacted generations and the +/// positions already recorded. +#[derive(Debug, Clone, Copy)] +pub(crate) struct ReadVersionState<'a> { + pub manifest: &'a Manifest, + pub indices: &'a [IndexMetadata], +} + /// An operation on a dataset. #[derive(Debug, Clone, DeepSizeOf)] pub enum Operation { @@ -319,6 +381,16 @@ pub enum Operation { }, /// Overwrite the entire dataset with the given fragments. This is also /// used when initially creating a table. + /// + /// The fragments are newly written ones and are assigned fresh ids at commit + /// time, continuing from the dataset's highest id ever used; the ids they + /// arrive with are ignored. + /// + /// A fragment carrying a deletion file is rejected. A deletion file's path + /// embeds the fragment id, so it cannot follow its fragment to the new id: + /// minting a fragment and giving it a deletion file are mutually exclusive in + /// one transaction. Use [`Self::Delete`] to commit deletions against existing + /// fragments, or [`Self::Merge`] to change their schema. Overwrite { fragments: Vec, schema: Schema, @@ -367,12 +439,23 @@ pub enum Operation { DataReplacement { replacements: Vec, }, + /// Attach overlay files to fragments, supplying new values for a subset of + /// `(physical offset, field)` cells without rewriting the fragments' base + /// data files. See [`DataOverlayFile`] and the Data Overlay Files + /// specification for resolution, coverage, and versioning rules. + DataOverlay { groups: Vec }, /// Merge a new column in /// 'fragments' is the final fragments include all data files, the new fragments must align with old ones at rows. /// 'schema' is not forced to include existed columns, which means we could use Merge to drop column data Merge { fragments: Vec, schema: Schema, + /// Set when this merge makes no nullability-affecting schema change: + /// it introduces no field that data staged against an earlier schema + /// could not safely omit. Without the assertion the merge conflicts + /// with concurrent appends in either commit order, since a stale + /// append omits new columns entirely and its rows read as null. + preserves_nullability: bool, }, /// Restore an old version of the database Restore { version: u64 }, @@ -407,8 +490,8 @@ pub enum Operation { new_fragments: Vec, /// The fields that have been modified fields_modified: Vec, - /// List of MemWAL region generations to mark as merged after this transaction - merged_generations: Vec, + /// MemWAL SSTables to mark as compacted after this transaction. + compacted_sstables: Vec, /// The fields that used to judge whether to preserve the new frag's id into /// the frag bitmap of the specified indices. fields_for_preserving_frag_bitmap: Vec, @@ -422,8 +505,16 @@ pub enum Operation { updated_fragment_offsets: Option, }, - /// Project to a new schema. This only changes the schema, not the data. - Project { schema: Schema }, + /// Project to a new schema. + Project { + schema: Schema, + /// Set when this projection makes no nullability-affecting schema + /// change, as a rename or a drop does not. A nullability tightening + /// must not set this: its producer proved the claim by scanning at its + /// read version, so a concurrent write can falsify it and the + /// projection conflicts with value-writes in either commit order. + preserves_nullability: bool, + }, /// Update the dataset configuration. UpdateConfig { @@ -432,11 +523,18 @@ pub enum Operation { schema_metadata_updates: Option, field_metadata_updates: HashMap, }, - /// Update merged generations in MemWAL index. + /// Update SSTable compaction progress in the MemWAL index. + /// /// This is used during merge-insert to atomically record which - /// generations have been merged to the base table. + /// SSTables have been compacted into the base table. UpdateMemWalState { - merged_generations: Vec, + compacted_sstables: Vec, + /// Requests the one-way migration to required index catch-up. + /// + /// One-way because returning to legacy semantics — where a missing + /// coverage entry reads as "fully caught up" — is unsafe once any + /// SSTable has been retired against a recorded catch-up position. + require_index_catchup: bool, }, /// Clone a dataset. @@ -499,6 +597,7 @@ impl std::fmt::Display for Operation { Self::Project { .. } => write!(f, "Project"), Self::UpdateConfig { .. } => write!(f, "UpdateConfig"), Self::DataReplacement { .. } => write!(f, "DataReplacement"), + Self::DataOverlay { .. } => write!(f, "DataOverlay"), Self::Clone { .. } => write!(f, "Clone"), Self::UpdateMemWalState { .. } => write!(f, "UpdateMemWalState"), Self::UpdateBases { .. } => write!(f, "UpdateBases"), @@ -614,12 +713,18 @@ impl PartialEq for Operation { Self::Merge { fragments: a_fragments, schema: a_schema, + preserves_nullability: a_preserves, }, Self::Merge { fragments: b_fragments, schema: b_schema, + preserves_nullability: b_preserves, }, - ) => compare_vec(a_fragments, b_fragments) && a_schema == b_schema, + ) => { + compare_vec(a_fragments, b_fragments) + && a_schema == b_schema + && a_preserves == b_preserves + } (Self::Restore { version: a }, Self::Restore { version: b }) => a == b, ( Self::ReserveFragments { num_fragments: a }, @@ -631,7 +736,7 @@ impl PartialEq for Operation { updated_fragments: a_updated, new_fragments: a_new, fields_modified: a_fields, - merged_generations: a_merged_generations, + compacted_sstables: a_compacted_sstables, fields_for_preserving_frag_bitmap: a_fields_for_preserving_frag_bitmap, update_mode: a_update_mode, inserted_rows_filter: a_inserted_rows_filter, @@ -642,7 +747,7 @@ impl PartialEq for Operation { updated_fragments: b_updated, new_fragments: b_new, fields_modified: b_fields, - merged_generations: b_merged_generations, + compacted_sstables: b_compacted_sstables, fields_for_preserving_frag_bitmap: b_fields_for_preserving_frag_bitmap, update_mode: b_update_mode, inserted_rows_filter: b_inserted_rows_filter, @@ -653,7 +758,7 @@ impl PartialEq for Operation { && compare_vec(a_updated, b_updated) && compare_vec(a_new, b_new) && compare_vec(a_fields, b_fields) - && compare_vec(a_merged_generations, b_merged_generations) + && compare_vec(a_compacted_sstables, b_compacted_sstables) && compare_vec( a_fields_for_preserving_frag_bitmap, b_fields_for_preserving_frag_bitmap, @@ -662,7 +767,16 @@ impl PartialEq for Operation { && a_inserted_rows_filter == b_inserted_rows_filter && a_updated_fragment_offsets == b_updated_fragment_offsets } - (Self::Project { schema: a }, Self::Project { schema: b }) => a == b, + ( + Self::Project { + schema: a, + preserves_nullability: a_preserves, + }, + Self::Project { + schema: b, + preserves_nullability: b_preserves, + }, + ) => a == b && a_preserves == b_preserves, ( Self::UpdateConfig { config_updates: a_config, @@ -1210,12 +1324,14 @@ impl PartialEq for Operation { } ( Self::UpdateMemWalState { - merged_generations: a_merged, + compacted_sstables: a_compacted, + require_index_catchup: a_activate, }, Self::UpdateMemWalState { - merged_generations: b_merged, + compacted_sstables: b_compacted, + require_index_catchup: b_activate, }, - ) => compare_vec(a_merged, b_merged), + ) => compare_vec(a_compacted, b_compacted) && a_activate == b_activate, (Self::Clone { .. }, Self::Append { .. }) => { std::mem::discriminant(self) == std::mem::discriminant(other) } @@ -1345,6 +1461,8 @@ impl PartialEq for Operation { (Self::Clone { .. }, Self::UpdateBases { .. }) => { std::mem::discriminant(self) == std::mem::discriminant(other) } + (Self::DataOverlay { groups: a }, Self::DataOverlay { groups: b }) => compare_vec(a, b), + (Self::DataOverlay { .. }, _) | (_, Self::DataOverlay { .. }) => false, } } } @@ -1521,6 +1639,7 @@ impl Operation { Self::Project { .. } => "Project", Self::UpdateConfig { .. } => "UpdateConfig", Self::DataReplacement { .. } => "DataReplacement", + Self::DataOverlay { .. } => "DataOverlay", Self::UpdateMemWalState { .. } => "UpdateMemWalState", Self::Clone { .. } => "Clone", Self::UpdateBases { .. } => "UpdateBases", @@ -1714,7 +1833,7 @@ impl Transaction { fn data_storage_format_from_files( fragments: &[Fragment], - user_requested: Option, + user_requested: Option, ) -> Result { if let Some(file_version) = Fragment::try_infer_version(fragments)? { // Ensure user-requested matches data files @@ -1748,15 +1867,333 @@ impl Transaction { .resolve_version_location(base_path, version, &object_store.inner) .await?; let mut manifest = read_manifest(object_store, &location.path, location.size).await?; + // Read below the reader validation boundary, so nothing else refuses a + // half-set manifest here: the flag reset would quietly drop the lone bit + // and republish an undefined state as legacy. + validate_mem_wal_index_catchup_flags(&manifest)?; manifest.set_timestamp(timestamp_to_nanos(config.timestamp)); manifest.transaction_file = Some(tx_path.to_string()); let indices = read_manifest_indexes(object_store, &location, &manifest).await?; manifest.max_fragment_id = manifest .max_fragment_id .max(current_manifest.max_fragment_id); + // A version from before catch-up was required carries MemWAL state this + // protocol never validated -- catch-up values activation deliberately + // cleared, or compaction progress it deliberately refused to trust. + // Keeping the bit would republish those as if this protocol had recorded + // them. Refuse instead: sanitizing is not possible here, because both + // fields would have to be re-derived from data Lance cannot see. + let current_requires = current_manifest.reader_feature_flags & FLAG_MEM_WAL_INDEX_CATCHUP + != 0 + || current_manifest.writer_feature_flags & FLAG_MEM_WAL_INDEX_CATCHUP != 0; + let restored_requires = manifest.reader_feature_flags & FLAG_MEM_WAL_INDEX_CATCHUP != 0 + && manifest.writer_feature_flags & FLAG_MEM_WAL_INDEX_CATCHUP != 0; + if current_requires && !restored_requires { + return Err(Error::invalid_input(format!( + "Cannot restore version {version}: this table requires MemWAL index \ + catch-up and that version predates it, so its recorded catch-up and \ + compaction progress were never validated by this protocol" + ))); + } + inherit_mem_wal_index_catchup(&mut manifest, current_manifest)?; Ok((manifest, indices)) } + /// Require index catch-up on a table that has never required it. + /// + /// One-way, because returning to legacy semantics -- where a missing + /// coverage entry reads as "fully caught up" -- is unsafe once any SSTable + /// has been retired against a recorded catch-up position. + fn require_index_catchup(final_indices: &mut [IndexMetadata], new_version: u64) -> Result<()> { + let Some(pos) = final_indices + .iter() + .position(|idx| idx.name == MEM_WAL_INDEX_NAME) + else { + return Err(Error::invalid_input(format!( + "Cannot require MemWAL index catch-up: the {} system index does \ + not exist on this table", + MEM_WAL_INDEX_NAME + ))); + }; + + let mut details = load_mem_wal_index_details(final_indices[pos].clone())?; + + // The beta protocol wrote compaction progress that was never an active + // retirement record, and Lance cannot check those numbers against WAL + // shard manifests. Trusting them would let the first trim after + // activation delete SSTables no commit copied in, so a table carrying + // them must be drained through an explicit migration instead. + if !details.compacted_sstables.is_empty() { + return Err(Error::invalid_input( + "Cannot require MemWAL index catch-up: the table already records \ + SSTable compaction progress from the beta protocol, which cannot \ + be validated. Drain or reset the table first.", + )); + } + + // Beta coverage was written under rules this protocol does not enforce, + // so it is not trustworthy. Left in place, a later compaction would find it + // already satisfied and could retire an SSTable that no index covers. + if details.index_catchup.is_empty() { + return Ok(()); + } + details.index_catchup.clear(); + final_indices[pos] = new_mem_wal_index_meta(new_version, details)?; + Ok(()) + } + + /// Every non-system logical index, mapped to what determines its coverage. + /// + /// A logical index may be backed by several physical segments, so "did this + /// index change" is a question about the whole set. Sorted by UUID so the + /// two sides compare positionally. + pub(crate) fn logical_index_segments(indices: &[IndexMetadata]) -> LogicalIndexSegments { + let mut by_name: LogicalIndexSegments = BTreeMap::new(); + for idx in indices.iter().filter(|idx| !is_system_index(idx)) { + by_name + .entry(idx.name.clone()) + .or_default() + .push(CoverageIdentity { + uuid: idx.uuid, + fragment_bitmap: idx.fragment_bitmap.clone(), + }); + } + for segments in by_name.values_mut() { + segments.sort_unstable_by_key(|segment| segment.uuid); + } + by_name + } + + /// Apply MemWAL index-coverage rules once the final index list is known. + /// + /// Coverage records that a base-table index contains the rows a compaction + /// copied in, and the WAL pod retires SSTables against it. + /// + /// It is derived, not reported. An index covering every fragment live at the + /// transaction's read version holds every row compaction had copied in by + /// then, so it is caught up to that version's `compacted_sstables`. That is + /// the only proof available: nothing maps a generation to the fragments its + /// rows landed in, so covering the table as the transaction read it is how + /// an index shows it covered those rows. Fragments appended since are a + /// later gap. + /// + /// Deriving rather than transmitting means no claim can go stale between + /// inspection and commit, the answer survives rebase (`read_version` is + /// fixed for a transaction's life), and any operation can earn coverage -- + /// an ordinary reindex that fully covers no longer has to throw its work + /// away and wait for a repair. + /// + /// Only meaningful once catch-up is required, where a missing entry means + /// "not caught up" and the SSTables stay. A legacy table reads a missing + /// entry as "fully caught up", so this leaves it untouched rather than + /// making the table look more covered than it is. + pub(crate) fn apply_mem_wal_index_coverage( + final_indices: &mut [IndexMetadata], + segments_before: &LogicalIndexSegments, + read_version_state: Option>, + index_catchup_required: bool, + new_version: u64, + ) -> Result<()> { + if !index_catchup_required { + return Ok(()); + } + + let Some(pos) = final_indices + .iter() + .position(|idx| idx.name == MEM_WAL_INDEX_NAME) + else { + // The system index went away with this transaction (MemWAL disable, + // or an overwrite). There is no coverage left to maintain. + return Ok(()); + }; + + let mut details = load_mem_wal_index_details(final_indices[pos].clone())?; + + // Nothing has ever been compacted, so no index can be behind and there + // is no coverage to invalidate. + if details.compacted_sstables.is_empty() && details.index_catchup.is_empty() { + return Ok(()); + } + + let segments_after = Self::logical_index_segments(final_indices); + let catchup_before = std::mem::take(&mut details.index_catchup); + + // Per shard: what this commit records as compacted, and the most the + // read version may credit. Generations compacted after that read landed + // in fragments no index under consideration has seen; the committed + // value caps it in turn, so a read version since rolled back cannot + // retire SSTables no live commit copied in. + let read_details = read_version_state + .map(|state| { + state + .indices + .iter() + .find(|idx| idx.name == MEM_WAL_INDEX_NAME) + .cloned() + .map(load_mem_wal_index_details) + .transpose() + }) + .transpose()? + .flatten(); + let shards: Vec<(Uuid, u64, u64)> = details + .compacted_sstables + .iter() + .map(|committed| { + let at_read = read_details + .as_ref() + .and_then(|read| { + read.compacted_sstables + .iter() + .find(|s| s.shard_id == committed.shard_id) + }) + .map_or(0, |s| s.generation); + ( + committed.shard_id, + committed.generation, + at_read.min(committed.generation), + ) + }) + .collect(); + + // Every fragment live when the transaction read the table. An index + // spanning all of them holds every row compacted by then. + let read_fragments: Option = read_version_state.map(|state| { + state + .manifest + .fragments + .iter() + .map(|fragment| fragment.id as u32) + .collect() + }); + + let covers_read_version = |segments: &[CoverageIdentity]| -> bool { + let Some(required) = read_fragments.as_ref() else { + return false; + }; + if required.is_empty() { + // Subset-of-empty is trivially true, so this would credit every + // index on a table with no fragments. Refused because an empty + // fragment list is not only what an emptied table looks like: + // it is also what a manifest written before #8438 looks like, + // where UpdateMemWalState published no fragments at all. On + // such a table the SSTables are the last copy of those rows, + // and crediting coverage would retire them. The cost is that a + // genuinely emptied table keeps its SSTables. + return false; + } + let mut covered = RoaringBitmap::new(); + for segment in segments { + match segment.fragment_bitmap.as_ref() { + Some(bitmap) => covered |= bitmap, + // An unknown bitmap cannot be shown to cover anything. + None => return false, + } + } + required.is_subset(&covered) + }; + + let mut rebuilt: Vec = Vec::new(); + for (name, after) in segments_after.iter() { + // Compared by [`CoverageIdentity`], not segment UUID: an Update + // that touches an indexed field prunes a segment's fragment bitmap + // in place while keeping its UUID, so a UUID-only comparison would + // carry a position forward that the index no longer earns. + let unchanged = segments_before.get(name) == Some(after); + let carried = unchanged + .then(|| catchup_before.iter().find(|e| e.index_name == *name)) + .flatten(); + let proven = covers_read_version(after); + + if carried.is_none() && !proven { + // Changed, and nothing shows the new index covers the read + // version. No entry: a missing one reads as "not caught up". + continue; + } + + let generations = shards + .iter() + .map(|&(shard_id, committed, creditable)| { + let prior = carried + .and_then(|entry| entry.caught_up_generation_for_shard(&shard_id)) + .unwrap_or(0); + let credited = if proven { creditable } else { 0 }; + // Takes the better of what this commit proves and what an + // unchanged index already held, so a commit reading an older + // version does not lower a position it cannot re-prove. The + // clamp is the exception: a position above what this commit + // records as compacted describes rows no live commit copied + // in. + CompactedSsTable::new(shard_id, prior.max(credited).min(committed)) + }) + .collect::>(); + if generations.iter().all(|g| g.generation == 0) { + continue; + } + rebuilt.push(IndexCatchupProgress::new(name.clone(), generations)); + } + rebuilt.sort_by(|a, b| a.index_name.cmp(&b.index_name)); + + let mut before_sorted = catchup_before; + before_sorted.sort_by(|a, b| a.index_name.cmp(&b.index_name)); + if rebuilt == before_sorted { + return Ok(()); + } + + let dropped: Vec<&str> = before_sorted + .iter() + .map(|e| e.index_name.as_str()) + .filter(|name| !rebuilt.iter().any(|kept| kept.index_name == *name)) + .collect(); + if !dropped.is_empty() { + // The first thing to check when SSTables stop becoming trimmable. + log::info!( + "MemWAL index catch-up invalidated at version {new_version} for {dropped:?}: \ + these indices changed and no longer cover the version this commit read" + ); + } + + details.index_catchup = rebuilt; + final_indices[pos] = new_mem_wal_index_meta(new_version, details)?; + Ok(()) + } + + /// Drop coverage for indices a post-`build_manifest` step narrowed. + /// + /// The derivation runs while the manifest is being built, but the index list + /// is not final there: `migrate_indices` can recalculate a segment's + /// fragment bitmap and keep its UUID, so an index can narrow after its + /// position was decided. It reports which ones it touched rather than the + /// caller re-snapshotting every bitmap to find out. Only ever removes. + pub(crate) fn withdraw_coverage_invalidated_after_build( + indices: &mut [IndexMetadata], + changed: &[String], + new_version: u64, + ) -> Result<()> { + if changed.is_empty() { + return Ok(()); + } + let Some(pos) = indices + .iter() + .position(|idx| idx.name == MEM_WAL_INDEX_NAME) + else { + return Ok(()); + }; + let mut details = load_mem_wal_index_details(indices[pos].clone())?; + let before = details.index_catchup.len(); + details + .index_catchup + .retain(|entry| !changed.contains(&entry.index_name)); + if details.index_catchup.len() == before { + return Ok(()); + } + log::info!( + "MemWAL index catch-up withdrawn at version {new_version} for {changed:?}: \ + these indices were recalculated after their coverage was derived" + ); + indices[pos] = new_mem_wal_index_meta(new_version, details)?; + Ok(()) + } + /// Create a new manifest from the current manifest and the transaction. /// /// `current_manifest` should only be None if the dataset does not yet exist. @@ -1766,6 +2203,29 @@ impl Transaction { current_indices: Vec, transaction_file_path: &str, config: &ManifestWriteConfig, + ) -> Result<(Manifest, Vec)> { + self.build_manifest_with_read_version( + current_manifest, + current_indices, + transaction_file_path, + config, + None, + ) + } + + /// [`Self::build_manifest`] with the version this transaction read. + /// + /// Supplied by the commit path, which already materializes that version. + /// `None` where there is none to read -- dataset creation and detached + /// commits -- in which case no index can be shown to cover it and coverage + /// is left as the invalidation rules put it. + pub(crate) fn build_manifest_with_read_version( + &self, + current_manifest: Option<&Manifest>, + current_indices: Vec, + transaction_file_path: &str, + config: &ManifestWriteConfig, + read_version_state: Option>, ) -> Result<(Manifest, Vec)> { if config.use_stable_row_ids && current_manifest @@ -1824,17 +2284,36 @@ impl Transaction { } }; - let mut fragment_id = if matches!(self.operation, Operation::Overwrite { .. }) { - 0 - } else { - current_manifest - .and_then(|m| m.max_fragment_id()) - .map(|id| id + 1) - .unwrap_or(0) - }; + // Fragment ids are a high water mark for the whole dataset history: an id + // must never name two different sets of rows, or per-fragment state keyed + // by id (caches, deletion files, row addresses) can be attributed to the + // wrong rows. + let mut fragment_id = current_manifest + .and_then(|m| m.max_fragment_id()) + .map(|id| id + 1) + .unwrap_or(0); let mut final_fragments = Vec::new(); let mut final_indices = current_indices; + // Both words must agree: a reader that keeps legacy semantics would read a + // missing entry as "fully caught up", so a half-set state is not safe mode. + let index_catchup_required = current_manifest + .map(|m| { + m.reader_feature_flags & FLAG_MEM_WAL_INDEX_CATCHUP != 0 + && m.writer_feature_flags & FLAG_MEM_WAL_INDEX_CATCHUP != 0 + }) + .unwrap_or(false); + + // Snapshot taken before the operation rewrites the list, so coverage can + // be compared against what each logical index looked like going in. Only + // tables in safe mode maintain coverage, so every other commit -- and the + // segment clones this costs -- pays nothing. + let mem_wal_segments_before = (index_catchup_required + && final_indices + .iter() + .any(|idx| idx.name == MEM_WAL_INDEX_NAME)) + .then(|| Self::logical_index_segments(&final_indices)); + let mut next_row_id = { // Only use row ids if the feature flag is set already or match (current_manifest, config.use_stable_row_ids) { @@ -1861,6 +2340,8 @@ impl Transaction { )) }); + let new_version = current_manifest.map_or(1, |m| m.version + 1); + match &self.operation { Operation::Clone { .. } => { return Err(Error::internal( @@ -1875,7 +2356,6 @@ impl Transaction { if let Some(next_row_id) = &mut next_row_id { Self::assign_row_ids(next_row_id, new_fragments.as_mut_slice())?; // Add version metadata for all new fragments - let new_version = current_manifest.map(|m| m.version + 1).unwrap_or(1); for fragment in new_fragments.iter_mut() { let version_meta = build_version_meta(fragment, new_version); fragment.last_updated_at_version_meta = version_meta.clone(); @@ -1890,13 +2370,15 @@ impl Transaction { .. } => { // Remove the deleted fragments + // Hash lookups keep this linear on tables with many fragments. + let deleted_ids: HashSet = deleted_fragment_ids.iter().copied().collect(); + let updated_by_id: HashMap = + updated_fragments.iter().map(|f| (f.id, f)).collect(); final_fragments.extend(maybe_existing_fragments?.clone()); - final_fragments.retain(|f| !deleted_fragment_ids.contains(&f.id)); + final_fragments.retain(|f| !deleted_ids.contains(&f.id)); final_fragments.iter_mut().for_each(|f| { - for updated in updated_fragments { - if updated.id == f.id { - *f = updated.clone(); - } + if let Some(updated) = updated_by_id.get(&f.id) { + *f = (*updated).clone(); } }); Self::retain_relevant_indices(&mut final_indices, &schema, &final_fragments) @@ -1906,7 +2388,7 @@ impl Transaction { updated_fragments, new_fragments, fields_modified, - merged_generations, + compacted_sstables, fields_for_preserving_frag_bitmap, update_mode, updated_fragment_offsets, @@ -1916,14 +2398,34 @@ impl Transaction { let existing_fragments = maybe_existing_fragments?; // Apply updates to existing fragments + // Hash lookups keep this linear on tables with many fragments. + let removed_ids: HashSet = removed_fragment_ids.iter().copied().collect(); + let mut updated_by_id: HashMap = + HashMap::with_capacity(updated_fragments.len()); + for fragment in updated_fragments { + updated_by_id.entry(fragment.id).or_insert(fragment); + } let updated_frags: Vec = existing_fragments .iter() .filter_map(|f| { - if removed_fragment_ids.contains(&f.id) { + if removed_ids.contains(&f.id) { return None; } - if let Some(updated) = updated_fragments.iter().find(|uf| uf.id == f.id) { - Some(updated.clone()) + if let Some(&updated) = updated_by_id.get(&f.id) { + let mut updated = updated.clone(); + // Carry forward the fragment's current overlays (which + // may include ones added by a concurrent commit). An + // in-place column rewrite then tombstones the overlaid + // fields it rewrote, since the fresh base values + // supersede them. + updated.overlays = f.overlays.clone(); + if matches!(update_mode, Some(RewriteColumns)) { + lance_table::format::overlay::tombstone_overlay_fields( + &mut updated.overlays, + fields_modified, + ); + } + Some(updated) } else { Some(f.clone()) } @@ -1945,12 +2447,18 @@ impl Transaction { && let Some(UpdatedFragmentOffsets(off_map)) = updated_fragment_offsets && !off_map.is_empty() { - let new_version = current_manifest.map(|m| m.version + 1).unwrap_or(1); let prev_version = current_manifest.map(|m| m.version).unwrap_or(0); for fragment in final_fragments.iter_mut() { let Some(bitmap) = off_map.get(&fragment.id) else { continue; }; + // Defense-in-depth: only stamp fragments that were actually + // rewritten. validate_operation enforces this invariant before + // build_manifest is called; this guard catches any path that + // bypasses validation. + if !updated_by_id.contains_key(&fragment.id) { + continue; + } if bitmap.is_empty() { continue; } @@ -1960,6 +2468,27 @@ impl Transaction { if fragment.last_updated_at_version_meta.is_none() { continue; } + let max_allowed = existing_fragments + .iter() + .find(|f| f.id == fragment.id) + .and_then(|f| f.physical_rows) + .unwrap_or(1 << 24); + if bitmap.len() as usize > max_allowed { + return Err(Error::invalid_input(format!( + "updatedFragmentOffsets cardinality {} exceeds fragment {} limit {}", + bitmap.len(), + fragment.id, + max_allowed + ))); + } + if let Some(max_off) = bitmap.max() + && max_off as usize >= max_allowed + { + return Err(Error::invalid_input(format!( + "updatedFragmentOffsets max offset {} exceeds fragment {} limit {}", + max_off, fragment.id, max_allowed + ))); + } let offsets: Vec = bitmap.iter().map(|o| o as usize).collect(); lance_table::rowids::version::refresh_row_latest_update_meta_for_partial_frag_rewrite_cols( fragment, @@ -1988,7 +2517,6 @@ impl Transaction { } if next_row_id.is_some() { - let new_version = current_manifest.map(|m| m.version + 1).unwrap_or(1); resolve_update_version_metadata( existing_fragments, new_fragments.as_mut_slice(), @@ -2010,12 +2538,27 @@ impl Transaction { .copied() .collect(); + // The original fragments that carried an overlay: their moved rows may have a + // stale index entry (see `register_pure_rewrite_rows_update_frags_in_indices`). + // Reuse the hash lookups built above instead of scanning + // `original_fragment_ids` per fragment. + let original_overlaid_frags: HashMap = existing_fragments + .iter() + .filter(|f| { + (removed_ids.contains(&f.id) || updated_by_id.contains_key(&f.id)) + && !f.overlays.is_empty() + }) + .map(|f| (f.id as u32, f)) + .collect(); + Self::register_pure_rewrite_rows_update_frags_in_indices( &mut final_indices, &pure_updated_frag_ids, &original_fragment_ids, fields_for_preserving_frag_bitmap, - ); + &original_overlaid_frags, + &schema, + )?; } if let Some(next_row_id) = &mut next_row_id { @@ -2030,22 +2573,28 @@ impl Transaction { final_fragments.extend(new_fragments); Self::retain_relevant_indices(&mut final_indices, &schema, &final_fragments); - if !merged_generations.is_empty() { - update_mem_wal_index_merged_generations( + if !compacted_sstables.is_empty() { + update_mem_wal_index_compacted_sstables( &mut final_indices, - current_manifest.map_or(1, |m| m.version + 1), - merged_generations.clone(), + new_version, + compacted_sstables.clone(), )?; } } Operation::Overwrite { fragments, .. } => { - let mut new_fragments = - Self::fragments_with_ids(fragments.clone(), &mut fragment_id) - .collect::>(); + // Every fragment in an overwrite is newly written, so all of them + // take fresh ids regardless of the id they arrive with. Fragments + // carried over from the dataset being replaced are rejected by + // `validate_operation`, which is what makes ignoring the incoming + // id safe here. + let mut new_fragments = fragments.clone(); + for fragment in new_fragments.iter_mut() { + fragment.id = fragment_id; + fragment_id += 1; + } if let Some(next_row_id) = &mut next_row_id { Self::assign_row_ids(next_row_id, new_fragments.as_mut_slice())?; // Add version metadata for all new fragments - let new_version = current_manifest.map(|m| m.version + 1).unwrap_or(1); for fragment in new_fragments.iter_mut() { let version_meta = build_version_meta(fragment, new_version); fragment.last_updated_at_version_meta = version_meta.clone(); @@ -2074,15 +2623,33 @@ impl Transaction { // We can re-use indices, but need to rewrite the fragment bitmaps debug_assert!(rewritten_indices.is_empty()); for index in final_indices.iter_mut() { + let results_are_row_addrs = index_results_are_row_addrs(index); if let Some(fragment_bitmap) = &mut index.fragment_bitmap { - *fragment_bitmap = - Self::recalculate_fragment_bitmap(fragment_bitmap, groups)?; + *fragment_bitmap = if results_are_row_addrs { + // Stable row ids survive a rewrite, so a row-id-domain index + // can simply follow its data to the new fragments. An + // address-domain index cannot: its stored addresses point into + // the fragments the rewrite dropped. Claiming coverage of the + // new fragments would make it answer queries with addresses + // that no longer resolve, so drop the rewritten fragments from + // its coverage instead and let the scanner fall back to a full + // scan for them. + Self::drop_rewritten_fragments(fragment_bitmap, groups) + } else { + Self::recalculate_fragment_bitmap(fragment_bitmap, groups)? + }; } } } else { Self::handle_rewrite_indices(&mut final_indices, rewritten_indices, groups)?; } + // A full compaction materializes a fragment's overlays into fresh + // base data. Any index older than one of those overlays was built on + // the pre-overlay values, so drop the rewritten fragment from its + // coverage to keep it from serving stale values. + Self::prune_overlay_stale_fields_from_indices(&mut final_indices, groups); + if let Some(frag_reuse_index) = frag_reuse_index { final_indices.retain(|idx| idx.name != frag_reuse_index.name); final_indices.push(frag_reuse_index.clone()); @@ -2091,6 +2658,7 @@ impl Transaction { Operation::CreateIndex { new_indices, removed_indices, + .. } => { final_fragments.extend(maybe_existing_fragments?.clone()); let removed_uuids = removed_indices @@ -2114,7 +2682,6 @@ impl Transaction { let existing_fragments = maybe_existing_fragments?; let mut merged_fragments = fragments.clone(); if next_row_id.is_some() { - let new_version = current_manifest.map(|m| m.version + 1).unwrap_or(1); let prev_by_id: HashMap = existing_fragments.iter().map(|f| (f.id, f)).collect(); for fragment in merged_fragments.iter_mut() { @@ -2260,11 +2827,8 @@ impl Transaction { // just add it to the final fragments. Push the DataFile as // given so every field (including base_id) is preserved. if columns_covered.is_disjoint(&new_file.fields.iter().collect()) { - LanceFileVersion::try_from_major_minor( - new_file.file_major_version, - new_file.file_minor_version, - ) - .expect("Expected valid file version"); + new_file.file_version()?; + new_frag.files.push(new_file.clone()); } @@ -2274,6 +2838,15 @@ impl Transaction { "Expected to modify the fragment but no changes were made. This means the new data files does not align with any exiting datafiles. Please check if the schema of the new data files matches the schema of the old data files including the file major and minor versions", )); } + + // New base values for these fields supersede any overlay + // still shadowing them; tombstone the overlaid fields so the + // replacement is not silently masked. + lance_table::format::overlay::tombstone_overlay_fields( + &mut new_frag.overlays, + &replaced_fields, + ); + final_fragments.push(new_frag); } @@ -2299,17 +2872,84 @@ impl Transaction { .cloned() .collect(); + // A replacement changes what its rows read as, so stamp them + // updated. Without this, get_updated_rows never reports them and + // an incremental consumer skips them for good. + if next_row_id.is_some() { + let new_version = current_manifest.map_or(1, |m| m.version + 1); + for fragment in final_fragments + .iter_mut() + .filter(|f| fragments_changed.contains(&f.id)) + { + lance_table::rowids::version::refresh_row_latest_update_meta_for_full_frag_rewrite_cols( + fragment, + new_version, + )?; + } + } + Self::prune_updated_fields_from_indices( &mut final_indices, &modified_fragments, &replaced_fields, ); } - Operation::UpdateMemWalState { merged_generations } => { - update_mem_wal_index_merged_generations( + Operation::DataOverlay { groups } => { + // Stamp each overlay with the version this commit is producing. + // build_manifest re-runs on every retry with an updated + // current_manifest, so this is naturally re-stamped on retry. + let new_version = current_manifest.map_or(1, |m| m.version + 1); + + let existing_fragments = maybe_existing_fragments?; + // Multiple groups may target the same fragment; merge them in + // order rather than letting a HashMap collapse drop all but the + // last group's overlays. + let mut overlays_by_fragment: HashMap> = HashMap::new(); + for group in groups { + overlays_by_fragment + .entry(group.fragment_id) + .or_default() + .extend(group.overlays.iter()); + } + + // Every group must target an existing fragment. Build a set of + // existing ids once so this is O(groups + fragments) rather than + // O(groups * fragments). + let existing_fragment_ids: HashSet = + existing_fragments.iter().map(|f| f.id).collect(); + for fragment_id in overlays_by_fragment.keys() { + if !existing_fragment_ids.contains(fragment_id) { + return Err(Error::invalid_input(format!( + "DataOverlay targets fragment {fragment_id}, which does not exist" + ))); + } + } + + for fragment in existing_fragments { + let mut fragment = fragment.clone(); + if let Some(new_overlays) = overlays_by_fragment.get(&fragment.id) { + // Appended (not replaced) so concurrently-written overlays + // survive; later entries are newer. + fragment + .overlays + .extend(new_overlays.iter().map(|&overlay| { + let mut overlay = overlay.clone(); + overlay.committed_version = new_version; + overlay + })); + } + final_fragments.push(fragment); + } + } + Operation::UpdateMemWalState { + compacted_sstables, .. + } => { + // Updates the MemWAL index only; the fragments are unchanged. + final_fragments.extend(maybe_existing_fragments?.clone()); + update_mem_wal_index_compacted_sstables( &mut final_indices, - current_manifest.map_or(1, |m| m.version + 1), - merged_generations.clone(), + new_version, + compacted_sstables.clone(), )?; } Operation::UpdateBases { .. } => { @@ -2325,13 +2965,36 @@ impl Transaction { // Clean up data files that only contain tombstoned fields Self::remove_tombstoned_data_files(&mut final_fragments); + // Enforce the newest-last overlay ordering invariant at the write + // boundary. Load normalizes with a sort; this rejects any commit path + // that assembled a fragment's overlays out of order. + for fragment in &final_fragments { + if !fragment.overlays.is_empty() { + lance_table::format::overlay::verify_overlays_newest_last(&fragment.overlays)?; + } + } + let user_requested_version = match (&config.storage_format, config.use_legacy_format) { - (Some(storage_format), _) => Some(storage_format.lance_file_version()?), - (None, Some(true)) => Some(LanceFileVersion::Legacy), - (None, Some(false)) => Some(LanceFileVersion::V2_0), + (Some(storage_format), _) => Some(storage_format.lance_file_format()), + (None, Some(true)) => Some(ConcreteFileVersion::V1), + (None, Some(false)) => Some(ConcreteFileVersion::V2_0), (None, None) => None, }; + // Applied once the final index list is known, so it sees exactly the + // indices this commit publishes rather than what any one operation arm + // intended. + if mem_wal_segments_before.is_some() { + let empty_segments = LogicalIndexSegments::new(); + Self::apply_mem_wal_index_coverage( + &mut final_indices, + mem_wal_segments_before.as_ref().unwrap_or(&empty_segments), + read_version_state, + index_catchup_required, + new_version, + )?; + } + let mut manifest = if let Some(current_manifest) = current_manifest { // OVERWRITE with initial_bases on existing dataset is not allowed (caught by validation) // So we always use new_from_previous which preserves base_paths @@ -2375,6 +3038,51 @@ impl Transaction { config.disable_transaction_file, )?; } + // Carried from the manifest this one is derived from. `new_from_previous` + // zeroes both feature words, so `apply_feature_flags` cannot see the + // previous state and every ordinary commit would otherwise drop the bit. + if let Some(current_manifest) = current_manifest { + inherit_mem_wal_index_catchup(&mut manifest, current_manifest)?; + } + + // Set after apply_feature_flags, which resets both flag words: activation + // is the one place the bit is turned on, and it must survive that reset. + if let Operation::UpdateMemWalState { + require_index_catchup: true, + .. + } = &self.operation + { + let reader_set = current_manifest + .map(|m| m.reader_feature_flags & FLAG_MEM_WAL_INDEX_CATCHUP != 0) + .unwrap_or(false); + let writer_set = current_manifest + .map(|m| m.writer_feature_flags & FLAG_MEM_WAL_INDEX_CATCHUP != 0) + .unwrap_or(false); + match (reader_set, writer_set) { + (false, false) => { + Self::require_index_catchup(&mut final_indices, new_version)?; + log::info!( + "MemWAL index catch-up is now required at version {new_version}; a \ + missing catch-up entry means an index is behind, not caught up. \ + This is one-way." + ); + } + // Already active. A retry whose first attempt landed but lost its + // response must not clear coverage repaired since, so this keeps + // every recorded generation. + (true, true) => {} + _ => { + return Err(Error::invalid_input( + "Cannot require MemWAL index catch-up: the table has only one of \ + the reader and writer feature bits set, so its catch-up \ + semantics are undefined", + )); + } + } + manifest.reader_feature_flags |= FLAG_MEM_WAL_INDEX_CATCHUP; + manifest.writer_feature_flags |= FLAG_MEM_WAL_INDEX_CATCHUP; + } + manifest.set_timestamp(timestamp_to_nanos(config.timestamp)); manifest.update_max_fragment_id(); @@ -2566,9 +3274,11 @@ impl Transaction { pure_update_frag_ids: &[u64], original_fragment_ids: &[u64], fields_for_preserving_frag_bitmap: &[u32], - ) { + original_overlaid_frags: &HashMap, + schema: &Schema, + ) -> Result<()> { if pure_update_frag_ids.is_empty() { - return; + return Ok(()); } let value_updated_field_set = fields_for_preserving_frag_bitmap @@ -2576,33 +3286,58 @@ impl Transaction { .collect::>(); for index in indices.iter_mut() { + // Physical row addresses cannot follow moved rows into a new fragment. + // Leave that fragment uncovered so the scanner reads it directly. + if index_results_are_row_addrs(index) { + continue; + } let index_covers_modified_field = index.fields.iter().any(|field_id| { value_updated_field_set.contains(&u32::try_from(*field_id).unwrap()) }); + if index_covers_modified_field { + continue; + } + let Some(fragment_bitmap) = index.fragment_bitmap.as_ref() else { + continue; + }; - if !index_covers_modified_field - && let Some(fragment_bitmap) = &mut index.fragment_bitmap - { - // check if all the original fragments contains the updating rows are covered - // by the index(index fragment bitmap contains these frag ids). - // if not, that means not all the updating rows are indexed, so we could not - // index them. - let index_covers_all_original_fragments = original_fragment_ids - .iter() - .all(|&fragment_id| fragment_bitmap.contains(fragment_id as u32)); + // Check that all the original fragments containing the updated rows are covered by + // the index. If not, some updated rows were not indexed, so we cannot index them. + let index_covers_all_original_fragments = original_fragment_ids + .iter() + .all(|&fragment_id| fragment_bitmap.contains(fragment_id as u32)); + if !index_covers_all_original_fragments { + continue; + } - if index_covers_all_original_fragments { - for fragment_id in pure_update_frag_ids.iter().map(|f| *f as u32) { - fragment_bitmap.insert(fragment_id); - } + // A rewrite materializes overlays. If any of those overlays touched the + // column being indexed then the rewrite will modify that column. As a + // result, that index will no longer cover the fragment and it does not + // count as a pure rewrite and we must exclude it from the index's fragment + // bitmap. + let mut overlay_stale = RoaringBitmap::new(); + collect_overlay_stale_frags( + index, + original_overlaid_frags, + &mut overlay_stale, + schema, + )?; + if !overlay_stale.is_empty() { + continue; + } + + if let Some(fragment_bitmap) = index.fragment_bitmap.as_mut() { + for fragment_id in pure_update_frag_ids.iter().map(|f| *f as u32) { + fragment_bitmap.insert(fragment_id); } } } + Ok(()) } /// If an operation modifies one or more fields in a fragment then we need to remove /// that fragment from any indices that cover one of the modified fields. - fn prune_updated_fields_from_indices( + pub(crate) fn prune_updated_fields_from_indices( indices: &mut [IndexMetadata], updated_fragments: &[Fragment], fields_modified: &[u32], @@ -2682,11 +3417,53 @@ impl Transaction { } } - fn is_vector_index(index: &IndexMetadata) -> bool { - if let Some(details) = &index.index_details { - details.type_url.ends_with("VectorIndexDetails") - } else { - false + /// After a `Rewrite` fully compacts a fragment, its data overlays are baked + /// into the new fragment's base data. An index built *before* one of those + /// overlays (`overlay.committed_version > index.dataset_version`) indexed the + /// stale pre-overlay values -- and unlike a live overlay, the compacted + /// fragment no longer signals that staleness to the query path. Drop each + /// rewritten (new) fragment from the coverage of any index covering a field + /// such an overlay supplied, so those rows fall back to a flat scan. + fn prune_overlay_stale_fields_from_indices( + indices: &mut [IndexMetadata], + groups: &[RewriteGroup], + ) { + for group in groups { + // field id -> newest overlay committed_version supplying that field + let mut overlaid_field_versions: HashMap = HashMap::new(); + for old_frag in &group.old_fragments { + for overlay in &old_frag.overlays { + for &field_id in overlay.data_file.fields.iter() { + if field_id < 0 { + // Tombstoned (obsolete) overlay field: supplies nothing. + continue; + } + let entry = overlaid_field_versions.entry(field_id).or_insert(0); + *entry = (*entry).max(overlay.committed_version); + } + } + } + if overlaid_field_versions.is_empty() { + continue; + } + + let new_fragment_ids = group + .new_fragments + .iter() + .map(|f| f.id as u32) + .collect::>(); + for index in indices.iter_mut() { + let is_stale = index.fields.iter().any(|field_id| { + overlaid_field_versions + .get(field_id) + .is_some_and(|&overlay_version| overlay_version > index.dataset_version) + }); + if is_stale && let Some(fragment_bitmap) = &mut index.fragment_bitmap { + for new_id in &new_fragment_ids { + fragment_bitmap.remove(*new_id); + } + } + } } } @@ -2760,15 +3537,19 @@ impl Transaction { }); if non_empty_indices.is_empty() { - // All indices are empty - for scalar indices, keep only the first (oldest) one - // For vector indices, remove all of them + // All indices are empty -- keep only the oldest definition. + // + // An empty index definition is still correct: the scanner + // falls back to scanning unindexed fragments, and normal + // index maintenance rebuilds coverage once rows accrue. + // Dropping the definition instead would silently lose the + // index whenever an operation replaces every fragment it + // covered (e.g. a full table rewrite), leaving the dataset + // without its declared index. let mut sorted_indices = empty_indices; sorted_indices.sort_by_key(|index: &&IndexMetadata| index.dataset_version); // Sort by ascending dataset_version - // Keep only the first (oldest) if it's not a vector index - if let Some(oldest) = sorted_indices.first() - && !Self::is_vector_index(oldest) - { + if let Some(oldest) = sorted_indices.first() { uuids_to_keep.insert(oldest.uuid); } } else { @@ -2778,18 +3559,10 @@ impl Transaction { } } } else { - // Single index - keep it unless it's an empty vector index + // Single index whose column is still in schema: keep it, even + // when its coverage is empty (see the all-empty note above). if let Some(index) = same_name_indices.first() { - let is_empty = index - .effective_fragment_bitmap(&existing_fragments) - .as_ref() - .is_none_or(|bitmap| bitmap.is_empty()); - let is_vector = Self::is_vector_index(index); - - // Keep the index unless it's an empty vector index - if !is_empty || !is_vector { - uuids_to_keep.insert(index.uuid); - } + uuids_to_keep.insert(index.uuid); } } } @@ -2834,8 +3607,20 @@ impl Transaction { Ok(new_bitmap) } - fn handle_rewrite_indices( - indices: &mut [IndexMetadata], + /// Coverage of an index that a rewrite invalidates: the rewritten fragments are + /// removed and the fragments they became are *not* added. + fn drop_rewritten_fragments(old: &RoaringBitmap, groups: &[RewriteGroup]) -> RoaringBitmap { + let mut new_bitmap = old.clone(); + for group in groups { + for old_fragment in &group.old_fragments { + new_bitmap.remove(old_fragment.id as u32); + } + } + new_bitmap + } + + fn handle_rewrite_indices( + indices: &mut [IndexMetadata], rewritten_indices: &[RewrittenIndex], groups: &[RewriteGroup], ) -> Result<()> { @@ -3015,7 +3800,7 @@ impl Transaction { let combined_sequence = RowIdSequence::from(row_ids.as_slice()); let serialized = write_row_ids(&combined_sequence); - fragment.row_id_meta = Some(RowIdMeta::Inline(serialized)); + fragment.row_id_meta = Some(RowIdMeta::Inline(serialized.into())); *next_row_id += remaining_rows; } Ordering::Greater => { @@ -3031,7 +3816,7 @@ impl Transaction { let sequence = RowIdSequence::from(row_ids); // TODO: write to a separate file if large. Possibly share a file with other fragments. let serialized = write_row_ids(&sequence); - fragment.row_id_meta = Some(RowIdMeta::Inline(serialized)); + fragment.row_id_meta = Some(RowIdMeta::Inline(serialized.into())); *next_row_id += physical_rows; } } @@ -3066,6 +3851,34 @@ impl TryFrom for DataReplacementGroup { } } +impl From<&DataOverlayGroup> for pb::transaction::DataOverlayGroup { + fn from(group: &DataOverlayGroup) -> Self { + Self { + fragment_id: group.fragment_id, + overlays: group + .overlays + .iter() + .map(pb::DataOverlayFile::from) + .collect(), + } + } +} + +impl TryFrom for DataOverlayGroup { + type Error = Error; + + fn try_from(message: pb::transaction::DataOverlayGroup) -> Result { + Ok(Self { + fragment_id: message.fragment_id, + overlays: message + .overlays + .into_iter() + .map(DataOverlayFile::try_from) + .collect::>>()?, + }) + } +} + impl TryFrom for Transaction { type Error = Error; @@ -3122,7 +3935,7 @@ impl TryFrom for Transaction { .into_iter() .map(Fragment::try_from) .collect::>>()?, - schema: Schema::from(&Fields(schema)), + schema: Schema::try_from(&Fields(schema))?, config_upsert_values: config_upsert_option, initial_bases: if initial_bases.is_empty() { None @@ -3185,12 +3998,17 @@ impl TryFrom for Transaction { fragments, schema, schema_metadata: _schema_metadata, // TODO: handle metadata + preserves_nullability, })) => Operation::Merge { fragments: fragments .into_iter() .map(Fragment::try_from) .collect::>>()?, - schema: Schema::from(&Fields(schema)), + schema: Schema::try_from(&Fields(schema))?, + // False for a writer that predates the field: no assertion, so + // a legacy required-field merge still conflicts and a legacy + // nullable merge over-conflicts, which only retries. + preserves_nullability, }, Some(pb::transaction::Operation::Restore(pb::transaction::Restore { version })) => { Operation::Restore { version } @@ -3200,11 +4018,12 @@ impl TryFrom for Transaction { updated_fragments, new_fragments, fields_modified, - merged_generations, + compacted_sstables, fields_for_preserving_frag_bitmap, update_mode, inserted_rows, updated_fragment_offsets, + updated_fragment_offset_bitmaps, })) => Operation::Update { removed_fragment_ids, updated_fragments: updated_fragments @@ -3216,9 +4035,9 @@ impl TryFrom for Transaction { .map(Fragment::try_from) .collect::>>()?, fields_modified, - merged_generations: merged_generations + compacted_sstables: compacted_sstables .into_iter() - .map(|m| MergedGeneration::try_from(m).unwrap()) + .map(|m| CompactedSsTable::try_from(m).unwrap()) .collect(), fields_for_preserving_frag_bitmap, update_mode: match update_mode { @@ -3230,11 +4049,31 @@ impl TryFrom for Transaction { .map(|ik| KeyExistenceFilter::try_from(&ik)) .transpose()?, updated_fragment_offsets: { - let m: HashMap = updated_fragment_offsets - .into_iter() - .filter(|(_, list)| !list.values.is_empty()) - .map(|(id, list)| (id, RoaringBitmap::from_iter(list.values))) - .collect(); + // Prefer field 10 (RoaringBitmap bytes); fall back to field 9 (UInt32List) + // for manifests written before this change. + let m: HashMap = + if !updated_fragment_offset_bitmaps.is_empty() { + updated_fragment_offset_bitmaps + .into_iter() + .filter(|(_, bytes)| !bytes.is_empty()) + .map(|(id, bytes)| { + let bitmap = RoaringBitmap::deserialize_from(bytes.as_slice()) + .map_err(|e| { + Error::invalid_input(format!( + "invalid updated_fragment_offset_bitmaps \ + for fragment {id}: {e}" + )) + })?; + Ok((id, bitmap)) + }) + .collect::>>()? + } else { + updated_fragment_offsets + .into_iter() + .filter(|(_, list)| !list.values.is_empty()) + .map(|(id, list)| (id, RoaringBitmap::from_iter(list.values))) + .collect() + }; if m.is_empty() { None } else { @@ -3242,11 +4081,16 @@ impl TryFrom for Transaction { } }, }, - Some(pb::transaction::Operation::Project(pb::transaction::Project { schema })) => { - Operation::Project { - schema: Schema::from(&Fields(schema)), - } - } + Some(pb::transaction::Operation::Project(pb::transaction::Project { + schema, + preserves_nullability, + })) => Operation::Project { + schema: Schema::try_from(&Fields(schema))?, + // False for a writer that predates the field: no assertion, so + // a legacy tightening still conflicts and a legacy rename + // over-conflicts, which only retries. + preserves_nullability, + }, Some(pb::transaction::Operation::UpdateConfig(update_config)) => { // Check if new-style fields are present let has_new_fields = update_config.config_updates.is_some() @@ -3336,18 +4180,41 @@ impl TryFrom for Transaction { .collect::>>()?, }, Some(pb::transaction::Operation::UpdateMemWalState( - pb::transaction::UpdateMemWalState { merged_generations }, + pb::transaction::UpdateMemWalState { + compacted_sstables, + require_index_catchup, + }, )) => Operation::UpdateMemWalState { - merged_generations: merged_generations + compacted_sstables: compacted_sstables .into_iter() - .map(|m| MergedGeneration::try_from(m).unwrap()) - .collect(), + .map(CompactedSsTable::try_from) + .collect::>()?, + // Absent is an ordinary progress update. Explicit `false` is + // refused rather than read as absent, so a caller cannot express + // "deactivate" -- the migration is one-way. + require_index_catchup: match require_index_catchup { + Some(false) => { + return Err(Error::invalid_input( + "require_index_catchup cannot be false: MemWAL index catch-up \ + cannot stop being required once it is", + )); + } + other => other.unwrap_or(false), + }, }, Some(pb::transaction::Operation::UpdateBases(pb::transaction::UpdateBases { new_bases, })) => Operation::UpdateBases { new_bases: new_bases.into_iter().map(BasePath::from).collect(), }, + Some(pb::transaction::Operation::DataOverlay(pb::transaction::DataOverlay { + groups, + })) => Operation::DataOverlay { + groups: groups + .into_iter() + .map(DataOverlayGroup::try_from) + .collect::>>()?, + }, None => { return Err(Error::internal( "Transaction message did not contain an operation".to_string(), @@ -3524,13 +4391,16 @@ impl From<&Transaction> for pb::Transaction { .map(pb::IndexMetadata::from) .collect(), }), - Operation::Merge { fragments, schema } => { - pb::transaction::Operation::Merge(pb::transaction::Merge { - fragments: fragments.iter().map(pb::DataFragment::from).collect(), - schema: Fields::from(schema).0, - schema_metadata: Default::default(), // TODO: handle metadata - }) - } + Operation::Merge { + fragments, + schema, + preserves_nullability, + } => pb::transaction::Operation::Merge(pb::transaction::Merge { + fragments: fragments.iter().map(pb::DataFragment::from).collect(), + schema: Fields::from(schema).0, + schema_metadata: Default::default(), // TODO: handle metadata + preserves_nullability: *preserves_nullability, + }), Operation::Restore { version } => { pb::transaction::Operation::Restore(pb::transaction::Restore { version: *version }) } @@ -3539,7 +4409,7 @@ impl From<&Transaction> for pb::Transaction { updated_fragments, new_fragments, fields_modified, - merged_generations, + compacted_sstables, fields_for_preserving_frag_bitmap, update_mode, inserted_rows_filter, @@ -3552,9 +4422,9 @@ impl From<&Transaction> for pb::Transaction { .collect(), new_fragments: new_fragments.iter().map(pb::DataFragment::from).collect(), fields_modified: fields_modified.clone(), - merged_generations: merged_generations + compacted_sstables: compacted_sstables .iter() - .map(pb::MergedGeneration::from) + .map(pb::CompactedSsTable::from) .collect(), fields_for_preserving_frag_bitmap: fields_for_preserving_frag_bitmap.clone(), update_mode: update_mode @@ -3565,24 +4435,31 @@ impl From<&Transaction> for pb::Transaction { }) .unwrap_or(0), inserted_rows: inserted_rows_filter.as_ref().map(|ik| ik.into()), - updated_fragment_offsets: updated_fragment_offsets + // Field 9: no longer written; kept empty for forward compat. + updated_fragment_offsets: HashMap::new(), + // Field 10: RoaringBitmap bytes. + updated_fragment_offset_bitmaps: updated_fragment_offsets .as_ref() .map(|UpdatedFragmentOffsets(m)| { m.iter() .filter(|(_, b)| !b.is_empty()) .map(|(frag_id, b)| { - let values: Vec = b.iter().collect(); - (*frag_id, pb::transaction::UInt32List { values }) + let mut buf = Vec::new(); + b.serialize_into(&mut buf) + .expect("RoaringBitmap serialization cannot fail"); + (*frag_id, buf) }) .collect::>() }) .unwrap_or_default(), }), - Operation::Project { schema } => { - pb::transaction::Operation::Project(pb::transaction::Project { - schema: Fields::from(schema).0, - }) - } + Operation::Project { + schema, + preserves_nullability, + } => pb::transaction::Operation::Project(pb::transaction::Project { + schema: Fields::from(schema).0, + preserves_nullability: *preserves_nullability, + }), Operation::UpdateConfig { config_updates, table_metadata_updates, @@ -3618,12 +4495,26 @@ impl From<&Transaction> for pb::Transaction { .collect(), }) } - Operation::UpdateMemWalState { merged_generations } => { + Operation::DataOverlay { groups } => { + pb::transaction::Operation::DataOverlay(pb::transaction::DataOverlay { + groups: groups + .iter() + .map(pb::transaction::DataOverlayGroup::from) + .collect(), + }) + } + Operation::UpdateMemWalState { + compacted_sstables, + require_index_catchup, + } => { pb::transaction::Operation::UpdateMemWalState(pb::transaction::UpdateMemWalState { - merged_generations: merged_generations + compacted_sstables: compacted_sstables .iter() - .map(pb::MergedGeneration::from) + .map(pb::CompactedSsTable::from) .collect::>(), + // Written only when requesting activation, so an ordinary + // progress update stays byte-identical to before. + require_index_catchup: require_index_catchup.then_some(true), }) } Operation::UpdateBases { new_bases } => { @@ -3703,6 +4594,7 @@ pub fn validate_operation(manifest: Option<&Manifest>, operation: &Operation) -> }, ) => { // Validate here because we are going to return early. + overwrite_fragments_valid(fragments)?; schema_fragments_valid(None, schema, fragments)?; return Ok(()); @@ -3722,19 +4614,19 @@ pub fn validate_operation(manifest: Option<&Manifest>, operation: &Operation) -> // Fragments must contain all fields in the schema schema_fragments_valid(Some(manifest), &manifest.schema, fragments) } - Operation::Project { schema } => { + Operation::Project { schema, .. } => { schema_fragments_valid(Some(manifest), schema, manifest.fragments.as_ref()) } - Operation::Merge { fragments, schema } => { + Operation::Merge { + fragments, schema, .. + } => { merge_fragments_valid(manifest, fragments)?; schema_fragments_valid(Some(manifest), schema, fragments) } Operation::Overwrite { - fragments, - schema, - config_upsert_values: None, - initial_bases: _, + fragments, schema, .. } => { + overwrite_fragments_valid(fragments)?; // Pass None for manifest because Overwrite replaces all fragments. // The old manifest's storage format is irrelevant for validating // the new fragments (e.g., LEGACY→STABLE transitions). @@ -3743,25 +4635,75 @@ pub fn validate_operation(manifest: Option<&Manifest>, operation: &Operation) -> Operation::Update { updated_fragments, new_fragments, + updated_fragment_offsets, + update_mode, .. } => { schema_fragments_valid(Some(manifest), &manifest.schema, updated_fragments)?; - schema_fragments_valid(Some(manifest), &manifest.schema, new_fragments) + schema_fragments_valid(Some(manifest), &manifest.schema, new_fragments)?; + // Key-presence check only applies to RewriteColumns: that is the only + // mode where build_manifest stamps version metadata using off_map keys, + // so a stray key can corrupt an unrelated fragment's metadata. + // Other modes (e.g. rewrite_rows) may supply offsets for fragments + // outside updated_fragments for their own purposes. + if matches!(update_mode, Some(UpdateMode::RewriteColumns)) + && let Some(UpdatedFragmentOffsets(off_map)) = updated_fragment_offsets + { + let updated_ids: HashSet = updated_fragments.iter().map(|f| f.id).collect(); + for &frag_id in off_map.keys() { + if !updated_ids.contains(&frag_id) { + return Err(Error::invalid_input(format!( + "updatedFragmentOffsets key {} is not in updated_fragments; \ + offsets must reference only fragments being rewritten", + frag_id + ))); + } + } + } + Ok(()) } _ => Ok(()), } } +// An overwrite's fragments are newly written, so they are given fresh ids at +// commit time. A deletion file cannot come along for that ride: its path embeds +// the fragment id, so renumbering the fragment would orphan the deletion vector +// and silently resurrect deleted rows. +fn overwrite_fragments_valid(fragments: &[Fragment]) -> Result<()> { + for fragment in fragments { + if let Some(deletion_file) = &fragment.deletion_file { + return Err(Error::invalid_input(format!( + "Overwrite fragments must be newly written, but fragment {} carries \ + deletion file {}. Use Delete to commit deletions against existing \ + fragments, or Merge to change their schema.", + fragment.id, + relative_deletion_file_path(fragment.id, deletion_file) + ))); + } + } + Ok(()) +} + fn schema_fragments_valid( manifest: Option<&Manifest>, schema: &Schema, fragments: &[Fragment], ) -> Result<()> { - if let Some(manifest) = manifest - && manifest.data_storage_format.lance_file_version()? == LanceFileVersion::Legacy - { - return schema_fragments_legacy_valid(schema, fragments); + if let Some(manifest) = manifest { + return super::versions::validate_fragment_schema( + manifest.data_storage_format.lance_file_format(), + schema, + fragments, + ); } + schema_fragments_modern_valid(schema, fragments) +} + +pub(crate) fn schema_fragments_modern_valid( + _schema: &Schema, + fragments: &[Fragment], +) -> Result<()> { // validate that each data file at least contains one field. for fragment in fragments { for data_file in &fragment.files { @@ -3779,7 +4721,7 @@ fn schema_fragments_valid( /// Check that each fragment contains all fields in the schema. /// It is not required that the schema contains all fields in the fragment. /// There may be masked fields. -fn schema_fragments_legacy_valid(schema: &Schema, fragments: &[Fragment]) -> Result<()> { +pub(crate) fn schema_fragments_legacy_valid(schema: &Schema, fragments: &[Fragment]) -> Result<()> { // TODO: add additional validation. Consider consolidating with various // validate() methods in the codebase. for fragment in fragments { @@ -3894,8 +4836,9 @@ mod tests { use lance_core::utils::address::RowAddress; use lance_core::utils::tempfile::TempStrDir; use lance_core::{ROW_ADDR, ROW_CREATED_AT_VERSION, ROW_LAST_UPDATED_AT_VERSION}; - use lance_file::version::LanceFileVersion; + use lance_file::version::{ConcreteFileVersion, LanceFileVersion}; use lance_io::utils::CachedFileSize; + use lance_table::format::overlay::OverlayCoverage; use lance_table::format::{ RowDatasetVersionMeta, RowDatasetVersionRun, RowDatasetVersionSequence, RowIdMeta, }; @@ -3910,11 +4853,15 @@ mod tests { use crate::session::Session; fn sample_manifest() -> Manifest { + sample_manifest_with_fragments(0..1) + } + + fn sample_manifest_with_fragments(ids: std::ops::Range) -> Manifest { let schema = ArrowSchema::new(vec![ArrowField::new("id", DataType::Int32, false)]); Manifest::new( LanceSchema::try_from(&schema).unwrap(), - Arc::new(vec![Fragment::new(0)]), - DataStorageFormat::new(LanceFileVersion::V2_0), + Arc::new(ids.map(Fragment::new).collect()), + DataStorageFormat::new(ConcreteFileVersion::V2_0), HashMap::new(), ) } @@ -4000,7 +4947,7 @@ mod tests { let manifest = Manifest::new( LanceSchema::try_from(&schema).unwrap(), Arc::new(original_fragments), - DataStorageFormat::new(LanceFileVersion::V2_0), + DataStorageFormat::new(ConcreteFileVersion::V2_0), HashMap::new(), ); @@ -4139,6 +5086,88 @@ mod tests { ); } + #[test] + fn test_update_build_manifest_replaces_and_removes_fragments() { + let manifest = sample_manifest_with_fragments(0..5); + + let mut updated2 = Fragment::new(2); + updated2.physical_rows = Some(42); + let mut updated4 = Fragment::new(4); + updated4.physical_rows = Some(43); + + let transaction = Transaction::new( + manifest.version, + Operation::Update { + removed_fragment_ids: vec![1], + // Fragment 99 does not exist in the dataset; it must be ignored, + // not appended. + updated_fragments: vec![updated2, updated4, Fragment::new(99)], + new_fragments: vec![], + fields_modified: vec![], + compacted_sstables: vec![], + fields_for_preserving_frag_bitmap: vec![], + update_mode: None, + inserted_rows_filter: None, + updated_fragment_offsets: None, + }, + None, + ); + + let (new_manifest, _) = transaction + .build_manifest( + Some(&manifest), + vec![], + "txn", + &ManifestWriteConfig::default(), + ) + .unwrap(); + + let ids: Vec = new_manifest.fragments.iter().map(|f| f.id).collect(); + assert_eq!(ids, vec![0, 2, 3, 4]); + let rows: Vec> = new_manifest + .fragments + .iter() + .map(|f| f.physical_rows) + .collect(); + assert_eq!(rows, vec![None, Some(42), None, Some(43)]); + } + + #[test] + fn test_delete_build_manifest_replaces_and_removes_fragments() { + let manifest = sample_manifest_with_fragments(0..5); + + let mut updated2 = Fragment::new(2); + updated2.physical_rows = Some(42); + + let transaction = Transaction::new( + manifest.version, + Operation::Delete { + updated_fragments: vec![updated2], + deleted_fragment_ids: vec![1, 3], + predicate: "id > 0".to_string(), + }, + None, + ); + + let (new_manifest, _) = transaction + .build_manifest( + Some(&manifest), + vec![], + "txn", + &ManifestWriteConfig::default(), + ) + .unwrap(); + + let ids: Vec = new_manifest.fragments.iter().map(|f| f.id).collect(); + assert_eq!(ids, vec![0, 2, 4]); + let rows: Vec> = new_manifest + .fragments + .iter() + .map(|f| f.physical_rows) + .collect(); + assert_eq!(rows, vec![None, Some(42), None]); + } + #[test] fn test_remove_tombstoned_data_files() { // Create a fragment with mixed data files: some normal, some fully tombstoned @@ -4207,6 +5236,7 @@ mod tests { physical_rows: Some(100), row_id_meta: None, files: vec![], + overlays: vec![], deletion_file: None, last_updated_at_version_meta: None, created_at_version_meta: None, @@ -4237,8 +5267,9 @@ mod tests { let mut fragments = vec![Fragment { id: 1, physical_rows: Some(50), - row_id_meta: Some(RowIdMeta::Inline(serialized)), + row_id_meta: Some(RowIdMeta::Inline(serialized.into())), files: vec![], + overlays: vec![], deletion_file: None, last_updated_at_version_meta: None, created_at_version_meta: None, @@ -4269,8 +5300,9 @@ mod tests { let mut fragments = vec![Fragment { id: 1, physical_rows: Some(50), // More physical rows than existing row IDs - row_id_meta: Some(RowIdMeta::Inline(serialized)), + row_id_meta: Some(RowIdMeta::Inline(serialized.into())), files: vec![], + overlays: vec![], deletion_file: None, last_updated_at_version_meta: None, created_at_version_meta: None, @@ -4304,8 +5336,9 @@ mod tests { let mut fragments = vec![Fragment { id: 1, physical_rows: Some(50), // Less physical rows than existing row IDs - row_id_meta: Some(RowIdMeta::Inline(serialized)), + row_id_meta: Some(RowIdMeta::Inline(serialized.into())), files: vec![], + overlays: vec![], deletion_file: None, last_updated_at_version_meta: None, created_at_version_meta: None, @@ -4334,6 +5367,7 @@ mod tests { physical_rows: Some(30), // No existing row IDs row_id_meta: None, files: vec![], + overlays: vec![], deletion_file: None, last_updated_at_version_meta: None, created_at_version_meta: None, @@ -4341,8 +5375,9 @@ mod tests { Fragment { id: 2, physical_rows: Some(25), // Partial existing row IDs - row_id_meta: Some(RowIdMeta::Inline(serialized)), + row_id_meta: Some(RowIdMeta::Inline(serialized.into())), files: vec![], + overlays: vec![], deletion_file: None, last_updated_at_version_meta: None, created_at_version_meta: None, @@ -4387,6 +5422,7 @@ mod tests { physical_rows: None, row_id_meta: None, files: vec![], + overlays: vec![], deletion_file: None, last_updated_at_version_meta: None, created_at_version_meta: None, @@ -4556,7 +5592,7 @@ mod tests { } #[test] - fn test_retain_single_empty_vector_index() { + fn test_retain_single_empty_vector_index_is_kept() { let schema = create_test_schema(&[1]); let fragments = vec![Fragment::new(1)]; @@ -4570,8 +5606,9 @@ mod tests { Transaction::retain_relevant_indices(&mut indices, &schema, &fragments); - // Single empty vector index should be removed - assert_eq!(indices.len(), 0); + // The empty definition is retained: coverage is empty but the index + // declaration must survive operations that replace every fragment. + assert_eq!(indices.len(), 1); } #[test] @@ -4614,9 +5651,10 @@ mod tests { Transaction::retain_relevant_indices(&mut scalar_indices, &schema, &fragments); Transaction::retain_relevant_indices(&mut vector_indices, &schema, &fragments); - // Scalar should be kept, vector should be removed + // Both kept: a None bitmap counts as empty coverage, and empty + // definitions are retained regardless of index type. assert_eq!(scalar_indices.len(), 1); - assert_eq!(vector_indices.len(), 0); + assert_eq!(vector_indices.len(), 1); } #[test] @@ -4638,7 +5676,7 @@ mod tests { } #[test] - fn test_retain_multiple_empty_vector_indices_removes_all() { + fn test_retain_multiple_empty_vector_indices_keeps_oldest() { let schema = create_test_schema(&[1]); let fragments = vec![Fragment::new(1)]; @@ -4650,8 +5688,10 @@ mod tests { Transaction::retain_relevant_indices(&mut indices, &schema, &fragments); - // All empty vector indices should be removed - assert_eq!(indices.len(), 0); + // Same as the scalar case: all deltas are empty, so only the oldest + // definition survives. + assert_eq!(indices.len(), 1); + assert_eq!(indices[0].dataset_version, 1); } #[test] @@ -4744,10 +5784,10 @@ mod tests { Transaction::retain_relevant_indices(&mut scalar_indices, &schema, &fragments); Transaction::retain_relevant_indices(&mut vector_indices, &schema, &fragments); - // Scalar should be kept (single index, even if effective bitmap is empty) - // Vector should be removed (empty effective bitmap) + // Both kept: a single index whose column is still in schema is + // retained even when its effective coverage is empty. assert_eq!(scalar_indices.len(), 1); - assert_eq!(vector_indices.len(), 0); + assert_eq!(vector_indices.len(), 1); } #[test] @@ -4763,11 +5803,12 @@ mod tests { Transaction::retain_relevant_indices(&mut indices, &schema, &fragments); - // idx_a (empty scalar) should be kept, idx_b (empty vector) removed, idx_c (non-empty) kept - assert_eq!(indices.len(), 2); + // All three kept: empty definitions are retained for scalar and + // vector indexes alike. + assert_eq!(indices.len(), 3); assert!(indices.iter().any(|idx| idx.name == "idx_a")); + assert!(indices.iter().any(|idx| idx.name == "idx_b")); assert!(indices.iter().any(|idx| idx.name == "idx_c")); - assert!(!indices.iter().any(|idx| idx.name == "idx_b")); } #[test] @@ -4795,7 +5836,10 @@ mod tests { Transaction::retain_relevant_indices(&mut indices, &schema, &fragments); - assert_eq!(indices.len(), 0); + // Only the bad-field index is dropped; the empty vector definitions + // are retained. + assert_eq!(indices.len(), 2); + assert!(!indices.iter().any(|idx| idx.name == "idx3")); } #[test] @@ -4810,7 +5854,7 @@ mod tests { create_test_index("idx_a", 1, 3, Some(RoaringBitmap::new()), false), create_test_index("idx_a", 1, 1, Some(RoaringBitmap::new()), false), // Oldest create_test_index("idx_a", 1, 2, Some(RoaringBitmap::new()), false), - // Group "vec_b" - all empty vectors, remove all + // Group "vec_b" - all empty vectors, keep oldest definition create_test_index("vec_b", 1, 1, Some(RoaringBitmap::new()), true), create_test_index("vec_b", 1, 2, Some(RoaringBitmap::new()), true), // Group "idx_c" - mixed empty/non-empty, keep non-empty @@ -4825,8 +5869,9 @@ mod tests { Transaction::retain_relevant_indices(&mut indices, &schema, &fragments); - // Expected: frag_reuse, idx_a (oldest), idx_c (2 non-empty), idx_d = 5 total - assert_eq!(indices.len(), 5); + // Expected: frag_reuse, idx_a (oldest), vec_b (oldest), idx_c (2 + // non-empty), idx_d = 6 total + assert_eq!(indices.len(), 6); // Verify system index kept assert!(indices.iter().any(|idx| idx.name == FRAG_REUSE_INDEX_NAME)); @@ -4836,8 +5881,10 @@ mod tests { assert_eq!(idx_a_indices.len(), 1); assert_eq!(idx_a_indices[0].dataset_version, 1); - // Verify vec_b all removed - assert!(!indices.iter().any(|idx| idx.name == "vec_b")); + // Verify vec_b kept oldest definition only + let vec_b_indices: Vec<_> = indices.iter().filter(|idx| idx.name == "vec_b").collect(); + assert_eq!(vec_b_indices.len(), 1); + assert_eq!(vec_b_indices[0].dataset_version, 1); // Verify idx_c kept non-empty only let idx_c_indices: Vec<_> = indices.iter().filter(|idx| idx.name == "idx_c").collect(); @@ -4886,14 +5933,21 @@ mod tests { #[test] fn test_partial_rewrite_skips_fragment_with_no_version_meta() { let row_ids = RowIdSequence::from([10u64, 11, 12, 13, 14].as_slice()); - let row_id_meta = Some(RowIdMeta::Inline(write_row_ids(&row_ids))); + let row_id_meta = Some(RowIdMeta::Inline(write_row_ids(&row_ids).into())); - let (major, minor) = lance_file::version::LanceFileVersion::Stable.to_numbers(); - let data_file = DataFile::new("data.lance", vec![0], vec![0], major, minor, None, None); + let data_file = DataFile::new( + "data.lance", + vec![0], + vec![0], + LanceFileVersion::Stable.resolve(), + None, + None, + ); let fragment = Fragment { id: 1, files: vec![data_file], + overlays: vec![], deletion_file: None, row_id_meta, physical_rows: Some(5), @@ -4912,7 +5966,7 @@ mod tests { updated_fragments: vec![fragment], new_fragments: vec![], fields_modified: vec![], - merged_generations: vec![], + compacted_sstables: vec![], fields_for_preserving_frag_bitmap: vec![], update_mode: Some(UpdateMode::RewriteColumns), inserted_rows_filter: None, @@ -4936,85 +5990,459 @@ mod tests { ); } - /// Partial RewriteColumns refresh in `build_manifest`: only matched physical - /// rows get `last_updated_at_version` bumped; same-fragment unmatched rows and - /// untouched fragments keep both version sequences. - #[tokio::test] - async fn test_build_manifest_partial_last_updated_rewrite_columns_stable_row_ids() { - let dir = TempStrDir::default(); - let uri = dir.as_str(); + #[test] + fn test_bitmap_cardinality_exceeds_physical_rows() { + let row_ids = RowIdSequence::from([10u64, 11, 12, 13, 14].as_slice()); + let row_id_meta = Some(RowIdMeta::Inline(write_row_ids(&row_ids).into())); - let schema = Arc::new(ArrowSchema::new(vec![ - ArrowField::new("i", DataType::Int32, false), - ArrowField::new("x", DataType::Int32, false), - ])); - let batch0 = RecordBatch::try_new( - schema.clone(), - vec![ - Arc::new(Int32Array::from_iter_values(0..8)), - Arc::new(Int32Array::from(vec![0_i32; 8])), - ], - ) - .unwrap(); - let reader0 = RecordBatchIterator::new(vec![Ok(batch0)], schema.clone()); - let write_params = WriteParams { - enable_stable_row_ids: true, - data_storage_version: Some(LanceFileVersion::Stable), - ..Default::default() + let data_file = DataFile::new( + "data.lance", + vec![0], + vec![0], + LanceFileVersion::Stable.resolve(), + None, + None, + ); + + let version_seq = RowDatasetVersionSequence::from_uniform_row_count(5, 1); + let version_meta = RowDatasetVersionMeta::from_sequence(&version_seq).unwrap(); + + let fragment = Fragment { + id: 1, + files: vec![data_file], + overlays: vec![], + deletion_file: None, + row_id_meta, + physical_rows: Some(5), + last_updated_at_version_meta: Some(version_meta.clone()), + created_at_version_meta: Some(version_meta), }; - let mut dataset = Dataset::write(reader0, uri, Some(write_params)) - .await - .unwrap(); - let batch1 = RecordBatch::try_new( - schema.clone(), - vec![ - Arc::new(Int32Array::from_iter_values(100..108)), - Arc::new(Int32Array::from(vec![0_i32; 8])), - ], - ) - .unwrap(); - let reader1 = RecordBatchIterator::new(vec![Ok(batch1)], schema.clone()); - dataset.append(reader1, None).await.unwrap(); + let manifest = make_stable_row_id_manifest(vec![fragment.clone()]); - let frags = dataset.get_fragments(); - assert_eq!( - frags.len(), - 2, - "expected two fragments (append creates a new fragment)" + // Bitmap with 10 offsets but fragment only has 5 physical rows. + let off_map = HashMap::from([(1u64, RoaringBitmap::from_iter(0u32..10))]); + let tx = Transaction::new( + manifest.version, + Operation::Update { + removed_fragment_ids: vec![], + updated_fragments: vec![fragment], + new_fragments: vec![], + fields_modified: vec![], + compacted_sstables: vec![], + fields_for_preserving_frag_bitmap: vec![], + update_mode: Some(UpdateMode::RewriteColumns), + inserted_rows_filter: None, + updated_fragment_offsets: Some(UpdatedFragmentOffsets(off_map)), + }, + None, ); - async fn scan_row_versions(ds: &Dataset) -> HashMap<(u32, u32), (u64, u64)> { - let mut scanner = ds.scan(); - scanner - .project(&[ - ROW_ADDR, - ROW_LAST_UPDATED_AT_VERSION, - ROW_CREATED_AT_VERSION, - ]) - .unwrap(); - let batches = scanner - .try_into_stream() - .await - .unwrap() - .try_collect::>() - .await - .unwrap(); - let mut out = HashMap::new(); - for batch in batches { - let addrs = batch - .column_by_name(ROW_ADDR) - .unwrap() - .as_primitive::(); - let last = batch - .column_by_name(ROW_LAST_UPDATED_AT_VERSION) - .unwrap() - .as_primitive::(); - let created = batch - .column_by_name(ROW_CREATED_AT_VERSION) - .unwrap() - .as_primitive::(); - for row in 0..batch.num_rows() { + let result = tx.build_manifest( + Some(&manifest), + vec![], + "txn", + &ManifestWriteConfig::default(), + ); + assert!(result.is_err()); + let msg = result.unwrap_err().to_string(); + assert!( + msg.contains("cardinality"), + "expected cardinality error, got: {msg}" + ); + } + + #[test] + fn test_bitmap_max_offset_exceeds_physical_rows() { + let row_ids = RowIdSequence::from([10u64, 11, 12, 13, 14].as_slice()); + let row_id_meta = Some(RowIdMeta::Inline(write_row_ids(&row_ids).into())); + + let data_file = DataFile::new( + "data.lance", + vec![0], + vec![0], + LanceFileVersion::Stable.resolve(), + None, + None, + ); + + let version_seq = RowDatasetVersionSequence::from_uniform_row_count(5, 1); + let version_meta = RowDatasetVersionMeta::from_sequence(&version_seq).unwrap(); + + let fragment = Fragment { + id: 1, + files: vec![data_file], + overlays: vec![], + deletion_file: None, + row_id_meta, + physical_rows: Some(5), + last_updated_at_version_meta: Some(version_meta.clone()), + created_at_version_meta: Some(version_meta), + }; + + let manifest = make_stable_row_id_manifest(vec![fragment.clone()]); + + // Only 2 offsets (within cardinality) but max offset 100 exceeds physical_rows 5. + let off_map = HashMap::from([(1u64, RoaringBitmap::from_iter([0u32, 100]))]); + let tx = Transaction::new( + manifest.version, + Operation::Update { + removed_fragment_ids: vec![], + updated_fragments: vec![fragment], + new_fragments: vec![], + fields_modified: vec![], + compacted_sstables: vec![], + fields_for_preserving_frag_bitmap: vec![], + update_mode: Some(UpdateMode::RewriteColumns), + inserted_rows_filter: None, + updated_fragment_offsets: Some(UpdatedFragmentOffsets(off_map)), + }, + None, + ); + + let result = tx.build_manifest( + Some(&manifest), + vec![], + "txn", + &ManifestWriteConfig::default(), + ); + assert!(result.is_err()); + let msg = result.unwrap_err().to_string(); + assert!( + msg.contains("max offset"), + "expected max offset error, got: {msg}" + ); + } + + #[test] + fn test_bitmap_at_exact_physical_rows_boundary_succeeds() { + let row_ids = RowIdSequence::from([10u64, 11, 12, 13, 14].as_slice()); + let row_id_meta = Some(RowIdMeta::Inline(write_row_ids(&row_ids).into())); + + let data_file = DataFile::new( + "data.lance", + vec![0], + vec![0], + LanceFileVersion::Stable.resolve(), + None, + None, + ); + + let version_seq = RowDatasetVersionSequence::from_uniform_row_count(5, 1); + let version_meta = RowDatasetVersionMeta::from_sequence(&version_seq).unwrap(); + + let fragment = Fragment { + id: 1, + files: vec![data_file], + overlays: vec![], + deletion_file: None, + row_id_meta, + physical_rows: Some(5), + last_updated_at_version_meta: Some(version_meta.clone()), + created_at_version_meta: Some(version_meta), + }; + + let manifest = make_stable_row_id_manifest(vec![fragment.clone()]); + + // All 5 offsets on a 5-row fragment — exactly at the boundary, should succeed. + let off_map = HashMap::from([(1u64, RoaringBitmap::from_iter(0u32..5))]); + let tx = Transaction::new( + manifest.version, + Operation::Update { + removed_fragment_ids: vec![], + updated_fragments: vec![fragment], + new_fragments: vec![], + fields_modified: vec![], + compacted_sstables: vec![], + fields_for_preserving_frag_bitmap: vec![], + update_mode: Some(UpdateMode::RewriteColumns), + inserted_rows_filter: None, + updated_fragment_offsets: Some(UpdatedFragmentOffsets(off_map)), + }, + None, + ); + + tx.build_manifest( + Some(&manifest), + vec![], + "txn", + &ManifestWriteConfig::default(), + ) + .expect("bitmap at exact physical_rows boundary should succeed"); + } + + #[test] + fn test_updated_fragment_offsets_key_not_in_updated_fragments_is_rejected() { + // Fragment A is being rewritten; fragment B exists in the manifest but is + // NOT in updated_fragments. Supplying an offset key for B must be rejected + // so that B's version metadata cannot be stamped by an unrelated commit. + let make_fragment = |id: u64| { + let row_ids = RowIdSequence::from([id * 10].as_slice()); + let row_id_meta = Some(RowIdMeta::Inline(write_row_ids(&row_ids).into())); + Fragment { + id, + files: vec![DataFile::new( + format!("{id}.lance"), + vec![0], + vec![0], + LanceFileVersion::Stable.resolve(), + None, + None, + )], + overlays: vec![], + deletion_file: None, + row_id_meta, + physical_rows: Some(5), + last_updated_at_version_meta: None, + created_at_version_meta: None, + } + }; + + let frag_a = make_fragment(1); + let frag_b = make_fragment(2); + let manifest = make_stable_row_id_manifest(vec![frag_a.clone(), frag_b.clone()]); + + // updated_fragments contains only A; offsets are keyed to B — must fail. + let off_map = HashMap::from([(frag_b.id, RoaringBitmap::from_iter([0u32, 1, 2]))]); + let operation = Operation::Update { + removed_fragment_ids: vec![], + updated_fragments: vec![frag_a], + new_fragments: vec![], + fields_modified: vec![], + compacted_sstables: vec![], + fields_for_preserving_frag_bitmap: vec![], + update_mode: Some(UpdateMode::RewriteColumns), + inserted_rows_filter: None, + updated_fragment_offsets: Some(UpdatedFragmentOffsets(off_map)), + }; + + let err = validate_operation(Some(&manifest), &operation).unwrap_err(); + assert!( + err.to_string().contains("not in updated_fragments"), + "expected key-presence error, got: {err}" + ); + } + + #[test] + fn test_proto_round_trip_field_10() { + let off_map = HashMap::from([ + (1u64, RoaringBitmap::from_iter([1u32, 3, 5])), + (2u64, RoaringBitmap::from_iter([0u32, 2, 4, 6])), + ]); + let tx = Transaction::new( + 1, + Operation::Update { + removed_fragment_ids: vec![], + updated_fragments: vec![], + new_fragments: vec![], + fields_modified: vec![], + compacted_sstables: vec![], + fields_for_preserving_frag_bitmap: vec![], + update_mode: Some(UpdateMode::RewriteColumns), + inserted_rows_filter: None, + updated_fragment_offsets: Some(UpdatedFragmentOffsets(off_map.clone())), + }, + None, + ); + + let pb_tx: pb::Transaction = pb::Transaction::from(&tx); + + // Field 9 must be empty; field 10 must be populated. + if let Some(pb::transaction::Operation::Update(ref update)) = pb_tx.operation { + assert!( + update.updated_fragment_offsets.is_empty(), + "field 9 should be empty" + ); + assert_eq!(update.updated_fragment_offset_bitmaps.len(), 2); + } else { + panic!("expected Update operation"); + } + + let tx2 = Transaction::try_from(pb_tx).unwrap(); + if let Operation::Update { + updated_fragment_offsets: Some(UpdatedFragmentOffsets(m)), + .. + } = &tx2.operation + { + assert_eq!(m.len(), 2); + assert_eq!(*m.get(&1).unwrap(), off_map[&1]); + assert_eq!(*m.get(&2).unwrap(), off_map[&2]); + } else { + panic!("expected Update with offsets"); + } + } + + #[test] + fn test_proto_legacy_field_9_read() { + // Simulate a manifest written by old Lance: only field 9, no field 10. + let pb_tx = pb::Transaction { + read_version: 1, + uuid: "test".to_string(), + tag: String::new(), + transaction_properties: HashMap::new(), + operation: Some(pb::transaction::Operation::Update( + pb::transaction::Update { + removed_fragment_ids: vec![], + updated_fragments: vec![], + new_fragments: vec![], + fields_modified: vec![], + compacted_sstables: vec![], + fields_for_preserving_frag_bitmap: vec![], + update_mode: 1, + inserted_rows: None, + updated_fragment_offsets: HashMap::from([( + 1u64, + pb::transaction::UInt32List { + values: vec![1, 3, 5], + }, + )]), + updated_fragment_offset_bitmaps: HashMap::new(), + }, + )), + }; + + let tx = Transaction::try_from(pb_tx).unwrap(); + if let Operation::Update { + updated_fragment_offsets: Some(UpdatedFragmentOffsets(m)), + .. + } = &tx.operation + { + assert_eq!(m.len(), 1); + let bitmap = m.get(&1).unwrap(); + let offsets: Vec = bitmap.iter().collect(); + assert_eq!(offsets, vec![1, 3, 5]); + } else { + panic!("expected Update with offsets from legacy field 9"); + } + } + + #[test] + fn test_proto_field_10_takes_precedence_over_field_9() { + // When both fields present, field 10 wins. + let mut bitmap_bytes = Vec::new(); + RoaringBitmap::from_iter([10u32, 20, 30]) + .serialize_into(&mut bitmap_bytes) + .unwrap(); + + let pb_tx = pb::Transaction { + read_version: 1, + uuid: "test".to_string(), + tag: String::new(), + transaction_properties: HashMap::new(), + operation: Some(pb::transaction::Operation::Update( + pb::transaction::Update { + removed_fragment_ids: vec![], + updated_fragments: vec![], + new_fragments: vec![], + fields_modified: vec![], + compacted_sstables: vec![], + fields_for_preserving_frag_bitmap: vec![], + update_mode: 1, + inserted_rows: None, + // Field 9 has different values than field 10. + updated_fragment_offsets: HashMap::from([( + 1u64, + pb::transaction::UInt32List { + values: vec![99, 100], + }, + )]), + updated_fragment_offset_bitmaps: HashMap::from([(1u64, bitmap_bytes)]), + }, + )), + }; + + let tx = Transaction::try_from(pb_tx).unwrap(); + if let Operation::Update { + updated_fragment_offsets: Some(UpdatedFragmentOffsets(m)), + .. + } = &tx.operation + { + let offsets: Vec = m.get(&1).unwrap().iter().collect(); + assert_eq!(offsets, vec![10, 20, 30], "field 10 should take precedence"); + } else { + panic!("expected Update with offsets from field 10"); + } + } + + /// Partial RewriteColumns refresh in `build_manifest`: only matched physical + /// rows get `last_updated_at_version` bumped; same-fragment unmatched rows and + /// untouched fragments keep both version sequences. + #[tokio::test] + async fn test_build_manifest_partial_last_updated_rewrite_columns_stable_row_ids() { + let dir = TempStrDir::default(); + let uri = dir.as_str(); + + let schema = Arc::new(ArrowSchema::new(vec![ + ArrowField::new("i", DataType::Int32, false), + ArrowField::new("x", DataType::Int32, false), + ])); + let batch0 = RecordBatch::try_new( + schema.clone(), + vec![ + Arc::new(Int32Array::from_iter_values(0..8)), + Arc::new(Int32Array::from(vec![0_i32; 8])), + ], + ) + .unwrap(); + let reader0 = RecordBatchIterator::new(vec![Ok(batch0)], schema.clone()); + let write_params = WriteParams { + enable_stable_row_ids: true, + data_storage_version: Some(LanceFileVersion::Stable), + ..Default::default() + }; + let mut dataset = Dataset::write(reader0, uri, Some(write_params)) + .await + .unwrap(); + + let batch1 = RecordBatch::try_new( + schema.clone(), + vec![ + Arc::new(Int32Array::from_iter_values(100..108)), + Arc::new(Int32Array::from(vec![0_i32; 8])), + ], + ) + .unwrap(); + let reader1 = RecordBatchIterator::new(vec![Ok(batch1)], schema.clone()); + dataset.append(reader1, None).await.unwrap(); + + let frags = dataset.get_fragments(); + assert_eq!( + frags.len(), + 2, + "expected two fragments (append creates a new fragment)" + ); + + async fn scan_row_versions(ds: &Dataset) -> HashMap<(u32, u32), (u64, u64)> { + let mut scanner = ds.scan(); + scanner + .project(&[ + ROW_ADDR, + ROW_LAST_UPDATED_AT_VERSION, + ROW_CREATED_AT_VERSION, + ]) + .unwrap(); + let batches = scanner + .try_into_stream() + .await + .unwrap() + .try_collect::>() + .await + .unwrap(); + let mut out = HashMap::new(); + for batch in batches { + let addrs = batch + .column_by_name(ROW_ADDR) + .unwrap() + .as_primitive::(); + let last = batch + .column_by_name(ROW_LAST_UPDATED_AT_VERSION) + .unwrap() + .as_primitive::(); + let created = batch + .column_by_name(ROW_CREATED_AT_VERSION) + .unwrap() + .as_primitive::(); + for row in 0..batch.num_rows() { let addr = RowAddress::from(addrs.value(row)); out.insert( (addr.fragment_id(), addr.row_offset()), @@ -5065,7 +6493,7 @@ mod tests { updated_fragments: vec![u.fragment], new_fragments: vec![], fields_modified: u.fields_modified, - merged_generations: Vec::new(), + compacted_sstables: Vec::new(), fields_for_preserving_frag_bitmap: vec![], update_mode: Some(UpdateMode::RewriteColumns), inserted_rows_filter: None, @@ -5145,7 +6573,7 @@ mod tests { let legacy_manifest = Manifest::new( schema.clone(), Arc::new(vec![Fragment::new(0)]), - DataStorageFormat::new(LanceFileVersion::Legacy), + DataStorageFormat::new(ConcreteFileVersion::V1), HashMap::new(), ); @@ -5157,12 +6585,12 @@ mod tests { "data.lance", vec![0, 1, 3, 4], // no field 2 (struct parent) vec![0, 1, 2, 3], - lance_file::format::MAJOR_VERSION as u32, - lance_file::format::MINOR_VERSION as u32, + ConcreteFileVersion::V1, None, None, )], physical_rows: Some(10), + overlays: vec![], deletion_file: None, row_id_meta: None, last_updated_at_version_meta: None, @@ -5188,7 +6616,7 @@ mod tests { let mut manifest = Manifest::new( LanceSchema::try_from(&schema).unwrap(), Arc::new(fragments), - DataStorageFormat::new(LanceFileVersion::V2_0), + DataStorageFormat::new(ConcreteFileVersion::V2_0), HashMap::new(), ); manifest.reader_feature_flags = FLAG_STABLE_ROW_IDS; @@ -5205,7 +6633,7 @@ mod tests { updated_fragments: vec![], new_fragments, fields_modified: vec![], - merged_generations: vec![], + compacted_sstables: vec![], fields_for_preserving_frag_bitmap: vec![], update_mode: None, inserted_rows_filter: None, @@ -5242,18 +6670,27 @@ mod tests { use lance_file::version::LanceFileVersion; use lance_table::feature_flags::FLAG_STABLE_ROW_IDS; - let (major, minor) = LanceFileVersion::Stable.to_numbers(); - let mk_file = |path: &str| DataFile::new(path, vec![0], vec![0], major, minor, None, None); + let mk_file = |path: &str| { + DataFile::new( + path, + vec![0], + vec![0], + LanceFileVersion::Stable.resolve(), + None, + None, + ) + }; let arrow_schema = ArrowSchema::new(vec![ArrowField::new("id", DataType::Int32, false)]); let lance_schema = LanceSchema::try_from(&arrow_schema).unwrap(); let row_ids = RowIdSequence::from([100u64, 101, 102, 103, 104].as_slice()); - let row_id_meta = Some(RowIdMeta::Inline(write_row_ids(&row_ids))); + let row_id_meta = Some(RowIdMeta::Inline(write_row_ids(&row_ids).into())); let prev_fragment = Fragment { id: 0, files: vec![mk_file("before.lance")], + overlays: vec![], deletion_file: None, row_id_meta, physical_rows: Some(5), @@ -5264,7 +6701,7 @@ mod tests { let mut manifest = Manifest::new( lance_schema.clone(), Arc::new(vec![prev_fragment.clone()]), - DataStorageFormat::new(LanceFileVersion::V2_0), + DataStorageFormat::new(ConcreteFileVersion::V2_0), HashMap::new(), ); manifest.reader_feature_flags |= FLAG_STABLE_ROW_IDS; @@ -5280,6 +6717,7 @@ mod tests { Operation::Merge { fragments: vec![merged_fragment], schema: lance_schema, + preserves_nullability: true, }, None, ); @@ -5311,14 +6749,20 @@ mod tests { use lance_table::feature_flags::FLAG_STABLE_ROW_IDS; use lance_table::rowids::version::{RowDatasetVersionMeta, RowDatasetVersionSequence}; - let (major, minor) = LanceFileVersion::Stable.to_numbers(); - let data_file = DataFile::new("same.lance", vec![0], vec![0], major, minor, None, None); + let data_file = DataFile::new( + "same.lance", + vec![0], + vec![0], + LanceFileVersion::Stable.resolve(), + None, + None, + ); let arrow_schema = ArrowSchema::new(vec![ArrowField::new("id", DataType::Int32, false)]); let lance_schema = LanceSchema::try_from(&arrow_schema).unwrap(); let row_ids = RowIdSequence::from([200u64, 201, 202, 203, 204].as_slice()); - let row_id_meta = Some(RowIdMeta::Inline(write_row_ids(&row_ids))); + let row_id_meta = Some(RowIdMeta::Inline(write_row_ids(&row_ids).into())); let uniform_v1 = RowDatasetVersionSequence::from_uniform_row_count(5, 1); let meta_v1 = RowDatasetVersionMeta::from_sequence(&uniform_v1).unwrap(); @@ -5326,6 +6770,7 @@ mod tests { let prev_fragment = Fragment { id: 0, files: vec![data_file.clone()], + overlays: vec![], deletion_file: None, row_id_meta: row_id_meta.clone(), physical_rows: Some(5), @@ -5336,7 +6781,7 @@ mod tests { let mut manifest = Manifest::new( lance_schema.clone(), Arc::new(vec![prev_fragment]), - DataStorageFormat::new(LanceFileVersion::V2_0), + DataStorageFormat::new(ConcreteFileVersion::V2_0), HashMap::new(), ); manifest.reader_feature_flags |= FLAG_STABLE_ROW_IDS; @@ -5345,6 +6790,7 @@ mod tests { let merged_fragment = Fragment { id: 0, files: vec![data_file], + overlays: vec![], deletion_file: None, row_id_meta, physical_rows: Some(5), @@ -5357,6 +6803,7 @@ mod tests { Operation::Merge { fragments: vec![merged_fragment], schema: lance_schema, + preserves_nullability: true, }, None, ); @@ -5385,8 +6832,16 @@ mod tests { use lance_file::version::LanceFileVersion; use lance_table::feature_flags::FLAG_STABLE_ROW_IDS; - let (major, minor) = LanceFileVersion::Stable.to_numbers(); - let mk_file = |path: &str| DataFile::new(path, vec![0], vec![0], major, minor, None, None); + let mk_file = |path: &str| { + DataFile::new( + path, + vec![0], + vec![0], + LanceFileVersion::Stable.resolve(), + None, + None, + ) + }; let arrow_schema = ArrowSchema::new(vec![ArrowField::new("id", DataType::Int32, false)]); let lance_schema = LanceSchema::try_from(&arrow_schema).unwrap(); @@ -5394,6 +6849,7 @@ mod tests { let prev_fragment = Fragment { id: 0, files: vec![mk_file("before.lance")], + overlays: vec![], deletion_file: None, row_id_meta: None, physical_rows: Some(5), @@ -5404,7 +6860,7 @@ mod tests { let manifest = Manifest::new( lance_schema.clone(), Arc::new(vec![prev_fragment.clone()]), - DataStorageFormat::new(LanceFileVersion::V2_0), + DataStorageFormat::new(ConcreteFileVersion::V2_0), HashMap::new(), ); assert_eq!( @@ -5423,6 +6879,7 @@ mod tests { Operation::Merge { fragments: vec![merged_fragment], schema: lance_schema, + preserves_nullability: true, }, None, ); @@ -5447,8 +6904,16 @@ mod tests { use lance_file::version::LanceFileVersion; use lance_table::feature_flags::FLAG_STABLE_ROW_IDS; - let (major, minor) = LanceFileVersion::Stable.to_numbers(); - let mk_file = |path: &str| DataFile::new(path, vec![0], vec![0], major, minor, None, None); + let mk_file = |path: &str| { + DataFile::new( + path, + vec![0], + vec![0], + LanceFileVersion::Stable.resolve(), + None, + None, + ) + }; let arrow_schema = ArrowSchema::new(vec![ArrowField::new("id", DataType::Int32, false)]); let lance_schema = LanceSchema::try_from(&arrow_schema).unwrap(); @@ -5458,8 +6923,9 @@ mod tests { let existing_fragment = Fragment { id: 0, files: vec![mk_file("existing.lance")], + overlays: vec![], deletion_file: None, - row_id_meta: Some(RowIdMeta::Inline(write_row_ids(&row_ids_0))), + row_id_meta: Some(RowIdMeta::Inline(write_row_ids(&row_ids_0).into())), physical_rows: Some(3), last_updated_at_version_meta: None, created_at_version_meta: None, @@ -5468,7 +6934,7 @@ mod tests { let mut manifest = Manifest::new( lance_schema.clone(), Arc::new(vec![existing_fragment.clone()]), - DataStorageFormat::new(LanceFileVersion::V2_0), + DataStorageFormat::new(ConcreteFileVersion::V2_0), HashMap::new(), ); manifest.reader_feature_flags |= FLAG_STABLE_ROW_IDS; @@ -5480,8 +6946,9 @@ mod tests { let new_fragment = Fragment { id: 1, files: vec![mk_file("new.lance")], + overlays: vec![], deletion_file: None, - row_id_meta: Some(RowIdMeta::Inline(write_row_ids(&row_ids_1))), + row_id_meta: Some(RowIdMeta::Inline(write_row_ids(&row_ids_1).into())), physical_rows: Some(4), last_updated_at_version_meta: None, created_at_version_meta: None, @@ -5492,6 +6959,7 @@ mod tests { Operation::Merge { fragments: vec![existing_fragment, new_fragment], schema: lance_schema, + preserves_nullability: true, }, None, ); @@ -5542,8 +7010,9 @@ mod tests { let existing_fragment = Fragment { id: 1, files: vec![], + overlays: vec![], deletion_file: None, - row_id_meta: Some(RowIdMeta::Inline(write_row_ids(&existing_seq))), + row_id_meta: Some(RowIdMeta::Inline(write_row_ids(&existing_seq).into())), physical_rows: Some(3), created_at_version_meta: Some( RowDatasetVersionMeta::from_sequence(&created_at_seq).unwrap(), @@ -5555,8 +7024,9 @@ mod tests { let new_fragment = Fragment { id: 10, files: vec![], + overlays: vec![], deletion_file: None, - row_id_meta: Some(RowIdMeta::Inline(write_row_ids(&new_seq))), + row_id_meta: Some(RowIdMeta::Inline(write_row_ids(&new_seq).into())), physical_rows: Some(2), created_at_version_meta: None, last_updated_at_version_meta: None, @@ -5597,8 +7067,9 @@ mod tests { Fragment { id: 1, files: vec![], + overlays: vec![], deletion_file: None, - row_id_meta: Some(RowIdMeta::Inline(write_row_ids(&frag_a_seq))), + row_id_meta: Some(RowIdMeta::Inline(write_row_ids(&frag_a_seq).into())), physical_rows: Some(2), created_at_version_meta: Some( RowDatasetVersionMeta::from_sequence(&frag_a_created).unwrap(), @@ -5608,8 +7079,9 @@ mod tests { Fragment { id: 2, files: vec![], + overlays: vec![], deletion_file: None, - row_id_meta: Some(RowIdMeta::Inline(write_row_ids(&frag_b_seq))), + row_id_meta: Some(RowIdMeta::Inline(write_row_ids(&frag_b_seq).into())), physical_rows: Some(3), created_at_version_meta: Some( RowDatasetVersionMeta::from_sequence(&frag_b_created).unwrap(), @@ -5623,8 +7095,9 @@ mod tests { let new_fragment = Fragment { id: 10, files: vec![], + overlays: vec![], deletion_file: None, - row_id_meta: Some(RowIdMeta::Inline(write_row_ids(&new_seq))), + row_id_meta: Some(RowIdMeta::Inline(write_row_ids(&new_seq).into())), physical_rows: Some(2), created_at_version_meta: None, last_updated_at_version_meta: None, @@ -5664,8 +7137,9 @@ mod tests { let existing_fragment = Fragment { id: 1, files: vec![], + overlays: vec![], deletion_file: None, - row_id_meta: Some(RowIdMeta::Inline(write_row_ids(&existing_seq))), + row_id_meta: Some(RowIdMeta::Inline(write_row_ids(&existing_seq).into())), physical_rows: Some(2), created_at_version_meta: Some( RowDatasetVersionMeta::from_sequence(&existing_created).unwrap(), @@ -5678,8 +7152,9 @@ mod tests { let new_fragment = Fragment { id: 10, files: vec![], + overlays: vec![], deletion_file: None, - row_id_meta: Some(RowIdMeta::Inline(write_row_ids(&new_seq))), + row_id_meta: Some(RowIdMeta::Inline(write_row_ids(&new_seq).into())), physical_rows: Some(2), created_at_version_meta: None, last_updated_at_version_meta: None, @@ -5722,8 +7197,9 @@ mod tests { let existing_fragment = Fragment { id: 1, files: vec![], + overlays: vec![], deletion_file: None, - row_id_meta: Some(RowIdMeta::Inline(write_row_ids(&existing_seq))), + row_id_meta: Some(RowIdMeta::Inline(write_row_ids(&existing_seq).into())), physical_rows: Some(2), created_at_version_meta: Some( RowDatasetVersionMeta::from_sequence(&existing_created).unwrap(), @@ -5735,8 +7211,9 @@ mod tests { let new_fragment = Fragment { id: 20, files: vec![], + overlays: vec![], deletion_file: None, - row_id_meta: Some(RowIdMeta::Inline(write_row_ids(&new_seq))), + row_id_meta: Some(RowIdMeta::Inline(write_row_ids(&new_seq).into())), physical_rows: Some(4), created_at_version_meta: None, last_updated_at_version_meta: None, @@ -5768,8 +7245,9 @@ mod tests { let existing_fragment = Fragment { id: 1, files: vec![], + overlays: vec![], deletion_file: None, - row_id_meta: Some(RowIdMeta::Inline(write_row_ids(&existing_seq))), + row_id_meta: Some(RowIdMeta::Inline(write_row_ids(&existing_seq).into())), physical_rows: Some(2), created_at_version_meta: None, last_updated_at_version_meta: None, @@ -5779,8 +7257,9 @@ mod tests { let new_fragment = Fragment { id: 10, files: vec![], + overlays: vec![], deletion_file: None, - row_id_meta: Some(RowIdMeta::Inline(write_row_ids(&new_seq))), + row_id_meta: Some(RowIdMeta::Inline(write_row_ids(&new_seq).into())), physical_rows: Some(1), created_at_version_meta: None, last_updated_at_version_meta: None, @@ -5807,8 +7286,9 @@ mod tests { let existing_fragment = Fragment { id: 1, files: vec![], + overlays: vec![], deletion_file: None, - row_id_meta: Some(RowIdMeta::Inline(write_row_ids(&existing_seq))), + row_id_meta: Some(RowIdMeta::Inline(write_row_ids(&existing_seq).into())), physical_rows: Some(2), created_at_version_meta: None, last_updated_at_version_meta: None, @@ -5817,6 +7297,7 @@ mod tests { let new_fragment = Fragment { id: 10, files: vec![], + overlays: vec![], deletion_file: None, row_id_meta: None, physical_rows: Some(3), @@ -5847,8 +7328,9 @@ mod tests { let existing_fragment = Fragment { id: 1, files: vec![], + overlays: vec![], deletion_file: None, - row_id_meta: Some(RowIdMeta::Inline(write_row_ids(&existing_seq))), + row_id_meta: Some(RowIdMeta::Inline(write_row_ids(&existing_seq).into())), physical_rows: Some(2), created_at_version_meta: Some(RowDatasetVersionMeta::Inline(Arc::from( vec![0xFFu8; 8].as_slice(), @@ -5860,8 +7342,9 @@ mod tests { let new_fragment = Fragment { id: 10, files: vec![], + overlays: vec![], deletion_file: None, - row_id_meta: Some(RowIdMeta::Inline(write_row_ids(&new_seq))), + row_id_meta: Some(RowIdMeta::Inline(write_row_ids(&new_seq).into())), physical_rows: Some(1), created_at_version_meta: None, last_updated_at_version_meta: None, @@ -5901,8 +7384,9 @@ mod tests { let in_range_frag = Fragment { id: 1, files: vec![], + overlays: vec![], deletion_file: None, - row_id_meta: Some(RowIdMeta::Inline(write_row_ids(&in_range_seq))), + row_id_meta: Some(RowIdMeta::Inline(write_row_ids(&in_range_seq).into())), physical_rows: Some(2), created_at_version_meta: Some( RowDatasetVersionMeta::from_sequence(&in_range_created).unwrap(), @@ -5921,8 +7405,9 @@ mod tests { let out_of_range_frag = Fragment { id: 2, files: vec![], + overlays: vec![], deletion_file: None, - row_id_meta: Some(RowIdMeta::Inline(write_row_ids(&out_of_range_seq))), + row_id_meta: Some(RowIdMeta::Inline(write_row_ids(&out_of_range_seq).into())), physical_rows: Some(2), created_at_version_meta: Some( RowDatasetVersionMeta::from_sequence(&out_of_range_created).unwrap(), @@ -5935,8 +7420,9 @@ mod tests { let new_frag = Fragment { id: 10, files: vec![], + overlays: vec![], deletion_file: None, - row_id_meta: Some(RowIdMeta::Inline(write_row_ids(&new_seq))), + row_id_meta: Some(RowIdMeta::Inline(write_row_ids(&new_seq).into())), physical_rows: Some(2), created_at_version_meta: None, last_updated_at_version_meta: None, @@ -5973,8 +7459,9 @@ mod tests { let existing = Fragment { id: 1, files: vec![], + overlays: vec![], deletion_file: None, - row_id_meta: Some(RowIdMeta::Inline(write_row_ids(&seq))), + row_id_meta: Some(RowIdMeta::Inline(write_row_ids(&seq).into())), physical_rows: Some(3), created_at_version_meta: Some(RowDatasetVersionMeta::from_sequence(&created).unwrap()), last_updated_at_version_meta: None, @@ -5985,8 +7472,9 @@ mod tests { let new_frag = Fragment { id: 10, files: vec![], + overlays: vec![], deletion_file: None, - row_id_meta: Some(RowIdMeta::Inline(write_row_ids(&new_seq))), + row_id_meta: Some(RowIdMeta::Inline(write_row_ids(&new_seq).into())), physical_rows: Some(2), created_at_version_meta: None, last_updated_at_version_meta: None, @@ -6033,8 +7521,9 @@ mod tests { let src_frag = Fragment { id: 1, files: vec![], + overlays: vec![], deletion_file: None, - row_id_meta: Some(RowIdMeta::Inline(write_row_ids(&src_seq))), + row_id_meta: Some(RowIdMeta::Inline(write_row_ids(&src_seq).into())), physical_rows: Some(100), created_at_version_meta: Some( RowDatasetVersionMeta::from_sequence(&src_created).unwrap(), @@ -6047,8 +7536,9 @@ mod tests { let new_frag = Fragment { id: 10, files: vec![], + overlays: vec![], deletion_file: None, - row_id_meta: Some(RowIdMeta::Inline(write_row_ids(&new_seq))), + row_id_meta: Some(RowIdMeta::Inline(write_row_ids(&new_seq).into())), physical_rows: Some(100), created_at_version_meta: None, last_updated_at_version_meta: None, @@ -6094,8 +7584,9 @@ mod tests { Fragment { id: 1, files: vec![], + overlays: vec![], deletion_file: None, - row_id_meta: Some(RowIdMeta::Inline(write_row_ids(&seq_a))), + row_id_meta: Some(RowIdMeta::Inline(write_row_ids(&seq_a).into())), physical_rows: Some(3), created_at_version_meta: Some( RowDatasetVersionMeta::from_sequence(&created_a).unwrap(), @@ -6105,8 +7596,9 @@ mod tests { Fragment { id: 2, files: vec![], + overlays: vec![], deletion_file: None, - row_id_meta: Some(RowIdMeta::Inline(write_row_ids(&seq_b))), + row_id_meta: Some(RowIdMeta::Inline(write_row_ids(&seq_b).into())), physical_rows: Some(3), created_at_version_meta: Some( RowDatasetVersionMeta::from_sequence(&created_b).unwrap(), @@ -6120,8 +7612,9 @@ mod tests { let new_frag = Fragment { id: 10, files: vec![], + overlays: vec![], deletion_file: None, - row_id_meta: Some(RowIdMeta::Inline(write_row_ids(&new_seq))), + row_id_meta: Some(RowIdMeta::Inline(write_row_ids(&new_seq).into())), physical_rows: Some(2), created_at_version_meta: None, last_updated_at_version_meta: None, @@ -6186,4 +7679,1023 @@ mod tests { assert!(!left.modifies_same_metadata(&different_key)); assert!(left.modifies_same_metadata(&replace)); } + + #[test] + fn test_data_overlay_operation_roundtrips() { + // A DataOverlay operation survives the protobuf round-trip, preserving + // the target fragment, the overlay's coverage, and its committed_version. + let mut bitmap = roaring::RoaringBitmap::new(); + bitmap.insert(1); + bitmap.insert(4); + let overlay = DataOverlayFile { + data_file: DataFile::new_legacy_from_fields("overlay-0.lance", vec![3], None), + coverage: OverlayCoverage::dense(bitmap.clone()), + committed_version: 6, + }; + let pb_overlay = pb::DataOverlayFile::from(&overlay); + + let message = pb::Transaction { + read_version: 1, + uuid: Uuid::new_v4().to_string(), + operation: Some(pb::transaction::Operation::DataOverlay( + pb::transaction::DataOverlay { + groups: vec![pb::transaction::DataOverlayGroup { + fragment_id: 7, + overlays: vec![pb_overlay], + }], + }, + )), + ..Default::default() + }; + + let txn = Transaction::try_from(message).unwrap(); + match txn.operation { + Operation::DataOverlay { groups } => { + assert_eq!(groups.len(), 1); + assert_eq!(groups[0].fragment_id, 7); + assert_eq!(groups[0].overlays.len(), 1); + assert_eq!(groups[0].overlays[0].committed_version, 6); + assert_eq!( + *groups[0].overlays[0].coverage_for_field(0).unwrap(), + bitmap + ); + } + other => panic!("expected DataOverlay, got {other:?}"), + } + } + + fn overlay_with_field(field: i32, committed_version: u64) -> DataOverlayFile { + DataOverlayFile { + data_file: DataFile::new_legacy_from_fields("o.lance", vec![field], None), + coverage: OverlayCoverage::dense(roaring::RoaringBitmap::from_iter([0u32])), + committed_version, + } + } + + #[test] + fn test_prune_overlay_stale_fields_from_indices() { + // Fragment 0 carried an overlay on field 1 committed at v5, and was + // fully compacted into new fragment 7. + let mut old_frag = Fragment::new(0); + old_frag.overlays = vec![overlay_with_field(1, 5)]; + let groups = vec![RewriteGroup { + old_fragments: vec![old_frag], + new_fragments: vec![Fragment::new(7)], + }]; + + // Post-remap state: every index already covers the new fragment (7). + let covering = || Some(RoaringBitmap::from_iter([7u32])); + let mut indices = vec![ + // Stale: covers the overlaid field 1, built (v2) before the overlay. + create_test_index("stale", 1, 2, covering(), false), + // Not stale: covers field 1 but built at the overlay's version (v5); + // `committed_version > dataset_version` is false at equality. + create_test_index("fresh", 1, 5, covering(), false), + // Unrelated: covers field 2, which the overlay never touched. + create_test_index("unrelated", 2, 2, covering(), false), + ]; + + Transaction::prune_overlay_stale_fields_from_indices(&mut indices, &groups); + + assert!( + !indices[0].fragment_bitmap.as_ref().unwrap().contains(7), + "stale index must drop the rewritten fragment from its coverage" + ); + assert!( + indices[1].fragment_bitmap.as_ref().unwrap().contains(7), + "an index built at/after the overlay is not stale" + ); + assert!( + indices[2].fragment_bitmap.as_ref().unwrap().contains(7), + "an index on an un-overlaid field is unaffected" + ); + } + + #[test] + fn test_data_overlay_build_manifest_multi_fragment() { + // Overlays targeting two distinct fragments are each applied and stamped. + // A targeted fragment already carrying an overlay (committed at v3) gets + // the new overlay appended and stamped while its existing overlay is + // preserved, and a fragment the operation does not target is passed + // through with its existing overlays untouched. + let mut frag0 = Fragment::new(0); + frag0.overlays = vec![overlay_with_field(5, 3)]; // targeted, pre-existing at v3 + let frag1 = Fragment::new(1); + let mut frag2 = Fragment::new(2); + frag2.overlays = vec![overlay_with_field(9, 3)]; // untargeted, committed at v3 + let schema = ArrowSchema::new(vec![ArrowField::new("id", DataType::Int32, false)]); + let mut manifest = Manifest::new( + LanceSchema::try_from(&schema).unwrap(), + Arc::new(vec![frag0, frag1, frag2]), + lance_table::format::DataStorageFormat::new(ConcreteFileVersion::V2_0), + HashMap::new(), + ); + // The pre-existing overlays were committed at v3, so the current + // manifest must be at least that version; the new commit then stamps + // its overlay at v4, keeping the fragment's overlays newest-last. + manifest.version = 3; + + let txn = Transaction::new( + manifest.version, + Operation::DataOverlay { + groups: vec![ + DataOverlayGroup { + fragment_id: 0, + overlays: vec![overlay_with_field(1, 0)], + }, + DataOverlayGroup { + fragment_id: 1, + overlays: vec![overlay_with_field(2, 0)], + }, + ], + }, + None, + ); + + let (result, _) = txn + .build_manifest( + Some(&manifest), + vec![], + "txn", + &ManifestWriteConfig::default(), + ) + .unwrap(); + + let frag = |id: u64| { + result + .fragments + .iter() + .find(|f| f.id == id) + .unwrap_or_else(|| panic!("fragment {id} missing from result")) + }; + // The already-overlaid target keeps its v3 overlay and appends the new + // one, stamped to the new version. + assert_eq!(frag(0).overlays.len(), 2); + assert_eq!(frag(0).overlays[0].committed_version, 3); + assert_eq!(frag(0).overlays[1].committed_version, result.version); + // The fresh target gets its overlay, stamped to the new version. + assert_eq!(frag(1).overlays.len(), 1); + assert_eq!(frag(1).overlays[0].committed_version, result.version); + // The untargeted fragment is unchanged: same overlay, original version. + assert_eq!(frag(2).overlays.len(), 1); + assert_eq!(frag(2).overlays[0].committed_version, 3); + assert!(result.version > manifest.version); + } + + #[test] + fn test_data_replacement_tombstones_overlaid_fields() { + // A DataReplacement writing new base values for field 5 must stop any + // overlay from shadowing those cells: field 5 is tombstoned in place + // (preserving the overlay's field 3), and an overlay covering only field + // 5 is dropped entirely. + let mut fragment = Fragment::new(0); + fragment.files = vec![ + DataFile::new_legacy_from_fields("f3.lance", vec![3], None), + DataFile::new_legacy_from_fields("f5.lance", vec![5], None), + ]; + fragment.overlays = vec![ + DataOverlayFile { + data_file: DataFile::new_legacy_from_fields("o35.lance", vec![3, 5], None), + coverage: OverlayCoverage::sparse(vec![ + roaring::RoaringBitmap::from_iter([0u32]), + roaring::RoaringBitmap::from_iter([0u32]), + ]), + committed_version: 3, + }, + DataOverlayFile { + data_file: DataFile::new_legacy_from_fields("o5.lance", vec![5], None), + coverage: OverlayCoverage::dense(roaring::RoaringBitmap::from_iter([0u32])), + committed_version: 3, + }, + ]; + + let schema = ArrowSchema::new(vec![ArrowField::new("id", DataType::Int32, false)]); + let manifest = Manifest::new( + LanceSchema::try_from(&schema).unwrap(), + Arc::new(vec![fragment]), + lance_table::format::DataStorageFormat::new(ConcreteFileVersion::V2_0), + HashMap::new(), + ); + + let txn = Transaction::new( + manifest.version, + Operation::DataReplacement { + replacements: vec![DataReplacementGroup( + 0, + DataFile::new_legacy_from_fields("f5-new.lance", vec![5], None), + )], + }, + None, + ); + + let (result, _) = txn + .build_manifest( + Some(&manifest), + vec![], + "txn", + &ManifestWriteConfig::default(), + ) + .unwrap(); + + let frag = &result.fragments[0]; + // The base data file for field 5 was swapped in. + assert!(frag.files.iter().any(|f| f.path == "f5-new.lance")); + // The [3, 5] overlay keeps field 3 and tombstones field 5; the [5]-only + // overlay is dropped. + assert_eq!(frag.overlays.len(), 1); + assert_eq!(frag.overlays[0].data_file.fields.as_ref(), &[3, -2]); + } + + #[test] + fn test_data_overlay_build_manifest_merges_duplicate_groups() { + // Two groups targeting the same fragment must both survive (a HashMap + // collapse would have dropped the first). + let manifest = sample_manifest(); + let txn = Transaction::new( + manifest.version, + Operation::DataOverlay { + groups: vec![ + DataOverlayGroup { + fragment_id: 0, + overlays: vec![overlay_with_field(1, 0)], + }, + DataOverlayGroup { + fragment_id: 0, + overlays: vec![overlay_with_field(2, 0)], + }, + ], + }, + None, + ); + + let (result, _) = txn + .build_manifest( + Some(&manifest), + vec![], + "txn", + &ManifestWriteConfig::default(), + ) + .unwrap(); + + let overlays = &result.fragments[0].overlays; + assert_eq!(overlays.len(), 2); + assert_eq!(overlays[0].data_file.fields.as_ref(), [1i32].as_slice()); + assert_eq!(overlays[1].data_file.fields.as_ref(), [2i32].as_slice()); + } + + #[test] + fn test_data_overlay_build_manifest_rejects_unknown_fragment() { + let manifest = sample_manifest(); + let txn = Transaction::new( + manifest.version, + Operation::DataOverlay { + groups: vec![DataOverlayGroup { + fragment_id: 99, + overlays: vec![overlay_with_field(1, 0)], + }], + }, + None, + ); + let err = txn + .build_manifest( + Some(&manifest), + vec![], + "txn", + &ManifestWriteConfig::default(), + ) + .unwrap_err(); + assert!(err.to_string().contains("does not exist"), "{err}"); + } + + #[test] + fn test_data_overlay_operation_eq() { + let overlay = |field: i32| Operation::DataOverlay { + groups: vec![DataOverlayGroup { + fragment_id: 0, + overlays: vec![overlay_with_field(field, 1)], + }], + }; + // Reflexive and value-based (the arm previously returned false for self). + assert_eq!(overlay(1), overlay(1)); + assert_ne!(overlay(1), overlay(2)); + // Not equal to a different operation kind (previously returned true vs Rewrite). + let rewrite = Operation::Rewrite { + groups: vec![], + rewritten_indices: vec![], + frag_reuse_index: None, + }; + assert_ne!(overlay(1), rewrite); + } + + #[test] + fn test_nullability_assertion_defaults_conservative() { + // A writer that predates the field encodes nothing, which decodes as + // false: no assertion, so a legacy tightening or required-field merge + // still conflicts. Only an explicit true skips the barrier. + for encoded in [false, true] { + let txn = Transaction::try_from(pb::Transaction { + read_version: 1, + uuid: "test".to_string(), + operation: Some(pb::transaction::Operation::Project( + pb::transaction::Project { + schema: vec![], + preserves_nullability: encoded, + }, + )), + ..Default::default() + }) + .unwrap(); + assert!( + matches!(txn.operation, Operation::Project { preserves_nullability, .. } if preserves_nullability == encoded), + "encoded={encoded:?}" + ); + + let txn = Transaction::try_from(pb::Transaction { + read_version: 1, + uuid: "test".to_string(), + operation: Some(pb::transaction::Operation::Merge(pb::transaction::Merge { + fragments: vec![], + schema: vec![], + schema_metadata: Default::default(), + preserves_nullability: encoded, + })), + ..Default::default() + }) + .unwrap(); + assert!( + matches!(txn.operation, Operation::Merge { preserves_nullability, .. } if preserves_nullability == encoded), + "encoded={encoded:?}" + ); + } + } + + mod mem_wal_index_coverage { + use super::*; + use lance_index::mem_wal::{ + CompactedSsTable, IndexCatchupProgress, MEM_WAL_INDEX_NAME, MemWalIndexDetails, + }; + use lance_table::feature_flags::FLAG_MEM_WAL_INDEX_CATCHUP; + + fn user_index(name: &str, uuid: Uuid, frags: &[u32]) -> IndexMetadata { + IndexMetadata { + uuid, + name: name.to_string(), + fields: vec![0], + dataset_version: 1, + fragment_bitmap: Some(RoaringBitmap::from_iter(frags.iter().copied())), + index_details: None, + index_version: 0, + created_at: None, + base_id: None, + files: None, + } + } + + fn mem_wal_index(details: MemWalIndexDetails) -> IndexMetadata { + crate::index::mem_wal::new_mem_wal_index_meta(1, details).unwrap() + } + + fn coverage_for(indices: &[IndexMetadata], name: &str) -> Option> { + let meta = indices + .iter() + .find(|idx| idx.name == MEM_WAL_INDEX_NAME) + .expect("mem wal index present"); + load_mem_wal_index_details(meta.clone()) + .unwrap() + .index_catchup + .into_iter() + .find(|entry| entry.index_name == name) + .map(|entry| entry.caught_up_generations) + } + + fn compacted(shard: Uuid, generation: u64) -> Vec { + vec![CompactedSsTable::new(shard, generation)] + } + + /// A manifest carrying exactly `frags`, standing in for the version a + /// transaction read. + fn manifest_with(frags: &[u32]) -> Manifest { + let fragments: Vec = + frags.iter().map(|id| Fragment::new(*id as u64)).collect(); + Manifest::new( + Schema::default(), + Arc::new(fragments), + DataStorageFormat::default(), + Default::default(), + ) + } + + /// Drives the production path, so these exercise the real derivation. + fn apply( + after: &mut [IndexMetadata], + before: &[IndexMetadata], + read_frags: &[u32], + read_indices: &[IndexMetadata], + required: bool, + ) -> Result<()> { + let manifest = manifest_with(read_frags); + let segments_before = Transaction::logical_index_segments(before); + Transaction::apply_mem_wal_index_coverage( + after, + &segments_before, + Some(ReadVersionState { + manifest: &manifest, + indices: read_indices, + }), + required, + 2, + ) + } + + fn table(idx_frags: &[u32], uuid: Uuid, details: MemWalIndexDetails) -> Vec { + vec![user_index("idx", uuid, idx_frags), mem_wal_index(details)] + } + + fn progress(shard: Uuid, generation: u64) -> MemWalIndexDetails { + MemWalIndexDetails { + compacted_sstables: compacted(shard, generation), + ..Default::default() + } + } + + fn progress_with_catchup(shard: Uuid, generation: u64, caught: u64) -> MemWalIndexDetails { + MemWalIndexDetails { + compacted_sstables: compacted(shard, generation), + index_catchup: vec![IndexCatchupProgress::new( + "idx".to_string(), + compacted(shard, caught), + )], + ..Default::default() + } + } + + /// An index spanning every fragment the transaction read is credited + /// with what that version had compacted. + #[test] + fn an_index_covering_the_read_version_is_credited() { + let shard = Uuid::new_v4(); + let read = table(&[0, 1], Uuid::new_v4(), progress(shard, 5)); + let mut after = table(&[0, 1], Uuid::new_v4(), progress(shard, 5)); + apply(&mut after, &read, &[0, 1], &read, true).unwrap(); + assert_eq!(coverage_for(&after, "idx"), Some(compacted(shard, 5))); + } + + /// An index short of the read version proves nothing, so it gets no + /// entry -- absence reads as "not caught up". + #[test] + fn an_index_short_of_the_read_version_is_not_credited() { + let shard = Uuid::new_v4(); + let read = table(&[0], Uuid::new_v4(), progress(shard, 5)); + let mut after = table(&[0], Uuid::new_v4(), progress(shard, 5)); + apply(&mut after, &read, &[0, 1], &read, true).unwrap(); + assert_eq!(coverage_for(&after, "idx"), None); + } + + /// The hazard that makes the comparison use whole metadata. + /// + /// `Operation::Update` prunes a segment's fragment bitmap in place when + /// it touches an indexed field, keeping the same UUID. A UUID-only + /// "unchanged" test carries the old position forward while the index + /// covers fewer fragments, and the WAL pod then trims on a position the + /// index no longer earns. Reachable from the ordinary SSTable merge. + #[test] + fn a_bitmap_pruned_in_place_does_not_keep_its_position() { + let shard = Uuid::new_v4(); + let uuid = Uuid::new_v4(); + let before = table(&[0, 1], uuid, progress_with_catchup(shard, 5, 5)); + // Same UUID, fragment 1 pruned away. + let mut after = table(&[0], uuid, progress_with_catchup(shard, 5, 5)); + apply(&mut after, &before, &[0, 1], &before, true).unwrap(); + assert_eq!( + coverage_for(&after, "idx"), + None, + "a shrunken index kept a position it no longer earns" + ); + } + + /// Carrying a position forward is not the same as extending it. An + /// index that has not moved still only holds the generations it caught + /// up to; the compaction that has landed since is in fragments it does + /// not span. + #[test] + fn an_unchanged_index_is_not_raised_beyond_what_it_proves() { + let shard = Uuid::new_v4(); + let uuid = Uuid::new_v4(); + // Recorded at generation 2; generation 5 has since been folded in. + let before = table(&[0], uuid, progress_with_catchup(shard, 5, 2)); + let mut after = before.clone(); + // Fragment 1 arrived with that compaction and this index lacks it. + apply(&mut after, &before, &[0, 1], &before, true).unwrap(); + assert_eq!(coverage_for(&after, "idx"), Some(compacted(shard, 2))); + } + + /// A recorded position above what this commit says was compacted is + /// clamped down. Nothing should produce one, but a position the base + /// table cannot back would retire SSTables whose rows are nowhere. + #[test] + fn a_carried_position_cannot_exceed_the_committed_progress() { + let shard = Uuid::new_v4(); + let uuid = Uuid::new_v4(); + let before = table(&[0], uuid, progress_with_catchup(shard, 3, 9)); + let mut after = before.clone(); + apply(&mut after, &before, &[0], &before, true).unwrap(); + assert_eq!(coverage_for(&after, "idx"), Some(compacted(shard, 3))); + } + + /// An unchanged index keeps what it recorded even when this commit's + /// own snapshot cannot prove as much. + #[test] + fn an_unchanged_index_is_never_lowered() { + let shard = Uuid::new_v4(); + let uuid = Uuid::new_v4(); + let before = table(&[0], uuid, progress_with_catchup(shard, 9, 9)); + let mut after = before.clone(); + apply(&mut after, &before, &[0, 1], &before, true).unwrap(); + assert_eq!(coverage_for(&after, "idx"), Some(compacted(shard, 9))); + } + + /// Credit never exceeds what this commit records as compacted, so a + /// read version since rolled back cannot retire SSTables no live commit + /// copied in. + #[test] + fn credit_is_capped_by_the_committed_progress() { + let shard = Uuid::new_v4(); + let read = table(&[0], Uuid::new_v4(), progress(shard, 9)); + let mut after = table(&[0], Uuid::new_v4(), progress(shard, 3)); + apply(&mut after, &read, &[0], &read, true).unwrap(); + assert_eq!(coverage_for(&after, "idx"), Some(compacted(shard, 3))); + } + + /// The cap is the read version's progress, not this commit's. A + /// compaction that landed while the index was being built put its rows + /// in fragments this transaction never inspected, so covering + /// everything it *did* read earns only what had been folded in by then. + #[test] + fn credit_never_reaches_past_the_read_version() { + let shard = Uuid::new_v4(); + // Read at generation 2; generation 5 landed while this ran. + let read = table(&[0], Uuid::new_v4(), progress(shard, 2)); + let mut after = table(&[0], Uuid::new_v4(), progress(shard, 5)); + apply(&mut after, &read, &[0], &read, true).unwrap(); + assert_eq!(coverage_for(&after, "idx"), Some(compacted(shard, 2))); + } + + /// One segment with an unknown bitmap makes the whole index unproven, + /// even when its siblings happen to span everything. Coverage that + /// cannot be read is not coverage that can be relied on. + #[test] + fn an_index_with_an_unknown_segment_is_not_credited() { + let shard = Uuid::new_v4(); + let mut unknown = user_index("idx", Uuid::new_v4(), &[]); + unknown.fragment_bitmap = None; + let read = vec![ + user_index("idx", Uuid::new_v4(), &[0, 1]), + unknown, + mem_wal_index(progress(shard, 5)), + ]; + let mut after = read.clone(); + apply(&mut after, &read, &[0, 1], &read, true).unwrap(); + assert_eq!(coverage_for(&after, "idx"), None); + } + + /// A dropped index has no coverage left to gate anything. + #[test] + fn a_dropped_index_loses_its_entry() { + let shard = Uuid::new_v4(); + let before = table(&[0], Uuid::new_v4(), progress_with_catchup(shard, 5, 5)); + let mut after = vec![mem_wal_index(progress_with_catchup(shard, 5, 5))]; + apply(&mut after, &before, &[0], &before, true).unwrap(); + assert_eq!(coverage_for(&after, "idx"), None); + } + + /// An index created by this commit is credited if it spans the read + /// version -- it was built over those fragments, so it holds their + /// rows. This is what the advance model could not express: an ordinary + /// build that fully covers had to throw its work away and wait. + #[test] + fn a_new_index_covering_the_read_version_is_credited() { + let shard = Uuid::new_v4(); + let before = vec![mem_wal_index(progress(shard, 5))]; + let mut after = table(&[0], Uuid::new_v4(), progress(shard, 5)); + // Covers the read version, but was not there when it was read. + apply(&mut after, &before, &[0], &before, true).unwrap(); + assert_eq!(coverage_for(&after, "idx"), Some(compacted(shard, 5))); + } + + /// A legacy table reads a missing entry as "fully caught up", so this + /// must leave it alone rather than make it look more covered. + #[test] + fn a_legacy_table_is_untouched() { + let shard = Uuid::new_v4(); + let before = table(&[0], Uuid::new_v4(), progress(shard, 5)); + let mut after = before.clone(); + let untouched = after.clone(); + apply(&mut after, &before, &[0], &before, false).unwrap(); + assert_eq!(after, untouched); + } + + /// Two shards, only one of them compacted. + #[test] + fn each_shard_is_credited_independently() { + let merged = Uuid::new_v4(); + let idle = Uuid::new_v4(); + let details = MemWalIndexDetails { + compacted_sstables: vec![ + CompactedSsTable::new(merged, 4), + CompactedSsTable::new(idle, 0), + ], + ..Default::default() + }; + let read = table(&[0], Uuid::new_v4(), details.clone()); + let mut after = table(&[0], Uuid::new_v4(), details); + apply(&mut after, &read, &[0], &read, true).unwrap(); + let coverage = coverage_for(&after, "idx").expect("credited"); + assert_eq!( + coverage + .iter() + .find(|g| g.shard_id == merged) + .map(|g| g.generation), + Some(4) + ); + assert_eq!( + coverage + .iter() + .find(|g| g.shard_id == idle) + .map(|g| g.generation), + Some(0) + ); + } + + /// Two indexes advance independently: one covering, one behind. + #[test] + fn indexes_are_credited_independently() { + let shard = Uuid::new_v4(); + let read = vec![ + user_index("fast", Uuid::new_v4(), &[0, 1]), + user_index("slow", Uuid::new_v4(), &[0]), + mem_wal_index(progress(shard, 6)), + ]; + let mut after = read.clone(); + apply(&mut after, &read, &[0, 1], &read, true).unwrap(); + assert_eq!(coverage_for(&after, "fast"), Some(compacted(shard, 6))); + assert_eq!(coverage_for(&after, "slow"), None); + } + + /// An index whose coverage is unknown cannot be shown to cover anything. + #[test] + fn an_index_without_a_bitmap_is_not_credited() { + let shard = Uuid::new_v4(); + let mut idx = user_index("idx", Uuid::new_v4(), &[0]); + idx.fragment_bitmap = None; + let read = vec![idx, mem_wal_index(progress(shard, 5))]; + let mut after = read.clone(); + apply(&mut after, &read, &[0], &read, true).unwrap(); + assert_eq!(coverage_for(&after, "idx"), None); + } + + /// Nothing compacted means nothing to be behind on. + #[test] + fn no_compaction_progress_writes_no_entries() { + let before = table(&[0], Uuid::new_v4(), MemWalIndexDetails::default()); + let mut after = before.clone(); + let untouched = after.clone(); + apply(&mut after, &before, &[0], &before, true).unwrap(); + assert_eq!(after, untouched); + } + + /// No MemWAL system index: nothing to maintain, and no error. + #[test] + fn a_table_without_mem_wal_is_a_no_op() { + let before = vec![user_index("idx", Uuid::new_v4(), &[0])]; + let mut after = before.clone(); + let untouched = after.clone(); + apply(&mut after, &before, &[0], &before, true).unwrap(); + assert_eq!(after, untouched); + } + + /// No read version -- dataset creation, detached commits -- credits + /// nothing and lowers nothing. + #[test] + fn without_a_read_version_nothing_changes() { + let shard = Uuid::new_v4(); + let uuid = Uuid::new_v4(); + let before = table(&[0], uuid, progress_with_catchup(shard, 5, 5)); + let mut after = before.clone(); + let segments_before = Transaction::logical_index_segments(&before); + Transaction::apply_mem_wal_index_coverage(&mut after, &segments_before, None, true, 2) + .unwrap(); + assert_eq!(coverage_for(&after, "idx"), Some(compacted(shard, 5))); + } + + /// An untrained index covers nothing that exists, so a sibling's work + /// is no evidence for it. + #[test] + fn an_untrained_index_earns_nothing() { + let shard = Uuid::new_v4(); + let read = vec![ + user_index("untrained", Uuid::new_v4(), &[]), + user_index("trained", Uuid::new_v4(), &[0]), + mem_wal_index(progress(shard, 10)), + ]; + let mut after = read.clone(); + apply(&mut after, &read, &[0], &read, true).unwrap(); + assert_eq!(coverage_for(&after, "untrained"), None); + assert_eq!(coverage_for(&after, "trained"), Some(compacted(shard, 10))); + } + + /// Shards move independently within one index: one advances on this + /// commit's proof while another keeps the position it already had. + #[test] + fn a_shard_keeps_its_position_while_another_advances() { + let (advancing, quiet) = (Uuid::new_v4(), Uuid::new_v4()); + let uuid = Uuid::new_v4(); + let details = |advancing_gen: u64| MemWalIndexDetails { + compacted_sstables: vec![ + CompactedSsTable::new(advancing, advancing_gen), + CompactedSsTable::new(quiet, 10), + ], + index_catchup: vec![IndexCatchupProgress::new( + "idx".to_string(), + vec![CompactedSsTable::new(quiet, 7)], + )], + ..Default::default() + }; + // The quiet shard was never compacted as of the read, so nothing + // this commit proves reaches it -- it keeps its recorded 7. + let read = vec![ + user_index("idx", uuid, &[0]), + mem_wal_index(MemWalIndexDetails { + compacted_sstables: vec![CompactedSsTable::new(advancing, 9)], + ..details(9) + }), + ]; + let mut after = vec![user_index("idx", uuid, &[0]), mem_wal_index(details(10))]; + apply(&mut after, &read, &[0], &read, true).unwrap(); + + let mut coverage = coverage_for(&after, "idx").expect("credited"); + coverage.sort_unstable_by_key(|sstable| sstable.shard_id); + let mut expected = vec![ + CompactedSsTable::new(advancing, 9), + CompactedSsTable::new(quiet, 7), + ]; + expected.sort_unstable_by_key(|sstable| sstable.shard_id); + assert_eq!(coverage, expected); + } + + /// The derivation drops coverage an index no longer earns, but it never + /// rejects the commit -- an ordinary index job must not be blocked by + /// a protocol it knows nothing about. + #[test] + fn an_ordinary_index_job_is_never_blocked() { + let shard = Uuid::new_v4(); + let before = table(&[0, 1], Uuid::new_v4(), progress_with_catchup(shard, 5, 5)); + // Rebuilt over a subset -- the shape a partial reindex leaves. + let mut after = table(&[0], Uuid::new_v4(), progress_with_catchup(shard, 5, 5)); + apply(&mut after, &before, &[0, 1], &before, true).unwrap(); + assert_eq!(coverage_for(&after, "idx"), None); + } + + /// A reader's rule is that a missing entry means "not caught up", so an + /// index caught up to nothing must be absent rather than present at + /// generation zero -- otherwise it reads as known-and-covered. + #[test] + fn an_index_caught_up_to_nothing_gets_no_entry() { + let shard = Uuid::new_v4(); + let uuid = Uuid::new_v4(); + let before = table(&[0], uuid, progress_with_catchup(shard, 5, 0)); + let mut after = before.clone(); + // Does not span the read version, so nothing lifts it off zero. + apply(&mut after, &before, &[0, 1], &before, true).unwrap(); + assert_eq!(coverage_for(&after, "idx"), None); + } + + /// Each shard carries its own position. Collapsing them to one value + /// would credit a lagging shard with a busier shard's progress. + #[test] + fn carried_positions_do_not_leak_between_shards() { + let (ahead, behind) = (Uuid::new_v4(), Uuid::new_v4()); + let uuid = Uuid::new_v4(); + let details = MemWalIndexDetails { + compacted_sstables: vec![ + CompactedSsTable::new(ahead, 10), + CompactedSsTable::new(behind, 10), + ], + index_catchup: vec![IndexCatchupProgress::new( + "idx".to_string(), + vec![ + CompactedSsTable::new(ahead, 8), + CompactedSsTable::new(behind, 2), + ], + )], + ..Default::default() + }; + let before = vec![user_index("idx", uuid, &[0]), mem_wal_index(details)]; + let mut after = before.clone(); + // Unchanged and unproven: both shards keep exactly what they had. + apply(&mut after, &before, &[0, 1], &before, true).unwrap(); + + let mut coverage = coverage_for(&after, "idx").expect("carried"); + coverage.sort_unstable_by_key(|sstable| sstable.shard_id); + let mut expected = vec![ + CompactedSsTable::new(ahead, 8), + CompactedSsTable::new(behind, 2), + ]; + expected.sort_unstable_by_key(|sstable| sstable.shard_id); + assert_eq!(coverage, expected); + } + + /// The derivation runs while the manifest is being built, but the + /// index list is not final there: `migrate_indices` recalculates a + /// segment's fragment bitmap and keeps its UUID. A position decided + /// before that must not survive the narrowing, or the WAL pod trims + /// against an index that no longer covers those rows. + #[test] + fn a_bitmap_narrowed_after_the_build_loses_its_position() { + let shard = Uuid::new_v4(); + let uuid = Uuid::new_v4(); + // What migrate_indices leaves behind: same UUID, fewer fragments, + // and it says so. + let mut migrated = table(&[0], uuid, progress_with_catchup(shard, 5, 5)); + + Transaction::withdraw_coverage_invalidated_after_build( + &mut migrated, + &["idx".to_string()], + 3, + ) + .unwrap(); + + assert_eq!(coverage_for(&migrated, "idx"), None); + } + + /// Migration routinely fills in file lists and inferred details. Those + /// do not change which rows an index answers for, so withdrawing on + /// them would drop coverage every commit for no reason. + #[test] + fn metadata_migration_that_does_not_narrow_keeps_its_position() { + let shard = Uuid::new_v4(); + let uuid = Uuid::new_v4(); + let mut migrated = table(&[0, 1], uuid, progress_with_catchup(shard, 5, 5)); + migrated[0].files = Some(Vec::new()); + migrated[0].created_at = Some(chrono::Utc::now()); + + // Nothing narrowed, so migration reports nothing. + Transaction::withdraw_coverage_invalidated_after_build(&mut migrated, &[], 3).unwrap(); + + assert_eq!(coverage_for(&migrated, "idx"), Some(compacted(shard, 5))); + } + + /// A commit that changes nothing must not churn the system index: a new + /// UUID on every append would invalidate its cache entry fleet-wide. + #[test] + fn an_unchanged_commit_does_not_rewrite_the_system_index() { + let shard = Uuid::new_v4(); + let uuid = Uuid::new_v4(); + let before = table(&[0], uuid, progress_with_catchup(shard, 5, 5)); + let mut after = before.clone(); + apply(&mut after, &before, &[0], &before, true).unwrap(); + + let system_uuid = |indices: &[IndexMetadata]| { + indices + .iter() + .find(|idx| idx.name == MEM_WAL_INDEX_NAME) + .unwrap() + .uuid + }; + assert_eq!(system_uuid(&after), system_uuid(&before)); + } + + /// Activation is what puts a table on the protocol. A table that has + /// never compacted is clean. + #[test] + fn activation_accepts_a_clean_table() { + let mut indices = vec![mem_wal_index(MemWalIndexDetails::default())]; + Transaction::require_index_catchup(&mut indices, 2).unwrap(); + } + + /// There is nothing to put on the protocol. + #[test] + fn activation_requires_the_mem_wal_index() { + let err = Transaction::require_index_catchup(&mut [], 2).unwrap_err(); + assert!(err.to_string().contains("does not exist"), "{err}"); + } + + /// Coverage recorded under the beta rules was written to a different + /// contract; keeping it would let the first trim run unchecked. + #[test] + fn activation_clears_beta_coverage() { + let shard = Uuid::new_v4(); + let mut indices = vec![mem_wal_index(MemWalIndexDetails { + index_catchup: vec![IndexCatchupProgress::new( + "idx".to_string(), + compacted(shard, 100), + )], + ..Default::default() + })]; + + Transaction::require_index_catchup(&mut indices, 2).unwrap(); + + assert!( + load_mem_wal_index_details(indices[0].clone()) + .unwrap() + .index_catchup + .is_empty() + ); + } + + /// Beta compaction progress means SSTables were folded in without any + /// coverage rule. No later commit can prove which indexes hold them. + #[test] + fn activation_rejects_pre_existing_beta_compaction_progress() { + let mut indices = vec![mem_wal_index(progress(Uuid::new_v4(), 4))]; + let err = Transaction::require_index_catchup(&mut indices, 2).unwrap_err(); + assert!(err.to_string().contains("beta protocol"), "{err}"); + } + + fn config_transaction(current: &Manifest) -> Transaction { + Transaction::new( + current.version, + Operation::UpdateConfig { + config_updates: None, + table_metadata_updates: None, + schema_metadata_updates: None, + field_metadata_updates: HashMap::new(), + }, + None, + ) + } + + /// One bit without the other is a manifest no writer should produce: + /// a reader-only bit lets an unaware writer trim, a writer-only bit + /// lets an unaware reader serve rows no index holds. + #[test] + fn a_half_set_feature_bit_is_refused() { + for (reader, writer) in [ + (FLAG_MEM_WAL_INDEX_CATCHUP, 0), + (0, FLAG_MEM_WAL_INDEX_CATCHUP), + ] { + let mut current = sample_manifest_with_fragments(0..1); + current.reader_feature_flags = reader; + current.writer_feature_flags = writer; + + let err = config_transaction(¤t) + .build_manifest( + Some(¤t), + vec![mem_wal_index(MemWalIndexDetails::default())], + "txn", + &ManifestWriteConfig::default(), + ) + .unwrap_err(); + + assert!(err.to_string().contains("only one of"), "{err}"); + } + } + + /// A writer that knows nothing about catch-up must not silently take a + /// table off the protocol. + #[test] + fn an_ordinary_commit_keeps_the_feature_bit() { + let mut current = sample_manifest_with_fragments(0..1); + current.reader_feature_flags = FLAG_MEM_WAL_INDEX_CATCHUP; + current.writer_feature_flags = FLAG_MEM_WAL_INDEX_CATCHUP; + + let (next, _) = config_transaction(¤t) + .build_manifest( + Some(¤t), + vec![mem_wal_index(MemWalIndexDetails::default())], + "txn", + &ManifestWriteConfig::default(), + ) + .unwrap(); + + assert_ne!(next.reader_feature_flags & FLAG_MEM_WAL_INDEX_CATCHUP, 0); + assert_ne!(next.writer_feature_flags & FLAG_MEM_WAL_INDEX_CATCHUP, 0); + } + + /// A commit with no read version still withdraws. It can prove nothing, + /// so an index it changed keeps no position -- the alternative leaves a + /// position describing an index that no longer exists. + #[test] + fn without_a_read_version_a_changed_index_still_loses_its_position() { + let shard = Uuid::new_v4(); + let before = table(&[0, 1], Uuid::new_v4(), progress_with_catchup(shard, 5, 5)); + let mut after = table(&[0], Uuid::new_v4(), progress_with_catchup(shard, 5, 5)); + let segments_before = Transaction::logical_index_segments(&before); + Transaction::apply_mem_wal_index_coverage(&mut after, &segments_before, None, true, 2) + .unwrap(); + assert_eq!(coverage_for(&after, "idx"), None); + } + + /// Two attempts against the same read version agree, which is what makes + /// a rebase safe: `read_version` is fixed for a transaction's life. + #[test] + fn the_derivation_is_stable_across_attempts() { + let shard = Uuid::new_v4(); + let read = table(&[0, 1], Uuid::new_v4(), progress(shard, 5)); + let mut first = table(&[0, 1], Uuid::new_v4(), progress(shard, 5)); + let mut second = first.clone(); + apply(&mut first, &read, &[0, 1], &read, true).unwrap(); + apply(&mut second, &read, &[0, 1], &read, true).unwrap(); + assert_eq!(coverage_for(&first, "idx"), coverage_for(&second, "idx")); + } + } } diff --git a/rust/lance/src/dataset/udtf.rs b/rust/lance/src/dataset/udtf.rs index 75c0388bc24..2144b108859 100644 --- a/rust/lance/src/dataset/udtf.rs +++ b/rust/lance/src/dataset/udtf.rs @@ -13,7 +13,6 @@ use lance_core::{Error, ROW_ADDR_FIELD, ROW_ID_FIELD}; use lance_index::scalar::FullTextSearchQuery; use lance_index::scalar::inverted::parser::from_json; use serde_json::Value; -use std::any::Any; use std::collections::HashMap; use std::fmt::Debug; use std::sync::Arc; @@ -61,10 +60,6 @@ impl FtsTableProvider { #[async_trait] impl TableProvider for FtsTableProvider { - fn as_any(&self) -> &dyn Any { - self - } - fn schema(&self) -> SchemaRef { self.full_schema.clone() } diff --git a/rust/lance/src/dataset/updater.rs b/rust/lance/src/dataset/updater.rs index 752a479e5b0..c5cd8db9600 100644 --- a/rust/lance/src/dataset/updater.rs +++ b/rust/lance/src/dataset/updater.rs @@ -12,7 +12,8 @@ use lance_table::utils::stream::ReadBatchFutStream; use super::Dataset; use super::fragment::FragmentReader; use super::scanner::get_default_batch_size; -use super::write::{GenericWriter, cleanup_data_fragments, open_update_writer}; +use super::versions; +use super::write::{GenericWriter, cleanup_data_fragments}; use crate::dataset::FileFragment; use crate::dataset::utils::SchemaAdapter; @@ -46,6 +47,8 @@ pub struct Updater { /// The adapter to convert the logical data to physical data. schema_adapter: Option, + allow_external_blob_outside_bases: bool, + finished: bool, deletion_restorer: DeletionRestorer, @@ -72,7 +75,13 @@ impl Updater { (None, None) }; - let legacy_batch_size = reader.legacy_num_rows_in_batch(0); + let storage_version = fragment + .dataset() + .manifest() + .data_storage_format + .lance_file_format(); + let legacy_batch_size = + versions::row_group_size_for_rewrite(storage_version, &fragment).await?; let batch_size = match (&legacy_batch_size, batch_size) { // If this is a v1 dataset we must use the row group size of the file @@ -95,6 +104,7 @@ impl Updater { // The schema adapter needs the data schema, not the logical schema, so it can't be // created until after the first batch is read. schema_adapter: None, + allow_external_blob_outside_bases: false, finished: false, deletion_restorer: DeletionRestorer::new(deletion_vector, legacy_batch_size), }) @@ -144,9 +154,21 @@ impl Updater { .dataset() .manifest() .data_storage_format - .lance_file_version()?; + .lance_file_format(); + + versions::open_update_writer( + data_storage_version, + self.dataset(), + &schema, + self.allow_external_blob_outside_bases, + ) + .await + } - open_update_writer(self.dataset(), &schema, data_storage_version).await + /// Allow trusted existing external blob references to pass through an update rewrite. + /// Callers must separately validate any newly supplied references before writing. + pub(super) fn allow_external_blob_outside_bases(&mut self) { + self.allow_external_blob_outside_bases = true; } /// Update one batch. @@ -229,12 +251,22 @@ impl Updater { } let mut fragment = Fragment::new(self.fragment.id() as u64); + let storage_version = self + .dataset() + .manifest() + .data_storage_format + .lance_file_format(); // cleanup_data_fragments only needs path/base_id to remove the unfinished // data file and any blob sidecars. Build a minimal synthetic fragment so // we can reuse the shared cleanup path without fabricating full metadata. - fragment - .files - .push(DataFile::new(path, vec![], vec![], 0, 0, None, base_id)); + fragment.files.push(DataFile::new( + path, + vec![], + vec![], + storage_version, + None, + base_id, + )); cleanup_data_fragments( &self.dataset().object_store, &self.dataset().base, diff --git a/rust/lance/src/dataset/versions/mod.rs b/rust/lance/src/dataset/versions/mod.rs new file mode 100644 index 00000000000..8eac709d74a --- /dev/null +++ b/rust/lance/src/dataset/versions/mod.rs @@ -0,0 +1,871 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright The Lance Authors + +//! Dataset policies that differ across exact Lance file versions. +//! +//! File grammar belongs to `lance_file::versions`. This module contains only +//! operation-level dataset choices whose behavior actually differs by version. + +use std::{collections::HashMap, ops::Range, sync::Arc}; + +use arrow_schema::{DataType, Field as ArrowField}; +use datafusion::execution::SendableRecordBatchStream; +use datafusion::physical_plan::ExecutionPlan; +use datafusion::physical_plan::metrics::ExecutionPlanMetricsSet; +use futures::{StreamExt, TryStreamExt}; +use lance_arrow::DataTypeExt; +use lance_core::{ + Error, Result, + datatypes::{Field, Projection, Schema, SchemaCompareOptions}, +}; +use lance_datafusion::chunker::{break_stream, chunk_stream}; +use lance_file::{ + version::ConcreteFileVersion, + versions as file_versions, + writer::{FileWriter, FileWriterOptions}, +}; +use lance_index::scalar::seed::IndexSeedWriter; +use lance_io::object_store::ObjectStore; +use lance_io::traits::Writer as ObjectWriter; +use lance_table::format::{DataFile, DataStorageFormat, Fragment, Manifest}; +use object_store::path::Path; + +use super::Dataset; +use super::fragment::{ + FileFragment, FragReadConfig, GenericFileReader, MetadataMode, V1FragmentReader, + write::FragmentCreateBuilder, +}; +use super::optimize::CompactionOptions; +use super::scanner::{PlannedFilteredScan, Scanner}; +use super::schema_evolution::optimize::{ + ChainedNewColumnTransformOptimizer, SqlToAllNullsOptimizer, +}; +use super::statistics::FieldStatistics; +use super::utils::SchemaAdapter; +use super::write::{self, GenericWriter, TargetBaseInfo, WriteParams, WriterOptions}; +use crate::io::exec::filtered_read::{FilteredReadExec, FilteredReadOptions}; +use crate::io::exec::{ + AddRowAddrExec, FilterPlan as ExprFilterPlan, LanceScanConfig, LanceStream, TakeExec, +}; + +#[allow(clippy::too_many_arguments)] +pub fn create_scan_stream( + version: ConcreteFileVersion, + dataset: Arc, + fragments: Arc>, + offsets: Option>, + projection: Arc, + config: LanceScanConfig, + metrics: &ExecutionPlanMetricsSet, + partition: usize, +) -> datafusion::error::Result { + match version { + ConcreteFileVersion::V1 => LanceStream::try_new_v1( + dataset, fragments, offsets, projection, config, metrics, partition, + ), + ConcreteFileVersion::V2_0 + | ConcreteFileVersion::V2_1 + | ConcreteFileVersion::V2_2 + | ConcreteFileVersion::V2_3 => LanceStream::try_new_v2( + dataset, fragments, offsets, projection, config, metrics, partition, + ), + } +} + +pub fn schema_compare_options(version: ConcreteFileVersion) -> SchemaCompareOptions { + match version { + ConcreteFileVersion::V1 => SchemaCompareOptions { + compare_dictionary: true, + ..Default::default() + }, + ConcreteFileVersion::V2_0 + | ConcreteFileVersion::V2_1 + | ConcreteFileVersion::V2_2 + | ConcreteFileVersion::V2_3 => SchemaCompareOptions::default(), + } +} + +async fn create_seed_writers( + version: ConcreteFileVersion, + dataset: Option<&Dataset>, + params: &WriteParams, +) -> Result>> { + match version { + ConcreteFileVersion::V1 => Ok(Vec::new()), + ConcreteFileVersion::V2_0 + | ConcreteFileVersion::V2_1 + | ConcreteFileVersion::V2_2 + | ConcreteFileVersion::V2_3 => write::create_seed_writers_current(dataset, params).await, + } +} + +fn create_current_file_writer( + version: ConcreteFileVersion, + object_writer: Box, + schema: Schema, + filename: String, + base_id: Option, +) -> Result<(FileWriter, DataFile)> { + let writer = + file_versions::create_writer(version, object_writer, schema, FileWriterOptions::default())?; + let mut data_file = DataFile::new_unstarted(filename, version); + data_file.base_id = base_id; + Ok((writer, data_file)) +} + +#[allow(clippy::too_many_arguments)] +pub async fn write_fragments( + version: ConcreteFileVersion, + dataset: Option<&Dataset>, + object_store: Arc, + base_dir: &Path, + normalized_schema: Schema, + data: SendableRecordBatchStream, + params: WriteParams, + target_bases_info: Option>, +) -> Result<(Vec, Schema)> { + let version_name = format!("{version:?}"); + let schema = write::prepare_write_schema( + dataset, + normalized_schema, + ¶ms, + schema_compare_options(version), + )?; + match version { + ConcreteFileVersion::V1 | ConcreteFileVersion::V2_0 | ConcreteFileVersion::V2_1 => { + write::validate_legacy_blob_write_schema(&schema, &version_name)?; + } + ConcreteFileVersion::V2_2 | ConcreteFileVersion::V2_3 => { + write::validate_blob_v2_write_schema(&schema)?; + } + } + let seed_writers = create_seed_writers(version, dataset, ¶ms).await?; + let fragments = write_fragments_direct( + version, + dataset, + object_store, + base_dir, + &schema, + data, + params, + target_bases_info, + seed_writers, + ) + .await?; + Ok((fragments, schema)) +} + +#[allow(clippy::too_many_arguments)] +pub async fn write_fragments_direct( + version: ConcreteFileVersion, + dataset: Option<&Dataset>, + object_store: Arc, + base_dir: &Path, + schema: &Schema, + data: SendableRecordBatchStream, + params: WriteParams, + target_bases_info: Option>, + seed_writers: Vec>, +) -> Result> { + let adapter = SchemaAdapter::new(data.schema()); + let data = adapter.to_physical_stream(data); + let buffered_reader = match version { + ConcreteFileVersion::V1 => chunk_stream(data, params.max_rows_per_group), + ConcreteFileVersion::V2_0 + | ConcreteFileVersion::V2_1 + | ConcreteFileVersion::V2_2 + | ConcreteFileVersion::V2_3 => break_stream(data, params.max_rows_per_file) + .map_ok(|batch| vec![batch]) + .boxed(), + }; + let external_base_resolver = match version { + ConcreteFileVersion::V2_2 | ConcreteFileVersion::V2_3 => { + write::blob_v2_external_base_resolver(dataset, ¶ms, schema).await? + } + ConcreteFileVersion::V1 | ConcreteFileVersion::V2_0 | ConcreteFileVersion::V2_1 => None, + }; + write::do_write_fragments_impl( + dataset, + object_store, + base_dir, + schema, + buffered_reader, + params, + move |object_store, schema, base_dir, options| async move { + open_writer(version, &object_store, &schema, &base_dir, options).await + }, + external_base_resolver, + target_bases_info, + seed_writers, + ) + .await +} + +fn binary_copy_files_match(fragments: &[Fragment], expected: ConcreteFileVersion) -> Result { + for fragment in fragments { + for data_file in &fragment.files { + if data_file.file_version()? != expected { + return Ok(false); + } + } + } + Ok(true) +} + +pub async fn can_use_binary_copy( + version: ConcreteFileVersion, + dataset: &Dataset, + options: &CompactionOptions, + fragments: &[Fragment], +) -> Result { + match version { + ConcreteFileVersion::V1 => Ok(false), + ConcreteFileVersion::V2_0 + | ConcreteFileVersion::V2_1 + | ConcreteFileVersion::V2_2 + | ConcreteFileVersion::V2_3 => { + if !binary_copy_files_match(fragments, version)? { + return Ok(false); + } + super::optimize::can_use_binary_copy_current(dataset, options, fragments).await + } + } +} + +pub async fn rewrite_files_binary_copy( + version: ConcreteFileVersion, + dataset: &Dataset, + fragments: &[Fragment], + params: &WriteParams, + read_batch_bytes: Option, +) -> Result> { + match version { + ConcreteFileVersion::V1 => Err(Error::not_supported( + "binary-copy compaction is not supported for Lance file version 1".to_string(), + )), + ConcreteFileVersion::V2_0 + | ConcreteFileVersion::V2_1 + | ConcreteFileVersion::V2_2 + | ConcreteFileVersion::V2_3 => { + super::optimize::binary_copy::rewrite_files_binary_copy( + version, + dataset, + fragments, + params, + read_batch_bytes, + ) + .await + } + } +} + +pub fn check_manifest_storage_version(manifest: &mut Manifest) -> Result<()> { + let version = manifest.data_storage_format.lance_file_format(); + match version { + ConcreteFileVersion::V1 => repair_legacy_manifest_storage(manifest), + ConcreteFileVersion::V2_0 + | ConcreteFileVersion::V2_1 + | ConcreteFileVersion::V2_2 + | ConcreteFileVersion::V2_3 => validate_exact_manifest_storage(manifest, version), + } +} + +pub fn validate_column_indices(manifest: &Manifest) -> Result<()> { + match manifest.data_storage_format.lance_file_format() { + ConcreteFileVersion::V1 | ConcreteFileVersion::V2_0 => Ok(()), + ConcreteFileVersion::V2_1 | ConcreteFileVersion::V2_2 | ConcreteFileVersion::V2_3 => { + validate_leaf_column_indices(manifest) + } + } +} + +fn validate_leaf_column_indices(manifest: &Manifest) -> Result<()> { + for fragment in manifest.fragments.iter() { + for data_file in &fragment.files { + let file_version = data_file.file_version()?; + if file_version == ConcreteFileVersion::V1 || data_file.column_indices.is_empty() { + continue; + } + if data_file.fields.len() != data_file.column_indices.len() { + return Err(Error::invalid_input(format!( + "Data file '{}' (fragment {}) has {} field ids but {} column indices. These must be the same length.", + data_file.path, + fragment.id, + data_file.fields.len(), + data_file.column_indices.len() + ))); + } + if file_version == ConcreteFileVersion::V2_0 { + continue; + } + for (field_id, column_index) in + data_file.fields.iter().zip(data_file.column_indices.iter()) + { + let Some(field) = manifest.schema.field_by_id(*field_id) else { + continue; + }; + let needs_column = field.is_leaf() || field.is_packed_struct() || field.is_blob(); + if needs_column && *column_index == -1 { + return Err(Error::invalid_input(format!( + "Field '{}' (id={}) in data file '{}' (fragment {}) has column_index=-1, but leaf fields, packed structs, and blob fields must have a valid column index in file format 2.1+.", + field.name, field_id, data_file.path, fragment.id + ))); + } + if !needs_column && *column_index != -1 { + return Err(Error::invalid_input(format!( + "Non-leaf field '{}' (id={}) in data file '{}' (fragment {}) has column_index={}, but non-leaf fields should have column_index=-1 in file format 2.1+.", + field.name, field_id, data_file.path, fragment.id, column_index + ))); + } + } + } + } + Ok(()) +} + +pub fn validate_fragment_schema( + version: ConcreteFileVersion, + schema: &Schema, + fragments: &[Fragment], +) -> Result<()> { + match version { + ConcreteFileVersion::V1 => { + super::transaction::schema_fragments_legacy_valid(schema, fragments) + } + ConcreteFileVersion::V2_0 + | ConcreteFileVersion::V2_1 + | ConcreteFileVersion::V2_2 + | ConcreteFileVersion::V2_3 => { + super::transaction::schema_fragments_modern_valid(schema, fragments) + } + } +} + +pub async fn write_fragment( + version: ConcreteFileVersion, + builder: &FragmentCreateBuilder<'_>, + stream: SendableRecordBatchStream, + schema: Schema, + id: u64, +) -> Result { + match version { + ConcreteFileVersion::V1 => builder.write_v1_impl(stream, schema, id).await, + ConcreteFileVersion::V2_0 + | ConcreteFileVersion::V2_1 + | ConcreteFileVersion::V2_2 + | ConcreteFileVersion::V2_3 => { + builder + .write_current_impl( + move |object_writer, schema, filename| { + create_current_file_writer(version, object_writer, schema, filename, None) + }, + stream, + schema, + id, + ) + .await + } + } +} + +pub async fn open_writer( + version: ConcreteFileVersion, + object_store: &ObjectStore, + schema: &Schema, + base_dir: &Path, + options: WriterOptions, +) -> Result> { + match version { + ConcreteFileVersion::V1 => { + write::open_v1_writer(object_store, schema, base_dir, options).await + } + ConcreteFileVersion::V2_0 | ConcreteFileVersion::V2_1 => { + write::open_current_writer( + move |object_writer, schema, filename, base_id| { + create_current_file_writer(version, object_writer, schema, filename, base_id) + }, + object_store, + schema, + base_dir, + options, + ) + .await + } + ConcreteFileVersion::V2_2 | ConcreteFileVersion::V2_3 => { + write::open_current_blob_v2_writer( + move |object_writer, schema, filename, base_id| { + create_current_file_writer(version, object_writer, schema, filename, base_id) + }, + object_store, + schema, + base_dir, + options, + ) + .await + } + } +} + +pub async fn open_update_writer( + version: ConcreteFileVersion, + dataset: &Dataset, + schema: &Schema, + allow_external_blob_outside_bases: bool, +) -> Result> { + let external_base_resolver = match version { + ConcreteFileVersion::V2_2 | ConcreteFileVersion::V2_3 => { + write::blob_v2_external_base_resolver(Some(dataset), &WriteParams::default(), schema) + .await? + } + ConcreteFileVersion::V1 | ConcreteFileVersion::V2_0 | ConcreteFileVersion::V2_1 => None, + }; + open_writer( + version, + &dataset.object_store, + schema, + &dataset.base, + WriterOptions::update( + dataset.session.store_registry(), + external_base_resolver, + allow_external_blob_outside_bases, + ), + ) + .await +} + +pub async fn create_fragment_from_file( + file_version: ConcreteFileVersion, + dataset_version: ConcreteFileVersion, + filename: &str, + dataset: &Dataset, + fragment_id: usize, + physical_rows: Option, +) -> Result { + if file_version != dataset_version { + return Err(Error::invalid_input(format!( + "File version mismatch. Dataset version: {:?} Fragment version: {:?}", + dataset_version, file_version + ))); + } + match file_version { + ConcreteFileVersion::V1 => { + FileFragment::create_from_v1_file(filename, dataset, fragment_id, physical_rows).await + } + ConcreteFileVersion::V2_0 + | ConcreteFileVersion::V2_1 + | ConcreteFileVersion::V2_2 + | ConcreteFileVersion::V2_3 => { + FileFragment::create_from_current_file(filename, dataset, fragment_id).await + } + } +} + +pub fn index_file_version(version: ConcreteFileVersion) -> ConcreteFileVersion { + match version { + ConcreteFileVersion::V1 | ConcreteFileVersion::V2_0 => ConcreteFileVersion::V2_0, + ConcreteFileVersion::V2_1 => ConcreteFileVersion::V2_1, + ConcreteFileVersion::V2_2 => ConcreteFileVersion::V2_2, + ConcreteFileVersion::V2_3 => ConcreteFileVersion::V2_3, + } +} + +pub async fn open_file_reader( + version: ConcreteFileVersion, + fragment: &FileFragment, + data_file: &DataFile, + projection: Option<&Schema>, + read_config: &FragReadConfig, + metadata_mode: MetadataMode, +) -> Result>> { + match version { + ConcreteFileVersion::V1 => fragment.open_v1_file_reader(data_file, projection).await, + ConcreteFileVersion::V2_0 + | ConcreteFileVersion::V2_1 + | ConcreteFileVersion::V2_2 + | ConcreteFileVersion::V2_3 => { + fragment + .open_current_file_reader(data_file, projection, read_config, metadata_mode) + .await + } + } +} + +pub async fn open_v1_fragment_reader( + fragment: &FileFragment, + projection: &Schema, + read_config: &FragReadConfig, +) -> Result { + for data_file in &fragment.metadata().files { + let actual = data_file.file_version()?; + if actual != ConcreteFileVersion::V1 { + return Err(Error::invalid_input(format!( + "Cannot open file {} with the v1 reader because it has version {}", + data_file.path, actual + ))); + } + } + fragment + .open_v1_fragment_reader(projection, read_config) + .await +} + +pub async fn row_group_size_for_rewrite( + version: ConcreteFileVersion, + fragment: &FileFragment, +) -> Result> { + match version { + ConcreteFileVersion::V1 => { + let reader = open_v1_fragment_reader( + fragment, + fragment.dataset().schema(), + &FragReadConfig::default(), + ) + .await?; + Ok(reader.num_rows_in_batch(0)) + } + ConcreteFileVersion::V2_0 + | ConcreteFileVersion::V2_1 + | ConcreteFileVersion::V2_2 + | ConcreteFileVersion::V2_3 => Ok(None), + } +} + +pub fn is_upcast_downcast( + version: ConcreteFileVersion, + from_type: &DataType, + to_type: &DataType, +) -> bool { + is_upcast_downcast_impl( + from_type, + to_type, + !matches!(version, ConcreteFileVersion::V1), + ) +} + +fn is_upcast_downcast_impl( + from_type: &DataType, + to_type: &DataType, + dictionary_materialization: bool, +) -> bool { + use DataType::*; + match (from_type, to_type) { + (_, Dictionary(_, _)) if !dictionary_materialization => false, + (Dictionary(_, from_value_type), _) => { + is_upcast_downcast_impl(from_value_type, to_type, dictionary_materialization) + } + (_, Dictionary(_, to_value_type)) => { + is_upcast_downcast_impl(from_type, to_value_type, dictionary_materialization) + } + (from, to) if from.is_integer() => to.is_integer(), + (from, to) if from.is_floating() => to.is_floating(), + (from, to) if from.is_temporal() => to.is_temporal(), + (Boolean, to) => matches!(to, Boolean), + (Utf8 | LargeUtf8, to) => matches!(to, Utf8 | LargeUtf8), + (Binary | LargeBinary, to) => matches!(to, Binary | LargeBinary), + (Decimal128(_, _) | Decimal256(_, _), to) => { + matches!(to, Decimal128(_, _) | Decimal256(_, _)) + } + (List(from_field) | LargeList(from_field) | FixedSizeList(from_field, _), to_type) => { + match to_type { + List(to_field) | LargeList(to_field) | FixedSizeList(to_field, _) => { + is_upcast_downcast_impl( + from_field.data_type(), + to_field.data_type(), + dictionary_materialization, + ) + } + _ => false, + } + } + _ => false, + } +} + +pub fn validate_nulls( + version: ConcreteFileVersion, + datatype: &DataType, + has_nulls: bool, +) -> Result<()> { + let supported = match version { + ConcreteFileVersion::V1 => matches!( + datatype, + DataType::Utf8 + | DataType::LargeUtf8 + | DataType::Binary + | DataType::List(_) + | DataType::FixedSizeBinary(_) + | DataType::FixedSizeList(_, _) + ), + ConcreteFileVersion::V2_0 => !matches!(datatype, DataType::Struct(..)), + ConcreteFileVersion::V2_1 | ConcreteFileVersion::V2_2 | ConcreteFileVersion::V2_3 => true, + }; + if has_nulls && !supported { + return Err(Error::invalid_input(format!( + "Join produced null values for type: {:?}, but storing nulls for this data type is not supported by the dataset's current Lance file format version: {:?}. This can be caused by an explicit null in the new data.", + datatype, version + ))); + } + Ok(()) +} + +fn reject_nested_column_add(field: &ArrowField, version: ConcreteFileVersion) -> Result<()> { + Err(Error::invalid_input(format!( + "Column {} is a struct col, add sub column is not supported in Lance file version {}", + field.name(), + version + ))) +} + +fn reject_nested_v1(field: &ArrowField) -> Result<()> { + reject_nested_column_add(field, ConcreteFileVersion::V1) +} + +fn reject_nested_v2_0(field: &ArrowField) -> Result<()> { + reject_nested_column_add(field, ConcreteFileVersion::V2_0) +} + +fn reject_nested_v2_1(field: &ArrowField) -> Result<()> { + reject_nested_column_add(field, ConcreteFileVersion::V2_1) +} + +fn allow_nested(_field: &ArrowField) -> Result<()> { + Ok(()) +} + +pub fn check_field_conflict( + version: ConcreteFileVersion, + left: &ArrowField, + right: &ArrowField, +) -> Result<()> { + let validate = match version { + ConcreteFileVersion::V1 => reject_nested_v1, + ConcreteFileVersion::V2_0 => reject_nested_v2_0, + ConcreteFileVersion::V2_1 => reject_nested_v2_1, + ConcreteFileVersion::V2_2 | ConcreteFileVersion::V2_3 => allow_nested, + }; + super::schema_evolution::check_field_conflict_with(left, right, validate) +} + +fn exclude_struct_field(field: &Field, other: &Field) -> Option { + field + .data_type() + .is_struct() + .then(|| field.exclude(other)) + .flatten() +} + +fn exclude_nested_field(field: &Field, other: &Field) -> Option { + field + .data_type() + .is_nested() + .then(|| field.exclude(other)) + .flatten() +} + +pub fn exclude_schema( + version: ConcreteFileVersion, + source: &Schema, + other: &Schema, +) -> Result { + let exclude = match version { + ConcreteFileVersion::V1 | ConcreteFileVersion::V2_0 | ConcreteFileVersion::V2_1 => { + exclude_struct_field + } + ConcreteFileVersion::V2_2 | ConcreteFileVersion::V2_3 => exclude_nested_field, + }; + super::schema_evolution::exclude_with(source, other, exclude) +} + +pub fn configure_new_column_optimizers( + version: ConcreteFileVersion, + optimizer: &mut ChainedNewColumnTransformOptimizer, +) { + match version { + ConcreteFileVersion::V1 => {} + ConcreteFileVersion::V2_0 + | ConcreteFileVersion::V2_1 + | ConcreteFileVersion::V2_2 + | ConcreteFileVersion::V2_3 => { + optimizer.add_optimizer(Box::new(SqlToAllNullsOptimizer::new())); + } + } +} + +pub fn validate_metadata_only_null_columns(version: ConcreteFileVersion) -> Result<()> { + match version { + ConcreteFileVersion::V1 => Err(Error::not_supported_source( + "Cannot add all-null columns to legacy dataset version.".into(), + )), + ConcreteFileVersion::V2_0 + | ConcreteFileVersion::V2_1 + | ConcreteFileVersion::V2_2 + | ConcreteFileVersion::V2_3 => Ok(()), + } +} + +#[allow(clippy::too_many_arguments)] +pub(in crate::dataset) async fn filtered_read( + version: ConcreteFileVersion, + scanner: &Scanner, + filter_plan: &ExprFilterPlan, + projection: Projection, + make_deletions_null: bool, + fragments: Option>>, + scan_range: Option>, + is_prefilter: bool, +) -> Result { + match version { + ConcreteFileVersion::V1 => { + scanner + .legacy_filtered_read( + filter_plan, + projection, + make_deletions_null, + fragments, + scan_range, + is_prefilter, + ) + .await + } + ConcreteFileVersion::V2_0 + | ConcreteFileVersion::V2_1 + | ConcreteFileVersion::V2_2 + | ConcreteFileVersion::V2_3 => { + let limit_pushed_down = scan_range.is_some(); + let plan = scanner + .new_filtered_read( + filter_plan, + projection, + make_deletions_null, + fragments, + scan_range, + ) + .await?; + Ok(PlannedFilteredScan { + filter_pushed_down: true, + limit_pushed_down, + plan, + }) + } + } +} + +pub fn take( + version: ConcreteFileVersion, + scanner: &Scanner, + input: Arc, + output_projection: Projection, +) -> Result> { + match version { + ConcreteFileVersion::V1 => scanner.take_legacy(input, output_projection), + ConcreteFileVersion::V2_0 + | ConcreteFileVersion::V2_1 + | ConcreteFileVersion::V2_2 + | ConcreteFileVersion::V2_3 => scanner.take_current(input, output_projection), + } +} + +pub async fn collect_data_stats( + version: ConcreteFileVersion, + dataset: &Arc, + field_stats: &mut HashMap, +) -> Result<()> { + match version { + ConcreteFileVersion::V1 => Ok(()), + ConcreteFileVersion::V2_0 + | ConcreteFileVersion::V2_1 + | ConcreteFileVersion::V2_2 + | ConcreteFileVersion::V2_3 => { + super::statistics::collect_current_data_stats(dataset, field_stats).await + } + } +} + +pub fn merge_insert_indexed_take( + version: ConcreteFileVersion, + dataset: Arc, + mut index_mapper: Arc, + projection: Projection, + add_row_addr: bool, +) -> Result> { + match version { + ConcreteFileVersion::V1 => { + if add_row_addr { + let position = index_mapper.schema().fields().len(); + index_mapper = Arc::new(AddRowAddrExec::try_new( + index_mapper, + dataset.clone(), + position, + )?); + } + Ok(Arc::new( + TakeExec::try_new(dataset, index_mapper, projection)?.ok_or_else(|| { + Error::internal("merge-insert legacy take unexpectedly needed no columns") + })?, + )) + } + ConcreteFileVersion::V2_0 + | ConcreteFileVersion::V2_1 + | ConcreteFileVersion::V2_2 + | ConcreteFileVersion::V2_3 => { + let mut projection = projection.with_row_id(); + if add_row_addr { + projection = projection.with_row_addr(); + } + Ok(Arc::new(FilteredReadExec::try_new( + dataset, + FilteredReadOptions::new(projection), + Some(index_mapper), + )?)) + } + } +} + +pub fn validate_row_stream_read(version: ConcreteFileVersion) -> Result<()> { + match version { + ConcreteFileVersion::V1 => Err(Error::not_supported_source( + "taking rows through FilteredReadExec requires the v2 storage format" + .to_string() + .into(), + )), + ConcreteFileVersion::V2_0 + | ConcreteFileVersion::V2_1 + | ConcreteFileVersion::V2_2 + | ConcreteFileVersion::V2_3 => Ok(()), + } +} + +fn repair_legacy_manifest_storage(manifest: &mut Manifest) -> Result<()> { + let declared = manifest.data_storage_format.lance_file_format(); + if let Some(actual) = Fragment::try_infer_version(&manifest.fragments) + .map_err(|error| { + Error::internal(format!( + "The dataset contains a mixture of file versions. You will need to rollback to an earlier version: {error}" + )) + })? + && actual != ConcreteFileVersion::V1 + { + log::warn!( + "Data storage version {} is less than the actual file version {}. This has been automatically updated.", + declared, + actual + ); + manifest.data_storage_format = DataStorageFormat::new(actual); + } + Ok(()) +} + +fn validate_exact_manifest_storage( + manifest: &Manifest, + expected: ConcreteFileVersion, +) -> Result<()> { + if let Some(actual) = Fragment::try_infer_version(&manifest.fragments)? + && actual != expected + { + return Err(Error::internal(format!( + "The operation added files with version {}. However, the data storage version is {}.", + actual, expected + ))); + } + Ok(()) +} diff --git a/rust/lance/src/dataset/write.rs b/rust/lance/src/dataset/write.rs index 80afa49ffdd..81b0fa37109 100644 --- a/rust/lance/src/dataset/write.rs +++ b/rust/lance/src/dataset/write.rs @@ -2,51 +2,51 @@ // SPDX-FileCopyrightText: Copyright The Lance Authors use arrow_array::RecordBatch; +use bytes::Bytes; use chrono::TimeDelta; use datafusion::physical_plan::SendableRecordBatchStream; -use datafusion::physical_plan::stream::RecordBatchStreamAdapter; -use futures::{Stream, StreamExt, TryStreamExt}; +use futures::StreamExt; use lance_arrow::{ ARROW_EXT_NAME_KEY, BLOB_DEDICATED_SIZE_THRESHOLD_META_KEY, BLOB_INLINE_SIZE_THRESHOLD_META_KEY, BLOB_META_KEY, BLOB_PACK_FILE_SIZE_THRESHOLD_META_KEY, BLOB_V2_EXT_NAME, }; -use lance_core::datatypes::{ - NullabilityComparison, OnMissing, OnTypeMismatch, SchemaCompareOptions, +use lance_core::datatypes::{NullabilityComparison, OnMissing, OnTypeMismatch}; +use lance_core::utils::tracing::{ + AUDIT_MODE_CREATE, AUDIT_MODE_DELETE, AUDIT_TYPE_DATA, TRACE_FILE_AUDIT, }; -use lance_core::error::LanceOptionExt; -use lance_core::utils::tempfile::TempDir; -use lance_core::utils::tracing::{AUDIT_MODE_CREATE, AUDIT_TYPE_DATA, TRACE_FILE_AUDIT}; use lance_core::{Error, Result, datatypes::Schema}; -use lance_datafusion::chunker::{break_stream, chunk_stream}; -use lance_datafusion::spill::{SpillReceiver, SpillSender, create_replay_spill}; use lance_datafusion::utils::StreamingWriteSource; -use lance_file::previous::writer::{ - FileWriter as PreviousFileWriter, ManifestProvider as PreviousManifestProvider, +use lance_file::version::{ConcreteFileVersion, LanceFileVersion}; +use lance_file::versions::v1::writer::{ + FileWriter as V1FileWriter, ManifestProvider as V1ManifestProvider, }; -use lance_file::version::LanceFileVersion; -use lance_file::writer::{self as current_writer, FileWriterOptions}; +use lance_file::writer::{self as current_writer}; use lance_io::object_store::{ ObjectStore, ObjectStoreParams, ObjectStoreRegistry, parse_base_scoped_key, }; -use lance_table::format::{BasePath, DataFile, Fragment}; +use lance_io::traits::Writer; +use lance_table::format::{BasePath, DataFile, Fragment, IndexMetadata}; use lance_table::io::commit::{CommitHandler, commit_handler_from_url}; use lance_table::io::manifest::ManifestDescribing; use object_store::path::Path; use std::borrow::Cow; use std::collections::{BTreeSet, HashMap, HashSet}; +use std::future::Future; use std::num::NonZero; use std::sync::Arc; use std::sync::atomic::AtomicUsize; use tracing::{info, instrument}; use crate::Dataset; -use crate::blob::normalize_prepared_blob_schema; +use crate::blob::prepared_to_logical_blob_schema; use crate::dataset::blob::{ BlobPreprocessor, ExternalBaseCandidate, ExternalBaseResolver, blob_dedicated_threshold_from_metadata, blob_inline_threshold_from_metadata, blob_pack_file_threshold_from_metadata, preprocess_blob_batches, }; +use crate::index::DatasetIndexExt; +use crate::index::scalar::{IndexDetails, fetch_index_details}; use crate::session::Session; use super::DATA_DIR; @@ -54,6 +54,7 @@ use super::fragment::write::generate_random_filename; use super::progress::{NoopFragmentWriteProgress, WriteFragmentProgress}; use super::transaction::Transaction; use super::utils::SchemaAdapter; +use super::versions; mod commit; pub mod delete; @@ -454,8 +455,8 @@ impl WriteParams { } } - pub fn storage_version_or_default(&self) -> LanceFileVersion { - self.data_storage_version.unwrap_or_default() + pub fn storage_version_or_default(&self) -> ConcreteFileVersion { + self.data_storage_version.unwrap_or_default().resolve() } pub fn store_registry(&self) -> Arc { @@ -595,39 +596,22 @@ pub async fn write_fragments( } #[allow(clippy::too_many_arguments)] -pub async fn do_write_fragments( +pub(super) async fn do_write_fragments_impl( dataset: Option<&Dataset>, object_store: Arc, base_dir: &Path, schema: &Schema, - data: SendableRecordBatchStream, + mut buffered_reader: futures::stream::BoxStream<'static, Result>>, params: WriteParams, - storage_version: LanceFileVersion, + open_writer: OpenWriter, + external_base_resolver: Option>, target_bases_info: Option>, -) -> Result> { - let adapter = SchemaAdapter::new(data.schema()); - let data = adapter.to_physical_stream(data); - - let mut buffered_reader = if storage_version == LanceFileVersion::Legacy { - // In v1 we split the stream into row group sized batches - chunk_stream(data, params.max_rows_per_group) - } else { - // In v2 we don't care about group size but we do want to break - // the stream on file boundaries - break_stream(data, params.max_rows_per_file) - .map_ok(|batch| vec![batch]) - .boxed() - }; - - let external_base_resolver = if storage_version >= LanceFileVersion::V2_2 - && schema.fields_pre_order().any(|field| field.is_blob_v2()) - { - Some(Arc::new( - build_external_base_resolver(dataset, ¶ms).await?, - )) - } else { - None - }; + mut seed_writers: Vec>, +) -> Result> +where + OpenWriter: Fn(Arc, Schema, Path, WriterOptions) -> OpenWriterFuture + Send + Sync, + OpenWriterFuture: Future>> + Send, +{ let source_store_registry = dataset .map(|ds| ds.session.store_registry()) .unwrap_or_else(|| params.store_registry()); @@ -639,7 +623,7 @@ pub async fn do_write_fragments( object_store.clone(), base_dir, schema, - storage_version, + open_writer, target_bases_info, external_base_resolver, params.allow_external_blob_outside_bases, @@ -669,6 +653,14 @@ pub async fn do_write_fragments( } writer.as_mut().unwrap().write(&batch_chunk).await?; + for seed_writer in seed_writers.iter_mut() { + let col_name = seed_writer.column_name().to_owned(); + for batch in &batch_chunk { + if let Some(col) = batch.column_by_name(&col_name) { + seed_writer.observe_batch(col)?; + } + } + } for batch in &batch_chunk { num_rows_in_current_file += batch.num_rows() as u32; } @@ -685,7 +677,9 @@ pub async fn do_write_fragments( if num_rows_in_current_file >= params.max_rows_per_file as u32 || writer.as_mut().unwrap().tell().await? >= params.max_bytes_per_file as u64 { - let (num_rows, data_file) = writer.take().unwrap().finish().await?; + let mut w = writer.take().unwrap(); + flush_seed_writers(w.as_mut(), &mut seed_writers).await?; + let (num_rows, data_file) = w.finish().await?; info!(target: TRACE_FILE_AUDIT, mode=AUDIT_MODE_CREATE, r#type=AUDIT_TYPE_DATA, path = &data_file.path); debug_assert_eq!(num_rows, num_rows_in_current_file); bytes_completed += data_file.file_size_bytes.get().map_or(0, |s| s.get()); @@ -727,6 +721,17 @@ pub async fn do_write_fragments( // Complete the final writer if let Some(mut writer) = writer.take() { + if let Err(e) = flush_seed_writers(writer.as_mut(), &mut seed_writers).await { + drop(writer); + cleanup_data_fragments( + &object_store, + base_dir, + cleanup_bases.as_deref(), + &fragments, + ) + .await; + return Err(e); + } match writer.finish().await { Ok((num_rows, data_file)) => { info!(target: TRACE_FILE_AUDIT, mode=AUDIT_MODE_CREATE, r#type=AUDIT_TYPE_DATA, path = &data_file.path); @@ -761,6 +766,23 @@ pub async fn do_write_fragments( Ok(fragments) } +/// Flush all seed writers into the given file writer, embedding seed buffers +/// and schema metadata before `finish()` is called. +async fn flush_seed_writers( + writer: &mut dyn GenericWriter, + seed_writers: &mut [Box], +) -> Result<()> { + for seed_writer in seed_writers.iter_mut() { + if let Some(bytes) = seed_writer.finish()? { + let buf_index = writer.add_global_buffer(bytes).await?; + let key = seed_writer.schema_metadata_key(); + let value = seed_writer.schema_metadata_value(buf_index); + writer.add_schema_metadata(key, value); + } + } + Ok(()) +} + /// Best-effort cleanup of data files for fragments that were written but not committed. /// /// Contract: @@ -781,6 +803,11 @@ pub(crate) async fn cleanup_data_fragments( let data_dir = base_dir.clone().join(DATA_DIR); let mut skipped_external = 0usize; for fragment in fragments { + // Deliberately not `referenced_lance_files()`: callers decide which + // files belong to the failed write. `schema_evolution` passes a live + // fragment whose `files` it narrowed to the newly written ones while + // leaving `overlays` untouched, so including overlays here would delete + // live data. for file in &fragment.files { let (store, file_dir) = if let Some(base_id) = file.base_id { match target_bases.and_then(|bases| bases.iter().find(|b| b.base_id == base_id)) { @@ -802,8 +829,13 @@ pub(crate) async fn cleanup_data_fragments( }; let path = file_dir.clone().join(file.path.as_str()); - if let Err(e) = store.delete(&path).await { - log::warn!("Failed to clean up orphaned data file '{}': {}", path, e); + match store.delete(&path).await { + Ok(()) => { + info!(target: TRACE_FILE_AUDIT, mode=AUDIT_MODE_DELETE, r#type=AUDIT_TYPE_DATA, path = file.path.as_str()); + } + Err(e) => { + log::warn!("Failed to clean up orphaned data file '{}': {}", path, e); + } } // Clean up any blob v2 sidecars that might exist for this data file. @@ -1220,6 +1252,20 @@ async fn build_external_base_resolver( Ok(ExternalBaseResolver::new(candidates, store_registry)) } +pub(super) async fn blob_v2_external_base_resolver( + dataset: Option<&Dataset>, + params: &WriteParams, + schema: &Schema, +) -> Result>> { + if schema.fields_pre_order().any(|field| field.is_blob_v2()) { + Ok(Some(Arc::new( + build_external_base_resolver(dataset, params).await?, + ))) + } else { + Ok(None) + } +} + /// Writes the given data to the dataset and returns fragments. /// /// NOTE: the fragments have not yet been assigned an ID. That must be done @@ -1229,8 +1275,13 @@ async fn build_external_base_resolver( /// This is a private variant that takes a `SendableRecordBatchStream` instead /// of a reader. We don't expose the stream at our interface because it is a /// DataFusion type. +/// +/// The caller must resolve `storage_version` once for the operation. Operations +/// that also select a commit format must reuse the same value when committing. +#[allow(clippy::too_many_arguments)] #[instrument(level = "debug", skip_all)] pub async fn write_fragments_internal( + storage_version: ConcreteFileVersion, dataset: Option<&Dataset>, object_store: Arc, base_dir: &Path, @@ -1256,91 +1307,113 @@ pub async fn write_fragments_internal( // Make sure the max rows per group is not larger than the max rows per file params.max_rows_per_group = std::cmp::min(params.max_rows_per_group, params.max_rows_per_file); validate_external_blob_write_params(¶ms)?; - let normalized_converted_schema = normalize_prepared_blob_schema(&converted_schema)?; - - let (schema, storage_version) = if let Some(dataset) = dataset { - match params.mode { - WriteMode::Append | WriteMode::Create => { - // Append mode, so we need to check compatibility - normalized_converted_schema.check_compatible( - dataset.schema(), - &SchemaCompareOptions { - // We don't care if the user claims their data is nullable / non-nullable. We will - // verify against the actual data. - compare_nullability: NullabilityComparison::Ignore, - allow_missing_if_nullable: true, - ignore_field_order: true, - compare_dictionary: dataset.is_legacy_storage(), - ..Default::default() - }, - )?; - validate_blob_threshold_metadata_for_append( - &normalized_converted_schema, - dataset.schema(), - )?; - let write_schema = dataset.schema().project_by_schema( - &normalized_converted_schema, - OnMissing::Error, - OnTypeMismatch::Error, - )?; - // Use the storage version from the dataset, ignoring any version from the user. - let data_storage_version = dataset - .manifest() - .data_storage_format - .lance_file_version()?; - (write_schema, data_storage_version) - } - WriteMode::Overwrite => { - // Overwrite, use the schema from the data. If the user specified - // a storage version use that. Otherwise use the version from the - // dataset. - let data_storage_version = params.data_storage_version.unwrap_or( - dataset - .manifest() - .data_storage_format - .lance_file_version()?, - ); - (normalized_converted_schema, data_storage_version) - } - } + let normalized_converted_schema = prepared_to_logical_blob_schema(&converted_schema)?; + + versions::write_fragments( + storage_version, + dataset, + object_store, + base_dir, + normalized_converted_schema, + data, + params, + target_bases_info, + ) + .await +} + +pub(super) fn prepare_write_schema( + dataset: Option<&Dataset>, + normalized_converted_schema: Schema, + params: &WriteParams, + mut schema_compare_options: lance_core::datatypes::SchemaCompareOptions, +) -> Result { + let schema = if let Some(dataset) = dataset + && matches!(params.mode, WriteMode::Append | WriteMode::Create) + { + schema_compare_options.compare_nullability = NullabilityComparison::Ignore; + schema_compare_options.allow_missing_if_nullable = true; + schema_compare_options.ignore_field_order = true; + normalized_converted_schema.check_compatible(dataset.schema(), &schema_compare_options)?; + validate_blob_threshold_metadata_for_append( + &normalized_converted_schema, + dataset.schema(), + )?; + dataset.schema().project_by_schema( + &normalized_converted_schema, + OnMissing::Error, + OnTypeMismatch::Error, + )? } else { - // Brand new dataset, use the schema from the data and the storage version - // from the user or the default. - ( - normalized_converted_schema, - params.storage_version_or_default(), - ) + normalized_converted_schema }; + Ok(schema) +} - if storage_version < LanceFileVersion::V2_2 && schema.fields_pre_order().any(|f| f.is_blob_v2()) - { +pub(super) fn validate_legacy_blob_write_schema( + schema: &Schema, + version_debug: &str, +) -> Result<()> { + if schema.fields_pre_order().any(|field| field.is_blob_v2()) { return Err(Error::invalid_input(format!( - "Blob v2 requires file version >= 2.2 (got {:?})", - storage_version + "Blob v2 requires file version >= 2.2 (got {version_debug})" ))); } + Ok(()) +} - if storage_version >= LanceFileVersion::V2_2 - && let Some(blob_field_path) = legacy_blob_field_path(&schema) - { +pub(super) fn validate_blob_v2_write_schema(schema: &Schema) -> Result<()> { + if let Some(blob_field_path) = legacy_blob_field_path(schema) { return Err(Error::invalid_input(format!( "Legacy blob columns (field metadata key {BLOB_META_KEY:?}) are not supported for file version >= 2.2. Found legacy blob field: {blob_field_path}. Use the blob v2 extension type (ARROW:extension:name = \"lance.blob.v2\") and the new blob APIs (e.g. lance::blob::blob_field / lance::blob::BlobArrayBuilder)." ))); } + Ok(()) +} - let fragments = do_write_fragments( - dataset, - object_store, - base_dir, - &schema, - data, - params, - storage_version, - target_bases_info, - ) - .await?; +pub(crate) async fn create_seed_writers_current( + dataset: Option<&Dataset>, + params: &WriteParams, +) -> Result>> { + // Seeds only make sense when appending to an existing dataset. + if !matches!(params.mode, WriteMode::Append) { + return Ok(Vec::new()); + } + let Some(dataset) = dataset else { + return Ok(Vec::new()); + }; - Ok((fragments, schema)) + let indices: Arc> = dataset.load_indices().await?; + let mut writers: Vec> = Vec::new(); + + for index in indices.iter() { + if index.fields.len() != 1 { + continue; + } + let field_id = index.fields[0]; + let Ok(field_path) = dataset.schema().field_path(field_id) else { + continue; + }; + let Some(data_type) = dataset.schema().field(&field_path).map(|f| f.data_type()) else { + continue; + }; + + let Ok(index_details) = fetch_index_details(dataset, &field_path, index).await else { + continue; + }; + let details = IndexDetails(index_details.clone()); + let Ok(plugin) = details.get_plugin() else { + continue; + }; + if let Some(writer) = plugin + .create_seed_writer(&field_path, &data_type, &index_details) + .await? + { + writers.push(writer); + } + } + + Ok(writers) } fn legacy_blob_field_path(schema: &Schema) -> Option { @@ -1367,13 +1440,23 @@ pub trait GenericWriter: Send { async fn tell(&mut self) -> Result; /// Finish writing the file (flush the remaining data and write footer) async fn finish(&mut self) -> Result<(u32, DataFile)>; + + /// Add a global buffer to the current file. Returns the 1-based buffer index. + /// Must be called before `finish`. No-op on legacy (V1) files (returns `Ok(1)`). + async fn add_global_buffer(&mut self, _buffer: Bytes) -> Result { + Ok(1) + } + + /// Add a key-value pair to the file's schema metadata. + /// Must be called before `finish`. No-op on legacy (V1) files. + fn add_schema_metadata(&mut self, _key: String, _value: String) {} } struct V1WriterAdapter where - M: PreviousManifestProvider + Send + Sync, + M: V1ManifestProvider + Send + Sync, { - writer: PreviousFileWriter, + writer: V1FileWriter, path: String, base_id: Option, } @@ -1381,7 +1464,7 @@ where #[async_trait::async_trait] impl GenericWriter for V1WriterAdapter where - M: PreviousManifestProvider + Send + Sync, + M: V1ManifestProvider + Send + Sync, { async fn write(&mut self, batches: &[RecordBatch]) -> Result<()> { self.writer.write(batches).await @@ -1408,8 +1491,7 @@ where struct V2WriterAdapter { writer: current_writer::FileWriter, - path: String, - base_id: Option, + data_file: Option, preprocessor: Option, } @@ -1429,7 +1511,10 @@ impl GenericWriter for V2WriterAdapter { Ok(()) } fn data_file_path(&self) -> (&str, Option) { - (&self.path, self.base_id) + self.data_file + .as_ref() + .map(|data_file| (data_file.path.as_str(), data_file.base_id)) + .unwrap_or(("", None)) } async fn tell(&mut self) -> Result { Ok(self.writer.tell().await?) @@ -1450,75 +1535,28 @@ impl GenericWriter for V2WriterAdapter { .iter() .map(|(_, column_index)| *column_index as i32) .collect::>(); - let (major, minor) = self.writer.version().to_numbers(); let write_summary = self.writer.finish().await?; - let data_file = DataFile::new( - std::mem::take(&mut self.path), - field_ids, - column_indices, - major, - minor, - NonZero::new(write_summary.size_bytes), - self.base_id, - ); + let mut data_file = self + .data_file + .take() + .ok_or_else(|| Error::internal("current writer was already finished"))?; + data_file.fields = field_ids.into(); + data_file.column_indices = column_indices.into(); + data_file.file_size_bytes = NonZero::new(write_summary.size_bytes).into(); Ok((write_summary.num_rows as u32, data_file)) } -} -pub async fn open_writer( - object_store: &ObjectStore, - schema: &Schema, - base_dir: &Path, - storage_version: LanceFileVersion, -) -> Result> { - open_writer_with_options( - object_store, - schema, - base_dir, - storage_version, - WriterOptions { - add_data_dir: true, - ..Default::default() - }, - ) - .await -} - -pub(super) async fn open_update_writer( - dataset: &Dataset, - schema: &Schema, - storage_version: LanceFileVersion, -) -> Result> { - // add_columns / alter_columns reuse the normal writer stack, but they do not - // flow through WriteParams. Rebuild the external base resolver here so blob - // v2 reference columns can resolve dataset-registered external URIs. - let external_base_resolver = if storage_version >= LanceFileVersion::V2_2 - && schema.fields_pre_order().any(|f| f.is_blob_v2()) - { - Some(Arc::new( - build_external_base_resolver(Some(dataset), &WriteParams::default()).await?, - )) - } else { - None - }; + async fn add_global_buffer(&mut self, buffer: Bytes) -> Result { + self.writer.add_global_buffer(buffer).await + } - open_writer_with_options( - &dataset.object_store, - schema, - &dataset.base, - storage_version, - WriterOptions { - add_data_dir: true, - external_base_resolver, - source_store_registry: dataset.session.store_registry(), - ..Default::default() - }, - ) - .await + fn add_schema_metadata(&mut self, key: String, value: String) { + self.writer.add_schema_metadata(key, value); + } } #[derive(Default)] -struct WriterOptions { +pub(crate) struct WriterOptions { add_data_dir: bool, base_id: Option, external_base_resolver: Option>, @@ -1529,13 +1567,94 @@ struct WriterOptions { blob_pack_file_size_threshold: Option, } -async fn open_writer_with_options( +impl WriterOptions { + pub(super) fn update( + source_store_registry: Arc, + external_base_resolver: Option>, + allow_external_blob_outside_bases: bool, + ) -> Self { + Self { + add_data_dir: true, + external_base_resolver, + allow_external_blob_outside_bases, + source_store_registry, + ..Default::default() + } + } +} + +pub(crate) async fn open_v1_writer( object_store: &ObjectStore, schema: &Schema, base_dir: &Path, - storage_version: LanceFileVersion, options: WriterOptions, ) -> Result> { + let WriterOptions { + add_data_dir, + base_id, + .. + } = options; + let (_data_file_key, filename, _data_dir, full_path) = + prepare_data_file_path(base_dir, add_data_dir); + Ok(Box::new(V1WriterAdapter { + writer: V1FileWriter::::try_new( + object_store, + &full_path, + schema.clone(), + &Default::default(), + ) + .await?, + path: filename, + base_id, + })) +} + +pub(in crate::dataset) async fn open_current_writer( + create_file_writer: F, + object_store: &ObjectStore, + schema: &Schema, + base_dir: &Path, + options: WriterOptions, +) -> Result> +where + F: FnOnce( + Box, + Schema, + String, + Option, + ) -> Result<(current_writer::FileWriter, DataFile)>, +{ + let WriterOptions { + add_data_dir, + base_id, + .. + } = options; + let (_data_file_key, filename, _data_dir, full_path) = + prepare_data_file_path(base_dir, add_data_dir); + let writer = object_store.create(&full_path).await?; + let (file_writer, data_file) = create_file_writer(writer, schema.clone(), filename, base_id)?; + Ok(Box::new(V2WriterAdapter { + writer: file_writer, + data_file: Some(data_file), + preprocessor: None, + })) +} + +pub(in crate::dataset) async fn open_current_blob_v2_writer( + create_file_writer: F, + object_store: &ObjectStore, + schema: &Schema, + base_dir: &Path, + options: WriterOptions, +) -> Result> +where + F: FnOnce( + Box, + Schema, + String, + Option, + ) -> Result<(current_writer::FileWriter, DataFile)>, +{ let WriterOptions { add_data_dir, base_id, @@ -1546,66 +1665,39 @@ async fn open_writer_with_options( source_store_params, blob_pack_file_size_threshold, } = options; + let (data_file_key, filename, data_dir, full_path) = + prepare_data_file_path(base_dir, add_data_dir); + let writer = object_store.create(&full_path).await?; + let (file_writer, data_file) = create_file_writer(writer, schema.clone(), filename, base_id)?; + let preprocessor = BlobPreprocessor::new( + object_store.clone(), + data_dir, + data_file_key, + schema, + external_base_resolver, + allow_external_blob_outside_bases, + external_blob_mode, + source_store_registry, + source_store_params, + blob_pack_file_size_threshold, + )?; + Ok(Box::new(V2WriterAdapter { + writer: file_writer, + data_file: Some(data_file), + preprocessor: Some(preprocessor), + })) +} +fn prepare_data_file_path(base_dir: &Path, add_data_dir: bool) -> (String, String, Path, Path) { let data_file_key = generate_random_filename(); let filename = format!("{}.lance", data_file_key); - let data_dir = if add_data_dir { base_dir.clone().join(DATA_DIR) } else { base_dir.clone() }; - let full_path = data_dir.clone().join(filename.as_str()); - - let writer = if storage_version == LanceFileVersion::Legacy { - Box::new(V1WriterAdapter { - writer: PreviousFileWriter::::try_new( - object_store, - &full_path, - schema.clone(), - &Default::default(), - ) - .await?, - path: filename, - base_id, - }) - } else { - let writer = object_store.create(&full_path).await?; - let enable_blob_v2 = storage_version >= LanceFileVersion::V2_2; - let file_writer = current_writer::FileWriter::try_new( - writer, - schema.clone(), - FileWriterOptions { - format_version: Some(storage_version), - ..Default::default() - }, - )?; - let preprocessor = if enable_blob_v2 { - Some(BlobPreprocessor::new( - object_store.clone(), - data_dir.clone(), - data_file_key.clone(), - schema, - external_base_resolver, - allow_external_blob_outside_bases, - external_blob_mode, - source_store_registry, - source_store_params, - blob_pack_file_size_threshold, - )?) - } else { - None - }; - let writer_adapter = V2WriterAdapter { - writer: file_writer, - path: filename, - base_id, - preprocessor, - }; - Box::new(writer_adapter) as Box - }; - Ok(writer) + (data_file_key, filename, data_dir, full_path) } /// Reserved base id that refers to the dataset's primary storage in @@ -1630,13 +1722,13 @@ pub struct TargetBaseInfo { pub is_dataset_root: bool, } -struct WriterGenerator { +struct WriterGenerator { /// Default object store (used when no target bases specified) object_store: Arc, /// Default base directory (used when no target bases specified) base_dir: Path, schema: Schema, - storage_version: LanceFileVersion, + open_writer: OpenWriter, /// Target base information (if writing to specific bases) target_bases_info: Option>, external_base_resolver: Option>, @@ -1649,13 +1741,17 @@ struct WriterGenerator { next_base_index: AtomicUsize, } -impl WriterGenerator { +impl WriterGenerator +where + OpenWriter: Fn(Arc, Schema, Path, WriterOptions) -> OpenWriterFuture + Send + Sync, + OpenWriterFuture: Future>> + Send, +{ #[allow(clippy::too_many_arguments)] pub fn new( object_store: Arc, base_dir: &Path, schema: &Schema, - storage_version: LanceFileVersion, + open_writer: OpenWriter, target_bases_info: Option>, external_base_resolver: Option>, allow_external_blob_outside_bases: bool, @@ -1668,7 +1764,7 @@ impl WriterGenerator { object_store, base_dir: base_dir.clone(), schema: schema.clone(), - storage_version, + open_writer, target_bases_info, external_base_resolver, allow_external_blob_outside_bases, @@ -1696,11 +1792,10 @@ impl WriterGenerator { let fragment = Fragment::new(0); let writer = if let Some(base_info) = self.select_target_base() { - open_writer_with_options( - &base_info.object_store, - &self.schema, - &base_info.base_dir, - self.storage_version, + (self.open_writer)( + base_info.object_store.clone(), + self.schema.clone(), + base_info.base_dir.clone(), WriterOptions { add_data_dir: base_info.is_dataset_root, // Primary-storage slots stamp no base id, like a write @@ -1716,11 +1811,10 @@ impl WriterGenerator { ) .await? } else { - open_writer_with_options( - &self.object_store, - &self.schema, - &self.base_dir, - self.storage_version, + (self.open_writer)( + self.object_store.clone(), + self.schema.clone(), + self.base_dir.clone(), WriterOptions { add_data_dir: true, base_id: None, @@ -1771,108 +1865,6 @@ async fn resolve_commit_handler( } } -/// Create an iterator of record batch streams from the given source. -/// -/// If `enable_retries` is true, then the source will be saved either in memory -/// or spilled to disk to allow replaying the source in case of a failure. The -/// source will be kept in memory if either (1) the size hint shows that -/// there is only one batch or (2) the stream contains less than 100MB of -/// data. Otherwise, the source will be spilled to a temporary file on disk. -/// -/// This is used to support retries on write operations. -async fn new_source_iter( - source: SendableRecordBatchStream, - enable_retries: bool, -) -> Result + Send + 'static>> { - if enable_retries { - let schema = source.schema(); - - // If size hint shows there is only one batch, spilling has no benefit, just keep that - // in memory. (This is a pretty common case.) - let size_hint = source.size_hint(); - if size_hint.0 == 1 && size_hint.1 == Some(1) { - let batches: Vec = source.try_collect().await?; - Ok(Box::new(std::iter::repeat_with(move || { - Box::pin(RecordBatchStreamAdapter::new( - schema.clone(), - futures::stream::iter(batches.clone().into_iter().map(Ok)), - )) as SendableRecordBatchStream - }))) - } else { - // Allow buffering up to 100MB in memory before spilling to disk. - Ok(Box::new( - SpillStreamIter::try_new(source, 100 * 1024 * 1024).await?, - )) - } - } else { - Ok(Box::new(std::iter::once(source))) - } -} - -struct SpillStreamIter { - receiver: SpillReceiver, - _sender_handle: tokio::task::JoinHandle, - // This temp dir is used to store the spilled data. It is kept alive by - // this struct. When this struct is dropped, the Drop implementation of - // tempfile::TempDir will delete the temp dir. - _tmp_dir: TempDir, -} - -impl SpillStreamIter { - pub async fn try_new( - mut source: SendableRecordBatchStream, - memory_limit: usize, - ) -> Result { - let tmp_dir = tokio::task::spawn_blocking(|| { - TempDir::try_new() - .map_err(|e| Error::invalid_input(format!("Failed to create temp dir: {}", e))) - }) - .await - .ok() - .expect_ok()??; - - let tmp_path = tmp_dir.std_path().join("spill.arrows"); - let (mut sender, receiver) = create_replay_spill(tmp_path, source.schema(), memory_limit); - - let sender_handle = tokio::task::spawn(async move { - while let Some(res) = source.next().await { - match res { - Ok(batch) => match sender.write(batch).await { - Ok(_) => {} - Err(e) => { - sender.send_error(e); - break; - } - }, - Err(e) => { - sender.send_error(e); - break; - } - } - } - - if let Err(err) = sender.finish().await { - sender.send_error(err); - } - sender - }); - - Ok(Self { - receiver, - _tmp_dir: tmp_dir, - _sender_handle: sender_handle, - }) - } -} - -impl Iterator for SpillStreamIter { - type Item = SendableRecordBatchStream; - - fn next(&mut self) -> Option { - Some(self.receiver.read()) - } -} - #[cfg(test)] mod tests { use super::*; @@ -1883,12 +1875,40 @@ mod tests { use datafusion::{error::DataFusionError, physical_plan::stream::RecordBatchStreamAdapter}; use datafusion_physical_plan::RecordBatchStream; use futures::TryStreamExt; + use lance_datafusion::chunker::chunk_stream; use lance_datagen::{BatchCount, RowCount, array, gen_batch}; - use lance_file::previous::reader::FileReader as PreviousFileReader; + use lance_file::version::ConcreteFileVersion; + use lance_file::versions::v1::reader::FileReader as V1FileReader; use lance_io::object_store::StorageOptionsAccessor; use lance_io::traits::Reader; use lance_table::format::BasePath; + async fn open_v2_1_test_writer( + object_store: Arc, + schema: Schema, + base_dir: Path, + options: WriterOptions, + ) -> Result> { + open_current_writer( + |object_writer, schema, filename, base_id| { + let writer = lance_file::versions::v2_1::create_writer( + object_writer, + schema, + lance_file::writer::FileWriterOptions::default(), + )? + .into(); + let mut data_file = DataFile::new_unstarted(filename, ConcreteFileVersion::V2_1); + data_file.base_id = base_id; + Ok((writer, data_file)) + }, + &object_store, + &schema, + &base_dir, + options, + ) + .await + } + #[test] fn test_auto_cleanup_disabled_by_default() { // Auto-cleanup must be off by default: the cleanup hook is expensive on @@ -2004,6 +2024,7 @@ mod tests { let object_store = Arc::new(ObjectStore::memory()); write_fragments_internal( + write_params.storage_version_or_default(), None, object_store, &Path::from("test"), @@ -2057,6 +2078,7 @@ mod tests { let object_store = Arc::new(ObjectStore::memory()); write_fragments_internal( + write_params.storage_version_or_default(), None, object_store, &Path::from("test"), @@ -2118,6 +2140,7 @@ mod tests { let object_store = Arc::new(ObjectStore::memory()); write_fragments_internal( + write_params.storage_version_or_default(), None, object_store, &Path::from("test"), @@ -2214,7 +2237,8 @@ mod tests { LanceFileVersion::Next, ]; for version in versions { - let (major, minor) = version.to_numbers(); + let (major, minor) = version.resolve().to_data_file_numbers(); + let write_params = WriteParams { data_storage_version: Some(version), // This parameter should be ignored @@ -2231,6 +2255,7 @@ mod tests { let object_store = Arc::new(ObjectStore::memory()); let (fragments, _) = write_fragments_internal( + version.resolve(), None, object_store, &Path::from("test"), @@ -2306,6 +2331,7 @@ mod tests { let object_store = Arc::new(ObjectStore::memory()); let base_path = Path::from("test"); let (fragments, _) = write_fragments_internal( + ConcreteFileVersion::V1, None, object_store.clone(), &base_path, @@ -2327,7 +2353,7 @@ mod tests { .join(DATA_DIR) .join(fragment.files[0].path.as_str()); let file_reader: Arc = object_store.open(&path).await.unwrap().into(); - let reader = PreviousFileReader::try_new_from_reader( + let reader = V1FileReader::try_new_from_reader( &path, file_reader, None, @@ -2515,7 +2541,7 @@ mod tests { object_store.clone(), &base_dir, &schema, - LanceFileVersion::Stable, + open_v2_1_test_writer, Some(target_bases), None, false, @@ -2561,11 +2587,11 @@ mod tests { let object_store = Arc::new(ObjectStore::memory()); let base_dir = Path::from("test/bucket2"); - let mut inner_writer = open_writer_with_options( + let mut inner_writer = versions::open_writer( + LanceFileVersion::Stable.resolve(), &object_store, &schema, &base_dir, - LanceFileVersion::Stable, WriterOptions { add_data_dir: false, // Don't add /data ..Default::default() @@ -2633,7 +2659,7 @@ mod tests { Arc::new(ObjectStore::memory()), &Path::from("default"), &schema, - LanceFileVersion::Stable, + open_v2_1_test_writer, Some(target_bases), None, false, @@ -3467,6 +3493,7 @@ mod tests { }; let result = write_fragments_internal( + write_params.storage_version_or_default(), None, object_store, &Path::from("test_empty"), @@ -3691,6 +3718,7 @@ mod tests { // Attempt to write data - should fail with IO error due to disk full let result = write_fragments_internal( + write_params.storage_version_or_default(), None, object_store, &Path::from("test_disk_full"), @@ -3733,6 +3761,7 @@ mod tests { async fn test_write_interruption_recovery() { use super::commit::CommitBuilder; use arrow_array::record_batch; + use lance_core::utils::tempfile::TempDir; // Create a temporary directory for testing let temp_dir = TempDir::default(); @@ -3871,15 +3900,16 @@ mod tests { futures::stream::iter(items), )); - let result = do_write_fragments( + let result = versions::write_fragments_direct( + ConcreteFileVersion::V2_1, None, object_store.clone(), &base_dir, &schema, stream, WriteParams::default(), - LanceFileVersion::V2_1, None, + Vec::new(), ) .await; @@ -3928,7 +3958,8 @@ mod tests { futures::stream::iter(items), )); - let result = do_write_fragments( + let result = versions::write_fragments_direct( + ConcreteFileVersion::V2_1, None, object_store.clone(), &base_dir, @@ -3938,8 +3969,8 @@ mod tests { max_rows_per_file: 3, ..Default::default() }, - LanceFileVersion::V2_1, None, + Vec::new(), ) .await; @@ -3974,12 +4005,14 @@ mod tests { // Sanity check: file is on disk. assert_eq!(count_data_files(test_uri), 1); - let mut external_file = DataFile::new_unstarted("external.lance", 2, 1); + let mut external_file = + DataFile::new_unstarted("external.lance", ConcreteFileVersion::V2_1); external_file.base_id = Some(42); - let local_file = DataFile::new_unstarted(local_filename, 2, 1); + let local_file = DataFile::new_unstarted(local_filename, ConcreteFileVersion::V2_1); let fragments = vec![Fragment { id: 0, files: vec![external_file, local_file], + overlays: vec![], deletion_file: None, row_id_meta: None, physical_rows: Some(0), @@ -4051,15 +4084,16 @@ mod tests { assert_eq!(count_data_files(base1_dir.as_str()), 1); assert_eq!(count_plain_files(base2_dir.as_str()), 1); - let mut base1_file = DataFile::new_unstarted("one.lance", 2, 1); + let mut base1_file = DataFile::new_unstarted("one.lance", ConcreteFileVersion::V2_1); base1_file.base_id = Some(1); - let mut base2_file = DataFile::new_unstarted("two.lance", 2, 1); + let mut base2_file = DataFile::new_unstarted("two.lance", ConcreteFileVersion::V2_1); base2_file.base_id = Some(2); - let mut unknown_file = DataFile::new_unstarted("unknown.lance", 2, 1); + let mut unknown_file = DataFile::new_unstarted("unknown.lance", ConcreteFileVersion::V2_1); unknown_file.base_id = Some(42); let fragments = vec![Fragment { id: 0, files: vec![base1_file, base2_file, unknown_file], + overlays: vec![], deletion_file: None, row_id_meta: None, physical_rows: Some(0), @@ -4152,7 +4186,8 @@ mod tests { is_dataset_root: true, }]; - let result = do_write_fragments( + let result = versions::write_fragments_direct( + ConcreteFileVersion::V2_1, None, object_store.clone(), &base_dir, @@ -4162,8 +4197,8 @@ mod tests { max_rows_per_file: 3, ..Default::default() }, - LanceFileVersion::V2_1, Some(target_bases), + vec![], ) .await; @@ -4479,4 +4514,116 @@ mod tests { .collect(); assert_eq!(file_bases, vec![None, Some(1), Some(2)]); } + + #[tokio::test] + async fn test_zone_map_seeds_used_during_update() { + use crate::Dataset; + use crate::index::DatasetIndexExt; + use crate::index::scalar::open_scalar_index; + use arrow::datatypes::Int32Type; + use lance_datagen::{BatchCount, RowCount}; + use lance_datagen::{array, gen_batch}; + use lance_file::reader::FileReaderOptions; + use lance_index::metrics::NoOpMetricsCollector; + use lance_index::scalar::seed::SEED_META_KEY_PREFIX; + use lance_index::{IndexType, scalar::ScalarIndexParams}; + use lance_io::scheduler::{ScanScheduler, SchedulerConfig}; + use lance_io::utils::CachedFileSize; + + let tmpdir = lance_core::utils::tempfile::TempStrDir::default(); + let uri = tmpdir.as_str(); + + // Step 1: Create initial dataset + let reader = gen_batch() + .col("val", array::step::()) + .into_reader_rows(RowCount::from(100), BatchCount::from(1)); + let mut dataset = Dataset::write(reader, uri, None).await.unwrap(); + + // Step 2: Create a zone map index with seeds explicitly enabled (Int32 defaults to off). + let params = ScalarIndexParams::for_builtin(lance_index::scalar::BuiltinIndexType::ZoneMap) + .with_params(&serde_json::json!({"use_seeds": true})); + dataset + .create_index(&["val"], IndexType::ZoneMap, None, ¶ms, false) + .await + .unwrap(); + // Step 3: Append new data - seeds should be written automatically + let reader = gen_batch() + .col("val", array::step::()) + .into_reader_rows(RowCount::from(50), BatchCount::from(1)); + let dataset = Dataset::write( + reader, + uri, + Some(WriteParams { + mode: WriteMode::Append, + data_storage_version: Some(lance_file::version::LanceFileVersion::V2_1), + ..Default::default() + }), + ) + .await + .unwrap(); + + // Step 4: Verify that the newly appended fragment has a seed embedded + let fragments = dataset.fragments(); + let new_fragment = fragments.last().unwrap(); + let data_file = new_fragment.files.first().unwrap(); + + let scheduler = ScanScheduler::new( + dataset.object_store.clone(), + SchedulerConfig::max_bandwidth(&dataset.object_store), + ); + let path = dataset + .base + .clone() + .join(super::DATA_DIR) + .join(data_file.path.as_str()); + let file_scheduler = scheduler + .open_file(&path, &CachedFileSize::unknown()) + .await + .unwrap(); + let reader = lance_file::reader::FileReader::try_open( + file_scheduler, + None, + Default::default(), + &dataset.metadata_cache.file_metadata_cache(&path), + FileReaderOptions::default(), + ) + .await + .unwrap(); + + let meta_key = format!("{}val", SEED_META_KEY_PREFIX); + let has_seed = reader + .metadata() + .file_schema + .metadata + .contains_key(&meta_key); + assert!( + has_seed, + "Newly appended fragment should have a zone map seed in metadata" + ); + + // Step 5: Optimize the index (should use seeds) + let mut dataset = Dataset::open(uri).await.unwrap(); + dataset.optimize_indices(&Default::default()).await.unwrap(); + + // Step 6: Query the updated index to verify it's correct + let dataset = Dataset::open(uri).await.unwrap(); + let indices = dataset.load_indices().await.unwrap(); + assert!( + !indices.is_empty(), + "Dataset should still have an index after optimization" + ); + + // Verify the index is a ZoneMap and covers all fragments + let index = indices.iter().find(|i| i.name.contains("val")).unwrap(); + let scalar_index = open_scalar_index(&dataset, "val", index, &NoOpMetricsCollector) + .await + .unwrap(); + assert_eq!( + scalar_index.index_type(), + IndexType::ZoneMap, + "Index should still be a ZoneMap after optimization" + ); + let frags = scalar_index.calculate_included_frags().await.unwrap(); + assert_eq!(frags.len(), 2, "Index should cover both fragments"); + } } diff --git a/rust/lance/src/dataset/write/commit.rs b/rust/lance/src/dataset/write/commit.rs index d76c2049873..4da568913d8 100644 --- a/rust/lance/src/dataset/write/commit.rs +++ b/rust/lance/src/dataset/write/commit.rs @@ -5,7 +5,7 @@ use std::collections::HashMap; use std::sync::Arc; use std::time::Duration; -use lance_file::version::LanceFileVersion; +use lance_file::version::{ConcreteFileVersion, LanceFileVersion}; use lance_io::object_store::{ObjectStore, ObjectStoreParams}; use lance_select::RowAddrTreeMap; use lance_table::{ @@ -13,6 +13,7 @@ use lance_table::{ io::commit::{CommitConfig, CommitHandler, ManifestNamingScheme}, }; +use crate::io::commit::DEFAULT_COMMIT_RETRY_TIMEOUT; use crate::{ Dataset, Error, Result, dataset::{ @@ -39,13 +40,15 @@ pub struct CommitBuilder<'a> { dest: WriteDestination<'a>, use_stable_row_ids: Option, enable_v2_manifest_paths: bool, - storage_format: Option, + storage_format: Option, commit_handler: Option>, store_params: Option, object_store: Option>, + source_store: Option>, session: Option>, detached: bool, commit_config: CommitConfig, + retry_timeout: Duration, affected_rows: Option, transaction_properties: Option>>, timeout: Option, @@ -64,9 +67,11 @@ impl<'a> CommitBuilder<'a> { commit_handler: None, store_params: None, object_store: None, + source_store: None, session: None, detached: false, commit_config: Default::default(), + retry_timeout: DEFAULT_COMMIT_RETRY_TIMEOUT, affected_rows: None, transaction_properties: None, timeout: Some(DEFAULT_COMMIT_TIMEOUT), @@ -93,6 +98,12 @@ impl<'a> CommitBuilder<'a> { /// All data files must use the same storage format as the existing dataset. /// If a different format is passed, an error will be returned. pub fn with_storage_format(mut self, storage_format: LanceFileVersion) -> Self { + self.storage_format = Some(storage_format.resolve()); + + self + } + + pub(crate) fn with_exact_storage_format(mut self, storage_format: ConcreteFileVersion) -> Self { self.storage_format = Some(storage_format); self } @@ -103,6 +114,18 @@ impl<'a> CommitBuilder<'a> { self } + /// Pass the object store of the dataset being cloned from. + /// + /// Only used by `Operation::Clone`: the source manifest is read through this store + /// while the new dataset is written through the destination store. This lets a clone + /// cross object stores/accounts (e.g. between two Azure accounts), where the source + /// is not reachable with the destination's credentials. Defaults to the destination + /// store when not set, preserving same-store behavior. + pub fn with_source_store(mut self, source_store: Arc) -> Self { + self.source_store = Some(source_store); + self + } + /// Pass a commit handler to use for the dataset. /// /// Takes precedence over the destination dataset's own handler. If not @@ -167,6 +190,26 @@ impl<'a> CommitBuilder<'a> { self } + /// Set the wall-clock budget used by commit conflict backoff. + /// + /// The first commit attempt is always allowed to complete. If it conflicts, + /// each backoff sleep is bounded by the time remaining in this budget. The + /// default is 30 seconds. + /// + /// # Examples + /// + /// ``` + /// use std::time::Duration; + /// use lance::dataset::CommitBuilder; + /// + /// let _builder = CommitBuilder::new("memory://dataset") + /// .with_retry_timeout(Duration::from_secs(10)); + /// ``` + pub fn with_retry_timeout(mut self, retry_timeout: Duration) -> Self { + self.retry_timeout = retry_timeout; + self + } + pub fn with_skip_auto_cleanup(mut self, skip_auto_cleanup: bool) -> Self { self.commit_config.skip_auto_cleanup = skip_auto_cleanup; self @@ -241,6 +284,9 @@ impl<'a> CommitBuilder<'a> { .or_else(|| self.dest.dataset().map(|ds| ds.session.clone())) .unwrap_or_default(); + // Store used to read the source manifest for a clone (see with_source_store). + let source_store = self.source_store.clone(); + let (object_store, base_path, commit_handler) = match &self.dest { WriteDestination::Dataset(dataset) => ( dataset.object_store.clone(), @@ -381,6 +427,7 @@ impl<'a> CommitBuilder<'a> { &transaction, &manifest_config, &self.commit_config, + self.retry_timeout, ) .await? } else { @@ -391,6 +438,7 @@ impl<'a> CommitBuilder<'a> { &transaction, &manifest_config, &self.commit_config, + self.retry_timeout, manifest_naming_scheme, self.affected_rows.as_ref(), ) @@ -404,6 +452,7 @@ impl<'a> CommitBuilder<'a> { } else { commit_new_dataset( object_store.as_ref(), + source_store.as_deref(), commit_handler.as_ref(), &base_path, &transaction, @@ -525,7 +574,10 @@ mod tests { use lance_io::utils::CachedFileSize; use lance_io::{assert_io_eq, assert_io_gt}; - use lance_table::format::{DataFile, Fragment}; + use lance_table::format::{ + DataFile, Fragment, IndexMetadata, Manifest, Transaction as TableTransaction, + }; + use lance_table::io::commit::{CommitError, ManifestLocation, ManifestWriter}; use std::time::Duration; use object_store::throttle::ThrottleConfig; @@ -537,7 +589,9 @@ mod tests { use super::*; fn sample_fragment() -> Fragment { - let (major_version, minor_version) = LanceFileVersion::Stable.to_numbers(); + let (major_version, minor_version) = + LanceFileVersion::Stable.resolve().to_data_file_numbers(); + Fragment { id: 0, files: vec![DataFile { @@ -549,6 +603,7 @@ mod tests { file_size_bytes: CachedFileSize::new(100), base_id: None, }], + overlays: vec![], deletion_file: None, row_id_meta: None, physical_rows: Some(10), @@ -569,6 +624,30 @@ mod tests { } } + #[derive(Debug)] + struct SlowConflictingCommitHandler; + + #[async_trait::async_trait] + impl CommitHandler for SlowConflictingCommitHandler { + fn is_version_not_found_definitive(&self) -> bool { + true + } + + async fn commit( + &self, + _manifest: &mut Manifest, + _indices: Option>, + _base_path: &object_store::path::Path, + _object_store: &ObjectStore, + _manifest_writer: ManifestWriter, + _naming_scheme: ManifestNamingScheme, + _transaction: Option, + ) -> std::result::Result { + tokio::time::sleep(Duration::from_millis(100)).await; + Err(CommitError::CommitConflict) + } + } + #[tokio::test] async fn test_reuse_session() { // Need to use in-memory for accurate IOPS tracking. @@ -629,8 +708,11 @@ mod tests { assert_eq!(new_ds.manifest().version, 7); // Session should still be re-used // However, the dataset needs to be loaded and the read version checked out. + // The read version's manifest body is served from the session cache (it + // was cached when v1 was first created), so the checkout only pays the + // version-resolution head, not a manifest read. let io_stats = dataset.object_store.as_ref().io_stats_incremental(); - assert_io_eq!(io_stats, read_iops, 4, "load dataset + check version"); + assert_io_eq!(io_stats, read_iops, 3, "load dataset + check version"); assert_io_eq!(io_stats, write_iops, 2, "write txn + manifest"); // Commit transaction with URI and new session. Re-use the store @@ -780,6 +862,13 @@ mod tests { assert_eq!(DEFAULT_COMMIT_TIMEOUT, Duration::from_secs(1800)); } + #[test] + fn test_commit_retry_timeout_default_is_thirty_seconds() { + let builder = CommitBuilder::new("memory://default-retry-timeout"); + assert_eq!(builder.retry_timeout, DEFAULT_COMMIT_RETRY_TIMEOUT); + assert_eq!(DEFAULT_COMMIT_RETRY_TIMEOUT, Duration::from_secs(30)); + } + #[tokio::test] async fn test_commit_timeout_zero_rejected() { let dataset = Arc::new( @@ -930,6 +1019,38 @@ mod tests { assert_eq!(new_ds.manifest.version, 2); } + #[tokio::test] + async fn test_commit_retry_timeout_interrupts_conflict_backoff() { + let dataset = InsertBuilder::new("memory://retry-timeout") + .execute(vec![ + RecordBatch::try_new( + Arc::new(ArrowSchema::new(vec![ArrowField::new( + "i", + DataType::Int32, + false, + )])), + vec![Arc::new(Int32Array::from_iter_values(0..10_i32))], + ) + .unwrap(), + ]) + .await + .unwrap(); + + let result = CommitBuilder::new(Arc::new(dataset)) + .with_commit_handler(Arc::new(SlowConflictingCommitHandler)) + .with_max_retries(3) + .with_retry_timeout(Duration::from_millis(150)) + .with_timeout(None) + .execute(sample_transaction(1)) + .await; + + let error = result.expect_err("conflict backoff should respect retry timeout"); + assert!( + matches!(&error, Error::TooMuchWriteContention { message, .. } if message.contains("failed on retry_timeout")), + "got {error:?}" + ); + } + #[tokio::test] async fn test_commit_batch() { // Create a dataset @@ -963,7 +1084,7 @@ mod tests { new_fragments: vec![], removed_fragment_ids: vec![], fields_modified: vec![], - merged_generations: Vec::new(), + compacted_sstables: Vec::new(), fields_for_preserving_frag_bitmap: vec![], update_mode: None, inserted_rows_filter: None, diff --git a/rust/lance/src/dataset/write/insert.rs b/rust/lance/src/dataset/write/insert.rs index 6e1db342f9c..c88d6a23729 100644 --- a/rust/lance/src/dataset/write/insert.rs +++ b/rust/lance/src/dataset/write/insert.rs @@ -7,11 +7,12 @@ use std::sync::Arc; use arrow_array::{RecordBatch, RecordBatchIterator}; use datafusion::execution::SendableRecordBatchStream; use humantime::format_duration; -use lance_core::datatypes::{NullabilityComparison, Schema, SchemaCompareOptions}; +use lance_core::datatypes::{NullabilityComparison, Schema}; +use lance_core::is_system_column; use lance_core::utils::tracing::{DATASET_WRITING_EVENT, TRACE_DATASET_EVENTS}; -use lance_core::{ROW_ADDR, ROW_ID, ROW_OFFSET}; use lance_datafusion::utils::StreamingWriteSource; -use lance_file::version::LanceFileVersion; +use lance_file::version::{ConcreteFileVersion, LanceFileVersion}; + use lance_io::object_store::ObjectStore; use lance_table::feature_flags::can_write_dataset; use lance_table::format::Fragment; @@ -19,7 +20,7 @@ use lance_table::io::commit::CommitHandler; use object_store::path::Path; use crate::Dataset; -use crate::blob::normalize_prepared_blob_schema; +use crate::blob::prepared_to_logical_blob_schema; use crate::dataset::ReadParams; use crate::dataset::builder::DatasetBuilder; use crate::dataset::transaction::{Operation, Transaction, TransactionBuilder}; @@ -136,7 +137,7 @@ impl<'a> InsertBuilder<'a> { async fn do_commit(context: &WriteContext<'_>, transaction: Transaction) -> Result { let mut commit_builder = CommitBuilder::new(context.dest.clone()) .use_stable_row_ids(context.params.enable_stable_row_ids) - .with_storage_format(context.storage_version) + .with_exact_storage_format(context.storage_version) .enable_v2_manifest_paths(context.params.enable_v2_manifest_paths) .with_commit_handler(context.commit_handler.clone()) .with_object_store(context.object_store.clone()) @@ -214,6 +215,7 @@ impl<'a> InsertBuilder<'a> { .await?; let (written_fragments, written_schema) = write_fragments_internal( + context.storage_version, context.dest.dataset(), context.object_store.clone(), &context.base_path, @@ -316,21 +318,22 @@ impl<'a> InsertBuilder<'a> { context.params.enable_stable_row_ids = dataset.manifest.uses_stable_row_ids(); } - let schema_cmp_opts = SchemaCompareOptions { - compare_dictionary: dataset.manifest.should_use_legacy_format(), - compare_nullability: NullabilityComparison::Ignore, - allow_missing_if_nullable: true, - ignore_field_order: true, - ..Default::default() - }; + let version = dataset.manifest.data_storage_format.lance_file_format(); + let mut schema_cmp_opts = crate::dataset::versions::schema_compare_options(version); + schema_cmp_opts.compare_nullability = NullabilityComparison::Ignore; + schema_cmp_opts.allow_missing_if_nullable = true; + schema_cmp_opts.ignore_field_order = true; - let normalized_data_schema = normalize_prepared_blob_schema(data_schema)?; + let normalized_data_schema = prepared_to_logical_blob_schema(data_schema)?; normalized_data_schema.check_compatible(dataset.schema(), &schema_cmp_opts)?; } - // Make sure we aren't using any reserved column names + // The system columns (`_rowid`, `_rowaddr`, `_rowoffset`, and the row-version + // columns) are virtual: they're injected into scan results at read time and + // never stored. A stored column sharing one of these names would collide with + // the system column on read, so reject it at write time. for field in data_schema.fields.iter() { - if field.name == ROW_ID || field.name == ROW_ADDR || field.name == ROW_OFFSET { + if is_system_column(&field.name) { return Err(Error::invalid_input_source( format!( "The column {} is a reserved name and cannot be used in a Lance dataset", @@ -365,7 +368,10 @@ impl<'a> InsertBuilder<'a> { WriteDestination::Dataset(dataset) => ( dataset.object_store.clone(), dataset.base.clone(), - dataset.commit_handler.clone(), + params + .commit_handler + .clone() + .unwrap_or_else(|| dataset.commit_handler.clone()), ), WriteDestination::Uri(uri) => { let registry = params @@ -413,15 +419,15 @@ impl<'a> InsertBuilder<'a> { (WriteMode::Overwrite, WriteDestination::Dataset(dataset)) => { // If overwriting an existing dataset, allow the user to specify but use // the existing version if they don't - params.data_storage_version.map(Ok).unwrap_or_else(|| { - let m = dataset.manifest.as_ref(); - m.data_storage_format.lance_file_version() - })? + params + .data_storage_version + .map(LanceFileVersion::resolve) + .unwrap_or_else(|| dataset.manifest.data_storage_format.lance_file_format()) } (_, WriteDestination::Dataset(dataset)) => { // If appending to an existing dataset, always use the dataset version let m = dataset.manifest.as_ref(); - m.data_storage_format.lance_file_version()? + m.data_storage_format.lance_file_format() } // Otherwise (no existing dataset) fallback to the default if the user didn't specify (_, WriteDestination::Uri(_)) => params.storage_version_or_default(), @@ -445,7 +451,7 @@ struct WriteContext<'a> { object_store: Arc, base_path: Path, commit_handler: Arc, - storage_version: LanceFileVersion, + storage_version: ConcreteFileVersion, } #[cfg(test)] @@ -455,6 +461,7 @@ mod test { use arrow_array::{ArrayRef, BinaryArray, Int32Array, RecordBatchReader, StructArray}; use arrow_schema::{ArrowError, DataType, Field, Schema}; use lance_arrow::BLOB_META_KEY; + use lance_table::io::commit::{RenameCommitHandler, commit_handler_from_url}; use crate::session::Session; @@ -506,6 +513,74 @@ mod test { ); } + #[tokio::test] + async fn dataset_destination_honors_explicit_commit_handler() { + let schema = Arc::new(Schema::new(vec![Field::new("id", DataType::Int32, false)])); + let initial_batch = + RecordBatch::try_new(schema.clone(), vec![Arc::new(Int32Array::from(vec![1]))]) + .unwrap(); + let mut dataset = InsertBuilder::new("memory://") + .execute_stream(RecordBatchIterator::new( + vec![Ok(initial_batch)], + schema.clone(), + )) + .await + .unwrap(); + dataset.commit_handler = commit_handler_from_url("cos://bucket/dataset", &None) + .await + .unwrap(); + + let append_batch = + RecordBatch::try_new(schema.clone(), vec![Arc::new(Int32Array::from(vec![2]))]) + .unwrap(); + let params = WriteParams { + mode: WriteMode::Append, + commit_handler: Some(Arc::new(RenameCommitHandler)), + ..Default::default() + }; + let dataset = Dataset::write( + RecordBatchIterator::new(vec![Ok(append_batch)], schema), + Arc::new(dataset), + Some(params), + ) + .await + .expect("the explicit per-write commit handler should override the dataset handler"); + + assert_eq!(dataset.count_rows(None).await.unwrap(), 2); + } + + #[rstest::rstest] + #[case::row_id("_rowid")] + #[case::row_addr("_rowaddr")] + #[case::row_offset("_rowoffset")] + #[case::row_created_at_version("_row_created_at_version")] + #[case::row_last_updated_at_version("_row_last_updated_at_version")] + #[tokio::test] + async fn rejects_reserved_system_column_names(#[case] reserved_name: &str) { + // Every system column name must be rejected on write. The row-version + // columns (`_row_created_at_version`, `_row_last_updated_at_version`) are + // computed at read time and appended by `Projection::to_schema`; a user + // data column sharing one of those names would otherwise pass ingest and + // later collide with the appended field. + let schema = Arc::new(Schema::new(vec![Field::new( + reserved_name, + DataType::Int32, + false, + )])); + let batch = RecordBatch::try_new(schema.clone(), vec![Arc::new(Int32Array::from(vec![1]))]) + .unwrap(); + + let result = InsertBuilder::new("memory://") + .execute_stream(RecordBatchIterator::new(vec![Ok(batch)], schema.clone())) + .await; + + let err = result.expect_err("writing a reserved system column name should fail"); + assert!( + err.to_string().contains("reserved name"), + "unexpected error for {reserved_name}: {err}" + ); + } + #[tokio::test] async fn allow_overwrite_to_v2_2_without_blob_upgrade() { let schema = Arc::new(Schema::new(vec![Field::new("id", DataType::Int32, false)])); diff --git a/rust/lance/src/dataset/write/merge_insert.rs b/rust/lance/src/dataset/write/merge_insert.rs index a356b8c1d26..be3bef8683b 100644 --- a/rust/lance/src/dataset/write/merge_insert.rs +++ b/rust/lance/src/dataset/write/merge_insert.rs @@ -55,13 +55,14 @@ use crate::{ Dataset, datafusion::dataframe::SessionContextExt, dataset::{ - fragment::{FileFragment, FragReadConfig}, + fragment::FileFragment, transaction::{Operation, Transaction}, - write::{merge_insert::logical_plan::MergeInsertPlanner, open_writer}, + versions, + write::merge_insert::logical_plan::MergeInsertPlanner, }, index::DatasetIndexInternalExt, io::exec::{ - AddRowAddrExec, Planner, TakeExec, project, + Planner, project, scalar_index::{IndexLookup, MapIndexExec}, utils::ReplayExec, }, @@ -76,6 +77,8 @@ use datafusion::common::NullEquality; use datafusion::common::tree_node::{Transformed, TreeNode}; use datafusion::error::DataFusionError; use datafusion::{ + catalog::{TableProvider, streaming::StreamingTable}, + datasource::MemTable, execution::{ context::{SessionConfig, SessionContext}, memory_pool::MemoryConsumer, @@ -89,6 +92,7 @@ use datafusion::{ repartition::RepartitionExec, sorts::sort::SortExec, stream::RecordBatchStreamAdapter, + streaming::PartitionStream, union::UnionExec, }, physical_planner::{DefaultPhysicalPlanner, PhysicalPlanner}, @@ -114,14 +118,16 @@ use lance_datafusion::{ chunker::chunk_stream, dataframe::BatchStreamGrouper, exec::{ - HardCapBatchSizeExec, LanceExecutionOptions, OneShotExec, analyze_plan, execute_plan, - get_session_context, + HardCapBatchSizeExec, LanceExecutionOptions, OneShotExec, OneShotPartitionStream, + analyze_plan, execute_plan, get_session_context, provider_to_stream, }, + spill::spilling_table_provider, utils::{StreamingWriteSource, reader_to_stream}, }; +#[cfg(test)] use lance_file::version::LanceFileVersion; use lance_index::IndexCriteria; -use lance_index::mem_wal::MergedGeneration; +use lance_index::mem_wal::CompactedSsTable; use lance_select::RowAddrTreeMap; use lance_table::format::{Fragment, IndexMetadata, RowIdMeta}; use log::info; @@ -130,9 +136,10 @@ use snafu::ResultExt; use std::collections::HashMap; use std::{ collections::{BTreeMap, HashSet}, + iter::Peekable, sync::{ Arc, Mutex, - atomic::{AtomicU32, Ordering}, + atomic::{AtomicU32, AtomicU64, Ordering}, }, time::Duration, }; @@ -143,6 +150,67 @@ mod assign_action; mod exec; mod logical_plan; +struct UpdatedRowAddrReconciler +where + I: Iterator, +{ + updated_rows: Peekable, +} + +impl UpdatedRowAddrReconciler +where + I: Iterator, +{ + fn new(updated_rows: I) -> Self { + Self { + updated_rows: updated_rows.peekable(), + } + } + + fn reconcile_batch(&mut self, original_row_addrs: &[u64]) -> Result> { + let mut indices = Vec::with_capacity(original_row_addrs.len()); + + for (original_offset, original_row_addr) in original_row_addrs.iter().enumerate() { + match self.updated_rows.peek().copied() { + Some((updated_row_addr, updated_row_index)) + if updated_row_addr == *original_row_addr => + { + self.updated_rows.next(); + indices.push(updated_row_index); + } + Some((updated_row_addr, _)) if updated_row_addr < *original_row_addr => { + return Err(Self::missing_row_error( + updated_row_addr, + Some(*original_row_addr), + )); + } + _ => indices.push((0, original_offset)), + } + } + + Ok(indices) + } + + fn finish(mut self) -> Result<()> { + if let Some((updated_row_addr, _)) = self.updated_rows.next() { + Err(Self::missing_row_error(updated_row_addr, None)) + } else { + Ok(()) + } + } + + fn missing_row_error(updated_row_addr: u64, next_original_row_addr: Option) -> Error { + let updated_row_addr = RowAddress::from(updated_row_addr); + let position = next_original_row_addr.map_or_else( + || "no target rows remain".to_string(), + |row_addr| format!("next target row address is {}", RowAddress::from(row_addr)), + ); + Error::internal(format!( + "Merge insert update row address {updated_row_addr} is missing from the target fragment; {position}" + )) + } +} + // "update if" expressions typically compare fields from the source table to the target table. // These tables have the same schema and so filter expressions need to differentiate. To do that // we wrap the left side and the right side in a struct and make a single "combined schema" @@ -227,6 +295,41 @@ pub fn create_duplicate_row_error( )))) } +/// Tracks non-null join keys for source rows that will be inserted. +/// +/// NULL join keys are deliberately not tracked because merge insert uses SQL +/// equality, where a key containing NULL does not equal another such key. +#[derive(Debug, Default)] +struct InsertedKeyTracker { + keys: HashSet>, +} + +impl InsertedKeyTracker { + /// Returns true when the row has a new key or a key containing NULL. + fn insert( + &mut self, + batch: &RecordBatch, + row_idx: usize, + on_columns: &[String], + ) -> datafusion::common::Result { + let mut key = Vec::with_capacity(on_columns.len()); + for column_name in on_columns { + let column = batch.column_by_name(column_name).ok_or_else(|| { + DataFusionError::Internal(format!( + "merge insert key column '{}' not found in source batch", + column_name + )) + })?; + let value = ScalarValue::try_from_array(column, row_idx)?; + if value.is_null() { + return Ok(true); + } + key.push(value); + } + Ok(self.keys.insert(key)) + } +} + /// Describes how rows should be handled when there is no matching row in the source table /// /// These are old rows which do not match any new data @@ -316,16 +419,21 @@ pub enum WhenNotMatched { DoNothing, } -/// Describes how to handle duplicate source rows that match the same target row. +/// Describes how to handle duplicate source rows. /// /// If the source contains duplicates and `FirstSeen` behavior doesn't match your needs, /// sort the source data before passing it to the merge insert operation. +/// Rows whose join keys contain NULL are not duplicates because merge insert uses SQL +/// equality, where NULL does not equal NULL. #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord, Default)] pub enum SourceDedupeBehavior { - /// Fail the operation if duplicates are found (default) + /// Fail if multiple source rows match the same target row (default) #[default] Fail, - /// Keep the first seen value and skip subsequent duplicates + /// Keep the first row for each join key and skip subsequent rows + /// + /// This applies both to rows that match a target row and to unmatched rows that + /// would otherwise insert the same non-null join key more than once. FirstSeen, } @@ -340,9 +448,15 @@ struct MergeInsertParams { // Controls whether data that is not matched by the source is deleted or not delete_not_matched_by_source: WhenNotMatchedBySource, conflict_retries: u32, + // When the source is a one-shot stream and `conflict_retries > 0`, the source + // is spilled (memory, then disk) so it can be replayed on each retry. Set to + // false to fail fast on contention instead of buffering the stream. Has no + // effect on re-scannable sources (materialized batches, files), which never + // spill. + spill_for_retry: bool, retry_timeout: Duration, - // List of MemWAL region generations to mark as merged when this commit succeeds. - merged_generations: Vec, + // MemWAL SSTables to mark as compacted when this commit succeeds. + compacted_sstables: Vec, // If true, skip auto cleanup during commits. This should be set to true // for high frequency writes to improve performance. This is also useful // if the writer does not have delete permissions and the clean up would @@ -477,8 +591,9 @@ impl MergeInsertBuilder { insert_not_matched: true, delete_not_matched_by_source: WhenNotMatchedBySource::Keep, conflict_retries: 10, + spill_for_retry: true, retry_timeout: Duration::from_secs(30), - merged_generations: Vec::new(), + compacted_sstables: Vec::new(), skip_auto_cleanup: false, use_index: true, source_dedupe_behavior: SourceDedupeBehavior::Fail, @@ -527,6 +642,27 @@ impl MergeInsertBuilder { self } + /// Controls whether a one-shot stream source is spilled so it can be replayed + /// across retries. + /// + /// When the source is a one-shot stream (e.g. [`MergeInsertJob::execute`]) and + /// `conflict_retries > 0`, the source is buffered in memory and spilled to disk + /// so each retry can re-read it. Set this to `false` to skip that buffering and + /// fail fast with a contention error instead of writing the stream to disk. + /// + /// This has no effect on re-scannable sources (materialized batches via + /// [`MergeInsertJob::execute_batches`], or a [`TableProvider`] via + /// [`MergeInsertJob::execute_provider`]), which are replayed directly and never + /// spill. + /// + /// Default is true. + /// + /// [`TableProvider`]: datafusion::catalog::TableProvider + pub fn spill_for_retry(&mut self, spill: bool) -> &mut Self { + self.params.spill_for_retry = spill; + self + } + /// Set the timeout used to limit retries. /// /// This is the maximum time to spend on the operation before giving up. At @@ -557,10 +693,13 @@ impl MergeInsertBuilder { self } - /// Specify how to handle duplicate source rows that match the same target row. + /// Specify how to handle duplicate source rows. /// - /// Default is `Fail` which errors on duplicates. - /// Use `FirstSeen` to keep the first encountered row and skip duplicates. + /// Default is `Fail`, which errors when multiple source rows match one target row. + /// Use `FirstSeen` to keep the first encountered row for each non-null join key, + /// including unmatched keys that will be inserted, and skip subsequent rows. + /// Join keys containing NULL are not deduplicated because merge insert uses SQL + /// equality, where NULL does not equal NULL. /// /// If the source contains duplicates and `FirstSeen` behavior doesn't match your needs, /// sort the source data before passing it to the merge insert operation. @@ -569,10 +708,19 @@ impl MergeInsertBuilder { self } - /// Mark MemWAL region generations as merged when this commit succeeds. - /// This updates the merged_generations in the MemWAL Index atomically with the data commit. - pub fn mark_generations_as_merged(&mut self, generations: Vec) -> &mut Self { - self.params.merged_generations.extend(generations); + /// Mark MemWAL SSTables as compacted when this commit succeeds. + /// + /// This updates `compacted_sstables` in the MemWAL index atomically with + /// the data commit. + /// + /// **For multi-pass compaction, call this only on the final successful + /// data-changing pass.** Intermediate passes must not carry compaction + /// progress. Lance cannot tell whether a caller has another pass planned, + /// so it cannot enforce this: if a delete pass carried the marker and the + /// process then died before the matching upsert, the recorded generation + /// would claim rows were copied in that never were. + pub fn mark_sstables_as_compacted(&mut self, sstables: Vec) -> &mut Self { + self.params.compacted_sstables.extend(sstables); self } @@ -698,6 +846,114 @@ enum SchemaComparison { Subschema, } +/// Wrap a one-shot stream in a non-replayable [`StreamingTable`] provider. +/// +/// The provider can only be scanned once (its single partition hands out the +/// underlying stream), so it must not be used where retries may re-scan it. +fn one_shot_provider(stream: SendableRecordBatchStream) -> Result> { + let schema = stream.schema(); + let partition = Arc::new(OneShotPartitionStream::new(stream)); + Ok(Arc::new(StreamingTable::try_new(schema, vec![partition])?)) +} + +/// Scans source partitions sequentially and removes duplicate non-null keys. +/// +/// Deduplicating before the join fixes the `FirstSeen` winner at the source +/// boundary, before DataFusion can reorder rows. The tracker retains only keys, +/// so this stays streaming without buffering source batches. +#[derive(Debug)] +struct DeduplicatingSourcePartitionStream { + input: Arc, + schema: Arc, + on_columns: Vec, + skipped_duplicates: Arc, +} + +impl DeduplicatingSourcePartitionStream { + fn new( + input: Arc, + on_columns: Vec, + skipped_duplicates: Arc, + ) -> Self { + let schema = input.schema(); + Self { + input, + schema, + on_columns, + skipped_duplicates, + } + } +} + +impl PartitionStream for DeduplicatingSourcePartitionStream { + fn schema(&self) -> &Arc { + &self.schema + } + + fn execute( + &self, + context: Arc, + ) -> SendableRecordBatchStream { + let input = self.input.clone(); + let partition_count = input.properties().output_partitioning().partition_count(); + let partition_streams = stream::iter(0..partition_count) + .map(move |partition| input.execute(partition, context.clone())) + .try_flatten(); + + let mut tracker = InsertedKeyTracker::default(); + let on_columns = self.on_columns.clone(); + let skipped_duplicates = self.skipped_duplicates.clone(); + skipped_duplicates.store(0, Ordering::Relaxed); + let deduplicated = partition_streams.map(move |batch| { + let batch = batch?; + let mut keep = Vec::with_capacity(batch.num_rows()); + let mut num_skipped = 0_u64; + for row_idx in 0..batch.num_rows() { + let is_first = tracker.insert(&batch, row_idx, &on_columns)?; + keep.push(is_first); + if !is_first { + num_skipped = num_skipped.checked_add(1).ok_or_else(|| { + DataFusionError::Execution( + "source duplicate count overflowed u64".to_string(), + ) + })?; + } + } + + let mut current = skipped_duplicates.load(Ordering::Relaxed); + loop { + let updated = current.checked_add(num_skipped).ok_or_else(|| { + DataFusionError::Execution(format!( + "source duplicate count overflow at {} with batch count {}", + current, num_skipped + )) + })?; + match skipped_duplicates.compare_exchange_weak( + current, + updated, + Ordering::Relaxed, + Ordering::Relaxed, + ) { + Ok(_) => break, + Err(actual) => current = actual, + } + } + + if num_skipped == 0 { + Ok(batch) + } else { + arrow::compute::filter_record_batch(&batch, &BooleanArray::from(keep)) + .map_err(DataFusionError::from) + } + }); + + Box::pin(RecordBatchStreamAdapter::new( + self.schema.clone(), + deduplicated, + )) + } +} + impl MergeInsertJob { pub async fn execute_reader( self, @@ -711,11 +967,13 @@ impl MergeInsertJob { let lance_schema: lance_core::datatypes::Schema = schema.try_into()?; let target_schema = self.dataset.schema(); - let mut options = SchemaCompareOptions { - compare_dictionary: self.dataset.is_legacy_storage(), - compare_nullability: NullabilityComparison::Ignore, - ..Default::default() - }; + let version = self + .dataset + .manifest() + .data_storage_format + .lance_file_format(); + let mut options = versions::schema_compare_options(version); + options.compare_nullability = NullabilityComparison::Ignore; // Try full schema match first. if lance_schema @@ -825,30 +1083,28 @@ impl MergeInsertJob { .iter() .map(|(col, idx)| IndexLookup::new(col.clone(), idx.name.clone())) .collect::>(); - let mut index_mapper: Arc = Arc::new(MapIndexExec::new_multi( + let index_mapper: Arc = Arc::new(MapIndexExec::new_multi( self.dataset.clone(), lookups, index_mapper_input, )); - // If requested, add row addresses to the output - if add_row_addr { - let pos = index_mapper.schema().fields().len(); // Add to end - index_mapper = Arc::new(AddRowAddrExec::try_new( - index_mapper, - self.dataset.clone(), - pos, - )?); - } - - // 4 - Take the mapped row ids + // 4 - Take the mapped row ids (TakeExec stays for legacy storage: + // the v1 reader cannot serve a FilteredReadExec) let projection = self .dataset .empty_projection() .union_arrow_schema(schema.as_ref(), OnMissing::Error)?; - let mut target = - Arc::new(TakeExec::try_new(self.dataset.clone(), index_mapper, projection)?.unwrap()) - as Arc; + let mut target = versions::merge_insert_indexed_take( + self.dataset + .manifest() + .data_storage_format + .lance_file_format(), + self.dataset.clone(), + index_mapper, + projection, + add_row_addr, + )?; // 5 - Take puts the row id and row addr at the beginning. A full scan (used when there is // no scalar index) puts the row id and addr at the end. We need to match these up so @@ -1113,7 +1369,7 @@ impl MergeInsertJob { // sort node so each input batch fits in the memory pool. let capped_plan = sorted_plan .transform_down(|node| { - if node.as_any().downcast_ref::().is_some() { + if node.downcast_ref::().is_some() { let children = node.children(); let new_children: Vec> = children .into_iter() @@ -1194,21 +1450,48 @@ impl MergeInsertJob { )?; let updated_rows: usize = batches.iter().map(|batch| batch.num_rows()).sum(); - if Some(updated_rows) == metadata.physical_rows { - // All rows have been updated and there are no deletions. So we - // don't need to merge in existing values. - // Also, because we already sorted by row address, the rows - // will be in the correct order. - - let data_storage_version = dataset - .manifest() - .data_storage_format - .lance_file_version()?; - let mut writer = open_writer( + + // This function is here to help rustc with lifetimes. + fn get_row_addr_iter( + batches: &[RecordBatch], + ) -> impl Iterator + '_ + Send { + batches.iter().enumerate().flat_map(|(batch_idx, batch)| { + // The index in source batches will be one more. + let batch_idx = batch_idx + 1; + let row_addrs = batch + .column_by_name(ROW_ADDR) + .unwrap() + .as_any() + .downcast_ref::() + .unwrap(); + row_addrs + .values() + .iter() + .enumerate() + .map(move |(offset, row_addr)| (*row_addr, (batch_idx, offset))) + }) + } + + let has_full_fragment_coverage = metadata.deletion_file.is_none() + && Some(updated_rows) == metadata.physical_rows + && get_row_addr_iter(&batches) + .map(|(row_addr, _)| row_addr) + .eq(RowAddress::address_range(metadata.id as u32).take(updated_rows)); + if has_full_fragment_coverage { + // Exact, deletion-free coverage can be written directly because the + // batches are sorted by row address. + + let data_storage_version = + dataset.manifest().data_storage_format.lance_file_format(); + let mut writer = versions::open_writer( + data_storage_version, &dataset.object_store, &write_schema, &dataset.base, - data_storage_version, + super::WriterOptions { + add_data_dir: true, + ..Default::default() + }, ) .await?; @@ -1237,16 +1520,12 @@ impl MergeInsertJob { } } - if data_storage_version == LanceFileVersion::Legacy { + if let Some(batch_size) = + versions::row_group_size_for_rewrite(data_storage_version, &fragment) + .await? + { // Need to match the existing batch size exactly, otherwise // we'll get errors. - let reader = fragment - .open( - dataset.schema(), - FragReadConfig::default().with_row_address(true), - ) - .await?; - let batch_size = reader.legacy_num_rows_in_batch(0).unwrap(); let stream = stream::iter(batches.into_iter().map(Ok)); let stream = Box::pin(RecordBatchStreamAdapter::new( Arc::new((&write_schema).into()), @@ -1282,6 +1561,7 @@ impl MergeInsertJob { Some(&read_columns), Some((write_schema, dataset.schema().clone())), None, + None, ) .await?; @@ -1289,33 +1569,20 @@ impl MergeInsertJob { // will be the original source data, and all subsequent batches // will be updates. let mut source_batches = Vec::with_capacity(batches.len() + 1); - source_batches.push(batches[0].clone()); // placeholder for source data + // Convert Arrow JSON columns (Utf8) to Lance JSON (LargeBinary) so every + // batch is in physical format, matching what the updater reads from the + // fragment. `convert_json_columns` is a no-op clone when there is nothing + // to convert, so it can be applied unconditionally. The first entry is a + // placeholder for the source data (overwritten each iteration below); it + // must be converted too, otherwise its schema would diverge from the rest. + source_batches.push(convert_json_columns(&batches[0]).map_err(Error::from)?); for batch in &batches { - source_batches.push(batch.drop_column(ROW_ADDR)?); + let dropped = batch.drop_column(ROW_ADDR)?; + source_batches.push(convert_json_columns(&dropped).map_err(Error::from)?); } - // This function is here to help rustc with lifetimes. - fn get_row_addr_iter( - batches: &[RecordBatch], - ) -> impl Iterator + '_ + Send - { - batches.iter().enumerate().flat_map(|(batch_idx, batch)| { - // The index in source batches will be one more. - let batch_idx = batch_idx + 1; - let row_addrs = batch - .column_by_name(ROW_ADDR) - .unwrap() - .as_any() - .downcast_ref::() - .unwrap(); - row_addrs - .values() - .iter() - .enumerate() - .map(move |(offset, row_addr)| (*row_addr, (batch_idx, offset))) - }) - } - let mut updated_row_addr_iter = get_row_addr_iter(&batches).peekable(); + let mut updated_rows = + UpdatedRowAddrReconciler::new(get_row_addr_iter(&batches)); while let Some(batch) = updater.next().await? { source_batches[0] = @@ -1327,34 +1594,13 @@ impl MergeInsertJob { .as_any() .downcast_ref::() .unwrap(); - let indices = original_row_addrs - .values() - .into_iter() - .enumerate() - .map(|(original_offset, row_addr)| { - match updated_row_addr_iter.peek() { - Some((updated_row_addr, _)) - if *updated_row_addr == *row_addr => - { - updated_row_addr_iter.next().unwrap().1 - } - // If we have passed the next updated row address, something went wrong. - Some((updated_row_addr, _)) => { - debug_assert!( - *updated_row_addr > *row_addr, - "Got updated row address that is not in the original batch" - ); - (0, original_offset) - } - _ => (0, original_offset), - } - }) - .collect::>(); + let indices = updated_rows.reconcile_batch(original_row_addrs.values())?; let updated_batch = interleave_batches(&source_batches, &indices)?; updater.update(updated_batch).await?; } + updated_rows.finish()?; let mut updated_fragment = updater.finish().await?; @@ -1423,6 +1669,7 @@ impl MergeInsertJob { )?; let (fragments, _) = write_fragments_internal( + dataset.manifest.data_storage_format.lance_file_format(), Some(dataset.as_ref()), dataset.object_store.clone(), &dataset.base, @@ -1590,24 +1837,139 @@ impl MergeInsertJob { )) } - /// Executes the merge insert job + /// Executes the merge insert job from a one-shot stream source. /// /// This will take in the source, merge it with the existing target data, and insert new - /// rows, update existing rows, and delete existing rows + /// rows, update existing rows, and delete existing rows. + /// + /// A stream can only be read once, so when `conflict_retries > 0` the stream is + /// spilled (in memory, then to disk) so it can be replayed on each retry. See + /// [`MergeInsertBuilder::spill_for_retry`] to fail fast instead, and + /// [`Self::execute_batches`] / [`Self::execute_provider`] for re-scannable + /// sources that never spill. pub async fn execute( self, source: SendableRecordBatchStream, ) -> Result<(Arc, MergeStats)> { - let source_iter = super::new_source_iter(source, self.params.conflict_retries > 0).await?; + let (provider, replayable) = self.stream_source_to_provider(source).await?; + self.execute_inner(provider, replayable).await + } + + /// Executes the merge insert job from a re-scannable [`TableProvider`]. + /// + /// This is the canonical entry point: [`Self::execute`] and + /// [`Self::execute_batches`] are thin wrappers that build a provider and call + /// this method. Because a provider can be scanned repeatedly, retries re-read + /// the source directly and never spill to disk. The provider's reported + /// statistics (e.g. from a [`MemTable`] or file source) also let DataFusion + /// optimize the merge join. + /// + /// [`MemTable`]: datafusion::datasource::MemTable + pub async fn execute_provider( + self, + provider: Arc, + ) -> Result<(Arc, MergeStats)> { + // A genuine TableProvider is re-scannable by contract, so retries are safe. + self.execute_inner(provider, true).await + } + + /// Executes the merge insert job from materialized record batches. + /// + /// The batches are wrapped in an in-memory [`MemTable`], which is re-scannable + /// (retries replay from memory, never spilling) and reports exact statistics to + /// the merge join. This is the preferred entry point when the full source is + /// already in memory. + pub async fn execute_batches( + self, + batches: Vec, + ) -> Result<(Arc, MergeStats)> { + let provider = self.batches_to_provider(batches)?; + self.execute_inner(provider, true).await + } + + /// Like [`Self::execute_batches`] but returns the uncommitted transaction. + /// + /// Use [`CommitBuilder`] to commit the returned transaction. + pub async fn execute_uncommitted_batches( + self, + batches: Vec, + ) -> Result { + let provider = self.batches_to_provider(batches)?; + self.execute_uncommitted_impl(provider).await + } + + /// Wrap materialized batches in a multi-partition in-memory [`MemTable`]. + fn batches_to_provider(&self, batches: Vec) -> Result> { + let schema = batches + .first() + .map(|batch| batch.schema()) + .unwrap_or_else(|| Arc::new(Schema::from(self.dataset.schema()))); + // FirstSeen needs a defined encounter order. Keep materialized batches in + // their caller-provided order; other modes retain parallel source scans. + let partitions = if self.params.source_dedupe_behavior == SourceDedupeBehavior::FirstSeen { + vec![batches] + } else { + Self::batches_into_partitions(batches) + }; + Ok(Arc::new(MemTable::try_new(schema, partitions)?)) + } + + /// Distribute batches round-robin across up to `num_compute_intensive_cpus` + /// partitions, so a [`MemTable`] built from them can be scanned in parallel. + /// Always returns at least one (possibly empty) partition so an empty source + /// still produces a valid provider. + fn batches_into_partitions(batches: Vec) -> Vec> { + let num_partitions = batches.len().min(get_num_compute_intensive_cpus()).max(1); + let mut partitions = vec![Vec::new(); num_partitions]; + for (idx, batch) in batches.into_iter().enumerate() { + partitions[idx % num_partitions].push(batch); + } + partitions + } + + /// Wrap a one-shot stream source in a provider, returning whether it can be + /// replayed across retries. + /// + /// With retries enabled and spilling allowed, the stream is drained into a + /// replayable spill (memory up to 100MB, then disk). Otherwise the stream is + /// wrapped in a non-replayable one-shot provider and any conflict fails fast. + async fn stream_source_to_provider( + &self, + source: SendableRecordBatchStream, + ) -> Result<(Arc, bool)> { + if self.params.conflict_retries > 0 && self.params.spill_for_retry { + // Allow buffering up to 100MB in memory before spilling to disk. + let provider = spilling_table_provider(source, 100 * 1024 * 1024).await?; + Ok((provider, true)) + } else { + Ok((one_shot_provider(source)?, false)) + } + } + + /// Run the retry loop against a provider, re-scanning it on each attempt. + /// + /// `replayable` indicates whether the provider can be scanned more than once. + /// When it cannot (a one-shot stream that was not spilled), retries are + /// disabled so we never scan it twice; the operation runs once and surfaces any + /// commit conflict directly. + async fn execute_inner( + self, + provider: Arc, + replayable: bool, + ) -> Result<(Arc, MergeStats)> { let dataset = self.dataset.clone(); let config = RetryConfig { - max_retries: self.params.conflict_retries, + max_retries: if replayable { + self.params.conflict_retries + } else { + 0 + }, retry_timeout: self.params.retry_timeout, }; - let wrapper = MergeInsertJobWithIterator { + let wrapper = MergeInsertJobWithProvider { job: self, - source_iter: Arc::new(Mutex::new(source_iter)), + provider, attempt_count: Arc::new(AtomicU32::new(0)), }; @@ -1622,7 +1984,8 @@ impl MergeInsertJob { source: impl StreamingWriteSource, ) -> Result { let stream = source.into_stream(); - self.execute_uncommitted_impl(stream).await + self.execute_uncommitted_impl(one_shot_provider(stream)?) + .await } fn create_plan_join_type(&self) -> JoinType { @@ -1640,18 +2003,32 @@ impl MergeInsertJob { } } - async fn create_plan( - self, - source: SendableRecordBatchStream, - ) -> Result> { + async fn create_plan(self, provider: Arc) -> Result> { // Goal: we shouldn't manually have to specify which columns to scan. // DataFusion's optimizer should be able to automatically perform // projection pushdown for us. // Goal: we shouldn't have to add new branches in this code to handle // indexed vs non-indexed cases. That should be handled by optimizer rules. - let session_config = SessionConfig::default(); - let session_ctx = SessionContext::new_with_config(session_config); - let scan = session_ctx.read_lance_unordered(self.dataset.clone(), true, true)?; + let session_ctx = SessionContext::new(); + let binary_blob_field_ids = self + .dataset + .schema() + .fields_pre_order() + .filter(|field| field.is_blob() && !field.is_blob_v2()) + .map(|field| field.id as u32) + .collect(); + let target_provider = Arc::new( + crate::datafusion::dataframe::LanceTableProvider::new_with_ordering( + self.dataset.clone(), + true, + true, + false, + ) + .with_blob_handling(lance_core::datatypes::BlobHandling::SomeBlobsBinary( + binary_blob_field_ids, + )), + ); + let scan = session_ctx.read_table(target_provider)?; // Wrap column names in double quotes to preserve case (DataFusion lowercases unquoted identifiers) let on_cols = self .params @@ -1660,7 +2037,28 @@ impl MergeInsertJob { .map(|name| format!("\"{}\"", name)) .collect::>(); let on_cols_refs = on_cols.iter().map(|s| s.as_str()).collect::>(); - let source_df = session_ctx.read_one_shot(source)?; + // FirstSeen must observe the caller's source order even though the join can + // reorder batches. Deduplicating source partitions sequentially before + // the join fixes the winner at that contract boundary. Other modes plan + // directly against the provider so its statistics reach the optimizer. + let deduplicate_source = + self.params.source_dedupe_behavior == SourceDedupeBehavior::FirstSeen; + let source_skipped_duplicates = Arc::new(AtomicU64::new(0)); + let source_df = if deduplicate_source { + let source_plan = provider.scan(&session_ctx.state(), None, &[], None).await?; + let deduplicated_partition = Arc::new(DeduplicatingSourcePartitionStream::new( + source_plan, + self.params.on.clone(), + source_skipped_duplicates.clone(), + )); + let deduplicated_provider = Arc::new(StreamingTable::try_new( + deduplicated_partition.schema().clone(), + vec![deduplicated_partition], + )?); + session_ctx.read_table(deduplicated_provider)? + } else { + session_ctx.read_table(provider)? + }; // Capture the source field names *before* aliasing / joining so we // can tell which dataset columns are missing from the source and // need to be filled from the target side of the join below. @@ -1720,6 +2118,7 @@ impl MergeInsertJob { logical_plan, self.dataset.clone(), self.params.clone(), + source_skipped_duplicates, ); let logical_plan = LogicalPlan::Extension(Extension { node: Arc::new(write_node), @@ -1739,14 +2138,14 @@ impl MergeInsertJob { async fn execute_uncommitted_v2( self, - source: SendableRecordBatchStream, + provider: Arc, ) -> Result<( Transaction, MergeStats, Option, Option, )> { - let plan = self.create_plan(source).await?; + let plan = self.create_plan(provider).await?; // Execute the plan // Assert that we have exactly one partition since we're designed for single-partition execution @@ -1780,8 +2179,7 @@ impl MergeInsertJob { // Extract merge stats from the execution plan let (stats, transaction, affected_rows, inserted_rows_filter) = if let Some(full_exec) = - plan.as_any() - .downcast_ref::() + plan.downcast_ref::() { let stats = full_exec.merge_stats().ok_or_else(|| { Error::internal("Merge stats not available - execution may not have completed") @@ -1792,10 +2190,7 @@ impl MergeInsertJob { let affected_rows = full_exec.affected_rows().map(RowAddrTreeMap::from); let inserted_rows_filter = full_exec.inserted_rows_filter(); (stats, transaction, affected_rows, inserted_rows_filter) - } else if let Some(delete_exec) = plan - .as_any() - .downcast_ref::() - { + } else if let Some(delete_exec) = plan.downcast_ref::() { let stats = delete_exec.merge_stats().ok_or_else(|| { Error::internal("Merge stats not available - execution may not have completed") })?; @@ -1935,14 +2330,14 @@ impl MergeInsertJob { async fn execute_uncommitted_impl( self, - source: SendableRecordBatchStream, + provider: Arc, ) -> Result { // Check if we can use the fast path - let can_use_fast_path = self.can_use_create_plan(source.schema().as_ref()).await?; + let can_use_fast_path = self.can_use_create_plan(provider.schema().as_ref()).await?; if can_use_fast_path { let (transaction, stats, affected_rows, inserted_rows_filter) = - self.execute_uncommitted_v2(source).await?; + self.execute_uncommitted_v2(provider).await?; return Ok(UncommittedMergeInsert { transaction, affected_rows, @@ -1953,6 +2348,8 @@ impl MergeInsertJob { let target_bases_info = resolve_target_bases(&self.dataset, &self.params).await?; + // The slow path consumes a single stream; adapt the provider back into one. + let source = provider_to_stream(provider).await?; let source_schema = source.schema(); let lance_schema = lance_core::datatypes::Schema::try_from(source_schema.as_ref())?; let full_schema = self.dataset.schema(); @@ -2014,7 +2411,7 @@ impl MergeInsertJob { } else { removed_row_ids }; - let removed_row_addrs = RoaringTreemap::from_iter(removed_row_addr_vec.into_iter()); + let removed_row_addrs = RoaringTreemap::from_iter(removed_row_addr_vec); let (updated_fragments, removed_fragment_ids) = Self::apply_deletions(&self.dataset, &removed_row_addrs).await?; @@ -2024,7 +2421,7 @@ impl MergeInsertJob { updated_fragments, new_fragments: vec![], fields_modified: vec![], - merged_generations: self.params.merged_generations.clone(), + compacted_sstables: self.params.compacted_sstables.clone(), fields_for_preserving_frag_bitmap: full_schema .fields .iter() @@ -2063,7 +2460,7 @@ impl MergeInsertJob { updated_fragments, new_fragments, fields_modified, - merged_generations: self.params.merged_generations.clone(), + compacted_sstables: self.params.compacted_sstables.clone(), fields_for_preserving_frag_bitmap: vec![], // in-place update do not affect preserving frag bitmap update_mode: Some(RewriteColumns), inserted_rows_filter: None, // not implemented for v1 @@ -2075,6 +2472,10 @@ impl MergeInsertJob { } else { let cleanup_bases = target_bases_info.clone(); let (mut new_fragments, _) = write_fragments_internal( + self.dataset + .manifest + .data_storage_format + .lance_file_format(), Some(&self.dataset), self.dataset.object_store.clone(), &self.dataset.base, @@ -2107,7 +2508,7 @@ impl MergeInsertJob { for (fragment, sequence) in new_fragments.iter_mut().zip(sequences) { let serialized = lance_table::rowids::write_row_ids(&sequence); - fragment.row_id_meta = Some(RowIdMeta::Inline(serialized)); + fragment.row_id_meta = Some(RowIdMeta::Inline(serialized.into())); } } @@ -2125,7 +2526,7 @@ impl MergeInsertJob { removed_row_ids }; - Ok(RoaringTreemap::from_iter(removed_row_addr_vec.into_iter())) + Ok(RoaringTreemap::from_iter(removed_row_addr_vec)) } .await; let removed_row_addrs = match post_write_result { @@ -2165,7 +2566,7 @@ impl MergeInsertJob { // On this path we only make deletions against updated_fragments and will not // modify any field values. fields_modified: vec![], - merged_generations: self.params.merged_generations.clone(), + compacted_sstables: self.params.compacted_sstables.clone(), fields_for_preserving_frag_bitmap: full_schema .fields .iter() @@ -2278,7 +2679,9 @@ impl MergeInsertJob { // Clone self since create_plan consumes the job let cloned_job = self.clone(); - let plan = cloned_job.create_plan(Box::pin(stream)).await?; + let plan = cloned_job + .create_plan(one_shot_provider(Box::pin(stream))?) + .await?; let display = DisplayableExecutionPlan::new(plan.as_ref()); Ok(format!("{}", display.indent(verbose))) @@ -2311,7 +2714,7 @@ impl MergeInsertJob { // Clone self since create_plan consumes the job let cloned_job = self.clone(); - let plan = cloned_job.create_plan(source).await?; + let plan = cloned_job.create_plan(one_shot_provider(source)?).await?; // Use the analyze_plan function from lance_datafusion, but strip out the wrapper lines let options = LanceExecutionOptions::default(); @@ -2361,15 +2764,15 @@ pub struct UncommittedMergeInsert { pub inserted_rows_filter: Option, } -/// Wrapper struct that combines MergeInsertJob with the source iterator for retry functionality +/// Wrapper struct that combines MergeInsertJob with the source provider for retry functionality #[derive(Clone)] -struct MergeInsertJobWithIterator { +struct MergeInsertJobWithProvider { job: MergeInsertJob, - source_iter: Arc + Send + 'static>>>, + provider: Arc, attempt_count: Arc, } -impl RetryExecutor for MergeInsertJobWithIterator { +impl RetryExecutor for MergeInsertJobWithProvider { type Data = UncommittedMergeInsert; type Result = (Arc, MergeStats); @@ -2377,10 +2780,11 @@ impl RetryExecutor for MergeInsertJobWithIterator { // Increment attempt counter self.attempt_count.fetch_add(1, Ordering::SeqCst); - // We need to get a fresh stream for each retry attempt - // The source_iter provides unlimited streams from the same source data - let stream = self.source_iter.lock().unwrap().next().unwrap(); - self.job.clone().execute_uncommitted_impl(stream).await + // Re-scan the provider on each retry attempt. + self.job + .clone() + .execute_uncommitted_impl(self.provider.clone()) + .await } async fn commit(&self, dataset: Arc, mut data: Self::Data) -> Result { @@ -2464,6 +2868,8 @@ struct Merger { enable_stable_row_ids: bool, /// Set to track processed row IDs to detect duplicates processed_row_ids: Arc>>, + /// Set to track non-null keys of rows inserted by FirstSeen mode + processed_insert_keys: Arc>, } impl Merger { @@ -2531,6 +2937,7 @@ impl Merger { output_schema, enable_stable_row_ids, processed_row_ids: Arc::new(Mutex::new(HashSet::new())), + processed_insert_keys: Arc::new(Mutex::new(InsertedKeyTracker::default())), }) } @@ -2792,7 +3199,24 @@ impl Merger { } } if self.params.insert_not_matched { - let not_matched = arrow::compute::filter_record_batch(&batch, &left_only)?; + let mut not_matched = arrow::compute::filter_record_batch(&batch, &left_only)?; + if self.params.source_dedupe_behavior == SourceDedupeBehavior::FirstSeen { + let mut processed_insert_keys = self.processed_insert_keys.lock().unwrap(); + let mut keep_indices = Vec::with_capacity(not_matched.num_rows()); + for row_idx in 0..not_matched.num_rows() { + if processed_insert_keys.insert(¬_matched, row_idx, &self.params.on)? { + keep_indices.push(row_idx as u32); + } else { + merge_statistics.num_skipped_duplicates += 1; + } + } + drop(processed_insert_keys); + + if keep_indices.len() != not_matched.num_rows() { + not_matched = + take_record_batch(¬_matched, &UInt32Array::from(keep_indices))?; + } + } let left_cols_with_id = left_cols .into_iter() .chain(row_addr_col) @@ -2867,7 +3291,7 @@ mod tests { use arrow_array::types::Float32Type; use arrow_array::{ Array, FixedSizeListArray, Float32Array, Float64Array, Int32Array, Int64Array, ListArray, - RecordBatchIterator, RecordBatchReader, StringArray, StructArray, UInt32Array, + NullArray, RecordBatchIterator, RecordBatchReader, StringArray, StructArray, UInt32Array, types::{Int32Type, UInt32Type}, }; use arrow_array::{RecordBatch, record_batch}; @@ -2896,6 +3320,88 @@ mod tests { t } + #[test] + fn test_inserted_key_tracker_preserves_logical_nulls() { + let batch = RecordBatch::try_new( + Arc::new(Schema::new(vec![Field::new("id", DataType::Null, true)])), + vec![Arc::new(NullArray::new(2))], + ) + .unwrap(); + let mut tracker = InsertedKeyTracker::default(); + let on_columns = ["id".to_string()]; + + assert!(tracker.insert(&batch, 0, &on_columns).unwrap()); + assert!(tracker.insert(&batch, 1, &on_columns).unwrap()); + } + + #[test] + fn test_updated_row_addr_missing_between_target_rows() { + let row_addr = |offset| u64::from(RowAddress::new_from_parts(3, offset)); + let mut updated_rows = UpdatedRowAddrReconciler::new([(row_addr(1), (1, 0))].into_iter()); + + let error = updated_rows + .reconcile_batch(&[row_addr(0), row_addr(2)]) + .unwrap_err(); + + assert!(matches!(error, Error::Internal { .. })); + let message = error.to_string(); + assert!(message.contains("update row address (3, 1) is missing")); + assert!(message.contains("next target row address is (3, 2)")); + } + + #[test] + fn test_updated_row_addr_missing_after_target_rows() { + let row_addr = |offset| u64::from(RowAddress::new_from_parts(7, offset)); + let mut updated_rows = UpdatedRowAddrReconciler::new([(row_addr(2), (1, 0))].into_iter()); + + assert_eq!( + updated_rows + .reconcile_batch(&[row_addr(0), row_addr(1)]) + .unwrap(), + vec![(0, 0), (0, 1)] + ); + let error = updated_rows.finish().unwrap_err(); + + assert!(matches!(error, Error::Internal { .. })); + let message = error.to_string(); + assert!(message.contains("update row address (7, 2) is missing")); + assert!(message.contains("no target rows remain")); + } + + #[tokio::test] + async fn test_updated_row_addr_missing_in_full_fragment_update() { + let initial = record_batch!(("value", Int32, [10, 20])).unwrap(); + let dataset = Arc::new( + InsertBuilder::new("memory://") + .execute(vec![initial]) + .await + .unwrap(), + ); + let fragment_id = dataset.get_fragments()[0].id() as u32; + let row_addr = |offset| u64::from(RowAddress::new_from_parts(fragment_id, offset)); + let updates = record_batch!( + (ROW_ADDR, UInt64, [row_addr(0), row_addr(2)]), + ("value", Int32, [100, 200]) + ) + .unwrap(); + let update_stream = + RecordBatchStreamAdapter::new(updates.schema(), futures::stream::iter([Ok(updates)])); + + let error = MergeInsertJob::update_fragments( + dataset.clone(), + Box::pin(update_stream), + dataset.manifest().version + 1, + None, + ) + .await + .unwrap_err(); + + assert!(matches!(error, Error::Internal { .. })); + let message = error.to_string(); + assert!(message.contains("update row address (0, 2) is missing")); + assert!(message.contains("no target rows remain")); + } + // An update-style merge_insert leaves the source and new fragments with // overlapping id ranges; a scattered delete punches holes in that range. A // filtered `with_row_id` scan must still resolve every id (round-tripped via take). @@ -3852,32 +4358,111 @@ mod tests { #[rstest::rstest] #[tokio::test] - async fn test_row_id_stability_across_update_and_merge_insert( - #[values(LanceFileVersion::Legacy, LanceFileVersion::V2_0)] version: LanceFileVersion, - #[values(true, false)] enable_stable_row_ids: bool, + async fn test_multi_batch_upsert_preserves_stable_row_ids( + #[values(true, false)] use_index: bool, ) { - let schema = create_test_schema(); - let test_uri = "memory://test_row_id_stability.lance"; - - let mut dataset = create_test_dataset(test_uri, version, enable_stable_row_ids).await; - - let target_key = 2u32; - let target_keys = vec![target_key]; - - let initial_row_ids = get_row_ids_for_keys(&dataset, &target_keys).await; - let initial_row_id = initial_row_ids.value(0); - - let initial_batch = dataset - .scan() - .filter(&format!("key = {}", target_key)) - .unwrap() - .with_row_id() - .try_into_batch() + let mut dataset = (*create_test_dataset( + "memory://test_multi_batch_upsert_row_ids", + LanceFileVersion::default(), + true, + ) + .await) + .clone(); + dataset + .create_index( + &["key"], + IndexType::Scalar, + None, + &ScalarIndexParams::default(), + false, + ) .await .unwrap(); + let dataset = Arc::new(dataset); - let initial_value = initial_batch - .column_by_name("value") + let initial_keys = [1, 2, 3, 4, 5, 6]; + let initial_row_ids = get_row_ids_for_keys(&dataset, &initial_keys).await; + let initial_row_ids = initial_row_ids + .values() + .iter() + .copied() + .collect::>(); + let updated_keys = [2, 4]; + let updated_row_ids_before = get_row_ids_for_keys(&dataset, &updated_keys).await; + + // Put inserts in the first source batch and updates in the second. Stable + // row-id assignment still relies on the join emitting all updates first. + let insert_batch = record_batch!( + ("key", UInt32, [7, 8]), + ("value", UInt32, [70, 80]), + ("filterme", Utf8, ["inserted", "inserted"]) + ) + .unwrap(); + let update_batch = record_batch!( + ("key", UInt32, [2, 4]), + ("value", UInt32, [20, 40]), + ("filterme", Utf8, ["updated", "updated"]) + ) + .unwrap(); + let source_schema = insert_batch.schema(); + let source = Box::new(RecordBatchIterator::new( + [Ok(insert_batch), Ok(update_batch)], + source_schema, + )); + + let (dataset, stats) = MergeInsertBuilder::try_new(dataset, vec!["key".to_string()]) + .unwrap() + .when_matched(WhenMatched::UpdateAll) + .when_not_matched(WhenNotMatched::InsertAll) + .use_index(use_index) + .try_build() + .unwrap() + .execute_reader(source) + .await + .unwrap(); + + assert_eq!(stats.num_updated_rows, 2); + assert_eq!(stats.num_inserted_rows, 2); + let updated_row_ids_after = get_row_ids_for_keys(&dataset, &updated_keys).await; + assert_eq!(updated_row_ids_after, updated_row_ids_before); + + let inserted_row_ids = get_row_ids_for_keys(&dataset, &[7, 8]).await; + assert!( + inserted_row_ids + .values() + .iter() + .all(|row_id| !initial_row_ids.contains(row_id)) + ); + } + + #[rstest::rstest] + #[tokio::test] + async fn test_row_id_stability_across_update_and_merge_insert( + #[values(LanceFileVersion::Legacy, LanceFileVersion::V2_0)] version: LanceFileVersion, + #[values(true, false)] enable_stable_row_ids: bool, + ) { + let schema = create_test_schema(); + let test_uri = "memory://test_row_id_stability.lance"; + + let mut dataset = create_test_dataset(test_uri, version, enable_stable_row_ids).await; + + let target_key = 2u32; + let target_keys = vec![target_key]; + + let initial_row_ids = get_row_ids_for_keys(&dataset, &target_keys).await; + let initial_row_id = initial_row_ids.value(0); + + let initial_batch = dataset + .scan() + .filter(&format!("key = {}", target_key)) + .unwrap() + .with_row_id() + .try_into_batch() + .await + .unwrap(); + + let initial_value = initial_batch + .column_by_name("value") .unwrap() .as_primitive::() .value(0); @@ -4353,6 +4938,79 @@ mod tests { assert_eq!(inserted, 1); } + /// A composite index probe can over-match the exact join key, but a target + /// row reached by more than one source batch must still enter the join once. + #[tokio::test] + async fn test_indexed_merge_insert_deduplicates_cross_batch_candidates() { + let initial = record_batch!( + ("a", Int32, [1, 1, 2, 2]), + ("b", Int32, [10, 20, 10, 20]), + ("value", Int32, [100, 200, 300, 400]) + ) + .unwrap(); + let schema = initial.schema(); + + let mut dataset = Dataset::write( + RecordBatchIterator::new(vec![Ok(initial)], schema.clone()), + "memory://", + None, + ) + .await + .unwrap(); + + let params = ScalarIndexParams::default(); + dataset + .create_index(&["a"], IndexType::Scalar, None, ¶ms, false) + .await + .unwrap(); + dataset + .create_index(&["b"], IndexType::Scalar, None, ¶ms, false) + .await + .unwrap(); + + let first = record_batch!( + ("a", Int32, [1]), + ("b", Int32, [10]), + ("value", Int32, [901]) + ) + .unwrap(); + // This batch probes `a IN (1, 2) AND b IN (20, 10)`, which reaches + // (1, 10) again even though that tuple is not present in this batch. + let second = record_batch!( + ("a", Int32, [1, 2]), + ("b", Int32, [20, 10]), + ("value", Int32, [902, 903]) + ) + .unwrap(); + + let (dataset, stats) = + MergeInsertBuilder::try_new(Arc::new(dataset), vec!["a".to_string(), "b".to_string()]) + .unwrap() + .when_matched(WhenMatched::UpdateAll) + .when_not_matched(WhenNotMatched::InsertAll) + .try_build() + .unwrap() + .execute_reader(Box::new(RecordBatchIterator::new( + vec![Ok(first), Ok(second)], + schema, + ))) + .await + .unwrap(); + + assert_eq!(stats.num_updated_rows, 3); + assert_eq!(stats.num_inserted_rows, 0); + assert_eq!(dataset.count_rows(None).await.unwrap(), 4); + for (a, b, value) in [(1, 10, 901), (1, 20, 902), (2, 10, 903), (2, 20, 400)] { + assert_eq!( + dataset + .count_rows(Some(format!("a = {a} AND b = {b} AND value = {value}"))) + .await + .unwrap(), + 1, + ); + } + } + /// Composite key merge_insert with no scalar index on any join column /// must keep working via the full-scan fallback. Guards against the /// indexed path becoming a hard requirement after this optimization. @@ -6524,7 +7182,10 @@ mod tests { let new_data = new_data.into_reader_rows(RowCount::from(512), BatchCount::from(16)); let new_data_stream = reader_to_stream(Box::new(new_data)); - let plan = merge_insert_job.create_plan(new_data_stream).await.unwrap(); + let plan = merge_insert_job + .create_plan(one_shot_provider(new_data_stream).unwrap()) + .await + .unwrap(); // Assert the plan structure using portable plan matching // The optimized plan should have: @@ -6577,7 +7238,10 @@ mod tests { let new_data_stream = reader_to_stream(Box::new(new_data)); // This should use the fast path (execute_uncommitted_v2) - let plan = merge_insert_job.create_plan(new_data_stream).await.unwrap(); + let plan = merge_insert_job + .create_plan(one_shot_provider(new_data_stream).unwrap()) + .await + .unwrap(); // The optimized plan should use Inner join instead of Right join since we're not // inserting unmatched rows. The sentinel IS NOT NULL condition is folded away by @@ -6625,7 +7289,10 @@ mod tests { let new_data_reader = new_data.into_reader_rows(RowCount::from(512), BatchCount::from(16)); let new_data_stream = reader_to_stream(Box::new(new_data_reader)); - let plan = merge_insert_job.create_plan(new_data_stream).await.unwrap(); + let plan = merge_insert_job + .create_plan(one_shot_provider(new_data_stream).unwrap()) + .await + .unwrap(); // The optimized plan should use Inner join and include the UpdateIf condition. // The sentinel IS NOT NULL condition is folded away (sentinel is lit(true)). @@ -6677,7 +7344,10 @@ mod tests { // Should reach the v2 fast path (`create_plan` + FullSchemaMergeInsertExec). // Dropping to v1 here would return an error from create_plan instead. - let plan = merge_insert_job.create_plan(new_data_stream).await.unwrap(); + let plan = merge_insert_job + .create_plan(one_shot_provider(new_data_stream).unwrap()) + .await + .unwrap(); // The join is Right because we keep unmatched source rows (InsertAll) // but discard unmatched target rows (DoNothing on when_matched, @@ -7913,7 +8583,12 @@ MergeInsert: on=[id], when_matched=DoNothing, when_not_matched=InsertAll, when_n let plan = merge_insert_job.explain_plan(None, false).await.unwrap(); assert!(plan.contains("HashJoinExec")); assert!(plan.contains("join_type=Full")); - assert!(plan.contains("projection=[_rowid")); + assert!( + plan.lines().any(|line| line.contains("HashJoinExec") + && line.contains("projection=[") + && line.contains("_rowid")), + "join should push down a projection that retains _rowid: {plan}" + ); assert!( plan.contains("LanceRead: uri=") && plan.contains("projection=[id]"), "target-side scan should prune the FSL payload from the join build side: {plan}" @@ -7981,7 +8656,12 @@ MergeInsert: on=[id], when_matched=DoNothing, when_not_matched=InsertAll, when_n let plan = merge_insert_job.explain_plan(None, false).await.unwrap(); assert!(plan.contains("HashJoinExec")); assert!(plan.contains("join_type=Full")); - assert!(plan.contains("projection=[_rowid")); + assert!( + plan.lines().any(|line| line.contains("HashJoinExec") + && line.contains("projection=[") + && line.contains("_rowid")), + "join should push down a projection that retains _rowid: {plan}" + ); assert!( plan.contains("LanceRead: uri=") && plan.contains("projection=[id]"), "target-side scan should prune the FSL payload from the join build side even when a scalar index exists: {plan}" @@ -8578,6 +9258,95 @@ MergeInsert: on=[id], when_matched=DoNothing, when_not_matched=InsertAll, when_n ); } + #[rstest::rstest] + #[case::v2(false)] + #[case::indexed_scan(true)] + #[tokio::test] + async fn test_first_seen_dedupes_unmatched_source_rows(#[case] with_index: bool) { + let initial = + record_batch!(("id", Int32, [Some(1)]), ("value", Int32, [Some(10)])).unwrap(); + let initial = if with_index { + initial + } else { + // Match the reported failure's empty-target setup on the v2 path. + initial.slice(0, 0) + }; + let mut dataset = Dataset::write( + RecordBatchIterator::new([Ok(initial.clone())], initial.schema()), + "memory://", + None, + ) + .await + .unwrap(); + + if with_index { + dataset + .create_index( + &["id"], + IndexType::Scalar, + None, + &ScalarIndexParams::default(), + false, + ) + .await + .unwrap(); + } + + // Split distinct duplicate values across batches to verify that FirstSeen + // preserves source order across the entire stream. On the v2 path, also + // verify that NULL keys remain distinct under SQL equality. + let (first, second, expected_inserted) = if with_index { + ( + record_batch!(("id", Int32, [Some(108)]), ("value", Int32, [Some(1)])).unwrap(), + record_batch!(("id", Int32, [Some(108)]), ("value", Int32, [Some(2)])).unwrap(), + 1, + ) + } else { + ( + record_batch!( + ("id", Int32, [Some(108), None]), + ("value", Int32, [Some(1), Some(3)]) + ) + .unwrap(), + record_batch!( + ("id", Int32, [Some(108), None]), + ("value", Int32, [Some(2), Some(4)]) + ) + .unwrap(), + 3, + ) + }; + + let (dataset, stats) = + MergeInsertBuilder::try_new(Arc::new(dataset), vec!["id".to_string()]) + .unwrap() + .when_matched(WhenMatched::UpdateAll) + .when_not_matched(WhenNotMatched::InsertAll) + .source_dedupe_behavior(SourceDedupeBehavior::FirstSeen) + .try_build() + .unwrap() + .execute_reader(Box::new(RecordBatchIterator::new( + [Ok(first.clone()), Ok(second)], + first.schema(), + ))) + .await + .unwrap(); + + assert_eq!(stats.num_inserted_rows, expected_inserted); + assert_eq!(stats.num_updated_rows, 0); + assert_eq!(stats.num_skipped_duplicates, 1); + + let inserted = dataset + .scan() + .filter("id = 108") + .unwrap() + .try_into_batch() + .await + .unwrap(); + assert_eq!(inserted.num_rows(), 1); + assert_eq!(inserted["value"].as_primitive::().value(0), 1); + } + #[tokio::test] async fn test_merge_insert_use_index() { let data = lance_datagen::gen_batch() @@ -9602,7 +10371,10 @@ MergeInsert: on=[id], when_matched=DoNothing, when_not_matched=InsertAll, when_n [Ok(new_batch.clone())], schema.clone(), ))); - let plan = plan_job.create_plan(plan_stream).await.unwrap(); + let plan = plan_job + .create_plan(one_shot_provider(plan_stream).unwrap()) + .await + .unwrap(); assert_plan_node_equals( plan, "DeleteOnlyMergeInsert: on=[key], when_matched=Delete, when_not_matched=DoNothing @@ -9684,7 +10456,10 @@ MergeInsert: on=[id], when_matched=DoNothing, when_not_matched=InsertAll, when_n [Ok(new_batch.clone())], id_only_schema.clone(), ))); - let plan = plan_job.create_plan(plan_stream).await.unwrap(); + let plan = plan_job + .create_plan(one_shot_provider(plan_stream).unwrap()) + .await + .unwrap(); assert_plan_node_equals( plan, "DeleteOnlyMergeInsert: on=[key], when_matched=Delete, when_not_matched=DoNothing @@ -9766,7 +10541,10 @@ MergeInsert: on=[id], when_matched=DoNothing, when_not_matched=InsertAll, when_n [Ok(new_batch.clone())], schema.clone(), ))); - let plan = plan_job.create_plan(plan_stream).await.unwrap(); + let plan = plan_job + .create_plan(one_shot_provider(plan_stream).unwrap()) + .await + .unwrap(); assert_plan_node_equals( plan, "MergeInsert: on=[key], when_matched=Delete, when_not_matched=InsertAll, when_not_matched_by_source=Keep...THEN 2 WHEN...THEN 3 ELSE 0 END as __action]...projection=[key, value, filterme]" @@ -9871,7 +10649,10 @@ MergeInsert: on=[id], when_matched=DoNothing, when_not_matched=InsertAll, when_n [Ok(non_matching_batch.clone())], schema.clone(), ))); - let plan = plan_job.create_plan(plan_stream).await.unwrap(); + let plan = plan_job + .create_plan(one_shot_provider(plan_stream).unwrap()) + .await + .unwrap(); assert_plan_node_equals( plan, "DeleteOnlyMergeInsert: on=[key], when_matched=Delete, when_not_matched=DoNothing @@ -9991,7 +10772,10 @@ MergeInsert: on=[id], when_matched=DoNothing, when_not_matched=InsertAll, when_n [Ok(new_batch)], schema.clone(), ))); - let plan = job.create_plan(plan_stream).await.unwrap(); + let plan = job + .create_plan(one_shot_provider(plan_stream).unwrap()) + .await + .unwrap(); let plan_str = datafusion::physical_plan::displayable(plan.as_ref()) .indent(true) @@ -10082,7 +10866,7 @@ MergeInsert: on=[id], when_matched=DoNothing, when_not_matched=InsertAll, when_n impl std::error::Error for MyTestError {} #[tokio::test] - async fn test_merge_insert_execute_reader_preserves_external_error() { + async fn test_merge_insert_execute_reader_preserves_error_message() { let schema = Arc::new(ArrowSchema::new(vec![ ArrowField::new("key", DataType::Int32, false), ArrowField::new("value", DataType::Int32, false), @@ -10119,14 +10903,14 @@ MergeInsert: on=[id], when_matched=DoNothing, when_not_matched=InsertAll, when_n .execute_reader(Box::new(reader) as Box) .await; - match result { - Err(Error::External { source }) => { - let original = source.downcast_ref::().unwrap(); - assert_eq!(original.code, error_code); - } - Err(other) => panic!("Expected External, got: {:?}", other), - Ok(_) => panic!("Expected error"), - } + // The source error is routed through the merge plan, which shares it + // across join partitions, so its concrete type is not recoverable. The + // message must still reach the caller. + let err = result.expect_err("expected the source error to surface"); + assert!( + err.to_string().contains("merge insert failure"), + "source error message should be preserved; got: {err}" + ); } } @@ -11311,4 +12095,580 @@ MergeInsert: on=[id], when_matched=DoNothing, when_not_matched=InsertAll, when_n take_meta_field ); } + + #[tokio::test] + async fn test_merge_insert_subschema_with_json_columns() { + use lance_arrow::ARROW_EXT_NAME_KEY; + use lance_arrow::json::ARROW_JSON_EXT_NAME; + + // Create a dataset with an Arrow JSON extension column + let test_dir = TempStrDir::default(); + let mut json_metadata = HashMap::new(); + json_metadata.insert( + ARROW_EXT_NAME_KEY.to_string(), + ARROW_JSON_EXT_NAME.to_string(), + ); + let schema = Arc::new(Schema::new(vec![ + Field::new("id", DataType::Int64, false), + Field::new("name", DataType::Utf8, true), + Field::new("score", DataType::Int64, true), + Field::new("meta", DataType::Utf8, true).with_metadata(json_metadata.clone()), + ])); + let batch = RecordBatch::try_new( + schema.clone(), + vec![ + Arc::new(Int64Array::from(vec![1, 2, 3, 4, 5])), + Arc::new(StringArray::from(vec!["a", "b", "c", "d", "e"])), + Arc::new(Int64Array::from(vec![10, 20, 30, 40, 50])), + Arc::new(StringArray::from(vec![ + r#"{"x":1}"#, + r#"{"x":2}"#, + r#"{"x":3}"#, + r#"{"x":4}"#, + r#"{"x":5}"#, + ])), + ], + ) + .unwrap(); + let reader = RecordBatchIterator::new(vec![Ok(batch)], schema); + let dataset = Arc::new( + Dataset::write(reader, test_dir.as_ref(), None) + .await + .unwrap(), + ); + + // Perform a subschema merge_insert: only update "meta" column (JSON type) + // This exercises the update_fragments path with interleave_batches + let update_schema = Arc::new(Schema::new(vec![ + Field::new("id", DataType::Int64, false), + Field::new("meta", DataType::Utf8, true).with_metadata(json_metadata), + ])); + let update_batch = RecordBatch::try_new( + update_schema.clone(), + vec![ + Arc::new(Int64Array::from(vec![2, 4])), + Arc::new(StringArray::from(vec![ + r#"{"updated":true,"id":2}"#, + r#"{"updated":true,"id":4}"#, + ])), + ], + ) + .unwrap(); + let update_reader: Box = Box::new(RecordBatchIterator::new( + vec![Ok(update_batch)], + update_schema, + )); + let stream = reader_to_stream(update_reader); + + // Execute merge_insert with subschema (only id + meta columns) + let mut builder = + MergeInsertBuilder::try_new(dataset.clone(), vec!["id".to_string()]).unwrap(); + builder.when_matched(WhenMatched::UpdateAll); + builder.when_not_matched(WhenNotMatched::DoNothing); + let job = builder.try_build().unwrap(); + let (updated_dataset, stats) = job.execute(stream).await.unwrap(); + + // Verify: the merge should not fail with type mismatch + assert_eq!(stats.num_updated_rows, 2); + + // Read back and verify the JSON column was updated correctly + let batches = updated_dataset + .scan() + .try_into_stream() + .await + .unwrap() + .try_collect::>() + .await + .unwrap(); + let result = concat_batches(&batches[0].schema(), &batches).unwrap(); + assert_eq!(result.num_rows(), 5); + + // Verify the "score" column (not in update) is preserved, and "meta" updated + let ids = result + .column_by_name("id") + .unwrap() + .as_any() + .downcast_ref::() + .unwrap(); + let scores = result + .column_by_name("score") + .unwrap() + .as_any() + .downcast_ref::() + .unwrap(); + let metas = result + .column_by_name("meta") + .unwrap() + .as_any() + .downcast_ref::() + .unwrap(); + for i in 0..5 { + let id = ids.value(i); + let score = scores.value(i); + let meta = metas.value(i); + // score = id * 10, regardless of row order + assert_eq!(score, id * 10, "id={} score mismatch", id); + if id == 2 || id == 4 { + assert!( + meta.contains("updated"), + "id={} should have updated meta, got: {}", + id, + meta + ); + } else { + assert!( + meta.contains("\"x\""), + "id={} should have original meta, got: {}", + id, + meta + ); + } + } + } + + fn id_value_schema() -> Arc { + Arc::new(Schema::new(vec![ + Field::new("id", DataType::UInt32, false), + Field::new("value", DataType::UInt32, false), + ])) + } + + /// `execute_provider` is the canonical entry point; a `MemTable` source merges + /// the same way a stream does. + #[tokio::test] + async fn test_merge_insert_execute_provider() { + let initial = + record_batch!(("id", UInt32, [0, 1, 2]), ("value", UInt32, [0, 0, 0])).unwrap(); + let dataset = Arc::new( + InsertBuilder::new("memory://") + .execute(vec![initial]) + .await + .unwrap(), + ); + + // Update id=1, insert id=3. + let new_data = record_batch!(("id", UInt32, [1, 3]), ("value", UInt32, [10, 30])).unwrap(); + let provider: Arc = Arc::new( + datafusion::datasource::MemTable::try_new(new_data.schema(), vec![vec![new_data]]) + .unwrap(), + ); + + let (merged, stats) = MergeInsertBuilder::try_new(dataset.clone(), vec!["id".to_string()]) + .unwrap() + .when_matched(WhenMatched::UpdateAll) + .when_not_matched(WhenNotMatched::InsertAll) + .try_build() + .unwrap() + .execute_provider(provider) + .await + .unwrap(); + + assert_eq!(stats.num_updated_rows, 1); + assert_eq!(stats.num_inserted_rows, 1); + + let batch = merged.scan().try_into_batch().await.unwrap(); + let ids = batch["id"].as_primitive::(); + let values = batch["value"].as_primitive::(); + let merged_rows: HashMap = ids + .values() + .iter() + .zip(values.values().iter()) + .map(|(id, value)| (*id, *value)) + .collect(); + assert_eq!( + merged_rows, + HashMap::from([(0, 0), (1, 10), (2, 0), (3, 30)]) + ); + } + + /// `execute_batches` merges materialized batches; multiple batches are spread + /// across partitions and merged correctly. + #[tokio::test] + async fn test_merge_insert_execute_batches() { + let initial = + record_batch!(("id", UInt32, [0, 1, 2]), ("value", UInt32, [0, 0, 0])).unwrap(); + let dataset = Arc::new( + InsertBuilder::new("memory://") + .execute(vec![initial]) + .await + .unwrap(), + ); + + // Two batches: update id=1 (batch 0), insert id=3 (batch 1). + let batch0 = record_batch!(("id", UInt32, [1]), ("value", UInt32, [10])).unwrap(); + let batch1 = record_batch!(("id", UInt32, [3]), ("value", UInt32, [30])).unwrap(); + + let (merged, stats) = MergeInsertBuilder::try_new(dataset.clone(), vec!["id".to_string()]) + .unwrap() + .when_matched(WhenMatched::UpdateAll) + .when_not_matched(WhenNotMatched::InsertAll) + .try_build() + .unwrap() + .execute_batches(vec![batch0, batch1]) + .await + .unwrap(); + + assert_eq!(stats.num_updated_rows, 1); + assert_eq!(stats.num_inserted_rows, 1); + + let batch = merged.scan().try_into_batch().await.unwrap(); + let ids = batch["id"].as_primitive::(); + let values = batch["value"].as_primitive::(); + let merged_rows: HashMap = ids + .values() + .iter() + .zip(values.values().iter()) + .map(|(id, value)| (*id, *value)) + .collect(); + assert_eq!( + merged_rows, + HashMap::from([(0, 0), (1, 10), (2, 0), (3, 30)]) + ); + } + + /// An empty batch list still produces a valid (single, empty) partition, so the + /// merge is a no-op and the target is unchanged. + #[tokio::test] + async fn test_merge_insert_execute_batches_empty() { + let initial = + record_batch!(("id", UInt32, [0, 1, 2]), ("value", UInt32, [0, 0, 0])).unwrap(); + let dataset = Arc::new( + InsertBuilder::new("memory://") + .execute(vec![initial]) + .await + .unwrap(), + ); + + let (merged, stats) = MergeInsertBuilder::try_new(dataset.clone(), vec!["id".to_string()]) + .unwrap() + .when_matched(WhenMatched::UpdateAll) + .when_not_matched(WhenNotMatched::InsertAll) + .try_build() + .unwrap() + .execute_batches(vec![]) + .await + .unwrap(); + + assert_eq!(stats.num_updated_rows, 0); + assert_eq!(stats.num_inserted_rows, 0); + + let batch = merged.scan().try_into_batch().await.unwrap(); + let ids = batch["id"].as_primitive::(); + let values = batch["value"].as_primitive::(); + let merged_rows: HashMap = ids + .values() + .iter() + .zip(values.values().iter()) + .map(|(id, value)| (*id, *value)) + .collect(); + assert_eq!(merged_rows, HashMap::from([(0, 0), (1, 0), (2, 0)])); + } + + fn collect_exact_row_counts(plan: &Arc, out: &mut Vec) { + if let Ok(stats) = plan.partition_statistics(None) + && let datafusion::common::stats::Precision::Exact(n) = stats.num_rows + { + out.push(n); + } + for child in plan.children() { + collect_exact_row_counts(child, out); + } + } + + /// Use case 3: planning against the provider exposes its exact source + /// statistics to the optimizer. + #[tokio::test] + async fn test_merge_insert_source_statistics_in_plan() { + let schema = id_value_schema(); + let target_rows = 1000u32; + let initial = RecordBatch::try_new( + schema.clone(), + vec![ + Arc::new(UInt32Array::from_iter_values(0..target_rows)), + Arc::new(UInt32Array::from_iter_values(std::iter::repeat_n( + 0, + target_rows as usize, + ))), + ], + ) + .unwrap(); + let dataset = Arc::new( + InsertBuilder::new("memory://") + .execute(vec![initial]) + .await + .unwrap(), + ); + + // A small source whose exact row count is distinct from the target's. + let source_rows = 10usize; + let new_data = RecordBatch::try_new( + schema.clone(), + vec![ + Arc::new(UInt32Array::from_iter_values(0..source_rows as u32)), + Arc::new(UInt32Array::from_iter_values(std::iter::repeat_n( + 1, + source_rows, + ))), + ], + ) + .unwrap(); + let provider: Arc = + Arc::new(MemTable::try_new(schema.clone(), vec![vec![new_data]]).unwrap()); + + let job = MergeInsertBuilder::try_new(dataset.clone(), vec!["id".to_string()]) + .unwrap() + .when_matched(WhenMatched::UpdateAll) + .try_build() + .unwrap(); + + // The provider's exact row count reaches the plan's statistics. + let plan = job.create_plan(provider).await.unwrap(); + let mut row_counts = Vec::new(); + collect_exact_row_counts(&plan, &mut row_counts); + assert!( + row_counts.contains(&source_rows), + "source provider's exact row count ({source_rows}) should reach the plan; got {row_counts:?}" + ); + } + + /// With a one-shot stream source and `spill_for_retry(false)`, a commit + /// conflict fails fast instead of replaying the stream. The non-replayable + /// one-shot provider must be scanned exactly once (scanning it twice would + /// panic), proving retries are disabled even though `conflict_retries > 0`. + #[tokio::test] + async fn test_merge_insert_spill_for_retry_false_fails_fast() { + let schema = Arc::new(Schema::new(vec![ + Field::new("id", DataType::UInt32, false).with_metadata( + vec![( + "lance-schema:unenforced-primary-key".to_string(), + "true".to_string(), + )] + .into_iter() + .collect(), + ), + Field::new("value", DataType::UInt32, false), + ])); + let initial = RecordBatch::try_new( + schema.clone(), + vec![ + Arc::new(UInt32Array::from(vec![0, 1, 2, 3])), + Arc::new(UInt32Array::from(vec![0, 0, 0, 0])), + ], + ) + .unwrap(); + let dataset = Arc::new( + InsertBuilder::new("memory://") + .execute(vec![initial]) + .await + .unwrap(), + ); + + // Merge insert job based on version 1, with retries enabled but spilling off. + let new_data = RecordBatch::try_new( + schema.clone(), + vec![ + Arc::new(UInt32Array::from(vec![100])), + Arc::new(UInt32Array::from(vec![1])), + ], + ) + .unwrap(); + let job = MergeInsertBuilder::try_new(dataset.clone(), vec!["id".to_string()]) + .unwrap() + .when_matched(WhenMatched::UpdateAll) + .when_not_matched(WhenNotMatched::InsertAll) + .conflict_retries(10) + .spill_for_retry(false) + .try_build() + .unwrap(); + + // An append commits first (version 2), so the merge built on version 1 hits + // an unresolvable conflict on commit. + let append_batch = RecordBatch::try_new( + schema.clone(), + vec![ + Arc::new(UInt32Array::from(vec![50])), + Arc::new(UInt32Array::from(vec![2])), + ], + ) + .unwrap(); + InsertBuilder::new(dataset.clone()) + .with_params(&WriteParams { + mode: WriteMode::Append, + ..Default::default() + }) + .execute(vec![append_batch]) + .await + .unwrap(); + + let source = RecordBatchStreamAdapter::new( + schema.clone(), + futures::stream::iter(vec![Ok(new_data)]), + ); + let merge_result = job + .execute(Box::pin(source) as SendableRecordBatchStream) + .await; + + assert!( + matches!( + merge_result, + Err(crate::Error::TooMuchWriteContention { .. }) + ), + "Expected fail-fast TooMuchWriteContention, got: {:?}", + merge_result + ); + } + + #[tokio::test] + async fn test_merge_insert_with_blob_v1_source_provides_blob() { + use arrow_array::LargeBinaryArray; + use arrow_schema::Schema as ArrowSchema; + use lance_arrow::BLOB_META_KEY; + + let test_dir = TempStrDir::default(); + let schema = Arc::new(ArrowSchema::new(vec![ + Field::new("blobs", DataType::LargeBinary, true).with_metadata(HashMap::from([( + BLOB_META_KEY.to_string(), + "true".to_string(), + )])), + Field::new("id", DataType::Int64, true), + Field::new("other", DataType::Int64, true), + ])); + let make_batch = |blob_values: Vec>, ids, others| { + RecordBatch::try_new( + schema.clone(), + vec![ + Arc::new(LargeBinaryArray::from(blob_values)), + Arc::new(Int64Array::from(ids)), + Arc::new(Int64Array::from(others)), + ], + ) + .unwrap() + }; + let dataset = Arc::new( + Dataset::write( + RecordBatchIterator::new( + vec![Ok(make_batch( + vec![Some(b"foo"), Some(b"bar")], + vec![0, 1], + vec![10, 20], + ))], + schema.clone(), + ), + &test_dir, + Some(WriteParams { + data_storage_version: Some(LanceFileVersion::V2_1), + ..Default::default() + }), + ) + .await + .unwrap(), + ); + let source = Box::new(RecordBatchIterator::new( + vec![Ok(make_batch( + vec![Some(b"baz"), Some(b"qux")], + vec![1, 2], + vec![200, 300], + ))], + schema, + )); + + let job = MergeInsertBuilder::try_new(dataset, vec!["id".to_string()]) + .unwrap() + .when_matched(WhenMatched::UpdateAll) + .when_not_matched(WhenNotMatched::InsertAll) + .try_build() + .unwrap(); + let (new_dataset, _) = job.execute_reader(source).await.unwrap(); + let blobs = new_dataset + .take_blobs_by_indices(&[0, 1, 2], "blobs") + .await + .unwrap(); + assert_eq!( + blobs[0].as_ref().unwrap().read().await.unwrap().as_ref(), + b"foo" + ); + assert_eq!( + blobs[1].as_ref().unwrap().read().await.unwrap().as_ref(), + b"baz" + ); + assert_eq!( + blobs[2].as_ref().unwrap().read().await.unwrap().as_ref(), + b"qux" + ); + } + + #[tokio::test] + async fn test_merge_insert_with_blob_v2_source_provides_blob() { + use crate::{BlobArrayBuilder, blob_field}; + use arrow_schema::Schema as ArrowSchema; + + let test_dir = TempStrDir::default(); + let schema = Arc::new(ArrowSchema::new(vec![ + blob_field("blobs", true), + Field::new("id", DataType::Int64, true), + Field::new("other", DataType::Int64, true), + ])); + let make_batch = |blob_values: &[&[u8]], ids, others| { + let mut blobs = BlobArrayBuilder::new(blob_values.len()); + for value in blob_values { + blobs.push_bytes(value).unwrap(); + } + RecordBatch::try_new( + schema.clone(), + vec![ + blobs.finish().unwrap(), + Arc::new(Int64Array::from(ids)), + Arc::new(Int64Array::from(others)), + ], + ) + .unwrap() + }; + let dataset = Arc::new( + Dataset::write( + RecordBatchIterator::new( + vec![Ok(make_batch(&[b"foo", b"bar"], vec![0, 1], vec![10, 20]))], + schema.clone(), + ), + &test_dir, + Some(WriteParams { + data_storage_version: Some(LanceFileVersion::V2_2), + ..Default::default() + }), + ) + .await + .unwrap(), + ); + let source = Box::new(RecordBatchIterator::new( + vec![Ok(make_batch( + &[b"baz", b"qux"], + vec![1, 2], + vec![200, 300], + ))], + schema, + )); + + let job = MergeInsertBuilder::try_new(dataset, vec!["id".to_string()]) + .unwrap() + .when_matched(WhenMatched::UpdateAll) + .when_not_matched(WhenNotMatched::InsertAll) + .try_build() + .unwrap(); + let (new_dataset, _) = job.execute_reader(source).await.unwrap(); + let blobs = new_dataset + .take_blobs_by_indices(&[0, 1, 2], "blobs") + .await + .unwrap(); + assert_eq!( + blobs[0].as_ref().unwrap().read().await.unwrap().as_ref(), + b"foo" + ); + assert_eq!( + blobs[1].as_ref().unwrap().read().await.unwrap().as_ref(), + b"baz" + ); + assert_eq!( + blobs[2].as_ref().unwrap().read().await.unwrap().as_ref(), + b"qux" + ); + } } diff --git a/rust/lance/src/dataset/write/merge_insert/exec/delete.rs b/rust/lance/src/dataset/write/merge_insert/exec/delete.rs index 07fad758902..b5df78d851e 100644 --- a/rust/lance/src/dataset/write/merge_insert/exec/delete.rs +++ b/rust/lance/src/dataset/write/merge_insert/exec/delete.rs @@ -1,7 +1,10 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright The Lance Authors -use std::sync::{Arc, Mutex}; +use std::sync::{ + Arc, Mutex, + atomic::{AtomicU64, Ordering}, +}; use arrow_array::{Array, RecordBatch, UInt8Array, UInt64Array}; use datafusion::common::Result as DFResult; @@ -49,6 +52,7 @@ pub struct DeleteOnlyMergeInsertExec { merge_stats: Arc>>, transaction: Arc>>, affected_rows: Arc>>, + source_skipped_duplicates: Arc, } impl DeleteOnlyMergeInsertExec { @@ -56,6 +60,7 @@ impl DeleteOnlyMergeInsertExec { input: Arc, dataset: Arc, params: MergeInsertParams, + source_skipped_duplicates: Arc, ) -> DFResult { let empty_schema = Arc::new(arrow_schema::Schema::empty()); let properties = Arc::new(PlanProperties::new( @@ -74,6 +79,7 @@ impl DeleteOnlyMergeInsertExec { merge_stats: Arc::new(Mutex::new(None)), transaction: Arc::new(Mutex::new(None)), affected_rows: Arc::new(Mutex::new(None)), + source_skipped_duplicates, }) } @@ -216,10 +222,6 @@ impl ExecutionPlan for DeleteOnlyMergeInsertExec { "DeleteOnlyMergeInsertExec" } - fn as_any(&self) -> &dyn std::any::Any { - self - } - fn schema(&self) -> arrow_schema::SchemaRef { Arc::new(arrow_schema::Schema::empty()) } @@ -246,6 +248,7 @@ impl ExecutionPlan for DeleteOnlyMergeInsertExec { merge_stats: self.merge_stats.clone(), transaction: self.transaction.clone(), affected_rows: self.affected_rows.clone(), + source_skipped_duplicates: self.source_skipped_duplicates.clone(), })) } @@ -283,9 +286,10 @@ impl ExecutionPlan for DeleteOnlyMergeInsertExec { let merge_stats_holder = self.merge_stats.clone(); let transaction_holder = self.transaction.clone(); let affected_rows_holder = self.affected_rows.clone(); - let merged_generations = self.params.merged_generations.clone(); + let compacted_sstables = self.params.compacted_sstables.clone(); let source_dedupe_behavior = self.params.source_dedupe_behavior; let on_columns = self.params.on.clone(); + let source_skipped_duplicates = self.source_skipped_duplicates.clone(); let result_stream = futures::stream::once(async move { // Delete-only merges write no data files, but still validate any @@ -308,7 +312,7 @@ impl ExecutionPlan for DeleteOnlyMergeInsertExec { updated_fragments, new_fragments: vec![], fields_modified: vec![], - merged_generations, + compacted_sstables, fields_for_preserving_frag_bitmap: dataset .schema() .fields @@ -323,6 +327,13 @@ impl ExecutionPlan for DeleteOnlyMergeInsertExec { let transaction = Transaction::new(dataset.manifest.version, operation, None); let num_deleted = delete_row_addrs.len(); + let num_skipped_duplicates = (skipped_duplicates.value() as u64) + .checked_add(source_skipped_duplicates.load(Ordering::Relaxed)) + .ok_or_else(|| { + datafusion::error::DataFusionError::Execution( + "merge insert skipped duplicate count overflowed u64".to_string(), + ) + })?; let stats = MergeStats { num_deleted_rows: num_deleted, num_inserted_rows: 0, @@ -330,7 +341,7 @@ impl ExecutionPlan for DeleteOnlyMergeInsertExec { bytes_written: 0, num_files_written: 0, num_attempts: 1, - num_skipped_duplicates: skipped_duplicates.value() as u64, + num_skipped_duplicates, }; if let Ok(mut transaction_guard) = transaction_holder.lock() { diff --git a/rust/lance/src/dataset/write/merge_insert/exec/write.rs b/rust/lance/src/dataset/write/merge_insert/exec/write.rs index d5b51b3d97f..38560bf0e1e 100644 --- a/rust/lance/src/dataset/write/merge_insert/exec/write.rs +++ b/rust/lance/src/dataset/write/merge_insert/exec/write.rs @@ -2,7 +2,10 @@ // SPDX-FileCopyrightText: Copyright The Lance Authors use std::collections::HashSet; -use std::sync::{Arc, Mutex}; +use std::sync::{ + Arc, Mutex, + atomic::{AtomicU64, Ordering}, +}; use arrow_array::{Array, RecordBatch, UInt8Array, UInt64Array}; use arrow_schema::Schema; @@ -29,7 +32,7 @@ use crate::dataset::write::merge_insert::inserted_rows::{ KeyExistenceFilter, KeyExistenceFilterBuilder, extract_key_value_from_batch, }; use crate::dataset::write::merge_insert::{ - MERGE_SOURCE_SENTINEL, SourceDedupeBehavior, create_duplicate_row_error, + InsertedKeyTracker, MERGE_SOURCE_SENTINEL, SourceDedupeBehavior, create_duplicate_row_error, format_key_values_on_columns, resolve_target_bases, }; use crate::{ @@ -63,6 +66,8 @@ struct MergeState { stable_row_ids: bool, /// Set to track processed row IDs to detect duplicates processed_row_ids: HashSet, + /// Set to track non-null keys of rows inserted by FirstSeen mode + processed_insert_keys: InsertedKeyTracker, /// The "on" column names for merge operation on_columns: Vec, /// How to handle duplicate source rows @@ -84,6 +89,7 @@ impl MergeState { metrics, stable_row_ids, processed_row_ids: HashSet::new(), + processed_insert_keys: InsertedKeyTracker::default(), on_columns, source_dedupe_behavior, } @@ -166,7 +172,15 @@ impl MergeState { Ok(Some(row_idx)) // Keep this row for writing } Action::Insert => { - // Insert action - just insert new data + if self.source_dedupe_behavior == SourceDedupeBehavior::FirstSeen + && !self + .processed_insert_keys + .insert(batch, row_idx, &self.on_columns)? + { + self.metrics.num_skipped_duplicates.add(1); + return Ok(None); + } + // Capture the key value for conflict detection (only for inserts, not updates) if let Some(key_value) = extract_key_value_from_batch(batch, row_idx, &self.on_columns) @@ -211,6 +225,7 @@ pub struct FullSchemaMergeInsertExec { transaction: Arc>>, affected_rows: Arc>>, inserted_rows_filter: Arc>>, + source_skipped_duplicates: Arc, /// Whether the ON columns match the schema's unenforced primary key. /// If true, inserted_rows_filter will be included in the transaction for conflict detection. is_primary_key: bool, @@ -221,6 +236,7 @@ impl FullSchemaMergeInsertExec { input: Arc, dataset: Arc, params: MergeInsertParams, + source_skipped_duplicates: Arc, ) -> DFResult { let empty_schema = Arc::new(arrow_schema::Schema::empty()); let properties = Arc::new(PlanProperties::new( @@ -254,6 +270,7 @@ impl FullSchemaMergeInsertExec { transaction: Arc::new(Mutex::new(None)), affected_rows: Arc::new(Mutex::new(None)), inserted_rows_filter: Arc::new(Mutex::new(None)), + source_skipped_duplicates, is_primary_key, }) } @@ -797,10 +814,6 @@ impl ExecutionPlan for FullSchemaMergeInsertExec { "FullSchemaMergeInsertExec" } - fn as_any(&self) -> &dyn std::any::Any { - self - } - fn schema(&self) -> arrow_schema::SchemaRef { Arc::new(arrow_schema::Schema::empty()) } @@ -828,6 +841,7 @@ impl ExecutionPlan for FullSchemaMergeInsertExec { transaction: self.transaction.clone(), affected_rows: self.affected_rows.clone(), inserted_rows_filter: self.inserted_rows_filter.clone(), + source_skipped_duplicates: self.source_skipped_duplicates.clone(), is_primary_key: self.is_primary_key, })) } @@ -868,6 +882,39 @@ impl ExecutionPlan for FullSchemaMergeInsertExec { // Execute the input plan to get the merge data stream let input_stream = self.input.execute(partition, context)?; + let has_blob_v2_columns = self + .dataset + .schema() + .fields_pre_order() + .any(|field| field.is_blob_v2()); + let input_stream = if has_blob_v2_columns { + let input_schema = input_stream.schema(); + let rewrite_plan = Arc::new( + crate::dataset::optimize::BlobV2BatchRewritePlan::try_new( + self.dataset.schema(), + input_schema.as_ref(), + true, + ) + .map_err(|error| DataFusionError::External(Box::new(error)))?, + ); + let output_schema = rewrite_plan.output_schema().clone(); + let dataset = self.dataset.clone(); + let transformed = input_stream.then(move |batch_result| { + let dataset = dataset.clone(); + let rewrite_plan = rewrite_plan.clone(); + async move { + let batch = batch_result?; + rewrite_plan + .transform_batch(&dataset, batch) + .await + .map_err(|error| DataFusionError::External(Box::new(error))) + } + }); + Box::pin(RecordBatchStreamAdapter::new(output_schema, transformed)) + as SendableRecordBatchStream + } else { + input_stream + }; // Step 1: Create shared state and streaming processor for row addresses and write data // Get field IDs for the ON columns from the dataset schema @@ -894,7 +941,8 @@ impl ExecutionPlan for FullSchemaMergeInsertExec { let transaction_holder = self.transaction.clone(); let affected_rows_holder = self.affected_rows.clone(); let inserted_rows_filter_holder = self.inserted_rows_filter.clone(); - let merged_generations = self.params.merged_generations.clone(); + let compacted_sstables = self.params.compacted_sstables.clone(); + let source_skipped_duplicates = self.source_skipped_duplicates.clone(); let is_primary_key = self.is_primary_key; let updating_row_ids = { let state = merge_state.lock().unwrap(); @@ -907,6 +955,7 @@ impl ExecutionPlan for FullSchemaMergeInsertExec { // Keep a copy so failures after the write can clean up routed files. let cleanup_bases = target_bases_info.clone(); let (mut new_fragments, _) = write_fragments_internal( + dataset.manifest.data_storage_format.lance_file_format(), Some(&dataset), dataset.object_store.clone(), &dataset.base, @@ -937,7 +986,7 @@ impl ExecutionPlan for FullSchemaMergeInsertExec { for (fragment, sequence) in new_fragments.iter_mut().zip(sequences) { let serialized = lance_table::rowids::write_row_ids(&sequence); - fragment.row_id_meta = Some(RowIdMeta::Inline(serialized)); + fragment.row_id_meta = Some(RowIdMeta::Inline(serialized.into())); } } Ok(()) @@ -994,7 +1043,7 @@ impl ExecutionPlan for FullSchemaMergeInsertExec { updated_fragments, new_fragments, fields_modified: vec![], // No fields are modified in schema for upsert - merged_generations, + compacted_sstables, // Use the full pre-order field list (not just top-level `fields`) so // that nested leaf field ids are included. A merge_insert rewrites whole // rows, so every field is potentially modified; omitting nested ids would @@ -1023,7 +1072,15 @@ impl ExecutionPlan for FullSchemaMergeInsertExec { .add(total_files_written); // Get the final stats from the shared state - let stats = MergeStats::from(&merge_state.metrics); + let mut stats = MergeStats::from(&merge_state.metrics); + stats.num_skipped_duplicates = stats + .num_skipped_duplicates + .checked_add(source_skipped_duplicates.load(Ordering::Relaxed)) + .ok_or_else(|| { + DataFusionError::Execution( + "merge insert skipped duplicate count overflowed u64".to_string(), + ) + })?; if let Ok(mut transaction_guard) = transaction_holder.lock() { transaction_guard.replace(transaction); diff --git a/rust/lance/src/dataset/write/merge_insert/logical_plan.rs b/rust/lance/src/dataset/write/merge_insert/logical_plan.rs index 7f67972ff7e..1588552ebd2 100644 --- a/rust/lance/src/dataset/write/merge_insert/logical_plan.rs +++ b/rust/lance/src/dataset/write/merge_insert/logical_plan.rs @@ -11,7 +11,10 @@ use datafusion::{ }; use datafusion_expr::{LogicalPlan, UserDefinedLogicalNode, UserDefinedLogicalNodeCore}; use lance_core::{ROW_ADDR, ROW_ID}; -use std::{cmp::Ordering, sync::Arc}; +use std::{ + cmp::Ordering, + sync::{Arc, atomic::AtomicU64}, +}; use crate::Dataset; use crate::dataset::write::merge_insert::exec::{ @@ -36,6 +39,7 @@ pub struct MergeInsertWriteNode { input: LogicalPlan, pub(crate) dataset: Arc, pub(crate) params: MergeInsertParams, + pub(crate) source_skipped_duplicates: Arc, schema: Arc, } @@ -67,13 +71,19 @@ impl PartialOrd for MergeInsertWriteNode { } impl MergeInsertWriteNode { - pub fn new(input: LogicalPlan, dataset: Arc, params: MergeInsertParams) -> Self { + pub fn new( + input: LogicalPlan, + dataset: Arc, + params: MergeInsertParams, + source_skipped_duplicates: Arc, + ) -> Self { let empty_schema = Arc::new(arrow_schema::Schema::empty()); let schema = Arc::new(DFSchema::try_from(empty_schema).unwrap()); Self { input, dataset, params, + source_skipped_duplicates, schema, } } @@ -143,6 +153,7 @@ impl UserDefinedLogicalNodeCore for MergeInsertWriteNode { inputs[0].clone(), self.dataset.clone(), self.params.clone(), + self.source_skipped_duplicates.clone(), )) } @@ -248,12 +259,14 @@ impl ExtensionPlanner for MergeInsertPlanner { physical_inputs[0].clone(), write_node.dataset.clone(), write_node.params.clone(), + write_node.source_skipped_duplicates.clone(), )?) } else { Arc::new(FullSchemaMergeInsertExec::try_new( physical_inputs[0].clone(), write_node.dataset.clone(), write_node.params.clone(), + write_node.source_skipped_duplicates.clone(), )?) }; Some(exec) diff --git a/rust/lance/src/dataset/write/retry.rs b/rust/lance/src/dataset/write/retry.rs index 1a72c95cec4..ca5bf09b51a 100644 --- a/rust/lance/src/dataset/write/retry.rs +++ b/rust/lance/src/dataset/write/retry.rs @@ -1,17 +1,14 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright The Lance Authors -use std::future::Future; use std::sync::Arc; use std::time::{Duration, Instant}; -use either::Either; -use futures::TryFutureExt; -use futures::future::FutureExt; use lance_core::utils::backoff::SlotBackoff; use lance_core::{Error, Result}; use crate::Dataset; +use crate::io::commit::{maybe_timeout, timeout_error}; /// Configuration for retry behavior #[derive(Debug, Clone)] @@ -44,33 +41,6 @@ pub trait RetryExecutor: Clone { fn update_dataset(&mut self, dataset: Arc); } -fn timeout_error(retry_timeout: Duration, attempts: u32) -> Error { - Error::too_much_write_contention(format!( - "Attempted {} times, but failed on retry_timeout of {:.3} seconds.", - attempts, - retry_timeout.as_secs_f32() - )) -} - -fn maybe_timeout( - backoff: &SlotBackoff, - start: Instant, - retry_timeout: Duration, - future: impl Future, -) -> impl Future> { - let attempt = backoff.attempt(); - if attempt == 0 { - // No timeout on first attempt - Either::Left(future.map(|res| Ok(res))) - } else { - let remaining = retry_timeout.saturating_sub(start.elapsed()); - Either::Right( - tokio::time::timeout(remaining, future) - .map_err(move |_| timeout_error(retry_timeout, attempt + 1)), - ) - } -} - /// Execute an operation with retry logic for commit conflicts pub async fn execute_with_retry( executor: E, @@ -86,11 +56,17 @@ pub async fn execute_with_retry( executor_clone.update_dataset(dataset_ref.clone()); let execute_fut = executor_clone.execute_impl(); - let execute_fut = maybe_timeout(&backoff, start, config.retry_timeout, execute_fut); + let execute_fut = + maybe_timeout(backoff.attempt(), start, config.retry_timeout, execute_fut); let data = execute_fut.await??; let commit_future = executor.commit(dataset_ref.clone(), data); - let commit_future = maybe_timeout(&backoff, start, config.retry_timeout, commit_future); + let commit_future = maybe_timeout( + backoff.attempt(), + start, + config.retry_timeout, + commit_future, + ); match commit_future.await? { Ok(result) => return Ok(result), @@ -111,7 +87,8 @@ pub async fn execute_with_retry( } let sleep_fut = tokio::time::sleep(backoff.next_backoff()); - let sleep_fut = maybe_timeout(&backoff, start, config.retry_timeout, sleep_fut); + let sleep_fut = + maybe_timeout(backoff.attempt(), start, config.retry_timeout, sleep_fut); sleep_fut.await?; let mut ds = dataset_ref.as_ref().clone(); diff --git a/rust/lance/src/dataset/write/update.rs b/rust/lance/src/dataset/write/update.rs index a87b3a4260b..c8f910d5a36 100644 --- a/rust/lance/src/dataset/write/update.rs +++ b/rust/lance/src/dataset/write/update.rs @@ -1,7 +1,7 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright The Lance Authors -use std::collections::{BTreeMap, HashMap}; +use std::collections::{BTreeMap, HashMap, HashSet}; use std::sync::Arc; use std::time::Duration; @@ -19,8 +19,8 @@ use arrow_schema::{ArrowError, DataType, Schema as ArrowSchema}; use datafusion::common::DFSchema; use datafusion::error::{DataFusionError, Result as DFResult}; use datafusion::logical_expr::ExprSchemable; -use datafusion::physical_plan::PhysicalExpr; use datafusion::physical_plan::stream::RecordBatchStreamAdapter; +use datafusion::physical_plan::{PhysicalExpr, SendableRecordBatchStream}; use datafusion::prelude::Expr; use datafusion::scalar::ScalarValue; use futures::StreamExt; @@ -133,6 +133,16 @@ impl UpdateBuilder { )) })?; + if crate::dataset::optimize::field_contains_blob_v2(field) { + return Err(Error::not_supported_source( + format!( + "Direct updates to column '{}' containing blob v2 values are not supported", + column.as_ref() + ) + .into(), + )); + } + // TODO: support nested column references. This is mostly blocked on the // ability to insert them into the RecordBatch properly. if column.as_ref().contains('.') { @@ -280,8 +290,25 @@ impl UpdateJob { async fn execute_impl(self) -> Result { let mut scanner = self.dataset.scan(); + let legacy_blob_ids = self + .dataset + .schema() + .fields_pre_order() + .filter(|field| field.is_blob() && !field.is_blob_v2()) + .filter_map(|field| u32::try_from(field.id).ok()) + .collect::>(); + if !legacy_blob_ids.is_empty() { + scanner.blob_handling(BlobHandling::SomeBlobsBinary(legacy_blob_ids)); + } + let has_blob_v2_columns = self + .dataset + .schema() + .fields_pre_order() + .any(|field| field.is_blob_v2()); + if has_blob_v2_columns { + scanner.with_row_address(); + } scanner.with_row_id(); - scanner.blob_handling(BlobHandling::AllBinary); if let Some(expr) = &self.condition { scanner.filter_expr(expr.clone()); @@ -296,16 +323,77 @@ impl UpdateJob { let (stream, row_id_rx) = make_rowid_capture_stream(stream, self.dataset.manifest.uses_stable_row_ids())?; - let schema = stream.schema(); - - let expected_schema = self.dataset.schema().into(); - if schema.as_ref() != &expected_schema { + let scan_schema = stream.schema(); + let expected_schema: ArrowSchema = self.dataset.schema().into(); + if !has_blob_v2_columns && scan_schema.as_ref() != &expected_schema { return Err(Error::internal(format!( "Expected schema {:?} but got {:?}", - expected_schema, schema + expected_schema, scan_schema ))); } + let stream = if has_blob_v2_columns { + let rewrite_plan = Arc::new(crate::dataset::optimize::BlobV2BatchRewritePlan::try_new( + self.dataset.schema(), + scan_schema.as_ref(), + false, + )?); + let output_schema = rewrite_plan.output_schema().clone(); + let dataset = self.dataset.clone(); + let transformed = stream.then(move |batch_result| { + let dataset = dataset.clone(); + let rewrite_plan = rewrite_plan.clone(); + async move { + let batch = batch_result?; + rewrite_plan + .transform_batch(&dataset, batch) + .await + .map_err(|error| DataFusionError::External(Box::new(error))) + } + }); + Box::pin(RecordBatchStreamAdapter::new(output_schema, transformed)) + as SendableRecordBatchStream + } else { + stream + }; + let schema = stream.schema(); + + let updated_blob_columns = self + .updates + .keys() + .filter(|column_name| { + self.dataset + .schema() + .field(column_name) + .is_some_and(crate::dataset::optimize::field_contains_blob_v2) + }) + .cloned() + .collect::>(); + let updated_blob_column_indices = schema + .fields() + .iter() + .enumerate() + .filter_map(|(column_idx, field)| { + updated_blob_columns + .contains(field.name()) + .then_some(column_idx) + }) + .collect::>(); + let write_params = WriteParams { + allow_external_blob_outside_bases: has_blob_v2_columns, + ..Default::default() + }; + let external_base_resolver = if updated_blob_column_indices.is_empty() { + None + } else { + super::blob_v2_external_base_resolver( + Some(self.dataset.as_ref()), + &write_params, + self.dataset.schema(), + ) + .await? + }; + let updates_ref = self.updates.clone(); let stream = stream .map(move |batch| { @@ -317,21 +405,39 @@ impl UpdateJob { Ok(Ok(batch)) => Ok(batch), Ok(Err(err)) => Err(err), Err(e) => Err(DataFusionError::ExecutionJoin(Box::new(e))), + }) + .then(move |batch_result| { + let external_base_resolver = external_base_resolver.clone(); + let updated_blob_column_indices = updated_blob_column_indices.clone(); + async move { + let batch = batch_result?; + if let Some(resolver) = external_base_resolver.as_deref() { + let updated_blob_batch = batch.project(&updated_blob_column_indices)?; + let selected_rows = vec![true; batch.num_rows()]; + crate::dataset::blob::validate_external_blob_references( + resolver, + &updated_blob_batch, + &selected_rows, + ) + .await + .map_err(|error| DataFusionError::External(Box::new(error)))?; + } + Ok(batch) + } }); let stream = RecordBatchStreamAdapter::new(schema, stream); - let version = self - .dataset - .manifest() - .data_storage_format - .lance_file_version()?; let (mut new_fragments, _) = write_fragments_internal( + self.dataset + .manifest + .data_storage_format + .lance_file_format(), Some(&self.dataset), self.dataset.object_store.clone(), &self.dataset.base, self.dataset.schema().clone(), Box::pin(stream), - WriteParams::with_storage_version(version), + write_params, None, // TODO: support multiple bases for update ) .await?; @@ -357,7 +463,7 @@ impl UpdateJob { })?; for (fragment, sequence) in new_fragments.iter_mut().zip(sequences) { let serialized = lance_table::rowids::write_row_ids(&sequence); - fragment.row_id_meta = Some(RowIdMeta::Inline(serialized)); + fragment.row_id_meta = Some(RowIdMeta::Inline(serialized.into())); } } @@ -419,7 +525,7 @@ impl UpdateJob { // are moved(deleted and appended). // so we do not need to handle the frag bitmap of the index about it. fields_modified: vec![], - merged_generations: Vec::new(), + compacted_sstables: Vec::new(), fields_for_preserving_frag_bitmap, update_mode: Some(RewriteRows), inserted_rows_filter: None, @@ -538,8 +644,10 @@ mod tests { array::AsArray, datatypes::{Int64Type, UInt32Type}, }; - use arrow_array::types::Float32Type; - use arrow_array::{Int64Array, RecordBatchIterator, StringArray, UInt32Array, UInt64Array}; + use arrow_array::types::{Float32Type, Int32Type}; + use arrow_array::{ + Int64Array, RecordBatchIterator, StringArray, StructArray, UInt32Array, UInt64Array, + }; use arrow_schema::{Field, Schema as ArrowSchema}; use arrow_select::concat::concat_batches; use futures::{TryStreamExt, future::try_join_all}; @@ -550,7 +658,7 @@ mod tests { use lance_datagen::{Dimension, RowCount}; use lance_file::version::LanceFileVersion; use lance_index::IndexType; - use lance_index::scalar::ScalarIndexParams; + use lance_index::scalar::{BuiltinIndexType, ScalarIndexParams}; use lance_io::object_store::ObjectStoreParams; use lance_linalg::distance::MetricType; use object_store::throttle::ThrottleConfig; @@ -1284,6 +1392,178 @@ mod tests { ); } + #[rstest] + #[case::zone_map(BuiltinIndexType::ZoneMap, "i < 100", 100)] + #[case::bloom_filter(BuiltinIndexType::BloomFilter, "i = 0", 1)] + #[tokio::test] + async fn test_addr_domain_index_does_not_cover_rewritten_update_fragment( + #[case] index_type: BuiltinIndexType, + #[case] query: &str, + #[case] expected_rows: usize, + ) { + let mut dataset = lance_datagen::gen_batch() + .col("i", lance_datagen::array::step::()) + .col("category", lance_datagen::array::step::()) + .into_ram_dataset_with_params( + FragmentCount::from(1), + FragmentRowCount::from(100), + Some(WriteParams { + max_rows_per_file: 100, + enable_stable_row_ids: true, + ..Default::default() + }), + ) + .await + .unwrap(); + + dataset + .create_index( + &["i"], + IndexType::Scalar, + Some("i_idx".to_string()), + &ScalarIndexParams::for_builtin(index_type), + true, + ) + .await + .unwrap(); + + let before = dataset + .scan() + .filter(query) + .unwrap() + .try_into_batch() + .await + .unwrap(); + assert_eq!(before.num_rows(), expected_rows); + + let dataset = UpdateBuilder::new(Arc::new(dataset)) + .update_where("i < 20") + .unwrap() + .set("category", "-1") + .unwrap() + .build() + .unwrap() + .execute() + .await + .unwrap() + .new_dataset; + + let indices = dataset.load_indices().await.unwrap(); + let index = indices.iter().find(|index| index.name == "i_idx").unwrap(); + assert_eq!( + index + .fragment_bitmap + .as_ref() + .unwrap() + .iter() + .collect::>(), + vec![0], + "the address-domain index must not cover the rewritten fragment" + ); + + // Regression for https://github.com/lance-format/lance/issues/8278: a later + // update must find rows moved out of the address-domain index's coverage. + let second_update = UpdateBuilder::new(dataset) + .update_where(query) + .unwrap() + .set("category", "-2") + .unwrap() + .build() + .unwrap() + .execute() + .await + .unwrap(); + assert_eq!(second_update.rows_updated, expected_rows as u64); + let dataset = second_update.new_dataset; + + let after = dataset + .scan() + .filter(query) + .unwrap() + .try_into_batch() + .await + .unwrap(); + assert_eq!(after.num_rows(), expected_rows); + + let updated = dataset + .scan() + .filter("category = -2") + .unwrap() + .try_into_batch() + .await + .unwrap(); + assert_eq!(updated.num_rows(), expected_rows); + } + + /// Regression test for https://github.com/lance-format/lance/issues/8076 + /// + /// A bloom filter index reports matches as physical row addresses. An update that + /// replaces every row of a fragment removes that fragment, but the index keeps the + /// addresses it holds for it, so translating its results to row ids has to tolerate + /// a fragment that is gone rather than fail with an internal error. + #[tokio::test] + async fn test_addr_domain_index_after_update_drops_fragment() { + let mut dataset = lance_datagen::gen_batch() + .col("i", lance_datagen::array::step::()) + .into_ram_dataset_with_params( + FragmentCount::from(2), + FragmentRowCount::from(3), + Some(WriteParams { + max_rows_per_file: 3, + enable_stable_row_ids: true, + ..Default::default() + }), + ) + .await + .unwrap(); + + dataset + .create_index( + &["i"], + IndexType::BloomFilter, + Some("i_idx".to_string()), + &ScalarIndexParams::for_builtin(BuiltinIndexType::BloomFilter), + true, + ) + .await + .unwrap(); + + // Rewrites all of fragment 1 (rows 3, 4, 5), which drops the fragment. + let dataset = UpdateBuilder::new(Arc::new(dataset)) + .update_where("i >= 3") + .unwrap() + .set("i", "-1") + .unwrap() + .build() + .unwrap() + .execute() + .await + .unwrap() + .new_dataset; + assert!(dataset.get_fragments().iter().all(|frag| frag.id() != 1)); + + // The index still holds a block for the dropped fragment, and a bloom filter + // cannot rule out a value it once held, so this query is the one that reaches + // the index with addresses in that fragment. + let matched = dataset + .scan() + .filter("i = 4") + .unwrap() + .try_into_batch() + .await + .unwrap(); + assert_eq!(matched.num_rows(), 0); + + let updated = dataset + .scan() + .filter("i = -1") + .unwrap() + .try_into_batch() + .await + .unwrap(); + assert_eq!(updated.num_rows(), 3); + } + #[tokio::test] async fn test_update_mixed_indexed_unindexed_fragments() { let mut dataset = lance_datagen::gen_batch() @@ -1774,4 +2054,104 @@ mod tests { let idx_foo = ids.values().iter().position(|&x| x == 0).unwrap(); assert_eq!(blobs.value(idx_foo), b"foo"); } + + #[rstest] + #[case::non_empty(0)] + #[case::empty(1)] + #[case::null(2)] + #[tokio::test] + async fn test_update_preserves_blob_v2(#[case] selected_id: i64) { + use crate::{BlobArrayBuilder, blob_field}; + + let make_blobs = || { + let mut builder = BlobArrayBuilder::new(3); + builder.push_bytes(b"one").unwrap(); + builder.push_bytes(b"").unwrap(); + builder.push_null().unwrap(); + builder.finish().unwrap() + }; + let nested_fields = vec![blob_field("blob", true)]; + let nested: Arc = Arc::new( + StructArray::try_new(nested_fields.clone().into(), vec![make_blobs()], None).unwrap(), + ); + let schema = Arc::new(ArrowSchema::new(vec![ + Field::new("id", DataType::Int64, false), + Field::new("body", DataType::Utf8, false), + blob_field("payload", true), + Field::new("info", DataType::Struct(nested_fields.into()), true), + ])); + let batch = RecordBatch::try_new( + schema.clone(), + vec![ + Arc::new(Int64Array::from(vec![0, 1, 2])), + Arc::new(StringArray::from(vec!["body-0", "body-1", "body-2"])), + make_blobs(), + nested, + ], + ) + .unwrap(); + let test_dir = TempStrDir::default(); + let dataset = Arc::new( + Dataset::write( + RecordBatchIterator::new(vec![Ok(batch)], schema), + &test_dir, + Some(WriteParams { + data_storage_version: Some(LanceFileVersion::V2_2), + ..Default::default() + }), + ) + .await + .unwrap(), + ); + + for column in ["payload", "info"] { + let error = UpdateBuilder::new(dataset.clone()) + .set(column, column) + .unwrap_err(); + assert!(matches!(error, Error::NotSupported { .. })); + assert!( + error.to_string().contains(&format!( + "Direct updates to column '{column}' containing blob v2 values are not supported" + )), + "unexpected error: {error}" + ); + } + + let result = UpdateBuilder::new(dataset) + .update_where(&format!("id = {selected_id}")) + .unwrap() + .set("body", "'updated'") + .unwrap() + .build() + .unwrap() + .execute() + .await + .unwrap(); + assert_eq!(result.rows_updated, 1); + + let mut scanner = result.new_dataset.scan(); + scanner.blob_handling(BlobHandling::AllBinary); + let batch = scanner.try_into_batch().await.unwrap(); + let ids = batch["id"].as_primitive::(); + let bodies = batch["body"].as_string::(); + let payloads = batch["payload"].as_binary::(); + let nested = batch["info"] + .as_struct() + .column_by_name("blob") + .unwrap() + .as_binary::(); + let expected = [Some(b"one".as_slice()), Some(b"".as_slice()), None]; + + for row_idx in 0..batch.num_rows() { + let id = ids.value(row_idx) as usize; + let expected_body = if id as i64 == selected_id { + "updated" + } else { + ["body-0", "body-1", "body-2"][id] + }; + assert_eq!(bodies.value(row_idx), expected_body); + assert_eq!(payloads.iter().nth(row_idx).unwrap(), expected[id]); + assert_eq!(nested.iter().nth(row_idx).unwrap(), expected[id]); + } + } } diff --git a/rust/lance/src/index.rs b/rust/lance/src/index.rs index f5afb48804c..8ee4c540eeb 100644 --- a/rust/lance/src/index.rs +++ b/rust/lance/src/index.rs @@ -7,25 +7,26 @@ use lance_core::utils::row_addr_remap::RowAddrRemap; use std::collections::{HashMap, HashSet}; use std::sync::{Arc, OnceLock}; +use std::time::Instant; use arrow_schema::DataType; use async_trait::async_trait; use datafusion::execution::SendableRecordBatchStream; -use futures::FutureExt; +use futures::{FutureExt, StreamExt, TryStreamExt}; use itertools::Itertools; -use lance_core::cache::CacheKey; +use lance_core::cache::{CacheKey, CacheKeySchema, KeyBuilder}; use lance_core::datatypes::Field; use lance_core::datatypes::Schema as LanceSchema; use lance_core::utils::parse::parse_env_as_bool; use lance_core::utils::tracing::{ IO_TYPE_OPEN_FRAG_REUSE, IO_TYPE_OPEN_MEM_WAL, IO_TYPE_OPEN_VECTOR, TRACE_IO_EVENTS, }; -use lance_file::previous::reader::FileReader as PreviousFileReader; use lance_file::reader::FileReaderOptions; +use lance_file::versions::v1::reader::FileReader as V1FileReader; use lance_index::INDEX_METADATA_SCHEMA_KEY; pub use lance_index::IndexParams; -use lance_index::frag_reuse::{FRAG_REUSE_INDEX_NAME, FragReuseIndex}; -use lance_index::mem_wal::{MEM_WAL_INDEX_NAME, MemWalIndex}; +use lance_index::frag_reuse::{FRAG_REUSE_INDEX_NAME, FragReuseIndex, FragReuseIndexHandle}; +use lance_index::mem_wal::{MEM_WAL_INDEX_NAME, MemWalIndex, MemWalIndexHandle}; use lance_index::optimize::OptimizeOptions; use lance_index::pb::index::Implementation; pub use lance_index::progress::{IndexBuildProgress, NoopIndexBuildProgress}; @@ -33,7 +34,7 @@ use lance_index::scalar::expression::{IndexInformationProvider, MultiQueryParser use lance_index::scalar::inverted::{InvertedIndex, InvertedIndexPlugin}; use lance_index::scalar::lance_format::LanceIndexStore; use lance_index::scalar::registry::{TrainingCriteria, TrainingOrdering}; -use lance_index::scalar::{CreatedIndex, ScalarIndex}; +use lance_index::scalar::{CreatedIndex, ScalarIndex, index_files_to_table, table_files_to_index}; use lance_index::vector::bq::builder::RabitQuantizer; use lance_index::vector::flat::index::{FlatBinQuantizer, FlatIndex, FlatQuantizer}; use lance_index::vector::hnsw::HNSW; @@ -41,7 +42,10 @@ use lance_index::vector::pq::ProductQuantizer; use lance_index::vector::quantizer::Quantization; use lance_index::vector::sq::ScalarQuantizer; use lance_index::vector::v3::subindex::IvfSubIndex; -use lance_index::{INDEX_FILE_NAME, Index, IndexType, PrewarmOptions, pb, vector::VectorIndex}; +use lance_index::{ + FtsPrewarmDiagnostics, FtsPrewarmOptions, FtsPrewarmResult, FtsPrewarmSegmentStatus, + INDEX_FILE_NAME, Index, IndexType, PrewarmOptions, pb, vector::VectorIndex, +}; use lance_index::{ IndexCriteria, is_system_index, metrics::{MetricsCollector, NoOpMetricsCollector}, @@ -54,16 +58,18 @@ use lance_io::utils::{ CachedFileSize, read_last_block, read_message, read_message_from_buf, read_metadata_offset, read_version, }; +use lance_table::feature_flags::FLAG_MEM_WAL_INDEX_CATCHUP; use lance_table::format::{Fragment, SelfDescribingFileReader}; use lance_table::format::{IndexFile, IndexMetadata, list_index_files_with_sizes}; use lance_table::io::manifest::read_manifest_indexes; use roaring::RoaringBitmap; use scalar::index_matches_criteria; use serde_json::json; -use tracing::{info, instrument}; +use tracing::{info, instrument, warn}; use uuid::Uuid; use vector::details::{ - derive_vector_index_type, infer_missing_vector_details, vector_details_as_json, + derive_vector_index_type, infer_missing_vector_details, needs_vector_details_inference, + vector_details_as_json, }; pub(crate) use vector::details::{vector_index_details, vector_index_details_default}; use vector::ivf::v2::{IVFIndex, IvfStateEntryBox}; @@ -84,14 +90,14 @@ use self::vector::remap_vector_index; use crate::dataset::index::LanceIndexStoreExt; use crate::dataset::optimize::RemappedIndex; use crate::dataset::optimize::remapping::RemapResult; -use crate::dataset::transaction::{Operation, Transaction, TransactionBuilder}; +use crate::dataset::transaction::{Operation, ReadVersionState, Transaction, TransactionBuilder}; pub use crate::index::api::{DatasetIndexExt, IndexSegment, IntoIndexSegment}; use crate::index::frag_reuse::{load_frag_reuse_index_details, open_frag_reuse_index}; use crate::index::mem_wal::open_mem_wal_index; pub use crate::index::prefilter::{FilterLoader, PreFilter}; use crate::index::scalar::{IndexDetails, fetch_index_details, load_training_data}; pub use crate::index::vector::{LogicalIvfView, LogicalVectorIndex}; -use crate::session::index_caches::{FragReuseIndexKey, IndexMetadataKey}; +use crate::session::index_caches::{FragReuseIndexKey, IndexMetadataKey, write_index_identity}; use crate::{Error, Result, dataset::Dataset}; pub use create::CreateIndexBuilder; pub use lance_index::IndexDescription; @@ -130,11 +136,109 @@ fn validate_segment_metadata(index_name: &str, segments: &[IndexMetadata]) -> Re Ok(()) } +fn collect_subtree_field_ids(field: &Field, field_ids: &mut HashSet) { + field_ids.insert(field.id); + for child in &field.children { + collect_subtree_field_ids(child, field_ids); + } +} + +fn fragment_field_paths<'a>( + fragment: &'a Fragment, + indexed_field_ids: &HashSet, +) -> HashMap { + fragment + .files + .iter() + .flat_map(|file| { + file.fields + .iter() + .filter(|field_id| indexed_field_ids.contains(field_id)) + .map(|field_id| (*field_id, file.path.as_str())) + }) + .collect() +} + +async fn prune_stale_segment_coverage( + dataset: &Dataset, + field_id: i32, + segments: &mut [IndexSegment], + prune_historically_missing: bool, + prune_newer_overlays: bool, +) -> Result<()> { + let field = dataset.schema().field_by_id(field_id).ok_or_else(|| { + Error::invalid_input(format!( + "CreateIndex: field id {field_id} does not exist in the current schema" + )) + })?; + let mut indexed_field_ids = HashSet::new(); + collect_subtree_field_ids(field, &mut indexed_field_ids); + + let current_fragments = dataset + .fragments() + .iter() + .map(|fragment| (fragment.id as u32, fragment)) + .collect::>(); + let historical_versions = segments + .iter() + .map(IndexSegment::dataset_version) + .filter(|version| *version < dataset.manifest.version) + .collect::>(); + + for version in historical_versions { + let historical = dataset.checkout_version(version).await.map_err(|error| { + Error::invalid_input(format!( + "CreateIndex: cannot validate segment coverage built at dataset version {version}: {error}" + )) + })?; + let historical_fragments = historical + .fragments() + .iter() + .map(|fragment| (fragment.id as u32, fragment)) + .collect::>(); + + for segment in segments + .iter_mut() + .filter(|segment| segment.dataset_version() == version) + { + let stale_fragments = segment + .fragment_bitmap() + .iter() + .filter(|fragment_id| { + let Some(historical_fragment) = historical_fragments.get(fragment_id) else { + return prune_historically_missing; + }; + let Some(current_fragment) = current_fragments.get(fragment_id) else { + return true; + }; + let changed_files = + fragment_field_paths(historical_fragment, &indexed_field_ids) + != fragment_field_paths(current_fragment, &indexed_field_ids); + let changed_overlays = prune_newer_overlays + && current_fragment.overlays.iter().any(|overlay| { + overlay.committed_version > version + && overlay + .data_file + .fields + .iter() + .any(|field_id| indexed_field_ids.contains(field_id)) + }); + changed_files || changed_overlays + }) + .collect::>(); + for fragment_id in stale_fragments { + segment.fragment_bitmap_mut().remove(fragment_id); + } + } + } + Ok(()) +} + pub(crate) async fn build_index_metadata_from_segments( dataset: &Dataset, index_name: &str, field_id: i32, - segments: Vec, + mut segments: Vec, ) -> Result> { if segments.is_empty() { return Err(Error::invalid_input( @@ -145,6 +249,22 @@ pub(crate) async fn build_index_metadata_from_segments( let mut seen_segment_ids = HashSet::with_capacity(segments.len()); let mut covered_fragments = RoaringBitmap::new(); for segment in &segments { + if segment.dataset_version() > dataset.manifest.version { + return Err(Error::invalid_input(format!( + "CreateIndex: segment {} was built at future dataset version {} (current version {})", + segment.uuid(), + segment.dataset_version(), + dataset.manifest.version + ))); + } + if segment.fields() != [field_id] { + return Err(Error::invalid_input(format!( + "CreateIndex: segment {} was built for fields {:?}, expected [{}]", + segment.uuid(), + segment.fields(), + field_id + ))); + } if !seen_segment_ids.insert(segment.uuid()) { return Err(Error::invalid_input(format!( "CreateIndex: duplicate segment uuid {} for index '{}'", @@ -161,16 +281,18 @@ pub(crate) async fn build_index_metadata_from_segments( covered_fragments |= segment.fragment_bitmap().clone(); } - let mut new_indices = Vec::with_capacity(segments.len()); - for segment in segments { - let (uuid, fragment_bitmap, index_details, index_version) = segment.into_parts(); + prune_stale_segment_coverage(dataset, field_id, &mut segments, false, false).await?; + + let new_indices = futures::stream::iter(segments.into_iter().map(|segment| async move { + let (uuid, fragment_bitmap, fields, index_details, index_version, dataset_version) = + segment.into_parts(); let is_inverted_index = index_details.type_url.ends_with("InvertedIndexDetails"); if is_inverted_index { let metadata = IndexMetadata { uuid, name: index_name.to_string(), - fields: vec![field_id], - dataset_version: dataset.manifest.version, + fields: fields.clone(), + dataset_version, fragment_bitmap: Some(fragment_bitmap.clone()), index_details: Some(index_details.clone()), index_version, @@ -181,24 +303,27 @@ pub(crate) async fn build_index_metadata_from_segments( crate::index::scalar::inverted::finalize_segment_files_if_needed(dataset, &metadata) .await?; } - let index_dir = dataset.indices_dir().clone().join(uuid.to_string()); + let index_dir = dataset.indices_dir().join(uuid.to_string()); let mut files = list_index_files_with_sizes(&dataset.object_store, &index_dir).await?; if is_inverted_index { retain_committed_inverted_files(&mut files); } - new_indices.push(IndexMetadata { + Ok::<_, Error>(IndexMetadata { uuid, name: index_name.to_string(), - fields: vec![field_id], - dataset_version: dataset.manifest.version, + fields, + dataset_version, fragment_bitmap: Some(fragment_bitmap), index_details: Some(index_details), index_version, created_at: Some(chrono::Utc::now()), base_id: None, files: Some(files), - }); - } + }) + })) + .buffered(dataset.object_store.io_parallelism()) + .try_collect::>() + .await?; Ok(new_indices) } @@ -207,6 +332,362 @@ fn retain_committed_inverted_files(files: &mut Vec) { files.retain(|file| !file.path.starts_with("staging/")); } +async fn prewarm_opened_index( + index: Arc, + options: Option<&PrewarmOptions>, +) -> Result<()> { + match options { + None => index.prewarm().await, + Some(PrewarmOptions::Fts(fts_options)) => { + let inverted = index + .as_any() + .downcast_ref::() + .ok_or_else(|| { + Error::invalid_input(format!( + "FTS prewarm options are only supported for inverted indices, got {:?}", + index.index_type() + )) + })?; + inverted.prewarm_with_options(fts_options).await + } + Some(_) => Err(Error::not_supported( + "unsupported prewarm options for this lance version".to_owned(), + )), + } +} + +struct SegmentPrewarmResult { + partition_count: usize, +} + +struct OpenedSegmentPrewarmResult { + index_uuid: Uuid, + index: Arc, + partition_count: usize, +} + +async fn prewarm_opened_index_result( + index: Arc, + options: &PrewarmOptions, +) -> Result { + match options { + PrewarmOptions::Fts(fts_options) => { + let inverted = index + .as_any() + .downcast_ref::() + .ok_or_else(|| { + Error::invalid_input(format!( + "FTS prewarm options are only supported for inverted indices, got {:?}", + index.index_type() + )) + })?; + inverted.prewarm_with_options_result(fts_options).await?; + Ok(SegmentPrewarmResult { + partition_count: inverted.partition_count(), + }) + } + _ => Err(Error::not_supported( + "unsupported prewarm options for this lance version".to_owned(), + )), + } +} + +async fn aggregate_fts_prewarm_results( + dataset: &Dataset, + segment_results: Vec, + options: Option<&PrewarmOptions>, +) -> Result { + let partition_count = segment_results + .iter() + .map(|result| result.partition_count) + .sum(); + let mut failing_segments = Vec::new(); + let mut failing_partitions = Vec::new(); + let mut best_effort = false; + + for segment in segment_results { + let segment_id = segment.index_uuid.to_string(); + let prewarmed_inverted = segment.index.as_any().downcast_ref::(); + let Some(fts_options) = (match options { + Some(PrewarmOptions::Fts(fts_options)) => Some(fts_options.clone()), + None if prewarmed_inverted.is_some() => Some(FtsPrewarmOptions::default()), + _ => None, + }) else { + continue; + }; + best_effort |= fts_options.mode.is_best_effort(); + + let cached_index = + scalar::cached_scalar_index_container(dataset, &segment.index_uuid).await; + let cached_inverted = cached_index + .as_ref() + .and_then(|index| index.as_any().downcast_ref::()); + let container_resident = cached_inverted.is_some(); + let container_matches_prewarmed = match (prewarmed_inverted, cached_inverted) { + (Some(prewarmed), Some(cached)) => std::ptr::addr_eq(prewarmed, cached), + _ => false, + }; + + if !container_resident || !container_matches_prewarmed { + failing_segments.push(FtsPrewarmSegmentStatus { + segment_id: segment_id.clone(), + scalar_index_container_resident: container_resident, + scalar_index_container_matches_prewarmed: container_matches_prewarmed, + }); + } + + if let Some(cached_inverted) = cached_inverted + && let Some(mut diagnostics) = cached_inverted + .prewarm_residency_result(fts_options.with_position) + .await + .diagnostics + { + failing_segments.append(&mut diagnostics.failing_segments); + failing_partitions.extend(diagnostics.failing_partitions.drain(..).map( + |mut partition| { + partition.segment_id = Some(segment_id.clone()); + partition + }, + )); + } + } + + let diagnostics = FtsPrewarmDiagnostics { + partition_count, + failing_segments, + failing_partitions, + }; + + if diagnostics.fully_resident() { + Ok(FtsPrewarmResult::fully_resident()) + } else if best_effort { + Ok(FtsPrewarmResult::partial(diagnostics)) + } else { + Err(Error::internal(diagnostics.to_string())) + } +} + +fn total_index_segment_size_bytes(indices: &[IndexMetadata]) -> Option { + let mut total = 0u64; + for index_meta in indices { + total += index_meta.total_size_bytes()?; + } + Some(total) +} + +fn cache_size_delta(after: usize, before: usize) -> i64 { + after.saturating_sub(before) as i64 - before.saturating_sub(after) as i64 +} + +fn prewarm_options_fields(options: Option<&PrewarmOptions>) -> (&'static str, bool) { + match options { + None => ("default", false), + Some(PrewarmOptions::Fts(fts_options)) => ("fts", fts_options.with_position), + Some(_) => ("unsupported", false), + } +} + +async fn prewarm_index_segments_by_metadata( + dataset: &Dataset, + name: &str, + indices: Vec, + options: Option<&PrewarmOptions>, + available_segment_count: usize, + requested_segment_count: Option, +) -> Result { + let request_started = Instant::now(); + let selected_segment_count = indices.len(); + let selected_size_bytes = total_index_segment_size_bytes(&indices); + let (prewarm_options, fts_with_position) = prewarm_options_fields(options); + let cache_stats_before = dataset.session.index_cache_stats().await; + info!( + index_name = name, + selected_segment_count, + available_segment_count, + requested_segment_count = requested_segment_count.unwrap_or(0), + segment_filter = requested_segment_count.is_some(), + selected_size_bytes = selected_size_bytes.unwrap_or(0), + selected_size_bytes_known = selected_size_bytes.is_some(), + index_cache_entries_before = cache_stats_before.num_entries, + index_cache_size_bytes_before = cache_stats_before.size_bytes, + prewarm_options, + fts_with_position, + "prewarm index segments started" + ); + + let result = futures::future::try_join_all(indices.into_iter().map(|index_meta| async move { + let index_uuid = index_meta.uuid; + let size_bytes = index_meta.total_size_bytes(); + let fragment_count = index_meta + .fragment_bitmap + .as_ref() + .map(|bitmap| bitmap.len()); + let segment_started = Instant::now(); + info!( + index_name = name, + %index_uuid, + index_version = index_meta.index_version, + dataset_version = index_meta.dataset_version, + fragment_count = fragment_count.unwrap_or(0), + fragment_count_known = fragment_count.is_some(), + size_bytes = size_bytes.unwrap_or(0), + size_bytes_known = size_bytes.is_some(), + "prewarm index segment started" + ); + + let index = match dataset + .open_generic_index(name, &index_uuid, &NoOpMetricsCollector) + .await + { + Ok(index) => { + info!( + index_name = name, + %index_uuid, + index_type = ?index.index_type(), + elapsed_ms = segment_started.elapsed().as_millis() as u64, + "opened index segment for prewarm" + ); + index + } + Err(err) => { + warn!( + index_name = name, + %index_uuid, + error = %err, + elapsed_ms = segment_started.elapsed().as_millis() as u64, + "failed to open index segment for prewarm" + ); + return Err(err); + } + }; + + let segment_result = match options { + Some(options) => match prewarm_opened_index_result(index.clone(), options).await { + Ok(result) => result, + Err(err) => { + warn!( + index_name = name, + %index_uuid, + error = %err, + elapsed_ms = segment_started.elapsed().as_millis() as u64, + "prewarm index segment failed" + ); + return Err(err); + } + }, + None => { + if let Err(err) = prewarm_opened_index(index.clone(), None).await { + warn!( + index_name = name, + %index_uuid, + error = %err, + elapsed_ms = segment_started.elapsed().as_millis() as u64, + "prewarm index segment failed" + ); + return Err(err); + } + SegmentPrewarmResult { partition_count: 0 } + } + }; + + info!( + index_name = name, + %index_uuid, + elapsed_ms = segment_started.elapsed().as_millis() as u64, + "prewarm index segment finished" + ); + Ok(OpenedSegmentPrewarmResult { + index_uuid, + index, + partition_count: segment_result.partition_count, + }) + })) + .await; + + match result { + Ok(segment_results) => { + let cache_stats_after = dataset.session.index_cache_stats().await; + let result = aggregate_fts_prewarm_results(dataset, segment_results, options).await?; + info!( + index_name = name, + selected_segment_count, + fts_fully_resident = result.fully_resident, + index_cache_entries_after = cache_stats_after.num_entries, + index_cache_entries_delta = cache_size_delta( + cache_stats_after.num_entries, + cache_stats_before.num_entries, + ), + index_cache_size_bytes_after = cache_stats_after.size_bytes, + index_cache_size_bytes_delta = + cache_size_delta(cache_stats_after.size_bytes, cache_stats_before.size_bytes,), + elapsed_ms = request_started.elapsed().as_millis() as u64, + "prewarm index segments finished" + ); + Ok(result) + } + Err(err) => { + let cache_stats_after = dataset.session.index_cache_stats().await; + warn!( + index_name = name, + selected_segment_count, + index_cache_entries_after = cache_stats_after.num_entries, + index_cache_entries_delta = cache_size_delta( + cache_stats_after.num_entries, + cache_stats_before.num_entries, + ), + index_cache_size_bytes_after = cache_stats_after.size_bytes, + index_cache_size_bytes_delta = cache_size_delta( + cache_stats_after.size_bytes, + cache_stats_before.size_bytes, + ), + error = %err, + elapsed_ms = request_started.elapsed().as_millis() as u64, + "prewarm index segments failed" + ); + Err(err) + } + } +} + +fn filter_index_segments_by_ids( + name: &str, + indices: Vec, + segment_ids: &[Uuid], +) -> Result> { + if segment_ids.is_empty() { + return Ok(Vec::new()); + } + + let requested = segment_ids.iter().copied().collect::>(); + let mut matched = HashSet::new(); + let filtered = indices + .into_iter() + .filter(|index_meta| { + if requested.contains(&index_meta.uuid) { + matched.insert(index_meta.uuid); + true + } else { + false + } + }) + .collect::>(); + + if matched.len() != requested.len() { + let mut missing = requested + .difference(&matched) + .map(ToString::to_string) + .collect::>(); + missing.sort(); + return Err(Error::index_not_found(format!( + "name={}, segment_ids=[{}]", + name, + missing.join(", ") + ))); + } + + Ok(filtered) +} + fn validate_segment_index_details(index_name: &str, segments: &[IndexMetadata]) -> Result<()> { let mut type_url = None::<&str>; for segment in segments { @@ -265,6 +746,13 @@ fn segment_has_bitmap_details(segment: &IndexMetadata) -> bool { .is_some_and(|details| details.type_url.ends_with("BitmapIndexDetails")) } +fn segment_has_bloomfilter_details(segment: &IndexMetadata) -> bool { + segment + .index_details + .as_ref() + .is_some_and(|details| details.type_url.ends_with("BloomFilterIndexDetails")) +} + /// Detect BTree segments, preserving a legacy pre-details fallback. fn segment_has_btree_details(segment: &IndexMetadata) -> bool { segment.index_details.as_ref().map_or_else( @@ -285,6 +773,15 @@ fn segment_has_zonemap_details(segment: &IndexMetadata) -> bool { .is_some_and(|details| details.type_url.ends_with("ZoneMapIndexDetails")) } +/// True when the index reports matches as physical row addresses rather than row ids +/// (`ScalarIndex::results_are_row_addresses`). +/// +/// Such an index cannot follow its data through a rewrite: the addresses it stores +/// name fragments and offsets, and neither kind supports remap. +pub(crate) fn index_results_are_row_addrs(index: &IndexMetadata) -> bool { + segment_has_zonemap_details(index) || segment_has_bloomfilter_details(index) +} + fn segment_has_fmindex_details(segment: &IndexMetadata) -> bool { segment .index_details @@ -299,6 +796,20 @@ fn segment_has_label_list_details(segment: &IndexMetadata) -> bool { .is_some_and(|details| details.type_url.ends_with("LabelListIndexDetails")) } +fn segment_has_rtree_details(segment: &IndexMetadata) -> bool { + segment + .index_details + .as_ref() + .is_some_and(|details| details.type_url.ends_with("RTreeIndexDetails")) +} + +fn segment_has_ngram_details(segment: &IndexMetadata) -> bool { + segment + .index_details + .as_ref() + .is_some_and(|details| details.type_url.ends_with("NGramIndexDetails")) +} + // Cache keys for different index types #[derive(Debug, Clone)] pub(crate) struct LegacyVectorIndexCacheKey<'a> { @@ -326,6 +837,14 @@ impl CacheKey for LegacyVectorIndexCacheKey<'_> { fn type_name() -> &'static str { "LegacyVectorIndex" } + + fn schema() -> CacheKeySchema { + CacheKeySchema::new("lance.index.legacy-vector-key", 1) + } + + fn write_key(&self, builder: &mut KeyBuilder) { + write_index_identity(builder, self.uuid, self.fri_uuid); + } } /// Sized cache key for `IvfIndexState`. @@ -333,6 +852,11 @@ impl CacheKey for LegacyVectorIndexCacheKey<'_> { /// Used for v0.3+ indices that support serialization. This key has a codec, /// so custom cache backends can serialize the state to disk/Redis/etc. /// Legacy indices use `LegacyVectorIndexCacheKey` instead (in-memory only). +/// +/// Note: legacy entries hold live readers bound to the object store that +/// opened them, so they are only valid for credential setups that refresh +/// internally (e.g. a credentials provider). Deployments that pass fresh +/// static credentials per dataset open should use v0.3+ index formats. #[derive(Debug, Clone)] pub(crate) struct IvfIndexStateCacheKey<'a> { uuid: &'a Uuid, @@ -360,6 +884,14 @@ impl CacheKey for IvfIndexStateCacheKey<'_> { } } + fn schema() -> CacheKeySchema { + CacheKeySchema::new("lance.index.ivf-state-key", 1) + } + + fn write_key(&self, builder: &mut KeyBuilder) { + write_index_identity(builder, self.uuid, self.fri_uuid); + } + fn codec() -> Option { Some(lance_core::cache::CacheCodec::from_impl::()) } @@ -396,6 +928,14 @@ impl CacheKey for FragReuseIndexCacheKey<'_> { fn type_name() -> &'static str { "FragReuseIndex" } + + fn schema() -> CacheKeySchema { + CacheKeySchema::new("lance.index.fragment-reuse-cache-key", 1) + } + + fn write_key(&self, builder: &mut KeyBuilder) { + write_index_identity(builder, self.uuid, self.fri_uuid); + } } #[derive(Debug, Clone)] @@ -424,6 +964,14 @@ impl CacheKey for MemWalCacheKey<'_> { fn type_name() -> &'static str { "MemWalIndex" } + + fn schema() -> CacheKeySchema { + CacheKeySchema::new("lance.index.mem-wal-key", 1) + } + + fn write_key(&self, builder: &mut KeyBuilder) { + write_index_identity(builder, self.uuid, self.fri_uuid); + } } // Whether to auto-migrate a dataset when we encounter corruption. @@ -605,7 +1153,7 @@ pub(crate) async fn remap_index( ) .unwrap(), index_version, - files, + files: table_files_to_index(files), } } _ => { @@ -621,7 +1169,7 @@ pub(crate) async fn remap_index( new_id, index_details: created_index.index_details, index_version: created_index.index_version, - files: Some(created_index.files), + files: Some(index_files_to_table(created_index.files)), })) } @@ -781,10 +1329,15 @@ impl IndexDescriptionImpl { let mut missing_fragment_refs = 0u64; for shard in &segments { - let fragment_bitmap = shard - .fragment_bitmap - .as_ref() - .ok_or_else(|| Error::index("Fragment bitmap is required for index description. This index must be retrained to support this method.".to_string()))?; + let Some(fragment_bitmap) = shard.fragment_bitmap.as_ref() else { + // A system index (e.g. __mem_wal) indexes no fragments, so a + // missing bitmap means zero indexed rows. For a data index it + // means unknown coverage — reject rather than fabricate a count. + if is_system_index(shard) { + continue; + } + return Err(Error::index("Fragment bitmap is required for index description. This index must be retrained to support this method.".to_string())); + }; indexed_fragment_refs += fragment_bitmap.len(); for fragment_id in fragment_bitmap.iter() { @@ -879,6 +1432,49 @@ impl IndexDescription for IndexDescriptionImpl { } } +impl Dataset { + /// Whether an otherwise empty commit would record a new MemWAL catch-up + /// position. + /// + /// Dry-runs the derivation rather than restating its conditions: a second + /// copy of "is this index behind" would be one more place to keep in step + /// with the real rule. A no-work optimize publishes no new segment, so the + /// index list it would commit is the one already loaded, and the version it + /// would read is the current one -- which makes the speculative answer the + /// same one the commit reaches. + fn mem_wal_catch_up_would_advance(&self, indices: &[IndexMetadata]) -> Result { + if self.manifest.reader_feature_flags & FLAG_MEM_WAL_INDEX_CATCHUP == 0 + || self.manifest.writer_feature_flags & FLAG_MEM_WAL_INDEX_CATCHUP == 0 + { + return Ok(false); + } + let catchup_of = |indices: &[IndexMetadata]| -> Result>> { + indices + .iter() + .find(|index| index.name == MEM_WAL_INDEX_NAME) + .cloned() + .map(|index| { + crate::index::mem_wal::load_mem_wal_index_details(index) + .map(|details| details.index_catchup) + }) + .transpose() + }; + + let mut speculative = indices.to_vec(); + Transaction::apply_mem_wal_index_coverage( + &mut speculative, + &Transaction::logical_index_segments(indices), + Some(ReadVersionState { + manifest: &self.manifest, + indices, + }), + true, + self.manifest.version + 1, + )?; + Ok(catchup_of(&speculative)? != catchup_of(indices)?) + } +} + #[async_trait] impl DatasetIndexExt for Dataset { type IndexBuilder<'a> = CreateIndexBuilder<'a>; @@ -974,49 +1570,93 @@ impl DatasetIndexExt for Dataset { return Err(Error::index_not_found(format!("name={}", name))); } - for index_meta in indices { - let index = self - .open_generic_index(name, &index_meta.uuid, &NoOpMetricsCollector) - .await?; - index.prewarm().await?; + let available_segment_count = indices.len(); + prewarm_index_segments_by_metadata(self, name, indices, None, available_segment_count, None) + .await + .map(|_| ()) + } + + async fn prewarm_index_with_options(&self, name: &str, options: &PrewarmOptions) -> Result<()> { + self.prewarm_index_with_options_result(name, options) + .await + .map(|_| ()) + } + + async fn prewarm_index_with_options_result( + &self, + name: &str, + options: &PrewarmOptions, + ) -> Result { + let indices = self.load_indices_by_name(name).await?; + if indices.is_empty() { + return Err(Error::index_not_found(format!("name={}", name))); } - Ok(()) + let available_segment_count = indices.len(); + prewarm_index_segments_by_metadata( + self, + name, + indices, + Some(options), + available_segment_count, + None, + ) + .await } - async fn prewarm_index_with_options(&self, name: &str, options: &PrewarmOptions) -> Result<()> { + async fn prewarm_index_segments(&self, name: &str, segment_ids: &[Uuid]) -> Result<()> { let indices = self.load_indices_by_name(name).await?; if indices.is_empty() { return Err(Error::index_not_found(format!("name={}", name))); } + let available_segment_count = indices.len(); + let indices = filter_index_segments_by_ids(name, indices, segment_ids)?; - for index_meta in indices { - let index = self - .open_generic_index(name, &index_meta.uuid, &NoOpMetricsCollector) - .await?; + prewarm_index_segments_by_metadata( + self, + name, + indices, + None, + available_segment_count, + Some(segment_ids.len()), + ) + .await + .map(|_| ()) + } - match options { - PrewarmOptions::Fts(fts_options) => { - let inverted = index - .as_any() - .downcast_ref::() - .ok_or_else(|| { - Error::invalid_input(format!( - "FTS prewarm options are only supported for inverted indices, got {:?}", - index.index_type() - )) - })?; - inverted.prewarm_with_options(fts_options).await?; - } - _ => { - return Err(Error::not_supported( - "unsupported prewarm options for this lance version".to_owned(), - )); - } - } + async fn prewarm_index_segments_with_options( + &self, + name: &str, + segment_ids: &[Uuid], + options: &PrewarmOptions, + ) -> Result<()> { + self.prewarm_index_segments_with_options_result(name, segment_ids, options) + .await + .map(|_| ()) + } + + async fn prewarm_index_segments_with_options_result( + &self, + name: &str, + segment_ids: &[Uuid], + options: &PrewarmOptions, + ) -> Result { + let indices = self.load_indices_by_name(name).await?; + if indices.is_empty() { + return Err(Error::index_not_found(format!("name={}", name))); } + let available_segment_count = indices.len(); + let indices = filter_index_segments_by_ids(name, indices, segment_ids)?; - Ok(()) + prewarm_index_segments_by_metadata( + self, + name, + indices, + Some(options), + available_segment_count, + Some(segment_ids.len()), + ) + .await } async fn describe_indices<'a, 'b>( @@ -1066,10 +1706,11 @@ impl DatasetIndexExt for Dataset { async fn load_indices(&self) -> Result>> { let metadata_key = IndexMetadataKey { version: self.version().version, + store_identity: &self.object_store.store_prefix, }; - let mut indices = match self.index_cache.get_with_key(&metadata_key).await { - Some(indices) => indices, - None => { + let mut indices = self + .index_cache + .get_or_insert_with_key(metadata_key, || async { let mut loaded_indices = read_manifest_indexes( &self.object_store, &self.manifest_location, @@ -1077,25 +1718,27 @@ impl DatasetIndexExt for Dataset { ) .await?; retain_supported_indices(&mut loaded_indices); - let loaded_indices = Arc::new(loaded_indices); - self.index_cache - .insert_with_key(&metadata_key, loaded_indices.clone()) - .await; - loaded_indices - } - }; + Ok(loaded_indices) + }) + .await?; // Infer details for legacy vector indices (once per index name, concurrently). // This may run on indices that were opportunistically cached during Dataset::open // before the full Dataset was available for inference. { - let mut updated = indices.as_ref().clone(); - infer_missing_vector_details(self, &mut updated).await; - if updated != *indices { - indices = Arc::new(updated); - self.index_cache - .insert_with_key(&metadata_key, indices.clone()) - .await; + let schema = self.schema(); + if indices + .iter() + .any(|idx| needs_vector_details_inference(idx, schema)) + { + let mut updated = indices.as_ref().clone(); + infer_missing_vector_details(self, &mut updated).await; + if updated != *indices { + indices = Arc::new(updated); + self.index_cache + .insert_with_key(&metadata_key, indices.clone()) + .await; + } } } @@ -1127,9 +1770,23 @@ impl DatasetIndexExt for Dataset { async fn merge_existing_index_segments( &self, - source_segments: Vec, + mut source_segments: Vec, ) -> Result { validate_segment_metadata("uncommitted", &source_segments)?; + if let Some(segment) = source_segments + .iter() + .find(|segment| segment.dataset_version > self.manifest.version) + { + return Err(Error::invalid_input(format!( + "merge_existing_index_segments: segment {} was built at future dataset version {} (current version {})", + segment.uuid, segment.dataset_version, self.manifest.version + ))); + } + let source_dataset_version = source_segments + .iter() + .map(|segment| segment.dataset_version) + .min() + .unwrap_or(self.manifest.version); let field_id = *source_segments[0].fields.first().ok_or_else(|| { Error::invalid_input(format!( "CreateIndex: segment {} is missing field ids", @@ -1147,17 +1804,23 @@ impl DatasetIndexExt for Dataset { let all_vector = source_segments.iter().all(segment_has_vector_details); let all_inverted = source_segments.iter().all(segment_has_inverted_details); let all_bitmap = source_segments.iter().all(segment_has_bitmap_details); + let all_bloomfilter = source_segments.iter().all(segment_has_bloomfilter_details); let all_btree = source_segments.iter().all(segment_has_btree_details); let all_fmindex = source_segments.iter().all(segment_has_fmindex_details); let all_zonemap = source_segments.iter().all(segment_has_zonemap_details); let all_label_list = source_segments.iter().all(segment_has_label_list_details); + let all_rtree = source_segments.iter().all(segment_has_rtree_details); + let all_ngram = source_segments.iter().all(segment_has_ngram_details); if !all_vector && !all_inverted && !all_bitmap + && !all_bloomfilter && !all_btree && !all_fmindex && !all_zonemap && !all_label_list + && !all_rtree + && !all_ngram { return Err(Error::invalid_input( "merge_existing_index_segments requires all segments to have the same supported index type" @@ -1165,6 +1828,21 @@ impl DatasetIndexExt for Dataset { )); } + let merged_dataset_version = if all_rtree { + let mut source_coverage = source_segments + .iter() + .cloned() + .map(IntoIndexSegment::into_index_segment) + .collect::>>()?; + prune_stale_segment_coverage(self, field_id, &mut source_coverage, true, true).await?; + for (source, coverage) in source_segments.iter_mut().zip(source_coverage) { + source.fragment_bitmap = Some(coverage.fragment_bitmap().clone()); + } + self.manifest.version + } else { + source_dataset_version + }; + let mut merged_segment = if all_vector { crate::index::vector::ivf::merge_segments( self.object_store.as_ref(), @@ -1178,14 +1856,29 @@ impl DatasetIndexExt for Dataset { crate::index::scalar::fmindex::merge_segments(self, source_segments).await? } else if all_bitmap { crate::index::scalar::bitmap::merge_segments(self, source_segments).await? + } else if all_bloomfilter { + crate::index::scalar::bloomfilter::merge_segments(self, source_segments).await? } else if all_label_list { crate::index::scalar::label_list::merge_segments(self, source_segments).await? } else if all_zonemap { crate::index::scalar::zonemap::merge_segments(self, source_segments).await? + } else if all_ngram { + crate::index::scalar::ngram::merge_segments(self, source_segments).await? + } else if all_rtree { + #[cfg(feature = "geo")] + { + crate::index::scalar::rtree::merge_segments(self, source_segments).await? + } + #[cfg(not(feature = "geo"))] + return Err(Error::not_supported( + "RTree segment merge requires the `geo` feature".to_string(), + )); } else { crate::index::scalar::btree::merge_segments(self, source_segments).await? }; - merged_segment.dataset_version = self.manifest.version; + if !all_ngram && !all_fmindex { + merged_segment.dataset_version = merged_dataset_version; + } merged_segment.fields = vec![field_id]; Ok(merged_segment) } @@ -1206,6 +1899,34 @@ impl DatasetIndexExt for Dataset { .into_iter() .map(IntoIndexSegment::into_index_segment) .collect::>>()?; + let dataset_fragments = self.fragment_bitmap.as_ref().clone(); + if segments.first().is_some_and(|segment| { + segment + .index_details() + .type_url + .ends_with("NGramIndexDetails") + }) { + let has_retired_coverage = segments + .iter() + .any(|segment| !(segment.fragment_bitmap() - &dataset_fragments).is_empty()); + let frag_reuse_index = self.open_frag_reuse_index(&NoOpMetricsCollector).await?; + let requires_rebuild = frag_reuse_index.as_ref().is_some_and(|frag_reuse_index| { + segments.iter().any(|segment| { + append::fragment_reuse_affects_segment( + frag_reuse_index, + segment.fragment_bitmap(), + segment.dataset_version(), + ) + }) + }); + if has_retired_coverage || requires_rebuild { + return Err(Error::invalid_input( + "CreateIndex: NGram segments built before compaction must be rebuilt or merged \ + with merge_existing_index_segments before commit" + .to_string(), + )); + } + } let new_indices = build_index_metadata_from_segments(self, index_name, field.id, segments).await?; validate_segment_metadata(index_name, &new_indices)?; @@ -1215,7 +1936,6 @@ impl DatasetIndexExt for Dataset { .index_details .as_ref() .map(|details| details.type_url.clone()); - let dataset_fragments = self.fragment_bitmap.as_ref().clone(); let mut incoming_fragments = RoaringBitmap::new(); for segment in &new_indices { if segment.fields != [field.id] { @@ -1239,15 +1959,39 @@ impl DatasetIndexExt for Dataset { please specify a different name" ))); } + let existing_different_type_url = existing_named_indices.iter().find_map(|idx| { + // Legacy metadata may omit index details. Its type cannot be proven + // compatible, so only a full replacement is safe. + let Some(existing_details) = idx.index_details.as_ref() else { + return Some("".to_owned()); + }; + let existing_type_url = existing_details.type_url.as_str(); + (Some(existing_type_url) != incoming_type_url.as_deref()) + .then(|| existing_type_url.to_owned()) + }); + let missing_fragments = &dataset_fragments - &incoming_fragments; + if let Some(existing_type_url) = &existing_different_type_url + && !missing_fragments.is_empty() + { + return Err(Error::invalid_input(format!( + "CreateIndex: cannot change index '{}' from type '{}' to type '{}' with partial fragment coverage; incoming segments are missing current fragments {:?}", + index_name, + existing_type_url, + incoming_type_url.as_deref().unwrap_or(""), + missing_fragments.iter().collect::>() + ))); + } + + let is_index_type_change = existing_different_type_url.is_some(); let removed_indices = existing_named_indices .into_iter() - .filter(|idx| { - idx.index_details - .as_ref() - .zip(incoming_type_url.as_deref()) - .is_none_or(|(details, expected)| details.type_url == expected) - }) .map(|idx| -> Result> { + // A logical index cannot combine segment types. Full current-fragment + // coverage was verified above, so a type change replaces every segment. + if is_index_type_change { + return Ok(Some(idx)); + } + let Some(existing_fragments) = idx.effective_fragment_bitmap(&dataset_fragments) else { if incoming_fragments != dataset_fragments { @@ -1367,6 +2111,7 @@ impl DatasetIndexExt for Dataset { } #[instrument(skip_all)] + async fn optimize_indices(&mut self, options: &OptimizeOptions) -> Result<()> { let dataset = Arc::new(self.clone()); let indices = self.load_indices().await?; @@ -1412,7 +2157,7 @@ impl DatasetIndexExt for Dataset { uuid: res.new_uuid, name: last_idx.name.clone(), // Keep the same name fields: last_idx.fields.clone(), - dataset_version: self.manifest.version, + dataset_version: res.new_dataset_version, fragment_bitmap: Some(res.new_fragment_bitmap), index_details: Some(Arc::new(res.new_index_details)), index_version: res.new_index_version, @@ -1424,7 +2169,14 @@ impl DatasetIndexExt for Dataset { new_indices.push(new_idx); } - if new_indices.is_empty() { + // A no-work optimize still has to commit on a table that requires + // catch-up. Coverage is derived at commit time, so an index that + // already spans the table records its position only if there is a + // commit to record it on -- and that is the ordinary case after a + // remap or a compaction that advanced a generation without changing + // fragments. Returning early there leaves the position missing forever + // and the repair rescheduling itself. + if new_indices.is_empty() && !self.mem_wal_catch_up_would_advance(&indices)? { return Ok(()); } @@ -1473,9 +2225,10 @@ impl DatasetIndexExt for Dataset { if indices.is_empty() { return Err(Error::index_not_found(format!("name={}", index_name))); } - let column = self.schema().field_by_id(indices[0].fields[0]).unwrap(); + let field_id = indices[0].fields[0]; + let field_path = self.schema().field_path(field_id)?; let logical_index = self - .open_logical_vector_index(&column.name, index_name) + .open_logical_vector_index(&field_path, index_name) .await?; logical_index .as_ivf()? @@ -1566,7 +2319,7 @@ async fn index_statistics_frag_reuse(ds: &Dataset) -> Result { .open_frag_reuse_index(&NoOpMetricsCollector) .await? .expect("FragmentReuse index does not exist"); - serialize_index_statistics(&index.statistics()?) + serialize_index_statistics(&FragReuseIndexHandle(index).statistics()?) } async fn index_statistics_mem_wal(ds: &Dataset) -> Result { @@ -1574,7 +2327,7 @@ async fn index_statistics_mem_wal(ds: &Dataset) -> Result { .open_mem_wal_index(&NoOpMetricsCollector) .await? .expect("MemWal index does not exist"); - serialize_index_statistics(&index.statistics()?) + serialize_index_statistics(&MemWalIndexHandle(index).statistics()?) } async fn index_statistics_scalar( @@ -1834,7 +2587,7 @@ impl DatasetIndexInternalExt for Dataset { let frag_reuse_cache_key = FragReuseIndexCacheKey::new(uuid, frag_reuse_uuid.as_ref()); if let Some(index) = self.index_cache.get_with_key(&frag_reuse_cache_key).await { - return Ok(index.as_index()); + return Ok(Arc::new(FragReuseIndexHandle(index)).as_index()); } // Sometimes we want to open an index and we don't care if it is a scalar or vector index. @@ -1906,7 +2659,7 @@ impl DatasetIndexInternalExt for Dataset { let state_key = IvfIndexStateCacheKey::new(uuid, frag_reuse_uuid.as_ref()); if let Some(entry) = self.index_cache.get_with_key(&state_key).await { log::debug!("Found IvfIndexState in cache uuid: {}", uuid); - let partition_cache = self.index_cache.with_key_prefix(&state_key.key()); + let partition_cache = self.index_cache.for_index(uuid, frag_reuse_uuid.as_ref()); let frag_reuse_index = self.open_frag_reuse_index(metrics).await?; return entry .0 @@ -1944,8 +2697,10 @@ impl DatasetIndexInternalExt for Dataset { let tailing_bytes = read_last_block(reader.as_ref()).await?; let (major_version, minor_version) = read_version(&tailing_bytes)?; - // Namespace the index cache by the UUID of the index. - let index_cache = self.index_cache.with_key_prefix(&cache_key.key()); + // Namespace the index cache by the UUID of the index. v2+ partition + // entries are store-free and remain reusable across object-store + // generations alongside their serializable state. + let index_cache = self.index_cache.for_index(uuid, frag_reuse_uuid.as_ref()); // Extract the cacheable state before type-erasing to Arc. fn wrap_ivf( @@ -1985,7 +2740,7 @@ impl DatasetIndexInternalExt for Dataset { (0, 2) => { info!(target: TRACE_IO_EVENTS, index_uuid=%uuid, r#type=IO_TYPE_OPEN_VECTOR, version="0.2", index_type="IVF_PQ"); - let reader = PreviousFileReader::try_new_self_described_from_reader( + let reader = V1FileReader::try_new_self_described_from_reader( reader.clone(), Some(&self.metadata_cache.file_metadata_cache(&index_file)), ) @@ -2003,8 +2758,8 @@ impl DatasetIndexInternalExt for Dataset { (0, 3) | (2, _) => { let scheduler = ScanScheduler::new( - self.object_store.clone(), - SchedulerConfig::max_bandwidth(&self.object_store), + object_store.clone(), + SchedulerConfig::max_bandwidth(&object_store), ); let cached_size = file_sizes .get(INDEX_FILE_NAME) @@ -2038,7 +2793,7 @@ impl DatasetIndexInternalExt for Dataset { "IVF_FLAT" => match element_type { DataType::Float16 | DataType::Float32 | DataType::Float64 => { let ivf = IVFIndex::::try_new( - self.object_store.clone(), + object_store.clone(), index_dir, uuid.to_owned(), frag_reuse_index, @@ -2051,7 +2806,7 @@ impl DatasetIndexInternalExt for Dataset { } DataType::UInt8 => { let ivf = IVFIndex::::try_new( - self.object_store.clone(), + object_store.clone(), index_dir, uuid.to_owned(), frag_reuse_index, @@ -2070,7 +2825,7 @@ impl DatasetIndexInternalExt for Dataset { "IVF_PQ" => { let ivf = IVFIndex::::try_new( - self.object_store.clone(), + object_store.clone(), index_dir, uuid.to_owned(), frag_reuse_index, @@ -2084,7 +2839,7 @@ impl DatasetIndexInternalExt for Dataset { "IVF_SQ" => { let ivf = IVFIndex::::try_new( - self.object_store.clone(), + object_store.clone(), index_dir, uuid.to_owned(), frag_reuse_index, @@ -2098,8 +2853,8 @@ impl DatasetIndexInternalExt for Dataset { "IVF_RQ" => { let ivf = IVFIndex::::try_new( - self.object_store.clone(), - self.indices_dir(), + object_store.clone(), + index_dir, uuid.to_owned(), frag_reuse_index, self.metadata_cache.as_ref(), @@ -2113,7 +2868,7 @@ impl DatasetIndexInternalExt for Dataset { "IVF_HNSW_FLAT" => match element_type { DataType::UInt8 => { let ivf = IVFIndex::::try_new( - self.object_store.clone(), + object_store.clone(), index_dir, uuid.to_owned(), frag_reuse_index, @@ -2126,7 +2881,7 @@ impl DatasetIndexInternalExt for Dataset { } _ => { let ivf = IVFIndex::::try_new( - self.object_store.clone(), + object_store.clone(), index_dir, uuid.to_owned(), frag_reuse_index, @@ -2141,7 +2896,7 @@ impl DatasetIndexInternalExt for Dataset { "IVF_HNSW_SQ" => { let ivf = IVFIndex::::try_new( - self.object_store.clone(), + object_store.clone(), index_dir, uuid.to_owned(), frag_reuse_index, @@ -2155,7 +2910,7 @@ impl DatasetIndexInternalExt for Dataset { "IVF_HNSW_PQ" => { let ivf = IVFIndex::::try_new( - self.object_store.clone(), + object_store.clone(), index_dir, uuid.to_owned(), frag_reuse_index, @@ -2190,7 +2945,6 @@ impl DatasetIndexInternalExt for Dataset { io_stats.add_scan_stats(&open_stats); } if let Some(ivf_entry) = ivf_entry { - let state_key = IvfIndexStateCacheKey::new(uuid, frag_reuse_uuid.as_ref()); self.index_cache .insert_with_key(&state_key, Arc::new(ivf_entry)) .await; @@ -2327,12 +3081,6 @@ impl DatasetIndexInternalExt for Dataset { // so the optimizer treats coverage as unknown. let mut fragment_bitmaps: HashMap<(String, String), Option> = HashMap::new(); for index in indices.iter().filter(|idx| { - let idx_schema = schema.project_by_ids(idx.fields.as_slice(), true); - let is_vector_index = idx_schema - .fields - .iter() - .any(|f| is_vector_field(f.data_type())); - // Check if this is an FTS index by looking at index details let is_fts_index = if let Some(details) = &idx.index_details { IndexDetails(details.clone()).supports_fts() @@ -2346,7 +3094,7 @@ impl DatasetIndexInternalExt for Dataset { !bitmap.is_empty() && !(bitmap & self.fragment_bitmap.as_ref()).is_empty() }); - idx.fields.len() == 1 && !is_vector_index && (has_non_empty_bitmap || is_fts_index) + idx.fields.len() == 1 && (has_non_empty_bitmap || is_fts_index) }) { let field = index.fields[0]; let field = schema.field_by_id(field).ok_or_else(|| { @@ -2491,7 +3239,7 @@ impl DatasetIndexInternalExt for Dataset { )) })?; - let mut field_names = Vec::new(); + let mut field_paths = Vec::with_capacity(source_index.fields.len()); for field_id in source_index.fields.iter() { let source_field = source_dataset .schema() @@ -2502,33 +3250,35 @@ impl DatasetIndexInternalExt for Dataset { field_id )) })?; + let source_field_path = source_dataset.schema().field_path(*field_id)?; - let target_field = self.schema().field(&source_field.name).ok_or_else(|| { + let target_field = self.schema().field(&source_field_path).ok_or_else(|| { Error::index(format!( "Field '{}' required by index '{}' not found in target dataset", - source_field.name, index_name + source_field_path, index_name )) })?; if source_field.data_type() != target_field.data_type() { return Err(Error::index(format!( "Field '{}' has different types in source ({:?}) and target ({:?}) datasets", - source_field.name, + source_field_path, source_field.data_type(), target_field.data_type() ))); } - field_names.push(source_field.name.as_str()); + field_paths.push(source_field_path); } - if field_names.is_empty() { + if field_paths.is_empty() { return Err(Error::index(format!( "Index '{}' has no fields", index_name ))); } + let field_names = field_paths.iter().map(String::as_str).collect::>(); if let Some(index_details) = &source_index.index_details { let index_details_wrapper = IndexDetails(index_details.clone()); @@ -2636,7 +3386,7 @@ mod tests { use crate::session::Session; use crate::utils::test::{DatagenExt, FragmentCount, FragmentRowCount, copy_test_data_to_tmp}; use arrow::array::AsArray; - use arrow::datatypes::{Float32Type, Int32Type}; + use arrow::datatypes::{Float32Type, Int32Type, Int64Type}; use arrow_array::Int32Array; use arrow_array::{ FixedSizeListArray, Float32Array, RecordBatch, RecordBatchIterator, StringArray, @@ -2647,8 +3397,12 @@ mod tests { use lance_core::utils::tempfile::TempStrDir; use lance_datagen::gen_batch; use lance_datagen::{BatchCount, ByteCount, Dimension, RowCount, array}; - use lance_index::pbold::BTreeIndexDetails; + use lance_index::pbold::{BTreeIndexDetails, InvertedIndexDetails}; use lance_index::scalar::bitmap::BITMAP_LOOKUP_NAME; + use lance_index::scalar::inverted::query::{FtsQuery, PhraseQuery}; + use lance_index::scalar::inverted::{ + INVERTED_INDEX_VERSION_V1, INVERTED_INDEX_VERSION_V2, INVERTED_INDEX_VERSION_V3, + }; use lance_index::scalar::{ BuiltinIndexType, FullTextSearchQuery, InvertedIndexParams, ScalarIndexParams, }; @@ -2715,20 +3469,10 @@ mod tests { } fn segment_from_metadata(metadata: &IndexMetadata) -> IndexSegment { - IndexSegment::new( - metadata.uuid, - metadata - .fragment_bitmap - .as_ref() - .expect("test segment metadata should have fragment coverage") - .iter(), - metadata - .index_details - .as_ref() - .expect("test segment metadata should have index details") - .clone(), - metadata.index_version, - ) + metadata + .clone() + .into_index_segment() + .expect("test segment metadata should convert to an index segment") } async fn write_fragmented_vector_dataset(uri: &str, dimension: i32) -> Dataset { @@ -3644,6 +4388,53 @@ mod tests { assert_eq!(stats["num_indices"], 1); } + #[rstest] + #[case::v1("v3.0.1/fts_v1", INVERTED_INDEX_VERSION_V1)] + #[case::v2("v4.0.1/fts_v2", INVERTED_INDEX_VERSION_V2)] + #[tokio::test] + async fn test_read_fts_format_fixture( + #[case] fixture_path: &str, + #[case] expected_version: u32, + ) { + async fn search_ids(dataset: &Dataset, query: FullTextSearchQuery) -> Vec { + let result = dataset + .scan() + .project(&["id"]) + .unwrap() + .full_text_search(query) + .unwrap() + .try_into_batch() + .await + .unwrap(); + let mut ids = result["id"].as_primitive::().values().to_vec(); + ids.sort_unstable(); + ids + } + + let test_dir = copy_test_data_to_tmp(fixture_path).unwrap(); + let dataset = Dataset::open(&test_dir.path_str()).await.unwrap(); + + let indices = dataset.load_indices().await.unwrap(); + assert_eq!(indices.len(), 1); + assert_eq!(indices[0].index_version, expected_version as i32); + + let match_ids = search_ids( + &dataset, + FullTextSearchQuery::new("compatibility".to_string()), + ) + .await; + assert_eq!(match_ids, (0..300).collect::>()); + + let phrase = + PhraseQuery::new("lance database".to_string()).with_column(Some("text".to_string())); + let phrase_ids = search_ids( + &dataset, + FullTextSearchQuery::new_query(FtsQuery::Phrase(phrase)), + ) + .await; + assert_eq!(phrase_ids, (0..300).step_by(3).collect::>()); + } + #[rstest] #[tokio::test] async fn test_optimize_fts(#[values(false, true)] with_position: bool) { @@ -4102,6 +4893,48 @@ mod tests { assert_eq!(indices2.len(), 1); } + #[tokio::test] + async fn test_load_indices_singleflights_concurrent_cache_misses() { + let session = Arc::new(Session::default()); + let write_params = WriteParams { + session: Some(session.clone()), + ..Default::default() + }; + let test_dir = TempStrDir::default(); + let schema = Arc::new(Schema::new(vec![Field::new("tag", DataType::Utf8, false)])); + let batch = RecordBatch::try_new( + schema.clone(), + vec![Arc::new(StringArray::from(vec!["a", "b", "c"]))], + ) + .unwrap(); + let mut dataset = Dataset::write( + RecordBatchIterator::new(vec![Ok(batch)], schema), + &test_dir, + Some(write_params), + ) + .await + .unwrap(); + dataset + .create_index( + &["tag"], + IndexType::Bitmap, + None, + &ScalarIndexParams::default(), + false, + ) + .await + .unwrap(); + + session.index_cache.clear().await; + let before = session.index_cache_stats().await; + let results = futures::future::join_all((0..32).map(|_| dataset.load_indices())).await; + assert!(results.iter().all(|result| result.is_ok())); + let after = session.index_cache_stats().await; + + assert_eq!(after.misses - before.misses, 1); + assert_eq!(after.hits - before.hits, 31); + } + #[tokio::test] async fn test_remap_empty() { let data = gen_batch() @@ -4488,6 +5321,7 @@ mod tests { // We commit by doing a delete("false") after replacing the cached indices. let metadata_key = crate::session::index_caches::IndexMetadataKey { version: dataset.version().version, + store_identity: &dataset.object_store.store_prefix, }; dataset .index_cache @@ -4715,6 +5549,83 @@ mod tests { ); } + #[rstest] + #[case::block_size_128(128)] + #[case::block_size_256(256)] + #[tokio::test] + async fn test_optimize_empty_code_fts_index_preserves_params(#[case] block_size: usize) { + let dir = TempStrDir::default(); + let schema = Arc::new(Schema::new(vec![ + Field::new("id", DataType::Int32, false), + Field::new("code", DataType::Utf8, false), + ])); + let batch = RecordBatch::try_new( + schema.clone(), + vec![ + Arc::new(Int32Array::from_iter_values([0, 1])), + Arc::new(StringArray::from_iter_values([ + "fn GetUserEmail() {}", + "fn ParseConfig() {}", + ])), + ], + ) + .unwrap(); + let reader = RecordBatchIterator::new(vec![Ok(batch)], schema); + let mut dataset = Dataset::write(reader, &dir, None).await.unwrap(); + + let params = InvertedIndexParams::default() + .analyzer("code") + .unwrap() + .block_size(block_size) + .unwrap() + .split_identifiers(true) + .preserve_original(false); + dataset + .create_index_builder(&["code"], IndexType::Inverted, ¶ms) + .name("code_idx".to_string()) + .train(false) + .await + .unwrap(); + + let indices = dataset.load_indices().await.unwrap(); + assert_eq!(indices.len(), 1); + assert_eq!(indices[0].index_version, INVERTED_INDEX_VERSION_V3 as i32); + + dataset.optimize_indices(&Default::default()).await.unwrap(); + + let indices = dataset.load_indices().await.unwrap(); + assert_eq!(indices.len(), 1); + assert_eq!(indices[0].index_version, INVERTED_INDEX_VERSION_V3 as i32); + let details = indices[0] + .index_details + .as_ref() + .expect("optimized FTS index should retain index details") + .to_msg::() + .unwrap(); + assert_eq!(details.base_tokenizer.as_deref(), Some("code")); + assert_eq!(details.block_size, Some(block_size as u32)); + let code_config = details + .code_config + .expect("optimized code FTS index should retain code configuration"); + assert!(code_config.split_identifiers); + assert_eq!(code_config.preserve_original, Some(false)); + + let stats: serde_json::Value = + serde_json::from_str(&dataset.index_statistics("code_idx").await.unwrap()).unwrap(); + assert_eq!(stats["num_unindexed_rows"], 0); + + let result = dataset + .scan() + .project(&["id"]) + .unwrap() + .full_text_search(FullTextSearchQuery::new("email".to_string())) + .unwrap() + .try_into_batch() + .await + .unwrap(); + assert_eq!(result["id"].as_primitive::().values(), &[0]); + } + /// Helper function to check if an index is being used in a query plan fn assert_index_usage(plan: &str, column_name: &str, should_use_index: bool, context: &str) { let index_used = if column_name == "text" { @@ -5857,6 +6768,68 @@ mod tests { ); } + #[rstest] + #[case::simple("value", "data.value")] + #[case::quoted("value.with.dot", "data.`value.with.dot`")] + #[tokio::test] + async fn test_initialize_index_on_nested_field( + #[case] nested_field_name: &str, + #[case] nested_field_path: &str, + ) { + let nested_field = Arc::new(Field::new(nested_field_name, DataType::Int32, false)); + let schema = Arc::new(Schema::new(vec![Field::new( + "data", + DataType::Struct(vec![nested_field.clone()].into()), + false, + )])); + let nested_values = arrow_array::StructArray::from(vec![( + nested_field, + Arc::new(Int32Array::from_iter_values(0..10)) as Arc, + )]); + let batch = RecordBatch::try_new(schema.clone(), vec![Arc::new(nested_values)]).unwrap(); + + let test_dir = TempStrDir::default(); + let source_uri = format!("{}/source", test_dir); + let target_uri = format!("{}/target", test_dir); + + let source_reader = + RecordBatchIterator::new(vec![batch.clone()].into_iter().map(Ok), schema.clone()); + let mut source_dataset = Dataset::write(source_reader, &source_uri, None) + .await + .unwrap(); + source_dataset + .create_index( + &[nested_field_path], + IndexType::BTree, + Some("nested_idx".to_string()), + &ScalarIndexParams::default(), + false, + ) + .await + .unwrap(); + let source_dataset = Dataset::open(&source_uri).await.unwrap(); + + let target_reader = + RecordBatchIterator::new(vec![batch].into_iter().map(Ok), schema.clone()); + let mut target_dataset = Dataset::write(target_reader, &target_uri, None) + .await + .unwrap(); + target_dataset + .initialize_index(&source_dataset, "nested_idx") + .await + .unwrap(); + + let target_indices = target_dataset.load_indices().await.unwrap(); + assert_eq!(target_indices.len(), 1); + assert_eq!( + target_dataset + .schema() + .field_path(target_indices[0].fields[0]) + .unwrap(), + nested_field_path + ); + } + #[tokio::test] async fn test_vector_index_on_nested_field_with_dots() { let dimensions = 16; @@ -6736,6 +7709,83 @@ mod tests { assert!(err.to_string().contains("at least one index segment")); } + #[tokio::test] + async fn test_commit_existing_index_segments_rejects_wrong_field_provenance() { + use lance_datagen::{BatchCount, RowCount, array}; + + let test_dir = tempfile::tempdir().unwrap(); + let reader = lance_datagen::gen_batch() + .col("id", array::step::()) + .col( + "vector", + array::rand_vec::(8.into()), + ) + .into_reader_rows(RowCount::from(10), BatchCount::from(1)); + let mut dataset = Dataset::write(reader, test_dir.path().to_str().unwrap(), None) + .await + .unwrap(); + + let vector_field_id = dataset.schema().field("vector").unwrap().id; + let mut metadata = write_vector_segment_metadata( + &dataset, + "vector_idx", + vector_field_id, + Uuid::new_v4(), + [0_u32], + b"segment", + ) + .await; + metadata.fields = vec![dataset.schema().field("id").unwrap().id]; + + let error = dataset + .commit_existing_index_segments( + "vector_idx", + "vector", + vec![segment_from_metadata(&metadata)], + ) + .await + .unwrap_err(); + assert!(error.to_string().contains("was built for fields")); + } + + #[tokio::test] + async fn test_commit_existing_index_segments_rejects_future_dataset_version() { + use lance_datagen::{BatchCount, RowCount, array}; + + let test_dir = tempfile::tempdir().unwrap(); + let reader = lance_datagen::gen_batch() + .col( + "vector", + array::rand_vec::(8.into()), + ) + .into_reader_rows(RowCount::from(10), BatchCount::from(1)); + let mut dataset = Dataset::write(reader, test_dir.path().to_str().unwrap(), None) + .await + .unwrap(); + + let field_id = dataset.schema().field("vector").unwrap().id; + let mut metadata = write_vector_segment_metadata( + &dataset, + "vector_idx", + field_id, + Uuid::new_v4(), + [0_u32], + b"segment", + ) + .await; + metadata.dataset_version = dataset.manifest.version + 1; + + let error = dataset + .commit_existing_index_segments( + "vector_idx", + "vector", + vec![segment_from_metadata(&metadata)], + ) + .await + .unwrap_err(); + assert!(error.to_string().contains("future dataset version")); + } + #[tokio::test] async fn test_commit_existing_index_segments_rejects_overlapping_fragment_coverage() { use lance_datagen::{BatchCount, RowCount, array}; @@ -6896,6 +7946,255 @@ mod tests { assert!(err.to_string().contains("would orphan fragments")); } + #[tokio::test] + async fn test_commit_existing_index_segments_replaces_different_index_type() { + use lance_datagen::{BatchCount, RowCount, array}; + + let test_dir = tempfile::tempdir().unwrap(); + let reader = lance_datagen::gen_batch() + .col("id", array::step::()) + .into_reader_rows(RowCount::from(10), BatchCount::from(1)); + let mut dataset = Dataset::write(reader, test_dir.path().to_str().unwrap(), None) + .await + .unwrap(); + + let index_name = "shared_name"; + let btree_params = ScalarIndexParams::for_builtin(BuiltinIndexType::BTree); + let original = dataset + .create_index_builder(&["id"], IndexType::BTree, &btree_params) + .name("staged_btree".to_string()) + .execute_uncommitted() + .await + .unwrap(); + dataset + .commit_existing_index_segments(index_name, "id", vec![original]) + .await + .unwrap(); + + let bitmap_params = ScalarIndexParams::for_builtin(BuiltinIndexType::Bitmap); + let replacement = dataset + .create_index_builder(&["id"], IndexType::Bitmap, &bitmap_params) + .name("staged_bitmap".to_string()) + .execute_uncommitted() + .await + .unwrap(); + let replacement_uuid = replacement.uuid; + let replacement_type_url = replacement.index_details.as_ref().unwrap().type_url.clone(); + + dataset + .commit_existing_index_segments(index_name, "id", vec![replacement]) + .await + .unwrap(); + + let committed = dataset + .load_index_by_name(index_name) + .await + .unwrap() + .unwrap(); + assert_eq!(committed.uuid, replacement_uuid); + assert_eq!( + committed.index_details.unwrap().type_url, + replacement_type_url + ); + } + + #[tokio::test] + async fn test_commit_existing_index_segments_rejects_partial_index_type_change() { + use lance_datagen::{BatchCount, RowCount, array}; + + let test_dir = tempfile::tempdir().unwrap(); + let reader = lance_datagen::gen_batch() + .col("id", array::step::()) + .into_reader_rows(RowCount::from(20), BatchCount::from(2)); + let mut dataset = Dataset::write( + reader, + test_dir.path().to_str().unwrap(), + Some(WriteParams { + max_rows_per_file: 20, + max_rows_per_group: 20, + ..Default::default() + }), + ) + .await + .unwrap(); + + let index_name = "shared_name"; + let fragments = dataset.get_fragments(); + assert_eq!(fragments.len(), 2); + let btree_params = ScalarIndexParams::for_builtin(BuiltinIndexType::BTree); + let mut original_segments = Vec::with_capacity(fragments.len()); + for fragment in &fragments { + original_segments.push( + dataset + .create_index_builder(&["id"], IndexType::BTree, &btree_params) + .fragments(vec![fragment.id() as u32]) + .execute_uncommitted() + .await + .unwrap(), + ); + } + dataset + .commit_existing_index_segments(index_name, "id", original_segments) + .await + .unwrap(); + let committed_version = dataset.manifest.version; + + let bitmap_params = ScalarIndexParams::for_builtin(BuiltinIndexType::Bitmap); + let replacement = dataset + .create_index_builder(&["id"], IndexType::Bitmap, &bitmap_params) + .fragments(vec![fragments[0].id() as u32]) + .execute_uncommitted() + .await + .unwrap(); + let error = dataset + .commit_existing_index_segments(index_name, "id", vec![replacement]) + .await + .unwrap_err(); + + assert!( + matches!(error, Error::InvalidInput { .. }), + "expected invalid input error, got: {error}" + ); + assert!( + error + .to_string() + .contains("cannot change index 'shared_name'") + ); + assert!(error.to_string().contains("missing current fragments [1]")); + assert_eq!(dataset.manifest.version, committed_version); + + let committed = + crate::index::scalar_logical::load_named_scalar_segments(&dataset, "id", index_name) + .await + .unwrap(); + assert_eq!(committed.len(), 2); + assert!(committed.iter().all(|segment| { + segment + .index_details + .as_ref() + .is_some_and(|details| details.type_url.ends_with("BTreeIndexDetails")) + })); + } + + #[tokio::test] + async fn test_partial_type_change_with_legacy_missing_details_is_rejected() { + use lance_datagen::{BatchCount, RowCount, array}; + + let test_dir = tempfile::tempdir().unwrap(); + let test_uri = test_dir.path().to_str().unwrap(); + let reader = lance_datagen::gen_batch() + .col("id", array::step::()) + .into_reader_rows(RowCount::from(10), BatchCount::from(1)); + let mut dataset = Dataset::write(reader, test_uri, None).await.unwrap(); + + let index_name = "shared_name"; + let btree_params = ScalarIndexParams::for_builtin(BuiltinIndexType::BTree); + let original = dataset + .create_index_builder(&["id"], IndexType::BTree, &btree_params) + .execute_uncommitted() + .await + .unwrap(); + dataset + .commit_existing_index_segments(index_name, "id", vec![original]) + .await + .unwrap(); + + let current = dataset.load_indices_by_name(index_name).await.unwrap(); + assert_eq!(current.len(), 1); + let original_uuid = current[0].uuid; + let mut legacy = current.clone(); + legacy[0].index_details = None; + legacy[0].index_version = 0; + let transaction = Transaction::new( + dataset.manifest.version, + Operation::CreateIndex { + new_indices: legacy, + removed_indices: current, + }, + None, + ); + dataset + .apply_commit(transaction, &Default::default(), &Default::default()) + .await + .unwrap(); + + let append_reader = lance_datagen::gen_batch() + .col("id", array::step::()) + .into_reader_rows(RowCount::from(10), BatchCount::from(1)); + let mut dataset = Dataset::write( + append_reader, + test_uri, + Some(WriteParams { + mode: WriteMode::Append, + ..Default::default() + }), + ) + .await + .unwrap(); + let fragments = dataset.get_fragments(); + assert_eq!(fragments.len(), 2); + + let bitmap_params = ScalarIndexParams::for_builtin(BuiltinIndexType::Bitmap); + let replacement = dataset + .create_index_builder(&["id"], IndexType::Bitmap, &bitmap_params) + .fragments(vec![fragments[1].id() as u32]) + .execute_uncommitted() + .await + .unwrap(); + let version_before = dataset.manifest.version; + let error = dataset + .commit_existing_index_segments(index_name, "id", vec![replacement]) + .await + .unwrap_err(); + + assert!( + matches!(error, Error::InvalidInput { .. }), + "expected invalid input error, got: {error}" + ); + assert!(error.to_string().contains("from type ''")); + assert!(error.to_string().contains("partial fragment coverage")); + assert!(error.to_string().contains("missing current fragments [0]")); + assert_eq!(dataset.manifest.version, version_before); + + let committed = dataset.load_indices_by_name(index_name).await.unwrap(); + assert_eq!(committed.len(), 1); + assert_eq!(committed[0].uuid, original_uuid); + assert!(committed[0].index_details.is_none()); + assert_eq!(committed[0].index_version, 0); + + let field_id = dataset.schema().field("id").unwrap().id; + let sentinel_segment = IndexSegment::new( + Uuid::new_v4(), + [fragments[1].id() as u32], + [field_id], + Arc::new(prost_types::Any { + type_url: "".to_string(), + value: Vec::new(), + }), + 0, + dataset.manifest.version, + ); + let error = dataset + .commit_existing_index_segments(index_name, "id", vec![sentinel_segment]) + .await + .unwrap_err(); + + assert!( + matches!(error, Error::InvalidInput { .. }), + "expected invalid input error, got: {error}" + ); + assert!(error.to_string().contains("from type ''")); + assert!(error.to_string().contains("to type ''")); + assert!(error.to_string().contains("missing current fragments [0]")); + assert_eq!(dataset.manifest.version, version_before); + + let committed = dataset.load_indices_by_name(index_name).await.unwrap(); + assert_eq!(committed.len(), 1); + assert_eq!(committed[0].uuid, original_uuid); + assert!(committed[0].index_details.is_none()); + assert_eq!(committed[0].index_version, 0); + } + #[tokio::test] async fn test_commit_existing_index_segments_removes_empty_segment() { use lance_datagen::{BatchCount, RowCount, array}; @@ -8091,4 +9390,91 @@ mod tests { ); } } + + #[tokio::test] + async fn test_optimize_rebuilds_dormant_vector_index_instead_of_merging_stale_rows() { + use crate::dataset::UpdateBuilder; + + let mut dataset = gen_batch() + .col("vec", array::rand_vec::(Dimension::from(4))) + .into_ram_dataset_with_params( + FragmentCount::from(2), + FragmentRowCount::from(3), + Some(WriteParams { + max_rows_per_file: 3, + enable_stable_row_ids: true, + ..Default::default() + }), + ) + .await + .unwrap(); + let original = dataset + .scan() + .project(&["vec"]) + .unwrap() + .limit(Some(1), None) + .unwrap() + .try_into_batch() + .await + .unwrap(); + let query = original["vec"].as_fixed_size_list().value(0); + + dataset + .create_index( + &["vec"], + IndexType::Vector, + Some("vec_idx".to_string()), + &VectorIndexParams::ivf_flat(1, MetricType::L2), + true, + ) + .await + .unwrap(); + + // A full rewrite replaces every covered fragment; the definition is + // retained as a dormant segment with empty effective coverage. + let mut updated = UpdateBuilder::new(Arc::new(dataset)) + .set("vec", "array[100.0, 100.0, 100.0, 100.0]") + .unwrap() + .build() + .unwrap() + .execute() + .await + .unwrap() + .new_dataset + .as_ref() + .clone(); + let retained = updated.load_indices_by_name("vec_idx").await.unwrap(); + assert_eq!(retained.len(), 1); + assert!( + retained[0] + .effective_fragment_bitmap(&updated.fragment_bitmap) + .unwrap() + .is_empty() + ); + + // Optimize must rebuild from live data and replace the dormant + // segment, not merge its stale postings back in. + updated.optimize_indices(&Default::default()).await.unwrap(); + let rebuilt = updated.load_indices_by_name("vec_idx").await.unwrap(); + assert_eq!(rebuilt.len(), 1); + assert!( + !rebuilt[0] + .effective_fragment_bitmap(&updated.fragment_bitmap) + .unwrap() + .is_empty() + ); + + let result = updated + .scan() + .nearest("vec", query.as_ref(), 1) + .unwrap() + .try_into_batch() + .await + .unwrap(); + let distance = result["_distance"].as_primitive::().value(0); + assert!( + distance > 1_000.0, + "nearest distance {distance} should reflect the rewritten vectors, not stale postings" + ); + } } diff --git a/rust/lance/src/index/api.rs b/rust/lance/src/index/api.rs index f856e9004f3..a6314829072 100644 --- a/rust/lance/src/index/api.rs +++ b/rust/lance/src/index/api.rs @@ -5,7 +5,9 @@ use std::sync::Arc; use async_trait::async_trait; use datafusion::execution::SendableRecordBatchStream; -use lance_index::{IndexParams, IndexType, PrewarmOptions, optimize::OptimizeOptions}; +use lance_index::{ + FtsPrewarmResult, IndexParams, IndexType, PrewarmOptions, optimize::OptimizeOptions, +}; use lance_table::format::IndexMetadata; use roaring::RoaringBitmap; use uuid::Uuid; @@ -15,37 +17,45 @@ use crate::{Error, Result}; /// A single physical segment of a logical index. /// /// Each segment is stored independently and will become one manifest entry when committed. -/// The logical index identity (name / target column / dataset version) is provided separately -/// by the commit API. +/// The logical index name is provided separately by the commit API, while physical field and +/// dataset-version provenance travel with the segment. #[derive(Debug, Clone, PartialEq)] pub struct IndexSegment { /// Unique ID of the physical segment. uuid: Uuid, /// The fragments covered by this segment. fragment_bitmap: RoaringBitmap, + /// Field IDs whose physical values are encoded in this segment. + fields: Vec, /// Metadata specific to the index type. index_details: Arc, /// The on-disk index version for this segment. index_version: i32, + /// Dataset version at which this segment's physical contents were built. + dataset_version: u64, } impl IndexSegment { - /// Create a fully described segment with the given UUID, fragment coverage, and index - /// metadata. - pub fn new( + /// Create a fully described segment with its physical build provenance. + pub fn new( uuid: Uuid, fragment_bitmap: I, + fields: F, index_details: Arc, index_version: i32, + dataset_version: u64, ) -> Self where I: IntoIterator, + F: IntoIterator, { Self { uuid, fragment_bitmap: fragment_bitmap.into_iter().collect(), + fields: fields.into_iter().collect(), index_details, index_version, + dataset_version, } } @@ -59,6 +69,15 @@ impl IndexSegment { &self.fragment_bitmap } + pub(crate) fn fragment_bitmap_mut(&mut self) -> &mut RoaringBitmap { + &mut self.fragment_bitmap + } + + /// Return the field IDs whose values are encoded in this segment. + pub fn fields(&self) -> &[i32] { + &self.fields + } + /// Return the serialized index details for this segment. pub fn index_details(&self) -> &Arc { &self.index_details @@ -69,13 +88,29 @@ impl IndexSegment { self.index_version } + /// Return the source dataset version for this segment. + pub fn dataset_version(&self) -> u64 { + self.dataset_version + } + /// Consume the segment and return its component parts. - pub fn into_parts(self) -> (Uuid, RoaringBitmap, Arc, i32) { + pub fn into_parts( + self, + ) -> ( + Uuid, + RoaringBitmap, + Vec, + Arc, + i32, + u64, + ) { ( self.uuid, self.fragment_bitmap, + self.fields, self.index_details, self.index_version, + self.dataset_version, ) } } @@ -110,8 +145,10 @@ impl IntoIndexSegment for IndexMetadata { Ok(IndexSegment::new( self.uuid, fragment_bitmap.iter(), + self.fields, index_details, self.index_version, + self.dataset_version, )) } } @@ -167,6 +204,48 @@ pub trait DatasetIndexExt { )) } + /// Prewarm an index by name with additional options and return the structured outcome. + async fn prewarm_index_with_options_result( + &self, + _name: &str, + _options: &PrewarmOptions, + ) -> Result { + Err(Error::not_supported( + "prewarm result reports are not supported by this dataset implementation".to_owned(), + )) + } + + /// Prewarm selected physical segments of an index by name. + async fn prewarm_index_segments(&self, _name: &str, _segment_ids: &[Uuid]) -> Result<()> { + Err(Error::not_supported( + "segment-level prewarm is not supported by this dataset implementation".to_owned(), + )) + } + + /// Prewarm selected physical segments of an index by name with additional options. + async fn prewarm_index_segments_with_options( + &self, + _name: &str, + _segment_ids: &[Uuid], + _options: &PrewarmOptions, + ) -> Result<()> { + Err(Error::not_supported( + "prewarm options are not supported by this dataset implementation".to_owned(), + )) + } + + /// Prewarm selected physical segments with options and return the structured outcome. + async fn prewarm_index_segments_with_options_result( + &self, + _name: &str, + _segment_ids: &[Uuid], + _options: &PrewarmOptions, + ) -> Result { + Err(Error::not_supported( + "prewarm result reports are not supported by this dataset implementation".to_owned(), + )) + } + /// Read all indices of this Dataset version. /// /// The indices are lazy loaded and cached in memory within the `Dataset` instance. @@ -254,3 +333,31 @@ pub trait DatasetIndexExt { with_vector: bool, ) -> Result; } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_index_metadata_conversion_preserves_provenance() { + let metadata = IndexMetadata { + uuid: Uuid::new_v4(), + name: "test".to_string(), + fields: vec![3, 7], + dataset_version: 42, + fragment_bitmap: Some(RoaringBitmap::from_iter([1, 2])), + index_details: Some(Arc::new(prost_types::Any { + type_url: "example.IndexDetails".to_string(), + value: vec![1, 2, 3], + })), + index_version: 5, + created_at: None, + base_id: None, + files: None, + }; + + let segment = metadata.into_index_segment().unwrap(); + assert_eq!(segment.fields(), [3, 7]); + assert_eq!(segment.dataset_version(), 42); + } +} diff --git a/rust/lance/src/index/append.rs b/rust/lance/src/index/append.rs index d3ecde030c1..6a85760bdba 100644 --- a/rust/lance/src/index/append.rs +++ b/rust/lance/src/index/append.rs @@ -5,28 +5,41 @@ use std::sync::Arc; use futures::{FutureExt, TryStreamExt}; use lance_core::{Error, Result}; +use lance_file::reader::FileReaderOptions; use lance_index::{ INDEX_FILE_NAME, IndexType, + frag_reuse::FragReuseIndex, metrics::NoOpMetricsCollector, optimize::OptimizeOptions, - progress::NoopIndexBuildProgress, + progress::{IndexBuildProgress, NoopIndexBuildProgress}, scalar::{ - CreatedIndex, OldIndexDataFilter, ScalarIndex, inverted::InvertedIndex, + CreatedIndex, OldIndexDataFilter, ScalarIndex, index_files_to_table, + inverted::InvertedIndex, lance_format::LanceIndexStore, + seed::{FragmentSeed, SEED_META_KEY_PREFIX}, + table_files_to_index, }, }; +use lance_io::scheduler::{ScanScheduler, SchedulerConfig}; +use lance_io::utils::CachedFileSize; use lance_select::{RowAddrTreeMap, RowSetOps}; use lance_table::format::{Fragment, IndexMetadata}; +use prost::Message; use roaring::RoaringBitmap; use uuid::Uuid; -use super::DatasetIndexInternalExt; -use super::vector::LogicalVectorIndex; -use super::vector::ivf::{optimize_vector_indices, select_segment_for_single_rebalance}; +use super::vector::ivf::{ + VectorSegmentCompatibility, index_type_for_segmented_optimize, optimize_vector_indices, + select_segment_for_single_rebalance, vector_segment_compatibility, +}; +use super::vector::{LogicalVectorIndex, fresh_vector_segment_params}; +use super::{CreateIndexBuilder, DatasetIndexInternalExt}; use crate::dataset::Dataset; use crate::dataset::index::LanceIndexStoreExt; use crate::dataset::rowids::load_row_id_sequences; -use crate::index::scalar::load_training_data; +use crate::index::scalar::{ + IndexDetails, fetch_index_details, load_fts_training_data, load_training_data, +}; use crate::index::vector_index_details_default; #[derive(Debug, Clone)] @@ -34,6 +47,7 @@ pub struct IndexMergeResults<'a> { pub new_uuid: Uuid, pub removed_indices: Vec<&'a IndexMetadata>, pub new_fragment_bitmap: RoaringBitmap, + pub new_dataset_version: u64, pub new_index_version: i32, pub new_index_details: prost_types::Any, /// List of files and their sizes for the merged index @@ -147,13 +161,65 @@ pub fn split_segment_coverage<'a>( (effective, deleted) } -/// Build one [`OldIndexDataFilter`] per segment, each derived from that segment's -/// *own* effective (still-live) and retired fragment coverage, plus the union of -/// every segment's still-live coverage. +pub fn fragment_reuse_affects_segments<'a>( + frag_reuse_index: &FragReuseIndex, + segments: impl IntoIterator, +) -> bool { + segments.into_iter().any(|segment| { + let Some(coverage) = segment.fragment_bitmap.as_ref() else { + return false; + }; + fragment_reuse_affects_segment(frag_reuse_index, coverage, segment.dataset_version) + }) +} + +pub fn fragment_reuse_affects_segment( + frag_reuse_index: &FragReuseIndex, + coverage: &RoaringBitmap, + dataset_version: u64, +) -> bool { + frag_reuse_index.details.versions.iter().any(|version| { + version.groups.iter().any(|group| { + if group.changed_row_addrs.is_empty() { + return false; + } + let covers_old = group + .old_frags + .iter() + .any(|fragment| coverage.contains(fragment.id as u32)); + let covers_new = group + .new_frags + .iter() + .any(|fragment| coverage.contains(fragment.id as u32)); + (version.dataset_version >= dataset_version && covers_old) + || (version.dataset_version > dataset_version && covers_new) + }) + }) +} + +/// Build one [`OldIndexDataFilter`] per segment and return their effective coverage. pub async fn build_per_segment_filters( dataset: &Dataset, segments: &[&IndexMetadata], ) -> Result<(RoaringBitmap, Vec>)> { + if dataset.manifest.uses_stable_row_ids() { + let mut effective_union = RoaringBitmap::new(); + let mut filters = Vec::with_capacity(segments.len()); + for segment in segments { + let effective = segment + .effective_fragment_bitmap(&dataset.fragment_bitmap) + .ok_or_else(|| { + Error::invalid_input(format!( + "CreateIndex: segment {} is missing fragment coverage", + segment.uuid + )) + })?; + effective_union |= &effective; + filters.push(build_old_data_filter(dataset, &effective, &RoaringBitmap::new()).await?); + } + return Ok((effective_union, filters)); + } + let mut effective_union = RoaringBitmap::new(); let mut filters = Vec::with_capacity(segments.len()); for segment in segments { @@ -175,6 +241,88 @@ pub async fn build_per_segment_filters( Ok((effective_union, filters)) } +/// Attempt to read seed buffers for `column_name` from `fragments`' data files. +/// +/// Returns `Some(vec)` only if every fragment has a seed entry; returns `None` +/// if any fragment is missing a seed or its data file cannot be opened. +/// Index-type-specific validation (e.g. `rows_per_zone` checks) is left to the +/// caller via [`FragmentSeed::metadata_value`]. +async fn try_harvest_seeds( + dataset: &Dataset, + fragments: &[Fragment], + column_name: &str, +) -> Result>> { + if fragments.is_empty() { + return Ok(Some(Vec::new())); + } + + let meta_key = format!("{}{}", SEED_META_KEY_PREFIX, column_name); + let mut seeds = Vec::with_capacity(fragments.len()); + + let scheduler = ScanScheduler::new( + dataset.object_store.clone(), + SchedulerConfig::max_bandwidth(&dataset.object_store), + ); + + for fragment in fragments { + let Some(data_file) = fragment.files.first() else { + return Ok(None); + }; + + let path = dataset + .base + .clone() + .join(crate::dataset::DATA_DIR) + .join(data_file.path.as_str()); + let Ok(file_scheduler) = scheduler.open_file(&path, &CachedFileSize::unknown()).await + else { + return Ok(None); + }; + + let Ok(reader) = lance_file::reader::FileReader::try_open( + file_scheduler, + None, + Default::default(), + &dataset.metadata_cache.file_metadata_cache(&path), + FileReaderOptions::default(), + ) + .await + else { + return Ok(None); + }; + + let Some(meta_value) = reader + .metadata() + .file_schema + .metadata + .get(&meta_key) + .cloned() + else { + return Ok(None); + }; + + // The buf_index is always the portion before the first ':'. + let Some(buf_index_str) = meta_value.split(':').next() else { + return Ok(None); + }; + let Ok(buf_index) = buf_index_str.parse::() else { + return Ok(None); + }; + + let Ok(bytes) = reader.read_global_buffer(buf_index).await else { + return Ok(None); + }; + + seeds.push(FragmentSeed { + fragment_id: fragment.id, + bytes, + metadata_value: meta_value, + }); + } + + Ok(Some(seeds)) +} + async fn load_unindexed_training_data( dataset: &Dataset, field_path: &str, @@ -293,7 +441,15 @@ async fn merge_scalar_indices<'a>( field_path: &str, column_name: &str, base_unindexed_bitmap: RoaringBitmap, -) -> Result, RoaringBitmap, CreatedIndex)>> { +) -> Result< + Option<( + Uuid, + Vec<&'a IndexMetadata>, + RoaringBitmap, + CreatedIndex, + u64, + )>, +> { if old_indices.is_empty() { return Err(Error::index( "merge_scalar_indices: no previous index found".to_string(), @@ -328,7 +484,12 @@ async fn merge_scalar_indices<'a>( // Scalar Index that expos an N:1 segment-merge primitive reachable without // rescanning the dataset - let has_segment_merge_primitive = matches!(index_type, IndexType::BTree); + let has_segment_merge_primitive = matches!(index_type, IndexType::BTree | IndexType::NGram); + let frag_reuse_index = dataset.open_frag_reuse_index(&NoOpMetricsCollector).await?; + let ngram_requires_rebuild = index_type == IndexType::NGram + && frag_reuse_index.as_ref().is_some_and(|frag_reuse_index| { + fragment_reuse_affects_segments(frag_reuse_index, selected_old_indices.iter().copied()) + }); // Merge new data into the existing segment(s) without rebuilding from // scratch, when all hold: @@ -343,53 +504,104 @@ async fn merge_scalar_indices<'a>( // primitive) the index is rebuilt from scratch over `frag_bitmap`. let can_merge_segments = !effective_old_frags.is_empty() && !update_criteria.requires_old_data + && !ngram_requires_rebuild && (has_segment_merge_primitive || selected_old_indices.len() == 1); - let created_index = if !can_merge_segments { - rebuild_scalar_segment( - dataset.as_ref(), - &reference_index, - field_path, - column_name, - new_uuid, - frag_bitmap.iter().collect(), + let (created_index, new_dataset_version) = if !can_merge_segments { + ( + rebuild_scalar_segment( + dataset.as_ref(), + &reference_index, + field_path, + column_name, + new_uuid, + frag_bitmap.iter().collect(), + ) + .await?, + dataset.manifest.version, ) - .await? } else { - let new_data_stream = - load_unindexed_training_data(dataset.as_ref(), field_path, &update_criteria, unindexed) - .await?; let new_store = LanceIndexStore::from_dataset_for_new(&dataset, &new_uuid)?; - match index_type { - IndexType::BTree => { - let (_, old_data_filters) = - build_per_segment_filters(dataset.as_ref(), &selected_old_indices).await?; - crate::index::scalar::btree::open_and_merge_segments( - dataset.as_ref(), - field_path, - &selected_old_indices, - new_data_stream, - &new_store, - &old_data_filters, - ) - .await? + // Try a seed-based update before falling back to a full column scan. + // Seeds are only available if every unindexed fragment had a seed buffer + // written during its data file write, and the plugin validates that the + // seeds are compatible with the current index configuration. + let index_details = + fetch_index_details(dataset.as_ref(), field_path, reference_idx).await?; + let details = IndexDetails(index_details.clone()); + let plugin = details.get_plugin()?; + // Only open data files looking for seeds when the plugin confirms this + // index type and configuration can actually produce them. + let maybe_created = if plugin.might_use_seeds(&index_details) { + if let Some(seeds) = try_harvest_seeds(dataset.as_ref(), unindexed, column_name).await? + { + plugin + .update_from_seeds(seeds, reference_index.clone(), &index_details, &new_store) + .await? + } else { + None } - // NOTE: IndexType::Inverted never reaches here -- it is handled by the - // dedicated arm in merge_indices_with_unindexed_frags before this - // function is called. - _ => { - let old_data_filter = build_old_data_filter( - dataset.as_ref(), - &effective_old_frags, - &deleted_old_frags, - ) - .await?; - reference_index - .update(new_data_stream, &new_store, old_data_filter) + } else { + None + }; + + let created_index = if let Some(created) = maybe_created { + created + } else { + let new_data_stream = load_unindexed_training_data( + dataset.as_ref(), + field_path, + &update_criteria, + unindexed, + ) + .await?; + + match index_type { + IndexType::BTree => { + let (_, old_data_filters) = + build_per_segment_filters(dataset.as_ref(), &selected_old_indices).await?; + crate::index::scalar::btree::open_and_merge_segments( + dataset.as_ref(), + field_path, + &selected_old_indices, + new_data_stream, + &new_store, + &old_data_filters, + ) + .await? + } + IndexType::NGram => { + let (_, old_data_filters) = + build_per_segment_filters(dataset.as_ref(), &selected_old_indices).await?; + crate::index::scalar::ngram::open_and_merge_segments( + dataset.as_ref(), + &selected_old_indices, + Some(new_data_stream), + &new_store, + &old_data_filters, + ) .await? + } + _ => { + let old_data_filter = build_old_data_filter( + dataset.as_ref(), + &effective_old_frags, + &deleted_old_frags, + ) + .await?; + reference_index + .update(new_data_stream, &new_store, old_data_filter) + .await? + } } - } + }; + let source_dataset_version = selected_old_indices + .iter() + .map(|index| index.dataset_version) + .min() + .unwrap_or(dataset.manifest.version); + (created_index, source_dataset_version) }; Ok(Some(( @@ -397,6 +609,7 @@ async fn merge_scalar_indices<'a>( selected_old_indices.to_vec(), frag_bitmap, created_index, + new_dataset_version, ))) } @@ -445,6 +658,87 @@ pub async fn merge_indices<'a>( .await } +async fn build_fresh_vector_segment( + dataset: &Dataset, + logical_index: &LogicalVectorIndex, + field_path: &str, + fragment_bitmap: &RoaringBitmap, + progress: Arc, +) -> Result { + let (reference_metadata, reference_index) = logical_index.iter().last().ok_or_else(|| { + Error::index(format!( + "Optimize vector index: logical index '{}' has no physical segments", + logical_index.name() + )) + })?; + let params = fresh_vector_segment_params(reference_metadata, reference_index.as_ref())?; + let mut build_dataset = dataset.clone(); + CreateIndexBuilder::new( + &mut build_dataset, + &[field_path], + IndexType::Vector, + ¶ms, + ) + .name(logical_index.name().to_string()) + .replace(true) + .fragments(fragment_bitmap.iter().collect()) + .progress(progress) + .execute_uncommitted() + .await +} + +async fn scan_vector_fragments( + dataset: &Dataset, + field_path: &str, + column_nullable: bool, + fragments: &[Fragment], +) -> Result { + let mut scanner = dataset.scan(); + scanner + .with_fragments(fragments.to_vec()) + .with_row_id() + .project(&[field_path])?; + if column_nullable { + let column_expr = lance_datafusion::logical_expr::field_path_to_expr(field_path)?; + scanner.filter_expr(column_expr.is_not_null()); + } + scanner.try_into_stream().await +} + +fn fresh_vector_segment_result<'a>( + segment: IndexMetadata, + expected_fragment_bitmap: &RoaringBitmap, + removed_indices: Vec<&'a IndexMetadata>, +) -> Result> { + let fragment_bitmap = segment.fragment_bitmap.ok_or_else(|| { + Error::index( + "Optimize vector index: newly built segment has no fragment bitmap".to_string(), + ) + })?; + if fragment_bitmap != *expected_fragment_bitmap { + return Err(Error::index(format!( + "Optimize vector index: newly built segment covers fragments {:?}, expected {:?}", + fragment_bitmap, expected_fragment_bitmap + ))); + } + let index_details = segment.index_details.ok_or_else(|| { + Error::index("Optimize vector index: newly built segment has no index details".to_string()) + })?; + let files = segment.files.ok_or_else(|| { + Error::index("Optimize vector index: newly built segment has no file metadata".to_string()) + })?; + + Ok(IndexMergeResults { + new_uuid: segment.uuid, + removed_indices, + new_fragment_bitmap: fragment_bitmap, + new_dataset_version: segment.dataset_version, + new_index_version: segment.index_version, + new_index_details: index_details.as_ref().clone(), + files, + }) +} + /// Merge a list of provided unindexed data, with a specific number of previous indices /// into a new index, to improve the query performance. pub async fn merge_indices_with_unindexed_frags<'a>( @@ -467,7 +761,33 @@ pub async fn merge_indices_with_unindexed_frags<'a>( old_indices[0].fields[0] )))?; - let field_path = dataset.schema().field_path(old_indices[0].fields[0])?; + let raw_field_path = dataset.schema().field_path(old_indices[0].fields[0])?; + let first_details = + super::scalar::fetch_index_details(dataset.as_ref(), &raw_field_path, old_indices[0]) + .await?; + let resolved_fts = if first_details.type_url.ends_with("InvertedIndexDetails") { + let details = + lance_index::pbold::InvertedIndexDetails::decode(first_details.value.as_slice()) + .map_err(|error| { + Error::io(format!( + "failed to decode InvertedIndexDetails payload: {error}" + )) + })?; + let granularity = lance_index::scalar::inverted::DocumentGranularity::try_from( + details.document_granularity, + )?; + Some(crate::index::scalar::inverted::resolve_fts_field_by_id( + dataset.schema(), + old_indices[0].fields[0], + granularity, + )?) + } else { + None + }; + let field_path = resolved_fts + .as_ref() + .map(|resolved| resolved.canonical_path.clone()) + .unwrap_or(raw_field_path); let first_is_vector_index = metadata_is_vector_index(dataset.as_ref(), old_indices[0]).await?; for idx in old_indices.iter().skip(1) { let is_vector_index = metadata_is_vector_index(dataset.as_ref(), idx).await?; @@ -484,61 +804,179 @@ pub async fn merge_indices_with_unindexed_frags<'a>( base_unindexed_bitmap.insert(frag.id as u32); }); - let (new_uuid, removed_indices, new_fragment_bitmap, created_index) = if first_is_vector_index { - let full_logical_index = dataset - .open_logical_vector_index(&field_path, &old_indices[0].name) - .await?; - let mut opened_indices_by_uuid = full_logical_index - .iter() - .map(|(metadata, index)| (metadata.uuid, (metadata.clone(), index.clone()))) - .collect::>(); - let mut selected_metadatas = Vec::with_capacity(old_indices.len()); - let mut selected_indices = Vec::with_capacity(old_indices.len()); - for metadata in old_indices { - let (selected_metadata, selected_index) = opened_indices_by_uuid.remove(&metadata.uuid).ok_or_else(|| { + let (new_uuid, removed_indices, new_fragment_bitmap, created_index, new_dataset_version) = + if first_is_vector_index { + // Segments with stored rows (raw coverage) but no live coverage + // (e.g. a definition retained through a full rewrite) are dormant. + // With stable row ids their stored postings still share ids with + // live rows, so merging them would resurrect stale vectors; they + // may only be replaced. A born-empty segment (deferred build) has + // no stored rows and stays mergeable. + let (live_segments, dormant_segments): (Vec<&IndexMetadata>, Vec<&IndexMetadata>) = + old_indices.iter().copied().partition(|idx| { + let has_stored_rows = idx + .fragment_bitmap + .as_ref() + .is_some_and(|bitmap| !bitmap.is_empty()); + let has_live_coverage = idx + .effective_fragment_bitmap(&dataset.fragment_bitmap) + .is_none_or(|bitmap| !bitmap.is_empty()); + !has_stored_rows || has_live_coverage + }); + if !dormant_segments.is_empty() && !live_segments.is_empty() && !options.retrain { + // Optimize the live segments as usual; the dormant segments + // are superseded by whatever that produces. + let mut merged = Box::pin(merge_indices_with_unindexed_frags( + dataset.clone(), + &live_segments, + unindexed, + options, + )) + .await?; + if let Some(results) = merged.as_mut() { + results.removed_indices.extend(dormant_segments); + } + return Ok(merged); + } + let rebuild_dormant = live_segments.is_empty() && !dormant_segments.is_empty(); + if rebuild_dormant && unindexed.is_empty() { + return Ok(None); + } + + let full_logical_index = dataset + .open_logical_vector_index(&field_path, &old_indices[0].name) + .await?; + let mut opened_indices_by_uuid = full_logical_index + .iter() + .map(|(metadata, index)| (metadata.uuid, (metadata.clone(), index.clone()))) + .collect::>(); + let mut selected_metadatas = Vec::with_capacity(old_indices.len()); + let mut selected_indices = Vec::with_capacity(old_indices.len()); + for metadata in old_indices { + let (selected_metadata, selected_index) = opened_indices_by_uuid.remove(&metadata.uuid).ok_or_else(|| { Error::index(format!( "Append index: logical vector index '{}' does not contain requested segment {}", old_indices[0].name, metadata.uuid )) })?; - selected_metadatas.push(selected_metadata); - selected_indices.push(selected_index); - } - let logical_index = LogicalVectorIndex::try_new( - old_indices[0].name.clone(), - field_path.clone(), - selected_metadatas - .into_iter() - .zip(selected_indices) - .collect(), - )?; - let ivf_view = logical_index.as_ivf()?; - - // Specialized vector no-op: when there is no new data and the caller - // hasn't asked for retrain or an explicit delta merge, the only useful - // work is rebalancing. Bail when no segment needs rebalancing so - // repeated optimize calls don't keep rewriting the same index. This - // matches the scalar gate in `Dataset::optimize_indices`, which also - // treats `OptimizeOptions::append()` (num_indices_to_merge=Some(0)) - // as "no explicit merge requested". - if unindexed.is_empty() - && !options.retrain - && options.num_indices_to_merge.is_none_or(|n| n == 0) - && select_segment_for_single_rebalance(&ivf_view)?.is_none() - { - return Ok(None); - } + selected_metadatas.push(selected_metadata); + selected_indices.push(selected_index); + } + let logical_index = LogicalVectorIndex::try_new( + old_indices[0].name.clone(), + field_path.clone(), + selected_metadatas + .into_iter() + .zip(selected_indices) + .collect(), + )?; - let use_single_segment_rebalance = logical_index.num_segments() > 1 - && options.num_indices_to_merge.is_none_or(|n| n == 0) - && !options.retrain - && unindexed.is_empty(); + if options.retrain || rebuild_dormant { + let fragment_bitmap = dataset.fragment_bitmap.as_ref().clone(); + let segment = build_fresh_vector_segment( + dataset.as_ref(), + &logical_index, + &field_path, + &fragment_bitmap, + options.progress.clone(), + ) + .await?; + return fresh_vector_segment_result( + segment, + &fragment_bitmap, + old_indices.to_vec(), + ) + .map(Some); + } + + let ivf_view = logical_index.as_ivf()?; + let compatibility = + vector_segment_compatibility(&ivf_view, "optimizing logical vector index")?; + let explicit_append = options.num_indices_to_merge == Some(0); + let default_append_for_heterogeneous_models = options.num_indices_to_merge.is_none() + && compatibility == VectorSegmentCompatibility::QueryCompatibleModelsDiffer; + + if !unindexed.is_empty() && (explicit_append || default_append_for_heterogeneous_models) + { + // CreateIndex commits append new segments at the end of the manifest's + // stable index order. Reusing the current suffix model keeps consecutive + // appends directly mergeable without requiring unrelated older segments + // to share that model. + let (reference_metadata, reference_index) = + logical_index.iter().last().ok_or_else(|| { + Error::index(format!( + "Optimize vector index: logical index '{}' has no physical segments", + logical_index.name() + )) + })?; + let reference_logical_index = LogicalVectorIndex::try_new( + logical_index.name().to_string(), + field_path.clone(), + vec![(reference_metadata.clone(), reference_index.clone())], + )?; + let reference_ivf_view = reference_logical_index.as_ivf()?; + let new_data_stream = scan_vector_fragments( + dataset.as_ref(), + &field_path, + column.nullable, + unindexed, + ) + .await?; + let mut append_options = options.clone(); + append_options.num_indices_to_merge = Some(0); + append_options.retrain = false; + let (new_uuid, indices_merged, files) = optimize_vector_indices( + dataset.as_ref().clone(), + Some(new_data_stream), + &field_path, + &reference_ivf_view, + &append_options, + ) + .boxed() + .await?; + if indices_merged != 0 { + return Err(Error::index(format!( + "Optimize vector index append unexpectedly merged {indices_merged} existing segments" + ))); + } + return Ok(Some(IndexMergeResults { + new_uuid, + removed_indices: Vec::new(), + new_fragment_bitmap: base_unindexed_bitmap, + new_dataset_version: dataset.manifest.version, + new_index_version: index_type_for_segmented_optimize(reference_index.as_ref())? + .version(), + new_index_details: reference_metadata + .index_details + .as_deref() + .cloned() + .unwrap_or_else(vector_index_details_default), + files, + })); + } - if use_single_segment_rebalance { - let Some(selected_segment_id) = select_segment_for_single_rebalance(&ivf_view)? else { + // Append is a steady-state no-op when every fragment is already + // indexed. Default optimize may still rebalance one segment. + if unindexed.is_empty() && options.num_indices_to_merge == Some(0) { return Ok(None); - }; - let removed_segment = old_indices + } + if unindexed.is_empty() + && options.num_indices_to_merge.is_none() + && select_segment_for_single_rebalance(&ivf_view)?.is_none() + { + return Ok(None); + } + + let use_single_segment_rebalance = logical_index.num_segments() > 1 + && options.num_indices_to_merge.is_none() + && unindexed.is_empty(); + + if use_single_segment_rebalance { + let Some(selected_segment_id) = select_segment_for_single_rebalance(&ivf_view)? + else { + return Ok(None); + }; + let removed_segment = old_indices .iter() .copied() .find(|metadata| metadata.uuid == selected_segment_id) @@ -548,7 +986,7 @@ pub async fn merge_indices_with_unindexed_frags<'a>( old_indices[0].name, selected_segment_id )) })?; - let (selected_metadata, selected_index) = logical_index + let (selected_metadata, selected_index) = logical_index .iter() .find(|(metadata, _)| metadata.uuid == selected_segment_id) .map(|(metadata, index)| (metadata.clone(), index.clone())) @@ -558,335 +996,1059 @@ pub async fn merge_indices_with_unindexed_frags<'a>( selected_segment_id, old_indices[0].name )) })?; - let selected_logical_index = LogicalVectorIndex::try_new( - old_indices[0].name.clone(), - field_path.clone(), - vec![(selected_metadata, selected_index)], - )?; - let selected_ivf_view = selected_logical_index.as_ivf()?; - let (new_uuid, indices_merged, files) = Box::pin(optimize_vector_indices( - dataset.as_ref().clone(), - Option::< - lance_io::stream::RecordBatchStreamAdapter< - futures::stream::Empty>, - >, - >::None, - &field_path, - &selected_ivf_view, - options, - )) - .await?; - if indices_merged == 0 { - return Ok(None); - } - - let new_fragment_bitmap = removed_segment - .effective_fragment_bitmap(&dataset.fragment_bitmap) - .or_else(|| removed_segment.fragment_bitmap.clone()) - .unwrap_or_default(); + let selected_logical_index = LogicalVectorIndex::try_new( + old_indices[0].name.clone(), + field_path.clone(), + vec![(selected_metadata, selected_index)], + )?; + let selected_ivf_view = selected_logical_index.as_ivf()?; + let (new_uuid, indices_merged, files) = Box::pin(optimize_vector_indices( + dataset.as_ref().clone(), + Option::< + lance_io::stream::RecordBatchStreamAdapter< + futures::stream::Empty>, + >, + >::None, + &field_path, + &selected_ivf_view, + options, + )) + .await?; + if indices_merged == 0 { + return Ok(None); + } - Ok(( - new_uuid, - vec![removed_segment], - new_fragment_bitmap, - CreatedIndex { - index_details: vector_index_details_default(), - index_version: lance_index::IndexType::Vector.version() as u32, - files, - }, - )) - } else { - let mut frag_bitmap = base_unindexed_bitmap.clone(); + let new_fragment_bitmap = removed_segment + .effective_fragment_bitmap(&dataset.fragment_bitmap) + .or_else(|| removed_segment.fragment_bitmap.clone()) + .unwrap_or_default(); - let new_data_stream = if unindexed.is_empty() { - None + Ok(( + new_uuid, + vec![removed_segment], + new_fragment_bitmap, + CreatedIndex { + index_details: removed_segment + .index_details + .as_deref() + .cloned() + .unwrap_or_else(vector_index_details_default), + index_version: removed_segment.index_version as u32, + files: table_files_to_index(files), + }, + removed_segment.dataset_version, + )) } else { - let mut scanner = dataset.scan(); - scanner - .with_fragments(unindexed.to_vec()) - .with_row_id() - .project(&[&field_path])?; - if column.nullable { - let column_expr = - lance_datafusion::logical_expr::field_path_to_expr(&field_path)?; - scanner.filter_expr(column_expr.is_not_null()); + let mut frag_bitmap = base_unindexed_bitmap.clone(); + let num_segments_to_merge = options + .num_indices_to_merge + .unwrap_or(logical_index.num_segments()) + .min(logical_index.num_segments()); + let merge_start = logical_index + .num_segments() + .saturating_sub(num_segments_to_merge); + let merge_logical_index = LogicalVectorIndex::try_new( + old_indices[0].name.clone(), + field_path.clone(), + logical_index + .iter() + .skip(merge_start) + .map(|(metadata, index)| (metadata.clone(), index.clone())) + .collect(), + )?; + let merge_ivf_view = merge_logical_index.as_ivf()?; + + let new_data_stream = if unindexed.is_empty() { + None + } else { + Some( + scan_vector_fragments( + dataset.as_ref(), + &field_path, + column.nullable, + unindexed, + ) + .await?, + ) + }; + + let (new_uuid, indices_merged, files) = optimize_vector_indices( + dataset.as_ref().clone(), + new_data_stream, + &field_path, + &merge_ivf_view, + options, + ) + .boxed() + .await?; + + let removed_indices = old_indices[old_indices.len() - indices_merged..].to_vec(); + let new_dataset_version = removed_indices + .iter() + .map(|index| index.dataset_version) + .min() + .unwrap_or(dataset.manifest.version); + removed_indices.iter().for_each(|idx| { + frag_bitmap.extend(idx.fragment_bitmap.as_ref().unwrap().iter()); + }); + for removed in removed_indices.iter() { + if let Some(effective) = + removed.effective_fragment_bitmap(&dataset.fragment_bitmap) + { + frag_bitmap |= &effective; + } } - Some(scanner.try_into_stream().await?) - }; - let (new_uuid, indices_merged, files) = optimize_vector_indices( - dataset.as_ref().clone(), - new_data_stream, - &field_path, - &ivf_view, - options, - ) - .boxed() - .await?; + // Metadata must come from an actual merge source. Older incompatible + // segments outside the selected suffix may describe a different model. + let (reference_metadata, reference_index) = + merge_logical_index.iter().last().ok_or_else(|| { + Error::index( + "Optimize vector index merge did not select a reference segment" + .to_string(), + ) + })?; + let index_details = removed_indices + .iter() + .rev() + .filter_map(|idx| idx.index_details.as_ref()) + .find(|d| !d.value.is_empty()) + .map(|d| d.as_ref().clone()) + .or_else(|| { + reference_metadata + .index_details + .as_deref() + .filter(|details| !details.value.is_empty()) + .cloned() + }) + .unwrap_or_else(vector_index_details_default); + let index_version = if let Some(metadata) = removed_indices.first() { + metadata.index_version as u32 + } else { + index_type_for_segmented_optimize(reference_index.as_ref())?.version() as u32 + }; - let removed_indices = old_indices[old_indices.len() - indices_merged..].to_vec(); - removed_indices.iter().for_each(|idx| { - frag_bitmap.extend(idx.fragment_bitmap.as_ref().unwrap().iter()); - }); - for removed in removed_indices.iter() { - if let Some(effective) = removed.effective_fragment_bitmap(&dataset.fragment_bitmap) + Ok(( + new_uuid, + removed_indices, + frag_bitmap, + CreatedIndex { + index_details, + index_version, + files: table_files_to_index(files), + }, + new_dataset_version, + )) + } + } else { + let mut indices = Vec::with_capacity(old_indices.len()); + for idx in old_indices { + match dataset + .open_generic_index(&field_path, &idx.uuid, &NoOpMetricsCollector) + .await { - frag_bitmap |= &effective; + Ok(index) => indices.push(index), + Err(e) => { + log::warn!( + "Cannot open index on column '{}': {}. \ + Skipping index merge for this column.", + field_path, + e + ); + return Ok(None); + } } } - // Carry forward existing index details, preferring the first segment - // that has populated (non-empty) details. - let index_details = old_indices - .iter() - .rev() - .filter_map(|idx| idx.index_details.as_ref()) - .find(|d| !d.value.is_empty()) - .map(|d| d.as_ref().clone()) - .unwrap_or_else(vector_index_details_default); - - Ok(( - new_uuid, - removed_indices, - frag_bitmap, - CreatedIndex { - index_details, - // retain_supported_indices guarantees all old_indices have - // index_version <= our max supported version, so we can safely - // write the current library's version for this index type. - index_version: lance_index::IndexType::Vector.version() as u32, - files, - }, - )) - } - } else { - let mut indices = Vec::with_capacity(old_indices.len()); - for idx in old_indices { - match dataset - .open_generic_index(&field_path, &idx.uuid, &NoOpMetricsCollector) - .await + if indices + .windows(2) + .any(|w| w[0].index_type() != w[1].index_type()) { - Ok(index) => indices.push(index), - Err(e) => { - log::warn!( - "Cannot open index on column '{}': {}. \ - Skipping index merge for this column.", - field_path, - e - ); - return Ok(None); - } + return Err(Error::index(format!( + "Append index: invalid index deltas: {:?}", + old_indices + ))); } - } - if indices - .windows(2) - .any(|w| w[0].index_type() != w[1].index_type()) - { - return Err(Error::index(format!( - "Append index: invalid index deltas: {:?}", - old_indices - ))); - } + let index_type = indices[0].index_type(); + match index_type { + IndexType::Inverted => { + let selected_old_indices = + select_segments_to_merge(dataset.as_ref(), old_indices, options); + if unindexed.is_empty() && selected_old_indices.len() <= 1 { + return Ok(None); + } + let reference_idx = selected_old_indices + .first() + .copied() + .unwrap_or(old_indices[old_indices.len() - 1]); + let reference_index = dataset + .open_scalar_index(&field_path, &reference_idx.uuid, &NoOpMetricsCollector) + .await?; + let update_criteria = reference_index.update_criteria(); + if update_criteria.requires_old_data { + let params = reference_index.derive_index_params()?; + let resolved = resolved_fts.as_ref().ok_or_else(|| { + Error::internal( + "Inverted index metadata did not resolve an FTS field".to_string(), + ) + })?; + let new_data_stream = load_fts_training_data( + dataset.as_ref(), + resolved, + &update_criteria.data_criteria, + None, + true, + None, + ) + .await?; + let new_uuid = Uuid::new_v4(); + let created_index = super::scalar::build_scalar_index( + dataset.as_ref(), + &resolved.canonical_path, + new_uuid, + ¶ms, + true, + None, + Some(new_data_stream), + Arc::new(NoopIndexBuildProgress), + ) + .await?; + return Ok(Some(IndexMergeResults { + new_uuid, + removed_indices: old_indices.to_vec(), + new_fragment_bitmap: dataset.fragment_bitmap.as_ref().clone(), + new_dataset_version: dataset.manifest.version, + new_index_version: created_index.index_version as i32, + new_index_details: created_index.index_details, + files: index_files_to_table(created_index.files), + })); + } - let index_type = indices[0].index_type(); - match index_type { - IndexType::Inverted => { - let selected_old_indices = - select_segments_to_merge(dataset.as_ref(), old_indices, options); - if unindexed.is_empty() && selected_old_indices.len() <= 1 { - return Ok(None); - } - let reference_idx = selected_old_indices - .first() - .copied() - .unwrap_or(old_indices[old_indices.len() - 1]); - let reference_index = dataset - .open_scalar_index(&field_path, &reference_idx.uuid, &NoOpMetricsCollector) - .await?; - let update_criteria = reference_index.update_criteria(); - if update_criteria.requires_old_data { - let params = reference_index.derive_index_params()?; - let new_data_stream = load_training_data( + let fragments = Some(unindexed.to_vec()); + let resolved = resolved_fts.as_ref().ok_or_else(|| { + Error::internal( + "Inverted index metadata did not resolve an FTS field".to_string(), + ) + })?; + let new_data_stream = load_fts_training_data( dataset.as_ref(), - &field_path, + resolved, &update_criteria.data_criteria, - None, + fragments, true, None, ) .await?; - let new_uuid = Uuid::new_v4(); - let created_index = super::scalar::build_scalar_index( - dataset.as_ref(), - column.name.as_str(), - new_uuid, - ¶ms, - true, - None, - Some(new_data_stream), - Arc::new(NoopIndexBuildProgress), - ) - .await?; - return Ok(Some(IndexMergeResults { - new_uuid, - removed_indices: old_indices.to_vec(), - new_fragment_bitmap: dataset.fragment_bitmap.as_ref().clone(), - new_index_version: created_index.index_version as i32, - new_index_details: created_index.index_details, - files: created_index.files, - })); - } - - let fragments = Some(unindexed.to_vec()); - let new_data_stream = load_training_data( - dataset.as_ref(), - &field_path, - &update_criteria.data_criteria, - fragments, - true, - None, - ) - .await?; - let mut frag_bitmap = base_unindexed_bitmap; - let mut effective_old_frags = RoaringBitmap::new(); - let mut selected_indices = Vec::with_capacity(selected_old_indices.len()); - for idx in &selected_old_indices { - if let Some(effective) = idx.effective_fragment_bitmap(&dataset.fragment_bitmap) - { - frag_bitmap |= &effective; - effective_old_frags |= &effective; + let mut frag_bitmap = base_unindexed_bitmap; + let mut effective_old_frags = RoaringBitmap::new(); + let mut selected_indices = Vec::with_capacity(selected_old_indices.len()); + for idx in &selected_old_indices { + if let Some(effective) = + idx.effective_fragment_bitmap(&dataset.fragment_bitmap) + { + frag_bitmap |= &effective; + effective_old_frags |= &effective; + } + let scalar_index = dataset + .open_scalar_index(&field_path, &idx.uuid, &NoOpMetricsCollector) + .await?; + let inverted_index = scalar_index + .as_any() + .downcast_ref::() + .ok_or_else(|| { + Error::index(format!( + "Append index: expected inverted index segment {}, got {:?}", + idx.uuid, + scalar_index.index_type() + )) + })?; + selected_indices.push(Arc::new(inverted_index.clone())); } - let scalar_index = dataset - .open_scalar_index(&field_path, &idx.uuid, &NoOpMetricsCollector) - .await?; - let inverted_index = scalar_index - .as_any() - .downcast_ref::() - .ok_or_else(|| { - Error::index(format!( - "Append index: expected inverted index segment {}, got {:?}", - idx.uuid, - scalar_index.index_type() - )) - })?; - selected_indices.push(Arc::new(inverted_index.clone())); - } - let old_data_filter = if selected_indices.is_empty() { - None - } else if dataset.manifest.uses_stable_row_ids() { - let valid_old_row_ids = - build_stable_row_id_filter(dataset.as_ref(), &effective_old_frags).await?; - Some(OldIndexDataFilter::RowIds(valid_old_row_ids)) - } else { - Some(OldIndexDataFilter::Fragments { - to_keep: effective_old_frags, - to_remove: RoaringBitmap::new(), - }) - }; + let old_data_filter = if selected_indices.is_empty() { + None + } else if dataset.manifest.uses_stable_row_ids() { + let valid_old_row_ids = + build_stable_row_id_filter(dataset.as_ref(), &effective_old_frags) + .await?; + Some(OldIndexDataFilter::RowIds(valid_old_row_ids)) + } else { + Some(OldIndexDataFilter::Fragments { + to_keep: effective_old_frags, + to_remove: RoaringBitmap::new(), + }) + }; - let new_uuid = Uuid::new_v4(); - let new_store = LanceIndexStore::from_dataset_for_new(&dataset, &new_uuid)?; - let created_index = if selected_indices.is_empty() { - let params = reference_index.derive_index_params()?; - super::scalar::build_scalar_index( - dataset.as_ref(), - column.name.as_str(), + let new_uuid = Uuid::new_v4(); + let new_store = LanceIndexStore::from_dataset_for_new(&dataset, &new_uuid)?; + let (created_index, new_dataset_version) = if selected_indices.is_empty() { + ( + super::scalar::build_scalar_index( + dataset.as_ref(), + &resolved.canonical_path, + new_uuid, + &reference_index.derive_index_params()?, + true, + None, + Some(new_data_stream), + Arc::new(NoopIndexBuildProgress), + ) + .await?, + dataset.manifest.version, + ) + } else { + ( + InvertedIndex::merge_segments( + &selected_indices, + new_data_stream, + &new_store, + old_data_filter, + options.progress.clone(), + ) + .await?, + selected_old_indices + .iter() + .map(|index| index.dataset_version) + .min() + .unwrap_or(dataset.manifest.version), + ) + }; + + Ok(( new_uuid, - ¶ms, - true, - None, - Some(new_data_stream), - Arc::new(NoopIndexBuildProgress), - ) - .await? - } else { - InvertedIndex::merge_segments( - &selected_indices, - new_data_stream, - &new_store, - old_data_filter, - options.progress.clone(), + selected_old_indices.to_vec(), + frag_bitmap, + created_index, + new_dataset_version, + )) + } + it if it.is_scalar() => { + let Some(result) = merge_scalar_indices( + dataset.clone(), + old_indices, + unindexed, + options, + it, + &field_path, + column.name.as_str(), + base_unindexed_bitmap, ) .await? - }; - - Ok(( - new_uuid, - selected_old_indices.to_vec(), - frag_bitmap, - created_index, - )) - } - it if it.is_scalar() => { - let Some(result) = merge_scalar_indices( - dataset.clone(), - old_indices, - unindexed, - options, - it, - &field_path, - column.name.as_str(), - base_unindexed_bitmap, - ) - .await? - else { - return Ok(None); - }; - Ok(result) + else { + return Ok(None); + }; + Ok(result) + } + _ => Err(Error::index(format!( + "Append index: invalid index type: {:?}", + indices[0].index_type() + ))), } - _ => Err(Error::index(format!( - "Append index: invalid index type: {:?}", - indices[0].index_type() - ))), - } - }?; + }?; Ok(Some(IndexMergeResults { new_uuid, removed_indices, new_fragment_bitmap, + new_dataset_version, new_index_version: created_index.index_version as i32, new_index_details: created_index.index_details, - files: created_index.files, + files: index_files_to_table(created_index.files), })) } -#[cfg(test)] -mod tests { - use super::*; +#[cfg(test)] +mod tests { + use super::*; + + use crate::index::DatasetIndexExt; + use crate::index::DatasetIndexInternalExt; + use arrow::datatypes::{Float32Type, UInt32Type}; + use arrow_array::cast::AsArray; + use arrow_array::{ + Array, ArrayRef, FixedSizeListArray, Float32Array, Int32Array, ListArray, RecordBatch, + RecordBatchIterator, StringArray, StructArray, UInt32Array, UInt64Array, + }; + use arrow_buffer::{BooleanBufferBuilder, NullBuffer, OffsetBuffer}; + use arrow_schema::{DataType, Field, Schema}; + use futures::TryStreamExt; + use lance_arrow::FixedSizeListArrayExt; + use lance_core::utils::tempfile::TempStrDir; + use lance_datafusion::utils::reader_to_stream; + use lance_datagen::{Dimension, RowCount, array}; + use lance_file::version::LanceFileVersion; + use lance_index::vector::hnsw::builder::HnswBuildParams; + use lance_index::vector::sq::builder::SQBuildParams; + use lance_index::{ + IndexType, + scalar::{ + BuiltinIndexType, InvertedIndexParams, ScalarIndexParams, SearchResult, TextQuery, + }, + vector::{ivf::IvfBuildParams, pq::PQBuildParams}, + }; + use lance_linalg::distance::MetricType; + use lance_testing::datagen::{generate_random_array, generate_random_array_with_seed}; + use rstest::rstest; + + use crate::dataset::builder::DatasetBuilder; + use crate::dataset::optimize::{CompactionOptions, compact_files}; + use crate::dataset::{MergeInsertBuilder, WhenMatched, WhenNotMatched, WriteMode, WriteParams}; + use crate::index::CreateIndexBuilder; + use crate::index::vector::VectorIndexParams; + use crate::utils::test::{DatagenExt, FragmentCount, FragmentRowCount}; + + #[test] + fn test_fragment_reuse_at_source_version_affects_segment() { + use lance_index::frag_reuse::{ + FragDigest, FragReuseGroup, FragReuseIndexDetails, FragReuseVersion, + }; + + let segment = IndexMetadata { + uuid: Uuid::new_v4(), + name: "text_ngram".to_string(), + fields: vec![0], + dataset_version: 5, + fragment_bitmap: Some(RoaringBitmap::from_iter([1u32])), + index_details: None, + index_version: 0, + created_at: None, + base_id: None, + files: None, + }; + let frag_reuse_index = FragReuseIndex { + uuid: Uuid::new_v4(), + row_id_maps: vec![], + details: FragReuseIndexDetails { + versions: vec![FragReuseVersion { + dataset_version: 5, + groups: vec![FragReuseGroup { + changed_row_addrs: vec![1], + old_frags: vec![FragDigest { + id: 1, + physical_rows: 1, + num_deleted_rows: 0, + }], + new_frags: vec![FragDigest { + id: 2, + physical_rows: 1, + num_deleted_rows: 0, + }], + }], + }], + }, + }; + + assert!(fragment_reuse_affects_segments( + &frag_reuse_index, + [&segment] + )); + + let rebuilt_segment = IndexMetadata { + dataset_version: 5, + fragment_bitmap: Some(RoaringBitmap::from_iter([2u32])), + ..segment + }; + assert!(!fragment_reuse_affects_segments( + &frag_reuse_index, + [&rebuilt_segment] + )); + + let stale_remapped_segment = IndexMetadata { + dataset_version: 4, + ..rebuilt_segment + }; + assert!(fragment_reuse_affects_segments( + &frag_reuse_index, + [&stale_remapped_segment] + )); + } + + fn clustered_vector_batch( + schema: Arc, + start_id: i32, + rows: usize, + dimension: usize, + center: f32, + ) -> (RecordBatch, Arc) { + let values = (0..rows) + .flat_map(|row| { + (0..dimension) + .map(move |column| center + row as f32 * 0.01 + column as f32 * 0.0001) + }) + .collect::>(); + let vectors = Arc::new( + FixedSizeListArray::try_new_from_values( + arrow_array::Float32Array::from(values), + dimension as i32, + ) + .unwrap(), + ); + let ids = Arc::new(Int32Array::from_iter_values( + start_id..start_id + rows as i32, + )); + let batch = RecordBatch::try_new(schema, vec![ids, vectors.clone() as ArrayRef]).unwrap(); + (batch, vectors) + } + + async fn nearest_id( + dataset: &Dataset, + query: &arrow_array::Float32Array, + num_probes: usize, + ) -> i32 { + let result = dataset + .scan() + .project(&["id"]) + .unwrap() + .nearest("vector", query, 1) + .unwrap() + .nprobes(num_probes) + .refine(1) + .try_into_batch() + .await + .unwrap(); + assert_eq!(result.num_rows(), 1); + result["id"] + .as_primitive::() + .value(0) + } + + #[rstest] + #[case::append(OptimizeOptions::append(), false)] + #[case::default_with_stable_row_ids_and_delete(OptimizeOptions::default(), true)] + #[tokio::test] + async fn test_vector_append_is_segment_set_native_with_distinct_models( + #[case] options: OptimizeOptions, + #[case] use_stable_row_ids: bool, + ) { + const DIMENSION: usize = 8; + const ROWS_PER_FRAGMENT: usize = 64; + const INDEX_NAME: &str = "vector_idx"; + + let test_dir = TempStrDir::default(); + let schema = Arc::new(Schema::new(vec![ + Field::new("id", DataType::Int32, false), + Field::new( + "vector", + DataType::FixedSizeList( + Arc::new(Field::new("item", DataType::Float32, true)), + DIMENSION as i32, + ), + false, + ), + ])); + let (first_batch, first_vectors) = + clustered_vector_batch(schema.clone(), 0, ROWS_PER_FRAGMENT, DIMENSION, 0.0); + let (second_batch, _) = clustered_vector_batch( + schema.clone(), + ROWS_PER_FRAGMENT as i32, + ROWS_PER_FRAGMENT, + DIMENSION, + 100.0, + ); + let reader = + RecordBatchIterator::new(vec![Ok(first_batch), Ok(second_batch)], schema.clone()); + let mut dataset = Dataset::write( + reader, + test_dir.as_str(), + Some(WriteParams { + enable_stable_row_ids: use_stable_row_ids, + max_rows_per_file: ROWS_PER_FRAGMENT, + ..Default::default() + }), + ) + .await + .unwrap(); + + let initial_fragments = dataset.get_fragments(); + assert_eq!(initial_fragments.len(), 2); + let params = VectorIndexParams::ivf_flat(1, MetricType::L2); + let mut initial_segments = Vec::with_capacity(initial_fragments.len()); + for fragment in &initial_fragments { + initial_segments.push( + CreateIndexBuilder::new(&mut dataset, &["vector"], IndexType::Vector, ¶ms) + .name(INDEX_NAME.to_string()) + .fragments(vec![fragment.id() as u32]) + .execute_uncommitted() + .await + .unwrap(), + ); + } + dataset + .commit_existing_index_segments(INDEX_NAME, "vector", initial_segments) + .await + .unwrap(); + dataset = DatasetBuilder::from_uri(test_dir.as_str()) + .load() + .await + .unwrap(); + + let old_segments = dataset.load_indices_by_name(INDEX_NAME).await.unwrap(); + assert_eq!(old_segments.len(), 2); + let logical_index = dataset + .open_logical_vector_index("vector", INDEX_NAME) + .await + .unwrap(); + let centroids = logical_index + .as_ivf() + .unwrap() + .segments() + .map(|(_, index)| index.ivf_model().centroids_array().unwrap().to_data()) + .collect::>(); + assert_ne!( + centroids[0], centroids[1], + "test setup must use distinct IVF centroid models" + ); + let version_before_default_optimize = dataset.version().version; + dataset + .optimize_indices(&OptimizeOptions::default()) + .await + .unwrap(); + assert_eq!( + dataset.version().version, + version_before_default_optimize, + "default optimize must leave incompatible steady-state segments unmerged" + ); + assert_eq!( + dataset.load_indices_by_name(INDEX_NAME).await.unwrap(), + old_segments, + "default optimize changed incompatible steady-state segments" + ); + + let appended_start_id = (2 * ROWS_PER_FRAGMENT) as i32; + let (appended_batch, appended_vectors) = clustered_vector_batch( + schema.clone(), + appended_start_id, + ROWS_PER_FRAGMENT, + DIMENSION, + 200.0, + ); + dataset + .append( + RecordBatchIterator::new(vec![Ok(appended_batch)], schema), + None, + ) + .await + .unwrap(); + let appended_fragment_id = dataset.get_fragments().last().unwrap().id() as u32; + if use_stable_row_ids { + dataset + .delete(&format!("id = {appended_start_id}")) + .await + .unwrap(); + } + + dataset.optimize_indices(&options).await.unwrap(); + + let mut dataset = DatasetBuilder::from_uri(test_dir.as_str()) + .load() + .await + .unwrap(); + let committed = dataset.load_indices_by_name(INDEX_NAME).await.unwrap(); + assert_eq!( + committed.len(), + 3, + "append must add one segment without rewriting existing segments" + ); + for old in &old_segments { + let retained = committed + .iter() + .find(|segment| segment.uuid == old.uuid) + .expect("old physical segment UUID must remain committed"); + assert_eq!(retained, old, "old physical segment metadata changed"); + } + + let new_segment = committed + .iter() + .find(|segment| { + old_segments + .iter() + .all(|old_segment| old_segment.uuid != segment.uuid) + }) + .expect("one new physical segment must be committed"); + assert_eq!( + new_segment.fragment_bitmap.as_ref().unwrap(), + &RoaringBitmap::from_iter([appended_fragment_id]), + "the appended segment must cover only the new fragment" + ); + + let logical_index = dataset + .open_logical_vector_index("vector", INDEX_NAME) + .await + .unwrap(); + let latest_old_uuid = old_segments.last().unwrap().uuid; + let latest_old_centroids = logical_index + .iter() + .find(|(metadata, _)| metadata.uuid == latest_old_uuid) + .unwrap() + .1 + .ivf_model() + .centroids_array() + .unwrap() + .to_data(); + let new_centroids = logical_index + .iter() + .find(|(metadata, _)| metadata.uuid == new_segment.uuid) + .unwrap() + .1 + .ivf_model() + .centroids_array() + .unwrap() + .to_data(); + assert_eq!( + new_centroids, latest_old_centroids, + "append must reuse the latest segment's complete IVF model" + ); + + let mut covered = RoaringBitmap::new(); + for segment in &committed { + let segment_coverage = segment.fragment_bitmap.as_ref().unwrap(); + assert!( + covered.is_disjoint(segment_coverage), + "physical vector segment coverage must be pairwise disjoint" + ); + covered |= segment_coverage; + } + assert_eq!( + covered, + dataset.fragment_bitmap.as_ref().clone(), + "physical vector segments must exactly cover indexed fragments" + ); + + let old_query = first_vectors.value(1); + assert_eq!( + nearest_id( + &dataset, + old_query.as_primitive::(), + committed.len() + ) + .await, + 1, + "query must return a representative row from an old segment" + ); + let appended_query_offset = usize::from(use_stable_row_ids); + let appended_query = appended_vectors.value(appended_query_offset); + assert_eq!( + nearest_id( + &dataset, + appended_query.as_primitive::(), + committed.len() + ) + .await, + appended_start_id + appended_query_offset as i32, + "query must return a representative row from the appended segment" + ); + if use_stable_row_ids { + assert_eq!( + dataset + .scan() + .filter(&format!("id = {appended_start_id}")) + .unwrap() + .count_rows() + .await + .unwrap(), + 0, + "deleted stable-row-id data must not reappear through the appended segment" + ); + } + + dataset + .optimize_indices(&OptimizeOptions::merge(2)) + .await + .unwrap(); + let merged_suffix = dataset.load_indices_by_name(INDEX_NAME).await.unwrap(); + assert_eq!( + merged_suffix.len(), + 2, + "the reference-compatible suffix should merge without rewriting the incompatible base" + ); + assert!( + merged_suffix + .iter() + .any(|segment| segment.uuid == old_segments[0].uuid), + "the incompatible base segment must remain unchanged" + ); + assert!( + merged_suffix + .iter() + .all(|segment| segment.uuid != latest_old_uuid && segment.uuid != new_segment.uuid), + "the compatible suffix segments must be replaced" + ); + + dataset + .optimize_indices(&OptimizeOptions::retrain()) + .await + .unwrap(); + let retrained = dataset.load_indices_by_name(INDEX_NAME).await.unwrap(); + assert_eq!(retrained.len(), 1); + assert_eq!( + retrained[0].fragment_bitmap.as_ref().unwrap(), + dataset.fragment_bitmap.as_ref(), + "explicit retrain must source-rebuild one segment over all current fragments" + ); + assert_eq!( + nearest_id( + &dataset, + old_query.as_primitive::(), + retrained.len() + ) + .await, + 1 + ); + assert_eq!( + nearest_id( + &dataset, + appended_query.as_primitive::(), + retrained.len() + ) + .await, + appended_start_id + appended_query_offset as i32 + ); + if use_stable_row_ids { + assert_eq!( + dataset + .scan() + .filter(&format!("id = {appended_start_id}")) + .unwrap() + .count_rows() + .await + .unwrap(), + 0, + "explicit retrain must not restore a deleted stable-row-id row" + ); + } + } + + #[rstest] + #[case::metric( + VectorIndexParams::ivf_flat(1, MetricType::L2), + VectorIndexParams::ivf_flat(1, MetricType::Cosine), + "has metric" + )] + #[case::index_family( + VectorIndexParams::ivf_flat(1, MetricType::L2), + VectorIndexParams::ivf_hnsw( + MetricType::L2, + IvfBuildParams::new(1), + HnswBuildParams::default() + ), + "has type" + )] + #[tokio::test] + async fn test_vector_append_validates_logical_query_compatibility( + #[case] first_params: VectorIndexParams, + #[case] second_params: VectorIndexParams, + #[case] expected_error: &str, + ) { + const DIMENSION: usize = 8; + const ROWS_PER_FRAGMENT: usize = 64; + const INDEX_NAME: &str = "vector_idx"; + + let test_dir = TempStrDir::default(); + let schema = Arc::new(Schema::new(vec![ + Field::new("id", DataType::Int32, false), + Field::new( + "vector", + DataType::FixedSizeList( + Arc::new(Field::new("item", DataType::Float32, true)), + DIMENSION as i32, + ), + false, + ), + ])); + let (first_batch, _) = + clustered_vector_batch(schema.clone(), 0, ROWS_PER_FRAGMENT, DIMENSION, 1.0); + let (second_batch, _) = clustered_vector_batch( + schema.clone(), + ROWS_PER_FRAGMENT as i32, + ROWS_PER_FRAGMENT, + DIMENSION, + 100.0, + ); + let reader = + RecordBatchIterator::new(vec![Ok(first_batch), Ok(second_batch)], schema.clone()); + let mut dataset = Dataset::write( + reader, + test_dir.as_str(), + Some(WriteParams { + max_rows_per_file: ROWS_PER_FRAGMENT, + ..Default::default() + }), + ) + .await + .unwrap(); + + let initial_fragments = dataset.get_fragments(); + let mut segments = Vec::with_capacity(initial_fragments.len()); + for (fragment, params) in initial_fragments.iter().zip([first_params, second_params]) { + segments.push( + CreateIndexBuilder::new(&mut dataset, &["vector"], IndexType::Vector, ¶ms) + .name(INDEX_NAME.to_string()) + .fragments(vec![fragment.id() as u32]) + .execute_uncommitted() + .await + .unwrap(), + ); + } + dataset + .commit_existing_index_segments(INDEX_NAME, "vector", segments) + .await + .unwrap(); + + let (appended_batch, _) = clustered_vector_batch( + schema.clone(), + (2 * ROWS_PER_FRAGMENT) as i32, + ROWS_PER_FRAGMENT, + DIMENSION, + 200.0, + ); + dataset + .append( + RecordBatchIterator::new(vec![Ok(appended_batch)], schema), + None, + ) + .await + .unwrap(); + + let version_before = dataset.version().version; + let segments_before = dataset.load_indices_by_name(INDEX_NAME).await.unwrap(); + let object_store = dataset.object_store.clone(); + let directories_before = object_store + .read_dir(dataset.indices_dir()) + .await + .unwrap() + .into_iter() + .collect::>(); + + let error = dataset + .optimize_indices(&OptimizeOptions::append()) + .await + .unwrap_err(); + assert!( + error.to_string().contains(expected_error), + "expected logical query compatibility error containing '{expected_error}', got {error}" + ); + + let latest = DatasetBuilder::from_uri(test_dir.as_str()) + .load() + .await + .unwrap(); + assert_eq!( + latest.version().version, + version_before, + "incompatible logical segments must fail before committing" + ); + let mut segments_after = latest.load_indices_by_name(INDEX_NAME).await.unwrap(); + let mut segments_before = segments_before; + for segment in segments_after.iter_mut().chain(segments_before.iter_mut()) { + segment.created_at = None; + } + assert_eq!( + segments_after, segments_before, + "incompatible logical segments must remain unchanged" + ); + let directories_after = object_store + .read_dir(latest.indices_dir()) + .await + .unwrap() + .into_iter() + .collect::>(); + assert_eq!( + directories_after, directories_before, + "query compatibility validation must run before staging a new segment" + ); + } + + #[tokio::test] + async fn test_vector_segment_native_append_preserves_commit_conflicts() { + const DIMENSION: usize = 8; + const ROWS_PER_FRAGMENT: usize = 64; + const INDEX_NAME: &str = "vector_idx"; + + let test_dir = TempStrDir::default(); + let schema = Arc::new(Schema::new(vec![ + Field::new("id", DataType::Int32, false), + Field::new( + "vector", + DataType::FixedSizeList( + Arc::new(Field::new("item", DataType::Float32, true)), + DIMENSION as i32, + ), + false, + ), + ])); + let (initial_batch, _) = + clustered_vector_batch(schema.clone(), 0, ROWS_PER_FRAGMENT, DIMENSION, 0.0); + let mut dataset = Dataset::write( + RecordBatchIterator::new(vec![Ok(initial_batch)], schema.clone()), + test_dir.as_str(), + Some(WriteParams { + max_rows_per_file: ROWS_PER_FRAGMENT, + ..Default::default() + }), + ) + .await + .unwrap(); + dataset + .create_index( + &["vector"], + IndexType::Vector, + Some(INDEX_NAME.to_string()), + &VectorIndexParams::ivf_flat(1, MetricType::L2), + true, + ) + .await + .unwrap(); + let old_uuid = dataset.load_indices_by_name(INDEX_NAME).await.unwrap()[0].uuid; + + let (first_append, _) = clustered_vector_batch( + schema.clone(), + ROWS_PER_FRAGMENT as i32, + ROWS_PER_FRAGMENT, + DIMENSION, + 100.0, + ); + dataset + .append( + RecordBatchIterator::new(vec![Ok(first_append)], schema.clone()), + None, + ) + .await + .unwrap(); + let stale_version = dataset.version().version; + let mut first_optimizer = dataset.checkout_version(stale_version).await.unwrap(); + let mut stale_optimizer = dataset.checkout_version(stale_version).await.unwrap(); + first_optimizer + .optimize_indices(&OptimizeOptions::append()) + .await + .unwrap(); - use crate::index::DatasetIndexExt; - use crate::index::DatasetIndexInternalExt; - use arrow::datatypes::{Float32Type, UInt32Type}; - use arrow_array::cast::AsArray; - use arrow_array::{ - FixedSizeListArray, Int32Array, RecordBatch, RecordBatchIterator, StringArray, UInt32Array, - }; - use arrow_schema::{DataType, Field, Schema}; - use futures::TryStreamExt; - use lance_arrow::FixedSizeListArrayExt; - use lance_core::utils::tempfile::TempStrDir; - use lance_datafusion::utils::reader_to_stream; - use lance_datagen::{Dimension, RowCount, array}; - use lance_index::vector::hnsw::builder::HnswBuildParams; - use lance_index::vector::sq::builder::SQBuildParams; - use lance_index::{ - IndexType, - scalar::{BuiltinIndexType, ScalarIndexParams, SearchResult, TextQuery}, - vector::{ivf::IvfBuildParams, pq::PQBuildParams}, - }; - use lance_linalg::distance::MetricType; - use lance_testing::datagen::generate_random_array; - use rstest::rstest; + let error = stale_optimizer + .optimize_indices(&OptimizeOptions::append()) + .await + .unwrap_err(); + assert!( + matches!(error, Error::RetryableCommitConflict { .. }), + "stale vector append optimize must retain retryable conflict semantics: {error}" + ); - use crate::dataset::builder::DatasetBuilder; - use crate::dataset::optimize::{CompactionOptions, compact_files}; - use crate::dataset::{MergeInsertBuilder, WhenMatched, WhenNotMatched, WriteMode, WriteParams}; - use crate::index::vector::VectorIndexParams; - use crate::utils::test::{DatagenExt, FragmentCount, FragmentRowCount}; + let latest = DatasetBuilder::from_uri(test_dir.as_str()) + .load() + .await + .unwrap(); + let committed = latest.load_indices_by_name(INDEX_NAME).await.unwrap(); + assert_eq!(committed.len(), 2); + assert!( + committed.iter().any(|segment| segment.uuid == old_uuid), + "the successful concurrent optimize must retain the old segment" + ); + assert_eq!( + latest.unindexed_fragments(INDEX_NAME).await.unwrap().len(), + 0, + "a failed stale commit must not publish the staged vector segment" + ); + } #[tokio::test] async fn test_append_index() { @@ -1005,6 +2167,118 @@ mod tests { assert_eq!(num_rows, 2000); } + #[tokio::test] + async fn test_optimize_append_preserves_case_sensitive_nullable_vector_column() { + const DIM: usize = 64; + const ROWS: usize = 1000; + + fn make_vectors(rows: usize, dim: usize, include_null: bool) -> FixedSizeListArray { + if include_null { + let mut nulls_builder = BooleanBufferBuilder::new(rows); + for row_idx in 0..rows { + nulls_builder.append(row_idx != 0); + } + let nulls = NullBuffer::new(nulls_builder.finish()); + FixedSizeListArray::try_new( + Arc::new(Field::new("item", DataType::Float32, true)), + dim as i32, + Arc::new(generate_random_array(rows * dim)), + Some(nulls), + ) + .unwrap() + } else { + FixedSizeListArray::try_new_from_values( + generate_random_array(rows * dim), + dim as i32, + ) + .unwrap() + } + } + + fn make_batch( + schema: Arc, + start_id: u32, + vectors: Arc, + ) -> RecordBatch { + let columns: Vec = vec![ + Arc::new(UInt32Array::from_iter_values( + start_id..start_id + ROWS as u32, + )) as ArrayRef, + vectors as ArrayRef, + ]; + RecordBatch::try_new(schema, columns).unwrap() + } + + let test_dir = TempStrDir::default(); + let test_uri = test_dir.as_str(); + let vector_type = DataType::FixedSizeList( + Arc::new(Field::new("item", DataType::Float32, true)), + DIM as i32, + ); + let schema = Arc::new(Schema::new(vec![ + Field::new("id", DataType::UInt32, false), + Field::new("VECTOR", vector_type, true), + ])); + + let initial_vectors = Arc::new(make_vectors(ROWS, DIM, false)); + let initial_batch = make_batch(schema.clone(), 0, initial_vectors); + let batches = RecordBatchIterator::new(std::iter::once(Ok(initial_batch)), schema.clone()); + let mut dataset = Dataset::write(batches, test_uri, None).await.unwrap(); + + let params = VectorIndexParams::with_ivf_pq_params( + MetricType::L2, + IvfBuildParams::new(2), + PQBuildParams { + num_sub_vectors: 2, + ..Default::default() + }, + ); + dataset + .create_index(&["VECTOR"], IndexType::Vector, None, ¶ms, true) + .await + .unwrap(); + + let appended_vectors = Arc::new(make_vectors(ROWS, DIM, true)); + let query = appended_vectors.value(5); + let appended_batch = make_batch(schema.clone(), ROWS as u32, appended_vectors); + let batches = RecordBatchIterator::new(std::iter::once(Ok(appended_batch)), schema); + dataset.append(batches, None).await.unwrap(); + + let index_name = dataset.load_indices().await.unwrap()[0].name.clone(); + assert!( + !dataset + .unindexed_fragments(&index_name) + .await + .unwrap() + .is_empty() + ); + + dataset + .optimize_indices(&OptimizeOptions::append()) + .await + .unwrap(); + + let dataset = DatasetBuilder::from_uri(test_uri).load().await.unwrap(); + assert!( + dataset + .unindexed_fragments(&index_name) + .await + .unwrap() + .is_empty() + ); + + let mut scanner = dataset.scan(); + scanner + .nearest("VECTOR", query.as_primitive::(), 10) + .unwrap(); + let results = scanner.try_into_batch().await.unwrap(); + assert_eq!( + results.num_rows(), + 10, + "expected the requested k=10 nearest-neighbor results" + ); + } + /// Regression: a second `OptimizeOptions::append()` call on a steady-state /// vector index used to fall through to `optimize_vector_indices` and write /// a new UUID directory + manifest even though nothing had changed. The @@ -1060,7 +2334,7 @@ mod tests { RecordBatchIterator::new(vec![make_batch()].into_iter().map(Ok), schema.clone()); dataset.append(batches, None).await.unwrap(); - // First append: folds the new fragment into a fresh delta segment. + // First append: writes the new fragment into a reference-compatible delta segment. dataset .optimize_indices(&OptimizeOptions::append()) .await @@ -1106,22 +2380,53 @@ mod tests { async fn test_query_delta_indices( #[values( VectorIndexParams::ivf_pq(2, 8, 4, MetricType::L2, 2), + VectorIndexParams::ivf_rq(2, 1, MetricType::L2), + VectorIndexParams::ivf_hnsw( + MetricType::L2, + IvfBuildParams::new(2), + HnswBuildParams { + max_level: 3, + m: 12, + ef_construction: 80, + prefetch_distance: Some(1), + } + ), + VectorIndexParams::with_ivf_hnsw_pq_params( + MetricType::L2, + IvfBuildParams::new(2), + HnswBuildParams { + max_level: 3, + m: 12, + ef_construction: 80, + prefetch_distance: Some(1), + }, + PQBuildParams { + num_sub_vectors: 4, + ..Default::default() + } + ), VectorIndexParams::with_ivf_hnsw_sq_params( MetricType::L2, IvfBuildParams::new(2), - HnswBuildParams::default(), + HnswBuildParams { + max_level: 3, + m: 12, + ef_construction: 80, + prefetch_distance: Some(1), + }, SQBuildParams::default() ) )] index_params: VectorIndexParams, ) { const DIM: usize = 64; - const TOTAL: usize = 1000; + const INITIAL_ROWS: usize = 1000; + const APPENDED_ROWS: usize = 64; let test_dir = TempStrDir::default(); let test_uri = test_dir.as_str(); - let vectors = generate_random_array(TOTAL * DIM); + let vectors = generate_random_array_with_seed::(INITIAL_ROWS * DIM, [42; 32]); let schema = Arc::new(Schema::new(vec![ Field::new( @@ -1139,7 +2444,7 @@ mod tests { schema.clone(), vec![ array.clone(), - Arc::new(UInt32Array::from_iter_values(0..TOTAL as u32)), + Arc::new(UInt32Array::from_iter_values(0..INITIAL_ROWS as u32)), ], ) .unwrap(); @@ -1159,9 +2464,9 @@ mod tests { let batch = RecordBatch::try_new( schema.clone(), vec![ - array.clone(), + Arc::new(array.slice(0, APPENDED_ROWS)), Arc::new(UInt32Array::from_iter_values( - TOTAL as u32..(TOTAL * 2) as u32, + INITIAL_ROWS as u32..(INITIAL_ROWS + APPENDED_ROWS) as u32, )), ], ) @@ -1169,6 +2474,7 @@ mod tests { let batches = RecordBatchIterator::new(vec![batch].into_iter().map(Ok), schema.clone()); dataset.append(batches, None).await.unwrap(); + let appended_fragment_id = dataset.get_fragments().last().unwrap().id() as u32; let stats: serde_json::Value = serde_json::from_str(&dataset.index_statistics("vector_idx").await.unwrap()).unwrap(); assert_eq!(stats["num_indices"], 1); @@ -1179,12 +2485,19 @@ mod tests { .optimize_indices(&OptimizeOptions::append()) .await .unwrap(); - let dataset = DatasetBuilder::from_uri(test_uri).load().await.unwrap(); + let mut dataset = DatasetBuilder::from_uri(test_uri).load().await.unwrap(); let stats: serde_json::Value = serde_json::from_str(&dataset.index_statistics("vector_idx").await.unwrap()).unwrap(); assert_eq!(stats["num_indices"], 2); assert_eq!(stats["num_indexed_fragments"], 2); assert_eq!(stats["num_unindexed_fragments"], 0); + let appended_segments = dataset.load_indices_by_name("vector_idx").await.unwrap(); + assert!( + appended_segments + .iter() + .all(|segment| segment.index_version == index_params.index_type().version()), + "append must preserve the storage type's index version" + ); let logical_index = dataset .open_logical_vector_index("vector", "vector_idx") .await @@ -1196,24 +2509,86 @@ mod tests { .into_iter() .map(|(_, num_rows)| num_rows) .sum::(), - 2000 + (INITIAL_ROWS + APPENDED_ROWS) as u64 ); + if matches!( + index_params.index_type(), + IndexType::IvfHnswFlat | IndexType::IvfHnswPq | IndexType::IvfHnswSq + ) { + let hnsw_params = logical_index + .iter() + .map(|(_, index)| index.statistics().unwrap()["sub_index"]["params"].clone()) + .collect::>(); + assert!( + hnsw_params.iter().all(|params| params == &hnsw_params[0]), + "append must preserve the reference segment's HNSW build parameters: {hnsw_params:?}" + ); + } - let results = dataset - .scan() + let mut fanout_scanner = dataset.scan(); + fanout_scanner .project(&["id"]) .unwrap() .nearest("vector", array.value(0).as_primitive::(), 2) .unwrap() .nprobes(2) - .refine(1) - .try_into_batch() + .refine(1); + let fanout_plan = fanout_scanner.explain_plan(true).await.unwrap(); + assert!( + fanout_plan.contains("ANNSubIndex: name=vector_idx, k=2, deltas=2"), + "logical vector query must fan out across both physical segments, plan was:\n{fanout_plan}" + ); + let results = fanout_scanner.try_into_batch().await.unwrap(); + assert_eq!(results.num_rows(), 2); + + for segment in &appended_segments { + let fragment_bitmap = segment + .fragment_bitmap + .as_ref() + .expect("vector segment must record fragment coverage"); + let expected_appended_row = fragment_bitmap.contains(appended_fragment_id); + let mut segment_scanner = dataset.scan(); + segment_scanner + .project(&["id"]) + .unwrap() + .nearest("vector", array.value(0).as_primitive::(), 1) + .unwrap() + .nprobes(2) + .refine(1) + .with_index_segments(vec![segment.uuid]) + .unwrap(); + let segment_result = segment_scanner.try_into_batch().await.unwrap(); + assert_eq!(segment_result.num_rows(), 1); + let id = segment_result["id"].as_primitive::().value(0); + assert_eq!( + id >= INITIAL_ROWS as u32, + expected_appended_row, + "segment {} returned row {id} outside its fragment coverage {:?}", + segment.uuid, + fragment_bitmap + ); + } + + dataset + .optimize_indices(&OptimizeOptions::merge(2)) .await .unwrap(); - assert_eq!(results.num_rows(), 2); - let mut id_arr = results["id"].as_primitive::().values().to_vec(); - id_arr.sort(); - assert_eq!(id_arr, vec![0, 1000]); + let merged = dataset.load_indices_by_name("vector_idx").await.unwrap(); + assert_eq!( + merged.len(), + 1, + "the reference-compatible append segment must merge with its source" + ); + assert_eq!( + merged[0].index_version, + index_params.index_type().version(), + "merge must preserve the storage type's index version" + ); + assert_eq!( + merged[0].fragment_bitmap.as_ref().unwrap(), + dataset.fragment_bitmap.as_ref(), + "the compatible merge must preserve exact fragment coverage" + ); } #[tokio::test] @@ -1664,25 +3039,121 @@ mod tests { let logical = crate::index::scalar_logical::open_named_scalar_index( &dataset, "text", - "text_fmindex", + "text_fmindex", + &NoOpMetricsCollector, + ) + .await + .unwrap(); + + for (pattern, expected) in [("old alpha", 1), ("new gamma", 1), ("needle", 2)] { + let query = TextQuery::StringContains(pattern.to_string()); + let result = logical.search(&query, &NoOpMetricsCollector).await.unwrap(); + let row_addrs = match result { + SearchResult::Exact(row_addrs) => row_addrs, + other => panic!("expected exact result for {pattern}, got {other:?}"), + }; + let count = row_addrs.true_rows().row_addrs().unwrap().count(); + assert_eq!( + count, expected, + "expected {expected} matches for {pattern}, got {count}" + ); + } + } + + #[tokio::test] + async fn test_optimize_ngram_merge_remaps_deferred_compaction() { + let test_dir = TempStrDir::default(); + let test_uri = test_dir.as_str(); + let schema = Arc::new(Schema::new(vec![Field::new("text", DataType::Utf8, true)])); + let make_batch = |values: &[&str]| { + RecordBatch::try_new( + schema.clone(), + vec![Arc::new(StringArray::from_iter_values( + values.iter().copied(), + ))], + ) + .unwrap() + }; + let reader = RecordBatchIterator::new( + vec![ + Ok(make_batch(&["alpha needle", "beta needle"])), + Ok(make_batch(&["gamma needle", "delta needle"])), + ], + schema.clone(), + ); + let mut dataset = Dataset::write( + reader, + test_uri, + Some(WriteParams { + max_rows_per_file: 2, + ..Default::default() + }), + ) + .await + .unwrap(); + dataset + .create_index( + &["text"], + IndexType::NGram, + Some("text_ngram".into()), + &ScalarIndexParams::for_builtin(BuiltinIndexType::NGram), + false, + ) + .await + .unwrap(); + + let metrics = compact_files( + &mut dataset, + CompactionOptions { + target_rows_per_fragment: 10, + defer_index_remap: true, + ..Default::default() + }, + None, + ) + .await + .unwrap(); + assert!(metrics.fragments_removed > 0 && metrics.fragments_added > 0); + + let appended = + RecordBatchIterator::new(vec![Ok(make_batch(&["epsilon needle"]))], schema.clone()); + let mut dataset = Dataset::write( + appended, + test_uri, + Some(WriteParams { + max_rows_per_file: 2, + mode: WriteMode::Append, + ..Default::default() + }), + ) + .await + .unwrap(); + dataset + .optimize_indices(&OptimizeOptions::merge(1)) + .await + .unwrap(); + + let dataset = DatasetBuilder::from_uri(test_uri).load().await.unwrap(); + let logical = crate::index::scalar_logical::open_named_scalar_index( + &dataset, + "text", + "text_ngram", &NoOpMetricsCollector, ) .await .unwrap(); - - for (pattern, expected) in [("old alpha", 1), ("new gamma", 1), ("needle", 2)] { - let query = TextQuery::StringContains(pattern.to_string()); - let result = logical.search(&query, &NoOpMetricsCollector).await.unwrap(); - let row_addrs = match result { - SearchResult::Exact(row_addrs) => row_addrs, - other => panic!("expected exact result for {pattern}, got {other:?}"), - }; - let count = row_addrs.true_rows().row_addrs().unwrap().count(); - assert_eq!( - count, expected, - "expected {expected} matches for {pattern}, got {count}" - ); - } + let result = logical + .search( + &TextQuery::StringContains("needle".to_string()), + &NoOpMetricsCollector, + ) + .await + .unwrap(); + let row_addrs = match result { + SearchResult::AtMost(row_addrs) => row_addrs, + other => panic!("expected AtMost result from ngram, got {other:?}"), + }; + assert_eq!(row_addrs.true_rows().row_addrs().unwrap().count(), 5); } #[tokio::test] @@ -1922,6 +3393,144 @@ mod tests { assert_eq!(rows, 2, "value 'd' lives in appended fragment"); } + #[tokio::test] + async fn test_optimize_append_with_backpressured_multichunk_read() { + const WIDE_DIMENSION: usize = 140_000; + const NARROW_DIMENSION: usize = 4_096; + const SHORT_ROWS: usize = 68; + const LONG_ROWS: usize = 128; + + fn make_batch(schema: Arc, rows: usize, base: usize) -> RecordBatch { + let docs_field = schema.field(1); + let DataType::List(item_field) = docs_field.data_type() else { + unreachable!("docs must be a list"); + }; + let DataType::Struct(doc_fields) = item_field.data_type() else { + unreachable!("docs items must be structs"); + }; + let wide_values = Float32Array::from_iter_values( + (0..rows * WIDE_DIMENSION).map(|index| ((index + base) % 1009) as f32), + ); + let narrow_values = Float32Array::from_iter_values( + (0..rows * NARROW_DIMENSION).map(|index| ((index + base) % 251) as f32), + ); + let docs = StructArray::new( + doc_fields.clone(), + vec![ + Arc::new( + FixedSizeListArray::try_new_from_values(wide_values, WIDE_DIMENSION as i32) + .unwrap(), + ), + Arc::new( + FixedSizeListArray::try_new_from_values( + narrow_values, + NARROW_DIMENSION as i32, + ) + .unwrap(), + ), + Arc::new(StringArray::from_iter_values( + (0..rows).map(|index| format!("document-{}", index + base)), + )), + ], + None, + ); + let docs = ListArray::new( + item_field.clone(), + OffsetBuffer::from_lengths(std::iter::repeat_n(1, rows)), + Arc::new(docs), + None, + ); + let ids = UInt64Array::from_iter_values(base as u64..(base + rows) as u64); + RecordBatch::try_new(schema, vec![Arc::new(ids), Arc::new(docs)]).unwrap() + } + + let vector_item = Arc::new(Field::new("item", DataType::Float32, true)); + let doc_fields = vec![ + Field::new( + "wide", + DataType::FixedSizeList(vector_item.clone(), WIDE_DIMENSION as i32), + true, + ), + Field::new( + "narrow", + DataType::FixedSizeList(vector_item, NARROW_DIMENSION as i32), + true, + ), + Field::new("text", DataType::Utf8, true), + ] + .into(); + let docs_item = Arc::new(Field::new("item", DataType::Struct(doc_fields), true)); + let schema = Arc::new(Schema::new(vec![ + Field::new("id", DataType::UInt64, false), + Field::new("docs", DataType::List(docs_item), true), + ])); + + let initial_batch = make_batch(schema.clone(), 1, 1_000); + let initial_reader = RecordBatchIterator::new(vec![Ok(initial_batch)], schema.clone()); + let mut dataset = Dataset::write( + initial_reader, + "memory://", + Some(WriteParams { + data_storage_version: Some(LanceFileVersion::V2_1), + ..Default::default() + }), + ) + .await + .unwrap(); + dataset + .create_index( + &["docs.text"], + IndexType::Inverted, + Some("docs_text".to_string()), + &InvertedIndexParams::default(), + true, + ) + .await + .unwrap(); + + let appended_batches = vec![ + Ok(make_batch(schema.clone(), SHORT_ROWS, 0)), + Ok(make_batch(schema.clone(), LONG_ROWS, SHORT_ROWS)), + ]; + let appended_reader = RecordBatchIterator::new(appended_batches, schema); + dataset + .append( + appended_reader, + Some(WriteParams { + data_storage_version: Some(LanceFileVersion::V2_1), + ..Default::default() + }), + ) + .await + .unwrap(); + // The deleted first batch makes the live delta start exactly at the + // second write-batch boundary, matching a merge-insert style update. + dataset.delete("id < 68").await.unwrap(); + + let optimize_options = OptimizeOptions::append(); + // Nested FTS reads the entire `docs` root. Under this budget the wide + // sibling is split into same-priority chunks, then the narrow sibling's + // higher-priority I/O consumes the remaining budget while the wide read + // is awaited. Admitted chunks must continue despite that backpressure. + let optimize = crate::index::scalar::TEST_TRAINING_IO_BUFFER_SIZE.scope( + 70 * 1024 * 1024, + dataset.optimize_indices(&optimize_options), + ); + tokio::time::timeout(std::time::Duration::from_secs(20), optimize) + .await + .expect("incremental nested FTS optimization timed out") + .unwrap(); + + let segments = dataset + .load_indices() + .await + .unwrap() + .iter() + .filter(|index| index.name == "docs_text") + .count(); + assert_eq!(segments, 2, "optimization should add one FTS delta segment"); + } + #[tokio::test] async fn test_optimize_btree_keeps_rows_with_stable_row_ids_after_compaction() { async fn query_id_count(dataset: &Dataset, id: &str) -> usize { @@ -2000,6 +3609,376 @@ mod tests { assert_eq!(query_id_count(&dataset, "song-42").await, 1); } + /// Updating an indexed vector column in place (`update_columns` + + /// `Operation::Update`) keeps the fragment id and the row address, so after the + /// fragment is pruned from the old segment's bitmap that segment still physically + /// holds the pre-update vector. Merging the segment must drop those rows: the merge + /// concatenates its physical rows with the freshly scanned ones and commits the + /// union of both coverages, so a surviving stale row is authorized by the same + /// bitmap as its fresh copy and no read-time filter can separate them. + #[rstest] + // IVF_FLAT keeps the vector verbatim, so the query distance tells which of the two + // copies survived. IVF_PQ re-uses the codebook trained before the update, which + // cannot represent the new value, so it only pins the row count and covers the + // transposed-code path in `take_partition_batches`. + #[case::ivf_flat(VectorIndexParams::ivf_flat(1, MetricType::L2), true)] + #[case::ivf_pq(VectorIndexParams::ivf_pq(1, 8, 16, MetricType::L2, 2), false)] + #[tokio::test] + async fn test_optimize_vector_index_drops_stale_rows_on_merge( + #[case] index_params: VectorIndexParams, + #[case] distance_is_exact: bool, + // The two row-id schemes take different filter branches: an address carries its + // fragment, a stable row id needs the fragments' persisted row-id sequences. + #[values(false, true)] enable_stable_row_ids: bool, + ) { + use crate::dataset::transaction::{Operation, UpdateMode, UpdatedFragmentOffsets}; + use arrow::datatypes::UInt64Type; + use arrow_array::{Float32Array, RecordBatchReader, UInt64Array}; + use lance_core::ROW_ID; + use lance_index::vector::DIST_COL; + use std::collections::HashMap; + + // Must match the sub-vector count in the IVF_PQ case above. + const DIM: usize = 16; + const BULK_ROWS: usize = 1000; + const UPDATED_ID: u32 = 10_000; + const STALE_VALUE: f32 = 2.0; + const FRESH_VALUE: f32 = 10.8; + + let test_dir = TempStrDir::default(); + let test_uri = test_dir.as_str(); + + let vector_type = DataType::FixedSizeList( + Arc::new(Field::new("item", DataType::Float32, true)), + DIM as i32, + ); + let schema = Arc::new(Schema::new(vec![ + Field::new("id", DataType::UInt32, false), + Field::new("vector", vector_type.clone(), true), + ])); + let constant_vector = |value: f32| { + FixedSizeListArray::try_new_from_values( + Float32Array::from_iter_values(std::iter::repeat_n(value, DIM)), + DIM as i32, + ) + .unwrap() + }; + + // Fragment 0: bulk vectors in [0, 1). None of them is near the query, so a + // surviving stale copy of the updated row ranks well inside top-k. + let bulk = generate_random_array_with_seed::(BULK_ROWS * DIM, [42; 32]); + let bulk_batch = RecordBatch::try_new( + schema.clone(), + vec![ + Arc::new(UInt32Array::from_iter_values(0..BULK_ROWS as u32)), + Arc::new(FixedSizeListArray::try_new_from_values(bulk, DIM as i32).unwrap()), + ], + ) + .unwrap(); + let mut dataset = Dataset::write( + RecordBatchIterator::new(vec![Ok(bulk_batch)], schema.clone()), + test_uri, + Some(WriteParams { + enable_stable_row_ids, + ..Default::default() + }), + ) + .await + .unwrap(); + + // Fragment 1: the single row that gets updated in place. + let stale_batch = RecordBatch::try_new( + schema.clone(), + vec![ + Arc::new(UInt32Array::from_iter_values([UPDATED_ID])), + Arc::new(constant_vector(STALE_VALUE)), + ], + ) + .unwrap(); + dataset + .append( + RecordBatchIterator::new(vec![Ok(stale_batch)], schema.clone()), + None, + ) + .await + .unwrap(); + assert_eq!(dataset.get_fragments().len(), 2); + + // One segment covering both fragments. + dataset + .create_index(&["vector"], IndexType::Vector, None, &index_params, true) + .await + .unwrap(); + + let mut fragment = dataset.get_fragment(1).unwrap(); + let mut fragment_scan = fragment.scan(); + fragment_scan.with_row_id(); + let updated_row_id = fragment_scan.try_into_batch().await.unwrap()[ROW_ID] + .as_primitive::() + .value(0); + + let update_schema = Arc::new(Schema::new(vec![ + Field::new(ROW_ID, DataType::UInt64, false), + Field::new("vector", vector_type, true), + ])); + let update_batch = RecordBatch::try_new( + update_schema.clone(), + vec![ + Arc::new(UInt64Array::from_iter_values([updated_row_id])), + Arc::new(constant_vector(FRESH_VALUE)), + ], + ) + .unwrap(); + let right_stream: Box = Box::new(RecordBatchIterator::new( + vec![Ok(update_batch)], + update_schema, + )); + let updated = fragment + .update_columns_with_offsets(right_stream, ROW_ID, ROW_ID) + .await + .unwrap(); + let updated_fragment_id = updated.fragment.id; + let dataset = Dataset::commit( + test_uri, + Operation::Update { + removed_fragment_ids: vec![], + updated_fragments: vec![updated.fragment], + new_fragments: vec![], + fields_modified: updated.fields_modified, + compacted_sstables: Vec::new(), + fields_for_preserving_frag_bitmap: vec![], + update_mode: Some(UpdateMode::RewriteColumns), + inserted_rows_filter: None, + updated_fragment_offsets: Some(UpdatedFragmentOffsets(HashMap::from([( + updated_fragment_id, + updated.matched_offsets, + )]))), + }, + Some(dataset.version().version), + None, + None, + Default::default(), + true, + ) + .await + .unwrap(); + + let mut dataset = dataset; + dataset + .optimize_indices(&OptimizeOptions::merge(1)) + .await + .unwrap(); + + let dataset = DatasetBuilder::from_uri(test_uri).load().await.unwrap(); + let segments = dataset.load_indices_by_name("vector_idx").await.unwrap(); + assert_eq!(segments.len(), 1, "merge must leave a single segment"); + + let mut scanner = dataset.scan(); + scanner + .nearest("vector", &constant_vector(FRESH_VALUE).value(0), 10) + .unwrap() + .with_row_id() + .project(&["id"]) + .unwrap(); + let results = scanner.try_into_batch().await.unwrap(); + let hits = results["id"] + .as_primitive::() + .values() + .iter() + .zip(results[DIST_COL].as_primitive::().values()) + .filter(|(id, _)| **id == UPDATED_ID) + .map(|(_, distance)| *distance) + .collect::>(); + assert_eq!( + hits.len(), + 1, + "updated row returned {} times after merge; row ids = {:?}", + hits.len(), + results[ROW_ID].as_primitive::().values() + ); + if distance_is_exact { + // The pre-update vector would sit at (FRESH - STALE)^2 * DIM from the query, + // so anything but ~0 means the stale copy is the one that survived. + assert!( + hits[0] < 1.0, + "surviving copy is the pre-update one: distance {} to the updated vector", + hits[0] + ); + } + } + + /// The ordinary update path (`UpdateBuilder`, `UpdateMode::RewriteRows`) commits + /// `fields_modified: vec![]`, so the old fragment keeps its place in the segment's + /// bitmap while its physical row is only deletion-marked. Under stable row ids the + /// rewritten copy reuses the same row id, so a merge that keeps every covered row + /// emits that id twice and the read-time filter admits both: the id is live again + /// at its new address. Coverage alone is therefore not a sufficient merge filter. + #[rstest] + // The two optimize modes read existing rows through different code, and both must + // filter: an explicit merge through `take_partition_batches`, and the default + // options over under-sized partitions through `partition_row_ids`, which feeds the + // join that rebuilds them. + #[case::merge(1, OptimizeOptions::merge(1))] + #[case::join(4, OptimizeOptions::new())] + #[tokio::test] + async fn test_optimize_vector_index_drops_rewritten_rows_on_merge( + #[case] num_partitions: usize, + #[case] optimize_options: OptimizeOptions, + // Only the stable-row-id scheme can duplicate: an address-domain rewrite lands + // at a new address, so the stale posting is masked at query time. + #[values(false, true)] enable_stable_row_ids: bool, + ) { + use crate::dataset::UpdateBuilder; + use arrow::datatypes::UInt64Type; + use arrow_array::Float32Array; + use lance_core::ROW_ID; + use lance_index::vector::DIST_COL; + + const DIM: usize = 4; + const BULK_ROWS: usize = 1000; + const UPDATED_ID: u32 = 10_000; + const STALE_VALUE: f32 = 2.0; + const FRESH_VALUE: f32 = 10.8; + + let test_dir = TempStrDir::default(); + let test_uri = test_dir.as_str(); + + let schema = Arc::new(Schema::new(vec![ + Field::new("id", DataType::UInt32, false), + Field::new( + "vector", + DataType::FixedSizeList( + Arc::new(Field::new("item", DataType::Float32, true)), + DIM as i32, + ), + true, + ), + ])); + let constant_vector = |value: f32| { + FixedSizeListArray::try_new_from_values( + Float32Array::from_iter_values(std::iter::repeat_n(value, DIM)), + DIM as i32, + ) + .unwrap() + }; + + // The updated row shares its fragment with the bulk rows, so the update leaves + // that fragment covered by the segment with one row deletion-marked instead of + // retiring it. The bulk vectors sit in [0, 1), far from the query, so a + // surviving stale copy of the updated row ranks well inside top-k. + let bulk = generate_random_array_with_seed::(BULK_ROWS * DIM, [42; 32]); + let batch = RecordBatch::try_new( + schema.clone(), + vec![ + Arc::new(UInt32Array::from_iter_values( + (0..BULK_ROWS as u32).chain(std::iter::once(UPDATED_ID)), + )), + Arc::new( + FixedSizeListArray::try_new_from_values( + Float32Array::from_iter_values( + bulk.values() + .iter() + .copied() + .chain(std::iter::repeat_n(STALE_VALUE, DIM)), + ), + DIM as i32, + ) + .unwrap(), + ), + ], + ) + .unwrap(); + let mut dataset = Dataset::write( + RecordBatchIterator::new(vec![Ok(batch)], schema.clone()), + test_uri, + Some(WriteParams { + enable_stable_row_ids, + ..Default::default() + }), + ) + .await + .unwrap(); + + dataset + .create_index( + &["vector"], + IndexType::Vector, + None, + &VectorIndexParams::ivf_flat(num_partitions, MetricType::L2), + true, + ) + .await + .unwrap(); + + let fresh_literal = format!( + "array[{}]", + std::iter::repeat_n(FRESH_VALUE.to_string(), DIM) + .collect::>() + .join(", ") + ); + let mut dataset = UpdateBuilder::new(Arc::new(dataset)) + .update_where(&format!("id = {UPDATED_ID}")) + .unwrap() + .set("vector", &fresh_literal) + .unwrap() + .build() + .unwrap() + .execute() + .await + .unwrap() + .new_dataset + .as_ref() + .clone(); + dataset.optimize_indices(&optimize_options).await.unwrap(); + + let dataset = DatasetBuilder::from_uri(test_uri).load().await.unwrap(); + let segments = dataset.load_indices_by_name("vector_idx").await.unwrap(); + assert_eq!(segments.len(), 1, "merge must leave a single segment"); + let partitions_after = dataset + .open_vector_index("vector", &segments[0].uuid, &NoOpMetricsCollector) + .await + .unwrap() + .ivf_model() + .num_partitions(); + // A join is what routes existing rows through `partition_row_ids`; without it + // this case would silently degrade into a second copy of the merge case. + assert!( + num_partitions == 1 || partitions_after < num_partitions, + "expected a join, but the index still has {partitions_after} partitions" + ); + + let mut scanner = dataset.scan(); + scanner + .nearest("vector", &constant_vector(FRESH_VALUE).value(0), 10) + .unwrap() + .with_row_id() + .project(&["id"]) + .unwrap(); + let results = scanner.try_into_batch().await.unwrap(); + let hits = results["id"] + .as_primitive::() + .values() + .iter() + .zip(results[DIST_COL].as_primitive::().values()) + .filter(|(id, _)| **id == UPDATED_ID) + .map(|(_, distance)| *distance) + .collect::>(); + assert_eq!( + hits.len(), + 1, + "rewritten row returned {} times after optimize; row ids = {:?}", + hits.len(), + results[ROW_ID].as_primitive::().values() + ); + // The pre-update vector would sit at (FRESH - STALE)^2 * DIM from the query, so + // anything but ~0 means the stale copy is the one that survived. + assert!( + hits[0] < 1.0, + "surviving copy is the pre-update one: distance {} to the updated vector", + hits[0] + ); + } + /// Under stable row ids, updating an indexed column and then calling /// `optimize_indices` must not leave stale entries (old value -> updated row) /// in the scalar index. An update deletes the old copy of each row and diff --git a/rust/lance/src/index/create.rs b/rust/lance/src/index/create.rs index 702a8c4e49f..16bb655a871 100644 --- a/rust/lance/src/index/create.rs +++ b/rust/lance/src/index/create.rs @@ -12,8 +12,8 @@ use crate::{ build_index_metadata_from_segments, scalar::{build_bitmap_index_segment, build_scalar_index}, vector::{ - LANCE_VECTOR_INDEX, VectorIndexParams, build_distributed_vector_index, - build_empty_vector_index, build_vector_index, + LANCE_VECTOR_INDEX, StageParams, VectorIndexParams, build_distributed_vector_index, + build_empty_vector_index, build_filtered_vector_index, build_vector_index, }, vector_index_details, vector_index_details_default, }, @@ -21,10 +21,15 @@ use crate::{ use futures::{FutureExt, future::BoxFuture}; use lance_core::datatypes::format_field_path; use lance_index::progress::{IndexBuildProgress, NoopIndexBuildProgress}; -use lance_index::{IndexParams, IndexType, scalar::CreatedIndex}; +use lance_index::{ + IndexParams, IndexType, registry::plugin_name_from_details_url, scalar::CreatedIndex, +}; use lance_index::{ metrics::NoOpMetricsCollector, - scalar::{LANCE_SCALAR_INDEX, ScalarIndexParams, inverted::tokenizer::InvertedIndexParams}, + scalar::{ + LANCE_SCALAR_INDEX, ScalarIndexParams, index_files_to_table, + inverted::tokenizer::InvertedIndexParams, table_files_to_index, + }, }; use lance_table::format::{IndexMetadata, list_index_files_with_sizes}; use std::{collections::HashMap, future::IntoFuture, sync::Arc}; @@ -44,6 +49,25 @@ fn default_index_name(fields: &[&str]) -> String { } } +fn resolved_inverted_params(params: &ScalarIndexParams) -> Result { + let mut merged = serde_json::to_value(InvertedIndexParams::default())?; + if let Some(raw_params) = params.params.as_deref() { + let provided = serde_json::from_str::(raw_params)?; + let merged = merged.as_object_mut().ok_or_else(|| { + Error::internal("default inverted index parameters are not a JSON object".to_string()) + })?; + let provided = provided.as_object().ok_or_else(|| { + Error::invalid_input("inverted index parameters must be a JSON object".to_string()) + })?; + merged.extend(provided.clone()); + } + Ok(serde_json::from_value(merged)?) +} + +fn scalar_params_from_inverted(params: &InvertedIndexParams) -> Result { + Ok(ScalarIndexParams::new("inverted".to_string()).with_params(¶ms.to_training_json()?)) +} + pub struct CreateIndexBuilder<'a> { dataset: &'a mut Dataset, columns: Vec, @@ -144,26 +168,98 @@ impl<'a> CreateIndexBuilder<'a> { )); } let column_input = &self.columns[0]; + let scalar_fts_request = self.index_type == IndexType::Scalar + && self + .params + .as_any() + .downcast_ref::() + .is_some_and(|params| { + params.index_type.eq_ignore_ascii_case("inverted") + || params.index_type.eq_ignore_ascii_case("fts") + }); + let inverted_params = if self.index_type == IndexType::Inverted { + if let Some(params) = self + .params + .as_any() + .downcast_ref::() + { + Some(params.clone()) + } else if let Some(params) = + self.params.as_any().downcast_ref::() + { + Some(resolved_inverted_params(params)?) + } else { + return Err(Error::index( + "Inverted index type must take InvertedIndexParams or ScalarIndexParams" + .to_string(), + )); + } + } else if scalar_fts_request { + Some(resolved_inverted_params( + self.params + .as_any() + .downcast_ref::() + .expect("scalar_fts_request verified scalar params"), + )?) + } else { + None + }; + let resolved_fts_field = if let Some(params) = &inverted_params { + Some(crate::index::scalar::inverted::resolve_fts_field( + self.dataset.schema(), + column_input, + params.get_document_granularity(), + )?) + } else { + None + }; + let lookup_column = resolved_fts_field + .as_ref() + .map(|resolved| resolved.root_column.as_str()) + .unwrap_or(column_input); // Use case-insensitive lookup for both simple and nested paths. // resolve_case_insensitive tries exact match first, then falls back to case-insensitive. - let Some(field_path) = self.dataset.schema().resolve_case_insensitive(column_input) else { + let Some(field_path) = self.dataset.schema().resolve_case_insensitive(lookup_column) else { return Err(Error::index(format!( "CreateIndex: column '{column_input}' does not exist" ))); }; - let field = *field_path.last().unwrap(); + let field = if let Some(resolved) = &resolved_fts_field { + self.dataset + .schema() + .field_by_id(resolved.final_field_id) + .ok_or_else(|| { + Error::index(format!( + "CreateIndex: FTS field path '{column_input}' resolved to missing field id {}", + resolved.final_field_id + )) + })? + } else { + *field_path.last().ok_or_else(|| { + Error::index(format!( + "CreateIndex: column '{column_input}' resolved to an empty field path" + )) + })? + }; // Reconstruct the column path with correct case from schema // Use quoted format for SQL parsing (special chars are quoted) let names: Vec<&str> = field_path.iter().map(|f| f.name.as_str()).collect(); let quoted_column: String = format_field_path(&names); - let column = quoted_column.as_str(); - - // If train is true but dataset is empty, automatically set train to false - let train = if self.train { - self.dataset.count_rows(None).await? > 0 - } else { - false - }; + let column = resolved_fts_field + .as_ref() + .map(|resolved| resolved.canonical_path.as_str()) + .unwrap_or(quoted_column.as_str()); + + let vector_fragments_for_validation = + is_builtin_vector_index(self.index_type, self.params) + .then_some(self.fragments.as_deref()) + .flatten(); + let train = should_train_index( + self.dataset, + self.train, + vector_fragments_for_validation, + ) + .await?; // Load indices from the disk. let indices = self.dataset.load_indices().await?; @@ -174,16 +270,28 @@ impl<'a> CreateIndexBuilder<'a> { let index_name = if let Some(name) = self.name.take() { name } else { - // Generate default name with collision handling - let column_path = default_index_name(&names); + // Generate default name with collision handling. + // A name is available when there is no existing index with: + // - the same name AND different fields, OR + // - the same name AND same field BUT a different index kind + let column_path = resolved_fts_field + .as_ref() + .map(|resolved| { + if resolved.document_granularity.is_list_element() { + format!("{}_list_element", resolved.canonical_path) + } else { + resolved.canonical_path.clone() + } + }) + .unwrap_or_else(|| default_index_name(&names)); let base_name = format!("{column_path}_idx"); let mut candidate = base_name.clone(); let mut counter = 2; // Start with no suffix, then use _2, _3, ... - // Find unique name by appending numeric suffix if needed - while indices - .iter() - .any(|idx| idx.name == candidate && idx.fields != [field.id]) - { + while indices.iter().any(|idx| { + idx.name == candidate + && (idx.fields != [field.id] + || !index_matches_type(idx, self.index_type, self.params)) + }) { candidate = format!("{base_name}_{counter}"); counter += 1; } @@ -237,7 +345,7 @@ impl<'a> CreateIndexBuilder<'a> { let base_params = ScalarIndexParams::for_builtin(self.index_type.try_into()?); // If custom params were provided, extract the params JSON and apply it - let params = if let Some(provided_params) = + let mut params = if let Some(provided_params) = self.params.as_any().downcast_ref::() { if let Some(params_json) = &provided_params.params { @@ -255,6 +363,9 @@ impl<'a> CreateIndexBuilder<'a> { } else { base_params }; + if let Some(inverted_params) = &inverted_params { + params = scalar_params_from_inverted(inverted_params)?; + } let preprocesssed_data = self .preprocessed_data @@ -307,12 +418,18 @@ impl<'a> CreateIndexBuilder<'a> { .downcast_ref::() .ok_or_else(|| { Error::index("Scalar index type must take a ScalarIndexParams".to_string()) - })?; + })? + .clone(); + let params = if let Some(inverted_params) = &inverted_params { + scalar_params_from_inverted(inverted_params)? + } else { + params + }; build_scalar_index( self.dataset, column, index_id, - params, + ¶ms, train, self.fragments.clone(), None, @@ -322,18 +439,11 @@ impl<'a> CreateIndexBuilder<'a> { } (IndexType::Inverted, _) => { // Inverted index params. - let inverted_params = self - .params - .as_any() - .downcast_ref::() - .ok_or_else(|| { - Error::index( - "Inverted index type must take a InvertedIndexParams".to_string(), - ) - })?; + let inverted_params = inverted_params + .as_ref() + .expect("IndexType::Inverted parameters were resolved above"); - let params = ScalarIndexParams::new("inverted".to_string()) - .with_params(&inverted_params.to_training_json()?); + let params = scalar_params_from_inverted(inverted_params)?; build_scalar_index( self.dataset, column, @@ -367,24 +477,40 @@ impl<'a> CreateIndexBuilder<'a> { })?; let index_version = vec_params.index_type().version() as u32; + let effective_fragments = + effective_vector_fragments(self.dataset, self.fragments.as_deref()); let files = if train { // Check if this is distributed indexing (fragment-level) - if let Some(fragments) = &self.fragments { - // For distributed indexing, build only on specified fragments - // This creates temporary index metadata without committing - let (segment_uuid, files) = Box::pin(build_distributed_vector_index( - self.dataset, - column, - &index_name, - index_id, - vec_params, - fri, - fragments, - self.progress.clone(), - )) - .await?; - output_index_uuid = segment_uuid; - files + if let Some(fragments) = effective_fragments.as_deref() { + if vector_params_have_precomputed_ivf(vec_params) { + // For distributed indexing, build only on specified fragments + // This creates temporary index metadata without committing + let (segment_uuid, files) = Box::pin(build_distributed_vector_index( + self.dataset, + column, + &index_name, + index_id, + vec_params, + fri, + fragments, + self.progress.clone(), + )) + .await?; + output_index_uuid = segment_uuid; + files + } else { + Box::pin(build_filtered_vector_index( + self.dataset, + column, + &index_name, + index_id, + vec_params, + fri, + fragments, + self.progress.clone(), + )) + .await? + } } else { // Standard full dataset indexing Box::pin(build_vector_index( @@ -412,7 +538,7 @@ impl<'a> CreateIndexBuilder<'a> { CreatedIndex { index_details: vector_index_details(vec_params), index_version, - files, + files: table_files_to_index(files), } } // Can't use if let Some(...) here because it's not stable yet. @@ -451,7 +577,7 @@ impl<'a> CreateIndexBuilder<'a> { CreatedIndex { index_details: vector_index_details_default(), index_version: self.index_type.version() as u32, - files, + files: table_files_to_index(files), } } (IndexType::FragmentReuse, _) => { @@ -484,7 +610,7 @@ impl<'a> CreateIndexBuilder<'a> { index_version: created_index.index_version as i32, created_at: Some(chrono::Utc::now()), base_id: None, - files: Some(created_index.files), + files: Some(index_files_to_table(created_index.files)), }) } .boxed() @@ -513,28 +639,15 @@ impl<'a> CreateIndexBuilder<'a> { } else { vec![] }; - let transaction = if uses_segment_commit_path(self.index_type, self.params) { - let dataset_version = new_idx.dataset_version; - TransactionBuilder::new( - dataset_version, - Operation::CreateIndex { - new_indices: vec![new_idx], - removed_indices, - }, - ) - .transaction_properties(self.transaction_properties.clone()) - .build() - } else { - TransactionBuilder::new( - new_idx.dataset_version, - Operation::CreateIndex { - new_indices: vec![new_idx], - removed_indices, - }, - ) - .transaction_properties(self.transaction_properties.clone()) - .build() - }; + let transaction = TransactionBuilder::new( + new_idx.dataset_version, + Operation::CreateIndex { + new_indices: vec![new_idx], + removed_indices, + }, + ) + .transaction_properties(self.transaction_properties.clone()) + .build(); self.dataset .apply_commit(transaction, &Default::default(), &Default::default()) @@ -651,7 +764,7 @@ impl<'a> CreateIndexBuilder<'a> { index_version: created_index.index_version as i32, created_at: Some(chrono::Utc::now()), base_id: None, - files: Some(created_index.files), + files: Some(index_files_to_table(created_index.files)), }; let segments = vec![metadata.into_index_segment()?]; let new_indices = @@ -720,7 +833,7 @@ impl<'a> CreateIndexBuilder<'a> { index_version: created_index.index_version as i32, created_at: Some(chrono::Utc::now()), base_id: None, - files: Some(created_index.files), + files: Some(index_files_to_table(created_index.files)), }); } @@ -733,8 +846,6 @@ impl<'a> CreateIndexBuilder<'a> { build_index_metadata_from_segments(self.dataset, &index_name, field.id, segments) .await?; - // Collect all same-name indices for removal when replace is set, - // matching the standard execute() path behavior. let removed_indices = if self.replace { existing_named_indices .into_iter() @@ -768,43 +879,33 @@ impl<'a> CreateIndexBuilder<'a> { } } -fn is_btree_scalar_params(params: &dyn IndexParams) -> bool { - params - .as_any() - .downcast_ref::() - .is_some_and(|p| p.index_type.eq_ignore_ascii_case("btree")) -} - -/// Validate that a user-supplied `index_uuid` is permitted for this build. -fn ensure_index_uuid_allowed( +/// Returns true if an existing `IndexMetadata` matches the given type +fn index_matches_type( + idx: &IndexMetadata, index_type: IndexType, params: &dyn IndexParams, - fragments: Option<&Vec>, - index_uuid: Option<&Uuid>, -) -> Result<()> { - let is_btree = index_type == IndexType::BTree - || params - .as_any() - .downcast_ref::() - .map(|params| params.index_type.eq_ignore_ascii_case("btree")) - .unwrap_or(false); - - if index_uuid.is_some() && fragments.is_some_and(|fragments| !fragments.is_empty()) && is_btree +) -> bool { + let Some(d) = &idx.index_details else { + // Fallback for legacy indexes, assume we are not trying to change the type + return true; + }; + // When index_type is Scalar the actual type is carried in ScalarIndexParams as a + // plugin name string (e.g. "zonemap"). The registry uses the same normalization + // for type_url lookup: lowercase the last path segment and strip "indexdetails". + // Compare directly instead of going through IndexType so this path stays valid + // as we move away from the IndexType enum. + if index_type == IndexType::Scalar + && let Some(scalar_params) = params.as_any().downcast_ref::() { - return Err(Error::invalid_input( - "index_uuid is no longer accepted for BTree distributed index builds; segment UUIDs \ - are generated by Lance and returned in the index metadata." - .to_string(), - )); + return scalar_params.index_type.to_lowercase() + == plugin_name_from_details_url(&d.type_url); } - Ok(()) + index_type.matches_details(d) } -fn uses_segment_commit_path(index_type: IndexType, params: &dyn IndexParams) -> bool { - let params_family = params.index_name(); - - if params_family == LANCE_VECTOR_INDEX +fn is_builtin_vector_index(index_type: IndexType, params: &dyn IndexParams) -> bool { + params.index_name() == LANCE_VECTOR_INDEX && matches!( index_type, IndexType::Vector @@ -817,19 +918,84 @@ fn uses_segment_commit_path(index_type: IndexType, params: &dyn IndexParams) -> | IndexType::IvfHnswSq ) && params.as_any().is::() - { - return true; +} + +async fn should_train_index( + dataset: &Dataset, + train: bool, + vector_fragments: Option<&[u32]>, +) -> Result { + if !train { + return Ok(false); } - if params_family == LANCE_SCALAR_INDEX { - match index_type { - IndexType::BTree => return true, - IndexType::Scalar if is_btree_scalar_params(params) => return true, - _ => {} - } + if dataset.fragment_bitmap.is_empty() { + return Ok(false); + } + + if let Some(fragment_ids) = vector_fragments { + dataset.get_fragments_from_ids(fragment_ids)?; + return Ok(true); + } + + Ok(dataset.count_rows(None).await? > 0) +} + +fn vector_params_have_precomputed_ivf(params: &VectorIndexParams) -> bool { + matches!( + params.stages.first(), + Some(StageParams::Ivf(ivf_params)) if ivf_params.centroids.is_some() + ) +} + +fn effective_vector_fragments(dataset: &Dataset, fragments: Option<&[u32]>) -> Option> { + let fragments = Dataset::normalize_fragment_ids(fragments?); + let fragment_bitmap: roaring::RoaringBitmap = fragments.iter().copied().collect(); + (fragment_bitmap != *dataset.fragment_bitmap).then_some(fragments) +} + +/// Validate that a user-supplied `index_uuid` is permitted for this build. +fn ensure_index_uuid_allowed( + index_type: IndexType, + params: &dyn IndexParams, + fragments: Option<&Vec>, + index_uuid: Option<&Uuid>, +) -> Result<()> { + let scalar_index_type = params + .as_any() + .downcast_ref::() + .map(|params| params.index_type.as_str()); + let unsafe_distributed_type = if index_type == IndexType::BTree + || scalar_index_type.is_some_and(|index_type| index_type.eq_ignore_ascii_case("btree")) + { + Some("BTree") + } else if index_type == IndexType::RTree + || scalar_index_type.is_some_and(|index_type| index_type.eq_ignore_ascii_case("rtree")) + { + Some("RTree") + } else if index_type == IndexType::NGram + || scalar_index_type.is_some_and(|index_type| index_type.eq_ignore_ascii_case("ngram")) + { + Some("NGram") + } else if index_type == IndexType::LabelList + || scalar_index_type.is_some_and(|index_type| index_type.eq_ignore_ascii_case("labellist")) + { + Some("LabelList") + } else { + None + }; + + if index_uuid.is_some() + && fragments.is_some_and(|fragments| !fragments.is_empty()) + && let Some(index_type) = unsafe_distributed_type + { + return Err(Error::invalid_input(format!( + "index_uuid is no longer accepted for {index_type} distributed index builds; \ + segment UUIDs are generated by Lance and returned in the index metadata." + ))); } - false + Ok(()) } impl<'a> IntoFuture for CreateIndexBuilder<'a> { @@ -859,12 +1025,14 @@ mod tests { use lance_index::optimize::OptimizeOptions; use lance_index::progress::IndexBuildProgress; use lance_index::scalar::{ - FullTextSearchQuery, SargableQuery, SearchResult, inverted::tokenizer::InvertedIndexParams, + BloomFilterQuery, FullTextSearchQuery, SargableQuery, SearchResult, + inverted::tokenizer::InvertedIndexParams, }; use lance_index::vector::hnsw::builder::HnswBuildParams; use lance_index::vector::ivf::IvfBuildParams; use lance_index::vector::kmeans::{KMeansParams, train_kmeans}; use lance_linalg::distance::{DistanceType, MetricType}; + use roaring::RoaringBitmap; use std::{collections::BTreeSet, ops::Bound, sync::Arc}; use uuid::Uuid; @@ -1152,6 +1320,50 @@ mod tests { IvfBuildParams::try_with_centroids(4, centroids).unwrap() } + async fn write_vector_fragment_dataset(uri: &str) -> Dataset { + let reader = gen_batch() + .col("id", lance_datagen::array::step::()) + .col( + "vector", + lance_datagen::array::rand_vec::(lance_datagen::Dimension::from(16)), + ) + .into_reader_rows( + lance_datagen::RowCount::from(256), + lance_datagen::BatchCount::from(4), + ); + Dataset::write( + reader, + uri, + Some(WriteParams { + max_rows_per_file: 64, + mode: WriteMode::Overwrite, + ..Default::default() + }), + ) + .await + .unwrap() + } + + #[tokio::test] + async fn test_get_frags_from_ordered_ids_accepts_unsorted_duplicates() { + let tmpdir = TempStrDir::default(); + let dataset_uri = format!("file://{}", tmpdir.as_str()); + let dataset = write_vector_fragment_dataset(&dataset_uri).await; + + let fragments = dataset.get_fragments(); + assert!(fragments.len() >= 2); + let first = fragments[0].id() as u32; + let second = fragments[1].id() as u32; + + let resolved = dataset.get_frags_from_ordered_ids(&[second, first, second, u32::MAX]); + + assert_eq!(resolved.len(), 4); + assert_eq!(resolved[0].as_ref().unwrap().id() as u32, second); + assert_eq!(resolved[1].as_ref().unwrap().id() as u32, first); + assert_eq!(resolved[2].as_ref().unwrap().id() as u32, second); + assert!(resolved[3].is_none()); + } + #[tokio::test] async fn test_execute_uncommitted() { // Test the complete workflow that covers the user's specified code pattern: @@ -1552,6 +1764,110 @@ mod tests { } } + #[tokio::test] + async fn test_label_list_distributed_index_uuid_collision_rejected() { + use lance_datagen::{Dimension, array}; + use lance_index::scalar::label_list::BITMAP_LOOKUP_NAME; + + let test_dir = TempStrDir::default(); + let labels = |label| { + array::cycle_vec_var( + array::cycle::(vec![label]), + Dimension::from(1), + Dimension::from(2), + ) + }; + let mut dataset = gen_batch() + .col("labels_a", labels(1)) + .col("labels_b", labels(3)) + .into_dataset( + test_dir.as_str(), + FragmentCount::from(2), + FragmentRowCount::from(4), + ) + .await + .unwrap(); + let params = + ScalarIndexParams::for_builtin(lance_index::scalar::BuiltinIndexType::LabelList); + let live_index = dataset + .create_index( + &["labels_a"], + IndexType::LabelList, + Some("labels_a_idx".to_string()), + ¶ms, + false, + ) + .await + .unwrap(); + let artifact_path = dataset + .indices_dir() + .join(live_index.uuid.to_string()) + .join(BITMAP_LOOKUP_NAME); + let artifact_before = dataset + .object_store + .read_one_all(&artifact_path) + .await + .unwrap(); + let version_before = dataset.version().version; + let fragment_id = dataset.get_fragments()[0].id() as u32; + + for index_type in [IndexType::LabelList, IndexType::Scalar] { + let err = dataset + .create_index_builder(&["labels_b"], index_type, ¶ms) + .name("labels_b_idx".to_string()) + .fragments(vec![fragment_id]) + .index_uuid(live_index.uuid) + .execute_uncommitted() + .await + .unwrap_err(); + + assert!( + matches!(err, Error::InvalidInput { .. }), + "expected invalid input error, got: {err}" + ); + assert!( + err.to_string().contains( + "index_uuid is no longer accepted for LabelList distributed index builds" + ), + "unexpected error: {err}" + ); + } + assert_eq!(dataset.version().version, version_before); + let artifact_after = dataset + .object_store + .read_one_all(&artifact_path) + .await + .unwrap(); + assert_eq!(artifact_after, artifact_before); + + let dataset = Dataset::open(test_dir.as_str()).await.unwrap(); + assert_eq!( + dataset + .count_rows(Some("array_has_any(labels_a, [1])".to_string())) + .await + .unwrap(), + 8 + ); + assert_eq!( + dataset + .count_rows(Some("array_has_any(labels_a, [3])".to_string())) + .await + .unwrap(), + 0 + ); + let plan = dataset + .scan() + .filter("array_has_any(labels_a, [1])") + .unwrap() + .explain_plan(false) + .await + .unwrap(); + assert!( + plan.contains("ScalarIndexQuery") && plan.contains("LabelList"), + "expected LabelList scalar index query in plan: {plan}" + ); + } + #[tokio::test] async fn test_range_based_btree_index_create() { use crate::dataset::scanner::ColumnOrdering; @@ -1707,42 +2023,206 @@ mod tests { } #[tokio::test] - async fn test_vector_execute_uncommitted_segments_commit_without_staging() { - let tmpdir = TempStrDir::default(); - let dataset_uri = format!("file://{}", tmpdir.as_str()); + async fn test_bloomfilter_distributed_segments_merge_and_query() { + #[derive(serde::Serialize)] + struct BloomParams { + number_of_items: u64, + probability: f64, + } - let reader = gen_batch() - .col("id", lance_datagen::array::step::()) - .col( - "vector", - lance_datagen::array::rand_vec::(lance_datagen::Dimension::from(16)), - ) - .into_reader_rows( - lance_datagen::RowCount::from(256), - lance_datagen::BatchCount::from(4), - ); - let mut dataset = Dataset::write( - reader, - &dataset_uri, - Some(WriteParams { - max_rows_per_file: 64, - mode: WriteMode::Overwrite, - ..Default::default() - }), - ) - .await - .unwrap(); + let mut dataset = gen_batch() + .col("value", lance_datagen::array::step::()) + .into_ram_dataset(FragmentCount::from(3), FragmentRowCount::from(8)) + .await + .unwrap(); + let params = + ScalarIndexParams::for_builtin(lance_index::scalar::BuiltinIndexType::BloomFilter) + .with_params(&BloomParams { + number_of_items: 4, + probability: 0.000_001, + }); let fragments = dataset.get_fragments(); - assert!(fragments.len() >= 2); - let params = VectorIndexParams::with_ivf_flat_params( - DistanceType::L2, - prepare_vector_ivf(&dataset, "vector").await, - ); - let mut input_segments = Vec::new(); - + assert_eq!(fragments.len(), 3); + let mut staged = Vec::with_capacity(fragments.len()); for fragment in &fragments { - let segment = + staged.push( + CreateIndexBuilder::new(&mut dataset, &["value"], IndexType::BloomFilter, ¶ms) + .name("value_bloom_segments".to_string()) + .fragments(vec![fragment.id() as u32]) + .execute_uncommitted() + .await + .unwrap(), + ); + } + + let mut future_segment = staged[1].clone(); + future_segment.dataset_version = dataset.manifest.version + 1; + let error = dataset + .merge_existing_index_segments(vec![staged[0].clone(), future_segment]) + .await + .unwrap_err(); + assert!( + error.to_string().contains("future dataset version"), + "unexpected merge error: {error}" + ); + + dataset + .commit_existing_index_segments("value_bloom_segments", "value", staged.clone()) + .await + .unwrap(); + let logical = crate::index::scalar_logical::open_named_scalar_index( + &dataset, + "value", + "value_bloom_segments", + &NoOpMetricsCollector, + ) + .await + .unwrap(); + assert_eq!( + logical.calculate_included_frags().await.unwrap(), + dataset.fragment_bitmap.as_ref().clone() + ); + + let mut trimmed = staged.clone(); + trimmed[0].fragment_bitmap = Some(RoaringBitmap::new()); + let trimmed_merged = dataset + .merge_existing_index_segments(trimmed) + .await + .unwrap(); + assert_eq!( + trimmed_merged.fragment_bitmap.as_ref().unwrap(), + &RoaringBitmap::from_iter(fragments[1..].iter().map(|fragment| fragment.id() as u32)) + ); + + let incompatible_params = + ScalarIndexParams::for_builtin(lance_index::scalar::BuiltinIndexType::BloomFilter) + .with_params(&BloomParams { + number_of_items: 8, + probability: 0.01, + }); + let incompatible = CreateIndexBuilder::new( + &mut dataset, + &["value"], + IndexType::BloomFilter, + &incompatible_params, + ) + .name("value_bloom_incompatible".to_string()) + .fragments(vec![fragments[1].id() as u32]) + .execute_uncommitted() + .await + .unwrap(); + let err = dataset + .merge_existing_index_segments(vec![staged[0].clone(), incompatible]) + .await + .unwrap_err(); + assert!( + err.to_string() + .contains("different parameters: number_of_items=4"), + "unexpected merge error: {err}" + ); + + let expected_dataset_version = staged + .iter() + .map(|segment| segment.dataset_version) + .min() + .unwrap(); + let merged = dataset.merge_existing_index_segments(staged).await.unwrap(); + assert_eq!(merged.dataset_version, expected_dataset_version); + assert_eq!( + merged.fragment_bitmap.as_ref().unwrap(), + dataset.fragment_bitmap.as_ref() + ); + assert!( + merged + .index_details + .as_ref() + .unwrap() + .type_url + .ends_with("BloomFilterIndexDetails") + ); + dataset + .commit_existing_index_segments("value_bloom_merged", "value", vec![merged]) + .await + .unwrap(); + let committed = dataset + .load_indices_by_name("value_bloom_merged") + .await + .unwrap(); + assert_eq!(committed[0].dataset_version, expected_dataset_version); + let merged_logical = crate::index::scalar_logical::open_named_scalar_index( + &dataset, + "value", + "value_bloom_merged", + &NoOpMetricsCollector, + ) + .await + .unwrap(); + let candidates = merged_logical + .search( + &BloomFilterQuery::Equals(ScalarValue::Int32(Some(17))), + &NoOpMetricsCollector, + ) + .await + .unwrap(); + assert!( + candidates + .row_addrs() + .true_rows() + .row_addrs() + .unwrap() + .map(u64::from) + .any(|row_addr| row_addr == (2_u64 << 32) + 1) + ); + + let result = dataset + .scan() + .filter("value = 17") + .unwrap() + .try_into_batch() + .await + .unwrap(); + assert_eq!(result.num_rows(), 1); + assert_eq!(result["value"].as_primitive::().value(0), 17); + } + + #[tokio::test] + async fn test_vector_execute_uncommitted_segments_commit_without_staging() { + let tmpdir = TempStrDir::default(); + let dataset_uri = format!("file://{}", tmpdir.as_str()); + + let reader = gen_batch() + .col("id", lance_datagen::array::step::()) + .col( + "vector", + lance_datagen::array::rand_vec::(lance_datagen::Dimension::from(16)), + ) + .into_reader_rows( + lance_datagen::RowCount::from(256), + lance_datagen::BatchCount::from(4), + ); + let mut dataset = Dataset::write( + reader, + &dataset_uri, + Some(WriteParams { + max_rows_per_file: 64, + mode: WriteMode::Overwrite, + ..Default::default() + }), + ) + .await + .unwrap(); + + let fragments = dataset.get_fragments(); + assert!(fragments.len() >= 2); + let params = VectorIndexParams::with_ivf_flat_params( + DistanceType::L2, + prepare_vector_ivf(&dataset, "vector").await, + ); + let mut input_segments = Vec::new(); + + for fragment in &fragments { + let segment = CreateIndexBuilder::new(&mut dataset, &["vector"], IndexType::Vector, ¶ms) .name("vector_idx".to_string()) .fragments(vec![fragment.id() as u32]) @@ -1809,6 +2289,330 @@ mod tests { assert!(result.num_rows() > 0); } + #[tokio::test] + async fn test_vector_explicit_all_fragments_uses_full_build_without_precomputed_ivf() { + let tmpdir = TempStrDir::default(); + let dataset_uri = format!("file://{}", tmpdir.as_str()); + let mut dataset = write_vector_fragment_dataset(&dataset_uri).await; + + let fragment_ids = dataset + .get_fragments() + .iter() + .map(|fragment| fragment.id() as u32) + .collect::>(); + assert!(fragment_ids.len() >= 2); + + let mut params = VectorIndexParams::ivf_pq(2, 8, 1, MetricType::L2, 10); + params.version(crate::index::vector::IndexFileVersion::Legacy); + let segment = + CreateIndexBuilder::new(&mut dataset, &["vector"], IndexType::Vector, ¶ms) + .name("vector_idx".to_string()) + .fragments(fragment_ids) + .execute_uncommitted() + .await + .unwrap(); + + assert_eq!( + segment.fragment_bitmap.as_ref().unwrap(), + dataset.fragment_bitmap.as_ref() + ); + } + + #[tokio::test] + async fn test_vector_precomputed_ivf_num_partitions_mismatch_errors() { + let tmpdir = TempStrDir::default(); + let dataset_uri = format!("file://{}", tmpdir.as_str()); + let mut dataset = write_vector_fragment_dataset(&dataset_uri).await; + + let mut ivf_params = prepare_vector_ivf(&dataset, "vector").await; + let centroid_count = ivf_params.centroids.as_ref().unwrap().len(); + ivf_params.num_partitions = Some(centroid_count + 1); + let params = VectorIndexParams::with_ivf_flat_params(DistanceType::L2, ivf_params); + + let err = CreateIndexBuilder::new(&mut dataset, &["vector"], IndexType::Vector, ¶ms) + .name("vector_idx".to_string()) + .execute_uncommitted() + .await + .unwrap_err(); + + assert!( + err.to_string().contains(&format!( + "num_partitions {} does not match precomputed IVF centroids length {}", + centroid_count + 1, + centroid_count + )), + "unexpected error: {err}" + ); + } + + #[tokio::test] + async fn test_vector_subset_legacy_ivf_pq_rejects_filtered_build() { + let tmpdir = TempStrDir::default(); + let dataset_uri = format!("file://{}", tmpdir.as_str()); + let mut dataset = write_vector_fragment_dataset(&dataset_uri).await; + + let fragments = dataset.get_fragments(); + assert!(fragments.len() >= 2); + let mut params = VectorIndexParams::ivf_pq(2, 8, 1, MetricType::L2, 10); + params.version(crate::index::vector::IndexFileVersion::Legacy); + + let err = CreateIndexBuilder::new(&mut dataset, &["vector"], IndexType::Vector, ¶ms) + .name("vector_idx".to_string()) + .fragments(vec![fragments[0].id() as u32]) + .execute_uncommitted() + .await + .unwrap_err(); + + assert!( + err.to_string() + .contains("filtered IVF_PQ builds do not support legacy format"), + "unexpected error: {err}" + ); + } + + #[tokio::test] + async fn test_vector_subset_fragments_train_without_precomputed_ivf() { + let tmpdir = TempStrDir::default(); + let dataset_uri = format!("file://{}", tmpdir.as_str()); + let mut dataset = write_vector_fragment_dataset(&dataset_uri).await; + + let fragments = dataset.get_fragments(); + assert!(fragments.len() >= 3); + let selected = vec![ + fragments[1].id() as u32, + fragments[0].id() as u32, + fragments[1].id() as u32, + ]; + let expected_bitmap = selected.iter().copied().collect::(); + + let params = VectorIndexParams::ivf_flat(2, DistanceType::L2); + let segment = + CreateIndexBuilder::new(&mut dataset, &["vector"], IndexType::Vector, ¶ms) + .name("vector_idx".to_string()) + .fragments(selected) + .execute_uncommitted() + .await + .unwrap(); + + assert_eq!(segment.fragment_bitmap.as_ref().unwrap(), &expected_bitmap); + } + + #[tokio::test] + async fn test_vector_merge_rejects_independently_trained_fragment_segments() { + let tmpdir = TempStrDir::default(); + let dataset_uri = format!("file://{}", tmpdir.as_str()); + let mut dataset = write_vector_fragment_dataset(&dataset_uri).await; + + let fragments = dataset.get_fragments(); + assert!(fragments.len() >= 2); + + let params = VectorIndexParams::ivf_flat(2, DistanceType::L2); + let mut segments = Vec::new(); + for fragment in fragments.iter().take(2) { + let segment = + CreateIndexBuilder::new(&mut dataset, &["vector"], IndexType::Vector, ¶ms) + .name("vector_idx".to_string()) + .fragments(vec![fragment.id() as u32]) + .execute_uncommitted() + .await + .unwrap(); + segments.push(segment); + } + + let err = dataset + .merge_existing_index_segments(segments) + .await + .unwrap_err(); + assert!( + err.to_string() + .contains("IVF centroids mismatch across shards"), + "unexpected error: {err}" + ); + } + + #[tokio::test] + async fn test_vector_optimize_rejects_independently_trained_fragment_segments() { + let tmpdir = TempStrDir::default(); + let dataset_uri = format!("file://{}", tmpdir.as_str()); + let mut dataset = write_vector_fragment_dataset(&dataset_uri).await; + + let fragments = dataset.get_fragments(); + assert!(fragments.len() >= 2); + + let params = VectorIndexParams::ivf_flat(2, DistanceType::L2); + let mut segments = Vec::new(); + for fragment in fragments.iter().take(2) { + let segment = + CreateIndexBuilder::new(&mut dataset, &["vector"], IndexType::Vector, ¶ms) + .name("vector_idx".to_string()) + .fragments(vec![fragment.id() as u32]) + .execute_uncommitted() + .await + .unwrap(); + segments.push(segment); + } + dataset + .commit_existing_index_segments("vector_idx", "vector", segments) + .await + .unwrap(); + + let version_before_optimize = dataset.version().version; + let segments_before_optimize = dataset.load_indices_by_name("vector_idx").await.unwrap(); + let err = dataset + .optimize_indices(&OptimizeOptions::merge(2)) + .await + .unwrap_err(); + assert!( + err.to_string() + .contains("vector index segments do not share IVF centroids"), + "unexpected error: {err}" + ); + assert_eq!(dataset.version().version, version_before_optimize); + assert_eq!( + dataset.load_indices_by_name("vector_idx").await.unwrap(), + segments_before_optimize, + "an incompatible merge must not partially commit index metadata" + ); + } + + #[tokio::test] + async fn test_vector_optimize_only_validates_requested_compatible_suffix() { + let tmpdir = TempStrDir::default(); + let dataset_uri = format!("file://{}", tmpdir.as_str()); + let mut dataset = write_vector_fragment_dataset(&dataset_uri).await; + let fragments = dataset.get_fragments(); + assert!(fragments.len() >= 4); + let tail_start = fragments.len() - 2; + + let mut independent_ivf = IvfBuildParams::new(2); + independent_ivf.max_iters = 7; + let independent_params = + VectorIndexParams::with_ivf_flat_params(DistanceType::L2, independent_ivf); + let independent_segment = CreateIndexBuilder::new( + &mut dataset, + &["vector"], + IndexType::Vector, + &independent_params, + ) + .name("vector_idx".to_string()) + .fragments( + fragments[..tail_start] + .iter() + .map(|fragment| fragment.id() as u32) + .collect(), + ) + .execute_uncommitted() + .await + .unwrap(); + + let mut shared_ivf = prepare_vector_ivf(&dataset, "vector").await; + shared_ivf.max_iters = 13; + let shared_params = VectorIndexParams::with_ivf_flat_params(DistanceType::L2, shared_ivf); + let mut compatible_tail = Vec::new(); + for fragment in fragments.iter().skip(tail_start) { + compatible_tail.push( + CreateIndexBuilder::new( + &mut dataset, + &["vector"], + IndexType::Vector, + &shared_params, + ) + .name("vector_idx".to_string()) + .fragments(vec![fragment.id() as u32]) + .execute_uncommitted() + .await + .unwrap(), + ); + } + + let independent_uuid = independent_segment.uuid; + let tail_uuids = compatible_tail + .iter() + .map(|segment| segment.uuid) + .collect::>(); + let expected_merged_fragments = compatible_tail + .iter() + .flat_map(|segment| segment.fragment_bitmap.as_ref().unwrap().iter()) + .collect::(); + let expected_merged_details = compatible_tail.last().unwrap().index_details.clone(); + assert_ne!( + independent_segment.index_details, expected_merged_details, + "test setup must distinguish base and suffix metadata" + ); + let mut segments = vec![independent_segment]; + segments.extend(compatible_tail); + dataset + .commit_existing_index_segments("vector_idx", "vector", segments) + .await + .unwrap(); + + dataset + .optimize_indices(&OptimizeOptions::merge(2)) + .await + .unwrap(); + + let optimized = dataset.load_indices_by_name("vector_idx").await.unwrap(); + assert_eq!(optimized.len(), 2); + assert!( + optimized + .iter() + .any(|segment| segment.uuid == independent_uuid), + "the incompatible base segment must not participate in the requested suffix merge" + ); + assert!( + tail_uuids + .iter() + .all(|uuid| optimized.iter().all(|segment| segment.uuid != *uuid)), + "both compatible suffix segments must be replaced" + ); + let merged = optimized + .iter() + .find(|segment| segment.uuid != independent_uuid) + .unwrap(); + assert_eq!( + merged.fragment_bitmap.as_ref().unwrap(), + &expected_merged_fragments + ); + assert_eq!( + merged.index_details, expected_merged_details, + "merged metadata must come from the selected suffix, not an incompatible base segment" + ); + } + + #[tokio::test] + async fn test_vector_empty_fragments_with_precomputed_ivf_builds_empty_segment() { + let tmpdir = TempStrDir::default(); + let dataset_uri = format!("file://{}", tmpdir.as_str()); + let mut dataset = write_vector_fragment_dataset(&dataset_uri).await; + + let mut ivf_params = prepare_vector_ivf(&dataset, "vector").await; + let expected_partitions = ivf_params.centroids.as_ref().unwrap().len(); + ivf_params.num_partitions = None; + let params = VectorIndexParams::with_ivf_flat_params(DistanceType::L2, ivf_params); + let segment = + CreateIndexBuilder::new(&mut dataset, &["vector"], IndexType::Vector, ¶ms) + .name("vector_idx".to_string()) + .fragments(vec![]) + .execute_uncommitted() + .await + .unwrap(); + + assert!(segment.fragment_bitmap.as_ref().unwrap().is_empty()); + dataset + .commit_existing_index_segments("vector_idx", "vector", vec![segment]) + .await + .unwrap(); + let logical_index = dataset + .open_logical_vector_index("vector", "vector_idx") + .await + .unwrap(); + let metadata = logical_index.metadatas().next().unwrap(); + assert_eq!( + logical_index.as_ivf().unwrap().num_partitions_per_segment(), + vec![(metadata.uuid, expected_partitions)] + ); + } + #[tokio::test] async fn test_commit_existing_index_segments_vector_commits_multi_segment_logical_index() { let tmpdir = TempStrDir::default(); @@ -2566,7 +3370,7 @@ mod tests { let mut legacy_segment = segment.clone(); legacy_segment.uuid = legacy_uuid; legacy_segment.index_version = LABEL_LIST_NULLS_MIN_VERSION; - legacy_segment.files = Some(vec![legacy_file]); + legacy_segment.files = Some(index_files_to_table(vec![legacy_file])); let err = dataset .merge_existing_index_segments(vec![legacy_segment]) @@ -2619,6 +3423,7 @@ mod tests { .await .unwrap(); + let source_dataset_version = dataset.manifest.version; dataset .commit_existing_index_segments( "vector_idx", @@ -2626,8 +3431,10 @@ mod tests { vec![IndexSegment::new( uuid, dataset.fragment_bitmap.as_ref().clone(), + [dataset.schema().field("vector").unwrap().id], Arc::new(vector_index_details(¶ms)), IndexType::IvfHnswFlat.version(), + source_dataset_version, )], ) .await @@ -2640,6 +3447,27 @@ mod tests { indices[0].fragment_bitmap.as_ref().unwrap(), dataset.fragment_bitmap.as_ref() ); + assert_eq!(indices[0].dataset_version, source_dataset_version); + + let err = dataset + .commit_existing_index_segments( + "future_vector_idx", + "vector", + vec![IndexSegment::new( + Uuid::new_v4(), + dataset.fragment_bitmap.as_ref().clone(), + [dataset.schema().field("vector").unwrap().id], + Arc::new(vector_index_details(¶ms)), + IndexType::IvfHnswFlat.version(), + dataset.manifest.version + 1, + )], + ) + .await + .unwrap_err(); + assert!( + err.to_string().contains("future dataset version"), + "unexpected error: {err}" + ); } #[tokio::test] @@ -2690,7 +3518,7 @@ mod tests { } #[tokio::test] - async fn test_create_index_ivf_rq_preserves_index_version_on_segment_commit_path() { + async fn test_create_index_ivf_rq_preserves_index_version() { let tmpdir = TempStrDir::default(); let dataset_uri = format!("file://{}", tmpdir.as_str()); diff --git a/rust/lance/src/index/mem_wal.rs b/rust/lance/src/index/mem_wal.rs index de2c70b62d2..8f90fd7d90b 100644 --- a/rust/lance/src/index/mem_wal.rs +++ b/rust/lance/src/index/mem_wal.rs @@ -5,18 +5,19 @@ //! //! The MemWAL Index stores: //! - Configuration (sharding_specs, maintained_indexes) -//! - Merge progress (merged_generations per shard) +//! - SSTable compaction progress //! - Shard state snapshots (eventually consistent) //! //! Writers no longer update the index on every write. Instead, they update //! shard manifests directly. This module provides functions to: //! - Load the MemWAL index -//! - Update merged generations (called during merge-insert commits) +//! - Update compacted SSTables (called during merge-insert commits) +use std::collections::HashSet; use std::sync::Arc; use lance_core::{Error, Result}; -use lance_index::mem_wal::{MEM_WAL_INDEX_NAME, MemWalIndex, MemWalIndexDetails, MergedGeneration}; +use lance_index::mem_wal::{CompactedSsTable, MEM_WAL_INDEX_NAME, MemWalIndex, MemWalIndexDetails}; use lance_table::format::{IndexMetadata, pb}; use uuid::Uuid; @@ -45,56 +46,86 @@ pub(crate) fn open_mem_wal_index(index: IndexMetadata) -> Result, +/// Update `compacted_sstables` in the MemWAL index. +/// +/// Called from the final data-changing merge-insert commit for a compaction +/// target, so the rows and the generation that describes them publish +/// together. +/// +/// A proposed generation must be **strictly greater** than the one the latest +/// state records for that shard, and a stale one fails the whole transaction. +/// Accepting it while keeping the larger marker would publish that worker's row +/// mutations under a generation it did not produce, and anything reading only +/// the marker could then stop serving SSTables whose rows were never inserted. +/// +/// Every other `MemWalIndexDetails` field is carried through untouched. +pub(crate) fn update_mem_wal_index_compacted_sstables( + indices: &mut [IndexMetadata], dataset_version: u64, - new_merged_generations: Vec, + new_compacted_sstables: Vec, ) -> Result<()> { - if new_merged_generations.is_empty() { + if new_compacted_sstables.is_empty() { return Ok(()); } + let mut seen_shards = HashSet::with_capacity(new_compacted_sstables.len()); + for sstable in &new_compacted_sstables { + if !seen_shards.insert(sstable.shard_id) { + return Err(Error::invalid_input(format!( + "Duplicate shard {} in one SSTable compaction update; each shard \ + may advance at most once per transaction", + sstable.shard_id + ))); + } + } + + // Default details would describe a table with no MemWAL shards at all, so + // the recorded generation would name a shard nothing can corroborate. + // Refuse instead of inventing metadata. let pos = indices .iter() - .position(|idx| idx.name == MEM_WAL_INDEX_NAME); - - let new_meta = if let Some(pos) = pos { - let current_meta = indices.remove(pos); - let mut details = load_mem_wal_index_details(current_meta)?; - - // Update merged_generations - for each shard, keep the higher generation - for new_mg in new_merged_generations { - if let Some(existing) = details - .merged_generations - .iter_mut() - .find(|mg| mg.shard_id == new_mg.shard_id) - { - if new_mg.generation > existing.generation { - existing.generation = new_mg.generation; - } - } else { - details.merged_generations.push(new_mg); + .position(|idx| idx.name == MEM_WAL_INDEX_NAME) + .ok_or_else(|| { + Error::invalid_input(format!( + "Cannot record SSTable compaction progress: the {} system index \ + does not exist on this table", + MEM_WAL_INDEX_NAME + )) + })?; + + // Validated against a copy so a rejected update leaves `indices` exactly as + // the caller passed it. + let mut details = load_mem_wal_index_details(indices[pos].clone())?; + + for new_sstable in new_compacted_sstables { + match details + .compacted_sstables + .iter_mut() + .find(|sstable| sstable.shard_id == new_sstable.shard_id) + { + Some(existing) if new_sstable.generation <= existing.generation => { + return Err(Error::invalid_input(format!( + "Stale SSTable compaction for shard {}: proposed generation {} \ + is not greater than the recorded generation {}", + new_sstable.shard_id, new_sstable.generation, existing.generation + ))); } + Some(existing) => existing.generation = new_sstable.generation, + None => details.compacted_sstables.push(new_sstable), } + } - new_mem_wal_index_meta(dataset_version, details)? - } else { - // Create new MemWAL index with just the merged generations - let details = MemWalIndexDetails { - merged_generations: new_merged_generations, - ..Default::default() - }; - new_mem_wal_index_meta(dataset_version, details)? - }; - - indices.push(new_meta); + // Replaced in place so the index list keeps its order. + indices[pos] = new_mem_wal_index_meta(dataset_version, details)?; Ok(()) } /// Create a new MemWAL index metadata entry. +/// +/// A fresh UUID is minted on every rewrite, including metadata-only updates. +/// The decoded-details cache is keyed on that UUID, so the change of identity +/// is what invalidates it; holding the UUID steady would leave a warmed reader +/// answering with the state from before the update. pub(crate) fn new_mem_wal_index_meta( dataset_version: u64, details: MemWalIndexDetails, @@ -120,6 +151,7 @@ pub(crate) fn new_mem_wal_index_meta( mod tests { use super::*; + use std::collections::HashMap; use std::sync::Arc; use crate::index::DatasetIndexExt; @@ -152,18 +184,77 @@ mod tests { .unwrap() } + /// A dataset with `__lance_mem_wal` already installed, as MemWAL + /// initialization leaves it. Recording compaction progress requires the + /// system index to exist, so any test that commits progress needs this + /// rather than a bare [`test_dataset`]. + async fn test_dataset_with_mem_wal() -> crate::Dataset { + let dataset = test_dataset().await; + let mem_wal_index = + new_mem_wal_index_meta(dataset.manifest.version, MemWalIndexDetails::default()) + .unwrap(); + let txn = Transaction::new( + dataset.manifest.version, + Operation::CreateIndex { + new_indices: vec![mem_wal_index], + removed_indices: vec![], + }, + None, + ); + CommitBuilder::new(Arc::new(dataset)) + .execute(txn) + .await + .unwrap() + } + + /// UpdateMemWalState touches indexes, not data, so it must carry the + /// fragment list forward. The operation builds its manifest from scratch, + /// and an unpopulated fragment list is published as an empty one. + #[tokio::test] + async fn test_update_mem_wal_state_preserves_fragments() { + let dataset = test_dataset_with_mem_wal().await; + let rows_before = dataset.count_rows(None).await.unwrap(); + let fragments_before: Vec = dataset.fragments().iter().map(|f| f.id).collect(); + assert!(rows_before > 0, "precondition: the table holds rows"); + + let txn = Transaction::new( + dataset.manifest.version, + Operation::UpdateMemWalState { + compacted_sstables: vec![CompactedSsTable::new(Uuid::new_v4(), 1)], + require_index_catchup: false, + }, + None, + ); + let dataset = CommitBuilder::new(Arc::new(dataset)) + .execute(txn) + .await + .unwrap(); + + assert_eq!( + dataset.fragments().iter().map(|f| f.id).collect::>(), + fragments_before, + "UpdateMemWalState dropped fragments" + ); + assert_eq!( + dataset.count_rows(None).await.unwrap(), + rows_before, + "UpdateMemWalState dropped rows" + ); + } + /// Test that UpdateMemWalState with lower generation than committed fails without retry. /// Per spec: If committed_generation >= to_commit_generation, abort without retry. #[tokio::test] async fn test_update_mem_wal_state_conflict_lower_generation_no_retry() { - let dataset = test_dataset().await; + let dataset = test_dataset_with_mem_wal().await; let shard = Uuid::new_v4(); // First commit UpdateMemWalState with generation 10 let txn1 = Transaction::new( dataset.manifest.version, Operation::UpdateMemWalState { - merged_generations: vec![MergedGeneration::new(shard, 10)], + compacted_sstables: vec![CompactedSsTable::new(shard, 10)], + require_index_catchup: false, }, None, ); @@ -177,7 +268,8 @@ mod tests { let txn2 = Transaction::new( dataset.manifest.version - 1, // Based on old version Operation::UpdateMemWalState { - merged_generations: vec![MergedGeneration::new(shard, 5)], + compacted_sstables: vec![CompactedSsTable::new(shard, 5)], + require_index_catchup: false, }, None, ); @@ -193,14 +285,15 @@ mod tests { /// Test that UpdateMemWalState with equal generation as committed fails without retry. #[tokio::test] async fn test_update_mem_wal_state_conflict_equal_generation_no_retry() { - let dataset = test_dataset().await; + let dataset = test_dataset_with_mem_wal().await; let shard = Uuid::new_v4(); // First commit UpdateMemWalState with generation 10 let txn1 = Transaction::new( dataset.manifest.version, Operation::UpdateMemWalState { - merged_generations: vec![MergedGeneration::new(shard, 10)], + compacted_sstables: vec![CompactedSsTable::new(shard, 10)], + require_index_catchup: false, }, None, ); @@ -213,7 +306,8 @@ mod tests { let txn2 = Transaction::new( dataset.manifest.version - 1, // Based on old version Operation::UpdateMemWalState { - merged_generations: vec![MergedGeneration::new(shard, 10)], + compacted_sstables: vec![CompactedSsTable::new(shard, 10)], + require_index_catchup: false, }, None, ); @@ -230,14 +324,15 @@ mod tests { /// Per spec: If committed_generation < to_commit_generation, retry is allowed. #[tokio::test] async fn test_update_mem_wal_state_conflict_higher_generation_retryable() { - let dataset = test_dataset().await; + let dataset = test_dataset_with_mem_wal().await; let shard = Uuid::new_v4(); // First commit UpdateMemWalState with generation 5 let txn1 = Transaction::new( dataset.manifest.version, Operation::UpdateMemWalState { - merged_generations: vec![MergedGeneration::new(shard, 5)], + compacted_sstables: vec![CompactedSsTable::new(shard, 5)], + require_index_catchup: false, }, None, ); @@ -251,7 +346,8 @@ mod tests { let txn2 = Transaction::new( dataset.manifest.version - 1, // Based on old version Operation::UpdateMemWalState { - merged_generations: vec![MergedGeneration::new(shard, 10)], + compacted_sstables: vec![CompactedSsTable::new(shard, 10)], + require_index_catchup: false, }, None, ); @@ -267,7 +363,7 @@ mod tests { /// Test that UpdateMemWalState on different shards don't conflict. #[tokio::test] async fn test_update_mem_wal_state_different_shards_no_conflict() { - let dataset = test_dataset().await; + let dataset = test_dataset_with_mem_wal().await; let shard1 = Uuid::new_v4(); let shard2 = Uuid::new_v4(); @@ -275,7 +371,8 @@ mod tests { let txn1 = Transaction::new( dataset.manifest.version, Operation::UpdateMemWalState { - merged_generations: vec![MergedGeneration::new(shard1, 10)], + compacted_sstables: vec![CompactedSsTable::new(shard1, 10)], + require_index_catchup: false, }, None, ); @@ -289,7 +386,8 @@ mod tests { let txn2 = Transaction::new( dataset.manifest.version - 1, // Based on old version Operation::UpdateMemWalState { - merged_generations: vec![MergedGeneration::new(shard2, 5)], + compacted_sstables: vec![CompactedSsTable::new(shard2, 5)], + require_index_catchup: false, }, None, ); @@ -312,21 +410,22 @@ mod tests { .unwrap() .clone(); let details = load_mem_wal_index_details(mem_wal_idx).unwrap(); - assert_eq!(details.merged_generations.len(), 2); + assert_eq!(details.compacted_sstables.len(), 2); } /// Test that CreateIndex of MemWalIndex can be rebased against UpdateMemWalState. - /// The merged_generations from UpdateMemWalState should be merged into CreateIndex. + /// The compacted_sstables from UpdateMemWalState should be included in CreateIndex. #[tokio::test] async fn test_create_index_rebase_against_update_mem_wal_state() { - let dataset = test_dataset().await; + let dataset = test_dataset_with_mem_wal().await; let shard = Uuid::new_v4(); // First commit UpdateMemWalState with generation 10 let txn1 = Transaction::new( dataset.manifest.version, Operation::UpdateMemWalState { - merged_generations: vec![MergedGeneration::new(shard, 10)], + compacted_sstables: vec![CompactedSsTable::new(shard, 10)], + require_index_catchup: false, }, None, ); @@ -336,7 +435,7 @@ mod tests { .unwrap(); // CreateIndex of MemWalIndex based on old version (before UpdateMemWalState) - // This should succeed and merge the generations + // This should succeed and combine the compaction progress. let details = MemWalIndexDetails { num_shards: 1, ..Default::default() @@ -359,7 +458,7 @@ mod tests { result ); - // Verify the merged_generations from UpdateMemWalState were merged into CreateIndex + // Verify the compacted_sstables from UpdateMemWalState were included in CreateIndex let dataset = result.unwrap(); let mem_wal_idx = dataset .load_indices() @@ -370,9 +469,9 @@ mod tests { .unwrap() .clone(); let details = load_mem_wal_index_details(mem_wal_idx).unwrap(); - assert_eq!(details.merged_generations.len(), 1); - assert_eq!(details.merged_generations[0].shard_id, shard); - assert_eq!(details.merged_generations[0].generation, 10); + assert_eq!(details.compacted_sstables.len(), 1); + assert_eq!(details.compacted_sstables[0].shard_id, shard); + assert_eq!(details.compacted_sstables[0].generation, 10); assert_eq!(details.num_shards, 1); // Config from CreateIndex preserved } @@ -382,9 +481,9 @@ mod tests { let dataset = test_dataset().await; let shard = Uuid::new_v4(); - // First commit CreateIndex of MemWalIndex with merged_generations + // First commit CreateIndex of MemWalIndex with compacted_sstables let details = MemWalIndexDetails { - merged_generations: vec![MergedGeneration::new(shard, 10)], + compacted_sstables: vec![CompactedSsTable::new(shard, 10)], ..Default::default() }; let mem_wal_index = new_mem_wal_index_meta(dataset.manifest.version, details).unwrap(); @@ -406,7 +505,8 @@ mod tests { let txn2 = Transaction::new( dataset.manifest.version - 1, // Based on old version Operation::UpdateMemWalState { - merged_generations: vec![MergedGeneration::new(shard, 5)], + compacted_sstables: vec![CompactedSsTable::new(shard, 5)], + require_index_catchup: false, }, None, ); @@ -419,75 +519,776 @@ mod tests { ); } + /// The bit must survive being written and read back, not just + /// `build_manifest`. `apply_feature_flags` runs a second time inside + /// `write_manifest_file`, so a version of this that stops at the in-memory + /// manifest passes while the stored table stays legacy. + #[tokio::test] + async fn required_catch_up_survives_a_persisted_round_trip() { + use crate::dataset::mem_wal::DatasetMemWalExt; + use lance_table::feature_flags::FLAG_MEM_WAL_INDEX_CATCHUP; + + // A real directory, not `memory://`: reopening by URI builds a fresh + // store registry, and the point of this test is to read back what was + // actually written. + let dir = tempfile::tempdir().unwrap(); + let uri = dir.path().to_str().unwrap(); + let write_params = WriteParams { + max_rows_per_file: 10, + ..Default::default() + }; + let data = RecordBatch::try_new( + Arc::new(Schema::new(vec![Field::new("a", DataType::Int32, false)])), + vec![Arc::new(Int32Array::from_iter_values(0..10_i32))], + ) + .unwrap(); + let dataset = InsertBuilder::new(uri) + .with_params(&write_params) + .execute(vec![data]) + .await + .unwrap(); + + // Install the system index, then require catch-up. + let mem_wal_index = + new_mem_wal_index_meta(dataset.manifest.version, MemWalIndexDetails::default()) + .unwrap(); + let txn = Transaction::new( + dataset.manifest.version, + Operation::CreateIndex { + new_indices: vec![mem_wal_index], + removed_indices: vec![], + }, + None, + ); + let mut dataset = CommitBuilder::new(Arc::new(dataset)) + .execute(txn) + .await + .unwrap(); + dataset.require_mem_wal_index_catchup().await.unwrap(); + + let reopened = crate::dataset::builder::DatasetBuilder::from_uri(uri) + .load() + .await + .unwrap(); + assert_ne!( + reopened.manifest.reader_feature_flags & FLAG_MEM_WAL_INDEX_CATCHUP, + 0, + "activation did not reach storage" + ); + assert_ne!( + reopened.manifest.writer_feature_flags & FLAG_MEM_WAL_INDEX_CATCHUP, + 0, + "activation did not reach storage" + ); + + // An ordinary commit must not walk it back. + let txn = Transaction::new( + reopened.manifest.version, + Operation::UpdateConfig { + config_updates: Some(crate::dataset::transaction::UpdateMap { + update_entries: vec![crate::dataset::transaction::UpdateMapEntry { + key: "k".to_string(), + value: Some("v".to_string()), + }], + replace: false, + }), + table_metadata_updates: None, + schema_metadata_updates: None, + field_metadata_updates: HashMap::new(), + }, + None, + ); + CommitBuilder::new(Arc::new(reopened)) + .execute(txn) + .await + .unwrap(); + + let reopened = crate::dataset::builder::DatasetBuilder::from_uri(uri) + .load() + .await + .unwrap(); + assert_ne!( + reopened.manifest.reader_feature_flags & FLAG_MEM_WAL_INDEX_CATCHUP, + 0, + "an ordinary commit downgraded the stored table" + ); + assert_ne!( + reopened.manifest.writer_feature_flags & FLAG_MEM_WAL_INDEX_CATCHUP, + 0, + "an ordinary commit downgraded the stored table" + ); + } + + /// A table on the catch-up protocol with `generation` folded into base. + /// + /// Activation must precede any compaction progress: it refuses a table that + /// already has some, since nothing can prove which indexes hold it. + async fn activated_dataset(shard: Uuid, generation: u64) -> crate::Dataset { + use crate::dataset::mem_wal::DatasetMemWalExt; + + let mut dataset = test_dataset_with_mem_wal().await; + dataset.require_mem_wal_index_catchup().await.unwrap(); + let txn = Transaction::new( + dataset.manifest.version, + Operation::UpdateMemWalState { + compacted_sstables: vec![CompactedSsTable::new(shard, generation)], + require_index_catchup: false, + }, + None, + ); + CommitBuilder::new(Arc::new(dataset)) + .execute(txn) + .await + .unwrap() + } + + /// An index segment spanning `fragments`, as a completed build leaves. + fn index_over(name: &str, fragments: &[u32]) -> IndexMetadata { + IndexMetadata { + uuid: Uuid::new_v4(), + name: name.to_string(), + fields: vec![0], + dataset_version: 1, + fragment_bitmap: Some(roaring::RoaringBitmap::from_iter(fragments.iter().copied())), + index_details: None, + index_version: 0, + created_at: None, + base_id: None, + files: None, + } + } + + async fn catch_up_generation(dataset: &crate::Dataset, index: &str) -> Option { + let meta = dataset + .load_indices() + .await + .unwrap() + .iter() + .find(|idx| idx.name == MEM_WAL_INDEX_NAME) + .unwrap() + .clone(); + load_mem_wal_index_details(meta) + .unwrap() + .index_catchup + .into_iter() + .find(|entry| entry.index_name == index) + .and_then(|entry| entry.caught_up_generations.first().map(|g| g.generation)) + } + + /// The commit path, not the derivation in isolation: `commit_transaction` + /// has to load the read version's indices and hand them down, and it only + /// does so for tables carrying the feature bit. + #[tokio::test] + async fn an_index_covering_the_table_earns_catch_up_on_commit() { + let shard = Uuid::new_v4(); + let dataset = activated_dataset(shard, 5).await; + + let txn = Transaction::new( + dataset.manifest.version, + Operation::CreateIndex { + new_indices: vec![index_over("idx", &[0])], + removed_indices: vec![], + }, + None, + ); + let dataset = CommitBuilder::new(Arc::new(dataset)) + .execute(txn) + .await + .unwrap(); + + assert_eq!(catch_up_generation(&dataset, "idx").await, Some(5)); + } + + /// A repair with nothing to rebuild still has to commit. + /// + /// Coverage is derived at commit time, so an index that already spans the + /// table records its position only if there is a commit to record it on. + /// That is the ordinary case after a remap, or after a compaction that + /// advanced a generation without changing which fragments exist: the + /// optimize finds no unindexed fragments and has no new segment to publish. + /// Skipping the commit there leaves the position missing forever, the + /// scheduler repeating the same repair, and the last SSTable unretirable. + #[tokio::test] + async fn a_no_work_repair_still_records_derived_catch_up() { + use lance_index::optimize::OptimizeOptions; + + let shard = Uuid::new_v4(); + let dataset = activated_dataset(shard, 7).await; + // Already spans the table, so the optimize below has nothing to build. + let txn = Transaction::new( + dataset.manifest.version, + Operation::CreateIndex { + new_indices: vec![index_over("idx", &[0])], + removed_indices: vec![], + }, + None, + ); + let dataset = CommitBuilder::new(Arc::new(dataset)) + .execute(txn) + .await + .unwrap(); + + // Compaction advances without adding a fragment, so the index still + // covers and the optimize stays a no-op. + let txn = Transaction::new( + dataset.manifest.version, + Operation::UpdateMemWalState { + compacted_sstables: vec![CompactedSsTable::new(shard, 9)], + require_index_catchup: false, + }, + None, + ); + let mut dataset = CommitBuilder::new(Arc::new(dataset)) + .execute(txn) + .await + .unwrap(); + + dataset + .optimize_indices(&OptimizeOptions::append().index_names(vec!["idx".to_string()])) + .await + .unwrap(); + + assert_eq!(catch_up_generation(&dataset, "idx").await, Some(9)); + + // And once it is current, the next pass must not commit again: + // periodic maintenance would otherwise mint a version forever. + let after_repair = dataset.manifest.version; + dataset + .optimize_indices(&OptimizeOptions::append().index_names(vec!["idx".to_string()])) + .await + .unwrap(); + assert_eq!(dataset.manifest.version, after_repair); + } + + /// A no-op optimize on a table that is not on the protocol must stay a + /// no-op: the early return is what keeps ordinary tables from committing an + /// empty version on every maintenance pass. + #[tokio::test] + async fn a_no_work_optimize_off_protocol_commits_nothing() { + use lance_index::optimize::OptimizeOptions; + + let dataset = test_dataset_with_mem_wal().await; + let txn = Transaction::new( + dataset.manifest.version, + Operation::CreateIndex { + new_indices: vec![index_over("idx", &[0])], + removed_indices: vec![], + }, + None, + ); + let mut dataset = CommitBuilder::new(Arc::new(dataset)) + .execute(txn) + .await + .unwrap(); + let before = dataset.manifest.version; + + dataset + .optimize_indices(&OptimizeOptions::append().index_names(vec!["idx".to_string()])) + .await + .unwrap(); + + assert_eq!(dataset.manifest.version, before); + } + + /// A legacy table reads a missing entry as "fully caught up". Writing one + /// there would be the first step toward a trim it never agreed to, so the + /// commit path must not even look. + #[tokio::test] + async fn a_legacy_table_earns_nothing_on_commit() { + let dataset = test_dataset_with_mem_wal().await; + let shard = Uuid::new_v4(); + let txn = Transaction::new( + dataset.manifest.version, + Operation::UpdateMemWalState { + compacted_sstables: vec![CompactedSsTable::new(shard, 5)], + require_index_catchup: false, + }, + None, + ); + let dataset = CommitBuilder::new(Arc::new(dataset)) + .execute(txn) + .await + .unwrap(); + + let txn = Transaction::new( + dataset.manifest.version, + Operation::CreateIndex { + new_indices: vec![index_over("idx", &[0])], + removed_indices: vec![], + }, + None, + ); + let dataset = CommitBuilder::new(Arc::new(dataset)) + .execute(txn) + .await + .unwrap(); + + assert_eq!(catch_up_generation(&dataset, "idx").await, None); + } + + /// What a commit earns is fixed by the version it read, and a rebase does + /// not move it. The builder inspected a one-fragment table with generation + /// 5 folded in; by the time it commits, an append has landed. It still + /// earns 5 -- judged against the table it never saw, it would earn nothing + /// and the SSTables would be retained forever. + #[tokio::test] + async fn credit_is_anchored_to_the_read_version_across_a_rebase() { + let shard = Uuid::new_v4(); + let dataset = activated_dataset(shard, 5).await; + let read_version = dataset.manifest.version; + + let data = RecordBatch::try_new( + Arc::new(Schema::from(dataset.schema())), + vec![ + Arc::new(Int32Array::from_iter_values(10..20_i32)), + Arc::new(Int32Array::from_iter_values(std::iter::repeat_n(0, 10))), + ], + ) + .unwrap(); + let dataset = InsertBuilder::new(Arc::new(dataset)) + .with_params(&WriteParams { + mode: crate::dataset::WriteMode::Append, + max_rows_per_file: 10, + ..Default::default() + }) + .execute(vec![data]) + .await + .unwrap(); + assert!( + dataset.get_fragments().len() > 1, + "the append must add a fragment the index has not seen" + ); + + // Built against `read_version`, and only ever saw fragment 0. + let txn = Transaction::new( + read_version, + Operation::CreateIndex { + new_indices: vec![index_over("idx", &[0])], + removed_indices: vec![], + }, + None, + ); + let dataset = CommitBuilder::new(Arc::new(dataset)) + .execute(txn) + .await + .unwrap(); + + assert_eq!(catch_up_generation(&dataset, "idx").await, Some(5)); + } + + /// A user index build that races a compaction commit is rejected outright + /// rather than rebased -- only the system index may rebase against + /// `UpdateMemWalState`. Anything scheduling catch-up work has to expect the + /// build to be thrown away and retried, so a busy shard needs the two kept + /// apart rather than merely retried. + #[tokio::test] + async fn a_user_index_build_cannot_rebase_past_a_compaction_commit() { + let shard = Uuid::new_v4(); + let dataset = activated_dataset(shard, 5).await; + let read_version = dataset.manifest.version; + + let txn = Transaction::new( + dataset.manifest.version, + Operation::UpdateMemWalState { + compacted_sstables: vec![CompactedSsTable::new(shard, 9)], + require_index_catchup: false, + }, + None, + ); + let dataset = CommitBuilder::new(Arc::new(dataset)) + .execute(txn) + .await + .unwrap(); + + let txn = Transaction::new( + read_version, + Operation::CreateIndex { + new_indices: vec![index_over("idx", &[0])], + removed_indices: vec![], + }, + None, + ); + let err = CommitBuilder::new(Arc::new(dataset)) + .execute(txn) + .await + .unwrap_err(); + + assert!( + err.to_string().contains("incompatible"), + "expected an incompatible-transaction error, got {err}" + ); + } + + /// One `__lance_mem_wal` entry carrying `details`, as a real table has. + fn indices_with(details: MemWalIndexDetails) -> Vec { + vec![new_mem_wal_index_meta(1, details).unwrap()] + } + + fn compacted_generation(indices: &[IndexMetadata], shard: Uuid) -> Option { + load_mem_wal_index_details(indices[0].clone()) + .unwrap() + .compacted_sstables + .iter() + .find(|sstable| sstable.shard_id == shard) + .map(|sstable| sstable.generation) + } + #[test] - fn test_update_merged_generations() { - let mut indices = Vec::new(); + fn test_update_compacted_sstables() { let shard1 = Uuid::new_v4(); let shard2 = Uuid::new_v4(); + let mut indices = indices_with(MemWalIndexDetails::default()); - // First update - creates new index - update_mem_wal_index_merged_generations( + update_mem_wal_index_compacted_sstables( &mut indices, 1, - vec![MergedGeneration::new(shard1, 5)], + vec![CompactedSsTable::new(shard1, 5)], ) .unwrap(); - assert_eq!(indices.len(), 1); - let details = load_mem_wal_index_details(indices[0].clone()).unwrap(); - assert_eq!(details.merged_generations.len(), 1); - assert_eq!(details.merged_generations[0].shard_id, shard1); - assert_eq!(details.merged_generations[0].generation, 5); + assert_eq!(compacted_generation(&indices, shard1), Some(5)); - // Second update - updates existing shard - update_mem_wal_index_merged_generations( + // Advancing an existing shard. + update_mem_wal_index_compacted_sstables( &mut indices, 2, - vec![MergedGeneration::new(shard1, 10)], + vec![CompactedSsTable::new(shard1, 10)], ) .unwrap(); + assert_eq!(compacted_generation(&indices, shard1), Some(10)); - assert_eq!(indices.len(), 1); - let details = load_mem_wal_index_details(indices[0].clone()).unwrap(); - assert_eq!(details.merged_generations.len(), 1); - assert_eq!(details.merged_generations[0].generation, 10); - - // Third update - adds new shard - update_mem_wal_index_merged_generations( + // A second shard is independent. + update_mem_wal_index_compacted_sstables( &mut indices, 3, - vec![MergedGeneration::new(shard2, 3)], + vec![CompactedSsTable::new(shard2, 3)], ) .unwrap(); + assert_eq!(compacted_generation(&indices, shard1), Some(10)); + assert_eq!(compacted_generation(&indices, shard2), Some(3)); + } - assert_eq!(indices.len(), 1); - let details = load_mem_wal_index_details(indices[0].clone()).unwrap(); - assert_eq!(details.merged_generations.len(), 2); + /// A stale proposal must fail the whole transaction: accepting it while + /// keeping generation 10 would publish the stale worker's rows under a + /// marker they did not produce. + #[test] + fn a_lower_generation_rejects_the_whole_update() { + let shard = Uuid::new_v4(); + let mut indices = indices_with(MemWalIndexDetails { + compacted_sstables: vec![CompactedSsTable::new(shard, 10)], + ..Default::default() + }); + let before = indices.clone(); + + let err = update_mem_wal_index_compacted_sstables( + &mut indices, + 2, + vec![CompactedSsTable::new(shard, 8)], + ) + .unwrap_err(); + + assert!( + err.to_string().contains("Stale SSTable compaction"), + "{err}" + ); + assert_eq!( + indices[0].uuid, before[0].uuid, + "a rejected update must leave the index list untouched" + ); + assert_eq!(compacted_generation(&indices, shard), Some(10)); + } + + /// Equal is also refused: it reports nothing new, and accepting it would let + /// a retry publish a second set of row mutations under the same marker. + #[test] + fn an_equal_generation_rejects() { + let shard = Uuid::new_v4(); + let mut indices = indices_with(MemWalIndexDetails { + compacted_sstables: vec![CompactedSsTable::new(shard, 10)], + ..Default::default() + }); + + let err = update_mem_wal_index_compacted_sstables( + &mut indices, + 2, + vec![CompactedSsTable::new(shard, 10)], + ) + .unwrap_err(); + + assert!( + err.to_string().contains("Stale SSTable compaction"), + "{err}" + ); + } + + #[test] + fn duplicate_shards_in_one_update_reject() { + let shard = Uuid::new_v4(); + let mut indices = indices_with(MemWalIndexDetails::default()); + + let err = update_mem_wal_index_compacted_sstables( + &mut indices, + 1, + vec![ + CompactedSsTable::new(shard, 5), + CompactedSsTable::new(shard, 6), + ], + ) + .unwrap_err(); + + assert!(err.to_string().contains("Duplicate shard"), "{err}"); + } + + /// Absent metadata must not be materialized: default details describe a + /// table with no MemWAL shards, so the recorded generation would name a + /// shard nothing can corroborate. + #[test] + fn a_missing_mem_wal_index_rejects() { + let mut indices: Vec = Vec::new(); + + let err = update_mem_wal_index_compacted_sstables( + &mut indices, + 1, + vec![CompactedSsTable::new(Uuid::new_v4(), 5)], + ) + .unwrap_err(); + + assert!(err.to_string().contains("does not exist"), "{err}"); + assert!(indices.is_empty(), "nothing should have been created"); + } + + /// Recording progress replaces the entry where it sits, so the index list + /// keeps its order. + #[test] + fn recording_progress_keeps_the_system_index_position() { + let shard = Uuid::new_v4(); + let mut indices = indices_with(MemWalIndexDetails::default()); + // A neighbour to show the entry is replaced in place, not moved. + indices.push(IndexMetadata { + name: "other_index".to_string(), + ..indices[0].clone() + }); + update_mem_wal_index_compacted_sstables( + &mut indices, + 2, + vec![CompactedSsTable::new(shard, 5)], + ) + .unwrap(); + + assert_eq!(indices[0].name, MEM_WAL_INDEX_NAME); + assert_eq!(indices[1].name, "other_index"); + } - // Fourth update - lower generation should not update - update_mem_wal_index_merged_generations( + /// Recording progress must not disturb any other saved MemWAL field. + #[test] + fn recording_progress_preserves_unrelated_mem_wal_state() { + let shard = Uuid::new_v4(); + let other_shard = Uuid::new_v4(); + let mut writer_config_defaults = HashMap::new(); + writer_config_defaults.insert("target_size".to_string(), "64MB".to_string()); + let details = MemWalIndexDetails { + snapshot_ts_millis: 12_345, + num_shards: 4, + maintained_indexes: vec!["vector_idx".to_string()], + compacted_sstables: vec![CompactedSsTable::new(other_shard, 7)], + writer_config_defaults: writer_config_defaults.clone(), + ..Default::default() + }; + let mut indices = indices_with(details); + + update_mem_wal_index_compacted_sstables( &mut indices, - 4, - vec![MergedGeneration::new(shard1, 8)], // lower than 10 + 2, + vec![CompactedSsTable::new(shard, 5)], ) .unwrap(); - let details = load_mem_wal_index_details(indices[0].clone()).unwrap(); - let r1_mg = details - .merged_generations - .iter() - .find(|mg| mg.shard_id == shard1) + let after = load_mem_wal_index_details(indices[0].clone()).unwrap(); + assert_eq!(after.snapshot_ts_millis, 12_345); + assert_eq!(after.num_shards, 4); + assert_eq!(after.maintained_indexes, vec!["vector_idx".to_string()]); + assert_eq!(after.writer_config_defaults, writer_config_defaults); + // The untouched shard keeps its generation. + assert_eq!(compacted_generation(&indices, other_shard), Some(7)); + } + + /// The hole this guards is a worker that refreshed to HEAD before + /// submitting: there is no intervening transaction, so the conflict + /// resolver sees nothing to reject and only apply-time validation stands + /// between a stale generation and a published commit. + /// + /// The row mutation is the point: were the stale generation accepted, this + /// commit would add a fragment while the recorded generation stayed at 10, + /// so the rows and the number describing them would disagree. + #[tokio::test] + async fn a_stale_generation_against_latest_head_publishes_nothing() { + use lance_table::format::Fragment; + + let shard = Uuid::new_v4(); + let dataset = test_dataset_with_mem_wal().await; + let version = dataset.manifest.version; + + // Record generation 10. + let dataset = CommitBuilder::new(Arc::new(dataset)) + .execute(Transaction::new( + version, + Operation::UpdateMemWalState { + compacted_sstables: vec![CompactedSsTable::new(shard, 10)], + require_index_catchup: false, + }, + None, + )) + .await .unwrap(); - assert_eq!(r1_mg.generation, 10); // Should still be 10 + + let version_before = dataset.manifest.version; + let fragments_before = dataset.get_fragments().len(); + + // Built against the LATEST version, so there is nothing stale for the + // conflict resolver to catch, and carrying a real row mutation. + let stale = Transaction::new( + version_before, + Operation::Update { + removed_fragment_ids: vec![], + updated_fragments: vec![], + new_fragments: vec![Fragment::new(999)], + fields_modified: vec![], + compacted_sstables: vec![CompactedSsTable::new(shard, 5)], + fields_for_preserving_frag_bitmap: vec![], + update_mode: None, + inserted_rows_filter: None, + updated_fragment_offsets: None, + }, + None, + ); + let mut dataset = dataset; + let result = CommitBuilder::new(Arc::new(dataset.clone())) + .execute(stale) + .await; + + // Asserting the reason, not just the failure: the commit must be + // refused by apply-time generation validation, not by something + // incidental about the fragment. + let err = result.expect_err("a stale generation must fail the whole commit"); + assert!( + err.to_string().contains("Stale SSTable compaction"), + "expected stale-generation rejection, got {err}" + ); + + // Neither the marker nor the fragment may have been published. + dataset.checkout_latest().await.unwrap(); + let latest = dataset; + assert_eq!( + latest.manifest.version, version_before, + "no new table version may be published" + ); + assert_eq!( + latest.get_fragments().len(), + fragments_before, + "the row mutation must not be published" + ); + let details = load_mem_wal_index_details( + latest + .load_indices() + .await + .unwrap() + .iter() + .find(|idx| idx.name == MEM_WAL_INDEX_NAME) + .unwrap() + .clone(), + ) + .unwrap(); + assert_eq!( + details.compacted_sstables[0].generation, 10, + "the recorded generation must be unchanged" + ); } + /// The system index holds the catch-up positions the WAL pod retires SSTables against. + /// Erasing it through the ordinary index API would leave the table claiming + /// nothing was ever compacted while the SSTables are already gone. + #[test] - fn test_empty_merged_generations_noop() { + fn test_empty_compacted_sstables_noop() { let mut indices = Vec::new(); - // Empty update should be a no-op - update_mem_wal_index_merged_generations(&mut indices, 1, vec![]).unwrap(); + // Empty update should be a no-op, even with no MemWAL index present. + update_mem_wal_index_compacted_sstables(&mut indices, 1, vec![]).unwrap(); assert!(indices.is_empty()); } + + /// Regression: a committed `__mem_wal` (legitimately `fragment_bitmap: + /// None`) must not break `describe_indices` — the path behind lancedb's + /// `list_indices`/`wait_for_index`. It's described as zero indexed rows, + /// like `__frag_reuse`. + #[tokio::test] + async fn test_describe_indices_includes_mem_wal_system_index() { + use crate::index::DatasetIndexExt; + use lance_index::IndexType; + use lance_index::scalar::ScalarIndexParams; + + let mut dataset = test_dataset_with_mem_wal().await; + + // A real user index that describe_indices must keep returning. + dataset + .create_index( + &["a"], + IndexType::Scalar, + None, + &ScalarIndexParams::default(), + true, + ) + .await + .unwrap(); + + // Commit a __mem_wal index, as WAL provisioning does in production. + let shard = Uuid::new_v4(); + let txn = Transaction::new( + dataset.manifest.version, + Operation::UpdateMemWalState { + compacted_sstables: vec![CompactedSsTable::new(shard, 1)], + require_index_catchup: false, + }, + None, + ); + let dataset = CommitBuilder::new(Arc::new(dataset)) + .execute(txn) + .await + .unwrap(); + + // The system index is present with no fragment_bitmap (by design). + let mem_wal = dataset + .load_indices() + .await + .unwrap() + .iter() + .find(|i| i.name == MEM_WAL_INDEX_NAME) + .unwrap() + .clone(); + assert!(mem_wal.fragment_bitmap.is_none()); + + // describe_indices describes the bitmap-less __mem_wal alongside the + // real index instead of erroring. + let descriptions = dataset.describe_indices(None).await.unwrap(); + let mem_wal_desc = descriptions + .iter() + .find(|d| d.name() == MEM_WAL_INDEX_NAME) + .expect("__mem_wal must be described, not skipped"); + assert_eq!( + mem_wal_desc.index_type(), + "MemWal", + "system index type must resolve via infer_system_index_type" + ); + assert_eq!( + mem_wal_desc.rows_indexed(), + 0, + "a bitmap-less system index indexes zero rows" + ); + assert_eq!( + descriptions.len(), + 2, + "both the real scalar index and __mem_wal must be listed" + ); + } } diff --git a/rust/lance/src/index/prefilter.rs b/rust/lance/src/index/prefilter.rs index 071f1b8893d..53c2f25ec86 100644 --- a/rust/lance/src/index/prefilter.rs +++ b/rust/lance/src/index/prefilter.rs @@ -52,6 +52,10 @@ pub struct DatasetPreFilter { // Fragment IDs whose data is still in the index but has been removed from the dataset. // Used by FTS merge-on-read to prune stale fragments at search time. pub(super) deleted_fragments: Option, + // Row addresses whose index entries are stale due to a newer data overlay committed after + // the index was built. Computed synchronously at plan time and ANDead into the final mask + // so the index never returns those rows. + pub(super) overlay_block: Option, // When the tasks are finished this is the combined filter pub(super) final_mask: Mutex>>, } @@ -65,7 +69,9 @@ impl DatasetPreFilter { let mut fragments = RoaringBitmap::new(); let all_have_bitmaps = indices.iter().all(|idx| idx.fragment_bitmap.is_some()); if !all_have_bitmaps { - fragments.insert_range(0..dataset.manifest.max_fragment_id.unwrap_or(0)); + if let Some(max_fragment_id) = dataset.manifest.max_fragment_id() { + fragments.insert_range(0..=max_fragment_id as u32); + } } else { indices.iter().for_each(|idx| { fragments |= idx.fragment_bitmap.as_ref().unwrap(); @@ -83,6 +89,7 @@ impl DatasetPreFilter { deleted_ids, filtered_ids, deleted_fragments: None, + overlay_block: None, final_mask: Mutex::new(OnceCell::new()), } } @@ -226,6 +233,13 @@ impl DatasetPreFilter { self.deleted_fragments = Some(fragments); } + /// Block specific row addresses from index results because their index entries are stale + /// due to a data overlay committed after the index was built. + pub fn with_overlay_block(mut self, block: RowAddrMask) -> Self { + self.overlay_block = Some(block); + self + } + /// Creates a task to load a mask that filters out deleted rows and, /// when `restrict_to_fragments` is true, also restricts results to only /// the given `fragments`. @@ -383,6 +397,9 @@ impl PreFilter for DatasetPreFilter { } combined = combined & RowAddrMask::from_block(block_list); } + if let Some(overlay_block) = &self.overlay_block { + combined = combined & overlay_block.clone(); + } Arc::new(combined) }); @@ -393,6 +410,7 @@ impl PreFilter for DatasetPreFilter { self.deleted_ids.is_none() && self.filtered_ids.is_none() && self.deleted_fragments.is_none() + && self.overlay_block.is_none() } /// Get the row id mask for this prefilter @@ -421,6 +439,7 @@ impl PreFilter for DatasetPreFilter { mod test { use lance_select::RowSetOps; use lance_testing::datagen::{BatchGenerator, IncrementingInt32}; + use rstest::rstest; use crate::dataset::WriteParams; @@ -529,6 +548,37 @@ mod test { assert_eq!(mask.block_list(), Some(&expected)); } + #[rstest] + #[case::stored_high_water_mark(false)] + #[case::computed_high_water_mark(true)] + #[tokio::test] + async fn test_legacy_index_mask_includes_max_fragment(#[case] unset_max_fragment_id: bool) { + let datasets = test_datasets(false).await; + let mut dataset = (*datasets.deletions_missing_frags).clone(); + if unset_max_fragment_id { + Arc::make_mut(&mut dataset.manifest).max_fragment_id = None; + } + let index = IndexMetadata { + uuid: uuid::Uuid::new_v4(), + fields: Vec::new(), + name: "legacy".to_string(), + dataset_version: dataset.manifest.version, + fragment_bitmap: None, + index_details: None, + index_version: 0, + created_at: None, + base_id: None, + files: None, + }; + let prefilter = DatasetPreFilter::new(Arc::new(dataset), &[index], None); + + prefilter.wait_for_ready().await.unwrap(); + + let mut expected = RowAddrTreeMap::from_iter(vec![(2 << 32) + 2]); + expected.insert_fragment(1); + assert_eq!(prefilter.mask().block_list(), Some(&expected)); + } + #[tokio::test] async fn test_deletion_mask_stable_row_id() { // Here, behavior is different. diff --git a/rust/lance/src/index/scalar.rs b/rust/lance/src/index/scalar.rs index b2ee8e0426a..4865436a442 100644 --- a/rust/lance/src/index/scalar.rs +++ b/rust/lance/src/index/scalar.rs @@ -5,13 +5,17 @@ //! pub(crate) mod bitmap; +pub(crate) mod bloomfilter; pub(crate) mod btree; pub(crate) mod fmindex; pub(crate) mod inverted; pub(crate) mod label_list; +pub(crate) mod ngram; +#[cfg(feature = "geo")] +pub(crate) mod rtree; pub(crate) mod zonemap; -pub use inverted::{load_segment_details, load_segments}; +pub use inverted::{load_segment_details, load_segment_params, load_segments}; pub use crate::index::scalar_logical::{LogicalScalarIndex, load_named_scalar_segments}; @@ -35,6 +39,7 @@ use lance_core::datatypes::Field; use lance_core::utils::tracing::{IO_TYPE_OPEN_SCALAR, TRACE_IO_EVENTS}; use lance_core::{Error, ROW_ADDR, ROW_ID, Result}; use lance_datafusion::exec::LanceExecutionOptions; +use lance_index::frag_reuse::FragReuseIndexHandle; use lance_index::metrics::{MetricsCollector, NoOpMetricsCollector}; use lance_index::pbold::{ BTreeIndexDetails, BitmapIndexDetails, InvertedIndexDetails, LabelListIndexDetails, @@ -47,7 +52,8 @@ use lance_index::scalar::label_list::{ LABEL_LIST_NULLS_METADATA_KEY, LABEL_LIST_NULLS_MIN_VERSION, }; use lance_index::scalar::registry::{ - ScalarIndexLoad, ScalarIndexPlugin, TrainingCriteria, TrainingOrdering, VALUE_COLUMN_NAME, + ScalarIndexCacheKey, ScalarIndexLoad, ScalarIndexPlugin, TrainingCriteria, TrainingOrdering, + VALUE_COLUMN_NAME, }; use lance_index::scalar::{BuiltinIndexType, CreatedIndex, InvertedIndexParams}; use lance_index::scalar::{ @@ -57,6 +63,7 @@ use lance_index::scalar::{ use lance_index::{IndexCriteria, IndexType}; use lance_table::format::{Fragment, IndexMetadata}; use log::info; +use prost::Message; use tracing::instrument; // Log an update every TRAINING_UPDATE_FREQ million rows processed @@ -122,6 +129,12 @@ impl TrainingRequest { } } +#[cfg(test)] +tokio::task_local! { + /// Overrides the scalar training scan's I/O budget without mutating process-wide state. + pub(crate) static TEST_TRAINING_IO_BUFFER_SIZE: u64; +} + pub(crate) async fn scan_training_data( dataset: &Dataset, column: &str, @@ -131,6 +144,10 @@ pub(crate) async fn scan_training_data( let num_rows = dataset.count_all_rows().await?; let mut scan = dataset.scan(); + #[cfg(test)] + if let Ok(io_buffer_size) = TEST_TRAINING_IO_BUFFER_SIZE.try_with(|size| *size) { + scan.io_buffer_size(io_buffer_size); + } // Fragment filtering is now handled in load_training_data function // This function just processes the fragments passed to it @@ -235,6 +252,35 @@ pub(crate) async fn load_training_data( } } +pub(crate) async fn load_fts_training_data( + dataset: &Dataset, + resolved: &inverted::ResolvedFtsField, + criteria: &TrainingCriteria, + fragments: Option>, + train: bool, + fragment_ids: Option>, +) -> Result { + let scan_column = if resolved.has_lists() { + resolved.root_column.as_str() + } else { + resolved.canonical_path.as_str() + }; + let stream = load_training_data( + dataset, + scan_column, + criteria, + fragments, + train, + fragment_ids, + ) + .await?; + if resolved.has_lists() { + inverted::transform_fts_document_stream(stream, resolved.clone()) + } else { + Ok(stream) + } +} + // TODO: Allow users to register their own plugins static SCALAR_INDEX_PLUGIN_REGISTRY: LazyLock> = LazyLock::new(IndexPluginRegistry::with_default_plugins); @@ -286,13 +332,40 @@ pub(super) async fn build_scalar_index( preprocessed_data: Option, progress: Arc, ) -> Result { - let field = dataset - .schema() - .field(column) - .ok_or(Error::invalid_input_source( - format!("No column with name {}", column).into(), - ))?; - let field: arrow_schema::Field = field.into(); + let inverted_params = (params.index_type.eq_ignore_ascii_case("inverted") + || params.index_type.eq_ignore_ascii_case("fts")) + .then(|| serde_json::from_str::(params.params.as_deref().unwrap_or("{}"))) + .transpose()?; + let resolved_fts_field = inverted_params + .as_ref() + .map(|params| { + inverted::resolve_fts_field(dataset.schema(), column, params.get_document_granularity()) + }) + .transpose()?; + let field = if let Some(resolved) = &resolved_fts_field { + let source = dataset + .schema() + .field_by_id(resolved.final_field_id) + .ok_or_else(|| { + Error::invalid_input(format!( + "FTS field id {} is missing from the dataset schema", + resolved.final_field_id + )) + })?; + if resolved.has_lists() { + arrow_schema::Field::new(VALUE_COLUMN_NAME, DataType::Utf8, true) + } else { + source.into() + } + } else { + let field = dataset + .schema() + .field(column) + .ok_or(Error::invalid_input_source( + format!("No column with name {}", column).into(), + ))?; + field.into() + }; let index_store = LanceIndexStore::from_dataset_for_new(dataset, &uuid)?; @@ -306,9 +379,20 @@ pub(super) async fn build_scalar_index( trainer.new_training_request(params.params.as_deref().unwrap_or("{}"), &field)?; progress.stage_start("load_data", None, "rows").await?; - let training_data = match preprocessed_data { - Some(preprocessed_data) => preprocessed_data, - None => { + let training_data = match (preprocessed_data, resolved_fts_field.as_ref()) { + (Some(preprocessed_data), _) => preprocessed_data, + (None, Some(resolved)) => { + load_fts_training_data( + dataset, + resolved, + training_request.criteria(), + None, + train, + fragment_ids.clone(), + ) + .await? + } + (None, None) => { load_training_data( dataset, column, @@ -401,6 +485,23 @@ pub async fn fetch_index_details( None => infer_scalar_index_details(dataset, column, index).await?, }; + if index_details.type_url.ends_with("InvertedIndexDetails") { + let details = + InvertedIndexDetails::decode(index_details.value.as_slice()).map_err(|err| { + Error::io(format!( + "failed to decode InvertedIndexDetails payload: {err}" + )) + })?; + let details = inverted::normalize_inverted_details(index, details)?; + return Ok(Arc::new(prost_types::Any::from_msg(&details).map_err( + |err| { + Error::io(format!( + "failed to encode InvertedIndexDetails payload: {err}" + )) + }, + )?)); + } + Ok(index_details) } @@ -460,7 +561,7 @@ pub async fn open_scalar_index( .for_index(&index.uuid, frag_reuse_index.as_ref().map(|f| &f.uuid)); let frag_reuse_index: Option> = - frag_reuse_index.map(|f| f as Arc); + frag_reuse_index.map(|f| Arc::new(FragReuseIndexHandle(f)) as Arc); // Runs only on a cold miss, and at most once even under concurrent opens // (the plugin coalesces). The compat check lives here because a warm hit was @@ -490,6 +591,17 @@ pub async fn open_scalar_index( .await } +pub(crate) async fn cached_scalar_index_container( + dataset: &Dataset, + uuid: &Uuid, +) -> Option> { + let frag_reuse_uuid = dataset.frag_reuse_index_uuid().await; + let index_cache = dataset + .index_cache + .for_index(uuid, frag_reuse_uuid.as_ref()); + index_cache.get_unsized_with_key(&ScalarIndexCacheKey).await +} + pub(crate) async fn infer_scalar_index_details( dataset: &Dataset, column: &str, @@ -515,11 +627,7 @@ pub(crate) async fn infer_scalar_index_details( let inverted_list_lookup = index_dir.clone().join(METADATA_FILE); let legacy_inverted_list_lookup = index_dir.clone().join(INVERT_LIST_FILE); let object_store = dataset.object_store_for_index(index).await?; - let index_details = if let DataType::List(_) = col.data_type() { - prost_types::Any::from_msg(&LabelListIndexDetails::default()).unwrap() - } else if object_store.exists(&bitmap_page_lookup).await? { - prost_types::Any::from_msg(&BitmapIndexDetails::default()).unwrap() - } else if object_store.exists(&inverted_list_lookup).await? { + let index_details = if object_store.exists(&inverted_list_lookup).await? { // Try to infer inverted index details from metadata file to capture with_position and other params // Fall back to defaults if anything goes wrong let default_details = prost_types::Any::from_msg(&InvertedIndexDetails::default()).unwrap(); @@ -536,6 +644,10 @@ pub(crate) async fn infer_scalar_index_details( parse_params().await.unwrap_or(default_details) } else if object_store.exists(&legacy_inverted_list_lookup).await? { prost_types::Any::from_msg(&InvertedIndexDetails::default()).unwrap() + } else if let DataType::List(_) | DataType::LargeList(_) = col.data_type() { + prost_types::Any::from_msg(&LabelListIndexDetails::default()).unwrap() + } else if object_store.exists(&bitmap_page_lookup).await? { + prost_types::Any::from_msg(&BitmapIndexDetails::default()).unwrap() } else { prost_types::Any::from_msg(&BTreeIndexDetails::default()).unwrap() }; @@ -572,16 +684,46 @@ pub fn index_matches_criteria( // return false just in case return Ok(false); } - let field = fields[0]; - // Build the full field path for nested fields - let field_path = if let Some(ancestors) = schema.field_ancestry_by_id(field.id) { - let field_refs: Vec<&str> = ancestors.iter().map(|f| f.name.as_str()).collect(); - lance_core::datatypes::format_field_path(&field_refs) + let is_fts_index = index + .index_details + .as_ref() + .is_some_and(|details| details.type_url.ends_with("InvertedIndexDetails")); + if criteria.must_support_fts && is_fts_index { + let requested_granularity = criteria + .fts_document_granularity + .unwrap_or(lance_index::scalar::inverted::DocumentGranularity::Row); + let requested = inverted::resolve_fts_field(schema, for_column, requested_granularity)?; + if index.fields[0] != requested.final_field_id { + return Ok(false); + } + let Some(details_any) = index.index_details.as_ref() else { + return Ok(false); + }; + let details = + InvertedIndexDetails::decode(details_any.value.as_slice()).map_err(|err| { + Error::io(format!( + "failed to decode InvertedIndexDetails payload: {err}" + )) + })?; + let details = inverted::normalize_inverted_details(index, details)?; + let stored_granularity = lance_index::scalar::inverted::DocumentGranularity::try_from( + details.document_granularity, + )?; + if stored_granularity != requested_granularity { + return Ok(false); + } } else { - field.name.clone() - }; - if for_column != field_path { - return Ok(false); + let field = fields[0]; + // Build the full field path for nested fields + let field_path = if let Some(ancestors) = schema.field_ancestry_by_id(field.id) { + let field_refs: Vec<&str> = ancestors.iter().map(|f| f.name.as_str()).collect(); + lance_core::datatypes::format_field_path(&field_refs) + } else { + field.name.clone() + }; + if for_column != field_path { + return Ok(false); + } } } @@ -651,10 +793,17 @@ pub async fn initialize_scalar_index( // Parse the JSON into InvertedIndexParams let inverted_params: InvertedIndexParams = serde_json::from_str(params_json)?; + let resolved = inverted::resolve_fts_field_by_id( + target_dataset.schema(), + *source_index.fields.first().ok_or_else(|| { + Error::index("Inverted index metadata has no indexed field".to_string()) + })?, + inverted_params.get_document_granularity(), + )?; target_dataset .create_index( - &[column_name], + &[&resolved.canonical_path], index_type, Some(source_index.name.clone()), &inverted_params, @@ -742,6 +891,7 @@ mod tests { let criteria = IndexCriteria { must_support_fts: false, + fts_document_granularity: None, must_support_exact_equality: false, for_column: None, has_name: None, @@ -768,6 +918,7 @@ mod tests { let criteria = IndexCriteria { must_support_fts: false, + fts_document_granularity: None, must_support_exact_equality: false, for_column: None, has_name: None, @@ -789,6 +940,7 @@ mod tests { // test for_column let mut criteria = IndexCriteria { must_support_fts: false, + fts_document_granularity: None, must_support_exact_equality: false, for_column: Some("mycol"), has_name: None, @@ -805,6 +957,7 @@ mod tests { // test has_name let mut criteria = IndexCriteria { must_support_fts: false, + fts_document_granularity: None, must_support_exact_equality: false, for_column: None, has_name: Some("btree_index"), @@ -827,6 +980,7 @@ mod tests { // test supports_exact_equality let mut criteria = IndexCriteria { must_support_fts: false, + fts_document_granularity: None, must_support_exact_equality: true, for_column: None, has_name: None, @@ -848,6 +1002,7 @@ mod tests { // test multiple indices let mut criteria = IndexCriteria { must_support_fts: false, + fts_document_granularity: None, must_support_exact_equality: false, for_column: None, has_name: None, @@ -2304,4 +2459,70 @@ mod tests { "Should have 0 rows with value='banana' after deletion" ); } + + // End-to-end: create index → delete a whole fragment → search (via index) → + // update index → search again. No deleted rows should ever appear. + #[tokio::test] + async fn test_zonemap_search_with_deleted_fragment_before_and_after_update() { + use arrow::datatypes::Int32Type; + use lance_datagen::array; + use lance_index::IndexType; + use lance_index::optimize::OptimizeOptions; + use lance_index::scalar::{BuiltinIndexType, ScalarIndexParams}; + + // 3 fragments × 10 rows: id 0-9 (frag 0), 10-19 (frag 1), 20-29 (frag 2). + let mut ds = lance_datagen::gen_batch() + .col("id", array::step::()) + .into_ram_dataset(FragmentCount::from(3), FragmentRowCount::from(10)) + .await + .unwrap(); + + let params = ScalarIndexParams::for_builtin(BuiltinIndexType::ZoneMap); + ds.create_index(&["id"], IndexType::Scalar, None, ¶ms, false) + .await + .unwrap(); + + // Delete the middle fragment entirely. + ds.delete("id >= 10 AND id < 20").await.unwrap(); + + // Helper: run a filter scan and return the sorted id values. + async fn live_ids(ds: &crate::Dataset) -> Vec { + let batch = ds + .scan() + .filter("id >= 0") + .unwrap() + .try_into_batch() + .await + .unwrap(); + let mut ids: Vec = batch["id"] + .as_any() + .downcast_ref::() + .unwrap() + .values() + .to_vec(); + ids.sort_unstable(); + ids + } + + // --- Before index update --- + let ids = live_ids(&ds).await; + assert_eq!(ids.len(), 20, "expected 20 live rows before index update"); + assert!( + ids.iter().all(|&id| !(10..20).contains(&id)), + "deleted fragment rows (id 10-19) must not appear before index update; got: {:?}", + ids + ); + + // Update the zone map index to reflect the deletion. + ds.optimize_indices(&OptimizeOptions::new()).await.unwrap(); + + // --- After index update --- + let ids = live_ids(&ds).await; + assert_eq!(ids.len(), 20, "expected 20 live rows after index update"); + assert!( + ids.iter().all(|&id| !(10..20).contains(&id)), + "deleted fragment rows (id 10-19) must not appear after index update; got: {:?}", + ids + ); + } } diff --git a/rust/lance/src/index/scalar/bitmap.rs b/rust/lance/src/index/scalar/bitmap.rs index 2eb5702ee28..84c8b6a8b91 100644 --- a/rust/lance/src/index/scalar/bitmap.rs +++ b/rust/lance/src/index/scalar/bitmap.rs @@ -3,6 +3,7 @@ use lance_index::metrics::NoOpMetricsCollector; use lance_index::scalar::bitmap::BitmapIndex; +use lance_index::scalar::index_files_to_table; use lance_index::scalar::lance_format::LanceIndexStore; use lance_table::format::IndexMetadata; use roaring::RoaringBitmap; @@ -70,7 +71,7 @@ pub(in crate::index) async fn merge_segments( index_version: created_index.index_version as i32, created_at: Some(chrono::Utc::now()), base_id: None, - files: Some(created_index.files), + files: Some(index_files_to_table(created_index.files)), ..segments[0].clone() }) } diff --git a/rust/lance/src/index/scalar/bloomfilter.rs b/rust/lance/src/index/scalar/bloomfilter.rs new file mode 100644 index 00000000000..4956a152622 --- /dev/null +++ b/rust/lance/src/index/scalar/bloomfilter.rs @@ -0,0 +1,99 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright The Lance Authors + +use std::sync::Arc; + +use lance_index::metrics::NoOpMetricsCollector; +use lance_index::scalar::bloomfilter::BloomFilterIndex; +use lance_index::scalar::index_files_to_table; +use lance_index::scalar::lance_format::LanceIndexStore; +use lance_table::format::IndexMetadata; +use roaring::RoaringBitmap; +use uuid::Uuid; + +use crate::{Dataset, Error, Result, dataset::index::LanceIndexStoreExt}; + +/// Merge one caller-defined group of source BloomFilter segments into a single segment. +pub(in crate::index) async fn merge_segments( + dataset: &Dataset, + segments: Vec, +) -> Result { + if segments.is_empty() { + return Err(Error::index("No segment metadata was provided".to_string())); + } + + let field_id = *segments[0].fields.first().ok_or_else(|| { + Error::invalid_input(format!( + "CreateIndex: segment {} is missing field ids", + segments[0].uuid + )) + })?; + let field_path = dataset.schema().field_path(field_id)?; + let dataset_version = segments + .iter() + .map(|segment| segment.dataset_version) + .min() + .unwrap_or(dataset.manifest.version); + let mut fragment_bitmap = RoaringBitmap::new(); + let dataset_fragments = dataset.fragment_bitmap.as_ref(); + let fragment_filters = segments + .iter() + .map(|segment| { + segment + .effective_fragment_bitmap(dataset_fragments) + .ok_or_else(|| { + Error::invalid_input(format!( + "CreateIndex: segment {} is missing fragment coverage", + segment.uuid + )) + }) + }) + .collect::>>()?; + for effective in &fragment_filters { + fragment_bitmap |= effective; + } + + let mut scalar_indices = Vec::with_capacity(segments.len()); + for (position, (segment, effective)) in segments.iter().zip(&fragment_filters).enumerate() { + if effective.is_empty() && !(fragment_bitmap.is_empty() && position == 0) { + continue; + } + let scalar_index = + super::open_scalar_index(dataset, &field_path, segment, &NoOpMetricsCollector).await?; + scalar_indices.push((segment.uuid, scalar_index, effective)); + } + + let mut source_indices = Vec::with_capacity(scalar_indices.len()); + for (segment_uuid, scalar_index, fragment_filter) in &scalar_indices { + let bloomfilter_index = scalar_index + .as_any() + .downcast_ref::() + .ok_or_else(|| { + Error::index(format!( + "merge_existing_index_segments: expected bloom filter segment {}, got {:?}", + segment_uuid, + scalar_index.index_type() + )) + })?; + source_indices.push((bloomfilter_index, *fragment_filter)); + } + + let new_uuid = Uuid::new_v4(); + let new_store = LanceIndexStore::from_dataset_for_new(dataset, &new_uuid)?; + let created_index = + lance_index::scalar::bloomfilter::merge_bloomfilter_indices(&source_indices, &new_store) + .await?; + + Ok(IndexMetadata { + uuid: new_uuid, + fields: vec![field_id], + dataset_version, + fragment_bitmap: Some(fragment_bitmap), + index_details: Some(Arc::new(created_index.index_details)), + index_version: created_index.index_version as i32, + created_at: Some(chrono::Utc::now()), + base_id: None, + files: Some(index_files_to_table(created_index.files)), + ..segments[0].clone() + }) +} diff --git a/rust/lance/src/index/scalar/btree.rs b/rust/lance/src/index/scalar/btree.rs index 4339b8c183b..7089997721c 100644 --- a/rust/lance/src/index/scalar/btree.rs +++ b/rust/lance/src/index/scalar/btree.rs @@ -15,7 +15,7 @@ use lance_index::pbold::BTreeIndexDetails; use lance_index::scalar::btree::BTreeIndex; use lance_index::scalar::lance_format::LanceIndexStore; use lance_index::scalar::registry::VALUE_COLUMN_NAME; -use lance_index::scalar::{CreatedIndex, OldIndexDataFilter}; +use lance_index::scalar::{CreatedIndex, OldIndexDataFilter, index_files_to_table}; use lance_table::format::IndexMetadata; use uuid::Uuid; @@ -161,6 +161,6 @@ pub(crate) async fn merge_segments( index_version: created_index.index_version as i32, created_at: Some(chrono::Utc::now()), base_id: None, - files: Some(created_index.files), + files: Some(index_files_to_table(created_index.files)), }) } diff --git a/rust/lance/src/index/scalar/fmindex.rs b/rust/lance/src/index/scalar/fmindex.rs index 32684ebf9ab..9c1baadbbb6 100644 --- a/rust/lance/src/index/scalar/fmindex.rs +++ b/rust/lance/src/index/scalar/fmindex.rs @@ -1,6 +1,7 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright The Lance Authors +use lance_index::scalar::index_files_to_table; use lance_table::format::IndexMetadata; use roaring::RoaringBitmap; use std::sync::Arc; @@ -76,7 +77,7 @@ pub(in crate::index) async fn merge_segments( index_version: created_index.index_version as i32, created_at: Some(chrono::Utc::now()), base_id: None, - files: Some(created_index.files), + files: Some(index_files_to_table(created_index.files)), ..segments[0].clone() }); } @@ -111,7 +112,7 @@ pub(in crate::index) async fn merge_segments( index_version: created_index.index_version as i32, created_at: Some(chrono::Utc::now()), base_id: None, - files: Some(created_index.files), + files: Some(index_files_to_table(created_index.files)), ..segments[0].clone() }) } diff --git a/rust/lance/src/index/scalar/inverted.rs b/rust/lance/src/index/scalar/inverted.rs index 000d2c3139c..7facf10a79b 100644 --- a/rust/lance/src/index/scalar/inverted.rs +++ b/rust/lance/src/index/scalar/inverted.rs @@ -3,15 +3,28 @@ #![allow(clippy::redundant_pub_crate)] -use std::sync::Arc; +use std::{collections::BTreeMap, sync::Arc}; -use arrow_schema::{Field as ArrowField, Schema as ArrowSchema}; +use arrow_array::cast::AsArray; +use arrow_array::{ + Array, ArrayRef, LargeStringArray, RecordBatch, StringArray, StringViewArray, UInt32Array, + UInt64Array, +}; +use arrow_schema::{DataType, Field as ArrowField, Schema as ArrowSchema}; +use datafusion::error::DataFusionError; use datafusion::execution::SendableRecordBatchStream; use datafusion::physical_plan::stream::RecordBatchStreamAdapter; -use lance_core::ROW_ID; +use futures::StreamExt; +use lance_core::{ + ROW_ID, + datatypes::{Field, LogicalType, Schema, format_field_path, parse_field_path}, +}; use lance_index::metrics::NoOpMetricsCollector; use lance_index::pbold::InvertedIndexDetails; -use lance_index::scalar::inverted::InvertedIndex; +use lance_index::scalar::index_files_to_table; +use lance_index::scalar::inverted::{ + DocumentGranularity, InvertedIndex, InvertedIndexParams, doc_index_storage_column, +}; use lance_index::scalar::lance_format::LanceIndexStore; use lance_index::scalar::registry::VALUE_COLUMN_NAME; use lance_table::format::IndexMetadata; @@ -25,6 +38,685 @@ use crate::{ index::{DatasetIndexExt, scalar::fetch_index_details}, }; +#[derive(Debug, Clone)] +enum FtsTraversal { + Text, + Struct { + child_index: usize, + child: Box, + }, + List { + child: Box, + }, +} + +impl FtsTraversal { + fn list_depth(&self) -> usize { + match self { + Self::Text => 0, + Self::Struct { child, .. } => child.list_depth(), + Self::List { child } => 1 + child.list_depth(), + } + } +} + +/// Schema-derived form of an FTS field path. +#[derive(Debug, Clone)] +pub(crate) struct ResolvedFtsField { + pub final_field_id: i32, + pub root_column: String, + pub canonical_path: String, + pub document_granularity: DocumentGranularity, + traversal: FtsTraversal, + list_depth: usize, +} + +/// One logical document extracted from a dataset row. +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) struct FtsDocument { + pub row_index: usize, + pub text: String, + pub doc_index: Vec, +} + +impl ResolvedFtsField { + pub fn has_lists(&self) -> bool { + self.list_depth > 0 + } + + pub fn coordinate_rank(&self) -> usize { + if self.document_granularity.is_list_element() { + self.list_depth + } else { + 0 + } + } + + pub fn documents_from_batch(&self, batch: &RecordBatch) -> Result> { + let column = batch.column_by_name(&self.root_column).ok_or_else(|| { + Error::invalid_input(format!( + "FTS root column '{}' is missing from the input batch", + self.root_column + )) + })?; + self.documents_from_array(column, batch.num_rows()) + } + + fn documents_from_array(&self, column: &ArrayRef, num_rows: usize) -> Result> { + let mut documents = Vec::new(); + match self.document_granularity { + DocumentGranularity::Row => { + documents.reserve(num_rows); + for row_index in 0..num_rows { + let mut text = String::new(); + append_row_text(&self.traversal, column.as_ref(), row_index, &mut text)?; + documents.push(FtsDocument { + row_index, + text, + doc_index: Vec::new(), + }); + } + } + DocumentGranularity::ListElement => { + for row_index in 0..num_rows { + collect_element_documents( + &self.traversal, + column.as_ref(), + row_index, + row_index, + 0, + self.list_depth, + &mut Vec::with_capacity(self.list_depth), + &mut documents, + )?; + } + } + } + Ok(documents) + } +} + +fn string_value(array: &dyn Array, index: usize) -> Result> { + if array.is_null(index) { + return Ok(None); + } + match array.data_type() { + DataType::Utf8 => Ok(Some( + array + .as_any() + .downcast_ref::() + .expect("Utf8 array type") + .value(index), + )), + DataType::LargeUtf8 => Ok(Some( + array + .as_any() + .downcast_ref::() + .expect("LargeUtf8 array type") + .value(index), + )), + DataType::Utf8View => Ok(Some( + array + .as_any() + .downcast_ref::() + .expect("Utf8View array type") + .value(index), + )), + data_type => Err(Error::internal(format!( + "FTS traversal expected a string array, got {data_type}" + ))), + } +} + +fn append_text(output: &mut String, text: &str) { + if !output.is_empty() { + output.push(' '); + } + output.push_str(text); +} + +fn append_row_text( + traversal: &FtsTraversal, + array: &dyn Array, + index: usize, + output: &mut String, +) -> Result<()> { + match traversal { + FtsTraversal::Text => { + if let Some(text) = string_value(array, index)? { + append_text(output, text); + } + } + FtsTraversal::Struct { child_index, child } => { + if !array.is_null(index) { + let structs = array.as_struct(); + append_row_text(child, structs.column(*child_index).as_ref(), index, output)?; + } + } + FtsTraversal::List { child } => match array.data_type() { + DataType::List(_) => { + let lists = array.as_list::(); + if !lists.is_null(index) { + let offsets = lists.value_offsets(); + let start = offsets[index] as usize; + let end = offsets[index + 1] as usize; + for element_index in start..end { + append_row_text(child, lists.values().as_ref(), element_index, output)?; + } + } + } + DataType::LargeList(_) => { + let lists = array.as_list::(); + if !lists.is_null(index) { + let offsets = lists.value_offsets(); + let start = offsets[index] as usize; + let end = offsets[index + 1] as usize; + for element_index in start..end { + append_row_text(child, lists.values().as_ref(), element_index, output)?; + } + } + } + data_type => { + return Err(Error::internal(format!( + "FTS traversal expected a list array, got {data_type}" + ))); + } + }, + } + Ok(()) +} + +#[allow(clippy::too_many_arguments)] +fn collect_element_documents( + traversal: &FtsTraversal, + array: &dyn Array, + index: usize, + row_index: usize, + list_depth: usize, + boundary_depth: usize, + doc_index: &mut Vec, + documents: &mut Vec, +) -> Result<()> { + match traversal { + FtsTraversal::Text => Err(Error::internal( + "ListElement FTS traversal did not encounter its list boundary".to_string(), + )), + FtsTraversal::Struct { child_index, child } => { + if !array.is_null(index) { + let structs = array.as_struct(); + collect_element_documents( + child, + structs.column(*child_index).as_ref(), + index, + row_index, + list_depth, + boundary_depth, + doc_index, + documents, + )?; + } + Ok(()) + } + FtsTraversal::List { child } => { + let mut visit = |values: &ArrayRef, start: usize, end: usize| -> Result<()> { + for (ordinal, element_index) in (start..end).enumerate() { + let ordinal = u32::try_from(ordinal).map_err(|_| { + Error::invalid_input(format!( + "FTS element ordinal overflow for row index {row_index}" + )) + })?; + doc_index.push(ordinal); + if list_depth + 1 == boundary_depth { + let mut text = String::new(); + append_row_text(child, values.as_ref(), element_index, &mut text)?; + documents.push(FtsDocument { + row_index, + text, + doc_index: doc_index.clone(), + }); + } else { + collect_element_documents( + child, + values.as_ref(), + element_index, + row_index, + list_depth + 1, + boundary_depth, + doc_index, + documents, + )?; + } + doc_index.pop(); + } + Ok(()) + }; + match array.data_type() { + DataType::List(_) => { + let lists = array.as_list::(); + if !lists.is_null(index) { + let offsets = lists.value_offsets(); + visit( + lists.values(), + offsets[index] as usize, + offsets[index + 1] as usize, + )?; + } + } + DataType::LargeList(_) => { + let lists = array.as_list::(); + if !lists.is_null(index) { + let offsets = lists.value_offsets(); + visit( + lists.values(), + offsets[index] as usize, + offsets[index + 1] as usize, + )?; + } + } + data_type => { + return Err(Error::internal(format!( + "FTS traversal expected a list array, got {data_type}" + ))); + } + } + Ok(()) + } + } +} + +fn find_child_case_insensitive<'a>(field: &'a Field, name: &str) -> Option<(usize, &'a Field)> { + field + .children + .iter() + .enumerate() + .find(|(_, child)| child.name == name) + .or_else(|| { + field + .children + .iter() + .enumerate() + .find(|(_, child)| child.name.eq_ignore_ascii_case(name)) + }) +} + +fn build_fts_traversal( + field: &Field, + remaining_names: &[String], + canonical_names: &mut Vec, + final_field_id: &mut i32, +) -> Result { + match field.data_type() { + DataType::List(_) | DataType::LargeList(_) => { + let child = field.children.first().ok_or_else(|| { + Error::invalid_input(format!( + "FTS list field '{}' does not have an item field", + field.name + )) + })?; + Ok(FtsTraversal::List { + child: Box::new(build_fts_traversal( + child, + remaining_names, + canonical_names, + final_field_id, + )?), + }) + } + DataType::Struct(_) => { + let Some((name, rest)) = remaining_names.split_first() else { + return Err(Error::invalid_input(format!( + "FTS field path ends at struct '{}'; specify the final text field", + field.name + ))); + }; + let (child_index, child) = + find_child_case_insensitive(field, name).ok_or_else(|| { + Error::index(format!( + "FTS field '{}' does not contain child '{}'", + field.name, name + )) + })?; + canonical_names.push(child.name.clone()); + *final_field_id = child.id; + Ok(FtsTraversal::Struct { + child_index, + child: Box::new(build_fts_traversal( + child, + rest, + canonical_names, + final_field_id, + )?), + }) + } + DataType::Utf8 | DataType::LargeUtf8 | DataType::Utf8View if remaining_names.is_empty() => { + Ok(FtsTraversal::Text) + } + _ if field.logical_type == LogicalType::from("json") && remaining_names.is_empty() => { + Ok(FtsTraversal::Text) + } + data_type if !remaining_names.is_empty() => Err(Error::invalid_input(format!( + "FTS field '{}' has type {data_type} and cannot contain child '{}'", + field.name, remaining_names[0] + ))), + data_type => Err(Error::invalid_input(format!( + "FTS field '{}' must resolve to Utf8, LargeUtf8, Utf8View, or JSON, got {data_type}", + field.name + ))), + } +} + +/// Resolve a public FTS field path and derive all list traversal from schema. +pub(crate) fn resolve_fts_field( + schema: &Schema, + path: &str, + document_granularity: DocumentGranularity, +) -> Result { + let names = parse_field_path(path)?; + let (root_name, remaining_names) = names + .split_first() + .ok_or_else(|| Error::invalid_input("FTS field path cannot be empty".to_string()))?; + let root = schema + .fields + .iter() + .find(|field| field.name == *root_name) + .or_else(|| { + schema + .fields + .iter() + .find(|field| field.name.eq_ignore_ascii_case(root_name)) + }) + .ok_or_else(|| { + Error::index(format!( + "FTS field path '{path}' does not exist in the dataset schema" + )) + })?; + + let mut canonical_names = vec![root.name.clone()]; + let mut final_field_id = root.id; + let traversal = build_fts_traversal( + root, + remaining_names, + &mut canonical_names, + &mut final_field_id, + )?; + let list_depth = traversal.list_depth(); + if document_granularity.is_list_element() && list_depth == 0 { + return Err(Error::invalid_input(format!( + "FTS field path '{}' has no List layer and cannot use ListElement document granularity", + format_field_path( + &canonical_names + .iter() + .map(String::as_str) + .collect::>() + ) + ))); + } + if list_depth > 0 && root.logical_type == LogicalType::from("json") { + return Err(Error::invalid_input( + "nested List traversal is not supported for JSON FTS sources".to_string(), + )); + } + let canonical_path = format_field_path( + &canonical_names + .iter() + .map(String::as_str) + .collect::>(), + ); + Ok(ResolvedFtsField { + final_field_id, + root_column: root.name.clone(), + canonical_path, + document_granularity, + traversal, + list_depth, + }) +} + +fn find_public_path_by_id(field: &Field, field_id: i32, path: &mut Vec) -> bool { + if field.id == field_id { + return true; + } + match field.data_type() { + DataType::List(_) | DataType::LargeList(_) => field + .children + .first() + .is_some_and(|child| find_physical_path_by_id(child, field_id, path)), + DataType::Struct(_) => field.children.iter().any(|child| { + path.push(child.name.clone()); + let found = find_public_path_by_id(child, field_id, path); + if !found { + path.pop(); + } + found + }), + _ => false, + } +} + +fn find_physical_path_by_id(field: &Field, field_id: i32, path: &mut Vec) -> bool { + if field.id == field_id { + return false; + } + match field.data_type() { + DataType::List(_) | DataType::LargeList(_) => field + .children + .first() + .is_some_and(|child| find_physical_path_by_id(child, field_id, path)), + DataType::Struct(_) => field.children.iter().any(|child| { + path.push(child.name.clone()); + let found = find_public_path_by_id(child, field_id, path); + if !found { + path.pop(); + } + found + }), + _ => false, + } +} + +/// Resolve an indexed final field id against the current schema, preserving +/// routing across field renames. +pub(crate) fn resolve_fts_field_by_id( + schema: &Schema, + field_id: i32, + document_granularity: DocumentGranularity, +) -> Result { + for root in &schema.fields { + let mut path = vec![root.name.clone()]; + if find_public_path_by_id(root, field_id, &mut path) { + let path_refs = path.iter().map(String::as_str).collect::>(); + return resolve_fts_field(schema, &format_field_path(&path_refs), document_granularity); + } + } + Err(Error::invalid_input(format!( + "FTS index refers to missing or Arrow-internal field id {field_id}" + ))) +} + +/// Return the persisted FTS document granularity for each logical index on a +/// public field path. Multiple physical segments with the same index name are +/// collapsed after validating that their metadata agrees. +pub(crate) async fn indexed_fts_document_granularities( + dataset: &Dataset, + column: &str, +) -> Result> { + let resolved = resolve_fts_field(dataset.schema(), column, DocumentGranularity::Row)?; + let mut by_name = BTreeMap::new(); + + for index in dataset.load_indices().await?.iter() { + if index.fields.as_slice() != [resolved.final_field_id] { + continue; + } + let details_any = fetch_index_details(dataset, &resolved.canonical_path, index).await?; + if !details_any.type_url.ends_with("InvertedIndexDetails") { + continue; + } + let details = + InvertedIndexDetails::decode(details_any.value.as_slice()).map_err(|error| { + Error::corrupt_file( + dataset.indices_dir().join(index.uuid.to_string()), + format!( + "failed to decode InvertedIndexDetails for FTS index '{}': {error}", + index.name + ), + ) + })?; + let document_granularity = DocumentGranularity::try_from(details.document_granularity)?; + if let Some(existing) = by_name.insert(index.name.clone(), document_granularity) + && existing != document_granularity + { + return Err(Error::corrupt_file( + dataset.indices_dir().join(index.uuid.to_string()), + format!( + "FTS index '{}' has inconsistent document granularity across segments: \ + {existing:?} and {document_granularity:?}", + index.name + ), + )); + } + } + + Ok(by_name.into_iter().collect()) +} + +/// Resolve an optional query granularity against persisted FTS index metadata. +/// +/// A unique indexed granularity is authoritative and also controls flat search +/// over unindexed fragments. An explicit request selects between coexisting +/// row and list-element indexes, but cannot contradict the only indexed +/// granularity. With no index, the established row default is retained. +pub(crate) async fn resolve_query_document_granularity( + dataset: &Dataset, + column: &str, + requested: Option, +) -> Result { + let indices = indexed_fts_document_granularities(dataset, column).await?; + let mut available = indices + .iter() + .map(|(_, document_granularity)| *document_granularity) + .collect::>(); + available.sort_by_key(|document_granularity| match document_granularity { + DocumentGranularity::Row => 0, + DocumentGranularity::ListElement => 1, + }); + available.dedup(); + + let resolved = match requested { + Some(requested) if available.is_empty() || available.contains(&requested) => requested, + Some(requested) => { + let indexed = indices + .iter() + .map(|(name, document_granularity)| format!("'{name}' ({document_granularity:?})")) + .collect::>() + .join(", "); + return Err(Error::invalid_input(format!( + "FTS query for field '{column}' requested {requested:?} document granularity, \ + but the existing FTS index uses a different granularity: {indexed}" + ))); + } + None if available.is_empty() => DocumentGranularity::Row, + None if available.len() == 1 => available[0], + None => { + let indexed = indices + .iter() + .map(|(name, document_granularity)| format!("'{name}' ({document_granularity:?})")) + .collect::>() + .join(", "); + return Err(Error::invalid_input(format!( + "FTS query for field '{column}' is ambiguous because Row and ListElement \ + indexes coexist: {indexed}; specify document_granularity" + ))); + } + }; + + resolve_fts_field(dataset.schema(), column, resolved)?; + Ok(resolved) +} + +pub(crate) fn fts_document_schema(coordinate_rank: usize) -> Arc { + let mut fields = vec![ + ArrowField::new(VALUE_COLUMN_NAME, DataType::Utf8, false), + ArrowField::new(ROW_ID, DataType::UInt64, false), + ]; + fields.extend( + (0..coordinate_rank) + .map(|rank| ArrowField::new(doc_index_storage_column(rank), DataType::UInt32, false)), + ); + Arc::new(ArrowSchema::new(fields)) +} + +pub(crate) fn transform_fts_document_stream( + input: SendableRecordBatchStream, + resolved: ResolvedFtsField, +) -> Result { + let input_schema = input.schema(); + if (input_schema + .column_with_name(&resolved.root_column) + .is_none() + && input_schema.column_with_name(VALUE_COLUMN_NAME).is_none()) + || input_schema.column_with_name(ROW_ID).is_none() + { + return Err(Error::internal( + "FTS document input must contain the root source column and _rowid".to_string(), + )); + } + let output_schema = fts_document_schema(resolved.coordinate_rank()); + let stream_schema = output_schema.clone(); + let stream = input.map(move |batch| { + let batch = batch?; + let source = batch + .column_by_name(&resolved.root_column) + .or_else(|| batch.column_by_name(VALUE_COLUMN_NAME)) + .expect("FTS document input schema was validated"); + let documents = resolved + .documents_from_array(source, batch.num_rows()) + .map_err(DataFusionError::from)?; + let input_row_ids = batch[ROW_ID].as_primitive::(); + let texts = + StringArray::from_iter_values(documents.iter().map(|document| document.text.as_str())); + let row_ids = UInt64Array::from_iter_values( + documents + .iter() + .map(|document| input_row_ids.value(document.row_index)), + ); + let mut columns = vec![Arc::new(texts) as ArrayRef, Arc::new(row_ids) as ArrayRef]; + for rank in 0..resolved.coordinate_rank() { + columns.push(Arc::new(UInt32Array::from_iter_values( + documents.iter().map(|document| document.doc_index[rank]), + )) as ArrayRef); + } + RecordBatch::try_new(stream_schema.clone(), columns).map_err(DataFusionError::from) + }); + Ok(Box::pin(RecordBatchStreamAdapter::new( + output_schema, + stream, + ))) +} + +/// Fill legacy posting-format metadata while leaving protobuf-default row +/// document granularity untouched. +pub(crate) fn normalize_inverted_details( + index: &IndexMetadata, + mut details: InvertedIndexDetails, +) -> Result { + if !matches!(index.index_version, 0..=3) { + return Err(Error::invalid_input(format!( + "FTS index '{}' has unsupported index_version {}; expected 0, 1, 2, or 3", + index.name, index.index_version + ))); + } + if details.posting_format_version.is_none() { + let posting_format_version = match index.index_version { + 0 | 1 => 1, + 2 => 2, + 3 => 3, + _ => unreachable!("index version was validated above"), + }; + details.posting_format_version = Some(posting_format_version); + } + Ok(details) +} + /// Build an empty update stream for the inverted merge API. /// /// `InvertedIndex::merge_segments` is shaped as "merge old segments plus new @@ -32,18 +724,25 @@ use crate::{ /// and `_rowid` fields. The stream intentionally contains no batches. fn empty_inverted_update_stream( dataset: &Dataset, - field_id: i32, + resolved: &ResolvedFtsField, ) -> Result { - let field = dataset.schema().field_by_id(field_id).ok_or_else(|| { - Error::invalid_input(format!( - "merge_existing_index_segments: field id {} does not exist", - field_id - )) - })?; - let schema = Arc::new(ArrowSchema::new(vec![ - ArrowField::new(VALUE_COLUMN_NAME, field.data_type(), true), - ArrowField::new(ROW_ID, arrow_schema::DataType::UInt64, false), - ])); + let field = dataset + .schema() + .field_by_id(resolved.final_field_id) + .ok_or_else(|| { + Error::invalid_input(format!( + "merge_existing_index_segments: field id {} does not exist", + resolved.final_field_id + )) + })?; + let schema = if resolved.has_lists() { + fts_document_schema(resolved.coordinate_rank()) + } else { + Arc::new(ArrowSchema::new(vec![ + ArrowField::new(VALUE_COLUMN_NAME, field.data_type(), true), + ArrowField::new(ROW_ID, arrow_schema::DataType::UInt64, false), + ])) + }; Ok(Box::pin(RecordBatchStreamAdapter::new( schema, futures::stream::empty(), @@ -90,7 +789,21 @@ pub(crate) async fn merge_segments( segments[0].uuid )) })?; - let field_path = dataset.schema().field_path(field_id)?; + let details = match segments[0].index_details.as_ref() { + Some(details_any) => { + let details = + InvertedIndexDetails::decode(details_any.value.as_slice()).map_err(|error| { + Error::io(format!( + "failed to decode InvertedIndexDetails payload: {error}" + )) + })?; + normalize_inverted_details(&segments[0], details)? + } + None => normalize_inverted_details(&segments[0], InvertedIndexDetails::default())?, + }; + let document_granularity = DocumentGranularity::try_from(details.document_granularity)?; + let resolved = resolve_fts_field_by_id(dataset.schema(), field_id, document_granularity)?; + load_segment_details(dataset, &resolved.canonical_path, &segments).await?; let mut source_indices = Vec::with_capacity(segments.len()); let mut fragment_bitmap = RoaringBitmap::new(); @@ -102,8 +815,19 @@ pub(crate) async fn merge_segments( segment.uuid )) })?; - let scalar_index = - super::open_scalar_index(dataset, &field_path, segment, &NoOpMetricsCollector).await?; + if segment.fields != segments[0].fields { + return Err(Error::invalid_input(format!( + "FTS index {} has inconsistent fields across segments", + segments[0].name + ))); + } + let scalar_index = super::open_scalar_index( + dataset, + &resolved.canonical_path, + segment, + &NoOpMetricsCollector, + ) + .await?; let inverted_index = scalar_index .as_any() .downcast_ref::() @@ -121,7 +845,7 @@ pub(crate) async fn merge_segments( let new_store = LanceIndexStore::from_dataset_for_new(dataset, &new_uuid)?; let created_index = InvertedIndex::merge_segments( &source_indices, - empty_inverted_update_stream(dataset, field_id)?, + empty_inverted_update_stream(dataset, &resolved)?, &new_store, None, lance_index::progress::noop_progress(), @@ -137,7 +861,7 @@ pub(crate) async fn merge_segments( index_version: created_index.index_version as i32, created_at: Some(chrono::Utc::now()), base_id: None, - files: Some(created_index.files), + files: Some(index_files_to_table(created_index.files)), ..segments[0].clone() }) } @@ -149,12 +873,17 @@ pub(crate) async fn merge_segments( /// exists, the returned vector contains every committed segment's /// [`IndexMetadata`] (UUID, fragment coverage, index details). All segments /// must share the same indexed fields; mismatched fields return an error. -pub async fn load_segments(dataset: &Dataset, column: &str) -> Result>> { +pub async fn load_segments( + dataset: &Dataset, + column: &str, + document_granularity: DocumentGranularity, +) -> Result>> { let Some(index_meta) = dataset .load_scalar_index( lance_index::IndexCriteria::default() .for_column(column) - .supports_fts(), + .supports_fts() + .with_fts_document_granularity(document_granularity), ) .await? else { @@ -179,12 +908,40 @@ pub async fn load_segments(dataset: &Dataset, column: &str) -> Result Result> { + let Some(segments) = load_segments(dataset, column, document_granularity).await? else { + return Ok(None); + }; + + let fragment_bitmap = + segments + .iter() + .try_fold(RoaringBitmap::new(), |mut coverage, segment| { + coverage |= segment.fragment_bitmap.as_ref()?.clone(); + Some(coverage) + }); + + Ok(fragment_bitmap) +} + /// Load and validate the shared [`InvertedIndexDetails`] across committed /// segments returned by [`load_segments`]. /// -/// All segments are required to agree on their decoded `InvertedIndexDetails` -/// payload (analyzer, tokenizer, position settings, etc.); inconsistent -/// segments return an error. Returns the canonical details that may be used +/// All segments are required to agree on their semantic `InvertedIndexDetails` +/// payload (tokenizer, position settings, etc.); inconsistent +/// segments return an error. Details are canonicalized before comparison so +/// legacy segments that omit default fields remain compatible with newly +/// written text FTS segments. Returns the canonical details that may be used /// when constructing a tokenizer or running a query against the index. pub async fn load_segment_details( dataset: &Dataset, @@ -200,6 +957,7 @@ pub async fn load_segment_details( "failed to decode InvertedIndexDetails payload: {err}" )) })?; + let details = canonicalize_inverted_index_details(details)?; match &expected_details { Some(expected) if expected != &details => { return Err(Error::invalid_input(format!( @@ -219,10 +977,38 @@ pub async fn load_segment_details( }) } +fn canonicalize_inverted_index_details( + details: InvertedIndexDetails, +) -> Result { + let params = InvertedIndexParams::try_from(&details)?; + InvertedIndexDetails::try_from(¶ms) +} + +/// Read one segment's [`InvertedIndexParams`] +pub async fn load_segment_params( + dataset: &Dataset, + segment: &IndexMetadata, +) -> Result { + let store = LanceIndexStore::from_dataset_for_existing(dataset, segment).await?; + InvertedIndex::load_params(&store).await +} + #[cfg(test)] mod tests { use super::*; + fn fts_test_schema() -> Schema { + let schema = ArrowSchema::new(vec![ + ArrowField::new("text", DataType::Utf8, true), + ArrowField::new( + "tags", + DataType::List(Arc::new(ArrowField::new("item", DataType::Utf8, true))), + true, + ), + ]); + Schema::try_from(&schema).unwrap() + } + #[test] fn decode_legacy_inverted_details_type_url() { let mut details_any = prost_types::Any::from_msg(&InvertedIndexDetails::default()).unwrap(); @@ -231,4 +1017,79 @@ mod tests { let decoded = InvertedIndexDetails::decode(details_any.value.as_slice()).unwrap(); assert_eq!(decoded, InvertedIndexDetails::default()); } + + #[test] + fn resolve_element_document_field() { + let schema = fts_test_schema(); + let tags = schema.field("tags").unwrap(); + let resolved = + resolve_fts_field(&schema, "tags", DocumentGranularity::ListElement).unwrap(); + assert_eq!(resolved.final_field_id, tags.id); + assert_eq!(resolved.root_column, "tags"); + assert_eq!(resolved.canonical_path, "tags"); + assert_eq!(resolved.coordinate_rank(), 1); + + let err = resolve_fts_field(&schema, "text", DocumentGranularity::ListElement).unwrap_err(); + assert!(err.to_string().contains("has no List layer"), "{err}"); + + let err = + resolve_fts_field(&schema, "tags[*]", DocumentGranularity::ListElement).unwrap_err(); + assert!(err.to_string().contains("does not exist"), "{err}"); + } + + #[test] + fn normalize_legacy_metadata_as_row_document() { + let metadata = IndexMetadata { + uuid: Uuid::new_v4(), + fields: vec![1], + name: "tags_idx".to_string(), + dataset_version: 1, + fragment_bitmap: None, + index_details: None, + index_version: 3, + created_at: None, + base_id: None, + files: None, + }; + let details = + normalize_inverted_details(&metadata, InvertedIndexDetails::default()).unwrap(); + assert_eq!( + DocumentGranularity::try_from(details.document_granularity).unwrap(), + DocumentGranularity::Row + ); + assert_eq!(details.posting_format_version, Some(3)); + } + + #[test] + fn normalize_rejects_unreleased_v4_metadata() { + let metadata = IndexMetadata { + uuid: Uuid::new_v4(), + fields: vec![1], + name: "tags_idx".to_string(), + dataset_version: 1, + fragment_bitmap: None, + index_details: None, + index_version: 4, + created_at: None, + base_id: None, + files: None, + }; + let details = InvertedIndexDetails::try_from(&InvertedIndexParams::default()).unwrap(); + + let err = normalize_inverted_details(&metadata, details).unwrap_err(); + + assert!(err.to_string().contains("unsupported index_version 4")); + } + + #[test] + fn canonicalize_inverted_details_accepts_legacy_empty_details() { + let legacy = InvertedIndexDetails::default(); + let current = InvertedIndexDetails::try_from(&InvertedIndexParams::default()).unwrap(); + + assert_ne!(legacy, current); + assert_eq!( + canonicalize_inverted_index_details(legacy).unwrap(), + canonicalize_inverted_index_details(current).unwrap() + ); + } } diff --git a/rust/lance/src/index/scalar/label_list.rs b/rust/lance/src/index/scalar/label_list.rs index 27bc49643bb..884346f0d6b 100644 --- a/rust/lance/src/index/scalar/label_list.rs +++ b/rust/lance/src/index/scalar/label_list.rs @@ -3,6 +3,7 @@ use lance_index::metrics::NoOpMetricsCollector; use lance_index::scalar::IndexStore; +use lance_index::scalar::index_files_to_table; use lance_index::scalar::label_list::{ BITMAP_LOOKUP_NAME, LABEL_LIST_NULLS_METADATA_KEY, LABEL_LIST_NULLS_MIN_VERSION, LabelListIndex, }; @@ -142,7 +143,7 @@ pub(in crate::index) async fn merge_segments( index_version: created_index.index_version as i32, created_at: Some(chrono::Utc::now()), base_id: None, - files: Some(created_index.files), + files: Some(index_files_to_table(created_index.files)), ..segments[0].clone() }) } diff --git a/rust/lance/src/index/scalar/ngram.rs b/rust/lance/src/index/scalar/ngram.rs new file mode 100644 index 00000000000..6fec83b09dd --- /dev/null +++ b/rust/lance/src/index/scalar/ngram.rs @@ -0,0 +1,170 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright The Lance Authors + +use std::sync::Arc; + +use datafusion::physical_plan::SendableRecordBatchStream; +use lance_index::metrics::NoOpMetricsCollector; +use lance_index::progress::NoopIndexBuildProgress; +use lance_index::scalar::lance_format::LanceIndexStore; +use lance_index::scalar::ngram::NGramIndex; +use lance_index::scalar::{ + BuiltinIndexType, CreatedIndex, IndexStore, OldIndexDataFilter, ScalarIndexParams, + index_files_to_table, +}; +use lance_table::format::IndexMetadata; +use roaring::RoaringBitmap; +use uuid::Uuid; + +use crate::{ + Dataset, Error, Result, dataset::index::LanceIndexStoreExt, index::DatasetIndexInternalExt, +}; + +async fn collect_ngram_segment_stores( + dataset: &Dataset, + segments: &[IndexMetadata], +) -> Result>> { + let mut stores: Vec> = Vec::with_capacity(segments.len()); + for segment in segments { + let store = LanceIndexStore::from_dataset_for_existing(dataset, segment).await?; + stores.push(Arc::new(store)); + } + Ok(stores) +} + +/// Merge one caller-defined group of source NGram segments into a single segment. +pub(in crate::index) async fn merge_segments( + dataset: &Dataset, + segments: Vec, +) -> Result { + if segments.is_empty() { + return Err(Error::index("No segment metadata was provided".to_string())); + } + + // All source segments must belong to the same column. + let reference_fields = segments[0].fields.as_slice(); + for segment in segments.iter().skip(1) { + if segment.fields.as_slice() != reference_fields { + return Err(Error::invalid_input(format!( + "NGram merge_segments: segment {} has fields {:?}, expected {:?}", + segment.uuid, segment.fields, reference_fields, + ))); + } + } + + let field_id = *segments[0].fields.first().ok_or_else(|| { + Error::invalid_input(format!( + "CreateIndex: segment {} is missing field ids", + segments[0].uuid + )) + })?; + let source_dataset_version = segments + .iter() + .map(|segment| segment.dataset_version) + .min() + .unwrap_or(dataset.manifest.version); + let segment_refs = segments.iter().collect::>(); + + let new_uuid = Uuid::new_v4(); + let frag_reuse_index = dataset.open_frag_reuse_index(&NoOpMetricsCollector).await?; + let has_retired_coverage = segments.iter().any(|segment| { + segment + .deleted_fragment_bitmap(&dataset.fragment_bitmap) + .is_some_and(|retired| !retired.is_empty()) + }); + let requires_rebuild = frag_reuse_index.as_ref().is_some_and(|frag_reuse_index| { + crate::index::append::fragment_reuse_affects_segments( + frag_reuse_index, + segment_refs.iter().copied(), + ) + }); + if has_retired_coverage && !requires_rebuild { + return Err(Error::invalid_input( + "NGram merge_segments: source segments cover retired fragments but no applicable \ + fragment-reuse mapping is available; rebuild the affected segments from the current \ + dataset" + .to_string(), + )); + } + let (created_index, dataset_version, fragment_bitmap) = if requires_rebuild { + let mut fragment_bitmap = segments + .iter() + .map(|segment| { + segment.fragment_bitmap.as_ref().cloned().ok_or_else(|| { + Error::invalid_input(format!( + "CreateIndex: segment {} is missing fragment coverage", + segment.uuid + )) + }) + }) + .collect::>>()? + .into_iter() + .fold(RoaringBitmap::new(), |coverage, segment| coverage | segment); + frag_reuse_index + .as_ref() + .expect("requires_rebuild implies a fragment-reuse index") + .remap_fragment_bitmap(&mut fragment_bitmap)?; + fragment_bitmap &= dataset.fragment_bitmap.as_ref(); + let field_path = dataset.schema().field_path(field_id)?; + ( + super::build_scalar_index( + dataset, + &field_path, + new_uuid, + &ScalarIndexParams::for_builtin(BuiltinIndexType::NGram), + true, + Some(fragment_bitmap.iter().collect()), + None, + Arc::new(NoopIndexBuildProgress), + ) + .await?, + dataset.manifest.version, + fragment_bitmap, + ) + } else { + let (fragment_bitmap, old_data_filters) = + crate::index::append::build_per_segment_filters(dataset, &segment_refs).await?; + let new_store = LanceIndexStore::from_dataset_for_new(dataset, &new_uuid)?; + ( + open_and_merge_segments(dataset, &segment_refs, None, &new_store, &old_data_filters) + .await?, + source_dataset_version, + fragment_bitmap, + ) + }; + + Ok(IndexMetadata { + uuid: new_uuid, + name: segments[0].name.clone(), + fields: vec![field_id], + dataset_version, + fragment_bitmap: Some(fragment_bitmap), + index_details: Some(Arc::new(created_index.index_details)), + index_version: created_index.index_version as i32, + created_at: Some(chrono::Utc::now()), + base_id: None, + files: Some(index_files_to_table(created_index.files)), + }) +} + +/// Merge the given NGram segments with optional newly appended data into a single +/// canonical segment. Pass `None` for `new_data` for a pure consolidation. +pub(in crate::index) async fn open_and_merge_segments( + dataset: &Dataset, + segments: &[&IndexMetadata], + new_data: Option, + new_store: &LanceIndexStore, + old_data_filters: &[Option], +) -> Result { + let segments = segments.iter().map(|&s| s.clone()).collect::>(); + let segment_stores = collect_ngram_segment_stores(dataset, &segments).await?; + let frag_reuse_index = dataset.open_frag_reuse_index(&NoOpMetricsCollector).await?; + NGramIndex::merge_segments( + &segment_stores, + new_data, + new_store, + old_data_filters, + frag_reuse_index, + ) + .await +} diff --git a/rust/lance/src/index/scalar/rtree.rs b/rust/lance/src/index/scalar/rtree.rs new file mode 100644 index 00000000000..de9fdece137 --- /dev/null +++ b/rust/lance/src/index/scalar/rtree.rs @@ -0,0 +1,93 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright The Lance Authors + +use std::sync::Arc; + +use lance_index::metrics::NoOpMetricsCollector; +use lance_index::scalar::lance_format::LanceIndexStore; +use lance_index::scalar::rtree::RTreeIndex; +use lance_index::scalar::{OldIndexDataFilter, index_files_to_table}; +use lance_select::RowSetOps; +use lance_table::format::IndexMetadata; +use uuid::Uuid; + +use crate::{Dataset, Error, Result, dataset::index::LanceIndexStoreExt}; + +fn filter_keeps_nothing(filter: &Option) -> bool { + match filter { + Some(OldIndexDataFilter::Fragments { to_keep, .. }) => to_keep.is_empty(), + Some(OldIndexDataFilter::RowIds(valid)) => valid.is_empty(), + None => false, + } +} + +/// Merge one caller-defined group of source RTree segments into a single segment. +pub(in crate::index) async fn merge_segments( + dataset: &Dataset, + segments: Vec, +) -> Result { + if segments.is_empty() { + return Err(Error::index("No segment metadata was provided".to_string())); + } + + let field_id = *segments[0].fields.first().ok_or_else(|| { + Error::invalid_input(format!( + "CreateIndex: segment {} is missing field ids", + segments[0].uuid + )) + })?; + let field_path = dataset.schema().field_path(field_id)?; + let dataset_version = segments + .iter() + .map(|segment| segment.dataset_version) + .min() + .unwrap_or(dataset.manifest.version); + let segment_refs = segments.iter().collect::>(); + let (fragment_bitmap, old_data_filters) = + crate::index::append::build_per_segment_filters(dataset, &segment_refs).await?; + + let mut source_indices = Vec::with_capacity(segments.len()); + let mut source_filters = Vec::with_capacity(old_data_filters.len()); + let all_keep_nothing = old_data_filters.iter().all(filter_keeps_nothing); + for (position, (segment, filter)) in segments.iter().zip(&old_data_filters).enumerate() { + if filter_keeps_nothing(filter) && !(all_keep_nothing && position == 0) { + continue; + } + let scalar_index = + super::open_scalar_index(dataset, &field_path, segment, &NoOpMetricsCollector).await?; + let rtree_index = scalar_index + .as_any() + .downcast_ref::() + .ok_or_else(|| { + Error::index(format!( + "merge_existing_index_segments: expected RTree segment {}, got {:?}", + segment.uuid, + scalar_index.index_type() + )) + })?; + source_indices.push(Arc::new(rtree_index.clone())); + source_filters.push(filter.clone()); + } + + let new_uuid = Uuid::new_v4(); + let new_store = LanceIndexStore::from_dataset_for_new(dataset, &new_uuid)?; + let created_index = lance_index::scalar::rtree::merge_rtree_indices( + &source_indices, + &new_store, + &source_filters, + ) + .await?; + + Ok(IndexMetadata { + uuid: new_uuid, + fields: vec![field_id], + dataset_version, + fragment_bitmap: Some(fragment_bitmap), + index_details: Some(Arc::new(created_index.index_details)), + index_version: created_index.index_version as i32, + created_at: Some(chrono::Utc::now()), + base_id: None, + files: Some(index_files_to_table(created_index.files)), + ..segments[0].clone() + }) +} diff --git a/rust/lance/src/index/scalar/zonemap.rs b/rust/lance/src/index/scalar/zonemap.rs index 0cbd98f2c40..a3524b4955e 100644 --- a/rust/lance/src/index/scalar/zonemap.rs +++ b/rust/lance/src/index/scalar/zonemap.rs @@ -4,6 +4,7 @@ use std::sync::Arc; use lance_index::metrics::NoOpMetricsCollector; +use lance_index::scalar::index_files_to_table; use lance_index::scalar::lance_format::LanceIndexStore; use lance_index::scalar::zonemap::ZoneMapIndex; use lance_table::format::IndexMetadata; @@ -80,7 +81,7 @@ pub(in crate::index) async fn merge_segments( index_version: created_index.index_version as i32, created_at: Some(chrono::Utc::now()), base_id: None, - files: Some(created_index.files), + files: Some(index_files_to_table(created_index.files)), ..segments[0].clone() }) } diff --git a/rust/lance/src/index/scalar_logical.rs b/rust/lance/src/index/scalar_logical.rs index 4ff0da7f091..3b7dd4fcbda 100644 --- a/rust/lance/src/index/scalar_logical.rs +++ b/rust/lance/src/index/scalar_logical.rs @@ -23,6 +23,21 @@ use crate::dataset::Dataset; use crate::index::scalar::fetch_index_details; use crate::index::{DatasetIndexExt, DatasetIndexInternalExt}; +/// Query-time view that exposes several physical scalar index segments as a single [`ScalarIndex`]. +/// +/// A named scalar index can be built incrementally, producing multiple physical segments that +/// each cover a disjoint set of fragments. When such an index is opened, the loader bundles +/// the segments into a `LogicalScalarIndex` so the scanner can treat them as one index: queries +/// are fanned out to every segment in parallel and the row-address results are unioned together. +/// +/// All segments must share the same [`IndexType`]; mixing types is rejected at construction. +/// Per-segment [`SearchResult`] precision is preserved when combining: a union of `Exact` +/// results stays `Exact`, a union containing `AtMost` results yields `AtMost`, and a union +/// containing `AtLeast` results yields `AtLeast`. Combining `AtMost` and `AtLeast` segments in +/// the same query is not supported. +/// +/// This is a read-only wrapper. [`ScalarIndex::remap`] and [`ScalarIndex::update`] both return +/// an error — callers must rebuild the index to consolidate segments before mutating it. #[derive(Debug)] pub struct LogicalScalarIndex { name: String, @@ -136,6 +151,12 @@ impl ScalarIndex for LogicalScalarIndex { combine_search_results(results) } + fn results_are_row_addresses(&self) -> bool { + // All segments of a logical index share the same underlying index type, + // so they agree on the result domain. + self.segments[0].results_are_row_addresses() + } + fn can_remap(&self) -> bool { false } @@ -274,6 +295,11 @@ fn union_fragment_bitmaps(indices: &[IndexMetadata], index_name: &str) -> Result Ok(combined) } +/// Return the union of fragment bitmaps across every usable segment of a named scalar index. +/// +/// Only segments whose fragment bitmap intersects the dataset's current fragment set are +/// considered. Returns `Ok(None)` when no such segment exists, `Ok(Some(bitmap))` otherwise. +/// Errors if the segments disagree on their underlying index type. pub async fn scalar_index_fragment_bitmap( dataset: &Dataset, column: &str, @@ -290,6 +316,13 @@ pub async fn scalar_index_fragment_bitmap( } } +/// Open a named scalar index, transparently bundling multiple segments when present. +/// +/// Loads every segment registered under `index_name` whose fragment bitmap intersects the +/// dataset. If exactly one usable segment exists it is returned directly; if multiple exist +/// they are wrapped in a [`LogicalScalarIndex`] so the caller sees a single [`ScalarIndex`]. +/// Errors if no usable segment exists (the scanner planned a query against an index that is +/// not present) or if the segments mix incompatible types. pub async fn open_named_scalar_index( dataset: &Dataset, column: &str, @@ -1462,6 +1495,224 @@ mod tests { ); } + #[tokio::test] + async fn test_ngram_segment_merge_rebuilds_after_deferred_compaction() { + let test_dir = TempStrDir::default(); + let schema = Arc::new(arrow_schema::Schema::new(vec![arrow_schema::Field::new( + "text", + arrow_schema::DataType::Utf8, + true, + )])); + let make_batch = |values| { + arrow_array::RecordBatch::try_new( + schema.clone(), + vec![Arc::new(arrow_array::StringArray::from(values))], + ) + .unwrap() + }; + let reader = arrow_array::RecordBatchIterator::new( + vec![Ok(make_batch(vec![Some("alpha needle"), None]))], + schema.clone(), + ); + Dataset::write( + reader, + test_dir.as_str(), + Some(WriteParams { + max_rows_per_file: 2, + ..Default::default() + }), + ) + .await + .unwrap(); + let appended = arrow_array::RecordBatchIterator::new( + vec![Ok(make_batch(vec![ + Some("beta needle"), + Some("gamma stack"), + ]))], + schema.clone(), + ); + let mut dataset = Dataset::write( + appended, + test_dir.as_str(), + Some(WriteParams { + mode: WriteMode::Append, + ..Default::default() + }), + ) + .await + .unwrap(); + + let params = ScalarIndexParams::for_builtin(BuiltinIndexType::NGram); + let mut segments = Vec::new(); + for fragment in dataset.get_fragments() { + segments.push( + CreateIndexBuilder::new(&mut dataset, &["text"], IndexType::NGram, ¶ms) + .name("text_ngram".to_string()) + .fragments(vec![fragment.id() as u32]) + .execute_uncommitted() + .await + .unwrap(), + ); + } + let source_version = segments[0].dataset_version; + dataset + .create_index( + &["text"], + IndexType::NGram, + Some("compaction_guard".to_string()), + ¶ms, + false, + ) + .await + .unwrap(); + + let metrics = compact_files( + &mut dataset, + CompactionOptions { + target_rows_per_fragment: 10, + defer_index_remap: true, + ..Default::default() + }, + None, + ) + .await + .unwrap(); + assert!(metrics.fragments_removed > 0 && metrics.fragments_added > 0); + assert!(dataset.version().version > source_version); + + let direct_commit_err = dataset + .commit_existing_index_segments("text_ngram_direct", "text", segments.clone()) + .await + .unwrap_err(); + assert!( + direct_commit_err + .to_string() + .contains("must be rebuilt or merged") + ); + + let rebuild_version = dataset.version().version; + let merged = dataset + .merge_existing_index_segments(segments) + .await + .unwrap(); + assert_eq!(merged.dataset_version, rebuild_version); + dataset + .commit_existing_index_segments("text_ngram", "text", vec![merged]) + .await + .unwrap(); + + let committed = dataset.load_indices_by_name("text_ngram").await.unwrap(); + assert_eq!(committed.len(), 1); + assert_eq!(committed[0].dataset_version, rebuild_version); + assert_eq!( + committed[0].fragment_bitmap.as_ref().unwrap(), + dataset.fragment_bitmap.as_ref() + ); + + let logical = + open_named_scalar_index(&dataset, "text", "text_ngram", &NoOpMetricsCollector) + .await + .unwrap(); + let result = logical + .search( + &lance_index::scalar::TextQuery::StringContains("needle".to_string()), + &NoOpMetricsCollector, + ) + .await + .unwrap(); + let row_addrs = match result { + SearchResult::AtMost(row_addrs) => row_addrs, + other => panic!("expected AtMost result from ngram, got {other:?}"), + }; + assert_eq!(row_addrs.true_rows().row_addrs().unwrap().count(), 2); + } + + #[tokio::test] + async fn test_ngram_segment_merge_rejects_retired_coverage_without_remap() { + let test_dir = TempStrDir::default(); + let schema = Arc::new(arrow_schema::Schema::new(vec![arrow_schema::Field::new( + "text", + arrow_schema::DataType::Utf8, + true, + )])); + let make_batch = |values| { + arrow_array::RecordBatch::try_new( + schema.clone(), + vec![Arc::new(arrow_array::StringArray::from(values))], + ) + .unwrap() + }; + let reader = arrow_array::RecordBatchIterator::new( + vec![Ok(make_batch(vec!["alpha", "beta"]))], + schema.clone(), + ); + Dataset::write( + reader, + test_dir.as_str(), + Some(WriteParams { + max_rows_per_file: 2, + ..Default::default() + }), + ) + .await + .unwrap(); + let appended = arrow_array::RecordBatchIterator::new( + vec![Ok(make_batch(vec!["gamma", "delta"]))], + schema, + ); + let mut dataset = Dataset::write( + appended, + test_dir.as_str(), + Some(WriteParams { + mode: WriteMode::Append, + ..Default::default() + }), + ) + .await + .unwrap(); + + let params = ScalarIndexParams::for_builtin(BuiltinIndexType::NGram); + let mut segments = Vec::new(); + for fragment in dataset.get_fragments() { + segments.push( + CreateIndexBuilder::new(&mut dataset, &["text"], IndexType::NGram, ¶ms) + .name("text_ngram".to_string()) + .fragments(vec![fragment.id() as u32]) + .execute_uncommitted() + .await + .unwrap(), + ); + } + + compact_files( + &mut dataset, + CompactionOptions { + target_rows_per_fragment: 10, + defer_index_remap: true, + ..Default::default() + }, + None, + ) + .await + .unwrap(); + + let direct_err = dataset + .commit_existing_index_segments("text_ngram", "text", segments.clone()) + .await + .unwrap_err(); + assert!(direct_err.to_string().contains("must be rebuilt or merged")); + + let merge_err = dataset + .merge_existing_index_segments(segments) + .await + .unwrap_err(); + assert!( + merge_err + .to_string() + .contains("no applicable fragment-reuse mapping is available") + ); + } + #[tokio::test] async fn test_fmindex_merge_single_segment_passthrough() { let test_dir = TempStrDir::default(); diff --git a/rust/lance/src/index/vector.rs b/rust/lance/src/index/vector.rs index c9920fb8adf..91231b69444 100644 --- a/rust/lance/src/index/vector.rs +++ b/rust/lance/src/index/vector.rs @@ -19,13 +19,14 @@ pub mod utils; mod fixture_test; use self::{ivf::*, pq::PQIndex}; +use arrow_array::Array; use arrow_schema::{DataType, Schema}; use builder::{IvfIndexBuilder, VectorIndexBuildSummary}; use datafusion::physical_plan::SendableRecordBatchStream; use datafusion::physical_plan::stream::RecordBatchStreamAdapter; use futures::stream; use lance_core::utils::tempfile::TempStdDir; -use lance_file::previous::reader::FileReader as PreviousFileReader; +use lance_file::versions::v1::reader::FileReader as V1FileReader; use lance_index::frag_reuse::FragReuseIndex; use lance_index::metrics::NoOpMetricsCollector; use lance_index::optimize::OptimizeOptions; @@ -516,6 +517,7 @@ async fn prepare_vector_segment_build( progress: Arc, mode: &str, require_precomputed_ivf: bool, + fragment_ids: Option<&[u32]>, ) -> Result<(DataType, IndexType, IvfBuildParams, Box)> { let stages = ¶ms.stages; @@ -557,15 +559,29 @@ async fn prepare_vector_segment_build( validate_supported_rq_num_bits(rq_params.num_bits)?; } - let num_rows = dataset.count_rows(None).await?; - let num_partitions = ivf_params0.num_partitions.unwrap_or_else(|| { - recommended_num_partitions( - num_rows, - ivf_params0 - .target_partition_size - .unwrap_or(index_type.target_partition_size()), - ) - }); + let num_partitions = match (ivf_params0.num_partitions, ivf_params0.centroids.as_ref()) { + (Some(num_partitions), Some(centroids)) if num_partitions != centroids.len() => { + return Err(Error::index(format!( + "{mode}: num_partitions {} does not match precomputed IVF centroids length {}", + num_partitions, + centroids.len() + ))); + } + (Some(num_partitions), _) => num_partitions, + (None, Some(centroids)) => centroids.len(), + (None, None) => { + let num_rows = match fragment_ids { + Some(fragment_ids) => dataset.count_rows_in_fragments(fragment_ids).await?, + None => dataset.count_rows(None).await?, + }; + recommended_num_partitions( + num_rows, + ivf_params0 + .target_partition_size + .unwrap_or(index_type.target_partition_size()), + ) + } + }; let mut ivf_params = ivf_params0.clone(); ivf_params.num_partitions = Some(num_partitions); @@ -602,6 +618,7 @@ pub(crate) async fn build_distributed_vector_index( progress.clone(), "Build Distributed Vector Index", true, + Some(fragment_ids), ) .await?; let stages = ¶ms.stages; @@ -945,6 +962,55 @@ pub(crate) async fn build_vector_index( params: &VectorIndexParams, frag_reuse_index: Option>, progress: Arc, +) -> Result> { + build_vector_index_impl( + dataset, + column, + name, + uuid, + params, + frag_reuse_index, + progress, + None, + ) + .await +} + +/// Build a standalone vector index segment over a subset of fragments. +#[allow(clippy::too_many_arguments)] +pub(crate) async fn build_filtered_vector_index( + dataset: &Dataset, + column: &str, + name: &str, + uuid: Uuid, + params: &VectorIndexParams, + frag_reuse_index: Option>, + fragment_ids: &[u32], + progress: Arc, +) -> Result> { + build_vector_index_impl( + dataset, + column, + name, + uuid, + params, + frag_reuse_index, + progress, + Some(fragment_ids), + ) + .await +} + +#[allow(clippy::too_many_arguments)] +async fn build_vector_index_impl( + dataset: &Dataset, + column: &str, + name: &str, + uuid: Uuid, + params: &VectorIndexParams, + frag_reuse_index: Option>, + progress: Arc, + fragment_ids: Option<&[u32]>, ) -> Result> { let (element_type, index_type, ivf_params, shuffler) = prepare_vector_segment_build( dataset, @@ -953,6 +1019,7 @@ pub(crate) async fn build_vector_index( progress.clone(), "Build Vector Index", false, + fragment_ids, ) .await?; let stages = ¶ms.stages; @@ -971,10 +1038,11 @@ pub(crate) async fn build_vector_index( (), frag_reuse_index, )? + .with_optional_fragment_filter(fragment_ids) .with_progress(progress.clone()) .build() .await?; - return Ok(summary.files); + Ok(summary.files) } DataType::UInt8 => { let summary = IvfIndexBuilder::::new( @@ -988,17 +1056,16 @@ pub(crate) async fn build_vector_index( (), frag_reuse_index, )? + .with_optional_fragment_filter(fragment_ids) .with_progress(progress.clone()) .build() .await?; - return Ok(summary.files); - } - _ => { - return Err(Error::index(format!( - "Build Vector Index: invalid data type: {:?}", - element_type - ))); + Ok(summary.files) } + _ => Err(Error::index(format!( + "Build Vector Index: invalid data type: {:?}", + element_type + ))), }, IndexType::IvfPq => { let len = stages.len(); @@ -1011,6 +1078,12 @@ pub(crate) async fn build_vector_index( match params.version { IndexFileVersion::Legacy => { + if fragment_ids.is_some() { + return Err(Error::index( + "Build Vector Index: filtered IVF_PQ builds do not support legacy format" + .to_string(), + )); + } let files = build_ivf_pq_index( dataset, column, @@ -1022,7 +1095,7 @@ pub(crate) async fn build_vector_index( progress.clone(), ) .await?; - return Ok(files); + Ok(files) } IndexFileVersion::V3 => { let mut builder = IvfIndexBuilder::::new( @@ -1039,10 +1112,11 @@ pub(crate) async fn build_vector_index( let summary = builder .with_transpose(!params.skip_transpose) + .with_optional_fragment_filter(fragment_ids) .with_progress(progress.clone()) .build() .await?; - return Ok(summary.files); + Ok(summary.files) } } } @@ -1065,10 +1139,11 @@ pub(crate) async fn build_vector_index( (), frag_reuse_index, )? + .with_optional_fragment_filter(fragment_ids) .with_progress(progress.clone()) .build() .await?; - return Ok(summary.files); + Ok(summary.files) } IndexType::IvfRq => { let StageParams::RQ(rq_params) = &stages[1] else { @@ -1092,10 +1167,11 @@ pub(crate) async fn build_vector_index( let summary = builder .with_transpose(!params.skip_transpose) + .with_optional_fragment_filter(fragment_ids) .with_progress(progress.clone()) .build() .await?; - return Ok(summary.files); + Ok(summary.files) } IndexType::IvfHnswFlat => { let StageParams::Hnsw(hnsw_params) = &stages[1] else { @@ -1117,10 +1193,11 @@ pub(crate) async fn build_vector_index( hnsw_params.clone(), frag_reuse_index, )? + .with_optional_fragment_filter(fragment_ids) .with_progress(progress.clone()) .build() .await?; - return Ok(summary.files); + Ok(summary.files) } _ => { let summary = IvfIndexBuilder::::new( @@ -1134,10 +1211,11 @@ pub(crate) async fn build_vector_index( hnsw_params.clone(), frag_reuse_index, )? + .with_optional_fragment_filter(fragment_ids) .with_progress(progress.clone()) .build() .await?; - return Ok(summary.files); + Ok(summary.files) } } } @@ -1165,10 +1243,11 @@ pub(crate) async fn build_vector_index( hnsw_params.clone(), frag_reuse_index, )? + .with_optional_fragment_filter(fragment_ids) .with_progress(progress.clone()) .build() .await?; - return Ok(summary.files); + Ok(summary.files) } IndexType::IvfHnswSq => { let StageParams::Hnsw(hnsw_params) = &stages[1] else { @@ -1194,17 +1273,16 @@ pub(crate) async fn build_vector_index( hnsw_params.clone(), frag_reuse_index, )? + .with_optional_fragment_filter(fragment_ids) .with_progress(progress.clone()) .build() .await?; - return Ok(summary.files); - } - _ => { - return Err(Error::index(format!( - "Build Vector Index: invalid index type: {:?}", - index_type - ))); + Ok(summary.files) } + _ => Err(Error::index(format!( + "Build Vector Index: invalid index type: {:?}", + index_type + ))), } } @@ -1633,7 +1711,7 @@ pub(crate) async fn open_vector_index_v2( dataset: Arc, column: &str, uuid: &Uuid, - reader: PreviousFileReader, + reader: V1FileReader, frag_reuse_index: Option>, ) -> Result> { let index_metadata = reader @@ -1951,7 +2029,7 @@ fn derive_rabit_params(rabit_quantizer: &RabitQuantizer) -> RQBuildParams { /// Extract HNSW build parameters from the source vector index statistics. /// Returns default parameters if extraction fails. /// TODO: support consistently deriving all the original parameters -fn derive_hnsw_params(source_index: &dyn VectorIndex) -> HnswBuildParams { +pub(crate) fn derive_hnsw_params(source_index: &dyn VectorIndex) -> HnswBuildParams { let default_params = HnswBuildParams { max_level: 4, m: 20, @@ -1984,18 +2062,124 @@ fn derive_hnsw_params(source_index: &dyn VectorIndex) -> HnswBuildParams { .and_then(|v| v.as_u64()) .map(|v| v as usize) .unwrap_or(100); + let prefetch_distance = params + .get("prefetch_distance") + .and_then(|v| v.as_u64()) + .map(|v| v as usize); return HnswBuildParams { max_level, m, ef_construction, - prefetch_distance: None, + prefetch_distance, }; } default_params } +fn vector_index_type(index: &dyn VectorIndex) -> IndexType { + match index.sub_index_type() { + (SubIndexType::Flat, QuantizationType::Flat | QuantizationType::FlatBin) => { + IndexType::IvfFlat + } + (SubIndexType::Flat, QuantizationType::Product) => IndexType::IvfPq, + (SubIndexType::Flat, QuantizationType::Scalar) => IndexType::IvfSq, + (SubIndexType::Flat, QuantizationType::Rabit) => IndexType::IvfRq, + (SubIndexType::Hnsw, QuantizationType::Flat | QuantizationType::FlatBin) => { + IndexType::IvfHnswFlat + } + (SubIndexType::Hnsw, QuantizationType::Product) => IndexType::IvfHnswPq, + (SubIndexType::Hnsw, QuantizationType::Scalar) => IndexType::IvfHnswSq, + (SubIndexType::Hnsw, QuantizationType::Rabit) => IndexType::Vector, + } +} + +/// Derive structural build parameters for a new, independently trained segment. +/// +/// Learned IVF centroids, quantizer codebooks, and rotations are deliberately +/// omitted so the new segment is valid for its own fragment set. +pub(crate) fn fresh_vector_segment_params( + metadata: &IndexMetadata, + index: &dyn VectorIndex, +) -> Result { + if let Some(params) = metadata + .index_details + .as_deref() + .and_then(details::vector_params_from_details) + && params.metric_type == index.metric_type() + && params.index_type() == vector_index_type(index) + { + return Ok(params); + } + + let mut ivf_params = derive_ivf_params(index.ivf_model()); + ivf_params.centroids = None; + #[allow(deprecated)] + { + ivf_params.retrain = false; + } + + let metric_type = index.metric_type(); + let quantizer = index.quantizer(); + Ok(match index.sub_index_type() { + (SubIndexType::Flat, QuantizationType::Flat | QuantizationType::FlatBin) => { + VectorIndexParams::with_ivf_flat_params(metric_type, ivf_params) + } + (SubIndexType::Flat, QuantizationType::Product) => { + let quantizer: ProductQuantizer = quantizer.try_into()?; + let mut pq_params = derive_pq_params(&quantizer); + pq_params.codebook = None; + VectorIndexParams::with_ivf_pq_params(metric_type, ivf_params, pq_params) + } + (SubIndexType::Flat, QuantizationType::Scalar) => { + let quantizer: ScalarQuantizer = quantizer.try_into()?; + VectorIndexParams::with_ivf_sq_params( + metric_type, + ivf_params, + derive_sq_params(&quantizer), + ) + } + (SubIndexType::Flat, QuantizationType::Rabit) => { + let quantizer: RabitQuantizer = quantizer.try_into()?; + VectorIndexParams::with_ivf_rq_params( + metric_type, + ivf_params, + derive_rabit_params(&quantizer), + ) + } + (SubIndexType::Hnsw, QuantizationType::Flat | QuantizationType::FlatBin) => { + VectorIndexParams::ivf_hnsw(metric_type, ivf_params, derive_hnsw_params(index)) + } + (SubIndexType::Hnsw, QuantizationType::Product) => { + let quantizer: ProductQuantizer = quantizer.try_into()?; + let mut pq_params = derive_pq_params(&quantizer); + pq_params.codebook = None; + VectorIndexParams::with_ivf_hnsw_pq_params( + metric_type, + ivf_params, + derive_hnsw_params(index), + pq_params, + ) + } + (SubIndexType::Hnsw, QuantizationType::Scalar) => { + let quantizer: ScalarQuantizer = quantizer.try_into()?; + VectorIndexParams::with_ivf_hnsw_sq_params( + metric_type, + ivf_params, + derive_hnsw_params(index), + derive_sq_params(&quantizer), + ) + } + (SubIndexType::Hnsw, QuantizationType::Rabit) => { + return Err(Error::index( + "Cannot build a fresh IVF_HNSW_RQ segment: this index type is unsupported" + .to_string(), + )); + } + }) +} + #[cfg(test)] mod tests { use super::*; @@ -2747,7 +2931,8 @@ mod tests { .await .unwrap(); let arrow_schema = ArrowSchema::new(vec![Field::new("dummy", ArrowDataType::Int32, true)]); - let mut v2w = lance_file::writer::FileWriter::try_new( + let mut v2w = lance_file::versions::create_writer( + dataset_format_version(&dataset), writer, lance_core::datatypes::Schema::try_from(&arrow_schema).unwrap(), FileWriterOptions::default(), diff --git a/rust/lance/src/index/vector/builder.rs b/rust/lance/src/index/vector/builder.rs index bf968d9744f..9c4c26e6ed8 100644 --- a/rust/lance/src/index/vector/builder.rs +++ b/rust/lance/src/index/vector/builder.rs @@ -27,8 +27,9 @@ use lance_core::datatypes::Schema; use lance_core::utils::tempfile::TempStdDir; use lance_core::utils::tokio::{get_num_compute_intensive_cpus, spawn_cpu}; use lance_core::{Error, ROW_ID_FIELD, Result}; -use lance_encoding::version::LanceFileVersion; -use lance_file::writer::{FileWriter, FileWriterOptions}; +use lance_file::version::ConcreteFileVersion; +use lance_file::versions as file_versions; +use lance_file::writer::FileWriterOptions; use lance_index::frag_reuse::FragReuseIndex; use lance_index::metrics::NoOpMetricsCollector; use lance_index::optimize::OptimizeOptions; @@ -63,7 +64,7 @@ use lance_index::{ }; use lance_index::{ INDEX_METADATA_SCHEMA_KEY, IndexMetadata, IndexType, MAX_PARTITION_SIZE_FACTOR, - MIN_PARTITION_SIZE_PERCENT, + MIN_PARTITION_SIZE_PERCENT, scalar::OldIndexDataFilter, }; use lance_io::local::to_local_path; use lance_io::stream::RecordBatchStream; @@ -74,11 +75,14 @@ use lance_table::format::IndexFile; use log::info; use object_store::path::Path; use prost::Message; +use roaring::RoaringBitmap; +use tokio::sync::OnceCell; use tracing::{Level, instrument, span}; use crate::Dataset; use crate::dataset::ProjectionRequest; use crate::dataset::index::dataset_format_version; +use crate::index::append::build_old_data_filter; use crate::index::vector::ivf::v2::PartitionEntry; use crate::index::vector::utils::infer_vector_dim; @@ -118,6 +122,89 @@ fn apply_centroid_splits( )?) } +/// An index segment an optimize pass reads existing rows from, paired with the rows +/// that segment is still allowed to contribute. +/// +/// A segment's index file keeps every row it was built with, but the rows it may still +/// contribute shrink afterwards: an update rewrites a row and either prunes its +/// fragment from the segment's bitmap (in-place column rewrite) or deletion-marks the +/// old physical row while the rewritten copy reuses its stable row id. Copying such a +/// row into a merged segment would duplicate it, because the merged coverage also spans +/// the fresh copy and nothing downstream can tell the two apart. +/// +/// The filter is resolved on first use rather than up front: under stable row ids +/// building it loads every covered fragment's row-id sequence, and the common optimize +/// pass appends a delta without reading a single existing row. +#[derive(Clone)] +pub struct ExistingIndex { + pub index: Arc, + coverage: Option>, +} + +/// The inputs [`ExistingIndex::old_data_filter`] needs, plus the filter once built. +struct SegmentCoverage { + dataset: Dataset, + effective_frags: RoaringBitmap, + deleted_frags: RoaringBitmap, + filter: OnceCell>, +} + +impl ExistingIndex { + /// An existing index whose rows are all still valid. + pub fn unfiltered(index: Arc) -> Self { + Self { + index, + coverage: None, + } + } + + /// An existing index that may only contribute rows still live in `effective_frags`. + pub fn with_coverage( + index: Arc, + dataset: Dataset, + effective_frags: RoaringBitmap, + deleted_frags: RoaringBitmap, + ) -> Self { + Self { + index, + coverage: Some(Arc::new(SegmentCoverage { + dataset, + effective_frags, + deleted_frags, + filter: OnceCell::new(), + })), + } + } + + /// Whether the filter has already been built. A pass that reads no existing rows + /// must never pay for one. + #[cfg(test)] + pub(crate) fn filter_is_built(&self) -> bool { + self.coverage + .as_deref() + .is_some_and(|coverage| coverage.filter.initialized()) + } + + /// The filter to apply to this segment's stored rows, or `None` when every row it + /// holds is still valid. Built once and shared by all partitions. + pub(crate) async fn old_data_filter(&self) -> Result> { + let Some(coverage) = self.coverage.as_deref() else { + return Ok(None); + }; + let filter = coverage + .filter + .get_or_try_init(|| { + build_old_data_filter( + &coverage.dataset, + &coverage.effective_frags, + &coverage.deleted_frags, + ) + }) + .await?; + Ok(filter.as_ref()) + } +} + // Builder for IVF index // The builder will train the IVF model and quantizer, shuffle the dataset, and build the sub index // for each partition. @@ -148,7 +235,7 @@ pub struct IvfIndexBuilder { shuffle_data_input: Mutex>, // fields for merging indices / remapping - existing_indices: Vec>, + existing_indices: Vec, frag_reuse_index: Option>, @@ -163,7 +250,7 @@ pub struct IvfIndexBuilder { transpose_codes: bool, // lance file version for writing index files - format_version: LanceFileVersion, + format_version: ConcreteFileVersion, progress: Arc, } @@ -278,7 +365,7 @@ impl IvfIndexBuilder quantizer: Some(ivf_index.quantizer().try_into()?), shuffle_reader: None, shuffle_data_input: Mutex::new(None), - existing_indices: vec![index], + existing_indices: vec![ExistingIndex::unfiltered(index)], frag_reuse_index: None, fragment_filter: None, optimize_options: None, @@ -344,7 +431,7 @@ impl IvfIndexBuilder }; log::info!("remap {} partitions", ivf.num_partitions()); - let existing_index = self.existing_indices[0].clone(); + let existing_index = self.existing_indices[0].index.clone(); let mapping = Arc::new(mapping.clone()); let build_iter = (0..ivf.num_partitions()).map(move |part_id| { @@ -388,14 +475,29 @@ impl IvfIndexBuilder self } + /// Read existing rows from `indices`, keeping every row they hold. pub fn with_existing_indices(&mut self, indices: Vec>) -> &mut Self { - self.existing_indices = indices; + self.existing_indices = indices.into_iter().map(ExistingIndex::unfiltered).collect(); + self + } + + /// Read existing rows from `sources`, keeping only the rows each segment is still + /// allowed to contribute. See [`ExistingIndex`]. + pub fn with_existing_index_sources(&mut self, sources: Vec) -> &mut Self { + self.existing_indices = sources; self } /// Set fragment filter for distributed indexing pub fn with_fragment_filter(&mut self, fragment_ids: Vec) -> &mut Self { - self.fragment_filter = Some(fragment_ids); + self.fragment_filter = Some(Dataset::normalize_fragment_ids(&fragment_ids)); + self + } + + pub fn with_optional_fragment_filter(&mut self, fragment_ids: Option<&[u32]>) -> &mut Self { + if let Some(fragment_ids) = fragment_ids { + self.fragment_filter = Some(Dataset::normalize_fragment_ids(fragment_ids)); + } self } @@ -553,19 +655,11 @@ impl IvfIndexBuilder return Ok(None); }; match &self.fragment_filter { - Some(fragment_ids) => { - let fragments: Vec<_> = dataset - .get_fragments() - .into_iter() - .filter(|f| fragment_ids.contains(&(f.id() as u32))) - .collect(); - let counts = futures::stream::iter(fragments) - .map(|f| async move { f.count_rows(None).await }) - .buffer_unordered(16) // ref: Dataset::count_all_rows() - .try_collect::>() - .await?; - Ok(Some(counts.iter().sum::() as u64)) - } + Some(fragment_ids) => Ok(Some( + dataset + .count_rows_in_existing_fragments(fragment_ids) + .await? as u64, + )), None => Ok(Some(dataset.count_rows(None).await? as u64)), } } @@ -603,14 +697,9 @@ impl IvfIndexBuilder "applying fragment filter for distributed indexing: {:?}", fragment_ids ); - // Filter fragments by converting fragment_ids to Fragment objects - let all_fragments = dataset.fragments(); - let filtered_fragments: Vec<_> = all_fragments - .iter() - .filter(|fragment| fragment_ids.contains(&(fragment.id as u32))) - .cloned() - .collect(); - builder.with_fragments(filtered_fragments); + builder.with_fragments( + dataset.get_existing_fragment_metadata_from_ids(fragment_ids), + ); } let (vector_type, _) = get_vector_type(dataset.schema(), &self.column)?; @@ -1030,12 +1119,13 @@ impl IvfIndexBuilder #[instrument(name = "take_partition_batches", level = "debug", skip_all)] async fn take_partition_batches( part_id: usize, - existing_indices: &[Arc], + existing_indices: &[ExistingIndex], reader: Option<&dyn ShuffleReader>, ) -> Result<(Vec, f64)> { let mut batches = Vec::new(); - for existing_index in existing_indices.iter() { - let existing_index = existing_index + for source in existing_indices.iter() { + let existing_index = source + .index .as_any() .downcast_ref::>() .ok_or(Error::invalid_input("existing index is not IVF index"))?; @@ -1046,6 +1136,11 @@ impl IvfIndexBuilder continue; } + // Resolved before the partition is decoded: partitions are built + // concurrently, so whichever one builds the filter would otherwise hold a + // decoded partition per in-flight task while the rest wait on it. + let old_data_filter = source.old_data_filter().await?; + let part_storage = existing_index.load_partition_storage(part_id, None).await?; let mut part_batches = part_storage.to_batches()?.collect::>(); // for PQ, the PQ codes are transposed, so we need to transpose them back @@ -1087,6 +1182,18 @@ impl IvfIndexBuilder _ => {} } + // Drop rows this segment may no longer contribute. They are physically + // still in its index file, and the merged segment covers their live copies + // again, so keeping them would emit the same row twice. + if let Some(filter) = old_data_filter { + for batch in part_batches.iter_mut() { + let keep = filter.filter_row_ids(batch[ROW_ID].as_primitive::()); + if keep.true_count() < batch.num_rows() { + *batch = arrow::compute::filter_record_batch(batch, &keep)?; + } + } + } + batches.extend(part_batches); } @@ -1140,23 +1247,22 @@ impl IvfIndexBuilder let storage_path = self.index_dir.clone().join(INDEX_AUXILIARY_FILE_NAME); let index_path = self.index_dir.clone().join(INDEX_FILE_NAME); - let writer_options = FileWriterOptions { - format_version: Some(self.format_version), - ..Default::default() - }; + let writer_options = FileWriterOptions::default(); let mut storage_writer = if is_flat { None } else { let mut fields = vec![ROW_ID_FIELD.clone(), quantizer.field()]; fields.extend(quantizer.extra_fields()); let storage_schema: Schema = (&arrow_schema::Schema::new(fields)).try_into()?; - Some(FileWriter::try_new( + Some(file_versions::create_writer( + self.format_version, self.store.create(&storage_path).await?, storage_schema, writer_options.clone(), )?) }; - let mut index_writer = FileWriter::try_new( + let mut index_writer = file_versions::create_writer( + self.format_version, self.store.create(&index_path).await?, S::schema().as_ref().try_into()?, writer_options.clone(), @@ -1224,7 +1330,8 @@ impl IvfIndexBuilder if storage_writer.is_none() { let storage_schema: Schema = batch.schema_ref().as_ref().try_into()?; - storage_writer = Some(FileWriter::try_new( + storage_writer = Some(file_versions::create_writer( + self.format_version, self.store.create(&storage_path).await?, storage_schema, writer_options.clone(), @@ -1292,7 +1399,8 @@ impl IvfIndexBuilder ), ]); let storage_schema: Schema = (&flat_schema).try_into()?; - storage_writer = Some(FileWriter::try_new( + storage_writer = Some(file_versions::create_writer( + self.format_version, self.store.create(&storage_path).await?, storage_schema, writer_options.clone(), @@ -1453,7 +1561,7 @@ impl IvfIndexBuilder fn check_partition_adjustment( ivf: &IvfModel, reader: &dyn ShuffleReader, - existing_indices: &[Arc], + existing_indices: &[ExistingIndex], ) -> Result<(Vec, Option)> { let index_type = IndexType::try_from( index_type_string(S::name().try_into()?, Q::quantization_type()).as_str(), @@ -1464,8 +1572,8 @@ impl IvfIndexBuilder let mut min_partition_size = usize::MAX; for partition in 0..ivf.num_partitions() { let mut num_rows = reader.partition_size(partition)?; - for index in existing_indices.iter() { - num_rows += index.partition_size(partition); + for source in existing_indices.iter() { + num_rows += source.index.partition_size(partition); } if num_rows > MAX_PARTITION_SIZE_FACTOR * index_type.target_partition_size() { split_partitions.push(partition); @@ -2032,7 +2140,8 @@ impl IvfIndexBuilder async fn partition_row_ids(&self, part_idx: usize) -> Result> { // existing part: read from the existing indices let mut row_ids = Vec::new(); - for index in self.existing_indices.iter() { + for source in self.existing_indices.iter() { + let index = &source.index; if part_idx >= index.ivf_model().num_partitions() { // there was a bug that may cause delta indices have different number of partitions, // it's safe to skip loading the extra partition, and split/join the existing partitions, @@ -2048,8 +2157,21 @@ impl IvfIndexBuilder let mut reader = index .partition_reader(part_idx, false, &NoOpMetricsCollector) .await?; + let old_data_filter = source.old_data_filter().await?; while let Some(batch) = reader.try_next().await? { - row_ids.extend(batch[ROW_ID].as_primitive::().values()); + let batch_row_ids = batch[ROW_ID].as_primitive::(); + match old_data_filter { + // Rows the segment may no longer contribute must not be reassigned + // to a partition; their live copy comes from another source. + Some(filter) => row_ids.extend( + batch_row_ids + .values() + .iter() + .zip(filter.filter_row_ids(batch_row_ids).values().iter()) + .filter_map(|(row_id, keep)| keep.then_some(row_id)), + ), + None => row_ids.extend(batch_row_ids.values()), + } } } diff --git a/rust/lance/src/index/vector/details.rs b/rust/lance/src/index/vector/details.rs index 63f9375792e..3ccfd6e0e1e 100644 --- a/rust/lance/src/index/vector/details.rs +++ b/rust/lance/src/index/vector/details.rs @@ -982,6 +982,88 @@ mod tests { assert_eq!(metric, None); } + // Schema with a vector column ("vec", FixedSizeList) and a scalar column + // ("tag", Utf8). Field ids are assigned by `set_field_id` during conversion, + // so look them up by name rather than hardcoding. + fn schema_with_vector_and_scalar() -> lance_core::datatypes::Schema { + use arrow_schema::{DataType, Field as ArrowField, Schema as ArrowSchema}; + let arrow = ArrowSchema::new(vec![ + ArrowField::new( + "vec", + DataType::FixedSizeList( + Arc::new(ArrowField::new("item", DataType::Float32, true)), + 8, + ), + true, + ), + ArrowField::new("tag", DataType::Utf8, true), + ]); + lance_core::datatypes::Schema::try_from(&arrow).unwrap() + } + + fn index_over_field(field_id: i32, index_details: Option) -> IndexMetadata { + IndexMetadata { + uuid: uuid::Uuid::new_v4(), + fields: vec![field_id], + name: "idx".to_string(), + dataset_version: 1, + fragment_bitmap: None, + index_details: index_details.map(Arc::new), + index_version: 1, + created_at: None, + base_id: None, + files: None, + } + } + + #[test] + fn test_needs_inference_missing_details_on_vector_field() { + // Oldest legacy case (<=0.19.2): no details, but the indexed field is a + // vector type => must infer. + let schema = schema_with_vector_and_scalar(); + let vec_id = schema.field("vec").unwrap().id; + let index = index_over_field(vec_id, None); + assert!(needs_vector_details_inference(&index, &schema)); + } + + #[test] + fn test_needs_inference_missing_details_on_scalar_field() { + // No details and the indexed field is not a vector type (e.g. an FTS or + // scalar index) => nothing to infer, so the load_indices fast path may + // skip the clone/infer/compare. + let schema = schema_with_vector_and_scalar(); + let tag_id = schema.field("tag").unwrap().id; + let index = index_over_field(tag_id, None); + assert!(!needs_vector_details_inference(&index, &schema)); + } + + #[test] + fn test_needs_inference_empty_vector_details() { + // Newer pre-details case: a VectorIndexDetails type_url with empty value + // bytes => must infer. + let schema = schema_with_vector_and_scalar(); + let vec_id = schema.field("vec").unwrap().id; + let index = index_over_field(vec_id, Some(vector_index_details_default())); + assert!(needs_vector_details_inference(&index, &schema)); + } + + #[test] + fn test_needs_inference_populated_vector_details() { + // Modern vector index with populated details => no inference needed. + let schema = schema_with_vector_and_scalar(); + let vec_id = schema.field("vec").unwrap().id; + let details = make_details( + VectorMetricType::L2, + None, + Some(Compression::Pq(ProductQuantization { + num_bits: 8, + num_sub_vectors: 16, + })), + ); + let index = index_over_field(vec_id, Some(details)); + assert!(!needs_vector_details_inference(&index, &schema)); + } + #[test] fn test_metric_type_from_index_metadata_all_metrics() { // Test all supported metric types. diff --git a/rust/lance/src/index/vector/fixture_test.rs b/rust/lance/src/index/vector/fixture_test.rs index a03cc313466..2af020e1df2 100644 --- a/rust/lance/src/index/vector/fixture_test.rs +++ b/rust/lance/src/index/vector/fixture_test.rs @@ -278,6 +278,7 @@ mod test { deleted_ids: None, filtered_ids: None, deleted_fragments: None, + overlay_block: None, final_mask: Mutex::new(OnceCell::new()), }), &NoOpMetricsCollector, diff --git a/rust/lance/src/index/vector/hamming.rs b/rust/lance/src/index/vector/hamming.rs index ba6ea98c42d..3240ab3021b 100644 --- a/rust/lance/src/index/vector/hamming.rs +++ b/rust/lance/src/index/vector/hamming.rs @@ -5,123 +5,229 @@ //! //! This module provides functionality to perform pairwise hamming distance //! computation and clustering on specific partitions of IVF_FLAT indices. - +//! +//! A logical IVF_FLAT index may consist of multiple physical segments (e.g. +//! delta segments created by `optimize_indices` in append mode, or segments +//! committed by distributed index builds). All segments of one logical index +//! are assumed to share the same global IVF centroids, so one partition id +//! refers to the same centroid region in every segment; this is validated and +//! an error is returned if the centroids differ. + +use std::sync::Arc; use std::time::Instant; -use arrow_array::RecordBatchReader; use arrow_array::cast::AsArray; use arrow_array::types::UInt64Type; +use arrow_array::{Array, FixedSizeListArray, RecordBatchReader}; use arrow_schema::DataType; use lance_core::{Error, Result}; use lance_index::metrics::NoOpMetricsCollector; use lance_index::vector::VectorIndex; use lance_index::vector::flat::index::{FlatBinQuantizer, FlatIndex}; use lance_index::vector::flat::storage::FLAT_COLUMN; +use lance_index::vector::ivf::storage::IvfModel; use lance_index::vector::storage::VectorStore; use lance_linalg::distance::{ - ClusteringResult, cluster_pairwise_result, extract_hashes_from_fixed_list, + BinaryHashValues, ClusteringResult, cluster_pairwise_result, + extract_binary_hashes_from_fixed_list, pairwise_hamming_distance_binary_parallel, pairwise_hamming_distance_parallel, }; +use lance_table::format::IndexMetadata; use rand::rng; use rand::seq::index::sample; +use uuid::Uuid; use crate::dataset::Dataset; -use crate::index::{DatasetIndexExt, DatasetIndexInternalExt}; +use crate::index::{DatasetIndexExt, DatasetIndexInternalExt, filter_index_segments_by_ids}; use super::ivf::v2::IVFIndex; -/// Perform pairwise hamming distance clustering on a partition of an IVF_FLAT index. -/// -/// This function loads a specific partition from an IVF_FLAT index on a hash column, -/// computes pairwise hamming distances between all hashes in the partition, -/// filters by threshold, and clusters the results using union-find. -/// -/// # Arguments -/// -/// * `dataset` - The Lance dataset -/// * `index_name` - Name of the IVF_FLAT index on the hash column -/// * `partition_id` - The partition ID within the IVF_FLAT index -/// * `hamming_threshold` - Maximum hamming distance to consider as similar -/// -/// # Returns -/// -/// A `RecordBatchReader` yielding batches with columns: -/// - `representative`: UInt64 - The representative row ID for each cluster -/// - `duplicates`: `List` - List of duplicate row IDs in each cluster -/// -/// # Errors +/// One opened physical segment of a logical IVF_FLAT binary index. +struct HashIndexSegment { + metadata: IndexMetadata, + index: Arc, +} + +impl HashIndexSegment { + fn ivf_flat_bin(&self) -> &IVFIndex { + self.index + .as_any() + .downcast_ref::>() + .expect("segment type validated in open_hash_index_segments") + } +} + +/// Validate that a column stores fixed-width binary hashes as +/// `FixedSizeList`, where `N` is a positive multiple of 8 bytes. +fn validate_hash_column(column: &str, data_type: &DataType) -> Result { + match data_type { + DataType::FixedSizeList(inner, size) if *inner.data_type() == DataType::UInt8 => { + if *size <= 0 || !(*size as usize).is_multiple_of(8) { + return Err(Error::invalid_input(format!( + "Column '{}' must be FixedSizeList where N is a positive \ + multiple of 8 bytes, got FixedSizeList", + column, size + ))); + } + Ok(*size as usize) + } + DataType::FixedSizeList(inner, size) => Err(Error::invalid_input(format!( + "Column '{}' must be FixedSizeList where N is a positive \ + multiple of 8 bytes, got FixedSizeList<{:?}, {}>", + column, + inner.data_type(), + size + ))), + _ => Err(Error::invalid_input(format!( + "Column '{}' must be FixedSizeList where N is a positive \ + multiple of 8 bytes, got {:?}", + column, data_type + ))), + } +} + +/// Validate that every segment of a logical IVF index shares the same global +/// centroids, so one partition id refers to the same centroid region in each +/// segment. Fails if any segment has no centroids or diverging centroids. +fn validate_shared_centroids<'a>( + index_name: &str, + models: impl IntoIterator, +) -> Result<()> { + struct Reference<'a> { + uuid: Uuid, + num_partitions: usize, + centroids: &'a FixedSizeListArray, + } + + let mut reference: Option> = None; + for (uuid, model) in models { + let centroids = model.centroids_array().ok_or_else(|| { + Error::invalid_input(format!( + "Index '{}' segment {} has no IVF centroids; hamming clustering requires \ + segments built from a shared global IVF model", + index_name, uuid + )) + })?; + match &reference { + None => { + reference = Some(Reference { + uuid, + num_partitions: model.num_partitions(), + centroids, + }); + } + Some(reference) => { + if centroids.to_data() != reference.centroids.to_data() { + return Err(Error::invalid_input(format!( + "Index '{}' segments do not share the same global IVF centroids: \ + segment {} ({} partitions) differs from segment {} ({} partitions); \ + retrain the index to merge segments before hamming clustering", + index_name, + uuid, + model.num_partitions(), + reference.uuid, + reference.num_partitions + ))); + } + } + } + } + Ok(()) +} + +/// Open the physical segments of a logical IVF_FLAT binary index. /// -/// Returns an error if: -/// - The index doesn't exist or is not an IVF_FLAT index -/// - The indexed column has wrong type (must be `FixedSizeList`) -/// - The partition ID is out of range -pub async fn hamming_clustering_for_ivf_partition( +/// When `segment_ids` is `None` all segments are opened; otherwise only the +/// requested segments are opened and every requested id must exist. Validates +/// that all selected segments index the same fixed-width binary hash column, +/// are IVF_FLAT indices for binary data, and share the same global centroids. +async fn open_hash_index_segments( dataset: &Dataset, index_name: &str, - partition_id: usize, - hamming_threshold: u32, -) -> Result> { - // Load indices and find the IVF_FLAT index - let indices = dataset.load_indices().await?; - let index_meta = indices - .iter() - .find(|idx| idx.name == index_name) - .ok_or_else(|| { - Error::invalid_input(format!("Index '{}' not found on dataset", index_name)) - })?; + segment_ids: Option<&[Uuid]>, +) -> Result> { + let metadatas = dataset.load_indices_by_name(index_name).await?; + if metadatas.is_empty() { + return Err(Error::invalid_input(format!( + "Index '{}' not found on dataset", + index_name + ))); + } - // Get the column name from the index metadata - let schema = dataset.schema(); - let field_id = index_meta - .fields + let metadatas = match segment_ids { + None => metadatas, + Some(segment_ids) => { + if segment_ids.is_empty() { + return Err(Error::invalid_input(format!( + "Segment selection for index '{}' must not be empty; \ + omit index_segments to use all segments", + index_name + ))); + } + filter_index_segments_by_ids(index_name, metadatas, segment_ids)? + } + }; + + let fields = metadatas[0].fields.clone(); + if let Some(mismatched) = metadatas.iter().find(|meta| meta.fields != fields) { + return Err(Error::invalid_input(format!( + "Index '{}' segments cover different fields: segment {} covers {:?} \ + while segment {} covers {:?}", + index_name, metadatas[0].uuid, fields, mismatched.uuid, mismatched.fields + ))); + } + + let field_id = fields .first() .ok_or_else(|| Error::invalid_input(format!("Index '{}' has no fields", index_name)))?; + let schema = dataset.schema(); let field = schema.field_by_id(*field_id).ok_or_else(|| { Error::invalid_input(format!( "Field with id {} not found in schema for index '{}'", field_id, index_name )) })?; - let column = &field.name; + validate_hash_column(&field.name, &field.data_type())?; - // Check column is FixedSizeList - let data_type = field.data_type(); - match data_type { - DataType::FixedSizeList(inner, 8) => { - if *inner.data_type() != DataType::UInt8 { - return Err(Error::invalid_input(format!( - "Column '{}' must be FixedSizeList, got FixedSizeList<{:?}, 8>", - column, - inner.data_type() - ))); - } - } - _ => { + let mut segments = Vec::with_capacity(metadatas.len()); + for metadata in metadatas { + let index = dataset + .open_vector_index(&field.name, &metadata.uuid, &NoOpMetricsCollector) + .await?; + if index + .as_any() + .downcast_ref::>() + .is_none() + { return Err(Error::invalid_input(format!( - "Column '{}' must be FixedSizeList, got {:?}", - column, data_type + "Index '{}' segment {} is not an IVF_FLAT index for binary data", + index_name, metadata.uuid ))); } + segments.push(HashIndexSegment { metadata, index }); } - // Open the vector index - let index = dataset - .open_vector_index(column, &index_meta.uuid, &NoOpMetricsCollector) - .await?; + validate_shared_centroids( + index_name, + segments + .iter() + .map(|segment| (segment.metadata.uuid, segment.ivf_flat_bin().ivf_model())), + )?; - // Try to downcast to IVFIndex (IVF_FLAT for binary data) - let ivf_index = index - .as_any() - .downcast_ref::>() - .ok_or_else(|| { - Error::invalid_input(format!( - "Index '{}' is not an IVF_FLAT index for binary data", - index_name - )) - })?; + Ok(segments) +} - // Check partition ID is valid - let num_partitions = ivf_index.ivf_model().num_partitions(); +async fn hamming_clustering_for_ivf_partition_impl( + dataset: &Dataset, + index_name: &str, + segment_ids: Option<&[Uuid]>, + partition_id: usize, + hamming_threshold: u32, +) -> Result> { + let segments = open_hash_index_segments(dataset, index_name, segment_ids).await?; + + // All segments share centroids, so the partition count is uniform. + let num_partitions = segments[0].ivf_flat_bin().ivf_model().num_partitions(); if partition_id >= num_partitions { return Err(Error::invalid_input(format!( "Partition ID {} is out of range (0..{})", @@ -129,45 +235,53 @@ pub async fn hamming_clustering_for_ivf_partition( ))); } - // Load the partition storage - let storage = ivf_index.load_partition_storage(partition_id, None).await?; - - // Get row IDs - let row_id_slice: Vec = storage.row_ids().copied().collect(); - - if row_id_slice.is_empty() { - let empty = ClusteringResult { - clusters: Vec::new(), - }; - return Ok(empty.into_reader(None)); + // Concatenate the partition's row ids and hashes across segments; identical + // hashes land in the same partition of every segment, so one pairwise pass + // over the union finds cross-segment duplicates. + let mut all_row_ids: Vec = Vec::new(); + let mut hash_chunks = Vec::new(); + let mut num_hashes = 0; + for segment in &segments { + let storage = segment + .ivf_flat_bin() + .load_partition_storage(partition_id, None) + .await?; + all_row_ids.extend(storage.row_ids().copied()); + for batch in storage.to_batches()? { + let vectors = batch + .column_by_name(FLAT_COLUMN) + .ok_or_else(|| { + Error::invalid_input(format!("Column '{}' not found in storage", FLAT_COLUMN)) + })? + .as_fixed_size_list(); + let hashes = extract_binary_hashes_from_fixed_list(vectors)?; + num_hashes += hashes.len(); + hash_chunks.push(hashes); + } + if all_row_ids.len() != num_hashes { + return Err(Error::internal(format!( + "Index '{}' segment {} partition {}: row id count {} does not match hash count {}", + index_name, + segment.metadata.uuid, + partition_id, + all_row_ids.len(), + num_hashes + ))); + } } - // Get vectors from the storage batches - let batches: Vec<_> = storage.to_batches()?.collect(); - if batches.is_empty() { + if all_row_ids.is_empty() { let empty = ClusteringResult { clusters: Vec::new(), }; return Ok(empty.into_reader(None)); } - - // Extract the hash vectors from the FLAT_COLUMN - let mut all_hashes = Vec::new(); - for batch in &batches { - let vectors = batch - .column_by_name(FLAT_COLUMN) - .ok_or_else(|| { - Error::invalid_input(format!("Column '{}' not found in storage", FLAT_COLUMN)) - })? - .as_fixed_size_list(); - let hashes = extract_hashes_from_fixed_list(vectors)?; - all_hashes.extend(hashes); - } + let all_hashes = BinaryHashValues::concat(&hash_chunks)?; // Compute pairwise hamming distances with threshold filtering - let pairwise_result = pairwise_hamming_distance_parallel( + let pairwise_result = pairwise_hamming_distance_binary_parallel( &all_hashes, - Some(&row_id_slice), + Some(&all_row_ids), Some(hamming_threshold), ); @@ -177,60 +291,125 @@ pub async fn hamming_clustering_for_ivf_partition( Ok(clustering.into_reader(None)) } -/// Get partition statistics for an IVF_FLAT index. -pub async fn get_ivf_partition_info( +/// Perform pairwise hamming distance clustering on a partition of an IVF_FLAT index. +/// +/// This function loads a specific partition from every segment of an IVF_FLAT +/// index on a hash column, computes pairwise hamming distances between all +/// hashes in the combined partition, filters by threshold, and clusters the +/// results using union-find. See [`hamming_clustering_for_ivf_partition_segments`] +/// to restrict the computation to selected segments. +/// +/// # Arguments +/// +/// * `dataset` - The Lance dataset +/// * `index_name` - Name of the IVF_FLAT index on the hash column +/// * `partition_id` - The partition ID within the IVF_FLAT index +/// * `hamming_threshold` - Maximum hamming distance to consider as similar +/// +/// # Returns +/// +/// A `RecordBatchReader` yielding batches with columns: +/// - `representative`: UInt64 - The representative row ID for each cluster +/// - `duplicates`: `List` - List of duplicate row IDs in each cluster +/// +/// # Errors +/// +/// Returns an error if: +/// - The index doesn't exist or is not an IVF_FLAT index +/// - The indexed column has wrong type (must be `FixedSizeList` where +/// `N` is a positive multiple of 8 bytes) +/// - The index segments do not share the same global IVF centroids +/// - The partition ID is out of range +pub async fn hamming_clustering_for_ivf_partition( dataset: &Dataset, index_name: &str, -) -> Result> { - let indices = dataset.load_indices().await?; - let index_meta = indices - .iter() - .find(|idx| idx.name == index_name) - .ok_or_else(|| { - Error::invalid_input(format!("Index '{}' not found on dataset", index_name)) - })?; - - // Get the column name from the index metadata - let schema = dataset.schema(); - let field_id = index_meta - .fields - .first() - .ok_or_else(|| Error::invalid_input(format!("Index '{}' has no fields", index_name)))?; - let field = schema.field_by_id(*field_id).ok_or_else(|| { - Error::invalid_input(format!( - "Field with id {} not found in schema for index '{}'", - field_id, index_name - )) - })?; - let column = &field.name; - - let index = dataset - .open_vector_index(column, &index_meta.uuid, &NoOpMetricsCollector) - .await?; - - let ivf_index = index - .as_any() - .downcast_ref::>() - .ok_or_else(|| { - Error::invalid_input(format!( - "Index '{}' is not an IVF_FLAT index for binary data", - index_name - )) - })?; + partition_id: usize, + hamming_threshold: u32, +) -> Result> { + hamming_clustering_for_ivf_partition_impl( + dataset, + index_name, + None, + partition_id, + hamming_threshold, + ) + .await +} - let num_partitions = ivf_index.ivf_model().num_partitions(); - let mut partition_infos = Vec::with_capacity(num_partitions); +/// Perform pairwise hamming distance clustering on a partition of selected +/// segments of an IVF_FLAT index. +/// +/// Same as [`hamming_clustering_for_ivf_partition`] but only the requested +/// physical segments contribute rows. Segment ids are the index UUIDs reported +/// by index descriptions; every requested id must belong to the named index and +/// the selection must not be empty. +pub async fn hamming_clustering_for_ivf_partition_segments( + dataset: &Dataset, + index_name: &str, + segment_ids: &[Uuid], + partition_id: usize, + hamming_threshold: u32, +) -> Result> { + hamming_clustering_for_ivf_partition_impl( + dataset, + index_name, + Some(segment_ids), + partition_id, + hamming_threshold, + ) + .await +} - for i in 0..num_partitions { - partition_infos.push(PartitionInfo { - partition_id: i, - size: ivf_index.ivf_model().partition_size(i), - }); +async fn get_ivf_partition_info_impl( + dataset: &Dataset, + index_name: &str, + segment_ids: Option<&[Uuid]>, +) -> Result> { + let segments = open_hash_index_segments(dataset, index_name, segment_ids).await?; + + let num_partitions = segments[0].ivf_flat_bin().ivf_model().num_partitions(); + let mut partition_infos: Vec = (0..num_partitions) + .map(|partition_id| PartitionInfo { + partition_id, + size: 0, + }) + .collect(); + for segment in &segments { + // Sizes come from the partition storage; the IVF model of a v3 index + // file does not carry partition lengths. + let index = segment.ivf_flat_bin(); + for info in partition_infos.iter_mut() { + info.size += index.partition_size(info.partition_id); + } } Ok(partition_infos) } +/// Get partition statistics for an IVF_FLAT index. +/// +/// Partition sizes are aggregated across all segments of the logical index. +/// See [`get_ivf_partition_info_segments`] to restrict the statistics to +/// selected segments. +pub async fn get_ivf_partition_info( + dataset: &Dataset, + index_name: &str, +) -> Result> { + get_ivf_partition_info_impl(dataset, index_name, None).await +} + +/// Get partition statistics for selected segments of an IVF_FLAT index. +/// +/// Same as [`get_ivf_partition_info`] but only the requested physical segments +/// contribute to the partition sizes. +pub async fn get_ivf_partition_info_segments( + dataset: &Dataset, + index_name: &str, + segment_ids: &[Uuid], +) -> Result> { + get_ivf_partition_info_impl(dataset, index_name, Some(segment_ids)).await +} + /// Information about an IVF partition. #[derive(Debug, Clone)] pub struct PartitionInfo { @@ -247,7 +426,8 @@ pub struct PartitionInfo { /// # Arguments /// /// * `dataset` - The Lance dataset -/// * `column` - Name of the hash column (must be `FixedSizeList`) +/// * `column` - Name of the hash column (must be `FixedSizeList` +/// where `N` is a positive multiple of 8 bytes) /// * `sample_size` - Number of rows to sample (if None or >= total rows, uses all rows) /// * `hamming_threshold` - Maximum hamming distance to consider as similar /// @@ -267,26 +447,7 @@ pub async fn hamming_clustering_for_sample( let field = schema.field(column).ok_or_else(|| { Error::invalid_input(format!("Column '{}' not found in dataset schema", column)) })?; - - // Check column is FixedSizeList - let data_type = field.data_type(); - match data_type { - DataType::FixedSizeList(inner, 8) => { - if *inner.data_type() != DataType::UInt8 { - return Err(Error::invalid_input(format!( - "Column '{}' must be FixedSizeList, got FixedSizeList<{:?}, 8>", - column, - inner.data_type() - ))); - } - } - _ => { - return Err(Error::invalid_input(format!( - "Column '{}' must be FixedSizeList, got {:?}", - column, data_type - ))); - } - } + validate_hash_column(column, &field.data_type())?; // Get total row count let total_rows: usize = dataset @@ -326,7 +487,7 @@ pub async fn hamming_clustering_for_sample( Error::invalid_input(format!("Column '{}' not found in result", column)) })?; let hashes_arr = hash_col.as_fixed_size_list(); - let hashes = extract_hashes_from_fixed_list(hashes_arr)?; + let hashes = extract_binary_hashes_from_fixed_list(hashes_arr)?; (hashes, row_id_vec) } else { @@ -348,7 +509,7 @@ pub async fn hamming_clustering_for_sample( Error::invalid_input(format!("Column '{}' not found in result", column)) })?; let hashes_arr = hash_col.as_fixed_size_list(); - let hashes = extract_hashes_from_fixed_list(hashes_arr)?; + let hashes = extract_binary_hashes_from_fixed_list(hashes_arr)?; (hashes, row_id_vec) }; @@ -362,7 +523,7 @@ pub async fn hamming_clustering_for_sample( // Compute pairwise hamming distances let pairwise = - pairwise_hamming_distance_parallel(&hashes, Some(&row_ids), Some(hamming_threshold)); + pairwise_hamming_distance_binary_parallel(&hashes, Some(&row_ids), Some(hamming_threshold)); // Cluster edges let clustering = cluster_pairwise_result(&pairwise); @@ -380,7 +541,8 @@ pub async fn hamming_clustering_for_sample( /// # Arguments /// /// * `dataset` - The Lance dataset -/// * `column` - Name of the hash column (must be `FixedSizeList`) +/// * `column` - Name of the hash column (must be `FixedSizeList` +/// where `N` is a positive multiple of 8 bytes) /// * `fragment_id` - The fragment ID to read from /// * `start_row` - The starting row offset within the fragment /// * `num_rows` - Number of rows to read from the start position @@ -396,7 +558,8 @@ pub async fn hamming_clustering_for_sample( /// /// Returns an error if: /// - The fragment doesn't exist -/// - The column has wrong type (must be `FixedSizeList`) +/// - The column has wrong type (must be `FixedSizeList` where `N` +/// is a positive multiple of 8 bytes) /// - The row range is out of bounds pub async fn hamming_clustering_for_range( dataset: &Dataset, @@ -411,26 +574,7 @@ pub async fn hamming_clustering_for_range( let field = schema.field(column).ok_or_else(|| { Error::invalid_input(format!("Column '{}' not found in dataset schema", column)) })?; - - // Check column is FixedSizeList - let data_type = field.data_type(); - match data_type { - DataType::FixedSizeList(inner, 8) => { - if *inner.data_type() != DataType::UInt8 { - return Err(Error::invalid_input(format!( - "Column '{}' must be FixedSizeList, got FixedSizeList<{:?}, 8>", - column, - inner.data_type() - ))); - } - } - _ => { - return Err(Error::invalid_input(format!( - "Column '{}' must be FixedSizeList, got {:?}", - column, data_type - ))); - } - } + validate_hash_column(column, &field.data_type())?; // Get the fragment let fragment = dataset.get_fragment(fragment_id).ok_or_else(|| { @@ -483,7 +627,7 @@ pub async fn hamming_clustering_for_range( .column_by_name(column) .ok_or_else(|| Error::invalid_input(format!("Column '{}' not found in result", column)))?; let hashes_arr = hash_col.as_fixed_size_list(); - let hashes = extract_hashes_from_fixed_list(hashes_arr)?; + let hashes = extract_binary_hashes_from_fixed_list(hashes_arr)?; if hashes.len() < 2 { let empty = ClusteringResult { @@ -493,8 +637,11 @@ pub async fn hamming_clustering_for_range( } // Compute pairwise hamming distances - let pairwise = - pairwise_hamming_distance_parallel(&hashes, Some(&row_id_vec), Some(hamming_threshold)); + let pairwise = pairwise_hamming_distance_binary_parallel( + &hashes, + Some(&row_id_vec), + Some(hamming_threshold), + ); // Cluster edges let clustering = cluster_pairwise_result(&pairwise); @@ -599,6 +746,31 @@ mod tests { clusters } + #[test] + fn test_validate_hash_column_generic_widths() { + use arrow_schema::{DataType, Field}; + use std::sync::Arc; + + fn hash_type(size: i32) -> DataType { + DataType::FixedSizeList(Arc::new(Field::new("item", DataType::UInt8, true)), size) + } + + assert_eq!(validate_hash_column("hash", &hash_type(8)).unwrap(), 8); + assert_eq!(validate_hash_column("hash", &hash_type(16)).unwrap(), 16); + assert_eq!(validate_hash_column("hash", &hash_type(32)).unwrap(), 32); + + let err = validate_hash_column("hash", &hash_type(12)).unwrap_err(); + assert!(err.to_string().contains("12"), "{}", err); + assert!(err.to_string().contains("multiple of 8 bytes"), "{}", err); + + let err = validate_hash_column("hash", &hash_type(4)).unwrap_err(); + assert!(err.to_string().contains("4"), "{}", err); + assert!(err.to_string().contains("multiple of 8 bytes"), "{}", err); + + let err = validate_hash_column("hash", &hash_type(-8)).unwrap_err(); + assert!(err.to_string().contains("-8"), "{}", err); + } + #[test] fn test_hamming_clustering_from_hashes_basic() { // Create some test hashes with known distances @@ -764,6 +936,200 @@ mod tests { assert!(err.to_string().contains("not found"), "Error: {}", err); } + #[test] + fn test_validate_shared_centroids() { + use arrow_array::UInt8Array; + use lance_arrow::FixedSizeListArrayExt; + + fn model_from_bytes(bytes: Vec) -> IvfModel { + let centroids = + FixedSizeListArray::try_new_from_values(UInt8Array::from(bytes), 8).unwrap(); + IvfModel::new(centroids, None) + } + + let uuid_a = Uuid::new_v4(); + let uuid_b = Uuid::new_v4(); + + let model_a = model_from_bytes(vec![0u8; 16]); + let model_b = model_from_bytes(vec![0u8; 16]); + validate_shared_centroids("idx", [(uuid_a, &model_a), (uuid_b, &model_b)]).unwrap(); + + let mut diverged = vec![0u8; 16]; + diverged[0] = 1; + let model_c = model_from_bytes(diverged); + let err = + validate_shared_centroids("idx", [(uuid_a, &model_a), (uuid_b, &model_c)]).unwrap_err(); + assert!( + err.to_string() + .contains("do not share the same global IVF centroids"), + "{}", + err + ); + assert!(err.to_string().contains(&uuid_b.to_string()), "{}", err); + + let model_d = model_from_bytes(vec![0u8; 24]); + let err = + validate_shared_centroids("idx", [(uuid_a, &model_a), (uuid_b, &model_d)]).unwrap_err(); + assert!(err.to_string().contains("2 partitions"), "{}", err); + assert!(err.to_string().contains("3 partitions"), "{}", err); + + let err = validate_shared_centroids("idx", [(uuid_a, &IvfModel::empty())]).unwrap_err(); + assert!(err.to_string().contains("has no IVF centroids"), "{}", err); + } + + #[tokio::test] + async fn test_hamming_clustering_for_ivf_partition_multi_segment_128_bit() { + use arrow_array::{FixedSizeListArray, RecordBatchIterator, UInt8Array}; + use arrow_schema::{Field, Schema}; + use lance_arrow::FixedSizeListArrayExt; + use lance_index::optimize::OptimizeOptions; + use lance_index::vector::ivf::IvfBuildParams; + use std::sync::Arc; + use tempfile::tempdir; + + const HASH_BYTES: i32 = 16; + + fn hash_batch(schema: Arc, values: &[u64]) -> arrow_array::RecordBatch { + let mut bytes = Vec::with_capacity(values.len() * HASH_BYTES as usize); + for value in values { + bytes.extend_from_slice(&value.to_le_bytes()); + let high = value.rotate_left(17) ^ 0xA5A5_A5A5_A5A5_A5A5; + bytes.extend_from_slice(&high.to_le_bytes()); + } + let array = + FixedSizeListArray::try_new_from_values(UInt8Array::from(bytes), HASH_BYTES) + .unwrap(); + arrow_array::RecordBatch::try_new(schema, vec![Arc::new(array)]).unwrap() + } + + let schema = Arc::new(Schema::new(vec![Field::new( + "hash", + arrow_schema::DataType::FixedSizeList( + Arc::new(Field::new("item", arrow_schema::DataType::UInt8, true)), + HASH_BYTES, + ), + false, + )])); + + // 25 distinct hash values, two copies each; the same batch is written to + // fragment 0 and appended as fragment 1, so every value has duplicates + // in both fragments. + let values: Vec = (0..50) + .map(|i| ((i / 2) as u64).wrapping_mul(0x9E3779B97F4A7C15)) + .collect(); + let num_values = 25; + + let temp_dir = tempdir().unwrap(); + let uri = temp_dir.path().to_str().unwrap(); + let reader = RecordBatchIterator::new( + vec![Ok(hash_batch(schema.clone(), &values))], + schema.clone(), + ); + let mut dataset = crate::Dataset::write(reader, uri, None).await.unwrap(); + + let params = crate::index::vector::VectorIndexParams::with_ivf_flat_params( + lance_linalg::distance::MetricType::Hamming, + IvfBuildParams::new(4), + ); + dataset + .create_index( + &["hash"], + crate::index::IndexType::Vector, + Some("hash_idx".into()), + ¶ms, + false, + ) + .await + .unwrap(); + + let reader = RecordBatchIterator::new( + vec![Ok(hash_batch(schema.clone(), &values))], + schema.clone(), + ); + dataset.append(reader, None).await.unwrap(); + dataset + .optimize_indices(&OptimizeOptions::append()) + .await + .unwrap(); + + let segments = dataset.load_indices_by_name("hash_idx").await.unwrap(); + assert_eq!( + segments.len(), + 2, + "expected a delta segment after optimize append" + ); + + // Partition sizes aggregate across both segments. + let infos = get_ivf_partition_info(&dataset, "hash_idx").await.unwrap(); + assert_eq!(infos.len(), 4); + assert_eq!(infos.iter().map(|info| info.size).sum::(), 100); + + // Clustering each partition with threshold 0 must group all four copies + // of every value, including the copies in the appended fragment. + const FRAG1_START: u64 = 1 << 32; + let mut clusters = Vec::new(); + for partition_id in 0..4 { + let reader = + hamming_clustering_for_ivf_partition(&dataset, "hash_idx", partition_id, 0) + .await + .unwrap(); + clusters.extend(collect_clusters(reader)); + } + assert_eq!(clusters.len(), num_values); + for (representative, duplicates) in &clusters { + assert_eq!(duplicates.len(), 3); + assert!(*representative < FRAG1_START); + assert!( + duplicates.iter().any(|row_id| *row_id >= FRAG1_START), + "cluster {} should contain rows from the appended fragment", + representative + ); + } + + // Selecting only the original segment reproduces the single-segment scope. + let first_segment = segments + .iter() + .find(|meta| meta.fragment_bitmap.as_ref().unwrap().contains(0)) + .unwrap(); + let mut old_clusters = Vec::new(); + for partition_id in 0..4 { + let reader = hamming_clustering_for_ivf_partition_segments( + &dataset, + "hash_idx", + &[first_segment.uuid], + partition_id, + 0, + ) + .await + .unwrap(); + old_clusters.extend(collect_clusters(reader)); + } + assert_eq!(old_clusters.len(), num_values); + for (representative, duplicates) in &old_clusters { + assert_eq!(duplicates.len(), 1); + assert!(*representative < FRAG1_START); + assert!(duplicates.iter().all(|row_id| *row_id < FRAG1_START)); + } + let infos = get_ivf_partition_info_segments(&dataset, "hash_idx", &[first_segment.uuid]) + .await + .unwrap(); + assert_eq!(infos.iter().map(|info| info.size).sum::(), 50); + + // Invalid segment selections are rejected. + let err = hamming_clustering_for_ivf_partition_segments(&dataset, "hash_idx", &[], 0, 0) + .await + .err() + .unwrap(); + assert!(err.to_string().contains("must not be empty"), "{}", err); + let missing = Uuid::new_v4(); + let err = + hamming_clustering_for_ivf_partition_segments(&dataset, "hash_idx", &[missing], 0, 0) + .await + .err() + .unwrap(); + assert!(err.to_string().contains(&missing.to_string()), "{}", err); + } + #[tokio::test] async fn test_hamming_clustering_for_sample_integration() { use arrow_array::{FixedSizeListArray, RecordBatchIterator, UInt8Array}; diff --git a/rust/lance/src/index/vector/ivf.rs b/rust/lance/src/index/vector/ivf.rs index 9b8db75fde3..8f3a6b8453b 100644 --- a/rust/lance/src/index/vector/ivf.rs +++ b/rust/lance/src/index/vector/ivf.rs @@ -4,12 +4,12 @@ //! IVF - Inverted File index. use super::{ - LogicalIvfView, + LogicalIvfView, derive_hnsw_params, pq::{PQIndex, build_pq_model}, utils::{filter_finite_training_data, maybe_sample_training_data}, }; use super::{ - builder::{IvfIndexBuilder, index_type_string}, + builder::{ExistingIndex, IvfIndexBuilder, index_type_string}, utils::PartitionLoadLock, }; use crate::dataset::index::dataset_format_version; @@ -19,7 +19,11 @@ use crate::index::vector::open_index_file; use crate::index::vector::utils::{get_vector_dim, get_vector_type}; use crate::{ dataset::Dataset, - index::{INDEX_FILE_NAME, pb, prefilter::PreFilter, vector::ivf::io::write_pq_partitions}, + index::{ + INDEX_FILE_NAME, pb, + prefilter::PreFilter, + vector::ivf::io::{write_pq_partition_payload, write_pq_partitions}, + }, }; use crate::{dataset::builder::DatasetBuilder, index::vector::IndexFileVersion}; use arrow::array::ArrayData; @@ -47,7 +51,7 @@ use lance_core::deepsize::DeepSizeOf; use lance_core::utils::row_addr_remap::RowAddrRemap; use lance_core::{ Error, ROW_ID_FIELD, Result, - cache::{LanceCache, UnsizedCacheKey, WeakLanceCache}, + cache::{CacheKeySchema, KeyBuilder, LanceCache, UnsizedCacheKey, WeakLanceCache}, traits::DatasetTakeRows, utils::parse::parse_env_as_bool, utils::tracing::{IO_TYPE_LOAD_VECTOR_PART, TRACE_IO_EVENTS}, @@ -55,10 +59,9 @@ use lance_core::{ use lance_encoding::decoder::FilterExpression; use lance_file::{ format::MAGIC, - previous::writer::{ - FileWriter as PreviousFileWriter, FileWriterOptions as PreviousFileWriterOptions, - }, reader::{FileReader as V2Reader, FileReaderOptions as V2ReaderOptions}, + versions as file_versions, + versions::v1::writer::{FileWriter as V1FileWriter, FileWriterOptions as V1FileWriterOptions}, writer::{FileWriter as V2Writer, FileWriterOptions as V2WriterOptions}, }; use lance_index::metrics::MetricsCollector; @@ -102,7 +105,6 @@ use lance_io::scheduler::{ScanScheduler, SchedulerConfig}; use lance_io::utils::CachedFileSize; use lance_io::{ ReadBatchParams, - encodings::plain::PlainEncoder, local::to_local_path, object_store::ObjectStore, stream::RecordBatchStream, @@ -157,6 +159,14 @@ impl UnsizedCacheKey for LegacyIVFPartitionKey { fn type_name() -> &'static str { "LegacyIVFPartition" } + + fn schema() -> CacheKeySchema { + CacheKeySchema::new("lance.index.legacy-ivf-partition-key", 1) + } + + fn write_key(&self, builder: &mut KeyBuilder) { + builder.write_u64(self.partition_id as u64); + } } /// IVF Index. @@ -322,7 +332,7 @@ fn candidate_is_better( } } -fn index_type_for_segmented_optimize(index: &dyn VectorIndex) -> Result { +pub(crate) fn index_type_for_segmented_optimize(index: &dyn VectorIndex) -> Result { let (sub_index_type, quantization_type) = index.sub_index_type(); IndexType::try_from(index_type_string(sub_index_type, quantization_type).as_str()) } @@ -386,6 +396,203 @@ pub(crate) fn select_segment_for_single_rebalance( Ok(selected.map(|candidate| candidate.segment_id)) } +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum VectorSegmentCompatibility { + SharedModel, + QueryCompatibleModelsDiffer, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum VectorModelMismatch { + StorageFormat, + IvfCentroids, + QuantizerMetadata, +} + +fn vector_index_dimension(index: &dyn VectorIndex) -> usize { + let ivf_dimension = index.ivf_model().dimension(); + if ivf_dimension != 0 { + return ivf_dimension; + } + + match index.quantizer() { + Quantizer::Flat(quantizer) => quantizer.metadata(None).dim, + Quantizer::FlatBin(quantizer) => quantizer.metadata(None).dim, + Quantizer::Product(quantizer) => quantizer.dimension, + Quantizer::Scalar(quantizer) => quantizer.metadata(None).dim, + Quantizer::Rabit(quantizer) => quantizer.metadata(None).rotated_dim(), + } +} + +fn validate_vector_query_compatibility( + indices: &[Arc], + operation: &str, +) -> Result<()> { + let Some(first) = indices.first() else { + return Ok(()); + }; + + let first_metric = first.metric_type(); + let first_dimension = vector_index_dimension(first.as_ref()); + let first_index_type = first.sub_index_type(); + let first_quantizer = first.quantizer(); + let first_quantizer_type = first_quantizer.quantization_type(); + + for (idx, index) in indices.iter().enumerate().skip(1) { + if index.metric_type() != first_metric { + return Err(Error::index(format!( + "{operation}: vector index segment {idx} has metric {:?}, expected {:?}", + index.metric_type(), + first_metric + ))); + } + let dimension = vector_index_dimension(index.as_ref()); + if dimension != first_dimension { + return Err(Error::index(format!( + "{operation}: vector index segment {idx} has dimension {dimension}, expected {first_dimension}" + ))); + } + let index_type = index.sub_index_type(); + if std::mem::discriminant(&index_type.0) != std::mem::discriminant(&first_index_type.0) + || index_type.1 != first_index_type.1 + { + return Err(Error::index(format!( + "{operation}: vector index segment {idx} has type {:?}, expected {:?}", + index_type, first_index_type + ))); + } + + let quantizer = index.quantizer(); + if quantizer.quantization_type() != first_quantizer_type { + return Err(Error::index(format!( + "{operation}: vector index segment {idx} has quantizer {:?}, expected {:?}", + quantizer.quantization_type(), + first_quantizer_type + ))); + } + } + + Ok(()) +} + +fn vector_model_mismatch(indices: &[Arc]) -> Option { + let first = indices.first()?; + let first_centroids = first.ivf_model().centroids_array(); + let first_quantizer = first.quantizer(); + + for index in indices.iter().skip(1) { + if first.as_any().type_id() != index.as_any().type_id() { + return Some(VectorModelMismatch::StorageFormat); + } + match (first_centroids, index.ivf_model().centroids_array()) { + (Some(expected), Some(actual)) if expected.to_data() != actual.to_data() => { + return Some(VectorModelMismatch::IvfCentroids); + } + (Some(_), None) | (None, Some(_)) => { + return Some(VectorModelMismatch::IvfCentroids); + } + _ => {} + } + + if !shared_quantizer_model(&first_quantizer, &index.quantizer()) { + return Some(VectorModelMismatch::QuantizerMetadata); + } + } + + None +} + +pub(crate) fn vector_segment_compatibility( + logical_index: &LogicalIvfView<'_>, + operation: &str, +) -> Result { + let indices = logical_index.indices().cloned().collect::>(); + validate_vector_query_compatibility(&indices, operation)?; + Ok(if vector_model_mismatch(&indices).is_none() { + VectorSegmentCompatibility::SharedModel + } else { + VectorSegmentCompatibility::QueryCompatibleModelsDiffer + }) +} + +fn validate_shared_vector_model(indices: &[Arc], operation: &str) -> Result<()> { + validate_vector_query_compatibility(indices, operation)?; + match vector_model_mismatch(indices) { + Some(VectorModelMismatch::StorageFormat) => Err(Error::index(format!( + "{operation}: vector index segments do not share a storage format" + ))), + Some(VectorModelMismatch::IvfCentroids) => Err(Error::index(format!( + "{operation}: vector index segments do not share IVF centroids" + ))), + Some(VectorModelMismatch::QuantizerMetadata) => Err(Error::index(format!( + "{operation}: vector index segments do not share quantizer metadata" + ))), + None => Ok(()), + } +} + +fn shared_quantizer_model(left: &Quantizer, right: &Quantizer) -> bool { + match (left, right) { + (Quantizer::Flat(left), Quantizer::Flat(right)) => { + left.metadata(None).dim == right.metadata(None).dim + } + (Quantizer::FlatBin(left), Quantizer::FlatBin(right)) => { + left.metadata(None).dim == right.metadata(None).dim + } + (Quantizer::Product(left), Quantizer::Product(right)) => { + left.num_sub_vectors == right.num_sub_vectors + && left.num_bits == right.num_bits + && left.dimension == right.dimension + && left.distance_type == right.distance_type + && left.codebook.to_data() == right.codebook.to_data() + } + (Quantizer::Scalar(left), Quantizer::Scalar(right)) => { + left.metadata(None) == right.metadata(None) + } + (Quantizer::Rabit(left), Quantizer::Rabit(right)) => { + let left = left.metadata(None); + let right = right.metadata(None); + left.rotation_type == right.rotation_type + && left.code_dim == right.code_dim + && left.num_bits == right.num_bits + && left.packed == right.packed + && left.query_estimator == right.query_estimator + && left.fast_rotation_signs == right.fast_rotation_signs + && match (&left.rotate_mat, &right.rotate_mat) { + (Some(left), Some(right)) => left.to_data() == right.to_data(), + (None, None) => true, + _ => false, + } + } + _ => false, + } +} + +/// Pair every segment with the coverage that decides which of its rows this optimize +/// pass may still copy into the new index. +/// +/// A segment that predates fragment bitmaps has unknown coverage, so it keeps every +/// row it holds. Turning coverage into a filter is deferred to the first partition +/// that actually reads the segment, because under stable row ids it costs a row-id +/// sequence load per covered fragment and most passes only append a delta. +fn existing_index_sources( + dataset: &Dataset, + logical_index: &LogicalIvfView<'_>, +) -> Vec { + logical_index + .segments() + .map(|(metadata, index)| { + let (Some(effective), Some(deleted)) = ( + metadata.effective_fragment_bitmap(&dataset.fragment_bitmap), + metadata.deleted_fragment_bitmap(&dataset.fragment_bitmap), + ) else { + return ExistingIndex::unfiltered(index.clone()); + }; + ExistingIndex::with_coverage(index.clone(), dataset.clone(), effective, deleted) + }) + .collect() +} + // TODO: move to `lance-index` crate. /// /// Returns (new_uuid, num_indices_merged, files) @@ -403,18 +610,14 @@ pub(crate) async fn optimize_vector_indices( "optimizing vector index: no existing index found".to_string(), )); } + validate_shared_vector_model(&existing_indices, "optimizing vector index")?; // try cast to v1 IVFIndex, // fallback to v2 IVFIndex if it's not v1 IVFIndex if !existing_indices[0].as_any().is::() { - return optimize_vector_indices_v2( - &dataset, - unindexed, - vector_column, - &existing_indices, - options, - ) - .await; + let sources = existing_index_sources(&dataset, logical_index); + return optimize_vector_indices_v2(&dataset, unindexed, vector_column, &sources, options) + .await; } let new_uuid = Uuid::new_v4(); @@ -483,7 +686,7 @@ pub(crate) async fn optimize_vector_indices_v2( dataset: &Dataset, unindexed: Option, vector_column: &str, - existing_indices: &[Arc], + existing_indices: &[ExistingIndex], options: &OptimizeOptions, ) -> Result<(Uuid, usize, Vec)> { // Sanity check the indices @@ -496,11 +699,12 @@ pub(crate) async fn optimize_vector_indices_v2( let new_uuid = Uuid::new_v4(); let index_dir = dataset.indices_dir().join(new_uuid.to_string()); - let ivf_model = existing_indices[0].ivf_model(); - let quantizer = existing_indices[0].quantizer(); - let distance_type = existing_indices[0].metric_type(); + let reference_index = &existing_indices[0].index; + let ivf_model = reference_index.ivf_model(); + let quantizer = reference_index.quantizer(); + let distance_type = reference_index.metric_type(); let num_partitions = ivf_model.num_partitions(); - let index_type = existing_indices[0].sub_index_type(); + let index_type = reference_index.sub_index_type(); let frag_reuse_index = dataset.open_frag_reuse_index(&NoOpMetricsCollector).await?; let format_version = dataset_format_version(dataset); @@ -526,7 +730,7 @@ pub(crate) async fn optimize_vector_indices_v2( )? .with_ivf(ivf_model.clone()) .with_quantizer(quantizer.try_into()?) - .with_existing_indices(existing_indices.clone()) + .with_existing_index_sources(existing_indices.clone()) .with_progress(options.progress.clone()) .shuffle_data_input(unindexed) .build() @@ -544,7 +748,7 @@ pub(crate) async fn optimize_vector_indices_v2( )? .with_ivf(ivf_model.clone()) .with_quantizer(quantizer.try_into()?) - .with_existing_indices(existing_indices.clone()) + .with_existing_index_sources(existing_indices.clone()) .with_progress(options.progress.clone()) .shuffle_data_input(unindexed) .build() @@ -565,7 +769,7 @@ pub(crate) async fn optimize_vector_indices_v2( )? .with_ivf(ivf_model.clone()) .with_quantizer(quantizer.try_into()?) - .with_existing_indices(existing_indices.clone()) + .with_existing_index_sources(existing_indices.clone()) .with_progress(options.progress.clone()) .shuffle_data_input(unindexed) .build() @@ -585,7 +789,7 @@ pub(crate) async fn optimize_vector_indices_v2( )? .with_ivf(ivf_model.clone()) .with_quantizer(quantizer.try_into()?) - .with_existing_indices(existing_indices.clone()) + .with_existing_index_sources(existing_indices.clone()) .with_progress(options.progress.clone()) .shuffle_data_input(unindexed) .build() @@ -605,7 +809,7 @@ pub(crate) async fn optimize_vector_indices_v2( )? .with_ivf(ivf_model.clone()) .with_quantizer(quantizer.try_into()?) - .with_existing_indices(existing_indices.clone()) + .with_existing_index_sources(existing_indices.clone()) .with_progress(options.progress.clone()) .shuffle_data_input(unindexed) .build() @@ -624,7 +828,7 @@ pub(crate) async fn optimize_vector_indices_v2( )? .with_ivf(ivf_model.clone()) .with_quantizer(quantizer.try_into()?) - .with_existing_indices(existing_indices.clone()) + .with_existing_index_sources(existing_indices.clone()) .with_progress(options.progress.clone()) .shuffle_data_input(unindexed) .build() @@ -639,13 +843,13 @@ pub(crate) async fn optimize_vector_indices_v2( index_dir, distance_type, shuffler, - HnswBuildParams::default(), + derive_hnsw_params(reference_index.as_ref()), frag_reuse_index, options.clone(), )? .with_ivf(ivf_model.clone()) .with_quantizer(quantizer.try_into()?) - .with_existing_indices(existing_indices.clone()) + .with_existing_index_sources(existing_indices.clone()) .with_progress(options.progress.clone()) .shuffle_data_input(unindexed) .build() @@ -657,13 +861,13 @@ pub(crate) async fn optimize_vector_indices_v2( index_dir, distance_type, shuffler, - HnswBuildParams::default(), + derive_hnsw_params(reference_index.as_ref()), frag_reuse_index, options.clone(), )? .with_ivf(ivf_model.clone()) .with_quantizer(quantizer.try_into()?) - .with_existing_indices(existing_indices.clone()) + .with_existing_index_sources(existing_indices.clone()) .with_progress(options.progress.clone()) .shuffle_data_input(unindexed) .build() @@ -678,13 +882,13 @@ pub(crate) async fn optimize_vector_indices_v2( index_dir, distance_type, shuffler, - HnswBuildParams::default(), + derive_hnsw_params(reference_index.as_ref()), frag_reuse_index, options.clone(), )? .with_ivf(ivf_model.clone()) .with_quantizer(quantizer.try_into()?) - .with_existing_indices(existing_indices.clone()) + .with_existing_index_sources(existing_indices.clone()) .with_progress(options.progress.clone()) .shuffle_data_input(unindexed) .build() @@ -698,13 +902,13 @@ pub(crate) async fn optimize_vector_indices_v2( index_dir, distance_type, shuffler, - HnswBuildParams::default(), + derive_hnsw_params(reference_index.as_ref()), frag_reuse_index, options.clone(), )? .with_ivf(ivf_model.clone()) .with_quantizer(quantizer.try_into()?) - .with_existing_indices(existing_indices.clone()) + .with_existing_index_sources(existing_indices.clone()) .with_progress(options.progress.clone()) .shuffle_data_input(unindexed) .build() @@ -870,11 +1074,8 @@ async fn optimize_ivf_hnsw_indices( // Prepare the HNSW writer let schema = lance_core::datatypes::Schema::try_from(HNSW::schema().as_ref())?; - let mut writer = PreviousFileWriter::with_object_writer( - writer, - schema, - &PreviousFileWriterOptions::default(), - )?; + let mut writer = + V1FileWriter::with_object_writer(writer, schema, &V1FileWriterOptions::default())?; writer.add_metadata( INDEX_METADATA_SCHEMA_KEY, json!(IndexMetadata { @@ -898,11 +1099,8 @@ async fn optimize_ivf_hnsw_indices( ), ]); let schema = lance_core::datatypes::Schema::try_from(&schema)?; - let mut aux_writer = PreviousFileWriter::with_object_writer( - aux_writer, - schema, - &PreviousFileWriterOptions::default(), - )?; + let mut aux_writer = + V1FileWriter::with_object_writer(aux_writer, schema, &V1FileWriterOptions::default())?; aux_writer.add_metadata( INDEX_METADATA_SCHEMA_KEY, json!(IndexMetadata { @@ -1744,8 +1942,12 @@ impl RemapPageTask { page.pq.code_dim(), page.row_ids.as_ref().unwrap().len(), ); - PlainEncoder::write(writer, &[&original_pq]).await?; - PlainEncoder::write(writer, &[page.row_ids.as_ref().unwrap().as_ref()]).await?; + write_pq_partition_payload( + writer, + &[&original_pq], + &[page.row_ids.as_ref().unwrap().as_ref()], + ) + .await?; Ok(()) } } @@ -1892,7 +2094,7 @@ pub(crate) async fn remap_index_file( let tasks = generate_remap_tasks(&index.ivf.offsets, &index.ivf.lengths)?; - let mut task_stream = stream::iter(tasks.into_iter()) + let mut task_stream = stream::iter(tasks) .map(|task| task.load_and_remap(reader.clone(), index, mapping)) .buffered(object_store.io_parallelism()); @@ -2064,11 +2266,8 @@ async fn write_ivf_hnsw_file( let writer = object_store.create(&path).await?; let schema = lance_core::datatypes::Schema::try_from(HNSW::schema().as_ref())?; - let mut writer = PreviousFileWriter::with_object_writer( - writer, - schema, - &PreviousFileWriterOptions::default(), - )?; + let mut writer = + V1FileWriter::with_object_writer(writer, schema, &V1FileWriterOptions::default())?; writer.add_metadata( INDEX_METADATA_SCHEMA_KEY, json!(IndexMetadata { @@ -2096,11 +2295,8 @@ async fn write_ivf_hnsw_file( ), ]); let schema = lance_core::datatypes::Schema::try_from(&schema)?; - let mut aux_writer = PreviousFileWriter::with_object_writer( - aux_writer, - schema, - &PreviousFileWriterOptions::default(), - )?; + let mut aux_writer = + V1FileWriter::with_object_writer(aux_writer, schema, &V1FileWriterOptions::default())?; aux_writer.add_metadata( INDEX_METADATA_SCHEMA_KEY, json!(IndexMetadata { @@ -2441,13 +2637,11 @@ async fn write_root_vector_index_from_auxiliary( // Schema for HNSW sub-index: include neighbors/dist fields; empty batch is fine. let arrow_schema = HNSW::schema(); let schema = lance_core::datatypes::Schema::try_from(arrow_schema.as_ref())?; - let mut v2_writer = V2Writer::try_new( + let mut v2_writer = file_versions::create_writer( + format_version, obj_writer, schema, - V2WriterOptions { - format_version: Some(format_version), - ..Default::default() - }, + V2WriterOptions::default(), )?; // For HNSW variants, attach per-partition metadata list; for FLAT-based @@ -4267,19 +4461,26 @@ async fn train_streaming_coreset_ivf_model( let coreset_len = coreset.len(); let (coreset_data, coreset_weights, coreset_losses) = coreset.into_fsl_parts(dimension)?; - let weighted_hierarchical_params = WeightedHierarchicalKMeansParams { - dimension, - target_k: num_partitions, - metric_type: DistanceType::L2, - max_iters: params.max_iters, - on_progress: on_progress.clone(), + // Scope `weighted_hierarchical_params` so the `on_progress` clone it holds + // (which owns a clone of `progress_tx`) is dropped as soon as training + // returns. Otherwise it would outlive the `progress_worker.await` below, + // keeping a channel sender alive so `progress_rx.recv()` never returns + // `None` and the progress worker — and thus this function — hangs forever. + let mut centroids = { + let weighted_hierarchical_params = WeightedHierarchicalKMeansParams { + dimension, + target_k: num_partitions, + metric_type: DistanceType::L2, + max_iters: params.max_iters, + on_progress: on_progress.clone(), + }; + train_weighted_hierarchical_f32_kmeans( + &coreset_data, + &coreset_weights, + &coreset_losses, + &weighted_hierarchical_params, + )? }; - let mut centroids = train_weighted_hierarchical_f32_kmeans( - &coreset_data, - &coreset_weights, - &coreset_losses, - &weighted_hierarchical_params, - )?; let refine_iters = 3; if refine_iters > 0 { let refined = refine_weighted_f32_kmeans( @@ -4559,6 +4760,7 @@ mod tests { use lance_datagen::{ArrayGeneratorExt, BatchCount, Dimension, RowCount, array, gen_batch}; use lance_index::VECTOR_INDEX_VERSION; use lance_index::metrics::NoOpMetricsCollector; + use lance_index::scalar::OldIndexDataFilter; use lance_index::vector::sq::builder::SQBuildParams; use lance_linalg::distance::l2_distance_batch; use lance_testing::datagen::{ @@ -4577,6 +4779,152 @@ mod tests { const DIM: usize = 32; + /// Building a merge filter loads a row-id sequence per covered fragment under + /// stable row ids, and an optimize pass that only appends a delta reads no existing + /// row at all. Such a pass must therefore build no filter, and a pass that does + /// merge must build one and reuse it across partitions. + #[tokio::test] + async fn test_optimize_builds_merge_filters_only_when_merging() { + let test_dir = TempStrDir::default(); + let test_uri = test_dir.as_str(); + + let make_batch = || { + gen_batch() + .col( + "vector", + array::rand_vec::(Dimension::from(DIM as u32)), + ) + .into_batch_rows(RowCount::from(256)) + .unwrap() + }; + let batch = make_batch(); + let schema = batch.schema(); + let mut dataset = Dataset::write( + RecordBatchIterator::new(vec![Ok(batch)], schema.clone()), + test_uri, + Some(WriteParams { + enable_stable_row_ids: true, + ..Default::default() + }), + ) + .await + .unwrap(); + // A single partition keeps `check_partition_adjustment` from selecting a split + // or a join, which would legitimately merge every segment. + dataset + .create_index( + &["vector"], + IndexType::Vector, + None, + &VectorIndexParams::ivf_flat(1, MetricType::L2), + true, + ) + .await + .unwrap(); + dataset + .append( + RecordBatchIterator::new(vec![Ok(make_batch())], schema), + None, + ) + .await + .unwrap(); + + let logical_index = dataset + .open_logical_vector_index("vector", "vector_idx") + .await + .unwrap(); + let ivf_view = logical_index.as_ivf().unwrap(); + let new_data = |fragments| async { + let mut scanner = dataset.scan(); + scanner + .with_fragments(fragments) + .with_row_id() + .project(&["vector"]) + .unwrap(); + scanner.try_into_stream().await.unwrap() + }; + let unindexed = dataset.unindexed_fragments("vector_idx").await.unwrap(); + assert_eq!( + unindexed.len(), + 1, + "the appended fragment must be unindexed" + ); + + // `ExistingIndex` shares its coverage behind an `Arc`, so the sources handed to + // the builder report what the builder actually did with them. + let sources = existing_index_sources(&dataset, &ivf_view); + optimize_vector_indices_v2( + &dataset, + Some(new_data(unindexed.clone()).await), + "vector", + &sources, + &OptimizeOptions::new(), + ) + .await + .unwrap(); + assert!( + sources.iter().all(|source| !source.filter_is_built()), + "a delta append reads no existing row, so it must build no filter" + ); + + let sources = existing_index_sources(&dataset, &ivf_view); + optimize_vector_indices_v2( + &dataset, + Some(new_data(unindexed).await), + "vector", + &sources, + &OptimizeOptions::merge(1), + ) + .await + .unwrap(); + assert!( + sources.iter().all(|source| source.filter_is_built()), + "a merge reads existing rows, so it must build a filter per merged segment" + ); + assert!( + matches!( + sources[0].old_data_filter().await.unwrap(), + Some(OldIndexDataFilter::RowIds(_)) + ), + "a stable-row-id segment must filter on exact row-id membership" + ); + } + + #[test] + fn test_shared_quantizer_model_compares_skipped_payloads() { + let codebook = |offset| { + let values = Float32Array::from_iter_values((0..32).map(|v| v as f32 + offset)); + FixedSizeListArray::try_new_from_values(values, 2).unwrap() + }; + let pq1 = Quantizer::Product(ProductQuantizer::new( + 1, + 4, + 2, + codebook(0.0), + DistanceType::L2, + )); + let pq2 = Quantizer::Product(ProductQuantizer::new( + 1, + 4, + 2, + codebook(100.0), + DistanceType::L2, + )); + assert!(!shared_quantizer_model(&pq1, &pq2)); + + let rq1 = Quantizer::Rabit(RabitQuantizer::new_with_rotation::( + 1, + 8, + lance_index::vector::bq::RQRotationType::Matrix, + )); + let rq2 = Quantizer::Rabit(RabitQuantizer::new_with_rotation::( + 1, + 8, + lance_index::vector::bq::RQRotationType::Matrix, + )); + assert!(!shared_quantizer_model(&rq1, &rq2)); + } + async fn compute_test_ivf_loss(dataset: &Dataset, column: &str, ivf: &IvfModel) -> f64 { let centroids = ivf .centroids_array() @@ -5661,6 +6009,94 @@ mod tests { ); } + /// Regression test for a hang in the streaming *coreset* trainer + /// (`train_streaming_coreset_ivf_model`, taken when `num_partitions > 256`). + /// + /// That function spawns a `progress_worker` task that loops on + /// `progress_rx.recv()` and only terminates once every clone of the mpsc + /// sender is dropped. The `on_progress` closure owns a sender clone, and + /// `WeightedHierarchicalKMeansParams` used to retain an `on_progress` clone + /// that outlived the `progress_worker.await` at the end of the function. + /// With a live sender remaining, `recv()` never returned `None`, the worker + /// never finished, and the trainer hung forever after all compute was done. + /// + /// The build is wrapped in a timeout so the regression fails fast rather + /// than hanging the test process indefinitely. + #[tokio::test(flavor = "multi_thread")] + async fn test_streaming_coreset_ivf_training_terminates() { + use lance_index::progress::IndexBuildProgress; + use std::sync::atomic::{AtomicU64, Ordering}; + use std::time::Duration; + + #[derive(Debug, Default)] + struct CountingProgress { + progress_calls: AtomicU64, + } + + #[async_trait::async_trait] + impl IndexBuildProgress for CountingProgress { + async fn stage_start(&self, _: &str, _: Option, _: &str) -> Result<()> { + Ok(()) + } + async fn stage_progress(&self, _: &str, _: u64) -> Result<()> { + self.progress_calls.fetch_add(1, Ordering::Relaxed); + Ok(()) + } + async fn stage_complete(&self, _: &str) -> Result<()> { + Ok(()) + } + } + + const SMALL_DIM: usize = 8; + + let test_dir = TempStrDir::default(); + let uri = format!("{}/ds", test_dir.as_str()); + let reader = gen_batch() + .col("id", array::step::()) + .col( + "vector", + array::rand_vec::((SMALL_DIM as u32).into()), + ) + .into_reader_rows(RowCount::from(2048), BatchCount::from(4)); + let dataset = Dataset::write(reader, &uri, None).await.unwrap(); + + // > 256 partitions routes through `train_streaming_coreset_ivf_model`. + let mut params = IvfBuildParams::new(257); + params.sample_rate = 8; + params.streaming_sample_rate = Some(4); + params.streaming_refine_passes = 1; + params.max_iters = 2; + + let progress = Arc::new(CountingProgress::default()); + + let ivf_model = tokio::time::timeout( + Duration::from_secs(120), + build_ivf_model( + &dataset, + "vector", + SMALL_DIM, + MetricType::L2, + ¶ms, + None, + progress.clone(), + ), + ) + .await + .expect( + "streaming coreset IVF training hung: progress worker never terminated after training", + ) + .unwrap(); + + assert_eq!(ivf_model.num_partitions(), 257); + assert_eq!(ivf_model.dimension(), SMALL_DIM); + // The progress worker must have processed reports and then joined + // cleanly (proven by `build_ivf_model` returning at all). + assert!( + progress.progress_calls.load(Ordering::Relaxed) > 0, + "expected the progress worker to receive at least one report" + ); + } + #[test] fn test_fixed_training_ranges_are_sorted_and_bounded() { let ranges = generate_fixed_training_ranges(10_000, 1_234, 1_024, 16); diff --git a/rust/lance/src/index/vector/ivf/builder.rs b/rust/lance/src/index/vector/ivf/builder.rs index 9bd1ba95803..d1961071315 100644 --- a/rust/lance/src/index/vector/ivf/builder.rs +++ b/rust/lance/src/index/vector/ivf/builder.rs @@ -13,7 +13,8 @@ use futures::{StreamExt, TryStreamExt}; use lance_arrow::{RecordBatchExt, SchemaExt}; use lance_core::utils::address::RowAddress; use lance_core::utils::tokio::{get_num_compute_intensive_cpus, spawn_cpu}; -use lance_file::previous::writer::FileWriter as PreviousFileWriter; +use lance_file::versions as file_versions; +use lance_file::versions::v1::writer::FileWriter as V1FileWriter; use lance_file::writer::FileWriterOptions; use lance_index::vector::PART_ID_COLUMN; use lance_index::vector::pq::ProductQuantizer; @@ -35,6 +36,7 @@ use lance_linalg::distance::{DistanceType, MetricType}; use crate::Dataset; use crate::dataset::builder::DatasetBuilder; +use crate::dataset::index::dataset_format_version; use crate::index::vector::ivf::io::write_pq_partitions; use super::io::write_hnsw_quantization_index_partitions; @@ -217,7 +219,11 @@ pub async fn write_vector_storage( data.boxed() }; - let mut writer = lance_file::writer::FileWriter::new_lazy(writer, FileWriterOptions::default()); + let mut writer = file_versions::create_lazy_writer( + dataset_format_version(dataset), + writer, + FileWriterOptions::default(), + )?; let mut transformed_stream = data .map_ok(move |batch| { let ivf_transformer = ivf_transformer.clone(); @@ -242,8 +248,8 @@ pub async fn write_vector_storage( #[instrument(level = "debug", skip(writer, auxiliary_writer, data, ivf, quantizer))] pub(super) async fn build_hnsw_partitions( dataset: Arc, - writer: &mut PreviousFileWriter, - auxiliary_writer: Option<&mut PreviousFileWriter>, + writer: &mut V1FileWriter, + auxiliary_writer: Option<&mut V1FileWriter>, data: impl RecordBatchStream + Unpin + 'static, column: &str, ivf: &mut IvfModel, diff --git a/rust/lance/src/index/vector/ivf/io.rs b/rust/lance/src/index/vector/ivf/io.rs index 612dd162281..09cde17d2b2 100644 --- a/rust/lance/src/index/vector/ivf/io.rs +++ b/rust/lance/src/index/vector/ivf/io.rs @@ -10,9 +10,10 @@ use super::IVFIndex; use crate::dataset::ROW_ID; use crate::index::vector::pq::{PQIndex, build_pq_storage}; use arrow::compute::concat; -use arrow_array::UInt64Array; use arrow_array::{ - Array, FixedSizeListArray, RecordBatch, UInt32Array, cast::AsArray, types::UInt64Type, + Array, FixedSizeListArray, PrimitiveArray, RecordBatch, UInt32Array, UInt64Array, + cast::AsArray, + types::{ArrowPrimitiveType, UInt8Type, UInt64Type}, }; use futures::stream::Peekable; use futures::{Stream, StreamExt, TryStreamExt}; @@ -22,10 +23,9 @@ use lance_core::datatypes::Schema; use lance_core::traits::DatasetTakeRows; use lance_core::utils::tempfile::TempStdDir; use lance_core::utils::tokio::{get_num_compute_intensive_cpus, spawn_cpu}; -use lance_file::previous::reader::FileReader as PreviousFileReader; -use lance_file::previous::writer::FileWriter as PreviousFileWriter; +use lance_file::versions::v1::reader::FileReader as V1FileReader; +use lance_file::versions::v1::writer::FileWriter as V1FileWriter; use lance_index::metrics::NoOpMetricsCollector; -use lance_index::scalar::IndexWriter; use lance_index::vector::hnsw::HNSW; use lance_index::vector::hnsw::{HnswMetadata, builder::HnswBuildParams}; use lance_index::vector::ivf::storage::IvfModel; @@ -35,7 +35,6 @@ use lance_index::vector::quantizer::{Quantization, Quantizer}; use lance_index::vector::v3::subindex::IvfSubIndex; use lance_index::vector::{PART_ID_COLUMN, PQ_CODE_COLUMN}; use lance_io::ReadBatchParams; -use lance_io::encodings::plain::PlainEncoder; use lance_io::object_store::ObjectStore; use lance_io::traits::Writer; use lance_linalg::distance::{DistanceType, MetricType}; @@ -43,7 +42,9 @@ use lance_linalg::kernels::normalize_fsl; use lance_table::format::SelfDescribingFileReader; use lance_table::io::manifest::ManifestDescribing; use object_store::path::Path; +use tokio::io::AsyncWriteExt; use tokio::sync::Semaphore; +use tokio::task::JoinHandle; use crate::Result; @@ -51,6 +52,75 @@ use crate::Result; static HNSW_PARTITIONS_BUILD_PARALLEL: LazyLock = LazyLock::new(get_num_compute_intensive_cpus); +async fn write_primitive_values( + writer: &mut dyn Writer, + array: &PrimitiveArray, +) -> Result<()> { + let data = array.to_data(); + let byte_width = std::mem::size_of::(); + let start = array.offset() * byte_width; + let end = start + array.len() * byte_width; + writer + .write_all(&data.buffers()[0].as_slice()[start..end]) + .await?; + Ok(()) +} + +async fn write_pq_codes(writer: &mut dyn Writer, arrays: &[&dyn Array]) -> Result<()> { + for array in arrays { + // Loaded legacy partitions expose transposed codes as flat UInt8 arrays, + // while newly shuffled partitions supply FixedSizeList. + if let Some(values) = array.as_any().downcast_ref::>() { + write_primitive_values(writer, values).await?; + } else if let Some(codes) = array.as_any().downcast_ref::() { + let value_offset = codes.value_offset(0) as usize; + let value_len = codes.len() * codes.value_length() as usize; + let values = codes.values().slice(value_offset, value_len); + let values = values + .as_any() + .downcast_ref::>() + .ok_or_else(|| { + Error::index(format!( + "legacy IVF PQ code values must be UInt8, found {}", + values.data_type() + )) + })?; + write_primitive_values(writer, values).await?; + } else { + return Err(Error::index(format!( + "legacy IVF PQ codes must be UInt8 or FixedSizeList, found {}", + array.data_type() + ))); + } + } + Ok(()) +} + +async fn write_row_ids(writer: &mut dyn Writer, arrays: &[&dyn Array]) -> Result<()> { + for array in arrays { + let row_ids = array + .as_any() + .downcast_ref::>() + .ok_or_else(|| { + Error::index(format!( + "legacy IVF row ids must be UInt64, found {}", + array.data_type() + )) + })?; + write_primitive_values(writer, row_ids).await?; + } + Ok(()) +} + +pub(super) async fn write_pq_partition_payload( + writer: &mut dyn Writer, + pq_codes: &[&dyn Array], + row_ids: &[&dyn Array], +) -> Result<()> { + write_pq_codes(writer, pq_codes).await?; + write_row_ids(writer, row_ids).await +} + /// Merge streams with the same partition id and collect PQ codes and row IDs. async fn merge_streams( streams_heap: &mut BinaryHeap<(Reverse, usize)>, @@ -222,10 +292,8 @@ pub(super) async fn write_pq_partitions( ivf.add_partition_with_offset(writer.tell().await?, total_records as u32); if total_records > 0 { let pq_refs = pq_array.iter().map(|a| a.as_ref()).collect::>(); - PlainEncoder::write(writer, &pq_refs).await?; - let row_ids_refs = row_id_array.iter().map(|a| a.as_ref()).collect::>(); - PlainEncoder::write(writer, row_ids_refs.as_slice()).await?; + write_pq_partition_payload(writer, &pq_refs, &row_ids_refs).await?; } log::info!( "Wrote partition {} in {} ms", @@ -242,8 +310,8 @@ pub(super) async fn write_hnsw_quantization_index_partitions( column: &str, distance_type: DistanceType, hnsw_params: &HnswBuildParams, - writer: &mut PreviousFileWriter, - mut auxiliary_writer: Option<&mut PreviousFileWriter>, + writer: &mut V1FileWriter, + mut auxiliary_writer: Option<&mut V1FileWriter>, ivf: &mut IvfModel, quantizer: Quantizer, streams: Option>>>, @@ -278,164 +346,234 @@ pub(super) async fn write_hnsw_quantization_index_partitions( } let object_store = ObjectStore::local(); - let mut part_files = Vec::with_capacity(ivf.num_partitions()); - let mut aux_part_files = Vec::with_capacity(ivf.num_partitions()); - let tmp_part_dir = Path::from_filesystem_path(TempStdDir::default())?; - let mut tasks = Vec::with_capacity(ivf.num_partitions()); - let sem = Arc::new(Semaphore::new(*HNSW_PARTITIONS_BUILD_PARALLEL)); - for part_id in 0..ivf.num_partitions() { - part_files.push(tmp_part_dir.clone().join(format!("hnsw_part_{}", part_id))); - aux_part_files.push( - tmp_part_dir - .clone() - .join(format!("hnsw_part_aux_{}", part_id)), - ); + // Partitions are staged in this scratch dir, then merged into the final index. + // Share the guard with every task via `Arc` so its `Drop` removes the dir only + // after the last task finishes -- never while one is still writing. + let tmp_part_dir_guard = Arc::new(TempStdDir::default()); + let tmp_part_dir = Path::from_filesystem_path(&**tmp_part_dir_guard)?; + + // `Option` per handle so the consume loop can `take()` each one, leaving the + // not-yet-consumed handles for the error-path drain. + let mut tasks: Vec>>> = + Vec::with_capacity(ivf.num_partitions()); + + let build_result: Result<(Vec, IvfModel)> = async { + let mut part_files = Vec::with_capacity(ivf.num_partitions()); + let mut aux_part_files = Vec::with_capacity(ivf.num_partitions()); + let sem = Arc::new(Semaphore::new(*HNSW_PARTITIONS_BUILD_PARALLEL)); + for part_id in 0..ivf.num_partitions() { + part_files.push(tmp_part_dir.clone().join(format!("hnsw_part_{}", part_id))); + aux_part_files.push( + tmp_part_dir + .clone() + .join(format!("hnsw_part_aux_{}", part_id)), + ); - let mut code_array: Vec> = vec![]; - let mut row_id_array: Vec> = vec![]; + let mut code_array: Vec> = vec![]; + let mut row_id_array: Vec> = vec![]; - // We don't transform vectors to SQ codes while shuffling, - // so we won't merge SQ codes from the stream. + // We don't transform vectors to SQ codes while shuffling, + // so we won't merge SQ codes from the stream. - if let Some(&previous_indices) = existing_indices.as_ref() { - for &idx in previous_indices.iter() { - let sub_index = idx - .load_partition(part_id, true, &NoOpMetricsCollector) - .await?; - let row_ids = Arc::new(UInt64Array::from_iter_values(sub_index.row_ids().cloned())); - row_id_array.push(row_ids); + if let Some(&previous_indices) = existing_indices.as_ref() { + for &idx in previous_indices.iter() { + let sub_index = idx + .load_partition(part_id, true, &NoOpMetricsCollector) + .await?; + let row_ids = + Arc::new(UInt64Array::from_iter_values(sub_index.row_ids().cloned())); + row_id_array.push(row_ids); + } } - } - let code_column = match &quantizer { - Quantizer::Product(pq) => Some(pq.column()), - _ => None, - }; - merge_streams( - &mut streams_heap, - &mut new_streams, - part_id as u32, - code_column, - &mut code_array, - &mut row_id_array, - ) - .await?; - - if row_id_array.is_empty() { - tasks.push(tokio::spawn(async { Ok(0) })); - continue; - } - - let (part_file, aux_part_file) = (&part_files[part_id], &aux_part_files[part_id]); - let part_writer = PreviousFileWriter::::try_new( - &object_store, - part_file, - Schema::try_from(writer.schema())?, - &Default::default(), - ) - .await?; + let code_column = match &quantizer { + Quantizer::Product(pq) => Some(pq.column()), + _ => None, + }; + merge_streams( + &mut streams_heap, + &mut new_streams, + part_id as u32, + code_column, + &mut code_array, + &mut row_id_array, + ) + .await?; - let aux_part_writer = match auxiliary_writer.as_ref() { - Some(writer) => Some( - PreviousFileWriter::::try_new( - &object_store, - aux_part_file, - Schema::try_from(writer.schema())?, - &Default::default(), - ) - .await?, - ), - None => None, - }; + if row_id_array.is_empty() { + tasks.push(Some(tokio::spawn(async { Ok(0) }))); + continue; + } - let dataset = dataset.clone(); - let column = column.to_owned(); - let hnsw_params = hnsw_params.clone(); - let quantizer = quantizer.clone(); - let sem = sem.clone(); - tasks.push(tokio::spawn(async move { - let _permit = sem.acquire().await.expect("semaphore error"); - - log::debug!("Building HNSW partition {}", part_id); - let result = build_hnsw_quantization_partition( - dataset, - &column, - distance_type, - hnsw_params, - part_writer, - aux_part_writer, - quantizer, - row_id_array, - code_array, + let (part_file, aux_part_file) = (&part_files[part_id], &aux_part_files[part_id]); + let part_writer = V1FileWriter::::try_new( + &object_store, + part_file, + Schema::try_from(writer.schema())?, + &Default::default(), ) - .await; - log::debug!("Finished building HNSW partition {}", part_id); - result - })); - } - - let mut aux_ivf = IvfModel::empty(); - let mut hnsw_metadata = Vec::with_capacity(ivf.num_partitions()); - for (part_id, task) in tasks.into_iter().enumerate() { - let offset = writer.len(); - let num_rows = task.await??; + .await?; - if num_rows == 0 { - ivf.add_partition(0); - aux_ivf.add_partition(0); - hnsw_metadata.push(HnswMetadata::default()); - continue; + let aux_part_writer = match auxiliary_writer.as_ref() { + Some(writer) => Some( + V1FileWriter::::try_new( + &object_store, + aux_part_file, + Schema::try_from(writer.schema())?, + &Default::default(), + ) + .await?, + ), + None => None, + }; + + let dataset = dataset.clone(); + let column = column.to_owned(); + let hnsw_params = hnsw_params.clone(); + let quantizer = quantizer.clone(); + let sem = sem.clone(); + let tmp_part_dir_guard = tmp_part_dir_guard.clone(); + tasks.push(Some(tokio::spawn(async move { + // Hold a guard clone so the scratch dir stays alive while this task writes. + let _tmp_part_dir_guard = tmp_part_dir_guard; + let _permit = sem.acquire().await.map_err(|err| { + Error::io(format!( + "failed to acquire HNSW partition build permit: {err}" + )) + })?; + + log::debug!("Building HNSW partition {}", part_id); + let result = build_hnsw_quantization_partition( + dataset, + &column, + distance_type, + hnsw_params, + part_writer, + aux_part_writer, + quantizer, + row_id_array, + code_array, + ) + .await; + log::debug!("Finished building HNSW partition {}", part_id); + result + }))); } - let (part_file, aux_part_file) = (&part_files[part_id], &aux_part_files[part_id]); - let part_reader = - PreviousFileReader::try_new_self_described(&object_store, part_file, None).await?; + let mut aux_ivf = IvfModel::empty(); + let mut hnsw_metadata = Vec::with_capacity(ivf.num_partitions()); + for (part_id, task) in tasks.iter_mut().enumerate() { + let task = task + .take() + .expect("each partition task is consumed exactly once"); + let offset = writer.len(); + let num_rows = task.await??; + + if num_rows == 0 { + ivf.add_partition(0); + aux_ivf.add_partition(0); + hnsw_metadata.push(HnswMetadata::default()); + continue; + } - let batches = futures::stream::iter(0..part_reader.num_batches()) - .map(|batch_id| { - part_reader.read_batch( - batch_id as i32, - ReadBatchParams::RangeFull, - part_reader.schema(), - ) - }) - .buffered(object_store.io_parallelism()) - .try_collect::>() - .await?; - writer.write(&batches).await?; - - ivf.add_partition((writer.len() - offset) as u32); - hnsw_metadata.push(serde_json::from_str( - part_reader.schema().metadata[HNSW::metadata_key()].as_str(), - )?); - std::mem::drop(part_reader); - object_store.delete(part_file).await?; - - if let Some(aux_writer) = auxiliary_writer.as_mut() { - let aux_part_reader = - PreviousFileReader::try_new_self_described(&object_store, aux_part_file, None) - .await?; + let (part_file, aux_part_file) = (&part_files[part_id], &aux_part_files[part_id]); + let part_reader = + V1FileReader::try_new_self_described(&object_store, part_file, None).await?; - let batches = futures::stream::iter(0..aux_part_reader.num_batches()) + let batches = futures::stream::iter(0..part_reader.num_batches()) .map(|batch_id| { - aux_part_reader.read_batch( + part_reader.read_batch( batch_id as i32, ReadBatchParams::RangeFull, - aux_part_reader.schema(), + part_reader.schema(), ) }) .buffered(object_store.io_parallelism()) .try_collect::>() .await?; - std::mem::drop(aux_part_reader); - object_store.delete(aux_part_file).await?; + writer.write(&batches).await?; + + ivf.add_partition((writer.len() - offset) as u32); + hnsw_metadata.push(serde_json::from_str( + part_reader.schema().metadata[HNSW::metadata_key()].as_str(), + )?); + std::mem::drop(part_reader); + object_store.delete(part_file).await?; + + if let Some(aux_writer) = auxiliary_writer.as_mut() { + let aux_part_reader = + V1FileReader::try_new_self_described(&object_store, aux_part_file, None) + .await?; + + let batches = futures::stream::iter(0..aux_part_reader.num_batches()) + .map(|batch_id| { + aux_part_reader.read_batch( + batch_id as i32, + ReadBatchParams::RangeFull, + aux_part_reader.schema(), + ) + }) + .buffered(object_store.io_parallelism()) + .try_collect::>() + .await?; + std::mem::drop(aux_part_reader); + object_store.delete(aux_part_file).await?; - aux_writer.write(&batches).await?; - aux_ivf.add_partition(num_rows as u32); + aux_writer.write(&batches).await?; + aux_ivf.add_partition(num_rows as u32); + } + } + + Ok((hnsw_metadata, aux_ivf)) + } + .await; + + // On error, abort and await the partition builds we never consumed so none + // keep running in the background; see `drain_partition_tasks`. + if build_result.is_err() { + for err in drain_partition_tasks(&mut tasks).await { + log::warn!( + "HNSW partition build task failed while draining after an earlier error: {err}" + ); } } - Ok((hnsw_metadata, aux_ivf)) + build_result +} + +/// Abort and await every still-outstanding partition-build task, returning the +/// non-cancellation errors they surfaced. +/// +/// A dropped [`JoinHandle`] detaches its task, so every handle is aborted up front +/// before any is awaited: otherwise a task slow to observe its own cancellation +/// would keep running -- and keep writing into the scratch dir -- while an earlier +/// handle is still being awaited. Awaiting then resolves only once each task has +/// actually stopped. Cancellation errors are the expected result of the abort and +/// dropped; failures and panics from tasks that had already finished before the +/// abort are returned so the caller can surface them (a task still in flight is +/// cancelled, so this best-effort drain only reports errors already produced). +async fn drain_partition_tasks(tasks: &mut [Option>>]) -> Vec { + for task in tasks.iter() { + if let Some(handle) = task.as_ref() { + handle.abort(); + } + } + + let mut errors = Vec::with_capacity(tasks.len()); + for task in tasks.iter_mut() { + let Some(handle) = task.take() else { + continue; + }; + match handle.await { + Ok(Ok(_)) => {} + Ok(Err(e)) => errors.push(e), + Err(join_err) if join_err.is_cancelled() => {} + Err(join_err) => errors.push(Error::io(format!( + "HNSW partition build task panicked: {join_err}" + ))), + } + } + errors } #[allow(clippy::too_many_arguments)] @@ -444,8 +582,8 @@ async fn build_hnsw_quantization_partition( column: &str, metric_type: MetricType, hnsw_params: Arc, - writer: PreviousFileWriter, - aux_writer: Option>, + writer: V1FileWriter, + aux_writer: Option>, quantizer: Quantizer, row_ids_array: Vec>, code_array: Vec>, @@ -473,24 +611,29 @@ async fn build_hnsw_quantization_partition( let build_hnsw = build_and_write_hnsw(vectors.clone(), (*hnsw_params).clone(), metric_type, writer); + // Build PQ storage as a child future, joined below: it writes `aux_writer`'s + // file into the scratch dir, so it is cancelled together with this task when the + // error-path drain aborts it, and the join surfaces its errors. let build_store = match quantizer { Quantizer::Flat(_) => { return Err(Error::index( "Flat quantizer is not supported for IVF_HNSW".to_string(), )); } - Quantizer::Product(pq) => tokio::spawn(build_and_write_pq_storage( - metric_type, - row_ids, - code_array, - pq, - aux_writer.unwrap(), - )), - - _ => unreachable!("IVF_HNSW_SQ has been moved to v2 index builder"), + Quantizer::Product(pq) => { + let aux_writer = aux_writer.ok_or_else(|| { + Error::index("IVF_HNSW_PQ requires an auxiliary writer for PQ storage".to_string()) + })?; + build_and_write_pq_storage(metric_type, row_ids, code_array, pq, aux_writer) + } + _ => { + return Err(Error::index( + "IVF_HNSW_SQ is not supported in the legacy HNSW partition writer".to_string(), + )); + } }; - let index_rows = futures::join!(build_hnsw, build_store).0?; + let (index_rows, ()) = futures::try_join!(build_hnsw, build_store)?; assert!( index_rows >= num_rows, "index rows {} must be greater than or equal to num rows {}", @@ -504,11 +647,11 @@ async fn build_and_write_hnsw( vectors: Arc, params: HnswBuildParams, distance_type: DistanceType, - mut writer: PreviousFileWriter, + mut writer: V1FileWriter, ) -> Result { let batch = params.build(vectors, distance_type).await?.to_batch()?; let metadata = batch.schema_ref().metadata().clone(); - writer.write_record_batch(batch).await?; + writer.write(&[batch]).await?; Ok(writer.finish_with_metadata(&metadata).await?.num_rows as usize) } @@ -517,11 +660,11 @@ async fn build_and_write_pq_storage( row_ids: Arc, code_array: Vec>, pq: ProductQuantizer, - mut writer: PreviousFileWriter, + mut writer: V1FileWriter, ) -> Result<()> { let storage = spawn_cpu(move || build_pq_storage(metric_type, row_ids, code_array, pq)).await?; - writer.write_record_batch(storage.batch().clone()).await?; + writer.write(&[storage.batch().clone()]).await?; writer.finish().await?; Ok(()) } @@ -530,16 +673,51 @@ async fn build_and_write_pq_storage( mod tests { use super::*; + use std::path::PathBuf; + use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; + use crate::Dataset; use crate::index::vector::ivf::v2; use crate::index::{DatasetIndexExt, DatasetIndexInternalExt, vector::VectorIndexParams}; - use arrow_array::RecordBatchIterator; + use arrow_array::{RecordBatchIterator, UInt8Array}; use arrow_schema::{Field, Schema}; - use lance_core::utils::tempfile::TempStrDir; + use lance_core::utils::tempfile::{TempObjFile, TempStrDir}; use lance_index::IndexType; use lance_index::metrics::NoOpMetricsCollector; + use lance_index::vector::ivf::IvfBuildParams; + use lance_index::vector::pq::PQBuildParams; use lance_testing::datagen::generate_random_array; + #[tokio::test] + async fn pq_partition_payload_preserves_sliced_values() { + let codes = + FixedSizeListArray::try_new_from_values(UInt8Array::from_iter_values(0..6), 2).unwrap(); + let codes = codes.slice(1, 2); + let flat_codes = UInt8Array::from_iter_values(6..9).slice(1, 2); + let row_ids = UInt64Array::from_iter_values([10, 11, 12]).slice(1, 2); + let flat_row_id = UInt64Array::from_iter_values([13]); + + let path = TempObjFile::default(); + let object_store = ObjectStore::local(); + let mut writer = object_store.create(&path).await.unwrap(); + write_pq_partition_payload( + writer.as_mut(), + &[&codes as &dyn Array, &flat_codes as &dyn Array], + &[&row_ids as &dyn Array, &flat_row_id as &dyn Array], + ) + .await + .unwrap(); + Writer::shutdown(&mut writer).await.unwrap(); + + let reader = object_store.open(&path).await.unwrap(); + let bytes = reader.get_range(0..30).await.unwrap(); + let mut expected = vec![2, 3, 4, 5, 7, 8]; + expected.extend_from_slice(&11_u64.to_le_bytes()); + expected.extend_from_slice(&12_u64.to_le_bytes()); + expected.extend_from_slice(&13_u64.to_le_bytes()); + assert_eq!(bytes.as_ref(), expected); + } + #[tokio::test] async fn test_merge_multiple_indices() { const DIM: usize = 32; @@ -589,4 +767,248 @@ mod tests { //let indices = /ds. } + + /// The scratch dir must outlive every partition task: dropping the caller-side + /// guard while tasks are still in flight must not remove it, because each task + /// still holds an `Arc` clone of the guard. + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn test_scratch_dir_outlives_partition_tasks() { + let tmp_part_dir_guard = Arc::new(TempStdDir::default()); + let scratch_path = tmp_part_dir_guard.to_path_buf(); + + // Park each task until the caller-side guard is dropped, so the dir's + // survival is attributable solely to the clones the tasks still hold. + let running = Arc::new(AtomicUsize::new(0)); + let released = Arc::new(AtomicBool::new(false)); + + const NUM_TASKS: usize = 3; + let mut tasks = Vec::with_capacity(NUM_TASKS); + for _ in 0..NUM_TASKS { + let task_guard = tmp_part_dir_guard.clone(); + let running = running.clone(); + let released = released.clone(); + tasks.push(tokio::spawn(async move { + running.fetch_add(1, Ordering::SeqCst); + while !released.load(Ordering::SeqCst) { + tokio::task::yield_now().await; + } + // Still holding `task_guard`, so the directory must be live. + task_guard.exists() + })); + } + + // Drop the caller-side guard only once every task is parked holding its clone. + while running.load(Ordering::SeqCst) < NUM_TASKS { + tokio::task::yield_now().await; + } + drop(tmp_part_dir_guard); + assert!( + scratch_path.exists(), + "scratch dir removed while partition tasks still held the guard" + ); + + released.store(true, Ordering::SeqCst); + for task in tasks { + assert!( + task.await.unwrap(), + "a partition task observed its scratch dir already removed" + ); + } + assert!( + !scratch_path.exists(), + "scratch dir was not removed after the last task's guard clone dropped" + ); + } + + /// The drain step must outlive every spawned task: it aborts and awaits each + /// one so that, once it returns, no task is still running against the scratch + /// directory. It must also surface late failures rather than swallow them. + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn test_drain_partition_tasks_waits_and_reports_errors() { + // Scratch dir guard analogous to the one held by + // `write_hnsw_quantization_index_partitions`; tasks read from it, and it + // is dropped only after the drain completes. + let scratch_guard = TempStdDir::default(); + let scratch_path = scratch_guard.to_path_buf(); + + // Count of live task futures. `LiveGuard` decrements on both completion and + // cancellation, so a zero count after the drain proves every task terminated + // rather than being detached. + let live = Arc::new(AtomicUsize::new(0)); + let saw_missing_dir = Arc::new(AtomicBool::new(false)); + + struct LiveGuard(Arc); + impl Drop for LiveGuard { + fn drop(&mut self) { + self.0.fetch_sub(1, Ordering::SeqCst); + } + } + + const NUM_SLOW: usize = 3; + let mut tasks: Vec>>> = Vec::with_capacity(NUM_SLOW + 1); + + // Tasks that never resolve on their own: the drain's abort is the only thing + // that can stop them, which is exactly what this test exercises. + for _ in 0..NUM_SLOW { + let live = live.clone(); + let saw_missing_dir = saw_missing_dir.clone(); + let scratch_path = scratch_path.clone(); + tasks.push(Some(tokio::spawn(async move { + live.fetch_add(1, Ordering::SeqCst); + let _guard = LiveGuard(live.clone()); + if !scratch_path.exists() { + saw_missing_dir.store(true, Ordering::SeqCst); + } + futures::future::pending::<()>().await; + Ok(0) + }))); + } + + // A task that fails; its error must be returned, not silently dropped. + // The drain aborts every handle up front, and an abort only preserves a + // task's output if the task has already finished -- an in-flight task is + // cancelled and its error lost. So wait until this task has actually + // completed before handing it to the drain; otherwise whether its error + // surfaces would depend on the scheduler and the test would be flaky. + let failing = + tokio::spawn(async move { Err(Error::io("late partition failure".to_string())) }); + while !failing.is_finished() { + tokio::task::yield_now().await; + } + tasks.push(Some(failing)); + + // Ensure all slow tasks are actually running before draining, so the + // drain has to await their cancellation rather than aborting them before + // they ever start. + while live.load(Ordering::SeqCst) < NUM_SLOW { + tokio::task::yield_now().await; + } + + let errors = drain_partition_tasks(&mut tasks).await; + + assert!(tasks.iter().all(Option::is_none), "handles left undrained"); + assert_eq!( + live.load(Ordering::SeqCst), + 0, + "a task was still running after the drain returned" + ); + assert!( + !saw_missing_dir.load(Ordering::SeqCst), + "scratch dir was removed while a task was still running" + ); + assert_eq!(errors.len(), 1, "expected exactly the one late failure"); + assert!( + errors[0].to_string().contains("late partition failure"), + "late failure was not surfaced: {}", + errors[0] + ); + + // The guard, not the drain, removes the scratch dir. + assert!(scratch_path.exists()); + drop(scratch_guard); + assert!(!scratch_path.exists()); + } + + /// `write_hnsw_quantization_index_partitions` stages each partition in a scratch + /// directory owned by a [`TempStdDir`] guard, which must remove it once the build + /// finishes so the OS temp dir does not grow without bound across legacy + /// IVF_HNSW_* builds. + /// + /// The OS temp dir is process-global, so an in-process check can't attribute a + /// leak to our own build. Instead we run the build in a child process with + /// `TMPDIR` pointed at an isolated dir we own, then assert nothing survives. + #[test] + fn test_hnsw_pq_scratch_dir_is_not_leaked() { + // Isolated temp root for the child. Owned here so it -- and anything the + // child leaks into it -- is removed when this guard drops at test end. + let isolated_root = TempStdDir::default(); + + let child_test = "index::vector::ivf::io::tests::build_legacy_hnsw_pq_in_child_process"; + let output = std::process::Command::new(std::env::current_exe().unwrap()) + .args([child_test, "--exact", "--ignored", "--nocapture"]) + .env("TMPDIR", isolated_root.as_ref()) + .env("LANCE_HNSW_LEAK_TEST_ROOT", isolated_root.as_ref()) + .output() + .expect("failed to spawn child test process"); + assert!( + output.status.success(), + "child build process failed:\n--- stdout ---\n{}\n--- stderr ---\n{}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr), + ); + + // Each build stages its partitions in one `.tmp*` dir directly under TMPDIR. + // Every guard removes its dir when it drops, so none should survive. + let leaked: Vec = std::fs::read_dir(&isolated_root) + .expect("read isolated temp root") + .flatten() + .map(|entry| entry.path()) + .filter(|path| { + path.is_dir() + && path + .file_name() + .and_then(|name| name.to_str()) + .is_some_and(|name| name.starts_with(".tmp")) + }) + .collect(); + + assert!( + leaked.is_empty(), + "legacy IVF_HNSW_PQ build leaked {} scratch director{} under the temp dir; \ + the TempStdDir guard should remove each one when it drops: {:?}", + leaked.len(), + if leaked.len() == 1 { "y" } else { "ies" }, + leaked, + ); + } + + /// Child half of [`test_hnsw_pq_scratch_dir_is_not_leaked`]. Ignored so it only + /// runs when the parent spawns it with `TMPDIR` and `LANCE_HNSW_LEAK_TEST_ROOT` + /// pointed at an isolated dir. Builds a few legacy IVF_HNSW_PQ indices; the + /// parent does the leak detection. + #[tokio::test] + #[ignore = "spawned as a child process by test_hnsw_pq_scratch_dir_is_not_leaked"] + async fn build_legacy_hnsw_pq_in_child_process() { + // Only do work when spawned by the parent; a bare `--ignored` run leaves + // the variable unset, so no-op rather than fail. + let Ok(root) = std::env::var("LANCE_HNSW_LEAK_TEST_ROOT") else { + return; + }; + + const DIM: usize = 32; + const ROWS: usize = 1024; + const NLIST: usize = 4; + const NUM_BUILDS: usize = 3; + + // Keep the dataset out of the temp dir's `.tmp*` namespace so the parent + // never confuses it with a leaked scratch directory. + let dataset_uri = format!("{root}/dataset"); + let values = generate_random_array(ROWS * DIM); + let fsl = Arc::new(FixedSizeListArray::try_new_from_values(values, DIM as i32).unwrap()); + let schema = Arc::new(Schema::new(vec![Field::new( + "vector", + fsl.data_type().clone(), + false, + )])); + let batch = RecordBatch::try_new(schema.clone(), vec![fsl]).unwrap(); + let batches = RecordBatchIterator::new(vec![Ok(batch)], schema.clone()); + let ds = Dataset::write(batches, &dataset_uri, Default::default()) + .await + .unwrap(); + + for _ in 0..NUM_BUILDS { + crate::index::vector::ivf::build_ivf_hnsw_pq_index( + &ds, + "vector", + "idx", + uuid::Uuid::new_v4(), + MetricType::L2, + &IvfBuildParams::new(NLIST), + &HnswBuildParams::default(), + &PQBuildParams::new(4, 8), + ) + .await + .unwrap(); + } + } } diff --git a/rust/lance/src/index/vector/ivf/partition_serde.rs b/rust/lance/src/index/vector/ivf/partition_serde.rs index ad737620a94..2d1fd21d3b0 100644 --- a/rust/lance/src/index/vector/ivf/partition_serde.rs +++ b/rust/lance/src/index/vector/ivf/partition_serde.rs @@ -1176,7 +1176,6 @@ mod tests { metadata: lance_index::vector::flat::index::FlatMetadata { dim: 2 }, sub_index_type: SubIndexType::Flat, quantization_type: QuantizationType::Flat, - cache_key_prefix: "prefix/".to_string(), index_file_size: 1024, aux_file_size: 512, rq_search_cache: empty_rabit_search_cache_cell(), diff --git a/rust/lance/src/index/vector/ivf/v2.rs b/rust/lance/src/index/vector/ivf/v2.rs index 0faa79dce5f..24505835b8a 100644 --- a/rust/lance/src/index/vector/ivf/v2.rs +++ b/rust/lance/src/index/vector/ivf/v2.rs @@ -9,7 +9,7 @@ use std::{ any::Any, borrow::Cow, collections::{BinaryHeap, HashMap}, - sync::{Arc, Mutex}, + sync::{Arc, LazyLock, Mutex}, }; use crate::index::vector::{IndexFileVersion, builder::index_type_string}; @@ -27,8 +27,8 @@ use futures::prelude::stream::{self, TryStreamExt}; use futures::{StreamExt, TryFutureExt}; use lance_arrow::RecordBatchExt; use lance_core::cache::{ - CacheCodec, CacheCodecImpl, CacheEntryReader, CacheEntryWriter, CacheKey, LanceCache, - WeakLanceCache, + CacheCodec, CacheCodecImpl, CacheEntryReader, CacheEntryWriter, CacheKey, CacheKeySchema, + KeyBuilder, LanceCache, WeakLanceCache, }; use lance_core::deepsize::DeepSizeOf; use lance_core::utils::tokio::{get_num_compute_intensive_cpus, spawn_cpu}; @@ -36,7 +36,7 @@ use lance_core::utils::tracing::{IO_TYPE_LOAD_VECTOR_PART, TRACE_IO_EVENTS}; use lance_core::{Error, ROW_ID, Result}; use lance_encoding::decoder::{DecoderPlugins, FilterExpression}; use lance_file::LanceEncodingsIo; -use lance_file::reader::{CachedFileMetadata, FileReader, FileReaderOptions}; +use lance_file::reader::{CachedFileMetadata, FileReader, FileReaderOptions, ReaderProjection}; use lance_index::cache_pb::IvfStateHeader; use lance_index::frag_reuse::FragReuseIndex; use lance_index::metrics::{LocalMetricsCollector, MetricsCollector, NoOpMetricsCollector}; @@ -112,8 +112,6 @@ pub(crate) struct IvfIndexState { pub(crate) metadata: ::Metadata, pub(crate) sub_index_type: SubIndexType, pub(crate) quantization_type: QuantizationType, - /// The cache key prefix used by the original index's WeakLanceCache. - pub(crate) cache_key_prefix: String, /// File sizes for the index and auxiliary files, used to avoid HEAD requests /// when reconstructing from cache. pub(crate) index_file_size: u64, @@ -122,6 +120,41 @@ pub(crate) struct IvfIndexState { pub(crate) rq_search_cache: RabitSearchCacheCell, } +/// Number of prepared partitions handed to a single `spawn_cpu` dispatch on the +/// streaming search path. +/// +/// The streaming path deliberately avoids per-partition CPU-task fan-out (a measured +/// 14-30% latency win, see #6475). Searching a batch of partitions per `spawn_cpu` +/// keeps most of that benefit — the per-dispatch overhead is paid once per +/// `STREAMING_SEARCH_BATCH_SIZE` partitions instead of once per partition — while +/// keeping the channel `recv`/`send` in async code so no CPU-pool thread ever parks on +/// a channel (which can deadlock the pool on small hosts, see #7642). `should_stop` is +/// still checked per partition, so early-stop granularity is unchanged. +/// +/// This is a tunable knob: larger batches amortize dispatch overhead further and keep +/// more work on a single CPU thread, at the cost of more prepared partitions held in +/// memory at once. The batch is an upper bound: the search loop greedily drains +/// whatever is already prepared rather than waiting for a full batch, so a slow +/// producer yields small batches (matching the old search-as-it-arrives latency) and +/// only a fast producer fills whole ones. Override with the +/// `LANCE_IVF_STREAMING_SEARCH_BATCH_SIZE` environment variable. +pub(crate) const DEFAULT_STREAMING_SEARCH_BATCH_SIZE: usize = 16; + +pub(crate) static STREAMING_SEARCH_BATCH_SIZE: LazyLock = LazyLock::new(|| { + let batch_size = std::env::var("LANCE_IVF_STREAMING_SEARCH_BATCH_SIZE") + .map(|value| { + value + .parse() + .expect("failed to parse LANCE_IVF_STREAMING_SEARCH_BATCH_SIZE") + }) + .unwrap_or(DEFAULT_STREAMING_SEARCH_BATCH_SIZE); + assert!( + batch_size > 0, + "LANCE_IVF_STREAMING_SEARCH_BATCH_SIZE must be greater than 0, got {batch_size}" + ); + batch_size +}); + struct PreparedPartitionSearch { query: Query, pre_filter: Arc, @@ -209,7 +242,6 @@ impl DeepSizeOf for IvfIndexState { + self.aux_ivf.deep_size_of_children(context) + self.sub_index_metadata.deep_size_of_children(context) + self.metadata.deep_size_of_children(context) - + self.cache_key_prefix.deep_size_of_children(context) + self .rq_search_cache .lock() @@ -321,7 +353,6 @@ impl CacheCodecImpl for IvfStateEntryBox { metadata, sub_index_type, quantization_type, - cache_key_prefix: header.cache_key_prefix, index_file_size: header.index_file_size, aux_file_size: header.aux_file_size, rq_search_cache: empty_rabit_search_cache_cell(), @@ -393,7 +424,6 @@ impl IvfStateEntry for IvfIndexState { sub_index_type: self.sub_index_type.to_string(), quantization_type: self.quantization_type.to_string(), quantizer_metadata_json, - cache_key_prefix: self.cache_key_prefix.clone(), index_file_size: self.index_file_size, aux_file_size: self.aux_file_size, }; @@ -451,43 +481,12 @@ impl CacheKey for FileMetadataCacheKey { fn key(&self) -> std::borrow::Cow<'_, str> { "".into() } -} - -/// Cached open file readers for the index and aux files. -/// -/// Stored in `file_metadata_cache` to avoid re-opening files on every reconstruction. -/// Not serializable (no codec); a cache miss just triggers a re-open. -struct CachedIndexReaders { - index_reader: Arc, - aux_reader: Arc, -} -impl lance_core::deepsize::DeepSizeOf for CachedIndexReaders { - fn deep_size_of_children(&self, context: &mut lance_core::deepsize::Context) -> usize { - // FileReader doesn't impl DeepSizeOf. We approximate by counting the - // fixed struct size for each reader plus the Arc - // heap contents. The metadata Arcs are also held by FileMetadataCacheKey - // entries, so this may over-count across cache entries, but - // over-counting is safer than under-counting for eviction purposes. - std::mem::size_of::() * 2 - + self.index_reader.metadata().deep_size_of_children(context) - + self.aux_reader.metadata().deep_size_of_children(context) + fn schema() -> CacheKeySchema { + CacheKeySchema::new("lance.index.ivf-file-metadata-key", 1) } -} - -struct CachedIndexReadersKey { - uuid: String, -} -impl CacheKey for CachedIndexReadersKey { - type ValueType = CachedIndexReaders; - fn type_name() -> &'static str { - "CachedIndexReaders" - } - fn key(&self) -> std::borrow::Cow<'_, str> { - self.uuid.as_str().into() - } - // No codec() override → in-memory only + fn write_key(&self, _builder: &mut KeyBuilder) {} } /// Open a FileReader, reusing cached file metadata if available. @@ -517,14 +516,20 @@ async fn open_reader_cached( .await } else { let file_scheduler = scheduler.open_file(path, &cached_size).await?; - FileReader::try_open( + let reader = FileReader::try_open( file_scheduler, None, Arc::::default(), cache, FileReaderOptions::default(), ) - .await + .await?; + // File metadata is store-free, so it outlives the reader opened here: + // cache it to spare later reconstructions the footer read. + file_cache + .insert_with_key(&FileMetadataCacheKey, reader.metadata().clone()) + .await; + Ok(reader) } } @@ -566,9 +571,23 @@ impl CacheKey for IVFPartit } fn type_name() -> &'static str { - // Using type_name is safe here: the impl is in the same crate as the - // types, so the monomorphized pointer is consistent. - std::any::type_name::>() + "IVFPartition" + } + + fn schema() -> CacheKeySchema { + CacheKeySchema::new("lance.index.ivf-partition-key", 1) + } + + fn write_key(&self, builder: &mut KeyBuilder) { + builder.write_str(S::name()); + builder.write_variant(match Q::quantization_type() { + QuantizationType::Flat => 0, + QuantizationType::FlatBin => 1, + QuantizationType::Product => 2, + QuantizationType::Scalar => 3, + QuantizationType::Rabit => 4, + }); + builder.write_u64(self.partition_id as u64); } fn codec() -> Option { @@ -590,6 +609,11 @@ pub struct IVFIndex { ivf: IvfModel, reader: FileReader, + /// Narrowed read of the index file, when the sub-index declares that + /// [`IvfSubIndex::load`] consumes only part of what it writes. `None` reads + /// every column. Built once here because it is fallible and only depends on + /// the file schema. + read_projection: Option, sub_index_metadata: Vec, storage: IvfQuantizationStorage, @@ -630,6 +654,18 @@ impl DeepSizeOf for IVFIndex { } impl IVFIndex { + fn read_projection(reader: &FileReader) -> Result> { + S::read_columns() + .map(|columns| { + lance_file::versions::reader_projection_from_column_names( + reader.metadata().version(), + reader.schema(), + columns, + ) + }) + .transpose() + } + fn use_query_residual( storage: &IvfQuantizationStorage, distance_type: DistanceType, @@ -1061,19 +1097,6 @@ impl IVFIndex { .insert_with_key(&FileMetadataCacheKey, storage.reader().metadata().clone()) .await; - // Cache open readers so the first reconstruction also skips file opens. - file_metadata_cache - .insert_with_key( - &CachedIndexReadersKey { - uuid: uuid_str.clone(), - }, - Arc::new(CachedIndexReaders { - index_reader: Arc::new(index_reader.clone()), - aux_reader: Arc::new(storage.reader().clone()), - }), - ) - .await; - let scratch_pool = Arc::new(Self::query_scratch_pool(&ivf, &storage)); let use_query_residual = Self::use_query_residual(&storage, distance_type); let use_residual_scratch = Self::use_residual_scratch(&ivf, use_query_residual); @@ -1085,6 +1108,8 @@ impl IVFIndex { // cumulative stats are exactly the one-time index-open I/O. let open_io_stats = scheduler.stats(); + let read_projection = Self::read_projection(&index_reader)?; + Ok(Self { uri: to_local_path(&uri), index_path: uri.as_ref().to_string(), @@ -1095,6 +1120,7 @@ impl IVFIndex { rq_search_cache, ivf, reader: index_reader, + read_projection, storage, sub_index_metadata, distance_type, @@ -1119,11 +1145,12 @@ impl IVFIndex { index_cache: LanceCache, io_parallelism: usize, rq_search_cache: Option>, - ) -> Self { + ) -> Result { let scratch_pool = Arc::new(Self::query_scratch_pool(&ivf, &storage)); let use_query_residual = Self::use_query_residual(&storage, distance_type); let use_residual_scratch = Self::use_residual_scratch(&ivf, use_query_residual); - Self { + let read_projection = Self::read_projection(&reader)?; + Ok(Self { uri, index_path, uuid, @@ -1133,6 +1160,7 @@ impl IVFIndex { rq_search_cache, ivf, reader, + read_projection, storage, sub_index_metadata, distance_type, @@ -1143,7 +1171,7 @@ impl IVFIndex { // and the first open via `try_new` already accounts for it). open_io_stats: ScanStats::default(), _marker: PhantomData, - } + }) } #[instrument(level = "debug", skip(self, metrics))] @@ -1164,20 +1192,27 @@ impl IVFIndex { let cache_key = IVFPartitionKey::::new(partition_id); if write_cache { - let entry = self + let result = self .index_cache - .get_or_insert_with_key(cache_key, || async { + .get_or_insert_with_key_hit(cache_key, || async { info!(target: TRACE_IO_EVENTS, r#type=IO_TYPE_LOAD_VECTOR_PART, index_type="ivf", part_id=partition_id); metrics.record_part_load(); self.load_partition_entry(partition_id, metrics.io_stats()) .await }) - .await?; + .await; + match &result { + Ok((_, true)) => metrics.record_index_cache_hit(), + _ => metrics.record_index_cache_miss(), + } + let (entry, _) = result?; Ok(entry as Arc) } else { if let Some(part_idx) = self.index_cache.get_with_key(&cache_key).await { + metrics.record_index_cache_hit(); return Ok(part_idx); } + metrics.record_index_cache_miss(); info!(target: TRACE_IO_EVENTS, r#type=IO_TYPE_LOAD_VECTOR_PART, index_type="ivf", part_id=partition_id); metrics.record_part_load(); Ok(Arc::new( @@ -1192,7 +1227,14 @@ impl IVFIndex { partition_id: usize, io_stats: Option, ) -> Result> { - let schema = Arc::new(self.reader.schema().as_ref().into()); + // `concat_batches` indexes the batches by this schema's field positions + // without comparing the two, so the schema has to describe exactly what + // was read: the full file schema over a projected read would index past + // the last column. + let schema = Arc::new(match &self.read_projection { + Some(projection) => projection.schema.as_ref().into(), + None => self.reader.schema().as_ref().into(), + }); let batch = match self.reader.metadata().num_rows { 0 => RecordBatch::new_empty(schema), _ => { @@ -1210,16 +1252,26 @@ impl IVFIndex { } None => Cow::Borrowed(&self.reader), }; - let batches = reader - .read_stream( - ReadBatchParams::Range(row_range), - u32::MAX, - 1, - FilterExpression::no_filter(), - ) - .await? - .try_collect::>() - .await?; + let params = ReadBatchParams::Range(row_range); + let stream = match &self.read_projection { + Some(projection) => { + reader + .read_stream_projected( + params, + u32::MAX, + 1, + projection.clone(), + FilterExpression::no_filter(), + ) + .await? + } + None => { + reader + .read_stream(params, u32::MAX, 1, FilterExpression::no_filter()) + .await? + } + }; + let batches = stream.try_collect::>().await?; concat_batches(&schema, batches.iter())? } } @@ -1271,7 +1323,6 @@ impl IVFIndex { metadata: self.storage.metadata().clone(), sub_index_type, quantization_type, - cache_key_prefix: self.index_cache.prefix().to_string(), index_file_size: self.reader.metadata().file_size(), aux_file_size: self.storage.reader().metadata().file_size(), rq_search_cache: rabit_search_cache_cell(self.rq_search_cache.clone()), @@ -1622,8 +1673,11 @@ impl VectorIndex for IVFInd ))); } + // The prepared channel holds a full search batch so that partitions prepared + // while the previous batch is being searched are ready for the next greedy + // drain, instead of serializing producer and consumer through a single slot. let (prepared_tx, mut prepared_rx) = - mpsc::channel::>>(1); + mpsc::channel::>>(*STREAMING_SEARCH_BATCH_SIZE); let (batch_tx, batch_rx) = mpsc::channel::>(1); let prepare_index = self.clone(); @@ -1665,61 +1719,139 @@ impl VectorIndex for IVFInd let use_query_residual = self.use_query_residual; let use_residual_scratch = self.use_residual_scratch; let search_metrics = metrics.clone(); - let batch_tx_for_search = batch_tx.clone(); let search_control = control.clone(); let scratch_pool = self.scratch_pool.clone(); + // Search prepared partitions in batches. Each batch is searched in a single + // `spawn_cpu` dispatch (amortizing the per-dispatch overhead the single-worker + // design in #6475 avoided), but the channel `recv`/`send` stay in async code so + // no CPU-pool thread ever parks on a channel — parking one can deadlock the pool + // on small hosts (#7642). `should_stop` is checked per partition, so early-stop + // granularity is unchanged. + // + // Batches are formed greedily: wait for one prepared partition, then drain + // whatever else is already prepared, up to the batch size. Waiting for a full + // batch instead would delay the first search (and the early-stop feedback it + // produces) behind up to a whole batch of prepare I/O, which is significant + // when prepare parallelism is low. tokio::spawn(async move { - let search_result = spawn_cpu(move || -> DataFusionResult<()> { - scratch_pool.with_scratch(|scratch| { - while let Some(prepared) = prepared_rx.blocking_recv() { - let prepared = match prepared { - Ok(prepared) => prepared, - Err(err) => { - let _ = batch_tx_for_search - .blocking_send(Err(DataFusionError::from(err))); - return Ok(()); - } - }; + loop { + // Stop pulling as soon as the search is done — or the receiver of our + // results is gone — so the producer stops preparing partitions we + // would never search. + if search_control + .as_ref() + .is_some_and(|control| control.should_stop()) + || batch_tx.is_closed() + { + return; + } - if search_control - .as_ref() - .is_some_and(|control| control.should_stop()) - { - return Ok(()); + let mut prepared_batch = Vec::with_capacity(*STREAMING_SEARCH_BATCH_SIZE); + let mut prepare_error = None; + let mut producer_done = false; + match prepared_rx.recv().await { + Some(Ok(prepared)) => prepared_batch.push(prepared), + Some(Err(err)) => prepare_error = Some(DataFusionError::from(err)), + None => producer_done = true, + } + while prepare_error.is_none() + && !producer_done + && prepared_batch.len() < *STREAMING_SEARCH_BATCH_SIZE + { + match prepared_rx.try_recv() { + Ok(Ok(prepared)) => prepared_batch.push(prepared), + Ok(Err(err)) => { + prepare_error = Some(DataFusionError::from(err)); } - - let batch = { - Self::run_prepared_partition_search( - use_query_residual, - use_residual_scratch, - prepared, - search_metrics.as_ref(), - scratch, - ) + // Nothing else is prepared yet; search what we have rather + // than waiting for more. + Err(mpsc::error::TryRecvError::Empty) => break, + Err(mpsc::error::TryRecvError::Disconnected) => { + producer_done = true; } - .map_err(DataFusionError::from); - match batch { - Ok(batch) => { - if let Some(control) = search_control.as_ref() { - control.record_batch(&batch); + } + } + + if !prepared_batch.is_empty() { + let scratch_pool = scratch_pool.clone(); + let search_metrics = search_metrics.clone(); + let search_control = search_control.clone(); + // `is_closed` is synchronously callable, so a sender clone lets the + // CPU loop notice a dropped receiver between partitions instead of + // searching out the whole batch for a cancelled query. (A `select!` + // on `closed()` would not help here: `spawn_cpu` closures are not + // cancellable, so abandoning the await leaves the work running.) + let cancel_probe = batch_tx.clone(); + let search_output = spawn_cpu(move || { + let mut outputs: Vec> = + Vec::with_capacity(prepared_batch.len()); + // `stopped` means the whole search should end (an error, an + // early-stop signal, or cancellation), not just this batch. + let mut stopped = false; + scratch_pool.with_scratch(|scratch| { + for prepared in prepared_batch { + if search_control + .as_ref() + .is_some_and(|control| control.should_stop()) + || cancel_probe.is_closed() + { + stopped = true; + break; } - if batch_tx_for_search.blocking_send(Ok(batch)).is_err() { - return Ok(()); + match Self::run_prepared_partition_search( + use_query_residual, + use_residual_scratch, + prepared, + search_metrics.as_ref(), + scratch, + ) + .map_err(DataFusionError::from) + { + Ok(batch) => { + if let Some(control) = search_control.as_ref() { + control.record_batch(&batch); + } + outputs.push(Ok(batch)); + } + Err(err) => { + outputs.push(Err(err)); + stopped = true; + break; + } } } - Err(err) => { - let _ = batch_tx_for_search.blocking_send(Err(err)); - return Ok(()); - } + }); + Ok::<_, DataFusionError>((outputs, stopped)) + }) + .await; + + let (outputs, stopped) = match search_output { + Ok(output) => output, + // Defensive: the closure always returns Ok (search errors are + // captured per partition in `outputs`), so this arm should be + // unreachable. Forward and stop rather than drop silently. + Err(err) => { + let _ = batch_tx.send(Err(err)).await; + return; + } + }; + for output in outputs { + if batch_tx.send(output).await.is_err() { + return; } } - Ok(()) - }) - }) - .await; + if stopped { + return; + } + } - if let Err(err) = search_result { - let _ = batch_tx.send(Err(err)).await; + if let Some(err) = prepare_error { + let _ = batch_tx.send(Err(err)).await; + return; + } + if producer_done { + return; + } } }); @@ -1846,43 +1978,25 @@ async fn reconstruct_typed( let dir: Path = parts.into_iter().collect(); let aux_path = dir.clone().join(INDEX_AUXILIARY_FILE_NAME); - let readers_key = CachedIndexReadersKey { - uuid: state.uuid.clone(), - }; - - let (index_reader, aux_reader) = - if let Some(cached) = file_metadata_cache.get_with_key(&readers_key).await { - // Warm path: reuse the cached readers directly, no file opens needed. - ((*cached.index_reader).clone(), (*cached.aux_reader).clone()) - } else { - // Cold path: open files, then cache the readers for future reconstructions. - let scheduler_config = SchedulerConfig::max_bandwidth(&object_store); - let scheduler = ScanScheduler::new(object_store, scheduler_config); - let index_reader = open_reader_cached( - &scheduler, - &index_path, - file_metadata_cache, - state.index_file_size, - ) - .await?; - let aux_reader = open_reader_cached( - &scheduler, - &aux_path, - file_metadata_cache, - state.aux_file_size, - ) - .await?; - file_metadata_cache - .insert_with_key( - &readers_key, - Arc::new(CachedIndexReaders { - index_reader: Arc::new(index_reader.clone()), - aux_reader: Arc::new(aux_reader.clone()), - }), - ) - .await; - (index_reader, aux_reader) - }; + // Readers carry a scheduler bound to an object store, so they cannot be + // shared across dataset opens. Reuse only portable file metadata and bind + // fresh readers to the object store supplied for this reconstruction. + let scheduler_config = SchedulerConfig::max_bandwidth(&object_store); + let scheduler = ScanScheduler::new(object_store, scheduler_config); + let index_reader = open_reader_cached( + &scheduler, + &index_path, + file_metadata_cache, + state.index_file_size, + ) + .await?; + let aux_reader = open_reader_cached( + &scheduler, + &aux_path, + file_metadata_cache, + state.aux_file_size, + ) + .await?; let storage = IvfQuantizationStorage::from_cached( aux_reader, @@ -1907,15 +2021,21 @@ async fn reconstruct_typed( index_cache, io_parallelism, rq_search_cache, - ); + )?; Ok(Arc::new(index)) } #[cfg(test)] mod tests { - use std::collections::HashSet; + use std::collections::{HashMap, HashSet}; use std::iter::repeat_n; - use std::{ops::Range, sync::Arc}; + use std::{ + ops::Range, + sync::{ + Arc, + atomic::{AtomicBool, AtomicUsize, Ordering}, + }, + }; use all_asserts::{assert_ge, assert_le, assert_lt}; use arrow::datatypes::{Float64Type, UInt8Type, UInt64Type}; @@ -1935,11 +2055,14 @@ mod tests { transform::{EX_ADD_FACTORS_COLUMN, EX_SCALE_FACTORS_COLUMN}, }; use lance_index::vector::storage::VectorStore; + use lance_index::vector::v3::subindex::IvfSubIndex; use crate::dataset::{InsertBuilder, UpdateBuilder, WriteMode, WriteParams}; use crate::index::DatasetIndexExt; use crate::index::DatasetIndexInternalExt; - use crate::index::vector::ivf::v2::IvfPq; + use crate::index::vector::ivf::v2::{ + IVFPartitionKey, IvfFlatIndex, IvfHnswSqIndex, IvfPq, IvfStateEntryBox, PartitionEntry, + }; use crate::utils::test::copy_test_data_to_tmp; use crate::{ Dataset, @@ -1949,20 +2072,23 @@ mod tests { dataset::optimize::{CompactionOptions, compact_files}, index::vector::IndexFileVersion, }; - use lance_core::cache::LanceCache; + use lance_core::cache::{CacheBackend, CacheCodecImpl, LanceCache}; use lance_core::utils::tempfile::TempStrDir; use lance_core::{ROW_ID, Result}; use lance_encoding::decoder::DecoderPlugins; use lance_file::reader::{FileReader, FileReaderOptions}; - use lance_file::writer::FileWriter; use lance_index::IndexType; + use lance_index::optimize::OptimizeOptions; use lance_index::progress::IndexBuildProgress; use lance_index::vector::DIST_COL; + use lance_index::vector::flat::index::FlatIndex; + use lance_index::vector::hnsw::HNSW; use lance_index::vector::hnsw::builder::HnswBuildParams; use lance_index::vector::ivf::IvfBuildParams; use lance_index::vector::kmeans::{KMeansParams, train_kmeans}; - use lance_index::vector::pq::PQBuildParams; + use lance_index::vector::pq::{PQBuildParams, ProductQuantizer}; use lance_index::vector::quantizer::QuantizerMetadata; + use lance_index::vector::sq::ScalarQuantizer; use lance_index::vector::sq::builder::SQBuildParams; use lance_index::vector::{ pq::storage::ProductQuantizationMetadata, @@ -1970,9 +2096,8 @@ mod tests { storage::STORAGE_METADATA_KEY, }; use lance_index::{INDEX_AUXILIARY_FILE_NAME, metrics::NoOpMetricsCollector}; - use lance_index::{optimize::OptimizeOptions, scalar::IndexReader}; use lance_io::{ - object_store::ObjectStore, + object_store::{ObjectStore, ObjectStoreParams, StorageOptionsAccessor}, scheduler::{ScanScheduler, SchedulerConfig}, utils::CachedFileSize, }; @@ -2757,6 +2882,17 @@ mod tests { .collect() } + async fn load_flat_partition_row_ids(index: &IvfFlatIndex, partition_idx: usize) -> Vec { + index + .storage + .load_partition(partition_idx, None) + .await + .unwrap() + .row_ids() + .copied() + .collect() + } + async fn delete_ids(dataset: &mut Dataset, ids: &[u64]) { if ids.is_empty() { return; @@ -4411,17 +4547,19 @@ mod tests { test_index_impl::(params, nlist, 0.75, -1.0..1.0, None).await; } - // RQ doesn't perform well for random data - // need to verify recall with real-world dataset (e.g. sift1m) + // These queries probe every partition, so recall here measures RaBitQ quantization + // error alone. At 1 bit per dimension it averages ~0.67 on this uniformly random, + // L2-normalized data, and each build draws a fresh random rotation, so no bar worth + // asserting sits clear of the spread. 5 bits lifts recall to ~0.97; its `ex_bits = 4` + // also covers a FastScan ex-code kernel that the multi-bit test below never reaches. #[rstest] - #[case(1, DistanceType::L2, 0.5)] - #[case(1, DistanceType::Cosine, 0.5)] - #[case(1, DistanceType::Dot, 0.5)] - #[case(4, DistanceType::L2, 0.5)] - #[case(4, DistanceType::Cosine, 0.5)] - #[case(4, DistanceType::Dot, 0.5)] + #[case(1, DistanceType::L2, 0.9)] + #[case(1, DistanceType::Cosine, 0.9)] + #[case(1, DistanceType::Dot, 0.9)] + #[case(4, DistanceType::L2, 0.9)] + #[case(4, DistanceType::Cosine, 0.9)] + #[case(4, DistanceType::Dot, 0.9)] #[tokio::test] - // #[ignore = "Temporarily skipping flaky 4-bit IVF_RQ tests"] async fn test_build_ivf_rq( #[case] nlist: usize, #[case] distance_type: DistanceType, @@ -4430,7 +4568,7 @@ mod tests { ) { let _ = env_logger::try_init(); let ivf_params = IvfBuildParams::new(nlist); - let rq_params = RQBuildParams::with_rotation_type(1, rotation_type); + let rq_params = RQBuildParams::with_rotation_type(5, rotation_type); let params = VectorIndexParams::with_ivf_rq_params(distance_type, ivf_params, rq_params); test_index(params.clone(), nlist, recall_requirement, None).await; if distance_type == DistanceType::Cosine { @@ -4653,6 +4791,226 @@ mod tests { } } + // `lance-index` keeps these crate-private; spelling them out here also pins + // the on-disk names, which are part of the index file contract. + const HNSW_VECTOR_ID_COL: &str = "__vector_id"; + const HNSW_NEIGHBORS_COL: &str = "__neighbors"; + + async fn build_ivf_hnsw_sq(test_uri: &str, nlist: usize) -> Dataset { + let (mut dataset, _) = generate_test_dataset::(test_uri, 0.0..1.0).await; + let params = VectorIndexParams::with_ivf_hnsw_sq_params( + DistanceType::L2, + IvfBuildParams::new(nlist), + HnswBuildParams::default(), + SQBuildParams::default(), + ); + dataset + .create_index(&["vector"], IndexType::Vector, None, ¶ms, true) + .await + .unwrap(); + dataset + } + + async fn open_ivf_hnsw_sq(dataset: &Dataset) -> Arc { + let indices = dataset.load_indices().await.unwrap(); + dataset + .open_vector_index("vector", &indices[0].uuid, &NoOpMetricsCollector) + .await + .unwrap() + } + + async fn assert_hnsw_columns(dataset: &Dataset, context: &str) { + let index = open_ivf_hnsw_sq(dataset).await; + let hnsw = index + .as_any() + .downcast_ref::() + .expect("IVF_HNSW_SQ should open as IvfHnswSqIndex"); + + let written = hnsw + .reader + .schema() + .fields + .iter() + .map(|f| f.name.as_str()) + .collect::>(); + assert_eq!( + written, + vec![HNSW_VECTOR_ID_COL, HNSW_NEIGHBORS_COL, DIST_COL], + "{context}: the written index file must keep every column" + ); + + // Every partition, not just the first: a projection that applied + // unevenly would leave the partition cache holding mixed schemas. + for partition_id in 0..hnsw.ivf.num_partitions() { + let entry = hnsw.load_partition_entry(partition_id, None).await.unwrap(); + let loaded = entry.index.to_batch().unwrap(); + let loaded_schema = loaded.schema(); + let read = loaded_schema + .fields() + .iter() + .map(|f| f.name().as_str()) + .collect::>(); + assert_eq!( + read, + vec![HNSW_VECTOR_ID_COL, HNSW_NEIGHBORS_COL], + "{context}: partition {partition_id} materialized the write-only distance column" + ); + } + } + + /// The index file keeps all three HNSW columns while a loaded partition + /// carries only the two the graph reads. Both halves matter: shrinking the + /// written schema would panic readers older than v8.0.0, and widening the + /// read back would undo the saving. + /// + /// Re-checked after a delta merge, because that is the one path that writes + /// a new index file while an already-projected index is open. + #[tokio::test] + async fn test_hnsw_partition_load_reads_only_graph_columns() { + let test_dir = TempStrDir::default(); + let mut dataset = build_ivf_hnsw_sq(test_dir.as_str(), 4).await; + assert_hnsw_columns(&dataset, "fresh index").await; + + append_dataset::(&mut dataset, 64, 0.0..1.0).await; + dataset + .optimize_indices(&OptimizeOptions::append()) + .await + .unwrap(); + dataset + .optimize_indices(&OptimizeOptions::merge(10)) + .await + .unwrap(); + assert_hnsw_columns(&dataset, "after delta merge").await; + } + + /// The saving is the point of the change, so pin it: reading a real + /// partition range through the declared projection must move strictly fewer + /// bytes than the full-schema read the index used to perform. + #[tokio::test] + async fn test_hnsw_read_projection_moves_fewer_bytes() { + use futures::TryStreamExt as _; + + let test_dir = TempStrDir::default(); + let dataset = build_ivf_hnsw_sq(test_dir.as_str(), 4).await; + let index = open_ivf_hnsw_sq(&dataset).await; + let hnsw = index + .as_any() + .downcast_ref::() + .expect("IVF_HNSW_SQ should open as IvfHnswSqIndex"); + + let projection = hnsw + .read_projection + .as_ref() + .expect("HNSW declares a read projection"); + assert_eq!(projection.schema.fields.len(), 2); + + let row_range = hnsw.ivf.row_range(0); + assert!(!row_range.is_empty(), "partition 0 should hold rows"); + let store = dataset.object_store.as_ref(); + + let read_bytes_for = async |projection: lance_file::reader::ReaderProjection| { + let _ = store.io_stats_incremental(); + hnsw.reader + .read_stream_projected( + lance_io::ReadBatchParams::Range(row_range.clone()), + u32::MAX, + 1, + projection, + lance_encoding::decoder::FilterExpression::no_filter(), + ) + .await + .unwrap() + .try_collect::>() + .await + .unwrap(); + store.io_stats_incremental().read_bytes + }; + + // Projected first on purpose: it then pays any first-touch metadata + // cost, so the comparison understates rather than flatters the saving. + let projected_bytes = read_bytes_for(projection.clone()).await; + let full_bytes = read_bytes_for(lance_file::versions::reader_projection_from_whole_schema( + hnsw.reader.schema(), + hnsw.reader.metadata().version(), + )) + .await; + + assert!( + projected_bytes > 0, + "the projected read still has to fetch the graph" + ); + assert_lt!(projected_bytes, full_bytes); + } + + /// The projection selects columns by field id, and `__neighbors` and + /// `_distance` are both 4-byte-item lists whose child fields share a name, + /// so a wrong column index would reinterpret distances as neighbor ids with + /// no type error to catch it. Compare the columns themselves, not just names. + #[tokio::test] + async fn test_hnsw_projected_read_matches_full_read() { + use futures::TryStreamExt as _; + + let test_dir = TempStrDir::default(); + let dataset = build_ivf_hnsw_sq(test_dir.as_str(), 4).await; + let index = open_ivf_hnsw_sq(&dataset).await; + let hnsw = index + .as_any() + .downcast_ref::() + .expect("IVF_HNSW_SQ should open as IvfHnswSqIndex"); + let projection = hnsw + .read_projection + .as_ref() + .expect("HNSW declares a read projection"); + + let read_range = async |proj: lance_file::reader::ReaderProjection, + range: std::ops::Range| { + let batches = hnsw + .reader + .read_stream_projected( + lance_io::ReadBatchParams::Range(range), + u32::MAX, + 1, + proj, + lance_encoding::decoder::FilterExpression::no_filter(), + ) + .await + .unwrap() + .try_collect::>() + .await + .unwrap(); + arrow::compute::concat_batches(&batches[0].schema(), &batches).unwrap() + }; + + let mut compared = 0; + for partition_id in 0..hnsw.ivf.num_partitions() { + let range = hnsw.ivf.row_range(partition_id); + if range.is_empty() { + continue; + } + let full = read_range( + lance_file::versions::reader_projection_from_whole_schema( + hnsw.reader.schema(), + hnsw.reader.metadata().version(), + ), + range.clone(), + ) + .await; + let projected = read_range(projection.clone(), range).await; + + assert_eq!(projected.num_columns(), 2); + assert_eq!(projected.num_rows(), full.num_rows()); + for name in [HNSW_VECTOR_ID_COL, HNSW_NEIGHBORS_COL] { + assert_eq!( + projected.column_by_name(name).unwrap(), + full.column_by_name(name).unwrap(), + "partition {partition_id}: {name} differs between the projected and full read" + ); + } + compared += 1; + } + assert!(compared > 0, "no non-empty partition was compared"); + } + async fn test_index_multivec(params: VectorIndexParams, nlist: usize, recall_requirement: f32) { // we introduce XTR for performance, which would reduce the recall a little bit let recall_requirement = recall_requirement * 0.9; @@ -4717,10 +5075,7 @@ mod tests { .as_primitive::() .values() .to_vec(); - let results = dists - .into_iter() - .zip(row_ids.clone().into_iter()) - .collect::>(); + let results = dists.into_iter().zip(row_ids.clone()).collect::>(); let row_ids = row_ids.into_iter().collect::>(); let gt = multivec_ground_truth(&vectors, &query, k, params.metric_type); @@ -5113,10 +5468,7 @@ mod tests { .as_primitive::() .values() .to_vec(); - let results = dists - .into_iter() - .zip(row_ids.into_iter()) - .collect::>(); + let results = dists.into_iter().zip(row_ids).collect::>(); let row_ids = results.iter().map(|(_, id)| *id).collect::>(); assert!(row_ids.len() == k); @@ -5172,11 +5524,25 @@ mod tests { // Rewrite auxiliary file with PQ codebook inlined into schema metadata. let mut metadata = reader.schema().metadata.clone(); - let batch = reader - .read_range(0..reader.num_rows() as usize, None) + let projection = lance_file::versions::reader_projection_from_whole_schema( + reader.schema(), + reader.metadata().version(), + ); + let batches = reader + .read_stream_projected( + lance_io::ReadBatchParams::RangeFull, + u32::MAX, + u32::MAX, + projection, + lance_encoding::decoder::FilterExpression::no_filter(), + ) .await?; + use futures::TryStreamExt as _; + let batches = batches.try_collect::>().await?; + let batch = arrow::compute::concat_batches(&batches[0].schema(), &batches)?; let new_aux_path = new_dir.clone().join(INDEX_AUXILIARY_FILE_NAME); - let mut writer = FileWriter::try_new( + let mut writer = lance_file::versions::create_writer( + reader.metadata().version(), obj_store.create(&new_aux_path).await?, batch.schema_ref().as_ref().try_into()?, Default::default(), @@ -6024,50 +6390,204 @@ mod tests { } } - #[tokio::test] - async fn test_prewarm_ivf_pq() { - use lance_io::assert_io_eq; + async fn row_ids_matching(dataset: &Dataset, predicate: &str) -> HashSet { + let mut scan = dataset.scan(); + scan.with_row_id(); + scan.filter(predicate).unwrap(); + let batch = scan.try_into_batch().await.unwrap(); + batch[ROW_ID] + .as_primitive::() + .values() + .iter() + .copied() + .collect() + } + + struct OptimizeAfterDelete { + deleted_row_ids: HashSet, + index_row_ids: HashSet, + num_partitions_after: usize, + stats_json: String, + } + /// Shared scenario for the issue-7701 regressions: stable-row-id dataset, + /// IVF_FLAT index, scattered delete, optimize. Asserts the invariants both + /// partition adjustments must hold -- no live row lost, no id that never + /// existed -- and returns the state for the mode-specific assertions. + async fn optimize_after_delete( + total_rows: usize, + nlist: usize, + delete_predicate: &str, + keep_predicate: &str, + ) -> OptimizeAfterDelete { + const INDEX_NAME: &str = "vector_idx"; let test_dir = TempStrDir::default(); let test_uri = test_dir.as_str(); - let (mut dataset, _) = generate_test_dataset::(test_uri, 0.0..1.0).await; - let params = VectorIndexParams::with_ivf_pq_params( - DistanceType::L2, - IvfBuildParams::new(4), - PQBuildParams::default(), - ); + let (batch, schema) = generate_batch::(total_rows, None, 0.0..1.0, false); + let batches = RecordBatchIterator::new(vec![batch].into_iter().map(Ok), schema); + let mut dataset = Dataset::write( + batches, + test_uri, + Some(WriteParams { + enable_stable_row_ids: true, + ..Default::default() + }), + ) + .await + .unwrap(); + + let params = VectorIndexParams::ivf_flat(nlist, DistanceType::L2); dataset .create_index( &["vector"], IndexType::Vector, - Some("my_idx".to_owned()), + Some(INDEX_NAME.to_string()), ¶ms, true, ) .await .unwrap(); - // Reset IO stats after index creation - dataset.object_store.as_ref().io_stats_incremental(); - - // Prewarm should perform IO to load all partitions into cache - dataset.prewarm_index("my_idx").await.unwrap(); - let stats = dataset.object_store.as_ref().io_stats_incremental(); - assert!( - stats.read_iops > 0, - "prewarm should have read from disk, but read_iops was 0" - ); + let deleted_row_ids = row_ids_matching(&dataset, delete_predicate).await; + let live_row_ids = row_ids_matching(&dataset, keep_predicate).await; + dataset.delete(delete_predicate).await.unwrap(); - // Can query index without IO - let q = Float32Array::from_iter_values(repeat_n(0.0, DIM)); dataset - .scan() - .nearest("vector", &q, 10) - .unwrap() - .project(&["_rowid"]) - .unwrap() - .try_into_batch() + .optimize_indices(&OptimizeOptions::new()) + .await + .unwrap(); + + let final_ctx = load_vector_index_context(&dataset, "vector", INDEX_NAME).await; + let num_partitions_after = final_ctx.num_partitions(); + let stats_json = final_ctx.stats_json().to_string(); + let flat = final_ctx + .index + .as_any() + .downcast_ref::() + .expect("expected IvfFlat index"); + let mut index_row_ids = HashSet::new(); + for part in 0..flat.ivf.num_partitions() { + index_row_ids.extend(load_flat_partition_row_ids(flat, part).await); + } + + for row_id in &live_row_ids { + assert!( + index_row_ids.contains(row_id), + "live row id {} missing from index after optimize", + row_id + ); + } + for row_id in &index_row_ids { + assert!( + live_row_ids.contains(row_id) || deleted_row_ids.contains(row_id), + "unexpected row id {} in index after optimize", + row_id + ); + } + + OptimizeAfterDelete { + deleted_row_ids, + index_row_ids, + num_partitions_after, + stats_json, + } + } + + #[tokio::test] + async fn test_optimize_join_after_delete_with_stable_row_ids() { + // Regression test for https://github.com/lance-format/lance/issues/7701: + // every partition (400 rows / 4) is under the IVF_FLAT join threshold, + // so optimize joins the smallest after a scattered delete. + let run = optimize_after_delete(400, 4, "id % 3 = 0", "id % 3 != 0").await; + + assert_eq!( + run.num_partitions_after, 3, + "optimize should have joined the smallest partition, got stats: {}", + run.stats_json + ); + + // The join reads every partition's stored rows through the merge filter, so + // deleted ids are dropped index-wide and not just from the joined partition. + for row_id in &run.deleted_row_ids { + assert!( + !run.index_row_ids.contains(row_id), + "deleted row id {} still in index after join", + row_id + ); + } + } + + #[tokio::test] + async fn test_optimize_split_after_delete_with_stable_row_ids() { + // Regression test for https://github.com/lance-format/lance/issues/7701: + // one partition holds more than 4x the IVF_FLAT target, so optimize + // splits it after a scattered delete. This path reaches + // filter_deleted_ids through reshuffle_partitions, unlike the join + // path's take_vectors. + let run = optimize_after_delete(20_000, 1, "id % 5 = 0", "id % 5 != 0").await; + + assert!( + run.num_partitions_after > 1, + "optimize should have split the oversized partition, got stats: {}", + run.stats_json + ); + + // The split rebuilds the whole partition from live rows: no deleted + // ids remain. + for row_id in &run.deleted_row_ids { + assert!( + !run.index_row_ids.contains(row_id), + "deleted row id {} still in index after split", + row_id + ); + } + } + + #[tokio::test] + async fn test_prewarm_ivf_pq() { + use lance_io::assert_io_eq; + + let test_dir = TempStrDir::default(); + let test_uri = test_dir.as_str(); + let (mut dataset, _) = generate_test_dataset::(test_uri, 0.0..1.0).await; + + let params = VectorIndexParams::with_ivf_pq_params( + DistanceType::L2, + IvfBuildParams::new(4), + PQBuildParams::default(), + ); + dataset + .create_index( + &["vector"], + IndexType::Vector, + Some("my_idx".to_owned()), + ¶ms, + true, + ) + .await + .unwrap(); + + // Reset IO stats after index creation + dataset.object_store.as_ref().io_stats_incremental(); + + // Prewarm should perform IO to load all partitions into cache + dataset.prewarm_index("my_idx").await.unwrap(); + let stats = dataset.object_store.as_ref().io_stats_incremental(); + assert!( + stats.read_iops > 0, + "prewarm should have read from disk, but read_iops was 0" + ); + + // Can query index without IO + let q = Float32Array::from_iter_values(repeat_n(0.0, DIM)); + dataset + .scan() + .nearest("vector", &q, 10) + .unwrap() + .project(&["_rowid"]) + .unwrap() + .try_into_batch() .await .unwrap(); let stats = dataset.object_store.as_ref().io_stats_incremental(); @@ -6199,53 +6719,120 @@ mod tests { assert_io_eq!(stats, read_iops, 0, "second prewarm should not perform IO"); } - type SerializedEntry = (Vec, lance_core::cache::CacheCodec, usize); - + /// Index-cache backend that can drop partition entries on demand. + /// + /// Used to simulate cache invalidation after a credential rotation: + /// partition keys are opaque digests, so the test supplies the exact + /// [`InternalCacheKey`] set to bypass once the index identity is known + /// (see [`ivf_partition_cache_keys`]). #[derive(Debug)] - struct SerializingBackend { - /// Serialized entries: key -> (bytes, codec, size). - serialized: tokio::sync::Mutex< - std::collections::HashMap, - >, - /// Fallback for entries without a codec. - passthrough: lance_core::cache::MokaCacheBackend, + struct PartitionBypassCacheBackend { + inner: lance_core::cache::MokaCacheBackend, + partition_keys: std::sync::Mutex>, + bypass_partitions: AtomicBool, + partition_hits: AtomicUsize, } - impl SerializingBackend { + impl PartitionBypassCacheBackend { fn new() -> Self { Self { - serialized: tokio::sync::Mutex::new(std::collections::HashMap::new()), - passthrough: lance_core::cache::MokaCacheBackend::with_capacity(256 * 1024 * 1024), + inner: lance_core::cache::MokaCacheBackend::with_capacity(256 * 1024 * 1024), + partition_keys: std::sync::Mutex::new(HashSet::new()), + bypass_partitions: AtomicBool::new(false), + partition_hits: AtomicUsize::new(0), } } - async fn serialized_entry_count(&self) -> usize { - self.serialized.lock().await.len() + fn set_partition_keys(&self, partition_keys: HashSet) { + *self.partition_keys.lock().unwrap() = partition_keys; + } + + fn is_partition(&self, key: &lance_core::cache::InternalCacheKey) -> bool { + self.partition_keys.lock().unwrap().contains(key) + } + + fn set_bypass_partitions(&self, bypass_partitions: bool) { + self.bypass_partitions + .store(bypass_partitions, Ordering::Relaxed); + } + + fn should_bypass(&self, key: &lance_core::cache::InternalCacheKey) -> bool { + self.bypass_partitions.load(Ordering::Relaxed) && self.is_partition(key) + } + + /// Whether the backend currently holds an entry for `key`. + async fn contains(&self, key: &lance_core::cache::InternalCacheKey) -> bool { + self.inner.get(key, None).await.is_some() } - async fn passthrough_entry_count(&self) -> usize { - use lance_core::cache::CacheBackend; - self.passthrough.num_entries().await + fn partition_hits(&self) -> usize { + self.partition_hits.load(Ordering::Relaxed) + } + } + + /// Derive the internal cache keys of the IVF partition entries for an + /// index, replicating the namespace path + /// `dataset URI -> index UUID -> frag-reuse UUID` used when opening the + /// index. V3 partitions use [`IVFPartitionKey`]; legacy (v0.1/v0.2) + /// indices use `LegacyIVFPartitionKey`. + fn ivf_partition_cache_keys( + dataset_uri: &str, + uuid: &uuid::Uuid, + fri_uuid: Option<&uuid::Uuid>, + num_partitions: usize, + index_version: &IndexFileVersion, + ) -> HashSet { + use lance_core::cache::{CacheKey, CacheNamespace, KeyBuilder, UnsizedCacheKey}; + + let mut namespace = CacheNamespace::root().child(dataset_uri); + namespace = namespace.child(uuid.as_hyphenated().to_string().as_str()); + if let Some(fri_uuid) = fri_uuid { + namespace = namespace.child(fri_uuid.as_hyphenated().to_string().as_str()); } + + (0..num_partitions) + .map(|partition_id| { + if matches!(index_version, IndexFileVersion::V3) { + let cache_key = + IVFPartitionKey::::new(partition_id); + let mut builder = KeyBuilder::new( + namespace, + IVFPartitionKey::::stable_type_id(), + IVFPartitionKey::::schema(), + ); + cache_key.write_key(&mut builder); + builder.finish() + } else { + let cache_key = + crate::index::vector::ivf::LegacyIVFPartitionKey::new(partition_id); + let mut builder = KeyBuilder::new( + namespace, + crate::index::vector::ivf::LegacyIVFPartitionKey::stable_type_id(), + crate::index::vector::ivf::LegacyIVFPartitionKey::schema(), + ); + cache_key.write_key(&mut builder); + builder.finish() + } + }) + .collect() } #[async_trait::async_trait] - impl lance_core::cache::CacheBackend for SerializingBackend { + impl lance_core::cache::CacheBackend for PartitionBypassCacheBackend { async fn get( &self, key: &lance_core::cache::InternalCacheKey, codec: Option, ) -> Option { - // Try serialized store first - let guard = self.serialized.lock().await; - if let Some((bytes, stored_codec, _)) = guard.get(key) { - return stored_codec - .deserialize(&bytes::Bytes::copy_from_slice(bytes)) - .hit(); + if self.should_bypass(key) { + None + } else { + let entry = self.inner.get(key, codec).await; + if entry.is_some() && self.is_partition(key) { + self.partition_hits.fetch_add(1, Ordering::Relaxed); + } + entry } - drop(guard); - // Fall through to passthrough - self.passthrough.get(key, codec).await } async fn insert( @@ -6255,17 +6842,8 @@ mod tests { size_bytes: usize, codec: Option, ) { - if let Some(codec) = codec { - let mut bytes = Vec::new(); - codec - .serialize(&entry, &mut bytes) - .expect("serialization should succeed"); - self.serialized - .lock() - .await - .insert(key.clone(), (bytes, codec, size_bytes)); - } else { - self.passthrough.insert(key, entry, size_bytes, None).await; + if !self.should_bypass(key) { + self.inner.insert(key, entry, size_bytes, codec).await; } } @@ -6281,40 +6859,38 @@ mod tests { >, codec: Option, ) -> Result<(lance_core::cache::CacheEntry, bool)> { - if let Some(entry) = self.get(key, codec).await { - return Ok((entry, true)); + if self.should_bypass(key) { + let (entry, _) = loader.await?; + Ok((entry, false)) + } else { + let result = self.inner.get_or_insert(key, loader, codec).await; + if result.as_ref().is_ok_and(|(_, is_cache_hit)| *is_cache_hit) + && self.is_partition(key) + { + self.partition_hits.fetch_add(1, Ordering::Relaxed); + } + result } - let (entry, size) = loader.await?; - self.insert(key, entry.clone(), size, codec).await; - Ok((entry, false)) - } - - async fn invalidate_prefix(&self, prefix: &str) { - self.serialized - .lock() - .await - .retain(|k, _| !k.starts_with(prefix)); - self.passthrough.invalidate_prefix(prefix).await; } async fn clear(&self) { - self.serialized.lock().await.clear(); - self.passthrough.clear().await; + self.inner.clear().await; } async fn num_entries(&self) -> usize { - self.serialized.lock().await.len() + self.passthrough.num_entries().await + self.inner.num_entries().await } async fn size_bytes(&self) -> usize { - let serialized: usize = self - .serialized - .lock() - .await - .values() - .map(|(_, _, s)| *s) - .sum(); - serialized + self.passthrough.size_bytes().await + self.inner.size_bytes().await + } + + fn approx_num_entries(&self) -> usize { + self.inner.approx_num_entries() + } + + fn approx_size_bytes(&self) -> usize { + self.inner.approx_size_bytes() } } @@ -6322,18 +6898,37 @@ mod tests { /// serializing cache backend, then query. Verifies that entries are /// serialized to bytes and that queries produce correct results after /// deserialization. + #[rstest] + #[case::ivf_pq( + VectorIndexParams::with_ivf_pq_params( + DistanceType::L2, + IvfBuildParams::new(4), + PQBuildParams::default(), + ), + as CacheCodecImpl>::TYPE_ID + )] + #[case::ivf_hnsw_sq( + VectorIndexParams::with_ivf_hnsw_sq_params( + DistanceType::L2, + IvfBuildParams::new(4), + HnswBuildParams::default(), + SQBuildParams::default(), + ), + as CacheCodecImpl>::TYPE_ID + )] #[tokio::test] - async fn test_prewarm_and_query_with_serializing_backend() { + async fn test_prewarm_and_query_with_serializing_backend( + #[case] params: VectorIndexParams, + #[case] partition_type_id: &'static str, + ) { + use crate::utils::test::serializing_cache::SerializingCacheBackend; + use lance_io::assert_io_eq; + let test_dir = TempStrDir::default(); let test_uri = test_dir.as_str(); // Create dataset with vector index using default cache let (mut dataset, _) = generate_test_dataset::(test_uri, 0.0..1.0).await; - let params = VectorIndexParams::with_ivf_pq_params( - DistanceType::L2, - IvfBuildParams::new(4), - PQBuildParams::default(), - ); dataset .create_index( &["vector"], @@ -6345,8 +6940,11 @@ mod tests { .await .unwrap(); + let q = Float32Array::from_iter_values(repeat_n(0.5, DIM)); + let expected = ground_truth(&dataset, "vector", &q, 10, DistanceType::L2).await; + // Re-open with the serializing backend - let backend = Arc::new(SerializingBackend::new()); + let backend = Arc::new(SerializingCacheBackend::new()); let session = Arc::new(crate::session::Session::with_index_cache_backend( backend.clone(), 128 * 1024 * 1024, @@ -6361,7 +6959,10 @@ mod tests { // Prewarm — this should serialize entries into the backend dataset.prewarm_index("serde_idx").await.unwrap(); let serialized = backend.serialized_entry_count().await; - let passthrough = backend.passthrough_entry_count().await; + let state_type_id = IvfStateEntryBox::TYPE_ID; + let state_inserts = backend.serialized_insert_count(state_type_id).await; + let partition_inserts = backend.serialized_insert_count(partition_type_id).await; + let passthrough = backend.l1_entry_count().await; assert!( serialized > 0, "prewarm should have serialized entries into the backend" @@ -6372,18 +6973,53 @@ mod tests { but found {passthrough} passthrough entries" ); - // Query — the backend will deserialize entries from bytes. - // After prewarm, all entries are in serialized form, so every - // cache hit involves a deserialization round-trip. - let q = Float32Array::from_iter_values(repeat_n(0.5, DIM)); + drop(dataset); + let backend = Arc::new(backend.restart()); + assert_eq!( + backend.l1_entry_count().await, + 0, + "restarting must discard the in-memory L1" + ); + assert_eq!( + backend.serialized_entry_count().await, + serialized, + "restarting must retain the serialized IVF state and partitions" + ); + let session = Arc::new(crate::session::Session::with_index_cache_backend( + backend.clone(), + 128 * 1024 * 1024, + Arc::new(lance_io::object_store::ObjectStoreRegistry::default()), + )); + let dataset = crate::DatasetBuilder::from_uri(test_uri) + .with_session(session) + .load() + .await + .unwrap(); + + // Query — the recreated backend will deserialize entries from bytes. + // All index entries are in serialized form, so every cache hit involves + // a deserialization round-trip. let results = dataset .scan() + .with_row_id() .nearest("vector", &q, 10) .unwrap() .nprobes(4) + .project(&["_rowid"]) + .unwrap() .try_into_batch() .await .unwrap(); + assert_eq!( + backend.serialized_insert_count(state_type_id).await, + state_inserts, + "the first restarted query must reuse the serialized IVF state" + ); + assert_eq!( + backend.serialized_insert_count(partition_type_id).await, + partition_inserts, + "the first restarted query must reuse every serialized IVF partition" + ); assert_eq!(results.num_rows(), 10, "should return 10 nearest neighbors"); // Verify distances are sorted (ascending for L2) @@ -6396,5 +7032,391 @@ mod tests { for w in distances.windows(2) { assert!(w[1] >= w[0], "distances should be sorted ascending"); } + + let row_ids = results[ROW_ID] + .as_primitive::() + .values() + .iter() + .copied() + .collect::>(); + let recall = row_ids.intersection(&expected).count() as f32 / expected.len() as f32; + assert_ge!( + recall, + 0.5, + "serialized IVF query recall is below threshold: {recall}" + ); + + dataset.object_store.as_ref().io_stats_incremental(); + dataset + .scan() + .nearest("vector", &q, 10) + .unwrap() + .nprobes(4) + .project(&["_rowid"]) + .unwrap() + .try_into_batch() + .await + .unwrap(); + let stats = dataset.object_store.as_ref().io_stats_incremental(); + assert_io_eq!( + stats, + read_iops, + 0, + "warmed IVF query should not perform IO after backend restart" + ); + } + + #[rstest] + #[case::v3(IndexFileVersion::V3)] + #[case::legacy(IndexFileVersion::Legacy)] + #[tokio::test] + async fn test_vector_cache_uses_current_object_store(#[case] index_version: IndexFileVersion) { + let test_dir = TempStrDir::default(); + let test_uri = test_dir.as_str(); + let (mut dataset, vectors) = generate_test_dataset::(test_uri, 0.0..1.0).await; + append_dataset::(&mut dataset, NUM_ROWS, 0.0..1.0).await; + assert_eq!(dataset.get_fragments().len(), 2); + + let params = VectorIndexParams::with_ivf_pq_params( + DistanceType::L2, + IvfBuildParams::new(4), + PQBuildParams::default(), + ) + .version(index_version.clone()) + .clone(); + dataset + .create_index( + &["vector"], + IndexType::Vector, + Some("credential_rotation_idx".to_owned()), + ¶ms, + true, + ) + .await + .unwrap(); + let index_meta = dataset + .load_indices_by_name("credential_rotation_idx") + .await + .unwrap() + .pop() + .unwrap(); + let query = vectors.value(0); + let ground_truth = ground_truth(&dataset, "vector", &query, 20, DistanceType::L2).await; + + let cache_backend = Arc::new(PartitionBypassCacheBackend::new()); + let session = Arc::new(crate::session::Session::with_index_cache_backend( + cache_backend.clone(), + 128 * 1024 * 1024, + Arc::new(lance_io::object_store::ObjectStoreRegistry::default()), + )); + let dataset = crate::DatasetBuilder::from_uri(test_uri) + .with_session(session) + .load() + .await + .unwrap(); + + let store_params_a = ObjectStoreParams { + storage_options_accessor: Some(Arc::new(StorageOptionsAccessor::with_static_options( + HashMap::from([( + "credential_generation".to_owned(), + "secret-generation-a".to_owned(), + )]), + ))), + ..Default::default() + }; + let (store_a, _) = ObjectStore::from_uri_and_params( + dataset.session().store_registry(), + dataset.uri(), + &store_params_a, + ) + .await + .unwrap(); + let dataset_a = dataset.with_object_store(store_a.clone(), Some(store_params_a)); + + let store_params_b = ObjectStoreParams { + storage_options_accessor: Some(Arc::new(StorageOptionsAccessor::with_static_options( + HashMap::from([( + "credential_generation".to_owned(), + "secret-generation-b".to_owned(), + )]), + ))), + ..Default::default() + }; + let (store_b, _) = ObjectStore::from_uri_and_params( + dataset.session().store_registry(), + dataset.uri(), + &store_params_b, + ) + .await + .unwrap(); + assert!(!Arc::ptr_eq(&store_a, &store_b)); + let dataset_b = dataset.with_object_store(store_b.clone(), Some(store_params_b)); + + let _ = store_a.io_stats_incremental(); + let _ = store_b.io_stats_incremental(); + + dataset_a + .scan() + .nearest("vector", &query, 20) + .unwrap() + .minimum_nprobes(4) + .with_row_id() + .try_into_batch() + .await + .unwrap(); + + let frag_reuse_uuid = dataset_a.frag_reuse_index_uuid().await; + let state_cache_key = + crate::index::IvfIndexStateCacheKey::new(&index_meta.uuid, frag_reuse_uuid.as_ref()); + let cached_state = if matches!(index_version, IndexFileVersion::V3) { + Some( + dataset_a + .index_cache + .get_with_key(&state_cache_key) + .await + .expect("V3 IVF state should be cached"), + ) + } else { + None + }; + let index_path_fragment = format!("_indices/{}", index_meta.uuid); + let first_store_stats = store_a.io_stats_incremental(); + assert!( + first_store_stats + .requests + .iter() + .any(|request| request.path.as_ref().contains(&index_path_fragment)), + "the first query should read the index through the first object store: {first_store_stats:#?}" + ); + let partition_keys = ivf_partition_cache_keys( + dataset.uri(), + &index_meta.uuid, + frag_reuse_uuid.as_ref(), + 4, + &index_version, + ); + cache_backend.set_partition_keys(partition_keys.clone()); + for partition_key in &partition_keys { + assert!( + cache_backend.contains(partition_key).await, + "the first query should populate portable partition entries" + ); + } + let index_entries_after_a = dataset.session().index_cache_stats().await.num_entries; + let metadata_entries_after_a = dataset.session().metadata_cache_stats().await.num_entries; + let _ = store_b.io_stats_incremental(); + + cache_backend.set_bypass_partitions(true); + let results = dataset_b + .scan() + .nearest("vector", &query, 20) + .unwrap() + .minimum_nprobes(4) + .with_row_id() + .try_into_batch() + .await + .unwrap(); + let row_ids = results[ROW_ID] + .as_primitive::() + .values() + .iter() + .copied() + .collect::>(); + let recall = row_ids.intersection(&ground_truth).count() as f32 / 20.0; + assert_ge!(recall, 0.5); + + let old_store_stats = store_a.io_stats_incremental(); + let old_store_index_reads = old_store_stats + .requests + .iter() + .filter(|request| request.path.as_ref().contains(&index_path_fragment)) + .count(); + let new_store_stats = store_b.io_stats_incremental(); + let new_store_index_reads = new_store_stats + .requests + .iter() + .filter(|request| request.path.as_ref().contains(&index_path_fragment)) + .count(); + if matches!(index_version, IndexFileVersion::V3) { + assert_eq!( + old_store_index_reads, 0, + "the new dataset query must not use readers bound to the old object store: {old_store_stats:#?}" + ); + assert!( + new_store_index_reads > 0, + "the new dataset query should read the index through the new object store: {new_store_stats:#?}" + ); + } else { + // Legacy live indices are shared across dataset opens: their + // readers stay bound to the object store that first populated the + // cache, so the second dataset keeps reading through the old store. + assert!( + old_store_index_reads > 0, + "the cached legacy index should keep reading through the original object store: {old_store_stats:#?}" + ); + assert_eq!( + new_store_index_reads, 0, + "the cached legacy index must not reopen through the new object store: {new_store_stats:#?}" + ); + } + if let Some(cached_state) = cached_state { + let state_after_rotation = dataset_b + .index_cache + .get_with_key(&state_cache_key) + .await + .expect("V3 IVF state should remain cached after rotation"); + assert!( + Arc::ptr_eq(&cached_state, &state_after_rotation), + "store-free IVF state should be reused across object-store generations" + ); + } + + // Re-query through the first dataset: V3 portable state is rebound to + // the store supplied by each reconstruction, while the cached legacy + // index keeps reading through its original store. Either way the second + // store must not be touched. + let _ = store_a.io_stats_incremental(); + let _ = store_b.io_stats_incremental(); + dataset_a + .scan() + .nearest("vector", &query, 20) + .unwrap() + .minimum_nprobes(4) + .with_row_id() + .try_into_batch() + .await + .unwrap(); + let store_a_stats = store_a.io_stats_incremental(); + let store_a_index_reads = store_a_stats + .requests + .iter() + .filter(|request| request.path.as_ref().contains(&index_path_fragment)) + .count(); + let store_b_stats = store_b.io_stats_incremental(); + let store_b_index_reads = store_b_stats + .requests + .iter() + .filter(|request| request.path.as_ref().contains(&index_path_fragment)) + .count(); + assert!( + store_a_index_reads > 0, + "re-querying the first dataset should read the index through its object store: {store_a_stats:#?}" + ); + assert_eq!( + store_b_index_reads, 0, + "re-querying the first dataset must not use the second object store: {store_b_stats:#?}" + ); + + // Cache keys are opaque digests, so they cannot embed credential + // material by construction. What rotation must not do is mint new + // entries: the same portable state, partitions, and file metadata + // serve both object-store generations. + let index_entries_after_rotation = dataset.session().index_cache_stats().await.num_entries; + let metadata_entries_after_rotation = + dataset.session().metadata_cache_stats().await.num_entries; + assert_eq!( + index_entries_after_rotation, index_entries_after_a, + "credential rotation must not create new index cache entries" + ); + assert_eq!( + metadata_entries_after_rotation, metadata_entries_after_a, + "credential rotation must not create new metadata cache entries" + ); + + cache_backend.set_bypass_partitions(false); + let partition_hits_before = cache_backend.partition_hits(); + dataset_b + .scan() + .nearest("vector", &query, 20) + .unwrap() + .minimum_nprobes(4) + .with_row_id() + .try_into_batch() + .await + .unwrap(); + assert!( + cache_backend.partition_hits() > partition_hits_before, + "the second store should reuse portable partitions populated by the first" + ); + } + + #[tokio::test] + async fn test_shallow_clone_ivf_rq_uses_resolved_index_directory() { + let test_dir = TempStrDir::default(); + let source_uri = format!("{}/source", test_dir.as_str()); + let clone_uri = format!("{}/clone", test_dir.as_str()); + let (mut source, vectors) = + generate_test_dataset::(&source_uri, 0.0..1.0).await; + append_dataset::(&mut source, NUM_ROWS, 0.0..1.0).await; + assert_eq!(source.get_fragments().len(), 2); + + let params = VectorIndexParams::ivf_rq(4, 5, DistanceType::L2); + source + .create_index( + &["vector"], + IndexType::Vector, + Some("ivf_rq_idx".to_owned()), + ¶ms, + true, + ) + .await + .unwrap(); + + let query = vectors.value(0); + let ground_truth = ground_truth(&source, "vector", &query, 20, DistanceType::L2).await; + source + .tags() + .create("with_ivf_rq", source.version().version) + .await + .unwrap(); + let cloned = source + .shallow_clone(&clone_uri, "with_ivf_rq", None) + .await + .unwrap(); + + let index_meta = cloned + .load_indices_by_name("ivf_rq_idx") + .await + .unwrap() + .pop() + .unwrap(); + assert!( + index_meta.base_id.is_some(), + "a shallow-cloned index should reference its source base" + ); + assert_eq!( + cloned.indice_files_dir(&index_meta).unwrap(), + source.indices_dir(), + "the cloned index should resolve its path through the source base" + ); + assert_ne!( + cloned.indice_files_dir(&index_meta).unwrap(), + cloned.indices_dir(), + "the cloned index should not use the clone's primary index directory" + ); + + let cloned = crate::DatasetBuilder::from_uri(&clone_uri) + .with_session(Arc::new(crate::session::Session::default())) + .load() + .await + .unwrap(); + + let results = cloned + .scan() + .nearest("vector", &query, 20) + .unwrap() + .minimum_nprobes(4) + .with_row_id() + .try_into_batch() + .await + .unwrap(); + let row_ids = results[ROW_ID] + .as_primitive::() + .values() + .iter() + .copied() + .collect::>(); + let recall = row_ids.intersection(&ground_truth).count() as f32 / 20.0; + assert_ge!(recall, 0.5); } } diff --git a/rust/lance/src/index/vector/pq.rs b/rust/lance/src/index/vector/pq.rs index 217667afbe9..862f81d2742 100644 --- a/rust/lance/src/index/vector/pq.rs +++ b/rust/lance/src/index/vector/pq.rs @@ -5,11 +5,15 @@ use lance_core::utils::row_addr_remap::RowAddrRemap; use std::any::Any; use std::sync::Arc; -use arrow::compute::concat; +use arrow::{ + array::{ArrayData, make_array}, + compute::concat, +}; use arrow_array::types::UInt64Type; use arrow_array::{ Array, FixedSizeListArray, RecordBatch, UInt8Array, UInt64Array, cast::{AsArray, as_primitive_array}, + new_empty_array, }; use arrow_array::{ArrayRef, Float32Array, UInt32Array}; use arrow_ord::sort::sort_to_indices; @@ -18,7 +22,7 @@ use arrow_select::take::take; use async_trait::async_trait; use datafusion::execution::SendableRecordBatchStream; use datafusion::physical_plan::stream::RecordBatchStreamAdapter; -use lance_arrow::FixedSizeListArrayExt; +use lance_arrow::{BufferExt, DataTypeExt, FixedSizeListArrayExt}; use lance_core::deepsize::DeepSizeOf; use lance_core::utils::address::RowAddress; use lance_core::utils::tokio::spawn_cpu; @@ -33,7 +37,7 @@ use lance_index::{ Index, IndexType, vector::{Query, pq::ProductQuantizer}, }; -use lance_io::{traits::Reader, utils::read_fixed_stride_array}; +use lance_io::traits::Reader; use lance_linalg::distance::{DistanceType, MetricType}; use log::{info, warn}; use roaring::RoaringBitmap; @@ -71,6 +75,36 @@ pub struct PQIndex { frag_reuse_index: Option>, } +async fn read_legacy_index_values( + reader: &dyn Reader, + data_type: &DataType, + offset: usize, + length: usize, +) -> Result { + if length == 0 { + return Ok(new_empty_array(data_type)); + } + + let byte_length = length + .checked_mul(data_type.byte_width()) + .ok_or_else(|| Error::index("legacy IVF page byte length overflow".to_string()))?; + let end = offset + .checked_add(byte_length) + .ok_or_else(|| Error::index("legacy IVF page offset overflow".to_string()))?; + let bytes = reader.get_range(offset..end).await?; + let buffer = if bytes.len() < byte_length { + arrow_buffer::Buffer::copy_bytes_bytes(bytes, byte_length) + } else { + arrow_buffer::Buffer::from_bytes_bytes(bytes, data_type.byte_width() as u64) + }; + let data = ArrayData::builder(data_type.clone()) + .len(length) + .null_count(0) + .add_buffer(buffer) + .build()?; + Ok(make_array(data)) +} + impl DeepSizeOf for PQIndex { fn deep_size_of_children(&self, context: &mut lance_core::deepsize::Context) -> usize { self.pq.deep_size_of_children(context) @@ -331,24 +365,14 @@ impl VectorIndex for PQIndex { length: usize, ) -> Result> { let pq_code_length = self.pq.code_dim() * length; - let pq_codes = read_fixed_stride_array( - reader.as_ref(), - &DataType::UInt8, - offset, - pq_code_length, - .., - ) - .await?; + let pq_codes = + read_legacy_index_values(reader.as_ref(), &DataType::UInt8, offset, pq_code_length) + .await?; let row_id_offset = offset + pq_code_length /* *1 */; - let row_ids = read_fixed_stride_array( - reader.as_ref(), - &DataType::UInt64, - row_id_offset, - length, - .., - ) - .await?; + let row_ids = + read_legacy_index_values(reader.as_ref(), &DataType::UInt64, row_id_offset, length) + .await?; let pq_codes = transpose( pq_codes.as_primitive(), @@ -651,8 +675,12 @@ mod tests { use arrow::datatypes::Float32Type; use arrow_array::RecordBatchIterator; use arrow_schema::{Field, Schema}; - use lance_core::utils::tempfile::TempStrDir; + use lance_core::utils::tempfile::{TempObjFile, TempStrDir}; + use lance_io::object_store::ObjectStore; + use lance_io::traits::Writer; use lance_linalg::kernels::normalize_fsl; + use object_store::path::Path; + use tokio::io::AsyncWriteExt; use crate::index::vector::ivf::build_ivf_model; use lance_index::metrics::NoOpMetricsCollector; @@ -664,6 +692,47 @@ mod tests { }; const DIM: usize = 128; + + #[tokio::test] + async fn empty_legacy_ivf_pq_partition_does_not_read_object_store() { + let object_store = ObjectStore::memory(); + let reader = object_store + .open(&Path::from("missing-index")) + .await + .unwrap(); + let codebook_values = Float32Array::from_iter_values((0..256).map(|value| value as f32)); + let codebook = FixedSizeListArray::try_new_from_values(codebook_values, 1).unwrap(); + let index = PQIndex::new( + ProductQuantizer::new(1, 8, 1, codebook, DistanceType::L2), + MetricType::L2, + None, + ); + + let loaded = index.load(reader.into(), 1, 0).await.unwrap(); + + assert_eq!(loaded.num_rows(), 0); + } + + #[tokio::test] + async fn legacy_index_values_are_read_from_the_requested_offset() { + let path = TempObjFile::default(); + let object_store = ObjectStore::local(); + let mut writer = object_store.create(&path).await.unwrap(); + writer.write_all(&[0xFF]).await.unwrap(); + writer.write_all(&11_u64.to_le_bytes()).await.unwrap(); + writer.write_all(&12_u64.to_le_bytes()).await.unwrap(); + Writer::shutdown(writer.as_mut()).await.unwrap(); + + let reader = object_store.open(&path).await.unwrap(); + let values = read_legacy_index_values(reader.as_ref(), &DataType::UInt64, 1, 2) + .await + .unwrap(); + assert_eq!( + values.as_primitive::(), + &UInt64Array::from_iter_values([11, 12]) + ); + } + async fn generate_dataset( test_uri: &str, range: Range, diff --git a/rust/lance/src/index/vector/utils.rs b/rust/lance/src/index/vector/utils.rs index 3046d0f3a83..012d8d3a703 100644 --- a/rust/lance/src/index/vector/utils.rs +++ b/rust/lance/src/index/vector/utils.rs @@ -1,16 +1,16 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright The Lance Authors -use std::collections::HashSet; use std::pin::Pin; use std::sync::Arc; +use std::sync::atomic::{AtomicUsize, Ordering}; use arrow::array::ArrayData; use arrow::datatypes::DataType; use arrow_array::new_empty_array; use arrow_array::{Array, ArrayRef, FixedSizeListArray, RecordBatch, cast::AsArray}; use arrow_buffer::{Buffer, MutableBuffer}; -use futures::{Stream, StreamExt, TryStreamExt, stream}; +use futures::{Stream, StreamExt, stream}; use lance_arrow::DataTypeExt; use lance_core::datatypes::Schema; use lance_linalg::distance::DistanceType; @@ -18,6 +18,7 @@ use log::{info, warn}; use rand::rngs::SmallRng; use rand::seq::{IteratorRandom, SliceRandom}; use rand::{Rng, SeedableRng}; +use roaring::RoaringTreemap; use tokio::sync::Mutex; use crate::dataset::{Dataset, ProjectionRequest, TakeBuilder, row_offsets_to_row_addresses}; @@ -264,38 +265,7 @@ fn infer_vector_element_type_impl( async fn count_rows(dataset: &Dataset, fragment_ids: Option<&[u32]>) -> Result { match fragment_ids { None => dataset.count_rows(None).await, - Some(fragment_ids) => { - let sorted_ids: Vec; - let sorted_fragment_ids = if fragment_ids.windows(2).all(|w| w[0] <= w[1]) { - fragment_ids - } else { - sorted_ids = { - let mut v = fragment_ids.to_vec(); - v.sort_unstable(); - v - }; - &sorted_ids - }; - let fragments = dataset.get_frags_from_ordered_ids(sorted_fragment_ids); - let valid_fragments = fragments - .into_iter() - .enumerate() - .map(|(i, frag)| { - frag.ok_or_else(|| { - Error::index(format!( - "Unexpectedly missing fragment {}", - sorted_fragment_ids[i] - )) - }) - }) - .collect::>>()?; - let cnts = stream::iter(valid_fragments) - .map(|f| async move { f.count_rows(None).await }) - .buffer_unordered(16) - .try_collect::>() - .await?; - Ok(cnts.iter().sum::()) - } + Some(fragment_ids) => dataset.count_rows_in_fragments(fragment_ids).await, } } @@ -309,8 +279,6 @@ pub async fn maybe_sample_training_data( sample_size_hint: usize, fragment_ids: Option<&[u32]>, ) -> Result { - let num_rows = count_rows(dataset, fragment_ids).await?; - let vector_field = dataset.schema().field(column).ok_or(Error::index(format!( "Sample training data: column {} does not exist in schema", column @@ -328,6 +296,8 @@ pub async fn maybe_sample_training_data( return Ok(new_empty_array(&fsl_type).as_fixed_size_list().clone()); } + let num_rows = count_rows(dataset, fragment_ids).await?; + let is_nullable = vector_field.nullable; let sample_size_hint = match vector_field.data_type() { @@ -486,18 +456,39 @@ async fn sample_training_data( ); return vector_column_to_fsl(&batch, column); } + // Rows the consumer still needs. The fragment producer sizes each + // prefetch round to this outstanding demand, keeping reads bounded by + // the requested sample size. + let still_needed = Arc::new(AtomicUsize::new(sample_size_hint)); let scan = sample_training_data_scan_from_fragments( dataset, column, - sample_size_hint, num_rows, fragment_ids, + still_needed.clone(), )?; return match vector_field.data_type() { DataType::FixedSizeList(_, _) => { - sample_nullable_fsl(column, sample_size_hint, byte_width, vector_field, scan).await + sample_nullable_fsl( + column, + sample_size_hint, + byte_width, + vector_field, + scan, + Some(still_needed), + ) + .await + } + _ => { + sample_nullable_fallback( + column, + sample_size_hint, + is_nullable, + scan, + Some(still_needed), + ) + .await } - _ => sample_nullable_fallback(column, sample_size_hint, is_nullable, scan).await, }; } @@ -516,12 +507,20 @@ async fn sample_training_data( DataType::FixedSizeList(_, _) => { let scan = sample_training_data_scan(dataset, column, sample_size_hint, num_rows, byte_width)?; - sample_nullable_fsl(column, sample_size_hint, byte_width, vector_field, scan).await + sample_nullable_fsl( + column, + sample_size_hint, + byte_width, + vector_field, + scan, + None, + ) + .await } _ => { let scan = sample_training_data_scan(dataset, column, sample_size_hint, num_rows, byte_width)?; - sample_nullable_fallback(column, sample_size_hint, is_nullable, scan).await + sample_nullable_fallback(column, sample_size_hint, is_nullable, scan, None).await } } } @@ -550,12 +549,19 @@ fn sample_training_data_scan( /// sampling must first map random offsets within the selected fragments to row /// addresses and then `take` those rows. Both nullable FSL and multivector /// paths reuse this stream to avoid duplicating fragment sampling logic. +/// +/// Each round is sized to `still_needed` (the consumer's outstanding demand), +/// so a low-null column reads at most the requested sample. Visited offsets +/// are tracked in a [`RoaringTreemap`] because sparse or all-null columns +/// force the stream to visit most selected rows before it can terminate, and +/// a compressed bitmap keeps that persistent state near `num_rows / 8` bytes +/// even when fully populated. fn sample_training_data_scan_from_fragments( dataset: &Dataset, column: &str, - sample_size_hint: usize, num_rows: usize, fragment_ids: &[u32], + still_needed: Arc, ) -> Result> + Send>>> { if fragment_ids.is_empty() { return Err(Error::invalid_input( @@ -563,21 +569,7 @@ fn sample_training_data_scan_from_fragments( )); } - let mut ordered_ids = fragment_ids.to_vec(); - ordered_ids.sort_unstable(); - ordered_ids.dedup(); - let selected_fragments = dataset - .get_frags_from_ordered_ids(&ordered_ids) - .into_iter() - .zip(ordered_ids.iter()) - .map(|(fragment, fragment_id)| { - fragment.ok_or_else(|| { - Error::invalid_input(format!( - "Unknown fragment id {fragment_id} in training fragment filter" - )) - }) - }) - .collect::>>()?; + let selected_fragments = dataset.get_fragments_from_ids(fragment_ids)?; let dataset = Arc::new(dataset.clone()); let projection = Arc::new( ProjectionRequest::from(dataset.schema().project(&[column])?) @@ -589,19 +581,36 @@ fn sample_training_data_scan_from_fragments( dataset, projection, selected_fragments, - HashSet::::with_capacity(sample_size_hint.min(num_rows)), + RoaringTreemap::new(), SmallRng::from_os_rng(), + still_needed, ), - move |(dataset, projection, selected_fragments, mut seen_offsets, mut rng)| async move { - if seen_offsets.len() >= num_rows { + move |( + dataset, + projection, + selected_fragments, + mut seen_offsets, + mut rng, + still_needed, + )| async move { + if seen_offsets.len() as usize >= num_rows { + return Ok(None); + } + let still = still_needed.load(Ordering::Relaxed); + if still == 0 { return Ok(None); } - let remaining = num_rows.saturating_sub(seen_offsets.len()); - let target = sample_size_hint.saturating_mul(2).min(remaining); + let remaining = num_rows.saturating_sub(seen_offsets.len() as usize); + // Sizing the round to the outstanding demand keeps a low-null + // column's reads bounded by the requested sample, matching the + // non-nullable path. + let target = still.min(remaining); let mut sampled_offsets = if remaining <= target.saturating_mul(4) { + // Few offsets remain unseen, so shuffling the unseen set is + // cheaper than repeatedly rejecting already-sampled offsets. let mut unseen_indices = (0..num_rows as u64) - .filter(|index| !seen_offsets.contains(index)) + .filter(|index| !seen_offsets.contains(*index)) .collect::>(); unseen_indices.shuffle(&mut rng); unseen_indices.truncate(target); @@ -635,7 +644,14 @@ fn sample_training_data_scan_from_fragments( .await?; Ok(Some(( batch, - (dataset, projection, selected_fragments, seen_offsets, rng), + ( + dataset, + projection, + selected_fragments, + seen_offsets, + rng, + still_needed, + ), ))) }, ); @@ -646,22 +662,7 @@ fn resolve_scan_fragments( dataset: &Dataset, fragment_ids: &[u32], ) -> Result> { - let mut ordered_ids = fragment_ids.to_vec(); - ordered_ids.sort_unstable(); - let fragments = dataset.get_frags_from_ordered_ids(&ordered_ids); - if let Some(missing_id) = fragments - .iter() - .zip(ordered_ids.iter()) - .find_map(|(fragment, fragment_id)| fragment.is_none().then_some(*fragment_id)) - { - return Err(Error::invalid_input(format!( - "Unknown fragment id {missing_id} in training fragment filter" - ))); - } - Ok(fragments - .into_iter() - .map(|fragment| fragment.unwrap().metadata().clone()) - .collect()) + dataset.get_fragment_metadata_from_ids(fragment_ids) } /// Build a FixedSizeListArray from raw flat value bytes. @@ -721,6 +722,7 @@ async fn sample_nullable_fsl( byte_width: usize, vector_field: &lance_core::datatypes::Field, mut scan: S, + still_needed: Option>, ) -> Result where S: Stream> + Unpin, @@ -731,6 +733,12 @@ where let mut rows_scanned: usize = 0; while num_non_null < sample_size_hint { + let remaining_rows = sample_size_hint - num_non_null; + // A fragment-limited producer sizes its next prefetch round to this + // outstanding demand. + if let Some(still_needed) = &still_needed { + still_needed.store(remaining_rows, Ordering::Relaxed); + } let Some(batch) = scan.next().await else { break; }; @@ -750,7 +758,16 @@ where continue; } let previous_num_non_null = num_non_null; - accumulate_fsl_values(&mut values_buf, &mut num_non_null, &array, byte_width, true)?; + // `remaining_rows` keeps `values_buf` within its pre-allocated + // `sample_size_hint * byte_width` capacity. + accumulate_fsl_values( + &mut values_buf, + &mut num_non_null, + &array, + byte_width, + true, + remaining_rows, + )?; info!( "Sample training data: batch {} read {} rows, accepted {} rows ({} scanned, {}/{} sampled after null filtering)", batch_count, @@ -762,6 +779,11 @@ where ); } + // Zero the demand so any further poll of the producer terminates instead + // of reading another round. + if let Some(still_needed) = &still_needed { + still_needed.store(0, Ordering::Relaxed); + } let num_rows_out = num_non_null.min(sample_size_hint); values_buf.truncate(num_rows_out * byte_width); @@ -797,7 +819,14 @@ async fn sample_fsl_uniform( for (chunk_idx, chunk) in indices.chunks(TAKE_CHUNK_SIZE).enumerate() { let batch = dataset.take(chunk, projection.clone()).await?; let array = get_column_from_batch(&batch, column)?; - accumulate_fsl_values(&mut values_buf, &mut total_rows, &array, byte_width, false)?; + accumulate_fsl_values( + &mut values_buf, + &mut total_rows, + &array, + byte_width, + false, + usize::MAX, + )?; info!( "Sample training data: batch {}/{} read {} rows ({}/{} sampled by uniform random sampling)", chunk_idx + 1, @@ -821,13 +850,20 @@ async fn sample_fsl_uniform( /// When `filter_nulls` is false and there are no nulls, copies raw bytes /// directly from the FSL values buffer (accounting for child array offset). /// When `filter_nulls` is true, uses Arrow's `filter` kernel to remove nulls. +/// At most `max_rows` rows are appended so callers can stop copying once their +/// sample is full; otherwise one oversized source batch can grow `values_buf` +/// far beyond the intended cap. fn accumulate_fsl_values( values_buf: &mut MutableBuffer, num_rows: &mut usize, array: &ArrayRef, byte_width: usize, filter_nulls: bool, + max_rows: usize, ) -> Result<()> { + if max_rows == 0 { + return Ok(()); + } let needs_filter = filter_nulls && array.null_count() > 0; if needs_filter { @@ -835,21 +871,29 @@ fn accumulate_fsl_values( let mask = arrow_array::BooleanArray::from(nulls.inner().clone()); let filtered = arrow::compute::filter(array, &mask)?; let fsl = filtered.as_fixed_size_list(); + let take = fsl.len().min(max_rows); + if take == 0 { + return Ok(()); + } let values_data = fsl.values().to_data(); - let value_bytes = &values_data.buffers()[0].as_slice()[..fsl.len() * byte_width]; + let value_bytes = &values_data.buffers()[0].as_slice()[..take * byte_width]; values_buf.extend_from_slice(value_bytes); - *num_rows += fsl.len(); + *num_rows += take; } else { // No nulls: copy raw bytes directly, accounting for child array offset. let fsl = array.as_fixed_size_list(); + let take = fsl.len().min(max_rows); + if take == 0 { + return Ok(()); + } let values = fsl.values(); let values_data = values.to_data(); let elem_size = byte_width / fsl.value_length() as usize; let offset_bytes = values_data.offset() * elem_size; - let total_bytes = fsl.len() * byte_width; + let total_bytes = take * byte_width; let buf = &values_data.buffers()[0].as_slice()[offset_bytes..offset_bytes + total_bytes]; values_buf.extend_from_slice(buf); - *num_rows += fsl.len(); + *num_rows += take; } Ok(()) } @@ -862,6 +906,7 @@ async fn sample_nullable_fallback( sample_size_hint: usize, is_nullable: bool, mut scan: S, + still_needed: Option>, ) -> Result where S: Stream> + Unpin, @@ -873,6 +918,12 @@ where let mut rows_scanned: usize = 0; while num_non_null < sample_size_hint { + let remaining_rows = sample_size_hint - num_non_null; + // A fragment-limited producer sizes its next prefetch round to this + // outstanding demand. + if let Some(still_needed) = &still_needed { + still_needed.store(remaining_rows, Ordering::Relaxed); + } let Some(batch) = scan.next().await else { break; }; @@ -898,7 +949,14 @@ where } else { batch }; - let accepted_rows = batch.num_rows(); + // Slicing to the outstanding demand keeps the retained batches, and + // the post-loop `concat_batches`, bounded by the sample size. + let accepted_rows = batch.num_rows().min(remaining_rows); + let batch = if accepted_rows < batch.num_rows() { + batch.slice(0, accepted_rows) + } else { + batch + }; num_non_null += accepted_rows; info!( "Sample training data (fallback): batch {} read {} rows, accepted {} rows ({} scanned, {}/{} sampled)", @@ -912,6 +970,12 @@ where filtered.push(batch); } + // Zero the demand so any further poll of the producer terminates instead + // of reading another round. + if let Some(still_needed) = &still_needed { + still_needed.store(0, Ordering::Relaxed); + } + let Some(schema) = schema else { return Err(Error::index("No non-null training data found".to_string())); }; @@ -1040,6 +1104,7 @@ mod tests { use crate::dataset::InsertBuilder; use arrow_array::{ArrayRef, Float32Array, types::Float32Type}; + use arrow_buffer::{BooleanBufferBuilder, NullBuffer}; use arrow_schema::{DataType, Field}; use lance_arrow::FixedSizeListArrayExt; use lance_datagen::{ArrayGeneratorExt, Dimension, RowCount, array, gen_batch}; @@ -1201,7 +1266,15 @@ mod tests { let mut buf = MutableBuffer::new(0); let mut num_rows = 0usize; let sliced_ref: ArrayRef = Arc::new(sliced); - accumulate_fsl_values(&mut buf, &mut num_rows, &sliced_ref, byte_width, false).unwrap(); + accumulate_fsl_values( + &mut buf, + &mut num_rows, + &sliced_ref, + byte_width, + false, + usize::MAX, + ) + .unwrap(); assert_eq!(num_rows, 4); let result: &[f32] = @@ -1327,4 +1400,140 @@ mod tests { let result = count_rows(&dataset, Some(&[ids[2], ids[0]])).await.unwrap(); assert_eq!(result, 250); } + + /// Nullable FSL with fragment-limited sampling must fill the requested sample + /// size when enough non-null rows exist, and terminate cleanly when all + /// selected rows are null. + #[tokio::test] + async fn test_maybe_sample_training_data_fsl_nullable_fragment_limited() { + let nrows: usize = 2000; + let dims: u32 = 8; + let sample_size: usize = 500; + + for (case, null_probability, expected_len) in + [("partial_nulls", 0.5, sample_size), ("all_nulls", 1.0, 0)] + { + let col_gen = array::rand_vec::(Dimension::from(dims)) + .with_random_nulls(null_probability); + let data = gen_batch() + .col("vec", col_gen) + .into_batch_rows(RowCount::from(nrows as u64)) + .unwrap(); + + let dataset = InsertBuilder::new("memory://") + .execute(vec![data]) + .await + .unwrap(); + + let fragment_ids: Vec = dataset + .get_fragments() + .iter() + .map(|f| f.id() as u32) + .collect(); + + let training_data = + maybe_sample_training_data(&dataset, "vec", sample_size, Some(&fragment_ids)) + .await + .unwrap(); + + assert_eq!(training_data.len(), expected_len, "{case}"); + assert_eq!(training_data.null_count(), 0, "{case}"); + assert_eq!(training_data.value_length(), dims as i32, "{case}"); + } + } + + /// Scan-side regression: each fragment-limited producer round must read at + /// most the consumer's outstanding demand. Driving the producer directly + /// with a fixed `still_needed` and inspecting the raw batch size catches + /// over-reads that a post-truncation output-length check cannot. + #[tokio::test] + async fn test_sample_fragment_scan_round_caps_at_still_needed() { + let nrows: usize = 4000; + let dims: u32 = 8; + let still: usize = 500; + + let col_gen = array::rand_vec::(Dimension::from(dims)).with_random_nulls(0.5); + let data = gen_batch() + .col("vec", col_gen) + .into_batch_rows(RowCount::from(nrows as u64)) + .unwrap(); + + let dataset = InsertBuilder::new("memory://fsl_scan_round_cap_test") + .execute(vec![data]) + .await + .unwrap(); + + let fragment_ids: Vec = dataset + .get_fragments() + .iter() + .map(|f| f.id() as u32) + .collect(); + let num_rows = count_rows(&dataset, Some(&fragment_ids)).await.unwrap(); + + // `still_needed` is left large enough that `num_rows` never bounds the + // round, so the batch size reflects the demand cap and nothing else. + let still_needed = Arc::new(AtomicUsize::new(still)); + let mut scan = sample_training_data_scan_from_fragments( + &dataset, + "vec", + num_rows, + &fragment_ids, + still_needed.clone(), + ) + .unwrap(); + + let batch = scan.next().await.unwrap().unwrap(); + assert!( + batch.num_rows() <= still, + "producer round read {} rows but only {} were outstanding", + batch.num_rows(), + still + ); + } + + #[test] + fn test_accumulate_fsl_values_respects_max_rows() { + let dim: usize = 4; + let total_rows: usize = 100; + let max_rows: usize = 16; + let byte_width = dim * std::mem::size_of::(); + + let values: Vec = (0..total_rows * dim).map(|i| i as f32).collect(); + let fsl = FixedSizeListArray::try_new_from_values(Float32Array::from(values), dim as i32) + .unwrap(); + let arr: ArrayRef = Arc::new(fsl); + + let mut buf = MutableBuffer::new(0); + let mut num_rows = 0usize; + accumulate_fsl_values(&mut buf, &mut num_rows, &arr, byte_width, true, max_rows).unwrap(); + + assert_eq!(num_rows, max_rows); + assert_eq!(buf.len(), max_rows * byte_width); + + let values: Vec = (0..total_rows * dim).map(|i| i as f32).collect(); + let item_field = Arc::new(Field::new("item", DataType::Float32, true)); + + // Every other row is null, leaving 50 non-null rows. + let mut nulls_builder = BooleanBufferBuilder::new(total_rows); + for i in 0..total_rows { + nulls_builder.append(i % 2 == 0); + } + let nulls = NullBuffer::new(nulls_builder.finish()); + + let fsl = FixedSizeListArray::try_new( + item_field, + dim as i32, + Arc::new(Float32Array::from(values)), + Some(nulls), + ) + .unwrap(); + let arr: ArrayRef = Arc::new(fsl); + + let mut buf = MutableBuffer::new(0); + let mut num_rows = 0usize; + accumulate_fsl_values(&mut buf, &mut num_rows, &arr, byte_width, true, max_rows).unwrap(); + + assert_eq!(num_rows, max_rows); + assert_eq!(buf.len(), max_rows * byte_width); + } } diff --git a/rust/lance/src/io/commit.rs b/rust/lance/src/io/commit.rs index 6ac6a0c362f..129d0aa21a0 100644 --- a/rust/lance/src/io/commit.rs +++ b/rust/lance/src/io/commit.rs @@ -20,23 +20,28 @@ //! alternative to [`CommitHandler`]. use std::collections::{HashMap, HashSet}; +use std::future::Future; use std::num::NonZero; use std::sync::Arc; -use std::time::Instant; +use std::time::{Duration, Instant}; use conflict_resolver::TransactionRebase; use lance_core::utils::backoff::{Backoff, SlotBackoff}; +use lance_core::utils::tracing::{AUDIT_MODE_DELETE, AUDIT_TYPE_TRANSACTION, TRACE_FILE_AUDIT}; +#[cfg(test)] use lance_file::version::LanceFileVersion; + use lance_index::metrics::NoOpMetricsCollector; use lance_io::utils::CachedFileSize; use lance_select::RowAddrTreeMap; use lance_table::format::{ - DETACHED_VERSION_MASK, DataStorageFormat, DeletionFile, Fragment, IndexMetadata, Manifest, - WriterVersion, is_detached_version, list_index_files_with_sizes, pb, + DETACHED_VERSION_MASK, DeletionFile, Fragment, IndexMetadata, Manifest, WriterVersion, + is_detached_version, list_index_files_with_sizes, pb, }; use lance_table::io::commit::{ CommitConfig, CommitError, CommitHandler, ManifestLocation, ManifestNamingScheme, }; +use lance_table::io::manifest::read_manifest; use rand::{Rng, rng}; use super::ObjectStore; @@ -56,12 +61,11 @@ use crate::session::Session; use crate::session::caches::DSMetadataCache; use crate::session::index_caches::IndexMetadataKey; use futures::future::Either; -use futures::{StreamExt, TryFutureExt, TryStreamExt}; +use futures::{FutureExt, StreamExt, TryFutureExt, TryStreamExt}; use lance_core::{Error, Result}; use lance_index::is_system_index; use lance_io::object_store::ObjectStoreRegistry; use log; -#[cfg(test)] use object_store::ObjectStoreExt; use object_store::path::Path; use prost::Message; @@ -75,8 +79,36 @@ pub mod namespace_manifest; #[cfg(all(feature = "dynamodb_tests", test))] mod s3_test; +/// Wall-clock budget for conflict retry backoff when callers do not override it. +pub(crate) const DEFAULT_COMMIT_RETRY_TIMEOUT: Duration = Duration::from_secs(30); + +pub(crate) fn timeout_error(retry_timeout: Duration, attempts: u32) -> Error { + Error::too_much_write_contention(format!( + "Attempted {} times, but failed on retry_timeout of {:.3} seconds.", + attempts, + retry_timeout.as_secs_f32() + )) +} + +pub(crate) fn maybe_timeout( + attempt: u32, + start: Instant, + retry_timeout: Duration, + future: impl Future, +) -> impl Future> { + if attempt == 0 { + // The first attempt establishes the observed latency used by SlotBackoff. + Either::Left(future.map(Ok)) + } else { + let remaining = retry_timeout.saturating_sub(start.elapsed()); + Either::Right( + tokio::time::timeout(remaining, future) + .map_err(move |_| timeout_error(retry_timeout, attempt + 1)), + ) + } +} + /// Read the transaction data from a transaction file. -#[cfg(test)] pub(crate) async fn read_transaction_file( object_store: &ObjectStore, base_path: &Path, @@ -97,6 +129,11 @@ pub(crate) async fn read_transaction_file( /// Logs a warning on failure rather than propagating the error, since the /// primary operation has already failed and the orphaned file will eventually /// be removed by GC. +/// +/// Callers must only invoke this for attempts whose commit is confirmed to +/// have NOT landed (see [`verify_commit_outcome`]): a landed manifest +/// references its transaction file by path, so deleting it would corrupt the +/// version. async fn cleanup_transaction_file( object_store: &ObjectStore, base_path: &Path, @@ -109,12 +146,174 @@ async fn cleanup_transaction_file( .clone() .join(TRANSACTIONS_DIR) .join(transaction_file); - if let Err(e) = object_store.delete(&path).await { - log::warn!( - "Failed to clean up orphaned transaction file '{}': {}", - transaction_file, - e - ); + match object_store.delete(&path).await { + Ok(()) => { + tracing::info!( + target: TRACE_FILE_AUDIT, + mode = AUDIT_MODE_DELETE, + r#type = AUDIT_TYPE_TRANSACTION, + path = transaction_file, + ); + } + Err(e) => { + log::warn!( + "Failed to clean up orphaned transaction file '{}': {}", + transaction_file, + e + ); + } + } +} + +/// Who owns the manifest at a version, checked after a failed commit attempt. +#[derive(Debug)] +enum CommitOutcome { + /// The manifest at the version is the one this attempt wrote: the commit + /// actually landed even though the store reported a failure (e.g. the + /// response to a successful conditional PUT was lost and an internal + /// retry surfaced "already exists"). + Ours { + manifest: Box, + location: ManifestLocation, + }, + /// A manifest exists at the version and records a different transaction + /// file: another writer definitely won the version. + Foreign, + /// No manifest exists at the version: this attempt definitely did not + /// land. + Absent, + /// The verification reads themselves kept failing; whether the commit + /// landed cannot be determined. Callers must not run destructive cleanup + /// in this state. + Unknown, +} + +/// Maximum verification read attempts in [`verify_commit_outcome`]. +const COMMIT_VERIFICATION_ATTEMPTS: u32 = 3; + +/// Determine whether a failed commit attempt actually landed, by comparing +/// the complete transaction recorded in the manifest at `version` with this +/// attempt's transaction. +/// +/// Never returns an error. Read failures and non-definitive not-found results +/// are retried briefly, then collapse to [`CommitOutcome::Unknown`]. +async fn verify_commit_outcome( + object_store: &ObjectStore, + commit_handler: &dyn CommitHandler, + base_path: &Path, + version: u64, + transaction: &Transaction, +) -> CommitOutcome { + enum VerificationFailure { + NotFound, + Read(Error), + } + + let mut backoff = Backoff::default(); + let failure = loop { + let failure = match try_read_manifest_at(object_store, commit_handler, base_path, version) + .await + { + Ok(Some((manifest, location))) => { + match read_manifest_transaction(object_store, base_path, &manifest, &location).await + { + Ok(Some(committed_transaction)) => { + return if committed_transaction == *transaction { + CommitOutcome::Ours { + manifest: Box::new(manifest), + location, + } + } else { + CommitOutcome::Foreign + }; + } + Ok(None) => return CommitOutcome::Foreign, + Err(error) => VerificationFailure::Read(error), + } + } + Ok(None) if commit_handler.is_version_not_found_definitive() => { + return CommitOutcome::Absent; + } + Ok(None) => VerificationFailure::NotFound, + Err(error) => VerificationFailure::Read(error), + }; + + if backoff.attempt() + 1 >= COMMIT_VERIFICATION_ATTEMPTS { + break failure; + } + tokio::time::sleep(backoff.next_backoff()).await; + }; + + match failure { + VerificationFailure::NotFound => { + log::warn!( + "The manifest for version {} was not visible after {} commit verification \ + attempts, and the commit handler does not guarantee definitive not-found \ + results; treating the commit status as unknown", + version, + COMMIT_VERIFICATION_ATTEMPTS + ); + CommitOutcome::Unknown + } + VerificationFailure::Read(error) => { + log::warn!( + "Could not verify the outcome of the commit attempt for version {} after {} \ + tries; treating the commit status as unknown: {}", + version, + COMMIT_VERIFICATION_ATTEMPTS, + error + ); + CommitOutcome::Unknown + } + } +} + +async fn read_manifest_transaction( + object_store: &ObjectStore, + base_path: &Path, + manifest: &Manifest, + location: &ManifestLocation, +) -> Result> { + if let Some(position) = manifest.transaction_section { + let reader = if let Some(size) = location.size { + object_store + .open_with_size(&location.path, size as usize) + .await? + } else { + object_store.open(&location.path).await? + }; + let transaction: pb::Transaction = + lance_io::utils::read_message(reader.as_ref(), position).await?; + Transaction::try_from(transaction).map(Some) + } else if let Some(transaction_file) = manifest.transaction_file.as_deref() { + read_transaction_file(object_store, base_path, transaction_file) + .await + .map(Some) + } else { + Ok(None) + } +} + +/// Read the manifest at `version`, distinguishing "no such version" +/// (`Ok(None)`) from transient read failures (`Err`). +async fn try_read_manifest_at( + object_store: &ObjectStore, + commit_handler: &dyn CommitHandler, + base_path: &Path, + version: u64, +) -> Result> { + let location = match commit_handler + .resolve_version_location(base_path, version, &object_store.inner) + .await + { + Ok(location) => location, + Err(Error::NotFound { .. }) => return Ok(None), + Err(e) => return Err(e), + }; + match read_manifest(object_store, &location.path, location.size).await { + Ok(manifest) => Ok(Some((manifest, location))), + Err(Error::NotFound { .. }) => Ok(None), + Err(e) => Err(e), } } @@ -122,7 +321,7 @@ async fn cleanup_transaction_file( pub(crate) async fn write_transaction_file( object_store: &ObjectStore, base_path: &Path, - transaction: &Transaction, + transaction: &pb::Transaction, ) -> Result { let file_name = format!("{}-{}.txn", transaction.read_version, transaction.uuid); let path = base_path @@ -130,16 +329,24 @@ pub(crate) async fn write_transaction_file( .join(TRANSACTIONS_DIR) .join(file_name.as_str()); - let message = pb::Transaction::from(transaction); - let buf = message.encode_to_vec(); + let buf = transaction.encode_to_vec(); object_store.put(&path, &buf).await?; Ok(file_name) } +/// Transactions serialized above this size are not inlined into the manifest. +#[cfg(not(test))] +pub(crate) const MAX_INLINE_TRANSACTION_BYTES: usize = 20 * 1024 * 1024; +/// Smaller threshold for unit tests so spill coverage does not need +/// multi-megabyte payloads. +#[cfg(test)] +pub(crate) const MAX_INLINE_TRANSACTION_BYTES: usize = 64 * 1024; + #[allow(clippy::too_many_arguments)] async fn do_commit_new_dataset( object_store: &ObjectStore, + source_store: Option<&ObjectStore>, commit_handler: &dyn CommitHandler, base_path: &Path, transaction: &Transaction, @@ -148,8 +355,11 @@ async fn do_commit_new_dataset( metadata_cache: &DSMetadataCache, store_registry: Arc, ) -> Result<(Manifest, ManifestLocation)> { + let pb_transaction = pb::Transaction::from(transaction); + let inline_transaction = pb_transaction.encoded_len() <= MAX_INLINE_TRANSACTION_BYTES; + let transaction_file = if !write_config.disable_transaction_file() { - write_transaction_file(object_store, base_path, transaction).await? + write_transaction_file(object_store, base_path, &pb_transaction).await? } else { String::new() }; @@ -163,15 +373,19 @@ async fn do_commit_new_dataset( .. } = &transaction.operation { + // The source manifest must be read through the source store, which may differ + // from the destination store when cloning across object stores/accounts. Falls + // back to the destination store for same-store clones. + let source_store = source_store.unwrap_or(object_store); let source_base_path = ObjectStore::extract_path_from_uri(store_registry, ref_path.as_str())?; let source_manifest_location = commit_handler - .resolve_version_location(&source_base_path, *ref_version, &object_store.inner) + .resolve_version_location(&source_base_path, *ref_version, &source_store.inner) .await?; let source_manifest = Dataset::load_manifest( - object_store, + source_store, &source_manifest_location, - base_path.to_string().as_str(), + ref_path.as_str(), &Session::default(), ) .await?; @@ -192,7 +406,7 @@ async fn do_commit_new_dataset( ); let updated_indices = if let Some(index_section_pos) = source_manifest.index_section { - let reader = object_store.open(&source_manifest_location.path).await?; + let reader = source_store.open(&source_manifest_location.path).await?; let section: pb::IndexSection = lance_io::utils::read_message(reader.as_ref(), index_section_pos).await?; section @@ -215,9 +429,11 @@ async fn do_commit_new_dataset( new_manifest.branch = None; new_manifest.tag = None; new_manifest.index_section = None; // will be rewritten below + new_manifest.transaction_file = + (!transaction_file.is_empty()).then_some(transaction_file.clone()); let mut new_frags = new_manifest.fragments.as_ref().clone(); for f in &mut new_frags { - for df in &mut f.files { + for df in f.referenced_lance_files_mut() { df.base_id = None; } if let Some(d) = f.deletion_file.as_mut() { @@ -229,7 +445,7 @@ async fn do_commit_new_dataset( // Indices: keep metadata but normalize base to local let mut updated_indices = Vec::new(); if let Some(index_section_pos) = source_manifest.index_section { - let reader = object_store.open(&source_manifest_location.path).await?; + let reader = source_store.open(&source_manifest_location.path).await?; let section: pb::IndexSection = lance_io::utils::read_message(reader.as_ref(), index_section_pos).await?; updated_indices = section @@ -262,7 +478,7 @@ async fn do_commit_new_dataset( }, write_config, manifest_naming_scheme, - Some(transaction), + inline_transaction.then(|| pb_transaction.into()), ) .await; @@ -270,36 +486,121 @@ async fn do_commit_new_dataset( // if there is a conflict. match result { Ok(manifest_location) => { - let tx_key = crate::session::caches::TransactionKey { - version: manifest.version, - }; - metadata_cache - .insert_with_key(&tx_key, Arc::new(transaction.clone())) - .await; - - let manifest_key = crate::session::caches::ManifestKey { - version: manifest_location.version, - e_tag: manifest_location.e_tag.as_deref(), - }; - metadata_cache - .insert_with_key(&manifest_key, Arc::new(manifest.clone())) + record_new_dataset_commit(metadata_cache, transaction, &manifest, &manifest_location) .await; Ok((manifest, manifest_location)) } Err(CommitError::CommitConflict) => { + // The dataset may "already exist" because this attempt's own + // manifest write landed but returned an ambiguous error. Verify + // before reporting a conflict (and before deleting the + // transaction file a landed manifest would reference). + match verify_commit_outcome( + object_store, + commit_handler, + base_path, + manifest.version, + transaction, + ) + .await + { + CommitOutcome::Ours { + manifest: committed_manifest, + location, + } => { + let committed_manifest = *committed_manifest; + record_new_dataset_commit( + metadata_cache, + transaction, + &committed_manifest, + &location, + ) + .await; + return Ok((committed_manifest, location)); + } + CommitOutcome::Foreign | CommitOutcome::Absent => {} + CommitOutcome::Unknown => { + return Err(Error::commit_status_unknown_source( + manifest.version, + "dataset creation reported a conflict but the manifest could not \ + be read back for verification" + .to_string() + .into(), + )); + } + } cleanup_transaction_file(object_store, base_path, &transaction_file).await; Err(crate::Error::dataset_already_exists(base_path.to_string())) } Err(CommitError::OtherError(err)) => { + match verify_commit_outcome( + object_store, + commit_handler, + base_path, + manifest.version, + transaction, + ) + .await + { + CommitOutcome::Ours { + manifest: committed_manifest, + location, + } => { + let committed_manifest = *committed_manifest; + record_new_dataset_commit( + metadata_cache, + transaction, + &committed_manifest, + &location, + ) + .await; + if commit_handler.propagate_commit_error_after_success() { + return Err(err); + } + return Ok((committed_manifest, location)); + } + CommitOutcome::Foreign | CommitOutcome::Absent => {} + CommitOutcome::Unknown => { + return Err(Error::commit_status_unknown_source( + manifest.version, + Box::new(err), + )); + } + } cleanup_transaction_file(object_store, base_path, &transaction_file).await; Err(err) } } } +/// Cache bookkeeping for a successful new-dataset commit, shared by the +/// direct-success and verified-own-commit paths of `do_commit_new_dataset`. +async fn record_new_dataset_commit( + metadata_cache: &DSMetadataCache, + transaction: &Transaction, + manifest: &Manifest, + location: &ManifestLocation, +) { + let tx_key = crate::session::caches::TransactionKey { + version: manifest.version, + }; + metadata_cache + .insert_with_key(&tx_key, Arc::new(transaction.clone())) + .await; + + let manifest_key = crate::session::caches::ManifestKey { + version: location.version, + e_tag: location.e_tag.as_deref(), + }; + metadata_cache + .insert_with_key(&manifest_key, Arc::new(manifest.clone())) + .await; +} + #[allow(clippy::too_many_arguments)] pub(crate) async fn commit_new_dataset( object_store: &ObjectStore, + source_store: Option<&ObjectStore>, commit_handler: &dyn CommitHandler, base_path: &Path, transaction: &Transaction, @@ -310,6 +611,7 @@ pub(crate) async fn commit_new_dataset( ) -> Result<(Manifest, ManifestLocation)> { do_commit_new_dataset( object_store, + source_store, commit_handler, base_path, transaction, @@ -368,98 +670,31 @@ async fn migrate_manifest( } fn check_storage_version(manifest: &mut Manifest) -> Result<()> { - let data_storage_version = manifest.data_storage_format.lance_file_version()?; - if manifest.data_storage_format.lance_file_version()? == LanceFileVersion::Legacy { - // Due to bugs in 0.16 it is possible the dataset's data storage version does not - // match the file version. As a result, we need to check and see if they are out - // of sync. - if let Some(actual_file_version) = - Fragment::try_infer_version(&manifest.fragments).map_err(|e| Error::internal(format!( - "The dataset contains a mixture of file versions. You will need to rollback to an earlier version: {}", - e - )))? - && actual_file_version > data_storage_version { - log::warn!( - "Data storage version {} is less than the actual file version {}. This has been automatically updated.", - data_storage_version, - actual_file_version - ); - manifest.data_storage_format = DataStorageFormat::new(actual_file_version); - } - } else { - // Otherwise, if we are on 2.0 or greater, we should ensure that the file versions - // match the data storage version. This is a sanity assertion to prevent data corruption. - if let Some(actual_file_version) = Fragment::try_infer_version(&manifest.fragments)? - && actual_file_version != data_storage_version - { - return Err(Error::internal(format!( - "The operation added files with version {}. However, the data storage version is {}.", - actual_file_version, data_storage_version - ))); - } + crate::dataset::versions::check_manifest_storage_version(manifest) +} + +/// Reject a manifest in which two fragments share an id. Per-fragment state is +/// keyed by fragment id — deletion file paths, cached row id sequences, row +/// addresses — so a duplicate makes it ambiguous which rows that state describes. +/// +/// Runs after the legacy fixups above, so a dataset that needs a rollback for some +/// other reason is diagnosed with that first. Relies on `build_manifest` leaving +/// the fragments sorted by id. +fn check_fragment_ids(manifest: &Manifest) -> Result<()> { + if let Some(pair) = manifest.fragments.windows(2).find(|p| p[0].id == p[1].id) { + return Err(Error::invalid_input(format!( + "The commit would produce two fragments with id {}. Fragment ids must be \ + unique. Datasets written by Lance 0.16 and earlier may already contain \ + duplicate ids; those have to be rewritten, or rolled back to a version \ + without the duplicate.", + pair[0].id + ))); } Ok(()) } fn check_column_indices(manifest: &Manifest) -> Result<()> { - let data_storage_version = manifest.data_storage_format.lance_file_version()?; - if data_storage_version < LanceFileVersion::V2_1 { - return Ok(()); - } - - for fragment in manifest.fragments.iter() { - for data_file in &fragment.files { - if data_file.is_legacy_file() || data_file.column_indices.is_empty() { - continue; - } - if data_file.fields.len() != data_file.column_indices.len() { - return Err(Error::invalid_input(format!( - "Data file '{}' (fragment {}) has {} field ids but {} column indices. \ - These must be the same length.", - data_file.path, - fragment.id, - data_file.fields.len(), - data_file.column_indices.len() - ))); - } - let file_version = LanceFileVersion::try_from_major_minor( - data_file.file_major_version, - data_file.file_minor_version, - )?; - if file_version < LanceFileVersion::V2_1 { - continue; - } - for (field_id, column_index) in - data_file.fields.iter().zip(data_file.column_indices.iter()) - { - // Field ids may not exist in the current schema after schema - // evolution (e.g. cast/drop column). Skip those. - let Some(field) = manifest.schema.field_by_id(*field_id) else { - continue; - }; - let needs_column = field.is_leaf() || field.is_packed_struct() || field.is_blob(); - if needs_column && *column_index == -1 { - return Err(Error::invalid_input(format!( - "Field '{}' (id={}) in data file '{}' (fragment {}) \ - has column_index=-1, but leaf fields, packed structs, \ - and blob fields must have a valid column index in \ - file format 2.1+.", - field.name, field_id, data_file.path, fragment.id - ))); - } - if !needs_column && *column_index != -1 { - return Err(Error::invalid_input(format!( - "Non-leaf field '{}' (id={}) in data file '{}' (fragment {}) \ - has column_index={}, but non-leaf fields should have \ - column_index=-1 in file format 2.1+. Only leaf fields, \ - packed structs, and blob fields should have column indices.", - field.name, field_id, data_file.path, fragment.id, column_index - ))); - } - } - } - } - Ok(()) + crate::dataset::versions::validate_column_indices(manifest) } /// Fix schema in case of duplicate field ids. @@ -489,13 +724,12 @@ fn fix_schema(manifest: &mut Manifest) -> Result<()> { } // Now, we need to remap the field ids to be unique. - let mut field_id_seed = manifest.max_field_id() + 1; let mut old_field_id_mapping: HashMap = HashMap::new(); let mut fields_with_duplicate_ids = fields_with_duplicate_ids.into_iter().collect::>(); fields_with_duplicate_ids.sort_unstable(); - for field_id in fields_with_duplicate_ids { + for (field_id_seed, field_id) in (manifest.max_field_id() + 1..).zip(fields_with_duplicate_ids) + { old_field_id_mapping.insert(field_id, field_id_seed); - field_id_seed += 1; } let mut fragments = manifest.fragments.as_ref().clone(); @@ -675,8 +909,14 @@ fn must_recalculate_fragment_bitmap( /// /// Indices might be missing `fragment_bitmap`, so this function will add it. /// Indices might also be missing `files` (file sizes), so this function will collect them. -async fn migrate_indices(dataset: &Dataset, indices: &mut [IndexMetadata]) -> Result<()> { +/// +/// Returns the logical indices whose `fragment_bitmap` this replaced. Those are +/// the only changes here that alter what an index covers, and the caller has to +/// withdraw MemWAL catch-up for them: this runs after the coverage derivation, +/// and it keeps the segment's UUID. +async fn migrate_indices(dataset: &Dataset, indices: &mut [IndexMetadata]) -> Result> { infer_missing_vector_details(dataset, indices).await; + let mut recovered_coverage = Vec::new(); let needs_recalculating = match detect_overlapping_fragments(indices) { Ok(()) => vec![], Err(BadFragmentBitmapError { bad_indices }) => { @@ -694,7 +934,11 @@ async fn migrate_indices(dataset: &Dataset, indices: &mut [IndexMetadata]) -> Re let idx = dataset .open_generic_index(&idx_field.name, &index.uuid, &NoOpMetricsCollector) .await?; - index.fragment_bitmap = Some(idx.calculate_included_frags().await?); + let recalculated = idx.calculate_included_frags().await?; + if index.fragment_bitmap.as_ref() != Some(&recalculated) { + recovered_coverage.push(index.name.clone()); + } + index.fragment_bitmap = Some(recalculated); } // We can't reliably recalculate the index type for label_list and bitmap indices and so we can't migrate this field. // However, we still log for visibility and to help potentially diagnose issues in the future if we grow to rely on the field. @@ -739,7 +983,7 @@ async fn migrate_indices(dataset: &Dataset, indices: &mut [IndexMetadata]) -> Re } } - Ok(()) + Ok(recovered_coverage) } pub(crate) struct BadFragmentBitmapError { @@ -783,17 +1027,27 @@ pub(crate) async fn do_commit_detached_transaction( transaction: &Transaction, write_config: &ManifestWriteConfig, commit_config: &CommitConfig, + retry_timeout: Duration, ) -> Result<(Manifest, ManifestLocation)> { + let pb_transaction = pb::Transaction::from(transaction); + let inline_transaction = pb_transaction.encoded_len() <= MAX_INLINE_TRANSACTION_BYTES; + // We don't strictly need a transaction file but we go ahead and create one for // record-keeping if nothing else. let transaction_file = if !write_config.disable_transaction_file() { - write_transaction_file(object_store, &dataset.base, transaction).await? + write_transaction_file(object_store, &dataset.base, &pb_transaction).await? } else { String::new() }; + // The inline copy is moved into the first commit attempt; a retry (only on + // a random-version collision) rebuilds it instead of cloning up front. + let mut inline_tx: Option = + inline_transaction.then(|| pb_transaction.into()); + // We still do a loop since we may have conflicts in the random version we pick let mut backoff = Backoff::default(); + let start = Instant::now(); while backoff.attempt() < commit_config.num_retries { // Pick a random u64 with the highest bit set to indicate it is detached let random_version = rng().random::() | DETACHED_VERSION_MASK; @@ -828,7 +1082,15 @@ pub(crate) async fn do_commit_detached_transaction( fix_schema(&mut manifest)?; check_storage_version(&mut manifest)?; check_column_indices(&manifest)?; - migrate_indices(dataset, &mut indices).await?; + check_fragment_ids(&manifest)?; + // Runs after the coverage derivation and can replace a fragment bitmap + // while keeping its UUID, so anything it narrowed loses its position. + let recovered_coverage = migrate_indices(dataset, &mut indices).await?; + Transaction::withdraw_coverage_invalidated_after_build( + &mut indices, + &recovered_coverage, + manifest.version, + )?; // Try to commit the manifest let result = write_manifest_file( @@ -843,7 +1105,7 @@ pub(crate) async fn do_commit_detached_transaction( }, write_config, ManifestNamingScheme::V2, - Some(transaction), + inline_tx.take(), ) .await; @@ -852,12 +1114,77 @@ pub(crate) async fn do_commit_detached_transaction( return Ok((manifest, location)); } Err(CommitError::CommitConflict) => { - // We pick a random u64 for the version, so it's possible (though extremely unlikely) - // that we have a conflict. In that case, we just try again. - tokio::time::sleep(backoff.next_backoff()).await; + // Either an (extremely unlikely) random-version collision, or + // our own write landed but returned an ambiguous error. + // Verify before retrying with a new random version. + match verify_commit_outcome( + object_store, + commit_handler, + &dataset.base, + manifest.version, + transaction, + ) + .await + { + CommitOutcome::Ours { + manifest: committed_manifest, + location, + } => { + return Ok((*committed_manifest, location)); + } + CommitOutcome::Foreign | CommitOutcome::Absent => {} + CommitOutcome::Unknown => { + return Err(Error::commit_status_unknown_source( + manifest.version, + "detached commit reported a conflict but the manifest could \ + not be read back for verification" + .to_string() + .into(), + )); + } + } + if start.elapsed() > retry_timeout { + cleanup_transaction_file(object_store, &dataset.base, &transaction_file).await; + return Err(timeout_error(retry_timeout, backoff.attempt() + 1)); + } + let sleep_fut = tokio::time::sleep(backoff.next_backoff()); + if let Err(error) = + maybe_timeout(backoff.attempt(), start, retry_timeout, sleep_fut).await + { + cleanup_transaction_file(object_store, &dataset.base, &transaction_file).await; + return Err(error); + } + // The inline copy was moved into the failed attempt; rebuild + // it for the retry with a new random version. + inline_tx = inline_transaction.then(|| pb::Transaction::from(transaction).into()); } Err(CommitError::OtherError(err)) => { - // If other error, return + match verify_commit_outcome( + object_store, + commit_handler, + &dataset.base, + manifest.version, + transaction, + ) + .await + { + CommitOutcome::Ours { + manifest: committed_manifest, + location, + } => { + if commit_handler.propagate_commit_error_after_success() { + return Err(err); + } + return Ok((*committed_manifest, location)); + } + CommitOutcome::Foreign | CommitOutcome::Absent => {} + CommitOutcome::Unknown => { + return Err(Error::commit_status_unknown_source( + manifest.version, + Box::new(err), + )); + } + } cleanup_transaction_file(object_store, &dataset.base, &transaction_file).await; return Err(err); } @@ -884,6 +1211,7 @@ pub(crate) async fn commit_detached_transaction( transaction: &Transaction, write_config: &ManifestWriteConfig, commit_config: &CommitConfig, + retry_timeout: Duration, ) -> Result<(Manifest, ManifestLocation)> { do_commit_detached_transaction( dataset, @@ -892,6 +1220,7 @@ pub(crate) async fn commit_detached_transaction( transaction, write_config, commit_config, + retry_timeout, ) .await } @@ -910,6 +1239,56 @@ async fn load_and_sort_new_transactions( Ok((new_ds, txns)) } +/// Success-path bookkeeping shared by the direct-success and +/// verified-own-commit paths of [`commit_transaction`]: populate the session +/// caches and run the auto-cleanup hook. +async fn record_successful_commit( + dataset: &Dataset, + transaction: &Transaction, + manifest: &Manifest, + location: &ManifestLocation, + indices: Vec, + skip_auto_cleanup: bool, +) { + let tx_key = crate::session::caches::TransactionKey { + version: manifest.version, + }; + dataset + .metadata_cache + .insert_with_key(&tx_key, Arc::new(transaction.clone())) + .await; + + let manifest_key = crate::session::caches::ManifestKey { + version: location.version, + e_tag: location.e_tag.as_deref(), + }; + dataset + .metadata_cache + .insert_with_key(&manifest_key, Arc::new(manifest.clone())) + .await; + if !indices.is_empty() { + let key = IndexMetadataKey { + version: manifest.version, + store_identity: &dataset.object_store.store_prefix, + }; + dataset + .index_cache + .insert_with_key(&key, Arc::new(indices)) + .await; + } + + if !skip_auto_cleanup { + // Note: We're using the old dataset here (before the new manifest is committed). + // This means cleanup runs based on the previous version's state, which may affect + // which versions are available for cleanup. + match auto_cleanup_hook(dataset, manifest).await { + Ok(Some(stats)) => log::info!("Auto cleanup triggered: {:?}", stats), + Err(e) => log::error!("Error encountered during auto_cleanup_hook: {}", e), + _ => {} + }; + } +} + /// Attempt to commit a transaction, with retries and conflict resolution. #[allow(clippy::too_many_arguments)] pub(crate) async fn commit_transaction( @@ -919,6 +1298,7 @@ pub(crate) async fn commit_transaction( transaction: &Transaction, write_config: &ManifestWriteConfig, commit_config: &CommitConfig, + retry_timeout: Duration, manifest_naming_scheme: ManifestNamingScheme, affected_rows: Option<&RowAddrTreeMap>, ) -> Result<(Manifest, ManifestLocation)> { @@ -943,6 +1323,31 @@ pub(crate) async fn commit_transaction( dataset.clone() }; + // The version this transaction read, captured before the retry loop moves + // `dataset` forward. MemWAL index catch-up is derived from it: an index + // covering every fragment live here holds every row compaction had copied + // in by then. + // + // Only tables on the protocol pay for the extra index load; the bit is on + // the manifest already in hand. + let read_version_dataset = dataset.clone(); + let read_version_indices = if read_version_dataset.manifest.reader_feature_flags + & lance_table::feature_flags::FLAG_MEM_WAL_INDEX_CATCHUP + != 0 + { + // The Arc is kept rather than cloned out: `load_indices` returns shared + // cached data, and this runs on every commit to an activated table. + Some(read_version_dataset.load_indices().await?) + } else { + None + }; + let read_version_state = read_version_indices.as_ref().map(|indices| { + crate::dataset::transaction::ReadVersionState { + manifest: read_version_dataset.manifest.as_ref(), + indices: indices.as_slice(), + } + }); + let mut transaction = transaction.clone(); let num_attempts = std::cmp::max(commit_config.num_retries, 1); @@ -982,8 +1387,13 @@ pub(crate) async fn commit_transaction( transaction = rebase.finish(&dataset).await?; } + // Recomputed every attempt: the rebase above may have rewritten the + // transaction. + let pb_transaction = pb::Transaction::from(&transaction); + let inline_transaction = pb_transaction.encoded_len() <= MAX_INLINE_TRANSACTION_BYTES; + current_transaction_file = if !write_config.disable_transaction_file() { - write_transaction_file(object_store, &dataset.base, &transaction).await? + write_transaction_file(object_store, &dataset.base, &pb_transaction).await? } else { String::new() }; @@ -1009,11 +1419,12 @@ pub(crate) async fn commit_transaction( ) .await? } - _ => transaction.build_manifest( + _ => transaction.build_manifest_with_read_version( Some(dataset.manifest.as_ref()), dataset.load_indices().await?.as_ref().clone(), transaction_file, write_config, + read_version_state, )?, }; @@ -1032,8 +1443,16 @@ pub(crate) async fn commit_transaction( check_storage_version(&mut manifest)?; check_column_indices(&manifest)?; + check_fragment_ids(&manifest)?; - migrate_indices(&dataset, &mut indices).await?; + // Runs after the coverage derivation and can replace a fragment bitmap + // while keeping its UUID, so anything it narrowed loses its position. + let recovered_coverage = migrate_indices(&dataset, &mut indices).await?; + Transaction::withdraw_coverage_invalidated_after_build( + &mut indices, + &recovered_coverage, + target_version, + )?; // Try to commit the manifest let result = write_manifest_file( @@ -1048,52 +1467,70 @@ pub(crate) async fn commit_transaction( }, write_config, manifest_naming_scheme, - Some(&transaction), + inline_transaction.then(|| pb_transaction.into()), ) .await; match result { Ok(manifest_location) => { - // Cache both the transaction file and manifest - let tx_key = crate::session::caches::TransactionKey { - version: target_version, - }; - dataset - .metadata_cache - .insert_with_key(&tx_key, Arc::new(transaction.clone())) - .await; - - let manifest_key = crate::session::caches::ManifestKey { - version: manifest_location.version, - e_tag: manifest_location.e_tag.as_deref(), - }; - dataset - .metadata_cache - .insert_with_key(&manifest_key, Arc::new(manifest.clone())) - .await; - if !indices.is_empty() { - let key = IndexMetadataKey { - version: target_version, - }; - dataset - .index_cache - .insert_with_key(&key, Arc::new(indices)) - .await; - } - - if !commit_config.skip_auto_cleanup { - // Note: We're using the old dataset here (before the new manifest is committed). - // This means cleanup runs based on the previous version's state, which may affect - // which versions are available for cleanup. - match auto_cleanup_hook(&dataset, &manifest).await { - Ok(Some(stats)) => log::info!("Auto cleanup triggered: {:?}", stats), - Err(e) => log::error!("Error encountered during auto_cleanup_hook: {}", e), - _ => {} - }; - } + record_successful_commit( + &dataset, + &transaction, + &manifest, + &manifest_location, + indices, + commit_config.skip_auto_cleanup, + ) + .await; return Ok((manifest, manifest_location)); } Err(CommitError::CommitConflict) => { + // The store may have applied this attempt's write and still + // reported a conflict (e.g. the response to a successful + // conditional PUT was lost and an internal retry saw + // "already exists"). Verify who owns the version before + // treating the attempt as lost: deleting the artifacts of a + // commit that actually landed corrupts the version. + match verify_commit_outcome( + object_store, + commit_handler, + &dataset.base, + target_version, + &transaction, + ) + .await + { + CommitOutcome::Ours { + manifest: committed_manifest, + location, + } => { + let committed_manifest = *committed_manifest; + record_successful_commit( + &dataset, + &transaction, + &committed_manifest, + &location, + indices, + commit_config.skip_auto_cleanup, + ) + .await; + return Ok((committed_manifest, location)); + } + // Confirmed loss: another writer owns the version (or, + // for handlers that detect conflicts before writing, + // the attempt never landed). Proceed with the normal + // rebase-and-retry path. + CommitOutcome::Foreign | CommitOutcome::Absent => {} + CommitOutcome::Unknown => { + return Err(Error::commit_status_unknown_source( + target_version, + "commit reported a conflict but the manifest at the target \ + version could not be read back for verification" + .to_string() + .into(), + )); + } + } let next_attempt_i = backoff.attempt() + 1; if backoff.attempt() == 0 { @@ -1113,16 +1550,63 @@ pub(crate) async fn commit_transaction( ¤t_transaction_file, ) .await; - tokio::time::sleep(backoff.next_backoff()).await; + if start.elapsed() > retry_timeout { + return Err(timeout_error(retry_timeout, backoff.attempt() + 1)); + } + let sleep_fut = tokio::time::sleep(backoff.next_backoff()); + maybe_timeout(backoff.attempt(), start, retry_timeout, sleep_fut).await?; continue; } else { break; } } Err(CommitError::OtherError(err)) => { - cleanup_transaction_file(object_store, &dataset.base, ¤t_transaction_file) - .await; - return Err(err); + match verify_commit_outcome( + object_store, + commit_handler, + &dataset.base, + target_version, + &transaction, + ) + .await + { + CommitOutcome::Ours { + manifest: committed_manifest, + location, + } => { + let committed_manifest = *committed_manifest; + record_successful_commit( + &dataset, + &transaction, + &committed_manifest, + &location, + indices, + commit_config.skip_auto_cleanup, + ) + .await; + if commit_handler.propagate_commit_error_after_success() { + return Err(err); + } + return Ok((committed_manifest, location)); + } + CommitOutcome::Foreign | CommitOutcome::Absent => { + // The attempt certainly did not land; its + // transaction file is orphaned. + cleanup_transaction_file( + object_store, + &dataset.base, + ¤t_transaction_file, + ) + .await; + return Err(err); + } + CommitOutcome::Unknown => { + return Err(Error::commit_status_unknown_source( + target_version, + Box::new(err), + )); + } + } } } } @@ -1150,11 +1634,13 @@ mod tests { use lance_core::datatypes::{Field, Schema}; use lance_core::utils::tempfile::TempStrDir; use lance_datagen::{BatchCount, RowCount, array, gen_batch}; + use lance_file::version::ConcreteFileVersion; use lance_index::IndexType; use lance_linalg::distance::MetricType; use lance_table::format::{DataFile, DataStorageFormat}; use lance_table::io::commit::{ CommitLease, CommitLock, ManifestWriter, RenameCommitHandler, UnsafeCommitHandler, + commit_handler_from_url, }; use lance_testing::datagen::generate_random_array; @@ -1302,9 +1788,13 @@ mod tests { Some("hello world".to_string()), ); - let file_name = write_transaction_file(&object_store, &base_path, &transaction) - .await - .unwrap(); + let file_name = write_transaction_file( + &object_store, + &base_path, + &pb::Transaction::from(&transaction), + ) + .await + .unwrap(); let read_transaction = read_transaction_file(&object_store, &base_path, &file_name) .await .unwrap(); @@ -1691,6 +2181,7 @@ mod tests { DataFile::new_legacy_from_fields("path1", vec![0, 1, 2], None), DataFile::new_legacy_from_fields("unused", vec![9], None), ], + overlays: vec![], deletion_file: None, row_id_meta: None, physical_rows: None, @@ -1703,6 +2194,7 @@ mod tests { DataFile::new_legacy_from_fields("path2", vec![0, 1, 2], None), DataFile::new_legacy_from_fields("path3", vec![2], None), ], + overlays: vec![], deletion_file: None, row_id_meta: None, physical_rows: None, @@ -1740,6 +2232,7 @@ mod tests { vec![0, 1, 10], None, )], + overlays: vec![], deletion_file: None, row_id_meta: None, physical_rows: None, @@ -1752,6 +2245,7 @@ mod tests { DataFile::new_legacy_from_fields("path2", vec![0, 1, 2], None), DataFile::new_legacy_from_fields("path3", vec![10], None), ], + overlays: vec![], deletion_file: None, row_id_meta: None, physical_rows: None, @@ -1770,6 +2264,10 @@ mod tests { #[async_trait::async_trait] impl CommitHandler for FailingCommitHandler { + fn is_version_not_found_definitive(&self) -> bool { + true + } + async fn commit( &self, _manifest: &mut Manifest, @@ -1833,6 +2331,372 @@ mod tests { extra = txn_files_after.saturating_sub(txn_files_before), ); } + + #[tokio::test] + async fn test_cos_commit_failure_preserves_error_and_cleans_up_transaction() { + let tmp = TempStrDir::default(); + let uri = tmp.as_str(); + let schema = simple_schema(); + let batch = simple_batch(&schema, vec![1, 2, 3]); + let commit_handler = commit_handler_from_url("cos://bucket/dataset", &None) + .await + .unwrap(); + let params = WriteParams { + commit_handler: Some(commit_handler), + ..Default::default() + }; + + let error = Dataset::write( + RecordBatchIterator::new(vec![Ok(batch)], schema), + uri, + Some(params), + ) + .await + .expect_err("the default Tencent COS handler should reject writes"); + + assert!( + matches!(&error, Error::NotSupported { .. }), + "expected NotSupported, got: {error:?}" + ); + assert!( + error.to_string().contains("distributed commit_lock"), + "unexpected error: {error}" + ); + assert_eq!( + count_txn_files(uri), + 0, + "a definitively failed COS commit must clean up its transaction file" + ); + } + + fn simple_batch(schema: &Arc, values: Vec) -> RecordBatch { + RecordBatch::try_new(schema.clone(), vec![Arc::new(Int32Array::from(values))]).unwrap() + } + + fn simple_schema() -> Arc { + Arc::new(ArrowSchema::new(vec![ArrowField::new( + "x", + DataType::Int32, + false, + )])) + } + + /// A commit whose manifest lands but is reported as a conflict (the + /// incident shape: successful conditional PUT, response lost, internal + /// retry sees "already exists") must be recognized as our own commit and + /// returned as success — with the rows appearing exactly once and the + /// transaction file left in place. + #[tokio::test] + async fn test_commit_succeeds_when_conflict_is_own_commit() { + use crate::utils::test::{AmbiguousCommitHandler, AmbiguousFailure}; + + let tmp = TempStrDir::default(); + let uri = tmp.as_str(); + let schema = simple_schema(); + let handler = Arc::new(AmbiguousCommitHandler::default()); + + let params = WriteParams { + commit_handler: Some(handler.clone()), + ..Default::default() + }; + let reader = RecordBatchIterator::new( + vec![Ok(simple_batch(&schema, vec![1, 2, 3]))], + schema.clone(), + ); + Dataset::write(reader, uri, Some(params)).await.unwrap(); + assert_eq!( + handler.resolve_calls(), + 0, + "an uncontended commit must not perform verification reads" + ); + let txn_files_before = count_txn_files(uri); + + handler.fail_next(AmbiguousFailure::LandAndConflict); + let params = WriteParams { + mode: WriteMode::Append, + commit_handler: Some(handler.clone()), + ..Default::default() + }; + let reader = RecordBatchIterator::new( + vec![Ok(simple_batch(&schema, vec![4, 5, 6]))], + schema.clone(), + ); + let ds = Dataset::write(reader, uri, Some(params)) + .await + .expect("a conflict with our own landed commit must be reported as success"); + + assert_eq!(ds.version().version, 2); + assert_eq!( + ds.count_rows(None).await.unwrap(), + 6, + "rows must appear exactly once (no duplicate re-commit)" + ); + assert_eq!( + count_txn_files(uri), + txn_files_before + 1, + "the landed commit's transaction file is referenced by the manifest and must survive" + ); + + // A fresh reader sees the committed version. + let ds2 = Dataset::open(uri).await.unwrap(); + assert_eq!(ds2.version().version, 2); + assert_eq!(ds2.count_rows(None).await.unwrap(), 6); + } + + /// Same as above, but the landed commit is reported as a plain I/O error + /// (e.g. the store's retries all returned 5xx while the first attempt had + /// landed). Verification must still recognize the commit as ours. + #[tokio::test] + async fn test_commit_succeeds_when_landed_with_other_error() { + use crate::utils::test::{AmbiguousCommitHandler, AmbiguousFailure}; + + let tmp = TempStrDir::default(); + let uri = tmp.as_str(); + let schema = simple_schema(); + let handler = Arc::new(AmbiguousCommitHandler::default()); + + let params = WriteParams { + commit_handler: Some(handler.clone()), + ..Default::default() + }; + let reader = RecordBatchIterator::new( + vec![Ok(simple_batch(&schema, vec![1, 2, 3]))], + schema.clone(), + ); + Dataset::write(reader, uri, Some(params)).await.unwrap(); + let txn_files_before = count_txn_files(uri); + + handler.fail_next(AmbiguousFailure::LandAndError); + let params = WriteParams { + mode: WriteMode::Append, + commit_handler: Some(handler.clone()), + ..Default::default() + }; + let reader = RecordBatchIterator::new( + vec![Ok(simple_batch(&schema, vec![4, 5, 6]))], + schema.clone(), + ); + let ds = Dataset::write(reader, uri, Some(params)) + .await + .expect("an errored commit that actually landed must be reported as success"); + + assert_eq!(ds.version().version, 2); + assert_eq!(ds.count_rows(None).await.unwrap(), 6); + assert_eq!(count_txn_files(uri), txn_files_before + 1); + } + + #[tokio::test] + async fn test_commit_retries_temporarily_invisible_manifest() { + use crate::utils::test::{AmbiguousCommitHandler, AmbiguousFailure}; + + let tmp = TempStrDir::default(); + let uri = tmp.as_str(); + let schema = simple_schema(); + let handler = Arc::new(AmbiguousCommitHandler::default()); + let params = WriteParams { + commit_handler: Some(handler.clone()), + ..Default::default() + }; + let reader = RecordBatchIterator::new( + vec![Ok(simple_batch(&schema, vec![1, 2, 3]))], + schema.clone(), + ); + Dataset::write(reader, uri, Some(params)).await.unwrap(); + + handler.fail_next(AmbiguousFailure::LandAndError); + handler.fail_next_resolves_with_not_found(2); + let calls_before = handler.resolve_calls(); + let params = WriteParams { + mode: WriteMode::Append, + commit_handler: Some(handler.clone()), + ..Default::default() + }; + let reader = + RecordBatchIterator::new(vec![Ok(simple_batch(&schema, vec![4, 5, 6]))], schema); + let dataset = Dataset::write(reader, uri, Some(params)) + .await + .expect("verification must retry a non-definitive NotFound result"); + + assert_eq!(handler.resolve_calls() - calls_before, 3); + assert_eq!(dataset.count_rows(None).await.unwrap(), 6); + } + + #[tokio::test] + async fn test_commit_verifies_inline_transaction_without_transaction_file() { + use crate::utils::test::{AmbiguousCommitHandler, AmbiguousFailure}; + + let tmp = TempStrDir::default(); + let uri = tmp.as_str(); + let schema = simple_schema(); + let handler = Arc::new(AmbiguousCommitHandler::default()); + let params = WriteParams { + commit_handler: Some(handler.clone()), + ..Default::default() + }; + let reader = + RecordBatchIterator::new(vec![Ok(simple_batch(&schema, vec![1, 2, 3]))], schema); + let dataset = Dataset::write(reader, uri, Some(params)).await.unwrap(); + let transaction = Transaction::new( + dataset.version().version, + Operation::Append { fragments: vec![] }, + None, + ); + let write_config = ManifestWriteConfig::default().with_transaction_file_disabled(); + + handler.fail_next(AmbiguousFailure::LandAndConflict); + let (manifest, _) = commit_transaction( + &dataset, + dataset.object_store.as_ref(), + handler.as_ref(), + &transaction, + &write_config, + &CommitConfig::default(), + DEFAULT_COMMIT_RETRY_TIMEOUT, + dataset.manifest_location.naming_scheme, + None, + ) + .await + .expect("the inline transaction must identify the landed commit"); + + assert_eq!(manifest.version, 2); + assert!(manifest.transaction_file.is_none()); + assert!(manifest.transaction_section.is_some()); + } + + /// A commit that errors without landing keeps today's behavior: + /// verification finds no manifest at the target version, the original + /// error propagates (not status-unknown), and the orphaned transaction + /// file is cleaned up. + #[tokio::test] + async fn test_commit_definite_failure_cleans_up_and_keeps_error() { + use crate::utils::test::{AmbiguousCommitHandler, AmbiguousFailure}; + + let tmp = TempStrDir::default(); + let uri = tmp.as_str(); + let schema = simple_schema(); + let handler = Arc::new(AmbiguousCommitHandler::default()); + + let params = WriteParams { + commit_handler: Some(handler.clone()), + ..Default::default() + }; + let reader = RecordBatchIterator::new( + vec![Ok(simple_batch(&schema, vec![1, 2, 3]))], + schema.clone(), + ); + Dataset::write(reader, uri, Some(params)).await.unwrap(); + let txn_files_before = count_txn_files(uri); + + handler.fail_next(AmbiguousFailure::FailOutright); + let params = WriteParams { + mode: WriteMode::Append, + commit_handler: Some(handler.clone()), + ..Default::default() + }; + let reader = RecordBatchIterator::new( + vec![Ok(simple_batch(&schema, vec![4, 5, 6]))], + schema.clone(), + ); + let result = Dataset::write(reader, uri, Some(params)).await; + let err = result.expect_err("commit that did not land must fail"); + assert!( + !err.is_commit_status_unknown(), + "a verified-absent commit is a definite failure, got: {:?}", + err + ); + assert_eq!( + count_txn_files(uri), + txn_files_before, + "orphaned transaction file of a definitely-failed commit must be cleaned up" + ); + } + + /// When the commit errors AND verification itself is unavailable, the + /// commit status is unknown: surface `CommitStatusUnknown` and delete + /// nothing. + #[tokio::test] + async fn test_commit_status_unknown_when_verification_unavailable() { + use crate::utils::test::{AmbiguousCommitHandler, AmbiguousFailure}; + + let tmp = TempStrDir::default(); + let uri = tmp.as_str(); + let schema = simple_schema(); + let handler = Arc::new(AmbiguousCommitHandler::default()); + + let params = WriteParams { + commit_handler: Some(handler.clone()), + ..Default::default() + }; + let reader = RecordBatchIterator::new( + vec![Ok(simple_batch(&schema, vec![1, 2, 3]))], + schema.clone(), + ); + Dataset::write(reader, uri, Some(params)).await.unwrap(); + let txn_files_before = count_txn_files(uri); + + // The commit lands but errors, and verification reads fail too. + handler.fail_next(AmbiguousFailure::LandAndError); + handler + .fail_resolve + .store(true, std::sync::atomic::Ordering::SeqCst); + let params = WriteParams { + mode: WriteMode::Append, + commit_handler: Some(handler.clone()), + ..Default::default() + }; + let reader = RecordBatchIterator::new( + vec![Ok(simple_batch(&schema, vec![4, 5, 6]))], + schema.clone(), + ); + let result = Dataset::write(reader, uri, Some(params)).await; + let err = result.expect_err("unknown status must not be reported as success"); + assert!( + err.is_commit_status_unknown(), + "expected CommitStatusUnknown, got: {:?}", + err + ); + assert_eq!( + count_txn_files(uri), + txn_files_before + 1, + "nothing may be deleted while the commit status is unknown" + ); + + // The commit did land: a fresh reader must see a consistent v2. + handler + .fail_resolve + .store(false, std::sync::atomic::Ordering::SeqCst); + let ds = Dataset::open(uri).await.unwrap(); + assert_eq!(ds.version().version, 2); + assert_eq!(ds.count_rows(None).await.unwrap(), 6); + } + + /// Dataset creation whose manifest lands but is reported as a conflict + /// must succeed instead of returning "dataset already exists". + #[tokio::test] + async fn test_create_dataset_succeeds_when_conflict_is_own_commit() { + use crate::utils::test::{AmbiguousCommitHandler, AmbiguousFailure}; + + let tmp = TempStrDir::default(); + let uri = tmp.as_str(); + let schema = simple_schema(); + let handler = Arc::new(AmbiguousCommitHandler::default()); + handler.fail_next(AmbiguousFailure::LandAndConflict); + + let params = WriteParams { + commit_handler: Some(handler.clone()), + ..Default::default() + }; + let reader = RecordBatchIterator::new( + vec![Ok(simple_batch(&schema, vec![1, 2, 3]))], + schema.clone(), + ); + let ds = Dataset::write(reader, uri, Some(params)) + .await + .expect("creation whose commit landed must succeed"); + assert_eq!(ds.version().version, 1); + assert_eq!(ds.count_rows(None).await.unwrap(), 3); + } + /// Helper to build a simple manifest for check_column_indices tests. fn make_manifest_with_file( schema: Schema, @@ -1842,6 +2706,7 @@ mod tests { let fragment = Fragment { id: 0, files: vec![data_file], + overlays: vec![], deletion_file: None, row_id_meta: None, physical_rows: Some(100), @@ -1851,7 +2716,7 @@ mod tests { Manifest::new( schema, Arc::new(vec![fragment]), - DataStorageFormat::new(data_storage_version), + DataStorageFormat::new(data_storage_version.resolve()), HashMap::new(), ) } @@ -1873,7 +2738,14 @@ mod tests { }; // field ids: struct=0, leaf=1; give struct a real column_index (wrong) - let data_file = DataFile::new("data.lance", vec![0, 1], vec![0, 1], 2, 1, None, None); + let data_file = DataFile::new( + "data.lance", + vec![0, 1], + vec![0, 1], + ConcreteFileVersion::V2_1, + None, + None, + ); let manifest = make_manifest_with_file(schema, data_file, LanceFileVersion::V2_1); let result = check_column_indices(&manifest); assert!( @@ -1901,7 +2773,14 @@ mod tests { }; // field ids: list=0, item=1; give list a real column_index (wrong) - let data_file = DataFile::new("data.lance", vec![0, 1], vec![0, 1], 2, 1, None, None); + let data_file = DataFile::new( + "data.lance", + vec![0, 1], + vec![0, 1], + ConcreteFileVersion::V2_1, + None, + None, + ); let manifest = make_manifest_with_file(schema, data_file, LanceFileVersion::V2_1); let result = check_column_indices(&manifest); assert!( @@ -1929,7 +2808,14 @@ mod tests { }; // struct=-1 (correct), leaf=0 (correct) - let data_file = DataFile::new("data.lance", vec![0, 1], vec![-1, 0], 2, 1, None, None); + let data_file = DataFile::new( + "data.lance", + vec![0, 1], + vec![-1, 0], + ConcreteFileVersion::V2_1, + None, + None, + ); let manifest = make_manifest_with_file(schema, data_file, LanceFileVersion::V2_1); assert!(check_column_indices(&manifest).is_ok()); } @@ -1954,7 +2840,14 @@ mod tests { }; // packed struct=0 (allowed), leaf=1 - let data_file = DataFile::new("data.lance", vec![0, 1], vec![0, 1], 2, 1, None, None); + let data_file = DataFile::new( + "data.lance", + vec![0, 1], + vec![0, 1], + ConcreteFileVersion::V2_1, + None, + None, + ); let manifest = make_manifest_with_file(schema, data_file, LanceFileVersion::V2_1); assert!(check_column_indices(&manifest).is_ok()); } @@ -1975,7 +2868,14 @@ mod tests { metadata: Default::default(), }; - let data_file = DataFile::new("data.lance", vec![0, 1], vec![0, 1], 2, 0, None, None); + let data_file = DataFile::new( + "data.lance", + vec![0, 1], + vec![0, 1], + ConcreteFileVersion::V2_0, + None, + None, + ); let manifest = make_manifest_with_file(schema, data_file, LanceFileVersion::V2_0); assert!(check_column_indices(&manifest).is_ok()); } @@ -1992,7 +2892,14 @@ mod tests { }; // 1 field id but 2 column indices - let data_file = DataFile::new("data.lance", vec![0], vec![0, 1], 2, 1, None, None); + let data_file = DataFile::new( + "data.lance", + vec![0], + vec![0, 1], + ConcreteFileVersion::V2_1, + None, + None, + ); let manifest = make_manifest_with_file(schema, data_file, LanceFileVersion::V2_1); let result = check_column_indices(&manifest); assert!(result.is_err(), "Expected error for mismatched lengths"); @@ -2012,7 +2919,14 @@ mod tests { }; // field id 99 does not exist in the schema — should be skipped - let data_file = DataFile::new("data.lance", vec![0, 99], vec![0, 1], 2, 1, None, None); + let data_file = DataFile::new( + "data.lance", + vec![0, 99], + vec![0, 1], + ConcreteFileVersion::V2_1, + None, + None, + ); let manifest = make_manifest_with_file(schema, data_file, LanceFileVersion::V2_1); assert!(check_column_indices(&manifest).is_ok()); } @@ -2034,7 +2948,14 @@ mod tests { }; // struct=-1 (correct), but leaf=-1 (wrong — leaf must have a real column) - let data_file = DataFile::new("data.lance", vec![0, 1], vec![-1, -1], 2, 1, None, None); + let data_file = DataFile::new( + "data.lance", + vec![0, 1], + vec![-1, -1], + ConcreteFileVersion::V2_1, + None, + None, + ); let manifest = make_manifest_with_file(schema, data_file, LanceFileVersion::V2_1); let result = check_column_indices(&manifest); assert!( diff --git a/rust/lance/src/io/commit/conflict_resolver.rs b/rust/lance/src/io/commit/conflict_resolver.rs index d95821dd130..cc61b33bd53 100644 --- a/rust/lance/src/io/commit/conflict_resolver.rs +++ b/rust/lance/src/io/commit/conflict_resolver.rs @@ -7,15 +7,17 @@ use crate::index::mem_wal::{load_mem_wal_index_details, new_mem_wal_index_meta}; use crate::io::deletion::read_dataset_deletion_file; use crate::{ Dataset, - dataset::transaction::{Operation, Transaction, UpdateMode}, + dataset::transaction::{DataOverlayGroup, Operation, Transaction, UpdateMode}, }; use futures::{StreamExt, TryStreamExt}; use lance_core::{Error, Result, utils::deletion::DeletionVector}; use lance_index::frag_reuse::FRAG_REUSE_INDEX_NAME; -use lance_index::mem_wal::{MEM_WAL_INDEX_NAME, MergedGeneration}; +use lance_index::mem_wal::{CompactedSsTable, MEM_WAL_INDEX_NAME}; use lance_select::{RowAddrTreeMap, RowSetOps}; use lance_table::format::IndexMetadata; +use lance_table::format::overlay::OverlayCoverage; use lance_table::{format::Fragment, io::deletion::write_deletion_file}; +use roaring::RoaringBitmap; use std::{ borrow::Cow, collections::{HashMap, HashSet}, @@ -32,9 +34,42 @@ pub struct TransactionRebase<'a> { modified_fragment_ids: HashSet, affected_rows: Option<&'a RowAddrTreeMap>, conflicting_frag_reuse_indices: Vec, - /// Merged generations from conflicting UpdateMemWalState transactions. + /// Compacted SSTables from conflicting UpdateMemWalState transactions. /// Used when rebasing CreateIndex of MemWalIndex. - conflicting_mem_wal_merged_gens: Vec, + conflicting_mem_wal_compacted_sstables: Vec, +} + +/// Whether `operation` may make a nullability-affecting schema change: a +/// projection or merge that does not assert `preserves_nullability`. A +/// tightening projection scanned for nulls at its read version, and a merge +/// may introduce a field that data staged against an earlier schema cannot +/// safely omit; either is falsified by a concurrent value-write. +fn may_alter_nullability(operation: &Operation) -> bool { + matches!( + operation, + Operation::Project { + preserves_nullability: false, + .. + } | Operation::Merge { + preserves_nullability: false, + .. + } + ) +} + +/// Whether `operation` can commit rows that falsify such a change: by writing +/// a null into a scanned field, or by omitting a required column entirely (a +/// stale append's fragments read as null for columns they predate). `Delete` +/// only removes rows and `Rewrite` preserves the values it moves; `Project` +/// already conflicts with projections and merges elsewhere. +fn supplies_values(operation: &Operation) -> bool { + matches!( + operation, + Operation::Append { .. } + | Operation::Update { .. } + | Operation::DataReplacement { .. } + | Operation::DataOverlay { .. } + ) } impl<'a> TransactionRebase<'a> { @@ -60,7 +95,7 @@ impl<'a> TransactionRebase<'a> { initial_fragments: HashMap::new(), modified_fragment_ids: HashSet::new(), conflicting_frag_reuse_indices: Vec::new(), - conflicting_mem_wal_merged_gens: Vec::new(), + conflicting_mem_wal_compacted_sstables: Vec::new(), }), Operation::Delete { updated_fragments, @@ -88,7 +123,7 @@ impl<'a> TransactionRebase<'a> { modified_fragment_ids, affected_rows: None, conflicting_frag_reuse_indices: Vec::new(), - conflicting_mem_wal_merged_gens: Vec::new(), + conflicting_mem_wal_compacted_sstables: Vec::new(), }); } @@ -101,7 +136,7 @@ impl<'a> TransactionRebase<'a> { initial_fragments, modified_fragment_ids, conflicting_frag_reuse_indices: Vec::new(), - conflicting_mem_wal_merged_gens: Vec::new(), + conflicting_mem_wal_compacted_sstables: Vec::new(), }) } Operation::Rewrite { groups, .. } => { @@ -119,7 +154,7 @@ impl<'a> TransactionRebase<'a> { initial_fragments, modified_fragment_ids, conflicting_frag_reuse_indices: Vec::new(), - conflicting_mem_wal_merged_gens: Vec::new(), + conflicting_mem_wal_compacted_sstables: Vec::new(), }) } Operation::DataReplacement { replacements } => { @@ -134,7 +169,22 @@ impl<'a> TransactionRebase<'a> { initial_fragments, modified_fragment_ids, conflicting_frag_reuse_indices: Vec::new(), - conflicting_mem_wal_merged_gens: Vec::new(), + conflicting_mem_wal_compacted_sstables: Vec::new(), + }) + } + Operation::DataOverlay { groups } => { + let modified_fragment_ids = + groups.iter().map(|g| g.fragment_id).collect::>(); + let initial_fragments = + initial_fragments_for_rebase(dataset, &transaction, &modified_fragment_ids) + .await; + Ok(Self { + transaction, + affected_rows, + initial_fragments, + modified_fragment_ids, + conflicting_frag_reuse_indices: Vec::new(), + conflicting_mem_wal_compacted_sstables: Vec::new(), }) } Operation::Merge { fragments, .. } => { @@ -148,7 +198,7 @@ impl<'a> TransactionRebase<'a> { initial_fragments, modified_fragment_ids, conflicting_frag_reuse_indices: Vec::new(), - conflicting_mem_wal_merged_gens: Vec::new(), + conflicting_mem_wal_compacted_sstables: Vec::new(), }) } } @@ -204,6 +254,15 @@ impl<'a> TransactionRebase<'a> { /// Will return an error if the transaction is not valid. Otherwise, it will /// return Ok(()). pub fn check_txn(&mut self, other_transaction: &Transaction, other_version: u64) -> Result<()> { + // Either order: the claim was checked without the write's data. + let ours = &self.transaction.operation; + let theirs = &other_transaction.operation; + if (may_alter_nullability(ours) && supplies_values(theirs)) + || (supplies_values(ours) && may_alter_nullability(theirs)) + { + return Err(self.retryable_conflict_err(other_transaction, other_version)); + } + let op = &self.transaction.operation; match op { Operation::Delete { .. } => self.check_delete_txn(other_transaction, other_version), @@ -219,6 +278,9 @@ impl<'a> TransactionRebase<'a> { Operation::DataReplacement { .. } => { self.check_data_replacement_txn(other_transaction, other_version) } + Operation::DataOverlay { .. } => { + self.check_data_overlay_txn(other_transaction, other_version) + } Operation::Merge { .. } => self.check_merge_txn(other_transaction, other_version), Operation::Restore { .. } => self.check_restore_txn(other_transaction, other_version), Operation::ReserveFragments { .. } => { @@ -251,6 +313,10 @@ impl<'a> TransactionRebase<'a> { | Operation::Project { .. } | Operation::Append { .. } | Operation::UpdateConfig { .. } + // A concurrent overlay is inert against the rows we delete + // (deletions take precedence over overlays) and otherwise + // preserves physical offsets, so it never conflicts. + | Operation::DataOverlay { .. } | Operation::UpdateBases { .. } => Ok(()), Operation::Rewrite { groups, .. } => { if groups @@ -345,7 +411,9 @@ impl<'a> TransactionRebase<'a> { ) -> Result<()> { if let Operation::Update { inserted_rows_filter: self_inserted_rows_filter, - merged_generations: self_merged_generations, + compacted_sstables: self_compacted_sstables, + new_fragments: self_new_fragments, + update_mode: self_update_mode, .. } = &self.transaction.operation { @@ -399,6 +467,49 @@ impl<'a> TransactionRebase<'a> { | Operation::Clone { .. } | Operation::UpdateConfig { .. } | Operation::UpdateBases { .. } => Ok(()), + Operation::DataOverlay { groups } => { + // Our update recomputed rows from the pre-overlay base, so if + // it commits over an overlay it would silently undo the + // overlay's values for any cell it recomputed. A row-moving + // update (RewriteRows) relocates the rows it touches out to + // new fragments; only the rows it actually moved lose their + // overlay, so we conflict only when the moved rows intersect + // the overlay's coverage. An in-place column rewrite + // (RewriteColumns) preserves offsets and just tombstones the + // overlaid fields at build time, so it never conflicts. + let moves_rows = !self_new_fragments.is_empty() + && matches!(self_update_mode, Some(UpdateMode::RewriteRows) | None); + if !moves_rows { + return Ok(()); + } + // `affected_rows` holds the physical offsets (per fragment) + // this update moved. The overlay's coverage is in the same + // physical-offset space, so we can intersect the two in + // memory. Without affected rows we cannot be precise, so we + // fall back to a fragment-granular conflict. + for group in groups { + if !self.modified_fragment_ids.contains(&group.fragment_id) { + continue; + } + let Some(affected_rows) = self.affected_rows else { + return Err( + self.retryable_conflict_err(other_transaction, other_version) + ); + }; + let Some(moved) = + affected_rows.get_fragment_bitmap(group.fragment_id as u32) + else { + continue; + }; + let coverage = overlay_group_coverage(group); + if !(moved & &coverage).is_empty() { + return Err( + self.retryable_conflict_err(other_transaction, other_version) + ); + } + } + Ok(()) + } Operation::Append { .. } => { // If current transaction has primary key conflict detection, // we can't safely commit against an Append because we don't @@ -487,10 +598,11 @@ impl<'a> TransactionRebase<'a> { Err(self.incompatible_conflict_err(other_transaction, other_version)) } Operation::UpdateMemWalState { - merged_generations: other_merged_generations, - } => self.check_merged_generations_conflict( - other_merged_generations, - self_merged_generations, + compacted_sstables: other_compacted_sstables, + .. + } => self.check_compacted_sstables_conflict( + other_compacted_sstables, + self_compacted_sstables, other_transaction, other_version, ), @@ -509,11 +621,15 @@ impl<'a> TransactionRebase<'a> { new_indices, removed_indices, .. - } = &self.transaction.operation + } = &mut self.transaction.operation { match &other_transaction.operation { Operation::Append { .. } | Operation::Clone { .. } + // An overlay committed after this index's version is newer than + // the index; the query path excludes its covered cells via the + // version gate, so the build does not conflict. + | Operation::DataOverlay { .. } | Operation::UpdateBases { .. } => Ok(()), Operation::CreateIndex { new_indices: created_indices, @@ -552,7 +668,19 @@ impl<'a> TransactionRebase<'a> { } // Although some of the rows we indexed may have been deleted / moved, // row ids are still valid, so we allow this optimistically. - Operation::Delete { .. } | Operation::Update { .. } => Ok(()), + Operation::Delete { .. } => Ok(()), + Operation::Update { + updated_fragments, + fields_modified, + .. + } => { + Transaction::prune_updated_fields_from_indices( + new_indices, + updated_fragments, + fields_modified, + ); + Ok(()) + } // Merge, reserve, and project don't change row ids, so this should be fine. Operation::Merge { .. } => Ok(()), Operation::ReserveFragments { .. } => Ok(()), @@ -571,6 +699,27 @@ impl<'a> TransactionRebase<'a> { // triggers a CreateIndex, and it needs to add the new reuse // version created by the rewrite if let Some(committed_fri) = frag_reuse_index { + let ngram_coverage = new_indices + .iter() + .filter(|idx| { + idx.index_details.as_ref().is_some_and(|details| { + details.type_url.ends_with("NGramIndexDetails") + }) + }) + .filter_map(|idx| idx.fragment_bitmap.as_ref()) + .fold(RoaringBitmap::new(), |coverage, fragments| { + coverage | fragments + }); + if groups + .iter() + .flat_map(|group| group.old_fragments.iter()) + .any(|fragment| ngram_coverage.contains(fragment.id as u32)) + { + return Err( + self.retryable_conflict_err(other_transaction, other_version) + ); + } + if new_indices .iter() .any(|idx| idx.name == FRAG_REUSE_INDEX_NAME) @@ -631,14 +780,15 @@ impl<'a> TransactionRebase<'a> { Ok(()) } Operation::UpdateMemWalState { - merged_generations: other_merged_gens, + compacted_sstables: other_compacted_sstables, + .. } => { // CreateIndex of MemWalIndex is compatible with UpdateMemWalState // as they can be rebased on each other if new_indices.iter().any(|idx| idx.name == MEM_WAL_INDEX_NAME) { - // Collect merged_generations from UpdateMemWalState for rebasing - self.conflicting_mem_wal_merged_gens - .extend(other_merged_gens.iter().cloned()); + // Collect compacted_sstables from UpdateMemWalState for rebasing + self.conflicting_mem_wal_compacted_sstables + .extend(other_compacted_sstables.iter().cloned()); Ok(()) } else { Err(self.incompatible_conflict_err(other_transaction, other_version)) @@ -695,6 +845,20 @@ impl<'a> TransactionRebase<'a> { Ok(()) } } + Operation::DataOverlay { groups } => { + // Rewriting a fragment changes its physical row addresses, so + // an overlay addressed by physical offset on that fragment is + // invalidated and must be re-applied against the new base. + if groups + .iter() + .map(|g| g.fragment_id) + .any(|id| self.modified_fragment_ids.contains(&id)) + { + Err(self.retryable_conflict_err(other_transaction, other_version)) + } else { + Ok(()) + } + } Operation::Rewrite { groups, frag_reuse_index: committed_fri, @@ -874,6 +1038,7 @@ impl<'a> TransactionRebase<'a> { | Operation::CreateIndex { .. } | Operation::Rewrite { .. } | Operation::DataReplacement { .. } + | Operation::DataOverlay { .. } | Operation::Merge { .. } | Operation::Restore { .. } | Operation::ReserveFragments { .. } @@ -907,7 +1072,8 @@ impl<'a> TransactionRebase<'a> { | Operation::Merge { .. } | Operation::UpdateConfig { .. } | Operation::Clone { .. } - | Operation::DataReplacement { .. } => Ok(()), + | Operation::DataReplacement { .. } + | Operation::DataOverlay { .. } => Ok(()), } } @@ -923,6 +1089,9 @@ impl<'a> TransactionRebase<'a> { | Operation::UpdateConfig { .. } | Operation::ReserveFragments { .. } | Operation::Project { .. } + // Both a column replacement and an overlay preserve physical row + // addresses; the overlay is newer and wins its covered cells. + | Operation::DataOverlay { .. } | Operation::UpdateBases { .. } => Ok(()), Operation::Merge { .. } => { // Merge rewrites the whole fragment list; always conflict @@ -1055,6 +1224,119 @@ impl<'a> TransactionRebase<'a> { } } + /// Conflict checks for our DataOverlay transaction against a concurrent one. + /// + /// Overlays are intentionally permissive (see the Data Overlay Files spec): + /// they stack with other overlays and tolerate appends, index builds, data + /// replacement, deletes, and in-place column rewrites (Update with + /// `RewriteColumns`), because overlay coverage is addressed by physical offset + /// and the version gate keeps indexes correct. A concurrent operation + /// conflicts when it takes precedence over the overlay for cells the overlay + /// covers, dropping the overlay's values: retryably when it rewrites the + /// physical layout of one of our fragments (Rewrite, Merge) or re-creates the + /// covered rows from the pre-overlay base (a row-moving Update — checked + /// row-by-row in `finish_data_overlay`), or removes an overlaid fragment + /// outright (a Delete / Update that drops the fragment); and incompatibly for + /// whole-dataset replacements (Overwrite / Restore) and MemWAL state updates + /// (UpdateMemWalState), which do not rebase against data operations. + fn check_data_overlay_txn( + &mut self, + other_transaction: &Transaction, + other_version: u64, + ) -> Result<()> { + match &other_transaction.operation { + Operation::Append { .. } + | Operation::CreateIndex { .. } + | Operation::ReserveFragments { .. } + | Operation::Project { .. } + | Operation::UpdateConfig { .. } + | Operation::UpdateBases { .. } + | Operation::Clone { .. } + | Operation::DataReplacement { .. } + | Operation::DataOverlay { .. } => Ok(()), + // A concurrent Delete only tombstones rows via a deletion vector, + // which preserves physical offsets; the overlay value for a deleted + // offset is simply inert. Conflict only if the whole overlaid + // fragment was removed, orphaning the overlay. + Operation::Delete { + deleted_fragment_ids, + .. + } => { + if deleted_fragment_ids + .iter() + .any(|id| self.modified_fragment_ids.contains(id)) + { + Err(self.retryable_conflict_err(other_transaction, other_version)) + } else { + Ok(()) + } + } + // A concurrent Update that removed an overlaid fragment orphans the + // overlay outright — conflict. A row-moving update (RewriteRows) + // deletes the rows it touches and re-creates them in new fragments; + // the update took precedence and the re-created rows were computed + // from the pre-overlay base, so the overlay's values for those cells + // are lost. That is a per-row problem, not an offset one: only the + // moved rows are affected. Comparing the moved rows against the + // overlay's coverage needs the update's deletion vectors, so we mark + // the fragment here and verify row-by-row in `finish_data_overlay`. + // An in-place column rewrite (RewriteColumns) preserves rows and just + // tombstones the overlaid fields at build time, so it never conflicts. + Operation::Update { + removed_fragment_ids, + updated_fragments, + new_fragments, + update_mode, + .. + } => { + let removed_ours = removed_fragment_ids + .iter() + .any(|id| self.modified_fragment_ids.contains(id)); + if removed_ours { + return Err(self.retryable_conflict_err(other_transaction, other_version)); + } + let moves_rows = !new_fragments.is_empty() + && matches!(update_mode, Some(UpdateMode::RewriteRows) | None); + if moves_rows { + for updated in updated_fragments { + if let Some((_, needs_row_check)) = + self.initial_fragments.get_mut(&updated.id) + { + *needs_row_check = true; + } + } + } + Ok(()) + } + Operation::Rewrite { groups, .. } => { + // A rewrite (compaction / fold) of a fragment we are overlaying + // changes its physical row addresses, so our offsets would be + // invalid. Conflict only if it touches one of our fragments. + let touches_our_fragment = groups + .iter() + .flat_map(|g| g.old_fragments.iter()) + .any(|f| self.modified_fragment_ids.contains(&f.id)); + if touches_our_fragment { + Err(self.retryable_conflict_err(other_transaction, other_version)) + } else { + Ok(()) + } + } + Operation::Merge { .. } => { + // Merge rewrites the whole fragment list; always conflict. + Err(self.retryable_conflict_err(other_transaction, other_version)) + } + // Overwrite/Restore replace the dataset; UpdateMemWalState does not + // rebase against data operations (mirroring check_update_mem_wal_state_txn, + // which likewise treats a concurrent DataOverlay as incompatible). + Operation::Overwrite { .. } + | Operation::Restore { .. } + | Operation::UpdateMemWalState { .. } => { + Err(self.incompatible_conflict_err(other_transaction, other_version)) + } + } + } + fn check_merge_txn( &mut self, other_transaction: &Transaction, @@ -1072,7 +1354,8 @@ impl<'a> TransactionRebase<'a> { | Operation::Delete { .. } | Operation::Rewrite { .. } | Operation::Merge { .. } - | Operation::DataReplacement { .. } => { + | Operation::DataReplacement { .. } + | Operation::DataOverlay { .. } => { Err(self.retryable_conflict_err(other_transaction, other_version)) } Operation::Overwrite { .. } @@ -1096,6 +1379,7 @@ impl<'a> TransactionRebase<'a> { | Operation::CreateIndex { .. } | Operation::Rewrite { .. } | Operation::DataReplacement { .. } + | Operation::DataOverlay { .. } | Operation::Merge { .. } | Operation::Restore { .. } | Operation::ReserveFragments { .. } @@ -1124,6 +1408,7 @@ impl<'a> TransactionRebase<'a> { | Operation::CreateIndex { .. } | Operation::Rewrite { .. } | Operation::DataReplacement { .. } + | Operation::DataOverlay { .. } | Operation::Merge { .. } | Operation::ReserveFragments { .. } | Operation::Update { .. } @@ -1148,6 +1433,7 @@ impl<'a> TransactionRebase<'a> { | Operation::UpdateConfig { .. } | Operation::CreateIndex { .. } | Operation::DataReplacement { .. } + | Operation::DataOverlay { .. } | Operation::Rewrite { .. } | Operation::Clone { .. } | Operation::ReserveFragments { .. } @@ -1212,6 +1498,7 @@ impl<'a> TransactionRebase<'a> { | Operation::CreateIndex { .. } | Operation::Rewrite { .. } | Operation::DataReplacement { .. } + | Operation::DataOverlay { .. } | Operation::Merge { .. } | Operation::Restore { .. } | Operation::ReserveFragments { .. } @@ -1230,45 +1517,49 @@ impl<'a> TransactionRebase<'a> { other_transaction: &Transaction, other_version: u64, ) -> Result<()> { + // Activation rebases like any other MemWAL state update; its preconditions + // are re-checked against the rebased index list when the commit applies. if let Operation::UpdateMemWalState { - merged_generations: self_merged_generations, + compacted_sstables: self_compacted_sstables, + .. } = &self.transaction.operation { match &other_transaction.operation { Operation::UpdateMemWalState { - merged_generations: other_merged_generations, + compacted_sstables: other_compacted_sstables, + .. } => { // Two UpdateMemWalState transactions conflict if they're updating - // the same shard's merged_generation - self.check_merged_generations_conflict( - other_merged_generations, - self_merged_generations, + // the same shard's compacted SSTable + self.check_compacted_sstables_conflict( + other_compacted_sstables, + self_compacted_sstables, other_transaction, other_version, ) } Operation::Update { - merged_generations: other_merged_generations, + compacted_sstables: other_compacted_sstables, .. } => { - // Update transactions with merged_generations can conflict - self.check_merged_generations_conflict( - other_merged_generations, - self_merged_generations, + // Update transactions with compacted_sstables can conflict + self.check_compacted_sstables_conflict( + other_compacted_sstables, + self_compacted_sstables, other_transaction, other_version, ) } Operation::CreateIndex { new_indices, .. } => { - // Check if CreateIndex has a MemWalIndex with merged_generations + // Check if CreateIndex has a MemWalIndex with compacted_sstables if let Some(mem_wal_idx) = new_indices .iter() .find(|idx| idx.name == MEM_WAL_INDEX_NAME) { let details = load_mem_wal_index_details(mem_wal_idx.clone())?; - self.check_merged_generations_conflict( - &details.merged_generations, - self_merged_generations, + self.check_compacted_sstables_conflict( + &details.compacted_sstables, + self_compacted_sstables, other_transaction, other_version, ) @@ -1284,6 +1575,7 @@ impl<'a> TransactionRebase<'a> { | Operation::Overwrite { .. } | Operation::Delete { .. } | Operation::DataReplacement { .. } + | Operation::DataOverlay { .. } | Operation::Merge { .. } | Operation::Restore { .. } | Operation::Clone { .. } @@ -1339,10 +1631,10 @@ impl<'a> TransactionRebase<'a> { } } - fn check_merged_generations_conflict( + fn check_compacted_sstables_conflict( &self, - committed: &[MergedGeneration], - to_commit: &[MergedGeneration], + committed: &[CompactedSsTable], + to_commit: &[CompactedSsTable], other_transaction: &Transaction, other_version: u64, ) -> Result<()> { @@ -1351,7 +1643,8 @@ impl<'a> TransactionRebase<'a> { for to_commit_mg in to_commit { if committed_mg.shard_id == to_commit_mg.shard_id { // Same shard being updated - // If committed >= to_commit, data already merged or superseded - abort without retry + // If committed >= to_commit, the SSTable is already compacted + // or superseded, so abort without retry. // If committed < to_commit, can retry with new state if committed_mg.generation >= to_commit_mg.generation { return Err( @@ -1374,6 +1667,7 @@ impl<'a> TransactionRebase<'a> { } Operation::CreateIndex { .. } => self.finish_create_index(dataset).await, Operation::Rewrite { .. } => self.finish_rewrite(dataset).await, + Operation::DataOverlay { .. } => self.finish_data_overlay(dataset).await, Operation::Append { .. } | Operation::Overwrite { .. } | Operation::DataReplacement { .. } @@ -1550,10 +1844,95 @@ impl<'a> TransactionRebase<'a> { } } + /// Verify no concurrent row-moving Update dropped the values of any cell + /// this overlay covers. `check_data_overlay_txn` flags (via the + /// `initial_fragments` needs-check bool) each overlaid fragment on which a + /// concurrent RewriteRows update relocated rows; here we read the deletion + /// vectors and conflict only when the moved rows intersect the overlay's + /// coverage. + /// + /// The moved rows are computed as the current deletion vector minus the + /// read-time one. In the rare case where both a concurrent Delete and a + /// concurrent Update touched the same flagged fragment, the Delete's rows are + /// also counted and may trigger an unnecessary retry — never data loss. Pure + /// concurrent deletes leave the fragment unflagged and are not examined here. + async fn finish_data_overlay(self, dataset: &Dataset) -> Result { + let fragments_to_check: HashSet = self + .initial_fragments + .iter() + .filter_map(|(id, (_, needs_check))| needs_check.then_some(*id)) + .collect(); + if fragments_to_check.is_empty() { + return Ok(Transaction { + read_version: dataset.manifest.version, + ..self.transaction + }); + } + + // Coverage (physical offsets, unioned across fields) per flagged fragment. + let Operation::DataOverlay { groups } = &self.transaction.operation else { + return Err(wrong_operation_err(&self.transaction.operation)); + }; + let mut coverage_by_fragment: HashMap = HashMap::new(); + for group in groups { + if !fragments_to_check.contains(&group.fragment_id) { + continue; + } + *coverage_by_fragment.entry(group.fragment_id).or_default() |= + overlay_group_coverage(group); + } + + for (fragment_id, coverage) in coverage_by_fragment { + let Some(current_fragment) = dataset + .fragments() + .as_slice() + .iter() + .find(|f| f.id == fragment_id) + else { + // The fragment is gone entirely; the overlay is orphaned. + return Err(crate::Error::retryable_commit_conflict_source( + dataset.manifest.version, + format!( + "This {} transaction was preempted: overlaid fragment {} was removed by a concurrent transaction. Please retry.", + self.transaction.uuid, fragment_id + ) + .into(), + )); + }; + let current_deletions = + read_fragment_deletion_bitmap(dataset, current_fragment).await?; + let initial_deletions = match self.initial_fragments.get(&fragment_id) { + Some((initial_fragment, _)) => { + read_fragment_deletion_bitmap(dataset, initial_fragment).await? + } + None => RoaringBitmap::new(), + }; + let moved_rows = ¤t_deletions - &initial_deletions; + let conflicting = &moved_rows & &coverage; + if !conflicting.is_empty() { + let sample: Vec = conflicting.iter().take(5).collect(); + return Err(crate::Error::retryable_commit_conflict_source( + dataset.manifest.version, + format!( + "This {} transaction was preempted by a concurrent update that moved overlaid rows on fragment {} (offsets {:?}). Please retry.", + self.transaction.uuid, fragment_id, sample.as_slice() + ) + .into(), + )); + } + } + + Ok(Transaction { + read_version: dataset.manifest.version, + ..self.transaction + }) + } + async fn finish_create_index(mut self, dataset: &Dataset) -> Result { if let Operation::CreateIndex { new_indices, removed_indices, + .. } = &mut self.transaction.operation { // Handle FRAG_REUSE_INDEX rebasing @@ -1608,33 +1987,35 @@ impl<'a> TransactionRebase<'a> { // Handle MEM_WAL_INDEX rebasing let has_mem_wal = new_indices.iter().any(|idx| idx.name == MEM_WAL_INDEX_NAME); - if has_mem_wal && !self.conflicting_mem_wal_merged_gens.is_empty() { + if has_mem_wal && !self.conflicting_mem_wal_compacted_sstables.is_empty() { let pos = new_indices .iter() .position(|idx| idx.name == MEM_WAL_INDEX_NAME) .unwrap(); - let current_meta = new_indices.remove(pos); - let mut details = load_mem_wal_index_details(current_meta)?; + let mut details = load_mem_wal_index_details(new_indices[pos].clone())?; - // Merge conflicting merged_generations - for each shard, keep higher generation - // We own self so we can consume conflicting_mem_wal_merged_gens directly - for new_mg in self.conflicting_mem_wal_merged_gens { + // Reconcile conflicting compacted_sstables by keeping each shard's higher + // generation. Both sides are already-committed facts here, so the higher one + // is correct; rejecting a stale proposal is the job of apply-time validation + // against the latest state, which runs after this rebase. + // We own self so we can consume conflicting_mem_wal_compacted_sstables directly + for new_sstable in self.conflicting_mem_wal_compacted_sstables { if let Some(existing) = details - .merged_generations + .compacted_sstables .iter_mut() - .find(|mg| mg.shard_id == new_mg.shard_id) + .find(|sstable| sstable.shard_id == new_sstable.shard_id) { - if new_mg.generation > existing.generation { - existing.generation = new_mg.generation; + if new_sstable.generation > existing.generation { + existing.generation = new_sstable.generation; } } else { - details.merged_generations.push(new_mg); + details.compacted_sstables.push(new_sstable); } } - let new_meta = new_mem_wal_index_meta(dataset.manifest.version, details)?; - new_indices.push(new_meta); + // Replaced in place so the index list keeps its order. + new_indices[pos] = new_mem_wal_index_meta(dataset.manifest.version, details)?; } for singleton_name in [FRAG_REUSE_INDEX_NAME, MEM_WAL_INDEX_NAME] { @@ -1774,6 +2155,40 @@ async fn initial_fragments_for_rebase( .collect::>() } +/// Read a fragment's deletion vector as a bitmap of physical offsets, or an +/// empty bitmap when the fragment has no deletion file. +async fn read_fragment_deletion_bitmap( + dataset: &Dataset, + fragment: &Fragment, +) -> Result { + match &fragment.deletion_file { + Some(deletion_file) => { + let dv = read_dataset_deletion_file(dataset, fragment.id, deletion_file).await?; + Ok(RoaringBitmap::from(dv.as_ref())) + } + None => Ok(RoaringBitmap::new()), + } +} + +/// The physical offsets a group's overlays cover, unioned across every overlay +/// and every field. This is the set of cells whose values the overlay supplies, +/// used to test whether a concurrent row-moving Update actually invalidates the +/// overlay. +fn overlay_group_coverage(group: &DataOverlayGroup) -> RoaringBitmap { + let mut union = RoaringBitmap::new(); + for overlay in &group.overlays { + match &overlay.coverage { + OverlayCoverage::Shared(bitmap) => union |= bitmap.as_ref(), + OverlayCoverage::PerField(bitmaps) => { + for bitmap in bitmaps { + union |= bitmap.as_ref(); + } + } + } + } + union +} + fn wrong_operation_err(op: &Operation) -> Error { Error::internal(format!("function called against a wrong operation: {}", op)) } @@ -1884,7 +2299,7 @@ mod tests { removed_fragment_ids: vec![], new_fragments: vec![], fields_modified: vec![], - merged_generations: Vec::new(), + compacted_sstables: Vec::new(), fields_for_preserving_frag_bitmap: vec![], update_mode: None, inserted_rows_filter: None, @@ -1897,7 +2312,7 @@ mod tests { removed_fragment_ids: vec![2], new_fragments: vec![], fields_modified: vec![], - merged_generations: Vec::new(), + compacted_sstables: Vec::new(), fields_for_preserving_frag_bitmap: vec![], update_mode: None, inserted_rows_filter: None, @@ -1913,7 +2328,7 @@ mod tests { updated_fragments: vec![Fragment::new(4)], new_fragments: vec![], fields_modified: vec![], - merged_generations: Vec::new(), + compacted_sstables: Vec::new(), fields_for_preserving_frag_bitmap: vec![], update_mode: None, inserted_rows_filter: None, @@ -2006,7 +2421,7 @@ mod tests { "path1", vec![0], vec![0], - &LanceFileVersion::Stable, + LanceFileVersion::Stable.resolve(), NonZero::new(10), ) .with_physical_rows(3); @@ -2016,7 +2431,7 @@ mod tests { removed_fragment_ids: vec![], new_fragments: vec![sample_file.clone()], fields_modified: vec![], - merged_generations: Vec::new(), + compacted_sstables: Vec::new(), fields_for_preserving_frag_bitmap: vec![], update_mode: None, inserted_rows_filter: None, @@ -2032,7 +2447,7 @@ mod tests { removed_fragment_ids: vec![], new_fragments: vec![sample_file], fields_modified: vec![], - merged_generations: Vec::new(), + compacted_sstables: Vec::new(), fields_for_preserving_frag_bitmap: vec![], update_mode: None, inserted_rows_filter: None, @@ -2149,7 +2564,7 @@ mod tests { "path1", vec![0], vec![0], - &LanceFileVersion::Stable, + LanceFileVersion::Stable.resolve(), NonZero::new(10), ) .with_physical_rows(3); @@ -2162,7 +2577,7 @@ mod tests { removed_fragment_ids: vec![0], new_fragments: vec![sample_file.clone()], fields_modified: vec![], - merged_generations: Vec::new(), + compacted_sstables: Vec::new(), fields_for_preserving_frag_bitmap: vec![], update_mode: None, inserted_rows_filter: None, @@ -2176,7 +2591,7 @@ mod tests { removed_fragment_ids: vec![], new_fragments: vec![sample_file.clone()], fields_modified: vec![], - merged_generations: Vec::new(), + compacted_sstables: Vec::new(), fields_for_preserving_frag_bitmap: vec![], update_mode: None, inserted_rows_filter: None, @@ -2313,6 +2728,7 @@ mod tests { Operation::Merge { fragments: vec![fragment0.clone(), fragment2.clone()], schema: lance_core::datatypes::Schema::default(), + preserves_nullability: true, }, Operation::Overwrite { fragments: vec![fragment0.clone(), fragment2.clone()], @@ -2337,7 +2753,7 @@ mod tests { updated_fragments: vec![fragment0.clone()], new_fragments: vec![fragment2.clone()], fields_modified: vec![0], - merged_generations: Vec::new(), + compacted_sstables: Vec::new(), fields_for_preserving_frag_bitmap: vec![], update_mode: None, inserted_rows_filter: None, @@ -2508,6 +2924,7 @@ mod tests { Operation::Merge { fragments: vec![fragment0.clone(), fragment2.clone()], schema: lance_core::datatypes::Schema::default(), + preserves_nullability: true, }, // Merge conflicts with everything except CreateIndex and ReserveFragments. [ @@ -2544,7 +2961,7 @@ mod tests { removed_fragment_ids: vec![], new_fragments: vec![fragment2], fields_modified: vec![0], - merged_generations: Vec::new(), + compacted_sstables: Vec::new(), fields_for_preserving_frag_bitmap: vec![], update_mode: None, inserted_rows_filter: None, @@ -2738,7 +3155,7 @@ mod tests { modified_fragment_ids: modified_fragment_ids(operation).collect::>(), affected_rows: None, conflicting_frag_reuse_indices: Vec::new(), - conflicting_mem_wal_merged_gens: Vec::new(), + conflicting_mem_wal_compacted_sstables: Vec::new(), }; for (other, expected_conflict) in other_transactions.iter().zip(expected_conflicts) { @@ -2782,74 +3199,702 @@ mod tests { } #[test] - fn test_create_index_conflicts_only_on_same_name() { - let index0 = IndexMetadata { - uuid: uuid::Uuid::new_v4(), - name: "test".to_string(), - fields: vec![0], - dataset_version: 1, - fragment_bitmap: None, - index_details: None, - index_version: 0, - created_at: None, - base_id: None, - files: None, + fn test_data_overlay_conflicts() { + use crate::dataset::transaction::{DataOverlayGroup, UpdateMode}; + use ConflictResult::*; + use lance_table::format::overlay::{DataOverlayFile, OverlayCoverage}; + use roaring::RoaringBitmap; + + // Our transaction overlays fragment 1. + let overlay_op = |fragment_id: u64| Operation::DataOverlay { + groups: vec![DataOverlayGroup { + fragment_id, + overlays: vec![DataOverlayFile { + data_file: DataFile::new_legacy_from_fields("overlay.lance", vec![0], None), + coverage: OverlayCoverage::dense(RoaringBitmap::from_iter([0u32])), + committed_version: 0, + }], + }], }; - let index1 = IndexMetadata { - uuid: uuid::Uuid::new_v4(), - name: "other".to_string(), - ..index0.clone() + let update_removing = |removed_fragment_ids: Vec| Operation::Update { + removed_fragment_ids, + updated_fragments: vec![], + new_fragments: vec![], + fields_modified: vec![], + compacted_sstables: Vec::new(), + fields_for_preserving_frag_bitmap: vec![], + update_mode: None, + inserted_rows_filter: None, + updated_fragment_offsets: None, }; - - let txn = Transaction::new( - 0, - Operation::CreateIndex { - new_indices: vec![index0.clone()], - removed_indices: vec![], - }, - None, - ); - let mut rebase = TransactionRebase { - transaction: txn, - initial_fragments: HashMap::new(), - modified_fragment_ids: HashSet::new(), - affected_rows: None, - conflicting_frag_reuse_indices: Vec::new(), - conflicting_mem_wal_merged_gens: Vec::new(), + let delete = |updated: Vec, deleted: Vec| Operation::Delete { + updated_fragments: updated, + deleted_fragment_ids: deleted, + predicate: "x > 2".to_string(), + }; + // A row-moving update (RewriteRows) relocates the updated rows into + // new_fragments; an in-place column rewrite (RewriteColumns) leaves rows + // where they are. + let update_moving = |updated: Vec, new: Vec| Operation::Update { + removed_fragment_ids: vec![], + updated_fragments: updated, + new_fragments: new, + fields_modified: vec![], + compacted_sstables: Vec::new(), + fields_for_preserving_frag_bitmap: vec![], + update_mode: Some(UpdateMode::RewriteRows), + inserted_rows_filter: None, + updated_fragment_offsets: None, + }; + let update_rewrite_columns = |updated: Vec| Operation::Update { + removed_fragment_ids: vec![], + updated_fragments: updated, + new_fragments: vec![], + fields_modified: vec![0], + compacted_sstables: Vec::new(), + fields_for_preserving_frag_bitmap: vec![], + update_mode: Some(UpdateMode::RewriteColumns), + inserted_rows_filter: None, + updated_fragment_offsets: None, + }; + let rewrite_of = |old: &Fragment| Operation::Rewrite { + groups: vec![RewriteGroup { + old_fragments: vec![old.clone()], + new_fragments: vec![], + }], + rewritten_indices: vec![], + frag_reuse_index: None, }; - let same_name = Transaction::new( - 0, - Operation::CreateIndex { - new_indices: vec![IndexMetadata { - uuid: uuid::Uuid::new_v4(), - ..index0 - }], - removed_indices: vec![], - }, - None, - ); - let different_name = Transaction::new( - 0, - Operation::CreateIndex { - new_indices: vec![index1], - removed_indices: vec![], - }, - None, - ); - - let same_name_result = rebase.check_txn(&same_name, 1); - assert!( - matches!(same_name_result, Err(Error::RetryableCommitConflict { .. })), - "Expected retryable conflict for same-name CreateIndex, got {:?}", - same_name_result - ); + let fragment0 = Fragment::new(0); + let fragment1 = Fragment::new(1); - let mut rebase = TransactionRebase { - transaction: Transaction::new( - 0, - Operation::CreateIndex { - new_indices: vec![IndexMetadata { + // Each case is checked against our overlay on fragment 1. + let cases: Vec<(Operation, ConflictResult)> = vec![ + // Permissive: preserves physical offsets / leaves fragment 1 in place. + ( + Operation::Append { + fragments: vec![fragment0.clone()], + }, + Compatible, + ), + ( + Operation::CreateIndex { + new_indices: vec![], + removed_indices: vec![], + }, + Compatible, + ), + ( + Operation::DataReplacement { + replacements: vec![DataReplacementGroup( + 1, + DataFile::new_legacy_from_fields("r.lance", vec![0], None), + )], + }, + Compatible, + ), + // Another overlay on the same fragment stacks rather than conflicts. + (overlay_op(1), Compatible), + // A Delete only tombstones rows (deletion vector) on fragment 1, and + // an in-place column rewrite preserves offsets, so both are compatible. + (delete(vec![fragment1.clone()], vec![]), Compatible), + (update_rewrite_columns(vec![fragment1.clone()]), Compatible), + (update_removing(vec![2]), Compatible), + // ...but removing our overlaid fragment 1 orphans the overlay -> conflict. + (delete(vec![], vec![1]), Retryable), + (update_removing(vec![1]), Retryable), + // A row-moving update re-creates the rows it touches from the + // pre-overlay base. Whether that actually drops any overlaid cell is + // a per-row question answered in `finish_data_overlay` (see + // test_data_overlay_finish_conflicts_with_row_moving_update), so the + // check itself defers rather than conflicting; a moving update on any + // fragment is compatible at this stage. + ( + update_moving(vec![fragment1.clone()], vec![fragment0.clone()]), + Compatible, + ), + ( + update_moving(vec![fragment0.clone()], vec![fragment0.clone()]), + Compatible, + ), + // Rewriting fragment 1 invalidates its physical offsets -> conflict; + // a rewrite of a different fragment does not. + (rewrite_of(&fragment1), Retryable), + (rewrite_of(&fragment0), Compatible), + // Merge rewrites the whole fragment list; Restore replaces the dataset. + ( + Operation::Merge { + fragments: vec![fragment1.clone()], + schema: lance_core::datatypes::Schema::default(), + preserves_nullability: true, + }, + Retryable, + ), + (Operation::Restore { version: 1 }, NotCompatible), + // Overwrite/Restore replace the dataset, and UpdateMemWalState does + // not rebase against data operations — all hard conflicts. + ( + Operation::Overwrite { + fragments: vec![fragment0.clone()], + schema: lance_core::datatypes::Schema::default(), + config_upsert_values: None, + initial_bases: None, + }, + NotCompatible, + ), + ( + Operation::UpdateMemWalState { + compacted_sstables: vec![], + require_index_catchup: false, + }, + NotCompatible, + ), + ]; + + for (other, expected) in cases { + let mut rebase = TransactionRebase { + transaction: Transaction::new(0, overlay_op(1), None), + initial_fragments: HashMap::new(), + modified_fragment_ids: modified_fragment_ids(&overlay_op(1)) + .collect::>(), + affected_rows: None, + conflicting_frag_reuse_indices: Vec::new(), + conflicting_mem_wal_compacted_sstables: Vec::new(), + }; + let other_txn = Transaction::new(0, other.clone(), None); + let result = rebase.check_txn(&other_txn, 1); + match expected { + Compatible => assert!( + result.is_ok(), + "overlay should be compatible with {other:?}, got {result:?}" + ), + Retryable => assert!( + matches!(result, Err(Error::RetryableCommitConflict { .. })), + "overlay should retryably conflict with {other:?}, got {result:?}" + ), + NotCompatible => assert!( + matches!(result, Err(Error::IncompatibleTransaction { .. })), + "overlay should be incompatible with {other:?}, got {result:?}" + ), + } + } + } + + #[test] + fn test_rewrite_conflicts_with_data_overlay() { + // Reverse direction of test_data_overlay_conflicts: our transaction is a + // Rewrite and a concurrent DataOverlay has already committed. A rewrite + // changes the physical row addresses of the fragments it touches, so an + // overlay on one of those fragments is invalidated (retryable); an + // overlay on any other fragment is unaffected. + use crate::dataset::transaction::DataOverlayGroup; + use lance_table::format::overlay::{DataOverlayFile, OverlayCoverage}; + use roaring::RoaringBitmap; + + let overlay_on = |fragment_id: u64| Operation::DataOverlay { + groups: vec![DataOverlayGroup { + fragment_id, + overlays: vec![DataOverlayFile { + data_file: DataFile::new_legacy_from_fields("overlay.lance", vec![0], None), + coverage: OverlayCoverage::dense(RoaringBitmap::from_iter([0u32])), + committed_version: 0, + }], + }], + }; + // Our transaction rewrites fragment 1. + let rewrite_op = Operation::Rewrite { + groups: vec![RewriteGroup { + old_fragments: vec![Fragment::new(1)], + new_fragments: vec![], + }], + rewritten_indices: vec![], + frag_reuse_index: None, + }; + + for (other, expect_conflict) in [(overlay_on(1), true), (overlay_on(0), false)] { + let mut rebase = TransactionRebase { + transaction: Transaction::new(0, rewrite_op.clone(), None), + initial_fragments: HashMap::new(), + modified_fragment_ids: modified_fragment_ids(&rewrite_op).collect::>(), + affected_rows: None, + conflicting_frag_reuse_indices: Vec::new(), + conflicting_mem_wal_compacted_sstables: Vec::new(), + }; + let other_txn = Transaction::new(0, other.clone(), None); + let result = rebase.check_txn(&other_txn, 1); + if expect_conflict { + assert!( + matches!(result, Err(Error::RetryableCommitConflict { .. })), + "rewrite of fragment 1 should retryably conflict with {other:?}, got {result:?}" + ); + } else { + assert!( + result.is_ok(), + "rewrite of fragment 1 should not conflict with {other:?}, got {result:?}" + ); + } + } + } + + #[test] + fn test_update_conflicts_with_data_overlay() { + // Reverse direction of test_data_overlay_conflicts: our transaction is an + // Update and a concurrent DataOverlay has already committed. A row-moving + // update relocates the rows it touches, so an overlay on one of those + // fragments can no longer be applied (retryable); an overlay on any other + // fragment, or an in-place column rewrite, is compatible. + use crate::dataset::transaction::{DataOverlayGroup, UpdateMode}; + use lance_table::format::overlay::{DataOverlayFile, OverlayCoverage}; + use roaring::RoaringBitmap; + + let overlay_on = |fragment_id: u64| Operation::DataOverlay { + groups: vec![DataOverlayGroup { + fragment_id, + overlays: vec![DataOverlayFile { + data_file: DataFile::new_legacy_from_fields("overlay.lance", vec![0], None), + coverage: OverlayCoverage::dense(RoaringBitmap::from_iter([0u32])), + committed_version: 0, + }], + }], + }; + // Our update always touches fragment 1. + let update = + |update_mode: Option, new_fragments: Vec| Operation::Update { + removed_fragment_ids: vec![], + updated_fragments: vec![Fragment::new(1)], + new_fragments, + fields_modified: vec![0], + compacted_sstables: Vec::new(), + fields_for_preserving_frag_bitmap: vec![], + update_mode, + inserted_rows_filter: None, + updated_fragment_offsets: None, + }; + + // The overlay covers physical offset 0 of its fragment. Row addresses + // pack the fragment id in the high 32 bits and the offset in the low 32. + let rows_on = |fragment_id: u64, offsets: &[u32]| { + let mut map = RowAddrTreeMap::new(); + map.insert_bitmap( + fragment_id as u32, + RoaringBitmap::from_iter(offsets.iter().copied()), + ); + map + }; + + // (update, committed overlay, moved rows the update carries, expect conflict) + let cases = [ + // Row-moving update whose moved rows include the overlaid cell -> the + // update would undo the overlay, so conflict. + ( + update(Some(UpdateMode::RewriteRows), vec![Fragment::new(2)]), + overlay_on(1), + Some(rows_on(1, &[0])), + true, + ), + // ...but if the moved rows miss the overlaid cell, the overlay survives. + ( + update(Some(UpdateMode::RewriteRows), vec![Fragment::new(2)]), + overlay_on(1), + Some(rows_on(1, &[5])), + false, + ), + // An overlay on a fragment the update did not touch is fine. + ( + update(Some(UpdateMode::RewriteRows), vec![Fragment::new(2)]), + overlay_on(0), + Some(rows_on(1, &[0])), + false, + ), + // An in-place column rewrite preserves rows -> compatible. + ( + update(Some(UpdateMode::RewriteColumns), vec![]), + overlay_on(1), + Some(rows_on(1, &[0])), + false, + ), + // Without affected rows we cannot be precise, so a row-moving update + // on the overlaid fragment falls back to a conservative conflict. + ( + update(Some(UpdateMode::RewriteRows), vec![Fragment::new(2)]), + overlay_on(1), + None, + true, + ), + ]; + + for (update_op, other, affected_rows, expect_conflict) in cases { + let mut rebase = TransactionRebase { + transaction: Transaction::new(0, update_op.clone(), None), + initial_fragments: HashMap::new(), + modified_fragment_ids: modified_fragment_ids(&update_op).collect::>(), + affected_rows: affected_rows.as_ref(), + conflicting_frag_reuse_indices: Vec::new(), + conflicting_mem_wal_compacted_sstables: Vec::new(), + }; + let other_txn = Transaction::new(0, other.clone(), None); + let result = rebase.check_txn(&other_txn, 1); + if expect_conflict { + assert!( + matches!(result, Err(Error::RetryableCommitConflict { .. })), + "update should retryably conflict with {other:?}, got {result:?}" + ); + } else { + assert!( + result.is_ok(), + "update should be compatible with {other:?}, got {result:?}" + ); + } + } + } + + /// An append is the one value-write that rebases across a committed merge, + /// so a merge introducing a required field must claim: the append's + /// fragments omit the new column and its rows would read as null. A merge + /// without the claim keeps the long-standing behavior of appends passing + /// over nullable column adds. The reverse order conflicts regardless of + /// the claim, because a rebasing merge rewrites the whole fragment list. + #[test] + fn test_merge_claim_blocks_stale_append() { + for claims in [true, false] { + let merge = Operation::Merge { + fragments: vec![Fragment::new(0)], + schema: lance_core::datatypes::Schema::default(), + preserves_nullability: !claims, + }; + let append = Operation::Append { + fragments: vec![Fragment::new(1)], + }; + + let mut append_rebase = TransactionRebase { + transaction: Transaction::new(0, append.clone(), None), + initial_fragments: HashMap::new(), + modified_fragment_ids: HashSet::new(), + affected_rows: None, + conflicting_frag_reuse_indices: Vec::new(), + conflicting_mem_wal_compacted_sstables: Vec::new(), + }; + let result = append_rebase.check_txn(&Transaction::new(0, merge.clone(), None), 1); + assert_eq!( + matches!(result, Err(Error::RetryableCommitConflict { .. })), + claims, + "append rebasing over merge/claims={claims}: got {result:?}" + ); + + let mut merge_rebase = TransactionRebase { + transaction: Transaction::new(0, merge, None), + initial_fragments: HashMap::new(), + modified_fragment_ids: HashSet::from_iter([0]), + affected_rows: None, + conflicting_frag_reuse_indices: Vec::new(), + conflicting_mem_wal_compacted_sstables: Vec::new(), + }; + let result = merge_rebase.check_txn(&Transaction::new(0, append, None), 1); + assert!( + matches!(result, Err(Error::RetryableCommitConflict { .. })), + "merge rebasing over append/claims={claims}: got {result:?}" + ); + } + } + + /// A claim conflicts with any write that can supply values, either order. + #[test] + fn test_non_null_claim_barrier() { + use crate::dataset::transaction::{DataOverlayGroup, UpdateMode}; + use lance_table::format::overlay::{DataOverlayFile, OverlayCoverage}; + use roaring::RoaringBitmap; + + let file = || DataFile::new_legacy_from_fields("w.lance", vec![0], None); + let writers = [ + ( + "append", + Operation::Append { + fragments: vec![Fragment::new(1)], + }, + ), + ( + "update", + Operation::Update { + removed_fragment_ids: vec![], + updated_fragments: vec![Fragment::new(0)], + new_fragments: vec![], + fields_modified: vec![0], + compacted_sstables: Vec::new(), + fields_for_preserving_frag_bitmap: vec![], + update_mode: Some(UpdateMode::RewriteColumns), + inserted_rows_filter: None, + updated_fragment_offsets: None, + }, + ), + ( + "replacement", + Operation::DataReplacement { + replacements: vec![DataReplacementGroup(0, file())], + }, + ), + ( + "overlay", + Operation::DataOverlay { + groups: vec![DataOverlayGroup { + fragment_id: 0, + overlays: vec![DataOverlayFile { + data_file: file(), + coverage: OverlayCoverage::dense(RoaringBitmap::from_iter([0u32])), + committed_version: 0, + }], + }], + }, + ), + ]; + + for (writer_name, writer) in &writers { + for claims in [true, false] { + let project = Operation::Project { + schema: lance_core::datatypes::Schema::default(), + preserves_nullability: !claims, + }; + for (order, ours, theirs) in [ + ("project-rebasing", project.clone(), writer.clone()), + ("writer-rebasing", writer.clone(), project.clone()), + ] { + let mut rebase = TransactionRebase { + transaction: Transaction::new(0, ours.clone(), None), + initial_fragments: HashMap::new(), + modified_fragment_ids: modified_fragment_ids(&ours).collect::>(), + affected_rows: None, + conflicting_frag_reuse_indices: Vec::new(), + conflicting_mem_wal_compacted_sstables: Vec::new(), + }; + let result = rebase.check_txn(&Transaction::new(0, theirs, None), 1); + assert_eq!( + matches!(result, Err(Error::RetryableCommitConflict { .. })), + claims, + "{writer_name}/claims={claims}/{order}: got {result:?}" + ); + } + } + } + } + + #[tokio::test] + #[rstest::rstest] + #[case::coverage_overlaps_moved_row(vec![0u32], true)] + #[case::coverage_disjoint_from_moved_row(vec![3u32], false)] + async fn test_data_overlay_finish_conflicts_with_row_moving_update( + #[case] coverage_offsets: Vec, + #[case] expect_conflict: bool, + ) { + // 5 rows in one fragment. A concurrent RewriteRows update moves row 0 out + // to a new fragment (deleting it from fragment 0). Our overlay on fragment + // 0 conflicts only when its coverage includes the moved row; the decision + // is made in finish, which reads the deletion vectors. + use crate::dataset::transaction::{DataOverlayGroup, UpdateMode}; + use lance_table::format::overlay::{DataOverlayFile, OverlayCoverage}; + use roaring::RoaringBitmap; + + let dataset = test_dataset(5, 1).await; + let mut fragment = dataset.fragments().as_slice()[0].clone(); + + let moved_fragment = Fragment::new(0) + .with_file( + "moved.lance", + vec![0], + vec![0], + LanceFileVersion::Stable.resolve(), + NonZero::new(10), + ) + .with_physical_rows(1); + let update_op = Operation::Update { + updated_fragments: vec![apply_deletion(&[0], &mut fragment, &dataset).await], + removed_fragment_ids: vec![], + new_fragments: vec![moved_fragment], + fields_modified: vec![], + compacted_sstables: Vec::new(), + fields_for_preserving_frag_bitmap: vec![], + update_mode: Some(UpdateMode::RewriteRows), + inserted_rows_filter: None, + updated_fragment_offsets: None, + }; + let update_txn = Transaction::new_from_version(dataset.manifest.version, update_op); + + let overlay_op = Operation::DataOverlay { + groups: vec![DataOverlayGroup { + fragment_id: 0, + overlays: vec![DataOverlayFile { + data_file: DataFile::new_legacy_from_fields("overlay.lance", vec![0], None), + coverage: OverlayCoverage::dense(RoaringBitmap::from_iter(coverage_offsets)), + committed_version: 0, + }], + }], + }; + let overlay_txn = Transaction::new_from_version(dataset.manifest.version, overlay_op); + + // Commit the update so the latest dataset reflects the moved (deleted) row. + let latest_dataset = CommitBuilder::new(Arc::new(dataset.clone())) + .execute(update_txn.clone()) + .await + .unwrap(); + + let mut rebase = TransactionRebase::try_new(&dataset, overlay_txn.clone(), None) + .await + .unwrap(); + // The check defers the row-level decision to finish, flagging fragment 0. + rebase.check_txn(&update_txn, 1).unwrap(); + assert_eq!( + rebase + .initial_fragments + .iter() + .map(|(id, (_, needs_check))| (*id, *needs_check)) + .collect::>(), + vec![(0, true)], + ); + + let res = rebase.finish(&latest_dataset).await; + if expect_conflict { + assert!( + matches!(res, Err(crate::Error::RetryableCommitConflict { .. })), + "overlay covering the moved row should conflict, got {res:?}" + ); + } else { + assert!( + res.is_ok(), + "overlay disjoint from the moved row should succeed, got {res:?}" + ); + } + } + + #[rstest::rstest] + #[test] + #[case::indexed_field_updated(0, vec![0])] + #[case::other_field_updated(1, vec![0, 1])] + fn test_create_index_rebase_prunes_updated_field_coverage( + #[case] field_modified: u32, + #[case] expected_fragment_ids: Vec, + ) { + let index = IndexMetadata { + uuid: Uuid::new_v4(), + name: "test".to_string(), + fields: vec![0], + dataset_version: 1, + fragment_bitmap: Some(RoaringBitmap::from_iter([0, 1])), + index_details: None, + index_version: 0, + created_at: None, + base_id: None, + files: None, + }; + let mut rebase = TransactionRebase { + transaction: Transaction::new( + 1, + Operation::CreateIndex { + new_indices: vec![index], + removed_indices: vec![], + }, + None, + ), + initial_fragments: HashMap::new(), + modified_fragment_ids: HashSet::new(), + affected_rows: None, + conflicting_frag_reuse_indices: Vec::new(), + conflicting_mem_wal_compacted_sstables: Vec::new(), + }; + let update = Transaction::new( + 1, + Operation::Update { + updated_fragments: vec![Fragment::new(1)], + removed_fragment_ids: vec![], + new_fragments: vec![], + fields_modified: vec![field_modified], + compacted_sstables: Vec::new(), + fields_for_preserving_frag_bitmap: vec![], + update_mode: Some(UpdateMode::RewriteColumns), + inserted_rows_filter: None, + updated_fragment_offsets: None, + }, + None, + ); + + rebase.check_txn(&update, 2).unwrap(); + + let Operation::CreateIndex { new_indices, .. } = &rebase.transaction.operation else { + panic!("expected CreateIndex operation"); + }; + assert_eq!( + new_indices[0].fragment_bitmap.as_ref().unwrap(), + &RoaringBitmap::from_iter(expected_fragment_ids) + ); + } + + #[test] + fn test_create_index_conflicts_only_on_same_name() { + let index0 = IndexMetadata { + uuid: uuid::Uuid::new_v4(), + name: "test".to_string(), + fields: vec![0], + dataset_version: 1, + fragment_bitmap: None, + index_details: None, + index_version: 0, + created_at: None, + base_id: None, + files: None, + }; + let index1 = IndexMetadata { + uuid: uuid::Uuid::new_v4(), + name: "other".to_string(), + ..index0.clone() + }; + + let txn = Transaction::new( + 0, + Operation::CreateIndex { + new_indices: vec![index0.clone()], + removed_indices: vec![], + }, + None, + ); + let mut rebase = TransactionRebase { + transaction: txn, + initial_fragments: HashMap::new(), + modified_fragment_ids: HashSet::new(), + affected_rows: None, + conflicting_frag_reuse_indices: Vec::new(), + conflicting_mem_wal_compacted_sstables: Vec::new(), + }; + + let same_name = Transaction::new( + 0, + Operation::CreateIndex { + new_indices: vec![IndexMetadata { + uuid: uuid::Uuid::new_v4(), + ..index0 + }], + removed_indices: vec![], + }, + None, + ); + let different_name = Transaction::new( + 0, + Operation::CreateIndex { + new_indices: vec![index1], + removed_indices: vec![], + }, + None, + ); + + let same_name_result = rebase.check_txn(&same_name, 1); + assert!( + matches!(same_name_result, Err(Error::RetryableCommitConflict { .. })), + "Expected retryable conflict for same-name CreateIndex, got {:?}", + same_name_result + ); + + let mut rebase = TransactionRebase { + transaction: Transaction::new( + 0, + Operation::CreateIndex { + new_indices: vec![IndexMetadata { uuid: uuid::Uuid::new_v4(), name: "test".to_string(), fields: vec![0], @@ -2869,7 +3914,7 @@ mod tests { modified_fragment_ids: HashSet::new(), affected_rows: None, conflicting_frag_reuse_indices: Vec::new(), - conflicting_mem_wal_merged_gens: Vec::new(), + conflicting_mem_wal_compacted_sstables: Vec::new(), }; let different_name_result = rebase.check_txn(&different_name, 1); assert!( @@ -2879,6 +3924,79 @@ mod tests { ); } + #[test] + fn test_create_ngram_index_conflicts_with_overlapping_deferred_rewrite() { + let ngram_index = |fragment_id| IndexMetadata { + uuid: Uuid::new_v4(), + name: "text_ngram".to_string(), + fields: vec![0], + dataset_version: 1, + fragment_bitmap: Some(RoaringBitmap::from_iter([fragment_id])), + index_details: Some(Arc::new(prost_types::Any { + type_url: "lance.index.NGramIndexDetails".to_string(), + value: Vec::new(), + })), + index_version: 0, + created_at: None, + base_id: None, + files: None, + }; + let frag_reuse_index = IndexMetadata { + uuid: Uuid::new_v4(), + name: FRAG_REUSE_INDEX_NAME.to_string(), + fields: vec![], + dataset_version: 2, + fragment_bitmap: Some(RoaringBitmap::from_iter([2u32])), + index_details: None, + index_version: 0, + created_at: None, + base_id: None, + files: None, + }; + let rewrite = Transaction::new( + 1, + Operation::Rewrite { + groups: vec![RewriteGroup { + old_fragments: vec![Fragment::new(1)], + new_fragments: vec![Fragment::new(2)], + }], + rewritten_indices: vec![], + frag_reuse_index: Some(frag_reuse_index), + }, + None, + ); + + for (covered_fragment, expect_conflict) in [(1u32, true), (3u32, false)] { + let mut rebase = TransactionRebase { + transaction: Transaction::new( + 1, + Operation::CreateIndex { + new_indices: vec![ngram_index(covered_fragment)], + removed_indices: vec![], + }, + None, + ), + initial_fragments: HashMap::new(), + modified_fragment_ids: HashSet::new(), + affected_rows: None, + conflicting_frag_reuse_indices: Vec::new(), + conflicting_mem_wal_compacted_sstables: Vec::new(), + }; + let result = rebase.check_txn(&rewrite, 2); + if expect_conflict { + assert!( + matches!(result, Err(Error::RetryableCommitConflict { .. })), + "overlapping staged NGram index should conflict, got {result:?}" + ); + } else { + assert!( + result.is_ok(), + "disjoint staged NGram index should remain compatible, got {result:?}" + ); + } + } + } + #[tokio::test] async fn test_add_bases_non_conflicting() { let dataset = test_dataset(10, 2).await; @@ -3067,7 +4185,7 @@ mod tests { removed_fragment_ids: vec![], new_fragments: vec![], fields_modified: vec![], - merged_generations: Vec::new(), + compacted_sstables: Vec::new(), fields_for_preserving_frag_bitmap: vec![], update_mode: None, inserted_rows_filter: None, @@ -3255,6 +4373,7 @@ mod tests { Operation::DataReplacement { replacements } => { Box::new(replacements.iter().map(|r| r.0)) } + Operation::DataOverlay { groups } => Box::new(groups.iter().map(|g| g.fragment_id)), } } @@ -3359,7 +4478,7 @@ mod tests { removed_fragment_ids: vec![], new_fragments: vec![], fields_modified: vec![2], - merged_generations: Vec::new(), + compacted_sstables: Vec::new(), fields_for_preserving_frag_bitmap: vec![], update_mode: Some(RewriteColumns), inserted_rows_filter: None, @@ -3378,7 +4497,7 @@ mod tests { removed_fragment_ids: vec![], new_fragments: vec![Fragment::new(5)], fields_modified: vec![2], - merged_generations: Vec::new(), + compacted_sstables: Vec::new(), fields_for_preserving_frag_bitmap: vec![], update_mode: Some(RewriteColumns), inserted_rows_filter: None, @@ -3396,7 +4515,7 @@ mod tests { removed_fragment_ids: vec![], new_fragments: vec![], fields_modified: vec![1], - merged_generations: Vec::new(), + compacted_sstables: Vec::new(), fields_for_preserving_frag_bitmap: vec![], update_mode: Some(RewriteColumns), inserted_rows_filter: None, @@ -3414,7 +4533,7 @@ mod tests { removed_fragment_ids: vec![], new_fragments: vec![Fragment::new(5)], fields_modified: vec![], - merged_generations: Vec::new(), + compacted_sstables: Vec::new(), fields_for_preserving_frag_bitmap: vec![], update_mode: Some(RewriteRows), inserted_rows_filter: None, @@ -3432,7 +4551,7 @@ mod tests { removed_fragment_ids: vec![0], new_fragments: vec![], fields_modified: vec![], - merged_generations: Vec::new(), + compacted_sstables: Vec::new(), fields_for_preserving_frag_bitmap: vec![], update_mode: None, inserted_rows_filter: None, @@ -3450,7 +4569,7 @@ mod tests { removed_fragment_ids: vec![], new_fragments: vec![Fragment::new(5)], fields_modified: vec![], - merged_generations: Vec::new(), + compacted_sstables: Vec::new(), fields_for_preserving_frag_bitmap: vec![], update_mode: Some(RewriteRows), inserted_rows_filter: None, @@ -3491,6 +4610,7 @@ mod tests { Operation::Merge { fragments: vec![Fragment::new(0)], schema: lance_core::datatypes::Schema::default(), + preserves_nullability: true, }, Retryable, ), @@ -3506,7 +4626,7 @@ mod tests { modified_fragment_ids: modified_fragment_ids(&op1).collect::>(), affected_rows: None, conflicting_frag_reuse_indices: Vec::new(), - conflicting_mem_wal_merged_gens: Vec::new(), + conflicting_mem_wal_compacted_sstables: Vec::new(), }; let result = rebase.check_txn(&txn2, 1); @@ -3542,7 +4662,7 @@ mod tests { } #[test] - fn test_merged_generations_conflict_lower_generation_fails() { + fn test_compacted_sstables_conflict_lower_generation_fails() { // Test: committed generation >= to_commit generation should be incompatible (no retry) let shard = Uuid::new_v4(); @@ -3550,7 +4670,8 @@ mod tests { let committed_txn = Transaction::new( 0, Operation::UpdateMemWalState { - merged_generations: vec![MergedGeneration::new(shard, 10)], + compacted_sstables: vec![CompactedSsTable::new(shard, 10)], + require_index_catchup: false, }, None, ); @@ -3558,7 +4679,8 @@ mod tests { let to_commit_txn = Transaction::new( 0, Operation::UpdateMemWalState { - merged_generations: vec![MergedGeneration::new(shard, 5)], + compacted_sstables: vec![CompactedSsTable::new(shard, 5)], + require_index_catchup: false, }, None, ); @@ -3569,7 +4691,7 @@ mod tests { modified_fragment_ids: HashSet::new(), affected_rows: None, conflicting_frag_reuse_indices: Vec::new(), - conflicting_mem_wal_merged_gens: Vec::new(), + conflicting_mem_wal_compacted_sstables: Vec::new(), }; let result = rebase.check_txn(&committed_txn, 1); @@ -3581,14 +4703,15 @@ mod tests { } #[test] - fn test_merged_generations_conflict_equal_generation_fails() { + fn test_compacted_sstables_conflict_equal_generation_fails() { // Test: committed generation == to_commit generation should be incompatible (no retry) let shard = Uuid::new_v4(); let committed_txn = Transaction::new( 0, Operation::UpdateMemWalState { - merged_generations: vec![MergedGeneration::new(shard, 10)], + compacted_sstables: vec![CompactedSsTable::new(shard, 10)], + require_index_catchup: false, }, None, ); @@ -3596,7 +4719,8 @@ mod tests { let to_commit_txn = Transaction::new( 0, Operation::UpdateMemWalState { - merged_generations: vec![MergedGeneration::new(shard, 10)], + compacted_sstables: vec![CompactedSsTable::new(shard, 10)], + require_index_catchup: false, }, None, ); @@ -3607,7 +4731,7 @@ mod tests { modified_fragment_ids: HashSet::new(), affected_rows: None, conflicting_frag_reuse_indices: Vec::new(), - conflicting_mem_wal_merged_gens: Vec::new(), + conflicting_mem_wal_compacted_sstables: Vec::new(), }; let result = rebase.check_txn(&committed_txn, 1); @@ -3619,7 +4743,7 @@ mod tests { } #[test] - fn test_merged_generations_conflict_higher_generation_retryable() { + fn test_compacted_sstables_conflict_higher_generation_retryable() { // Test: committed generation < to_commit generation should be retryable let shard = Uuid::new_v4(); @@ -3627,7 +4751,8 @@ mod tests { let committed_txn = Transaction::new( 0, Operation::UpdateMemWalState { - merged_generations: vec![MergedGeneration::new(shard, 5)], + compacted_sstables: vec![CompactedSsTable::new(shard, 5)], + require_index_catchup: false, }, None, ); @@ -3635,7 +4760,8 @@ mod tests { let to_commit_txn = Transaction::new( 0, Operation::UpdateMemWalState { - merged_generations: vec![MergedGeneration::new(shard, 10)], + compacted_sstables: vec![CompactedSsTable::new(shard, 10)], + require_index_catchup: false, }, None, ); @@ -3646,7 +4772,7 @@ mod tests { modified_fragment_ids: HashSet::new(), affected_rows: None, conflicting_frag_reuse_indices: Vec::new(), - conflicting_mem_wal_merged_gens: Vec::new(), + conflicting_mem_wal_compacted_sstables: Vec::new(), }; let result = rebase.check_txn(&committed_txn, 1); @@ -3658,7 +4784,7 @@ mod tests { } #[test] - fn test_merged_generations_different_shards_ok() { + fn test_compacted_sstables_different_shards_ok() { // Test: different shards should not conflict let shard1 = Uuid::new_v4(); let shard2 = Uuid::new_v4(); @@ -3666,7 +4792,8 @@ mod tests { let committed_txn = Transaction::new( 0, Operation::UpdateMemWalState { - merged_generations: vec![MergedGeneration::new(shard1, 10)], + compacted_sstables: vec![CompactedSsTable::new(shard1, 10)], + require_index_catchup: false, }, None, ); @@ -3674,7 +4801,8 @@ mod tests { let to_commit_txn = Transaction::new( 0, Operation::UpdateMemWalState { - merged_generations: vec![MergedGeneration::new(shard2, 5)], + compacted_sstables: vec![CompactedSsTable::new(shard2, 5)], + require_index_catchup: false, }, None, ); @@ -3685,7 +4813,7 @@ mod tests { modified_fragment_ids: HashSet::new(), affected_rows: None, conflicting_frag_reuse_indices: Vec::new(), - conflicting_mem_wal_merged_gens: Vec::new(), + conflicting_mem_wal_compacted_sstables: Vec::new(), }; let result = rebase.check_txn(&committed_txn, 1); @@ -3697,15 +4825,15 @@ mod tests { } #[test] - fn test_update_mem_wal_state_vs_create_index_with_merged_generations() { + fn test_update_mem_wal_state_vs_create_index_with_compacted_sstables() { use crate::index::mem_wal::new_mem_wal_index_meta; use lance_index::mem_wal::MemWalIndexDetails; let shard = Uuid::new_v4(); - // Create a MemWalIndex with merged_generations + // Create a MemWalIndex with compacted_sstables let details = MemWalIndexDetails { - merged_generations: vec![MergedGeneration::new(shard, 10)], + compacted_sstables: vec![CompactedSsTable::new(shard, 10)], ..Default::default() }; let mem_wal_index = new_mem_wal_index_meta(1, details).unwrap(); @@ -3724,7 +4852,8 @@ mod tests { let to_commit_txn = Transaction::new( 0, Operation::UpdateMemWalState { - merged_generations: vec![MergedGeneration::new(shard, 5)], + compacted_sstables: vec![CompactedSsTable::new(shard, 5)], + require_index_catchup: false, }, None, ); @@ -3735,7 +4864,7 @@ mod tests { modified_fragment_ids: HashSet::new(), affected_rows: None, conflicting_frag_reuse_indices: Vec::new(), - conflicting_mem_wal_merged_gens: Vec::new(), + conflicting_mem_wal_compacted_sstables: Vec::new(), }; let result = rebase.check_txn(&committed_txn, 1); @@ -3749,7 +4878,8 @@ mod tests { let to_commit_txn_higher = Transaction::new( 0, Operation::UpdateMemWalState { - merged_generations: vec![MergedGeneration::new(shard, 15)], + compacted_sstables: vec![CompactedSsTable::new(shard, 15)], + require_index_catchup: false, }, None, ); @@ -3760,7 +4890,7 @@ mod tests { modified_fragment_ids: HashSet::new(), affected_rows: None, conflicting_frag_reuse_indices: Vec::new(), - conflicting_mem_wal_merged_gens: Vec::new(), + conflicting_mem_wal_compacted_sstables: Vec::new(), }; let result_higher = rebase_higher.check_txn(&committed_txn, 1); @@ -3778,7 +4908,7 @@ mod tests { let shard = Uuid::new_v4(); - // CreateIndex with MemWalIndex (no merged_generations initially) + // CreateIndex with MemWalIndex (no compacted_sstables initially) let details = MemWalIndexDetails::default(); let mem_wal_index = new_mem_wal_index_meta(1, details).unwrap(); @@ -3795,7 +4925,8 @@ mod tests { let committed_txn = Transaction::new( 0, Operation::UpdateMemWalState { - merged_generations: vec![MergedGeneration::new(shard, 10)], + compacted_sstables: vec![CompactedSsTable::new(shard, 10)], + require_index_catchup: false, }, None, ); @@ -3806,11 +4937,11 @@ mod tests { modified_fragment_ids: HashSet::new(), affected_rows: None, conflicting_frag_reuse_indices: Vec::new(), - conflicting_mem_wal_merged_gens: Vec::new(), + conflicting_mem_wal_compacted_sstables: Vec::new(), }; // CreateIndex of MemWalIndex should be compatible with UpdateMemWalState - // and should collect the merged_generations for rebasing + // and should collect the compacted_sstables for rebasing let result = rebase.check_txn(&committed_txn, 1); assert!( result.is_ok(), @@ -3818,10 +4949,16 @@ mod tests { result ); - // Verify that merged_generations were collected - assert_eq!(rebase.conflicting_mem_wal_merged_gens.len(), 1); - assert_eq!(rebase.conflicting_mem_wal_merged_gens[0].shard_id, shard); - assert_eq!(rebase.conflicting_mem_wal_merged_gens[0].generation, 10); + // Verify that compacted_sstables were collected + assert_eq!(rebase.conflicting_mem_wal_compacted_sstables.len(), 1); + assert_eq!( + rebase.conflicting_mem_wal_compacted_sstables[0].shard_id, + shard + ); + assert_eq!( + rebase.conflicting_mem_wal_compacted_sstables[0].generation, + 10 + ); } #[tokio::test] diff --git a/rust/lance/src/io/commit/external_manifest.rs b/rust/lance/src/io/commit/external_manifest.rs index 850d10f9a23..9efecbda5e9 100644 --- a/rust/lance/src/io/commit/external_manifest.rs +++ b/rust/lance/src/io/commit/external_manifest.rs @@ -14,10 +14,11 @@ mod test { use futures::stream::BoxStream; use futures::{StreamExt, TryStreamExt, future::join_all}; use lance_core::{Error, Result}; + use lance_io::object_store::ObjectStore; use lance_table::io::commit::external_manifest::{ ExternalManifestCommitHandler, ExternalManifestStore, }; - use lance_table::io::commit::{CommitHandler, ManifestNamingScheme}; + use lance_table::io::commit::{CommitHandler, ManifestLocation, ManifestNamingScheme}; use lance_testing::datagen::{BatchGenerator, IncrementingInt32}; use object_store::memory::InMemory; use object_store::{ @@ -125,6 +126,76 @@ mod test { } } + #[derive(Debug)] + struct StaticExternalManifestStore { + location: ManifestLocation, + verify_store: Option>, + } + + #[async_trait] + impl ExternalManifestStore for StaticExternalManifestStore { + async fn get(&self, _uri: &str, version: u64) -> Result { + if version == self.location.version { + Ok(self.location.path.to_string()) + } else { + Err(Error::not_found(format!("version {}", version))) + } + } + + async fn get_manifest_location( + &self, + _base_uri: &str, + version: u64, + ) -> Result { + if version == self.location.version { + Ok(self.location.clone()) + } else { + Err(Error::not_found(format!("version {}", version))) + } + } + + async fn get_latest_version(&self, _uri: &str) -> Result> { + Ok(Some(( + self.location.version, + self.location.path.to_string(), + ))) + } + + async fn get_latest_manifest_location( + &self, + _base_uri: &str, + ) -> Result> { + Ok(Some(self.location.clone())) + } + + async fn put_if_not_exists( + &self, + _uri: &str, + _version: u64, + _path: &str, + _size: u64, + _e_tag: Option, + ) -> Result<()> { + Ok(()) + } + + async fn put_if_exists( + &self, + _uri: &str, + _version: u64, + path: &str, + size: u64, + e_tag: Option, + ) -> Result<()> { + if let Some(store) = &self.verify_store { + let final_meta = store.head(&Path::from(path)).await?; + assert_eq!(size, final_meta.size); + assert_eq!(e_tag, final_meta.e_tag); + } + Ok(()) + } + } + fn read_params(handler: Arc) -> ReadParams { ReadParams { commit_handler: Some(handler), @@ -139,6 +210,242 @@ mod test { } } + #[tokio::test] + async fn finalized_external_manifest_location_is_head_checked() { + let sleepy_store = SleepyExternalManifestStore::new(); + let inner_store = sleepy_store.store.clone(); + let handler = ExternalManifestCommitHandler { + external_manifest_store: Arc::new(sleepy_store), + }; + let object_store = ObjectStore::memory(); + let base_path = Path::from("repro"); + let version = 7; + let final_path = ManifestNamingScheme::V2.manifest_path(&base_path, version); + let body = b"manifest body"; + + object_store + .inner + .put(&final_path, PutPayload::from_static(body)) + .await + .expect("seed finalized manifest"); + inner_store + .lock() + .await + .insert((base_path.to_string(), version), final_path.to_string()); + + let location = handler + .resolve_latest_location(&base_path, &object_store) + .await + .expect("resolve latest finalized manifest"); + + assert_eq!(location.path, final_path); + assert_eq!(location.size, Some(body.len() as u64)); + assert_eq!(location.naming_scheme, ManifestNamingScheme::V2); + } + + #[tokio::test] + async fn finalized_external_manifest_location_falls_back_to_v1() { + let sleepy_store = SleepyExternalManifestStore::new(); + let inner_store = sleepy_store.store.clone(); + let handler = ExternalManifestCommitHandler { + external_manifest_store: Arc::new(sleepy_store), + }; + let object_store = ObjectStore::memory(); + let base_path = Path::from("repro"); + let version = 7; + let missing_v2_path = ManifestNamingScheme::V2.manifest_path(&base_path, version); + let v1_path = ManifestNamingScheme::V1.manifest_path(&base_path, version); + + object_store + .inner + .put(&v1_path, PutPayload::from_static(b"v1 manifest body")) + .await + .expect("seed V1 manifest"); + inner_store.lock().await.insert( + (base_path.to_string(), version), + missing_v2_path.to_string(), + ); + + let latest_location = handler + .resolve_latest_location(&base_path, &object_store) + .await + .expect("resolve latest should fall back to V1"); + assert_eq!(latest_location.path, v1_path); + assert_eq!(latest_location.naming_scheme, ManifestNamingScheme::V1); + + let version_location = handler + .resolve_version_location(&base_path, version, object_store.inner.as_ref()) + .await + .expect("resolve version should fall back to V1"); + assert_eq!(version_location.path, v1_path); + assert_eq!(version_location.naming_scheme, ManifestNamingScheme::V1); + } + + #[tokio::test] + async fn finalized_external_manifest_location_rejects_size_mismatch() { + let object_store = ObjectStore::memory(); + let base_path = Path::from("repro"); + let version = 7; + let final_path = ManifestNamingScheme::V2.manifest_path(&base_path, version); + let body = b"manifest body"; + + object_store + .inner + .put(&final_path, PutPayload::from_static(body)) + .await + .expect("seed finalized manifest"); + let handler = ExternalManifestCommitHandler { + external_manifest_store: Arc::new(StaticExternalManifestStore { + location: ManifestLocation { + version, + path: final_path, + size: Some(body.len() as u64 + 1), + naming_scheme: ManifestNamingScheme::V2, + e_tag: None, + }, + verify_store: None, + }), + }; + + let err = handler + .resolve_latest_location(&base_path, &object_store) + .await + .expect_err("stale external manifest size should be rejected"); + assert!(matches!(err, Error::CorruptFile { .. }), "{err:?}"); + assert!(err.to_string().contains("Manifest size mismatch"), "{err}"); + } + + #[tokio::test] + async fn finalized_external_manifest_location_rejects_etag_mismatch() { + let object_store = ObjectStore::memory(); + let base_path = Path::from("repro"); + let version = 7; + let final_path = ManifestNamingScheme::V2.manifest_path(&base_path, version); + let body = b"manifest body"; + + object_store + .inner + .put(&final_path, PutPayload::from_static(body)) + .await + .expect("seed finalized manifest"); + let handler = ExternalManifestCommitHandler { + external_manifest_store: Arc::new(StaticExternalManifestStore { + location: ManifestLocation { + version, + path: final_path, + size: Some(body.len() as u64), + naming_scheme: ManifestNamingScheme::V2, + e_tag: Some("stale-etag".to_string()), + }, + verify_store: None, + }), + }; + + let err = handler + .resolve_latest_location(&base_path, &object_store) + .await + .expect_err("stale external manifest e_tag should be rejected"); + assert!(matches!(err, Error::CorruptFile { .. }), "{err:?}"); + assert!(err.to_string().contains("Manifest e_tag mismatch"), "{err}"); + } + + #[tokio::test] + async fn external_manifest_store_put_records_destination_metadata() { + let object_store: Arc = Arc::new(InMemory::new()); + let base_path = Path::from("repro"); + let staging_path = Path::from("repro/_versions/1.manifest.staging-abcd"); + object_store + .put(&staging_path, PutPayload::from_static(b"manifest body")) + .await + .expect("seed staging manifest"); + let staging_meta = object_store + .head(&staging_path) + .await + .expect("read staging metadata"); + + let external_store = StaticExternalManifestStore { + location: ManifestLocation { + version: 1, + path: staging_path.clone(), + size: Some(staging_meta.size), + naming_scheme: ManifestNamingScheme::V2, + e_tag: staging_meta.e_tag.clone(), + }, + verify_store: Some(object_store.clone()), + }; + let location = external_store + .put( + &base_path, + 1, + &staging_path, + staging_meta.size, + staging_meta.e_tag.clone(), + object_store.as_ref(), + ManifestNamingScheme::V2, + ) + .await + .expect("finalize manifest"); + let final_meta = object_store + .head(&location.path) + .await + .expect("read finalized metadata"); + + assert_ne!( + staging_meta.e_tag, final_meta.e_tag, + "test store must assign a new ETag to the copied object" + ); + assert_eq!(location.size, Some(final_meta.size)); + assert_eq!(location.e_tag, final_meta.e_tag); + } + + #[tokio::test] + async fn external_manifest_handler_finalize_records_destination_metadata() { + let object_store = ObjectStore::memory(); + let base_path = Path::from("repro"); + let version = 1; + let staging_path = Path::from("repro/_versions/1.manifest.staging-abcd"); + object_store + .inner + .put(&staging_path, PutPayload::from_static(b"manifest body")) + .await + .expect("seed staging manifest"); + let staging_meta = object_store + .inner + .head(&staging_path) + .await + .expect("read staging metadata"); + + let handler = ExternalManifestCommitHandler { + external_manifest_store: Arc::new(StaticExternalManifestStore { + location: ManifestLocation { + version, + path: staging_path, + size: Some(staging_meta.size), + naming_scheme: ManifestNamingScheme::V2, + e_tag: staging_meta.e_tag.clone(), + }, + verify_store: Some(object_store.inner.clone()), + }), + }; + + let location = handler + .resolve_version_location(&base_path, version, object_store.inner.as_ref()) + .await + .expect("finalize manifest"); + let final_meta = object_store + .inner + .head(&location.path) + .await + .expect("read finalized metadata"); + + assert_ne!( + staging_meta.e_tag, final_meta.e_tag, + "test store must assign a new ETag to the copied object" + ); + assert_eq!(location.size, Some(final_meta.size)); + assert_eq!(location.e_tag, final_meta.e_tag); + } + #[tokio::test] async fn test_dataset_can_onboard_external_store() { // First write a dataset WITHOUT external store diff --git a/rust/lance/src/io/commit/s3_test.rs b/rust/lance/src/io/commit/s3_test.rs index b5b1a09c776..6a714e8e576 100644 --- a/rust/lance/src/io/commit/s3_test.rs +++ b/rust/lance/src/io/commit/s3_test.rs @@ -304,9 +304,12 @@ async fn test_ddb_open_iops() { // * write staged file // * copy to final file // * delete staged file + // Commit: 2 read IOPs: + // * list versions before creating the dataset + // * HEAD the finalized manifest to record destination metadata after copy let io_stats = committed_ds.object_store.as_ref().io_stats_incremental(); assert_io_eq!(io_stats, write_iops, 4); - assert_io_eq!(io_stats, read_iops, 1); + assert_io_eq!(io_stats, read_iops, 2); let dataset = DatasetBuilder::from_uri(&uri) .with_read_params(ReadParams { @@ -317,9 +320,10 @@ async fn test_ddb_open_iops() { .await .unwrap(); let io_stats = dataset.object_store.as_ref().io_stats_incremental(); - // Open dataset can be read with 1 IOP, just to read the manifest. - // Looking up latest manifest is handled in dynamodb. - assert_io_eq!(io_stats, read_iops, 1); + // Open dataset can be read with 2 IOPs: HEAD verifies that the + // finalized path returned by DynamoDB still exists, then the manifest + // itself is read. Looking up the latest manifest is handled in DynamoDB. + assert_io_eq!(io_stats, read_iops, 2); assert_io_eq!(io_stats, write_iops, 0); // Append @@ -334,14 +338,17 @@ async fn test_ddb_open_iops() { let io_stats = dataset.object_store.as_ref().io_stats_incremental(); // Append: 5 IOPS: data file, transaction file, 3x manifest file assert_io_eq!(io_stats, write_iops, 5); + // Append reads once to list versions and once to HEAD the finalized manifest + // so the destination metadata can be recorded after copy. // TODO: we can reduce this by implementing a specialized CommitHandler::list_manifest_locations() // for the DDB commit handler. - assert_io_eq!(io_stats, read_iops, 1); + assert_io_eq!(io_stats, read_iops, 2); // Checkout original version dataset.checkout_version(1).await.unwrap(); let io_stats = dataset.object_store.as_ref().io_stats_incremental(); - // Checkout: 1 IOPS: manifest file + // Checkout: 1 read IOP. Version resolution HEAD-checks the finalized path, + // while the manifest body is served from the Session metadata cache. assert_io_eq!(io_stats, read_iops, 1); assert_io_eq!(io_stats, write_iops, 0); } diff --git a/rust/lance/src/io/exec.rs b/rust/lance/src/io/exec.rs index a477d60d56d..6923f09c34f 100644 --- a/rust/lance/src/io/exec.rs +++ b/rust/lance/src/io/exec.rs @@ -36,6 +36,7 @@ pub use optimizer::get_physical_optimizer; pub use projection::project; pub use pushdown_scan::{LancePushdownScanExec, ScanConfig}; pub use rowids::{AddRowAddrExec, AddRowOffsetExec}; +pub(crate) use scan::LanceStream; pub use scan::{LanceScanConfig, LanceScanExec}; pub use take::TakeExec; pub use utils::PreFilterSource; diff --git a/rust/lance/src/io/exec/count_from_mask.rs b/rust/lance/src/io/exec/count_from_mask.rs index 0b7aeb11111..e2f7f00efb5 100644 --- a/rust/lance/src/io/exec/count_from_mask.rs +++ b/rust/lance/src/io/exec/count_from_mask.rs @@ -395,10 +395,6 @@ impl ExecutionPlan for CountFromMaskExec { "CountFromMaskExec" } - fn as_any(&self) -> &dyn std::any::Any { - self - } - fn schema(&self) -> SchemaRef { self.schema.clone() } @@ -463,11 +459,11 @@ impl ExecutionPlan for CountFromMaskExec { fn partition_statistics( &self, _partition: Option, - ) -> datafusion::error::Result { - Ok(datafusion::physical_plan::Statistics { + ) -> datafusion::error::Result> { + Ok(Arc::new(datafusion::physical_plan::Statistics { num_rows: datafusion::common::stats::Precision::Exact(1), ..datafusion::physical_plan::Statistics::new_unknown(&self.schema) - }) + })) } fn metrics(&self) -> Option { @@ -494,7 +490,6 @@ mod tests { use datafusion::logical_expr::lit; use datafusion::physical_expr::execution_props::ExecutionProps; use datafusion::physical_plan::ExecutionPlan; - use datafusion::physical_planner::create_aggregate_expr_and_maybe_filter; use datafusion::scalar::ScalarValue; use futures::TryStreamExt; use lance_core::utils::tempfile::TempStrDir; @@ -512,8 +507,13 @@ mod tests { use crate::index::DatasetIndexExt; use crate::io::exec::scalar_index::ScalarIndexExec; use crate::utils::test::{DatagenExt, FragmentCount, FragmentRowCount}; + #[allow(deprecated)] + use datafusion::physical_planner::create_aggregate_expr_and_maybe_filter; /// Build an `AggregateFunctionExpr` matching `COUNT(*)`. + // TODO(datafusion-54): migrate off the deprecated + // create_aggregate_expr_and_maybe_filter to LoweredAggregateBuilder. + #[allow(deprecated)] fn count_star_expr(input_schema: &SchemaRef) -> Arc { let expr = functions_aggregate::count::count(lit(1)); let df_schema = DFSchema::try_from(input_schema.as_ref().clone()).unwrap(); diff --git a/rust/lance/src/io/exec/count_pushdown.rs b/rust/lance/src/io/exec/count_pushdown.rs index d5d90b5881a..9da8f4842f1 100644 --- a/rust/lance/src/io/exec/count_pushdown.rs +++ b/rust/lance/src/io/exec/count_pushdown.rs @@ -13,16 +13,17 @@ //! enough to be reused. //! //! Two rewritten shapes are emitted depending on whether the scalar index -//! backing the filter covers every dataset fragment. +//! backing the filter covers every fragment targeted by the scan. //! -//! **Full coverage** (index ⊇ dataset, or no filter at all): +//! **Full coverage** (index ⊇ targeted fragments, or no filter at all): //! //! ```text //! AggregateExec(Final, aggs=[count(...)], group_by=[]) //! └── CountFromMaskExec { prefilter_input = index_input } //! ``` //! -//! **Partial coverage** (index ⊊ dataset — typically appended fragments): +//! **Partial coverage** (index misses some targeted fragments — typically +//! appended fragments): //! //! ```text //! AggregateExec(Final, aggs=[count(...)], group_by=[]) @@ -83,7 +84,7 @@ impl PhysicalOptimizerRule for CountPushdown { ) -> DFResult> { Ok(plan .transform_down(|plan| { - let Some(agg) = plan.as_any().downcast_ref::() else { + let Some(agg) = plan.downcast_ref::() else { return Ok(Transformed::no(plan)); }; if let Some(rewritten) = try_rewrite(agg)? { @@ -147,6 +148,11 @@ fn try_rewrite(agg: &AggregateExec) -> DFResult>> }; let options = filtered_read.options(); + // We don't currently support count pushdown when the row selector + // is a row stream. + if filtered_read.row_stream_input().is_some() { + return Ok(None); + } // A refine filter is a residual the index couldn't fully evaluate — it // needs column data to apply, which we can't. if options.refine_filter.is_some() { @@ -173,21 +179,38 @@ fn try_rewrite(agg: &AggregateExec) -> DFResult>> ); return Ok(None); } - // Same story for an explicit fragment subset: legitimate, but unexpected - // alongside an aggregate, and we lose the pushdown opportunity. - if options.fragments.is_some() { - warn!( - "count_pushdown: skipped because the FilteredReadExec was scoped \ - to an explicit fragment subset; the count will be computed via a \ - full scan. Intersecting that subset into the coverage logic would \ - let this query be answered from index metadata." - ); - return Ok(None); - } - let dataset = filtered_read.dataset().clone(); let dataset_fragments: RoaringBitmap = dataset.fragments().iter().map(|f| f.id as u32).collect(); + let fragment_scope = if let Some(fragments) = options.fragments.as_ref() { + let fragment_scope = fragments + .iter() + .map(|fragment| fragment.id as u32) + .collect::(); + // A bitmap cannot preserve duplicate fragments, and CountFromMaskExec + // resolves fragment IDs against the current manifest instead of using + // the descriptors supplied to FilteredReadExec. + let has_duplicate_fragments = fragment_scope.len() != fragments.len() as u64; + let descriptors_are_current = fragments.iter().all(|fragment| { + let Ok(fragment_id) = u32::try_from(fragment.id) else { + return false; + }; + if !dataset.fragment_bitmap.contains(fragment_id) { + return false; + } + let fragment_index = dataset.fragment_bitmap.rank(fragment_id) as usize - 1; + dataset.fragments().get(fragment_index) == Some(fragment) + }); + if has_duplicate_fragments || !descriptors_are_current { + return Ok(None); + } + Some(fragment_scope) + } else { + None + }; + let target_fragments = fragment_scope + .clone() + .unwrap_or_else(|| dataset_fragments.clone()); let prefilter_input = filtered_read.index_input().cloned(); // If there is a prefilter, inspect its ScalarIndexExpr leaves: @@ -202,15 +225,12 @@ fn try_rewrite(agg: &AggregateExec) -> DFResult>> let index_coverage = match &prefilter_input { None => None, Some(input) => { - let scalar_exec = input - .as_any() - .downcast_ref::() - .ok_or_else(|| { - datafusion::error::DataFusionError::Internal( - "count_pushdown: FilteredReadExec.index_input is not a ScalarIndexExec" - .to_string(), - ) - })?; + let scalar_exec = input.downcast_ref::().ok_or_else(|| { + datafusion::error::DataFusionError::Internal( + "count_pushdown: FilteredReadExec.index_input is not a ScalarIndexExec" + .to_string(), + ) + })?; if scalar_exec.expr().needs_recheck() { return Ok(None); } @@ -226,11 +246,11 @@ fn try_rewrite(agg: &AggregateExec) -> DFResult>> // Decide on the plan shape. Three cases: // // 1. No prefilter (no filter at all): single pushdown branch over every - // dataset fragment. Always safe. - // 2. Prefilter + index covers every dataset fragment: single pushdown + // targeted fragment. Always safe. + // 2. Prefilter + index covers every targeted fragment: single pushdown // branch, prefilter feeds in directly. - // 3. Prefilter + index covers a strict subset: split into pushdown over - // indexed fragments + parallel scan over unindexed fragments. + // 3. Prefilter + index covers a strict subset of the target: split into + // pushdown over indexed fragments + parallel scan over unindexed fragments. let (partial_stream, partial_state_schema): (Arc, _) = match index_coverage { None => { // No prefilter at all (verified above): nothing to restrict. @@ -238,32 +258,36 @@ fn try_rewrite(agg: &AggregateExec) -> DFResult>> dataset, aggr_exprs.clone(), prefilter_input, - None, + fragment_scope, )?; let schema = exec.schema(); (Arc::new(exec), schema) } - Some(coverage) if (&dataset_fragments - &coverage).is_empty() => { - // Prefilter exists and the index covers every dataset fragment — + Some(coverage) if (&target_fragments - &coverage).is_empty() => { + // Prefilter exists and the index covers every targeted fragment — // safe to push the whole count down. let exec = CountFromMaskExec::try_new_restricted( dataset, aggr_exprs.clone(), prefilter_input, - None, + fragment_scope, )?; let schema = exec.schema(); (Arc::new(exec), schema) } Some(coverage) => { - // Split plan: CountFromMaskExec for the indexed fragments, a - // normal scan + AggregateExec(Partial) for the rest. - let uncovered = &dataset_fragments - &coverage; + // Split plan: CountFromMaskExec for the targeted indexed fragments, + // a normal scan + AggregateExec(Partial) for the targeted remainder. + let covered = &target_fragments & &coverage; + if covered.is_empty() { + return Ok(None); + } + let uncovered = &target_fragments - &coverage; let pushdown_exec = CountFromMaskExec::try_new_restricted( dataset, aggr_exprs.clone(), prefilter_input, - Some(&dataset_fragments & &coverage), + Some(covered), )?; let partial_state_schema = pushdown_exec.schema(); let pushdown_branch: Arc = Arc::new(pushdown_exec); @@ -367,20 +391,21 @@ fn build_scan_branch( fn strip_row_preserving_wrappers(plan: &Arc) -> Option<&FilteredReadExec> { let mut current: &dyn ExecutionPlan = plan.as_ref(); loop { - if let Some(filtered_read) = current.as_any().downcast_ref::() { + if let Some(filtered_read) = current.downcast_ref::() { return Some(filtered_read); } let next: &Arc = - if let Some(inner) = current.as_any().downcast_ref::() { + if let Some(inner) = current.downcast_ref::() { inner.input() } else if let Some(inner) = { #[allow(deprecated)] - current.as_any().downcast_ref::() + current.downcast_ref::() } { inner.input() - } else if let Some(inner) = current.as_any().downcast_ref::() { + } else if let Some(inner) = current.downcast_ref::() { inner.input() - } else if let Some(proj) = current.as_any().downcast_ref::() { + } else { + let proj = current.downcast_ref::()?; // Only walk through projections that are row-preserving: every // output expression is a direct column reference back to the // input. (Empty projections trivially qualify — DataFusion uses @@ -390,7 +415,6 @@ fn strip_row_preserving_wrappers(plan: &Arc) -> Option<&Filte let identity = proj.expr().iter().all(|projection_expr| { projection_expr .expr - .as_any() .downcast_ref::() .is_some_and(|c| c.name() == input_schema.field(c.index()).name()) }); @@ -398,8 +422,6 @@ fn strip_row_preserving_wrappers(plan: &Arc) -> Option<&Filte return None; } proj.input() - } else { - return None; }; current = next.as_ref(); } @@ -434,7 +456,7 @@ fn is_count_star(af: &Arc) -> bool { if args.len() != 1 { return false; } - let Some(lit) = args[0].as_any().downcast_ref::() else { + let Some(lit) = args[0].downcast_ref::() else { return false; }; // `COUNT(NULL)` would always return 0; rule it out so we don't accidentally @@ -497,7 +519,7 @@ mod tests { fn plan_contains_pushdown(plan: &Arc) -> bool { let mut found = false; plan.apply(|node| { - if node.as_any().is::() { + if node.is::() { found = true; Ok(TreeNodeRecursion::Stop) } else { @@ -511,7 +533,7 @@ mod tests { fn plan_contains_union(plan: &Arc) -> bool { let mut found = false; plan.apply(|node| { - if node.as_any().is::() { + if node.is::() { found = true; Ok(TreeNodeRecursion::Stop) } else { @@ -587,6 +609,54 @@ mod tests { ); } + #[tokio::test] + async fn rule_fires_when_filter_is_scoped_to_fragment() { + let fixture = make_fixture().await; + let mut scanner = fixture.dataset.get_fragments()[1].scan(); + scanner.empty_project().unwrap().with_row_id(); + scanner.filter("ordered < 25").unwrap(); + + let (plan, count) = run_count(&mut scanner).await; + + assert_eq!(count, 10); + assert!( + plan_contains_pushdown(&plan), + "expected CountFromMaskExec for a fragment-scoped count: {}", + displayable(plan.as_ref()).indent(true) + ); + assert!( + !plan_contains_union(&plan), + "no union expected when the index covers the requested fragment, got: {}", + displayable(plan.as_ref()).indent(true) + ); + } + + #[tokio::test] + async fn count_matches_scan_for_stale_fragment_descriptor() { + let fixture = make_fixture().await; + let mut dataset = fixture.dataset.as_ref().clone(); + let stale_fragment = dataset.fragments()[0].clone(); + dataset.delete("ordered = 0").await.unwrap(); + let dataset = Arc::new(dataset); + + let mut scan = dataset.scan(); + scan.with_fragments(vec![stale_fragment.clone()]); + scan.filter("ordered < 10").unwrap(); + let scanned_rows = scan.try_into_batch().await.unwrap().num_rows() as i64; + + let mut count_scan = dataset.scan(); + count_scan.with_fragments(vec![stale_fragment]); + count_scan.filter("ordered < 10").unwrap(); + let (plan, count) = run_count(&mut count_scan).await; + + assert_eq!(count, scanned_rows); + assert!( + !plan_contains_pushdown(&plan), + "a stale fragment descriptor must retain the original scan plan: {}", + displayable(plan.as_ref()).indent(true) + ); + } + #[tokio::test] async fn rule_emits_split_plan_for_partial_index_coverage() { // Build index over 4 fragments, then append a 5th — the index now @@ -648,6 +718,43 @@ mod tests { "expected UnionExec for partial-coverage split, got: {}", displayable(plan.as_ref()).indent(true) ); + + let fragments = dataset.fragments(); + let mut indexed_fragment_scanner = dataset.scan(); + indexed_fragment_scanner + .with_fragments(vec![fragments[1].clone()]) + .filter("ordered < 100") + .unwrap(); + let (plan, count) = run_count(&mut indexed_fragment_scanner).await; + assert_eq!(count, 10); + assert!( + plan_contains_pushdown(&plan), + "expected pushdown when the index covers the requested fragment: {}", + displayable(plan.as_ref()).indent(true) + ); + assert!( + !plan_contains_union(&plan), + "unindexed fragments outside the requested scope must not add a scan branch: {}", + displayable(plan.as_ref()).indent(true) + ); + + let mut mixed_fragment_scanner = dataset.scan(); + mixed_fragment_scanner + .with_fragments(vec![fragments[1].clone(), fragments[4].clone()]) + .filter("ordered < 100") + .unwrap(); + let (plan, count) = run_count(&mut mixed_fragment_scanner).await; + assert_eq!(count, 20); + assert!( + plan_contains_pushdown(&plan), + "expected pushdown for the indexed requested fragment: {}", + displayable(plan.as_ref()).indent(true) + ); + assert!( + plan_contains_union(&plan), + "expected a scan branch for the unindexed requested fragment: {}", + displayable(plan.as_ref()).indent(true) + ); } #[tokio::test] @@ -730,6 +837,19 @@ mod tests { "rule should fire under stable row IDs with a filter, got plan: {}", displayable(plan.as_ref()).indent(true) ); + + let mut fragment_scanner = dataset.scan(); + fragment_scanner + .with_fragments(vec![dataset.fragments()[1].clone()]) + .filter("ordered >= 0") + .unwrap(); + let (plan, count) = run_count(&mut fragment_scanner).await; + assert_eq!(count, 9); + assert!( + plan_contains_pushdown(&plan), + "rule should push down a fragment-scoped count under stable row IDs: {}", + displayable(plan.as_ref()).indent(true) + ); } #[tokio::test] diff --git a/rust/lance/src/io/exec/filter.rs b/rust/lance/src/io/exec/filter.rs index 3a36f8d6712..71f1a5b2b4a 100644 --- a/rust/lance/src/io/exec/filter.rs +++ b/rust/lance/src/io/exec/filter.rs @@ -48,10 +48,6 @@ impl ExecutionPlan for LanceFilterExec { "LanceFilterExec" } - fn as_any(&self) -> &dyn std::any::Any { - self - } - fn properties(&self) -> &Arc { self.filter.properties() } @@ -71,7 +67,6 @@ impl ExecutionPlan for LanceFilterExec { // Rewrap the result in a LanceFilterExec to preserve the logical expression let new_filter_plan = self.filter.clone().with_new_children(children)?; let new_filter = new_filter_plan - .as_any() .downcast_ref::() .expect("FilterExec::with_new_children should return FilterExec") .clone(); @@ -93,7 +88,7 @@ impl ExecutionPlan for LanceFilterExec { self.filter.metrics() } - fn partition_statistics(&self, partition: Option) -> DataFusionResult { + fn partition_statistics(&self, partition: Option) -> DataFusionResult> { self.filter.partition_statistics(partition) } diff --git a/rust/lance/src/io/exec/filtered_read.rs b/rust/lance/src/io/exec/filtered_read.rs index 059fa4e2b36..4c9dc743f3d 100644 --- a/rust/lance/src/io/exec/filtered_read.rs +++ b/rust/lance/src/io/exec/filtered_read.rs @@ -1,20 +1,24 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright The Lance Authors -use std::any::Any; use std::collections::{BTreeMap, HashMap}; -use std::pin::Pin; use std::sync::Mutex; use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; -use std::{ops::Range, sync::Arc}; +use std::task::Poll; +use std::{ + ops::{Range, RangeInclusive}, + sync::Arc, +}; -use arrow_array::RecordBatch; -use arrow_schema::SchemaRef; +use arrow_array::cast::AsArray; +use arrow_array::types::UInt64Type; +use arrow_array::{Array, BooleanArray, RecordBatch, UInt32Array}; +use arrow_schema::{Schema as ArrowSchema, SchemaRef}; use datafusion::common::runtime::SpawnedTask; use datafusion::common::stats::Precision; use datafusion::error::{DataFusionError, Result as DataFusionResult}; use datafusion::execution::{SendableRecordBatchStream, TaskContext}; use datafusion::physical_plan::metrics::ExecutionPlanMetricsSet; -use datafusion::physical_plan::stream::RecordBatchStreamAdapter; +use datafusion::physical_plan::stream::{RecordBatchReceiverStream, RecordBatchStreamAdapter}; use datafusion::physical_plan::{ DisplayAs, DisplayFormatType, ExecutionPlan, PlanProperties, execution_plan::{Boundedness, EmissionType}, @@ -31,7 +35,9 @@ use lance_core::datatypes::OnMissing; use lance_core::utils::deletion::DeletionVector; use lance_core::utils::futures::FinallyStreamExt; use lance_core::utils::tokio::get_num_compute_intensive_cpus; -use lance_core::{Error, Result, datatypes::Projection}; +use lance_core::{ + Error, ROW_ADDR, ROW_ADDR_FIELD, ROW_ID, ROW_ID_FIELD, Result, datatypes::Projection, +}; use lance_datafusion::planner::Planner; use lance_datafusion::utils::{ ExecutionPlanMetricsSetExt, FRAGMENTS_SCANNED_METRIC, RANGES_SCANNED_METRIC, @@ -41,7 +47,8 @@ use lance_file::reader::FileReaderOptions; use lance_index::scalar::expression::FilterPlan; use lance_io::scheduler::{ScanScheduler, SchedulerConfig}; use lance_select::{ - IndexExprResult, RowAddrSelection, RowAddrTreeMap, bitmap_to_ranges, ranges_to_bitmap, + IndexExprResult, RowAddrMask, RowAddrSelection, RowAddrTreeMap, bitmap_to_ranges, + ranges_to_bitmap, result::IndexExprResultWireFormat, }; use lance_table::format::Fragment; use lance_table::rowids::RowIdSequence; @@ -57,9 +64,17 @@ use crate::dataset::scanner::{ BATCH_SIZE_FALLBACK, DEFAULT_FRAGMENT_READAHEAD, get_default_batch_size, get_default_io_buffer_size_override, }; +use crate::dataset::versions; use super::utils::IoMetrics; +fn public_blob_v2_binary_projection_schema(projection: &Projection) -> SchemaRef { + let schema = projection.to_schema(); + let schema = crate::dataset::blob::public_blob_v2_binary_output_schema(&schema); + let schema: ArrowSchema = (&schema).into(); + Arc::new(schema) +} + #[derive(Debug)] pub struct EvaluatedIndex { index_result: IndexExprResult, @@ -86,6 +101,18 @@ impl EvaluatedIndex { applicable_fragments, }) } + + /// Block `rows` (stale overlay row addresses) from the index result so the index never + /// emits them. Their fragments stay in the covered set, so non-stale rows keep the index; + /// the blocked rows are re-evaluated against their current (overlay-merged) values on a + /// separate targeted take path built by the scanner. + fn without_rows(mut self, block_list: &RowAddrTreeMap) -> Self { + self.index_result.upper = + std::mem::take(&mut self.index_result.upper).also_block(block_list.clone()); + self.index_result.lower = + std::mem::take(&mut self.index_result.lower).also_block(block_list.clone()); + self + } } /// A fragment along with ranges of row offsets to read @@ -328,6 +355,116 @@ struct FilteredReadStream { threading_mode: FilteredReadThreadingMode, /// Range to apply to the result stream if not already pushed down in planning phase scan_range_after_filter: Option>, + /// Fragments planned non-empty, and their total planned rows; the output + /// side uses these to detect take-shaped plans (batch size resolves at + /// execute time, so the detection lives there too) + touched_fragments: usize, + planned_rows: u64, +} + +/// Below this many fragments there are too few handoffs to be worth +/// consolidating +const CONSOLIDATE_MIN_FRAGMENTS: usize = 8; + +/// Above this per-fragment average, batches are big enough to amortize +/// their handoff +const CONSOLIDATE_MAX_AVG_PLANNED_ROWS_PER_FRAGMENT: u64 = 1024; + +/// Pump a take-shaped read on a spawned task, handing the consumer +/// consolidated batches. Inline polling would otherwise execute the +/// per-batch pipeline work on the consumer, which serializes concurrent +/// small reads. +fn consolidated_stream( + inner: SendableRecordBatchStream, + target: usize, +) -> SendableRecordBatchStream { + let mut builder = RecordBatchReceiverStream::builder(inner.schema(), 4); + let tx = builder.tx(); + builder.spawn(async move { + let mut stream = coalesce_batches(inner, target).boxed(); + while let Some(item) = stream.next().await { + if tx.send(item).await.is_err() { + // Receiver dropped: the query was cancelled + break; + } + } + Ok(()) + }); + builder.build() +} + +/// Merge batches up to `target` rows; batches already at the target pass +/// through whole (never split). Order is preserved. +pub fn coalesce_batches( + input: SendableRecordBatchStream, + target: usize, +) -> impl Stream> { + struct Coalescer { + input: SendableRecordBatchStream, + schema: SchemaRef, + target: usize, + buffered: Vec, + buffered_rows: usize, + exhausted: bool, + } + + impl Coalescer { + fn ready_to_emit(&self) -> bool { + self.buffered_rows >= self.target || (self.exhausted && !self.buffered.is_empty()) + } + + fn buffer(&mut self, batch: RecordBatch) { + self.buffered_rows += batch.num_rows(); + self.buffered.push(batch); + } + + fn emit(&mut self) -> DataFusionResult { + self.buffered_rows = 0; + if self.buffered.len() > 1 { + let batch = arrow::compute::concat_batches(&self.schema, self.buffered.iter())?; + self.buffered.clear(); + Ok(batch) + } else { + self.buffered.pop().ok_or_else(|| { + DataFusionError::Internal( + "coalesce_batches emitted with an empty buffer".to_string(), + ) + }) + } + } + } + + let schema = input.schema(); + let coalescer = Coalescer { + input, + schema, + target, + buffered: Vec::new(), + buffered_rows: 0, + exhausted: false, + }; + futures::stream::try_unfold(coalescer, |mut this| async move { + loop { + if this.ready_to_emit() { + return Ok(Some((this.emit()?, this))); + } + if this.exhausted { + return Ok(None); + } + match this.input.try_next().await? { + Some(batch) if batch.num_rows() >= this.target && !this.buffered.is_empty() => { + // Emit the partial buffer on its own; the large batch + // then passes through whole on the next iteration + let out = this.emit()?; + this.buffer(batch); + return Ok(Some((out, this))); + } + Some(batch) if batch.num_rows() > 0 => this.buffer(batch), + Some(_) => {} + None => this.exhausted = true, + } + } + }) } impl std::fmt::Debug for FilteredReadStream { @@ -337,16 +474,22 @@ impl std::fmt::Debug for FilteredReadStream { } impl FilteredReadStream { - /// Create a new FilteredReadStream from a pre-computed internal plan + /// Create a new FilteredReadStream from a pre-computed internal plan. + /// Fragment handles are constructed I/O-free from the manifest + /// descriptors, only for the fragments the plan selects. A `None` + /// scheduler is created here; the row-stream path injects its per-query + /// shared one and a per-batch priority offset. #[instrument(name = "init_filtered_read_stream", skip_all)] - async fn try_new( + fn try_new( dataset: Arc, options: FilteredReadOptions, - metrics: &ExecutionPlanMetricsSet, + global_metrics: Arc, plan: FilteredReadInternalPlan, - ) -> DataFusionResult { - let global_metrics = Arc::new(FilteredReadGlobalMetrics::new(metrics)); - + scan_scheduler: Option>, + priority_offset: Option, + ) -> Self { + let scan_scheduler = + scan_scheduler.unwrap_or_else(|| Self::make_scan_scheduler(&dataset, &options)); let threading_mode = options.threading_mode; let io_parallelism = dataset.object_store.io_parallelism(); @@ -355,62 +498,36 @@ impl FilteredReadStream { .unwrap_or_else(|| (*DEFAULT_FRAGMENT_READAHEAD).unwrap_or(io_parallelism * 2)) .max(1); - let fragments = options + let fragment_descriptors = options .fragments .clone() .unwrap_or_else(|| dataset.fragments().clone()); log::debug!( "Filtered read on {} fragments with frag_readahead={} and io_parallelism={}", - fragments.len(), + fragment_descriptors.len(), fragment_readahead, io_parallelism ); - // Ideally we don't need to collect here but if we don't we get "implementation of FnOnce is - // not general enough" false positives from rustc - let frag_futs = fragments - .iter() - .map(|frag| { - Result::Ok(Self::load_fragment( - dataset.clone(), - frag.clone(), - options.with_deleted_rows, - )) - }) - .collect::>(); - let loaded_fragments = futures::stream::iter(frag_futs) - // Cannot use unordered because we need to populate logical_offset based on user-provided order - .try_buffered(io_parallelism) - .try_collect::>() - .await?; - - let output_schema = Arc::new(options.projection.to_arrow_schema()); - - let obj_store = dataset.object_store.clone(); - // Explicit options take precedence; otherwise fall back to the - // LANCE_DEFAULT_IO_BUFFER_SIZE env var if set; otherwise max_bandwidth. - let scheduler_config = if let Some(io_buffer_size_bytes) = options - .io_buffer_size_bytes - .or_else(get_default_io_buffer_size_override) - { - SchedulerConfig::new(io_buffer_size_bytes) - } else { - SchedulerConfig::max_bandwidth(obj_store.as_ref()) - }; - let scan_scheduler = ScanScheduler::new(obj_store, scheduler_config); + let output_schema = public_blob_v2_binary_projection_schema(&options.projection); // Get scan_range_after_filter from the plan let scan_range_after_filter = plan.scan_range_after_filter.clone(); // Convert plan to scoped fragments for I/O - let scoped_fragments = Self::plan_to_scoped_fragments( + let mut scoped_fragments = Self::plan_to_scoped_fragments( &plan, - &loaded_fragments, + &fragment_descriptors, &dataset, &options, scan_scheduler.clone(), ); + if let Some(priority_offset) = priority_offset.filter(|offset| *offset != 0) { + for scoped in &mut scoped_fragments { + scoped.priority = scoped.priority.saturating_add(priority_offset); + } + } let global_metrics_clone = global_metrics.clone(); @@ -420,8 +537,10 @@ impl FilteredReadStream { move |scoped_fragment| { let metrics = global_metrics_clone.clone(); let limit = scan_range_after_filter.as_ref().map(|r| r.end); + let dataset = dataset.clone(); SpawnedTask::spawn( - Self::read_fragment(scoped_fragment, metrics, limit).in_current_span(), + Self::read_fragment(dataset, scoped_fragment, metrics, limit) + .in_current_span(), ) .map(|thread_result| thread_result.unwrap()) } @@ -429,7 +548,23 @@ impl FilteredReadStream { .buffered(fragment_readahead); let task_stream = fragment_streams.try_flatten().boxed(); - Ok(Self { + // A batch never spans fragments, so a plan touching many fragments + // with few rows each emits one tiny batch per fragment. Fragments + // planned empty produce no batch and don't count. Filtered scans + // stay dense here: their planned rows are a pre-refine upper bound. + let (touched_fragments, planned_rows) = + plan.rows + .values() + .fold((0usize, 0u64), |(fragments, rows), ranges| { + let fragment_rows: u64 = + ranges.iter().map(|range| range.end - range.start).sum(); + if fragment_rows > 0 { + (fragments + 1, rows + fragment_rows) + } else { + (fragments, rows) + } + }); + Self { output_schema, task_stream: Arc::new(AsyncMutex::new(task_stream)), scan_scheduler, @@ -437,7 +572,62 @@ impl FilteredReadStream { active_partitions_counter: Arc::new(AtomicUsize::new(0)), threading_mode, scan_range_after_filter, - }) + touched_fragments, + planned_rows, + } + } + + /// Drain the entire read into batches (used by the row-stream path, + /// which is the stream's only consumer and records metrics per batch) + async fn collect_all(&self, decode_parallelism: usize) -> Result> { + let mut task_stream = self.task_stream.lock().await; + (&mut *task_stream) + .try_buffered(decode_parallelism) + .try_collect() + .await + } + + async fn load_all_fragments( + dataset: &Arc, + options: &FilteredReadOptions, + ) -> Result> { + let io_parallelism = dataset.object_store.io_parallelism(); + let fragments = options + .fragments + .clone() + .unwrap_or_else(|| dataset.fragments().clone()); + // Ideally we don't need to collect here but if we don't we get "implementation of FnOnce is + // not general enough" false positives from rustc + let frag_futs = fragments + .iter() + .map(|frag| { + Result::Ok(Self::load_fragment( + dataset.clone(), + frag.clone(), + options.with_deleted_rows, + )) + }) + .collect::>(); + futures::stream::iter(frag_futs) + // Cannot use unordered because we need to populate logical_offset based on user-provided order + .try_buffered(io_parallelism) + .try_collect::>() + .await + } + + /// Create the I/O scheduler for a read (explicit option → env override → + /// max bandwidth) + fn make_scan_scheduler(dataset: &Dataset, options: &FilteredReadOptions) -> Arc { + let obj_store = dataset.object_store.clone(); + let scheduler_config = if let Some(io_buffer_size_bytes) = options + .io_buffer_size_bytes + .or_else(get_default_io_buffer_size_override) + { + SchedulerConfig::new(io_buffer_size_bytes) + } else { + SchedulerConfig::max_bandwidth(obj_store.as_ref()) + }; + ScanScheduler::new(obj_store, scheduler_config) } async fn load_fragment( @@ -640,10 +830,13 @@ impl FilteredReadStream { } } + /// Handles are constructed here, I/O-free, only for the fragments the + /// plan selects; priority is the fragment's position in the candidate + /// list so a sparse plan keeps the original I/O order. fn plan_to_scoped_fragments( plan: &FilteredReadInternalPlan, - fragments: &[LoadedFragment], - dataset: &Dataset, + fragments: &[Fragment], + dataset: &Arc, options: &FilteredReadOptions, scan_scheduler: Arc, ) -> Vec { @@ -659,7 +852,7 @@ impl FilteredReadStream { let mut scoped_fragments = Vec::new(); for (priority, fragment) in fragments.iter().enumerate() { - let fragment_id = fragment.fragment.id() as u32; + let fragment_id = fragment.id as u32; // Check if this fragment is in the plan if let Some(ranges) = plan.rows.get(&fragment_id) { @@ -671,7 +864,7 @@ impl FilteredReadStream { let filter = plan.filters.get(&fragment_id).map(|f| (**f).clone()); scoped_fragments.push(ScopedFragmentRead { - fragment: fragment.fragment.clone(), + fragment: Arc::new(FileFragment::new(dataset.clone(), fragment.clone())), ranges: ranges.clone(), projection: projection.clone(), with_deleted_rows: options.with_deleted_rows, @@ -1076,11 +1269,13 @@ impl FilteredReadStream { // Reads a single fragment into a stream of batch tasks #[instrument(name = "read_fragment", level = "debug", skip_all)] async fn read_fragment( + dataset: Arc, mut fragment_read_task: ScopedFragmentRead, global_metrics: Arc, fragment_soft_limit: Option, ) -> Result>> { - let output_schema = Arc::new(fragment_read_task.projection.to_arrow_schema()); + let output_schema = + public_blob_v2_binary_projection_schema(fragment_read_task.projection.as_ref()); if let Some(filter) = &fragment_read_task.filter { let filter_cols = Planner::column_names_in_expr(filter); @@ -1095,10 +1290,22 @@ impl FilteredReadStream { } } - let read_schema = fragment_read_task.projection.to_bare_schema(); + let output_read_schema = Arc::new(fragment_read_task.projection.to_schema()); + let bare_read_schema = fragment_read_task.projection.to_bare_schema(); + let materialize_blob_v2_binary = + crate::dataset::blob::schema_has_blob_v2_binary_view(&bare_read_schema); + let read_schema = if materialize_blob_v2_binary { + crate::dataset::blob::blob_v2_descriptor_schema(&bare_read_schema) + } else { + bare_read_schema + }; + let mut frag_read_config = fragment_read_task.frag_read_config(); + if materialize_blob_v2_binary { + frag_read_config = frag_read_config.with_row_address(true); + } let mut fragment_reader = fragment_read_task .fragment - .open(&read_schema, fragment_read_task.frag_read_config()) + .open(&read_schema, frag_read_config) .await?; if fragment_read_task.with_deleted_rows { @@ -1112,8 +1319,9 @@ impl FilteredReadStream { let physical_filter = fragment_read_task .filter .map(|filter| { - let planner = - Planner::new(Arc::new(fragment_read_task.projection.to_arrow_schema())); + let planner = Planner::new(public_blob_v2_binary_projection_schema( + fragment_read_task.projection.as_ref(), + )); planner.create_physical_expr(&filter) }) .transpose()?; @@ -1136,7 +1344,7 @@ impl FilteredReadStream { let global_metrics = global_metrics.clone(); let fragment_counted = fragment_counted.clone(); let range_tracker = range_tracker.clone(); - batch_fut + let batch_fut = batch_fut .inspect_ok(move |batch| { let num_rows = batch.num_rows(); global_metrics.rows_scanned.add(num_rows); @@ -1151,7 +1359,23 @@ impl FilteredReadStream { global_metrics.ranges_scanned.add(additional_ranges); } }) - .boxed() + .boxed(); + if materialize_blob_v2_binary { + let dataset = dataset.clone(); + let output_read_schema = output_read_schema.clone(); + batch_fut + .and_then(move |batch| async move { + crate::dataset::blob::materialize_blob_v2_binary_batch( + &dataset, + output_read_schema.as_ref(), + batch, + ) + .await + }) + .boxed() + } else { + batch_fut + } }) .zip(futures::stream::repeat(( physical_filter.clone(), @@ -1159,12 +1383,11 @@ impl FilteredReadStream { ))) .map(|(batch_fut, args)| Self::wrap_with_filter(batch_fut, args.0, args.1)); - let result: Pin> + Send>> = - if let Some(limit) = fragment_soft_limit { - Box::pin(Self::apply_soft_limit(fragment_stream, limit)) - } else { - Box::pin(fragment_stream) - }; + let result = if let Some(limit) = fragment_soft_limit { + Self::apply_soft_limit(fragment_stream, limit).boxed() + } else { + fragment_stream.boxed() + }; Ok(result) } @@ -1270,7 +1493,7 @@ pub struct FilteredReadOptions { pub scan_range_before_filter: Option>, /// The range of rows to read after applying the filter. pub scan_range_after_filter: Option>, - /// Include deleted rows in the scan + /// Include deleted rows in the scan; they are returned with a null row id pub with_deleted_rows: bool, /// The maximum number of rows per batch pub batch_size: Option, @@ -1295,6 +1518,12 @@ pub struct FilteredReadOptions { pub io_buffer_size_bytes: Option, /// If true, skip fragments that are not covered by the scalar index result. pub only_indexed_fragments: bool, + /// Row addresses whose index entries may be stale because an overlay committed after the + /// index was built touches an indexed field. They are blocked from the index result so the + /// index never emits them; the scanner re-evaluates just these rows against their current + /// (overlay-merged) values on a targeted take path. Their fragments stay in the covered set, + /// so non-stale rows keep the index. `None` on the common no-overlay fast path. + pub overlay_block: Option, } impl FilteredReadOptions { @@ -1324,12 +1553,20 @@ impl FilteredReadOptions { full_filter: None, io_buffer_size_bytes: None, only_indexed_fragments: false, + overlay_block: None, threading_mode: FilteredReadThreadingMode::OnePartitionMultipleThreads( get_num_compute_intensive_cpus(), ), } } + /// Block the given stale overlay row addresses (see the `overlay_block` field) from the + /// scalar index result so the index never emits them. + pub fn with_overlay_block(mut self, block: RowAddrMask) -> Self { + self.overlay_block = Some(block); + self + } + /// Include deleted rows in the scan /// /// This is currently only supported if there is no scan_range specified @@ -1482,6 +1719,19 @@ impl FilteredReadOptions { self.only_indexed_fragments = true; self } + + /// Specify the threading mode to use for the scan. + /// + /// This controls how decode work is parallelized. For the default single-partition + /// scan, the parameter of [`FilteredReadThreadingMode::OnePartitionMultipleThreads`] + /// bounds how many batch-decode tasks are buffered in flight (via `try_buffered`). + /// + /// The parallelism must be greater than 0. A value of 0 is rejected by + /// [`FilteredReadExec::try_new`]. + pub fn with_threading_mode(mut self, threading_mode: FilteredReadThreadingMode) -> Self { + self.threading_mode = threading_mode; + self + } } /// A plan node that reads a dataset, applying an optional filter and projection. @@ -1504,7 +1754,7 @@ pub struct FilteredReadExec { options: FilteredReadOptions, properties: Arc, metrics: ExecutionPlanMetricsSet, - index_input: Option>, + input: RowSelector, // Precomputed internal plan plan: Arc>, // When execute is first called we will initialize the FilteredReadStream. In order to support @@ -1512,6 +1762,49 @@ pub struct FilteredReadExec { running_stream: Arc>>, } +/// Describes which rows a [`FilteredReadExec`] should read +#[derive(Debug)] +enum RowSelector { + /// Every live row of the dataset (no input plan) + AllRows, + /// A set of rows: one serialized [`IndexExprResult`] batch. Output is in + /// storage order and deduplicated. + RowSet(Arc), + /// A stream of rows: record batches with a `_rowid`/`_rowaddr` column + /// and other payload columns (just carried) + RowStream(Arc), +} + +impl RowSelector { + fn row_set_plan(&self) -> Option<&Arc> { + match self { + Self::RowSet(plan) => Some(plan), + _ => None, + } + } + + fn child(&self) -> Option<&Arc> { + match self { + Self::AllRows => None, + Self::RowSet(plan) => Some(plan), + Self::RowStream(source) => Some(&source.plan), + } + } +} + +/// State derived at construction for a row-stream source +#[derive(Debug)] +struct RowStreamSource { + plan: Arc, + /// The stream column identifying rows: [`ROW_ID`] or [`ROW_ADDR`] + key_column: &'static str, + /// Options for the internal fragment read; carries the projection that + /// reflects the actual columns to read (plus the alignment key column) + read_options: FilteredReadOptions, + /// The schema for newly read columns + new_fields_schema: SchemaRef, +} + /// Public plan for distributed execution - uses bitmap for flexibility #[derive(Clone)] pub struct FilteredReadPlan { @@ -1558,11 +1851,166 @@ impl FilteredReadInternalPlan { } impl FilteredReadExec { + /// Create a new filtered read pub fn try_new( + dataset: Arc, + options: FilteredReadOptions, + input: Option>, + ) -> Result { + match input { + Some(input) if Self::is_index_query_schema(input.schema().as_ref()) => { + Self::try_new_scan(dataset, options, Some(input)) + } + Some(input) => Self::try_new_row_stream(dataset, options, input), + None => Self::try_new_scan(dataset, options, None), + } + } + + /// Whether `schema` is one of the serialized [`IndexExprResult`] wire + /// layouts (see [`IndexExprResultWireFormat`]) + fn is_index_query_schema(schema: &arrow_schema::Schema) -> bool { + [ + IndexExprResultWireFormat::TwoMask, + IndexExprResultWireFormat::ThreeVariant, + ] + .iter() + .any(|format| schema.fields() == format.schema().fields()) + } + + /// The input columns that carry through to the output: identity columns + /// appear iff their flag is requested, ordinary columns always carry + fn carried_schema(input_schema: &arrow_schema::Schema, projection: &Projection) -> SchemaRef { + Arc::new(arrow_schema::Schema::new( + input_schema + .fields() + .iter() + .filter(|f| { + (f.name() != ROW_ID || projection.with_row_id) + && (f.name() != ROW_ADDR || projection.with_row_addr) + }) + .cloned() + .collect::>(), + )) + } + + /// Construct a read over a row-stream source + fn try_new_row_stream( + dataset: Arc, + options: FilteredReadOptions, + input: Arc, + ) -> Result { + versions::validate_row_stream_read( + dataset.manifest().data_storage_format.lance_file_format(), + )?; + if options.refine_filter.is_some() || options.full_filter.is_some() { + return Err(Error::invalid_input_source( + "filters are not supported when taking rows from an input plan".into(), + )); + } + // A limit is safer to apply upstream, on the cheap keyed rows + if options.scan_range_before_filter.is_some() || options.scan_range_after_filter.is_some() { + return Err(Error::invalid_input_source( + "scan ranges are not supported when taking rows from an input plan".into(), + )); + } + // Row-stream reads do not support deleted rows yet; deleted rows are + // excluded from the output by default + if options.with_deleted_rows || options.only_indexed_fragments { + return Err(Error::invalid_input_source( + "with_deleted_rows / only_indexed_fragments are not supported when taking rows from an input plan".into(), + )); + } + let input_schema = input.schema(); + let key_column = if input_schema.column_with_name(ROW_ID).is_some() { + ROW_ID + } else if input_schema.column_with_name(ROW_ADDR).is_some() { + ROW_ADDR + } else { + return Err(Error::invalid_input_source( + format!( + "a row-stream input plan must have a column named '{}' or '{}'", + ROW_ADDR, ROW_ID + ) + .into(), + )); + }; + + let fields_to_read = options + .projection + .clone() + .subtract_arrow_schema(input_schema.as_ref(), OnMissing::Ignore)?; + let synthesize_row_id = fields_to_read.with_row_id; + let synthesize_row_addr = fields_to_read.with_row_addr; + if !fields_to_read.has_data_fields() && !synthesize_row_id && !synthesize_row_addr { + return Err(Error::invalid_input_source( + "the input plan already contains every projected field; there is nothing to read" + .into(), + )); + } + + let carried_schema = Self::carried_schema(input_schema.as_ref(), &options.projection); + + // Output = carried columns ⊕ fetched fields ⊕ synthesized identity + let output_schema = Arc::new(arrow_schema::Schema::from( + &super::TakeExec::calculate_output_schema( + dataset.schema(), + carried_schema.as_ref(), + &fields_to_read, + ), + )); + + // Partitioning and emission behavior follow the input + let properties = Arc::new( + input + .properties() + .as_ref() + .clone() + .with_eq_properties(EquivalenceProperties::new(output_schema)), + ); + + let bare_schema = arrow_schema::Schema::from(&fields_to_read.to_bare_schema()); + let mut new_fields = bare_schema.fields().iter().cloned().collect::>(); + if synthesize_row_id { + new_fields.push(Arc::new(ROW_ID_FIELD.clone())); + } + if synthesize_row_addr { + new_fields.push(Arc::new(ROW_ADDR_FIELD.clone())); + } + let new_fields_schema = Arc::new(arrow_schema::Schema::new(new_fields)); + + // fields_to_read keeps the synthesis flags; add the key column on top + let mut read_options = options.clone(); + read_options.projection = if key_column == ROW_ID { + fields_to_read.with_row_id() + } else { + fields_to_read.with_row_addr() + }; + + Ok(Self { + dataset, + options, + properties, + metrics: ExecutionPlanMetricsSet::new(), + input: RowSelector::RowStream(Arc::new(RowStreamSource { + plan: input, + key_column, + read_options, + new_fields_schema, + })), + plan: Arc::new(OnceCell::new()), + running_stream: Arc::new(AsyncMutex::new(None)), + }) + } + + fn try_new_scan( dataset: Arc, mut options: FilteredReadOptions, index_input: Option>, ) -> Result { + let input = match index_input { + Some(plan) => RowSelector::RowSet(plan), + None => RowSelector::AllRows, + }; if options.with_deleted_rows { // Ensure we have the row id column if with_deleted_rows is set options.projection = options.projection.with_row_id(); @@ -1573,11 +2021,28 @@ impl FilteredReadExec { .into())); } + // A parallelism of 0 would cause `try_buffered(0)` to hang forever instead of erroring + match options.threading_mode { + FilteredReadThreadingMode::OnePartitionMultipleThreads(0) => { + return Err(Error::invalid_input_source( + "FilteredReadThreadingMode::OnePartitionMultipleThreads must be greater than 0, got 0" + .into(), + )); + } + FilteredReadThreadingMode::MultiplePartitions(0) => { + return Err(Error::invalid_input_source( + "FilteredReadThreadingMode::MultiplePartitions must be greater than 0, got 0" + .into(), + )); + } + _ => {} + } + if options.scan_range_after_filter.is_some() { // Validate that there's a filter when using scan_range_after_filter if options.full_filter.is_none() && options.refine_filter.is_none() - && index_input.is_none() + && input.row_set_plan().is_none() { return Err(Error::invalid_input_source("scan_range_after_filter requires a filter to be applied. Use scan_range_before_filter for unfiltered scans." .into())); @@ -1595,7 +2060,7 @@ impl FilteredReadExec { )); } } - let output_schema = Arc::new(options.projection.to_arrow_schema()); + let output_schema = public_blob_v2_binary_projection_schema(&options.projection); let num_partitions = match options.threading_mode { FilteredReadThreadingMode::OnePartitionMultipleThreads(_) => 1, FilteredReadThreadingMode::MultiplePartitions(n) => n, @@ -1616,7 +2081,7 @@ impl FilteredReadExec { properties, running_stream: Arc::new(AsyncMutex::new(None)), metrics, - index_input, + input, plan: Arc::new(OnceCell::new()), }) } @@ -1675,9 +2140,15 @@ impl FilteredReadExec { let index_search_result = index_search.next().await.ok_or_else(|| { Error::internal("Index search did not yield any results".to_string()) })??; - evaluated_index = Some(Arc::new(EvaluatedIndex::try_from_arrow( - &index_search_result, - )?)); + let mut idx = EvaluatedIndex::try_from_arrow(&index_search_result)?; + // `overlay_block` is always constructed as a block list (see + // `Scanner::stale_rows_block_mask`), so `block_list()` is always `Some`. + if let Some(block_list) = + options.overlay_block.as_ref().and_then(|b| b.block_list()) + { + idx = idx.without_rows(block_list); + } + evaluated_index = Some(Arc::new(idx)); } // Load fragments to compute the plan @@ -1703,7 +2174,9 @@ impl FilteredReadExec { .try_collect::>() .await?; - // Plan the scan + // Plan the scan; the metadata loaded here drops when planning + // finishes — stream construction rebuilds I/O-free handles + // from the manifest descriptors Ok(FilteredReadStream::plan_scan( &loaded_fragments, &evaluated_index, @@ -1715,11 +2188,18 @@ impl FilteredReadExec { /// Get the existing plan or create it if it doesn't exist pub async fn get_or_create_plan(&self, ctx: Arc) -> Result { + if self.row_stream_input().is_some() { + return Err(Error::not_supported_source( + "a FilteredReadExec with a row-stream source does not have a precomputable plan" + .to_string() + .into(), + )); + } let internal_plan = Self::get_or_create_plan_impl( &self.plan, self.dataset.clone(), &self.options, - self.index_input.as_ref(), + self.input.row_set_plan(), 0, ctx, ) @@ -1746,12 +2226,13 @@ impl FilteredReadExec { n.min(target_partitions).max(1), ); } + let batch_size_rows = options.batch_size; let batch_size_bytes = options .file_reader_options .as_ref() .and_then(|o| o.batch_size_bytes); let metrics = self.metrics.clone(); - let index_input = self.index_input.clone(); + let index_input = self.input.row_set_plan().cloned(); let plan_cell = self.plan.clone(); let stream = futures::stream::once(async move { @@ -1769,16 +2250,42 @@ impl FilteredReadExec { ) .await .map_err(|e| DataFusionError::External(e.into()))?; - let new_running_stream = - FilteredReadStream::try_new(dataset, options, &metrics, plan.clone()) - .await - .map_err(|e| DataFusionError::External(e.into()))?; + let new_running_stream = FilteredReadStream::try_new( + dataset, + options, + Arc::new(FilteredReadGlobalMetrics::new(&metrics)), + plan.clone(), + None, + None, + ); let first_stream = new_running_stream.get_stream(&metrics, partition); *running_stream = Some(new_running_stream); first_stream }; - let stream: SendableRecordBatchStream = match batch_size_bytes { - Some(target) => { + // Only masked reads consolidate; plain scans keep their batch + // boundaries, and the byte-based rechunk merges on its own + let consolidate = if index_input.is_some() && batch_size_bytes.is_none() { + running_stream.as_ref().and_then(|running| { + // Explicit option → lance env default → session batch size + let batch_target_rows = batch_size_rows + .map(|batch_size| batch_size as usize) + .or_else(get_default_batch_size) + .unwrap_or_else(|| context.session_config().batch_size()); + let is_sparse_plan = batch_target_rows > 0 + && running.touched_fragments >= CONSOLIDATE_MIN_FRAGMENTS + && running.planned_rows + < running.touched_fragments as u64 + * CONSOLIDATE_MAX_AVG_PLANNED_ROWS_PER_FRAGMENT; + is_sparse_plan.then_some(batch_target_rows) + }) + } else { + None + }; + drop(running_stream); + + let stream = match (consolidate, batch_size_bytes) { + (Some(target), _) => consolidated_stream(inner, target), + (None, Some(bytes)) => { let schema = inner.schema(); Box::pin(RecordBatchStreamAdapter::new( schema.clone(), @@ -1786,11 +2293,11 @@ impl FilteredReadExec { inner, schema, 0, - target as usize, + bytes as usize, ), )) } - None => inner, + (None, None) => inner, }; DataFusionResult::::Ok(stream) }) @@ -1808,17 +2315,476 @@ impl FilteredReadExec { } pub fn index_input(&self) -> Option<&Arc> { - self.index_input.as_ref() + self.input.row_set_plan() + } + + pub fn row_stream_input(&self) -> Option<&Arc> { + match &self.input { + RowSelector::RowStream(source) => Some(&source.plan), + _ => None, + } } /// Return the pre-computed plan if one exists, without triggering initialization. pub fn plan(&self) -> Option { self.plan.get().map(|p| p.to_external_plan()) } + + fn execute_row_stream( + &self, + source: &Arc, + partition: usize, + context: Arc, + ) -> DataFusionResult { + let input_stream = source.plan.execute(partition, context)?; + let dataset = self.dataset.clone(); + let source = source.clone(); + let carried_schema = + Self::carried_schema(source.plan.schema().as_ref(), &self.options.projection); + let output_schema = self.schema(); + let metrics = self.metrics.clone(); + + let lazy_stream = futures::stream::once(async move { + let row_stream_read = Arc::new(RowStreamRead::new( + dataset, + source, + carried_schema, + output_schema, + &metrics, + partition, + )); + row_stream_read.apply(input_stream) + }) + .flatten(); + Ok(Box::pin(RecordBatchStreamAdapter::new( + self.schema(), + lazy_stream, + ))) + } +} + +/// How many batches run concurrently. Each batch's read already carries +/// the full fragment-readahead and decode parallelism, so a shallow pipeline +/// keeps the I/O pipe full; running every batch at once only multiplies that +/// into lock contention +const ROW_STREAM_CONCURRENT_BATCHES: usize = 4; + +/// Fragment metadata, loaded on the first batch and reused afterwards +struct StreamFragments { + /// All dataset (or scoped) fragments, in dataset order + fragments: Vec, + /// Fragment id → position in `fragments` + positions: HashMap, + /// Each fragment's row-id span, for skipping fragments a batch cannot + /// touch (None = empty fragment) + id_spans: Vec>>, +} + +impl StreamFragments { + fn get(&self, fragment_id: u32) -> Option<&LoadedFragment> { + self.positions + .get(&fragment_id) + .map(|position| &self.fragments[*position]) + } +} + +/// Executes a [`FilteredReadExec`] over a row-stream source +struct RowStreamRead { + dataset: Arc, + source: Arc, + /// The input columns that carry through to the output + carried_schema: SchemaRef, + output_schema: SchemaRef, + scan_scheduler: Arc, + loaded_fragments: OnceCell, + global_metrics: Arc, + baseline_metrics: BaselineMetrics, +} + +impl RowStreamRead { + fn new( + dataset: Arc, + source: Arc, + carried_schema: SchemaRef, + output_schema: SchemaRef, + metrics: &ExecutionPlanMetricsSet, + partition: usize, + ) -> Self { + let scan_scheduler = + FilteredReadStream::make_scan_scheduler(&dataset, &source.read_options); + Self { + dataset, + source, + carried_schema, + output_schema, + scan_scheduler, + loaded_fragments: OnceCell::new(), + global_metrics: Arc::new(FilteredReadGlobalMetrics::new(metrics)), + baseline_metrics: BaselineMetrics::new(metrics, partition), + } + } + + async fn load_fragments(&self) -> Result<&StreamFragments> { + self.loaded_fragments + .get_or_try_init(|| async { + let fragments = FilteredReadStream::load_all_fragments( + &self.dataset, + &self.source.read_options, + ) + .await?; + let positions = fragments + .iter() + .enumerate() + .map(|(position, fragment)| (fragment.fragment.id() as u32, position)) + .collect(); + let id_spans = fragments + .iter() + .map(|fragment| fragment.row_id_sequence.row_id_range()) + .collect(); + Ok(StreamFragments { + fragments, + positions, + id_spans, + }) + }) + .await + } + + /// Build a batch's read ranges directly from physical row addresses + fn plan_batch_from_addresses( + addrs: &RowAddrTreeMap, + fragments: &StreamFragments, + ) -> FilteredReadInternalPlan { + let mut rows: BTreeMap>> = BTreeMap::new(); + for (fragment_id, requested) in addrs.iter() { + // Unknown fragments (e.g. fully deleted) drop like stale keys + let Some(fragment) = fragments.get(*fragment_id) else { + continue; + }; + let requested = match requested { + RowAddrSelection::Full => vec![0..fragment.num_physical_rows], + RowAddrSelection::Partial(bitmap) => bitmap_to_ranges(bitmap), + }; + let valid = FilteredReadStream::full_frag_range( + fragment.num_physical_rows, + &fragment.deletion_vector, + ); + let matched = FilteredReadStream::intersect_ranges(&valid, &requested); + if !matched.is_empty() { + rows.insert(*fragment_id, matched); + } + } + FilteredReadInternalPlan { + rows, + filters: HashMap::new(), + scan_range_after_filter: None, + } + } + + /// Build a batch's read ranges by resolving stable row ids through the + /// fragments' row-id sequences + fn plan_batch_from_row_ids( + ids: RowAddrTreeMap, + keys: &arrow_array::PrimitiveArray, + fragments: &StreamFragments, + ) -> FilteredReadInternalPlan { + let mut rows: BTreeMap>> = BTreeMap::new(); + let (Some(min_key), Some(max_key)) = (arrow::compute::min(keys), arrow::compute::max(keys)) + else { + // Every key is null + return FilteredReadInternalPlan { + rows, + filters: HashMap::new(), + scan_range_after_filter: None, + }; + }; + let requested = RowAddrMask::from_allowed(ids); + for (fragment, id_span) in fragments.fragments.iter().zip(&fragments.id_spans) { + // Only fragments whose id span overlaps the batch's key range + // can hold requested rows + let Some(id_span) = id_span else { continue }; + if *id_span.end() < min_key || *id_span.start() > max_key { + continue; + } + let offsets = fragment.row_id_sequence.mask_to_offset_ranges(&requested); + if offsets.is_empty() { + continue; + } + let valid = FilteredReadStream::full_frag_range( + fragment.num_physical_rows, + &fragment.deletion_vector, + ); + let matched = FilteredReadStream::intersect_ranges(&valid, &offsets); + if !matched.is_empty() { + rows.insert(fragment.fragment.id() as u32, matched); + } + } + FilteredReadInternalPlan { + rows, + filters: HashMap::new(), + scan_range_after_filter: None, + } + } + + fn key_array<'a>( + &self, + batch: &'a RecordBatch, + producer: &str, + ) -> DataFusionResult<&'a arrow_array::PrimitiveArray> { + let keys = batch + .column_by_name(self.source.key_column) + .ok_or_else(|| { + DataFusionError::Internal(format!( + "the row-stream {} is missing the '{}' column", + producer, self.source.key_column + )) + })?; + keys.as_primitive_opt::().ok_or_else(|| { + DataFusionError::Internal(format!( + "expected the row-stream column '{}' to be UInt64 but it was {}", + self.source.key_column, + keys.data_type() + )) + }) + } + + async fn plan_batch( + &self, + keys: &arrow_array::PrimitiveArray, + ) -> DataFusionResult { + let compute_timer = self.baseline_metrics.elapsed_compute().timer(); + // Null keys are excluded; attach_columns drops their rows + let batch_keys = if keys.null_count() == 0 { + RowAddrTreeMap::from_iter(keys.values().iter().copied()) + } else { + RowAddrTreeMap::from_iter(keys.iter().flatten()) + }; + drop(compute_timer); + + let fragments = self.load_fragments().await?; + // Row ids equal row addresses when the dataset does not use stable + // row ids, so either key resolves directly by position + if self.source.key_column == ROW_ADDR || !self.dataset.manifest.uses_stable_row_ids() { + Ok(Self::plan_batch_from_addresses(&batch_keys, fragments)) + } else { + Ok(Self::plan_batch_from_row_ids(batch_keys, keys, fragments)) + } + } + + /// Read the batch's planned ranges through the same executor as a scan, + /// returning the rows in storage order, deduplicated, with the key + /// column included + async fn read_batch( + &self, + internal_plan: FilteredReadInternalPlan, + batch_index: u32, + ) -> DataFusionResult { + let fragment_count = self.load_fragments().await?.fragments.len(); + // I/O priority: earlier batches strictly first (output emits in batch + // order), fragments keep dataset order within a batch + let priority_offset = batch_index.saturating_mul(fragment_count as u32); + let read = FilteredReadStream::try_new( + self.dataset.clone(), + self.source.read_options.clone(), + self.global_metrics.clone(), + internal_plan, + Some(self.scan_scheduler.clone()), + Some(priority_offset), + ); + let decode_parallelism = match self.source.read_options.threading_mode { + FilteredReadThreadingMode::OnePartitionMultipleThreads(n) => n, + FilteredReadThreadingMode::MultiplePartitions(n) => n, + }; + let read_batches = read.collect_all(decode_parallelism.max(1)).await?; + Ok(arrow::compute::concat_batches( + &read.output_schema, + read_batches.iter(), + )?) + } + + /// Align the read rows back to the batch's row order and merge the + /// fetched columns on + fn attach_columns( + &self, + batch: RecordBatch, + read_data: RecordBatch, + ) -> DataFusionResult { + let _compute_timer = self.baseline_metrics.elapsed_compute().timer(); + let keys = self.key_array(&batch, "input")?; + let read_keys = self.key_array(&read_data, "read")?; + attach_read_columns( + &batch, + keys, + &read_data, + read_keys, + self.carried_schema.as_ref(), + self.source.new_fields_schema.as_ref(), + &self.output_schema, + ) + } + + async fn execute_batch( + self: Arc, + batch: RecordBatch, + batch_index: u32, + ) -> DataFusionResult { + if batch.num_rows() == 0 { + return Ok(RecordBatch::new_empty(self.output_schema.clone())); + } + let internal_plan = self.plan_batch(self.key_array(&batch, "input")?).await?; + let read_data = self.read_batch(internal_plan, batch_index).await?; + self.attach_columns(batch, read_data) + } + + fn apply( + self: Arc, + input: SendableRecordBatchStream, + ) -> impl Stream> { + let batch_target_rows = self + .source + .read_options + .batch_size + .map(|batch_size| batch_size as usize) + .unwrap_or_else(|| get_default_batch_size().unwrap_or(BATCH_SIZE_FALLBACK)); + let on_result = self.clone(); + let on_done = self.clone(); + coalesce_batches(input, batch_target_rows) + .enumerate() + .map(move |(batch_index, batch)| { + let batch = batch?; + let this = self.clone(); + DataFusionResult::Ok( + // SpawnedTask aborts on drop: cancelling the query + // cancels in-flight batches + SpawnedTask::spawn( + this.execute_batch(batch, batch_index as u32) + .in_current_span(), + ) + .map(|res| match res { + Ok(result) => result, + Err(join_error) => Err(DataFusionError::External(Box::new(join_error))), + }), + ) + }) + .boxed() + .try_buffered(ROW_STREAM_CONCURRENT_BATCHES) + .map(move |result| { + on_result + .global_metrics + .io_metrics + .record(&on_result.scan_scheduler); + match on_result + .baseline_metrics + .record_poll(Poll::Ready(Some(result))) + { + Poll::Ready(Some(result)) => result, + _ => unreachable!("record_poll returned a different poll state"), + } + }) + .finally(move || { + on_done.baseline_metrics.done(); + on_done + .global_metrics + .io_metrics + .record(&on_done.scan_scheduler); + }) + } +} + +/// Align `read_data` rows back to a keyed batch's row order and merge the +/// fetched columns on. +/// +/// `keys` is the row-id-space key of each `batch` row (usually the batch's +/// own key column; a caller that keys by address passes the resolved ids +/// instead) and `read_keys` the key of each `read_data` row, which must be +/// unique. Input rows whose key has no read row — null or stale keys — are +/// DROPPED; duplicate input keys re-expand through the gather. Output +/// columns follow `output_schema`: `carried_schema` names come from `batch`, +/// `new_fields_schema` names from `read_data`. +pub fn attach_read_columns( + batch: &RecordBatch, + keys: &arrow_array::PrimitiveArray, + read_data: &RecordBatch, + read_keys: &arrow_array::PrimitiveArray, + carried_schema: &arrow_schema::Schema, + new_fields_schema: &arrow_schema::Schema, + output_schema: &SchemaRef, +) -> DataFusionResult { + // Fast path: one read row per input row with an identical key sequence — + // already aligned, skip the hash map and the permutation + if keys.null_count() == 0 + && read_data.num_rows() == batch.num_rows() + && read_keys.values() == keys.values() + { + let new_data = read_data.project_by_schema(new_fields_schema)?; + let carried = batch.project_by_schema(carried_schema)?; + return Ok(carried.merge_with_schema(&new_data, output_schema.as_ref())?); + } + + let key_to_index: HashMap = read_keys + .values() + .iter() + .enumerate() + .map(|(index, key)| (*key, index as u32)) + .collect(); + + // Sizes differ only when some input keys have no live row (null or + // stale keys): drop those input rows first + let (batch, keys) = if read_data.num_rows() != batch.num_rows() { + let matched: BooleanArray = keys + .iter() + .map(|key| key.map(|key| key_to_index.contains_key(&key))) + .collect(); + let keys = arrow::compute::filter(keys, &matched)? + .as_primitive::() + .clone(); + (arrow::compute::filter_record_batch(batch, &matched)?, keys) + } else { + (batch.clone(), keys.clone()) + }; + if batch.num_rows() == 0 { + return Ok(RecordBatch::new_empty(output_schema.clone())); + } + + // Gather the read rows into input order — every remaining key hits + let indices = UInt32Array::from_iter_values(keys.values().iter().map(|key| key_to_index[key])); + let new_data = arrow_select::take::take_record_batch(read_data, &indices)?; + let new_data = new_data.project_by_schema(new_fields_schema)?; + let carried = batch.project_by_schema(carried_schema)?; + Ok(carried.merge_with_schema(&new_data, output_schema.as_ref())?) } impl DisplayAs for FilteredReadExec { fn fmt_as(&self, t: DisplayFormatType, f: &mut std::fmt::Formatter) -> std::fmt::Result { + if let RowSelector::RowStream(source) = &self.input { + let columns = source + .new_fields_schema + .fields + .iter() + .map(|f| f.name().as_str()) + .collect::>() + .join(", "); + return match t { + DisplayFormatType::Default | DisplayFormatType::Verbose => { + write!( + f, + "LanceRead: uri={}, projection=[{}], source=stream({})", + self.dataset.data_dir(), + columns, + source.key_column, + ) + } + DisplayFormatType::TreeRender => { + write!( + f, + "LanceRead\nuri={}\nprojection=[{}]\nsource=stream({})", + self.dataset.data_dir(), + columns, + source.key_column, + ) + } + }; + } let columns = self .options .projection @@ -1892,22 +2858,24 @@ impl ExecutionPlan for FilteredReadExec { "FilteredReadExec" } - fn as_any(&self) -> &dyn Any { - self - } - fn properties(&self) -> &Arc { &self.properties } fn children(&self) -> Vec<&Arc> { - if let Some(index_input) = &self.index_input { - vec![index_input] + if let Some(child) = self.input.child() { + vec![child] } else { vec![] } } + fn benefits_from_input_partitioning(&self) -> Vec { + // Partitioning a row-stream read would create multiple I/O schedulers + // (RAM heavy); the other selectors have no row input + vec![false; self.children().len()] + } + fn metrics(&self) -> Option { Some(self.metrics.clone_inner()) } @@ -1915,7 +2883,14 @@ impl ExecutionPlan for FilteredReadExec { fn partition_statistics( &self, partition: Option, - ) -> datafusion::error::Result { + ) -> datafusion::error::Result> { + if let RowSelector::RowStream(source) = &self.input { + // At most one output row per input row + return Ok(Arc::new(Statistics { + num_rows: source.plan.partition_statistics(partition)?.num_rows, + ..Statistics::new_unknown(self.schema().as_ref()) + })); + } let fragments = self .options .fragments @@ -1952,10 +2927,10 @@ impl ExecutionPlan for FilteredReadExec { total_rows }; - return Ok(Statistics { + return Ok(Arc::new(Statistics { num_rows: Precision::Exact(total_rows as usize), ..datafusion::physical_plan::Statistics::new_unknown(self.schema().as_ref()) - }); + })); }; // We could evaluate the indexed filter here but this is still during the planning @@ -1973,7 +2948,7 @@ impl ExecutionPlan for FilteredReadExec { .clone() .union_columns(filter_columns, OnMissing::Error)?; - let read_schema = Arc::new(read_projection.to_arrow_schema()); + let read_schema = public_blob_v2_binary_projection_schema(&read_projection); let planner = Arc::new(Planner::new(read_schema.clone())); let physical_filter = planner.create_physical_expr(filter)?; @@ -1990,7 +2965,7 @@ impl ExecutionPlan for FilteredReadExec { None, )?); let df_filter_exec = FilterExec::try_new(physical_filter, mock_input)?; - let mut df_stats = df_filter_exec.partition_statistics(partition)?; + let mut df_stats = Arc::unwrap_or_clone(df_filter_exec.partition_statistics(partition)?); // If we have an after-filter range, we should apply it to the stats (the before-filter range // is applied in the mock input) @@ -2026,7 +3001,7 @@ impl ExecutionPlan for FilteredReadExec { } }); - Ok(df_stats) + Ok(Arc::new(df_stats)) } fn with_new_children( @@ -2038,18 +3013,12 @@ impl ExecutionPlan for FilteredReadExec { Error::internal("A FilteredReadExec cannot have two children".to_string()).into(), )) } else { - let index_input = children.into_iter().next(); - Ok(Arc::new(Self { - dataset: self.dataset.clone(), - options: self.options.clone(), - properties: self.properties.clone(), - metrics: self.metrics.clone(), - // Seems unlikely this would already be initialized but clear it - // out just in case - running_stream: Arc::new(AsyncMutex::new(None)), - index_input, - plan: Arc::new(OnceCell::new()), - })) + // Rebuild via try_new so the selector and derived state are + // re-derived from the new child's schema + let child = children.into_iter().next(); + let rebuilt = Self::try_new(self.dataset.clone(), self.options.clone(), child) + .map_err(|e| DataFusionError::External(e.into()))?; + Ok(Arc::new(rebuilt)) } } @@ -2058,10 +3027,16 @@ impl ExecutionPlan for FilteredReadExec { partition: usize, context: Arc, ) -> DataFusionResult { - Ok(self.obtain_stream(partition, context)) + match &self.input { + RowSelector::RowStream(source) => self.execute_row_stream(source, partition, context), + _ => Ok(self.obtain_stream(partition, context)), + } } fn fetch(&self) -> Option { + if self.row_stream_input().is_some() { + return None; + } if self.options.full_filter.is_none() { self.options .scan_range_before_filter @@ -2076,13 +3051,15 @@ impl ExecutionPlan for FilteredReadExec { } fn supports_limit_pushdown(&self) -> bool { - // This is to push the limit through the node and into an upstream node. - // The only upstream node is the index search and we can't push the limit - // to that node. - false + // A limit pushes through to a row-stream input (one output row per + // input row); the other selectors have no node to push it to + self.row_stream_input().is_some() } fn with_fetch(&self, limit: Option) -> Option> { + if self.row_stream_input().is_some() { + return None; + } // TODO: Support multiple partitions in the future by coordinating limits across partitions if matches!( self.options.threading_mode, @@ -2113,7 +3090,7 @@ impl ExecutionPlan for FilteredReadExec { match Self::try_new( self.dataset.clone(), updated_options, - self.index_input.clone(), + self.input.row_set_plan().cloned(), ) { Ok(exec) => Some(Arc::new(exec)), Err(e) => { @@ -2142,7 +3119,9 @@ mod tests { }; use itertools::Itertools; use lance_core::datatypes::OnMissing; + use lance_core::utils::address::RowAddress; use lance_core::utils::tempfile::TempStrDir; + use lance_datafusion::exec::OneShotExec; use lance_datagen::{BatchCount, Dimension, RowCount, array, gen_batch}; use lance_index::{ IndexType, @@ -2150,6 +3129,7 @@ mod tests { scalar::{ScalarIndexParams, expression::PlannerIndexExt}, }; use lance_select::result::IndexExprResultWireFormat; + use lance_select::{RowAddrMask, RowAddrTreeMap}; use crate::{ dataset::{InsertBuilder, WriteDestination, WriteMode, WriteParams}, @@ -2376,6 +3356,77 @@ mod tests { )) } + /// Take-shaped masked reads consolidate their tiny per-fragment batches; + /// few-fragment and dense masked reads keep per-fragment boundaries. + #[test_log::test(tokio::test)] + async fn test_take_shaped_mask_consolidation() { + // 20 fragments x 2000 rows, value = global row number + let tmp_path = TempStrDir::default(); + let data = gen_batch() + .col("value", array::step::()) + .into_reader_rows(RowCount::from(2000), BatchCount::from(20)); + let dataset = Arc::new( + Dataset::write( + data, + tmp_path.as_str(), + Some(WriteParams { + max_rows_per_file: 2000, + ..Default::default() + }), + ) + .await + .unwrap(), + ); + + let mask_input = |addrs: Vec| -> Arc { + let covered: RoaringBitmap = dataset.fragments().iter().map(|f| f.id as u32).collect(); + let batch = + IndexExprResult::exact(RowAddrMask::from_allowed(RowAddrTreeMap::from_iter(addrs))) + .serialize(&covered, IndexExprResultWireFormat::default()) + .unwrap(); + let schema = batch.schema(); + let stream = futures::stream::once(async move { Ok(batch) }); + Arc::new(OneShotExec::new(Box::pin(RecordBatchStreamAdapter::new( + schema, stream, + )))) + }; + let run = |input: Arc| { + let dataset = dataset.clone(); + async move { + // Pin the batch size so batch-count assertions don't depend + // on LANCE_DEFAULT_BATCH_SIZE + let options = FilteredReadOptions::basic_full_read(&dataset).with_batch_size(2000); + let plan = + FilteredReadExec::try_new(dataset.clone(), options, Some(input)).unwrap(); + let stream = plan.execute(0, Arc::new(TaskContext::default())).unwrap(); + stream.try_collect::>().await.unwrap() + } + }; + let addr = |frag: u32, offset: u32| u64::from(RowAddress::new_from_parts(frag, offset)); + + // Take shape: 20 fragments, 2 rows each -> one consolidated batch, + // rows in fragment order + let addrs: Vec = (0..20u32).flat_map(|f| [addr(f, 3), addr(f, 7)]).collect(); + let batches = run(mask_input(addrs)).await; + assert_eq!(batches.iter().map(|b| b.num_rows()).sum::(), 40); + assert_eq!(batches.len(), 1); + let expected = + UInt32Array::from_iter_values((0..20u32).flat_map(|f| [f * 2000 + 3, f * 2000 + 7])); + assert_eq!(batches[0].column(0).as_ref(), &expected); + + // Too few fragments -> inline path, one batch per fragment + let batches = run(mask_input(vec![addr(0, 3), addr(1, 7)])).await; + assert_eq!(batches.len(), 2); + + // Dense (2000 planned rows per fragment) -> inline path + let addrs: Vec = (0..8u32) + .flat_map(|f| (0..2000u32).map(move |o| addr(f, o))) + .collect(); + let batches = run(mask_input(addrs)).await; + assert_eq!(batches.len(), 8); + assert_eq!(batches.iter().map(|b| b.num_rows()).sum::(), 16000); + } + /// Round-trip every interval shape through the arrow wire format and /// confirm the endpoints survive. Exercises both /// `IndexExprResult::serialize` and `EvaluatedIndex::try_from_arrow` @@ -2410,15 +3461,15 @@ mod tests { // robust, but the canonical builders preserve representation. assert_eq!( decoded.index_result.lower, original.lower, - "{name}: lower endpoint changed across round-trip", + "{name}: lower endpoint changed across batch-trip", ); assert_eq!( decoded.index_result.upper, original.upper, - "{name}: upper endpoint changed across round-trip", + "{name}: upper endpoint changed across batch-trip", ); assert_eq!( decoded.applicable_fragments, frags, - "{name}: applicable fragments changed across round-trip", + "{name}: applicable fragments changed across batch-trip", ); } } @@ -2765,6 +3816,48 @@ mod tests { assert_eq!(num_rows, 300); } + /// A stale (not rebuilt after a delete) index hit drops on the live view + /// and returns as a null-_rowid tombstone with with_deleted_rows + #[test_log::test(tokio::test)] + async fn test_with_deleted_rows_stale_index() { + let fixture = Arc::new(TestFixture::new().await); + let base_options = FilteredReadOptions::basic_full_read(&fixture.dataset); + + // Row 220 is deletion-vector-deleted but still in the index + let filter_plan = fixture.filter_plan("fully_indexed == 220", true).await; + + // Live view: the stale index hit drops + fixture + .test_plan( + base_options.clone().with_filter_plan(filter_plan), + &u32s(vec![]), + ) + .await; + + // Physical view: the tombstone returns + let filter_plan = fixture.filter_plan("fully_indexed == 220", true).await; + let options = base_options + .with_deleted_rows() + .unwrap() + .with_filter_plan(filter_plan); + let plan = fixture.make_plan(options).await; + let stream = plan.execute(0, Arc::new(TaskContext::default())).unwrap(); + let batches = stream.try_collect::>().await.unwrap(); + let total_rows: usize = batches.iter().map(|b| b.num_rows()).sum(); + assert_eq!(total_rows, 1); + let batch = batches.iter().find(|b| b.num_rows() > 0).unwrap(); + let values = batch + .column_by_name("fully_indexed") + .unwrap() + .as_primitive::(); + assert_eq!(values.value(0), 220); + let row_ids = batch + .column_by_name(ROW_ID) + .unwrap() + .as_primitive::(); + assert!(row_ids.is_null(0)); + } + #[test] fn test_dv_to_ranges() { let dv = Arc::new(DeletionVector::from_iter(vec![1])); @@ -2967,10 +4060,7 @@ mod tests { assert_eq!(plan.options().scan_range_before_filter, None); assert_eq!(plan.fetch(), None); let new_plan = plan.with_fetch(Some(100)).unwrap(); - let new_plan = new_plan - .as_any() - .downcast_ref::() - .unwrap(); + let new_plan = new_plan.downcast_ref::().unwrap(); assert_eq!(new_plan.options().scan_range_before_filter, Some(0..100)); assert_eq!(new_plan.fetch(), Some(100)); } @@ -2998,10 +4088,7 @@ mod tests { assert_eq!(plan.options().scan_range_after_filter, None); assert_eq!(plan.fetch(), None); let new_plan = plan.with_fetch(Some(50)).unwrap(); - let new_plan = new_plan - .as_any() - .downcast_ref::() - .unwrap(); + let new_plan = new_plan.downcast_ref::().unwrap(); assert_eq!(new_plan.options().scan_range_after_filter, Some(0..50)); assert_eq!(new_plan.fetch(), Some(50)); } @@ -3051,14 +4138,11 @@ mod tests { let options = base_options.with_filter_plan(filter_plan); let plan = fixture.make_plan(options).await; - assert!(plan.index_input.is_some()); + assert!(plan.index_input().is_some()); assert!(plan.options().refine_filter.is_some()); let limited_plan = plan.with_fetch(Some(10)).unwrap(); - let limited_plan = limited_plan - .as_any() - .downcast_ref::() - .unwrap(); + let limited_plan = limited_plan.downcast_ref::().unwrap(); assert_eq!(limited_plan.options().scan_range_after_filter, Some(0..10)); let stream = limited_plan @@ -3780,7 +4864,7 @@ mod tests { /// Test that direct execution gives the same result as get_plan + execute_with_plan #[test_log::test(tokio::test)] - async fn test_plan_round_trip() { + async fn test_plan_batch_trip() { let fixture = TestFixture::new().await; let ctx = Arc::new(TaskContext::default()); @@ -3893,4 +4977,863 @@ mod tests { assert_eq!(default_result.num_rows(), capped_result.num_rows()); } + + // Row-stream selector tests + + mod row_stream { + use super::*; + use arrow_array::{Float32Array, StringArray, UInt64Array}; + use arrow_schema::{DataType, Field as ArrowField, Fields, Schema as ArrowSchema}; + use datafusion::physical_plan::stream::RecordBatchStreamAdapter; + use lance_datafusion::exec::OneShotExec; + use rstest::rstest; + + use crate::dataset::{Dataset, WriteParams}; + use crate::utils::test::NoContextTestFixture; + + struct TakeFixture { + dataset: Arc, + _tmp_dir: TempStrDir, + } + + /// 30 rows across 3 fragments with columns i, s, and struct{x, y} + async fn take_fixture(stable_row_ids: bool) -> TakeFixture { + let struct_fields = Fields::from(vec![ + Arc::new(ArrowField::new("x", DataType::Int32, false)), + Arc::new(ArrowField::new("y", DataType::Int32, false)), + ]); + let schema = Arc::new(ArrowSchema::new(vec![ + ArrowField::new("i", DataType::Int32, false), + ArrowField::new("s", DataType::Utf8, false), + ArrowField::new("struct", DataType::Struct(struct_fields.clone()), false), + ])); + let batches: Vec = (0..3) + .map(|batch_id| { + let value_range = batch_id * 10..batch_id * 10 + 10; + RecordBatch::try_new( + schema.clone(), + vec![ + Arc::new(Int32Array::from_iter_values(value_range.clone())), + Arc::new(StringArray::from_iter_values( + value_range.clone().map(|v| format!("s-{v}")), + )), + Arc::new(arrow_array::StructArray::new( + struct_fields.clone(), + vec![ + Arc::new(Int32Array::from_iter(value_range.clone())), + Arc::new(Int32Array::from_iter(value_range)), + ], + None, + )), + ], + ) + .unwrap() + }) + .collect(); + + let tmp_dir = TempStrDir::default(); + let uri = tmp_dir.as_str(); + let params = WriteParams { + max_rows_per_file: 10, + enable_stable_row_ids: stable_row_ids, + ..Default::default() + }; + let reader = RecordBatchIterator::new(batches.into_iter().map(Ok), schema); + Dataset::write(reader, uri, Some(params)).await.unwrap(); + TakeFixture { + dataset: Arc::new(Dataset::open(uri).await.unwrap()), + _tmp_dir: tmp_dir, + } + } + + /// Wrap batches of (payload, key) rows into an input plan + fn rows_input(batches: Vec) -> Arc { + let schema = batches[0].schema(); + let stream = futures::stream::iter(batches.into_iter().map(Ok)); + let stream = Box::pin(RecordBatchStreamAdapter::new(schema, stream)); + Arc::new(OneShotExec::new(stream)) + } + + fn take_plan( + dataset: &Arc, + input: Arc, + columns: &[&str], + ) -> Result { + let projection = dataset + .empty_projection() + .union_columns(columns, OnMissing::Error) + .unwrap(); + FilteredReadExec::try_new( + dataset.clone(), + FilteredReadOptions::new(projection), + Some(input), + ) + } + + fn take_plan_sized( + dataset: &Arc, + input: Arc, + columns: &[&str], + batch_size: u32, + ) -> Result { + let projection = dataset + .empty_projection() + .union_columns(columns, OnMissing::Error) + .unwrap(); + FilteredReadExec::try_new( + dataset.clone(), + FilteredReadOptions::new(projection).with_batch_size(batch_size), + Some(input), + ) + } + + async fn run(plan: &FilteredReadExec) -> Vec { + plan.execute(0, Arc::new(TaskContext::default())) + .unwrap() + .try_collect::>() + .await + .unwrap() + } + + /// A sparse plan constructs fragment handles only for the fragments + /// it selects, keeping their candidate-list position as priority — + /// no metadata is loaded or retained for unselected fragments + #[tokio::test] + async fn sparse_plan_scopes_only_selected_fragments() { + let fixture = take_fixture(false).await; + let dataset = &fixture.dataset; + let descriptors = dataset.fragments().clone(); + assert_eq!(descriptors.len(), 3); + + let mut rows = BTreeMap::new(); + rows.insert(2u32, vec![0u64..5]); + let plan = FilteredReadInternalPlan { + rows, + filters: HashMap::new(), + scan_range_after_filter: None, + }; + let options = FilteredReadOptions::basic_full_read(dataset); + let scheduler = FilteredReadStream::make_scan_scheduler(dataset, &options); + + let scoped = FilteredReadStream::plan_to_scoped_fragments( + &plan, + &descriptors, + dataset, + &options, + scheduler, + ); + assert_eq!(scoped.len(), 1); + assert_eq!(scoped[0].fragment.id(), 2); + assert_eq!(scoped[0].priority, 2); + } + + /// Output preserves the input's row order, duplicates, and payload + #[rstest] + #[case::by_row_addr(false, ROW_ADDR)] + #[case::by_row_id(false, ROW_ID)] + #[case::stable_by_row_addr(true, ROW_ADDR)] + #[case::stable_by_row_id(true, ROW_ID)] + #[tokio::test] + async fn take_preserves_order_dups_and_payload( + #[case] stable_row_ids: bool, + #[case] key: &str, + ) { + let fixture = take_fixture(stable_row_ids).await; + + // Stable row ids are assigned sequentially on write, so the id of + // row `i` is `i`; without them id == address + let addr = |frag: u64, off: u64| (frag << 32) | off; + let keys: Vec = if key == ROW_ID && stable_row_ids { + vec![21, 3, 15, 21, 0] + } else { + vec![ + addr(2, 1), // i = 21 + addr(0, 3), // i = 3 + addr(1, 5), // i = 15 + addr(2, 1), // i = 21 (duplicate) + addr(0, 0), // i = 0 + ] + }; + let expected_i: Vec = vec![21, 3, 15, 21, 0]; + + let input_schema = Arc::new(ArrowSchema::new(vec![ + ArrowField::new("payload", DataType::Float32, false), + ArrowField::new(key, DataType::UInt64, true), + ])); + let payload: Vec = (0..keys.len()).map(|v| v as f32 * 0.5).collect(); + let batch = RecordBatch::try_new( + input_schema.clone(), + vec![ + Arc::new(Float32Array::from(payload.clone())), + Arc::new(UInt64Array::from(keys.clone())), + ], + ) + .unwrap(); + let batches = vec![batch.slice(0, 3), batch.slice(3, 2)]; + + let plan = take_plan(&fixture.dataset, rows_input(batches), &["s", "i"]).unwrap(); + assert!(plan.row_stream_input().is_some()); + // Input columns, then new fields; the unrequested key is stripped + assert_eq!( + plan.schema() + .fields() + .iter() + .map(|f| f.name().clone()) + .collect::>(), + vec!["payload", "i", "s"] + ); + + let result = run(&plan).await; + assert_eq!(result.len(), 1); + assert_eq!(result[0].num_rows(), 5); + let result = concat_batches(&plan.schema(), &result).unwrap(); + + let i_col = result.column_by_name("i").unwrap(); + assert_eq!( + i_col.as_primitive::().values(), + &expected_i[..] + ); + let s_col = result.column_by_name("s").unwrap().as_string::(); + for (row, i) in expected_i.iter().enumerate() { + assert_eq!(s_col.value(row), format!("s-{i}")); + } + let payload_col = result + .column_by_name("payload") + .unwrap() + .as_primitive::(); + assert_eq!(payload_col.values(), &payload[..]); + } + + /// Tiny input batches merge up to the target and oversized ones pass + /// through whole, preserving order across the boundaries + #[tokio::test] + async fn take_coalesces_input_to_batch_size() { + let fixture = take_fixture(false).await; + + let addr = |frag: u64, off: u64| (frag << 32) | off; + let keys: Vec = vec![ + addr(2, 3), // i = 23 + addr(0, 1), // i = 1 + addr(1, 4), // i = 14 + addr(0, 7), // i = 7 + addr(2, 0), // i = 20 + addr(1, 1), // i = 11 + addr(0, 2), // i = 2 + ]; + let expected_i: Vec = vec![23, 1, 14, 7, 20, 11, 2]; + let input_schema = Arc::new(ArrowSchema::new(vec![ArrowField::new( + ROW_ADDR, + DataType::UInt64, + true, + )])); + let batch = RecordBatch::try_new( + input_schema.clone(), + vec![Arc::new(UInt64Array::from(keys.clone()))], + ) + .unwrap(); + + let assert_batches = + |result: Vec, schema: SchemaRef, expected_sizes: Vec| { + assert_eq!( + result.iter().map(|b| b.num_rows()).collect::>(), + expected_sizes + ); + let merged = concat_batches(&schema, &result).unwrap(); + let i_col = merged.column_by_name("i").unwrap(); + assert_eq!( + i_col.as_primitive::().values(), + &expected_i[..] + ); + }; + + // Seven one-row batches merge whenever the buffer reaches 3 rows + let tiny = (0..7).map(|i| batch.slice(i, 1)).collect::>(); + let plan = take_plan_sized(&fixture.dataset, rows_input(tiny), &["i"], 3).unwrap(); + assert_batches(run(&plan).await, plan.schema(), vec![3, 3, 1]); + + // One oversized batch passes through whole — never split + let plan = + take_plan_sized(&fixture.dataset, rows_input(vec![batch.clone()]), &["i"], 3) + .unwrap(); + assert_batches(run(&plan).await, plan.schema(), vec![7]); + + // A large batch flushes the partial buffer and passes through + let mixed = vec![batch.slice(0, 2), batch.slice(2, 5)]; + let plan = take_plan_sized(&fixture.dataset, rows_input(mixed), &["i"], 3).unwrap(); + assert_batches(run(&plan).await, plan.schema(), vec![2, 5]); + } + + /// Storage-ordered input exercises the aligned fast path + #[rstest] + #[case::by_row_addr(ROW_ADDR)] + #[case::by_row_id(ROW_ID)] + #[tokio::test] + async fn take_aligned_input_fast_path(#[case] key: &str) { + let fixture = take_fixture(false).await; + + let addr = |frag: u64, off: u64| (frag << 32) | off; + let keys: Vec = vec![ + addr(0, 0), // i = 0 + addr(0, 3), // i = 3 + addr(1, 5), // i = 15 + addr(2, 1), // i = 21 + addr(2, 9), // i = 29 + ]; + let expected_i: Vec = vec![0, 3, 15, 21, 29]; + + let input_schema = Arc::new(ArrowSchema::new(vec![ + ArrowField::new("payload", DataType::Float32, false), + ArrowField::new(key, DataType::UInt64, true), + ])); + let payload: Vec = (0..keys.len()).map(|v| v as f32 * 0.5).collect(); + let batch = RecordBatch::try_new( + input_schema, + vec![ + Arc::new(Float32Array::from(payload.clone())), + Arc::new(UInt64Array::from(keys)), + ], + ) + .unwrap(); + + let plan = take_plan(&fixture.dataset, rows_input(vec![batch]), &["i"]).unwrap(); + let result = concat_batches(&plan.schema(), &run(&plan).await).unwrap(); + assert_eq!(result.num_rows(), 5); + let i_col = result.column_by_name("i").unwrap(); + assert_eq!( + i_col.as_primitive::().values(), + &expected_i[..] + ); + let payload_col = result + .column_by_name("payload") + .unwrap() + .as_primitive::(); + assert_eq!(payload_col.values(), &payload[..]); + } + + /// A fragment-scoped take reads from the scoped fragments only; keys + /// pointing outside the scope drop like stale rows + #[rstest] + #[case::by_row_addr(false, ROW_ADDR)] + #[case::by_row_id(false, ROW_ID)] + #[case::stable_by_row_addr(true, ROW_ADDR)] + #[case::stable_by_row_id(true, ROW_ID)] + #[tokio::test] + async fn take_scoped_to_fragments(#[case] stable_row_ids: bool, #[case] key: &str) { + let fixture = take_fixture(stable_row_ids).await; + let subset = Arc::new(vec![fixture.dataset.fragments()[1].clone()]); + + let addr = |frag: u64, off: u64| (frag << 32) | off; + // i = 12 inside the scoped fragment, i = 3 outside the scope + let keys: Vec = if key == ROW_ID && stable_row_ids { + vec![12, 3] + } else { + vec![addr(1, 2), addr(0, 3)] + }; + let input_schema = Arc::new(ArrowSchema::new(vec![ArrowField::new( + key, + DataType::UInt64, + true, + )])); + let batch = RecordBatch::try_new(input_schema, vec![Arc::new(UInt64Array::from(keys))]) + .unwrap(); + + let projection = fixture + .dataset + .empty_projection() + .union_columns(["i"], OnMissing::Error) + .unwrap(); + let plan = FilteredReadExec::try_new( + fixture.dataset.clone(), + FilteredReadOptions::new(projection).with_fragments(subset), + Some(rows_input(vec![batch])), + ) + .unwrap(); + + let result = concat_batches(&plan.schema(), &run(&plan).await).unwrap(); + assert_eq!(result.num_rows(), 1); + let i_col = result + .column_by_name("i") + .unwrap() + .as_primitive::(); + assert_eq!(i_col.value(0), 12); + } + + /// A batch whose keys span the whole id range but hit only two rows: + /// the span prefilter must not misread coverage as membership + #[tokio::test] + async fn take_stable_ids_wide_key_span() { + let fixture = take_fixture(true).await; + // Last row of the last fragment, first row of the first: every + // fragment's span overlaps, only two rows match + let keys: Vec = vec![29, 0]; + let input_schema = Arc::new(ArrowSchema::new(vec![ArrowField::new( + ROW_ID, + DataType::UInt64, + true, + )])); + let batch = RecordBatch::try_new(input_schema, vec![Arc::new(UInt64Array::from(keys))]) + .unwrap(); + + let plan = take_plan(&fixture.dataset, rows_input(vec![batch]), &["i"]).unwrap(); + let result = concat_batches(&plan.schema(), &run(&plan).await).unwrap(); + let i_col = result + .column_by_name("i") + .unwrap() + .as_primitive::(); + assert_eq!(i_col.values(), &[29, 0]); + } + + /// Identity flags: requested-but-missing columns are synthesized, + /// carried ones kept, unrequested carried ones stripped + #[rstest] + #[case::unstable(false)] + #[case::stable(true)] + #[tokio::test] + async fn take_identity_flags(#[case] stable_row_ids: bool) { + let fixture = take_fixture(stable_row_ids).await; + let addr = |frag: u64, off: u64| (frag << 32) | off; + + let ids: Vec = if stable_row_ids { + vec![21, 3] + } else { + vec![addr(2, 1), addr(0, 3)] + }; + let expected_addrs: Vec = vec![addr(2, 1), addr(0, 3)]; + let expected_i: Vec = vec![21, 3]; + let id_schema = Arc::new(ArrowSchema::new(vec![ArrowField::new( + ROW_ID, + DataType::UInt64, + true, + )])); + let id_batch = + RecordBatch::try_new(id_schema, vec![Arc::new(UInt64Array::from(ids.clone()))]) + .unwrap(); + + // Keep the carried _rowid and synthesize _rowaddr + let projection = fixture + .dataset + .empty_projection() + .union_columns(["i"], OnMissing::Error) + .unwrap() + .with_row_id() + .with_row_addr(); + let plan = FilteredReadExec::try_new( + fixture.dataset.clone(), + FilteredReadOptions::new(projection), + Some(rows_input(vec![id_batch.clone()])), + ) + .unwrap(); + assert_eq!( + plan.schema() + .fields() + .iter() + .map(|f| f.name().clone()) + .collect::>(), + vec![ROW_ID, "i", ROW_ADDR] + ); + let result = concat_batches(&plan.schema(), &run(&plan).await).unwrap(); + let id_col = result + .column_by_name(ROW_ID) + .unwrap() + .as_primitive::(); + assert_eq!(id_col.values(), &ids[..]); + let addr_col = result + .column_by_name(ROW_ADDR) + .unwrap() + .as_primitive::(); + assert_eq!(addr_col.values(), &expected_addrs[..]); + let i_col = result + .column_by_name("i") + .unwrap() + .as_primitive::(); + assert_eq!(i_col.values(), &expected_i[..]); + + // Synthesize _rowaddr but strip the unrequested carried _rowid + let projection = fixture + .dataset + .empty_projection() + .union_columns(["i"], OnMissing::Error) + .unwrap() + .with_row_addr(); + let plan = FilteredReadExec::try_new( + fixture.dataset.clone(), + FilteredReadOptions::new(projection), + Some(rows_input(vec![id_batch.clone()])), + ) + .unwrap(); + assert_eq!( + plan.schema() + .fields() + .iter() + .map(|f| f.name().clone()) + .collect::>(), + vec!["i", ROW_ADDR] + ); + let result = concat_batches(&plan.schema(), &run(&plan).await).unwrap(); + let addr_col = result + .column_by_name(ROW_ADDR) + .unwrap() + .as_primitive::(); + assert_eq!(addr_col.values(), &expected_addrs[..]); + + // Address-keyed input, synthesize _rowid (the reverse direction) + let addr_schema = Arc::new(ArrowSchema::new(vec![ArrowField::new( + ROW_ADDR, + DataType::UInt64, + true, + )])); + let addr_batch = RecordBatch::try_new( + addr_schema, + vec![Arc::new(UInt64Array::from(expected_addrs.clone()))], + ) + .unwrap(); + let projection = fixture + .dataset + .empty_projection() + .union_columns(["i"], OnMissing::Error) + .unwrap() + .with_row_id(); + let plan = FilteredReadExec::try_new( + fixture.dataset.clone(), + FilteredReadOptions::new(projection), + Some(rows_input(vec![addr_batch])), + ) + .unwrap(); + assert_eq!( + plan.schema() + .fields() + .iter() + .map(|f| f.name().clone()) + .collect::>(), + vec!["i", ROW_ID] + ); + let result = concat_batches(&plan.schema(), &run(&plan).await).unwrap(); + let id_col = result + .column_by_name(ROW_ID) + .unwrap() + .as_primitive::(); + assert_eq!(id_col.values(), &ids[..]); + + // Fetch nothing, synthesize only (the AddRowAddrExec shape) + let projection = fixture + .dataset + .empty_projection() + .with_row_id() + .with_row_addr(); + let plan = FilteredReadExec::try_new( + fixture.dataset.clone(), + FilteredReadOptions::new(projection), + Some(rows_input(vec![id_batch])), + ) + .unwrap(); + assert_eq!( + plan.schema() + .fields() + .iter() + .map(|f| f.name().clone()) + .collect::>(), + vec![ROW_ID, ROW_ADDR] + ); + let result = concat_batches(&plan.schema(), &run(&plan).await).unwrap(); + let addr_col = result + .column_by_name(ROW_ADDR) + .unwrap() + .as_primitive::(); + assert_eq!(addr_col.values(), &expected_addrs[..]); + } + + /// New sub-fields merge into an existing struct column + #[tokio::test] + async fn take_merges_nested_struct() { + let fixture = take_fixture(false).await; + + let data = fixture + .dataset + .scan() + .project(&["struct"]) + .unwrap() + .with_row_id() + .try_into_batch() + .await + .unwrap(); + // Rebuild the input with only struct.y so struct.x must be taken + let full_struct = data.column_by_name("struct").unwrap().as_struct(); + let y_only = arrow_array::StructArray::new( + Fields::from(vec![Arc::new(ArrowField::new("y", DataType::Int32, false))]), + vec![full_struct.column_by_name("y").unwrap().clone()], + None, + ); + let input_schema = Arc::new(ArrowSchema::new(vec![ + ArrowField::new("struct", y_only.data_type().clone(), false), + ArrowField::new(ROW_ID, DataType::UInt64, true), + ])); + let data = RecordBatch::try_new( + input_schema, + vec![ + Arc::new(y_only), + data.column_by_name(ROW_ID).unwrap().clone(), + ], + ) + .unwrap(); + + let projection = fixture + .dataset + .empty_projection() + .union_column("struct.x", OnMissing::Error) + .unwrap(); + let plan = FilteredReadExec::try_new( + fixture.dataset.clone(), + FilteredReadOptions::new(projection), + Some(rows_input(vec![data])), + ) + .unwrap(); + + let expected_struct_type = DataType::Struct(Fields::from(vec![ + Arc::new(ArrowField::new("x", DataType::Int32, false)), + Arc::new(ArrowField::new("y", DataType::Int32, false)), + ])); + assert_eq!( + plan.schema().field_with_name("struct").unwrap().data_type(), + &expected_struct_type + ); + + let result = concat_batches(&plan.schema(), &run(&plan).await).unwrap(); + assert_eq!(result.num_rows(), 30); + let struct_col = result.column_by_name("struct").unwrap().as_struct(); + assert_eq!( + struct_col.column_by_name("x").unwrap(), + struct_col.column_by_name("y").unwrap() + ); + } + + /// Input rows whose key no longer exists (deleted rows) are dropped + #[rstest] + #[case::by_row_addr(false, ROW_ADDR)] + #[case::by_row_id(false, ROW_ID)] + #[case::stable_by_row_addr(true, ROW_ADDR)] + #[case::stable_by_row_id(true, ROW_ID)] + #[tokio::test] + async fn take_drops_stale_keys(#[case] stable_row_ids: bool, #[case] key: &str) { + let fixture = take_fixture(stable_row_ids).await; + let mut dataset = fixture.dataset.as_ref().clone(); + dataset.delete("i = 15").await.unwrap(); + let dataset = Arc::new(dataset); + + let addr = |frag: u64, off: u64| (frag << 32) | off; + // The pre-delete identifiers of rows 15 (now deleted) and 16 + let keys: Vec = if key == ROW_ID && stable_row_ids { + vec![15, 16] + } else { + vec![addr(1, 5), addr(1, 6)] + }; + let input_schema = Arc::new(ArrowSchema::new(vec![ArrowField::new( + key, + DataType::UInt64, + true, + )])); + let batch = RecordBatch::try_new(input_schema, vec![Arc::new(UInt64Array::from(keys))]) + .unwrap(); + + let plan = take_plan(&dataset, rows_input(vec![batch]), &["i"]).unwrap(); + let result = concat_batches(&plan.schema(), &run(&plan).await).unwrap(); + assert_eq!(result.num_rows(), 1); + let i_col = result + .column_by_name("i") + .unwrap() + .as_primitive::(); + assert_eq!(i_col.value(0), 16); + } + + /// Keys of a fully deleted fragment (gone from the manifest) are + /// dropped like stale rows + #[rstest] + #[case::by_row_addr(false, ROW_ADDR)] + #[case::by_row_id(false, ROW_ID)] + #[case::stable_by_row_addr(true, ROW_ADDR)] + #[case::stable_by_row_id(true, ROW_ID)] + #[tokio::test] + async fn take_drops_keys_of_deleted_fragment( + #[case] stable_row_ids: bool, + #[case] key: &str, + ) { + let fixture = take_fixture(stable_row_ids).await; + let mut dataset = fixture.dataset.as_ref().clone(); + dataset.delete("i >= 10 and i < 20").await.unwrap(); + let dataset = Arc::new(dataset); + + let addr = |frag: u64, off: u64| (frag << 32) | off; + // The pre-delete identifiers of row 15 (fragment 1, now gone + // from the manifest) and row 20 (fragment 2, still live) + let keys: Vec = if key == ROW_ID && stable_row_ids { + vec![15, 20] + } else { + vec![addr(1, 5), addr(2, 0)] + }; + let input_schema = Arc::new(ArrowSchema::new(vec![ArrowField::new( + key, + DataType::UInt64, + true, + )])); + let batch = RecordBatch::try_new(input_schema, vec![Arc::new(UInt64Array::from(keys))]) + .unwrap(); + + let plan = take_plan(&dataset, rows_input(vec![batch]), &["i"]).unwrap(); + let result = concat_batches(&plan.schema(), &run(&plan).await).unwrap(); + assert_eq!(result.num_rows(), 1); + let i_col = result + .column_by_name("i") + .unwrap() + .as_primitive::(); + assert_eq!(i_col.value(0), 20); + } + + /// After delete + compaction the stable row-id sequences are no + /// longer simple contiguous ranges; ids must still resolve to the + /// moved rows and deleted ids must still drop + #[tokio::test] + async fn take_stable_ids_after_compaction() { + use crate::dataset::optimize::{CompactionOptions, compact_files}; + + let fixture = take_fixture(true).await; + let mut dataset = fixture.dataset.as_ref().clone(); + // Punch holes, then rewrite all fragments into one + dataset.delete("i % 3 = 0").await.unwrap(); + compact_files(&mut dataset, CompactionOptions::default(), None) + .await + .unwrap(); + let dataset = Arc::new(dataset); + assert_eq!(dataset.get_fragments().len(), 1); + + // Survivors in scattered order, plus a compacted-away id (15) + let keys: Vec = vec![25, 1, 15, 14]; + let input_schema = Arc::new(ArrowSchema::new(vec![ArrowField::new( + ROW_ID, + DataType::UInt64, + true, + )])); + let batch = RecordBatch::try_new(input_schema, vec![Arc::new(UInt64Array::from(keys))]) + .unwrap(); + + let plan = take_plan(&dataset, rows_input(vec![batch]), &["i"]).unwrap(); + let result = concat_batches(&plan.schema(), &run(&plan).await).unwrap(); + let i_col = result + .column_by_name("i") + .unwrap() + .as_primitive::(); + assert_eq!(i_col.values(), &[25, 1, 14]); + } + + /// with_deleted_rows is rejected for a row-stream read + #[rstest] + #[case::unstable(false)] + #[case::stable(true)] + #[tokio::test] + async fn take_rejects_with_deleted_rows(#[case] stable_row_ids: bool) { + let fixture = take_fixture(stable_row_ids).await; + let input_schema = Arc::new(ArrowSchema::new(vec![ArrowField::new( + ROW_ADDR, + DataType::UInt64, + true, + )])); + let batch = + RecordBatch::try_new(input_schema, vec![Arc::new(UInt64Array::from(vec![0_u64]))]) + .unwrap(); + let projection = fixture + .dataset + .empty_projection() + .union_columns(["i"], OnMissing::Error) + .unwrap(); + let err = FilteredReadExec::try_new( + fixture.dataset, + FilteredReadOptions::new(projection) + .with_deleted_rows() + .unwrap(), + Some(rows_input(vec![batch])), + ) + .unwrap_err(); + assert!(matches!(err, Error::InvalidInput { .. }), "{err}"); + assert!(err.to_string().contains("with_deleted_rows")); + } + + /// Construction errors: no key column, nothing to read + #[tokio::test] + async fn take_construction_errors() { + let fixture = take_fixture(false).await; + let no_key_schema = Arc::new(ArrowSchema::new(vec![ArrowField::new( + "payload", + DataType::UInt64, + true, + )])); + let batch = RecordBatch::try_new( + no_key_schema, + vec![Arc::new(UInt64Array::from(vec![0_u64]))], + ) + .unwrap(); + let err = take_plan(&fixture.dataset, rows_input(vec![batch]), &["s"]).unwrap_err(); + assert!(matches!(err, Error::InvalidInput { .. }), "{err}"); + assert!(err.to_string().contains("must have a column")); + + // Taking fields the input already has: nothing to read + let with_s_schema = Arc::new(ArrowSchema::new(vec![ + ArrowField::new(ROW_ADDR, DataType::UInt64, true), + ArrowField::new("s", DataType::Utf8, false), + ])); + let with_s_batch = RecordBatch::try_new( + with_s_schema, + vec![ + Arc::new(UInt64Array::from(vec![0_u64])), + Arc::new(StringArray::from(vec!["x"])), + ], + ) + .unwrap(); + let err = + take_plan(&fixture.dataset, rows_input(vec![with_s_batch]), &["s"]).unwrap_err(); + assert!(matches!(err, Error::InvalidInput { .. }), "{err}"); + assert!(err.to_string().contains("nothing to read")); + } + + /// with_new_children re-derives the row-stream source and preserves the schema + #[tokio::test] + async fn take_with_new_children() { + let fixture = take_fixture(false).await; + let input_schema = Arc::new(ArrowSchema::new(vec![ArrowField::new( + ROW_ID, + DataType::UInt64, + true, + )])); + let batch = RecordBatch::try_new( + input_schema, + vec![Arc::new(UInt64Array::from(vec![0_u64, 1]))], + ) + .unwrap(); + let input = rows_input(vec![batch]); + let plan: Arc = + Arc::new(take_plan(&fixture.dataset, input.clone(), &["s"]).unwrap()); + let rebuilt = plan.clone().with_new_children(vec![input]).unwrap(); + assert_eq!(plan.schema(), rebuilt.schema()); + assert!( + rebuilt + .downcast_ref::() + .unwrap() + .row_stream_input() + .is_some() + ); + } + + /// Take-mode nodes can be created and executed without an active + /// tokio runtime (required for DataFusion foreign table providers) + #[test] + fn no_context_take_rows() { + use lance_datafusion::datagen::DatafusionDatagenExt; + use lance_datagen::{BatchCount, RowCount}; + + let fixture = NoContextTestFixture::new(); + let dataset = Arc::new(fixture.dataset); + let input = lance_datagen::gen_batch() + .col(ROW_ID, lance_datagen::array::step::()) + .into_df_exec(RowCount::from(50), BatchCount::from(2)); + let plan = take_plan(&dataset, input, &["text"]).unwrap(); + plan.execute(0, Arc::new(TaskContext::default())).unwrap(); + } + } } diff --git a/rust/lance/src/io/exec/filtered_read_proto.rs b/rust/lance/src/io/exec/filtered_read_proto.rs index 4eb329506aa..f351e42b1a6 100644 --- a/rust/lance/src/io/exec/filtered_read_proto.rs +++ b/rust/lance/src/io/exec/filtered_read_proto.rs @@ -727,6 +727,59 @@ mod tests { ); } + /// A row-stream (take) exec serializes like any other: the input plan + /// travels as the node's child through the plan codec, and decoding + /// re-derives the row-stream selector from the child's schema + #[tokio::test] + async fn test_exec_to_proto_roundtrip_row_stream() { + use datafusion::physical_plan::stream::RecordBatchStreamAdapter; + use lance_core::{ROW_ID, ROW_ID_FIELD}; + use lance_datafusion::exec::OneShotExec; + + fn keys_input() -> Arc { + let schema = Arc::new(ArrowSchema::new(vec![ROW_ID_FIELD.clone()])); + let batch = arrow_array::RecordBatch::try_new( + schema.clone(), + vec![Arc::new(arrow_array::UInt64Array::from(vec![3u64, 1, 4]))], + ) + .unwrap(); + let stream = futures::stream::iter(vec![Ok(batch)]); + Arc::new(OneShotExec::new(Box::pin(RecordBatchStreamAdapter::new( + schema, stream, + )))) + } + + let dataset = make_test_dataset().await; + let ctx = SessionContext::new(); + let state = ctx.state(); + + let options = FilteredReadOptions::basic_full_read(&dataset); + let exec = FilteredReadExec::try_new(dataset.clone(), options, Some(keys_input())).unwrap(); + assert!(exec.row_stream_input().is_some()); + + let proto = filtered_read_exec_to_proto(&exec, &state).await.unwrap(); + + // The codec hands the decoded child back; the selector re-derives + // from its schema + let back = + filtered_read_exec_from_proto(proto, Some(dataset.clone()), Some(keys_input()), &state) + .await + .unwrap(); + assert!(back.row_stream_input().is_some()); + assert_eq!(exec.schema(), back.schema()); + assert_eq!( + exec.options().projection.field_ids, + back.options().projection.field_ids + ); + assert!( + back.row_stream_input() + .unwrap() + .schema() + .column_with_name(ROW_ID) + .is_some() + ); + } + #[tokio::test] async fn test_plan_proto_roundtrip() { let dataset = make_test_dataset().await; diff --git a/rust/lance/src/io/exec/fts.rs b/rust/lance/src/io/exec/fts.rs index add864c0ea9..138ee3bca6a 100644 --- a/rust/lance/src/io/exec/fts.rs +++ b/rust/lance/src/io/exec/fts.rs @@ -1,27 +1,29 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright The Lance Authors -use std::collections::HashMap; -use std::sync::Arc; +use std::cmp::Ordering; +use std::collections::{HashMap, HashSet}; +use std::sync::{Arc, OnceLock}; -use arrow::array::{AsArray, BooleanBuilder}; +use arrow::array::{AsArray, BooleanBuilder, ListBuilder, UInt32Builder}; use arrow::datatypes::{Float32Type, UInt64Type}; use arrow_array::{Array, BooleanArray, Float32Array, OffsetSizeTrait, RecordBatch, UInt64Array}; -use arrow_schema::{DataType, SchemaRef}; +use arrow_schema::{DataType, Field, SchemaRef}; use datafusion::common::{NullEquality, Statistics}; use datafusion::error::{DataFusionError, Result as DataFusionResult}; use datafusion::execution::SendableRecordBatchStream; use datafusion::physical_plan::empty::EmptyExec; use datafusion::physical_plan::execution_plan::{Boundedness, EmissionType}; -use datafusion::physical_plan::metrics::{ExecutionPlanMetricsSet, MetricsSet}; +use datafusion::physical_plan::metrics::{ExecutionPlanMetricsSet, Gauge, MetricsSet}; use datafusion::physical_plan::repartition::RepartitionExec; use datafusion::physical_plan::stream::RecordBatchStreamAdapter; use datafusion::physical_plan::union::UnionExec; use datafusion::physical_plan::{DisplayAs, DisplayFormatType, ExecutionPlan, PlanProperties}; use datafusion_physical_expr::expressions::Column; -use datafusion_physical_expr::{Distribution, EquivalenceProperties, Partitioning}; +use datafusion_physical_expr::{Distribution, EquivalenceProperties, Partitioning, PhysicalExpr}; +use datafusion_physical_plan::ExecutionPlanProperties; use datafusion_physical_plan::joins::{HashJoinExec, PartitionMode}; -use datafusion_physical_plan::metrics::{BaselineMetrics, Count}; +use datafusion_physical_plan::metrics::{BaselineMetrics, Count, Time}; use futures::future::try_join_all; use futures::stream::{self}; use futures::{FutureExt, StreamExt, TryStreamExt}; @@ -35,27 +37,217 @@ use lance_table::format::IndexMetadata; use super::PreFilterSource; use super::utils::{IndexMetrics, build_prefilter}; -use crate::index::scalar::inverted::{load_segment_details, load_segments}; +use crate::index::scalar::inverted::{ + ResolvedFtsField, fts_document_schema, load_segment_details, load_segments, + transform_fts_document_stream, +}; use crate::{Dataset, index::DatasetIndexInternalExt}; use lance_index::metrics::{ AND_CANDIDATES_PRUNED_BEFORE_RETURN_METRIC, AND_CANDIDATES_SEEN_METRIC, AND_FULL_SCORES_METRIC, - FREQS_COLLECTED_METRIC, MetricsCollector, + COMPOUND_ADDRESS_RESOLUTION_BATCHES_METRIC, COMPOUND_ADDRESSES_RESOLVED_METRIC, + COMPOUND_PEAK_ADDRESS_RESOLUTION_BATCH_SIZE_METRIC, COMPOUND_PEAK_BUFFERED_CANDIDATES_METRIC, + COMPOUND_SCORE_FLOOR_OVERFLOWS_METRIC, FREQS_COLLECTED_METRIC, MetricsCollector, }; use lance_index::scalar::inverted::builder::ScoredDoc; use lance_index::scalar::inverted::builder::document_input; use lance_index::scalar::inverted::document_tokenizer::{DocType, JsonTokenizer, LanceTokenizer}; use lance_index::scalar::inverted::query::{ - BoostQuery, FtsSearchParams, MatchQuery, PhraseQuery, Tokens, collect_query_tokens, - has_query_token, + BoostQuery, FtsQuery, FtsQueryNode, FtsSearchParams, MatchQuery, Operator, PhraseQuery, Tokens, + collect_query_tokens, has_query_token, }; use lance_index::scalar::inverted::tokenizer::document_tokenizer::TextTokenizer; use lance_index::scalar::inverted::{ - FTS_SCHEMA, InvertedIndex, MemBM25Scorer, SCORE_COL, build_global_bm25_scorer, - flat_bm25_search_stream_with_metrics, + DOC_INDEX_COL, DocumentGranularity, FTS_SCHEMA, FlatBm25SearchOptions, InvertedIndex, + MemBM25Scorer, SCORE_COL, build_global_bm25_scorer, compound_search, + compound_search_with_base_scorer, flat_bm25_search_stream_with_options_and_scorer, fts_schema, }; use lance_index::{prefilter::PreFilter, scalar::inverted::query::BooleanQuery}; +use lance_select::RowAddrMask; use lance_tokenizer::{SimpleTokenizer, TextAnalyzer}; use tracing::instrument; +use uuid::Uuid; + +#[derive(Debug, Clone, PartialEq, Eq)] +struct TokenWithPosition { + text: String, + position: u32, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +struct TokenizedQuery(Vec); + +impl TokenizedQuery { + fn from_tokens(tokens: &Tokens) -> Self { + let mut token_positions = Vec::with_capacity(tokens.len()); + for index in 0..tokens.len() { + token_positions.push(TokenWithPosition { + text: tokens.get_token(index).to_string(), + position: tokens.position(index), + }); + } + Self(token_positions) + } +} + +impl std::fmt::Display for TokenizedQuery { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "[")?; + for (index, token) in self.0.iter().enumerate() { + if index > 0 { + write!(f, ", ")?; + } + write!(f, "({:?}, {})", token.text, token.position)?; + } + write!(f, "]") + } +} + +fn record_tokenized_query(snapshot: &OnceLock, tokens: &Tokens) { + snapshot.get_or_init(|| TokenizedQuery::from_tokens(tokens)); +} + +fn fmt_tokenized_query( + snapshot: &OnceLock, + separator: &str, + f: &mut std::fmt::Formatter<'_>, +) -> std::fmt::Result { + if let Some(tokens) = snapshot.get() { + write!(f, "{separator}tokenized_query={tokens}")?; + } + Ok(()) +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum TokenizedLeafKind { + Match, + Phrase, +} + +impl std::fmt::Display for TokenizedLeafKind { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::Match => write!(f, "Match"), + Self::Phrase => write!(f, "Phrase"), + } + } +} + +#[derive(Debug, Clone, PartialEq, Eq)] +struct TokenizedQueryLeaf { + kind: TokenizedLeafKind, + column: Option, + tokens: TokenizedQuery, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +struct TokenizedCompoundQuery(Vec); + +impl std::fmt::Display for TokenizedCompoundQuery { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "[")?; + for (index, leaf) in self.0.iter().enumerate() { + if index > 0 { + write!(f, ", ")?; + } + write!( + f, + "{}(column={:?}, tokens={})", + leaf.kind, + leaf.column.as_deref().unwrap_or_default(), + leaf.tokens + )?; + } + write!(f, "]") + } +} + +fn fmt_tokenized_compound_query( + snapshot: &OnceLock, + separator: &str, + f: &mut std::fmt::Formatter<'_>, +) -> std::fmt::Result { + if let Some(tokens) = snapshot.get() { + write!(f, "{separator}tokenized_query={tokens}")?; + } + Ok(()) +} + +/// Expands a schema-derived nested FTS source into one canonical row per +/// logical document before flat search or index building consumes it. +#[derive(Debug)] +pub struct FtsDocumentExec { + input: Arc, + resolved: ResolvedFtsField, + properties: Arc, +} + +impl FtsDocumentExec { + pub(crate) fn new(input: Arc, resolved: ResolvedFtsField) -> Self { + let schema = fts_document_schema(resolved.coordinate_rank()); + let properties = Arc::new(PlanProperties::new( + EquivalenceProperties::new(schema), + input.output_partitioning().clone(), + EmissionType::Incremental, + Boundedness::Bounded, + )); + Self { + input, + resolved, + properties, + } + } +} + +impl DisplayAs for FtsDocumentExec { + fn fmt_as(&self, _t: DisplayFormatType, f: &mut std::fmt::Formatter) -> std::fmt::Result { + write!( + f, + "FtsDocument: column={}, granularity={:?}", + self.resolved.canonical_path, self.resolved.document_granularity + ) + } +} + +impl ExecutionPlan for FtsDocumentExec { + fn name(&self) -> &str { + "FtsDocumentExec" + } + + fn children(&self) -> Vec<&Arc> { + vec![&self.input] + } + + fn with_new_children( + self: Arc, + mut children: Vec>, + ) -> DataFusionResult> { + if children.len() != 1 { + return Err(DataFusionError::Internal( + "FtsDocumentExec expects one child".to_string(), + )); + } + Ok(Arc::new(Self::new( + children.pop().unwrap(), + self.resolved.clone(), + ))) + } + + fn execute( + &self, + partition: usize, + context: Arc, + ) -> DataFusionResult { + transform_fts_document_stream( + self.input.execute(partition, context)?, + self.resolved.clone(), + ) + .map_err(DataFusionError::from) + } + + fn properties(&self) -> &Arc { + &self.properties + } +} /// Open one FTS segment as an [`InvertedIndex`]. async fn open_fts_segment( @@ -105,7 +297,7 @@ async fn search_segments( pre_filter: Arc, metrics: Arc, base_scorer: Arc, -) -> Result<(Vec, Vec)> { +) -> Result> { let limit = params.limit.unwrap_or(usize::MAX); let mut candidates = std::collections::BinaryHeap::new(); let searches = indices @@ -119,7 +311,7 @@ async fn search_segments( let base_scorer = base_scorer.clone(); async move { index - .bm25_search( + .bm25_search_documents( tokens, params, operator, @@ -134,13 +326,13 @@ async fn search_segments( let searches = stream::iter(searches).buffer_unordered(get_num_compute_intensive_cpus()); let mut searches = searches; - while let Some((doc_ids, scores)) = searches.try_next().await? { - for (row_id, score) in doc_ids.into_iter().zip(scores.into_iter()) { + while let Some(documents) = searches.try_next().await? { + for document in documents { if candidates.len() < limit { - candidates.push(std::cmp::Reverse(ScoredDoc::new(row_id, score))); - } else if candidates.peek().unwrap().0.score.0 < score { + candidates.push(std::cmp::Reverse(document)); + } else if candidates.peek().unwrap().0.score < document.score { candidates.pop(); - candidates.push(std::cmp::Reverse(ScoredDoc::new(row_id, score))); + candidates.push(std::cmp::Reverse(document)); } } } @@ -148,8 +340,448 @@ async fn search_segments( Ok(candidates .into_sorted_vec() .into_iter() - .map(|std::cmp::Reverse(doc)| (doc.row_id, doc.score.0)) - .unzip()) + .map(|std::cmp::Reverse(document)| document) + .collect()) +} + +fn scored_documents_batch(schema: SchemaRef, documents: Vec) -> Result { + let row_ids = UInt64Array::from_iter_values(documents.iter().map(|document| document.row_id)); + let scores = Float32Array::from_iter_values(documents.iter().map(|document| document.score.0)); + let mut columns = vec![Arc::new(row_ids) as Arc]; + if schema.field_with_name(DOC_INDEX_COL).is_ok() { + let mut builder = ListBuilder::new(UInt32Builder::new()).with_field(Field::new( + "item", + DataType::UInt32, + false, + )); + for document in &documents { + builder.values().append_slice(&document.doc_index); + builder.append(true); + } + columns.push(Arc::new(builder.finish())); + } + columns.push(Arc::new(scores)); + Ok(RecordBatch::try_new(schema, columns)?) +} + +#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)] +struct DocumentKey { + row_id: u64, + doc_index: Vec, +} + +fn batch_document_keys(batch: &RecordBatch) -> Result> { + let row_ids = batch[ROW_ID].as_primitive::(); + let doc_indices = batch + .column_by_name(DOC_INDEX_COL) + .map(|column| column.as_list::()); + (0..batch.num_rows()) + .map(|row| { + let doc_index = if let Some(doc_indices) = doc_indices { + if doc_indices.is_null(row) { + return Err(Error::internal( + "element-document FTS produced a null document coordinate".to_string(), + )); + } + doc_indices + .value(row) + .as_primitive::() + .values() + .to_vec() + } else { + Vec::new() + }; + Ok(DocumentKey { + row_id: row_ids.value(row), + doc_index, + }) + }) + .collect() +} + +fn batch_scored_document_keys(batch: &RecordBatch) -> Result> { + let keys = batch_document_keys(batch)?; + let scores = batch[SCORE_COL].as_primitive::(); + Ok(keys + .into_iter() + .enumerate() + .map(|(index, key)| (key, scores.value(index))) + .collect()) +} + +fn batch_scored_document_keys_sum_scores(batch: &RecordBatch) -> Result> { + let keys = batch_document_keys(batch)?; + let schema = batch.schema(); + let score_columns = schema + .fields() + .iter() + .enumerate() + .filter(|(_, field)| field.name() == SCORE_COL) + .map(|(index, _)| batch.column(index).as_primitive::()) + .collect::>(); + if score_columns.is_empty() { + return Err(Error::internal(format!( + "Boolean MUST result is missing required {SCORE_COL} columns" + ))); + } + keys.into_iter() + .enumerate() + .map(|(row, key)| { + let score: f32 = score_columns.iter().map(|scores| scores.value(row)).sum(); + if !score.is_finite() { + return Err(Error::internal(format!( + "Boolean MUST score sum must be finite, got {score} for row_id={}", + key.row_id + ))); + } + Ok((key, score)) + }) + .collect() +} + +fn document_key_scores_batch( + schema: SchemaRef, + values: impl IntoIterator, +) -> Result { + scored_documents_batch( + schema, + values + .into_iter() + .map(|(key, score)| ScoredDoc::with_doc_index(key.row_id, key.doc_index, score)) + .collect(), + ) +} + +fn compare_scored_documents( + (left_key, left_score): &(DocumentKey, f32), + (right_key, right_score): &(DocumentKey, f32), +) -> Ordering { + right_score + .total_cmp(left_score) + .then_with(|| left_key.cmp(right_key)) +} + +fn count_fts_leaves(query: &FtsQuery) -> usize { + match query { + FtsQuery::Match(_) | FtsQuery::Phrase(_) => 1, + FtsQuery::Boost(query) => { + count_fts_leaves(&query.positive) + count_fts_leaves(&query.negative) + } + FtsQuery::MultiMatch(query) => query.match_queries.len(), + FtsQuery::Boolean(query) => query + .should + .iter() + .chain(&query.must) + .chain(&query.must_not) + .map(count_fts_leaves) + .sum(), + } +} + +/// One DataFusion boundary around a posting-backed compound scorer tree. +#[derive(Debug)] +pub struct CompoundQueryExec { + dataset: Arc, + query: FtsQuery, + tokenized_query: Arc>, + params: FtsSearchParams, + prefilter_source: PreFilterSource, + /// When set, leaf scorers use this instead of building one from the + /// searched segments — see [`MatchQueryExec::with_base_scorer`]. + base_scorer: Option>, + segment_selection: FtsSegmentSelection, + properties: Arc, + metrics: ExecutionPlanMetricsSet, +} + +impl CompoundQueryExec { + pub fn new_with_segments( + dataset: Arc, + query: FtsQuery, + params: FtsSearchParams, + prefilter_source: PreFilterSource, + segments: Vec, + ) -> Self { + Self::new_inner( + dataset, + query, + params, + prefilter_source, + FtsSegmentSelection::ExactResolved(Arc::from(segments)), + ) + } + + pub fn new_with_segment_uuids( + dataset: Arc, + query: FtsQuery, + params: FtsSearchParams, + prefilter_source: PreFilterSource, + segment_uuids: Vec, + ) -> Self { + Self::new_inner( + dataset, + query, + params, + prefilter_source, + FtsSegmentSelection::exact_uuids(segment_uuids), + ) + } + + fn new_inner( + dataset: Arc, + query: FtsQuery, + params: FtsSearchParams, + prefilter_source: PreFilterSource, + segment_selection: FtsSegmentSelection, + ) -> Self { + Self { + dataset, + query, + tokenized_query: Arc::new(OnceLock::new()), + params, + prefilter_source, + base_scorer: None, + segment_selection, + properties: Arc::new(PlanProperties::new( + EquivalenceProperties::new(FTS_SCHEMA.clone()), + Partitioning::RoundRobinBatch(1), + EmissionType::Final, + Boundedness::Bounded, + )), + metrics: ExecutionPlanMetricsSet::new(), + } + } + + /// Override locally computed BM25 statistics with a corpus-wide scorer. + /// + /// The scorer must cover every token in every query leaf, including fuzzy + /// expansions. Execution returns an error when any required token is absent. + pub fn with_base_scorer(mut self, scorer: Arc) -> Self { + self.base_scorer = Some(scorer); + self + } + + pub fn dataset(&self) -> &Arc { + &self.dataset + } + + pub fn query(&self) -> &FtsQuery { + &self.query + } + + pub fn params(&self) -> &FtsSearchParams { + &self.params + } + + pub fn prefilter_source(&self) -> &PreFilterSource { + &self.prefilter_source + } + + pub fn base_scorer(&self) -> Option<&Arc> { + self.base_scorer.as_ref() + } + + /// See [`MatchQueryExec::explicit_segment_uuids`]. + pub fn explicit_segment_uuids(&self) -> Option> { + self.segment_selection.explicit_segment_uuids() + } +} + +impl DisplayAs for CompoundQueryExec { + fn fmt_as(&self, t: DisplayFormatType, f: &mut std::fmt::Formatter) -> std::fmt::Result { + match t { + DisplayFormatType::Default | DisplayFormatType::Verbose => { + write!(f, "CompoundFtsScorer: query={}", self.query)?; + fmt_tokenized_compound_query(&self.tokenized_query, ", ", f) + } + DisplayFormatType::TreeRender => { + write!(f, "CompoundFtsScorer\nquery={}", self.query)?; + fmt_tokenized_compound_query(&self.tokenized_query, "\n", f) + } + } + } +} + +impl ExecutionPlan for CompoundQueryExec { + fn name(&self) -> &str { + "CompoundQueryExec" + } + + fn children(&self) -> Vec<&Arc> { + match &self.prefilter_source { + PreFilterSource::None => vec![], + PreFilterSource::FilteredRowIds(source) | PreFilterSource::ScalarIndexQuery(source) => { + vec![source] + } + } + } + + fn required_input_distribution(&self) -> Vec { + self.children() + .iter() + .map(|_| Distribution::SinglePartition) + .collect() + } + + fn with_new_children( + self: Arc, + mut children: Vec>, + ) -> DataFusionResult> { + let prefilter_source = match children.len() { + 0 if matches!(self.prefilter_source, PreFilterSource::None) => PreFilterSource::None, + 1 => { + let Some(source) = children.pop() else { + return Err(DataFusionError::Internal( + "compound FTS lost its prefilter child".to_string(), + )); + }; + match &self.prefilter_source { + PreFilterSource::FilteredRowIds(_) => PreFilterSource::FilteredRowIds(source), + PreFilterSource::ScalarIndexQuery(_) => { + PreFilterSource::ScalarIndexQuery(source) + } + PreFilterSource::None => { + return Err(DataFusionError::Internal( + "compound FTS received an unexpected prefilter child".to_string(), + )); + } + } + } + count => { + return Err(DataFusionError::Internal(format!( + "compound FTS expected at most one prefilter child, got {count}" + ))); + } + }; + Ok(Arc::new(Self { + dataset: self.dataset.clone(), + query: self.query.clone(), + tokenized_query: self.tokenized_query.clone(), + params: self.params.clone(), + prefilter_source, + base_scorer: self.base_scorer.clone(), + segment_selection: self.segment_selection.clone(), + properties: self.properties.clone(), + metrics: ExecutionPlanMetricsSet::new(), + })) + } + + #[instrument(name = "compound_fts_scorer_exec", level = "debug", skip_all)] + fn execute( + &self, + partition: usize, + context: Arc, + ) -> DataFusionResult { + let dataset = self.dataset.clone(); + let query = self.query.clone(); + let tokenized_query = self.tokenized_query.clone(); + let params = self.params.clone(); + let prefilter_source = self.prefilter_source.clone(); + let base_scorer = self.base_scorer.clone(); + let segment_selection = self.segment_selection.clone(); + let metrics = Arc::new(FtsIndexMetrics::new(&self.metrics, partition)); + + let stream = stream::once(async move { + let _timer = metrics.baseline_metrics.elapsed_compute().timer(); + let columns = query.columns(); + let column = columns.iter().next().ok_or_else(|| { + DataFusionError::Execution( + "compound FTS query does not reference an indexed column".to_string(), + ) + })?; + if columns.len() != 1 { + return Err(DataFusionError::Execution( + "posting-backed compound FTS requires exactly one column".to_string(), + )); + } + let segments = segment_selection + .resolve( + &dataset, + column, + DocumentGranularity::Row, + &metrics.segment_bind_duration, + ) + .await?; + let _details = load_segment_details(&dataset, column, &segments).await?; + let indices = + open_fts_segments(&dataset, column, &segments, &metrics.index_metrics).await?; + if let Some(first_index) = indices.first() { + tokenized_query + .get_or_init(|| tokenize_compound_query(&query, first_index.as_ref())); + } + let mut prefilter = build_prefilter( + context, + partition, + &prefilter_source, + dataset, + &segments, + None, + )?; + let deleted_fragments = + indices + .iter() + .fold(roaring::RoaringBitmap::new(), |mut deleted, index| { + deleted |= index.deleted_fragments().clone(); + deleted + }); + if !deleted_fragments.is_empty() { + let prefilter = Arc::get_mut(&mut prefilter).ok_or_else(|| { + DataFusionError::Internal( + "compound FTS prefilter was unexpectedly shared before initialization" + .to_string(), + ) + })?; + prefilter.set_deleted_fragments(deleted_fragments); + } + metrics.record_parts_searched( + indices + .iter() + .map(|index| index.partition_count()) + .sum::() + .saturating_mul(count_fts_leaves(&query)), + ); + let (row_ids, scores) = match base_scorer { + Some(base_scorer) => { + compound_search_with_base_scorer( + &indices, + &query, + ¶ms, + prefilter, + metrics.clone(), + base_scorer, + ) + .await? + } + None => { + compound_search(&indices, &query, ¶ms, prefilter, metrics.clone()).await? + } + }; + metrics.baseline_metrics.record_output(row_ids.len()); + Ok::<_, DataFusionError>(RecordBatch::try_new( + FTS_SCHEMA.clone(), + vec![ + Arc::new(UInt64Array::from(row_ids)), + Arc::new(Float32Array::from(scores)), + ], + )?) + }); + Ok(Box::pin(RecordBatchStreamAdapter::new( + self.schema(), + stream.stream_in_current_span().boxed(), + ))) + } + + fn metrics(&self) -> Option { + Some(self.metrics.clone_inner()) + } + + fn properties(&self) -> &Arc { + &self.properties + } + + fn supports_limit_pushdown(&self) -> bool { + false + } } /// Fall back to the default simple tokenizer when no on-disk FTS segment exists. @@ -159,6 +791,261 @@ fn default_text_tokenizer() -> Box { )) } +fn tokenizer_for_match_query( + index: &InvertedIndex, + fuzziness: Option, +) -> Box { + if !matches!(fuzziness, Some(distance) if distance != 0) { + return index.tokenizer(); + } + + let analyzer = TextAnalyzer::from(SimpleTokenizer::default()); + match index.tokenizer().doc_type() { + DocType::Text => Box::new(TextTokenizer::new(analyzer)), + DocType::Json => Box::new(JsonTokenizer::new(analyzer)), + } +} + +fn tokenize_compound_query(query: &FtsQuery, index: &InvertedIndex) -> TokenizedCompoundQuery { + fn visit(query: &FtsQuery, index: &InvertedIndex, leaves: &mut Vec) { + match query { + FtsQuery::Match(query) => { + let mut tokenizer = tokenizer_for_match_query(index, query.fuzziness); + let tokens = collect_query_tokens(&query.terms, &mut tokenizer); + leaves.push(TokenizedQueryLeaf { + kind: TokenizedLeafKind::Match, + column: query.column.clone(), + tokens: TokenizedQuery::from_tokens(&tokens), + }); + } + FtsQuery::Phrase(query) => { + let mut tokenizer = index.tokenizer(); + let tokens = collect_query_tokens(&query.terms, &mut tokenizer); + leaves.push(TokenizedQueryLeaf { + kind: TokenizedLeafKind::Phrase, + column: query.column.clone(), + tokens: TokenizedQuery::from_tokens(&tokens), + }); + } + FtsQuery::Boost(query) => { + visit(&query.positive, index, leaves); + visit(&query.negative, index, leaves); + } + FtsQuery::MultiMatch(query) => { + for query in &query.match_queries { + let mut tokenizer = tokenizer_for_match_query(index, query.fuzziness); + let tokens = collect_query_tokens(&query.terms, &mut tokenizer); + leaves.push(TokenizedQueryLeaf { + kind: TokenizedLeafKind::Match, + column: query.column.clone(), + tokens: TokenizedQuery::from_tokens(&tokens), + }); + } + } + FtsQuery::Boolean(query) => { + for query in query + .should + .iter() + .chain(&query.must) + .chain(&query.must_not) + { + visit(query, index, leaves); + } + } + } + } + + let mut leaves = Vec::with_capacity(count_fts_leaves(query)); + visit(query, index, &mut leaves); + TokenizedCompoundQuery(leaves) +} + +type SharedScorerResult = std::result::Result, Arc>; + +/// Coordinates BM25 corpus statistics between the indexed and flat branches +/// of a mixed search. The flat branch extends the indexed statistics with the +/// unindexed documents, then publishes the resulting corpus-wide scorer. +#[derive(Debug)] +pub(crate) struct SharedFtsScorer { + sender: tokio::sync::watch::Sender>, +} + +impl SharedFtsScorer { + pub(crate) fn new() -> Self { + let (sender, _) = tokio::sync::watch::channel(None); + Self { sender } + } + + fn publish(&self, scorer: MemBM25Scorer) { + self.sender.send_replace(Some(Ok(Arc::new(scorer)))); + } + + fn publish_error(&self, error: &DataFusionError) { + self.sender + .send_replace(Some(Err(Arc::from(error.to_string())))); + } + + async fn wait(&self) -> DataFusionResult> { + let mut receiver = self.sender.subscribe(); + loop { + let result = receiver.borrow_and_update().clone(); + if let Some(result) = result { + return result.map_err(|message| DataFusionError::Execution(message.to_string())); + } + receiver.changed().await.map_err(|_| { + DataFusionError::Execution( + "mixed FTS corpus scorer producer stopped before publishing statistics" + .to_string(), + ) + })?; + } + } +} + +struct SharedFtsScorerProducer { + scorer: Arc, + completed: bool, +} + +impl SharedFtsScorerProducer { + fn new(scorer: Arc) -> Self { + Self { + scorer, + completed: false, + } + } + + fn publish(mut self, scorer: MemBM25Scorer) { + self.scorer.publish(scorer); + self.completed = true; + } + + fn publish_error(mut self, error: &DataFusionError) { + self.scorer.publish_error(error); + self.completed = true; + } +} + +impl Drop for SharedFtsScorerProducer { + fn drop(&mut self) { + if !self.completed { + self.scorer.sender.send_replace(Some(Err(Arc::from( + "mixed FTS corpus scorer producer was cancelled before publishing statistics", + )))); + } + } +} + +/// Time spent resolving an exact ordered UUID selection to committed FTS segments. +pub const FTS_SEGMENT_BIND_DURATION_METRIC: &str = "fts_segment_bind_duration"; + +#[derive(Debug, Clone)] +enum FtsSegmentSelection { + AllCommitted, + ExactResolved(Arc<[IndexMetadata]>), + ExactUuids(Arc<[Uuid]>), +} + +impl FtsSegmentSelection { + fn exact_uuids(mut uuids: Vec) -> Self { + let mut seen = HashSet::with_capacity(uuids.len()); + uuids.retain(|uuid| seen.insert(*uuid)); + Self::ExactUuids(Arc::from(uuids)) + } + + fn preset_segments(&self) -> Option<&[IndexMetadata]> { + match self { + Self::ExactResolved(segments) => Some(segments), + Self::AllCommitted | Self::ExactUuids(_) => None, + } + } + + fn explicit_segment_uuids(&self) -> Option> { + match self { + Self::AllCommitted => None, + Self::ExactResolved(segments) => { + Some(segments.iter().map(|segment| segment.uuid).collect()) + } + Self::ExactUuids(uuids) => Some(uuids.to_vec()), + } + } + + async fn resolve( + &self, + dataset: &Dataset, + column: &str, + document_granularity: DocumentGranularity, + segment_bind_duration: &Time, + ) -> DataFusionResult> { + let segments = match self { + Self::AllCommitted => load_segments(dataset, column, document_granularity) + .await? + .map(Arc::from) + .ok_or_else(|| { + DataFusionError::Execution(format!( + "No Inverted index found for column {}", + column, + )) + }), + Self::ExactResolved(segments) => Ok(segments.clone()), + Self::ExactUuids(uuids) => { + let _timer = segment_bind_duration.timer(); + let dataset_version = dataset.version_id(); + if uuids.is_empty() { + return Err(DataFusionError::Execution(format!( + "Exact FTS segment selection for column {} at dataset version {} \ + requires at least one segment UUID", + column, dataset_version + ))); + } + + let committed_segments = load_segments(dataset, column, document_granularity) + .await? + .ok_or_else(|| { + DataFusionError::Execution(format!( + "Cannot resolve exact FTS segment selection for column {} at dataset \ + version {}: no Inverted index found", + column, dataset_version + )) + })?; + let mut segments_by_uuid = HashMap::with_capacity(committed_segments.len()); + for segment in committed_segments { + let uuid = segment.uuid; + if segments_by_uuid.insert(uuid, segment).is_some() { + return Err(DataFusionError::Execution(format!( + "FTS metadata for column {} at dataset version {} contains duplicate \ + segment UUID {}", + column, dataset_version, uuid + ))); + } + } + + let mut resolved = Vec::with_capacity(uuids.len()); + for uuid in uuids.iter() { + let segment = segments_by_uuid.get(uuid).ok_or_else(|| { + DataFusionError::Execution(format!( + "Requested FTS segment UUID {} for column {} is not committed in \ + dataset version {}", + uuid, column, dataset_version + )) + })?; + resolved.push(segment.clone()); + } + Ok(Arc::from(resolved)) + } + }?; + let details = load_segment_details(dataset, column, &segments).await?; + let indexed_granularity = DocumentGranularity::try_from(details.document_granularity)?; + if indexed_granularity != document_granularity { + return Err(DataFusionError::Execution(format!( + "FTS segments selected for column {column} use {indexed_granularity:?} document \ + granularity, but the query was resolved as {document_granularity:?}" + ))); + } + Ok(segments) + } +} + pub struct FtsIndexMetrics { index_metrics: IndexMetrics, partitions_searched: Count, @@ -166,6 +1053,15 @@ pub struct FtsIndexMetrics { and_candidates_pruned_before_return: Count, and_full_scores: Count, freqs_collected: Count, + compound_addresses_resolved: Count, + compound_address_resolution_batches: Count, + compound_peak_address_resolution_batch_size: Gauge, + compound_score_floor_overflows: Count, + compound_peak_buffered_candidates: Gauge, + /// Wall time (ms) of the exec-local `build_global_bm25_scorer` + /// fallback; zero when a preset base scorer was injected. + scorer_build_ms: Gauge, + segment_bind_duration: Time, baseline_metrics: BaselineMetrics, } @@ -179,6 +1075,20 @@ impl FtsIndexMetrics { .new_count(AND_CANDIDATES_PRUNED_BEFORE_RETURN_METRIC, partition), and_full_scores: metrics.new_count(AND_FULL_SCORES_METRIC, partition), freqs_collected: metrics.new_count(FREQS_COLLECTED_METRIC, partition), + compound_addresses_resolved: metrics + .new_count(COMPOUND_ADDRESSES_RESOLVED_METRIC, partition), + compound_address_resolution_batches: metrics + .new_count(COMPOUND_ADDRESS_RESOLUTION_BATCHES_METRIC, partition), + compound_peak_address_resolution_batch_size: metrics.new_gauge( + COMPOUND_PEAK_ADDRESS_RESOLUTION_BATCH_SIZE_METRIC, + partition, + ), + compound_score_floor_overflows: metrics + .new_count(COMPOUND_SCORE_FLOOR_OVERFLOWS_METRIC, partition), + compound_peak_buffered_candidates: metrics + .new_gauge(COMPOUND_PEAK_BUFFERED_CANDIDATES_METRIC, partition), + scorer_build_ms: metrics.new_gauge("scorer_build_ms", partition), + segment_bind_duration: metrics.new_time(FTS_SEGMENT_BIND_DURATION_METRIC, partition), baseline_metrics: BaselineMetrics::new(metrics, partition), } } @@ -186,6 +1096,10 @@ impl FtsIndexMetrics { pub fn record_parts_searched(&self, num_parts: usize) { self.partitions_searched.add(num_parts); } + + pub fn record_scorer_build(&self, elapsed: std::time::Duration) { + self.scorer_build_ms.set(elapsed.as_millis() as usize); + } } impl MetricsCollector for FtsIndexMetrics { @@ -201,6 +1115,14 @@ impl MetricsCollector for FtsIndexMetrics { self.index_metrics.record_comparisons(num_comparisons); } + fn record_index_cache_hits(&self, num_hits: usize) { + self.index_metrics.record_index_cache_hits(num_hits); + } + + fn record_index_cache_misses(&self, num_misses: usize) { + self.index_metrics.record_index_cache_misses(num_misses); + } + fn record_and_candidates_seen(&self, num_candidates: usize) { self.and_candidates_seen.add(num_candidates); } @@ -213,8 +1135,30 @@ impl MetricsCollector for FtsIndexMetrics { self.and_full_scores.add(num_scores); } - fn record_freqs_collected(&self, num_collections: usize) { - self.freqs_collected.add(num_collections); + fn record_freqs_collected(&self, num_collections: usize) { + self.freqs_collected.add(num_collections); + } + + fn record_compound_addresses_resolved(&self, num_addresses: usize) { + self.compound_addresses_resolved.add(num_addresses); + } + + fn record_compound_address_resolution_batches(&self, num_batches: usize) { + self.compound_address_resolution_batches.add(num_batches); + } + + fn record_compound_peak_address_resolution_batch_size(&self, num_addresses: usize) { + self.compound_peak_address_resolution_batch_size + .set_max(num_addresses); + } + + fn record_compound_score_floor_overflows(&self, num_overflows: usize) { + self.compound_score_floor_overflows.add(num_overflows); + } + + fn record_compound_peak_buffered_candidates(&self, num_candidates: usize) { + self.compound_peak_buffered_candidates + .set_max(num_candidates); } } @@ -222,14 +1166,19 @@ impl MetricsCollector for FtsIndexMetrics { pub struct MatchQueryExec { dataset: Arc, query: MatchQuery, + tokenized_query: Arc>, params: FtsSearchParams, prefilter_source: PreFilterSource, /// When set, `execute()` skips `build_global_bm25_scorer` and threads this /// scorer down to `InvertedIndex::bm25_search`. base_scorer: Option>, - /// When set, `execute()` skips `load_segments` and searches exactly these - /// segments. - preset_segments: Option>, + /// Corpus-wide scorer published by the flat branch of a mixed search. + shared_scorer: Option>, + segment_selection: FtsSegmentSelection, + /// Rows whose indexed values were superseded by newer data overlays. + overlay_block: Option, + document_granularity: DocumentGranularity, + schema: SchemaRef, properties: Arc, metrics: ExecutionPlanMetricsSet, @@ -241,10 +1190,11 @@ impl DisplayAs for MatchQueryExec { DisplayFormatType::Default | DisplayFormatType::Verbose => { write!( f, - "MatchQuery: column={}, query={}", + "MatchQuery: column={}, query=[{}]", self.query.column.as_deref().unwrap_or_default(), self.query.terms - ) + )?; + fmt_tokenized_query(&self.tokenized_query, ", ", f) } DisplayFormatType::TreeRender => { write!( @@ -252,7 +1202,8 @@ impl DisplayAs for MatchQueryExec { "MatchQuery\ncolumn={}\nquery={}", self.query.column.as_deref().unwrap_or_default(), self.query.terms - ) + )?; + fmt_tokenized_query(&self.tokenized_query, "\n", f) } } } @@ -273,9 +1224,29 @@ impl MatchQueryExec { query: MatchQuery, params: FtsSearchParams, prefilter_source: PreFilterSource, + ) -> Result { + let document_granularity = query.document_granularity.ok_or_else(|| { + Error::invalid_input("MatchQuery document granularity must be resolved".to_string()) + })?; + Ok(Self::new_with_document_granularity( + dataset, + query, + params, + prefilter_source, + document_granularity, + )) + } + + pub fn new_with_document_granularity( + dataset: Arc, + query: MatchQuery, + params: FtsSearchParams, + prefilter_source: PreFilterSource, + document_granularity: DocumentGranularity, ) -> Self { + let schema = fts_schema(document_granularity); let properties = Arc::new(PlanProperties::new( - EquivalenceProperties::new(FTS_SCHEMA.clone()), + EquivalenceProperties::new(schema.clone()), Partitioning::RoundRobinBatch(1), EmissionType::Final, Boundedness::Bounded, @@ -284,10 +1255,15 @@ impl MatchQueryExec { Self { dataset, query, + tokenized_query: Arc::new(OnceLock::new()), params, prefilter_source, base_scorer: None, - preset_segments: None, + shared_scorer: None, + segment_selection: FtsSegmentSelection::AllCommitted, + overlay_block: None, + document_granularity, + schema, properties, metrics: ExecutionPlanMetricsSet::new(), } @@ -308,9 +1284,31 @@ impl MatchQueryExec { params: FtsSearchParams, prefilter_source: PreFilterSource, segments: Vec, + ) -> Result { + let document_granularity = query.document_granularity.ok_or_else(|| { + Error::invalid_input("MatchQuery document granularity must be resolved".to_string()) + })?; + Ok(Self::new_with_segments_and_document_granularity( + dataset, + query, + params, + prefilter_source, + segments, + document_granularity, + )) + } + + pub fn new_with_segments_and_document_granularity( + dataset: Arc, + query: MatchQuery, + params: FtsSearchParams, + prefilter_source: PreFilterSource, + segments: Vec, + document_granularity: DocumentGranularity, ) -> Self { + let schema = fts_schema(document_granularity); let properties = Arc::new(PlanProperties::new( - EquivalenceProperties::new(FTS_SCHEMA.clone()), + EquivalenceProperties::new(schema.clone()), Partitioning::RoundRobinBatch(1), EmissionType::Final, Boundedness::Bounded, @@ -319,15 +1317,62 @@ impl MatchQueryExec { Self { dataset, query, + tokenized_query: Arc::new(OnceLock::new()), params, prefilter_source, base_scorer: None, - preset_segments: Some(segments), + shared_scorer: None, + segment_selection: FtsSegmentSelection::ExactResolved(Arc::from(segments)), + overlay_block: None, + document_granularity, + schema, properties, metrics: ExecutionPlanMetricsSet::new(), } } + /// Construct a `MatchQueryExec` bound to an exact ordered set of committed + /// FTS segment UUIDs. + /// + /// The UUIDs are resolved from this exec's dataset snapshot when the output + /// stream is polled. Duplicate UUIDs are removed while preserving their + /// first-occurrence order. Resolution fails if the list is empty or any UUID + /// is not committed for the query column. + pub fn new_with_segment_uuids( + dataset: Arc, + query: MatchQuery, + params: FtsSearchParams, + prefilter_source: PreFilterSource, + segment_uuids: Vec, + ) -> Result { + let document_granularity = query.document_granularity.ok_or_else(|| { + Error::invalid_input("MatchQuery document granularity must be resolved".to_string()) + })?; + let schema = fts_schema(document_granularity); + let properties = Arc::new(PlanProperties::new( + EquivalenceProperties::new(schema.clone()), + Partitioning::RoundRobinBatch(1), + EmissionType::Final, + Boundedness::Bounded, + )); + let params = Self::effective_params(&query, params); + Ok(Self { + dataset, + query, + tokenized_query: Arc::new(OnceLock::new()), + params, + prefilter_source, + base_scorer: None, + shared_scorer: None, + segment_selection: FtsSegmentSelection::exact_uuids(segment_uuids), + overlay_block: None, + document_granularity, + schema, + properties, + metrics: ExecutionPlanMetricsSet::new(), + }) + } + /// Override the BM25 scorer used by `execute()`. When set, the local /// `build_global_bm25_scorer` call is skipped and the supplied scorer is /// threaded down to `InvertedIndex::bm25_search`. @@ -345,6 +1390,17 @@ impl MatchQueryExec { self } + pub(crate) fn with_shared_scorer(mut self, scorer: Arc) -> Self { + self.shared_scorer = Some(scorer); + self + } + + /// Exclude rows whose indexed text was superseded by a newer data overlay. + pub(crate) fn with_overlay_block(mut self, overlay_block: RowAddrMask) -> Self { + self.overlay_block = Some(overlay_block); + self + } + pub fn query(&self) -> &MatchQuery { &self.query } @@ -366,7 +1422,16 @@ impl MatchQueryExec { } pub fn preset_segments(&self) -> Option<&[IndexMetadata]> { - self.preset_segments.as_deref() + self.segment_selection.preset_segments() + } + + /// Return the ordered segment UUIDs for an explicit selection. + /// + /// Returns `None` when this exec searches all committed segments. UUID-based + /// selections omit duplicates while preserving first-occurrence order. + /// Pre-resolved selections preserve the supplied metadata order. + pub fn explicit_segment_uuids(&self) -> Option> { + self.segment_selection.explicit_segment_uuids() } } @@ -375,10 +1440,6 @@ impl ExecutionPlan for MatchQueryExec { "MatchQueryExec" } - fn as_any(&self) -> &dyn std::any::Any { - self - } - fn children(&self) -> Vec<&Arc> { match &self.prefilter_source { PreFilterSource::None => vec![], @@ -410,10 +1471,15 @@ impl ExecutionPlan for MatchQueryExec { Self { dataset: self.dataset.clone(), query: self.query.clone(), + tokenized_query: self.tokenized_query.clone(), params: self.params.clone(), prefilter_source: PreFilterSource::None, base_scorer: self.base_scorer.clone(), - preset_segments: self.preset_segments.clone(), + shared_scorer: self.shared_scorer.clone(), + segment_selection: self.segment_selection.clone(), + overlay_block: self.overlay_block.clone(), + document_granularity: self.document_granularity, + schema: self.schema.clone(), properties: self.properties.clone(), metrics: ExecutionPlanMetricsSet::new(), } @@ -437,10 +1503,15 @@ impl ExecutionPlan for MatchQueryExec { Self { dataset: self.dataset.clone(), query: self.query.clone(), + tokenized_query: self.tokenized_query.clone(), params: self.params.clone(), prefilter_source, base_scorer: self.base_scorer.clone(), - preset_segments: self.preset_segments.clone(), + shared_scorer: self.shared_scorer.clone(), + segment_selection: self.segment_selection.clone(), + overlay_block: self.overlay_block.clone(), + document_granularity: self.document_granularity, + schema: self.schema.clone(), properties: self.properties.clone(), metrics: ExecutionPlanMetricsSet::new(), } @@ -461,11 +1532,16 @@ impl ExecutionPlan for MatchQueryExec { context: Arc, ) -> DataFusionResult { let query = self.query.clone(); + let tokenized_query = self.tokenized_query.clone(); let params = self.params.clone(); let ds = self.dataset.clone(); let prefilter_source = self.prefilter_source.clone(); let preset_base_scorer = self.base_scorer.clone(); - let preset_segments = self.preset_segments.clone(); + let shared_scorer = self.shared_scorer.clone(); + let segment_selection = self.segment_selection.clone(); + let overlay_block = self.overlay_block.clone(); + let document_granularity = self.document_granularity; + let schema = self.schema.clone(); let metrics = Arc::new(FtsIndexMetrics::new(&self.metrics, partition)); let column = query.column.ok_or(DataFusionError::Execution(format!( "column not set for MatchQuery {}", @@ -473,21 +1549,25 @@ impl ExecutionPlan for MatchQueryExec { )))?; let stream = stream::once(async move { let _timer = metrics.baseline_metrics.elapsed_compute().timer(); - let segments = match preset_segments { - Some(segments) => segments, - None => load_segments(&ds, &column) - .await? - .ok_or(DataFusionError::Execution(format!( - "No Inverted index found for column {}", - column, - )))?, - }; - let _details = load_segment_details(&ds, &column, &segments).await?; + let segments = segment_selection + .resolve( + &ds, + &column, + document_granularity, + &metrics.segment_bind_duration, + ) + .await?; let indices = open_fts_segments(&ds, &column, &segments, &metrics.index_metrics).await?; - let mut pre_filter = - build_prefilter(context.clone(), partition, &prefilter_source, ds, &segments)?; + let mut pre_filter = build_prefilter( + context.clone(), + partition, + &prefilter_source, + ds, + &segments, + overlay_block, + )?; let deleted_fragments = indices .iter() @@ -503,39 +1583,37 @@ impl ExecutionPlan for MatchQueryExec { metrics .record_parts_searched(indices.iter().map(|index| index.partition_count()).sum()); - let is_fuzzy = matches!(query.fuzziness, Some(n) if n != 0); let first_index = indices.first().ok_or(DataFusionError::Execution(format!( "FTS index for column {} has no segments", column )))?; - let mut tokenizer = match is_fuzzy { - false => first_index.tokenizer(), - true => { - let tokenizer = TextAnalyzer::from(SimpleTokenizer::default()); - match first_index.tokenizer().doc_type() { - DocType::Text => { - Box::new(TextTokenizer::new(tokenizer)) as Box - } - DocType::Json => { - Box::new(JsonTokenizer::new(tokenizer)) as Box - } - } - } - }; + let mut tokenizer = tokenizer_for_match_query(first_index, query.fuzziness); let tokens = collect_query_tokens(&query.terms, &mut tokenizer); - let base_scorer = match preset_base_scorer { - Some(scorer) => scorer, - None => Arc::new( - build_global_bm25_scorer(&indices, &tokens, ¶ms) + record_tokenized_query(&tokenized_query, &tokens); + let base_scorer = match (preset_base_scorer, shared_scorer) { + (Some(scorer), _) => scorer, + (None, Some(shared_scorer)) => shared_scorer.wait().await?, + (None, None) => { + let scorer_start = std::time::Instant::now(); + let scorer = Arc::new( + build_global_bm25_scorer( + &indices, + &tokens, + ¶ms, + Some(metrics.as_ref()), + ) .boxed() .await?, - ), + ); + metrics.record_scorer_build(scorer_start.elapsed()); + scorer + } }; pre_filter.wait_for_ready().await?; let tokens = Arc::new(tokens); let params = Arc::new(params); - let (doc_ids, mut scores) = search_segments( + let mut documents = search_segments( &indices, tokens, params, @@ -545,18 +1623,12 @@ impl ExecutionPlan for MatchQueryExec { base_scorer, ) .await?; - scores.iter_mut().for_each(|s| { - *s *= query.boost; + documents.iter_mut().for_each(|document| { + document.score.0 *= query.boost; }); - metrics.baseline_metrics.record_output(doc_ids.len()); + metrics.baseline_metrics.record_output(documents.len()); - let batch = RecordBatch::try_new( - FTS_SCHEMA.clone(), - vec![ - Arc::new(UInt64Array::from(doc_ids)), - Arc::new(Float32Array::from(scores)), - ], - )?; + let batch = scored_documents_batch(schema, documents)?; Ok::<_, DataFusionError>(batch) }); @@ -579,22 +1651,66 @@ impl ExecutionPlan for MatchQueryExec { } } -/// Filters the input, removing rows that do not share tokens with the query +/// Filters the input according to a match query's token operator. #[derive(Debug)] pub struct FlatMatchFilterExec { dataset: Arc, input: Arc, query: MatchQuery, + tokenized_query: Arc>, params: FtsSearchParams, /// Optional pre-resolved segment list. See /// [`MatchQueryExec::new_with_segments`]. `FlatMatchFilterExec` only /// uses the first segment's tokenizer, but the full list is preserved so /// the field round-trips through `with_new_children`. preset_segments: Option>, + document_column: String, + resolved_field: Option, metrics: ExecutionPlanMetricsSet, } +struct FlatMatchFilterStreamOptions { + dataset: Arc, + query: MatchQuery, + tokenized_query: Arc>, + document_column: String, + preset_segments: Option>, + resolved_field: Option, + metrics_set: ExecutionPlanMetricsSet, +} + +fn document_matches_query( + text: &str, + tokenizer: &mut Box, + query_tokens: &Tokens, + operator: Operator, +) -> bool { + match operator { + Operator::Or => has_query_token(text, tokenizer, query_tokens), + Operator::And => { + let mut remaining_positions = (0..query_tokens.len()) + .map(|index| query_tokens.position(index)) + .collect::>(); + if remaining_positions.is_empty() { + return false; + } + let mut stream = tokenizer.token_stream_for_doc(text); + while let Some(token) = stream.next() { + for index in 0..query_tokens.len() { + if token.text == query_tokens.get_token(index) { + remaining_positions.remove(&query_tokens.position(index)); + } + } + if remaining_positions.is_empty() { + return true; + } + } + false + } + } +} + impl DisplayAs for FlatMatchFilterExec { fn fmt_as(&self, t: DisplayFormatType, f: &mut std::fmt::Formatter) -> std::fmt::Result { match t { @@ -604,7 +1720,8 @@ impl DisplayAs for FlatMatchFilterExec { "FlatMatchFilter: column={}, query={}", self.query.column.as_deref().unwrap_or_default(), self.query.terms - ) + )?; + fmt_tokenized_query(&self.tokenized_query, ", ", f) } DisplayFormatType::TreeRender => { write!( @@ -612,7 +1729,8 @@ impl DisplayAs for FlatMatchFilterExec { "FlatMatchFilter\ncolumn={}\nquery={}", self.query.column.as_deref().unwrap_or_default(), self.query.terms - ) + )?; + fmt_tokenized_query(&self.tokenized_query, "\n", f) } } } @@ -622,9 +1740,10 @@ impl FlatMatchFilterExec { async fn load_tokenizer( dataset: &Dataset, column: &str, + document_granularity: DocumentGranularity, metrics: &IndexMetrics, ) -> DataFusionResult> { - if let Some(segments) = load_segments(dataset, column).await? { + if let Some(segments) = load_segments(dataset, column, document_granularity).await? { let index_meta = segments.first().ok_or_else(|| { DataFusionError::Execution(format!( "FTS index for column {} has no segments", @@ -657,13 +1776,47 @@ impl FlatMatchFilterExec { dataset: Arc, query: MatchQuery, params: FtsSearchParams, + ) -> Self { + let document_column = query.column.clone().unwrap_or_default(); + Self::new_with_document_column(input, dataset, query, params, document_column) + } + + pub fn new_with_document_column( + input: Arc, + dataset: Arc, + query: MatchQuery, + params: FtsSearchParams, + document_column: String, + ) -> Self { + Self { + dataset, + input, + query, + tokenized_query: Arc::new(OnceLock::new()), + params, + preset_segments: None, + document_column, + resolved_field: None, + metrics: ExecutionPlanMetricsSet::new(), + } + } + + pub(crate) fn new_with_resolved_field( + input: Arc, + dataset: Arc, + query: MatchQuery, + params: FtsSearchParams, + resolved_field: ResolvedFtsField, ) -> Self { Self { dataset, input, query, + tokenized_query: Arc::new(OnceLock::new()), params, preset_segments: None, + document_column: resolved_field.root_column.clone(), + resolved_field: Some(resolved_field), metrics: ExecutionPlanMetricsSet::new(), } } @@ -678,12 +1831,16 @@ impl FlatMatchFilterExec { params: FtsSearchParams, segments: Vec, ) -> Self { + let document_column = query.column.clone().unwrap_or_default(); Self { dataset, input, query, + tokenized_query: Arc::new(OnceLock::new()), params, preset_segments: Some(segments), + document_column, + resolved_field: None, metrics: ExecutionPlanMetricsSet::new(), } } @@ -708,12 +1865,20 @@ impl FlatMatchFilterExec { text_col: &dyn Array, tokenizer: &mut Box, query_tokens: &Tokens, + operator: Operator, ) -> BooleanArray { let text_col = text_col.as_string::(); let mut predicate = BooleanBuilder::with_capacity(text_col.len()); for idx in 0..text_col.len() { - let value = text_col.value(idx); - predicate.append_value(has_query_token(value, tokenizer, query_tokens)); + predicate.append_value( + !text_col.is_null(idx) + && document_matches_query( + text_col.value(idx), + tokenizer, + query_tokens, + operator, + ), + ); } predicate.finish() } @@ -722,11 +1887,17 @@ impl FlatMatchFilterExec { input: SendableRecordBatchStream, partition: usize, schema: SchemaRef, - dataset: Arc, - query: MatchQuery, - preset_segments: Option>, - metrics_set: ExecutionPlanMetricsSet, + options: FlatMatchFilterStreamOptions, ) -> DataFusionResult { + let FlatMatchFilterStreamOptions { + dataset, + query, + tokenized_query, + document_column, + preset_segments, + resolved_field, + metrics_set, + } = options; let metrics = Arc::new(FtsIndexMetrics::new(&metrics_set, partition)); let column = query .column @@ -735,6 +1906,21 @@ impl FlatMatchFilterExec { "column not set for MatchQuery {}", query.terms )))?; + if query.fuzziness != Some(0) { + return Err(DataFusionError::NotImplemented(format!( + "Fuzzy MatchQuery is not supported when FTS is used as a post-filter: column={}, fuzziness={:?}", + column, query.fuzziness + ))); + } + let document_granularity = resolved_field + .as_ref() + .map(|resolved| resolved.document_granularity) + .or(query.document_granularity) + .ok_or_else(|| { + DataFusionError::Execution( + "MatchQuery document granularity was not resolved".to_string(), + ) + })?; let mut tokenizer = match preset_segments { Some(segments) => { Self::load_tokenizer_from_preset_segments( @@ -745,33 +1931,72 @@ impl FlatMatchFilterExec { ) .await? } - None => Self::load_tokenizer(&dataset, &column, &metrics.index_metrics).await?, + None => { + Self::load_tokenizer( + &dataset, + &column, + document_granularity, + &metrics.index_metrics, + ) + .await? + } }; let query_tokens = Arc::new(collect_query_tokens(&query.terms, &mut tokenizer)); + record_tokenized_query(&tokenized_query, &query_tokens); let baseline = BaselineMetrics::new(&metrics_set, partition); let elapsed_compute = baseline.elapsed_compute().clone(); let stream = input.then(move |batch_result| { - let column = column.clone(); + let column = document_column.clone(); let query_tokens = query_tokens.clone(); let mut tokenizer = tokenizer.box_clone(); let elapsed_compute = elapsed_compute.clone(); + let resolved_field = resolved_field.clone(); + let query_operator = query.operator; async move { let batch = batch_result?; let _t = elapsed_compute.timer(); + if let Some(resolved_field) = resolved_field { + let documents = resolved_field + .documents_from_batch(&batch) + .map_err(DataFusionError::from)?; + let mut matches = vec![false; batch.num_rows()]; + for document in documents { + if document_matches_query( + &document.text, + &mut tokenizer, + &query_tokens, + query_operator, + ) { + matches[document.row_index] = true; + } + } + let predicate = BooleanArray::from(matches); + return Ok(arrow::compute::filter_record_batch(&batch, &predicate)?); + } let text_column = batch.column_by_name(&column).ok_or_else(|| { DataFusionError::Execution(format!("Column {} not found in batch", column,)) })?; let predicate = match text_column.data_type() { DataType::Utf8 => { - Self::find_matches::(text_column, &mut tokenizer, &query_tokens) + Self::find_matches::( + text_column, + &mut tokenizer, + &query_tokens, + query_operator, + ) } DataType::LargeUtf8 => { - Self::find_matches::(text_column, &mut tokenizer, &query_tokens) + Self::find_matches::( + text_column, + &mut tokenizer, + &query_tokens, + query_operator, + ) } _ => { return Err(DataFusionError::Execution(format!( - "Column {} is not a string", + "FTS document column {} is not a string; nested List inputs must be expanded before filtering", column, ))); } @@ -795,10 +2020,6 @@ impl ExecutionPlan for FlatMatchFilterExec { "FlatMatchFilterExec" } - fn as_any(&self) -> &dyn std::any::Any { - self - } - fn children(&self) -> Vec<&Arc> { vec![&self.input] } @@ -820,8 +2041,11 @@ impl ExecutionPlan for FlatMatchFilterExec { dataset: self.dataset.clone(), input, query: self.query.clone(), + tokenized_query: self.tokenized_query.clone(), params: self.params.clone(), preset_segments: self.preset_segments.clone(), + document_column: self.document_column.clone(), + resolved_field: self.resolved_field.clone(), metrics: ExecutionPlanMetricsSet::new(), })) } @@ -838,10 +2062,15 @@ impl ExecutionPlan for FlatMatchFilterExec { input, partition, schema.clone(), - self.dataset.clone(), - self.query.clone(), - self.preset_segments.clone(), - self.metrics.clone(), + FlatMatchFilterStreamOptions { + dataset: self.dataset.clone(), + query: self.query.clone(), + tokenized_query: self.tokenized_query.clone(), + document_column: self.document_column.clone(), + preset_segments: self.preset_segments.clone(), + resolved_field: self.resolved_field.clone(), + metrics_set: self.metrics.clone(), + }, ); let stream = stream::once(stream_fut) .try_flatten() @@ -850,7 +2079,7 @@ impl ExecutionPlan for FlatMatchFilterExec { Ok(Box::pin(RecordBatchStreamAdapter::new(schema, stream))) } - fn partition_statistics(&self, partition: Option) -> DataFusionResult { + fn partition_statistics(&self, partition: Option) -> DataFusionResult> { self.input.partition_statistics(partition) } @@ -872,14 +2101,20 @@ impl ExecutionPlan for FlatMatchFilterExec { pub struct FlatMatchQueryExec { dataset: Arc, query: MatchQuery, + tokenized_query: Arc>, params: FtsSearchParams, unindexed_input: Arc, /// Optional override for the BM25 scorer normally built locally inside /// `execute()`. See [`MatchQueryExec::with_base_scorer`]. base_scorer: Option>, + /// Publishes the scorer extended with this flat branch's documents. + shared_scorer: Option>, /// Optional pre-resolved segment list. See /// [`MatchQueryExec::new_with_segments`]. preset_segments: Option>, + document_granularity: DocumentGranularity, + document_column: String, + schema: SchemaRef, properties: Arc, metrics: ExecutionPlanMetricsSet, @@ -894,7 +2129,8 @@ impl DisplayAs for FlatMatchQueryExec { "FlatMatchQuery: column={}, query={}", self.query.column.as_deref().unwrap_or_default(), self.query.terms - ) + )?; + fmt_tokenized_query(&self.tokenized_query, ", ", f) } DisplayFormatType::TreeRender => { write!( @@ -902,7 +2138,8 @@ impl DisplayAs for FlatMatchQueryExec { "FlatMatchQuery\ncolumn={}\nquery={}", self.query.column.as_deref().unwrap_or_default(), self.query.terms - ) + )?; + fmt_tokenized_query(&self.tokenized_query, "\n", f) } } } @@ -914,9 +2151,32 @@ impl FlatMatchQueryExec { query: MatchQuery, params: FtsSearchParams, unindexed_input: Arc, + ) -> Result { + let document_column = query.column.clone().unwrap_or_default(); + let document_granularity = query.document_granularity.ok_or_else(|| { + Error::invalid_input("MatchQuery document granularity must be resolved".to_string()) + })?; + Ok(Self::new_with_document_granularity( + dataset, + query, + params, + unindexed_input, + document_granularity, + document_column, + )) + } + + pub fn new_with_document_granularity( + dataset: Arc, + query: MatchQuery, + params: FtsSearchParams, + unindexed_input: Arc, + document_granularity: DocumentGranularity, + document_column: String, ) -> Self { + let schema = fts_schema(document_granularity); let properties = Arc::new(PlanProperties::new( - EquivalenceProperties::new(FTS_SCHEMA.clone()), + EquivalenceProperties::new(schema.clone()), Partitioning::RoundRobinBatch(1), EmissionType::Incremental, Boundedness::Bounded, @@ -924,10 +2184,15 @@ impl FlatMatchQueryExec { Self { dataset, query, + tokenized_query: Arc::new(OnceLock::new()), params, unindexed_input, base_scorer: None, + shared_scorer: None, preset_segments: None, + document_granularity, + document_column, + schema, properties, metrics: ExecutionPlanMetricsSet::new(), } @@ -940,9 +2205,34 @@ impl FlatMatchQueryExec { params: FtsSearchParams, unindexed_input: Arc, segments: Vec, + ) -> Result { + let document_column = query.column.clone().unwrap_or_default(); + let document_granularity = query.document_granularity.ok_or_else(|| { + Error::invalid_input("MatchQuery document granularity must be resolved".to_string()) + })?; + Ok(Self::new_with_segments_and_document_granularity( + dataset, + query, + params, + unindexed_input, + segments, + document_granularity, + document_column, + )) + } + + pub fn new_with_segments_and_document_granularity( + dataset: Arc, + query: MatchQuery, + params: FtsSearchParams, + unindexed_input: Arc, + segments: Vec, + document_granularity: DocumentGranularity, + document_column: String, ) -> Self { + let schema = fts_schema(document_granularity); let properties = Arc::new(PlanProperties::new( - EquivalenceProperties::new(FTS_SCHEMA.clone()), + EquivalenceProperties::new(schema.clone()), Partitioning::RoundRobinBatch(1), EmissionType::Incremental, Boundedness::Bounded, @@ -950,10 +2240,15 @@ impl FlatMatchQueryExec { Self { dataset, query, + tokenized_query: Arc::new(OnceLock::new()), params, unindexed_input, base_scorer: None, + shared_scorer: None, preset_segments: Some(segments), + document_granularity, + document_column, + schema, properties, metrics: ExecutionPlanMetricsSet::new(), } @@ -965,6 +2260,11 @@ impl FlatMatchQueryExec { self } + pub(crate) fn with_shared_scorer(mut self, scorer: Arc) -> Self { + self.shared_scorer = Some(scorer); + self + } + pub fn query(&self) -> &MatchQuery { &self.query } @@ -991,10 +2291,6 @@ impl ExecutionPlan for FlatMatchQueryExec { "FlatMatchQueryExec" } - fn as_any(&self) -> &dyn std::any::Any { - self - } - fn children(&self) -> Vec<&Arc> { vec![&self.unindexed_input] } @@ -1020,10 +2316,15 @@ impl ExecutionPlan for FlatMatchQueryExec { Ok(Arc::new(Self { dataset: self.dataset.clone(), query: self.query.clone(), + tokenized_query: self.tokenized_query.clone(), params: self.params.clone(), unindexed_input, base_scorer: self.base_scorer.clone(), + shared_scorer: self.shared_scorer.clone(), preset_segments: self.preset_segments.clone(), + document_granularity: self.document_granularity, + document_column: self.document_column.clone(), + schema: self.schema.clone(), properties: self.properties.clone(), metrics: ExecutionPlanMetricsSet::new(), })) @@ -1036,12 +2337,17 @@ impl ExecutionPlan for FlatMatchQueryExec { context: Arc, ) -> DataFusionResult { let query = self.query.clone(); + let tokenized_query = self.tokenized_query.clone(); let ds = self.dataset.clone(); let preset_base_scorer = self.base_scorer.clone(); + let shared_scorer_producer = self.shared_scorer.clone().map(SharedFtsScorerProducer::new); let preset_segments = self.preset_segments.clone(); let metrics = Arc::new(FtsIndexMetrics::new(&self.metrics, partition)); let metrics_clone = metrics.clone(); let target_batch_size = context.session_config().batch_size(); + let document_granularity = self.document_granularity; + let document_column = self.document_column.clone(); + let phrase_slop = self.params.phrase_slop; // CPU time accumulator passed into `flat_bm25_search_stream_with_metrics` // so it can attribute the spawn_cpu tokenize work and synchronous @@ -1054,57 +2360,92 @@ impl ExecutionPlan for FlatMatchQueryExec { "column not set for MatchQuery {}", query.terms )))?; - let unindexed_input = - document_input(self.unindexed_input.execute(partition, context)?, &column)?; + let unindexed_input = document_input( + self.unindexed_input.execute(partition, context)?, + &document_column, + )?; let stream = stream::once(async move { - let segments = match preset_segments { - Some(segments) => Some(segments), - None => load_segments(&ds, &column).await?, - }; - let (tokenizer, base_scorer) = match segments { - Some(segments) => { - let _details = load_segment_details(&ds, &column, &segments).await?; - let indices = - open_fts_segments(&ds, &column, &segments, &metrics.index_metrics).await?; - metrics.record_parts_searched( - indices.iter().map(|index| index.partition_count()).sum(), - ); - let first_index = indices.first().ok_or(DataFusionError::Execution( - format!("FTS index for column {} has no segments", column), - ))?; - let mut tokenizer = first_index.tokenizer(); - let base_scorer = match preset_base_scorer { - Some(scorer) => (*scorer).clone(), - None => { - let query_tokens = collect_query_tokens(&query.terms, &mut tokenizer); - build_global_bm25_scorer( - &indices, - &query_tokens, - &FtsSearchParams::new(), - ) - .boxed() - .await? - } - }; - (tokenizer, Some(base_scorer)) - } - None => ( - default_text_tokenizer(), - preset_base_scorer.map(|s| (*s).clone()), - ), - }; + let shared_scorer_producer = shared_scorer_producer; + let result = async { + let segments = match preset_segments { + Some(segments) => Some(segments), + None => load_segments(&ds, &column, document_granularity).await?, + }; + let (tokenizer, base_scorer) = match segments { + Some(segments) => { + let _details = load_segment_details(&ds, &column, &segments).await?; + let indices = + open_fts_segments(&ds, &column, &segments, &metrics.index_metrics) + .await?; + metrics.record_parts_searched( + indices.iter().map(|index| index.partition_count()).sum(), + ); + let first_index = indices.first().ok_or(DataFusionError::Execution( + format!("FTS index for column {} has no segments", column), + ))?; + let mut tokenizer = first_index.tokenizer(); + let query_tokens = collect_query_tokens(&query.terms, &mut tokenizer); + record_tokenized_query(&tokenized_query, &query_tokens); + let base_scorer = match preset_base_scorer { + Some(scorer) => (*scorer).clone(), + None => { + let scorer_start = std::time::Instant::now(); + let scorer = build_global_bm25_scorer( + &indices, + &query_tokens, + &FtsSearchParams::new(), + Some(metrics.as_ref()), + ) + .boxed() + .await?; + metrics.record_scorer_build(scorer_start.elapsed()); + scorer + } + }; + (tokenizer, Some(base_scorer)) + } + None => { + let mut tokenizer = default_text_tokenizer(); + let query_tokens = collect_query_tokens(&query.terms, &mut tokenizer); + record_tokenized_query(&tokenized_query, &query_tokens); + (tokenizer, preset_base_scorer.map(|s| (*s).clone())) + } + }; - flat_bm25_search_stream_with_metrics( - unindexed_input, - column, - query.terms, - tokenizer, - base_scorer, - target_batch_size, - Some(elapsed_compute), - ) - .await + flat_bm25_search_stream_with_options_and_scorer( + unindexed_input, + document_column, + query.terms, + tokenizer, + base_scorer, + FlatBm25SearchOptions { + target_batch_size, + elapsed_compute: Some(elapsed_compute), + operator: query.operator, + boost: query.boost, + document_granularity, + phrase_slop, + }, + ) + .await + } + .await; + + match result { + Ok((stream, scorer)) => { + if let Some(producer) = shared_scorer_producer { + producer.publish(scorer); + } + Ok(stream) + } + Err(error) => { + if let Some(producer) = shared_scorer_producer { + producer.publish_error(&error); + } + Err(error) + } + } }) .try_flatten() .map(move |batch| { @@ -1142,14 +2483,19 @@ impl ExecutionPlan for FlatMatchQueryExec { pub struct PhraseQueryExec { dataset: Arc, query: PhraseQuery, + tokenized_query: Arc>, params: FtsSearchParams, prefilter_source: PreFilterSource, /// Optional override for the BM25 scorer normally built locally inside /// `execute()`. See [`MatchQueryExec::with_base_scorer`]. base_scorer: Option>, - /// Optional pre-resolved segment list. See - /// [`MatchQueryExec::new_with_segments`]. - preset_segments: Option>, + /// Corpus-wide scorer published by the flat branch of a mixed search. + shared_scorer: Option>, + segment_selection: FtsSegmentSelection, + /// Rows whose indexed values were superseded by newer data overlays. + overlay_block: Option, + document_granularity: DocumentGranularity, + schema: SchemaRef, properties: Arc, metrics: ExecutionPlanMetricsSet, } @@ -1163,7 +2509,8 @@ impl DisplayAs for PhraseQueryExec { "PhraseQuery: column={}, query={}", self.query.column.as_deref().unwrap_or_default(), self.query.terms - ) + )?; + fmt_tokenized_query(&self.tokenized_query, ", ", f) } DisplayFormatType::TreeRender => { write!( @@ -1171,65 +2518,161 @@ impl DisplayAs for PhraseQueryExec { "PhraseQuery\ncolumn={}\nquery={}", self.query.column.as_deref().unwrap_or_default(), self.query.terms - ) + )?; + fmt_tokenized_query(&self.tokenized_query, "\n", f) } } } -} +} + +impl PhraseQueryExec { + pub fn new( + dataset: Arc, + query: PhraseQuery, + params: FtsSearchParams, + prefilter_source: PreFilterSource, + ) -> Result { + let document_granularity = query.document_granularity.ok_or_else(|| { + Error::invalid_input("PhraseQuery document granularity must be resolved".to_string()) + })?; + Ok(Self::new_with_document_granularity( + dataset, + query, + params, + prefilter_source, + document_granularity, + )) + } + + pub fn new_with_document_granularity( + dataset: Arc, + query: PhraseQuery, + params: FtsSearchParams, + prefilter_source: PreFilterSource, + document_granularity: DocumentGranularity, + ) -> Self { + let schema = fts_schema(document_granularity); + let properties = Arc::new(PlanProperties::new( + EquivalenceProperties::new(schema.clone()), + Partitioning::RoundRobinBatch(1), + EmissionType::Final, + Boundedness::Bounded, + )); + let params = params.with_phrase_slop(Some(query.slop)); + + Self { + dataset, + query, + tokenized_query: Arc::new(OnceLock::new()), + params, + prefilter_source, + base_scorer: None, + shared_scorer: None, + segment_selection: FtsSegmentSelection::AllCommitted, + overlay_block: None, + document_granularity, + schema, + properties, + metrics: ExecutionPlanMetricsSet::new(), + } + } + + /// See [`MatchQueryExec::new_with_segments`]. + pub fn new_with_segments( + dataset: Arc, + query: PhraseQuery, + params: FtsSearchParams, + prefilter_source: PreFilterSource, + segments: Vec, + ) -> Result { + let document_granularity = query.document_granularity.ok_or_else(|| { + Error::invalid_input("PhraseQuery document granularity must be resolved".to_string()) + })?; + Ok(Self::new_with_segments_and_document_granularity( + dataset, + query, + params, + prefilter_source, + segments, + document_granularity, + )) + } -impl PhraseQueryExec { - pub fn new( + pub fn new_with_segments_and_document_granularity( dataset: Arc, query: PhraseQuery, - mut params: FtsSearchParams, + params: FtsSearchParams, prefilter_source: PreFilterSource, + segments: Vec, + document_granularity: DocumentGranularity, ) -> Self { + let schema = fts_schema(document_granularity); let properties = Arc::new(PlanProperties::new( - EquivalenceProperties::new(FTS_SCHEMA.clone()), + EquivalenceProperties::new(schema.clone()), Partitioning::RoundRobinBatch(1), EmissionType::Final, Boundedness::Bounded, )); - params = params.with_phrase_slop(Some(query.slop)); + let params = params.with_phrase_slop(Some(query.slop)); Self { dataset, query, + tokenized_query: Arc::new(OnceLock::new()), params, prefilter_source, base_scorer: None, - preset_segments: None, + shared_scorer: None, + segment_selection: FtsSegmentSelection::ExactResolved(Arc::from(segments)), + overlay_block: None, + document_granularity, + schema, properties, metrics: ExecutionPlanMetricsSet::new(), } } - /// See [`MatchQueryExec::new_with_segments`]. - pub fn new_with_segments( + /// Construct a `PhraseQueryExec` bound to an exact ordered set of committed + /// FTS segment UUIDs. + /// + /// The UUIDs are resolved from this exec's dataset snapshot when the output + /// stream is polled. Duplicate UUIDs are removed while preserving their + /// first-occurrence order. Resolution fails if the list is empty or any UUID + /// is not committed for the query column. + pub fn new_with_segment_uuids( dataset: Arc, query: PhraseQuery, mut params: FtsSearchParams, prefilter_source: PreFilterSource, - segments: Vec, - ) -> Self { + segment_uuids: Vec, + ) -> Result { + let document_granularity = query.document_granularity.ok_or_else(|| { + Error::invalid_input("PhraseQuery document granularity must be resolved".to_string()) + })?; + let schema = fts_schema(document_granularity); let properties = Arc::new(PlanProperties::new( - EquivalenceProperties::new(FTS_SCHEMA.clone()), + EquivalenceProperties::new(schema.clone()), Partitioning::RoundRobinBatch(1), EmissionType::Final, Boundedness::Bounded, )); params = params.with_phrase_slop(Some(query.slop)); - Self { + Ok(Self { dataset, query, + tokenized_query: Arc::new(OnceLock::new()), params, prefilter_source, base_scorer: None, - preset_segments: Some(segments), + shared_scorer: None, + segment_selection: FtsSegmentSelection::exact_uuids(segment_uuids), + overlay_block: None, + document_granularity, + schema, properties, metrics: ExecutionPlanMetricsSet::new(), - } + }) } /// Override the local BM25 scorer; see [`MatchQueryExec::with_base_scorer`]. @@ -1238,6 +2681,17 @@ impl PhraseQueryExec { self } + pub(crate) fn with_shared_scorer(mut self, scorer: Arc) -> Self { + self.shared_scorer = Some(scorer); + self + } + + /// Exclude rows whose indexed text was superseded by a newer data overlay. + pub(crate) fn with_overlay_block(mut self, overlay_block: RowAddrMask) -> Self { + self.overlay_block = Some(overlay_block); + self + } + pub fn query(&self) -> &PhraseQuery { &self.query } @@ -1259,7 +2713,16 @@ impl PhraseQueryExec { } pub fn preset_segments(&self) -> Option<&[IndexMetadata]> { - self.preset_segments.as_deref() + self.segment_selection.preset_segments() + } + + /// Return the ordered segment UUIDs for an explicit selection. + /// + /// Returns `None` when this exec searches all committed segments. UUID-based + /// selections omit duplicates while preserving first-occurrence order. + /// Pre-resolved selections preserve the supplied metadata order. + pub fn explicit_segment_uuids(&self) -> Option> { + self.segment_selection.explicit_segment_uuids() } } @@ -1268,10 +2731,6 @@ impl ExecutionPlan for PhraseQueryExec { "PhraseQueryExec" } - fn as_any(&self) -> &dyn std::any::Any { - self - } - fn children(&self) -> Vec<&Arc> { match &self.prefilter_source { PreFilterSource::None => vec![], @@ -1296,10 +2755,15 @@ impl ExecutionPlan for PhraseQueryExec { 0 => Self { dataset: self.dataset.clone(), query: self.query.clone(), + tokenized_query: self.tokenized_query.clone(), params: self.params.clone(), prefilter_source: PreFilterSource::None, base_scorer: self.base_scorer.clone(), - preset_segments: self.preset_segments.clone(), + shared_scorer: self.shared_scorer.clone(), + segment_selection: self.segment_selection.clone(), + overlay_block: self.overlay_block.clone(), + document_granularity: self.document_granularity, + schema: self.schema.clone(), properties: self.properties.clone(), metrics: ExecutionPlanMetricsSet::new(), }, @@ -1321,10 +2785,15 @@ impl ExecutionPlan for PhraseQueryExec { Self { dataset: self.dataset.clone(), query: self.query.clone(), + tokenized_query: self.tokenized_query.clone(), params: self.params.clone(), prefilter_source, base_scorer: self.base_scorer.clone(), - preset_segments: self.preset_segments.clone(), + shared_scorer: self.shared_scorer.clone(), + segment_selection: self.segment_selection.clone(), + overlay_block: self.overlay_block.clone(), + document_granularity: self.document_granularity, + schema: self.schema.clone(), properties: self.properties.clone(), metrics: ExecutionPlanMetricsSet::new(), } @@ -1345,11 +2814,16 @@ impl ExecutionPlan for PhraseQueryExec { context: Arc, ) -> DataFusionResult { let query = self.query.clone(); + let tokenized_query = self.tokenized_query.clone(); let params = self.params.clone(); let ds = self.dataset.clone(); let prefilter_source = self.prefilter_source.clone(); let preset_base_scorer = self.base_scorer.clone(); - let preset_segments = self.preset_segments.clone(); + let shared_scorer = self.shared_scorer.clone(); + let segment_selection = self.segment_selection.clone(); + let overlay_block = self.overlay_block.clone(); + let document_granularity = self.document_granularity; + let schema = self.schema.clone(); let metrics = Arc::new(FtsIndexMetrics::new(&self.metrics, partition)); let stream = stream::once(async move { let _timer = metrics.baseline_metrics.elapsed_compute().timer(); @@ -1357,21 +2831,25 @@ impl ExecutionPlan for PhraseQueryExec { "column not set for PhraseQuery {}", query.terms )))?; - let segments = match preset_segments { - Some(segments) => segments, - None => load_segments(&ds, &column) - .await? - .ok_or(DataFusionError::Execution(format!( - "No Inverted index found for column {}", - column, - )))?, - }; - let _details = load_segment_details(&ds, &column, &segments).await?; + let segments = segment_selection + .resolve( + &ds, + &column, + document_granularity, + &metrics.segment_bind_duration, + ) + .await?; let indices = open_fts_segments(&ds, &column, &segments, &metrics.index_metrics).await?; - let mut pre_filter = - build_prefilter(context.clone(), partition, &prefilter_source, ds, &segments)?; + let mut pre_filter = build_prefilter( + context.clone(), + partition, + &prefilter_source, + ds, + &segments, + overlay_block, + )?; let deleted_fragments = indices .iter() @@ -1393,19 +2871,31 @@ impl ExecutionPlan for PhraseQueryExec { )))?; let mut tokenizer = first_index.tokenizer(); let tokens = collect_query_tokens(&query.terms, &mut tokenizer); - let base_scorer = match preset_base_scorer { - Some(scorer) => scorer, - None => Arc::new( - build_global_bm25_scorer(&indices, &tokens, ¶ms) + record_tokenized_query(&tokenized_query, &tokens); + let base_scorer = match (preset_base_scorer, shared_scorer) { + (Some(scorer), _) => scorer, + (None, Some(shared_scorer)) => shared_scorer.wait().await?, + (None, None) => { + let scorer_start = std::time::Instant::now(); + let scorer = Arc::new( + build_global_bm25_scorer( + &indices, + &tokens, + ¶ms, + Some(metrics.as_ref()), + ) .boxed() .await?, - ), + ); + metrics.record_scorer_build(scorer_start.elapsed()); + scorer + } }; pre_filter.wait_for_ready().await?; let tokens = Arc::new(tokens); let params = Arc::new(params); - let (doc_ids, scores) = search_segments( + let documents = search_segments( &indices, tokens, params, @@ -1415,14 +2905,8 @@ impl ExecutionPlan for PhraseQueryExec { base_scorer, ) .await?; - metrics.baseline_metrics.record_output(doc_ids.len()); - let batch = RecordBatch::try_new( - FTS_SCHEMA.clone(), - vec![ - Arc::new(UInt64Array::from(doc_ids)), - Arc::new(Float32Array::from(scores)), - ], - )?; + metrics.baseline_metrics.record_output(documents.len()); + let batch = scored_documents_batch(schema, documents)?; Ok::<_, DataFusionError>(batch) }); Ok(Box::pin(RecordBatchStreamAdapter::new( @@ -1450,6 +2934,7 @@ pub struct BoostQueryExec { params: FtsSearchParams, positive: Arc, negative: Arc, + schema: SchemaRef, properties: Arc, metrics: ExecutionPlanMetricsSet, @@ -1483,8 +2968,9 @@ impl BoostQueryExec { positive: Arc, negative: Arc, ) -> Self { + let schema = positive.schema(); let properties = Arc::new(PlanProperties::new( - EquivalenceProperties::new(FTS_SCHEMA.clone()), + EquivalenceProperties::new(schema.clone()), Partitioning::RoundRobinBatch(1), EmissionType::Final, Boundedness::Bounded, @@ -1494,6 +2980,7 @@ impl BoostQueryExec { params, positive, negative, + schema, properties, metrics: ExecutionPlanMetricsSet::new(), } @@ -1521,10 +3008,6 @@ impl ExecutionPlan for BoostQueryExec { "BoostQueryExec" } - fn as_any(&self) -> &dyn std::any::Any { - self - } - fn children(&self) -> Vec<&Arc> { vec![&self.positive, &self.negative] } @@ -1555,6 +3038,7 @@ impl ExecutionPlan for BoostQueryExec { params: self.params.clone(), positive, negative, + schema: self.schema.clone(), properties: self.properties.clone(), metrics: ExecutionPlanMetricsSet::new(), })) @@ -1570,6 +3054,7 @@ impl ExecutionPlan for BoostQueryExec { let params = self.params.clone(); let positive = self.positive.execute(partition, context.clone())?; let negative = self.negative.execute(partition, context)?; + let schema = self.schema.clone(); let metrics = Arc::new(FtsIndexMetrics::new(&self.metrics, partition)); let stream = stream::once(async move { let positive = positive.try_collect::>().await?; @@ -1578,38 +3063,26 @@ impl ExecutionPlan for BoostQueryExec { let _timer = metrics.baseline_metrics.elapsed_compute().timer(); let mut res = HashMap::new(); for batch in positive { - let doc_ids = batch[ROW_ID].as_primitive::().values(); - let scores = batch[SCORE_COL].as_primitive::().values(); - - for (doc_id, score) in std::iter::zip(doc_ids, scores) { - res.insert(*doc_id, *score); + for (key, score) in batch_scored_document_keys(&batch)? { + res.insert(key, score); } } for batch in negative { - let doc_ids = batch[ROW_ID].as_primitive::().values(); - let scores = batch[SCORE_COL].as_primitive::().values(); - - for (doc_id, neg_score) in std::iter::zip(doc_ids, scores) { - if let Some(score) = res.get_mut(doc_id) { + for (key, neg_score) in batch_scored_document_keys(&batch)? { + if let Some(score) = res.get_mut(&key) { *score -= query.negative_boost * neg_score; } } } - let (doc_ids, scores): (Vec<_>, Vec<_>) = res + let documents = res .into_iter() - .sorted_unstable_by(|(_, a), (_, b)| b.total_cmp(a)) + .sorted_unstable_by(compare_scored_documents) .take(params.limit.unwrap_or(usize::MAX)) - .unzip(); - metrics.baseline_metrics.record_output(doc_ids.len()); + .collect::>(); + metrics.baseline_metrics.record_output(documents.len()); - let batch = RecordBatch::try_new( - FTS_SCHEMA.clone(), - vec![ - Arc::new(UInt64Array::from(doc_ids)), - Arc::new(Float32Array::from(scores)), - ], - )?; + let batch = document_key_scores_batch(schema, documents)?; Ok::<_, DataFusionError>(batch) }); Ok(Box::pin(RecordBatchStreamAdapter::new( @@ -1655,13 +3128,21 @@ pub enum BoolSlot { /// lookups). Returns `Result>>` so the /// `Must` slot's `None` case is naturally expressible. pub fn build_boolean_query_children( + slot: BoolSlot, + children: Vec>, +) -> Result>> { + build_boolean_query_children_with_schema(slot, children, FTS_SCHEMA.clone()) +} + +pub fn build_boolean_query_children_with_schema( slot: BoolSlot, mut children: Vec>, + schema: SchemaRef, ) -> Result>> { match slot { BoolSlot::Should | BoolSlot::MustNot => { if children.is_empty() { - Ok(Some(Arc::new(EmptyExec::new(FTS_SCHEMA.clone())))) + Ok(Some(Arc::new(EmptyExec::new(schema)))) } else if children.len() == 1 { Ok(Some(children.pop().unwrap())) } else { @@ -1676,13 +3157,20 @@ pub fn build_boolean_query_children( let mut joined: Option> = None; for plan in children { if let Some(left) = joined { + let mut on: Vec<(Arc, Arc)> = vec![( + Arc::new(Column::new_with_schema(ROW_ID, &schema)?), + Arc::new(Column::new_with_schema(ROW_ID, &schema)?), + )]; + if schema.field_with_name(DOC_INDEX_COL).is_ok() { + on.push(( + Arc::new(Column::new_with_schema(DOC_INDEX_COL, &schema)?), + Arc::new(Column::new_with_schema(DOC_INDEX_COL, &schema)?), + )); + } joined = Some(Arc::new(HashJoinExec::try_new( left, plan, - vec![( - Arc::new(Column::new_with_schema(ROW_ID, &FTS_SCHEMA)?), - Arc::new(Column::new_with_schema(ROW_ID, &FTS_SCHEMA)?), - )], + on, None, &datafusion_expr::JoinType::Inner, None, @@ -1706,6 +3194,7 @@ pub struct BooleanQueryExec { should: Arc, must: Option>, must_not: Arc, + schema: SchemaRef, properties: Arc, metrics: ExecutionPlanMetricsSet, @@ -1746,8 +3235,9 @@ impl BooleanQueryExec { must: Option>, must_not: Arc, ) -> Self { + let schema = should.schema(); let properties = Arc::new(PlanProperties::new( - EquivalenceProperties::new(FTS_SCHEMA.clone()), + EquivalenceProperties::new(schema.clone()), Partitioning::RoundRobinBatch(1), EmissionType::Final, Boundedness::Bounded, @@ -1758,6 +3248,7 @@ impl BooleanQueryExec { must, should, must_not, + schema, properties, metrics: ExecutionPlanMetricsSet::new(), } @@ -1789,10 +3280,6 @@ impl ExecutionPlan for BooleanQueryExec { "BooleanQueryExec" } - fn as_any(&self) -> &dyn std::any::Any { - self - } - fn children(&self) -> Vec<&Arc> { match &self.must { Some(must) => vec![&self.should, &self.must_not, must], @@ -1822,6 +3309,7 @@ impl ExecutionPlan for BooleanQueryExec { should, must: None, must_not: self.must_not.clone(), + schema: self.schema.clone(), properties: self.properties.clone(), metrics: ExecutionPlanMetricsSet::new(), })) @@ -1835,6 +3323,7 @@ impl ExecutionPlan for BooleanQueryExec { should, must: None, must_not, + schema: self.schema.clone(), properties: self.properties.clone(), metrics: ExecutionPlanMetricsSet::new(), })) @@ -1849,6 +3338,7 @@ impl ExecutionPlan for BooleanQueryExec { should, must: Some(must), must_not, + schema: self.schema.clone(), properties: self.properties.clone(), metrics: ExecutionPlanMetricsSet::new(), })) @@ -1877,6 +3367,7 @@ impl ExecutionPlan for BooleanQueryExec { let mut should = self.should.execute(partition, context.clone())?; let mut must_not = self.must_not.execute(partition, context)?; let metrics = Arc::new(FtsIndexMetrics::new(&self.metrics, partition)); + let schema = self.schema.clone(); let stream = stream::once(async move { let elapsed_time = metrics.baseline_metrics.elapsed_compute(); @@ -1886,25 +3377,17 @@ impl ExecutionPlan for BooleanQueryExec { if let Some(mut must) = must { while let Some(batch) = must.try_next().await? { let _timer = elapsed_time.timer(); - let row_ids = batch[ROW_ID].as_primitive::().values(); - let scores = batch[SCORE_COL].as_primitive::().values(); - res.extend(std::iter::zip( - row_ids.iter().copied(), - scores.iter().copied(), - )); + res.extend(batch_scored_document_keys_sum_scores(&batch)?); } } // add the scores from the should clause while let Some(batch) = should.try_next().await? { let _timer = elapsed_time.timer(); - let row_ids = batch[ROW_ID].as_primitive::().values(); - let scores = batch[SCORE_COL].as_primitive::().values(); - - for (row_id, score) in std::iter::zip(row_ids, scores) { - let entry = res.entry(*row_id).and_modify(|e| *e += score); + for (key, score) in batch_scored_document_keys(&batch)? { + let entry = res.entry(key).and_modify(|value| *value += score); if !has_must { - entry.or_insert(*score); + entry.or_insert(score); } } } @@ -1912,9 +3395,8 @@ impl ExecutionPlan for BooleanQueryExec { // remove the results from the must_not clause while let Some(batch) = must_not.try_next().await? { let _timer = elapsed_time.timer(); - let row_ids = batch[ROW_ID].as_primitive::().values(); - for row_id in row_ids { - res.remove(row_id); + for key in batch_document_keys(&batch)? { + res.remove(&key); } } @@ -1936,19 +3418,13 @@ impl ExecutionPlan for BooleanQueryExec { // sort the results and take the top k let _timer = elapsed_time.timer(); - let (row_ids, scores): (Vec<_>, Vec<_>) = res + let documents = res .into_iter() - .sorted_unstable_by(|(_, a), (_, b)| b.total_cmp(a)) + .sorted_unstable_by(compare_scored_documents) .take(params.limit.unwrap_or(usize::MAX)) - .unzip(); - metrics.baseline_metrics.record_output(row_ids.len()); - let batch = RecordBatch::try_new( - FTS_SCHEMA.clone(), - vec![ - Arc::new(UInt64Array::from(row_ids)), - Arc::new(Float32Array::from(scores)), - ], - )?; + .collect::>(); + metrics.baseline_metrics.record_output(documents.len()); + let batch = document_key_scores_batch(schema, documents)?; Ok::<_, DataFusionError>(batch) }); Ok(Box::pin(RecordBatchStreamAdapter::new( @@ -1976,10 +3452,11 @@ mod tests { UInt64Array, }; use arrow_schema::DataType; + use datafusion::error::{DataFusionError, Result as DataFusionResult}; use datafusion::physical_plan::metrics::ExecutionPlanMetricsSet; use datafusion::{execution::TaskContext, physical_plan::ExecutionPlan}; use futures::TryStreamExt; - use lance_core::ROW_ID; + use lance_core::{ROW_ID, utils::address::RowAddress}; use lance_datafusion::datagen::DatafusionDatagenExt; use lance_datafusion::exec::{ExecutionStatsCallback, ExecutionSummaryCounts}; use lance_datafusion::utils::PARTITIONS_SEARCHED_METRIC; @@ -1990,11 +3467,13 @@ mod tests { PhraseQuery, collect_query_tokens, has_query_token, }; use lance_index::scalar::inverted::{ - FTS_SCHEMA, InvertedIndex, Language, SCORE_COL, build_global_bm25_scorer, + DocumentGranularity, FTS_SCHEMA, InvertedIndex, Language, SCORE_COL, + build_global_bm25_scorer, }; use lance_index::scalar::{FullTextSearchQuery, InvertedIndexParams}; use lance_index::{IndexCriteria, IndexType}; use lance_table::format::IndexMetadata; + use uuid::Uuid; use crate::{ Dataset, @@ -2006,8 +3485,9 @@ mod tests { }; use super::{ - BoolSlot, BoostQueryExec, FlatMatchFilterExec, FlatMatchQueryExec, MatchQueryExec, - PhraseQueryExec, build_boolean_query_children, open_fts_segments, + BoolSlot, BoostQueryExec, CompoundQueryExec, FTS_SEGMENT_BIND_DURATION_METRIC, + FlatMatchFilterExec, FlatMatchQueryExec, MatchQueryExec, PhraseQueryExec, + build_boolean_query_children, default_text_tokenizer, open_fts_segments, }; use crate::io::exec::utils::IndexMetrics; use datafusion::physical_plan::empty::EmptyExec; @@ -2033,6 +3513,223 @@ mod tests { } } + async fn create_segment_selection_fixture() -> (Arc, Vec, Vec) { + let mut dataset = lance_datagen::gen_batch() + .col( + "text", + lance_datagen::array::cycle_utf8_literals(&["quick brown fox"]), + ) + .col( + "other", + lance_datagen::array::cycle_utf8_literals(&["not indexed"]), + ) + .into_ram_dataset(FragmentCount::from(3), FragmentRowCount::from(2)) + .await + .unwrap(); + let fragment_ids = dataset + .get_fragments() + .iter() + .map(|fragment| fragment.id() as u32) + .collect::>(); + assert_eq!(fragment_ids.len(), 3); + + let params = InvertedIndexParams::default().with_position(true); + let mut segments = Vec::with_capacity(fragment_ids.len()); + for fragment_id in &fragment_ids { + let mut builder = dataset + .create_index_builder(&["text"], IndexType::Inverted, ¶ms) + .name("segment_selection_fts".to_string()) + .fragments(vec![*fragment_id]); + segments.push(builder.execute_uncommitted().await.unwrap()); + } + dataset + .commit_existing_index_segments("segment_selection_fts", "text", segments.clone()) + .await + .unwrap(); + + let committed = crate::index::scalar::inverted::load_segments( + &dataset, + "text", + DocumentGranularity::Row, + ) + .await + .unwrap() + .unwrap(); + assert_eq!(committed.len(), fragment_ids.len()); + (Arc::new(dataset), committed, fragment_ids) + } + + fn tokenized_query_index_params() -> InvertedIndexParams { + InvertedIndexParams::new("simple".to_string(), Language::English) + .with_position(true) + .lower_case(true) + .stem(false) + .remove_stop_words(true) + .ascii_folding(false) + } + + async fn create_tokenized_query_fixture(with_unindexed_append: bool) -> Dataset { + let mut dataset = lance_datagen::gen_batch() + .col( + "text", + lance_datagen::array::cycle_utf8_literals(&["first and second"]), + ) + .into_ram_dataset(FragmentCount::from(1), FragmentRowCount::from(2)) + .await + .unwrap(); + dataset + .create_index( + &["text"], + IndexType::Inverted, + None, + &tokenized_query_index_params(), + true, + ) + .await + .unwrap(); + + if with_unindexed_append { + let appended = lance_datagen::gen_batch() + .col( + "text", + lance_datagen::array::cycle_utf8_literals(&["first and second"]), + ) + .into_reader_rows(RowCount::from(2), BatchCount::from(1)); + dataset.append(appended, None).await.unwrap(); + } + dataset + } + + fn find_plan_line<'a>(analysis: &'a str, node: &str) -> &'a str { + analysis + .lines() + .find(|line| line.trim_start().starts_with(node)) + .unwrap_or_else(|| panic!("{node} missing from plan:\n{analysis}")) + } + + fn segment_uuid_for_fragment(segments: &[IndexMetadata], fragment_id: u32) -> Uuid { + segments + .iter() + .find(|segment| { + segment + .fragment_bitmap + .as_ref() + .is_some_and(|fragments| fragments.contains(fragment_id)) + }) + .map(|segment| segment.uuid) + .unwrap() + } + + fn expected_row_ids(fragment_ids: &[u32]) -> Vec { + let mut row_ids = fragment_ids + .iter() + .flat_map(|fragment_id| { + (0..2).map(|offset| u64::from(RowAddress::new_from_parts(*fragment_id, offset))) + }) + .collect::>(); + row_ids.sort_unstable(); + row_ids + } + + async fn execute_results(plan: &dyn ExecutionPlan) -> DataFusionResult> { + let batches: Vec = plan + .execute(0, Arc::new(TaskContext::default()))? + .try_collect() + .await?; + let mut results = Vec::new(); + for batch in batches { + let row_ids = batch[ROW_ID] + .as_any() + .downcast_ref::() + .unwrap(); + let scores = batch[SCORE_COL] + .as_any() + .downcast_ref::() + .unwrap(); + results.extend( + row_ids + .values() + .iter() + .copied() + .zip(scores.values().iter().copied()), + ); + } + results.sort_by_key(|(row_id, _)| *row_id); + Ok(results) + } + + async fn execute_row_ids(plan: &dyn ExecutionPlan) -> DataFusionResult> { + Ok(execute_results(plan) + .await? + .into_iter() + .map(|(row_id, _)| row_id) + .collect()) + } + + fn metric_value(plan: &dyn ExecutionPlan, name: &str) -> usize { + plan.metrics() + .unwrap() + .iter() + .find(|metric| metric.value().name() == name) + .unwrap() + .value() + .as_usize() + } + + fn assert_execution_error(error: DataFusionError, expected_message: &str) { + assert!( + matches!(&error, DataFusionError::Execution(_)), + "expected execution error, got {error:?}" + ); + assert!( + error.to_string().contains(expected_message), + "expected error containing {expected_message:?}, got {error}" + ); + } + + #[test] + fn document_match_filter_respects_document_boundary() { + let mut tokenizer = default_text_tokenizer(); + let query_tokens = collect_query_tokens("alpha", &mut tokenizer); + assert!(super::document_matches_query( + "alpha beta", + &mut tokenizer, + &query_tokens, + Operator::Or, + )); + + let mut tokenizer = default_text_tokenizer(); + let query_tokens = collect_query_tokens("alpha beta", &mut tokenizer); + assert!(!super::document_matches_query( + "alpha", + &mut tokenizer, + &query_tokens, + Operator::And, + )); + assert!(super::document_matches_query( + "alpha beta", + &mut tokenizer, + &query_tokens, + Operator::And, + )); + } + + #[tokio::test] + async fn shared_fts_scorer_reports_cancelled_producer() { + let scorer = Arc::new(super::SharedFtsScorer::new()); + let producer = super::SharedFtsScorerProducer::new(scorer.clone()); + drop(producer); + + let error = tokio::time::timeout(std::time::Duration::from_secs(1), scorer.wait()) + .await + .expect("cancelled producer must wake scorer waiters") + .unwrap_err(); + assert!( + error.to_string().contains("producer was cancelled"), + "{error}" + ); + } + #[test] fn execute_without_context() { // These tests ensure we can create nodes and call execute without a tokio Runtime @@ -2041,10 +3738,13 @@ mod tests { let fixture = NoContextTestFixture::new(); let match_query = MatchQueryExec::new( Arc::new(fixture.dataset.clone()), - MatchQuery::new("blah".to_string()).with_column(Some("text".to_string())), + MatchQuery::new("blah".to_string()) + .with_column(Some("text".to_string())) + .with_document_granularity(DocumentGranularity::Row), FtsSearchParams::default(), PreFilterSource::None, - ); + ) + .unwrap(); match_query .execute(0, Arc::new(TaskContext::default())) .unwrap(); @@ -2060,10 +3760,13 @@ mod tests { let flat_match_query = FlatMatchQueryExec::new( Arc::new(fixture.dataset.clone()), - MatchQuery::new("blah".to_string()).with_column(Some("text".to_string())), + MatchQuery::new("blah".to_string()) + .with_column(Some("text".to_string())) + .with_document_granularity(DocumentGranularity::Row), FtsSearchParams::default(), flat_input, - ); + ) + .unwrap(); flat_match_query .execute(0, Arc::new(TaskContext::default())) .unwrap(); @@ -2072,10 +3775,12 @@ mod tests { let phrase_query = PhraseQueryExec::new( Arc::new(fixture.dataset.clone()), - PhraseQuery::new("blah".to_string()), + PhraseQuery::new("blah".to_string()) + .with_document_granularity(DocumentGranularity::Row), FtsSearchParams::new().with_phrase_slop(Some(0)), PreFilterSource::None, - ); + ) + .unwrap(); phrase_query .execute(0, Arc::new(TaskContext::default())) .unwrap(); @@ -2084,17 +3789,23 @@ mod tests { let boost_input_one = MatchQueryExec::new( Arc::new(fixture.dataset.clone()), - MatchQuery::new("blah".to_string()).with_column(Some("text".to_string())), + MatchQuery::new("blah".to_string()) + .with_column(Some("text".to_string())) + .with_document_granularity(DocumentGranularity::Row), FtsSearchParams::default(), PreFilterSource::None, - ); + ) + .unwrap(); let boost_input_two = MatchQueryExec::new( Arc::new(fixture.dataset), - MatchQuery::new("blah".to_string()).with_column(Some("text".to_string())), + MatchQuery::new("blah".to_string()) + .with_column(Some("text".to_string())) + .with_document_granularity(DocumentGranularity::Row), FtsSearchParams::default(), PreFilterSource::None, - ); + ) + .unwrap(); let boost_query = BoostQueryExec::new( BoostQuery::new( @@ -2129,8 +3840,12 @@ mod tests { let text_col = LargeStringArray::from(vec!["hello world", "no match here", "say hello there"]); - let result = - FlatMatchFilterExec::find_matches::(&text_col, &mut tokenizer, &query_tokens); + let result = FlatMatchFilterExec::find_matches::( + &text_col, + &mut tokenizer, + &query_tokens, + Operator::Or, + ); assert_eq!(result.len(), 3); assert!(result.value(0), "expected match in 'hello world'"); @@ -2182,14 +3897,24 @@ mod tests { .unwrap(); let metrics = IndexMetrics::new(&ExecutionPlanMetricsSet::new(), 0); - let mut tokenizer = FlatMatchFilterExec::load_tokenizer(&dataset, "text", &metrics) - .await - .unwrap(); + let mut tokenizer = FlatMatchFilterExec::load_tokenizer( + &dataset, + "text", + DocumentGranularity::Row, + &metrics, + ) + .await + .unwrap(); let query_tokens = collect_query_tokens("hello", &mut tokenizer); - let mut tokenizer = FlatMatchFilterExec::load_tokenizer(&dataset, "text", &metrics) - .await - .unwrap(); + let mut tokenizer = FlatMatchFilterExec::load_tokenizer( + &dataset, + "text", + DocumentGranularity::Row, + &metrics, + ) + .await + .unwrap(); assert!(has_query_token("hello", &mut tokenizer, &query_tokens)); assert!( !has_query_token("HELLO", &mut tokenizer, &query_tokens), @@ -2260,6 +3985,92 @@ mod tests { assert!(analysis.contains(PARTITIONS_SEARCHED_METRIC)); } + #[tokio::test] + async fn test_analyze_plan_shows_indexed_and_flat_match_tokens() { + let dataset = create_tokenized_query_fixture(true).await; + let query = MatchQuery::new("FIRST and SECOND".to_string()) + .with_column(Some("text".to_string())) + .with_operator(Operator::And); + let mut scanner = dataset.scan(); + scanner + .full_text_search(FullTextSearchQuery::new_query(query.into())) + .unwrap(); + + let explained = scanner.explain_plan(false).await.unwrap(); + assert!( + !explained.contains("tokenized_query="), + "explain_plan should not claim runtime tokenization: {explained}" + ); + + let analysis = scanner.analyze_plan().await.unwrap(); + let expected = r#"tokenized_query=[("first", 0), ("second", 2)]"#; + assert!( + find_plan_line(&analysis, "MatchQuery:").contains(expected), + "indexed MatchQuery is missing token positions:\n{analysis}" + ); + assert!( + find_plan_line(&analysis, "FlatMatchQuery:").contains(expected), + "flat MatchQuery is missing token positions:\n{analysis}" + ); + } + + #[tokio::test] + async fn test_analyze_plan_shows_indexed_and_flat_phrase_tokens() { + let dataset = create_tokenized_query_fixture(true).await; + let query = + PhraseQuery::new("FIRST and SECOND".to_string()).with_column(Some("text".to_string())); + let mut scanner = dataset.scan(); + scanner + .full_text_search(FullTextSearchQuery::new_query(query.into())) + .unwrap(); + + let analysis = scanner.analyze_plan().await.unwrap(); + let expected = r#"tokenized_query=[("first", 0), ("second", 2)]"#; + assert!( + find_plan_line(&analysis, "PhraseQuery:").contains(expected), + "indexed PhraseQuery is missing token positions:\n{analysis}" + ); + assert!( + find_plan_line(&analysis, "FlatMatchQuery:").contains(expected), + "flat phrase path is missing token positions:\n{analysis}" + ); + } + + #[tokio::test] + async fn test_analyze_plan_shows_compound_leaf_tokens() { + let dataset = create_tokenized_query_fixture(false).await; + let query = BooleanQuery::new([ + ( + Occur::Should, + MatchQuery::new("FIRST and SECOND".to_string()) + .with_column(Some("text".to_string())) + .into(), + ), + ( + Occur::Must, + PhraseQuery::new("SECOND FIRST".to_string()) + .with_column(Some("text".to_string())) + .with_slop(2) + .into(), + ), + ]); + let mut scanner = dataset.scan(); + scanner + .full_text_search(FullTextSearchQuery::new_query(query.into())) + .unwrap(); + + let analysis = scanner.analyze_plan().await.unwrap(); + let compound = find_plan_line(&analysis, "CompoundFtsScorer:"); + assert!( + compound.contains(r#"Match(column="text", tokens=[("first", 0), ("second", 2)])"#), + "compound Match leaf is missing token positions: {compound}" + ); + assert!( + compound.contains(r#"Phrase(column="text", tokens=[("second", 0), ("first", 1)])"#), + "compound Phrase leaf is missing token positions: {compound}" + ); + } + #[tokio::test] async fn test_boolean_query_parts_searched_metrics() { let mut dataset = lance_datagen::gen_batch() @@ -2318,13 +4129,398 @@ mod tests { .full_text_search(FullTextSearchQuery::new_query(query.into())) .unwrap(); let analysis = scanner.analyze_plan().await.unwrap(); - let boolean_line = analysis + let compound_line = analysis .lines() - .find(|line| line.contains("BooleanQuery")) + .find(|line| line.contains("CompoundFtsScorer")) + .unwrap(); + assert!( + compound_line.contains(&format!("{PARTITIONS_SEARCHED_METRIC}={expected_total}")), + "compound FTS scorer metrics missing partitions_searched: {compound_line}" + ); + } + + #[tokio::test] + async fn test_match_query_exec_segment_selection() { + let (dataset, segments, fragment_ids) = create_segment_selection_fixture().await; + let query = MatchQuery::new("quick".to_string()) + .with_column(Some("text".to_string())) + .with_document_granularity(DocumentGranularity::Row); + let params = FtsSearchParams::default().with_limit(Some(20)); + let committed_uuids = segments + .iter() + .map(|segment| segment.uuid) + .collect::>(); + + let all_committed = MatchQueryExec::new( + dataset.clone(), + query.clone(), + params.clone(), + PreFilterSource::None, + ) + .unwrap(); + assert!(all_committed.preset_segments().is_none()); + assert!(all_committed.explicit_segment_uuids().is_none()); + let all_results = execute_results(&all_committed).await.unwrap(); + assert_eq!( + all_results + .iter() + .map(|(row_id, _)| *row_id) + .collect::>(), + expected_row_ids(&fragment_ids) + ); + assert_eq!( + metric_value(&all_committed, FTS_SEGMENT_BIND_DURATION_METRIC), + 0 + ); + + let exact_resolved = MatchQueryExec::new_with_segments( + dataset.clone(), + query.clone(), + params.clone(), + PreFilterSource::None, + segments.clone(), + ) + .unwrap(); + assert_eq!(exact_resolved.preset_segments(), Some(segments.as_slice())); + assert_eq!( + exact_resolved.explicit_segment_uuids(), + Some(committed_uuids.clone()) + ); + assert_eq!(execute_results(&exact_resolved).await.unwrap(), all_results); + assert_eq!( + metric_value(&exact_resolved, FTS_SEGMENT_BIND_DURATION_METRIC), + 0 + ); + + let mismatched_granularity = MatchQueryExec::new_with_segments_and_document_granularity( + dataset.clone(), + query.clone(), + params.clone(), + PreFilterSource::None, + segments.clone(), + DocumentGranularity::ListElement, + ); + assert_execution_error( + execute_row_ids(&mismatched_granularity).await.unwrap_err(), + "use Row document granularity", + ); + + let selected_fragment = fragment_ids[1]; + let selected_uuid = segment_uuid_for_fragment(&segments, selected_fragment); + let unpolled = MatchQueryExec::new_with_segment_uuids( + dataset.clone(), + query.clone(), + params.clone(), + PreFilterSource::None, + vec![selected_uuid], + ) + .unwrap(); + drop( + unpolled + .execute(0, Arc::new(TaskContext::default())) + .unwrap(), + ); + assert_eq!( + metric_value(&unpolled, FTS_SEGMENT_BIND_DURATION_METRIC), + 0, + "UUID binding should not start until the output stream is polled" + ); + + let exact_uuids = MatchQueryExec::new_with_segment_uuids( + dataset.clone(), + query.clone(), + params.clone(), + PreFilterSource::None, + vec![selected_uuid], + ) + .unwrap(); + assert!(exact_uuids.preset_segments().is_none()); + assert_eq!( + exact_uuids.explicit_segment_uuids(), + Some(vec![selected_uuid]) + ); + assert_eq!( + execute_row_ids(&exact_uuids).await.unwrap(), + expected_row_ids(&[selected_fragment]) + ); + assert!( + metric_value(&exact_uuids, FTS_SEGMENT_BIND_DURATION_METRIC) > 0, + "successful UUID binding should record a duration" + ); + + let input_uuids = vec![ + segment_uuid_for_fragment(&segments, fragment_ids[2]), + segment_uuid_for_fragment(&segments, fragment_ids[0]), + segment_uuid_for_fragment(&segments, fragment_ids[2]), + ]; + let deduplicated_uuids = input_uuids[..2].to_vec(); + let ordered_plan = Arc::new( + MatchQueryExec::new_with_segment_uuids( + dataset.clone(), + query.clone(), + params.clone(), + PreFilterSource::None, + input_uuids, + ) + .unwrap(), + ) + .with_new_children(vec![]) + .unwrap(); + let rewritten = ordered_plan.downcast_ref::().unwrap(); + assert_eq!( + rewritten.explicit_segment_uuids(), + Some(deduplicated_uuids.clone()) + ); + assert_eq!( + execute_row_ids(rewritten).await.unwrap(), + expected_row_ids(&[fragment_ids[2], fragment_ids[0]]) + ); + let resolver_metrics_set = ExecutionPlanMetricsSet::new(); + let resolver_metrics = super::FtsIndexMetrics::new(&resolver_metrics_set, 0); + let resolved = rewritten + .segment_selection + .resolve( + &dataset, + "text", + DocumentGranularity::Row, + &resolver_metrics.segment_bind_duration, + ) + .await .unwrap(); + assert_eq!( + resolved + .iter() + .map(|segment| segment.uuid) + .collect::>(), + deduplicated_uuids + ); + + let empty = MatchQueryExec::new_with_segment_uuids( + dataset.clone(), + query.clone(), + params.clone(), + PreFilterSource::None, + vec![], + ) + .unwrap(); + assert_execution_error( + execute_row_ids(&empty).await.unwrap_err(), + "requires at least one segment UUID", + ); + + let missing_uuid = Uuid::new_v4(); + let missing = MatchQueryExec::new_with_segment_uuids( + dataset.clone(), + query, + params.clone(), + PreFilterSource::None, + vec![missing_uuid], + ) + .unwrap(); + assert_execution_error( + execute_row_ids(&missing).await.unwrap_err(), + &missing_uuid.to_string(), + ); + + let wrong_column = MatchQueryExec::new_with_segment_uuids( + dataset, + MatchQuery::new("quick".to_string()) + .with_column(Some("other".to_string())) + .with_document_granularity(DocumentGranularity::Row), + params, + PreFilterSource::None, + vec![selected_uuid], + ) + .unwrap(); + assert_execution_error( + execute_row_ids(&wrong_column).await.unwrap_err(), + "no Inverted index found", + ); + } + + #[tokio::test] + async fn test_phrase_query_exec_segment_selection() { + let (dataset, segments, fragment_ids) = create_segment_selection_fixture().await; + let query = PhraseQuery::new("quick brown".to_string()) + .with_column(Some("text".to_string())) + .with_document_granularity(DocumentGranularity::Row); + let params = FtsSearchParams::default().with_limit(Some(20)); + let committed_uuids = segments + .iter() + .map(|segment| segment.uuid) + .collect::>(); + + let all_committed = PhraseQueryExec::new( + dataset.clone(), + query.clone(), + params.clone(), + PreFilterSource::None, + ) + .unwrap(); + assert!(all_committed.preset_segments().is_none()); + assert!(all_committed.explicit_segment_uuids().is_none()); + let all_results = execute_results(&all_committed).await.unwrap(); + assert_eq!( + all_results + .iter() + .map(|(row_id, _)| *row_id) + .collect::>(), + expected_row_ids(&fragment_ids) + ); + assert_eq!( + metric_value(&all_committed, FTS_SEGMENT_BIND_DURATION_METRIC), + 0 + ); + + let exact_resolved = PhraseQueryExec::new_with_segments( + dataset.clone(), + query.clone(), + params.clone(), + PreFilterSource::None, + segments.clone(), + ) + .unwrap(); + assert_eq!(exact_resolved.preset_segments(), Some(segments.as_slice())); + assert_eq!( + exact_resolved.explicit_segment_uuids(), + Some(committed_uuids) + ); + assert_eq!(execute_results(&exact_resolved).await.unwrap(), all_results); + assert_eq!( + metric_value(&exact_resolved, FTS_SEGMENT_BIND_DURATION_METRIC), + 0 + ); + + let selected_fragment = fragment_ids[1]; + let selected_uuid = segment_uuid_for_fragment(&segments, selected_fragment); + let unpolled = PhraseQueryExec::new_with_segment_uuids( + dataset.clone(), + query.clone(), + params.clone(), + PreFilterSource::None, + vec![selected_uuid], + ) + .unwrap(); + drop( + unpolled + .execute(0, Arc::new(TaskContext::default())) + .unwrap(), + ); + assert_eq!( + metric_value(&unpolled, FTS_SEGMENT_BIND_DURATION_METRIC), + 0, + "UUID binding should not start until the output stream is polled" + ); + + let exact_uuids = PhraseQueryExec::new_with_segment_uuids( + dataset.clone(), + query.clone(), + params.clone(), + PreFilterSource::None, + vec![selected_uuid], + ) + .unwrap(); + assert!(exact_uuids.preset_segments().is_none()); + assert_eq!( + exact_uuids.explicit_segment_uuids(), + Some(vec![selected_uuid]) + ); + assert_eq!( + execute_row_ids(&exact_uuids).await.unwrap(), + expected_row_ids(&[selected_fragment]) + ); assert!( - boolean_line.contains(&format!("{PARTITIONS_SEARCHED_METRIC}={expected_total}")), - "BooleanQuery metrics missing partitions_searched: {boolean_line}" + metric_value(&exact_uuids, FTS_SEGMENT_BIND_DURATION_METRIC) > 0, + "successful UUID binding should record a duration" + ); + + let input_uuids = vec![ + segment_uuid_for_fragment(&segments, fragment_ids[2]), + segment_uuid_for_fragment(&segments, fragment_ids[0]), + segment_uuid_for_fragment(&segments, fragment_ids[2]), + ]; + let deduplicated_uuids = input_uuids[..2].to_vec(); + let ordered_plan = Arc::new( + PhraseQueryExec::new_with_segment_uuids( + dataset.clone(), + query.clone(), + params.clone(), + PreFilterSource::None, + input_uuids, + ) + .unwrap(), + ) + .with_new_children(vec![]) + .unwrap(); + let rewritten = ordered_plan.downcast_ref::().unwrap(); + assert_eq!( + rewritten.explicit_segment_uuids(), + Some(deduplicated_uuids.clone()) + ); + assert_eq!( + execute_row_ids(rewritten).await.unwrap(), + expected_row_ids(&[fragment_ids[2], fragment_ids[0]]) + ); + let resolver_metrics_set = ExecutionPlanMetricsSet::new(); + let resolver_metrics = super::FtsIndexMetrics::new(&resolver_metrics_set, 0); + let resolved = rewritten + .segment_selection + .resolve( + &dataset, + "text", + DocumentGranularity::Row, + &resolver_metrics.segment_bind_duration, + ) + .await + .unwrap(); + assert_eq!( + resolved + .iter() + .map(|segment| segment.uuid) + .collect::>(), + deduplicated_uuids + ); + + let empty = PhraseQueryExec::new_with_segment_uuids( + dataset.clone(), + query.clone(), + params.clone(), + PreFilterSource::None, + vec![], + ) + .unwrap(); + assert_execution_error( + execute_row_ids(&empty).await.unwrap_err(), + "requires at least one segment UUID", + ); + + let missing_uuid = Uuid::new_v4(); + let missing = PhraseQueryExec::new_with_segment_uuids( + dataset.clone(), + query, + params.clone(), + PreFilterSource::None, + vec![missing_uuid], + ) + .unwrap(); + assert_execution_error( + execute_row_ids(&missing).await.unwrap_err(), + &missing_uuid.to_string(), + ); + + let wrong_column = PhraseQueryExec::new_with_segment_uuids( + dataset, + PhraseQuery::new("quick brown".to_string()) + .with_column(Some("other".to_string())) + .with_document_granularity(DocumentGranularity::Row), + params, + PreFilterSource::None, + vec![selected_uuid], + ) + .unwrap(); + assert_execution_error( + execute_row_ids(&wrong_column).await.unwrap_err(), + "no Inverted index found", ); } @@ -2411,7 +4607,9 @@ mod tests { ); let dataset = Arc::new(ds); - let query = MatchQuery::new("lance".to_string()).with_column(Some("text".to_string())); + let query = MatchQuery::new("lance".to_string()) + .with_column(Some("text".to_string())) + .with_document_granularity(DocumentGranularity::Row); let search_params = FtsSearchParams::default().with_limit(Some(10)); // Baseline: the existing path that builds the global scorer locally. @@ -2420,7 +4618,8 @@ mod tests { query.clone(), search_params.clone(), PreFilterSource::None, - ); + ) + .unwrap(); let baseline_batches: Vec = baseline_exec .execute(0, Arc::new(TaskContext::default())) .unwrap() @@ -2435,10 +4634,14 @@ mod tests { // Override: build the global scorer manually via the public helper, then // construct the exec with the preset segments and the preset scorer. - let preset_segments = crate::index::scalar::inverted::load_segments(&dataset, "text") - .await - .unwrap() - .expect("FTS index just created"); + let preset_segments = crate::index::scalar::inverted::load_segments( + &dataset, + "text", + DocumentGranularity::Row, + ) + .await + .unwrap() + .expect("FTS index just created"); let metrics_set = ExecutionPlanMetricsSet::new(); let metrics = IndexMetrics::new(&metrics_set, 0); let indices = open_fts_segments(&dataset, "text", &preset_segments, &metrics) @@ -2452,7 +4655,7 @@ mod tests { let mut tokenizer = indices[0].tokenizer(); let tokens = collect_query_tokens(&query.terms, &mut tokenizer); let global_scorer = Arc::new( - build_global_bm25_scorer(&indices, &tokens, &search_params) + build_global_bm25_scorer(&indices, &tokens, &search_params, None) .await .unwrap(), ); @@ -2464,6 +4667,7 @@ mod tests { PreFilterSource::None, preset_segments, ) + .unwrap() .with_base_scorer(global_scorer); let override_batches: Vec = override_exec .execute(0, Arc::new(TaskContext::default())) @@ -2532,6 +4736,124 @@ mod tests { } } + #[tokio::test] + async fn test_compound_query_exec_validates_base_scorer() { + let (dataset, segments, _) = create_segment_selection_fixture().await; + let search_params = FtsSearchParams::default().with_limit(Some(10)); + let metrics_set = ExecutionPlanMetricsSet::new(); + let metrics = IndexMetrics::new(&metrics_set, 0); + let indices = open_fts_segments(&dataset, "text", &segments, &metrics) + .await + .unwrap(); + + let query: FtsQuery = BooleanQuery::new([ + ( + Occur::Should, + MatchQuery::new("quick".to_string()) + .with_column(Some("text".to_string())) + .into(), + ), + ( + Occur::Should, + MatchQuery::new("brown".to_string()) + .with_column(Some("text".to_string())) + .into(), + ), + ]) + .into(); + + let baseline = CompoundQueryExec::new_with_segments( + dataset.clone(), + query.clone(), + search_params.clone(), + PreFilterSource::None, + segments.clone(), + ); + let baseline_results = execute_results(&baseline).await.unwrap(); + + let mut tokenizer = indices[0].tokenizer(); + let complete_tokens = collect_query_tokens("quick brown", &mut tokenizer); + let complete_scorer = Arc::new( + build_global_bm25_scorer(&indices, &complete_tokens, &search_params, None) + .await + .unwrap(), + ); + let complete_override = CompoundQueryExec::new_with_segments( + dataset.clone(), + query.clone(), + search_params.clone(), + PreFilterSource::None, + segments.clone(), + ) + .with_base_scorer(complete_scorer); + assert_eq!( + execute_results(&complete_override).await.unwrap(), + baseline_results + ); + + let mut tokenizer = indices[0].tokenizer(); + let incomplete_tokens = collect_query_tokens("quick", &mut tokenizer); + let incomplete_scorer = Arc::new( + build_global_bm25_scorer(&indices, &incomplete_tokens, &search_params, None) + .await + .unwrap(), + ); + let incomplete_override = CompoundQueryExec::new_with_segments( + dataset.clone(), + query, + search_params.clone(), + PreFilterSource::None, + segments.clone(), + ) + .with_base_scorer(incomplete_scorer); + + let error = execute_results(&incomplete_override).await.unwrap_err(); + assert!( + error + .to_string() + .contains("injected BM25 scorer is missing compound FTS token 'brown'"), + "unexpected incomplete-scorer error: {error}" + ); + + let mut tokenizer = indices[0].tokenizer(); + let brown_tokens = collect_query_tokens("brown", &mut tokenizer); + let scorer_without_fuzzy_expansion = Arc::new( + build_global_bm25_scorer(&indices, &brown_tokens, &search_params, None) + .await + .unwrap(), + ); + let fuzzy_query = BooleanQuery::new([ + ( + Occur::Should, + MatchQuery::new("quik".to_string()) + .with_column(Some("text".to_string())) + .with_fuzziness(Some(1)) + .into(), + ), + ( + Occur::Should, + MatchQuery::new("brown".to_string()) + .with_column(Some("text".to_string())) + .into(), + ), + ]); + let fuzzy_override = CompoundQueryExec::new_with_segments( + dataset, + fuzzy_query.into(), + search_params, + PreFilterSource::None, + segments, + ) + .with_base_scorer(scorer_without_fuzzy_expansion); + let error = execute_results(&fuzzy_override).await.unwrap_err(); + assert!( + error + .to_string() + .contains("injected BM25 scorer is missing compound FTS token 'quick'"), + "unexpected fuzzy-scorer error: {error}" + ); + } + fn empty_fts_child() -> Arc { Arc::new(EmptyExec::new(FTS_SCHEMA.clone())) } @@ -2542,7 +4864,7 @@ mod tests { .unwrap() .expect("Should slot always returns Some"); assert!( - plan.as_any().downcast_ref::().is_some(), + plan.downcast_ref::().is_some(), "expected EmptyExec for empty Should slot, got {plan:?}" ); } @@ -2570,12 +4892,10 @@ mod tests { .unwrap() .expect("Should slot always returns Some"); let repartition = plan - .as_any() .downcast_ref::() .expect("multi-child Should should be wrapped in RepartitionExec"); let inner = repartition .input() - .as_any() .downcast_ref::() .expect("RepartitionExec should wrap a UnionExec"); assert_eq!(inner.children().len(), 2); @@ -2614,7 +4934,7 @@ mod tests { // there are N-1 joins. let mut joins = 0usize; let mut current: Arc = plan; - while let Some(join) = current.clone().as_any().downcast_ref::() { + while let Some(join) = current.clone().downcast_ref::() { joins += 1; current = join.children()[0].clone(); } @@ -2630,12 +4950,10 @@ mod tests { .unwrap() .expect("MustNot slot always returns Some"); let repartition = plan - .as_any() .downcast_ref::() .expect("multi-child MustNot should be wrapped in RepartitionExec"); let inner = repartition .input() - .as_any() .downcast_ref::() .expect("RepartitionExec should wrap a UnionExec"); assert_eq!(inner.children().len(), 2); diff --git a/rust/lance/src/io/exec/knn.rs b/rust/lance/src/io/exec/knn.rs index f6332ada94e..b2561aa8f1a 100644 --- a/rust/lance/src/io/exec/knn.rs +++ b/rust/lance/src/io/exec/knn.rs @@ -3,7 +3,6 @@ #[cfg(test)] use lance_core::utils::row_addr_remap::RowAddrRemap; -use std::any::Any; use std::cmp::Ordering as CmpOrdering; use std::collections::{BinaryHeap, HashMap, HashSet}; use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; @@ -61,6 +60,8 @@ use lance_table::format::IndexMetadata; use tokio::sync::Notify; use uuid::Uuid; +use lance_select::RowAddrMask; + use crate::dataset::Dataset; use crate::index::DatasetIndexInternalExt; use crate::index::prefilter::{DatasetPreFilter, FilterLoader}; @@ -829,10 +830,6 @@ impl ExecutionPlan for KNNVectorDistanceExec { "KNNVectorDistanceExec" } - fn as_any(&self) -> &dyn Any { - self - } - /// Flat KNN inherits the schema from input node, and add one distance column. fn schema(&self) -> arrow_schema::SchemaRef { self.output_schema.clone() @@ -948,7 +945,7 @@ impl ExecutionPlan for KNNVectorDistanceExec { Ok(Box::pin(RecordBatchStreamAdapter::new(schema, stream))) } - fn partition_statistics(&self, partition: Option) -> DataFusionResult { + fn partition_statistics(&self, partition: Option) -> DataFusionResult> { let inner_stats = self.input.partition_statistics(partition)?; let input_schema = self.input.schema(); let input_stats_by_name = inner_stats @@ -985,11 +982,11 @@ impl ExecutionPlan for KNNVectorDistanceExec { } }) .collect::>(); - Ok(Statistics { + Ok(Arc::new(Statistics { num_rows: inner_stats.num_rows, column_statistics, ..Statistics::new_unknown(self.schema().as_ref()) - }) + })) } fn metrics(&self) -> Option { @@ -1104,11 +1101,14 @@ pub static KNN_PARTITION_SCHEMA: LazyLock = LazyLock::new(|| { ])) }); +/// Create a new ANN execution node. `overlay_block`, when `Some`, excludes rows whose index +/// entries may be stale due to a newer data overlay (see [`ANNIvfSubIndexExec::with_overlay_block`]). pub fn new_knn_exec( dataset: Arc, indices: &[IndexMetadata], query: &Query, prefilter_source: PreFilterSource, + overlay_block: Option, ) -> Result> { let ivf_node = ANNIvfPartitionExec::try_new( dataset.clone(), @@ -1116,13 +1116,16 @@ pub fn new_knn_exec( query.clone(), )?; - let sub_index = ANNIvfSubIndexExec::try_new( + let mut sub_index = ANNIvfSubIndexExec::try_new( Arc::new(ivf_node), dataset, indices.to_vec(), query.clone(), prefilter_source, )?; + if let Some(overlay_block) = overlay_block { + sub_index = sub_index.with_overlay_block(overlay_block); + } Ok(Arc::new(sub_index)) } @@ -1225,10 +1228,6 @@ impl ExecutionPlan for ANNIvfPartitionExec { "ANNIVFPartitionExec" } - fn as_any(&self) -> &dyn Any { - self - } - fn schema(&self) -> SchemaRef { KNN_PARTITION_SCHEMA.clone() } @@ -1237,11 +1236,11 @@ impl ExecutionPlan for ANNIvfPartitionExec { &self.properties } - fn partition_statistics(&self, _partition: Option) -> DataFusionResult { - Ok(Statistics { + fn partition_statistics(&self, _partition: Option) -> DataFusionResult> { + Ok(Arc::new(Statistics { num_rows: Precision::Exact(self.query.minimum_nprobes), ..Statistics::new_unknown(self.schema().as_ref()) - }) + })) } fn metrics(&self) -> Option { @@ -1386,6 +1385,10 @@ pub struct ANNIvfSubIndexExec { /// Prefiltering input prefilter_source: PreFilterSource, + /// Row addresses whose index entries are stale due to a newer data overlay. Blocked from + /// index results at execution time via [`DatasetPreFilter::with_overlay_block`]. + overlay_block: Option, + /// Datafusion Plan Properties properties: Arc, @@ -1418,11 +1421,18 @@ impl ANNIvfSubIndexExec { indices, query, prefilter_source, + overlay_block: None, properties, metrics: ExecutionPlanMetricsSet::new(), }) } + /// Block stale row addresses (see the `overlay_block` field) from index results. + pub fn with_overlay_block(mut self, overlay_block: RowAddrMask) -> Self { + self.overlay_block = Some(overlay_block); + self + } + /// Returns a reference to the vector query. pub fn query(&self) -> &Query { &self.query @@ -1631,12 +1641,15 @@ impl ANNIvfSubIndexExec { .unwrap_or(partitions.len()) .min(partitions.len()); let min_nprobes = query.minimum_nprobes.min(max_nprobes); + + // Every delta must reach the barrier, even if it has no partitions left + // to search, so that siblings waiting for the initial search can proceed. + let found_so_far = state.wait_for_minimum_to_finish().await; if max_nprobes <= min_nprobes { // We've already searched all partitions, no late search needed return futures::stream::empty().boxed(); } - let found_so_far = state.wait_for_minimum_to_finish().await; if found_so_far >= query.k { // We found enough results, no need for late search return futures::stream::empty().boxed(); @@ -1843,10 +1856,6 @@ impl ExecutionPlan for ANNIvfSubIndexExec { "ANNSubIndexExec" } - fn as_any(&self) -> &dyn Any { - self - } - fn schema(&self) -> arrow_schema::SchemaRef { KNN_INDEX_SCHEMA.clone() } @@ -1893,6 +1902,7 @@ impl ExecutionPlan for ANNIvfSubIndexExec { indices: self.indices.clone(), query: self.query.clone(), prefilter_source, + overlay_block: self.overlay_block.clone(), properties: self.properties.clone(), metrics: ExecutionPlanMetricsSet::new(), } @@ -1973,11 +1983,13 @@ impl ExecutionPlan for ANNIvfSubIndexExec { PreFilterSource::None => None, }; - let pre_filter = Arc::new(DatasetPreFilter::new( - ds.clone(), - &indices, - prefilter_loader, - )); + let pre_filter = { + let mut pf = DatasetPreFilter::new(ds.clone(), &indices, prefilter_loader); + if let Some(block) = self.overlay_block.clone() { + pf = pf.with_overlay_block(block); + } + Arc::new(pf) + }; let state = Arc::new(ANNIvfEarlySearchResults::new(indices.len(), query.k)); @@ -2043,8 +2055,8 @@ impl ExecutionPlan for ANNIvfSubIndexExec { fn partition_statistics( &self, partition: Option, - ) -> DataFusionResult { - Ok(Statistics { + ) -> DataFusionResult> { + Ok(Arc::new(Statistics { num_rows: Precision::Exact( self.query.k * self.query.refine_factor.unwrap_or(1) as usize @@ -2056,7 +2068,7 @@ impl ExecutionPlan for ANNIvfSubIndexExec { .unwrap_or(&1), ), ..Statistics::new_unknown(self.schema().as_ref()) - }) + })) } fn metrics(&self) -> Option { @@ -2139,10 +2151,6 @@ impl ExecutionPlan for MultivectorScoringExec { "MultivectorScoringExec" } - fn as_any(&self) -> &dyn Any { - self - } - fn schema(&self) -> arrow_schema::SchemaRef { KNN_INDEX_SCHEMA.clone() } @@ -2290,6 +2298,8 @@ impl ExecutionPlan for MultivectorScoringExec { mod tests { use super::*; + use std::any::Any; + use crate::index::DatasetIndexExt; use arrow::compute::{concat_batches, sort_to_indices, take_record_batch}; use arrow::datatypes::Float32Type; @@ -2321,6 +2331,7 @@ mod tests { use crate::dataset::{WriteMode, WriteParams}; use crate::index::vector::VectorIndexParams; + use crate::index::vector::ivf::v2::STREAMING_SEARCH_BATCH_SIZE; use crate::io::exec::testing::TestingExec; fn base_query() -> Query { @@ -2634,7 +2645,6 @@ mod tests { _metrics: Arc, ) -> Result { let (batch_tx, batch_rx) = mpsc::channel(1); - let batch_tx_for_search = batch_tx.clone(); let prepared_partition_ids = (start_idx..end_idx) .map(|idx| partitions.value(idx) as usize) .collect::>(); @@ -2642,42 +2652,74 @@ mod tests { .lock() .unwrap() .extend(prepared_partition_ids.iter().copied()); + // Mirror the production streaming path (v2.rs): search prepared partitions + // in batches of STREAMING_SEARCH_BATCH_SIZE, one `spawn_cpu` per batch, with + // the channel send in async code so no CPU-pool thread parks (#7642). tokio::spawn(async move { - let search_result = spawn_cpu(move || -> DataFusionResult<()> { - for partition_id in prepared_partition_ids { - if control - .as_ref() - .is_some_and(|control| control.should_stop()) - { - return Ok(()); - } - let batch = self - .search_prepared_partition( - Box::new(partition_id), - &lance_index::metrics::NoOpMetricsCollector, - ) - .map_err(datafusion::error::DataFusionError::from); - match batch { - Ok(batch) => { - if let Some(control) = control.as_ref() { - control.record_batch(&batch); + for chunk in prepared_partition_ids.chunks(*STREAMING_SEARCH_BATCH_SIZE) { + if control + .as_ref() + .is_some_and(|control| control.should_stop()) + || batch_tx.is_closed() + { + return; + } + let chunk = chunk.to_vec(); + let index = self.clone(); + let control_for_search = control.clone(); + let cancel_probe = batch_tx.clone(); + let search_output = spawn_cpu(move || { + let mut outputs: Vec> = + Vec::with_capacity(chunk.len()); + let mut stopped = false; + for partition_id in chunk { + if control_for_search + .as_ref() + .is_some_and(|control| control.should_stop()) + || cancel_probe.is_closed() + { + stopped = true; + break; + } + match index + .search_prepared_partition( + Box::new(partition_id), + &lance_index::metrics::NoOpMetricsCollector, + ) + .map_err(datafusion::error::DataFusionError::from) + { + Ok(batch) => { + if let Some(control) = control_for_search.as_ref() { + control.record_batch(&batch); + } + outputs.push(Ok(batch)); } - if batch_tx_for_search.blocking_send(Ok(batch)).is_err() { - return Ok(()); + Err(err) => { + outputs.push(Err(err)); + stopped = true; + break; } } - Err(err) => { - let _ = batch_tx_for_search.blocking_send(Err(err)); - return Ok(()); - } } - } - Ok(()) - }) - .await; + Ok::<_, datafusion::error::DataFusionError>((outputs, stopped)) + }) + .await; - if let Err(err) = search_result { - let _ = batch_tx.send(Err(err)).await; + let (outputs, stopped) = match search_output { + Ok(output) => output, + Err(err) => { + let _ = batch_tx.send(Err(err)).await; + return; + } + }; + for output in outputs { + if batch_tx.send(output).await.is_err() { + return; + } + } + if stopped { + return; + } } }); @@ -2871,19 +2913,29 @@ mod tests { ); } + // All partitions fit in a single search batch, so they are searched in one + // `spawn_cpu` dispatch and therefore share one cpu thread. The partition count + // adapts to the configured batch size so the single-batch property holds under + // any valid `LANCE_IVF_STREAMING_SEARCH_BATCH_SIZE`, including 1. #[tokio::test] async fn test_sequential_initial_search_prepares_all_then_searches_on_one_cpu_thread() { + let num_partitions = 3.min(*STREAMING_SEARCH_BATCH_SIZE); + let row_ids = (0..num_partitions).map(|i| 10 + i as u64).collect(); let (index, prepared_partitions, searched_partitions, search_threads) = - prepared_index(vec![10, 11, 12]); + prepared_index(row_ids); let mut query = base_query(); - query.minimum_nprobes = 3; + query.minimum_nprobes = num_partitions; let state = Arc::new(ANNIvfEarlySearchResults::new(1, query.k)); + let partition_idx = (0..num_partitions as u32).collect::>(); + let q_c_dists = (0..num_partitions) + .map(|i| i as f32 * 0.1) + .collect::>(); let batches = ANNIvfSubIndexExec::initial_search( index, query, - Arc::new(UInt32Array::from(vec![0, 1, 2])), - Arc::new(Float32Array::from(vec![0.1, 0.2, 0.3])), + Arc::new(UInt32Array::from(partition_idx)), + Arc::new(Float32Array::from(q_c_dists)), empty_prefilter().await, prepared_metrics(), state, @@ -2893,11 +2945,12 @@ mod tests { .await .unwrap(); - assert_eq!(batches.len(), 3); - assert_eq!(*prepared_partitions.lock().unwrap(), vec![0, 1, 2]); - assert_eq!(*searched_partitions.lock().unwrap(), vec![0, 1, 2]); + let expected: Vec = (0..num_partitions).collect(); + assert_eq!(batches.len(), num_partitions); + assert_eq!(*prepared_partitions.lock().unwrap(), expected); + assert_eq!(*searched_partitions.lock().unwrap(), expected); let search_threads = search_threads.lock().unwrap().clone(); - assert_eq!(search_threads.len(), 3); + assert_eq!(search_threads.len(), num_partitions); assert!( search_threads.iter().all(|name| name.contains("lance-cpu")), "expected prepared searches to run on the cpu runtime, got threads {search_threads:?}", @@ -2908,6 +2961,52 @@ mod tests { ); } + // Regression guard for the batched streaming search (#7642): with more partitions + // than a single batch, the search spans multiple `spawn_cpu` dispatches. Verify that + // every partition is still prepared and searched in order across the batch boundary, + // and that all search work stays on the cpu runtime. + // + // Note: this does not reproduce the single-thread-pool deadlock the async recv/send + // fixes -- that requires a 1-thread CPU pool, which is a process-global singleton and + // impractical to force in a unit test (same limitation noted for the #7423 fix). + #[tokio::test] + async fn test_sequential_search_spans_multiple_cpu_batches() { + let num_partitions = *STREAMING_SEARCH_BATCH_SIZE + 3; + let row_ids = (0..num_partitions).map(|i| i as u64 * 10).collect(); + let (index, prepared_partitions, searched_partitions, search_threads) = + prepared_index(row_ids); + let mut query = base_query(); + query.minimum_nprobes = num_partitions; + let state = Arc::new(ANNIvfEarlySearchResults::new(1, query.k)); + + let partition_idx = (0..num_partitions as u32).collect::>(); + let q_c_dists = (0..num_partitions).map(|i| i as f32).collect::>(); + let batches = ANNIvfSubIndexExec::initial_search( + index, + query, + Arc::new(UInt32Array::from(partition_idx.clone())), + Arc::new(Float32Array::from(q_c_dists)), + empty_prefilter().await, + prepared_metrics(), + state, + usize::MAX, + ) + .try_collect::>() + .await + .unwrap(); + + let expected: Vec = (0..num_partitions).collect(); + assert_eq!(batches.len(), num_partitions); + assert_eq!(*prepared_partitions.lock().unwrap(), expected); + assert_eq!(*searched_partitions.lock().unwrap(), expected); + let search_threads = search_threads.lock().unwrap().clone(); + assert_eq!(search_threads.len(), num_partitions); + assert!( + search_threads.iter().all(|name| name.contains("lance-cpu")), + "expected prepared searches to run on the cpu runtime, got threads {search_threads:?}", + ); + } + #[tokio::test] async fn test_sequential_late_search_prepares_all_then_stops_search_early() { let (index, prepared_partitions, searched_partitions, _search_threads) = @@ -2948,6 +3047,57 @@ mod tests { assert_eq!(state.num_results_found.load(Ordering::Relaxed), 2); } + #[tokio::test] + async fn test_delta_skipping_late_search_releases_sibling() { + let prefilter = empty_prefilter().await; + let state = Arc::new(ANNIvfEarlySearchResults::new(2, 4)); + + let (index_a, _prepared_a, searched_a, _threads_a) = prepared_index(vec![21]); + let mut query_a = base_query(); + query_a.k = 4; + query_a.minimum_nprobes = 1; + let delta_a = ANNIvfSubIndexExec::late_search( + index_a, + query_a, + Arc::new(UInt32Array::from(vec![0])), + Arc::new(Float32Array::from(vec![0.1])), + prefilter.clone(), + prepared_metrics(), + state.clone(), + usize::MAX, + ) + .try_collect::>(); + + let (index_b, _prepared_b, searched_b, _threads_b) = prepared_index(vec![31, 32, 33, 34]); + let mut query_b = base_query(); + query_b.k = 4; + query_b.minimum_nprobes = 1; + query_b.maximum_nprobes = Some(4); + let delta_b = ANNIvfSubIndexExec::late_search( + index_b, + query_b, + Arc::new(UInt32Array::from(vec![0, 1, 2, 3])), + Arc::new(Float32Array::from(vec![0.1, 0.2, 0.3, 0.4])), + prefilter, + prepared_metrics(), + state, + usize::MAX, + ) + .try_collect::>(); + + let (result_a, result_b) = tokio::time::timeout( + std::time::Duration::from_secs(5), + futures::future::join(delta_a, delta_b), + ) + .await + .expect("late search deadlocked because one delta skipped the shared barrier"); + + result_a.unwrap(); + result_b.unwrap(); + assert!(searched_a.lock().unwrap().is_empty()); + assert!(!searched_b.lock().unwrap().is_empty()); + } + #[tokio::test] async fn knn_flat_search() { let schema = Arc::new(ArrowSchema::new(vec![ diff --git a/rust/lance/src/io/exec/optimizer.rs b/rust/lance/src/io/exec/optimizer.rs index 72488f3a14e..528cc2fa709 100644 --- a/rust/lance/src/io/exec/optimizer.rs +++ b/rust/lance/src/io/exec/optimizer.rs @@ -6,6 +6,7 @@ use std::sync::Arc; use super::TakeExec; +use super::filtered_read::FilteredReadExec; use arrow_schema::Schema as ArrowSchema; #[allow(deprecated)] use datafusion::physical_plan::coalesce_batches::CoalesceBatchesExec; @@ -18,17 +19,38 @@ use datafusion::{ }; use datafusion_physical_expr::{PhysicalExpr, expressions::Column}; -/// Rule that eliminates [TakeExec] nodes that are immediately followed by another [TakeExec]. +/// Rule that eliminates take nodes that are immediately followed by another +/// take node, fetching the union of the columns in a single node instead. +/// +/// A "take" is either a [TakeExec] (legacy storage) or a [FilteredReadExec] +/// with a row-stream source (see `FilteredReadExec::row_stream_input`); the +/// scanner emits stacked takes in some plan shapes (e.g. filter columns then +/// projection columns). #[derive(Debug)] pub struct CoalesceTake; impl CoalesceTake { + /// Whether `plan` is a take node this rule knows how to collapse + fn as_take(plan: &Arc) -> Option<&dyn ExecutionPlan> { + if plan.downcast_ref::().is_some() { + Some(plan.as_ref()) + } else if let Some(filtered_read) = plan.downcast_ref::() { + filtered_read + .row_stream_input() + .is_some() + .then_some(plan.as_ref()) + } else { + None + } + } + fn field_order_differs(old_schema: &ArrowSchema, new_schema: &ArrowSchema) -> bool { - old_schema - .fields - .iter() - .zip(&new_schema.fields) - .any(|(old, new)| old.name() != new.name()) + old_schema.fields.len() != new_schema.fields.len() + || old_schema + .fields + .iter() + .zip(&new_schema.fields) + .any(|(old, new)| old.name() != new.name()) } fn remap_collapsed_output( @@ -47,24 +69,39 @@ impl CoalesceTake { Arc::new(ProjectionExec::try_new(project_exprs, plan).unwrap()) } + /// Collapse two stacked takes into one, or return None when the rebuilt + /// node would not produce every column of the original output (the + /// rebuild re-derives what to fetch from the outer take's projection, so + /// a column only the inner take fetched can go missing if the outer + /// projection doesn't cover it) fn collapse_takes( - inner_take: &TakeExec, - outer_take: &TakeExec, + inner_take: &dyn ExecutionPlan, + outer_take: &dyn ExecutionPlan, outer_exec: Arc, - ) -> Arc { + ) -> Option> { let inner_take_input = inner_take.children()[0].clone(); let old_output_schema = outer_take.schema(); - let collapsed = outer_exec - .with_new_children(vec![inner_take_input]) - .unwrap(); + let collapsed = outer_exec.with_new_children(vec![inner_take_input]).ok()?; let new_output_schema = collapsed.schema(); + if old_output_schema + .fields() + .iter() + .any(|field| new_output_schema.field_with_name(field.name()).is_err()) + { + return None; + } + // It's possible that collapsing the take can change the field order. This disturbs DF's planner and // so we must restore it. if Self::field_order_differs(&old_output_schema, &new_output_schema) { - Self::remap_collapsed_output(&old_output_schema, &new_output_schema, collapsed) + Some(Self::remap_collapsed_output( + &old_output_schema, + &new_output_schema, + collapsed, + )) } else { - collapsed + Some(collapsed) } } } @@ -78,26 +115,23 @@ impl PhysicalOptimizerRule for CoalesceTake { ) -> DFResult> { Ok(plan .transform_down(|plan| { - if let Some(outer_take) = plan.as_any().downcast_ref::() { + if let Some(outer_take) = Self::as_take(&plan) { let child = outer_take.children()[0]; - // Case 1: TakeExec -> TakeExec - if let Some(inner_take) = child.as_any().downcast_ref::() { - return Ok(Transformed::yes(Self::collapse_takes( - inner_take, - outer_take, - plan.clone(), - ))); - // Case 2: TakeExec -> CoalesceBatchesExec -> TakeExec - } else if let Some(exec_child) = - child.as_any().downcast_ref::() - { + // Case 1: take -> take + if let Some(inner_take) = Self::as_take(child) { + if let Some(collapsed) = + Self::collapse_takes(inner_take, outer_take, plan.clone()) + { + return Ok(Transformed::yes(collapsed)); + } + // Case 2: take -> CoalesceBatchesExec -> take + } else if let Some(exec_child) = child.downcast_ref::() { let inner_child = exec_child.children()[0].clone(); - if let Some(inner_take) = inner_child.as_any().downcast_ref::() { - return Ok(Transformed::yes(Self::collapse_takes( - inner_take, - outer_take, - plan.clone(), - ))); + if let Some(inner_take) = Self::as_take(&inner_child) + && let Some(collapsed) = + Self::collapse_takes(inner_take, outer_take, plan.clone()) + { + return Ok(Transformed::yes(collapsed)); } } } @@ -128,7 +162,7 @@ impl PhysicalOptimizerRule for SimplifyProjection { ) -> DFResult> { Ok(plan .transform_down(|plan| { - if let Some(proj) = plan.as_any().downcast_ref::() { + if let Some(proj) = plan.downcast_ref::() { let children = proj.children(); if children.len() != 1 { return Ok(Transformed::no(plan)); @@ -145,7 +179,7 @@ impl PhysicalOptimizerRule for SimplifyProjection { } if proj.expr().iter().enumerate().all(|(index, proj_expr)| { - if let Some(expr) = proj_expr.expr.as_any().downcast_ref::() { + if let Some(expr) = proj_expr.expr.downcast_ref::() { // no renaming, no reordering expr.index() == index && expr.name() == proj_expr.alias } else { @@ -184,3 +218,196 @@ pub fn get_physical_optimizer() -> PhysicalOptimizer { Arc::new(datafusion::physical_optimizer::enforce_distribution::EnforceDistribution::new()), ]) } + +#[cfg(test)] +mod tests { + use super::*; + + use arrow_array::cast::AsArray; + use arrow_array::{Int32Array, RecordBatch, RecordBatchIterator, StringArray, UInt64Array}; + use arrow_schema::{DataType, Field as ArrowField, Schema as ArrowSchema}; + use arrow_select::concat::concat_batches; + use datafusion::execution::TaskContext; + use datafusion::physical_plan::stream::RecordBatchStreamAdapter; + use futures::TryStreamExt; + use lance_core::ROW_ID; + use lance_core::datatypes::OnMissing; + use lance_core::utils::tempfile::TempStrDir; + use lance_datafusion::exec::OneShotExec; + use lance_file::version::LanceFileVersion; + + use crate::dataset::{Dataset, WriteParams}; + use crate::io::exec::filtered_read::{FilteredReadExec, FilteredReadOptions}; + + /// 20 rows, one fragment, columns i (Int32) and s (Utf8) + async fn fixture(storage_version: LanceFileVersion) -> (Arc, TempStrDir) { + let schema = Arc::new(ArrowSchema::new(vec![ + ArrowField::new("i", DataType::Int32, false), + ArrowField::new("s", DataType::Utf8, false), + ])); + let batch = RecordBatch::try_new( + schema.clone(), + vec![ + Arc::new(Int32Array::from_iter_values(0..20)), + Arc::new(StringArray::from_iter_values( + (0..20).map(|v| format!("s-{v}")), + )), + ], + ) + .unwrap(); + let tmp_dir = TempStrDir::default(); + let uri = tmp_dir.as_str(); + let params = WriteParams { + data_storage_version: Some(storage_version), + ..Default::default() + }; + let reader = RecordBatchIterator::new(vec![Ok(batch)], schema); + Dataset::write(reader, uri, Some(params)).await.unwrap(); + (Arc::new(Dataset::open(uri).await.unwrap()), tmp_dir) + } + + /// An input plan producing one batch of `_rowid` keys + fn keys_input(keys: Vec) -> Arc { + let schema = Arc::new(ArrowSchema::new(vec![ArrowField::new( + ROW_ID, + DataType::UInt64, + true, + )])); + let batch = + RecordBatch::try_new(schema.clone(), vec![Arc::new(UInt64Array::from(keys))]).unwrap(); + let stream = futures::stream::iter(vec![Ok(batch)]); + let stream = Box::pin(RecordBatchStreamAdapter::new(schema, stream)); + Arc::new(OneShotExec::new(stream)) + } + + fn row_stream_take( + dataset: &Arc, + input: Arc, + columns: &[&str], + ) -> Arc { + // Mirror Scanner::take: full target projection, carried identity kept + let mut projection = dataset + .empty_projection() + .union_columns(columns, OnMissing::Error) + .unwrap(); + projection.with_row_id = true; + Arc::new( + FilteredReadExec::try_new( + dataset.clone(), + FilteredReadOptions::new(projection), + Some(input), + ) + .unwrap(), + ) + } + + async fn run(plan: &Arc) -> RecordBatch { + let stream = plan.execute(0, Arc::new(TaskContext::default())).unwrap(); + let schema = stream.schema(); + let batches: Vec<_> = stream.try_collect().await.unwrap(); + concat_batches(&schema, batches.iter()).unwrap() + } + + fn count_takes(plan: &Arc) -> usize { + let self_count = CoalesceTake::as_take(plan).map(|_| 1).unwrap_or(0); + self_count + + plan + .children() + .iter() + .map(|child| count_takes(child)) + .sum::() + } + + /// Stacked row-stream takes collapse into one node fetching both takes' + /// columns, preserving the output schema and values + #[tokio::test] + async fn collapse_row_stream_takes() { + let (dataset, _tmp) = fixture(LanceFileVersion::Stable).await; + + // OneShotExec inputs are single-use: build the plan fresh per run + let build = |dataset: &Arc| { + let inner = row_stream_take(dataset, keys_input(vec![3, 1, 4]), &["s"]); + row_stream_take(dataset, inner, &["i", "s"]) + }; + let outer = build(&dataset); + let expected_schema = outer.schema(); + assert_eq!(count_takes(&outer), 2); + let expected = run(&outer).await; + + let optimized = CoalesceTake + .optimize(build(&dataset), &ConfigOptions::default()) + .unwrap(); + assert_eq!(count_takes(&optimized), 1); + assert_eq!(optimized.schema(), expected_schema); + + let result = run(&optimized).await; + assert_eq!(result, expected); + let i_col = result + .column_by_name("i") + .unwrap() + .as_primitive::(); + assert_eq!(i_col.values(), &[3, 1, 4]); + } + + /// When the outer take's projection does not cover a column the inner + /// take fetched, the collapse is skipped instead of dropping the column + #[tokio::test] + async fn collapse_skipped_when_column_would_drop() { + let (dataset, _tmp) = fixture(LanceFileVersion::Stable).await; + + // Outer target deliberately omits the inner take's "s" + let build = |dataset: &Arc| { + let inner = row_stream_take(dataset, keys_input(vec![3, 1, 4]), &["s"]); + row_stream_take(dataset, inner, &["i"]) + }; + let outer = build(&dataset); + assert_eq!(count_takes(&outer), 2); + let expected = run(&outer).await; + + let optimized = CoalesceTake + .optimize(build(&dataset), &ConfigOptions::default()) + .unwrap(); + assert_eq!(count_takes(&optimized), 2); + assert_eq!(run(&optimized).await, expected); + assert!(expected.column_by_name("s").is_some()); + } + + /// Legacy TakeExec pairs still collapse (through the CoalesceBatchesExec + /// the scanner inserts on that path) + #[tokio::test] + async fn collapse_legacy_takes() { + let (dataset, _tmp) = fixture(LanceFileVersion::Legacy).await; + + let build = |dataset: &Arc| -> Arc { + let inner_proj = dataset + .empty_projection() + .union_columns(["s"], OnMissing::Error) + .unwrap(); + let inner: Arc = Arc::new( + TakeExec::try_new(dataset.clone(), keys_input(vec![3, 1, 4]), inner_proj) + .unwrap() + .unwrap(), + ); + let outer_proj = dataset + .empty_projection() + .union_columns(["i", "s"], OnMissing::Error) + .unwrap(); + Arc::new( + TakeExec::try_new(dataset.clone(), inner, outer_proj) + .unwrap() + .unwrap(), + ) + }; + let outer = build(&dataset); + let expected_schema = outer.schema(); + assert_eq!(count_takes(&outer), 2); + let expected = run(&outer).await; + + let optimized = CoalesceTake + .optimize(build(&dataset), &ConfigOptions::default()) + .unwrap(); + assert_eq!(count_takes(&optimized), 1); + assert_eq!(optimized.schema(), expected_schema); + assert_eq!(run(&optimized).await, expected); + } +} diff --git a/rust/lance/src/io/exec/projection.rs b/rust/lance/src/io/exec/projection.rs index 3106fcfac61..06bc0b8de67 100644 --- a/rust/lance/src/io/exec/projection.rs +++ b/rust/lance/src/io/exec/projection.rs @@ -44,7 +44,7 @@ pub fn project(input: Arc, projection: &ArrowSchema) -> Resul let field_names = projection.fields().iter().map(|f| f.name()).cloned(); - for (name, selection) in field_names.zip(selections.into_iter()) { + for (name, selection) in field_names.zip(selections) { let expr = selection_as_expr(&selection, input_schema.fields(), None); exprs.push((expr, name)); } diff --git a/rust/lance/src/io/exec/pushdown_scan.rs b/rust/lance/src/io/exec/pushdown_scan.rs index b0b0eacded6..3d72c3291a3 100644 --- a/rust/lance/src/io/exec/pushdown_scan.rs +++ b/rust/lance/src/io/exec/pushdown_scan.rs @@ -2,7 +2,7 @@ // SPDX-FileCopyrightText: Copyright The Lance Authors use std::collections::HashMap; -use std::{any::Any, sync::Arc}; +use std::sync::Arc; use arrow_array::cast::AsArray; use arrow_array::types::{Int64Type, UInt64Type}; @@ -25,7 +25,8 @@ use datafusion::{ }; use datafusion_functions::core::expr_ext::FieldAccessor; use datafusion_physical_expr::EquivalenceProperties; -use futures::{FutureExt, Stream, StreamExt, TryStreamExt}; +use futures::stream::BoxStream; +use futures::{FutureExt, StreamExt, TryStreamExt}; use lance_arrow::{RecordBatchExt, SchemaExt}; use lance_core::utils::tokio::get_num_compute_intensive_cpus; use lance_core::{ROW_ADDR, ROW_ADDR_FIELD, ROW_ID_FIELD}; @@ -40,7 +41,8 @@ use crate::{ Dataset, dataset::{ ROW_ID, - fragment::{FileFragment, FragmentReader}, + fragment::{FileFragment, V1FragmentReader}, + versions, }, datatypes::Schema, }; @@ -158,10 +160,6 @@ impl ExecutionPlan for LancePushdownScanExec { "LancePushdownScanExec" } - fn as_any(&self) -> &dyn Any { - self - } - fn schema(&self) -> SchemaRef { self.output_schema.clone() } @@ -283,7 +281,7 @@ struct FragmentScanner { predicate_projection: Arc, predicate: Expr, config: ScanConfig, - reader: FragmentReader, + reader: V1FragmentReader, stats: Option, } @@ -304,15 +302,14 @@ impl FragmentScanner { if let Some(file_reader_options) = config.file_reader_options.clone() { frag_config = frag_config.with_file_reader_options(file_reader_options); } - let mut reader = fragment.open(dataset.schema(), frag_config).await?; + let mut reader = + versions::open_v1_fragment_reader(&fragment, dataset.schema(), &frag_config).await?; if config.make_deletions_null { reader.with_make_deletions_null(); } // We only need the statistics for the predicate projection. - let stats = reader - .legacy_read_page_stats(Some(&predicate_projection)) - .await?; + let stats = reader.read_page_stats(Some(&predicate_projection)).await?; Ok(Self { fragment, @@ -325,7 +322,7 @@ impl FragmentScanner { }) } - pub fn scan(self) -> Result> + 'static + Send> { + pub fn scan(self) -> Result>> { let batch_readahead = self.config.batch_readahead; let simplified_predicates = self.simplified_predicates()?; let ordered_output = self.config.ordered_output; @@ -381,7 +378,7 @@ impl FragmentScanner { projection_reader.with_row_address(); } let batch = projection_reader - .legacy_read_batch_projected(batch_id, .., &self.projection) + .read_batch_projected(batch_id, .., &self.projection) .await?; let batch = self.final_projection(batch)?; Ok(Some(batch)) @@ -414,7 +411,7 @@ impl FragmentScanner { reader.with_row_address(); let batch = reader - .legacy_read_batch_projected(batch_id, .., &predicate_projection) + .read_batch_projected(batch_id, .., &predicate_projection) .await?; // 2. Evaluate predicate @@ -488,7 +485,7 @@ impl FragmentScanner { self.projection.project_by_ids(&remaining_fields, true); Some( self.reader - .legacy_read_batch_projected( + .read_batch_projected( batch_id, selection.clone(), &remaining_projection, @@ -658,20 +655,20 @@ impl FragmentScanner { } fn simplified_predicates(&self) -> Result> { - let num_batches = self.reader.legacy_num_batches(); + let num_batches = self.reader.num_batches(); if let Some(stats) = &self.stats { let batch_sizes: Vec = (0..num_batches as u32) .map(|batch_id| { self.reader - .legacy_num_rows_in_batch(batch_id) + .num_rows_in_batch(batch_id) .expect("Operation does not yet support v2 fragments") as usize }) .collect(); let schema = Arc::new(ArrowSchema::from(self.predicate_projection.as_ref()).try_into()?); - let context = SimplifyContext::default().with_schema(schema); + let context = SimplifyContext::builder().with_schema(schema).build(); let mut simplifier = ExprSimplifier::new(context); let mut predicates = Vec::with_capacity(num_batches); diff --git a/rust/lance/src/io/exec/rowids.rs b/rust/lance/src/io/exec/rowids.rs index 837d0b81fa3..a0add27d3f7 100644 --- a/rust/lance/src/io/exec/rowids.rs +++ b/rust/lance/src/io/exec/rowids.rs @@ -242,10 +242,6 @@ impl ExecutionPlan for AddRowAddrExec { "AddRowAddrExec" } - fn as_any(&self) -> &dyn std::any::Any { - self - } - fn schema(&self) -> Arc { self.output_schema.clone() } @@ -291,8 +287,8 @@ impl ExecutionPlan for AddRowAddrExec { fn partition_statistics( &self, partition: Option, - ) -> Result { - let mut stats = self.input.partition_statistics(partition)?; + ) -> Result> { + let mut stats = Arc::unwrap_or_clone(self.input.partition_statistics(partition)?); let row_id_col_stats = stats.column_statistics.get(self.rowid_pos).ok_or_else(|| { DataFusionError::Internal("RowAddrExec: rowid column stats not found".into()) @@ -327,7 +323,7 @@ impl ExecutionPlan for AddRowAddrExec { .column_statistics .insert(self.rowaddr_pos, row_addr_col_stats); - Ok(stats) + Ok(Arc::new(stats)) } fn metrics(&self) -> Option { @@ -442,15 +438,19 @@ impl AddRowOffsetExec { frag_id_to_offset: &HashMap, ) -> Result { let row_addr_values = row_addr.as_primitive::().values(); - let mut row_offsets = Vec::with_capacity(row_addr_values.len()); + let mut row_offsets = vec![0; row_addr_values.len()]; + // The deletion iterator only moves forward, so compute in address order and + // scatter the offsets back into input order. + let mut sorted_row_indices = (0..row_addr_values.len()).collect::>(); + sorted_row_indices.sort_unstable_by_key(|index| row_addr_values[*index]); let mut last_frag_id = u32::MAX; let mut last_frag_offset = 0; let mut last_frag_delete_count = 0; let mut dv_iter = None; - for addr in row_addr_values { - let addr = RowAddress::new_from_u64(*addr); + for row_index in sorted_row_indices { + let addr = RowAddress::new_from_u64(row_addr_values[row_index]); let frag_id = addr.fragment_id(); if frag_id != last_frag_id { last_frag_id = frag_id; @@ -480,10 +480,10 @@ impl AddRowOffsetExec { break; } } - row_offsets - .push(last_frag_offset + row_offset as u64 - last_frag_delete_count as u64); + row_offsets[row_index] = + last_frag_offset + row_offset as u64 - last_frag_delete_count as u64; } else { - row_offsets.push(last_frag_offset + row_offset as u64); + row_offsets[row_index] = last_frag_offset + row_offset as u64; } } @@ -506,10 +506,6 @@ impl ExecutionPlan for AddRowOffsetExec { "AddRowOffsetExec" } - fn as_any(&self) -> &dyn std::any::Any { - self - } - fn properties(&self) -> &Arc { &self.properties } @@ -526,7 +522,7 @@ impl ExecutionPlan for AddRowOffsetExec { vec![false] } - fn partition_statistics(&self, partition: Option) -> Result { + fn partition_statistics(&self, partition: Option) -> Result> { self.input.partition_statistics(partition) } @@ -735,4 +731,27 @@ mod test { .fold(0, |acc, col| acc + col.get_array_memory_size()); assert_eq!(stats.total_byte_size, Precision::Exact(actual_byte_size)); } + + #[test] + fn test_row_offsets_with_unsorted_addresses() { + let row_addrs: ArrayRef = Arc::new(UInt64Array::from(vec![ + u64::from(RowAddress::new_from_parts(0, 100)), + u64::from(RowAddress::new_from_parts(0, 50)), + ])); + let frag_id_to_offset = HashMap::from([( + 0, + FragInfo { + row_offset: 1_000, + deletion_vector: Some(Arc::new(DeletionVector::from_iter([10, 60]))), + }, + )]); + + let row_offsets = + AddRowOffsetExec::compute_row_offsets(&row_addrs, &frag_id_to_offset).unwrap(); + + assert_eq!( + row_offsets.as_primitive::().values(), + &[1_098, 1_049] + ); + } } diff --git a/rust/lance/src/io/exec/scalar_index.rs b/rust/lance/src/io/exec/scalar_index.rs index 0f74a13478d..b1d8ed2855e 100644 --- a/rust/lance/src/io/exec/scalar_index.rs +++ b/rust/lance/src/io/exec/scalar_index.rs @@ -6,17 +6,18 @@ use std::sync::{Arc, LazyLock}; use super::utils::{IndexMetrics, InstrumentedRecordBatchStreamAdapter}; use crate::{ Dataset, - dataset::rowids::load_row_id_sequences, + dataset::rowids::{load_row_id_sequences, translate_addr_treemap_to_row_ids}, index::{ prefilter::DatasetPreFilter, scalar_logical::{open_named_scalar_index, scalar_index_fragment_bitmap}, }, }; -use arrow_array::{Array, RecordBatch, UInt64Array}; +use arrow_array::{Array, RecordBatch, UInt64Array, cast::AsArray, types::UInt64Type}; use arrow_schema::{Schema, SchemaRef}; use async_recursion::async_recursion; use async_trait::async_trait; use datafusion::{ + execution::memory_pool::{MemoryConsumer, MemoryReservation}, physical_plan::{ DisplayAs, DisplayFormatType, ExecutionPlan, Partitioning, PlanProperties, execution_plan::{Boundedness, EmissionType}, @@ -27,7 +28,7 @@ use datafusion::{ }; use datafusion_physical_expr::EquivalenceProperties; use futures::{StreamExt, TryFutureExt, TryStreamExt, stream::BoxStream}; -use lance_core::{Error, ROW_ID_FIELD, Result, utils::address::RowAddress}; +use lance_core::{Error, ROW_ID_FIELD, Result, deepsize::DeepSizeOf, utils::address::RowAddress}; use lance_datafusion::{ chunker::break_stream, utils::{ @@ -42,7 +43,8 @@ use lance_index::{ }, }; use lance_select::{ - IndexExprResult, RowAddrMask, RowAddrTreeMap, RowSetOps, result::IndexExprResultWireFormat, + IndexExprResult, NullableIndexExprResult, NullableRowAddrMask, NullableRowAddrSet, RowAddrMask, + RowAddrTreeMap, RowSetOps, result::IndexExprResultWireFormat, }; use lance_table::format::Fragment; use roaring::RoaringBitmap; @@ -58,6 +60,50 @@ impl ScalarIndexLoader for Dataset { ) -> Result> { open_named_scalar_index(self, column, index_name, metrics).await } + + async fn row_addr_result_to_row_ids( + &self, + result: NullableIndexExprResult, + ) -> Result { + // Addresses and row ids only diverge under stable row ids; otherwise the + // address is the row id and there is nothing to translate. + if !self.manifest.uses_stable_row_ids() { + return Ok(result); + } + + let NullableIndexExprResult { lower, upper, .. } = result; + let lower = translate_addr_mask_to_row_ids(self, lower).await?; + let upper = translate_addr_mask_to_row_ids(self, upper).await?; + Ok(NullableIndexExprResult::new(lower, upper)) + } +} + +/// Translate an address-domain [`NullableRowAddrMask`] into the row-id domain +/// +/// Address-domain index results are always positive allow-lists (`AtMost`), so +/// a block-list here would mean a boolean op was applied before translation, +/// which is unsupported. +async fn translate_addr_mask_to_row_ids( + dataset: &Dataset, + mask: NullableRowAddrMask, +) -> Result { + match mask { + NullableRowAddrMask::AllowList(set) => Ok(NullableRowAddrMask::AllowList( + translate_addr_set_to_row_ids(dataset, set).await?, + )), + NullableRowAddrMask::BlockList(_) => Err(Error::internal( + "cannot translate a block-list address mask to the row-id domain", + )), + } +} + +async fn translate_addr_set_to_row_ids( + dataset: &Dataset, + set: NullableRowAddrSet, +) -> Result { + let selected = translate_addr_treemap_to_row_ids(dataset, set.selected_rows()).await?; + let nulls = translate_addr_treemap_to_row_ids(dataset, set.null_rows()).await?; + Ok(NullableRowAddrSet::new(selected, nulls)) } /// An execution node that performs a scalar index search @@ -182,10 +228,6 @@ impl ExecutionPlan for ScalarIndexExec { "ScalarIndexExec" } - fn as_any(&self) -> &dyn std::any::Any { - self - } - fn schema(&self) -> SchemaRef { self.result_format.schema().clone() } @@ -233,11 +275,11 @@ impl ExecutionPlan for ScalarIndexExec { fn partition_statistics( &self, _partition: Option, - ) -> datafusion::error::Result { - Ok(datafusion::physical_plan::Statistics { + ) -> datafusion::error::Result> { + Ok(Arc::new(datafusion::physical_plan::Statistics { num_rows: datafusion::common::stats::Precision::Exact(2), ..datafusion::physical_plan::Statistics::new_unknown(self.result_format.schema()) - }) + })) } fn metrics(&self) -> Option { @@ -267,6 +309,82 @@ pub struct IndexLookup { pub index_name: String, } +const MAP_INDEX_CANDIDATES_MEMORY_CONSUMER: &str = "MapIndexExecCandidates"; + +/// Per-address allowance for cooperative pool accounting of the retained map. +/// This conservatively bounds its approximate [`DeepSizeOf`] growth, but is not +/// an allocator-level peak-memory bound. The reservation is shrunk to the +/// measured retained-map size after each input batch. +const ROW_ADDR_INSERT_RESERVATION_BYTES: usize = 256; + +#[derive(Debug)] +struct DistinctRowAddrs { + emitted: RowAddrTreeMap, + reservation: MemoryReservation, +} + +impl DistinctRowAddrs { + fn new(reservation: MemoryReservation) -> Self { + Self { + emitted: RowAddrTreeMap::new(), + reservation, + } + } + + fn retain_unseen(&mut self, row_addrs: &UInt64Array) -> datafusion::error::Result { + let initial_reservation_size = self.reservation.size(); + // The first batch also has to account for the empty map's inline size, + // which is not part of the initially empty reservation. + let retained_size = initial_reservation_size.max(std::mem::size_of::()); + let unaccounted_candidates = row_addrs + .values() + .iter() + .filter(|row_addr| !self.emitted.contains(**row_addr)) + .count(); + let provisional_size = unaccounted_candidates + .checked_mul(ROW_ADDR_INSERT_RESERVATION_BYTES) + .and_then(|additional| retained_size.checked_add(additional)) + .ok_or_else(|| { + datafusion::error::DataFusionError::ResourcesExhausted(format!( + "Candidate memory reservation overflowed for {MAP_INDEX_CANDIDATES_MEMORY_CONSUMER}" + )) + })?; + self.reservation.try_resize(provisional_size)?; + + // Allocate output storage only after the complete retained-state + // reservation succeeds. + let mut unseen = Vec::with_capacity(unaccounted_candidates); + for row_addr in row_addrs.values() { + if self.emitted.insert(*row_addr) { + unseen.push(*row_addr); + } + } + + let measured_size = self.emitted.deep_size_of(); + if measured_size > provisional_size { + self.rollback_batch(&unseen, initial_reservation_size); + return Err(datafusion::error::DataFusionError::ResourcesExhausted( + format!( + "MapIndexExecCandidates batch exceeded its {ROW_ADDR_INSERT_RESERVATION_BYTES}-byte per-candidate reservation" + ), + )); + } + self.reservation.resize(measured_size); + Ok(UInt64Array::from(unseen)) + } + + fn rollback_batch(&mut self, unseen: &[u64], reservation_size: usize) { + for row_addr in unseen { + let is_removed = self.emitted.remove(*row_addr); + debug_assert!( + is_removed, + "a newly inserted MapIndexExec candidate must be removable" + ); + } + self.reservation.resize(reservation_size); + } +} + impl IndexLookup { pub fn new(column: impl Into, index_name: impl Into) -> Self { Self { @@ -285,7 +403,8 @@ impl IndexLookup { /// row addresses where every column's value is present in its respective /// index — that is, the AND of the per-column index probes. This lets a /// composite-key join trim the candidate row set with every available -/// scalar index before the downstream take. +/// scalar index before the downstream take. A row address reached by more +/// than one input batch is emitted only once. #[derive(Debug)] pub struct MapIndexExec { dataset: Arc, @@ -366,6 +485,7 @@ impl MapIndexExec { lookups: Vec, index_metrics: Arc, metrics_set: ExecutionPlanMetricsSet, + candidate_reservation: MemoryReservation, ) -> datafusion::error::Result { // A row can be found by the composite probe only if it lives in a // fragment covered by *every* index in `lookups`; restrict the @@ -412,6 +532,19 @@ impl MapIndexExec { Self::map_batch(lookups, dataset, deletion_mask, batch, metrics).await } }); + let mut distinct_row_addrs = DistinctRowAddrs::new(candidate_reservation); + let stream = stream.and_then(move |batch| { + // Each batch's index result is already a set. Retain first + // occurrences here to make the complete candidate stream a set too. + let row_addrs = batch.column(0).as_primitive::(); + let result = distinct_row_addrs + .retain_unseen(row_addrs) + .and_then(|unseen| { + RecordBatch::try_new(INDEX_LOOKUP_SCHEMA.clone(), vec![Arc::new(unseen)]) + .map_err(datafusion::error::DataFusionError::from) + }); + futures::future::ready(result) + }); let stream = stream.map(move |batch| { let poll = baseline.record_poll(std::task::Poll::Ready(Some(batch))); match poll { @@ -489,10 +622,6 @@ impl ExecutionPlan for MapIndexExec { "MapIndexExec" } - fn as_any(&self) -> &dyn std::any::Any { - self - } - fn schema(&self) -> SchemaRef { INDEX_LOOKUP_SCHEMA.clone() } @@ -523,6 +652,10 @@ impl ExecutionPlan for MapIndexExec { partition: usize, context: Arc, ) -> datafusion::error::Result { + // Cross-batch deduplication retains every emitted candidate until the + // stream ends, so this state is not covered by per-batch reservations. + let candidate_reservation = MemoryConsumer::new(MAP_INDEX_CANDIDATES_MEMORY_CONSUMER) + .register(context.memory_pool()); let input = self.input.execute(partition, context)?; let stream_fut = Self::build_stream( input, @@ -531,6 +664,7 @@ impl ExecutionPlan for MapIndexExec { self.lookups.clone(), Arc::new(IndexMetrics::new(&self.metrics, partition)), self.metrics.clone(), + candidate_reservation, ); let stream = futures::stream::once(stream_fut).try_flatten(); Ok(Box::pin(RecordBatchStreamAdapter::new( @@ -561,6 +695,10 @@ pub struct MaterializeIndexExec { dataset: Arc, expr: ScalarIndexExpr, fragments: Arc>, + /// Row addresses blocked from the index result due to data overlay files committed after the + /// index was built. ANDead into the candidate mask before row ID materialisation so that stale + /// index entries never reach downstream operators. + overlay_block: Option, properties: Arc, metrics: ExecutionPlanMetricsSet, } @@ -633,16 +771,24 @@ impl MaterializeIndexExec { dataset, expr, fragments, + overlay_block: None, properties, metrics: ExecutionPlanMetricsSet::new(), } } + /// Block specific row addresses (see the `overlay_block` field) from the index result. + pub fn with_overlay_block(mut self, block: RowAddrMask) -> Self { + self.overlay_block = Some(block); + self + } + #[instrument(name = "materialize_scalar_index", skip_all, level = "debug")] async fn do_execute( expr: ScalarIndexExpr, dataset: Arc, fragments: Arc>, + overlay_block: Option, metrics: Arc, ) -> Result { let expr_result = expr.evaluate(dataset.as_ref(), metrics.as_ref()); @@ -670,12 +816,15 @@ impl MaterializeIndexExec { } Ok(result.upper) }; - let mask = if let Some(prefilter) = prefilter { + let mut mask = if let Some(prefilter) = prefilter { let (expr_result, prefilter) = futures::try_join!(expr_result, prefilter)?; take_upper(expr_result)? & (*prefilter).clone() } else { take_upper(expr_result.await?)? }; + if let Some(block) = overlay_block { + mask = mask & block; + } let ids = row_ids_for_mask(mask, &dataset, &fragments).await?; let ids = UInt64Array::from(ids); Ok(RecordBatch::try_new( @@ -776,10 +925,6 @@ impl ExecutionPlan for MaterializeIndexExec { "MaterializeIndexExec" } - fn as_any(&self) -> &dyn std::any::Any { - self - } - fn schema(&self) -> SchemaRef { MATERIALIZE_INDEX_SCHEMA.clone() } @@ -811,6 +956,7 @@ impl ExecutionPlan for MaterializeIndexExec { self.expr.clone(), self.dataset.clone(), self.fragments.clone(), + self.overlay_block.clone(), metrics, ); let stream = futures::stream::iter(vec![batch_fut]) @@ -849,13 +995,24 @@ mod tests { use crate::index::DatasetIndexExt; use arrow::datatypes::UInt64Type; + use arrow::record_batch::RecordBatchIterator; + use arrow_array::{ArrayRef, Int32Array, RecordBatch, UInt64Array}; use arrow_schema::Schema; use datafusion::{ - execution::TaskContext, physical_plan::ExecutionPlan, prelude::SessionConfig, + error::DataFusionError, + execution::{ + TaskContext, + memory_pool::{GreedyMemoryPool, MemoryConsumer, MemoryPool}, + }, + physical_plan::ExecutionPlan, + prelude::SessionConfig, scalar::ScalarValue, }; use futures::TryStreamExt; - use lance_core::utils::tempfile::TempStrDir; + use lance_core::{ + deepsize::DeepSizeOf, + utils::{address::RowAddress, tempfile::TempStrDir}, + }; use lance_datagen::gen_batch; use lance_index::{ IndexType, @@ -864,15 +1021,19 @@ mod tests { expression::{ScalarIndexExpr, ScalarIndexSearch}, }, }; - use lance_select::result::IndexExprResultWireFormat; + use lance_select::{RowAddrTreeMap, RowSetOps, result::IndexExprResultWireFormat}; use crate::{ Dataset, + dataset::WriteParams, io::exec::scalar_index::MaterializeIndexExec, utils::test::{DatagenExt, FragmentCount, FragmentRowCount, NoContextTestFixture}, }; - use super::{MapIndexExec, ScalarIndexExec}; + use super::{ + DistinctRowAddrs, MAP_INDEX_CANDIDATES_MEMORY_CONSUMER, MapIndexExec, + ROW_ADDR_INSERT_RESERVATION_BYTES, ScalarIndexExec, + }; struct TestFixture { dataset: Arc, @@ -910,6 +1071,66 @@ mod tests { } } + #[test] + fn test_map_index_candidates_accept_empty_first_batch() { + let pool: Arc = Arc::new(GreedyMemoryPool::new(1024)); + let reservation = MemoryConsumer::new(MAP_INDEX_CANDIDATES_MEMORY_CONSUMER).register(&pool); + let mut candidates = DistinctRowAddrs::new(reservation); + + let empty = candidates + .retain_unseen(&UInt64Array::from(Vec::::new())) + .unwrap(); + assert!(empty.is_empty()); + assert_eq!(pool.reserved(), RowAddrTreeMap::new().deep_size_of()); + } + + #[test] + fn test_map_index_candidates_respect_memory_pool() { + let mut first_candidate = RowAddrTreeMap::new(); + first_candidate.insert(1); + let first_candidate_size = first_candidate.deep_size_of(); + let pool: Arc = Arc::new(GreedyMemoryPool::new( + ROW_ADDR_INSERT_RESERVATION_BYTES + first_candidate_size, + )); + let reservation = MemoryConsumer::new(MAP_INDEX_CANDIDATES_MEMORY_CONSUMER).register(&pool); + let mut candidates = DistinctRowAddrs::new(reservation); + + let first = candidates + .retain_unseen(&UInt64Array::from(vec![1])) + .unwrap(); + assert_eq!(first.values(), &[1]); + assert_eq!(pool.reserved(), first_candidate_size); + + let second = candidates + .retain_unseen(&UInt64Array::from(vec![2])) + .unwrap(); + assert_eq!(second.values(), &[2]); + let retained_size = pool.reserved(); + assert!(retained_size > first_candidate_size); + + let error = candidates + .retain_unseen(&UInt64Array::from(vec![3])) + .unwrap_err(); + assert!(matches!(error, DataFusionError::ResourcesExhausted(_))); + assert!( + error + .to_string() + .contains(MAP_INDEX_CANDIDATES_MEMORY_CONSUMER) + ); + assert!(candidates.emitted.contains(1)); + assert!(candidates.emitted.contains(2)); + assert!(!candidates.emitted.contains(3)); + assert_eq!(pool.reserved(), retained_size); + + let duplicate = candidates + .retain_unseen(&UInt64Array::from(vec![1, 2])) + .unwrap(); + assert!(duplicate.values().is_empty()); + + drop(candidates); + assert_eq!(pool.reserved(), 0); + } + #[tokio::test] async fn test_materialize_index_exec() { let TestFixture { @@ -928,7 +1149,6 @@ mod tests { needs_recheck: false, fragment_bitmap: None, }); - let fragments = dataset.fragments().clone(); let plan = MaterializeIndexExec::new(dataset, query, fragments); @@ -949,6 +1169,51 @@ mod tests { assert_eq!(batches[0].num_rows(), 5); } + #[tokio::test] + async fn test_translate_addr_treemap_to_stable_row_ids() { + let test_dir = TempStrDir::default(); + let batch = RecordBatch::try_from_iter(vec![( + "id", + Arc::new(Int32Array::from((0..10).collect::>())) as ArrayRef, + )]) + .unwrap(); + let reader = RecordBatchIterator::new(vec![Ok(batch.clone())], batch.schema()); + let write_params = WriteParams { + enable_stable_row_ids: true, + max_rows_per_file: 5, + ..Default::default() + }; + let dataset = Dataset::write(reader, test_dir.as_str(), Some(write_params)) + .await + .unwrap(); + let fragment_id = dataset.get_fragments()[1].id() as u32; + + let mut full_fragment = RowAddrTreeMap::new(); + full_fragment.insert_fragment(fragment_id); + let translated = super::translate_addr_treemap_to_row_ids(&dataset, &full_fragment) + .await + .unwrap(); + let row_ids = translated + .get_fragment_bitmap(0) + .unwrap() + .iter() + .collect::>(); + assert_eq!(row_ids, vec![5, 6, 7, 8, 9]); + + let mut partial_fragment = RowAddrTreeMap::new(); + partial_fragment.insert(RowAddress::new_from_parts(fragment_id, 1).into()); + partial_fragment.insert(RowAddress::new_from_parts(fragment_id, 3).into()); + let translated = super::translate_addr_treemap_to_row_ids(&dataset, &partial_fragment) + .await + .unwrap(); + let row_ids = translated + .get_fragment_bitmap(0) + .unwrap() + .iter() + .collect::>(); + assert_eq!(row_ids, vec![6, 8]); + } + /// `ScalarIndexExec::schema()` (and the stream it emits) must advertise /// the same schema the batch actually carries — otherwise downstream /// consumers that trust `ExecutionPlan::schema()` will see a different diff --git a/rust/lance/src/io/exec/scan.rs b/rust/lance/src/io/exec/scan.rs index 3ec63ce04cc..f250245baec 100644 --- a/rust/lance/src/io/exec/scan.rs +++ b/rust/lance/src/io/exec/scan.rs @@ -1,7 +1,6 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright The Lance Authors -use std::any::Any; use std::ops::Range; use std::pin::Pin; use std::sync::Arc; @@ -26,7 +25,10 @@ use futures::{StreamExt, TryStreamExt}; use lance_arrow::SchemaExt; use lance_core::utils::tokio::get_num_compute_intensive_cpus; use lance_core::utils::tracing::StreamTracingExt; -use lance_core::{Error, ROW_ADDR_FIELD, ROW_ID_FIELD}; +use lance_core::{ + Error, ROW_ADDR_FIELD, ROW_CREATED_AT_VERSION_FIELD, ROW_ID_FIELD, + ROW_LAST_UPDATED_AT_VERSION_FIELD, +}; use lance_file::reader::FileReaderOptions; use lance_io::scheduler::{ScanScheduler, SchedulerConfig}; use lance_table::format::Fragment; @@ -166,18 +168,10 @@ impl LanceStream { metrics: &ExecutionPlanMetricsSet, partition: usize, ) -> Result { - let is_v2_scan = fragments - .iter() - .filter_map(|frag| frag.files.first().map(|f| !f.is_legacy_file())) - .next() - .unwrap_or(false); - if is_v2_scan { - Self::try_new_v2( - dataset, fragments, offsets, projection, config, metrics, partition, - ) - } else { - Self::try_new_v1(dataset, fragments, projection, config, metrics, partition) - } + let version = dataset.manifest().data_storage_format.lance_file_format(); + crate::dataset::versions::create_scan_stream( + version, dataset, fragments, offsets, projection, config, metrics, partition, + ) } #[allow(clippy::too_many_arguments)] @@ -192,7 +186,36 @@ impl LanceStream { ) -> Result { let scan_metrics = ScanMetrics::new(metrics, partition); let timer = scan_metrics.baseline_metrics.elapsed_compute().timer(); - let project_schema = projection.clone(); + let materialize_blob_v2_binary = + crate::dataset::blob::schema_has_blob_v2_binary_view(projection.as_ref()); + let read_projection = if materialize_blob_v2_binary { + Arc::new(crate::dataset::blob::blob_v2_descriptor_schema( + projection.as_ref(), + )) + } else { + projection.clone() + }; + let project_schema = read_projection; + let output_projection = if materialize_blob_v2_binary { + let mut output_projection = projection.as_ref().clone(); + let mut system_fields = Vec::with_capacity(4); + if config.with_row_id { + system_fields.push(ROW_ID_FIELD.clone()); + } + if config.with_row_address { + system_fields.push(ROW_ADDR_FIELD.clone()); + } + if config.with_row_last_updated_at_version { + system_fields.push(ROW_LAST_UPDATED_AT_VERSION_FIELD.clone()); + } + if config.with_row_created_at_version { + system_fields.push(ROW_CREATED_AT_VERSION_FIELD.clone()); + } + output_projection.extend(&system_fields)?; + Arc::new(output_projection) + } else { + projection.clone() + }; let io_parallelism = dataset.object_store.io_parallelism(); // First, use the value specified by the user in the call // Second, use the default from the environment variable, if specified @@ -275,12 +298,14 @@ impl LanceStream { let scan_scheduler_clone = scan_scheduler.clone(); + let materialize_dataset = dataset; let config_for_stream = config.clone(); let batches = stream::iter(file_fragments.into_iter().enumerate()) .map(move |(priority, file_fragment)| { let project_schema = project_schema.clone(); let scan_scheduler = scan_scheduler.clone(); let config = config_for_stream.clone(); + let force_row_address = materialize_blob_v2_binary; #[allow(clippy::type_complexity)] let frag_task: BoxFuture< Result>>>>, @@ -288,7 +313,7 @@ impl LanceStream { (async move { let mut frag_config = FragReadConfig::default() .with_row_id(config.with_row_id) - .with_row_address(config.with_row_address) + .with_row_address(config.with_row_address || force_row_address) .with_row_last_updated_at_version( config.with_row_last_updated_at_version, ) @@ -349,6 +374,25 @@ impl LanceStream { ) .stream_in_current_span() .boxed(); + let inner_stream = if materialize_blob_v2_binary { + inner_stream + .and_then(move |batch| { + let dataset = materialize_dataset.clone(); + let output_projection = output_projection.clone(); + async move { + crate::dataset::blob::materialize_blob_v2_binary_batch( + &dataset, + output_projection.as_ref(), + batch, + ) + .await + .map_err(DataFusionError::from) + } + }) + .boxed() + } else { + inner_stream + }; timer.done(); Ok(Self { @@ -364,6 +408,7 @@ impl LanceStream { pub fn try_new_v1( dataset: Arc, fragments: Arc>, + _offsets: Option>, projection: Arc, config: LanceScanConfig, metrics: &ExecutionPlanMetricsSet, @@ -482,7 +527,9 @@ impl core::fmt::Debug for LanceStream { impl RecordBatchStream for LanceStream { fn schema(&self) -> SchemaRef { - let mut schema: ArrowSchema = self.projection.as_ref().into(); + let output_projection = + crate::dataset::blob::public_blob_v2_binary_output_schema(self.projection.as_ref()); + let mut schema: ArrowSchema = (&output_projection).into(); if self.config.with_row_id { schema = schema.try_with_column(ROW_ID_FIELD.clone()).unwrap(); } @@ -602,7 +649,9 @@ impl LanceScanExec { projection: Arc, config: LanceScanConfig, ) -> Self { - let mut output_schema: ArrowSchema = projection.as_ref().into(); + let output_projection = + crate::dataset::blob::public_blob_v2_binary_output_schema(projection.as_ref()); + let mut output_schema: ArrowSchema = (&output_projection).into(); if config.with_row_id { output_schema = output_schema.try_with_column(ROW_ID_FIELD.clone()).unwrap(); @@ -673,10 +722,6 @@ impl ExecutionPlan for LanceScanExec { "LanceScanExec" } - fn as_any(&self) -> &dyn Any { - self - } - fn schema(&self) -> SchemaRef { self.output_schema.clone() } @@ -727,7 +772,7 @@ impl ExecutionPlan for LanceScanExec { ))) } - fn partition_statistics(&self, _partition: Option) -> Result { + fn partition_statistics(&self, _partition: Option) -> Result> { // Some fragments from older datasets might have the row count stats missing. let (row_count, is_exact) = self.fragments @@ -744,10 +789,10 @@ impl ExecutionPlan for LanceScanExec { false => Precision::Absent, }; - Ok(Statistics { + Ok(Arc::new(Statistics { num_rows, ..Statistics::new_unknown(self.schema().as_ref()) - }) + })) } fn metrics(&self) -> Option { diff --git a/rust/lance/src/io/exec/take.rs b/rust/lance/src/io/exec/take.rs index c3642cdb043..fea9e45b3ad 100644 --- a/rust/lance/src/io/exec/take.rs +++ b/rust/lance/src/io/exec/take.rs @@ -70,6 +70,10 @@ struct TakeStream { dataset: Arc, /// The fields to take from the input stream fields_to_take: Arc, + /// The descriptor-view schema used for storage reads when blob payloads + /// must be materialized after take. + read_fields: Arc, + materialize_blob_v2_binary: bool, /// The output schema, needed for us to merge the new columns /// into the input data in the correct order output_schema: SchemaRef, @@ -92,9 +96,20 @@ impl TakeStream { metrics: &ExecutionPlanMetricsSet, partition: usize, ) -> Self { + let materialize_blob_v2_binary = + crate::dataset::blob::schema_has_blob_v2_binary_view(fields_to_take.as_ref()); + let read_fields = if materialize_blob_v2_binary { + Arc::new(crate::dataset::blob::blob_v2_descriptor_schema( + fields_to_take.as_ref(), + )) + } else { + fields_to_take.clone() + }; Self { dataset, fields_to_take, + read_fields, + materialize_blob_v2_binary, output_schema, readers_cache: Arc::new(Mutex::new(HashMap::new())), scan_scheduler, @@ -131,14 +146,12 @@ impl TakeStream { )) })?; - let reader = Arc::new( - fragment - .open( - &self.fields_to_take, - FragReadConfig::default().with_scan_scheduler(self.scan_scheduler.clone()), - ) - .await?, - ); + let mut read_config = + FragReadConfig::default().with_scan_scheduler(self.scan_scheduler.clone()); + if self.materialize_blob_v2_binary { + read_config = read_config.with_row_address(true); + } + let reader = Arc::new(fragment.open(&self.read_fields, read_config).await?); let mut readers = self.readers_cache.lock().unwrap(); readers.insert(fragment_id, reader.clone()); @@ -355,6 +368,15 @@ impl TakeStream { (None, None) => {} } + if self.materialize_blob_v2_binary { + new_data = crate::dataset::blob::materialize_blob_v2_binary_batch( + &self.dataset, + self.fields_to_take.as_ref(), + new_data, + ) + .await?; + } + Ok(batch.merge_with_schema(&new_data, self.output_schema.as_ref())?) } @@ -487,10 +509,10 @@ impl TakeExec { projection ); - let output_schema = Arc::new(Self::calculate_output_schema( - dataset.schema(), - &input.schema(), - &projection, + let output_schema = + Self::calculate_output_schema(dataset.schema(), &input.schema(), &projection); + let output_schema = Arc::new(crate::dataset::blob::public_blob_v2_binary_output_schema( + &output_schema, )); let output_arrow = Arc::new(ArrowSchema::from(output_schema.as_ref())); let properties = Arc::new( @@ -521,7 +543,7 @@ impl TakeExec { /// /// If this happens the order of the new nested fields will match the order defined in /// the dataset schema. - fn calculate_output_schema( + pub(crate) fn calculate_output_schema( dataset_schema: &Schema, input_schema: &ArrowSchema, projection: &Projection, @@ -576,10 +598,6 @@ impl ExecutionPlan for TakeExec { "TakeExec" } - fn as_any(&self) -> &dyn std::any::Any { - self - } - fn schema(&self) -> SchemaRef { self.output_schema.clone() } @@ -663,11 +681,11 @@ impl ExecutionPlan for TakeExec { fn partition_statistics( &self, partition: Option, - ) -> Result { - Ok(Statistics { + ) -> Result> { + Ok(Arc::new(Statistics { num_rows: self.input.partition_statistics(partition)?.num_rows, ..Statistics::new_unknown(self.schema().as_ref()) - }) + })) } fn properties(&self) -> &Arc { diff --git a/rust/lance/src/io/exec/testing.rs b/rust/lance/src/io/exec/testing.rs index 2d5911a4e46..f979c403431 100644 --- a/rust/lance/src/io/exec/testing.rs +++ b/rust/lance/src/io/exec/testing.rs @@ -4,7 +4,6 @@ //! Testing Node //! -use std::any::Any; use std::sync::Arc; use arrow_array::RecordBatch; @@ -51,10 +50,6 @@ impl ExecutionPlan for TestingExec { "TestingExec" } - fn as_any(&self) -> &dyn Any { - self - } - fn schema(&self) -> arrow_schema::SchemaRef { self.batches[0].schema() } diff --git a/rust/lance/src/io/exec/utils.rs b/rust/lance/src/io/exec/utils.rs index 6e2d50d3736..44092f75459 100644 --- a/rust/lance/src/io/exec/utils.rs +++ b/rust/lance/src/io/exec/utils.rs @@ -2,8 +2,9 @@ // SPDX-FileCopyrightText: Copyright The Lance Authors use lance_datafusion::utils::{ - BYTES_READ_METRIC, ExecutionPlanMetricsSetExt, INDEX_COMPARISONS_METRIC, INDICES_LOADED_METRIC, - IOPS_METRIC, PARTS_LOADED_METRIC, REQUESTS_METRIC, + BYTES_READ_METRIC, ExecutionPlanMetricsSetExt, INDEX_CACHE_HITS_METRIC, + INDEX_CACHE_MISSES_METRIC, INDEX_COMPARISONS_METRIC, INDICES_LOADED_METRIC, IOPS_METRIC, + PARTS_LOADED_METRIC, REQUESTS_METRIC, }; use lance_index::metrics::MetricsCollector; use lance_io::scheduler::{IoStats, ScanScheduler, ScanStats}; @@ -51,6 +52,7 @@ pub(crate) fn build_prefilter( prefilter_source: &PreFilterSource, ds: Arc, index_meta: &[IndexMetadata], + overlay_block: Option, ) -> Result> { let prefilter_loader = match &prefilter_source { PreFilterSource::FilteredRowIds(src_node) => { @@ -63,11 +65,11 @@ pub(crate) fn build_prefilter( } PreFilterSource::None => None, }; - Ok(Arc::new(DatasetPreFilter::new( - ds, - index_meta, - prefilter_loader, - ))) + let mut prefilter = DatasetPreFilter::new(ds, index_meta, prefilter_loader); + if let Some(overlay_block) = overlay_block { + prefilter = prefilter.with_overlay_block(overlay_block); + } + Ok(Arc::new(prefilter)) } // Utility to convert an input (containing row ids) into a prefilter @@ -421,10 +423,6 @@ impl ExecutionPlan for ReplayExec { "ReplayExec" } - fn as_any(&self) -> &dyn std::any::Any { - self - } - fn schema(&self) -> arrow_schema::SchemaRef { self.input.schema() } @@ -521,6 +519,8 @@ pub struct IndexMetrics { indices_loaded: Count, parts_loaded: Count, index_comparisons: Count, + index_cache_hits: Count, + index_cache_misses: Count, /// Per-query sink that accumulates exact index-file I/O as partitions are /// loaded from storage. Shared by all clones of this `IndexMetrics`, so /// concurrent partition loads all funnel into the same counters. Published @@ -535,6 +535,8 @@ impl IndexMetrics { indices_loaded: metrics.new_count(INDICES_LOADED_METRIC, partition), parts_loaded: metrics.new_count(PARTS_LOADED_METRIC, partition), index_comparisons: metrics.new_count(INDEX_COMPARISONS_METRIC, partition), + index_cache_hits: metrics.new_count(INDEX_CACHE_HITS_METRIC, partition), + index_cache_misses: metrics.new_count(INDEX_CACHE_MISSES_METRIC, partition), io_stats: IoStats::new(), io_metrics: IoMetrics::new(metrics, partition), } @@ -559,6 +561,12 @@ impl MetricsCollector for IndexMetrics { fn record_comparisons(&self, num_comparisons: usize) { self.index_comparisons.add(num_comparisons); } + fn record_index_cache_hits(&self, num_hits: usize) { + self.index_cache_hits.add(num_hits); + } + fn record_index_cache_misses(&self, num_misses: usize) { + self.index_cache_misses.add(num_misses); + } fn io_stats(&self) -> Option { Some(self.io_stats.clone()) } diff --git a/rust/lance/src/lib.rs b/rust/lance/src/lib.rs index 2a5d0d2c822..66bc876bd88 100644 --- a/rust/lance/src/lib.rs +++ b/rust/lance/src/lib.rs @@ -81,6 +81,8 @@ pub mod datafusion; pub mod dataset; pub mod index; pub mod io; +#[cfg(feature = "metrics")] +pub mod metrics; pub mod session; pub mod table; pub mod utils; diff --git a/rust/lance/src/metrics.md b/rust/lance/src/metrics.md new file mode 100644 index 00000000000..05d9df0c0e3 --- /dev/null +++ b/rust/lance/src/metrics.md @@ -0,0 +1,40 @@ +Lance publishes metrics through the [`metrics`](https://docs.rs/metrics) crate +facade. Install any recorder (Prometheus, OpenTelemetry, etc.) in your +application and Lance will emit into it; when no recorder is installed, emission +is a cheap no-op. Metrics are only emitted when Lance is built with the +`metrics` feature. + +## Object store metrics + +These track I/O against the underlying object store. The `base` label +identifies the store; its cardinality is controlled by the +`LANCE_OBJECT_STORE_METRICS_LABEL` environment variable: + +- `scheme` (default) — the scheme only (`s3`, `gs`, `az`, `file`, `memory`); + low, bounded cardinality. +- `full` — the store's unique prefix (`s3$my-bucket`, `az$container@account` + where Azure's account also matters), so multiple buckets on the same cloud + can be told apart. Cardinality grows with the number of stores accessed. +- `off` — omit the `base` label entirely. + +`operation` is one of `get`, `put`, `put_part`, `head`, `list`, `delete`, +`copy`, `rename`, `complete_multipart`, or `abort_multipart`. + +Request counts are per logical operation: a `list` or `delete` that spans many +objects is one request, matching how backends batch them. + +| Metric | Type | Labels | Description | +|--------|------|--------|-------------| +| `lance_object_store_requests_total` | counter | `operation`, `base` | Object store requests issued. | +| `lance_object_store_request_bytes_total` | counter | `operation`, `base` | Bytes transferred by `get`/`put` requests. A `get` is counted once its response body has been fully read. | +| `lance_object_store_request_duration_seconds` | histogram | `operation`, `base` | Per-request latency, in seconds. For `get` this covers the full body transfer, not just time-to-first-byte. | +| `lance_object_store_errors_total` | counter | `operation`, `base` | Requests that returned an error. | +| `lance_object_store_in_flight_requests` | gauge | `operation`, `base` | Requests currently in flight. | +| `lance_object_store_throttle_total` | counter | `status`, `base` | Throttle responses (HTTP 429 / 503) seen at the HTTP layer, counted per attempt including retries. The `status` label is the numeric HTTP status. | +| `lance_object_store_retryable_responses_total` | counter | `status`, `base` | Retryable responses (HTTP 5xx / 429 / 408) seen at the HTTP layer, counted per attempt including retries. A superset of `throttle_total`; 409 (conflict) is excluded so commit conflicts are not counted. | + +`lance_object_store_throttle_total` and +`lance_object_store_retryable_responses_total` are recorded only for the native +cloud stores (S3, GCS, Azure); Opendal-backed stores bypass the HTTP client +where the counters are installed, so they report the other object store metrics +but not throttle/retryable counts. diff --git a/rust/lance/src/metrics.rs b/rust/lance/src/metrics.rs new file mode 100644 index 00000000000..1d48865e859 --- /dev/null +++ b/rust/lance/src/metrics.rs @@ -0,0 +1,10 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright The Lance Authors + +//! Metrics published by Lance. +#![doc = include_str!("metrics.md")] +//! +//! The metrics themselves are emitted from the relevant subsystems (for +//! example object store I/O is instrumented in +//! [`lance_io::object_store::metrics`]); this module exists to document the +//! full catalogue of metric names, types, and labels in one place. diff --git a/rust/lance/src/session.rs b/rust/lance/src/session.rs index c6214298341..8328ba20b22 100644 --- a/rust/lance/src/session.rs +++ b/rust/lance/src/session.rs @@ -4,7 +4,7 @@ use std::collections::HashMap; use std::sync::Arc; -use lance_core::cache::{CacheBackend, CacheKeyIterator, LanceCache}; +use lance_core::cache::{CacheBackend, LanceCache, QuickCacheBackend}; use lance_core::deepsize::DeepSizeOf; use lance_core::{Error, Result}; use lance_index::IndexType; @@ -21,6 +21,17 @@ pub(crate) mod caches; pub mod index_caches; pub(crate) mod index_extension; +/// Cache selection for one session cache tier. +#[derive(Clone, Debug)] +pub enum CacheSpec { + /// Use the Lance-level default capacity for the tier. + Default, + /// Use the default in-memory backend with this capacity. + Size(usize), + /// Use an already constructed backend. + Backend(Arc), +} + /// A user session holds the runtime state for a [`crate::Dataset`] /// /// A session will be created automatically when a Dataset is opened. However, you @@ -95,8 +106,10 @@ impl Session { /// /// Parameters: /// - /// - ***index_cache_size***: the size of the index cache. - /// - ***metadata_cache_size***: the size of the metadata cache. + /// - ***index_cache_size***: the size of the index cache, backed by + /// [`QuickCacheBackend`]. + /// - ***metadata_cache_size***: the size of the metadata cache, backed by + /// [`QuickCacheBackend`]. /// - ***store_registry***: the object store registry to use when opening /// datasets. This determines which schemes are available, and also allows /// re-using object stores. @@ -106,8 +119,12 @@ impl Session { store_registry: Arc, ) -> Self { Self { - index_cache: GlobalIndexCache(LanceCache::with_capacity(index_cache_size)), - metadata_cache: GlobalMetadataCache(LanceCache::with_capacity(metadata_cache_size)), + index_cache: GlobalIndexCache(LanceCache::with_backend(Arc::new( + QuickCacheBackend::with_capacity(index_cache_size), + ))), + metadata_cache: GlobalMetadataCache(LanceCache::with_backend(Arc::new( + QuickCacheBackend::with_capacity(metadata_cache_size), + ))), index_extensions: HashMap::new(), store_registry, spill_store: Arc::new(LocalSpillStore::default()), @@ -117,7 +134,7 @@ impl Session { /// Create a session with a custom index cache backend. /// /// The provided backend will be used for caching index data. The metadata - /// cache will use the default Moka-based backend with the given capacity. + /// cache uses a [`QuickCacheBackend`] with the given capacity. pub fn with_index_cache_backend( index_cache_backend: Arc, metadata_cache_size: usize, @@ -125,7 +142,9 @@ impl Session { ) -> Self { Self { index_cache: GlobalIndexCache(LanceCache::with_backend(index_cache_backend)), - metadata_cache: GlobalMetadataCache(LanceCache::with_capacity(metadata_cache_size)), + metadata_cache: GlobalMetadataCache(LanceCache::with_backend(Arc::new( + QuickCacheBackend::with_capacity(metadata_cache_size), + ))), index_extensions: HashMap::new(), store_registry, spill_store: Arc::new(LocalSpillStore::default()), @@ -157,6 +176,64 @@ impl Session { &*self.spill_store } + /// Create a session with custom backends for both caches. + /// + /// Each [`CacheSpec`] controls one tier. [`CacheSpec::Default`] uses that + /// tier's Lance-level default capacity, [`CacheSpec::Size`] uses the + /// default in-memory backend with an explicit capacity, and + /// [`CacheSpec::Backend`] uses a caller-provided backend. This keeps size + /// and backend selection mutually exclusive. + /// + /// This is the recommended constructor when a caller has already resolved + /// backend selection through + /// [`build_from_config`](lance_core::cache::build_from_config) or + /// [`build_from_uri`](lance_core::cache::build_from_uri) — the resulting + /// `Arc` can be plugged in for either or both caches. + /// + /// # Examples + /// + /// ``` + /// # use lance::session::{CacheSpec, Session}; + /// # use lance_core::cache::build_from_uri; + /// # fn example() -> lance_core::Result<()> { + /// let index_backend = build_from_uri("moka://?capacity=1048576")?; + /// let session = Session::with_cache_backends( + /// CacheSpec::Backend(index_backend), + /// CacheSpec::Default, + /// Default::default(), + /// ); + /// # let _ = session; + /// # Ok(()) + /// # } + /// ``` + pub fn with_cache_backends( + index_cache: CacheSpec, + metadata_cache: CacheSpec, + store_registry: Arc, + ) -> Self { + let index_cache = Self::build_cache(index_cache, DEFAULT_INDEX_CACHE_SIZE); + let metadata_cache = Self::build_cache(metadata_cache, DEFAULT_METADATA_CACHE_SIZE); + Self { + index_cache: GlobalIndexCache(index_cache), + metadata_cache: GlobalMetadataCache(metadata_cache), + index_extensions: HashMap::new(), + store_registry, + spill_store: Arc::new(LocalSpillStore::default()), + } + } + + fn build_cache(spec: CacheSpec, default_size: usize) -> LanceCache { + match spec { + CacheSpec::Default => { + LanceCache::with_backend(Arc::new(QuickCacheBackend::with_capacity(default_size))) + } + CacheSpec::Size(size) => { + LanceCache::with_backend(Arc::new(QuickCacheBackend::with_capacity(size))) + } + CacheSpec::Backend(backend) => LanceCache::with_backend(backend), + } + } + /// Register a new index extension. /// /// A name can only be registered once per type of index extension. @@ -239,44 +316,6 @@ impl Session { pub async fn index_cache_stats(&self) -> lance_core::cache::CacheStats { self.index_cache.0.stats().await } - - /// Return an iterator over keys currently held by the index cache. - /// - /// Returns `None` when the index cache backend does not support key - /// inventory. - /// - /// # Examples - /// - /// ``` - /// # use lance::session::Session; - /// # async fn example() { - /// let session = Session::default(); - /// let keys = session.index_cache_keys().await; - /// assert!(keys.is_some()); - /// # } - /// ``` - pub async fn index_cache_keys(&self) -> Option> { - self.index_cache.0.keys().await - } - - /// Return an iterator over keys currently held by the metadata cache. - /// - /// Returns `None` when the metadata cache backend does not support key - /// inventory. - /// - /// # Examples - /// - /// ``` - /// # use lance::session::Session; - /// # async fn example() { - /// let session = Session::default(); - /// let keys = session.metadata_cache_keys().await; - /// assert!(keys.is_some()); - /// # } - /// ``` - pub async fn metadata_cache_keys(&self) -> Option> { - self.metadata_cache.0.keys().await - } } impl Default for Session { @@ -306,7 +345,7 @@ mod tests { } fn type_name() -> &'static str { - "TestVec" + "Test" } } @@ -334,41 +373,72 @@ mod tests { ); } + /// `with_cache_backends` should honor whichever tier the caller + /// provided a backend for and fall back to that tier's default on the + /// other tier. + #[tokio::test] + async fn test_with_cache_backends_uses_provided_and_default() { + use lance_core::cache::build_from_uri; + + let index_backend = build_from_uri("moka://?capacity=1048576").unwrap(); + let session = Session::with_cache_backends( + CacheSpec::Backend(index_backend), + CacheSpec::Default, + Default::default(), + ); + + let value = Arc::new(vec![1, 2, 3]); + session + .index_cache + .insert_with_key(&TestKey("injected-index-backend"), value.clone()) + .await; + assert_eq!( + session + .index_cache + .get_with_key(&TestKey("injected-index-backend")) + .await + .as_deref(), + Some(value.as_ref()) + ); + // Metadata cache fell back to a size-based default. We can only + // sanity-check that the session was constructed without panicking. + let stats = session.metadata_cache.0.stats().await; + assert_eq!(stats.num_entries, 0); + } + #[tokio::test] - async fn test_session_cache_keys() { - let session = Session::new(10_000, 10_000, Default::default()); + async fn test_with_cache_backends_uses_explicit_size() { + let session = Session::with_cache_backends( + CacheSpec::Size(0), + CacheSpec::Size(2048), + Default::default(), + ); session .index_cache - .insert_with_key(&TestKey("index-key"), Arc::new(vec![1])) + .insert_with_key(&TestKey("disabled-index-cache"), Arc::new(vec![1, 2, 3])) .await; + assert!( + session + .index_cache + .get_with_key(&TestKey("disabled-index-cache")) + .await + .is_none() + ); + session .metadata_cache .0 - .insert_with_key(&TestKey("metadata-key"), Arc::new(vec![2])) + .insert_with_key(&TestKey("metadata-cache"), Arc::new(vec![4, 5, 6])) .await; - - let index_keys = session - .index_cache_keys() - .await - .unwrap() - .collect::>(); - assert_eq!(index_keys.len(), 1); - assert_eq!(index_keys[0].prefix(), ""); - assert_eq!(index_keys[0].key(), "index-key"); - assert_eq!(index_keys[0].type_name(), "TestVec"); - - let metadata_keys = session - .metadata_cache_keys() - .await - .unwrap() - .collect::>(); - assert_eq!(metadata_keys.len(), 1); - assert_eq!(metadata_keys[0].prefix(), ""); - assert_eq!(metadata_keys[0].key(), "metadata-key"); - assert_eq!(metadata_keys[0].type_name(), "TestVec"); - - assert_ne!(index_keys, metadata_keys); + assert!( + session + .metadata_cache + .0 + .get_with_key(&TestKey("metadata-cache")) + .await + .is_some() + ); } #[tokio::test] diff --git a/rust/lance/src/session/caches.rs b/rust/lance/src/session/caches.rs index a2dda6069ab..330dbdbd0b0 100644 --- a/rust/lance/src/session/caches.rs +++ b/rust/lance/src/session/caches.rs @@ -14,12 +14,12 @@ use std::{borrow::Cow, ops::Deref}; use lance_core::deepsize::{Context, DeepSizeOf}; use lance_core::{ - cache::{CacheKey, LanceCache}, + cache::{CacheKey, CacheKeySchema, KeyBuilder, LanceCache}, utils::deletion::DeletionVector, }; use lance_select::RowAddrMask; use lance_table::{ - format::{DeletionFile, Manifest}, + format::{DeletionFile, DeletionFileType, Manifest, RowIdMeta}, rowids::{RowIdIndex, RowIdSequence}, }; use object_store::path::Path; @@ -80,6 +80,20 @@ impl CacheKey for ManifestKey<'_> { fn type_name() -> &'static str { "Manifest" } + + fn schema() -> CacheKeySchema { + CacheKeySchema::new("lance.dataset.manifest-key", 1) + } + + fn write_key(&self, builder: &mut KeyBuilder) { + builder.write_u64(self.version); + if let Some(e_tag) = self.e_tag { + builder.write_some(); + builder.write_str(e_tag); + } else { + builder.write_none(); + } + } } #[derive(Debug)] @@ -95,6 +109,14 @@ impl CacheKey for TransactionKey { fn type_name() -> &'static str { "Transaction" } + + fn schema() -> CacheKeySchema { + CacheKeySchema::new("lance.dataset.transaction-key", 1) + } + + fn write_key(&self, builder: &mut KeyBuilder) { + builder.write_u64(self.version); + } } #[derive(Debug)] @@ -117,6 +139,26 @@ impl CacheKey for DeletionFileKey<'_> { fn type_name() -> &'static str { "DeletionVector" } + + fn schema() -> CacheKeySchema { + CacheKeySchema::new("lance.dataset.deletion-file-key", 1) + } + + fn write_key(&self, builder: &mut KeyBuilder) { + builder.write_u64(self.fragment_id); + builder.write_u64(self.deletion_file.read_version); + builder.write_u64(self.deletion_file.id); + builder.write_variant(match &self.deletion_file.file_type { + DeletionFileType::Array => 0, + DeletionFileType::Bitmap => 1, + }); + if let Some(base_id) = self.deletion_file.base_id { + builder.write_some(); + builder.write_u32(base_id); + } else { + builder.write_none(); + } + } } #[derive(Debug)] @@ -139,6 +181,20 @@ impl CacheKey for RowAddrMaskKey { fn type_name() -> &'static str { "RowAddrMask" } + + fn schema() -> CacheKeySchema { + CacheKeySchema::new("lance.dataset.row-address-mask-key", 1) + } + + fn write_key(&self, builder: &mut KeyBuilder) { + builder.write_u64(self.version); + if let Some(restrict_hash) = self.restrict_hash { + builder.write_some(); + builder.write_u64(restrict_hash); + } else { + builder.write_none(); + } + } } #[derive(Debug)] @@ -154,21 +210,68 @@ impl CacheKey for RowIdIndexKey { fn type_name() -> &'static str { "RowIdIndex" } + + fn schema() -> CacheKeySchema { + CacheKeySchema::new("lance.dataset.row-id-index-key", 1) + } + + fn write_key(&self, builder: &mut KeyBuilder) { + builder.write_u64(self.version); + } } #[derive(Debug)] -pub struct RowIdSequenceKey { +pub struct RowIdSequenceKey<'a> { pub fragment_id: u64, + /// Where the sequence is stored. A fragment id alone is not enough: this + /// cache is namespaced by dataset URI only (see + /// [`GlobalMetadataCache::for_dataset`]), and a dataset dropped and + /// recreated at the same URI restarts fragment ids at 0, so a reused id + /// would otherwise be served the earlier generation's sequence (#7645). + /// The `row_id_meta` differentiates generations of dataset fragments. + /// + /// Any operation that changes which row ids a fragment holds also writes it + /// new `row_id_meta`, so generations stay distinct; operations that leave + /// row ids alone (deletes, added columns) leave it untouched and keep + /// hitting the cache. + /// + /// For inline metadata the identity of the sequence *is* its encoded bytes, + /// so the key uses + /// [`InlineRowIds::digest`](lance_table::format::InlineRowIds::digest), + /// which those bytes memoize on first use — an array-encoded sequence is + /// 8 bytes per row, too much to rehash on every lookup. + pub row_id_meta: &'a RowIdMeta, } -impl CacheKey for RowIdSequenceKey { +impl CacheKey for RowIdSequenceKey<'_> { type ValueType = RowIdSequence; + // Only the legacy display form. Identity comes from `write_key` below. fn key(&self) -> Cow<'_, str> { Cow::Owned(format!("row_id_sequence/{}", self.fragment_id)) } fn type_name() -> &'static str { "RowIdSequence" } + + fn schema() -> CacheKeySchema { + CacheKeySchema::new("lance.dataset.row-id-sequence-key", 2) + } + + fn write_key(&self, builder: &mut KeyBuilder) { + builder.write_u64(self.fragment_id); + match self.row_id_meta { + RowIdMeta::Inline(data) => { + builder.write_variant(0); + builder.write_fixed_bytes(data.digest()); + } + RowIdMeta::External(file) => { + builder.write_variant(1); + builder.write_str(&file.path); + builder.write_u64(file.offset); + builder.write_u64(file.size); + } + } + } } impl DSMetadataCache { @@ -178,3 +281,121 @@ impl DSMetadataCache { self.0.with_key_prefix(prefix.as_ref()) } } + +#[cfg(test)] +mod tests { + use std::sync::Arc; + + use lance_table::format::ExternalFile; + use lance_table::rowids::write_row_ids; + + use super::*; + + #[tokio::test] + async fn deletion_file_key_separates_storage_bases() { + let cache = LanceCache::with_capacity(4096); + let deletion_file = DeletionFile { + read_version: 3, + id: 4, + file_type: DeletionFileType::Bitmap, + num_deleted_rows: Some(1), + base_id: None, + }; + cache + .insert_with_key( + &DeletionFileKey { + fragment_id: 2, + deletion_file: &deletion_file, + }, + Arc::new(DeletionVector::NoDeletions), + ) + .await; + + let deletion_file_on_other_base = DeletionFile { + base_id: Some(7), + ..deletion_file + }; + assert!( + cache + .get_with_key(&DeletionFileKey { + fragment_id: 2, + deletion_file: &deletion_file_on_other_base, + }) + .await + .is_none() + ); + } + + #[tokio::test] + async fn row_id_sequence_key_separates_fragment_generations() { + // A dataset dropped and recreated at the same URI restarts fragment ids, + // so the same id must not resolve to the earlier generation's sequence. + let cache = LanceCache::with_capacity(4096); + let first_generation = RowIdMeta::Inline(write_row_ids(&(0..100).into()).into()); + let key = RowIdSequenceKey { + fragment_id: 0, + row_id_meta: &first_generation, + }; + cache + .insert_with_key(&key, Arc::new(RowIdSequence::from(0..100))) + .await; + assert!(cache.get_with_key(&key).await.is_some()); + + let second_generation = RowIdMeta::Inline(write_row_ids(&(100..160).into()).into()); + assert!( + cache + .get_with_key(&RowIdSequenceKey { + fragment_id: 0, + row_id_meta: &second_generation, + }) + .await + .is_none() + ); + } + + #[tokio::test] + async fn row_id_sequence_key_separates_external_slices() { + // External metadata is a read-only legacy shape, but the same slice of + // the same file is the only thing that may share a cache entry. + let cache = LanceCache::with_capacity(4096); + let external = |offset| { + RowIdMeta::External(ExternalFile { + path: "_row_ids/1.rowids".into(), + offset, + size: 16, + }) + }; + let first_slice = external(0); + cache + .insert_with_key( + &RowIdSequenceKey { + fragment_id: 0, + row_id_meta: &first_slice, + }, + Arc::new(RowIdSequence::from(0..100)), + ) + .await; + + let second_slice = external(16); + assert!( + cache + .get_with_key(&RowIdSequenceKey { + fragment_id: 0, + row_id_meta: &second_slice, + }) + .await + .is_none() + ); + // An inline sequence never aliases an external one. + let inline = RowIdMeta::Inline(write_row_ids(&(0..100).into()).into()); + assert!( + cache + .get_with_key(&RowIdSequenceKey { + fragment_id: 0, + row_id_meta: &inline, + }) + .await + .is_none() + ); + } +} diff --git a/rust/lance/src/session/index_caches.rs b/rust/lance/src/session/index_caches.rs index e93b7208b09..f4a17918d2e 100644 --- a/rust/lance/src/session/index_caches.rs +++ b/rust/lance/src/session/index_caches.rs @@ -12,7 +12,7 @@ use std::{borrow::Cow, ops::Deref, sync::Arc}; -use lance_core::cache::{CacheKey, LanceCache}; +use lance_core::cache::{CacheKey, CacheKeySchema, KeyBuilder, LanceCache}; use lance_core::deepsize::{Context, DeepSizeOf}; use lance_index::frag_reuse::FragReuseIndex; use lance_table::format::IndexMetadata; @@ -64,17 +64,31 @@ impl Deref for DSIndexCache { impl DSIndexCache { /// Create an index-specific cache with the given UUID prefix. pub fn for_index(&self, uuid: &Uuid, fri_uuid: Option<&Uuid>) -> LanceCache { + let mut uuid_buffer = Uuid::encode_buffer(); + let cache = self + .0 + .with_key_prefix(uuid.as_hyphenated().encode_lower(&mut uuid_buffer)); if let Some(fri_uuid) = fri_uuid { // If a FRI UUID is provided, use it to create a more specific cache key. - let cache_key = format!("{}-{}", uuid, fri_uuid); - self.0.with_key_prefix(&cache_key) + let mut fri_uuid_buffer = Uuid::encode_buffer(); + cache.with_key_prefix(fri_uuid.as_hyphenated().encode_lower(&mut fri_uuid_buffer)) } else { // Otherwise, just use the index UUID as the key prefix. - self.0.with_key_prefix(&uuid.to_string()) + cache } } } +pub(crate) fn write_index_identity(builder: &mut KeyBuilder, uuid: &Uuid, fri_uuid: Option<&Uuid>) { + builder.write_fixed_bytes(uuid.as_bytes()); + if let Some(fri_uuid) = fri_uuid { + builder.write_some(); + builder.write_fixed_bytes(fri_uuid.as_bytes()); + } else { + builder.write_none(); + } +} + // Cache key types for type-safe cache access #[derive(Debug)] @@ -92,24 +106,47 @@ impl CacheKey for FragReuseIndexKey<'_> { fn type_name() -> &'static str { "FragReuseIndex" } + + fn schema() -> CacheKeySchema { + CacheKeySchema::new("lance.index.fragment-reuse-key", 1) + } + + fn write_key(&self, builder: &mut KeyBuilder) { + builder.write_fixed_bytes(self.uuid.as_bytes()); + } } -#[derive(Debug)] -pub struct IndexMetadataKey { +#[derive(Clone, Copy, Debug)] +pub struct IndexMetadataKey<'a> { pub version: u64, + pub store_identity: &'a str, } -impl CacheKey for IndexMetadataKey { +impl CacheKey for IndexMetadataKey<'_> { type ValueType = Vec; fn key(&self) -> Cow<'_, str> { - Cow::Owned(self.version.to_string()) + Cow::Owned(format!( + "{}:{}/{}", + self.store_identity.len(), + self.store_identity, + self.version + )) } fn type_name() -> &'static str { "Vec" } + fn schema() -> CacheKeySchema { + CacheKeySchema::new("lance.index.metadata-key", 1) + } + + fn write_key(&self, builder: &mut KeyBuilder) { + builder.write_str(self.store_identity); + builder.write_u64(self.version); + } + fn codec() -> Option { Some(lance_table::format::index_metadata_codec()) } @@ -144,4 +181,31 @@ impl CacheKey for ScalarIndexDetailsKey<'_> { fn type_name() -> &'static str { "ScalarIndexDetails" } + + fn schema() -> CacheKeySchema { + CacheKeySchema::new("lance.index.scalar-details-key", 1) + } + + fn write_key(&self, builder: &mut KeyBuilder) { + builder.write_fixed_bytes(self.uuid.as_bytes()); + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn index_metadata_key_isolates_object_store_identity() { + let first = IndexMetadataKey { + version: 7, + store_identity: "s3$first-options", + }; + let second = IndexMetadataKey { + version: 7, + store_identity: "s3$second-options", + }; + + assert_ne!(first.key(), second.key()); + } } diff --git a/rust/lance/src/session/index_extension.rs b/rust/lance/src/session/index_extension.rs index 0f4d2ecf310..c93ae3269ef 100644 --- a/rust/lance/src/session/index_extension.rs +++ b/rust/lance/src/session/index_extension.rs @@ -7,7 +7,7 @@ use std::sync::Arc; use lance_core::Result; use lance_core::deepsize::DeepSizeOf; -use lance_file::previous::reader::FileReader as PreviousFileReader; +use lance_file::versions::v1::reader::FileReader as V1FileReader; use lance_index::{IndexParams, IndexType, vector::VectorIndex}; use uuid::Uuid; @@ -48,7 +48,7 @@ pub trait VectorIndexExtension: IndexExtension { dataset: Arc, column: &str, uuid: &Uuid, - reader: PreviousFileReader, + reader: V1FileReader, ) -> Result>; } @@ -72,10 +72,10 @@ mod test { use arrow_schema::Schema; use datafusion::execution::SendableRecordBatchStream; use lance_core::deepsize::DeepSizeOf; - use lance_file::previous::writer::{ - FileWriter as PreviousFileWriter, FileWriterOptions as PreviousFileWriterOptions, - }; use lance_file::version::LanceFileVersion; + use lance_file::versions::v1::writer::{ + FileWriter as V1FileWriter, FileWriterOptions as V1FileWriterOptions, + }; use lance_index::vector::v3::subindex::SubIndexType; use lance_index::{ INDEX_FILE_NAME, INDEX_METADATA_SCHEMA_KEY, Index, IndexMetadata, IndexType, @@ -270,13 +270,9 @@ mod test { let arrow_schema = Arc::new(Schema::new(vec![VECTOR_ID_FIELD.clone()])); let schema = lance_core::datatypes::Schema::try_from(arrow_schema.as_ref()).unwrap(); - let mut writer: PreviousFileWriter = - PreviousFileWriter::with_object_writer( - writer, - schema, - &PreviousFileWriterOptions::default(), - ) - .unwrap(); + let mut writer: V1FileWriter = + V1FileWriter::with_object_writer(writer, schema, &V1FileWriterOptions::default()) + .unwrap(); writer.add_metadata( INDEX_METADATA_SCHEMA_KEY, json!(IndexMetadata { @@ -304,7 +300,7 @@ mod test { _dataset: Arc, _column: &str, _uuid: &Uuid, - _reader: PreviousFileReader, + _reader: V1FileReader, ) -> Result> { self.load_index_called .store(true, std::sync::atomic::Ordering::Release); diff --git a/rust/lance/src/utils/test.rs b/rust/lance/src/utils/test.rs index 3338eee07a8..1bdd14734b6 100644 --- a/rust/lance/src/utils/test.rs +++ b/rust/lance/src/utils/test.rs @@ -1,7 +1,8 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright The Lance Authors -use std::sync::Arc; +use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; +use std::sync::{Arc, Mutex}; use lance_core::utils::tempfile::{TempDir, TempStrDir}; @@ -12,7 +13,12 @@ use lance_arrow::RecordBatchExt; use lance_core::datatypes::Schema; use lance_datagen::{BatchCount, BatchGeneratorBuilder, ByteCount, RowCount}; use lance_file::version::LanceFileVersion; -use lance_table::format::Fragment; +use lance_table::format::pb::transaction::Operation as PbOperation; +use lance_table::format::{Fragment, Transaction as TableTransaction}; +use lance_table::io::commit::{ + CommitError, CommitHandler, ConditionalPutCommitHandler, ManifestLocation, + ManifestNamingScheme, ManifestWriter, +}; use rand::prelude::SliceRandom; use rand::{Rng, SeedableRng}; @@ -22,6 +28,7 @@ use crate::dataset::fragment::write::FragmentCreateBuilder; use crate::dataset::transaction::Operation; mod failing_store; +pub mod serializing_cache; mod throttle_store; pub use failing_store::FailingProxyStore; @@ -243,6 +250,7 @@ impl TestDatasetGenerator { Fragment { id: 0, files, + overlays: vec![], deletion_file: None, row_id_meta: None, physical_rows: Some(batch.num_rows()), @@ -632,17 +640,8 @@ mod tests { .flat_map(|file| file.fields.iter()) .cloned() .collect::>(); - let mut field_ids = schema - .fields_pre_order() - .filter_map(|f| { - if data_storage_version < LanceFileVersion::V2_1 || f.children.is_empty() { - Some(f.id) - } else { - // In 2.1+, struct / list fields don't have their own column - None - } - }) - .collect::>(); + let (mut field_ids, _) = + lance_file::versions::data_file_columns(data_storage_version.resolve(), &schema); field_ids_frags.sort_unstable(); field_ids.sort_unstable(); assert_eq!(field_ids_frags, field_ids); @@ -729,3 +728,201 @@ mod tests { } } } + +/// How [`AmbiguousCommitHandler`] should treat the next commit. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum AmbiguousFailure { + /// Apply the commit (the manifest lands), then report a conflict — the + /// store equivalent of a successful conditional PUT whose response was + /// lost and whose internal retry saw "already exists". + LandAndConflict, + /// Apply the commit, then report an I/O error. + LandAndError, + /// Do not apply the commit; report an I/O error. + FailOutright, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum AmbiguousFailureTarget { + Any, + Rewrite, + ReserveFragments, +} + +/// A commit handler for tests that can make a commit physically land while +/// reporting a failure, and can make commit-outcome verification unavailable. +/// +/// Delegates real work to [`ConditionalPutCommitHandler`]. +#[derive(Debug)] +pub struct AmbiguousCommitHandler { + /// Failure to inject into the next commit; taken (reset to `None`) when + /// the commit runs. + fail_next_commit: Mutex>, + /// When set, version resolution fails, making commit-outcome verification + /// impossible. + pub fail_resolve: AtomicBool, + /// Number of upcoming resolution calls that should return NotFound. + resolve_not_found_remaining: AtomicUsize, + /// Whether a persistent NotFound proves that the requested version is + /// absent. Tests can disable this to model an eventually visible resolver. + not_found_is_definitive: AtomicBool, + /// Number of version resolutions requested (a proxy for how often commit + /// verification ran). + resolve_calls: AtomicUsize, +} + +impl Default for AmbiguousCommitHandler { + fn default() -> Self { + Self { + fail_next_commit: Mutex::new(None), + fail_resolve: AtomicBool::new(false), + resolve_not_found_remaining: AtomicUsize::new(0), + not_found_is_definitive: AtomicBool::new(true), + resolve_calls: AtomicUsize::new(0), + } + } +} + +impl AmbiguousCommitHandler { + pub fn fail_next(&self, failure: AmbiguousFailure) { + *self.fail_next_commit.lock().unwrap() = Some((failure, AmbiguousFailureTarget::Any)); + } + + /// Arm the failure for the next Rewrite commit only. + pub fn fail_next_rewrite(&self, failure: AmbiguousFailure) { + *self.fail_next_commit.lock().unwrap() = Some((failure, AmbiguousFailureTarget::Rewrite)); + } + + /// Arm the failure for the next ReserveFragments commit only. + pub fn fail_next_reserve(&self, failure: AmbiguousFailure) { + *self.fail_next_commit.lock().unwrap() = + Some((failure, AmbiguousFailureTarget::ReserveFragments)); + } + + pub fn fail_next_resolves_with_not_found(&self, count: usize) { + self.resolve_not_found_remaining + .store(count, Ordering::SeqCst); + self.not_found_is_definitive.store(false, Ordering::SeqCst); + } + + pub fn resolve_calls(&self) -> usize { + self.resolve_calls.load(Ordering::SeqCst) + } +} + +#[async_trait::async_trait] +impl CommitHandler for AmbiguousCommitHandler { + fn is_version_not_found_definitive(&self) -> bool { + self.not_found_is_definitive.load(Ordering::SeqCst) + } + + fn propagate_commit_error_after_success(&self) -> bool { + false + } + + async fn commit( + &self, + manifest: &mut lance_table::format::Manifest, + indices: Option>, + base_path: &object_store::path::Path, + object_store: &lance_io::object_store::ObjectStore, + manifest_writer: ManifestWriter, + naming_scheme: ManifestNamingScheme, + transaction: Option, + ) -> std::result::Result { + let operation = transaction + .as_ref() + .and_then(|t| t.as_pb().operation.as_ref()) + .map(|operation| match operation { + PbOperation::Rewrite(_) => AmbiguousFailureTarget::Rewrite, + PbOperation::ReserveFragments(_) => AmbiguousFailureTarget::ReserveFragments, + _ => AmbiguousFailureTarget::Any, + }); + let failure = { + let mut armed = self.fail_next_commit.lock().unwrap(); + let matches_target = armed.as_ref().is_some_and(|(_, target)| { + *target == AmbiguousFailureTarget::Any || operation == Some(*target) + }); + matches_target.then(|| armed.take().unwrap().0) + }; + match failure { + None => { + ConditionalPutCommitHandler + .commit( + manifest, + indices, + base_path, + object_store, + manifest_writer, + naming_scheme, + transaction, + ) + .await + } + Some(AmbiguousFailure::LandAndConflict) => { + ConditionalPutCommitHandler + .commit( + manifest, + indices, + base_path, + object_store, + manifest_writer, + naming_scheme, + transaction, + ) + .await?; + Err(CommitError::CommitConflict) + } + Some(AmbiguousFailure::LandAndError) => { + ConditionalPutCommitHandler + .commit( + manifest, + indices, + base_path, + object_store, + manifest_writer, + naming_scheme, + transaction, + ) + .await?; + Err(CommitError::OtherError(lance_core::Error::io( + "simulated ambiguous commit failure", + ))) + } + Some(AmbiguousFailure::FailOutright) => Err(CommitError::OtherError( + lance_core::Error::io("simulated outright commit failure"), + )), + } + } + + async fn resolve_version_location( + &self, + base_path: &object_store::path::Path, + version: u64, + object_store: &dyn object_store::ObjectStore, + ) -> lance_core::Result { + self.resolve_calls.fetch_add(1, Ordering::SeqCst); + if self.fail_resolve.load(Ordering::SeqCst) { + return Err(lance_core::Error::io("simulated verification outage")); + } + let mut remaining = self.resolve_not_found_remaining.load(Ordering::SeqCst); + while remaining > 0 { + match self.resolve_not_found_remaining.compare_exchange_weak( + remaining, + remaining - 1, + Ordering::SeqCst, + Ordering::SeqCst, + ) { + Ok(_) => { + return Err(lance_core::Error::not_found( + "simulated temporarily invisible manifest", + )); + } + Err(actual) => remaining = actual, + } + } + ConditionalPutCommitHandler + .resolve_version_location(base_path, version, object_store) + .await + } +} diff --git a/rust/lance/src/utils/test/failing_store.rs b/rust/lance/src/utils/test/failing_store.rs index 07b5f7da92d..feabc4a4e74 100644 --- a/rust/lance/src/utils/test/failing_store.rs +++ b/rust/lance/src/utils/test/failing_store.rs @@ -89,6 +89,12 @@ impl FailingProxyStore { }), ); } + + /// Stop failing calls configured by [`Self::fail_when`]. + pub fn clear_fail_when(&self, method: &str, path_substr: &str) { + let mut policy = self.policy.lock().unwrap(); + policy.clear_before_policy(&format!("fail_{}_{}", method, path_substr)); + } } impl WrappingObjectStore for FailingProxyStore { diff --git a/rust/lance/src/utils/test/serializing_cache.rs b/rust/lance/src/utils/test/serializing_cache.rs new file mode 100644 index 00000000000..89f2b2cbb74 --- /dev/null +++ b/rust/lance/src/utils/test/serializing_cache.rs @@ -0,0 +1,245 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright The Lance Authors + +use std::collections::HashMap; +use std::pin::Pin; +use std::sync::Arc; + +use async_trait::async_trait; +use bytes::Bytes; +use futures::Future; +use lance_core::Result; +use lance_core::cache::{CacheBackend, CacheCodec, CacheEntry, InternalCacheKey, MokaCacheBackend}; + +#[derive(Debug)] +struct SerializedEntry { + bytes: Bytes, + size_bytes: usize, +} + +#[derive(Debug, Default)] +struct SerializedStore { + entries: tokio::sync::Mutex>, + insert_counts: tokio::sync::Mutex>, +} + +/// Test-only cache backend that forces codec-backed entries through bytes. +/// +/// The serialized store can survive [`restart`](Self::restart), while entries +/// without a codec live only in the in-memory L1. This deliberately models the +/// persistence boundary, not a production cache: +/// - it has no capacity limit or eviction policy for serialized entries; +/// - concurrent serialized misses are not single-flight; +/// - accounting uses the logical size supplied by the cache layer. +#[derive(Debug)] +pub struct SerializingCacheBackend { + serialized: Arc, + l1: MokaCacheBackend, +} + +impl SerializingCacheBackend { + pub fn new() -> Self { + Self { + serialized: Arc::new(SerializedStore::default()), + l1: MokaCacheBackend::with_capacity(256 * 1024 * 1024), + } + } + + /// Recreate the backend over the same serialized bytes and an empty L1. + pub fn restart(&self) -> Self { + Self { + serialized: self.serialized.clone(), + l1: MokaCacheBackend::with_capacity(256 * 1024 * 1024), + } + } + + pub async fn serialized_entry_count(&self) -> usize { + self.serialized.entries.lock().await.len() + } + + pub async fn serialized_insert_count(&self, type_id: &'static str) -> usize { + self.serialized + .insert_counts + .lock() + .await + .get(type_id) + .copied() + .unwrap_or(0) + } + + pub async fn l1_entry_count(&self) -> usize { + self.l1.num_entries().await + } +} + +#[async_trait] +impl CacheBackend for SerializingCacheBackend { + async fn get(&self, key: &InternalCacheKey, codec: Option) -> Option { + let Some(codec) = codec else { + return self.l1.get(key, None).await; + }; + let bytes = self + .serialized + .entries + .lock() + .await + .get(key) + .map(|entry| entry.bytes.clone())?; + codec.deserialize(&bytes).hit() + } + + async fn insert( + &self, + key: &InternalCacheKey, + entry: CacheEntry, + size_bytes: usize, + codec: Option, + ) { + let Some(codec) = codec else { + self.l1.insert(key, entry, size_bytes, None).await; + return; + }; + let mut bytes = Vec::new(); + codec + .serialize(&entry, &mut bytes) + .expect("test cache entry serialization should succeed"); + let mut insert_counts = self.serialized.insert_counts.lock().await; + *insert_counts.entry(codec.type_id()).or_default() += 1; + drop(insert_counts); + self.serialized.entries.lock().await.insert( + *key, + SerializedEntry { + bytes: Bytes::from(bytes), + size_bytes, + }, + ); + } + + async fn get_or_insert<'a>( + &self, + key: &InternalCacheKey, + loader: Pin> + Send + 'a>>, + codec: Option, + ) -> Result<(CacheEntry, bool)> { + if let Some(entry) = self.get(key, codec).await { + return Ok((entry, true)); + } + let (entry, size_bytes) = loader.await?; + self.insert(key, entry.clone(), size_bytes, codec).await; + Ok((entry, false)) + } + + async fn clear(&self) { + self.serialized.entries.lock().await.clear(); + self.l1.clear().await; + } + + async fn num_entries(&self) -> usize { + self.serialized.entries.lock().await.len() + self.l1.num_entries().await + } + + async fn size_bytes(&self) -> usize { + let serialized_size = self + .serialized + .entries + .lock() + .await + .values() + .map(|entry| entry.size_bytes) + .sum::(); + serialized_size.saturating_add(self.l1.size_bytes().await) + } +} + +#[cfg(test)] +mod tests { + use lance_core::cache::{CacheCodecImpl, CacheEntryReader, CacheEntryWriter}; + use lance_core::{Error, Result}; + + use super::*; + + #[derive(Debug)] + struct StoredValue(u32); + + impl CacheCodecImpl for StoredValue { + const TYPE_ID: &'static str = "test.LookupCodec"; + const CURRENT_VERSION: u32 = 1; + + fn serialize(&self, writer: &mut CacheEntryWriter<'_>) -> Result<()> { + writer.write_raw(&self.0.to_le_bytes()) + } + + fn deserialize(_reader: &mut CacheEntryReader<'_>) -> Result { + Err(Error::internal( + "the codec retained at insert time must not be used for lookup", + )) + } + } + + #[derive(Debug, PartialEq)] + struct LookupValue(u32); + + impl CacheCodecImpl for LookupValue { + const TYPE_ID: &'static str = "test.LookupCodec"; + const CURRENT_VERSION: u32 = 1; + + fn serialize(&self, writer: &mut CacheEntryWriter<'_>) -> Result<()> { + writer.write_raw(&self.0.to_le_bytes()) + } + + fn deserialize(reader: &mut CacheEntryReader<'_>) -> Result { + let bytes = reader.read_raw()?; + let value = u32::from_le_bytes( + bytes + .as_ref() + .try_into() + .map_err(|_| Error::internal("invalid test cache value"))?, + ); + Ok(Self(value)) + } + } + + #[tokio::test] + async fn restart_keeps_bytes_uses_lookup_codec_and_discards_l1() { + let backend = SerializingCacheBackend::new(); + let serialized_key = InternalCacheKey::from_bytes([1; 16]); + backend + .insert( + &serialized_key, + Arc::new(StoredValue(42)), + 4, + Some(CacheCodec::from_impl::()), + ) + .await; + let l1_key = InternalCacheKey::from_bytes([2; 16]); + backend.insert(&l1_key, Arc::new(7_u32), 4, None).await; + + let restarted = backend.restart(); + assert_eq!(restarted.serialized_entry_count().await, 1); + assert_eq!( + restarted + .serialized_insert_count(StoredValue::TYPE_ID) + .await, + 1 + ); + assert_eq!(restarted.l1_entry_count().await, 0); + assert!(restarted.get(&l1_key, None).await.is_none()); + + let decoded = restarted + .get( + &serialized_key, + Some(CacheCodec::from_impl::()), + ) + .await + .unwrap() + .downcast::() + .unwrap(); + assert_eq!(*decoded, LookupValue(42)); + assert_eq!( + restarted + .serialized_insert_count(StoredValue::TYPE_ID) + .await, + 1 + ); + } +} diff --git a/rust/lance/tests/count_pushdown/mod.rs b/rust/lance/tests/count_pushdown/mod.rs index aaa3f5f539e..d8afa051bca 100644 --- a/rust/lance/tests/count_pushdown/mod.rs +++ b/rust/lance/tests/count_pushdown/mod.rs @@ -81,7 +81,7 @@ fn lance_aware_context(dataset: Arc) -> SessionContext { fn plan_contains_pushdown(plan: &Arc) -> bool { let mut found = false; plan.apply(|node| { - if node.as_any().is::() { + if node.is::() { found = true; Ok(TreeNodeRecursion::Stop) } else { diff --git a/rust/lance/tests/integration_tests.rs b/rust/lance/tests/integration_tests.rs index 7a6d3e71ca4..e6ea4e321ae 100644 --- a/rust/lance/tests/integration_tests.rs +++ b/rust/lance/tests/integration_tests.rs @@ -4,6 +4,7 @@ // NOTE: we only create one integration test binary, to keep compilation overhead down. mod count_pushdown; +mod mem_wal; #[cfg(feature = "slow_tests")] mod query; #[cfg(feature = "slow_tests")] diff --git a/rust/lance/tests/mem_wal/mod.rs b/rust/lance/tests/mem_wal/mod.rs new file mode 100644 index 00000000000..c617c2a1589 --- /dev/null +++ b/rust/lance/tests/mem_wal/mod.rs @@ -0,0 +1,131 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright The Lance Authors + +//! End-to-end MemWAL tests through the public [`ShardWriter`] surface, as +//! opposed to the lib-level unit tests that can reach internals directly. + +use std::time::Duration; + +use arrow_array::cast::AsArray; +use arrow_array::record_batch; +use arrow_array::types::Int32Type; +use lance::dataset::mem_wal::{ShardWriter, ShardWriterConfig}; +use lance_core::FenceReason; +use lance_io::object_store::ObjectStore; +use uuid::Uuid; + +fn durable_writer_config(shard_id: Uuid) -> ShardWriterConfig { + ShardWriterConfig { + shard_id, + durable_write: true, + max_wal_buffer_size: 64 * 1024 * 1024, + max_wal_flush_interval: Some(Duration::from_millis(10)), + max_memtable_size: 64 * 1024 * 1024, + manifest_scan_batch_size: 2, + ..Default::default() + } +} + +#[tokio::test] +async fn durable_put_is_readable_through_public_scan() { + let base_uri = "memory://".to_string(); + let (store, base_path) = ObjectStore::from_uri(&base_uri).await.unwrap(); + let batch = record_batch!(("id", Int32, [1, 2, 3])).unwrap(); + let writer = ShardWriter::open( + store, + base_path, + base_uri, + durable_writer_config(Uuid::new_v4()), + batch.schema(), + vec![], + ) + .await + .unwrap(); + + writer.put(vec![batch]).await.unwrap(); + + let scanned = writer.scan().await.unwrap().try_into_batch().await.unwrap(); + let scanned_ids = scanned["id"].as_primitive::(); + assert_eq!( + scanned_ids.values(), + &[1, 2, 3], + "a durable put must be readable through the public scan API" + ); + + writer.close().await.unwrap(); +} + +/// A durable put into a post-rotation MemTable must be acknowledged only by +/// its own WAL flush, never by a flush from an older generation. +/// +/// Batch positions restart at 0 in every MemTable generation. A durability +/// watch keyed on that generation-local position is satisfied by the previous +/// generation's flush at the same position — so the first durable put after a +/// rotation used to return success before its own WAL append completed. Here +/// a peer writer has claimed a higher epoch in between, so the false success +/// is observable: the put's own flush is fenced, and the put must surface +/// that fence instead of reporting durability. +#[tokio::test] +async fn durable_put_does_not_alias_across_memtable_generations() { + let base_uri = "memory://".to_string(); + let (store, base_path) = ObjectStore::from_uri(&base_uri).await.unwrap(); + let shard_id = Uuid::new_v4(); + let first_batch = record_batch!(("id", Int32, [1])).unwrap(); + let schema = first_batch.schema(); + let config = durable_writer_config(shard_id); + + let writer_a = ShardWriter::open( + store.clone(), + base_path.clone(), + base_uri.clone(), + config.clone(), + schema.clone(), + vec![], + ) + .await + .unwrap(); + + // Write and fully flush generation N, then rotate to generation N+1. + writer_a.put(vec![first_batch]).await.unwrap(); + let first_generation = writer_a.memtable_stats().await.unwrap().generation; + writer_a.force_seal_active().await.unwrap(); + writer_a.wait_for_flush_drain().await.unwrap(); + let current_generation = writer_a.memtable_stats().await.unwrap().generation; + assert_eq!( + current_generation, + first_generation + 1, + "expected rotation from generation {first_generation}, got {current_generation}" + ); + + // A peer claims a higher epoch, fencing writer A's next WAL append. + let writer_b = ShardWriter::open(store, base_path, base_uri, config, schema, vec![]) + .await + .unwrap(); + assert!( + writer_b.epoch() > writer_a.epoch(), + "expected peer epoch to increase: writer_a={}, writer_b={}", + writer_a.epoch(), + writer_b.epoch() + ); + + // Generation N+1's first durable put lands at the same local batch + // position (0..1) that generation N already flushed. It must wait for its + // own flush and therefore surface the fence, not ack from the stale + // generation. + let second_batch = record_batch!(("id", Int32, [2])).unwrap(); + let error = writer_a + .put(vec![second_batch]) + .await + .expect_err("generation-2 durable put must wait for its own WAL flush"); + assert_eq!( + error.fence_reason(), + Some(FenceReason::PeerClaimedEpoch), + "expected a peer-claimed-epoch fence, got: {error}" + ); + assert!( + error.to_string().contains("Writer fenced"), + "expected the writer-fenced error prefix, got: {error}" + ); + + writer_b.close().await.unwrap(); +} diff --git a/rust/lance/tests/query/inverted.rs b/rust/lance/tests/query/inverted.rs index 4392278c0d8..c51f715f1e3 100644 --- a/rust/lance/tests/query/inverted.rs +++ b/rust/lance/tests/query/inverted.rs @@ -5,15 +5,27 @@ use std::sync::Arc; use arrow_array::cast::AsArray; use arrow_array::{ - ArrayRef, Int32Array, RecordBatch, RecordBatchIterator, StringArray, UInt32Array, + ArrayRef, FixedSizeListArray, Float32Array, Int32Array, ListArray, RecordBatch, + RecordBatchIterator, StringArray, StructArray, UInt32Array, + builder::{ListBuilder, StringBuilder}, }; +use arrow_buffer::{OffsetBuffer, ScalarBuffer}; +use arrow_schema::{DataType, Field as ArrowField, Fields as ArrowFields}; use lance::Dataset; -use lance::dataset::scanner::ColumnOrdering; -use lance::dataset::{InsertBuilder, WriteParams}; -use lance::index::DatasetIndexExt; +use lance::dataset::optimize::{CompactionOptions, compact_files}; +use lance::dataset::scanner::{ColumnOrdering, QueryFilter}; +use lance::dataset::{ColumnAlteration, InsertBuilder, WriteMode, WriteParams}; +use lance::index::{DatasetIndexExt, DatasetIndexInternalExt}; +use lance_arrow::FixedSizeListArrayExt; use lance_index::IndexType; -use lance_index::scalar::inverted::Language; -use lance_index::scalar::inverted::query::{FtsQuery, PhraseQuery}; +use lance_index::metrics::NoOpMetricsCollector; +use lance_index::optimize::OptimizeOptions; +use lance_index::prefilter::NoFilter; +use lance_index::scalar::inverted::query::{ + BooleanQuery, BoostQuery, FtsQuery, FtsSearchParams, MatchQuery, MultiMatchQuery, Occur, + Operator, PhraseQuery, collect_query_tokens, +}; +use lance_index::scalar::inverted::{DocumentGranularity, Language}; use lance_index::scalar::{FullTextSearchQuery, InvertedIndexParams}; use lance_table::format::IndexMetadata; @@ -41,6 +53,46 @@ fn params_for(base_tokenizer: &str, lower_case: bool, with_position: bool) -> In .max_token_length(None) } +fn list_element_params(with_position: bool) -> InvertedIndexParams { + base_inverted_params(with_position).document_granularity(DocumentGranularity::ListElement) +} + +fn list_element_match_node(column: &str, terms: &str) -> MatchQuery { + MatchQuery::new(terms.to_string()) + .with_column(Some(column.to_string())) + .with_document_granularity(DocumentGranularity::ListElement) +} + +fn list_element_match(column: &str, terms: &str) -> FullTextSearchQuery { + FullTextSearchQuery::new_query(FtsQuery::Match(list_element_match_node(column, terms))) +} + +fn list_element_phrase(column: &str, terms: &str) -> FullTextSearchQuery { + FullTextSearchQuery::new_query(FtsQuery::Phrase( + PhraseQuery::new(terms.to_string()) + .with_column(Some(column.to_string())) + .with_document_granularity(DocumentGranularity::ListElement), + )) +} + +fn row_match_node(column: &str, terms: &str) -> MatchQuery { + MatchQuery::new(terms.to_string()) + .with_column(Some(column.to_string())) + .with_document_granularity(DocumentGranularity::Row) +} + +fn row_match(column: &str, terms: &str) -> FullTextSearchQuery { + FullTextSearchQuery::new_query(FtsQuery::Match(row_match_node(column, terms))) +} + +fn row_phrase(column: &str, terms: &str) -> FullTextSearchQuery { + FullTextSearchQuery::new_query(FtsQuery::Phrase( + PhraseQuery::new(terms.to_string()) + .with_column(Some(column.to_string())) + .with_document_granularity(DocumentGranularity::Row), + )) +} + // Execute a full-text search with optional filter and deterministic id ordering. async fn run_fts(ds: &Dataset, query: FullTextSearchQuery, filter: Option<&str>) -> RecordBatch { let mut scanner = ds.scan(); @@ -75,6 +127,891 @@ async fn assert_fts_expected( assert_eq!(&expected, &scanned); } +fn string_lists(values: &[Option>>]) -> ArrayRef { + let mut builder = ListBuilder::new(StringBuilder::new()); + for value in values { + match value { + Some(elements) => { + for element in elements { + match element { + Some(element) => builder.values().append_value(element), + None => builder.values().append_null(), + } + } + builder.append(true); + } + None => builder.append(false), + } + } + Arc::new(builder.finish()) +} + +fn element_hits(batch: &RecordBatch) -> Vec<(i32, Vec)> { + let ids = batch["id"].as_primitive::(); + let coordinates = batch["_doc_index"] + .as_any() + .downcast_ref::() + .unwrap(); + let mut hits = (0..batch.num_rows()) + .map(|row| { + let coordinate = coordinates.value(row); + ( + ids.value(row), + coordinate + .as_primitive::() + .values() + .to_vec(), + ) + }) + .collect::>(); + hits.sort_unstable(); + hits +} + +fn element_scored_hits(batch: &RecordBatch) -> Vec<(i32, Vec, f32)> { + let ids = batch["id"].as_primitive::(); + let coordinates = batch["_doc_index"].as_list::(); + let scores = batch["_score"].as_primitive::(); + let mut hits = (0..batch.num_rows()) + .map(|row| { + ( + ids.value(row), + coordinates + .value(row) + .as_primitive::() + .values() + .to_vec(), + scores.value(row), + ) + }) + .collect::>(); + hits.sort_unstable_by(|left, right| (&left.0, &left.1).cmp(&(&right.0, &right.1))); + hits +} + +fn assert_same_element_scores(left: &RecordBatch, right: &RecordBatch) { + let left = element_scored_hits(left); + let right = element_scored_hits(right); + assert_eq!(left.len(), right.len()); + for (left, right) in left.iter().zip(&right) { + assert_eq!((&left.0, &left.1), (&right.0, &right.1)); + assert!((left.2 - right.2).abs() < 1e-5, "{left:?} != {right:?}"); + } +} + +fn assert_element_coordinates_point_to(batch: &RecordBatch, column: &str, term: &str) { + let values = batch[column].as_list::(); + let coordinates = batch["_doc_index"].as_list::(); + for row in 0..batch.num_rows() { + let coordinate = coordinates + .value(row) + .as_primitive::() + .value(0) as usize; + let elements = values.value(row); + let element = elements.as_string::().value(coordinate); + assert!( + element.contains(term), + "coordinate {coordinate} selected {element:?}" + ); + } +} + +fn expected_bm25_score( + num_docs: usize, + token_docs: usize, + total_tokens: u32, + doc_tokens: u32, +) -> f32 { + let num_docs = num_docs as f32; + let idf = ((num_docs - token_docs as f32 + 0.5) / (token_docs as f32 + 0.5) + 1.0).ln(); + let avg_doc_length = total_tokens as f32 / num_docs; + let doc_norm = 1.2 * (1.0 - 0.75 + 0.75 * doc_tokens as f32 / avg_doc_length); + idf * 2.2 / (1.0 + doc_norm) +} + +#[tokio::test] +async fn test_row_document_raw_list_is_consistent_across_index_coverage() { + let batch = RecordBatch::try_from_iter(vec![ + ("id", Arc::new(Int32Array::from(vec![0])) as ArrayRef), + ("tags", string_lists(&[Some(vec![Some("a"), Some("b")])])), + ]) + .unwrap(); + let test_dir = tempfile::tempdir().unwrap(); + let mut ds = Dataset::write( + RecordBatchIterator::new(vec![Ok(batch.clone())], batch.schema()), + test_dir.path().to_str().unwrap(), + None, + ) + .await + .unwrap(); + + ds.create_index( + &["tags"], + IndexType::Inverted, + None, + ¶ms_for("raw", false, false), + true, + ) + .await + .unwrap(); + + let appended = RecordBatch::try_from_iter(vec![ + ("id", Arc::new(Int32Array::from(vec![1])) as ArrayRef), + ("tags", string_lists(&[Some(vec![Some("a"), Some("b")])])), + ]) + .unwrap(); + ds = InsertBuilder::new(Arc::new(ds)) + .with_params(&WriteParams { + mode: WriteMode::Append, + ..Default::default() + }) + .execute(vec![appended]) + .await + .unwrap(); + + let joined = run_fts(&ds, row_match("tags", "a b"), None).await; + assert_eq!( + joined["id"] + .as_primitive::() + .values(), + &[0, 1] + ); + let element = run_fts(&ds, row_match("tags", "a"), None).await; + assert_eq!(element.num_rows(), 0); +} + +#[tokio::test] +async fn test_element_document_fts_flat_indexed_and_mixed() { + let ids = Arc::new(Int32Array::from(vec![0, 1, 2, 3, 4, 5])); + let tags = string_lists(&[ + Some(vec![ + Some("alpha beta"), + Some("gamma alpha"), + None, + Some(""), + Some("delta"), + ]), + Some(vec![Some("beta"), Some("gamma")]), + None, + Some(vec![]), + Some(vec![None, None]), + Some(vec![Some("!!!"), Some("epsilon")]), + ]); + let batch = RecordBatch::try_from_iter(vec![("id", ids as ArrayRef), ("tags", tags)]).unwrap(); + let schema = batch.schema(); + let test_dir = tempfile::tempdir().unwrap(); + let mut ds = Dataset::write( + RecordBatchIterator::new(vec![Ok(batch)], schema), + test_dir.path().to_str().unwrap(), + None, + ) + .await + .unwrap(); + + let element_query = list_element_match("tags", "alpha"); + let element_and_query = FullTextSearchQuery::new_query(FtsQuery::Match( + list_element_match_node("tags", "alpha beta").with_operator(Operator::And), + )); + let flat = run_fts(&ds, element_query.clone(), None).await; + assert_eq!(element_hits(&flat), vec![(0, vec![0]), (0, vec![1])]); + let flat_and = run_fts(&ds, element_and_query.clone(), None).await; + assert_eq!(element_hits(&flat_and), vec![(0, vec![0])]); + let flat_phrase = run_fts(&ds, list_element_phrase("tags", "alpha beta"), None).await; + assert_eq!(element_hits(&flat_phrase), vec![(0, vec![0])]); + let flat_cross_element_phrase = + run_fts(&ds, list_element_phrase("tags", "beta gamma"), None).await; + assert_eq!(flat_cross_element_phrase.num_rows(), 0); + let row_flat_phrase = run_fts( + &ds, + FullTextSearchQuery::new_query(FtsQuery::Phrase( + PhraseQuery::new("beta gamma".to_string()).with_column(Some("tags".to_string())), + )), + None, + ) + .await; + assert_eq!( + row_flat_phrase["id"] + .as_primitive::() + .values(), + &[0, 1] + ); + assert_element_coordinates_point_to(&flat, "tags", "alpha"); + let params = list_element_params(true); + let err = ds + .create_index(&["id"], IndexType::Inverted, None, ¶ms, true) + .await + .unwrap_err(); + assert!(err.to_string().contains("must resolve to Utf8"), "{err}"); + let err = ds + .create_index(&["tags[*]"], IndexType::Inverted, None, ¶ms, true) + .await + .unwrap_err(); + assert!(err.to_string().contains("tags[*]"), "{err}"); + ds.create_index(&["tags"], IndexType::Inverted, None, ¶ms, true) + .await + .unwrap(); + let mut element_only_auto = ds.scan(); + element_only_auto + .full_text_search(FullTextSearchQuery::new("alpha".to_string())) + .unwrap(); + let err = element_only_auto.try_into_batch().await.unwrap_err(); + assert!( + err.to_string() + .contains("unless an INVERTED index has been created"), + "{err}" + ); + let inferred_element = run_fts( + &ds, + FullTextSearchQuery::new_query(FtsQuery::Match( + MatchQuery::new("alpha".to_string()).with_column(Some("tags".to_string())), + )), + None, + ) + .await; + assert_eq!( + element_hits(&inferred_element), + vec![(0, vec![0]), (0, vec![1])] + ); + let mut mismatched_row = ds.scan(); + mismatched_row + .full_text_search(row_match("tags", "alpha")) + .unwrap(); + let err = mismatched_row.try_into_batch().await.unwrap_err(); + assert!( + err.to_string().contains("requested Row") && err.to_string().contains("ListElement"), + "{err}" + ); + + ds.create_index( + &["tags"], + IndexType::Inverted, + None, + &base_inverted_params(true), + true, + ) + .await + .unwrap(); + let names = ds + .load_indices() + .await + .unwrap() + .iter() + .map(|index| index.name.clone()) + .collect::>(); + assert!(names.contains(&"tags_idx".to_string())); + assert!(names.contains(&"tags_list_element_idx".to_string())); + + let auto_row = run_fts(&ds, FullTextSearchQuery::new("alpha".to_string()), None).await; + assert_eq!(auto_row.num_rows(), 1); + assert!(auto_row.column_by_name("_doc_index").is_none()); + let mut ambiguous = ds.scan(); + ambiguous + .full_text_search( + FullTextSearchQuery::new("alpha".to_string()) + .with_column("tags".to_string()) + .unwrap(), + ) + .unwrap(); + let err = ambiguous.try_into_batch().await.unwrap_err(); + assert!( + err.to_string().contains("ambiguous") + && err.to_string().contains("specify document_granularity"), + "{err}" + ); + let mut row_projection = ds.scan(); + row_projection + .full_text_search(row_match("tags", "alpha")) + .unwrap(); + row_projection.project(&["_doc_index"]).unwrap(); + let err = row_projection.try_into_batch().await.unwrap_err(); + assert!(err.to_string().contains("_doc_index"), "{err}"); + + let indexed = run_fts(&ds, element_query.clone(), None).await; + assert_eq!(element_hits(&indexed), vec![(0, vec![0]), (0, vec![1])]); + let indexed_and = run_fts(&ds, element_and_query, None).await; + assert_eq!(element_hits(&indexed_and), vec![(0, vec![0])]); + assert_element_coordinates_point_to(&indexed, "tags", "alpha"); + let element_index = ds + .load_indices_by_name("tags_list_element_idx") + .await + .unwrap() + .pop() + .unwrap(); + assert_eq!( + element_index.index_version, + lance_index::scalar::inverted::INVERTED_INDEX_VERSION_V3 as i32 + ); + let element_index = ds + .open_scalar_index("tags", &element_index.uuid, &NoOpMetricsCollector) + .await + .unwrap(); + assert_eq!(element_index.statistics().unwrap()["num_docs"], 11); + let element_index = element_index + .as_any() + .downcast_ref::() + .unwrap(); + let (total_tokens, num_docs, token_docs) = element_index + .bm25_stats_for_terms(&["alpha".to_string()], None) + .await + .unwrap(); + assert_eq!((total_tokens, num_docs, token_docs), (8, 11, vec![2])); + let mut tokenizer = element_index.tokenizer(); + let tokens = Arc::new(collect_query_tokens("alpha", &mut tokenizer)); + let documents = element_index + .bm25_search_documents( + tokens, + Arc::new(FtsSearchParams::default()), + Operator::Or, + Arc::new(NoFilter), + Arc::new(NoOpMetricsCollector), + None, + ) + .await + .unwrap(); + let expected_score = expected_bm25_score(11, 2, 8, 2); + assert!( + documents + .iter() + .all(|document| (document.score.0 - expected_score).abs() < 1e-5), + "{documents:?}" + ); + assert_same_element_scores(&flat, &indexed); + let indexed_phrase = run_fts(&ds, list_element_phrase("tags", "alpha beta"), None).await; + assert_eq!(element_hits(&indexed_phrase), vec![(0, vec![0])]); + let filtered = run_fts(&ds, element_query.clone(), Some("id = 0")).await; + assert_eq!(element_hits(&filtered), vec![(0, vec![0]), (0, vec![1])]); + let ordinal = run_fts(&ds, list_element_match("tags", "delta"), None).await; + assert_eq!(element_hits(&ordinal), vec![(0, vec![4])]); + let punctuation_gap = run_fts(&ds, list_element_match("tags", "epsilon"), None).await; + assert_eq!(element_hits(&punctuation_gap), vec![(5, vec![1])]); + let limited = run_fts(&ds, element_query.clone().limit(Some(1)), None).await; + assert_eq!(limited.num_rows(), 1); + assert_eq!( + limited["id"] + .as_primitive::() + .value(0), + 0 + ); + assert_eq!(element_hits(&limited).len(), 1); + + let element_match = |terms: &str| FtsQuery::Match(list_element_match_node("tags", terms)); + let boolean = BooleanQuery::new([ + (Occur::Must, element_match("alpha")), + (Occur::Must, element_match("beta")), + ]); + let boolean = run_fts( + &ds, + FullTextSearchQuery::new_query(FtsQuery::Boolean(boolean)), + None, + ) + .await; + assert_eq!(element_hits(&boolean), vec![(0, vec![0])]); + + let boost = BoostQuery::new(element_match("alpha"), element_match("gamma"), Some(0.5)); + let boost = run_fts( + &ds, + FullTextSearchQuery::new_query(FtsQuery::Boost(boost)), + None, + ) + .await; + assert_eq!(element_hits(&boost), vec![(0, vec![0]), (0, vec![1])]); + + let phrase = PhraseQuery::new("beta gamma".to_string()) + .with_column(Some("tags".to_string())) + .with_document_granularity(DocumentGranularity::ListElement); + let element_phrase = run_fts( + &ds, + FullTextSearchQuery::new_query(FtsQuery::Phrase(phrase)), + None, + ) + .await; + assert_eq!(element_phrase.num_rows(), 0); + + let row_phrase = run_fts(&ds, row_phrase("tags", "beta gamma"), None).await; + assert_eq!( + row_phrase["id"] + .as_primitive::() + .values(), + &[0, 1] + ); + assert!(row_phrase.column_by_name("_doc_index").is_none()); + + let cross_target = BooleanQuery::new([ + (Occur::Must, element_match("alpha")), + (Occur::Must, FtsQuery::Match(row_match_node("tags", "beta"))), + ]); + let mut scanner = ds.scan(); + scanner + .full_text_search(FullTextSearchQuery::new_query(FtsQuery::Boolean( + cross_target, + ))) + .unwrap(); + let err = scanner + .try_into_batch() + .await + .expect_err("mixed document granularities must be rejected"); + assert!( + err.to_string() + .contains("cannot mix Row and ListElement document granularities"), + "{err}" + ); + + let mut multi_match = + MultiMatchQuery::try_new("alpha".to_string(), vec!["tags".to_string()]).unwrap(); + multi_match.match_queries[0].document_granularity = Some(DocumentGranularity::ListElement); + let mut scanner = ds.scan(); + scanner + .full_text_search(FullTextSearchQuery::new_query(FtsQuery::MultiMatch( + multi_match, + ))) + .unwrap(); + let err = scanner + .try_into_batch() + .await + .expect_err("ListElement MultiMatch must be rejected"); + assert!( + err.to_string() + .contains("MultiMatch does not support ListElement document granularity"), + "{err}" + ); + + let mut row_multi_match = + MultiMatchQuery::try_new("beta".to_string(), vec!["tags".to_string()]).unwrap(); + row_multi_match.match_queries[0].document_granularity = Some(DocumentGranularity::Row); + let mixed_multi_match = BooleanQuery::new([ + (Occur::Must, element_match("alpha")), + (Occur::Must, FtsQuery::MultiMatch(row_multi_match)), + ]); + let mut scanner = ds.scan(); + scanner + .full_text_search(FullTextSearchQuery::new_query(FtsQuery::Boolean( + mixed_multi_match, + ))) + .unwrap(); + let err = scanner + .try_into_batch() + .await + .expect_err("mixed document granularities must be rejected"); + assert!( + err.to_string() + .contains("cannot mix Row and ListElement document granularities"), + "{err}" + ); + + let appended = RecordBatch::try_from_iter(vec![ + ("id", Arc::new(Int32Array::from(vec![6])) as ArrayRef), + ( + "tags", + string_lists(&[Some(vec![Some("alpha beta"), Some("alpha again")])]), + ), + ]) + .unwrap(); + ds = InsertBuilder::new(Arc::new(ds)) + .with_params(&WriteParams { + mode: WriteMode::Append, + ..Default::default() + }) + .execute(vec![appended]) + .await + .unwrap(); + + let mixed = run_fts(&ds, element_query, None).await; + assert_eq!( + element_hits(&mixed), + vec![(0, vec![0]), (0, vec![1]), (6, vec![0]), (6, vec![1])] + ); + assert_element_coordinates_point_to(&mixed, "tags", "alpha"); + let mixed_phrase = run_fts(&ds, list_element_phrase("tags", "alpha beta"), None).await; + assert_eq!( + element_hits(&mixed_phrase), + vec![(0, vec![0]), (6, vec![0])] + ); + let all_data = ds.scan().try_into_batch().await.unwrap(); + let flat_reference_dir = tempfile::tempdir().unwrap(); + let flat_reference_ds = Dataset::write( + RecordBatchIterator::new(vec![Ok(all_data.clone())], all_data.schema()), + flat_reference_dir.path().to_str().unwrap(), + None, + ) + .await + .unwrap(); + let flat_reference = run_fts( + &flat_reference_ds, + list_element_match("tags", "alpha"), + None, + ) + .await; + assert_same_element_scores(&mixed, &flat_reference); + let flat_phrase_reference = run_fts( + &flat_reference_ds, + list_element_phrase("tags", "alpha beta"), + None, + ) + .await; + assert_same_element_scores(&mixed_phrase, &flat_phrase_reference); + + ds.optimize_indices(&OptimizeOptions::append()) + .await + .unwrap(); + assert_eq!( + ds.load_indices_by_name("tags_list_element_idx") + .await + .unwrap() + .len(), + 2 + ); + let optimized = run_fts(&ds, list_element_match("tags", "alpha"), None).await; + assert_same_element_scores(&optimized, &flat_reference); + assert_element_coordinates_point_to(&optimized, "tags", "alpha"); + + ds.optimize_indices( + &OptimizeOptions::merge(2).index_names(vec!["tags_list_element_idx".to_string()]), + ) + .await + .unwrap(); + assert_eq!( + ds.load_indices_by_name("tags_list_element_idx") + .await + .unwrap() + .len(), + 1 + ); + let merged = run_fts(&ds, list_element_match("tags", "alpha"), None).await; + assert_same_element_scores(&merged, &flat_reference); + assert_element_coordinates_point_to(&merged, "tags", "alpha"); + + ds.delete("id = 0").await.unwrap(); + compact_files(&mut ds, CompactionOptions::default(), None) + .await + .unwrap(); + let compacted = run_fts(&ds, list_element_match("tags", "alpha"), None).await; + assert_eq!(element_hits(&compacted), vec![(6, vec![0]), (6, vec![1])]); + + ds.alter_columns(&[ColumnAlteration::new("tags".into()).rename("labels".into())]) + .await + .unwrap(); + let renamed = run_fts(&ds, list_element_match("labels", "alpha"), None).await; + assert_eq!(element_hits(&renamed), vec![(6, vec![0]), (6, vec![1])]); + let mut old_name_scanner = ds.scan(); + old_name_scanner + .full_text_search(list_element_match("tags", "alpha")) + .unwrap(); + let err = old_name_scanner + .try_into_batch() + .await + .expect_err("renamed element target must reject the old path"); + assert!(err.to_string().contains("tags"), "{err}"); +} + +#[tokio::test] +async fn test_element_document_persists_empty_and_zero_token_corpora() { + async fn assert_empty_query(tags: ArrayRef, expected_documents: usize, with_position: bool) { + let ids = Arc::new(Int32Array::from_iter_values(0..tags.len() as i32)); + let batch = + RecordBatch::try_from_iter(vec![("id", ids as ArrayRef), ("tags", tags)]).unwrap(); + let test_dir = tempfile::tempdir().unwrap(); + let mut ds = Dataset::write( + RecordBatchIterator::new(vec![Ok(batch.clone())], batch.schema()), + test_dir.path().to_str().unwrap(), + None, + ) + .await + .unwrap(); + + ds.create_index( + &["tags"], + IndexType::Inverted, + None, + &list_element_params(with_position), + true, + ) + .await + .unwrap(); + + let result = run_fts(&ds, list_element_match("tags", "alpha"), None).await; + assert_eq!(result.num_rows(), 0); + + let metadata = ds + .load_indices_by_name("tags_list_element_idx") + .await + .unwrap() + .pop() + .unwrap(); + let index = ds + .open_scalar_index("tags", &metadata.uuid, &NoOpMetricsCollector) + .await + .unwrap(); + assert_eq!(index.statistics().unwrap()["num_docs"], expected_documents); + } + + for with_position in [false, true] { + assert_empty_query(string_lists(&[None, Some(vec![])]), 0, with_position).await; + assert_empty_query( + string_lists(&[Some(vec![None, Some(""), Some("!!!")])]), + 3, + with_position, + ) + .await; + } +} + +#[tokio::test] +async fn test_element_document_nested_lists_use_deepest_boundary() { + let doc_fields = ArrowFields::from(vec![ArrowField::new("content", DataType::Utf8, true)]); + let doc_values = StructArray::new( + doc_fields.clone(), + vec![Arc::new(StringArray::from(vec![ + Some("alpha"), + Some("beta"), + Some("gamma"), + Some("alpha delta"), + Some("alpha"), + ])) as ArrayRef], + None, + ); + let doc_item = Arc::new(ArrowField::new("item", DataType::Struct(doc_fields), true)); + let docs_type = DataType::List(doc_item.clone()); + let docs = ListArray::new( + doc_item, + OffsetBuffer::new(ScalarBuffer::from(vec![0i32, 2, 4, 4, 5])), + Arc::new(doc_values), + None, + ); + let group_fields = ArrowFields::from(vec![ArrowField::new("docs", docs_type, true)]); + let group_values = + StructArray::new(group_fields.clone(), vec![Arc::new(docs) as ArrayRef], None); + let group_item = Arc::new(ArrowField::new( + "item", + DataType::Struct(group_fields), + true, + )); + let groups = ListArray::new( + group_item, + OffsetBuffer::new(ScalarBuffer::from(vec![0i32, 2, 4])), + Arc::new(group_values), + None, + ); + let batch = RecordBatch::try_from_iter(vec![ + ("id", Arc::new(Int32Array::from(vec![0, 1])) as ArrayRef), + ("groups", Arc::new(groups) as ArrayRef), + ]) + .unwrap(); + let test_dir = tempfile::tempdir().unwrap(); + let mut ds = Dataset::write( + RecordBatchIterator::new(vec![Ok(batch.clone())], batch.schema()), + test_dir.path().to_str().unwrap(), + None, + ) + .await + .unwrap(); + + let path = "groups.docs.content"; + let expected = vec![(0, vec![0, 0]), (0, vec![1, 1]), (1, vec![1, 0])]; + let flat = run_fts(&ds, list_element_match(path, "alpha"), None).await; + assert_eq!(element_hits(&flat), expected); + + let row_flat = run_fts( + &ds, + FullTextSearchQuery::new("alpha".to_string()) + .with_column(path.to_string()) + .unwrap(), + None, + ) + .await; + assert_eq!( + row_flat["id"] + .as_primitive::() + .values(), + &[0, 1] + ); + assert!(row_flat.column_by_name("_doc_index").is_none()); + + ds.create_index( + &[path], + IndexType::Inverted, + None, + &list_element_params(true), + true, + ) + .await + .unwrap(); + ds.create_index( + &[path], + IndexType::Inverted, + None, + &base_inverted_params(true), + true, + ) + .await + .unwrap(); + + let indexed = run_fts(&ds, list_element_match(path, "alpha"), None).await; + assert_eq!(element_hits(&indexed), expected); + let indices = ds.load_indices().await.unwrap(); + let row = indices + .iter() + .find(|index| index.name == "groups.docs.content_idx") + .unwrap(); + let elements = indices + .iter() + .find(|index| index.name == "groups.docs.content_list_element_idx") + .unwrap(); + let groups = ds.schema().field("groups").unwrap(); + let group_item = groups.children.first().unwrap(); + let docs = group_item + .children + .iter() + .find(|field| field.name == "docs") + .unwrap(); + let doc_item = docs.children.first().unwrap(); + let content = doc_item + .children + .iter() + .find(|field| field.name == "content") + .unwrap(); + assert_eq!(row.fields, elements.fields); + assert_eq!(row.fields, vec![content.id]); + assert_ne!(row.fields, vec![docs.children[0].id]); +} + +#[tokio::test] +async fn test_element_document_bm25_uses_element_corpus() { + let batch = RecordBatch::try_from_iter(vec![ + ("id", Arc::new(Int32Array::from(vec![0, 1])) as ArrayRef), + ( + "tags", + string_lists(&[ + Some(vec![ + Some("needle"), + Some("filler filler filler filler filler filler filler filler filler"), + ]), + Some(vec![Some("needle filler")]), + ]), + ), + ]) + .unwrap(); + let schema = batch.schema(); + let test_dir = tempfile::tempdir().unwrap(); + let mut ds = Dataset::write( + RecordBatchIterator::new(vec![Ok(batch)], schema), + test_dir.path().to_str().unwrap(), + None, + ) + .await + .unwrap(); + let params = base_inverted_params(false); + ds.create_index(&["tags"], IndexType::Inverted, None, ¶ms, true) + .await + .unwrap(); + ds.create_index( + &["tags"], + IndexType::Inverted, + None, + &list_element_params(false), + true, + ) + .await + .unwrap(); + + let row = run_fts(&ds, row_match("tags", "needle"), None).await; + let element = run_fts(&ds, list_element_match("tags", "needle"), None).await; + let row_scores = row["_score"].as_primitive::(); + let element_scores = element["_score"].as_primitive::(); + + let expected_row_long = expected_bm25_score(2, 2, 12, 10); + let expected_row_short = expected_bm25_score(2, 2, 12, 2); + let expected_element_short = expected_bm25_score(3, 2, 12, 1); + let expected_element_long = expected_bm25_score(3, 2, 12, 2); + assert!((row_scores.value(0) - expected_row_long).abs() < 1e-5); + assert!((row_scores.value(1) - expected_row_short).abs() < 1e-5); + assert!((element_scores.value(0) - expected_element_short).abs() < 1e-5); + assert!((element_scores.value(1) - expected_element_long).abs() < 1e-5); + assert!(row_scores.value(1) > row_scores.value(0)); + assert!(element_scores.value(0) > element_scores.value(1)); +} + +#[tokio::test] +async fn test_element_document_fts_vector_prefilter_deduplicates_parent_rows() { + let vectors = + FixedSizeListArray::try_new_from_values(Float32Array::from(vec![0.0, 0.0, 1.0, 1.0]), 2) + .unwrap(); + let batch = RecordBatch::try_from_iter(vec![ + ("id", Arc::new(Int32Array::from(vec![0, 1])) as ArrayRef), + ( + "tags", + string_lists(&[ + Some(vec![Some("alpha"), Some("alpha again")]), + Some(vec![Some("beta")]), + ]), + ), + ("vector", Arc::new(vectors) as ArrayRef), + ]) + .unwrap(); + let schema = batch.schema(); + let test_dir = tempfile::tempdir().unwrap(); + let mut ds = Dataset::write( + RecordBatchIterator::new(vec![Ok(batch)], schema), + test_dir.path().to_str().unwrap(), + None, + ) + .await + .unwrap(); + ds.create_index( + &["tags"], + IndexType::Inverted, + None, + &list_element_params(false), + true, + ) + .await + .unwrap(); + + let query_vector = Float32Array::from(vec![0.0, 0.0]); + let mut scanner = ds.scan(); + scanner + .nearest("vector", &query_vector, 10) + .unwrap() + .filter_query(QueryFilter::Fts(list_element_match("tags", "alpha"))) + .unwrap() + .prefilter(true); + let result = scanner.try_into_batch().await.unwrap(); + + assert_eq!( + result["id"] + .as_primitive::() + .values(), + &[0] + ); + assert!(result.column_by_name("_doc_index").is_none()); + + ds.create_index( + &["tags"], + IndexType::Inverted, + None, + &base_inverted_params(false), + true, + ) + .await + .unwrap(); + let mut scanner = ds.scan(); + scanner + .nearest("vector", &query_vector, 10) + .unwrap() + .filter_query(QueryFilter::Fts(FullTextSearchQuery::new( + "alpha".to_string(), + ))) + .unwrap() + .prefilter(true); + let result = scanner.try_into_batch().await.unwrap(); + assert_eq!( + result["id"] + .as_primitive::() + .values(), + &[0] + ); + assert!(result.column_by_name("_doc_index").is_none()); +} + #[tokio::test] // Ensure indexed and non-indexed full-text search return the same ids. async fn test_inverted_basic_equivalence() { diff --git a/rust/lance/tests/query/primitives.rs b/rust/lance/tests/query/primitives.rs index 65fa6f4e4d3..608a59abb61 100644 --- a/rust/lance/tests/query/primitives.rs +++ b/rust/lance/tests/query/primitives.rs @@ -10,8 +10,8 @@ use arrow_array::{ }; use arrow_schema::DataType; use lance::Dataset; -use lance::dataset::WriteParams; use lance::dataset::optimize::{CompactionOptions, compact_files}; +use lance::dataset::{InsertBuilder, WriteParams}; use lance::index::DatasetIndexExt; use lance_datagen::{ArrayGeneratorExt, RowCount, array, gen_batch}; @@ -260,7 +260,9 @@ async fn test_query_date(#[case] data_type: DataType) { test_scan(&original, &ds).await; test_take(&original, &ds).await; test_filter(&original, &ds, "value < current_date()").await; - test_filter(&original, &ds, "value > DATE '2024-01-01'").await; + // Mid-range literal: rand_type samples dates from the fixed range + // [2023-01-01, 2024-01-01), so this splits the generated values + test_filter(&original, &ds, "value > DATE '2023-07-01'").await; test_filter(&original, &ds, "value is null").await; test_filter(&original, &ds, "value is not null").await; }) @@ -295,7 +297,9 @@ async fn test_query_timestamp(#[case] data_type: DataType) { test_scan(&original, &ds).await; test_take(&original, &ds).await; test_filter(&original, &ds, "value < current_timestamp()").await; - test_filter(&original, &ds, "value > TIMESTAMP '2024-01-01 00:00:00'").await; + // Mid-range literal: rand_type samples timestamps from the fixed range + // [2023-01-01, 2024-01-01), so this splits the generated values + test_filter(&original, &ds, "value > TIMESTAMP '2023-07-01 00:00:00'").await; test_filter(&original, &ds, "value is null").await; test_filter(&original, &ds, "value is not null").await; }) @@ -513,3 +517,87 @@ async fn test_filtered_scan_after_compact_with_srid() { results.num_rows() ); } + +/// Verifies that a zone map index on a string column is used (ScalarIndexQuery +/// in the plan) for both IS NULL and IS NOT NULL predicate filters. +/// +/// IS NOT NULL must not silently fall back to a full scan when a zone map +/// index exists — both predicates should leverage the index. +#[tokio::test] +async fn test_zone_map_null_index_used() { + // 6 non-null strings and 4 null values across 10 rows. + let string_values = vec![ + Some("alpha"), + None, + Some("beta"), + Some("gamma"), + None, + Some("delta"), + None, + Some("epsilon"), + Some("zeta"), + None, + ]; + let value_array = Arc::new(StringArray::from(string_values)) as ArrayRef; + let id_array = Arc::new(Int32Array::from((0..10).collect::>())) as ArrayRef; + let batch = RecordBatch::try_from_iter(vec![("id", id_array), ("value", value_array)]).unwrap(); + + let mut ds = InsertBuilder::new("memory://") + .execute(vec![batch]) + .await + .unwrap(); + + ds.create_index( + &["value"], + IndexType::ZoneMap, + None, + &lance_index::scalar::ScalarIndexParams::default(), + true, + ) + .await + .unwrap(); + + // IS NULL: the zone map index must appear in the plan. + let plan = ds + .scan() + .filter("value IS NULL") + .unwrap() + .explain_plan(false) + .await + .unwrap(); + assert!( + plan.contains("ScalarIndexQuery"), + "IS NULL should use zone map index, got plan:\n{}", + plan + ); + let null_batch = ds + .scan() + .filter("value IS NULL") + .unwrap() + .try_into_batch() + .await + .unwrap(); + assert_eq!(null_batch.num_rows(), 4); + + // IS NOT NULL: the zone map index must also appear in the plan. + let plan = ds + .scan() + .filter("value IS NOT NULL") + .unwrap() + .explain_plan(false) + .await + .unwrap(); + assert!( + plan.contains("ScalarIndexQuery"), + "IS NOT NULL should use zone map index, got plan:\n{}", + plan + ); + let non_null_batch = ds + .scan() + .filter("value IS NOT NULL") + .unwrap() + .try_into_batch() + .await + .unwrap(); + assert_eq!(non_null_batch.num_rows(), 6); +} diff --git a/rust/lance/tests/scalar_index_spill.rs b/rust/lance/tests/scalar_index_spill.rs new file mode 100644 index 00000000000..595e625477e --- /dev/null +++ b/rust/lance/tests/scalar_index_spill.rs @@ -0,0 +1,69 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright The Lance Authors + +//! Regression test: training a BTREE scalar index on a large string column +//! must succeed when the sort spills under a small bounded memory pool. +//! +//! Sorting far more data than the pool holds produces many spilled runs; the +//! external-sort merge phase then needs pool memory on top of the runs. +//! Before the sort spill reservation was sized to the pool (#7675), the merge +//! reservation could exceed the whole `FairSpillPool`, failing index creation +//! with `ResourcesExhausted` unless spilling was bypassed entirely via +//! `LANCE_BYPASS_SPILLING`. +//! +//! This file must stay a single-test integration binary: the training path +//! only reads the pool size from the process-global `LANCE_MEM_POOL_SIZE`, so +//! any sibling test could race the `set_var` or inherit the tiny pool. + +use lance::Dataset; +use lance::dataset::WriteParams; +use lance::index::DatasetIndexExt; +use lance_datafusion::exec::LanceExecutionOptions; +use lance_datagen::{BatchCount, ByteCount, RowCount, array, gen_batch}; +use lance_index::IndexType; +use lance_index::scalar::{BuiltinIndexType, ScalarIndexParams}; + +const MEM_POOL_SIZE: u64 = 4 * 1024 * 1024; + +#[tokio::test] +async fn test_btree_training_sort_spill_merge_fits_pool() { + // 4 MiB pool vs ~36 MiB of sort input (512K rows of 64-byte strings plus + // row ids) forces many spilled sort runs. + unsafe { + std::env::set_var("LANCE_MEM_POOL_SIZE", MEM_POOL_SIZE.to_string()); + // The historical workaround for this very bug; if it leaks in from the + // environment the bounded pool is skipped and the test checks nothing. + std::env::remove_var("LANCE_BYPASS_SPILLING"); + } + // The training scan builds its execution options from the env vars above; + // fail loudly if that plumbing ever changes, otherwise the sort would run + // against the default (much larger) pool and pass vacuously. + let options = LanceExecutionOptions { + use_spilling: true, + ..Default::default() + }; + assert_eq!(options.mem_pool_size(), MEM_POOL_SIZE); + assert!(options.use_spilling()); + + let data = gen_batch() + .col("value", array::rand_utf8(ByteCount::from(64), false)) + .into_reader_rows(RowCount::from(8192), BatchCount::from(64)); + + let write_params = WriteParams { + max_rows_per_file: 256 * 1024, + ..Default::default() + }; + let mut dataset = Dataset::write(data, "memory://", Some(write_params)) + .await + .unwrap(); + assert!(dataset.get_fragments().len() > 1); + + let params = ScalarIndexParams::for_builtin(BuiltinIndexType::BTree); + dataset + .create_index(&["value"], IndexType::BTree, None, ¶ms, false) + .await + .unwrap(); + + let indices = dataset.load_indices().await.unwrap(); + assert_eq!(indices.len(), 1); +} diff --git a/test_data/fri_straddle_pre_6610/datagen/Cargo.lock b/test_data/fri_straddle_pre_6610/datagen/Cargo.lock index 6fcff711a06..531e90b410f 100644 --- a/test_data/fri_straddle_pre_6610/datagen/Cargo.lock +++ b/test_data/fri_straddle_pre_6610/datagen/Cargo.lock @@ -1039,6 +1039,17 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724" +[[package]] +name = "chacha20" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d524456ba66e72eb8b115ff89e01e497f8e6d11d78b70b1aa13c0fbd97540a81" +dependencies = [ + "cfg-if", + "cpufeatures 0.3.0", + "rand_core 0.10.1", +] + [[package]] name = "chrono" version = "0.4.44" @@ -1124,9 +1135,9 @@ dependencies = [ [[package]] name = "cmov" -version = "0.5.3" +version = "0.5.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3f88a43d011fc4a6876cb7344703e297c71dda42494fee094d5f7c76bf13f746" +checksum = "0c9ea0ac24bc397ab3c98583a3c9ba74fa56b09a4449bbe172b9b1ddb016027a" [[package]] name = "colorchoice" @@ -2541,11 +2552,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd" dependencies = [ "cfg-if", - "js-sys", "libc", "r-efi 5.3.0", "wasip2", - "wasm-bindgen", ] [[package]] @@ -2555,10 +2564,13 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0de51e6874e94e7bf76d726fc5d13ba782deca734ff60d5bb2fb2607c7406555" dependencies = [ "cfg-if", + "js-sys", "libc", "r-efi 6.0.0", + "rand_core 0.10.1", "wasip2", "wasip3", + "wasm-bindgen", ] [[package]] @@ -4594,14 +4606,15 @@ dependencies = [ [[package]] name = "quinn-proto" -version = "0.11.14" +version = "0.11.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "434b42fec591c96ef50e21e886936e66d3cc3f737104fdb9b737c40ffb94c098" +checksum = "2f4bfc015262b9df63c8845072ce59068853ff5872180c2ce2f13038b970e560" dependencies = [ "bytes", - "getrandom 0.3.4", + "getrandom 0.4.2", "lru-slab", - "rand 0.9.4", + "rand 0.10.2", + "rand_pcg", "ring", "rustc-hash", "rustls", @@ -4675,6 +4688,17 @@ dependencies = [ "rand_core 0.9.5", ] +[[package]] +name = "rand" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c7f5fa3a058cd35567ef9bfa5e75732bee0f9e4c55fa90477bef2dfcdbc4be80" +dependencies = [ + "chacha20", + "getrandom 0.4.2", + "rand_core 0.10.1", +] + [[package]] name = "rand_chacha" version = "0.3.1" @@ -4713,6 +4737,12 @@ dependencies = [ "getrandom 0.3.4", ] +[[package]] +name = "rand_core" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63b8176103e19a2643978565ca18b50549f6101881c443590420e4dc998a3c69" + [[package]] name = "rand_distr" version = "0.5.1" @@ -4723,6 +4753,15 @@ dependencies = [ "rand 0.9.4", ] +[[package]] +name = "rand_pcg" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "caa0f4137e1c0a72f4c651489402276c8e8e1cf081f3b0ba156d2cbeef09e86a" +dependencies = [ + "rand_core 0.10.1", +] + [[package]] name = "rand_xoshiro" version = "0.7.0" diff --git a/test_data/readme.md b/test_data/readme.md index 8d041a96db5..69f153c22d1 100644 --- a/test_data/readme.md +++ b/test_data/readme.md @@ -30,4 +30,9 @@ folder contains a `datagen.py` script that generates one or more lance datasets. * `v0.27.1/pq_in_schema`: This dataset uses the old method of storing the PQ metadata in the schema metadata in the index file. We switched to storing them in a global buffer in https://github.com/lancedb/lance/pull/3829, but still - need to be able to read the old format. \ No newline at end of file + need to be able to read the old format. +* `v3.0.1/fts_v1` and `v4.0.1/fts_v2`: These datasets cover the supported FTS + layouts written by stable Lance releases. Each fixture contains posting lists + that cross the layout's block boundary and positions used by phrase queries. + The v1 fixture also retains the retired `skip_merge` parameter written by Lance + 3.0.1. diff --git a/test_data/v3.0.1/datagen.py b/test_data/v3.0.1/datagen.py new file mode 100644 index 00000000000..2815351490c --- /dev/null +++ b/test_data/v3.0.1/datagen.py @@ -0,0 +1,45 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright The Lance Authors + +import os +import shutil +from pathlib import Path + +import lance +import pyarrow as pa +from lance.query import PhraseQuery + +EXPECTED_LANCE_VERSION = "3.0.1" +EXPECTED_FTS_VERSION = 1 +NUM_ROWS = 300 + +assert lance.__version__ == EXPECTED_LANCE_VERSION + +os.environ["LANCE_FTS_FORMAT_VERSION"] = str(EXPECTED_FTS_VERSION) + +dataset_path = Path(__file__).parent / "fts_v1" +shutil.rmtree(dataset_path, ignore_errors=True) + +row_ids = list(range(NUM_ROWS)) +texts = [ + "lance database compatibility shared" + if row_id % 3 == 0 + else "database lance compatibility shared" + for row_id in row_ids +] +dataset = lance.write_dataset(pa.table({"id": row_ids, "text": texts}), dataset_path) +dataset.create_scalar_index( + "text", + "INVERTED", + with_position=True, + skip_merge=True, +) + +index = dataset.describe_indices()[0] +assert index.segments[0].index_version == EXPECTED_FTS_VERSION + +matches = dataset.to_table(full_text_query="compatibility") +assert set(matches["id"].to_pylist()) == set(row_ids) + +phrase_matches = dataset.to_table(full_text_query=PhraseQuery("lance database", "text")) +assert set(phrase_matches["id"].to_pylist()) == set(range(0, NUM_ROWS, 3)) diff --git a/test_data/v3.0.1/fts_v1/_indices/450d1cd4-9eb2-4a11-8eac-ab1b64b1ea31/metadata.lance b/test_data/v3.0.1/fts_v1/_indices/450d1cd4-9eb2-4a11-8eac-ab1b64b1ea31/metadata.lance new file mode 100644 index 00000000000..10d6e21f91a Binary files /dev/null and b/test_data/v3.0.1/fts_v1/_indices/450d1cd4-9eb2-4a11-8eac-ab1b64b1ea31/metadata.lance differ diff --git a/test_data/v3.0.1/fts_v1/_indices/450d1cd4-9eb2-4a11-8eac-ab1b64b1ea31/part_1_docs.lance b/test_data/v3.0.1/fts_v1/_indices/450d1cd4-9eb2-4a11-8eac-ab1b64b1ea31/part_1_docs.lance new file mode 100644 index 00000000000..d7314b6b7c2 Binary files /dev/null and b/test_data/v3.0.1/fts_v1/_indices/450d1cd4-9eb2-4a11-8eac-ab1b64b1ea31/part_1_docs.lance differ diff --git a/test_data/v3.0.1/fts_v1/_indices/450d1cd4-9eb2-4a11-8eac-ab1b64b1ea31/part_1_invert.lance b/test_data/v3.0.1/fts_v1/_indices/450d1cd4-9eb2-4a11-8eac-ab1b64b1ea31/part_1_invert.lance new file mode 100644 index 00000000000..609d3cbcbb9 Binary files /dev/null and b/test_data/v3.0.1/fts_v1/_indices/450d1cd4-9eb2-4a11-8eac-ab1b64b1ea31/part_1_invert.lance differ diff --git a/test_data/v3.0.1/fts_v1/_indices/450d1cd4-9eb2-4a11-8eac-ab1b64b1ea31/part_1_tokens.lance b/test_data/v3.0.1/fts_v1/_indices/450d1cd4-9eb2-4a11-8eac-ab1b64b1ea31/part_1_tokens.lance new file mode 100644 index 00000000000..614e7e40872 Binary files /dev/null and b/test_data/v3.0.1/fts_v1/_indices/450d1cd4-9eb2-4a11-8eac-ab1b64b1ea31/part_1_tokens.lance differ diff --git a/test_data/v3.0.1/fts_v1/_transactions/0-0e238eef-4390-4c40-9c7e-3c1437b4735b.txn b/test_data/v3.0.1/fts_v1/_transactions/0-0e238eef-4390-4c40-9c7e-3c1437b4735b.txn new file mode 100644 index 00000000000..2f01cf7c9b2 Binary files /dev/null and b/test_data/v3.0.1/fts_v1/_transactions/0-0e238eef-4390-4c40-9c7e-3c1437b4735b.txn differ diff --git a/test_data/v3.0.1/fts_v1/_transactions/1-d331315a-f1a6-4735-a453-90470239bd60.txn b/test_data/v3.0.1/fts_v1/_transactions/1-d331315a-f1a6-4735-a453-90470239bd60.txn new file mode 100644 index 00000000000..333e934e259 Binary files /dev/null and b/test_data/v3.0.1/fts_v1/_transactions/1-d331315a-f1a6-4735-a453-90470239bd60.txn differ diff --git a/test_data/v3.0.1/fts_v1/_versions/18446744073709551613.manifest b/test_data/v3.0.1/fts_v1/_versions/18446744073709551613.manifest new file mode 100644 index 00000000000..d61e99244a2 Binary files /dev/null and b/test_data/v3.0.1/fts_v1/_versions/18446744073709551613.manifest differ diff --git a/test_data/v3.0.1/fts_v1/_versions/18446744073709551614.manifest b/test_data/v3.0.1/fts_v1/_versions/18446744073709551614.manifest new file mode 100644 index 00000000000..7839e3ac728 Binary files /dev/null and b/test_data/v3.0.1/fts_v1/_versions/18446744073709551614.manifest differ diff --git a/test_data/v3.0.1/fts_v1/data/110110010101010101010101b80a6f4c4d804a62b40de56ffb.lance b/test_data/v3.0.1/fts_v1/data/110110010101010101010101b80a6f4c4d804a62b40de56ffb.lance new file mode 100644 index 00000000000..96e80c7b9a8 Binary files /dev/null and b/test_data/v3.0.1/fts_v1/data/110110010101010101010101b80a6f4c4d804a62b40de56ffb.lance differ diff --git a/test_data/v4.0.1/datagen.py b/test_data/v4.0.1/datagen.py new file mode 100644 index 00000000000..accaa57cadf --- /dev/null +++ b/test_data/v4.0.1/datagen.py @@ -0,0 +1,40 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright The Lance Authors + +import os +import shutil +from pathlib import Path + +import lance +import pyarrow as pa +from lance.query import PhraseQuery + +EXPECTED_LANCE_VERSION = "4.0.1" +EXPECTED_FTS_VERSION = 2 +NUM_ROWS = 300 + +assert lance.__version__ == EXPECTED_LANCE_VERSION + +os.environ["LANCE_FTS_FORMAT_VERSION"] = str(EXPECTED_FTS_VERSION) + +dataset_path = Path(__file__).parent / "fts_v2" +shutil.rmtree(dataset_path, ignore_errors=True) + +row_ids = list(range(NUM_ROWS)) +texts = [ + "lance database compatibility shared" + if row_id % 3 == 0 + else "database lance compatibility shared" + for row_id in row_ids +] +dataset = lance.write_dataset(pa.table({"id": row_ids, "text": texts}), dataset_path) +dataset.create_scalar_index("text", "INVERTED", with_position=True) + +index = dataset.describe_indices()[0] +assert index.segments[0].index_version == EXPECTED_FTS_VERSION + +matches = dataset.to_table(full_text_query="compatibility") +assert set(matches["id"].to_pylist()) == set(row_ids) + +phrase_matches = dataset.to_table(full_text_query=PhraseQuery("lance database", "text")) +assert set(phrase_matches["id"].to_pylist()) == set(range(0, NUM_ROWS, 3)) diff --git a/test_data/v4.0.1/fts_v2/_indices/1fb8e405-43d0-4707-be37-64eadc3d5e97/metadata.lance b/test_data/v4.0.1/fts_v2/_indices/1fb8e405-43d0-4707-be37-64eadc3d5e97/metadata.lance new file mode 100644 index 00000000000..1f3d5698f43 Binary files /dev/null and b/test_data/v4.0.1/fts_v2/_indices/1fb8e405-43d0-4707-be37-64eadc3d5e97/metadata.lance differ diff --git a/test_data/v4.0.1/fts_v2/_indices/1fb8e405-43d0-4707-be37-64eadc3d5e97/part_6_docs.lance b/test_data/v4.0.1/fts_v2/_indices/1fb8e405-43d0-4707-be37-64eadc3d5e97/part_6_docs.lance new file mode 100644 index 00000000000..d7314b6b7c2 Binary files /dev/null and b/test_data/v4.0.1/fts_v2/_indices/1fb8e405-43d0-4707-be37-64eadc3d5e97/part_6_docs.lance differ diff --git a/test_data/v4.0.1/fts_v2/_indices/1fb8e405-43d0-4707-be37-64eadc3d5e97/part_6_invert.lance b/test_data/v4.0.1/fts_v2/_indices/1fb8e405-43d0-4707-be37-64eadc3d5e97/part_6_invert.lance new file mode 100644 index 00000000000..196ad67f15a Binary files /dev/null and b/test_data/v4.0.1/fts_v2/_indices/1fb8e405-43d0-4707-be37-64eadc3d5e97/part_6_invert.lance differ diff --git a/test_data/v4.0.1/fts_v2/_indices/1fb8e405-43d0-4707-be37-64eadc3d5e97/part_6_tokens.lance b/test_data/v4.0.1/fts_v2/_indices/1fb8e405-43d0-4707-be37-64eadc3d5e97/part_6_tokens.lance new file mode 100644 index 00000000000..614e7e40872 Binary files /dev/null and b/test_data/v4.0.1/fts_v2/_indices/1fb8e405-43d0-4707-be37-64eadc3d5e97/part_6_tokens.lance differ diff --git a/test_data/v4.0.1/fts_v2/_transactions/0-fbd896dd-b250-46fa-b60d-9b1474d7fc7b.txn b/test_data/v4.0.1/fts_v2/_transactions/0-fbd896dd-b250-46fa-b60d-9b1474d7fc7b.txn new file mode 100644 index 00000000000..f550fa2c193 Binary files /dev/null and b/test_data/v4.0.1/fts_v2/_transactions/0-fbd896dd-b250-46fa-b60d-9b1474d7fc7b.txn differ diff --git a/test_data/v4.0.1/fts_v2/_transactions/1-ebe80000-4421-4ed6-8072-093abeb55eaf.txn b/test_data/v4.0.1/fts_v2/_transactions/1-ebe80000-4421-4ed6-8072-093abeb55eaf.txn new file mode 100644 index 00000000000..3cbfb8b41cb Binary files /dev/null and b/test_data/v4.0.1/fts_v2/_transactions/1-ebe80000-4421-4ed6-8072-093abeb55eaf.txn differ diff --git a/test_data/v4.0.1/fts_v2/_versions/18446744073709551613.manifest b/test_data/v4.0.1/fts_v2/_versions/18446744073709551613.manifest new file mode 100644 index 00000000000..6fa1d8bb6e7 Binary files /dev/null and b/test_data/v4.0.1/fts_v2/_versions/18446744073709551613.manifest differ diff --git a/test_data/v4.0.1/fts_v2/_versions/18446744073709551614.manifest b/test_data/v4.0.1/fts_v2/_versions/18446744073709551614.manifest new file mode 100644 index 00000000000..cb8ff78ef70 Binary files /dev/null and b/test_data/v4.0.1/fts_v2/_versions/18446744073709551614.manifest differ diff --git a/test_data/v4.0.1/fts_v2/data/000100000010011011000110c47acb4569aa4d2c1957105328.lance b/test_data/v4.0.1/fts_v2/data/000100000010011011000110c47acb4569aa4d2c1957105328.lance new file mode 100644 index 00000000000..96e80c7b9a8 Binary files /dev/null and b/test_data/v4.0.1/fts_v2/data/000100000010011011000110c47acb4569aa4d2c1957105328.lance differ